diff --git a/.codespellrc b/.codespellrc deleted file mode 100644 index 771985af191..00000000000 --- a/.codespellrc +++ /dev/null @@ -1,3 +0,0 @@ -[codespell] -skip = .git,target,./crates/storage/libmdbx-rs/mdbx-sys/libmdbx,Cargo.toml,Cargo.lock -ignore-words-list = crate,ser,ratatui diff --git a/.config/nextest.toml b/.config/nextest.toml index e107857a351..26b4a000b93 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -5,3 +5,13 @@ slow-timeout = { period = "30s", terminate-after = 4 } [[profile.default.overrides]] filter = "test(general_state_tests)" slow-timeout = { period = "1m", terminate-after = 10 } + +[[profile.default.overrides]] +filter = "test(eest_fixtures)" +slow-timeout = { period = "2m", terminate-after = 10 } + +# E2E tests using the testsuite framework from crates/e2e-test-utils +# These tests are located in tests/e2e-testsuite/ directories across various crates +[[profile.default.overrides]] +filter = "binary(e2e_testsuite)" +slow-timeout = { period = "2m", terminate-after = 3 } diff --git a/.config/zepter.yaml b/.config/zepter.yaml index b754d06a062..a4179c0a8fd 100644 --- a/.config/zepter.yaml +++ b/.config/zepter.yaml @@ -12,7 +12,7 @@ workflows: # Check that `A` activates the features of `B`. "propagate-feature", # These are the features to check: - "--features=std,op,dev,asm-keccak,jemalloc,jemalloc-prof,tracy-allocator,serde-bincode-compat,serde,test-utils,arbitrary,bench,alloy-compat", + "--features=std,op,dev,asm-keccak,jemalloc,jemalloc-prof,tracy-allocator,serde-bincode-compat,serde,test-utils,arbitrary,bench,alloy-compat,min-error-logs,min-warn-logs,min-info-logs,min-debug-logs,min-trace-logs,otlp,js-tracer", # Do not try to add a new section to `[features]` of `A` only because `B` exposes that feature. There are edge-cases where this is still needed, but we can add them manually. "--left-side-feature-missing=ignore", # Ignore the case that `A` it outside of the workspace. Otherwise it will report errors in external dependencies that we have no influence on. diff --git a/.gitattributes b/.gitattributes index 52ee28d3ba9..17286acb516 100644 --- a/.gitattributes +++ b/.gitattributes @@ -2,3 +2,5 @@ book/cli/**/*.md linguist-vendored book/cli/cli.md -linguist-vendored crates/storage/libmdbx-rs/mdbx-sys/libmdbx/** linguist-vendored + +bun.lock linguist-language=JSON-with-Comments diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 1596d90b30f..eed64b157f3 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,50 +1,44 @@ * @gakonst -bin/ @onbjerg -crates/blockchain-tree/ @rakita @rkrasiuk @mattsse @Rjected -crates/blockchain-tree-api/ @rakita @rkrasiuk @mattsse @Rjected +crates/blockchain-tree-api/ @rakita @mattsse @Rjected +crates/blockchain-tree/ @rakita @mattsse @Rjected +crates/chain-state/ @fgimenez @mattsse crates/chainspec/ @Rjected @joshieDo @mattsse -crates/chain-state/ @fgimenez @mattsse @rkrasiuk -crates/cli/ @onbjerg @mattsse -crates/config/ @onbjerg -crates/consensus/ @rkrasiuk @mattsse @Rjected -crates/engine @rkrasiuk @mattsse @Rjected -crates/e2e-test-utils/ @mattsse @Rjected -crates/engine/ @rkrasiuk @mattsse @Rjected @fgimenez +crates/cli/ @mattsse +crates/consensus/ @mattsse @Rjected +crates/e2e-test-utils/ @mattsse @Rjected @klkvr @fgimenez +crates/engine/ @mattsse @Rjected @fgimenez @mediocregopher @yongkangc +crates/era/ @mattsse @RomanHodulak crates/errors/ @mattsse -crates/era/ @mattsse -crates/ethereum/ @mattsse @Rjected crates/ethereum-forks/ @mattsse @Rjected +crates/ethereum/ @mattsse @Rjected crates/etl/ @joshieDo @shekhirin crates/evm/ @rakita @mattsse @Rjected -crates/exex/ @onbjerg @shekhirin -crates/fs-util/ @onbjerg -crates/metrics/ @onbjerg +crates/exex/ @shekhirin crates/net/ @mattsse @Rjected -crates/net/downloaders/ @onbjerg @rkrasiuk -crates/node/ @mattsse @Rjected @onbjerg @klkvr +crates/net/downloaders/ @Rjected +crates/node/ @mattsse @Rjected @klkvr crates/optimism/ @mattsse @Rjected @fgimenez crates/payload/ @mattsse @Rjected +crates/primitives-traits/ @Rjected @RomanHodulak @mattsse @klkvr crates/primitives/ @Rjected @mattsse @klkvr -crates/primitives-traits/ @Rjected @joshieDo @mattsse @klkvr crates/prune/ @shekhirin @joshieDo +crates/ress @shekhirin @Rjected crates/revm/ @mattsse @rakita -crates/rpc/ @mattsse @Rjected -crates/stages/ @onbjerg @rkrasiuk @shekhirin +crates/rpc/ @mattsse @Rjected @RomanHodulak +crates/stages/ @shekhirin @mediocregopher crates/static-file/ @joshieDo @shekhirin crates/storage/codecs/ @joshieDo -crates/storage/db/ @joshieDo @rakita crates/storage/db-api/ @joshieDo @rakita -crates/storage/db-common/ @Rjected @onbjerg -crates/storage/errors/ @rakita @onbjerg +crates/storage/db-common/ @Rjected +crates/storage/db/ @joshieDo @rakita +crates/storage/errors/ @rakita crates/storage/libmdbx-rs/ @rakita @shekhirin crates/storage/nippy-jar/ @joshieDo @shekhirin crates/storage/provider/ @rakita @joshieDo @shekhirin -crates/storage/storage-api/ @joshieDo @rkrasiuk +crates/storage/storage-api/ @joshieDo crates/tasks/ @mattsse crates/tokio-util/ @fgimenez -crates/tracing/ @onbjerg -crates/transaction-pool/ @mattsse -crates/trie/ @rkrasiuk @Rjected @shekhirin -crates/ress @rkrasiuk -etc/ @Rjected @onbjerg @shekhirin -.github/ @onbjerg @gakonst @DaniPopes +crates/transaction-pool/ @mattsse @yongkangc +crates/trie/ @Rjected @shekhirin @mediocregopher +etc/ @Rjected @shekhirin +.github/ @gakonst @DaniPopes diff --git a/.github/assets/check_wasm.sh b/.github/assets/check_wasm.sh index b47655f2dd0..874b7d508c6 100755 --- a/.github/assets/check_wasm.sh +++ b/.github/assets/check_wasm.sh @@ -11,6 +11,7 @@ exclude_crates=( # The following require investigation if they can be fixed reth-basic-payload-builder reth-bench + reth-bench-compare reth-cli reth-cli-commands reth-cli-runner @@ -40,6 +41,7 @@ exclude_crates=( reth-node-events reth-node-metrics reth-optimism-cli + reth-optimism-flashblocks reth-optimism-node reth-optimism-payload-builder reth-optimism-rpc @@ -48,6 +50,8 @@ exclude_crates=( reth-rpc-api reth-rpc-api-testing-util reth-rpc-builder + reth-rpc-convert + reth-rpc-e2e-tests reth-rpc-engine-api reth-rpc-eth-api reth-rpc-eth-types @@ -58,21 +62,26 @@ exclude_crates=( reth-ress-provider # The following are not supposed to be working reth # all of the crates below + reth-storage-rpc-provider reth-invalid-block-hooks # reth-provider reth-libmdbx # mdbx reth-mdbx-sys # mdbx reth-payload-builder # reth-metrics reth-provider # tokio reth-prune # tokio + reth-prune-static-files # reth-provider reth-stages-api # reth-provider, reth-prune reth-static-file # tokio reth-transaction-pool # c-kzg reth-payload-util # reth-transaction-pool reth-trie-parallel # tokio + reth-trie-sparse-parallel # rayon reth-testing-utils reth-optimism-txpool # reth-transaction-pool reth-era-downloader # tokio reth-era-utils # tokio + reth-tracing-otlp + reth-node-ethstats ) # Array to hold the results diff --git a/.github/assets/hive/build_simulators.sh b/.github/assets/hive/build_simulators.sh index 44792bde076..d65e609e700 100755 --- a/.github/assets/hive/build_simulators.sh +++ b/.github/assets/hive/build_simulators.sh @@ -11,7 +11,8 @@ go build . # Run each hive command in the background for each simulator and wait echo "Building images" -./hive -client reth --sim "ethereum/eest" --sim.buildarg fixtures=https://github.com/ethereum/execution-spec-tests/releases/download/v4.4.0/fixtures_develop.tar.gz --sim.buildarg branch=v4.4.0 -sim.timelimit 1s || true & +# TODO: test code has been moved from https://github.com/ethereum/execution-spec-tests to https://github.com/ethereum/execution-specs we need to pin eels branch with `--sim.buildarg branch=` once we have the fusaka release tagged on the new repo +./hive -client reth --sim "ethereum/eels" --sim.buildarg fixtures=https://github.com/ethereum/execution-spec-tests/releases/download/v5.3.0/fixtures_develop.tar.gz -sim.timelimit 1s || true & ./hive -client reth --sim "ethereum/engine" -sim.timelimit 1s || true & ./hive -client reth --sim "devp2p" -sim.timelimit 1s || true & ./hive -client reth --sim "ethereum/rpc-compat" -sim.timelimit 1s || true & @@ -27,8 +28,8 @@ docker save hive/hiveproxy:latest -o ../hive_assets/hiveproxy.tar & saving_pids+ docker save hive/simulators/devp2p:latest -o ../hive_assets/devp2p.tar & saving_pids+=( $! ) docker save hive/simulators/ethereum/engine:latest -o ../hive_assets/engine.tar & saving_pids+=( $! ) docker save hive/simulators/ethereum/rpc-compat:latest -o ../hive_assets/rpc_compat.tar & saving_pids+=( $! ) -docker save hive/simulators/ethereum/eest/consume-engine:latest -o ../hive_assets/eest_engine.tar & saving_pids+=( $! ) -docker save hive/simulators/ethereum/eest/consume-rlp:latest -o ../hive_assets/eest_rlp.tar & saving_pids+=( $! ) +docker save hive/simulators/ethereum/eels/consume-engine:latest -o ../hive_assets/eels_engine.tar & saving_pids+=( $! ) +docker save hive/simulators/ethereum/eels/consume-rlp:latest -o ../hive_assets/eels_rlp.tar & saving_pids+=( $! ) docker save hive/simulators/smoke/genesis:latest -o ../hive_assets/smoke_genesis.tar & saving_pids+=( $! ) docker save hive/simulators/smoke/network:latest -o ../hive_assets/smoke_network.tar & saving_pids+=( $! ) docker save hive/simulators/ethereum/sync:latest -o ../hive_assets/ethereum_sync.tar & saving_pids+=( $! ) diff --git a/.github/assets/hive/expected_failures.yaml b/.github/assets/hive/expected_failures.yaml index 2610fc69cdc..db18aa9ceda 100644 --- a/.github/assets/hive/expected_failures.yaml +++ b/.github/assets/hive/expected_failures.yaml @@ -6,16 +6,16 @@ rpc-compat: - debug_getRawReceipts/get-block-n (reth) - debug_getRawTransaction/get-invalid-hash (reth) - - eth_call/call-callenv (reth) - eth_getStorageAt/get-storage-invalid-key-too-large (reth) - eth_getStorageAt/get-storage-invalid-key (reth) - - eth_getTransactionReceipt/get-access-list (reth) - - eth_getTransactionReceipt/get-blob-tx (reth) - - eth_getTransactionReceipt/get-dynamic-fee (reth) - eth_getTransactionReceipt/get-legacy-contract (reth) - eth_getTransactionReceipt/get-legacy-input (reth) - eth_getTransactionReceipt/get-legacy-receipt (reth) + # after https://github.com/paradigmxyz/reth/pull/16742 we start the node in + # syncing mode, the test expects syncing to be false on start + - eth_syncing/check-syncing (reth) + # no fix due to https://github.com/paradigmxyz/reth/issues/8732 engine-withdrawals: - Withdrawals Fork On Genesis (Paris) (reth) @@ -28,39 +28,47 @@ engine-withdrawals: - Withdraw zero amount (Paris) (reth) - Empty Withdrawals (Paris) (reth) - Corrupted Block Hash Payload (INVALID) (Paris) (reth) - - Withdrawals Fork on Block 1 - 8 Block Re-Org NewPayload (Paris) (reth) - - Withdrawals Fork on Block 1 - 8 Block Re-Org, Sync (Paris) (reth) - - Withdrawals Fork on Block 8 - 10 Block Re-Org NewPayload (Paris) (reth) - - Withdrawals Fork on Block 8 - 10 Block Re-Org Sync (Paris) (reth) - Withdrawals Fork on Canonical Block 8 / Side Block 7 - 10 Block Re-Org (Paris) (reth) - - Withdrawals Fork on Canonical Block 8 / Side Block 7 - 10 Block Re-Org Sync (Paris) (reth) - - Withdrawals Fork on Canonical Block 8 / Side Block 9 - 10 Block Re-Org (Paris) (reth) - - Withdrawals Fork on Canonical Block 8 / Side Block 9 - 10 Block Re-Org Sync (Paris) (reth) -engine-api: [] +engine-api: [ ] # no fix due to https://github.com/paradigmxyz/reth/issues/8732 engine-cancun: - Invalid PayloadAttributes, Missing BeaconRoot, Syncing=True (Cancun) (reth) - - Invalid NewPayload, ExcessBlobGas, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth) + # the test fails with older versions of the code for which it passed before, probably related to changes + # in hive or its dependencies + - Blob Transaction Ordering, Multiple Clients (Cancun) (reth) -sync: [] +sync: [ ] -# https://github.com/ethereum/hive/issues/1277 -engine-auth: - - "JWT Authentication: No time drift, correct secret (Paris) (reth)" - - "JWT Authentication: Negative time drift, within limit, correct secret (Paris) (reth)" - - "JWT Authentication: Positive time drift, within limit, correct secret (Paris) (reth)" +engine-auth: [ ] -# 7702 test - no fix: it’s too expensive to check whether the storage is empty on each creation -# 6110 related tests - may start passing when fixtures improve -# 7002 related tests - post-fork test, should fix for spec compliance but not -# realistic on mainnet -# 7251 related tests - modified contract, not necessarily practical on mainnet, -# worth re-visiting when more of these related tests are passing -eest/consume-engine: +# EIP-7610 related tests (Revert creation in case of non-empty storage): +# +# tests/prague/eip7702_set_code_tx/test_set_code_txs.py::test_set_code_to_non_empty_storage +# The test artificially creates an empty account with storage, then tests EIP-7610's behavior. +# On mainnet, ~25 such accounts exist as contract addresses (derived from keccak(prefix, caller, +# nonce/salt), not from public keys). No private key exists for contract addresses. To trigger +# this with EIP-7702, you'd need to recover a private key from one of the already deployed contract addresses - mathematically impossible. +# +# tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_* +# Requires hash collision on create2 address to target already deployed accounts with storage. +# ~20-30 such accounts exist from before the state-clear EIP. Creating new accounts targeting +# these requires hash collision - mathematically impossible to trigger on mainnet. +# ref: https://github.com/ethereum/go-ethereum/pull/28666#issuecomment-1891997143 +# +# System contract tests (already fixed and deployed): +# +# tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout and test_invalid_log_length +# System contract is already fixed and deployed; tests cover scenarios where contract is +# malformed which can't happen retroactively. No point in adding checks. +# +# tests/prague/eip7002_el_triggerable_withdrawals/test_contract_deployment.py::test_system_contract_deployment +# tests/prague/eip7251_consolidations/test_contract_deployment.py::test_system_contract_deployment +# Post-fork system contract deployment tests. Should fix for spec compliance but not realistic +# on mainnet as these contracts are already deployed at the correct addresses. +eels/consume-engine: - tests/prague/eip7702_set_code_tx/test_set_code_txs.py::test_set_code_to_non_empty_storage[fork_Prague-blockchain_test_engine-zero_nonce]-reth - - tests/prague/eip7251_consolidations/test_modified_consolidation_contract.py::test_system_contract_errors[fork_Prague-blockchain_test_engine-system_contract_reaches_gas_limit-system_contract_0x0000bbddc7ce488642fb579f8b00f3a590007251]-reth - tests/prague/eip7251_consolidations/test_contract_deployment.py::test_system_contract_deployment[fork_CancunToPragueAtTime15k-blockchain_test_engine-deploy_after_fork-nonzero_balance]-reth - tests/prague/eip7251_consolidations/test_contract_deployment.py::test_system_contract_deployment[fork_CancunToPragueAtTime15k-blockchain_test_engine-deploy_after_fork-zero_balance]-reth - tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Prague-blockchain_test_engine-log_argument_amount_offset-value_zero]-reth @@ -68,7 +76,6 @@ eest/consume-engine: - tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Prague-blockchain_test_engine-log_argument_index_offset-value_zero]-reth - tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Prague-blockchain_test_engine-log_argument_index_size-value_zero]-reth - tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Prague-blockchain_test_engine-log_argument_pubkey_offset-value_zero]-reth - - tests/prague/eip7002_el_triggerable_withdrawals/test_modified_withdrawal_contract.py::test_system_contract_errors[fork_Prague-blockchain_test_engine-system_contract_reaches_gas_limit-system_contract_0x00000961ef480eb55e80d19ad83579a64c007002]-reth - tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Prague-blockchain_test_engine-log_argument_pubkey_size-value_zero]-reth - tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Prague-blockchain_test_engine-log_argument_signature_offset-value_zero]-reth - tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Prague-blockchain_test_engine-log_argument_signature_size-value_zero]-reth @@ -78,11 +85,76 @@ eest/consume-engine: - tests/prague/eip7002_el_triggerable_withdrawals/test_contract_deployment.py::test_system_contract_deployment[fork_CancunToPragueAtTime15k-blockchain_test_engine-deploy_after_fork-zero_balance]-reth - tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_log_length[fork_Prague-blockchain_test_engine-slice_bytes_False]-reth - tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_log_length[fork_Prague-blockchain_test_engine-slice_bytes_True]-reth - # the next test expects a concrete new format in the error message, there is no spec for this message, so it is ok to ignore - - tests/cancun/eip4844_blobs/test_blob_txs.py::test_blob_type_tx_pre_fork[fork_ShanghaiToCancunAtTime15k-blockchain_test_engine_from_state_test-one_blob_tx]-reth -# 7702 test - no fix: it’s too expensive to check whether the storage is empty on each creation -# rest of tests - see above -eest/consume-rlp: + - tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Osaka-blockchain_test_engine-log_argument_amount_offset-value_zero]-reth + - tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Osaka-blockchain_test_engine-log_argument_amount_size-value_zero]-reth + - tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Osaka-blockchain_test_engine-log_argument_index_offset-value_zero]-reth + - tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Osaka-blockchain_test_engine-log_argument_index_size-value_zero]-reth + - tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Osaka-blockchain_test_engine-log_argument_pubkey_offset-value_zero]-reth + - tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Osaka-blockchain_test_engine-log_argument_pubkey_size-value_zero]-reth + - tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Osaka-blockchain_test_engine-log_argument_signature_offset-value_zero]-reth + - tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Osaka-blockchain_test_engine-log_argument_signature_size-value_zero]-reth + - tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Osaka-blockchain_test_engine-log_argument_withdrawal_credentials_offset-value_zero]-reth + - tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Osaka-blockchain_test_engine-log_argument_withdrawal_credentials_size-value_zero]-reth + - tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_log_length[fork_Osaka-blockchain_test_engine-slice_bytes_False]-reth + - tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_log_length[fork_Osaka-blockchain_test_engine-slice_bytes_True]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Osaka-tx_type_0-blockchain_test_engine_from_state_test-non-empty-balance-revert-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Prague-tx_type_0-blockchain_test_engine_from_state_test-non-empty-balance-correct-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Paris-tx_type_1-blockchain_test_engine_from_state_test-non-empty-balance-correct-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Paris-tx_type_2-blockchain_test_engine_from_state_test-non-empty-balance-correct-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Shanghai-tx_type_1-blockchain_test_engine_from_state_test-non-empty-balance-correct-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Shanghai-tx_type_2-blockchain_test_engine_from_state_test-non-empty-balance-revert-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Cancun-blockchain_test_engine_from_state_test-opcode_CREATE-non-empty-balance-revert-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Shanghai-tx_type_1-blockchain_test_engine_from_state_test-non-empty-balance-revert-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Shanghai-tx_type_2-blockchain_test_engine_from_state_test-non-empty-balance-correct-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Cancun-blockchain_test_engine_from_state_test-opcode_CREATE2-non-empty-balance-revert-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Cancun-blockchain_test_engine_from_state_test-opcode_CREATE-non-empty-balance-correct-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Osaka-blockchain_test_engine_from_state_test-opcode_CREATE-non-empty-balance-correct-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Osaka-tx_type_1-blockchain_test_engine_from_state_test-non-empty-balance-correct-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Cancun-blockchain_test_engine_from_state_test-opcode_CREATE2-non-empty-balance-correct-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Osaka-blockchain_test_engine_from_state_test-opcode_CREATE2-non-empty-balance-correct-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Osaka-blockchain_test_engine_from_state_test-opcode_CREATE-non-empty-balance-revert-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Paris-blockchain_test_engine_from_state_test-opcode_CREATE-non-empty-balance-correct-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Paris-blockchain_test_engine_from_state_test-opcode_CREATE-non-empty-balance-revert-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Paris-blockchain_test_engine_from_state_test-opcode_CREATE2-non-empty-balance-revert-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Paris-blockchain_test_engine_from_state_test-opcode_CREATE2-non-empty-balance-correct-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Cancun-tx_type_1-blockchain_test_engine_from_state_test-non-empty-balance-correct-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Osaka-blockchain_test_engine_from_state_test-opcode_CREATE2-non-empty-balance-revert-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Prague-blockchain_test_engine_from_state_test-opcode_CREATE-non-empty-balance-correct-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Prague-blockchain_test_engine_from_state_test-opcode_CREATE-non-empty-balance-revert-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Prague-blockchain_test_engine_from_state_test-opcode_CREATE2-non-empty-balance-correct-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Prague-blockchain_test_engine_from_state_test-opcode_CREATE2-non-empty-balance-revert-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Shanghai-blockchain_test_engine_from_state_test-opcode_CREATE-non-empty-balance-correct-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Shanghai-blockchain_test_engine_from_state_test-opcode_CREATE-non-empty-balance-revert-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Shanghai-blockchain_test_engine_from_state_test-opcode_CREATE2-non-empty-balance-correct-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Shanghai-blockchain_test_engine_from_state_test-opcode_CREATE2-non-empty-balance-revert-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Cancun-tx_type_0-blockchain_test_engine_from_state_test-non-empty-balance-revert-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Osaka-tx_type_2-blockchain_test_engine_from_state_test-non-empty-balance-correct-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Osaka-tx_type_1-blockchain_test_engine_from_state_test-non-empty-balance-revert-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Cancun-tx_type_2-blockchain_test_engine_from_state_test-non-empty-balance-correct-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Cancun-tx_type_1-blockchain_test_engine_from_state_test-non-empty-balance-revert-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Paris-tx_type_0-blockchain_test_engine_from_state_test-non-empty-balance-correct-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Osaka-tx_type_2-blockchain_test_engine_from_state_test-non-empty-balance-revert-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Cancun-tx_type_2-blockchain_test_engine_from_state_test-non-empty-balance-revert-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Osaka-tx_type_0-blockchain_test_engine_from_state_test-non-empty-balance-correct-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Paris-tx_type_1-blockchain_test_engine_from_state_test-non-empty-balance-revert-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Paris-tx_type_0-blockchain_test_engine_from_state_test-non-empty-balance-revert-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Paris-tx_type_2-blockchain_test_engine_from_state_test-non-empty-balance-revert-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Cancun-tx_type_0-blockchain_test_engine_from_state_test-non-empty-balance-correct-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Prague-tx_type_1-blockchain_test_engine_from_state_test-non-empty-balance-correct-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Prague-tx_type_0-blockchain_test_engine_from_state_test-non-empty-balance-revert-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Shanghai-tx_type_0-blockchain_test_engine_from_state_test-non-empty-balance-revert-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Prague-tx_type_2-blockchain_test_engine_from_state_test-non-empty-balance-correct-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Prague-tx_type_1-blockchain_test_engine_from_state_test-non-empty-balance-revert-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Prague-tx_type_2-blockchain_test_engine_from_state_test-non-empty-balance-revert-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Shanghai-tx_type_0-blockchain_test_engine_from_state_test-non-empty-balance-correct-initcode]-reth + +# Blob limit tests: +# +# tests/osaka/eip7594_peerdas/test_max_blob_per_tx.py::test_max_blobs_per_tx_fork_transition[fork_PragueToOsakaAtTime15k-blob_count_7-blockchain_test] +# this test inserts a chain via chain.rlp where the last block is invalid, but expects import to stop there, this doesn't work properly with our pipeline import approach hence the import fails when the invalid block is detected. +#. In other words, if this test fails, this means we're correctly rejecting the block. +#. The same test exists in the consume-engine simulator where it is passing as expected +eels/consume-rlp: - tests/prague/eip7702_set_code_tx/test_set_code_txs.py::test_set_code_to_non_empty_storage[fork_Prague-blockchain_test-zero_nonce]-reth - tests/prague/eip7251_consolidations/test_modified_consolidation_contract.py::test_system_contract_errors[fork_Prague-blockchain_test_engine-system_contract_reaches_gas_limit-system_contract_0x0000bbddc7ce488642fb579f8b00f3a590007251]-reth - tests/prague/eip7251_consolidations/test_contract_deployment.py::test_system_contract_deployment[fork_CancunToPragueAtTime15k-blockchain_test_engine-deploy_after_fork-nonzero_balance]-reth @@ -98,13 +170,74 @@ eest/consume-rlp: - tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Prague-blockchain_test_engine-log_argument_signature_size-value_zero]-reth - tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Prague-blockchain_test_engine-log_argument_withdrawal_credentials_offset-value_zero]-reth - tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Prague-blockchain_test_engine-log_argument_withdrawal_credentials_size-value_zero]-reth + - tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Osaka-blockchain_test_engine-log_argument_amount_offset-value_zero]-reth + - tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Osaka-blockchain_test_engine-log_argument_amount_size-value_zero]-reth + - tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Osaka-blockchain_test_engine-log_argument_index_offset-value_zero]-reth + - tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Osaka-blockchain_test_engine-log_argument_index_size-value_zero]-reth + - tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Osaka-blockchain_test_engine-log_argument_pubkey_offset-value_zero]-reth + - tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Osaka-blockchain_test_engine-log_argument_pubkey_size-value_zero]-reth + - tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Osaka-blockchain_test_engine-log_argument_signature_offset-value_zero]-reth + - tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Osaka-blockchain_test_engine-log_argument_signature_size-value_zero]-reth + - tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Osaka-blockchain_test_engine-log_argument_withdrawal_credentials_offset-value_zero]-reth + - tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Osaka-blockchain_test_engine-log_argument_withdrawal_credentials_size-value_zero]-reth - tests/prague/eip7002_el_triggerable_withdrawals/test_contract_deployment.py::test_system_contract_deployment[fork_CancunToPragueAtTime15k-blockchain_test_engine-deploy_after_fork-nonzero_balance]-reth - tests/prague/eip7002_el_triggerable_withdrawals/test_contract_deployment.py::test_system_contract_deployment[fork_CancunToPragueAtTime15k-blockchain_test_engine-deploy_after_fork-zero_balance]-reth - tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_log_length[fork_Prague-blockchain_test_engine-slice_bytes_False]-reth - tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_log_length[fork_Prague-blockchain_test_engine-slice_bytes_True]-reth - - tests/prague/eip7251_consolidations/test_modified_consolidation_contract.py::test_system_contract_errors[fork_Prague-blockchain_test-system_contract_reaches_gas_limit-system_contract_0x0000bbddc7ce488642fb579f8b00f3a590007251]-reth + - tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_log_length[fork_Osaka-blockchain_test_engine-slice_bytes_False]-reth + - tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_log_length[fork_Osaka-blockchain_test_engine-slice_bytes_True]-reth + - tests/osaka/eip7594_peerdas/test_max_blob_per_tx.py::test_max_blobs_per_tx_fork_transition[fork_PragueToOsakaAtTime15k-blob_count_7-blockchain_test]-reth - tests/prague/eip7251_consolidations/test_contract_deployment.py::test_system_contract_deployment[fork_CancunToPragueAtTime15k-blockchain_test-deploy_after_fork-nonzero_balance]-reth - tests/prague/eip7251_consolidations/test_contract_deployment.py::test_system_contract_deployment[fork_CancunToPragueAtTime15k-blockchain_test-deploy_after_fork-zero_balance]-reth - - tests/prague/eip7002_el_triggerable_withdrawals/test_modified_withdrawal_contract.py::test_system_contract_errors[fork_Prague-blockchain_test-system_contract_reaches_gas_limit-system_contract_0x00000961ef480eb55e80d19ad83579a64c007002]-reth - tests/prague/eip7002_el_triggerable_withdrawals/test_contract_deployment.py::test_system_contract_deployment[fork_CancunToPragueAtTime15k-blockchain_test-deploy_after_fork-nonzero_balance]-reth - tests/prague/eip7002_el_triggerable_withdrawals/test_contract_deployment.py::test_system_contract_deployment[fork_CancunToPragueAtTime15k-blockchain_test-deploy_after_fork-zero_balance]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Shanghai-tx_type_1-blockchain_test_from_state_test-non-empty-balance-correct-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Shanghai-tx_type_0-blockchain_test_from_state_test-non-empty-balance-revert-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Prague-tx_type_0-blockchain_test_from_state_test-non-empty-balance-correct-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Paris-tx_type_2-blockchain_test_from_state_test-non-empty-balance-correct-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Paris-tx_type_1-blockchain_test_from_state_test-non-empty-balance-correct-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Prague-tx_type_1-blockchain_test_from_state_test-non-empty-balance-correct-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Shanghai-tx_type_1-blockchain_test_from_state_test-non-empty-balance-revert-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Shanghai-tx_type_2-blockchain_test_from_state_test-non-empty-balance-correct-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Paris-tx_type_1-blockchain_test_from_state_test-non-empty-balance-revert-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Paris-tx_type_2-blockchain_test_from_state_test-non-empty-balance-revert-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Osaka-tx_type_0-blockchain_test_from_state_test-non-empty-balance-revert-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Cancun-blockchain_test_from_state_test-opcode_CREATE-non-empty-balance-correct-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Cancun-blockchain_test_from_state_test-opcode_CREATE2-non-empty-balance-correct-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Shanghai-tx_type_2-blockchain_test_from_state_test-non-empty-balance-revert-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Osaka-tx_type_1-blockchain_test_from_state_test-non-empty-balance-correct-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Cancun-blockchain_test_from_state_test-opcode_CREATE-non-empty-balance-revert-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Osaka-blockchain_test_from_state_test-opcode_CREATE-non-empty-balance-revert-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Cancun-blockchain_test_from_state_test-opcode_CREATE2-non-empty-balance-revert-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Osaka-blockchain_test_from_state_test-opcode_CREATE-non-empty-balance-correct-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Osaka-blockchain_test_from_state_test-opcode_CREATE2-non-empty-balance-revert-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Paris-blockchain_test_from_state_test-opcode_CREATE-non-empty-balance-correct-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Osaka-blockchain_test_from_state_test-opcode_CREATE2-non-empty-balance-correct-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Paris-blockchain_test_from_state_test-opcode_CREATE-non-empty-balance-revert-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Paris-blockchain_test_from_state_test-opcode_CREATE2-non-empty-balance-revert-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Paris-blockchain_test_from_state_test-opcode_CREATE2-non-empty-balance-correct-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Prague-blockchain_test_from_state_test-opcode_CREATE-non-empty-balance-correct-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Prague-blockchain_test_from_state_test-opcode_CREATE-non-empty-balance-revert-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Prague-blockchain_test_from_state_test-opcode_CREATE2-non-empty-balance-correct-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Prague-blockchain_test_from_state_test-opcode_CREATE2-non-empty-balance-revert-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Shanghai-blockchain_test_from_state_test-opcode_CREATE-non-empty-balance-revert-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Shanghai-blockchain_test_from_state_test-opcode_CREATE-non-empty-balance-correct-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Cancun-tx_type_0-blockchain_test_from_state_test-non-empty-balance-revert-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Shanghai-blockchain_test_from_state_test-opcode_CREATE2-non-empty-balance-correct-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Shanghai-blockchain_test_from_state_test-opcode_CREATE2-non-empty-balance-revert-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Cancun-tx_type_1-blockchain_test_from_state_test-non-empty-balance-correct-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Osaka-tx_type_2-blockchain_test_from_state_test-non-empty-balance-correct-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Osaka-tx_type_1-blockchain_test_from_state_test-non-empty-balance-revert-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Cancun-tx_type_2-blockchain_test_from_state_test-non-empty-balance-correct-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Cancun-tx_type_1-blockchain_test_from_state_test-non-empty-balance-revert-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Osaka-tx_type_2-blockchain_test_from_state_test-non-empty-balance-revert-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Osaka-tx_type_0-blockchain_test_from_state_test-non-empty-balance-correct-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Paris-tx_type_0-blockchain_test_from_state_test-non-empty-balance-correct-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Cancun-tx_type_2-blockchain_test_from_state_test-non-empty-balance-revert-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Paris-tx_type_0-blockchain_test_from_state_test-non-empty-balance-revert-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Prague-tx_type_0-blockchain_test_from_state_test-non-empty-balance-revert-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Cancun-tx_type_0-blockchain_test_from_state_test-non-empty-balance-correct-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Prague-tx_type_2-blockchain_test_from_state_test-non-empty-balance-correct-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Prague-tx_type_1-blockchain_test_from_state_test-non-empty-balance-revert-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Prague-tx_type_2-blockchain_test_from_state_test-non-empty-balance-revert-initcode]-reth + - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Shanghai-tx_type_0-blockchain_test_from_state_test-non-empty-balance-correct-initcode]-reth diff --git a/.github/assets/hive/ignored_tests.yaml b/.github/assets/hive/ignored_tests.yaml new file mode 100644 index 00000000000..22de89312f2 --- /dev/null +++ b/.github/assets/hive/ignored_tests.yaml @@ -0,0 +1,36 @@ +# Ignored Tests Configuration +# +# This file contains tests that should be ignored for various reasons (flaky, known issues, etc). +# These tests will be IGNORED in the CI results - they won't cause the build to fail +# regardless of whether they pass or fail. +# +# Format +# test_suite: +# - "test name 1" +# - "test name 2" +# +# When a test should no longer be ignored, remove it from this list. + +# flaky +engine-withdrawals: + - Withdrawals Fork on Block 1 - 8 Block Re-Org NewPayload (Paris) (reth) + - Withdrawals Fork on Block 8 - 10 Block Re-Org NewPayload (Paris) (reth) + - Withdrawals Fork on Canonical Block 8 / Side Block 7 - 10 Block Re-Org (Paris) (reth) + - Sync after 128 blocks - Withdrawals on Block 2 - Multiple Withdrawal Accounts (Paris) (reth) +engine-cancun: + - Transaction Re-Org, New Payload on Revert Back (Cancun) (reth) + - Transaction Re-Org, Re-Org to Different Block (Cancun) (reth) + - Transaction Re-Org, Re-Org Out (Cancun) (reth) + - Invalid Missing Ancestor ReOrg, StateRoot, EmptyTxs=False, Invalid P9 (Cancun) (reth) + - Multiple New Payloads Extending Canonical Chain, Wait for Canonical Payload (Cancun) (reth) +engine-api: + - Transaction Re-Org, Re-Org Out (Paris) (reth) + - Transaction Re-Org, Re-Org to Different Block (Paris) (reth) + - Transaction Re-Org, New Payload on Revert Back (Paris) (reth) + - Transaction Re-Org, Re-Org to Different Block (Paris) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Transaction Nonce, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Paris) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Transaction Signature, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Paris) (reth) + - Invalid Missing Ancestor Syncing ReOrg, Transaction Signature, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Paris) (reth) + - Invalid Missing Ancestor ReOrg, StateRoot, EmptyTxs=True, Invalid P10 (Paris) (reth) + - Multiple New Payloads Extending Canonical Chain, Wait for Canonical Payload (Paris) (reth) + - Multiple New Payloads Extending Canonical Chain, Set Head to First Payload Received (Paris) (reth) diff --git a/.github/assets/hive/load_images.sh b/.github/assets/hive/load_images.sh index 37a2f82de54..e7dd7c99f4a 100755 --- a/.github/assets/hive/load_images.sh +++ b/.github/assets/hive/load_images.sh @@ -11,8 +11,8 @@ IMAGES=( "/tmp/smoke_genesis.tar" "/tmp/smoke_network.tar" "/tmp/ethereum_sync.tar" - "/tmp/eest_engine.tar" - "/tmp/eest_rlp.tar" + "/tmp/eels_engine.tar" + "/tmp/eels_rlp.tar" "/tmp/reth_image.tar" ) diff --git a/.github/assets/hive/parse.py b/.github/assets/hive/parse.py index c408a4d1336..11a30ae095b 100644 --- a/.github/assets/hive/parse.py +++ b/.github/assets/hive/parse.py @@ -7,6 +7,7 @@ parser = argparse.ArgumentParser(description="Check for unexpected test results based on an exclusion list.") parser.add_argument("report_json", help="Path to the hive report JSON file.") parser.add_argument("--exclusion", required=True, help="Path to the exclusion YAML file.") +parser.add_argument("--ignored", required=True, help="Path to the ignored tests YAML file.") args = parser.parse_args() # Load hive JSON @@ -18,13 +19,30 @@ exclusion_data = yaml.safe_load(file) exclusions = exclusion_data.get(report['name'], []) +# Load ignored tests YAML +with open(args.ignored, 'r') as file: + ignored_data = yaml.safe_load(file) + ignored_tests = ignored_data.get(report['name'], []) + # Collect unexpected failures and passes unexpected_failures = [] unexpected_passes = [] +ignored_results = {'passed': [], 'failed': []} for test in report['testCases'].values(): test_name = test['name'] test_pass = test['summaryResult']['pass'] + + # Check if this is an ignored test + if test_name in ignored_tests: + # Track ignored test results for informational purposes + if test_pass: + ignored_results['passed'].append(test_name) + else: + ignored_results['failed'].append(test_name) + continue # Skip this test - don't count it as unexpected + + # Check against expected failures if test_name in exclusions: if test_pass: unexpected_passes.append(test_name) @@ -32,6 +50,19 @@ if not test_pass: unexpected_failures.append(test_name) +# Print summary of ignored tests if any were ignored +if ignored_results['passed'] or ignored_results['failed']: + print("Ignored Tests:") + if ignored_results['passed']: + print(f" Passed ({len(ignored_results['passed'])} tests):") + for test in ignored_results['passed']: + print(f" {test}") + if ignored_results['failed']: + print(f" Failed ({len(ignored_results['failed'])} tests):") + for test in ignored_results['failed']: + print(f" {test}") + print() + # Check if there are any unexpected failures or passes and exit with error if unexpected_failures or unexpected_passes: if unexpected_failures: diff --git a/.github/assets/kurtosis_op_network_params.yaml b/.github/assets/kurtosis_op_network_params.yaml index 87670587395..c90be9e1ad2 100644 --- a/.github/assets/kurtosis_op_network_params.yaml +++ b/.github/assets/kurtosis_op_network_params.yaml @@ -4,7 +4,6 @@ ethereum_package: el_extra_params: - "--rpc.eth-proof-window=100" cl_type: teku - cl_image: "consensys/teku:25.4.0" network_params: preset: minimal genesis_delay: 5 @@ -19,12 +18,19 @@ ethereum_package: }' optimism_package: chains: - - participants: - - el_type: op-geth - cl_type: op-node - - el_type: op-reth - cl_type: op-node - el_image: "ghcr.io/paradigmxyz/op-reth:kurtosis-ci" + chain0: + participants: + node0: + el: + type: op-geth + cl: + type: op-node + node1: + el: + type: op-reth + image: "ghcr.io/paradigmxyz/op-reth:kurtosis-ci" + cl: + type: op-node network_params: holocene_time_offset: 0 isthmus_time_offset: 0 diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 0215bf304c1..0203a4654a0 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -12,17 +12,13 @@ env: BASELINE: base SEED: reth -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - name: bench jobs: codspeed: runs-on: group: Reth steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: submodules: true - uses: rui314/setup-mold@v1 @@ -37,7 +33,8 @@ jobs: - name: Build the benchmark target(s) run: ./.github/scripts/codspeed-build.sh - name: Run the benchmarks - uses: CodSpeedHQ/action@v3 + uses: CodSpeedHQ/action@v4 with: run: cargo codspeed run --workspace + mode: instrumentation token: ${{ secrets.CODSPEED_TOKEN }} diff --git a/.github/workflows/book.yml b/.github/workflows/book.yml index 837d47e9f84..c4262cbb3ad 100644 --- a/.github/workflows/book.yml +++ b/.github/workflows/book.yml @@ -7,115 +7,53 @@ on: branches: [main] pull_request: branches: [main] + types: [opened, reopened, synchronize, closed] merge_group: jobs: - test: - runs-on: ubuntu-latest - name: test - timeout-minutes: 60 - - steps: - - uses: actions/checkout@v4 - - - name: Install mdbook - run: | - mkdir mdbook - curl -sSL https://github.com/rust-lang/mdBook/releases/download/v0.4.14/mdbook-v0.4.14-x86_64-unknown-linux-gnu.tar.gz | tar -xz --directory=./mdbook - echo $(pwd)/mdbook >> $GITHUB_PATH - - - name: Install mdbook-template - run: | - mkdir mdbook-template - curl -sSL https://github.com/sgoudham/mdbook-template/releases/latest/download/mdbook-template-x86_64-unknown-linux-gnu.tar.gz | tar -xz --directory=./mdbook-template - echo $(pwd)/mdbook-template >> $GITHUB_PATH - - - name: Run tests - run: mdbook test - - lint: - runs-on: ubuntu-latest - name: lint - timeout-minutes: 60 - - steps: - - uses: actions/checkout@v4 - - - name: Install mdbook-linkcheck - run: | - mkdir mdbook-linkcheck - curl -sSL -o mdbook-linkcheck.zip https://github.com/Michael-F-Bryan/mdbook-linkcheck/releases/latest/download/mdbook-linkcheck.x86_64-unknown-linux-gnu.zip - unzip mdbook-linkcheck.zip -d ./mdbook-linkcheck - chmod +x $(pwd)/mdbook-linkcheck/mdbook-linkcheck - echo $(pwd)/mdbook-linkcheck >> $GITHUB_PATH - - - name: Run linkcheck - run: mdbook-linkcheck --standalone - build: runs-on: ubuntu-latest - timeout-minutes: 60 + timeout-minutes: 90 steps: - - uses: actions/checkout@v4 - - uses: rui314/setup-mold@v1 - - uses: dtolnay/rust-toolchain@nightly - - name: Install mdbook - run: | - mkdir mdbook - curl -sSL https://github.com/rust-lang/mdBook/releases/download/v0.4.14/mdbook-v0.4.14-x86_64-unknown-linux-gnu.tar.gz | tar -xz --directory=./mdbook - echo $(pwd)/mdbook >> $GITHUB_PATH - - - name: Install mdbook-template - run: | - mkdir mdbook-template - curl -sSL https://github.com/sgoudham/mdbook-template/releases/latest/download/mdbook-template-x86_64-unknown-linux-gnu.tar.gz | tar -xz --directory=./mdbook-template - echo $(pwd)/mdbook-template >> $GITHUB_PATH + - name: Checkout + uses: actions/checkout@v5 - - uses: Swatinem/rust-cache@v2 + - name: Install bun + uses: oven-sh/setup-bun@v2 with: - cache-on-failure: true + bun-version: v1.2.23 - - name: Build book - run: mdbook build + - name: Install Playwright browsers + # Required for rehype-mermaid to render Mermaid diagrams during build + run: | + cd docs/vocs/ + bun i + npx playwright install --with-deps chromium + + - name: Install Rust nightly + uses: dtolnay/rust-toolchain@nightly - name: Build docs - run: cargo docs --exclude "example-*" - env: - # Keep in sync with ./ci.yml:jobs.docs - RUSTDOCFLAGS: --cfg docsrs --show-type-layout --generate-link-to-definition --enable-index-page -Zunstable-options + run: cd docs/vocs && bash scripts/build-cargo-docs.sh - - name: Move docs to book folder + - name: Build Vocs run: | - mv target/doc target/book/docs + cd docs/vocs/ && bun run build + echo "Vocs Build Complete" - - name: Archive artifact - shell: sh - run: | - chmod -c -R +rX "target/book" | - while read line; do - echo "::warning title=Invalid file permissions automatically fixed::$line" - done - tar \ - --dereference --hard-dereference \ - --directory "target/book" \ - -cvf "$RUNNER_TEMP/artifact.tar" \ - --exclude=.git \ - --exclude=.github \ - . + - name: Setup Pages + uses: actions/configure-pages@v5 - name: Upload artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-pages-artifact@v4 with: - name: github-pages - path: ${{ runner.temp }}/artifact.tar - retention-days: 1 - if-no-files-found: error + path: "./docs/vocs/docs/dist" deploy: # Only deploy if a push to main if: github.ref_name == 'main' && github.event_name == 'push' runs-on: ubuntu-latest - needs: [test, lint, build] + needs: [build] # Grant GITHUB_TOKEN the permissions required to make a Pages deployment permissions: diff --git a/.github/workflows/build-release-binaries.yml b/.github/workflows/build-release-binaries.yml deleted file mode 100644 index 92b26406169..00000000000 --- a/.github/workflows/build-release-binaries.yml +++ /dev/null @@ -1,57 +0,0 @@ -name: build release binaries - -on: - workflow_dispatch: - -env: - CARGO_TERM_COLOR: always - -jobs: - build: - name: build release - runs-on: ${{ matrix.configs.os }} - strategy: - matrix: - configs: - - target: x86_64-unknown-linux-gnu - os: ubuntu-24.04 - profile: maxperf - - target: aarch64-unknown-linux-gnu - os: ubuntu-24.04 - profile: maxperf - - target: x86_64-apple-darwin - os: macos-13 - profile: maxperf - - target: aarch64-apple-darwin - os: macos-14 - profile: maxperf - - target: x86_64-pc-windows-gnu - os: ubuntu-24.04 - profile: maxperf - build: - - command: build - binary: reth - - command: op-build - binary: op-reth - steps: - - uses: actions/checkout@v4 - - uses: rui314/setup-mold@v1 - - uses: dtolnay/rust-toolchain@stable - with: - target: ${{ matrix.configs.target }} - - name: Install cross main - id: cross_main - run: | - cargo install cross --git https://github.com/cross-rs/cross - - uses: Swatinem/rust-cache@v2 - with: - cache-on-failure: true - - - name: Apple M1 setup - if: matrix.configs.target == 'aarch64-apple-darwin' - run: | - echo "SDKROOT=$(xcrun -sdk macosx --show-sdk-path)" >> $GITHUB_ENV - echo "MACOSX_DEPLOYMENT_TARGET=$(xcrun -sdk macosx --show-sdk-platform-version)" >> $GITHUB_ENV - - - name: Build Reth - run: make PROFILE=${{ matrix.configs.profile }} ${{ matrix.build.command }}-${{ matrix.configs.target }} diff --git a/.github/workflows/compact.yml b/.github/workflows/compact.yml index 06ddec20cb7..8a18df872d2 100644 --- a/.github/workflows/compact.yml +++ b/.github/workflows/compact.yml @@ -31,7 +31,7 @@ jobs: with: cache-on-failure: true - name: Checkout base - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: ref: ${{ github.base_ref || 'main' }} # On `main` branch, generates test vectors and serializes them to disk using `Compact`. @@ -39,7 +39,7 @@ jobs: run: | ${{ matrix.bin }} -- test-vectors compact --write - name: Checkout PR - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: clean: false # On incoming merge try to read and decode previously generated vectors with `Compact` diff --git a/.github/workflows/docker-git.yml b/.github/workflows/docker-git.yml index 7542c84f4c3..62830608d67 100644 --- a/.github/workflows/docker-git.yml +++ b/.github/workflows/docker-git.yml @@ -33,7 +33,7 @@ jobs: - name: 'Build and push the git-sha-tagged op-reth image' command: 'make IMAGE_NAME=$OP_IMAGE_NAME DOCKER_IMAGE_NAME=$OP_DOCKER_IMAGE_NAME GIT_SHA=$GIT_SHA PROFILE=maxperf op-docker-build-push-git-sha' steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: rui314/setup-mold@v1 - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 diff --git a/.github/workflows/docker-nightly.yml b/.github/workflows/docker-nightly.yml index 490bb583aa9..213b2314060 100644 --- a/.github/workflows/docker-nightly.yml +++ b/.github/workflows/docker-nightly.yml @@ -35,7 +35,7 @@ jobs: - name: 'Build and push the nightly profiling op-reth image' command: 'make IMAGE_NAME=$OP_IMAGE_NAME DOCKER_IMAGE_NAME=$OP_DOCKER_IMAGE_NAME PROFILE=profiling op-docker-build-push-nightly-profiling' steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Remove bloatware uses: laverdet/remove-bloatware@v1.0.0 with: diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 788e0e60417..0768ea8e79a 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -8,7 +8,6 @@ on: - v* env: - REPO_NAME: ${{ github.repository_owner }}/reth IMAGE_NAME: ${{ github.repository_owner }}/reth OP_IMAGE_NAME: ${{ github.repository_owner }}/op-reth CARGO_TERM_COLOR: always @@ -17,8 +16,45 @@ env: DOCKER_USERNAME: ${{ github.actor }} jobs: + build-rc: + if: contains(github.ref, '-rc') + name: build and push as release candidate + runs-on: ubuntu-24.04 + permissions: + packages: write + contents: read + strategy: + fail-fast: false + matrix: + build: + - name: "Build and push reth image" + command: "make IMAGE_NAME=$IMAGE_NAME DOCKER_IMAGE_NAME=$DOCKER_IMAGE_NAME PROFILE=maxperf docker-build-push" + - name: "Build and push op-reth image" + command: "make IMAGE_NAME=$OP_IMAGE_NAME DOCKER_IMAGE_NAME=$OP_DOCKER_IMAGE_NAME PROFILE=maxperf op-docker-build-push" + steps: + - uses: actions/checkout@v5 + - uses: rui314/setup-mold@v1 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + - name: Install cross main + id: cross_main + run: | + cargo install cross --git https://github.com/cross-rs/cross + - name: Log in to Docker + run: | + echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io --username ${DOCKER_USERNAME} --password-stdin + - name: Set up Docker builder + run: | + docker run --privileged --rm tonistiigi/binfmt --install arm64,amd64 + docker buildx create --use --name cross-builder + - name: Build and push ${{ matrix.build.name }} + run: ${{ matrix.build.command }} + build: - name: build and push + if: ${{ !contains(github.ref, '-rc') }} + name: build and push as latest runs-on: ubuntu-24.04 permissions: packages: write @@ -27,16 +63,12 @@ jobs: fail-fast: false matrix: build: - - name: 'Build and push reth image' - command: 'make PROFILE=maxperf docker-build-push' - - name: 'Build and push reth image, tag as "latest"' - command: 'make PROFILE=maxperf docker-build-push-latest' - - name: 'Build and push op-reth image' - command: 'make IMAGE_NAME=$OP_IMAGE_NAME DOCKER_IMAGE_NAME=$OP_DOCKER_IMAGE_NAME PROFILE=maxperf op-docker-build-push' - - name: 'Build and push op-reth image, tag as "latest"' - command: 'make IMAGE_NAME=$OP_IMAGE_NAME DOCKER_IMAGE_NAME=$OP_DOCKER_IMAGE_NAME PROFILE=maxperf op-docker-build-push-latest' + - name: "Build and push reth image" + command: "make IMAGE_NAME=$IMAGE_NAME DOCKER_IMAGE_NAME=$DOCKER_IMAGE_NAME PROFILE=maxperf docker-build-push-latest" + - name: "Build and push op-reth image" + command: "make IMAGE_NAME=$OP_IMAGE_NAME DOCKER_IMAGE_NAME=$OP_DOCKER_IMAGE_NAME PROFILE=maxperf op-docker-build-push-latest" steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: rui314/setup-mold@v1 - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 00000000000..16c9fb2f613 --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,46 @@ +# Runs e2e tests using the testsuite framework + +name: e2e + +on: + pull_request: + merge_group: + push: + branches: [main] + +env: + CARGO_TERM_COLOR: always + SEED: rustethereumethereumrust + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + test: + name: e2e-testsuite + runs-on: + group: Reth + env: + RUST_BACKTRACE: 1 + timeout-minutes: 90 + steps: + - uses: actions/checkout@v5 + - uses: dtolnay/rust-toolchain@stable + - uses: taiki-e/install-action@nextest + - uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + - name: Run e2e tests + run: | + cargo nextest run \ + --locked --features "asm-keccak" \ + --workspace \ + --exclude 'example-*' \ + --exclude 'exex-subscription' \ + --exclude 'reth-bench' \ + --exclude 'ef-tests' \ + --exclude 'op-reth' \ + --exclude 'reth' \ + -E 'binary(e2e_testsuite)' + diff --git a/.github/workflows/hive.yml b/.github/workflows/hive.yml index 095facc7240..7d0ac65bee7 100644 --- a/.github/workflows/hive.yml +++ b/.github/workflows/hive.yml @@ -5,8 +5,7 @@ name: hive on: workflow_dispatch: schedule: - # run every 12 hours - - cron: "0 */12 * * *" + - cron: "0 */6 * * *" env: CARGO_TERM_COLOR: always @@ -28,29 +27,53 @@ jobs: runs-on: group: Reth steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Checkout hive tests - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: repository: ethereum/hive - ref: master path: hivetests - - uses: actions/setup-go@v5 + - name: Get hive commit hash + id: hive-commit + run: echo "hash=$(cd hivetests && git rev-parse HEAD)" >> $GITHUB_OUTPUT + + - uses: actions/setup-go@v6 with: go-version: "^1.13.1" - run: go version + - name: Restore hive assets cache + id: cache-hive + uses: actions/cache@v4 + with: + path: ./hive_assets + key: hive-assets-${{ steps.hive-commit.outputs.hash }}-${{ hashFiles('.github/assets/hive/build_simulators.sh') }} + - name: Build hive assets + if: steps.cache-hive.outputs.cache-hit != 'true' run: .github/assets/hive/build_simulators.sh + - name: Load cached Docker images + if: steps.cache-hive.outputs.cache-hit == 'true' + run: | + cd hive_assets + for tar_file in *.tar; do + if [ -f "$tar_file" ]; then + echo "Loading $tar_file..." + docker load -i "$tar_file" + fi + done + # Make hive binary executable + chmod +x hive + - name: Upload hive assets - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v5 with: name: hive_assets path: ./hive_assets test: - timeout-minutes: 60 + timeout-minutes: 120 strategy: fail-fast: false matrix: @@ -62,17 +85,19 @@ jobs: - sim: ethereum/sync - sim: devp2p limit: discv4 - - sim: devp2p - limit: eth - include: - # failures tracked in https://github.com/paradigmxyz/reth/issues/14825 - - Status - - GetBlockHeaders - - ZeroRequestID - - GetBlockBodies - - MaliciousHandshake - - Transaction - - NewPooledTxs + # started failing after https://github.com/ethereum/go-ethereum/pull/31843, no + # action on our side, remove from here when we get unxpected passes on these tests + # - sim: devp2p + # limit: eth + # include: + # - MaliciousHandshake + # # failures tracked in https://github.com/paradigmxyz/reth/issues/14825 + # - Status + # - GetBlockHeaders + # - ZeroRequestID + # - GetBlockBodies + # - Transaction + # - NewPooledTxs - sim: devp2p limit: discv5 include: @@ -112,36 +137,44 @@ jobs: - debug_ # consume-engine - - sim: ethereum/eest/consume-engine + - sim: ethereum/eels/consume-engine + limit: .*tests/osaka.* + - sim: ethereum/eels/consume-engine limit: .*tests/prague.* - - sim: ethereum/eest/consume-engine + - sim: ethereum/eels/consume-engine limit: .*tests/cancun.* - - sim: ethereum/eest/consume-engine + - sim: ethereum/eels/consume-engine limit: .*tests/shanghai.* - - sim: ethereum/eest/consume-engine + - sim: ethereum/eels/consume-engine limit: .*tests/berlin.* - - sim: ethereum/eest/consume-engine + - sim: ethereum/eels/consume-engine limit: .*tests/istanbul.* - - sim: ethereum/eest/consume-engine + - sim: ethereum/eels/consume-engine limit: .*tests/homestead.* - - sim: ethereum/eest/consume-engine + - sim: ethereum/eels/consume-engine limit: .*tests/frontier.* + - sim: ethereum/eels/consume-engine + limit: .*tests/paris.* # consume-rlp - - sim: ethereum/eest/consume-rlp + - sim: ethereum/eels/consume-rlp + limit: .*tests/osaka.* + - sim: ethereum/eels/consume-rlp limit: .*tests/prague.* - - sim: ethereum/eest/consume-rlp + - sim: ethereum/eels/consume-rlp limit: .*tests/cancun.* - - sim: ethereum/eest/consume-rlp + - sim: ethereum/eels/consume-rlp limit: .*tests/shanghai.* - - sim: ethereum/eest/consume-rlp + - sim: ethereum/eels/consume-rlp limit: .*tests/berlin.* - - sim: ethereum/eest/consume-rlp + - sim: ethereum/eels/consume-rlp limit: .*tests/istanbul.* - - sim: ethereum/eest/consume-rlp + - sim: ethereum/eels/consume-rlp limit: .*tests/homestead.* - - sim: ethereum/eest/consume-rlp + - sim: ethereum/eels/consume-rlp limit: .*tests/frontier.* + - sim: ethereum/eels/consume-rlp + limit: .*tests/paris.* needs: - prepare-reth - prepare-hive @@ -151,18 +184,18 @@ jobs: permissions: issues: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: fetch-depth: 0 - name: Download hive assets - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v6 with: name: hive_assets path: /tmp - name: Download reth image - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v6 with: name: artifacts path: /tmp @@ -176,7 +209,7 @@ jobs: chmod +x /usr/local/bin/hive - name: Checkout hive tests - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: repository: ethereum/hive ref: master @@ -200,7 +233,7 @@ jobs: - name: Parse hive output run: | - find hivetests/workspace/logs -type f -name "*.json" ! -name "hive.json" | xargs -I {} python .github/assets/hive/parse.py {} --exclusion .github/assets/hive/expected_failures.yaml + find hivetests/workspace/logs -type f -name "*.json" ! -name "hive.json" | xargs -I {} python .github/assets/hive/parse.py {} --exclusion .github/assets/hive/expected_failures.yaml --ignored .github/assets/hive/ignored_tests.yaml - name: Print simulator output if: ${{ failure() }} diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index aad87b0fea8..90e3287917e 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -8,8 +8,8 @@ on: push: branches: [main] schedule: - # Run once a day at 3:00 UTC - - cron: '0 3 * * *' + # Run once a day at 3:00 UTC + - cron: "0 3 * * *" env: CARGO_TERM_COLOR: always @@ -32,7 +32,7 @@ jobs: network: ["ethereum", "optimism"] timeout-minutes: 60 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: rui314/setup-mold@v1 - uses: dtolnay/rust-toolchain@stable - name: Install Geth @@ -47,7 +47,7 @@ jobs: cargo nextest run \ --locked --features "asm-keccak ${{ matrix.network }}" \ --workspace --exclude ef-tests \ - -E "kind(test)" + -E "kind(test) and not binary(e2e_testsuite)" - if: matrix.network == 'optimism' name: Run tests run: | @@ -66,17 +66,17 @@ jobs: with: jobs: ${{ toJSON(needs) }} - era-files: + era-files: name: era1 file integration tests once a day if: github.event_name == 'schedule' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: rui314/setup-mold@v1 - - uses: dtolnay/rust-toolchain@stable - - uses: taiki-e/install-action@nextest - - uses: Swatinem/rust-cache@v2 - with: - cache-on-failure: true - - name: run era1 files integration tests - run: cargo nextest run --package reth-era --test it -- --ignored + - uses: actions/checkout@v5 + - uses: rui314/setup-mold@v1 + - uses: dtolnay/rust-toolchain@stable + - uses: taiki-e/install-action@nextest + - uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + - name: run era1 files integration tests + run: cargo nextest run --package reth-era --test it -- --ignored diff --git a/.github/workflows/kurtosis-op.yml b/.github/workflows/kurtosis-op.yml index d9f3d64e102..7477e759209 100644 --- a/.github/workflows/kurtosis-op.yml +++ b/.github/workflows/kurtosis-op.yml @@ -5,7 +5,7 @@ name: kurtosis-op on: workflow_dispatch: schedule: - - cron: "0 */12 * * *" + - cron: "0 */6 * * *" push: tags: @@ -37,12 +37,12 @@ jobs: needs: - prepare-reth steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: fetch-depth: 0 - name: Download reth image - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v6 with: name: artifacts path: /tmp @@ -64,8 +64,8 @@ jobs: kurtosis engine start kurtosis run --enclave op-devnet github.com/ethpandaops/optimism-package --args-file .github/assets/kurtosis_op_network_params.yaml ENCLAVE_ID=$(curl http://127.0.0.1:9779/api/enclaves | jq --raw-output 'keys[0]') - GETH_PORT=$(curl "http://127.0.0.1:9779/api/enclaves/$ENCLAVE_ID/services" | jq '."op-el-2151908-1-op-geth-op-node-op-kurtosis".public_ports.rpc.number') - RETH_PORT=$(curl "http://127.0.0.1:9779/api/enclaves/$ENCLAVE_ID/services" | jq '."op-el-2151908-2-op-reth-op-node-op-kurtosis".public_ports.rpc.number') + GETH_PORT=$(curl "http://127.0.0.1:9779/api/enclaves/$ENCLAVE_ID/services" | jq '."op-el-2151908-node0-op-geth".public_ports.rpc.number') + RETH_PORT=$(curl "http://127.0.0.1:9779/api/enclaves/$ENCLAVE_ID/services" | jq '."op-el-2151908-node1-op-reth".public_ports.rpc.number') echo "GETH_RPC=http://127.0.0.1:$GETH_PORT" >> $GITHUB_ENV echo "RETH_RPC=http://127.0.0.1:$RETH_PORT" >> $GITHUB_ENV diff --git a/.github/workflows/kurtosis.yml b/.github/workflows/kurtosis.yml index 75c20a16c04..b45e997ef73 100644 --- a/.github/workflows/kurtosis.yml +++ b/.github/workflows/kurtosis.yml @@ -5,7 +5,7 @@ name: kurtosis on: workflow_dispatch: schedule: - - cron: "0 */12 * * *" + - cron: "0 */6 * * *" push: tags: @@ -35,12 +35,12 @@ jobs: needs: - prepare-reth steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: fetch-depth: 0 - name: Download reth image - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v6 with: name: artifacts path: /tmp diff --git a/.github/workflows/label-pr.yml b/.github/workflows/label-pr.yml index 07727173531..d4b4bf07cc4 100644 --- a/.github/workflows/label-pr.yml +++ b/.github/workflows/label-pr.yml @@ -11,12 +11,12 @@ jobs: issues: write pull-requests: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: fetch-depth: 0 - name: Label PRs - uses: actions/github-script@v7 + uses: actions/github-script@v8 with: script: | const label_pr = require('./.github/assets/label_pr.js') diff --git a/.github/workflows/lint-actions.yml b/.github/workflows/lint-actions.yml index 4c2171784d3..f408c4f50a5 100644 --- a/.github/workflows/lint-actions.yml +++ b/.github/workflows/lint-actions.yml @@ -12,7 +12,7 @@ jobs: actionlint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Download actionlint id: get_actionlint run: bash <(curl https://raw.githubusercontent.com/rhysd/actionlint/main/scripts/download-actionlint.bash) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 5d67bee3de0..309a25218b7 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -20,11 +20,8 @@ jobs: - type: ethereum args: --workspace --lib --examples --tests --benches --locked features: "ethereum asm-keccak jemalloc jemalloc-prof min-error-logs min-warn-logs min-info-logs min-debug-logs min-trace-logs" - - type: book - args: --manifest-path book/sources/Cargo.toml --workspace --bins - features: "" steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: rui314/setup-mold@v1 - uses: dtolnay/rust-toolchain@clippy with: @@ -46,7 +43,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: rui314/setup-mold@v1 - uses: dtolnay/rust-toolchain@nightly with: @@ -62,7 +59,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: rui314/setup-mold@v1 - uses: dtolnay/rust-toolchain@stable with: @@ -81,7 +78,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 60 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: rui314/setup-mold@v1 - uses: dtolnay/rust-toolchain@stable with: @@ -98,7 +95,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: rui314/setup-mold@v1 - uses: dtolnay/rust-toolchain@stable - uses: taiki-e/install-action@cargo-hack @@ -117,11 +114,11 @@ jobs: - binary: reth - binary: op-reth steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: rui314/setup-mold@v1 - uses: dtolnay/rust-toolchain@master with: - toolchain: "1.86" # MSRV + toolchain: "1.88" # MSRV - uses: Swatinem/rust-cache@v2 with: cache-on-failure: true @@ -134,7 +131,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: rui314/setup-mold@v1 - uses: dtolnay/rust-toolchain@nightly - uses: Swatinem/rust-cache@v2 @@ -151,22 +148,20 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: rui314/setup-mold@v1 - uses: dtolnay/rust-toolchain@nightly with: components: rustfmt - name: Run fmt run: cargo fmt --all --check - - name: Run fmt on book sources - run: cargo fmt --manifest-path book/sources/Cargo.toml --all --check udeps: name: udeps runs-on: ubuntu-latest timeout-minutes: 30 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: rui314/setup-mold@v1 - uses: dtolnay/rust-toolchain@nightly - uses: Swatinem/rust-cache@v2 @@ -180,7 +175,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: rui314/setup-mold@v1 - uses: dtolnay/rust-toolchain@nightly - uses: Swatinem/rust-cache@v2 @@ -189,27 +184,25 @@ jobs: - run: cargo build --bin reth --workspace --features ethereum env: RUSTFLAGS: -D warnings - - run: ./book/cli/update.sh target/debug/reth - - name: Check book changes + - run: ./docs/cli/update.sh target/debug/reth + - name: Check docs changes run: git diff --exit-code - codespell: + typos: runs-on: ubuntu-latest timeout-minutes: 30 steps: - - uses: actions/checkout@v4 - - uses: codespell-project/actions-codespell@v2 - with: - skip: "*.json" + - uses: actions/checkout@v5 + - uses: crate-ci/typos@v1 check-toml: runs-on: ubuntu-latest timeout-minutes: 30 steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Run dprint - uses: dprint/check@v2.2 + uses: dprint/check@v2.3 with: config-path: dprint.json @@ -217,7 +210,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Check dashboard JSON with jq uses: sergeysova/jq-action@v2 with: @@ -227,7 +220,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: rui314/setup-mold@v1 - uses: dtolnay/rust-toolchain@stable - name: Ensure no arbitrary or proptest dependency on default build @@ -239,7 +232,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: rui314/setup-mold@v1 - uses: dtolnay/rust-toolchain@clippy - uses: Swatinem/rust-cache@v2 @@ -256,16 +249,15 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 20 steps: - - uses: actions/checkout@v4 - - name: fetch deps - run: | - # Eagerly pull dependencies - time cargo metadata --format-version=1 --locked > /dev/null - - name: run zepter - run: | - cargo install zepter -f --locked - zepter --version - time zepter run check + - uses: actions/checkout@v5 + - uses: dtolnay/rust-toolchain@stable + - uses: rui314/setup-mold@v1 + - uses: taiki-e/cache-cargo-install-action@v2 + with: + tool: zepter + - name: Eagerly pull dependencies + run: cargo metadata --format-version=1 --locked > /dev/null + - run: zepter run check deny: uses: ithacaxyz/ci/.github/workflows/deny.yml@main @@ -283,7 +275,7 @@ jobs: - fmt - udeps - book - - codespell + - typos - grafana - no-test-deps - features diff --git a/.github/workflows/pr-title.yml b/.github/workflows/pr-title.yml index 1612ecd92a9..e30045423bd 100644 --- a/.github/workflows/pr-title.yml +++ b/.github/workflows/pr-title.yml @@ -21,20 +21,22 @@ jobs: steps: - name: Check title id: lint_pr_title - uses: amannn/action-semantic-pull-request@v5 + uses: amannn/action-semantic-pull-request@v6 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: types: | - fix feat + fix chore test + bench perf refactor docs ci revert + deps continue-on-error: true - name: Add PR Comment for Invalid Title if: steps.lint_pr_title.outcome == 'failure' @@ -54,22 +56,24 @@ jobs: - `fix`: Patches a bug - `chore`: General maintenance tasks or updates - `test`: Adding new tests or modifying existing tests + - `bench`: Adding new benchmarks or modifying existing benchmarks - `perf`: Performance improvements - `refactor`: Changes to improve code structure - `docs`: Documentation updates - `ci`: Changes to CI/CD configurations - `revert`: Reverts a previously merged PR - + - `deps`: Updates dependencies + **Breaking Changes** Breaking changes are noted by using an exclamation mark. For example: - `feat!: changed the API` - `chore(node)!: Removed unused public function` - + **Help** For more information, follow the guidelines here: https://www.conventionalcommits.org/en/v1.0.0/ - + - name: Remove Comment for Valid Title if: steps.lint_pr_title.outcome == 'success' uses: marocchino/sticky-pull-request-comment@v2 diff --git a/.github/workflows/prepare-reth.yml b/.github/workflows/prepare-reth.yml index 422eba19d16..17be3767dce 100644 --- a/.github/workflows/prepare-reth.yml +++ b/.github/workflows/prepare-reth.yml @@ -29,7 +29,7 @@ jobs: runs-on: group: Reth steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - run: mkdir artifacts - name: Set up Docker Buildx @@ -51,7 +51,7 @@ jobs: - name: Upload reth image id: upload - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v5 with: name: artifacts path: ./artifacts diff --git a/.github/workflows/release-dist.yml b/.github/workflows/release-dist.yml index f7df80e81f9..57a6f311d0b 100644 --- a/.github/workflows/release-dist.yml +++ b/.github/workflows/release-dist.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Update Homebrew formula - uses: dawidd6/action-homebrew-bump-formula@v4 + uses: dawidd6/action-homebrew-bump-formula@v5 with: token: ${{ secrets.HOMEBREW }} no_fork: true diff --git a/.github/workflows/release-reproducible.yml b/.github/workflows/release-reproducible.yml new file mode 100644 index 00000000000..e0e7f78aa58 --- /dev/null +++ b/.github/workflows/release-reproducible.yml @@ -0,0 +1,56 @@ +# This workflow is for building and pushing reproducible Docker images for releases. + +name: release-reproducible + +on: + push: + tags: + - v* + +env: + DOCKER_REPRODUCIBLE_IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/reth-reproducible + +jobs: + extract-version: + name: extract version + runs-on: ubuntu-latest + steps: + - name: Extract version + run: echo "VERSION=$(echo ${GITHUB_REF#refs/tags/})" >> $GITHUB_OUTPUT + id: extract_version + outputs: + VERSION: ${{ steps.extract_version.outputs.VERSION }} + + build-reproducible: + name: build and push reproducible image + runs-on: ubuntu-latest + needs: extract-version + permissions: + packages: write + contents: read + steps: + - uses: actions/checkout@v5 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push reproducible image + uses: docker/build-push-action@v6 + with: + context: . + file: ./Dockerfile.reproducible + push: true + tags: | + ${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${{ needs.extract-version.outputs.VERSION }} + ${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:latest + cache-from: type=gha + cache-to: type=gha,mode=max + provenance: false + env: + DOCKER_BUILD_RECORD_UPLOAD: false diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f0fd709585b..b59b967b086 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,55 +7,99 @@ on: push: tags: - v* + workflow_dispatch: + inputs: + dry_run: + description: "Enable dry run mode (builds artifacts but skips uploads and release creation)" + type: boolean + default: false env: REPO_NAME: ${{ github.repository_owner }}/reth - OP_IMAGE_NAME: ${{ github.repository_owner }}/op-reth IMAGE_NAME: ${{ github.repository_owner }}/reth + OP_IMAGE_NAME: ${{ github.repository_owner }}/op-reth + REPRODUCIBLE_IMAGE_NAME: ${{ github.repository_owner }}/reth-reproducible CARGO_TERM_COLOR: always - DOCKER_IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/reth - DOCKER_REPRODUCIBLE_IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/reth-reproducible + DOCKER_IMAGE_NAME_URL: https://ghcr.io/${{ github.repository_owner }}/reth + DOCKER_OP_IMAGE_NAME_URL: https://ghcr.io/${{ github.repository_owner }}/op-reth jobs: + dry-run: + name: check dry run + runs-on: ubuntu-latest + steps: + - run: | + echo "Dry run value: ${{ github.event.inputs.dry_run }}" + echo "Dry run enabled: ${{ github.event.inputs.dry_run == 'true'}}" + echo "Dry run disabled: ${{ github.event.inputs.dry_run != 'true'}}" + extract-version: name: extract version runs-on: ubuntu-latest steps: - name: Extract version - run: echo "VERSION=$(echo ${GITHUB_REF#refs/tags/})" >> $GITHUB_OUTPUT + run: echo "VERSION=${GITHUB_REF_NAME}" >> $GITHUB_OUTPUT id: extract_version outputs: VERSION: ${{ steps.extract_version.outputs.VERSION }} + check-version: + name: check version + runs-on: ubuntu-latest + needs: extract-version + if: ${{ github.event.inputs.dry_run != 'true' }} + steps: + - uses: actions/checkout@v5 + - uses: dtolnay/rust-toolchain@stable + - name: Verify crate version matches tag + # Check that the Cargo version starts with the tag, + # so that Cargo version 1.4.8 can be matched against both v1.4.8 and v1.4.8-rc.1 + run: | + tag="${{ needs.extract-version.outputs.VERSION }}" + tag=${tag#v} + cargo_ver=$(cargo metadata --no-deps --format-version 1 | jq -r '.packages[0].version') + [[ "$tag" == "$cargo_ver"* ]] || { echo "Tag $tag doesn’t match the Cargo version $cargo_ver"; exit 1; } + build: name: build release runs-on: ${{ matrix.configs.os }} needs: extract-version + continue-on-error: ${{ matrix.configs.allow_fail }} strategy: + fail-fast: true matrix: configs: - target: x86_64-unknown-linux-gnu os: ubuntu-24.04 profile: maxperf + allow_fail: false - target: aarch64-unknown-linux-gnu os: ubuntu-24.04 profile: maxperf + allow_fail: false - target: x86_64-apple-darwin - os: macos-13 + os: macos-14 profile: maxperf + allow_fail: false - target: aarch64-apple-darwin os: macos-14 profile: maxperf + allow_fail: false - target: x86_64-pc-windows-gnu os: ubuntu-24.04 profile: maxperf + allow_fail: false + - target: riscv64gc-unknown-linux-gnu + os: ubuntu-24.04 + profile: maxperf + allow_fail: true build: - command: build binary: reth - command: op-build binary: op-reth steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: rui314/setup-mold@v1 - uses: dtolnay/rust-toolchain@stable with: @@ -96,55 +140,24 @@ jobs: shell: bash - name: Upload artifact - uses: actions/upload-artifact@v4 + if: ${{ github.event.inputs.dry_run != 'true' }} + uses: actions/upload-artifact@v5 with: name: ${{ matrix.build.binary }}-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.configs.target }}.tar.gz path: ${{ matrix.build.binary }}-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.configs.target }}.tar.gz - name: Upload signature - uses: actions/upload-artifact@v4 + if: ${{ github.event.inputs.dry_run != 'true' }} + uses: actions/upload-artifact@v5 with: name: ${{ matrix.build.binary }}-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.configs.target }}.tar.gz.asc path: ${{ matrix.build.binary }}-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.configs.target }}.tar.gz.asc - build-reproducible: - name: build and push reproducible image - runs-on: ubuntu-latest - needs: extract-version - permissions: - packages: write - contents: read - steps: - - uses: actions/checkout@v4 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Log in to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and push reproducible image - uses: docker/build-push-action@v6 - with: - context: . - file: ./Dockerfile.reproducible - push: true - tags: | - ${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${{ needs.extract-version.outputs.VERSION }} - ${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:latest - cache-from: type=gha - cache-to: type=gha,mode=max - provenance: false - env: - DOCKER_BUILD_RECORD_UPLOAD: false - draft-release: name: draft release - needs: [build, build-reproducible, extract-version] runs-on: ubuntu-latest + needs: [build, extract-version] + if: ${{ github.event.inputs.dry_run != 'true' }} env: VERSION: ${{ needs.extract-version.outputs.VERSION }} permissions: @@ -153,11 +166,11 @@ jobs: steps: # This is necessary for generating the changelog. # It has to come before "Download Artifacts" or else it deletes the artifacts. - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: fetch-depth: 0 - name: Download artifacts - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v6 - name: Generate full changelog id: changelog run: | @@ -171,6 +184,11 @@ jobs: # The formatting here is borrowed from Lighthouse (which is borrowed from OpenEthereum): # https://github.com/openethereum/openethereum/blob/6c2d392d867b058ff867c4373e40850ca3f96969/.github/workflows/build.yml run: | + prerelease_flag="" + if [[ "${GITHUB_REF}" == *-rc* ]]; then + prerelease_flag="--prerelease" + fi + body=$(cat <<- "ENDBODY" ![image](https://raw.githubusercontent.com/paradigmxyz/reth/main/assets/reth-prod.png) @@ -205,7 +223,7 @@ jobs: | Payload Builders | | | Non-Payload Builders | | - *See [Update Priorities](https://paradigmxyz.github.io/reth/installation/priorities.html) for more information about this table.* + *See [Update Priorities](https://reth.rs/installation/priorities) for more information about this table.* ## All Changes @@ -213,21 +231,31 @@ jobs: ## Binaries - [See pre-built binaries documentation.](https://paradigmxyz.github.io/reth/installation/binaries.html) + [See pre-built binaries documentation.](https://reth.rs/installation/binaries) The binaries are signed with the PGP key: `50FB 7CC5 5B2E 8AFA 59FE 03B7 AA5E D56A 7FBF 253E` + ### Reth + + | System | Architecture | Binary | PGP Signature | + |:---:|:---:|:---:|:---| + | | x86_64 | [reth-${{ env.VERSION }}-x86_64-unknown-linux-gnu.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-x86_64-unknown-linux-gnu.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-x86_64-unknown-linux-gnu.tar.gz.asc) | + | | aarch64 | [reth-${{ env.VERSION }}-aarch64-unknown-linux-gnu.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-aarch64-unknown-linux-gnu.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-aarch64-unknown-linux-gnu.tar.gz.asc) | + | | x86_64 | [reth-${{ env.VERSION }}-x86_64-pc-windows-gnu.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-x86_64-pc-windows-gnu.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-x86_64-pc-windows-gnu.tar.gz.asc) | + | | x86_64 | [reth-${{ env.VERSION }}-x86_64-apple-darwin.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-x86_64-apple-darwin.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-x86_64-apple-darwin.tar.gz.asc) | + | | aarch64 | [reth-${{ env.VERSION }}-aarch64-apple-darwin.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-aarch64-apple-darwin.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-aarch64-apple-darwin.tar.gz.asc) | + | | Docker | [${{ env.IMAGE_NAME }}](${{ env.DOCKER_IMAGE_NAME_URL }}) | - | + + ### OP-Reth + | System | Architecture | Binary | PGP Signature | |:---:|:---:|:---:|:---| - | | x86_64 | [reth-${{ env.VERSION }}-x86_64-unknown-linux-gnu.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-x86_64-unknown-linux-gnu.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-x86_64-unknown-linux-gnu.tar.gz.asc) | - | | aarch64 | [reth-${{ env.VERSION }}-aarch64-unknown-linux-gnu.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-aarch64-unknown-linux-gnu.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-aarch64-unknown-linux-gnu.tar.gz.asc) | - | | x86_64 | [reth-${{ env.VERSION }}-x86_64-pc-windows-gnu.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-x86_64-pc-windows-gnu.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-x86_64-pc-windows-gnu.tar.gz.asc) | - | | x86_64 | [reth-${{ env.VERSION }}-x86_64-apple-darwin.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-x86_64-apple-darwin.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-x86_64-apple-darwin.tar.gz.asc) | - | | aarch64 | [reth-${{ env.VERSION }}-aarch64-apple-darwin.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-aarch64-apple-darwin.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-aarch64-apple-darwin.tar.gz.asc) | - | | | | | - | **System** | **Option** | - | **Resource** | - | | Docker | | [${{ env.IMAGE_NAME }}](https://github.com/paradigmxyz/reth/pkgs/container/reth) | - | | Docker (Reproducible) | | [${{ env.IMAGE_NAME }}-reproducible](https://github.com/paradigmxyz/reth/pkgs/container/reth-reproducible) | + | | x86_64 | [op-reth-${{ env.VERSION }}-x86_64-unknown-linux-gnu.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/op-reth-${{ env.VERSION }}-x86_64-unknown-linux-gnu.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/op-reth-${{ env.VERSION }}-x86_64-unknown-linux-gnu.tar.gz.asc) | + | | aarch64 | [op-reth-${{ env.VERSION }}-aarch64-unknown-linux-gnu.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/op-reth-${{ env.VERSION }}-aarch64-unknown-linux-gnu.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/op-reth-${{ env.VERSION }}-aarch64-unknown-linux-gnu.tar.gz.asc) | + | | x86_64 | [op-reth-${{ env.VERSION }}-x86_64-pc-windows-gnu.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/op-reth-${{ env.VERSION }}-x86_64-pc-windows-gnu.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/op-reth-${{ env.VERSION }}-x86_64-pc-windows-gnu.tar.gz.asc) | + | | x86_64 | [op-reth-${{ env.VERSION }}-x86_64-apple-darwin.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/op-reth-${{ env.VERSION }}-x86_64-apple-darwin.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/op-reth-${{ env.VERSION }}-x86_64-apple-darwin.tar.gz.asc) | + | | aarch64 | [op-reth-${{ env.VERSION }}-aarch64-apple-darwin.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/op-reth-${{ env.VERSION }}-aarch64-apple-darwin.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/op-reth-${{ env.VERSION }}-aarch64-apple-darwin.tar.gz.asc) | + | | Docker | [${{ env.OP_IMAGE_NAME }}](${{ env.DOCKER_OP_IMAGE_NAME_URL }}) | - | ENDBODY ) assets=() @@ -235,4 +263,26 @@ jobs: assets+=("$asset/$asset") done tag_name="${{ env.VERSION }}" - echo "$body" | gh release create --draft -t "Reth $tag_name" -F "-" "$tag_name" "${assets[@]}" + echo "$body" | gh release create --draft $prerelease_flag -t "Reth $tag_name" -F "-" "$tag_name" "${assets[@]}" + + dry-run-summary: + name: dry run summary + runs-on: ubuntu-latest + needs: [build, extract-version] + if: ${{ github.event.inputs.dry_run == 'true' }} + env: + VERSION: ${{ needs.extract-version.outputs.VERSION }} + steps: + - name: Summarize dry run + run: | + echo "## 🧪 Release Dry Run Summary" + echo "" + echo "✅ Successfully completed dry run for commit ${{ github.sha }}" + echo "" + echo "### What would happen in a real release:" + echo "- Binary artifacts would be uploaded to GitHub" + echo "- Docker images would be pushed to registry" + echo "- A draft release would be created" + echo "" + echo "### Next Steps" + echo "To perform a real release, push a git tag." diff --git a/.github/workflows/reproducible-build.yml b/.github/workflows/reproducible-build.yml index c8e70513441..b4a93cedaba 100644 --- a/.github/workflows/reproducible-build.yml +++ b/.github/workflows/reproducible-build.yml @@ -10,7 +10,7 @@ jobs: name: build reproducible binaries runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: rui314/setup-mold@v1 - uses: dtolnay/rust-toolchain@stable with: diff --git a/.github/workflows/stage.yml b/.github/workflows/stage.yml index 5c326282762..7225d84cffa 100644 --- a/.github/workflows/stage.yml +++ b/.github/workflows/stage.yml @@ -29,7 +29,7 @@ jobs: RUST_BACKTRACE: 1 timeout-minutes: 60 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: rui314/setup-mold@v1 - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 38cca2fb1a9..297339f53e6 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -14,7 +14,7 @@ jobs: issues: write pull-requests: write steps: - - uses: actions/stale@v9 + - uses: actions/stale@v10 with: days-before-stale: 21 days-before-close: 7 diff --git a/.github/workflows/sync-era.yml b/.github/workflows/sync-era.yml new file mode 100644 index 00000000000..f2539b2fdc2 --- /dev/null +++ b/.github/workflows/sync-era.yml @@ -0,0 +1,67 @@ +# Runs sync tests with ERA stage enabled. + +name: sync-era test + +on: + workflow_dispatch: + schedule: + - cron: "0 */6 * * *" + +env: + CARGO_TERM_COLOR: always + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + sync: + name: sync (${{ matrix.chain.bin }}) + runs-on: + group: Reth + env: + RUST_LOG: info,sync=error + RUST_BACKTRACE: 1 + timeout-minutes: 60 + strategy: + matrix: + chain: + - build: install + bin: reth + chain: mainnet + tip: "0x91c90676cab257a59cd956d7cb0bceb9b1a71d79755c23c7277a0697ccfaf8c4" + block: 100000 + unwind-target: "0x52e0509d33a988ef807058e2980099ee3070187f7333aae12b64d4d675f34c5a" + - build: install-op + bin: op-reth + chain: base + tip: "0xbb9b85352c7ebca6ba8efc63bd66cecd038c92ec8ebd02e153a3e0b197e672b7" + block: 10000 + unwind-target: "0x118a6e922a8c6cab221fc5adfe5056d2b72d58c6580e9c5629de55299e2cf8de" + steps: + - uses: actions/checkout@v5 + - uses: rui314/setup-mold@v1 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + - name: Build ${{ matrix.chain.bin }} + run: make ${{ matrix.chain.build }} + - name: Run sync with ERA enabled + run: | + ${{ matrix.chain.bin }} node \ + --chain ${{ matrix.chain.chain }} \ + --debug.tip ${{ matrix.chain.tip }} \ + --debug.max-block ${{ matrix.chain.block }} \ + --debug.terminate \ + --era.enable + - name: Verify the target block hash + run: | + ${{ matrix.chain.bin }} db --chain ${{ matrix.chain.chain }} get static-file headers ${{ matrix.chain.block }} \ + | grep ${{ matrix.chain.tip }} + - name: Run stage unwind for 100 blocks + run: | + ${{ matrix.chain.bin }} stage unwind num-blocks 100 --chain ${{ matrix.chain.chain }} + - name: Run stage unwind to block hash + run: | + ${{ matrix.chain.bin }} stage unwind to-block ${{ matrix.chain.unwind-target }} --chain ${{ matrix.chain.chain }} diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 952cab361ab..e57082b83e7 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -3,7 +3,9 @@ name: sync test on: - merge_group: + workflow_dispatch: + schedule: + - cron: "0 */6 * * *" env: CARGO_TERM_COLOR: always @@ -37,7 +39,7 @@ jobs: block: 10000 unwind-target: "0x118a6e922a8c6cab221fc5adfe5056d2b72d58c6580e9c5629de55299e2cf8de" steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: rui314/setup-mold@v1 - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 diff --git a/.github/workflows/unit.yml b/.github/workflows/unit.yml index 767a3e5c0ad..d9aca93f21c 100644 --- a/.github/workflows/unit.yml +++ b/.github/workflows/unit.yml @@ -42,13 +42,9 @@ jobs: args: --features "asm-keccak" --locked --exclude reth --exclude reth-bench --exclude "example-*" --exclude "reth-ethereum-*" --exclude "*-ethereum" partition: 2 total_partitions: 2 - - type: book - args: --manifest-path book/sources/Cargo.toml - partition: 1 - total_partitions: 1 timeout-minutes: 30 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: rui314/setup-mold@v1 - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 @@ -65,7 +61,7 @@ jobs: ${{ matrix.args }} --workspace \ --exclude ef-tests --no-tests=warn \ --partition hash:${{ matrix.partition }}/2 \ - -E "!kind(test)" + -E "!kind(test) and not binary(e2e_testsuite)" state: name: Ethereum state tests @@ -76,15 +72,24 @@ jobs: RUST_BACKTRACE: 1 timeout-minutes: 30 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Checkout ethereum/tests - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: repository: ethereum/tests ref: 81862e4848585a438d64f911a19b3825f0f4cd95 path: testing/ef-tests/ethereum-tests submodules: recursive fetch-depth: 1 + - name: Download & extract EEST fixtures (public) + shell: bash + env: + EEST_TESTS_TAG: v4.5.0 + run: | + set -euo pipefail + mkdir -p testing/ef-tests/execution-spec-tests + URL="https://github.com/ethereum/execution-spec-tests/releases/download/${EEST_TESTS_TAG}/fixtures_stable.tar.gz" + curl -L "$URL" | tar -xz --strip-components=1 -C testing/ef-tests/execution-spec-tests - uses: rui314/setup-mold@v1 - uses: dtolnay/rust-toolchain@stable - uses: taiki-e/install-action@nextest @@ -101,7 +106,7 @@ jobs: RUST_BACKTRACE: 1 timeout-minutes: 30 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: rui314/setup-mold@v1 - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 diff --git a/.github/workflows/update-superchain.yml b/.github/workflows/update-superchain.yml new file mode 100644 index 00000000000..f682f35a17d --- /dev/null +++ b/.github/workflows/update-superchain.yml @@ -0,0 +1,36 @@ +name: Update Superchain Config + +on: + schedule: + - cron: '0 3 * * 0' + workflow_dispatch: + +permissions: + contents: write + +jobs: + update-superchain: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v5 + + - name: Install required tools + run: | + sudo apt-get update + sudo apt-get install -y jq zstd qpdf yq + + - name: Run fetch_superchain_config.sh + run: | + chmod +x crates/optimism/chainspec/res/fetch_superchain_config.sh + cd crates/optimism/chainspec/res + ./fetch_superchain_config.sh + + - name: Create Pull Request + uses: peter-evans/create-pull-request@v7 + with: + commit-message: "chore: update superchain config" + title: "chore: update superchain config" + body: "This PR updates the superchain configs via scheduled workflow." + branch: "ci/update-superchain-config" + delete-branch: true diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 20258cfa721..81181c2cb1a 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -15,7 +15,7 @@ jobs: timeout-minutes: 60 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: rui314/setup-mold@v1 - uses: dtolnay/rust-toolchain@stable with: @@ -34,7 +34,7 @@ jobs: timeout-minutes: 60 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: rui314/setup-mold@v1 - uses: dtolnay/rust-toolchain@stable with: diff --git a/.gitignore b/.gitignore index 1072d75dfaa..a9b9f4768d5 100644 --- a/.gitignore +++ b/.gitignore @@ -54,5 +54,21 @@ rustc-ice-* # Book sources should be able to build with the latest version book/sources/Cargo.lock +# vocs node_modules +docs/vocs/node_modules + # Cargo chef recipe file recipe.json + +_ +# broken links report +links-report.json + +# Python cache +__pycache__/ +*.py[cod] +*$py.class + +# direnv +.envrc +.direnv/ diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000000..c7a709c6713 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,393 @@ +# Reth Development Guide for AI Agents + +This guide provides comprehensive instructions for AI agents working on the Reth codebase. It covers the architecture, development workflows, and critical guidelines for effective contributions. + +## Project Overview + +Reth is a high-performance Ethereum execution client written in Rust, focusing on modularity, performance, and contributor-friendliness. The codebase is organized into well-defined crates with clear boundaries and responsibilities. + +## Architecture Overview + +### Core Components + +1. **Consensus (`crates/consensus/`)**: Validates blocks according to Ethereum consensus rules +2. **Storage (`crates/storage/`)**: Hybrid database using MDBX + static files for optimal performance +3. **Networking (`crates/net/`)**: P2P networking stack with discovery, sync, and transaction propagation +4. **RPC (`crates/rpc/`)**: JSON-RPC server supporting all standard Ethereum APIs +5. **Execution (`crates/evm/`, `crates/ethereum/`)**: Transaction execution and state transitions +6. **Pipeline (`crates/stages/`)**: Staged sync architecture for blockchain synchronization +7. **Trie (`crates/trie/`)**: Merkle Patricia Trie implementation with parallel state root computation +8. **Node Builder (`crates/node/`)**: High-level node orchestration and configuration +9 **The Consensus Engine (`crates/engine/`)**: Handles processing blocks received from the consensus layer with the Engine API (newPayload, forkchoiceUpdated) + +### Key Design Principles + +- **Modularity**: Each crate can be used as a standalone library +- **Performance**: Extensive use of parallelism, memory-mapped I/O, and optimized data structures +- **Extensibility**: Traits and generic types allow for different implementations (Ethereum, Optimism, etc.) +- **Type Safety**: Strong typing throughout with minimal use of dynamic dispatch + +## Development Workflow + +### Code Style and Standards + +1. **Formatting**: Always use nightly rustfmt + ```bash + cargo +nightly fmt --all + ``` + +2. **Linting**: Run clippy with all features + ```bash + RUSTFLAGS="-D warnings" cargo +nightly clippy --workspace --lib --examples --tests --benches --all-features --locked + ``` + +3. **Testing**: Use nextest for faster test execution + ```bash + cargo nextest run --workspace + ``` + +### Common Contribution Types + +Based on actual recent PRs, here are typical contribution patterns: + +#### 1. Small Bug Fixes (1-10 lines) +Real example: Fixing beacon block root handling ([#16767](https://github.com/paradigmxyz/reth/pull/16767)) +```rust +// Changed a single line to fix logic error +- parent_beacon_block_root: parent.parent_beacon_block_root(), ++ parent_beacon_block_root: parent.parent_beacon_block_root().map(|_| B256::ZERO), +``` + +#### 2. Integration with Upstream Changes +Real example: Integrating revm updates ([#16752](https://github.com/paradigmxyz/reth/pull/16752)) +```rust +// Update code to use new APIs from dependencies +- if self.fork_tracker.is_shanghai_activated() { +- if let Err(err) = transaction.ensure_max_init_code_size(MAX_INIT_CODE_BYTE_SIZE) { ++ if let Some(init_code_size_limit) = self.fork_tracker.max_initcode_size() { ++ if let Err(err) = transaction.ensure_max_init_code_size(init_code_size_limit) { +``` + +#### 3. Adding Comprehensive Tests +Real example: ETH69 protocol tests ([#16759](https://github.com/paradigmxyz/reth/pull/16759)) +```rust +#[tokio::test(flavor = "multi_thread")] +async fn test_eth69_peers_can_connect() { + // Create test network with specific protocol versions + let p0 = PeerConfig::with_protocols(NoopProvider::default(), Some(EthVersion::Eth69.into())); + // Test connection and version negotiation +} +``` + +#### 4. Making Components Generic +Real example: Making EthEvmConfig generic over chainspec ([#16758](https://github.com/paradigmxyz/reth/pull/16758)) +```rust +// Before: Hardcoded to ChainSpec +- pub struct EthEvmConfig { +- pub executor_factory: EthBlockExecutorFactory, EvmFactory>, + +// After: Generic over any chain spec type ++ pub struct EthEvmConfig ++ where ++ C: EthereumHardforks, ++ { ++ pub executor_factory: EthBlockExecutorFactory, EvmFactory>, +``` + +#### 5. Resource Management Improvements +Real example: ETL directory cleanup ([#16770](https://github.com/paradigmxyz/reth/pull/16770)) +```rust +// Add cleanup logic on startup ++ if let Err(err) = fs::remove_dir_all(&etl_path) { ++ warn!(target: "reth::cli", ?etl_path, %err, "Failed to remove ETL path on launch"); ++ } +``` + +#### 6. Feature Additions +Real example: Sharded mempool support ([#16756](https://github.com/paradigmxyz/reth/pull/16756)) +```rust +// Add new filtering policies for transaction announcements +pub struct ShardedMempoolAnnouncementFilter { + pub inner: T, + pub shard_bits: u8, + pub node_id: Option, +} +``` + +### Testing Guidelines + +1. **Unit Tests**: Test individual functions and components +2. **Integration Tests**: Test interactions between components +3. **Benchmarks**: For performance-critical code +4. **Fuzz Tests**: For parsing and serialization code +5. **Property Tests**: For checking component correctness on a wide variety of inputs + +Example test structure: +```rust +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_component_behavior() { + // Arrange + let component = Component::new(); + + // Act + let result = component.operation(); + + // Assert + assert_eq!(result, expected); + } +} +``` + +### Performance Considerations + +1. **Avoid Allocations in Hot Paths**: Use references and borrowing +2. **Parallel Processing**: Use rayon for CPU-bound parallel work +3. **Async/Await**: Use tokio for I/O-bound operations +4. **File Operations**: Use `reth_fs_util` instead of `std::fs` for better error handling + +### Common Pitfalls + +1. **Don't Block Async Tasks**: Use `spawn_blocking` for CPU-intensive work or work with lots of blocking I/O +2. **Handle Errors Properly**: Use `?` operator and proper error types + +### What to Avoid + +Based on PR patterns, avoid: + +1. **Large, sweeping changes**: Keep PRs focused and reviewable +2. **Mixing unrelated changes**: One logical change per PR +3. **Ignoring CI failures**: All checks must pass +4. **Incomplete implementations**: Finish features before submitting +5. **Modifying libmdbx sources**: Never modify files in `crates/storage/libmdbx-rs/mdbx-sys/libmdbx/` - this is vendored third-party code + +### CI Requirements + +Before submitting changes, ensure: + +1. **Format Check**: `cargo +nightly fmt --all --check` +2. **Clippy**: No warnings with `RUSTFLAGS="-D warnings"` +3. **Tests Pass**: All unit and integration tests +4. **Documentation**: Update relevant docs and add doc comments with `cargo docs --document-private-items` +5. **Commit Messages**: Follow conventional format (feat:, fix:, chore:, etc.) + + +### Opening PRs against + +Label PRs appropriately, first check the available labels and then apply the relevant ones: +* when changes are RPC related, add A-rpc label +* when changes are docs related, add C-docs label +* when changes are optimism related (e.g. new feature or exclusive changes to crates/optimism), add A-op-reth label +* ... and so on, check the available labels for more options. +* if being tasked to open a pr, ensure that all changes are properly formatted: `cargo +nightly fmt --all` + +If changes in reth include changes to dependencies, run commands `zepter` and `make lint-toml` before finalizing the pr. Assume `zepter` binary is installed. + +### Debugging Tips + +1. **Logging**: Use `tracing` crate with appropriate levels + ```rust + tracing::debug!(target: "reth::component", ?value, "description"); + ``` + +2. **Metrics**: Add metrics for monitoring + ```rust + metrics::counter!("reth_component_operations").increment(1); + ``` + +3. **Test Isolation**: Use separate test databases/directories + +### Finding Where to Contribute + +1. **Check Issues**: Look for issues labeled `good-first-issue` or `help-wanted` +2. **Review TODOs**: Search for `TODO` comments in the codebase +3. **Improve Tests**: Areas with low test coverage are good targets +4. **Documentation**: Improve code comments and documentation +5. **Performance**: Profile and optimize hot paths (with benchmarks) + +### Common PR Patterns + +#### Small, Focused Changes +Most PRs change only 1-5 files. Examples: +- Single-line bug fixes +- Adding a missing trait implementation +- Updating error messages +- Adding test cases for edge conditions + +#### Integration Work +When dependencies update (especially revm), code needs updating: +- Check for breaking API changes +- Update to use new features (like EIP implementations) +- Ensure compatibility with new versions + +#### Test Improvements +Tests often need expansion for: +- New protocol versions (ETH68, ETH69) +- Edge cases in state transitions +- Network behavior under specific conditions +- Concurrent operations + +#### Making Code More Generic +Common refactoring pattern: +- Replace concrete types with generics +- Add trait bounds for flexibility +- Enable reuse across different chain types (Ethereum, Optimism) + +#### When to Comment + +Write comments that remain valuable after the PR is merged. Future readers won't have PR context - they only see the current code. + +##### ✅ DO: Add Value + +**Explain WHY and non-obvious behavior:** +```rust +// Process must handle allocations atomically to prevent race conditions +// between dealloc on drop and concurrent limit checks +unsafe impl GlobalAlloc for LimitedAllocator { ... } + +// Binary search requires sorted input. Panics on unsorted slices. +fn find_index(items: &[Item], target: &Item) -> Option + +// Timeout set to 5s to match EVM block processing limits +const TRACER_TIMEOUT: Duration = Duration::from_secs(5); +``` + +**Document constraints and assumptions:** +```rust +/// Returns heap size estimate. +/// +/// Note: May undercount shared references (Rc/Arc). For precise +/// accounting, combine with an allocator-based approach. +fn deep_size_of(&self) -> usize +``` + +**Explain complex logic:** +```rust +// We reset limits at task start because tokio reuses threads in +// spawn_blocking pool. Without reset, second task inherits first +// task's allocation count and immediately hits limit. +THREAD_ALLOCATED.with(|allocated| allocated.set(0)); +``` + +##### ❌ DON'T: Describe Changes +```rust +// ❌ BAD - Describes the change, not the code +// Changed from Vec to HashMap for O(1) lookups + +// ✅ GOOD - Explains the decision +// HashMap provides O(1) symbol lookups during trace replay +``` +```rust +// ❌ BAD - PR-specific context +// Fix for issue #234 where memory wasn't freed + +// ✅ GOOD - Documents the actual behavior +// Explicitly drop allocations before limit check to ensure +// accurate accounting +``` +```rust +// ❌ BAD - States the obvious +// Increment counter +counter += 1; + +// ✅ GOOD - Explains non-obvious purpose +// Track allocations across all threads for global limit enforcement +GLOBAL_COUNTER.fetch_add(1, Ordering::SeqCst); +``` + +✅ **Comment when:** +- Non-obvious behavior or edge cases +- Performance trade-offs +- Safety requirements (unsafe blocks must always be documented) +- Limitations or gotchas +- Why simpler alternatives don't work + +❌ **Don't comment when:** +- Code is self-explanatory +- Just restating the code in English +- Describing what changed in this PR + +##### The Test: "Will this make sense in 6 months?" + +Before adding a comment, ask: Would someone reading just the current code (no PR, no history) find this helpful? + + +### Example Contribution Workflow + +Let's say you want to fix a bug where external IP resolution fails on startup: + +1. **Create a branch**: + ```bash + git checkout -b fix-external-ip-resolution + ``` + +2. **Find the relevant code**: + ```bash + # Search for IP resolution code + rg "external.*ip" --type rust + ``` + +3. **Reason about the problem, when the problem is identified, make the fix**: + ```rust + // In crates/net/discv4/src/lib.rs + pub fn resolve_external_ip() -> Option { + // Add fallback mechanism + nat::external_ip() + .or_else(|| nat::external_ip_from_stun()) + .or_else(|| Some(DEFAULT_IP)) + } + ``` + +4. **Add a test**: + ```rust + #[test] + fn test_external_ip_fallback() { + // Test that resolution has proper fallbacks + } + ``` + +5. **Run checks**: + ```bash + cargo +nightly fmt --all + cargo clippy --all-features + cargo test -p reth-discv4 + ``` + +6. **Commit with clear message**: + ```bash + git commit -m "fix: add fallback for external IP resolution + + Previously, node startup could fail if external IP resolution + failed. This adds fallback mechanisms to ensure the node can + always start with a reasonable default." + ``` + +## Quick Reference + +### Essential Commands + +```bash +# Format code +cargo +nightly fmt --all + +# Run lints +RUSTFLAGS="-D warnings" cargo +nightly clippy --workspace --all-features --locked + +# Run tests +cargo nextest run --workspace + +# Run specific benchmark +cargo bench --bench bench_name + +# Build optimized binary +cargo build --release --features "jemalloc asm-keccak" + +# Check compilation for all features +cargo check --workspace --all-features + +# Check documentation +cargo docs --document-private-items +``` diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 53f5c9075bc..8c51b03d19a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -3,7 +3,7 @@ Thanks for your interest in improving Reth! There are multiple opportunities to contribute at any level. It doesn't matter if you are just getting started with Rust -or are the most weathered expert, we can use your help. +or if you are already the most weathered expert, we can use your help. **No contribution is too small and all contributions are valued.** @@ -55,7 +55,7 @@ If you have reviewed existing documentation and still have questions, or you are *opening a discussion**. This repository comes with a discussions board where you can also ask for help. Click the " Discussions" tab at the top. -As Reth is still in heavy development, the documentation can be a bit scattered. The [Reth Book][reth-book] is our +As Reth is still in heavy development, the documentation can be a bit scattered. The [Reth Docs][reth-docs] is our current best-effort attempt at keeping up-to-date information. ### Submitting a bug report @@ -235,7 +235,7 @@ _Adapted from the [Foundry contributing guide][foundry-contributing]_. [dev-tg]: https://t.me/paradigm_reth -[reth-book]: https://github.com/paradigmxyz/reth/tree/main/book +[reth-docs]: https://github.com/paradigmxyz/reth/tree/main/docs [mcve]: https://stackoverflow.com/help/mcve diff --git a/Cargo.lock b/Cargo.lock index 021462e1781..1ac1afefcdc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,15 +2,6 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "addr2line" -version = "0.24.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" -dependencies = [ - "gimli", -] - [[package]] name = "adler2" version = "2.0.1" @@ -59,7 +50,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", - "getrandom 0.3.3", + "getrandom 0.3.4", "once_cell", "version_check", "zerocopy", @@ -67,13 +58,22 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" dependencies = [ "memchr", ] +[[package]] +name = "aligned-vec" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc890384c8602f339876ded803c97ad529f3842aba97f6392b3dba0dd171769b" +dependencies = [ + "equator", +] + [[package]] name = "alloc-no-stdlib" version = "2.0.4" @@ -97,9 +97,9 @@ checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] name = "alloy-chains" -version = "0.2.9" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef8ff73a143281cb77c32006b04af9c047a6b8fe5860e85a88ad325328965355" +checksum = "6068f356948cd84b5ad9ac30c50478e433847f14a50714d2b68f15d052724049" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -112,15 +112,16 @@ dependencies = [ [[package]] name = "alloy-consensus" -version = "0.15.11" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32c3f3bc4f2a6b725970cd354e78e9738ea1e8961a91898f57bf6317970b1915" +checksum = "90d103d3e440ad6f703dd71a5b58a6abd24834563bde8a5fabe706e00242f810" dependencies = [ - "alloy-eips 0.15.11", + "alloy-eips", "alloy-primitives", "alloy-rlp", - "alloy-serde 0.15.11", - "alloy-trie 0.8.1", + "alloy-serde", + "alloy-trie", + "alloy-tx-macros", "arbitrary", "auto_impl", "c-kzg", @@ -129,91 +130,55 @@ dependencies = [ "k256", "once_cell", "rand 0.8.5", - "secp256k1", + "secp256k1 0.30.0", "serde", + "serde_json", "serde_with", - "thiserror 2.0.16", -] - -[[package]] -name = "alloy-consensus" -version = "1.0.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d213580c17d239ae83c0d897ac3315db7cda83d2d4936a9823cc3517552f2e24" -dependencies = [ - "alloy-eips 1.0.30", - "alloy-primitives", - "alloy-rlp", - "alloy-serde 1.0.30", - "alloy-trie 0.9.1", - "alloy-tx-macros", - "auto_impl", - "c-kzg", - "derive_more", - "either", - "k256", - "once_cell", - "rand 0.8.5", - "secp256k1", - "serde", - "thiserror 2.0.16", + "thiserror 2.0.17", ] [[package]] name = "alloy-consensus-any" -version = "0.15.11" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dda014fb5591b8d8d24cab30f52690117d238e52254c6fb40658e91ea2ccd6c3" +checksum = "48ead76c8c84ab3a50c31c56bc2c748c2d64357ad2131c32f9b10ab790a25e1a" dependencies = [ - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", + "alloy-consensus", + "alloy-eips", "alloy-primitives", "alloy-rlp", - "alloy-serde 0.15.11", + "alloy-serde", "arbitrary", "serde", ] -[[package]] -name = "alloy-consensus-any" -version = "1.0.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81443e3b8dccfeac7cd511aced15928c97ff253f4177acbb97de97178e543f6c" -dependencies = [ - "alloy-consensus 1.0.30", - "alloy-eips 1.0.30", - "alloy-primitives", - "alloy-rlp", - "alloy-serde 1.0.30", - "serde", -] - [[package]] name = "alloy-contract" -version = "0.15.11" +version = "1.0.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9668ce1176f0b87a5e5fc805b3d198954f495de2e99b70a44bed691ba2b0a9d8" +checksum = "c98d21aeef3e0783046c207abd3eb6cb41f6e77e0c0fc8077ebecd6df4f9d171" dependencies = [ - "alloy-consensus 0.15.11", + "alloy-consensus", "alloy-dyn-abi", "alloy-json-abi", "alloy-network", - "alloy-network-primitives 0.15.11", + "alloy-network-primitives", "alloy-primitives", "alloy-provider", - "alloy-rpc-types-eth 0.15.11", + "alloy-rpc-types-eth", "alloy-sol-types", "alloy-transport", "futures", "futures-util", - "thiserror 2.0.16", + "serde_json", + "thiserror 2.0.17", ] [[package]] name = "alloy-dyn-abi" -version = "1.3.1" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3f56873f3cac7a2c63d8e98a4314b8311aa96adb1a0f82ae923eb2119809d2c" +checksum = "3fdff496dd4e98a81f4861e66f7eaf5f2488971848bb42d9c892f871730245c8" dependencies = [ "alloy-json-abi", "alloy-primitives", @@ -238,7 +203,7 @@ dependencies = [ "crc", "rand 0.8.5", "serde", - "thiserror 2.0.16", + "thiserror 2.0.17", ] [[package]] @@ -267,41 +232,21 @@ dependencies = [ "rand 0.8.5", "serde", "serde_with", - "thiserror 2.0.16", -] - -[[package]] -name = "alloy-eips" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "609515c1955b33af3d78d26357540f68c5551a90ef58fd53def04f2aa074ec43" -dependencies = [ - "alloy-eip2124", - "alloy-eip2930", - "alloy-eip7702", - "alloy-primitives", - "alloy-rlp", - "alloy-serde 0.14.0", - "auto_impl", - "c-kzg", - "derive_more", - "either", - "serde", - "sha2 0.10.9", + "thiserror 2.0.17", ] [[package]] name = "alloy-eips" -version = "0.15.11" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f7b2f7010581f29bcace81776cf2f0e022008d05a7d326884763f16f3044620" +checksum = "7bdbec74583d0067798d77afa43d58f00d93035335d7ceaa5d3f93857d461bb9" dependencies = [ "alloy-eip2124", "alloy-eip2930", "alloy-eip7702", "alloy-primitives", "alloy-rlp", - "alloy-serde 0.15.11", + "alloy-serde", "arbitrary", "auto_impl", "c-kzg", @@ -310,67 +255,52 @@ dependencies = [ "ethereum_ssz", "ethereum_ssz_derive", "serde", - "sha2 0.10.9", -] - -[[package]] -name = "alloy-eips" -version = "1.0.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a15b4b0f6bab47aae017d52bb5a739bda381553c09fb9918b7172721ef5f5de" -dependencies = [ - "alloy-eip2124", - "alloy-eip2930", - "alloy-eip7702", - "alloy-primitives", - "alloy-rlp", - "alloy-serde 1.0.30", - "auto_impl", - "c-kzg", - "derive_more", - "either", - "serde", "serde_with", - "sha2 0.10.9", - "thiserror 2.0.16", + "sha2", + "thiserror 2.0.17", ] [[package]] name = "alloy-evm" -version = "0.7.0" -source = "git+https://github.com/mantle-xyz/evm?tag=v2.0.1#d8c600f278d14f271fd5df120708737eb2ef2483" +version = "0.23.3" +source = "git+https://github.com/mantle-xyz/evm?tag=v2.1.2#5c8cd52229f0c82c653c1a8e496169e8ff0e2e30" dependencies = [ - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", + "alloy-consensus", + "alloy-eips", "alloy-hardforks", + "alloy-op-hardforks", "alloy-primitives", + "alloy-rpc-types-engine", + "alloy-rpc-types-eth", "alloy-sol-types", "auto_impl", "derive_more", "op-alloy-consensus", + "op-alloy-rpc-types-engine", "op-revm", "revm", - "thiserror 2.0.16", + "thiserror 2.0.17", ] [[package]] name = "alloy-genesis" -version = "0.15.11" +version = "1.0.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7f723856b1c4ad5473f065650ab9be557c96fbc77e89180fbdac003e904a8d6" +checksum = "675b163946b343ed2ddde4416114ad61fabc8b2a50d08423f38aa0ac2319e800" dependencies = [ - "alloy-eips 0.15.11", + "alloy-eips", "alloy-primitives", - "alloy-serde 0.15.11", - "alloy-trie 0.8.1", + "alloy-serde", + "alloy-trie", "serde", + "serde_with", ] [[package]] name = "alloy-hardforks" -version = "0.2.13" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3165210652f71dfc094b051602bafd691f506c54050a174b1cba18fb5ef706a3" +checksum = "1e29d7eacf42f89c21d7f089916d0bdb4f36139a31698790e8837d2dbbd4b2c3" dependencies = [ "alloy-chains", "alloy-eip2124", @@ -382,9 +312,9 @@ dependencies = [ [[package]] name = "alloy-json-abi" -version = "1.3.1" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "125a1c373261b252e53e04d6e92c37d881833afc1315fceab53fd46045695640" +checksum = "5513d5e6bd1cba6bdcf5373470f559f320c05c8c59493b6e98912fbe6733943f" dependencies = [ "alloy-primitives", "alloy-sol-type-parser", @@ -394,33 +324,34 @@ dependencies = [ [[package]] name = "alloy-json-rpc" -version = "0.15.11" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca1e31b50f4ed9a83689ae97263d366b15b935a67c4acb5dd46d5b1c3b27e8e6" +checksum = "003f46c54f22854a32b9cc7972660a476968008ad505427eabab49225309ec40" dependencies = [ "alloy-primitives", "alloy-sol-types", + "http", "serde", "serde_json", - "thiserror 2.0.16", + "thiserror 2.0.17", "tracing", ] [[package]] name = "alloy-network" -version = "0.15.11" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "879afc0f4a528908c8fe6935b2ab0bc07f77221a989186f71583f7592831689e" +checksum = "612296e6b723470bb1101420a73c63dfd535aa9bf738ce09951aedbd4ab7292e" dependencies = [ - "alloy-consensus 0.15.11", - "alloy-consensus-any 0.15.11", - "alloy-eips 0.15.11", + "alloy-consensus", + "alloy-consensus-any", + "alloy-eips", "alloy-json-rpc", - "alloy-network-primitives 0.15.11", + "alloy-network-primitives", "alloy-primitives", "alloy-rpc-types-any", - "alloy-rpc-types-eth 0.15.11", - "alloy-serde 0.15.11", + "alloy-rpc-types-eth", + "alloy-serde", "alloy-signer", "alloy-sol-types", "async-trait", @@ -429,42 +360,29 @@ dependencies = [ "futures-utils-wasm", "serde", "serde_json", - "thiserror 2.0.16", -] - -[[package]] -name = "alloy-network-primitives" -version = "0.15.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec185bac9d32df79c1132558a450d48f6db0bfb5adef417dbb1a0258153f879b" -dependencies = [ - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", - "alloy-primitives", - "alloy-serde 0.15.11", - "serde", + "thiserror 2.0.17", ] [[package]] name = "alloy-network-primitives" -version = "1.0.30" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b14fa9ba5774e0b30ae6a04176d998211d516c8af69c9c530af7c6c42a8c508" +checksum = "a0e7918396eecd69d9c907046ec8a93fb09b89e2f325d5e7ea9c4e3929aa0dd2" dependencies = [ - "alloy-consensus 1.0.30", - "alloy-eips 1.0.30", + "alloy-consensus", + "alloy-eips", "alloy-primitives", - "alloy-serde 1.0.30", + "alloy-serde", "serde", ] [[package]] name = "alloy-op-evm" -version = "0.7.0" -source = "git+https://github.com/mantle-xyz/evm?tag=v2.0.1#d8c600f278d14f271fd5df120708737eb2ef2483" +version = "0.23.3" +source = "git+https://github.com/mantle-xyz/evm?tag=v2.1.2#5c8cd52229f0c82c653c1a8e496169e8ff0e2e30" dependencies = [ - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", + "alloy-consensus", + "alloy-eips", "alloy-evm", "alloy-op-hardforks", "alloy-primitives", @@ -472,25 +390,27 @@ dependencies = [ "op-alloy-consensus", "op-revm", "revm", + "thiserror 2.0.17", ] [[package]] name = "alloy-op-hardforks" -version = "0.2.13" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3417f4187eaf7f7fb0d7556f0197bca26f0b23c4bb3aca0c9d566dc1c5d727a2" +checksum = "95ac97adaba4c26e17192d81f49186ac20c1e844e35a00e169c8d3d58bc84e6b" dependencies = [ "alloy-chains", "alloy-hardforks", + "alloy-primitives", "auto_impl", "serde", ] [[package]] name = "alloy-primitives" -version = "1.3.1" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc9485c56de23438127a731a6b4c87803d49faf1a7068dcd1d8768aca3a9edb9" +checksum = "355bf68a433e0fd7f7d33d5a9fc2583fde70bf5c530f63b80845f8da5505cf28" dependencies = [ "alloy-rlp", "arbitrary", @@ -498,19 +418,19 @@ dependencies = [ "cfg-if", "const-hex", "derive_more", - "foldhash", - "getrandom 0.3.3", - "hashbrown 0.15.5", - "indexmap 2.11.1", + "foldhash 0.2.0", + "getrandom 0.3.4", + "hashbrown 0.16.0", + "indexmap 2.12.0", "itoa", "k256", "keccak-asm", "paste", "proptest", - "proptest-derive", + "proptest-derive 0.6.0", "rand 0.9.2", "ruint", - "rustc-hash 2.1.1", + "rustc-hash", "serde", "sha3", "tiny-keccak", @@ -518,21 +438,23 @@ dependencies = [ [[package]] name = "alloy-provider" -version = "0.15.11" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2d918534afe9cc050eabd8309c107dafd161aa77357782eca4f218bef08a660" +checksum = "55c1313a527a2e464d067c031f3c2ec073754ef615cc0eabca702fd0fe35729c" dependencies = [ "alloy-chains", - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", + "alloy-consensus", + "alloy-eips", "alloy-json-rpc", "alloy-network", - "alloy-network-primitives 0.15.11", + "alloy-network-primitives", "alloy-primitives", "alloy-pubsub", "alloy-rpc-client", + "alloy-rpc-types-debug", "alloy-rpc-types-engine", - "alloy-rpc-types-eth 0.15.11", + "alloy-rpc-types-eth", + "alloy-rpc-types-trace", "alloy-signer", "alloy-sol-types", "alloy-transport", @@ -552,7 +474,7 @@ dependencies = [ "reqwest", "serde", "serde_json", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "tracing", "url", @@ -561,13 +483,14 @@ dependencies = [ [[package]] name = "alloy-pubsub" -version = "0.15.11" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d77a92001c6267261a5e493d33db794b2206ebebf7c2dfbbe3825ebaec6a96d" +checksum = "f77d20cdbb68a614c7a86b3ffef607b37d087bb47a03c58f4c3f8f99bc3ace3b" dependencies = [ "alloy-json-rpc", "alloy-primitives", "alloy-transport", + "auto_impl", "bimap", "futures", "parking_lot", @@ -575,7 +498,7 @@ dependencies = [ "serde_json", "tokio", "tokio-stream", - "tower 0.5.2", + "tower", "tracing", "wasmtimer", ] @@ -599,14 +522,14 @@ checksum = "64b728d511962dda67c1bc7ea7c03736ec275ed2cf4c35d9585298ac9ccf3b73" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] name = "alloy-rpc-client" -version = "0.15.11" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a15e30dcada47c04820b64f63de2423506c5c74f9ab59b115277ef5ad595a6fc" +checksum = "31c89883fe6b7381744cbe80fef638ac488ead4f1956a4278956a1362c71cd2e" dependencies = [ "alloy-json-rpc", "alloy-primitives", @@ -615,7 +538,6 @@ dependencies = [ "alloy-transport-http", "alloy-transport-ipc", "alloy-transport-ws", - "async-stream", "futures", "pin-project", "reqwest", @@ -623,31 +545,30 @@ dependencies = [ "serde_json", "tokio", "tokio-stream", - "tower 0.5.2", + "tower", "tracing", - "tracing-futures", "url", "wasmtimer", ] [[package]] name = "alloy-rpc-types" -version = "0.15.11" +version = "1.0.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4aa10e26554ad7f79a539a6a8851573aedec5289f1f03244aad0bdbc324bfe5c" +checksum = "fe106e50522980bc9e7cc9016f445531edf1a53e0fdba904c833b98c6fdff3f0" dependencies = [ "alloy-primitives", "alloy-rpc-types-engine", - "alloy-rpc-types-eth 0.15.11", - "alloy-serde 0.15.11", + "alloy-rpc-types-eth", + "alloy-serde", "serde", ] [[package]] name = "alloy-rpc-types-admin" -version = "0.15.11" +version = "1.0.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35d28eaf48f002c822c02e3376254fd7f56228577e1c294c271c88299eff85e5" +checksum = "e8b67bf1ed8cac6fde7dd017ca0a1c33be846e613a265956089f983af1354f13" dependencies = [ "alloy-genesis", "alloy-primitives", @@ -657,71 +578,74 @@ dependencies = [ [[package]] name = "alloy-rpc-types-anvil" -version = "0.15.11" +version = "1.0.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6cd4346521aa1e2e76963bbf0c1d311223f6eb565269359a6f9232c9044d1f7" +checksum = "c1cf94d581b3aa13ebacb90ea52e0179985b7c20d8a522319e7d40768d56667a" dependencies = [ "alloy-primitives", - "alloy-rpc-types-eth 0.15.11", - "alloy-serde 0.15.11", + "alloy-rpc-types-eth", + "alloy-serde", "serde", ] [[package]] name = "alloy-rpc-types-any" -version = "0.15.11" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5a8f1efd77116915dad61092f9ef9295accd0b0b251062390d9c4e81599344" +checksum = "cdbf6d1766ca41e90ac21c4bc5cbc5e9e965978a25873c3f90b3992d905db4cb" dependencies = [ - "alloy-consensus-any 0.15.11", - "alloy-rpc-types-eth 0.15.11", - "alloy-serde 0.15.11", + "alloy-consensus-any", + "alloy-rpc-types-eth", + "alloy-serde", ] [[package]] name = "alloy-rpc-types-beacon" -version = "0.15.11" +version = "1.0.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df7b4a021b4daff504208b3b43a25270d0a5a27490d6535655052f32c419ba0a" +checksum = "440655ffd9ff8724fa76a07c7dbe18cb4353617215c23e3921163516b6c07ff8" dependencies = [ - "alloy-eips 0.15.11", + "alloy-eips", "alloy-primitives", "alloy-rpc-types-engine", + "derive_more", "ethereum_ssz", "ethereum_ssz_derive", "serde", + "serde_json", "serde_with", - "thiserror 2.0.16", + "thiserror 2.0.17", "tree_hash", "tree_hash_derive", ] [[package]] name = "alloy-rpc-types-debug" -version = "0.15.11" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07c142af843da7422e5e979726dcfeb78064949fdc6cb651457cc95034806a52" +checksum = "1b2ca3a434a6d49910a7e8e51797eb25db42ef8a5578c52d877fcb26d0afe7bc" dependencies = [ "alloy-primitives", + "derive_more", "serde", + "serde_with", ] [[package]] name = "alloy-rpc-types-engine" -version = "0.15.11" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "246c225f878dbcb8ad405949a30c403b3bb43bdf974f7f0331b276f835790a5e" +checksum = "07da696cc7fbfead4b1dda8afe408685cae80975cbb024f843ba74d9639cd0d3" dependencies = [ - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", + "alloy-consensus", + "alloy-eips", "alloy-primitives", "alloy-rlp", - "alloy-serde 0.15.11", + "alloy-serde", "arbitrary", "derive_more", "ethereum_ssz", "ethereum_ssz_derive", - "jsonrpsee-types", "jsonwebtoken", "rand 0.8.5", "serde", @@ -730,119 +654,72 @@ dependencies = [ [[package]] name = "alloy-rpc-types-eth" -version = "0.15.11" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc1323310d87f9d950fb3ff58d943fdf832f5e10e6f902f405c0eaa954ffbaf1" +checksum = "a15e4831b71eea9d20126a411c1c09facf1d01d5cac84fd51d532d3c429cfc26" dependencies = [ - "alloy-consensus 0.15.11", - "alloy-consensus-any 0.15.11", - "alloy-eips 0.15.11", - "alloy-network-primitives 0.15.11", + "alloy-consensus", + "alloy-consensus-any", + "alloy-eips", + "alloy-network-primitives", "alloy-primitives", "alloy-rlp", - "alloy-serde 0.15.11", + "alloy-serde", "alloy-sol-types", "arbitrary", "itertools 0.14.0", - "jsonrpsee-types", - "serde", - "serde_json", - "thiserror 2.0.16", -] - -[[package]] -name = "alloy-rpc-types-eth" -version = "1.0.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd1e1b4dcdf13eaa96343e5c0dafc2d2e8ce5d20b90347169d46a1df0dec210" -dependencies = [ - "alloy-consensus 1.0.30", - "alloy-consensus-any 1.0.30", - "alloy-eips 1.0.30", - "alloy-network-primitives 1.0.30", - "alloy-primitives", - "alloy-rlp", - "alloy-serde 1.0.30", - "alloy-sol-types", - "itertools 0.14.0", "serde", "serde_json", "serde_with", - "thiserror 2.0.16", -] - -[[package]] -name = "alloy-rpc-types-mev" -version = "0.15.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e25683e41684a568317b10d6ba00f60ef088460cc96ddad270478397334cd6f3" -dependencies = [ - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", - "alloy-primitives", - "alloy-rpc-types-eth 0.15.11", - "alloy-serde 0.15.11", - "serde", - "serde_json", + "thiserror 2.0.17", ] [[package]] name = "alloy-rpc-types-mev" -version = "1.0.30" +version = "1.0.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01620baa48d3f49fc908c781eb91ded71f3226e719bb6404697c2851cac4e098" +checksum = "6c89422163337ff64d9aaa13f3e4df53d60d789004044cd64ebc7dc4d5765a64" dependencies = [ - "alloy-consensus 1.0.30", - "alloy-eips 1.0.30", + "alloy-consensus", + "alloy-eips", "alloy-primitives", - "alloy-rpc-types-eth 1.0.30", - "alloy-serde 1.0.30", + "alloy-rpc-types-eth", + "alloy-serde", "serde", "serde_json", ] [[package]] name = "alloy-rpc-types-trace" -version = "0.15.11" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d57f1b7da0af516ad2c02233ed1274527f9b0536dbda300acea2e8a1e7ac20c8" +checksum = "fb0c800e2ce80829fca1491b3f9063c29092850dc6cf19249d5f678f0ce71bb0" dependencies = [ "alloy-primitives", - "alloy-rpc-types-eth 0.15.11", - "alloy-serde 0.15.11", + "alloy-rpc-types-eth", + "alloy-serde", "serde", "serde_json", - "thiserror 2.0.16", + "thiserror 2.0.17", ] [[package]] name = "alloy-rpc-types-txpool" -version = "0.15.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d4f4239235c4afdd8fa930a757ed81816799ddcc93d4e33cd0dae3b44f83f3e" -dependencies = [ - "alloy-primitives", - "alloy-rpc-types-eth 0.15.11", - "alloy-serde 0.15.11", - "serde", -] - -[[package]] -name = "alloy-serde" -version = "0.14.0" +version = "1.0.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4dba6ff08916bc0a9cbba121ce21f67c0b554c39cf174bc7b9df6c651bd3c3b" +checksum = "4c208cbe2ea28368c3f61bd1e27b14238b7b03796e90370de3c0d8722e0f9830" dependencies = [ "alloy-primitives", + "alloy-rpc-types-eth", + "alloy-serde", "serde", - "serde_json", ] [[package]] name = "alloy-serde" -version = "0.15.11" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d05ace2ef3da874544c3ffacfd73261cdb1405d8631765deb991436a53ec6069" +checksum = "a6f180c399ca7c1e2fe17ea58343910cad0090878a696ff5a50241aee12fc529" dependencies = [ "alloy-primitives", "arbitrary", @@ -850,22 +727,11 @@ dependencies = [ "serde_json", ] -[[package]] -name = "alloy-serde" -version = "1.0.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1b3b1078b8775077525bc9fe9f6577e815ceaecd6c412a4f3b4d8aa2836e8f6" -dependencies = [ - "alloy-primitives", - "serde", - "serde_json", -] - [[package]] name = "alloy-signer" -version = "0.15.11" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67fdabad99ad3c71384867374c60bcd311fc1bb90ea87f5f9c779fd8c7ec36aa" +checksum = "ecc39ad2c0a3d2da8891f4081565780703a593f090f768f884049aa3aa929cbc" dependencies = [ "alloy-primitives", "async-trait", @@ -873,16 +739,16 @@ dependencies = [ "either", "elliptic-curve", "k256", - "thiserror 2.0.16", + "thiserror 2.0.17", ] [[package]] name = "alloy-signer-local" -version = "0.15.11" +version = "1.0.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acb3f4e72378566b189624d54618c8adf07afbcf39d5f368f4486e35a66725b3" +checksum = "590dcaeb290cdce23155e68af4791d093afc3754b1a331198a25d2d44c5456e8" dependencies = [ - "alloy-consensus 0.15.11", + "alloy-consensus", "alloy-network", "alloy-primitives", "alloy-signer", @@ -891,46 +757,47 @@ dependencies = [ "coins-bip39", "k256", "rand 0.8.5", - "thiserror 2.0.16", + "thiserror 2.0.17", + "zeroize", ] [[package]] name = "alloy-sol-macro" -version = "1.3.1" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d20d867dcf42019d4779519a1ceb55eba8d7f3d0e4f0a89bcba82b8f9eb01e48" +checksum = "f3ce480400051b5217f19d6e9a82d9010cdde20f1ae9c00d53591e4a1afbb312" dependencies = [ "alloy-sol-macro-expander", "alloy-sol-macro-input", "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] name = "alloy-sol-macro-expander" -version = "1.3.1" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b74e91b0b553c115d14bd0ed41898309356dc85d0e3d4b9014c4e7715e48c8ad" +checksum = "6d792e205ed3b72f795a8044c52877d2e6b6e9b1d13f431478121d8d4eaa9028" dependencies = [ "alloy-sol-macro-input", "const-hex", "heck", - "indexmap 2.11.1", + "indexmap 2.12.0", "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", "syn-solidity", "tiny-keccak", ] [[package]] name = "alloy-sol-macro-input" -version = "1.3.1" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84194d31220803f5f62d0a00f583fd3a062b36382e2bea446f1af96727754565" +checksum = "0bd1247a8f90b465ef3f1207627547ec16940c35597875cdc09c49d58b19693c" dependencies = [ "const-hex", "dunce", @@ -938,15 +805,15 @@ dependencies = [ "macro-string", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", "syn-solidity", ] [[package]] name = "alloy-sol-type-parser" -version = "1.3.1" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe8c27b3cf6b2bb8361904732f955bc7c05e00be5f469cec7e2280b6167f3ff0" +checksum = "954d1b2533b9b2c7959652df3076954ecb1122a28cc740aa84e7b0a49f6ac0a9" dependencies = [ "serde", "winnow", @@ -954,9 +821,9 @@ dependencies = [ [[package]] name = "alloy-sol-types" -version = "1.3.1" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5383d34ea00079e6dd89c652bcbdb764db160cef84e6250926961a0b2295d04" +checksum = "70319350969a3af119da6fb3e9bddb1bce66c9ea933600cb297c8b1850ad2a3c" dependencies = [ "alloy-json-abi", "alloy-primitives", @@ -966,12 +833,12 @@ dependencies = [ [[package]] name = "alloy-transport" -version = "0.15.11" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6964d85cd986cfc015b96887b89beed9e06d0d015b75ee2b7bfbd64341aab874" +checksum = "cae82426d98f8bc18f53c5223862907cac30ab8fc5e4cd2bb50808e6d3ab43d8" dependencies = [ "alloy-json-rpc", - "alloy-primitives", + "auto_impl", "base64 0.22.1", "derive_more", "futures", @@ -979,9 +846,9 @@ dependencies = [ "parking_lot", "serde", "serde_json", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", - "tower 0.5.2", + "tower", "tracing", "url", "wasmtimer", @@ -989,24 +856,24 @@ dependencies = [ [[package]] name = "alloy-transport-http" -version = "0.15.11" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef7c5ea7bda4497abe4ea92dcb8c76e9f052c178f3c82aa6976bcb264675f73c" +checksum = "90aa6825760905898c106aba9c804b131816a15041523e80b6d4fe7af6380ada" dependencies = [ "alloy-json-rpc", "alloy-transport", "reqwest", "serde_json", - "tower 0.5.2", + "tower", "tracing", "url", ] [[package]] name = "alloy-transport-ipc" -version = "0.15.11" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c506a002d77e522ccab7b7068089a16e24694ea04cb89c0bfecf3cc3603fccf" +checksum = "6ace83a4a6bb896e5894c3479042e6ba78aa5271dde599aa8c36a021d49cc8cc" dependencies = [ "alloy-json-rpc", "alloy-pubsub", @@ -1024,9 +891,9 @@ dependencies = [ [[package]] name = "alloy-transport-ws" -version = "0.15.11" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ff1bb1182601fa5e7b0f8bac03dcd496441ed23859387731462b17511c6680" +checksum = "86c9ab4c199e3a8f3520b60ba81aa67bb21fed9ed0d8304e0569094d0758a56f" dependencies = [ "alloy-pubsub", "alloy-transport", @@ -1042,9 +909,9 @@ dependencies = [ [[package]] name = "alloy-trie" -version = "0.8.1" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "983d99aa81f586cef9dae38443245e585840fcf0fc58b09aee0b1f27aed1d500" +checksum = "e3412d52bb97c6c6cc27ccc28d4e6e8cf605469101193b50b0bd5813b1f990b5" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -1052,25 +919,9 @@ dependencies = [ "arrayvec", "derive_arbitrary", "derive_more", - "nybbles 0.3.4", + "nybbles", "proptest", - "proptest-derive", - "serde", - "smallvec", - "tracing", -] - -[[package]] -name = "alloy-trie" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3412d52bb97c6c6cc27ccc28d4e6e8cf605469101193b50b0bd5813b1f990b5" -dependencies = [ - "alloy-primitives", - "alloy-rlp", - "arrayvec", - "derive_more", - "nybbles 0.4.4", + "proptest-derive 0.5.1", "serde", "smallvec", "tracing", @@ -1078,15 +929,14 @@ dependencies = [ [[package]] name = "alloy-tx-macros" -version = "1.0.32" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e434e0917dce890f755ea774f59d6f12557bc8c7dd9fa06456af80cfe0f0181e" +checksum = "ae109e33814b49fc0a62f2528993aa8a2dd346c26959b151f05441dc0b9da292" dependencies = [ - "alloy-primitives", "darling 0.21.3", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -1106,9 +956,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstream" -version = "0.6.20" +version = "0.6.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ae563653d1938f79b1ab1b5e668c87c76a9930414574a6583a7b7e11a8e6192" +checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" dependencies = [ "anstyle", "anstyle-parse", @@ -1121,9 +971,9 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.11" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd" +checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" [[package]] name = "anstyle-parse" @@ -1156,9 +1006,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.99" +version = "1.0.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0674a1ddeecb70197781e945de4b3b8ffb61fa939a5597bcf48503737663100" +checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" [[package]] name = "aquamarine" @@ -1171,7 +1021,7 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -1313,7 +1163,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62945a2f7e6de02a31fe400aa489f0e0f5b2502e69f95f853adb82a96c7a6b60" dependencies = [ "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -1351,7 +1201,7 @@ dependencies = [ "num-traits", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -1440,7 +1290,7 @@ checksum = "213888f660fddcca0d257e88e54ac05bca01885f258ccdf695bafd77031bb69d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -1513,9 +1363,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.30" +version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "977eb15ea9efd848bb8a4a1a2500347ed7f0bf794edf0dc3ddcf439f43d36b23" +checksum = "5a89bce6054c720275ac2432fbba080a66a2106a44a1b804553930ca6909f4e0" dependencies = [ "compression-codecs", "compression-core", @@ -1531,7 +1381,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e6fa871e4334a622afd6bb2f611635e8083a6f5e2936c0f90f37c7ef9856298" dependencies = [ "async-channel", - "futures-lite", + "futures-lite 1.13.0", "http-types", "log", "memchr", @@ -1557,7 +1407,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -1568,7 +1418,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -1606,7 +1456,7 @@ checksum = "ffdcb70bdbc4d478427380519163274ac86e52916e10f0a8889adf0f96d3fee7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -1616,28 +1466,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] -name = "backon" -version = "1.5.2" +name = "az" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "592277618714fbcecda9a02ba7a8781f319d26532a88553bbacc77ba5d2b3a8d" -dependencies = [ - "fastrand 2.3.0", - "tokio", -] +checksum = "7b7e4c2464d97fe331d41de9d5db0def0a96f4d823b8b32a2efd503578988973" [[package]] -name = "backtrace" -version = "0.3.75" +name = "backon" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6806a6321ec58106fea15becdad98371e28d92ccbc7c8f1b3b6dd724fe8f1002" +checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" dependencies = [ - "addr2line", - "cfg-if", - "libc", - "miniz_oxide", - "object", - "rustc-demangle", - "windows-targets 0.52.6", + "fastrand 2.3.0", + "tokio", ] [[package]] @@ -1652,6 +1493,16 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" +[[package]] +name = "base256emoji" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e9430d9a245a77c92176e649af6e275f20839a48389859d1661e9a128d077c" +dependencies = [ + "const-str", + "match-lookup", +] + [[package]] name = "base64" version = "0.13.1" @@ -1719,20 +1570,38 @@ dependencies = [ [[package]] name = "bindgen" -version = "0.70.1" +version = "0.71.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f49d8fed880d473ea71efb9bf597651e77201bdd4893efe54c9e5d65ae04ce6f" +checksum = "5f58bf3d7db68cfbac37cfc485a8d711e87e064c3d0fe0435b92f7a407f9d6b3" dependencies = [ - "bitflags 2.9.4", + "bitflags 2.10.0", "cexpr", "clang-sys", "itertools 0.13.0", "proc-macro2", "quote", "regex", - "rustc-hash 1.1.0", + "rustc-hash", "shlex", - "syn 2.0.106", + "syn 2.0.108", +] + +[[package]] +name = "bindgen" +version = "0.72.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +dependencies = [ + "bitflags 2.10.0", + "cexpr", + "clang-sys", + "itertools 0.13.0", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn 2.0.108", ] [[package]] @@ -1774,12 +1643,12 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.9.4" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2261d10cca569e4643e526d8dc2e62e433cc8aba21ab764233731f8d369bf394" +checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" dependencies = [ "arbitrary", - "serde", + "serde_core", ] [[package]] @@ -1795,28 +1664,6 @@ dependencies = [ "wyz", ] -[[package]] -name = "blake3" -version = "1.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3888aaa89e4b2a40fca9848e400f6a658a5a3978de7be858e209cafa8be9a4a0" -dependencies = [ - "arrayref", - "arrayvec", - "cc", - "cfg-if", - "constant_time_eq", -] - -[[package]] -name = "block-buffer" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" -dependencies = [ - "generic-array", -] - [[package]] name = "block-buffer" version = "0.10.4" @@ -1835,11 +1682,20 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdeb9d870516001442e364c5220d3574d2da8dc765554b4a617230d33fa58ef5" +dependencies = [ + "objc2", +] + [[package]] name = "blst" -version = "0.3.15" +version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fd49896f12ac9b6dcd7a5998466b9b58263a695a3dd1ecc1aaca2e12a90b080" +checksum = "dcdb4c7013139a150f9fc55d123186dbfaba0d912817466282c73ac49e71fb45" dependencies = [ "cc", "glob", @@ -1849,140 +1705,142 @@ dependencies = [ [[package]] name = "boa_ast" -version = "0.20.0" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c340fe0f0b267787095cbe35240c6786ff19da63ec7b69367ba338eace8169b" +checksum = "bc119a5ad34c3f459062a96907f53358989b173d104258891bb74f95d93747e8" dependencies = [ - "bitflags 2.9.4", + "bitflags 2.10.0", "boa_interner", "boa_macros", "boa_string", - "indexmap 2.11.1", + "indexmap 2.12.0", "num-bigint", - "rustc-hash 2.1.1", + "rustc-hash", ] [[package]] name = "boa_engine" -version = "0.20.0" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f620c3f06f51e65c0504ddf04978be1b814ac6586f0b45f6019801ab5efd37f9" +checksum = "e637ec52ea66d76b0ca86180c259d6c7bb6e6a6e14b2f36b85099306d8b00cc3" dependencies = [ + "aligned-vec", "arrayvec", - "bitflags 2.9.4", + "bitflags 2.10.0", "boa_ast", "boa_gc", "boa_interner", "boa_macros", "boa_parser", - "boa_profiler", "boa_string", "bytemuck", "cfg-if", + "cow-utils", "dashmap 6.1.0", + "dynify", "fast-float2", - "hashbrown 0.15.5", - "icu_normalizer 1.5.0", - "indexmap 2.11.1", + "float16", + "futures-channel", + "futures-concurrency", + "futures-lite 2.6.1", + "hashbrown 0.16.0", + "icu_normalizer", + "indexmap 2.12.0", "intrusive-collections", - "itertools 0.13.0", + "itertools 0.14.0", "num-bigint", "num-integer", "num-traits", "num_enum", - "once_cell", - "pollster", + "paste", "portable-atomic", - "rand 0.8.5", + "rand 0.9.2", "regress", - "rustc-hash 2.1.1", + "rustc-hash", "ryu-js", "serde", "serde_json", - "sptr", + "small_btree", "static_assertions", + "tag_ptr", "tap", "thin-vec", - "thiserror 2.0.16", + "thiserror 2.0.17", "time", + "xsum", ] [[package]] name = "boa_gc" -version = "0.20.0" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2425c0b7720d42d73eaa6a883fbb77a5c920da8694964a3d79a67597ac55cce2" +checksum = "f1179f690cbfcbe5364cceee5f1cb577265bb6f07b0be6f210aabe270adcf9da" dependencies = [ "boa_macros", - "boa_profiler", "boa_string", - "hashbrown 0.15.5", + "hashbrown 0.16.0", "thin-vec", ] [[package]] name = "boa_interner" -version = "0.20.0" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42407a3b724cfaecde8f7d4af566df4b56af32a2f11f0956f5570bb974e7f749" +checksum = "9626505d33dc63d349662437297df1d3afd9d5fc4a2b3ad34e5e1ce879a78848" dependencies = [ "boa_gc", "boa_macros", - "hashbrown 0.15.5", - "indexmap 2.11.1", + "hashbrown 0.16.0", + "indexmap 2.12.0", "once_cell", "phf", - "rustc-hash 2.1.1", + "rustc-hash", "static_assertions", ] [[package]] name = "boa_macros" -version = "0.20.0" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fd3f870829131332587f607a7ff909f1af5fc523fd1b192db55fbbdf52e8d3c" +checksum = "7f36418a46544b152632c141b0a0b7a453cd69ca150caeef83aee9e2f4b48b7d" dependencies = [ + "cfg-if", + "cow-utils", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", "synstructure", ] [[package]] name = "boa_parser" -version = "0.20.0" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cc142dac798cdc6e2dbccfddeb50f36d2523bb977a976e19bdb3ae19b740804" +checksum = "02f99bf5b684f0de946378fcfe5f38c3a0fbd51cbf83a0f39ff773a0e218541f" dependencies = [ - "bitflags 2.9.4", + "bitflags 2.10.0", "boa_ast", "boa_interner", "boa_macros", - "boa_profiler", "fast-float2", - "icu_properties 1.5.1", + "icu_properties", "num-bigint", "num-traits", "regress", - "rustc-hash 2.1.1", + "rustc-hash", ] -[[package]] -name = "boa_profiler" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4064908e7cdf9b6317179e9b04dcb27f1510c1c144aeab4d0394014f37a0f922" - [[package]] name = "boa_string" -version = "0.20.0" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7debc13fbf7997bf38bf8e9b20f1ad5e2a7d27a900e1f6039fe244ce30f589b5" +checksum = "45ce9d7aa5563a2e14eab111e2ae1a06a69a812f6c0c3d843196c9d03fbef440" dependencies = [ "fast-float2", + "itoa", "paste", - "rustc-hash 2.1.1", - "sptr", + "rustc-hash", + "ryu-js", "static_assertions", ] @@ -2022,15 +1880,15 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" dependencies = [ - "sha2 0.10.9", + "sha2", "tinyvec", ] [[package]] name = "bstr" -version = "1.12.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "234113d19d0d7d613b40e86fb654acf958910802bcceab913a4f9e7cda03b1a4" +checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab" dependencies = [ "memchr", "regex-automata", @@ -2057,22 +1915,22 @@ checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" [[package]] name = "bytemuck" -version = "1.23.2" +version = "1.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3995eaeebcdf32f91f980d360f78732ddc061097ab4e39991ae7a6ace9194677" +checksum = "1fbdf580320f38b612e485521afda1ee26d10cc9884efaaa750d383e13e3c5f4" dependencies = [ "bytemuck_derive", ] [[package]] name = "bytemuck_derive" -version = "1.10.1" +version = "1.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f154e572231cb6ba2bd1176980827e3d5dc04cc183a75dea38109fbdd672d29" +checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -2092,9 +1950,9 @@ dependencies = [ [[package]] name = "c-kzg" -version = "2.1.1" +version = "2.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7318cfa722931cb5fe0838b98d3ce5621e75f6a6408abc21721d80de9223f2e4" +checksum = "e00bf4b112b07b505472dbefd19e37e53307e2bfed5a79e0cc161d58ccd0e687" dependencies = [ "arbitrary", "blst", @@ -2108,11 +1966,11 @@ dependencies = [ [[package]] name = "camino" -version = "1.1.12" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd0b03af37dad7a14518b7691d81acb0f8222604ad3d1b02f6b4bed5188c0cd5" +checksum = "276a59bf2b2c967788139340c9f0c5b12d7fd6630315c15c217e559de85d2609" dependencies = [ - "serde", + "serde_core", ] [[package]] @@ -2132,7 +1990,7 @@ checksum = "4acbb09d9ee8e23699b9634375c72795d095bf268439da88562cf9b501f181fa" dependencies = [ "camino", "cargo-platform", - "semver 1.0.26", + "semver 1.0.27", "serde", "serde_json", ] @@ -2145,10 +2003,10 @@ checksum = "dd5eb614ed4c27c5d706420e4320fbe3216ab31fa1c33cd8246ac36dae4479ba" dependencies = [ "camino", "cargo-platform", - "semver 1.0.26", + "semver 1.0.27", "serde", "serde_json", - "thiserror 2.0.16", + "thiserror 2.0.17", ] [[package]] @@ -2200,9 +2058,9 @@ dependencies = [ [[package]] name = "cfg-if" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fd1289c04a9ea8cb22300a459a72a385d7c73d3259e2ed7dcb2af674838cfa9" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "cfg_aliases" @@ -2221,7 +2079,7 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-link 0.2.0", + "windows-link 0.2.1", ] [[package]] @@ -2274,9 +2132,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.47" +version = "4.5.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7eac00902d9d136acd712710d71823fb8ac8004ca445a89e73a41d45aa712931" +checksum = "4c26d721170e0295f191a69bd9a1f93efcdb0aff38684b61ab5750468972e5f5" dependencies = [ "clap_builder", "clap_derive", @@ -2284,9 +2142,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.47" +version = "4.5.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ad9bbf750e73b5884fb8a211a9424a1906c1e156724260fdae972f31d70e1d6" +checksum = "75835f0c7bf681bfd05abe44e965760fea999a5286c6eb2d59883634fd02011a" dependencies = [ "anstream", "anstyle", @@ -2296,21 +2154,21 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.47" +version = "4.5.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbfd7eae0b0f1a6e63d4b13c9c478de77c2eb546fba158ad50b4203dc24b9f9c" +checksum = "2a0b5487afeab2deb2ff4e03a807ad1a03ac532ff5a2cee5d86884440c7f7671" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] name = "clap_lex" -version = "0.7.5" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" +checksum = "a1d728cc89cf3aee9ff92b05e62b19ee65a02b5702cff7d5a377e32c6ae29d8d" [[package]] name = "cmake" @@ -2357,7 +2215,7 @@ dependencies = [ "hmac", "k256", "serde", - "sha2 0.10.9", + "sha2", "thiserror 1.0.69", ] @@ -2373,7 +2231,7 @@ dependencies = [ "once_cell", "pbkdf2", "rand 0.8.5", - "sha2 0.10.9", + "sha2", "thiserror 1.0.69", ] @@ -2391,7 +2249,7 @@ dependencies = [ "generic-array", "ripemd", "serde", - "sha2 0.10.9", + "sha2", "sha3", "thiserror 1.0.69", ] @@ -2449,9 +2307,9 @@ dependencies = [ [[package]] name = "compression-codecs" -version = "0.4.30" +version = "0.4.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "485abf41ac0c8047c07c87c72c8fb3eb5197f6e9d7ded615dfd1a00ae00a0f64" +checksum = "ef8a506ec4b81c460798f572caead636d57d3d7e940f998160f52bd254bf2d23" dependencies = [ "brotli", "compression-core", @@ -2499,15 +2357,14 @@ dependencies = [ [[package]] name = "const-hex" -version = "1.15.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dccd746bf9b1038c0507b7cec21eb2b11222db96a2902c96e8c185d6d20fb9c4" +checksum = "3bb320cac8a0750d7f25280aa97b09c26edfe161164238ecbbb31092b079e735" dependencies = [ "cfg-if", "cpufeatures", - "hex", "proptest", - "serde", + "serde_core", ] [[package]] @@ -2517,10 +2374,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" [[package]] -name = "constant_time_eq" -version = "0.3.1" +name = "const-str" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f421161cb492475f1661ddc9815a745a1c894592070661180fdec3d4872e9c3" + +[[package]] +name = "const_format" +version = "0.2.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" +checksum = "7faa7469a93a566e9ccc1c73fe783b4a65c274c5ace346038dca9c39fe0030ad" +dependencies = [ + "const_format_proc_macros", +] + +[[package]] +name = "const_format_proc_macros" +version = "0.2.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d57c2eccfb16dbac1f4e61e206105db5820c9d26c3c472bc17c774259ef7744" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] [[package]] name = "convert_case" @@ -2531,6 +2408,16 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "cordyceps" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "688d7fbb8092b8de775ef2536f36c8c31f2bc4006ece2e8d8ad2d17d00ce0a2a" +dependencies = [ + "loom", + "tracing", +] + [[package]] name = "core-foundation" version = "0.10.1" @@ -2556,6 +2443,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "cow-utils" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "417bef24afe1460300965a25ff4a24b8b45ad011948302ec221e8a0a81eb2c79" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -2673,7 +2566,7 @@ version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" dependencies = [ - "bitflags 2.9.4", + "bitflags 2.10.0", "crossterm_winapi", "mio", "parking_lot", @@ -2689,7 +2582,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8b9f2e4c67f833b660cdb0a3523065869fb35570177239812ed4c905aeff87b" dependencies = [ - "bitflags 2.9.4", + "bitflags 2.10.0", "crossterm_winapi", "document-features", "parking_lot", @@ -2708,9 +2601,9 @@ dependencies = [ [[package]] name = "crunchy" -version = "0.2.2" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" [[package]] name = "crypto-bigint" @@ -2737,21 +2630,21 @@ dependencies = [ [[package]] name = "csv" -version = "1.3.1" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acdc4883a9c96732e4733212c01447ebd805833b7275a73ca3ee080fd77afdaf" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" dependencies = [ "csv-core", "itoa", "ryu", - "serde", + "serde_core", ] [[package]] name = "csv-core" -version = "0.1.12" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d02f3b0da4c6504f86e9cd789d8dbafab48c2321be74e9987593de5a894d93d" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" dependencies = [ "memchr", ] @@ -2765,6 +2658,17 @@ dependencies = [ "cipher", ] +[[package]] +name = "ctrlc" +version = "3.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73736a89c4aff73035ba2ed2e565061954da00d4970fc9ac25dcc85a2a20d790" +dependencies = [ + "dispatch2", + "nix 0.30.1", + "windows-sys 0.61.2", +] + [[package]] name = "curve25519-dalek" version = "4.1.3" @@ -2789,7 +2693,20 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", +] + +[[package]] +name = "custom-hardforks" +version = "0.1.0" +dependencies = [ + "alloy-consensus", + "alloy-eips", + "alloy-genesis", + "alloy-primitives", + "reth-chainspec", + "reth-network-peers", + "serde", ] [[package]] @@ -2823,7 +2740,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -2838,7 +2755,7 @@ dependencies = [ "quote", "serde", "strsim", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -2849,7 +2766,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core 0.20.11", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -2860,7 +2777,7 @@ checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" dependencies = [ "darling_core 0.21.3", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -2913,7 +2830,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d162beedaa69905488a8da94f5ac3edb4dd4788b732fadb7bd120b2625c1976" dependencies = [ "data-encoding", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -2945,12 +2862,12 @@ dependencies = [ [[package]] name = "deranged" -version = "0.5.3" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d630bccd429a5bb5a64b5e94f693bfc48c9f8566418fda4c494cc94f911f87cc" +checksum = "ececcb659e7ba858fb4f10388c250a7252eb0a27373f1a72b8748afdd248e587" dependencies = [ "powerfmt", - "serde", + "serde_core", ] [[package]] @@ -2972,7 +2889,7 @@ checksum = "ef941ded77d15ca19b40374869ac6000af1c9f2a4c0f3d4c70926287e6364a8f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -2983,7 +2900,7 @@ checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -3004,7 +2921,7 @@ dependencies = [ "darling 0.20.11", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -3014,7 +2931,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" dependencies = [ "derive_builder_core", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -3035,10 +2952,16 @@ dependencies = [ "convert_case", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", "unicode-xid", ] +[[package]] +name = "diatomic-waker" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab03c107fafeb3ee9f5925686dbb7a73bc76e3932abb0d2b365cb64b169cf04c" + [[package]] name = "diff" version = "0.1.13" @@ -3060,7 +2983,7 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer 0.10.4", + "block-buffer", "const-oid", "crypto-common", "subtle", @@ -3110,9 +3033,9 @@ dependencies = [ [[package]] name = "discv5" -version = "0.9.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4b4e7798d2ff74e29cee344dc490af947ae657d6ab5273dde35d58ce06a4d71" +checksum = "f170f4f6ed0e1df52bf43b403899f0081917ecf1500bfe312505cc3b515a8899" dependencies = [ "aes", "aes-gcm", @@ -3141,6 +3064,18 @@ dependencies = [ "zeroize", ] +[[package]] +name = "dispatch2" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89a09f22a6c6069a18470eb92d2298acf25463f14256d24778e1230d789a2aec" +dependencies = [ + "bitflags 2.10.0", + "block2", + "libc", + "objc2", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -3149,7 +3084,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -3160,9 +3095,9 @@ checksum = "aac81fa3e28d21450aa4d2ac065992ba96a1d7303efbce51a95f4fd175b67562" [[package]] name = "document-features" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95249b50c6c185bee49034bcb378a49dc2b5dff0be90ff6616d31d64febab05d" +checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61" dependencies = [ "litrs", ] @@ -3179,6 +3114,26 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "dynify" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81acb15628a3e22358bf73de5e7e62360b8a777dbcb5fc9ac7dfa9ae73723747" +dependencies = [ + "dynify-macros", +] + +[[package]] +name = "dynify-macros" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ec431cd708430d5029356535259c5d645d60edd3d39c54e5eea9782d46caa7d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.108", +] + [[package]] name = "ecdsa" version = "0.16.9" @@ -3214,7 +3169,7 @@ dependencies = [ "ed25519", "rand_core 0.6.4", "serde", - "sha2 0.10.9", + "sha2", "subtle", "zeroize", ] @@ -3228,19 +3183,26 @@ dependencies = [ "enum-ordinalize", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", +] + +[[package]] +name = "ef-test-runner" +version = "1.9.3" +dependencies = [ + "clap", + "ef-tests", ] [[package]] name = "ef-tests" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", + "alloy-consensus", + "alloy-eips", "alloy-genesis", "alloy-primitives", "alloy-rlp", - "crunchy", "rayon", "reth-chainspec", "reth-consensus", @@ -3261,8 +3223,7 @@ dependencies = [ "revm", "serde", "serde_json", - "thiserror 2.0.16", - "tracing", + "thiserror 2.0.17", "walkdir", ] @@ -3315,7 +3276,7 @@ dependencies = [ "k256", "log", "rand 0.8.5", - "secp256k1", + "secp256k1 0.30.0", "serde", "sha3", "zeroize", @@ -3330,27 +3291,47 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] name = "enum-ordinalize" -version = "4.3.0" +version = "4.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fea0dcfa4e54eeb516fe454635a95753ddd39acda650ce703031c6973e315dd5" +checksum = "4a1091a7bb1f8f2c4b28f1fe2cef4980ca2d410a3d727d67ecc3178c9b0800f0" dependencies = [ "enum-ordinalize-derive", ] [[package]] name = "enum-ordinalize-derive" -version = "4.3.1" +version = "4.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ca9601fb2d62598ee17836250842873a413586e5d7ed88b356e38ddbb0ec631" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.108", +] + +[[package]] +name = "equator" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4711b213838dfee0117e3be6ac926007d7f433d7bbe33595975d4190cb07e6fc" +dependencies = [ + "equator-macro", +] + +[[package]] +name = "equator-macro" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d28318a75d4aead5c4db25382e8ef717932d0346600cacae6357eb5941bc5ff" +checksum = "44f23cf4b44bfce11a86ace86f8a73ffdec849c9fd00a386a53d278bd9e81fb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -3366,7 +3347,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -3386,7 +3367,7 @@ checksum = "c853bd72c9e5787f8aafc3df2907c2ed03cff3150c3acd94e2e53a98ab70a8ab" dependencies = [ "cpufeatures", "ring", - "sha2 0.10.9", + "sha2", ] [[package]] @@ -3404,9 +3385,9 @@ dependencies = [ [[package]] name = "ethereum_ssz" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ca8ba45b63c389c6e115b095ca16381534fdcc03cf58176a3f8554db2dbe19b" +checksum = "0dcddb2554d19cde19b099fadddde576929d7a4d0c1cd3512d1fd95cf174375c" dependencies = [ "alloy-primitives", "ethereum_serde_utils", @@ -3419,14 +3400,14 @@ dependencies = [ [[package]] name = "ethereum_ssz_derive" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dd55d08012b4e0dfcc92b8d6081234df65f2986ad34cc76eeed69c5e2ce7506" +checksum = "a657b6b3b7e153637dc6bdc6566ad9279d9ee11a15b12cfb24a2e04360637e9f" dependencies = [ "darling 0.20.11", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -3439,18 +3420,18 @@ checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" name = "example-beacon-api-sidecar-fetcher" version = "0.1.0" dependencies = [ - "alloy-consensus 0.15.11", + "alloy-consensus", + "alloy-eips", "alloy-primitives", "alloy-rpc-types-beacon", "clap", "eyre", "futures-util", "reqwest", - "reth", "reth-ethereum", "serde", "serde_json", - "thiserror 2.0.16", + "thiserror 2.0.17", ] [[package]] @@ -3474,7 +3455,6 @@ dependencies = [ "alloy-rlp", "alloy-rpc-types", "bytes", - "derive_more", "futures", "reth-chainspec", "reth-discv4", @@ -3491,10 +3471,10 @@ dependencies = [ "reth-primitives-traits", "reth-provider", "reth-tracing", - "secp256k1", + "secp256k1 0.30.0", "serde", "serde_json", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "tokio-stream", "tracing", @@ -3504,12 +3484,11 @@ dependencies = [ name = "example-custom-beacon-withdrawals" version = "0.0.0" dependencies = [ - "alloy-eips 0.15.11", + "alloy-eips", "alloy-evm", "alloy-sol-macro", "alloy-sol-types", "eyre", - "reth", "reth-ethereum", ] @@ -3521,7 +3500,6 @@ dependencies = [ "alloy-primitives", "eyre", "futures-util", - "reth", "reth-ethereum", "serde_json", "tokio", @@ -3531,21 +3509,18 @@ dependencies = [ name = "example-custom-engine-types" version = "0.0.0" dependencies = [ - "alloy-eips 0.15.11", + "alloy-eips", "alloy-genesis", "alloy-primitives", "alloy-rpc-types", "eyre", - "reth", "reth-basic-payload-builder", - "reth-engine-local", "reth-ethereum", "reth-ethereum-payload-builder", "reth-payload-builder", "reth-tracing", - "reth-trie-db", "serde", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", ] @@ -3557,7 +3532,6 @@ dependencies = [ "alloy-genesis", "alloy-primitives", "eyre", - "reth", "reth-ethereum", "reth-tracing", "tokio", @@ -3567,13 +3541,12 @@ dependencies = [ name = "example-custom-inspector" version = "0.0.0" dependencies = [ - "alloy-eips 0.15.11", + "alloy-eips", "alloy-evm", "alloy-primitives", - "alloy-rpc-types-eth 0.15.11", + "alloy-rpc-types-eth", "clap", "futures-util", - "reth", "reth-ethereum", ] @@ -3581,30 +3554,35 @@ dependencies = [ name = "example-custom-node" version = "0.0.0" dependencies = [ - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", + "alloy-consensus", + "alloy-eips", "alloy-evm", "alloy-genesis", + "alloy-network", "alloy-op-evm", "alloy-primitives", "alloy-rlp", "alloy-rpc-types-engine", - "alloy-serde 0.15.11", + "alloy-rpc-types-eth", + "alloy-serde", "async-trait", "derive_more", "eyre", "jsonrpsee", "modular-bitfield", "op-alloy-consensus", + "op-alloy-rpc-types", "op-alloy-rpc-types-engine", "op-revm", "reth-chain-state", "reth-codecs", + "reth-db-api", + "reth-engine-primitives", "reth-ethereum", "reth-network-peers", "reth-node-builder", "reth-op", - "reth-optimism-consensus", + "reth-optimism-flashblocks", "reth-optimism-forks", "reth-payload-builder", "reth-rpc-api", @@ -3612,7 +3590,7 @@ dependencies = [ "revm", "revm-primitives", "serde", - "test-fuzz", + "thiserror 2.0.17", ] [[package]] @@ -3620,7 +3598,6 @@ name = "example-custom-node-components" version = "0.0.0" dependencies = [ "eyre", - "reth", "reth-ethereum", "reth-tracing", ] @@ -3629,10 +3606,9 @@ dependencies = [ name = "example-custom-payload-builder" version = "0.0.0" dependencies = [ - "alloy-eips 0.15.11", + "alloy-eips", "eyre", "futures-util", - "reth", "reth-basic-payload-builder", "reth-ethereum", "reth-ethereum-payload-builder", @@ -3647,7 +3623,6 @@ dependencies = [ "alloy-primitives", "eyre", "futures", - "reth", "reth-ethereum", "tokio", "tokio-stream", @@ -3663,6 +3638,17 @@ dependencies = [ "reth-ethereum", ] +[[package]] +name = "example-engine-api-access" +version = "0.0.0" +dependencies = [ + "reth-db", + "reth-node-builder", + "reth-optimism-chainspec", + "reth-optimism-node", + "tokio", +] + [[package]] name = "example-exex-hello-world" version = "0.0.0" @@ -3670,7 +3656,6 @@ dependencies = [ "clap", "eyre", "futures", - "reth", "reth-ethereum", "reth-op", "reth-tracing", @@ -3689,18 +3674,26 @@ dependencies = [ "tokio", ] +[[package]] +name = "example-full-contract-state" +version = "1.9.3" +dependencies = [ + "eyre", + "reth-ethereum", +] + [[package]] name = "example-manual-p2p" version = "0.0.0" dependencies = [ - "alloy-consensus 0.15.11", + "alloy-consensus", "eyre", "futures", "reth-discv4", "reth-ecies", "reth-ethereum", "reth-network-peers", - "secp256k1", + "secp256k1 0.30.0", "tokio", ] @@ -3734,6 +3727,13 @@ dependencies = [ "tokio", ] +[[package]] +name = "example-node-builder-api" +version = "0.0.0" +dependencies = [ + "reth-ethereum", +] + [[package]] name = "example-node-custom-rpc" version = "0.0.0" @@ -3741,6 +3741,7 @@ dependencies = [ "clap", "jsonrpsee", "reth-ethereum", + "serde_json", "tokio", ] @@ -3751,6 +3752,14 @@ dependencies = [ "reth-ethereum", ] +[[package]] +name = "example-op-db-access" +version = "0.0.0" +dependencies = [ + "eyre", + "reth-op", +] + [[package]] name = "example-polygon-p2p" version = "0.0.0" @@ -3759,7 +3768,7 @@ dependencies = [ "reth-discv4", "reth-ethereum", "reth-tracing", - "secp256k1", + "secp256k1 0.30.0", "serde_json", "tokio", "tokio-stream", @@ -3774,7 +3783,6 @@ dependencies = [ "alloy-primitives", "eyre", "parking_lot", - "reth", "reth-ethereum", "reth-tracing", "schnellru", @@ -3788,7 +3796,6 @@ dependencies = [ "eyre", "futures", "jsonrpsee", - "reth", "reth-ethereum", "tokio", ] @@ -3797,17 +3804,18 @@ dependencies = [ name = "example-txpool-tracing" version = "0.0.0" dependencies = [ + "alloy-network", "alloy-primitives", "alloy-rpc-types-trace", "clap", + "eyre", "futures-util", - "reth", "reth-ethereum", ] [[package]] name = "exex-subscription" -version = "1.3.12" +version = "1.9.3" dependencies = [ "alloy-primitives", "clap", @@ -3816,6 +3824,7 @@ dependencies = [ "jsonrpsee", "reth-ethereum", "serde", + "serde_json", "tokio", "tracing", ] @@ -3923,16 +3932,32 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + [[package]] name = "flate2" -version = "1.1.2" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a3d7db9596fecd151c5f638c0ee5d5bd487b6e0ea232e5dc96d5250f6f94b1d" +checksum = "bfe33edd8e85a12a67454e37f8c75e730830d83e313556ab9ebf9ee7fbeb3bfb" dependencies = [ "crc32fast", "miniz_oxide", ] +[[package]] +name = "float16" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bffafbd079d520191c7c2779ae9cf757601266cf4167d3f659ff09617ff8483" +dependencies = [ + "cfg-if", + "rustc_version 0.2.3", +] + [[package]] name = "fnv" version = "1.0.7" @@ -3945,6 +3970,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -3984,6 +4015,19 @@ dependencies = [ "futures-util", ] +[[package]] +name = "futures-buffered" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8e0e1f38ec07ba4abbde21eed377082f17ccb988be9d988a5adbf4bafc118fd" +dependencies = [ + "cordyceps", + "diatomic-waker", + "futures-core", + "pin-project-lite", + "spin", +] + [[package]] name = "futures-channel" version = "0.3.31" @@ -3994,6 +4038,21 @@ dependencies = [ "futures-sink", ] +[[package]] +name = "futures-concurrency" +version = "7.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eb68017df91f2e477ed4bea586c59eaecaa47ed885a770d0444e21e62572cd2" +dependencies = [ + "fixedbitset", + "futures-buffered", + "futures-core", + "futures-lite 2.6.1", + "pin-project", + "slab", + "smallvec", +] + [[package]] name = "futures-core" version = "0.3.31" @@ -4033,14 +4092,27 @@ dependencies = [ ] [[package]] -name = "futures-macro" -version = "0.3.31" +name = "futures-lite" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad" +dependencies = [ + "fastrand 2.3.0", + "futures-core", + "futures-io", + "parking", + "pin-project-lite", +] + +[[package]] +name = "futures-macro" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -4109,7 +4181,6 @@ version = "0.14.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ - "serde", "typenum", "version_check", "zeroize", @@ -4141,15 +4212,15 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", "js-sys", "libc", "r-efi", - "wasi 0.14.7+wasi-0.2.4", + "wasip2", "wasm-bindgen", ] @@ -4163,19 +4234,13 @@ dependencies = [ "polyval", ] -[[package]] -name = "gimli" -version = "0.31.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" - [[package]] name = "git2" version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2deb07a133b1520dc1a5690e9bd08950108873d7ed5de38dcc74d3b5ebffa110" dependencies = [ - "bitflags 2.9.4", + "bitflags 2.10.0", "libc", "libgit2-sys", "log", @@ -4234,6 +4299,16 @@ dependencies = [ "web-sys", ] +[[package]] +name = "gmp-mpfr-sys" +version = "1.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60f8970a75c006bb2f8ae79c6768a116dd215fa8346a87aed99bf9d82ca43394" +dependencies = [ + "libc", + "windows-sys 0.60.2", +] + [[package]] name = "group" version = "0.13.0" @@ -4257,7 +4332,7 @@ dependencies = [ "futures-core", "futures-sink", "http", - "indexmap 2.11.1", + "indexmap 2.12.0", "slab", "tokio", "tokio-util", @@ -4266,12 +4341,13 @@ dependencies = [ [[package]] name = "half" -version = "2.6.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" dependencies = [ "cfg-if", "crunchy", + "zerocopy", ] [[package]] @@ -4309,7 +4385,18 @@ checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ "allocator-api2", "equivalent", - "foldhash", + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", "serde", ] @@ -4349,9 +4436,6 @@ name = "hex" version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" -dependencies = [ - "serde", -] [[package]] name = "hex-conservative" @@ -4381,7 +4465,7 @@ dependencies = [ "rand 0.9.2", "ring", "serde", - "thiserror 2.0.16", + "thiserror 2.0.17", "tinyvec", "tokio", "tracing", @@ -4405,7 +4489,7 @@ dependencies = [ "resolv-conf", "serde", "smallvec", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "tracing", ] @@ -4477,7 +4561,7 @@ dependencies = [ "anyhow", "async-channel", "base64 0.13.1", - "futures-lite", + "futures-lite 1.13.0", "infer", "pin-project-lite", "rand 0.7.3", @@ -4561,7 +4645,20 @@ dependencies = [ "tokio", "tokio-rustls", "tower-service", - "webpki-roots 1.0.2", + "webpki-roots 1.0.4", +] + +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", ] [[package]] @@ -4582,7 +4679,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.0", + "socket2 0.6.1", "tokio", "tower-service", "tracing", @@ -4612,18 +4709,6 @@ dependencies = [ "cc", ] -[[package]] -name = "icu_collections" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" -dependencies = [ - "displaydoc", - "yoke 0.7.5", - "zerofrom", - "zerovec 0.10.4", -] - [[package]] name = "icu_collections" version = "2.0.0" @@ -4632,96 +4717,42 @@ checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47" dependencies = [ "displaydoc", "potential_utf", - "yoke 0.8.0", + "yoke", "zerofrom", - "zerovec 0.11.4", + "zerovec", ] [[package]] name = "icu_locale_core" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a" -dependencies = [ - "displaydoc", - "litemap 0.8.0", - "tinystr 0.8.1", - "writeable 0.6.1", - "zerovec 0.11.4", -] - -[[package]] -name = "icu_locid" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" -dependencies = [ - "displaydoc", - "litemap 0.7.5", - "tinystr 0.7.6", - "writeable 0.5.5", - "zerovec 0.10.4", -] - -[[package]] -name = "icu_locid_transform" -version = "1.5.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" dependencies = [ "displaydoc", - "icu_locid", - "icu_locid_transform_data", - "icu_provider 1.5.0", - "tinystr 0.7.6", - "zerovec 0.10.4", + "litemap", + "serde", + "tinystr", + "writeable", + "zerovec", ] -[[package]] -name = "icu_locid_transform_data" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7515e6d781098bf9f7205ab3fc7e9709d34554ae0b21ddbcb5febfa4bc7df11d" - [[package]] name = "icu_normalizer" -version = "1.5.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +checksum = "8b24a59706036ba941c9476a55cd57b82b77f38a3c667d637ee7cabbc85eaedc" dependencies = [ "displaydoc", - "icu_collections 1.5.0", - "icu_normalizer_data 1.5.1", - "icu_properties 1.5.1", - "icu_provider 1.5.0", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", "smallvec", "utf16_iter", - "utf8_iter", "write16", - "zerovec 0.10.4", -] - -[[package]] -name = "icu_normalizer" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "436880e8e18df4d7bbc06d58432329d6458cc84531f7ac5f024e93deadb37979" -dependencies = [ - "displaydoc", - "icu_collections 2.0.0", - "icu_normalizer_data 2.0.0", - "icu_properties 2.0.1", - "icu_provider 2.0.0", - "smallvec", - "zerovec 0.11.4", + "zerovec", ] -[[package]] -name = "icu_normalizer_data" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5e8338228bdc8ab83303f16b797e177953730f601a96c25d10cb3ab0daa0cb7" - [[package]] name = "icu_normalizer_data" version = "2.0.0" @@ -4730,41 +4761,20 @@ checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3" [[package]] name = "icu_properties" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" -dependencies = [ - "displaydoc", - "icu_collections 1.5.0", - "icu_locid_transform", - "icu_properties_data 1.5.1", - "icu_provider 1.5.0", - "tinystr 0.7.6", - "zerovec 0.10.4", -] - -[[package]] -name = "icu_properties" -version = "2.0.1" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "016c619c1eeb94efb86809b015c58f479963de65bdb6253345c1a1276f22e32b" +checksum = "f5a97b8ac6235e69506e8dacfb2adf38461d2ce6d3e9bd9c94c4cbc3cd4400a4" dependencies = [ "displaydoc", - "icu_collections 2.0.0", + "icu_collections", "icu_locale_core", - "icu_properties_data 2.0.1", - "icu_provider 2.0.0", + "icu_properties_data", + "icu_provider", "potential_utf", "zerotrie", - "zerovec 0.11.4", + "zerovec", ] -[[package]] -name = "icu_properties_data" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85fb8799753b75aee8d2a21d7c14d9f38921b54b3dbda10f5a3c7a7b82dba5e2" - [[package]] name = "icu_properties_data" version = "2.0.1" @@ -4773,47 +4783,19 @@ checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632" [[package]] name = "icu_provider" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" -dependencies = [ - "displaydoc", - "icu_locid", - "icu_provider_macros", - "stable_deref_trait", - "tinystr 0.7.6", - "writeable 0.5.5", - "yoke 0.7.5", - "zerofrom", - "zerovec 0.10.4", -] - -[[package]] -name = "icu_provider" -version = "2.0.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af" +checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" dependencies = [ "displaydoc", "icu_locale_core", + "serde", "stable_deref_trait", - "tinystr 0.8.1", - "writeable 0.6.1", - "yoke 0.8.0", + "writeable", + "yoke", "zerofrom", "zerotrie", - "zerovec 0.11.4", -] - -[[package]] -name = "icu_provider_macros" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.106", + "zerovec", ] [[package]] @@ -4839,15 +4821,15 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" dependencies = [ - "icu_normalizer 2.0.0", - "icu_properties 2.0.1", + "icu_normalizer", + "icu_properties", ] [[package]] name = "if-addrs" -version = "0.13.4" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69b2eeee38fef3aa9b4cc5f1beea8a2444fc00e7377cafae396de3f5c2065e24" +checksum = "bf39cc0423ee66021dc5eccface85580e4a001e0c5288bae8bea7ecb69225e90" dependencies = [ "libc", "windows-sys 0.59.0", @@ -4870,7 +4852,7 @@ checksum = "a0eb5a3343abf848c0984fe4604b2b105da9539376e24fc0a3b0007411ae4fd9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -4911,21 +4893,25 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.11.1" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "206a8042aec68fa4a62e8d3f7aa4ceb508177d9324faf261e1959e495b7a1921" +checksum = "6717a8d2a5a929a1a2eb43a12812498ed141a0bcfb7e8f7844fbdbe4303bba9f" dependencies = [ "arbitrary", "equivalent", - "hashbrown 0.15.5", + "hashbrown 0.16.0", "serde", + "serde_core", ] [[package]] name = "indoc" -version = "2.0.6" +version = "2.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4c7245a08504955605670dbf141fceab975f15ca21570696aebe9d2e71576bd" +checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706" +dependencies = [ + "rustversion", +] [[package]] name = "infer" @@ -4939,7 +4925,7 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f37dccff2791ab604f9babef0ba14fbe0be30bd368dc541e2b08d07c8aa908f3" dependencies = [ - "bitflags 2.9.4", + "bitflags 2.10.0", "inotify-sys", "libc", ] @@ -4973,7 +4959,7 @@ dependencies = [ "indoc", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -5009,17 +4995,6 @@ dependencies = [ "memoffset", ] -[[package]] -name = "io-uring" -version = "0.7.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046fa2d4d00aea763528b4950358d0ead425372445dc8ff86312b3c69ff7727b" -dependencies = [ - "bitflags 2.9.4", - "cfg-if", - "libc", -] - [[package]] name = "ipconfig" version = "0.3.2" @@ -5050,9 +5025,9 @@ dependencies = [ [[package]] name = "is-terminal" -version = "0.4.16" +version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9" +checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", @@ -5061,9 +5036,9 @@ dependencies = [ [[package]] name = "is_terminal_polyfill" -version = "1.70.1" +version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" [[package]] name = "itertools" @@ -5126,15 +5101,15 @@ version = "0.1.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" dependencies = [ - "getrandom 0.3.3", + "getrandom 0.3.4", "libc", ] [[package]] name = "js-sys" -version = "0.3.79" +version = "0.3.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6247da8b8658ad4e73a186e747fcc5fc2a29f979d6fe6269127fdb5fd08298d0" +checksum = "b011eec8cc36da2aab2d5cff675ec18454fad408585853910a202391cf9f8e65" dependencies = [ "once_cell", "wasm-bindgen", @@ -5142,9 +5117,9 @@ dependencies = [ [[package]] name = "jsonrpsee" -version = "0.24.9" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37b26c20e2178756451cfeb0661fb74c47dd5988cb7e3939de7e9241fd604d42" +checksum = "3f3f48dc3e6b8bd21e15436c1ddd0bc22a6a54e8ec46fedd6adf3425f396ec6a" dependencies = [ "jsonrpsee-client-transport", "jsonrpsee-core", @@ -5160,9 +5135,9 @@ dependencies = [ [[package]] name = "jsonrpsee-client-transport" -version = "0.24.9" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bacb85abf4117092455e1573625e21b8f8ef4dec8aff13361140b2dc266cdff2" +checksum = "cf36eb27f8e13fa93dcb50ccb44c417e25b818cfa1a481b5470cd07b19c60b98" dependencies = [ "base64 0.22.1", "futures-channel", @@ -5175,7 +5150,7 @@ dependencies = [ "rustls-pki-types", "rustls-platform-verifier", "soketto", - "thiserror 1.0.69", + "thiserror 2.0.17", "tokio", "tokio-rustls", "tokio-util", @@ -5185,9 +5160,9 @@ dependencies = [ [[package]] name = "jsonrpsee-core" -version = "0.24.9" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "456196007ca3a14db478346f58c7238028d55ee15c1df15115596e411ff27925" +checksum = "316c96719901f05d1137f19ba598b5fe9c9bc39f4335f67f6be8613921946480" dependencies = [ "async-trait", "bytes", @@ -5199,24 +5174,24 @@ dependencies = [ "jsonrpsee-types", "parking_lot", "pin-project", - "rand 0.8.5", - "rustc-hash 2.1.1", + "rand 0.9.2", + "rustc-hash", "serde", "serde_json", - "thiserror 1.0.69", + "thiserror 2.0.17", "tokio", "tokio-stream", + "tower", "tracing", "wasm-bindgen-futures", ] [[package]] name = "jsonrpsee-http-client" -version = "0.24.9" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c872b6c9961a4ccc543e321bb5b89f6b2d2c7fe8b61906918273a3333c95400c" +checksum = "790bedefcec85321e007ff3af84b4e417540d5c87b3c9779b9e247d1bcc3dab8" dependencies = [ - "async-trait", "base64 0.22.1", "http-body", "hyper", @@ -5228,31 +5203,30 @@ dependencies = [ "rustls-platform-verifier", "serde", "serde_json", - "thiserror 1.0.69", + "thiserror 2.0.17", "tokio", - "tower 0.4.13", - "tracing", + "tower", "url", ] [[package]] name = "jsonrpsee-proc-macros" -version = "0.24.9" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e65763c942dfc9358146571911b0cd1c361c2d63e2d2305622d40d36376ca80" +checksum = "2da3f8ab5ce1bb124b6d082e62dffe997578ceaf0aeb9f3174a214589dc00f07" dependencies = [ "heck", "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] name = "jsonrpsee-server" -version = "0.24.9" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55e363146da18e50ad2b51a0a7925fc423137a0b1371af8235b1c231a0647328" +checksum = "4c51b7c290bb68ce3af2d029648148403863b982f138484a73f02a9dd52dbd7f" dependencies = [ "futures-util", "http", @@ -5267,47 +5241,49 @@ dependencies = [ "serde", "serde_json", "soketto", - "thiserror 1.0.69", + "thiserror 2.0.17", "tokio", "tokio-stream", "tokio-util", - "tower 0.4.13", + "tower", "tracing", ] [[package]] name = "jsonrpsee-types" -version = "0.24.9" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08a8e70baf945b6b5752fc8eb38c918a48f1234daf11355e07106d963f860089" +checksum = "bc88ff4688e43cc3fa9883a8a95c6fa27aa2e76c96e610b737b6554d650d7fd5" dependencies = [ "http", "serde", "serde_json", - "thiserror 1.0.69", + "thiserror 2.0.17", ] [[package]] name = "jsonrpsee-wasm-client" -version = "0.24.9" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6558a9586cad43019dafd0b6311d0938f46efc116b34b28c74778bc11a2edf6" +checksum = "7902885de4779f711a95d82c8da2d7e5f9f3a7c7cfa44d51c067fd1c29d72a3c" dependencies = [ "jsonrpsee-client-transport", "jsonrpsee-core", "jsonrpsee-types", + "tower", ] [[package]] name = "jsonrpsee-ws-client" -version = "0.24.9" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01b3323d890aa384f12148e8d2a1fd18eb66e9e7e825f9de4fa53bcc19b93eef" +checksum = "9b6fceceeb05301cc4c065ab3bd2fa990d41ff4eb44e4ca1b30fa99c057c3e79" dependencies = [ "http", "jsonrpsee-client-transport", "jsonrpsee-core", "jsonrpsee-types", + "tower", "url", ] @@ -5337,7 +5313,7 @@ dependencies = [ "elliptic-curve", "once_cell", "serdect", - "sha2 0.10.9", + "sha2", "signature", ] @@ -5388,9 +5364,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.175" +version = "0.2.177" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543" +checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" [[package]] name = "libgit2-sys" @@ -5406,12 +5382,12 @@ dependencies = [ [[package]] name = "libloading" -version = "0.8.8" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" dependencies = [ "cfg-if", - "windows-targets 0.53.3", + "windows-link 0.2.1", ] [[package]] @@ -5433,19 +5409,19 @@ dependencies = [ "k256", "multihash", "quick-protobuf", - "sha2 0.10.9", - "thiserror 2.0.16", + "sha2", + "thiserror 2.0.17", "tracing", "zeroize", ] [[package]] name = "libproc" -version = "0.14.10" +version = "0.14.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e78a09b56be5adbcad5aa1197371688dc6bb249a26da3bca2011ee2fb987ebfb" +checksum = "a54ad7278b8bc5301d5ffd2a94251c004feb971feba96c971ea4063645990757" dependencies = [ - "bindgen", + "bindgen 0.72.1", "errno", "libc", ] @@ -5456,57 +5432,11 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "416f7e718bdb06000964960ffa43b4335ad4012ae8b99060261aa4a8088d5ccb" dependencies = [ - "bitflags 2.9.4", + "bitflags 2.10.0", "libc", "redox_syscall", ] -[[package]] -name = "libsecp256k1" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e79019718125edc905a079a70cfa5f3820bc76139fc91d6f9abc27ea2a887139" -dependencies = [ - "arrayref", - "base64 0.22.1", - "digest 0.9.0", - "libsecp256k1-core", - "libsecp256k1-gen-ecmult", - "libsecp256k1-gen-genmult", - "rand 0.8.5", - "serde", - "sha2 0.9.9", -] - -[[package]] -name = "libsecp256k1-core" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5be9b9bb642d8522a44d533eab56c16c738301965504753b03ad1de3425d5451" -dependencies = [ - "crunchy", - "digest 0.9.0", - "subtle", -] - -[[package]] -name = "libsecp256k1-gen-ecmult" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3038c808c55c87e8a172643a7d87187fc6c4174468159cb3090659d55bcb4809" -dependencies = [ - "libsecp256k1-core", -] - -[[package]] -name = "libsecp256k1-gen-genmult" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3db8d6ba2cec9eacc40e6e8ccc98931840301f1006e95647ceb2dd5c3aa06f7c" -dependencies = [ - "libsecp256k1-core", -] - [[package]] name = "libz-sys" version = "1.1.22" @@ -5549,29 +5479,22 @@ checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" [[package]] name = "litemap" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856" - -[[package]] -name = "litemap" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" +checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" [[package]] name = "litrs" -version = "0.4.2" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5e54036fe321fd421e10d732f155734c4e4afd610dd556d9a82833ab3ee0bed" +checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" [[package]] name = "lock_api" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96936507f153605bddfcda068dd804796c84324ed2510809e5b2a624c81da765" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" dependencies = [ - "autocfg", "scopeguard", "serde", ] @@ -5646,9 +5569,9 @@ checksum = "08ab2867e3eeeca90e844d1940eab391c9dc5228783db2ed999acbc0a9ed375a" [[package]] name = "mach2" -version = "0.4.3" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d640282b302c0bb0a2a8e0233ead9035e3bed871f0b7e81fe4a1ec829765db44" +checksum = "6a1b95cd5421ec55b445b5ae102f5ea0e768de1f82bd3001e11f426c269c3aea" dependencies = [ "libc", ] @@ -5661,7 +5584,18 @@ checksum = "1b27834086c65ec3f9387b096d66e99f221cf081c2b738042aa252bcd41204e3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", +] + +[[package]] +name = "match-lookup" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1265724d8cb29dbbc2b0f06fffb8bf1a8c0cf73a78eede9ba73a4a66c52a981e" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", ] [[package]] @@ -5675,15 +5609,15 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.5" +version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" +checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" [[package]] name = "memmap2" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843a98750cd611cc2965a8213b53b43e715f13c37a9e096c6408e69990961db7" +checksum = "744133e4a0e0a658e1374cf3bf8e415c4052a15a111acd372764c55b4177d490" dependencies = [ "libc", ] @@ -5716,7 +5650,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -5726,7 +5660,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd7399781913e5393588a8d8c6a2867bf85fb38eaf2502fdce465aad2dc6f034" dependencies = [ "base64 0.22.1", - "indexmap 2.11.1", + "indexmap 2.12.0", "metrics", "metrics-util", "quanta", @@ -5735,18 +5669,18 @@ dependencies = [ [[package]] name = "metrics-process" -version = "2.4.0" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a82c8add4382f29a122fa64fff1891453ed0f6b2867d971e7d60cb8dfa322ff" +checksum = "f615e08e049bd14a44c4425415782efb9bcd479fc1e19ddeb971509074c060d0" dependencies = [ "libc", "libproc", "mach2", "metrics", "once_cell", - "procfs", + "procfs 0.18.0", "rlimit", - "windows 0.58.0", + "windows 0.62.2", ] [[package]] @@ -5758,7 +5692,7 @@ dependencies = [ "crossbeam-epoch", "crossbeam-utils", "hashbrown 0.15.5", - "indexmap 2.11.1", + "indexmap 2.12.0", "metrics", "ordered-float", "quanta", @@ -5773,7 +5707,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd9e517b6c1d1143b35b716ec1107a493b2ce1143a35cbb9788e81f69c6f574c" dependencies = [ - "alloy-rpc-types-mev 1.0.30", + "alloy-rpc-types-mev", "async-sse", "bytes", "futures-util", @@ -5782,7 +5716,7 @@ dependencies = [ "reqwest", "serde", "serde_json", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "tracing", ] @@ -5832,18 +5766,19 @@ checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" dependencies = [ "adler2", "serde", + "simd-adler32", ] [[package]] name = "mio" -version = "1.0.4" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" +checksum = "69d83b0086dc8ecf3ce9ae2874b2d1290252e2a30720bea58a5c6639b0092873" dependencies = [ "libc", "log", "wasi 0.11.1+wasi-snapshot-preview1", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -5869,20 +5804,19 @@ dependencies = [ [[package]] name = "moka" -version = "0.12.10" +version = "0.12.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9321642ca94a4282428e6ea4af8cc2ca4eac48ac7a6a4ea8f33f76d0ce70926" +checksum = "8261cd88c312e0004c1d51baad2980c66528dfdb2bee62003e643a4d8f86b077" dependencies = [ "crossbeam-channel", "crossbeam-epoch", "crossbeam-utils", - "loom", + "equivalent", "parking_lot", "portable-atomic", "rustc_version 0.4.1", "smallvec", "tagptr", - "thiserror 1.0.69", "uuid", ] @@ -5913,11 +5847,12 @@ dependencies = [ [[package]] name = "multibase" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b3539ec3c1f04ac9748a260728e855f261b4977f5c3406612c884564f329404" +checksum = "8694bb4835f452b0e3bb06dbebb1d6fc5385b6ca1caf2e55fd165c042390ec77" dependencies = [ "base-x", + "base256emoji", "data-encoding", "data-encoding-macro", ] @@ -5932,6 +5867,30 @@ dependencies = [ "unsigned-varint", ] +[[package]] +name = "nix" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" +dependencies = [ + "bitflags 2.10.0", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "nix" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" +dependencies = [ + "bitflags 2.10.0", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "nom" version = "7.1.3" @@ -5948,7 +5907,7 @@ version = "8.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d3d07927151ff8575b7087f245456e549fea62edf0ec4e565a5ee50c8402bc3" dependencies = [ - "bitflags 2.9.4", + "bitflags 2.10.0", "fsevent-sys", "inotify", "kqueue", @@ -5977,11 +5936,11 @@ dependencies = [ [[package]] name = "nu-ansi-term" -version = "0.50.1" +version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4a28e057d01f97e61255210fcff094d74ed0466038633e95017f5beb68e4399" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -6077,9 +6036,9 @@ dependencies = [ [[package]] name = "num_enum" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a973b4e44ce6cad84ce69d797acf9a044532e4184c4f267913d1b546a0727b7a" +checksum = "b1207a7e20ad57b847bbddc6776b968420d38292bbfe2089accff5e19e82454c" dependencies = [ "num_enum_derive", "rustversion", @@ -6087,14 +6046,14 @@ dependencies = [ [[package]] name = "num_enum_derive" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77e878c846a8abae00dd069496dbe8751b16ac1c3d6bd2a7283a938e8228f90d" +checksum = "ff32365de1b6743cb203b710788263c44a03de03802daf96092f2da4fe6ba4d7" dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -6108,38 +6067,33 @@ dependencies = [ [[package]] name = "nybbles" -version = "0.3.4" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8983bb634df7248924ee0c4c3a749609b5abcb082c28fffe3254b3eb3602b307" +checksum = "2c4b5ecbd0beec843101bffe848217f770e8b8da81d8355b7d6e226f2199b3dc" dependencies = [ "alloy-rlp", "arbitrary", - "const-hex", + "cfg-if", "proptest", + "ruint", "serde", "smallvec", ] [[package]] -name = "nybbles" -version = "0.4.4" +name = "objc2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0418987d1aaed324d95b4beffc93635e19be965ed5d63ec07a35980fe3b71a4" +checksum = "b7c2599ce0ec54857b29ce62166b0ed9b4f6f1a70ccc9a71165b6154caca8c05" dependencies = [ - "cfg-if", - "ruint", - "serde", - "smallvec", + "objc2-encode", ] [[package]] -name = "object" -version = "0.36.7" +name = "objc2-encode" +version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" -dependencies = [ - "memchr", -] +checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" [[package]] name = "once_cell" @@ -6153,9 +6107,9 @@ dependencies = [ [[package]] name = "once_cell_polyfill" -version = "1.70.1" +version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" [[package]] name = "oorandom" @@ -6165,21 +6119,21 @@ checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" [[package]] name = "op-alloy-consensus" -version = "0.15.4" -source = "git+https://github.com/mantle-xyz/op-alloy?tag=v2.0.1#66375676a7c10acf6ab521e776b2e8d1728f7225" +version = "0.22.1" +source = "git+https://github.com/mantle-xyz/op-alloy?tag=v2.1.0#2ec6d06b7445975bd45ed68ceb73520fbc67579d" dependencies = [ - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", + "alloy-consensus", + "alloy-eips", "alloy-network", "alloy-primitives", "alloy-rlp", - "alloy-rpc-types-eth 0.15.11", - "alloy-serde 0.15.11", + "alloy-rpc-types-eth", + "alloy-serde", "arbitrary", "derive_more", "serde", "serde_with", - "thiserror 2.0.16", + "thiserror 2.0.17", ] [[package]] @@ -6190,13 +6144,14 @@ checksum = "a79f352fc3893dcd670172e615afef993a41798a1d3fc0db88a3e60ef2e70ecc" [[package]] name = "op-alloy-network" -version = "0.15.4" -source = "git+https://github.com/mantle-xyz/op-alloy?tag=v2.0.1#66375676a7c10acf6ab521e776b2e8d1728f7225" +version = "0.22.1" +source = "git+https://github.com/mantle-xyz/op-alloy?tag=v2.1.0#2ec6d06b7445975bd45ed68ceb73520fbc67579d" dependencies = [ - "alloy-consensus 0.15.11", + "alloy-consensus", "alloy-network", "alloy-primitives", - "alloy-rpc-types-eth 0.15.11", + "alloy-provider", + "alloy-rpc-types-eth", "alloy-signer", "op-alloy-consensus", "op-alloy-rpc-types", @@ -6204,8 +6159,8 @@ dependencies = [ [[package]] name = "op-alloy-rpc-jsonrpsee" -version = "0.15.4" -source = "git+https://github.com/mantle-xyz/op-alloy?tag=v2.0.1#66375676a7c10acf6ab521e776b2e8d1728f7225" +version = "0.22.1" +source = "git+https://github.com/mantle-xyz/op-alloy?tag=v2.1.0#2ec6d06b7445975bd45ed68ceb73520fbc67579d" dependencies = [ "alloy-primitives", "jsonrpsee", @@ -6213,44 +6168,47 @@ dependencies = [ [[package]] name = "op-alloy-rpc-types" -version = "0.15.4" -source = "git+https://github.com/mantle-xyz/op-alloy?tag=v2.0.1#66375676a7c10acf6ab521e776b2e8d1728f7225" +version = "0.22.1" +source = "git+https://github.com/mantle-xyz/op-alloy?tag=v2.1.0#2ec6d06b7445975bd45ed68ceb73520fbc67579d" dependencies = [ - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", - "alloy-network-primitives 0.15.11", + "alloy-consensus", + "alloy-eips", + "alloy-network-primitives", "alloy-primitives", - "alloy-rpc-types-eth 0.15.11", - "alloy-serde 0.15.11", + "alloy-rpc-types-eth", + "alloy-serde", + "arbitrary", "derive_more", "op-alloy-consensus", "serde", "serde_json", + "thiserror 2.0.17", ] [[package]] name = "op-alloy-rpc-types-engine" -version = "0.15.4" -source = "git+https://github.com/mantle-xyz/op-alloy?tag=v2.0.1#66375676a7c10acf6ab521e776b2e8d1728f7225" +version = "0.22.1" +source = "git+https://github.com/mantle-xyz/op-alloy?tag=v2.1.0#2ec6d06b7445975bd45ed68ceb73520fbc67579d" dependencies = [ - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", + "alloy-consensus", + "alloy-eips", "alloy-primitives", "alloy-rlp", "alloy-rpc-types-engine", - "alloy-serde 0.15.11", + "alloy-serde", "arbitrary", "derive_more", "ethereum_ssz", + "ethereum_ssz_derive", "op-alloy-consensus", "serde", "snap", - "thiserror 2.0.16", + "thiserror 2.0.17", ] [[package]] name = "op-reth" -version = "1.3.12" +version = "1.9.3" dependencies = [ "clap", "reth-cli-util", @@ -6268,12 +6226,11 @@ dependencies = [ [[package]] name = "op-revm" -version = "4.0.2" -source = "git+https://github.com/mantle-xyz/revm?tag=v2.0.0#7412a0d6707ec96aa9ae9cb3e6cb8e975029f1f0" +version = "12.0.2" +source = "git+https://github.com/mantle-xyz/revm?tag=v2.1.2#ba33cb80cdced50ee342186fa5ade478b75f1050" dependencies = [ "alloy-sol-types", "auto_impl", - "once_cell", "revm", "serde", ] @@ -6290,6 +6247,86 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" +[[package]] +name = "opentelemetry" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b84bcd6ae87133e903af7ef497404dda70c60d0ea14895fc8a5e6722754fc2a0" +dependencies = [ + "futures-core", + "futures-sink", + "js-sys", + "pin-project-lite", + "thiserror 2.0.17", + "tracing", +] + +[[package]] +name = "opentelemetry-http" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7a6d09a73194e6b66df7c8f1b680f156d916a1a942abf2de06823dd02b7855d" +dependencies = [ + "async-trait", + "bytes", + "http", + "opentelemetry", + "reqwest", +] + +[[package]] +name = "opentelemetry-otlp" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2366db2dca4d2ad033cad11e6ee42844fd727007af5ad04a1730f4cb8163bf" +dependencies = [ + "http", + "opentelemetry", + "opentelemetry-http", + "opentelemetry-proto", + "opentelemetry_sdk", + "prost", + "reqwest", + "thiserror 2.0.17", + "tokio", + "tonic", + "tracing", +] + +[[package]] +name = "opentelemetry-proto" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7175df06de5eaee9909d4805a3d07e28bb752c34cab57fa9cff549da596b30f" +dependencies = [ + "opentelemetry", + "opentelemetry_sdk", + "prost", + "tonic", + "tonic-prost", +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e62e29dfe041afb8ed2a6c9737ab57db4907285d999ef8ad3a59092a36bdc846" + +[[package]] +name = "opentelemetry_sdk" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14ae4f5991976fd48df6d843de219ca6d31b01daaab2dad5af2badeded372bd" +dependencies = [ + "futures-channel", + "futures-executor", + "futures-util", + "opentelemetry", + "percent-encoding", + "rand 0.9.2", + "thiserror 2.0.17", +] + [[package]] name = "option-ext" version = "0.2.0" @@ -6314,7 +6351,7 @@ dependencies = [ "ecdsa", "elliptic-curve", "primeorder", - "sha2 0.10.9", + "sha2", ] [[package]] @@ -6329,17 +6366,19 @@ dependencies = [ [[package]] name = "parity-scale-codec" -version = "3.6.12" +version = "3.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "306800abfa29c7f16596b5970a588435e3d5b3149683d00c12b699cc19f895ee" +checksum = "799781ae679d79a948e13d4824a40970bfa500058d245760dd857301059810fa" dependencies = [ "arbitrary", "arrayvec", "bitvec", "byte-slice-cast", "bytes", + "const_format", "impl-trait-for-tuples", "parity-scale-codec-derive", + "rustversion", "serde", ] @@ -6352,7 +6391,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -6363,9 +6402,9 @@ checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" [[package]] name = "parking_lot" -version = "0.12.4" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70d58bf43669b5795d1576d0641cfb6fbb2057bf629506267a92807158584a13" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" dependencies = [ "lock_api", "parking_lot_core", @@ -6373,15 +6412,15 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.11" +version = "0.9.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ "cfg-if", "libc", "redox_syscall", "smallvec", - "windows-targets 0.52.6", + "windows-link 0.2.1", ] [[package]] @@ -6402,12 +6441,12 @@ dependencies = [ [[package]] name = "pem" -version = "3.0.5" +version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38af38e8470ac9dee3ce1bae1af9c1671fffc44ddfd8bd1d0a3445bf349a8ef3" +checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" dependencies = [ "base64 0.22.1", - "serde", + "serde_core", ] [[package]] @@ -6418,12 +6457,11 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pest" -version = "2.8.2" +version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21e0a3a33733faeaf8651dfee72dd0f388f0c8e5ad496a3478fa5a922f49cfa8" +checksum = "989e7521a040efde50c3ab6bbadafbe15ab6dc042686926be59ac35d74607df4" dependencies = [ "memchr", - "thiserror 2.0.16", "ucd-trie", ] @@ -6439,9 +6477,9 @@ dependencies = [ [[package]] name = "phf" -version = "0.11.3" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" dependencies = [ "phf_macros", "phf_shared", @@ -6450,32 +6488,32 @@ dependencies = [ [[package]] name = "phf_generator" -version = "0.11.3" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737" dependencies = [ + "fastrand 2.3.0", "phf_shared", - "rand 0.8.5", ] [[package]] name = "phf_macros" -version = "0.11.3" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" +checksum = "812f032b54b1e759ccd5f8b6677695d5268c588701effba24601f6932f8269ef" dependencies = [ "phf_generator", "phf_shared", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] name = "phf_shared" -version = "0.11.3" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" dependencies = [ "siphasher", ] @@ -6497,7 +6535,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -6565,12 +6603,6 @@ dependencies = [ "plotters-backend", ] -[[package]] -name = "pollster" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f3a9f18d041e6d0e102a0a46750538147e5e8992d3b4873aaafee2520b00ce3" - [[package]] name = "polyval" version = "0.6.2" @@ -6591,11 +6623,11 @@ checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" [[package]] name = "potential_utf" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84df19adbe5b5a0782edcab45899906947ab039ccf4573713735ee7de1e6b08a" +checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" dependencies = [ - "zerovec 0.11.4", + "zerovec", ] [[package]] @@ -6630,7 +6662,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -6659,7 +6691,7 @@ version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "219cb19e96be00ab2e37d6e299658a0cfa83e52429179969b0f0121b4ac46983" dependencies = [ - "toml_edit 0.23.4", + "toml_edit 0.23.7", ] [[package]] @@ -6681,14 +6713,14 @@ dependencies = [ "proc-macro-error-attr2", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] name = "proc-macro2" -version = "1.0.101" +version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de" +checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8" dependencies = [ "unicode-ident", ] @@ -6699,35 +6731,55 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc5b72d8145275d844d4b5f6d4e1eef00c8cd889edb6035c21675d1bb1f45c9f" dependencies = [ - "bitflags 2.9.4", + "bitflags 2.10.0", "chrono", "flate2", "hex", - "procfs-core", + "procfs-core 0.17.0", "rustix 0.38.44", ] +[[package]] +name = "procfs" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25485360a54d6861439d60facef26de713b1e126bf015ec8f98239467a2b82f7" +dependencies = [ + "bitflags 2.10.0", + "procfs-core 0.18.0", + "rustix 1.1.2", +] + [[package]] name = "procfs-core" version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "239df02d8349b06fc07398a3a1697b06418223b1c7725085e801e7c0fc6a12ec" dependencies = [ - "bitflags 2.9.4", + "bitflags 2.10.0", "chrono", "hex", ] +[[package]] +name = "procfs-core" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6401bf7b6af22f78b563665d15a22e9aef27775b79b149a66ca022468a4e405" +dependencies = [ + "bitflags 2.10.0", + "hex", +] + [[package]] name = "proptest" -version = "1.7.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fcdab19deb5195a31cf7726a210015ff1496ba1464fd42cb4f537b8b01b471f" +checksum = "bee689443a2bd0a16ab0348b52ee43e3b2d1b1f931c8aa5c9f8de4c86fbe8c40" dependencies = [ "bit-set", "bit-vec", - "bitflags 2.9.4", - "lazy_static", + "bitflags 2.10.0", "num-traits", "rand 0.9.2", "rand_chacha 0.9.0", @@ -6756,7 +6808,41 @@ checksum = "4ee1c9ac207483d5e7db4940700de86a9aae46ef90c48b57f99fe7edb8345e49" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", +] + +[[package]] +name = "proptest-derive" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "095a99f75c69734802359b682be8daaf8980296731f6470434ea2c652af1dd30" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.108", +] + +[[package]] +name = "prost" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7231bd9b3d3d33c86b58adbac74b5ec0ad9f496b19d22801d773636feaa95f3d" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9120690fafc389a67ba3803df527d0ec9cbbc9cc45e4cc20b332996dfb672425" +dependencies = [ + "anyhow", + "itertools 0.14.0", + "proc-macro2", + "quote", + "syn 2.0.108", ] [[package]] @@ -6765,7 +6851,7 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57206b407293d2bcd3af849ce869d52068623f19e1b5ff8e8778e3309439682b" dependencies = [ - "bitflags 2.9.4", + "bitflags 2.10.0", "memchr", "unicase", ] @@ -6811,10 +6897,10 @@ dependencies = [ "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash 2.1.1", + "rustc-hash", "rustls", - "socket2 0.6.0", - "thiserror 2.0.16", + "socket2 0.6.1", + "thiserror 2.0.17", "tokio", "tracing", "web-time", @@ -6827,15 +6913,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" dependencies = [ "bytes", - "getrandom 0.3.3", + "getrandom 0.3.4", "lru-slab", "rand 0.9.2", "ring", - "rustc-hash 2.1.1", + "rustc-hash", "rustls", "rustls-pki-types", "slab", - "thiserror 2.0.16", + "thiserror 2.0.17", "tinyvec", "tracing", "web-time", @@ -6850,16 +6936,16 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.0", + "socket2 0.6.1", "tracing", "windows-sys 0.60.2", ] [[package]] name = "quote" -version = "1.0.40" +version = "1.0.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +checksum = "ce25767e7b499d1b604768e7cde645d14cc8584231ea6b295e9c9eb22c02e1d1" dependencies = [ "proc-macro2", ] @@ -6966,7 +7052,7 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" dependencies = [ - "getrandom 0.3.3", + "getrandom 0.3.4", "serde", ] @@ -7003,7 +7089,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eabd94c2f37801c20583fc49dd5cd6b0ba68c716787c2dd6ed18571e1e63117b" dependencies = [ - "bitflags 2.9.4", + "bitflags 2.10.0", "cassowary", "compact_str", "crossterm 0.28.1", @@ -7024,7 +7110,7 @@ version = "11.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" dependencies = [ - "bitflags 2.9.4", + "bitflags 2.10.0", ] [[package]] @@ -7055,11 +7141,11 @@ checksum = "d3edd4d5d42c92f0a659926464d4cce56b562761267ecf0f469d85b7de384175" [[package]] name = "redox_syscall" -version = "0.5.17" +version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5407465600fb0548f1442edf71dd20683c6ed326200ace4b1ef0763521bb3b77" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.9.4", + "bitflags 2.10.0", ] [[package]] @@ -7081,34 +7167,34 @@ checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" dependencies = [ "getrandom 0.2.16", "libredox", - "thiserror 2.0.16", + "thiserror 2.0.17", ] [[package]] name = "ref-cast" -version = "1.0.24" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a0ae411dbe946a674d89546582cea4ba2bb8defac896622d6496f14c23ba5cf" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" dependencies = [ "ref-cast-impl", ] [[package]] name = "ref-cast-impl" -version = "1.0.24" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1165225c21bff1f3bbce98f5a1f889949bc902d3575308cc7b0de30b4f6d27c7" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] name = "regex" -version = "1.11.2" +version = "1.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23d7fd106d8c02486a8d64e778353d1cffe08ce79ac2e82f540c86d0facf6912" +checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" dependencies = [ "aho-corasick", "memchr", @@ -7118,9 +7204,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.10" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b9458fa0bfeeac22b5ca447c63aaf45f28439a709ccd244698632f9aa6394d6" +checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" dependencies = [ "aho-corasick", "memchr", @@ -7129,17 +7215,17 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.6" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caf4aa5b0f434c91fe5c7f1ecb6a5ece2130b02ad2a590589dda5146df959001" +checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" [[package]] name = "regress" -version = "0.10.4" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "145bb27393fe455dd64d6cbc8d059adfa392590a45eadf079c01b11857e7b010" +checksum = "2057b2325e68a893284d1538021ab90279adac1139957ca2a74426c6f118fb48" dependencies = [ - "hashbrown 0.15.5", + "hashbrown 0.16.0", "memchr", ] @@ -7151,9 +7237,9 @@ checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" [[package]] name = "reqwest" -version = "0.12.23" +version = "0.12.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d429f34c8092b2d42c7c93cec323bb4adeb7c67698f70839adec842ec10c7ceb" +checksum = "9d0946410b9f7b082a427e4ef5c8ff541a88b357bc6c637c40db3a68ac70a36f" dependencies = [ "base64 0.22.1", "bytes", @@ -7181,7 +7267,7 @@ dependencies = [ "tokio", "tokio-rustls", "tokio-util", - "tower 0.5.2", + "tower", "tower-http", "tower-service", "url", @@ -7189,7 +7275,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots 1.0.2", + "webpki-roots 1.0.4", ] [[package]] @@ -7200,7 +7286,7 @@ checksum = "6b3789b30bd25ba102de4beabd95d21ac45b69b1be7d14522bab988c526d6799" [[package]] name = "reth" -version = "1.3.12" +version = "1.9.3" dependencies = [ "alloy-rpc-types", "aquamarine", @@ -7234,13 +7320,12 @@ dependencies = [ "reth-rpc", "reth-rpc-api", "reth-rpc-builder", + "reth-rpc-convert", "reth-rpc-eth-types", "reth-rpc-server-types", - "reth-rpc-types-compat", "reth-tasks", "reth-tokio-util", "reth-transaction-pool", - "similar-asserts", "tempfile", "tokio", "tracing", @@ -7248,10 +7333,10 @@ dependencies = [ [[package]] name = "reth-basic-payload-builder" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", + "alloy-consensus", + "alloy-eips", "alloy-primitives", "futures-core", "futures-util", @@ -7271,9 +7356,9 @@ dependencies = [ [[package]] name = "reth-bench" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-eips 0.15.11", + "alloy-eips", "alloy-json-rpc", "alloy-primitives", "alloy-provider", @@ -7291,6 +7376,7 @@ dependencies = [ "futures", "humantime", "op-alloy-consensus", + "op-alloy-rpc-types-engine", "reqwest", "reth-cli-runner", "reth-cli-util", @@ -7301,21 +7387,48 @@ dependencies = [ "reth-tracing", "serde", "serde_json", - "thiserror 2.0.16", + "thiserror 2.0.17", + "tokio", + "tower", + "tracing", +] + +[[package]] +name = "reth-bench-compare" +version = "1.9.3" +dependencies = [ + "alloy-primitives", + "alloy-provider", + "alloy-rpc-types-eth", + "chrono", + "clap", + "csv", + "ctrlc", + "eyre", + "nix 0.29.0", + "reth-chainspec", + "reth-cli-runner", + "reth-cli-util", + "reth-node-core", + "reth-tracing", + "serde", + "serde_json", + "shellexpand", + "shlex", "tokio", - "tower 0.4.13", "tracing", ] [[package]] name = "reth-chain-state" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", + "alloy-consensus", + "alloy-eips", "alloy-primitives", "alloy-signer", "alloy-signer-local", + "codspeed-criterion-compat", "derive_more", "metrics", "parking_lot", @@ -7332,6 +7445,7 @@ dependencies = [ "reth-trie", "revm-database", "revm-state", + "serde", "tokio", "tokio-stream", "tracing", @@ -7339,16 +7453,16 @@ dependencies = [ [[package]] name = "reth-chainspec" -version = "1.3.12" +version = "1.9.3" dependencies = [ "alloy-chains", - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", + "alloy-consensus", + "alloy-eips", "alloy-evm", "alloy-genesis", "alloy-primitives", "alloy-rlp", - "alloy-trie 0.8.1", + "alloy-trie", "auto_impl", "derive_more", "reth-ethereum-forks", @@ -7359,7 +7473,7 @@ dependencies = [ [[package]] name = "reth-cli" -version = "1.3.12" +version = "1.9.3" dependencies = [ "alloy-genesis", "clap", @@ -7372,12 +7486,11 @@ dependencies = [ [[package]] name = "reth-cli-commands" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "ahash", "alloy-chains", - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", + "alloy-consensus", + "alloy-eips", "alloy-primitives", "alloy-rlp", "arbitrary", @@ -7389,6 +7502,7 @@ dependencies = [ "fdlimit", "futures", "human_bytes", + "humantime", "itertools 0.14.0", "lz4", "proptest", @@ -7409,6 +7523,7 @@ dependencies = [ "reth-discv5", "reth-downloaders", "reth-ecies", + "reth-era", "reth-era-downloader", "reth-era-utils", "reth-eth-wire", @@ -7431,6 +7546,7 @@ dependencies = [ "reth-provider", "reth-prune", "reth-prune-types", + "reth-revm", "reth-stages", "reth-stages-types", "reth-static-file", @@ -7438,19 +7554,21 @@ dependencies = [ "reth-trie", "reth-trie-common", "reth-trie-db", - "secp256k1", + "secp256k1 0.30.0", "serde", "serde_json", "tar", + "tempfile", "tokio", "tokio-stream", "toml", "tracing", + "zstd", ] [[package]] name = "reth-cli-runner" -version = "1.3.12" +version = "1.9.3" dependencies = [ "reth-tasks", "tokio", @@ -7459,9 +7577,9 @@ dependencies = [ [[package]] name = "reth-cli-util" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-eips 0.15.11", + "alloy-eips", "alloy-primitives", "cfg-if", "eyre", @@ -7469,23 +7587,23 @@ dependencies = [ "rand 0.8.5", "rand 0.9.2", "reth-fs-util", - "secp256k1", + "secp256k1 0.30.0", "serde", "snmalloc-rs", - "thiserror 2.0.16", + "thiserror 2.0.17", "tikv-jemallocator", "tracy-client", ] [[package]] name = "reth-codecs" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", + "alloy-consensus", + "alloy-eips", "alloy-genesis", "alloy-primitives", - "alloy-trie 0.8.1", + "alloy-trie", "arbitrary", "bytes", "modular-bitfield", @@ -7503,18 +7621,17 @@ dependencies = [ [[package]] name = "reth-codecs-derive" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "convert_case", "proc-macro2", "quote", "similar-asserts", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] name = "reth-config" -version = "1.3.12" +version = "1.9.3" dependencies = [ "alloy-primitives", "eyre", @@ -7526,26 +7643,27 @@ dependencies = [ "serde", "tempfile", "toml", + "url", ] [[package]] name = "reth-consensus" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", + "alloy-consensus", "alloy-primitives", "auto_impl", "reth-execution-types", "reth-primitives-traits", - "thiserror 2.0.16", + "thiserror 2.0.17", ] [[package]] name = "reth-consensus-common" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", + "alloy-consensus", + "alloy-eips", "alloy-primitives", "rand 0.9.2", "reth-chainspec", @@ -7556,14 +7674,15 @@ dependencies = [ [[package]] name = "reth-consensus-debug-client" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", + "alloy-consensus", + "alloy-eips", "alloy-json-rpc", "alloy-primitives", "alloy-provider", "alloy-rpc-types-engine", + "alloy-transport", "auto_impl", "derive_more", "eyre", @@ -7574,14 +7693,15 @@ dependencies = [ "reth-tracing", "ringbuffer", "serde", + "serde_json", "tokio", ] [[package]] name = "reth-db" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", + "alloy-consensus", "alloy-primitives", "arbitrary", "assert_matches", @@ -7598,23 +7718,24 @@ dependencies = [ "reth-metrics", "reth-nippy-jar", "reth-primitives-traits", + "reth-prune-types", "reth-static-file-types", "reth-storage-errors", "reth-tracing", - "rustc-hash 2.1.1", + "rustc-hash", "serde", "serde_json", "strum 0.27.2", "sysinfo", "tempfile", - "thiserror 2.0.16", + "thiserror 2.0.17", ] [[package]] name = "reth-db-api" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", + "alloy-consensus", "alloy-genesis", "alloy-primitives", "arbitrary", @@ -7642,9 +7763,9 @@ dependencies = [ [[package]] name = "reth-db-common" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", + "alloy-consensus", "alloy-genesis", "alloy-primitives", "boyer-moore-magiclen", @@ -7655,7 +7776,9 @@ dependencies = [ "reth-db", "reth-db-api", "reth-etl", + "reth-execution-errors", "reth-fs-util", + "reth-mantle-forks", "reth-node-types", "reth-primitives-traits", "reth-provider", @@ -7663,17 +7786,18 @@ dependencies = [ "reth-static-file-types", "reth-trie", "reth-trie-db", + "revm", "serde", "serde_json", - "thiserror 2.0.16", + "thiserror 2.0.17", "tracing", ] [[package]] name = "reth-db-models" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-eips 0.15.11", + "alloy-eips", "alloy-primitives", "arbitrary", "bytes", @@ -7683,19 +7807,17 @@ dependencies = [ "reth-codecs", "reth-primitives-traits", "serde", - "test-fuzz", ] [[package]] name = "reth-discv4" -version = "1.3.12" +version = "1.9.3" dependencies = [ "alloy-primitives", "alloy-rlp", "assert_matches", "discv5", "enr", - "generic-array", "itertools 0.14.0", "parking_lot", "rand 0.8.5", @@ -7705,9 +7827,9 @@ dependencies = [ "reth-network-peers", "reth-tracing", "schnellru", - "secp256k1", + "secp256k1 0.30.0", "serde", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "tokio-stream", "tracing", @@ -7715,7 +7837,7 @@ dependencies = [ [[package]] name = "reth-discv5" -version = "1.3.12" +version = "1.9.3" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -7732,15 +7854,15 @@ dependencies = [ "reth-metrics", "reth-network-peers", "reth-tracing", - "secp256k1", - "thiserror 2.0.16", + "secp256k1 0.30.0", + "thiserror 2.0.17", "tokio", "tracing", ] [[package]] name = "reth-dns-discovery" -version = "1.3.12" +version = "1.9.3" dependencies = [ "alloy-chains", "alloy-primitives", @@ -7757,10 +7879,10 @@ dependencies = [ "reth-tokio-util", "reth-tracing", "schnellru", - "secp256k1", + "secp256k1 0.30.0", "serde", "serde_with", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "tokio-stream", "tracing", @@ -7768,13 +7890,14 @@ dependencies = [ [[package]] name = "reth-downloaders" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", + "alloy-consensus", + "alloy-eips", "alloy-primitives", "alloy-rlp", "assert_matches", + "async-compression", "futures", "futures-util", "itertools 0.14.0", @@ -7785,8 +7908,6 @@ dependencies = [ "reth-chainspec", "reth-config", "reth-consensus", - "reth-db", - "reth-db-api", "reth-ethereum-primitives", "reth-metrics", "reth-network-p2p", @@ -7798,7 +7919,7 @@ dependencies = [ "reth-testing-utils", "reth-tracing", "tempfile", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "tokio-stream", "tokio-util", @@ -7807,14 +7928,16 @@ dependencies = [ [[package]] name = "reth-e2e-test-utils" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", + "alloy-consensus", + "alloy-eips", "alloy-network", "alloy-primitives", + "alloy-provider", + "alloy-rlp", "alloy-rpc-types-engine", - "alloy-rpc-types-eth 0.15.11", + "alloy-rpc-types-eth", "alloy-signer", "alloy-signer-local", "derive_more", @@ -7822,10 +7945,16 @@ dependencies = [ "futures-util", "jsonrpsee", "reth-chainspec", + "reth-cli-commands", + "reth-config", + "reth-consensus", "reth-db", + "reth-db-common", "reth-engine-local", + "reth-engine-primitives", "reth-ethereum-primitives", "reth-network-api", + "reth-network-p2p", "reth-network-peers", "reth-node-api", "reth-node-builder", @@ -7834,10 +7963,12 @@ dependencies = [ "reth-payload-builder", "reth-payload-builder-primitives", "reth-payload-primitives", + "reth-primitives", + "reth-primitives-traits", "reth-provider", "reth-rpc-api", + "reth-rpc-builder", "reth-rpc-eth-api", - "reth-rpc-layer", "reth-rpc-server-types", "reth-stages-types", "reth-tasks", @@ -7845,6 +7976,7 @@ dependencies = [ "reth-tracing", "revm", "serde_json", + "tempfile", "tokio", "tokio-stream", "tracing", @@ -7853,7 +7985,7 @@ dependencies = [ [[package]] name = "reth-ecies" -version = "1.3.12" +version = "1.9.3" dependencies = [ "aes", "alloy-primitives", @@ -7870,10 +8002,10 @@ dependencies = [ "pin-project", "rand 0.8.5", "reth-network-peers", - "secp256k1", - "sha2 0.10.9", + "secp256k1 0.30.0", + "sha2", "sha3", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "tokio-stream", "tokio-util", @@ -7883,28 +8015,21 @@ dependencies = [ [[package]] name = "reth-engine-local" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", + "alloy-consensus", "alloy-primitives", "alloy-rpc-types-engine", "eyre", "futures-util", "op-alloy-rpc-types-engine", "reth-chainspec", - "reth-consensus", "reth-engine-primitives", - "reth-engine-service", - "reth-engine-tree", "reth-ethereum-engine-primitives", - "reth-evm", - "reth-node-types", "reth-optimism-chainspec", "reth-payload-builder", "reth-payload-primitives", - "reth-provider", - "reth-prune", - "reth-stages-api", + "reth-storage-api", "reth-transaction-pool", "tokio", "tokio-stream", @@ -7913,9 +8038,10 @@ dependencies = [ [[package]] name = "reth-engine-primitives" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", + "alloy-consensus", + "alloy-eips", "alloy-primitives", "alloy-rpc-types-engine", "auto_impl", @@ -7923,20 +8049,20 @@ dependencies = [ "reth-chain-state", "reth-errors", "reth-ethereum-primitives", + "reth-evm", "reth-execution-types", "reth-payload-builder-primitives", "reth-payload-primitives", "reth-primitives-traits", - "reth-trie", "reth-trie-common", "serde", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", ] [[package]] name = "reth-engine-service" -version = "1.3.12" +version = "1.9.3" dependencies = [ "futures", "pin-project", @@ -7959,17 +8085,16 @@ dependencies = [ "reth-prune", "reth-stages-api", "reth-tasks", - "thiserror 2.0.16", "tokio", "tokio-stream", ] [[package]] name = "reth-engine-tree" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", + "alloy-consensus", + "alloy-eips", "alloy-evm", "alloy-primitives", "alloy-rlp", @@ -7977,10 +8102,12 @@ dependencies = [ "assert_matches", "codspeed-criterion-compat", "crossbeam-channel", + "dashmap 6.1.0", "derive_more", + "eyre", "futures", - "itertools 0.14.0", "metrics", + "metrics-util", "mini-moka", "parking_lot", "proptest", @@ -7992,6 +8119,7 @@ dependencies = [ "reth-consensus", "reth-db", "reth-db-common", + "reth-e2e-test-utils", "reth-engine-primitives", "reth-errors", "reth-ethereum-consensus", @@ -7999,7 +8127,9 @@ dependencies = [ "reth-ethereum-primitives", "reth-evm", "reth-evm-ethereum", + "reth-execution-types", "reth-exex-types", + "reth-mantle-forks", "reth-metrics", "reth-network-p2p", "reth-node-ethereum", @@ -8010,7 +8140,6 @@ dependencies = [ "reth-prune", "reth-prune-types", "reth-revm", - "reth-rpc-types-compat", "reth-stages", "reth-stages-api", "reth-static-file", @@ -8018,22 +8147,25 @@ dependencies = [ "reth-testing-utils", "reth-tracing", "reth-trie", - "reth-trie-db", "reth-trie-parallel", "reth-trie-sparse", + "reth-trie-sparse-parallel", + "revm", "revm-primitives", "revm-state", "schnellru", - "thiserror 2.0.16", + "serde_json", + "smallvec", + "thiserror 2.0.17", "tokio", "tracing", ] [[package]] name = "reth-engine-util" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", + "alloy-consensus", "alloy-rpc-types-engine", "eyre", "futures", @@ -8041,6 +8173,7 @@ dependencies = [ "pin-project", "reth-chainspec", "reth-engine-primitives", + "reth-engine-tree", "reth-errors", "reth-evm", "reth-fs-util", @@ -8057,10 +8190,10 @@ dependencies = [ [[package]] name = "reth-era" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", + "alloy-consensus", + "alloy-eips", "alloy-primitives", "alloy-rlp", "ethereum_ssz", @@ -8073,13 +8206,13 @@ dependencies = [ "snap", "tempfile", "test-case", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", ] [[package]] name = "reth-era-downloader" -version = "1.3.12" +version = "1.9.3" dependencies = [ "alloy-primitives", "bytes", @@ -8088,7 +8221,7 @@ dependencies = [ "futures-util", "reqwest", "reth-fs-util", - "sha2 0.10.9", + "sha2", "tempfile", "test-case", "tokio", @@ -8096,12 +8229,12 @@ dependencies = [ [[package]] name = "reth-era-utils" -version = "1.3.12" +version = "1.9.3" dependencies = [ + "alloy-consensus", "alloy-primitives", "bytes", "eyre", - "futures", "futures-util", "reqwest", "reth-db-api", @@ -8112,29 +8245,31 @@ dependencies = [ "reth-fs-util", "reth-primitives-traits", "reth-provider", + "reth-stages-types", "reth-storage-api", "tempfile", "tokio", + "tokio-util", "tracing", ] [[package]] name = "reth-errors" -version = "1.3.12" +version = "1.9.3" dependencies = [ "reth-consensus", "reth-execution-errors", "reth-storage-errors", - "thiserror 2.0.16", + "thiserror 2.0.17", ] [[package]] name = "reth-eth-wire" -version = "1.3.12" +version = "1.9.3" dependencies = [ "alloy-chains", - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", + "alloy-consensus", + "alloy-eips", "alloy-primitives", "alloy-rlp", "arbitrary", @@ -8155,11 +8290,11 @@ dependencies = [ "reth-network-peers", "reth-primitives-traits", "reth-tracing", - "secp256k1", + "secp256k1 0.30.0", "serde", "snap", "test-fuzz", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "tokio-stream", "tokio-util", @@ -8168,11 +8303,11 @@ dependencies = [ [[package]] name = "reth-eth-wire-types" -version = "1.3.12" +version = "1.9.3" dependencies = [ "alloy-chains", - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", + "alloy-consensus", + "alloy-eips", "alloy-genesis", "alloy-hardforks", "alloy-primitives", @@ -8188,18 +8323,22 @@ dependencies = [ "reth-ethereum-primitives", "reth-primitives-traits", "serde", - "thiserror 2.0.16", + "thiserror 2.0.17", ] [[package]] name = "reth-ethereum" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-rpc-types-eth 0.15.11", + "alloy-rpc-types-engine", + "alloy-rpc-types-eth", "reth-chainspec", + "reth-cli-util", + "reth-codecs", "reth-consensus", "reth-consensus-common", "reth-db", + "reth-engine-local", "reth-eth-wire", "reth-ethereum-cli", "reth-ethereum-consensus", @@ -8210,6 +8349,7 @@ dependencies = [ "reth-network", "reth-network-api", "reth-node-api", + "reth-node-builder", "reth-node-core", "reth-node-ethereum", "reth-primitives-traits", @@ -8220,75 +8360,42 @@ dependencies = [ "reth-rpc-builder", "reth-rpc-eth-types", "reth-storage-api", + "reth-tasks", "reth-transaction-pool", "reth-trie", + "reth-trie-db", ] [[package]] name = "reth-ethereum-cli" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", - "alloy-primitives", - "alloy-rlp", - "alloy-rpc-types", - "backon", "clap", "eyre", - "futures", - "reth-basic-payload-builder", "reth-chainspec", "reth-cli", "reth-cli-commands", "reth-cli-runner", - "reth-cli-util", - "reth-config", - "reth-consensus", "reth-db", - "reth-db-api", - "reth-downloaders", - "reth-errors", - "reth-ethereum-payload-builder", - "reth-ethereum-primitives", - "reth-evm", - "reth-execution-types", - "reth-exex", - "reth-fs-util", - "reth-network", - "reth-network-api", - "reth-network-p2p", "reth-node-api", "reth-node-builder", "reth-node-core", "reth-node-ethereum", - "reth-node-events", "reth-node-metrics", - "reth-payload-builder", - "reth-primitives-traits", - "reth-provider", - "reth-prune", - "reth-revm", - "reth-stages", - "reth-static-file", - "reth-tasks", + "reth-rpc-server-types", "reth-tracing", - "reth-transaction-pool", - "reth-trie", - "reth-trie-db", - "serde_json", - "similar-asserts", + "reth-tracing-otlp", "tempfile", - "tokio", "tracing", + "url", ] [[package]] name = "reth-ethereum-consensus" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", + "alloy-consensus", + "alloy-eips", "alloy-primitives", "reth-chainspec", "reth-consensus", @@ -8301,9 +8408,9 @@ dependencies = [ [[package]] name = "reth-ethereum-engine-primitives" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-eips 0.15.11", + "alloy-eips", "alloy-primitives", "alloy-rlp", "alloy-rpc-types-engine", @@ -8313,12 +8420,13 @@ dependencies = [ "reth-primitives-traits", "serde", "serde_json", - "sha2 0.10.9", + "sha2", + "thiserror 2.0.17", ] [[package]] name = "reth-ethereum-forks" -version = "1.3.12" +version = "1.9.3" dependencies = [ "alloy-eip2124", "alloy-hardforks", @@ -8326,19 +8434,21 @@ dependencies = [ "arbitrary", "auto_impl", "once_cell", - "rustc-hash 2.1.1", + "rustc-hash", ] [[package]] name = "reth-ethereum-payload-builder" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", + "alloy-consensus", + "alloy-eips", "alloy-primitives", + "alloy-rlp", "alloy-rpc-types-engine", "reth-basic-payload-builder", "reth-chainspec", + "reth-consensus-common", "reth-errors", "reth-ethereum-primitives", "reth-evm", @@ -8357,12 +8467,14 @@ dependencies = [ [[package]] name = "reth-ethereum-primitives" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", + "alloy-consensus", + "alloy-eips", "alloy-primitives", "alloy-rlp", + "alloy-rpc-types-eth", + "alloy-serde", "arbitrary", "bincode 1.3.3", "derive_more", @@ -8374,15 +8486,15 @@ dependencies = [ "reth-codecs", "reth-primitives-traits", "reth-zstd-compressors", - "secp256k1", + "secp256k1 0.30.0", "serde", + "serde_json", "serde_with", - "test-fuzz", ] [[package]] name = "reth-etl" -version = "1.3.12" +version = "1.9.3" dependencies = [ "alloy-primitives", "rayon", @@ -8392,18 +8504,16 @@ dependencies = [ [[package]] name = "reth-evm" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", + "alloy-consensus", + "alloy-eips", "alloy-evm", "alloy-primitives", "auto_impl", "derive_more", "futures-util", "metrics", - "metrics-util", - "op-revm", "reth-ethereum-forks", "reth-ethereum-primitives", "reth-execution-errors", @@ -8418,13 +8528,14 @@ dependencies = [ [[package]] name = "reth-evm-ethereum" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", + "alloy-consensus", + "alloy-eips", "alloy-evm", "alloy-genesis", "alloy-primitives", + "alloy-rpc-types-engine", "derive_more", "parking_lot", "reth-chainspec", @@ -8433,29 +8544,30 @@ dependencies = [ "reth-evm", "reth-execution-types", "reth-primitives-traits", + "reth-storage-errors", "reth-testing-utils", "revm", - "secp256k1", + "secp256k1 0.30.0", ] [[package]] name = "reth-execution-errors" -version = "1.3.12" +version = "1.9.3" dependencies = [ "alloy-evm", "alloy-primitives", "alloy-rlp", - "nybbles 0.3.4", + "nybbles", "reth-storage-errors", - "thiserror 2.0.16", + "thiserror 2.0.17", ] [[package]] name = "reth-execution-types" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", + "alloy-consensus", + "alloy-eips", "alloy-evm", "alloy-primitives", "arbitrary", @@ -8472,10 +8584,10 @@ dependencies = [ [[package]] name = "reth-exex" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", + "alloy-consensus", + "alloy-eips", "alloy-genesis", "alloy-primitives", "eyre", @@ -8506,9 +8618,9 @@ dependencies = [ "reth-testing-utils", "reth-tracing", "rmp-serde", - "secp256k1", + "secp256k1 0.30.0", "tempfile", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "tokio-util", "tracing", @@ -8516,9 +8628,9 @@ dependencies = [ [[package]] name = "reth-exex-test-utils" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-eips 0.15.11", + "alloy-eips", "eyre", "futures-util", "reth-chainspec", @@ -8540,17 +8652,16 @@ dependencies = [ "reth-provider", "reth-tasks", "reth-transaction-pool", - "reth-trie-db", "tempfile", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", ] [[package]] name = "reth-exex-types" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-eips 0.15.11", + "alloy-eips", "alloy-primitives", "arbitrary", "bincode 1.3.3", @@ -8565,18 +8676,19 @@ dependencies = [ [[package]] name = "reth-fs-util" -version = "1.3.12" +version = "1.9.3" dependencies = [ "serde", "serde_json", - "thiserror 2.0.16", + "thiserror 2.0.17", ] [[package]] name = "reth-invalid-block-hooks" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", + "alloy-consensus", + "alloy-eips", "alloy-primitives", "alloy-rlp", "alloy-rpc-types-debug", @@ -8586,24 +8698,28 @@ dependencies = [ "pretty_assertions", "reth-chainspec", "reth-engine-primitives", + "reth-ethereum-primitives", "reth-evm", + "reth-evm-ethereum", "reth-primitives-traits", "reth-provider", "reth-revm", "reth-rpc-api", + "reth-testing-utils", "reth-tracing", "reth-trie", + "revm", "revm-bytecode", "revm-database", "serde", "serde_json", + "tempfile", ] [[package]] name = "reth-ipc" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "async-trait", "bytes", "futures", "futures-util", @@ -8614,55 +8730,66 @@ dependencies = [ "reth-tracing", "serde", "serde_json", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "tokio-stream", "tokio-util", - "tower 0.4.13", + "tower", "tracing", ] [[package]] name = "reth-libmdbx" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "bitflags 2.9.4", + "bitflags 2.10.0", "byteorder", "codspeed-criterion-compat", "dashmap 6.1.0", "derive_more", - "indexmap 2.11.1", "parking_lot", "rand 0.9.2", "reth-mdbx-sys", "smallvec", "tempfile", - "thiserror 2.0.16", + "thiserror 2.0.17", "tracing", ] [[package]] name = "reth-mantle-forks" -version = "1.3.12" +version = "1.9.3" dependencies = [ "alloy-chains", "alloy-hardforks", + "alloy-op-evm", + "alloy-primitives", "auto_impl", + "eyre", + "op-revm", + "reth-db", + "reth-db-api", "reth-optimism-forks", + "reth-primitives-traits", + "reth-provider", + "reth-trie", + "reth-trie-db", + "revm", "serde", + "tracing", ] [[package]] name = "reth-mdbx-sys" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "bindgen", + "bindgen 0.71.1", "cc", ] [[package]] name = "reth-metrics" -version = "1.3.12" +version = "1.9.3" dependencies = [ "futures", "metrics", @@ -8673,31 +8800,31 @@ dependencies = [ [[package]] name = "reth-net-banlist" -version = "1.3.12" +version = "1.9.3" dependencies = [ "alloy-primitives", ] [[package]] name = "reth-net-nat" -version = "1.3.12" +version = "1.9.3" dependencies = [ "futures-util", "if-addrs", "reqwest", "reth-tracing", "serde_with", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "tracing", ] [[package]] name = "reth-network" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", + "alloy-consensus", + "alloy-eips", "alloy-genesis", "alloy-primitives", "alloy-rlp", @@ -8739,13 +8866,12 @@ dependencies = [ "reth-tokio-util", "reth-tracing", "reth-transaction-pool", - "rustc-hash 2.1.1", + "rustc-hash", "schnellru", - "secp256k1", + "secp256k1 0.30.0", "serde", "smallvec", - "tempfile", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "tokio-stream", "tokio-util", @@ -8755,10 +8881,12 @@ dependencies = [ [[package]] name = "reth-network-api" -version = "1.3.12" +version = "1.9.3" dependencies = [ + "alloy-consensus", "alloy-primitives", "alloy-rpc-types-admin", + "alloy-rpc-types-eth", "auto_impl", "derive_more", "enr", @@ -8770,17 +8898,17 @@ dependencies = [ "reth-network-types", "reth-tokio-util", "serde", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "tokio-stream", ] [[package]] name = "reth-network-p2p" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", + "alloy-consensus", + "alloy-eips", "alloy-primitives", "auto_impl", "derive_more", @@ -8799,24 +8927,24 @@ dependencies = [ [[package]] name = "reth-network-peers" -version = "1.3.12" +version = "1.9.3" dependencies = [ "alloy-primitives", "alloy-rlp", "enr", "rand 0.8.5", "rand 0.9.2", - "secp256k1", + "secp256k1 0.30.0", "serde_json", "serde_with", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "url", ] [[package]] name = "reth-network-types" -version = "1.3.12" +version = "1.9.3" dependencies = [ "alloy-eip2124", "humantime-serde", @@ -8829,7 +8957,7 @@ dependencies = [ [[package]] name = "reth-nippy-jar" -version = "1.3.12" +version = "1.9.3" dependencies = [ "anyhow", "bincode 1.3.3", @@ -8840,14 +8968,14 @@ dependencies = [ "reth-fs-util", "serde", "tempfile", - "thiserror 2.0.16", + "thiserror 2.0.17", "tracing", "zstd", ] [[package]] name = "reth-node-api" -version = "1.3.12" +version = "1.9.3" dependencies = [ "alloy-rpc-types-engine", "eyre", @@ -8870,10 +8998,10 @@ dependencies = [ [[package]] name = "reth-node-builder" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", + "alloy-consensus", + "alloy-eips", "alloy-primitives", "alloy-provider", "alloy-rpc-types", @@ -8896,10 +9024,13 @@ dependencies = [ "reth-db-common", "reth-downloaders", "reth-engine-local", + "reth-engine-primitives", "reth-engine-service", "reth-engine-tree", "reth-engine-util", + "reth-ethereum-engine-primitives", "reth-evm", + "reth-evm-ethereum", "reth-exex", "reth-fs-util", "reth-invalid-block-hooks", @@ -8908,9 +9039,12 @@ dependencies = [ "reth-network-p2p", "reth-node-api", "reth-node-core", + "reth-node-ethereum", + "reth-node-ethstats", "reth-node-events", "reth-node-metrics", "reth-payload-builder", + "reth-primitives-traits", "reth-provider", "reth-prune", "reth-rpc", @@ -8925,7 +9059,7 @@ dependencies = [ "reth-tokio-util", "reth-tracing", "reth-transaction-pool", - "secp256k1", + "secp256k1 0.30.0", "serde_json", "tempfile", "tokio", @@ -8935,10 +9069,10 @@ dependencies = [ [[package]] name = "reth-node-core" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", + "alloy-consensus", + "alloy-eips", "alloy-primitives", "alloy-rpc-types-engine", "clap", @@ -8956,6 +9090,7 @@ dependencies = [ "reth-db", "reth-discv4", "reth-discv5", + "reth-engine-local", "reth-engine-primitives", "reth-ethereum-forks", "reth-net-nat", @@ -8964,48 +9099,51 @@ dependencies = [ "reth-network-peers", "reth-primitives-traits", "reth-prune-types", + "reth-rpc-convert", "reth-rpc-eth-types", "reth-rpc-server-types", - "reth-rpc-types-compat", "reth-stages-types", "reth-storage-api", "reth-storage-errors", "reth-tracing", + "reth-tracing-otlp", "reth-transaction-pool", - "secp256k1", + "secp256k1 0.30.0", "serde", "shellexpand", "strum 0.27.2", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "toml", "tracing", + "url", "vergen", "vergen-git2", ] [[package]] name = "reth-node-ethereum" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", + "alloy-consensus", "alloy-contract", - "alloy-eips 0.15.11", + "alloy-eips", "alloy-genesis", + "alloy-network", "alloy-primitives", "alloy-provider", "alloy-rpc-types-beacon", "alloy-rpc-types-engine", - "alloy-rpc-types-eth 0.15.11", + "alloy-rpc-types-eth", "alloy-signer", "alloy-sol-types", "eyre", "futures", "rand 0.9.2", "reth-chainspec", - "reth-consensus", "reth-db", "reth-e2e-test-utils", + "reth-engine-local", "reth-engine-primitives", "reth-ethereum-consensus", "reth-ethereum-engine-primitives", @@ -9031,18 +9169,40 @@ dependencies = [ "reth-tasks", "reth-tracing", "reth-transaction-pool", - "reth-trie-db", "revm", "serde_json", "tokio", ] +[[package]] +name = "reth-node-ethstats" +version = "1.9.3" +dependencies = [ + "alloy-consensus", + "alloy-primitives", + "chrono", + "futures-util", + "reth-chain-state", + "reth-network-api", + "reth-primitives-traits", + "reth-storage-api", + "reth-transaction-pool", + "serde", + "serde_json", + "thiserror 2.0.17", + "tokio", + "tokio-stream", + "tokio-tungstenite", + "tracing", + "url", +] + [[package]] name = "reth-node-events" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", + "alloy-consensus", + "alloy-eips", "alloy-primitives", "alloy-rpc-types-engine", "derive_more", @@ -9062,7 +9222,7 @@ dependencies = [ [[package]] name = "reth-node-metrics" -version = "1.3.12" +version = "1.9.3" dependencies = [ "eyre", "http", @@ -9071,42 +9231,46 @@ dependencies = [ "metrics-exporter-prometheus", "metrics-process", "metrics-util", - "procfs", + "procfs 0.17.0", "reqwest", "reth-metrics", "reth-tasks", "socket2 0.5.10", "tikv-jemalloc-ctl", "tokio", - "tower 0.4.13", + "tower", "tracing", ] [[package]] name = "reth-node-types" -version = "1.3.12" +version = "1.9.3" dependencies = [ "reth-chainspec", "reth-db-api", "reth-engine-primitives", "reth-payload-primitives", "reth-primitives-traits", - "reth-trie-db", ] [[package]] name = "reth-op" -version = "1.3.12" +version = "1.9.3" dependencies = [ "reth-chainspec", + "reth-cli-util", + "reth-codecs", "reth-consensus", "reth-consensus-common", "reth-db", + "reth-engine-local", "reth-eth-wire", "reth-evm", + "reth-exex", "reth-network", "reth-network-api", "reth-node-api", + "reth-node-builder", "reth-node-core", "reth-optimism-chainspec", "reth-optimism-cli", @@ -9123,23 +9287,27 @@ dependencies = [ "reth-rpc-builder", "reth-rpc-eth-types", "reth-storage-api", + "reth-tasks", "reth-transaction-pool", "reth-trie", + "reth-trie-db", ] [[package]] name = "reth-optimism-chainspec" -version = "1.3.12" +version = "1.9.3" dependencies = [ "alloy-chains", - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", + "alloy-consensus", + "alloy-eips", "alloy-genesis", "alloy-hardforks", + "alloy-op-hardforks", "alloy-primitives", - "alloy-serde 0.15.11", + "alloy-serde", "derive_more", "miniz_oxide", + "op-alloy-consensus", "op-alloy-rpc-types", "paste", "reth-chainspec", @@ -9152,15 +9320,15 @@ dependencies = [ "serde", "serde_json", "tar-no-std", - "thiserror 2.0.16", + "thiserror 2.0.17", ] [[package]] name = "reth-optimism-cli" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", + "alloy-consensus", + "alloy-eips", "alloy-primitives", "alloy-rlp", "clap", @@ -9192,31 +9360,33 @@ dependencies = [ "reth-primitives-traits", "reth-provider", "reth-prune", + "reth-rpc-server-types", "reth-stages", "reth-static-file", "reth-static-file-types", "reth-tracing", + "reth-tracing-otlp", "serde", "tempfile", "tokio", "tokio-util", "tracing", + "url", ] [[package]] name = "reth-optimism-consensus" -version = "1.3.12" +version = "1.9.3" dependencies = [ "alloy-chains", - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", + "alloy-consensus", + "alloy-eips", "alloy-primitives", - "alloy-trie 0.8.1", + "alloy-trie", "op-alloy-consensus", "reth-chainspec", "reth-consensus", "reth-consensus-common", - "reth-db-api", "reth-db-common", "reth-execution-types", "reth-optimism-chainspec", @@ -9231,21 +9401,22 @@ dependencies = [ "reth-trie", "reth-trie-common", "revm", - "thiserror 2.0.16", + "thiserror 2.0.17", "tracing", ] [[package]] name = "reth-optimism-evm" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", + "alloy-consensus", + "alloy-eips", "alloy-evm", "alloy-genesis", "alloy-op-evm", "alloy-primitives", "op-alloy-consensus", + "op-alloy-rpc-types-engine", "op-revm", "reth-chainspec", "reth-evm", @@ -9258,13 +9429,55 @@ dependencies = [ "reth-optimism-primitives", "reth-primitives-traits", "reth-revm", + "reth-rpc-eth-api", + "reth-storage-errors", "revm", - "thiserror 2.0.16", + "thiserror 2.0.17", + "tracing", +] + +[[package]] +name = "reth-optimism-flashblocks" +version = "1.9.3" +dependencies = [ + "alloy-consensus", + "alloy-eips", + "alloy-primitives", + "alloy-rpc-types-engine", + "alloy-serde", + "brotli", + "derive_more", + "eyre", + "futures-util", + "metrics", + "reth-chain-state", + "reth-engine-primitives", + "reth-errors", + "reth-evm", + "reth-execution-types", + "reth-metrics", + "reth-optimism-evm", + "reth-optimism-payload-builder", + "reth-optimism-primitives", + "reth-payload-primitives", + "reth-primitives-traits", + "reth-revm", + "reth-rpc-eth-types", + "reth-storage-api", + "reth-tasks", + "ringbuffer", + "serde", + "serde_json", + "test-case", + "tokio", + "tokio-tungstenite", + "tracing", + "url", ] [[package]] name = "reth-optimism-forks" -version = "1.3.12" +version = "1.9.3" dependencies = [ "alloy-op-hardforks", "alloy-primitives", @@ -9274,19 +9487,19 @@ dependencies = [ [[package]] name = "reth-optimism-node" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", + "alloy-consensus", "alloy-genesis", "alloy-network", "alloy-primitives", "alloy-rpc-types-engine", - "alloy-rpc-types-eth 0.15.11", + "alloy-rpc-types-eth", "clap", "eyre", "futures", "op-alloy-consensus", + "op-alloy-network", "op-alloy-rpc-types-engine", "op-revm", "reth-chainspec", @@ -9308,35 +9521,36 @@ dependencies = [ "reth-optimism-payload-builder", "reth-optimism-primitives", "reth-optimism-rpc", + "reth-optimism-storage", "reth-optimism-txpool", "reth-payload-builder", "reth-payload-util", - "reth-payload-validator", "reth-primitives-traits", "reth-provider", "reth-revm", + "reth-rpc", "reth-rpc-api", "reth-rpc-engine-api", - "reth-rpc-eth-api", "reth-rpc-eth-types", "reth-rpc-server-types", "reth-tasks", "reth-tracing", "reth-transaction-pool", "reth-trie-common", - "reth-trie-db", "revm", "serde", "serde_json", "tokio", + "url", ] [[package]] name = "reth-optimism-payload-builder" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", + "alloy-consensus", + "alloy-eips", + "alloy-evm", "alloy-primitives", "alloy-rlp", "alloy-rpc-types-debug", @@ -9364,17 +9578,17 @@ dependencies = [ "reth-transaction-pool", "revm", "serde", - "sha2 0.10.9", - "thiserror 2.0.16", + "sha2", + "thiserror 2.0.17", "tracing", ] [[package]] name = "reth-optimism-primitives" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", + "alloy-consensus", + "alloy-eips", "alloy-primitives", "alloy-rlp", "arbitrary", @@ -9390,7 +9604,7 @@ dependencies = [ "reth-primitives-traits", "reth-zstd-compressors", "rstest", - "secp256k1", + "secp256k1 0.30.0", "serde", "serde_json", "serde_with", @@ -9398,42 +9612,44 @@ dependencies = [ [[package]] name = "reth-optimism-rpc" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", + "alloy-consensus", + "alloy-eips", "alloy-json-rpc", + "alloy-op-hardforks", "alloy-primitives", "alloy-rpc-client", "alloy-rpc-types-debug", "alloy-rpc-types-engine", - "alloy-rpc-types-eth 0.15.11", - "alloy-signer", + "alloy-rpc-types-eth", "alloy-transport", "alloy-transport-http", "async-trait", "derive_more", "eyre", + "futures", "jsonrpsee", "jsonrpsee-core", "jsonrpsee-types", + "metrics", "op-alloy-consensus", "op-alloy-network", "op-alloy-rpc-jsonrpsee", "op-alloy-rpc-types", "op-alloy-rpc-types-engine", "op-revm", - "parking_lot", "reqwest", "reth-chain-state", "reth-chainspec", "reth-evm", - "reth-network-api", + "reth-mantle-forks", + "reth-metrics", "reth-node-api", "reth-node-builder", "reth-optimism-chainspec", - "reth-optimism-consensus", "reth-optimism-evm", + "reth-optimism-flashblocks", "reth-optimism-forks", "reth-optimism-payload-builder", "reth-optimism-primitives", @@ -9449,23 +9665,21 @@ dependencies = [ "reth-tasks", "reth-transaction-pool", "revm", - "thiserror 2.0.16", + "serde_json", + "thiserror 2.0.17", "tokio", + "tokio-stream", + "tower", "tracing", ] [[package]] name = "reth-optimism-storage" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", - "alloy-primitives", - "reth-chainspec", + "alloy-consensus", "reth-codecs", - "reth-db-api", - "reth-optimism-forks", "reth-optimism-primitives", - "reth-primitives-traits", "reth-prune-types", "reth-stages-types", "reth-storage-api", @@ -9473,21 +9687,22 @@ dependencies = [ [[package]] name = "reth-optimism-txpool" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", + "alloy-consensus", + "alloy-eips", "alloy-json-rpc", "alloy-primitives", "alloy-rpc-client", - "alloy-rpc-types-eth 0.15.11", - "alloy-serde 0.15.11", + "alloy-rpc-types-eth", + "alloy-serde", "c-kzg", "derive_more", "futures-util", "metrics", "op-alloy-consensus", "op-alloy-flz", + "op-alloy-rpc-types", "op-revm", "parking_lot", "reth-chain-state", @@ -9503,16 +9718,16 @@ dependencies = [ "reth-storage-api", "reth-transaction-pool", "serde", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "tracing", ] [[package]] name = "reth-payload-builder" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", + "alloy-consensus", "alloy-primitives", "alloy-rpc-types", "futures-util", @@ -9530,7 +9745,7 @@ dependencies = [ [[package]] name = "reth-payload-builder-primitives" -version = "1.3.12" +version = "1.9.3" dependencies = [ "pin-project", "reth-payload-primitives", @@ -9541,47 +9756,48 @@ dependencies = [ [[package]] name = "reth-payload-primitives" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-eips 0.15.11", + "alloy-eips", "alloy-primitives", "alloy-rpc-types-engine", "assert_matches", "auto_impl", + "either", "op-alloy-rpc-types-engine", "reth-chain-state", "reth-chainspec", "reth-errors", "reth-primitives-traits", "serde", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", ] [[package]] name = "reth-payload-util" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", + "alloy-consensus", "alloy-primitives", "reth-transaction-pool", ] [[package]] name = "reth-payload-validator" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", + "alloy-consensus", "alloy-rpc-types-engine", "reth-primitives-traits", ] [[package]] name = "reth-primitives" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", + "alloy-consensus", + "alloy-eips", "alloy-genesis", "alloy-primitives", "alloy-rlp", @@ -9600,14 +9816,15 @@ dependencies = [ [[package]] name = "reth-primitives-traits" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", + "alloy-consensus", + "alloy-eips", "alloy-genesis", "alloy-primitives", "alloy-rlp", - "alloy-trie 0.8.1", + "alloy-rpc-types-eth", + "alloy-trie", "arbitrary", "auto_impl", "bincode 1.3.3", @@ -9627,20 +9844,19 @@ dependencies = [ "revm-bytecode", "revm-primitives", "revm-state", - "secp256k1", + "secp256k1 0.30.0", "serde", "serde_json", "serde_with", - "test-fuzz", - "thiserror 2.0.16", + "thiserror 2.0.17", ] [[package]] name = "reth-provider" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", + "alloy-consensus", + "alloy-eips", "alloy-primitives", "alloy-rpc-types-engine", "assert_matches", @@ -9660,7 +9876,6 @@ dependencies = [ "reth-errors", "reth-ethereum-engine-primitives", "reth-ethereum-primitives", - "reth-evm", "reth-execution-types", "reth-fs-util", "reth-metrics", @@ -9686,16 +9901,15 @@ dependencies = [ [[package]] name = "reth-prune" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", + "alloy-consensus", + "alloy-eips", "alloy-primitives", "assert_matches", "itertools 0.14.0", "metrics", "rayon", - "reth-chainspec", "reth-config", "reth-db", "reth-db-api", @@ -9710,15 +9924,19 @@ dependencies = [ "reth-testing-utils", "reth-tokio-util", "reth-tracing", - "rustc-hash 2.1.1", - "thiserror 2.0.16", + "rustc-hash", + "thiserror 2.0.17", "tokio", "tracing", ] +[[package]] +name = "reth-prune-db" +version = "1.9.3" + [[package]] name = "reth-prune-types" -version = "1.3.12" +version = "1.9.3" dependencies = [ "alloy-primitives", "arbitrary", @@ -9730,16 +9948,16 @@ dependencies = [ "reth-codecs", "serde", "serde_json", - "test-fuzz", - "thiserror 2.0.16", + "strum 0.27.2", + "thiserror 2.0.17", "toml", ] [[package]] name = "reth-ress-protocol" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", + "alloy-consensus", "alloy-primitives", "alloy-rlp", "arbitrary", @@ -9763,21 +9981,22 @@ dependencies = [ [[package]] name = "reth-ress-provider" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", + "alloy-consensus", "alloy-primitives", "eyre", "futures", "parking_lot", "reth-chain-state", + "reth-errors", "reth-ethereum-primitives", "reth-evm", "reth-node-api", "reth-primitives-traits", - "reth-provider", "reth-ress-protocol", "reth-revm", + "reth-storage-api", "reth-tasks", "reth-tokio-util", "reth-trie", @@ -9788,9 +10007,9 @@ dependencies = [ [[package]] name = "reth-revm" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", + "alloy-consensus", "alloy-primitives", "reth-ethereum-forks", "reth-primitives-traits", @@ -9802,34 +10021,37 @@ dependencies = [ [[package]] name = "reth-rpc" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", + "alloy-consensus", "alloy-dyn-abi", - "alloy-eips 0.15.11", + "alloy-eips", "alloy-evm", "alloy-genesis", "alloy-network", "alloy-primitives", "alloy-rlp", + "alloy-rpc-client", "alloy-rpc-types", "alloy-rpc-types-admin", "alloy-rpc-types-beacon", "alloy-rpc-types-debug", "alloy-rpc-types-engine", - "alloy-rpc-types-eth 0.15.11", - "alloy-rpc-types-mev 0.15.11", + "alloy-rpc-types-eth", + "alloy-rpc-types-mev", "alloy-rpc-types-trace", "alloy-rpc-types-txpool", - "alloy-serde 0.15.11", + "alloy-serde", "alloy-signer", "alloy-signer-local", "async-trait", "derive_more", + "dyn-clone", "futures", "http", "http-body", "hyper", + "itertools 0.14.0", "jsonrpsee", "jsonrpsee-types", "jsonwebtoken", @@ -9839,6 +10061,8 @@ dependencies = [ "reth-chain-state", "reth-chainspec", "reth-consensus", + "reth-consensus-common", + "reth-db-api", "reth-engine-primitives", "reth-errors", "reth-ethereum-primitives", @@ -9854,33 +10078,35 @@ dependencies = [ "reth-provider", "reth-revm", "reth-rpc-api", + "reth-rpc-convert", "reth-rpc-engine-api", "reth-rpc-eth-api", "reth-rpc-eth-types", "reth-rpc-server-types", - "reth-rpc-types-compat", "reth-storage-api", "reth-tasks", "reth-testing-utils", "reth-transaction-pool", + "reth-trie-common", "revm", "revm-inspectors", "revm-primitives", "serde", "serde_json", - "thiserror 2.0.16", + "sha2", + "thiserror 2.0.17", "tokio", "tokio-stream", - "tower 0.4.13", + "tower", "tracing", "tracing-futures", ] [[package]] name = "reth-rpc-api" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-eips 0.15.11", + "alloy-eips", "alloy-genesis", "alloy-json-rpc", "alloy-primitives", @@ -9890,24 +10116,26 @@ dependencies = [ "alloy-rpc-types-beacon", "alloy-rpc-types-debug", "alloy-rpc-types-engine", - "alloy-rpc-types-eth 0.15.11", - "alloy-rpc-types-mev 0.15.11", + "alloy-rpc-types-eth", + "alloy-rpc-types-mev", "alloy-rpc-types-trace", "alloy-rpc-types-txpool", - "alloy-serde 0.15.11", + "alloy-serde", "jsonrpsee", + "reth-chain-state", "reth-engine-primitives", "reth-network-peers", "reth-rpc-eth-api", + "reth-trie-common", ] [[package]] name = "reth-rpc-api-testing-util" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-eips 0.15.11", + "alloy-eips", "alloy-primitives", - "alloy-rpc-types-eth 0.15.11", + "alloy-rpc-types-eth", "alloy-rpc-types-trace", "futures", "jsonrpsee", @@ -9922,16 +10150,17 @@ dependencies = [ [[package]] name = "reth-rpc-builder" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-eips 0.15.11", + "alloy-eips", "alloy-network", "alloy-primitives", "alloy-provider", "alloy-rpc-types-engine", - "alloy-rpc-types-eth 0.15.11", + "alloy-rpc-types-eth", "alloy-rpc-types-trace", "clap", + "dyn-clone", "http", "jsonrpsee", "metrics", @@ -9950,36 +10179,82 @@ dependencies = [ "reth-network-peers", "reth-node-core", "reth-node-ethereum", - "reth-payload-builder", - "reth-primitives-traits", - "reth-provider", - "reth-rpc", + "reth-payload-builder", + "reth-primitives-traits", + "reth-provider", + "reth-rpc", + "reth-rpc-api", + "reth-rpc-engine-api", + "reth-rpc-eth-api", + "reth-rpc-eth-types", + "reth-rpc-layer", + "reth-rpc-server-types", + "reth-storage-api", + "reth-tasks", + "reth-tracing", + "reth-transaction-pool", + "serde", + "serde_json", + "thiserror 2.0.17", + "tokio", + "tokio-util", + "tower", + "tower-http", + "tracing", +] + +[[package]] +name = "reth-rpc-convert" +version = "1.9.3" +dependencies = [ + "alloy-consensus", + "alloy-json-rpc", + "alloy-network", + "alloy-primitives", + "alloy-rpc-types-eth", + "alloy-signer", + "auto_impl", + "dyn-clone", + "jsonrpsee-types", + "op-alloy-consensus", + "op-alloy-network", + "op-alloy-rpc-types", + "op-revm", + "reth-ethereum-primitives", + "reth-evm", + "reth-optimism-primitives", + "reth-primitives-traits", + "reth-storage-api", + "revm-context", + "serde_json", + "thiserror 2.0.17", +] + +[[package]] +name = "reth-rpc-e2e-tests" +version = "1.9.3" +dependencies = [ + "alloy-genesis", + "alloy-rpc-types-engine", + "eyre", + "futures-util", + "jsonrpsee", + "reth-chainspec", + "reth-e2e-test-utils", + "reth-node-api", + "reth-node-ethereum", "reth-rpc-api", - "reth-rpc-engine-api", - "reth-rpc-eth-api", - "reth-rpc-eth-types", - "reth-rpc-layer", - "reth-rpc-server-types", - "reth-rpc-types-compat", - "reth-storage-api", - "reth-tasks", "reth-tracing", - "reth-transaction-pool", - "serde", "serde_json", - "thiserror 2.0.16", "tokio", - "tokio-util", - "tower 0.4.13", - "tower-http", "tracing", ] [[package]] name = "reth-rpc-engine-api" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-eips 0.15.11", + "alloy-eips", "alloy-primitives", "alloy-rlp", "alloy-rpc-types-engine", @@ -10001,31 +10276,31 @@ dependencies = [ "reth-primitives-traits", "reth-provider", "reth-rpc-api", - "reth-rpc-server-types", "reth-storage-api", "reth-tasks", "reth-testing-utils", "reth-transaction-pool", "serde", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "tracing", ] [[package]] name = "reth-rpc-eth-api" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", + "alloy-consensus", "alloy-dyn-abi", - "alloy-eips 0.15.11", + "alloy-eips", + "alloy-evm", "alloy-json-rpc", "alloy-network", "alloy-primitives", "alloy-rlp", - "alloy-rpc-types-eth 0.15.11", - "alloy-rpc-types-mev 0.15.11", - "alloy-serde 0.15.11", + "alloy-rpc-types-eth", + "alloy-rpc-types-mev", + "alloy-serde", "async-trait", "auto_impl", "dyn-clone", @@ -10033,18 +10308,18 @@ dependencies = [ "jsonrpsee", "jsonrpsee-types", "parking_lot", + "reth-chain-state", "reth-chainspec", "reth-errors", "reth-evm", "reth-network-api", "reth-node-api", - "reth-payload-builder", "reth-primitives-traits", - "reth-provider", "reth-revm", + "reth-rpc-convert", "reth-rpc-eth-types", "reth-rpc-server-types", - "reth-rpc-types-compat", + "reth-storage-api", "reth-tasks", "reth-transaction-pool", "reth-trie-common", @@ -10057,13 +10332,17 @@ dependencies = [ [[package]] name = "reth-rpc-eth-types" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", + "alloy-consensus", + "alloy-eips", + "alloy-evm", + "alloy-network", "alloy-primitives", - "alloy-rpc-types-eth 0.15.11", + "alloy-rpc-client", + "alloy-rpc-types-eth", "alloy-sol-types", + "alloy-transport", "derive_more", "futures", "itertools 0.14.0", @@ -10071,6 +10350,7 @@ dependencies = [ "jsonrpsee-types", "metrics", "rand 0.9.2", + "reqwest", "reth-chain-state", "reth-chainspec", "reth-errors", @@ -10080,8 +10360,8 @@ dependencies = [ "reth-metrics", "reth-primitives-traits", "reth-revm", + "reth-rpc-convert", "reth-rpc-server-types", - "reth-rpc-types-compat", "reth-storage-api", "reth-tasks", "reth-transaction-pool", @@ -10091,7 +10371,7 @@ dependencies = [ "schnellru", "serde", "serde_json", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "tokio-stream", "tracing", @@ -10099,7 +10379,7 @@ dependencies = [ [[package]] name = "reth-rpc-layer" -version = "1.3.12" +version = "1.9.3" dependencies = [ "alloy-rpc-types-engine", "http", @@ -10109,16 +10389,16 @@ dependencies = [ "pin-project", "reqwest", "tokio", - "tower 0.4.13", + "tower", "tower-http", "tracing", ] [[package]] name = "reth-rpc-server-types" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-eips 0.15.11", + "alloy-eips", "alloy-primitives", "alloy-rpc-types-engine", "jsonrpsee-core", @@ -10129,30 +10409,18 @@ dependencies = [ "strum 0.27.2", ] -[[package]] -name = "reth-rpc-types-compat" -version = "1.3.12" -dependencies = [ - "alloy-consensus 0.15.11", - "alloy-primitives", - "alloy-rpc-types-eth 0.15.11", - "jsonrpsee-types", - "reth-primitives-traits", - "serde", -] - [[package]] name = "reth-stages" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", + "alloy-consensus", + "alloy-eips", "alloy-primitives", "alloy-rlp", "assert_matches", "bincode 1.3.3", - "blake3", "codspeed-criterion-compat", + "eyre", "futures-util", "itertools 0.14.0", "num-traits", @@ -10167,12 +10435,14 @@ dependencies = [ "reth-db", "reth-db-api", "reth-downloaders", + "reth-era", + "reth-era-downloader", + "reth-era-utils", "reth-ethereum-consensus", "reth-ethereum-primitives", "reth-etl", "reth-evm", "reth-evm-ethereum", - "reth-execution-errors", "reth-execution-types", "reth-exex", "reth-fs-util", @@ -10188,21 +10458,19 @@ dependencies = [ "reth-static-file-types", "reth-storage-errors", "reth-testing-utils", - "reth-tracing", "reth-trie", "reth-trie-db", - "serde", "tempfile", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "tracing", ] [[package]] name = "reth-stages-api" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-eips 0.15.11", + "alloy-eips", "alloy-primitives", "aquamarine", "assert_matches", @@ -10221,7 +10489,7 @@ dependencies = [ "reth-static-file-types", "reth-testing-utils", "reth-tokio-util", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "tokio-stream", "tracing", @@ -10229,7 +10497,7 @@ dependencies = [ [[package]] name = "reth-stages-types" -version = "1.3.12" +version = "1.9.3" dependencies = [ "alloy-primitives", "arbitrary", @@ -10241,43 +10509,44 @@ dependencies = [ "reth-codecs", "reth-trie-common", "serde", - "test-fuzz", ] [[package]] name = "reth-stateless" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", + "alloy-consensus", "alloy-primitives", "alloy-rlp", "alloy-rpc-types-debug", - "alloy-trie 0.8.1", + "alloy-trie", "itertools 0.14.0", + "k256", "reth-chainspec", "reth-consensus", "reth-errors", "reth-ethereum-consensus", "reth-ethereum-primitives", "reth-evm", - "reth-evm-ethereum", "reth-primitives-traits", "reth-revm", "reth-trie-common", "reth-trie-sparse", - "thiserror 2.0.16", + "secp256k1 0.30.0", + "serde", + "serde_with", + "thiserror 2.0.17", ] [[package]] name = "reth-static-file" -version = "1.3.12" +version = "1.9.3" dependencies = [ "alloy-primitives", "assert_matches", "parking_lot", "rayon", "reth-codecs", - "reth-db", "reth-db-api", "reth-primitives-traits", "reth-provider", @@ -10294,7 +10563,7 @@ dependencies = [ [[package]] name = "reth-static-file-types" -version = "1.3.12" +version = "1.9.3" dependencies = [ "alloy-primitives", "clap", @@ -10306,10 +10575,10 @@ dependencies = [ [[package]] name = "reth-storage-api" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", + "alloy-consensus", + "alloy-eips", "alloy-primitives", "alloy-rpc-types-engine", "auto_impl", @@ -10323,15 +10592,14 @@ dependencies = [ "reth-stages-types", "reth-storage-errors", "reth-trie-common", - "reth-trie-db", "revm-database", ] [[package]] name = "reth-storage-errors" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-eips 0.15.11", + "alloy-eips", "alloy-primitives", "alloy-rlp", "derive_more", @@ -10339,12 +10607,41 @@ dependencies = [ "reth-prune-types", "reth-static-file-types", "revm-database-interface", - "thiserror 2.0.16", + "thiserror 2.0.17", +] + +[[package]] +name = "reth-storage-rpc-provider" +version = "1.9.3" +dependencies = [ + "alloy-consensus", + "alloy-eips", + "alloy-network", + "alloy-primitives", + "alloy-provider", + "alloy-rpc-types", + "alloy-rpc-types-engine", + "parking_lot", + "reth-chainspec", + "reth-db-api", + "reth-errors", + "reth-execution-types", + "reth-node-types", + "reth-primitives", + "reth-provider", + "reth-prune-types", + "reth-rpc-convert", + "reth-stages-types", + "reth-storage-api", + "reth-trie", + "revm", + "tokio", + "tracing", ] [[package]] name = "reth-tasks" -version = "1.3.12" +version = "1.9.3" dependencies = [ "auto_impl", "dyn-clone", @@ -10353,7 +10650,7 @@ dependencies = [ "pin-project", "rayon", "reth-metrics", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "tracing", "tracing-futures", @@ -10361,22 +10658,22 @@ dependencies = [ [[package]] name = "reth-testing-utils" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", + "alloy-consensus", + "alloy-eips", "alloy-genesis", "alloy-primitives", "rand 0.8.5", "rand 0.9.2", "reth-ethereum-primitives", "reth-primitives-traits", - "secp256k1", + "secp256k1 0.30.0", ] [[package]] name = "reth-tokio-util" -version = "1.3.12" +version = "1.9.3" dependencies = [ "tokio", "tokio-stream", @@ -10385,35 +10682,55 @@ dependencies = [ [[package]] name = "reth-tracing" -version = "1.3.12" +version = "1.9.3" dependencies = [ "clap", "eyre", + "reth-tracing-otlp", "rolling-file", "tracing", "tracing-appender", "tracing-journald", "tracing-logfmt", "tracing-subscriber 0.3.20", + "url", +] + +[[package]] +name = "reth-tracing-otlp" +version = "1.9.3" +dependencies = [ + "clap", + "eyre", + "opentelemetry", + "opentelemetry-otlp", + "opentelemetry-semantic-conventions", + "opentelemetry_sdk", + "tracing", + "tracing-opentelemetry", + "tracing-subscriber 0.3.20", + "url", ] [[package]] name = "reth-transaction-pool" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", + "alloy-consensus", + "alloy-eips", "alloy-primitives", "alloy-rlp", "aquamarine", "assert_matches", "auto_impl", - "bitflags 2.9.4", + "bitflags 2.10.0", "codspeed-criterion-compat", + "futures", "futures-util", "metrics", "parking_lot", "paste", + "pin-project", "proptest", "proptest-arbitrary-interop", "rand 0.9.2", @@ -10431,13 +10748,13 @@ dependencies = [ "reth-tracing", "revm-interpreter", "revm-primitives", - "rustc-hash 2.1.1", + "rustc-hash", "schnellru", "serde", "serde_json", "smallvec", "tempfile", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "tokio-stream", "tracing", @@ -10445,13 +10762,14 @@ dependencies = [ [[package]] name = "reth-trie" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", - "alloy-eips 0.15.11", + "alloy-consensus", + "alloy-eips", "alloy-primitives", "alloy-rlp", - "alloy-trie 0.8.1", + "alloy-trie", + "assert_matches", "auto_impl", "codspeed-criterion-compat", "itertools 0.14.0", @@ -10477,23 +10795,24 @@ dependencies = [ [[package]] name = "reth-trie-common" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", + "alloy-consensus", "alloy-genesis", "alloy-primitives", "alloy-rlp", - "alloy-rpc-types-eth 0.15.11", - "alloy-serde 0.15.11", - "alloy-trie 0.8.1", + "alloy-rpc-types-eth", + "alloy-serde", + "alloy-trie", "arbitrary", + "arrayvec", "bincode 1.3.3", "bytes", "codspeed-criterion-compat", "derive_more", "hash-db", "itertools 0.14.0", - "nybbles 0.3.4", + "nybbles", "plain_hasher", "proptest", "proptest-arbitrary-interop", @@ -10509,9 +10828,9 @@ dependencies = [ [[package]] name = "reth-trie-db" -version = "1.3.12" +version = "1.9.3" dependencies = [ - "alloy-consensus 0.15.11", + "alloy-consensus", "alloy-primitives", "alloy-rlp", "proptest", @@ -10522,7 +10841,6 @@ dependencies = [ "reth-execution-errors", "reth-primitives-traits", "reth-provider", - "reth-storage-errors", "reth-trie", "reth-trie-common", "revm", @@ -10535,11 +10853,13 @@ dependencies = [ [[package]] name = "reth-trie-parallel" -version = "1.3.12" +version = "1.9.3" dependencies = [ "alloy-primitives", "alloy-rlp", "codspeed-criterion-compat", + "crossbeam-channel", + "dashmap 6.1.0", "derive_more", "itertools 0.14.0", "metrics", @@ -10547,7 +10867,6 @@ dependencies = [ "proptest-arbitrary-interop", "rand 0.9.2", "rayon", - "reth-db-api", "reth-execution-errors", "reth-metrics", "reth-primitives-traits", @@ -10557,17 +10876,18 @@ dependencies = [ "reth-trie-common", "reth-trie-db", "reth-trie-sparse", - "thiserror 2.0.16", + "thiserror 2.0.17", "tokio", "tracing", ] [[package]] name = "reth-trie-sparse" -version = "1.3.12" +version = "1.9.3" dependencies = [ "alloy-primitives", "alloy-rlp", + "alloy-trie", "arbitrary", "assert_matches", "auto_impl", @@ -10579,6 +10899,7 @@ dependencies = [ "proptest-arbitrary-interop", "rand 0.8.5", "rand 0.9.2", + "rayon", "reth-execution-errors", "reth-metrics", "reth-primitives-traits", @@ -10593,17 +10914,46 @@ dependencies = [ "tracing", ] +[[package]] +name = "reth-trie-sparse-parallel" +version = "1.9.3" +dependencies = [ + "alloy-primitives", + "alloy-rlp", + "alloy-trie", + "arbitrary", + "assert_matches", + "itertools 0.14.0", + "metrics", + "pretty_assertions", + "proptest", + "proptest-arbitrary-interop", + "rand 0.8.5", + "rand 0.9.2", + "rayon", + "reth-execution-errors", + "reth-metrics", + "reth-primitives-traits", + "reth-provider", + "reth-trie", + "reth-trie-common", + "reth-trie-db", + "reth-trie-sparse", + "smallvec", + "tracing", +] + [[package]] name = "reth-zstd-compressors" -version = "1.3.12" +version = "1.9.3" dependencies = [ "zstd", ] [[package]] name = "revm" -version = "23.1.0" -source = "git+https://github.com/mantle-xyz/revm?tag=v2.0.0#7412a0d6707ec96aa9ae9cb3e6cb8e975029f1f0" +version = "31.0.2" +source = "git+https://github.com/mantle-xyz/revm?tag=v2.1.2#ba33cb80cdced50ee342186fa5ade478b75f1050" dependencies = [ "revm-bytecode", "revm-context", @@ -10620,11 +10970,10 @@ dependencies = [ [[package]] name = "revm-bytecode" -version = "4.0.0" -source = "git+https://github.com/mantle-xyz/revm?tag=v2.0.0#7412a0d6707ec96aa9ae9cb3e6cb8e975029f1f0" +version = "7.1.1" +source = "git+https://github.com/mantle-xyz/revm?tag=v2.1.2#ba33cb80cdced50ee342186fa5ade478b75f1050" dependencies = [ "bitvec", - "once_cell", "phf", "revm-primitives", "serde", @@ -10632,9 +10981,10 @@ dependencies = [ [[package]] name = "revm-context" -version = "4.1.0" -source = "git+https://github.com/mantle-xyz/revm?tag=v2.0.0#7412a0d6707ec96aa9ae9cb3e6cb8e975029f1f0" +version = "11.0.2" +source = "git+https://github.com/mantle-xyz/revm?tag=v2.1.2#ba33cb80cdced50ee342186fa5ade478b75f1050" dependencies = [ + "bitvec", "cfg-if", "derive-where", "revm-bytecode", @@ -10647,8 +10997,8 @@ dependencies = [ [[package]] name = "revm-context-interface" -version = "4.1.0" -source = "git+https://github.com/mantle-xyz/revm?tag=v2.0.0#7412a0d6707ec96aa9ae9cb3e6cb8e975029f1f0" +version = "12.0.1" +source = "git+https://github.com/mantle-xyz/revm?tag=v2.1.2#ba33cb80cdced50ee342186fa5ade478b75f1050" dependencies = [ "alloy-eip2930", "alloy-eip7702", @@ -10662,10 +11012,10 @@ dependencies = [ [[package]] name = "revm-database" -version = "4.0.0" -source = "git+https://github.com/mantle-xyz/revm?tag=v2.0.0#7412a0d6707ec96aa9ae9cb3e6cb8e975029f1f0" +version = "9.0.5" +source = "git+https://github.com/mantle-xyz/revm?tag=v2.1.2#ba33cb80cdced50ee342186fa5ade478b75f1050" dependencies = [ - "alloy-eips 0.14.0", + "alloy-eips", "revm-bytecode", "revm-database-interface", "revm-primitives", @@ -10675,10 +11025,11 @@ dependencies = [ [[package]] name = "revm-database-interface" -version = "4.0.0" -source = "git+https://github.com/mantle-xyz/revm?tag=v2.0.0#7412a0d6707ec96aa9ae9cb3e6cb8e975029f1f0" +version = "8.0.5" +source = "git+https://github.com/mantle-xyz/revm?tag=v2.1.2#ba33cb80cdced50ee342186fa5ade478b75f1050" dependencies = [ "auto_impl", + "either", "revm-primitives", "revm-state", "serde", @@ -10686,10 +11037,11 @@ dependencies = [ [[package]] name = "revm-handler" -version = "4.1.0" -source = "git+https://github.com/mantle-xyz/revm?tag=v2.0.0#7412a0d6707ec96aa9ae9cb3e6cb8e975029f1f0" +version = "12.0.2" +source = "git+https://github.com/mantle-xyz/revm?tag=v2.1.2#ba33cb80cdced50ee342186fa5ade478b75f1050" dependencies = [ "auto_impl", + "derive-where", "revm-bytecode", "revm-context", "revm-context-interface", @@ -10703,10 +11055,11 @@ dependencies = [ [[package]] name = "revm-inspector" -version = "4.1.0" -source = "git+https://github.com/mantle-xyz/revm?tag=v2.0.0#7412a0d6707ec96aa9ae9cb3e6cb8e975029f1f0" +version = "12.0.2" +source = "git+https://github.com/mantle-xyz/revm?tag=v2.1.2#ba33cb80cdced50ee342186fa5ade478b75f1050" dependencies = [ "auto_impl", + "either", "revm-context", "revm-database-interface", "revm-handler", @@ -10719,11 +11072,11 @@ dependencies = [ [[package]] name = "revm-inspectors" -version = "0.22.3" -source = "git+https://github.com/mantle-xyz/revm-inspectors?tag=v2.0.0#aa1a12e72e3b76f7204fa4b952d2ca4daf40c595" +version = "0.32.0" +source = "git+https://github.com/mantle-xyz/revm-inspectors?tag=v2.1.1#745481367432662c601739bf94678081ad991347" dependencies = [ "alloy-primitives", - "alloy-rpc-types-eth 0.15.11", + "alloy-rpc-types-eth", "alloy-rpc-types-trace", "alloy-sol-types", "boa_engine", @@ -10731,60 +11084,62 @@ dependencies = [ "colorchoice", "revm", "serde_json", - "thiserror 2.0.16", + "thiserror 2.0.17", ] [[package]] name = "revm-interpreter" -version = "19.1.0" -source = "git+https://github.com/mantle-xyz/revm?tag=v2.0.0#7412a0d6707ec96aa9ae9cb3e6cb8e975029f1f0" +version = "29.0.1" +source = "git+https://github.com/mantle-xyz/revm?tag=v2.1.2#ba33cb80cdced50ee342186fa5ade478b75f1050" dependencies = [ "revm-bytecode", "revm-context-interface", "revm-primitives", + "revm-state", "serde", ] [[package]] name = "revm-precompile" -version = "20.1.0" -source = "git+https://github.com/mantle-xyz/revm?tag=v2.0.0#7412a0d6707ec96aa9ae9cb3e6cb8e975029f1f0" +version = "29.0.1" +source = "git+https://github.com/mantle-xyz/revm?tag=v2.1.2#ba33cb80cdced50ee342186fa5ade478b75f1050" dependencies = [ "ark-bls12-381", "ark-bn254", "ark-ec", "ark-ff 0.5.0", "ark-serialize 0.5.0", + "arrayref", "aurora-engine-modexp", "blst", "c-kzg", "cfg-if", "k256", - "libsecp256k1", - "once_cell", "p256", "revm-primitives", "ripemd", - "secp256k1", - "sha2 0.10.9", + "rug", + "secp256k1 0.31.1", + "sha2", ] [[package]] name = "revm-primitives" -version = "19.0.0" -source = "git+https://github.com/mantle-xyz/revm?tag=v2.0.0#7412a0d6707ec96aa9ae9cb3e6cb8e975029f1f0" +version = "21.0.2" +source = "git+https://github.com/mantle-xyz/revm?tag=v2.1.2#ba33cb80cdced50ee342186fa5ade478b75f1050" dependencies = [ "alloy-primitives", "num_enum", + "once_cell", "serde", ] [[package]] name = "revm-state" -version = "4.0.0" -source = "git+https://github.com/mantle-xyz/revm?tag=v2.0.0#7412a0d6707ec96aa9ae9cb3e6cb8e975029f1f0" +version = "8.1.1" +source = "git+https://github.com/mantle-xyz/revm?tag=v2.1.2#ba33cb80cdced50ee342186fa5ade478b75f1050" dependencies = [ - "bitflags 2.9.4", + "bitflags 2.10.0", "revm-bytecode", "revm-primitives", "serde", @@ -10921,20 +11276,33 @@ dependencies = [ "regex", "relative-path", "rustc_version 0.4.1", - "syn 2.0.106", + "syn 2.0.108", "unicode-ident", ] +[[package]] +name = "rug" +version = "1.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58ad2e973fe3c3214251a840a621812a4f40468da814b1a3d6947d433c2af11f" +dependencies = [ + "az", + "gmp-mpfr-sys", + "libc", + "libm", +] + [[package]] name = "ruint" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ecb38f82477f20c5c3d62ef52d7c4e536e38ea9b73fb570a20c5cae0e14bcf6" +checksum = "a68df0380e5c9d20ce49534f292a36a7514ae21350726efe1865bdb1fa91d278" dependencies = [ "alloy-rlp", "arbitrary", "ark-ff 0.3.0", "ark-ff 0.4.2", + "ark-ff 0.5.0", "bytes", "fastrlp 0.3.1", "fastrlp 0.4.0", @@ -10948,7 +11316,7 @@ dependencies = [ "rand 0.9.2", "rlp", "ruint-macro", - "serde", + "serde_core", "valuable", "zeroize", ] @@ -10965,12 +11333,6 @@ version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace" -[[package]] -name = "rustc-hash" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" - [[package]] name = "rustc-hash" version = "2.1.1" @@ -10986,6 +11348,15 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e75f6a532d0fd9f7f13144f392b6ad56a32696bfcd9c78f797f16bbb6f072d6" +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver 0.9.0", +] + [[package]] name = "rustc_version" version = "0.3.3" @@ -11001,7 +11372,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" dependencies = [ - "semver 1.0.26", + "semver 1.0.27", ] [[package]] @@ -11010,7 +11381,7 @@ version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags 2.9.4", + "bitflags 2.10.0", "errno", "libc", "linux-raw-sys 0.4.15", @@ -11023,18 +11394,18 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" dependencies = [ - "bitflags 2.9.4", + "bitflags 2.10.0", "errno", "libc", "linux-raw-sys 0.11.0", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] name = "rustls" -version = "0.23.31" +version = "0.23.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0ebcbd2f03de0fc1122ad9bb24b127a5a6cd51d72604a3f3c50ac459762b6cc" +checksum = "6a9586e9ee2b4f8fab52a0048ca7334d7024eef48e2cb9407e3497bb7cab7fa7" dependencies = [ "log", "once_cell", @@ -11047,9 +11418,9 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcff2dd52b58a8d98a70243663a0d234c4e2b79235637849d15913394a247d3" +checksum = "9980d917ebb0c0536119ba501e90834767bffc3d60641457fd84a1f3fd337923" dependencies = [ "openssl-probe", "rustls-pki-types", @@ -11059,9 +11430,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.12.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79" +checksum = "94182ad936a0c91c324cd46c6511b9510ed16af436d7b5bab34beab0afd55f7a" dependencies = [ "web-time", "zeroize", @@ -11096,9 +11467,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.6" +version = "0.103.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8572f3c2cb9934231157b45499fc41e1f58c589fdfb81a844ba873265e80f8eb" +checksum = "2ffdfa2f5286e2247234e03f680868ac2815974dc39e00ea15adc445d0aafe52" dependencies = [ "ring", "rustls-pki-types", @@ -11113,9 +11484,9 @@ checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" [[package]] name = "rusty-fork" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb3dcc6e454c328bb824492db107ab7c0ae8fcffe4ad210136ef014458c1bc4f" +checksum = "cc6bf79ff24e648f6da1f8d1f011e9cac26491b619e6b9280f2b47f1774e6ee2" dependencies = [ "fnv", "quick-error", @@ -11150,7 +11521,7 @@ version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" dependencies = [ - "windows-sys 0.61.0", + "windows-sys 0.61.2", ] [[package]] @@ -11223,10 +11594,21 @@ checksum = "b50c5943d326858130af85e049f2661ba3c78b26589b8ab98e65e80ae44a1252" dependencies = [ "bitcoin_hashes", "rand 0.8.5", - "secp256k1-sys", + "secp256k1-sys 0.10.1", "serde", ] +[[package]] +name = "secp256k1" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c3c81b43dc2d8877c216a3fccf76677ee1ebccd429566d3e67447290d0c42b2" +dependencies = [ + "bitcoin_hashes", + "rand 0.9.2", + "secp256k1-sys 0.11.0", +] + [[package]] name = "secp256k1-sys" version = "0.10.1" @@ -11236,13 +11618,22 @@ dependencies = [ "cc", ] +[[package]] +name = "secp256k1-sys" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcb913707158fadaf0d8702c2db0e857de66eb003ccfdda5924b5f5ac98efb38" +dependencies = [ + "cc", +] + [[package]] name = "security-framework" -version = "3.4.0" +version = "3.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60b369d18893388b345804dc0007963c99b7d665ae71d275812d828c6f089640" +checksum = "b3297343eaf830f66ede390ea39da1d462b6b0c1b000f420d0a83f898bbbe6ef" dependencies = [ - "bitflags 2.9.4", + "bitflags 2.10.0", "core-foundation", "core-foundation-sys", "libc", @@ -11259,24 +11650,40 @@ dependencies = [ "libc", ] +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser 0.7.0", +] + [[package]] name = "semver" version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f301af10236f6df4160f7c3f04eec6dbc70ace82d23326abad5edee88801c6b6" dependencies = [ - "semver-parser", + "semver-parser 0.10.3", ] [[package]] name = "semver" -version = "1.0.26" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" +checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" dependencies = [ "serde", + "serde_core", ] +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + [[package]] name = "semver-parser" version = "0.10.3" @@ -11300,10 +11707,11 @@ checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" [[package]] name = "serde" -version = "1.0.215" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" dependencies = [ + "serde_core", "serde_derive", ] @@ -11316,28 +11724,38 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + [[package]] name = "serde_derive" -version = "1.0.215" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] name = "serde_json" -version = "1.0.143" +version = "1.0.145" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d401abef1d108fbd9cbaebc3e46611f4b1021f714a0597a71f41ee463f5f4a5a" +checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" dependencies = [ - "indexmap 2.11.1", + "indexmap 2.12.0", "itoa", "memchr", "ryu", "serde", + "serde_core", ] [[package]] @@ -11374,19 +11792,18 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.14.0" +version = "3.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2c45cd61fefa9db6f254525d46e392b852e0e61d9a1fd36e5bd183450a556d5" +checksum = "aa66c845eee442168b2c8134fec70ac50dc20e760769c8ba0ad1319ca1959b04" dependencies = [ "base64 0.22.1", "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.11.1", + "indexmap 2.12.0", "schemars 0.9.0", "schemars 1.0.4", - "serde", - "serde_derive", + "serde_core", "serde_json", "serde_with_macros", "time", @@ -11394,14 +11811,14 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.14.0" +version = "3.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de90945e6565ce0d9a25098082ed4ee4002e047cb59892c318d66821e14bb30f" +checksum = "b91a903660542fced4e99881aa481bdbaec1634568ee02e0b8bd57c64cb38955" dependencies = [ - "darling 0.20.11", + "darling 0.21.3", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -11425,19 +11842,6 @@ dependencies = [ "digest 0.10.7", ] -[[package]] -name = "sha2" -version = "0.9.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" -dependencies = [ - "block-buffer 0.9.0", - "cfg-if", - "cpufeatures", - "digest 0.9.0", - "opaque-debug", -] - [[package]] name = "sha2" version = "0.10.9" @@ -11505,9 +11909,9 @@ dependencies = [ [[package]] name = "signal-hook-mio" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34db1a06d485c9142248b7a054f034b349b212551f3dfd19c94d45a754a217cd" +checksum = "b75a19a7a740b25bc7944bdee6172368f988763b744e3d4dfe753f6b4ece40cc" dependencies = [ "libc", "mio", @@ -11533,6 +11937,12 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" + [[package]] name = "similar" version = "2.7.0" @@ -11562,7 +11972,7 @@ checksum = "297f631f50729c8c99b84667867963997ec0b50f32b2a7dbcab828ef0541e8bb" dependencies = [ "num-bigint", "num-traits", - "thiserror 2.0.16", + "thiserror 2.0.17", "time", ] @@ -11599,6 +12009,15 @@ version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" +[[package]] +name = "small_btree" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ba60d2df92ba73864714808ca68c059734853e6ab722b40e1cf543ebb3a057a" +dependencies = [ + "arrayvec", +] + [[package]] name = "smallvec" version = "1.15.1" @@ -11646,12 +12065,12 @@ dependencies = [ [[package]] name = "socket2" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "233504af464074f9d066d7b5416c5f9b894a5862a6506e306f7b816cdd6f1807" +checksum = "17129e116933cf371d018bb80ae557e889637989d8638274fb25622827b03881" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -11670,6 +12089,12 @@ dependencies = [ "sha1", ] +[[package]] +name = "spin" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591" + [[package]] name = "spki" version = "0.7.3" @@ -11680,17 +12105,11 @@ dependencies = [ "der", ] -[[package]] -name = "sptr" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b9b39299b249ad65f3b7e96443bad61c02ca5cd3589f46cb6d610a0fd6c0d6a" - [[package]] name = "stable_deref_trait" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "static_assertions" @@ -11732,7 +12151,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -11744,7 +12163,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -11766,9 +12185,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.106" +version = "2.0.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6" +checksum = "da58917d35242480a05c2897064da0a80589a2a0476c9a3f2fdc83b53502e917" dependencies = [ "proc-macro2", "quote", @@ -11777,14 +12196,14 @@ dependencies = [ [[package]] name = "syn-solidity" -version = "1.3.1" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0b198d366dbec045acfcd97295eb653a7a2b40e4dc764ef1e79aafcad439d3c" +checksum = "ff790eb176cc81bb8936aed0f7b9f14fc4670069a2d371b3e3b0ecce908b2cb3" dependencies = [ "paste", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -11804,7 +12223,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -11820,6 +12239,12 @@ dependencies = [ "windows 0.57.0", ] +[[package]] +name = "tag_ptr" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0e973b34477b7823833469eb0f5a3a60370fef7a453e02d751b59180d0a5a05" + [[package]] name = "tagptr" version = "0.2.0" @@ -11849,22 +12274,22 @@ version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac9ee8b664c9f1740cd813fea422116f8ba29997bb7c878d1940424889802897" dependencies = [ - "bitflags 2.9.4", + "bitflags 2.10.0", "log", "num-traits", ] [[package]] name = "tempfile" -version = "3.22.0" +version = "3.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84fa4d11fadde498443cca10fd3ac23c951f0dc59e080e9f4b93d4df4e4eea53" +checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16" dependencies = [ "fastrand 2.3.0", - "getrandom 0.3.3", + "getrandom 0.3.4", "once_cell", "rustix 1.1.2", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -11885,7 +12310,7 @@ dependencies = [ "cfg-if", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -11896,7 +12321,7 @@ checksum = "5c89e72a01ed4c579669add59014b9a524d609c0c88c6a585ce37485879f6ffb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", "test-case-core", ] @@ -11936,7 +12361,7 @@ dependencies = [ "prettyplease", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -11969,11 +12394,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.16" +version = "2.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3467d614147380f2e4e374161426ff399c91084acd2363eaf549172b3d5e60c0" +checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" dependencies = [ - "thiserror-impl 2.0.16", + "thiserror-impl 2.0.17", ] [[package]] @@ -11984,18 +12409,18 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] name = "thiserror-impl" -version = "2.0.16" +version = "2.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c5e1be1c48b9172ee610da68fd9cd2770e7a4056cb3fc98710ee6906f0c7960" +checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -12018,9 +12443,9 @@ dependencies = [ [[package]] name = "tikv-jemalloc-ctl" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f21f216790c8df74ce3ab25b534e0718da5a1916719771d3fec23315c99e468b" +checksum = "661f1f6a57b3a36dc9174a2c10f19513b4866816e13425d3e418b11cc37bc24c" dependencies = [ "libc", "paste", @@ -12029,9 +12454,9 @@ dependencies = [ [[package]] name = "tikv-jemalloc-sys" -version = "0.6.0+5.3.0-1-ge13ca993e8ccb9ba9847cc330696e02839f328f7" +version = "0.6.1+5.3.0-1-ge13ca993e8ccb9ba9847cc330696e02839f328f7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd3c60906412afa9c2b5b5a48ca6a5abe5736aec9eb48ad05037a677e52e4e2d" +checksum = "cd8aa5b2ab86a2cefa406d889139c162cbb230092f7d1d7cbc1716405d852a3b" dependencies = [ "cc", "libc", @@ -12039,9 +12464,9 @@ dependencies = [ [[package]] name = "tikv-jemallocator" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cec5ff18518d81584f477e9bfdf957f5bb0979b0bac3af4ca30b5b3ae2d2865" +checksum = "0359b4327f954e0567e69fb191cf1436617748813819c94b8cd4a431422d053a" dependencies = [ "libc", "tikv-jemalloc-sys", @@ -12049,11 +12474,12 @@ dependencies = [ [[package]] name = "time" -version = "0.3.43" +version = "0.3.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83bde6f1ec10e72d583d91623c939f623002284ef622b87de38cfd546cbf2031" +checksum = "91e7d9e3bb61134e77bde20dd4825b97c010155709965fedf0f49bb138e52a9d" dependencies = [ "deranged", + "itoa", "js-sys", "libc", "num-conv", @@ -12091,22 +12517,13 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" -dependencies = [ - "displaydoc", - "zerovec 0.10.4", -] - -[[package]] -name = "tinystr" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b" +checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" dependencies = [ "displaydoc", - "zerovec 0.11.4", + "serde_core", + "zerovec", ] [[package]] @@ -12136,40 +12553,37 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.47.1" +version = "1.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89e49afdadebb872d3145a5638b59eb0691ea23e46ca484037cfab3b76b95038" +checksum = "ff360e02eab121e0bc37a2d3b4d4dc622e6eda3a8e5253d5435ecf5bd4c68408" dependencies = [ - "backtrace", "bytes", - "io-uring", "libc", "mio", "parking_lot", "pin-project-lite", "signal-hook-registry", - "slab", - "socket2 0.6.0", + "socket2 0.6.1", "tokio-macros", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] name = "tokio-macros" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" +checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] name = "tokio-rustls" -version = "0.26.2" +version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" dependencies = [ "rustls", "tokio", @@ -12196,6 +12610,7 @@ dependencies = [ "futures-util", "log", "rustls", + "rustls-native-certs", "rustls-pki-types", "tokio", "tokio-rustls", @@ -12241,11 +12656,11 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.7.0" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bade1c3e902f58d73d3f294cd7f20391c1cb2fbcb643b73566bc773971df91e3" +checksum = "f2cdb639ebbc97961c51720f858597f7f24c4fc295327923af55b74c3c724533" dependencies = [ - "serde", + "serde_core", ] [[package]] @@ -12254,7 +12669,7 @@ version = "0.22.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ - "indexmap 2.11.1", + "indexmap 2.12.0", "serde", "serde_spanned", "toml_datetime 0.6.11", @@ -12264,21 +12679,21 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.23.4" +version = "0.23.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7211ff1b8f0d3adae1663b7da9ffe396eabe1ca25f0b0bee42b0da29a9ddce93" +checksum = "6485ef6d0d9b5d0ec17244ff7eb05310113c3f316f2d14200d4de56b3cb98f8d" dependencies = [ - "indexmap 2.11.1", - "toml_datetime 0.7.0", + "indexmap 2.12.0", + "toml_datetime 0.7.3", "toml_parser", "winnow", ] [[package]] name = "toml_parser" -version = "1.0.2" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b551886f449aa90d4fe2bdaa9f4a2577ad2dde302c61ecf262d80b116db95c10" +checksum = "c0cbe268d35bdb4bb5a56a2de88d0ad0eb70af5384a99d648cd4b3d04039800e" dependencies = [ "winnow", ] @@ -12290,26 +12705,42 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" [[package]] -name = "tower" -version = "0.4.13" +name = "tonic" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +checksum = "eb7613188ce9f7df5bfe185db26c5814347d110db17920415cf2fbcad85e7203" dependencies = [ - "futures-core", - "futures-util", - "hdrhistogram", - "indexmap 1.9.3", + "async-trait", + "base64 0.22.1", + "bytes", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-timeout", + "hyper-util", + "percent-encoding", "pin-project", - "pin-project-lite", - "rand 0.8.5", - "slab", + "sync_wrapper", "tokio", - "tokio-util", + "tokio-stream", + "tower", "tower-layer", "tower-service", "tracing", ] +[[package]] +name = "tonic-prost" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66bd50ad6ce1252d87ef024b3d64fe4c3cf54a86fb9ef4c631fdd0ded7aeaa67" +dependencies = [ + "bytes", + "prost", + "tonic", +] + [[package]] name = "tower" version = "0.5.2" @@ -12318,11 +12749,16 @@ checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" dependencies = [ "futures-core", "futures-util", + "hdrhistogram", + "indexmap 2.12.0", "pin-project-lite", + "slab", "sync_wrapper", "tokio", + "tokio-util", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -12333,7 +12769,7 @@ checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" dependencies = [ "async-compression", "base64 0.22.1", - "bitflags 2.9.4", + "bitflags 2.10.0", "bytes", "futures-core", "futures-util", @@ -12349,7 +12785,7 @@ dependencies = [ "pin-project-lite", "tokio", "tokio-util", - "tower 0.5.2", + "tower", "tower-layer", "tower-service", "tracing", @@ -12400,7 +12836,7 @@ checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -12419,8 +12855,6 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" dependencies = [ - "futures", - "futures-task", "pin-project", "tracing", ] @@ -12459,6 +12893,25 @@ dependencies = [ "tracing-subscriber 0.3.20", ] +[[package]] +name = "tracing-opentelemetry" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6e5658463dd88089aba75c7791e1d3120633b1bfde22478b28f625a9bb1b8e" +dependencies = [ + "js-sys", + "opentelemetry", + "opentelemetry_sdk", + "rustversion", + "smallvec", + "thiserror 2.0.17", + "tracing", + "tracing-core", + "tracing-log", + "tracing-subscriber 0.3.20", + "web-time", +] + [[package]] name = "tracing-serde" version = "0.2.0" @@ -12518,7 +12971,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "319c70195101a93f56db4c74733e272d720768e13471f400c78406a326b172b0" dependencies = [ "cc", - "windows-targets 0.52.6", + "windows-targets 0.48.5", ] [[package]] @@ -12543,7 +12996,7 @@ dependencies = [ "darling 0.20.11", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -12558,9 +13011,9 @@ dependencies = [ [[package]] name = "triomphe" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef8f7726da4807b58ea5c96fdc122f80702030edc33b35aff9190a51148ccc85" +checksum = "dd69c5aa8f924c7519d6372789a74eac5b94fb0f8fcf0d4a97eb0bfc3e785f39" [[package]] name = "try-lock" @@ -12583,15 +13036,15 @@ dependencies = [ "rustls", "rustls-pki-types", "sha1", - "thiserror 2.0.16", + "thiserror 2.0.17", "utf-8", ] [[package]] name = "typenum" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" +checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" [[package]] name = "ucd-trie" @@ -12637,9 +13090,9 @@ checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" [[package]] name = "unicode-ident" -version = "1.0.19" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f63a545481291138910575129486daeaf8ac54aee4387fe7906919f7830c7d9d" +checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" [[package]] name = "unicode-segmentation" @@ -12746,7 +13199,7 @@ version = "1.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2" dependencies = [ - "getrandom 0.3.3", + "getrandom 0.3.4", "js-sys", "wasm-bindgen", ] @@ -12824,7 +13277,7 @@ checksum = "d674d135b4a8c1d7e813e2f8d1c9a58308aee4a680323066025e53132218bd91" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -12873,15 +13326,6 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" -[[package]] -name = "wasi" -version = "0.14.7+wasi-0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "883478de20367e224c0090af9cf5f9fa85bed63a95c1abf3afc5c083ebc06e8c" -dependencies = [ - "wasip2", -] - [[package]] name = "wasip2" version = "1.0.1+wasi-0.2.4" @@ -12893,9 +13337,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.102" +version = "0.2.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ad224d2776649cfb4f4471124f8176e54c1cca67a88108e30a0cd98b90e7ad3" +checksum = "da95793dfc411fbbd93f5be7715b0578ec61fe87cb1a42b12eb625caa5c5ea60" dependencies = [ "cfg-if", "once_cell", @@ -12904,25 +13348,11 @@ dependencies = [ "wasm-bindgen-shared", ] -[[package]] -name = "wasm-bindgen-backend" -version = "0.2.102" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a1364104bdcd3c03f22b16a3b1c9620891469f5e9f09bc38b2db121e593e732" -dependencies = [ - "bumpalo", - "log", - "proc-macro2", - "quote", - "syn 2.0.106", - "wasm-bindgen-shared", -] - [[package]] name = "wasm-bindgen-futures" -version = "0.4.52" +version = "0.4.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c0a08ecf5d99d5604a6666a70b3cde6ab7cc6142f5e641a8ef48fc744ce8854" +checksum = "551f88106c6d5e7ccc7cd9a16f312dd3b5d36ea8b4954304657d5dfba115d4a0" dependencies = [ "cfg-if", "js-sys", @@ -12933,9 +13363,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.102" +version = "0.2.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d7ab4ca3e367bb1ed84ddbd83cc6e41e115f8337ed047239578210214e36c76" +checksum = "04264334509e04a7bf8690f2384ef5265f05143a4bff3889ab7a3269adab59c2" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -12943,22 +13373,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.102" +version = "0.2.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a518014843a19e2dbbd0ed5dfb6b99b23fb886b14e6192a00803a3e14c552b0" +checksum = "420bc339d9f322e562942d52e115d57e950d12d88983a14c79b86859ee6c7ebc" dependencies = [ + "bumpalo", "proc-macro2", "quote", - "syn 2.0.106", - "wasm-bindgen-backend", + "syn 2.0.108", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.102" +version = "0.2.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "255eb0aa4cc2eea3662a00c2bbd66e93911b7361d5e0fcd62385acfd7e15dcee" +checksum = "76f218a38c84bcb33c25ec7059b07847d465ce0e0a76b995e134a45adcb6af76" dependencies = [ "unicode-ident", ] @@ -12992,9 +13422,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.79" +version = "0.3.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50462a022f46851b81d5441d1a6f5bac0b21a1d72d64bd4906fbdd4bf7230ec7" +checksum = "3a1f95c0d03a47f4ae1f7a64643a6bb97465d9b740f0fa8f90ea33915c99a9a1" dependencies = [ "js-sys", "wasm-bindgen", @@ -13016,14 +13446,14 @@ version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75c7f0ef91146ebfb530314f5f1d24528d7f0767efbfd31dce919275413e393e" dependencies = [ - "webpki-root-certs 1.0.2", + "webpki-root-certs 1.0.4", ] [[package]] name = "webpki-root-certs" -version = "1.0.2" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e4ffd8df1c57e87c325000a3d6ef93db75279dc3a231125aac571650f22b12a" +checksum = "ee3e3b5f5e80bc89f30ce8d0343bf4e5f12341c51f3e26cbeecbc7c85443e85b" dependencies = [ "rustls-pki-types", ] @@ -13034,23 +13464,23 @@ version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "webpki-roots 1.0.2", + "webpki-roots 1.0.4", ] [[package]] name = "webpki-roots" -version = "1.0.2" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e8983c3ab33d6fb807cfcdad2491c4ea8cbc8ed839181c7dfd9c67c83e261b2" +checksum = "b2878ef029c47c6e8cf779119f20fcf52bde7ad42a731b2a304bc221df17571e" dependencies = [ "rustls-pki-types", ] [[package]] name = "widestring" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd7cf3379ca1aac9eea11fba24fd7e315d621f8dfe35c8d7d2be8b793726e07d" +checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" [[package]] name = "winapi" @@ -13095,25 +13525,27 @@ dependencies = [ [[package]] name = "windows" -version = "0.58.0" +version = "0.61.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd04d41d93c4992d421894c18c8b43496aa748dd4c081bac0dc93eb0489272b6" +checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" dependencies = [ - "windows-core 0.58.0", - "windows-targets 0.52.6", + "windows-collections 0.2.0", + "windows-core 0.61.2", + "windows-future 0.2.1", + "windows-link 0.1.3", + "windows-numerics 0.2.0", ] [[package]] name = "windows" -version = "0.61.3" +version = "0.62.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" +checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" dependencies = [ - "windows-collections", - "windows-core 0.61.2", - "windows-future", - "windows-link 0.1.3", - "windows-numerics", + "windows-collections 0.3.2", + "windows-core 0.62.2", + "windows-future 0.3.2", + "windows-numerics 0.3.1", ] [[package]] @@ -13125,6 +13557,15 @@ dependencies = [ "windows-core 0.61.2", ] +[[package]] +name = "windows-collections" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" +dependencies = [ + "windows-core 0.62.2", +] + [[package]] name = "windows-core" version = "0.57.0" @@ -13139,28 +13580,28 @@ dependencies = [ [[package]] name = "windows-core" -version = "0.58.0" +version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ba6d44ec8c2591c134257ce647b7ea6b20335bf6379a27dac5f1641fcf59f99" +checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" dependencies = [ - "windows-implement 0.58.0", - "windows-interface 0.58.0", - "windows-result 0.2.0", - "windows-strings 0.1.0", - "windows-targets 0.52.6", + "windows-implement 0.60.2", + "windows-interface 0.59.3", + "windows-link 0.1.3", + "windows-result 0.3.4", + "windows-strings 0.4.2", ] [[package]] name = "windows-core" -version = "0.61.2" +version = "0.62.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" dependencies = [ - "windows-implement 0.60.0", - "windows-interface 0.59.1", - "windows-link 0.1.3", - "windows-result 0.3.4", - "windows-strings 0.4.2", + "windows-implement 0.60.2", + "windows-interface 0.59.3", + "windows-link 0.2.1", + "windows-result 0.4.1", + "windows-strings 0.5.1", ] [[package]] @@ -13171,40 +13612,40 @@ checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" dependencies = [ "windows-core 0.61.2", "windows-link 0.1.3", - "windows-threading", + "windows-threading 0.1.0", ] [[package]] -name = "windows-implement" -version = "0.57.0" +name = "windows-future" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7" +checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.106", + "windows-core 0.62.2", + "windows-link 0.2.1", + "windows-threading 0.2.1", ] [[package]] name = "windows-implement" -version = "0.58.0" +version = "0.57.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bbd5b46c938e506ecbce286b6628a02171d56153ba733b6c741fc627ec9579b" +checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] name = "windows-implement" -version = "0.60.0" +version = "0.60.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -13215,29 +13656,18 @@ checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", -] - -[[package]] -name = "windows-interface" -version = "0.58.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "053c4c462dc91d3b1504c6fe5a726dd15e216ba718e84a0e46a88fbe5ded3515" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] name = "windows-interface" -version = "0.59.1" +version = "0.59.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -13248,9 +13678,9 @@ checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" [[package]] name = "windows-link" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45e46c0661abb7180e7b9c281db115305d49ca1709ab8242adf09666d2173c65" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" [[package]] name = "windows-numerics" @@ -13263,19 +13693,20 @@ dependencies = [ ] [[package]] -name = "windows-result" -version = "0.1.2" +name = "windows-numerics" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8" +checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" dependencies = [ - "windows-targets 0.52.6", + "windows-core 0.62.2", + "windows-link 0.2.1", ] [[package]] name = "windows-result" -version = "0.2.0" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" +checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8" dependencies = [ "windows-targets 0.52.6", ] @@ -13290,13 +13721,12 @@ dependencies = [ ] [[package]] -name = "windows-strings" -version = "0.1.0" +name = "windows-result" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" dependencies = [ - "windows-result 0.2.0", - "windows-targets 0.52.6", + "windows-link 0.2.1", ] [[package]] @@ -13308,6 +13738,15 @@ dependencies = [ "windows-link 0.1.3", ] +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link 0.2.1", +] + [[package]] name = "windows-sys" version = "0.45.0" @@ -13350,16 +13789,16 @@ version = "0.60.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" dependencies = [ - "windows-targets 0.53.3", + "windows-targets 0.53.5", ] [[package]] name = "windows-sys" -version = "0.61.0" +version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e201184e40b2ede64bc2ea34968b28e33622acdbbf37104f0e4a33f7abe657aa" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" dependencies = [ - "windows-link 0.2.0", + "windows-link 0.2.1", ] [[package]] @@ -13410,19 +13849,19 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.53.3" +version = "0.53.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5fe6031c4041849d7c496a8ded650796e7b6ecc19df1a431c1a363342e5dc91" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" dependencies = [ - "windows-link 0.1.3", - "windows_aarch64_gnullvm 0.53.0", - "windows_aarch64_msvc 0.53.0", - "windows_i686_gnu 0.53.0", - "windows_i686_gnullvm 0.53.0", - "windows_i686_msvc 0.53.0", - "windows_x86_64_gnu 0.53.0", - "windows_x86_64_gnullvm 0.53.0", - "windows_x86_64_msvc 0.53.0", + "windows-link 0.2.1", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", ] [[package]] @@ -13434,6 +13873,15 @@ dependencies = [ "windows-link 0.1.3", ] +[[package]] +name = "windows-threading" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" +dependencies = [ + "windows-link 0.2.1", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.42.2" @@ -13454,9 +13902,9 @@ checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] name = "windows_aarch64_gnullvm" -version = "0.53.0" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" [[package]] name = "windows_aarch64_msvc" @@ -13478,9 +13926,9 @@ checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] name = "windows_aarch64_msvc" -version = "0.53.0" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" [[package]] name = "windows_i686_gnu" @@ -13502,9 +13950,9 @@ checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" [[package]] name = "windows_i686_gnu" -version = "0.53.0" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" [[package]] name = "windows_i686_gnullvm" @@ -13514,9 +13962,9 @@ checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] name = "windows_i686_gnullvm" -version = "0.53.0" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" [[package]] name = "windows_i686_msvc" @@ -13538,9 +13986,9 @@ checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] name = "windows_i686_msvc" -version = "0.53.0" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" [[package]] name = "windows_x86_64_gnu" @@ -13562,9 +14010,9 @@ checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] name = "windows_x86_64_gnu" -version = "0.53.0" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" [[package]] name = "windows_x86_64_gnullvm" @@ -13586,9 +14034,9 @@ checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] name = "windows_x86_64_gnullvm" -version = "0.53.0" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" [[package]] name = "windows_x86_64_msvc" @@ -13610,9 +14058,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "windows_x86_64_msvc" -version = "0.53.0" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" [[package]] name = "winnow" @@ -13647,15 +14095,9 @@ checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" [[package]] name = "writeable" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" - -[[package]] -name = "writeable" -version = "0.6.1" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" +checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" [[package]] name = "ws_stream_wasm" @@ -13670,7 +14112,7 @@ dependencies = [ "pharos", "rustc_version 0.4.1", "send_wrapper 0.6.0", - "thiserror 2.0.16", + "thiserror 2.0.17", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", @@ -13687,65 +14129,46 @@ dependencies = [ [[package]] name = "xattr" -version = "1.5.1" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af3a19837351dc82ba89f8a125e22a3c475f05aba604acc023d62b2739ae2909" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" dependencies = [ "libc", "rustix 1.1.2", ] [[package]] -name = "yansi" -version = "1.0.1" +name = "xsum" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" +checksum = "0637d3a5566a82fa5214bae89087bc8c9fb94cd8e8a3c07feb691bb8d9c632db" [[package]] -name = "yoke" -version = "0.7.5" +name = "yansi" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" -dependencies = [ - "serde", - "stable_deref_trait", - "yoke-derive 0.7.5", - "zerofrom", -] +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" [[package]] name = "yoke" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc" +checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" dependencies = [ - "serde", "stable_deref_trait", - "yoke-derive 0.8.0", + "yoke-derive", "zerofrom", ] [[package]] name = "yoke-derive" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.106", - "synstructure", -] - -[[package]] -name = "yoke-derive" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" +checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", "synstructure", ] @@ -13766,7 +14189,7 @@ checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] @@ -13786,15 +14209,15 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", "synstructure", ] [[package]] name = "zeroize" -version = "1.8.1" +version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" dependencies = [ "zeroize_derive", ] @@ -13807,62 +14230,41 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] name = "zerotrie" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595" +checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" dependencies = [ "displaydoc", - "yoke 0.8.0", - "zerofrom", -] - -[[package]] -name = "zerovec" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" -dependencies = [ - "yoke 0.7.5", + "yoke", "zerofrom", - "zerovec-derive 0.10.3", ] [[package]] name = "zerovec" -version = "0.11.4" +version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7aa2bd55086f1ab526693ecbe444205da57e25f4489879da80635a46d90e73b" +checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" dependencies = [ - "yoke 0.8.0", + "serde", + "yoke", "zerofrom", - "zerovec-derive 0.11.1", -] - -[[package]] -name = "zerovec-derive" -version = "0.10.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.106", + "zerovec-derive", ] [[package]] name = "zerovec-derive" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" +checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.108", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index fb79283e695..133f6492212 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [workspace.package] -version = "1.3.12" -edition = "2021" -rust-version = "1.86" +version = "1.9.3" +edition = "2024" +rust-version = "1.88" license = "MIT OR Apache-2.0" homepage = "https://paradigmxyz.github.io/reth" repository = "https://github.com/paradigmxyz/reth" @@ -11,7 +11,9 @@ authors = ["Mantle Core Contributors"] [workspace] members = [ "bin/reth-bench/", + "bin/reth-bench-compare/", "bin/reth/", + "crates/storage/rpc-provider/", "crates/chain-state/", "crates/chainspec/", "crates/cli/cli/", @@ -67,6 +69,7 @@ members = [ "crates/node/api/", "crates/node/builder/", "crates/node/core/", + "crates/node/ethstats", "crates/node/events/", "crates/node/metrics", "crates/node/types", @@ -75,6 +78,7 @@ members = [ "crates/optimism/cli", "crates/optimism/consensus", "crates/optimism/evm/", + "crates/optimism/flashblocks/", "crates/optimism/hardforks/", "crates/mantle-hardforks/", "crates/optimism/node/", @@ -92,6 +96,7 @@ members = [ "crates/payload/util/", "crates/primitives-traits/", "crates/primitives/", + "crates/prune/db", "crates/prune/prune", "crates/prune/types", "crates/ress/protocol", @@ -106,7 +111,8 @@ members = [ "crates/rpc/rpc-layer", "crates/rpc/rpc-server-types/", "crates/rpc/rpc-testing-util/", - "crates/rpc/rpc-types-compat/", + "crates/rpc/rpc-e2e-tests/", + "crates/rpc/rpc-convert/", "crates/rpc/rpc/", "crates/stages/api/", "crates/stages/stages/", @@ -135,6 +141,7 @@ members = [ "crates/trie/db", "crates/trie/parallel/", "crates/trie/sparse", + "crates/trie/sparse-parallel/", "crates/trie/trie", "examples/beacon-api-sidecar-fetcher/", "examples/beacon-api-sse/", @@ -143,21 +150,26 @@ members = [ "examples/custom-node/", "examples/custom-engine-types/", "examples/custom-evm/", + "examples/custom-hardforks/", "examples/custom-inspector/", "examples/custom-node-components/", "examples/custom-payload-builder/", "examples/custom-rlpx-subprotocol", "examples/custom-node", "examples/db-access", + "examples/engine-api-access", "examples/exex-hello-world", "examples/exex-subscription", "examples/exex-test", + "examples/full-contract-state", "examples/manual-p2p/", "examples/network-txpool/", "examples/network/", "examples/network-proxy/", + "examples/node-builder-api/", "examples/node-custom-rpc/", "examples/node-event-hooks/", + "examples/op-db-access/", "examples/polygon-p2p/", "examples/rpc-db/", "examples/precompile-cache/", @@ -165,9 +177,11 @@ members = [ "examples/custom-beacon-withdrawals", "testing/ef-tests/", "testing/testing-utils", + "testing/runner", + "crates/tracing-otlp", ] default-members = ["bin/reth"] -exclude = ["book/sources", "book/cli"] +exclude = ["docs/cli"] # Explicitly set the resolver to version 2, which is the default for packages with edition >= 2021 # https://doc.rust-lang.org/edition-guide/rust-2021/default-cargo-resolver.html @@ -179,6 +193,7 @@ rust.missing_docs = "warn" rust.rust_2018_idioms = { level = "deny", priority = -1 } rust.unreachable_pub = "warn" rust.unused_must_use = "deny" +rust.rust_2024_incompatible_pat = "warn" rustdoc.all = "warn" # rust.unnameable-types = "warn" @@ -319,8 +334,10 @@ codegen-units = 1 # reth op-reth = { path = "crates/optimism/bin" } reth = { path = "bin/reth" } +reth-storage-rpc-provider = { path = "crates/storage/rpc-provider" } reth-basic-payload-builder = { path = "crates/payload/basic" } reth-bench = { path = "bin/reth-bench" } +reth-bench-compare = { path = "bin/reth-bench-compare" } reth-chain-state = { path = "crates/chain-state" } reth-chainspec = { path = "crates/chainspec", default-features = false } reth-cli = { path = "crates/cli/cli" } @@ -354,7 +371,7 @@ reth-era-utils = { path = "crates/era-utils" } reth-errors = { path = "crates/errors" } reth-eth-wire = { path = "crates/net/eth-wire" } reth-eth-wire-types = { path = "crates/net/eth-wire-types" } -reth-ethereum-cli = { path = "crates/ethereum/cli" } +reth-ethereum-cli = { path = "crates/ethereum/cli", default-features = false } reth-ethereum-consensus = { path = "crates/ethereum/consensus", default-features = false } reth-ethereum-engine-primitives = { path = "crates/ethereum/engine-primitives", default-features = false } reth-ethereum-forks = { path = "crates/ethereum/hardforks", default-features = false } @@ -388,13 +405,14 @@ reth-node-api = { path = "crates/node/api" } reth-node-builder = { path = "crates/node/builder" } reth-node-core = { path = "crates/node/core" } reth-node-ethereum = { path = "crates/ethereum/node" } +reth-node-ethstats = { path = "crates/node/ethstats" } reth-node-events = { path = "crates/node/events" } reth-node-metrics = { path = "crates/node/metrics" } reth-optimism-node = { path = "crates/optimism/node" } reth-node-types = { path = "crates/node/types" } reth-op = { path = "crates/optimism/reth", default-features = false } reth-optimism-chainspec = { path = "crates/optimism/chainspec", default-features = false } -reth-optimism-cli = { path = "crates/optimism/cli" } +reth-optimism-cli = { path = "crates/optimism/cli", default-features = false } reth-optimism-consensus = { path = "crates/optimism/consensus", default-features = false } reth-optimism-forks = { path = "crates/optimism/hardforks", default-features = false } reth-mantle-forks = { path = "crates/mantle-hardforks", default-features = false } @@ -418,16 +436,18 @@ reth-rpc = { path = "crates/rpc/rpc" } reth-rpc-api = { path = "crates/rpc/rpc-api" } reth-rpc-api-testing-util = { path = "crates/rpc/rpc-testing-util" } reth-rpc-builder = { path = "crates/rpc/rpc-builder" } +reth-rpc-e2e-tests = { path = "crates/rpc/rpc-e2e-tests" } reth-rpc-engine-api = { path = "crates/rpc/rpc-engine-api" } reth-rpc-eth-api = { path = "crates/rpc/rpc-eth-api" } reth-rpc-eth-types = { path = "crates/rpc/rpc-eth-types", default-features = false } reth-rpc-layer = { path = "crates/rpc/rpc-layer" } +reth-optimism-flashblocks = { path = "crates/optimism/flashblocks" } reth-rpc-server-types = { path = "crates/rpc/rpc-server-types" } -reth-rpc-types-compat = { path = "crates/rpc/rpc-types-compat" } +reth-rpc-convert = { path = "crates/rpc/rpc-convert" } reth-stages = { path = "crates/stages/stages" } reth-stages-api = { path = "crates/stages/api" } reth-stages-types = { path = "crates/stages/types", default-features = false } -reth-stateless = { path = "crates/stateless" } +reth-stateless = { path = "crates/stateless", default-features = false } reth-static-file = { path = "crates/static-file/static-file" } reth-static-file-types = { path = "crates/static-file/types", default-features = false } reth-storage-api = { path = "crates/storage/storage-api", default-features = false } @@ -435,91 +455,115 @@ reth-storage-errors = { path = "crates/storage/errors", default-features = false reth-tasks = { path = "crates/tasks" } reth-testing-utils = { path = "testing/testing-utils" } reth-tokio-util = { path = "crates/tokio-util" } -reth-tracing = { path = "crates/tracing" } +reth-tracing = { path = "crates/tracing", default-features = false } +reth-tracing-otlp = { path = "crates/tracing-otlp" } reth-transaction-pool = { path = "crates/transaction-pool" } reth-trie = { path = "crates/trie/trie" } reth-trie-common = { path = "crates/trie/common", default-features = false } reth-trie-db = { path = "crates/trie/db" } reth-trie-parallel = { path = "crates/trie/parallel" } reth-trie-sparse = { path = "crates/trie/sparse", default-features = false } +reth-trie-sparse-parallel = { path = "crates/trie/sparse-parallel" } reth-zstd-compressors = { path = "crates/storage/zstd-compressors", default-features = false } reth-ress-protocol = { path = "crates/ress/protocol" } reth-ress-provider = { path = "crates/ress/provider" } # revm -revm = { git = "https://github.com/mantle-xyz/revm", tag = "v2.0.0", default-features = false } -revm-bytecode = { git = "https://github.com/mantle-xyz/revm", tag = "v2.0.0", default-features = false } -revm-state = { git = "https://github.com/mantle-xyz/revm", tag = "v2.0.0", default-features = false } -revm-primitives = { git = "https://github.com/mantle-xyz/revm", tag = "v2.0.0", default-features = false } -revm-interpreter = { git = "https://github.com/mantle-xyz/revm", tag = "v2.0.0", default-features = false } -revm-inspector = { git = "https://github.com/mantle-xyz/revm", tag = "v2.0.0", default-features = false } -revm-context = { git = "https://github.com/mantle-xyz/revm", tag = "v2.0.0", default-features = false } -revm-context-interface = { git = "https://github.com/mantle-xyz/revm", tag = "v2.0.0", default-features = false } -revm-database = { git = "https://github.com/mantle-xyz/revm", tag = "v2.0.0", default-features = false } -revm-database-interface = { git = "https://github.com/mantle-xyz/revm", tag = "v2.0.0", default-features = false } -op-revm = { git = "https://github.com/mantle-xyz/revm", tag = "v2.0.0", default-features = false } -revm-inspectors = { git = "https://github.com/mantle-xyz/revm-inspectors", tag = "v2.0.0", default-features = false } +# revm = { version = "31.0.2", default-features = false } +# revm-bytecode = { version = "7.1.1", default-features = false } +# revm-database = { version = "9.0.5", default-features = false } +# revm-state = { version = "8.1.1", default-features = false } +# revm-primitives = { version = "21.0.2", default-features = false } +# revm-interpreter = { version = "29.0.1", default-features = false } +# revm-inspector = { version = "12.0.2", default-features = false } +# revm-context = { version = "11.0.2", default-features = false } +# revm-context-interface = { version = "12.0.1", default-features = false } +# revm-database-interface = { version = "8.0.5", default-features = false } +# op-revm = { version = "12.0.2", default-features = false } +# revm-inspectors = "0.32.0" +revm = { git = "https://github.com/mantle-xyz/revm", tag = "v2.1.2", default-features = false } +revm-bytecode = { git = "https://github.com/mantle-xyz/revm", tag = "v2.1.2", default-features = false } +revm-state = { git = "https://github.com/mantle-xyz/revm", tag = "v2.1.2", default-features = false } +revm-primitives = { git = "https://github.com/mantle-xyz/revm", tag = "v2.1.2", default-features = false } +revm-interpreter = { git = "https://github.com/mantle-xyz/revm", tag = "v2.1.2", default-features = false } +revm-inspector = { git = "https://github.com/mantle-xyz/revm", tag = "v2.1.2", default-features = false } +revm-context = { git = "https://github.com/mantle-xyz/revm", tag = "v2.1.2", default-features = false } +revm-context-interface = { git = "https://github.com/mantle-xyz/revm", tag = "v2.1.2", default-features = false } +revm-database = { git = "https://github.com/mantle-xyz/revm", tag = "v2.1.2", default-features = false } +revm-database-interface = { git = "https://github.com/mantle-xyz/revm", tag = "v2.1.2", default-features = false } +op-revm = { git = "https://github.com/mantle-xyz/revm", tag = "v2.1.2", default-features = false } +revm-inspectors = { git = "https://github.com/mantle-xyz/revm-inspectors", tag = "v2.1.1", default-features = false } + # eth -alloy-chains = { version = "0.2.0", default-features = false } -alloy-dyn-abi = "1.1.0" +alloy-chains = { version = "0.2.5", default-features = false } +alloy-dyn-abi = "1.4.1" alloy-eip2124 = { version = "0.2.0", default-features = false } -alloy-evm = { git = "https://github.com/mantle-xyz/evm", tag = "v2.0.1", default-features = false } -alloy-primitives = { version = "1.1.0", default-features = false, features = ["map-foldhash"] } +# alloy-evm = { version = "0.23.0", default-features = false } +alloy-evm = { git = "https://github.com/mantle-xyz/evm", tag = "v2.1.2", default-features = false } +alloy-primitives = { version = "1.4.1", default-features = false, features = ["map-foldhash"] } alloy-rlp = { version = "0.3.10", default-features = false, features = ["core-net"] } -alloy-sol-macro = "1.1.0" -alloy-sol-types = { version = "1.1.0", default-features = false } -alloy-trie = { version = "0.8.1", default-features = false } - -alloy-hardforks = "0.2.0" - -alloy-consensus = { version = "0.15.10", default-features = false } -alloy-contract = { version = "0.15.10", default-features = false } -alloy-eips = { version = "0.15.10", default-features = false } -alloy-genesis = { version = "0.15.10", default-features = false } -alloy-json-rpc = { version = "0.15.10", default-features = false } -alloy-network = { version = "0.15.10", default-features = false } -alloy-network-primitives = { version = "0.15.10", default-features = false } -alloy-provider = { version = "0.15.10", features = ["reqwest"], default-features = false } -alloy-pubsub = { version = "0.15.10", default-features = false } -alloy-rpc-client = { version = "0.15.10", default-features = false } -alloy-rpc-types = { version = "0.15.10", features = ["eth"], default-features = false } -alloy-rpc-types-admin = { version = "0.15.10", default-features = false } -alloy-rpc-types-anvil = { version = "0.15.10", default-features = false } -alloy-rpc-types-beacon = { version = "0.15.10", default-features = false } -alloy-rpc-types-debug = { version = "0.15.10", default-features = false } -alloy-rpc-types-engine = { version = "0.15.10", default-features = false } -alloy-rpc-types-eth = { version = "0.15.10", default-features = false } -alloy-rpc-types-mev = { version = "0.15.10", default-features = false } -alloy-rpc-types-trace = { version = "0.15.10", default-features = false } -alloy-rpc-types-txpool = { version = "0.15.10", default-features = false } -alloy-serde = { version = "0.15.10", default-features = false } -alloy-signer = { version = "0.15.10", default-features = false } -alloy-signer-local = { version = "0.15.10", default-features = false } -alloy-transport = { version = "0.15.10" } -alloy-transport-http = { version = "0.15.10", features = ["reqwest-rustls-tls"], default-features = false } -alloy-transport-ipc = { version = "0.15.10", default-features = false } -alloy-transport-ws = { version = "0.15.10", default-features = false } +alloy-sol-macro = "1.4.1" +alloy-sol-types = { version = "1.4.1", default-features = false } +alloy-trie = { version = "0.9.1", default-features = false } + +alloy-hardforks = "0.4.4" + +alloy-consensus = { version = "1.0.41", default-features = false } +alloy-contract = { version = "1.0.41", default-features = false } +alloy-eips = { version = "1.0.41", default-features = false } +alloy-genesis = { version = "1.0.41", default-features = false } +alloy-json-rpc = { version = "1.0.41", default-features = false } +alloy-network = { version = "1.0.41", default-features = false } +alloy-network-primitives = { version = "1.0.41", default-features = false } +alloy-provider = { version = "1.0.41", features = ["reqwest"], default-features = false } +alloy-pubsub = { version = "1.0.41", default-features = false } +alloy-rpc-client = { version = "1.0.41", default-features = false } +alloy-rpc-types = { version = "1.0.41", features = ["eth"], default-features = false } +alloy-rpc-types-admin = { version = "1.0.41", default-features = false } +alloy-rpc-types-anvil = { version = "1.0.41", default-features = false } +alloy-rpc-types-beacon = { version = "1.0.41", default-features = false } +alloy-rpc-types-debug = { version = "1.0.41", default-features = false } +alloy-rpc-types-engine = { version = "1.0.41", default-features = false } +alloy-rpc-types-eth = { version = "1.0.41", default-features = false } +alloy-rpc-types-mev = { version = "1.0.41", default-features = false } +alloy-rpc-types-trace = { version = "1.0.41", default-features = false } +alloy-rpc-types-txpool = { version = "1.0.41", default-features = false } +alloy-serde = { version = "1.0.41", default-features = false } +alloy-signer = { version = "1.0.41", default-features = false } +alloy-signer-local = { version = "1.0.41", default-features = false } +alloy-transport = { version = "1.0.41" } +alloy-transport-http = { version = "1.0.41", features = ["reqwest-rustls-tls"], default-features = false } +alloy-transport-ipc = { version = "1.0.41", default-features = false } +alloy-transport-ws = { version = "1.0.41", default-features = false } # op -alloy-op-hardforks = "0.2.0" -alloy-op-evm = { git = "https://github.com/mantle-xyz/evm", tag = "v2.0.1", default-features = false } -op-alloy-rpc-types = { git = "https://github.com/mantle-xyz/op-alloy", tag = "v2.0.1", default-features = false } -op-alloy-rpc-types-engine = { git = "https://github.com/mantle-xyz/op-alloy", tag = "v2.0.1", default-features = false } -op-alloy-network = { git = "https://github.com/mantle-xyz/op-alloy", tag = "v2.0.1", default-features = false } -op-alloy-consensus = { git = "https://github.com/mantle-xyz/op-alloy", tag = "v2.0.1", default-features = false } -op-alloy-rpc-jsonrpsee = { git = "https://github.com/mantle-xyz/op-alloy", tag = "v2.0.1", default-features = false } -op-alloy-flz = { version = "0.13.0", default-features = false } +alloy-op-evm = { git = "https://github.com/mantle-xyz/evm", tag = "v2.1.2", default-features = false } +alloy-op-hardforks = "0.4.4" +op-alloy-rpc-types = { git = "https://github.com/mantle-xyz/op-alloy", tag = "v2.1.0", default-features = false } +op-alloy-rpc-types-engine = { git = "https://github.com/mantle-xyz/op-alloy", tag = "v2.1.0", default-features = false } +op-alloy-network = { git = "https://github.com/mantle-xyz/op-alloy", tag = "v2.1.0", default-features = false } +op-alloy-consensus = { git = "https://github.com/mantle-xyz/op-alloy", tag = "v2.1.0", default-features = false } +op-alloy-rpc-jsonrpsee = { git = "https://github.com/mantle-xyz/op-alloy", tag = "v2.1.0", default-features = false } +# alloy-op-evm = { version = "0.23.0", default-features = false } +# op-alloy-rpc-types = { version = "0.22.0", default-features = false } +# op-alloy-rpc-types-engine = { version = "0.22.0", default-features = false } +# op-alloy-network = { version = "0.22.0", default-features = false } +# op-alloy-consensus = { version = "0.22.0", default-features = false } +# op-alloy-rpc-jsonrpsee = { version = "0.22.0", default-features = false } +op-alloy-flz = { version = "0.13.1", default-features = false } # misc +either = { version = "1.15.0", default-features = false } +arrayvec = { version = "0.7.6", default-features = false } aquamarine = "0.6" auto_impl = "1" backon = { version = "1.2", default-features = false, features = ["std-blocking-sleep", "tokio-sleep"] } bincode = "1.3" bitflags = "2.4" -blake3 = "1.5.5" boyer-moore-magiclen = "0.2.16" bytes = { version = "1.5", default-features = false } +brotli = "8" cfg-if = "1.0" clap = "4" dashmap = "6.0" @@ -528,7 +572,8 @@ dirs-next = "2.0.0" dyn-clone = "1.0.17" eyre = "0.6" fdlimit = "0.3.0" -generic-array = "0.14" +# pinned until downstream crypto libs migrate to 1.0 because 0.14.8 marks all types as deprecated +generic-array = "=0.14.7" humantime = "2.1" humantime-serde = "1.1" itertools = { version = "0.14", default-features = false } @@ -536,7 +581,7 @@ linked_hash_set = "0.1" lz4 = "1.28.1" modular-bitfield = "0.11.2" notify = { version = "8.0.0", default-features = false, features = ["macos_fsevent"] } -nybbles = { version = "0.3.0", default-features = false } +nybbles = { version = "0.4.2", default-features = false } once_cell = { version = "1.19", default-features = false, features = ["critical-section"] } parking_lot = "0.12" paste = "1.0" @@ -544,11 +589,12 @@ rand = "0.9" rayon = "1.7" rustc-hash = { version = "2.0", default-features = false } schnellru = "0.2" -serde = { version = "=1.0.215", default-features = false } +serde = { version = "^1.0.226", default-features = false } serde_json = { version = "1.0", default-features = false, features = ["alloc"] } serde_with = { version = "3", default-features = false, features = ["macros"] } sha2 = { version = "0.10", default-features = false } shellexpand = "3.0.0" +shlex = "1.3" smallvec = "1" strum = { version = "0.27", default-features = false } strum_macros = "0.27" @@ -563,6 +609,7 @@ byteorder = "1" mini-moka = "0.10" tar-no-std = { version = "0.3.2", default-features = false } miniz_oxide = { version = "0.8.4", default-features = false } +chrono = "0.4.41" # metrics metrics = "0.24.0" @@ -578,9 +625,11 @@ quote = "1.0" # tokio tokio = { version = "1.44.2", default-features = false } tokio-stream = "0.1.11" +tokio-tungstenite = "0.26.2" tokio-util = { version = "0.7.4", features = ["codec"] } # async +async-compression = { version = "0.4", default-features = false } async-stream = "0.3" async-trait = "0.1.68" futures = "0.3" @@ -591,19 +640,19 @@ hyper-util = "0.1.5" pin-project = "1.0.12" reqwest = { version = "0.12", default-features = false } tracing-futures = "0.2" -tower = "0.4" +tower = "0.5" tower-http = "0.6" # p2p -discv5 = "0.9" -if-addrs = "0.13" +discv5 = "0.10" +if-addrs = "0.14" # rpc -jsonrpsee = "0.24.9" -jsonrpsee-core = "0.24.9" -jsonrpsee-server = "0.24.9" -jsonrpsee-http-client = "0.24.9" -jsonrpsee-types = "0.24.9" +jsonrpsee = "0.26.0" +jsonrpsee-core = "0.26.0" +jsonrpsee-server = "0.26.0" +jsonrpsee-http-client = "0.26.0" +jsonrpsee-types = "0.26.0" # http http = "1.0" @@ -620,19 +669,26 @@ secp256k1 = { version = "0.30", default-features = false, features = ["global-co rand_08 = { package = "rand", version = "0.8" } # for eip-4844 -c-kzg = "2.1.1" +c-kzg = "2.1.5" # config toml = "0.8" +# otlp obs +opentelemetry_sdk = "0.31" +opentelemetry = "0.31" +opentelemetry-otlp = "0.31" +opentelemetry-semantic-conventions = "0.31" +tracing-opentelemetry = "0.32" + # misc-testing arbitrary = "1.3" assert_matches = "1.5.0" criterion = { package = "codspeed-criterion-compat", version = "2.7" } -proptest = "1.4" +proptest = "1.7" proptest-derive = "0.5" similar-asserts = { version = "1.5.0", features = ["serde"] } -tempfile = "3.8" +tempfile = "3.20" test-fuzz = "7" rstest = "0.24.0" test-case = "3" @@ -647,24 +703,19 @@ tikv-jemallocator = "0.6" tracy-client = "0.18.0" snmalloc-rs = { version = "0.3.7", features = ["build_cc"] } -# TODO: When we build for a windows target on an ubuntu runner, crunchy tries to -# get the wrong path, update this when the workflow has been updated -# -# See: https://github.com/eira-fransham/crunchy/issues/13 -crunchy = "=0.2.2" aes = "0.8.1" ahash = "0.8" anyhow = "1.0" -bindgen = { version = "0.70", default-features = false } +bindgen = { version = "0.71", default-features = false } block-padding = "0.3.2" cc = "=1.2.15" cipher = "0.4.3" comfy-table = "7.0" concat-kdf = "0.1.0" -convert_case = "0.7.0" crossbeam-channel = "0.5.13" crossterm = "0.28.0" csv = "1.3.0" +ctrlc = "3.4" ctr = "0.9.2" data-encoding = "2" delegate = "0.13" @@ -756,3 +807,12 @@ vergen-git2 = "1.0.5" # op-alloy-rpc-types-engine = { git = "https://github.com/alloy-rs/op-alloy", rev = "ad607c1" } # # revm-inspectors = { git = "https://github.com/paradigmxyz/revm-inspectors", rev = "1207e33" } +# +# jsonrpsee = { git = "https://github.com/paradigmxyz/jsonrpsee", branch = "matt/make-rpc-service-pub" } +# jsonrpsee-core = { git = "https://github.com/paradigmxyz/jsonrpsee", branch = "matt/make-rpc-service-pub" } +# jsonrpsee-server = { git = "https://github.com/paradigmxyz/jsonrpsee", branch = "matt/make-rpc-service-pub" } +# jsonrpsee-http-client = { git = "https://github.com/paradigmxyz/jsonrpsee", branch = "matt/make-rpc-service-pub" } +# jsonrpsee-types = { git = "https://github.com/paradigmxyz/jsonrpsee", branch = "matt/make-rpc-service-pub" } + +# alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "a69f0b45a6b0286e16072cb8399e02ce6ceca353" } +# alloy-op-evm = { git = "https://github.com/alloy-rs/evm", rev = "a69f0b45a6b0286e16072cb8399e02ce6ceca353" } diff --git a/Cross.toml b/Cross.toml index c48dd846864..9b4fd44f752 100644 --- a/Cross.toml +++ b/Cross.toml @@ -1,12 +1,38 @@ [build] pre-build = [ + # Use HTTPS for package sources + "apt-get update && apt-get install --assume-yes --no-install-recommends ca-certificates", + "find /etc/apt/ -type f \\( -name '*.list' -o -name '*.sources' \\) -exec sed -i 's|http://|https://|g' {} +", + + # Configure APT retries and timeouts to handle network issues + "echo 'Acquire::Retries \"3\";' > /etc/apt/apt.conf.d/80-retries", + "echo 'Acquire::http::Timeout \"60\";' >> /etc/apt/apt.conf.d/80-retries", + "echo 'Acquire::ftp::Timeout \"60\";' >> /etc/apt/apt.conf.d/80-retries", + # rust-bindgen dependencies: llvm-dev libclang-dev (>= 10) clang (>= 10) # See: https://github.com/cross-rs/cross/wiki/FAQ#using-clang--bindgen for # recommended clang versions for the given cross and bindgen version. - "apt-get update && apt-get install --assume-yes --no-install-recommends llvm-dev libclang-10-dev clang-10", + "apt-get update && apt-get install --assume-yes --no-install-recommends llvm-dev libclang-dev clang", ] -[build.env] -passthrough = [ - "JEMALLOC_SYS_WITH_LG_PAGE", +[target.x86_64-pc-windows-gnu] +# Why do we need a custom Dockerfile on Windows: +# 1. `reth-libmdbx` stopped working with MinGW 9.3 that cross image comes with. +# 2. To be able to update the version of MinGW, we need to also update the Ubuntu that the image is based on. +# +# Also see https://github.com/cross-rs/cross/issues/1667 +# Inspired by https://github.com/cross-rs/cross/blob/9e2298e17170655342d3248a9c8ac37ef92ba38f/docker/Dockerfile.x86_64-pc-windows-gnu#L51 +dockerfile = "./Dockerfile.x86_64-pc-windows-gnu" + +[target.riscv64gc-unknown-linux-gnu] +image = "ubuntu:24.04" +pre-build = [ + "apt update", + "apt install --yes gcc gcc-riscv64-linux-gnu libclang-dev make", +] +env.passthrough = [ + "CARGO_TARGET_RISCV64GC_UNKNOWN_LINUX_GNU_LINKER=riscv64-linux-gnu-gcc", ] + +[build.env] +passthrough = ["JEMALLOC_SYS_WITH_LG_PAGE"] diff --git a/Dockerfile b/Dockerfile index beea3301cfe..b61c177525b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,7 @@ LABEL org.opencontainers.image.source=https://github.com/paradigmxyz/reth LABEL org.opencontainers.image.licenses="MIT OR Apache-2.0" # Install system dependencies -RUN apt-get update && apt-get -y upgrade && apt-get install -y libclang-dev pkg-config +RUN apt-get update && apt-get install -y libclang-dev pkg-config # Builds a cargo-chef plan FROM chef AS planner @@ -33,7 +33,7 @@ ENV FEATURES=$FEATURES RUN cargo chef cook --profile $BUILD_PROFILE --features "$FEATURES" --recipe-path recipe.json # Build application -COPY --exclude=.git --exclude=dist . . +COPY --exclude=dist . . RUN cargo build --profile $BUILD_PROFILE --features "$FEATURES" --locked --bin reth # ARG is not resolved in COPY so we have to hack around it by copying the diff --git a/Dockerfile.reproducible b/Dockerfile.reproducible index 26addba287e..a0d4a17b5bb 100644 --- a/Dockerfile.reproducible +++ b/Dockerfile.reproducible @@ -1,8 +1,8 @@ -# Use the Rust 1.86 image based on Debian Bullseye -FROM rust:1.86-bullseye AS builder +# Use the Rust 1.88 image based on Debian Bookworm +FROM rust:1.88-bookworm AS builder # Install specific version of libclang-dev -RUN apt-get update && apt-get install -y libclang-dev=1:11.0-51+nmu5 +RUN apt-get update && apt-get install -y libclang-dev=1:14.0-55.7~deb12u1 # Copy the project to the container COPY ./ /app diff --git a/Dockerfile.x86_64-pc-windows-gnu b/Dockerfile.x86_64-pc-windows-gnu new file mode 100644 index 00000000000..c4611c249ff --- /dev/null +++ b/Dockerfile.x86_64-pc-windows-gnu @@ -0,0 +1,79 @@ +FROM ubuntu:24.04 AS cross-base +ENV DEBIAN_FRONTEND=noninteractive + +# Use HTTPS for package sources +RUN apt-get update && apt-get install --assume-yes --no-install-recommends ca-certificates +RUN find /etc/apt/ -type f \( -name '*.list' -o -name '*.sources' \) -exec sed -i 's|http://|https://|g' {} + + +# Configure APT retries and timeouts to handle network issues +RUN echo 'Acquire::Retries \"3\";' > /etc/apt/apt.conf.d/80-retries && \ + echo 'Acquire::http::Timeout \"60\";' >> /etc/apt/apt.conf.d/80-retries && \ + echo 'Acquire::ftp::Timeout \"60\";' >> /etc/apt/apt.conf.d/80-retries + +# configure fallback mirrors +RUN sed -i 's|URIs: https://archive.ubuntu.com/ubuntu/|URIs: https://mirror.cov.ukservers.com/ubuntu/ https://archive.ubuntu.com/ubuntu/ https://mirror.ox.ac.uk/sites/archive.ubuntu.com/ubuntu/|g' /etc/apt/sources.list.d/ubuntu.sources + +RUN apt-get update && apt-get install --assume-yes --no-install-recommends git + +RUN git clone https://github.com/cross-rs/cross /cross +WORKDIR /cross/docker +RUN git checkout baf457efc2555225af47963475bd70e8d2f5993f + +# xargo doesn't work with Rust 1.89 and higher: https://github.com/cross-rs/cross/issues/1701. +# +# When this PR https://github.com/cross-rs/cross/pull/1580 is merged, +# we can update the checkout above and remove this replacement. +RUN sed -i 's|sh rustup-init.sh -y --no-modify-path --profile minimal|sh rustup-init.sh -y --no-modify-path --profile minimal --default-toolchain=1.88.0|' xargo.sh + +RUN cp common.sh lib.sh / && /common.sh +RUN cp cmake.sh / && /cmake.sh +RUN cp xargo.sh / && /xargo.sh + +FROM cross-base AS build + +RUN apt-get install --assume-yes --no-install-recommends libz-mingw-w64-dev g++-mingw-w64-x86-64 gfortran-mingw-w64-x86-64 + +# Install Wine using OpenSUSE repository because official one is often lagging behind +RUN dpkg --add-architecture i386 && \ + apt-get install --assume-yes --no-install-recommends wget gpg && \ + mkdir -pm755 /etc/apt/keyrings && curl -fsSL \ + https://download.opensuse.org/repositories/Emulators:/Wine:/Debian/xUbuntu_24.04/Release.key \ + | tee /etc/apt/keyrings/obs-winehq.key >/dev/null && \ + echo "deb [arch=amd64,i386 signed-by=/etc/apt/keyrings/obs-winehq.key] \ + https://download.opensuse.org/repositories/Emulators:/Wine:/Debian/xUbuntu_24.04/ ./" \ + | tee /etc/apt/sources.list.d/obs-winehq.list && \ + apt-get update && apt-get install --assume-yes --install-recommends winehq-stable + +# run-detectors are responsible for calling the correct interpreter for exe +# files. For some reason it does not work inside a docker container (it works +# fine in the host). So we replace the usual paths of run-detectors to run wine +# directly. This only affects the guest, we are not messing up with the host. +# +# See /usr/share/doc/binfmt-support/detectors +RUN mkdir -p /usr/lib/binfmt-support/ && \ + rm -f /usr/lib/binfmt-support/run-detectors /usr/bin/run-detectors && \ + ln -s /usr/bin/wine /usr/lib/binfmt-support/run-detectors && \ + ln -s /usr/bin/wine /usr/bin/run-detectors + +RUN cp windows-entry.sh / +ENTRYPOINT ["/windows-entry.sh"] + +RUN cp toolchain.cmake /opt/toolchain.cmake + +# for why we always link with pthread support, see: +# https://github.com/cross-rs/cross/pull/1123#issuecomment-1312287148 +ENV CROSS_TOOLCHAIN_PREFIX=x86_64-w64-mingw32- +ENV CROSS_TOOLCHAIN_SUFFIX=-posix +ENV CROSS_SYSROOT=/usr/x86_64-w64-mingw32 +ENV CROSS_TARGET_RUNNER="env -u CARGO_TARGET_X86_64_PC_WINDOWS_GNU_RUNNER wine" +ENV CARGO_TARGET_X86_64_PC_WINDOWS_GNU_LINKER="$CROSS_TOOLCHAIN_PREFIX"gcc"$CROSS_TOOLCHAIN_SUFFIX" \ + CARGO_TARGET_X86_64_PC_WINDOWS_GNU_RUNNER="$CROSS_TARGET_RUNNER" \ + AR_x86_64_pc_windows_gnu="$CROSS_TOOLCHAIN_PREFIX"ar \ + CC_x86_64_pc_windows_gnu="$CROSS_TOOLCHAIN_PREFIX"gcc"$CROSS_TOOLCHAIN_SUFFIX" \ + CXX_x86_64_pc_windows_gnu="$CROSS_TOOLCHAIN_PREFIX"g++"$CROSS_TOOLCHAIN_SUFFIX" \ + CMAKE_TOOLCHAIN_FILE_x86_64_pc_windows_gnu=/opt/toolchain.cmake \ + BINDGEN_EXTRA_CLANG_ARGS_x86_64_pc_windows_gnu="--sysroot=$CROSS_SYSROOT -idirafter/usr/include" \ + CROSS_CMAKE_SYSTEM_NAME=Windows \ + CROSS_CMAKE_SYSTEM_PROCESSOR=AMD64 \ + CROSS_CMAKE_CRT=gnu \ + CROSS_CMAKE_OBJECT_FLAGS="-ffunction-sections -fdata-sections -m64" diff --git a/DockerfileOp b/DockerfileOp index e7b2bf4f96b..8255debe2d3 100644 --- a/DockerfileOp +++ b/DockerfileOp @@ -6,13 +6,13 @@ LABEL org.opencontainers.image.licenses="MIT OR Apache-2.0" RUN apt-get update && apt-get -y upgrade && apt-get install -y libclang-dev pkg-config +# Builds a cargo-chef plan FROM chef AS planner COPY . . RUN cargo chef prepare --recipe-path recipe.json FROM chef AS builder COPY --from=planner /app/recipe.json recipe.json -COPY . . ARG BUILD_PROFILE=release ENV BUILD_PROFILE=$BUILD_PROFILE @@ -20,10 +20,13 @@ ENV BUILD_PROFILE=$BUILD_PROFILE ARG RUSTFLAGS="" ENV RUSTFLAGS="$RUSTFLAGS" -RUN cargo chef cook --profile $BUILD_PROFILE --recipe-path recipe.json --manifest-path /app/crates/optimism/bin/Cargo.toml +ARG FEATURES="" +ENV FEATURES=$FEATURES + +RUN cargo chef cook --profile $BUILD_PROFILE --features "$FEATURES" --recipe-path recipe.json --manifest-path /app/crates/optimism/bin/Cargo.toml COPY . . -RUN cargo build --profile $BUILD_PROFILE --bin op-reth --manifest-path /app/crates/optimism/bin/Cargo.toml +RUN cargo build --profile $BUILD_PROFILE --features "$FEATURES" --bin op-reth --manifest-path /app/crates/optimism/bin/Cargo.toml RUN ls -la /app/target/$BUILD_PROFILE/op-reth RUN cp /app/target/$BUILD_PROFILE/op-reth /app/op-reth diff --git a/HARDFORK-CHECKLIST.md b/HARDFORK-CHECKLIST.md index fa69107a2c1..0b6361221bb 100644 --- a/HARDFORK-CHECKLIST.md +++ b/HARDFORK-CHECKLIST.md @@ -30,12 +30,12 @@ Opstack tries to be as close to the L1 engine API as much as possible. Isthmus (Prague equivalent) introduced the first deviation from the L1 engine API with an additional field in the `ExecutionPayload`. For this reason the op engine API -has it's own server traits `OpEngineApi`. +has its own server traits `OpEngineApi`. Adding a new versioned endpoint requires the same changes as for L1 just for the dedicated OP types. ### Hardforks -Opstack has dedicated hardkfors (e.g. Isthmus), that can be entirely opstack specific (e.g. Holocene) or can be an L1 +Opstack has dedicated hardforks (e.g. Isthmus), that can be entirely opstack specific (e.g. Holocene) or can be an L1 equivalent hardfork. Since opstack sticks to the L1 header primitive, a new L1 equivalent hardfork also requires new equivalent consensus checks. For this reason these `OpHardfork` must be mapped to L1 `EthereumHardfork`, for example: -`OpHardfork::Isthmus` corresponds to `EthereumHardfork::Prague`. These mappings must be defined in the `ChainSpec`. \ No newline at end of file +`OpHardfork::Isthmus` corresponds to `EthereumHardfork::Prague`. These mappings must be defined in the `ChainSpec`. diff --git a/Makefile b/Makefile index 4ffc126b345..30f6b0aa478 100644 --- a/Makefile +++ b/Makefile @@ -30,6 +30,11 @@ EF_TESTS_TAG := v17.0 EF_TESTS_URL := https://github.com/ethereum/tests/archive/refs/tags/$(EF_TESTS_TAG).tar.gz EF_TESTS_DIR := ./testing/ef-tests/ethereum-tests +# The release tag of https://github.com/ethereum/execution-spec-tests to use for EEST tests +EEST_TESTS_TAG := v4.5.0 +EEST_TESTS_URL := https://github.com/ethereum/execution-spec-tests/releases/download/$(EEST_TESTS_TAG)/fixtures_stable.tar.gz +EEST_TESTS_DIR := ./testing/ef-tests/execution-spec-tests + # The docker image name DOCKER_IMAGE_NAME ?= ghcr.io/paradigmxyz/reth @@ -42,14 +47,14 @@ help: ## Display this help. ##@ Build .PHONY: install -install: ## Build and install the reth binary under `~/.cargo/bin`. +install: ## Build and install the reth binary under `$(CARGO_HOME)/bin`. cargo install --path bin/reth --bin reth --force --locked \ --features "$(FEATURES)" \ --profile "$(PROFILE)" \ $(CARGO_INSTALL_EXTRA_FLAGS) .PHONY: install-op -install-op: ## Build and install the op-reth binary under `~/.cargo/bin`. +install-op: ## Build and install the op-reth binary under `$(CARGO_HOME)/bin`. cargo install --path crates/optimism/bin --bin op-reth --force --locked \ --features "$(FEATURES)" \ --profile "$(PROFILE)" \ @@ -65,7 +70,7 @@ RUST_BUILD_FLAGS = # Enable static linking to ensure reproducibility across builds RUST_BUILD_FLAGS += --C target-feature=+crt-static # Set the linker to use static libgcc to ensure reproducibility across builds -RUST_BUILD_FLAGS += -Clink-arg=-static-libgcc +RUST_BUILD_FLAGS += -C link-arg=-static-libgcc # Remove build ID from the binary to ensure reproducibility across builds RUST_BUILD_FLAGS += -C link-arg=-Wl,--build-id=none # Remove metadata hash from symbol names to ensure reproducible builds @@ -202,9 +207,30 @@ $(EF_TESTS_DIR): tar -xzf ethereum-tests.tar.gz --strip-components=1 -C $(EF_TESTS_DIR) rm ethereum-tests.tar.gz +# Downloads and unpacks EEST tests in the `$(EEST_TESTS_DIR)` directory. +# +# Requires `wget` and `tar` +$(EEST_TESTS_DIR): + mkdir $(EEST_TESTS_DIR) + wget $(EEST_TESTS_URL) -O execution-spec-tests.tar.gz + tar -xzf execution-spec-tests.tar.gz --strip-components=1 -C $(EEST_TESTS_DIR) + rm execution-spec-tests.tar.gz + .PHONY: ef-tests -ef-tests: $(EF_TESTS_DIR) ## Runs Ethereum Foundation tests. - cargo nextest run -p ef-tests --features ef-tests +ef-tests: $(EF_TESTS_DIR) $(EEST_TESTS_DIR) ## Runs Legacy and EEST tests. + cargo nextest run -p ef-tests --release --features ef-tests + +##@ reth-bench + +.PHONY: reth-bench +reth-bench: ## Build the reth-bench binary into the `target` directory. + cargo build --manifest-path bin/reth-bench/Cargo.toml --features "$(FEATURES)" --profile "$(PROFILE)" + +.PHONY: install-reth-bench +install-reth-bench: ## Build and install the reth binary under `$(CARGO_HOME)/bin`. + cargo install --path bin/reth-bench --bin reth-bench --force --locked \ + --features "$(FEATURES)" \ + --profile "$(PROFILE)" ##@ Docker @@ -356,7 +382,7 @@ db-tools: ## Compile MDBX debugging tools. .PHONY: update-book-cli update-book-cli: build-debug ## Update book cli documentation. @echo "Updating book cli doc..." - @./book/cli/update.sh $(CARGO_TARGET_DIR)/debug/reth + @./docs/cli/update.sh $(CARGO_TARGET_DIR)/debug/reth .PHONY: profiling profiling: ## Builds `reth` with optimisations, but also symbols. @@ -392,12 +418,23 @@ clippy: --all-features \ -- -D warnings -lint-codespell: ensure-codespell - codespell --skip "*.json" --skip "./testing/ef-tests/ethereum-tests" +clippy-op-dev: + cargo +nightly clippy \ + --bin op-reth \ + --workspace \ + --lib \ + --examples \ + --tests \ + --benches \ + --locked \ + --all-features + +lint-typos: ensure-typos + typos -ensure-codespell: - @if ! command -v codespell &> /dev/null; then \ - echo "codespell not found. Please install it by running the command `pip install codespell` or refer to the following link for more information: https://github.com/codespell-project/codespell" \ +ensure-typos: + @if ! command -v typos &> /dev/null; then \ + echo "typos not found. Please install it by running the command 'cargo install typos-cli' or refer to the following link for more information: https://github.com/crate-ci/typos"; \ exit 1; \ fi @@ -416,14 +453,14 @@ lint-toml: ensure-dprint ensure-dprint: @if ! command -v dprint &> /dev/null; then \ - echo "dprint not found. Please install it by running the command `cargo install --locked dprint` or refer to the following link for more information: https://github.com/dprint/dprint" \ + echo "dprint not found. Please install it by running the command 'cargo install --locked dprint' or refer to the following link for more information: https://github.com/dprint/dprint"; \ exit 1; \ fi lint: make fmt && \ make clippy && \ - make lint-codespell && \ + make lint-typos && \ make lint-toml clippy-fix: diff --git a/README.md b/README.md index abac066fd16..4f9f63c2d04 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ ![](./assets/reth-prod.png) **[Install](https://paradigmxyz.github.io/reth/installation/installation.html)** -| [User Book](https://reth.rs) +| [User Docs](https://reth.rs) | [Developer Docs](./docs) | [Crate Docs](https://reth.rs/docs) @@ -20,7 +20,7 @@ ## What is Reth? -Reth (short for Rust Ethereum, [pronunciation](https://twitter.com/kelvinfichter/status/1597653609411268608)) is a new Ethereum full node implementation that is focused on being user-friendly, highly modular, as well as being fast and efficient. Reth is an Execution Layer (EL) and is compatible with all Ethereum Consensus Layer (CL) implementations that support the [Engine API](https://github.com/ethereum/execution-apis/tree/a0d03086564ab1838b462befbc083f873dcf0c0f/src/engine). It is originally built and driven forward by [Paradigm](https://paradigm.xyz/), and is licensed under the Apache and MIT licenses. +Reth (short for Rust Ethereum, [pronunciation](https://x.com/kelvinfichter/status/1597653609411268608)) is a new Ethereum full node implementation that is focused on being user-friendly, highly modular, as well as being fast and efficient. Reth is an Execution Layer (EL) and is compatible with all Ethereum Consensus Layer (CL) implementations that support the [Engine API](https://github.com/ethereum/execution-apis/tree/a0d03086564ab1838b462befbc083f873dcf0c0f/src/engine). It is originally built and driven forward by [Paradigm](https://paradigm.xyz/), and is licensed under the Apache and MIT licenses. ## Goals @@ -29,7 +29,7 @@ As a full Ethereum node, Reth allows users to connect to the Ethereum network an More concretely, our goals are: 1. **Modularity**: Every component of Reth is built to be used as a library: well-tested, heavily documented and benchmarked. We envision that developers will import the node's crates, mix and match, and innovate on top of them. Examples of such usage include but are not limited to spinning up standalone P2P networks, talking directly to a node's database, or "unbundling" the node into the components you need. To achieve that, we are licensing Reth under the Apache/MIT permissive license. You can learn more about the project's components [here](./docs/repo/layout.md). -2. **Performance**: Reth aims to be fast, so we used Rust and the [Erigon staged-sync](https://erigon.substack.com/p/erigon-stage-sync-and-control-flows) node architecture. We also use our Ethereum libraries (including [Alloy](https://github.com/alloy-rs/alloy/) and [revm](https://github.com/bluealloy/revm/)) which we’ve battle-tested and optimized via [Foundry](https://github.com/foundry-rs/foundry/). +2. **Performance**: Reth aims to be fast, so we use Rust and the [Erigon staged-sync](https://erigon.substack.com/p/erigon-stage-sync-and-control-flows) node architecture. We also use our Ethereum libraries (including [Alloy](https://github.com/alloy-rs/alloy/) and [revm](https://github.com/bluealloy/revm/)) which we've battle-tested and optimized via [Foundry](https://github.com/foundry-rs/foundry/). 3. **Free for anyone to use any way they want**: Reth is free open source software, built for the community, by the community. By licensing the software under the Apache/MIT license, we want developers to use it without being bound by business licenses, or having to think about the implications of GPL-like licenses. 4. **Client Diversity**: The Ethereum protocol becomes more antifragile when no node implementation dominates. This ensures that if there's a software bug, the network does not finalize a bad block. By building a new client, we hope to contribute to Ethereum's antifragility. 5. **Support as many EVM chains as possible**: We aspire that Reth can full-sync not only Ethereum, but also other chains like Optimism, Polygon, BNB Smart Chain, and more. If you're working on any of these projects, please reach out. @@ -40,17 +40,18 @@ More concretely, our goals are: Reth is production ready, and suitable for usage in mission-critical environments such as staking or high-uptime services. We also actively recommend professional node operators to switch to Reth in production for performance and cost reasons in use cases where high performance with great margins is required such as RPC, MEV, Indexing, Simulations, and P2P activities. More historical context below: -* We released 1.0 "production-ready" stable Reth in June 2024. - * Reth completed an audit with [Sigma Prime](https://sigmaprime.io/), the developers of [Lighthouse](https://github.com/sigp/lighthouse), the Rust Consensus Layer implementation. Find it [here](./audit/sigma_prime_audit_v2.pdf). - * Revm (the EVM used in Reth) underwent an audit with [Guido Vranken](https://twitter.com/guidovranken) (#1 [Ethereum Bug Bounty](https://ethereum.org/en/bug-bounty)). We will publish the results soon. -* We released multiple iterative beta versions, up to [beta.9](https://github.com/paradigmxyz/reth/releases/tag/v0.2.0-beta.9) on Monday June 3rd 2024 the last beta release. -* We released [beta](https://github.com/paradigmxyz/reth/releases/tag/v0.2.0-beta.1) on Monday March 4th 2024, our first breaking change to the database model, providing faster query speed, smaller database footprint, and allowing "history" to be mounted on separate drives. -* We shipped iterative improvements until the last alpha release on February 28th 2024, [0.1.0-alpha.21](https://github.com/paradigmxyz/reth/releases/tag/v0.1.0-alpha.21). -* We [initially announced](https://www.paradigm.xyz/2023/06/reth-alpha) [0.1.0-alpha.1](https://github.com/paradigmxyz/reth/releases/tag/v0.1.0-alpha.1) in June 20th 2023. + +- We released 1.0 "production-ready" stable Reth in June 2024. + - Reth completed an audit with [Sigma Prime](https://sigmaprime.io/), the developers of [Lighthouse](https://github.com/sigp/lighthouse), the Rust Consensus Layer implementation. Find it [here](./audit/sigma_prime_audit_v2.pdf). + - Revm (the EVM used in Reth) underwent an audit with [Guido Vranken](https://x.com/guidovranken) (#1 [Ethereum Bug Bounty](https://ethereum.org/en/bug-bounty)). We will publish the results soon. +- We released multiple iterative beta versions, up to [beta.9](https://github.com/paradigmxyz/reth/releases/tag/v0.2.0-beta.9) on Monday June 3, 2024,the last beta release. +- We released [beta](https://github.com/paradigmxyz/reth/releases/tag/v0.2.0-beta.1) on Monday March 4, 2024, our first breaking change to the database model, providing faster query speed, smaller database footprint, and allowing "history" to be mounted on separate drives. +- We shipped iterative improvements until the last alpha release on February 28, 2024, [0.1.0-alpha.21](https://github.com/paradigmxyz/reth/releases/tag/v0.1.0-alpha.21). +- We [initially announced](https://www.paradigm.xyz/2023/06/reth-alpha) [0.1.0-alpha.1](https://github.com/paradigmxyz/reth/releases/tag/v0.1.0-alpha.1) on June 20, 2023. ### Database compatibility -We do not have any breaking database changes since beta.1, and do not plan any in the near future. +We do not have any breaking database changes since beta.1, and we do not plan any in the near future. Reth [v0.2.0-beta.1](https://github.com/paradigmxyz/reth/releases/tag/v0.2.0-beta.1) includes a [set of breaking database changes](https://github.com/paradigmxyz/reth/pull/5191) that makes it impossible to use database files produced by earlier versions. @@ -60,7 +61,7 @@ If you had a database produced by alpha versions of Reth, you need to drop it wi ## For Users -See the [Reth Book](https://paradigmxyz.github.io/reth) for instructions on how to install and run Reth. +See the [Reth documentation](https://reth.rs/) for instructions on how to install and run Reth. ## For Developers @@ -68,7 +69,7 @@ See the [Reth Book](https://paradigmxyz.github.io/reth) for instructions on how You can use individual crates of reth in your project. -The crate docs can be found [here](https://paradigmxyz.github.io/reth/docs). +The crate docs can be found [here](https://reth.rs/docs/). For a general overview of the crates, see [Project Layout](./docs/repo/layout.md). @@ -76,21 +77,20 @@ For a general overview of the crates, see [Project Layout](./docs/repo/layout.md If you want to contribute, or follow along with contributor discussion, you can use our [main telegram](https://t.me/paradigm_reth) to chat with us about the development of Reth! -- Our contributor guidelines can be found in [`CONTRIBUTING.md`](./CONTRIBUTING.md). -- See our [contributor docs](./docs) for more information on the project. A good starting point is [Project Layout](./docs/repo/layout.md). +- Our contributor guidelines can be found in [`CONTRIBUTING.md`](./CONTRIBUTING.md). +- See our [contributor docs](./docs) for more information on the project. A good starting point is [Project Layout](./docs/repo/layout.md). ### Building and testing -The Minimum Supported Rust Version (MSRV) of this project is [1.86.0](https://blog.rust-lang.org/2025/04/03/Rust-1.86.0/). +The Minimum Supported Rust Version (MSRV) of this project is [1.88.0](https://blog.rust-lang.org/2025/06/26/Rust-1.88.0/). -See the book for detailed instructions on how to [build from source](https://paradigmxyz.github.io/reth/installation/source.html). +See the docs for detailed instructions on how to [build from source](https://reth.rs/installation/source/). To fully test Reth, you will need to have [Geth installed](https://geth.ethereum.org/docs/getting-started/installing-geth), but it is possible to run a subset of tests without Geth. @@ -119,13 +119,13 @@ Using `cargo test` to run tests may work fine, but this is not tested and does n ## Getting Help -If you have any questions, first see if the answer to your question can be found in the [book][book]. +If you have any questions, first see if the answer to your question can be found in the [docs][book]. If the answer is not there: -- Join the [Telegram][tg-url] to get help, or -- Open a [discussion](https://github.com/paradigmxyz/reth/discussions/new) with your question, or -- Open an issue with [the bug](https://github.com/paradigmxyz/reth/issues/new?assignees=&labels=C-bug%2CS-needs-triage&projects=&template=bug.yml) +- Join the [Telegram][tg-url] to get help, or +- Open a [discussion](https://github.com/paradigmxyz/reth/discussions/new) with your question, or +- Open an issue with [the bug](https://github.com/paradigmxyz/reth/issues/new?assignees=&labels=C-bug%2CS-needs-triage&projects=&template=bug.yml) ## Security @@ -137,13 +137,13 @@ Reth is a new implementation of the Ethereum protocol. In the process of develop None of this would have been possible without them, so big shoutout to the teams below: -- [Geth](https://github.com/ethereum/go-ethereum/): We would like to express our heartfelt gratitude to the go-ethereum team for their outstanding contributions to Ethereum over the years. Their tireless efforts and dedication have helped to shape the Ethereum ecosystem and make it the vibrant and innovative community it is today. Thank you for your hard work and commitment to the project. -- [Erigon](https://github.com/ledgerwatch/erigon) (fka Turbo-Geth): Erigon pioneered the ["Staged Sync" architecture](https://erigon.substack.com/p/erigon-stage-sync-and-control-flows) that Reth is using, as well as [introduced MDBX](https://github.com/ledgerwatch/erigon/wiki/Choice-of-storage-engine) as the database of choice. We thank Erigon for pushing the state of the art research on the performance limits of Ethereum nodes. -- [Akula](https://github.com/akula-bft/akula/): Reth uses forks of the Apache versions of Akula's [MDBX Bindings](https://github.com/paradigmxyz/reth/pull/132), [FastRLP](https://github.com/paradigmxyz/reth/pull/63) and [ECIES](https://github.com/paradigmxyz/reth/pull/80) . Given that these packages were already released under the Apache License, and they implement standardized solutions, we decided not to reimplement them to iterate faster. We thank the Akula team for their contributions to the Rust Ethereum ecosystem and for publishing these packages. +- [Geth](https://github.com/ethereum/go-ethereum/): We would like to express our heartfelt gratitude to the go-ethereum team for their outstanding contributions to Ethereum over the years. Their tireless efforts and dedication have helped to shape the Ethereum ecosystem and make it the vibrant and innovative community it is today. Thank you for your hard work and commitment to the project. +- [Erigon](https://github.com/ledgerwatch/erigon) (fka Turbo-Geth): Erigon pioneered the ["Staged Sync" architecture](https://erigon.substack.com/p/erigon-stage-sync-and-control-flows) that Reth is using, as well as [introduced MDBX](https://github.com/ledgerwatch/erigon/wiki/Choice-of-storage-engine) as the database of choice. We thank Erigon for pushing the state of the art research on the performance limits of Ethereum nodes. +- [Akula](https://github.com/akula-bft/akula/): Reth uses forks of the Apache versions of Akula's [MDBX Bindings](https://github.com/paradigmxyz/reth/pull/132), [FastRLP](https://github.com/paradigmxyz/reth/pull/63) and [ECIES](https://github.com/paradigmxyz/reth/pull/80). Given that these packages were already released under the Apache License, and they implement standardized solutions, we decided not to reimplement them to iterate faster. We thank the Akula team for their contributions to the Rust Ethereum ecosystem and for publishing these packages. ## Warning The `NippyJar` and `Compact` encoding formats and their implementations are designed for storing and retrieving data internally. They are not hardened to safely read potentially malicious data. -[book]: https://paradigmxyz.github.io/reth/ +[book]: https://reth.rs/ [tg-url]: https://t.me/paradigm_reth diff --git a/bin/reth-bench-compare/Cargo.toml b/bin/reth-bench-compare/Cargo.toml new file mode 100644 index 00000000000..11d9b4f8bdb --- /dev/null +++ b/bin/reth-bench-compare/Cargo.toml @@ -0,0 +1,96 @@ +[package] +name = "reth-bench-compare" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true +description = "Automated reth benchmark comparison between git references" + +[lints] +workspace = true + +[[bin]] +name = "reth-bench-compare" +path = "src/main.rs" + +[dependencies] +# reth +reth-cli-runner.workspace = true +reth-cli-util.workspace = true +reth-node-core.workspace = true +reth-tracing.workspace = true +reth-chainspec.workspace = true + +# alloy +alloy-provider = { workspace = true, features = ["reqwest-rustls-tls"], default-features = false } +alloy-rpc-types-eth.workspace = true +alloy-primitives.workspace = true + +# CLI and argument parsing +clap = { workspace = true, features = ["derive", "env"] } +eyre.workspace = true + +# Async runtime +tokio = { workspace = true, features = ["full"] } +tracing.workspace = true + +# Serialization +serde = { workspace = true, features = ["derive"] } +serde_json.workspace = true + +# Time handling +chrono = { workspace = true, features = ["serde"] } + +# Path manipulation +shellexpand.workspace = true + +# CSV handling +csv.workspace = true + +# Process management +ctrlc.workspace = true +shlex.workspace = true + +[target.'cfg(unix)'.dependencies] +nix = { version = "0.29", features = ["signal", "process"] } + +[features] +default = ["jemalloc"] + +asm-keccak = [ + "reth-node-core/asm-keccak", + "alloy-primitives/asm-keccak", +] + +jemalloc = [ + "reth-cli-util/jemalloc", + "reth-node-core/jemalloc", +] +jemalloc-prof = ["reth-cli-util/jemalloc-prof"] +tracy-allocator = ["reth-cli-util/tracy-allocator"] + +min-error-logs = [ + "tracing/release_max_level_error", + "reth-node-core/min-error-logs", +] +min-warn-logs = [ + "tracing/release_max_level_warn", + "reth-node-core/min-warn-logs", +] +min-info-logs = [ + "tracing/release_max_level_info", + "reth-node-core/min-info-logs", +] +min-debug-logs = [ + "tracing/release_max_level_debug", + "reth-node-core/min-debug-logs", +] +min-trace-logs = [ + "tracing/release_max_level_trace", + "reth-node-core/min-trace-logs", +] + +# no-op feature flag for switching between the `optimism` and default functionality in CI matrices +ethereum = [] diff --git a/bin/reth-bench-compare/src/benchmark.rs b/bin/reth-bench-compare/src/benchmark.rs new file mode 100644 index 00000000000..e1b971f5792 --- /dev/null +++ b/bin/reth-bench-compare/src/benchmark.rs @@ -0,0 +1,296 @@ +//! Benchmark execution using reth-bench. + +use crate::cli::Args; +use eyre::{eyre, Result, WrapErr}; +use std::{ + path::Path, + sync::{Arc, Mutex}, +}; +use tokio::{ + fs::File as AsyncFile, + io::{AsyncBufReadExt, AsyncWriteExt, BufReader}, + process::Command, +}; +use tracing::{debug, error, info, warn}; + +/// Manages benchmark execution using reth-bench +pub(crate) struct BenchmarkRunner { + rpc_url: String, + jwt_secret: String, + wait_time: Option, + warmup_blocks: u64, +} + +impl BenchmarkRunner { + /// Create a new `BenchmarkRunner` from CLI arguments + pub(crate) fn new(args: &Args) -> Self { + Self { + rpc_url: args.get_rpc_url(), + jwt_secret: args.jwt_secret_path().to_string_lossy().to_string(), + wait_time: args.wait_time.clone(), + warmup_blocks: args.get_warmup_blocks(), + } + } + + /// Clear filesystem caches (page cache, dentries, and inodes) + pub(crate) async fn clear_fs_caches() -> Result<()> { + info!("Clearing filesystem caches..."); + + // First sync to ensure all pending writes are flushed + let sync_output = + Command::new("sync").output().await.wrap_err("Failed to execute sync command")?; + + if !sync_output.status.success() { + return Err(eyre!("sync command failed")); + } + + // Drop caches - requires sudo/root permissions + // 3 = drop pagecache, dentries, and inodes + let drop_caches_cmd = Command::new("sudo") + .args(["-n", "sh", "-c", "echo 3 > /proc/sys/vm/drop_caches"]) + .output() + .await; + + match drop_caches_cmd { + Ok(output) if output.status.success() => { + info!("Successfully cleared filesystem caches"); + Ok(()) + } + Ok(output) => { + let stderr = String::from_utf8_lossy(&output.stderr); + if stderr.contains("sudo: a password is required") { + warn!("Unable to clear filesystem caches: sudo password required"); + warn!( + "For optimal benchmarking, configure passwordless sudo for cache clearing:" + ); + warn!(" echo '$USER ALL=(ALL) NOPASSWD: /bin/sh -c echo\\\\ [0-9]\\\\ \\\\>\\\\ /proc/sys/vm/drop_caches' | sudo tee /etc/sudoers.d/drop_caches"); + Ok(()) + } else { + Err(eyre!("Failed to clear filesystem caches: {}", stderr)) + } + } + Err(e) => { + warn!("Unable to clear filesystem caches: {}", e); + Ok(()) + } + } + } + + /// Run a warmup benchmark for cache warming + pub(crate) async fn run_warmup(&self, from_block: u64) -> Result<()> { + let to_block = from_block + self.warmup_blocks; + info!( + "Running warmup benchmark from block {} to {} ({} blocks)", + from_block, to_block, self.warmup_blocks + ); + + // Build the reth-bench command for warmup (no output flag) + let mut cmd = Command::new("reth-bench"); + cmd.args([ + "new-payload-fcu", + "--rpc-url", + &self.rpc_url, + "--jwt-secret", + &self.jwt_secret, + "--from", + &from_block.to_string(), + "--to", + &to_block.to_string(), + ]); + + // Add wait-time argument if provided + if let Some(ref wait_time) = self.wait_time { + cmd.args(["--wait-time", wait_time]); + } + + cmd.stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .kill_on_drop(true); + + // Set process group for consistent signal handling + #[cfg(unix)] + { + cmd.process_group(0); + } + + debug!("Executing warmup reth-bench command: {:?}", cmd); + + // Execute the warmup benchmark + let mut child = cmd.spawn().wrap_err("Failed to start warmup reth-bench process")?; + + // Stream output at debug level + if let Some(stdout) = child.stdout.take() { + tokio::spawn(async move { + let reader = BufReader::new(stdout); + let mut lines = reader.lines(); + while let Ok(Some(line)) = lines.next_line().await { + debug!("[WARMUP] {}", line); + } + }); + } + + if let Some(stderr) = child.stderr.take() { + tokio::spawn(async move { + let reader = BufReader::new(stderr); + let mut lines = reader.lines(); + while let Ok(Some(line)) = lines.next_line().await { + debug!("[WARMUP] {}", line); + } + }); + } + + let status = child.wait().await.wrap_err("Failed to wait for warmup reth-bench")?; + + if !status.success() { + return Err(eyre!("Warmup reth-bench failed with exit code: {:?}", status.code())); + } + + info!("Warmup completed successfully"); + Ok(()) + } + + /// Run a benchmark for the specified block range + pub(crate) async fn run_benchmark( + &self, + from_block: u64, + to_block: u64, + output_dir: &Path, + ) -> Result<()> { + info!( + "Running benchmark from block {} to {} (output: {:?})", + from_block, to_block, output_dir + ); + + // Ensure output directory exists + std::fs::create_dir_all(output_dir) + .wrap_err_with(|| format!("Failed to create output directory: {output_dir:?}"))?; + + // Create log file path for reth-bench output + let log_file_path = output_dir.join("reth_bench.log"); + info!("reth-bench logs will be saved to: {:?}", log_file_path); + + // Build the reth-bench command + let mut cmd = Command::new("reth-bench"); + cmd.args([ + "new-payload-fcu", + "--rpc-url", + &self.rpc_url, + "--jwt-secret", + &self.jwt_secret, + "--from", + &from_block.to_string(), + "--to", + &to_block.to_string(), + "--output", + &output_dir.to_string_lossy(), + ]); + + // Add wait-time argument if provided + if let Some(ref wait_time) = self.wait_time { + cmd.args(["--wait-time", wait_time]); + } + + cmd.stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .kill_on_drop(true); + + // Set process group for consistent signal handling + #[cfg(unix)] + { + cmd.process_group(0); + } + + // Debug log the command + debug!("Executing reth-bench command: {:?}", cmd); + + // Execute the benchmark + let mut child = cmd.spawn().wrap_err("Failed to start reth-bench process")?; + + // Capture stdout and stderr for error reporting + let stdout_lines = Arc::new(Mutex::new(Vec::new())); + let stderr_lines = Arc::new(Mutex::new(Vec::new())); + + // Stream stdout with prefix at debug level, capture for error reporting, and write to log + // file + if let Some(stdout) = child.stdout.take() { + let stdout_lines_clone = stdout_lines.clone(); + let log_file = AsyncFile::create(&log_file_path) + .await + .wrap_err(format!("Failed to create log file: {:?}", log_file_path))?; + tokio::spawn(async move { + let reader = BufReader::new(stdout); + let mut lines = reader.lines(); + let mut log_file = log_file; + while let Ok(Some(line)) = lines.next_line().await { + debug!("[RETH-BENCH] {}", line); + if let Ok(mut captured) = stdout_lines_clone.lock() { + captured.push(line.clone()); + } + // Write to log file (reth-bench output already has timestamps if needed) + let log_line = format!("{}\n", line); + if let Err(e) = log_file.write_all(log_line.as_bytes()).await { + debug!("Failed to write to log file: {}", e); + } + } + }); + } + + // Stream stderr with prefix at debug level, capture for error reporting, and write to log + // file + if let Some(stderr) = child.stderr.take() { + let stderr_lines_clone = stderr_lines.clone(); + let log_file = AsyncFile::options() + .create(true) + .append(true) + .open(&log_file_path) + .await + .wrap_err(format!("Failed to open log file for stderr: {:?}", log_file_path))?; + tokio::spawn(async move { + let reader = BufReader::new(stderr); + let mut lines = reader.lines(); + let mut log_file = log_file; + while let Ok(Some(line)) = lines.next_line().await { + debug!("[RETH-BENCH] {}", line); + if let Ok(mut captured) = stderr_lines_clone.lock() { + captured.push(line.clone()); + } + // Write to log file (reth-bench output already has timestamps if needed) + let log_line = format!("{}\n", line); + if let Err(e) = log_file.write_all(log_line.as_bytes()).await { + debug!("Failed to write to log file: {}", e); + } + } + }); + } + + let status = child.wait().await.wrap_err("Failed to wait for reth-bench")?; + + if !status.success() { + // Print all captured output when command fails + error!("reth-bench failed with exit code: {:?}", status.code()); + + if let Ok(stdout) = stdout_lines.lock() && + !stdout.is_empty() + { + error!("reth-bench stdout:"); + for line in stdout.iter() { + error!(" {}", line); + } + } + + if let Ok(stderr) = stderr_lines.lock() && + !stderr.is_empty() + { + error!("reth-bench stderr:"); + for line in stderr.iter() { + error!(" {}", line); + } + } + + return Err(eyre!("reth-bench failed with exit code: {:?}", status.code())); + } + + info!("Benchmark completed"); + Ok(()) + } +} diff --git a/bin/reth-bench-compare/src/cli.rs b/bin/reth-bench-compare/src/cli.rs new file mode 100644 index 00000000000..ecb7125c46d --- /dev/null +++ b/bin/reth-bench-compare/src/cli.rs @@ -0,0 +1,931 @@ +//! CLI argument parsing and main command orchestration. + +use alloy_provider::{Provider, ProviderBuilder}; +use clap::Parser; +use eyre::{eyre, Result, WrapErr}; +use reth_chainspec::Chain; +use reth_cli_runner::CliContext; +use reth_node_core::args::{DatadirArgs, LogArgs}; +use reth_tracing::FileWorkerGuard; +use std::{net::TcpListener, path::PathBuf, str::FromStr}; +use tokio::process::Command; +use tracing::{debug, info, warn}; + +use crate::{ + benchmark::BenchmarkRunner, comparison::ComparisonGenerator, compilation::CompilationManager, + git::GitManager, node::NodeManager, +}; + +/// Target for disabling the --debug.startup-sync-state-idle flag +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum DisableStartupSyncStateIdle { + /// Disable for baseline and warmup runs + Baseline, + /// Disable for feature runs only + Feature, + /// Disable for all runs + All, +} + +impl FromStr for DisableStartupSyncStateIdle { + type Err = String; + + fn from_str(s: &str) -> std::result::Result { + match s.to_lowercase().as_str() { + "baseline" => Ok(Self::Baseline), + "feature" => Ok(Self::Feature), + "all" => Ok(Self::All), + _ => Err(format!("Invalid value '{}'. Expected 'baseline', 'feature', or 'all'", s)), + } + } +} + +impl std::fmt::Display for DisableStartupSyncStateIdle { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Baseline => write!(f, "baseline"), + Self::Feature => write!(f, "feature"), + Self::All => write!(f, "all"), + } + } +} + +/// Automated reth benchmark comparison between git references +#[derive(Debug, Parser)] +#[command( + name = "reth-bench-compare", + about = "Compare reth performance between two git references (branches or tags)", + version +)] +pub(crate) struct Args { + /// Git reference (branch or tag) to use as baseline for comparison + #[arg(long, value_name = "REF")] + pub baseline_ref: String, + + /// Git reference (branch or tag) to compare against the baseline + #[arg(long, value_name = "REF")] + pub feature_ref: String, + + #[command(flatten)] + pub datadir: DatadirArgs, + + /// Number of blocks to benchmark + #[arg(long, value_name = "N", default_value = "100")] + pub blocks: u64, + + /// RPC endpoint for fetching block data + #[arg(long, value_name = "URL")] + pub rpc_url: Option, + + /// JWT secret file path + /// + /// If not provided, defaults to `//jwt.hex`. + /// If the file doesn't exist, it will be created automatically. + #[arg(long, value_name = "PATH")] + pub jwt_secret: Option, + + /// Output directory for benchmark results + #[arg(long, value_name = "PATH", default_value = "./reth-bench-compare")] + pub output_dir: String, + + /// Skip git branch validation (useful for testing) + #[arg(long)] + pub skip_git_validation: bool, + + /// Port for reth metrics endpoint + #[arg(long, value_name = "PORT", default_value = "5005")] + pub metrics_port: u16, + + /// The chain this node is running. + /// + /// Possible values are either a built-in chain name or numeric chain ID. + #[arg(long, value_name = "CHAIN", default_value = "mainnet", required = false)] + pub chain: Chain, + + /// Run reth binary with sudo (for elevated privileges) + #[arg(long)] + pub sudo: bool, + + /// Generate comparison charts using Python script + #[arg(long)] + pub draw: bool, + + /// Enable CPU profiling with samply during benchmark runs + #[arg(long)] + pub profile: bool, + + /// Wait time between engine API calls (passed to reth-bench) + #[arg(long, value_name = "DURATION")] + pub wait_time: Option, + + /// Number of blocks to run for cache warmup after clearing caches. + /// If not specified, defaults to the same as --blocks + #[arg(long, value_name = "N")] + pub warmup_blocks: Option, + + /// Disable filesystem cache clearing before warmup phase. + /// By default, filesystem caches are cleared before warmup to ensure consistent benchmarks. + #[arg(long)] + pub no_clear_cache: bool, + + #[command(flatten)] + pub logs: LogArgs, + + /// Additional arguments to pass to baseline reth node command + /// + /// Example: `--baseline-args "--debug.tip 0xabc..."` + #[arg(long, value_name = "ARGS")] + pub baseline_args: Option, + + /// Additional arguments to pass to feature reth node command + /// + /// Example: `--feature-args "--debug.tip 0xdef..."` + #[arg(long, value_name = "ARGS")] + pub feature_args: Option, + + /// Additional arguments to pass to reth node command (applied to both baseline and feature) + /// + /// All arguments after `--` will be passed directly to the reth node command. + /// Example: `reth-bench-compare --baseline-ref main --feature-ref pr/123 -- --debug.tip + /// 0xabc...` + #[arg(trailing_var_arg = true, allow_hyphen_values = true)] + pub reth_args: Vec, + + /// Comma-separated list of features to enable during reth compilation + /// + /// Example: `jemalloc,asm-keccak` + #[arg(long, value_name = "FEATURES", default_value = "jemalloc,asm-keccak")] + pub features: String, + + /// Disable automatic --debug.startup-sync-state-idle flag for specific runs. + /// Can be "baseline", "feature", or "all". + /// By default, the flag is passed to warmup, baseline, and feature runs. + /// When "baseline" is specified, the flag is NOT passed to warmup OR baseline. + /// When "feature" is specified, the flag is NOT passed to feature. + /// When "all" is specified, the flag is NOT passed to any run. + #[arg(long, value_name = "TARGET")] + pub disable_startup_sync_state_idle: Option, +} + +impl Args { + /// Initializes tracing with the configured options. + pub(crate) fn init_tracing(&self) -> Result> { + let guard = self.logs.init_tracing()?; + Ok(guard) + } + + /// Build additional arguments for a specific ref type, conditionally including + /// --debug.startup-sync-state-idle based on the configuration + pub(crate) fn build_additional_args( + &self, + ref_type: &str, + base_args_str: Option<&String>, + ) -> Vec { + // Parse the base arguments string if provided + let mut args = base_args_str.map(|s| parse_args_string(s)).unwrap_or_default(); + + // Determine if we should add the --debug.startup-sync-state-idle flag + let should_add_flag = match self.disable_startup_sync_state_idle { + None => true, // By default, add the flag + Some(DisableStartupSyncStateIdle::All) => false, + Some(DisableStartupSyncStateIdle::Baseline) => { + ref_type != "baseline" && ref_type != "warmup" + } + Some(DisableStartupSyncStateIdle::Feature) => ref_type != "feature", + }; + + if should_add_flag { + args.push("--debug.startup-sync-state-idle".to_string()); + debug!("Adding --debug.startup-sync-state-idle flag for ref_type: {}", ref_type); + } else { + debug!("Skipping --debug.startup-sync-state-idle flag for ref_type: {}", ref_type); + } + + args + } + + /// Get the default RPC URL for a given chain + const fn get_default_rpc_url(chain: &Chain) -> &'static str { + match chain.id() { + 8453 => "https://base-mainnet.rpc.ithaca.xyz", // base + 84532 => "https://base-sepolia.rpc.ithaca.xyz", // base-sepolia + 27082 => "https://rpc.hoodi.ethpandaops.io", // hoodi + _ => "https://reth-ethereum.ithaca.xyz/rpc", // mainnet and fallback + } + } + + /// Get the RPC URL, using chain-specific default if not provided + pub(crate) fn get_rpc_url(&self) -> String { + self.rpc_url.clone().unwrap_or_else(|| Self::get_default_rpc_url(&self.chain).to_string()) + } + + /// Get the JWT secret path - either provided or derived from datadir + pub(crate) fn jwt_secret_path(&self) -> PathBuf { + match &self.jwt_secret { + Some(path) => { + let jwt_secret_str = path.to_string_lossy(); + let expanded = shellexpand::tilde(&jwt_secret_str); + PathBuf::from(expanded.as_ref()) + } + None => { + // Use the same logic as reth: //jwt.hex + let chain_path = self.datadir.clone().resolve_datadir(self.chain); + chain_path.jwt() + } + } + } + + /// Get the resolved datadir path using the chain + pub(crate) fn datadir_path(&self) -> PathBuf { + let chain_path = self.datadir.clone().resolve_datadir(self.chain); + chain_path.data_dir().to_path_buf() + } + + /// Get the expanded output directory path + pub(crate) fn output_dir_path(&self) -> PathBuf { + let expanded = shellexpand::tilde(&self.output_dir); + PathBuf::from(expanded.as_ref()) + } + + /// Get the effective warmup blocks value - either specified or defaults to blocks + pub(crate) fn get_warmup_blocks(&self) -> u64 { + self.warmup_blocks.unwrap_or(self.blocks) + } +} + +/// Validate that the RPC endpoint chain ID matches the specified chain +async fn validate_rpc_chain_id(rpc_url: &str, expected_chain: &Chain) -> Result<()> { + // Create Alloy provider + let url = rpc_url.parse().map_err(|e| eyre!("Invalid RPC URL '{}': {}", rpc_url, e))?; + let provider = ProviderBuilder::new().connect_http(url); + + // Query chain ID using Alloy + let rpc_chain_id = provider + .get_chain_id() + .await + .map_err(|e| eyre!("Failed to get chain ID from RPC endpoint {}: {:?}", rpc_url, e))?; + + let expected_chain_id = expected_chain.id(); + + if rpc_chain_id != expected_chain_id { + return Err(eyre!( + "RPC endpoint chain ID mismatch!\n\ + Expected: {} (chain: {})\n\ + Found: {} at RPC endpoint: {}\n\n\ + Please use an RPC endpoint for the correct network or change the --chain argument.", + expected_chain_id, + expected_chain, + rpc_chain_id, + rpc_url + )); + } + + info!("Validated RPC endpoint chain ID"); + Ok(()) +} + +/// Main comparison workflow execution +pub(crate) async fn run_comparison(args: Args, _ctx: CliContext) -> Result<()> { + // Create a new process group for this process and all its children + #[cfg(unix)] + { + use nix::unistd::{getpid, setpgid}; + if let Err(e) = setpgid(getpid(), getpid()) { + warn!("Failed to create process group: {e}"); + } + } + + info!( + "Starting benchmark comparison between '{}' and '{}'", + args.baseline_ref, args.feature_ref + ); + + if args.sudo { + info!("Running in sudo mode - reth commands will use elevated privileges"); + } + + // Initialize Git manager + let git_manager = GitManager::new()?; + // Fetch all branches, tags, and commits + git_manager.fetch_all()?; + + // Initialize compilation manager + let output_dir = args.output_dir_path(); + let compilation_manager = CompilationManager::new( + git_manager.repo_root().to_string(), + output_dir.clone(), + git_manager.clone(), + args.features.clone(), + )?; + // Initialize node manager + let mut node_manager = NodeManager::new(&args); + + let benchmark_runner = BenchmarkRunner::new(&args); + let mut comparison_generator = ComparisonGenerator::new(&args); + + // Set the comparison directory in node manager to align with results directory + node_manager.set_comparison_dir(comparison_generator.get_output_dir()); + + // Store original git state for restoration + let original_ref = git_manager.get_current_ref()?; + info!("Current git reference: {}", original_ref); + + // Validate git state + if !args.skip_git_validation { + git_manager.validate_clean_state()?; + git_manager.validate_refs(&[&args.baseline_ref, &args.feature_ref])?; + } + + // Validate RPC endpoint chain ID matches the specified chain + let rpc_url = args.get_rpc_url(); + validate_rpc_chain_id(&rpc_url, &args.chain).await?; + + // Setup signal handling for cleanup + let git_manager_cleanup = git_manager.clone(); + let original_ref_cleanup = original_ref.clone(); + ctrlc::set_handler(move || { + eprintln!("Received interrupt signal, cleaning up..."); + + // Send SIGTERM to entire process group to ensure all children exit + #[cfg(unix)] + { + use nix::{ + sys::signal::{kill, Signal}, + unistd::Pid, + }; + + // Send SIGTERM to our process group (negative PID = process group) + let current_pid = std::process::id() as i32; + let pgid = Pid::from_raw(-current_pid); + if let Err(e) = kill(pgid, Signal::SIGTERM) { + eprintln!("Failed to send SIGTERM to process group: {e}"); + } + } + + // Give a moment for any ongoing git operations to complete + std::thread::sleep(std::time::Duration::from_millis(200)); + + if let Err(e) = git_manager_cleanup.switch_ref(&original_ref_cleanup) { + eprintln!("Failed to restore original git reference: {e}"); + eprintln!("You may need to manually run: git checkout {original_ref_cleanup}"); + } + std::process::exit(1); + })?; + + let result = run_benchmark_workflow( + &git_manager, + &compilation_manager, + &mut node_manager, + &benchmark_runner, + &mut comparison_generator, + &args, + ) + .await; + + // Always restore original git reference + info!("Restoring original git reference: {}", original_ref); + git_manager.switch_ref(&original_ref)?; + + // Handle any errors from the workflow + result?; + + Ok(()) +} + +/// Parse a string of arguments into a vector of strings +fn parse_args_string(args_str: &str) -> Vec { + shlex::split(args_str).unwrap_or_else(|| { + // Fallback to simple whitespace splitting if shlex fails + args_str.split_whitespace().map(|s| s.to_string()).collect() + }) +} + +/// Run compilation phase for both baseline and feature binaries +async fn run_compilation_phase( + git_manager: &GitManager, + compilation_manager: &CompilationManager, + args: &Args, + is_optimism: bool, +) -> Result<(String, String)> { + info!("=== Running compilation phase ==="); + + // Ensure required tools are available (only need to check once) + compilation_manager.ensure_reth_bench_available()?; + if args.profile { + compilation_manager.ensure_samply_available()?; + } + + let refs = [&args.baseline_ref, &args.feature_ref]; + let ref_types = ["baseline", "feature"]; + + // First, resolve all refs to commits using a HashMap to avoid race conditions where a ref is + // pushed to mid-run. + let mut ref_commits = std::collections::HashMap::new(); + for &git_ref in &refs { + if !ref_commits.contains_key(git_ref) { + git_manager.switch_ref(git_ref)?; + let commit = git_manager.get_current_commit()?; + ref_commits.insert(git_ref.clone(), commit); + info!("Reference {} resolves to commit: {}", git_ref, &ref_commits[git_ref][..8]); + } + } + + // Now compile each ref using the resolved commits + for (i, &git_ref) in refs.iter().enumerate() { + let ref_type = ref_types[i]; + let commit = &ref_commits[git_ref]; + + info!( + "Compiling {} binary for reference: {} (commit: {})", + ref_type, + git_ref, + &commit[..8] + ); + + // Switch to target reference + git_manager.switch_ref(git_ref)?; + + // Compile reth (with caching) + compilation_manager.compile_reth(commit, is_optimism)?; + + info!("Completed compilation for {} reference", ref_type); + } + + let baseline_commit = ref_commits[&args.baseline_ref].clone(); + let feature_commit = ref_commits[&args.feature_ref].clone(); + + info!("Compilation phase completed"); + Ok((baseline_commit, feature_commit)) +} + +/// Run warmup phase to warm up caches before benchmarking +async fn run_warmup_phase( + git_manager: &GitManager, + compilation_manager: &CompilationManager, + node_manager: &mut NodeManager, + benchmark_runner: &BenchmarkRunner, + args: &Args, + is_optimism: bool, + baseline_commit: &str, +) -> Result<()> { + info!("=== Running warmup phase ==="); + + // Use baseline for warmup + let warmup_ref = &args.baseline_ref; + + // Switch to baseline reference + git_manager.switch_ref(warmup_ref)?; + + // Get the cached binary path for baseline (should already be compiled) + let binary_path = + compilation_manager.get_cached_binary_path_for_commit(baseline_commit, is_optimism); + + // Verify the cached binary exists + if !binary_path.exists() { + return Err(eyre!( + "Cached baseline binary not found at {:?}. Compilation phase should have created it.", + binary_path + )); + } + + info!("Using cached baseline binary for warmup (commit: {})", &baseline_commit[..8]); + + // Build additional args with conditional --debug.startup-sync-state-idle flag + let additional_args = args.build_additional_args("warmup", args.baseline_args.as_ref()); + + // Start reth node for warmup + let mut node_process = + node_manager.start_node(&binary_path, warmup_ref, "warmup", &additional_args).await?; + + // Wait for node to be ready and get its current tip + let current_tip = node_manager.wait_for_node_ready_and_get_tip().await?; + info!("Warmup node is ready at tip: {}", current_tip); + + // Store the tip we'll unwind back to + let original_tip = current_tip; + + // Clear filesystem caches before warmup run only (unless disabled) + if args.no_clear_cache { + info!("Skipping filesystem cache clearing (--no-clear-cache flag set)"); + } else { + BenchmarkRunner::clear_fs_caches().await?; + } + + // Run warmup to warm up caches + benchmark_runner.run_warmup(current_tip).await?; + + // Stop node before unwinding (node must be stopped to release database lock) + node_manager.stop_node(&mut node_process).await?; + + // Unwind back to starting block after warmup + node_manager.unwind_to_block(original_tip).await?; + + info!("Warmup phase completed"); + Ok(()) +} + +/// Execute the complete benchmark workflow for both branches +async fn run_benchmark_workflow( + git_manager: &GitManager, + compilation_manager: &CompilationManager, + node_manager: &mut NodeManager, + benchmark_runner: &BenchmarkRunner, + comparison_generator: &mut ComparisonGenerator, + args: &Args, +) -> Result<()> { + // Detect if this is an Optimism chain once at the beginning + let rpc_url = args.get_rpc_url(); + let is_optimism = compilation_manager.detect_optimism_chain(&rpc_url).await?; + + // Run compilation phase for both binaries + let (baseline_commit, feature_commit) = + run_compilation_phase(git_manager, compilation_manager, args, is_optimism).await?; + + // Run warmup phase before benchmarking (skip if warmup_blocks is 0) + if args.get_warmup_blocks() > 0 { + run_warmup_phase( + git_manager, + compilation_manager, + node_manager, + benchmark_runner, + args, + is_optimism, + &baseline_commit, + ) + .await?; + } else { + info!("Skipping warmup phase (warmup_blocks is 0)"); + } + + let refs = [&args.baseline_ref, &args.feature_ref]; + let ref_types = ["baseline", "feature"]; + let commits = [&baseline_commit, &feature_commit]; + + for (i, &git_ref) in refs.iter().enumerate() { + let ref_type = ref_types[i]; + let commit = commits[i]; + info!("=== Processing {} reference: {} ===", ref_type, git_ref); + + // Switch to target reference + git_manager.switch_ref(git_ref)?; + + // Get the cached binary path for this git reference (should already be compiled) + let binary_path = + compilation_manager.get_cached_binary_path_for_commit(commit, is_optimism); + + // Verify the cached binary exists + if !binary_path.exists() { + return Err(eyre!( + "Cached {} binary not found at {:?}. Compilation phase should have created it.", + ref_type, + binary_path + )); + } + + info!("Using cached {} binary (commit: {})", ref_type, &commit[..8]); + + // Get reference-specific base arguments string + let base_args_str = match ref_type { + "baseline" => args.baseline_args.as_ref(), + "feature" => args.feature_args.as_ref(), + _ => None, + }; + + // Build additional args with conditional --debug.startup-sync-state-idle flag + let additional_args = args.build_additional_args(ref_type, base_args_str); + + // Start reth node + let mut node_process = + node_manager.start_node(&binary_path, git_ref, ref_type, &additional_args).await?; + + // Wait for node to be ready and get its current tip (wherever it is) + let current_tip = node_manager.wait_for_node_ready_and_get_tip().await?; + info!("Node is ready at tip: {}", current_tip); + + // Store the tip we'll unwind back to + let original_tip = current_tip; + + // Calculate benchmark range + // Note: reth-bench has an off-by-one error where it consumes the first block + // of the range, so we add 1 to compensate and get exactly args.blocks blocks + let from_block = original_tip; + let to_block = original_tip + args.blocks; + + // Run benchmark + let output_dir = comparison_generator.get_ref_output_dir(ref_type); + + // Capture start timestamp for the benchmark run + let benchmark_start = chrono::Utc::now(); + + // Run benchmark (comparison logic is handled separately by ComparisonGenerator) + benchmark_runner.run_benchmark(from_block, to_block, &output_dir).await?; + + // Capture end timestamp for the benchmark run + let benchmark_end = chrono::Utc::now(); + + // Stop node + node_manager.stop_node(&mut node_process).await?; + + // Unwind back to original tip + node_manager.unwind_to_block(original_tip).await?; + + // Store results for comparison + comparison_generator.add_ref_results(ref_type, &output_dir)?; + + // Set the benchmark run timestamps + comparison_generator.set_ref_timestamps(ref_type, benchmark_start, benchmark_end)?; + + info!("Completed {} reference benchmark", ref_type); + } + + // Generate comparison report + comparison_generator.generate_comparison_report().await?; + + // Generate charts if requested + if args.draw { + generate_comparison_charts(comparison_generator).await?; + } + + // Start samply servers if profiling was enabled + if args.profile { + start_samply_servers(args).await?; + } + + Ok(()) +} + +/// Generate comparison charts using the Python script +async fn generate_comparison_charts(comparison_generator: &ComparisonGenerator) -> Result<()> { + info!("Generating comparison charts with Python script..."); + + let baseline_output_dir = comparison_generator.get_ref_output_dir("baseline"); + let feature_output_dir = comparison_generator.get_ref_output_dir("feature"); + + let baseline_csv = baseline_output_dir.join("combined_latency.csv"); + let feature_csv = feature_output_dir.join("combined_latency.csv"); + + // Check if CSV files exist + if !baseline_csv.exists() { + return Err(eyre!("Baseline CSV not found: {:?}", baseline_csv)); + } + if !feature_csv.exists() { + return Err(eyre!("Feature CSV not found: {:?}", feature_csv)); + } + + let output_dir = comparison_generator.get_output_dir(); + let chart_output = output_dir.join("latency_comparison.png"); + + let script_path = "bin/reth-bench/scripts/compare_newpayload_latency.py"; + + info!("Running Python comparison script with uv..."); + let mut cmd = Command::new("uv"); + cmd.args([ + "run", + script_path, + &baseline_csv.to_string_lossy(), + &feature_csv.to_string_lossy(), + "-o", + &chart_output.to_string_lossy(), + ]); + + // Set process group for consistent signal handling + #[cfg(unix)] + { + cmd.process_group(0); + } + + let output = cmd.output().await.map_err(|e| { + eyre!("Failed to execute Python script with uv: {}. Make sure uv is installed.", e) + })?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + let stdout = String::from_utf8_lossy(&output.stdout); + return Err(eyre!( + "Python script failed with exit code {:?}:\nstdout: {}\nstderr: {}", + output.status.code(), + stdout, + stderr + )); + } + + let stdout = String::from_utf8_lossy(&output.stdout); + if !stdout.trim().is_empty() { + info!("Python script output:\n{}", stdout); + } + + info!("Comparison chart generated: {:?}", chart_output); + Ok(()) +} + +/// Start samply servers for viewing profiles +async fn start_samply_servers(args: &Args) -> Result<()> { + info!("Starting samply servers for profile viewing..."); + + let output_dir = args.output_dir_path(); + let profiles_dir = output_dir.join("profiles"); + + // Build profile paths + let baseline_profile = profiles_dir.join("baseline.json.gz"); + let feature_profile = profiles_dir.join("feature.json.gz"); + + // Check if profiles exist + if !baseline_profile.exists() { + warn!("Baseline profile not found: {:?}", baseline_profile); + return Ok(()); + } + if !feature_profile.exists() { + warn!("Feature profile not found: {:?}", feature_profile); + return Ok(()); + } + + // Find two consecutive available ports starting from 3000 + let (baseline_port, feature_port) = find_consecutive_ports(3000)?; + info!("Found available ports: {} and {}", baseline_port, feature_port); + + // Get samply path + let samply_path = get_samply_path().await?; + + // Start baseline server + info!("Starting samply server for baseline '{}' on port {}", args.baseline_ref, baseline_port); + let mut baseline_cmd = Command::new(&samply_path); + baseline_cmd + .args(["load", "--port", &baseline_port.to_string(), &baseline_profile.to_string_lossy()]) + .kill_on_drop(true); + + // Set process group for consistent signal handling + #[cfg(unix)] + { + baseline_cmd.process_group(0); + } + + // Conditionally pipe output based on log level + if tracing::enabled!(tracing::Level::DEBUG) { + baseline_cmd.stdout(std::process::Stdio::piped()).stderr(std::process::Stdio::piped()); + } else { + baseline_cmd.stdout(std::process::Stdio::null()).stderr(std::process::Stdio::null()); + } + + // Debug log the command + debug!("Executing samply load command: {:?}", baseline_cmd); + + let mut baseline_child = + baseline_cmd.spawn().wrap_err("Failed to start samply server for baseline")?; + + // Stream baseline samply output if debug logging is enabled + if tracing::enabled!(tracing::Level::DEBUG) { + if let Some(stdout) = baseline_child.stdout.take() { + tokio::spawn(async move { + use tokio::io::{AsyncBufReadExt, BufReader}; + let reader = BufReader::new(stdout); + let mut lines = reader.lines(); + while let Ok(Some(line)) = lines.next_line().await { + debug!("[SAMPLY-BASELINE] {}", line); + } + }); + } + + if let Some(stderr) = baseline_child.stderr.take() { + tokio::spawn(async move { + use tokio::io::{AsyncBufReadExt, BufReader}; + let reader = BufReader::new(stderr); + let mut lines = reader.lines(); + while let Ok(Some(line)) = lines.next_line().await { + debug!("[SAMPLY-BASELINE] {}", line); + } + }); + } + } + + // Start feature server + info!("Starting samply server for feature '{}' on port {}", args.feature_ref, feature_port); + let mut feature_cmd = Command::new(&samply_path); + feature_cmd + .args(["load", "--port", &feature_port.to_string(), &feature_profile.to_string_lossy()]) + .kill_on_drop(true); + + // Set process group for consistent signal handling + #[cfg(unix)] + { + feature_cmd.process_group(0); + } + + // Conditionally pipe output based on log level + if tracing::enabled!(tracing::Level::DEBUG) { + feature_cmd.stdout(std::process::Stdio::piped()).stderr(std::process::Stdio::piped()); + } else { + feature_cmd.stdout(std::process::Stdio::null()).stderr(std::process::Stdio::null()); + } + + // Debug log the command + debug!("Executing samply load command: {:?}", feature_cmd); + + let mut feature_child = + feature_cmd.spawn().wrap_err("Failed to start samply server for feature")?; + + // Stream feature samply output if debug logging is enabled + if tracing::enabled!(tracing::Level::DEBUG) { + if let Some(stdout) = feature_child.stdout.take() { + tokio::spawn(async move { + use tokio::io::{AsyncBufReadExt, BufReader}; + let reader = BufReader::new(stdout); + let mut lines = reader.lines(); + while let Ok(Some(line)) = lines.next_line().await { + debug!("[SAMPLY-FEATURE] {}", line); + } + }); + } + + if let Some(stderr) = feature_child.stderr.take() { + tokio::spawn(async move { + use tokio::io::{AsyncBufReadExt, BufReader}; + let reader = BufReader::new(stderr); + let mut lines = reader.lines(); + while let Ok(Some(line)) = lines.next_line().await { + debug!("[SAMPLY-FEATURE] {}", line); + } + }); + } + } + + // Give servers time to start + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + + // Print access information + println!("\n=== SAMPLY PROFILE SERVERS STARTED ==="); + println!("Baseline '{}': http://127.0.0.1:{}", args.baseline_ref, baseline_port); + println!("Feature '{}': http://127.0.0.1:{}", args.feature_ref, feature_port); + println!("\nOpen the URLs in your browser to view the profiles."); + println!("Press Ctrl+C to stop the servers and exit."); + println!("=========================================\n"); + + // Wait for Ctrl+C or process termination + let ctrl_c = tokio::signal::ctrl_c(); + let baseline_wait = baseline_child.wait(); + let feature_wait = feature_child.wait(); + + tokio::select! { + _ = ctrl_c => { + info!("Received Ctrl+C, shutting down samply servers..."); + } + result = baseline_wait => { + match result { + Ok(status) => info!("Baseline samply server exited with status: {}", status), + Err(e) => warn!("Baseline samply server error: {}", e), + } + } + result = feature_wait => { + match result { + Ok(status) => info!("Feature samply server exited with status: {}", status), + Err(e) => warn!("Feature samply server error: {}", e), + } + } + } + + // Ensure both processes are terminated + let _ = baseline_child.kill().await; + let _ = feature_child.kill().await; + + info!("Samply servers stopped."); + Ok(()) +} + +/// Find two consecutive available ports starting from the given port +fn find_consecutive_ports(start_port: u16) -> Result<(u16, u16)> { + for port in start_port..=65533 { + // Check if both port and port+1 are available + if is_port_available(port) && is_port_available(port + 1) { + return Ok((port, port + 1)); + } + } + Err(eyre!("Could not find two consecutive available ports starting from {}", start_port)) +} + +/// Check if a port is available by attempting to bind to it +fn is_port_available(port: u16) -> bool { + TcpListener::bind(("127.0.0.1", port)).is_ok() +} + +/// Get the absolute path to samply using 'which' command +async fn get_samply_path() -> Result { + let output = Command::new("which") + .arg("samply") + .output() + .await + .wrap_err("Failed to execute 'which samply' command")?; + + if !output.status.success() { + return Err(eyre!("samply not found in PATH")); + } + + let samply_path = String::from_utf8(output.stdout) + .wrap_err("samply path is not valid UTF-8")? + .trim() + .to_string(); + + if samply_path.is_empty() { + return Err(eyre!("which samply returned empty path")); + } + + Ok(samply_path) +} diff --git a/bin/reth-bench-compare/src/comparison.rs b/bin/reth-bench-compare/src/comparison.rs new file mode 100644 index 00000000000..316609569bf --- /dev/null +++ b/bin/reth-bench-compare/src/comparison.rs @@ -0,0 +1,484 @@ +//! Results comparison and report generation. + +use crate::cli::Args; +use chrono::{DateTime, Utc}; +use csv::Reader; +use eyre::{eyre, Result, WrapErr}; +use serde::{Deserialize, Serialize}; +use std::{ + collections::HashMap, + fs, + path::{Path, PathBuf}, +}; +use tracing::{info, warn}; + +/// Manages comparison between baseline and feature reference results +pub(crate) struct ComparisonGenerator { + output_dir: PathBuf, + timestamp: String, + baseline_ref_name: String, + feature_ref_name: String, + baseline_results: Option, + feature_results: Option, +} + +/// Represents the results from a single benchmark run +#[derive(Debug, Clone)] +pub(crate) struct BenchmarkResults { + pub ref_name: String, + pub combined_latency_data: Vec, + pub summary: BenchmarkSummary, + pub start_timestamp: Option>, + pub end_timestamp: Option>, +} + +/// Combined latency CSV row structure +#[derive(Debug, Clone, Deserialize, Serialize)] +pub(crate) struct CombinedLatencyRow { + pub block_number: u64, + pub gas_used: u64, + pub new_payload_latency: u128, +} + +/// Total gas CSV row structure +#[derive(Debug, Clone, Deserialize, Serialize)] +pub(crate) struct TotalGasRow { + pub block_number: u64, + pub gas_used: u64, + pub time: u128, +} + +/// Summary statistics for a benchmark run +#[derive(Debug, Clone, Serialize)] +pub(crate) struct BenchmarkSummary { + pub total_blocks: u64, + pub total_gas_used: u64, + pub total_duration_ms: u128, + pub avg_new_payload_latency_ms: f64, + pub gas_per_second: f64, + pub blocks_per_second: f64, +} + +/// Comparison report between two benchmark runs +#[derive(Debug, Serialize)] +pub(crate) struct ComparisonReport { + pub timestamp: String, + pub baseline: RefInfo, + pub feature: RefInfo, + pub comparison_summary: ComparisonSummary, + pub per_block_comparisons: Vec, +} + +/// Information about a reference in the comparison +#[derive(Debug, Serialize)] +pub(crate) struct RefInfo { + pub ref_name: String, + pub summary: BenchmarkSummary, + pub start_timestamp: Option>, + pub end_timestamp: Option>, +} + +/// Summary of the comparison between references +#[derive(Debug, Serialize)] +pub(crate) struct ComparisonSummary { + pub new_payload_latency_change_percent: f64, + pub gas_per_second_change_percent: f64, + pub blocks_per_second_change_percent: f64, +} + +/// Per-block comparison data +#[derive(Debug, Serialize)] +pub(crate) struct BlockComparison { + pub block_number: u64, + pub baseline_new_payload_latency: u128, + pub feature_new_payload_latency: u128, + pub new_payload_latency_change_percent: f64, +} + +impl ComparisonGenerator { + /// Create a new comparison generator + pub(crate) fn new(args: &Args) -> Self { + let now: DateTime = Utc::now(); + let timestamp = now.format("%Y%m%d_%H%M%S").to_string(); + + Self { + output_dir: args.output_dir_path(), + timestamp, + baseline_ref_name: args.baseline_ref.clone(), + feature_ref_name: args.feature_ref.clone(), + baseline_results: None, + feature_results: None, + } + } + + /// Get the output directory for a specific reference + pub(crate) fn get_ref_output_dir(&self, ref_type: &str) -> PathBuf { + self.output_dir.join("results").join(&self.timestamp).join(ref_type) + } + + /// Get the main output directory for this comparison run + pub(crate) fn get_output_dir(&self) -> PathBuf { + self.output_dir.join("results").join(&self.timestamp) + } + + /// Add benchmark results for a reference + pub(crate) fn add_ref_results(&mut self, ref_type: &str, output_path: &Path) -> Result<()> { + let ref_name = match ref_type { + "baseline" => &self.baseline_ref_name, + "feature" => &self.feature_ref_name, + _ => return Err(eyre!("Unknown reference type: {}", ref_type)), + }; + + let results = self.load_benchmark_results(ref_name, output_path)?; + + match ref_type { + "baseline" => self.baseline_results = Some(results), + "feature" => self.feature_results = Some(results), + _ => return Err(eyre!("Unknown reference type: {}", ref_type)), + } + + info!("Loaded benchmark results for {} reference", ref_type); + + Ok(()) + } + + /// Set the benchmark run timestamps for a reference + pub(crate) fn set_ref_timestamps( + &mut self, + ref_type: &str, + start: DateTime, + end: DateTime, + ) -> Result<()> { + match ref_type { + "baseline" => { + if let Some(ref mut results) = self.baseline_results { + results.start_timestamp = Some(start); + results.end_timestamp = Some(end); + } else { + return Err(eyre!("Baseline results not loaded yet")); + } + } + "feature" => { + if let Some(ref mut results) = self.feature_results { + results.start_timestamp = Some(start); + results.end_timestamp = Some(end); + } else { + return Err(eyre!("Feature results not loaded yet")); + } + } + _ => return Err(eyre!("Unknown reference type: {}", ref_type)), + } + + Ok(()) + } + + /// Generate the final comparison report + pub(crate) async fn generate_comparison_report(&self) -> Result<()> { + info!("Generating comparison report..."); + + let baseline = + self.baseline_results.as_ref().ok_or_else(|| eyre!("Baseline results not loaded"))?; + + let feature = + self.feature_results.as_ref().ok_or_else(|| eyre!("Feature results not loaded"))?; + + // Generate comparison + let comparison_summary = + self.calculate_comparison_summary(&baseline.summary, &feature.summary)?; + let per_block_comparisons = self.calculate_per_block_comparisons(baseline, feature)?; + + let report = ComparisonReport { + timestamp: self.timestamp.clone(), + baseline: RefInfo { + ref_name: baseline.ref_name.clone(), + summary: baseline.summary.clone(), + start_timestamp: baseline.start_timestamp, + end_timestamp: baseline.end_timestamp, + }, + feature: RefInfo { + ref_name: feature.ref_name.clone(), + summary: feature.summary.clone(), + start_timestamp: feature.start_timestamp, + end_timestamp: feature.end_timestamp, + }, + comparison_summary, + per_block_comparisons, + }; + + // Write reports + self.write_comparison_reports(&report).await?; + + // Print summary to console + self.print_comparison_summary(&report); + + Ok(()) + } + + /// Load benchmark results from CSV files + fn load_benchmark_results( + &self, + ref_name: &str, + output_path: &Path, + ) -> Result { + let combined_latency_path = output_path.join("combined_latency.csv"); + let total_gas_path = output_path.join("total_gas.csv"); + + let combined_latency_data = self.load_combined_latency_csv(&combined_latency_path)?; + let total_gas_data = self.load_total_gas_csv(&total_gas_path)?; + + let summary = self.calculate_summary(&combined_latency_data, &total_gas_data)?; + + Ok(BenchmarkResults { + ref_name: ref_name.to_string(), + combined_latency_data, + summary, + start_timestamp: None, + end_timestamp: None, + }) + } + + /// Load combined latency CSV data + fn load_combined_latency_csv(&self, path: &Path) -> Result> { + let mut reader = Reader::from_path(path) + .wrap_err_with(|| format!("Failed to open combined latency CSV: {path:?}"))?; + + let mut rows = Vec::new(); + for result in reader.deserialize() { + let row: CombinedLatencyRow = result + .wrap_err_with(|| format!("Failed to parse combined latency row in {path:?}"))?; + rows.push(row); + } + + if rows.is_empty() { + return Err(eyre!("No data found in combined latency CSV: {:?}", path)); + } + + Ok(rows) + } + + /// Load total gas CSV data + fn load_total_gas_csv(&self, path: &Path) -> Result> { + let mut reader = Reader::from_path(path) + .wrap_err_with(|| format!("Failed to open total gas CSV: {path:?}"))?; + + let mut rows = Vec::new(); + for result in reader.deserialize() { + let row: TotalGasRow = + result.wrap_err_with(|| format!("Failed to parse total gas row in {path:?}"))?; + rows.push(row); + } + + if rows.is_empty() { + return Err(eyre!("No data found in total gas CSV: {:?}", path)); + } + + Ok(rows) + } + + /// Calculate summary statistics for a benchmark run + fn calculate_summary( + &self, + combined_data: &[CombinedLatencyRow], + total_gas_data: &[TotalGasRow], + ) -> Result { + if combined_data.is_empty() || total_gas_data.is_empty() { + return Err(eyre!("Cannot calculate summary for empty data")); + } + + let total_blocks = combined_data.len() as u64; + let total_gas_used: u64 = combined_data.iter().map(|r| r.gas_used).sum(); + + let total_duration_ms = total_gas_data.last().unwrap().time / 1000; // Convert microseconds to milliseconds + + let avg_new_payload_latency_ms: f64 = + combined_data.iter().map(|r| r.new_payload_latency as f64 / 1000.0).sum::() / + total_blocks as f64; + + let total_duration_seconds = total_duration_ms as f64 / 1000.0; + let gas_per_second = if total_duration_seconds > f64::EPSILON { + total_gas_used as f64 / total_duration_seconds + } else { + 0.0 + }; + + let blocks_per_second = if total_duration_seconds > f64::EPSILON { + total_blocks as f64 / total_duration_seconds + } else { + 0.0 + }; + + Ok(BenchmarkSummary { + total_blocks, + total_gas_used, + total_duration_ms, + avg_new_payload_latency_ms, + gas_per_second, + blocks_per_second, + }) + } + + /// Calculate comparison summary between baseline and feature + fn calculate_comparison_summary( + &self, + baseline: &BenchmarkSummary, + feature: &BenchmarkSummary, + ) -> Result { + let calc_percent_change = |baseline: f64, feature: f64| -> f64 { + if baseline.abs() > f64::EPSILON { + ((feature - baseline) / baseline) * 100.0 + } else { + 0.0 + } + }; + + Ok(ComparisonSummary { + new_payload_latency_change_percent: calc_percent_change( + baseline.avg_new_payload_latency_ms, + feature.avg_new_payload_latency_ms, + ), + gas_per_second_change_percent: calc_percent_change( + baseline.gas_per_second, + feature.gas_per_second, + ), + blocks_per_second_change_percent: calc_percent_change( + baseline.blocks_per_second, + feature.blocks_per_second, + ), + }) + } + + /// Calculate per-block comparisons + fn calculate_per_block_comparisons( + &self, + baseline: &BenchmarkResults, + feature: &BenchmarkResults, + ) -> Result> { + let mut baseline_map: HashMap = HashMap::new(); + for row in &baseline.combined_latency_data { + baseline_map.insert(row.block_number, row); + } + + let mut comparisons = Vec::new(); + for feature_row in &feature.combined_latency_data { + if let Some(baseline_row) = baseline_map.get(&feature_row.block_number) { + let calc_percent_change = |baseline: u128, feature: u128| -> f64 { + if baseline > 0 { + ((feature as f64 - baseline as f64) / baseline as f64) * 100.0 + } else { + 0.0 + } + }; + + let comparison = BlockComparison { + block_number: feature_row.block_number, + baseline_new_payload_latency: baseline_row.new_payload_latency, + feature_new_payload_latency: feature_row.new_payload_latency, + new_payload_latency_change_percent: calc_percent_change( + baseline_row.new_payload_latency, + feature_row.new_payload_latency, + ), + }; + comparisons.push(comparison); + } else { + warn!("Block {} not found in baseline data", feature_row.block_number); + } + } + + Ok(comparisons) + } + + /// Write comparison reports to files + async fn write_comparison_reports(&self, report: &ComparisonReport) -> Result<()> { + let report_dir = self.output_dir.join("results").join(&self.timestamp); + fs::create_dir_all(&report_dir) + .wrap_err_with(|| format!("Failed to create report directory: {report_dir:?}"))?; + + // Write JSON report + let json_path = report_dir.join("comparison_report.json"); + let json_content = serde_json::to_string_pretty(report) + .wrap_err("Failed to serialize comparison report to JSON")?; + fs::write(&json_path, json_content) + .wrap_err_with(|| format!("Failed to write JSON report: {json_path:?}"))?; + + // Write CSV report for per-block comparisons + let csv_path = report_dir.join("per_block_comparison.csv"); + let mut writer = csv::Writer::from_path(&csv_path) + .wrap_err_with(|| format!("Failed to create CSV writer: {csv_path:?}"))?; + + for comparison in &report.per_block_comparisons { + writer.serialize(comparison).wrap_err("Failed to write comparison row to CSV")?; + } + writer.flush().wrap_err("Failed to flush CSV writer")?; + + info!("Comparison reports written to: {:?}", report_dir); + Ok(()) + } + + /// Print comparison summary to console + fn print_comparison_summary(&self, report: &ComparisonReport) { + // Parse and format timestamp nicely + let formatted_timestamp = if let Ok(dt) = chrono::DateTime::parse_from_str( + &format!("{} +0000", report.timestamp.replace('_', " ")), + "%Y%m%d %H%M%S %z", + ) { + dt.format("%Y-%m-%d %H:%M:%S UTC").to_string() + } else { + // Fallback to original if parsing fails + report.timestamp.clone() + }; + + println!("\n=== BENCHMARK COMPARISON SUMMARY ==="); + println!("Timestamp: {formatted_timestamp}"); + println!("Baseline: {}", report.baseline.ref_name); + println!("Feature: {}", report.feature.ref_name); + println!(); + + let summary = &report.comparison_summary; + + println!("Performance Changes:"); + println!(" NewPayload Latency: {:+.2}%", summary.new_payload_latency_change_percent); + println!(" Gas/Second: {:+.2}%", summary.gas_per_second_change_percent); + println!(" Blocks/Second: {:+.2}%", summary.blocks_per_second_change_percent); + println!(); + + println!("Baseline Summary:"); + let baseline = &report.baseline.summary; + println!( + " Blocks: {}, Gas: {}, Duration: {:.2}s", + baseline.total_blocks, + baseline.total_gas_used, + baseline.total_duration_ms as f64 / 1000.0 + ); + println!(" Avg NewPayload: {:.2}ms", baseline.avg_new_payload_latency_ms); + if let (Some(start), Some(end)) = + (&report.baseline.start_timestamp, &report.baseline.end_timestamp) + { + println!( + " Started: {}, Ended: {}", + start.format("%Y-%m-%d %H:%M:%S UTC"), + end.format("%Y-%m-%d %H:%M:%S UTC") + ); + } + println!(); + + println!("Feature Summary:"); + let feature = &report.feature.summary; + println!( + " Blocks: {}, Gas: {}, Duration: {:.2}s", + feature.total_blocks, + feature.total_gas_used, + feature.total_duration_ms as f64 / 1000.0 + ); + println!(" Avg NewPayload: {:.2}ms", feature.avg_new_payload_latency_ms); + if let (Some(start), Some(end)) = + (&report.feature.start_timestamp, &report.feature.end_timestamp) + { + println!( + " Started: {}, Ended: {}", + start.format("%Y-%m-%d %H:%M:%S UTC"), + end.format("%Y-%m-%d %H:%M:%S UTC") + ); + } + println!(); + } +} diff --git a/bin/reth-bench-compare/src/compilation.rs b/bin/reth-bench-compare/src/compilation.rs new file mode 100644 index 00000000000..0bd9f70ce64 --- /dev/null +++ b/bin/reth-bench-compare/src/compilation.rs @@ -0,0 +1,354 @@ +//! Compilation operations for reth and reth-bench. + +use crate::git::GitManager; +use alloy_primitives::address; +use alloy_provider::{Provider, ProviderBuilder}; +use eyre::{eyre, Result, WrapErr}; +use std::{fs, path::PathBuf, process::Command}; +use tracing::{debug, error, info, warn}; + +/// Manages compilation operations for reth components +#[derive(Debug)] +pub(crate) struct CompilationManager { + repo_root: String, + output_dir: PathBuf, + git_manager: GitManager, + features: String, +} + +impl CompilationManager { + /// Create a new `CompilationManager` + pub(crate) const fn new( + repo_root: String, + output_dir: PathBuf, + git_manager: GitManager, + features: String, + ) -> Result { + Ok(Self { repo_root, output_dir, git_manager, features }) + } + + /// Detect if the RPC endpoint is an Optimism chain + pub(crate) async fn detect_optimism_chain(&self, rpc_url: &str) -> Result { + info!("Detecting chain type from RPC endpoint..."); + + // Create Alloy provider + let url = rpc_url.parse().map_err(|e| eyre!("Invalid RPC URL '{}': {}", rpc_url, e))?; + let provider = ProviderBuilder::new().connect_http(url); + + // Check for Optimism predeploy at address 0x420000000000000000000000000000000000000F + let is_optimism = !provider + .get_code_at(address!("0x420000000000000000000000000000000000000F")) + .await? + .is_empty(); + + if is_optimism { + info!("Detected Optimism chain"); + } else { + info!("Detected Ethereum chain"); + } + + Ok(is_optimism) + } + + /// Get the path to the cached binary using explicit commit hash + pub(crate) fn get_cached_binary_path_for_commit( + &self, + commit: &str, + is_optimism: bool, + ) -> PathBuf { + let identifier = &commit[..8]; // Use first 8 chars of commit + + let binary_name = if is_optimism { + format!("op-reth_{}", identifier) + } else { + format!("reth_{}", identifier) + }; + + self.output_dir.join("bin").join(binary_name) + } + + /// Compile reth using cargo build and cache the binary + pub(crate) fn compile_reth(&self, commit: &str, is_optimism: bool) -> Result<()> { + // Validate that current git commit matches the expected commit + let current_commit = self.git_manager.get_current_commit()?; + if current_commit != commit { + return Err(eyre!( + "Git commit mismatch! Expected: {}, but currently at: {}", + &commit[..8], + ¤t_commit[..8] + )); + } + + let cached_path = self.get_cached_binary_path_for_commit(commit, is_optimism); + + // Check if cached binary already exists (since path contains commit hash, it's valid) + if cached_path.exists() { + info!("Using cached binary (commit: {})", &commit[..8]); + return Ok(()); + } + + info!("No cached binary found, compiling (commit: {})...", &commit[..8]); + + let binary_name = if is_optimism { "op-reth" } else { "reth" }; + + info!( + "Compiling {} with profiling configuration (commit: {})...", + binary_name, + &commit[..8] + ); + + let mut cmd = Command::new("cargo"); + cmd.arg("build").arg("--profile").arg("profiling"); + + // Add features + cmd.arg("--features").arg(&self.features); + info!("Using features: {}", self.features); + + // Add bin-specific arguments for optimism + if is_optimism { + cmd.arg("--bin") + .arg("op-reth") + .arg("--manifest-path") + .arg("crates/optimism/bin/Cargo.toml"); + } + + cmd.current_dir(&self.repo_root); + + // Set RUSTFLAGS for native CPU optimization + cmd.env("RUSTFLAGS", "-C target-cpu=native"); + + // Debug log the command + debug!("Executing cargo command: {:?}", cmd); + + let output = cmd.output().wrap_err("Failed to execute cargo build command")?; + + // Print stdout and stderr with prefixes at debug level + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + + for line in stdout.lines() { + if !line.trim().is_empty() { + debug!("[CARGO] {}", line); + } + } + + for line in stderr.lines() { + if !line.trim().is_empty() { + debug!("[CARGO] {}", line); + } + } + + if !output.status.success() { + // Print all output when compilation fails + error!("Cargo build failed with exit code: {:?}", output.status.code()); + + if !stdout.trim().is_empty() { + error!("Cargo stdout:"); + for line in stdout.lines() { + error!(" {}", line); + } + } + + if !stderr.trim().is_empty() { + error!("Cargo stderr:"); + for line in stderr.lines() { + error!(" {}", line); + } + } + + return Err(eyre!("Compilation failed with exit code: {:?}", output.status.code())); + } + + info!("{} compilation completed", binary_name); + + // Copy the compiled binary to cache + let source_path = + PathBuf::from(&self.repo_root).join(format!("target/profiling/{}", binary_name)); + if !source_path.exists() { + return Err(eyre!("Compiled binary not found at {:?}", source_path)); + } + + // Create bin directory if it doesn't exist + let bin_dir = self.output_dir.join("bin"); + fs::create_dir_all(&bin_dir).wrap_err("Failed to create bin directory")?; + + // Copy binary to cache + fs::copy(&source_path, &cached_path).wrap_err("Failed to copy binary to cache")?; + + // Make the cached binary executable + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mut perms = fs::metadata(&cached_path)?.permissions(); + perms.set_mode(0o755); + fs::set_permissions(&cached_path, perms)?; + } + + info!("Cached compiled binary at: {:?}", cached_path); + Ok(()) + } + + /// Check if reth-bench is available in PATH + pub(crate) fn is_reth_bench_available(&self) -> bool { + match Command::new("which").arg("reth-bench").output() { + Ok(output) => { + if output.status.success() { + let path = String::from_utf8_lossy(&output.stdout); + info!("Found reth-bench: {}", path.trim()); + true + } else { + false + } + } + Err(_) => false, + } + } + + /// Check if samply is available in PATH + pub(crate) fn is_samply_available(&self) -> bool { + match Command::new("which").arg("samply").output() { + Ok(output) => { + if output.status.success() { + let path = String::from_utf8_lossy(&output.stdout); + info!("Found samply: {}", path.trim()); + true + } else { + false + } + } + Err(_) => false, + } + } + + /// Install samply using cargo + pub(crate) fn install_samply(&self) -> Result<()> { + info!("Installing samply via cargo..."); + + let mut cmd = Command::new("cargo"); + cmd.args(["install", "--locked", "samply"]); + + // Debug log the command + debug!("Executing cargo command: {:?}", cmd); + + let output = cmd.output().wrap_err("Failed to execute cargo install samply command")?; + + // Print stdout and stderr with prefixes at debug level + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + + for line in stdout.lines() { + if !line.trim().is_empty() { + debug!("[CARGO-SAMPLY] {}", line); + } + } + + for line in stderr.lines() { + if !line.trim().is_empty() { + debug!("[CARGO-SAMPLY] {}", line); + } + } + + if !output.status.success() { + // Print all output when installation fails + error!("Cargo install samply failed with exit code: {:?}", output.status.code()); + + if !stdout.trim().is_empty() { + error!("Cargo stdout:"); + for line in stdout.lines() { + error!(" {}", line); + } + } + + if !stderr.trim().is_empty() { + error!("Cargo stderr:"); + for line in stderr.lines() { + error!(" {}", line); + } + } + + return Err(eyre!( + "samply installation failed with exit code: {:?}", + output.status.code() + )); + } + + info!("Samply installation completed"); + Ok(()) + } + + /// Ensure samply is available, installing if necessary + pub(crate) fn ensure_samply_available(&self) -> Result<()> { + if self.is_samply_available() { + Ok(()) + } else { + warn!("samply not found in PATH, installing..."); + self.install_samply() + } + } + + /// Ensure reth-bench is available, compiling if necessary + pub(crate) fn ensure_reth_bench_available(&self) -> Result<()> { + if self.is_reth_bench_available() { + Ok(()) + } else { + warn!("reth-bench not found in PATH, compiling and installing..."); + self.compile_reth_bench() + } + } + + /// Compile and install reth-bench using `make install-reth-bench` + pub(crate) fn compile_reth_bench(&self) -> Result<()> { + info!("Compiling and installing reth-bench..."); + + let mut cmd = Command::new("make"); + cmd.arg("install-reth-bench").current_dir(&self.repo_root); + + // Debug log the command + debug!("Executing make command: {:?}", cmd); + + let output = cmd.output().wrap_err("Failed to execute make install-reth-bench command")?; + + // Print stdout and stderr with prefixes at debug level + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + + for line in stdout.lines() { + if !line.trim().is_empty() { + debug!("[MAKE-BENCH] {}", line); + } + } + + for line in stderr.lines() { + if !line.trim().is_empty() { + debug!("[MAKE-BENCH] {}", line); + } + } + + if !output.status.success() { + // Print all output when compilation fails + error!("Make install-reth-bench failed with exit code: {:?}", output.status.code()); + + if !stdout.trim().is_empty() { + error!("Make stdout:"); + for line in stdout.lines() { + error!(" {}", line); + } + } + + if !stderr.trim().is_empty() { + error!("Make stderr:"); + for line in stderr.lines() { + error!(" {}", line); + } + } + + return Err(eyre!( + "reth-bench compilation failed with exit code: {:?}", + output.status.code() + )); + } + + info!("Reth-bench compilation completed"); + Ok(()) + } +} diff --git a/bin/reth-bench-compare/src/git.rs b/bin/reth-bench-compare/src/git.rs new file mode 100644 index 00000000000..0da82b14018 --- /dev/null +++ b/bin/reth-bench-compare/src/git.rs @@ -0,0 +1,330 @@ +//! Git operations for branch management. + +use eyre::{eyre, Result, WrapErr}; +use std::process::Command; +use tracing::{info, warn}; + +/// Manages git operations for branch switching +#[derive(Debug, Clone)] +pub(crate) struct GitManager { + repo_root: String, +} + +impl GitManager { + /// Create a new `GitManager`, detecting the repository root + pub(crate) fn new() -> Result { + let output = Command::new("git") + .args(["rev-parse", "--show-toplevel"]) + .output() + .wrap_err("Failed to execute git command - is git installed?")?; + + if !output.status.success() { + return Err(eyre!("Not in a git repository or git command failed")); + } + + let repo_root = String::from_utf8(output.stdout) + .wrap_err("Git output is not valid UTF-8")? + .trim() + .to_string(); + + let manager = Self { repo_root }; + info!( + "Detected git repository at: {}, current reference: {}", + manager.repo_root(), + manager.get_current_ref()? + ); + + Ok(manager) + } + + /// Get the current git branch name + pub(crate) fn get_current_branch(&self) -> Result { + let output = Command::new("git") + .args(["branch", "--show-current"]) + .current_dir(&self.repo_root) + .output() + .wrap_err("Failed to get current branch")?; + + if !output.status.success() { + return Err(eyre!("Failed to determine current branch")); + } + + let branch = String::from_utf8(output.stdout) + .wrap_err("Branch name is not valid UTF-8")? + .trim() + .to_string(); + + if branch.is_empty() { + return Err(eyre!("Not on a named branch (detached HEAD?)")); + } + + Ok(branch) + } + + /// Get the current git reference (branch name, tag, or commit hash) + pub(crate) fn get_current_ref(&self) -> Result { + // First try to get branch name + if let Ok(branch) = self.get_current_branch() { + return Ok(branch); + } + + // If not on a branch, check if we're on a tag + let tag_output = Command::new("git") + .args(["describe", "--exact-match", "--tags", "HEAD"]) + .current_dir(&self.repo_root) + .output() + .wrap_err("Failed to check for tag")?; + + if tag_output.status.success() { + let tag = String::from_utf8(tag_output.stdout) + .wrap_err("Tag name is not valid UTF-8")? + .trim() + .to_string(); + return Ok(tag); + } + + // If not on a branch or tag, return the commit hash + let commit_output = Command::new("git") + .args(["rev-parse", "HEAD"]) + .current_dir(&self.repo_root) + .output() + .wrap_err("Failed to get current commit")?; + + if !commit_output.status.success() { + return Err(eyre!("Failed to get current commit hash")); + } + + let commit_hash = String::from_utf8(commit_output.stdout) + .wrap_err("Commit hash is not valid UTF-8")? + .trim() + .to_string(); + + Ok(commit_hash) + } + + /// Check if the git working directory has uncommitted changes to tracked files + pub(crate) fn validate_clean_state(&self) -> Result<()> { + let output = Command::new("git") + .args(["status", "--porcelain"]) + .current_dir(&self.repo_root) + .output() + .wrap_err("Failed to check git status")?; + + if !output.status.success() { + return Err(eyre!("Git status command failed")); + } + + let status_output = + String::from_utf8(output.stdout).wrap_err("Git status output is not valid UTF-8")?; + + // Check for uncommitted changes to tracked files + // Status codes: M = modified, A = added, D = deleted, R = renamed, C = copied, U = updated + // ?? = untracked files (we want to ignore these) + let has_uncommitted_changes = status_output.lines().any(|line| { + if line.len() >= 2 { + let status = &line[0..2]; + // Ignore untracked files (??) and ignored files (!!) + !matches!(status, "??" | "!!") + } else { + false + } + }); + + if has_uncommitted_changes { + warn!("Git working directory has uncommitted changes to tracked files:"); + for line in status_output.lines() { + if line.len() >= 2 && !matches!(&line[0..2], "??" | "!!") { + warn!(" {}", line); + } + } + return Err(eyre!( + "Git working directory has uncommitted changes to tracked files. Please commit or stash changes before running benchmark comparison." + )); + } + + // Check if there are untracked files and log them as info + let untracked_files: Vec<&str> = + status_output.lines().filter(|line| line.starts_with("??")).collect(); + + if !untracked_files.is_empty() { + info!( + "Git working directory has {} untracked files (this is OK)", + untracked_files.len() + ); + } + + info!("Git working directory is clean (no uncommitted changes to tracked files)"); + Ok(()) + } + + /// Fetch all refs from remote to ensure we have latest branches and tags + pub(crate) fn fetch_all(&self) -> Result<()> { + let output = Command::new("git") + .args(["fetch", "--all", "--tags", "--quiet", "--force"]) + .current_dir(&self.repo_root) + .output() + .wrap_err("Failed to fetch latest refs")?; + + if output.status.success() { + info!("Fetched latest refs"); + } else { + let stderr = String::from_utf8_lossy(&output.stderr); + // Only warn if there's actual error content, not just fetch progress + if !stderr.trim().is_empty() && !stderr.contains("-> origin/") { + warn!("Git fetch encountered issues (continuing anyway): {}", stderr); + } + } + + Ok(()) + } + + /// Validate that the specified git references exist (branches, tags, or commits) + pub(crate) fn validate_refs(&self, refs: &[&str]) -> Result<()> { + for &git_ref in refs { + // Try branch first, then tag, then commit + let branch_check = Command::new("git") + .args(["rev-parse", "--verify", &format!("refs/heads/{git_ref}")]) + .current_dir(&self.repo_root) + .output(); + + let tag_check = Command::new("git") + .args(["rev-parse", "--verify", &format!("refs/tags/{git_ref}")]) + .current_dir(&self.repo_root) + .output(); + + let commit_check = Command::new("git") + .args(["rev-parse", "--verify", &format!("{git_ref}^{{commit}}")]) + .current_dir(&self.repo_root) + .output(); + + let found = if let Ok(output) = branch_check && + output.status.success() + { + info!("Validated branch exists: {}", git_ref); + true + } else if let Ok(output) = tag_check && + output.status.success() + { + info!("Validated tag exists: {}", git_ref); + true + } else if let Ok(output) = commit_check && + output.status.success() + { + info!("Validated commit exists: {}", git_ref); + true + } else { + false + }; + + if !found { + return Err(eyre!( + "Git reference '{}' does not exist as branch, tag, or commit", + git_ref + )); + } + } + + Ok(()) + } + + /// Switch to the specified git reference (branch, tag, or commit) + pub(crate) fn switch_ref(&self, git_ref: &str) -> Result<()> { + // First checkout the reference + let output = Command::new("git") + .args(["checkout", git_ref]) + .current_dir(&self.repo_root) + .output() + .wrap_err_with(|| format!("Failed to switch to reference '{git_ref}'"))?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(eyre!("Failed to switch to reference '{}': {}", git_ref, stderr)); + } + + // Check if this is a branch that tracks a remote and pull latest changes + let is_branch = Command::new("git") + .args(["show-ref", "--verify", "--quiet", &format!("refs/heads/{git_ref}")]) + .current_dir(&self.repo_root) + .status() + .map(|s| s.success()) + .unwrap_or(false); + + if is_branch { + // Check if the branch tracks a remote + let tracking_output = Command::new("git") + .args([ + "rev-parse", + "--abbrev-ref", + "--symbolic-full-name", + &format!("{git_ref}@{{upstream}}"), + ]) + .current_dir(&self.repo_root) + .output(); + + if let Ok(output) = tracking_output && + output.status.success() + { + let upstream = String::from_utf8_lossy(&output.stdout).trim().to_string(); + if !upstream.is_empty() && upstream != format!("{git_ref}@{{upstream}}") { + // Branch tracks a remote, pull latest changes + info!("Pulling latest changes for branch: {}", git_ref); + + let pull_output = Command::new("git") + .args(["pull", "--ff-only"]) + .current_dir(&self.repo_root) + .output() + .wrap_err_with(|| { + format!("Failed to pull latest changes for branch '{git_ref}'") + })?; + + if pull_output.status.success() { + info!("Successfully pulled latest changes for branch: {}", git_ref); + } else { + let stderr = String::from_utf8_lossy(&pull_output.stderr); + warn!("Failed to pull latest changes for branch '{}': {}", git_ref, stderr); + // Continue anyway, we'll use whatever version we have + } + } + } + } + + // Verify the checkout succeeded by checking the current commit + let current_commit_output = Command::new("git") + .args(["rev-parse", "HEAD"]) + .current_dir(&self.repo_root) + .output() + .wrap_err("Failed to get current commit")?; + + if !current_commit_output.status.success() { + return Err(eyre!("Failed to verify git checkout")); + } + + info!("Switched to reference: {}", git_ref); + Ok(()) + } + + /// Get the current commit hash + pub(crate) fn get_current_commit(&self) -> Result { + let output = Command::new("git") + .args(["rev-parse", "HEAD"]) + .current_dir(&self.repo_root) + .output() + .wrap_err("Failed to get current commit")?; + + if !output.status.success() { + return Err(eyre!("Failed to get current commit hash")); + } + + let commit_hash = String::from_utf8(output.stdout) + .wrap_err("Commit hash is not valid UTF-8")? + .trim() + .to_string(); + + Ok(commit_hash) + } + + /// Get the repository root path + pub(crate) fn repo_root(&self) -> &str { + &self.repo_root + } +} diff --git a/bin/reth-bench-compare/src/main.rs b/bin/reth-bench-compare/src/main.rs new file mode 100644 index 00000000000..e866afb2509 --- /dev/null +++ b/bin/reth-bench-compare/src/main.rs @@ -0,0 +1,45 @@ +//! # reth-bench-compare +//! +//! Automated tool for comparing reth performance between two git branches. +//! This tool automates the complete workflow of compiling, running, and benchmarking +//! reth on different branches to provide meaningful performance comparisons. + +#![doc( + html_logo_url = "https://raw.githubusercontent.com/paradigmxyz/reth/main/assets/reth-docs.png", + html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256", + issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" +)] +#![cfg_attr(not(test), warn(unused_crate_dependencies))] + +#[global_allocator] +static ALLOC: reth_cli_util::allocator::Allocator = reth_cli_util::allocator::new_allocator(); + +mod benchmark; +mod cli; +mod comparison; +mod compilation; +mod git; +mod node; + +use clap::Parser; +use cli::{run_comparison, Args}; +use eyre::Result; +use reth_cli_runner::CliRunner; + +fn main() -> Result<()> { + // Enable backtraces unless a RUST_BACKTRACE value has already been explicitly provided. + if std::env::var_os("RUST_BACKTRACE").is_none() { + unsafe { + std::env::set_var("RUST_BACKTRACE", "1"); + } + } + + let args = Args::parse(); + + // Initialize tracing + let _guard = args.init_tracing()?; + + // Run until either exit or sigint or sigterm + let runner = CliRunner::try_default_runtime()?; + runner.run_command_until_exit(|ctx| run_comparison(args, ctx)) +} diff --git a/bin/reth-bench-compare/src/node.rs b/bin/reth-bench-compare/src/node.rs new file mode 100644 index 00000000000..01eb9961f9f --- /dev/null +++ b/bin/reth-bench-compare/src/node.rs @@ -0,0 +1,511 @@ +//! Node management for starting, stopping, and controlling reth instances. + +use crate::cli::Args; +use alloy_provider::{Provider, ProviderBuilder}; +use alloy_rpc_types_eth::SyncStatus; +use eyre::{eyre, OptionExt, Result, WrapErr}; +#[cfg(unix)] +use nix::sys::signal::{killpg, Signal}; +#[cfg(unix)] +use nix::unistd::Pid; +use reth_chainspec::Chain; +use std::{fs, path::PathBuf, time::Duration}; +use tokio::{ + fs::File as AsyncFile, + io::{AsyncBufReadExt, AsyncWriteExt, BufReader as AsyncBufReader}, + process::Command, + time::{sleep, timeout}, +}; +use tracing::{debug, info, warn}; + +/// Manages reth node lifecycle and operations +pub(crate) struct NodeManager { + datadir: Option, + metrics_port: u16, + chain: Chain, + use_sudo: bool, + binary_path: Option, + enable_profiling: bool, + output_dir: PathBuf, + additional_reth_args: Vec, + comparison_dir: Option, +} + +impl NodeManager { + /// Create a new `NodeManager` with configuration from CLI args + pub(crate) fn new(args: &Args) -> Self { + Self { + datadir: Some(args.datadir_path().to_string_lossy().to_string()), + metrics_port: args.metrics_port, + chain: args.chain, + use_sudo: args.sudo, + binary_path: None, + enable_profiling: args.profile, + output_dir: args.output_dir_path(), + additional_reth_args: args.reth_args.clone(), + comparison_dir: None, + } + } + + /// Set the comparison directory path for logging + pub(crate) fn set_comparison_dir(&mut self, dir: PathBuf) { + self.comparison_dir = Some(dir); + } + + /// Get the log file path for a given reference type + fn get_log_file_path(&self, ref_type: &str) -> Result { + let comparison_dir = self + .comparison_dir + .as_ref() + .ok_or_eyre("Comparison directory not set. Call set_comparison_dir first.")?; + + // The comparison directory already contains the full path to results/ + let log_dir = comparison_dir.join(ref_type); + + // Create the directory if it doesn't exist + fs::create_dir_all(&log_dir) + .wrap_err(format!("Failed to create log directory: {:?}", log_dir))?; + + let log_file = log_dir.join("reth_node.log"); + Ok(log_file) + } + + /// Get the perf event max sample rate from the system, capped at 10000 + fn get_perf_sample_rate(&self) -> Option { + let perf_rate_file = "/proc/sys/kernel/perf_event_max_sample_rate"; + if let Ok(content) = fs::read_to_string(perf_rate_file) { + let rate_str = content.trim(); + if !rate_str.is_empty() { + if let Ok(system_rate) = rate_str.parse::() { + let capped_rate = std::cmp::min(system_rate, 10000); + info!( + "Detected perf_event_max_sample_rate: {}, using: {}", + system_rate, capped_rate + ); + return Some(capped_rate.to_string()); + } + warn!("Failed to parse perf_event_max_sample_rate: {}", rate_str); + } + } + None + } + + /// Get the absolute path to samply using 'which' command + async fn get_samply_path(&self) -> Result { + let output = Command::new("which") + .arg("samply") + .output() + .await + .wrap_err("Failed to execute 'which samply' command")?; + + if !output.status.success() { + return Err(eyre!("samply not found in PATH")); + } + + let samply_path = String::from_utf8(output.stdout) + .wrap_err("samply path is not valid UTF-8")? + .trim() + .to_string(); + + if samply_path.is_empty() { + return Err(eyre!("which samply returned empty path")); + } + + Ok(samply_path) + } + + /// Build reth arguments as a vector of strings + fn build_reth_args( + &self, + binary_path_str: &str, + additional_args: &[String], + ) -> (Vec, String) { + let mut reth_args = vec![binary_path_str.to_string(), "node".to_string()]; + + // Add chain argument (skip for mainnet as it's the default) + let chain_str = self.chain.to_string(); + if chain_str != "mainnet" { + reth_args.extend_from_slice(&["--chain".to_string(), chain_str.clone()]); + } + + // Add datadir if specified + if let Some(ref datadir) = self.datadir { + reth_args.extend_from_slice(&["--datadir".to_string(), datadir.clone()]); + } + + // Add reth-specific arguments + let metrics_arg = format!("0.0.0.0:{}", self.metrics_port); + reth_args.extend_from_slice(&[ + "--engine.accept-execution-requests-hash".to_string(), + "--metrics".to_string(), + metrics_arg, + "--http".to_string(), + "--http.api".to_string(), + "eth".to_string(), + "--disable-discovery".to_string(), + "--trusted-only".to_string(), + ]); + + // Add any additional arguments passed via command line (common to both baseline and + // feature) + reth_args.extend_from_slice(&self.additional_reth_args); + + // Add reference-specific additional arguments + reth_args.extend_from_slice(additional_args); + + (reth_args, chain_str) + } + + /// Create a command for profiling mode + async fn create_profiling_command( + &self, + ref_type: &str, + reth_args: &[String], + ) -> Result { + // Create profiles directory if it doesn't exist + let profile_dir = self.output_dir.join("profiles"); + fs::create_dir_all(&profile_dir).wrap_err("Failed to create profiles directory")?; + + let profile_path = profile_dir.join(format!("{}.json.gz", ref_type)); + info!("Starting reth node with samply profiling..."); + info!("Profile output: {:?}", profile_path); + + // Get absolute path to samply + let samply_path = self.get_samply_path().await?; + + let mut cmd = if self.use_sudo { + let mut sudo_cmd = Command::new("sudo"); + sudo_cmd.arg(&samply_path); + sudo_cmd + } else { + Command::new(&samply_path) + }; + + // Add samply arguments + cmd.args(["record", "--save-only", "-o", &profile_path.to_string_lossy()]); + + // Add rate argument if available + if let Some(rate) = self.get_perf_sample_rate() { + cmd.args(["--rate", &rate]); + } + + // Add separator and complete reth command + cmd.arg("--"); + cmd.args(reth_args); + + Ok(cmd) + } + + /// Create a command for direct reth execution + fn create_direct_command(&self, reth_args: &[String]) -> Command { + let binary_path = &reth_args[0]; + + if self.use_sudo { + info!("Starting reth node with sudo..."); + let mut cmd = Command::new("sudo"); + cmd.args(reth_args); + cmd + } else { + info!("Starting reth node..."); + let mut cmd = Command::new(binary_path); + cmd.args(&reth_args[1..]); // Skip the binary path since it's the command + cmd + } + } + + /// Start a reth node using the specified binary path and return the process handle + pub(crate) async fn start_node( + &mut self, + binary_path: &std::path::Path, + _git_ref: &str, + ref_type: &str, + additional_args: &[String], + ) -> Result { + // Store the binary path for later use (e.g., in unwind_to_block) + self.binary_path = Some(binary_path.to_path_buf()); + + let binary_path_str = binary_path.to_string_lossy(); + let (reth_args, _) = self.build_reth_args(&binary_path_str, additional_args); + + // Log additional arguments if any + if !self.additional_reth_args.is_empty() { + info!("Using common additional reth arguments: {:?}", self.additional_reth_args); + } + if !additional_args.is_empty() { + info!("Using reference-specific additional reth arguments: {:?}", additional_args); + } + + let mut cmd = if self.enable_profiling { + self.create_profiling_command(ref_type, &reth_args).await? + } else { + self.create_direct_command(&reth_args) + }; + + // Set process group for better signal handling + #[cfg(unix)] + { + cmd.process_group(0); + } + + debug!("Executing reth command: {cmd:?}"); + + let mut child = cmd + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .kill_on_drop(true) // Kill on drop so that on Ctrl-C for parent process we stop all child processes + .spawn() + .wrap_err("Failed to start reth node")?; + + info!( + "Reth node started with PID: {:?} (binary: {})", + child.id().ok_or_eyre("Reth node is not running")?, + binary_path_str + ); + + // Prepare log file path + let log_file_path = self.get_log_file_path(ref_type)?; + info!("Reth node logs will be saved to: {:?}", log_file_path); + + // Stream stdout and stderr with prefixes at debug level and to log file + if let Some(stdout) = child.stdout.take() { + let log_file = AsyncFile::create(&log_file_path) + .await + .wrap_err(format!("Failed to create log file: {:?}", log_file_path))?; + tokio::spawn(async move { + let reader = AsyncBufReader::new(stdout); + let mut lines = reader.lines(); + let mut log_file = log_file; + while let Ok(Some(line)) = lines.next_line().await { + debug!("[RETH] {}", line); + // Write to log file (reth already includes timestamps) + let log_line = format!("{}\n", line); + if let Err(e) = log_file.write_all(log_line.as_bytes()).await { + debug!("Failed to write to log file: {}", e); + } + } + }); + } + + if let Some(stderr) = child.stderr.take() { + let log_file = AsyncFile::options() + .create(true) + .append(true) + .open(&log_file_path) + .await + .wrap_err(format!("Failed to open log file for stderr: {:?}", log_file_path))?; + tokio::spawn(async move { + let reader = AsyncBufReader::new(stderr); + let mut lines = reader.lines(); + let mut log_file = log_file; + while let Ok(Some(line)) = lines.next_line().await { + debug!("[RETH] {}", line); + // Write to log file (reth already includes timestamps) + let log_line = format!("{}\n", line); + if let Err(e) = log_file.write_all(log_line.as_bytes()).await { + debug!("Failed to write to log file: {}", e); + } + } + }); + } + + // Give the node a moment to start up + sleep(Duration::from_secs(5)).await; + + Ok(child) + } + + /// Wait for the node to be ready and return its current tip + pub(crate) async fn wait_for_node_ready_and_get_tip(&self) -> Result { + info!("Waiting for node to be ready and synced..."); + + let max_wait = Duration::from_secs(120); // 2 minutes to allow for sync + let check_interval = Duration::from_secs(2); + let rpc_url = "http://localhost:8545"; + + // Create Alloy provider + let url = rpc_url.parse().map_err(|e| eyre!("Invalid RPC URL '{}': {}", rpc_url, e))?; + let provider = ProviderBuilder::new().connect_http(url); + + timeout(max_wait, async { + loop { + // First check if RPC is up and node is not syncing + match provider.syncing().await { + Ok(sync_result) => { + match sync_result { + SyncStatus::Info(sync_info) => { + debug!("Node is still syncing {sync_info:?}, waiting..."); + } + _ => { + // Node is not syncing, now get the tip + match provider.get_block_number().await { + Ok(tip) => { + info!("Node is ready and not syncing at block: {}", tip); + return Ok(tip); + } + Err(e) => { + debug!("Failed to get block number: {}", e); + } + } + } + } + } + Err(e) => { + debug!("Node RPC not ready yet or failed to check sync status: {}", e); + } + } + + sleep(check_interval).await; + } + }) + .await + .wrap_err("Timed out waiting for node to be ready and synced")? + } + + /// Stop the reth node gracefully + pub(crate) async fn stop_node(&self, child: &mut tokio::process::Child) -> Result<()> { + let pid = child.id().expect("Child process ID should be available"); + + // Check if the process has already exited + match child.try_wait() { + Ok(Some(status)) => { + info!("Reth node (PID: {}) has already exited with status: {:?}", pid, status); + return Ok(()); + } + Ok(None) => { + // Process is still running, proceed to stop it + info!("Stopping process gracefully with SIGINT (PID: {})...", pid); + } + Err(e) => { + return Err(eyre!("Failed to check process status: {}", e)); + } + } + + #[cfg(unix)] + { + // Send SIGINT to process group to mimic Ctrl-C behavior + let nix_pgid = Pid::from_raw(pid as i32); + + match killpg(nix_pgid, Signal::SIGINT) { + Ok(()) => {} + Err(nix::errno::Errno::ESRCH) => { + info!("Process group {} has already exited", pid); + } + Err(e) => { + return Err(eyre!("Failed to send SIGINT to process group {}: {}", pid, e)); + } + } + } + + #[cfg(not(unix))] + { + // On non-Unix systems, fall back to using external kill command + let output = Command::new("taskkill") + .args(["/PID", &pid.to_string(), "/F"]) + .output() + .await + .wrap_err("Failed to execute taskkill command")?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + // Check if the error is because the process doesn't exist + if stderr.contains("not found") || stderr.contains("not exist") { + info!("Process {} has already exited", pid); + } else { + return Err(eyre!("Failed to kill process {}: {}", pid, stderr)); + } + } + } + + // Wait for the process to exit + match child.wait().await { + Ok(status) => { + info!("Reth node (PID: {}) exited with status: {:?}", pid, status); + } + Err(e) => { + // If we get an error here, it might be because the process already exited + debug!("Error waiting for process exit (may have already exited): {}", e); + } + } + + Ok(()) + } + + /// Unwind the node to a specific block + pub(crate) async fn unwind_to_block(&self, block_number: u64) -> Result<()> { + if self.use_sudo { + info!("Unwinding node to block: {} (with sudo)", block_number); + } else { + info!("Unwinding node to block: {}", block_number); + } + + // Use the binary path from the last start_node call, or fallback to default + let binary_path = self + .binary_path + .as_ref() + .map(|p| p.to_string_lossy().to_string()) + .unwrap_or_else(|| "./target/profiling/reth".to_string()); + + let mut cmd = if self.use_sudo { + let mut sudo_cmd = Command::new("sudo"); + sudo_cmd.args([&binary_path, "stage", "unwind"]); + sudo_cmd + } else { + let mut reth_cmd = Command::new(&binary_path); + reth_cmd.args(["stage", "unwind"]); + reth_cmd + }; + + // Add chain argument (skip for mainnet as it's the default) + let chain_str = self.chain.to_string(); + if chain_str != "mainnet" { + cmd.args(["--chain", &chain_str]); + } + + // Add datadir if specified + if let Some(ref datadir) = self.datadir { + cmd.args(["--datadir", datadir]); + } + + cmd.args(["to-block", &block_number.to_string()]); + + // Debug log the command + debug!("Executing reth unwind command: {:?}", cmd); + + let mut child = cmd + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .wrap_err("Failed to start unwind command")?; + + // Stream stdout and stderr with prefixes in real-time + if let Some(stdout) = child.stdout.take() { + tokio::spawn(async move { + let reader = AsyncBufReader::new(stdout); + let mut lines = reader.lines(); + while let Ok(Some(line)) = lines.next_line().await { + debug!("[RETH-UNWIND] {}", line); + } + }); + } + + if let Some(stderr) = child.stderr.take() { + tokio::spawn(async move { + let reader = AsyncBufReader::new(stderr); + let mut lines = reader.lines(); + while let Ok(Some(line)) = lines.next_line().await { + debug!("[RETH-UNWIND] {}", line); + } + }); + } + + // Wait for the command to complete + let status = child.wait().await.wrap_err("Failed to wait for unwind command")?; + + if !status.success() { + return Err(eyre!("Unwind command failed with exit code: {:?}", status.code())); + } + + info!("Unwound to block: {}", block_number); + Ok(()) + } +} diff --git a/bin/reth-bench/Cargo.toml b/bin/reth-bench/Cargo.toml index 640c582b7f4..a07d0f5200e 100644 --- a/bin/reth-bench/Cargo.toml +++ b/bin/reth-bench/Cargo.toml @@ -35,6 +35,7 @@ alloy-transport-ipc.workspace = true alloy-transport-ws.workspace = true alloy-transport.workspace = true op-alloy-consensus = { workspace = true, features = ["alloy-compat"] } +op-alloy-rpc-types-engine = { workspace = true, features = ["serde"] } # reqwest reqwest = { workspace = true, default-features = false, features = ["rustls-tls-native-roots"] } @@ -64,7 +65,6 @@ humantime.workspace = true csv.workspace = true [dev-dependencies] -reth-tracing.workspace = true [features] default = ["jemalloc"] @@ -81,11 +81,26 @@ jemalloc = [ jemalloc-prof = ["reth-cli-util/jemalloc-prof"] tracy-allocator = ["reth-cli-util/tracy-allocator"] -min-error-logs = ["tracing/release_max_level_error"] -min-warn-logs = ["tracing/release_max_level_warn"] -min-info-logs = ["tracing/release_max_level_info"] -min-debug-logs = ["tracing/release_max_level_debug"] -min-trace-logs = ["tracing/release_max_level_trace"] +min-error-logs = [ + "tracing/release_max_level_error", + "reth-node-core/min-error-logs", +] +min-warn-logs = [ + "tracing/release_max_level_warn", + "reth-node-core/min-warn-logs", +] +min-info-logs = [ + "tracing/release_max_level_info", + "reth-node-core/min-info-logs", +] +min-debug-logs = [ + "tracing/release_max_level_debug", + "reth-node-core/min-debug-logs", +] +min-trace-logs = [ + "tracing/release_max_level_trace", + "reth-node-core/min-trace-logs", +] # no-op feature flag for switching between the `optimism` and default functionality in CI matrices ethereum = [] diff --git a/bin/reth-bench/README.md b/bin/reth-bench/README.md index 3f7ae7f0377..b8176749fc7 100644 --- a/bin/reth-bench/README.md +++ b/bin/reth-bench/README.md @@ -49,7 +49,7 @@ reth stage unwind to-block 21000000 The following `reth-bench` command would then start the benchmark at block 21,000,000: ```bash -reth-bench new-payload-fcu --rpc-url --from 21000000 --to --jwtsecret +reth-bench new-payload-fcu --rpc-url --from 21000000 --to --jwt-secret ``` Finally, make sure that reth is built using a build profile suitable for what you are trying to measure. @@ -80,11 +80,11 @@ RUSTFLAGS="-C target-cpu=native" cargo build --profile profiling --no-default-fe ### Run the Benchmark: First, start the reth node. Here is an example that runs `reth` compiled with the `profiling` profile, runs `samply`, and configures `reth` to run with metrics enabled: ```bash -samply record -p 3001 target/profiling/reth node --metrics localhost:9001 --authrpc.jwtsecret +samply record -p 3001 target/profiling/reth node --metrics localhost:9001 --authrpc.jwt-secret ``` ```bash -reth-bench new-payload-fcu --rpc-url --from --to --jwtsecret +reth-bench new-payload-fcu --rpc-url --from --to --jwt-secret ``` Replace ``, ``, and `` with the appropriate values for your testing environment. `` should be the URL of an RPC endpoint that can provide the blocks that will be used during the execution. @@ -92,6 +92,18 @@ This should NOT be the node that is being used for the benchmark. The node behin the benchmark. The node being benchmarked will not have these blocks. Note that this assumes that the benchmark node's engine API is running on `http://127.0.0.1:8551`, which is set as a default value in `reth-bench`. To configure this value, use the `--engine-rpc-url` flag. +#### Using the `--advance` argument + +The `--advance` argument allows you to benchmark a relative number of blocks from the current head, without manually specifying `--from` and `--to`. + +```bash +# Benchmark the next 10 blocks from the current head +reth-bench new-payload-fcu --advance 10 --jwt-secret --rpc-url + +# Benchmark the next 50 blocks with a different subcommand +reth-bench new-payload-only --advance 50 --jwt-secret --rpc-url +``` + ### Observe Outputs After running the command, `reth-bench` will output benchmark results, showing processing speeds and gas usage, which are useful metrics for analyzing the node's performance. diff --git a/bin/reth-bench/scripts/compare_newpayload_latency.py b/bin/reth-bench/scripts/compare_newpayload_latency.py new file mode 100755 index 00000000000..f434d034b9a --- /dev/null +++ b/bin/reth-bench/scripts/compare_newpayload_latency.py @@ -0,0 +1,384 @@ +#!/usr/bin/env -S uv run +# /// script +# requires-python = ">=3.8" +# dependencies = [ +# "pandas", +# "matplotlib", +# "numpy", +# ] +# /// + +# A simple script which plots graphs comparing two combined_latency.csv files +# output by reth-bench. The graphs which are plotted are: +# +# - A histogram of the percent change between latencies, bucketed by 1% +# increments. +# +# - A simple line graph plotting the latencies of the two files against each +# other. +# +# - A gas per second (gas/s) chart showing throughput over time. + + +import argparse +import pandas as pd +import matplotlib.pyplot as plt +import numpy as np +import sys +import os +from matplotlib.ticker import FuncFormatter + +def get_output_filename(base_path, suffix=None): + """Generate output filename with optional suffix.""" + if suffix is None: + return base_path + + # Split the base path into directory, name, and extension + dir_name = os.path.dirname(base_path) + base_name = os.path.basename(base_path) + name, ext = os.path.splitext(base_name) + + # Create new filename with suffix + new_name = f"{name}_{suffix}{ext}" + return os.path.join(dir_name, new_name) if dir_name else new_name + +def format_gas_units(value, pos): + """Format gas values with appropriate units (gas, Kgas, Mgas, Ggas, Tgas).""" + if value == 0: + return '0' + + # Define unit thresholds and labels + units = [ + (1e12, 'Tgas'), # Teragas + (1e9, 'Ggas'), # Gigagas + (1e6, 'Mgas'), # Megagas + (1e3, 'Kgas'), # Kilogas + (1, 'gas') # gas + ] + + abs_value = abs(value) + for threshold, unit in units: + if abs_value >= threshold: + scaled_value = value / threshold + # Format with appropriate precision + if scaled_value >= 100: + return f'{scaled_value:.0f}{unit}/s' + elif scaled_value >= 10: + return f'{scaled_value:.1f}{unit}/s' + else: + return f'{scaled_value:.2f}{unit}/s' + + return f'{value:.0f}gas/s' + +def moving_average(data, window_size): + """Calculate moving average with given window size.""" + if window_size <= 1: + return data + + # Use pandas for efficient rolling mean calculation + series = pd.Series(data) + return series.rolling(window=window_size, center=True, min_periods=1).mean().values + +def main(): + parser = argparse.ArgumentParser(description='Generate histogram of total_latency percent differences between two CSV files') + parser.add_argument('baseline_csv', help='First CSV file, used as the baseline/control') + parser.add_argument('comparison_csv', help='Second CSV file, which is being compared to the baseline') + parser.add_argument('-o', '--output', default='latency.png', help='Output image file (default: latency.png)') + parser.add_argument('--graphs', default='all', help='Comma-separated list of graphs to plot: histogram, line, gas, all (default: all)') + parser.add_argument('--average', type=int, metavar='N', help='Apply moving average over N blocks to smooth line and gas charts') + parser.add_argument('--separate', action='store_true', help='Output each chart as a separate file') + + args = parser.parse_args() + + # Parse graph selection + if args.graphs.lower() == 'all': + selected_graphs = {'histogram', 'line', 'gas'} + else: + selected_graphs = set(graph.strip().lower() for graph in args.graphs.split(',')) + valid_graphs = {'histogram', 'line', 'gas'} + invalid_graphs = selected_graphs - valid_graphs + if invalid_graphs: + print(f"Error: Invalid graph types: {', '.join(invalid_graphs)}. Valid options are: histogram, line, gas, all", file=sys.stderr) + sys.exit(1) + + try: + df1 = pd.read_csv(args.baseline_csv) + df2 = pd.read_csv(args.comparison_csv) + except FileNotFoundError as e: + print(f"Error: {e}", file=sys.stderr) + sys.exit(1) + except Exception as e: + print(f"Error reading CSV files: {e}", file=sys.stderr) + sys.exit(1) + + if 'total_latency' not in df1.columns: + print(f"Error: 'total_latency' column not found in {args.baseline_csv}", file=sys.stderr) + sys.exit(1) + + if 'total_latency' not in df2.columns: + print(f"Error: 'total_latency' column not found in {args.comparison_csv}", file=sys.stderr) + sys.exit(1) + + # Check for gas_used column if gas graph is selected + if 'gas' in selected_graphs: + if 'gas_used' not in df1.columns: + print(f"Error: 'gas_used' column not found in {args.baseline_csv} (required for gas graph)", file=sys.stderr) + sys.exit(1) + if 'gas_used' not in df2.columns: + print(f"Error: 'gas_used' column not found in {args.comparison_csv} (required for gas graph)", file=sys.stderr) + sys.exit(1) + + if len(df1) != len(df2): + print("Warning: CSV files have different number of rows. Using minimum length.", file=sys.stderr) + min_len = min(len(df1), len(df2)) + df1 = df1.head(min_len) + df2 = df2.head(min_len) + + # Convert from microseconds to milliseconds for better readability + latency1 = df1['total_latency'].values / 1000.0 + latency2 = df2['total_latency'].values / 1000.0 + + # Handle division by zero + with np.errstate(divide='ignore', invalid='ignore'): + percent_diff = ((latency2 - latency1) / latency1) * 100 + + # Remove infinite and NaN values + percent_diff = percent_diff[np.isfinite(percent_diff)] + + if len(percent_diff) == 0: + print("Error: No valid percent differences could be calculated", file=sys.stderr) + sys.exit(1) + + # Calculate statistics once for use in graphs and output + mean_diff = np.mean(percent_diff) + median_diff = np.median(percent_diff) + + # Determine number of subplots and create figure + num_plots = len(selected_graphs) + if num_plots == 0: + print("Error: No valid graphs selected", file=sys.stderr) + sys.exit(1) + + # Store output filenames + output_files = [] + + if args.separate: + # We'll create individual figures for each graph + pass + else: + # Create combined figure + if num_plots == 1: + fig, ax = plt.subplots(1, 1, figsize=(12, 6)) + axes = [ax] + else: + fig, axes = plt.subplots(num_plots, 1, figsize=(12, 6 * num_plots)) + + plot_idx = 0 + + # Plot histogram if selected + if 'histogram' in selected_graphs: + if args.separate: + fig, ax = plt.subplots(1, 1, figsize=(12, 6)) + else: + ax = axes[plot_idx] + + min_diff = np.floor(percent_diff.min()) + max_diff = np.ceil(percent_diff.max()) + + # Create histogram with 1% buckets + bins = np.arange(min_diff, max_diff + 1, 1) + + ax.hist(percent_diff, bins=bins, edgecolor='black', alpha=0.7) + ax.set_xlabel('Percent Difference (%)') + ax.set_ylabel('Number of Blocks') + ax.set_title(f'Total Latency Percent Difference Histogram\n({args.baseline_csv} vs {args.comparison_csv})') + ax.grid(True, alpha=0.3) + + # Add statistics to the histogram + ax.axvline(mean_diff, color='red', linestyle='--', label=f'Mean: {mean_diff:.2f}%') + ax.axvline(median_diff, color='orange', linestyle='--', label=f'Median: {median_diff:.2f}%') + ax.legend() + + if args.separate: + plt.tight_layout() + output_file = get_output_filename(args.output, 'histogram') + plt.savefig(output_file, dpi=300, bbox_inches='tight') + output_files.append(output_file) + plt.close(fig) + else: + plot_idx += 1 + + # Plot line graph if selected + if 'line' in selected_graphs: + if args.separate: + fig, ax = plt.subplots(1, 1, figsize=(12, 6)) + else: + ax = axes[plot_idx] + + # Determine comparison color based on median change. The median being + # negative means processing time got faster, so that becomes green. + comparison_color = 'green' if median_diff < 0 else 'red' + + # Apply moving average if requested + plot_latency1 = latency1[:len(percent_diff)] + plot_latency2 = latency2[:len(percent_diff)] + + if args.average: + plot_latency1 = moving_average(plot_latency1, args.average) + plot_latency2 = moving_average(plot_latency2, args.average) + if 'block_number' in df1.columns and 'block_number' in df2.columns: + block_numbers = df1['block_number'].values[:len(percent_diff)] + ax.plot(block_numbers, plot_latency1, 'orange', alpha=0.7, label=f'Baseline ({args.baseline_csv})') + ax.plot(block_numbers, plot_latency2, comparison_color, alpha=0.7, label=f'Comparison ({args.comparison_csv})') + ax.set_xlabel('Block Number') + ax.set_ylabel('Total Latency (ms)') + title = 'Total Latency vs Block Number' + if args.average: + title += f' ({args.average}-block moving average)' + ax.set_title(title) + ax.grid(True, alpha=0.3) + ax.legend() + else: + # If no block_number column, use index + indices = np.arange(len(percent_diff)) + ax.plot(indices, plot_latency1, 'orange', alpha=0.7, label=f'Baseline ({args.baseline_csv})') + ax.plot(indices, plot_latency2, comparison_color, alpha=0.7, label=f'Comparison ({args.comparison_csv})') + ax.set_xlabel('Block Index') + ax.set_ylabel('Total Latency (ms)') + title = 'Total Latency vs Block Index' + if args.average: + title += f' ({args.average}-block moving average)' + ax.set_title(title) + ax.grid(True, alpha=0.3) + ax.legend() + + if args.separate: + plt.tight_layout() + output_file = get_output_filename(args.output, 'line') + plt.savefig(output_file, dpi=300, bbox_inches='tight') + output_files.append(output_file) + plt.close(fig) + else: + plot_idx += 1 + + # Plot gas/s graph if selected + if 'gas' in selected_graphs: + if args.separate: + fig, ax = plt.subplots(1, 1, figsize=(12, 6)) + else: + ax = axes[plot_idx] + + # Calculate gas per second (gas/s) + # latency is in microseconds, so convert to seconds for gas/s calculation + gas1 = df1['gas_used'].values[:len(percent_diff)] + gas2 = df2['gas_used'].values[:len(percent_diff)] + + # Convert latency from microseconds to seconds + latency1_sec = df1['total_latency'].values[:len(percent_diff)] / 1_000_000.0 + latency2_sec = df2['total_latency'].values[:len(percent_diff)] / 1_000_000.0 + + # Calculate gas per second + gas_per_sec1 = gas1 / latency1_sec + gas_per_sec2 = gas2 / latency2_sec + + # Store original values for statistics before averaging + original_gas_per_sec1 = gas_per_sec1.copy() + original_gas_per_sec2 = gas_per_sec2.copy() + + # Apply moving average if requested + if args.average: + gas_per_sec1 = moving_average(gas_per_sec1, args.average) + gas_per_sec2 = moving_average(gas_per_sec2, args.average) + + # Calculate median gas/s for color determination (use original values) + median_gas_per_sec1 = np.median(original_gas_per_sec1) + median_gas_per_sec2 = np.median(original_gas_per_sec2) + comparison_color = 'green' if median_gas_per_sec2 > median_gas_per_sec1 else 'red' + + if 'block_number' in df1.columns and 'block_number' in df2.columns: + block_numbers = df1['block_number'].values[:len(percent_diff)] + ax.plot(block_numbers, gas_per_sec1, 'orange', alpha=0.7, label=f'Baseline ({args.baseline_csv})') + ax.plot(block_numbers, gas_per_sec2, comparison_color, alpha=0.7, label=f'Comparison ({args.comparison_csv})') + ax.set_xlabel('Block Number') + ax.set_ylabel('Gas Throughput') + title = 'Gas Throughput vs Block Number' + if args.average: + title += f' ({args.average}-block moving average)' + ax.set_title(title) + ax.grid(True, alpha=0.3) + ax.legend() + + # Format Y-axis with gas units + formatter = FuncFormatter(format_gas_units) + ax.yaxis.set_major_formatter(formatter) + else: + # If no block_number column, use index + indices = np.arange(len(percent_diff)) + ax.plot(indices, gas_per_sec1, 'orange', alpha=0.7, label=f'Baseline ({args.baseline_csv})') + ax.plot(indices, gas_per_sec2, comparison_color, alpha=0.7, label=f'Comparison ({args.comparison_csv})') + ax.set_xlabel('Block Index') + ax.set_ylabel('Gas Throughput') + title = 'Gas Throughput vs Block Index' + if args.average: + title += f' ({args.average}-block moving average)' + ax.set_title(title) + ax.grid(True, alpha=0.3) + ax.legend() + + # Format Y-axis with gas units + formatter = FuncFormatter(format_gas_units) + ax.yaxis.set_major_formatter(formatter) + + if args.separate: + plt.tight_layout() + output_file = get_output_filename(args.output, 'gas') + plt.savefig(output_file, dpi=300, bbox_inches='tight') + output_files.append(output_file) + plt.close(fig) + else: + plot_idx += 1 + + # Save combined figure if not using separate files + if not args.separate: + plt.tight_layout() + plt.savefig(args.output, dpi=300, bbox_inches='tight') + output_files.append(args.output) + + # Create graph type description for output message + graph_types = [] + if 'histogram' in selected_graphs: + graph_types.append('histogram') + if 'line' in selected_graphs: + graph_types.append('latency graph') + if 'gas' in selected_graphs: + graph_types.append('gas/s graph') + graph_desc = ' and '.join(graph_types) + + # Print output file(s) information + if args.separate: + print(f"Saved {len(output_files)} separate files:") + for output_file in output_files: + print(f" - {output_file}") + else: + print(f"{graph_desc.capitalize()} saved to {args.output}") + + # Always print statistics + print(f"\nStatistics:") + print(f"Mean percent difference: {mean_diff:.2f}%") + print(f"Median percent difference: {median_diff:.2f}%") + print(f"Standard deviation: {np.std(percent_diff):.2f}%") + print(f"Min: {percent_diff.min():.2f}%") + print(f"Max: {percent_diff.max():.2f}%") + print(f"Total blocks analyzed: {len(percent_diff)}") + + # Print gas/s statistics if gas data is available + if 'gas' in selected_graphs: + # Use original values for statistics (not averaged) + print(f"\nGas/s Statistics:") + print(f"Baseline median gas/s: {median_gas_per_sec1:,.0f}") + print(f"Comparison median gas/s: {median_gas_per_sec2:,.0f}") + gas_diff_percent = ((median_gas_per_sec2 - median_gas_per_sec1) / median_gas_per_sec1) * 100 + print(f"Gas/s percent change: {gas_diff_percent:+.2f}%") + +if __name__ == '__main__': + main() diff --git a/bin/reth-bench/src/bench/context.rs b/bin/reth-bench/src/bench/context.rs index e5b1b363449..1d53ce8e1a3 100644 --- a/bin/reth-bench/src/bench/context.rs +++ b/bin/reth-bench/src/bench/context.rs @@ -3,9 +3,11 @@ use crate::{authenticated_transport::AuthenticatedTransportConnect, bench_mode::BenchMode}; use alloy_eips::BlockNumberOrTag; +use alloy_primitives::address; use alloy_provider::{network::AnyNetwork, Provider, RootProvider}; use alloy_rpc_client::ClientBuilder; use alloy_rpc_types_engine::JwtSecret; +use alloy_transport::layers::RetryBackoffLayer; use reqwest::Url; use reth_node_core::args::BenchmarkArgs; use tracing::info; @@ -25,6 +27,8 @@ pub(crate) struct BenchContext { pub(crate) benchmark_mode: BenchMode, /// The next block to fetch. pub(crate) next_block: u64, + /// Whether the chain is an OP rollup. + pub(crate) is_optimism: bool, } impl BenchContext { @@ -33,26 +37,35 @@ impl BenchContext { pub(crate) async fn new(bench_args: &BenchmarkArgs, rpc_url: String) -> eyre::Result { info!("Running benchmark using data from RPC URL: {}", rpc_url); - // Ensure that output directory is a directory + // Ensure that output directory exists and is a directory if let Some(output) = &bench_args.output { if output.is_file() { return Err(eyre::eyre!("Output path must be a directory")); } + // Create the directory if it doesn't exist + if !output.exists() { + std::fs::create_dir_all(output)?; + info!("Created output directory: {:?}", output); + } } // set up alloy client for blocks - let client = ClientBuilder::default().http(rpc_url.parse()?); + let client = ClientBuilder::default() + .layer(RetryBackoffLayer::new(10, 800, u64::MAX)) + .http(rpc_url.parse()?); let block_provider = RootProvider::::new(client); - // If neither `--from` nor `--to` are provided, we will run the benchmark continuously, - // starting at the latest block. - let mut benchmark_mode = BenchMode::new(bench_args.from, bench_args.to)?; + // Check if this is an OP chain by checking code at a predeploy address. + let is_optimism = !block_provider + .get_code_at(address!("0x420000000000000000000000000000000000000F")) + .await? + .is_empty(); // construct the authenticated provider let auth_jwt = bench_args .auth_jwtsecret .clone() - .ok_or_else(|| eyre::eyre!("--jwtsecret must be provided for authenticated RPC"))?; + .ok_or_else(|| eyre::eyre!("--jwt-secret must be provided for authenticated RPC"))?; // fetch jwt from file // @@ -69,6 +82,31 @@ impl BenchContext { let client = ClientBuilder::default().connect_with(auth_transport).await?; let auth_provider = RootProvider::::new(client); + // Computes the block range for the benchmark. + // + // - If `--advance` is provided, fetches the latest block and sets: + // - `from = head + 1` + // - `to = head + advance` + // - Otherwise, uses the values from `--from` and `--to`. + let (from, to) = if let Some(advance) = bench_args.advance { + if advance == 0 { + return Err(eyre::eyre!("--advance must be greater than 0")); + } + + let head_block = auth_provider + .get_block_by_number(BlockNumberOrTag::Latest) + .await? + .ok_or_else(|| eyre::eyre!("Failed to fetch latest block for --advance"))?; + let head_number = head_block.header.number; + (Some(head_number), Some(head_number + advance)) + } else { + (bench_args.from, bench_args.to) + }; + + // If neither `--from` nor `--to` are provided, we will run the benchmark continuously, + // starting at the latest block. + let mut benchmark_mode = BenchMode::new(from, to)?; + let first_block = match benchmark_mode { BenchMode::Continuous => { // fetch Latest block @@ -94,6 +132,6 @@ impl BenchContext { }; let next_block = first_block.header.number + 1; - Ok(Self { auth_provider, block_provider, benchmark_mode, next_block }) + Ok(Self { auth_provider, block_provider, benchmark_mode, next_block, is_optimism }) } } diff --git a/bin/reth-bench/src/bench/mod.rs b/bin/reth-bench/src/bench/mod.rs index afc76b3b6ac..da3ccb1a8bb 100644 --- a/bin/reth-bench/src/bench/mod.rs +++ b/bin/reth-bench/src/bench/mod.rs @@ -38,7 +38,7 @@ pub enum Subcommands { /// /// One powerful use case is pairing this command with the `cast block` command, for example: /// - /// `cast block latest--full --json | reth-bench send-payload --rpc-url localhost:5000 + /// `cast block latest --full --json | reth-bench send-payload --rpc-url localhost:5000 /// --jwt-secret $(cat ~/.local/share/reth/mainnet/jwt.hex)` SendPayload(send_payload::Command), } diff --git a/bin/reth-bench/src/bench/new_payload_fcu.rs b/bin/reth-bench/src/bench/new_payload_fcu.rs index 76166197a73..1d1bf59b365 100644 --- a/bin/reth-bench/src/bench/new_payload_fcu.rs +++ b/bin/reth-bench/src/bench/new_payload_fcu.rs @@ -9,12 +9,13 @@ use crate::{ GAS_OUTPUT_SUFFIX, }, }, - valid_payload::{call_forkchoice_updated, call_new_payload}, + valid_payload::{block_to_new_payload, call_forkchoice_updated, call_new_payload}, }; use alloy_provider::Provider; -use alloy_rpc_types_engine::{ExecutionPayload, ForkchoiceState}; +use alloy_rpc_types_engine::ForkchoiceState; use clap::Parser; use csv::Writer; +use eyre::{Context, OptionExt}; use humantime::parse_duration; use reth_cli_runner::CliContext; use reth_node_core::args::BenchmarkArgs; @@ -29,8 +30,18 @@ pub struct Command { rpc_url: String, /// How long to wait after a forkchoice update before sending the next payload. - #[arg(long, value_name = "WAIT_TIME", value_parser = parse_duration, verbatim_doc_comment)] - wait_time: Option, + #[arg(long, value_name = "WAIT_TIME", value_parser = parse_duration, default_value = "250ms", verbatim_doc_comment)] + wait_time: Duration, + + /// The size of the block buffer (channel capacity) for prefetching blocks from the RPC + /// endpoint. + #[arg( + long = "rpc-block-buffer-size", + value_name = "RPC_BLOCK_BUFFER_SIZE", + default_value = "20", + verbatim_doc_comment + )] + rpc_block_buffer_size: usize, #[command(flatten)] benchmark: BenchmarkArgs, @@ -39,32 +50,46 @@ pub struct Command { impl Command { /// Execute `benchmark new-payload-fcu` command pub async fn execute(self, _ctx: CliContext) -> eyre::Result<()> { - let BenchContext { benchmark_mode, block_provider, auth_provider, mut next_block } = - BenchContext::new(&self.benchmark, self.rpc_url).await?; + let BenchContext { + benchmark_mode, + block_provider, + auth_provider, + mut next_block, + is_optimism, + } = BenchContext::new(&self.benchmark, self.rpc_url).await?; + + let buffer_size = self.rpc_block_buffer_size; + + // Use a oneshot channel to propagate errors from the spawned task + let (error_sender, mut error_receiver) = tokio::sync::oneshot::channel(); + let (sender, mut receiver) = tokio::sync::mpsc::channel(buffer_size); - let (sender, mut receiver) = tokio::sync::mpsc::channel(1000); tokio::task::spawn(async move { while benchmark_mode.contains(next_block) { - let block_res = block_provider.get_block_by_number(next_block.into()).full().await; - let block = block_res.unwrap().unwrap(); - - let block = block - .into_inner() - .map_header(|header| header.map(|h| h.into_header_with_defaults())) - .try_map_transactions(|tx| { - // try to convert unknowns into op type so that we can also support optimism - tx.try_into_either::() - }) - .unwrap() - .into_consensus(); - - let blob_versioned_hashes = - block.body.blob_versioned_hashes_iter().copied().collect::>(); - - // Convert to execution payload - let (payload, sidecar) = ExecutionPayload::from_block_slow(&block); - let header = block.header; - let head_block_hash = payload.block_hash(); + let block_res = block_provider + .get_block_by_number(next_block.into()) + .full() + .await + .wrap_err_with(|| format!("Failed to fetch block by number {next_block}")); + let block = match block_res.and_then(|opt| opt.ok_or_eyre("Block not found")) { + Ok(block) => block, + Err(e) => { + tracing::error!("Failed to fetch block {next_block}: {e}"); + let _ = error_sender.send(e); + break; + } + }; + let header = block.header.clone(); + + let (version, params) = match block_to_new_payload(block, is_optimism) { + Ok(result) => result, + Err(e) => { + tracing::error!("Failed to convert block to new payload: {e}"); + let _ = error_sender.send(e); + break; + } + }; + let head_block_hash = header.hash; let safe_block_hash = block_provider.get_block_by_number(header.number.saturating_sub(32).into()); @@ -73,23 +98,31 @@ impl Command { let (safe, finalized) = tokio::join!(safe_block_hash, finalized_block_hash,); - let safe_block_hash = safe.unwrap().expect("finalized block exists").header.hash; - let finalized_block_hash = - finalized.unwrap().expect("finalized block exists").header.hash; + let safe_block_hash = match safe { + Ok(Some(block)) => block.header.hash, + Ok(None) | Err(_) => head_block_hash, + }; + + let finalized_block_hash = match finalized { + Ok(Some(block)) => block.header.hash, + Ok(None) | Err(_) => head_block_hash, + }; next_block += 1; - sender + if let Err(e) = sender .send(( header, - blob_versioned_hashes, - payload, - sidecar, + version, + params, head_block_hash, safe_block_hash, finalized_block_hash, )) .await - .unwrap(); + { + tracing::error!("Failed to send block data: {e}"); + break; + } } }); @@ -98,7 +131,7 @@ impl Command { let total_benchmark_duration = Instant::now(); let mut total_wait_time = Duration::ZERO; - while let Some((header, versioned_hashes, payload, sidecar, head, safe, finalized)) = { + while let Some((header, version, params, head, safe, finalized)) = { let wait_start = Instant::now(); let result = receiver.recv().await; total_wait_time += wait_start.elapsed(); @@ -118,19 +151,11 @@ impl Command { }; let start = Instant::now(); - let message_version = call_new_payload( - &auth_provider, - payload, - sidecar, - header.parent_beacon_block_root, - versioned_hashes, - ) - .await?; + call_new_payload(&auth_provider, version, params).await?; let new_payload_result = NewPayloadResult { gas_used, latency: start.elapsed() }; - call_forkchoice_updated(&auth_provider, message_version, forkchoice_state, None) - .await?; + call_forkchoice_updated(&auth_provider, version, forkchoice_state, None).await?; // calculate the total duration and the fcu latency, record let total_latency = start.elapsed(); @@ -145,16 +170,19 @@ impl Command { // convert gas used to gigagas, then compute gigagas per second info!(%combined_result); - // wait if we need to - if let Some(wait_time) = self.wait_time { - tokio::time::sleep(wait_time).await; - } + // wait before sending the next payload + tokio::time::sleep(self.wait_time).await; // record the current result let gas_row = TotalGasRow { block_number, gas_used, time: current_duration }; results.push((gas_row, combined_result)); } + // Check if the spawned task encountered an error + if let Ok(error) = error_receiver.try_recv() { + return Err(error); + } + let (gas_output_results, combined_results): (_, Vec) = results.into_iter().unzip(); @@ -182,7 +210,7 @@ impl Command { } // accumulate the results and calculate the overall Ggas/s - let gas_output = TotalGasOutput::new(gas_output_results); + let gas_output = TotalGasOutput::new(gas_output_results)?; info!( total_duration=?gas_output.total_duration, total_gas_used=?gas_output.total_gas_used, diff --git a/bin/reth-bench/src/bench/new_payload_only.rs b/bin/reth-bench/src/bench/new_payload_only.rs index 099ef8112e1..3dfa619ec7b 100644 --- a/bin/reth-bench/src/bench/new_payload_only.rs +++ b/bin/reth-bench/src/bench/new_payload_only.rs @@ -8,12 +8,12 @@ use crate::{ NEW_PAYLOAD_OUTPUT_SUFFIX, }, }, - valid_payload::call_new_payload, + valid_payload::{block_to_new_payload, call_new_payload}, }; use alloy_provider::Provider; -use alloy_rpc_types_engine::ExecutionPayload; use clap::Parser; use csv::Writer; +use eyre::{Context, OptionExt}; use reth_cli_runner::CliContext; use reth_node_core::args::BenchmarkArgs; use std::time::{Duration, Instant}; @@ -26,6 +26,16 @@ pub struct Command { #[arg(long, value_name = "RPC_URL", verbatim_doc_comment)] rpc_url: String, + /// The size of the block buffer (channel capacity) for prefetching blocks from the RPC + /// endpoint. + #[arg( + long = "rpc-block-buffer-size", + value_name = "RPC_BLOCK_BUFFER_SIZE", + default_value = "20", + verbatim_doc_comment + )] + rpc_block_buffer_size: usize, + #[command(flatten)] benchmark: BenchmarkArgs, } @@ -33,29 +43,51 @@ pub struct Command { impl Command { /// Execute `benchmark new-payload-only` command pub async fn execute(self, _ctx: CliContext) -> eyre::Result<()> { - let BenchContext { benchmark_mode, block_provider, auth_provider, mut next_block } = - BenchContext::new(&self.benchmark, self.rpc_url).await?; + let BenchContext { + benchmark_mode, + block_provider, + auth_provider, + mut next_block, + is_optimism, + } = BenchContext::new(&self.benchmark, self.rpc_url).await?; + + let buffer_size = self.rpc_block_buffer_size; + + // Use a oneshot channel to propagate errors from the spawned task + let (error_sender, mut error_receiver) = tokio::sync::oneshot::channel(); + let (sender, mut receiver) = tokio::sync::mpsc::channel(buffer_size); - let (sender, mut receiver) = tokio::sync::mpsc::channel(1000); tokio::task::spawn(async move { while benchmark_mode.contains(next_block) { - let block_res = block_provider.get_block_by_number(next_block.into()).full().await; - let block = block_res.unwrap().unwrap(); - let block = block - .into_inner() - .map_header(|header| header.map(|h| h.into_header_with_defaults())) - .try_map_transactions(|tx| { - tx.try_into_either::() - }) - .unwrap() - .into_consensus(); - - let blob_versioned_hashes = - block.body.blob_versioned_hashes_iter().copied().collect::>(); - let (payload, sidecar) = ExecutionPayload::from_block_slow(&block); + let block_res = block_provider + .get_block_by_number(next_block.into()) + .full() + .await + .wrap_err_with(|| format!("Failed to fetch block by number {next_block}")); + let block = match block_res.and_then(|opt| opt.ok_or_eyre("Block not found")) { + Ok(block) => block, + Err(e) => { + tracing::error!("Failed to fetch block {next_block}: {e}"); + let _ = error_sender.send(e); + break; + } + }; + let header = block.header.clone(); + + let (version, params) = match block_to_new_payload(block, is_optimism) { + Ok(result) => result, + Err(e) => { + tracing::error!("Failed to convert block to new payload: {e}"); + let _ = error_sender.send(e); + break; + } + }; next_block += 1; - sender.send((block.header, blob_versioned_hashes, payload, sidecar)).await.unwrap(); + if let Err(e) = sender.send((header, version, params)).await { + tracing::error!("Failed to send block data: {e}"); + break; + } } }); @@ -64,7 +96,7 @@ impl Command { let total_benchmark_duration = Instant::now(); let mut total_wait_time = Duration::ZERO; - while let Some((header, versioned_hashes, payload, sidecar)) = { + while let Some((header, version, params)) = { let wait_start = Instant::now(); let result = receiver.recv().await; total_wait_time += wait_start.elapsed(); @@ -73,7 +105,7 @@ impl Command { // just put gas used here let gas_used = header.gas_used; - let block_number = payload.block_number(); + let block_number = header.number; debug!( target: "reth-bench", @@ -82,14 +114,7 @@ impl Command { ); let start = Instant::now(); - call_new_payload( - &auth_provider, - payload, - sidecar, - header.parent_beacon_block_root, - versioned_hashes, - ) - .await?; + call_new_payload(&auth_provider, version, params).await?; let new_payload_result = NewPayloadResult { gas_used, latency: start.elapsed() }; info!(%new_payload_result); @@ -103,6 +128,11 @@ impl Command { results.push((row, new_payload_result)); } + // Check if the spawned task encountered an error + if let Ok(error) = error_receiver.try_recv() { + return Err(error); + } + let (gas_output_results, new_payload_results): (_, Vec) = results.into_iter().unzip(); @@ -130,7 +160,7 @@ impl Command { } // accumulate the results and calculate the overall Ggas/s - let gas_output = TotalGasOutput::new(gas_output_results); + let gas_output = TotalGasOutput::new(gas_output_results)?; info!( total_duration=?gas_output.total_duration, total_gas_used=?gas_output.total_gas_used, diff --git a/bin/reth-bench/src/bench/output.rs b/bin/reth-bench/src/bench/output.rs index 4fe463e91a5..794cd2768df 100644 --- a/bin/reth-bench/src/bench/output.rs +++ b/bin/reth-bench/src/bench/output.rs @@ -1,6 +1,7 @@ //! Contains various benchmark output formats, either for logging or for //! serialization to / from files. +use eyre::OptionExt; use reth_primitives_traits::constants::GIGAGAS; use serde::{ser::SerializeStruct, Serialize}; use std::time::Duration; @@ -52,7 +53,7 @@ impl Serialize for NewPayloadResult { { // convert the time to microseconds let time = self.latency.as_micros(); - let mut state = serializer.serialize_struct("NewPayloadResult", 3)?; + let mut state = serializer.serialize_struct("NewPayloadResult", 2)?; state.serialize_field("gas_used", &self.gas_used)?; state.serialize_field("latency", &time)?; state.end() @@ -145,15 +146,14 @@ pub(crate) struct TotalGasOutput { impl TotalGasOutput { /// Create a new [`TotalGasOutput`] from a list of [`TotalGasRow`]. - pub(crate) fn new(rows: Vec) -> Self { + pub(crate) fn new(rows: Vec) -> eyre::Result { // the duration is obtained from the last row - let total_duration = - rows.last().map(|row| row.time).expect("the row has at least one element"); + let total_duration = rows.last().map(|row| row.time).ok_or_eyre("empty results")?; let blocks_processed = rows.len() as u64; let total_gas_used: u64 = rows.into_iter().map(|row| row.gas_used).sum(); let total_gas_per_second = total_gas_used as f64 / total_duration.as_secs_f64(); - Self { total_gas_used, total_duration, total_gas_per_second, blocks_processed } + Ok(Self { total_gas_used, total_duration, total_gas_per_second, blocks_processed }) } /// Return the total gigagas per second. diff --git a/bin/reth-bench/src/main.rs b/bin/reth-bench/src/main.rs index f146af0f70d..89fea3c381c 100644 --- a/bin/reth-bench/src/main.rs +++ b/bin/reth-bench/src/main.rs @@ -9,7 +9,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #[global_allocator] static ALLOC: reth_cli_util::allocator::Allocator = reth_cli_util::allocator::new_allocator(); @@ -26,7 +26,9 @@ use reth_cli_runner::CliRunner; fn main() { // Enable backtraces unless a RUST_BACKTRACE value has already been explicitly provided. if std::env::var_os("RUST_BACKTRACE").is_none() { - std::env::set_var("RUST_BACKTRACE", "1"); + unsafe { + std::env::set_var("RUST_BACKTRACE", "1"); + } } // Run until either exit or sigint or sigterm diff --git a/bin/reth-bench/src/valid_payload.rs b/bin/reth-bench/src/valid_payload.rs index 8a7021da7df..d253506b22b 100644 --- a/bin/reth-bench/src/valid_payload.rs +++ b/bin/reth-bench/src/valid_payload.rs @@ -2,71 +2,38 @@ //! response. This is useful for benchmarking, as it allows us to wait for a payload to be valid //! before sending additional calls. -use alloy_eips::eip7685::RequestsOrHash; -use alloy_primitives::B256; -use alloy_provider::{ext::EngineApi, Network, Provider}; +use alloy_eips::eip7685::Requests; +use alloy_provider::{ext::EngineApi, network::AnyRpcBlock, Network, Provider}; use alloy_rpc_types_engine::{ - ExecutionPayload, ExecutionPayloadInputV2, ExecutionPayloadSidecar, ExecutionPayloadV1, - ExecutionPayloadV3, ForkchoiceState, ForkchoiceUpdated, PayloadAttributes, PayloadStatus, + ExecutionPayload, ExecutionPayloadInputV2, ForkchoiceState, ForkchoiceUpdated, + PayloadAttributes, PayloadStatus, }; use alloy_transport::TransportResult; +use op_alloy_rpc_types_engine::OpExecutionPayloadV4; use reth_node_api::EngineApiMessageVersion; use tracing::error; /// An extension trait for providers that implement the engine API, to wait for a VALID response. #[async_trait::async_trait] pub trait EngineApiValidWaitExt: Send + Sync { - /// Calls `engine_newPayloadV1` with the given [ExecutionPayloadV1], and waits until the - /// response is VALID. - async fn new_payload_v1_wait( - &self, - payload: ExecutionPayloadV1, - ) -> TransportResult; - - /// Calls `engine_newPayloadV2` with the given [ExecutionPayloadInputV2], and waits until the - /// response is VALID. - async fn new_payload_v2_wait( - &self, - payload: ExecutionPayloadInputV2, - ) -> TransportResult; - - /// Calls `engine_newPayloadV3` with the given [ExecutionPayloadV3], parent beacon block root, - /// and versioned hashes, and waits until the response is VALID. - async fn new_payload_v3_wait( - &self, - payload: ExecutionPayloadV3, - versioned_hashes: Vec, - parent_beacon_block_root: B256, - ) -> TransportResult; - - /// Calls `engine_newPayloadV4` with the given [ExecutionPayloadV3], parent beacon block root, - /// versioned hashes, and requests hash, and waits until the response is VALID. - async fn new_payload_v4_wait( - &self, - payload: ExecutionPayloadV3, - versioned_hashes: Vec, - parent_beacon_block_root: B256, - requests_hash: B256, - ) -> TransportResult; - - /// Calls `engine_forkChoiceUpdatedV1` with the given [ForkchoiceState] and optional - /// [PayloadAttributes], and waits until the response is VALID. + /// Calls `engine_forkChoiceUpdatedV1` with the given [`ForkchoiceState`] and optional + /// [`PayloadAttributes`], and waits until the response is VALID. async fn fork_choice_updated_v1_wait( &self, fork_choice_state: ForkchoiceState, payload_attributes: Option, ) -> TransportResult; - /// Calls `engine_forkChoiceUpdatedV2` with the given [ForkchoiceState] and optional - /// [PayloadAttributes], and waits until the response is VALID. + /// Calls `engine_forkChoiceUpdatedV2` with the given [`ForkchoiceState`] and optional + /// [`PayloadAttributes`], and waits until the response is VALID. async fn fork_choice_updated_v2_wait( &self, fork_choice_state: ForkchoiceState, payload_attributes: Option, ) -> TransportResult; - /// Calls `engine_forkChoiceUpdatedV3` with the given [ForkchoiceState] and optional - /// [PayloadAttributes], and waits until the response is VALID. + /// Calls `engine_forkChoiceUpdatedV3` with the given [`ForkchoiceState`] and optional + /// [`PayloadAttributes`], and waits until the response is VALID. async fn fork_choice_updated_v3_wait( &self, fork_choice_state: ForkchoiceState, @@ -80,122 +47,6 @@ where N: Network, P: Provider + EngineApi, { - async fn new_payload_v1_wait( - &self, - payload: ExecutionPayloadV1, - ) -> TransportResult { - let mut status = self.new_payload_v1(payload.clone()).await?; - while !status.is_valid() { - if status.is_invalid() { - error!(?status, ?payload, "Invalid newPayloadV1",); - panic!("Invalid newPayloadV1: {status:?}"); - } - status = self.new_payload_v1(payload.clone()).await?; - } - Ok(status) - } - - async fn new_payload_v2_wait( - &self, - payload: ExecutionPayloadInputV2, - ) -> TransportResult { - let mut status = self.new_payload_v2(payload.clone()).await?; - while !status.is_valid() { - if status.is_invalid() { - error!(?status, ?payload, "Invalid newPayloadV2",); - panic!("Invalid newPayloadV2: {status:?}"); - } - status = self.new_payload_v2(payload.clone()).await?; - } - Ok(status) - } - - async fn new_payload_v3_wait( - &self, - payload: ExecutionPayloadV3, - versioned_hashes: Vec, - parent_beacon_block_root: B256, - ) -> TransportResult { - let mut status = self - .new_payload_v3(payload.clone(), versioned_hashes.clone(), parent_beacon_block_root) - .await?; - while !status.is_valid() { - if status.is_invalid() { - error!( - ?status, - ?payload, - ?versioned_hashes, - ?parent_beacon_block_root, - "Invalid newPayloadV3", - ); - panic!("Invalid newPayloadV3: {status:?}"); - } - if status.is_syncing() { - return Err(alloy_json_rpc::RpcError::UnsupportedFeature( - "invalid range: no canonical state found for parent of requested block", - )) - } - status = self - .new_payload_v3(payload.clone(), versioned_hashes.clone(), parent_beacon_block_root) - .await?; - } - Ok(status) - } - - async fn new_payload_v4_wait( - &self, - payload: ExecutionPayloadV3, - versioned_hashes: Vec, - parent_beacon_block_root: B256, - requests_hash: B256, - ) -> TransportResult { - // We cannot use `self.new_payload_v4` because it does not support sending - // `RequestsOrHash::Hash` - - let mut status: PayloadStatus = self - .client() - .request( - "engine_newPayloadV4", - ( - payload.clone(), - versioned_hashes.clone(), - parent_beacon_block_root, - RequestsOrHash::Hash(requests_hash), - ), - ) - .await?; - while !status.is_valid() { - if status.is_invalid() { - error!( - ?status, - ?payload, - ?versioned_hashes, - ?parent_beacon_block_root, - "Invalid newPayloadV4", - ); - panic!("Invalid newPayloadV4: {status:?}"); - } - if status.is_syncing() { - return Err(alloy_json_rpc::RpcError::UnsupportedFeature( - "invalid range: no canonical state found for parent of requested block", - )) - } - status = self - .client() - .request( - "engine_newPayloadV4", - ( - payload.clone(), - versioned_hashes.clone(), - parent_beacon_block_root, - RequestsOrHash::Hash(requests_hash), - ), - ) - .await?; - } - Ok(status) - } - async fn fork_choice_updated_v1_wait( &self, fork_choice_state: ForkchoiceState, @@ -282,39 +133,60 @@ where } } -/// Calls the correct `engine_newPayload` method depending on the given [`ExecutionPayload`] and its -/// versioned variant. Returns the [`EngineApiMessageVersion`] depending on the payload's version. -/// -/// # Panics -/// If the given payload is a V3 payload, but a parent beacon block root is provided as `None`. -pub(crate) async fn call_new_payload>( - provider: P, - payload: ExecutionPayload, - sidecar: ExecutionPayloadSidecar, - parent_beacon_block_root: Option, - versioned_hashes: Vec, -) -> TransportResult { - match payload { +pub(crate) fn block_to_new_payload( + block: AnyRpcBlock, + is_optimism: bool, +) -> eyre::Result<(EngineApiMessageVersion, serde_json::Value)> { + let block = block + .into_inner() + .map_header(|header| header.map(|h| h.into_header_with_defaults())) + .try_map_transactions(|tx| { + // try to convert unknowns into op type so that we can also support optimism + tx.try_into_either::() + })? + .into_consensus(); + + // Convert to execution payload + let (payload, sidecar) = ExecutionPayload::from_block_slow(&block); + + let (version, params) = match payload { ExecutionPayload::V3(payload) => { - // We expect the caller - let parent_beacon_block_root = parent_beacon_block_root - .expect("parent_beacon_block_root is required for V3 payloads and higher"); + let cancun = sidecar.cancun().unwrap(); - if let Some(requests_hash) = sidecar.requests_hash() { - provider - .new_payload_v4_wait( - payload, - versioned_hashes, - parent_beacon_block_root, - requests_hash, + if let Some(prague) = sidecar.prague() { + if is_optimism { + ( + EngineApiMessageVersion::V4, + serde_json::to_value(( + OpExecutionPayloadV4 { + payload_inner: payload, + withdrawals_root: block.withdrawals_root.unwrap(), + }, + cancun.versioned_hashes.clone(), + cancun.parent_beacon_block_root, + Requests::default(), + ))?, + ) + } else { + ( + EngineApiMessageVersion::V4, + serde_json::to_value(( + payload, + cancun.versioned_hashes.clone(), + cancun.parent_beacon_block_root, + prague.requests.requests_hash(), + ))?, ) - .await?; - Ok(EngineApiMessageVersion::V4) + } } else { - provider - .new_payload_v3_wait(payload, versioned_hashes, parent_beacon_block_root) - .await?; - Ok(EngineApiMessageVersion::V3) + ( + EngineApiMessageVersion::V3, + serde_json::to_value(( + payload, + cancun.versioned_hashes.clone(), + cancun.parent_beacon_block_root, + ))?, + ) } } ExecutionPayload::V2(payload) => { @@ -323,16 +195,43 @@ pub(crate) async fn call_new_payload>( withdrawals: Some(payload.withdrawals), }; - provider.new_payload_v2_wait(input).await?; - - Ok(EngineApiMessageVersion::V2) + (EngineApiMessageVersion::V2, serde_json::to_value((input,))?) } ExecutionPayload::V1(payload) => { - provider.new_payload_v1_wait(payload).await?; + (EngineApiMessageVersion::V1, serde_json::to_value((payload,))?) + } + }; - Ok(EngineApiMessageVersion::V1) + Ok((version, params)) +} + +/// Calls the correct `engine_newPayload` method depending on the given [`ExecutionPayload`] and its +/// versioned variant. Returns the [`EngineApiMessageVersion`] depending on the payload's version. +/// +/// # Panics +/// If the given payload is a V3 payload, but a parent beacon block root is provided as `None`. +pub(crate) async fn call_new_payload>( + provider: P, + version: EngineApiMessageVersion, + params: serde_json::Value, +) -> TransportResult<()> { + let method = version.method_name(); + + let mut status: PayloadStatus = provider.client().request(method, ¶ms).await?; + + while !status.is_valid() { + if status.is_invalid() { + error!(?status, ?params, "Invalid {method}",); + panic!("Invalid {method}: {status:?}"); + } + if status.is_syncing() { + return Err(alloy_json_rpc::RpcError::UnsupportedFeature( + "invalid range: no canonical state found for parent of requested block", + )) } + status = provider.client().request(method, ¶ms).await?; } + Ok(()) } /// Calls the correct `engine_forkchoiceUpdated` method depending on the given @@ -345,7 +244,7 @@ pub(crate) async fn call_forkchoice_updated>( payload_attributes: Option, ) -> TransportResult { match message_version { - EngineApiMessageVersion::V3 | EngineApiMessageVersion::V4 => { + EngineApiMessageVersion::V3 | EngineApiMessageVersion::V4 | EngineApiMessageVersion::V5 => { provider.fork_choice_updated_v3_wait(forkchoice_state, payload_attributes).await } EngineApiMessageVersion::V2 => { diff --git a/bin/reth/Cargo.toml b/bin/reth/Cargo.toml index 4d93ca5d73c..eb0cf0bd2b2 100644 --- a/bin/reth/Cargo.toml +++ b/bin/reth/Cargo.toml @@ -27,7 +27,7 @@ reth-cli-util.workspace = true reth-consensus-common.workspace = true reth-rpc-builder.workspace = true reth-rpc.workspace = true -reth-rpc-types-compat.workspace = true +reth-rpc-convert.workspace = true reth-rpc-api = { workspace = true, features = ["client"] } reth-rpc-eth-types.workspace = true reth-rpc-server-types.workspace = true @@ -40,7 +40,7 @@ reth-node-api.workspace = true reth-node-core.workspace = true reth-ethereum-payload-builder.workspace = true reth-ethereum-primitives.workspace = true -reth-node-ethereum = { workspace = true, features = ["js-tracer"] } +reth-node-ethereum.workspace = true reth-node-builder.workspace = true reth-node-metrics.workspace = true reth-consensus.workspace = true @@ -64,11 +64,21 @@ eyre.workspace = true [dev-dependencies] backon.workspace = true -similar-asserts.workspace = true tempfile.workspace = true [features] -default = ["jemalloc", "reth-revm/portable"] +default = ["jemalloc", "otlp", "reth-revm/portable", "js-tracer"] + +otlp = [ + "reth-ethereum-cli/otlp", + "reth-node-core/otlp", +] +js-tracer = [ + "reth-node-builder/js-tracer", + "reth-node-ethereum/js-tracer", + "reth-rpc/js-tracer", + "reth-rpc-eth-types/js-tracer", +] dev = ["reth-ethereum-cli/dev"] @@ -76,6 +86,7 @@ asm-keccak = [ "reth-node-core/asm-keccak", "reth-primitives/asm-keccak", "reth-ethereum-cli/asm-keccak", + "reth-node-ethereum/asm-keccak", ] jemalloc = [ @@ -109,22 +120,27 @@ snmalloc-native = [ min-error-logs = [ "tracing/release_max_level_error", "reth-ethereum-cli/min-error-logs", + "reth-node-core/min-error-logs", ] min-warn-logs = [ "tracing/release_max_level_warn", "reth-ethereum-cli/min-warn-logs", + "reth-node-core/min-warn-logs", ] min-info-logs = [ "tracing/release_max_level_info", "reth-ethereum-cli/min-info-logs", + "reth-node-core/min-info-logs", ] min-debug-logs = [ "tracing/release_max_level_debug", "reth-ethereum-cli/min-debug-logs", + "reth-node-core/min-debug-logs", ] min-trace-logs = [ "tracing/release_max_level_trace", "reth-ethereum-cli/min-trace-logs", + "reth-node-core/min-trace-logs", ] [[bin]] diff --git a/bin/reth/src/lib.rs b/bin/reth/src/lib.rs index 11a50acd3a7..10744b877dd 100644 --- a/bin/reth/src/lib.rs +++ b/bin/reth/src/lib.rs @@ -25,7 +25,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] pub mod cli; @@ -175,9 +175,9 @@ pub mod rpc { pub use reth_rpc_server_types::result::*; } - /// Re-exported from `reth_rpc_types_compat`. + /// Re-exported from `reth_rpc_convert`. pub mod compat { - pub use reth_rpc_types_compat::*; + pub use reth_rpc_convert::*; } } diff --git a/bin/reth/src/ress.rs b/bin/reth/src/ress.rs index 43ddcb6a3a8..88d3e2aa698 100644 --- a/bin/reth/src/ress.rs +++ b/bin/reth/src/ress.rs @@ -2,7 +2,7 @@ use reth_ethereum_primitives::EthPrimitives; use reth_evm::ConfigureEvm; use reth_network::{protocol::IntoRlpxSubProtocol, NetworkProtocols}; use reth_network_api::FullNetwork; -use reth_node_api::BeaconConsensusEngineEvent; +use reth_node_api::ConsensusEngineEvent; use reth_node_core::args::RessArgs; use reth_provider::providers::{BlockchainProvider, ProviderNodeTypes}; use reth_ress_protocol::{NodeType, ProtocolState, RessProtocolHandler}; @@ -19,7 +19,7 @@ pub fn install_ress_subprotocol( evm_config: E, network: N, task_executor: TaskExecutor, - engine_events: EventStream>, + engine_events: EventStream>, ) -> eyre::Result<()> where P: ProviderNodeTypes, diff --git a/book/SUMMARY.md b/book/SUMMARY.md deleted file mode 100644 index 310eebb0285..00000000000 --- a/book/SUMMARY.md +++ /dev/null @@ -1,84 +0,0 @@ -# Reth Book - -- [Introduction](./intro.md) -- [Installation](./installation/installation.md) - - [Pre-Built Binaries](./installation/binaries.md) - - [Docker](./installation/docker.md) - - [Build from Source](./installation/source.md) - - [Build for ARM devices](./installation/build-for-arm-devices.md) - - [Update Priorities](./installation/priorities.md) -- [Run a Node](./run/run-a-node.md) - - [Mainnet or official testnets](./run/mainnet.md) - - [OP Stack](./run/optimism.md) - - [Run an OP Mainnet Node](./run/sync-op-mainnet.md) - - [Private testnet](./run/private-testnet.md) - - [Metrics](./run/observability.md) - - [Configuring Reth](./run/config.md) - - [Transaction types](./run/transactions.md) - - [Pruning & Full Node](./run/pruning.md) - - [Ports](./run/ports.md) - - [Troubleshooting](./run/troubleshooting.md) -- [Interacting with Reth over JSON-RPC](./jsonrpc/intro.md) - - [eth](./jsonrpc/eth.md) - - [web3](./jsonrpc/web3.md) - - [net](./jsonrpc/net.md) - - [txpool](./jsonrpc/txpool.md) - - [debug](./jsonrpc/debug.md) - - [trace](./jsonrpc/trace.md) - - [admin](./jsonrpc/admin.md) - - [rpc](./jsonrpc/rpc.md) -- [CLI Reference](./cli/cli.md) - - [`reth`](./cli/reth.md) - - [`reth node`](./cli/reth/node.md) - - [`reth init`](./cli/reth/init.md) - - [`reth init-state`](./cli/reth/init-state.md) - - [`reth import`](./cli/reth/import.md) - - [`reth import-era`](./cli/reth/import-era.md) - - [`reth dump-genesis`](./cli/reth/dump-genesis.md) - - [`reth db`](./cli/reth/db.md) - - [`reth db stats`](./cli/reth/db/stats.md) - - [`reth db list`](./cli/reth/db/list.md) - - [`reth db checksum`](./cli/reth/db/checksum.md) - - [`reth db diff`](./cli/reth/db/diff.md) - - [`reth db get`](./cli/reth/db/get.md) - - [`reth db get mdbx`](./cli/reth/db/get/mdbx.md) - - [`reth db get static-file`](./cli/reth/db/get/static-file.md) - - [`reth db drop`](./cli/reth/db/drop.md) - - [`reth db clear`](./cli/reth/db/clear.md) - - [`reth db clear mdbx`](./cli/reth/db/clear/mdbx.md) - - [`reth db clear static-file`](./cli/reth/db/clear/static-file.md) - - [`reth db version`](./cli/reth/db/version.md) - - [`reth db path`](./cli/reth/db/path.md) - - [`reth download`](./cli/reth/download.md) - - [`reth stage`](./cli/reth/stage.md) - - [`reth stage run`](./cli/reth/stage/run.md) - - [`reth stage drop`](./cli/reth/stage/drop.md) - - [`reth stage dump`](./cli/reth/stage/dump.md) - - [`reth stage dump execution`](./cli/reth/stage/dump/execution.md) - - [`reth stage dump storage-hashing`](./cli/reth/stage/dump/storage-hashing.md) - - [`reth stage dump account-hashing`](./cli/reth/stage/dump/account-hashing.md) - - [`reth stage dump merkle`](./cli/reth/stage/dump/merkle.md) - - [`reth stage unwind`](./cli/reth/stage/unwind.md) - - [`reth stage unwind to-block`](./cli/reth/stage/unwind/to-block.md) - - [`reth stage unwind num-blocks`](./cli/reth/stage/unwind/num-blocks.md) - - [`reth p2p`](./cli/reth/p2p.md) - - [`reth p2p header`](./cli/reth/p2p/header.md) - - [`reth p2p body`](./cli/reth/p2p/body.md) - - [`reth p2p rlpx`](./cli/reth/p2p/rlpx.md) - - [`reth p2p rlpx ping`](./cli/reth/p2p/rlpx/ping.md) - - [`reth config`](./cli/reth/config.md) - - [`reth debug`](./cli/reth/debug.md) - - [`reth debug execution`](./cli/reth/debug/execution.md) - - [`reth debug merkle`](./cli/reth/debug/merkle.md) - - [`reth debug in-memory-merkle`](./cli/reth/debug/in-memory-merkle.md) - - [`reth debug build-block`](./cli/reth/debug/build-block.md) - - [`reth recover`](./cli/reth/recover.md) - - [`reth recover storage-tries`](./cli/reth/recover/storage-tries.md) - - [`reth prune`](./cli/reth/prune.md) -- [Developers](./developers/developers.md) - - [Execution Extensions](./developers/exex/exex.md) - - [How do ExExes work?](./developers/exex/how-it-works.md) - - [Hello World](./developers/exex/hello-world.md) - - [Tracking State](./developers/exex/tracking-state.md) - - [Remote](./developers/exex/remote.md) - - [Contribute](./developers/contribute.md) diff --git a/book/cli/SUMMARY.md b/book/cli/SUMMARY.md deleted file mode 100644 index aa625298590..00000000000 --- a/book/cli/SUMMARY.md +++ /dev/null @@ -1,47 +0,0 @@ -- [`reth`](./reth.md) - - [`reth node`](./reth/node.md) - - [`reth init`](./reth/init.md) - - [`reth init-state`](./reth/init-state.md) - - [`reth import`](./reth/import.md) - - [`reth import-era`](./reth/import-era.md) - - [`reth dump-genesis`](./reth/dump-genesis.md) - - [`reth db`](./reth/db.md) - - [`reth db stats`](./reth/db/stats.md) - - [`reth db list`](./reth/db/list.md) - - [`reth db checksum`](./reth/db/checksum.md) - - [`reth db diff`](./reth/db/diff.md) - - [`reth db get`](./reth/db/get.md) - - [`reth db get mdbx`](./reth/db/get/mdbx.md) - - [`reth db get static-file`](./reth/db/get/static-file.md) - - [`reth db drop`](./reth/db/drop.md) - - [`reth db clear`](./reth/db/clear.md) - - [`reth db clear mdbx`](./reth/db/clear/mdbx.md) - - [`reth db clear static-file`](./reth/db/clear/static-file.md) - - [`reth db version`](./reth/db/version.md) - - [`reth db path`](./reth/db/path.md) - - [`reth download`](./reth/download.md) - - [`reth stage`](./reth/stage.md) - - [`reth stage run`](./reth/stage/run.md) - - [`reth stage drop`](./reth/stage/drop.md) - - [`reth stage dump`](./reth/stage/dump.md) - - [`reth stage dump execution`](./reth/stage/dump/execution.md) - - [`reth stage dump storage-hashing`](./reth/stage/dump/storage-hashing.md) - - [`reth stage dump account-hashing`](./reth/stage/dump/account-hashing.md) - - [`reth stage dump merkle`](./reth/stage/dump/merkle.md) - - [`reth stage unwind`](./reth/stage/unwind.md) - - [`reth stage unwind to-block`](./reth/stage/unwind/to-block.md) - - [`reth stage unwind num-blocks`](./reth/stage/unwind/num-blocks.md) - - [`reth p2p`](./reth/p2p.md) - - [`reth p2p header`](./reth/p2p/header.md) - - [`reth p2p body`](./reth/p2p/body.md) - - [`reth p2p rlpx`](./reth/p2p/rlpx.md) - - [`reth p2p rlpx ping`](./reth/p2p/rlpx/ping.md) - - [`reth config`](./reth/config.md) - - [`reth debug`](./reth/debug.md) - - [`reth debug execution`](./reth/debug/execution.md) - - [`reth debug merkle`](./reth/debug/merkle.md) - - [`reth debug in-memory-merkle`](./reth/debug/in-memory-merkle.md) - - [`reth debug build-block`](./reth/debug/build-block.md) - - [`reth recover`](./reth/recover.md) - - [`reth recover storage-tries`](./reth/recover/storage-tries.md) - - [`reth prune`](./reth/prune.md) diff --git a/book/cli/reth/debug/execution.md b/book/cli/reth/debug/execution.md deleted file mode 100644 index ef7069f8173..00000000000 --- a/book/cli/reth/debug/execution.md +++ /dev/null @@ -1,328 +0,0 @@ -# reth debug execution - -Debug the roundtrip execution of blocks as well as the generated data - -```bash -$ reth debug execution --help -``` -```txt -Usage: reth debug execution [OPTIONS] --to - -Options: - -h, --help - Print help (see a summary with '-h') - -Datadir: - --datadir - The path to the data dir for all reth files and subdirectories. - - Defaults to the OS-specific data directory: - - - Linux: `$XDG_DATA_HOME/reth/` or `$HOME/.local/share/reth/` - - Windows: `{FOLDERID_RoamingAppData}/reth/` - - macOS: `$HOME/Library/Application Support/reth/` - - [default: default] - - --datadir.static-files - The absolute path to store static files in. - - --config - The path to the configuration file to use - - --chain - The chain this node is running. - Possible values are either a built-in chain or the path to a chain specification file. - - Built-in chains: - mainnet, sepolia, holesky, hoodi, dev - - [default: mainnet] - -Database: - --db.log-level - Database logging level. Levels higher than "notice" require a debug build - - Possible values: - - fatal: Enables logging for critical conditions, i.e. assertion failures - - error: Enables logging for error conditions - - warn: Enables logging for warning conditions - - notice: Enables logging for normal but significant condition - - verbose: Enables logging for verbose informational - - debug: Enables logging for debug-level messages - - trace: Enables logging for trace debug-level messages - - extra: Enables logging for extra debug-level messages - - --db.exclusive - Open environment in exclusive/monopolistic mode. Makes it possible to open a database on an NFS volume - - [possible values: true, false] - - --db.max-size - Maximum database size (e.g., 4TB, 8MB) - - --db.growth-step - Database growth step (e.g., 4GB, 4KB) - - --db.read-transaction-timeout - Read transaction timeout in seconds, 0 means no timeout - -Networking: - -d, --disable-discovery - Disable the discovery service - - --disable-dns-discovery - Disable the DNS discovery - - --disable-discv4-discovery - Disable Discv4 discovery - - --enable-discv5-discovery - Enable Discv5 discovery - - --disable-nat - Disable Nat discovery - - --discovery.addr - The UDP address to use for devp2p peer discovery version 4 - - [default: 0.0.0.0] - - --discovery.port - The UDP port to use for devp2p peer discovery version 4 - - [default: 30303] - - --discovery.v5.addr - The UDP IPv4 address to use for devp2p peer discovery version 5. Overwritten by `RLPx` address, if it's also IPv4 - - --discovery.v5.addr.ipv6 - The UDP IPv6 address to use for devp2p peer discovery version 5. Overwritten by `RLPx` address, if it's also IPv6 - - --discovery.v5.port - The UDP IPv4 port to use for devp2p peer discovery version 5. Not used unless `--addr` is IPv4, or `--discovery.v5.addr` is set - - [default: 9200] - - --discovery.v5.port.ipv6 - The UDP IPv6 port to use for devp2p peer discovery version 5. Not used unless `--addr` is IPv6, or `--discovery.addr.ipv6` is set - - [default: 9200] - - --discovery.v5.lookup-interval - The interval in seconds at which to carry out periodic lookup queries, for the whole run of the program - - [default: 20] - - --discovery.v5.bootstrap.lookup-interval - The interval in seconds at which to carry out boost lookup queries, for a fixed number of times, at bootstrap - - [default: 5] - - --discovery.v5.bootstrap.lookup-countdown - The number of times to carry out boost lookup queries at bootstrap - - [default: 200] - - --trusted-peers - Comma separated enode URLs of trusted peers for P2P connections. - - --trusted-peers enode://abcd@192.168.0.1:30303 - - --trusted-only - Connect to or accept from trusted peers only - - --bootnodes - Comma separated enode URLs for P2P discovery bootstrap. - - Will fall back to a network-specific default if not specified. - - --dns-retries - Amount of DNS resolution requests retries to perform when peering - - [default: 0] - - --peers-file - The path to the known peers file. Connected peers are dumped to this file on nodes - shutdown, and read on startup. Cannot be used with `--no-persist-peers`. - - --identity - Custom node identity - - [default: reth/-/] - - --p2p-secret-key - Secret key to use for this node. - - This will also deterministically set the peer ID. If not specified, it will be set in the data dir for the chain being used. - - --no-persist-peers - Do not persist peers. - - --nat - NAT resolution method (any|none|upnp|publicip|extip:\) - - [default: any] - - --addr - Network listening address - - [default: 0.0.0.0] - - --port - Network listening port - - [default: 30303] - - --max-outbound-peers - Maximum number of outbound requests. default: 100 - - --max-inbound-peers - Maximum number of inbound requests. default: 30 - - --max-tx-reqs - Max concurrent `GetPooledTransactions` requests. - - [default: 130] - - --max-tx-reqs-peer - Max concurrent `GetPooledTransactions` requests per peer. - - [default: 1] - - --max-seen-tx-history - Max number of seen transactions to remember per peer. - - Default is 320 transaction hashes. - - [default: 320] - - --max-pending-imports - Max number of transactions to import concurrently. - - [default: 4096] - - --pooled-tx-response-soft-limit - Experimental, for usage in research. Sets the max accumulated byte size of transactions - to pack in one response. - Spec'd at 2MiB. - - [default: 2097152] - - --pooled-tx-pack-soft-limit - Experimental, for usage in research. Sets the max accumulated byte size of transactions to - request in one request. - - Since `RLPx` protocol version 68, the byte size of a transaction is shared as metadata in a - transaction announcement (see `RLPx` specs). This allows a node to request a specific size - response. - - By default, nodes request only 128 KiB worth of transactions, but should a peer request - more, up to 2 MiB, a node will answer with more than 128 KiB. - - Default is 128 KiB. - - [default: 131072] - - --max-tx-pending-fetch - Max capacity of cache of hashes for transactions pending fetch. - - [default: 25600] - - --net-if.experimental - Name of network interface used to communicate with peers. - - If flag is set, but no value is passed, the default interface for docker `eth0` is tried. - - --tx-propagation-policy - Transaction Propagation Policy - - The policy determines which peers transactions are gossiped to. - - [default: All] - - --to - The maximum block height - - --interval - The block interval for sync and unwind. Defaults to `1000` - - [default: 1000] - -Logging: - --log.stdout.format - The format to use for logs written to stdout - - [default: terminal] - - Possible values: - - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - - terminal: Represents terminal-friendly formatting for logs - - --log.stdout.filter - The filter to use for logs written to stdout - - [default: ] - - --log.file.format - The format to use for logs written to the log file - - [default: terminal] - - Possible values: - - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - - terminal: Represents terminal-friendly formatting for logs - - --log.file.filter - The filter to use for logs written to the log file - - [default: debug] - - --log.file.directory - The path to put log files in - - [default: /logs] - - --log.file.max-size - The maximum size (in MB) of one log file - - [default: 200] - - --log.file.max-files - The maximum amount of log files that will be stored. If set to 0, background file logging is disabled - - [default: 5] - - --log.journald - Write logs to journald - - --log.journald.filter - The filter to use for logs written to journald - - [default: error] - - --color - Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting - - [default: always] - - Possible values: - - always: Colors on - - auto: Colors on - - never: Colors off - -Display: - -v, --verbosity... - Set the minimum log level. - - -v Errors - -vv Warnings - -vvv Info - -vvvv Debug - -vvvvv Traces (warning: very verbose!) - - -q, --quiet - Silence all log output -``` \ No newline at end of file diff --git a/book/cli/reth/debug/in-memory-merkle.md b/book/cli/reth/debug/in-memory-merkle.md deleted file mode 100644 index 7db3b2d2ba8..00000000000 --- a/book/cli/reth/debug/in-memory-merkle.md +++ /dev/null @@ -1,328 +0,0 @@ -# reth debug in-memory-merkle - -Debug in-memory state root calculation - -```bash -$ reth debug in-memory-merkle --help -``` -```txt -Usage: reth debug in-memory-merkle [OPTIONS] - -Options: - -h, --help - Print help (see a summary with '-h') - -Datadir: - --datadir - The path to the data dir for all reth files and subdirectories. - - Defaults to the OS-specific data directory: - - - Linux: `$XDG_DATA_HOME/reth/` or `$HOME/.local/share/reth/` - - Windows: `{FOLDERID_RoamingAppData}/reth/` - - macOS: `$HOME/Library/Application Support/reth/` - - [default: default] - - --datadir.static-files - The absolute path to store static files in. - - --config - The path to the configuration file to use - - --chain - The chain this node is running. - Possible values are either a built-in chain or the path to a chain specification file. - - Built-in chains: - mainnet, sepolia, holesky, hoodi, dev - - [default: mainnet] - -Database: - --db.log-level - Database logging level. Levels higher than "notice" require a debug build - - Possible values: - - fatal: Enables logging for critical conditions, i.e. assertion failures - - error: Enables logging for error conditions - - warn: Enables logging for warning conditions - - notice: Enables logging for normal but significant condition - - verbose: Enables logging for verbose informational - - debug: Enables logging for debug-level messages - - trace: Enables logging for trace debug-level messages - - extra: Enables logging for extra debug-level messages - - --db.exclusive - Open environment in exclusive/monopolistic mode. Makes it possible to open a database on an NFS volume - - [possible values: true, false] - - --db.max-size - Maximum database size (e.g., 4TB, 8MB) - - --db.growth-step - Database growth step (e.g., 4GB, 4KB) - - --db.read-transaction-timeout - Read transaction timeout in seconds, 0 means no timeout - -Networking: - -d, --disable-discovery - Disable the discovery service - - --disable-dns-discovery - Disable the DNS discovery - - --disable-discv4-discovery - Disable Discv4 discovery - - --enable-discv5-discovery - Enable Discv5 discovery - - --disable-nat - Disable Nat discovery - - --discovery.addr - The UDP address to use for devp2p peer discovery version 4 - - [default: 0.0.0.0] - - --discovery.port - The UDP port to use for devp2p peer discovery version 4 - - [default: 30303] - - --discovery.v5.addr - The UDP IPv4 address to use for devp2p peer discovery version 5. Overwritten by `RLPx` address, if it's also IPv4 - - --discovery.v5.addr.ipv6 - The UDP IPv6 address to use for devp2p peer discovery version 5. Overwritten by `RLPx` address, if it's also IPv6 - - --discovery.v5.port - The UDP IPv4 port to use for devp2p peer discovery version 5. Not used unless `--addr` is IPv4, or `--discovery.v5.addr` is set - - [default: 9200] - - --discovery.v5.port.ipv6 - The UDP IPv6 port to use for devp2p peer discovery version 5. Not used unless `--addr` is IPv6, or `--discovery.addr.ipv6` is set - - [default: 9200] - - --discovery.v5.lookup-interval - The interval in seconds at which to carry out periodic lookup queries, for the whole run of the program - - [default: 20] - - --discovery.v5.bootstrap.lookup-interval - The interval in seconds at which to carry out boost lookup queries, for a fixed number of times, at bootstrap - - [default: 5] - - --discovery.v5.bootstrap.lookup-countdown - The number of times to carry out boost lookup queries at bootstrap - - [default: 200] - - --trusted-peers - Comma separated enode URLs of trusted peers for P2P connections. - - --trusted-peers enode://abcd@192.168.0.1:30303 - - --trusted-only - Connect to or accept from trusted peers only - - --bootnodes - Comma separated enode URLs for P2P discovery bootstrap. - - Will fall back to a network-specific default if not specified. - - --dns-retries - Amount of DNS resolution requests retries to perform when peering - - [default: 0] - - --peers-file - The path to the known peers file. Connected peers are dumped to this file on nodes - shutdown, and read on startup. Cannot be used with `--no-persist-peers`. - - --identity - Custom node identity - - [default: reth/-/] - - --p2p-secret-key - Secret key to use for this node. - - This will also deterministically set the peer ID. If not specified, it will be set in the data dir for the chain being used. - - --no-persist-peers - Do not persist peers. - - --nat - NAT resolution method (any|none|upnp|publicip|extip:\) - - [default: any] - - --addr - Network listening address - - [default: 0.0.0.0] - - --port - Network listening port - - [default: 30303] - - --max-outbound-peers - Maximum number of outbound requests. default: 100 - - --max-inbound-peers - Maximum number of inbound requests. default: 30 - - --max-tx-reqs - Max concurrent `GetPooledTransactions` requests. - - [default: 130] - - --max-tx-reqs-peer - Max concurrent `GetPooledTransactions` requests per peer. - - [default: 1] - - --max-seen-tx-history - Max number of seen transactions to remember per peer. - - Default is 320 transaction hashes. - - [default: 320] - - --max-pending-imports - Max number of transactions to import concurrently. - - [default: 4096] - - --pooled-tx-response-soft-limit - Experimental, for usage in research. Sets the max accumulated byte size of transactions - to pack in one response. - Spec'd at 2MiB. - - [default: 2097152] - - --pooled-tx-pack-soft-limit - Experimental, for usage in research. Sets the max accumulated byte size of transactions to - request in one request. - - Since `RLPx` protocol version 68, the byte size of a transaction is shared as metadata in a - transaction announcement (see `RLPx` specs). This allows a node to request a specific size - response. - - By default, nodes request only 128 KiB worth of transactions, but should a peer request - more, up to 2 MiB, a node will answer with more than 128 KiB. - - Default is 128 KiB. - - [default: 131072] - - --max-tx-pending-fetch - Max capacity of cache of hashes for transactions pending fetch. - - [default: 25600] - - --net-if.experimental - Name of network interface used to communicate with peers. - - If flag is set, but no value is passed, the default interface for docker `eth0` is tried. - - --tx-propagation-policy - Transaction Propagation Policy - - The policy determines which peers transactions are gossiped to. - - [default: All] - - --retries - The number of retries per request - - [default: 5] - - --skip-node-depth - The depth after which we should start comparing branch nodes - -Logging: - --log.stdout.format - The format to use for logs written to stdout - - [default: terminal] - - Possible values: - - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - - terminal: Represents terminal-friendly formatting for logs - - --log.stdout.filter - The filter to use for logs written to stdout - - [default: ] - - --log.file.format - The format to use for logs written to the log file - - [default: terminal] - - Possible values: - - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - - terminal: Represents terminal-friendly formatting for logs - - --log.file.filter - The filter to use for logs written to the log file - - [default: debug] - - --log.file.directory - The path to put log files in - - [default: /logs] - - --log.file.max-size - The maximum size (in MB) of one log file - - [default: 200] - - --log.file.max-files - The maximum amount of log files that will be stored. If set to 0, background file logging is disabled - - [default: 5] - - --log.journald - Write logs to journald - - --log.journald.filter - The filter to use for logs written to journald - - [default: error] - - --color - Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting - - [default: always] - - Possible values: - - always: Colors on - - auto: Colors on - - never: Colors off - -Display: - -v, --verbosity... - Set the minimum log level. - - -v Errors - -vv Warnings - -vvv Info - -vvvv Debug - -vvvvv Traces (warning: very verbose!) - - -q, --quiet - Silence all log output -``` \ No newline at end of file diff --git a/book/cli/reth/debug/replay-engine.md b/book/cli/reth/debug/replay-engine.md deleted file mode 100644 index da36f11cc0e..00000000000 --- a/book/cli/reth/debug/replay-engine.md +++ /dev/null @@ -1,332 +0,0 @@ -# reth debug replay-engine - -Debug engine API by replaying stored messages - -```bash -$ reth debug replay-engine --help -``` -```txt -Usage: reth debug replay-engine [OPTIONS] --engine-api-store - -Options: - --instance - Add a new instance of a node. - - Configures the ports of the node to avoid conflicts with the defaults. This is useful for running multiple nodes on the same machine. - - Max number of instances is 200. It is chosen in a way so that it's not possible to have port numbers that conflict with each other. - - Changes to the following port numbers: - `DISCOVERY_PORT`: default + `instance` - 1 - `AUTH_PORT`: default + `instance` * 100 - 100 - `HTTP_RPC_PORT`: default - `instance` + 1 - `WS_RPC_PORT`: default + `instance` * 2 - 2 - - [default: 1] - - -h, --help - Print help (see a summary with '-h') - -Datadir: - --datadir - The path to the data dir for all reth files and subdirectories. - - Defaults to the OS-specific data directory: - - - Linux: `$XDG_DATA_HOME/reth/` or `$HOME/.local/share/reth/` - - Windows: `{FOLDERID_RoamingAppData}/reth/` - - macOS: `$HOME/Library/Application Support/reth/` - - [default: default] - - --datadir.static-files - The absolute path to store static files in. - - --config - The path to the configuration file to use - - --chain - The chain this node is running. - Possible values are either a built-in chain or the path to a chain specification file. - - Built-in chains: - mainnet, sepolia, holesky, dev - - [default: mainnet] - -Database: - --db.log-level - Database logging level. Levels higher than "notice" require a debug build - - Possible values: - - fatal: Enables logging for critical conditions, i.e. assertion failures - - error: Enables logging for error conditions - - warn: Enables logging for warning conditions - - notice: Enables logging for normal but significant condition - - verbose: Enables logging for verbose informational - - debug: Enables logging for debug-level messages - - trace: Enables logging for trace debug-level messages - - extra: Enables logging for extra debug-level messages - - --db.exclusive - Open environment in exclusive/monopolistic mode. Makes it possible to open a database on an NFS volume - - [possible values: true, false] - - --db.max-size - Maximum database size (e.g., 4TB, 8MB) - - --db.growth-step - Database growth step (e.g., 4GB, 4KB) - - --db.read-transaction-timeout - Read transaction timeout in seconds, 0 means no timeout - -Networking: - -d, --disable-discovery - Disable the discovery service - - --disable-dns-discovery - Disable the DNS discovery - - --disable-discv4-discovery - Disable Discv4 discovery - - --enable-discv5-discovery - Enable Discv5 discovery - - --disable-nat - Disable Nat discovery - - --discovery.addr - The UDP address to use for devp2p peer discovery version 4 - - [default: 0.0.0.0] - - --discovery.port - The UDP port to use for devp2p peer discovery version 4 - - [default: 30303] - - --discovery.v5.addr - The UDP IPv4 address to use for devp2p peer discovery version 5. Overwritten by `RLPx` address, if it's also IPv4 - - --discovery.v5.addr.ipv6 - The UDP IPv6 address to use for devp2p peer discovery version 5. Overwritten by `RLPx` address, if it's also IPv6 - - --discovery.v5.port - The UDP IPv4 port to use for devp2p peer discovery version 5. Not used unless `--addr` is IPv4, or `--discovery.v5.addr` is set - - [default: 9200] - - --discovery.v5.port.ipv6 - The UDP IPv6 port to use for devp2p peer discovery version 5. Not used unless `--addr` is IPv6, or `--discovery.addr.ipv6` is set - - [default: 9200] - - --discovery.v5.lookup-interval - The interval in seconds at which to carry out periodic lookup queries, for the whole run of the program - - [default: 20] - - --discovery.v5.bootstrap.lookup-interval - The interval in seconds at which to carry out boost lookup queries, for a fixed number of times, at bootstrap - - [default: 5] - - --discovery.v5.bootstrap.lookup-countdown - The number of times to carry out boost lookup queries at bootstrap - - [default: 200] - - --trusted-peers - Comma separated enode URLs of trusted peers for P2P connections. - - --trusted-peers enode://abcd@192.168.0.1:30303 - - --trusted-only - Connect to or accept from trusted peers only - - --bootnodes - Comma separated enode URLs for P2P discovery bootstrap. - - Will fall back to a network-specific default if not specified. - - --dns-retries - Amount of DNS resolution requests retries to perform when peering - - [default: 0] - - --peers-file - The path to the known peers file. Connected peers are dumped to this file on nodes - shutdown, and read on startup. Cannot be used with `--no-persist-peers`. - - --identity - Custom node identity - - [default: reth/-/] - - --p2p-secret-key - Secret key to use for this node. - - This will also deterministically set the peer ID. If not specified, it will be set in the data dir for the chain being used. - - --no-persist-peers - Do not persist peers. - - --nat - NAT resolution method (any|none|upnp|publicip|extip:\) - - [default: any] - - --addr - Network listening address - - [default: 0.0.0.0] - - --port - Network listening port - - [default: 30303] - - --max-outbound-peers - Maximum number of outbound requests. default: 100 - - --max-inbound-peers - Maximum number of inbound requests. default: 30 - - --max-tx-reqs - Max concurrent `GetPooledTransactions` requests. - - [default: 130] - - --max-tx-reqs-peer - Max concurrent `GetPooledTransactions` requests per peer. - - [default: 1] - - --max-seen-tx-history - Max number of seen transactions to remember per peer. - - Default is 320 transaction hashes. - - [default: 320] - - --max-pending-imports - Max number of transactions to import concurrently. - - [default: 4096] - - --pooled-tx-response-soft-limit - Experimental, for usage in research. Sets the max accumulated byte size of transactions - to pack in one response. - Spec'd at 2MiB. - - [default: 2097152] - - --pooled-tx-pack-soft-limit - Experimental, for usage in research. Sets the max accumulated byte size of transactions to - request in one request. - - Since `RLPx` protocol version 68, the byte size of a transaction is shared as metadata in a - transaction announcement (see `RLPx` specs). This allows a node to request a specific size - response. - - By default, nodes request only 128 KiB worth of transactions, but should a peer request - more, up to 2 MiB, a node will answer with more than 128 KiB. - - Default is 128 KiB. - - [default: 131072] - - --max-tx-pending-fetch - Max capacity of cache of hashes for transactions pending fetch. - - [default: 25600] - - --net-if.experimental - Name of network interface used to communicate with peers. - - If flag is set, but no value is passed, the default interface for docker `eth0` is tried. - - --engine-api-store - The path to read engine API messages from - - --interval - The number of milliseconds between Engine API messages - - [default: 1000] - -Logging: - --log.stdout.format - The format to use for logs written to stdout - - [default: terminal] - - Possible values: - - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - - terminal: Represents terminal-friendly formatting for logs - - --log.stdout.filter - The filter to use for logs written to stdout - - [default: ] - - --log.file.format - The format to use for logs written to the log file - - [default: terminal] - - Possible values: - - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - - terminal: Represents terminal-friendly formatting for logs - - --log.file.filter - The filter to use for logs written to the log file - - [default: debug] - - --log.file.directory - The path to put log files in - - [default: /logs] - - --log.file.max-size - The maximum size (in MB) of one log file - - [default: 200] - - --log.file.max-files - The maximum amount of log files that will be stored. If set to 0, background file logging is disabled - - [default: 5] - - --log.journald - Write logs to journald - - --log.journald.filter - The filter to use for logs written to journald - - [default: error] - - --color - Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting - - [default: always] - - Possible values: - - always: Colors on - - auto: Colors on - - never: Colors off - -Display: - -v, --verbosity... - Set the minimum log level. - - -v Errors - -vv Warnings - -vvv Info - -vvvv Debug - -vvvvv Traces (warning: very verbose!) - - -q, --quiet - Silence all log output -``` \ No newline at end of file diff --git a/book/cli/reth/import-receipts-op.md b/book/cli/reth/import-receipts-op.md deleted file mode 100644 index 0b7135e1d7a..00000000000 --- a/book/cli/reth/import-receipts-op.md +++ /dev/null @@ -1,133 +0,0 @@ -# op-reth import-receipts-op - -This imports non-standard RLP encoded receipts from a file. -The supported RLP encoding, is the non-standard encoding used -for receipt export in . -Supports import of OVM receipts from the Bedrock datadir. - -```bash -$ op-reth import-receipts-op --help -Usage: op-reth import-receipts-op [OPTIONS] - -Options: - --datadir - The path to the data dir for all reth files and subdirectories. - - Defaults to the OS-specific data directory: - - - Linux: `$XDG_DATA_HOME/reth/` or `$HOME/.local/share/reth/` - - Windows: `{FOLDERID_RoamingAppData}/reth/` - - macOS: `$HOME/Library/Application Support/reth/` - - [default: default] - - --chunk-len - Chunk byte length to read from file. - - [default: 1GB] - - -h, --help - Print help (see a summary with '-h') - -Database: - --db.log-level - Database logging level. Levels higher than "notice" require a debug build - - Possible values: - - fatal: Enables logging for critical conditions, i.e. assertion failures - - error: Enables logging for error conditions - - warn: Enables logging for warning conditions - - notice: Enables logging for normal but significant condition - - verbose: Enables logging for verbose informational - - debug: Enables logging for debug-level messages - - trace: Enables logging for trace debug-level messages - - extra: Enables logging for extra debug-level messages - - --db.exclusive - Open environment in exclusive/monopolistic mode. Makes it possible to open a database on an NFS volume - - [possible values: true, false] - - - The path to a receipts file for import. File must use `OpGethReceiptFileCodec` (used for - exporting OP chain segment below Bedrock block via testinprod/op-geth). - - - -Logging: - --log.stdout.format - The format to use for logs written to stdout - - [default: terminal] - - Possible values: - - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - - terminal: Represents terminal-friendly formatting for logs - - --log.stdout.filter - The filter to use for logs written to stdout - - [default: ] - - --log.file.format - The format to use for logs written to the log file - - [default: terminal] - - Possible values: - - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - - terminal: Represents terminal-friendly formatting for logs - - --log.file.filter - The filter to use for logs written to the log file - - [default: debug] - - --log.file.directory - The path to put log files in - - [default: /logs] - - --log.file.max-size - The maximum size (in MB) of one log file - - [default: 200] - - --log.file.max-files - The maximum amount of log files that will be stored. If set to 0, background file logging is disabled - - [default: 5] - - --log.journald - Write logs to journald - - --log.journald.filter - The filter to use for logs written to journald - - [default: error] - - --color - Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting - - [default: always] - - Possible values: - - always: Colors on - - auto: Colors on - - never: Colors off - -Display: - -v, --verbosity... - Set the minimum log level. - - -v Errors - -vv Warnings - -vvv Info - -vvvv Debug - -vvvvv Traces (warning: very verbose!) - - -q, --quiet - Silence all log output -``` \ No newline at end of file diff --git a/book/cli/reth/p2p/body.md b/book/cli/reth/p2p/body.md deleted file mode 100644 index e5092f274ea..00000000000 --- a/book/cli/reth/p2p/body.md +++ /dev/null @@ -1,95 +0,0 @@ -# reth p2p body - -Download block body - -```bash -$ reth p2p body --help -``` -```txt -Usage: reth p2p body [OPTIONS] - -Arguments: - - The block number or hash - -Options: - -h, --help - Print help (see a summary with '-h') - -Logging: - --log.stdout.format - The format to use for logs written to stdout - - [default: terminal] - - Possible values: - - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - - terminal: Represents terminal-friendly formatting for logs - - --log.stdout.filter - The filter to use for logs written to stdout - - [default: ] - - --log.file.format - The format to use for logs written to the log file - - [default: terminal] - - Possible values: - - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - - terminal: Represents terminal-friendly formatting for logs - - --log.file.filter - The filter to use for logs written to the log file - - [default: debug] - - --log.file.directory - The path to put log files in - - [default: /logs] - - --log.file.max-size - The maximum size (in MB) of one log file - - [default: 200] - - --log.file.max-files - The maximum amount of log files that will be stored. If set to 0, background file logging is disabled - - [default: 5] - - --log.journald - Write logs to journald - - --log.journald.filter - The filter to use for logs written to journald - - [default: error] - - --color - Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting - - [default: always] - - Possible values: - - always: Colors on - - auto: Colors on - - never: Colors off - -Display: - -v, --verbosity... - Set the minimum log level. - - -v Errors - -vv Warnings - -vvv Info - -vvvv Debug - -vvvvv Traces (warning: very verbose!) - - -q, --quiet - Silence all log output -``` \ No newline at end of file diff --git a/book/cli/reth/p2p/header.md b/book/cli/reth/p2p/header.md deleted file mode 100644 index 8b1f6b96cd8..00000000000 --- a/book/cli/reth/p2p/header.md +++ /dev/null @@ -1,95 +0,0 @@ -# reth p2p header - -Download block header - -```bash -$ reth p2p header --help -``` -```txt -Usage: reth p2p header [OPTIONS] - -Arguments: - - The header number or hash - -Options: - -h, --help - Print help (see a summary with '-h') - -Logging: - --log.stdout.format - The format to use for logs written to stdout - - [default: terminal] - - Possible values: - - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - - terminal: Represents terminal-friendly formatting for logs - - --log.stdout.filter - The filter to use for logs written to stdout - - [default: ] - - --log.file.format - The format to use for logs written to the log file - - [default: terminal] - - Possible values: - - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - - terminal: Represents terminal-friendly formatting for logs - - --log.file.filter - The filter to use for logs written to the log file - - [default: debug] - - --log.file.directory - The path to put log files in - - [default: /logs] - - --log.file.max-size - The maximum size (in MB) of one log file - - [default: 200] - - --log.file.max-files - The maximum amount of log files that will be stored. If set to 0, background file logging is disabled - - [default: 5] - - --log.journald - Write logs to journald - - --log.journald.filter - The filter to use for logs written to journald - - [default: error] - - --color - Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting - - [default: always] - - Possible values: - - always: Colors on - - auto: Colors on - - never: Colors off - -Display: - -v, --verbosity... - Set the minimum log level. - - -v Errors - -vv Warnings - -vvv Info - -vvvv Debug - -vvvvv Traces (warning: very verbose!) - - -q, --quiet - Silence all log output -``` \ No newline at end of file diff --git a/book/cli/reth/test-vectors.md b/book/cli/reth/test-vectors.md deleted file mode 100644 index 844c5ed8455..00000000000 --- a/book/cli/reth/test-vectors.md +++ /dev/null @@ -1,113 +0,0 @@ -# reth test-vectors - -Generate Test Vectors - -```bash -$ reth test-vectors --help -Usage: reth test-vectors [OPTIONS] - -Commands: - tables Generates test vectors for specified tables. If no table is specified, generate for all - help Print this message or the help of the given subcommand(s) - -Options: - --chain - The chain this node is running. - Possible values are either a built-in chain or the path to a chain specification file. - - Built-in chains: - mainnet, sepolia, holesky, dev - - [default: mainnet] - - --instance - Add a new instance of a node. - - Configures the ports of the node to avoid conflicts with the defaults. This is useful for running multiple nodes on the same machine. - - Max number of instances is 200. It is chosen in a way so that it's not possible to have port numbers that conflict with each other. - - Changes to the following port numbers: - `DISCOVERY_PORT`: default + `instance` - 1 - `AUTH_PORT`: default + `instance` * 100 - 100 - `HTTP_RPC_PORT`: default - `instance` + 1 - `WS_RPC_PORT`: default + `instance` * 2 - 2 - - [default: 1] - - -h, --help - Print help (see a summary with '-h') - -Logging: - --log.stdout.format - The format to use for logs written to stdout - - [default: terminal] - - Possible values: - - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - - terminal: Represents terminal-friendly formatting for logs - - --log.stdout.filter - The filter to use for logs written to stdout - - [default: ] - - --log.file.format - The format to use for logs written to the log file - - [default: terminal] - - Possible values: - - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - - terminal: Represents terminal-friendly formatting for logs - - --log.file.filter - The filter to use for logs written to the log file - - [default: debug] - - --log.file.directory - The path to put log files in - - [default: /logs] - - --log.file.max-size - The maximum size (in MB) of one log file - - [default: 200] - - --log.file.max-files - The maximum amount of log files that will be stored. If set to 0, background file logging is disabled - - [default: 5] - - --log.journald - Write logs to journald - - --log.journald.filter - The filter to use for logs written to journald - - [default: error] - - --color - Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting - - [default: always] - - Possible values: - - always: Colors on - - auto: Colors on - - never: Colors off - -Display: - -v, --verbosity... - Set the minimum log level. - - -v Errors - -vv Warnings - -vvv Info - -vvvv Debug - -vvvvv Traces (warning: very verbose!) - - -q, --quiet - Silence all log output -``` \ No newline at end of file diff --git a/book/cli/update.sh b/book/cli/update.sh deleted file mode 100755 index 6e792df0f2b..00000000000 --- a/book/cli/update.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail - -BOOK_ROOT="$(dirname "$(dirname "$0")")" -RETH=${1:-"$(dirname "$BOOK_ROOT")/target/debug/reth"} - -cmd=( - "$(dirname "$0")/help.rs" - --root-dir "$BOOK_ROOT/" - --root-indentation 2 - --root-summary - --out-dir "$BOOK_ROOT/cli/" - "$RETH" -) -echo "Running: $" "${cmd[*]}" -"${cmd[@]}" diff --git a/book/developers/contribute.md b/book/developers/contribute.md deleted file mode 100644 index 74f00e69a1a..00000000000 --- a/book/developers/contribute.md +++ /dev/null @@ -1,9 +0,0 @@ -# Contribute - - - -Reth has docs specifically geared for developers and contributors, including documentation on the structure and architecture of reth, the general workflow we employ, and other useful tips. - -You can find these docs [here](https://github.com/paradigmxyz/reth/tree/main/docs). - -Check out our contributing guidelines [here](https://github.com/paradigmxyz/reth/blob/main/CONTRIBUTING.md). diff --git a/book/developers/developers.md b/book/developers/developers.md deleted file mode 100644 index 9d8c5a9c673..00000000000 --- a/book/developers/developers.md +++ /dev/null @@ -1,3 +0,0 @@ -# Developers - -Reth is composed of several crates that can be used in standalone projects. If you are interested in using one or more of the crates, you can get an overview of them in the [developer docs](https://github.com/paradigmxyz/reth/tree/main/docs), or take a look at the [crate docs](https://paradigmxyz.github.io/reth/docs). diff --git a/book/installation/priorities.md b/book/installation/priorities.md deleted file mode 100644 index f7444e79d63..00000000000 --- a/book/installation/priorities.md +++ /dev/null @@ -1,18 +0,0 @@ -# Update Priorities - -When publishing releases, reth will include an "Update Priority" section in the release notes, in the same manner Lighthouse does. - -The "Update Priority" section will include a table which may appear like so: - -| User Class | Priority | -|----------------------|-----------------| -| Payload Builders | Medium Priority | -| Non-Payload Builders | Low Priority | - -To understand this table, the following terms are important: - -- *Payload builders* are those who use reth to build and validate payloads. -- *Non-payload builders* are those who run reth for other purposes (e.g., data analysis, RPC or applications). -- *High priority* updates should be completed as soon as possible (e.g., hours or days). -- *Medium priority* updates should be completed at the next convenience (e.g., days or a week). -- *Low priority* updates should be completed in the next routine update cycle (e.g., two weeks). diff --git a/book/run/mainnet.md b/book/run/mainnet.md deleted file mode 100644 index c4908971f69..00000000000 --- a/book/run/mainnet.md +++ /dev/null @@ -1,96 +0,0 @@ -# Running Reth on Ethereum Mainnet or testnets - -Reth is an [_execution client_](https://ethereum.org/en/developers/docs/nodes-and-clients/#execution-clients). After Ethereum's transition to Proof of Stake (aka the Merge) it became required to run a [_consensus client_](https://ethereum.org/en/developers/docs/nodes-and-clients/#consensus-clients) along your execution client in order to sync into any "post-Merge" network. This is because the Ethereum execution layer now outsources consensus to a separate component, known as the consensus client. - -Consensus clients decide what blocks are part of the chain, while execution clients only validate that transactions and blocks are valid in themselves and with respect to the world state. In other words, execution clients execute blocks and transactions and check their validity, while consensus clients determine which valid blocks should be part of the chain. Therefore, running a consensus client in parallel with the execution client is necessary to ensure synchronization and participation in the network. - -By running both an execution client like Reth and a consensus client, such as Lighthouse 🦀 (which we will assume for this guide), you can effectively contribute to the Ethereum network and participate in the consensus process, even if you don't intend to run validators. - -| Client | Role | -|-------------|--------------------------------------------------| -| Execution | Validates transactions and blocks | -| | (checks their validity and global state) | -| Consensus | Determines which blocks are part of the chain | -| | (makes consensus decisions) | - -## Running the Reth Node - -First, ensure that you have Reth installed by following the [installation instructions][installation]. - -Now, to start the archive node, run: - -```bash -reth node -``` - -And to start the full node, run: -```bash -reth node --full -``` - -On differences between archive and full nodes, see [Pruning & Full Node](./pruning.md#basic-concepts) section. - -> Note that these commands will not open any HTTP/WS ports by default. You can change this by adding the `--http`, `--ws` flags, respectively and using the `--http.api` and `--ws.api` flags to enable various [JSON-RPC APIs](../jsonrpc/intro.md). For more commands, see the [`reth node` CLI reference](../cli/reth/node.md). - -The EL <> CL communication happens over the [Engine API](https://github.com/ethereum/execution-apis/blob/main/src/engine/common.md), which is by default exposed at `http://localhost:8551`. The connection is authenticated over JWT using a JWT secret which is auto-generated by Reth and placed in a file called `jwt.hex` in the data directory, which on Linux by default is `$HOME/.local/share/reth/` (`/Users//Library/Application Support/reth/mainnet/jwt.hex` in Mac). - -You can override this path using the `--authrpc.jwtsecret` option. You MUST use the same JWT secret in BOTH Reth and the chosen Consensus Layer. If you want to override the address or port, you can use the `--authrpc.addr` and `--authrpc.port` options, respectively. - -So one might do: - -```bash -reth node \ - --authrpc.jwtsecret /path/to/secret \ - --authrpc.addr 127.0.0.1 \ - --authrpc.port 8551 -``` - -At this point, our Reth node has started discovery, and even discovered some new peers. But it will not start syncing until you spin up the consensus layer! - -## Running the Consensus Layer - -First, make sure you have Lighthouse installed. Sigma Prime provides excellent [installation](https://lighthouse-book.sigmaprime.io/installation.html) and [node operation](https://lighthouse-book.sigmaprime.io/run_a_node.html) instructions. - -Assuming you have done that, run: - -```bash -lighthouse bn \ - --checkpoint-sync-url https://mainnet.checkpoint.sigp.io \ - --execution-endpoint http://localhost:8551 \ - --execution-jwt /path/to/secret -``` - -If you don't intend on running validators on your node you can add: - -``` bash - --disable-deposit-contract-sync -``` - -The `--checkpoint-sync-url` argument value can be replaced with any checkpoint sync endpoint from a [community maintained list](https://eth-clients.github.io/checkpoint-sync-endpoints/#mainnet). - -Your Reth node should start receiving "fork choice updated" messages, and begin syncing the chain. - -## Verify the chain is growing - -You can easily verify that by inspecting the logs, and seeing that headers are arriving in Reth. Sit back now and wait for the stages to run! -In the meantime, consider setting up [observability](./observability.md) to monitor your node's health or [test the JSON RPC API](../jsonrpc/intro.md). - - - -[installation]: ./../installation/installation.md -[docs]: https://github.com/paradigmxyz/reth/tree/main/docs -[metrics]: https://github.com/paradigmxyz/reth/blob/main/docs/design/metrics.md#current-metrics - -## Running without a Consensus Layer - -We provide a method for running Reth without a Consensus Layer via the `--debug.tip ` parameter. If you provide that to your node, it will simulate sending an `engine_forkchoiceUpdated` message _once_ and will trigger syncing to the provided block hash. This is useful for testing and debugging purposes, but in order to have a node that can keep up with the tip you'll need to run a CL alongside it. At the moment we have no plans of including a Consensus Layer implementation in Reth, and we are open to including light clients other methods of syncing like importing Lighthouse as a library. - -## Running with Etherscan as Block Source - -You can use `--debug.etherscan` to run Reth with a fake consensus client that advances the chain using recent blocks on Etherscan. This requires an Etherscan API key (set via `ETHERSCAN_API_KEY` environment variable). Optionally, specify a custom API URL with `--debug.etherscan `. - -Example: -```bash -export ETHERSCAN_API_KEY=your_api_key_here -reth node --debug.etherscan -``` \ No newline at end of file diff --git a/book/run/ports.md b/book/run/ports.md deleted file mode 100644 index 5239a5262c4..00000000000 --- a/book/run/ports.md +++ /dev/null @@ -1,38 +0,0 @@ -# Ports - -This section provides essential information about the ports used by the system, their primary purposes, and recommendations for exposure settings. - -## Peering Ports - -- **Port:** 30303 -- **Protocol:** TCP and UDP -- **Purpose:** Peering with other nodes for synchronization of blockchain data. Nodes communicate through this port to maintain network consensus and share updated information. -- **Exposure Recommendation:** This port should be exposed to enable seamless interaction and synchronization with other nodes in the network. - -## Metrics Port - -- **Port:** 9001 -- **Protocol:** TCP -- **Purpose:** This port is designated for serving metrics related to the system's performance and operation. It allows internal monitoring and data collection for analysis. -- **Exposure Recommendation:** By default, this port should not be exposed to the public. It is intended for internal monitoring and analysis purposes. - -## HTTP RPC Port - -- **Port:** 8545 -- **Protocol:** TCP -- **Purpose:** Port 8545 provides an HTTP-based Remote Procedure Call (RPC) interface. It enables external applications to interact with the blockchain by sending requests over HTTP. -- **Exposure Recommendation:** Similar to the metrics port, exposing this port to the public is not recommended by default due to security considerations. - -## WS RPC Port - -- **Port:** 8546 -- **Protocol:** TCP -- **Purpose:** Port 8546 offers a WebSocket-based Remote Procedure Call (RPC) interface. It allows real-time communication between external applications and the blockchain. -- **Exposure Recommendation:** As with the HTTP RPC port, the WS RPC port should not be exposed by default for security reasons. - -## Engine API Port - -- **Port:** 8551 -- **Protocol:** TCP -- **Purpose:** Port 8551 facilitates communication between specific components, such as "reth" and "CL" (assuming their definitions are understood within the context of the system). It enables essential internal processes. -- **Exposure Recommendation:** This port is not meant to be exposed to the public by default. It should be reserved for internal communication between vital components of the system. diff --git a/book/run/run-a-node.md b/book/run/run-a-node.md deleted file mode 100644 index d8981e15522..00000000000 --- a/book/run/run-a-node.md +++ /dev/null @@ -1,15 +0,0 @@ -# Run a Node - -Congratulations, now that you have installed Reth, it's time to run it! - -In this chapter we'll go through a few different topics you'll encounter when running Reth, including: -1. [Running on mainnet or official testnets](./mainnet.md) -1. [Running on OP Stack chains](./optimism.md) -1. [Logs and Observability](./observability.md) -1. [Configuring reth.toml](./config.md) -1. [Transaction types](./transactions.md) -1. [Pruning & Full Node](./pruning.md) -1. [Ports](./ports.md) -1. [Troubleshooting](./troubleshooting.md) - -In the future, we also intend to support the [OP Stack](https://docs.optimism.io/get-started/superchain), which will allow you to run Reth as a Layer 2 client. More there soon! diff --git a/book/sources/Cargo.toml b/book/sources/Cargo.toml deleted file mode 100644 index b374ad798b5..00000000000 --- a/book/sources/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[workspace] -members = ["exex/hello-world", "exex/remote", "exex/tracking-state"] - -# Explicitly set the resolver to version 2, which is the default for packages with edition >= 2021 -# https://doc.rust-lang.org/edition-guide/rust-2021/default-cargo-resolver.html -resolver = "2" - -[patch.'https://github.com/paradigmxyz/reth'] -reth = { path = "../../bin/reth" } -reth-exex = { path = "../../crates/exex/exex" } -reth-node-ethereum = { path = "../../crates/ethereum/node" } -reth-tracing = { path = "../../crates/tracing" } -reth-node-api = { path = "../../crates/node/api" } diff --git a/book/templates/source_and_github.md b/book/templates/source_and_github.md deleted file mode 100644 index c4abbaa3894..00000000000 --- a/book/templates/source_and_github.md +++ /dev/null @@ -1,4 +0,0 @@ -[File: [[ #path ]]](https://github.com/paradigmxyz/reth/blob/main/[[ #path ]]) -```rust,no_run,noplayground -{{#include [[ #path_to_root ]][[ #path ]]:[[ #anchor ]]}} -``` \ No newline at end of file diff --git a/book/theme/head.hbs b/book/theme/head.hbs deleted file mode 100644 index 37667d80f6e..00000000000 --- a/book/theme/head.hbs +++ /dev/null @@ -1,5 +0,0 @@ - - - - - diff --git a/clippy.toml b/clippy.toml index bdc50bb3fda..9ddf1014802 100644 --- a/clippy.toml +++ b/clippy.toml @@ -1,4 +1,3 @@ -msrv = "1.86" too-large-for-stack = 128 doc-valid-idents = [ "P2P", diff --git a/crates/chain-state/Cargo.toml b/crates/chain-state/Cargo.toml index 4d6e493fe34..d21c83ae7c4 100644 --- a/crates/chain-state/Cargo.toml +++ b/crates/chain-state/Cargo.toml @@ -41,6 +41,7 @@ derive_more.workspace = true metrics.workspace = true parking_lot.workspace = true pin-project.workspace = true +serde = { workspace = true, optional = true } # optional deps for test-utils alloy-signer = { workspace = true, optional = true } @@ -52,10 +53,26 @@ reth-primitives-traits = { workspace = true, features = ["test-utils"] } reth-testing-utils.workspace = true alloy-signer.workspace = true alloy-signer-local.workspace = true -alloy-consensus.workspace = true rand.workspace = true +revm-state.workspace = true +criterion.workspace = true [features] +serde = [ + "dep:serde", + "alloy-consensus/serde", + "alloy-eips/serde", + "alloy-primitives/serde", + "parking_lot/serde", + "rand?/serde", + "reth-ethereum-primitives/serde", + "reth-execution-types/serde", + "reth-primitives-traits/serde", + "reth-trie/serde", + "revm-database/serde", + "revm-state?/serde", + "reth-storage-api/serde", +] test-utils = [ "alloy-primitives/getrandom", "alloy-signer", @@ -67,3 +84,8 @@ test-utils = [ "reth-trie/test-utils", "reth-ethereum-primitives/test-utils", ] + +[[bench]] +name = "canonical_hashes_range" +harness = false +required-features = ["test-utils"] diff --git a/crates/chain-state/benches/canonical_hashes_range.rs b/crates/chain-state/benches/canonical_hashes_range.rs new file mode 100644 index 00000000000..c19ce25ec4f --- /dev/null +++ b/crates/chain-state/benches/canonical_hashes_range.rs @@ -0,0 +1,96 @@ +#![allow(missing_docs)] + +use criterion::{black_box, criterion_group, criterion_main, Criterion}; +use reth_chain_state::{ + test_utils::TestBlockBuilder, ExecutedBlock, MemoryOverlayStateProviderRef, +}; +use reth_ethereum_primitives::EthPrimitives; +use reth_storage_api::{noop::NoopProvider, BlockHashReader}; + +criterion_group!(benches, bench_canonical_hashes_range); +criterion_main!(benches); + +fn bench_canonical_hashes_range(c: &mut Criterion) { + let mut group = c.benchmark_group("canonical_hashes_range"); + + let scenarios = [("small", 10), ("medium", 100), ("large", 1000)]; + + for (name, num_blocks) in scenarios { + group.bench_function(format!("{}_blocks_{}", name, num_blocks), |b| { + let (provider, blocks) = setup_provider_with_blocks(num_blocks); + let start_block = blocks[0].recovered_block().number; + let end_block = blocks[num_blocks / 2].recovered_block().number; + + b.iter(|| { + black_box( + provider + .canonical_hashes_range(black_box(start_block), black_box(end_block)) + .unwrap(), + ) + }) + }); + } + + let (provider, blocks) = setup_provider_with_blocks(500); + let base_block = blocks[100].recovered_block().number; + + let range_sizes = [1, 10, 50, 100, 250]; + for range_size in range_sizes { + group.bench_function(format!("range_size_{}", range_size), |b| { + let end_block = base_block + range_size; + + b.iter(|| { + black_box( + provider + .canonical_hashes_range(black_box(base_block), black_box(end_block)) + .unwrap(), + ) + }) + }); + } + + // Benchmark edge cases + group.bench_function("no_in_memory_matches", |b| { + let (provider, blocks) = setup_provider_with_blocks(100); + let first_block = blocks[0].recovered_block().number; + let start_block = first_block - 50; + let end_block = first_block - 10; + + b.iter(|| { + black_box( + provider + .canonical_hashes_range(black_box(start_block), black_box(end_block)) + .unwrap(), + ) + }) + }); + + group.bench_function("all_in_memory_matches", |b| { + let (provider, blocks) = setup_provider_with_blocks(100); + let first_block = blocks[0].recovered_block().number; + let last_block = blocks[blocks.len() - 1].recovered_block().number; + + b.iter(|| { + black_box( + provider + .canonical_hashes_range(black_box(first_block), black_box(last_block + 1)) + .unwrap(), + ) + }) + }); + + group.finish(); +} + +fn setup_provider_with_blocks( + num_blocks: usize, +) -> (MemoryOverlayStateProviderRef<'static, EthPrimitives>, Vec>) { + let mut builder = TestBlockBuilder::::default(); + + let blocks: Vec<_> = builder.get_executed_blocks(1000..1000 + num_blocks as u64).collect(); + + let historical = Box::new(NoopProvider::default()); + let provider = MemoryOverlayStateProviderRef::new(historical, blocks.clone()); + + (provider, blocks) +} diff --git a/crates/chain-state/src/chain_info.rs b/crates/chain-state/src/chain_info.rs index a8a08430566..dd6afc8db1a 100644 --- a/crates/chain-state/src/chain_info.rs +++ b/crates/chain-state/src/chain_info.rs @@ -77,22 +77,22 @@ where self.inner.finalized_block.borrow().clone() } - /// Returns the canonical head of the chain. + /// Returns the `BlockNumHash` of the canonical head. pub fn get_canonical_num_hash(&self) -> BlockNumHash { self.inner.canonical_head.read().num_hash() } - /// Returns the canonical head of the chain. + /// Returns the block number of the canonical head. pub fn get_canonical_block_number(&self) -> BlockNumber { self.inner.canonical_head_number.load(Ordering::Relaxed) } - /// Returns the safe header of the chain. + /// Returns the `BlockNumHash` of the safe header. pub fn get_safe_num_hash(&self) -> Option { self.inner.safe_block.borrow().as_ref().map(SealedHeader::num_hash) } - /// Returns the finalized header of the chain. + /// Returns the `BlockNumHash` of the finalized header. pub fn get_finalized_num_hash(&self) -> Option { self.inner.finalized_block.borrow().as_ref().map(SealedHeader::num_hash) } diff --git a/crates/chain-state/src/in_memory.rs b/crates/chain-state/src/in_memory.rs index 14e3b02a3d8..a6c85538107 100644 --- a/crates/chain-state/src/in_memory.rs +++ b/crates/chain-state/src/in_memory.rs @@ -5,15 +5,16 @@ use crate::{ ChainInfoTracker, MemoryOverlayStateProvider, }; use alloy_consensus::{transaction::TransactionMeta, BlockHeader}; -use alloy_eips::{eip2718::Encodable2718, BlockHashOrNumber, BlockNumHash}; -use alloy_primitives::{map::HashMap, TxHash, B256}; +use alloy_eips::{BlockHashOrNumber, BlockNumHash}; +use alloy_primitives::{map::HashMap, BlockNumber, TxHash, B256}; use parking_lot::RwLock; use reth_chainspec::ChainInfo; use reth_ethereum_primitives::EthPrimitives; use reth_execution_types::{Chain, ExecutionOutcome}; use reth_metrics::{metrics::Gauge, Metrics}; use reth_primitives_traits::{ - BlockBody as _, NodePrimitives, RecoveredBlock, SealedBlock, SealedHeader, SignedTransaction, + BlockBody as _, IndexedTx, NodePrimitives, RecoveredBlock, SealedBlock, SealedHeader, + SignedTransaction, }; use reth_storage_api::StateProviderBox; use reth_trie::{updates::TrieUpdates, HashedPostState}; @@ -42,8 +43,9 @@ pub(crate) struct InMemoryStateMetrics { /// /// # Locking behavior on state updates /// -/// All update calls must be atomic, meaning that they must acquire all locks at once, before -/// modifying the state. This is to ensure that the internal state is always consistent. +/// All update calls must acquire all locks at once before modifying state to ensure the internal +/// state remains consistent. This prevents readers from observing partially updated state where +/// the numbers and blocks maps are out of sync. /// Update functions ensure that the numbers write lock is always acquired first, because lookup by /// numbers first read the numbers map and then the blocks map. /// By acquiring the numbers lock first, we ensure that read-only lookups don't deadlock updates. @@ -159,7 +161,7 @@ impl CanonicalInMemoryStateInner { } type PendingBlockAndReceipts = - (SealedBlock<::Block>, Vec>); + (RecoveredBlock<::Block>, Vec>); /// This type is responsible for providing the blocks, receipts, and state for /// all canonical blocks not on disk yet and keeps track of the block range that @@ -240,7 +242,7 @@ impl CanonicalInMemoryState { /// Updates the pending block with the given block. /// /// Note: This assumes that the parent block of the pending block is canonical. - pub fn set_pending_block(&self, pending: ExecutedBlockWithTrieUpdates) { + pub fn set_pending_block(&self, pending: ExecutedBlock) { // fetch the state of the pending block's parent block let parent = self.state_by_hash(pending.recovered_block().parent_hash()); let pending = BlockState::with_parent(pending, parent); @@ -256,7 +258,7 @@ impl CanonicalInMemoryState { /// them to their parent blocks. fn update_blocks(&self, new_blocks: I, reorged: R) where - I: IntoIterator>, + I: IntoIterator>, R: IntoIterator>, { { @@ -480,7 +482,7 @@ impl CanonicalInMemoryState { pub fn pending_block_and_receipts(&self) -> Option> { self.pending_state().map(|block_state| { ( - block_state.block_ref().recovered_block().sealed_block().clone(), + block_state.block_ref().recovered_block().clone(), block_state.executed_block_receipts(), ) }) @@ -553,24 +555,8 @@ impl CanonicalInMemoryState { tx_hash: TxHash, ) -> Option<(N::SignedTx, TransactionMeta)> { for block_state in self.canonical_chain() { - if let Some((index, tx)) = block_state - .block_ref() - .recovered_block() - .body() - .transactions_iter() - .enumerate() - .find(|(_, tx)| tx.trie_hash() == tx_hash) - { - let meta = TransactionMeta { - tx_hash, - index: index as u64, - block_hash: block_state.hash(), - block_number: block_state.block_ref().recovered_block().number(), - base_fee: block_state.block_ref().recovered_block().base_fee_per_gas(), - timestamp: block_state.block_ref().recovered_block().timestamp(), - excess_blob_gas: block_state.block_ref().recovered_block().excess_blob_gas(), - }; - return Some((tx.clone(), meta)) + if let Some(indexed) = block_state.find_indexed(tx_hash) { + return Some((indexed.tx().clone(), indexed.meta())); } } None @@ -582,41 +568,38 @@ impl CanonicalInMemoryState { #[derive(Debug, PartialEq, Eq, Clone)] pub struct BlockState { /// The executed block that determines the state after this block has been executed. - block: ExecutedBlockWithTrieUpdates, + block: ExecutedBlock, /// The block's parent block if it exists. - parent: Option>>, + parent: Option>, } impl BlockState { /// [`BlockState`] constructor. - pub const fn new(block: ExecutedBlockWithTrieUpdates) -> Self { + pub const fn new(block: ExecutedBlock) -> Self { Self { block, parent: None } } /// [`BlockState`] constructor with parent. - pub const fn with_parent( - block: ExecutedBlockWithTrieUpdates, - parent: Option>, - ) -> Self { + pub const fn with_parent(block: ExecutedBlock, parent: Option>) -> Self { Self { block, parent } } /// Returns the hash and block of the on disk block this state can be traced back to. pub fn anchor(&self) -> BlockNumHash { - if let Some(parent) = &self.parent { - parent.anchor() - } else { - self.block.recovered_block().parent_num_hash() + let mut current = self; + while let Some(parent) = ¤t.parent { + current = parent; } + current.block.recovered_block().parent_num_hash() } /// Returns the executed block that determines the state. - pub fn block(&self) -> ExecutedBlockWithTrieUpdates { + pub fn block(&self) -> ExecutedBlock { self.block.clone() } /// Returns a reference to the executed block that determines the state. - pub const fn block_ref(&self) -> &ExecutedBlockWithTrieUpdates { + pub const fn block_ref(&self) -> &ExecutedBlock { &self.block } @@ -725,30 +708,14 @@ impl BlockState { tx_hash: TxHash, ) -> Option<(N::SignedTx, TransactionMeta)> { self.chain().find_map(|block_state| { - block_state - .block_ref() - .recovered_block() - .body() - .transactions_iter() - .enumerate() - .find(|(_, tx)| tx.trie_hash() == tx_hash) - .map(|(index, tx)| { - let meta = TransactionMeta { - tx_hash, - index: index as u64, - block_hash: block_state.hash(), - block_number: block_state.block_ref().recovered_block().number(), - base_fee: block_state.block_ref().recovered_block().base_fee_per_gas(), - timestamp: block_state.block_ref().recovered_block().timestamp(), - excess_blob_gas: block_state - .block_ref() - .recovered_block() - .excess_blob_gas(), - }; - (tx.clone(), meta) - }) + block_state.find_indexed(tx_hash).map(|indexed| (indexed.tx().clone(), indexed.meta())) }) } + + /// Finds a transaction by hash and returns it with its index and block context. + pub fn find_indexed(&self, tx_hash: TxHash) -> Option> { + self.block_ref().recovered_block().find_indexed(tx_hash) + } } /// Represents an executed block stored in-memory. @@ -760,6 +727,8 @@ pub struct ExecutedBlock { pub execution_output: Arc>, /// Block's hashed state. pub hashed_state: Arc, + /// Trie updates that result from calculating the state root for the block. + pub trie_updates: Arc, } impl Default for ExecutedBlock { @@ -768,6 +737,7 @@ impl Default for ExecutedBlock { recovered_block: Default::default(), execution_output: Default::default(), hashed_state: Default::default(), + trie_updates: Default::default(), } } } @@ -796,53 +766,17 @@ impl ExecutedBlock { pub fn hashed_state(&self) -> &HashedPostState { &self.hashed_state } -} -/// An [`ExecutedBlock`] with its [`TrieUpdates`]. -/// -/// We store it as separate type because [`TrieUpdates`] are only available for blocks stored in -/// memory and can't be obtained for canonical persisted blocks. -#[derive( - Clone, - Debug, - PartialEq, - Eq, - Default, - derive_more::Deref, - derive_more::DerefMut, - derive_more::Into, -)] -pub struct ExecutedBlockWithTrieUpdates { - /// Inner [`ExecutedBlock`]. - #[deref] - #[deref_mut] - #[into] - pub block: ExecutedBlock, - /// Trie updates that result of applying the block. - pub trie: Arc, -} - -impl ExecutedBlockWithTrieUpdates { - /// [`ExecutedBlock`] constructor. - pub const fn new( - recovered_block: Arc>, - execution_output: Arc>, - hashed_state: Arc, - trie: Arc, - ) -> Self { - Self { block: ExecutedBlock { recovered_block, execution_output, hashed_state }, trie } - } - - /// Returns a reference to the trie updates for the block + /// Returns a reference to the trie updates resulting from the execution outcome #[inline] pub fn trie_updates(&self) -> &TrieUpdates { - &self.trie + &self.trie_updates } - /// Converts the value into [`SealedBlock`]. - pub fn into_sealed_block(self) -> SealedBlock { - let block = Arc::unwrap_or_clone(self.block.recovered_block); - block.into_sealed_block() + /// Returns a [`BlockNumber`] of the block. + #[inline] + pub fn block_number(&self) -> BlockNumber { + self.recovered_block.header().number() } } @@ -852,32 +786,28 @@ pub enum NewCanonicalChain { /// A simple append to the current canonical head Commit { /// all blocks that lead back to the canonical head - new: Vec>, + new: Vec>, }, /// A reorged chain consists of two chains that trace back to a shared ancestor block at which /// point they diverge. Reorg { /// All blocks of the _new_ chain - new: Vec>, + new: Vec>, /// All blocks of the _old_ chain - /// - /// These are not [`ExecutedBlockWithTrieUpdates`] because we don't always have the trie - /// updates for the old canonical chain. For example, in case of node being restarted right - /// before the reorg [`TrieUpdates`] can't be fetched from database. old: Vec>, }, } impl> NewCanonicalChain { /// Returns the length of the new chain. - pub fn new_block_count(&self) -> usize { + pub const fn new_block_count(&self) -> usize { match self { Self::Commit { new } | Self::Reorg { new, .. } => new.len(), } } /// Returns the length of the reorged chain. - pub fn reorged_block_count(&self) -> usize { + pub const fn reorged_block_count(&self) -> usize { match self { Self::Commit { .. } => 0, Self::Reorg { old, .. } => old.len(), @@ -941,8 +871,8 @@ mod tests { use reth_ethereum_primitives::{EthPrimitives, Receipt}; use reth_primitives_traits::{Account, Bytecode}; use reth_storage_api::{ - AccountReader, BlockHashReader, HashedPostStateProvider, StateProofProvider, StateProvider, - StateRootProvider, StorageRootProvider, + AccountReader, BlockHashReader, BytecodeReader, HashedPostStateProvider, + StateProofProvider, StateProvider, StateRootProvider, StorageRootProvider, }; use reth_trie::{ AccountProof, HashedStorage, MultiProof, MultiProofTargets, StorageMultiProof, @@ -990,7 +920,9 @@ mod tests { ) -> ProviderResult> { Ok(None) } + } + impl BytecodeReader for MockStateProvider { fn bytecode_by_hash(&self, _code_hash: &B256) -> ProviderResult> { Ok(None) } @@ -1224,7 +1156,7 @@ mod tests { block1.recovered_block().hash() ); - let chain = NewCanonicalChain::Reorg { new: vec![block2.clone()], old: vec![block1.block] }; + let chain = NewCanonicalChain::Reorg { new: vec![block2.clone()], old: vec![block1] }; state.update_chain(chain); assert_eq!( state.head_state().unwrap().block_ref().recovered_block().hash(), @@ -1290,7 +1222,7 @@ mod tests { // Check the pending block and receipts assert_eq!( state.pending_block_and_receipts().unwrap(), - (block2.recovered_block().sealed_block().clone(), vec![]) + (block2.recovered_block().clone(), vec![]) ); } @@ -1347,8 +1279,7 @@ mod tests { #[test] fn test_canonical_in_memory_state_canonical_chain_empty() { let state: CanonicalInMemoryState = CanonicalInMemoryState::empty(); - let chain: Vec<_> = state.canonical_chain().collect(); - assert!(chain.is_empty()); + assert!(state.canonical_chain().next().is_none()); } #[test] @@ -1507,7 +1438,7 @@ mod tests { // Test reorg notification let chain_reorg = NewCanonicalChain::Reorg { new: vec![block1a.clone(), block2a.clone()], - old: vec![block1.block.clone(), block2.block.clone()], + old: vec![block1.clone(), block2.clone()], }; assert_eq!( diff --git a/crates/chain-state/src/lib.rs b/crates/chain-state/src/lib.rs index f2325c087db..091201f5fa9 100644 --- a/crates/chain-state/src/lib.rs +++ b/crates/chain-state/src/lib.rs @@ -6,7 +6,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] mod in_memory; pub use in_memory::*; diff --git a/crates/chain-state/src/memory_overlay.rs b/crates/chain-state/src/memory_overlay.rs index e8f85905afb..254edb248b4 100644 --- a/crates/chain-state/src/memory_overlay.rs +++ b/crates/chain-state/src/memory_overlay.rs @@ -1,11 +1,11 @@ -use super::ExecutedBlockWithTrieUpdates; +use super::ExecutedBlock; use alloy_consensus::BlockHeader; use alloy_primitives::{keccak256, Address, BlockNumber, Bytes, StorageKey, StorageValue, B256}; use reth_errors::ProviderResult; use reth_primitives_traits::{Account, Bytecode, NodePrimitives}; use reth_storage_api::{ - AccountReader, BlockHashReader, HashedPostStateProvider, StateProofProvider, StateProvider, - StateRootProvider, StorageRootProvider, + AccountReader, BlockHashReader, BytecodeReader, HashedPostStateProvider, StateProofProvider, + StateProvider, StateRootProvider, StorageRootProvider, }; use reth_trie::{ updates::TrieUpdates, AccountProof, HashedPostState, HashedStorage, MultiProof, @@ -21,12 +21,12 @@ pub struct MemoryOverlayStateProviderRef< 'a, N: NodePrimitives = reth_ethereum_primitives::EthPrimitives, > { - /// Historical state provider for state lookups that are not found in in-memory blocks. + /// Historical state provider for state lookups that are not found in memory blocks. pub(crate) historical: Box, /// The collection of executed parent blocks. Expected order is newest to oldest. - pub(crate) in_memory: Vec>, + pub(crate) in_memory: Vec>, /// Lazy-loaded in-memory trie data. - pub(crate) trie_state: OnceLock, + pub(crate) trie_input: OnceLock, } /// A state provider that stores references to in-memory blocks along with their state as well as @@ -41,11 +41,8 @@ impl<'a, N: NodePrimitives> MemoryOverlayStateProviderRef<'a, N> { /// - `in_memory` - the collection of executed ancestor blocks in reverse. /// - `historical` - a historical state provider for the latest ancestor block stored in the /// database. - pub fn new( - historical: Box, - in_memory: Vec>, - ) -> Self { - Self { historical, in_memory, trie_state: OnceLock::new() } + pub fn new(historical: Box, in_memory: Vec>) -> Self { + Self { historical, in_memory, trie_input: OnceLock::new() } } /// Turn this state provider into a state provider @@ -54,16 +51,23 @@ impl<'a, N: NodePrimitives> MemoryOverlayStateProviderRef<'a, N> { } /// Return lazy-loaded trie state aggregated from in-memory blocks. - fn trie_state(&self) -> &MemoryOverlayTrieState { - self.trie_state.get_or_init(|| { - let mut trie_state = MemoryOverlayTrieState::default(); - for block in self.in_memory.iter().rev() { - trie_state.state.extend_ref(block.hashed_state.as_ref()); - trie_state.nodes.extend_ref(block.trie.as_ref()); - } - trie_state + fn trie_input(&self) -> &TrieInput { + self.trie_input.get_or_init(|| { + TrieInput::from_blocks( + self.in_memory + .iter() + .rev() + .map(|block| (block.hashed_state.as_ref(), block.trie_updates.as_ref())), + ) }) } + + fn merged_hashed_storage(&self, address: Address, storage: HashedStorage) -> HashedStorage { + let state = &self.trie_input().state; + let mut hashed = state.storages.get(&keccak256(address)).cloned().unwrap_or_default(); + hashed.extend(&storage); + hashed + } } impl BlockHashReader for MemoryOverlayStateProviderRef<'_, N> { @@ -84,14 +88,22 @@ impl BlockHashReader for MemoryOverlayStateProviderRef<'_, N> ) -> ProviderResult> { let range = start..end; let mut earliest_block_number = None; - let mut in_memory_hashes = Vec::new(); + let mut in_memory_hashes = Vec::with_capacity(range.size_hint().0); + + // iterate in ascending order (oldest to newest = low to high) for block in &self.in_memory { - if range.contains(&block.recovered_block().number()) { - in_memory_hashes.insert(0, block.recovered_block().hash()); - earliest_block_number = Some(block.recovered_block().number()); + let block_num = block.recovered_block().number(); + if range.contains(&block_num) { + in_memory_hashes.push(block.recovered_block().hash()); + earliest_block_number = Some(block_num); } } + // `self.in_memory` stores executed blocks in ascending order (oldest to newest). + // However, `in_memory_hashes` should be constructed in descending order (newest to oldest), + // so we reverse the vector after collecting the hashes. + in_memory_hashes.reverse(); + let mut hashes = self.historical.canonical_hashes_range(start, earliest_block_number.unwrap_or(end))?; hashes.append(&mut in_memory_hashes); @@ -117,8 +129,7 @@ impl StateRootProvider for MemoryOverlayStateProviderRef<'_, } fn state_root_from_nodes(&self, mut input: TrieInput) -> ProviderResult { - let MemoryOverlayTrieState { nodes, state } = self.trie_state().clone(); - input.prepend_cached(nodes, state); + input.prepend_self(self.trie_input().clone()); self.historical.state_root_from_nodes(input) } @@ -133,8 +144,7 @@ impl StateRootProvider for MemoryOverlayStateProviderRef<'_, &self, mut input: TrieInput, ) -> ProviderResult<(B256, TrieUpdates)> { - let MemoryOverlayTrieState { nodes, state } = self.trie_state().clone(); - input.prepend_cached(nodes, state); + input.prepend_self(self.trie_input().clone()); self.historical.state_root_from_nodes_with_updates(input) } } @@ -142,11 +152,8 @@ impl StateRootProvider for MemoryOverlayStateProviderRef<'_, impl StorageRootProvider for MemoryOverlayStateProviderRef<'_, N> { // TODO: Currently this does not reuse available in-memory trie nodes. fn storage_root(&self, address: Address, storage: HashedStorage) -> ProviderResult { - let state = &self.trie_state().state; - let mut hashed_storage = - state.storages.get(&keccak256(address)).cloned().unwrap_or_default(); - hashed_storage.extend(&storage); - self.historical.storage_root(address, hashed_storage) + let merged = self.merged_hashed_storage(address, storage); + self.historical.storage_root(address, merged) } // TODO: Currently this does not reuse available in-memory trie nodes. @@ -156,11 +163,8 @@ impl StorageRootProvider for MemoryOverlayStateProviderRef<'_ slot: B256, storage: HashedStorage, ) -> ProviderResult { - let state = &self.trie_state().state; - let mut hashed_storage = - state.storages.get(&keccak256(address)).cloned().unwrap_or_default(); - hashed_storage.extend(&storage); - self.historical.storage_proof(address, slot, hashed_storage) + let merged = self.merged_hashed_storage(address, storage); + self.historical.storage_proof(address, slot, merged) } // TODO: Currently this does not reuse available in-memory trie nodes. @@ -170,11 +174,8 @@ impl StorageRootProvider for MemoryOverlayStateProviderRef<'_ slots: &[B256], storage: HashedStorage, ) -> ProviderResult { - let state = &self.trie_state().state; - let mut hashed_storage = - state.storages.get(&keccak256(address)).cloned().unwrap_or_default(); - hashed_storage.extend(&storage); - self.historical.storage_multiproof(address, slots, hashed_storage) + let merged = self.merged_hashed_storage(address, storage); + self.historical.storage_multiproof(address, slots, merged) } } @@ -185,8 +186,7 @@ impl StateProofProvider for MemoryOverlayStateProviderRef<'_, address: Address, slots: &[B256], ) -> ProviderResult { - let MemoryOverlayTrieState { nodes, state } = self.trie_state().clone(); - input.prepend_cached(nodes, state); + input.prepend_self(self.trie_input().clone()); self.historical.proof(input, address, slots) } @@ -195,14 +195,12 @@ impl StateProofProvider for MemoryOverlayStateProviderRef<'_, mut input: TrieInput, targets: MultiProofTargets, ) -> ProviderResult { - let MemoryOverlayTrieState { nodes, state } = self.trie_state().clone(); - input.prepend_cached(nodes, state); + input.prepend_self(self.trie_input().clone()); self.historical.multiproof(input, targets) } fn witness(&self, mut input: TrieInput, target: HashedPostState) -> ProviderResult> { - let MemoryOverlayTrieState { nodes, state } = self.trie_state().clone(); - input.prepend_cached(nodes, state); + input.prepend_self(self.trie_input().clone()); self.historical.witness(input, target) } } @@ -227,7 +225,9 @@ impl StateProvider for MemoryOverlayStateProviderRef<'_, N> { self.historical.storage(address, storage_key) } +} +impl BytecodeReader for MemoryOverlayStateProviderRef<'_, N> { fn bytecode_by_hash(&self, code_hash: &B256) -> ProviderResult> { for block in &self.in_memory { if let Some(contract) = block.execution_output.bytecode(code_hash) { @@ -238,12 +238,3 @@ impl StateProvider for MemoryOverlayStateProviderRef<'_, N> { self.historical.bytecode_by_hash(code_hash) } } - -/// The collection of data necessary for trie-related operations for [`MemoryOverlayStateProvider`]. -#[derive(Clone, Default, Debug)] -pub(crate) struct MemoryOverlayTrieState { - /// The collection of aggregated in-memory trie updates. - pub(crate) nodes: TrieUpdates, - /// The collection of hashed state from in-memory blocks. - pub(crate) state: HashedPostState, -} diff --git a/crates/chain-state/src/notifications.rs b/crates/chain-state/src/notifications.rs index b03513a7520..1d2f4df10fa 100644 --- a/crates/chain-state/src/notifications.rs +++ b/crates/chain-state/src/notifications.rs @@ -81,6 +81,8 @@ impl Stream for CanonStateNotificationStream { /// The notification contains at least one [`Chain`] with the imported segment. If some blocks were /// reverted (e.g. during a reorg), the old chain is also returned. #[derive(Clone, Debug, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(bound = ""))] pub enum CanonStateNotification { /// The canonical chain was extended. Commit { @@ -120,16 +122,36 @@ impl CanonStateNotification { } } - /// Get the new tip of the chain. + /// Gets the new tip of the chain. /// /// Returns the new tip for [`Self::Reorg`] and [`Self::Commit`] variants which commit at least /// 1 new block. + /// + /// # Panics + /// + /// If chain doesn't have any blocks. pub fn tip(&self) -> &RecoveredBlock { match self { Self::Commit { new } | Self::Reorg { new, .. } => new.tip(), } } + /// Gets the new tip of the chain. + /// + /// If the chain has no blocks, it returns `None`. Otherwise, it returns the new tip for + /// [`Self::Reorg`] and [`Self::Commit`] variants. + pub fn tip_checked(&self) -> Option<&RecoveredBlock> { + match self { + Self::Commit { new } | Self::Reorg { new, .. } => { + if new.is_empty() { + None + } else { + Some(new.tip()) + } + } + } + } + /// Get receipts in the reverted and newly imported chain segments with their corresponding /// block numbers and transaction hashes. /// @@ -357,6 +379,7 @@ mod tests { block_receipts[0].0, BlockReceipts { block: block1.num_hash(), + timestamp: block1.timestamp, tx_receipts: vec![( // Transaction hash of a Transaction::default() b256!("0x20b5378c6fe992c118b557d2f8e8bbe0b7567f6fe5483a8f0f1c51e93a9d91ab"), @@ -442,6 +465,7 @@ mod tests { block_receipts[0].0, BlockReceipts { block: old_block1.num_hash(), + timestamp: old_block1.timestamp, tx_receipts: vec![( // Transaction hash of a Transaction::default() b256!("0x20b5378c6fe992c118b557d2f8e8bbe0b7567f6fe5483a8f0f1c51e93a9d91ab"), @@ -458,6 +482,7 @@ mod tests { block_receipts[1].0, BlockReceipts { block: new_block1.num_hash(), + timestamp: new_block1.timestamp, tx_receipts: vec![( // Transaction hash of a Transaction::default() b256!("0x20b5378c6fe992c118b557d2f8e8bbe0b7567f6fe5483a8f0f1c51e93a9d91ab"), diff --git a/crates/chain-state/src/test_utils.rs b/crates/chain-state/src/test_utils.rs index ae0455b9c23..5d318aca56c 100644 --- a/crates/chain-state/src/test_utils.rs +++ b/crates/chain-state/src/test_utils.rs @@ -1,10 +1,8 @@ use crate::{ - in_memory::ExecutedBlockWithTrieUpdates, CanonStateNotification, CanonStateNotifications, + in_memory::ExecutedBlock, CanonStateNotification, CanonStateNotifications, CanonStateSubscriptions, }; -use alloy_consensus::{ - Header, SignableTransaction, Transaction as _, TxEip1559, TxReceipt, EMPTY_ROOT_HASH, -}; +use alloy_consensus::{Header, SignableTransaction, TxEip1559, TxReceipt, EMPTY_ROOT_HASH}; use alloy_eips::{ eip1559::{ETHEREUM_BLOCK_GAS_LIMIT_30M, INITIAL_BASE_FEE}, eip7685::Requests, @@ -29,7 +27,6 @@ use reth_trie::{root::state_root_unhashed, updates::TrieUpdates, HashedPostState use revm_database::BundleState; use revm_state::AccountInfo; use std::{ - collections::HashMap, ops::Range, sync::{Arc, Mutex}, }; @@ -148,12 +145,10 @@ impl TestBlockBuilder { mix_hash: B256::random(), gas_limit: ETHEREUM_BLOCK_GAS_LIMIT_30M, base_fee_per_gas: Some(INITIAL_BASE_FEE), - transactions_root: calculate_transaction_root( - &transactions.clone().into_iter().map(|tx| tx.into_inner()).collect::>(), - ), + transactions_root: calculate_transaction_root(&transactions), receipts_root: calculate_receipt_root(&receipts), beneficiary: Address::random(), - state_root: state_root_unhashed(HashMap::from([( + state_root: state_root_unhashed([( self.signer, Account { balance: initial_signer_balance - signer_balance_decrease, @@ -161,7 +156,7 @@ impl TestBlockBuilder { ..Default::default() } .into_trie_account(EMPTY_ROOT_HASH), - )])), + )]), // use the number as the timestamp so it is monotonically increasing timestamp: number + EthereumHardfork::Cancun.activation_timestamp(self.chain_spec.chain).unwrap(), @@ -203,45 +198,45 @@ impl TestBlockBuilder { fork } - /// Gets an [`ExecutedBlockWithTrieUpdates`] with [`BlockNumber`], receipts and parent hash. + /// Gets an [`ExecutedBlock`] with [`BlockNumber`], receipts and parent hash. fn get_executed_block( &mut self, block_number: BlockNumber, receipts: Vec>, parent_hash: B256, - ) -> ExecutedBlockWithTrieUpdates { + ) -> ExecutedBlock { let block_with_senders = self.generate_random_block(block_number, parent_hash); let (block, senders) = block_with_senders.split_sealed(); - ExecutedBlockWithTrieUpdates::new( - Arc::new(RecoveredBlock::new_sealed(block, senders)), - Arc::new(ExecutionOutcome::new( + ExecutedBlock { + recovered_block: Arc::new(RecoveredBlock::new_sealed(block, senders)), + execution_output: Arc::new(ExecutionOutcome::new( BundleState::default(), receipts, block_number, vec![Requests::default()], )), - Arc::new(HashedPostState::default()), - Arc::new(TrieUpdates::default()), - ) + hashed_state: Arc::new(HashedPostState::default()), + trie_updates: Arc::new(TrieUpdates::default()), + } } - /// Generates an [`ExecutedBlockWithTrieUpdates`] that includes the given receipts. + /// Generates an [`ExecutedBlock`] that includes the given receipts. pub fn get_executed_block_with_receipts( &mut self, receipts: Vec>, parent_hash: B256, - ) -> ExecutedBlockWithTrieUpdates { + ) -> ExecutedBlock { let number = rand::rng().random::(); self.get_executed_block(number, receipts, parent_hash) } - /// Generates an [`ExecutedBlockWithTrieUpdates`] with the given [`BlockNumber`]. + /// Generates an [`ExecutedBlock`] with the given [`BlockNumber`]. pub fn get_executed_block_with_number( &mut self, block_number: BlockNumber, parent_hash: B256, - ) -> ExecutedBlockWithTrieUpdates { + ) -> ExecutedBlock { self.get_executed_block(block_number, vec![vec![]], parent_hash) } @@ -249,7 +244,7 @@ impl TestBlockBuilder { pub fn get_executed_blocks( &mut self, range: Range, - ) -> impl Iterator + '_ { + ) -> impl Iterator + '_ { let mut parent_hash = B256::default(); range.map(move |number| { let current_parent_hash = parent_hash; @@ -266,6 +261,16 @@ impl TestBlockBuilder { &mut self, block: RecoveredBlock, ) -> ExecutionOutcome { + let num_txs = block.body().transactions.len() as u64; + let single_cost = Self::single_tx_cost(); + + let mut final_balance = self.signer_execute_account_info.balance; + for _ in 0..num_txs { + final_balance -= single_cost; + } + + let final_nonce = self.signer_execute_account_info.nonce + num_txs; + let receipts = block .body() .transactions @@ -279,26 +284,18 @@ impl TestBlockBuilder { }) .collect::>(); - let mut bundle_state_builder = BundleState::builder(block.number..=block.number); - - for tx in &block.body().transactions { - self.signer_execute_account_info.balance -= Self::single_tx_cost(); - bundle_state_builder = bundle_state_builder.state_present_account_info( + let bundle_state = BundleState::builder(block.number..=block.number) + .state_present_account_info( self.signer, - AccountInfo { - nonce: tx.nonce(), - balance: self.signer_execute_account_info.balance, - ..Default::default() - }, - ); - } + AccountInfo { nonce: final_nonce, balance: final_balance, ..Default::default() }, + ) + .build(); - let execution_outcome = ExecutionOutcome::new( - bundle_state_builder.build(), - vec![vec![]], - block.number, - Vec::new(), - ); + self.signer_execute_account_info.balance = final_balance; + self.signer_execute_account_info.nonce = final_nonce; + + let execution_outcome = + ExecutionOutcome::new(bundle_state, vec![vec![]], block.number, Vec::new()); execution_outcome.with_receipts(vec![receipts]) } diff --git a/crates/chainspec/Cargo.toml b/crates/chainspec/Cargo.toml index 6d09d71c634..4d3c23117b3 100644 --- a/crates/chainspec/Cargo.toml +++ b/crates/chainspec/Cargo.toml @@ -35,7 +35,6 @@ derive_more.workspace = true alloy-trie = { workspace = true, features = ["arbitrary"] } alloy-eips = { workspace = true, features = ["arbitrary"] } alloy-rlp = { workspace = true, features = ["arrayvec"] } -alloy-genesis.workspace = true [features] default = ["std"] diff --git a/crates/chainspec/src/api.rs b/crates/chainspec/src/api.rs index 41bce1bcabb..ce035518bba 100644 --- a/crates/chainspec/src/api.rs +++ b/crates/chainspec/src/api.rs @@ -1,19 +1,19 @@ use crate::{ChainSpec, DepositContract}; use alloc::{boxed::Box, vec::Vec}; use alloy_chains::Chain; -use alloy_consensus::Header; -use alloy_eips::{eip1559::BaseFeeParams, eip7840::BlobParams}; +use alloy_eips::{calc_next_block_base_fee, eip1559::BaseFeeParams, eip7840::BlobParams}; use alloy_genesis::Genesis; use alloy_primitives::{B256, U256}; use core::fmt::{Debug, Display}; use reth_ethereum_forks::EthereumHardforks; use reth_network_peers::NodeRecord; +use reth_primitives_traits::{AlloyBlockHeader, BlockHeader}; /// Trait representing type configuring a chain spec. #[auto_impl::auto_impl(&, Arc)] pub trait EthChainSpec: Send + Sync + Unpin + Debug { /// The header type of the network. - type Header; + type Header: BlockHeader; /// Returns the [`Chain`] object this spec targets. fn chain(&self) -> Chain; @@ -23,9 +23,6 @@ pub trait EthChainSpec: Send + Sync + Unpin + Debug { self.chain().id() } - /// Get the [`BaseFeeParams`] for the chain at the given block. - fn base_fee_params_at_block(&self, block_number: u64) -> BaseFeeParams; - /// Get the [`BaseFeeParams`] for the chain at the given timestamp. fn base_fee_params_at_timestamp(&self, timestamp: u64) -> BaseFeeParams; @@ -65,19 +62,25 @@ pub trait EthChainSpec: Send + Sync + Unpin + Debug { /// Returns the final total difficulty if the Paris hardfork is known. fn final_paris_total_difficulty(&self) -> Option; + + /// See [`calc_next_block_base_fee`]. + fn next_block_base_fee(&self, parent: &Self::Header, target_timestamp: u64) -> Option { + Some(calc_next_block_base_fee( + parent.gas_used(), + parent.gas_limit(), + parent.base_fee_per_gas()?, + self.base_fee_params_at_timestamp(target_timestamp), + )) + } } -impl EthChainSpec for ChainSpec { - type Header = Header; +impl EthChainSpec for ChainSpec { + type Header = H; fn chain(&self) -> Chain { self.chain } - fn base_fee_params_at_block(&self, block_number: u64) -> BaseFeeParams { - self.base_fee_params_at_block(block_number) - } - fn base_fee_params_at_timestamp(&self, timestamp: u64) -> BaseFeeParams { self.base_fee_params_at_timestamp(timestamp) } @@ -85,6 +88,8 @@ impl EthChainSpec for ChainSpec { fn blob_params_at_timestamp(&self, timestamp: u64) -> Option { if let Some(blob_param) = self.blob_params.active_scheduled_params_at_timestamp(timestamp) { Some(*blob_param) + } else if self.is_osaka_active_at_timestamp(timestamp) { + Some(self.blob_params.osaka) } else if self.is_prague_active_at_timestamp(timestamp) { Some(self.blob_params.prague) } else if self.is_cancun_active_at_timestamp(timestamp) { diff --git a/crates/chainspec/src/lib.rs b/crates/chainspec/src/lib.rs index d140bf88bee..96db768a1c2 100644 --- a/crates/chainspec/src/lib.rs +++ b/crates/chainspec/src/lib.rs @@ -6,7 +6,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(feature = "std"), no_std)] extern crate alloc; @@ -145,4 +145,34 @@ mod tests { let chain: Chain = NamedChain::Holesky.into(); assert_eq!(s, chain.public_dns_network_protocol().unwrap().as_str()); } + + #[test] + fn test_centralized_base_fee_calculation() { + use crate::{ChainSpec, EthChainSpec}; + use alloy_consensus::Header; + use alloy_eips::eip1559::INITIAL_BASE_FEE; + + fn parent_header() -> Header { + Header { + gas_used: 15_000_000, + gas_limit: 30_000_000, + base_fee_per_gas: Some(INITIAL_BASE_FEE), + timestamp: 1_000, + ..Default::default() + } + } + + let spec = ChainSpec::default(); + let parent = parent_header(); + + // For testing, assume next block has timestamp 12 seconds later + let next_timestamp = parent.timestamp + 12; + + let expected = parent + .next_block_base_fee(spec.base_fee_params_at_timestamp(next_timestamp)) + .unwrap_or_default(); + + let got = spec.next_block_base_fee(&parent, next_timestamp).unwrap_or_default(); + assert_eq!(expected, got, "Base fee calculation does not match expected value"); + } } diff --git a/crates/chainspec/src/spec.rs b/crates/chainspec/src/spec.rs index df723864799..74d7a7fe0cf 100644 --- a/crates/chainspec/src/spec.rs +++ b/crates/chainspec/src/spec.rs @@ -3,20 +3,25 @@ use alloy_evm::eth::spec::EthExecutorSpec; use crate::{ constants::{MAINNET_DEPOSIT_CONTRACT, MAINNET_PRUNE_DELETE_LIMIT}, + ethereum::SEPOLIA_PARIS_TTD, + holesky, hoodi, mainnet, + mainnet::{MAINNET_PARIS_BLOCK, MAINNET_PARIS_TTD}, + sepolia, + sepolia::SEPOLIA_PARIS_BLOCK, EthChainSpec, }; -use alloc::{boxed::Box, sync::Arc, vec::Vec}; +use alloc::{boxed::Box, format, sync::Arc, vec::Vec}; use alloy_chains::{Chain, NamedChain}; use alloy_consensus::{ constants::{ - DEV_GENESIS_HASH, EMPTY_WITHDRAWALS, HOLESKY_GENESIS_HASH, HOODI_GENESIS_HASH, - MAINNET_GENESIS_HASH, SEPOLIA_GENESIS_HASH, + EMPTY_WITHDRAWALS, HOLESKY_GENESIS_HASH, HOODI_GENESIS_HASH, MAINNET_GENESIS_HASH, + SEPOLIA_GENESIS_HASH, }, Header, }; use alloy_eips::{ - eip1559::INITIAL_BASE_FEE, eip6110::MAINNET_DEPOSIT_CONTRACT_ADDRESS, - eip7685::EMPTY_REQUESTS_HASH, eip7892::BlobScheduleBlobParams, + eip1559::INITIAL_BASE_FEE, eip7685::EMPTY_REQUESTS_HASH, eip7840::BlobParams, + eip7892::BlobScheduleBlobParams, }; use alloy_genesis::Genesis; use alloy_primitives::{address, b256, Address, BlockNumber, B256, U256}; @@ -31,7 +36,7 @@ use reth_network_peers::{ holesky_nodes, hoodi_nodes, mainnet_nodes, op_nodes, op_testnet_nodes, sepolia_nodes, NodeRecord, }; -use reth_primitives_traits::{sync::LazyLock, SealedHeader}; +use reth_primitives_traits::{sync::LazyLock, BlockHeader, SealedHeader}; /// Helper method building a [`Header`] given [`Genesis`] and [`ChainHardforks`]. pub fn make_genesis_header(genesis: &Genesis, hardforks: &ChainHardforks) -> Header { @@ -100,19 +105,18 @@ pub static MAINNET: LazyLock> = LazyLock::new(|| { genesis, // paris_block_and_final_difficulty: Some(( - 15537394, + MAINNET_PARIS_BLOCK, U256::from(58_750_003_716_598_352_816_469u128), )), hardforks, // https://etherscan.io/tx/0xe75fb554e433e03763a1560646ee22dcb74e5274b34c5ad644e7c0f619a7e1d0 - deposit_contract: Some(DepositContract::new( - MAINNET_DEPOSIT_CONTRACT_ADDRESS, - 11052984, - b256!("0x649bbc62d0e31342afea4e5cd82d4049e7e1ee912fc0889aa790803be39038c5"), - )), + deposit_contract: Some(MAINNET_DEPOSIT_CONTRACT), base_fee_params: BaseFeeParamsKind::Constant(BaseFeeParams::ethereum()), prune_delete_limit: MAINNET_PRUNE_DELETE_LIMIT, - blob_params: BlobScheduleBlobParams::default(), + blob_params: BlobScheduleBlobParams::default().with_scheduled([ + (mainnet::MAINNET_BPO1_TIMESTAMP, BlobParams::bpo1()), + (mainnet::MAINNET_BPO2_TIMESTAMP, BlobParams::bpo2()), + ]), }; spec.genesis.config.dao_fork_support = true; spec.into() @@ -131,7 +135,10 @@ pub static SEPOLIA: LazyLock> = LazyLock::new(|| { ), genesis, // - paris_block_and_final_difficulty: Some((1450409, U256::from(17_000_018_015_853_232u128))), + paris_block_and_final_difficulty: Some(( + SEPOLIA_PARIS_BLOCK, + U256::from(17_000_018_015_853_232u128), + )), hardforks, // https://sepolia.etherscan.io/tx/0x025ecbf81a2f1220da6285d1701dc89fb5a956b62562ee922e1a9efd73eb4b14 deposit_contract: Some(DepositContract::new( @@ -141,7 +148,10 @@ pub static SEPOLIA: LazyLock> = LazyLock::new(|| { )), base_fee_params: BaseFeeParamsKind::Constant(BaseFeeParams::ethereum()), prune_delete_limit: 10000, - blob_params: BlobScheduleBlobParams::default(), + blob_params: BlobScheduleBlobParams::default().with_scheduled([ + (sepolia::SEPOLIA_BPO1_TIMESTAMP, BlobParams::bpo1()), + (sepolia::SEPOLIA_BPO2_TIMESTAMP, BlobParams::bpo2()), + ]), }; spec.genesis.config.dao_fork_support = true; spec.into() @@ -168,7 +178,10 @@ pub static HOLESKY: LazyLock> = LazyLock::new(|| { )), base_fee_params: BaseFeeParamsKind::Constant(BaseFeeParams::ethereum()), prune_delete_limit: 10000, - blob_params: BlobScheduleBlobParams::default(), + blob_params: BlobScheduleBlobParams::default().with_scheduled([ + (holesky::HOLESKY_BPO1_TIMESTAMP, BlobParams::bpo1()), + (holesky::HOLESKY_BPO2_TIMESTAMP, BlobParams::bpo2()), + ]), }; spec.genesis.config.dao_fork_support = true; spec.into() @@ -197,7 +210,10 @@ pub static HOODI: LazyLock> = LazyLock::new(|| { )), base_fee_params: BaseFeeParamsKind::Constant(BaseFeeParams::ethereum()), prune_delete_limit: 10000, - blob_params: BlobScheduleBlobParams::default(), + blob_params: BlobScheduleBlobParams::default().with_scheduled([ + (hoodi::HOODI_BPO1_TIMESTAMP, BlobParams::bpo1()), + (hoodi::HOODI_BPO2_TIMESTAMP, BlobParams::bpo2()), + ]), }; spec.genesis.config.dao_fork_support = true; spec.into() @@ -213,13 +229,10 @@ pub static DEV: LazyLock> = LazyLock::new(|| { let hardforks = DEV_HARDFORKS.clone(); ChainSpec { chain: Chain::dev(), - genesis_header: SealedHeader::new( - make_genesis_header(&genesis, &hardforks), - DEV_GENESIS_HASH, - ), + genesis_header: SealedHeader::seal_slow(make_genesis_header(&genesis, &hardforks)), genesis, paris_block_and_final_difficulty: Some((0, U256::from(0))), - hardforks: DEV_HARDFORKS.clone(), + hardforks, base_fee_params: BaseFeeParamsKind::Constant(BaseFeeParams::ethereum()), deposit_contract: None, // TODO: do we even have? ..Default::default() @@ -261,7 +274,7 @@ impl From for BaseFeeParamsKind { #[derive(Clone, Debug, PartialEq, Eq, From)] pub struct ForkBaseFeeParams(Vec<(Box, BaseFeeParams)>); -impl core::ops::Deref for ChainSpec { +impl core::ops::Deref for ChainSpec { type Target = ChainHardforks; fn deref(&self) -> &Self::Target { @@ -277,7 +290,7 @@ impl core::ops::Deref for ChainSpec { /// - The genesis block of the chain ([`Genesis`]) /// - What hardforks are activated, and under which conditions #[derive(Debug, Clone, PartialEq, Eq)] -pub struct ChainSpec { +pub struct ChainSpec { /// The chain ID pub chain: Chain, @@ -285,7 +298,7 @@ pub struct ChainSpec { pub genesis: Genesis, /// The header corresponding to the genesis block. - pub genesis_header: SealedHeader, + pub genesis_header: SealedHeader, /// The block at which [`EthereumHardfork::Paris`] was activated and the final difficulty at /// this block. @@ -307,7 +320,7 @@ pub struct ChainSpec { pub blob_params: BlobScheduleBlobParams, } -impl Default for ChainSpec { +impl Default for ChainSpec { fn default() -> Self { Self { chain: Default::default(), @@ -329,6 +342,13 @@ impl ChainSpec { genesis.into() } + /// Build a chainspec using [`ChainSpecBuilder`] + pub fn builder() -> ChainSpecBuilder { + ChainSpecBuilder::default() + } +} + +impl ChainSpec { /// Get information about the chain itself pub const fn chain(&self) -> Chain { self.chain @@ -366,12 +386,12 @@ impl ChainSpec { } /// Get the header for the genesis block. - pub fn genesis_header(&self) -> &Header { + pub fn genesis_header(&self) -> &H { &self.genesis_header } /// Get the sealed header for the genesis block. - pub fn sealed_genesis_header(&self) -> SealedHeader { + pub fn sealed_genesis_header(&self) -> SealedHeader { SealedHeader::new(self.genesis_header().clone(), self.genesis_hash()) } @@ -404,25 +424,6 @@ impl ChainSpec { } } - /// Get the [`BaseFeeParams`] for the chain at the given block number - pub fn base_fee_params_at_block(&self, block_number: u64) -> BaseFeeParams { - match self.base_fee_params { - BaseFeeParamsKind::Constant(bf_params) => bf_params, - BaseFeeParamsKind::Variable(ForkBaseFeeParams(ref bf_params)) => { - // Walk through the base fee params configuration in reverse order, and return the - // first one that corresponds to a hardfork that is active at the - // given timestamp. - for (fork, params) in bf_params.iter().rev() { - if self.hardforks.is_fork_active_at_block(fork.clone(), block_number) { - return *params - } - } - - bf_params.first().map(|(_, params)| *params).unwrap_or(BaseFeeParams::ethereum()) - } - } - } - /// Get the hash of the genesis block. pub fn genesis_hash(&self) -> B256 { self.genesis_header.hash() @@ -439,7 +440,7 @@ impl ChainSpec { } /// Get the fork filter for the given hardfork - pub fn hardfork_fork_filter(&self, fork: H) -> Option { + pub fn hardfork_fork_filter(&self, fork: HF) -> Option { match self.hardforks.fork(fork.clone()) { ForkCondition::Never => None, _ => Some(self.fork_filter(self.satisfy(self.hardforks.fork(fork)))), @@ -448,12 +449,31 @@ impl ChainSpec { /// Returns the hardfork display helper. pub fn display_hardforks(&self) -> DisplayHardforks { - DisplayHardforks::new(self.hardforks.forks_iter()) + // Create an iterator with hardfork, condition, and optional blob metadata + let hardforks_with_meta = self.hardforks.forks_iter().map(|(fork, condition)| { + // Generate blob metadata for timestamp-based hardforks that have blob params + let metadata = match condition { + ForkCondition::Timestamp(timestamp) => { + // Try to get blob params for this timestamp + // This automatically handles all hardforks with blob support + EthChainSpec::blob_params_at_timestamp(self, timestamp).map(|params| { + format!( + "blob: (target: {}, max: {}, fraction: {})", + params.target_blob_count, params.max_blob_count, params.update_fraction + ) + }) + } + _ => None, + }; + (fork, condition, metadata) + }); + + DisplayHardforks::with_meta(hardforks_with_meta) } /// Get the fork id for the given hardfork. #[inline] - pub fn hardfork_fork_id(&self, fork: H) -> Option { + pub fn hardfork_fork_id(&self, fork: HF) -> Option { let condition = self.hardforks.fork(fork); match condition { ForkCondition::Never => None, @@ -485,8 +505,8 @@ impl ChainSpec { /// Creates a [`ForkFilter`] for the block described by [Head]. pub fn fork_filter(&self, head: Head) -> ForkFilter { let forks = self.hardforks.forks_iter().filter_map(|(_, condition)| { - // We filter out TTD-based forks w/o a pre-known block since those do not show up in the - // fork filter. + // We filter out TTD-based forks w/o a pre-known block since those do not show up in + // the fork filter. Some(match condition { ForkCondition::Block(block) | ForkCondition::TTD { fork_block: Some(block), .. } => ForkFilterKey::Block(block), @@ -618,11 +638,6 @@ impl ChainSpec { None } - /// Build a chainspec using [`ChainSpecBuilder`] - pub fn builder() -> ChainSpecBuilder { - ChainSpecBuilder::default() - } - /// Returns the known bootnode records for the given chain. pub fn bootnodes(&self) -> Option> { use NamedChain as C; @@ -644,6 +659,32 @@ impl ChainSpec { _ => None, } } + + /// Convert header to another type. + pub fn map_header(self, f: impl FnOnce(H) -> NewH) -> ChainSpec { + let Self { + chain, + genesis, + genesis_header, + paris_block_and_final_difficulty, + hardforks, + deposit_contract, + base_fee_params, + prune_delete_limit, + blob_params, + } = self; + ChainSpec { + chain, + genesis, + genesis_header: SealedHeader::new_unhashed(f(genesis_header.into_header())), + paris_block_and_final_difficulty, + hardforks, + deposit_contract, + base_fee_params, + prune_delete_limit, + blob_params, + } + } } impl From for ChainSpec { @@ -673,26 +714,50 @@ impl From for ChainSpec { // We expect no new networks to be configured with the merge, so we ignore the TTD field // and merge netsplit block from external genesis files. All existing networks that have // merged should have a static ChainSpec already (namely mainnet and sepolia). - let paris_block_and_final_difficulty = - if let Some(ttd) = genesis.config.terminal_total_difficulty { - hardforks.push(( - EthereumHardfork::Paris.boxed(), - ForkCondition::TTD { - // NOTE: this will not work properly if the merge is not activated at - // genesis, and there is no merge netsplit block - activation_block_number: genesis - .config - .merge_netsplit_block - .unwrap_or_default(), - total_difficulty: ttd, - fork_block: genesis.config.merge_netsplit_block, - }, - )); + let paris_block_and_final_difficulty = if let Some(ttd) = + genesis.config.terminal_total_difficulty + { + hardforks.push(( + EthereumHardfork::Paris.boxed(), + ForkCondition::TTD { + // NOTE: this will not work properly if the merge is not activated at + // genesis, and there is no merge netsplit block + activation_block_number: genesis + .config + .merge_netsplit_block + .or_else(|| { + // due to this limitation we can't determine the merge block, + // this is the case for perfnet testing for example + // at the time of this fix, only two networks transitioned: MAINNET + + // SEPOLIA and this parsing from genesis is used for shadowforking, so + // we can reasonably assume that if the TTD and the chainid matches + // those networks we use the activation + // blocks of those networks + match genesis.config.chain_id { + 1 => { + if ttd == MAINNET_PARIS_TTD { + return Some(MAINNET_PARIS_BLOCK) + } + } + 11155111 => { + if ttd == SEPOLIA_PARIS_TTD { + return Some(SEPOLIA_PARIS_BLOCK) + } + } + _ => {} + }; + None + }) + .unwrap_or_default(), + total_difficulty: ttd, + fork_block: genesis.config.merge_netsplit_block, + }, + )); - genesis.config.merge_netsplit_block.map(|block| (block, ttd)) - } else { - None - }; + genesis.config.merge_netsplit_block.map(|block| (block, ttd)) + } else { + None + }; // Time-based hardforks let time_hardfork_opts = [ @@ -700,6 +765,11 @@ impl From for ChainSpec { (EthereumHardfork::Cancun.boxed(), genesis.config.cancun_time), (EthereumHardfork::Prague.boxed(), genesis.config.prague_time), (EthereumHardfork::Osaka.boxed(), genesis.config.osaka_time), + (EthereumHardfork::Bpo1.boxed(), genesis.config.bpo1_time), + (EthereumHardfork::Bpo2.boxed(), genesis.config.bpo2_time), + (EthereumHardfork::Bpo3.boxed(), genesis.config.bpo3_time), + (EthereumHardfork::Bpo4.boxed(), genesis.config.bpo4_time), + (EthereumHardfork::Bpo5.boxed(), genesis.config.bpo5_time), ]; let mut time_hardforks = time_hardfork_opts @@ -751,8 +821,8 @@ impl From for ChainSpec { } } -impl Hardforks for ChainSpec { - fn fork(&self, fork: H) -> ForkCondition { +impl Hardforks for ChainSpec { + fn fork(&self, fork: HF) -> ForkCondition { self.hardforks.fork(fork) } @@ -773,7 +843,7 @@ impl Hardforks for ChainSpec { } } -impl EthereumHardforks for ChainSpec { +impl EthereumHardforks for ChainSpec { fn ethereum_fork_activation(&self, fork: EthereumHardfork) -> ForkCondition { self.fork(fork) } @@ -815,6 +885,12 @@ impl ChainSpecBuilder { self } + /// Resets any existing hardforks from the builder. + pub fn reset(mut self) -> Self { + self.hardforks = ChainHardforks::default(); + self + } + /// Set the genesis block. pub fn genesis(mut self, genesis: Genesis) -> Self { self.genesis = Some(genesis); @@ -835,7 +911,7 @@ impl ChainSpecBuilder { /// Remove the given fork from the spec. pub fn without_fork(mut self, fork: H) -> Self { - self.hardforks.remove(fork); + self.hardforks.remove(&fork); self } @@ -953,6 +1029,12 @@ impl ChainSpecBuilder { self } + /// Enable Prague at the given timestamp. + pub fn with_prague_at(mut self, timestamp: u64) -> Self { + self.hardforks.insert(EthereumHardfork::Prague, ForkCondition::Timestamp(timestamp)); + self + } + /// Enable Osaka at genesis. pub fn osaka_activated(mut self) -> Self { self = self.prague_activated(); @@ -960,6 +1042,12 @@ impl ChainSpecBuilder { self } + /// Enable Osaka at the given timestamp. + pub fn with_osaka_at(mut self, timestamp: u64) -> Self { + self.hardforks.insert(EthereumHardfork::Osaka, ForkCondition::Timestamp(timestamp)); + self + } + /// Build the resulting [`ChainSpec`]. /// /// # Panics @@ -1002,7 +1090,7 @@ impl From<&Arc> for ChainSpecBuilder { } } -impl EthExecutorSpec for ChainSpec { +impl EthExecutorSpec for ChainSpec { fn deposit_contract_address(&self) -> Option
{ self.deposit_contract.map(|deposit_contract| deposit_contract.address) } @@ -1051,11 +1139,7 @@ mod tests { use alloy_trie::{TrieAccount, EMPTY_ROOT_HASH}; use core::ops::Deref; use reth_ethereum_forks::{ForkCondition, ForkHash, ForkId, Head}; - use std::{ - collections::{BTreeMap, HashMap}, - str::FromStr, - string::String, - }; + use std::{collections::HashMap, str::FromStr}; fn test_hardfork_fork_ids(spec: &ChainSpec, cases: &[(EthereumHardfork, ForkId)]) { for (hardfork, expected_id) in cases { @@ -1065,9 +1149,9 @@ mod tests { "Expected fork ID {expected_id:?}, computed fork ID {computed_id:?} for hardfork {hardfork}" ); if matches!(hardfork, EthereumHardfork::Shanghai) { - if let Some(shangai_id) = spec.shanghai_fork_id() { + if let Some(shanghai_id) = spec.shanghai_fork_id() { assert_eq!( - expected_id, &shangai_id, + expected_id, &shanghai_id, "Expected fork ID {expected_id:?}, computed fork ID {computed_id:?} for Shanghai hardfork" ); } else { @@ -1101,8 +1185,11 @@ Merge hard forks: - Paris @58750000000000000000000 (network is known to be merged) Post-merge hard forks (timestamp based): - Shanghai @1681338455 -- Cancun @1710338135 -- Prague @1746612311" +- Cancun @1710338135 blob: (target: 3, max: 6, fraction: 3338477) +- Prague @1746612311 blob: (target: 6, max: 9, fraction: 5007716) +- Osaka @1764798551 blob: (target: 6, max: 9, fraction: 5007716) +- Bpo1 @1765290071 blob: (target: 10, max: 15, fraction: 8346193) +- Bpo2 @1767747671 blob: (target: 14, max: 21, fraction: 11684671)" ); } @@ -1242,7 +1329,7 @@ Post-merge hard forks (timestamp based): Head { number: 101, timestamp: 11313123, ..Default::default() }; assert_eq!( fork_cond_ttd_blocknum_head, fork_cond_ttd_blocknum_expected, - "expected satisfy() to return {fork_cond_ttd_blocknum_expected:#?}, but got {fork_cond_ttd_blocknum_expected:#?} ", + "expected satisfy() to return {fork_cond_ttd_blocknum_expected:#?}, but got {fork_cond_ttd_blocknum_head:#?} ", ); // spec w/ only ForkCondition::Block - test the match arm for ForkCondition::Block to ensure @@ -1271,7 +1358,7 @@ Post-merge hard forks (timestamp based): Head { total_difficulty: U256::from(10_790_000), ..Default::default() }; assert_eq!( fork_cond_ttd_no_new_spec, fork_cond_ttd_no_new_spec_expected, - "expected satisfy() to return {fork_cond_ttd_blocknum_expected:#?}, but got {fork_cond_ttd_blocknum_expected:#?} ", + "expected satisfy() to return {fork_cond_ttd_no_new_spec_expected:#?}, but got {fork_cond_ttd_no_new_spec:#?} ", ); } @@ -1346,7 +1433,10 @@ Post-merge hard forks (timestamp based): ), ( EthereumHardfork::Prague, - ForkId { hash: ForkHash([0xc3, 0x76, 0xcf, 0x8b]), next: 0 }, + ForkId { + hash: ForkHash([0xc3, 0x76, 0xcf, 0x8b]), + next: mainnet::MAINNET_OSAKA_TIMESTAMP, + }, ), ], ); @@ -1411,7 +1501,10 @@ Post-merge hard forks (timestamp based): ), ( EthereumHardfork::Prague, - ForkId { hash: ForkHash([0xed, 0x88, 0xb5, 0xfd]), next: 0 }, + ForkId { + hash: ForkHash([0xed, 0x88, 0xb5, 0xfd]), + next: sepolia::SEPOLIA_OSAKA_TIMESTAMP, + }, ), ], ); @@ -1486,13 +1579,23 @@ Post-merge hard forks (timestamp based): ), // First Prague block ( - Head { number: 20000002, timestamp: 1746612311, ..Default::default() }, - ForkId { hash: ForkHash([0xc3, 0x76, 0xcf, 0x8b]), next: 0 }, + Head { number: 20000004, timestamp: 1746612311, ..Default::default() }, + ForkId { + hash: ForkHash([0xc3, 0x76, 0xcf, 0x8b]), + next: mainnet::MAINNET_OSAKA_TIMESTAMP, + }, ), - // Future Prague block + // Osaka block ( - Head { number: 20000002, timestamp: 2000000000, ..Default::default() }, - ForkId { hash: ForkHash([0xc3, 0x76, 0xcf, 0x8b]), next: 0 }, + Head { + number: 20000004, + timestamp: mainnet::MAINNET_OSAKA_TIMESTAMP, + ..Default::default() + }, + ForkId { + hash: ForkHash(hex!("0x5167e2a6")), + next: mainnet::MAINNET_BPO1_TIMESTAMP, + }, ), ], ); @@ -1510,7 +1613,22 @@ Post-merge hard forks (timestamp based): // First Prague block ( Head { number: 0, timestamp: 1742999833, ..Default::default() }, - ForkId { hash: ForkHash([0x09, 0x29, 0xe2, 0x4e]), next: 0 }, + ForkId { + hash: ForkHash([0x09, 0x29, 0xe2, 0x4e]), + next: hoodi::HOODI_OSAKA_TIMESTAMP, + }, + ), + // First Osaka block + ( + Head { + number: 0, + timestamp: hoodi::HOODI_OSAKA_TIMESTAMP, + ..Default::default() + }, + ForkId { + hash: ForkHash(hex!("0xe7e0e7ff")), + next: hoodi::HOODI_BPO1_TIMESTAMP, + }, ), ], ) @@ -1558,7 +1676,22 @@ Post-merge hard forks (timestamp based): // First Prague block ( Head { number: 123, timestamp: 1740434112, ..Default::default() }, - ForkId { hash: ForkHash([0xdf, 0xbd, 0x9b, 0xed]), next: 0 }, + ForkId { + hash: ForkHash([0xdf, 0xbd, 0x9b, 0xed]), + next: holesky::HOLESKY_OSAKA_TIMESTAMP, + }, + ), + // First Osaka block + ( + Head { + number: 123, + timestamp: holesky::HOLESKY_OSAKA_TIMESTAMP, + ..Default::default() + }, + ForkId { + hash: ForkHash(hex!("0x783def52")), + next: holesky::HOLESKY_BPO1_TIMESTAMP, + }, ), ], ) @@ -1608,7 +1741,22 @@ Post-merge hard forks (timestamp based): // First Prague block ( Head { number: 1735377, timestamp: 1741159776, ..Default::default() }, - ForkId { hash: ForkHash([0xed, 0x88, 0xb5, 0xfd]), next: 0 }, + ForkId { + hash: ForkHash([0xed, 0x88, 0xb5, 0xfd]), + next: sepolia::SEPOLIA_OSAKA_TIMESTAMP, + }, + ), + // First Osaka block + ( + Head { + number: 1735377, + timestamp: sepolia::SEPOLIA_OSAKA_TIMESTAMP, + ..Default::default() + }, + ForkId { + hash: ForkHash(hex!("0xe2ae4999")), + next: sepolia::SEPOLIA_BPO1_TIMESTAMP, + }, ), ], ); @@ -1620,7 +1768,7 @@ Post-merge hard forks (timestamp based): &DEV, &[( Head { number: 0, ..Default::default() }, - ForkId { hash: ForkHash([0x45, 0xb8, 0x36, 0x12]), next: 0 }, + ForkId { hash: ForkHash([0x0b, 0x1a, 0x4e, 0xf7]), next: 0 }, )], ) } @@ -1756,11 +1904,22 @@ Post-merge hard forks (timestamp based): ), // First Prague block ( Head { number: 20000004, timestamp: 1746612311, ..Default::default() }, - ForkId { hash: ForkHash([0xc3, 0x76, 0xcf, 0x8b]), next: 0 }, - ), // Future Prague block + ForkId { + hash: ForkHash([0xc3, 0x76, 0xcf, 0x8b]), + next: mainnet::MAINNET_OSAKA_TIMESTAMP, + }, + ), + // Osaka block ( - Head { number: 20000004, timestamp: 2000000000, ..Default::default() }, - ForkId { hash: ForkHash([0xc3, 0x76, 0xcf, 0x8b]), next: 0 }, + Head { + number: 20000004, + timestamp: mainnet::MAINNET_OSAKA_TIMESTAMP, + ..Default::default() + }, + ForkId { + hash: ForkHash(hex!("0x5167e2a6")), + next: mainnet::MAINNET_BPO1_TIMESTAMP, + }, ), ], ); @@ -2383,7 +2542,7 @@ Post-merge hard forks (timestamp based): #[test] fn check_fork_id_chainspec_with_fork_condition_never() { - let spec = ChainSpec { + let spec: ChainSpec = ChainSpec { chain: Chain::mainnet(), genesis: Genesis::default(), hardforks: ChainHardforks::new(vec![( @@ -2400,7 +2559,7 @@ Post-merge hard forks (timestamp based): #[test] fn check_fork_filter_chainspec_with_fork_condition_never() { - let spec = ChainSpec { + let spec: ChainSpec = ChainSpec { chain: Chain::mainnet(), genesis: Genesis::default(), hardforks: ChainHardforks::new(vec![( @@ -2417,10 +2576,26 @@ Post-merge hard forks (timestamp based): #[test] fn latest_eth_mainnet_fork_id() { - assert_eq!( - ForkId { hash: ForkHash([0xc3, 0x76, 0xcf, 0x8b]), next: 0 }, - MAINNET.latest_fork_id() - ) + // BPO2 + assert_eq!(ForkId { hash: ForkHash(hex!("0x07c9462e")), next: 0 }, MAINNET.latest_fork_id()) + } + + #[test] + fn latest_hoodi_mainnet_fork_id() { + // BPO2 + assert_eq!(ForkId { hash: ForkHash(hex!("0x23aa1351")), next: 0 }, HOODI.latest_fork_id()) + } + + #[test] + fn latest_holesky_mainnet_fork_id() { + // BPO2 + assert_eq!(ForkId { hash: ForkHash(hex!("0x9bc6cb31")), next: 0 }, HOLESKY.latest_fork_id()) + } + + #[test] + fn latest_sepolia_mainnet_fork_id() { + // BPO2 + assert_eq!(ForkId { hash: ForkHash(hex!("0x268956b6")), next: 0 }, SEPOLIA.latest_fork_id()) } #[test] @@ -2521,34 +2696,107 @@ Post-merge hard forks (timestamp based): #[test] fn blob_params_from_genesis() { let s = r#"{ - "cancun":{ - "baseFeeUpdateFraction":3338477, - "max":6, - "target":3 - }, - "prague":{ - "baseFeeUpdateFraction":3338477, - "max":6, - "target":3 - } - }"#; - let schedule: BTreeMap = serde_json::from_str(s).unwrap(); - let hardfork_params = BlobScheduleBlobParams::from_schedule(&schedule); + "blobSchedule": { + "cancun":{ + "baseFeeUpdateFraction":3338477, + "max":6, + "target":3 + }, + "prague":{ + "baseFeeUpdateFraction":3338477, + "max":6, + "target":3 + } + } + }"#; + let config: ChainConfig = serde_json::from_str(s).unwrap(); + let hardfork_params = config.blob_schedule_blob_params(); let expected = BlobScheduleBlobParams { cancun: BlobParams { target_blob_count: 3, max_blob_count: 6, update_fraction: 3338477, min_blob_fee: BLOB_TX_MIN_BLOB_GASPRICE, + max_blobs_per_tx: 6, + blob_base_cost: 0, }, prague: BlobParams { target_blob_count: 3, max_blob_count: 6, update_fraction: 3338477, min_blob_fee: BLOB_TX_MIN_BLOB_GASPRICE, + max_blobs_per_tx: 6, + blob_base_cost: 0, }, - scheduled: Default::default(), + ..Default::default() }; assert_eq!(hardfork_params, expected); } + + #[test] + fn parse_perf_net_genesis() { + let s = r#"{ + "config": { + "chainId": 1, + "homesteadBlock": 1150000, + "daoForkBlock": 1920000, + "daoForkSupport": true, + "eip150Block": 2463000, + "eip150Hash": "0x2086799aeebeae135c246c65021c82b4e15a2c451340993aacfd2751886514f0", + "eip155Block": 2675000, + "eip158Block": 2675000, + "byzantiumBlock": 4370000, + "constantinopleBlock": 7280000, + "petersburgBlock": 7280000, + "istanbulBlock": 9069000, + "muirGlacierBlock": 9200000, + "berlinBlock": 12244000, + "londonBlock": 12965000, + "arrowGlacierBlock": 13773000, + "grayGlacierBlock": 15050000, + "terminalTotalDifficulty": 58750000000000000000000, + "terminalTotalDifficultyPassed": true, + "shanghaiTime": 1681338455, + "cancunTime": 1710338135, + "pragueTime": 1746612311, + "ethash": {}, + "depositContractAddress": "0x00000000219ab540356cBB839Cbe05303d7705Fa", + "blobSchedule": { + "cancun": { + "target": 3, + "max": 6, + "baseFeeUpdateFraction": 3338477 + }, + "prague": { + "target": 6, + "max": 9, + "baseFeeUpdateFraction": 5007716 + } + } + }, + "nonce": "0x42", + "timestamp": "0x0", + "extraData": "0x11bbe8db4e347b4e8c937c1c8370e4b5ed33adb3db69cbdb7a38e1e50b1b82fa", + "gasLimit": "0x1388", + "difficulty": "0x400000000", + "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "coinbase": "0x0000000000000000000000000000000000000000", + "number": "0x0", + "gasUsed": "0x0", + "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "baseFeePerGas": null +}"#; + + let genesis = serde_json::from_str::(s).unwrap(); + let chainspec = ChainSpec::from_genesis(genesis); + let activation = chainspec.hardforks.fork(EthereumHardfork::Paris); + assert_eq!( + activation, + ForkCondition::TTD { + activation_block_number: MAINNET_PARIS_BLOCK, + total_difficulty: MAINNET_PARIS_TTD, + fork_block: None, + } + ) + } } diff --git a/crates/cli/cli/src/chainspec.rs b/crates/cli/cli/src/chainspec.rs index b70430a9102..4a76bb8a5ec 100644 --- a/crates/cli/cli/src/chainspec.rs +++ b/crates/cli/cli/src/chainspec.rs @@ -39,6 +39,11 @@ pub trait ChainSpecParser: Clone + Send + Sync + 'static { /// List of supported chains. const SUPPORTED_CHAINS: &'static [&'static str]; + /// The default value for the chain spec parser. + fn default_value() -> Option<&'static str> { + Self::SUPPORTED_CHAINS.first().copied() + } + /// Parses the given string into a chain spec. /// /// # Arguments diff --git a/crates/cli/cli/src/lib.rs b/crates/cli/cli/src/lib.rs index 52e97289112..b95fea9fa42 100644 --- a/crates/cli/cli/src/lib.rs +++ b/crates/cli/cli/src/lib.rs @@ -6,7 +6,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] use clap::{Error, Parser}; use reth_cli_runner::CliRunner; diff --git a/crates/cli/commands/Cargo.toml b/crates/cli/commands/Cargo.toml index b8e4d397697..da1a5318f25 100644 --- a/crates/cli/commands/Cargo.toml +++ b/crates/cli/commands/Cargo.toml @@ -21,9 +21,10 @@ reth-consensus.workspace = true reth-db = { workspace = true, features = ["mdbx"] } reth-db-api.workspace = true reth-db-common.workspace = true -reth-downloaders.workspace = true +reth-downloaders = { workspace = true, features = ["file-client"] } reth-ecies.workspace = true reth-eth-wire.workspace = true +reth-era.workspace = true reth-era-downloader.workspace = true reth-era-utils.workspace = true reth-etl.workspace = true @@ -43,13 +44,14 @@ reth-ethereum-primitives = { workspace = true, optional = true } reth-provider.workspace = true reth-prune.workspace = true reth-prune-types = { workspace = true, optional = true } +reth-revm.workspace = true reth-stages.workspace = true reth-stages-types = { workspace = true, optional = true } reth-static-file-types = { workspace = true, features = ["clap"] } reth-static-file.workspace = true reth-trie = { workspace = true, features = ["metrics"] } reth-trie-db = { workspace = true, features = ["metrics"] } -reth-trie-common = { workspace = true, optional = true } +reth-trie-common.workspace = true reth-primitives-traits.workspace = true reth-discv4.workspace = true reth-discv5.workspace = true @@ -66,11 +68,12 @@ futures.workspace = true tokio.workspace = true # misc -ahash.workspace = true +humantime.workspace = true human_bytes.workspace = true eyre.workspace = true clap = { workspace = true, features = ["derive", "env"] } lz4.workspace = true +zstd.workspace = true serde.workspace = true serde_json.workspace = true tar.workspace = true @@ -96,6 +99,8 @@ proptest-arbitrary-interop = { workspace = true, optional = true } [dev-dependencies] reth-ethereum-cli.workspace = true +reth-provider = { workspace = true, features = ["test-utils"] } +tempfile.workspace = true [features] default = [] @@ -117,7 +122,7 @@ arbitrary = [ "reth-codecs/arbitrary", "reth-prune-types?/arbitrary", "reth-stages-types?/arbitrary", - "reth-trie-common?/arbitrary", + "reth-trie-common/arbitrary", "alloy-consensus/arbitrary", "reth-primitives-traits/arbitrary", "reth-ethereum-primitives/arbitrary", diff --git a/crates/cli/commands/src/common.rs b/crates/cli/commands/src/common.rs index c19ed0834e6..5b8cfce7716 100644 --- a/crates/cli/commands/src/common.rs +++ b/crates/cli/commands/src/common.rs @@ -5,11 +5,13 @@ use clap::Parser; use reth_chainspec::EthChainSpec; use reth_cli::chainspec::ChainSpecParser; use reth_config::{config::EtlConfig, Config}; -use reth_consensus::{noop::NoopConsensus, ConsensusError, FullConsensus}; +use reth_consensus::noop::NoopConsensus; use reth_db::{init_db, open_db_read_only, DatabaseEnv}; use reth_db_common::init::init_genesis; use reth_downloaders::{bodies::noop::NoopBodiesDownloader, headers::noop::NoopHeaderDownloader}; +use reth_eth_wire::NetPrimitivesFor; use reth_evm::{noop::NoopEvmConfig, ConfigureEvm}; +use reth_network::NetworkEventListenerProvider; use reth_node_api::FullNodeTypesAdapter; use reth_node_builder::{ Node, NodeComponents, NodeComponentsBuilder, NodeTypes, NodeTypesWithDBAdapter, @@ -46,7 +48,7 @@ pub struct EnvironmentArgs { long, value_name = "CHAIN_OR_PATH", long_help = C::help_message(), - default_value = C::SUPPORTED_CHAINS[0], + default_value = C::default_value(), value_parser = C::parser(), global = true )] @@ -85,6 +87,9 @@ impl EnvironmentArgs { if config.stages.etl.dir.is_none() { config.stages.etl.dir = Some(EtlConfig::from_datadir(data_dir.data_dir())); } + if config.stages.era.folder.is_none() { + config.stages.era = config.stages.era.with_datadir(data_dir.data_dir()); + } info!(target: "reth::cli", ?db_path, ?sf_path, "Opening storage"); let (db, sfp) = match access { @@ -121,9 +126,8 @@ impl EnvironmentArgs { where C: ChainSpecParser, { - let has_receipt_pruning = config.prune.as_ref().is_some_and(|a| a.has_receipts_pruning()); - let prune_modes = - config.prune.as_ref().map(|prune| prune.segments.clone()).unwrap_or_default(); + let has_receipt_pruning = config.prune.has_receipts_pruning(); + let prune_modes = config.prune.segments.clone(); let factory = ProviderFactory::>>::new( db, self.chain.clone(), @@ -164,6 +168,7 @@ impl EnvironmentArgs { NoopEvmConfig::::default(), config.stages.clone(), prune_modes.clone(), + None, )) .build(factory.clone(), StaticFileProducer::new(factory.clone(), prune_modes)); @@ -210,10 +215,22 @@ type FullTypesAdapter = FullNodeTypesAdapter< BlockchainProvider>>, >; +/// Trait for block headers that can be modified through CLI operations. +pub trait CliHeader { + fn set_number(&mut self, number: u64); +} + +impl CliHeader for alloy_consensus::Header { + fn set_number(&mut self, number: u64) { + self.number = number; + } +} + /// Helper trait with a common set of requirements for the /// [`NodeTypes`] in CLI. -pub trait CliNodeTypes: NodeTypes + NodeTypesForProvider { +pub trait CliNodeTypes: Node> + NodeTypesForProvider { type Evm: ConfigureEvm; + type NetworkPrimitives: NetPrimitivesFor; } impl CliNodeTypes for N @@ -221,34 +238,47 @@ where N: Node> + NodeTypesForProvider, { type Evm = <>>::Components as NodeComponents>>::Evm; + type NetworkPrimitives = <<>>::Components as NodeComponents>>::Network as NetworkEventListenerProvider>::Primitives; } -/// Helper trait aggregating components required for the CLI. -pub trait CliNodeComponents { - /// Evm to use. - type Evm: ConfigureEvm + 'static; - /// Consensus implementation. - type Consensus: FullConsensus + Clone + 'static; +type EvmFor = <<>>::ComponentsBuilder as NodeComponentsBuilder< + FullTypesAdapter, +>>::Components as NodeComponents>>::Evm; + +type ConsensusFor = + <<>>::ComponentsBuilder as NodeComponentsBuilder< + FullTypesAdapter, + >>::Components as NodeComponents>>::Consensus; +/// Helper trait aggregating components required for the CLI. +pub trait CliNodeComponents: Send + Sync + 'static { /// Returns the configured EVM. - fn evm_config(&self) -> &Self::Evm; + fn evm_config(&self) -> &EvmFor; /// Returns the consensus implementation. - fn consensus(&self) -> &Self::Consensus; + fn consensus(&self) -> &ConsensusFor; } -impl CliNodeComponents for (E, C) -where - E: ConfigureEvm + 'static, - C: FullConsensus + Clone + 'static, -{ - type Evm = E; - type Consensus = C; - - fn evm_config(&self) -> &Self::Evm { +impl CliNodeComponents for (EvmFor, ConsensusFor) { + fn evm_config(&self) -> &EvmFor { &self.0 } - fn consensus(&self) -> &Self::Consensus { + fn consensus(&self) -> &ConsensusFor { &self.1 } } + +/// Helper trait alias for an [`FnOnce`] producing [`CliNodeComponents`]. +pub trait CliComponentsBuilder: + FnOnce(Arc) -> Self::Components + Send + Sync + 'static +{ + type Components: CliNodeComponents; +} + +impl CliComponentsBuilder for F +where + F: FnOnce(Arc) -> Comp + Send + Sync + 'static, + Comp: CliNodeComponents, +{ + type Components = Comp; +} diff --git a/crates/cli/commands/src/db/checksum.rs b/crates/cli/commands/src/db/checksum.rs index 40f0d22f6df..e5ed9d909cd 100644 --- a/crates/cli/commands/src/db/checksum.rs +++ b/crates/cli/commands/src/db/checksum.rs @@ -2,7 +2,7 @@ use crate::{ common::CliNodeTypes, db::get::{maybe_json_value_parser, table_key}, }; -use ahash::RandomState; +use alloy_primitives::map::foldhash::fast::FixedState; use clap::Parser; use reth_chainspec::EthereumHardforks; use reth_db::DatabaseEnv; @@ -102,7 +102,7 @@ impl TableViewer<(u64, Duration)> for ChecksumViewer<'_, N }; let start_time = Instant::now(); - let mut hasher = RandomState::with_seeds(1, 2, 3, 4).build_hasher(); + let mut hasher = FixedState::with_seed(u64::from_be_bytes(*b"RETHRETH")).build_hasher(); let mut total = 0; let limit = self.limit.unwrap_or(usize::MAX); @@ -111,7 +111,7 @@ impl TableViewer<(u64, Duration)> for ChecksumViewer<'_, N for (index, entry) in walker.enumerate() { let (k, v): (RawKey, RawValue) = entry?; - if index % 100_000 == 0 { + if index.is_multiple_of(100_000) { info!("Hashed {index} entries."); } diff --git a/crates/cli/commands/src/db/get.rs b/crates/cli/commands/src/db/get.rs index 9da53433135..9d06a35dcaa 100644 --- a/crates/cli/commands/src/db/get.rs +++ b/crates/cli/commands/src/db/get.rs @@ -1,15 +1,17 @@ -use alloy_consensus::Header; use alloy_primitives::{hex, BlockHash}; use clap::Parser; -use reth_db::static_file::{ - ColumnSelectorOne, ColumnSelectorTwo, HeaderWithHashMask, ReceiptMask, TransactionMask, +use reth_db::{ + static_file::{ + ColumnSelectorOne, ColumnSelectorTwo, HeaderWithHashMask, ReceiptMask, TransactionMask, + }, + RawDupSort, }; use reth_db_api::{ table::{Decompress, DupSort, Table}, tables, RawKey, RawTable, Receipts, TableViewer, Transactions, }; use reth_db_common::DbTool; -use reth_node_api::{ReceiptTy, TxTy}; +use reth_node_api::{HeaderTy, ReceiptTy, TxTy}; use reth_node_builder::NodeTypesWithDB; use reth_provider::{providers::ProviderNodeTypes, StaticFileProviderFactory}; use reth_static_file_types::StaticFileSegment; @@ -63,16 +65,16 @@ impl Command { } Subcommand::StaticFile { segment, key, raw } => { let (key, mask): (u64, _) = match segment { - StaticFileSegment::Headers => { - (table_key::(&key)?, >::MASK) - } + StaticFileSegment::Headers => ( + table_key::(&key)?, + >>::MASK, + ), StaticFileSegment::Transactions => { (table_key::(&key)?, >>::MASK) } StaticFileSegment::Receipts => { (table_key::(&key)?, >>::MASK) } - StaticFileSegment::BlockMeta => todo!(), }; let content = tool.provider_factory.static_file_provider().find_static_file( @@ -94,7 +96,7 @@ impl Command { } else { match segment { StaticFileSegment::Headers => { - let header = Header::decompress(content[0].as_slice())?; + let header = HeaderTy::::decompress(content[0].as_slice())?; let block_hash = BlockHash::decompress(content[1].as_slice())?; println!( "Header\n{}\n\nBlockHash\n{}", @@ -114,9 +116,6 @@ impl Command { )?; println!("{}", serde_json::to_string_pretty(&receipt)?); } - StaticFileSegment::BlockMeta => { - todo!() - } } } } @@ -181,9 +180,21 @@ impl TableViewer<()> for GetValueViewer<'_, N> { // process dupsort table let subkey = table_subkey::(self.subkey.as_deref())?; - match self.tool.get_dup::(key, subkey)? { + let content = if self.raw { + self.tool + .get_dup::>(RawKey::from(key), RawKey::from(subkey))? + .map(|content| hex::encode_prefixed(content.raw_value())) + } else { + self.tool + .get_dup::(key, subkey)? + .as_ref() + .map(serde_json::to_string_pretty) + .transpose()? + }; + + match content { Some(content) => { - println!("{}", serde_json::to_string_pretty(&content)?); + println!("{content}"); } None => { error!(target: "reth::cli", "No content for the given table subkey."); diff --git a/crates/cli/commands/src/db/list.rs b/crates/cli/commands/src/db/list.rs index 9288a56a86c..2540e77c111 100644 --- a/crates/cli/commands/src/db/list.rs +++ b/crates/cli/commands/src/db/list.rs @@ -97,7 +97,7 @@ impl TableViewer<()> for ListTableViewer<'_, N> { fn view(&self) -> Result<(), Self::Error> { self.tool.provider_factory.db_ref().view(|tx| { let table_db = tx.inner.open_db(Some(self.args.table.name())).wrap_err("Could not open db.")?; - let stats = tx.inner.db_stat(&table_db).wrap_err(format!("Could not find table: {}", stringify!($table)))?; + let stats = tx.inner.db_stat(&table_db).wrap_err(format!("Could not find table: {}", self.args.table.name()))?; let total_entries = stats.entries(); let final_entry_idx = total_entries.saturating_sub(1); if self.args.skip > final_entry_idx { diff --git a/crates/cli/commands/src/db/mod.rs b/crates/cli/commands/src/db/mod.rs index 67b060f7e9a..1ea66b2f550 100644 --- a/crates/cli/commands/src/db/mod.rs +++ b/crates/cli/commands/src/db/mod.rs @@ -13,6 +13,7 @@ mod clear; mod diff; mod get; mod list; +mod repair_trie; mod stats; /// DB List TUI mod tui; @@ -48,6 +49,8 @@ pub enum Subcommands { }, /// Deletes all table entries Clear(clear::Command), + /// Verifies trie consistency and outputs any inconsistencies + RepairTrie(repair_trie::Command), /// Lists current and local database versions Version, /// Returns the full database path @@ -59,7 +62,7 @@ macro_rules! db_ro_exec { ($env:expr, $tool:ident, $N:ident, $command:block) => { let Environment { provider_factory, .. } = $env.init::<$N>(AccessRights::RO)?; - let $tool = DbTool::new(provider_factory.clone())?; + let $tool = DbTool::new(provider_factory)?; $command; }; } @@ -135,6 +138,12 @@ impl> Command let Environment { provider_factory, .. } = self.env.init::(AccessRights::RW)?; command.execute(provider_factory)?; } + Subcommands::RepairTrie(command) => { + let access_rights = + if command.dry_run { AccessRights::RO } else { AccessRights::RW }; + let Environment { provider_factory, .. } = self.env.init::(access_rights)?; + command.execute(provider_factory)?; + } Subcommands::Version => { let local_db_version = match get_db_version(&db_path) { Ok(version) => Some(version), diff --git a/crates/cli/commands/src/db/repair_trie.rs b/crates/cli/commands/src/db/repair_trie.rs new file mode 100644 index 00000000000..f7dea67b76f --- /dev/null +++ b/crates/cli/commands/src/db/repair_trie.rs @@ -0,0 +1,249 @@ +use clap::Parser; +use reth_db_api::{ + cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO}, + database::Database, + tables, + transaction::{DbTx, DbTxMut}, +}; +use reth_node_builder::NodeTypesWithDB; +use reth_provider::{providers::ProviderNodeTypes, ProviderFactory, StageCheckpointReader}; +use reth_stages::StageId; +use reth_trie::{ + verify::{Output, Verifier}, + Nibbles, +}; +use reth_trie_common::{StorageTrieEntry, StoredNibbles, StoredNibblesSubKey}; +use reth_trie_db::{DatabaseHashedCursorFactory, DatabaseTrieCursorFactory}; +use std::time::{Duration, Instant}; +use tracing::{info, warn}; + +const PROGRESS_PERIOD: Duration = Duration::from_secs(5); + +/// The arguments for the `reth db repair-trie` command +#[derive(Parser, Debug)] +pub struct Command { + /// Only show inconsistencies without making any repairs + #[arg(long)] + pub(crate) dry_run: bool, +} + +impl Command { + /// Execute `db repair-trie` command + pub fn execute( + self, + provider_factory: ProviderFactory, + ) -> eyre::Result<()> { + if self.dry_run { + verify_only(provider_factory)? + } else { + verify_and_repair(provider_factory)? + } + + Ok(()) + } +} + +fn verify_only(provider_factory: ProviderFactory) -> eyre::Result<()> { + // Get a database transaction directly from the database + let db = provider_factory.db_ref(); + let mut tx = db.tx()?; + tx.disable_long_read_transaction_safety(); + + // Create the verifier + let hashed_cursor_factory = DatabaseHashedCursorFactory::new(&tx); + let trie_cursor_factory = DatabaseTrieCursorFactory::new(&tx); + let verifier = Verifier::new(&trie_cursor_factory, hashed_cursor_factory)?; + + let mut inconsistent_nodes = 0; + let start_time = Instant::now(); + let mut last_progress_time = Instant::now(); + + // Iterate over the verifier and repair inconsistencies + for output_result in verifier { + let output = output_result?; + + if let Output::Progress(path) = output { + if last_progress_time.elapsed() > PROGRESS_PERIOD { + output_progress(path, start_time, inconsistent_nodes); + last_progress_time = Instant::now(); + } + } else { + warn!("Inconsistency found: {output:?}"); + inconsistent_nodes += 1; + } + } + + info!("Found {} inconsistencies (dry run - no changes made)", inconsistent_nodes); + + Ok(()) +} + +/// Checks that the merkle stage has completed running up to the account and storage hashing stages. +fn verify_checkpoints(provider: impl StageCheckpointReader) -> eyre::Result<()> { + let account_hashing_checkpoint = + provider.get_stage_checkpoint(StageId::AccountHashing)?.unwrap_or_default(); + let storage_hashing_checkpoint = + provider.get_stage_checkpoint(StageId::StorageHashing)?.unwrap_or_default(); + let merkle_checkpoint = + provider.get_stage_checkpoint(StageId::MerkleExecute)?.unwrap_or_default(); + + if account_hashing_checkpoint.block_number != merkle_checkpoint.block_number { + return Err(eyre::eyre!( + "MerkleExecute stage checkpoint ({}) != AccountHashing stage checkpoint ({}), you must first complete the pipeline sync by running `reth node`", + merkle_checkpoint.block_number, + account_hashing_checkpoint.block_number, + )) + } + + if storage_hashing_checkpoint.block_number != merkle_checkpoint.block_number { + return Err(eyre::eyre!( + "MerkleExecute stage checkpoint ({}) != StorageHashing stage checkpoint ({}), you must first complete the pipeline sync by running `reth node`", + merkle_checkpoint.block_number, + storage_hashing_checkpoint.block_number, + )) + } + + let merkle_checkpoint_progress = + provider.get_stage_checkpoint_progress(StageId::MerkleExecute)?; + if merkle_checkpoint_progress.is_some_and(|progress| !progress.is_empty()) { + return Err(eyre::eyre!( + "MerkleExecute sync stage in-progress, you must first complete the pipeline sync by running `reth node`", + )) + } + + Ok(()) +} + +fn verify_and_repair( + provider_factory: ProviderFactory, +) -> eyre::Result<()> { + // Get a read-write database provider + let mut provider_rw = provider_factory.provider_rw()?; + + // Check that a pipeline sync isn't in progress. + verify_checkpoints(provider_rw.as_ref())?; + + // Create cursors for making modifications with + let tx = provider_rw.tx_mut(); + tx.disable_long_read_transaction_safety(); + let mut account_trie_cursor = tx.cursor_write::()?; + let mut storage_trie_cursor = tx.cursor_dup_write::()?; + + // Create the cursor factories. These cannot accept the `&mut` tx above because they require it + // to be AsRef. + let tx = provider_rw.tx_ref(); + let hashed_cursor_factory = DatabaseHashedCursorFactory::new(tx); + let trie_cursor_factory = DatabaseTrieCursorFactory::new(tx); + + // Create the verifier + let verifier = Verifier::new(&trie_cursor_factory, hashed_cursor_factory)?; + + let mut inconsistent_nodes = 0; + let start_time = Instant::now(); + let mut last_progress_time = Instant::now(); + + // Iterate over the verifier and repair inconsistencies + for output_result in verifier { + let output = output_result?; + + if !matches!(output, Output::Progress(_)) { + warn!("Inconsistency found, will repair: {output:?}"); + inconsistent_nodes += 1; + } + + match output { + Output::AccountExtra(path, _node) => { + // Extra account node in trie, remove it + let nibbles = StoredNibbles(path); + if account_trie_cursor.seek_exact(nibbles)?.is_some() { + account_trie_cursor.delete_current()?; + } + } + Output::StorageExtra(account, path, _node) => { + // Extra storage node in trie, remove it + let nibbles = StoredNibblesSubKey(path); + if storage_trie_cursor + .seek_by_key_subkey(account, nibbles.clone())? + .filter(|e| e.nibbles == nibbles) + .is_some() + { + storage_trie_cursor.delete_current()?; + } + } + Output::AccountWrong { path, expected: node, .. } | + Output::AccountMissing(path, node) => { + // Wrong/missing account node value, upsert it + let nibbles = StoredNibbles(path); + account_trie_cursor.upsert(nibbles, &node)?; + } + Output::StorageWrong { account, path, expected: node, .. } | + Output::StorageMissing(account, path, node) => { + // Wrong/missing storage node value, upsert it + // (We can't just use `upsert` method with a dup cursor, it's not properly + // supported) + let nibbles = StoredNibblesSubKey(path); + let entry = StorageTrieEntry { nibbles: nibbles.clone(), node }; + if storage_trie_cursor + .seek_by_key_subkey(account, nibbles.clone())? + .filter(|v| v.nibbles == nibbles) + .is_some() + { + storage_trie_cursor.delete_current()?; + } + storage_trie_cursor.upsert(account, &entry)?; + } + Output::Progress(path) => { + if last_progress_time.elapsed() > PROGRESS_PERIOD { + output_progress(path, start_time, inconsistent_nodes); + last_progress_time = Instant::now(); + } + } + } + } + + if inconsistent_nodes == 0 { + info!("No inconsistencies found"); + } else { + info!("Repaired {} inconsistencies, committing changes", inconsistent_nodes); + provider_rw.commit()?; + } + + Ok(()) +} + +/// Output progress information based on the last seen account path. +fn output_progress(last_account: Nibbles, start_time: Instant, inconsistent_nodes: u64) { + // Calculate percentage based on position in the trie path space + // For progress estimation, we'll use the first few nibbles as an approximation + + // Convert the first 16 nibbles (8 bytes) to a u64 for progress calculation + let mut current_value: u64 = 0; + let nibbles_to_use = last_account.len().min(16); + + for i in 0..nibbles_to_use { + current_value = (current_value << 4) | (last_account.get(i).unwrap_or(0) as u64); + } + // Shift left to fill remaining bits if we have fewer than 16 nibbles + if nibbles_to_use < 16 { + current_value <<= (16 - nibbles_to_use) * 4; + } + + let progress_percent = current_value as f64 / u64::MAX as f64 * 100.0; + let progress_percent_str = format!("{progress_percent:.2}"); + + // Calculate ETA based on current speed + let elapsed = start_time.elapsed(); + let elapsed_secs = elapsed.as_secs_f64(); + + let estimated_total_time = + if progress_percent > 0.0 { elapsed_secs / (progress_percent / 100.0) } else { 0.0 }; + let remaining_time = estimated_total_time - elapsed_secs; + let eta_duration = Duration::from_secs(remaining_time as u64); + + info!( + progress_percent = progress_percent_str, + eta = %humantime::format_duration(eta_duration), + inconsistent_nodes, + "Repairing trie tables", + ); +} diff --git a/crates/cli/commands/src/download.rs b/crates/cli/commands/src/download.rs index 08c21d9eb83..20bc7081f05 100644 --- a/crates/cli/commands/src/download.rs +++ b/crates/cli/commands/src/download.rs @@ -7,37 +7,125 @@ use reth_chainspec::{EthChainSpec, EthereumHardforks}; use reth_cli::chainspec::ChainSpecParser; use reth_fs_util as fs; use std::{ + borrow::Cow, io::{self, Read, Write}, path::Path, - sync::Arc, + sync::{Arc, OnceLock}, time::{Duration, Instant}, }; use tar::Archive; use tokio::task; use tracing::info; +use zstd::stream::read::Decoder as ZstdDecoder; const BYTE_UNITS: [&str; 4] = ["B", "KB", "MB", "GB"]; const MERKLE_BASE_URL: &str = "https://downloads.merkle.io"; -const EXTENSION_TAR_FILE: &str = ".tar.lz4"; +const EXTENSION_TAR_LZ4: &str = ".tar.lz4"; +const EXTENSION_TAR_ZSTD: &str = ".tar.zst"; + +/// Global static download defaults +static DOWNLOAD_DEFAULTS: OnceLock = OnceLock::new(); + +/// Download configuration defaults +/// +/// Global defaults can be set via [`DownloadDefaults::try_init`]. +#[derive(Debug, Clone)] +pub struct DownloadDefaults { + /// List of available snapshot sources + pub available_snapshots: Vec>, + /// Default base URL for snapshots + pub default_base_url: Cow<'static, str>, + /// Optional custom long help text that overrides the generated help + pub long_help: Option, +} + +impl DownloadDefaults { + /// Initialize the global download defaults with this configuration + pub fn try_init(self) -> Result<(), Self> { + DOWNLOAD_DEFAULTS.set(self) + } + + /// Get a reference to the global download defaults + pub fn get_global() -> &'static DownloadDefaults { + DOWNLOAD_DEFAULTS.get_or_init(DownloadDefaults::default_download_defaults) + } + + /// Default download configuration with defaults from merkle.io and publicnode + pub fn default_download_defaults() -> Self { + Self { + available_snapshots: vec![ + Cow::Borrowed("https://www.merkle.io/snapshots (default, mainnet archive)"), + Cow::Borrowed("https://publicnode.com/snapshots (full nodes & testnets)"), + ], + default_base_url: Cow::Borrowed(MERKLE_BASE_URL), + long_help: None, + } + } + + /// Generates the long help text for the download URL argument using these defaults. + /// + /// If a custom long_help is set, it will be returned. Otherwise, help text is generated + /// from the available_snapshots list. + pub fn long_help(&self) -> String { + if let Some(ref custom_help) = self.long_help { + return custom_help.clone(); + } + + let mut help = String::from( + "Specify a snapshot URL or let the command propose a default one.\n\nAvailable snapshot sources:\n", + ); + + for source in &self.available_snapshots { + help.push_str("- "); + help.push_str(source); + help.push('\n'); + } + + help.push_str( + "\nIf no URL is provided, the latest mainnet archive snapshot\nwill be proposed for download from ", + ); + help.push_str(self.default_base_url.as_ref()); + help + } + + /// Add a snapshot source to the list + pub fn with_snapshot(mut self, source: impl Into>) -> Self { + self.available_snapshots.push(source.into()); + self + } + + /// Replace all snapshot sources + pub fn with_snapshots(mut self, sources: Vec>) -> Self { + self.available_snapshots = sources; + self + } + + /// Set the default base URL, e.g. `https://downloads.merkle.io`. + pub fn with_base_url(mut self, url: impl Into>) -> Self { + self.default_base_url = url.into(); + self + } + + /// Builder: Set custom long help text, overriding the generated help + pub fn with_long_help(mut self, help: impl Into) -> Self { + self.long_help = Some(help.into()); + self + } +} + +impl Default for DownloadDefaults { + fn default() -> Self { + Self::default_download_defaults() + } +} #[derive(Debug, Parser)] pub struct DownloadCommand { #[command(flatten)] env: EnvironmentArgs, - #[arg( - long, - short, - help = "Custom URL to download the snapshot from", - long_help = "Specify a snapshot URL or let the command propose a default one.\n\ - \n\ - Available snapshot sources:\n\ - - https://downloads.merkle.io (default, mainnet archive)\n\ - - https://publicnode.com/snapshots (full nodes & testnets)\n\ - \n\ - If no URL is provided, the latest mainnet archive snapshot\n\ - will be proposed for download from merkle.io" - )] + /// Custom URL to download the snapshot from + #[arg(long, short, long_help = DownloadDefaults::get_global().long_help())] url: Option, } @@ -139,16 +227,36 @@ impl ProgressReader { impl Read for ProgressReader { fn read(&mut self, buf: &mut [u8]) -> io::Result { let bytes = self.reader.read(buf)?; - if bytes > 0 { - if let Err(e) = self.progress.update(bytes as u64) { - return Err(io::Error::other(e)); - } + if bytes > 0 && + let Err(e) = self.progress.update(bytes as u64) + { + return Err(io::Error::other(e)); } Ok(bytes) } } -/// Downloads and extracts a snapshot with blocking approach +/// Supported compression formats for snapshots +#[derive(Debug, Clone, Copy)] +enum CompressionFormat { + Lz4, + Zstd, +} + +impl CompressionFormat { + /// Detect compression format from file extension + fn from_url(url: &str) -> Result { + if url.ends_with(EXTENSION_TAR_LZ4) { + Ok(Self::Lz4) + } else if url.ends_with(EXTENSION_TAR_ZSTD) { + Ok(Self::Zstd) + } else { + Err(eyre::eyre!("Unsupported file format. Expected .tar.lz4 or .tar.zst, got: {}", url)) + } + } +} + +/// Downloads and extracts a snapshot, blocking until finished. fn blocking_download_and_extract(url: &str, target_dir: &Path) -> Result<()> { let client = reqwest::blocking::Client::builder().build()?; let response = client.get(url).send()?.error_for_status()?; @@ -160,11 +268,18 @@ fn blocking_download_and_extract(url: &str, target_dir: &Path) -> Result<()> { })?; let progress_reader = ProgressReader::new(response, total_size); + let format = CompressionFormat::from_url(url)?; - let decoder = Decoder::new(progress_reader)?; - let mut archive = Archive::new(decoder); - - archive.unpack(target_dir)?; + match format { + CompressionFormat::Lz4 => { + let decoder = Decoder::new(progress_reader)?; + Archive::new(decoder).unpack(target_dir)?; + } + CompressionFormat::Zstd => { + let decoder = ZstdDecoder::new(progress_reader)?; + Archive::new(decoder).unpack(target_dir)?; + } + } info!(target: "reth::cli", "Extraction complete."); Ok(()) @@ -178,9 +293,10 @@ async fn stream_and_extract(url: &str, target_dir: &Path) -> Result<()> { Ok(()) } -// Builds default URL for latest mainnet archive snapshot +// Builds default URL for latest mainnet archive snapshot using configured defaults async fn get_latest_snapshot_url() -> Result { - let latest_url = format!("{MERKLE_BASE_URL}/latest.txt"); + let base_url = &DownloadDefaults::get_global().default_base_url; + let latest_url = format!("{base_url}/latest.txt"); let filename = Client::new() .get(latest_url) .send() @@ -191,9 +307,64 @@ async fn get_latest_snapshot_url() -> Result { .trim() .to_string(); - if !filename.ends_with(EXTENSION_TAR_FILE) { - return Err(eyre::eyre!("Unexpected snapshot filename format: {}", filename)); + Ok(format!("{base_url}/{filename}")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_download_defaults_builder() { + let defaults = DownloadDefaults::default() + .with_snapshot("https://example.com/snapshots (example)") + .with_base_url("https://example.com"); + + assert_eq!(defaults.default_base_url, "https://example.com"); + assert_eq!(defaults.available_snapshots.len(), 3); // 2 defaults + 1 added } - Ok(format!("{MERKLE_BASE_URL}/{filename}")) + #[test] + fn test_download_defaults_replace_snapshots() { + let defaults = DownloadDefaults::default().with_snapshots(vec![ + Cow::Borrowed("https://custom1.com"), + Cow::Borrowed("https://custom2.com"), + ]); + + assert_eq!(defaults.available_snapshots.len(), 2); + assert_eq!(defaults.available_snapshots[0], "https://custom1.com"); + } + + #[test] + fn test_long_help_generation() { + let defaults = DownloadDefaults::default(); + let help = defaults.long_help(); + + assert!(help.contains("Available snapshot sources:")); + assert!(help.contains("merkle.io")); + assert!(help.contains("publicnode.com")); + } + + #[test] + fn test_long_help_override() { + let custom_help = "This is custom help text for downloading snapshots."; + let defaults = DownloadDefaults::default().with_long_help(custom_help); + + let help = defaults.long_help(); + assert_eq!(help, custom_help); + assert!(!help.contains("Available snapshot sources:")); + } + + #[test] + fn test_builder_chaining() { + let defaults = DownloadDefaults::default() + .with_base_url("https://custom.example.com") + .with_snapshot("https://snapshot1.com") + .with_snapshot("https://snapshot2.com") + .with_long_help("Custom help for snapshots"); + + assert_eq!(defaults.default_base_url, "https://custom.example.com"); + assert_eq!(defaults.available_snapshots.len(), 4); // 2 defaults + 2 added + assert_eq!(defaults.long_help, Some("Custom help for snapshots".to_string())); + } } diff --git a/crates/cli/commands/src/dump_genesis.rs b/crates/cli/commands/src/dump_genesis.rs index 20e8b19d0b2..8130975f20a 100644 --- a/crates/cli/commands/src/dump_genesis.rs +++ b/crates/cli/commands/src/dump_genesis.rs @@ -15,7 +15,7 @@ pub struct DumpGenesisCommand { long, value_name = "CHAIN_OR_PATH", long_help = C::help_message(), - default_value = C::SUPPORTED_CHAINS[0], + default_value = C::default_value(), value_parser = C::parser() )] chain: Arc, diff --git a/crates/cli/commands/src/export_era.rs b/crates/cli/commands/src/export_era.rs new file mode 100644 index 00000000000..dbedf1852e5 --- /dev/null +++ b/crates/cli/commands/src/export_era.rs @@ -0,0 +1,109 @@ +//! Command exporting block data to convert them to ERA1 files. + +use crate::common::{AccessRights, CliNodeTypes, Environment, EnvironmentArgs}; +use clap::{Args, Parser}; +use reth_chainspec::{EthChainSpec, EthereumHardforks}; +use reth_cli::chainspec::ChainSpecParser; +use reth_era::execution_types::MAX_BLOCKS_PER_ERA1; +use reth_era_utils as era1; +use reth_provider::DatabaseProviderFactory; +use std::{path::PathBuf, sync::Arc}; +use tracing::info; + +// Default folder name for era1 export files +const ERA1_EXPORT_FOLDER_NAME: &str = "era1-export"; + +#[derive(Debug, Parser)] +pub struct ExportEraCommand { + #[command(flatten)] + env: EnvironmentArgs, + + #[clap(flatten)] + export: ExportArgs, +} + +#[derive(Debug, Args)] +pub struct ExportArgs { + /// Optional first block number to export from the db. + /// It is by default 0. + #[arg(long, value_name = "first-block-number", verbatim_doc_comment)] + first_block_number: Option, + /// Optional last block number to export from the db. + /// It is by default 8191. + #[arg(long, value_name = "last-block-number", verbatim_doc_comment)] + last_block_number: Option, + /// The maximum number of blocks per file, it can help you to decrease the size of the files. + /// Must be less than or equal to 8192. + #[arg(long, value_name = "max-blocks-per-file", verbatim_doc_comment)] + max_blocks_per_file: Option, + /// The directory path where to export era1 files. + /// The block data are read from the database. + #[arg(long, value_name = "EXPORT_ERA1_PATH", verbatim_doc_comment)] + path: Option, +} + +impl> ExportEraCommand { + /// Execute `export-era` command + pub async fn execute(self) -> eyre::Result<()> + where + N: CliNodeTypes, + { + let Environment { provider_factory, .. } = self.env.init::(AccessRights::RO)?; + + // Either specified path or default to `//era1-export/` + let data_dir = match &self.export.path { + Some(path) => path.clone(), + None => self + .env + .datadir + .resolve_datadir(self.env.chain.chain()) + .data_dir() + .join(ERA1_EXPORT_FOLDER_NAME), + }; + + let export_config = era1::ExportConfig { + network: self.env.chain.chain().to_string(), + first_block_number: self.export.first_block_number.unwrap_or(0), + last_block_number: self + .export + .last_block_number + .unwrap_or(MAX_BLOCKS_PER_ERA1 as u64 - 1), + max_blocks_per_file: self + .export + .max_blocks_per_file + .unwrap_or(MAX_BLOCKS_PER_ERA1 as u64), + dir: data_dir, + }; + + export_config.validate()?; + + info!( + target: "reth::cli", + "Starting ERA1 block export: blocks {}-{} to {}", + export_config.first_block_number, + export_config.last_block_number, + export_config.dir.display() + ); + + // Only read access is needed for the database provider + let provider = provider_factory.database_provider_ro()?; + + let exported_files = era1::export(&provider, &export_config)?; + + info!( + target: "reth::cli", + "Successfully exported {} ERA1 files to {}", + exported_files.len(), + export_config.dir.display() + ); + + Ok(()) + } +} + +impl ExportEraCommand { + /// Returns the underlying chain being used to run this command + pub fn chain_spec(&self) -> Option<&Arc> { + Some(&self.env.chain) + } +} diff --git a/crates/cli/commands/src/import.rs b/crates/cli/commands/src/import.rs index 6eff43acd68..e8493c9ab33 100644 --- a/crates/cli/commands/src/import.rs +++ b/crates/cli/commands/src/import.rs @@ -1,38 +1,18 @@ //! Command that initializes the node by importing a chain from a file. -use crate::common::{AccessRights, CliNodeComponents, CliNodeTypes, Environment, EnvironmentArgs}; -use alloy_primitives::B256; +use crate::{ + common::{AccessRights, CliNodeComponents, CliNodeTypes, Environment, EnvironmentArgs}, + import_core::{import_blocks_from_file, ImportConfig}, +}; use clap::Parser; -use futures::{Stream, StreamExt}; -use reth_chainspec::{EthChainSpec, EthereumHardforks}; +use reth_chainspec::{ChainSpecProvider, EthChainSpec, EthereumHardforks}; use reth_cli::chainspec::ChainSpecParser; -use reth_config::Config; -use reth_consensus::{ConsensusError, FullConsensus}; -use reth_db_api::{tables, transaction::DbTx}; -use reth_downloaders::{ - bodies::bodies::BodiesDownloaderBuilder, - file_client::{ChunkedFileReader, FileClient, DEFAULT_BYTE_LEN_CHUNK_CHAIN_FILE}, - headers::reverse_headers::ReverseHeadersDownloaderBuilder, -}; -use reth_evm::ConfigureEvm; -use reth_network_p2p::{ - bodies::downloader::BodyDownloader, - headers::downloader::{HeaderDownloader, SyncTarget}, -}; -use reth_node_api::BlockTy; -use reth_node_core::version::SHORT_VERSION; -use reth_node_events::node::NodeEvent; -use reth_provider::{ - providers::ProviderNodeTypes, BlockNumReader, ChainSpecProvider, HeaderProvider, ProviderError, - ProviderFactory, StageCheckpointReader, -}; -use reth_prune::PruneModes; -use reth_stages::{prelude::*, Pipeline, StageId, StageSet}; -use reth_static_file::StaticFileProducer; +use reth_node_core::version::version_metadata; use std::{path::PathBuf, sync::Arc}; -use tokio::sync::watch; -use tracing::{debug, error, info}; +use tracing::info; + +pub use crate::import_core::build_import_pipeline_impl as build_import_pipeline; -/// Syncs RLP encoded blocks from a file. +/// Syncs RLP encoded blocks from a file or files. #[derive(Debug, Parser)] pub struct ImportCommand { #[command(flatten)] @@ -46,118 +26,80 @@ pub struct ImportCommand { #[arg(long, value_name = "CHUNK_LEN", verbatim_doc_comment)] chunk_len: Option, - /// The path to a block file for import. + /// The path(s) to block file(s) for import. /// /// The online stages (headers and bodies) are replaced by a file import, after which the - /// remaining stages are executed. - #[arg(value_name = "IMPORT_PATH", verbatim_doc_comment)] - path: PathBuf, + /// remaining stages are executed. Multiple files will be imported sequentially. + #[arg(value_name = "IMPORT_PATH", required = true, num_args = 1.., verbatim_doc_comment)] + paths: Vec, } impl> ImportCommand { /// Execute `import` command - pub async fn execute(self, components: F) -> eyre::Result<()> + pub async fn execute( + self, + components: impl FnOnce(Arc) -> Comp, + ) -> eyre::Result<()> where N: CliNodeTypes, Comp: CliNodeComponents, - F: FnOnce(Arc) -> Comp, { - info!(target: "reth::cli", "reth {} starting", SHORT_VERSION); - - if self.no_state { - info!(target: "reth::cli", "Disabled stages requiring state"); - } - - debug!(target: "reth::cli", - chunk_byte_len=self.chunk_len.unwrap_or(DEFAULT_BYTE_LEN_CHUNK_CHAIN_FILE), - "Chunking chain import" - ); + info!(target: "reth::cli", "reth {} starting", version_metadata().short_version); let Environment { provider_factory, config, .. } = self.env.init::(AccessRights::RW)?; let components = components(provider_factory.chain_spec()); + + info!(target: "reth::cli", "Starting import of {} file(s)", self.paths.len()); + + let import_config = ImportConfig { no_state: self.no_state, chunk_len: self.chunk_len }; + let executor = components.evm_config().clone(); let consensus = Arc::new(components.consensus().clone()); - info!(target: "reth::cli", "Consensus engine initialized"); - - // open file - let mut reader = ChunkedFileReader::new(&self.path, self.chunk_len).await?; + let mut total_imported_blocks = 0; + let mut total_imported_txns = 0; let mut total_decoded_blocks = 0; let mut total_decoded_txns = 0; - let mut sealed_header = provider_factory - .sealed_header(provider_factory.last_block_number()?)? - .expect("should have genesis"); + // Import each file sequentially + for (index, path) in self.paths.iter().enumerate() { + info!(target: "reth::cli", "Importing file {} of {}: {}", index + 1, self.paths.len(), path.display()); - while let Some(file_client) = - reader.next_chunk::>(consensus.clone(), Some(sealed_header)).await? - { - // create a new FileClient from chunk read from file - info!(target: "reth::cli", - "Importing chain file chunk" - ); - - let tip = file_client.tip().ok_or(eyre::eyre!("file client has no tip"))?; - info!(target: "reth::cli", "Chain file chunk read"); - - total_decoded_blocks += file_client.headers_len(); - total_decoded_txns += file_client.total_transactions(); - - let (mut pipeline, events) = build_import_pipeline( - &config, + let result = import_blocks_from_file( + path, + import_config.clone(), provider_factory.clone(), - &consensus, - Arc::new(file_client), - StaticFileProducer::new(provider_factory.clone(), PruneModes::default()), - self.no_state, + &config, executor.clone(), - )?; - - // override the tip - pipeline.set_tip(tip); - debug!(target: "reth::cli", ?tip, "Tip manually set"); - - let provider = provider_factory.provider()?; - - let latest_block_number = - provider.get_stage_checkpoint(StageId::Finish)?.map(|ch| ch.block_number); - tokio::spawn(reth_node_events::node::handle_events(None, latest_block_number, events)); - - // Run pipeline - info!(target: "reth::cli", "Starting sync pipeline"); - tokio::select! { - res = pipeline.run() => res?, - _ = tokio::signal::ctrl_c() => {}, + consensus.clone(), + ) + .await?; + + total_imported_blocks += result.total_imported_blocks; + total_imported_txns += result.total_imported_txns; + total_decoded_blocks += result.total_decoded_blocks; + total_decoded_txns += result.total_decoded_txns; + + if !result.is_complete() { + return Err(eyre::eyre!( + "Chain was partially imported from file: {}. Imported {}/{} blocks, {}/{} transactions", + path.display(), + result.total_imported_blocks, + result.total_decoded_blocks, + result.total_imported_txns, + result.total_decoded_txns + )); } - sealed_header = provider_factory - .sealed_header(provider_factory.last_block_number()?)? - .expect("should have genesis"); - } - - let provider = provider_factory.provider()?; - - let total_imported_blocks = provider.tx_ref().entries::()?; - let total_imported_txns = provider.tx_ref().entries::()?; - - if total_decoded_blocks != total_imported_blocks || - total_decoded_txns != total_imported_txns - { - error!(target: "reth::cli", - total_decoded_blocks, - total_imported_blocks, - total_decoded_txns, - total_imported_txns, - "Chain was partially imported" - ); + info!(target: "reth::cli", + "Successfully imported file {}: {} blocks, {} transactions", + path.display(), result.total_imported_blocks, result.total_imported_txns); } info!(target: "reth::cli", - total_imported_blocks, - total_imported_txns, - "Chain file imported" - ); + "All files imported successfully. Total: {}/{} blocks, {}/{} transactions", + total_imported_blocks, total_decoded_blocks, total_imported_txns, total_decoded_txns); Ok(()) } @@ -170,81 +112,6 @@ impl ImportCommand { } } -/// Builds import pipeline. -/// -/// If configured to execute, all stages will run. Otherwise, only stages that don't require state -/// will run. -pub fn build_import_pipeline( - config: &Config, - provider_factory: ProviderFactory, - consensus: &Arc, - file_client: Arc>>, - static_file_producer: StaticFileProducer>, - disable_exec: bool, - evm_config: E, -) -> eyre::Result<(Pipeline, impl Stream>)> -where - N: ProviderNodeTypes, - C: FullConsensus + 'static, - E: ConfigureEvm + 'static, -{ - if !file_client.has_canonical_blocks() { - eyre::bail!("unable to import non canonical blocks"); - } - - // Retrieve latest header found in the database. - let last_block_number = provider_factory.last_block_number()?; - let local_head = provider_factory - .sealed_header(last_block_number)? - .ok_or_else(|| ProviderError::HeaderNotFound(last_block_number.into()))?; - - let mut header_downloader = ReverseHeadersDownloaderBuilder::new(config.stages.headers) - .build(file_client.clone(), consensus.clone()) - .into_task(); - // TODO: The pipeline should correctly configure the downloader on its own. - // Find the possibility to remove unnecessary pre-configuration. - header_downloader.update_local_head(local_head); - header_downloader.update_sync_target(SyncTarget::Tip(file_client.tip().unwrap())); - - let mut body_downloader = BodiesDownloaderBuilder::new(config.stages.bodies) - .build(file_client.clone(), consensus.clone(), provider_factory.clone()) - .into_task(); - // TODO: The pipeline should correctly configure the downloader on its own. - // Find the possibility to remove unnecessary pre-configuration. - body_downloader - .set_download_range(file_client.min_block().unwrap()..=file_client.max_block().unwrap()) - .expect("failed to set download range"); - - let (tip_tx, tip_rx) = watch::channel(B256::ZERO); - - let max_block = file_client.max_block().unwrap_or(0); - - let pipeline = Pipeline::builder() - .with_tip_sender(tip_tx) - // we want to sync all blocks the file client provides or 0 if empty - .with_max_block(max_block) - .with_fail_on_unwind(true) - .add_stages( - DefaultStages::new( - provider_factory.clone(), - tip_rx, - consensus.clone(), - header_downloader, - body_downloader, - evm_config, - config.stages.clone(), - PruneModes::default(), - ) - .builder() - .disable_all_if(&StageId::STATE_REQUIRED, || disable_exec), - ) - .build(provider_factory, static_file_producer); - - let events = pipeline.events().map(Into::into); - - Ok((pipeline, events)) -} - #[cfg(test)] mod tests { use super::*; @@ -262,4 +129,14 @@ mod tests { ); } } + + #[test] + fn parse_import_command_with_multiple_paths() { + let args: ImportCommand = + ImportCommand::parse_from(["reth", "file1.rlp", "file2.rlp", "file3.rlp"]); + assert_eq!(args.paths.len(), 3); + assert_eq!(args.paths[0], PathBuf::from("file1.rlp")); + assert_eq!(args.paths[1], PathBuf::from("file2.rlp")); + assert_eq!(args.paths[2], PathBuf::from("file3.rlp")); + } } diff --git a/crates/cli/commands/src/import_core.rs b/crates/cli/commands/src/import_core.rs new file mode 100644 index 00000000000..98f888bb9e3 --- /dev/null +++ b/crates/cli/commands/src/import_core.rs @@ -0,0 +1,260 @@ +//! Core import functionality without CLI dependencies. + +use alloy_primitives::B256; +use futures::StreamExt; +use reth_config::Config; +use reth_consensus::FullConsensus; +use reth_db_api::{tables, transaction::DbTx}; +use reth_downloaders::{ + bodies::bodies::BodiesDownloaderBuilder, + file_client::{ChunkedFileReader, FileClient, DEFAULT_BYTE_LEN_CHUNK_CHAIN_FILE}, + headers::reverse_headers::ReverseHeadersDownloaderBuilder, +}; +use reth_evm::ConfigureEvm; +use reth_network_p2p::{ + bodies::downloader::BodyDownloader, + headers::downloader::{HeaderDownloader, SyncTarget}, +}; +use reth_node_api::BlockTy; +use reth_node_events::node::NodeEvent; +use reth_provider::{ + providers::ProviderNodeTypes, BlockNumReader, HeaderProvider, ProviderError, ProviderFactory, + StageCheckpointReader, +}; +use reth_prune::PruneModes; +use reth_stages::{prelude::*, Pipeline, StageId, StageSet}; +use reth_static_file::StaticFileProducer; +use std::{path::Path, sync::Arc}; +use tokio::sync::watch; +use tracing::{debug, error, info}; + +/// Configuration for importing blocks from RLP files. +#[derive(Debug, Clone, Default)] +pub struct ImportConfig { + /// Disables stages that require state. + pub no_state: bool, + /// Chunk byte length to read from file. + pub chunk_len: Option, +} + +/// Result of an import operation. +#[derive(Debug)] +pub struct ImportResult { + /// Total number of blocks decoded from the file. + pub total_decoded_blocks: usize, + /// Total number of transactions decoded from the file. + pub total_decoded_txns: usize, + /// Total number of blocks imported into the database. + pub total_imported_blocks: usize, + /// Total number of transactions imported into the database. + pub total_imported_txns: usize, +} + +impl ImportResult { + /// Returns true if all blocks and transactions were imported successfully. + pub fn is_complete(&self) -> bool { + self.total_decoded_blocks == self.total_imported_blocks && + self.total_decoded_txns == self.total_imported_txns + } +} + +/// Imports blocks from an RLP-encoded file into the database. +/// +/// This function reads RLP-encoded blocks from a file in chunks and imports them +/// using the pipeline infrastructure. It's designed to be used both from the CLI +/// and from test code. +pub async fn import_blocks_from_file( + path: &Path, + import_config: ImportConfig, + provider_factory: ProviderFactory, + config: &Config, + executor: impl ConfigureEvm + 'static, + consensus: Arc< + impl FullConsensus + 'static, + >, +) -> eyre::Result +where + N: ProviderNodeTypes, +{ + if import_config.no_state { + info!(target: "reth::import", "Disabled stages requiring state"); + } + + debug!(target: "reth::import", + chunk_byte_len=import_config.chunk_len.unwrap_or(DEFAULT_BYTE_LEN_CHUNK_CHAIN_FILE), + "Chunking chain import" + ); + + info!(target: "reth::import", "Consensus engine initialized"); + + // open file + let mut reader = ChunkedFileReader::new(path, import_config.chunk_len).await?; + + let provider = provider_factory.provider()?; + let init_blocks = provider.tx_ref().entries::()?; + let init_txns = provider.tx_ref().entries::()?; + drop(provider); + + let mut total_decoded_blocks = 0; + let mut total_decoded_txns = 0; + + let mut sealed_header = provider_factory + .sealed_header(provider_factory.last_block_number()?)? + .expect("should have genesis"); + + let static_file_producer = + StaticFileProducer::new(provider_factory.clone(), PruneModes::default()); + + while let Some(file_client) = + reader.next_chunk::>(consensus.clone(), Some(sealed_header)).await? + { + // create a new FileClient from chunk read from file + info!(target: "reth::import", + "Importing chain file chunk" + ); + + let tip = file_client.tip().ok_or(eyre::eyre!("file client has no tip"))?; + info!(target: "reth::import", "Chain file chunk read"); + + total_decoded_blocks += file_client.headers_len(); + total_decoded_txns += file_client.total_transactions(); + + let (mut pipeline, events) = build_import_pipeline_impl( + config, + provider_factory.clone(), + &consensus, + Arc::new(file_client), + static_file_producer.clone(), + import_config.no_state, + executor.clone(), + )?; + + // override the tip + pipeline.set_tip(tip); + debug!(target: "reth::import", ?tip, "Tip manually set"); + + let latest_block_number = + provider_factory.get_stage_checkpoint(StageId::Finish)?.map(|ch| ch.block_number); + tokio::spawn(reth_node_events::node::handle_events(None, latest_block_number, events)); + + // Run pipeline + info!(target: "reth::import", "Starting sync pipeline"); + tokio::select! { + res = pipeline.run() => res?, + _ = tokio::signal::ctrl_c() => { + info!(target: "reth::import", "Import interrupted by user"); + break; + }, + } + + sealed_header = provider_factory + .sealed_header(provider_factory.last_block_number()?)? + .expect("should have genesis"); + } + + let provider = provider_factory.provider()?; + let total_imported_blocks = provider.tx_ref().entries::()? - init_blocks; + let total_imported_txns = + provider.tx_ref().entries::()? - init_txns; + + let result = ImportResult { + total_decoded_blocks, + total_decoded_txns, + total_imported_blocks, + total_imported_txns, + }; + + if !result.is_complete() { + error!(target: "reth::import", + total_decoded_blocks, + total_imported_blocks, + total_decoded_txns, + total_imported_txns, + "Chain was partially imported" + ); + } else { + info!(target: "reth::import", + total_imported_blocks, + total_imported_txns, + "Chain was fully imported" + ); + } + + Ok(result) +} + +/// Builds import pipeline. +/// +/// If configured to execute, all stages will run. Otherwise, only stages that don't require state +/// will run. +pub fn build_import_pipeline_impl( + config: &Config, + provider_factory: ProviderFactory, + consensus: &Arc, + file_client: Arc>>, + static_file_producer: StaticFileProducer>, + disable_exec: bool, + evm_config: E, +) -> eyre::Result<(Pipeline, impl futures::Stream> + use)> +where + N: ProviderNodeTypes, + C: FullConsensus + 'static, + E: ConfigureEvm + 'static, +{ + if !file_client.has_canonical_blocks() { + eyre::bail!("unable to import non canonical blocks"); + } + + // Retrieve latest header found in the database. + let last_block_number = provider_factory.last_block_number()?; + let local_head = provider_factory + .sealed_header(last_block_number)? + .ok_or_else(|| ProviderError::HeaderNotFound(last_block_number.into()))?; + + let mut header_downloader = ReverseHeadersDownloaderBuilder::new(config.stages.headers) + .build(file_client.clone(), consensus.clone()) + .into_task(); + // TODO: The pipeline should correctly configure the downloader on its own. + // Find the possibility to remove unnecessary pre-configuration. + header_downloader.update_local_head(local_head); + header_downloader.update_sync_target(SyncTarget::Tip(file_client.tip().unwrap())); + + let mut body_downloader = BodiesDownloaderBuilder::new(config.stages.bodies) + .build(file_client.clone(), consensus.clone(), provider_factory.clone()) + .into_task(); + // TODO: The pipeline should correctly configure the downloader on its own. + // Find the possibility to remove unnecessary pre-configuration. + body_downloader + .set_download_range(file_client.min_block().unwrap()..=file_client.max_block().unwrap()) + .expect("failed to set download range"); + + let (tip_tx, tip_rx) = watch::channel(B256::ZERO); + + let max_block = file_client.max_block().unwrap_or(0); + + let pipeline = Pipeline::builder() + .with_tip_sender(tip_tx) + // we want to sync all blocks the file client provides or 0 if empty + .with_max_block(max_block) + .with_fail_on_unwind(true) + .add_stages( + DefaultStages::new( + provider_factory.clone(), + tip_rx, + consensus.clone(), + header_downloader, + body_downloader, + evm_config, + config.stages.clone(), + PruneModes::default(), + None, + ) + .builder() + .disable_all_if(&StageId::STATE_REQUIRED, || disable_exec), + ) + .build(provider_factory, static_file_producer); + + let events = pipeline.events().map(Into::into); + + Ok((pipeline, events)) +} diff --git a/crates/cli/commands/src/import_era.rs b/crates/cli/commands/src/import_era.rs index 3d5097649cf..2bf1fe1c9f7 100644 --- a/crates/cli/commands/src/import_era.rs +++ b/crates/cli/commands/src/import_era.rs @@ -2,14 +2,17 @@ use crate::common::{AccessRights, CliNodeTypes, Environment, EnvironmentArgs}; use alloy_chains::{ChainKind, NamedChain}; use clap::{Args, Parser}; -use eyre::{eyre, OptionExt}; +use eyre::eyre; use reqwest::{Client, Url}; use reth_chainspec::{EthChainSpec, EthereumHardforks}; use reth_cli::chainspec::ChainSpecParser; use reth_era_downloader::{read_dir, EraClient, EraStream, EraStreamConfig}; use reth_era_utils as era; use reth_etl::Collector; -use reth_node_core::{dirs::data_dir, version::SHORT_VERSION}; +use reth_fs_util as fs; +use reth_node_core::version::version_metadata; +use reth_provider::StaticFileProviderFactory; +use reth_static_file_types::StaticFileSegment; use std::{path::PathBuf, sync::Arc}; use tracing::info; @@ -48,10 +51,11 @@ impl TryFromChain for ChainKind { fn try_to_url(&self) -> eyre::Result { Ok(match self { ChainKind::Named(NamedChain::Mainnet) => { - Url::parse("https://era.ithaca.xyz/era1/").expect("URL should be valid") + Url::parse("https://era.ithaca.xyz/era1/index.html").expect("URL should be valid") } ChainKind::Named(NamedChain::Sepolia) => { - Url::parse("https://era.ithaca.xyz/sepolia-era1/").expect("URL should be valid") + Url::parse("https://era.ithaca.xyz/sepolia-era1/index.html") + .expect("URL should be valid") } chain => return Err(eyre!("No known host for ERA files on chain {chain:?}")), }) @@ -64,28 +68,37 @@ impl> ImportEraC where N: CliNodeTypes, { - info!(target: "reth::cli", "reth {} starting", SHORT_VERSION); + info!(target: "reth::cli", "reth {} starting", version_metadata().short_version); let Environment { provider_factory, config, .. } = self.env.init::(AccessRights::RW)?; - let hash_collector = Collector::new(config.stages.etl.file_size, config.stages.etl.dir); - let provider_factory = &provider_factory.provider_rw()?.0; + let mut hash_collector = Collector::new(config.stages.etl.file_size, config.stages.etl.dir); + + let next_block = provider_factory + .static_file_provider() + .get_highest_static_file_block(StaticFileSegment::Headers) + .unwrap_or_default() + + 1; if let Some(path) = self.import.path { - let stream = read_dir(path)?; + let stream = read_dir(path, next_block)?; - era::import(stream, provider_factory, hash_collector)?; + era::import(stream, &provider_factory, &mut hash_collector)?; } else { let url = match self.import.url { Some(url) => url, None => self.env.chain.chain().kind().try_to_url()?, }; - let folder = data_dir().ok_or_eyre("Missing data directory")?.join("era"); - let folder = folder.into_boxed_path(); + let folder = + self.env.datadir.resolve_datadir(self.env.chain.chain()).data_dir().join("era"); + + fs::create_dir_all(&folder)?; + + let config = EraStreamConfig::default().start_from(next_block); let client = EraClient::new(Client::new(), url, folder); - let stream = EraStream::new(client, EraStreamConfig::default()); + let stream = EraStream::new(client, config); - era::import(stream, provider_factory, hash_collector)?; + era::import(stream, &provider_factory, &mut hash_collector)?; } Ok(()) diff --git a/crates/cli/commands/src/init_state/mod.rs b/crates/cli/commands/src/init_state/mod.rs index 4f0ae05d8bc..4b5c51585b3 100644 --- a/crates/cli/commands/src/init_state/mod.rs +++ b/crates/cli/commands/src/init_state/mod.rs @@ -1,17 +1,19 @@ //! Command that initializes the node from a genesis file. -use crate::common::{AccessRights, CliNodeTypes, Environment, EnvironmentArgs}; -use alloy_primitives::{B256, U256}; +use crate::common::{AccessRights, CliHeader, CliNodeTypes, Environment, EnvironmentArgs}; +use alloy_consensus::BlockHeader as AlloyBlockHeader; +use alloy_primitives::{Sealable, B256}; use clap::Parser; use reth_chainspec::{EthChainSpec, EthereumHardforks}; use reth_cli::chainspec::ChainSpecParser; use reth_db_common::init::init_from_state_dump; use reth_node_api::NodePrimitives; -use reth_primitives_traits::SealedHeader; +use reth_primitives_traits::{BlockHeader, SealedHeader}; use reth_provider::{ - BlockNumReader, DatabaseProviderFactory, StaticFileProviderFactory, StaticFileWriter, + BlockNumReader, DBProvider, DatabaseProviderFactory, StaticFileProviderFactory, + StaticFileWriter, }; -use std::{io::BufReader, path::PathBuf, str::FromStr, sync::Arc}; +use std::{io::BufReader, path::PathBuf, sync::Arc}; use tracing::info; pub mod without_evm; @@ -56,13 +58,9 @@ pub struct InitStateCommand { #[arg(long, value_name = "HEADER_FILE", verbatim_doc_comment)] pub header: Option, - /// Total difficulty of the header. - #[arg(long, value_name = "TOTAL_DIFFICULTY", verbatim_doc_comment)] - pub total_difficulty: Option, - /// Hash of the header. #[arg(long, value_name = "HEADER_HASH", verbatim_doc_comment)] - pub header_hash: Option, + pub header_hash: Option, } impl> InitStateCommand { @@ -71,7 +69,7 @@ impl> InitStateC where N: CliNodeTypes< ChainSpec = C::ChainSpec, - Primitives: NodePrimitives, + Primitives: NodePrimitives, >, { info!(target: "reth::cli", "Reth init-state starting"); @@ -84,16 +82,11 @@ impl> InitStateC if self.without_evm { // ensure header, total difficulty and header hash are provided let header = self.header.ok_or_else(|| eyre::eyre!("Header file must be provided"))?; - let header = without_evm::read_header_from_file(header)?; - - let header_hash = - self.header_hash.ok_or_else(|| eyre::eyre!("Header hash must be provided"))?; - let header_hash = B256::from_str(&header_hash)?; + let header = without_evm::read_header_from_file::< + ::BlockHeader, + >(&header)?; - let total_difficulty = self - .total_difficulty - .ok_or_else(|| eyre::eyre!("Total difficulty must be provided"))?; - let total_difficulty = U256::from_str(&total_difficulty)?; + let header_hash = self.header_hash.unwrap_or_else(|| header.hash_slow()); let last_block_number = provider_rw.last_block_number()?; @@ -101,7 +94,12 @@ impl> InitStateC without_evm::setup_without_evm( &provider_rw, SealedHeader::new(header, header_hash), - total_difficulty, + |number| { + let mut header = + <::BlockHeader>::default(); + header.set_number(number); + header + }, )?; // SAFETY: it's safe to commit static files, since in the event of a crash, they @@ -110,7 +108,7 @@ impl> InitStateC // Necessary to commit, so the header is accessible to provider_rw and // init_state_dump static_file_provider.commit()?; - } else if last_block_number > 0 && last_block_number < header.number { + } else if last_block_number > 0 && last_block_number < header.number() { return Err(eyre::eyre!( "Data directory should be empty when calling init-state with --without-evm-history." )); @@ -136,3 +134,32 @@ impl InitStateCommand { Some(&self.env.chain) } } + +#[cfg(test)] +mod tests { + use super::*; + use alloy_primitives::b256; + use reth_ethereum_cli::chainspec::EthereumChainSpecParser; + + #[test] + fn parse_init_state_command_with_without_evm() { + let cmd: InitStateCommand = InitStateCommand::parse_from([ + "reth", + "--chain", + "sepolia", + "--without-evm", + "--header", + "header.rlp", + "--header-hash", + "0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", + "state.jsonl", + ]); + assert_eq!(cmd.state.to_str().unwrap(), "state.jsonl"); + assert!(cmd.without_evm); + assert_eq!(cmd.header.unwrap().to_str().unwrap(), "header.rlp"); + assert_eq!( + cmd.header_hash.unwrap(), + b256!("0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef") + ); + } +} diff --git a/crates/cli/commands/src/init_state/without_evm.rs b/crates/cli/commands/src/init_state/without_evm.rs index 294c8796239..6ecf71a6337 100644 --- a/crates/cli/commands/src/init_state/without_evm.rs +++ b/crates/cli/commands/src/init_state/without_evm.rs @@ -1,49 +1,59 @@ -use alloy_consensus::{BlockHeader, Header}; -use alloy_primitives::{BlockNumber, B256, U256}; +use alloy_consensus::BlockHeader; +use alloy_primitives::{BlockNumber, B256}; use alloy_rlp::Decodable; use reth_codecs::Compact; use reth_node_builder::NodePrimitives; use reth_primitives_traits::{SealedBlock, SealedHeader, SealedHeaderFor}; use reth_provider::{ providers::StaticFileProvider, BlockWriter, ProviderResult, StageCheckpointWriter, - StaticFileProviderFactory, StaticFileWriter, StorageLocation, + StaticFileProviderFactory, StaticFileWriter, }; use reth_stages::{StageCheckpoint, StageId}; use reth_static_file_types::StaticFileSegment; -use std::{fs::File, io::Read, path::PathBuf}; +use std::path::Path; use tracing::info; /// Reads the header RLP from a file and returns the Header. -pub fn read_header_from_file(path: PathBuf) -> Result { - let mut file = File::open(path)?; - let mut buf = Vec::new(); - file.read_to_end(&mut buf)?; - - let header = Header::decode(&mut &buf[..])?; +pub fn read_header_from_file(path: &Path) -> Result +where + H: Decodable, +{ + let buf = if let Ok(content) = reth_fs_util::read_to_string(path) { + alloy_primitives::hex::decode(content.trim())? + } else { + // If UTF-8 decoding fails, read as raw bytes + reth_fs_util::read(path)? + }; + + let header = H::decode(&mut &buf[..])?; Ok(header) } /// Creates a dummy chain (with no transactions) up to the last EVM block and appends the /// first valid block. -pub fn setup_without_evm( +pub fn setup_without_evm( provider_rw: &Provider, header: SealedHeader<::BlockHeader>, - total_difficulty: U256, + header_factory: F, ) -> ProviderResult<()> where - Provider: StaticFileProviderFactory> + Provider: StaticFileProviderFactory + StageCheckpointWriter + BlockWriter::Block>, + F: Fn(BlockNumber) -> ::BlockHeader + + Send + + Sync + + 'static, { info!(target: "reth::cli", new_tip = ?header.num_hash(), "Setting up dummy EVM chain before importing state."); let static_file_provider = provider_rw.static_file_provider(); // Write EVM dummy data up to `header - 1` block - append_dummy_chain(&static_file_provider, header.number() - 1)?; + append_dummy_chain(&static_file_provider, header.number() - 1, header_factory)?; info!(target: "reth::cli", "Appending first valid block."); - append_first_block(provider_rw, &header, total_difficulty)?; + append_first_block(provider_rw, &header)?; for stage in StageId::ALL { provider_rw.save_stage_checkpoint(stage, StageCheckpoint::new(header.number()))?; @@ -61,7 +71,6 @@ where fn append_first_block( provider_rw: &Provider, header: &SealedHeaderFor, - total_difficulty: U256, ) -> ProviderResult<()> where Provider: BlockWriter::Block> @@ -74,21 +83,12 @@ where ) .try_recover() .expect("no senders or txes"), - StorageLocation::Database, )?; let sf_provider = provider_rw.static_file_provider(); - sf_provider.latest_writer(StaticFileSegment::Headers)?.append_header( - header, - total_difficulty, - &header.hash(), - )?; - sf_provider.latest_writer(StaticFileSegment::Receipts)?.increment_block(header.number())?; - sf_provider.latest_writer(StaticFileSegment::Transactions)?.increment_block(header.number())?; - Ok(()) } @@ -97,10 +97,15 @@ where /// * Headers: It will push an empty block. /// * Transactions: It will not push any tx, only increments the end block range. /// * Receipts: It will not push any receipt, only increments the end block range. -fn append_dummy_chain>( +fn append_dummy_chain( sf_provider: &StaticFileProvider, target_height: BlockNumber, -) -> ProviderResult<()> { + header_factory: F, +) -> ProviderResult<()> +where + N: NodePrimitives, + F: Fn(BlockNumber) -> N::BlockHeader + Send + Sync + 'static, +{ let (tx, rx) = std::sync::mpsc::channel(); // Spawn jobs for incrementing the block end range of transactions and receipts @@ -122,12 +127,11 @@ fn append_dummy_chain>( // Spawn job for appending empty headers let provider = sf_provider.clone(); std::thread::spawn(move || { - let mut empty_header = Header::default(); let result = provider.latest_writer(StaticFileSegment::Headers).and_then(|mut writer| { for block_num in 1..=target_height { // TODO: should we fill with real parent_hash? - empty_header.number = block_num; - writer.append_header(&empty_header, U256::ZERO, &B256::ZERO)?; + let header = header_factory(block_num); + writer.append_header(&header, &B256::ZERO)?; } Ok(()) }); @@ -157,3 +161,85 @@ fn append_dummy_chain>( Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + use alloy_consensus::Header; + use alloy_primitives::{address, b256}; + use reth_db_common::init::init_genesis; + use reth_provider::{test_utils::create_test_provider_factory, DatabaseProviderFactory}; + use std::io::Write; + use tempfile::NamedTempFile; + + #[test] + fn test_read_header_from_file_hex_string() { + let header_rlp = "0xf90212a00d84d79f59fc384a1f6402609a5b7253b4bfe7a4ae12608ed107273e5422b6dda01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d493479471562b71999873db5b286df957af199ec94617f7a0f496f3d199c51a1aaee67dac95f24d92ac13c60d25181e1eecd6eca5ddf32ac0a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421b9010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000808206a4840365908a808468e975f09ad983011003846765746888676f312e32352e308664617277696ea06f485a167165ec12e0ab3e6ab59a7b88560b90306ac98a26eb294abf95a8c59b88000000000000000007"; + + let mut temp_file = NamedTempFile::new().unwrap(); + temp_file.write_all(header_rlp.as_bytes()).unwrap(); + temp_file.flush().unwrap(); + + let header: Header = read_header_from_file(temp_file.path()).unwrap(); + + assert_eq!(header.number, 1700); + assert_eq!( + header.parent_hash, + b256!("0d84d79f59fc384a1f6402609a5b7253b4bfe7a4ae12608ed107273e5422b6dd") + ); + assert_eq!(header.beneficiary, address!("71562b71999873db5b286df957af199ec94617f7")); + } + + #[test] + fn test_read_header_from_file_raw_bytes() { + let header_rlp = "0xf90212a00d84d79f59fc384a1f6402609a5b7253b4bfe7a4ae12608ed107273e5422b6dda01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d493479471562b71999873db5b286df957af199ec94617f7a0f496f3d199c51a1aaee67dac95f24d92ac13c60d25181e1eecd6eca5ddf32ac0a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421b9010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000808206a4840365908a808468e975f09ad983011003846765746888676f312e32352e308664617277696ea06f485a167165ec12e0ab3e6ab59a7b88560b90306ac98a26eb294abf95a8c59b88000000000000000007"; + let header_bytes = + alloy_primitives::hex::decode(header_rlp.trim_start_matches("0x")).unwrap(); + + let mut temp_file = NamedTempFile::new().unwrap(); + temp_file.write_all(&header_bytes).unwrap(); + temp_file.flush().unwrap(); + + let header: Header = read_header_from_file(temp_file.path()).unwrap(); + + assert_eq!(header.number, 1700); + assert_eq!( + header.parent_hash, + b256!("0d84d79f59fc384a1f6402609a5b7253b4bfe7a4ae12608ed107273e5422b6dd") + ); + assert_eq!(header.beneficiary, address!("71562b71999873db5b286df957af199ec94617f7")); + } + + #[test] + fn test_setup_without_evm_succeeds() { + let header_rlp = "0xf90212a00d84d79f59fc384a1f6402609a5b7253b4bfe7a4ae12608ed107273e5422b6dda01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d493479471562b71999873db5b286df957af199ec94617f7a0f496f3d199c51a1aaee67dac95f24d92ac13c60d25181e1eecd6eca5ddf32ac0a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421b9010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000808206a4840365908a808468e975f09ad983011003846765746888676f312e32352e308664617277696ea06f485a167165ec12e0ab3e6ab59a7b88560b90306ac98a26eb294abf95a8c59b88000000000000000007"; + let header_bytes = + alloy_primitives::hex::decode(header_rlp.trim_start_matches("0x")).unwrap(); + + let mut temp_file = NamedTempFile::new().unwrap(); + temp_file.write_all(&header_bytes).unwrap(); + temp_file.flush().unwrap(); + + let header: Header = read_header_from_file(temp_file.path()).unwrap(); + let header_hash = b256!("4f05e4392969fc82e41f6d6a8cea379323b0b2d3ddf7def1a33eec03883e3a33"); + + let provider_factory = create_test_provider_factory(); + + init_genesis(&provider_factory).unwrap(); + + let provider_rw = provider_factory.database_provider_rw().unwrap(); + + setup_without_evm(&provider_rw, SealedHeader::new(header, header_hash), |number| Header { + number, + ..Default::default() + }) + .unwrap(); + + let static_files = provider_factory.static_file_provider(); + let writer = static_files.latest_writer(StaticFileSegment::Headers).unwrap(); + let actual_next_height = writer.next_block_number(); + let expected_next_height = 1701; + + assert_eq!(actual_next_height, expected_next_height); + } +} diff --git a/crates/cli/commands/src/launcher.rs b/crates/cli/commands/src/launcher.rs new file mode 100644 index 00000000000..d782334546b --- /dev/null +++ b/crates/cli/commands/src/launcher.rs @@ -0,0 +1,91 @@ +use futures::Future; +use reth_cli::chainspec::ChainSpecParser; +use reth_db::DatabaseEnv; +use reth_node_builder::{NodeBuilder, WithLaunchContext}; +use std::{fmt, sync::Arc}; + +/// A trait for launching a reth node with custom configuration strategies. +/// +/// This trait allows defining node configuration through various object types rather than just +/// functions. By implementing this trait on your own structures, you can: +/// +/// - Create flexible configurations that connect necessary components without creating separate +/// closures +/// - Take advantage of decomposition to break complex configurations into a series of methods +/// - Encapsulate configuration logic in dedicated types with their own state and behavior +/// - Reuse configuration patterns across different parts of your application +pub trait Launcher +where + C: ChainSpecParser, + Ext: clap::Args + fmt::Debug, +{ + /// Entry point for launching a node with custom configuration. + /// + /// Consumes `self` to use pre-configured state, takes a builder and arguments, + /// and returns an async future. + /// + /// # Arguments + /// + /// * `builder` - Node builder with launch context + /// * `builder_args` - Extension arguments for configuration + fn entrypoint( + self, + builder: WithLaunchContext, C::ChainSpec>>, + builder_args: Ext, + ) -> impl Future>; +} + +/// A function-based adapter implementation of the [`Launcher`] trait. +/// +/// This struct adapts existing closures to work with the new [`Launcher`] trait, +/// maintaining backward compatibility with current node implementations while +/// enabling the transition to the more flexible trait-based approach. +pub struct FnLauncher { + /// The function to execute when launching the node + func: F, +} + +impl FnLauncher { + /// Creates a new function launcher adapter. + /// + /// Type parameters `C` and `Ext` help the compiler infer correct types + /// since they're not stored in the struct itself. + /// + /// # Arguments + /// + /// * `func` - Function that configures and launches a node + pub fn new(func: F) -> Self + where + C: ChainSpecParser, + F: AsyncFnOnce( + WithLaunchContext, C::ChainSpec>>, + Ext, + ) -> eyre::Result<()>, + { + Self { func } + } +} + +impl fmt::Debug for FnLauncher { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("FnLauncher").field("func", &"").finish() + } +} + +impl Launcher for FnLauncher +where + C: ChainSpecParser, + Ext: clap::Args + fmt::Debug, + F: AsyncFnOnce( + WithLaunchContext, C::ChainSpec>>, + Ext, + ) -> eyre::Result<()>, +{ + fn entrypoint( + self, + builder: WithLaunchContext, C::ChainSpec>>, + builder_args: Ext, + ) -> impl Future> { + (self.func)(builder, builder_args) + } +} diff --git a/crates/cli/commands/src/lib.rs b/crates/cli/commands/src/lib.rs index 2789ad41bb7..88bd28ac9a9 100644 --- a/crates/cli/commands/src/lib.rs +++ b/crates/cli/commands/src/lib.rs @@ -6,21 +6,24 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] pub mod common; pub mod config_cmd; pub mod db; pub mod download; pub mod dump_genesis; +pub mod export_era; pub mod import; +pub mod import_core; pub mod import_era; pub mod init_cmd; pub mod init_state; +pub mod launcher; pub mod node; pub mod p2p; pub mod prune; -pub mod recover; +pub mod re_execute; pub mod stage; #[cfg(feature = "arbitrary")] pub mod test_vectors; diff --git a/crates/cli/commands/src/node.rs b/crates/cli/commands/src/node.rs index 5b8e0055fec..240bb3c2893 100644 --- a/crates/cli/commands/src/node.rs +++ b/crates/cli/commands/src/node.rs @@ -1,21 +1,21 @@ //! Main node command for launching a node +use crate::launcher::Launcher; use clap::{value_parser, Args, Parser}; use reth_chainspec::{EthChainSpec, EthereumHardforks}; use reth_cli::chainspec::ChainSpecParser; use reth_cli_runner::CliContext; -use reth_cli_util::parse_socket_address; -use reth_db::{init_db, DatabaseEnv}; -use reth_node_builder::{NodeBuilder, WithLaunchContext}; +use reth_db::init_db; +use reth_node_builder::NodeBuilder; use reth_node_core::{ args::{ - DatabaseArgs, DatadirArgs, DebugArgs, DevArgs, EngineArgs, NetworkArgs, PayloadBuilderArgs, - PruningArgs, RpcServerArgs, TxPoolArgs, + DatabaseArgs, DatadirArgs, DebugArgs, DevArgs, EngineArgs, EraArgs, MetricArgs, + NetworkArgs, PayloadBuilderArgs, PruningArgs, RpcServerArgs, TxPoolArgs, }, node_config::NodeConfig, version, }; -use std::{ffi::OsString, fmt, future::Future, net::SocketAddr, path::PathBuf, sync::Arc}; +use std::{ffi::OsString, fmt, path::PathBuf, sync::Arc}; /// Start the node #[derive(Debug, Parser)] @@ -31,18 +31,16 @@ pub struct NodeCommand, - /// Enable Prometheus metrics. - /// - /// The metrics will be served at the given interface and port. - #[arg(long, value_name = "SOCKET", value_parser = parse_socket_address, help_heading = "Metrics")] - pub metrics: Option, + /// Prometheus metrics configuration. + #[command(flatten)] + pub metrics: MetricArgs, /// Add a new instance of a node. /// @@ -58,7 +56,7 @@ pub struct NodeCommand, /// Sets all ports to unused, allowing the OS to choose random unused ports when sockets are @@ -108,6 +106,10 @@ pub struct NodeCommand(self, ctx: CliContext, launcher: L) -> eyre::Result<()> + /// launcher. + pub async fn execute(self, ctx: CliContext, launcher: L) -> eyre::Result<()> where - L: FnOnce(WithLaunchContext, C::ChainSpec>>, Ext) -> Fut, - Fut: Future>, + L: Launcher, { - tracing::info!(target: "reth::cli", version = ?version::SHORT_VERSION, "Starting reth"); + tracing::info!(target: "reth::cli", version = ?version::version_metadata().short_version, "Starting reth"); let Self { datadir, @@ -163,6 +164,7 @@ where pruning, ext, engine, + era, } = self; // set up node config @@ -181,6 +183,7 @@ where dev, pruning, engine, + era, }; let data_dir = node_config.datadir(); @@ -197,7 +200,7 @@ where .with_database(database) .with_launch_context(ctx.task_executor); - launcher(builder, ext).await + launcher.entrypoint(builder, ext).await } } @@ -207,6 +210,7 @@ impl NodeCommand { Some(&self.chain) } } + /// No Additional arguments #[derive(Debug, Clone, Copy, Default, Args)] #[non_exhaustive] @@ -218,7 +222,7 @@ mod tests { use reth_discv4::DEFAULT_DISCOVERY_PORT; use reth_ethereum_cli::chainspec::{EthereumChainSpecParser, SUPPORTED_CHAINS}; use std::{ - net::{IpAddr, Ipv4Addr}, + net::{IpAddr, Ipv4Addr, SocketAddr}, path::Path, }; @@ -279,15 +283,24 @@ mod tests { fn parse_metrics_port() { let cmd: NodeCommand = NodeCommand::try_parse_args_from(["reth", "--metrics", "9001"]).unwrap(); - assert_eq!(cmd.metrics, Some(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 9001))); + assert_eq!( + cmd.metrics.prometheus, + Some(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 9001)) + ); let cmd: NodeCommand = NodeCommand::try_parse_args_from(["reth", "--metrics", ":9001"]).unwrap(); - assert_eq!(cmd.metrics, Some(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 9001))); + assert_eq!( + cmd.metrics.prometheus, + Some(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 9001)) + ); let cmd: NodeCommand = NodeCommand::try_parse_args_from(["reth", "--metrics", "localhost:9001"]).unwrap(); - assert_eq!(cmd.metrics, Some(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 9001))); + assert_eq!( + cmd.metrics.prometheus, + Some(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 9001)) + ); } #[test] diff --git a/crates/cli/commands/src/p2p/bootnode.rs b/crates/cli/commands/src/p2p/bootnode.rs index 9be60aca658..8e4fb5ad2d3 100644 --- a/crates/cli/commands/src/p2p/bootnode.rs +++ b/crates/cli/commands/src/p2p/bootnode.rs @@ -1,29 +1,32 @@ //! Standalone bootnode command use clap::Parser; +use reth_cli_util::{get_secret_key, load_secret_key::rng_secret_key}; use reth_discv4::{DiscoveryUpdate, Discv4, Discv4Config}; use reth_discv5::{discv5::Event, Config, Discv5}; use reth_net_nat::NatResolver; use reth_network_peers::NodeRecord; -use std::{net::SocketAddr, str::FromStr}; +use secp256k1::SecretKey; +use std::{net::SocketAddr, path::PathBuf}; use tokio::select; use tokio_stream::StreamExt; use tracing::info; -/// Satrt a discovery only bootnode. +/// Start a discovery only bootnode. #[derive(Parser, Debug)] pub struct Command { - /// Listen address for the bootnode (default: ":30301"). - #[arg(long, default_value = ":30301")] - pub addr: String, + /// Listen address for the bootnode (default: "0.0.0.0:30301"). + #[arg(long, default_value = "0.0.0.0:30301")] + pub addr: SocketAddr, - /// Generate a new node key and save it to the specified file. - #[arg(long, default_value = "")] - pub gen_key: String, - - /// Private key filename for the node. - #[arg(long, default_value = "")] - pub node_key: String, + /// Secret key to use for the bootnode. + /// + /// This will also deterministically set the peer ID. + /// If a path is provided but no key exists at that path, + /// a new random secret will be generated and stored there. + /// If no path is specified, a new ephemeral random secret will be used. + #[arg(long, value_name = "PATH")] + pub p2p_secret_key: Option, /// NAT resolution method (any|none|upnp|publicip|extip:\) #[arg(long, default_value = "any")] @@ -37,17 +40,16 @@ pub struct Command { impl Command { /// Execute the bootnode command. pub async fn execute(self) -> eyre::Result<()> { - info!("Bootnode started with config: {:?}", self); - let sk = reth_network::config::rng_secret_key(); - let socket_addr = SocketAddr::from_str(&self.addr)?; - let local_enr = NodeRecord::from_secret_key(socket_addr, &sk); + info!("Bootnode started with config: {self:?}"); + + let sk = self.network_secret()?; + let local_enr = NodeRecord::from_secret_key(self.addr, &sk); let config = Discv4Config::builder().external_ip_resolver(Some(self.nat)).build(); - let (_discv4, mut discv4_service) = - Discv4::bind(socket_addr, local_enr, sk, config).await?; + let (_discv4, mut discv4_service) = Discv4::bind(self.addr, local_enr, sk, config).await?; - info!("Started discv4 at address:{:?}", socket_addr); + info!("Started discv4 at address: {local_enr:?}"); let mut discv4_updates = discv4_service.update_stream(); discv4_service.spawn(); @@ -57,7 +59,7 @@ impl Command { if self.v5 { info!("Starting discv5"); - let config = Config::builder(socket_addr).build(); + let config = Config::builder(self.addr).build(); let (_discv5, updates, _local_enr_discv5) = Discv5::start(&sk, config).await?; discv5_updates = Some(updates); }; @@ -104,4 +106,11 @@ impl Command { Ok(()) } + + fn network_secret(&self) -> eyre::Result { + match &self.p2p_secret_key { + Some(path) => Ok(get_secret_key(path)?), + None => Ok(rng_secret_key()), + } + } } diff --git a/crates/cli/commands/src/p2p/mod.rs b/crates/cli/commands/src/p2p/mod.rs index 5e4d31464b1..c72ceca78e6 100644 --- a/crates/cli/commands/src/p2p/mod.rs +++ b/crates/cli/commands/src/p2p/mod.rs @@ -2,6 +2,7 @@ use std::{path::PathBuf, sync::Arc}; +use crate::common::CliNodeTypes; use alloy_eips::BlockHashOrNumber; use backon::{ConstantBuilder, Retryable}; use clap::{Parser, Subcommand}; @@ -9,74 +10,162 @@ use reth_chainspec::{EthChainSpec, EthereumHardforks, Hardforks}; use reth_cli::chainspec::ChainSpecParser; use reth_cli_util::{get_secret_key, hash_or_num_value_parser}; use reth_config::Config; -use reth_network::{BlockDownloaderProvider, NetworkConfigBuilder, NetworkPrimitives}; +use reth_network::{BlockDownloaderProvider, NetworkConfigBuilder}; use reth_network_p2p::bodies::client::BodiesClient; use reth_node_core::{ - args::{DatabaseArgs, DatadirArgs, NetworkArgs}, + args::{DatadirArgs, NetworkArgs}, utils::get_single_header, }; pub mod bootnode; -mod rlpx; +pub mod rlpx; /// `reth p2p` command #[derive(Debug, Parser)] pub struct Command { - /// The path to the configuration file to use. - #[arg(long, value_name = "FILE", verbatim_doc_comment)] - config: Option, + #[command(subcommand)] + command: Subcommands, +} - /// The chain this node is running. - /// - /// Possible values are either a built-in chain or the path to a chain specification file. - #[arg( - long, - value_name = "CHAIN_OR_PATH", - long_help = C::help_message(), - default_value = C::SUPPORTED_CHAINS[0], - value_parser = C::parser() - )] - chain: Arc, +impl> Command { + /// Execute `p2p` command + pub async fn execute>(self) -> eyre::Result<()> { + match self.command { + Subcommands::Header { args, id } => { + let handle = args.launch_network::().await?; + let fetch_client = handle.fetch_client().await?; + let backoff = args.backoff(); - /// The number of retries per request - #[arg(long, default_value = "5")] - retries: usize, + let header = (move || get_single_header(fetch_client.clone(), id)) + .retry(backoff) + .notify(|err, _| tracing::warn!(target: "reth::cli", error = %err, "Error requesting header. Retrying...")) + .await?; + tracing::info!(target: "reth::cli", ?header, "Successfully downloaded header"); + } - #[command(flatten)] - network: NetworkArgs, + Subcommands::Body { args, id } => { + let handle = args.launch_network::().await?; + let fetch_client = handle.fetch_client().await?; + let backoff = args.backoff(); - #[command(flatten)] - datadir: DatadirArgs, + let hash = match id { + BlockHashOrNumber::Hash(hash) => hash, + BlockHashOrNumber::Number(number) => { + tracing::info!(target: "reth::cli", "Block number provided. Downloading header first..."); + let client = fetch_client.clone(); + let header = (move || { + get_single_header(client.clone(), BlockHashOrNumber::Number(number)) + }) + .retry(backoff) + .notify(|err, _| tracing::warn!(target: "reth::cli", error = %err, "Error requesting header. Retrying...")) + .await?; + header.hash() + } + }; + let (_, result) = (move || { + let client = fetch_client.clone(); + client.get_block_bodies(vec![hash]) + }) + .retry(backoff) + .notify(|err, _| tracing::warn!(target: "reth::cli", error = %err, "Error requesting block. Retrying...")) + .await? + .split(); + if result.len() != 1 { + eyre::bail!( + "Invalid number of headers received. Expected: 1. Received: {}", + result.len() + ) + } + let body = result.into_iter().next().unwrap(); + tracing::info!(target: "reth::cli", ?body, "Successfully downloaded body") + } + Subcommands::Rlpx(command) => { + command.execute().await?; + } + Subcommands::Bootnode(command) => { + command.execute().await?; + } + } - #[command(flatten)] - db: DatabaseArgs, + Ok(()) + } +} - #[command(subcommand)] - command: Subcommands, +impl Command { + /// Returns the underlying chain being used to run this command + pub fn chain_spec(&self) -> Option<&Arc> { + match &self.command { + Subcommands::Header { args, .. } => Some(&args.chain), + Subcommands::Body { args, .. } => Some(&args.chain), + Subcommands::Rlpx(_) => None, + Subcommands::Bootnode(_) => None, + } + } } /// `reth p2p` subcommands #[derive(Subcommand, Debug)] -pub enum Subcommands { +pub enum Subcommands { /// Download block header Header { + #[command(flatten)] + args: DownloadArgs, /// The header number or hash #[arg(value_parser = hash_or_num_value_parser)] id: BlockHashOrNumber, }, /// Download block body Body { + #[command(flatten)] + args: DownloadArgs, /// The block number or hash #[arg(value_parser = hash_or_num_value_parser)] id: BlockHashOrNumber, }, // RLPx utilities Rlpx(rlpx::Command), + /// Bootnode command + Bootnode(bootnode::Command), } -impl> Command { - /// Execute `p2p` command - pub async fn execute(self) -> eyre::Result<()> { +#[derive(Debug, Clone, Parser)] +pub struct DownloadArgs { + /// The number of retries per request + #[arg(long, default_value = "5")] + retries: usize, + + #[command(flatten)] + network: NetworkArgs, + + #[command(flatten)] + datadir: DatadirArgs, + + /// The path to the configuration file to use. + #[arg(long, value_name = "FILE", verbatim_doc_comment)] + config: Option, + + /// The chain this node is running. + /// + /// Possible values are either a built-in chain or the path to a chain specification file. + #[arg( + long, + value_name = "CHAIN_OR_PATH", + long_help = C::help_message(), + default_value = C::default_value(), + value_parser = C::parser() + )] + chain: Arc, +} + +impl DownloadArgs { + /// Creates and spawns the network and returns the handle. + pub async fn launch_network( + &self, + ) -> eyre::Result> + where + C::ChainSpec: EthChainSpec + Hardforks + EthereumHardforks + Send + Sync + 'static, + N: CliNodeTypes, + { let data_dir = self.datadir.clone().resolve_datadir(self.chain.chain()); let config_path = self.config.clone().unwrap_or_else(|| data_dir.config()); @@ -100,76 +189,42 @@ impl let rlpx_socket = (self.network.addr, self.network.port).into(); let boot_nodes = self.chain.bootnodes().unwrap_or_default(); - let net = NetworkConfigBuilder::::new(p2p_secret_key) + let net = NetworkConfigBuilder::::new(p2p_secret_key) .peer_config(config.peers_config_with_basic_nodes_from_file(None)) .external_ip_resolver(self.network.nat) - .disable_discv4_discovery_if(self.chain.chain().is_optimism()) + .network_id(self.network.network_id) .boot_nodes(boot_nodes.clone()) .apply(|builder| { self.network.discovery.apply_to_builder(builder, rlpx_socket, boot_nodes) }) - .build_with_noop_provider(self.chain) + .build_with_noop_provider(self.chain.clone()) .manager() .await?; - let network = net.handle().clone(); + let handle = net.handle().clone(); tokio::task::spawn(net); - let fetch_client = network.fetch_client().await?; - let retries = self.retries.max(1); - let backoff = ConstantBuilder::default().with_max_times(retries); - - match self.command { - Subcommands::Header { id } => { - let header = (move || get_single_header(fetch_client.clone(), id)) - .retry(backoff) - .notify(|err, _| println!("Error requesting header: {err}. Retrying...")) - .await?; - println!("Successfully downloaded header: {header:?}"); - } - Subcommands::Body { id } => { - let hash = match id { - BlockHashOrNumber::Hash(hash) => hash, - BlockHashOrNumber::Number(number) => { - println!("Block number provided. Downloading header first..."); - let client = fetch_client.clone(); - let header = (move || { - get_single_header(client.clone(), BlockHashOrNumber::Number(number)) - }) - .retry(backoff) - .notify(|err, _| println!("Error requesting header: {err}. Retrying...")) - .await?; - header.hash() - } - }; - let (_, result) = (move || { - let client = fetch_client.clone(); - client.get_block_bodies(vec![hash]) - }) - .retry(backoff) - .notify(|err, _| println!("Error requesting block: {err}. Retrying...")) - .await? - .split(); - if result.len() != 1 { - eyre::bail!( - "Invalid number of headers received. Expected: 1. Received: {}", - result.len() - ) - } - let body = result.into_iter().next().unwrap(); - println!("Successfully downloaded body: {body:?}") - } - Subcommands::Rlpx(command) => { - command.execute().await?; - } - } + Ok(handle) + } - Ok(()) + pub fn backoff(&self) -> ConstantBuilder { + ConstantBuilder::default().with_max_times(self.retries.max(1)) } } -impl Command { - /// Returns the underlying chain being used to run this command - pub fn chain_spec(&self) -> Option<&Arc> { - Some(&self.chain) +#[cfg(test)] +mod tests { + use super::*; + use reth_ethereum_cli::chainspec::EthereumChainSpecParser; + + #[test] + fn parse_header_cmd() { + let _args: Command = + Command::parse_from(["reth", "header", "--chain", "mainnet", "1000"]); + } + + #[test] + fn parse_body_cmd() { + let _args: Command = + Command::parse_from(["reth", "body", "--chain", "mainnet", "1000"]); } } diff --git a/crates/cli/commands/src/prune.rs b/crates/cli/commands/src/prune.rs index de60fbfdb3b..cae0fa00901 100644 --- a/crates/cli/commands/src/prune.rs +++ b/crates/cli/commands/src/prune.rs @@ -1,5 +1,5 @@ //! Command that runs pruning without any limits. -use crate::common::{AccessRights, CliNodeTypes, Environment, EnvironmentArgs}; +use crate::common::{AccessRights, CliNodeTypes, EnvironmentArgs}; use clap::Parser; use reth_chainspec::{EthChainSpec, EthereumHardforks}; use reth_cli::chainspec::ChainSpecParser; @@ -18,22 +18,23 @@ pub struct PruneCommand { impl> PruneCommand { /// Execute the `prune` command pub async fn execute>(self) -> eyre::Result<()> { - let Environment { config, provider_factory, .. } = self.env.init::(AccessRights::RW)?; - let prune_config = config.prune.unwrap_or_default(); + let env = self.env.init::(AccessRights::RW)?; + let provider_factory = env.provider_factory; + let config = env.config.prune; // Copy data from database to static files info!(target: "reth::cli", "Copying data from database to static files..."); let static_file_producer = - StaticFileProducer::new(provider_factory.clone(), prune_config.segments.clone()); + StaticFileProducer::new(provider_factory.clone(), config.segments.clone()); let lowest_static_file_height = static_file_producer.lock().copy_to_static_files()?.min_block_num(); info!(target: "reth::cli", ?lowest_static_file_height, "Copied data from database to static files"); // Delete data which has been copied to static files. if let Some(prune_tip) = lowest_static_file_height { - info!(target: "reth::cli", ?prune_tip, ?prune_config, "Pruning data from database..."); + info!(target: "reth::cli", ?prune_tip, ?config, "Pruning data from database..."); // Run the pruner according to the configuration, and don't enforce any limits on it - let mut pruner = PrunerBuilder::new(prune_config) + let mut pruner = PrunerBuilder::new(config) .delete_limit(usize::MAX) .build_with_provider_factory(provider_factory); diff --git a/crates/cli/commands/src/re_execute.rs b/crates/cli/commands/src/re_execute.rs new file mode 100644 index 00000000000..3b8ba305a42 --- /dev/null +++ b/crates/cli/commands/src/re_execute.rs @@ -0,0 +1,222 @@ +//! Re-execute blocks from database in parallel. + +use crate::common::{ + AccessRights, CliComponentsBuilder, CliNodeComponents, CliNodeTypes, Environment, + EnvironmentArgs, +}; +use alloy_consensus::{transaction::TxHashRef, BlockHeader, TxReceipt}; +use clap::Parser; +use eyre::WrapErr; +use reth_chainspec::{EthChainSpec, EthereumHardforks, Hardforks}; +use reth_cli::chainspec::ChainSpecParser; +use reth_consensus::FullConsensus; +use reth_evm::{execute::Executor, ConfigureEvm}; +use reth_primitives_traits::{format_gas_throughput, BlockBody, GotExpected}; +use reth_provider::{ + BlockNumReader, BlockReader, ChainSpecProvider, DatabaseProviderFactory, ReceiptProvider, + StaticFileProviderFactory, TransactionVariant, +}; +use reth_revm::database::StateProviderDatabase; +use reth_stages::stages::calculate_gas_used_from_headers; +use std::{ + sync::Arc, + time::{Duration, Instant}, +}; +use tokio::{sync::mpsc, task::JoinSet}; +use tracing::*; + +/// `reth re-execute` command +/// +/// Re-execute blocks in parallel to verify historical sync correctness. +#[derive(Debug, Parser)] +pub struct Command { + #[command(flatten)] + env: EnvironmentArgs, + + /// The height to start at. + #[arg(long, default_value = "1")] + from: u64, + + /// The height to end at. Defaults to the latest block. + #[arg(long)] + to: Option, + + /// Number of tasks to run in parallel + #[arg(long, default_value = "10")] + num_tasks: u64, +} + +impl Command { + /// Returns the underlying chain being used to run this command + pub fn chain_spec(&self) -> Option<&Arc> { + Some(&self.env.chain) + } +} + +impl> Command { + /// Execute `re-execute` command + pub async fn execute(self, components: impl CliComponentsBuilder) -> eyre::Result<()> + where + N: CliNodeTypes, + { + let Environment { provider_factory, .. } = self.env.init::(AccessRights::RO)?; + + let provider = provider_factory.database_provider_ro()?; + let components = components(provider_factory.chain_spec()); + + let min_block = self.from; + let max_block = self.to.unwrap_or(provider.best_block_number()?); + + let total_blocks = max_block - min_block; + let total_gas = calculate_gas_used_from_headers( + &provider_factory.static_file_provider(), + min_block..=max_block, + )?; + let blocks_per_task = total_blocks / self.num_tasks; + + let db_at = { + let provider_factory = provider_factory.clone(); + move |block_number: u64| { + StateProviderDatabase( + provider_factory.history_by_block_number(block_number).unwrap(), + ) + } + }; + + let (stats_tx, mut stats_rx) = mpsc::unbounded_channel(); + + let mut tasks = JoinSet::new(); + for i in 0..self.num_tasks { + let start_block = min_block + i * blocks_per_task; + let end_block = + if i == self.num_tasks - 1 { max_block } else { start_block + blocks_per_task }; + + // Spawn thread executing blocks + let provider_factory = provider_factory.clone(); + let evm_config = components.evm_config().clone(); + let consensus = components.consensus().clone(); + let db_at = db_at.clone(); + let stats_tx = stats_tx.clone(); + tasks.spawn_blocking(move || { + let mut executor = evm_config.batch_executor(db_at(start_block - 1)); + for block in start_block..end_block { + let block = provider_factory + .recovered_block(block.into(), TransactionVariant::NoHash)? + .unwrap(); + let result = executor.execute_one(&block)?; + + if let Err(err) = consensus + .validate_block_post_execution(&block, &result) + .wrap_err_with(|| format!("Failed to validate block {}", block.number())) + { + let correct_receipts = + provider_factory.receipts_by_block(block.number().into())?.unwrap(); + + for (i, (receipt, correct_receipt)) in + result.receipts.iter().zip(correct_receipts.iter()).enumerate() + { + if receipt != correct_receipt { + let tx_hash = block.body().transactions()[i].tx_hash(); + error!( + ?receipt, + ?correct_receipt, + index = i, + ?tx_hash, + "Invalid receipt" + ); + let expected_gas_used = correct_receipt.cumulative_gas_used() - + if i == 0 { + 0 + } else { + correct_receipts[i - 1].cumulative_gas_used() + }; + let got_gas_used = receipt.cumulative_gas_used() - + if i == 0 { + 0 + } else { + result.receipts[i - 1].cumulative_gas_used() + }; + if got_gas_used != expected_gas_used { + let mismatch = GotExpected { + expected: expected_gas_used, + got: got_gas_used, + }; + + error!(number=?block.number(), ?mismatch, "Gas usage mismatch"); + return Err(err); + } + } else { + continue; + } + } + + return Err(err); + } + let _ = stats_tx.send(block.gas_used()); + + // Reset DB once in a while to avoid OOM + if executor.size_hint() > 1_000_000 { + executor = evm_config.batch_executor(db_at(block.number())); + } + } + + eyre::Ok(()) + }); + } + + let instant = Instant::now(); + let mut total_executed_blocks = 0; + let mut total_executed_gas = 0; + + let mut last_logged_gas = 0; + let mut last_logged_blocks = 0; + let mut last_logged_time = Instant::now(); + + let mut interval = tokio::time::interval(Duration::from_secs(10)); + + loop { + tokio::select! { + Some(gas_used) = stats_rx.recv() => { + total_executed_blocks += 1; + total_executed_gas += gas_used; + } + result = tasks.join_next() => { + if let Some(result) = result { + if matches!(result, Err(_) | Ok(Err(_))) { + error!(?result); + return Err(eyre::eyre!("Re-execution failed: {result:?}")); + } + } else { + break; + } + } + _ = interval.tick() => { + let blocks_executed = total_executed_blocks - last_logged_blocks; + let gas_executed = total_executed_gas - last_logged_gas; + + if blocks_executed > 0 { + let progress = 100.0 * total_executed_gas as f64 / total_gas as f64; + info!( + throughput=?format_gas_throughput(gas_executed, last_logged_time.elapsed()), + progress=format!("{progress:.2}%"), + "Executed {blocks_executed} blocks" + ); + } + + last_logged_blocks = total_executed_blocks; + last_logged_gas = total_executed_gas; + last_logged_time = Instant::now(); + } + } + } + + info!( + start_block = min_block, + end_block = max_block, + throughput=?format_gas_throughput(total_executed_gas, instant.elapsed()), + "Re-executed successfully" + ); + + Ok(()) + } +} diff --git a/crates/cli/commands/src/recover/mod.rs b/crates/cli/commands/src/recover/mod.rs deleted file mode 100644 index dde0d6c448f..00000000000 --- a/crates/cli/commands/src/recover/mod.rs +++ /dev/null @@ -1,45 +0,0 @@ -//! `reth recover` command. - -use crate::common::CliNodeTypes; -use clap::{Parser, Subcommand}; -use reth_chainspec::{EthChainSpec, EthereumHardforks}; -use reth_cli::chainspec::ChainSpecParser; -use reth_cli_runner::CliContext; -use std::sync::Arc; - -mod storage_tries; - -/// `reth recover` command -#[derive(Debug, Parser)] -pub struct Command { - #[command(subcommand)] - command: Subcommands, -} - -/// `reth recover` subcommands -#[derive(Subcommand, Debug)] -pub enum Subcommands { - /// Recover the node by deleting dangling storage tries. - StorageTries(storage_tries::Command), -} - -impl> Command { - /// Execute `recover` command - pub async fn execute>( - self, - ctx: CliContext, - ) -> eyre::Result<()> { - match self.command { - Subcommands::StorageTries(command) => command.execute::(ctx).await, - } - } -} - -impl Command { - /// Returns the underlying chain being used to run this command - pub fn chain_spec(&self) -> Option<&Arc> { - match &self.command { - Subcommands::StorageTries(command) => command.chain_spec(), - } - } -} diff --git a/crates/cli/commands/src/recover/storage_tries.rs b/crates/cli/commands/src/recover/storage_tries.rs deleted file mode 100644 index 9974f2fd72c..00000000000 --- a/crates/cli/commands/src/recover/storage_tries.rs +++ /dev/null @@ -1,76 +0,0 @@ -use crate::common::{AccessRights, CliNodeTypes, Environment, EnvironmentArgs}; -use alloy_consensus::BlockHeader; -use clap::Parser; -use reth_chainspec::{EthChainSpec, EthereumHardforks}; -use reth_cli::chainspec::ChainSpecParser; -use reth_cli_runner::CliContext; -use reth_db_api::{ - cursor::{DbCursorRO, DbDupCursorRW}, - tables, - transaction::DbTx, -}; -use reth_provider::{BlockNumReader, HeaderProvider, ProviderError}; -use reth_trie::StateRoot; -use reth_trie_db::DatabaseStateRoot; -use std::sync::Arc; -use tracing::*; - -/// `reth recover storage-tries` command -#[derive(Debug, Parser)] -pub struct Command { - #[command(flatten)] - env: EnvironmentArgs, -} - -impl> Command { - /// Execute `storage-tries` recovery command - pub async fn execute>( - self, - _ctx: CliContext, - ) -> eyre::Result<()> { - let Environment { provider_factory, .. } = self.env.init::(AccessRights::RW)?; - - let mut provider = provider_factory.provider_rw()?; - let best_block = provider.best_block_number()?; - let best_header = provider - .sealed_header(best_block)? - .ok_or_else(|| ProviderError::HeaderNotFound(best_block.into()))?; - - let mut deleted_tries = 0; - let tx_mut = provider.tx_mut(); - let mut hashed_account_cursor = tx_mut.cursor_read::()?; - let mut storage_trie_cursor = tx_mut.cursor_dup_read::()?; - let mut entry = storage_trie_cursor.first()?; - - info!(target: "reth::cli", "Starting pruning of storage tries"); - while let Some((hashed_address, _)) = entry { - if hashed_account_cursor.seek_exact(hashed_address)?.is_none() { - deleted_tries += 1; - storage_trie_cursor.delete_current_duplicates()?; - } - - entry = storage_trie_cursor.next()?; - } - - let state_root = StateRoot::from_tx(tx_mut).root()?; - if state_root != best_header.state_root() { - eyre::bail!( - "Recovery failed. Incorrect state root. Expected: {:?}. Received: {:?}", - best_header.state_root(), - state_root - ); - } - - provider.commit()?; - info!(target: "reth::cli", deleted = deleted_tries, "Finished recovery"); - - Ok(()) - } -} - -impl Command { - /// Returns the underlying chain being used to run this command - pub fn chain_spec(&self) -> Option<&Arc> { - Some(&self.env.chain) - } -} diff --git a/crates/cli/commands/src/stage/drop.rs b/crates/cli/commands/src/stage/drop.rs index 1684264213d..2c6e911d7bd 100644 --- a/crates/cli/commands/src/stage/drop.rs +++ b/crates/cli/commands/src/stage/drop.rs @@ -15,9 +15,7 @@ use reth_db_common::{ }; use reth_node_api::{HeaderTy, ReceiptTy, TxTy}; use reth_node_core::args::StageEnum; -use reth_provider::{ - writer::UnifiedStorageWriter, DatabaseProviderFactory, StaticFileProviderFactory, -}; +use reth_provider::{DBProvider, DatabaseProviderFactory, StaticFileProviderFactory, TrieWriter}; use reth_prune::PruneSegment; use reth_stages::StageId; use reth_static_file_types::StaticFileSegment; @@ -72,7 +70,6 @@ impl Command { StageEnum::Headers => { tx.clear::()?; tx.clear::>>()?; - tx.clear::()?; tx.clear::()?; reset_stage_checkpoint(tx, StageId::Headers)?; @@ -81,7 +78,6 @@ impl Command { StageEnum::Bodies => { tx.clear::()?; tx.clear::>>()?; - reset_prune_checkpoint(tx, PruneSegment::Transactions)?; tx.clear::()?; tx.clear::>>()?; @@ -140,6 +136,10 @@ impl Command { None, )?; } + StageEnum::MerkleChangeSets => { + provider_rw.clear_trie_changesets()?; + reset_stage_checkpoint(tx, StageId::MerkleChangeSets)?; + } StageEnum::AccountHistory | StageEnum::StorageHistory => { tx.clear::()?; tx.clear::()?; @@ -160,7 +160,7 @@ impl Command { tx.put::(StageId::Finish.to_string(), Default::default())?; - UnifiedStorageWriter::commit_unwind(provider_rw)?; + provider_rw.commit()?; Ok(()) } diff --git a/crates/cli/commands/src/stage/dump/execution.rs b/crates/cli/commands/src/stage/dump/execution.rs index 921af75c78b..9e8e68e9800 100644 --- a/crates/cli/commands/src/stage/dump/execution.rs +++ b/crates/cli/commands/src/stage/dump/execution.rs @@ -69,13 +69,6 @@ fn import_tables_with_range( to, ) })??; - output_db.update(|tx| { - tx.import_table_with_range::( - &db_tool.provider_factory.db_ref().tx()?, - Some(from), - to, - ) - })??; output_db.update(|tx| { tx.import_table_with_range::( &db_tool.provider_factory.db_ref().tx()?, diff --git a/crates/cli/commands/src/stage/dump/merkle.rs b/crates/cli/commands/src/stage/dump/merkle.rs index 904d43dbade..1815b0b0348 100644 --- a/crates/cli/commands/src/stage/dump/merkle.rs +++ b/crates/cli/commands/src/stage/dump/merkle.rs @@ -1,12 +1,12 @@ use std::sync::Arc; use super::setup; -use alloy_primitives::BlockNumber; +use alloy_primitives::{Address, BlockNumber}; use eyre::Result; use reth_config::config::EtlConfig; use reth_consensus::{ConsensusError, FullConsensus}; use reth_db::DatabaseEnv; -use reth_db_api::{database::Database, table::TableImporter, tables}; +use reth_db_api::{database::Database, models::BlockNumberAddress, table::TableImporter, tables}; use reth_db_common::DbTool; use reth_evm::ConfigureEvm; use reth_exex::ExExManagerHandle; @@ -18,7 +18,7 @@ use reth_provider::{ use reth_stages::{ stages::{ AccountHashingStage, ExecutionStage, MerkleStage, StorageHashingStage, - MERKLE_STAGE_DEFAULT_CLEAN_THRESHOLD, + MERKLE_STAGE_DEFAULT_REBUILD_THRESHOLD, }, ExecutionStageThresholds, Stage, StageCheckpoint, UnwindInput, }; @@ -92,10 +92,8 @@ fn unwind_and_copy( reth_stages::ExecInput { target: Some(to), checkpoint: Some(StageCheckpoint::new(from)) }; // Unwind hashes all the way to FROM - - StorageHashingStage::default().unwind(&provider, unwind).unwrap(); - AccountHashingStage::default().unwind(&provider, unwind).unwrap(); - + StorageHashingStage::default().unwind(&provider, unwind)?; + AccountHashingStage::default().unwind(&provider, unwind)?; MerkleStage::default_unwind().unwind(&provider, unwind)?; // Bring Plainstate to TO (hashing stage execution requires it) @@ -108,7 +106,7 @@ fn unwind_and_copy( max_cumulative_gas: None, max_duration: None, }, - MERKLE_STAGE_DEFAULT_CLEAN_THRESHOLD, + MERKLE_STAGE_DEFAULT_REBUILD_THRESHOLD, ExExManagerHandle::empty(), ); @@ -127,21 +125,23 @@ fn unwind_and_copy( commit_threshold: u64::MAX, etl_config: EtlConfig::default(), } - .execute(&provider, execute_input) - .unwrap(); + .execute(&provider, execute_input)?; StorageHashingStage { clean_threshold: u64::MAX, commit_threshold: u64::MAX, etl_config: EtlConfig::default(), } - .execute(&provider, execute_input) - .unwrap(); + .execute(&provider, execute_input)?; let unwind_inner_tx = provider.into_tx(); - // TODO optimize we can actually just get the entries we need - output_db - .update(|tx| tx.import_dupsort::(&unwind_inner_tx))??; + output_db.update(|tx| { + tx.import_table_with_range::( + &unwind_inner_tx, + Some(BlockNumberAddress((from, Address::ZERO))), + BlockNumberAddress((to, Address::repeat_byte(0xff))), + ) + })??; output_db.update(|tx| tx.import_table::(&unwind_inner_tx))??; output_db.update(|tx| tx.import_dupsort::(&unwind_inner_tx))??; @@ -161,7 +161,8 @@ where let mut stage = MerkleStage::Execution { // Forces updating the root instead of calculating from scratch - clean_threshold: u64::MAX, + rebuild_threshold: u64::MAX, + incremental_threshold: u64::MAX, }; loop { diff --git a/crates/cli/commands/src/stage/mod.rs b/crates/cli/commands/src/stage/mod.rs index 09d6ea9c091..129a84733f6 100644 --- a/crates/cli/commands/src/stage/mod.rs +++ b/crates/cli/commands/src/stage/mod.rs @@ -7,7 +7,6 @@ use clap::{Parser, Subcommand}; use reth_chainspec::{EthChainSpec, EthereumHardforks, Hardforks}; use reth_cli::chainspec::ChainSpecParser; use reth_cli_runner::CliContext; -use reth_eth_wire::NetPrimitivesFor; pub mod drop; pub mod dump; @@ -18,7 +17,7 @@ pub mod unwind; #[derive(Debug, Parser)] pub struct Command { #[command(subcommand)] - command: Subcommands, + pub command: Subcommands, } /// `reth stage` subcommands @@ -41,15 +40,17 @@ pub enum Subcommands { impl> Command { /// Execute `stage` command - pub async fn execute(self, ctx: CliContext, components: F) -> eyre::Result<()> + pub async fn execute( + self, + ctx: CliContext, + components: impl FnOnce(Arc) -> Comp, + ) -> eyre::Result<()> where N: CliNodeTypes, Comp: CliNodeComponents, - F: FnOnce(Arc) -> Comp, - P: NetPrimitivesFor, { match self.command { - Subcommands::Run(command) => command.execute::(ctx, components).await, + Subcommands::Run(command) => command.execute::(ctx, components).await, Subcommands::Drop(command) => command.execute::().await, Subcommands::Dump(command) => command.execute::(components).await, Subcommands::Unwind(command) => command.execute::(components).await, diff --git a/crates/cli/commands/src/stage/run.rs b/crates/cli/commands/src/stage/run.rs index e21f3996edc..f25338d30ef 100644 --- a/crates/cli/commands/src/stage/run.rs +++ b/crates/cli/commands/src/stage/run.rs @@ -16,16 +16,12 @@ use reth_downloaders::{ bodies::bodies::BodiesDownloaderBuilder, headers::reverse_headers::ReverseHeadersDownloaderBuilder, }; -use reth_eth_wire::NetPrimitivesFor; use reth_exex::ExExManagerHandle; use reth_network::BlockDownloaderProvider; use reth_network_p2p::HeadersClient; use reth_node_core::{ args::{NetworkArgs, StageEnum}, - version::{ - BUILD_PROFILE_NAME, CARGO_PKG_VERSION, VERGEN_BUILD_TIMESTAMP, VERGEN_CARGO_FEATURES, - VERGEN_CARGO_TARGET_TRIPLE, VERGEN_GIT_SHA, - }, + version::version_metadata, }; use reth_node_metrics::{ chain::ChainSpecInfo, @@ -34,8 +30,8 @@ use reth_node_metrics::{ version::VersionInfo, }; use reth_provider::{ - writer::UnifiedStorageWriter, ChainSpecProvider, DatabaseProviderFactory, - StageCheckpointReader, StageCheckpointWriter, StaticFileProviderFactory, + ChainSpecProvider, DBProvider, DatabaseProviderFactory, StageCheckpointReader, + StageCheckpointWriter, StaticFileProviderFactory, }; use reth_stages::{ stages::{ @@ -103,12 +99,11 @@ pub struct Command { impl> Command { /// Execute `stage` command - pub async fn execute(self, ctx: CliContext, components: F) -> eyre::Result<()> + pub async fn execute(self, ctx: CliContext, components: F) -> eyre::Result<()> where N: CliNodeTypes, Comp: CliNodeComponents, F: FnOnce(Arc) -> Comp, - P: NetPrimitivesFor, { // Raise the fd limit of the process. // Does not do anything on windows. @@ -121,16 +116,15 @@ impl let components = components(provider_factory.chain_spec()); if let Some(listen_addr) = self.metrics { - info!(target: "reth::cli", "Starting metrics endpoint at {}", listen_addr); let config = MetricServerConfig::new( listen_addr, VersionInfo { - version: CARGO_PKG_VERSION, - build_timestamp: VERGEN_BUILD_TIMESTAMP, - cargo_features: VERGEN_CARGO_FEATURES, - git_sha: VERGEN_GIT_SHA, - target_triple: VERGEN_CARGO_TARGET_TRIPLE, - build_profile: BUILD_PROFILE_NAME, + version: version_metadata().cargo_pkg_version.as_ref(), + build_timestamp: version_metadata().vergen_build_timestamp.as_ref(), + cargo_features: version_metadata().vergen_cargo_features.as_ref(), + git_sha: version_metadata().vergen_git_sha.as_ref(), + target_triple: version_metadata().vergen_cargo_target_triple.as_ref(), + build_profile: version_metadata().build_profile_name.as_ref(), }, ChainSpecInfo { name: provider_factory.chain_spec().chain().to_string() }, ctx.task_executor, @@ -156,7 +150,7 @@ impl let batch_size = self.batch_size.unwrap_or(self.to.saturating_sub(self.from) + 1); let etl_config = config.stages.etl.clone(); - let prune_modes = config.prune.clone().map(|prune| prune.segments).unwrap_or_default(); + let prune_modes = config.prune.segments.clone(); let (mut exec_stage, mut unwind_stage): (Box>, Option>>) = match self.stage { @@ -174,7 +168,7 @@ impl let network = self .network - .network_config::

( + .network_config::( &config, provider_factory.chain_spec(), p2p_secret_key, @@ -186,7 +180,7 @@ impl let fetch_client = Arc::new(network.fetch_client().await?); // Use `to` as the tip for the stage - let tip: P::BlockHeader = loop { + let tip = loop { match fetch_client.get_header(BlockHashOrNumber::Number(self.to)).await { Ok(header) => { if let Some(header) = header.into_data() { @@ -229,7 +223,7 @@ impl let network = self .network - .network_config::

( + .network_config::( &config, provider_factory.chain_spec(), p2p_secret_key, @@ -271,7 +265,7 @@ impl max_cumulative_gas: None, max_duration: None, }, - config.stages.merkle.clean_threshold, + config.stages.merkle.incremental_threshold, ExExManagerHandle::empty(), )), None, @@ -299,7 +293,10 @@ impl None, ), StageEnum::Merkle => ( - Box::new(MerkleStage::new_execution(config.stages.merkle.clean_threshold)), + Box::new(MerkleStage::new_execution( + config.stages.merkle.rebuild_threshold, + config.stages.merkle.incremental_threshold, + )), Some(Box::new(MerkleStage::default_unwind())), ), StageEnum::AccountHistory => ( @@ -344,7 +341,7 @@ impl } if self.commit { - UnifiedStorageWriter::commit_unwind(provider_rw)?; + provider_rw.commit()?; provider_rw = provider_factory.database_provider_rw()?; } } @@ -367,7 +364,7 @@ impl provider_rw.save_stage_checkpoint(exec_stage.id(), checkpoint)?; } if self.commit { - UnifiedStorageWriter::commit(provider_rw)?; + provider_rw.commit()?; provider_rw = provider_factory.database_provider_rw()?; } diff --git a/crates/cli/commands/src/stage/unwind.rs b/crates/cli/commands/src/stage/unwind.rs index 0218c2c3bb7..ffd8e330062 100644 --- a/crates/cli/commands/src/stage/unwind.rs +++ b/crates/cli/commands/src/stage/unwind.rs @@ -15,10 +15,7 @@ use reth_db::DatabaseEnv; use reth_downloaders::{bodies::noop::NoopBodiesDownloader, headers::noop::NoopHeaderDownloader}; use reth_evm::ConfigureEvm; use reth_exex::ExExManagerHandle; -use reth_provider::{ - providers::ProviderNodeTypes, BlockExecutionWriter, BlockNumReader, ChainStateBlockReader, - ChainStateBlockWriter, ProviderFactory, StaticFileProviderFactory, StorageLocation, -}; +use reth_provider::{providers::ProviderNodeTypes, BlockNumReader, ProviderFactory}; use reth_stages::{ sets::{DefaultStages, OfflineStages}, stages::ExecutionStage, @@ -60,53 +57,21 @@ impl> Command let components = components(provider_factory.chain_spec()); - let highest_static_file_block = provider_factory - .static_file_provider() - .get_highest_static_files() - .max_block_num() - .filter(|highest_static_file_block| *highest_static_file_block > target); - - // Execute a pipeline unwind if the start of the range overlaps the existing static - // files. If that's the case, then copy all available data from MDBX to static files, and - // only then, proceed with the unwind. - // - // We also execute a pipeline unwind if `offline` is specified, because we need to only - // unwind the data associated with offline stages. - if highest_static_file_block.is_some() || self.offline { - if self.offline { - info!(target: "reth::cli", "Performing an unwind for offline-only data!"); - } - - if let Some(highest_static_file_block) = highest_static_file_block { - info!(target: "reth::cli", ?target, ?highest_static_file_block, "Executing a pipeline unwind."); - } else { - info!(target: "reth::cli", ?target, "Executing a pipeline unwind."); - } - - // This will build an offline-only pipeline if the `offline` flag is enabled - let mut pipeline = - self.build_pipeline(config, provider_factory, components.evm_config().clone())?; - - // Move all applicable data from database to static files. - pipeline.move_to_static_files()?; - - pipeline.unwind(target, None)?; - } else { - info!(target: "reth::cli", ?target, "Executing a database unwind."); - let provider = provider_factory.provider_rw()?; + if self.offline { + info!(target: "reth::cli", "Performing an unwind for offline-only data!"); + } - provider - .remove_block_and_execution_above(target, StorageLocation::Both) - .map_err(|err| eyre::eyre!("Transaction error on unwind: {err}"))?; + let highest_static_file_block = provider_factory.provider()?.last_block_number()?; + info!(target: "reth::cli", ?target, ?highest_static_file_block, prune_config=?config.prune, "Executing a pipeline unwind."); - // update finalized block if needed - let last_saved_finalized_block_number = provider.last_finalized_block_number()?; - if last_saved_finalized_block_number.is_none_or(|f| f > target) { - provider.save_finalized_block_number(target)?; - } + // This will build an offline-only pipeline if the `offline` flag is enabled + let mut pipeline = + self.build_pipeline(config, provider_factory, components.evm_config().clone())?; - provider.commit()?; - } + // Move all applicable data from database to static files. + pipeline.move_to_static_files()?; + + pipeline.unwind(target, None)?; info!(target: "reth::cli", ?target, "Unwound blocks"); @@ -120,7 +85,7 @@ impl> Command evm_config: impl ConfigureEvm + 'static, ) -> Result, eyre::Error> { let stage_conf = &config.stages; - let prune_modes = config.prune.clone().map(|prune| prune.segments).unwrap_or_default(); + let prune_modes = config.prune.segments.clone(); let (tip_tx, tip_rx) = watch::channel(B256::ZERO); @@ -146,6 +111,7 @@ impl> Command evm_config.clone(), stage_conf.clone(), prune_modes.clone(), + None, ) .set(ExecutionStage::new( evm_config, @@ -208,7 +174,9 @@ impl Subcommands { Self::NumBlocks { amount } => last.saturating_sub(*amount), }; if target > last { - eyre::bail!("Target block number is higher than the latest block number") + eyre::bail!( + "Target block number {target} is higher than the latest block number {last}" + ) } Ok(target) } @@ -216,9 +184,9 @@ impl Subcommands { #[cfg(test)] mod tests { - use reth_ethereum_cli::chainspec::EthereumChainSpecParser; - use super::*; + use reth_chainspec::SEPOLIA; + use reth_ethereum_cli::chainspec::EthereumChainSpecParser; #[test] fn parse_unwind() { @@ -240,4 +208,13 @@ mod tests { ]); assert_eq!(cmd.command, Subcommands::NumBlocks { amount: 100 }); } + + #[test] + fn parse_unwind_chain() { + let cmd = Command::::parse_from([ + "reth", "--chain", "sepolia", "to-block", "100", + ]); + assert_eq!(cmd.command, Subcommands::ToBlock { target: BlockHashOrNumber::Number(100) }); + assert_eq!(cmd.env.chain.chain_id(), SEPOLIA.chain_id()); + } } diff --git a/crates/cli/commands/src/test_vectors/compact.rs b/crates/cli/commands/src/test_vectors/compact.rs index abd3635e4fd..f4636f5f83b 100644 --- a/crates/cli/commands/src/test_vectors/compact.rs +++ b/crates/cli/commands/src/test_vectors/compact.rs @@ -94,7 +94,6 @@ compact_types!( Log, // BranchNodeCompact, // todo requires arbitrary TrieMask, - // TxDeposit, TODO(joshie): optimism // reth_prune_types PruneCheckpoint, PruneMode, @@ -284,7 +283,7 @@ pub fn type_name() -> String { // With alloy type transition the types are renamed, we map them here to the original name so that test vector files remain consistent let name = std::any::type_name::(); match name { - "alloy_consensus::transaction::typed::EthereumTypedTransaction" => "Transaction".to_string(), + "alloy_consensus::transaction::envelope::EthereumTypedTransaction" => "Transaction".to_string(), "alloy_consensus::transaction::envelope::EthereumTxEnvelope" => "TransactionSigned".to_string(), name => { name.split("::").last().unwrap_or(std::any::type_name::()).to_string() diff --git a/crates/cli/commands/src/test_vectors/tables.rs b/crates/cli/commands/src/test_vectors/tables.rs index 1bbd2604f97..10b94695399 100644 --- a/crates/cli/commands/src/test_vectors/tables.rs +++ b/crates/cli/commands/src/test_vectors/tables.rs @@ -54,7 +54,7 @@ pub fn generate_vectors(mut tables: Vec) -> Result<()> { match table.as_str() { $( stringify!($table_type) => { - println!("Generating test vectors for {} <{}>.", stringify!($table_or_dup), tables::$table_type$(::<$($generic),+>)?::NAME); + tracing::info!(target: "reth::cli", "Generating test vectors for {} <{}>.", stringify!($table_or_dup), tables::$table_type$(::<$($generic),+>)?::NAME); generate_vector!($table_type$(<$($generic),+>)?, $per_table, $table_or_dup); }, @@ -69,7 +69,6 @@ pub fn generate_vectors(mut tables: Vec) -> Result<()> { generate!([ (CanonicalHeaders, PER_TABLE, TABLE), - (HeaderTerminalDifficulties, PER_TABLE, TABLE), (HeaderNumbers, PER_TABLE, TABLE), (Headers

, PER_TABLE, TABLE), (BlockBodyIndices, PER_TABLE, TABLE), diff --git a/crates/cli/runner/src/lib.rs b/crates/cli/runner/src/lib.rs index 48caf171eea..79dc6b21142 100644 --- a/crates/cli/runner/src/lib.rs +++ b/crates/cli/runner/src/lib.rs @@ -6,7 +6,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] //! Entrypoint for running commands. @@ -36,11 +36,15 @@ impl CliRunner { pub const fn from_runtime(tokio_runtime: tokio::runtime::Runtime) -> Self { Self { tokio_runtime } } -} -// === impl CliRunner === + /// Executes an async block on the runtime and blocks until completion. + pub fn block_on(&self, fut: F) -> T + where + F: Future, + { + self.tokio_runtime.block_on(fut) + } -impl CliRunner { /// Executes the given _async_ command on the tokio runtime until the command future resolves or /// until the process receives a `SIGINT` or `SIGTERM` signal. /// @@ -99,8 +103,7 @@ impl CliRunner { F: Future>, E: Send + Sync + From + 'static, { - let tokio_runtime = tokio_runtime()?; - tokio_runtime.block_on(run_until_ctrl_c(fut))?; + self.tokio_runtime.block_on(run_until_ctrl_c(fut))?; Ok(()) } @@ -113,7 +116,7 @@ impl CliRunner { F: Future> + Send + 'static, E: Send + Sync + From + 'static, { - let tokio_runtime = tokio_runtime()?; + let tokio_runtime = self.tokio_runtime; let handle = tokio_runtime.handle().clone(); let fut = tokio_runtime.handle().spawn_blocking(move || handle.block_on(fut)); tokio_runtime @@ -174,8 +177,10 @@ where { let fut = pin!(fut); tokio::select! { - err = tasks => { - return Err(err.into()) + task_manager_result = tasks => { + if let Err(panicked_error) = task_manager_result { + return Err(panicked_error.into()); + } }, res = fut => res?, } diff --git a/crates/cli/util/src/lib.rs b/crates/cli/util/src/lib.rs index a82c3ba57f7..7e0d69c1868 100644 --- a/crates/cli/util/src/lib.rs +++ b/crates/cli/util/src/lib.rs @@ -6,7 +6,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] pub mod allocator; diff --git a/crates/cli/util/src/sigsegv_handler.rs b/crates/cli/util/src/sigsegv_handler.rs index b0a195391ff..dabbf866cee 100644 --- a/crates/cli/util/src/sigsegv_handler.rs +++ b/crates/cli/util/src/sigsegv_handler.rs @@ -7,7 +7,7 @@ use std::{ fmt, mem, ptr, }; -extern "C" { +unsafe extern "C" { fn backtrace_symbols_fd(buffer: *const *mut libc::c_void, size: libc::c_int, fd: libc::c_int); } diff --git a/crates/config/Cargo.toml b/crates/config/Cargo.toml index 25d42ac9022..65bca139012 100644 --- a/crates/config/Cargo.toml +++ b/crates/config/Cargo.toml @@ -24,6 +24,9 @@ humantime-serde = { workspace = true, optional = true } toml = { workspace = true, optional = true } eyre = { workspace = true, optional = true } +# value objects +url.workspace = true + [features] serde = [ "dep:serde", @@ -34,6 +37,7 @@ serde = [ "reth-prune-types/serde", "reth-stages-types/serde", "alloy-primitives/serde", + "url/serde", ] [dev-dependencies] diff --git a/crates/config/src/config.rs b/crates/config/src/config.rs index ad6bdb6ef04..5ff2431bb56 100644 --- a/crates/config/src/config.rs +++ b/crates/config/src/config.rs @@ -6,6 +6,7 @@ use std::{ path::{Path, PathBuf}, time::Duration, }; +use url::Url; #[cfg(feature = "serde")] const EXTENSION: &str = "toml"; @@ -22,8 +23,8 @@ pub struct Config { // TODO(onbjerg): Can we make this easier to maintain when we add/remove stages? pub stages: StageConfig, /// Configuration for pruning. - #[cfg_attr(feature = "serde", serde(skip_serializing_if = "Option::is_none"))] - pub prune: Option, + #[cfg_attr(feature = "serde", serde(default))] + pub prune: PruneConfig, /// Configuration for the discovery service. pub peers: PeersConfig, /// Configuration for peer sessions. @@ -32,8 +33,8 @@ pub struct Config { impl Config { /// Sets the pruning configuration. - pub fn update_prune_config(&mut self, prune_config: PruneConfig) { - self.prune = Some(prune_config); + pub fn set_prune_config(&mut self, prune_config: PruneConfig) { + self.prune = prune_config; } } @@ -100,6 +101,8 @@ impl Config { #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[cfg_attr(feature = "serde", serde(default))] pub struct StageConfig { + /// ERA stage configuration. + pub era: EraConfig, /// Header stage configuration. pub headers: HeadersConfig, /// Body stage configuration. @@ -133,12 +136,39 @@ impl StageConfig { /// `ExecutionStage` pub fn execution_external_clean_threshold(&self) -> u64 { self.merkle - .clean_threshold + .incremental_threshold .max(self.account_hashing.clean_threshold) .max(self.storage_hashing.clean_threshold) } } +/// ERA stage configuration. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(default))] +pub struct EraConfig { + /// Path to a local directory where ERA1 files are located. + /// + /// Conflicts with `url`. + pub path: Option, + /// The base URL of an ERA1 file host to download from. + /// + /// Conflicts with `path`. + pub url: Option, + /// Path to a directory where files downloaded from `url` will be stored until processed. + /// + /// Required for `url`. + pub folder: Option, +} + +impl EraConfig { + /// Sets `folder` for temporary downloads as a directory called "era" inside `dir`. + pub fn with_datadir(mut self, dir: impl AsRef) -> Self { + self.folder = Some(dir.as_ref().join("era")); + self + } +} + /// Header stage configuration. #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] @@ -312,14 +342,22 @@ impl Default for HashingConfig { #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[cfg_attr(feature = "serde", serde(default))] pub struct MerkleConfig { + /// The number of blocks we will run the incremental root method for when we are catching up on + /// the merkle stage for a large number of blocks. + /// + /// When we are catching up for a large number of blocks, we can only run the incremental root + /// for a limited number of blocks, otherwise the incremental root method may cause the node to + /// OOM. This number determines how many blocks in a row we will run the incremental root + /// method for. + pub incremental_threshold: u64, /// The threshold (in number of blocks) for switching from incremental trie building of changes /// to whole rebuild. - pub clean_threshold: u64, + pub rebuild_threshold: u64, } impl Default for MerkleConfig { fn default() -> Self { - Self { clean_threshold: 5_000 } + Self { incremental_threshold: 7_000, rebuild_threshold: 100_000 } } } @@ -402,11 +440,16 @@ pub struct PruneConfig { impl Default for PruneConfig { fn default() -> Self { - Self { block_interval: DEFAULT_BLOCK_INTERVAL, segments: PruneModes::none() } + Self { block_interval: DEFAULT_BLOCK_INTERVAL, segments: PruneModes::default() } } } impl PruneConfig { + /// Returns whether this configuration is the default one. + pub fn is_default(&self) -> bool { + self == &Self::default() + } + /// Returns whether there is any kind of receipt pruning configuration. pub fn has_receipts_pruning(&self) -> bool { self.segments.receipts.is_some() || !self.segments.receipts_log_filter.is_empty() @@ -414,8 +457,7 @@ impl PruneConfig { /// Merges another `PruneConfig` into this one, taking values from the other config if and only /// if the corresponding value in this config is not set. - pub fn merge(&mut self, other: Option) { - let Some(other) = other else { return }; + pub fn merge(&mut self, other: Self) { let Self { block_interval, segments: @@ -425,6 +467,8 @@ impl PruneConfig { receipts, account_history, storage_history, + bodies_history, + merkle_changesets, receipts_log_filter, }, } = other; @@ -440,6 +484,9 @@ impl PruneConfig { self.segments.receipts = self.segments.receipts.or(receipts); self.segments.account_history = self.segments.account_history.or(account_history); self.segments.storage_history = self.segments.storage_history.or(storage_history); + self.segments.bodies_history = self.segments.bodies_history.or(bodies_history); + // Merkle changesets is not optional, so we just replace it if provided + self.segments.merkle_changesets = merkle_changesets; if self.segments.receipts_log_filter.0.is_empty() && !receipts_log_filter.0.is_empty() { self.segments.receipts_log_filter = receipts_log_filter; @@ -960,6 +1007,8 @@ receipts = 'full' receipts: Some(PruneMode::Distance(1000)), account_history: None, storage_history: Some(PruneMode::Before(5000)), + bodies_history: None, + merkle_changesets: PruneMode::Before(0), receipts_log_filter: ReceiptsLogPruneConfig(BTreeMap::from([( Address::random(), PruneMode::Full, @@ -975,6 +1024,8 @@ receipts = 'full' receipts: Some(PruneMode::Full), account_history: Some(PruneMode::Distance(2000)), storage_history: Some(PruneMode::Distance(3000)), + bodies_history: None, + merkle_changesets: PruneMode::Distance(10000), receipts_log_filter: ReceiptsLogPruneConfig(BTreeMap::from([ (Address::random(), PruneMode::Distance(1000)), (Address::random(), PruneMode::Before(2000)), @@ -983,7 +1034,7 @@ receipts = 'full' }; let original_filter = config1.segments.receipts_log_filter.clone(); - config1.merge(Some(config2)); + config1.merge(config2); // Check that the configuration has been merged. Any configuration present in config1 // should not be overwritten by config2 @@ -993,6 +1044,7 @@ receipts = 'full' assert_eq!(config1.segments.receipts, Some(PruneMode::Distance(1000))); assert_eq!(config1.segments.account_history, Some(PruneMode::Distance(2000))); assert_eq!(config1.segments.storage_history, Some(PruneMode::Before(5000))); + assert_eq!(config1.segments.merkle_changesets, PruneMode::Distance(10000)); assert_eq!(config1.segments.receipts_log_filter, original_filter); } diff --git a/crates/config/src/lib.rs b/crates/config/src/lib.rs index 1e81e18ec42..df2dd6ec5b2 100644 --- a/crates/config/src/lib.rs +++ b/crates/config/src/lib.rs @@ -6,7 +6,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] pub mod config; pub use config::{BodiesConfig, Config, PruneConfig}; diff --git a/crates/consensus/common/Cargo.toml b/crates/consensus/common/Cargo.toml index 96dea6c232f..901e8697cd5 100644 --- a/crates/consensus/common/Cargo.toml +++ b/crates/consensus/common/Cargo.toml @@ -23,7 +23,6 @@ alloy-eips.workspace = true [dev-dependencies] alloy-primitives = { workspace = true, features = ["rand"] } reth-ethereum-primitives.workspace = true -alloy-consensus.workspace = true rand.workspace = true [features] diff --git a/crates/consensus/common/src/lib.rs b/crates/consensus/common/src/lib.rs index b6971a0d528..cff441c3e96 100644 --- a/crates/consensus/common/src/lib.rs +++ b/crates/consensus/common/src/lib.rs @@ -6,7 +6,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(feature = "std"), no_std)] /// Collection of consensus validation methods. diff --git a/crates/consensus/common/src/validation.rs b/crates/consensus/common/src/validation.rs index a2a7e14f8dd..e14a3164279 100644 --- a/crates/consensus/common/src/validation.rs +++ b/crates/consensus/common/src/validation.rs @@ -1,16 +1,26 @@ //! Collection of methods for block validation. use alloy_consensus::{ - constants::MAXIMUM_EXTRA_DATA_SIZE, BlockHeader as _, EMPTY_OMMER_ROOT_HASH, + constants::MAXIMUM_EXTRA_DATA_SIZE, BlockHeader as _, Transaction, EMPTY_OMMER_ROOT_HASH, }; use alloy_eips::{eip4844::DATA_GAS_PER_BLOB, eip7840::BlobParams}; -use reth_chainspec::{EthChainSpec, EthereumHardforks}; -use reth_consensus::ConsensusError; +use reth_chainspec::{EthChainSpec, EthereumHardfork, EthereumHardforks}; +use reth_consensus::{ConsensusError, TxGasLimitTooHighErr}; use reth_primitives_traits::{ - constants::MAXIMUM_GAS_LIMIT_BLOCK, Block, BlockBody, BlockHeader, GotExpected, SealedBlock, - SealedHeader, + constants::{ + GAS_LIMIT_BOUND_DIVISOR, MAXIMUM_GAS_LIMIT_BLOCK, MAX_TX_GAS_LIMIT_OSAKA, MINIMUM_GAS_LIMIT, + }, + transaction::TxHashRef, + Block, BlockBody, BlockHeader, GotExpected, SealedBlock, SealedHeader, }; +/// The maximum RLP length of a block, defined in [EIP-7934](https://eips.ethereum.org/EIPS/eip-7934). +/// +/// Calculated as `MAX_BLOCK_SIZE` - `SAFETY_MARGIN` where +/// `MAX_BLOCK_SIZE` = `10_485_760` +/// `SAFETY_MARGIN` = `2_097_152` +pub const MAX_RLP_BLOCK_SIZE: usize = 8_388_608; + /// Gas used needs to be less than gas limit. Gas used is going to be checked after execution. #[inline] pub fn validate_header_gas(header: &H) -> Result<(), ConsensusError> { @@ -132,11 +142,49 @@ where /// - Compares the ommer hash in the block header to the block body /// - Compares the transactions root in the block header to the block body /// - Pre-execution transaction validation -/// - (Optionally) Compares the receipts root in the block header to the block body pub fn validate_block_pre_execution( block: &SealedBlock, chain_spec: &ChainSpec, ) -> Result<(), ConsensusError> +where + B: Block, + ChainSpec: EthereumHardforks, +{ + post_merge_hardfork_fields(block, chain_spec)?; + + // Check transaction root + if let Err(error) = block.ensure_transaction_root_valid() { + return Err(ConsensusError::BodyTransactionRootDiff(error.into())) + } + // EIP-7825 validation + if chain_spec.is_osaka_active_at_timestamp(block.timestamp()) { + for tx in block.body().transactions() { + if tx.gas_limit() > MAX_TX_GAS_LIMIT_OSAKA { + return Err(TxGasLimitTooHighErr { + tx_hash: *tx.tx_hash(), + gas_limit: tx.gas_limit(), + max_allowed: MAX_TX_GAS_LIMIT_OSAKA, + } + .into()); + } + } + } + + Ok(()) +} + +/// Validates the ommers hash and other fork-specific fields. +/// +/// These fork-specific validations are: +/// * EIP-4895 withdrawals validation, if shanghai is active based on the given chainspec. See more +/// information about the specific checks in [`validate_shanghai_withdrawals`]. +/// * EIP-4844 blob gas validation, if cancun is active based on the given chainspec. See more +/// information about the specific checks in [`validate_cancun_gas`]. +/// * EIP-7934 block size limit validation, if osaka is active based on the given chainspec. +pub fn post_merge_hardfork_fields( + block: &SealedBlock, + chain_spec: &ChainSpec, +) -> Result<(), ConsensusError> where B: Block, ChainSpec: EthereumHardforks, @@ -153,11 +201,6 @@ where )) } - // Check transaction root - if let Err(error) = block.ensure_transaction_root_valid() { - return Err(ConsensusError::BodyTransactionRootDiff(error.into())) - } - // EIP-4895: Beacon chain push withdrawals as operations if chain_spec.is_shanghai_active_at_timestamp(block.timestamp()) { validate_shanghai_withdrawals(block)?; @@ -167,6 +210,15 @@ where validate_cancun_gas(block)?; } + if chain_spec.is_osaka_active_at_timestamp(block.timestamp()) && + block.rlp_length() > MAX_RLP_BLOCK_SIZE + { + return Err(ConsensusError::BlockTooLarge { + rlp_length: block.rlp_length(), + max_rlp_length: MAX_RLP_BLOCK_SIZE, + }) + } + Ok(()) } @@ -185,28 +237,18 @@ pub fn validate_4844_header_standalone( blob_params: BlobParams, ) -> Result<(), ConsensusError> { let blob_gas_used = header.blob_gas_used().ok_or(ConsensusError::BlobGasUsedMissing)?; - let excess_blob_gas = header.excess_blob_gas().ok_or(ConsensusError::ExcessBlobGasMissing)?; if header.parent_beacon_block_root().is_none() { return Err(ConsensusError::ParentBeaconBlockRootMissing) } - if blob_gas_used % DATA_GAS_PER_BLOB != 0 { + if !blob_gas_used.is_multiple_of(DATA_GAS_PER_BLOB) { return Err(ConsensusError::BlobGasUsedNotMultipleOfBlobGasPerBlob { blob_gas_used, blob_gas_per_blob: DATA_GAS_PER_BLOB, }) } - // `excess_blob_gas` must also be a multiple of `DATA_GAS_PER_BLOB`. This will be checked later - // (via `calc_excess_blob_gas`), but it doesn't hurt to catch the problem sooner. - if excess_blob_gas % DATA_GAS_PER_BLOB != 0 { - return Err(ConsensusError::ExcessBlobGasNotMultipleOfBlobGasPerBlob { - excess_blob_gas, - blob_gas_per_blob: DATA_GAS_PER_BLOB, - }) - } - if blob_gas_used > blob_params.max_blob_gas_per_block() { return Err(ConsensusError::BlobGasUsedExceedsMaxBlobGasPerBlock { blob_gas_used, @@ -259,15 +301,31 @@ pub fn validate_against_parent_hash_number( /// Validates the base fee against the parent and EIP-1559 rules. #[inline] -pub fn validate_against_parent_eip1559_base_fee< - H: BlockHeader, - ChainSpec: EthChainSpec + EthereumHardforks, ->( - _header: &H, - _parent: &H, - _chain_spec: &ChainSpec, +pub fn validate_against_parent_eip1559_base_fee( + header: &ChainSpec::Header, + parent: &ChainSpec::Header, + chain_spec: &ChainSpec, ) -> Result<(), ConsensusError> { - // TODO - implement this + if chain_spec.is_london_active_at_block(header.number()) { + let base_fee = header.base_fee_per_gas().ok_or(ConsensusError::BaseFeeMissing)?; + + let expected_base_fee = if chain_spec + .ethereum_fork_activation(EthereumHardfork::London) + .transitions_at_block(header.number()) + { + alloy_eips::eip1559::INITIAL_BASE_FEE + } else { + chain_spec + .next_block_base_fee(parent, header.timestamp()) + .ok_or(ConsensusError::BaseFeeMissing)? + }; + if expected_base_fee != base_fee { + return Err(ConsensusError::BaseFeeDiff(GotExpected { + expected: expected_base_fee, + got: base_fee, + })) + } + } Ok(()) } @@ -287,6 +345,54 @@ pub fn validate_against_parent_timestamp( Ok(()) } +/// Validates gas limit against parent gas limit. +/// +/// The maximum allowable difference between self and parent gas limits is determined by the +/// parent's gas limit divided by the [`GAS_LIMIT_BOUND_DIVISOR`]. +#[inline] +pub fn validate_against_parent_gas_limit< + H: BlockHeader, + ChainSpec: EthChainSpec + EthereumHardforks, +>( + header: &SealedHeader, + parent: &SealedHeader, + chain_spec: &ChainSpec, +) -> Result<(), ConsensusError> { + // Determine the parent gas limit, considering elasticity multiplier on the London fork. + let parent_gas_limit = if !chain_spec.is_london_active_at_block(parent.number()) && + chain_spec.is_london_active_at_block(header.number()) + { + parent.gas_limit() * + chain_spec.base_fee_params_at_timestamp(header.timestamp()).elasticity_multiplier + as u64 + } else { + parent.gas_limit() + }; + + // Check for an increase in gas limit beyond the allowed threshold. + if header.gas_limit() > parent_gas_limit { + if header.gas_limit() - parent_gas_limit >= parent_gas_limit / GAS_LIMIT_BOUND_DIVISOR { + return Err(ConsensusError::GasLimitInvalidIncrease { + parent_gas_limit, + child_gas_limit: header.gas_limit(), + }) + } + } + // Check for a decrease in gas limit beyond the allowed threshold. + else if parent_gas_limit - header.gas_limit() >= parent_gas_limit / GAS_LIMIT_BOUND_DIVISOR { + return Err(ConsensusError::GasLimitInvalidDecrease { + parent_gas_limit, + child_gas_limit: header.gas_limit(), + }) + } + // Check if the self gas limit is below the minimum required limit. + else if header.gas_limit() < MINIMUM_GAS_LIMIT { + return Err(ConsensusError::GasLimitInvalidMinimum { child_gas_limit: header.gas_limit() }) + } + + Ok(()) +} + /// Validates that the EIP-4844 header fields are correct with respect to the parent block. This /// ensures that the `blob_gas_used` and `excess_blob_gas` fields exist in the child header, and /// that the `excess_blob_gas` field matches the expected `excess_blob_gas` calculated from the @@ -310,8 +416,12 @@ pub fn validate_against_parent_4844( } let excess_blob_gas = header.excess_blob_gas().ok_or(ConsensusError::ExcessBlobGasMissing)?; - let expected_excess_blob_gas = - blob_params.next_block_excess_blob_gas(parent_excess_blob_gas, parent_blob_gas_used); + let parent_base_fee_per_gas = parent.base_fee_per_gas().unwrap_or(0); + let expected_excess_blob_gas = blob_params.next_block_excess_blob_gas_osaka( + parent_excess_blob_gas, + parent_blob_gas_used, + parent_base_fee_per_gas, + ); if expected_excess_blob_gas != excess_blob_gas { return Err(ConsensusError::ExcessBlobGasDiff { diff: GotExpected { got: excess_blob_gas, expected: expected_excess_blob_gas }, @@ -368,7 +478,9 @@ mod tests { base_fee_per_gas: Some(1337), withdrawals_root: Some(proofs::calculate_withdrawals_root(&[])), blob_gas_used: Some(1), - transactions_root: proofs::calculate_transaction_root(&[transaction.clone()]), + transactions_root: proofs::calculate_transaction_root(std::slice::from_ref( + &transaction, + )), ..Default::default() }; let body = BlockBody { diff --git a/crates/consensus/consensus/src/lib.rs b/crates/consensus/consensus/src/lib.rs index 7bf171d4797..b3dfa30e61b 100644 --- a/crates/consensus/consensus/src/lib.rs +++ b/crates/consensus/consensus/src/lib.rs @@ -6,12 +6,12 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(feature = "std"), no_std)] extern crate alloc; -use alloc::{fmt::Debug, string::String, vec::Vec}; +use alloc::{boxed::Box, fmt::Debug, string::String, vec::Vec}; use alloy_consensus::Header; use alloy_primitives::{BlockHash, BlockNumber, Bloom, B256}; use reth_execution_types::BlockExecutionResult; @@ -62,8 +62,9 @@ pub trait Consensus: HeaderValidator { /// Validate a block disregarding world state, i.e. things that can be checked before sender /// recovery and execution. /// - /// See the Yellow Paper sections 4.3.2 "Holistic Validity", 4.3.4 "Block Header Validity", and - /// 11.1 "Ommer Validation". + /// See the Yellow Paper sections 4.4.2 "Holistic Validity", 4.4.4 "Block Header Validity". + /// Note: Ommer Validation (previously section 11.1) has been deprecated since the Paris hard + /// fork transition to proof of stake. /// /// **This should not be called for the genesis block**. /// @@ -71,7 +72,7 @@ pub trait Consensus: HeaderValidator { fn validate_block_pre_execution(&self, block: &SealedBlock) -> Result<(), Self::Error>; } -/// HeaderValidator is a protocol that validates headers and their relationships. +/// `HeaderValidator` is a protocol that validates headers and their relationships. #[auto_impl::auto_impl(&, Arc)] pub trait HeaderValidator: Debug + Send + Sync { /// Validate if header is correct and follows consensus specification. @@ -87,7 +88,7 @@ pub trait HeaderValidator: Debug + Send + Sync { /// /// **This should not be called for the genesis block**. /// - /// Note: Validating header against its parent does not include other HeaderValidator + /// Note: Validating header against its parent does not include other `HeaderValidator` /// validations. fn validate_header_against_parent( &self, @@ -320,17 +321,6 @@ pub enum ConsensusError { blob_gas_per_blob: u64, }, - /// Error when excess blob gas is not a multiple of blob gas per blob. - #[error( - "excess blob gas {excess_blob_gas} is not a multiple of blob gas per blob {blob_gas_per_blob}" - )] - ExcessBlobGasNotMultipleOfBlobGasPerBlob { - /// The actual excess blob gas. - excess_blob_gas: u64, - /// The blob gas per blob. - blob_gas_per_blob: u64, - }, - /// Error when the blob gas used in the header does not match the expected blob gas used. #[error("blob gas used mismatch: {0}")] BlobGasUsedDiff(GotExpected), @@ -406,6 +396,17 @@ pub enum ConsensusError { /// The block's timestamp. timestamp: u64, }, + /// Error when the block is too large. + #[error("block is too large: {rlp_length} > {max_rlp_length}")] + BlockTooLarge { + /// The actual RLP length of the block. + rlp_length: usize, + /// The maximum allowed RLP length. + max_rlp_length: usize, + }, + /// EIP-7825: Transaction gas limit exceeds maximum allowed + #[error(transparent)] + TransactionGasLimitTooHigh(Box), /// Other, likely an injected L2 error. #[error("{0}")] Other(String), @@ -424,7 +425,25 @@ impl From for ConsensusError { } } +impl From for ConsensusError { + fn from(value: TxGasLimitTooHighErr) -> Self { + Self::TransactionGasLimitTooHigh(Box::new(value)) + } +} + /// `HeaderConsensusError` combines a `ConsensusError` with the `SealedHeader` it relates to. #[derive(thiserror::Error, Debug)] #[error("Consensus error: {0}, Invalid header: {1:?}")] pub struct HeaderConsensusError(ConsensusError, SealedHeader); + +/// EIP-7825: Transaction gas limit exceeds maximum allowed +#[derive(thiserror::Error, Debug, Eq, PartialEq, Clone)] +#[error("transaction gas limit ({gas_limit}) is greater than the cap ({max_allowed})")] +pub struct TxGasLimitTooHighErr { + /// Hash of the transaction that violates the rule + pub tx_hash: B256, + /// The gas limit of the transaction + pub gas_limit: u64, + /// The maximum allowed gas limit + pub max_allowed: u64, +} diff --git a/crates/consensus/consensus/src/noop.rs b/crates/consensus/consensus/src/noop.rs index 259fae27d67..3d6818ca306 100644 --- a/crates/consensus/consensus/src/noop.rs +++ b/crates/consensus/consensus/src/noop.rs @@ -1,9 +1,32 @@ +//! A consensus implementation that does nothing. +//! +//! This module provides `NoopConsensus`, a consensus implementation that performs no validation +//! and always returns `Ok(())` for all validation methods. Useful for testing and scenarios +//! where consensus validation is not required. +//! +//! # Examples +//! +//! ```rust +//! use reth_consensus::noop::NoopConsensus; +//! use std::sync::Arc; +//! +//! let consensus = NoopConsensus::default(); +//! let consensus_arc = NoopConsensus::arc(); +//! ``` +//! +//! # Warning +//! +//! **Not for production use** - provides no security guarantees or consensus validation. + use crate::{Consensus, ConsensusError, FullConsensus, HeaderValidator}; use alloc::sync::Arc; use reth_execution_types::BlockExecutionResult; use reth_primitives_traits::{Block, NodePrimitives, RecoveredBlock, SealedBlock, SealedHeader}; /// A Consensus implementation that does nothing. +/// +/// Always returns `Ok(())` for all validation methods. Suitable for testing and scenarios +/// where consensus validation is not required. #[derive(Debug, Copy, Clone, Default)] #[non_exhaustive] pub struct NoopConsensus; @@ -16,10 +39,12 @@ impl NoopConsensus { } impl HeaderValidator for NoopConsensus { + /// Validates a header (no-op implementation). fn validate_header(&self, _header: &SealedHeader) -> Result<(), ConsensusError> { Ok(()) } + /// Validates a header against its parent (no-op implementation). fn validate_header_against_parent( &self, _header: &SealedHeader, @@ -32,6 +57,7 @@ impl HeaderValidator for NoopConsensus { impl Consensus for NoopConsensus { type Error = ConsensusError; + /// Validates body against header (no-op implementation). fn validate_body_against_header( &self, _body: &B::Body, @@ -40,12 +66,14 @@ impl Consensus for NoopConsensus { Ok(()) } + /// Validates block before execution (no-op implementation). fn validate_block_pre_execution(&self, _block: &SealedBlock) -> Result<(), Self::Error> { Ok(()) } } impl FullConsensus for NoopConsensus { + /// Validates block after execution (no-op implementation). fn validate_block_post_execution( &self, _block: &RecoveredBlock, diff --git a/crates/consensus/debug-client/Cargo.toml b/crates/consensus/debug-client/Cargo.toml index 784c52c3b53..3783793a29f 100644 --- a/crates/consensus/debug-client/Cargo.toml +++ b/crates/consensus/debug-client/Cargo.toml @@ -20,6 +20,7 @@ reth-primitives-traits.workspace = true alloy-consensus = { workspace = true, features = ["serde"] } alloy-eips.workspace = true alloy-provider = { workspace = true, features = ["ws"] } +alloy-transport.workspace = true alloy-rpc-types-engine.workspace = true alloy-json-rpc.workspace = true alloy-primitives.workspace = true @@ -28,8 +29,9 @@ auto_impl.workspace = true derive_more.workspace = true futures.workspace = true eyre.workspace = true -reqwest = { workspace = true, features = ["rustls-tls", "json"] } +reqwest = { workspace = true, features = ["rustls-tls"] } serde = { workspace = true, features = ["derive"] } tokio = { workspace = true, features = ["time"] } +serde_json.workspace = true ringbuffer.workspace = true diff --git a/crates/consensus/debug-client/src/client.rs b/crates/consensus/debug-client/src/client.rs index b27a1afd8bf..b77d7db94f4 100644 --- a/crates/consensus/debug-client/src/client.rs +++ b/crates/consensus/debug-client/src/client.rs @@ -1,8 +1,8 @@ use alloy_consensus::Sealable; use alloy_primitives::B256; use reth_node_api::{ - BeaconConsensusEngineHandle, BuiltPayload, EngineApiMessageVersion, ExecutionPayload, - NodePrimitives, PayloadTypes, + BuiltPayload, ConsensusEngineHandle, EngineApiMessageVersion, ExecutionPayload, NodePrimitives, + PayloadTypes, }; use reth_primitives_traits::{Block, SealedBlock}; use reth_tracing::tracing::warn; @@ -62,7 +62,7 @@ pub trait BlockProvider: Send + Sync + 'static { #[derive(Debug)] pub struct DebugConsensusClient { /// Handle to execution client. - engine_handle: BeaconConsensusEngineHandle, + engine_handle: ConsensusEngineHandle, /// Provider to get consensus blocks from. block_provider: P, } @@ -70,7 +70,7 @@ pub struct DebugConsensusClient { impl DebugConsensusClient { /// Create a new debug consensus client with the given handle to execution /// client and block provider. - pub const fn new(engine_handle: BeaconConsensusEngineHandle, block_provider: P) -> Self { + pub const fn new(engine_handle: ConsensusEngineHandle, block_provider: P) -> Self { Self { engine_handle, block_provider } } } @@ -84,7 +84,6 @@ where /// blocks. pub async fn run(self) { let mut previous_block_hashes = AllocRingBuffer::new(64); - let mut block_stream = { let (tx, rx) = mpsc::channel::(64); let block_provider = self.block_provider.clone(); diff --git a/crates/consensus/debug-client/src/lib.rs b/crates/consensus/debug-client/src/lib.rs index bc244fafeb0..16dc9d34578 100644 --- a/crates/consensus/debug-client/src/lib.rs +++ b/crates/consensus/debug-client/src/lib.rs @@ -10,7 +10,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] mod client; mod providers; diff --git a/crates/consensus/debug-client/src/providers/etherscan.rs b/crates/consensus/debug-client/src/providers/etherscan.rs index c52ee609d20..ea21d95e73d 100644 --- a/crates/consensus/debug-client/src/providers/etherscan.rs +++ b/crates/consensus/debug-client/src/providers/etherscan.rs @@ -3,7 +3,7 @@ use alloy_consensus::BlockHeader; use alloy_eips::BlockNumberOrTag; use alloy_json_rpc::{Response, ResponsePayload}; use reqwest::Client; -use reth_tracing::tracing::warn; +use reth_tracing::tracing::{debug, warn}; use serde::{de::DeserializeOwned, Serialize}; use std::{sync::Arc, time::Duration}; use tokio::{sync::mpsc, time::interval}; @@ -14,6 +14,7 @@ pub struct EtherscanBlockProvider { http_client: Client, base_url: String, api_key: String, + chain_id: u64, interval: Duration, #[debug(skip)] convert: Arc PrimitiveBlock + Send + Sync>, @@ -27,12 +28,14 @@ where pub fn new( base_url: String, api_key: String, + chain_id: u64, convert: impl Fn(RpcBlock) -> PrimitiveBlock + Send + Sync + 'static, ) -> Self { Self { http_client: Client::new(), base_url, api_key, + chain_id, interval: Duration::from_secs(3), convert: Arc::new(convert), } @@ -56,20 +59,26 @@ where tag => tag.to_string(), }; - let resp: Response = self - .http_client - .get(&self.base_url) - .query(&[ - ("module", "proxy"), - ("action", "eth_getBlockByNumber"), - ("tag", &tag), - ("boolean", "true"), - ("apikey", &self.api_key), - ]) - .send() - .await? - .json() - .await?; + let mut req = self.http_client.get(&self.base_url).query(&[ + ("module", "proxy"), + ("action", "eth_getBlockByNumber"), + ("tag", &tag), + ("boolean", "true"), + ("apikey", &self.api_key), + ]); + + if !self.base_url.contains("chainid=") { + // only append chainid if not part of the base url already + req = req.query(&[("chainid", &self.chain_id.to_string())]); + } + + let resp = req.send().await?.text().await?; + + debug!(target: "etherscan", %resp, "fetched block from etherscan"); + + let resp: Response = serde_json::from_str(&resp).inspect_err(|err| { + warn!(target: "etherscan", "Failed to parse block response from etherscan: {}", err); + })?; let payload = resp.payload; match payload { diff --git a/crates/consensus/debug-client/src/providers/rpc.rs b/crates/consensus/debug-client/src/providers/rpc.rs index 6c5e0a35dca..0c9dfbce7de 100644 --- a/crates/consensus/debug-client/src/providers/rpc.rs +++ b/crates/consensus/debug-client/src/providers/rpc.rs @@ -1,9 +1,9 @@ use crate::BlockProvider; -use alloy_consensus::BlockHeader; use alloy_provider::{Network, Provider, ProviderBuilder}; -use futures::StreamExt; +use alloy_transport::TransportResult; +use futures::{Stream, StreamExt}; use reth_node_api::Block; -use reth_tracing::tracing::warn; +use reth_tracing::tracing::{debug, warn}; use std::sync::Arc; use tokio::sync::mpsc::Sender; @@ -25,17 +25,33 @@ impl RpcBlockProvider { convert: impl Fn(N::BlockResponse) -> PrimitiveBlock + Send + Sync + 'static, ) -> eyre::Result { Ok(Self { - provider: Arc::new( - ProviderBuilder::new() - .disable_recommended_fillers() - .network::() - .connect(rpc_url) - .await?, - ), + provider: Arc::new(ProviderBuilder::default().connect(rpc_url).await?), url: rpc_url.to_string(), convert: Arc::new(convert), }) } + + /// Obtains a full block stream. + /// + /// This first attempts to obtain an `eth_subscribe` subscription, if that fails because the + /// connection is not a websocket, this falls back to poll based subscription. + async fn full_block_stream( + &self, + ) -> TransportResult>> { + // first try to obtain a regular subscription + match self.provider.subscribe_full_blocks().full().into_stream().await { + Ok(sub) => Ok(sub.left_stream()), + Err(err) => { + debug!( + target: "consensus::debug-client", + %err, + url=%self.url, + "Failed to establish block subscription", + ); + Ok(self.provider.watch_full_blocks().await?.full().into_stream().right_stream()) + } + } + } } impl BlockProvider for RpcBlockProvider @@ -45,22 +61,21 @@ where type Block = PrimitiveBlock; async fn subscribe_blocks(&self, tx: Sender) { - let mut stream = match self.provider.subscribe_blocks().await { - Ok(sub) => sub.into_stream(), - Err(err) => { - warn!( - target: "consensus::debug-client", - %err, - url=%self.url, - "Failed to subscribe to blocks", - ); - return; - } + let Ok(mut stream) = self.full_block_stream().await.inspect_err(|err| { + warn!( + target: "consensus::debug-client", + %err, + url=%self.url, + "Failed to subscribe to blocks", + ); + }) else { + return }; - while let Some(header) = stream.next().await { - match self.get_block(header.number()).await { + + while let Some(res) = stream.next().await { + match res { Ok(block) => { - if tx.send(block).await.is_err() { + if tx.send((self.convert)(block)).await.is_err() { // Channel closed. break; } diff --git a/crates/e2e-test-utils/Cargo.toml b/crates/e2e-test-utils/Cargo.toml index a324f1d4612..673193ddd9a 100644 --- a/crates/e2e-test-utils/Cargo.toml +++ b/crates/e2e-test-utils/Cargo.toml @@ -15,8 +15,9 @@ reth-chainspec.workspace = true reth-tracing.workspace = true reth-db = { workspace = true, features = ["test-utils"] } reth-network-api.workspace = true -reth-rpc-layer.workspace = true +reth-network-p2p.workspace = true reth-rpc-server-types.workspace = true +reth-rpc-builder.workspace = true reth-rpc-eth-api.workspace = true reth-rpc-api = { workspace = true, features = ["client"] } reth-payload-builder = { workspace = true, features = ["test-utils"] } @@ -30,11 +31,19 @@ reth-tokio-util.workspace = true reth-stages-types.workspace = true reth-network-peers.workspace = true reth-engine-local.workspace = true +reth-engine-primitives.workspace = true reth-tasks.workspace = true reth-node-ethereum.workspace = true reth-ethereum-primitives.workspace = true +reth-cli-commands.workspace = true +reth-config.workspace = true +reth-consensus.workspace = true +reth-primitives.workspace = true +reth-db-common.workspace = true +reth-primitives-traits.workspace = true revm.workspace = true +tempfile.workspace = true # rpc jsonrpsee.workspace = true @@ -43,17 +52,23 @@ url.workspace = true # ethereum alloy-primitives.workspace = true alloy-eips.workspace = true - -futures-util.workspace = true -eyre.workspace = true -tokio.workspace = true -tokio-stream.workspace = true -serde_json.workspace = true +alloy-rlp.workspace = true alloy-signer.workspace = true alloy-signer-local = { workspace = true, features = ["mnemonic"] } alloy-rpc-types-eth.workspace = true alloy-rpc-types-engine.workspace = true alloy-network.workspace = true alloy-consensus = { workspace = true, features = ["kzg"] } +alloy-provider = { workspace = true, features = ["reqwest"] } + +futures-util.workspace = true +eyre.workspace = true +tokio.workspace = true +tokio-stream.workspace = true +serde_json.workspace = true tracing.workspace = true derive_more.workspace = true + +[[test]] +name = "e2e_testsuite" +path = "tests/e2e-testsuite/main.rs" diff --git a/crates/e2e-test-utils/src/lib.rs b/crates/e2e-test-utils/src/lib.rs index 4073b6c8b1d..57d03f70fa5 100644 --- a/crates/e2e-test-utils/src/lib.rs +++ b/crates/e2e-test-utils/src/lib.rs @@ -1,23 +1,19 @@ //! Utilities for end-to-end tests. use node::NodeTestContext; -use reth_chainspec::{ChainSpec, EthChainSpec}; +use reth_chainspec::ChainSpec; use reth_db::{test_utils::TempDatabase, DatabaseEnv}; use reth_engine_local::LocalPayloadAttributesBuilder; use reth_network_api::test_utils::PeersHandleProvider; use reth_node_builder::{ components::NodeComponentsBuilder, rpc::{EngineValidatorAddOn, RethRpcAddOns}, - EngineNodeLauncher, FullNodeTypesAdapter, Node, NodeAdapter, NodeBuilder, NodeComponents, - NodeConfig, NodeHandle, NodePrimitives, NodeTypes, NodeTypesWithDBAdapter, + FullNodeTypesAdapter, Node, NodeAdapter, NodeComponents, NodeTypes, NodeTypesWithDBAdapter, PayloadAttributesBuilder, PayloadTypes, }; -use reth_node_core::args::{DiscoveryArgs, NetworkArgs, RpcServerArgs}; use reth_provider::providers::{BlockchainProvider, NodeTypesForProvider}; -use reth_rpc_server_types::RpcModuleSelection; use reth_tasks::TaskManager; use std::sync::Arc; -use tracing::{span, Level}; use wallet::Wallet; /// Wrapper type to create test nodes @@ -33,12 +29,22 @@ pub mod wallet; /// Helper for payload operations mod payload; +/// Helper for setting up nodes with pre-imported chain data +pub mod setup_import; + /// Helper for network operations mod network; /// Helper for rpc operations mod rpc; +/// Utilities for creating and writing RLP test data +pub mod test_rlp_utils; + +/// Builder for configuring test node setups +mod setup_builder; +pub use setup_builder::E2ETestSetupBuilder; + /// Creates the initial setup with `num_nodes` started and interconnected. pub async fn setup( num_nodes: usize, @@ -47,59 +53,14 @@ pub async fn setup( attributes_generator: impl Fn(u64) -> <::Payload as PayloadTypes>::PayloadBuilderAttributes + Send + Sync + Copy + 'static, ) -> eyre::Result<(Vec>, TaskManager, Wallet)> where - N: Default + Node> + NodeTypesForProvider + NodeTypes, - N::ComponentsBuilder: NodeComponentsBuilder< - TmpNodeAdapter, - Components: NodeComponents, Network: PeersHandleProvider>, - >, - N::AddOns: RethRpcAddOns> + EngineValidatorAddOn>, + N: NodeBuilderHelper, LocalPayloadAttributesBuilder: PayloadAttributesBuilder<<::Payload as PayloadTypes>::PayloadAttributes>, { - let tasks = TaskManager::current(); - let exec = tasks.executor(); - - let network_config = NetworkArgs { - discovery: DiscoveryArgs { disable_discovery: true, ..DiscoveryArgs::default() }, - ..NetworkArgs::default() - }; - - // Create nodes and peer them - let mut nodes: Vec> = Vec::with_capacity(num_nodes); - - for idx in 0..num_nodes { - let node_config = NodeConfig::new(chain_spec.clone()) - .with_network(network_config.clone()) - .with_unused_ports() - .with_rpc(RpcServerArgs::default().with_unused_ports().with_http()) - .set_dev(is_dev); - - let span = span!(Level::INFO, "node", idx); - let _enter = span.enter(); - let NodeHandle { node, node_exit_future: _ } = NodeBuilder::new(node_config.clone()) - .testing_node(exec.clone()) - .node(Default::default()) - .launch() - .await?; - - let mut node = NodeTestContext::new(node, attributes_generator).await?; - - // Connect each node in a chain. - if let Some(previous_node) = nodes.last_mut() { - previous_node.connect(&mut node).await; - } - - // Connect last node with the first if there are more than two - if idx + 1 == num_nodes && num_nodes > 2 { - if let Some(first_node) = nodes.first_mut() { - node.connect(first_node).await; - } - } - - nodes.push(node); - } - - Ok((nodes, tasks, Wallet::default().with_chain_id(chain_spec.chain().into()))) + E2ETestSetupBuilder::new(num_nodes, chain_spec, attributes_generator) + .with_node_config_modifier(move |config| config.set_dev(is_dev)) + .build() + .await } /// Creates the initial setup with `num_nodes` started and interconnected. @@ -107,6 +68,7 @@ pub async fn setup_engine( num_nodes: usize, chain_spec: Arc, is_dev: bool, + tree_config: reth_node_api::TreeConfig, attributes_generator: impl Fn(u64) -> <::Payload as PayloadTypes>::PayloadBuilderAttributes + Send + Sync + Copy + 'static, ) -> eyre::Result<( Vec>>>, @@ -118,68 +80,41 @@ where LocalPayloadAttributesBuilder: PayloadAttributesBuilder<::PayloadAttributes>, { - let tasks = TaskManager::current(); - let exec = tasks.executor(); - - let network_config = NetworkArgs { - discovery: DiscoveryArgs { disable_discovery: true, ..DiscoveryArgs::default() }, - ..NetworkArgs::default() - }; - - // Create nodes and peer them - let mut nodes: Vec> = Vec::with_capacity(num_nodes); - - for idx in 0..num_nodes { - let node_config = NodeConfig::new(chain_spec.clone()) - .with_network(network_config.clone()) - .with_unused_ports() - .with_rpc( - RpcServerArgs::default() - .with_unused_ports() - .with_http() - .with_http_api(RpcModuleSelection::All), - ) - .set_dev(is_dev); - - let span = span!(Level::INFO, "node", idx); - let _enter = span.enter(); - let node = N::default(); - let NodeHandle { node, node_exit_future: _ } = NodeBuilder::new(node_config.clone()) - .testing_node(exec.clone()) - .with_types_and_provider::>() - .with_components(node.components_builder()) - .with_add_ons(node.add_ons()) - .launch_with_fn(|builder| { - let launcher = EngineNodeLauncher::new( - builder.task_executor().clone(), - builder.config().datadir(), - Default::default(), - ); - builder.launch_with(launcher) - }) - .await?; - - let mut node = NodeTestContext::new(node, attributes_generator).await?; - - let genesis = node.block_hash(0); - node.update_forkchoice(genesis, genesis).await?; - - // Connect each node in a chain. - if let Some(previous_node) = nodes.last_mut() { - previous_node.connect(&mut node).await; - } - - // Connect last node with the first if there are more than two - if idx + 1 == num_nodes && num_nodes > 2 { - if let Some(first_node) = nodes.first_mut() { - node.connect(first_node).await; - } - } - - nodes.push(node); - } + setup_engine_with_connection::( + num_nodes, + chain_spec, + is_dev, + tree_config, + attributes_generator, + true, + ) + .await +} - Ok((nodes, tasks, Wallet::default().with_chain_id(chain_spec.chain().into()))) +/// Creates the initial setup with `num_nodes` started and optionally interconnected. +pub async fn setup_engine_with_connection( + num_nodes: usize, + chain_spec: Arc, + is_dev: bool, + tree_config: reth_node_api::TreeConfig, + attributes_generator: impl Fn(u64) -> <::Payload as PayloadTypes>::PayloadBuilderAttributes + Send + Sync + Copy + 'static, + connect_nodes: bool, +) -> eyre::Result<( + Vec>>>, + TaskManager, + Wallet, +)> +where + N: NodeBuilderHelper, + LocalPayloadAttributesBuilder: + PayloadAttributesBuilder<::PayloadAttributes>, +{ + E2ETestSetupBuilder::new(num_nodes, chain_spec, attributes_generator) + .with_tree_config_modifier(move |_| tree_config.clone()) + .with_node_config_modifier(move |config| config.set_dev(is_dev)) + .with_connect_nodes(connect_nodes) + .build() + .await } // Type aliases @@ -205,19 +140,12 @@ pub type NodeHelperType, >, > + Node< TmpNodeAdapter>>, - Primitives: NodePrimitives< - BlockHeader = alloy_consensus::Header, - BlockBody = alloy_consensus::BlockBody< - ::SignedTx, - >, - >, ComponentsBuilder: NodeComponentsBuilder< TmpNodeAdapter>>, Components: NodeComponents< @@ -240,19 +168,12 @@ where impl NodeBuilderHelper for T where Self: Default - + NodeTypesForProvider - + NodeTypes< + + NodeTypesForProvider< Payload: PayloadTypes< PayloadBuilderAttributes: From, >, > + Node< TmpNodeAdapter>>, - Primitives: NodePrimitives< - BlockHeader = alloy_consensus::Header, - BlockBody = alloy_consensus::BlockBody< - ::SignedTx, - >, - >, ComponentsBuilder: NodeComponentsBuilder< TmpNodeAdapter>>, Components: NodeComponents< diff --git a/crates/e2e-test-utils/src/node.rs b/crates/e2e-test-utils/src/node.rs index 293beb55be4..4dd1ae63e1a 100644 --- a/crates/e2e-test-utils/src/node.rs +++ b/crates/e2e-test-utils/src/node.rs @@ -1,12 +1,12 @@ use crate::{network::NetworkTestContext, payload::PayloadTestContext, rpc::RpcTestContext}; -use alloy_consensus::BlockHeader; +use alloy_consensus::{transaction::TxHashRef, BlockHeader}; use alloy_eips::BlockId; use alloy_primitives::{BlockHash, BlockNumber, Bytes, Sealable, B256}; use alloy_rpc_types_engine::ForkchoiceState; use alloy_rpc_types_eth::BlockNumberOrTag; use eyre::Ok; use futures_util::Future; -use jsonrpsee::http_client::{transport::HttpBackend, HttpClient}; +use jsonrpsee::http_client::HttpClient; use reth_chainspec::EthereumHardforks; use reth_network_api::test_utils::PeersHandleProvider; use reth_node_api::{ @@ -14,20 +14,20 @@ use reth_node_api::{ PrimitivesTy, }; use reth_node_builder::{rpc::RethRpcAddOns, FullNode, NodeTypes}; -use reth_node_core::primitives::SignedTransaction; + use reth_payload_primitives::{BuiltPayload, PayloadBuilderAttributes}; use reth_provider::{ BlockReader, BlockReaderIdExt, CanonStateNotificationStream, CanonStateSubscriptions, - StageCheckpointReader, + HeaderProvider, StageCheckpointReader, }; +use reth_rpc_builder::auth::AuthServerHandle; use reth_rpc_eth_api::helpers::{EthApiSpec, EthTransactions, TraceExt}; -use reth_rpc_layer::AuthClientService; use reth_stages_types::StageId; use std::pin::Pin; use tokio_stream::StreamExt; use url::Url; -/// An helper struct to handle node actions +/// A helper struct to handle node actions #[expect(missing_debug_implementations)] pub struct NodeTestContext where @@ -150,19 +150,18 @@ where loop { tokio::time::sleep(std::time::Duration::from_millis(20)).await; - if !check && wait_finish_checkpoint { - if let Some(checkpoint) = - self.inner.provider.get_stage_checkpoint(StageId::Finish)? - { - if checkpoint.block_number >= number { - check = true - } - } + if !check && + wait_finish_checkpoint && + let Some(checkpoint) = + self.inner.provider.get_stage_checkpoint(StageId::Finish)? && + checkpoint.block_number >= number + { + check = true } if check { - if let Some(latest_block) = self.inner.provider.block_by_number(number)? { - assert_eq!(latest_block.header().hash_slow(), expected_block_hash); + if let Some(latest_header) = self.inner.provider.header_by_number(number)? { + assert_eq!(latest_header.hash_slow(), expected_block_hash); break } assert!( @@ -178,10 +177,10 @@ where pub async fn wait_unwind(&self, number: BlockNumber) -> eyre::Result<()> { loop { tokio::time::sleep(std::time::Duration::from_millis(10)).await; - if let Some(checkpoint) = self.inner.provider.get_stage_checkpoint(StageId::Headers)? { - if checkpoint.block_number == number { - break - } + if let Some(checkpoint) = self.inner.provider.get_stage_checkpoint(StageId::Headers)? && + checkpoint.block_number == number + { + break } } Ok(()) @@ -207,14 +206,13 @@ where // wait for the block to commit tokio::time::sleep(std::time::Duration::from_millis(20)).await; if let Some(latest_block) = - self.inner.provider.block_by_number_or_tag(BlockNumberOrTag::Latest)? + self.inner.provider.block_by_number_or_tag(BlockNumberOrTag::Latest)? && + latest_block.header().number() == block_number { - if latest_block.header().number() == block_number { - // make sure the block hash we submitted via FCU engine api is the new latest - // block using an RPC call - assert_eq!(latest_block.header().hash_slow(), block_hash); - break - } + // make sure the block hash we submitted via FCU engine api is the new latest + // block using an RPC call + assert_eq!(latest_block.header().hash_slow(), block_hash); + break } } Ok(()) @@ -302,7 +300,23 @@ where } /// Returns an Engine API client. - pub fn engine_api_client(&self) -> HttpClient> { - self.inner.auth_server_handle().http_client() + pub fn auth_server_handle(&self) -> AuthServerHandle { + self.inner.auth_server_handle().clone() + } + + /// Creates a [`crate::testsuite::NodeClient`] from this test context. + /// + /// This helper method extracts the necessary handles and creates a client + /// that can interact with both the regular RPC and Engine API endpoints. + /// It automatically includes the beacon engine handle for direct consensus engine interaction. + pub fn to_node_client(&self) -> eyre::Result> { + let rpc = self + .rpc_client() + .ok_or_else(|| eyre::eyre!("Failed to create HTTP RPC client for node"))?; + let auth = self.auth_server_handle(); + let url = self.rpc_url(); + let beacon_handle = self.inner.add_ons_handle.beacon_engine_handle.clone(); + + Ok(crate::testsuite::NodeClient::new_with_beacon_engine(rpc, auth, url, beacon_handle)) } } diff --git a/crates/e2e-test-utils/src/payload.rs b/crates/e2e-test-utils/src/payload.rs index b3f9b027fba..4e185ce9693 100644 --- a/crates/e2e-test-utils/src/payload.rs +++ b/crates/e2e-test-utils/src/payload.rs @@ -57,8 +57,9 @@ impl PayloadTestContext { /// Wait until the best built payload is ready pub async fn wait_for_built_payload(&self, payload_id: PayloadId) { loop { - let payload = self.payload_builder.best_payload(payload_id).await.unwrap().unwrap(); - if payload.block().body().transactions().is_empty() { + let payload = + self.payload_builder.best_payload(payload_id).await.transpose().ok().flatten(); + if payload.is_none_or(|p| p.block().body().transactions().is_empty()) { tokio::time::sleep(std::time::Duration::from_millis(20)).await; continue } diff --git a/crates/e2e-test-utils/src/rpc.rs b/crates/e2e-test-utils/src/rpc.rs index 96dda811735..ff030c390b9 100644 --- a/crates/e2e-test-utils/src/rpc.rs +++ b/crates/e2e-test-utils/src/rpc.rs @@ -1,4 +1,5 @@ -use alloy_consensus::TxEnvelope; +use alloy_consensus::{EthereumTxEnvelope, TxEip4844Variant}; +use alloy_eips::eip7594::BlobTransactionSidecarVariant; use alloy_network::eip2718::Decodable2718; use alloy_primitives::{Bytes, B256}; use reth_chainspec::EthereumHardforks; @@ -30,9 +31,12 @@ where } /// Retrieves a transaction envelope by its hash - pub async fn envelope_by_hash(&self, hash: B256) -> eyre::Result { + pub async fn envelope_by_hash( + &self, + hash: B256, + ) -> eyre::Result>> { let tx = self.inner.debug_api().raw_transaction(hash).await?.unwrap(); let tx = tx.to_vec(); - Ok(TxEnvelope::decode_2718(&mut tx.as_ref()).unwrap()) + Ok(EthereumTxEnvelope::decode_2718(&mut tx.as_ref()).unwrap()) } } diff --git a/crates/e2e-test-utils/src/setup_builder.rs b/crates/e2e-test-utils/src/setup_builder.rs new file mode 100644 index 00000000000..8de2280fe41 --- /dev/null +++ b/crates/e2e-test-utils/src/setup_builder.rs @@ -0,0 +1,210 @@ +//! Builder for configuring and creating test node setups. +//! +//! This module provides a flexible builder API for setting up test nodes with custom +//! configurations through closures that modify `NodeConfig` and `TreeConfig`. + +use crate::{node::NodeTestContext, wallet::Wallet, NodeBuilderHelper, NodeHelperType, TmpDB}; +use reth_chainspec::EthChainSpec; +use reth_engine_local::LocalPayloadAttributesBuilder; +use reth_node_builder::{ + EngineNodeLauncher, NodeBuilder, NodeConfig, NodeHandle, NodeTypes, NodeTypesWithDBAdapter, + PayloadAttributesBuilder, PayloadTypes, +}; +use reth_node_core::args::{DiscoveryArgs, NetworkArgs, RpcServerArgs}; +use reth_provider::providers::BlockchainProvider; +use reth_rpc_server_types::RpcModuleSelection; +use reth_tasks::TaskManager; +use std::sync::Arc; +use tracing::{span, Level}; + +/// Type alias for tree config modifier closure +type TreeConfigModifier = + Box reth_node_api::TreeConfig + Send + Sync>; + +/// Type alias for node config modifier closure +type NodeConfigModifier = Box) -> NodeConfig + Send + Sync>; + +/// Builder for configuring and creating test node setups. +/// +/// This builder allows customizing test node configurations through closures that +/// modify `NodeConfig` and `TreeConfig`. It avoids code duplication by centralizing +/// the node creation logic. +pub struct E2ETestSetupBuilder +where + N: NodeBuilderHelper, + F: Fn(u64) -> <::Payload as PayloadTypes>::PayloadBuilderAttributes + + Send + + Sync + + Copy + + 'static, + LocalPayloadAttributesBuilder: + PayloadAttributesBuilder<::PayloadAttributes>, +{ + num_nodes: usize, + chain_spec: Arc, + attributes_generator: F, + connect_nodes: bool, + tree_config_modifier: Option, + node_config_modifier: Option>, +} + +impl E2ETestSetupBuilder +where + N: NodeBuilderHelper, + F: Fn(u64) -> <::Payload as PayloadTypes>::PayloadBuilderAttributes + + Send + + Sync + + Copy + + 'static, + LocalPayloadAttributesBuilder: + PayloadAttributesBuilder<::PayloadAttributes>, +{ + /// Creates a new builder with the required parameters. + pub fn new(num_nodes: usize, chain_spec: Arc, attributes_generator: F) -> Self { + Self { + num_nodes, + chain_spec, + attributes_generator, + connect_nodes: true, + tree_config_modifier: None, + node_config_modifier: None, + } + } + + /// Sets whether nodes should be interconnected (default: true). + pub const fn with_connect_nodes(mut self, connect_nodes: bool) -> Self { + self.connect_nodes = connect_nodes; + self + } + + /// Sets a modifier function for the tree configuration. + /// + /// The closure receives the base tree config and returns a modified version. + pub fn with_tree_config_modifier(mut self, modifier: G) -> Self + where + G: Fn(reth_node_api::TreeConfig) -> reth_node_api::TreeConfig + Send + Sync + 'static, + { + self.tree_config_modifier = Some(Box::new(modifier)); + self + } + + /// Sets a modifier function for the node configuration. + /// + /// The closure receives the base node config and returns a modified version. + pub fn with_node_config_modifier(mut self, modifier: G) -> Self + where + G: Fn(NodeConfig) -> NodeConfig + Send + Sync + 'static, + { + self.node_config_modifier = Some(Box::new(modifier)); + self + } + + /// Builds and launches the test nodes. + pub async fn build( + self, + ) -> eyre::Result<( + Vec>>>, + TaskManager, + Wallet, + )> { + let tasks = TaskManager::current(); + let exec = tasks.executor(); + + let network_config = NetworkArgs { + discovery: DiscoveryArgs { disable_discovery: true, ..DiscoveryArgs::default() }, + ..NetworkArgs::default() + }; + + // Apply tree config modifier if present + let tree_config = if let Some(modifier) = self.tree_config_modifier { + modifier(reth_node_api::TreeConfig::default()) + } else { + reth_node_api::TreeConfig::default() + }; + + let mut nodes: Vec> = Vec::with_capacity(self.num_nodes); + + for idx in 0..self.num_nodes { + // Create base node config + let base_config = NodeConfig::new(self.chain_spec.clone()) + .with_network(network_config.clone()) + .with_unused_ports() + .with_rpc( + RpcServerArgs::default() + .with_unused_ports() + .with_http() + .with_http_api(RpcModuleSelection::All), + ); + + // Apply node config modifier if present + let node_config = if let Some(modifier) = &self.node_config_modifier { + modifier(base_config) + } else { + base_config + }; + + let span = span!(Level::INFO, "node", idx); + let _enter = span.enter(); + let node = N::default(); + let NodeHandle { node, node_exit_future: _ } = NodeBuilder::new(node_config) + .testing_node(exec.clone()) + .with_types_and_provider::>() + .with_components(node.components_builder()) + .with_add_ons(node.add_ons()) + .launch_with_fn(|builder| { + let launcher = EngineNodeLauncher::new( + builder.task_executor().clone(), + builder.config().datadir(), + tree_config.clone(), + ); + builder.launch_with(launcher) + }) + .await?; + + let mut node = NodeTestContext::new(node, self.attributes_generator).await?; + + let genesis = node.block_hash(0); + node.update_forkchoice(genesis, genesis).await?; + + // Connect nodes if requested + if self.connect_nodes { + if let Some(previous_node) = nodes.last_mut() { + previous_node.connect(&mut node).await; + } + + // Connect last node with the first if there are more than two + if idx + 1 == self.num_nodes && + self.num_nodes > 2 && + let Some(first_node) = nodes.first_mut() + { + node.connect(first_node).await; + } + } + + nodes.push(node); + } + + Ok((nodes, tasks, Wallet::default().with_chain_id(self.chain_spec.chain().into()))) + } +} + +impl std::fmt::Debug for E2ETestSetupBuilder +where + N: NodeBuilderHelper, + F: Fn(u64) -> <::Payload as PayloadTypes>::PayloadBuilderAttributes + + Send + + Sync + + Copy + + 'static, + LocalPayloadAttributesBuilder: + PayloadAttributesBuilder<::PayloadAttributes>, +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("E2ETestSetupBuilder") + .field("num_nodes", &self.num_nodes) + .field("connect_nodes", &self.connect_nodes) + .field("tree_config_modifier", &self.tree_config_modifier.as_ref().map(|_| "")) + .field("node_config_modifier", &self.node_config_modifier.as_ref().map(|_| "")) + .finish_non_exhaustive() + } +} diff --git a/crates/e2e-test-utils/src/setup_import.rs b/crates/e2e-test-utils/src/setup_import.rs new file mode 100644 index 00000000000..81e5a386aac --- /dev/null +++ b/crates/e2e-test-utils/src/setup_import.rs @@ -0,0 +1,576 @@ +//! Setup utilities for importing RLP chain data before starting nodes. + +use crate::{node::NodeTestContext, NodeHelperType, Wallet}; +use reth_chainspec::ChainSpec; +use reth_cli_commands::import_core::{import_blocks_from_file, ImportConfig}; +use reth_config::Config; +use reth_db::DatabaseEnv; +use reth_node_api::{NodeTypesWithDBAdapter, TreeConfig}; +use reth_node_builder::{EngineNodeLauncher, Node, NodeBuilder, NodeConfig, NodeHandle}; +use reth_node_core::args::{DiscoveryArgs, NetworkArgs, RpcServerArgs}; +use reth_node_ethereum::EthereumNode; +use reth_provider::{ + providers::BlockchainProvider, DatabaseProviderFactory, ProviderFactory, StageCheckpointReader, + StaticFileProviderFactory, +}; +use reth_rpc_server_types::RpcModuleSelection; +use reth_stages_types::StageId; +use reth_tasks::TaskManager; +use std::{path::Path, sync::Arc}; +use tempfile::TempDir; +use tracing::{debug, info, span, Level}; + +/// Setup result containing nodes and temporary directories that must be kept alive +pub struct ChainImportResult { + /// The nodes that were created + pub nodes: Vec>, + /// The task manager + pub task_manager: TaskManager, + /// The wallet for testing + pub wallet: Wallet, + /// Temporary directories that must be kept alive for the duration of the test + pub _temp_dirs: Vec, +} + +impl std::fmt::Debug for ChainImportResult { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ChainImportResult") + .field("nodes", &self.nodes.len()) + .field("wallet", &self.wallet) + .field("temp_dirs", &self._temp_dirs.len()) + .finish() + } +} + +/// Creates a test setup with Ethereum nodes that have pre-imported chain data from RLP files. +/// +/// This function: +/// 1. Creates a temporary datadir for each node +/// 2. Imports the specified RLP chain data into the datadir +/// 3. Starts the nodes with the pre-populated database +/// 4. Returns the running nodes ready for testing +/// +/// Note: This function is currently specific to `EthereumNode` because the import process +/// uses Ethereum-specific consensus and block format. It can be made generic in the future +/// by abstracting the import process. +/// It uses `NoopConsensus` during import to bypass validation checks like gas limit constraints, +/// which allows importing test chains that may not strictly conform to mainnet consensus rules. The +/// nodes themselves still run with proper consensus when started. +pub async fn setup_engine_with_chain_import( + num_nodes: usize, + chain_spec: Arc, + is_dev: bool, + tree_config: TreeConfig, + rlp_path: &Path, + attributes_generator: impl Fn(u64) -> reth_payload_builder::EthPayloadBuilderAttributes + + Send + + Sync + + Copy + + 'static, +) -> eyre::Result { + let tasks = TaskManager::current(); + let exec = tasks.executor(); + + let network_config = NetworkArgs { + discovery: DiscoveryArgs { disable_discovery: true, ..DiscoveryArgs::default() }, + ..NetworkArgs::default() + }; + + // Create nodes with imported data + let mut nodes: Vec> = Vec::with_capacity(num_nodes); + let mut temp_dirs = Vec::with_capacity(num_nodes); // Keep temp dirs alive + + for idx in 0..num_nodes { + // Create a temporary datadir for this node + let temp_dir = TempDir::new()?; + let datadir = temp_dir.path().to_path_buf(); + + let mut node_config = NodeConfig::new(chain_spec.clone()) + .with_network(network_config.clone()) + .with_unused_ports() + .with_rpc( + RpcServerArgs::default() + .with_unused_ports() + .with_http() + .with_http_api(RpcModuleSelection::All), + ) + .set_dev(is_dev); + + // Set the datadir + node_config.datadir.datadir = + reth_node_core::dirs::MaybePlatformPath::from(datadir.clone()); + debug!(target: "e2e::import", "Node {idx} datadir: {datadir:?}"); + + let span = span!(Level::INFO, "node", idx); + let _enter = span.enter(); + + // First, import the chain data into this datadir + info!(target: "test", "Importing chain data from {:?} for node {} into {:?}", rlp_path, idx, datadir); + + // Create database path and static files path + let db_path = datadir.join("db"); + let static_files_path = datadir.join("static_files"); + + // Initialize the database using init_db (same as CLI import command) + // Use the same database arguments as the node will use + let db_args = reth_node_core::args::DatabaseArgs::default().database_args(); + let db_env = reth_db::init_db(&db_path, db_args)?; + let db = Arc::new(db_env); + + // Create a provider factory with the initialized database (use regular DB, not + // TempDatabase) We need to specify the node types properly for the adapter + let provider_factory = ProviderFactory::< + NodeTypesWithDBAdapter>, + >::new( + db.clone(), + chain_spec.clone(), + reth_provider::providers::StaticFileProvider::read_write(static_files_path.clone())?, + ); + + // Initialize genesis if needed + reth_db_common::init::init_genesis(&provider_factory)?; + + // Import the chain data + // Use no_state to skip state validation for test chains + let import_config = ImportConfig::default(); + let config = Config::default(); + + // Create EVM and consensus for Ethereum + let evm_config = reth_node_ethereum::EthEvmConfig::new(chain_spec.clone()); + // Use NoopConsensus to skip gas limit validation for test imports + let consensus = reth_consensus::noop::NoopConsensus::arc(); + + let result = import_blocks_from_file( + rlp_path, + import_config, + provider_factory.clone(), + &config, + evm_config, + consensus, + ) + .await?; + + info!( + target: "test", + "Imported {} blocks and {} transactions for node {}", + result.total_imported_blocks, + result.total_imported_txns, + idx + ); + + debug!(target: "e2e::import", + "Import result for node {}: decoded {} blocks, imported {} blocks, complete: {}", + idx, + result.total_decoded_blocks, + result.total_imported_blocks, + result.is_complete() + ); + + if result.total_decoded_blocks != result.total_imported_blocks { + debug!(target: "e2e::import", + "Import block count mismatch: decoded {} != imported {}", + result.total_decoded_blocks, result.total_imported_blocks + ); + return Err(eyre::eyre!("Chain import block count mismatch for node {}", idx)); + } + + if result.total_decoded_txns != result.total_imported_txns { + debug!(target: "e2e::import", + "Import transaction count mismatch: decoded {} != imported {}", + result.total_decoded_txns, result.total_imported_txns + ); + return Err(eyre::eyre!("Chain import transaction count mismatch for node {}", idx)); + } + + // Verify the database was properly initialized by checking stage checkpoints + { + let provider = provider_factory.database_provider_ro()?; + let headers_checkpoint = provider.get_stage_checkpoint(StageId::Headers)?; + if headers_checkpoint.is_none() { + return Err(eyre::eyre!("Headers stage checkpoint is missing after import!")); + } + debug!(target: "e2e::import", "Headers stage checkpoint after import: {headers_checkpoint:?}"); + drop(provider); + } + + // IMPORTANT: We need to properly flush and close the static files provider + // The static files provider may have open file handles that need to be closed + // before we can reopen the database in the node launcher + { + let static_file_provider = provider_factory.static_file_provider(); + // This will ensure all static file writers are properly closed + drop(static_file_provider); + } + + // Close all database handles to release locks before launching the node + drop(provider_factory); + drop(db); + + // Give the OS a moment to release file locks + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + + // Now launch the node with the pre-populated datadir + debug!(target: "e2e::import", "Launching node with datadir: {:?}", datadir); + + // Use the testing_node_with_datadir method which properly handles opening existing + // databases + let node = EthereumNode::default(); + + let NodeHandle { node, node_exit_future: _ } = NodeBuilder::new(node_config.clone()) + .testing_node_with_datadir(exec.clone(), datadir.clone()) + .with_types_and_provider::>() + .with_components(node.components_builder()) + .with_add_ons(node.add_ons()) + .launch_with_fn(|builder| { + let launcher = EngineNodeLauncher::new( + builder.task_executor().clone(), + builder.config().datadir(), + tree_config.clone(), + ); + builder.launch_with(launcher) + }) + .await?; + + let node_ctx = NodeTestContext::new(node, attributes_generator).await?; + + nodes.push(node_ctx); + temp_dirs.push(temp_dir); // Keep temp dir alive + } + + Ok(ChainImportResult { + nodes, + task_manager: tasks, + wallet: crate::Wallet::default().with_chain_id(chain_spec.chain.id()), + _temp_dirs: temp_dirs, + }) +} + +/// Helper to load forkchoice state from a JSON file +pub fn load_forkchoice_state(path: &Path) -> eyre::Result { + let json_str = std::fs::read_to_string(path)?; + let fcu_data: serde_json::Value = serde_json::from_str(&json_str)?; + + // The headfcu.json file contains a JSON-RPC request with the forkchoice state in params[0] + let state = &fcu_data["params"][0]; + Ok(alloy_rpc_types_engine::ForkchoiceState { + head_block_hash: state["headBlockHash"] + .as_str() + .ok_or_else(|| eyre::eyre!("missing headBlockHash"))? + .parse()?, + safe_block_hash: state["safeBlockHash"] + .as_str() + .ok_or_else(|| eyre::eyre!("missing safeBlockHash"))? + .parse()?, + finalized_block_hash: state["finalizedBlockHash"] + .as_str() + .ok_or_else(|| eyre::eyre!("missing finalizedBlockHash"))? + .parse()?, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_rlp_utils::{create_fcu_json, generate_test_blocks, write_blocks_to_rlp}; + use reth_chainspec::{ChainSpecBuilder, MAINNET}; + use reth_db::mdbx::DatabaseArguments; + use reth_payload_builder::EthPayloadBuilderAttributes; + use reth_primitives::SealedBlock; + use reth_provider::{ + test_utils::MockNodeTypesWithDB, BlockHashReader, BlockNumReader, BlockReaderIdExt, + }; + use std::path::PathBuf; + + #[tokio::test] + async fn test_stage_checkpoints_persistence() { + // This test specifically verifies that stage checkpoints are persisted correctly + // when reopening the database + reth_tracing::init_test_tracing(); + + let chain_spec = Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis( + serde_json::from_str(include_str!("testsuite/assets/genesis.json")).unwrap(), + ) + .london_activated() + .shanghai_activated() + .build(), + ); + + // Generate test blocks + let test_blocks = generate_test_blocks(&chain_spec, 5); + + // Create temporary files for RLP data + let temp_dir = tempfile::tempdir().expect("Failed to create temp dir"); + let rlp_path = temp_dir.path().join("test_chain.rlp"); + write_blocks_to_rlp(&test_blocks, &rlp_path).expect("Failed to write RLP data"); + + // Create a persistent datadir that won't be deleted + let datadir = temp_dir.path().join("datadir"); + std::fs::create_dir_all(&datadir).unwrap(); + let db_path = datadir.join("db"); + let static_files_path = datadir.join("static_files"); + + // Import the chain + { + let db_env = reth_db::init_db(&db_path, DatabaseArguments::default()).unwrap(); + let db = Arc::new(db_env); + + let provider_factory: ProviderFactory< + NodeTypesWithDBAdapter>, + > = ProviderFactory::new( + db.clone(), + chain_spec.clone(), + reth_provider::providers::StaticFileProvider::read_write(static_files_path.clone()) + .unwrap(), + ); + + // Initialize genesis + reth_db_common::init::init_genesis(&provider_factory).unwrap(); + + // Import the chain data + let import_config = ImportConfig::default(); + let config = Config::default(); + let evm_config = reth_node_ethereum::EthEvmConfig::new(chain_spec.clone()); + // Use NoopConsensus to skip gas limit validation for test imports + let consensus = reth_consensus::noop::NoopConsensus::arc(); + + let result = import_blocks_from_file( + &rlp_path, + import_config, + provider_factory.clone(), + &config, + evm_config, + consensus, + ) + .await + .unwrap(); + + assert_eq!(result.total_decoded_blocks, 5); + assert_eq!(result.total_imported_blocks, 5); + + // Verify stage checkpoints exist + let provider = provider_factory.database_provider_ro().unwrap(); + let headers_checkpoint = provider.get_stage_checkpoint(StageId::Headers).unwrap(); + assert!(headers_checkpoint.is_some(), "Headers checkpoint should exist after import"); + assert_eq!( + headers_checkpoint.unwrap().block_number, + 5, + "Headers checkpoint should be at block 5" + ); + drop(provider); + + // Properly close static files to release all file handles + let static_file_provider = provider_factory.static_file_provider(); + drop(static_file_provider); + + drop(provider_factory); + drop(db); + } + + // Give the OS a moment to release file locks + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + + // Now reopen the database and verify checkpoints are still there + { + let db_env = reth_db::init_db(&db_path, DatabaseArguments::default()).unwrap(); + let db = Arc::new(db_env); + + let provider_factory: ProviderFactory< + NodeTypesWithDBAdapter>, + > = ProviderFactory::new( + db, + chain_spec.clone(), + reth_provider::providers::StaticFileProvider::read_only(static_files_path, false) + .unwrap(), + ); + + let provider = provider_factory.database_provider_ro().unwrap(); + + // Check that stage checkpoints are still present + let headers_checkpoint = provider.get_stage_checkpoint(StageId::Headers).unwrap(); + assert!( + headers_checkpoint.is_some(), + "Headers checkpoint should still exist after reopening database" + ); + assert_eq!( + headers_checkpoint.unwrap().block_number, + 5, + "Headers checkpoint should still be at block 5" + ); + + // Verify we can read blocks + let block_5_hash = provider.block_hash(5).unwrap(); + assert!(block_5_hash.is_some(), "Block 5 should exist in database"); + assert_eq!(block_5_hash.unwrap(), test_blocks[4].hash(), "Block 5 hash should match"); + + // Check all stage checkpoints + debug!(target: "e2e::import", "All stage checkpoints after reopening:"); + for stage in StageId::ALL { + let checkpoint = provider.get_stage_checkpoint(stage).unwrap(); + debug!(target: "e2e::import", " Stage {stage:?}: {checkpoint:?}"); + } + } + } + + /// Helper to create test chain spec + fn create_test_chain_spec() -> Arc { + Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis( + serde_json::from_str(include_str!("testsuite/assets/genesis.json")).unwrap(), + ) + .london_activated() + .shanghai_activated() + .build(), + ) + } + + /// Helper to setup test blocks and write to RLP + async fn setup_test_blocks_and_rlp( + chain_spec: &ChainSpec, + block_count: u64, + temp_dir: &Path, + ) -> (Vec, PathBuf) { + let test_blocks = generate_test_blocks(chain_spec, block_count); + assert_eq!( + test_blocks.len(), + block_count as usize, + "Should have generated expected blocks" + ); + + let rlp_path = temp_dir.join("test_chain.rlp"); + write_blocks_to_rlp(&test_blocks, &rlp_path).expect("Failed to write RLP data"); + + let rlp_size = std::fs::metadata(&rlp_path).expect("RLP file should exist").len(); + debug!(target: "e2e::import", "Wrote RLP file with size: {rlp_size} bytes"); + + (test_blocks, rlp_path) + } + + #[tokio::test] + async fn test_import_blocks_only() { + // Tests just the block import functionality without full node setup + reth_tracing::init_test_tracing(); + + let chain_spec = create_test_chain_spec(); + let temp_dir = tempfile::tempdir().expect("Failed to create temp dir"); + let (test_blocks, rlp_path) = + setup_test_blocks_and_rlp(&chain_spec, 10, temp_dir.path()).await; + + // Create a test database + let datadir = temp_dir.path().join("datadir"); + std::fs::create_dir_all(&datadir).unwrap(); + let db_path = datadir.join("db"); + let db_env = reth_db::init_db(&db_path, DatabaseArguments::default()).unwrap(); + let db = Arc::new(reth_db::test_utils::TempDatabase::new(db_env, db_path)); + + // Create static files path + let static_files_path = datadir.join("static_files"); + + // Create a provider factory + let provider_factory: ProviderFactory = ProviderFactory::new( + db.clone(), + chain_spec.clone(), + reth_provider::providers::StaticFileProvider::read_write(static_files_path).unwrap(), + ); + + // Initialize genesis + reth_db_common::init::init_genesis(&provider_factory).unwrap(); + + // Import the chain data + let import_config = ImportConfig::default(); + let config = Config::default(); + let evm_config = reth_node_ethereum::EthEvmConfig::new(chain_spec.clone()); + // Use NoopConsensus to skip gas limit validation for test imports + let consensus = reth_consensus::noop::NoopConsensus::arc(); + + let result = import_blocks_from_file( + &rlp_path, + import_config, + provider_factory.clone(), + &config, + evm_config, + consensus, + ) + .await + .unwrap(); + + debug!(target: "e2e::import", + "Import result: decoded {} blocks, imported {} blocks", + result.total_decoded_blocks, result.total_imported_blocks + ); + + // Verify the import was successful + assert_eq!(result.total_decoded_blocks, 10); + assert_eq!(result.total_imported_blocks, 10); + assert_eq!(result.total_decoded_txns, 0); + assert_eq!(result.total_imported_txns, 0); + + // Verify we can read the imported blocks + let provider = provider_factory.database_provider_ro().unwrap(); + let latest_block = provider.last_block_number().unwrap(); + assert_eq!(latest_block, 10, "Should have imported up to block 10"); + + let block_10_hash = provider.block_hash(10).unwrap().expect("Block 10 should exist"); + assert_eq!(block_10_hash, test_blocks[9].hash(), "Block 10 hash should match"); + } + + #[tokio::test] + async fn test_import_with_node_integration() { + // Tests the full integration with node setup, forkchoice updates, and syncing + reth_tracing::init_test_tracing(); + + let chain_spec = create_test_chain_spec(); + let temp_dir = tempfile::tempdir().expect("Failed to create temp dir"); + let (test_blocks, rlp_path) = + setup_test_blocks_and_rlp(&chain_spec, 10, temp_dir.path()).await; + + // Create FCU data for the tip + let tip = test_blocks.last().expect("Should have generated blocks"); + let fcu_path = temp_dir.path().join("test_fcu.json"); + std::fs::write(&fcu_path, create_fcu_json(tip).to_string()) + .expect("Failed to write FCU data"); + + // Setup nodes with imported chain + let result = setup_engine_with_chain_import( + 1, + chain_spec, + false, + TreeConfig::default(), + &rlp_path, + |_| EthPayloadBuilderAttributes::default(), + ) + .await + .expect("Failed to setup nodes with chain import"); + + // Load and apply forkchoice state + let fcu_state = load_forkchoice_state(&fcu_path).expect("Failed to load forkchoice state"); + + let node = &result.nodes[0]; + + // Send forkchoice update to make the imported chain canonical + node.update_forkchoice(fcu_state.finalized_block_hash, fcu_state.head_block_hash) + .await + .expect("Failed to update forkchoice"); + + // Wait for the node to sync to the head + node.sync_to(fcu_state.head_block_hash).await.expect("Failed to sync to head"); + + // Verify the chain tip + let latest = node + .inner + .provider + .sealed_header_by_id(alloy_eips::BlockId::latest()) + .expect("Failed to get latest header") + .expect("No latest header found"); + + assert_eq!( + latest.hash(), + fcu_state.head_block_hash, + "Chain tip does not match expected head" + ); + } +} diff --git a/crates/e2e-test-utils/src/test_rlp_utils.rs b/crates/e2e-test-utils/src/test_rlp_utils.rs new file mode 100644 index 00000000000..bcfb9faa9d8 --- /dev/null +++ b/crates/e2e-test-utils/src/test_rlp_utils.rs @@ -0,0 +1,184 @@ +//! Utilities for creating and writing RLP test data + +use alloy_consensus::{constants::EMPTY_WITHDRAWALS, BlockHeader, Header}; +use alloy_eips::eip4895::Withdrawals; +use alloy_primitives::{Address, B256, B64, U256}; +use alloy_rlp::Encodable; +use reth_chainspec::{ChainSpec, EthereumHardforks}; +use reth_ethereum_primitives::{Block, BlockBody}; +use reth_primitives::SealedBlock; +use reth_primitives_traits::Block as BlockTrait; +use std::{io::Write, path::Path}; +use tracing::debug; + +/// Generate test blocks for a given chain spec +pub fn generate_test_blocks(chain_spec: &ChainSpec, count: u64) -> Vec { + let mut blocks: Vec = Vec::new(); + let genesis_header = chain_spec.sealed_genesis_header(); + let mut parent_hash = genesis_header.hash(); + let mut parent_number = genesis_header.number(); + let mut parent_base_fee = genesis_header.base_fee_per_gas; + let mut parent_gas_limit = genesis_header.gas_limit; + + debug!(target: "e2e::import", + "Genesis header base fee: {:?}, gas limit: {}, state root: {:?}", + parent_base_fee, + parent_gas_limit, + genesis_header.state_root() + ); + + for i in 1..=count { + // Create a simple header + let mut header = Header { + parent_hash, + number: parent_number + 1, + gas_limit: parent_gas_limit, // Use parent's gas limit + gas_used: 0, // Empty blocks use no gas + timestamp: genesis_header.timestamp() + i * 12, // 12 second blocks + beneficiary: Address::ZERO, + receipts_root: alloy_consensus::constants::EMPTY_RECEIPTS, + logs_bloom: Default::default(), + difficulty: U256::from(1), // Will be overridden for post-merge + // Use the same state root as parent for now (empty state changes) + state_root: if i == 1 { + genesis_header.state_root() + } else { + blocks.last().unwrap().state_root + }, + transactions_root: alloy_consensus::constants::EMPTY_TRANSACTIONS, + ommers_hash: alloy_consensus::constants::EMPTY_OMMER_ROOT_HASH, + mix_hash: B256::ZERO, + nonce: B64::from(0u64), + extra_data: Default::default(), + base_fee_per_gas: None, + withdrawals_root: None, + blob_gas_used: None, + excess_blob_gas: None, + parent_beacon_block_root: None, + requests_hash: None, + }; + + // Set required fields based on chain spec + if chain_spec.is_london_active_at_block(header.number) { + // Calculate base fee based on parent block + if let Some(parent_fee) = parent_base_fee { + // For the first block, we need to use the exact expected base fee + // The consensus rules expect it to be calculated from the genesis + let (parent_gas_used, parent_gas_limit) = if i == 1 { + // Genesis block parameters + (genesis_header.gas_used, genesis_header.gas_limit) + } else { + let last_block = blocks.last().unwrap(); + (last_block.gas_used, last_block.gas_limit) + }; + header.base_fee_per_gas = Some(alloy_eips::calc_next_block_base_fee( + parent_gas_used, + parent_gas_limit, + parent_fee, + chain_spec.base_fee_params_at_timestamp(header.timestamp), + )); + debug!(target: "e2e::import", "Block {} calculated base fee: {:?} (parent gas used: {}, parent gas limit: {}, parent base fee: {})", + i, header.base_fee_per_gas, parent_gas_used, parent_gas_limit, parent_fee); + parent_base_fee = header.base_fee_per_gas; + } + } + + // For post-merge blocks + if chain_spec.is_paris_active_at_block(header.number) { + header.difficulty = U256::ZERO; + header.nonce = B64::ZERO; + } + + // For post-shanghai blocks + if chain_spec.is_shanghai_active_at_timestamp(header.timestamp) { + header.withdrawals_root = Some(EMPTY_WITHDRAWALS); + } + + // For post-cancun blocks + if chain_spec.is_cancun_active_at_timestamp(header.timestamp) { + header.blob_gas_used = Some(0); + header.excess_blob_gas = Some(0); + header.parent_beacon_block_root = Some(B256::ZERO); + } + + // Create an empty block body + let body = BlockBody { + transactions: vec![], + ommers: vec![], + withdrawals: header.withdrawals_root.is_some().then(Withdrawals::default), + }; + + // Create the block + let block = Block { header: header.clone(), body: body.clone() }; + let sealed_block = BlockTrait::seal_slow(block); + + debug!(target: "e2e::import", + "Generated block {} with hash {:?}", + sealed_block.number(), + sealed_block.hash() + ); + debug!(target: "e2e::import", + " Body has {} transactions, {} ommers, withdrawals: {}", + body.transactions.len(), + body.ommers.len(), + body.withdrawals.is_some() + ); + + // Update parent for next iteration + parent_hash = sealed_block.hash(); + parent_number = sealed_block.number(); + parent_gas_limit = sealed_block.gas_limit; + if header.base_fee_per_gas.is_some() { + parent_base_fee = header.base_fee_per_gas; + } + + blocks.push(sealed_block); + } + + blocks +} + +/// Write blocks to RLP file +pub fn write_blocks_to_rlp(blocks: &[SealedBlock], path: &Path) -> std::io::Result<()> { + let mut file = std::fs::File::create(path)?; + let mut total_bytes = 0; + + for (i, block) in blocks.iter().enumerate() { + // Convert SealedBlock to Block before encoding + let block_for_encoding = block.clone().unseal(); + + let mut buf = Vec::new(); + block_for_encoding.encode(&mut buf); + debug!(target: "e2e::import", + "Block {} has {} transactions, encoded to {} bytes", + i, + block.body().transactions.len(), + buf.len() + ); + + // Debug: check what's in the encoded data + if buf.len() < 20 { + debug!(target: "e2e::import", " Raw bytes: {:?}", &buf); + } else { + debug!(target: "e2e::import", " First 20 bytes: {:?}", &buf[..20]); + } + + total_bytes += buf.len(); + file.write_all(&buf)?; + } + + file.flush()?; + debug!(target: "e2e::import", "Total RLP bytes written: {total_bytes}"); + Ok(()) +} + +/// Create FCU JSON for the tip of the chain +pub fn create_fcu_json(tip: &SealedBlock) -> serde_json::Value { + serde_json::json!({ + "params": [{ + "headBlockHash": format!("0x{:x}", tip.hash()), + "safeBlockHash": format!("0x{:x}", tip.hash()), + "finalizedBlockHash": format!("0x{:x}", tip.hash()), + }] + }) +} diff --git a/crates/e2e-test-utils/src/testsuite/README.md b/crates/e2e-test-utils/src/testsuite/README.md new file mode 100644 index 00000000000..b9e4927de88 --- /dev/null +++ b/crates/e2e-test-utils/src/testsuite/README.md @@ -0,0 +1,787 @@ +# E2E Test Suite Framework + +This directory contains the framework for writing end-to-end (e2e) tests in Reth. The framework provides utilities for setting up test environments, performing actions, and verifying blockchain behavior. + +## Test Organization + +E2E tests using this framework follow a consistent structure across the codebase: + +### Directory Structure +Each crate that requires e2e tests should organize them as follows: +``` +/ +├── src/ +│ └── ... (implementation code) +├── tests/ +│ └── e2e-testsuite/ +│ └── main.rs (or other test files) +└── Cargo.toml +``` + +### Cargo.toml Configuration +In your crate's `Cargo.toml`, define the e2e test binary: +```toml +[[test]] +name = "e2e_testsuite" +path = "tests/e2e-testsuite/main.rs" +harness = true +``` + +**Important**: The test binary MUST be named `e2e_testsuite` to be properly recognized by the nextest filter and CI workflows. + +## Running E2E Tests + +### Run all e2e tests across the workspace +```bash +cargo nextest run --workspace \ + --exclude 'example-*' \ + --exclude 'exex-subscription' \ + --exclude 'reth-bench' \ + --exclude 'ef-tests' \ + --exclude 'op-reth' \ + --exclude 'reth' \ + -E 'binary(e2e_testsuite)' +``` + +Note: The `--exclude` flags prevent compilation of crates that don't contain e2e tests (examples, benchmarks, binaries, and EF tests), significantly reducing build time. + +### Run e2e tests for a specific crate +```bash +cargo nextest run -p -E 'binary(e2e_testsuite)' +``` + +### Run with additional features +```bash +cargo nextest run --locked --features "asm-keccak" --workspace -E 'binary(e2e_testsuite)' +``` + +### Run a specific test +```bash +cargo nextest run --workspace -E 'binary(e2e_testsuite) and test(test_name)' +``` + +## Writing E2E Tests + +Tests use the framework components from this directory: + +```rust +use reth_e2e_test_utils::{setup_import, Environment, TestBuilder}; + +#[tokio::test] +async fn test_example() -> eyre::Result<()> { + // Create test environment + let (mut env, mut handle) = TestBuilder::new() + .build() + .await?; + + // Perform test actions... + + Ok(()) +} +``` + +## Framework Components + +- **Environment**: Core test environment managing nodes and network state +- **TestBuilder**: Builder pattern for configuring test environments +- **Actions** (`actions/`): Pre-built test actions like block production, reorgs, etc. +- **Setup utilities**: Helper functions for common test scenarios + +## CI Integration + +E2E tests run in a dedicated GitHub Actions workflow (`.github/workflows/e2e.yml`) with: +- Extended timeouts (2 minutes per test, with 3 retries) +- Isolation from unit and integration tests +- Parallel execution support + +## Nextest Configuration + +The framework uses custom nextest settings (`.config/nextest.toml`): +```toml +[[profile.default.overrides]] +filter = "binary(e2e_testsuite)" +slow-timeout = { period = "2m", terminate-after = 3 } +``` + +This ensures all e2e tests get appropriate timeouts for complex blockchain operations. + +## E2E Test Actions Reference + +This section provides comprehensive documentation for all available end-to-end (e2e) test actions in the Reth testing framework. These actions enable developers to write complex blockchain integration tests by performing operations and making assertions in a single step. + +### Overview + +The e2e test framework provides a rich set of actions organized into several categories: + +- **Block Production Actions**: Create and manage blocks +- **Fork Management Actions**: Handle blockchain forks and reorgs +- **Node Operations**: Multi-node coordination and validation +- **Engine API Actions**: Test execution layer interactions +- **RPC Compatibility Actions**: Test RPC methods against execution-apis test data +- **Custom FCU Actions**: Advanced forkchoice update scenarios + +### Action Categories + +#### Block Production Actions + +##### `AssertMineBlock` +Mines a single block with specified transactions and verifies successful creation. + +```rust +use reth_e2e_test_utils::testsuite::actions::AssertMineBlock; + +let action = AssertMineBlock::new( + node_idx, // Node index to mine on + transactions, // Vec - transactions to include + expected_hash, // Option - expected block hash + payload_attributes, // Engine::PayloadAttributes +); +``` + +##### `ProduceBlocks` +Produces a sequence of blocks using the available clients. + +```rust +use reth_e2e_test_utils::testsuite::actions::ProduceBlocks; + +let action = ProduceBlocks::new(num_blocks); // Number of blocks to produce +``` + +##### `ProduceBlocksLocally` +Produces blocks locally without broadcasting to other nodes. + +```rust +use reth_e2e_test_utils::testsuite::actions::ProduceBlocksLocally; + +let action = ProduceBlocksLocally::new(num_blocks); +``` + +##### `ProduceInvalidBlocks` +Produces a sequence of blocks where some blocks are intentionally invalid. + +```rust +use reth_e2e_test_utils::testsuite::actions::ProduceInvalidBlocks; + +let action = ProduceInvalidBlocks::new( + num_blocks, // Total number of blocks + invalid_indices, // HashSet - indices of invalid blocks +); + +// Or create with a single invalid block +let action = ProduceInvalidBlocks::with_invalid_at(num_blocks, invalid_index); +``` + +##### `PickNextBlockProducer` +Selects the next block producer based on round-robin selection. + +```rust +use reth_e2e_test_utils::testsuite::actions::PickNextBlockProducer; + +let action = PickNextBlockProducer::new(); +``` + +##### `GeneratePayloadAttributes` +Generates and stores payload attributes for the next block. + +```rust +use reth_e2e_test_utils::testsuite::actions::GeneratePayloadAttributes; + +let action = GeneratePayloadAttributes::new(); +``` + +##### `GenerateNextPayload` +Generates the next execution payload using stored attributes. + +```rust +use reth_e2e_test_utils::testsuite::actions::GenerateNextPayload; + +let action = GenerateNextPayload::new(); +``` + +##### `BroadcastLatestForkchoice` +Broadcasts the latest fork choice state to all clients. + +```rust +use reth_e2e_test_utils::testsuite::actions::BroadcastLatestForkchoice; + +let action = BroadcastLatestForkchoice::new(); +``` + +##### `BroadcastNextNewPayload` +Broadcasts the next new payload to nodes. + +```rust +use reth_e2e_test_utils::testsuite::actions::BroadcastNextNewPayload; + +// Broadcast to all nodes +let action = BroadcastNextNewPayload::new(); + +// Broadcast only to active node +let action = BroadcastNextNewPayload::with_active_node(); +``` + +##### `CheckPayloadAccepted` +Verifies that a broadcasted payload has been accepted by nodes. + +```rust +use reth_e2e_test_utils::testsuite::actions::CheckPayloadAccepted; + +let action = CheckPayloadAccepted::new(); +``` + +##### `UpdateBlockInfo` +Syncs environment state with the node's canonical chain via RPC. + +```rust +use reth_e2e_test_utils::testsuite::actions::UpdateBlockInfo; + +let action = UpdateBlockInfo::new(); +``` + +##### `UpdateBlockInfoToLatestPayload` +Updates environment state using the locally produced payload. + +```rust +use reth_e2e_test_utils::testsuite::actions::UpdateBlockInfoToLatestPayload; + +let action = UpdateBlockInfoToLatestPayload::new(); +``` + +##### `MakeCanonical` +Makes the current latest block canonical by broadcasting a forkchoice update. + +```rust +use reth_e2e_test_utils::testsuite::actions::MakeCanonical; + +// Broadcast to all nodes +let action = MakeCanonical::new(); + +// Only apply to active node +let action = MakeCanonical::with_active_node(); +``` + +##### `CaptureBlock` +Captures the current block and tags it with a name for later reference. + +```rust +use reth_e2e_test_utils::testsuite::actions::CaptureBlock; + +let action = CaptureBlock::new("block_tag"); +``` + +#### Fork Management Actions + +##### `CreateFork` +Creates a fork from a specified block and produces blocks on top. + +```rust +use reth_e2e_test_utils::testsuite::actions::CreateFork; + +// Create fork from block number +let action = CreateFork::new(fork_base_block, num_blocks); + +// Create fork from tagged block +let action = CreateFork::new_from_tag("block_tag", num_blocks); +``` + +##### `SetForkBase` +Sets the fork base block in the environment. + +```rust +use reth_e2e_test_utils::testsuite::actions::SetForkBase; + +let action = SetForkBase::new(fork_base_block); +``` + +##### `SetForkBaseFromBlockInfo` +Sets the fork base from existing block information. + +```rust +use reth_e2e_test_utils::testsuite::actions::SetForkBaseFromBlockInfo; + +let action = SetForkBaseFromBlockInfo::new(block_info); +``` + +##### `ValidateFork` +Validates that a fork was created correctly. + +```rust +use reth_e2e_test_utils::testsuite::actions::ValidateFork; + +let action = ValidateFork::new(fork_base_number); +``` + +#### Reorg Actions + +##### `ReorgTo` +Performs a reorg by setting a new head block as canonical. + +```rust +use reth_e2e_test_utils::testsuite::actions::ReorgTo; + +// Reorg to specific block hash +let action = ReorgTo::new(target_hash); + +// Reorg to tagged block +let action = ReorgTo::new_from_tag("block_tag"); +``` + +##### `SetReorgTarget` +Sets the reorg target block in the environment. + +```rust +use reth_e2e_test_utils::testsuite::actions::SetReorgTarget; + +let action = SetReorgTarget::new(target_block_info); +``` + +#### Node Operations + +##### `SelectActiveNode` +Selects which node should be active for subsequent operations. + +```rust +use reth_e2e_test_utils::testsuite::actions::SelectActiveNode; + +let action = SelectActiveNode::new(node_idx); +``` + +##### `CompareNodeChainTips` +Compares chain tips between two nodes. + +```rust +use reth_e2e_test_utils::testsuite::actions::CompareNodeChainTips; + +// Expect nodes to have the same chain tip +let action = CompareNodeChainTips::expect_same(node_a, node_b); + +// Expect nodes to have different chain tips +let action = CompareNodeChainTips::expect_different(node_a, node_b); +``` + +##### `CaptureBlockOnNode` +Captures a block with a tag, associating it with a specific node. + +```rust +use reth_e2e_test_utils::testsuite::actions::CaptureBlockOnNode; + +let action = CaptureBlockOnNode::new("tag_name", node_idx); +``` + +##### `ValidateBlockTag` +Validates that a block tag exists and optionally came from a specific node. + +```rust +use reth_e2e_test_utils::testsuite::actions::ValidateBlockTag; + +// Just validate tag exists +let action = ValidateBlockTag::exists("tag_name"); + +// Validate tag came from specific node +let action = ValidateBlockTag::from_node("tag_name", node_idx); +``` + +##### `WaitForSync` +Waits for two nodes to sync and have the same chain tip. + +```rust +use reth_e2e_test_utils::testsuite::actions::WaitForSync; + +// With default timeouts (30s timeout, 1s poll interval) +let action = WaitForSync::new(node_a, node_b); + +// With custom timeouts +let action = WaitForSync::new(node_a, node_b) + .with_timeout(60) // 60 second timeout + .with_poll_interval(2); // 2 second poll interval +``` + +##### `AssertChainTip` +Asserts that the current chain tip is at a specific block number. + +```rust +use reth_e2e_test_utils::testsuite::actions::AssertChainTip; + +let action = AssertChainTip::new(expected_block_number); +``` + +#### Engine API Actions + +##### `SendNewPayload` +Sends a newPayload request to a specific node. + +```rust +use reth_e2e_test_utils::testsuite::actions::{SendNewPayload, ExpectedPayloadStatus}; + +let action = SendNewPayload::new( + node_idx, // Target node index + block_number, // Block number to send + source_node_idx, // Source node to get block from + ExpectedPayloadStatus::Valid, // Expected status +); +``` + +##### `SendNewPayloads` +Sends multiple blocks to a node in a specific order. + +```rust +use reth_e2e_test_utils::testsuite::actions::SendNewPayloads; + +let action = SendNewPayloads::new() + .with_target_node(node_idx) + .with_source_node(source_idx) + .with_start_block(1) + .with_total_blocks(5); + +// Send in reverse order +let action = SendNewPayloads::new() + .with_target_node(node_idx) + .with_source_node(source_idx) + .with_start_block(1) + .with_total_blocks(5) + .in_reverse_order(); + +// Send specific block numbers +let action = SendNewPayloads::new() + .with_target_node(node_idx) + .with_source_node(source_idx) + .with_block_numbers(vec![1, 3, 5]); +``` + +#### RPC Compatibility Actions + +##### `RunRpcCompatTests` +Runs RPC compatibility tests from execution-apis test data. + +```rust +use reth_rpc_e2e_tests::rpc_compat::RunRpcCompatTests; + +// Test specific RPC methods +let action = RunRpcCompatTests::new( + vec!["eth_getLogs".to_string(), "eth_syncing".to_string()], + test_data_path, +); + +// With fail-fast option +let action = RunRpcCompatTests::new(methods, test_data_path) + .with_fail_fast(true); +``` + +##### `InitializeFromExecutionApis` +Initializes the chain from execution-apis test data. + +```rust +use reth_rpc_e2e_tests::rpc_compat::InitializeFromExecutionApis; + +// With default paths +let action = InitializeFromExecutionApis::new(); + +// With custom paths +let action = InitializeFromExecutionApis::new() + .with_chain_rlp("path/to/chain.rlp") + .with_fcu_json("path/to/headfcu.json"); +``` + +#### Custom FCU Actions + +##### `SendForkchoiceUpdate` +Sends a custom forkchoice update with specific finalized, safe, and head blocks. + +```rust +use reth_e2e_test_utils::testsuite::actions::{SendForkchoiceUpdate, BlockReference}; + +let action = SendForkchoiceUpdate::new( + BlockReference::Hash(finalized_hash), + BlockReference::Hash(safe_hash), + BlockReference::Hash(head_hash), +); + +// With expected status +let action = SendForkchoiceUpdate::new( + BlockReference::Tag("finalized"), + BlockReference::Tag("safe"), + BlockReference::Tag("head"), +).with_expected_status(PayloadStatusEnum::Valid); + +// Send to specific node +let action = SendForkchoiceUpdate::new( + BlockReference::Latest, + BlockReference::Latest, + BlockReference::Latest, +).with_node_idx(node_idx); +``` + +##### `FinalizeBlock` +Finalizes a specific block with a given head. + +```rust +use reth_e2e_test_utils::testsuite::actions::FinalizeBlock; + +let action = FinalizeBlock::new(BlockReference::Hash(block_hash)); + +// With different head +let action = FinalizeBlock::new(BlockReference::Hash(block_hash)) + .with_head(BlockReference::Hash(head_hash)); + +// Send to specific node +let action = FinalizeBlock::new(BlockReference::Tag("block_tag")) + .with_node_idx(node_idx); +``` + +#### FCU Status Testing Actions + +##### `TestFcuToTag` +Tests forkchoice update to a tagged block with expected status. + +```rust +use reth_e2e_test_utils::testsuite::actions::TestFcuToTag; + +let action = TestFcuToTag::new("block_tag", PayloadStatusEnum::Valid); +``` + +##### `ExpectFcuStatus` +Expects a specific FCU status when targeting a tagged block. + +```rust +use reth_e2e_test_utils::testsuite::actions::ExpectFcuStatus; + +// Expect valid status +let action = ExpectFcuStatus::valid("block_tag"); + +// Expect invalid status +let action = ExpectFcuStatus::invalid("block_tag"); + +// Expect syncing status +let action = ExpectFcuStatus::syncing("block_tag"); + +// Expect accepted status +let action = ExpectFcuStatus::accepted("block_tag"); +``` + +##### `ValidateCanonicalTag` +Validates that a tagged block remains canonical. + +```rust +use reth_e2e_test_utils::testsuite::actions::ValidateCanonicalTag; + +let action = ValidateCanonicalTag::new("block_tag"); +``` + +### Block Reference Types + +#### `BlockReference` +Used to reference blocks in various actions: + +```rust +use reth_e2e_test_utils::testsuite::actions::BlockReference; + +// Direct block hash +let reference = BlockReference::Hash(block_hash); + +// Tagged block reference +let reference = BlockReference::Tag("block_tag".to_string()); + +// Latest block on active node +let reference = BlockReference::Latest; +``` + +#### `ForkBase` +Used to specify fork base in fork creation: + +```rust +use reth_e2e_test_utils::testsuite::actions::ForkBase; + +// Block number +let fork_base = ForkBase::Number(block_number); + +// Tagged block +let fork_base = ForkBase::Tag("block_tag".to_string()); +``` + +#### `ReorgTarget` +Used to specify reorg targets: + +```rust +use reth_e2e_test_utils::testsuite::actions::ReorgTarget; + +// Direct block hash +let target = ReorgTarget::Hash(block_hash); + +// Tagged block reference +let target = ReorgTarget::Tag("block_tag".to_string()); +``` + +### Expected Payload Status + +#### `ExpectedPayloadStatus` +Used to specify expected payload status in engine API actions: + +```rust +use reth_e2e_test_utils::testsuite::actions::ExpectedPayloadStatus; + +// Expect valid payload +let status = ExpectedPayloadStatus::Valid; + +// Expect invalid payload +let status = ExpectedPayloadStatus::Invalid; + +// Expect syncing or accepted (buffered) +let status = ExpectedPayloadStatus::SyncingOrAccepted; +``` + +### Usage Examples + +#### Basic Block Production Test + +```rust +use reth_e2e_test_utils::testsuite::{ + actions::{ProduceBlocks, MakeCanonical, AssertChainTip}, + setup::{NetworkSetup, Setup}, + TestBuilder, +}; + +#[tokio::test] +async fn test_basic_block_production() -> eyre::Result<()> { + let setup = Setup::default() + .with_chain_spec(chain_spec) + .with_network(NetworkSetup::single_node()); + + let test = TestBuilder::new() + .with_setup(setup) + .with_action(ProduceBlocks::new(5)) + .with_action(MakeCanonical::new()) + .with_action(AssertChainTip::new(5)); + + test.run::().await?; + Ok(()) +} +``` + +#### Fork and Reorg Test + +```rust +use reth_e2e_test_utils::testsuite::{ + actions::{ProduceBlocks, CreateFork, CaptureBlock, ReorgTo, MakeCanonical}, + setup::{NetworkSetup, Setup}, + TestBuilder, +}; + +#[tokio::test] +async fn test_fork_and_reorg() -> eyre::Result<()> { + let setup = Setup::default() + .with_chain_spec(chain_spec) + .with_network(NetworkSetup::single_node()); + + let test = TestBuilder::new() + .with_setup(setup) + .with_action(ProduceBlocks::new(3)) // Produce blocks 1, 2, 3 + .with_action(MakeCanonical::new()) // Make main chain canonical + .with_action(CreateFork::new(1, 2)) // Fork from block 1, produce 2 blocks + .with_action(CaptureBlock::new("fork_tip")) // Tag the fork tip + .with_action(ReorgTo::new_from_tag("fork_tip")); // Reorg to fork tip + + test.run::().await?; + Ok(()) +} +``` + +#### Multi-Node Test + +```rust +use reth_e2e_test_utils::testsuite::{ + actions::{SelectActiveNode, ProduceBlocks, CompareNodeChainTips, CaptureBlockOnNode}, + setup::{NetworkSetup, Setup}, + TestBuilder, +}; + +#[tokio::test] +async fn test_multi_node_coordination() -> eyre::Result<()> { + let setup = Setup::default() + .with_chain_spec(chain_spec) + .with_network(NetworkSetup::multi_node(2)); // 2 nodes + + let test = TestBuilder::new() + .with_setup(setup) + .with_action(CompareNodeChainTips::expect_same(0, 1)) // Both start at genesis + .with_action(SelectActiveNode::new(0)) // Select node 0 + .with_action(ProduceBlocks::new(3)) // Produce blocks on node 0 + .with_action(CaptureBlockOnNode::new("node0_tip", 0)) // Tag node 0's tip + .with_action(CompareNodeChainTips::expect_same(0, 1)); // Verify sync + + test.run::().await?; + Ok(()) +} +``` + +#### Engine API Test + +```rust +use reth_e2e_test_utils::testsuite::{ + actions::{SendNewPayload, ExpectedPayloadStatus}, + setup::{NetworkSetup, Setup}, + TestBuilder, +}; + +#[tokio::test] +async fn test_engine_api() -> eyre::Result<()> { + let setup = Setup::default() + .with_chain_spec(chain_spec) + .with_network(NetworkSetup::multi_node(2)); + + let test = TestBuilder::new() + .with_setup(setup) + .with_action(SendNewPayload::new( + 1, // Target node + 1, // Block number + 0, // Source node + ExpectedPayloadStatus::Valid, // Expected status + )); + + test.run::().await?; + Ok(()) +} +``` + +#### RPC Compatibility Test + +```rust +use reth_e2e_test_utils::testsuite::{ + actions::{MakeCanonical, UpdateBlockInfo}, + setup::{NetworkSetup, Setup}, + TestBuilder, +}; +use reth_rpc_e2e_tests::rpc_compat::{InitializeFromExecutionApis, RunRpcCompatTests}; + +#[tokio::test] +async fn test_rpc_compatibility() -> eyre::Result<()> { + let test_data_path = "path/to/execution-apis/tests"; + + let setup = Setup::default() + .with_chain_spec(chain_spec) + .with_network(NetworkSetup::single_node()); + + let test = TestBuilder::new() + .with_setup_and_import(setup, "path/to/chain.rlp") + .with_action(UpdateBlockInfo::default()) + .with_action(InitializeFromExecutionApis::new() + .with_fcu_json("path/to/headfcu.json")) + .with_action(MakeCanonical::new()) + .with_action(RunRpcCompatTests::new( + vec!["eth_getLogs".to_string()], + test_data_path, + )); + + test.run::().await?; + Ok(()) +} +``` + +### Best Practices + +1. **Use Tagged Blocks**: Use `CaptureBlock` or `CaptureBlockOnNode` to tag important blocks for later reference in reorgs and forks. + +2. **Make Blocks Canonical**: After producing blocks, use `MakeCanonical` to ensure they become part of the canonical chain. + +3. **Update Block Info**: Use `UpdateBlockInfo` or `UpdateBlockInfoToLatestPayload` to keep the environment state synchronized with the node. + +4. **Multi-Node Coordination**: Use `SelectActiveNode` to control which node performs operations, and `CompareNodeChainTips` to verify synchronization. diff --git a/crates/e2e-test-utils/src/testsuite/actions.rs b/crates/e2e-test-utils/src/testsuite/actions.rs deleted file mode 100644 index 186849a26b9..00000000000 --- a/crates/e2e-test-utils/src/testsuite/actions.rs +++ /dev/null @@ -1,562 +0,0 @@ -//! Actions that can be performed in tests. - -use crate::testsuite::Environment; -use alloy_primitives::{Bytes, B256}; -use alloy_rpc_types_engine::{ - ExecutionPayloadV3, ForkchoiceState, PayloadAttributes, PayloadStatusEnum, -}; -use alloy_rpc_types_eth::{Block, Header, Receipt, Transaction}; -use eyre::Result; -use futures_util::future::BoxFuture; -use reth_node_api::{EngineTypes, PayloadTypes}; -use reth_rpc_api::clients::{EngineApiClient, EthApiClient}; -use std::{future::Future, marker::PhantomData, time::Duration}; -use tokio::time::sleep; -use tracing::debug; - -/// An action that can be performed on an instance. -/// -/// Actions execute operations and potentially make assertions in a single step. -/// The action name indicates what it does (e.g., `AssertMineBlock` would both -/// mine a block and assert it worked). -pub trait Action: Send + 'static { - /// Executes the action - fn execute<'a>(&'a mut self, env: &'a mut Environment) -> BoxFuture<'a, Result<()>>; -} - -/// Simplified action container for storage in tests -#[expect(missing_debug_implementations)] -pub struct ActionBox(Box>); - -impl ActionBox { - /// Constructor for [`ActionBox`]. - pub fn new>(action: A) -> Self { - Self(Box::new(action)) - } - - /// Executes an [`ActionBox`] with the given [`Environment`] reference. - pub async fn execute(mut self, env: &mut Environment) -> Result<()> { - self.0.execute(env).await - } -} - -/// Implementation of `Action` for any function/closure that takes an Environment -/// reference and returns a Future resolving to Result<()>. -/// -/// This allows using closures directly as actions with `.with_action(async move |env| {...})`. -impl Action for F -where - F: FnMut(&Environment) -> Fut + Send + 'static, - Fut: Future> + Send + 'static, -{ - fn execute<'a>(&'a mut self, env: &'a mut Environment) -> BoxFuture<'a, Result<()>> { - Box::pin(self(env)) - } -} - -/// Mine a single block with the given transactions and verify the block was created -/// successfully. -#[derive(Debug)] -pub struct AssertMineBlock -where - Engine: PayloadTypes, -{ - /// The node index to mine - pub node_idx: usize, - /// Transactions to include in the block - pub transactions: Vec, - /// Expected block hash (optional) - pub expected_hash: Option, - /// Block's payload attributes - // TODO: refactor once we have actions to generate payload attributes. - pub payload_attributes: Engine::PayloadAttributes, - /// Tracks engine type - _phantom: PhantomData, -} - -impl AssertMineBlock -where - Engine: PayloadTypes, -{ - /// Create a new `AssertMineBlock` action - pub fn new( - node_idx: usize, - transactions: Vec, - expected_hash: Option, - payload_attributes: Engine::PayloadAttributes, - ) -> Self { - Self { - node_idx, - transactions, - expected_hash, - payload_attributes, - _phantom: Default::default(), - } - } -} - -impl Action for AssertMineBlock -where - Engine: EngineTypes, -{ - fn execute<'a>(&'a mut self, env: &'a mut Environment) -> BoxFuture<'a, Result<()>> { - Box::pin(async move { - if self.node_idx >= env.node_clients.len() { - return Err(eyre::eyre!("Node index out of bounds: {}", self.node_idx)); - } - - let node_client = &env.node_clients[self.node_idx]; - let rpc_client = &node_client.rpc; - let engine_client = &node_client.engine; - - // get the latest block to use as parent - let latest_block = - EthApiClient::::block_by_number( - rpc_client, - alloy_eips::BlockNumberOrTag::Latest, - false, - ) - .await?; - - let latest_block = latest_block.ok_or_else(|| eyre::eyre!("Latest block not found"))?; - let parent_hash = latest_block.header.hash; - - debug!("Latest block hash: {parent_hash}"); - - // create a simple forkchoice state with the latest block as head - let fork_choice_state = ForkchoiceState { - head_block_hash: parent_hash, - safe_block_hash: parent_hash, - finalized_block_hash: parent_hash, - }; - - let fcu_result = EngineApiClient::::fork_choice_updated_v2( - engine_client, - fork_choice_state, - Some(self.payload_attributes.clone()), - ) - .await?; - - debug!("FCU result: {:?}", fcu_result); - - // check if we got a valid payload ID - match fcu_result.payload_status.status { - PayloadStatusEnum::Valid => { - if let Some(payload_id) = fcu_result.payload_id { - debug!("Got payload ID: {payload_id}"); - - // get the payload that was built - let _engine_payload = - EngineApiClient::::get_payload_v2(engine_client, payload_id) - .await?; - Ok(()) - } else { - Err(eyre::eyre!("No payload ID returned from forkchoiceUpdated")) - } - } - _ => Err(eyre::eyre!("Payload status not valid: {:?}", fcu_result.payload_status)), - } - }) - } -} -/// Pick the next block producer based on the latest block information. -#[derive(Debug, Default)] -pub struct PickNextBlockProducer {} - -impl PickNextBlockProducer { - /// Create a new `PickNextBlockProducer` action - pub const fn new() -> Self { - Self {} - } -} - -impl Action for PickNextBlockProducer -where - Engine: EngineTypes, -{ - fn execute<'a>(&'a mut self, env: &'a mut Environment) -> BoxFuture<'a, Result<()>> { - Box::pin(async move { - let num_clients = env.node_clients.len(); - if num_clients == 0 { - return Err(eyre::eyre!("No node clients available")); - } - - let latest_info = env - .latest_block_info - .as_ref() - .ok_or_else(|| eyre::eyre!("No latest block information available"))?; - - // Calculate the starting index based on the latest block number - let start_idx = ((latest_info.number + 1) % num_clients as u64) as usize; - - for i in 0..num_clients { - let idx = (start_idx + i) % num_clients; - let node_client = &env.node_clients[idx]; - let rpc_client = &node_client.rpc; - - let latest_block = - EthApiClient::::block_by_number( - rpc_client, - alloy_eips::BlockNumberOrTag::Latest, - false, - ) - .await?; - - if let Some(block) = latest_block { - let block_number = block.header.number; - let block_hash = block.header.hash; - - // Check if the block hash and number match the latest block info - if block_hash == latest_info.hash && block_number == latest_info.number { - env.last_producer_idx = Some(idx); - debug!("Selected node {} as the next block producer", idx); - return Ok(()); - } - } - } - - Err(eyre::eyre!("No suitable block producer found")) - }) - } -} - -/// Store payload attributes for the next block. -#[derive(Debug, Default)] -pub struct GeneratePayloadAttributes {} - -impl Action for GeneratePayloadAttributes -where - Engine: EngineTypes, -{ - fn execute<'a>(&'a mut self, env: &'a mut Environment) -> BoxFuture<'a, Result<()>> { - Box::pin(async move { - let latest_block = env - .latest_block_info - .as_ref() - .ok_or_else(|| eyre::eyre!("No latest block information available"))?; - let block_number = latest_block.number; - let timestamp = env.latest_header_time + env.block_timestamp_increment; - let payload_attributes = alloy_rpc_types_engine::PayloadAttributes { - timestamp, - prev_randao: B256::random(), - suggested_fee_recipient: alloy_primitives::Address::random(), - withdrawals: Some(vec![]), - parent_beacon_block_root: Some(B256::ZERO), - }; - - env.payload_attributes.insert(latest_block.number + 1, payload_attributes); - debug!("Stored payload attributes for block {}", block_number + 1); - Ok(()) - }) - } -} -/// Action that generates the next payload -#[derive(Debug, Default)] -pub struct GenerateNextPayload {} - -impl Action for GenerateNextPayload -where - Engine: EngineTypes + PayloadTypes, - reth_node_ethereum::engine::EthPayloadAttributes: - From<::ExecutionPayloadEnvelopeV3>, -{ - fn execute<'a>(&'a mut self, env: &'a mut Environment) -> BoxFuture<'a, Result<()>> { - Box::pin(async move { - let latest_block = env - .latest_block_info - .as_ref() - .ok_or_else(|| eyre::eyre!("No latest block information available"))?; - - let parent_hash = latest_block.hash; - debug!("Latest block hash: {parent_hash}"); - - let fork_choice_state = ForkchoiceState { - head_block_hash: parent_hash, - safe_block_hash: parent_hash, - finalized_block_hash: parent_hash, - }; - - let payload_attributes: PayloadAttributes = env - .payload_attributes - .get(&latest_block.number) - .cloned() - .ok_or_else(|| eyre::eyre!("No payload attributes found for latest block"))?; - - let fcu_result = EngineApiClient::::fork_choice_updated_v3( - &env.node_clients[0].engine, - fork_choice_state, - Some(payload_attributes.clone()), - ) - .await?; - - debug!("FCU result: {:?}", fcu_result); - - let payload_id = fcu_result - .payload_id - .ok_or_else(|| eyre::eyre!("No payload ID returned from forkChoiceUpdated"))?; - - debug!("Received payload ID: {:?}", payload_id); - env.next_payload_id = Some(payload_id); - - sleep(Duration::from_secs(1)).await; - - let built_payload: PayloadAttributes = - EngineApiClient::::get_payload_v3(&env.node_clients[0].engine, payload_id) - .await? - .into(); - env.payload_id_history.insert(latest_block.number + 1, payload_id); - env.latest_payload_built = Some(built_payload); - - Ok(()) - }) - } -} - -///Action that broadcasts the latest fork choice state to all clients -#[derive(Debug, Default)] -pub struct BroadcastLatestForkchoice {} - -impl Action for BroadcastLatestForkchoice -where - Engine: EngineTypes + PayloadTypes, - reth_node_ethereum::engine::EthPayloadAttributes: - From<::ExecutionPayloadEnvelopeV3>, -{ - fn execute<'a>(&'a mut self, env: &'a mut Environment) -> BoxFuture<'a, Result<()>> { - Box::pin(async move { - let payload = env.latest_payload_executed.clone(); - - if env.node_clients.is_empty() { - return Err(eyre::eyre!("No node clients available")); - } - let latest_block = env - .latest_block_info - .as_ref() - .ok_or_else(|| eyre::eyre!("No latest block information available"))?; - - let parent_hash = latest_block.hash; - debug!("Latest block hash: {parent_hash}"); - - let fork_choice_state = ForkchoiceState { - head_block_hash: parent_hash, - safe_block_hash: parent_hash, - finalized_block_hash: parent_hash, - }; - debug!( - "Broadcasting forkchoice update to {} clients. Head: {:?}", - env.node_clients.len(), - fork_choice_state.head_block_hash - ); - - for (idx, client) in env.node_clients.iter().enumerate() { - let engine_client = &client.engine; - - match EngineApiClient::::fork_choice_updated_v3( - engine_client, - fork_choice_state, - payload.clone(), - ) - .await - { - Ok(resp) => { - debug!( - "Client {}: Forkchoice update status: {:?}", - idx, resp.payload_status.status - ); - } - Err(err) => { - return Err(eyre::eyre!( - "Client {}: Failed to broadcast forkchoice: {:?}", - idx, - err - )); - } - } - } - debug!("Forkchoice update broadcasted successfully"); - Ok(()) - }) - } -} - -/// Action that produces a sequence of blocks using the available clients -#[derive(Debug)] -pub struct ProduceBlocks { - /// Number of blocks to produce - pub num_blocks: u64, - /// Tracks engine type - _phantom: PhantomData, -} - -impl ProduceBlocks { - /// Create a new `ProduceBlocks` action - pub fn new(num_blocks: u64) -> Self { - Self { num_blocks, _phantom: Default::default() } - } -} - -impl Default for ProduceBlocks { - fn default() -> Self { - Self::new(0) - } -} - -impl Action for ProduceBlocks -where - Engine: EngineTypes, -{ - fn execute<'a>(&'a mut self, env: &'a mut Environment) -> BoxFuture<'a, Result<()>> { - Box::pin(async move { - // Create a sequence for producing a single block - let mut sequence = Sequence::new(vec![ - Box::new(PickNextBlockProducer::default()), - Box::new(GeneratePayloadAttributes::default()), - ]); - for _ in 0..self.num_blocks { - sequence.execute(env).await?; - } - Ok(()) - }) - } -} - -/// Run a sequence of actions in series. -#[expect(missing_debug_implementations)] -pub struct Sequence { - /// Actions to execute in sequence - pub actions: Vec>>, -} - -impl Sequence { - /// Create a new sequence of actions - pub fn new(actions: Vec>>) -> Self { - Self { actions } - } -} - -impl Action for Sequence { - fn execute<'a>(&'a mut self, env: &'a mut Environment) -> BoxFuture<'a, Result<()>> { - Box::pin(async move { - // Execute each action in sequence - for action in &mut self.actions { - action.execute(env).await?; - } - - Ok(()) - }) - } -} - -/// Action that broadcasts the next new payload -#[derive(Debug, Default)] -pub struct BroadcastNextNewPayload {} - -impl Action for BroadcastNextNewPayload -where - Engine: EngineTypes + PayloadTypes, - reth_node_ethereum::engine::EthPayloadAttributes: - From<::ExecutionPayloadEnvelopeV3>, -{ - fn execute<'a>(&'a mut self, env: &'a mut Environment) -> BoxFuture<'a, Result<()>> { - Box::pin(async move { - // Get the next new payload to broadcast - let next_new_payload = env - .latest_payload_built - .as_ref() - .ok_or_else(|| eyre::eyre!("No next built payload found"))?; - let parent_beacon_block_root = next_new_payload - .parent_beacon_block_root - .ok_or_else(|| eyre::eyre!("No parent beacon block root for next new payload"))?; - - // Loop through all clients and broadcast the next new payload - let mut successful_broadcast: bool = false; - - for client in &env.node_clients { - let engine = &client.engine; - let rpc_client = &client.rpc; - - // Get latest block from the client - let rpc_latest_block = - EthApiClient::::block_by_number( - rpc_client, - alloy_eips::BlockNumberOrTag::Latest, - false, - ) - .await? - .ok_or_else(|| eyre::eyre!("No latest block found from rpc"))?; - - let latest_block = reth_ethereum_primitives::Block { - header: rpc_latest_block.header.inner, - body: reth_ethereum_primitives::BlockBody { - transactions: rpc_latest_block - .transactions - .into_transactions() - .map(|tx| tx.inner.into_inner().into()) - .collect(), - ommers: Default::default(), - withdrawals: rpc_latest_block.withdrawals, - }, - }; - - // Validate block number matches expected - let latest_block_info = env - .latest_block_info - .as_ref() - .ok_or_else(|| eyre::eyre!("No latest block info found"))?; - - if latest_block.header.number != latest_block_info.number { - return Err(eyre::eyre!( - "Client block number {} does not match expected block number {}", - latest_block.header.number, - latest_block_info.number - )); - } - - // Validate parent beacon block root - let latest_block_parent_beacon_block_root = - latest_block.parent_beacon_block_root.ok_or_else(|| { - eyre::eyre!("No parent beacon block root for latest block") - })?; - - if parent_beacon_block_root != latest_block_parent_beacon_block_root { - return Err(eyre::eyre!( - "Parent beacon block root mismatch: expected {:?}, got {:?}", - parent_beacon_block_root, - latest_block_parent_beacon_block_root - )); - } - - // Construct and broadcast the execution payload from the latest block - // The latest block should contain the latest_payload_built - let execution_payload = ExecutionPayloadV3::from_block_slow(&latest_block); - let result = EngineApiClient::::new_payload_v3( - engine, - execution_payload, - vec![], - parent_beacon_block_root, - ) - .await?; - - // Check if broadcast was successful - if result.status == PayloadStatusEnum::Valid { - successful_broadcast = true; - // We don't need to update the latest payload built since it should be the same. - // env.latest_payload_built = Some(next_new_payload.clone()); - env.latest_payload_executed = Some(next_new_payload.clone()); - break; - } else if let PayloadStatusEnum::Invalid { validation_error } = result.status { - debug!( - "Invalid payload status returned from broadcast: {:?}", - validation_error - ); - } - } - - if !successful_broadcast { - return Err(eyre::eyre!("Failed to successfully broadcast payload to any client")); - } - - Ok(()) - }) - } -} diff --git a/crates/e2e-test-utils/src/testsuite/actions/custom_fcu.rs b/crates/e2e-test-utils/src/testsuite/actions/custom_fcu.rs new file mode 100644 index 00000000000..397947caef1 --- /dev/null +++ b/crates/e2e-test-utils/src/testsuite/actions/custom_fcu.rs @@ -0,0 +1,226 @@ +//! Custom forkchoice update actions for testing specific FCU scenarios. + +use crate::testsuite::{Action, Environment}; +use alloy_primitives::B256; +use alloy_rpc_types_engine::{ForkchoiceState, PayloadStatusEnum}; +use eyre::Result; +use futures_util::future::BoxFuture; +use reth_node_api::EngineTypes; +use reth_rpc_api::clients::EngineApiClient; +use std::marker::PhantomData; +use tracing::debug; + +/// Reference to a block for forkchoice update +#[derive(Debug, Clone)] +pub enum BlockReference { + /// Direct block hash + Hash(B256), + /// Tagged block reference + Tag(String), + /// Latest block on the active node + Latest, +} + +/// Helper function to resolve a block reference to a hash +pub fn resolve_block_reference( + reference: &BlockReference, + env: &Environment, +) -> Result { + match reference { + BlockReference::Hash(hash) => Ok(*hash), + BlockReference::Tag(tag) => { + let (block_info, _) = env + .block_registry + .get(tag) + .ok_or_else(|| eyre::eyre!("Block tag '{tag}' not found in registry"))?; + Ok(block_info.hash) + } + BlockReference::Latest => { + let block_info = env + .current_block_info() + .ok_or_else(|| eyre::eyre!("No current block information available"))?; + Ok(block_info.hash) + } + } +} + +/// Action to send a custom forkchoice update with specific finalized, safe, and head blocks +#[derive(Debug)] +pub struct SendForkchoiceUpdate { + /// The finalized block reference + pub finalized: BlockReference, + /// The safe block reference + pub safe: BlockReference, + /// The head block reference + pub head: BlockReference, + /// Expected payload status (None means accept any non-error) + pub expected_status: Option, + /// Node index to send to (None means active node) + pub node_idx: Option, + /// Tracks engine type + _phantom: PhantomData, +} + +impl SendForkchoiceUpdate { + /// Create a new custom forkchoice update action + pub const fn new( + finalized: BlockReference, + safe: BlockReference, + head: BlockReference, + ) -> Self { + Self { finalized, safe, head, expected_status: None, node_idx: None, _phantom: PhantomData } + } + + /// Set expected status for the FCU response + pub fn with_expected_status(mut self, status: PayloadStatusEnum) -> Self { + self.expected_status = Some(status); + self + } + + /// Set the target node index + pub const fn with_node_idx(mut self, idx: usize) -> Self { + self.node_idx = Some(idx); + self + } +} + +impl Action for SendForkchoiceUpdate +where + Engine: EngineTypes, +{ + fn execute<'a>(&'a mut self, env: &'a mut Environment) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + let finalized_hash = resolve_block_reference(&self.finalized, env)?; + let safe_hash = resolve_block_reference(&self.safe, env)?; + let head_hash = resolve_block_reference(&self.head, env)?; + + let fork_choice_state = ForkchoiceState { + head_block_hash: head_hash, + safe_block_hash: safe_hash, + finalized_block_hash: finalized_hash, + }; + + debug!( + "Sending FCU - finalized: {finalized_hash}, safe: {safe_hash}, head: {head_hash}" + ); + + let node_idx = self.node_idx.unwrap_or(env.active_node_idx); + if node_idx >= env.node_clients.len() { + return Err(eyre::eyre!("Node index {node_idx} out of bounds")); + } + + let engine = env.node_clients[node_idx].engine.http_client(); + let fcu_response = + EngineApiClient::::fork_choice_updated_v3(&engine, fork_choice_state, None) + .await?; + + debug!( + "Node {node_idx}: FCU response - status: {:?}, latest_valid_hash: {:?}", + fcu_response.payload_status.status, fcu_response.payload_status.latest_valid_hash + ); + + // If we have an expected status, validate it + if let Some(expected) = &self.expected_status { + match (&fcu_response.payload_status.status, expected) { + (PayloadStatusEnum::Valid, PayloadStatusEnum::Valid) => { + debug!("Node {node_idx}: FCU returned VALID as expected"); + } + ( + PayloadStatusEnum::Invalid { validation_error }, + PayloadStatusEnum::Invalid { .. }, + ) => { + debug!( + "Node {node_idx}: FCU returned INVALID as expected: {validation_error:?}" + ); + } + (PayloadStatusEnum::Syncing, PayloadStatusEnum::Syncing) => { + debug!("Node {node_idx}: FCU returned SYNCING as expected"); + } + (PayloadStatusEnum::Accepted, PayloadStatusEnum::Accepted) => { + debug!("Node {node_idx}: FCU returned ACCEPTED as expected"); + } + (actual, expected) => { + return Err(eyre::eyre!( + "Node {node_idx}: FCU status mismatch. Expected {expected:?}, got {actual:?}" + )); + } + } + } else { + // Just validate it's not an error + if matches!(fcu_response.payload_status.status, PayloadStatusEnum::Invalid { .. }) { + return Err(eyre::eyre!( + "Node {node_idx}: FCU returned unexpected INVALID status: {:?}", + fcu_response.payload_status.status + )); + } + } + + Ok(()) + }) + } +} + +/// Action to finalize a specific block with a given head +#[derive(Debug)] +pub struct FinalizeBlock { + /// Block to finalize + pub block_to_finalize: BlockReference, + /// Current head block (if None, uses the finalized block) + pub head: Option, + /// Node index to send to (None means active node) + pub node_idx: Option, + /// Tracks engine type + _phantom: PhantomData, +} + +impl FinalizeBlock { + /// Create a new finalize block action + pub const fn new(block_to_finalize: BlockReference) -> Self { + Self { block_to_finalize, head: None, node_idx: None, _phantom: PhantomData } + } + + /// Set the head block (if different from finalized) + pub fn with_head(mut self, head: BlockReference) -> Self { + self.head = Some(head); + self + } + + /// Set the target node index + pub const fn with_node_idx(mut self, idx: usize) -> Self { + self.node_idx = Some(idx); + self + } +} + +impl Action for FinalizeBlock +where + Engine: EngineTypes, +{ + fn execute<'a>(&'a mut self, env: &'a mut Environment) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + let finalized_hash = resolve_block_reference(&self.block_to_finalize, env)?; + let head_hash = if let Some(ref head_ref) = self.head { + resolve_block_reference(head_ref, env)? + } else { + finalized_hash + }; + + // Use SendForkchoiceUpdate to do the actual work + let mut fcu_action = SendForkchoiceUpdate::new( + BlockReference::Hash(finalized_hash), + BlockReference::Hash(finalized_hash), // safe = finalized + BlockReference::Hash(head_hash), + ); + + if let Some(idx) = self.node_idx { + fcu_action = fcu_action.with_node_idx(idx); + } + + fcu_action.execute(env).await?; + + debug!("Block {finalized_hash} successfully finalized with head at {head_hash}"); + + Ok(()) + }) + } +} diff --git a/crates/e2e-test-utils/src/testsuite/actions/engine_api.rs b/crates/e2e-test-utils/src/testsuite/actions/engine_api.rs new file mode 100644 index 00000000000..d4053228d9c --- /dev/null +++ b/crates/e2e-test-utils/src/testsuite/actions/engine_api.rs @@ -0,0 +1,358 @@ +//! Engine API specific actions for testing. + +use crate::testsuite::{Action, Environment}; +use alloy_primitives::B256; +use alloy_rpc_types_engine::{ + ExecutionPayloadV1, ExecutionPayloadV2, ExecutionPayloadV3, PayloadStatusEnum, +}; +use alloy_rpc_types_eth::{Block, Header, Receipt, Transaction, TransactionRequest}; +use eyre::Result; +use futures_util::future::BoxFuture; +use reth_ethereum_primitives::TransactionSigned; +use reth_node_api::{EngineTypes, PayloadTypes}; +use reth_rpc_api::clients::{EngineApiClient, EthApiClient}; +use std::marker::PhantomData; +use tracing::debug; + +/// Action that sends a newPayload request to a specific node. +#[derive(Debug)] +pub struct SendNewPayload +where + Engine: EngineTypes, +{ + /// The node index to send to + pub node_idx: usize, + /// The block number to send + pub block_number: u64, + /// The source node to get the block from + pub source_node_idx: usize, + /// Expected payload status + pub expected_status: ExpectedPayloadStatus, + _phantom: PhantomData, +} + +/// Expected status for a payload +#[derive(Debug, Clone)] +pub enum ExpectedPayloadStatus { + /// Expect the payload to be valid + Valid, + /// Expect the payload to be invalid + Invalid, + /// Expect the payload to be syncing or accepted (buffered) + SyncingOrAccepted, +} + +impl SendNewPayload +where + Engine: EngineTypes, +{ + /// Create a new `SendNewPayload` action + pub fn new( + node_idx: usize, + block_number: u64, + source_node_idx: usize, + expected_status: ExpectedPayloadStatus, + ) -> Self { + Self { + node_idx, + block_number, + source_node_idx, + expected_status, + _phantom: Default::default(), + } + } +} + +impl Action for SendNewPayload +where + Engine: EngineTypes + PayloadTypes, +{ + fn execute<'a>(&'a mut self, env: &'a mut Environment) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + if self.node_idx >= env.node_clients.len() { + return Err(eyre::eyre!("Target node index out of bounds: {}", self.node_idx)); + } + if self.source_node_idx >= env.node_clients.len() { + return Err(eyre::eyre!( + "Source node index out of bounds: {}", + self.source_node_idx + )); + } + + // Get the block from the source node with retries + let source_rpc = &env.node_clients[self.source_node_idx].rpc; + let mut block = None; + let mut retries = 0; + const MAX_RETRIES: u32 = 5; + + while retries < MAX_RETRIES { + match EthApiClient::< + TransactionRequest, + Transaction, + Block, + Receipt, + Header, + TransactionSigned, + >::block_by_number( + source_rpc, + alloy_eips::BlockNumberOrTag::Number(self.block_number), + true, // include transactions + ) + .await + { + Ok(Some(b)) => { + block = Some(b); + break; + } + Ok(None) => { + debug!( + "Block {} not found on source node {} (attempt {}/{})", + self.block_number, + self.source_node_idx, + retries + 1, + MAX_RETRIES + ); + retries += 1; + if retries < MAX_RETRIES { + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + } + } + Err(e) => return Err(e.into()), + } + } + + let block = block.ok_or_else(|| { + eyre::eyre!( + "Block {} not found on source node {} after {} retries", + self.block_number, + self.source_node_idx, + MAX_RETRIES + ) + })?; + + // Convert block to ExecutionPayloadV3 + let payload = block_to_payload_v3(block.clone()); + + // Send the payload to the target node + let target_engine = env.node_clients[self.node_idx].engine.http_client(); + let result = EngineApiClient::::new_payload_v3( + &target_engine, + payload, + vec![], + B256::ZERO, // parent_beacon_block_root + ) + .await?; + + debug!( + "Node {}: new_payload for block {} response - status: {:?}, latest_valid_hash: {:?}", + self.node_idx, self.block_number, result.status, result.latest_valid_hash + ); + + // Validate the response based on expectations + match (&result.status, &self.expected_status) { + (PayloadStatusEnum::Valid, ExpectedPayloadStatus::Valid) => { + debug!( + "Node {}: Block {} marked as VALID as expected", + self.node_idx, self.block_number + ); + Ok(()) + } + ( + PayloadStatusEnum::Invalid { validation_error }, + ExpectedPayloadStatus::Invalid, + ) => { + debug!( + "Node {}: Block {} marked as INVALID as expected: {:?}", + self.node_idx, self.block_number, validation_error + ); + Ok(()) + } + ( + PayloadStatusEnum::Syncing | PayloadStatusEnum::Accepted, + ExpectedPayloadStatus::SyncingOrAccepted, + ) => { + debug!( + "Node {}: Block {} marked as SYNCING/ACCEPTED as expected (buffered)", + self.node_idx, self.block_number + ); + Ok(()) + } + (status, expected) => Err(eyre::eyre!( + "Node {}: Unexpected payload status for block {}. Got {:?}, expected {:?}", + self.node_idx, + self.block_number, + status, + expected + )), + } + }) + } +} + +/// Action that sends multiple blocks to a node in a specific order. +#[derive(Debug)] +pub struct SendNewPayloads +where + Engine: EngineTypes, +{ + /// The node index to send to + target_node: Option, + /// The source node to get the blocks from + source_node: Option, + /// The starting block number + start_block: Option, + /// The total number of blocks to send + total_blocks: Option, + /// Whether to send in reverse order + reverse_order: bool, + /// Custom block numbers to send (if not using `start_block` + `total_blocks`) + custom_block_numbers: Option>, + _phantom: PhantomData, +} + +impl SendNewPayloads +where + Engine: EngineTypes, +{ + /// Create a new `SendNewPayloads` action builder + pub fn new() -> Self { + Self { + target_node: None, + source_node: None, + start_block: None, + total_blocks: None, + reverse_order: false, + custom_block_numbers: None, + _phantom: Default::default(), + } + } + + /// Set the target node index + pub const fn with_target_node(mut self, node_idx: usize) -> Self { + self.target_node = Some(node_idx); + self + } + + /// Set the source node index + pub const fn with_source_node(mut self, node_idx: usize) -> Self { + self.source_node = Some(node_idx); + self + } + + /// Set the starting block number + pub const fn with_start_block(mut self, block_num: u64) -> Self { + self.start_block = Some(block_num); + self + } + + /// Set the total number of blocks to send + pub const fn with_total_blocks(mut self, count: u64) -> Self { + self.total_blocks = Some(count); + self + } + + /// Send blocks in reverse order (useful for testing buffering) + pub const fn in_reverse_order(mut self) -> Self { + self.reverse_order = true; + self + } + + /// Set custom block numbers to send + pub fn with_block_numbers(mut self, block_numbers: Vec) -> Self { + self.custom_block_numbers = Some(block_numbers); + self + } +} + +impl Default for SendNewPayloads +where + Engine: EngineTypes, +{ + fn default() -> Self { + Self::new() + } +} + +impl Action for SendNewPayloads +where + Engine: EngineTypes + PayloadTypes, +{ + fn execute<'a>(&'a mut self, env: &'a mut Environment) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + // Validate required fields + let target_node = + self.target_node.ok_or_else(|| eyre::eyre!("Target node not specified"))?; + let source_node = + self.source_node.ok_or_else(|| eyre::eyre!("Source node not specified"))?; + + // Determine block numbers to send + let block_numbers = if let Some(custom_numbers) = &self.custom_block_numbers { + custom_numbers.clone() + } else { + let start = + self.start_block.ok_or_else(|| eyre::eyre!("Start block not specified"))?; + let count = + self.total_blocks.ok_or_else(|| eyre::eyre!("Total blocks not specified"))?; + + if self.reverse_order { + // Send blocks in reverse order (e.g., for count=2, start=1: [2, 1]) + (0..count).map(|i| start + count - 1 - i).collect() + } else { + // Send blocks in normal order + (0..count).map(|i| start + i).collect() + } + }; + + for &block_number in &block_numbers { + // For the first block in reverse order, expect buffering + // For subsequent blocks, they might connect immediately + let expected_status = + if self.reverse_order && block_number == *block_numbers.first().unwrap() { + ExpectedPayloadStatus::SyncingOrAccepted + } else { + ExpectedPayloadStatus::Valid + }; + + let mut action = SendNewPayload::::new( + target_node, + block_number, + source_node, + expected_status, + ); + + action.execute(env).await?; + } + + Ok(()) + }) + } +} + +/// Helper function to convert a block to `ExecutionPayloadV3` +fn block_to_payload_v3(block: Block) -> ExecutionPayloadV3 { + use alloy_primitives::U256; + + ExecutionPayloadV3 { + payload_inner: ExecutionPayloadV2 { + payload_inner: ExecutionPayloadV1 { + parent_hash: block.header.inner.parent_hash, + fee_recipient: block.header.inner.beneficiary, + state_root: block.header.inner.state_root, + receipts_root: block.header.inner.receipts_root, + logs_bloom: block.header.inner.logs_bloom, + prev_randao: block.header.inner.mix_hash, + block_number: block.header.inner.number, + gas_limit: block.header.inner.gas_limit, + gas_used: block.header.inner.gas_used, + timestamp: block.header.inner.timestamp, + extra_data: block.header.inner.extra_data.clone(), + base_fee_per_gas: U256::from(block.header.inner.base_fee_per_gas.unwrap_or(0)), + block_hash: block.header.hash, + transactions: vec![], // No transactions needed for buffering tests + }, + withdrawals: block.withdrawals.unwrap_or_default().to_vec(), + }, + blob_gas_used: block.header.inner.blob_gas_used.unwrap_or(0), + excess_blob_gas: block.header.inner.excess_blob_gas.unwrap_or(0), + } +} diff --git a/crates/e2e-test-utils/src/testsuite/actions/fork.rs b/crates/e2e-test-utils/src/testsuite/actions/fork.rs new file mode 100644 index 00000000000..154b695adde --- /dev/null +++ b/crates/e2e-test-utils/src/testsuite/actions/fork.rs @@ -0,0 +1,288 @@ +//! Fork creation actions for the e2e testing framework. + +use crate::testsuite::{ + actions::{produce_blocks::ProduceBlocks, Sequence}, + Action, BlockInfo, Environment, +}; +use alloy_rpc_types_engine::{ForkchoiceState, PayloadAttributes}; +use alloy_rpc_types_eth::{Block, Header, Receipt, Transaction, TransactionRequest}; +use eyre::Result; +use futures_util::future::BoxFuture; +use reth_ethereum_primitives::TransactionSigned; +use reth_node_api::{EngineTypes, PayloadTypes}; +use reth_rpc_api::clients::EthApiClient; +use std::marker::PhantomData; +use tracing::debug; + +/// Fork base target for fork creation +#[derive(Debug, Clone)] +pub enum ForkBase { + /// Block number + Number(u64), + /// Tagged block reference + Tag(String), +} + +/// Action to create a fork from a specified block and produce blocks on top +#[derive(Debug)] +pub struct CreateFork { + /// Fork base specification (either block number or tag) + pub fork_base: ForkBase, + /// Number of blocks to produce on top of the fork base + pub num_blocks: u64, + /// Tracks engine type + _phantom: PhantomData, +} + +impl CreateFork { + /// Create a new `CreateFork` action from a block number + pub fn new(fork_base_block: u64, num_blocks: u64) -> Self { + Self { + fork_base: ForkBase::Number(fork_base_block), + num_blocks, + _phantom: Default::default(), + } + } + + /// Create a new `CreateFork` action from a tagged block + pub fn new_from_tag(tag: impl Into, num_blocks: u64) -> Self { + Self { fork_base: ForkBase::Tag(tag.into()), num_blocks, _phantom: Default::default() } + } +} + +impl Action for CreateFork +where + Engine: EngineTypes + PayloadTypes, + Engine::PayloadAttributes: From + Clone, + Engine::ExecutionPayloadEnvelopeV3: + Into, +{ + fn execute<'a>(&'a mut self, env: &'a mut Environment) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + // resolve the fork base and execute the appropriate sequence + match &self.fork_base { + ForkBase::Number(block_number) => { + // store the fork base for later validation on the active node + env.active_node_state_mut()?.current_fork_base = Some(*block_number); + + let mut sequence = Sequence::new(vec![ + Box::new(SetForkBase::new(*block_number)), + Box::new(ProduceBlocks::new(self.num_blocks)), + ]); + sequence.execute(env).await + } + ForkBase::Tag(tag) => { + let (block_info, _node_idx) = + env.block_registry.get(tag).copied().ok_or_else(|| { + eyre::eyre!("Block tag '{}' not found in registry", tag) + })?; + + // store the fork base for later validation on the active node + env.active_node_state_mut()?.current_fork_base = Some(block_info.number); + + let mut sequence = Sequence::new(vec![ + Box::new(SetForkBaseFromBlockInfo::new(block_info)), + Box::new(ProduceBlocks::new(self.num_blocks)), + ]); + sequence.execute(env).await + } + } + }) + } +} + +/// Sub-action to set the fork base block in the environment +#[derive(Debug)] +pub struct SetForkBase { + /// Block number to use as the base of the fork + pub fork_base_block: u64, +} + +/// Sub-action to set the fork base block from existing block info +#[derive(Debug)] +pub struct SetForkBaseFromBlockInfo { + /// Complete block info to use as the base of the fork + pub fork_base_info: BlockInfo, +} + +impl SetForkBase { + /// Create a new `SetForkBase` action + pub const fn new(fork_base_block: u64) -> Self { + Self { fork_base_block } + } +} + +impl SetForkBaseFromBlockInfo { + /// Create a new `SetForkBaseFromBlockInfo` action + pub const fn new(fork_base_info: BlockInfo) -> Self { + Self { fork_base_info } + } +} + +impl Action for SetForkBase +where + Engine: EngineTypes, +{ + fn execute<'a>(&'a mut self, env: &'a mut Environment) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + if env.node_clients.is_empty() { + return Err(eyre::eyre!("No node clients available")); + } + + // get the block at the fork base number to establish the fork point + let rpc_client = &env.node_clients[0].rpc; + let fork_base_block = EthApiClient::< + TransactionRequest, + Transaction, + Block, + Receipt, + Header, + TransactionSigned, + >::block_by_number( + rpc_client, + alloy_eips::BlockNumberOrTag::Number(self.fork_base_block), + false, + ) + .await? + .ok_or_else(|| eyre::eyre!("Fork base block {} not found", self.fork_base_block))?; + + // update active node state to point to the fork base block + let active_node_state = env.active_node_state_mut()?; + active_node_state.current_block_info = Some(BlockInfo { + hash: fork_base_block.header.hash, + number: fork_base_block.header.number, + timestamp: fork_base_block.header.timestamp, + }); + + active_node_state.latest_header_time = fork_base_block.header.timestamp; + + // update fork choice state to the fork base + active_node_state.latest_fork_choice_state = ForkchoiceState { + head_block_hash: fork_base_block.header.hash, + safe_block_hash: fork_base_block.header.hash, + finalized_block_hash: fork_base_block.header.hash, + }; + + debug!( + "Set fork base to block {} (hash: {})", + self.fork_base_block, fork_base_block.header.hash + ); + + Ok(()) + }) + } +} + +impl Action for SetForkBaseFromBlockInfo +where + Engine: EngineTypes, +{ + fn execute<'a>(&'a mut self, env: &'a mut Environment) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + let block_info = self.fork_base_info; + + debug!( + "Set fork base from block info: block {} (hash: {})", + block_info.number, block_info.hash + ); + + // update active node state to point to the fork base block + let active_node_state = env.active_node_state_mut()?; + active_node_state.current_block_info = Some(block_info); + active_node_state.latest_header_time = block_info.timestamp; + + // update fork choice state to the fork base + active_node_state.latest_fork_choice_state = ForkchoiceState { + head_block_hash: block_info.hash, + safe_block_hash: block_info.hash, + finalized_block_hash: block_info.hash, + }; + + debug!("Set fork base to block {} (hash: {})", block_info.number, block_info.hash); + + Ok(()) + }) + } +} + +/// Sub-action to validate that a fork was created correctly +#[derive(Debug)] +pub struct ValidateFork { + /// Number of the fork base block (stored here since we need it for validation) + pub fork_base_number: u64, +} + +impl ValidateFork { + /// Create a new `ValidateFork` action + pub const fn new(fork_base_number: u64) -> Self { + Self { fork_base_number } + } +} + +impl Action for ValidateFork +where + Engine: EngineTypes, +{ + fn execute<'a>(&'a mut self, env: &'a mut Environment) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + let current_block_info = env + .current_block_info() + .ok_or_else(|| eyre::eyre!("No current block information available"))?; + + // verify that the current tip is at or ahead of the fork base + if current_block_info.number < self.fork_base_number { + return Err(eyre::eyre!( + "Fork validation failed: current block number {} is behind fork base {}", + current_block_info.number, + self.fork_base_number + )); + } + + // get the fork base hash from the environment's fork choice state + // we assume the fork choice state was set correctly by SetForkBase + let fork_base_hash = + env.active_node_state()?.latest_fork_choice_state.finalized_block_hash; + + // trace back from current tip to verify it's a descendant of the fork base + let rpc_client = &env.node_clients[0].rpc; + let mut current_hash = current_block_info.hash; + let mut current_number = current_block_info.number; + + // walk backwards through the chain until we reach the fork base + while current_number > self.fork_base_number { + let block = EthApiClient::< + TransactionRequest, + Transaction, + Block, + Receipt, + Header, + TransactionSigned, + >::block_by_hash(rpc_client, current_hash, false) + .await? + .ok_or_else(|| { + eyre::eyre!("Block with hash {} not found during fork validation", current_hash) + })?; + + current_hash = block.header.parent_hash; + current_number = block.header.number.saturating_sub(1); + } + + // verify we reached the expected fork base + if current_hash != fork_base_hash { + return Err(eyre::eyre!( + "Fork validation failed: expected fork base hash {}, but found {} at block {}", + fork_base_hash, + current_hash, + current_number + )); + } + + debug!( + "Fork validation successful: tip block {} is descendant of fork base {} ({})", + current_block_info.number, self.fork_base_number, fork_base_hash + ); + + Ok(()) + }) + } +} diff --git a/crates/e2e-test-utils/src/testsuite/actions/mod.rs b/crates/e2e-test-utils/src/testsuite/actions/mod.rs new file mode 100644 index 00000000000..d4916265692 --- /dev/null +++ b/crates/e2e-test-utils/src/testsuite/actions/mod.rs @@ -0,0 +1,320 @@ +//! Actions that can be performed in tests. + +use crate::testsuite::Environment; +use alloy_rpc_types_engine::{ForkchoiceState, ForkchoiceUpdated, PayloadStatusEnum}; +use eyre::Result; +use futures_util::future::BoxFuture; +use reth_node_api::EngineTypes; +use reth_rpc_api::clients::EngineApiClient; +use std::future::Future; +use tracing::debug; + +pub mod custom_fcu; +pub mod engine_api; +pub mod fork; +pub mod node_ops; +pub mod produce_blocks; +pub mod reorg; + +pub use custom_fcu::{BlockReference, FinalizeBlock, SendForkchoiceUpdate}; +pub use engine_api::{ExpectedPayloadStatus, SendNewPayload, SendNewPayloads}; +pub use fork::{CreateFork, ForkBase, SetForkBase, SetForkBaseFromBlockInfo, ValidateFork}; +pub use node_ops::{ + AssertChainTip, CaptureBlockOnNode, CompareNodeChainTips, SelectActiveNode, ValidateBlockTag, + WaitForSync, +}; +pub use produce_blocks::{ + AssertMineBlock, BroadcastLatestForkchoice, BroadcastNextNewPayload, CheckPayloadAccepted, + ExpectFcuStatus, GenerateNextPayload, GeneratePayloadAttributes, PickNextBlockProducer, + ProduceBlocks, ProduceBlocksLocally, ProduceInvalidBlocks, TestFcuToTag, UpdateBlockInfo, + UpdateBlockInfoToLatestPayload, ValidateCanonicalTag, +}; +pub use reorg::{ReorgTarget, ReorgTo, SetReorgTarget}; + +/// An action that can be performed on an instance. +/// +/// Actions execute operations and potentially make assertions in a single step. +/// The action name indicates what it does (e.g., `AssertMineBlock` would both +/// mine a block and assert it worked). +pub trait Action: Send + 'static +where + I: EngineTypes, +{ + /// Executes the action + fn execute<'a>(&'a mut self, env: &'a mut Environment) -> BoxFuture<'a, Result<()>>; +} + +/// Simplified action container for storage in tests +#[expect(missing_debug_implementations)] +pub struct ActionBox(Box>); + +impl ActionBox +where + I: EngineTypes + 'static, +{ + /// Constructor for [`ActionBox`]. + pub fn new>(action: A) -> Self { + Self(Box::new(action)) + } + + /// Executes an [`ActionBox`] with the given [`Environment`] reference. + pub async fn execute(mut self, env: &mut Environment) -> Result<()> { + self.0.execute(env).await + } +} + +/// Implementation of `Action` for any function/closure that takes an Environment +/// reference and returns a Future resolving to Result<()>. +/// +/// This allows using closures directly as actions with `.with_action(async move |env| {...})`. +impl Action for F +where + I: EngineTypes, + F: FnMut(&Environment) -> Fut + Send + 'static, + Fut: Future> + Send + 'static, +{ + fn execute<'a>(&'a mut self, env: &'a mut Environment) -> BoxFuture<'a, Result<()>> { + Box::pin(self(env)) + } +} + +/// Run a sequence of actions in series. +#[expect(missing_debug_implementations)] +pub struct Sequence { + /// Actions to execute in sequence + pub actions: Vec>>, +} + +impl Sequence { + /// Create a new sequence of actions + pub fn new(actions: Vec>>) -> Self { + Self { actions } + } +} + +impl Action for Sequence +where + I: EngineTypes + Sync + Send + 'static, +{ + fn execute<'a>(&'a mut self, env: &'a mut Environment) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + // Execute each action in sequence + for action in &mut self.actions { + action.execute(env).await?; + } + + Ok(()) + }) + } +} + +/// Action that makes the current latest block canonical by broadcasting a forkchoice update +#[derive(Debug, Default)] +pub struct MakeCanonical { + /// If true, only send to the active node. If false, broadcast to all nodes. + active_node_only: bool, +} + +impl MakeCanonical { + /// Create a new `MakeCanonical` action + pub const fn new() -> Self { + Self { active_node_only: false } + } + + /// Create a new `MakeCanonical` action that only applies to the active node + pub const fn with_active_node() -> Self { + Self { active_node_only: true } + } +} + +impl Action for MakeCanonical +where + Engine: EngineTypes + reth_node_api::PayloadTypes, + Engine::PayloadAttributes: From + Clone, + Engine::ExecutionPayloadEnvelopeV3: + Into, +{ + fn execute<'a>(&'a mut self, env: &'a mut Environment) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + if self.active_node_only { + // Only update the active node + let latest_block = env + .current_block_info() + .ok_or_else(|| eyre::eyre!("No latest block information available"))?; + + let fork_choice_state = ForkchoiceState { + head_block_hash: latest_block.hash, + safe_block_hash: latest_block.hash, + finalized_block_hash: latest_block.hash, + }; + + let active_idx = env.active_node_idx; + let engine = env.node_clients[active_idx].engine.http_client(); + + let fcu_response = EngineApiClient::::fork_choice_updated_v3( + &engine, + fork_choice_state, + None, + ) + .await?; + + debug!( + "Active node {}: Forkchoice update status: {:?}", + active_idx, fcu_response.payload_status.status + ); + + validate_fcu_response(&fcu_response, &format!("Active node {active_idx}"))?; + + Ok(()) + } else { + // Original broadcast behavior + let mut actions: Vec>> = vec![ + Box::new(BroadcastLatestForkchoice::default()), + Box::new(UpdateBlockInfo::default()), + ]; + + // if we're on a fork, validate it now that it's canonical + if let Ok(active_state) = env.active_node_state() && + let Some(fork_base) = active_state.current_fork_base + { + debug!("MakeCanonical: Adding fork validation from base block {}", fork_base); + actions.push(Box::new(ValidateFork::new(fork_base))); + // clear the fork base since we're now canonical + env.active_node_state_mut()?.current_fork_base = None; + } + + let mut sequence = Sequence::new(actions); + sequence.execute(env).await + } + }) + } +} + +/// Action that captures the current block and tags it with a name for later reference +#[derive(Debug)] +pub struct CaptureBlock { + /// Tag name to associate with the current block + pub tag: String, +} + +impl CaptureBlock { + /// Create a new `CaptureBlock` action + pub fn new(tag: impl Into) -> Self { + Self { tag: tag.into() } + } +} + +impl Action for CaptureBlock +where + Engine: EngineTypes, +{ + fn execute<'a>(&'a mut self, env: &'a mut Environment) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + let current_block = env + .current_block_info() + .ok_or_else(|| eyre::eyre!("No current block information available"))?; + + env.block_registry.insert(self.tag.clone(), (current_block, env.active_node_idx)); + + debug!( + "Captured block {} (hash: {}) from active node {} with tag '{}'", + current_block.number, current_block.hash, env.active_node_idx, self.tag + ); + + Ok(()) + }) + } +} + +/// Validates a forkchoice update response and returns an error if invalid +pub fn validate_fcu_response(response: &ForkchoiceUpdated, context: &str) -> Result<()> { + match &response.payload_status.status { + PayloadStatusEnum::Valid => { + debug!("{}: FCU accepted as valid", context); + Ok(()) + } + PayloadStatusEnum::Invalid { validation_error } => { + Err(eyre::eyre!("{}: FCU rejected as invalid: {:?}", context, validation_error)) + } + PayloadStatusEnum::Syncing => { + debug!("{}: FCU accepted, node is syncing", context); + Ok(()) + } + PayloadStatusEnum::Accepted => { + debug!("{}: FCU accepted for processing", context); + Ok(()) + } + } +} + +/// Expects that the `ForkchoiceUpdated` response status is VALID. +pub fn expect_fcu_valid(response: &ForkchoiceUpdated, context: &str) -> Result<()> { + match &response.payload_status.status { + PayloadStatusEnum::Valid => { + debug!("{}: FCU status is VALID as expected.", context); + Ok(()) + } + other_status => { + Err(eyre::eyre!("{}: Expected FCU status VALID, but got {:?}", context, other_status)) + } + } +} + +/// Expects that the `ForkchoiceUpdated` response status is INVALID. +pub fn expect_fcu_invalid(response: &ForkchoiceUpdated, context: &str) -> Result<()> { + match &response.payload_status.status { + PayloadStatusEnum::Invalid { validation_error } => { + debug!("{}: FCU status is INVALID as expected: {:?}", context, validation_error); + Ok(()) + } + other_status => { + Err(eyre::eyre!("{}: Expected FCU status INVALID, but got {:?}", context, other_status)) + } + } +} + +/// Expects that the `ForkchoiceUpdated` response status is either SYNCING or ACCEPTED. +pub fn expect_fcu_syncing_or_accepted(response: &ForkchoiceUpdated, context: &str) -> Result<()> { + match &response.payload_status.status { + PayloadStatusEnum::Syncing => { + debug!("{}: FCU status is SYNCING as expected (SYNCING or ACCEPTED).", context); + Ok(()) + } + PayloadStatusEnum::Accepted => { + debug!("{}: FCU status is ACCEPTED as expected (SYNCING or ACCEPTED).", context); + Ok(()) + } + other_status => Err(eyre::eyre!( + "{}: Expected FCU status SYNCING or ACCEPTED, but got {:?}", + context, + other_status + )), + } +} + +/// Expects that the `ForkchoiceUpdated` response status is not SYNCING and not ACCEPTED. +pub fn expect_fcu_not_syncing_or_accepted( + response: &ForkchoiceUpdated, + context: &str, +) -> Result<()> { + match &response.payload_status.status { + PayloadStatusEnum::Valid => { + debug!("{}: FCU status is VALID as expected (not SYNCING or ACCEPTED).", context); + Ok(()) + } + PayloadStatusEnum::Invalid { validation_error } => { + debug!( + "{}: FCU status is INVALID as expected (not SYNCING or ACCEPTED): {:?}", + context, validation_error + ); + Ok(()) + } + syncing_or_accepted_status @ (PayloadStatusEnum::Syncing | PayloadStatusEnum::Accepted) => { + Err(eyre::eyre!( + "{}: Expected FCU status not SYNCING or ACCEPTED (i.e., VALID or INVALID), but got {:?}", + context, + syncing_or_accepted_status + )) + } + } +} diff --git a/crates/e2e-test-utils/src/testsuite/actions/node_ops.rs b/crates/e2e-test-utils/src/testsuite/actions/node_ops.rs new file mode 100644 index 00000000000..da1cf98e617 --- /dev/null +++ b/crates/e2e-test-utils/src/testsuite/actions/node_ops.rs @@ -0,0 +1,395 @@ +//! Node-specific operations for multi-node testing. + +use crate::testsuite::{Action, Environment}; +use alloy_rpc_types_eth::{Block, Header, Receipt, Transaction, TransactionRequest}; +use eyre::Result; +use futures_util::future::BoxFuture; +use reth_ethereum_primitives::TransactionSigned; +use reth_node_api::EngineTypes; +use reth_rpc_api::clients::EthApiClient; +use std::time::Duration; +use tokio::time::{sleep, timeout}; +use tracing::debug; + +/// Action to select which node should be active for subsequent single-node operations. +#[derive(Debug)] +pub struct SelectActiveNode { + /// Node index to set as active + pub node_idx: usize, +} + +impl SelectActiveNode { + /// Create a new `SelectActiveNode` action + pub const fn new(node_idx: usize) -> Self { + Self { node_idx } + } +} + +impl Action for SelectActiveNode +where + Engine: EngineTypes, +{ + fn execute<'a>(&'a mut self, env: &'a mut Environment) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + env.set_active_node(self.node_idx)?; + debug!("Set active node to {}", self.node_idx); + Ok(()) + }) + } +} + +/// Action to compare chain tips between two nodes. +#[derive(Debug)] +pub struct CompareNodeChainTips { + /// First node index + pub node_a: usize, + /// Second node index + pub node_b: usize, + /// Whether tips should be the same or different + pub should_be_equal: bool, +} + +impl CompareNodeChainTips { + /// Create a new action expecting nodes to have the same chain tip + pub const fn expect_same(node_a: usize, node_b: usize) -> Self { + Self { node_a, node_b, should_be_equal: true } + } + + /// Create a new action expecting nodes to have different chain tips + pub const fn expect_different(node_a: usize, node_b: usize) -> Self { + Self { node_a, node_b, should_be_equal: false } + } +} + +impl Action for CompareNodeChainTips +where + Engine: EngineTypes, +{ + fn execute<'a>(&'a mut self, env: &'a mut Environment) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + if self.node_a >= env.node_count() || self.node_b >= env.node_count() { + return Err(eyre::eyre!("Node index out of bounds")); + } + + let node_a_client = &env.node_clients[self.node_a]; + let node_b_client = &env.node_clients[self.node_b]; + + // Get latest block from each node + let block_a = EthApiClient::< + TransactionRequest, + Transaction, + Block, + Receipt, + Header, + TransactionSigned, + >::block_by_number( + &node_a_client.rpc, alloy_eips::BlockNumberOrTag::Latest, false + ) + .await? + .ok_or_else(|| eyre::eyre!("Failed to get latest block from node {}", self.node_a))?; + + let block_b = EthApiClient::< + TransactionRequest, + Transaction, + Block, + Receipt, + Header, + TransactionSigned, + >::block_by_number( + &node_b_client.rpc, alloy_eips::BlockNumberOrTag::Latest, false + ) + .await? + .ok_or_else(|| eyre::eyre!("Failed to get latest block from node {}", self.node_b))?; + + let tips_equal = block_a.header.hash == block_b.header.hash; + + debug!( + "Node {} chain tip: {} (block {}), Node {} chain tip: {} (block {})", + self.node_a, + block_a.header.hash, + block_a.header.number, + self.node_b, + block_b.header.hash, + block_b.header.number + ); + + if self.should_be_equal && !tips_equal { + return Err(eyre::eyre!( + "Expected nodes {} and {} to have the same chain tip, but node {} has {} and node {} has {}", + self.node_a, self.node_b, self.node_a, block_a.header.hash, self.node_b, block_b.header.hash + )); + } + + if !self.should_be_equal && tips_equal { + return Err(eyre::eyre!( + "Expected nodes {} and {} to have different chain tips, but both have {}", + self.node_a, + self.node_b, + block_a.header.hash + )); + } + + Ok(()) + }) + } +} + +/// Action to capture a block with a tag, associating it with a specific node. +#[derive(Debug)] +pub struct CaptureBlockOnNode { + /// Tag name to associate with the block + pub tag: String, + /// Node index to capture the block from + pub node_idx: usize, +} + +impl CaptureBlockOnNode { + /// Create a new `CaptureBlockOnNode` action + pub fn new(tag: impl Into, node_idx: usize) -> Self { + Self { tag: tag.into(), node_idx } + } +} + +impl Action for CaptureBlockOnNode +where + Engine: EngineTypes, +{ + fn execute<'a>(&'a mut self, env: &'a mut Environment) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + let node_state = env.node_state(self.node_idx)?; + let current_block = node_state.current_block_info.ok_or_else(|| { + eyre::eyre!("No current block information available for node {}", self.node_idx) + })?; + + env.block_registry.insert(self.tag.clone(), (current_block, self.node_idx)); + + debug!( + "Captured block {} (hash: {}) from node {} with tag '{}'", + current_block.number, current_block.hash, self.node_idx, self.tag + ); + + Ok(()) + }) + } +} + +/// Action to get a block by tag and verify which node it came from. +#[derive(Debug)] +pub struct ValidateBlockTag { + /// Tag to look up + pub tag: String, + /// Expected node index (optional) + pub expected_node_idx: Option, +} + +impl ValidateBlockTag { + /// Create a new action to validate a block tag exists + pub fn exists(tag: impl Into) -> Self { + Self { tag: tag.into(), expected_node_idx: None } + } + + /// Create a new action to validate a block tag came from a specific node + pub fn from_node(tag: impl Into, node_idx: usize) -> Self { + Self { tag: tag.into(), expected_node_idx: Some(node_idx) } + } +} + +impl Action for ValidateBlockTag +where + Engine: EngineTypes, +{ + fn execute<'a>(&'a mut self, env: &'a mut Environment) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + let (block_info, node_idx) = env + .block_registry + .get(&self.tag) + .copied() + .ok_or_else(|| eyre::eyre!("Block tag '{}' not found in registry", self.tag))?; + + if let Some(expected_node) = self.expected_node_idx && + node_idx != expected_node + { + return Err(eyre::eyre!( + "Block tag '{}' came from node {} but expected node {}", + self.tag, + node_idx, + expected_node + )); + } + + debug!( + "Validated block tag '{}': block {} (hash: {}) from node {}", + self.tag, block_info.number, block_info.hash, node_idx + ); + + Ok(()) + }) + } +} + +/// Action that waits for two nodes to sync and have the same chain tip. +#[derive(Debug)] +pub struct WaitForSync { + /// First node index + pub node_a: usize, + /// Second node index + pub node_b: usize, + /// Maximum time to wait for sync (default: 30 seconds) + pub timeout_secs: u64, + /// Polling interval (default: 1 second) + pub poll_interval_secs: u64, +} + +impl WaitForSync { + /// Create a new `WaitForSync` action with default timeouts + pub const fn new(node_a: usize, node_b: usize) -> Self { + Self { node_a, node_b, timeout_secs: 30, poll_interval_secs: 1 } + } + + /// Set custom timeout + pub const fn with_timeout(mut self, timeout_secs: u64) -> Self { + self.timeout_secs = timeout_secs; + self + } + + /// Set custom poll interval + pub const fn with_poll_interval(mut self, poll_interval_secs: u64) -> Self { + self.poll_interval_secs = poll_interval_secs; + self + } +} + +impl Action for WaitForSync +where + Engine: EngineTypes, +{ + fn execute<'a>(&'a mut self, env: &'a mut Environment) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + if self.node_a >= env.node_count() || self.node_b >= env.node_count() { + return Err(eyre::eyre!("Node index out of bounds")); + } + + let timeout_duration = Duration::from_secs(self.timeout_secs); + let poll_interval = Duration::from_secs(self.poll_interval_secs); + + debug!( + "Waiting for nodes {} and {} to sync (timeout: {}s, poll interval: {}s)", + self.node_a, self.node_b, self.timeout_secs, self.poll_interval_secs + ); + + let sync_check = async { + loop { + let node_a_client = &env.node_clients[self.node_a]; + let node_b_client = &env.node_clients[self.node_b]; + + // Get latest block from each node + let block_a = EthApiClient::< + TransactionRequest, + Transaction, + Block, + Receipt, + Header, + TransactionSigned, + >::block_by_number( + &node_a_client.rpc, + alloy_eips::BlockNumberOrTag::Latest, + false, + ) + .await? + .ok_or_else(|| { + eyre::eyre!("Failed to get latest block from node {}", self.node_a) + })?; + + let block_b = EthApiClient::< + TransactionRequest, + Transaction, + Block, + Receipt, + Header, + TransactionSigned, + >::block_by_number( + &node_b_client.rpc, + alloy_eips::BlockNumberOrTag::Latest, + false, + ) + .await? + .ok_or_else(|| { + eyre::eyre!("Failed to get latest block from node {}", self.node_b) + })?; + + debug!( + "Sync check: Node {} tip: {} (block {}), Node {} tip: {} (block {})", + self.node_a, + block_a.header.hash, + block_a.header.number, + self.node_b, + block_b.header.hash, + block_b.header.number + ); + + if block_a.header.hash == block_b.header.hash { + debug!( + "Nodes {} and {} successfully synced to block {} (hash: {})", + self.node_a, self.node_b, block_a.header.number, block_a.header.hash + ); + return Ok(()); + } + + sleep(poll_interval).await; + } + }; + + match timeout(timeout_duration, sync_check).await { + Ok(result) => result, + Err(_) => Err(eyre::eyre!( + "Timeout waiting for nodes {} and {} to sync after {}s", + self.node_a, + self.node_b, + self.timeout_secs + )), + } + }) + } +} + +/// Action to assert the current chain tip is at a specific block number. +#[derive(Debug)] +pub struct AssertChainTip { + /// Expected block number + pub expected_block_number: u64, +} + +impl AssertChainTip { + /// Create a new `AssertChainTip` action + pub const fn new(expected_block_number: u64) -> Self { + Self { expected_block_number } + } +} + +impl Action for AssertChainTip +where + Engine: EngineTypes, +{ + fn execute<'a>(&'a mut self, env: &'a mut Environment) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + let current_block = env + .current_block_info() + .ok_or_else(|| eyre::eyre!("No current block information available"))?; + + if current_block.number != self.expected_block_number { + return Err(eyre::eyre!( + "Expected chain tip to be at block {}, but found block {}", + self.expected_block_number, + current_block.number + )); + } + + debug!( + "Chain tip verified at block {} (hash: {})", + current_block.number, current_block.hash + ); + + Ok(()) + }) + } +} diff --git a/crates/e2e-test-utils/src/testsuite/actions/produce_blocks.rs b/crates/e2e-test-utils/src/testsuite/actions/produce_blocks.rs new file mode 100644 index 00000000000..fe9e9133aec --- /dev/null +++ b/crates/e2e-test-utils/src/testsuite/actions/produce_blocks.rs @@ -0,0 +1,1150 @@ +//! Block production actions for the e2e testing framework. + +use crate::testsuite::{ + actions::{expect_fcu_not_syncing_or_accepted, validate_fcu_response, Action, Sequence}, + BlockInfo, Environment, +}; +use alloy_primitives::{Bytes, B256}; +use alloy_rpc_types_engine::{ + payload::ExecutionPayloadEnvelopeV3, ForkchoiceState, PayloadAttributes, PayloadStatusEnum, +}; +use alloy_rpc_types_eth::{Block, Header, Receipt, Transaction, TransactionRequest}; +use eyre::Result; +use futures_util::future::BoxFuture; +use reth_ethereum_primitives::TransactionSigned; +use reth_node_api::{EngineTypes, PayloadTypes}; +use reth_rpc_api::clients::{EngineApiClient, EthApiClient}; +use std::{collections::HashSet, marker::PhantomData, time::Duration}; +use tokio::time::sleep; +use tracing::debug; + +/// Mine a single block with the given transactions and verify the block was created +/// successfully. +#[derive(Debug)] +pub struct AssertMineBlock +where + Engine: PayloadTypes, +{ + /// The node index to mine + pub node_idx: usize, + /// Transactions to include in the block + pub transactions: Vec, + /// Expected block hash (optional) + pub expected_hash: Option, + /// Block's payload attributes + // TODO: refactor once we have actions to generate payload attributes. + pub payload_attributes: Engine::PayloadAttributes, + /// Tracks engine type + _phantom: PhantomData, +} + +impl AssertMineBlock +where + Engine: PayloadTypes, +{ + /// Create a new `AssertMineBlock` action + pub fn new( + node_idx: usize, + transactions: Vec, + expected_hash: Option, + payload_attributes: Engine::PayloadAttributes, + ) -> Self { + Self { + node_idx, + transactions, + expected_hash, + payload_attributes, + _phantom: Default::default(), + } + } +} + +impl Action for AssertMineBlock +where + Engine: EngineTypes, +{ + fn execute<'a>(&'a mut self, env: &'a mut Environment) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + if self.node_idx >= env.node_clients.len() { + return Err(eyre::eyre!("Node index out of bounds: {}", self.node_idx)); + } + + let node_client = &env.node_clients[self.node_idx]; + let rpc_client = &node_client.rpc; + let engine_client = node_client.engine.http_client(); + + // get the latest block to use as parent + let latest_block = EthApiClient::< + TransactionRequest, + Transaction, + Block, + Receipt, + Header, + TransactionSigned, + >::block_by_number( + rpc_client, alloy_eips::BlockNumberOrTag::Latest, false + ) + .await?; + + let latest_block = latest_block.ok_or_else(|| eyre::eyre!("Latest block not found"))?; + let parent_hash = latest_block.header.hash; + + debug!("Latest block hash: {parent_hash}"); + + // create a simple forkchoice state with the latest block as head + let fork_choice_state = ForkchoiceState { + head_block_hash: parent_hash, + safe_block_hash: parent_hash, + finalized_block_hash: parent_hash, + }; + + // Try v2 first for backwards compatibility, fall back to v3 on error. + match EngineApiClient::::fork_choice_updated_v2( + &engine_client, + fork_choice_state, + Some(self.payload_attributes.clone()), + ) + .await + { + Ok(fcu_result) => { + debug!(?fcu_result, "FCU v2 result"); + match fcu_result.payload_status.status { + PayloadStatusEnum::Valid => { + if let Some(payload_id) = fcu_result.payload_id { + debug!(id=%payload_id, "Got payload"); + let _engine_payload = EngineApiClient::::get_payload_v2( + &engine_client, + payload_id, + ) + .await?; + Ok(()) + } else { + Err(eyre::eyre!("No payload ID returned from forkchoiceUpdated")) + } + } + _ => Err(eyre::eyre!( + "Payload status not valid: {:?}", + fcu_result.payload_status + ))?, + } + } + Err(_) => { + // If v2 fails due to unsupported fork/missing fields, try v3 + let fcu_result = EngineApiClient::::fork_choice_updated_v3( + &engine_client, + fork_choice_state, + Some(self.payload_attributes.clone()), + ) + .await?; + + debug!(?fcu_result, "FCU v3 result"); + match fcu_result.payload_status.status { + PayloadStatusEnum::Valid => { + if let Some(payload_id) = fcu_result.payload_id { + debug!(id=%payload_id, "Got payload"); + let _engine_payload = EngineApiClient::::get_payload_v3( + &engine_client, + payload_id, + ) + .await?; + Ok(()) + } else { + Err(eyre::eyre!("No payload ID returned from forkchoiceUpdated")) + } + } + _ => Err(eyre::eyre!( + "Payload status not valid: {:?}", + fcu_result.payload_status + )), + } + } + } + }) + } +} + +/// Pick the next block producer based on the latest block information. +#[derive(Debug, Default)] +pub struct PickNextBlockProducer {} + +impl PickNextBlockProducer { + /// Create a new `PickNextBlockProducer` action + pub const fn new() -> Self { + Self {} + } +} + +impl Action for PickNextBlockProducer +where + Engine: EngineTypes, +{ + fn execute<'a>(&'a mut self, env: &'a mut Environment) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + let num_clients = env.node_clients.len(); + if num_clients == 0 { + return Err(eyre::eyre!("No node clients available")); + } + + let latest_info = env + .current_block_info() + .ok_or_else(|| eyre::eyre!("No latest block information available"))?; + + // simple round-robin selection based on next block number + let next_producer_idx = ((latest_info.number + 1) % num_clients as u64) as usize; + + env.last_producer_idx = Some(next_producer_idx); + debug!( + "Selected node {} as the next block producer for block {}", + next_producer_idx, + latest_info.number + 1 + ); + + Ok(()) + }) + } +} + +/// Store payload attributes for the next block. +#[derive(Debug, Default)] +pub struct GeneratePayloadAttributes {} + +impl Action for GeneratePayloadAttributes +where + Engine: EngineTypes + PayloadTypes, + Engine::PayloadAttributes: From, +{ + fn execute<'a>(&'a mut self, env: &'a mut Environment) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + let latest_block = env + .current_block_info() + .ok_or_else(|| eyre::eyre!("No latest block information available"))?; + let block_number = latest_block.number; + let timestamp = + env.active_node_state()?.latest_header_time + env.block_timestamp_increment; + let payload_attributes = PayloadAttributes { + timestamp, + prev_randao: B256::random(), + suggested_fee_recipient: alloy_primitives::Address::random(), + withdrawals: Some(vec![]), + parent_beacon_block_root: Some(B256::ZERO), + }; + + env.active_node_state_mut()? + .payload_attributes + .insert(latest_block.number + 1, payload_attributes); + debug!("Stored payload attributes for block {}", block_number + 1); + Ok(()) + }) + } +} + +/// Action that generates the next payload +#[derive(Debug, Default)] +pub struct GenerateNextPayload {} + +impl Action for GenerateNextPayload +where + Engine: EngineTypes + PayloadTypes, + Engine::PayloadAttributes: From + Clone, +{ + fn execute<'a>(&'a mut self, env: &'a mut Environment) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + let latest_block = env + .current_block_info() + .ok_or_else(|| eyre::eyre!("No latest block information available"))?; + + let parent_hash = latest_block.hash; + debug!("Latest block hash: {parent_hash}"); + + let fork_choice_state = ForkchoiceState { + head_block_hash: parent_hash, + safe_block_hash: parent_hash, + finalized_block_hash: parent_hash, + }; + + let payload_attributes = env + .active_node_state()? + .payload_attributes + .get(&(latest_block.number + 1)) + .cloned() + .ok_or_else(|| eyre::eyre!("No payload attributes found for next block"))?; + + let producer_idx = + env.last_producer_idx.ok_or_else(|| eyre::eyre!("No block producer selected"))?; + + let fcu_result = EngineApiClient::::fork_choice_updated_v3( + &env.node_clients[producer_idx].engine.http_client(), + fork_choice_state, + Some(payload_attributes.clone().into()), + ) + .await?; + + debug!("FCU result: {:?}", fcu_result); + + // validate the FCU status before proceeding + // Note: In the context of GenerateNextPayload, Syncing usually means the engine + // doesn't have the requested head block, which should be an error + expect_fcu_not_syncing_or_accepted(&fcu_result, "GenerateNextPayload")?; + + let payload_id = if let Some(payload_id) = fcu_result.payload_id { + debug!("Received new payload ID: {:?}", payload_id); + payload_id + } else { + debug!("No payload ID returned, generating fresh payload attributes for forking"); + + let fresh_payload_attributes = PayloadAttributes { + timestamp: env.active_node_state()?.latest_header_time + + env.block_timestamp_increment, + prev_randao: B256::random(), + suggested_fee_recipient: alloy_primitives::Address::random(), + withdrawals: Some(vec![]), + parent_beacon_block_root: Some(B256::ZERO), + }; + + let fresh_fcu_result = EngineApiClient::::fork_choice_updated_v3( + &env.node_clients[producer_idx].engine.http_client(), + fork_choice_state, + Some(fresh_payload_attributes.clone().into()), + ) + .await?; + + debug!("Fresh FCU result: {:?}", fresh_fcu_result); + + // validate the fresh FCU status + expect_fcu_not_syncing_or_accepted( + &fresh_fcu_result, + "GenerateNextPayload (fresh)", + )?; + + if let Some(payload_id) = fresh_fcu_result.payload_id { + payload_id + } else { + debug!("Engine considers the fork base already canonical, skipping payload generation"); + return Ok(()); + } + }; + + env.active_node_state_mut()?.next_payload_id = Some(payload_id); + + sleep(Duration::from_secs(1)).await; + + let built_payload_envelope = EngineApiClient::::get_payload_v3( + &env.node_clients[producer_idx].engine.http_client(), + payload_id, + ) + .await?; + + // Store the payload attributes that were used to generate this payload + let built_payload = payload_attributes.clone(); + env.active_node_state_mut()? + .payload_id_history + .insert(latest_block.number + 1, payload_id); + env.active_node_state_mut()?.latest_payload_built = Some(built_payload); + env.active_node_state_mut()?.latest_payload_envelope = Some(built_payload_envelope); + + Ok(()) + }) + } +} + +/// Action that broadcasts the latest fork choice state to all clients +#[derive(Debug, Default)] +pub struct BroadcastLatestForkchoice {} + +impl Action for BroadcastLatestForkchoice +where + Engine: EngineTypes + PayloadTypes, + Engine::PayloadAttributes: From + Clone, + Engine::ExecutionPayloadEnvelopeV3: Into, +{ + fn execute<'a>(&'a mut self, env: &'a mut Environment) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + if env.node_clients.is_empty() { + return Err(eyre::eyre!("No node clients available")); + } + + // use the hash of the newly executed payload if available + let head_hash = if let Some(payload_envelope) = + &env.active_node_state()?.latest_payload_envelope + { + let execution_payload_envelope: ExecutionPayloadEnvelopeV3 = + payload_envelope.clone().into(); + let new_block_hash = execution_payload_envelope + .execution_payload + .payload_inner + .payload_inner + .block_hash; + debug!("Using newly executed block hash as head: {new_block_hash}"); + new_block_hash + } else { + // fallback to RPC query + let rpc_client = &env.node_clients[0].rpc; + let current_head_block = EthApiClient::< + TransactionRequest, + Transaction, + Block, + Receipt, + Header, + TransactionSigned, + >::block_by_number( + rpc_client, alloy_eips::BlockNumberOrTag::Latest, false + ) + .await? + .ok_or_else(|| eyre::eyre!("No latest block found from RPC"))?; + debug!("Using RPC latest block hash as head: {}", current_head_block.header.hash); + current_head_block.header.hash + }; + + let fork_choice_state = ForkchoiceState { + head_block_hash: head_hash, + safe_block_hash: head_hash, + finalized_block_hash: head_hash, + }; + debug!( + "Broadcasting forkchoice update to {} clients. Head: {:?}", + env.node_clients.len(), + fork_choice_state.head_block_hash + ); + + for (idx, client) in env.node_clients.iter().enumerate() { + match EngineApiClient::::fork_choice_updated_v3( + &client.engine.http_client(), + fork_choice_state, + None, + ) + .await + { + Ok(resp) => { + debug!( + "Client {}: Forkchoice update status: {:?}", + idx, resp.payload_status.status + ); + // validate that the forkchoice update was accepted + validate_fcu_response(&resp, &format!("Client {idx}"))?; + } + Err(err) => { + return Err(eyre::eyre!( + "Client {}: Failed to broadcast forkchoice: {:?}", + idx, + err + )); + } + } + } + debug!("Forkchoice update broadcasted successfully"); + Ok(()) + }) + } +} + +/// Action that syncs environment state with the node's canonical chain via RPC. +/// +/// This queries the latest canonical block from the node and updates the environment +/// to match. Typically used after forkchoice operations to ensure the environment +/// is in sync with the node's view of the canonical chain. +#[derive(Debug, Default)] +pub struct UpdateBlockInfo {} + +impl Action for UpdateBlockInfo +where + Engine: EngineTypes, +{ + fn execute<'a>(&'a mut self, env: &'a mut Environment) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + // get the latest block from the first client to update environment state + let rpc_client = &env.node_clients[0].rpc; + let latest_block = EthApiClient::< + TransactionRequest, + Transaction, + Block, + Receipt, + Header, + TransactionSigned, + >::block_by_number( + rpc_client, alloy_eips::BlockNumberOrTag::Latest, false + ) + .await? + .ok_or_else(|| eyre::eyre!("No latest block found from RPC"))?; + + // update environment with the new block information + env.set_current_block_info(BlockInfo { + hash: latest_block.header.hash, + number: latest_block.header.number, + timestamp: latest_block.header.timestamp, + })?; + + env.active_node_state_mut()?.latest_header_time = latest_block.header.timestamp; + env.active_node_state_mut()?.latest_fork_choice_state.head_block_hash = + latest_block.header.hash; + + debug!( + "Updated environment to block {} (hash: {})", + latest_block.header.number, latest_block.header.hash + ); + + Ok(()) + }) + } +} + +/// Action that updates environment state using the locally produced payload. +/// +/// This uses the execution payload stored in the environment rather than querying RPC, +/// making it more efficient and reliable during block production. Preferred over +/// `UpdateBlockInfo` when we have just produced a block and have the payload available. +#[derive(Debug, Default)] +pub struct UpdateBlockInfoToLatestPayload {} + +impl Action for UpdateBlockInfoToLatestPayload +where + Engine: EngineTypes + PayloadTypes, + Engine::ExecutionPayloadEnvelopeV3: Into, +{ + fn execute<'a>(&'a mut self, env: &'a mut Environment) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + let payload_envelope = env + .active_node_state()? + .latest_payload_envelope + .as_ref() + .ok_or_else(|| eyre::eyre!("No execution payload envelope available"))?; + + let execution_payload_envelope: ExecutionPayloadEnvelopeV3 = + payload_envelope.clone().into(); + let execution_payload = execution_payload_envelope.execution_payload; + + let block_hash = execution_payload.payload_inner.payload_inner.block_hash; + let block_number = execution_payload.payload_inner.payload_inner.block_number; + let block_timestamp = execution_payload.payload_inner.payload_inner.timestamp; + + // update environment with the new block information from the payload + env.set_current_block_info(BlockInfo { + hash: block_hash, + number: block_number, + timestamp: block_timestamp, + })?; + + env.active_node_state_mut()?.latest_header_time = block_timestamp; + env.active_node_state_mut()?.latest_fork_choice_state.head_block_hash = block_hash; + + debug!( + "Updated environment to newly produced block {} (hash: {})", + block_number, block_hash + ); + + Ok(()) + }) + } +} + +/// Action that checks whether the broadcasted new payload has been accepted +#[derive(Debug, Default)] +pub struct CheckPayloadAccepted {} + +impl Action for CheckPayloadAccepted +where + Engine: EngineTypes, + Engine::ExecutionPayloadEnvelopeV3: Into, +{ + fn execute<'a>(&'a mut self, env: &'a mut Environment) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + let mut accepted_check: bool = false; + + let latest_block = env + .current_block_info() + .ok_or_else(|| eyre::eyre!("No latest block information available"))?; + + let payload_id = *env + .active_node_state()? + .payload_id_history + .get(&(latest_block.number + 1)) + .ok_or_else(|| eyre::eyre!("Cannot find payload_id"))?; + + let node_clients = env.node_clients.clone(); + for (idx, client) in node_clients.iter().enumerate() { + let rpc_client = &client.rpc; + + // get the last header by number using latest_head_number + let rpc_latest_header = EthApiClient::< + TransactionRequest, + Transaction, + Block, + Receipt, + Header, + TransactionSigned, + >::header_by_number( + rpc_client, alloy_eips::BlockNumberOrTag::Latest + ) + .await? + .ok_or_else(|| eyre::eyre!("No latest header found from rpc"))?; + + // perform several checks + let next_new_payload = env + .active_node_state()? + .latest_payload_built + .as_ref() + .ok_or_else(|| eyre::eyre!("No next built payload found"))?; + + let built_payload = EngineApiClient::::get_payload_v3( + &client.engine.http_client(), + payload_id, + ) + .await?; + + let execution_payload_envelope: ExecutionPayloadEnvelopeV3 = built_payload.into(); + let new_payload_block_hash = execution_payload_envelope + .execution_payload + .payload_inner + .payload_inner + .block_hash; + + if rpc_latest_header.hash != new_payload_block_hash { + debug!( + "Client {}: The hash is not matched: {:?} {:?}", + idx, rpc_latest_header.hash, new_payload_block_hash + ); + continue; + } + + if rpc_latest_header.inner.difficulty != alloy_primitives::U256::ZERO { + debug!( + "Client {}: difficulty != 0: {:?}", + idx, rpc_latest_header.inner.difficulty + ); + continue; + } + + if rpc_latest_header.inner.mix_hash != next_new_payload.prev_randao { + debug!( + "Client {}: The mix_hash and prev_randao is not same: {:?} {:?}", + idx, rpc_latest_header.inner.mix_hash, next_new_payload.prev_randao + ); + continue; + } + + let extra_len = rpc_latest_header.inner.extra_data.len(); + if extra_len <= 32 { + debug!("Client {}: extra_len is fewer than 32. extra_len: {}", idx, extra_len); + continue; + } + + // at least one client passes all the check, save the header in Env + if !accepted_check { + accepted_check = true; + // save the current block info in Env + env.set_current_block_info(BlockInfo { + hash: rpc_latest_header.hash, + number: rpc_latest_header.inner.number, + timestamp: rpc_latest_header.inner.timestamp, + })?; + + // align latest header time and forkchoice state with the accepted canonical + // head + env.active_node_state_mut()?.latest_header_time = + rpc_latest_header.inner.timestamp; + env.active_node_state_mut()?.latest_fork_choice_state.head_block_hash = + rpc_latest_header.hash; + } + } + + if accepted_check { + Ok(()) + } else { + Err(eyre::eyre!("No clients passed payload acceptance checks")) + } + }) + } +} + +/// Action that broadcasts the next new payload +#[derive(Debug, Default)] +pub struct BroadcastNextNewPayload { + /// If true, only send to the active node. If false, broadcast to all nodes. + active_node_only: bool, +} + +impl BroadcastNextNewPayload { + /// Create a new `BroadcastNextNewPayload` action that only sends to the active node + pub const fn with_active_node() -> Self { + Self { active_node_only: true } + } +} + +impl Action for BroadcastNextNewPayload +where + Engine: EngineTypes + PayloadTypes, + Engine::PayloadAttributes: From + Clone, + Engine::ExecutionPayloadEnvelopeV3: Into, +{ + fn execute<'a>(&'a mut self, env: &'a mut Environment) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + // Get the next new payload to broadcast + let next_new_payload = env + .active_node_state()? + .latest_payload_built + .as_ref() + .ok_or_else(|| eyre::eyre!("No next built payload found"))? + .clone(); + let parent_beacon_block_root = next_new_payload + .parent_beacon_block_root + .ok_or_else(|| eyre::eyre!("No parent beacon block root for next new payload"))?; + + let payload_envelope = env + .active_node_state()? + .latest_payload_envelope + .as_ref() + .ok_or_else(|| eyre::eyre!("No execution payload envelope available"))? + .clone(); + + let execution_payload_envelope: ExecutionPayloadEnvelopeV3 = payload_envelope.into(); + let execution_payload = execution_payload_envelope.execution_payload; + + if self.active_node_only { + // Send only to the active node + let active_idx = env.active_node_idx; + let engine = env.node_clients[active_idx].engine.http_client(); + + let result = EngineApiClient::::new_payload_v3( + &engine, + execution_payload.clone(), + vec![], + parent_beacon_block_root, + ) + .await?; + + debug!("Active node {}: new_payload status: {:?}", active_idx, result.status); + + // Validate the response + match result.status { + PayloadStatusEnum::Valid => { + env.active_node_state_mut()?.latest_payload_executed = + Some(next_new_payload); + Ok(()) + } + other => Err(eyre::eyre!( + "Active node {}: Unexpected payload status: {:?}", + active_idx, + other + )), + } + } else { + // Loop through all clients and broadcast the next new payload + let mut broadcast_results = Vec::new(); + let mut first_valid_seen = false; + + for (idx, client) in env.node_clients.iter().enumerate() { + let engine = client.engine.http_client(); + + // Broadcast the execution payload + let result = EngineApiClient::::new_payload_v3( + &engine, + execution_payload.clone(), + vec![], + parent_beacon_block_root, + ) + .await?; + + broadcast_results.push((idx, result.status.clone())); + debug!("Node {}: new_payload broadcast status: {:?}", idx, result.status); + + // Check if this node accepted the payload + if result.status == PayloadStatusEnum::Valid && !first_valid_seen { + first_valid_seen = true; + } else if let PayloadStatusEnum::Invalid { validation_error } = result.status { + debug!( + "Node {}: Invalid payload status returned from broadcast: {:?}", + idx, validation_error + ); + } + } + + // Update the executed payload state after broadcasting to all nodes + if first_valid_seen { + env.active_node_state_mut()?.latest_payload_executed = Some(next_new_payload); + } + + // Check if at least one node accepted the payload + let any_valid = + broadcast_results.iter().any(|(_, status)| *status == PayloadStatusEnum::Valid); + if !any_valid { + return Err(eyre::eyre!( + "Failed to successfully broadcast payload to any client" + )); + } + + debug!("Broadcast complete. Results: {:?}", broadcast_results); + + Ok(()) + } + }) + } +} + +/// Action that produces a sequence of blocks using the available clients +#[derive(Debug)] +pub struct ProduceBlocks { + /// Number of blocks to produce + pub num_blocks: u64, + /// Tracks engine type + _phantom: PhantomData, +} + +impl ProduceBlocks { + /// Create a new `ProduceBlocks` action + pub fn new(num_blocks: u64) -> Self { + Self { num_blocks, _phantom: Default::default() } + } +} + +impl Default for ProduceBlocks { + fn default() -> Self { + Self::new(0) + } +} + +impl Action for ProduceBlocks +where + Engine: EngineTypes + PayloadTypes, + Engine::PayloadAttributes: From + Clone, + Engine::ExecutionPayloadEnvelopeV3: Into, +{ + fn execute<'a>(&'a mut self, env: &'a mut Environment) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + for _ in 0..self.num_blocks { + // create a fresh sequence for each block to avoid state pollution + // Note: This produces blocks but does NOT make them canonical + // Use MakeCanonical action explicitly if canonicalization is needed + let mut sequence = Sequence::new(vec![ + Box::new(PickNextBlockProducer::default()), + Box::new(GeneratePayloadAttributes::default()), + Box::new(GenerateNextPayload::default()), + Box::new(BroadcastNextNewPayload::default()), + Box::new(UpdateBlockInfoToLatestPayload::default()), + ]); + sequence.execute(env).await?; + } + Ok(()) + }) + } +} + +/// Action to test forkchoice update to a tagged block with expected status +#[derive(Debug)] +pub struct TestFcuToTag { + /// Tag name of the target block + pub tag: String, + /// Expected payload status + pub expected_status: PayloadStatusEnum, +} + +impl TestFcuToTag { + /// Create a new `TestFcuToTag` action + pub fn new(tag: impl Into, expected_status: PayloadStatusEnum) -> Self { + Self { tag: tag.into(), expected_status } + } +} + +impl Action for TestFcuToTag +where + Engine: EngineTypes, +{ + fn execute<'a>(&'a mut self, env: &'a mut Environment) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + // get the target block from the registry + let (target_block, _node_idx) = env + .block_registry + .get(&self.tag) + .copied() + .ok_or_else(|| eyre::eyre!("Block tag '{}' not found in registry", self.tag))?; + + let engine_client = env.node_clients[0].engine.http_client(); + let fcu_state = ForkchoiceState { + head_block_hash: target_block.hash, + safe_block_hash: target_block.hash, + finalized_block_hash: target_block.hash, + }; + + let fcu_response = + EngineApiClient::::fork_choice_updated_v2(&engine_client, fcu_state, None) + .await?; + + // validate the response matches expected status + match (&fcu_response.payload_status.status, &self.expected_status) { + (PayloadStatusEnum::Valid, PayloadStatusEnum::Valid) => { + debug!("FCU to '{}' returned VALID as expected", self.tag); + } + (PayloadStatusEnum::Invalid { .. }, PayloadStatusEnum::Invalid { .. }) => { + debug!("FCU to '{}' returned INVALID as expected", self.tag); + } + (PayloadStatusEnum::Syncing, PayloadStatusEnum::Syncing) => { + debug!("FCU to '{}' returned SYNCING as expected", self.tag); + } + (PayloadStatusEnum::Accepted, PayloadStatusEnum::Accepted) => { + debug!("FCU to '{}' returned ACCEPTED as expected", self.tag); + } + (actual, expected) => { + return Err(eyre::eyre!( + "FCU to '{}': expected status {:?}, but got {:?}", + self.tag, + expected, + actual + )); + } + } + + Ok(()) + }) + } +} + +/// Action to expect a specific FCU status when targeting a tagged block +#[derive(Debug)] +pub struct ExpectFcuStatus { + /// Tag name of the target block + pub target_tag: String, + /// Expected payload status + pub expected_status: PayloadStatusEnum, +} + +impl ExpectFcuStatus { + /// Create a new `ExpectFcuStatus` action expecting VALID status + pub fn valid(target_tag: impl Into) -> Self { + Self { target_tag: target_tag.into(), expected_status: PayloadStatusEnum::Valid } + } + + /// Create a new `ExpectFcuStatus` action expecting INVALID status + pub fn invalid(target_tag: impl Into) -> Self { + Self { + target_tag: target_tag.into(), + expected_status: PayloadStatusEnum::Invalid { + validation_error: "corrupted block".to_string(), + }, + } + } + + /// Create a new `ExpectFcuStatus` action expecting SYNCING status + pub fn syncing(target_tag: impl Into) -> Self { + Self { target_tag: target_tag.into(), expected_status: PayloadStatusEnum::Syncing } + } + + /// Create a new `ExpectFcuStatus` action expecting ACCEPTED status + pub fn accepted(target_tag: impl Into) -> Self { + Self { target_tag: target_tag.into(), expected_status: PayloadStatusEnum::Accepted } + } +} + +impl Action for ExpectFcuStatus +where + Engine: EngineTypes, +{ + fn execute<'a>(&'a mut self, env: &'a mut Environment) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + let mut test_fcu = TestFcuToTag::new(&self.target_tag, self.expected_status.clone()); + test_fcu.execute(env).await + }) + } +} + +/// Action to validate that a tagged block remains canonical by performing FCU to it +#[derive(Debug)] +pub struct ValidateCanonicalTag { + /// Tag name of the block to validate as canonical + pub tag: String, +} + +impl ValidateCanonicalTag { + /// Create a new `ValidateCanonicalTag` action + pub fn new(tag: impl Into) -> Self { + Self { tag: tag.into() } + } +} + +impl Action for ValidateCanonicalTag +where + Engine: EngineTypes, +{ + fn execute<'a>(&'a mut self, env: &'a mut Environment) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + let mut expect_valid = ExpectFcuStatus::valid(&self.tag); + expect_valid.execute(env).await?; + + debug!("Successfully validated that '{}' remains canonical", self.tag); + Ok(()) + }) + } +} + +/// Action that produces blocks locally without broadcasting to other nodes +/// This sends the payload only to the active node to ensure it's available locally +#[derive(Debug)] +pub struct ProduceBlocksLocally { + /// Number of blocks to produce + pub num_blocks: u64, + /// Tracks engine type + _phantom: PhantomData, +} + +impl ProduceBlocksLocally { + /// Create a new `ProduceBlocksLocally` action + pub fn new(num_blocks: u64) -> Self { + Self { num_blocks, _phantom: Default::default() } + } +} + +impl Default for ProduceBlocksLocally { + fn default() -> Self { + Self::new(0) + } +} + +impl Action for ProduceBlocksLocally +where + Engine: EngineTypes + PayloadTypes, + Engine::PayloadAttributes: From + Clone, + Engine::ExecutionPayloadEnvelopeV3: Into, +{ + fn execute<'a>(&'a mut self, env: &'a mut Environment) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + // Remember the active node to ensure all blocks are produced on the same node + let producer_idx = env.active_node_idx; + + for _ in 0..self.num_blocks { + // Ensure we always use the same producer + env.last_producer_idx = Some(producer_idx); + + // create a sequence that produces blocks and sends only to active node + let mut sequence = Sequence::new(vec![ + // Skip PickNextBlockProducer to maintain the same producer + Box::new(GeneratePayloadAttributes::default()), + Box::new(GenerateNextPayload::default()), + // Send payload only to the active node to make it available + Box::new(BroadcastNextNewPayload::with_active_node()), + Box::new(UpdateBlockInfoToLatestPayload::default()), + ]); + sequence.execute(env).await?; + } + Ok(()) + }) + } +} + +/// Action that produces a sequence of blocks where some blocks are intentionally invalid +#[derive(Debug)] +pub struct ProduceInvalidBlocks { + /// Number of blocks to produce + pub num_blocks: u64, + /// Set of indices (0-based) where blocks should be made invalid + pub invalid_indices: HashSet, + /// Tracks engine type + _phantom: PhantomData, +} + +impl ProduceInvalidBlocks { + /// Create a new `ProduceInvalidBlocks` action + pub fn new(num_blocks: u64, invalid_indices: HashSet) -> Self { + Self { num_blocks, invalid_indices, _phantom: Default::default() } + } + + /// Create a new `ProduceInvalidBlocks` action with a single invalid block at the specified + /// index + pub fn with_invalid_at(num_blocks: u64, invalid_index: u64) -> Self { + let mut invalid_indices = HashSet::new(); + invalid_indices.insert(invalid_index); + Self::new(num_blocks, invalid_indices) + } +} + +impl Action for ProduceInvalidBlocks +where + Engine: EngineTypes + PayloadTypes, + Engine::PayloadAttributes: From + Clone, + Engine::ExecutionPayloadEnvelopeV3: Into, +{ + fn execute<'a>(&'a mut self, env: &'a mut Environment) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + for block_index in 0..self.num_blocks { + let is_invalid = self.invalid_indices.contains(&block_index); + + if is_invalid { + debug!("Producing invalid block at index {}", block_index); + + // produce a valid block first, then corrupt it + let mut sequence = Sequence::new(vec![ + Box::new(PickNextBlockProducer::default()), + Box::new(GeneratePayloadAttributes::default()), + Box::new(GenerateNextPayload::default()), + ]); + sequence.execute(env).await?; + + // get the latest payload and corrupt it + let latest_envelope = + env.active_node_state()?.latest_payload_envelope.as_ref().ok_or_else( + || eyre::eyre!("No payload envelope available to corrupt"), + )?; + + let envelope_v3: ExecutionPayloadEnvelopeV3 = latest_envelope.clone().into(); + let mut corrupted_payload = envelope_v3.execution_payload; + + // corrupt the state root to make the block invalid + corrupted_payload.payload_inner.payload_inner.state_root = B256::random(); + + debug!( + "Corrupted state root for block {} to: {}", + block_index, corrupted_payload.payload_inner.payload_inner.state_root + ); + + // send the corrupted payload via newPayload + let engine_client = env.node_clients[0].engine.http_client(); + // for simplicity, we'll use empty versioned hashes for invalid block testing + let versioned_hashes = Vec::new(); + // use a random parent beacon block root since this is for invalid block testing + let parent_beacon_block_root = B256::random(); + + let new_payload_response = EngineApiClient::::new_payload_v3( + &engine_client, + corrupted_payload.clone(), + versioned_hashes, + parent_beacon_block_root, + ) + .await?; + + // expect the payload to be rejected as invalid + match new_payload_response.status { + PayloadStatusEnum::Invalid { validation_error } => { + debug!( + "Block {} correctly rejected as invalid: {:?}", + block_index, validation_error + ); + } + other_status => { + return Err(eyre::eyre!( + "Expected block {} to be rejected as INVALID, but got: {:?}", + block_index, + other_status + )); + } + } + + // update block info with the corrupted block (for potential future reference) + env.set_current_block_info(BlockInfo { + hash: corrupted_payload.payload_inner.payload_inner.block_hash, + number: corrupted_payload.payload_inner.payload_inner.block_number, + timestamp: corrupted_payload.timestamp(), + })?; + } else { + debug!("Producing valid block at index {}", block_index); + + // produce a valid block normally + let mut sequence = Sequence::new(vec![ + Box::new(PickNextBlockProducer::default()), + Box::new(GeneratePayloadAttributes::default()), + Box::new(GenerateNextPayload::default()), + Box::new(BroadcastNextNewPayload::default()), + Box::new(UpdateBlockInfoToLatestPayload::default()), + ]); + sequence.execute(env).await?; + } + } + Ok(()) + }) + } +} diff --git a/crates/e2e-test-utils/src/testsuite/actions/reorg.rs b/crates/e2e-test-utils/src/testsuite/actions/reorg.rs new file mode 100644 index 00000000000..337734c3282 --- /dev/null +++ b/crates/e2e-test-utils/src/testsuite/actions/reorg.rs @@ -0,0 +1,123 @@ +//! Reorg actions for the e2e testing framework. + +use crate::testsuite::{ + actions::{produce_blocks::BroadcastLatestForkchoice, Action, Sequence}, + BlockInfo, Environment, +}; +use alloy_primitives::B256; +use alloy_rpc_types_engine::{ForkchoiceState, PayloadAttributes}; +use eyre::Result; +use futures_util::future::BoxFuture; +use reth_node_api::{EngineTypes, PayloadTypes}; +use std::marker::PhantomData; +use tracing::debug; + +/// Target for reorg operation +#[derive(Debug, Clone)] +pub enum ReorgTarget { + /// Direct block hash + Hash(B256), + /// Tagged block reference + Tag(String), +} + +/// Action that performs a reorg by setting a new head block as canonical +#[derive(Debug)] +pub struct ReorgTo { + /// Target for the reorg operation + pub target: ReorgTarget, + /// Tracks engine type + _phantom: PhantomData, +} + +impl ReorgTo { + /// Create a new `ReorgTo` action with a direct block hash + pub const fn new(target_hash: B256) -> Self { + Self { target: ReorgTarget::Hash(target_hash), _phantom: PhantomData } + } + + /// Create a new `ReorgTo` action with a tagged block reference + pub fn new_from_tag(tag: impl Into) -> Self { + Self { target: ReorgTarget::Tag(tag.into()), _phantom: PhantomData } + } +} + +impl Action for ReorgTo +where + Engine: EngineTypes + PayloadTypes, + Engine::PayloadAttributes: From + Clone, + Engine::ExecutionPayloadEnvelopeV3: + Into, +{ + fn execute<'a>(&'a mut self, env: &'a mut Environment) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + // resolve the target block info from either direct hash or tag + let target_block_info = match &self.target { + ReorgTarget::Hash(_hash) => { + return Err(eyre::eyre!( + "Direct hash reorgs are not supported. Use CaptureBlock to tag the target block first, then use ReorgTo::new_from_tag()" + )); + } + ReorgTarget::Tag(tag) => { + let (block_info, _node_idx) = + env.block_registry.get(tag).copied().ok_or_else(|| { + eyre::eyre!("Block tag '{}' not found in registry", tag) + })?; + block_info + } + }; + + let mut sequence = Sequence::new(vec![ + Box::new(SetReorgTarget::new(target_block_info)), + Box::new(BroadcastLatestForkchoice::default()), + ]); + + sequence.execute(env).await + }) + } +} + +/// Sub-action to set the reorg target block in the environment +#[derive(Debug)] +pub struct SetReorgTarget { + /// Complete block info for the reorg target + pub target_block_info: BlockInfo, +} + +impl SetReorgTarget { + /// Create a new `SetReorgTarget` action + pub const fn new(target_block_info: BlockInfo) -> Self { + Self { target_block_info } + } +} + +impl Action for SetReorgTarget +where + Engine: EngineTypes, +{ + fn execute<'a>(&'a mut self, env: &'a mut Environment) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + let block_info = self.target_block_info; + + debug!( + "Setting reorg target to block {} (hash: {})", + block_info.number, block_info.hash + ); + + // update active node state to point to the target block + let active_node_state = env.active_node_state_mut()?; + active_node_state.current_block_info = Some(block_info); + active_node_state.latest_header_time = block_info.timestamp; + + // update fork choice state to make the target block canonical + active_node_state.latest_fork_choice_state = ForkchoiceState { + head_block_hash: block_info.hash, + safe_block_hash: block_info.hash, + finalized_block_hash: block_info.hash, + }; + + debug!("Set reorg target to block {}", block_info.hash); + Ok(()) + }) + } +} diff --git a/crates/e2e-test-utils/src/testsuite/examples.rs b/crates/e2e-test-utils/src/testsuite/examples.rs deleted file mode 100644 index 0d9343482dd..00000000000 --- a/crates/e2e-test-utils/src/testsuite/examples.rs +++ /dev/null @@ -1,72 +0,0 @@ -//! Example tests using the test suite framework. - -use crate::testsuite::{ - actions::{AssertMineBlock, ProduceBlocks}, - setup::{NetworkSetup, Setup}, - TestBuilder, -}; -use alloy_primitives::{Address, B256}; -use alloy_rpc_types_engine::PayloadAttributes; -use eyre::Result; -use reth_chainspec::{ChainSpecBuilder, MAINNET}; -use reth_node_ethereum::{EthEngineTypes, EthereumNode}; -use std::sync::Arc; - -#[tokio::test] -async fn test_testsuite_assert_mine_block() -> Result<()> { - reth_tracing::init_test_tracing(); - - let setup = Setup::default() - .with_chain_spec(Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(serde_json::from_str(include_str!("assets/genesis.json")).unwrap()) - .paris_activated() - .build(), - )) - .with_network(NetworkSetup::single_node()); - - let test = - TestBuilder::new().with_setup(setup).with_action(AssertMineBlock::::new( - 0, - vec![], - Some(B256::ZERO), - // TODO: refactor once we have actions to generate payload attributes. - PayloadAttributes { - timestamp: std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_secs(), - prev_randao: B256::random(), - suggested_fee_recipient: Address::random(), - withdrawals: None, - parent_beacon_block_root: None, - }, - )); - - test.run::().await?; - - Ok(()) -} - -#[tokio::test] -async fn test_testsuite_produce_blocks() -> Result<()> { - reth_tracing::init_test_tracing(); - - let setup = Setup::default() - .with_chain_spec(Arc::new( - ChainSpecBuilder::default() - .chain(MAINNET.chain) - .genesis(serde_json::from_str(include_str!("assets/genesis.json")).unwrap()) - .cancun_activated() - .build(), - )) - .with_network(NetworkSetup::single_node()); - - let test = - TestBuilder::new().with_setup(setup).with_action(ProduceBlocks::::new(0)); - - test.run::().await?; - - Ok(()) -} diff --git a/crates/e2e-test-utils/src/testsuite/mod.rs b/crates/e2e-test-utils/src/testsuite/mod.rs index db3d98883ca..79e906ef592 100644 --- a/crates/e2e-test-utils/src/testsuite/mod.rs +++ b/crates/e2e-test-utils/src/testsuite/mod.rs @@ -6,104 +6,307 @@ use crate::{ }; use alloy_primitives::B256; use eyre::Result; -use jsonrpsee::http_client::{transport::HttpBackend, HttpClient}; +use jsonrpsee::http_client::HttpClient; use reth_engine_local::LocalPayloadAttributesBuilder; -use reth_node_api::{NodeTypes, PayloadTypes}; +use reth_node_api::{EngineTypes, NodeTypes, PayloadTypes}; use reth_payload_builder::PayloadId; -use reth_rpc_layer::AuthClientService; -use setup::Setup; use std::{collections::HashMap, marker::PhantomData}; pub mod actions; pub mod setup; +use crate::testsuite::setup::Setup; +use alloy_provider::{Provider, ProviderBuilder}; use alloy_rpc_types_engine::{ForkchoiceState, PayloadAttributes}; - -#[cfg(test)] -mod examples; +use reth_engine_primitives::ConsensusEngineHandle; +use reth_rpc_builder::auth::AuthServerHandle; +use std::sync::Arc; +use url::Url; /// Client handles for both regular RPC and Engine API endpoints -#[derive(Debug)] -pub struct NodeClient { +#[derive(Clone)] +pub struct NodeClient +where + Payload: PayloadTypes, +{ /// Regular JSON-RPC client pub rpc: HttpClient, /// Engine API client - pub engine: HttpClient>, + pub engine: AuthServerHandle, + /// Beacon consensus engine handle for direct interaction with the consensus engine + pub beacon_engine_handle: Option>, + /// Alloy provider for interacting with the node + provider: Arc, +} + +impl NodeClient +where + Payload: PayloadTypes, +{ + /// Instantiates a new [`NodeClient`] with the given handles and RPC URL + pub fn new(rpc: HttpClient, engine: AuthServerHandle, url: Url) -> Self { + let provider = + Arc::new(ProviderBuilder::new().connect_http(url)) as Arc; + Self { rpc, engine, beacon_engine_handle: None, provider } + } + + /// Instantiates a new [`NodeClient`] with the given handles, RPC URL, and beacon engine handle + pub fn new_with_beacon_engine( + rpc: HttpClient, + engine: AuthServerHandle, + url: Url, + beacon_engine_handle: ConsensusEngineHandle, + ) -> Self { + let provider = + Arc::new(ProviderBuilder::new().connect_http(url)) as Arc; + Self { rpc, engine, beacon_engine_handle: Some(beacon_engine_handle), provider } + } + + /// Get a block by number using the alloy provider + pub async fn get_block_by_number( + &self, + number: alloy_eips::BlockNumberOrTag, + ) -> Result> { + self.provider + .get_block_by_number(number) + .await + .map_err(|e| eyre::eyre!("Failed to get block by number: {}", e)) + } + + /// Check if the node is ready by attempting to get the latest block + pub async fn is_ready(&self) -> bool { + self.get_block_by_number(alloy_eips::BlockNumberOrTag::Latest).await.is_ok() + } } -/// Represents the latest block information. -#[derive(Debug, Clone)] -pub struct LatestBlockInfo { - /// Hash of the latest block +impl std::fmt::Debug for NodeClient +where + Payload: PayloadTypes, +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("NodeClient") + .field("rpc", &self.rpc) + .field("engine", &self.engine) + .field("beacon_engine_handle", &self.beacon_engine_handle.is_some()) + .field("provider", &"") + .finish() + } +} + +/// Represents complete block information. +#[derive(Debug, Clone, Copy)] +pub struct BlockInfo { + /// Hash of the block pub hash: B256, - /// Number of the latest block + /// Number of the block pub number: u64, + /// Timestamp of the block + pub timestamp: u64, +} + +/// Per-node state tracking for multi-node environments +#[derive(Clone)] +pub struct NodeState +where + I: EngineTypes, +{ + /// Current block information for this node + pub current_block_info: Option, + /// Stores payload attributes indexed by block number for this node + pub payload_attributes: HashMap, + /// Tracks the latest block header timestamp for this node + pub latest_header_time: u64, + /// Stores payload IDs returned by this node, indexed by block number + pub payload_id_history: HashMap, + /// Stores the next expected payload ID for this node + pub next_payload_id: Option, + /// Stores the latest fork choice state for this node + pub latest_fork_choice_state: ForkchoiceState, + /// Stores the most recent built execution payload for this node + pub latest_payload_built: Option, + /// Stores the most recent executed payload for this node + pub latest_payload_executed: Option, + /// Stores the most recent built execution payload envelope for this node + pub latest_payload_envelope: Option, + /// Fork base block number for validation (if this node is currently on a fork) + pub current_fork_base: Option, } + +impl Default for NodeState +where + I: EngineTypes, +{ + fn default() -> Self { + Self { + current_block_info: None, + payload_attributes: HashMap::new(), + latest_header_time: 0, + payload_id_history: HashMap::new(), + next_payload_id: None, + latest_fork_choice_state: ForkchoiceState::default(), + latest_payload_built: None, + latest_payload_executed: None, + latest_payload_envelope: None, + current_fork_base: None, + } + } +} + +impl std::fmt::Debug for NodeState +where + I: EngineTypes, +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("NodeState") + .field("current_block_info", &self.current_block_info) + .field("payload_attributes", &self.payload_attributes) + .field("latest_header_time", &self.latest_header_time) + .field("payload_id_history", &self.payload_id_history) + .field("next_payload_id", &self.next_payload_id) + .field("latest_fork_choice_state", &self.latest_fork_choice_state) + .field("latest_payload_built", &self.latest_payload_built) + .field("latest_payload_executed", &self.latest_payload_executed) + .field("latest_payload_envelope", &"") + .field("current_fork_base", &self.current_fork_base) + .finish() + } +} + /// Represents a test environment. #[derive(Debug)] -pub struct Environment { +pub struct Environment +where + I: EngineTypes, +{ /// Combined clients with both RPC and Engine API endpoints - pub node_clients: Vec, + pub node_clients: Vec>, + /// Per-node state tracking + pub node_states: Vec>, /// Tracks instance generic. _phantom: PhantomData, - /// Latest block information - pub latest_block_info: Option, /// Last producer index pub last_producer_idx: Option, - /// Stores payload attributes indexed by block number - pub payload_attributes: HashMap, - /// Tracks the latest block header timestamp - pub latest_header_time: u64, /// Defines the increment for block timestamps (default: 2 seconds) pub block_timestamp_increment: u64, - /// Stores payload IDs returned by block producers, indexed by block number - pub payload_id_history: HashMap, - /// Stores the next expected payload ID - pub next_payload_id: Option, - /// Stores the latest fork choice state - pub latest_fork_choice_state: ForkchoiceState, - /// Stores the most recent built execution payload - pub latest_payload_built: Option, - /// Stores the most recent executed payload - pub latest_payload_executed: Option, /// Number of slots until a block is considered safe pub slots_to_safe: u64, /// Number of slots until a block is considered finalized pub slots_to_finalized: u64, + /// Registry for tagged blocks, mapping tag names to block info and node index + pub block_registry: HashMap, + /// Currently active node index for backward compatibility with single-node actions + pub active_node_idx: usize, } -impl Default for Environment { +impl Default for Environment +where + I: EngineTypes, +{ fn default() -> Self { Self { node_clients: vec![], + node_states: vec![], _phantom: Default::default(), - latest_block_info: None, last_producer_idx: None, - payload_attributes: Default::default(), - latest_header_time: 0, block_timestamp_increment: 2, - payload_id_history: HashMap::new(), - next_payload_id: None, - latest_fork_choice_state: ForkchoiceState::default(), - latest_payload_built: None, - latest_payload_executed: None, slots_to_safe: 0, slots_to_finalized: 0, + block_registry: HashMap::new(), + active_node_idx: 0, + } + } +} + +impl Environment +where + I: EngineTypes, +{ + /// Get the number of nodes in the environment + pub const fn node_count(&self) -> usize { + self.node_clients.len() + } + + /// Get mutable reference to a specific node's state + pub fn node_state_mut(&mut self, node_idx: usize) -> Result<&mut NodeState, eyre::Error> { + let node_count = self.node_count(); + self.node_states.get_mut(node_idx).ok_or_else(|| { + eyre::eyre!("Node index {} out of bounds (have {} nodes)", node_idx, node_count) + }) + } + + /// Get immutable reference to a specific node's state + pub fn node_state(&self, node_idx: usize) -> Result<&NodeState, eyre::Error> { + self.node_states.get(node_idx).ok_or_else(|| { + eyre::eyre!("Node index {} out of bounds (have {} nodes)", node_idx, self.node_count()) + }) + } + + /// Get the currently active node's state + pub fn active_node_state(&self) -> Result<&NodeState, eyre::Error> { + self.node_state(self.active_node_idx) + } + + /// Get mutable reference to the currently active node's state + pub fn active_node_state_mut(&mut self) -> Result<&mut NodeState, eyre::Error> { + let idx = self.active_node_idx; + self.node_state_mut(idx) + } + + /// Set the active node index + pub fn set_active_node(&mut self, node_idx: usize) -> Result<(), eyre::Error> { + if node_idx >= self.node_count() { + return Err(eyre::eyre!( + "Node index {} out of bounds (have {} nodes)", + node_idx, + self.node_count() + )); } + self.active_node_idx = node_idx; + Ok(()) + } + + /// Initialize node states when nodes are created + pub fn initialize_node_states(&mut self, node_count: usize) { + self.node_states = (0..node_count).map(|_| NodeState::default()).collect(); + } + + /// Get current block info from active node + pub fn current_block_info(&self) -> Option { + self.active_node_state().ok()?.current_block_info + } + + /// Set current block info on active node + pub fn set_current_block_info(&mut self, block_info: BlockInfo) -> Result<(), eyre::Error> { + self.active_node_state_mut()?.current_block_info = Some(block_info); + Ok(()) } } /// Builder for creating test scenarios #[expect(missing_debug_implementations)] -#[derive(Default)] -pub struct TestBuilder { +pub struct TestBuilder +where + I: EngineTypes, +{ setup: Option>, actions: Vec>, env: Environment, } -impl TestBuilder { +impl Default for TestBuilder +where + I: EngineTypes, +{ + fn default() -> Self { + Self { setup: None, actions: Vec::new(), env: Default::default() } + } +} + +impl TestBuilder +where + I: EngineTypes + 'static, +{ /// Create a new test builder pub fn new() -> Self { - Self { setup: None, actions: Vec::new(), env: Default::default() } + Self::default() } /// Set the test setup @@ -112,6 +315,17 @@ impl TestBuilder { self } + /// Set the test setup with chain import from RLP file + pub fn with_setup_and_import( + mut self, + mut setup: Setup, + rlp_path: impl Into, + ) -> Self { + setup.import_rlp_path = Some(rlp_path.into()); + self.setup = Some(setup); + self + } + /// Add an action to the test pub fn with_action(mut self, action: A) -> Self where @@ -134,7 +348,7 @@ impl TestBuilder { /// Run the test scenario pub async fn run(mut self) -> Result<()> where - N: NodeBuilderHelper, + N: NodeBuilderHelper, LocalPayloadAttributesBuilder: PayloadAttributesBuilder< <::Payload as PayloadTypes>::PayloadAttributes, >, diff --git a/crates/e2e-test-utils/src/testsuite/setup.rs b/crates/e2e-test-utils/src/testsuite/setup.rs index 8da4f44d3eb..94f661753b5 100644 --- a/crates/e2e-test-utils/src/testsuite/setup.rs +++ b/crates/e2e-test-utils/src/testsuite/setup.rs @@ -1,27 +1,29 @@ //! Test setup utilities for configuring the initial state. -use crate::{setup_engine, testsuite::Environment, NodeBuilderHelper, PayloadAttributesBuilder}; +use crate::{ + setup_engine_with_connection, testsuite::Environment, NodeBuilderHelper, + PayloadAttributesBuilder, +}; use alloy_eips::BlockNumberOrTag; use alloy_primitives::B256; -use alloy_rpc_types_engine::PayloadAttributes; -use alloy_rpc_types_eth::{Block as RpcBlock, Header, Receipt, Transaction}; +use alloy_rpc_types_engine::{ForkchoiceState, PayloadAttributes}; use eyre::{eyre, Result}; use reth_chainspec::ChainSpec; use reth_engine_local::LocalPayloadAttributesBuilder; use reth_ethereum_primitives::Block; -use reth_node_api::{NodeTypes, PayloadTypes}; +use reth_network_p2p::sync::{NetworkSyncUpdater, SyncState}; +use reth_node_api::{EngineTypes, NodeTypes, PayloadTypes, TreeConfig}; use reth_node_core::primitives::RecoveredBlock; use reth_payload_builder::EthPayloadBuilderAttributes; -use reth_rpc_api::clients::EthApiClient; use revm::state::EvmState; -use std::{marker::PhantomData, sync::Arc}; +use std::{marker::PhantomData, path::Path, sync::Arc}; use tokio::{ sync::mpsc, time::{sleep, Duration}, }; -use tracing::{debug, error}; +use tracing::debug; -/// Configuration for setting upa test environment +/// Configuration for setting up test environment #[derive(Debug)] pub struct Setup { /// Chain specification to use @@ -34,12 +36,19 @@ pub struct Setup { pub state: Option, /// Network configuration pub network: NetworkSetup, + /// Engine tree configuration + pub tree_config: TreeConfig, /// Shutdown channel to stop nodes when setup is dropped shutdown_tx: Option>, /// Is this setup in dev mode pub is_dev: bool, /// Tracks instance generic. _phantom: PhantomData, + /// Holds the import result to keep nodes alive when using imported chain + /// This is stored as an option to avoid lifetime issues with `tokio::spawn` + import_result_holder: Option, + /// Path to RLP file to import during setup + pub import_rlp_path: Option, } impl Default for Setup { @@ -50,9 +59,12 @@ impl Default for Setup { blocks: Vec::new(), state: None, network: NetworkSetup::default(), + tree_config: TreeConfig::default(), shutdown_tx: None, is_dev: true, _phantom: Default::default(), + import_result_holder: None, + import_rlp_path: None, } } } @@ -66,12 +78,10 @@ impl Drop for Setup { } } -impl Setup { - /// Create a new setup with default values - pub fn new() -> Self { - Self::default() - } - +impl Setup +where + I: EngineTypes, +{ /// Set the chain specification pub fn with_chain_spec(mut self, chain_spec: Arc) -> Self { self.chain_spec = Some(chain_spec); @@ -114,42 +124,106 @@ impl Setup { self } + /// Set the engine tree configuration + pub const fn with_tree_config(mut self, tree_config: TreeConfig) -> Self { + self.tree_config = tree_config; + self + } + + /// Apply setup using pre-imported chain data from RLP file + pub async fn apply_with_import( + &mut self, + env: &mut Environment, + rlp_path: &Path, + ) -> Result<()> + where + N: NodeBuilderHelper, + LocalPayloadAttributesBuilder: PayloadAttributesBuilder< + <::Payload as PayloadTypes>::PayloadAttributes, + >, + { + // Note: this future is quite large so we box it + Box::pin(self.apply_with_import_::(env, rlp_path)).await + } + + /// Apply setup using pre-imported chain data from RLP file + async fn apply_with_import_( + &mut self, + env: &mut Environment, + rlp_path: &Path, + ) -> Result<()> + where + N: NodeBuilderHelper, + LocalPayloadAttributesBuilder: PayloadAttributesBuilder< + <::Payload as PayloadTypes>::PayloadAttributes, + >, + { + // Create nodes with imported chain data + let import_result = self.create_nodes_with_import::(rlp_path).await?; + + // Extract node clients + let mut node_clients = Vec::new(); + let nodes = &import_result.nodes; + for node in nodes { + let rpc = node + .rpc_client() + .ok_or_else(|| eyre!("Failed to create HTTP RPC client for node"))?; + let auth = node.auth_server_handle(); + let url = node.rpc_url(); + // TODO: Pass beacon_engine_handle once import system supports generic types + node_clients.push(crate::testsuite::NodeClient::new(rpc, auth, url)); + } + + // Store the import result to keep nodes alive + // They will be dropped when the Setup is dropped + self.import_result_holder = Some(import_result); + + // Finalize setup - this will wait for nodes and initialize states + self.finalize_setup(env, node_clients, true).await + } + /// Apply the setup to the environment pub async fn apply(&mut self, env: &mut Environment) -> Result<()> where - N: NodeBuilderHelper, + N: NodeBuilderHelper, + LocalPayloadAttributesBuilder: PayloadAttributesBuilder< + <::Payload as PayloadTypes>::PayloadAttributes, + >, + { + // Note: this future is quite large so we box it + Box::pin(self.apply_::(env)).await + } + + /// Apply the setup to the environment + async fn apply_(&mut self, env: &mut Environment) -> Result<()> + where + N: NodeBuilderHelper, LocalPayloadAttributesBuilder: PayloadAttributesBuilder< <::Payload as PayloadTypes>::PayloadAttributes, >, { + // If import_rlp_path is set, use apply_with_import instead + if let Some(rlp_path) = self.import_rlp_path.take() { + return self.apply_with_import::(env, &rlp_path).await; + } let chain_spec = self.chain_spec.clone().ok_or_else(|| eyre!("Chain specification is required"))?; let (shutdown_tx, mut shutdown_rx) = mpsc::channel(1); - self.shutdown_tx = Some(shutdown_tx); let is_dev = self.is_dev; let node_count = self.network.node_count; - let attributes_generator = move |timestamp| { - let attributes = PayloadAttributes { - timestamp, - prev_randao: B256::ZERO, - suggested_fee_recipient: alloy_primitives::Address::ZERO, - withdrawals: Some(vec![]), - parent_beacon_block_root: Some(B256::ZERO), - }; - <::Payload as PayloadTypes>::PayloadBuilderAttributes::from( - EthPayloadBuilderAttributes::new(B256::ZERO, attributes), - ) - }; + let attributes_generator = Self::create_static_attributes_generator::(); - let result = setup_engine::( + let result = setup_engine_with_connection::( node_count, Arc::::new((*chain_spec).clone().into()), is_dev, + self.tree_config.clone(), attributes_generator, + self.network.connect_nodes, ) .await; @@ -158,12 +232,7 @@ impl Setup { Ok((nodes, executor, _wallet)) => { // create HTTP clients for each node's RPC and Engine API endpoints for node in &nodes { - let rpc = node - .rpc_client() - .ok_or_else(|| eyre!("Failed to create HTTP RPC client for node"))?; - let engine = node.engine_api_client(); - - node_clients.push(crate::testsuite::NodeClient { rpc, engine }); + node_clients.push(node.to_node_client()?); } // spawn a separate task just to handle the shutdown @@ -177,53 +246,195 @@ impl Setup { }); } Err(e) => { - error!("Failed to setup nodes: {}", e); return Err(eyre!("Failed to setup nodes: {}", e)); } } + // Finalize setup + self.finalize_setup(env, node_clients, false).await + } + + /// Create nodes with imported chain data + /// + /// Note: Currently this only supports `EthereumNode` due to the import process + /// being Ethereum-specific. The generic parameter N is kept for consistency + /// with other methods but is not used. + async fn create_nodes_with_import( + &self, + rlp_path: &Path, + ) -> Result + where + N: NodeBuilderHelper, + LocalPayloadAttributesBuilder: PayloadAttributesBuilder< + <::Payload as PayloadTypes>::PayloadAttributes, + >, + { + let chain_spec = + self.chain_spec.clone().ok_or_else(|| eyre!("Chain specification is required"))?; + + let attributes_generator = move |timestamp| { + let attributes = PayloadAttributes { + timestamp, + prev_randao: B256::ZERO, + suggested_fee_recipient: alloy_primitives::Address::ZERO, + withdrawals: Some(vec![]), + parent_beacon_block_root: Some(B256::ZERO), + }; + EthPayloadBuilderAttributes::new(B256::ZERO, attributes) + }; + + crate::setup_import::setup_engine_with_chain_import( + self.network.node_count, + chain_spec, + self.is_dev, + self.tree_config.clone(), + rlp_path, + attributes_generator, + ) + .await + } + + /// Create a static attributes generator that doesn't capture any instance data + fn create_static_attributes_generator( + ) -> impl Fn(u64) -> <::Payload as PayloadTypes>::PayloadBuilderAttributes + + Copy + + use + where + N: NodeBuilderHelper, + LocalPayloadAttributesBuilder: PayloadAttributesBuilder< + <::Payload as PayloadTypes>::PayloadAttributes, + >, + { + move |timestamp| { + let attributes = PayloadAttributes { + timestamp, + prev_randao: B256::ZERO, + suggested_fee_recipient: alloy_primitives::Address::ZERO, + withdrawals: Some(vec![]), + parent_beacon_block_root: Some(B256::ZERO), + }; + <::Payload as PayloadTypes>::PayloadBuilderAttributes::from( + EthPayloadBuilderAttributes::new(B256::ZERO, attributes), + ) + } + } + + /// Common finalization logic for both apply methods + async fn finalize_setup( + &self, + env: &mut Environment, + node_clients: Vec>, + use_latest_block: bool, + ) -> Result<()> { if node_clients.is_empty() { return Err(eyre!("No nodes were created")); } - // wait for all nodes to be ready to accept RPC requests before proceeding + // Wait for all nodes to be ready + self.wait_for_nodes_ready(&node_clients).await?; + + env.node_clients = node_clients; + env.initialize_node_states(self.network.node_count); + + // Get initial block info (genesis or latest depending on use_latest_block) + let (initial_block_info, genesis_block_info) = if use_latest_block { + // For imported chain, get both latest and genesis + let latest = + self.get_block_info(&env.node_clients[0], BlockNumberOrTag::Latest).await?; + let genesis = + self.get_block_info(&env.node_clients[0], BlockNumberOrTag::Number(0)).await?; + (latest, genesis) + } else { + // For fresh chain, both are genesis + let genesis = + self.get_block_info(&env.node_clients[0], BlockNumberOrTag::Number(0)).await?; + (genesis, genesis) + }; + + // Initialize all node states + for (node_idx, node_state) in env.node_states.iter_mut().enumerate() { + node_state.current_block_info = Some(initial_block_info); + node_state.latest_header_time = initial_block_info.timestamp; + node_state.latest_fork_choice_state = ForkchoiceState { + head_block_hash: initial_block_info.hash, + safe_block_hash: initial_block_info.hash, + finalized_block_hash: genesis_block_info.hash, + }; + + debug!( + "Node {} initialized with block {} (hash: {})", + node_idx, initial_block_info.number, initial_block_info.hash + ); + } + + debug!( + "Environment initialized with {} nodes, starting from block {} (hash: {})", + self.network.node_count, initial_block_info.number, initial_block_info.hash + ); + + // In test environments, explicitly set sync state to Idle after initialization + // This ensures that eth_syncing returns false as expected by tests + if let Some(import_result) = &self.import_result_holder { + for (idx, node_ctx) in import_result.nodes.iter().enumerate() { + debug!("Setting sync state to Idle for node {}", idx); + node_ctx.inner.network.update_sync_state(SyncState::Idle); + } + } + + Ok(()) + } + + /// Wait for all nodes to be ready to accept RPC requests + async fn wait_for_nodes_ready

( + &self, + node_clients: &[crate::testsuite::NodeClient

], + ) -> Result<()> + where + P: PayloadTypes, + { for (idx, client) in node_clients.iter().enumerate() { let mut retry_count = 0; - const MAX_RETRIES: usize = 5; - let mut last_error = None; + const MAX_RETRIES: usize = 10; while retry_count < MAX_RETRIES { - match EthApiClient::::block_by_number( - &client.rpc, - BlockNumberOrTag::Latest, - false, - ) - .await - { - Ok(_) => { - debug!("Node {idx} RPC endpoint is ready"); - break; - } - Err(e) => { - last_error = Some(e); - retry_count += 1; - debug!( - "Node {idx} RPC endpoint not ready, retry {retry_count}/{MAX_RETRIES}" - ); - sleep(Duration::from_millis(500)).await; - } + if client.is_ready().await { + debug!("Node {idx} RPC endpoint is ready"); + break; } + + retry_count += 1; + debug!("Node {idx} RPC endpoint not ready, retry {retry_count}/{MAX_RETRIES}"); + sleep(Duration::from_millis(500)).await; } + if retry_count == MAX_RETRIES { - return Err(eyre!("Failed to connect to node {idx} RPC endpoint after {MAX_RETRIES} retries: {:?}", last_error)); + return Err(eyre!( + "Failed to connect to node {idx} RPC endpoint after {MAX_RETRIES} retries" + )); } } + Ok(()) + } - env.node_clients = node_clients; - - // TODO: For each block in self.blocks, replay it on the node + /// Get block info for a given block number or tag + async fn get_block_info

( + &self, + client: &crate::testsuite::NodeClient

, + block: BlockNumberOrTag, + ) -> Result + where + P: PayloadTypes, + { + let block = client + .get_block_by_number(block) + .await? + .ok_or_else(|| eyre!("Block {:?} not found", block))?; - Ok(()) + Ok(crate::testsuite::BlockInfo { + hash: block.header.hash, + number: block.header.number, + timestamp: block.header.timestamp, + }) } } @@ -236,16 +447,23 @@ pub struct Genesis {} pub struct NetworkSetup { /// Number of nodes to create pub node_count: usize, + /// Whether nodes should be connected to each other + pub connect_nodes: bool, } impl NetworkSetup { /// Create a new network setup with a single node pub const fn single_node() -> Self { - Self { node_count: 1 } + Self { node_count: 1, connect_nodes: true } } - /// Create a new network setup with multiple nodes + /// Create a new network setup with multiple nodes (connected) pub const fn multi_node(count: usize) -> Self { - Self { node_count: count } + Self { node_count: count, connect_nodes: true } + } + + /// Create a new network setup with multiple nodes (disconnected) + pub const fn multi_node_unconnected(count: usize) -> Self { + Self { node_count: count, connect_nodes: false } } } diff --git a/crates/e2e-test-utils/src/transaction.rs b/crates/e2e-test-utils/src/transaction.rs index 1fc05fb5382..dd49ac76195 100644 --- a/crates/e2e-test-utils/src/transaction.rs +++ b/crates/e2e-test-utils/src/transaction.rs @@ -1,5 +1,7 @@ -use alloy_consensus::{EnvKzgSettings, SidecarBuilder, SimpleCoder, TxEip4844Variant, TxEnvelope}; -use alloy_eips::eip7702::SignedAuthorization; +use alloy_consensus::{ + EnvKzgSettings, EthereumTxEnvelope, SidecarBuilder, SimpleCoder, TxEip4844Variant, TxEnvelope, +}; +use alloy_eips::{eip7594::BlobTransactionSidecarVariant, eip7702::SignedAuthorization}; use alloy_network::{ eip2718::Encodable2718, Ethereum, EthereumWallet, TransactionBuilder, TransactionBuilder4844, }; @@ -16,7 +18,17 @@ pub struct TransactionTestContext; impl TransactionTestContext { /// Creates a static transfer and signs it, returning an envelope. pub async fn transfer_tx(chain_id: u64, wallet: PrivateKeySigner) -> TxEnvelope { - let tx = tx(chain_id, 21000, None, None, 0); + let tx = tx(chain_id, 21000, None, None, 0, Some(20e9 as u128)); + Self::sign_tx(wallet, tx).await + } + + /// Same as `transfer_tx`, but could set max fee per gas. + pub async fn transfer_tx_with_gas_fee( + chain_id: u64, + max_fee_per_gas: Option, + wallet: PrivateKeySigner, + ) -> TxEnvelope { + let tx = tx(chain_id, 21000, None, None, 0, max_fee_per_gas); Self::sign_tx(wallet, tx).await } @@ -33,7 +45,7 @@ impl TransactionTestContext { init_code: Bytes, wallet: PrivateKeySigner, ) -> TxEnvelope { - let tx = tx(chain_id, gas, Some(init_code), None, 0); + let tx = tx(chain_id, gas, Some(init_code), None, 0, Some(20e9 as u128)); Self::sign_tx(wallet, tx).await } @@ -61,7 +73,14 @@ impl TransactionTestContext { let signature = wallet .sign_hash_sync(&authorization.signature_hash()) .expect("could not sign authorization"); - let tx = tx(chain_id, 48100, None, Some(authorization.into_signed(signature)), 0); + let tx = tx( + chain_id, + 48100, + None, + Some(authorization.into_signed(signature)), + 0, + Some(20e9 as u128), + ); Self::sign_tx(wallet, tx).await } @@ -82,7 +101,7 @@ impl TransactionTestContext { chain_id: u64, wallet: PrivateKeySigner, ) -> eyre::Result { - let mut tx = tx(chain_id, 210000, None, None, 0); + let mut tx = tx(chain_id, 210000, None, None, 0, Some(20e9 as u128)); let mut builder = SidecarBuilder::::new(); builder.ingest(b"dummy blob"); @@ -118,7 +137,7 @@ impl TransactionTestContext { let l1_block_info = Bytes::from_static(&hex!( "7ef9015aa044bae9d41b8380d781187b426c6fe43df5fb2fb57bd4466ef6a701e1f01e015694deaddeaddeaddeaddeaddeaddeaddeaddead000194420000000000000000000000000000000000001580808408f0d18001b90104015d8eb900000000000000000000000000000000000000000000000000000000008057650000000000000000000000000000000000000000000000000000000063d96d10000000000000000000000000000000000000000000000000000000000009f35273d89754a1e0387b89520d989d3be9c37c1f32495a88faf1ea05c61121ab0d1900000000000000000000000000000000000000000000000000000000000000010000000000000000000000002d679b567db6187c0c8323fa982cfb88b74dbcc7000000000000000000000000000000000000000000000000000000000000083400000000000000000000000000000000000000000000000000000000000f4240" )); - let tx = tx(chain_id, 210000, Some(l1_block_info), None, nonce); + let tx = tx(chain_id, 210000, Some(l1_block_info), None, nonce, Some(20e9 as u128)); let signer = EthereumWallet::from(wallet); >::build(tx, &signer) .await @@ -129,11 +148,13 @@ impl TransactionTestContext { /// Validates the sidecar of a given tx envelope and returns the versioned hashes #[track_caller] - pub fn validate_sidecar(tx: TxEnvelope) -> Vec { + pub fn validate_sidecar( + tx: EthereumTxEnvelope>, + ) -> Vec { let proof_setting = EnvKzgSettings::Default; match tx { - TxEnvelope::Eip4844(signed) => match signed.tx() { + EthereumTxEnvelope::Eip4844(signed) => match signed.tx() { TxEip4844Variant::TxEip4844WithSidecar(tx) => { tx.validate_blob(proof_setting.get()).unwrap(); tx.sidecar.versioned_hashes().collect() @@ -152,13 +173,14 @@ fn tx( data: Option, delegate_to: Option, nonce: u64, + max_fee_per_gas: Option, ) -> TransactionRequest { TransactionRequest { nonce: Some(nonce), value: Some(U256::from(100)), to: Some(TxKind::Call(Address::random())), gas: Some(gas), - max_fee_per_gas: Some(20e9 as u128), + max_fee_per_gas, max_priority_fee_per_gas: Some(20e9 as u128), chain_id: Some(chain_id), input: TransactionInput { input: None, data }, diff --git a/crates/e2e-test-utils/src/wallet.rs b/crates/e2e-test-utils/src/wallet.rs index 099918ad637..92af590e705 100644 --- a/crates/e2e-test-utils/src/wallet.rs +++ b/crates/e2e-test-utils/src/wallet.rs @@ -43,7 +43,7 @@ impl Wallet { let builder = builder.clone().derivation_path(format!("{derivation_path}{idx}")).unwrap(); let wallet = builder.build().unwrap().with_chain_id(Some(self.chain_id)); - wallets.push(wallet) + wallets.push(wallet); } wallets } diff --git a/crates/e2e-test-utils/tests/e2e-testsuite/main.rs b/crates/e2e-test-utils/tests/e2e-testsuite/main.rs new file mode 100644 index 00000000000..4a2ac77ec65 --- /dev/null +++ b/crates/e2e-test-utils/tests/e2e-testsuite/main.rs @@ -0,0 +1,389 @@ +//! Example tests using the test suite framework. + +use alloy_primitives::{Address, B256}; +use alloy_rpc_types_engine::PayloadAttributes; +use eyre::Result; +use reth_chainspec::{ChainSpecBuilder, MAINNET}; +use reth_e2e_test_utils::{ + test_rlp_utils::{generate_test_blocks, write_blocks_to_rlp}, + testsuite::{ + actions::{ + Action, AssertChainTip, AssertMineBlock, CaptureBlock, CaptureBlockOnNode, + CompareNodeChainTips, CreateFork, MakeCanonical, ProduceBlocks, ReorgTo, + SelectActiveNode, UpdateBlockInfo, + }, + setup::{NetworkSetup, Setup}, + Environment, TestBuilder, + }, + E2ETestSetupBuilder, +}; +use reth_node_api::TreeConfig; +use reth_node_ethereum::{EthEngineTypes, EthereumNode}; +use reth_payload_builder::EthPayloadBuilderAttributes; +use std::sync::Arc; +use tempfile::TempDir; +use tracing::debug; + +#[tokio::test] +async fn test_apply_with_import() -> Result<()> { + reth_tracing::init_test_tracing(); + + // Create test chain spec + let chain_spec = Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis( + serde_json::from_str(include_str!( + "../../../../crates/e2e-test-utils/src/testsuite/assets/genesis.json" + )) + .unwrap(), + ) + .london_activated() + .shanghai_activated() + .cancun_activated() + .build(), + ); + + // Generate test blocks + let test_blocks = generate_test_blocks(&chain_spec, 10); + + // Write blocks to RLP file + let temp_dir = TempDir::new()?; + let rlp_path = temp_dir.path().join("test_chain.rlp"); + write_blocks_to_rlp(&test_blocks, &rlp_path)?; + + // Create setup with imported chain + let mut setup = + Setup::default().with_chain_spec(chain_spec).with_network(NetworkSetup::single_node()); + + // Create environment and apply setup with import + let mut env = Environment::::default(); + setup.apply_with_import::(&mut env, &rlp_path).await?; + + // Now run test actions on the environment with imported chain + // First check what block we're at after import + debug!("Current block info after import: {:?}", env.current_block_info()); + + // Update block info to sync environment state with the node + let mut update_block_info = UpdateBlockInfo::default(); + update_block_info.execute(&mut env).await?; + + // Make the imported chain canonical first + let mut make_canonical = MakeCanonical::new(); + make_canonical.execute(&mut env).await?; + + // Wait for the pipeline to finish processing all stages + debug!("Waiting for pipeline to finish processing imported blocks..."); + let start = std::time::Instant::now(); + loop { + // Check if we can get the block from RPC (indicates pipeline finished) + let client = &env.node_clients[0]; + let block_result = reth_rpc_api::clients::EthApiClient::< + alloy_rpc_types_eth::TransactionRequest, + alloy_rpc_types_eth::Transaction, + alloy_rpc_types_eth::Block, + alloy_rpc_types_eth::Receipt, + alloy_rpc_types_eth::Header, + reth_ethereum_primitives::TransactionSigned, + >::block_by_number( + &client.rpc, + alloy_eips::BlockNumberOrTag::Number(10), + true, // Include full transaction details + ) + .await; + + if let Ok(Some(block)) = block_result && + block.header.number == 10 + { + debug!("Pipeline finished, block 10 is fully available"); + break; + } + + if start.elapsed() > std::time::Duration::from_secs(10) { + return Err(eyre::eyre!("Timeout waiting for pipeline to finish")); + } + + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + + // Update block info again after making canonical + let mut update_block_info_2 = UpdateBlockInfo::default(); + update_block_info_2.execute(&mut env).await?; + + // Assert we're at block 10 after import + let mut assert_tip = AssertChainTip::new(10); + assert_tip.execute(&mut env).await?; + + debug!("Successfully imported chain to block 10"); + + // Produce 5 more blocks + let mut produce_blocks = ProduceBlocks::::new(5); + produce_blocks.execute(&mut env).await?; + + // Assert we're now at block 15 + let mut assert_new_tip = AssertChainTip::new(15); + assert_new_tip.execute(&mut env).await?; + + Ok(()) +} + +#[tokio::test] +async fn test_testsuite_assert_mine_block() -> Result<()> { + reth_tracing::init_test_tracing(); + + let setup = Setup::default() + .with_chain_spec(Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis( + serde_json::from_str(include_str!( + "../../../../crates/e2e-test-utils/src/testsuite/assets/genesis.json" + )) + .unwrap(), + ) + .paris_activated() + .build(), + )) + .with_network(NetworkSetup::single_node()); + + let test = + TestBuilder::new().with_setup(setup).with_action(AssertMineBlock::::new( + 0, + vec![], + Some(B256::ZERO), + // TODO: refactor once we have actions to generate payload attributes. + PayloadAttributes { + timestamp: std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs(), + prev_randao: B256::random(), + suggested_fee_recipient: Address::random(), + withdrawals: None, + parent_beacon_block_root: None, + }, + )); + + test.run::().await?; + + Ok(()) +} + +#[tokio::test] +async fn test_testsuite_produce_blocks() -> Result<()> { + reth_tracing::init_test_tracing(); + + let setup = Setup::default() + .with_chain_spec(Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis( + serde_json::from_str(include_str!( + "../../../../crates/e2e-test-utils/src/testsuite/assets/genesis.json" + )) + .unwrap(), + ) + .cancun_activated() + .build(), + )) + .with_network(NetworkSetup::single_node()); + + let test = TestBuilder::new() + .with_setup(setup) + .with_action(ProduceBlocks::::new(5)) + .with_action(MakeCanonical::new()); + + test.run::().await?; + + Ok(()) +} + +#[tokio::test] +async fn test_testsuite_create_fork() -> Result<()> { + reth_tracing::init_test_tracing(); + + let setup = Setup::default() + .with_chain_spec(Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis( + serde_json::from_str(include_str!( + "../../../../crates/e2e-test-utils/src/testsuite/assets/genesis.json" + )) + .unwrap(), + ) + .cancun_activated() + .build(), + )) + .with_network(NetworkSetup::single_node()); + + let test = TestBuilder::new() + .with_setup(setup) + .with_action(ProduceBlocks::::new(2)) + .with_action(MakeCanonical::new()) + .with_action(CreateFork::::new(1, 3)); + + test.run::().await?; + + Ok(()) +} + +#[tokio::test] +async fn test_testsuite_reorg_with_tagging() -> Result<()> { + reth_tracing::init_test_tracing(); + + let setup = Setup::default() + .with_chain_spec(Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis( + serde_json::from_str(include_str!( + "../../../../crates/e2e-test-utils/src/testsuite/assets/genesis.json" + )) + .unwrap(), + ) + .cancun_activated() + .build(), + )) + .with_network(NetworkSetup::single_node()); + + let test = TestBuilder::new() + .with_setup(setup) + .with_action(ProduceBlocks::::new(3)) // produce blocks 1, 2, 3 + .with_action(MakeCanonical::new()) // make main chain tip canonical + .with_action(CreateFork::::new(1, 2)) // fork from block 1, produce blocks 2', 3' + .with_action(CaptureBlock::new("fork_tip")) // tag fork tip + .with_action(ReorgTo::::new_from_tag("fork_tip")); // reorg to fork tip + + test.run::().await?; + + Ok(()) +} + +#[tokio::test] +async fn test_testsuite_deep_reorg() -> Result<()> { + reth_tracing::init_test_tracing(); + + let setup = Setup::default() + .with_chain_spec(Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis( + serde_json::from_str(include_str!( + "../../../../crates/e2e-test-utils/src/testsuite/assets/genesis.json" + )) + .unwrap(), + ) + .cancun_activated() + .build(), + )) + .with_network(NetworkSetup::single_node()) + .with_tree_config(TreeConfig::default().with_state_root_fallback(true)); + + let test = TestBuilder::new() + .with_setup(setup) + // receive newPayload and forkchoiceUpdated with block height 1 + .with_action(ProduceBlocks::::new(1)) + .with_action(MakeCanonical::new()) + .with_action(CaptureBlock::new("block1")) + // receive forkchoiceUpdated with block hash A as head (block A at height 2) + .with_action(CreateFork::::new(1, 1)) + .with_action(CaptureBlock::new("blockA_height2")) + .with_action(MakeCanonical::new()) + // receive newPayload with block hash B and height 2 + .with_action(ReorgTo::::new_from_tag("block1")) + .with_action(CreateFork::::new(1, 1)) + .with_action(CaptureBlock::new("blockB_height2")) + // receive forkchoiceUpdated with block hash B as head + .with_action(ReorgTo::::new_from_tag("blockB_height2")); + + test.run::().await?; + + Ok(()) +} + +/// Multi-node test demonstrating block creation and coordination across multiple nodes. +/// +/// This test demonstrates the working multi-node framework: +/// - Multiple nodes start from the same genesis +/// - Nodes can be selected for specific operations +/// - Block production can happen on different nodes +/// - Chain tips can be compared between nodes +/// - Node-specific state is properly tracked +#[tokio::test] +async fn test_testsuite_multinode_block_production() -> Result<()> { + reth_tracing::init_test_tracing(); + + let setup = Setup::default() + .with_chain_spec(Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis( + serde_json::from_str(include_str!( + "../../../../crates/e2e-test-utils/src/testsuite/assets/genesis.json" + )) + .unwrap(), + ) + .cancun_activated() + .build(), + )) + .with_network(NetworkSetup::multi_node(2)) // Create 2 nodes + .with_tree_config(TreeConfig::default().with_state_root_fallback(true)); + + let test = TestBuilder::new() + .with_setup(setup) + // both nodes start from genesis + .with_action(CaptureBlock::new("genesis")) + .with_action(CompareNodeChainTips::expect_same(0, 1)) + // build main chain (blocks 1-3) + .with_action(SelectActiveNode::new(0)) + .with_action(ProduceBlocks::::new(3)) + .with_action(MakeCanonical::new()) + .with_action(CaptureBlockOnNode::new("node0_tip", 0)) + .with_action(CompareNodeChainTips::expect_same(0, 1)) + // node 0 already has the state and can continue producing blocks + .with_action(ProduceBlocks::::new(2)) + .with_action(MakeCanonical::new()) + .with_action(CaptureBlockOnNode::new("node0_tip_2", 0)) + // verify both nodes remain in sync + .with_action(CompareNodeChainTips::expect_same(0, 1)); + + test.run::().await?; + + Ok(()) +} + +#[tokio::test] +async fn test_setup_builder_with_custom_tree_config() -> Result<()> { + reth_tracing::init_test_tracing(); + + let chain_spec = Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis( + serde_json::from_str(include_str!( + "../../../../crates/e2e-test-utils/src/testsuite/assets/genesis.json" + )) + .unwrap(), + ) + .cancun_activated() + .build(), + ); + + let (nodes, _tasks, _wallet) = + E2ETestSetupBuilder::::new(1, chain_spec, |_| { + EthPayloadBuilderAttributes::default() + }) + .with_tree_config_modifier(|config| { + config.with_persistence_threshold(0).with_memory_block_buffer_target(5) + }) + .build() + .await?; + + assert_eq!(nodes.len(), 1); + + let genesis_hash = nodes[0].block_hash(0); + assert_ne!(genesis_hash, B256::ZERO); + + Ok(()) +} diff --git a/crates/engine/invalid-block-hooks/Cargo.toml b/crates/engine/invalid-block-hooks/Cargo.toml index 02b4b2c4460..5b3563c7ac3 100644 --- a/crates/engine/invalid-block-hooks/Cargo.toml +++ b/crates/engine/invalid-block-hooks/Cargo.toml @@ -12,8 +12,8 @@ workspace = true [dependencies] # reth +revm.workspace = true revm-bytecode.workspace = true -reth-chainspec.workspace = true revm-database.workspace = true reth-engine-primitives.workspace = true reth-evm.workspace = true @@ -39,3 +39,13 @@ jsonrpsee.workspace = true pretty_assertions.workspace = true serde.workspace = true serde_json.workspace = true + +[dev-dependencies] +alloy-eips.workspace = true +reth-chainspec.workspace = true +reth-ethereum-primitives.workspace = true +reth-evm-ethereum.workspace = true +reth-provider = { workspace = true, features = ["test-utils"] } +reth-revm = { workspace = true, features = ["test-utils"] } +reth-testing-utils.workspace = true +tempfile.workspace = true diff --git a/crates/engine/invalid-block-hooks/src/witness.rs b/crates/engine/invalid-block-hooks/src/witness.rs index 7ddf593d337..d00f3b8287b 100644 --- a/crates/engine/invalid-block-hooks/src/witness.rs +++ b/crates/engine/invalid-block-hooks/src/witness.rs @@ -1,32 +1,50 @@ use alloy_consensus::BlockHeader; -use alloy_primitives::{keccak256, Address, B256, U256}; +use alloy_primitives::{keccak256, Address, Bytes, B256, U256}; use alloy_rpc_types_debug::ExecutionWitness; use pretty_assertions::Comparison; -use reth_chainspec::{EthChainSpec, EthereumHardforks}; use reth_engine_primitives::InvalidBlockHook; use reth_evm::{execute::Executor, ConfigureEvm}; use reth_primitives_traits::{NodePrimitives, RecoveredBlock, SealedHeader}; -use reth_provider::{BlockExecutionOutput, ChainSpecProvider, StateProviderFactory}; -use reth_revm::{database::StateProviderDatabase, db::BundleState, state::AccountInfo}; +use reth_provider::{BlockExecutionOutput, StateProvider, StateProviderFactory}; +use reth_revm::{ + database::StateProviderDatabase, + db::{BundleState, State}, +}; use reth_rpc_api::DebugApiClient; use reth_tracing::tracing::warn; use reth_trie::{updates::TrieUpdates, HashedStorage}; +use revm::state::AccountInfo; use revm_bytecode::Bytecode; -use revm_database::states::{ - reverts::{AccountInfoRevert, RevertToSlot}, - AccountStatus, StorageSlot, +use revm_database::{ + states::{reverts::AccountInfoRevert, StorageSlot}, + AccountStatus, RevertToSlot, }; use serde::Serialize; use std::{collections::BTreeMap, fmt::Debug, fs::File, io::Write, path::PathBuf}; +type CollectionResult = + (BTreeMap, BTreeMap, reth_trie::HashedPostState, BundleState); + +/// Serializable version of `BundleState` for deterministic comparison #[derive(Debug, PartialEq, Eq)] -struct AccountRevertSorted { - pub account: AccountInfoRevert, - pub storage: BTreeMap, - pub previous_status: AccountStatus, - pub wipe_storage: bool, +struct BundleStateSorted { + /// Account state + pub state: BTreeMap, + /// All created contracts in this block. + pub contracts: BTreeMap, + /// Changes to revert + /// + /// **Note**: Inside vector is *not* sorted by address. + /// + /// But it is unique by address. + pub reverts: Vec>, + /// The size of the plain state in the bundle state + pub state_size: usize, + /// The size of reverts in the bundle state + pub reverts_size: usize, } +/// Serializable version of `BundleAccount` #[derive(Debug, PartialEq, Eq)] struct BundleAccountSorted { pub info: Option, @@ -41,76 +59,120 @@ struct BundleAccountSorted { pub status: AccountStatus, } +/// Serializable version of `AccountRevert` #[derive(Debug, PartialEq, Eq)] -struct BundleStateSorted { - /// Account state - pub state: BTreeMap, - /// All created contracts in this block. - pub contracts: BTreeMap, - /// Changes to revert - /// - /// **Note**: Inside vector is *not* sorted by address. - /// - /// But it is unique by address. - pub reverts: Vec>, - /// The size of the plain state in the bundle state - pub state_size: usize, - /// The size of reverts in the bundle state - pub reverts_size: usize, +struct AccountRevertSorted { + pub account: AccountInfoRevert, + pub storage: BTreeMap, + pub previous_status: AccountStatus, + pub wipe_storage: bool, } -impl BundleStateSorted { - fn from_bundle_state(bundle_state: &BundleState) -> Self { - let state = bundle_state +/// Converts bundle state to sorted format for deterministic comparison +fn sort_bundle_state_for_comparison(bundle_state: &BundleState) -> BundleStateSorted { + BundleStateSorted { + state: bundle_state .state - .clone() - .into_iter() - .map(|(address, account)| { - { - ( - address, - BundleAccountSorted { - info: account.info, - original_info: account.original_info, - status: account.status, - storage: BTreeMap::from_iter(account.storage), - }, - ) - } + .iter() + .map(|(addr, acc)| { + ( + *addr, + BundleAccountSorted { + info: acc.info.clone(), + original_info: acc.original_info.clone(), + storage: BTreeMap::from_iter(acc.storage.clone()), + status: acc.status, + }, + ) }) - .collect(); - - let contracts = BTreeMap::from_iter(bundle_state.contracts.clone()); - - let reverts = bundle_state + .collect(), + contracts: BTreeMap::from_iter(bundle_state.contracts.clone()), + reverts: bundle_state .reverts .iter() .map(|block| { block .iter() - .map(|(address, account_revert)| { + .map(|(addr, rev)| { ( - *address, + *addr, AccountRevertSorted { - account: account_revert.account.clone(), - previous_status: account_revert.previous_status, - wipe_storage: account_revert.wipe_storage, - storage: BTreeMap::from_iter(account_revert.storage.clone()), + account: rev.account.clone(), + storage: BTreeMap::from_iter(rev.storage.clone()), + previous_status: rev.previous_status, + wipe_storage: rev.wipe_storage, }, ) }) .collect() }) - .collect(); + .collect(), + state_size: bundle_state.state_size, + reverts_size: bundle_state.reverts_size, + } +} + +/// Extracts execution data including codes, preimages, and hashed state from database +fn collect_execution_data( + mut db: State>>, +) -> eyre::Result { + let bundle_state = db.take_bundle(); + let mut codes = BTreeMap::new(); + let mut preimages = BTreeMap::new(); + let mut hashed_state = db.database.hashed_post_state(&bundle_state); + + // Collect codes + db.cache.contracts.values().chain(bundle_state.contracts.values()).for_each(|code| { + let code_bytes = code.original_bytes(); + codes.insert(keccak256(&code_bytes), code_bytes); + }); - let state_size = bundle_state.state_size; - let reverts_size = bundle_state.reverts_size; + // Collect preimages + for (address, account) in db.cache.accounts { + let hashed_address = keccak256(address); + hashed_state + .accounts + .insert(hashed_address, account.account.as_ref().map(|a| a.info.clone().into())); - Self { state, contracts, reverts, state_size, reverts_size } + if let Some(account_data) = account.account { + preimages.insert(hashed_address, alloy_rlp::encode(address).into()); + let storage = hashed_state + .storages + .entry(hashed_address) + .or_insert_with(|| HashedStorage::new(account.status.was_destroyed())); + + for (slot, value) in account_data.storage { + let slot_bytes = B256::from(slot); + let hashed_slot = keccak256(slot_bytes); + storage.storage.insert(hashed_slot, value); + preimages.insert(hashed_slot, alloy_rlp::encode(slot_bytes).into()); + } + } } + + Ok((codes, preimages, hashed_state, bundle_state)) } -/// Generates a witness for the given block and saves it to a file. +/// Generates execution witness from collected codes, preimages, and hashed state +fn generate( + codes: BTreeMap, + preimages: BTreeMap, + hashed_state: reth_trie::HashedPostState, + state_provider: Box, +) -> eyre::Result { + let state = state_provider.witness(Default::default(), hashed_state)?; + Ok(ExecutionWitness { + state, + codes: codes.into_values().collect(), + keys: preimages.into_values().collect(), + ..Default::default() + }) +} + +/// Hook for generating execution witnesses when invalid blocks are detected. +/// +/// This hook captures the execution state and generates witness data that can be used +/// for debugging and analysis of invalid block execution. #[derive(Debug)] pub struct InvalidBlockWitnessHook { /// The provider to read the historical state and do the EVM execution. @@ -138,111 +200,55 @@ impl InvalidBlockWitnessHook { impl InvalidBlockWitnessHook where - P: StateProviderFactory - + ChainSpecProvider - + Send - + Sync - + 'static, + P: StateProviderFactory + Send + Sync + 'static, E: ConfigureEvm + 'static, N: NodePrimitives, { - fn on_invalid_block( + /// Re-executes the block and collects execution data + fn re_execute_block( &self, parent_header: &SealedHeader, block: &RecoveredBlock, - output: &BlockExecutionOutput, - trie_updates: Option<(&TrieUpdates, B256)>, - ) -> eyre::Result<()> - where - N: NodePrimitives, - { - // TODO(alexey): unify with `DebugApi::debug_execution_witness` - + ) -> eyre::Result<(ExecutionWitness, BundleState)> { let mut executor = self.evm_config.batch_executor(StateProviderDatabase::new( self.provider.state_by_block_hash(parent_header.hash())?, )); executor.execute_one(block)?; + let db = executor.into_state(); + let (codes, preimages, hashed_state, bundle_state) = collect_execution_data(db)?; - // Take the bundle state - let mut db = executor.into_state(); - let mut bundle_state = db.take_bundle(); - - // Initialize a map of preimages. - let mut state_preimages = Vec::default(); - - // Get codes - let codes = db - .cache - .contracts - .values() - .map(|code| code.original_bytes()) - .chain( - // cache state does not have all the contracts, especially when - // a contract is created within the block - // the contract only exists in bundle state, therefore we need - // to include them as well - bundle_state.contracts.values().map(|code| code.original_bytes()), - ) - .collect(); - - // Grab all account proofs for the data accessed during block execution. - // - // Note: We grab *all* accounts in the cache here, as the `BundleState` prunes - // referenced accounts + storage slots. - let mut hashed_state = db.database.hashed_post_state(&bundle_state); - for (address, account) in db.cache.accounts { - let hashed_address = keccak256(address); - hashed_state - .accounts - .insert(hashed_address, account.account.as_ref().map(|a| a.info.clone().into())); + let state_provider = self.provider.state_by_block_hash(parent_header.hash())?; + let witness = generate(codes, preimages, hashed_state, state_provider)?; - let storage = hashed_state - .storages - .entry(hashed_address) - .or_insert_with(|| HashedStorage::new(account.status.was_destroyed())); - - if let Some(account) = account.account { - state_preimages.push(alloy_rlp::encode(address).into()); - - for (slot, value) in account.storage { - let slot = B256::from(slot); - let hashed_slot = keccak256(slot); - storage.storage.insert(hashed_slot, value); + Ok((witness, bundle_state)) + } - state_preimages.push(alloy_rlp::encode(slot).into()); - } - } - } + /// Handles witness generation, saving, and comparison with healthy node + fn handle_witness_operations( + &self, + witness: &ExecutionWitness, + block_prefix: &str, + block_number: u64, + ) -> eyre::Result<()> { + let filename = format!("{}.witness.re_executed.json", block_prefix); + let re_executed_witness_path = self.save_file(filename, witness)?; - // Generate an execution witness for the aggregated state of accessed accounts. - // Destruct the cache database to retrieve the state provider. - let state_provider = db.database.into_inner(); - let state = state_provider.witness(Default::default(), hashed_state.clone())?; - - // Write the witness to the output directory. - let response = - ExecutionWitness { state, codes, keys: state_preimages, ..Default::default() }; - let re_executed_witness_path = self.save_file( - format!("{}_{}.witness.re_executed.json", block.number(), block.hash()), - &response, - )?; if let Some(healthy_node_client) = &self.healthy_node_client { - // Compare the witness against the healthy node. let healthy_node_witness = futures::executor::block_on(async move { - DebugApiClient::debug_execution_witness(healthy_node_client, block.number().into()) - .await + DebugApiClient::<()>::debug_execution_witness( + healthy_node_client, + block_number.into(), + ) + .await })?; - let healthy_path = self.save_file( - format!("{}_{}.witness.healthy.json", block.number(), block.hash()), - &healthy_node_witness, - )?; + let filename = format!("{}.witness.healthy.json", block_prefix); + let healthy_path = self.save_file(filename, &healthy_node_witness)?; - // If the witnesses are different, write the diff to the output directory. - if response != healthy_node_witness { - let filename = format!("{}_{}.witness.diff", block.number(), block.hash()); - let diff_path = self.save_diff(filename, &response, &healthy_node_witness)?; + if witness != &healthy_node_witness { + let filename = format!("{}.witness.diff", block_prefix); + let diff_path = self.save_diff(filename, witness, &healthy_node_witness)?; warn!( target: "engine::invalid_block_hooks::witness", diff_path = %diff_path.display(), @@ -252,39 +258,26 @@ where ); } } + Ok(()) + } - // The bundle state after re-execution should match the original one. - // - // NOTE: This should not be needed if `Reverts` had a comparison method that sorted first, - // or otherwise did not care about order. - // - // See: https://github.com/bluealloy/revm/issues/1813 - let mut output = output.clone(); - for reverts in output.state.reverts.iter_mut() { - reverts.sort_by(|left, right| left.0.cmp(&right.0)); - } - - // We also have to sort the `bundle_state` reverts - for reverts in bundle_state.reverts.iter_mut() { - reverts.sort_by(|left, right| left.0.cmp(&right.0)); - } - - if bundle_state != output.state { - let original_path = self.save_file( - format!("{}_{}.bundle_state.original.json", block.number(), block.hash()), - &output.state, - )?; - let re_executed_path = self.save_file( - format!("{}_{}.bundle_state.re_executed.json", block.number(), block.hash()), - &bundle_state, - )?; - - let filename = format!("{}_{}.bundle_state.diff", block.number(), block.hash()); - // Convert bundle state to sorted struct which has BTreeMap instead of HashMap to - // have deterministric ordering - let bundle_state_sorted = BundleStateSorted::from_bundle_state(&bundle_state); - let output_state_sorted = BundleStateSorted::from_bundle_state(&output.state); + /// Validates that the bundle state after re-execution matches the original + fn validate_bundle_state( + &self, + re_executed_state: &BundleState, + original_state: &BundleState, + block_prefix: &str, + ) -> eyre::Result<()> { + if re_executed_state != original_state { + let original_filename = format!("{}.bundle_state.original.json", block_prefix); + let original_path = self.save_file(original_filename, original_state)?; + let re_executed_filename = format!("{}.bundle_state.re_executed.json", block_prefix); + let re_executed_path = self.save_file(re_executed_filename, re_executed_state)?; + // Convert bundle state to sorted format for deterministic comparison + let bundle_state_sorted = sort_bundle_state_for_comparison(re_executed_state); + let output_state_sorted = sort_bundle_state_for_comparison(original_state); + let filename = format!("{}.bundle_state.diff", block_prefix); let diff_path = self.save_diff(filename, &bundle_state_sorted, &output_state_sorted)?; warn!( @@ -295,35 +288,44 @@ where "Bundle state mismatch after re-execution" ); } + Ok(()) + } - // Calculate the state root and trie updates after re-execution. They should match - // the original ones. + /// Validates state root and trie updates after re-execution + fn validate_state_root_and_trie( + &self, + parent_header: &SealedHeader, + block: &RecoveredBlock, + bundle_state: &BundleState, + trie_updates: Option<(&TrieUpdates, B256)>, + block_prefix: &str, + ) -> eyre::Result<()> { + let state_provider = self.provider.state_by_block_hash(parent_header.hash())?; + let hashed_state = state_provider.hashed_post_state(bundle_state); let (re_executed_root, trie_output) = state_provider.state_root_with_updates(hashed_state)?; + if let Some((original_updates, original_root)) = trie_updates { if re_executed_root != original_root { - let filename = format!("{}_{}.state_root.diff", block.number(), block.hash()); + let filename = format!("{}.state_root.diff", block_prefix); let diff_path = self.save_diff(filename, &re_executed_root, &original_root)?; warn!(target: "engine::invalid_block_hooks::witness", ?original_root, ?re_executed_root, diff_path = %diff_path.display(), "State root mismatch after re-execution"); } - // If the re-executed state root does not match the _header_ state root, also log that. if re_executed_root != block.state_root() { - let filename = - format!("{}_{}.header_state_root.diff", block.number(), block.hash()); + let filename = format!("{}.header_state_root.diff", block_prefix); let diff_path = self.save_diff(filename, &re_executed_root, &block.state_root())?; warn!(target: "engine::invalid_block_hooks::witness", header_state_root=?block.state_root(), ?re_executed_root, diff_path = %diff_path.display(), "Re-executed state root does not match block state root"); } if &trie_output != original_updates { - // Trie updates are too big to diff, so we just save the original and re-executed let original_path = self.save_file( - format!("{}_{}.trie_updates.original.json", block.number(), block.hash()), - original_updates, + format!("{}.trie_updates.original.json", block_prefix), + &original_updates.into_sorted_ref(), )?; let re_executed_path = self.save_file( - format!("{}_{}.trie_updates.re_executed.json", block.number(), block.hash()), - &trie_output, + format!("{}.trie_updates.re_executed.json", block_prefix), + &trie_output.into_sorted_ref(), )?; warn!( target: "engine::invalid_block_hooks::witness", @@ -333,11 +335,44 @@ where ); } } + Ok(()) + } + + fn on_invalid_block( + &self, + parent_header: &SealedHeader, + block: &RecoveredBlock, + output: &BlockExecutionOutput, + trie_updates: Option<(&TrieUpdates, B256)>, + ) -> eyre::Result<()> { + // TODO(alexey): unify with `DebugApi::debug_execution_witness` + let (witness, bundle_state) = self.re_execute_block(parent_header, block)?; + + let block_prefix = format!("{}_{}", block.number(), block.hash()); + self.handle_witness_operations(&witness, &block_prefix, block.number())?; + + self.validate_bundle_state(&bundle_state, &output.state, &block_prefix)?; + + self.validate_state_root_and_trie( + parent_header, + block, + &bundle_state, + trie_updates, + &block_prefix, + )?; Ok(()) } - /// Saves the diff of two values into a file with the given name in the output directory. + /// Serializes and saves a value to a JSON file in the output directory + fn save_file(&self, filename: String, value: &T) -> eyre::Result { + let path = self.output_directory.join(filename); + File::create(&path)?.write_all(serde_json::to_string(value)?.as_bytes())?; + + Ok(path) + } + + /// Compares two values and saves their diff to a file in the output directory fn save_diff( &self, filename: String, @@ -350,22 +385,11 @@ where Ok(path) } - - fn save_file(&self, filename: String, value: &T) -> eyre::Result { - let path = self.output_directory.join(filename); - File::create(&path)?.write_all(serde_json::to_string(value)?.as_bytes())?; - - Ok(path) - } } impl InvalidBlockHook for InvalidBlockWitnessHook where - P: StateProviderFactory - + ChainSpecProvider - + Send - + Sync - + 'static, + P: StateProviderFactory + Send + Sync + 'static, E: ConfigureEvm + 'static, { fn on_invalid_block( @@ -380,3 +404,655 @@ where } } } + +#[cfg(test)] +mod tests { + use super::*; + use alloy_eips::eip7685::Requests; + use alloy_primitives::{map::HashMap, Address, Bytes, B256, U256}; + use reth_chainspec::ChainSpec; + use reth_ethereum_primitives::EthPrimitives; + use reth_evm_ethereum::EthEvmConfig; + use reth_provider::test_utils::MockEthProvider; + use reth_revm::db::{BundleAccount, BundleState}; + use revm_database::states::reverts::AccountRevert; + use tempfile::TempDir; + + use reth_revm::test_utils::StateProviderTest; + use reth_testing_utils::generators::{self, random_block, random_eoa_accounts, BlockParams}; + use revm_bytecode::Bytecode; + + /// Creates a test `BundleState` with realistic accounts, contracts, and reverts + fn create_bundle_state() -> BundleState { + let mut rng = generators::rng(); + let mut bundle_state = BundleState::default(); + + // Generate realistic EOA accounts using generators + let accounts = random_eoa_accounts(&mut rng, 3); + + for (i, (addr, account)) in accounts.into_iter().enumerate() { + // Create storage entries for each account + let mut storage = HashMap::default(); + let storage_key = U256::from(i + 1); + storage.insert( + storage_key, + StorageSlot { + present_value: U256::from((i + 1) * 10), + previous_or_original_value: U256::from((i + 1) * 15), + }, + ); + + let bundle_account = BundleAccount { + info: Some(AccountInfo { + balance: account.balance, + nonce: account.nonce, + code_hash: account.bytecode_hash.unwrap_or_default(), + code: None, + }), + original_info: (i == 0).then(|| AccountInfo { + balance: account.balance.checked_div(U256::from(2)).unwrap_or(U256::ZERO), + nonce: 0, + code_hash: account.bytecode_hash.unwrap_or_default(), + code: None, + }), + storage, + status: AccountStatus::default(), + }; + + bundle_state.state.insert(addr, bundle_account); + } + + // Generate realistic contract bytecode using generators + let contract_hashes: Vec = (0..3).map(|_| B256::random()).collect(); + for (i, hash) in contract_hashes.iter().enumerate() { + let bytecode = match i { + 0 => Bytes::from(vec![0x60, 0x80, 0x60, 0x40, 0x52]), // Simple contract + 1 => Bytes::from(vec![0x61, 0x81, 0x60, 0x00, 0x39]), // Another contract + _ => Bytes::from(vec![0x60, 0x00, 0x60, 0x00, 0xfd]), // REVERT contract + }; + bundle_state.contracts.insert(*hash, Bytecode::new_raw(bytecode)); + } + + // Add reverts for multiple blocks using different accounts + let addresses: Vec

= bundle_state.state.keys().copied().collect(); + for (i, addr) in addresses.iter().take(2).enumerate() { + let revert = AccountRevert { + wipe_storage: i == 0, // First account has storage wiped + ..AccountRevert::default() + }; + bundle_state.reverts.push(vec![(*addr, revert)]); + } + + // Set realistic sizes + bundle_state.state_size = bundle_state.state.len(); + bundle_state.reverts_size = bundle_state.reverts.len(); + + bundle_state + } + #[test] + fn test_sort_bundle_state_for_comparison() { + // Use the fixture function to create test data + let bundle_state = create_bundle_state(); + + // Call the function under test + let sorted = sort_bundle_state_for_comparison(&bundle_state); + + // Verify state_size and reverts_size values match the fixture + assert_eq!(sorted.state_size, 3); + assert_eq!(sorted.reverts_size, 2); + + // Verify state contains our mock accounts + assert_eq!(sorted.state.len(), 3); // We added 3 accounts + + // Verify contracts contains our mock contracts + assert_eq!(sorted.contracts.len(), 3); // We added 3 contracts + + // Verify reverts is an array with multiple blocks of reverts + let reverts = &sorted.reverts; + assert_eq!(reverts.len(), 2); // Fixture has two blocks of reverts + + // Verify that the state accounts have the expected structure + for account_data in sorted.state.values() { + // BundleAccountSorted has info, original_info, storage, and status fields + // Just verify the structure exists by accessing the fields + let _info = &account_data.info; + let _original_info = &account_data.original_info; + let _storage = &account_data.storage; + let _status = &account_data.status; + } + } + + #[test] + fn test_data_collector_collect() { + // Create test data using the fixture function + let bundle_state = create_bundle_state(); + + // Create a State with StateProviderTest + let state_provider = StateProviderTest::default(); + let mut state = State::builder() + .with_database(StateProviderDatabase::new( + Box::new(state_provider) as Box + )) + .with_bundle_update() + .build(); + + // Insert contracts from the fixture into the state cache + for (code_hash, bytecode) in &bundle_state.contracts { + state.cache.contracts.insert(*code_hash, bytecode.clone()); + } + + // Manually set the bundle state in the state object + state.bundle_state = bundle_state; + + // Call the collect function + let result = collect_execution_data(state); + // Verify the function returns successfully + assert!(result.is_ok()); + + let (codes, _preimages, _hashed_state, returned_bundle_state) = result.unwrap(); + + // Verify that the returned data contains expected values + // Since we used the fixture data, we should have some codes and state + assert!(!codes.is_empty(), "Expected some bytecode entries"); + assert!(!returned_bundle_state.state.is_empty(), "Expected some state entries"); + + // Verify the bundle state structure matches our fixture + assert_eq!(returned_bundle_state.state.len(), 3, "Expected 3 accounts from fixture"); + assert_eq!(returned_bundle_state.contracts.len(), 3, "Expected 3 contracts from fixture"); + } + + #[test] + fn test_re_execute_block() { + // Create hook instance + let (hook, _output_directory, _temp_dir) = create_test_hook(); + + // Setup to call re_execute_block + let mut rng = generators::rng(); + let parent_header = generators::random_header(&mut rng, 1, None); + + // Create a random block that inherits from the parent header + let recovered_block = random_block( + &mut rng, + 2, // block number + BlockParams { + parent: Some(parent_header.hash()), + tx_count: Some(0), + ..Default::default() + }, + ) + .try_recover() + .unwrap(); + + let result = hook.re_execute_block(&parent_header, &recovered_block); + + // Verify the function behavior with mock data + assert!(result.is_ok(), "re_execute_block should return Ok"); + } + + /// Creates test `InvalidBlockWitnessHook` with temporary directory + fn create_test_hook() -> ( + InvalidBlockWitnessHook, EthEvmConfig>, + PathBuf, + TempDir, + ) { + let temp_dir = TempDir::new().expect("Failed to create temp dir"); + let output_directory = temp_dir.path().to_path_buf(); + + let provider = MockEthProvider::::default(); + let evm_config = EthEvmConfig::mainnet(); + + let hook = + InvalidBlockWitnessHook::new(provider, evm_config, output_directory.clone(), None); + + (hook, output_directory, temp_dir) + } + + #[test] + fn test_handle_witness_operations_with_healthy_client_mock() { + // Create hook instance with mock healthy client + let (hook, output_directory, _temp_dir) = create_test_hook(); + + // Create sample ExecutionWitness with correct types + let witness = ExecutionWitness { + state: vec![Bytes::from("state_data")], + codes: vec![Bytes::from("code_data")], + keys: vec![Bytes::from("key_data")], + ..Default::default() + }; + + // Call handle_witness_operations + let result = hook.handle_witness_operations(&witness, "test_block_healthy", 67890); + + // Should succeed + assert!(result.is_ok()); + + // Check that witness file was created + let witness_file = output_directory.join("test_block_healthy.witness.re_executed.json"); + assert!(witness_file.exists()); + } + + #[test] + fn test_handle_witness_operations_file_creation() { + // Test file creation and content validation + let (hook, output_directory, _temp_dir) = create_test_hook(); + + let witness = ExecutionWitness { + state: vec![Bytes::from("test_state")], + codes: vec![Bytes::from("test_code")], + keys: vec![Bytes::from("test_key")], + ..Default::default() + }; + + let block_prefix = "file_test_block"; + let block_number = 11111; + + // Call handle_witness_operations + let result = hook.handle_witness_operations(&witness, block_prefix, block_number); + assert!(result.is_ok()); + + // Verify file was created with correct name + let expected_file = + output_directory.join(format!("{}.witness.re_executed.json", block_prefix)); + assert!(expected_file.exists()); + + // Read and verify file content is valid JSON and contains witness structure + let file_content = std::fs::read_to_string(&expected_file).expect("Failed to read file"); + let parsed_witness: serde_json::Value = + serde_json::from_str(&file_content).expect("File should contain valid JSON"); + + // Verify the JSON structure contains expected fields + assert!(parsed_witness.get("state").is_some(), "JSON should contain 'state' field"); + assert!(parsed_witness.get("codes").is_some(), "JSON should contain 'codes' field"); + assert!(parsed_witness.get("keys").is_some(), "JSON should contain 'keys' field"); + } + + #[test] + fn test_proof_generator_generate() { + // Use existing MockEthProvider + let mock_provider = MockEthProvider::default(); + let state_provider: Box = Box::new(mock_provider); + + // Mock Data + let mut codes = BTreeMap::new(); + codes.insert(B256::from([1u8; 32]), Bytes::from("contract_code_1")); + codes.insert(B256::from([2u8; 32]), Bytes::from("contract_code_2")); + + let mut preimages = BTreeMap::new(); + preimages.insert(B256::from([3u8; 32]), Bytes::from("preimage_1")); + preimages.insert(B256::from([4u8; 32]), Bytes::from("preimage_2")); + + let hashed_state = reth_trie::HashedPostState::default(); + + // Call generate function + let result = generate(codes.clone(), preimages.clone(), hashed_state, state_provider); + + // Verify result + assert!(result.is_ok(), "generate function should succeed"); + let execution_witness = result.unwrap(); + + assert!(execution_witness.state.is_empty(), "State should be empty from MockEthProvider"); + + let expected_codes: Vec = codes.into_values().collect(); + assert_eq!( + execution_witness.codes.len(), + expected_codes.len(), + "Codes length should match" + ); + for code in &expected_codes { + assert!( + execution_witness.codes.contains(code), + "Codes should contain expected bytecode" + ); + } + + let expected_keys: Vec = preimages.into_values().collect(); + assert_eq!(execution_witness.keys.len(), expected_keys.len(), "Keys length should match"); + for key in &expected_keys { + assert!(execution_witness.keys.contains(key), "Keys should contain expected preimage"); + } + } + + #[test] + fn test_validate_bundle_state_matching() { + let (hook, _output_dir, _temp_dir) = create_test_hook(); + let bundle_state = create_bundle_state(); + let block_prefix = "test_block_123"; + + // Test with identical states - should not produce any warnings or files + let result = hook.validate_bundle_state(&bundle_state, &bundle_state, block_prefix); + assert!(result.is_ok()); + } + + #[test] + fn test_validate_bundle_state_mismatch() { + let (hook, output_dir, _temp_dir) = create_test_hook(); + let original_state = create_bundle_state(); + let mut modified_state = create_bundle_state(); + + // Modify the state to create a mismatch + let addr = Address::from([1u8; 20]); + if let Some(account) = modified_state.state.get_mut(&addr) && + let Some(ref mut info) = account.info + { + info.balance = U256::from(999); + } + + let block_prefix = "test_block_mismatch"; + + // Test with different states - should save files and log warning + let result = hook.validate_bundle_state(&modified_state, &original_state, block_prefix); + assert!(result.is_ok()); + + // Verify that files were created + let original_file = output_dir.join(format!("{}.bundle_state.original.json", block_prefix)); + let re_executed_file = + output_dir.join(format!("{}.bundle_state.re_executed.json", block_prefix)); + let diff_file = output_dir.join(format!("{}.bundle_state.diff", block_prefix)); + + assert!(original_file.exists(), "Original bundle state file should be created"); + assert!(re_executed_file.exists(), "Re-executed bundle state file should be created"); + assert!(diff_file.exists(), "Diff file should be created"); + } + + /// Creates test `TrieUpdates` with account nodes and removed nodes + fn create_test_trie_updates() -> TrieUpdates { + use alloy_primitives::map::HashMap; + use reth_trie::{updates::TrieUpdates, BranchNodeCompact, Nibbles}; + use std::collections::HashSet; + + let mut account_nodes = HashMap::default(); + let nibbles = Nibbles::from_nibbles_unchecked([0x1, 0x2, 0x3]); + let branch_node = BranchNodeCompact::new( + 0b1010, // state_mask + 0b1010, // tree_mask - must be subset of state_mask + 0b1000, // hash_mask + vec![B256::from([1u8; 32])], // hashes + None, // root_hash + ); + account_nodes.insert(nibbles, branch_node); + + let mut removed_nodes = HashSet::default(); + removed_nodes.insert(Nibbles::from_nibbles_unchecked([0x4, 0x5, 0x6])); + + TrieUpdates { account_nodes, removed_nodes, storage_tries: HashMap::default() } + } + + #[test] + fn test_validate_state_root_and_trie_with_trie_updates() { + let (hook, _output_dir, _temp_dir) = create_test_hook(); + let bundle_state = create_bundle_state(); + + // Generate test data + let mut rng = generators::rng(); + let parent_header = generators::random_header(&mut rng, 1, None); + let recovered_block = random_block( + &mut rng, + 2, + BlockParams { + parent: Some(parent_header.hash()), + tx_count: Some(0), + ..Default::default() + }, + ) + .try_recover() + .unwrap(); + + let trie_updates = create_test_trie_updates(); + let original_root = B256::from([2u8; 32]); // Different from what will be computed + let block_prefix = "test_state_root_with_trie"; + + // Test with trie updates - this will likely produce warnings due to mock data + let result = hook.validate_state_root_and_trie( + &parent_header, + &recovered_block, + &bundle_state, + Some((&trie_updates, original_root)), + block_prefix, + ); + assert!(result.is_ok()); + } + + #[test] + fn test_on_invalid_block_calls_all_validation_methods() { + let (hook, output_dir, _temp_dir) = create_test_hook(); + let bundle_state = create_bundle_state(); + + // Generate test data + let mut rng = generators::rng(); + let parent_header = generators::random_header(&mut rng, 1, None); + let recovered_block = random_block( + &mut rng, + 2, + BlockParams { + parent: Some(parent_header.hash()), + tx_count: Some(0), + ..Default::default() + }, + ) + .try_recover() + .unwrap(); + + // Create mock BlockExecutionOutput + let output = BlockExecutionOutput { + state: bundle_state, + result: reth_provider::BlockExecutionResult { + receipts: vec![], + requests: Requests::default(), + gas_used: 0, + blob_gas_used: 0, + }, + }; + + // Create test trie updates + let trie_updates = create_test_trie_updates(); + let state_root = B256::random(); + + // Test that on_invalid_block attempts to call all its internal methods + // by checking that it doesn't panic and tries to create files + let files_before = output_dir.read_dir().unwrap().count(); + + let _result = hook.on_invalid_block( + &parent_header, + &recovered_block, + &output, + Some((&trie_updates, state_root)), + ); + + // Verify that the function attempted to process the block: + // Either it succeeded, or it created some output files during processing + let files_after = output_dir.read_dir().unwrap().count(); + + // The function should attempt to execute its workflow + assert!( + files_after >= files_before, + "on_invalid_block should attempt to create output files during processing" + ); + } + + #[test] + fn test_handle_witness_operations_with_empty_witness() { + let (hook, _output_dir, _temp_dir) = create_test_hook(); + let witness = ExecutionWitness::default(); + let block_prefix = "empty_witness_test"; + let block_number = 12345; + + let result = hook.handle_witness_operations(&witness, block_prefix, block_number); + assert!(result.is_ok()); + } + + #[test] + fn test_handle_witness_operations_with_zero_block_number() { + let (hook, _output_dir, _temp_dir) = create_test_hook(); + let witness = ExecutionWitness { + state: vec![Bytes::from("test_state")], + codes: vec![Bytes::from("test_code")], + keys: vec![Bytes::from("test_key")], + ..Default::default() + }; + let block_prefix = "zero_block_test"; + let block_number = 0; + + let result = hook.handle_witness_operations(&witness, block_prefix, block_number); + assert!(result.is_ok()); + } + + #[test] + fn test_handle_witness_operations_with_large_witness_data() { + let (hook, _output_dir, _temp_dir) = create_test_hook(); + let large_data = vec![0u8; 10000]; // 10KB of data + let witness = ExecutionWitness { + state: vec![Bytes::from(large_data.clone())], + codes: vec![Bytes::from(large_data.clone())], + keys: vec![Bytes::from(large_data)], + ..Default::default() + }; + let block_prefix = "large_witness_test"; + let block_number = 999999; + + let result = hook.handle_witness_operations(&witness, block_prefix, block_number); + assert!(result.is_ok()); + } + + #[test] + fn test_validate_bundle_state_with_empty_states() { + let (hook, _output_dir, _temp_dir) = create_test_hook(); + let empty_state = BundleState::default(); + let block_prefix = "empty_states_test"; + + let result = hook.validate_bundle_state(&empty_state, &empty_state, block_prefix); + assert!(result.is_ok()); + } + + #[test] + fn test_validate_bundle_state_with_different_contract_counts() { + let (hook, output_dir, _temp_dir) = create_test_hook(); + let state1 = create_bundle_state(); + let mut state2 = create_bundle_state(); + + // Add extra contract to state2 + let extra_contract_hash = B256::random(); + state2.contracts.insert( + extra_contract_hash, + Bytecode::new_raw(Bytes::from(vec![0x60, 0x00, 0x60, 0x00, 0xfd])), // REVERT opcode + ); + + let block_prefix = "different_contracts_test"; + let result = hook.validate_bundle_state(&state1, &state2, block_prefix); + assert!(result.is_ok()); + + // Verify diff files were created + let diff_file = output_dir.join(format!("{}.bundle_state.diff", block_prefix)); + assert!(diff_file.exists()); + } + + #[test] + fn test_save_diff_with_identical_values() { + let (hook, output_dir, _temp_dir) = create_test_hook(); + let value1 = "identical_value"; + let value2 = "identical_value"; + let filename = "identical_diff_test".to_string(); + + let result = hook.save_diff(filename.clone(), &value1, &value2); + assert!(result.is_ok()); + + let diff_file = output_dir.join(filename); + assert!(diff_file.exists()); + } + + #[test] + fn test_validate_state_root_and_trie_without_trie_updates() { + let (hook, _output_dir, _temp_dir) = create_test_hook(); + let bundle_state = create_bundle_state(); + + let mut rng = generators::rng(); + let parent_header = generators::random_header(&mut rng, 1, None); + let recovered_block = random_block( + &mut rng, + 2, + BlockParams { + parent: Some(parent_header.hash()), + tx_count: Some(0), + ..Default::default() + }, + ) + .try_recover() + .unwrap(); + + let block_prefix = "no_trie_updates_test"; + + // Test without trie updates (None case) + let result = hook.validate_state_root_and_trie( + &parent_header, + &recovered_block, + &bundle_state, + None, + block_prefix, + ); + assert!(result.is_ok()); + } + + #[test] + fn test_complete_invalid_block_workflow() { + let (hook, _output_dir, _temp_dir) = create_test_hook(); + let mut rng = generators::rng(); + + // Create a realistic block scenario + let parent_header = generators::random_header(&mut rng, 100, None); + let invalid_block = random_block( + &mut rng, + 101, + BlockParams { + parent: Some(parent_header.hash()), + tx_count: Some(3), + ..Default::default() + }, + ) + .try_recover() + .unwrap(); + + let bundle_state = create_bundle_state(); + let trie_updates = create_test_trie_updates(); + + // Test validation methods + let validation_result = + hook.validate_bundle_state(&bundle_state, &bundle_state, "integration_test"); + assert!(validation_result.is_ok(), "Bundle state validation should succeed"); + + let state_root_result = hook.validate_state_root_and_trie( + &parent_header, + &invalid_block, + &bundle_state, + Some((&trie_updates, B256::random())), + "integration_test", + ); + assert!(state_root_result.is_ok(), "State root validation should succeed"); + } + + #[test] + fn test_integration_workflow_components() { + let (hook, _output_dir, _temp_dir) = create_test_hook(); + let mut rng = generators::rng(); + + // Create test data + let parent_header = generators::random_header(&mut rng, 50, None); + let _invalid_block = random_block( + &mut rng, + 51, + BlockParams { + parent: Some(parent_header.hash()), + tx_count: Some(2), + ..Default::default() + }, + ) + .try_recover() + .unwrap(); + + let bundle_state = create_bundle_state(); + let _trie_updates = create_test_trie_updates(); + + // Test individual components that would be part of the complete flow + let validation_result = + hook.validate_bundle_state(&bundle_state, &bundle_state, "integration_component_test"); + assert!(validation_result.is_ok(), "Component validation should succeed"); + } +} diff --git a/crates/engine/local/Cargo.toml b/crates/engine/local/Cargo.toml index 5d0eb22baca..dd708dee905 100644 --- a/crates/engine/local/Cargo.toml +++ b/crates/engine/local/Cargo.toml @@ -11,19 +11,12 @@ exclude.workspace = true [dependencies] # reth reth-chainspec.workspace = true -reth-consensus.workspace = true -reth-engine-primitives.workspace = true -reth-engine-service.workspace = true -reth-engine-tree.workspace = true -reth-node-types.workspace = true -reth-evm.workspace = true +reth-engine-primitives = { workspace = true, features = ["std"] } reth-ethereum-engine-primitives.workspace = true reth-payload-builder.workspace = true reth-payload-primitives.workspace = true -reth-provider.workspace = true -reth-prune.workspace = true +reth-storage-api.workspace = true reth-transaction-pool.workspace = true -reth-stages-api.workspace = true # alloy alloy-consensus.workspace = true @@ -50,5 +43,4 @@ op = [ "dep:op-alloy-rpc-types-engine", "dep:reth-optimism-chainspec", "reth-payload-primitives/op", - "reth-evm/op", ] diff --git a/crates/engine/local/src/lib.rs b/crates/engine/local/src/lib.rs index 26c84d50c85..c80789cc248 100644 --- a/crates/engine/local/src/lib.rs +++ b/crates/engine/local/src/lib.rs @@ -6,12 +6,10 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] pub mod miner; pub mod payload; -pub mod service; -pub use miner::MiningMode; +pub use miner::{LocalMiner, MiningMode}; pub use payload::LocalPayloadAttributesBuilder; -pub use service::LocalEngineService; diff --git a/crates/engine/local/src/miner.rs b/crates/engine/local/src/miner.rs index 7b907d29492..d6298502fb5 100644 --- a/crates/engine/local/src/miner.rs +++ b/crates/engine/local/src/miner.rs @@ -5,41 +5,51 @@ use alloy_primitives::{TxHash, B256}; use alloy_rpc_types_engine::ForkchoiceState; use eyre::OptionExt; use futures_util::{stream::Fuse, StreamExt}; -use reth_engine_primitives::BeaconEngineMessage; +use reth_engine_primitives::ConsensusEngineHandle; use reth_payload_builder::PayloadBuilderHandle; use reth_payload_primitives::{ BuiltPayload, EngineApiMessageVersion, PayloadAttributesBuilder, PayloadKind, PayloadTypes, }; -use reth_provider::BlockReader; +use reth_storage_api::BlockReader; use reth_transaction_pool::TransactionPool; use std::{ + collections::VecDeque, future::Future, pin::Pin, task::{Context, Poll}, time::{Duration, UNIX_EPOCH}, }; -use tokio::{ - sync::{mpsc::UnboundedSender, oneshot}, - time::Interval, -}; +use tokio::time::Interval; use tokio_stream::wrappers::ReceiverStream; use tracing::error; /// A mining mode for the local dev engine. #[derive(Debug)] -pub enum MiningMode { +pub enum MiningMode { /// In this mode a block is built as soon as /// a valid transaction reaches the pool. - Instant(Fuse>), + /// If `max_transactions` is set, a block is built when that many transactions have + /// accumulated. + Instant { + /// The transaction pool. + pool: Pool, + /// Stream of transaction notifications. + rx: Fuse>, + /// Maximum number of transactions to accumulate before mining a block. + /// If None, mine immediately when any transaction arrives. + max_transactions: Option, + /// Counter for accumulated transactions (only used when `max_transactions` is set). + accumulated: usize, + }, /// In this mode a block is built at a fixed interval. Interval(Interval), } -impl MiningMode { +impl MiningMode { /// Constructor for a [`MiningMode::Instant`] - pub fn instant(pool: Pool) -> Self { + pub fn instant(pool: Pool, max_transactions: Option) -> Self { let rx = pool.pending_transactions_listener(); - Self::Instant(ReceiverStream::new(rx).fuse()) + Self::Instant { pool, rx: ReceiverStream::new(rx).fuse(), max_transactions, accumulated: 0 } } /// Constructor for a [`MiningMode::Interval`] @@ -49,16 +59,29 @@ impl MiningMode { } } -impl Future for MiningMode { +impl Future for MiningMode { type Output = (); fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { let this = self.get_mut(); match this { - Self::Instant(rx) => { - // drain all transactions notifications - if let Poll::Ready(Some(_)) = rx.poll_next_unpin(cx) { - return Poll::Ready(()) + Self::Instant { pool, rx, max_transactions, accumulated } => { + // Poll for new transaction notifications + while let Poll::Ready(Some(_)) = rx.poll_next_unpin(cx) { + if pool.pending_and_queued_txn_count().0 == 0 { + continue; + } + if let Some(max_tx) = max_transactions { + *accumulated += 1; + // If we've reached the max transactions threshold, mine a block + if *accumulated >= *max_tx { + *accumulated = 0; // Reset counter for next block + return Poll::Ready(()); + } + } else { + // If no max_transactions is set, mine immediately + return Poll::Ready(()); + } } Poll::Pending } @@ -72,54 +95,52 @@ impl Future for MiningMode { } } -/// Local miner advancing the chain/ +/// Local miner advancing the chain #[derive(Debug)] -pub struct LocalMiner { +pub struct LocalMiner { /// The payload attribute builder for the engine payload_attributes_builder: B, /// Sender for events to engine. - to_engine: UnboundedSender>, + to_engine: ConsensusEngineHandle, /// The mining mode for the engine - mode: MiningMode, + mode: MiningMode, /// The payload builder for the engine payload_builder: PayloadBuilderHandle, /// Timestamp for the next block. last_timestamp: u64, /// Stores latest mined blocks. - last_block_hashes: Vec, + last_block_hashes: VecDeque, } -impl LocalMiner +impl LocalMiner where T: PayloadTypes, B: PayloadAttributesBuilder<::PayloadAttributes>, + Pool: TransactionPool + Unpin, { /// Spawns a new [`LocalMiner`] with the given parameters. - pub fn spawn_new( + pub fn new( provider: impl BlockReader, payload_attributes_builder: B, - to_engine: UnboundedSender>, - mode: MiningMode, + to_engine: ConsensusEngineHandle, + mode: MiningMode, payload_builder: PayloadBuilderHandle, - ) { + ) -> Self { let latest_header = provider.sealed_header(provider.best_block_number().unwrap()).unwrap().unwrap(); - let miner = Self { + Self { payload_attributes_builder, to_engine, mode, payload_builder, last_timestamp: latest_header.timestamp(), - last_block_hashes: vec![latest_header.hash()], - }; - - // Spawn the miner - tokio::spawn(miner.run()); + last_block_hashes: VecDeque::from([latest_header.hash()]), + } } /// Runs the [`LocalMiner`] in a loop, polling the miner and building payloads. - async fn run(mut self) { + pub async fn run(mut self) { let mut fcu_interval = tokio::time::interval(Duration::from_secs(1)); loop { tokio::select! { @@ -142,7 +163,7 @@ where /// Returns current forkchoice state. fn forkchoice_state(&self) -> ForkchoiceState { ForkchoiceState { - head_block_hash: *self.last_block_hashes.last().expect("at least 1 block exists"), + head_block_hash: *self.last_block_hashes.back().expect("at least 1 block exists"), safe_block_hash: *self .last_block_hashes .get(self.last_block_hashes.len().saturating_sub(32)) @@ -156,17 +177,14 @@ where /// Sends a FCU to the engine. async fn update_forkchoice_state(&self) -> eyre::Result<()> { - let (tx, rx) = oneshot::channel(); - self.to_engine.send(BeaconEngineMessage::ForkchoiceUpdated { - state: self.forkchoice_state(), - payload_attrs: None, - tx, - version: EngineApiMessageVersion::default(), - })?; - - let res = rx.await??; - if !res.forkchoice_status().is_valid() { - eyre::bail!("Invalid fork choice update") + let state = self.forkchoice_state(); + let res = self + .to_engine + .fork_choice_updated(state, None, EngineApiMessageVersion::default()) + .await?; + + if !res.is_valid() { + eyre::bail!("Invalid fork choice update {state:?}: {res:?}") } Ok(()) @@ -176,23 +194,23 @@ where /// through newPayload. async fn advance(&mut self) -> eyre::Result<()> { let timestamp = std::cmp::max( - self.last_timestamp + 1, + self.last_timestamp.saturating_add(1), std::time::SystemTime::now() .duration_since(UNIX_EPOCH) .expect("cannot be earlier than UNIX_EPOCH") .as_secs(), ); - let (tx, rx) = oneshot::channel(); - self.to_engine.send(BeaconEngineMessage::ForkchoiceUpdated { - state: self.forkchoice_state(), - payload_attrs: Some(self.payload_attributes_builder.build(timestamp)), - tx, - version: EngineApiMessageVersion::default(), - })?; + let res = self + .to_engine + .fork_choice_updated( + self.forkchoice_state(), + Some(self.payload_attributes_builder.build(timestamp)), + EngineApiMessageVersion::default(), + ) + .await?; - let res = rx.await??.await?; - if !res.payload_status.is_valid() { + if !res.is_valid() { eyre::bail!("Invalid payload status") } @@ -206,22 +224,18 @@ where let block = payload.block(); - let (tx, rx) = oneshot::channel(); let payload = T::block_to_payload(payload.block().clone()); - self.to_engine.send(BeaconEngineMessage::NewPayload { payload, tx })?; - - let res = rx.await??; + let res = self.to_engine.new_payload(payload).await?; if !res.is_valid() { eyre::bail!("Invalid payload") } self.last_timestamp = timestamp; - self.last_block_hashes.push(block.hash()); + self.last_block_hashes.push_back(block.hash()); // ensure we keep at most 64 blocks if self.last_block_hashes.len() > 64 { - self.last_block_hashes = - self.last_block_hashes.split_off(self.last_block_hashes.len() - 64); + self.last_block_hashes.pop_front(); } Ok(()) diff --git a/crates/engine/local/src/payload.rs b/crates/engine/local/src/payload.rs index ea6c63083a9..34deaf3e10c 100644 --- a/crates/engine/local/src/payload.rs +++ b/crates/engine/local/src/payload.rs @@ -1,5 +1,5 @@ //! The implementation of the [`PayloadAttributesBuilder`] for the -//! [`LocalEngineService`](super::service::LocalEngineService). +//! [`LocalMiner`](super::LocalMiner). use alloy_primitives::{Address, B256}; use reth_chainspec::EthereumHardforks; @@ -11,7 +11,8 @@ use std::sync::Arc; #[derive(Debug)] #[non_exhaustive] pub struct LocalPayloadAttributesBuilder { - chain_spec: Arc, + /// The chainspec + pub chain_spec: Arc, } impl LocalPayloadAttributesBuilder { @@ -60,22 +61,7 @@ where no_tx_pool: None, gas_limit: None, eip_1559_params: None, + min_base_fee: None, } } } - -/// A temporary workaround to support local payload engine launcher for arbitrary payload -/// attributes. -// TODO(mattsse): This should be reworked so that LocalPayloadAttributesBuilder can be implemented -// for any -pub trait UnsupportedLocalAttributes: Send + Sync + 'static {} - -impl PayloadAttributesBuilder for LocalPayloadAttributesBuilder -where - ChainSpec: Send + Sync + 'static, - T: UnsupportedLocalAttributes, -{ - fn build(&self, _: u64) -> T { - panic!("Unsupported payload attributes") - } -} diff --git a/crates/engine/local/src/service.rs b/crates/engine/local/src/service.rs deleted file mode 100644 index 90fe47f94af..00000000000 --- a/crates/engine/local/src/service.rs +++ /dev/null @@ -1,163 +0,0 @@ -//! Provides a local dev service engine that can be used to run a dev chain. -//! -//! [`LocalEngineService`] polls the payload builder based on a mining mode -//! which can be set to `Instant` or `Interval`. The `Instant` mode will -//! constantly poll the payload builder and initiate block building -//! with a single transaction. The `Interval` mode will initiate block -//! building at a fixed interval. - -use core::fmt; -use std::{ - fmt::{Debug, Formatter}, - pin::Pin, - sync::Arc, - task::{Context, Poll}, -}; - -use crate::miner::{LocalMiner, MiningMode}; -use futures_util::{Stream, StreamExt}; -use reth_chainspec::EthChainSpec; -use reth_consensus::{ConsensusError, FullConsensus}; -use reth_engine_primitives::{BeaconConsensusEngineEvent, BeaconEngineMessage, EngineValidator}; -use reth_engine_service::service::EngineMessageStream; -use reth_engine_tree::{ - chain::{ChainEvent, HandlerEvent}, - engine::{ - EngineApiKind, EngineApiRequest, EngineApiRequestHandler, EngineRequestHandler, FromEngine, - RequestHandlerEvent, - }, - persistence::PersistenceHandle, - tree::{EngineApiTreeHandler, InvalidBlockHook, TreeConfig}, -}; -use reth_evm::ConfigureEvm; -use reth_node_types::BlockTy; -use reth_payload_builder::PayloadBuilderHandle; -use reth_payload_primitives::{PayloadAttributesBuilder, PayloadTypes}; -use reth_provider::{ - providers::{BlockchainProvider, ProviderNodeTypes}, - ChainSpecProvider, ProviderFactory, -}; -use reth_prune::PrunerWithFactory; -use reth_stages_api::MetricEventsSender; -use tokio::sync::mpsc::UnboundedSender; -use tracing::error; - -/// Provides a local dev service engine that can be used to drive the -/// chain forward. -/// -/// This service both produces and consumes [`BeaconEngineMessage`]s. This is done to allow -/// modifications of the stream -pub struct LocalEngineService -where - N: ProviderNodeTypes, -{ - /// Processes requests. - /// - /// This type is responsible for processing incoming requests. - handler: EngineApiRequestHandler, N::Primitives>, - /// Receiver for incoming requests (from the engine API endpoint) that need to be processed. - incoming_requests: EngineMessageStream, -} - -impl LocalEngineService -where - N: ProviderNodeTypes, -{ - /// Constructor for [`LocalEngineService`]. - #[expect(clippy::too_many_arguments)] - pub fn new( - consensus: Arc>, - provider: ProviderFactory, - blockchain_db: BlockchainProvider, - pruner: PrunerWithFactory>, - payload_builder: PayloadBuilderHandle, - payload_validator: V, - tree_config: TreeConfig, - invalid_block_hook: Box>, - sync_metrics_tx: MetricEventsSender, - to_engine: UnboundedSender>, - from_engine: EngineMessageStream, - mode: MiningMode, - payload_attributes_builder: B, - evm_config: C, - ) -> Self - where - B: PayloadAttributesBuilder<::PayloadAttributes>, - V: EngineValidator>, - C: ConfigureEvm + 'static, - { - let chain_spec = provider.chain_spec(); - let engine_kind = - if chain_spec.is_optimism() { EngineApiKind::OpStack } else { EngineApiKind::Ethereum }; - - let persistence_handle = - PersistenceHandle::::spawn_service(provider, pruner, sync_metrics_tx); - let canonical_in_memory_state = blockchain_db.canonical_in_memory_state(); - - let (to_tree_tx, from_tree) = EngineApiTreeHandler::::spawn_new( - blockchain_db.clone(), - consensus, - payload_validator, - persistence_handle, - payload_builder.clone(), - canonical_in_memory_state, - tree_config, - invalid_block_hook, - engine_kind, - evm_config, - ); - - let handler = EngineApiRequestHandler::new(to_tree_tx, from_tree); - - LocalMiner::spawn_new( - blockchain_db, - payload_attributes_builder, - to_engine, - mode, - payload_builder, - ); - - Self { handler, incoming_requests: from_engine } - } -} - -impl Stream for LocalEngineService -where - N: ProviderNodeTypes, -{ - type Item = ChainEvent>; - - fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - let this = self.get_mut(); - - if let Poll::Ready(ev) = this.handler.poll(cx) { - return match ev { - RequestHandlerEvent::HandlerEvent(ev) => match ev { - HandlerEvent::BackfillAction(_) => { - error!(target: "engine::local", "received backfill request in local engine"); - Poll::Ready(Some(ChainEvent::FatalError)) - } - HandlerEvent::Event(ev) => Poll::Ready(Some(ChainEvent::Handler(ev))), - HandlerEvent::FatalError => Poll::Ready(Some(ChainEvent::FatalError)), - }, - RequestHandlerEvent::Download(_) => { - error!(target: "engine::local", "received download request in local engine"); - Poll::Ready(Some(ChainEvent::FatalError)) - } - } - } - - // forward incoming requests to the handler - while let Poll::Ready(Some(req)) = this.incoming_requests.poll_next_unpin(cx) { - this.handler.on_event(FromEngine::Request(req.into())); - } - - Poll::Pending - } -} - -impl Debug for LocalEngineService { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - f.debug_struct("LocalEngineService").finish_non_exhaustive() - } -} diff --git a/crates/engine/primitives/Cargo.toml b/crates/engine/primitives/Cargo.toml index 43c0cf69dcd..795118083b5 100644 --- a/crates/engine/primitives/Cargo.toml +++ b/crates/engine/primitives/Cargo.toml @@ -12,13 +12,13 @@ workspace = true [dependencies] # reth +reth-evm.workspace = true reth-execution-types.workspace = true reth-payload-primitives.workspace = true reth-payload-builder-primitives.workspace = true reth-primitives-traits.workspace = true reth-ethereum-primitives.workspace = true reth-chain-state.workspace = true -reth-trie.workspace = true reth-errors.workspace = true reth-trie-common.workspace = true @@ -26,10 +26,11 @@ reth-trie-common.workspace = true alloy-primitives.workspace = true alloy-consensus.workspace = true alloy-rpc-types-engine.workspace = true +alloy-eips.workspace = true # async -tokio = { workspace = true, features = ["sync"] } -futures.workspace = true +tokio = { workspace = true, features = ["sync"], optional = true } +futures = { workspace = true, optional = true } # misc auto_impl.workspace = true @@ -46,7 +47,10 @@ std = [ "alloy-primitives/std", "alloy-consensus/std", "alloy-rpc-types-engine/std", + "alloy-eips/std", "futures/std", + "tokio", "serde/std", "thiserror/std", + "reth-evm/std", ] diff --git a/crates/engine/primitives/src/config.rs b/crates/engine/primitives/src/config.rs index 1c2fce0a49c..0b9b7d9f821 100644 --- a/crates/engine/primitives/src/config.rs +++ b/crates/engine/primitives/src/config.rs @@ -4,16 +4,42 @@ pub const DEFAULT_PERSISTENCE_THRESHOLD: u64 = 2; /// How close to the canonical head we persist blocks. -pub const DEFAULT_MEMORY_BLOCK_BUFFER_TARGET: u64 = 2; +pub const DEFAULT_MEMORY_BLOCK_BUFFER_TARGET: u64 = 0; -/// Default maximum concurrency for proof tasks -pub const DEFAULT_MAX_PROOF_TASK_CONCURRENCY: u64 = 256; +/// Minimum number of workers we allow configuring explicitly. +pub const MIN_WORKER_COUNT: usize = 32; + +/// Returns the default number of storage worker threads based on available parallelism. +fn default_storage_worker_count() -> usize { + #[cfg(feature = "std")] + { + std::thread::available_parallelism().map_or(8, |n| n.get() * 2).min(MIN_WORKER_COUNT) + } + #[cfg(not(feature = "std"))] + { + 8 + } +} + +/// Returns the default number of account worker threads. +/// +/// Account workers coordinate storage proof collection and account trie traversal. +/// They are set to the same count as storage workers for simplicity. +fn default_account_worker_count() -> usize { + default_storage_worker_count() +} + +/// The size of proof targets chunk to spawn in one multiproof calculation. +pub const DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE: usize = 10; /// Default number of reserved CPU cores for non-reth processes. /// -/// This will be deducated from the thread count of main reth global threadpool. +/// This will be deducted from the thread count of main reth global threadpool. pub const DEFAULT_RESERVED_CPU_CORES: usize = 1; +/// Default maximum concurrency for prewarm task. +pub const DEFAULT_PREWARM_MAX_CONCURRENCY: usize = 16; + const DEFAULT_BLOCK_BUFFER_LIMIT: u32 = 256; const DEFAULT_MAX_INVALID_HEADER_CACHE_LENGTH: u32 = 256; const DEFAULT_MAX_EXECUTE_BLOCK_BATCH_SIZE: usize = 4; @@ -37,7 +63,7 @@ pub fn has_enough_parallelism() -> bool { } /// The configuration of the engine tree. -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct TreeConfig { /// Maximum number of blocks to be kept only in memory without triggering /// persistence. @@ -63,18 +89,48 @@ pub struct TreeConfig { /// Whether to always compare trie updates from the state root task to the trie updates from /// the regular state root calculation. always_compare_trie_updates: bool, - /// Whether to disable cross-block caching and parallel prewarming. - disable_caching_and_prewarming: bool, + /// Whether to disable parallel prewarming. + disable_prewarming: bool, + /// Whether to disable the parallel sparse trie state root algorithm. + disable_parallel_sparse_trie: bool, /// Whether to enable state provider metrics. state_provider_metrics: bool, /// Cross-block cache size in bytes. cross_block_cache_size: u64, /// Whether the host has enough parallelism to run state root task. has_enough_parallelism: bool, - /// Maximum number of concurrent proof tasks - max_proof_task_concurrency: u64, + /// Whether multiproof task should chunk proof targets. + multiproof_chunking_enabled: bool, + /// Multiproof task chunk size for proof targets. + multiproof_chunk_size: usize, /// Number of reserved CPU cores for non-reth processes reserved_cpu_cores: usize, + /// Whether to disable the precompile cache + precompile_cache_disabled: bool, + /// Whether to use state root fallback for testing + state_root_fallback: bool, + /// Whether to always process payload attributes and begin a payload build process + /// even if `forkchoiceState.headBlockHash` is already the canonical head or an ancestor. + /// + /// The Engine API specification generally states that client software "MUST NOT begin a + /// payload build process if `forkchoiceState.headBlockHash` references a `VALID` + /// ancestor of the head of canonical chain". + /// See: (Rule 2) + /// + /// This flag allows overriding that behavior. + /// This is useful for specific chain configurations (e.g., OP Stack where proposers + /// can reorg their own chain), various custom chains, or for development/testing purposes + /// where immediate payload regeneration is desired despite the head not changing or moving to + /// an ancestor. + always_process_payload_attributes_on_canonical_head: bool, + /// Maximum concurrency for the prewarm task. + prewarm_max_concurrency: usize, + /// Whether to unwind canonical header to ancestor during forkchoice updates. + allow_unwind_canonical_header: bool, + /// Number of storage proof worker threads. + storage_worker_count: usize, + /// Number of account proof worker threads. + account_worker_count: usize, } impl Default for TreeConfig { @@ -87,12 +143,21 @@ impl Default for TreeConfig { max_execute_block_batch_size: DEFAULT_MAX_EXECUTE_BLOCK_BATCH_SIZE, legacy_state_root: false, always_compare_trie_updates: false, - disable_caching_and_prewarming: false, + disable_prewarming: false, + disable_parallel_sparse_trie: false, state_provider_metrics: false, cross_block_cache_size: DEFAULT_CROSS_BLOCK_CACHE_SIZE, has_enough_parallelism: has_enough_parallelism(), - max_proof_task_concurrency: DEFAULT_MAX_PROOF_TASK_CONCURRENCY, + multiproof_chunking_enabled: true, + multiproof_chunk_size: DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE, reserved_cpu_cores: DEFAULT_RESERVED_CPU_CORES, + precompile_cache_disabled: false, + state_root_fallback: false, + always_process_payload_attributes_on_canonical_head: false, + prewarm_max_concurrency: DEFAULT_PREWARM_MAX_CONCURRENCY, + allow_unwind_canonical_header: false, + storage_worker_count: default_storage_worker_count(), + account_worker_count: default_account_worker_count(), } } } @@ -108,12 +173,21 @@ impl TreeConfig { max_execute_block_batch_size: usize, legacy_state_root: bool, always_compare_trie_updates: bool, - disable_caching_and_prewarming: bool, + disable_prewarming: bool, + disable_parallel_sparse_trie: bool, state_provider_metrics: bool, cross_block_cache_size: u64, has_enough_parallelism: bool, - max_proof_task_concurrency: u64, + multiproof_chunking_enabled: bool, + multiproof_chunk_size: usize, reserved_cpu_cores: usize, + precompile_cache_disabled: bool, + state_root_fallback: bool, + always_process_payload_attributes_on_canonical_head: bool, + prewarm_max_concurrency: usize, + allow_unwind_canonical_header: bool, + storage_worker_count: usize, + account_worker_count: usize, ) -> Self { Self { persistence_threshold, @@ -123,12 +197,21 @@ impl TreeConfig { max_execute_block_batch_size, legacy_state_root, always_compare_trie_updates, - disable_caching_and_prewarming, + disable_prewarming, + disable_parallel_sparse_trie, state_provider_metrics, cross_block_cache_size, has_enough_parallelism, - max_proof_task_concurrency, + multiproof_chunking_enabled, + multiproof_chunk_size, reserved_cpu_cores, + precompile_cache_disabled, + state_root_fallback, + always_process_payload_attributes_on_canonical_head, + prewarm_max_concurrency, + allow_unwind_canonical_header, + storage_worker_count, + account_worker_count, } } @@ -157,9 +240,14 @@ impl TreeConfig { self.max_execute_block_batch_size } - /// Return the maximum proof task concurrency. - pub const fn max_proof_task_concurrency(&self) -> u64 { - self.max_proof_task_concurrency + /// Return whether the multiproof task chunking is enabled. + pub const fn multiproof_chunking_enabled(&self) -> bool { + self.multiproof_chunking_enabled + } + + /// Return the multiproof task chunk size. + pub const fn multiproof_chunk_size(&self) -> usize { + self.multiproof_chunk_size } /// Return the number of reserved CPU cores for non-reth processes @@ -178,9 +266,14 @@ impl TreeConfig { self.state_provider_metrics } - /// Returns whether or not cross-block caching and parallel prewarming should be used. - pub const fn disable_caching_and_prewarming(&self) -> bool { - self.disable_caching_and_prewarming + /// Returns whether or not the parallel sparse trie is disabled. + pub const fn disable_parallel_sparse_trie(&self) -> bool { + self.disable_parallel_sparse_trie + } + + /// Returns whether or not parallel prewarming should be used. + pub const fn disable_prewarming(&self) -> bool { + self.disable_prewarming } /// Returns whether to always compare trie updates from the state root task to the trie updates @@ -189,11 +282,42 @@ impl TreeConfig { self.always_compare_trie_updates } - /// Return the cross-block cache size. + /// Returns the cross-block cache size. pub const fn cross_block_cache_size(&self) -> u64 { self.cross_block_cache_size } + /// Returns whether precompile cache is disabled. + pub const fn precompile_cache_disabled(&self) -> bool { + self.precompile_cache_disabled + } + + /// Returns whether to use state root fallback. + pub const fn state_root_fallback(&self) -> bool { + self.state_root_fallback + } + + /// Sets whether to always process payload attributes when the FCU head is already canonical. + pub const fn with_always_process_payload_attributes_on_canonical_head( + mut self, + always_process_payload_attributes_on_canonical_head: bool, + ) -> Self { + self.always_process_payload_attributes_on_canonical_head = + always_process_payload_attributes_on_canonical_head; + self + } + + /// Returns true if payload attributes should always be processed even when the FCU head is + /// canonical. + pub const fn always_process_payload_attributes_on_canonical_head(&self) -> bool { + self.always_process_payload_attributes_on_canonical_head + } + + /// Returns true if canonical header should be unwound to ancestor during forkchoice updates. + pub const fn unwind_canonical_header(&self) -> bool { + self.allow_unwind_canonical_header + } + /// Setter for persistence threshold. pub const fn with_persistence_threshold(mut self, persistence_threshold: u64) -> Self { self.persistence_threshold = persistence_threshold; @@ -239,12 +363,9 @@ impl TreeConfig { self } - /// Setter for whether to disable cross-block caching and parallel prewarming. - pub const fn without_caching_and_prewarming( - mut self, - disable_caching_and_prewarming: bool, - ) -> Self { - self.disable_caching_and_prewarming = disable_caching_and_prewarming; + /// Setter for whether to disable parallel prewarming. + pub const fn without_prewarming(mut self, disable_prewarming: bool) -> Self { + self.disable_prewarming = disable_prewarming; self } @@ -276,12 +397,27 @@ impl TreeConfig { self } - /// Setter for maximum number of concurrent proof tasks. - pub const fn with_max_proof_task_concurrency( + /// Setter for whether to disable the parallel sparse trie + pub const fn with_disable_parallel_sparse_trie( mut self, - max_proof_task_concurrency: u64, + disable_parallel_sparse_trie: bool, ) -> Self { - self.max_proof_task_concurrency = max_proof_task_concurrency; + self.disable_parallel_sparse_trie = disable_parallel_sparse_trie; + self + } + + /// Setter for whether multiproof task should chunk proof targets. + pub const fn with_multiproof_chunking_enabled( + mut self, + multiproof_chunking_enabled: bool, + ) -> Self { + self.multiproof_chunking_enabled = multiproof_chunking_enabled; + self + } + + /// Setter for multiproof task chunk size for proof targets. + pub const fn with_multiproof_chunk_size(mut self, multiproof_chunk_size: usize) -> Self { + self.multiproof_chunk_size = multiproof_chunk_size; self } @@ -291,8 +427,59 @@ impl TreeConfig { self } + /// Setter for whether to disable the precompile cache. + pub const fn without_precompile_cache(mut self, precompile_cache_disabled: bool) -> Self { + self.precompile_cache_disabled = precompile_cache_disabled; + self + } + + /// Setter for whether to use state root fallback, useful for testing. + pub const fn with_state_root_fallback(mut self, state_root_fallback: bool) -> Self { + self.state_root_fallback = state_root_fallback; + self + } + + /// Setter for whether to unwind canonical header to ancestor during forkchoice updates. + pub const fn with_unwind_canonical_header(mut self, unwind_canonical_header: bool) -> Self { + self.allow_unwind_canonical_header = unwind_canonical_header; + self + } + /// Whether or not to use state root task pub const fn use_state_root_task(&self) -> bool { self.has_enough_parallelism && !self.legacy_state_root } + + /// Setter for prewarm max concurrency. + pub const fn with_prewarm_max_concurrency(mut self, prewarm_max_concurrency: usize) -> Self { + self.prewarm_max_concurrency = prewarm_max_concurrency; + self + } + + /// Return the prewarm max concurrency. + pub const fn prewarm_max_concurrency(&self) -> usize { + self.prewarm_max_concurrency + } + + /// Return the number of storage proof worker threads. + pub const fn storage_worker_count(&self) -> usize { + self.storage_worker_count + } + + /// Setter for the number of storage proof worker threads. + pub fn with_storage_worker_count(mut self, storage_worker_count: usize) -> Self { + self.storage_worker_count = storage_worker_count.max(MIN_WORKER_COUNT); + self + } + + /// Return the number of account proof worker threads. + pub const fn account_worker_count(&self) -> usize { + self.account_worker_count + } + + /// Setter for the number of account proof worker threads. + pub fn with_account_worker_count(mut self, account_worker_count: usize) -> Self { + self.account_worker_count = account_worker_count.max(MIN_WORKER_COUNT); + self + } } diff --git a/crates/engine/primitives/src/event.rs b/crates/engine/primitives/src/event.rs index d8165bed1c8..8cced031524 100644 --- a/crates/engine/primitives/src/event.rs +++ b/crates/engine/primitives/src/event.rs @@ -3,25 +3,32 @@ use crate::ForkchoiceStatus; use alloc::boxed::Box; use alloy_consensus::BlockHeader; +use alloy_eips::BlockNumHash; use alloy_primitives::B256; use alloy_rpc_types_engine::ForkchoiceState; use core::{ fmt::{Display, Formatter, Result}, time::Duration, }; -use reth_chain_state::ExecutedBlockWithTrieUpdates; +use reth_chain_state::ExecutedBlock; use reth_ethereum_primitives::EthPrimitives; use reth_primitives_traits::{NodePrimitives, SealedBlock, SealedHeader}; +/// Type alias for backwards compat +#[deprecated(note = "Use ConsensusEngineEvent instead")] +pub type BeaconConsensusEngineEvent = ConsensusEngineEvent; + /// Events emitted by the consensus engine. #[derive(Clone, Debug)] -pub enum BeaconConsensusEngineEvent { +pub enum ConsensusEngineEvent { /// The fork choice state was updated, and the current fork choice status ForkchoiceUpdated(ForkchoiceState, ForkchoiceStatus), /// A block was added to the fork chain. - ForkBlockAdded(ExecutedBlockWithTrieUpdates, Duration), + ForkBlockAdded(ExecutedBlock, Duration), + /// A new block was received from the consensus engine + BlockReceived(BlockNumHash), /// A block was added to the canonical chain, and the elapsed time validating the block - CanonicalBlockAdded(ExecutedBlockWithTrieUpdates, Duration), + CanonicalBlockAdded(ExecutedBlock, Duration), /// A canonical chain was committed, and the elapsed time committing the data CanonicalChainCommitted(Box>, Duration), /// The consensus engine processed an invalid block. @@ -30,9 +37,9 @@ pub enum BeaconConsensusEngineEvent { LiveSyncProgress(ConsensusEngineLiveSyncProgress), } -impl BeaconConsensusEngineEvent { +impl ConsensusEngineEvent { /// Returns the canonical header if the event is a - /// [`BeaconConsensusEngineEvent::CanonicalChainCommitted`]. + /// [`ConsensusEngineEvent::CanonicalChainCommitted`]. pub const fn canonical_header(&self) -> Option<&SealedHeader> { match self { Self::CanonicalChainCommitted(header, _) => Some(header), @@ -41,7 +48,7 @@ impl BeaconConsensusEngineEvent { } } -impl Display for BeaconConsensusEngineEvent +impl Display for ConsensusEngineEvent where N: NodePrimitives, { @@ -69,6 +76,9 @@ where Self::LiveSyncProgress(progress) => { write!(f, "LiveSyncProgress({progress:?})") } + Self::BlockReceived(num_hash) => { + write!(f, "BlockReceived({num_hash:?})") + } } } } @@ -80,7 +90,7 @@ pub enum ConsensusEngineLiveSyncProgress { DownloadingBlocks { /// The number of blocks remaining to download. remaining_blocks: u64, - /// The target block hash and number to download. + /// The target block hash to download. target: B256, }, } diff --git a/crates/engine/primitives/src/forkchoice.rs b/crates/engine/primitives/src/forkchoice.rs index 2fe47d807c5..69cb5990711 100644 --- a/crates/engine/primitives/src/forkchoice.rs +++ b/crates/engine/primitives/src/forkchoice.rs @@ -56,7 +56,7 @@ impl ForkchoiceStateTracker { self.latest_status().is_some_and(|s| s.is_syncing()) } - /// Returns whether the latest received FCU is syncing: [`ForkchoiceStatus::Invalid`] + /// Returns whether the latest received FCU is invalid: [`ForkchoiceStatus::Invalid`] pub fn is_latest_invalid(&self) -> bool { self.latest_status().is_some_and(|s| s.is_invalid()) } diff --git a/crates/engine/primitives/src/invalid_block_hook.rs b/crates/engine/primitives/src/invalid_block_hook.rs index 767fc83304f..c981f34ed65 100644 --- a/crates/engine/primitives/src/invalid_block_hook.rs +++ b/crates/engine/primitives/src/invalid_block_hook.rs @@ -1,7 +1,8 @@ +use alloc::{boxed::Box, fmt, vec::Vec}; use alloy_primitives::B256; use reth_execution_types::BlockExecutionOutput; use reth_primitives_traits::{NodePrimitives, RecoveredBlock, SealedHeader}; -use reth_trie::updates::TrieUpdates; +use reth_trie_common::updates::TrieUpdates; /// An invalid block hook. pub trait InvalidBlockHook: Send + Sync { @@ -36,3 +37,42 @@ where self(parent_header, block, output, trie_updates) } } + +/// A no-op [`InvalidBlockHook`] that does nothing. +#[derive(Debug, Default)] +#[non_exhaustive] +pub struct NoopInvalidBlockHook; + +impl InvalidBlockHook for NoopInvalidBlockHook { + fn on_invalid_block( + &self, + _parent_header: &SealedHeader, + _block: &RecoveredBlock, + _output: &BlockExecutionOutput, + _trie_updates: Option<(&TrieUpdates, B256)>, + ) { + } +} + +/// Multiple [`InvalidBlockHook`]s that are executed in order. +pub struct InvalidBlockHooks(pub Vec>>); + +impl fmt::Debug for InvalidBlockHooks { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("InvalidBlockHooks").field("len", &self.0.len()).finish() + } +} + +impl InvalidBlockHook for InvalidBlockHooks { + fn on_invalid_block( + &self, + parent_header: &SealedHeader, + block: &RecoveredBlock, + output: &BlockExecutionOutput, + trie_updates: Option<(&TrieUpdates, B256)>, + ) { + for hook in &self.0 { + hook.on_invalid_block(parent_header, block, output, trie_updates); + } + } +} diff --git a/crates/engine/primitives/src/lib.rs b/crates/engine/primitives/src/lib.rs index a30d41ae382..196a3baa18d 100644 --- a/crates/engine/primitives/src/lib.rs +++ b/crates/engine/primitives/src/lib.rs @@ -6,7 +6,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(feature = "std"), no_std)] extern crate alloc; @@ -22,6 +22,7 @@ use reth_trie_common::HashedPostState; use serde::{de::DeserializeOwned, Serialize}; // Re-export [`ExecutionPayload`] moved to `reth_payload_primitives` +pub use reth_evm::{ConfigureEngineEvm, ExecutableTxIterator}; pub use reth_payload_primitives::ExecutionPayload; mod error; @@ -30,14 +31,16 @@ pub use error::*; mod forkchoice; pub use forkchoice::{ForkchoiceStateHash, ForkchoiceStateTracker, ForkchoiceStatus}; +#[cfg(feature = "std")] mod message; +#[cfg(feature = "std")] pub use message::*; mod event; pub use event::*; mod invalid_block_hook; -pub use invalid_block_hook::InvalidBlockHook; +pub use invalid_block_hook::{InvalidBlockHook, InvalidBlockHooks, NoopInvalidBlockHook}; pub mod config; pub use config::*; @@ -57,7 +60,8 @@ pub trait EngineTypes: BuiltPayload: TryInto + TryInto + TryInto - + TryInto, + + TryInto + + TryInto, > + DeserializeOwned + Serialize { @@ -93,17 +97,40 @@ pub trait EngineTypes: + Send + Sync + 'static; + /// Execution Payload V5 envelope type. + type ExecutionPayloadEnvelopeV5: DeserializeOwned + + Serialize + + Clone + + Unpin + + Send + + Sync + + 'static; +} + +/// Type that validates the payloads processed by the engine API. +pub trait EngineApiValidator: Send + Sync + Unpin + 'static { + /// Validates the presence or exclusion of fork-specific fields based on the payload attributes + /// and the message version. + fn validate_version_specific_fields( + &self, + version: EngineApiMessageVersion, + payload_or_attrs: PayloadOrAttributes<'_, Types::ExecutionData, Types::PayloadAttributes>, + ) -> Result<(), EngineObjectValidationError>; + + /// Ensures that the payload attributes are valid for the given [`EngineApiMessageVersion`]. + fn ensure_well_formed_attributes( + &self, + version: EngineApiMessageVersion, + attributes: &Types::PayloadAttributes, + ) -> Result<(), EngineObjectValidationError>; } /// Type that validates an [`ExecutionPayload`]. #[auto_impl::auto_impl(&, Arc)] -pub trait PayloadValidator: Send + Sync + Unpin + 'static { +pub trait PayloadValidator: Send + Sync + Unpin + 'static { /// The block type used by the engine. type Block: Block; - /// The execution payload type used by the engine. - type ExecutionData; - /// Ensures that the given payload does not violate any consensus rules that concern the block's /// layout. /// @@ -114,7 +141,7 @@ pub trait PayloadValidator: Send + Sync + Unpin + 'static { /// engine-API specification. fn ensure_well_formed_payload( &self, - payload: Self::ExecutionData, + payload: Types::ExecutionData, ) -> Result, NewPayloadError>; /// Verifies payload post-execution w.r.t. hashed state updates. @@ -126,30 +153,6 @@ pub trait PayloadValidator: Send + Sync + Unpin + 'static { // method not used by l1 Ok(()) } -} - -/// Type that validates the payloads processed by the engine. -pub trait EngineValidator: - PayloadValidator -{ - /// Validates the presence or exclusion of fork-specific fields based on the payload attributes - /// and the message version. - fn validate_version_specific_fields( - &self, - version: EngineApiMessageVersion, - payload_or_attrs: PayloadOrAttributes< - '_, - Types::ExecutionData, - ::PayloadAttributes, - >, - ) -> Result<(), EngineObjectValidationError>; - - /// Ensures that the payload attributes are valid for the given [`EngineApiMessageVersion`]. - fn ensure_well_formed_attributes( - &self, - version: EngineApiMessageVersion, - attributes: &::PayloadAttributes, - ) -> Result<(), EngineObjectValidationError>; /// Validates the payload attributes with respect to the header. /// @@ -159,10 +162,10 @@ pub trait EngineValidator: /// > timestamp /// > of a block referenced by forkchoiceState.headBlockHash. /// - /// See also [engine api spec](https://github.com/ethereum/execution-apis/tree/fe8e13c288c592ec154ce25c534e26cb7ce0530d/src/engine) + /// See also: fn validate_payload_attributes_against_header( &self, - attr: &::PayloadAttributes, + attr: &Types::PayloadAttributes, header: &::Header, ) -> Result<(), InvalidPayloadAttributesError> { if attr.timestamp() <= header.timestamp() { diff --git a/crates/engine/primitives/src/message.rs b/crates/engine/primitives/src/message.rs index 283f6a4135b..5e7d97c8c05 100644 --- a/crates/engine/primitives/src/message.rs +++ b/crates/engine/primitives/src/message.rs @@ -1,6 +1,5 @@ use crate::{ - error::BeaconForkChoiceUpdateError, BeaconOnNewPayloadError, EngineApiMessageVersion, - ExecutionPayload, ForkchoiceStatus, + error::BeaconForkChoiceUpdateError, BeaconOnNewPayloadError, ExecutionPayload, ForkchoiceStatus, }; use alloy_rpc_types_engine::{ ForkChoiceUpdateResult, ForkchoiceState, ForkchoiceUpdateError, ForkchoiceUpdated, PayloadId, @@ -15,9 +14,13 @@ use core::{ use futures::{future::Either, FutureExt, TryFutureExt}; use reth_errors::RethResult; use reth_payload_builder_primitives::PayloadBuilderError; -use reth_payload_primitives::PayloadTypes; +use reth_payload_primitives::{EngineApiMessageVersion, PayloadTypes}; use tokio::sync::{mpsc::UnboundedSender, oneshot}; +/// Type alias for backwards compat +#[deprecated(note = "Use ConsensusEngineHandle instead")] +pub type BeaconConsensusEngineHandle = ConsensusEngineHandle; + /// Represents the outcome of forkchoice update. /// /// This is a future that resolves to [`ForkChoiceUpdateResult`] @@ -192,14 +195,14 @@ impl Display for BeaconEngineMessage { /// /// This type mirrors consensus related functions of the engine API. #[derive(Debug, Clone)] -pub struct BeaconConsensusEngineHandle +pub struct ConsensusEngineHandle where Payload: PayloadTypes, { to_engine: UnboundedSender>, } -impl BeaconConsensusEngineHandle +impl ConsensusEngineHandle where Payload: PayloadTypes, { diff --git a/crates/engine/service/Cargo.toml b/crates/engine/service/Cargo.toml index e2932ec6faa..6c7b746c741 100644 --- a/crates/engine/service/Cargo.toml +++ b/crates/engine/service/Cargo.toml @@ -31,7 +31,6 @@ futures.workspace = true pin-project.workspace = true # misc -thiserror.workspace = true [dev-dependencies] reth-engine-tree = { workspace = true, features = ["test-utils"] } @@ -39,7 +38,6 @@ reth-ethereum-consensus.workspace = true reth-ethereum-engine-primitives.workspace = true reth-evm-ethereum.workspace = true reth-exex-types.workspace = true -reth-chainspec.workspace = true reth-primitives-traits.workspace = true reth-node-ethereum.workspace = true diff --git a/crates/engine/service/src/lib.rs b/crates/engine/service/src/lib.rs index a707ae9ff93..cd61b0354ee 100644 --- a/crates/engine/service/src/lib.rs +++ b/crates/engine/service/src/lib.rs @@ -5,7 +5,7 @@ html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256", issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(test), warn(unused_crate_dependencies))] /// Engine Service diff --git a/crates/engine/service/src/service.rs b/crates/engine/service/src/service.rs index d9093aed30a..ff9eb66f100 100644 --- a/crates/engine/service/src/service.rs +++ b/crates/engine/service/src/service.rs @@ -2,13 +2,13 @@ use futures::{Stream, StreamExt}; use pin_project::pin_project; use reth_chainspec::EthChainSpec; use reth_consensus::{ConsensusError, FullConsensus}; -use reth_engine_primitives::{BeaconConsensusEngineEvent, BeaconEngineMessage, EngineValidator}; +use reth_engine_primitives::{BeaconEngineMessage, ConsensusEngineEvent}; use reth_engine_tree::{ backfill::PipelineSync, download::BasicBlockDownloader, engine::{EngineApiKind, EngineApiRequest, EngineApiRequestHandler, EngineHandler}, persistence::PersistenceHandle, - tree::{EngineApiTreeHandler, InvalidBlockHook, TreeConfig}, + tree::{EngineApiTreeHandler, EngineValidator, TreeConfig}, }; pub use reth_engine_tree::{ chain::{ChainEvent, ChainOrchestrator}, @@ -51,7 +51,7 @@ type EngineServiceType = ChainOrchestrator< /// The type that drives the chain forward and communicates progress. #[pin_project] #[expect(missing_debug_implementations)] -// TODO(mattsse): remove hidde once fixed : +// TODO(mattsse): remove hidden once fixed : // otherwise rustdoc fails to resolve the alias #[doc(hidden)] pub struct EngineService @@ -82,12 +82,11 @@ where payload_builder: PayloadBuilderHandle, payload_validator: V, tree_config: TreeConfig, - invalid_block_hook: Box>, sync_metrics_tx: MetricEventsSender, evm_config: C, ) -> Self where - V: EngineValidator>, + V: EngineValidator, C: ConfigureEvm + 'static, { let engine_kind = @@ -108,7 +107,6 @@ where payload_builder, canonical_in_memory_state, tree_config, - invalid_block_hook, engine_kind, evm_config, ); @@ -132,7 +130,7 @@ where N: ProviderNodeTypes, Client: BlockClient> + 'static, { - type Item = ChainEvent>; + type Item = ChainEvent>; fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { let mut orchestrator = self.project().orchestrator; @@ -140,17 +138,12 @@ where } } -/// Potential error returned by `EngineService`. -#[derive(Debug, thiserror::Error)] -#[error("Engine service error.")] -pub struct EngineServiceError {} - #[cfg(test)] mod tests { use super::*; use reth_chainspec::{ChainSpecBuilder, MAINNET}; - use reth_engine_primitives::BeaconEngineMessage; - use reth_engine_tree::{test_utils::TestPipelineBuilder, tree::NoopInvalidBlockHook}; + use reth_engine_primitives::{BeaconEngineMessage, NoopInvalidBlockHook}; + use reth_engine_tree::{test_utils::TestPipelineBuilder, tree::BasicEngineValidator}; use reth_ethereum_consensus::EthBeaconConsensus; use reth_ethereum_engine_primitives::EthEngineTypes; use reth_evm_ethereum::EthEvmConfig; @@ -195,6 +188,15 @@ mod tests { let pruner = Pruner::new_with_factory(provider_factory.clone(), vec![], 0, 0, None, rx); let evm_config = EthEvmConfig::new(chain_spec.clone()); + let engine_validator = BasicEngineValidator::new( + blockchain_db.clone(), + consensus.clone(), + evm_config.clone(), + engine_payload_validator, + TreeConfig::default(), + Box::new(NoopInvalidBlockHook::default()), + ); + let (sync_metrics_tx, _sync_metrics_rx) = unbounded_channel(); let (tx, _rx) = unbounded_channel(); let _eth_service = EngineService::new( @@ -208,9 +210,8 @@ mod tests { blockchain_db, pruner, PayloadBuilderHandle::new(tx), - engine_payload_validator, + engine_validator, TreeConfig::default(), - Box::new(NoopInvalidBlockHook::default()), sync_metrics_tx, evm_config, ); diff --git a/crates/engine/tree/Cargo.toml b/crates/engine/tree/Cargo.toml index ae9bf53319a..c55ce01cd47 100644 --- a/crates/engine/tree/Cargo.toml +++ b/crates/engine/tree/Cargo.toml @@ -18,6 +18,7 @@ reth-consensus.workspace = true reth-db.workspace = true reth-engine-primitives.workspace = true reth-errors.workspace = true +reth-execution-types.workspace = true reth-evm = { workspace = true, features = ["metrics"] } reth-network-p2p.workspace = true reth-payload-builder.workspace = true @@ -29,10 +30,11 @@ reth-prune.workspace = true reth-revm.workspace = true reth-stages-api.workspace = true reth-tasks.workspace = true -reth-trie-db.workspace = true reth-trie-parallel.workspace = true reth-trie-sparse = { workspace = true, features = ["std", "metrics"] } +reth-trie-sparse-parallel = { workspace = true, features = ["std"] } reth-trie.workspace = true +reth-mantle-forks = { workspace = true, features = ["std"] } # alloy alloy-evm.workspace = true @@ -42,6 +44,7 @@ alloy-primitives.workspace = true alloy-rlp.workspace = true alloy-rpc-types-engine.workspace = true +revm.workspace = true revm-primitives.workspace = true # common @@ -49,18 +52,21 @@ futures.workspace = true thiserror.workspace = true tokio = { workspace = true, features = ["rt", "rt-multi-thread", "sync", "macros"] } mini-moka = { workspace = true, features = ["sync"] } +smallvec.workspace = true # metrics metrics.workspace = true reth-metrics = { workspace = true, features = ["common"] } # misc +dashmap.workspace = true schnellru.workspace = true rayon.workspace = true tracing.workspace = true derive_more.workspace = true parking_lot.workspace = true -itertools.workspace = true +crossbeam-channel.workspace = true +eyre.workspace = true # optional deps for test-utils reth-prune-types = { workspace = true, optional = true } @@ -70,31 +76,31 @@ reth-tracing = { workspace = true, optional = true } [dev-dependencies] # reth -reth-evm-ethereum.workspace = true +reth-evm-ethereum = { workspace = true, features = ["test-utils"] } reth-chain-state = { workspace = true, features = ["test-utils"] } reth-chainspec.workspace = true reth-db-common.workspace = true reth-ethereum-consensus.workspace = true +metrics-util = { workspace = true, features = ["debugging"] } reth-ethereum-engine-primitives.workspace = true reth-evm = { workspace = true, features = ["test-utils"] } reth-exex-types.workspace = true reth-network-p2p = { workspace = true, features = ["test-utils"] } reth-prune-types.workspace = true -reth-prune.workspace = true -reth-rpc-types-compat.workspace = true reth-stages = { workspace = true, features = ["test-utils"] } reth-static-file.workspace = true reth-testing-utils.workspace = true reth-tracing.workspace = true -reth-trie-db.workspace = true reth-node-ethereum.workspace = true +reth-e2e-test-utils.workspace = true # alloy -alloy-rlp.workspace = true revm-state.workspace = true assert_matches.workspace = true criterion.workspace = true +eyre.workspace = true +serde_json.workspace = true crossbeam-channel.workspace = true proptest.workspace = true rand.workspace = true @@ -129,9 +135,12 @@ test-utils = [ "reth-trie/test-utils", "reth-trie-sparse/test-utils", "reth-prune-types?/test-utils", - "reth-trie-db/test-utils", "reth-trie-parallel/test-utils", "reth-ethereum-primitives/test-utils", "reth-node-ethereum/test-utils", "reth-evm-ethereum/test-utils", ] + +[[test]] +name = "e2e_testsuite" +path = "tests/e2e-testsuite/main.rs" diff --git a/crates/engine/tree/benches/channel_perf.rs b/crates/engine/tree/benches/channel_perf.rs index 74067d4de70..41dd651c890 100644 --- a/crates/engine/tree/benches/channel_perf.rs +++ b/crates/engine/tree/benches/channel_perf.rs @@ -5,7 +5,7 @@ use alloy_primitives::{B256, U256}; use criterion::{criterion_group, criterion_main, BatchSize, BenchmarkId, Criterion}; use proptest::test_runner::TestRunner; -use rand_08::Rng; +use rand::Rng; use revm_primitives::{Address, HashMap}; use revm_state::{Account, AccountInfo, AccountStatus, EvmState, EvmStorage, EvmStorageSlot}; use std::{hint::black_box, thread}; @@ -18,17 +18,18 @@ fn create_bench_state(num_accounts: usize) -> EvmState { for i in 0..num_accounts { let storage = - EvmStorage::from_iter([(U256::from(i), EvmStorageSlot::new(U256::from(i + 1)))]); + EvmStorage::from_iter([(U256::from(i), EvmStorageSlot::new(U256::from(i + 1), 0))]); let account = Account { info: AccountInfo { balance: U256::from(100), nonce: 10, - code_hash: B256::from_slice(&rng.r#gen::<[u8; 32]>()), + code_hash: B256::from_slice(&rng.random::<[u8; 32]>()), code: Default::default(), }, storage, - status: AccountStatus::Loaded, + status: AccountStatus::empty(), + transaction_id: 0, }; let address = Address::with_last_byte(i as u8); diff --git a/crates/engine/tree/benches/state_root_task.rs b/crates/engine/tree/benches/state_root_task.rs index 1b6596f28d3..b6306678b5b 100644 --- a/crates/engine/tree/benches/state_root_task.rs +++ b/crates/engine/tree/benches/state_root_task.rs @@ -8,22 +8,22 @@ use alloy_evm::block::StateChangeSource; use alloy_primitives::{Address, B256}; use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion}; use proptest::test_runner::TestRunner; -use rand_08::Rng; -use reth_chain_state::EthPrimitives; +use rand::Rng; use reth_chainspec::ChainSpec; use reth_db_common::init::init_genesis; use reth_engine_tree::tree::{ - executor::WorkloadExecutor, PayloadProcessor, StateProviderBuilder, TreeConfig, + executor::WorkloadExecutor, precompile_cache::PrecompileCacheMap, PayloadProcessor, + StateProviderBuilder, TreeConfig, }; +use reth_ethereum_primitives::TransactionSigned; use reth_evm::OnStateHook; use reth_evm_ethereum::EthEvmConfig; -use reth_primitives_traits::{Account as RethAccount, StorageEntry}; +use reth_primitives_traits::{Account as RethAccount, Recovered, StorageEntry}; use reth_provider::{ - providers::{BlockchainProvider, ConsistentDbView}, + providers::{BlockchainProvider, OverlayStateProviderFactory}, test_utils::{create_test_provider_factory_with_chain_spec, MockNodeTypesWithDB}, AccountReader, ChainSpecProvider, HashingWriter, ProviderFactory, }; -use reth_trie::TrieInput; use revm_primitives::{HashMap, U256}; use revm_state::{Account as RevmAccount, AccountInfo, AccountStatus, EvmState, EvmStorageSlot}; use std::{hint::black_box, sync::Arc}; @@ -41,51 +41,50 @@ struct BenchParams { fn create_bench_state_updates(params: &BenchParams) -> Vec { let mut runner = TestRunner::deterministic(); let mut rng = runner.rng().clone(); - let all_addresses: Vec
= (0..params.num_accounts) - .map(|_| { - // TODO: rand08 - Address::random() - }) - .collect(); - let mut updates = Vec::new(); + let all_addresses: Vec
= + (0..params.num_accounts).map(|_| Address::random_with(&mut rng)).collect(); + let mut updates = Vec::with_capacity(params.updates_per_account); for _ in 0..params.updates_per_account { let mut state_update = EvmState::default(); - let num_accounts_in_update = rng.gen_range(1..=params.num_accounts); + let num_accounts_in_update = rng.random_range(1..=params.num_accounts); // regular updates for randomly selected accounts for &address in &all_addresses[0..num_accounts_in_update] { // randomly choose to self-destruct with probability // (selfdestructs/accounts) - let is_selfdestruct = - rng.gen_bool(params.selfdestructs_per_update as f64 / params.num_accounts as f64); + let is_selfdestruct = rng + .random_bool(params.selfdestructs_per_update as f64 / params.num_accounts as f64); let account = if is_selfdestruct { RevmAccount { info: AccountInfo::default(), storage: HashMap::default(), status: AccountStatus::SelfDestructed, + transaction_id: 0, } } else { RevmAccount { info: AccountInfo { - balance: U256::from(rng.r#gen::()), - nonce: rng.r#gen::(), + balance: U256::from(rng.random::()), + nonce: rng.random::(), code_hash: KECCAK_EMPTY, code: Some(Default::default()), }, - storage: (0..rng.gen_range(0..=params.storage_slots_per_account)) + storage: (0..rng.random_range(0..=params.storage_slots_per_account)) .map(|_| { ( - U256::from(rng.r#gen::()), + U256::from(rng.random::()), EvmStorageSlot::new_changed( U256::ZERO, - U256::from(rng.r#gen::()), + U256::from(rng.random::()), + 0, ), ) }) .collect(), status: AccountStatus::Touched, + transaction_id: 0, } }; @@ -122,7 +121,7 @@ fn setup_provider( for update in state_updates { let provider_rw = factory.provider_rw()?; - let mut account_updates = Vec::new(); + let mut account_updates = Vec::with_capacity(update.len()); for (address, account) in update { // only process self-destructs if account exists, always process @@ -216,23 +215,25 @@ fn bench_state_root(c: &mut Criterion) { let state_updates = create_bench_state_updates(params); setup_provider(&factory, &state_updates).expect("failed to setup provider"); - let payload_processor = PayloadProcessor::::new( + let payload_processor = PayloadProcessor::new( WorkloadExecutor::default(), EthEvmConfig::new(factory.chain_spec()), &TreeConfig::default(), + PrecompileCacheMap::default(), ); let provider = BlockchainProvider::new(factory).unwrap(); (genesis_hash, payload_processor, provider, state_updates) }, - |(genesis_hash, payload_processor, provider, state_updates)| { + |(genesis_hash, mut payload_processor, provider, state_updates)| { black_box({ let mut handle = payload_processor.spawn( Default::default(), - Default::default(), + core::iter::empty::< + Result, core::convert::Infallible>, + >(), StateProviderBuilder::new(provider.clone(), genesis_hash, None), - ConsistentDbView::new_with_latest_tip(provider).unwrap(), - TrieInput::default(), + OverlayStateProviderFactory::new(provider), &TreeConfig::default(), ); diff --git a/crates/engine/tree/docs/mermaid/state-root-task.mmd b/crates/engine/tree/docs/mermaid/state-root-task.mmd index 011196d9e0d..d1993035f21 100644 --- a/crates/engine/tree/docs/mermaid/state-root-task.mmd +++ b/crates/engine/tree/docs/mermaid/state-root-task.mmd @@ -4,7 +4,7 @@ flowchart TD StateRootMessage::PrefetchProofs StateRootMessage::EmptyProof StateRootMessage::ProofCalculated - StataRootMessage::FinishedStateUpdates + StateRootMessage::FinishedStateUpdates end subgraph StateRootTask[State Root Task thread] @@ -40,5 +40,5 @@ flowchart TD StateRootMessage::ProofCalculated --> NewProof NewProof ---> MultiProofCompletion ProofSequencerCondition -->|Yes, send multiproof and state update| SparseTrieUpdate - StataRootMessage::FinishedStateUpdates --> EndCondition1 + StateRootMessage::FinishedStateUpdates --> EndCondition1 EndCondition3 -->|Close SparseTrieUpdate channel| SparseTrieUpdate diff --git a/crates/engine/tree/docs/root.md b/crates/engine/tree/docs/root.md index d3c4e1e5757..a5b9bcb1d48 100644 --- a/crates/engine/tree/docs/root.md +++ b/crates/engine/tree/docs/root.md @@ -10,7 +10,7 @@ root of the new state. 4. Compares the root with the one received in the block header. 5. Considers the block valid. -This document describes the lifecycle of a payload with the focus on state root calculation, +This document describes the lifecycle of a payload with a focus on state root calculation, from the moment the payload is received, to the moment we have a new state root. We will look at the following components: @@ -26,7 +26,7 @@ We will look at the following components: It all starts with the `engine_newPayload` request coming from the [Consensus Client](https://ethereum.org/en/developers/docs/nodes-and-clients/#consensus-clients). We extract the block from the payload, and eventually pass it to the `EngineApiTreeHandler::insert_block_inner` -method which executes the block and calculates the state root. +method that executes the block and calculates the state root. https://github.com/paradigmxyz/reth/blob/2ba54bf1c1f38c7173838f37027315a09287c20a/crates/engine/tree/src/tree/mod.rs#L2359-L2362 Let's walk through the steps involved in the process. @@ -166,7 +166,7 @@ and send `StateRootMessage::ProofCalculated` to the [State Root Task](#state-roo ### Exhausting the pending queue -To exhaust the pending queue from the step 2 of the `spawn_or_queue` described above, +To exhaust the pending queue from step 2 of the `spawn_or_queue` described above, the [State Root Task](#state-root-task) calls into another method `on_calculation_complete` every time a proof is calculated. https://github.com/paradigmxyz/reth/blob/2ba54bf1c1f38c7173838f37027315a09287c20a/crates/engine/tree/src/tree/root.rs#L379-L387 @@ -230,11 +230,11 @@ https://github.com/paradigmxyz/reth/blob/2ba54bf1c1f38c7173838f37027315a09287c20 https://github.com/paradigmxyz/reth/blob/2ba54bf1c1f38c7173838f37027315a09287c20a/crates/engine/tree/src/tree/root.rs#L1093 3. Update accounts trie https://github.com/paradigmxyz/reth/blob/2ba54bf1c1f38c7173838f37027315a09287c20a/crates/engine/tree/src/tree/root.rs#L1133 -4. Calculate keccak hashes of the nodes below the certain level +4. Calculate keccak hashes of the nodes below a certain level https://github.com/paradigmxyz/reth/blob/2ba54bf1c1f38c7173838f37027315a09287c20a/crates/engine/tree/src/tree/root.rs#L1139 As you can see, we do not calculate the state root hash of the accounts trie -(the one that will be the result of the whole task), but instead calculate only the certain hashes. +(the one that will be the result of the whole task), but instead calculate only certain hashes. This is an optimization that comes from the fact that we will likely update the top 2-3 levels of the trie in every transaction, so doing that work every time would be wasteful. diff --git a/crates/engine/tree/src/chain.rs b/crates/engine/tree/src/chain.rs index e2893bb976a..3e6207c9d40 100644 --- a/crates/engine/tree/src/chain.rs +++ b/crates/engine/tree/src/chain.rs @@ -71,7 +71,7 @@ where /// Internal function used to advance the chain. /// /// Polls the `ChainOrchestrator` for the next event. - #[tracing::instrument(level = "debug", name = "ChainOrchestrator::poll", skip(self, cx))] + #[tracing::instrument(level = "debug", target = "engine::tree::chain_orchestrator", skip_all)] fn poll_next_event(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { let this = self.get_mut(); diff --git a/crates/engine/tree/src/download.rs b/crates/engine/tree/src/download.rs index 5d7d52af848..b7c147e4524 100644 --- a/crates/engine/tree/src/download.rs +++ b/crates/engine/tree/src/download.rs @@ -121,7 +121,7 @@ where self.download_full_block(hash); } else { trace!( - target: "consensus::engine", + target: "engine::download", ?hash, ?count, "start downloading full block range." @@ -152,7 +152,7 @@ where }); trace!( - target: "consensus::engine::sync", + target: "engine::download", ?hash, "Start downloading full block" ); @@ -213,7 +213,7 @@ where for idx in (0..self.inflight_full_block_requests.len()).rev() { let mut request = self.inflight_full_block_requests.swap_remove(idx); if let Poll::Ready(block) = request.poll_unpin(cx) { - trace!(target: "consensus::engine", block=?block.num_hash(), "Received single full block, buffering"); + trace!(target: "engine::download", block=?block.num_hash(), "Received single full block, buffering"); self.set_buffered_blocks.push(Reverse(block.into())); } else { // still pending @@ -225,7 +225,7 @@ where for idx in (0..self.inflight_block_range_requests.len()).rev() { let mut request = self.inflight_block_range_requests.swap_remove(idx); if let Poll::Ready(blocks) = request.poll_unpin(cx) { - trace!(target: "consensus::engine", len=?blocks.len(), first=?blocks.first().map(|b| b.num_hash()), last=?blocks.last().map(|b| b.num_hash()), "Received full block range, buffering"); + trace!(target: "engine::download", len=?blocks.len(), first=?blocks.first().map(|b| b.num_hash()), last=?blocks.last().map(|b| b.num_hash()), "Received full block range, buffering"); self.set_buffered_blocks.extend( blocks .into_iter() diff --git a/crates/engine/tree/src/engine.rs b/crates/engine/tree/src/engine.rs index 16ba1034399..f08195b205e 100644 --- a/crates/engine/tree/src/engine.rs +++ b/crates/engine/tree/src/engine.rs @@ -7,8 +7,8 @@ use crate::{ }; use alloy_primitives::B256; use futures::{Stream, StreamExt}; -use reth_chain_state::ExecutedBlockWithTrieUpdates; -use reth_engine_primitives::{BeaconConsensusEngineEvent, BeaconEngineMessage}; +use reth_chain_state::ExecutedBlock; +use reth_engine_primitives::{BeaconEngineMessage, ConsensusEngineEvent}; use reth_ethereum_primitives::EthPrimitives; use reth_payload_primitives::PayloadTypes; use reth_primitives_traits::{Block, NodePrimitives, RecoveredBlock}; @@ -191,7 +191,7 @@ impl EngineRequestHandler for EngineApiRequestHandle where Request: Send, { - type Event = BeaconConsensusEngineEvent; + type Event = ConsensusEngineEvent; type Request = Request; type Block = N::Block; @@ -246,7 +246,7 @@ pub enum EngineApiRequest { /// A request received from the consensus engine. Beacon(BeaconEngineMessage), /// Request to insert an already executed block, e.g. via payload building. - InsertExecutedBlock(ExecutedBlockWithTrieUpdates), + InsertExecutedBlock(ExecutedBlock), } impl Display for EngineApiRequest { @@ -279,7 +279,7 @@ impl From> pub enum EngineApiEvent { /// Event from the consensus engine. // TODO(mattsse): find a more appropriate name for this variant, consider phasing it out. - BeaconConsensus(BeaconConsensusEngineEvent), + BeaconConsensus(ConsensusEngineEvent), /// Backfill action is needed. BackfillAction(BackfillAction), /// Block download is needed. @@ -293,8 +293,8 @@ impl EngineApiEvent { } } -impl From> for EngineApiEvent { - fn from(event: BeaconConsensusEngineEvent) -> Self { +impl From> for EngineApiEvent { + fn from(event: ConsensusEngineEvent) -> Self { Self::BeaconConsensus(event) } } diff --git a/crates/engine/tree/src/lib.rs b/crates/engine/tree/src/lib.rs index f197dd764aa..43f29b8e0ba 100644 --- a/crates/engine/tree/src/lib.rs +++ b/crates/engine/tree/src/lib.rs @@ -5,7 +5,7 @@ //! The components in this crate are involved in: //! * Handling and reacting to incoming consensus events ([`EngineHandler`](engine::EngineHandler)) //! * Advancing the chain ([`ChainOrchestrator`](chain::ChainOrchestrator)) -//! * Keeping track of the chain structure in-memory ([`TreeState`](tree::TreeState)) +//! * Keeping track of the chain structure in-memory ([`TreeState`](tree::state::TreeState)) //! * Performing backfill sync and handling its progress ([`BackfillSync`](backfill::BackfillSync)) //! * Downloading blocks ([`BlockDownloader`](download::BlockDownloader)), and //! * Persisting blocks and performing pruning @@ -58,10 +58,10 @@ //! //! ## Chain representation //! -//! The chain is represented by the [`TreeState`](tree::TreeState) data structure, which keeps -//! tracks of blocks by hash and number, as well as keeping track of parent-child relationships -//! between blocks. The hash and number of the current head of the canonical chain is also tracked -//! in the [`TreeState`](tree::TreeState). +//! The chain is represented by the [`TreeState`](tree::state::TreeState) data structure, which +//! keeps tracks of blocks by hash and number, as well as keeping track of parent-child +//! relationships between blocks. The hash and number of the current head of the canonical chain is +//! also tracked in the [`TreeState`](tree::state::TreeState). //! //! ## Persistence model //! @@ -89,7 +89,7 @@ html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256", issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(test), warn(unused_crate_dependencies))] /// Support for backfill sync mode. diff --git a/crates/engine/tree/src/persistence.rs b/crates/engine/tree/src/persistence.rs index 1c417e357fb..12482b1a162 100644 --- a/crates/engine/tree/src/persistence.rs +++ b/crates/engine/tree/src/persistence.rs @@ -1,13 +1,13 @@ use crate::metrics::PersistenceMetrics; use alloy_consensus::BlockHeader; use alloy_eips::BlockNumHash; -use reth_chain_state::ExecutedBlockWithTrieUpdates; +use reth_chain_state::ExecutedBlock; use reth_errors::ProviderError; use reth_ethereum_primitives::EthPrimitives; use reth_primitives_traits::NodePrimitives; use reth_provider::{ - providers::ProviderNodeTypes, writer::UnifiedStorageWriter, BlockHashReader, - ChainStateBlockWriter, DatabaseProviderFactory, ProviderFactory, StaticFileProviderFactory, + providers::ProviderNodeTypes, BlockExecutionWriter, BlockHashReader, ChainStateBlockWriter, + DBProvider, DatabaseProviderFactory, ProviderFactory, }; use reth_prune::{PrunerError, PrunerOutput, PrunerWithFactory}; use reth_stages_api::{MetricEvent, MetricEventsSender}; @@ -57,7 +57,7 @@ where Self { provider, incoming, pruner, metrics: PersistenceMetrics::default(), sync_metrics_tx } } - /// Prunes block data before the given block hash according to the configured prune + /// Prunes block data before the given block number according to the configured prune /// configuration. fn prune_before(&mut self, block_num: u64) -> Result { debug!(target: "engine::persistence", ?block_num, "Running pruner"); @@ -128,11 +128,10 @@ where debug!(target: "engine::persistence", ?new_tip_num, "Removing blocks"); let start_time = Instant::now(); let provider_rw = self.provider.database_provider_rw()?; - let sf_provider = self.provider.static_file_provider(); let new_tip_hash = provider_rw.block_hash(new_tip_num)?; - UnifiedStorageWriter::from(&provider_rw, &sf_provider).remove_blocks_above(new_tip_num)?; - UnifiedStorageWriter::commit_unwind(provider_rw)?; + provider_rw.remove_block_and_execution_above(new_tip_num)?; + provider_rw.commit()?; debug!(target: "engine::persistence", ?new_tip_num, ?new_tip_hash, "Removed blocks from disk"); self.metrics.remove_blocks_above_duration_seconds.record(start_time.elapsed()); @@ -141,9 +140,12 @@ where fn on_save_blocks( &self, - blocks: Vec>, + blocks: Vec>, ) -> Result, PersistenceError> { - debug!(target: "engine::persistence", first=?blocks.first().map(|b| b.recovered_block.num_hash()), last=?blocks.last().map(|b| b.recovered_block.num_hash()), "Saving range of blocks"); + let first_block_hash = blocks.first().map(|b| b.recovered_block.num_hash()); + let last_block_hash = blocks.last().map(|b| b.recovered_block.num_hash()); + debug!(target: "engine::persistence", first=?first_block_hash, last=?last_block_hash, "Saving range of blocks"); + let start_time = Instant::now(); let last_block_hash_num = blocks.last().map(|block| BlockNumHash { hash: block.recovered_block().hash(), @@ -152,11 +154,13 @@ where if last_block_hash_num.is_some() { let provider_rw = self.provider.database_provider_rw()?; - let static_file_provider = self.provider.static_file_provider(); - UnifiedStorageWriter::from(&provider_rw, &static_file_provider).save_blocks(blocks)?; - UnifiedStorageWriter::commit(provider_rw)?; + provider_rw.save_blocks(blocks)?; + provider_rw.commit()?; } + + debug!(target: "engine::persistence", first=?first_block_hash, last=?last_block_hash, "Saved range of blocks"); + self.metrics.save_blocks_duration_seconds.record(start_time.elapsed()); Ok(last_block_hash_num) } @@ -182,7 +186,7 @@ pub enum PersistenceAction { /// /// First, header, transaction, and receipt-related data should be written to static files. /// Then the execution history-related data will be written to the database. - SaveBlocks(Vec>, oneshot::Sender>), + SaveBlocks(Vec>, oneshot::Sender>), /// Removes block data above the given block number from the database. /// @@ -259,7 +263,7 @@ impl PersistenceHandle { /// If there are no blocks to persist, then `None` is sent in the sender. pub fn save_blocks( &self, - blocks: Vec>, + blocks: Vec>, tx: oneshot::Sender>, ) -> Result<(), SendError>> { self.send_action(PersistenceAction::SaveBlocks(blocks, tx)) @@ -273,7 +277,7 @@ impl PersistenceHandle { self.send_action(PersistenceAction::SaveFinalizedBlock(finalized_block)) } - /// Persists the finalized block number on disk. + /// Persists the safe block number on disk. pub fn save_safe_block_number( &self, safe_block: u64, diff --git a/crates/engine/tree/src/test_utils.rs b/crates/engine/tree/src/test_utils.rs index 2ec00f9b918..e011a54b73c 100644 --- a/crates/engine/tree/src/test_utils.rs +++ b/crates/engine/tree/src/test_utils.rs @@ -3,9 +3,8 @@ use reth_chainspec::ChainSpec; use reth_ethereum_primitives::BlockBody; use reth_network_p2p::test_utils::TestFullBlockClient; use reth_primitives_traits::SealedHeader; -use reth_provider::{ - test_utils::{create_test_provider_factory_with_chain_spec, MockNodeTypesWithDB}, - ExecutionOutcome, +use reth_provider::test_utils::{ + create_test_provider_factory_with_chain_spec, MockNodeTypesWithDB, }; use reth_prune_types::PruneModes; use reth_stages::{test_utils::TestStages, ExecOutput, StageError}; @@ -18,13 +17,12 @@ use tokio::sync::watch; #[derive(Default, Debug)] pub struct TestPipelineBuilder { pipeline_exec_outputs: VecDeque>, - executor_results: Vec, } impl TestPipelineBuilder { /// Create a new [`TestPipelineBuilder`]. pub const fn new() -> Self { - Self { pipeline_exec_outputs: VecDeque::new(), executor_results: Vec::new() } + Self { pipeline_exec_outputs: VecDeque::new() } } /// Set the pipeline execution outputs to use for the test consensus engine. @@ -37,8 +35,14 @@ impl TestPipelineBuilder { } /// Set the executor results to use for the test consensus engine. - pub fn with_executor_results(mut self, executor_results: Vec) -> Self { - self.executor_results = executor_results; + #[deprecated( + note = "no-op: executor results are not used and will be removed in a future release" + )] + pub fn with_executor_results( + self, + executor_results: Vec, + ) -> Self { + let _ = executor_results; self } diff --git a/crates/engine/tree/src/tree/block_buffer.rs b/crates/engine/tree/src/tree/block_buffer.rs index 6da92818e21..5c168198611 100644 --- a/crates/engine/tree/src/tree/block_buffer.rs +++ b/crates/engine/tree/src/tree/block_buffer.rs @@ -74,9 +74,7 @@ impl BlockBuffer { if self.block_queue.len() >= self.max_blocks { // Evict oldest block if limit is hit if let Some(evicted_hash) = self.block_queue.pop_front() { - if let Some(evicted_block) = self.remove_block(&evicted_hash) { - self.remove_from_parent(evicted_block.parent_hash(), &evicted_hash); - } + self.remove_block(&evicted_hash); } } self.block_queue.push_back(hash); @@ -495,4 +493,57 @@ mod tests { assert_buffer_lengths(&buffer, 3); } + + #[test] + fn eviction_parent_child_cleanup() { + let mut rng = generators::rng(); + + let main_parent = BlockNumHash::new(9, rng.random()); + let block1 = create_block(&mut rng, 10, main_parent.hash); + let block2 = create_block(&mut rng, 11, block1.hash()); + // Unrelated block to trigger eviction + let unrelated_parent = rng.random(); + let unrelated_block = create_block(&mut rng, 12, unrelated_parent); + + // Capacity 2 so third insert evicts the oldest (block1) + let mut buffer = BlockBuffer::new(2); + + buffer.insert_block(block1.clone()); + buffer.insert_block(block2.clone()); + + // Pre-eviction: parent_to_child contains main_parent -> {block1}, block1 -> {block2} + assert!(buffer + .parent_to_child + .get(&main_parent.hash) + .and_then(|s| s.get(&block1.hash())) + .is_some()); + assert!(buffer + .parent_to_child + .get(&block1.hash()) + .and_then(|s| s.get(&block2.hash())) + .is_some()); + + // Insert unrelated block to evict block1 + buffer.insert_block(unrelated_block); + + // Evicted block1 should be fully removed from collections + assert_block_removal(&buffer, &block1); + + // Cleanup: parent_to_child must no longer have (main_parent -> block1) + assert!(buffer + .parent_to_child + .get(&main_parent.hash) + .and_then(|s| s.get(&block1.hash())) + .is_none()); + + // But the mapping (block1 -> block2) must remain so descendants can still be tracked + assert!(buffer + .parent_to_child + .get(&block1.hash()) + .and_then(|s| s.get(&block2.hash())) + .is_some()); + + // And lowest ancestor for block2 becomes itself after its parent is evicted + assert_eq!(buffer.lowest_ancestor(&block2.hash()), Some(&block2)); + } } diff --git a/crates/engine/tree/src/tree/cached_state.rs b/crates/engine/tree/src/tree/cached_state.rs index a6e16a7503a..fd9999b9eba 100644 --- a/crates/engine/tree/src/tree/cached_state.rs +++ b/crates/engine/tree/src/tree/cached_state.rs @@ -1,4 +1,4 @@ -//! Implements a state provider that has a shared cache in front of it. +//! Execution cache implementation for block processing. use alloy_primitives::{Address, StorageKey, StorageValue, B256}; use metrics::Gauge; use mini_moka::sync::CacheBuilder; @@ -6,8 +6,8 @@ use reth_errors::ProviderResult; use reth_metrics::Metrics; use reth_primitives_traits::{Account, Bytecode}; use reth_provider::{ - AccountReader, BlockHashReader, HashedPostStateProvider, StateProofProvider, StateProvider, - StateRootProvider, StorageRootProvider, + AccountReader, BlockHashReader, BytecodeReader, HashedPostStateProvider, StateProofProvider, + StateProvider, StateRootProvider, StorageRootProvider, }; use reth_revm::db::BundleState; use reth_trie::{ @@ -15,8 +15,8 @@ use reth_trie::{ MultiProofTargets, StorageMultiProof, StorageProof, TrieInput, }; use revm_primitives::map::DefaultHashBuilder; -use std::time::Duration; -use tracing::trace; +use std::{sync::Arc, time::Duration}; +use tracing::{debug_span, instrument, trace}; pub(crate) type Cache = mini_moka::sync::Cache; @@ -27,7 +27,7 @@ pub(crate) struct CachedStateProvider { state_provider: S, /// The caches used for the provider - caches: ProviderCaches, + caches: ExecutionCache, /// Metrics for the cached state provider metrics: CachedStateMetrics, @@ -37,11 +37,11 @@ impl CachedStateProvider where S: StateProvider, { - /// Creates a new [`CachedStateProvider`] from a [`ProviderCaches`], state provider, and + /// Creates a new [`CachedStateProvider`] from an [`ExecutionCache`], state provider, and /// [`CachedStateMetrics`]. pub(crate) const fn new_with_caches( state_provider: S, - caches: ProviderCaches, + caches: ExecutionCache, metrics: CachedStateMetrics, ) -> Self { Self { state_provider, caches, metrics } @@ -128,14 +128,14 @@ impl AccountReader for CachedStateProvider { } } -/// Represents the status of a storage slot in the cache +/// Represents the status of a storage slot in the cache. #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) enum SlotStatus { - /// The account's storage cache doesn't exist + /// The account's storage cache doesn't exist. NotCached, - /// The storage slot is empty (either not in cache or explicitly None) + /// The storage slot exists in cache and is empty (value is zero). Empty, - /// The storage slot has a value + /// The storage slot exists in cache and has a specific non-zero value. Value(StorageValue), } @@ -162,7 +162,9 @@ impl StateProvider for CachedStateProvider { } } } +} +impl BytecodeReader for CachedStateProvider { fn bytecode_by_hash(&self, code_hash: &B256) -> ProviderResult> { if let Some(res) = self.caches.code_cache.get(code_hash) { self.metrics.code_cache_hits.increment(1); @@ -246,6 +248,18 @@ impl StorageRootProvider for CachedStateProvider { self.state_provider.storage_proof(address, slot, hashed_storage) } + /// Generate a storage multiproof for multiple storage slots. + /// + /// A **storage multiproof** is a cryptographic proof that can verify the values + /// of multiple storage slots for a single account in a single verification step. + /// Instead of generating separate proofs for each slot (which would be inefficient), + /// a multiproof bundles the necessary trie nodes to prove all requested slots. + /// + /// ## How it works: + /// 1. Takes an account address and a list of storage slot keys + /// 2. Traverses the account's storage trie to collect proof nodes + /// 3. Returns a [`StorageMultiProof`] containing the minimal set of trie nodes needed to verify + /// all the requested storage slots fn storage_multiproof( &self, address: Address, @@ -276,20 +290,25 @@ impl HashedPostStateProvider for CachedStateProvider } } -/// The set of caches that are used in the [`CachedStateProvider`]. +/// Execution cache used during block processing. +/// +/// Optimizes state access by maintaining in-memory copies of frequently accessed +/// accounts, storage slots, and bytecode. Works in conjunction with prewarming +/// to reduce database I/O during block execution. #[derive(Debug, Clone)] -pub(crate) struct ProviderCaches { - /// The cache for bytecode +pub(crate) struct ExecutionCache { + /// Cache for contract bytecode, keyed by code hash. code_cache: Cache>, - /// The cache for storage, organized hierarchically by account - storage_cache: Cache, + /// Per-account storage cache: outer cache keyed by Address, inner cache tracks that account’s + /// storage slots. + storage_cache: Cache>, - /// The cache for basic accounts + /// Cache for basic account information (nonce, balance, code hash). account_cache: Cache>, } -impl ProviderCaches { +impl ExecutionCache { /// Get storage value from hierarchical cache. /// /// Returns a `SlotStatus` indicating whether: @@ -310,12 +329,26 @@ impl ProviderCaches { key: StorageKey, value: Option, ) { - let account_cache = self.storage_cache.get(&address).unwrap_or_else(|| { - let account_cache = AccountStorageCache::default(); - self.storage_cache.insert(address, account_cache.clone()); - account_cache - }); - account_cache.insert_storage(key, value); + self.insert_storage_bulk(address, [(key, value)]); + } + + /// Insert multiple storage values into hierarchical cache for a single account + /// + /// This method is optimized for inserting multiple storage values for the same address + /// by doing the account cache lookup only once instead of for each key-value pair. + pub(crate) fn insert_storage_bulk(&self, address: Address, storage_entries: I) + where + I: IntoIterator)>, + { + let account_cache = self.storage_cache.get(&address).unwrap_or_default(); + + for (key, value) in storage_entries { + account_cache.insert_storage(key, value); + } + + // Insert to the cache so that moka picks up on the changed size, even though the actual + // value (the Arc) is the same + self.storage_cache.insert(address, account_cache); } /// Invalidate storage for specific account @@ -328,24 +361,43 @@ impl ProviderCaches { self.storage_cache.iter().map(|addr| addr.len()).sum() } - /// Inserts the [`BundleState`] entries into the cache. + /// Inserts the post-execution state changes into the cache. + /// + /// This method is called after transaction execution to update the cache with + /// the touched and modified state. The insertion order is critical: /// - /// Entries are inserted in the following order: - /// 1. Bytecodes - /// 2. Storage slots - /// 3. Accounts + /// 1. Bytecodes: Insert contract code first + /// 2. Storage slots: Update storage values for each account + /// 3. Accounts: Update account info (nonce, balance, code hash) /// - /// The order is important, because the access patterns are Account -> Bytecode and Account -> - /// Storage slot. If we update the account first, it may point to a code hash that doesn't have - /// the associated bytecode anywhere yet. + /// ## Why This Order Matters /// - /// Returns an error if the state can't be cached and should be discarded. + /// Account information references bytecode via code hash. If we update accounts + /// before bytecode, we might create cache entries pointing to non-existent code. + /// The current order ensures cache consistency. + /// + /// ## Error Handling + /// + /// Returns an error if the state updates are inconsistent and should be discarded. + #[instrument(level = "debug", target = "engine::caching", skip_all)] pub(crate) fn insert_state(&self, state_updates: &BundleState) -> Result<(), ()> { + let _enter = + debug_span!(target: "engine::tree", "contracts", len = state_updates.contracts.len()) + .entered(); // Insert bytecodes for (code_hash, bytecode) in &state_updates.contracts { self.code_cache.insert(*code_hash, Some(Bytecode(bytecode.clone()))); } - + drop(_enter); + + let _enter = debug_span!( + target: "engine::tree", + "accounts", + accounts = state_updates.state.len(), + storages = + state_updates.state.values().map(|account| account.storage.len()).sum::() + ) + .entered(); for (addr, account) in &state_updates.state { // If the account was not modified, as in not changed and not destroyed, then we have // nothing to do w.r.t. this particular account and can move on @@ -371,11 +423,14 @@ impl ProviderCaches { }; // Now we iterate over all storage and make updates to the cached storage values - for (storage_key, slot) in &account.storage { + // Use bulk insertion to optimize cache lookups - only lookup the account cache once + // instead of for each storage key + let storage_entries = account.storage.iter().map(|(storage_key, slot)| { // We convert the storage key from U256 to B256 because that is how it's represented // in the cache - self.insert_storage(*addr, (*storage_key).into(), Some(slot.present_value)); - } + ((*storage_key).into(), Some(slot.present_value)) + }); + self.insert_storage_bulk(*addr, storage_entries); // Insert will update if present, so we just use the new account info as the new value // for the account cache @@ -386,9 +441,9 @@ impl ProviderCaches { } } -/// A builder for [`ProviderCaches`]. +/// A builder for [`ExecutionCache`]. #[derive(Debug)] -pub(crate) struct ProviderCacheBuilder { +pub(crate) struct ExecutionCacheBuilder { /// Code cache entries code_cache_entries: u64, @@ -399,9 +454,9 @@ pub(crate) struct ProviderCacheBuilder { account_cache_entries: u64, } -impl ProviderCacheBuilder { - /// Build a [`ProviderCaches`] struct, so that provider caches can be easily cloned. - pub(crate) fn build_caches(self, total_cache_size: u64) -> ProviderCaches { +impl ExecutionCacheBuilder { + /// Build an [`ExecutionCache`] struct, so that execution caches can be easily cloned. + pub(crate) fn build_caches(self, total_cache_size: u64) -> ExecutionCache { let storage_cache_size = (total_cache_size * 8888) / 10000; // 88.88% of total let account_cache_size = (total_cache_size * 556) / 10000; // 5.56% of total let code_cache_size = (total_cache_size * 556) / 10000; // 5.56% of total @@ -410,7 +465,7 @@ impl ProviderCacheBuilder { const TIME_TO_IDLE: Duration = Duration::from_secs(3600); // 1 hour let storage_cache = CacheBuilder::new(self.storage_cache_entries) - .weigher(|_key: &Address, value: &AccountStorageCache| -> u32 { + .weigher(|_key: &Address, value: &Arc| -> u32 { // values based on results from measure_storage_cache_overhead test let base_weight = 39_000; let slots_weight = value.len() * 218; @@ -423,24 +478,8 @@ impl ProviderCacheBuilder { let account_cache = CacheBuilder::new(self.account_cache_entries) .weigher(|_key: &Address, value: &Option| -> u32 { - match value { - Some(account) => { - let mut weight = 40; - if account.nonce != 0 { - weight += 32; - } - if !account.balance.is_zero() { - weight += 32; - } - if account.bytecode_hash.is_some() { - weight += 33; // size of Option - } else { - weight += 8; // size of None variant - } - weight as u32 - } - None => 8, // size of None variant - } + // Account has a fixed size (none, balance,code_hash) + 20 + size_of_val(value) as u32 }) .max_capacity(account_cache_size) .time_to_live(EXPIRY_TIME) @@ -449,24 +488,30 @@ impl ProviderCacheBuilder { let code_cache = CacheBuilder::new(self.code_cache_entries) .weigher(|_key: &B256, value: &Option| -> u32 { - match value { + let code_size = match value { Some(bytecode) => { - // base weight + actual bytecode size - (40 + bytecode.len()) as u32 + // base weight + actual (padded) bytecode size + size of the jump table + (size_of_val(value) + + bytecode.bytecode().len() + + bytecode + .legacy_jump_table() + .map(|table| table.as_slice().len()) + .unwrap_or_default()) as u32 } - None => 8, // size of None variant - } + None => size_of_val(value) as u32, + }; + 32 + code_size }) .max_capacity(code_cache_size) .time_to_live(EXPIRY_TIME) .time_to_idle(TIME_TO_IDLE) .build_with_hasher(DefaultHashBuilder::default()); - ProviderCaches { code_cache, storage_cache, account_cache } + ExecutionCache { code_cache, storage_cache, account_cache } } } -impl Default for ProviderCacheBuilder { +impl Default for ExecutionCacheBuilder { fn default() -> Self { // With weigher and max_capacity in place, these numbers represent // the maximum number of entries that can be stored, not the actual @@ -491,20 +536,20 @@ pub(crate) struct SavedCache { hash: B256, /// The caches used for the provider. - caches: ProviderCaches, + caches: ExecutionCache, /// Metrics for the cached state provider metrics: CachedStateMetrics, + + /// A guard to track in-flight usage of this cache. + /// The cache is considered available if the strong count is 1. + usage_guard: Arc<()>, } impl SavedCache { /// Creates a new instance with the internals - pub(super) const fn new( - hash: B256, - caches: ProviderCaches, - metrics: CachedStateMetrics, - ) -> Self { - Self { hash, caches, metrics } + pub(super) fn new(hash: B256, caches: ExecutionCache, metrics: CachedStateMetrics) -> Self { + Self { hash, caches, metrics, usage_guard: Arc::new(()) } } /// Returns the hash for this cache @@ -513,16 +558,26 @@ impl SavedCache { } /// Splits the cache into its caches and metrics, consuming it. - pub(crate) fn split(self) -> (ProviderCaches, CachedStateMetrics) { + pub(crate) fn split(self) -> (ExecutionCache, CachedStateMetrics) { (self.caches, self.metrics) } - /// Returns the [`ProviderCaches`] belonging to the tracked hash. - pub(crate) const fn cache(&self) -> &ProviderCaches { + /// Returns true if the cache is available for use (no other tasks are currently using it). + pub(crate) fn is_available(&self) -> bool { + Arc::strong_count(&self.usage_guard) == 1 + } + + /// Returns the [`ExecutionCache`] belonging to the tracked hash. + pub(crate) const fn cache(&self) -> &ExecutionCache { &self.caches } - /// Updates the metrics for the [`ProviderCaches`]. + /// Returns the metrics associated with this cache. + pub(crate) const fn metrics(&self) -> &CachedStateMetrics { + &self.metrics + } + + /// Updates the metrics for the [`ExecutionCache`]. pub(crate) fn update_metrics(&self) { self.metrics.storage_cache_size.set(self.caches.total_storage_slots() as f64); self.metrics.account_cache_size.set(self.caches.account_cache.entry_count() as f64); @@ -530,10 +585,20 @@ impl SavedCache { } } -/// Cache for an account's storage slots +#[cfg(test)] +impl SavedCache { + fn clone_guard_for_test(&self) -> Arc<()> { + self.usage_guard.clone() + } +} + +/// Cache for an individual account's storage slots. +/// +/// This represents the second level of the hierarchical storage cache. +/// Each account gets its own `AccountStorageCache` to store accessed storage slots. #[derive(Debug, Clone)] pub(crate) struct AccountStorageCache { - /// The storage slots for this account + /// Map of storage keys to their cached values. slots: Cache>, } @@ -619,7 +684,7 @@ mod tests { unsafe impl GlobalAlloc for TrackingAllocator { unsafe fn alloc(&self, layout: Layout) -> *mut u8 { - let ret = self.inner.alloc(layout); + let ret = unsafe { self.inner.alloc(layout) }; if !ret.is_null() { self.allocated.fetch_add(layout.size(), Ordering::SeqCst); self.total_allocated.fetch_add(layout.size(), Ordering::SeqCst); @@ -629,7 +694,7 @@ mod tests { unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { self.allocated.fetch_sub(layout.size(), Ordering::SeqCst); - self.inner.dealloc(ptr, layout) + unsafe { self.inner.dealloc(ptr, layout) } } } } @@ -690,7 +755,7 @@ mod tests { let provider = MockEthProvider::default(); provider.extend_accounts(vec![(address, account)]); - let caches = ProviderCacheBuilder::default().build_caches(1000); + let caches = ExecutionCacheBuilder::default().build_caches(1000); let state_provider = CachedStateProvider::new_with_caches(provider, caches, CachedStateMetrics::zeroed()); @@ -713,11 +778,11 @@ mod tests { let provider = MockEthProvider::default(); provider.extend_accounts(vec![(address, account)]); - let caches = ProviderCacheBuilder::default().build_caches(1000); + let caches = ExecutionCacheBuilder::default().build_caches(1000); let state_provider = CachedStateProvider::new_with_caches(provider, caches, CachedStateMetrics::zeroed()); - // check that the storage is empty + // check that the storage returns the expected value let res = state_provider.storage(address, storage_key); assert!(res.is_ok()); assert_eq!(res.unwrap(), Some(storage_value)); @@ -731,10 +796,10 @@ mod tests { let storage_value = U256::from(1); // insert into caches directly - let caches = ProviderCacheBuilder::default().build_caches(1000); + let caches = ExecutionCacheBuilder::default().build_caches(1000); caches.insert_storage(address, storage_key, Some(storage_value)); - // check that the storage is empty + // check that the storage returns the cached value let slot_status = caches.get_storage(&address, &storage_key); assert_eq!(slot_status, SlotStatus::Value(storage_value)); } @@ -746,9 +811,9 @@ mod tests { let address = Address::random(); // just create empty caches - let caches = ProviderCacheBuilder::default().build_caches(1000); + let caches = ExecutionCacheBuilder::default().build_caches(1000); - // check that the storage is empty + // check that the storage is not cached let slot_status = caches.get_storage(&address, &storage_key); assert_eq!(slot_status, SlotStatus::NotCached); } @@ -761,11 +826,52 @@ mod tests { let storage_key = StorageKey::random(); // insert into caches directly - let caches = ProviderCacheBuilder::default().build_caches(1000); + let caches = ExecutionCacheBuilder::default().build_caches(1000); caches.insert_storage(address, storage_key, None); // check that the storage is empty let slot_status = caches.get_storage(&address, &storage_key); assert_eq!(slot_status, SlotStatus::Empty); } + + // Tests for SavedCache locking mechanism + #[test] + fn test_saved_cache_is_available() { + let execution_cache = ExecutionCacheBuilder::default().build_caches(1000); + let cache = SavedCache::new(B256::ZERO, execution_cache, CachedStateMetrics::zeroed()); + + // Initially, the cache should be available (only one reference) + assert!(cache.is_available(), "Cache should be available initially"); + + // Clone the usage guard (simulating it being handed out) + let _guard = cache.clone_guard_for_test(); + + // Now the cache should not be available (two references) + assert!(!cache.is_available(), "Cache should not be available with active guard"); + } + + #[test] + fn test_saved_cache_multiple_references() { + let execution_cache = ExecutionCacheBuilder::default().build_caches(1000); + let cache = + SavedCache::new(B256::from([2u8; 32]), execution_cache, CachedStateMetrics::zeroed()); + + // Create multiple references to the usage guard + let guard1 = cache.clone_guard_for_test(); + let guard2 = cache.clone_guard_for_test(); + let guard3 = guard1.clone(); + + // Cache should not be available with multiple guards + assert!(!cache.is_available()); + + // Drop guards one by one + drop(guard1); + assert!(!cache.is_available()); // Still not available + + drop(guard2); + assert!(!cache.is_available()); // Still not available + + drop(guard3); + assert!(cache.is_available()); // Now available + } } diff --git a/crates/engine/tree/src/tree/error.rs b/crates/engine/tree/src/tree/error.rs index f5edc3b860f..8589bc59d3d 100644 --- a/crates/engine/tree/src/tree/error.rs +++ b/crates/engine/tree/src/tree/error.rs @@ -4,6 +4,7 @@ use alloy_consensus::BlockHeader; use reth_consensus::ConsensusError; use reth_errors::{BlockExecutionError, BlockValidationError, ProviderError}; use reth_evm::execute::InternalBlockExecutionError; +use reth_payload_primitives::NewPayloadError; use reth_primitives_traits::{Block, BlockBody, SealedBlock}; use tokio::sync::oneshot::error::TryRecvError; @@ -174,3 +175,14 @@ pub enum InsertBlockValidationError { #[error(transparent)] Validation(#[from] BlockValidationError), } + +/// Errors that may occur when inserting a payload. +#[derive(Debug, thiserror::Error)] +pub enum InsertPayloadError { + /// Block validation error + #[error(transparent)] + Block(#[from] InsertBlockError), + /// Payload validation error + #[error(transparent)] + Payload(#[from] NewPayloadError), +} diff --git a/crates/engine/tree/src/tree/instrumented_state.rs b/crates/engine/tree/src/tree/instrumented_state.rs index ab6707972ec..9d96aca3a2e 100644 --- a/crates/engine/tree/src/tree/instrumented_state.rs +++ b/crates/engine/tree/src/tree/instrumented_state.rs @@ -5,8 +5,8 @@ use reth_errors::ProviderResult; use reth_metrics::Metrics; use reth_primitives_traits::{Account, Bytecode}; use reth_provider::{ - AccountReader, BlockHashReader, HashedPostStateProvider, StateProofProvider, StateProvider, - StateRootProvider, StorageRootProvider, + AccountReader, BlockHashReader, BytecodeReader, HashedPostStateProvider, StateProofProvider, + StateProvider, StateRootProvider, StorageRootProvider, }; use reth_trie::{ updates::TrieUpdates, AccountProof, HashedPostState, HashedStorage, MultiProof, @@ -191,7 +191,9 @@ impl StateProvider for InstrumentedStateProvider { self.record_storage_fetch(start.elapsed()); res } +} +impl BytecodeReader for InstrumentedStateProvider { fn bytecode_by_hash(&self, code_hash: &B256) -> ProviderResult> { let start = Instant::now(); let res = self.state_provider.bytecode_by_hash(code_hash); diff --git a/crates/engine/tree/src/tree/invalid_block_hook.rs b/crates/engine/tree/src/tree/invalid_block_hook.rs deleted file mode 100644 index 0670e855342..00000000000 --- a/crates/engine/tree/src/tree/invalid_block_hook.rs +++ /dev/null @@ -1,44 +0,0 @@ -use alloy_primitives::B256; -use reth_engine_primitives::InvalidBlockHook; -use reth_primitives_traits::{NodePrimitives, RecoveredBlock, SealedHeader}; -use reth_provider::BlockExecutionOutput; -use reth_trie::updates::TrieUpdates; - -/// A no-op [`InvalidBlockHook`] that does nothing. -#[derive(Debug, Default)] -#[non_exhaustive] -pub struct NoopInvalidBlockHook; - -impl InvalidBlockHook for NoopInvalidBlockHook { - fn on_invalid_block( - &self, - _parent_header: &SealedHeader, - _block: &RecoveredBlock, - _output: &BlockExecutionOutput, - _trie_updates: Option<(&TrieUpdates, B256)>, - ) { - } -} - -/// Multiple [`InvalidBlockHook`]s that are executed in order. -pub struct InvalidBlockHooks(pub Vec>>); - -impl std::fmt::Debug for InvalidBlockHooks { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("InvalidBlockHooks").field("len", &self.0.len()).finish() - } -} - -impl InvalidBlockHook for InvalidBlockHooks { - fn on_invalid_block( - &self, - parent_header: &SealedHeader, - block: &RecoveredBlock, - output: &BlockExecutionOutput, - trie_updates: Option<(&TrieUpdates, B256)>, - ) { - for hook in &self.0 { - hook.on_invalid_block(parent_header, block, output, trie_updates); - } - } -} diff --git a/crates/engine/tree/src/tree/metrics.rs b/crates/engine/tree/src/tree/metrics.rs index 51ef3eb7568..1d1e208b0a6 100644 --- a/crates/engine/tree/src/tree/metrics.rs +++ b/crates/engine/tree/src/tree/metrics.rs @@ -1,9 +1,22 @@ -use reth_evm::metrics::ExecutorMetrics; +use crate::tree::MeteredStateHook; +use alloy_consensus::transaction::TxHashRef; +use alloy_evm::{ + block::{BlockExecutor, ExecutableTx}, + Evm, +}; +use core::borrow::BorrowMut; +use reth_errors::BlockExecutionError; +use reth_evm::{metrics::ExecutorMetrics, OnStateHook}; +use reth_execution_types::BlockExecutionOutput; use reth_metrics::{ metrics::{Counter, Gauge, Histogram}, Metrics, }; +use reth_primitives_traits::SignedTransaction; use reth_trie::updates::TrieUpdates; +use revm::database::{states::bundle_state::BundleRetention, State}; +use std::time::Instant; +use tracing::{debug_span, trace}; /// Metrics for the `EngineApi`. #[derive(Debug, Default)] @@ -14,14 +27,95 @@ pub(crate) struct EngineApiMetrics { pub(crate) executor: ExecutorMetrics, /// Metrics for block validation pub(crate) block_validation: BlockValidationMetrics, - /// A copy of legacy blockchain tree metrics, to be replaced when we replace the old tree - pub(crate) tree: TreeMetrics, + /// Canonical chain and reorg related metrics + pub tree: TreeMetrics, +} + +impl EngineApiMetrics { + /// Helper function for metered execution + fn metered(&self, f: F) -> R + where + F: FnOnce() -> (u64, R), + { + // Execute the block and record the elapsed time. + let execute_start = Instant::now(); + let (gas_used, output) = f(); + let execution_duration = execute_start.elapsed().as_secs_f64(); + + // Update gas metrics. + self.executor.gas_processed_total.increment(gas_used); + self.executor.gas_per_second.set(gas_used as f64 / execution_duration); + self.executor.gas_used_histogram.record(gas_used as f64); + self.executor.execution_histogram.record(execution_duration); + self.executor.execution_duration.set(execution_duration); + + output + } + + /// Execute the given block using the provided [`BlockExecutor`] and update metrics for the + /// execution. + /// + /// This method updates metrics for execution time, gas usage, and the number + /// of accounts, storage slots and bytecodes loaded and updated. + pub(crate) fn execute_metered( + &self, + executor: E, + transactions: impl Iterator, BlockExecutionError>>, + state_hook: Box, + ) -> Result, BlockExecutionError> + where + DB: alloy_evm::Database, + E: BlockExecutor>>, Transaction: SignedTransaction>, + { + // clone here is cheap, all the metrics are Option>. additionally + // they are globally registered so that the data recorded in the hook will + // be accessible. + let wrapper = MeteredStateHook { metrics: self.executor.clone(), inner_hook: state_hook }; + + let mut executor = executor.with_state_hook(Some(Box::new(wrapper))); + + let f = || { + executor.apply_pre_execution_changes()?; + for tx in transactions { + let tx = tx?; + let span = + debug_span!(target: "engine::tree", "execute tx", tx_hash=?tx.tx().tx_hash()); + let _enter = span.enter(); + trace!(target: "engine::tree", "Executing transaction"); + executor.execute_transaction(tx)?; + } + executor.finish().map(|(evm, result)| (evm.into_db(), result)) + }; + + // Use metered to execute and track timing/gas metrics + let (mut db, result) = self.metered(|| { + let res = f(); + let gas_used = res.as_ref().map(|r| r.1.gas_used).unwrap_or(0); + (gas_used, res) + })?; + + // merge transitions into bundle state + db.borrow_mut().merge_transitions(BundleRetention::Reverts); + let output = BlockExecutionOutput { result, state: db.borrow_mut().take_bundle() }; + + // Update the metrics for the number of accounts, storage slots and bytecodes updated + let accounts = output.state.state.len(); + let storage_slots = + output.state.state.values().map(|account| account.storage.len()).sum::(); + let bytecodes = output.state.contracts.len(); + + self.executor.accounts_updated_histogram.record(accounts as f64); + self.executor.storage_slots_updated_histogram.record(storage_slots as f64); + self.executor.bytecodes_updated_histogram.record(bytecodes as f64); + + Ok(output) + } } /// Metrics for the entire blockchain tree #[derive(Metrics)] #[metrics(scope = "blockchain_tree")] -pub(super) struct TreeMetrics { +pub(crate) struct TreeMetrics { /// The highest block number in the canonical chain pub canonical_chain_height: Gauge, /// The number of reorgs @@ -46,6 +140,10 @@ pub(crate) struct EngineMetrics { pub(crate) pipeline_runs: Counter, /// The total count of forkchoice updated messages received. pub(crate) forkchoice_updated_messages: Counter, + /// The total count of forkchoice updated messages with payload received. + pub(crate) forkchoice_with_attributes_updated_messages: Counter, + /// Newly arriving block hash is not present in executed blocks cache storage + pub(crate) executed_new_block_cache_miss: Counter, /// The total count of new payload messages received. pub(crate) new_payload_messages: Counter, /// Histogram of persistence operation durations (in seconds) @@ -58,7 +156,8 @@ pub(crate) struct EngineMetrics { pub(crate) failed_new_payload_response_deliveries: Counter, /// Tracks the how often we failed to deliver a forkchoice update response. pub(crate) failed_forkchoice_updated_response_deliveries: Counter, - // TODO add latency metrics + /// block insert duration + pub(crate) block_insert_total_duration: Histogram, } /// Metrics for non-execution related block validation. @@ -69,12 +168,22 @@ pub(crate) struct BlockValidationMetrics { pub(crate) state_root_storage_tries_updated_total: Counter, /// Total number of times the parallel state root computation fell back to regular. pub(crate) state_root_parallel_fallback_total: Counter, - /// Histogram of state root duration - pub(crate) state_root_histogram: Histogram, - /// Latest state root duration + /// Latest state root duration, ie the time spent blocked waiting for the state root. pub(crate) state_root_duration: Gauge, + /// Histogram for state root duration ie the time spent blocked waiting for the state root + pub(crate) state_root_histogram: Histogram, /// Trie input computation duration pub(crate) trie_input_duration: Histogram, + /// Payload conversion and validation latency + pub(crate) payload_validation_duration: Gauge, + /// Histogram of payload validation latency + pub(crate) payload_validation_histogram: Histogram, + /// Payload processor spawning duration + pub(crate) spawn_payload_processor: Histogram, + /// Post-execution validation duration + pub(crate) post_execution_validation_duration: Histogram, + /// Total duration of the new payload call + pub(crate) total_duration: Histogram, } impl BlockValidationMetrics { @@ -85,6 +194,13 @@ impl BlockValidationMetrics { self.state_root_duration.set(elapsed_as_secs); self.state_root_histogram.record(elapsed_as_secs); } + + /// Records a new payload validation time, updating both the histogram and the payload + /// validation gauge + pub(crate) fn record_payload_validation(&self, elapsed_as_secs: f64) { + self.payload_validation_duration.set(elapsed_as_secs); + self.payload_validation_histogram.record(elapsed_as_secs); + } } /// Metrics for the blockchain tree block buffer @@ -94,3 +210,233 @@ pub(crate) struct BlockBufferMetrics { /// Total blocks in the block buffer pub blocks: Gauge, } + +#[cfg(test)] +mod tests { + use super::*; + use alloy_eips::eip7685::Requests; + use alloy_evm::block::StateChangeSource; + use alloy_primitives::{B256, U256}; + use metrics_util::debugging::{DebuggingRecorder, Snapshotter}; + use reth_ethereum_primitives::{Receipt, TransactionSigned}; + use reth_evm_ethereum::EthEvm; + use reth_execution_types::BlockExecutionResult; + use reth_primitives_traits::RecoveredBlock; + use revm::{ + context::result::{ExecutionResult, Output, ResultAndState, SuccessReason}, + database::State, + database_interface::EmptyDB, + inspector::NoOpInspector, + state::{Account, AccountInfo, AccountStatus, EvmState, EvmStorage, EvmStorageSlot}, + Context, MainBuilder, MainContext, + }; + use revm_primitives::Bytes; + use std::sync::mpsc; + + /// A simple mock executor for testing that doesn't require complex EVM setup + struct MockExecutor { + state: EvmState, + hook: Option>, + } + + impl MockExecutor { + fn new(state: EvmState) -> Self { + Self { state, hook: None } + } + } + + // Mock Evm type for testing + type MockEvm = EthEvm, NoOpInspector>; + + impl BlockExecutor for MockExecutor { + type Transaction = TransactionSigned; + type Receipt = Receipt; + type Evm = MockEvm; + + fn apply_pre_execution_changes(&mut self) -> Result<(), BlockExecutionError> { + Ok(()) + } + + fn execute_transaction_without_commit( + &mut self, + _tx: impl ExecutableTx, + ) -> Result::HaltReason>, BlockExecutionError> { + // Call hook with our mock state for each transaction + if let Some(hook) = self.hook.as_mut() { + hook.on_state(StateChangeSource::Transaction(0), &self.state); + } + + Ok(ResultAndState::new( + ExecutionResult::Success { + reason: SuccessReason::Return, + gas_used: 1000, // Mock gas used + gas_refunded: 0, + logs: vec![], + output: Output::Call(Bytes::from(vec![])), + }, + Default::default(), + )) + } + + fn commit_transaction( + &mut self, + _output: ResultAndState<::HaltReason>, + _tx: impl ExecutableTx, + ) -> Result { + Ok(1000) + } + + fn finish( + self, + ) -> Result<(Self::Evm, BlockExecutionResult), BlockExecutionError> { + let Self { hook, state, .. } = self; + + // Call hook with our mock state + if let Some(mut hook) = hook { + hook.on_state(StateChangeSource::Transaction(0), &state); + } + + // Create a mock EVM + let db = State::builder() + .with_database(EmptyDB::default()) + .with_bundle_update() + .without_state_clear() + .build(); + let evm = EthEvm::new( + Context::mainnet().with_db(db).build_mainnet_with_inspector(NoOpInspector {}), + false, + ); + + // Return successful result like the original tests + Ok(( + evm, + BlockExecutionResult { + receipts: vec![], + requests: Requests::default(), + gas_used: 1000, + blob_gas_used: 0, + }, + )) + } + + fn set_state_hook(&mut self, hook: Option>) { + self.hook = hook; + } + + fn evm(&self) -> &Self::Evm { + panic!("Mock executor evm() not implemented") + } + + fn evm_mut(&mut self) -> &mut Self::Evm { + panic!("Mock executor evm_mut() not implemented") + } + } + + struct ChannelStateHook { + output: i32, + sender: mpsc::Sender, + } + + impl OnStateHook for ChannelStateHook { + fn on_state(&mut self, _source: StateChangeSource, _state: &EvmState) { + let _ = self.sender.send(self.output); + } + } + + fn setup_test_recorder() -> Snapshotter { + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + recorder.install().unwrap(); + snapshotter + } + + #[test] + fn test_executor_metrics_hook_called() { + let metrics = EngineApiMetrics::default(); + let input = RecoveredBlock::::default(); + + let (tx, rx) = mpsc::channel(); + let expected_output = 42; + let state_hook = Box::new(ChannelStateHook { sender: tx, output: expected_output }); + + let state = EvmState::default(); + let executor = MockExecutor::new(state); + + // This will fail to create the EVM but should still call the hook + let _result = metrics.execute_metered::<_, EmptyDB>( + executor, + input.clone_transactions_recovered().map(Ok::<_, BlockExecutionError>), + state_hook, + ); + + // Check if hook was called (it might not be if finish() fails early) + match rx.try_recv() { + Ok(actual_output) => assert_eq!(actual_output, expected_output), + Err(_) => { + // Hook wasn't called, which is expected if the mock fails early + // The test still validates that the code compiles and runs + } + } + } + + #[test] + fn test_executor_metrics_hook_metrics_recorded() { + let snapshotter = setup_test_recorder(); + let metrics = EngineApiMetrics::default(); + + // Pre-populate some metrics to ensure they exist + metrics.executor.gas_processed_total.increment(0); + metrics.executor.gas_per_second.set(0.0); + metrics.executor.gas_used_histogram.record(0.0); + + let input = RecoveredBlock::::default(); + + let (tx, _rx) = mpsc::channel(); + let state_hook = Box::new(ChannelStateHook { sender: tx, output: 42 }); + + // Create a state with some data + let state = { + let mut state = EvmState::default(); + let storage = + EvmStorage::from_iter([(U256::from(1), EvmStorageSlot::new(U256::from(2), 0))]); + state.insert( + Default::default(), + Account { + info: AccountInfo { + balance: U256::from(100), + nonce: 10, + code_hash: B256::random(), + code: Default::default(), + }, + storage, + status: AccountStatus::default(), + transaction_id: 0, + }, + ); + state + }; + + let executor = MockExecutor::new(state); + + // Execute (will fail but should still update some metrics) + let _result = metrics.execute_metered::<_, EmptyDB>( + executor, + input.clone_transactions_recovered().map(Ok::<_, BlockExecutionError>), + state_hook, + ); + + let snapshot = snapshotter.snapshot().into_vec(); + + // Verify that metrics were registered + let mut found_metrics = false; + for (key, _unit, _desc, _value) in snapshot { + let metric_name = key.key().name(); + if metric_name.starts_with("sync.execution") { + found_metrics = true; + break; + } + } + + assert!(found_metrics, "Expected to find sync.execution metrics"); + } +} diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index d9d705e3a3d..ca8a93df079 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -3,56 +3,41 @@ use crate::{ chain::FromOrchestrator, engine::{DownloadRequest, EngineApiEvent, EngineApiKind, EngineApiRequest, FromEngine}, persistence::PersistenceHandle, - tree::{ - cached_state::CachedStateProvider, executor::WorkloadExecutor, metrics::EngineApiMetrics, - }, + tree::{error::InsertPayloadError, metrics::EngineApiMetrics, payload_validator::TreeCtx}, }; use alloy_consensus::BlockHeader; -use alloy_eips::{merge::EPOCH_SLOTS, BlockNumHash, NumHash}; -use alloy_primitives::{ - map::{HashMap, HashSet}, - BlockNumber, B256, -}; +use alloy_eips::{eip1898::BlockWithParent, merge::EPOCH_SLOTS, BlockNumHash, NumHash}; +use alloy_evm::block::StateChangeSource; +use alloy_primitives::B256; use alloy_rpc_types_engine::{ ForkchoiceState, PayloadStatus, PayloadStatusEnum, PayloadValidationError, }; -use error::{InsertBlockError, InsertBlockErrorKind, InsertBlockFatalError}; -use instrumented_state::InstrumentedStateProvider; -use payload_processor::sparse_trie::StateRootComputeOutcome; -use persistence_state::CurrentPersistenceAction; +use error::{InsertBlockError, InsertBlockFatalError}; use reth_chain_state::{ - CanonicalInMemoryState, ExecutedBlock, ExecutedBlockWithTrieUpdates, - MemoryOverlayStateProvider, NewCanonicalChain, + CanonicalInMemoryState, ExecutedBlock, MemoryOverlayStateProvider, NewCanonicalChain, }; use reth_consensus::{Consensus, FullConsensus}; -pub use reth_engine_primitives::InvalidBlockHook; use reth_engine_primitives::{ - BeaconConsensusEngineEvent, BeaconEngineMessage, BeaconOnNewPayloadError, EngineValidator, - ExecutionPayload, ForkchoiceStateTracker, OnForkChoiceUpdated, + BeaconEngineMessage, BeaconOnNewPayloadError, ConsensusEngineEvent, ExecutionPayload, + ForkchoiceStateTracker, OnForkChoiceUpdated, }; use reth_errors::{ConsensusError, ProviderResult}; -use reth_ethereum_primitives::EthPrimitives; -use reth_evm::ConfigureEvm; +use reth_evm::{ConfigureEvm, OnStateHook}; use reth_payload_builder::PayloadBuilderHandle; -use reth_payload_primitives::{EngineApiMessageVersion, PayloadBuilderAttributes, PayloadTypes}; -use reth_primitives_traits::{ - Block, GotExpected, NodePrimitives, RecoveredBlock, SealedBlock, SealedHeader, +use reth_payload_primitives::{ + BuiltPayload, EngineApiMessageVersion, NewPayloadError, PayloadBuilderAttributes, PayloadTypes, }; +use reth_primitives_traits::{NodePrimitives, RecoveredBlock, SealedBlock, SealedHeader}; use reth_provider::{ - providers::ConsistentDbView, BlockNumReader, BlockReader, DBProvider, DatabaseProviderFactory, - ExecutionOutcome, HashedPostStateProvider, ProviderError, StateCommitmentProvider, - StateProvider, StateProviderBox, StateProviderFactory, StateReader, StateRootProvider, - TransactionVariant, + BlockReader, DatabaseProviderFactory, HashedPostStateProvider, ProviderError, StateProviderBox, + StateProviderFactory, StateReader, TransactionVariant, TrieReader, }; -use reth_revm::{database::StateProviderDatabase, State}; +use reth_revm::database::StateProviderDatabase; use reth_stages_api::ControlFlow; -use reth_trie::{updates::TrieUpdates, HashedPostState, TrieInput}; -use reth_trie_db::{DatabaseHashedPostState, StateCommitment}; -use reth_trie_parallel::root::{ParallelStateRoot, ParallelStateRootError}; +use revm::state::EvmState; +use state::TreeState; use std::{ - collections::{btree_map, hash_map, BTreeMap, VecDeque}, fmt::Debug, - ops::Bound, sync::{ mpsc::{Receiver, RecvError, RecvTimeoutError, Sender}, Arc, @@ -69,23 +54,27 @@ mod block_buffer; mod cached_state; pub mod error; mod instrumented_state; -mod invalid_block_hook; mod invalid_headers; mod metrics; mod payload_processor; +pub mod payload_validator; mod persistence_state; +pub mod precompile_cache; +#[cfg(test)] +mod tests; // TODO(alexey): compare trie updates in `insert_block_inner` #[expect(unused)] mod trie_updates; use crate::tree::error::AdvancePersistenceError; pub use block_buffer::BlockBuffer; -pub use invalid_block_hook::{InvalidBlockHooks, NoopInvalidBlockHook}; pub use invalid_headers::InvalidHeaderCache; pub use payload_processor::*; +pub use payload_validator::{BasicEngineValidator, EngineValidator}; pub use persistence_state::PersistenceState; pub use reth_engine_primitives::TreeConfig; -use reth_evm::execute::BlockExecutionOutput; + +pub mod state; /// The largest gap for which the tree will be used to sync individual blocks by downloading them. /// @@ -98,352 +87,6 @@ use reth_evm::execute::BlockExecutionOutput; /// backfill this gap. pub(crate) const MIN_BLOCKS_FOR_PIPELINE_RUN: u64 = EPOCH_SLOTS; -/// Keeps track of the state of the tree. -/// -/// ## Invariants -/// -/// - This only stores blocks that are connected to the canonical chain. -/// - All executed blocks are valid and have been executed. -#[derive(Debug, Default)] -pub struct TreeState { - /// __All__ unique executed blocks by block hash that are connected to the canonical chain. - /// - /// This includes blocks of all forks. - blocks_by_hash: HashMap>, - /// Executed blocks grouped by their respective block number. - /// - /// This maps unique block number to all known blocks for that height. - /// - /// Note: there can be multiple blocks at the same height due to forks. - blocks_by_number: BTreeMap>>, - /// Map of any parent block hash to its children. - parent_to_child: HashMap>, - /// Map of hash to trie updates for canonical blocks that are persisted but not finalized. - /// - /// Contains the block number for easy removal. - persisted_trie_updates: HashMap)>, - /// Currently tracked canonical head of the chain. - current_canonical_head: BlockNumHash, -} - -impl TreeState { - /// Returns a new, empty tree state that points to the given canonical head. - fn new(current_canonical_head: BlockNumHash) -> Self { - Self { - blocks_by_hash: HashMap::default(), - blocks_by_number: BTreeMap::new(), - current_canonical_head, - parent_to_child: HashMap::default(), - persisted_trie_updates: HashMap::default(), - } - } - - /// Resets the state and points to the given canonical head. - fn reset(&mut self, current_canonical_head: BlockNumHash) { - *self = Self::new(current_canonical_head); - } - - /// Returns the number of executed blocks stored. - fn block_count(&self) -> usize { - self.blocks_by_hash.len() - } - - /// Returns the [`ExecutedBlockWithTrieUpdates`] by hash. - fn executed_block_by_hash(&self, hash: B256) -> Option<&ExecutedBlockWithTrieUpdates> { - self.blocks_by_hash.get(&hash) - } - - /// Returns the block by hash. - fn block_by_hash(&self, hash: B256) -> Option>> { - self.blocks_by_hash.get(&hash).map(|b| Arc::new(b.recovered_block().sealed_block().clone())) - } - - /// Returns all available blocks for the given hash that lead back to the canonical chain, from - /// newest to oldest. And the parent hash of the oldest block that is missing from the buffer. - /// - /// Returns `None` if the block for the given hash is not found. - fn blocks_by_hash(&self, hash: B256) -> Option<(B256, Vec>)> { - let block = self.blocks_by_hash.get(&hash).cloned()?; - let mut parent_hash = block.recovered_block().parent_hash(); - let mut blocks = vec![block]; - while let Some(executed) = self.blocks_by_hash.get(&parent_hash) { - parent_hash = executed.recovered_block().parent_hash(); - blocks.push(executed.clone()); - } - - Some((parent_hash, blocks)) - } - - /// Insert executed block into the state. - fn insert_executed(&mut self, executed: ExecutedBlockWithTrieUpdates) { - let hash = executed.recovered_block().hash(); - let parent_hash = executed.recovered_block().parent_hash(); - let block_number = executed.recovered_block().number(); - - if self.blocks_by_hash.contains_key(&hash) { - return; - } - - self.blocks_by_hash.insert(hash, executed.clone()); - - self.blocks_by_number.entry(block_number).or_default().push(executed); - - self.parent_to_child.entry(parent_hash).or_default().insert(hash); - - for children in self.parent_to_child.values_mut() { - children.retain(|child| self.blocks_by_hash.contains_key(child)); - } - } - - /// Remove single executed block by its hash. - /// - /// ## Returns - /// - /// The removed block and the block hashes of its children. - fn remove_by_hash( - &mut self, - hash: B256, - ) -> Option<(ExecutedBlockWithTrieUpdates, HashSet)> { - let executed = self.blocks_by_hash.remove(&hash)?; - - // Remove this block from collection of children of its parent block. - let parent_entry = self.parent_to_child.entry(executed.recovered_block().parent_hash()); - if let hash_map::Entry::Occupied(mut entry) = parent_entry { - entry.get_mut().remove(&hash); - - if entry.get().is_empty() { - entry.remove(); - } - } - - // Remove point to children of this block. - let children = self.parent_to_child.remove(&hash).unwrap_or_default(); - - // Remove this block from `blocks_by_number`. - let block_number_entry = self.blocks_by_number.entry(executed.recovered_block().number()); - if let btree_map::Entry::Occupied(mut entry) = block_number_entry { - // We have to find the index of the block since it exists in a vec - if let Some(index) = entry.get().iter().position(|b| b.recovered_block().hash() == hash) - { - entry.get_mut().swap_remove(index); - - // If there are no blocks left then remove the entry for this block - if entry.get().is_empty() { - entry.remove(); - } - } - } - - Some((executed, children)) - } - - /// Returns whether or not the hash is part of the canonical chain. - pub(crate) fn is_canonical(&self, hash: B256) -> bool { - let mut current_block = self.current_canonical_head.hash; - if current_block == hash { - return true - } - - while let Some(executed) = self.blocks_by_hash.get(¤t_block) { - current_block = executed.recovered_block().parent_hash(); - if current_block == hash { - return true - } - } - - false - } - - /// Removes canonical blocks below the upper bound, only if the last persisted hash is - /// part of the canonical chain. - pub(crate) fn remove_canonical_until( - &mut self, - upper_bound: BlockNumber, - last_persisted_hash: B256, - ) { - debug!(target: "engine::tree", ?upper_bound, ?last_persisted_hash, "Removing canonical blocks from the tree"); - - // If the last persisted hash is not canonical, then we don't want to remove any canonical - // blocks yet. - if !self.is_canonical(last_persisted_hash) { - return - } - - // First, let's walk back the canonical chain and remove canonical blocks lower than the - // upper bound - let mut current_block = self.current_canonical_head.hash; - while let Some(executed) = self.blocks_by_hash.get(¤t_block) { - current_block = executed.recovered_block().parent_hash(); - if executed.recovered_block().number() <= upper_bound { - debug!(target: "engine::tree", num_hash=?executed.recovered_block().num_hash(), "Attempting to remove block walking back from the head"); - if let Some((removed, _)) = self.remove_by_hash(executed.recovered_block().hash()) { - debug!(target: "engine::tree", num_hash=?removed.recovered_block().num_hash(), "Removed block walking back from the head"); - // finally, move the trie updates - self.persisted_trie_updates.insert( - removed.recovered_block().hash(), - (removed.recovered_block().number(), removed.trie), - ); - } - } - } - debug!(target: "engine::tree", ?upper_bound, ?last_persisted_hash, "Removed canonical blocks from the tree"); - } - - /// Removes all blocks that are below the finalized block, as well as removing non-canonical - /// sidechains that fork from below the finalized block. - pub(crate) fn prune_finalized_sidechains(&mut self, finalized_num_hash: BlockNumHash) { - let BlockNumHash { number: finalized_num, hash: finalized_hash } = finalized_num_hash; - - // We remove disconnected sidechains in three steps: - // * first, remove everything with a block number __below__ the finalized block. - // * next, we populate a vec with parents __at__ the finalized block. - // * finally, we iterate through the vec, removing children until the vec is empty - // (BFS). - - // We _exclude_ the finalized block because we will be dealing with the blocks __at__ - // the finalized block later. - let blocks_to_remove = self - .blocks_by_number - .range((Bound::Unbounded, Bound::Excluded(finalized_num))) - .flat_map(|(_, blocks)| blocks.iter().map(|b| b.recovered_block().hash())) - .collect::>(); - for hash in blocks_to_remove { - if let Some((removed, _)) = self.remove_by_hash(hash) { - debug!(target: "engine::tree", num_hash=?removed.recovered_block().num_hash(), "Removed finalized sidechain block"); - } - } - - // remove trie updates that are below the finalized block - self.persisted_trie_updates.retain(|_, (block_num, _)| *block_num > finalized_num); - - // The only block that should remain at the `finalized` number now, is the finalized - // block, if it exists. - // - // For all other blocks, we first put their children into this vec. - // Then, we will iterate over them, removing them, adding their children, etc, - // until the vec is empty. - let mut blocks_to_remove = self.blocks_by_number.remove(&finalized_num).unwrap_or_default(); - - // re-insert the finalized hash if we removed it - if let Some(position) = - blocks_to_remove.iter().position(|b| b.recovered_block().hash() == finalized_hash) - { - let finalized_block = blocks_to_remove.swap_remove(position); - self.blocks_by_number.insert(finalized_num, vec![finalized_block]); - } - - let mut blocks_to_remove = blocks_to_remove - .into_iter() - .map(|e| e.recovered_block().hash()) - .collect::>(); - while let Some(block) = blocks_to_remove.pop_front() { - if let Some((removed, children)) = self.remove_by_hash(block) { - debug!(target: "engine::tree", num_hash=?removed.recovered_block().num_hash(), "Removed finalized sidechain child block"); - blocks_to_remove.extend(children); - } - } - } - - /// Remove all blocks up to __and including__ the given block number. - /// - /// If a finalized hash is provided, the only non-canonical blocks which will be removed are - /// those which have a fork point at or below the finalized hash. - /// - /// Canonical blocks below the upper bound will still be removed. - /// - /// NOTE: if the finalized block is greater than the upper bound, the only blocks that will be - /// removed are canonical blocks and sidechains that fork below the `upper_bound`. This is the - /// same behavior as if the `finalized_num` were `Some(upper_bound)`. - pub(crate) fn remove_until( - &mut self, - upper_bound: BlockNumHash, - last_persisted_hash: B256, - finalized_num_hash: Option, - ) { - debug!(target: "engine::tree", ?upper_bound, ?finalized_num_hash, "Removing blocks from the tree"); - - // If the finalized num is ahead of the upper bound, and exists, we need to instead ensure - // that the only blocks removed, are canonical blocks less than the upper bound - let finalized_num_hash = finalized_num_hash.map(|mut finalized| { - if upper_bound.number < finalized.number { - finalized = upper_bound; - debug!(target: "engine::tree", ?finalized, "Adjusted upper bound"); - } - finalized - }); - - // We want to do two things: - // * remove canonical blocks that are persisted - // * remove forks whose root are below the finalized block - // We can do this in 2 steps: - // * remove all canonical blocks below the upper bound - // * fetch the number of the finalized hash, removing any sidechains that are __below__ the - // finalized block - self.remove_canonical_until(upper_bound.number, last_persisted_hash); - - // Now, we have removed canonical blocks (assuming the upper bound is above the finalized - // block) and only have sidechains below the finalized block. - if let Some(finalized_num_hash) = finalized_num_hash { - self.prune_finalized_sidechains(finalized_num_hash); - } - } - - /// Determines if the second block is a direct descendant of the first block. - /// - /// If the two blocks are the same, this returns `false`. - fn is_descendant(&self, first: BlockNumHash, second: &N::BlockHeader) -> bool { - // If the second block's parent is the first block's hash, then it is a direct descendant - // and we can return early. - if second.parent_hash() == first.hash { - return true - } - - // If the second block is lower than, or has the same block number, they are not - // descendants. - if second.number() <= first.number { - return false - } - - // iterate through parents of the second until we reach the number - let Some(mut current_block) = self.block_by_hash(second.parent_hash()) else { - // If we can't find its parent in the tree, we can't continue, so return false - return false - }; - - while current_block.number() > first.number + 1 { - let Some(block) = self.block_by_hash(current_block.header().parent_hash()) else { - // If we can't find its parent in the tree, we can't continue, so return false - return false - }; - - current_block = block; - } - - // Now the block numbers should be equal, so we compare hashes. - current_block.parent_hash() == first.hash - } - - /// Updates the canonical head to the given block. - const fn set_canonical_head(&mut self, new_head: BlockNumHash) { - self.current_canonical_head = new_head; - } - - /// Returns the tracked canonical head. - const fn canonical_head(&self) -> &BlockNumHash { - &self.current_canonical_head - } - - /// Returns the block hash of the canonical head. - const fn canonical_block_hash(&self) -> B256 { - self.canonical_head().hash - } - - /// Returns the block number of the canonical head. - const fn canonical_block_number(&self) -> BlockNumber { - self.canonical_head().number - } -} - /// A builder for creating state providers that can be used across threads. #[derive(Clone, Debug)] pub struct StateProviderBuilder { @@ -452,7 +95,7 @@ pub struct StateProviderBuilder { /// The historical block hash to fetch state from. historical: B256, /// The blocks that form the chain from historical to target and are in memory. - overlay: Option>>, + overlay: Option>>, } impl StateProviderBuilder { @@ -461,7 +104,7 @@ impl StateProviderBuilder { pub const fn new( provider_factory: P, historical: B256, - overlay: Option>>, + overlay: Option>>, ) -> Self { Self { provider_factory, historical, overlay } } @@ -469,7 +112,7 @@ impl StateProviderBuilder { impl StateProviderBuilder where - P: BlockReader + StateProviderFactory + StateReader + StateCommitmentProvider + Clone, + P: BlockReader + StateProviderFactory + StateReader + Clone, { /// Creates a new state provider from this builder. pub fn build(&self) -> ProviderResult { @@ -502,11 +145,12 @@ impl EngineApiTreeState { block_buffer_limit: u32, max_invalid_header_cache_length: u32, canonical_block: BlockNumHash, + engine_kind: EngineApiKind, ) -> Self { Self { invalid_headers: InvalidHeaderCache::new(max_invalid_header_cache_length), buffer: BlockBuffer::new(block_buffer_limit), - tree_state: TreeState::new(canonical_block), + tree_state: TreeState::new(canonical_block, engine_kind), forkchoice_state_tracker: ForkchoiceStateTracker::default(), } } @@ -562,6 +206,28 @@ pub enum TreeAction { }, } +/// Wrapper struct that combines metrics and state hook +struct MeteredStateHook { + metrics: reth_evm::metrics::ExecutorMetrics, + inner_hook: Box, +} + +impl OnStateHook for MeteredStateHook { + fn on_state(&mut self, source: StateChangeSource, state: &EvmState) { + // Update the metrics for the number of accounts, storage slots and bytecodes loaded + let accounts = state.keys().len(); + let storage_slots = state.values().map(|account| account.storage.len()).sum::(); + let bytecodes = state.values().filter(|account| !account.info.is_empty_code_hash()).count(); + + self.metrics.accounts_loaded_histogram.record(accounts as f64); + self.metrics.storage_slots_loaded_histogram.record(storage_slots as f64); + self.metrics.bytecodes_loaded_histogram.record(bytecodes as f64); + + // Call the original state hook + self.inner_hook.on_state(source, state); + } +} + /// The engine API tree handler implementation. /// /// This type is responsible for processing engine API requests, maintaining the canonical state and @@ -570,6 +236,7 @@ pub struct EngineApiTreeHandler where N: NodePrimitives, T: PayloadTypes, + C: ConfigureEvm + 'static, { provider: P, consensus: Arc>, @@ -605,20 +272,17 @@ where config: TreeConfig, /// Metrics for the engine api. metrics: EngineApiMetrics, - /// An invalid block hook. - invalid_block_hook: Box>, /// The engine API variant of this handler engine_kind: EngineApiKind, - /// The type responsible for processing new payloads - payload_processor: PayloadProcessor, /// The EVM configuration. evm_config: C, } -impl std::fmt::Debug +impl std::fmt::Debug for EngineApiTreeHandler where N: NodePrimitives, + C: Debug + ConfigureEvm, { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("EngineApiTreeHandler") @@ -634,8 +298,8 @@ where .field("payload_builder", &self.payload_builder) .field("config", &self.config) .field("metrics", &self.metrics) - .field("invalid_block_hook", &format!("{:p}", self.invalid_block_hook)) .field("engine_kind", &self.engine_kind) + .field("evm_config", &self.evm_config) .finish() } } @@ -647,15 +311,15 @@ where + BlockReader + StateProviderFactory + StateReader - + StateCommitmentProvider + HashedPostStateProvider + + TrieReader + Clone + 'static,

::Provider: BlockReader, C: ConfigureEvm + 'static, - T: PayloadTypes, - V: EngineValidator, + T: PayloadTypes>, + V: EngineValidator, { /// Creates a new [`EngineApiTreeHandler`]. #[expect(clippy::too_many_arguments)] @@ -675,9 +339,6 @@ where ) -> Self { let (incoming_tx, incoming) = std::sync::mpsc::channel(); - let payload_processor = - PayloadProcessor::new(WorkloadExecutor::default(), evm_config.clone(), &config); - Self { provider, consensus, @@ -693,18 +354,11 @@ where config, metrics: Default::default(), incoming_tx, - invalid_block_hook: Box::new(NoopInvalidBlockHook), engine_kind, - payload_processor, evm_config, } } - /// Sets the invalid block hook. - fn set_invalid_block_hook(&mut self, invalid_block_hook: Box>) { - self.invalid_block_hook = invalid_block_hook; - } - /// Creates a new [`EngineApiTreeHandler`] instance and spawns it in its /// own thread. /// @@ -719,7 +373,6 @@ where payload_builder: PayloadBuilderHandle, canonical_in_memory_state: CanonicalInMemoryState, config: TreeConfig, - invalid_block_hook: Box>, kind: EngineApiKind, evm_config: C, ) -> (Sender, N::Block>>, UnboundedReceiver>) @@ -737,9 +390,10 @@ where config.block_buffer_limit(), config.max_invalid_header_cache_length(), header.num_hash(), + kind, ); - let mut task = Self::new( + let task = Self::new( provider, consensus, payload_validator, @@ -753,9 +407,8 @@ where kind, evm_config, ); - task.set_invalid_block_hook(invalid_block_hook); let incoming = task.incoming_tx.clone(); - std::thread::Builder::new().name("Tree Task".to_string()).spawn(|| task.run()).unwrap(); + std::thread::Builder::new().name("Engine Task".to_string()).spawn(|| task.run()).unwrap(); (incoming, outgoing) } @@ -830,7 +483,8 @@ where /// When the Consensus layer receives a new block via the consensus gossip protocol, /// the transactions in the block are sent to the execution layer in the form of a - /// [`PayloadTypes::ExecutionData`](reth_payload_primitives::PayloadTypes::ExecutionData). The + /// [`PayloadTypes::ExecutionData`], for example + /// [`ExecutionData`](reth_payload_primitives::PayloadTypes::ExecutionData). The /// Execution layer executes the transactions and validates the state in the block header, /// then passes validation data back to Consensus layer, that adds the block to the head of /// its own blockchain and attests to it. The block is then broadcast over the consensus p2p @@ -841,7 +495,12 @@ where /// /// This returns a [`PayloadStatus`] that represents the outcome of a processed new payload and /// returns an error if an internal error occurred. - #[instrument(level = "trace", skip_all, fields(block_hash = %payload.block_hash(), block_num = %payload.block_number(),), target = "engine::tree")] + #[instrument( + level = "debug", + target = "engine::tree", + skip_all, + fields(block_hash = %payload.block_hash(), block_num = %payload.block_number()), + )] fn on_new_payload( &mut self, payload: T::ExecutionData, @@ -849,6 +508,9 @@ where trace!(target: "engine::tree", "invoked new payload"); self.metrics.engine.new_payload_messages.increment(1); + // start timing for the new payload process + let start = Instant::now(); + // Ensures that the given payload does not violate any consensus rules that concern the // block's layout, like: // - missing or invalid base fee @@ -874,77 +536,32 @@ where // null}` if the expected and the actual arrays don't match. // // This validation **MUST** be instantly run in all cases even during active sync process. - let parent_hash = payload.parent_hash(); - let block = match self.payload_validator.ensure_well_formed_payload(payload) { - Ok(block) => block, - Err(error) => { - error!(target: "engine::tree", %error, "Invalid payload"); - // we need to convert the error to a payload status (response to the CL) - - let latest_valid_hash = - if error.is_block_hash_mismatch() || error.is_invalid_versioned_hashes() { - // Engine-API rules: - // > `latestValidHash: null` if the blockHash validation has failed () - // > `latestValidHash: null` if the expected and the actual arrays don't match () - None - } else { - self.latest_valid_hash_for_invalid_payload(parent_hash)? - }; - let status = PayloadStatusEnum::from(error); - return Ok(TreeOutcome::new(PayloadStatus::new(status, latest_valid_hash))) - } - }; + let num_hash = payload.num_hash(); + let engine_event = ConsensusEngineEvent::BlockReceived(num_hash); + self.emit_event(EngineApiEvent::BeaconConsensus(engine_event)); - let block_hash = block.hash(); - let mut lowest_buffered_ancestor = self.lowest_buffered_ancestor_or(block_hash); - if lowest_buffered_ancestor == block_hash { - lowest_buffered_ancestor = block.parent_hash(); - } + let block_hash = num_hash.hash; - // now check the block itself - if let Some(status) = - self.check_invalid_ancestor_with_head(lowest_buffered_ancestor, &block)? - { - return Ok(TreeOutcome::new(status)) + // Check for invalid ancestors + if let Some(invalid) = self.find_invalid_ancestor(&payload) { + let status = self.handle_invalid_ancestor_payload(payload, invalid)?; + return Ok(TreeOutcome::new(status)); } - let status = if self.backfill_sync_state.is_idle() { - let mut latest_valid_hash = None; - let num_hash = block.num_hash(); - match self.insert_block(block) { - Ok(status) => { - let status = match status { - InsertPayloadOk::Inserted(BlockStatus::Valid) => { - latest_valid_hash = Some(block_hash); - self.try_connect_buffered_blocks(num_hash)?; - PayloadStatusEnum::Valid - } - InsertPayloadOk::AlreadySeen(BlockStatus::Valid) => { - latest_valid_hash = Some(block_hash); - PayloadStatusEnum::Valid - } - InsertPayloadOk::Inserted(BlockStatus::Disconnected { .. }) | - InsertPayloadOk::AlreadySeen(BlockStatus::Disconnected { .. }) => { - // not known to be invalid, but we don't know anything else - PayloadStatusEnum::Syncing - } - }; + // record pre-execution phase duration + self.metrics.block_validation.record_payload_validation(start.elapsed().as_secs_f64()); - PayloadStatus::new(status, latest_valid_hash) - } - Err(error) => self.on_insert_block_error(error)?, - } - } else if let Err(error) = self.buffer_block(block) { - self.on_insert_block_error(error)? + let status = if self.backfill_sync_state.is_idle() { + self.try_insert_payload(payload)? } else { - PayloadStatus::from_status(PayloadStatusEnum::Syncing) + self.try_buffer_payload(payload)? }; let mut outcome = TreeOutcome::new(status); // if the block is valid and it is the current sync target head, make it canonical if outcome.outcome.is_valid() && self.is_sync_target_head(block_hash) { - // but only if it isn't already the canonical head + // Only create the canonical event if this block isn't already the canonical head if self.state.tree_state.canonical_block_hash() != block_hash { outcome = outcome.with_event(TreeEvent::TreeAction(TreeAction::MakeCanonical { sync_target_head: block_hash, @@ -952,9 +569,85 @@ where } } + // record total newPayload duration + self.metrics.block_validation.total_duration.record(start.elapsed().as_secs_f64()); + Ok(outcome) } + /// Processes a payload during normal sync operation. + /// + /// Returns: + /// - `Valid`: Payload successfully validated and inserted + /// - `Syncing`: Parent missing, payload buffered for later + /// - Error status: Payload is invalid + #[instrument(level = "debug", target = "engine::tree", skip_all)] + fn try_insert_payload( + &mut self, + payload: T::ExecutionData, + ) -> Result { + let block_hash = payload.block_hash(); + let num_hash = payload.num_hash(); + let parent_hash = payload.parent_hash(); + let mut latest_valid_hash = None; + + match self.insert_payload(payload) { + Ok(status) => { + let status = match status { + InsertPayloadOk::Inserted(BlockStatus::Valid) => { + latest_valid_hash = Some(block_hash); + self.try_connect_buffered_blocks(num_hash)?; + PayloadStatusEnum::Valid + } + InsertPayloadOk::AlreadySeen(BlockStatus::Valid) => { + latest_valid_hash = Some(block_hash); + PayloadStatusEnum::Valid + } + InsertPayloadOk::Inserted(BlockStatus::Disconnected { .. }) | + InsertPayloadOk::AlreadySeen(BlockStatus::Disconnected { .. }) => { + // not known to be invalid, but we don't know anything else + PayloadStatusEnum::Syncing + } + }; + + Ok(PayloadStatus::new(status, latest_valid_hash)) + } + Err(error) => match error { + InsertPayloadError::Block(error) => Ok(self.on_insert_block_error(error)?), + InsertPayloadError::Payload(error) => { + Ok(self.on_new_payload_error(error, parent_hash)?) + } + }, + } + } + + /// Stores a payload for later processing during backfill sync. + /// + /// During backfill, the node lacks the state needed to validate payloads, + /// so they are buffered (stored in memory) until their parent blocks are synced. + /// + /// Returns: + /// - `Syncing`: Payload successfully buffered + /// - Error status: Payload is malformed or invalid + fn try_buffer_payload( + &mut self, + payload: T::ExecutionData, + ) -> Result { + let parent_hash = payload.parent_hash(); + + match self.payload_validator.ensure_well_formed_payload(payload) { + // if the block is well-formed, buffer it for later + Ok(block) => { + if let Err(error) = self.buffer_block(block) { + Ok(self.on_insert_block_error(error)?) + } else { + Ok(PayloadStatus::from_status(PayloadStatusEnum::Syncing)) + } + } + Err(error) => Ok(self.on_new_payload_error(error, parent_hash)?), + } + } + /// Returns the new chain for the given head. /// /// This also handles reorgs. @@ -964,6 +657,8 @@ where fn on_new_head(&self, new_head: B256) -> ProviderResult>> { // get the executed new head block let Some(new_head_block) = self.state.tree_state.blocks_by_hash.get(&new_head) else { + debug!(target: "engine::tree", new_head=?new_head, "New head block not found in inmemory tree state"); + self.metrics.engine.executed_new_block_cache_miss.increment(1); return Ok(None) }; @@ -988,7 +683,7 @@ where warn!(target: "engine::tree", current_hash=?current_hash, "Sidechain block not found in TreeState"); // This should never happen as we're walking back a chain that should connect to // the canonical chain - return Ok(None); + return Ok(None) } } @@ -998,7 +693,7 @@ where new_chain.reverse(); // Simple extension of the current chain - return Ok(Some(NewCanonicalChain::Commit { new: new_chain })); + return Ok(Some(NewCanonicalChain::Commit { new: new_chain })) } // We have a reorg. Walk back both chains to find the fork point. @@ -1015,7 +710,7 @@ where } else { // This shouldn't happen as we're walking back the canonical chain warn!(target: "engine::tree", current_hash=?old_hash, "Canonical block not found in TreeState"); - return Ok(None); + return Ok(None) } } @@ -1031,7 +726,7 @@ where } else { // This shouldn't happen as we're walking back the canonical chain warn!(target: "engine::tree", current_hash=?old_hash, "Canonical block not found in TreeState"); - return Ok(None); + return Ok(None) } if let Some(block) = self.state.tree_state.executed_block_by_hash(current_hash).cloned() @@ -1041,7 +736,7 @@ where } else { // This shouldn't happen as we've already walked this path warn!(target: "engine::tree", invalid_hash=?current_hash, "New chain block not found in TreeState"); - return Ok(None); + return Ok(None) } } new_chain.reverse(); @@ -1050,59 +745,226 @@ where Ok(Some(NewCanonicalChain::Reorg { new: new_chain, old: old_chain })) } - /// Determines if the given block is part of a fork by checking that these - /// conditions are true: - /// * walking back from the target hash to verify that the target hash is not part of an - /// extension of the canonical chain. - /// * walking back from the current head to verify that the target hash is not already part of - /// the canonical chain. - fn is_fork(&self, target_hash: B256) -> ProviderResult { - // verify that the given hash is not part of an extension of the canon chain. - let canonical_head = self.state.tree_state.canonical_head(); - let mut current_hash = target_hash; - while let Some(current_block) = self.sealed_header_by_hash(current_hash)? { - if current_block.hash() == canonical_head.hash { - return Ok(false) - } - // We already passed the canonical head - if current_block.number() <= canonical_head.number { - break + /// Updates the latest block state to the specified canonical ancestor. + /// + /// This method ensures that the latest block tracks the given canonical header by resetting + /// + /// # Arguments + /// * `canonical_header` - The canonical header to set as the new head + /// + /// # Returns + /// * `ProviderResult<()>` - Ok(()) on success, error if state update fails + /// + /// Caution: This unwinds the canonical chain + fn update_latest_block_to_canonical_ancestor( + &mut self, + canonical_header: &SealedHeader, + ) -> ProviderResult<()> { + debug!(target: "engine::tree", head = ?canonical_header.num_hash(), "Update latest block to canonical ancestor"); + let current_head_number = self.state.tree_state.canonical_block_number(); + let new_head_number = canonical_header.number(); + let new_head_hash = canonical_header.hash(); + + // Update tree state with the new canonical head + self.state.tree_state.set_canonical_head(canonical_header.num_hash()); + + // Handle the state update based on whether this is an unwind scenario + if new_head_number < current_head_number { + debug!( + target: "engine::tree", + current_head = current_head_number, + new_head = new_head_number, + new_head_hash = ?new_head_hash, + "FCU unwind detected: reverting to canonical ancestor" + ); + + self.handle_canonical_chain_unwind(current_head_number, canonical_header) + } else { + debug!( + target: "engine::tree", + previous_head = current_head_number, + new_head = new_head_number, + new_head_hash = ?new_head_hash, + "Advancing latest block to canonical ancestor" + ); + self.handle_chain_advance_or_same_height(canonical_header) + } + } + + /// Handles chain unwind scenarios by collecting blocks to remove and performing an unwind back + /// to the canonical header + fn handle_canonical_chain_unwind( + &self, + current_head_number: u64, + canonical_header: &SealedHeader, + ) -> ProviderResult<()> { + let new_head_number = canonical_header.number(); + debug!( + target: "engine::tree", + from = current_head_number, + to = new_head_number, + "Handling unwind: collecting blocks to remove from in-memory state" + ); + + // Collect blocks that need to be removed from memory + let old_blocks = + self.collect_blocks_for_canonical_unwind(new_head_number, current_head_number); + + // Load and apply the canonical ancestor block + self.apply_canonical_ancestor_via_reorg(canonical_header, old_blocks) + } + + /// Collects blocks from memory that need to be removed during an unwind to a canonical block. + fn collect_blocks_for_canonical_unwind( + &self, + new_head_number: u64, + current_head_number: u64, + ) -> Vec> { + let mut old_blocks = Vec::new(); + + for block_num in (new_head_number + 1)..=current_head_number { + if let Some(block_state) = self.canonical_in_memory_state.state_by_number(block_num) { + let executed_block = block_state.block_ref().clone(); + old_blocks.push(executed_block); + debug!( + target: "engine::tree", + block_number = block_num, + "Collected block for removal from in-memory state" + ); } - current_hash = current_block.parent_hash(); } - // verify that the given hash is not already part of canonical chain stored in memory - if self.canonical_in_memory_state.header_by_hash(target_hash).is_some() { - return Ok(false) + if old_blocks.is_empty() { + debug!( + target: "engine::tree", + "No blocks found in memory to remove, will clear and reset state" + ); } - // verify that the given hash is not already part of persisted canonical chain - if self.provider.block_number(target_hash)?.is_some() { - return Ok(false) + old_blocks + } + + /// Applies the canonical ancestor block via a reorg operation. + fn apply_canonical_ancestor_via_reorg( + &self, + canonical_header: &SealedHeader, + old_blocks: Vec>, + ) -> ProviderResult<()> { + let new_head_hash = canonical_header.hash(); + let new_head_number = canonical_header.number(); + + // Try to load the canonical ancestor's block + match self.canonical_block_by_hash(new_head_hash)? { + Some(executed_block) => { + // Perform the reorg to properly handle the unwind + self.canonical_in_memory_state.update_chain(NewCanonicalChain::Reorg { + new: vec![executed_block], + old: old_blocks, + }); + + // CRITICAL: Update the canonical head after the reorg + // This ensures get_canonical_head() returns the correct block + self.canonical_in_memory_state.set_canonical_head(canonical_header.clone()); + + debug!( + target: "engine::tree", + block_number = new_head_number, + block_hash = ?new_head_hash, + "Successfully loaded canonical ancestor into memory via reorg" + ); + } + None => { + // Fallback: update header only if block cannot be found + warn!( + target: "engine::tree", + block_hash = ?new_head_hash, + "Could not find canonical ancestor block, updating header only" + ); + self.canonical_in_memory_state.set_canonical_head(canonical_header.clone()); + } } - Ok(true) + Ok(()) } - /// Returns the persisting kind for the input block. - fn persisting_kind_for(&self, block: &N::BlockHeader) -> PersistingKind { - // Check that we're currently persisting. - let Some(action) = self.persistence_state.current_action() else { - return PersistingKind::NotPersisting - }; - // Check that the persistince action is saving blocks, not removing them. - let CurrentPersistenceAction::SavingBlocks { highest } = action else { - return PersistingKind::PersistingNotDescendant - }; + /// Handles chain advance or same height scenarios. + fn handle_chain_advance_or_same_height( + &self, + canonical_header: &SealedHeader, + ) -> ProviderResult<()> { + let new_head_number = canonical_header.number(); + let new_head_hash = canonical_header.hash(); - // The block being validated can only be a descendant if its number is higher than - // the highest block persisting. Otherwise, it's likely a fork of a lower block. - if block.number() > highest.number && self.state.tree_state.is_descendant(*highest, block) { - return PersistingKind::PersistingDescendant - } + // Update the canonical head header + self.canonical_in_memory_state.set_canonical_head(canonical_header.clone()); - // In all other cases, the block is not a descendant. - PersistingKind::PersistingNotDescendant + // Load the block into memory if it's not already present + self.ensure_block_in_memory(new_head_number, new_head_hash) + } + + /// Ensures a block is loaded into memory if not already present. + fn ensure_block_in_memory(&self, block_number: u64, block_hash: B256) -> ProviderResult<()> { + // Check if block is already in memory + if self.canonical_in_memory_state.state_by_number(block_number).is_some() { + return Ok(()); + } + + // Try to load the block from storage + if let Some(executed_block) = self.canonical_block_by_hash(block_hash)? { + self.canonical_in_memory_state + .update_chain(NewCanonicalChain::Commit { new: vec![executed_block] }); + + debug!( + target: "engine::tree", + block_number, + block_hash = ?block_hash, + "Added canonical block to in-memory state" + ); + } + + Ok(()) + } + + /// Determines if the given block is part of a fork by checking that these + /// conditions are true: + /// * walking back from the target hash to verify that the target hash is not part of an + /// extension of the canonical chain. + /// * walking back from the current head to verify that the target hash is not already part of + /// the canonical chain. + /// + /// The header is required as an arg, because we might be checking that the header is a fork + /// block before it's in the tree state and before it's in the database. + fn is_fork(&self, target: BlockWithParent) -> ProviderResult { + let target_hash = target.block.hash; + // verify that the given hash is not part of an extension of the canon chain. + let canonical_head = self.state.tree_state.canonical_head(); + let mut current_hash; + let mut current_block = target; + loop { + if current_block.block.hash == canonical_head.hash { + return Ok(false) + } + // We already passed the canonical head + if current_block.block.number <= canonical_head.number { + break + } + current_hash = current_block.parent; + + let Some(next_block) = self.sealed_header_by_hash(current_hash)? else { break }; + current_block = next_block.block_with_parent(); + } + + // verify that the given hash is not already part of canonical chain stored in memory + if self.canonical_in_memory_state.header_by_hash(target_hash).is_some() { + return Ok(false) + } + + // verify that the given hash is not already part of persisted canonical chain + if self.provider.block_number(target_hash)?.is_some() { + return Ok(false) + } + + Ok(true) } /// Invoked when we receive a new forkchoice update message. Calls into the blockchain tree @@ -1113,7 +975,7 @@ where /// `engine_forkchoiceUpdated`](https://github.com/ethereum/execution-apis/blob/main/src/engine/paris.md#specification-1). /// /// Returns an error if an internal error occurred like a database error. - #[instrument(level = "trace", skip_all, fields(head = % state.head_block_hash, safe = % state.safe_block_hash,finalized = % state.finalized_block_hash), target = "engine::tree")] + #[instrument(level = "debug", target = "engine::tree", skip_all, fields(head = % state.head_block_hash, safe = % state.safe_block_hash,finalized = % state.finalized_block_hash))] fn on_forkchoice_updated( &mut self, state: ForkchoiceState, @@ -1121,20 +983,79 @@ where version: EngineApiMessageVersion, ) -> ProviderResult> { trace!(target: "engine::tree", ?attrs, "invoked forkchoice update"); + + // Record metrics + self.record_forkchoice_metrics(&attrs); + + // Pre-validation of forkchoice state + if let Some(early_result) = self.validate_forkchoice_state(state)? { + return Ok(TreeOutcome::new(early_result)); + } + + // Return early if we are on the correct fork + if let Some(result) = self.handle_canonical_head(state, &attrs, version)? { + return Ok(result); + } + + // Attempt to apply a chain update when the head differs from our canonical chain. + // This handles reorgs and chain extensions by making the specified head canonical. + if let Some(result) = self.apply_chain_update(state, &attrs, version)? { + return Ok(result); + } + + // Fallback that ensures to catch up to the network's state. + self.handle_missing_block(state) + } + + /// Records metrics for forkchoice updated calls + fn record_forkchoice_metrics(&self, attrs: &Option) { self.metrics.engine.forkchoice_updated_messages.increment(1); + if attrs.is_some() { + self.metrics.engine.forkchoice_with_attributes_updated_messages.increment(1); + } self.canonical_in_memory_state.on_forkchoice_update_received(); + } - if let Some(on_updated) = self.pre_validate_forkchoice_update(state)? { - return Ok(TreeOutcome::new(on_updated)) + /// Pre-validates the forkchoice state and returns early if validation fails. + /// + /// Returns `Some(OnForkChoiceUpdated)` if validation fails and an early response should be + /// returned. Returns `None` if validation passes and processing should continue. + fn validate_forkchoice_state( + &mut self, + state: ForkchoiceState, + ) -> ProviderResult> { + if state.head_block_hash.is_zero() { + return Ok(Some(OnForkChoiceUpdated::invalid_state())); } - let valid_outcome = |head| { - TreeOutcome::new(OnForkChoiceUpdated::valid(PayloadStatus::new( - PayloadStatusEnum::Valid, - Some(head), - ))) - }; + // Check if the new head hash is connected to any ancestor that we previously marked as + // invalid + let lowest_buffered_ancestor_fcu = self.lowest_buffered_ancestor_or(state.head_block_hash); + if let Some(status) = self.check_invalid_ancestor(lowest_buffered_ancestor_fcu)? { + return Ok(Some(OnForkChoiceUpdated::with_invalid(status))); + } + + if !self.backfill_sync_state.is_idle() { + // We can only process new forkchoice updates if the pipeline is idle, since it requires + // exclusive access to the database + trace!(target: "engine::tree", "Pipeline is syncing, skipping forkchoice update"); + return Ok(Some(OnForkChoiceUpdated::syncing())); + } + + Ok(None) + } + /// Handles the case where the forkchoice head is already canonical. + /// + /// Returns `Some(TreeOutcome)` if the head is already canonical and + /// processing is complete. Returns `None` if the head is not canonical and processing + /// should continue. + fn handle_canonical_head( + &self, + state: ForkchoiceState, + attrs: &Option, // Changed to reference + version: EngineApiMessageVersion, + ) -> ProviderResult>> { // Process the forkchoice update by trying to make the head block canonical // // We can only process this forkchoice update if: @@ -1149,89 +1070,153 @@ where // - emitting a canonicalization event for the new chain (including reorg) // - if we have payload attributes, delegate them to the payload service - // 1. ensure we have a new head block - if self.state.tree_state.canonical_block_hash() == state.head_block_hash { - trace!(target: "engine::tree", "fcu head hash is already canonical"); + if self.state.tree_state.canonical_block_hash() != state.head_block_hash { + return Ok(None); + } - // update the safe and finalized blocks and ensure their values are valid - if let Err(outcome) = self.ensure_consistent_forkchoice_state(state) { - // safe or finalized hashes are invalid - return Ok(TreeOutcome::new(outcome)) - } + trace!(target: "engine::tree", "fcu head hash is already canonical"); - // we still need to process payload attributes if the head is already canonical - if let Some(attr) = attrs { - let tip = self - .block_by_hash(self.state.tree_state.canonical_block_hash())? - .ok_or_else(|| { - // If we can't find the canonical block, then something is wrong and we need - // to return an error - ProviderError::HeaderNotFound(state.head_block_hash.into()) - })?; - let updated = self.process_payload_attributes(attr, tip.header(), state, version); - return Ok(TreeOutcome::new(updated)) - } + // Update the safe and finalized blocks and ensure their values are valid + if let Err(outcome) = self.ensure_consistent_forkchoice_state(state) { + // safe or finalized hashes are invalid + return Ok(Some(TreeOutcome::new(outcome))); + } - // the head block is already canonical - return Ok(valid_outcome(state.head_block_hash)) + // Process payload attributes if the head is already canonical + if let Some(attr) = attrs { + let tip = self + .sealed_header_by_hash(self.state.tree_state.canonical_block_hash())? + .ok_or_else(|| { + // If we can't find the canonical block, then something is wrong and we need + // to return an error + ProviderError::HeaderNotFound(state.head_block_hash.into()) + })?; + // Clone only when we actually need to process the attributes + let updated = self.process_payload_attributes(attr.clone(), &tip, state, version); + return Ok(Some(TreeOutcome::new(updated))); } - // 2. check if the head is already part of the canonical chain + // The head block is already canonical + let outcome = TreeOutcome::new(OnForkChoiceUpdated::valid(PayloadStatus::new( + PayloadStatusEnum::Valid, + Some(state.head_block_hash), + ))); + Ok(Some(outcome)) + } + + /// Applies chain update for the new head block and processes payload attributes. + /// + /// This method handles the case where the forkchoice head differs from our current canonical + /// head. It attempts to make the specified head block canonical by: + /// - Checking if the head is already part of the canonical chain + /// - Applying chain reorganizations (reorgs) if necessary + /// - Processing payload attributes if provided + /// - Returning the appropriate forkchoice update response + /// + /// Returns `Some(TreeOutcome)` if a chain update was successfully applied. + /// Returns `None` if no chain update was needed or possible. + fn apply_chain_update( + &mut self, + state: ForkchoiceState, + attrs: &Option, + version: EngineApiMessageVersion, + ) -> ProviderResult>> { + // Check if the head is already part of the canonical chain if let Ok(Some(canonical_header)) = self.find_canonical_header(state.head_block_hash) { debug!(target: "engine::tree", head = canonical_header.number(), "fcu head block is already canonical"); - // For OpStack the proposers are allowed to reorg their own chain at will, so we need to - // always trigger a new payload job if requested. - if self.engine_kind.is_opstack() { + // For OpStack, or if explicitly configured, the proposers are allowed to reorg their + // own chain at will, so we need to always trigger a new payload job if requested. + if self.engine_kind.is_opstack() || + self.config.always_process_payload_attributes_on_canonical_head() + { if let Some(attr) = attrs { debug!(target: "engine::tree", head = canonical_header.number(), "handling payload attributes for canonical head"); - let updated = - self.process_payload_attributes(attr, &canonical_header, state, version); - return Ok(TreeOutcome::new(updated)) + // Clone only when we actually need to process the attributes + let updated = self.process_payload_attributes( + attr.clone(), + &canonical_header, + state, + version, + ); + return Ok(Some(TreeOutcome::new(updated))); + } + + // At this point, no alternative block has been triggered, so we need effectively + // unwind the _canonical_ chain to the FCU's head, which is part of the canonical + // chain. We need to update the latest block state to reflect the + // canonical ancestor. This ensures that state providers and the + // transaction pool operate with the correct chain state after + // forkchoice update processing. + + if self.config.unwind_canonical_header() { + self.update_latest_block_to_canonical_ancestor(&canonical_header)?; } } - // 2. Client software MAY skip an update of the forkchoice state and MUST NOT begin a - // payload build process if `forkchoiceState.headBlockHash` references a `VALID` - // ancestor of the head of canonical chain, i.e. the ancestor passed payload - // validation process and deemed `VALID`. In the case of such an event, client - // software MUST return `{payloadStatus: {status: VALID, latestValidHash: - // forkchoiceState.headBlockHash, validationError: null}, payloadId: null}` + // According to the Engine API specification, client software MAY skip an update of the + // forkchoice state and MUST NOT begin a payload build process if + // `forkchoiceState.headBlockHash` references a `VALID` ancestor of the head + // of canonical chain, i.e. the ancestor passed payload validation process + // and deemed `VALID`. In the case of such an event, client software MUST + // return `{payloadStatus: {status: VALID, latestValidHash: + // forkchoiceState.headBlockHash, validationError: null}, payloadId: null}` + + // The head block is already canonical and we're not processing payload attributes, + // so we're not triggering a payload job and can return right away - // the head block is already canonical, so we're not triggering a payload job and can - // return right away - return Ok(valid_outcome(state.head_block_hash)) + let outcome = TreeOutcome::new(OnForkChoiceUpdated::valid(PayloadStatus::new( + PayloadStatusEnum::Valid, + Some(state.head_block_hash), + ))); + return Ok(Some(outcome)); } - // 3. ensure we can apply a new chain update for the head block + // Ensure we can apply a new chain update for the head block if let Some(chain_update) = self.on_new_head(state.head_block_hash)? { let tip = chain_update.tip().clone_sealed_header(); self.on_canonical_chain_update(chain_update); - // update the safe and finalized blocks and ensure their values are valid + // Update the safe and finalized blocks and ensure their values are valid if let Err(outcome) = self.ensure_consistent_forkchoice_state(state) { // safe or finalized hashes are invalid - return Ok(TreeOutcome::new(outcome)) + return Ok(Some(TreeOutcome::new(outcome))); } if let Some(attr) = attrs { - let updated = self.process_payload_attributes(attr, &tip, state, version); - return Ok(TreeOutcome::new(updated)) + // Clone only when we actually need to process the attributes + let updated = self.process_payload_attributes(attr.clone(), &tip, state, version); + return Ok(Some(TreeOutcome::new(updated))); } - return Ok(valid_outcome(state.head_block_hash)) + let outcome = TreeOutcome::new(OnForkChoiceUpdated::valid(PayloadStatus::new( + PayloadStatusEnum::Valid, + Some(state.head_block_hash), + ))); + return Ok(Some(outcome)); } - // 4. we don't have the block to perform the update - // we assume the FCU is valid and at least the head is missing, + Ok(None) + } + + /// Handles the case where the head block is missing and needs to be downloaded. + /// + /// This is the fallback case when all other forkchoice update scenarios have been exhausted. + /// Returns a `TreeOutcome` with syncing status and download event. + fn handle_missing_block( + &self, + state: ForkchoiceState, + ) -> ProviderResult> { + // We don't have the block to perform the forkchoice update + // We assume the FCU is valid and at least the head is missing, // so we need to start syncing to it // // find the appropriate target to sync to, if we don't have the safe block hash then we // start syncing to the safe block via backfill first let target = if self.state.forkchoice_state_tracker.is_empty() && - // check that safe block is valid and missing - !state.safe_block_hash.is_zero() && - self.find_canonical_header(state.safe_block_hash).ok().flatten().is_none() + // check that safe block is valid and missing + !state.safe_block_hash.is_zero() && + self.find_canonical_header(state.safe_block_hash).ok().flatten().is_none() { debug!(target: "engine::tree", "missing safe block on initial FCU, downloading safe block"); state.safe_block_hash @@ -1288,7 +1273,7 @@ where /// Helper method to save blocks and set the persistence state. This ensures we keep track of /// the current persistence action while we're saving blocks. - fn persist_blocks(&mut self, blocks_to_persist: Vec>) { + fn persist_blocks(&mut self, blocks_to_persist: Vec>) { if blocks_to_persist.is_empty() { debug!(target: "engine::tree", "Returned empty set of blocks to persist"); return @@ -1301,7 +1286,7 @@ where .map(|b| b.recovered_block().num_hash()) .expect("Checked non-empty persisting blocks"); - debug!(target: "engine::tree", blocks = ?blocks_to_persist.iter().map(|block| block.recovered_block().num_hash()).collect::>(), "Persisting blocks"); + debug!(target: "engine::tree", count=blocks_to_persist.len(), blocks = ?blocks_to_persist.iter().map(|block| block.recovered_block().num_hash()).collect::>(), "Persisting blocks"); let (tx, rx) = oneshot::channel(); let _ = self.persistence.save_blocks(blocks_to_persist, tx); @@ -1350,7 +1335,7 @@ where if let Some(new_tip_num) = self.find_disk_reorg()? { self.remove_blocks(new_tip_num) } else if self.should_persist() { - let blocks_to_persist = self.get_canonical_blocks_to_persist(); + let blocks_to_persist = self.get_canonical_blocks_to_persist()?; self.persist_blocks(blocks_to_persist); } } @@ -1397,7 +1382,7 @@ where self.state.tree_state.insert_executed(block.clone()); self.metrics.engine.inserted_already_executed_blocks.increment(1); self.emit_event(EngineApiEvent::BeaconConsensus( - BeaconConsensusEngineEvent::CanonicalBlockAdded(block, now.elapsed()), + ConsensusEngineEvent::CanonicalBlockAdded(block, now.elapsed()), )); } EngineApiRequest::Beacon(request) => { @@ -1418,7 +1403,7 @@ where .set_latest(state, res.outcome.forkchoice_status()); // emit an event about the handled FCU - self.emit_event(BeaconConsensusEngineEvent::ForkchoiceUpdated( + self.emit_event(ConsensusEngineEvent::ForkchoiceUpdated( state, res.outcome.forkchoice_status(), )); @@ -1492,18 +1477,18 @@ where debug!(target: "engine::tree", "received backfill sync finished event"); self.backfill_sync_state = BackfillSyncState::Idle; - // backfill height is the block number that the backfill finished at - let mut backfill_height = ctrl.block_number(); - // Pipeline unwound, memorize the invalid block and wait for CL for next sync target. - if let ControlFlow::Unwind { bad_block, target } = &ctrl { + let backfill_height = if let ControlFlow::Unwind { bad_block, target } = &ctrl { warn!(target: "engine::tree", invalid_block=?bad_block, "Bad block detected in unwind"); // update the `invalid_headers` cache with the new invalid header self.state.invalid_headers.insert(**bad_block); // if this was an unwind then the target is the new height - backfill_height = Some(*target); - } + Some(*target) + } else { + // backfill height is the block number that the backfill finished at + ctrl.block_number() + }; // backfill height is the block number that the backfill finished at let Some(backfill_height) = backfill_height else { return Ok(()) }; @@ -1589,6 +1574,32 @@ where return Ok(()) }; + // Check if there are more blocks to sync between current head and FCU target + if let Some(lowest_buffered) = + self.state.buffer.lowest_ancestor(&sync_target_state.head_block_hash) + { + let current_head_num = self.state.tree_state.current_canonical_head.number; + let target_head_num = lowest_buffered.number(); + + if let Some(distance) = self.distance_from_local_tip(current_head_num, target_head_num) + { + // There are blocks between current head and FCU target, download them + debug!( + target: "engine::tree", + %current_head_num, + %target_head_num, + %distance, + "Backfill complete, downloading remaining blocks to reach FCU target" + ); + + self.emit_event(EngineApiEvent::Download(DownloadRequest::BlockRange( + lowest_buffered.parent_hash(), + distance, + ))); + return Ok(()); + } + } + // try to close the gap by executing buffered blocks that are child blocks of the new head self.try_connect_buffered_blocks(self.state.tree_state.current_canonical_head) } @@ -1677,19 +1688,32 @@ where } /// Returns a batch of consecutive canonical blocks to persist in the range - /// `(last_persisted_number .. canonical_head - threshold]` . The expected + /// `(last_persisted_number .. canonical_head - threshold]`. The expected /// order is oldest -> newest. - fn get_canonical_blocks_to_persist(&self) -> Vec> { + fn get_canonical_blocks_to_persist( + &self, + ) -> Result>, AdvancePersistenceError> { + // We will calculate the state root using the database, so we need to be sure there are no + // changes + debug_assert!(!self.persistence_state.in_progress()); + let mut blocks_to_persist = Vec::new(); let mut current_hash = self.state.tree_state.canonical_block_hash(); let last_persisted_number = self.persistence_state.last_persisted_block.number; - let canonical_head_number = self.state.tree_state.canonical_block_number(); + // Persist only up to block buffer target let target_number = canonical_head_number.saturating_sub(self.config.memory_block_buffer_target()); - debug!(target: "engine::tree", ?last_persisted_number, ?canonical_head_number, ?target_number, ?current_hash, "Returning canonical blocks to persist"); + debug!( + target: "engine::tree", + ?current_hash, + ?last_persisted_number, + ?canonical_head_number, + ?target_number, + "Returning canonical blocks to persist" + ); while let Some(block) = self.state.tree_state.blocks_by_hash.get(¤t_hash) { if block.recovered_block().number() <= last_persisted_number { break; @@ -1702,10 +1726,10 @@ where current_hash = block.recovered_block().parent_hash(); } - // reverse the order so that the oldest block comes first + // Reverse the order so that the oldest block comes first blocks_to_persist.reverse(); - blocks_to_persist + Ok(blocks_to_persist) } /// This clears the blocks from the in-memory tree state that have been persisted to the @@ -1742,8 +1766,8 @@ where fn canonical_block_by_hash(&self, hash: B256) -> ProviderResult>> { trace!(target: "engine::tree", ?hash, "Fetching executed block by hash"); // check memory first - if let Some(block) = self.state.tree_state.executed_block_by_hash(hash).cloned() { - return Ok(Some(block.block)) + if let Some(block) = self.state.tree_state.executed_block_by_hash(hash) { + return Ok(Some(block.clone())) } let (block, senders) = self @@ -1756,50 +1780,31 @@ where .get_state(block.header().number())? .ok_or_else(|| ProviderError::StateForNumberNotFound(block.header().number()))?; let hashed_state = self.provider.hashed_post_state(execution_output.state()); + let trie_updates = self.provider.get_block_trie_updates(block.number())?; Ok(Some(ExecutedBlock { recovered_block: Arc::new(RecoveredBlock::new_sealed(block, senders)), execution_output: Arc::new(execution_output), hashed_state: Arc::new(hashed_state), + trie_updates: Arc::new(trie_updates.into()), })) } - /// Return sealed block from database or in-memory state by hash. + /// Return sealed block header from in-memory state or database by hash. fn sealed_header_by_hash( &self, hash: B256, ) -> ProviderResult>> { // check memory first - let block = self - .state - .tree_state - .block_by_hash(hash) - .map(|block| block.as_ref().clone_sealed_header()); + let header = self.state.tree_state.sealed_header_by_hash(&hash); - if block.is_some() { - Ok(block) + if header.is_some() { + Ok(header) } else { self.provider.sealed_header_by_hash(hash) } } - /// Return block from database or in-memory state by hash. - fn block_by_hash(&self, hash: B256) -> ProviderResult> { - // check database first - let mut block = self.provider.block_by_hash(hash)?; - if block.is_none() { - // Note: it's fine to return the unsealed block because the caller already has - // the hash - block = self - .state - .tree_state - .block_by_hash(hash) - // TODO: clone for compatibility. should we return an Arc here? - .map(|block| block.as_ref().clone().into_block()); - } - Ok(block) - } - /// Return the parent hash of the lowest buffered ancestor for the requested block, if there /// are any buffered ancestors. If there are no buffered ancestors, and the block itself does /// not exist in the buffer, this returns the hash that is passed in. @@ -1829,7 +1834,7 @@ where parent_hash: B256, ) -> ProviderResult> { // Check if parent exists in side chain or in canonical chain. - if self.block_by_hash(parent_hash)?.is_some() { + if self.sealed_header_by_hash(parent_hash)?.is_some() { return Ok(Some(parent_hash)) } @@ -1843,7 +1848,7 @@ where // If current_header is None, then the current_hash does not have an invalid // ancestor in the cache, check its presence in blockchain tree - if current_block.is_none() && self.block_by_hash(current_hash)?.is_some() { + if current_block.is_none() && self.sealed_header_by_hash(current_hash)?.is_some() { return Ok(Some(current_hash)) } } @@ -1856,10 +1861,10 @@ where fn prepare_invalid_response(&mut self, mut parent_hash: B256) -> ProviderResult { // Edge case: the `latestValid` field is the zero hash if the parent block is the terminal // PoW block, which we need to identify by looking at the parent's block difficulty - if let Some(parent) = self.block_by_hash(parent_hash)? { - if !parent.header().difficulty().is_zero() { - parent_hash = B256::ZERO; - } + if let Some(parent) = self.sealed_header_by_hash(parent_hash)? && + !parent.difficulty().is_zero() + { + parent_hash = B256::ZERO; } let valid_parent_hash = self.latest_valid_hash_for_invalid_payload(parent_hash)?; @@ -1892,14 +1897,74 @@ where // check if the check hash was previously marked as invalid let Some(header) = self.state.invalid_headers.get(&check) else { return Ok(None) }; + Ok(Some(self.on_invalid_new_payload(head.clone(), header)?)) + } + + /// Invoked when a new payload received is invalid. + fn on_invalid_new_payload( + &mut self, + head: SealedBlock, + invalid: BlockWithParent, + ) -> ProviderResult { // populate the latest valid hash field - let status = self.prepare_invalid_response(header.parent)?; + let status = self.prepare_invalid_response(invalid.parent)?; // insert the head block into the invalid header cache - self.state.invalid_headers.insert_with_invalid_ancestor(head.hash(), header); - self.emit_event(BeaconConsensusEngineEvent::InvalidBlock(Box::new(head.clone()))); + self.state.invalid_headers.insert_with_invalid_ancestor(head.hash(), invalid); + self.emit_event(ConsensusEngineEvent::InvalidBlock(Box::new(head))); + + Ok(status) + } + + /// Finds any invalid ancestor for the given payload. + /// + /// This function walks up the chain of buffered ancestors from the payload's block + /// hash and checks if any ancestor is marked as invalid in the tree state. + /// + /// The check works by: + /// 1. Finding the lowest buffered ancestor for the given block hash + /// 2. If the ancestor is the same as the block hash itself, using the parent hash instead + /// 3. Checking if this ancestor is in the `invalid_headers` map + /// + /// Returns the invalid ancestor block info if found, or None if no invalid ancestor exists. + fn find_invalid_ancestor(&mut self, payload: &T::ExecutionData) -> Option { + let parent_hash = payload.parent_hash(); + let block_hash = payload.block_hash(); + let mut lowest_buffered_ancestor = self.lowest_buffered_ancestor_or(block_hash); + if lowest_buffered_ancestor == block_hash { + lowest_buffered_ancestor = parent_hash; + } + + // Check if the block has an invalid ancestor + self.state.invalid_headers.get(&lowest_buffered_ancestor) + } + + /// Handles a payload that has an invalid ancestor. + /// + /// This function validates the payload and processes it according to whether it's + /// well-formed or malformed: + /// 1. **Well-formed payload**: The payload is marked as invalid since it descends from a + /// known-bad block, which violates consensus rules + /// 2. **Malformed payload**: Returns an appropriate error status since the payload cannot be + /// validated due to its own structural issues + fn handle_invalid_ancestor_payload( + &mut self, + payload: T::ExecutionData, + invalid: BlockWithParent, + ) -> Result { + let parent_hash = payload.parent_hash(); - Ok(Some(status)) + // Here we might have 2 cases + // 1. the block is well formed and indeed links to an invalid header, meaning we should + // remember it as invalid + // 2. the block is not well formed (i.e block hash is incorrect), and we should just return + // an error and forget it + let block = match self.payload_validator.ensure_well_formed_payload(payload) { + Ok(block) => block, + Err(error) => return Ok(self.on_new_payload_error(error, parent_hash)?), + }; + + Ok(self.on_invalid_new_payload(block.into_sealed_block(), invalid)?) } /// Checks if the given `head` points to an invalid header, which requires a specific response @@ -1907,8 +1972,18 @@ where fn check_invalid_ancestor(&mut self, head: B256) -> ProviderResult> { // check if the head was previously marked as invalid let Some(header) = self.state.invalid_headers.get(&head) else { return Ok(None) }; - // populate the latest valid hash field - Ok(Some(self.prepare_invalid_response(header.parent)?)) + + // Try to prepare invalid response, but handle errors gracefully + match self.prepare_invalid_response(header.parent) { + Ok(status) => Ok(Some(status)), + Err(err) => { + debug!(target: "engine::tree", %err, "Failed to prepare invalid response for ancestor check"); + // Return a basic invalid status without latest valid hash + Ok(Some(PayloadStatus::from_status(PayloadStatusEnum::Invalid { + validation_error: PayloadValidationError::LinksToRejectedPayload.to_string(), + }))) + } + } } /// Validate if block is correct and satisfies all the consensus rules that concern the header @@ -1928,7 +2003,7 @@ where } /// Attempts to connect any buffered blocks that are connected to the given parent hash. - #[instrument(level = "trace", skip(self), target = "engine::tree")] + #[instrument(level = "debug", target = "engine::tree", skip(self))] fn try_connect_buffered_blocks( &mut self, parent: BlockNumHash, @@ -1954,10 +2029,12 @@ where } } Err(err) => { - debug!(target: "engine::tree", ?err, "failed to connect buffered block to tree"); - if let Err(fatal) = self.on_insert_block_error(err) { - warn!(target: "engine::tree", %fatal, "fatal error occurred while connecting buffered blocks"); - return Err(fatal) + if let InsertPayloadError::Block(err) = err { + debug!(target: "engine::tree", ?err, "failed to connect buffered block to tree"); + if let Err(fatal) = self.on_insert_block_error(err) { + warn!(target: "engine::tree", %fatal, "fatal error occurred while connecting buffered blocks"); + return Err(fatal) + } } } } @@ -2013,65 +2090,66 @@ where ) -> Option { let sync_target_state = self.state.forkchoice_state_tracker.sync_target_state(); - // check if the distance exceeds the threshold for backfill sync - let mut exceeds_backfill_threshold = - self.exceeds_backfill_run_threshold(canonical_tip_num, target_block_number); - // check if the downloaded block is the tracked finalized block - if let Some(buffered_finalized) = sync_target_state - .as_ref() - .and_then(|state| self.state.buffer.block(&state.finalized_block_hash)) - { - // if we have buffered the finalized block, we should check how far - // we're off - exceeds_backfill_threshold = - self.exceeds_backfill_run_threshold(canonical_tip_num, buffered_finalized.number()); - } - - // If this is invoked after we downloaded a block we can check if this block is the - // finalized block - if let (Some(downloaded_block), Some(ref state)) = (downloaded_block, sync_target_state) { - if downloaded_block.hash == state.finalized_block_hash { - // we downloaded the finalized block and can now check how far we're off - exceeds_backfill_threshold = - self.exceeds_backfill_run_threshold(canonical_tip_num, downloaded_block.number); - } - } + let exceeds_backfill_threshold = + match (downloaded_block.as_ref(), sync_target_state.as_ref()) { + // if we downloaded the finalized block we can now check how far we're off + (Some(downloaded_block), Some(state)) + if downloaded_block.hash == state.finalized_block_hash => + { + self.exceeds_backfill_run_threshold(canonical_tip_num, downloaded_block.number) + } + _ => match sync_target_state + .as_ref() + .and_then(|state| self.state.buffer.block(&state.finalized_block_hash)) + { + Some(buffered_finalized) => { + // if we have buffered the finalized block, we should check how far we're + // off + self.exceeds_backfill_run_threshold( + canonical_tip_num, + buffered_finalized.number(), + ) + } + None => { + // check if the distance exceeds the threshold for backfill sync + self.exceeds_backfill_run_threshold(canonical_tip_num, target_block_number) + } + }, + }; // if the number of missing blocks is greater than the max, trigger backfill - if exceeds_backfill_threshold { - if let Some(state) = sync_target_state { - // if we have already canonicalized the finalized block, we should skip backfill - match self.provider.header_by_hash_or_number(state.finalized_block_hash.into()) { - Err(err) => { - warn!(target: "engine::tree", %err, "Failed to get finalized block header"); + if exceeds_backfill_threshold && let Some(state) = sync_target_state { + // if we have already canonicalized the finalized block, we should skip backfill + match self.provider.header_by_hash_or_number(state.finalized_block_hash.into()) { + Err(err) => { + warn!(target: "engine::tree", %err, "Failed to get finalized block header"); + } + Ok(None) => { + // ensure the finalized block is known (not the zero hash) + if !state.finalized_block_hash.is_zero() { + // we don't have the block yet and the distance exceeds the allowed + // threshold + return Some(state.finalized_block_hash) } - Ok(None) => { - // ensure the finalized block is known (not the zero hash) - if !state.finalized_block_hash.is_zero() { - // we don't have the block yet and the distance exceeds the allowed - // threshold - return Some(state.finalized_block_hash) - } - // OPTIMISTIC SYNCING - // - // It can happen when the node is doing an - // optimistic sync, where the CL has no knowledge of the finalized hash, - // but is expecting the EL to sync as high - // as possible before finalizing. - // - // This usually doesn't happen on ETH mainnet since CLs use the more - // secure checkpoint syncing. - // - // However, optimism chains will do this. The risk of a reorg is however - // low. - debug!(target: "engine::tree", hash=?state.head_block_hash, "Setting head hash as an optimistic backfill target."); - return Some(state.head_block_hash) - } - Ok(Some(_)) => { - // we're fully synced to the finalized block - } + // OPTIMISTIC SYNCING + // + // It can happen when the node is doing an + // optimistic sync, where the CL has no knowledge of the finalized hash, + // but is expecting the EL to sync as high + // as possible before finalizing. + // + // This usually doesn't happen on ETH mainnet since CLs use the more + // secure checkpoint syncing. + // + // However, optimism chains will do this. The risk of a reorg is however + // low. + debug!(target: "engine::tree", hash=?state.head_block_hash, "Setting head hash as an optimistic backfill target."); + return Some(state.head_block_hash) + } + Ok(Some(_)) => { + // we're fully synced to the finalized block } } } @@ -2146,21 +2224,7 @@ where self.update_reorg_metrics(old.len()); self.reinsert_reorged_blocks(new.clone()); - // Try reinserting the reorged canonical chain. This is only possible if we have - // `persisted_trie_updates` for those blocks. - let old = old - .iter() - .filter_map(|block| { - let (_, trie) = self - .state - .tree_state - .persisted_trie_updates - .get(&block.recovered_block.hash()) - .cloned()?; - Some(ExecutedBlockWithTrieUpdates { block: block.clone(), trie }) - }) - .collect::>(); - self.reinsert_reorged_blocks(old); + self.reinsert_reorged_blocks(old.clone()); } // update the tracked in-memory state with the new chain @@ -2174,7 +2238,7 @@ where self.canonical_in_memory_state.notify_canon_state(notification); // emit event - self.emit_event(BeaconConsensusEngineEvent::CanonicalChainCommitted( + self.emit_event(ConsensusEngineEvent::CanonicalChainCommitted( Box::new(tip), start.elapsed(), )); @@ -2187,7 +2251,7 @@ where } /// This reinserts any blocks in the new chain that do not already exist in the tree - fn reinsert_reorged_blocks(&mut self, new_chain: Vec>) { + fn reinsert_reorged_blocks(&mut self, new_chain: Vec>) { for block in new_chain { if self .state @@ -2201,21 +2265,6 @@ where } } - /// Invoke the invalid block hook if this is a new invalid block. - fn on_invalid_block( - &mut self, - parent_header: &SealedHeader, - block: &RecoveredBlock, - output: &BlockExecutionOutput, - trie_updates: Option<(&TrieUpdates, B256)>, - ) { - if self.state.invalid_headers.get(&block.hash()).is_some() { - // we already marked this block as invalid - return; - } - self.invalid_block_hook.on_invalid_block(parent_header, block, output, trie_updates); - } - /// This handles downloaded blocks that are shown to be disconnected from the canonical chain. /// /// This mainly compares the missing parent of the downloaded block with the current canonical @@ -2263,7 +2312,7 @@ where /// Returns an event with the appropriate action to take, such as: /// - download more missing blocks /// - try to canonicalize the target if the `block` is the tracked target (head) block. - #[instrument(level = "trace", skip_all, fields(block_hash = %block.hash(), block_num = %block.number(),), target = "engine::tree")] + #[instrument(level = "debug", target = "engine::tree", skip_all, fields(block_hash = %block.hash(), block_num = %block.number()))] fn on_downloaded_block( &mut self, block: RecoveredBlock, @@ -2309,453 +2358,164 @@ where trace!(target: "engine::tree", "downloaded block already executed"); } Err(err) => { - debug!(target: "engine::tree", err=%err.kind(), "failed to insert downloaded block"); - if let Err(fatal) = self.on_insert_block_error(err) { - warn!(target: "engine::tree", %fatal, "fatal error occurred while inserting downloaded block"); - return Err(fatal) + if let InsertPayloadError::Block(err) = err { + debug!(target: "engine::tree", err=%err.kind(), "failed to insert downloaded block"); + if let Err(fatal) = self.on_insert_block_error(err) { + warn!(target: "engine::tree", %fatal, "fatal error occurred while inserting downloaded block"); + return Err(fatal) + } } } } Ok(None) } + /// Inserts a payload into the tree and executes it. + /// + /// This function validates the payload's basic structure, then executes it using the + /// payload validator. The execution includes running all transactions in the payload + /// and validating the resulting state transitions. + /// + /// Returns `InsertPayloadOk` if the payload was successfully inserted and executed, + /// or `InsertPayloadError` if validation or execution failed. + fn insert_payload( + &mut self, + payload: T::ExecutionData, + ) -> Result> { + self.insert_block_or_payload( + payload.block_with_parent(), + payload, + |validator, payload, ctx| validator.validate_payload(payload, ctx), + |this, payload| Ok(this.payload_validator.ensure_well_formed_payload(payload)?), + ) + } + fn insert_block( &mut self, block: RecoveredBlock, - ) -> Result> { - match self.insert_block_inner(block) { - Ok(result) => Ok(result), - Err((kind, block)) => Err(InsertBlockError::new(block.into_sealed_block(), kind)), - } + ) -> Result> { + self.insert_block_or_payload( + block.block_with_parent(), + block, + |validator, block, ctx| validator.validate_block(block, ctx), + |_, block| Ok(block), + ) } - fn insert_block_inner( + /// Inserts a block or payload into the blockchain tree with full execution. + /// + /// This is a generic function that handles both blocks and payloads by accepting + /// a block identifier, input data, and execution/validation functions. It performs + /// comprehensive checks and execution: + /// + /// - Validates that the block doesn't already exist in the tree + /// - Ensures parent state is available, buffering if necessary + /// - Executes the block/payload using the provided execute function + /// - Handles both canonical and fork chain insertions + /// - Updates pending block state when appropriate + /// - Emits consensus engine events and records metrics + /// + /// Returns `InsertPayloadOk::Inserted(BlockStatus::Valid)` on successful execution, + /// `InsertPayloadOk::AlreadySeen` if the block already exists, or + /// `InsertPayloadOk::Inserted(BlockStatus::Disconnected)` if parent state is missing. + #[instrument(level = "debug", target = "engine::tree", skip_all, fields(block_id))] + fn insert_block_or_payload( &mut self, - block: RecoveredBlock, - ) -> Result)> { - /// A helper macro that returns the block in case there was an error - macro_rules! ensure_ok { - ($expr:expr) => { - match $expr { - Ok(val) => val, - Err(e) => return Err((e.into(), block)), - } - }; - } + block_id: BlockWithParent, + input: Input, + execute: impl FnOnce(&mut V, Input, TreeCtx<'_, N>) -> Result, Err>, + convert_to_block: impl FnOnce(&mut Self, Input) -> Result, Err>, + ) -> Result + where + Err: From>, + { + let block_insert_start = Instant::now(); + let block_num_hash = block_id.block; + debug!(target: "engine::tree", block=?block_num_hash, parent = ?block_id.parent, "Inserting new block into tree"); - let block_num_hash = block.num_hash(); - debug!(target: "engine::tree", block=?block_num_hash, parent = ?block.parent_hash(), state_root = ?block.state_root(), "Inserting new block into tree"); + match self.sealed_header_by_hash(block_num_hash.hash) { + Err(err) => { + let block = convert_to_block(self, input)?; + return Err(InsertBlockError::new(block.into_sealed_block(), err.into()).into()); + } + Ok(Some(_)) => { + // We now assume that we already have this block in the tree. However, we need to + // run the conversion to ensure that the block hash is valid. + convert_to_block(self, input)?; + return Ok(InsertPayloadOk::AlreadySeen(BlockStatus::Valid)) + } + _ => {} + }; - if ensure_ok!(self.block_by_hash(block.hash())).is_some() { - return Ok(InsertPayloadOk::AlreadySeen(BlockStatus::Valid)) - } + // Ensure that the parent state is available. + match self.state_provider_builder(block_id.parent) { + Err(err) => { + let block = convert_to_block(self, input)?; + return Err(InsertBlockError::new(block.into_sealed_block(), err.into()).into()); + } + Ok(None) => { + let block = convert_to_block(self, input)?; - let start = Instant::now(); + // we don't have the state required to execute this block, buffering it and find the + // missing parent block + let missing_ancestor = self + .state + .buffer + .lowest_ancestor(&block.parent_hash()) + .map(|block| block.parent_num_hash()) + .unwrap_or_else(|| block.parent_num_hash()); - trace!(target: "engine::tree", block=?block_num_hash, "Validating block consensus"); + self.state.buffer.insert_block(block); - // validate block consensus rules - ensure_ok!(self.validate_block(&block)); + return Ok(InsertPayloadOk::Inserted(BlockStatus::Disconnected { + head: self.state.tree_state.current_canonical_head, + missing_ancestor, + })) + } + Ok(Some(_)) => {} + } - trace!(target: "engine::tree", block=?block_num_hash, parent=?block.parent_hash(), "Fetching block state provider"); - let Some(provider_builder) = ensure_ok!(self.state_provider_builder(block.parent_hash())) - else { - // we don't have the state required to execute this block, buffering it and find the - // missing parent block - let missing_ancestor = self - .state - .buffer - .lowest_ancestor(&block.parent_hash()) - .map(|block| block.parent_num_hash()) - .unwrap_or_else(|| block.parent_num_hash()); + // determine whether we are on a fork chain + let is_fork = match self.is_fork(block_id) { + Err(err) => { + let block = convert_to_block(self, input)?; + return Err(InsertBlockError::new(block.into_sealed_block(), err.into()).into()); + } + Ok(is_fork) => is_fork, + }; - self.state.buffer.insert_block(block); + let ctx = TreeCtx::new(&mut self.state, &self.canonical_in_memory_state); - return Ok(InsertPayloadOk::Inserted(BlockStatus::Disconnected { - head: self.state.tree_state.current_canonical_head, - missing_ancestor, - })) - }; + let start = Instant::now(); - // now validate against the parent - let Some(parent_block) = ensure_ok!(self.sealed_header_by_hash(block.parent_hash())) else { - return Err(( - InsertBlockErrorKind::Provider(ProviderError::HeaderNotFound( - block.parent_hash().into(), - )), - block, - )) - }; + let executed = execute(&mut self.payload_validator, input, ctx)?; - if let Err(e) = - self.consensus.validate_header_against_parent(block.sealed_header(), &parent_block) + // if the parent is the canonical head, we can insert the block as the pending block + if self.state.tree_state.canonical_block_hash() == executed.recovered_block().parent_hash() { - warn!(target: "engine::tree", ?block, "Failed to validate header {} against parent: {e}", block.hash()); - return Err((e.into(), block)) + debug!(target: "engine::tree", pending=?block_num_hash, "updating pending block"); + self.canonical_in_memory_state.set_pending_block(executed.clone()); } - let state_provider = ensure_ok!(provider_builder.build()); - - // We only run the parallel state root if we are not currently persisting any blocks or - // persisting blocks that are all ancestors of the one we are executing. - // - // If we're committing ancestor blocks, then: any trie updates being committed are a subset - // of the in-memory trie updates collected before fetching reverts. So any diff in - // reverts (pre vs post commit) is already covered by the in-memory trie updates we - // collect in `compute_state_root_parallel`. - // - // See https://github.com/paradigmxyz/reth/issues/12688 for more details - let persisting_kind = self.persisting_kind_for(block.header()); - let run_parallel_state_root = persisting_kind.can_run_parallel_state_root(); - - // use prewarming background task - let header = block.clone_sealed_header(); - let txs = block.clone_transactions_recovered().collect(); - let mut handle = if run_parallel_state_root && self.config.use_state_root_task() { - // use background tasks for state root calc - let consistent_view = - ensure_ok!(ConsistentDbView::new_with_latest_tip(self.provider.clone())); - - // Compute trie input - let trie_input_start = Instant::now(); - let res = self.compute_trie_input( - persisting_kind, - consistent_view.clone(), - block.header().parent_hash(), - ); - let trie_input = match res { - Ok(val) => val, - Err(e) => return Err((InsertBlockErrorKind::Other(Box::new(e)), block)), - }; + self.state.tree_state.insert_executed(executed.clone()); + self.metrics.engine.executed_blocks.set(self.state.tree_state.block_count() as f64); - self.metrics - .block_validation - .trie_input_duration - .record(trie_input_start.elapsed().as_secs_f64()); - - self.payload_processor.spawn( - header, - txs, - provider_builder, - consistent_view, - trie_input, - &self.config, - ) + // emit insert event + let elapsed = start.elapsed(); + let engine_event = if is_fork { + ConsensusEngineEvent::ForkBlockAdded(executed, elapsed) } else { - self.payload_processor.spawn_cache_exclusive(header, txs, provider_builder) + ConsensusEngineEvent::CanonicalBlockAdded(executed, elapsed) }; + self.emit_event(EngineApiEvent::BeaconConsensus(engine_event)); - // Use cached state provider before executing, used in execution after prewarming threads - // complete - let state_provider = CachedStateProvider::new_with_caches( - state_provider, - handle.caches(), - handle.cache_metrics(), - ); - - let (output, execution_finish) = if self.config.state_provider_metrics() { - let state_provider = InstrumentedStateProvider::from_state_provider(&state_provider); - let (output, execution_finish) = - ensure_ok!(self.execute_block(&state_provider, &block, &handle)); - state_provider.record_total_latency(); - (output, execution_finish) - } else { - let (output, execution_finish) = - ensure_ok!(self.execute_block(&state_provider, &block, &handle)); - (output, execution_finish) - }; - - // after executing the block we can stop executing transactions - handle.stop_prewarming_execution(); - - if let Err(err) = self.consensus.validate_block_post_execution(&block, &output) { - // call post-block hook - self.on_invalid_block(&parent_block, &block, &output, None); - return Err((err.into(), block)) - } - - let hashed_state = self.provider.hashed_post_state(&output.state); - - if let Err(err) = self - .payload_validator - .validate_block_post_execution_with_hashed_state(&hashed_state, &block) - { - // call post-block hook - self.on_invalid_block(&parent_block, &block, &output, None); - return Err((err.into(), block)) - } - - debug!(target: "engine::tree", block=?block_num_hash, "Calculating block state root"); - - let root_time = Instant::now(); - - let mut maybe_state_root = None; - - if run_parallel_state_root { - // if we new payload extends the current canonical change we attempt to use the - // background task or try to compute it in parallel - if self.config.use_state_root_task() { - match handle.state_root() { - Ok(StateRootComputeOutcome { state_root, trie_updates }) => { - let elapsed = execution_finish.elapsed(); - info!(target: "engine::tree", ?state_root, ?elapsed, "State root task finished"); - // we double check the state root here for good measure - if state_root == block.header().state_root() { - maybe_state_root = Some((state_root, trie_updates, elapsed)) - } else { - warn!( - target: "engine::tree", - ?state_root, - block_state_root = ?block.header().state_root(), - "State root task returned incorrect state root" - ); - } - } - Err(error) => { - debug!(target: "engine::tree", %error, "Background parallel state root computation failed"); - } - } - } else { - match self.compute_state_root_parallel( - persisting_kind, - block.header().parent_hash(), - &hashed_state, - ) { - Ok(result) => { - info!( - target: "engine::tree", - block = ?block_num_hash, - regular_state_root = ?result.0, - "Regular root task finished" - ); - maybe_state_root = Some((result.0, result.1, root_time.elapsed())); - } - Err(ParallelStateRootError::Provider(ProviderError::ConsistentView(error))) => { - debug!(target: "engine::tree", %error, "Parallel state root computation failed consistency check, falling back"); - } - Err(error) => return Err((InsertBlockErrorKind::Other(Box::new(error)), block)), - } - } - } - - let (state_root, trie_output, root_elapsed) = if let Some(maybe_state_root) = - maybe_state_root - { - maybe_state_root - } else { - // fallback is to compute the state root regularly in sync - warn!(target: "engine::tree", block=?block_num_hash, ?persisting_kind, "Failed to compute state root in parallel"); - self.metrics.block_validation.state_root_parallel_fallback_total.increment(1); - let (root, updates) = - ensure_ok!(state_provider.state_root_with_updates(hashed_state.clone())); - (root, updates, root_time.elapsed()) - }; - - self.metrics.block_validation.record_state_root(&trie_output, root_elapsed.as_secs_f64()); - debug!(target: "engine::tree", ?root_elapsed, block=?block_num_hash, "Calculated state root"); - - // ensure state root matches - if state_root != block.header().state_root() { - // call post-block hook - self.on_invalid_block(&parent_block, &block, &output, Some((&trie_output, state_root))); - return Err(( - ConsensusError::BodyStateRootDiff( - GotExpected { got: state_root, expected: block.header().state_root() }.into(), - ) - .into(), - block, - )) - } - - // terminate prewarming task with good state output - handle.terminate_caching(Some(output.state.clone())); - - let executed: ExecutedBlockWithTrieUpdates = ExecutedBlockWithTrieUpdates { - block: ExecutedBlock { - recovered_block: Arc::new(block), - execution_output: Arc::new(ExecutionOutcome::from((output, block_num_hash.number))), - hashed_state: Arc::new(hashed_state), - }, - trie: Arc::new(trie_output), - }; - - // if the parent is the canonical head, we can insert the block as the pending block - if self.state.tree_state.canonical_block_hash() == executed.recovered_block().parent_hash() - { - debug!(target: "engine::tree", pending=?block_num_hash, "updating pending block"); - self.canonical_in_memory_state.set_pending_block(executed.clone()); - } - - self.state.tree_state.insert_executed(executed.clone()); - self.metrics.engine.executed_blocks.set(self.state.tree_state.block_count() as f64); - - // emit insert event - let elapsed = start.elapsed(); - let is_fork = match self.is_fork(block_num_hash.hash) { - Ok(val) => val, - Err(e) => return Err((e.into(), executed.block.recovered_block().clone())), - }; - let engine_event = if is_fork { - BeaconConsensusEngineEvent::ForkBlockAdded(executed, elapsed) - } else { - BeaconConsensusEngineEvent::CanonicalBlockAdded(executed, elapsed) - }; - self.emit_event(EngineApiEvent::BeaconConsensus(engine_event)); - - debug!(target: "engine::tree", block=?block_num_hash, "Finished inserting block"); - Ok(InsertPayloadOk::Inserted(BlockStatus::Valid)) - } - - /// Executes a block with the given state provider - fn execute_block( - &self, - state_provider: S, - block: &RecoveredBlock, - handle: &PayloadHandle, - ) -> Result<(BlockExecutionOutput, Instant), InsertBlockErrorKind> { - debug!(target: "engine::tree", block=?block.num_hash(), "Executing block"); - let mut db = State::builder() - .with_database(StateProviderDatabase::new(&state_provider)) - .with_bundle_update() - .without_state_clear() - .build(); - let executor = self.evm_config.executor_for_block(&mut db, block); - let execution_start = Instant::now(); - let output = self.metrics.executor.execute_metered( - executor, - block, - Box::new(handle.state_hook()), - )?; - let execution_finish = Instant::now(); - let execution_time = execution_finish.duration_since(execution_start); - debug!(target: "engine::tree", elapsed = ?execution_time, number=?block.number(), "Executed block"); - Ok((output, execution_finish)) - } - - /// Compute state root for the given hashed post state in parallel. - /// - /// # Returns - /// - /// Returns `Ok(_)` if computed successfully. - /// Returns `Err(_)` if error was encountered during computation. - /// `Err(ProviderError::ConsistentView(_))` can be safely ignored and fallback computation - /// should be used instead. - fn compute_state_root_parallel( - &self, - persisting_kind: PersistingKind, - parent_hash: B256, - hashed_state: &HashedPostState, - ) -> Result<(B256, TrieUpdates), ParallelStateRootError> { - let consistent_view = ConsistentDbView::new_with_latest_tip(self.provider.clone())?; - - let mut input = - self.compute_trie_input(persisting_kind, consistent_view.clone(), parent_hash)?; - // Extend with block we are validating root for. - input.append_ref(hashed_state); - - ParallelStateRoot::new(consistent_view, input).incremental_root_with_updates() - } - - /// Computes the trie input at the provided parent hash. - /// - /// The goal of this function is to take in-memory blocks and generate a [`TrieInput`] that - /// serves as an overlay to the database blocks. - /// - /// It works as follows: - /// 1. Collect in-memory blocks that are descendants of the provided parent hash using - /// [`TreeState::blocks_by_hash`]. - /// 2. If the persistence is in progress, and the block that we're computing the trie input for - /// is a descendant of the currently persisting blocks, we need to be sure that in-memory - /// blocks are not overlapping with the database blocks that may have been already persisted. - /// To do that, we're filtering out in-memory blocks that are lower than the highest database - /// block. - /// 3. Once in-memory blocks are collected and optionally filtered, we compute the - /// [`HashedPostState`] from them. - fn compute_trie_input( - &self, - persisting_kind: PersistingKind, - consistent_view: ConsistentDbView

, - parent_hash: B256, - ) -> Result { - let mut input = TrieInput::default(); - - let provider = consistent_view.provider_ro()?; - let best_block_number = provider.best_block_number()?; - - let (mut historical, mut blocks) = self - .state - .tree_state - .blocks_by_hash(parent_hash) - .map_or_else(|| (parent_hash.into(), vec![]), |(hash, blocks)| (hash.into(), blocks)); - - // If the current block is a descendant of the currently persisting blocks, then we need to - // filter in-memory blocks, so that none of them are already persisted in the database. - if persisting_kind.is_descendant() { - // Iterate over the blocks from oldest to newest. - while let Some(block) = blocks.last() { - let recovered_block = block.recovered_block(); - if recovered_block.number() <= best_block_number { - // Remove those blocks that lower than or equal to the highest database - // block. - blocks.pop(); - } else { - // If the block is higher than the best block number, stop filtering, as it's - // the first block that's not in the database. - break - } - } - - historical = if let Some(block) = blocks.last() { - // If there are any in-memory blocks left after filtering, set the anchor to the - // parent of the oldest block. - (block.recovered_block().number() - 1).into() - } else { - // Otherwise, set the anchor to the original provided parent hash. - parent_hash.into() - }; - } - - if blocks.is_empty() { - debug!(target: "engine::tree", %parent_hash, "Parent found on disk"); - } else { - debug!(target: "engine::tree", %parent_hash, %historical, blocks = blocks.len(), "Parent found in memory"); - } - - // Convert the historical block to the block number. - let block_number = provider - .convert_hash_or_number(historical)? - .ok_or_else(|| ProviderError::BlockHashNotFound(historical.as_hash().unwrap()))?; - - // Retrieve revert state for historical block. - let revert_state = if block_number == best_block_number { - // We do not check against the `last_block_number` here because - // `HashedPostState::from_reverts` only uses the database tables, and not static files. - debug!(target: "engine::tree", block_number, best_block_number, "Empty revert state"); - HashedPostState::default() - } else { - let revert_state = HashedPostState::from_reverts::< - ::KeyHasher, - >(provider.tx_ref(), block_number + 1) - .map_err(ProviderError::from)?; - debug!( - target: "engine::tree", - block_number, - best_block_number, - accounts = revert_state.accounts.len(), - storages = revert_state.storages.len(), - "Non-empty revert state" - ); - revert_state - }; - input.append(revert_state); - - // Extend with contents of parent in-memory blocks. - for block in blocks.iter().rev() { - input.append_cached_ref(block.trie_updates(), block.hashed_state()) - } - - Ok(input) - } + self.metrics + .engine + .block_insert_total_duration + .record(block_insert_start.elapsed().as_secs_f64()); + debug!(target: "engine::tree", block=?block_num_hash, "Finished inserting block"); + Ok(InsertPayloadOk::Inserted(BlockStatus::Valid)) + } /// Handles an error that occurred while inserting a block. /// @@ -2786,15 +2546,39 @@ where // keep track of the invalid header self.state.invalid_headers.insert(block.block_with_parent()); - self.emit_event(EngineApiEvent::BeaconConsensus(BeaconConsensusEngineEvent::InvalidBlock( + self.emit_event(EngineApiEvent::BeaconConsensus(ConsensusEngineEvent::InvalidBlock( Box::new(block), ))); + Ok(PayloadStatus::new( PayloadStatusEnum::Invalid { validation_error: validation_err.to_string() }, latest_valid_hash, )) } + /// Handles a [`NewPayloadError`] by converting it to a [`PayloadStatus`]. + fn on_new_payload_error( + &mut self, + error: NewPayloadError, + parent_hash: B256, + ) -> ProviderResult { + error!(target: "engine::tree", %error, "Invalid payload"); + // we need to convert the error to a payload status (response to the CL) + + let latest_valid_hash = + if error.is_block_hash_mismatch() || error.is_invalid_versioned_hashes() { + // Engine-API rules: + // > `latestValidHash: null` if the blockHash validation has failed () + // > `latestValidHash: null` if the expected and the actual arrays don't match () + None + } else { + self.latest_valid_hash_for_invalid_payload(parent_hash)? + }; + + let status = PayloadStatusEnum::from(error); + Ok(PayloadStatus::new(status, latest_valid_hash)) + } + /// Attempts to find the header for the given block hash if it is canonical. pub fn find_canonical_header( &self, @@ -2803,7 +2587,7 @@ where let mut canonical = self.canonical_in_memory_state.header_by_hash(hash); if canonical.is_none() { - canonical = self.provider.header(&hash)?.map(|header| SealedHeader::new(header, hash)); + canonical = self.provider.header(hash)?.map(|header| SealedHeader::new(header, hash)); } Ok(canonical) @@ -2901,35 +2685,6 @@ where self.update_safe_block(state.safe_block_hash) } - /// Pre-validate forkchoice update and check whether it can be processed. - /// - /// This method returns the update outcome if validation fails or - /// the node is syncing and the update cannot be processed at the moment. - fn pre_validate_forkchoice_update( - &mut self, - state: ForkchoiceState, - ) -> ProviderResult> { - if state.head_block_hash.is_zero() { - return Ok(Some(OnForkChoiceUpdated::invalid_state())) - } - - // check if the new head hash is connected to any ancestor that we previously marked as - // invalid - let lowest_buffered_ancestor_fcu = self.lowest_buffered_ancestor_or(state.head_block_hash); - if let Some(status) = self.check_invalid_ancestor(lowest_buffered_ancestor_fcu)? { - return Ok(Some(OnForkChoiceUpdated::with_invalid(status))) - } - - if !self.backfill_sync_state.is_idle() { - // We can only process new forkchoice updates if the pipeline is idle, since it requires - // exclusive access to the database - trace!(target: "engine::tree", "Pipeline is syncing, skipping forkchoice update"); - return Ok(Some(OnForkChoiceUpdated::syncing())) - } - - Ok(None) - } - /// Validates the payload attributes with respect to the header and fork choice state. /// /// Note: At this point, the fork choice update is considered to be VALID, however, we can still @@ -3018,7 +2773,7 @@ where hash: B256, ) -> ProviderResult>> where - P: BlockReader + StateProviderFactory + StateReader + StateCommitmentProvider + Clone, + P: BlockReader + StateProviderFactory + StateReader + Clone, { if let Some((historical, blocks)) = self.state.tree_state.blocks_by_hash(hash) { debug!(target: "engine::tree", %hash, %historical, "found canonical state for block in memory, creating provider builder"); @@ -3031,7 +2786,7 @@ where } // Check if the block is persisted - if let Some(header) = self.provider.header(&hash)? { + if let Some(header) = self.provider.header(hash)? { debug!(target: "engine::tree", %hash, number = %header.number(), "found canonical state for block in database, creating provider builder"); // For persisted blocks, we create a builder that will fetch state directly from the // database @@ -3063,8 +2818,8 @@ pub enum BlockStatus { /// How a payload was inserted if it was valid. /// -/// If the payload was valid, but has already been seen, [`InsertPayloadOk::AlreadySeen(_)`] is -/// returned, otherwise [`InsertPayloadOk::Inserted(_)`] is returned. +/// If the payload was valid, but has already been seen, [`InsertPayloadOk::AlreadySeen`] is +/// returned, otherwise [`InsertPayloadOk::Inserted`] is returned. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum InsertPayloadOk { /// The payload was valid, but we have already seen it. @@ -3072,1674 +2827,3 @@ pub enum InsertPayloadOk { /// The payload was valid and inserted into the tree. Inserted(BlockStatus), } - -/// Whether or not the blocks are currently persisting and the input block is a descendant. -#[derive(Debug, Clone, Copy)] -pub enum PersistingKind { - /// The blocks are not currently persisting. - NotPersisting, - /// The blocks are currently persisting but the input block is not a descendant. - PersistingNotDescendant, - /// The blocks are currently persisting and the input block is a descendant. - PersistingDescendant, -} - -impl PersistingKind { - /// Returns true if the parallel state root can be run. - /// - /// We only run the parallel state root if we are not currently persisting any blocks or - /// persisting blocks that are all ancestors of the one we are calculating the state root for. - pub const fn can_run_parallel_state_root(&self) -> bool { - matches!(self, Self::NotPersisting | Self::PersistingDescendant) - } - - /// Returns true if the blocks are currently being persisted and the input block is a - /// descendant. - pub const fn is_descendant(&self) -> bool { - matches!(self, Self::PersistingDescendant) - } -} -#[cfg(test)] -mod tests { - use super::*; - use crate::persistence::PersistenceAction; - use alloy_consensus::Header; - use alloy_primitives::Bytes; - use alloy_rlp::Decodable; - use alloy_rpc_types_engine::{ - CancunPayloadFields, ExecutionData, ExecutionPayloadSidecar, ExecutionPayloadV1, - ExecutionPayloadV3, - }; - use assert_matches::assert_matches; - use reth_chain_state::{test_utils::TestBlockBuilder, BlockState}; - use reth_chainspec::{ChainSpec, HOLESKY, MAINNET}; - use reth_engine_primitives::ForkchoiceStatus; - use reth_ethereum_consensus::EthBeaconConsensus; - use reth_ethereum_engine_primitives::EthEngineTypes; - use reth_ethereum_primitives::{Block, EthPrimitives}; - use reth_evm_ethereum::MockEvmConfig; - use reth_node_ethereum::EthereumEngineValidator; - use reth_primitives_traits::Block as _; - use reth_provider::test_utils::MockEthProvider; - use reth_trie::{updates::TrieUpdates, HashedPostState}; - use std::{ - str::FromStr, - sync::mpsc::{channel, Sender}, - }; - - /// This is a test channel that allows you to `release` any value that is in the channel. - /// - /// If nothing has been sent, then the next value will be immediately sent. - struct TestChannel { - /// If an item is sent to this channel, an item will be released in the wrapped channel - release: Receiver<()>, - /// The sender channel - tx: Sender, - /// The receiver channel - rx: Receiver, - } - - impl TestChannel { - /// Creates a new test channel - fn spawn_channel() -> (Sender, Receiver, TestChannelHandle) { - let (original_tx, original_rx) = channel(); - let (wrapped_tx, wrapped_rx) = channel(); - let (release_tx, release_rx) = channel(); - let handle = TestChannelHandle::new(release_tx); - let test_channel = Self { release: release_rx, tx: wrapped_tx, rx: original_rx }; - // spawn the task that listens and releases stuff - std::thread::spawn(move || test_channel.intercept_loop()); - (original_tx, wrapped_rx, handle) - } - - /// Runs the intercept loop, waiting for the handle to release a value - fn intercept_loop(&self) { - while self.release.recv() == Ok(()) { - let Ok(value) = self.rx.recv() else { return }; - - let _ = self.tx.send(value); - } - } - } - - struct TestChannelHandle { - /// The sender to use for releasing values - release: Sender<()>, - } - - impl TestChannelHandle { - /// Returns a [`TestChannelHandle`] - const fn new(release: Sender<()>) -> Self { - Self { release } - } - - /// Signals to the channel task that a value should be released - #[expect(dead_code)] - fn release(&self) { - let _ = self.release.send(()); - } - } - - struct TestHarness { - tree: EngineApiTreeHandler< - EthPrimitives, - MockEthProvider, - EthEngineTypes, - EthereumEngineValidator, - MockEvmConfig, - >, - to_tree_tx: Sender, Block>>, - from_tree_rx: UnboundedReceiver, - blocks: Vec, - action_rx: Receiver, - evm_config: MockEvmConfig, - block_builder: TestBlockBuilder, - provider: MockEthProvider, - } - - impl TestHarness { - fn new(chain_spec: Arc) -> Self { - let (action_tx, action_rx) = channel(); - Self::with_persistence_channel(chain_spec, action_tx, action_rx) - } - - #[expect(dead_code)] - fn with_test_channel(chain_spec: Arc) -> (Self, TestChannelHandle) { - let (action_tx, action_rx, handle) = TestChannel::spawn_channel(); - (Self::with_persistence_channel(chain_spec, action_tx, action_rx), handle) - } - - fn with_persistence_channel( - chain_spec: Arc, - action_tx: Sender, - action_rx: Receiver, - ) -> Self { - let persistence_handle = PersistenceHandle::new(action_tx); - - let consensus = Arc::new(EthBeaconConsensus::new(chain_spec.clone())); - - let provider = MockEthProvider::default(); - - let payload_validator = EthereumEngineValidator::new(chain_spec.clone()); - - let (from_tree_tx, from_tree_rx) = unbounded_channel(); - - let header = chain_spec.genesis_header().clone(); - let header = SealedHeader::seal_slow(header); - let engine_api_tree_state = EngineApiTreeState::new(10, 10, header.num_hash()); - let canonical_in_memory_state = CanonicalInMemoryState::with_head(header, None, None); - - let (to_payload_service, _payload_command_rx) = unbounded_channel(); - let payload_builder = PayloadBuilderHandle::new(to_payload_service); - - let evm_config = MockEvmConfig::default(); - - let tree = EngineApiTreeHandler::new( - provider.clone(), - consensus, - payload_validator, - from_tree_tx, - engine_api_tree_state, - canonical_in_memory_state, - persistence_handle, - PersistenceState::default(), - payload_builder, - // TODO: fix tests for state root task https://github.com/paradigmxyz/reth/issues/14376 - // always assume enough parallelism for tests - TreeConfig::default() - .with_legacy_state_root(true) - .with_has_enough_parallelism(true), - EngineApiKind::Ethereum, - evm_config.clone(), - ); - - let block_builder = TestBlockBuilder::default().with_chain_spec((*chain_spec).clone()); - Self { - to_tree_tx: tree.incoming_tx.clone(), - tree, - from_tree_rx, - blocks: vec![], - action_rx, - evm_config, - block_builder, - provider, - } - } - - fn with_blocks(mut self, blocks: Vec) -> Self { - let mut blocks_by_hash = HashMap::default(); - let mut blocks_by_number = BTreeMap::new(); - let mut state_by_hash = HashMap::default(); - let mut hash_by_number = BTreeMap::new(); - let mut parent_to_child: HashMap> = HashMap::default(); - let mut parent_hash = B256::ZERO; - - for block in &blocks { - let sealed_block = block.recovered_block(); - let hash = sealed_block.hash(); - let number = sealed_block.number; - blocks_by_hash.insert(hash, block.clone()); - blocks_by_number.entry(number).or_insert_with(Vec::new).push(block.clone()); - state_by_hash.insert(hash, Arc::new(BlockState::new(block.clone()))); - hash_by_number.insert(number, hash); - parent_to_child.entry(parent_hash).or_default().insert(hash); - parent_hash = hash; - } - - self.tree.state.tree_state = TreeState { - blocks_by_hash, - blocks_by_number, - current_canonical_head: blocks.last().unwrap().recovered_block().num_hash(), - parent_to_child, - persisted_trie_updates: HashMap::default(), - }; - - let last_executed_block = blocks.last().unwrap().clone(); - let pending = Some(BlockState::new(last_executed_block)); - self.tree.canonical_in_memory_state = - CanonicalInMemoryState::new(state_by_hash, hash_by_number, pending, None, None); - - self.blocks = blocks.clone(); - - let recovered_blocks = - blocks.iter().map(|b| b.recovered_block().clone()).collect::>(); - - self.persist_blocks(recovered_blocks); - - self - } - - const fn with_backfill_state(mut self, state: BackfillSyncState) -> Self { - self.tree.backfill_sync_state = state; - self - } - - fn extend_execution_outcome( - &self, - execution_outcomes: impl IntoIterator>, - ) { - self.evm_config.extend(execution_outcomes); - } - - fn insert_block( - &mut self, - block: RecoveredBlock, - ) -> Result> { - let execution_outcome = self.block_builder.get_execution_outcome(block.clone()); - self.extend_execution_outcome([execution_outcome]); - self.tree.provider.add_state_root(block.state_root); - self.tree.insert_block(block) - } - - async fn fcu_to(&mut self, block_hash: B256, fcu_status: impl Into) { - let fcu_status = fcu_status.into(); - - self.send_fcu(block_hash, fcu_status).await; - - self.check_fcu(block_hash, fcu_status).await; - } - - async fn send_fcu(&mut self, block_hash: B256, fcu_status: impl Into) { - let fcu_state = self.fcu_state(block_hash); - - let (tx, rx) = oneshot::channel(); - self.tree - .on_engine_message(FromEngine::Request( - BeaconEngineMessage::ForkchoiceUpdated { - state: fcu_state, - payload_attrs: None, - tx, - version: EngineApiMessageVersion::default(), - } - .into(), - )) - .unwrap(); - - let response = rx.await.unwrap().unwrap().await.unwrap(); - match fcu_status.into() { - ForkchoiceStatus::Valid => assert!(response.payload_status.is_valid()), - ForkchoiceStatus::Syncing => assert!(response.payload_status.is_syncing()), - ForkchoiceStatus::Invalid => assert!(response.payload_status.is_invalid()), - } - } - - async fn check_fcu(&mut self, block_hash: B256, fcu_status: impl Into) { - let fcu_state = self.fcu_state(block_hash); - - // check for ForkchoiceUpdated event - let event = self.from_tree_rx.recv().await.unwrap(); - match event { - EngineApiEvent::BeaconConsensus(BeaconConsensusEngineEvent::ForkchoiceUpdated( - state, - status, - )) => { - assert_eq!(state, fcu_state); - assert_eq!(status, fcu_status.into()); - } - _ => panic!("Unexpected event: {event:#?}"), - } - } - - const fn fcu_state(&self, block_hash: B256) -> ForkchoiceState { - ForkchoiceState { - head_block_hash: block_hash, - safe_block_hash: block_hash, - finalized_block_hash: block_hash, - } - } - - async fn send_new_payload( - &mut self, - block: RecoveredBlock, - ) { - let payload = ExecutionPayloadV3::from_block_unchecked( - block.hash(), - &block.clone_sealed_block().into_block(), - ); - self.tree - .on_new_payload(ExecutionData { - payload: payload.into(), - sidecar: ExecutionPayloadSidecar::v3(CancunPayloadFields { - parent_beacon_block_root: block.parent_beacon_block_root.unwrap(), - versioned_hashes: vec![], - }), - }) - .unwrap(); - } - - async fn insert_chain( - &mut self, - chain: impl IntoIterator> + Clone, - ) { - for block in chain.clone() { - self.insert_block(block.clone()).unwrap(); - } - self.check_canon_chain_insertion(chain).await; - } - - async fn check_canon_commit(&mut self, hash: B256) { - let event = self.from_tree_rx.recv().await.unwrap(); - match event { - EngineApiEvent::BeaconConsensus( - BeaconConsensusEngineEvent::CanonicalChainCommitted(header, _), - ) => { - assert_eq!(header.hash(), hash); - } - _ => panic!("Unexpected event: {event:#?}"), - } - } - - async fn check_fork_chain_insertion( - &mut self, - chain: impl IntoIterator> + Clone, - ) { - for block in chain { - self.check_fork_block_added(block.hash()).await; - } - } - - async fn check_canon_chain_insertion( - &mut self, - chain: impl IntoIterator> + Clone, - ) { - for block in chain.clone() { - self.check_canon_block_added(block.hash()).await; - } - } - - async fn check_canon_block_added(&mut self, expected_hash: B256) { - let event = self.from_tree_rx.recv().await.unwrap(); - match event { - EngineApiEvent::BeaconConsensus( - BeaconConsensusEngineEvent::CanonicalBlockAdded(executed, _), - ) => { - assert_eq!(executed.recovered_block.hash(), expected_hash); - } - _ => panic!("Unexpected event: {event:#?}"), - } - } - - async fn check_fork_block_added(&mut self, expected_hash: B256) { - let event = self.from_tree_rx.recv().await.unwrap(); - match event { - EngineApiEvent::BeaconConsensus(BeaconConsensusEngineEvent::ForkBlockAdded( - executed, - _, - )) => { - assert_eq!(executed.recovered_block.hash(), expected_hash); - } - _ => panic!("Unexpected event: {event:#?}"), - } - } - - async fn check_invalid_block(&mut self, expected_hash: B256) { - let event = self.from_tree_rx.recv().await.unwrap(); - match event { - EngineApiEvent::BeaconConsensus(BeaconConsensusEngineEvent::InvalidBlock( - block, - )) => { - assert_eq!(block.hash(), expected_hash); - } - _ => panic!("Unexpected event: {event:#?}"), - } - } - - fn persist_blocks(&self, blocks: Vec>) { - let mut block_data: Vec<(B256, Block)> = Vec::with_capacity(blocks.len()); - let mut headers_data: Vec<(B256, Header)> = Vec::with_capacity(blocks.len()); - - for block in &blocks { - block_data.push((block.hash(), block.clone_block())); - headers_data.push((block.hash(), block.header().clone())); - } - - self.provider.extend_blocks(block_data); - self.provider.extend_headers(headers_data); - } - - fn setup_range_insertion_for_valid_chain( - &mut self, - chain: Vec>, - ) { - self.setup_range_insertion_for_chain(chain, None) - } - - fn setup_range_insertion_for_invalid_chain( - &mut self, - chain: Vec>, - index: usize, - ) { - self.setup_range_insertion_for_chain(chain, Some(index)) - } - - fn setup_range_insertion_for_chain( - &mut self, - chain: Vec>, - invalid_index: Option, - ) { - // setting up execution outcomes for the chain, the blocks will be - // executed starting from the oldest, so we need to reverse. - let mut chain_rev = chain; - chain_rev.reverse(); - - let mut execution_outcomes = Vec::with_capacity(chain_rev.len()); - for (index, block) in chain_rev.iter().enumerate() { - let execution_outcome = self.block_builder.get_execution_outcome(block.clone()); - let state_root = if invalid_index.is_some() && invalid_index.unwrap() == index { - B256::random() - } else { - block.state_root - }; - self.tree.provider.add_state_root(state_root); - execution_outcomes.push(execution_outcome); - } - self.extend_execution_outcome(execution_outcomes); - } - - fn check_canon_head(&self, head_hash: B256) { - assert_eq!(self.tree.state.tree_state.canonical_head().hash, head_hash); - } - } - - #[test] - fn test_tree_persist_block_batch() { - let tree_config = TreeConfig::default(); - let chain_spec = MAINNET.clone(); - let mut test_block_builder = TestBlockBuilder::eth().with_chain_spec((*chain_spec).clone()); - - // we need more than tree_config.persistence_threshold() +1 blocks to - // trigger the persistence task. - let blocks: Vec<_> = test_block_builder - .get_executed_blocks(1..tree_config.persistence_threshold() + 2) - .collect(); - let mut test_harness = TestHarness::new(chain_spec).with_blocks(blocks); - - let mut blocks = vec![]; - for idx in 0..tree_config.max_execute_block_batch_size() * 2 { - blocks.push(test_block_builder.generate_random_block(idx as u64, B256::random())); - } - - test_harness.to_tree_tx.send(FromEngine::DownloadedBlocks(blocks)).unwrap(); - - // process the message - let msg = test_harness.tree.try_recv_engine_message().unwrap().unwrap(); - test_harness.tree.on_engine_message(msg).unwrap(); - - // we now should receive the other batch - let msg = test_harness.tree.try_recv_engine_message().unwrap().unwrap(); - match msg { - FromEngine::DownloadedBlocks(blocks) => { - assert_eq!(blocks.len(), tree_config.max_execute_block_batch_size()); - } - _ => panic!("unexpected message: {msg:#?}"), - } - } - - #[tokio::test] - async fn test_tree_persist_blocks() { - let tree_config = TreeConfig::default(); - let chain_spec = MAINNET.clone(); - let mut test_block_builder = TestBlockBuilder::eth().with_chain_spec((*chain_spec).clone()); - - // we need more than tree_config.persistence_threshold() +1 blocks to - // trigger the persistence task. - let blocks: Vec<_> = test_block_builder - .get_executed_blocks(1..tree_config.persistence_threshold() + 2) - .collect(); - let test_harness = TestHarness::new(chain_spec).with_blocks(blocks.clone()); - std::thread::Builder::new() - .name("Tree Task".to_string()) - .spawn(|| test_harness.tree.run()) - .unwrap(); - - // send a message to the tree to enter the main loop. - test_harness.to_tree_tx.send(FromEngine::DownloadedBlocks(vec![])).unwrap(); - - let received_action = - test_harness.action_rx.recv().expect("Failed to receive save blocks action"); - if let PersistenceAction::SaveBlocks(saved_blocks, _) = received_action { - // only blocks.len() - tree_config.memory_block_buffer_target() will be - // persisted - let expected_persist_len = - blocks.len() - tree_config.memory_block_buffer_target() as usize; - assert_eq!(saved_blocks.len(), expected_persist_len); - assert_eq!(saved_blocks, blocks[..expected_persist_len]); - } else { - panic!("unexpected action received {received_action:?}"); - } - } - - #[tokio::test] - async fn test_in_memory_state_trait_impl() { - let blocks: Vec<_> = TestBlockBuilder::eth().get_executed_blocks(0..10).collect(); - let test_harness = TestHarness::new(MAINNET.clone()).with_blocks(blocks.clone()); - - for executed_block in blocks { - let sealed_block = executed_block.recovered_block(); - - let expected_state = BlockState::new(executed_block.clone()); - - let actual_state_by_hash = test_harness - .tree - .canonical_in_memory_state - .state_by_hash(sealed_block.hash()) - .unwrap(); - assert_eq!(expected_state, *actual_state_by_hash); - - let actual_state_by_number = test_harness - .tree - .canonical_in_memory_state - .state_by_number(sealed_block.number) - .unwrap(); - assert_eq!(expected_state, *actual_state_by_number); - } - } - - #[tokio::test] - async fn test_engine_request_during_backfill() { - let tree_config = TreeConfig::default(); - let blocks: Vec<_> = TestBlockBuilder::eth() - .get_executed_blocks(0..tree_config.persistence_threshold()) - .collect(); - let mut test_harness = TestHarness::new(MAINNET.clone()) - .with_blocks(blocks) - .with_backfill_state(BackfillSyncState::Active); - - let (tx, rx) = oneshot::channel(); - test_harness - .tree - .on_engine_message(FromEngine::Request( - BeaconEngineMessage::ForkchoiceUpdated { - state: ForkchoiceState { - head_block_hash: B256::random(), - safe_block_hash: B256::random(), - finalized_block_hash: B256::random(), - }, - payload_attrs: None, - tx, - version: EngineApiMessageVersion::default(), - } - .into(), - )) - .unwrap(); - - let resp = rx.await.unwrap().unwrap().await.unwrap(); - assert!(resp.payload_status.is_syncing()); - } - - #[test] - fn test_disconnected_payload() { - let s = include_str!("../../test-data/holesky/2.rlp"); - let data = Bytes::from_str(s).unwrap(); - let block = Block::decode(&mut data.as_ref()).unwrap(); - let sealed = block.seal_slow(); - let hash = sealed.hash(); - let payload = ExecutionPayloadV1::from_block_unchecked(hash, &sealed.clone().into_block()); - - let mut test_harness = TestHarness::new(HOLESKY.clone()); - - let outcome = test_harness - .tree - .on_new_payload(ExecutionData { - payload: payload.into(), - sidecar: ExecutionPayloadSidecar::none(), - }) - .unwrap(); - assert!(outcome.outcome.is_syncing()); - - // ensure block is buffered - let buffered = test_harness.tree.state.buffer.block(&hash).unwrap(); - assert_eq!(buffered.clone_sealed_block(), sealed); - } - - #[test] - fn test_disconnected_block() { - let s = include_str!("../../test-data/holesky/2.rlp"); - let data = Bytes::from_str(s).unwrap(); - let block = Block::decode(&mut data.as_ref()).unwrap(); - let sealed = block.seal_slow().try_recover().unwrap(); - - let mut test_harness = TestHarness::new(HOLESKY.clone()); - - let outcome = test_harness.tree.insert_block(sealed.clone()).unwrap(); - assert_eq!( - outcome, - InsertPayloadOk::Inserted(BlockStatus::Disconnected { - head: test_harness.tree.state.tree_state.current_canonical_head, - missing_ancestor: sealed.parent_num_hash() - }) - ); - } - - #[tokio::test] - async fn test_holesky_payload() { - let s = include_str!("../../test-data/holesky/1.rlp"); - let data = Bytes::from_str(s).unwrap(); - let block: Block = Block::decode(&mut data.as_ref()).unwrap(); - let sealed = block.seal_slow(); - let payload = - ExecutionPayloadV1::from_block_unchecked(sealed.hash(), &sealed.clone().into_block()); - - let mut test_harness = - TestHarness::new(HOLESKY.clone()).with_backfill_state(BackfillSyncState::Active); - - let (tx, rx) = oneshot::channel(); - test_harness - .tree - .on_engine_message(FromEngine::Request( - BeaconEngineMessage::NewPayload { - payload: ExecutionData { - payload: payload.clone().into(), - sidecar: ExecutionPayloadSidecar::none(), - }, - tx, - } - .into(), - )) - .unwrap(); - - let resp = rx.await.unwrap().unwrap(); - assert!(resp.is_syncing()); - } - - #[test] - fn test_tree_state_normal_descendant() { - let mut tree_state = TreeState::new(BlockNumHash::default()); - let blocks: Vec<_> = TestBlockBuilder::eth().get_executed_blocks(1..4).collect(); - - tree_state.insert_executed(blocks[0].clone()); - assert!(tree_state.is_descendant( - blocks[0].recovered_block().num_hash(), - blocks[1].recovered_block().header() - )); - - tree_state.insert_executed(blocks[1].clone()); - - assert!(tree_state.is_descendant( - blocks[0].recovered_block().num_hash(), - blocks[2].recovered_block().header() - )); - assert!(tree_state.is_descendant( - blocks[1].recovered_block().num_hash(), - blocks[2].recovered_block().header() - )); - } - - #[tokio::test] - async fn test_tree_state_insert_executed() { - let mut tree_state = TreeState::new(BlockNumHash::default()); - let blocks: Vec<_> = TestBlockBuilder::eth().get_executed_blocks(1..4).collect(); - - tree_state.insert_executed(blocks[0].clone()); - tree_state.insert_executed(blocks[1].clone()); - - assert_eq!( - tree_state.parent_to_child.get(&blocks[0].recovered_block().hash()), - Some(&HashSet::from_iter([blocks[1].recovered_block().hash()])) - ); - - assert!(!tree_state.parent_to_child.contains_key(&blocks[1].recovered_block().hash())); - - tree_state.insert_executed(blocks[2].clone()); - - assert_eq!( - tree_state.parent_to_child.get(&blocks[1].recovered_block().hash()), - Some(&HashSet::from_iter([blocks[2].recovered_block().hash()])) - ); - assert!(tree_state.parent_to_child.contains_key(&blocks[1].recovered_block().hash())); - - assert!(!tree_state.parent_to_child.contains_key(&blocks[2].recovered_block().hash())); - } - - #[tokio::test] - async fn test_tree_state_insert_executed_with_reorg() { - let mut tree_state = TreeState::new(BlockNumHash::default()); - let mut test_block_builder = TestBlockBuilder::eth(); - let blocks: Vec<_> = test_block_builder.get_executed_blocks(1..6).collect(); - - for block in &blocks { - tree_state.insert_executed(block.clone()); - } - assert_eq!(tree_state.blocks_by_hash.len(), 5); - - let fork_block_3 = test_block_builder - .get_executed_block_with_number(3, blocks[1].recovered_block().hash()); - let fork_block_4 = test_block_builder - .get_executed_block_with_number(4, fork_block_3.recovered_block().hash()); - let fork_block_5 = test_block_builder - .get_executed_block_with_number(5, fork_block_4.recovered_block().hash()); - - tree_state.insert_executed(fork_block_3.clone()); - tree_state.insert_executed(fork_block_4.clone()); - tree_state.insert_executed(fork_block_5.clone()); - - assert_eq!(tree_state.blocks_by_hash.len(), 8); - assert_eq!(tree_state.blocks_by_number[&3].len(), 2); // two blocks at height 3 (original and fork) - assert_eq!(tree_state.parent_to_child[&blocks[1].recovered_block().hash()].len(), 2); // block 2 should have two children - - // verify that we can insert the same block again without issues - tree_state.insert_executed(fork_block_4.clone()); - assert_eq!(tree_state.blocks_by_hash.len(), 8); - - assert!(tree_state.parent_to_child[&fork_block_3.recovered_block().hash()] - .contains(&fork_block_4.recovered_block().hash())); - assert!(tree_state.parent_to_child[&fork_block_4.recovered_block().hash()] - .contains(&fork_block_5.recovered_block().hash())); - - assert_eq!(tree_state.blocks_by_number[&4].len(), 2); - assert_eq!(tree_state.blocks_by_number[&5].len(), 2); - } - - #[tokio::test] - async fn test_tree_state_remove_before() { - let start_num_hash = BlockNumHash::default(); - let mut tree_state = TreeState::new(start_num_hash); - let blocks: Vec<_> = TestBlockBuilder::eth().get_executed_blocks(1..6).collect(); - - for block in &blocks { - tree_state.insert_executed(block.clone()); - } - - let last = blocks.last().unwrap(); - - // set the canonical head - tree_state.set_canonical_head(last.recovered_block().num_hash()); - - // inclusive bound, so we should remove anything up to and including 2 - tree_state.remove_until( - BlockNumHash::new(2, blocks[1].recovered_block().hash()), - start_num_hash.hash, - Some(blocks[1].recovered_block().num_hash()), - ); - - assert!(!tree_state.blocks_by_hash.contains_key(&blocks[0].recovered_block().hash())); - assert!(!tree_state.blocks_by_hash.contains_key(&blocks[1].recovered_block().hash())); - assert!(!tree_state.blocks_by_number.contains_key(&1)); - assert!(!tree_state.blocks_by_number.contains_key(&2)); - - assert!(tree_state.blocks_by_hash.contains_key(&blocks[2].recovered_block().hash())); - assert!(tree_state.blocks_by_hash.contains_key(&blocks[3].recovered_block().hash())); - assert!(tree_state.blocks_by_hash.contains_key(&blocks[4].recovered_block().hash())); - assert!(tree_state.blocks_by_number.contains_key(&3)); - assert!(tree_state.blocks_by_number.contains_key(&4)); - assert!(tree_state.blocks_by_number.contains_key(&5)); - - assert!(!tree_state.parent_to_child.contains_key(&blocks[0].recovered_block().hash())); - assert!(!tree_state.parent_to_child.contains_key(&blocks[1].recovered_block().hash())); - assert!(tree_state.parent_to_child.contains_key(&blocks[2].recovered_block().hash())); - assert!(tree_state.parent_to_child.contains_key(&blocks[3].recovered_block().hash())); - assert!(!tree_state.parent_to_child.contains_key(&blocks[4].recovered_block().hash())); - - assert_eq!( - tree_state.parent_to_child.get(&blocks[2].recovered_block().hash()), - Some(&HashSet::from_iter([blocks[3].recovered_block().hash()])) - ); - assert_eq!( - tree_state.parent_to_child.get(&blocks[3].recovered_block().hash()), - Some(&HashSet::from_iter([blocks[4].recovered_block().hash()])) - ); - } - - #[tokio::test] - async fn test_tree_state_remove_before_finalized() { - let start_num_hash = BlockNumHash::default(); - let mut tree_state = TreeState::new(start_num_hash); - let blocks: Vec<_> = TestBlockBuilder::eth().get_executed_blocks(1..6).collect(); - - for block in &blocks { - tree_state.insert_executed(block.clone()); - } - - let last = blocks.last().unwrap(); - - // set the canonical head - tree_state.set_canonical_head(last.recovered_block().num_hash()); - - // we should still remove everything up to and including 2 - tree_state.remove_until( - BlockNumHash::new(2, blocks[1].recovered_block().hash()), - start_num_hash.hash, - None, - ); - - assert!(!tree_state.blocks_by_hash.contains_key(&blocks[0].recovered_block().hash())); - assert!(!tree_state.blocks_by_hash.contains_key(&blocks[1].recovered_block().hash())); - assert!(!tree_state.blocks_by_number.contains_key(&1)); - assert!(!tree_state.blocks_by_number.contains_key(&2)); - - assert!(tree_state.blocks_by_hash.contains_key(&blocks[2].recovered_block().hash())); - assert!(tree_state.blocks_by_hash.contains_key(&blocks[3].recovered_block().hash())); - assert!(tree_state.blocks_by_hash.contains_key(&blocks[4].recovered_block().hash())); - assert!(tree_state.blocks_by_number.contains_key(&3)); - assert!(tree_state.blocks_by_number.contains_key(&4)); - assert!(tree_state.blocks_by_number.contains_key(&5)); - - assert!(!tree_state.parent_to_child.contains_key(&blocks[0].recovered_block().hash())); - assert!(!tree_state.parent_to_child.contains_key(&blocks[1].recovered_block().hash())); - assert!(tree_state.parent_to_child.contains_key(&blocks[2].recovered_block().hash())); - assert!(tree_state.parent_to_child.contains_key(&blocks[3].recovered_block().hash())); - assert!(!tree_state.parent_to_child.contains_key(&blocks[4].recovered_block().hash())); - - assert_eq!( - tree_state.parent_to_child.get(&blocks[2].recovered_block().hash()), - Some(&HashSet::from_iter([blocks[3].recovered_block().hash()])) - ); - assert_eq!( - tree_state.parent_to_child.get(&blocks[3].recovered_block().hash()), - Some(&HashSet::from_iter([blocks[4].recovered_block().hash()])) - ); - } - - #[tokio::test] - async fn test_tree_state_remove_before_lower_finalized() { - let start_num_hash = BlockNumHash::default(); - let mut tree_state = TreeState::new(start_num_hash); - let blocks: Vec<_> = TestBlockBuilder::eth().get_executed_blocks(1..6).collect(); - - for block in &blocks { - tree_state.insert_executed(block.clone()); - } - - let last = blocks.last().unwrap(); - - // set the canonical head - tree_state.set_canonical_head(last.recovered_block().num_hash()); - - // we have no forks so we should still remove anything up to and including 2 - tree_state.remove_until( - BlockNumHash::new(2, blocks[1].recovered_block().hash()), - start_num_hash.hash, - Some(blocks[0].recovered_block().num_hash()), - ); - - assert!(!tree_state.blocks_by_hash.contains_key(&blocks[0].recovered_block().hash())); - assert!(!tree_state.blocks_by_hash.contains_key(&blocks[1].recovered_block().hash())); - assert!(!tree_state.blocks_by_number.contains_key(&1)); - assert!(!tree_state.blocks_by_number.contains_key(&2)); - - assert!(tree_state.blocks_by_hash.contains_key(&blocks[2].recovered_block().hash())); - assert!(tree_state.blocks_by_hash.contains_key(&blocks[3].recovered_block().hash())); - assert!(tree_state.blocks_by_hash.contains_key(&blocks[4].recovered_block().hash())); - assert!(tree_state.blocks_by_number.contains_key(&3)); - assert!(tree_state.blocks_by_number.contains_key(&4)); - assert!(tree_state.blocks_by_number.contains_key(&5)); - - assert!(!tree_state.parent_to_child.contains_key(&blocks[0].recovered_block().hash())); - assert!(!tree_state.parent_to_child.contains_key(&blocks[1].recovered_block().hash())); - assert!(tree_state.parent_to_child.contains_key(&blocks[2].recovered_block().hash())); - assert!(tree_state.parent_to_child.contains_key(&blocks[3].recovered_block().hash())); - assert!(!tree_state.parent_to_child.contains_key(&blocks[4].recovered_block().hash())); - - assert_eq!( - tree_state.parent_to_child.get(&blocks[2].recovered_block().hash()), - Some(&HashSet::from_iter([blocks[3].recovered_block().hash()])) - ); - assert_eq!( - tree_state.parent_to_child.get(&blocks[3].recovered_block().hash()), - Some(&HashSet::from_iter([blocks[4].recovered_block().hash()])) - ); - } - - #[tokio::test] - async fn test_tree_state_on_new_head_reorg() { - reth_tracing::init_test_tracing(); - let chain_spec = MAINNET.clone(); - - // Set persistence_threshold to 1 - let mut test_harness = TestHarness::new(chain_spec); - test_harness.tree.config = test_harness - .tree - .config - .with_persistence_threshold(1) - .with_memory_block_buffer_target(1); - let mut test_block_builder = TestBlockBuilder::eth(); - let blocks: Vec<_> = test_block_builder.get_executed_blocks(1..6).collect(); - - for block in &blocks { - test_harness.tree.state.tree_state.insert_executed(block.clone()); - } - - // set block 3 as the current canonical head - test_harness - .tree - .state - .tree_state - .set_canonical_head(blocks[2].recovered_block().num_hash()); - - // create a fork from block 2 - let fork_block_3 = test_block_builder - .get_executed_block_with_number(3, blocks[1].recovered_block().hash()); - let fork_block_4 = test_block_builder - .get_executed_block_with_number(4, fork_block_3.recovered_block().hash()); - let fork_block_5 = test_block_builder - .get_executed_block_with_number(5, fork_block_4.recovered_block().hash()); - - test_harness.tree.state.tree_state.insert_executed(fork_block_3.clone()); - test_harness.tree.state.tree_state.insert_executed(fork_block_4.clone()); - test_harness.tree.state.tree_state.insert_executed(fork_block_5.clone()); - - // normal (non-reorg) case - let result = test_harness.tree.on_new_head(blocks[4].recovered_block().hash()).unwrap(); - assert!(matches!(result, Some(NewCanonicalChain::Commit { .. }))); - if let Some(NewCanonicalChain::Commit { new }) = result { - assert_eq!(new.len(), 2); - assert_eq!(new[0].recovered_block().hash(), blocks[3].recovered_block().hash()); - assert_eq!(new[1].recovered_block().hash(), blocks[4].recovered_block().hash()); - } - - // should be a None persistence action before we advance persistence - let current_action = test_harness.tree.persistence_state.current_action(); - assert_eq!(current_action, None); - - // let's attempt to persist and check that it attempts to save blocks - // - // since in-memory block buffer target and persistence_threshold are both 1, this should - // save all but the current tip of the canonical chain (up to blocks[1]) - test_harness.tree.advance_persistence().unwrap(); - let current_action = test_harness.tree.persistence_state.current_action().cloned(); - assert_eq!( - current_action, - Some(CurrentPersistenceAction::SavingBlocks { - highest: blocks[1].recovered_block().num_hash() - }) - ); - - // get rid of the prev action - let received_action = test_harness.action_rx.recv().unwrap(); - let PersistenceAction::SaveBlocks(saved_blocks, sender) = received_action else { - panic!("received wrong action"); - }; - assert_eq!(saved_blocks, vec![blocks[0].clone(), blocks[1].clone()]); - - // send the response so we can advance again - sender.send(Some(blocks[1].recovered_block().num_hash())).unwrap(); - - // we should be persisting blocks[1] because we threw out the prev action - let current_action = test_harness.tree.persistence_state.current_action().cloned(); - assert_eq!( - current_action, - Some(CurrentPersistenceAction::SavingBlocks { - highest: blocks[1].recovered_block().num_hash() - }) - ); - - // after advancing persistence, we should be at `None` for the next action - test_harness.tree.advance_persistence().unwrap(); - let current_action = test_harness.tree.persistence_state.current_action().cloned(); - assert_eq!(current_action, None); - - // reorg case - let result = test_harness.tree.on_new_head(fork_block_5.recovered_block().hash()).unwrap(); - assert!(matches!(result, Some(NewCanonicalChain::Reorg { .. }))); - - if let Some(NewCanonicalChain::Reorg { new, old }) = result { - assert_eq!(new.len(), 3); - assert_eq!(new[0].recovered_block().hash(), fork_block_3.recovered_block().hash()); - assert_eq!(new[1].recovered_block().hash(), fork_block_4.recovered_block().hash()); - assert_eq!(new[2].recovered_block().hash(), fork_block_5.recovered_block().hash()); - - assert_eq!(old.len(), 1); - assert_eq!(old[0].recovered_block().hash(), blocks[2].recovered_block().hash()); - } - - // The canonical block has not changed, so we will not get any active persistence action - test_harness.tree.advance_persistence().unwrap(); - let current_action = test_harness.tree.persistence_state.current_action().cloned(); - assert_eq!(current_action, None); - - // Let's change the canonical head and advance persistence - test_harness - .tree - .state - .tree_state - .set_canonical_head(fork_block_5.recovered_block().num_hash()); - - // The canonical block has changed now, we should get fork_block_4 due to the persistence - // threshold and in memory block buffer target - test_harness.tree.advance_persistence().unwrap(); - let current_action = test_harness.tree.persistence_state.current_action().cloned(); - assert_eq!( - current_action, - Some(CurrentPersistenceAction::SavingBlocks { - highest: fork_block_4.recovered_block().num_hash() - }) - ); - } - - #[test] - fn test_tree_state_on_new_head_deep_fork() { - reth_tracing::init_test_tracing(); - - let chain_spec = MAINNET.clone(); - let mut test_harness = TestHarness::new(chain_spec); - let mut test_block_builder = TestBlockBuilder::eth(); - - let blocks: Vec<_> = test_block_builder.get_executed_blocks(0..5).collect(); - - for block in &blocks { - test_harness.tree.state.tree_state.insert_executed(block.clone()); - } - - // set last block as the current canonical head - let last_block = blocks.last().unwrap().recovered_block().clone(); - - test_harness.tree.state.tree_state.set_canonical_head(last_block.num_hash()); - - // create a fork chain from last_block - let chain_a = test_block_builder.create_fork(&last_block, 10); - let chain_b = test_block_builder.create_fork(&last_block, 10); - - for block in &chain_a { - test_harness.tree.state.tree_state.insert_executed(ExecutedBlockWithTrieUpdates { - block: ExecutedBlock { - recovered_block: Arc::new(block.clone()), - execution_output: Arc::new(ExecutionOutcome::default()), - hashed_state: Arc::new(HashedPostState::default()), - }, - trie: Arc::new(TrieUpdates::default()), - }); - } - test_harness.tree.state.tree_state.set_canonical_head(chain_a.last().unwrap().num_hash()); - - for block in &chain_b { - test_harness.tree.state.tree_state.insert_executed(ExecutedBlockWithTrieUpdates { - block: ExecutedBlock { - recovered_block: Arc::new(block.clone()), - execution_output: Arc::new(ExecutionOutcome::default()), - hashed_state: Arc::new(HashedPostState::default()), - }, - trie: Arc::new(TrieUpdates::default()), - }); - } - - // for each block in chain_b, reorg to it and then back to canonical - let mut expected_new = Vec::new(); - for block in &chain_b { - // reorg to chain from block b - let result = test_harness.tree.on_new_head(block.hash()).unwrap(); - assert_matches!(result, Some(NewCanonicalChain::Reorg { .. })); - - expected_new.push(block); - if let Some(NewCanonicalChain::Reorg { new, old }) = result { - assert_eq!(new.len(), expected_new.len()); - for (index, block) in expected_new.iter().enumerate() { - assert_eq!(new[index].recovered_block().hash(), block.hash()); - } - - assert_eq!(old.len(), chain_a.len()); - for (index, block) in chain_a.iter().enumerate() { - assert_eq!(old[index].recovered_block().hash(), block.hash()); - } - } - - // set last block of chain a as canonical head - test_harness.tree.on_new_head(chain_a.last().unwrap().hash()).unwrap(); - } - } - - #[tokio::test] - async fn test_get_canonical_blocks_to_persist() { - let chain_spec = MAINNET.clone(); - let mut test_harness = TestHarness::new(chain_spec); - let mut test_block_builder = TestBlockBuilder::eth(); - - let canonical_head_number = 9; - let blocks: Vec<_> = - test_block_builder.get_executed_blocks(0..canonical_head_number + 1).collect(); - test_harness = test_harness.with_blocks(blocks.clone()); - - let last_persisted_block_number = 3; - test_harness.tree.persistence_state.last_persisted_block = - blocks[last_persisted_block_number as usize].recovered_block.num_hash(); - - let persistence_threshold = 4; - let memory_block_buffer_target = 3; - test_harness.tree.config = TreeConfig::default() - .with_persistence_threshold(persistence_threshold) - .with_memory_block_buffer_target(memory_block_buffer_target); - - let blocks_to_persist = test_harness.tree.get_canonical_blocks_to_persist(); - - let expected_blocks_to_persist_length: usize = - (canonical_head_number - memory_block_buffer_target - last_persisted_block_number) - .try_into() - .unwrap(); - - assert_eq!(blocks_to_persist.len(), expected_blocks_to_persist_length); - for (i, item) in - blocks_to_persist.iter().enumerate().take(expected_blocks_to_persist_length) - { - assert_eq!(item.recovered_block().number, last_persisted_block_number + i as u64 + 1); - } - - // make sure only canonical blocks are included - let fork_block = test_block_builder.get_executed_block_with_number(4, B256::random()); - let fork_block_hash = fork_block.recovered_block().hash(); - test_harness.tree.state.tree_state.insert_executed(fork_block); - - assert!(test_harness.tree.state.tree_state.block_by_hash(fork_block_hash).is_some()); - - let blocks_to_persist = test_harness.tree.get_canonical_blocks_to_persist(); - assert_eq!(blocks_to_persist.len(), expected_blocks_to_persist_length); - - // check that the fork block is not included in the blocks to persist - assert!(!blocks_to_persist.iter().any(|b| b.recovered_block().hash() == fork_block_hash)); - - // check that the original block 4 is still included - assert!(blocks_to_persist.iter().any(|b| b.recovered_block().number == 4 && - b.recovered_block().hash() == blocks[4].recovered_block().hash())); - - // check that if we advance persistence, the persistence action is the correct value - test_harness.tree.advance_persistence().expect("advancing persistence should succeed"); - assert_eq!( - test_harness.tree.persistence_state.current_action().cloned(), - Some(CurrentPersistenceAction::SavingBlocks { - highest: blocks_to_persist.last().unwrap().recovered_block().num_hash() - }) - ); - } - - #[tokio::test] - async fn test_engine_tree_fcu_missing_head() { - let chain_spec = MAINNET.clone(); - let mut test_harness = TestHarness::new(chain_spec.clone()); - - let mut test_block_builder = TestBlockBuilder::eth().with_chain_spec((*chain_spec).clone()); - - let blocks: Vec<_> = test_block_builder.get_executed_blocks(0..5).collect(); - test_harness = test_harness.with_blocks(blocks); - - let missing_block = test_block_builder - .generate_random_block(6, test_harness.blocks.last().unwrap().recovered_block().hash()); - - test_harness.fcu_to(missing_block.hash(), PayloadStatusEnum::Syncing).await; - - // after FCU we receive an EngineApiEvent::Download event to get the missing block. - let event = test_harness.from_tree_rx.recv().await.unwrap(); - match event { - EngineApiEvent::Download(DownloadRequest::BlockSet(actual_block_set)) => { - let expected_block_set = HashSet::from_iter([missing_block.hash()]); - assert_eq!(actual_block_set, expected_block_set); - } - _ => panic!("Unexpected event: {event:#?}"), - } - } - - #[tokio::test] - async fn test_engine_tree_fcu_canon_chain_insertion() { - let chain_spec = MAINNET.clone(); - let mut test_harness = TestHarness::new(chain_spec.clone()); - - let base_chain: Vec<_> = test_harness.block_builder.get_executed_blocks(0..1).collect(); - test_harness = test_harness.with_blocks(base_chain.clone()); - - test_harness - .fcu_to(base_chain.last().unwrap().recovered_block().hash(), ForkchoiceStatus::Valid) - .await; - - // extend main chain - let main_chain = test_harness.block_builder.create_fork(base_chain[0].recovered_block(), 3); - - test_harness.insert_chain(main_chain).await; - } - - #[tokio::test] - async fn test_engine_tree_fcu_reorg_with_all_blocks() { - let chain_spec = MAINNET.clone(); - let mut test_harness = TestHarness::new(chain_spec.clone()); - - let main_chain: Vec<_> = test_harness.block_builder.get_executed_blocks(0..5).collect(); - test_harness = test_harness.with_blocks(main_chain.clone()); - - let fork_chain = test_harness.block_builder.create_fork(main_chain[2].recovered_block(), 3); - let fork_chain_last_hash = fork_chain.last().unwrap().hash(); - - // add fork blocks to the tree - for block in &fork_chain { - test_harness.insert_block(block.clone()).unwrap(); - } - - test_harness.send_fcu(fork_chain_last_hash, ForkchoiceStatus::Valid).await; - - // check for ForkBlockAdded events, we expect fork_chain.len() blocks added - test_harness.check_fork_chain_insertion(fork_chain.clone()).await; - - // check for CanonicalChainCommitted event - test_harness.check_canon_commit(fork_chain_last_hash).await; - - test_harness.check_fcu(fork_chain_last_hash, ForkchoiceStatus::Valid).await; - - // new head is the tip of the fork chain - test_harness.check_canon_head(fork_chain_last_hash); - } - - #[tokio::test] - async fn test_engine_tree_live_sync_transition_required_blocks_requested() { - reth_tracing::init_test_tracing(); - - let chain_spec = MAINNET.clone(); - let mut test_harness = TestHarness::new(chain_spec.clone()); - - let base_chain: Vec<_> = test_harness.block_builder.get_executed_blocks(0..1).collect(); - test_harness = test_harness.with_blocks(base_chain.clone()); - - test_harness - .fcu_to(base_chain.last().unwrap().recovered_block().hash(), ForkchoiceStatus::Valid) - .await; - - // extend main chain with enough blocks to trigger pipeline run but don't insert them - let main_chain = test_harness - .block_builder - .create_fork(base_chain[0].recovered_block(), MIN_BLOCKS_FOR_PIPELINE_RUN + 10); - - let main_chain_last_hash = main_chain.last().unwrap().hash(); - test_harness.send_fcu(main_chain_last_hash, ForkchoiceStatus::Syncing).await; - - test_harness.check_fcu(main_chain_last_hash, ForkchoiceStatus::Syncing).await; - - // create event for backfill finished - let backfill_finished_block_number = MIN_BLOCKS_FOR_PIPELINE_RUN + 1; - let backfill_finished = FromOrchestrator::BackfillSyncFinished(ControlFlow::Continue { - block_number: backfill_finished_block_number, - }); - - let backfill_tip_block = main_chain[(backfill_finished_block_number - 1) as usize].clone(); - // add block to mock provider to enable persistence clean up. - test_harness.provider.add_block(backfill_tip_block.hash(), backfill_tip_block.into_block()); - test_harness.tree.on_engine_message(FromEngine::Event(backfill_finished)).unwrap(); - - let event = test_harness.from_tree_rx.recv().await.unwrap(); - match event { - EngineApiEvent::Download(DownloadRequest::BlockSet(hash_set)) => { - assert_eq!(hash_set, HashSet::from_iter([main_chain_last_hash])); - } - _ => panic!("Unexpected event: {event:#?}"), - } - - test_harness - .tree - .on_engine_message(FromEngine::DownloadedBlocks(vec![main_chain - .last() - .unwrap() - .clone()])) - .unwrap(); - - let event = test_harness.from_tree_rx.recv().await.unwrap(); - match event { - EngineApiEvent::Download(DownloadRequest::BlockRange(initial_hash, total_blocks)) => { - assert_eq!( - total_blocks, - (main_chain.len() - backfill_finished_block_number as usize - 1) as u64 - ); - assert_eq!(initial_hash, main_chain.last().unwrap().parent_hash); - } - _ => panic!("Unexpected event: {event:#?}"), - } - } - - #[tokio::test] - async fn test_engine_tree_live_sync_transition_eventually_canonical() { - reth_tracing::init_test_tracing(); - - let chain_spec = MAINNET.clone(); - let mut test_harness = TestHarness::new(chain_spec.clone()); - test_harness.tree.config = test_harness.tree.config.with_max_execute_block_batch_size(100); - - // create base chain and setup test harness with it - let base_chain: Vec<_> = test_harness.block_builder.get_executed_blocks(0..1).collect(); - test_harness = test_harness.with_blocks(base_chain.clone()); - - // fcu to the tip of base chain - test_harness - .fcu_to(base_chain.last().unwrap().recovered_block().hash(), ForkchoiceStatus::Valid) - .await; - - // create main chain, extension of base chain, with enough blocks to - // trigger backfill sync - let main_chain = test_harness - .block_builder - .create_fork(base_chain[0].recovered_block(), MIN_BLOCKS_FOR_PIPELINE_RUN + 10); - - let main_chain_last = main_chain.last().unwrap(); - let main_chain_last_hash = main_chain_last.hash(); - let main_chain_backfill_target = - main_chain.get(MIN_BLOCKS_FOR_PIPELINE_RUN as usize).unwrap(); - let main_chain_backfill_target_hash = main_chain_backfill_target.hash(); - - // fcu to the element of main chain that should trigger backfill sync - test_harness.send_fcu(main_chain_backfill_target_hash, ForkchoiceStatus::Syncing).await; - test_harness.check_fcu(main_chain_backfill_target_hash, ForkchoiceStatus::Syncing).await; - - // check download request for target - let event = test_harness.from_tree_rx.recv().await.unwrap(); - match event { - EngineApiEvent::Download(DownloadRequest::BlockSet(hash_set)) => { - assert_eq!(hash_set, HashSet::from_iter([main_chain_backfill_target_hash])); - } - _ => panic!("Unexpected event: {event:#?}"), - } - - // send message to tell the engine the requested block was downloaded - test_harness - .tree - .on_engine_message(FromEngine::DownloadedBlocks(vec![ - main_chain_backfill_target.clone() - ])) - .unwrap(); - - // check that backfill is triggered - let event = test_harness.from_tree_rx.recv().await.unwrap(); - match event { - EngineApiEvent::BackfillAction(BackfillAction::Start( - reth_stages::PipelineTarget::Sync(target_hash), - )) => { - assert_eq!(target_hash, main_chain_backfill_target_hash); - } - _ => panic!("Unexpected event: {event:#?}"), - } - - // persist blocks of main chain, same as the backfill operation would do - let backfilled_chain: Vec<_> = - main_chain.clone().drain(0..(MIN_BLOCKS_FOR_PIPELINE_RUN + 1) as usize).collect(); - test_harness.persist_blocks(backfilled_chain.clone()); - - test_harness.setup_range_insertion_for_valid_chain(backfilled_chain); - - // send message to mark backfill finished - test_harness - .tree - .on_engine_message(FromEngine::Event(FromOrchestrator::BackfillSyncFinished( - ControlFlow::Continue { block_number: main_chain_backfill_target.number }, - ))) - .unwrap(); - - // send fcu to the tip of main - test_harness.fcu_to(main_chain_last_hash, ForkchoiceStatus::Syncing).await; - - let event = test_harness.from_tree_rx.recv().await.unwrap(); - match event { - EngineApiEvent::Download(DownloadRequest::BlockSet(target_hash)) => { - assert_eq!(target_hash, HashSet::from_iter([main_chain_last_hash])); - } - _ => panic!("Unexpected event: {event:#?}"), - } - - // tell engine main chain tip downloaded - test_harness - .tree - .on_engine_message(FromEngine::DownloadedBlocks(vec![main_chain_last.clone()])) - .unwrap(); - - // check download range request - let event = test_harness.from_tree_rx.recv().await.unwrap(); - match event { - EngineApiEvent::Download(DownloadRequest::BlockRange(initial_hash, total_blocks)) => { - assert_eq!( - total_blocks, - (main_chain.len() - MIN_BLOCKS_FOR_PIPELINE_RUN as usize - 2) as u64 - ); - assert_eq!(initial_hash, main_chain_last.parent_hash); - } - _ => panic!("Unexpected event: {event:#?}"), - } - - let remaining: Vec<_> = main_chain - .clone() - .drain((MIN_BLOCKS_FOR_PIPELINE_RUN + 1) as usize..main_chain.len()) - .collect(); - - test_harness.setup_range_insertion_for_valid_chain(remaining.clone()); - - // tell engine block range downloaded - test_harness - .tree - .on_engine_message(FromEngine::DownloadedBlocks(remaining.clone())) - .unwrap(); - - test_harness.check_canon_chain_insertion(remaining).await; - - // check canonical chain committed event with the hash of the latest block - test_harness.check_canon_commit(main_chain_last_hash).await; - - // new head is the tip of the main chain - test_harness.check_canon_head(main_chain_last_hash); - } - - #[tokio::test] - async fn test_engine_tree_live_sync_fcu_extends_canon_chain() { - reth_tracing::init_test_tracing(); - - let chain_spec = MAINNET.clone(); - let mut test_harness = TestHarness::new(chain_spec.clone()); - - // create base chain and setup test harness with it - let base_chain: Vec<_> = test_harness.block_builder.get_executed_blocks(0..1).collect(); - test_harness = test_harness.with_blocks(base_chain.clone()); - - // fcu to the tip of base chain - test_harness - .fcu_to(base_chain.last().unwrap().recovered_block().hash(), ForkchoiceStatus::Valid) - .await; - - // create main chain, extension of base chain - let main_chain = - test_harness.block_builder.create_fork(base_chain[0].recovered_block(), 10); - // determine target in the middle of main hain - let target = main_chain.get(5).unwrap(); - let target_hash = target.hash(); - let main_last = main_chain.last().unwrap(); - let main_last_hash = main_last.hash(); - - // insert main chain - test_harness.insert_chain(main_chain).await; - - // send fcu to target - test_harness.send_fcu(target_hash, ForkchoiceStatus::Valid).await; - - test_harness.check_canon_commit(target_hash).await; - test_harness.check_fcu(target_hash, ForkchoiceStatus::Valid).await; - - // send fcu to main tip - test_harness.send_fcu(main_last_hash, ForkchoiceStatus::Valid).await; - - test_harness.check_canon_commit(main_last_hash).await; - test_harness.check_fcu(main_last_hash, ForkchoiceStatus::Valid).await; - test_harness.check_canon_head(main_last_hash); - } - - #[tokio::test] - async fn test_engine_tree_valid_forks_with_older_canonical_head() { - reth_tracing::init_test_tracing(); - - let chain_spec = MAINNET.clone(); - let mut test_harness = TestHarness::new(chain_spec.clone()); - - // create base chain and setup test harness with it - let base_chain: Vec<_> = test_harness.block_builder.get_executed_blocks(0..1).collect(); - test_harness = test_harness.with_blocks(base_chain.clone()); - - let old_head = base_chain.first().unwrap().recovered_block(); - - // extend base chain - let extension_chain = test_harness.block_builder.create_fork(old_head, 5); - let fork_block = extension_chain.last().unwrap().clone_sealed_block(); - - test_harness.setup_range_insertion_for_valid_chain(extension_chain.clone()); - test_harness.insert_chain(extension_chain).await; - - // fcu to old_head - test_harness.fcu_to(old_head.hash(), ForkchoiceStatus::Valid).await; - - // create two competing chains starting from fork_block - let chain_a = test_harness.block_builder.create_fork(&fork_block, 10); - let chain_b = test_harness.block_builder.create_fork(&fork_block, 10); - - // insert chain A blocks using newPayload - test_harness.setup_range_insertion_for_valid_chain(chain_a.clone()); - for block in &chain_a { - test_harness.send_new_payload(block.clone()).await; - } - - test_harness.check_canon_chain_insertion(chain_a.clone()).await; - - // insert chain B blocks using newPayload - test_harness.setup_range_insertion_for_valid_chain(chain_b.clone()); - for block in &chain_b { - test_harness.send_new_payload(block.clone()).await; - } - - test_harness.check_canon_chain_insertion(chain_b.clone()).await; - - // send FCU to make the tip of chain B the new head - let chain_b_tip_hash = chain_b.last().unwrap().hash(); - test_harness.send_fcu(chain_b_tip_hash, ForkchoiceStatus::Valid).await; - - // check for CanonicalChainCommitted event - test_harness.check_canon_commit(chain_b_tip_hash).await; - - // verify FCU was processed - test_harness.check_fcu(chain_b_tip_hash, ForkchoiceStatus::Valid).await; - - // verify the new canonical head - test_harness.check_canon_head(chain_b_tip_hash); - - // verify that chain A is now considered a fork - assert!(test_harness.tree.is_fork(chain_a.last().unwrap().hash()).unwrap()); - } - - #[tokio::test] - async fn test_engine_tree_buffered_blocks_are_eventually_connected() { - let chain_spec = MAINNET.clone(); - let mut test_harness = TestHarness::new(chain_spec.clone()); - - let base_chain: Vec<_> = test_harness.block_builder.get_executed_blocks(0..1).collect(); - test_harness = test_harness.with_blocks(base_chain.clone()); - - // side chain consisting of two blocks, the last will be inserted first - // so that we force it to be buffered - let side_chain = - test_harness.block_builder.create_fork(base_chain.last().unwrap().recovered_block(), 2); - - // buffer last block of side chain - let buffered_block = side_chain.last().unwrap(); - let buffered_block_hash = buffered_block.hash(); - - test_harness.setup_range_insertion_for_valid_chain(vec![buffered_block.clone()]); - test_harness.send_new_payload(buffered_block.clone()).await; - - assert!(test_harness.tree.state.buffer.block(&buffered_block_hash).is_some()); - - let non_buffered_block = side_chain.first().unwrap(); - let non_buffered_block_hash = non_buffered_block.hash(); - - // insert block that continues the canon chain, should not be buffered - test_harness.setup_range_insertion_for_valid_chain(vec![non_buffered_block.clone()]); - test_harness.send_new_payload(non_buffered_block.clone()).await; - assert!(test_harness.tree.state.buffer.block(&non_buffered_block_hash).is_none()); - - // the previously buffered block should be connected now - assert!(test_harness.tree.state.buffer.block(&buffered_block_hash).is_none()); - - // both blocks are added to the canon chain in order - test_harness.check_canon_block_added(non_buffered_block_hash).await; - test_harness.check_canon_block_added(buffered_block_hash).await; - } - - #[tokio::test] - async fn test_engine_tree_valid_and_invalid_forks_with_older_canonical_head() { - reth_tracing::init_test_tracing(); - - let chain_spec = MAINNET.clone(); - let mut test_harness = TestHarness::new(chain_spec.clone()); - - // create base chain and setup test harness with it - let base_chain: Vec<_> = test_harness.block_builder.get_executed_blocks(0..1).collect(); - test_harness = test_harness.with_blocks(base_chain.clone()); - - let old_head = base_chain.first().unwrap().recovered_block(); - - // extend base chain - let extension_chain = test_harness.block_builder.create_fork(old_head, 5); - let fork_block = extension_chain.last().unwrap().clone_sealed_block(); - test_harness.insert_chain(extension_chain).await; - - // fcu to old_head - test_harness.fcu_to(old_head.hash(), ForkchoiceStatus::Valid).await; - - // create two competing chains starting from fork_block, one of them invalid - let total_fork_elements = 10; - let chain_a = test_harness.block_builder.create_fork(&fork_block, total_fork_elements); - let chain_b = test_harness.block_builder.create_fork(&fork_block, total_fork_elements); - - // insert chain B blocks using newPayload - test_harness.setup_range_insertion_for_valid_chain(chain_b.clone()); - for block in &chain_b { - test_harness.send_new_payload(block.clone()).await; - test_harness.send_fcu(block.hash(), ForkchoiceStatus::Valid).await; - test_harness.check_canon_block_added(block.hash()).await; - test_harness.check_canon_commit(block.hash()).await; - test_harness.check_fcu(block.hash(), ForkchoiceStatus::Valid).await; - } - - // insert chain A blocks using newPayload, one of the blocks will be invalid - let invalid_index = 3; - test_harness.setup_range_insertion_for_invalid_chain(chain_a.clone(), invalid_index); - for block in &chain_a { - test_harness.send_new_payload(block.clone()).await; - } - - // check canon chain insertion up to the invalid index and taking into - // account reversed ordering - test_harness - .check_fork_chain_insertion( - chain_a[..chain_a.len() - invalid_index - 1].iter().cloned(), - ) - .await; - for block in &chain_a[chain_a.len() - invalid_index - 1..] { - test_harness.check_invalid_block(block.hash()).await; - } - - // send FCU to make the tip of chain A, expect invalid - let chain_a_tip_hash = chain_a.last().unwrap().hash(); - test_harness.fcu_to(chain_a_tip_hash, ForkchoiceStatus::Invalid).await; - - // send FCU to make the tip of chain B the new head - let chain_b_tip_hash = chain_b.last().unwrap().hash(); - - // verify the new canonical head - test_harness.check_canon_head(chain_b_tip_hash); - - // verify the canonical head didn't change - test_harness.check_canon_head(chain_b_tip_hash); - } - - #[tokio::test] - async fn test_engine_tree_reorg_with_missing_ancestor_expecting_valid() { - reth_tracing::init_test_tracing(); - let chain_spec = MAINNET.clone(); - let mut test_harness = TestHarness::new(chain_spec.clone()); - - let base_chain: Vec<_> = test_harness.block_builder.get_executed_blocks(0..6).collect(); - test_harness = test_harness.with_blocks(base_chain.clone()); - - // create a side chain with an invalid block - let side_chain = test_harness - .block_builder - .create_fork(base_chain.last().unwrap().recovered_block(), 15); - let invalid_index = 9; - - test_harness.setup_range_insertion_for_invalid_chain(side_chain.clone(), invalid_index); - - for (index, block) in side_chain.iter().enumerate() { - test_harness.send_new_payload(block.clone()).await; - - if index < side_chain.len() - invalid_index - 1 { - test_harness.send_fcu(block.hash(), ForkchoiceStatus::Valid).await; - } - } - - // Try to do a forkchoice update to a block after the invalid one - let fork_tip_hash = side_chain.last().unwrap().hash(); - test_harness.send_fcu(fork_tip_hash, ForkchoiceStatus::Invalid).await; - } -} diff --git a/crates/engine/tree/src/tree/payload_processor/configured_sparse_trie.rs b/crates/engine/tree/src/tree/payload_processor/configured_sparse_trie.rs new file mode 100644 index 00000000000..b587a721398 --- /dev/null +++ b/crates/engine/tree/src/tree/payload_processor/configured_sparse_trie.rs @@ -0,0 +1,188 @@ +//! Configured sparse trie enum for switching between serial and parallel implementations. + +use alloy_primitives::B256; +use reth_trie::{Nibbles, TrieNode}; +use reth_trie_sparse::{ + errors::SparseTrieResult, provider::TrieNodeProvider, LeafLookup, LeafLookupError, + RevealedSparseNode, SerialSparseTrie, SparseTrieInterface, SparseTrieUpdates, TrieMasks, +}; +use reth_trie_sparse_parallel::ParallelSparseTrie; +use std::borrow::Cow; + +/// Enum for switching between serial and parallel sparse trie implementations. +/// +/// This type allows runtime selection between different sparse trie implementations, +/// providing flexibility in choosing the appropriate implementation based on workload +/// characteristics. +#[derive(Debug, Clone)] +pub(crate) enum ConfiguredSparseTrie { + /// Serial implementation of the sparse trie. + Serial(Box), + /// Parallel implementation of the sparse trie. + Parallel(Box), +} + +impl From for ConfiguredSparseTrie { + fn from(trie: SerialSparseTrie) -> Self { + Self::Serial(Box::new(trie)) + } +} + +impl From for ConfiguredSparseTrie { + fn from(trie: ParallelSparseTrie) -> Self { + Self::Parallel(Box::new(trie)) + } +} + +impl Default for ConfiguredSparseTrie { + fn default() -> Self { + Self::Serial(Default::default()) + } +} + +impl SparseTrieInterface for ConfiguredSparseTrie { + fn with_root( + self, + root: TrieNode, + masks: TrieMasks, + retain_updates: bool, + ) -> SparseTrieResult { + match self { + Self::Serial(trie) => { + trie.with_root(root, masks, retain_updates).map(|t| Self::Serial(Box::new(t))) + } + Self::Parallel(trie) => { + trie.with_root(root, masks, retain_updates).map(|t| Self::Parallel(Box::new(t))) + } + } + } + + fn with_updates(self, retain_updates: bool) -> Self { + match self { + Self::Serial(trie) => Self::Serial(Box::new(trie.with_updates(retain_updates))), + Self::Parallel(trie) => Self::Parallel(Box::new(trie.with_updates(retain_updates))), + } + } + + fn reserve_nodes(&mut self, additional: usize) { + match self { + Self::Serial(trie) => trie.reserve_nodes(additional), + Self::Parallel(trie) => trie.reserve_nodes(additional), + } + } + + fn reveal_node( + &mut self, + path: Nibbles, + node: TrieNode, + masks: TrieMasks, + ) -> SparseTrieResult<()> { + match self { + Self::Serial(trie) => trie.reveal_node(path, node, masks), + Self::Parallel(trie) => trie.reveal_node(path, node, masks), + } + } + + fn reveal_nodes(&mut self, nodes: Vec) -> SparseTrieResult<()> { + match self { + Self::Serial(trie) => trie.reveal_nodes(nodes), + Self::Parallel(trie) => trie.reveal_nodes(nodes), + } + } + + fn update_leaf( + &mut self, + full_path: Nibbles, + value: Vec, + provider: P, + ) -> SparseTrieResult<()> { + match self { + Self::Serial(trie) => trie.update_leaf(full_path, value, provider), + Self::Parallel(trie) => trie.update_leaf(full_path, value, provider), + } + } + + fn remove_leaf( + &mut self, + full_path: &Nibbles, + provider: P, + ) -> SparseTrieResult<()> { + match self { + Self::Serial(trie) => trie.remove_leaf(full_path, provider), + Self::Parallel(trie) => trie.remove_leaf(full_path, provider), + } + } + + fn root(&mut self) -> B256 { + match self { + Self::Serial(trie) => trie.root(), + Self::Parallel(trie) => trie.root(), + } + } + + fn update_subtrie_hashes(&mut self) { + match self { + Self::Serial(trie) => trie.update_subtrie_hashes(), + Self::Parallel(trie) => trie.update_subtrie_hashes(), + } + } + + fn get_leaf_value(&self, full_path: &Nibbles) -> Option<&Vec> { + match self { + Self::Serial(trie) => trie.get_leaf_value(full_path), + Self::Parallel(trie) => trie.get_leaf_value(full_path), + } + } + + fn find_leaf( + &self, + full_path: &Nibbles, + expected_value: Option<&Vec>, + ) -> Result { + match self { + Self::Serial(trie) => trie.find_leaf(full_path, expected_value), + Self::Parallel(trie) => trie.find_leaf(full_path, expected_value), + } + } + + fn take_updates(&mut self) -> SparseTrieUpdates { + match self { + Self::Serial(trie) => trie.take_updates(), + Self::Parallel(trie) => trie.take_updates(), + } + } + + fn wipe(&mut self) { + match self { + Self::Serial(trie) => trie.wipe(), + Self::Parallel(trie) => trie.wipe(), + } + } + + fn clear(&mut self) { + match self { + Self::Serial(trie) => trie.clear(), + Self::Parallel(trie) => trie.clear(), + } + } + + fn updates_ref(&self) -> Cow<'_, SparseTrieUpdates> { + match self { + Self::Serial(trie) => trie.updates_ref(), + Self::Parallel(trie) => trie.updates_ref(), + } + } + fn shrink_nodes_to(&mut self, size: usize) { + match self { + Self::Serial(trie) => trie.shrink_nodes_to(size), + Self::Parallel(trie) => trie.shrink_nodes_to(size), + } + } + + fn shrink_values_to(&mut self, size: usize) { + match self { + Self::Serial(trie) => trie.shrink_values_to(size), + Self::Parallel(trie) => trie.shrink_values_to(size), + } + } +} diff --git a/crates/engine/tree/src/tree/payload_processor/executor.rs b/crates/engine/tree/src/tree/payload_processor/executor.rs index c7fb37f8a98..28165d5e8f2 100644 --- a/crates/engine/tree/src/tree/payload_processor/executor.rs +++ b/crates/engine/tree/src/tree/payload_processor/executor.rs @@ -1,17 +1,15 @@ //! Executor for mixed I/O and CPU workloads. -use rayon::ThreadPool as RayonPool; -use std::sync::{Arc, OnceLock}; +use std::{sync::OnceLock, time::Duration}; use tokio::{ - runtime::{Handle, Runtime}, + runtime::{Builder, Handle, Runtime}, task::JoinHandle, }; /// An executor for mixed I/O and CPU workloads. /// -/// This type has access to its own rayon pool and uses tokio to spawn blocking tasks. -/// -/// It will reuse an existing tokio runtime if available or create its own. +/// This type uses tokio to spawn blocking tasks and will reuse an existing tokio +/// runtime if available or create its own. #[derive(Debug, Clone)] pub struct WorkloadExecutor { inner: WorkloadExecutorInner, @@ -19,21 +17,11 @@ pub struct WorkloadExecutor { impl Default for WorkloadExecutor { fn default() -> Self { - Self { inner: WorkloadExecutorInner::new(rayon::ThreadPoolBuilder::new().build().unwrap()) } + Self { inner: WorkloadExecutorInner::new() } } } impl WorkloadExecutor { - /// Creates a new executor with the given number of threads for cpu bound work (rayon). - #[expect(unused)] - pub(super) fn with_num_cpu_threads(cpu_threads: usize) -> Self { - Self { - inner: WorkloadExecutorInner::new( - rayon::ThreadPoolBuilder::new().num_threads(cpu_threads).build().unwrap(), - ), - } - } - /// Returns the handle to the tokio runtime pub(super) const fn handle(&self) -> &Handle { &self.inner.handle @@ -48,33 +36,38 @@ impl WorkloadExecutor { { self.inner.handle.spawn_blocking(func) } - - /// Returns access to the rayon pool - #[expect(unused)] - pub(super) const fn rayon_pool(&self) -> &Arc { - &self.inner.rayon_pool - } } #[derive(Debug, Clone)] struct WorkloadExecutorInner { handle: Handle, - rayon_pool: Arc, } impl WorkloadExecutorInner { - fn new(rayon_pool: rayon::ThreadPool) -> Self { + fn new() -> Self { fn get_runtime_handle() -> Handle { Handle::try_current().unwrap_or_else(|_| { - // Create a new runtime if now runtime is available + // Create a new runtime if no runtime is available static RT: OnceLock = OnceLock::new(); - let rt = RT.get_or_init(|| Runtime::new().unwrap()); + let rt = RT.get_or_init(|| { + Builder::new_multi_thread() + .enable_all() + // Keep the threads alive for at least the block time, which is 12 seconds + // at the time of writing, plus a little extra. + // + // This is to prevent the costly process of spawning new threads on every + // new block, and instead reuse the existing + // threads. + .thread_keep_alive(Duration::from_secs(15)) + .build() + .unwrap() + }); rt.handle().clone() }) } - Self { handle: get_runtime_handle(), rayon_pool: Arc::new(rayon_pool) } + Self { handle: get_runtime_handle() } } } diff --git a/crates/engine/tree/src/tree/payload_processor/mod.rs b/crates/engine/tree/src/tree/payload_processor/mod.rs index 6ecd529ddcf..d1f7531e9dd 100644 --- a/crates/engine/tree/src/tree/payload_processor/mod.rs +++ b/crates/engine/tree/src/tree/payload_processor/mod.rs @@ -1,7 +1,11 @@ //! Entrypoint for payload processing. +use super::precompile_cache::PrecompileCacheMap; use crate::tree::{ - cached_state::{CachedStateMetrics, ProviderCacheBuilder, ProviderCaches, SavedCache}, + cached_state::{ + CachedStateMetrics, ExecutionCache as StateExecutionCache, ExecutionCacheBuilder, + SavedCache, + }, payload_processor::{ prewarm::{PrewarmCacheTask, PrewarmContext, PrewarmTaskEvent}, sparse_trie::StateRootComputeOutcome, @@ -9,43 +13,86 @@ use crate::tree::{ sparse_trie::SparseTrieTask, StateProviderBuilder, TreeConfig, }; -use alloy_consensus::{transaction::Recovered, BlockHeader}; -use alloy_evm::block::StateChangeSource; +use alloy_evm::{block::StateChangeSource, ToTxEnv}; use alloy_primitives::B256; +use crossbeam_channel::Sender as CrossbeamSender; use executor::WorkloadExecutor; -use multiproof::*; +use multiproof::{SparseTrieUpdate, *}; use parking_lot::RwLock; use prewarm::PrewarmMetrics; -use reth_evm::{ConfigureEvm, OnStateHook}; -use reth_primitives_traits::{NodePrimitives, SealedHeaderFor}; -use reth_provider::{ - providers::ConsistentDbView, BlockReader, DatabaseProviderFactory, StateCommitmentProvider, - StateProviderFactory, StateReader, +use reth_engine_primitives::ExecutableTxIterator; +use reth_evm::{ + execute::{ExecutableTxFor, WithTxEnv}, + ConfigureEvm, EvmEnvFor, OnStateHook, SpecFor, TxEnvFor, }; +use reth_primitives_traits::NodePrimitives; +use reth_provider::{BlockReader, DatabaseProviderROFactory, StateProviderFactory, StateReader}; use reth_revm::{db::BundleState, state::EvmState}; -use reth_trie::TrieInput; +use reth_trie::{hashed_cursor::HashedCursorFactory, trie_cursor::TrieCursorFactory}; use reth_trie_parallel::{ - proof_task::{ProofTaskCtx, ProofTaskManager}, + proof_task::{ProofTaskCtx, ProofWorkerHandle}, root::ParallelStateRootError, }; +use reth_trie_sparse::{ + provider::{TrieNodeProvider, TrieNodeProviderFactory}, + ClearedSparseStateTrie, SparseStateTrie, SparseTrie, +}; +use reth_trie_sparse_parallel::{ParallelSparseTrie, ParallelismThresholds}; use std::{ - collections::VecDeque, sync::{ atomic::AtomicBool, - mpsc, - mpsc::{channel, Sender}, + mpsc::{self, channel}, Arc, }, + time::Instant, }; +use tracing::{debug, debug_span, instrument, warn}; +mod configured_sparse_trie; pub mod executor; pub mod multiproof; pub mod prewarm; pub mod sparse_trie; +use configured_sparse_trie::ConfiguredSparseTrie; + +/// Default parallelism thresholds to use with the [`ParallelSparseTrie`]. +/// +/// These values were determined by performing benchmarks using gradually increasing values to judge +/// the affects. Below 100 throughput would generally be equal or slightly less, while above 150 it +/// would deteriorate to the point where PST might as well not be used. +pub const PARALLEL_SPARSE_TRIE_PARALLELISM_THRESHOLDS: ParallelismThresholds = + ParallelismThresholds { min_revealed_nodes: 100, min_updated_nodes: 100 }; + +/// Default node capacity for shrinking the sparse trie. This is used to limit the number of trie +/// nodes in allocated sparse tries. +/// +/// Node maps have a key of `Nibbles` and value of `SparseNode`. +/// The `size_of::` is 40, and `size_of::` is 80. +/// +/// If we have 1 million entries of 120 bytes each, this conservative estimate comes out at around +/// 120MB. +pub const SPARSE_TRIE_MAX_NODES_SHRINK_CAPACITY: usize = 1_000_000; + +/// Default value capacity for shrinking the sparse trie. This is used to limit the number of values +/// in allocated sparse tries. +/// +/// There are storage and account values, the largest of the two being account values, which are +/// essentially `TrieAccount`s. +/// +/// Account value maps have a key of `Nibbles` and value of `TrieAccount`. +/// The `size_of::` is 40, and `size_of::` is 104. +/// +/// If we have 1 million entries of 144 bytes each, this conservative estimate comes out at around +/// 144MB. +pub const SPARSE_TRIE_MAX_VALUES_SHRINK_CAPACITY: usize = 1_000_000; + /// Entrypoint for executing the payload. -#[derive(Debug, Clone)] -pub struct PayloadProcessor { +#[derive(Debug)] +pub struct PayloadProcessor +where + Evm: ConfigureEvm, +{ /// The executor used by to spawn tasks. executor: WorkloadExecutor, /// The most recent cache used for execution. @@ -58,25 +105,52 @@ pub struct PayloadProcessor { disable_transaction_prewarming: bool, /// Determines how to configure the evm for execution. evm_config: Evm, - _marker: std::marker::PhantomData, + /// Whether precompile cache should be disabled. + precompile_cache_disabled: bool, + /// Precompile cache map. + precompile_cache_map: PrecompileCacheMap>, + /// A cleared `SparseStateTrie`, kept around to be reused for the state root computation so + /// that allocations can be minimized. + sparse_state_trie: Arc< + parking_lot::Mutex< + Option>, + >, + >, + /// Whether to disable the parallel sparse trie. + disable_parallel_sparse_trie: bool, + /// Maximum concurrency for prewarm task. + prewarm_max_concurrency: usize, } -impl PayloadProcessor { +impl PayloadProcessor +where + N: NodePrimitives, + Evm: ConfigureEvm, +{ /// Creates a new payload processor. - pub fn new(executor: WorkloadExecutor, evm_config: Evm, config: &TreeConfig) -> Self { + pub fn new( + executor: WorkloadExecutor, + evm_config: Evm, + config: &TreeConfig, + precompile_cache_map: PrecompileCacheMap>, + ) -> Self { Self { executor, execution_cache: Default::default(), trie_metrics: Default::default(), cross_block_cache_size: config.cross_block_cache_size(), - disable_transaction_prewarming: config.disable_caching_and_prewarming(), + disable_transaction_prewarming: config.disable_prewarming(), evm_config, - _marker: Default::default(), + precompile_cache_disabled: config.precompile_cache_disabled(), + precompile_cache_map, + sparse_state_trie: Arc::default(), + disable_parallel_sparse_trie: config.disable_parallel_sparse_trie(), + prewarm_max_concurrency: config.prewarm_max_concurrency(), } } } -impl PayloadProcessor +impl PayloadProcessor where N: NodePrimitives, Evm: ConfigureEvm + 'static, @@ -113,161 +187,194 @@ where /// /// This returns a handle to await the final state root and to interact with the tasks (e.g. /// canceling) - pub fn spawn

( - &self, - header: SealedHeaderFor, - transactions: VecDeque>, + #[allow(clippy::type_complexity)] + #[instrument( + level = "debug", + target = "engine::tree::payload_processor", + name = "payload processor", + skip_all + )] + pub fn spawn>( + &mut self, + env: ExecutionEnv, + transactions: I, provider_builder: StateProviderBuilder, - consistent_view: ConsistentDbView

, - trie_input: TrieInput, + multiproof_provider_factory: F, config: &TreeConfig, - ) -> PayloadHandle + ) -> PayloadHandle, I::Tx>, I::Error> where - P: DatabaseProviderFactory - + BlockReader - + StateProviderFactory - + StateReader - + StateCommitmentProvider + P: BlockReader + StateProviderFactory + StateReader + Clone + 'static, + F: DatabaseProviderROFactory + Clone + + Send + 'static, { + let span = tracing::Span::current(); let (to_sparse_trie, sparse_trie_rx) = channel(); - // spawn multiproof task - let state_root_config = MultiProofConfig::new_from_input(consistent_view, trie_input); + + // We rely on the cursor factory to provide whatever DB overlay is necessary to see a + // consistent view of the database, including the trie tables. Because of this there is no + // need for an overarching prefix set to invalidate any section of the trie tables, and so + // we use an empty prefix set. // Create and spawn the storage proof task - let task_ctx = ProofTaskCtx::new( - state_root_config.nodes_sorted.clone(), - state_root_config.state_sorted.clone(), - state_root_config.prefix_sets.clone(), - ); - let max_proof_task_concurrency = config.max_proof_task_concurrency() as usize; - let proof_task = ProofTaskManager::new( + let task_ctx = ProofTaskCtx::new(multiproof_provider_factory); + let storage_worker_count = config.storage_worker_count(); + let account_worker_count = config.account_worker_count(); + let proof_handle = ProofWorkerHandle::new( self.executor.handle().clone(), - state_root_config.consistent_view.clone(), task_ctx, - max_proof_task_concurrency, + storage_worker_count, + account_worker_count, ); - // We set it to half of the proof task concurrency, because often for each multiproof we - // spawn one Tokio task for the account proof, and one Tokio task for the storage proof. - let max_multi_proof_task_concurrency = max_proof_task_concurrency / 2; let multi_proof_task = MultiProofTask::new( - state_root_config, - self.executor.clone(), - proof_task.handle(), + proof_handle.clone(), to_sparse_trie, - max_multi_proof_task_concurrency, + config.multiproof_chunking_enabled().then_some(config.multiproof_chunk_size()), ); // wire the multiproof task to the prewarm task let to_multi_proof = Some(multi_proof_task.state_root_message_sender()); - let prewarm_handle = - self.spawn_caching_with(header, transactions, provider_builder, to_multi_proof.clone()); + let (prewarm_rx, execution_rx, transaction_count_hint) = + self.spawn_tx_iterator(transactions); + + let prewarm_handle = self.spawn_caching_with( + env, + prewarm_rx, + transaction_count_hint, + provider_builder, + to_multi_proof.clone(), + ); // spawn multi-proof task self.executor.spawn_blocking(move || { + let _enter = span.entered(); multi_proof_task.run(); }); - let mut sparse_trie_task = SparseTrieTask::new( - self.executor.clone(), - sparse_trie_rx, - proof_task.handle(), - self.trie_metrics.clone(), - ); - // wire the sparse trie to the state root response receiver let (state_root_tx, state_root_rx) = channel(); - self.executor.spawn_blocking(move || { - let res = sparse_trie_task.run(); - let _ = state_root_tx.send(res); - }); - // spawn the proof task - self.executor.spawn_blocking(move || { - if let Err(err) = proof_task.run() { - // At least log if there is an error at any point - tracing::error!( - target: "engine::root", - ?err, - "Storage proof task returned an error" - ); - } - }); + // Spawn the sparse trie task using any stored trie and parallel trie configuration. + self.spawn_sparse_trie_task(sparse_trie_rx, proof_handle, state_root_tx); - PayloadHandle { to_multi_proof, prewarm_handle, state_root: Some(state_root_rx) } + PayloadHandle { + to_multi_proof, + prewarm_handle, + state_root: Some(state_root_rx), + transactions: execution_rx, + } } - /// Spawn cache prewarming exclusively. + /// Spawns a task that exclusively handles cache prewarming for transaction execution. /// /// Returns a [`PayloadHandle`] to communicate with the task. - pub(super) fn spawn_cache_exclusive

( + #[instrument(level = "debug", target = "engine::tree::payload_processor", skip_all)] + pub(super) fn spawn_cache_exclusive>( &self, - header: SealedHeaderFor, - transactions: VecDeque>, + env: ExecutionEnv, + transactions: I, provider_builder: StateProviderBuilder, - ) -> PayloadHandle + ) -> PayloadHandle, I::Tx>, I::Error> where - P: BlockReader - + StateProviderFactory - + StateReader - + StateCommitmentProvider - + Clone - + 'static, + P: BlockReader + StateProviderFactory + StateReader + Clone + 'static, { - let prewarm_handle = self.spawn_caching_with(header, transactions, provider_builder, None); - PayloadHandle { to_multi_proof: None, prewarm_handle, state_root: None } + let (prewarm_rx, execution_rx, size_hint) = self.spawn_tx_iterator(transactions); + let prewarm_handle = + self.spawn_caching_with(env, prewarm_rx, size_hint, provider_builder, None); + PayloadHandle { + to_multi_proof: None, + prewarm_handle, + state_root: None, + transactions: execution_rx, + } + } + + /// Spawns a task advancing transaction env iterator and streaming updates through a channel. + #[expect(clippy::type_complexity)] + fn spawn_tx_iterator>( + &self, + transactions: I, + ) -> ( + mpsc::Receiver, I::Tx>>, + mpsc::Receiver, I::Tx>, I::Error>>, + usize, + ) { + // Get the transaction count for prewarming task + // Use upper bound if available (more accurate), otherwise use lower bound + let (lower, upper) = transactions.size_hint(); + let transaction_count_hint = upper.unwrap_or(lower); + + let (prewarm_tx, prewarm_rx) = mpsc::channel(); + let (execute_tx, execute_rx) = mpsc::channel(); + self.executor.spawn_blocking(move || { + for tx in transactions { + let tx = tx.map(|tx| WithTxEnv { tx_env: tx.to_tx_env(), tx: Arc::new(tx) }); + // only send Ok(_) variants to prewarming task + if let Ok(tx) = &tx { + let _ = prewarm_tx.send(tx.clone()); + } + let _ = execute_tx.send(tx); + } + }); + + (prewarm_rx, execute_rx, transaction_count_hint) } /// Spawn prewarming optionally wired to the multiproof task for target updates. fn spawn_caching_with

( &self, - header: SealedHeaderFor, - mut transactions: VecDeque>, + env: ExecutionEnv, + mut transactions: mpsc::Receiver + Clone + Send + 'static>, + transaction_count_hint: usize, provider_builder: StateProviderBuilder, - to_multi_proof: Option>, + to_multi_proof: Option>, ) -> CacheTaskHandle where - P: BlockReader - + StateProviderFactory - + StateReader - + StateCommitmentProvider - + Clone - + 'static, + P: BlockReader + StateProviderFactory + StateReader + Clone + 'static, { if self.disable_transaction_prewarming { // if no transactions should be executed we clear them but still spawn the task for // caching updates - transactions.clear(); + transactions = mpsc::channel().1; } - let (cache, cache_metrics) = self.cache_for(header.parent_hash()).split(); + let saved_cache = self.cache_for(env.parent_hash); + let cache = saved_cache.cache().clone(); + let cache_metrics = saved_cache.metrics().clone(); // configure prewarming let prewarm_ctx = PrewarmContext { - header, + env, evm_config: self.evm_config.clone(), - cache: cache.clone(), - cache_metrics: cache_metrics.clone(), + saved_cache, provider: provider_builder, metrics: PrewarmMetrics::default(), terminate_execution: Arc::new(AtomicBool::new(false)), + precompile_cache_disabled: self.precompile_cache_disabled, + precompile_cache_map: self.precompile_cache_map.clone(), }; - let prewarm_task = PrewarmCacheTask::new( + let (prewarm_task, to_prewarm_task) = PrewarmCacheTask::new( self.executor.clone(), self.execution_cache.clone(), prewarm_ctx, to_multi_proof, - transactions, + transaction_count_hint, + self.prewarm_max_concurrency, ); - let to_prewarm_task = prewarm_task.actions_tx(); // spawn pre-warm task - self.executor.spawn_blocking(move || { - prewarm_task.run(); - }); + { + let to_prewarm_task = to_prewarm_task.clone(); + let span = debug_span!(target: "engine::tree::payload_processor", "prewarm task"); + self.executor.spawn_blocking(move || { + let _enter = span.entered(); + prewarm_task.run(transactions, to_prewarm_task); + }); + } + CacheTaskHandle { cache, to_prewarm_task: Some(to_prewarm_task), cache_metrics } } @@ -275,31 +382,103 @@ where /// /// If the given hash is different then what is recently cached, then this will create a new /// instance. + #[instrument(level = "debug", target = "engine::caching", skip(self))] fn cache_for(&self, parent_hash: B256) -> SavedCache { - self.execution_cache.get_cache_for(parent_hash).unwrap_or_else(|| { - let cache = ProviderCacheBuilder::default().build_caches(self.cross_block_cache_size); + if let Some(cache) = self.execution_cache.get_cache_for(parent_hash) { + debug!("reusing execution cache"); + cache + } else { + debug!("creating new execution cache on cache miss"); + let cache = ExecutionCacheBuilder::default().build_caches(self.cross_block_cache_size); SavedCache::new(parent_hash, cache, CachedStateMetrics::zeroed()) - }) + } + } + + /// Spawns the [`SparseTrieTask`] for this payload processor. + #[instrument(level = "debug", target = "engine::tree::payload_processor", skip_all)] + fn spawn_sparse_trie_task( + &self, + sparse_trie_rx: mpsc::Receiver, + proof_worker_handle: BPF, + state_root_tx: mpsc::Sender>, + ) where + BPF: TrieNodeProviderFactory + Clone + Send + Sync + 'static, + BPF::AccountNodeProvider: TrieNodeProvider + Send + Sync, + BPF::StorageNodeProvider: TrieNodeProvider + Send + Sync, + { + // Reuse a stored SparseStateTrie, or create a new one using the desired configuration if + // there's none to reuse. + let cleared_sparse_trie = Arc::clone(&self.sparse_state_trie); + let sparse_state_trie = cleared_sparse_trie.lock().take().unwrap_or_else(|| { + let default_trie = SparseTrie::blind_from(if self.disable_parallel_sparse_trie { + ConfiguredSparseTrie::Serial(Default::default()) + } else { + ConfiguredSparseTrie::Parallel(Box::new( + ParallelSparseTrie::default() + .with_parallelism_thresholds(PARALLEL_SPARSE_TRIE_PARALLELISM_THRESHOLDS), + )) + }); + ClearedSparseStateTrie::from_state_trie( + SparseStateTrie::new() + .with_accounts_trie(default_trie.clone()) + .with_default_storage_trie(default_trie) + .with_updates(true), + ) + }); + + let task = + SparseTrieTask::<_, ConfiguredSparseTrie, ConfiguredSparseTrie>::new_with_cleared_trie( + sparse_trie_rx, + proof_worker_handle, + self.trie_metrics.clone(), + sparse_state_trie, + ); + + let span = tracing::Span::current(); + self.executor.spawn_blocking(move || { + let _enter = span.entered(); + + let (result, trie) = task.run(); + // Send state root computation result + let _ = state_root_tx.send(result); + + // Clear the SparseStateTrie, shrink, and replace it back into the mutex _after_ sending + // results to the next step, so that time spent clearing doesn't block the step after + // this one. + let _enter = debug_span!(target: "engine::tree::payload_processor", "clear").entered(); + let mut cleared_trie = ClearedSparseStateTrie::from_state_trie(trie); + + // Shrink the sparse trie so that we don't have ever increasing memory. + cleared_trie.shrink_to( + SPARSE_TRIE_MAX_NODES_SHRINK_CAPACITY, + SPARSE_TRIE_MAX_VALUES_SHRINK_CAPACITY, + ); + + cleared_sparse_trie.lock().replace(cleared_trie); + }); } } /// Handle to all the spawned tasks. #[derive(Debug)] -pub struct PayloadHandle { +pub struct PayloadHandle { /// Channel for evm state updates - to_multi_proof: Option>, + to_multi_proof: Option>, // must include the receiver of the state root wired to the sparse trie prewarm_handle: CacheTaskHandle, /// Receiver for the state root state_root: Option>>, + /// Stream of block transactions + transactions: mpsc::Receiver>, } -impl PayloadHandle { +impl PayloadHandle { /// Awaits the state root /// /// # Panics /// /// If payload processing was started without background tasks. + #[instrument(level = "debug", target = "engine::tree::payload_processor", skip_all)] pub fn state_root(&mut self) -> Result { self.state_root .take() @@ -323,10 +502,11 @@ impl PayloadHandle { } /// Returns a clone of the caches used by prewarming - pub(super) fn caches(&self) -> ProviderCaches { + pub(super) fn caches(&self) -> StateExecutionCache { self.prewarm_handle.cache.clone() } + /// Returns a clone of the cache metrics used by prewarming pub(super) fn cache_metrics(&self) -> CachedStateMetrics { self.prewarm_handle.cache_metrics.clone() } @@ -341,20 +521,27 @@ impl PayloadHandle { /// Terminates the entire caching task. /// /// If the [`BundleState`] is provided it will update the shared cache. - pub(super) fn terminate_caching(&mut self, block_output: Option) { + pub(super) fn terminate_caching(&mut self, block_output: Option<&BundleState>) { self.prewarm_handle.terminate_caching(block_output) } + + /// Returns iterator yielding transactions from the stream. + pub fn iter_transactions(&mut self) -> impl Iterator> + '_ { + core::iter::repeat_with(|| self.transactions.recv()) + .take_while(|res| res.is_ok()) + .map(|res| res.unwrap()) + } } /// Access to the spawned [`PrewarmCacheTask`]. #[derive(Debug)] pub(crate) struct CacheTaskHandle { /// The shared cache the task operates with. - cache: ProviderCaches, + cache: StateExecutionCache, /// Metrics for the caches cache_metrics: CachedStateMetrics, /// Channel to the spawned prewarm task if any - to_prewarm_task: Option>, + to_prewarm_task: Option>, } impl CacheTaskHandle { @@ -370,10 +557,12 @@ impl CacheTaskHandle { /// Terminates the entire pre-warming task. /// /// If the [`BundleState`] is provided it will update the shared cache. - pub(super) fn terminate_caching(&mut self, block_output: Option) { - self.to_prewarm_task - .take() - .map(|tx| tx.send(PrewarmTaskEvent::Terminate { block_output }).ok()); + pub(super) fn terminate_caching(&mut self, block_output: Option<&BundleState>) { + if let Some(tx) = self.to_prewarm_task.take() { + // Only clone when we have an active task and a state to send + let event = PrewarmTaskEvent::Terminate { block_output: block_output.cloned() }; + let _ = tx.send(event); + } } } @@ -391,6 +580,24 @@ impl Drop for CacheTaskHandle { /// - Update cache upon successful payload execution /// /// This process assumes that payloads are received sequentially. +/// +/// ## Cache Safety +/// +/// **CRITICAL**: Cache update operations require exclusive access. All concurrent cache users +/// (such as prewarming tasks) must be terminated before calling `update_with_guard`, otherwise +/// the cache may be corrupted or cleared. +/// +/// ## Cache vs Prewarming Distinction +/// +/// **`ExecutionCache`**: +/// - Stores parent block's execution state after completion +/// - Used to fetch parent data for next block's execution +/// - Must be exclusively accessed during save operations +/// +/// **`PrewarmCacheTask`**: +/// - Speculatively loads accounts/storage that might be used in transaction execution +/// - Prepares data for state root proof computation +/// - Runs concurrently but must not interfere with cache saves #[derive(Clone, Debug, Default)] struct ExecutionCache { /// Guarded cloneable cache identified by a block hash. @@ -398,12 +605,25 @@ struct ExecutionCache { } impl ExecutionCache { - /// Returns the cache if the currently store cache is for the given `parent_hash` + /// Returns the cache for `parent_hash` if it's available for use. + /// + /// A cache is considered available when: + /// - It exists and matches the requested parent hash + /// - No other tasks are currently using it (checked via Arc reference count) + #[instrument(level = "debug", target = "engine::tree::payload_processor", skip(self))] pub(crate) fn get_cache_for(&self, parent_hash: B256) -> Option { + let start = Instant::now(); let cache = self.inner.read(); + + let elapsed = start.elapsed(); + if elapsed.as_millis() > 5 { + warn!(blocked_for=?elapsed, "Blocked waiting for execution cache mutex"); + } + cache .as_ref() - .and_then(|cache| (cache.executed_block_hash() == parent_hash).then(|| cache.clone())) + .filter(|c| c.executed_block_hash() == parent_hash && c.is_available()) + .cloned() } /// Clears the tracked cache @@ -412,44 +632,157 @@ impl ExecutionCache { self.inner.write().take(); } - /// Stores the provider cache - pub(crate) fn save_cache(&self, cache: SavedCache) { - self.inner.write().replace(cache); + /// Updates the cache with a closure that has exclusive access to the guard. + /// This ensures that all cache operations happen atomically. + /// + /// ## CRITICAL SAFETY REQUIREMENT + /// + /// **Before calling this method, you MUST ensure there are no other active cache users.** + /// This includes: + /// - No running [`PrewarmCacheTask`] instances that could write to the cache + /// - No concurrent transactions that might access the cached state + /// - All prewarming operations must be completed or cancelled + /// + /// Violating this requirement can result in cache corruption, incorrect state data, + /// and potential consensus failures. + pub(crate) fn update_with_guard(&self, update_fn: F) + where + F: FnOnce(&mut Option), + { + let mut guard = self.inner.write(); + update_fn(&mut guard); + } +} + +/// EVM context required to execute a block. +#[derive(Debug, Clone)] +pub struct ExecutionEnv { + /// Evm environment. + pub evm_env: EvmEnvFor, + /// Hash of the block being executed. + pub hash: B256, + /// Hash of the parent block. + pub parent_hash: B256, +} + +impl Default for ExecutionEnv +where + EvmEnvFor: Default, +{ + fn default() -> Self { + Self { + evm_env: Default::default(), + hash: Default::default(), + parent_hash: Default::default(), + } } } #[cfg(test)] mod tests { - use std::sync::Arc; - + use super::ExecutionCache; use crate::tree::{ + cached_state::{CachedStateMetrics, ExecutionCacheBuilder, SavedCache}, payload_processor::{ evm_state_to_hashed_post_state, executor::WorkloadExecutor, PayloadProcessor, }, + precompile_cache::PrecompileCacheMap, StateProviderBuilder, TreeConfig, }; use alloy_evm::block::StateChangeSource; use rand::Rng; use reth_chainspec::ChainSpec; use reth_db_common::init::init_genesis; - use reth_ethereum_primitives::EthPrimitives; + use reth_ethereum_primitives::TransactionSigned; use reth_evm::OnStateHook; use reth_evm_ethereum::EthEvmConfig; - use reth_primitives_traits::{Account, StorageEntry}; + use reth_primitives_traits::{Account, Recovered, StorageEntry}; use reth_provider::{ - providers::{BlockchainProvider, ConsistentDbView}, + providers::{BlockchainProvider, OverlayStateProviderFactory}, test_utils::create_test_provider_factory_with_chain_spec, ChainSpecProvider, HashingWriter, }; use reth_testing_utils::generators; - use reth_trie::{test_utils::state_root, HashedPostState, TrieInput}; + use reth_trie::{test_utils::state_root, HashedPostState}; use revm_primitives::{Address, HashMap, B256, KECCAK_EMPTY, U256}; use revm_state::{AccountInfo, AccountStatus, EvmState, EvmStorageSlot}; + use std::sync::Arc; + + fn make_saved_cache(hash: B256) -> SavedCache { + let execution_cache = ExecutionCacheBuilder::default().build_caches(1_000); + SavedCache::new(hash, execution_cache, CachedStateMetrics::zeroed()) + } + + #[test] + fn execution_cache_allows_single_checkout() { + let execution_cache = ExecutionCache::default(); + let hash = B256::from([1u8; 32]); + + execution_cache.update_with_guard(|slot| *slot = Some(make_saved_cache(hash))); + + let first = execution_cache.get_cache_for(hash); + assert!(first.is_some(), "expected initial checkout to succeed"); + + let second = execution_cache.get_cache_for(hash); + assert!(second.is_none(), "second checkout should be blocked while guard is active"); + + drop(first); + + let third = execution_cache.get_cache_for(hash); + assert!(third.is_some(), "third checkout should succeed after guard is dropped"); + } + + #[test] + fn execution_cache_checkout_releases_on_drop() { + let execution_cache = ExecutionCache::default(); + let hash = B256::from([2u8; 32]); + + execution_cache.update_with_guard(|slot| *slot = Some(make_saved_cache(hash))); + + { + let guard = execution_cache.get_cache_for(hash); + assert!(guard.is_some(), "expected checkout to succeed"); + // Guard dropped at end of scope + } + + let retry = execution_cache.get_cache_for(hash); + assert!(retry.is_some(), "checkout should succeed after guard drop"); + } + + #[test] + fn execution_cache_mismatch_parent_returns_none() { + let execution_cache = ExecutionCache::default(); + let hash = B256::from([3u8; 32]); + + execution_cache.update_with_guard(|slot| *slot = Some(make_saved_cache(hash))); + + let miss = execution_cache.get_cache_for(B256::from([4u8; 32])); + assert!(miss.is_none(), "checkout should fail for different parent hash"); + } + + #[test] + fn execution_cache_update_after_release_succeeds() { + let execution_cache = ExecutionCache::default(); + let initial = B256::from([5u8; 32]); + + execution_cache.update_with_guard(|slot| *slot = Some(make_saved_cache(initial))); + + let guard = + execution_cache.get_cache_for(initial).expect("expected initial checkout to succeed"); + + drop(guard); + + let updated = B256::from([6u8; 32]); + execution_cache.update_with_guard(|slot| *slot = Some(make_saved_cache(updated))); + + let new_checkout = execution_cache.get_cache_for(updated); + assert!(new_checkout.is_some(), "new checkout should succeed after release and update"); + } fn create_mock_state_updates(num_accounts: usize, updates_per_account: usize) -> Vec { let mut rng = generators::rng(); let all_addresses: Vec

= (0..num_accounts).map(|_| rng.random()).collect(); - let mut updates = Vec::new(); + let mut updates = Vec::with_capacity(updates_per_account); for _ in 0..updates_per_account { let num_accounts_in_update = rng.random_range(1..=num_accounts); @@ -467,6 +800,7 @@ mod tests { EvmStorageSlot::new_changed( U256::ZERO, U256::from(rng.random::()), + 0, ), ); } @@ -481,6 +815,7 @@ mod tests { }, storage, status: AccountStatus::Touched, + transaction_id: 0, }; state_update.insert(address, account); @@ -544,18 +879,20 @@ mod tests { } } - let payload_processor = PayloadProcessor::::new( + let mut payload_processor = PayloadProcessor::new( WorkloadExecutor::default(), EthEvmConfig::new(factory.chain_spec()), &TreeConfig::default(), + PrecompileCacheMap::default(), ); - let provider = BlockchainProvider::new(factory).unwrap(); + + let provider_factory = BlockchainProvider::new(factory).unwrap(); + let mut handle = payload_processor.spawn( Default::default(), - Default::default(), - StateProviderBuilder::new(provider.clone(), genesis_hash, None), - ConsistentDbView::new_with_latest_tip(provider).unwrap(), - TrieInput::from_state(hashed_state), + core::iter::empty::, core::convert::Infallible>>(), + StateProviderBuilder::new(provider_factory.clone(), genesis_hash, None), + OverlayStateProviderFactory::new(provider_factory), &TreeConfig::default(), ); diff --git a/crates/engine/tree/src/tree/payload_processor/multiproof.rs b/crates/engine/tree/src/tree/payload_processor/multiproof.rs index a430c221d3d..7da199dd636 100644 --- a/crates/engine/tree/src/tree/payload_processor/multiproof.rs +++ b/crates/engine/tree/src/tree/payload_processor/multiproof.rs @@ -1,39 +1,31 @@ //! Multiproof task related functionality. -use crate::tree::payload_processor::executor::WorkloadExecutor; use alloy_evm::block::StateChangeSource; use alloy_primitives::{ keccak256, map::{B256Set, HashSet}, B256, }; +use crossbeam_channel::{unbounded, Receiver as CrossbeamReceiver, Sender as CrossbeamSender}; +use dashmap::DashMap; use derive_more::derive::Deref; -use metrics::Histogram; -use reth_errors::ProviderError; +use metrics::{Gauge, Histogram}; use reth_metrics::Metrics; -use reth_provider::{ - providers::ConsistentDbView, BlockReader, DatabaseProviderFactory, FactoryTx, - StateCommitmentProvider, -}; use reth_revm::state::EvmState; use reth_trie::{ - prefix_set::TriePrefixSetsMut, updates::TrieUpdatesSorted, HashedPostState, - HashedPostStateSorted, HashedStorage, MultiProof, MultiProofTargets, TrieInput, + added_removed_keys::MultiAddedRemovedKeys, prefix_set::TriePrefixSetsMut, + updates::TrieUpdatesSorted, DecodedMultiProof, HashedPostState, HashedPostStateSorted, + HashedStorage, MultiProofTargets, TrieInput, }; -use reth_trie_parallel::{proof::ParallelProof, proof_task::ProofTaskManagerHandle}; -use std::{ - collections::{BTreeMap, VecDeque}, - ops::DerefMut, - sync::{ - mpsc::{channel, Receiver, Sender}, - Arc, +use reth_trie_parallel::{ + proof::ParallelProof, + proof_task::{ + AccountMultiproofInput, ProofResultContext, ProofResultMessage, ProofWorkerHandle, + StorageProofInput, }, - time::{Duration, Instant}, }; -use tracing::{debug, error, trace}; - -/// The size of proof targets chunk to spawn in one calculation. -const MULTIPROOF_TARGETS_CHUNK_SIZE: usize = 10; +use std::{collections::BTreeMap, ops::DerefMut, sync::Arc, time::Instant}; +use tracing::{debug, error, instrument, trace}; /// A trie update that can be applied to sparse trie alongside the proofs for touched parts of the /// state. @@ -42,7 +34,7 @@ pub struct SparseTrieUpdate { /// The state update that was used to calculate the proof pub(crate) state: HashedPostState, /// The calculated multiproof - pub(crate) multiproof: MultiProof, + pub(crate) multiproof: DecodedMultiProof, } impl SparseTrieUpdate { @@ -53,8 +45,8 @@ impl SparseTrieUpdate { /// Construct update from multiproof. #[cfg(test)] - pub(super) fn from_multiproof(multiproof: MultiProof) -> Self { - Self { multiproof, ..Default::default() } + pub(super) fn from_multiproof(multiproof: reth_trie::MultiProof) -> alloy_rlp::Result { + Ok(Self { multiproof: multiproof.try_into()?, ..Default::default() }) } /// Extend update with contents of the other. @@ -65,10 +57,8 @@ impl SparseTrieUpdate { } /// Common configuration for multi proof tasks -#[derive(Debug, Clone)] -pub(super) struct MultiProofConfig { - /// View over the state in the database. - pub consistent_view: ConsistentDbView, +#[derive(Debug, Clone, Default)] +pub(crate) struct MultiProofConfig { /// The sorted collection of cached in-memory intermediate trie nodes that /// can be reused for computation. pub nodes_sorted: Arc, @@ -80,18 +70,18 @@ pub(super) struct MultiProofConfig { pub prefix_sets: Arc, } -impl MultiProofConfig { - /// Creates a new state root config from the consistent view and the trie input. - pub(super) fn new_from_input( - consistent_view: ConsistentDbView, - input: TrieInput, - ) -> Self { - Self { - consistent_view, - nodes_sorted: Arc::new(input.nodes.into_sorted()), - state_sorted: Arc::new(input.state.into_sorted()), - prefix_sets: Arc::new(input.prefix_sets), - } +impl MultiProofConfig { + /// Creates a new state root config from the trie input. + /// + /// This returns a cleared [`TrieInput`] so that we can reuse any allocated space in the + /// [`TrieInput`]. + pub(crate) fn from_input(mut input: TrieInput) -> (TrieInput, Self) { + let config = Self { + nodes_sorted: Arc::new(input.nodes.drain_into_sorted()), + state_sorted: Arc::new(input.state.drain_into_sorted()), + prefix_sets: Arc::new(input.prefix_sets.clone()), + }; + (input.cleared(), config) } } @@ -112,10 +102,6 @@ pub(super) enum MultiProofMessage { /// The state update that was used to calculate the proof state: HashedPostState, }, - /// Proof calculation completed for a specific state update - ProofCalculated(Box), - /// Error during proof calculation - ProofCalculationError(ProviderError), /// Signals state update stream end. /// /// This is triggered by block execution, indicating that no additional state updates are @@ -123,17 +109,6 @@ pub(super) enum MultiProofMessage { FinishedStateUpdates, } -/// Message about completion of proof calculation for a specific state update -#[derive(Debug)] -pub(super) struct ProofCalculated { - /// The index of this proof in the sequence of state updates - sequence_number: u64, - /// Sparse trie update - update: SparseTrieUpdate, - /// The time taken to calculate the proof. - elapsed: Duration, -} - /// Handle to track proof calculation ordering. #[derive(Debug, Default)] struct ProofSequencer { @@ -196,10 +171,10 @@ impl ProofSequencer { /// This should trigger once the block has been executed (after) the last state update has been /// sent. This triggers the exit condition of the multi proof task. #[derive(Deref, Debug)] -pub(super) struct StateHookSender(Sender); +pub(super) struct StateHookSender(CrossbeamSender); impl StateHookSender { - pub(crate) const fn new(inner: Sender) -> Self { + pub(crate) const fn new(inner: CrossbeamSender) -> Self { Self(inner) } } @@ -217,7 +192,7 @@ pub(crate) fn evm_state_to_hashed_post_state(update: EvmState) -> HashedPostStat for (address, account) in update { if account.is_touched() { let hashed_address = keccak256(address); - trace!(target: "engine::root", ?address, ?hashed_address, "Adding account to state update"); + trace!(target: "engine::tree::payload_processor::multiproof", ?address, ?hashed_address, "Adding account to state update"); let destroyed = account.is_selfdestructed(); let info = if destroyed { None } else { Some(account.info.into()) }; @@ -245,14 +220,14 @@ pub(crate) fn evm_state_to_hashed_post_state(update: EvmState) -> HashedPostStat /// A pending multiproof task, either [`StorageMultiproofInput`] or [`MultiproofInput`]. #[derive(Debug)] -enum PendingMultiproofTask { +enum PendingMultiproofTask { /// A storage multiproof task input. - Storage(StorageMultiproofInput), + Storage(StorageMultiproofInput), /// A regular multiproof task input. - Regular(MultiproofInput), + Regular(MultiproofInput), } -impl PendingMultiproofTask { +impl PendingMultiproofTask { /// Returns the proof sequence number of the task. const fn proof_sequence_number(&self) -> u64 { match self { @@ -278,31 +253,30 @@ impl PendingMultiproofTask { } } -impl From> for PendingMultiproofTask { - fn from(input: StorageMultiproofInput) -> Self { +impl From for PendingMultiproofTask { + fn from(input: StorageMultiproofInput) -> Self { Self::Storage(input) } } -impl From> for PendingMultiproofTask { - fn from(input: MultiproofInput) -> Self { +impl From for PendingMultiproofTask { + fn from(input: MultiproofInput) -> Self { Self::Regular(input) } } -/// Input parameters for spawning a dedicated storage multiproof calculation. +/// Input parameters for dispatching a dedicated storage multiproof calculation. #[derive(Debug)] -struct StorageMultiproofInput { - config: MultiProofConfig, - source: Option, +struct StorageMultiproofInput { hashed_state_update: HashedPostState, hashed_address: B256, proof_targets: B256Set, proof_sequence_number: u64, - state_root_message_sender: Sender, + state_root_message_sender: CrossbeamSender, + multi_added_removed_keys: Arc, } -impl StorageMultiproofInput { +impl StorageMultiproofInput { /// Destroys the input and sends a [`MultiProofMessage::EmptyProof`] message to the sender. fn send_empty_proof(self) { let _ = self.state_root_message_sender.send(MultiProofMessage::EmptyProof { @@ -312,18 +286,18 @@ impl StorageMultiproofInput { } } -/// Input parameters for spawning a multiproof calculation. +/// Input parameters for dispatching a multiproof calculation. #[derive(Debug)] -struct MultiproofInput { - config: MultiProofConfig, +struct MultiproofInput { source: Option, hashed_state_update: HashedPostState, proof_targets: MultiProofTargets, proof_sequence_number: u64, - state_root_message_sender: Sender, + state_root_message_sender: CrossbeamSender, + multi_added_removed_keys: Option>, } -impl MultiproofInput { +impl MultiproofInput { /// Destroys the input and sends a [`MultiProofMessage::EmptyProof`] message to the sender. fn send_empty_proof(self) { let _ = self.state_root_message_sender.send(MultiProofMessage::EmptyProof { @@ -333,51 +307,60 @@ impl MultiproofInput { } } -/// Manages concurrent multiproof calculations. -/// Takes care of not having more calculations in flight than a given maximum -/// concurrency, further calculation requests are queued and spawn later, after -/// availability has been signaled. +/// Coordinates multiproof dispatch between `MultiProofTask` and the parallel trie workers. +/// +/// # Flow +/// 1. `MultiProofTask` asks the manager to dispatch either storage or account proof work. +/// 2. The manager builds the request, clones `proof_result_tx`, and hands everything to +/// [`ProofWorkerHandle`]. +/// 3. A worker finishes the proof and sends a [`ProofResultMessage`] through the channel included +/// in the job. +/// 4. `MultiProofTask` consumes the message from the same channel and sequences it with +/// `ProofSequencer`. #[derive(Debug)] -pub struct MultiproofManager { - /// Maximum number of concurrent calculations. - max_concurrent: usize, - /// Currently running calculations. - inflight: usize, - /// Queued calculations. - pending: VecDeque>, - /// Executor for tasks - executor: WorkloadExecutor, - /// Sender to the storage proof task. - storage_proof_task_handle: ProofTaskManagerHandle>, +pub struct MultiproofManager { + /// Handle to the proof worker pools (storage and account). + proof_worker_handle: ProofWorkerHandle, + /// Cached storage proof roots for missed leaves; this maps + /// hashed (missed) addresses to their storage proof roots. + /// + /// It is important to cache these. Otherwise, a common account + /// (popular ERC-20, etc.) having missed leaves in its path would + /// repeatedly calculate these proofs per interacting transaction + /// (same account different slots). + /// + /// This also works well with chunking multiproofs, which may break + /// a big account change into different chunks, which may repeatedly + /// revisit missed leaves. + missed_leaves_storage_roots: Arc>, + /// Channel sender cloned into each dispatched job so workers can send back the + /// `ProofResultMessage`. + proof_result_tx: CrossbeamSender, /// Metrics metrics: MultiProofTaskMetrics, } -impl MultiproofManager -where - Factory: - DatabaseProviderFactory + StateCommitmentProvider + Clone + 'static, -{ +impl MultiproofManager { /// Creates a new [`MultiproofManager`]. fn new( - executor: WorkloadExecutor, metrics: MultiProofTaskMetrics, - storage_proof_task_handle: ProofTaskManagerHandle>, - max_concurrent: usize, + proof_worker_handle: ProofWorkerHandle, + proof_result_tx: CrossbeamSender, ) -> Self { + // Initialize the max worker gauges with the worker pool sizes + metrics.max_storage_workers.set(proof_worker_handle.total_storage_workers() as f64); + metrics.max_account_workers.set(proof_worker_handle.total_account_workers() as f64); + Self { - pending: VecDeque::with_capacity(max_concurrent), - max_concurrent, - executor, - inflight: 0, metrics, - storage_proof_task_handle, + proof_worker_handle, + missed_leaves_storage_roots: Default::default(), + proof_result_tx, } } - /// Spawns a new multiproof calculation or enqueues it for later if - /// `max_concurrent` are already inflight. - fn spawn_or_queue(&mut self, input: PendingMultiproofTask) { + /// Dispatches a new multiproof calculation to worker pools. + fn dispatch(&self, input: PendingMultiproofTask) { // If there are no proof targets, we can just send an empty multiproof back immediately if input.proof_targets_is_empty() { debug!( @@ -388,185 +371,180 @@ where return } - if self.inflight >= self.max_concurrent { - self.pending.push_back(input); - self.metrics.pending_multiproofs_histogram.record(self.pending.len() as f64); - return; - } - - self.spawn_multiproof_task(input); - } - - /// Signals that a multiproof calculation has finished and there's room to - /// spawn a new calculation if needed. - fn on_calculation_complete(&mut self) { - self.inflight = self.inflight.saturating_sub(1); - self.metrics.inflight_multiproofs_histogram.record(self.inflight as f64); - - if let Some(input) = self.pending.pop_front() { - self.metrics.pending_multiproofs_histogram.record(self.pending.len() as f64); - self.spawn_multiproof_task(input); - } - } - - /// Spawns a multiproof task, dispatching to `spawn_storage_proof` if the input is a storage - /// multiproof, and dispatching to `spawn_multiproof` otherwise. - fn spawn_multiproof_task(&mut self, input: PendingMultiproofTask) { match input { PendingMultiproofTask::Storage(storage_input) => { - self.spawn_storage_proof(storage_input); + self.dispatch_storage_proof(storage_input); } PendingMultiproofTask::Regular(multiproof_input) => { - self.spawn_multiproof(multiproof_input); + self.dispatch_multiproof(multiproof_input); } } } - /// Spawns a single storage proof calculation task. - fn spawn_storage_proof(&mut self, storage_multiproof_input: StorageMultiproofInput) { + /// Dispatches a single storage proof calculation to worker pool. + fn dispatch_storage_proof(&self, storage_multiproof_input: StorageMultiproofInput) { let StorageMultiproofInput { - config, - source, hashed_state_update, hashed_address, proof_targets, proof_sequence_number, - state_root_message_sender, + multi_added_removed_keys, + state_root_message_sender: _, } = storage_multiproof_input; - let storage_proof_task_handle = self.storage_proof_task_handle.clone(); + let storage_targets = proof_targets.len(); - self.executor.spawn_blocking(move || { - let storage_targets = proof_targets.len(); + trace!( + target: "engine::tree::payload_processor::multiproof", + proof_sequence_number, + ?proof_targets, + storage_targets, + "Dispatching storage proof to workers" + ); - trace!( - target: "engine::root", - proof_sequence_number, - ?proof_targets, - storage_targets, - "Starting dedicated storage proof calculation", - ); - let start = Instant::now(); - let result = ParallelProof::new( - config.consistent_view, - config.nodes_sorted, - config.state_sorted, - config.prefix_sets, - storage_proof_task_handle.clone(), - ) - .with_branch_node_masks(true) - .storage_proof(hashed_address, proof_targets); - let elapsed = start.elapsed(); - trace!( - target: "engine::root", + let start = Instant::now(); + + // Create prefix set from targets + let prefix_set = reth_trie::prefix_set::PrefixSetMut::from( + proof_targets.iter().map(reth_trie::Nibbles::unpack), + ); + let prefix_set = prefix_set.freeze(); + + // Build computation input (data only) + let input = StorageProofInput::new( + hashed_address, + prefix_set, + proof_targets, + true, // with_branch_node_masks + Some(multi_added_removed_keys), + ); + + // Dispatch to storage worker + if let Err(e) = self.proof_worker_handle.dispatch_storage_proof( + input, + ProofResultContext::new( + self.proof_result_tx.clone(), proof_sequence_number, - ?elapsed, - ?source, - storage_targets, - "Storage multiproofs calculated", - ); + hashed_state_update, + start, + ), + ) { + error!(target: "engine::tree::payload_processor::multiproof", ?e, "Failed to dispatch storage proof"); + return; + } - match result { - Ok(proof) => { - let _ = state_root_message_sender.send(MultiProofMessage::ProofCalculated( - Box::new(ProofCalculated { - sequence_number: proof_sequence_number, - update: SparseTrieUpdate { - state: hashed_state_update, - multiproof: MultiProof::from_storage_proof(hashed_address, proof), - }, - elapsed, - }), - )); - } - Err(error) => { - let _ = state_root_message_sender - .send(MultiProofMessage::ProofCalculationError(error.into())); - } - } - }); + self.metrics + .active_storage_workers_histogram + .record(self.proof_worker_handle.active_storage_workers() as f64); + self.metrics + .active_account_workers_histogram + .record(self.proof_worker_handle.active_account_workers() as f64); + self.metrics + .pending_storage_multiproofs_histogram + .record(self.proof_worker_handle.pending_storage_tasks() as f64); + self.metrics + .pending_account_multiproofs_histogram + .record(self.proof_worker_handle.pending_account_tasks() as f64); + } - self.inflight += 1; - self.metrics.inflight_multiproofs_histogram.record(self.inflight as f64); + /// Signals that a multiproof calculation has finished. + fn on_calculation_complete(&self) { + self.metrics + .active_storage_workers_histogram + .record(self.proof_worker_handle.active_storage_workers() as f64); + self.metrics + .active_account_workers_histogram + .record(self.proof_worker_handle.active_account_workers() as f64); + self.metrics + .pending_storage_multiproofs_histogram + .record(self.proof_worker_handle.pending_storage_tasks() as f64); + self.metrics + .pending_account_multiproofs_histogram + .record(self.proof_worker_handle.pending_account_tasks() as f64); } - /// Spawns a single multiproof calculation task. - fn spawn_multiproof(&mut self, multiproof_input: MultiproofInput) { + /// Dispatches a single multiproof calculation to worker pool. + fn dispatch_multiproof(&self, multiproof_input: MultiproofInput) { let MultiproofInput { - config, source, hashed_state_update, proof_targets, proof_sequence_number, - state_root_message_sender, + state_root_message_sender: _, + multi_added_removed_keys, } = multiproof_input; - let storage_proof_task_handle = self.storage_proof_task_handle.clone(); - self.executor.spawn_blocking(move || { - let account_targets = proof_targets.len(); - let storage_targets = proof_targets.values().map(|slots| slots.len()).sum::(); + let missed_leaves_storage_roots = self.missed_leaves_storage_roots.clone(); + let account_targets = proof_targets.len(); + let storage_targets = proof_targets.values().map(|slots| slots.len()).sum::(); - trace!( - target: "engine::root", - proof_sequence_number, - ?proof_targets, - account_targets, - storage_targets, - "Starting multiproof calculation", - ); - let start = Instant::now(); - let result = ParallelProof::new( - config.consistent_view, - config.nodes_sorted, - config.state_sorted, - config.prefix_sets, - storage_proof_task_handle.clone(), - ) - .with_branch_node_masks(true) - .multiproof(proof_targets); - let elapsed = start.elapsed(); - trace!( - target: "engine::root", + trace!( + target: "engine::tree::payload_processor::multiproof", + proof_sequence_number, + ?proof_targets, + account_targets, + storage_targets, + ?source, + "Dispatching multiproof to workers" + ); + + let start = Instant::now(); + + // Extend prefix sets with targets + let frozen_prefix_sets = + ParallelProof::extend_prefix_sets_with_targets(&Default::default(), &proof_targets); + + // Dispatch account multiproof to worker pool with result sender + let input = AccountMultiproofInput { + targets: proof_targets, + prefix_sets: frozen_prefix_sets, + collect_branch_node_masks: true, + multi_added_removed_keys, + missed_leaves_storage_roots, + // Workers will send ProofResultMessage directly to proof_result_rx + proof_result_sender: ProofResultContext::new( + self.proof_result_tx.clone(), proof_sequence_number, - ?elapsed, - ?source, - account_targets, - storage_targets, - "Multiproof calculated", - ); + hashed_state_update, + start, + ), + }; - match result { - Ok(proof) => { - let _ = state_root_message_sender.send(MultiProofMessage::ProofCalculated( - Box::new(ProofCalculated { - sequence_number: proof_sequence_number, - update: SparseTrieUpdate { - state: hashed_state_update, - multiproof: proof, - }, - elapsed, - }), - )); - } - Err(error) => { - let _ = state_root_message_sender - .send(MultiProofMessage::ProofCalculationError(error.into())); - } - } - }); + if let Err(e) = self.proof_worker_handle.dispatch_account_multiproof(input) { + error!(target: "engine::tree::payload_processor::multiproof", ?e, "Failed to dispatch account multiproof"); + return; + } - self.inflight += 1; - self.metrics.inflight_multiproofs_histogram.record(self.inflight as f64); + self.metrics + .active_storage_workers_histogram + .record(self.proof_worker_handle.active_storage_workers() as f64); + self.metrics + .active_account_workers_histogram + .record(self.proof_worker_handle.active_account_workers() as f64); + self.metrics + .pending_storage_multiproofs_histogram + .record(self.proof_worker_handle.pending_storage_tasks() as f64); + self.metrics + .pending_account_multiproofs_histogram + .record(self.proof_worker_handle.pending_account_tasks() as f64); } } #[derive(Metrics, Clone)] #[metrics(scope = "tree.root")] pub(crate) struct MultiProofTaskMetrics { - /// Histogram of inflight multiproofs. - pub inflight_multiproofs_histogram: Histogram, - /// Histogram of pending multiproofs. - pub pending_multiproofs_histogram: Histogram, + /// Histogram of active storage workers processing proofs. + pub active_storage_workers_histogram: Histogram, + /// Histogram of active account workers processing proofs. + pub active_account_workers_histogram: Histogram, + /// Gauge for the maximum number of storage workers in the pool. + pub max_storage_workers: Gauge, + /// Gauge for the maximum number of account workers in the pool. + pub max_account_workers: Gauge, + /// Histogram of pending storage multiproofs in the queue. + pub pending_storage_multiproofs_histogram: Histogram, + /// Histogram of pending account multiproofs in the queue. + pub pending_account_multiproofs_histogram: Histogram, /// Histogram of the number of prefetch proof target accounts. pub prefetch_proof_targets_accounts_histogram: Histogram, @@ -607,77 +585,182 @@ pub(crate) struct MultiProofTaskMetrics { /// Standalone task that receives a transaction state stream and updates relevant /// data structures to calculate state root. /// -/// It is responsible of initializing a blinded sparse trie and subscribe to -/// transaction state stream. As it receives transaction execution results, it -/// fetches the proofs for relevant accounts from the database and reveal them -/// to the tree. -/// Then it updates relevant leaves according to the result of the transaction. -/// This feeds updates to the sparse trie task. +/// ## Architecture: Dual-Channel Multiproof System +/// +/// This task orchestrates parallel proof computation using a dual-channel architecture that +/// separates control messages from proof computation results: +/// +/// ```text +/// ┌─────────────────────────────────────────────────────────────────┐ +/// │ MultiProofTask │ +/// │ Event Loop (crossbeam::select!) │ +/// └──┬──────────────────────────────────────────────────────────▲───┘ +/// │ │ +/// │ (1) Send proof request │ +/// │ via tx (control channel) │ +/// │ │ +/// ▼ │ +/// ┌──────────────────────────────────────────────────────────────┐ │ +/// │ MultiproofManager │ │ +/// │ - Deduplicates against fetched_proof_targets │ │ +/// │ - Routes to appropriate worker pool │ │ +/// └──┬───────────────────────────────────────────────────────────┘ │ +/// │ │ +/// │ (2) Dispatch to workers │ +/// │ OR send EmptyProof (fast path) │ +/// ▼ │ +/// ┌──────────────────────────────────────────────────────────────┐ │ +/// │ ProofWorkerHandle │ │ +/// │ ┌─────────────────────┐ ┌────────────────────────┐ │ │ +/// │ │ Storage Worker Pool │ │ Account Worker Pool │ │ │ +/// │ │ (spawn_blocking) │ │ (spawn_blocking) │ │ │ +/// │ └─────────────────────┘ └────────────────────────┘ │ │ +/// └──┬───────────────────────────────────────────────────────────┘ │ +/// │ │ +/// │ (3) Compute proofs in parallel │ +/// │ Send results back │ +/// │ │ +/// ▼ │ +/// ┌──────────────────────────────────────────────────────────────┐ │ +/// │ proof_result_tx (crossbeam unbounded channel) │ │ +/// │ → ProofResultMessage { multiproof, sequence_number, ... } │ │ +/// └──────────────────────────────────────────────────────────────┘ │ +/// │ +/// (4) Receive via crossbeam::select! on two channels: ───────────┘ +/// - rx: Control messages (PrefetchProofs, StateUpdate, +/// EmptyProof, FinishedStateUpdates) +/// - proof_result_rx: Computed proof results from workers +/// ``` +/// +/// ## Component Responsibilities +/// +/// - **[`MultiProofTask`]**: Event loop coordinator +/// - Receives state updates from transaction execution +/// - Deduplicates proof targets against already-fetched proofs +/// - Sequences proofs to maintain transaction ordering +/// - Feeds sequenced updates to sparse trie task +/// +/// - **[`MultiproofManager`]**: Calculation orchestrator +/// - Decides between fast path ([`EmptyProof`]) and worker dispatch +/// - Routes storage-only vs full multiproofs to appropriate workers +/// - Records metrics for monitoring +/// +/// - **[`ProofWorkerHandle`]**: Worker pool manager +/// - Maintains separate pools for storage and account proofs +/// - Dispatches work to blocking threads (CPU-intensive) +/// - Sends results directly via `proof_result_tx` (bypasses control channel) +/// +/// [`EmptyProof`]: MultiProofMessage::EmptyProof +/// [`ProofWorkerHandle`]: reth_trie_parallel::proof_task::ProofWorkerHandle +/// +/// ## Dual-Channel Design Rationale +/// +/// The system uses two separate crossbeam channels: +/// +/// 1. **Control Channel (`tx`/`rx`)**: For orchestration messages +/// - `PrefetchProofs`: Pre-fetch proofs before execution +/// - `StateUpdate`: New transaction execution results +/// - `EmptyProof`: Fast path when all targets already fetched +/// - `FinishedStateUpdates`: Signal to drain pending work +/// +/// 2. **Proof Result Channel (`proof_result_tx`/`proof_result_rx`)**: For worker results +/// - `ProofResultMessage`: Computed multiproofs from worker pools +/// - Direct path from workers to event loop (no intermediate hops) +/// - Keeps control messages separate from high-throughput proof data +/// +/// This separation enables: +/// - **Non-blocking control**: Control messages never wait behind large proof data +/// - **Backpressure management**: Each channel can apply different policies +/// - **Clear ownership**: Workers only need proof result sender, not control channel +/// +/// ## Initialization and Lifecycle +/// +/// The task initializes a blinded sparse trie and subscribes to transaction state streams. +/// As it receives transaction execution results, it fetches proofs for relevant accounts +/// from the database and reveals them to the tree, then updates relevant leaves according +/// to transaction results. This feeds updates to the sparse trie task. +/// +/// See the `run()` method documentation for detailed lifecycle flow. #[derive(Debug)] -pub(super) struct MultiProofTask { - /// Task configuration. - config: MultiProofConfig, - /// Receiver for state root related messages. - rx: Receiver, +pub(super) struct MultiProofTask { + /// The size of proof targets chunk to spawn in one calculation. + /// If None, chunking is disabled and all targets are processed in a single proof. + chunk_size: Option, + /// Receiver for state root related messages (prefetch, state updates, finish signal). + rx: CrossbeamReceiver, /// Sender for state root related messages. - tx: Sender, + tx: CrossbeamSender, + /// Receiver for proof results directly from workers. + proof_result_rx: CrossbeamReceiver, /// Sender for state updates emitted by this type. - to_sparse_trie: Sender, + to_sparse_trie: std::sync::mpsc::Sender, /// Proof targets that have been already fetched. fetched_proof_targets: MultiProofTargets, + /// Tracks keys which have been added and removed throughout the entire block. + multi_added_removed_keys: MultiAddedRemovedKeys, /// Proof sequencing handler. proof_sequencer: ProofSequencer, /// Manages calculation of multiproofs. - multiproof_manager: MultiproofManager, + multiproof_manager: MultiproofManager, /// multi proof task metrics metrics: MultiProofTaskMetrics, } -impl MultiProofTask -where - Factory: - DatabaseProviderFactory + StateCommitmentProvider + Clone + 'static, -{ +impl MultiProofTask { /// Creates a new multi proof task with the unified message channel pub(super) fn new( - config: MultiProofConfig, - executor: WorkloadExecutor, - proof_task_handle: ProofTaskManagerHandle>, - to_sparse_trie: Sender, - max_concurrency: usize, + proof_worker_handle: ProofWorkerHandle, + to_sparse_trie: std::sync::mpsc::Sender, + chunk_size: Option, ) -> Self { - let (tx, rx) = channel(); + let (tx, rx) = unbounded(); + let (proof_result_tx, proof_result_rx) = unbounded(); let metrics = MultiProofTaskMetrics::default(); Self { - config, + chunk_size, rx, tx, + proof_result_rx, to_sparse_trie, fetched_proof_targets: Default::default(), + multi_added_removed_keys: MultiAddedRemovedKeys::new(), proof_sequencer: ProofSequencer::default(), multiproof_manager: MultiproofManager::new( - executor, metrics.clone(), - proof_task_handle, - max_concurrency, + proof_worker_handle, + proof_result_tx, ), metrics, } } - /// Returns a [`Sender`] that can be used to send arbitrary [`MultiProofMessage`]s to this task. - pub(super) fn state_root_message_sender(&self) -> Sender { + /// Returns a sender that can be used to send arbitrary [`MultiProofMessage`]s to this task. + pub(super) fn state_root_message_sender(&self) -> CrossbeamSender { self.tx.clone() } /// Handles request for proof prefetch. /// /// Returns a number of proofs that were spawned. + #[instrument( + level = "debug", + target = "engine::tree::payload_processor::multiproof", + skip_all, + fields(accounts = targets.len(), chunks = 0) + )] fn on_prefetch_proof(&mut self, targets: MultiProofTargets) -> u64 { let proof_targets = self.get_prefetch_proof_targets(targets); self.fetched_proof_targets.extend_ref(&proof_targets); + // Make sure all target accounts have an `AddedRemovedKeySet` in the + // [`MultiAddedRemovedKeys`]. Even if there are not any known removed keys for the account, + // we still want to optimistically fetch extension children for the leaf addition case. + self.multi_added_removed_keys.touch_accounts(proof_targets.keys().copied()); + + // Clone+Arc MultiAddedRemovedKeys for sharing with the dispatched multiproof tasks + let multi_added_removed_keys = Arc::new(self.multi_added_removed_keys.clone()); + self.metrics.prefetch_proof_targets_accounts_histogram.record(proof_targets.len() as f64); self.metrics .prefetch_proof_targets_storages_histogram @@ -685,20 +768,42 @@ where // Process proof targets in chunks. let mut chunks = 0; - for proof_targets_chunk in proof_targets.chunks(MULTIPROOF_TARGETS_CHUNK_SIZE) { - self.multiproof_manager.spawn_or_queue( + + // Only chunk if multiple account or storage workers are available to take advantage of + // parallelism. + let should_chunk = self.multiproof_manager.proof_worker_handle.available_account_workers() > + 1 || + self.multiproof_manager.proof_worker_handle.available_storage_workers() > 1; + + let mut dispatch = |proof_targets| { + self.multiproof_manager.dispatch( MultiproofInput { - config: self.config.clone(), source: None, hashed_state_update: Default::default(), - proof_targets: proof_targets_chunk, + proof_targets, proof_sequence_number: self.proof_sequencer.next_sequence(), state_root_message_sender: self.tx.clone(), + multi_added_removed_keys: Some(multi_added_removed_keys.clone()), } .into(), ); chunks += 1; + }; + + if should_chunk && + let Some(chunk_size) = self.chunk_size && + proof_targets.chunking_length() > chunk_size + { + let mut chunks = 0usize; + for proof_targets_chunk in proof_targets.chunks(chunk_size) { + dispatch(proof_targets_chunk); + chunks += 1; + } + tracing::Span::current().record("chunks", chunks); + } else { + dispatch(proof_targets); } + self.metrics.prefetch_proof_chunks_histogram.record(chunks as f64); chunks @@ -715,8 +820,8 @@ where let all_proofs_processed = proofs_processed >= state_update_proofs_requested + prefetch_proofs_requested; let no_pending = !self.proof_sequencer.has_pending(); - debug!( - target: "engine::root", + trace!( + target: "engine::tree::payload_processor::multiproof", proofs_processed, state_update_proofs_requested, prefetch_proofs_requested, @@ -771,7 +876,7 @@ where } if duplicates > 0 { - trace!(target: "engine::root", duplicates, "Removed duplicate prefetch proof targets"); + trace!(target: "engine::tree::payload_processor::multiproof", duplicates, "Removed duplicate prefetch proof targets"); } targets @@ -780,16 +885,26 @@ where /// Handles state updates. /// /// Returns a number of proofs that were spawned. + #[instrument( + level = "debug", + target = "engine::tree::payload_processor::multiproof", + skip(self, update), + fields(accounts = update.len(), chunks = 0) + )] fn on_state_update(&mut self, source: StateChangeSource, update: EvmState) -> u64 { let hashed_state_update = evm_state_to_hashed_post_state(update); + + // Update removed keys based on the state update. + self.multi_added_removed_keys.update_with_state(&hashed_state_update); + // Split the state update into already fetched and not fetched according to the proof // targets. - let (fetched_state_update, not_fetched_state_update) = - hashed_state_update.partition_by_targets(&self.fetched_proof_targets); + let (fetched_state_update, not_fetched_state_update) = hashed_state_update + .partition_by_targets(&self.fetched_proof_targets, &self.multi_added_removed_keys); let mut state_updates = 0; // If there are any accounts or storage slots that we already fetched the proofs for, - // send them immediately, as they don't require spawning any additional multiproofs. + // send them immediately, as they don't require dispatching any additional multiproofs. if !fetched_state_update.is_empty() { let _ = self.tx.send(MultiProofMessage::EmptyProof { sequence_number: self.proof_sequencer.next_sequence(), @@ -798,25 +913,55 @@ where state_updates += 1; } + // Clone+Arc MultiAddedRemovedKeys for sharing with the dispatched multiproof tasks + let multi_added_removed_keys = Arc::new(self.multi_added_removed_keys.clone()); + // Process state updates in chunks. let mut chunks = 0; + let mut spawned_proof_targets = MultiProofTargets::default(); - for chunk in not_fetched_state_update.chunks(MULTIPROOF_TARGETS_CHUNK_SIZE) { - let proof_targets = get_proof_targets(&chunk, &self.fetched_proof_targets); + + // Only chunk if multiple account or storage workers are available to take advantage of + // parallelism. + let should_chunk = self.multiproof_manager.proof_worker_handle.available_account_workers() > + 1 || + self.multiproof_manager.proof_worker_handle.available_storage_workers() > 1; + + let mut dispatch = |hashed_state_update| { + let proof_targets = get_proof_targets( + &hashed_state_update, + &self.fetched_proof_targets, + &multi_added_removed_keys, + ); spawned_proof_targets.extend_ref(&proof_targets); - self.multiproof_manager.spawn_or_queue( + self.multiproof_manager.dispatch( MultiproofInput { - config: self.config.clone(), source: Some(source), - hashed_state_update: chunk, + hashed_state_update, proof_targets, proof_sequence_number: self.proof_sequencer.next_sequence(), state_root_message_sender: self.tx.clone(), + multi_added_removed_keys: Some(multi_added_removed_keys.clone()), } .into(), ); + chunks += 1; + }; + + if should_chunk && + let Some(chunk_size) = self.chunk_size && + not_fetched_state_update.chunking_length() > chunk_size + { + let mut chunks = 0usize; + for chunk in not_fetched_state_update.chunks(chunk_size) { + dispatch(chunk); + chunks += 1; + } + tracing::Span::current().record("chunks", chunks); + } else { + dispatch(not_fetched_state_update); } self.metrics @@ -864,15 +1009,14 @@ where /// so that the proofs for accounts and storage slots that were already fetched are not /// requested again. /// 2. Using the proof targets, a new multiproof is calculated using - /// [`MultiproofManager::spawn_or_queue`]. + /// [`MultiproofManager::dispatch`]. /// * If the list of proof targets is empty, the [`MultiProofMessage::EmptyProof`] message is /// sent back to this task along with the original state update. - /// * Otherwise, the multiproof is calculated and the [`MultiProofMessage::ProofCalculated`] - /// message is sent back to this task along with the resulting multiproof, proof targets - /// and original state update. - /// 3. Either [`MultiProofMessage::EmptyProof`] or [`MultiProofMessage::ProofCalculated`] is - /// received. - /// * The multiproof is added to the (proof sequencer)[`ProofSequencer`]. + /// * Otherwise, the multiproof is dispatched to worker pools and results are sent directly + /// to this task via the `proof_result_rx` channel as [`ProofResultMessage`]. + /// 3. Either [`MultiProofMessage::EmptyProof`] (via control channel) or [`ProofResultMessage`] + /// (via proof result channel) is received. + /// * The multiproof is added to the [`ProofSequencer`]. /// * If the proof sequencer has a contiguous sequence of multiproofs in the same order as /// state updates arrived (i.e. transaction order), such sequence is returned. /// 4. Once there's a sequence of contiguous multiproofs along with the proof targets and state @@ -881,10 +1025,15 @@ where /// 5. Steps above are repeated until this task receives a /// [`MultiProofMessage::FinishedStateUpdates`]. /// * Once this message is received, on every [`MultiProofMessage::EmptyProof`] and - /// [`MultiProofMessage::ProofCalculated`] message, we check if there are any proofs are - /// currently being calculated, or if there are any pending proofs in the proof sequencer - /// left to be revealed by checking the pending tasks. + /// [`ProofResultMessage`], we check if all proofs have been processed and if there are any + /// pending proofs in the proof sequencer left to be revealed. /// 6. This task exits after all pending proofs are processed. + #[instrument( + level = "debug", + name = "MultiProofTask::run", + target = "engine::tree::payload_processor::multiproof", + skip_all + )] pub(crate) fn run(mut self) { // TODO convert those into fields let mut prefetch_proofs_requested = 0; @@ -902,156 +1051,171 @@ where let mut updates_finished_time = None; loop { - trace!(target: "engine::root", "entering main channel receiving loop"); - match self.rx.recv() { - Ok(message) => match message { - MultiProofMessage::PrefetchProofs(targets) => { - trace!(target: "engine::root", "processing MultiProofMessage::PrefetchProofs"); - if first_update_time.is_none() { - // record the wait time - self.metrics - .first_update_wait_time_histogram - .record(start.elapsed().as_secs_f64()); - first_update_time = Some(Instant::now()); - debug!(target: "engine::root", "Started state root calculation"); - } - - let account_targets = targets.len(); - let storage_targets = - targets.values().map(|slots| slots.len()).sum::(); - prefetch_proofs_requested += self.on_prefetch_proof(targets); - debug!( - target: "engine::root", - account_targets, - storage_targets, - prefetch_proofs_requested, - "Prefetching proofs" - ); - } - MultiProofMessage::StateUpdate(source, update) => { - trace!(target: "engine::root", "processing MultiProofMessage::StateUpdate"); - if first_update_time.is_none() { - // record the wait time - self.metrics - .first_update_wait_time_histogram - .record(start.elapsed().as_secs_f64()); - first_update_time = Some(Instant::now()); - debug!(target: "engine::root", "Started state root calculation"); - } - - let len = update.len(); - state_update_proofs_requested += self.on_state_update(source, update); - debug!( - target: "engine::root", - ?source, - len, - ?state_update_proofs_requested, - "Received new state update" - ); - } - MultiProofMessage::FinishedStateUpdates => { - trace!(target: "engine::root", "processing MultiProofMessage::FinishedStateUpdates"); - updates_finished = true; - updates_finished_time = Some(Instant::now()); - if self.is_done( - proofs_processed, - state_update_proofs_requested, - prefetch_proofs_requested, - updates_finished, - ) { - debug!( - target: "engine::root", - "State updates finished and all proofs processed, ending calculation" - ); - break - } - } - MultiProofMessage::EmptyProof { sequence_number, state } => { - trace!(target: "engine::root", "processing MultiProofMessage::EmptyProof"); + trace!(target: "engine::tree::payload_processor::multiproof", "entering main channel receiving loop"); - proofs_processed += 1; + crossbeam_channel::select_biased! { + recv(self.proof_result_rx) -> proof_msg => { + match proof_msg { + Ok(proof_result) => { + proofs_processed += 1; - if let Some(combined_update) = self.on_proof( - sequence_number, - SparseTrieUpdate { state, multiproof: MultiProof::default() }, - ) { - let _ = self.to_sparse_trie.send(combined_update); + self.metrics + .proof_calculation_duration_histogram + .record(proof_result.elapsed); + + self.multiproof_manager.on_calculation_complete(); + + // Convert ProofResultMessage to SparseTrieUpdate + match proof_result.result { + Ok(proof_result_data) => { + debug!( + target: "engine::tree::payload_processor::multiproof", + sequence = proof_result.sequence_number, + total_proofs = proofs_processed, + "Processing calculated proof from worker" + ); + + let update = SparseTrieUpdate { + state: proof_result.state, + multiproof: proof_result_data.into_multiproof(), + }; + + if let Some(combined_update) = + self.on_proof(proof_result.sequence_number, update) + { + let _ = self.to_sparse_trie.send(combined_update); + } + } + Err(error) => { + error!(target: "engine::tree::payload_processor::multiproof", ?error, "proof calculation error from worker"); + return + } + } + + if self.is_done( + proofs_processed, + state_update_proofs_requested, + prefetch_proofs_requested, + updates_finished, + ) { + debug!( + target: "engine::tree::payload_processor::multiproof", + "State updates finished and all proofs processed, ending calculation" + ); + break + } } - - if self.is_done( - proofs_processed, - state_update_proofs_requested, - prefetch_proofs_requested, - updates_finished, - ) { - debug!( - target: "engine::root", - "State updates finished and all proofs processed, ending calculation" - ); - break + Err(_) => { + error!(target: "engine::tree::payload_processor::multiproof", "Proof result channel closed unexpectedly"); + return } } - MultiProofMessage::ProofCalculated(proof_calculated) => { - trace!(target: "engine::root", "processing - MultiProofMessage::ProofCalculated"); - - // we increment proofs_processed for both state updates and prefetches, - // because both are used for the root termination condition. - proofs_processed += 1; - - self.metrics - .proof_calculation_duration_histogram - .record(proof_calculated.elapsed); - - debug!( - target: "engine::root", - sequence = proof_calculated.sequence_number, - total_proofs = proofs_processed, - "Processing calculated proof" - ); - - self.multiproof_manager.on_calculation_complete(); - - if let Some(combined_update) = - self.on_proof(proof_calculated.sequence_number, proof_calculated.update) - { - let _ = self.to_sparse_trie.send(combined_update); - } - - if self.is_done( - proofs_processed, - state_update_proofs_requested, - prefetch_proofs_requested, - updates_finished, - ) { - debug!( - target: "engine::root", - "State updates finished and all proofs processed, ending calculation"); - break + }, + recv(self.rx) -> message => { + match message { + Ok(msg) => match msg { + MultiProofMessage::PrefetchProofs(targets) => { + trace!(target: "engine::tree::payload_processor::multiproof", "processing MultiProofMessage::PrefetchProofs"); + + if first_update_time.is_none() { + // record the wait time + self.metrics + .first_update_wait_time_histogram + .record(start.elapsed().as_secs_f64()); + first_update_time = Some(Instant::now()); + debug!(target: "engine::tree::payload_processor::multiproof", "Started state root calculation"); + } + + let account_targets = targets.len(); + let storage_targets = + targets.values().map(|slots| slots.len()).sum::(); + prefetch_proofs_requested += self.on_prefetch_proof(targets); + debug!( + target: "engine::tree::payload_processor::multiproof", + account_targets, + storage_targets, + prefetch_proofs_requested, + "Prefetching proofs" + ); + } + MultiProofMessage::StateUpdate(source, update) => { + trace!(target: "engine::tree::payload_processor::multiproof", "processing MultiProofMessage::StateUpdate"); + + if first_update_time.is_none() { + // record the wait time + self.metrics + .first_update_wait_time_histogram + .record(start.elapsed().as_secs_f64()); + first_update_time = Some(Instant::now()); + debug!(target: "engine::tree::payload_processor::multiproof", "Started state root calculation"); + } + + let len = update.len(); + state_update_proofs_requested += self.on_state_update(source, update); + debug!( + target: "engine::tree::payload_processor::multiproof", + ?source, + len, + ?state_update_proofs_requested, + "Received new state update" + ); + } + MultiProofMessage::FinishedStateUpdates => { + trace!(target: "engine::tree::payload_processor::multiproof", "processing MultiProofMessage::FinishedStateUpdates"); + + updates_finished = true; + updates_finished_time = Some(Instant::now()); + + if self.is_done( + proofs_processed, + state_update_proofs_requested, + prefetch_proofs_requested, + updates_finished, + ) { + debug!( + target: "engine::tree::payload_processor::multiproof", + "State updates finished and all proofs processed, ending calculation" + ); + break + } + } + MultiProofMessage::EmptyProof { sequence_number, state } => { + trace!(target: "engine::tree::payload_processor::multiproof", "processing MultiProofMessage::EmptyProof"); + + proofs_processed += 1; + + if let Some(combined_update) = self.on_proof( + sequence_number, + SparseTrieUpdate { state, multiproof: Default::default() }, + ) { + let _ = self.to_sparse_trie.send(combined_update); + } + + if self.is_done( + proofs_processed, + state_update_proofs_requested, + prefetch_proofs_requested, + updates_finished, + ) { + debug!( + target: "engine::tree::payload_processor::multiproof", + "State updates finished and all proofs processed, ending calculation" + ); + break + } + } + }, + Err(_) => { + error!(target: "engine::tree::payload_processor::multiproof", "State root related message channel closed unexpectedly"); + return } } - MultiProofMessage::ProofCalculationError(err) => { - error!( - target: "engine::root", - ?err, - "proof calculation error" - ); - return - } - }, - Err(_) => { - // this means our internal message channel is closed, which shouldn't happen - // in normal operation since we hold both ends - error!( - target: "engine::root", - "Internal message channel closed unexpectedly" - ); } } } debug!( - target: "engine::root", + target: "engine::tree::payload_processor::multiproof", total_updates = state_update_proofs_requested, total_proofs = proofs_processed, total_time = ?first_update_time.map(|t|t.elapsed()), @@ -1080,6 +1244,7 @@ where fn get_proof_targets( state_update: &HashedPostState, fetched_proof_targets: &MultiProofTargets, + multi_added_removed_keys: &MultiAddedRemovedKeys, ) -> MultiProofTargets { let mut targets = MultiProofTargets::default(); @@ -1093,10 +1258,14 @@ fn get_proof_targets( // then process storage slots for all accounts in the state update for (hashed_address, storage) in &state_update.storages { let fetched = fetched_proof_targets.get(hashed_address); + let storage_added_removed_keys = multi_added_removed_keys.get_storage(hashed_address); let mut changed_slots = storage .storage .keys() - .filter(|slot| !fetched.is_some_and(|f| f.contains(*slot))) + .filter(|slot| { + !fetched.is_some_and(|f| f.contains(*slot)) || + storage_added_removed_keys.is_some_and(|k| k.is_removed(slot)) + }) .peekable(); // If the storage is wiped, we still need to fetch the account proof. @@ -1116,50 +1285,43 @@ fn get_proof_targets( mod tests { use super::*; use alloy_primitives::map::B256Set; - use reth_provider::{providers::ConsistentDbView, test_utils::create_test_provider_factory}; - use reth_trie::TrieInput; - use reth_trie_parallel::proof_task::{ProofTaskCtx, ProofTaskManager}; + use reth_provider::{ + providers::OverlayStateProviderFactory, test_utils::create_test_provider_factory, + BlockReader, DatabaseProviderFactory, PruneCheckpointReader, StageCheckpointReader, + TrieReader, + }; + use reth_trie::MultiProof; + use reth_trie_parallel::proof_task::{ProofTaskCtx, ProofWorkerHandle}; use revm_primitives::{B256, U256}; - use std::sync::Arc; - - fn create_state_root_config(factory: F, input: TrieInput) -> MultiProofConfig - where - F: DatabaseProviderFactory - + StateCommitmentProvider - + Clone - + 'static, - { - let consistent_view = ConsistentDbView::new(factory, None); - let nodes_sorted = Arc::new(input.nodes.clone().into_sorted()); - let state_sorted = Arc::new(input.state.clone().into_sorted()); - let prefix_sets = Arc::new(input.prefix_sets); - - MultiProofConfig { consistent_view, nodes_sorted, state_sorted, prefix_sets } + use std::sync::OnceLock; + use tokio::runtime::{Handle, Runtime}; + + /// Get a handle to the test runtime, creating it if necessary + fn get_test_runtime_handle() -> Handle { + static TEST_RT: OnceLock = OnceLock::new(); + TEST_RT + .get_or_init(|| { + tokio::runtime::Builder::new_multi_thread().enable_all().build().unwrap() + }) + .handle() + .clone() } - fn create_test_state_root_task(factory: F) -> MultiProofTask + fn create_test_state_root_task(factory: F) -> MultiProofTask where - F: DatabaseProviderFactory - + StateCommitmentProvider - + Clone + F: DatabaseProviderFactory< + Provider: BlockReader + TrieReader + StageCheckpointReader + PruneCheckpointReader, + > + Clone + + Send + 'static, { - let executor = WorkloadExecutor::default(); - let config = create_state_root_config(factory, TrieInput::default()); - let task_ctx = ProofTaskCtx::new( - config.nodes_sorted.clone(), - config.state_sorted.clone(), - config.prefix_sets.clone(), - ); - let proof_task = ProofTaskManager::new( - executor.handle().clone(), - config.consistent_view.clone(), - task_ctx, - 1, - ); - let channel = channel(); + let rt_handle = get_test_runtime_handle(); + let overlay_factory = OverlayStateProviderFactory::new(factory); + let task_ctx = ProofTaskCtx::new(overlay_factory); + let proof_handle = ProofWorkerHandle::new(rt_handle, task_ctx, 1, 1); + let (to_sparse_trie, _receiver) = std::sync::mpsc::channel(); - MultiProofTask::new(config, executor, proof_task.handle(), channel.0, 1) + MultiProofTask::new(proof_handle, to_sparse_trie, Some(1)) } #[test] @@ -1169,11 +1331,11 @@ mod tests { let proof2 = MultiProof::default(); sequencer.next_sequence = 2; - let ready = sequencer.add_proof(0, SparseTrieUpdate::from_multiproof(proof1)); + let ready = sequencer.add_proof(0, SparseTrieUpdate::from_multiproof(proof1).unwrap()); assert_eq!(ready.len(), 1); assert!(!sequencer.has_pending()); - let ready = sequencer.add_proof(1, SparseTrieUpdate::from_multiproof(proof2)); + let ready = sequencer.add_proof(1, SparseTrieUpdate::from_multiproof(proof2).unwrap()); assert_eq!(ready.len(), 1); assert!(!sequencer.has_pending()); } @@ -1186,15 +1348,15 @@ mod tests { let proof3 = MultiProof::default(); sequencer.next_sequence = 3; - let ready = sequencer.add_proof(2, SparseTrieUpdate::from_multiproof(proof3)); + let ready = sequencer.add_proof(2, SparseTrieUpdate::from_multiproof(proof3).unwrap()); assert_eq!(ready.len(), 0); assert!(sequencer.has_pending()); - let ready = sequencer.add_proof(0, SparseTrieUpdate::from_multiproof(proof1)); + let ready = sequencer.add_proof(0, SparseTrieUpdate::from_multiproof(proof1).unwrap()); assert_eq!(ready.len(), 1); assert!(sequencer.has_pending()); - let ready = sequencer.add_proof(1, SparseTrieUpdate::from_multiproof(proof2)); + let ready = sequencer.add_proof(1, SparseTrieUpdate::from_multiproof(proof2).unwrap()); assert_eq!(ready.len(), 2); assert!(!sequencer.has_pending()); } @@ -1206,10 +1368,10 @@ mod tests { let proof3 = MultiProof::default(); sequencer.next_sequence = 3; - let ready = sequencer.add_proof(0, SparseTrieUpdate::from_multiproof(proof1)); + let ready = sequencer.add_proof(0, SparseTrieUpdate::from_multiproof(proof1).unwrap()); assert_eq!(ready.len(), 1); - let ready = sequencer.add_proof(2, SparseTrieUpdate::from_multiproof(proof3)); + let ready = sequencer.add_proof(2, SparseTrieUpdate::from_multiproof(proof3).unwrap()); assert_eq!(ready.len(), 0); assert!(sequencer.has_pending()); } @@ -1220,10 +1382,10 @@ mod tests { let proof1 = MultiProof::default(); let proof2 = MultiProof::default(); - let ready = sequencer.add_proof(0, SparseTrieUpdate::from_multiproof(proof1)); + let ready = sequencer.add_proof(0, SparseTrieUpdate::from_multiproof(proof1).unwrap()); assert_eq!(ready.len(), 1); - let ready = sequencer.add_proof(0, SparseTrieUpdate::from_multiproof(proof2)); + let ready = sequencer.add_proof(0, SparseTrieUpdate::from_multiproof(proof2).unwrap()); assert_eq!(ready.len(), 0); assert!(!sequencer.has_pending()); } @@ -1234,12 +1396,13 @@ mod tests { let proofs: Vec<_> = (0..5).map(|_| MultiProof::default()).collect(); sequencer.next_sequence = 5; - sequencer.add_proof(4, SparseTrieUpdate::from_multiproof(proofs[4].clone())); - sequencer.add_proof(2, SparseTrieUpdate::from_multiproof(proofs[2].clone())); - sequencer.add_proof(1, SparseTrieUpdate::from_multiproof(proofs[1].clone())); - sequencer.add_proof(3, SparseTrieUpdate::from_multiproof(proofs[3].clone())); + sequencer.add_proof(4, SparseTrieUpdate::from_multiproof(proofs[4].clone()).unwrap()); + sequencer.add_proof(2, SparseTrieUpdate::from_multiproof(proofs[2].clone()).unwrap()); + sequencer.add_proof(1, SparseTrieUpdate::from_multiproof(proofs[1].clone()).unwrap()); + sequencer.add_proof(3, SparseTrieUpdate::from_multiproof(proofs[3].clone()).unwrap()); - let ready = sequencer.add_proof(0, SparseTrieUpdate::from_multiproof(proofs[0].clone())); + let ready = + sequencer.add_proof(0, SparseTrieUpdate::from_multiproof(proofs[0].clone()).unwrap()); assert_eq!(ready.len(), 5); assert!(!sequencer.has_pending()); } @@ -1267,7 +1430,7 @@ mod tests { let state = create_get_proof_targets_state(); let fetched = MultiProofTargets::default(); - let targets = get_proof_targets(&state, &fetched); + let targets = get_proof_targets(&state, &fetched, &MultiAddedRemovedKeys::new()); // should return all accounts as targets since nothing was fetched before assert_eq!(targets.len(), state.accounts.len()); @@ -1281,7 +1444,7 @@ mod tests { let state = create_get_proof_targets_state(); let fetched = MultiProofTargets::default(); - let targets = get_proof_targets(&state, &fetched); + let targets = get_proof_targets(&state, &fetched, &MultiAddedRemovedKeys::new()); // verify storage slots are included for accounts with storage for (addr, storage) in &state.storages { @@ -1309,7 +1472,7 @@ mod tests { // mark the account as already fetched fetched.insert(*fetched_addr, HashSet::default()); - let targets = get_proof_targets(&state, &fetched); + let targets = get_proof_targets(&state, &fetched, &MultiAddedRemovedKeys::new()); // should not include the already fetched account since it has no storage updates assert!(!targets.contains_key(fetched_addr)); @@ -1329,7 +1492,7 @@ mod tests { fetched_slots.insert(fetched_slot); fetched.insert(*addr, fetched_slots); - let targets = get_proof_targets(&state, &fetched); + let targets = get_proof_targets(&state, &fetched, &MultiAddedRemovedKeys::new()); // should not include the already fetched storage slot let target_slots = &targets[addr]; @@ -1342,7 +1505,7 @@ mod tests { let state = HashedPostState::default(); let fetched = MultiProofTargets::default(); - let targets = get_proof_targets(&state, &fetched); + let targets = get_proof_targets(&state, &fetched, &MultiAddedRemovedKeys::new()); assert!(targets.is_empty()); } @@ -1369,7 +1532,7 @@ mod tests { fetched_slots.insert(slot1); fetched.insert(addr1, fetched_slots); - let targets = get_proof_targets(&state, &fetched); + let targets = get_proof_targets(&state, &fetched, &MultiAddedRemovedKeys::new()); assert!(targets.contains_key(&addr2)); assert!(!targets[&addr1].contains(&slot1)); @@ -1395,7 +1558,7 @@ mod tests { assert!(!state.accounts.contains_key(&addr)); assert!(!fetched.contains_key(&addr)); - let targets = get_proof_targets(&state, &fetched); + let targets = get_proof_targets(&state, &fetched, &MultiAddedRemovedKeys::new()); // verify that we still get the storage slots for the unmodified account assert!(targets.contains_key(&addr)); @@ -1417,8 +1580,8 @@ mod tests { let addr2 = B256::random(); let slot1 = B256::random(); let slot2 = B256::random(); - targets.insert(addr1, vec![slot1].into_iter().collect()); - targets.insert(addr2, vec![slot2].into_iter().collect()); + targets.insert(addr1, std::iter::once(slot1).collect()); + targets.insert(addr2, std::iter::once(slot2).collect()); let prefetch_proof_targets = test_state_root_task.get_prefetch_proof_targets(targets.clone()); @@ -1430,7 +1593,7 @@ mod tests { // add a different addr and slot to fetched proof targets let addr3 = B256::random(); let slot3 = B256::random(); - test_state_root_task.fetched_proof_targets.insert(addr3, vec![slot3].into_iter().collect()); + test_state_root_task.fetched_proof_targets.insert(addr3, std::iter::once(slot3).collect()); let prefetch_proof_targets = test_state_root_task.get_prefetch_proof_targets(targets.clone()); @@ -1451,11 +1614,11 @@ mod tests { let addr2 = B256::random(); let slot1 = B256::random(); let slot2 = B256::random(); - targets.insert(addr1, vec![slot1].into_iter().collect()); - targets.insert(addr2, vec![slot2].into_iter().collect()); + targets.insert(addr1, std::iter::once(slot1).collect()); + targets.insert(addr2, std::iter::once(slot2).collect()); // add a subset of the first target to fetched proof targets - test_state_root_task.fetched_proof_targets.insert(addr1, vec![slot1].into_iter().collect()); + test_state_root_task.fetched_proof_targets.insert(addr1, std::iter::once(slot1).collect()); let prefetch_proof_targets = test_state_root_task.get_prefetch_proof_targets(targets.clone()); @@ -1478,12 +1641,119 @@ mod tests { assert!(prefetch_proof_targets.contains_key(&addr1)); assert_eq!( *prefetch_proof_targets.get(&addr1).unwrap(), - vec![slot3].into_iter().collect::() + std::iter::once(slot3).collect::() ); assert!(prefetch_proof_targets.contains_key(&addr2)); assert_eq!( *prefetch_proof_targets.get(&addr2).unwrap(), - vec![slot2].into_iter().collect::() + std::iter::once(slot2).collect::() ); } + + #[test] + fn test_get_proof_targets_with_removed_storage_keys() { + let mut state = HashedPostState::default(); + let mut fetched = MultiProofTargets::default(); + let mut multi_added_removed_keys = MultiAddedRemovedKeys::new(); + + let addr = B256::random(); + let slot1 = B256::random(); + let slot2 = B256::random(); + + // add account to state + state.accounts.insert(addr, Some(Default::default())); + + // add storage updates + let mut storage = HashedStorage::default(); + storage.storage.insert(slot1, U256::from(100)); + storage.storage.insert(slot2, U256::from(200)); + state.storages.insert(addr, storage); + + // mark slot1 as already fetched + let mut fetched_slots = HashSet::default(); + fetched_slots.insert(slot1); + fetched.insert(addr, fetched_slots); + + // update multi_added_removed_keys to mark slot1 as removed + let mut removed_state = HashedPostState::default(); + let mut removed_storage = HashedStorage::default(); + removed_storage.storage.insert(slot1, U256::ZERO); // U256::ZERO marks as removed + removed_state.storages.insert(addr, removed_storage); + multi_added_removed_keys.update_with_state(&removed_state); + + let targets = get_proof_targets(&state, &fetched, &multi_added_removed_keys); + + // slot1 should be included despite being fetched, because it's marked as removed + assert!(targets.contains_key(&addr)); + let target_slots = &targets[&addr]; + assert_eq!(target_slots.len(), 2); + assert!(target_slots.contains(&slot1)); // included because it's removed + assert!(target_slots.contains(&slot2)); // included because it's not fetched + } + + #[test] + fn test_get_proof_targets_with_wiped_storage() { + let mut state = HashedPostState::default(); + let fetched = MultiProofTargets::default(); + let multi_added_removed_keys = MultiAddedRemovedKeys::new(); + + let addr = B256::random(); + let slot1 = B256::random(); + + // add account to state + state.accounts.insert(addr, Some(Default::default())); + + // add wiped storage + let mut storage = HashedStorage::new(true); + storage.storage.insert(slot1, U256::from(100)); + state.storages.insert(addr, storage); + + let targets = get_proof_targets(&state, &fetched, &multi_added_removed_keys); + + // account should be included because storage is wiped and account wasn't fetched + assert!(targets.contains_key(&addr)); + let target_slots = &targets[&addr]; + assert_eq!(target_slots.len(), 1); + assert!(target_slots.contains(&slot1)); + } + + #[test] + fn test_get_proof_targets_removed_keys_not_in_state_update() { + let mut state = HashedPostState::default(); + let mut fetched = MultiProofTargets::default(); + let mut multi_added_removed_keys = MultiAddedRemovedKeys::new(); + + let addr = B256::random(); + let slot1 = B256::random(); + let slot2 = B256::random(); + let slot3 = B256::random(); + + // add account to state + state.accounts.insert(addr, Some(Default::default())); + + // add storage updates for slot1 and slot2 only + let mut storage = HashedStorage::default(); + storage.storage.insert(slot1, U256::from(100)); + storage.storage.insert(slot2, U256::from(200)); + state.storages.insert(addr, storage); + + // mark all slots as already fetched + let mut fetched_slots = HashSet::default(); + fetched_slots.insert(slot1); + fetched_slots.insert(slot2); + fetched_slots.insert(slot3); // slot3 is fetched but not in state update + fetched.insert(addr, fetched_slots); + + // mark slot3 as removed (even though it's not in the state update) + let mut removed_state = HashedPostState::default(); + let mut removed_storage = HashedStorage::default(); + removed_storage.storage.insert(slot3, U256::ZERO); + removed_state.storages.insert(addr, removed_storage); + multi_added_removed_keys.update_with_state(&removed_state); + + let targets = get_proof_targets(&state, &fetched, &multi_added_removed_keys); + + // only slots in the state update can be included, so slot3 should not appear + assert!(!targets.contains_key(&addr)); + } } diff --git a/crates/engine/tree/src/tree/payload_processor/prewarm.rs b/crates/engine/tree/src/tree/payload_processor/prewarm.rs index 4d94f7b84cd..ddbfc0715a1 100644 --- a/crates/engine/tree/src/tree/payload_processor/prewarm.rs +++ b/crates/engine/tree/src/tree/payload_processor/prewarm.rs @@ -1,184 +1,332 @@ //! Caching and prewarming related functionality. +//! +//! Prewarming executes transactions in parallel before the actual block execution +//! to populate the execution cache with state that will likely be accessed during +//! block processing. +//! +//! ## How Prewarming Works +//! +//! 1. Incoming transactions are split into two streams: one for prewarming (executed in parallel) +//! and one for actual execution (executed sequentially) +//! 2. Prewarming tasks execute transactions in parallel using shared caches +//! 3. When actual block execution happens, it benefits from the warmed cache use crate::tree::{ - cached_state::{CachedStateMetrics, CachedStateProvider, ProviderCaches, SavedCache}, + cached_state::{CachedStateProvider, SavedCache}, payload_processor::{ - executor::WorkloadExecutor, multiproof::MultiProofMessage, ExecutionCache, + executor::WorkloadExecutor, multiproof::MultiProofMessage, + ExecutionCache as PayloadExecutionCache, }, - StateProviderBuilder, + precompile_cache::{CachedPrecompile, PrecompileCacheMap}, + ExecutionEnv, StateProviderBuilder, }; -use alloy_consensus::transaction::Recovered; +use alloy_consensus::transaction::TxHashRef; +use alloy_eips::Typed2718; use alloy_evm::Database; use alloy_primitives::{keccak256, map::B256Set, B256}; -use itertools::Itertools; -use metrics::{Gauge, Histogram}; -use reth_evm::{ConfigureEvm, Evm, EvmFor}; +use crossbeam_channel::Sender as CrossbeamSender; +use metrics::{Counter, Gauge, Histogram}; +use reth_evm::{execute::ExecutableTxFor, ConfigureEvm, Evm, EvmFor, SpecFor}; use reth_metrics::Metrics; -use reth_primitives_traits::{header::SealedHeaderFor, NodePrimitives, SignedTransaction}; -use reth_provider::{BlockReader, StateCommitmentProvider, StateProviderFactory, StateReader}; +use reth_primitives_traits::NodePrimitives; +use reth_provider::{BlockReader, StateProviderFactory, StateReader}; use reth_revm::{database::StateProviderDatabase, db::BundleState, state::EvmState}; use reth_trie::MultiProofTargets; use std::{ - collections::VecDeque, sync::{ atomic::{AtomicBool, Ordering}, - mpsc::{channel, Receiver, Sender}, + mpsc::{self, channel, Receiver, Sender}, Arc, }, time::Instant, }; -use tracing::{debug, trace}; +use tracing::{debug, debug_span, instrument, trace, warn}; + +/// A wrapper for transactions that includes their index in the block. +#[derive(Clone)] +struct IndexedTransaction { + /// The transaction index in the block. + index: usize, + /// The wrapped transaction. + tx: Tx, +} + +/// Maximum standard Ethereum transaction type value. +/// +/// Standard transaction types are: +/// - Type 0: Legacy transactions (original Ethereum) +/// - Type 1: EIP-2930 (access list transactions) +/// - Type 2: EIP-1559 (dynamic fee transactions) +/// - Type 3: EIP-4844 (blob transactions) +/// - Type 4: EIP-7702 (set code authorization transactions) +/// +/// Any transaction with a type > 4 is considered a non-standard/system transaction, +/// typically used by L2s for special purposes (e.g., Optimism deposit transactions use type 126). +const MAX_STANDARD_TX_TYPE: u8 = 4; /// A task that is responsible for caching and prewarming the cache by executing transactions /// individually in parallel. /// /// Note: This task runs until cancelled externally. -pub(super) struct PrewarmCacheTask { +pub(super) struct PrewarmCacheTask +where + N: NodePrimitives, + Evm: ConfigureEvm, +{ /// The executor used to spawn execution tasks. executor: WorkloadExecutor, /// Shared execution cache. - execution_cache: ExecutionCache, - /// Transactions pending execution. - pending: VecDeque>, + execution_cache: PayloadExecutionCache, /// Context provided to execution tasks ctx: PrewarmContext, /// How many transactions should be executed in parallel max_concurrency: usize, + /// The number of transactions to be processed + transaction_count_hint: usize, /// Sender to emit evm state outcome messages, if any. - to_multi_proof: Option>, + to_multi_proof: Option>, /// Receiver for events produced by tx execution actions_rx: Receiver, - /// Sender the transactions use to send their result back - actions_tx: Sender, - /// Total prewarming tasks spawned - prewarm_outcomes_left: usize, } impl PrewarmCacheTask where N: NodePrimitives, - P: BlockReader + StateProviderFactory + StateReader + StateCommitmentProvider + Clone + 'static, + P: BlockReader + StateProviderFactory + StateReader + Clone + 'static, Evm: ConfigureEvm + 'static, { /// Initializes the task with the given transactions pending execution pub(super) fn new( executor: WorkloadExecutor, - execution_cache: ExecutionCache, + execution_cache: PayloadExecutionCache, ctx: PrewarmContext, - to_multi_proof: Option>, - pending: VecDeque>, - ) -> Self { + to_multi_proof: Option>, + transaction_count_hint: usize, + max_concurrency: usize, + ) -> (Self, Sender) { let (actions_tx, actions_rx) = channel(); - Self { - executor, - execution_cache, - pending, - ctx, - max_concurrency: 64, - to_multi_proof, - actions_rx, - actions_tx, - prewarm_outcomes_left: 0, - } - } - /// Returns the sender that can communicate with this task. - pub(super) fn actions_tx(&self) -> Sender { - self.actions_tx.clone() + trace!( + target: "engine::tree::payload_processor::prewarm", + max_concurrency, + transaction_count_hint, + "Initialized prewarm task" + ); + + ( + Self { + executor, + execution_cache, + ctx, + max_concurrency, + transaction_count_hint, + to_multi_proof, + actions_rx, + }, + actions_tx, + ) } /// Spawns all pending transactions as blocking tasks by first chunking them. - fn spawn_all(&mut self) { - let chunk_size = (self.pending.len() / self.max_concurrency).max(1); + /// + /// For Optimism chains, special handling is applied to the first transaction if it's a + /// deposit transaction (type 0x7E/126) which sets critical metadata that affects all + /// subsequent transactions in the block. + fn spawn_all(&self, pending: mpsc::Receiver, actions_tx: Sender) + where + Tx: ExecutableTxFor + Clone + Send + 'static, + { + let executor = self.executor.clone(); + let ctx = self.ctx.clone(); + let max_concurrency = self.max_concurrency; + let transaction_count_hint = self.transaction_count_hint; + let span = tracing::Span::current(); + + self.executor.spawn_blocking(move || { + let _enter = debug_span!(target: "engine::tree::payload_processor::prewarm", parent: span, "spawn_all").entered(); + + let (done_tx, done_rx) = mpsc::channel(); + + // When transaction_count_hint is 0, it means the count is unknown. In this case, spawn + // max workers to handle potentially many transactions in parallel rather + // than bottlenecking on a single worker. + let workers_needed = if transaction_count_hint == 0 { + max_concurrency + } else { + transaction_count_hint.min(max_concurrency) + }; - for chunk in &self.pending.drain(..).chunks(chunk_size) { - let sender = self.actions_tx.clone(); - let ctx = self.ctx.clone(); - let pending_chunk = chunk.collect::>(); + // Initialize worker handles container + let mut handles = Vec::with_capacity(workers_needed); - self.prewarm_outcomes_left += pending_chunk.len(); - self.executor.spawn_blocking(move || { - ctx.transact_batch(&pending_chunk, sender); - }); - } + // Only spawn initial workers as needed + for i in 0..workers_needed { + handles.push(ctx.spawn_worker(i, &executor, actions_tx.clone(), done_tx.clone())); + } + + // Distribute transactions to workers + let mut tx_index = 0usize; + while let Ok(tx) = pending.recv() { + // Stop distributing if termination was requested + if ctx.terminate_execution.load(Ordering::Relaxed) { + trace!( + target: "engine::tree::payload_processor::prewarm", + "Termination requested, stopping transaction distribution" + ); + break; + } + + let indexed_tx = IndexedTransaction { index: tx_index, tx }; + let is_system_tx = indexed_tx.tx.tx().ty() > MAX_STANDARD_TX_TYPE; + + // System transactions (type > 4) in the first position set critical metadata + // that affects all subsequent transactions (e.g., L1 block info on L2s). + // Broadcast the first system transaction to all workers to ensure they have + // the critical state. This is particularly important for L2s like Optimism + // where the first deposit transaction (type 126) contains essential block metadata. + if tx_index == 0 && is_system_tx { + for handle in &handles { + // Ignore send errors: workers listen to terminate_execution and may + // exit early when signaled. Sending to a disconnected worker is + // possible and harmless and should happen at most once due to + // the terminate_execution check above. + let _ = handle.send(indexed_tx.clone()); + } + } else { + // Round-robin distribution for all other transactions + let worker_idx = tx_index % workers_needed; + // Ignore send errors: workers listen to terminate_execution and may + // exit early when signaled. Sending to a disconnected worker is + // possible and harmless and should happen at most once due to + // the terminate_execution check above. + let _ = handles[worker_idx].send(indexed_tx); + } + + tx_index += 1; + } + + // drop handle and wait for all tasks to finish and drop theirs + drop(done_tx); + drop(handles); + while done_rx.recv().is_ok() {} + + let _ = actions_tx + .send(PrewarmTaskEvent::FinishedTxExecution { executed_transactions: tx_index }); + }); + } + + /// Returns true if prewarming was terminated and no more transactions should be prewarmed. + fn is_execution_terminated(&self) -> bool { + self.ctx.terminate_execution.load(Ordering::Relaxed) } /// If configured and the tx returned proof targets, emit the targets the transaction produced fn send_multi_proof_targets(&self, targets: Option) { + if self.is_execution_terminated() { + // if execution is already terminated then we dont need to send more proof fetch + // messages + return + } + if let Some((proof_targets, to_multi_proof)) = targets.zip(self.to_multi_proof.as_ref()) { let _ = to_multi_proof.send(MultiProofMessage::PrefetchProofs(proof_targets)); } } - /// Save the state to the shared cache for the given block. + /// This method calls `ExecutionCache::update_with_guard` which requires exclusive access. + /// It should only be called after ensuring that: + /// 1. All prewarming tasks have completed execution + /// 2. No other concurrent operations are accessing the cache + /// + /// Saves the warmed caches back into the shared slot after prewarming completes. + /// + /// This consumes the `SavedCache` held by the task, which releases its usage guard and allows + /// the new, warmed cache to be inserted. + /// + /// This method is called from `run()` only after all execution tasks are complete. + #[instrument(level = "debug", target = "engine::tree::payload_processor::prewarm", skip_all)] fn save_cache(self, state: BundleState) { let start = Instant::now(); - let cache = SavedCache::new( - self.ctx.header.hash(), - self.ctx.cache.clone(), - self.ctx.cache_metrics.clone(), - ); - if cache.cache().insert_state(&state).is_err() { - return - } - cache.update_metrics(); + let Self { execution_cache, ctx: PrewarmContext { env, metrics, saved_cache, .. }, .. } = + self; + let hash = env.hash; + + debug!(target: "engine::caching", parent_hash=?hash, "Updating execution cache"); + // Perform all cache operations atomically under the lock + execution_cache.update_with_guard(|cached| { + // consumes the `SavedCache` held by the prewarming task, which releases its usage guard + let (caches, cache_metrics) = saved_cache.split(); + let new_cache = SavedCache::new(hash, caches, cache_metrics); + + // Insert state into cache while holding the lock + if new_cache.cache().insert_state(&state).is_err() { + // Clear the cache on error to prevent having a polluted cache + *cached = None; + debug!(target: "engine::caching", "cleared execution cache on update error"); + return; + } - debug!(target: "engine::caching", "Updated state caches"); + new_cache.update_metrics(); - // update the cache for the executed block - self.execution_cache.save_cache(cache); - self.ctx.metrics.cache_saving_duration.set(start.elapsed().as_secs_f64()); - } + // Replace the shared cache with the new one; the previous cache (if any) is dropped. + *cached = Some(new_cache); + }); - /// Removes the `actions_tx` currently stored in the struct, replacing it with a new one that - /// does not point to any active receiver. - /// - /// This is used to drop the `actions_tx` after all tasks have been spawned, and should not be - /// used in any context other than the `run` method. - fn drop_actions_tx(&mut self) { - self.actions_tx = channel().0; + let elapsed = start.elapsed(); + debug!(target: "engine::caching", parent_hash=?hash, elapsed=?elapsed, "Updated execution cache"); + + metrics.cache_saving_duration.set(elapsed.as_secs_f64()); } /// Executes the task. /// /// This will execute the transactions until all transactions have been processed or the task /// was cancelled. - pub(super) fn run(mut self) { - self.ctx.metrics.transactions.set(self.pending.len() as f64); - self.ctx.metrics.transactions_histogram.record(self.pending.len() as f64); - + #[instrument( + level = "debug", + target = "engine::tree::payload_processor::prewarm", + name = "prewarm", + skip_all + )] + pub(super) fn run( + self, + pending: mpsc::Receiver + Clone + Send + 'static>, + actions_tx: Sender, + ) { // spawn execution tasks. - self.spawn_all(); - - // drop the actions sender after we've spawned all execution tasks. This is so that the - // following loop can terminate even if one of the prewarm tasks ends in an error (i.e., - // does not return an Outcome) or panics. - self.drop_actions_tx(); + self.spawn_all(pending, actions_tx); let mut final_block_output = None; + let mut finished_execution = false; while let Ok(event) = self.actions_rx.recv() { match event { PrewarmTaskEvent::TerminateTransactionExecution => { // stop tx processing + debug!(target: "engine::tree::prewarm", "Terminating prewarm execution"); self.ctx.terminate_execution.store(true, Ordering::Relaxed); } PrewarmTaskEvent::Outcome { proof_targets } => { // completed executing a set of transactions self.send_multi_proof_targets(proof_targets); + } + PrewarmTaskEvent::Terminate { block_output } => { + trace!(target: "engine::tree::payload_processor::prewarm", "Received termination signal"); + final_block_output = Some(block_output); - // decrement the number of tasks left - self.prewarm_outcomes_left -= 1; - - if self.prewarm_outcomes_left == 0 && final_block_output.is_some() { - // all tasks are done, and we have the block output, we can exit + if finished_execution { + // all tasks are done, we can exit, which will save caches and exit break } } - PrewarmTaskEvent::Terminate { block_output } => { - final_block_output = Some(block_output); + PrewarmTaskEvent::FinishedTxExecution { executed_transactions } => { + trace!(target: "engine::tree::payload_processor::prewarm", "Finished prewarm execution signal"); + self.ctx.metrics.transactions.set(executed_transactions as f64); + self.ctx.metrics.transactions_histogram.record(executed_transactions as f64); + + finished_execution = true; - if self.prewarm_outcomes_left == 0 { + if final_block_output.is_some() { // all tasks are done, we can exit, which will save caches and exit break } @@ -186,6 +334,8 @@ where } } + debug!(target: "engine::tree::payload_processor::prewarm", "Completed prewarm execution"); + // save caches and finish if let Some(Some(state)) = final_block_output { self.save_cache(state); @@ -195,44 +345,49 @@ where /// Context required by tx execution tasks. #[derive(Debug, Clone)] -pub(super) struct PrewarmContext { - pub(super) header: SealedHeaderFor, +pub(super) struct PrewarmContext +where + N: NodePrimitives, + Evm: ConfigureEvm, +{ + pub(super) env: ExecutionEnv, pub(super) evm_config: Evm, - pub(super) cache: ProviderCaches, - pub(super) cache_metrics: CachedStateMetrics, + pub(super) saved_cache: SavedCache, /// Provider to obtain the state pub(super) provider: StateProviderBuilder, pub(super) metrics: PrewarmMetrics, /// An atomic bool that tells prewarm tasks to not start any more execution. pub(super) terminate_execution: Arc, + pub(super) precompile_cache_disabled: bool, + pub(super) precompile_cache_map: PrecompileCacheMap>, } impl PrewarmContext where N: NodePrimitives, - P: BlockReader + StateProviderFactory + StateReader + StateCommitmentProvider + Clone + 'static, + P: BlockReader + StateProviderFactory + StateReader + Clone + 'static, Evm: ConfigureEvm + 'static, { /// Splits this context into an evm, an evm config, metrics, and the atomic bool for terminating /// execution. - fn evm_for_ctx( - self, - ) -> Option<(EvmFor, Evm, PrewarmMetrics, Arc)> { + #[instrument(level = "debug", target = "engine::tree::payload_processor::prewarm", skip_all)] + fn evm_for_ctx(self) -> Option<(EvmFor, PrewarmMetrics, Arc)> { let Self { - header, + env, evm_config, - cache: caches, - cache_metrics, + saved_cache, provider, metrics, terminate_execution, + precompile_cache_disabled, + mut precompile_cache_map, } = self; let state_provider = match provider.build() { Ok(provider) => provider, Err(err) => { trace!( - target: "engine::tree", + target: "engine::tree::payload_processor::prewarm", %err, "Failed to build state provider in prewarm thread" ); @@ -241,67 +396,144 @@ where }; // Use the caches to create a new provider with caching + let caches = saved_cache.cache().clone(); + let cache_metrics = saved_cache.metrics().clone(); let state_provider = CachedStateProvider::new_with_caches(state_provider, caches, cache_metrics); let state_provider = StateProviderDatabase::new(state_provider); - let mut evm_env = evm_config.evm_env(&header); + let mut evm_env = env.evm_env; // we must disable the nonce check so that we can execute the transaction even if the nonce // doesn't match what's on chain. evm_env.cfg_env.disable_nonce_check = true; // create a new executor and disable nonce checks in the env - let evm = evm_config.evm_with_env(state_provider, evm_env); + let spec_id = *evm_env.spec_id(); + let mut evm = evm_config.evm_with_env(state_provider, evm_env); + + if !precompile_cache_disabled { + // Only cache pure precompiles to avoid issues with stateful precompiles + evm.precompiles_mut().map_pure_precompiles(|address, precompile| { + CachedPrecompile::wrap( + precompile, + precompile_cache_map.cache_for_address(*address), + spec_id, + None, // No metrics for prewarm + ) + }); + } - Some((evm, evm_config, metrics, terminate_execution)) + Some((evm, metrics, terminate_execution)) } - /// Transacts the vec of transactions and returns the state outcome. + /// Accepts an [`mpsc::Receiver`] of transactions and a handle to prewarm task. Executes + /// transactions and streams [`PrewarmTaskEvent::Outcome`] messages for each transaction. /// /// Returns `None` if executing the transactions failed to a non Revert error. /// Returns the touched+modified state of the transaction. /// - /// Note: Since here are no ordering guarantees this won't the state the txs produce when - /// executed sequentially. - fn transact_batch(self, txs: &[Recovered], sender: Sender) { - let Some((mut evm, evm_config, metrics, terminate_execution)) = self.evm_for_ctx() else { - return - }; + /// Note: There are no ordering guarantees; this does not reflect the state produced by + /// sequential execution. + #[instrument(level = "debug", target = "engine::tree::payload_processor::prewarm", skip_all)] + fn transact_batch( + self, + txs: mpsc::Receiver>, + sender: Sender, + done_tx: Sender<()>, + ) where + Tx: ExecutableTxFor, + { + let Some((mut evm, metrics, terminate_execution)) = self.evm_for_ctx() else { return }; + + while let Ok(IndexedTransaction { index, tx }) = { + let _enter = debug_span!(target: "engine::tree::payload_processor::prewarm", "recv tx") + .entered(); + txs.recv() + } { + let _enter = + debug_span!(target: "engine::tree::payload_processor::prewarm", "prewarm tx", index, tx_hash=%tx.tx().tx_hash()) + .entered(); + + // create the tx env + let start = Instant::now(); - for tx in txs { // If the task was cancelled, stop execution, send an empty result to notify the task, // and exit. if terminate_execution.load(Ordering::Relaxed) { let _ = sender.send(PrewarmTaskEvent::Outcome { proof_targets: None }); - return + break } - // create the tx env - let tx_env = evm_config.tx_env(tx); - let start = Instant::now(); - let res = match evm.transact(tx_env) { + let res = match evm.transact(&tx) { Ok(res) => res, Err(err) => { trace!( - target: "engine::tree", + target: "engine::tree::payload_processor::prewarm", %err, - tx_hash=%tx.tx_hash(), + tx_hash=%tx.tx().tx_hash(), sender=%tx.signer(), "Error when executing prewarm transaction", ); - return + // Track transaction execution errors + metrics.transaction_errors.increment(1); + // skip error because we can ignore these errors and continue with the next tx + continue } }; metrics.execution_duration.record(start.elapsed()); - let (targets, storage_targets) = multiproof_targets_from_state(res.state); - metrics.prefetch_storage_targets.record(storage_targets as f64); - metrics.total_runtime.record(start.elapsed()); + drop(_enter); - let _ = sender.send(PrewarmTaskEvent::Outcome { proof_targets: Some(targets) }); + // If the task was cancelled, stop execution, send an empty result to notify the task, + // and exit. + if terminate_execution.load(Ordering::Relaxed) { + let _ = sender.send(PrewarmTaskEvent::Outcome { proof_targets: None }); + break + } + + // Only send outcome for transactions after the first txn + // as the main execution will be just as fast + if index > 0 { + let _enter = + debug_span!(target: "engine::tree::payload_processor::prewarm", "prewarm outcome", index, tx_hash=%tx.tx().tx_hash()) + .entered(); + let (targets, storage_targets) = multiproof_targets_from_state(res.state); + metrics.prefetch_storage_targets.record(storage_targets as f64); + let _ = sender.send(PrewarmTaskEvent::Outcome { proof_targets: Some(targets) }); + drop(_enter); + } + + metrics.total_runtime.record(start.elapsed()); } + + // send a message to the main task to flag that we're done + let _ = done_tx.send(()); + } + + /// Spawns a worker task for transaction execution and returns its sender channel. + fn spawn_worker( + &self, + idx: usize, + executor: &WorkloadExecutor, + actions_tx: Sender, + done_tx: Sender<()>, + ) -> mpsc::Sender> + where + Tx: ExecutableTxFor + Send + 'static, + { + let (tx, rx) = mpsc::channel(); + let ctx = self.clone(); + let span = + debug_span!(target: "engine::tree::payload_processor::prewarm", "prewarm worker", idx); + + executor.spawn_blocking(move || { + let _enter = span.entered(); + ctx.transact_batch(rx, actions_tx, done_tx); + }); + + tx } } @@ -355,6 +587,11 @@ pub(super) enum PrewarmTaskEvent { /// The prepared proof targets based on the evm state outcome proof_targets: Option, }, + /// Finished executing all transactions + FinishedTxExecution { + /// Number of transactions executed + executed_transactions: usize, + }, } /// Metrics for transactions prewarming. @@ -373,4 +610,6 @@ pub(crate) struct PrewarmMetrics { pub(crate) prefetch_storage_targets: Histogram, /// A histogram of duration for cache saving pub(crate) cache_saving_duration: Gauge, + /// Counter for transaction execution errors during prewarming + pub(crate) transaction_errors: Counter, } diff --git a/crates/engine/tree/src/tree/payload_processor/sparse_trie.rs b/crates/engine/tree/src/tree/payload_processor/sparse_trie.rs index 7119c519c19..6302abde5fb 100644 --- a/crates/engine/tree/src/tree/payload_processor/sparse_trie.rs +++ b/crates/engine/tree/src/tree/payload_processor/sparse_trie.rs @@ -1,66 +1,54 @@ //! Sparse Trie task related functionality. -use crate::tree::payload_processor::{ - executor::WorkloadExecutor, - multiproof::{MultiProofTaskMetrics, SparseTrieUpdate}, -}; +use crate::tree::payload_processor::multiproof::{MultiProofTaskMetrics, SparseTrieUpdate}; use alloy_primitives::B256; use rayon::iter::{ParallelBridge, ParallelIterator}; use reth_trie::{updates::TrieUpdates, Nibbles}; use reth_trie_parallel::root::ParallelStateRootError; use reth_trie_sparse::{ - blinded::{BlindedProvider, BlindedProviderFactory}, errors::{SparseStateTrieResult, SparseTrieErrorKind}, - SparseStateTrie, + provider::{TrieNodeProvider, TrieNodeProviderFactory}, + ClearedSparseStateTrie, SerialSparseTrie, SparseStateTrie, SparseTrieInterface, }; +use smallvec::SmallVec; use std::{ sync::mpsc, time::{Duration, Instant}, }; -use tracing::{debug, trace, trace_span}; - -/// The level below which the sparse trie hashes are calculated in -/// [`update_sparse_trie`]. -const SPARSE_TRIE_INCREMENTAL_LEVEL: usize = 2; +use tracing::{debug, debug_span, instrument, trace}; /// A task responsible for populating the sparse trie. -pub(super) struct SparseTrieTask +pub(super) struct SparseTrieTask where - BPF: BlindedProviderFactory + Send + Sync, - BPF::AccountNodeProvider: BlindedProvider + Send + Sync, - BPF::StorageNodeProvider: BlindedProvider + Send + Sync, + BPF: TrieNodeProviderFactory + Send + Sync, + BPF::AccountNodeProvider: TrieNodeProvider + Send + Sync, + BPF::StorageNodeProvider: TrieNodeProvider + Send + Sync, { - /// Executor used to spawn subtasks. - #[expect(unused)] // TODO use this for spawning trie tasks - pub(super) executor: WorkloadExecutor, /// Receives updates from the state root task. pub(super) updates: mpsc::Receiver, - /// Sparse Trie initialized with the blinded provider factory. - /// - /// It's kept as a field on the struct to prevent blocking on de-allocation in [`Self::run`]. - pub(super) trie: SparseStateTrie, + /// `SparseStateTrie` used for computing the state root. + pub(super) trie: SparseStateTrie, pub(super) metrics: MultiProofTaskMetrics, + /// Trie node provider factory. + blinded_provider_factory: BPF, } -impl SparseTrieTask +impl SparseTrieTask where - BPF: BlindedProviderFactory + Send + Sync, - BPF::AccountNodeProvider: BlindedProvider + Send + Sync, - BPF::StorageNodeProvider: BlindedProvider + Send + Sync, + BPF: TrieNodeProviderFactory + Send + Sync + Clone, + BPF::AccountNodeProvider: TrieNodeProvider + Send + Sync, + BPF::StorageNodeProvider: TrieNodeProvider + Send + Sync, + A: SparseTrieInterface + Send + Sync + Default, + S: SparseTrieInterface + Send + Sync + Default + Clone, { - /// Creates a new sparse trie task. - pub(super) fn new( - executor: WorkloadExecutor, + /// Creates a new sparse trie, pre-populating with a [`ClearedSparseStateTrie`]. + pub(super) fn new_with_cleared_trie( updates: mpsc::Receiver, blinded_provider_factory: BPF, metrics: MultiProofTaskMetrics, + sparse_state_trie: ClearedSparseStateTrie, ) -> Self { - Self { - executor, - updates, - metrics, - trie: SparseStateTrie::new(blinded_provider_factory).with_updates(true), - } + Self { updates, metrics, trie: sparse_state_trie.into_inner(), blinded_provider_factory } } /// Runs the sparse trie task to completion. @@ -69,9 +57,27 @@ where /// /// This concludes once the last trie update has been received. /// - /// NOTE: This function does not take `self` by value to prevent blocking on [`SparseStateTrie`] - /// drop. - pub(super) fn run(&mut self) -> Result { + /// # Returns + /// + /// - State root computation outcome. + /// - `SparseStateTrie` that needs to be cleared and reused to avoid reallocations. + #[instrument( + level = "debug", + target = "engine::tree::payload_processor::sparse_trie", + skip_all + )] + pub(super) fn run( + mut self, + ) -> (Result, SparseStateTrie) { + // run the main loop to completion + let result = self.run_inner(); + (result, self.trie) + } + + /// Inner function to run the sparse trie task to completion. + /// + /// See [`Self::run`] for more information. + fn run_inner(&mut self) -> Result { let now = Instant::now(); let mut num_iterations = 0; @@ -79,10 +85,14 @@ where while let Ok(mut update) = self.updates.recv() { num_iterations += 1; let mut num_updates = 1; + let _enter = + debug_span!(target: "engine::tree::payload_processor::sparse_trie", "drain updates") + .entered(); while let Ok(next) = self.updates.try_recv() { update.extend(next); num_updates += 1; } + drop(_enter); debug!( target: "engine::root", @@ -92,9 +102,13 @@ where "Updating sparse trie" ); - let elapsed = update_sparse_trie(&mut self.trie, update).map_err(|e| { - ParallelStateRootError::Other(format!("could not calculate state root: {e:?}")) - })?; + let elapsed = + update_sparse_trie(&mut self.trie, update, &self.blinded_provider_factory) + .map_err(|e| { + ParallelStateRootError::Other(format!( + "could not calculate state root: {e:?}" + )) + })?; self.metrics.sparse_trie_update_duration_histogram.record(elapsed); trace!(target: "engine::root", ?elapsed, num_iterations, "Root calculation completed"); } @@ -102,9 +116,10 @@ where debug!(target: "engine::root", num_iterations, "All proofs processed, ending calculation"); let start = Instant::now(); - let (state_root, trie_updates) = self.trie.root_with_updates().map_err(|e| { - ParallelStateRootError::Other(format!("could not calculate state root: {e:?}")) - })?; + let (state_root, trie_updates) = + self.trie.root_with_updates(&self.blinded_provider_factory).map_err(|e| { + ParallelStateRootError::Other(format!("could not calculate state root: {e:?}")) + })?; self.metrics.sparse_trie_final_update_duration_histogram.record(start.elapsed()); self.metrics.sparse_trie_total_duration_histogram.record(now.elapsed()); @@ -124,20 +139,24 @@ pub struct StateRootComputeOutcome { } /// Updates the sparse trie with the given proofs and state, and returns the elapsed time. -pub(crate) fn update_sparse_trie( - trie: &mut SparseStateTrie, +#[instrument(level = "debug", target = "engine::tree::payload_processor::sparse_trie", skip_all)] +pub(crate) fn update_sparse_trie( + trie: &mut SparseStateTrie, SparseTrieUpdate { mut state, multiproof }: SparseTrieUpdate, + blinded_provider_factory: &BPF, ) -> SparseStateTrieResult where - BPF: BlindedProviderFactory + Send + Sync, - BPF::AccountNodeProvider: BlindedProvider + Send + Sync, - BPF::StorageNodeProvider: BlindedProvider + Send + Sync, + BPF: TrieNodeProviderFactory + Send + Sync, + BPF::AccountNodeProvider: TrieNodeProvider + Send + Sync, + BPF::StorageNodeProvider: TrieNodeProvider + Send + Sync, + A: SparseTrieInterface + Send + Sync + Default, + S: SparseTrieInterface + Send + Sync + Default + Clone, { trace!(target: "engine::root::sparse", "Updating sparse trie"); let started_at = Instant::now(); // Reveal new accounts and storage slots. - trie.reveal_multiproof(multiproof)?; + trie.reveal_decoded_multiproof(multiproof)?; let reveal_multiproof_elapsed = started_at.elapsed(); trace!( target: "engine::root::sparse", @@ -146,6 +165,7 @@ where ); // Update storage slots with new values and calculate storage roots. + let span = tracing::Span::current(); let (tx, rx) = mpsc::channel(); state .storages @@ -153,35 +173,68 @@ where .map(|(address, storage)| (address, storage, trie.take_storage_trie(&address))) .par_bridge() .map(|(address, storage, storage_trie)| { - let span = trace_span!(target: "engine::root::sparse", "Storage trie", ?address); - let _enter = span.enter(); - trace!(target: "engine::root::sparse", "Updating storage"); + let _enter = + debug_span!(target: "engine::tree::payload_processor::sparse_trie", parent: span.clone(), "storage trie", ?address) + .entered(); + + trace!(target: "engine::tree::payload_processor::sparse_trie", "Updating storage"); + let storage_provider = blinded_provider_factory.storage_node_provider(address); let mut storage_trie = storage_trie.ok_or(SparseTrieErrorKind::Blind)?; if storage.wiped { - trace!(target: "engine::root::sparse", "Wiping storage"); + trace!(target: "engine::tree::payload_processor::sparse_trie", "Wiping storage"); storage_trie.wipe()?; } + + // Defer leaf removals until after updates/additions, so that we don't delete an + // intermediate branch node during a removal and then re-add that branch back during a + // later leaf addition. This is an optimization, but also a requirement inherited from + // multiproof generating, which can't know the order that leaf operations happen in. + let mut removed_slots = SmallVec::<[Nibbles; 8]>::new(); + for (slot, value) in storage.storage { let slot_nibbles = Nibbles::unpack(slot); + if value.is_zero() { - trace!(target: "engine::root::sparse", ?slot, "Removing storage slot"); - storage_trie.remove_leaf(&slot_nibbles)?; - } else { - trace!(target: "engine::root::sparse", ?slot, "Updating storage slot"); - storage_trie - .update_leaf(slot_nibbles, alloy_rlp::encode_fixed_size(&value).to_vec())?; + removed_slots.push(slot_nibbles); + continue; } + + trace!(target: "engine::tree::payload_processor::sparse_trie", ?slot_nibbles, "Updating storage slot"); + storage_trie.update_leaf( + slot_nibbles, + alloy_rlp::encode_fixed_size(&value).to_vec(), + &storage_provider, + )?; + } + + for slot_nibbles in removed_slots { + trace!(target: "engine::root::sparse", ?slot_nibbles, "Removing storage slot"); + storage_trie.remove_leaf(&slot_nibbles, &storage_provider)?; } storage_trie.root(); SparseStateTrieResult::Ok((address, storage_trie)) }) - .for_each_init(|| tx.clone(), |tx, result| tx.send(result).unwrap()); + .for_each_init( + || tx.clone(), + |tx, result| { + let _ = tx.send(result); + }, + ); drop(tx); + // Defer leaf removals until after updates/additions, so that we don't delete an intermediate + // branch node during a removal and then re-add that branch back during a later leaf addition. + // This is an optimization, but also a requirement inherited from multiproof generating, which + // can't know the order that leaf operations happen in. + let mut removed_accounts = Vec::new(); + // Update account storage roots + let _enter = + tracing::debug_span!(target: "engine::tree::payload_processor::sparse_trie", "account trie") + .entered(); for result in rx { let (address, storage_trie) = result?; trie.insert_storage_trie(address, storage_trie); @@ -190,33 +243,48 @@ where // If the account itself has an update, remove it from the state update and update in // one go instead of doing it down below. trace!(target: "engine::root::sparse", ?address, "Updating account and its storage root"); - trie.update_account(address, account.unwrap_or_default())?; + if !trie.update_account( + address, + account.unwrap_or_default(), + blinded_provider_factory, + )? { + removed_accounts.push(address); + } } else if trie.is_account_revealed(address) { // Otherwise, if the account is revealed, only update its storage root. trace!(target: "engine::root::sparse", ?address, "Updating account storage root"); - trie.update_account_storage_root(address)?; + if !trie.update_account_storage_root(address, blinded_provider_factory)? { + removed_accounts.push(address); + } } } // Update accounts for (address, account) in state.accounts { trace!(target: "engine::root::sparse", ?address, "Updating account"); - trie.update_account(address, account.unwrap_or_default())?; + if !trie.update_account(address, account.unwrap_or_default(), blinded_provider_factory)? { + removed_accounts.push(address); + } + } + + // Remove accounts + for address in removed_accounts { + trace!(target: "engine::root::sparse", ?address, "Removing account"); + let nibbles = Nibbles::unpack(address); + trie.remove_account_leaf(&nibbles, blinded_provider_factory)?; } let elapsed_before = started_at.elapsed(); trace!( - target: "engine::root:sparse", - level=SPARSE_TRIE_INCREMENTAL_LEVEL, - "Calculating intermediate nodes below trie level" + target: "engine::root::sparse", + "Calculating subtries" ); - trie.calculate_below_level(SPARSE_TRIE_INCREMENTAL_LEVEL); + trie.calculate_subtries(); let elapsed = started_at.elapsed(); let below_level_elapsed = elapsed - elapsed_before; trace!( - target: "engine::root:sparse", - level=SPARSE_TRIE_INCREMENTAL_LEVEL, + target: "engine::root::sparse", ?below_level_elapsed, "Intermediate nodes calculated" ); diff --git a/crates/engine/tree/src/tree/payload_validator.rs b/crates/engine/tree/src/tree/payload_validator.rs new file mode 100644 index 00000000000..c8f5a5d6209 --- /dev/null +++ b/crates/engine/tree/src/tree/payload_validator.rs @@ -0,0 +1,1146 @@ +//! Types and traits for validating blocks and payloads. + +use crate::tree::{ + cached_state::CachedStateProvider, + error::{InsertBlockError, InsertBlockErrorKind, InsertPayloadError}, + executor::WorkloadExecutor, + instrumented_state::InstrumentedStateProvider, + payload_processor::{multiproof::MultiProofConfig, PayloadProcessor}, + precompile_cache::{CachedPrecompile, CachedPrecompileMetrics, PrecompileCacheMap}, + sparse_trie::StateRootComputeOutcome, + EngineApiMetrics, EngineApiTreeState, ExecutionEnv, PayloadHandle, StateProviderBuilder, + StateProviderDatabase, TreeConfig, +}; +use alloy_consensus::transaction::Either; +use alloy_eips::{eip1898::BlockWithParent, NumHash}; +use alloy_evm::Evm; +use alloy_primitives::B256; +use reth_chain_state::{CanonicalInMemoryState, ExecutedBlock}; +use reth_consensus::{ConsensusError, FullConsensus}; +use reth_engine_primitives::{ + ConfigureEngineEvm, ExecutableTxIterator, ExecutionPayload, InvalidBlockHook, PayloadValidator, +}; +use reth_errors::{BlockExecutionError, ProviderResult}; +use reth_evm::{ + block::BlockExecutor, execute::ExecutableTxFor, ConfigureEvm, EvmEnvFor, ExecutionCtxFor, + SpecFor, +}; +use reth_payload_primitives::{ + BuiltPayload, InvalidPayloadAttributesError, NewPayloadError, PayloadTypes, +}; +use reth_primitives_traits::{ + AlloyBlockHeader, BlockTy, GotExpected, NodePrimitives, RecoveredBlock, SealedHeader, +}; +use reth_provider::{ + providers::OverlayStateProviderFactory, BlockExecutionOutput, BlockReader, + DatabaseProviderFactory, ExecutionOutcome, HashedPostStateProvider, ProviderError, + PruneCheckpointReader, StageCheckpointReader, StateProvider, StateProviderFactory, StateReader, + StateRootProvider, TrieReader, +}; +use reth_revm::db::State; +use reth_trie::{updates::TrieUpdates, HashedPostState, TrieInput}; +use reth_trie_parallel::root::{ParallelStateRoot, ParallelStateRootError}; +use std::{collections::HashMap, sync::Arc, time::Instant}; +use tracing::{debug, debug_span, error, info, instrument, trace, warn}; + +/// Context providing access to tree state during validation. +/// +/// This context is provided to the [`EngineValidator`] and includes the state of the tree's +/// internals +pub struct TreeCtx<'a, N: NodePrimitives> { + /// The engine API tree state + state: &'a mut EngineApiTreeState, + /// Reference to the canonical in-memory state + canonical_in_memory_state: &'a CanonicalInMemoryState, +} + +impl<'a, N: NodePrimitives> std::fmt::Debug for TreeCtx<'a, N> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TreeCtx") + .field("state", &"EngineApiTreeState") + .field("canonical_in_memory_state", &self.canonical_in_memory_state) + .finish() + } +} + +impl<'a, N: NodePrimitives> TreeCtx<'a, N> { + /// Creates a new tree context + pub const fn new( + state: &'a mut EngineApiTreeState, + canonical_in_memory_state: &'a CanonicalInMemoryState, + ) -> Self { + Self { state, canonical_in_memory_state } + } + + /// Returns a reference to the engine tree state + pub const fn state(&self) -> &EngineApiTreeState { + &*self.state + } + + /// Returns a mutable reference to the engine tree state + pub const fn state_mut(&mut self) -> &mut EngineApiTreeState { + self.state + } + + /// Returns a reference to the canonical in-memory state + pub const fn canonical_in_memory_state(&self) -> &'a CanonicalInMemoryState { + self.canonical_in_memory_state + } +} + +/// A helper type that provides reusable payload validation logic for network-specific validators. +/// +/// This type satisfies [`EngineValidator`] and is responsible for executing blocks/payloads. +/// +/// This type contains common validation, execution, and state root computation logic that can be +/// used by network-specific payload validators (e.g., Ethereum, Optimism). It is not meant to be +/// used as a standalone component, but rather as a building block for concrete implementations. +#[derive(derive_more::Debug)] +pub struct BasicEngineValidator +where + Evm: ConfigureEvm, +{ + /// Provider for database access. + provider: P, + /// Consensus implementation for validation. + consensus: Arc>, + /// EVM configuration. + evm_config: Evm, + /// Configuration for the tree. + config: TreeConfig, + /// Payload processor for state root computation. + payload_processor: PayloadProcessor, + /// Precompile cache map. + precompile_cache_map: PrecompileCacheMap>, + /// Precompile cache metrics. + precompile_cache_metrics: HashMap, + /// Hook to call when invalid blocks are encountered. + #[debug(skip)] + invalid_block_hook: Box>, + /// Metrics for the engine api. + metrics: EngineApiMetrics, + /// Validator for the payload. + validator: V, + /// A cleared trie input, kept around to be reused so allocations can be minimized. + trie_input: Option, +} + +impl BasicEngineValidator +where + N: NodePrimitives, + P: DatabaseProviderFactory< + Provider: BlockReader + TrieReader + StageCheckpointReader + PruneCheckpointReader, + > + BlockReader
+ + StateProviderFactory + + StateReader + + HashedPostStateProvider + + Clone + + 'static, + Evm: ConfigureEvm + 'static, +{ + /// Creates a new `TreePayloadValidator`. + #[allow(clippy::too_many_arguments)] + pub fn new( + provider: P, + consensus: Arc>, + evm_config: Evm, + validator: V, + config: TreeConfig, + invalid_block_hook: Box>, + ) -> Self { + let precompile_cache_map = PrecompileCacheMap::default(); + let payload_processor = PayloadProcessor::new( + WorkloadExecutor::default(), + evm_config.clone(), + &config, + precompile_cache_map.clone(), + ); + Self { + provider, + consensus, + evm_config, + payload_processor, + precompile_cache_map, + precompile_cache_metrics: HashMap::new(), + config, + invalid_block_hook, + metrics: EngineApiMetrics::default(), + validator, + trie_input: Default::default(), + } + } + + /// Converts a [`BlockOrPayload`] to a recovered block. + pub fn convert_to_block>>( + &self, + input: BlockOrPayload, + ) -> Result, NewPayloadError> + where + V: PayloadValidator, + { + match input { + BlockOrPayload::Payload(payload) => self.validator.ensure_well_formed_payload(payload), + BlockOrPayload::Block(block) => Ok(block), + } + } + + /// Returns EVM environment for the given payload or block. + pub fn evm_env_for>>( + &self, + input: &BlockOrPayload, + ) -> Result, Evm::Error> + where + V: PayloadValidator, + Evm: ConfigureEngineEvm, + { + match input { + BlockOrPayload::Payload(payload) => Ok(self.evm_config.evm_env_for_payload(payload)?), + BlockOrPayload::Block(block) => Ok(self.evm_config.evm_env(block.header())?), + } + } + + /// Returns [`ExecutableTxIterator`] for the given payload or block. + pub fn tx_iterator_for<'a, T: PayloadTypes>>( + &'a self, + input: &'a BlockOrPayload, + ) -> Result + 'a, NewPayloadError> + where + V: PayloadValidator, + Evm: ConfigureEngineEvm, + { + match input { + BlockOrPayload::Payload(payload) => Ok(Either::Left( + self.evm_config + .tx_iterator_for_payload(payload) + .map_err(NewPayloadError::other)? + .map(|res| res.map(Either::Left)), + )), + BlockOrPayload::Block(block) => { + let transactions = block.clone_transactions_recovered().collect::>(); + Ok(Either::Right(transactions.into_iter().map(|tx| Ok(Either::Right(tx))))) + } + } + } + + /// Returns a [`ExecutionCtxFor`] for the given payload or block. + pub fn execution_ctx_for<'a, T: PayloadTypes>>( + &self, + input: &'a BlockOrPayload, + ) -> Result, Evm::Error> + where + V: PayloadValidator, + Evm: ConfigureEngineEvm, + { + match input { + BlockOrPayload::Payload(payload) => Ok(self.evm_config.context_for_payload(payload)?), + BlockOrPayload::Block(block) => Ok(self.evm_config.context_for_block(block)?), + } + } + + /// Handles execution errors by checking if header validation errors should take precedence. + /// + /// When an execution error occurs, this function checks if there are any header validation + /// errors that should be reported instead, as header validation errors have higher priority. + fn handle_execution_error>>( + &self, + input: BlockOrPayload, + execution_err: InsertBlockErrorKind, + parent_block: &SealedHeader, + ) -> Result, InsertPayloadError> + where + V: PayloadValidator, + { + debug!( + target: "engine::tree::payload_validator", + ?execution_err, + block = ?input.num_hash(), + "Block execution failed, checking for header validation errors" + ); + + // If execution failed, we should first check if there are any header validation + // errors that take precedence over the execution error + let block = self.convert_to_block(input)?; + + // Validate block consensus rules which includes header validation + if let Err(consensus_err) = self.validate_block_inner(&block) { + // Header validation error takes precedence over execution error + return Err(InsertBlockError::new(block.into_sealed_block(), consensus_err.into()).into()) + } + + // Also validate against the parent + if let Err(consensus_err) = + self.consensus.validate_header_against_parent(block.sealed_header(), parent_block) + { + // Parent validation error takes precedence over execution error + return Err(InsertBlockError::new(block.into_sealed_block(), consensus_err.into()).into()) + } + + // No header validation errors, return the original execution error + Err(InsertBlockError::new(block.into_sealed_block(), execution_err).into()) + } + + /// Validates a block that has already been converted from a payload. + /// + /// This method performs: + /// - Consensus validation + /// - Block execution + /// - State root computation + /// - Fork detection + #[instrument( + level = "debug", + target = "engine::tree::payload_validator", + skip_all, + fields( + parent = ?input.parent_hash(), + type_name = ?input.type_name(), + ) + )] + pub fn validate_block_with_state>>( + &mut self, + input: BlockOrPayload, + mut ctx: TreeCtx<'_, N>, + ) -> ValidationOutcome> + where + V: PayloadValidator, + Evm: ConfigureEngineEvm, + { + /// A helper macro that returns the block in case there was an error + /// This macro is used for early returns before block conversion + macro_rules! ensure_ok { + ($expr:expr) => { + match $expr { + Ok(val) => val, + Err(e) => { + let block = self.convert_to_block(input)?; + return Err( + InsertBlockError::new(block.into_sealed_block(), e.into()).into() + ) + } + } + }; + } + + /// A helper macro for handling errors after the input has been converted to a block + macro_rules! ensure_ok_post_block { + ($expr:expr, $block:expr) => { + match $expr { + Ok(val) => val, + Err(e) => { + return Err( + InsertBlockError::new($block.into_sealed_block(), e.into()).into() + ) + } + } + }; + } + + let parent_hash = input.parent_hash(); + let block_num_hash = input.num_hash(); + + trace!(target: "engine::tree::payload_validator", "Fetching block state provider"); + let _enter = + debug_span!(target: "engine::tree::payload_validator", "state provider").entered(); + let Some(provider_builder) = + ensure_ok!(self.state_provider_builder(parent_hash, ctx.state())) + else { + // this is pre-validated in the tree + return Err(InsertBlockError::new( + self.convert_to_block(input)?.into_sealed_block(), + ProviderError::HeaderNotFound(parent_hash.into()).into(), + ) + .into()) + }; + let state_provider = ensure_ok!(provider_builder.build()); + drop(_enter); + + // fetch parent block + let Some(parent_block) = ensure_ok!(self.sealed_header_by_hash(parent_hash, ctx.state())) + else { + return Err(InsertBlockError::new( + self.convert_to_block(input)?.into_sealed_block(), + ProviderError::HeaderNotFound(parent_hash.into()).into(), + ) + .into()) + }; + + let evm_env = debug_span!(target: "engine::tree::payload_validator", "evm env") + .in_scope(|| self.evm_env_for(&input)) + .map_err(NewPayloadError::other)?; + + let env = ExecutionEnv { evm_env, hash: input.hash(), parent_hash: input.parent_hash() }; + + // Plan the strategy used for state root computation. + let strategy = self.plan_state_root_computation(); + + debug!( + target: "engine::tree::payload_validator", + ?strategy, + "Deciding which state root algorithm to run" + ); + + // use prewarming background task + let txs = self.tx_iterator_for(&input)?; + + // Spawn the appropriate processor based on strategy + let mut handle = ensure_ok!(self.spawn_payload_processor( + env.clone(), + txs, + provider_builder, + parent_hash, + ctx.state(), + strategy, + )); + + // Use cached state provider before executing, used in execution after prewarming threads + // complete + let state_provider = CachedStateProvider::new_with_caches( + state_provider, + handle.caches(), + handle.cache_metrics(), + ); + + // Execute the block and handle any execution errors + let output = match if self.config.state_provider_metrics() { + let state_provider = InstrumentedStateProvider::from_state_provider(&state_provider); + let result = self.execute_block(&state_provider, env, &input, &mut handle); + state_provider.record_total_latency(); + result + } else { + self.execute_block(&state_provider, env, &input, &mut handle) + } { + Ok(output) => output, + Err(err) => return self.handle_execution_error(input, err, &parent_block), + }; + + // after executing the block we can stop executing transactions + handle.stop_prewarming_execution(); + + let block = self.convert_to_block(input)?; + + let hashed_state = ensure_ok_post_block!( + self.validate_post_execution(&block, &parent_block, &output, &mut ctx), + block + ); + + debug!(target: "engine::tree::payload_validator", "Calculating block state root"); + + let root_time = Instant::now(); + + let mut maybe_state_root = None; + + match strategy { + StateRootStrategy::StateRootTask => { + debug!(target: "engine::tree::payload_validator", "Using sparse trie state root algorithm"); + match handle.state_root() { + Ok(StateRootComputeOutcome { state_root, trie_updates }) => { + let elapsed = root_time.elapsed(); + info!(target: "engine::tree::payload_validator", ?state_root, ?elapsed, "State root task finished"); + // we double check the state root here for good measure + if state_root == block.header().state_root() { + maybe_state_root = Some((state_root, trie_updates, elapsed)) + } else { + warn!( + target: "engine::tree::payload_validator", + ?state_root, + block_state_root = ?block.header().state_root(), + "State root task returned incorrect state root" + ); + } + } + Err(error) => { + debug!(target: "engine::tree::payload_validator", %error, "State root task failed"); + } + } + } + StateRootStrategy::Parallel => { + debug!(target: "engine::tree::payload_validator", "Using parallel state root algorithm"); + match self.compute_state_root_parallel( + block.parent_hash(), + &hashed_state, + ctx.state(), + ) { + Ok(result) => { + let elapsed = root_time.elapsed(); + info!( + target: "engine::tree::payload_validator", + regular_state_root = ?result.0, + ?elapsed, + "Regular root task finished" + ); + maybe_state_root = Some((result.0, result.1, elapsed)); + } + Err(error) => { + debug!(target: "engine::tree::payload_validator", %error, "Parallel state root computation failed"); + } + } + } + StateRootStrategy::Synchronous => {} + } + + // Determine the state root. + // If the state root was computed in parallel, we use it. + // Otherwise, we fall back to computing it synchronously. + let (state_root, trie_output, root_elapsed) = if let Some(maybe_state_root) = + maybe_state_root + { + maybe_state_root + } else { + // fallback is to compute the state root regularly in sync + if self.config.state_root_fallback() { + debug!(target: "engine::tree::payload_validator", "Using state root fallback for testing"); + } else { + warn!(target: "engine::tree::payload_validator", "Failed to compute state root in parallel"); + self.metrics.block_validation.state_root_parallel_fallback_total.increment(1); + } + + let (root, updates) = ensure_ok_post_block!( + state_provider.state_root_with_updates(hashed_state.clone()), + block + ); + (root, updates, root_time.elapsed()) + }; + + self.metrics.block_validation.record_state_root(&trie_output, root_elapsed.as_secs_f64()); + debug!(target: "engine::tree::payload_validator", ?root_elapsed, "Calculated state root"); + + // ensure state root matches + if state_root != block.header().state_root() { + // ===== Mantle Hardfork: Export full state on mismatch for debugging ===== + warn!(target: "engine::tree::payload_validator", + "State root mismatch detected, exporting full state for debugging"); + + if let Err(e) = self.export_state_on_mismatch( + &block, + state_root, + block.header().state_root(), + &hashed_state, + &output, + ) { + warn!(target: "engine::tree::payload_validator", + error = ?e, "Failed to export state on mismatch"); + } + // ===== Mantle Hardfork: End of addition ===== + + // call post-block hook + self.on_invalid_block( + &parent_block, + &block, + &output, + Some((&trie_output, state_root)), + ctx.state_mut(), + ); + let block_state_root = block.header().state_root(); + return Err(InsertBlockError::new( + block.into_sealed_block(), + ConsensusError::BodyStateRootDiff( + GotExpected { got: state_root, expected: block_state_root }.into(), + ) + .into(), + ) + .into()) + } + + // terminate prewarming task with good state output + handle.terminate_caching(Some(&output.state)); + + Ok(ExecutedBlock { + recovered_block: Arc::new(block), + execution_output: Arc::new(ExecutionOutcome::from((output, block_num_hash.number))), + hashed_state: Arc::new(hashed_state), + trie_updates: Arc::new(trie_output), + }) + } + + /// Return sealed block header from database or in-memory state by hash. + fn sealed_header_by_hash( + &self, + hash: B256, + state: &EngineApiTreeState, + ) -> ProviderResult>> { + // check memory first + let header = state.tree_state.sealed_header_by_hash(&hash); + + if header.is_some() { + Ok(header) + } else { + self.provider.sealed_header_by_hash(hash) + } + } + + /// Validate if block is correct and satisfies all the consensus rules that concern the header + /// and block body itself. + fn validate_block_inner(&self, block: &RecoveredBlock) -> Result<(), ConsensusError> { + if let Err(e) = self.consensus.validate_header(block.sealed_header()) { + error!(target: "engine::tree::payload_validator", ?block, "Failed to validate header {}: {e}", block.hash()); + return Err(e) + } + + if let Err(e) = self.consensus.validate_block_pre_execution(block.sealed_block()) { + error!(target: "engine::tree::payload_validator", ?block, "Failed to validate block {}: {e}", block.hash()); + return Err(e) + } + + Ok(()) + } + + /// Executes a block with the given state provider + #[instrument(level = "debug", target = "engine::tree::payload_validator", skip_all)] + fn execute_block( + &mut self, + state_provider: S, + env: ExecutionEnv, + input: &BlockOrPayload, + handle: &mut PayloadHandle, Err>, + ) -> Result, InsertBlockErrorKind> + where + S: StateProvider, + Err: core::error::Error + Send + Sync + 'static, + V: PayloadValidator, + T: PayloadTypes>, + Evm: ConfigureEngineEvm, + { + debug!(target: "engine::tree::payload_validator", "Executing block"); + + let mut db = State::builder() + .with_database(StateProviderDatabase::new(&state_provider)) + .with_bundle_update() + .without_state_clear() + .build(); + + let evm = self.evm_config.evm_with_env(&mut db, env.evm_env.clone()); + let ctx = + self.execution_ctx_for(input).map_err(|e| InsertBlockErrorKind::Other(Box::new(e)))?; + let mut executor = self.evm_config.create_executor(evm, ctx); + + if !self.config.precompile_cache_disabled() { + // Only cache pure precompiles to avoid issues with stateful precompiles + executor.evm_mut().precompiles_mut().map_pure_precompiles(|address, precompile| { + let metrics = self + .precompile_cache_metrics + .entry(*address) + .or_insert_with(|| CachedPrecompileMetrics::new_with_address(*address)) + .clone(); + CachedPrecompile::wrap( + precompile, + self.precompile_cache_map.cache_for_address(*address), + *env.evm_env.spec_id(), + Some(metrics), + ) + }); + } + + let execution_start = Instant::now(); + let state_hook = Box::new(handle.state_hook()); + let output = self.metrics.execute_metered( + executor, + handle.iter_transactions().map(|res| res.map_err(BlockExecutionError::other)), + state_hook, + )?; + let execution_finish = Instant::now(); + let execution_time = execution_finish.duration_since(execution_start); + debug!(target: "engine::tree::payload_validator", elapsed = ?execution_time, "Executed block"); + Ok(output) + } + + /// Compute state root for the given hashed post state in parallel. + /// + /// # Returns + /// + /// Returns `Ok(_)` if computed successfully. + /// Returns `Err(_)` if error was encountered during computation. + /// `Err(ProviderError::ConsistentView(_))` can be safely ignored and fallback computation + /// should be used instead. + #[instrument(level = "debug", target = "engine::tree::payload_validator", skip_all)] + fn compute_state_root_parallel( + &self, + parent_hash: B256, + hashed_state: &HashedPostState, + state: &EngineApiTreeState, + ) -> Result<(B256, TrieUpdates), ParallelStateRootError> { + let (mut input, block_hash) = self.compute_trie_input(parent_hash, state, None)?; + + // Extend with block we are validating root for. + input.append_ref(hashed_state); + + // Convert the TrieInput into a MultProofConfig, since everything uses the sorted + // forms of the state/trie fields. + let (_, multiproof_config) = MultiProofConfig::from_input(input); + + let factory = OverlayStateProviderFactory::new(self.provider.clone()) + .with_block_hash(Some(block_hash)) + .with_trie_overlay(Some(multiproof_config.nodes_sorted)) + .with_hashed_state_overlay(Some(multiproof_config.state_sorted)); + + // The `hashed_state` argument is already taken into account as part of the overlay, but we + // need to use the prefix sets which were generated from it to indicate to the + // ParallelStateRoot which parts of the trie need to be recomputed. + let prefix_sets = Arc::into_inner(multiproof_config.prefix_sets) + .expect("MultiProofConfig was never cloned") + .freeze(); + + ParallelStateRoot::new(factory, prefix_sets).incremental_root_with_updates() + } + + /// Validates the block after execution. + /// + /// This performs: + /// - parent header validation + /// - post-execution consensus validation + /// - state-root based post-execution validation + fn validate_post_execution>>( + &self, + block: &RecoveredBlock, + parent_block: &SealedHeader, + output: &BlockExecutionOutput, + ctx: &mut TreeCtx<'_, N>, + ) -> Result + where + V: PayloadValidator, + { + let start = Instant::now(); + + trace!(target: "engine::tree::payload_validator", block=?block.num_hash(), "Validating block consensus"); + // validate block consensus rules + if let Err(e) = self.validate_block_inner(block) { + return Err(e.into()) + } + + // now validate against the parent + if let Err(e) = + self.consensus.validate_header_against_parent(block.sealed_header(), parent_block) + { + warn!(target: "engine::tree::payload_validator", ?block, "Failed to validate header {} against parent: {e}", block.hash()); + return Err(e.into()) + } + + if let Err(err) = self.consensus.validate_block_post_execution(block, output) { + // call post-block hook + self.on_invalid_block(parent_block, block, output, None, ctx.state_mut()); + return Err(err.into()) + } + + let hashed_state = self.provider.hashed_post_state(&output.state); + + if let Err(err) = + self.validator.validate_block_post_execution_with_hashed_state(&hashed_state, block) + { + // call post-block hook + self.on_invalid_block(parent_block, block, output, None, ctx.state_mut()); + return Err(err.into()) + } + + // record post-execution validation duration + self.metrics + .block_validation + .post_execution_validation_duration + .record(start.elapsed().as_secs_f64()); + + Ok(hashed_state) + } + + /// Spawns a payload processor task based on the state root strategy. + /// + /// This method determines how to execute the block and compute its state root based on + /// the selected strategy: + /// - `StateRootTask`: Uses a dedicated task for state root computation with proof generation + /// - `Parallel`: Computes state root in parallel with block execution + /// - `Synchronous`: Falls back to sequential execution and state root computation + /// + /// The method handles strategy fallbacks if the preferred approach fails, ensuring + /// block execution always completes with a valid state root. + #[allow(clippy::too_many_arguments)] + #[instrument( + level = "debug", + target = "engine::tree::payload_validator", + skip_all, + fields(strategy) + )] + fn spawn_payload_processor>( + &mut self, + env: ExecutionEnv, + txs: T, + provider_builder: StateProviderBuilder, + parent_hash: B256, + state: &EngineApiTreeState, + strategy: StateRootStrategy, + ) -> Result< + PayloadHandle< + impl ExecutableTxFor + use, + impl core::error::Error + Send + Sync + 'static + use, + >, + InsertBlockErrorKind, + > { + match strategy { + StateRootStrategy::StateRootTask => { + // get allocated trie input if it exists + let allocated_trie_input = self.trie_input.take(); + + // Compute trie input + let trie_input_start = Instant::now(); + let (trie_input, block_hash) = + self.compute_trie_input(parent_hash, state, allocated_trie_input)?; + + self.metrics + .block_validation + .trie_input_duration + .record(trie_input_start.elapsed().as_secs_f64()); + + // Convert the TrieInput into a MultProofConfig, since everything uses the sorted + // forms of the state/trie fields. + let (trie_input, multiproof_config) = MultiProofConfig::from_input(trie_input); + self.trie_input.replace(trie_input); + + // Create OverlayStateProviderFactory with the multiproof config, for use with + // multiproofs. + let multiproof_provider_factory = + OverlayStateProviderFactory::new(self.provider.clone()) + .with_block_hash(Some(block_hash)) + .with_trie_overlay(Some(multiproof_config.nodes_sorted)) + .with_hashed_state_overlay(Some(multiproof_config.state_sorted)); + + // Use state root task only if prefix sets are empty, otherwise proof generation is + // too expensive because it requires walking all paths in every proof. + let spawn_start = Instant::now(); + let handle = self.payload_processor.spawn( + env, + txs, + provider_builder, + multiproof_provider_factory, + &self.config, + ); + + // record prewarming initialization duration + self.metrics + .block_validation + .spawn_payload_processor + .record(spawn_start.elapsed().as_secs_f64()); + + Ok(handle) + } + StateRootStrategy::Parallel | StateRootStrategy::Synchronous => { + let start = Instant::now(); + let handle = + self.payload_processor.spawn_cache_exclusive(env, txs, provider_builder); + + // Record prewarming initialization duration + self.metrics + .block_validation + .spawn_payload_processor + .record(start.elapsed().as_secs_f64()); + + Ok(handle) + } + } + } + + /// Creates a `StateProviderBuilder` for the given parent hash. + /// + /// This method checks if the parent is in the tree state (in-memory) or persisted to disk, + /// and creates the appropriate provider builder. + fn state_provider_builder( + &self, + hash: B256, + state: &EngineApiTreeState, + ) -> ProviderResult>> { + if let Some((historical, blocks)) = state.tree_state.blocks_by_hash(hash) { + debug!(target: "engine::tree::payload_validator", %hash, %historical, "found canonical state for block in memory, creating provider builder"); + // the block leads back to the canonical chain + return Ok(Some(StateProviderBuilder::new( + self.provider.clone(), + historical, + Some(blocks), + ))) + } + + // Check if the block is persisted + if let Some(header) = self.provider.header(hash)? { + debug!(target: "engine::tree::payload_validator", %hash, number = %header.number(), "found canonical state for block in database, creating provider builder"); + // For persisted blocks, we create a builder that will fetch state directly from the + // database + return Ok(Some(StateProviderBuilder::new(self.provider.clone(), hash, None))) + } + + debug!(target: "engine::tree::payload_validator", %hash, "no canonical state found for block"); + Ok(None) + } + + /// Determines the state root computation strategy based on configuration. + #[instrument(level = "debug", target = "engine::tree::payload_validator", skip_all)] + fn plan_state_root_computation(&self) -> StateRootStrategy { + let strategy = if self.config.state_root_fallback() { + StateRootStrategy::Synchronous + } else if self.config.use_state_root_task() { + StateRootStrategy::StateRootTask + } else { + StateRootStrategy::Parallel + }; + + debug!( + target: "engine::tree::payload_validator", + ?strategy, + "Planned state root computation strategy" + ); + + strategy + } + + /// Export full state (parent state + current changes) on state root mismatch for debugging + fn export_state_on_mismatch( + &self, + block: &RecoveredBlock, + computed_root: B256, + expected_root: B256, + _hashed_state: &HashedPostState, + output: &BlockExecutionOutput, + ) -> eyre::Result<()> { + use reth_mantle_forks::debug::state_export::export_full_state_with_bundle; + + let block_number = block.number(); + let filename = format!("full_state_{}.json", block_number); + + info!(target: "engine::tree::payload_validator", + block_number = block_number, + computed_root = ?computed_root, + expected_root = ?expected_root, + filename = %filename, + "Exporting state (parent + changes) with original keys from bundle_state, only bundle account storage exported"); + + // Get database provider + let provider = self.provider.database_provider_ro()?; + + // Call export function (using bundle_state), pass computed state root + // Only export storage details for accounts in bundle_state to avoid OOM + export_full_state_with_bundle(&provider, &output.state, &filename, Some(computed_root), true)?; + + info!(target: "engine::tree::payload_validator", + filename = %filename, + "State exported successfully with original storage keys from bundle_state"); + + Ok(()) + } + + /// Called when an invalid block is encountered during validation. + fn on_invalid_block( + &self, + parent_header: &SealedHeader, + block: &RecoveredBlock, + output: &BlockExecutionOutput, + trie_updates: Option<(&TrieUpdates, B256)>, + state: &mut EngineApiTreeState, + ) { + if state.invalid_headers.get(&block.hash()).is_some() { + // we already marked this block as invalid + return + } + self.invalid_block_hook.on_invalid_block(parent_header, block, output, trie_updates); + } + + /// Computes the trie input at the provided parent hash, as well as the block number of the + /// highest persisted ancestor. + /// + /// The goal of this function is to take in-memory blocks and generate a [`TrieInput`] that + /// serves as an overlay to the database blocks. + /// + /// It works as follows: + /// 1. Collect in-memory blocks that are descendants of the provided parent hash using + /// [`crate::tree::TreeState::blocks_by_hash`]. This returns the highest persisted ancestor + /// hash (`block_hash`) and the list of in-memory descendant blocks. + /// 2. Extend the `TrieInput` with the contents of these in-memory blocks (from oldest to + /// newest) to build the overlay state and trie updates that sit on top of the database view + /// anchored at `block_hash`. + #[instrument( + level = "debug", + target = "engine::tree::payload_validator", + skip_all, + fields(parent_hash) + )] + fn compute_trie_input( + &self, + parent_hash: B256, + state: &EngineApiTreeState, + allocated_trie_input: Option, + ) -> ProviderResult<(TrieInput, B256)> { + // get allocated trie input or use a default trie input + let mut input = allocated_trie_input.unwrap_or_default(); + + let (block_hash, blocks) = + state.tree_state.blocks_by_hash(parent_hash).unwrap_or_else(|| (parent_hash, vec![])); + + if blocks.is_empty() { + debug!(target: "engine::tree::payload_validator", "Parent found on disk"); + } else { + debug!(target: "engine::tree::payload_validator", historical = ?block_hash, blocks = blocks.len(), "Parent found in memory"); + } + + // Extend with contents of parent in-memory blocks. + input.extend_with_blocks( + blocks.iter().rev().map(|block| (block.hashed_state(), block.trie_updates())), + ); + + Ok((input, block_hash)) + } +} + +/// Output of block or payload validation. +pub type ValidationOutcome>> = Result, E>; + +/// Strategy describing how to compute the state root. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum StateRootStrategy { + /// Use the state root task (background sparse trie computation). + StateRootTask, + /// Run the parallel state root computation on the calling thread. + Parallel, + /// Fall back to synchronous computation via the state provider. + Synchronous, +} + +/// Type that validates the payloads processed by the engine. +/// +/// This provides the necessary functions for validating/executing payloads/blocks. +pub trait EngineValidator< + Types: PayloadTypes, + N: NodePrimitives = <::BuiltPayload as BuiltPayload>::Primitives, +>: Send + Sync + 'static +{ + /// Validates the payload attributes with respect to the header. + /// + /// By default, this enforces that the payload attributes timestamp is greater than the + /// timestamp according to: + /// > 7. Client software MUST ensure that payloadAttributes.timestamp is greater than + /// > timestamp + /// > of a block referenced by forkchoiceState.headBlockHash. + /// + /// See also: + fn validate_payload_attributes_against_header( + &self, + attr: &Types::PayloadAttributes, + header: &N::BlockHeader, + ) -> Result<(), InvalidPayloadAttributesError>; + + /// Ensures that the given payload does not violate any consensus rules that concern the block's + /// layout. + /// + /// This function must convert the payload into the executable block and pre-validate its + /// fields. + /// + /// Implementers should ensure that the checks are done in the order that conforms with the + /// engine-API specification. + fn ensure_well_formed_payload( + &self, + payload: Types::ExecutionData, + ) -> Result, NewPayloadError>; + + /// Validates a payload received from engine API. + fn validate_payload( + &mut self, + payload: Types::ExecutionData, + ctx: TreeCtx<'_, N>, + ) -> ValidationOutcome; + + /// Validates a block downloaded from the network. + fn validate_block( + &mut self, + block: RecoveredBlock, + ctx: TreeCtx<'_, N>, + ) -> ValidationOutcome; +} + +impl EngineValidator for BasicEngineValidator +where + P: DatabaseProviderFactory< + Provider: BlockReader + TrieReader + StageCheckpointReader + PruneCheckpointReader, + > + BlockReader
+ + StateProviderFactory + + StateReader + + HashedPostStateProvider + + Clone + + 'static, + N: NodePrimitives, + V: PayloadValidator, + Evm: ConfigureEngineEvm + 'static, + Types: PayloadTypes>, +{ + fn validate_payload_attributes_against_header( + &self, + attr: &Types::PayloadAttributes, + header: &N::BlockHeader, + ) -> Result<(), InvalidPayloadAttributesError> { + self.validator.validate_payload_attributes_against_header(attr, header) + } + + fn ensure_well_formed_payload( + &self, + payload: Types::ExecutionData, + ) -> Result, NewPayloadError> { + let block = self.validator.ensure_well_formed_payload(payload)?; + Ok(block) + } + + fn validate_payload( + &mut self, + payload: Types::ExecutionData, + ctx: TreeCtx<'_, N>, + ) -> ValidationOutcome { + self.validate_block_with_state(BlockOrPayload::Payload(payload), ctx) + } + + fn validate_block( + &mut self, + block: RecoveredBlock, + ctx: TreeCtx<'_, N>, + ) -> ValidationOutcome { + self.validate_block_with_state(BlockOrPayload::Block(block), ctx) + } +} + +/// Enum representing either block or payload being validated. +#[derive(Debug)] +pub enum BlockOrPayload { + /// Payload. + Payload(T::ExecutionData), + /// Block. + Block(RecoveredBlock::Primitives>>), +} + +impl BlockOrPayload { + /// Returns the hash of the block. + pub fn hash(&self) -> B256 { + match self { + Self::Payload(payload) => payload.block_hash(), + Self::Block(block) => block.hash(), + } + } + + /// Returns the number and hash of the block. + pub fn num_hash(&self) -> NumHash { + match self { + Self::Payload(payload) => payload.num_hash(), + Self::Block(block) => block.num_hash(), + } + } + + /// Returns the parent hash of the block. + pub fn parent_hash(&self) -> B256 { + match self { + Self::Payload(payload) => payload.parent_hash(), + Self::Block(block) => block.parent_hash(), + } + } + + /// Returns [`BlockWithParent`] for the block. + pub fn block_with_parent(&self) -> BlockWithParent { + match self { + Self::Payload(payload) => payload.block_with_parent(), + Self::Block(block) => block.block_with_parent(), + } + } + + /// Returns a string showing whether or not this is a block or payload. + pub const fn type_name(&self) -> &'static str { + match self { + Self::Payload(_) => "payload", + Self::Block(_) => "block", + } + } +} diff --git a/crates/engine/tree/src/tree/persistence_state.rs b/crates/engine/tree/src/tree/persistence_state.rs index e7b4dc0ad19..82a8078447d 100644 --- a/crates/engine/tree/src/tree/persistence_state.rs +++ b/crates/engine/tree/src/tree/persistence_state.rs @@ -1,3 +1,25 @@ +//! Persistence state management for background database operations. +//! +//! This module manages the state of background tasks that persist cached data +//! to the database. The persistence system works asynchronously to avoid blocking +//! block execution while ensuring data durability. +//! +//! ## Background Persistence +//! +//! The execution engine maintains an in-memory cache of state changes that need +//! to be persisted to disk. Rather than writing synchronously (which would slow +//! down block processing), persistence happens in background tasks. +//! +//! ## Persistence Actions +//! +//! - **Saving Blocks**: Persist newly executed blocks and their state changes +//! - **Removing Blocks**: Remove invalid blocks during chain reorganizations +//! +//! ## Coordination +//! +//! The [`PersistenceState`] tracks ongoing persistence operations and coordinates +//! between the main execution thread and background persistence workers. + use alloy_eips::BlockNumHash; use alloy_primitives::B256; use std::time::Instant; @@ -45,6 +67,7 @@ impl PersistenceState { /// Returns the current persistence action. If there is no persistence task in progress, then /// this returns `None`. + #[cfg(test)] pub(crate) fn current_action(&self) -> Option<&CurrentPersistenceAction> { self.rx.as_ref().map(|rx| &rx.2) } diff --git a/crates/engine/tree/src/tree/precompile_cache.rs b/crates/engine/tree/src/tree/precompile_cache.rs new file mode 100644 index 00000000000..1183dfbe983 --- /dev/null +++ b/crates/engine/tree/src/tree/precompile_cache.rs @@ -0,0 +1,404 @@ +//! Contains a precompile cache backed by `schnellru::LruMap` (LRU by length). + +use alloy_primitives::Bytes; +use parking_lot::Mutex; +use reth_evm::precompiles::{DynPrecompile, Precompile, PrecompileInput}; +use revm::precompile::{PrecompileId, PrecompileOutput, PrecompileResult}; +use revm_primitives::Address; +use schnellru::LruMap; +use std::{ + collections::HashMap, + hash::{Hash, Hasher}, + sync::Arc, +}; + +/// Default max cache size for [`PrecompileCache`] +const MAX_CACHE_SIZE: u32 = 10_000; + +/// Stores caches for each precompile. +#[derive(Debug, Clone, Default)] +pub struct PrecompileCacheMap(HashMap>) +where + S: Eq + Hash + std::fmt::Debug + Send + Sync + Clone; + +impl PrecompileCacheMap +where + S: Eq + Hash + std::fmt::Debug + Send + Sync + Clone + 'static, +{ + pub(crate) fn cache_for_address(&mut self, address: Address) -> PrecompileCache { + self.0.entry(address).or_default().clone() + } +} + +/// Cache for precompiles, for each input stores the result. +/// +/// [`LruMap`] requires a mutable reference on `get` since it updates the LRU order, +/// so we use a [`Mutex`] instead of an `RwLock`. +#[derive(Debug, Clone)] +pub struct PrecompileCache(Arc, CacheEntry>>>) +where + S: Eq + Hash + std::fmt::Debug + Send + Sync + Clone; + +impl Default for PrecompileCache +where + S: Eq + Hash + std::fmt::Debug + Send + Sync + Clone + 'static, +{ + fn default() -> Self { + Self(Arc::new(Mutex::new(LruMap::new(schnellru::ByLength::new(MAX_CACHE_SIZE))))) + } +} + +impl PrecompileCache +where + S: Eq + Hash + std::fmt::Debug + Send + Sync + Clone + 'static, +{ + fn get(&self, key: &CacheKeyRef<'_, S>) -> Option { + self.0.lock().get(key).cloned() + } + + /// Inserts the given key and value into the cache, returning the new cache size. + fn insert(&self, key: CacheKey, value: CacheEntry) -> usize { + let mut cache = self.0.lock(); + cache.insert(key, value); + cache.len() + } +} + +/// Cache key, spec id and precompile call input. spec id is included in the key to account for +/// precompile repricing across fork activations. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct CacheKey((S, Bytes)); + +impl CacheKey { + const fn new(spec_id: S, input: Bytes) -> Self { + Self((spec_id, input)) + } +} + +/// Cache key reference, used to avoid cloning the input bytes when looking up using a [`CacheKey`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CacheKeyRef<'a, S>((S, &'a [u8])); + +impl<'a, S> CacheKeyRef<'a, S> { + const fn new(spec_id: S, input: &'a [u8]) -> Self { + Self((spec_id, input)) + } +} + +impl PartialEq> for CacheKeyRef<'_, S> { + fn eq(&self, other: &CacheKey) -> bool { + self.0 .0 == other.0 .0 && self.0 .1 == other.0 .1.as_ref() + } +} + +impl<'a, S: Hash> Hash for CacheKeyRef<'a, S> { + fn hash(&self, state: &mut H) { + self.0 .0.hash(state); + self.0 .1.hash(state); + } +} + +/// Cache entry, precompile successful output. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CacheEntry(PrecompileOutput); + +impl CacheEntry { + const fn gas_used(&self) -> u64 { + self.0.gas_used + } + + fn to_precompile_result(&self) -> PrecompileResult { + Ok(self.0.clone()) + } +} + +/// A cache for precompile inputs / outputs. +#[derive(Debug)] +pub(crate) struct CachedPrecompile +where + S: Eq + Hash + std::fmt::Debug + Send + Sync + Clone + 'static, +{ + /// Cache for precompile results and gas bounds. + cache: PrecompileCache, + /// The precompile. + precompile: DynPrecompile, + /// Cache metrics. + metrics: Option, + /// Spec id associated to the EVM from which this cached precompile was created. + spec_id: S, +} + +impl CachedPrecompile +where + S: Eq + Hash + std::fmt::Debug + Send + Sync + Clone + 'static, +{ + /// `CachedPrecompile` constructor. + pub(crate) const fn new( + precompile: DynPrecompile, + cache: PrecompileCache, + spec_id: S, + metrics: Option, + ) -> Self { + Self { precompile, cache, spec_id, metrics } + } + + pub(crate) fn wrap( + precompile: DynPrecompile, + cache: PrecompileCache, + spec_id: S, + metrics: Option, + ) -> DynPrecompile { + let precompile_id = precompile.precompile_id().clone(); + let wrapped = Self::new(precompile, cache, spec_id, metrics); + (precompile_id, move |input: PrecompileInput<'_>| -> PrecompileResult { + wrapped.call(input) + }) + .into() + } + + fn increment_by_one_precompile_cache_hits(&self) { + if let Some(metrics) = &self.metrics { + metrics.precompile_cache_hits.increment(1); + } + } + + fn increment_by_one_precompile_cache_misses(&self) { + if let Some(metrics) = &self.metrics { + metrics.precompile_cache_misses.increment(1); + } + } + + fn set_precompile_cache_size_metric(&self, to: f64) { + if let Some(metrics) = &self.metrics { + metrics.precompile_cache_size.set(to); + } + } + + fn increment_by_one_precompile_errors(&self) { + if let Some(metrics) = &self.metrics { + metrics.precompile_errors.increment(1); + } + } +} + +impl Precompile for CachedPrecompile +where + S: Eq + Hash + std::fmt::Debug + Send + Sync + Clone + 'static, +{ + fn precompile_id(&self) -> &PrecompileId { + self.precompile.precompile_id() + } + + fn call(&self, input: PrecompileInput<'_>) -> PrecompileResult { + let key = CacheKeyRef::new(self.spec_id.clone(), input.data); + + if let Some(entry) = &self.cache.get(&key) { + self.increment_by_one_precompile_cache_hits(); + if input.gas >= entry.gas_used() { + return entry.to_precompile_result() + } + } + + let calldata = input.data; + let result = self.precompile.call(input); + + match &result { + Ok(output) => { + let key = CacheKey::new(self.spec_id.clone(), Bytes::copy_from_slice(calldata)); + let size = self.cache.insert(key, CacheEntry(output.clone())); + self.set_precompile_cache_size_metric(size as f64); + self.increment_by_one_precompile_cache_misses(); + } + _ => { + self.increment_by_one_precompile_errors(); + } + } + result + } +} + +/// Metrics for the cached precompile. +#[derive(reth_metrics::Metrics, Clone)] +#[metrics(scope = "sync.caching")] +pub(crate) struct CachedPrecompileMetrics { + /// Precompile cache hits + precompile_cache_hits: metrics::Counter, + + /// Precompile cache misses + precompile_cache_misses: metrics::Counter, + + /// Precompile cache size. Uses the LRU cache length as the size metric. + precompile_cache_size: metrics::Gauge, + + /// Precompile execution errors. + precompile_errors: metrics::Counter, +} + +impl CachedPrecompileMetrics { + /// Creates a new instance of [`CachedPrecompileMetrics`] with the given address. + /// + /// Adds address as an `address` label padded with zeros to at least two hex symbols, prefixed + /// by `0x`. + pub(crate) fn new_with_address(address: Address) -> Self { + Self::new_with_labels(&[("address", format!("0x{address:02x}"))]) + } +} + +#[cfg(test)] +mod tests { + use std::hash::DefaultHasher; + + use super::*; + use reth_evm::{EthEvmFactory, Evm, EvmEnv, EvmFactory}; + use reth_revm::db::EmptyDB; + use revm::{context::TxEnv, precompile::PrecompileOutput}; + use revm_primitives::hardfork::SpecId; + + #[test] + fn test_cache_key_ref_hash() { + let key1 = CacheKey::new(SpecId::PRAGUE, b"test_input".into()); + let key2 = CacheKeyRef::new(SpecId::PRAGUE, b"test_input"); + assert!(PartialEq::eq(&key2, &key1)); + + let mut hasher = DefaultHasher::new(); + key1.hash(&mut hasher); + let hash1 = hasher.finish(); + + let mut hasher = DefaultHasher::new(); + key2.hash(&mut hasher); + let hash2 = hasher.finish(); + + assert_eq!(hash1, hash2); + } + + #[test] + fn test_precompile_cache_basic() { + let dyn_precompile: DynPrecompile = (|_input: PrecompileInput<'_>| -> PrecompileResult { + Ok(PrecompileOutput { gas_used: 0, bytes: Bytes::default(), reverted: false }) + }) + .into(); + + let cache = + CachedPrecompile::new(dyn_precompile, PrecompileCache::default(), SpecId::PRAGUE, None); + + let output = PrecompileOutput { + gas_used: 50, + bytes: alloy_primitives::Bytes::copy_from_slice(b"cached_result"), + reverted: false, + }; + + let key = CacheKey::new(SpecId::PRAGUE, b"test_input".into()); + let expected = CacheEntry(output); + cache.cache.insert(key, expected.clone()); + + let key = CacheKeyRef::new(SpecId::PRAGUE, b"test_input"); + let actual = cache.cache.get(&key).unwrap(); + + assert_eq!(actual, expected); + } + + #[test] + fn test_precompile_cache_map_separate_addresses() { + let mut evm = EthEvmFactory::default().create_evm(EmptyDB::default(), EvmEnv::default()); + let input_data = b"same_input"; + let gas_limit = 100_000; + + let address1 = Address::repeat_byte(1); + let address2 = Address::repeat_byte(2); + + let mut cache_map = PrecompileCacheMap::default(); + + // create the first precompile with a specific output + let precompile1: DynPrecompile = (PrecompileId::custom("custom"), { + move |input: PrecompileInput<'_>| -> PrecompileResult { + assert_eq!(input.data, input_data); + + Ok(PrecompileOutput { + gas_used: 5000, + bytes: alloy_primitives::Bytes::copy_from_slice(b"output_from_precompile_1"), + reverted: false, + }) + } + }) + .into(); + + // create the second precompile with a different output + let precompile2: DynPrecompile = (PrecompileId::custom("custom"), { + move |input: PrecompileInput<'_>| -> PrecompileResult { + assert_eq!(input.data, input_data); + + Ok(PrecompileOutput { + gas_used: 7000, + bytes: alloy_primitives::Bytes::copy_from_slice(b"output_from_precompile_2"), + reverted: false, + }) + } + }) + .into(); + + let wrapped_precompile1 = CachedPrecompile::wrap( + precompile1, + cache_map.cache_for_address(address1), + SpecId::PRAGUE, + None, + ); + let wrapped_precompile2 = CachedPrecompile::wrap( + precompile2, + cache_map.cache_for_address(address2), + SpecId::PRAGUE, + None, + ); + + let precompile1_address = Address::with_last_byte(1); + let precompile2_address = Address::with_last_byte(2); + + evm.precompiles_mut().apply_precompile(&precompile1_address, |_| Some(wrapped_precompile1)); + evm.precompiles_mut().apply_precompile(&precompile2_address, |_| Some(wrapped_precompile2)); + + // first invocation of precompile1 (cache miss) + let result1 = evm + .transact_raw(TxEnv { + caller: Address::ZERO, + gas_limit, + data: input_data.into(), + kind: precompile1_address.into(), + ..Default::default() + }) + .unwrap() + .result + .into_output() + .unwrap(); + assert_eq!(result1.as_ref(), b"output_from_precompile_1"); + + // first invocation of precompile2 with the same input (should be a cache miss) + // if cache was incorrectly shared, we'd get precompile1's result + let result2 = evm + .transact_raw(TxEnv { + caller: Address::ZERO, + gas_limit, + data: input_data.into(), + kind: precompile2_address.into(), + ..Default::default() + }) + .unwrap() + .result + .into_output() + .unwrap(); + assert_eq!(result2.as_ref(), b"output_from_precompile_2"); + + // second invocation of precompile1 (should be a cache hit) + let result3 = evm + .transact_raw(TxEnv { + caller: Address::ZERO, + gas_limit, + data: input_data.into(), + kind: precompile1_address.into(), + ..Default::default() + }) + .unwrap() + .result + .into_output() + .unwrap(); + assert_eq!(result3.as_ref(), b"output_from_precompile_1"); + } +} diff --git a/crates/engine/tree/src/tree/state.rs b/crates/engine/tree/src/tree/state.rs new file mode 100644 index 00000000000..0a13207e660 --- /dev/null +++ b/crates/engine/tree/src/tree/state.rs @@ -0,0 +1,600 @@ +//! Functionality related to tree state. + +use crate::engine::EngineApiKind; +use alloy_eips::BlockNumHash; +use alloy_primitives::{ + map::{HashMap, HashSet}, + BlockNumber, B256, +}; +use reth_chain_state::{EthPrimitives, ExecutedBlock}; +use reth_primitives_traits::{AlloyBlockHeader, NodePrimitives, SealedHeader}; +use std::{ + collections::{btree_map, hash_map, BTreeMap, VecDeque}, + ops::Bound, +}; +use tracing::debug; + +/// Keeps track of the state of the tree. +/// +/// ## Invariants +/// +/// - This only stores blocks that are connected to the canonical chain. +/// - All executed blocks are valid and have been executed. +#[derive(Debug, Default)] +pub struct TreeState { + /// __All__ unique executed blocks by block hash that are connected to the canonical chain. + /// + /// This includes blocks of all forks. + pub(crate) blocks_by_hash: HashMap>, + /// Executed blocks grouped by their respective block number. + /// + /// This maps unique block number to all known blocks for that height. + /// + /// Note: there can be multiple blocks at the same height due to forks. + pub(crate) blocks_by_number: BTreeMap>>, + /// Map of any parent block hash to its children. + pub(crate) parent_to_child: HashMap>, + /// Currently tracked canonical head of the chain. + pub(crate) current_canonical_head: BlockNumHash, + /// The engine API variant of this handler + pub(crate) engine_kind: EngineApiKind, +} + +impl TreeState { + /// Returns a new, empty tree state that points to the given canonical head. + pub(crate) fn new(current_canonical_head: BlockNumHash, engine_kind: EngineApiKind) -> Self { + Self { + blocks_by_hash: HashMap::default(), + blocks_by_number: BTreeMap::new(), + current_canonical_head, + parent_to_child: HashMap::default(), + engine_kind, + } + } + + /// Resets the state and points to the given canonical head. + pub(crate) fn reset(&mut self, current_canonical_head: BlockNumHash) { + *self = Self::new(current_canonical_head, self.engine_kind); + } + + /// Returns the number of executed blocks stored. + pub(crate) fn block_count(&self) -> usize { + self.blocks_by_hash.len() + } + + /// Returns the [`ExecutedBlock`] by hash. + pub(crate) fn executed_block_by_hash(&self, hash: B256) -> Option<&ExecutedBlock> { + self.blocks_by_hash.get(&hash) + } + + /// Returns the sealed block header by hash. + pub(crate) fn sealed_header_by_hash( + &self, + hash: &B256, + ) -> Option> { + self.blocks_by_hash.get(hash).map(|b| b.sealed_block().sealed_header().clone()) + } + + /// Returns all available blocks for the given hash that lead back to the canonical chain, from + /// newest to oldest, and the parent hash of the oldest returned block. This parent hash is the + /// highest persisted block connected to this chain. + /// + /// Returns `None` if the block for the given hash is not found. + pub(crate) fn blocks_by_hash(&self, hash: B256) -> Option<(B256, Vec>)> { + let block = self.blocks_by_hash.get(&hash).cloned()?; + let mut parent_hash = block.recovered_block().parent_hash(); + let mut blocks = vec![block]; + while let Some(executed) = self.blocks_by_hash.get(&parent_hash) { + parent_hash = executed.recovered_block().parent_hash(); + blocks.push(executed.clone()); + } + + Some((parent_hash, blocks)) + } + + /// Insert executed block into the state. + pub(crate) fn insert_executed(&mut self, executed: ExecutedBlock) { + let hash = executed.recovered_block().hash(); + let parent_hash = executed.recovered_block().parent_hash(); + let block_number = executed.recovered_block().number(); + + if self.blocks_by_hash.contains_key(&hash) { + return; + } + + self.blocks_by_hash.insert(hash, executed.clone()); + + self.blocks_by_number.entry(block_number).or_default().push(executed); + + self.parent_to_child.entry(parent_hash).or_default().insert(hash); + } + + /// Remove single executed block by its hash. + /// + /// ## Returns + /// + /// The removed block and the block hashes of its children. + fn remove_by_hash(&mut self, hash: B256) -> Option<(ExecutedBlock, HashSet)> { + let executed = self.blocks_by_hash.remove(&hash)?; + + // Remove this block from collection of children of its parent block. + let parent_entry = self.parent_to_child.entry(executed.recovered_block().parent_hash()); + if let hash_map::Entry::Occupied(mut entry) = parent_entry { + entry.get_mut().remove(&hash); + + if entry.get().is_empty() { + entry.remove(); + } + } + + // Remove point to children of this block. + let children = self.parent_to_child.remove(&hash).unwrap_or_default(); + + // Remove this block from `blocks_by_number`. + let block_number_entry = self.blocks_by_number.entry(executed.recovered_block().number()); + if let btree_map::Entry::Occupied(mut entry) = block_number_entry { + // We have to find the index of the block since it exists in a vec + if let Some(index) = entry.get().iter().position(|b| b.recovered_block().hash() == hash) + { + entry.get_mut().swap_remove(index); + + // If there are no blocks left then remove the entry for this block + if entry.get().is_empty() { + entry.remove(); + } + } + } + + Some((executed, children)) + } + + /// Returns whether or not the hash is part of the canonical chain. + pub(crate) fn is_canonical(&self, hash: B256) -> bool { + let mut current_block = self.current_canonical_head.hash; + if current_block == hash { + return true + } + + while let Some(executed) = self.blocks_by_hash.get(¤t_block) { + current_block = executed.recovered_block().parent_hash(); + if current_block == hash { + return true + } + } + + false + } + + /// Removes canonical blocks below the upper bound, only if the last persisted hash is + /// part of the canonical chain. + pub(crate) fn remove_canonical_until( + &mut self, + upper_bound: BlockNumber, + last_persisted_hash: B256, + ) { + debug!(target: "engine::tree", ?upper_bound, ?last_persisted_hash, "Removing canonical blocks from the tree"); + + // If the last persisted hash is not canonical, then we don't want to remove any canonical + // blocks yet. + if !self.is_canonical(last_persisted_hash) { + return + } + + // First, let's walk back the canonical chain and remove canonical blocks lower than the + // upper bound + let mut current_block = self.current_canonical_head.hash; + while let Some(executed) = self.blocks_by_hash.get(¤t_block) { + current_block = executed.recovered_block().parent_hash(); + if executed.recovered_block().number() <= upper_bound { + let num_hash = executed.recovered_block().num_hash(); + debug!(target: "engine::tree", ?num_hash, "Attempting to remove block walking back from the head"); + self.remove_by_hash(executed.recovered_block().hash()); + } + } + debug!(target: "engine::tree", ?upper_bound, ?last_persisted_hash, "Removed canonical blocks from the tree"); + } + + /// Removes all blocks that are below the finalized block, as well as removing non-canonical + /// sidechains that fork from below the finalized block. + pub(crate) fn prune_finalized_sidechains(&mut self, finalized_num_hash: BlockNumHash) { + let BlockNumHash { number: finalized_num, hash: finalized_hash } = finalized_num_hash; + + // We remove disconnected sidechains in three steps: + // * first, remove everything with a block number __below__ the finalized block. + // * next, we populate a vec with parents __at__ the finalized block. + // * finally, we iterate through the vec, removing children until the vec is empty + // (BFS). + + // We _exclude_ the finalized block because we will be dealing with the blocks __at__ + // the finalized block later. + let blocks_to_remove = self + .blocks_by_number + .range((Bound::Unbounded, Bound::Excluded(finalized_num))) + .flat_map(|(_, blocks)| blocks.iter().map(|b| b.recovered_block().hash())) + .collect::>(); + for hash in blocks_to_remove { + if let Some((removed, _)) = self.remove_by_hash(hash) { + debug!(target: "engine::tree", num_hash=?removed.recovered_block().num_hash(), "Removed finalized sidechain block"); + } + } + + // The only block that should remain at the `finalized` number now, is the finalized + // block, if it exists. + // + // For all other blocks, we first put their children into this vec. + // Then, we will iterate over them, removing them, adding their children, etc, + // until the vec is empty. + let mut blocks_to_remove = self.blocks_by_number.remove(&finalized_num).unwrap_or_default(); + + // re-insert the finalized hash if we removed it + if let Some(position) = + blocks_to_remove.iter().position(|b| b.recovered_block().hash() == finalized_hash) + { + let finalized_block = blocks_to_remove.swap_remove(position); + self.blocks_by_number.insert(finalized_num, vec![finalized_block]); + } + + let mut blocks_to_remove = blocks_to_remove + .into_iter() + .map(|e| e.recovered_block().hash()) + .collect::>(); + while let Some(block) = blocks_to_remove.pop_front() { + if let Some((removed, children)) = self.remove_by_hash(block) { + debug!(target: "engine::tree", num_hash=?removed.recovered_block().num_hash(), "Removed finalized sidechain child block"); + blocks_to_remove.extend(children); + } + } + } + + /// Remove all blocks up to __and including__ the given block number. + /// + /// If a finalized hash is provided, the only non-canonical blocks which will be removed are + /// those which have a fork point at or below the finalized hash. + /// + /// Canonical blocks below the upper bound will still be removed. + /// + /// NOTE: if the finalized block is greater than the upper bound, the only blocks that will be + /// removed are canonical blocks and sidechains that fork below the `upper_bound`. This is the + /// same behavior as if the `finalized_num` were `Some(upper_bound)`. + pub(crate) fn remove_until( + &mut self, + upper_bound: BlockNumHash, + last_persisted_hash: B256, + finalized_num_hash: Option, + ) { + debug!(target: "engine::tree", ?upper_bound, ?finalized_num_hash, "Removing blocks from the tree"); + + // If the finalized num is ahead of the upper bound, and exists, we need to instead ensure + // that the only blocks removed, are canonical blocks less than the upper bound + let finalized_num_hash = finalized_num_hash.map(|mut finalized| { + if upper_bound.number < finalized.number { + finalized = upper_bound; + debug!(target: "engine::tree", ?finalized, "Adjusted upper bound"); + } + finalized + }); + + // We want to do two things: + // * remove canonical blocks that are persisted + // * remove forks whose root are below the finalized block + // We can do this in 2 steps: + // * remove all canonical blocks below the upper bound + // * fetch the number of the finalized hash, removing any sidechains that are __below__ the + // finalized block + self.remove_canonical_until(upper_bound.number, last_persisted_hash); + + // Now, we have removed canonical blocks (assuming the upper bound is above the finalized + // block) and only have sidechains below the finalized block. + if let Some(finalized_num_hash) = finalized_num_hash { + self.prune_finalized_sidechains(finalized_num_hash); + } + } + + /// Updates the canonical head to the given block. + pub(crate) const fn set_canonical_head(&mut self, new_head: BlockNumHash) { + self.current_canonical_head = new_head; + } + + /// Returns the tracked canonical head. + pub(crate) const fn canonical_head(&self) -> &BlockNumHash { + &self.current_canonical_head + } + + /// Returns the block hash of the canonical head. + pub(crate) const fn canonical_block_hash(&self) -> B256 { + self.canonical_head().hash + } + + /// Returns the block number of the canonical head. + pub(crate) const fn canonical_block_number(&self) -> BlockNumber { + self.canonical_head().number + } +} + +#[cfg(test)] +impl TreeState { + /// Determines if the second block is a descendant of the first block. + /// + /// If the two blocks are the same, this returns `false`. + pub(crate) fn is_descendant( + &self, + first: BlockNumHash, + second: alloy_eips::eip1898::BlockWithParent, + ) -> bool { + // If the second block's parent is the first block's hash, then it is a direct child + // and we can return early. + if second.parent == first.hash { + return true + } + + // If the second block is lower than, or has the same block number, they are not + // descendants. + if second.block.number <= first.number { + return false + } + + // iterate through parents of the second until we reach the number + let Some(mut current_block) = self.blocks_by_hash.get(&second.parent) else { + // If we can't find its parent in the tree, we can't continue, so return false + return false + }; + + while current_block.recovered_block().number() > first.number + 1 { + let Some(block) = + self.blocks_by_hash.get(¤t_block.recovered_block().parent_hash()) + else { + // If we can't find its parent in the tree, we can't continue, so return false + return false + }; + + current_block = block; + } + + // Now the block numbers should be equal, so we compare hashes. + current_block.recovered_block().parent_hash() == first.hash + } +} + +#[cfg(test)] +mod tests { + use super::*; + use reth_chain_state::test_utils::TestBlockBuilder; + + #[test] + fn test_tree_state_normal_descendant() { + let mut tree_state = TreeState::new(BlockNumHash::default(), EngineApiKind::Ethereum); + let blocks: Vec<_> = TestBlockBuilder::eth().get_executed_blocks(1..4).collect(); + + tree_state.insert_executed(blocks[0].clone()); + assert!(tree_state.is_descendant( + blocks[0].recovered_block().num_hash(), + blocks[1].recovered_block().block_with_parent() + )); + + tree_state.insert_executed(blocks[1].clone()); + + assert!(tree_state.is_descendant( + blocks[0].recovered_block().num_hash(), + blocks[2].recovered_block().block_with_parent() + )); + assert!(tree_state.is_descendant( + blocks[1].recovered_block().num_hash(), + blocks[2].recovered_block().block_with_parent() + )); + } + + #[tokio::test] + async fn test_tree_state_insert_executed() { + let mut tree_state = TreeState::new(BlockNumHash::default(), EngineApiKind::Ethereum); + let blocks: Vec<_> = TestBlockBuilder::eth().get_executed_blocks(1..4).collect(); + + tree_state.insert_executed(blocks[0].clone()); + tree_state.insert_executed(blocks[1].clone()); + + assert_eq!( + tree_state.parent_to_child.get(&blocks[0].recovered_block().hash()), + Some(&HashSet::from_iter([blocks[1].recovered_block().hash()])) + ); + + assert!(!tree_state.parent_to_child.contains_key(&blocks[1].recovered_block().hash())); + + tree_state.insert_executed(blocks[2].clone()); + + assert_eq!( + tree_state.parent_to_child.get(&blocks[1].recovered_block().hash()), + Some(&HashSet::from_iter([blocks[2].recovered_block().hash()])) + ); + assert!(tree_state.parent_to_child.contains_key(&blocks[1].recovered_block().hash())); + + assert!(!tree_state.parent_to_child.contains_key(&blocks[2].recovered_block().hash())); + } + + #[tokio::test] + async fn test_tree_state_insert_executed_with_reorg() { + let mut tree_state = TreeState::new(BlockNumHash::default(), EngineApiKind::Ethereum); + let mut test_block_builder = TestBlockBuilder::eth(); + let blocks: Vec<_> = test_block_builder.get_executed_blocks(1..6).collect(); + + for block in &blocks { + tree_state.insert_executed(block.clone()); + } + assert_eq!(tree_state.blocks_by_hash.len(), 5); + + let fork_block_3 = test_block_builder + .get_executed_block_with_number(3, blocks[1].recovered_block().hash()); + let fork_block_4 = test_block_builder + .get_executed_block_with_number(4, fork_block_3.recovered_block().hash()); + let fork_block_5 = test_block_builder + .get_executed_block_with_number(5, fork_block_4.recovered_block().hash()); + + tree_state.insert_executed(fork_block_3.clone()); + tree_state.insert_executed(fork_block_4.clone()); + tree_state.insert_executed(fork_block_5.clone()); + + assert_eq!(tree_state.blocks_by_hash.len(), 8); + assert_eq!(tree_state.blocks_by_number[&3].len(), 2); // two blocks at height 3 (original and fork) + assert_eq!(tree_state.parent_to_child[&blocks[1].recovered_block().hash()].len(), 2); // block 2 should have two children + + // verify that we can insert the same block again without issues + tree_state.insert_executed(fork_block_4.clone()); + assert_eq!(tree_state.blocks_by_hash.len(), 8); + + assert!(tree_state.parent_to_child[&fork_block_3.recovered_block().hash()] + .contains(&fork_block_4.recovered_block().hash())); + assert!(tree_state.parent_to_child[&fork_block_4.recovered_block().hash()] + .contains(&fork_block_5.recovered_block().hash())); + + assert_eq!(tree_state.blocks_by_number[&4].len(), 2); + assert_eq!(tree_state.blocks_by_number[&5].len(), 2); + } + + #[tokio::test] + async fn test_tree_state_remove_before() { + let start_num_hash = BlockNumHash::default(); + let mut tree_state = TreeState::new(start_num_hash, EngineApiKind::Ethereum); + let blocks: Vec<_> = TestBlockBuilder::eth().get_executed_blocks(1..6).collect(); + + for block in &blocks { + tree_state.insert_executed(block.clone()); + } + + let last = blocks.last().unwrap(); + + // set the canonical head + tree_state.set_canonical_head(last.recovered_block().num_hash()); + + // inclusive bound, so we should remove anything up to and including 2 + tree_state.remove_until( + BlockNumHash::new(2, blocks[1].recovered_block().hash()), + start_num_hash.hash, + Some(blocks[1].recovered_block().num_hash()), + ); + + assert!(!tree_state.blocks_by_hash.contains_key(&blocks[0].recovered_block().hash())); + assert!(!tree_state.blocks_by_hash.contains_key(&blocks[1].recovered_block().hash())); + assert!(!tree_state.blocks_by_number.contains_key(&1)); + assert!(!tree_state.blocks_by_number.contains_key(&2)); + + assert!(tree_state.blocks_by_hash.contains_key(&blocks[2].recovered_block().hash())); + assert!(tree_state.blocks_by_hash.contains_key(&blocks[3].recovered_block().hash())); + assert!(tree_state.blocks_by_hash.contains_key(&blocks[4].recovered_block().hash())); + assert!(tree_state.blocks_by_number.contains_key(&3)); + assert!(tree_state.blocks_by_number.contains_key(&4)); + assert!(tree_state.blocks_by_number.contains_key(&5)); + + assert!(!tree_state.parent_to_child.contains_key(&blocks[0].recovered_block().hash())); + assert!(!tree_state.parent_to_child.contains_key(&blocks[1].recovered_block().hash())); + assert!(tree_state.parent_to_child.contains_key(&blocks[2].recovered_block().hash())); + assert!(tree_state.parent_to_child.contains_key(&blocks[3].recovered_block().hash())); + assert!(!tree_state.parent_to_child.contains_key(&blocks[4].recovered_block().hash())); + + assert_eq!( + tree_state.parent_to_child.get(&blocks[2].recovered_block().hash()), + Some(&HashSet::from_iter([blocks[3].recovered_block().hash()])) + ); + assert_eq!( + tree_state.parent_to_child.get(&blocks[3].recovered_block().hash()), + Some(&HashSet::from_iter([blocks[4].recovered_block().hash()])) + ); + } + + #[tokio::test] + async fn test_tree_state_remove_before_finalized() { + let start_num_hash = BlockNumHash::default(); + let mut tree_state = TreeState::new(start_num_hash, EngineApiKind::Ethereum); + let blocks: Vec<_> = TestBlockBuilder::eth().get_executed_blocks(1..6).collect(); + + for block in &blocks { + tree_state.insert_executed(block.clone()); + } + + let last = blocks.last().unwrap(); + + // set the canonical head + tree_state.set_canonical_head(last.recovered_block().num_hash()); + + // we should still remove everything up to and including 2 + tree_state.remove_until( + BlockNumHash::new(2, blocks[1].recovered_block().hash()), + start_num_hash.hash, + None, + ); + + assert!(!tree_state.blocks_by_hash.contains_key(&blocks[0].recovered_block().hash())); + assert!(!tree_state.blocks_by_hash.contains_key(&blocks[1].recovered_block().hash())); + assert!(!tree_state.blocks_by_number.contains_key(&1)); + assert!(!tree_state.blocks_by_number.contains_key(&2)); + + assert!(tree_state.blocks_by_hash.contains_key(&blocks[2].recovered_block().hash())); + assert!(tree_state.blocks_by_hash.contains_key(&blocks[3].recovered_block().hash())); + assert!(tree_state.blocks_by_hash.contains_key(&blocks[4].recovered_block().hash())); + assert!(tree_state.blocks_by_number.contains_key(&3)); + assert!(tree_state.blocks_by_number.contains_key(&4)); + assert!(tree_state.blocks_by_number.contains_key(&5)); + + assert!(!tree_state.parent_to_child.contains_key(&blocks[0].recovered_block().hash())); + assert!(!tree_state.parent_to_child.contains_key(&blocks[1].recovered_block().hash())); + assert!(tree_state.parent_to_child.contains_key(&blocks[2].recovered_block().hash())); + assert!(tree_state.parent_to_child.contains_key(&blocks[3].recovered_block().hash())); + assert!(!tree_state.parent_to_child.contains_key(&blocks[4].recovered_block().hash())); + + assert_eq!( + tree_state.parent_to_child.get(&blocks[2].recovered_block().hash()), + Some(&HashSet::from_iter([blocks[3].recovered_block().hash()])) + ); + assert_eq!( + tree_state.parent_to_child.get(&blocks[3].recovered_block().hash()), + Some(&HashSet::from_iter([blocks[4].recovered_block().hash()])) + ); + } + + #[tokio::test] + async fn test_tree_state_remove_before_lower_finalized() { + let start_num_hash = BlockNumHash::default(); + let mut tree_state = TreeState::new(start_num_hash, EngineApiKind::Ethereum); + let blocks: Vec<_> = TestBlockBuilder::eth().get_executed_blocks(1..6).collect(); + + for block in &blocks { + tree_state.insert_executed(block.clone()); + } + + let last = blocks.last().unwrap(); + + // set the canonical head + tree_state.set_canonical_head(last.recovered_block().num_hash()); + + // we have no forks so we should still remove anything up to and including 2 + tree_state.remove_until( + BlockNumHash::new(2, blocks[1].recovered_block().hash()), + start_num_hash.hash, + Some(blocks[0].recovered_block().num_hash()), + ); + + assert!(!tree_state.blocks_by_hash.contains_key(&blocks[0].recovered_block().hash())); + assert!(!tree_state.blocks_by_hash.contains_key(&blocks[1].recovered_block().hash())); + assert!(!tree_state.blocks_by_number.contains_key(&1)); + assert!(!tree_state.blocks_by_number.contains_key(&2)); + + assert!(tree_state.blocks_by_hash.contains_key(&blocks[2].recovered_block().hash())); + assert!(tree_state.blocks_by_hash.contains_key(&blocks[3].recovered_block().hash())); + assert!(tree_state.blocks_by_hash.contains_key(&blocks[4].recovered_block().hash())); + assert!(tree_state.blocks_by_number.contains_key(&3)); + assert!(tree_state.blocks_by_number.contains_key(&4)); + assert!(tree_state.blocks_by_number.contains_key(&5)); + + assert!(!tree_state.parent_to_child.contains_key(&blocks[0].recovered_block().hash())); + assert!(!tree_state.parent_to_child.contains_key(&blocks[1].recovered_block().hash())); + assert!(tree_state.parent_to_child.contains_key(&blocks[2].recovered_block().hash())); + assert!(tree_state.parent_to_child.contains_key(&blocks[3].recovered_block().hash())); + assert!(!tree_state.parent_to_child.contains_key(&blocks[4].recovered_block().hash())); + + assert_eq!( + tree_state.parent_to_child.get(&blocks[2].recovered_block().hash()), + Some(&HashSet::from_iter([blocks[3].recovered_block().hash()])) + ); + assert_eq!( + tree_state.parent_to_child.get(&blocks[3].recovered_block().hash()), + Some(&HashSet::from_iter([blocks[4].recovered_block().hash()])) + ); + } +} diff --git a/crates/engine/tree/src/tree/tests.rs b/crates/engine/tree/src/tree/tests.rs new file mode 100644 index 00000000000..7fbae4cac5c --- /dev/null +++ b/crates/engine/tree/src/tree/tests.rs @@ -0,0 +1,1993 @@ +use super::*; +use crate::{ + persistence::PersistenceAction, + tree::{ + payload_validator::{BasicEngineValidator, TreeCtx, ValidationOutcome}, + persistence_state::CurrentPersistenceAction, + TreeConfig, + }, +}; +use alloy_consensus::Header; +use alloy_eips::eip1898::BlockWithParent; +use alloy_primitives::{ + map::{HashMap, HashSet}, + Bytes, B256, +}; +use alloy_rlp::Decodable; +use alloy_rpc_types_engine::{ + ExecutionData, ExecutionPayloadSidecar, ExecutionPayloadV1, ForkchoiceState, +}; +use assert_matches::assert_matches; +use reth_chain_state::{test_utils::TestBlockBuilder, BlockState}; +use reth_chainspec::{ChainSpec, HOLESKY, MAINNET}; +use reth_engine_primitives::{EngineApiValidator, ForkchoiceStatus, NoopInvalidBlockHook}; +use reth_ethereum_consensus::EthBeaconConsensus; +use reth_ethereum_engine_primitives::EthEngineTypes; +use reth_ethereum_primitives::{Block, EthPrimitives}; +use reth_evm_ethereum::MockEvmConfig; +use reth_primitives_traits::Block as _; +use reth_provider::{test_utils::MockEthProvider, ExecutionOutcome}; +use reth_trie::{updates::TrieUpdates, HashedPostState}; +use std::{ + collections::BTreeMap, + str::FromStr, + sync::{ + mpsc::{channel, Receiver, Sender}, + Arc, + }, +}; +use tokio::sync::oneshot; + +/// Mock engine validator for tests +#[derive(Debug, Clone)] +struct MockEngineValidator; + +impl reth_engine_primitives::PayloadValidator for MockEngineValidator { + type Block = Block; + + fn ensure_well_formed_payload( + &self, + payload: ExecutionData, + ) -> Result< + reth_primitives_traits::RecoveredBlock, + reth_payload_primitives::NewPayloadError, + > { + // For tests, convert the execution payload to a block + let block = reth_ethereum_primitives::Block::try_from(payload.payload).map_err(|e| { + reth_payload_primitives::NewPayloadError::Other(format!("{e:?}").into()) + })?; + let sealed = block.seal_slow(); + + sealed.try_recover().map_err(|e| reth_payload_primitives::NewPayloadError::Other(e.into())) + } +} + +impl EngineApiValidator for MockEngineValidator { + fn validate_version_specific_fields( + &self, + _version: reth_payload_primitives::EngineApiMessageVersion, + _payload_or_attrs: reth_payload_primitives::PayloadOrAttributes< + '_, + alloy_rpc_types_engine::ExecutionData, + alloy_rpc_types_engine::PayloadAttributes, + >, + ) -> Result<(), reth_payload_primitives::EngineObjectValidationError> { + // Mock implementation - always valid + Ok(()) + } + + fn ensure_well_formed_attributes( + &self, + _version: reth_payload_primitives::EngineApiMessageVersion, + _attributes: &alloy_rpc_types_engine::PayloadAttributes, + ) -> Result<(), reth_payload_primitives::EngineObjectValidationError> { + // Mock implementation - always valid + Ok(()) + } +} + +/// This is a test channel that allows you to `release` any value that is in the channel. +/// +/// If nothing has been sent, then the next value will be immediately sent. +struct TestChannel { + /// If an item is sent to this channel, an item will be released in the wrapped channel + release: Receiver<()>, + /// The sender channel + tx: Sender, + /// The receiver channel + rx: Receiver, +} + +impl TestChannel { + /// Creates a new test channel + fn spawn_channel() -> (Sender, Receiver, TestChannelHandle) { + let (original_tx, original_rx) = channel(); + let (wrapped_tx, wrapped_rx) = channel(); + let (release_tx, release_rx) = channel(); + let handle = TestChannelHandle::new(release_tx); + let test_channel = Self { release: release_rx, tx: wrapped_tx, rx: original_rx }; + // spawn the task that listens and releases stuff + std::thread::spawn(move || test_channel.intercept_loop()); + (original_tx, wrapped_rx, handle) + } + + /// Runs the intercept loop, waiting for the handle to release a value + fn intercept_loop(&self) { + while self.release.recv() == Ok(()) { + let Ok(value) = self.rx.recv() else { return }; + + let _ = self.tx.send(value); + } + } +} + +struct TestChannelHandle { + /// The sender to use for releasing values + release: Sender<()>, +} + +impl TestChannelHandle { + /// Returns a [`TestChannelHandle`] + const fn new(release: Sender<()>) -> Self { + Self { release } + } + + /// Signals to the channel task that a value should be released + #[expect(dead_code)] + fn release(&self) { + let _ = self.release.send(()); + } +} + +struct TestHarness { + tree: EngineApiTreeHandler< + EthPrimitives, + MockEthProvider, + EthEngineTypes, + BasicEngineValidator, + MockEvmConfig, + >, + to_tree_tx: Sender, Block>>, + from_tree_rx: UnboundedReceiver, + blocks: Vec, + action_rx: Receiver, + block_builder: TestBlockBuilder, + provider: MockEthProvider, +} + +impl TestHarness { + fn new(chain_spec: Arc) -> Self { + let (action_tx, action_rx) = channel(); + Self::with_persistence_channel(chain_spec, action_tx, action_rx) + } + + #[expect(dead_code)] + fn with_test_channel(chain_spec: Arc) -> (Self, TestChannelHandle) { + let (action_tx, action_rx, handle) = TestChannel::spawn_channel(); + (Self::with_persistence_channel(chain_spec, action_tx, action_rx), handle) + } + + fn with_persistence_channel( + chain_spec: Arc, + action_tx: Sender, + action_rx: Receiver, + ) -> Self { + let persistence_handle = PersistenceHandle::new(action_tx); + + let consensus = Arc::new(EthBeaconConsensus::new(chain_spec.clone())); + + let provider = MockEthProvider::default(); + + let payload_validator = MockEngineValidator; + + let (from_tree_tx, from_tree_rx) = unbounded_channel(); + + let header = chain_spec.genesis_header().clone(); + let header = SealedHeader::seal_slow(header); + let engine_api_tree_state = + EngineApiTreeState::new(10, 10, header.num_hash(), EngineApiKind::Ethereum); + let canonical_in_memory_state = CanonicalInMemoryState::with_head(header, None, None); + + let (to_payload_service, _payload_command_rx) = unbounded_channel(); + let payload_builder = PayloadBuilderHandle::new(to_payload_service); + + let evm_config = MockEvmConfig::default(); + let engine_validator = BasicEngineValidator::new( + provider.clone(), + consensus.clone(), + evm_config.clone(), + payload_validator, + TreeConfig::default(), + Box::new(NoopInvalidBlockHook::default()), + ); + + let tree = EngineApiTreeHandler::new( + provider.clone(), + consensus, + engine_validator, + from_tree_tx, + engine_api_tree_state, + canonical_in_memory_state, + persistence_handle, + PersistenceState::default(), + payload_builder, + // always assume enough parallelism for tests + TreeConfig::default().with_legacy_state_root(false).with_has_enough_parallelism(true), + EngineApiKind::Ethereum, + evm_config, + ); + + let block_builder = TestBlockBuilder::default().with_chain_spec((*chain_spec).clone()); + Self { + to_tree_tx: tree.incoming_tx.clone(), + tree, + from_tree_rx, + blocks: vec![], + action_rx, + block_builder, + provider, + } + } + + fn with_blocks(mut self, blocks: Vec) -> Self { + let mut blocks_by_hash = HashMap::default(); + let mut blocks_by_number = BTreeMap::new(); + let mut state_by_hash = HashMap::default(); + let mut hash_by_number = BTreeMap::new(); + let mut parent_to_child: HashMap> = HashMap::default(); + let mut parent_hash = B256::ZERO; + + for block in &blocks { + let sealed_block = block.recovered_block(); + let hash = sealed_block.hash(); + let number = sealed_block.number; + blocks_by_hash.insert(hash, block.clone()); + blocks_by_number.entry(number).or_insert_with(Vec::new).push(block.clone()); + state_by_hash.insert(hash, Arc::new(BlockState::new(block.clone()))); + hash_by_number.insert(number, hash); + parent_to_child.entry(parent_hash).or_default().insert(hash); + parent_hash = hash; + } + + self.tree.state.tree_state = TreeState { + blocks_by_hash, + blocks_by_number, + current_canonical_head: blocks.last().unwrap().recovered_block().num_hash(), + parent_to_child, + engine_kind: EngineApiKind::Ethereum, + }; + + let last_executed_block = blocks.last().unwrap().clone(); + let pending = Some(BlockState::new(last_executed_block)); + self.tree.canonical_in_memory_state = + CanonicalInMemoryState::new(state_by_hash, hash_by_number, pending, None, None); + + self.blocks = blocks.clone(); + + let recovered_blocks = + blocks.iter().map(|b| b.recovered_block().clone()).collect::>(); + + self.persist_blocks(recovered_blocks); + + self + } + + const fn with_backfill_state(mut self, state: BackfillSyncState) -> Self { + self.tree.backfill_sync_state = state; + self + } + + async fn fcu_to(&mut self, block_hash: B256, fcu_status: impl Into) { + let fcu_status = fcu_status.into(); + + self.send_fcu(block_hash, fcu_status).await; + + self.check_fcu(block_hash, fcu_status).await; + } + + async fn send_fcu(&mut self, block_hash: B256, fcu_status: impl Into) { + let fcu_state = self.fcu_state(block_hash); + + let (tx, rx) = oneshot::channel(); + self.tree + .on_engine_message(FromEngine::Request( + BeaconEngineMessage::ForkchoiceUpdated { + state: fcu_state, + payload_attrs: None, + tx, + version: EngineApiMessageVersion::default(), + } + .into(), + )) + .unwrap(); + + let response = rx.await.unwrap().unwrap().await.unwrap(); + match fcu_status.into() { + ForkchoiceStatus::Valid => assert!(response.payload_status.is_valid()), + ForkchoiceStatus::Syncing => assert!(response.payload_status.is_syncing()), + ForkchoiceStatus::Invalid => assert!(response.payload_status.is_invalid()), + } + } + + async fn check_fcu(&mut self, block_hash: B256, fcu_status: impl Into) { + let fcu_state = self.fcu_state(block_hash); + + // check for ForkchoiceUpdated event + let event = self.from_tree_rx.recv().await.unwrap(); + match event { + EngineApiEvent::BeaconConsensus(ConsensusEngineEvent::ForkchoiceUpdated( + state, + status, + )) => { + assert_eq!(state, fcu_state); + assert_eq!(status, fcu_status.into()); + } + _ => panic!("Unexpected event: {event:#?}"), + } + } + + const fn fcu_state(&self, block_hash: B256) -> ForkchoiceState { + ForkchoiceState { + head_block_hash: block_hash, + safe_block_hash: block_hash, + finalized_block_hash: block_hash, + } + } + + fn persist_blocks(&self, blocks: Vec>) { + let mut block_data: Vec<(B256, Block)> = Vec::with_capacity(blocks.len()); + let mut headers_data: Vec<(B256, Header)> = Vec::with_capacity(blocks.len()); + + for block in &blocks { + block_data.push((block.hash(), block.clone_block())); + headers_data.push((block.hash(), block.header().clone())); + } + + self.provider.extend_blocks(block_data); + self.provider.extend_headers(headers_data); + } +} + +/// Simplified test metrics for validation calls +#[derive(Debug, Default)] +struct TestMetrics { + /// Count of successful `validate_block_direct` calls + validation_calls: usize, + /// Count of validation errors + validation_errors: usize, +} + +impl TestMetrics { + fn record_validation(&mut self, success: bool) { + if success { + self.validation_calls += 1; + } else { + self.validation_errors += 1; + } + } + + fn total_calls(&self) -> usize { + self.validation_calls + self.validation_errors + } +} + +/// Extended test harness with direct `validate_block_with_state` access +pub(crate) struct ValidatorTestHarness { + /// Basic test harness + harness: TestHarness, + /// Direct access to validator for `validate_block_with_state` calls + validator: BasicEngineValidator, + /// Simple validation metrics + metrics: TestMetrics, +} + +impl ValidatorTestHarness { + fn new(chain_spec: Arc) -> Self { + let harness = TestHarness::new(chain_spec.clone()); + + // Create validator identical to the one in TestHarness + let consensus = Arc::new(EthBeaconConsensus::new(chain_spec)); + let provider = harness.provider.clone(); + let payload_validator = MockEngineValidator; + let evm_config = MockEvmConfig::default(); + + let validator = BasicEngineValidator::new( + provider, + consensus, + evm_config, + payload_validator, + TreeConfig::default(), + Box::new(NoopInvalidBlockHook::default()), + ); + + Self { harness, validator, metrics: TestMetrics::default() } + } + + /// Configure `PersistenceState` for specific persistence scenarios + fn start_persistence_operation(&mut self, action: CurrentPersistenceAction) { + use tokio::sync::oneshot; + + // Create a dummy receiver for testing - it will never receive a value + let (_tx, rx) = oneshot::channel(); + + match action { + CurrentPersistenceAction::SavingBlocks { highest } => { + self.harness.tree.persistence_state.start_save(highest, rx); + } + CurrentPersistenceAction::RemovingBlocks { new_tip_num } => { + self.harness.tree.persistence_state.start_remove(new_tip_num, rx); + } + } + } + + /// Check if persistence is currently in progress + fn is_persistence_in_progress(&self) -> bool { + self.harness.tree.persistence_state.in_progress() + } + + /// Call `validate_block_with_state` directly with block + fn validate_block_direct( + &mut self, + block: RecoveredBlock, + ) -> ValidationOutcome { + let ctx = TreeCtx::new( + &mut self.harness.tree.state, + &self.harness.tree.canonical_in_memory_state, + ); + let result = self.validator.validate_block(block, ctx); + self.metrics.record_validation(result.is_ok()); + result + } + + /// Get validation metrics for testing + fn validation_call_count(&self) -> usize { + self.metrics.total_calls() + } +} + +/// Factory for creating test blocks with controllable properties +struct TestBlockFactory { + builder: TestBlockBuilder, +} + +impl TestBlockFactory { + fn new(chain_spec: ChainSpec) -> Self { + Self { builder: TestBlockBuilder::eth().with_chain_spec(chain_spec) } + } + + /// Create block that triggers consensus violation by corrupting state root + fn create_invalid_consensus_block(&mut self, parent_hash: B256) -> RecoveredBlock { + let mut block = self.builder.generate_random_block(1, parent_hash).into_block(); + + // Corrupt state root to trigger consensus violation + block.header.state_root = B256::random(); + + block.seal_slow().try_recover().unwrap() + } + + /// Create block that triggers execution failure + fn create_invalid_execution_block(&mut self, parent_hash: B256) -> RecoveredBlock { + let mut block = self.builder.generate_random_block(1, parent_hash).into_block(); + + // Create transaction that will fail execution + // This is simplified - in practice we'd create a transaction with insufficient gas, etc. + block.header.gas_used = block.header.gas_limit + 1; // Gas used exceeds limit + + block.seal_slow().try_recover().unwrap() + } + + /// Create valid block + fn create_valid_block(&mut self, parent_hash: B256) -> RecoveredBlock { + let block = self.builder.generate_random_block(1, parent_hash).into_block(); + block.seal_slow().try_recover().unwrap() + } +} + +#[test] +fn test_tree_persist_block_batch() { + let tree_config = TreeConfig::default(); + let chain_spec = MAINNET.clone(); + let mut test_block_builder = TestBlockBuilder::eth().with_chain_spec((*chain_spec).clone()); + + // we need more than tree_config.persistence_threshold() +1 blocks to + // trigger the persistence task. + let blocks: Vec<_> = test_block_builder + .get_executed_blocks(1..tree_config.persistence_threshold() + 2) + .collect(); + let mut test_harness = TestHarness::new(chain_spec).with_blocks(blocks); + + let mut blocks = vec![]; + for idx in 0..tree_config.max_execute_block_batch_size() * 2 { + blocks.push(test_block_builder.generate_random_block(idx as u64, B256::random())); + } + + test_harness.to_tree_tx.send(FromEngine::DownloadedBlocks(blocks)).unwrap(); + + // process the message + let msg = test_harness.tree.try_recv_engine_message().unwrap().unwrap(); + test_harness.tree.on_engine_message(msg).unwrap(); + + // we now should receive the other batch + let msg = test_harness.tree.try_recv_engine_message().unwrap().unwrap(); + match msg { + FromEngine::DownloadedBlocks(blocks) => { + assert_eq!(blocks.len(), tree_config.max_execute_block_batch_size()); + } + _ => panic!("unexpected message: {msg:#?}"), + } +} + +#[tokio::test] +async fn test_tree_persist_blocks() { + let tree_config = TreeConfig::default(); + let chain_spec = MAINNET.clone(); + let mut test_block_builder = TestBlockBuilder::eth().with_chain_spec((*chain_spec).clone()); + + // we need more than tree_config.persistence_threshold() +1 blocks to + // trigger the persistence task. + let blocks: Vec<_> = test_block_builder + .get_executed_blocks(1..tree_config.persistence_threshold() + 2) + .collect(); + let test_harness = TestHarness::new(chain_spec).with_blocks(blocks.clone()); + std::thread::Builder::new() + .name("Engine Task".to_string()) + .spawn(|| test_harness.tree.run()) + .unwrap(); + + // send a message to the tree to enter the main loop. + test_harness.to_tree_tx.send(FromEngine::DownloadedBlocks(vec![])).unwrap(); + + let received_action = + test_harness.action_rx.recv().expect("Failed to receive save blocks action"); + if let PersistenceAction::SaveBlocks(saved_blocks, _) = received_action { + // only blocks.len() - tree_config.memory_block_buffer_target() will be + // persisted + let expected_persist_len = blocks.len() - tree_config.memory_block_buffer_target() as usize; + assert_eq!(saved_blocks.len(), expected_persist_len); + assert_eq!(saved_blocks, blocks[..expected_persist_len]); + } else { + panic!("unexpected action received {received_action:?}"); + } +} + +#[tokio::test] +async fn test_in_memory_state_trait_impl() { + let blocks: Vec<_> = TestBlockBuilder::eth().get_executed_blocks(0..10).collect(); + let test_harness = TestHarness::new(MAINNET.clone()).with_blocks(blocks.clone()); + + for executed_block in blocks { + let sealed_block = executed_block.recovered_block(); + + let expected_state = BlockState::new(executed_block.clone()); + + let actual_state_by_hash = + test_harness.tree.canonical_in_memory_state.state_by_hash(sealed_block.hash()).unwrap(); + assert_eq!(expected_state, *actual_state_by_hash); + + let actual_state_by_number = test_harness + .tree + .canonical_in_memory_state + .state_by_number(sealed_block.number) + .unwrap(); + assert_eq!(expected_state, *actual_state_by_number); + } +} + +#[tokio::test] +async fn test_engine_request_during_backfill() { + let tree_config = TreeConfig::default(); + let blocks: Vec<_> = TestBlockBuilder::eth() + .get_executed_blocks(0..tree_config.persistence_threshold()) + .collect(); + let mut test_harness = TestHarness::new(MAINNET.clone()) + .with_blocks(blocks) + .with_backfill_state(BackfillSyncState::Active); + + let (tx, rx) = oneshot::channel(); + test_harness + .tree + .on_engine_message(FromEngine::Request( + BeaconEngineMessage::ForkchoiceUpdated { + state: ForkchoiceState { + head_block_hash: B256::random(), + safe_block_hash: B256::random(), + finalized_block_hash: B256::random(), + }, + payload_attrs: None, + tx, + version: EngineApiMessageVersion::default(), + } + .into(), + )) + .unwrap(); + + let resp = rx.await.unwrap().unwrap().await.unwrap(); + assert!(resp.payload_status.is_syncing()); +} + +#[test] +fn test_disconnected_payload() { + let s = include_str!("../../test-data/holesky/2.rlp"); + let data = Bytes::from_str(s).unwrap(); + let block = Block::decode(&mut data.as_ref()).unwrap(); + let sealed = block.seal_slow(); + let hash = sealed.hash(); + let sealed_clone = sealed.clone(); + let block = sealed.into_block(); + let payload = ExecutionPayloadV1::from_block_unchecked(hash, &block); + + let mut test_harness = TestHarness::new(HOLESKY.clone()); + + let outcome = test_harness + .tree + .on_new_payload(ExecutionData { + payload: payload.into(), + sidecar: ExecutionPayloadSidecar::none(), + }) + .unwrap(); + assert!(outcome.outcome.is_syncing()); + + // ensure block is buffered + let buffered = test_harness.tree.state.buffer.block(&hash).unwrap(); + assert_eq!(buffered.clone_sealed_block(), sealed_clone); +} + +#[test] +fn test_disconnected_block() { + let s = include_str!("../../test-data/holesky/2.rlp"); + let data = Bytes::from_str(s).unwrap(); + let block = Block::decode(&mut data.as_ref()).unwrap(); + let sealed = block.seal_slow().try_recover().unwrap(); + + let mut test_harness = TestHarness::new(HOLESKY.clone()); + + let outcome = test_harness.tree.insert_block(sealed.clone()).unwrap(); + assert_eq!( + outcome, + InsertPayloadOk::Inserted(BlockStatus::Disconnected { + head: test_harness.tree.state.tree_state.current_canonical_head, + missing_ancestor: sealed.parent_num_hash() + }) + ); +} + +#[tokio::test] +async fn test_holesky_payload() { + let s = include_str!("../../test-data/holesky/1.rlp"); + let data = Bytes::from_str(s).unwrap(); + let block: Block = Block::decode(&mut data.as_ref()).unwrap(); + let sealed = block.seal_slow(); + let hash = sealed.hash(); + let block = sealed.into_block(); + let payload = ExecutionPayloadV1::from_block_unchecked(hash, &block); + + let mut test_harness = + TestHarness::new(HOLESKY.clone()).with_backfill_state(BackfillSyncState::Active); + + let (tx, rx) = oneshot::channel(); + test_harness + .tree + .on_engine_message(FromEngine::Request( + BeaconEngineMessage::NewPayload { + payload: ExecutionData { + payload: payload.clone().into(), + sidecar: ExecutionPayloadSidecar::none(), + }, + tx, + } + .into(), + )) + .unwrap(); + + let resp = rx.await.unwrap().unwrap(); + assert!(resp.is_syncing()); +} + +#[tokio::test] +async fn test_tree_state_on_new_head_reorg() { + reth_tracing::init_test_tracing(); + let chain_spec = MAINNET.clone(); + + // Set persistence_threshold to 1 + let mut test_harness = TestHarness::new(chain_spec); + test_harness.tree.config = + test_harness.tree.config.with_persistence_threshold(1).with_memory_block_buffer_target(1); + let mut test_block_builder = TestBlockBuilder::eth(); + let blocks: Vec<_> = test_block_builder.get_executed_blocks(1..6).collect(); + + for block in &blocks { + test_harness.tree.state.tree_state.insert_executed(block.clone()); + } + + // set block 3 as the current canonical head + test_harness.tree.state.tree_state.set_canonical_head(blocks[2].recovered_block().num_hash()); + + // create a fork from block 2 + let fork_block_3 = + test_block_builder.get_executed_block_with_number(3, blocks[1].recovered_block().hash()); + let fork_block_4 = + test_block_builder.get_executed_block_with_number(4, fork_block_3.recovered_block().hash()); + let fork_block_5 = + test_block_builder.get_executed_block_with_number(5, fork_block_4.recovered_block().hash()); + + test_harness.tree.state.tree_state.insert_executed(fork_block_3.clone()); + test_harness.tree.state.tree_state.insert_executed(fork_block_4.clone()); + test_harness.tree.state.tree_state.insert_executed(fork_block_5.clone()); + + // normal (non-reorg) case + let result = test_harness.tree.on_new_head(blocks[4].recovered_block().hash()).unwrap(); + assert!(matches!(result, Some(NewCanonicalChain::Commit { .. }))); + if let Some(NewCanonicalChain::Commit { new }) = result { + assert_eq!(new.len(), 2); + assert_eq!(new[0].recovered_block().hash(), blocks[3].recovered_block().hash()); + assert_eq!(new[1].recovered_block().hash(), blocks[4].recovered_block().hash()); + } + + // should be a None persistence action before we advance persistence + let current_action = test_harness.tree.persistence_state.current_action(); + assert_eq!(current_action, None); + + // let's attempt to persist and check that it attempts to save blocks + // + // since in-memory block buffer target and persistence_threshold are both 1, this should + // save all but the current tip of the canonical chain (up to blocks[1]) + test_harness.tree.advance_persistence().unwrap(); + let current_action = test_harness.tree.persistence_state.current_action().cloned(); + assert_eq!( + current_action, + Some(CurrentPersistenceAction::SavingBlocks { + highest: blocks[1].recovered_block().num_hash() + }) + ); + + // get rid of the prev action + let received_action = test_harness.action_rx.recv().unwrap(); + let PersistenceAction::SaveBlocks(saved_blocks, sender) = received_action else { + panic!("received wrong action"); + }; + assert_eq!(saved_blocks, vec![blocks[0].clone(), blocks[1].clone()]); + + // send the response so we can advance again + sender.send(Some(blocks[1].recovered_block().num_hash())).unwrap(); + + // we should be persisting blocks[1] because we threw out the prev action + let current_action = test_harness.tree.persistence_state.current_action().cloned(); + assert_eq!( + current_action, + Some(CurrentPersistenceAction::SavingBlocks { + highest: blocks[1].recovered_block().num_hash() + }) + ); + + // after advancing persistence, we should be at `None` for the next action + test_harness.tree.advance_persistence().unwrap(); + let current_action = test_harness.tree.persistence_state.current_action().cloned(); + assert_eq!(current_action, None); + + // reorg case + let result = test_harness.tree.on_new_head(fork_block_5.recovered_block().hash()).unwrap(); + assert!(matches!(result, Some(NewCanonicalChain::Reorg { .. }))); + + if let Some(NewCanonicalChain::Reorg { new, old }) = result { + assert_eq!(new.len(), 3); + assert_eq!(new[0].recovered_block().hash(), fork_block_3.recovered_block().hash()); + assert_eq!(new[1].recovered_block().hash(), fork_block_4.recovered_block().hash()); + assert_eq!(new[2].recovered_block().hash(), fork_block_5.recovered_block().hash()); + + assert_eq!(old.len(), 1); + assert_eq!(old[0].recovered_block().hash(), blocks[2].recovered_block().hash()); + } + + // The canonical block has not changed, so we will not get any active persistence action + test_harness.tree.advance_persistence().unwrap(); + let current_action = test_harness.tree.persistence_state.current_action().cloned(); + assert_eq!(current_action, None); + + // Let's change the canonical head and advance persistence + test_harness + .tree + .state + .tree_state + .set_canonical_head(fork_block_5.recovered_block().num_hash()); + + // The canonical block has changed now, we should get fork_block_4 due to the persistence + // threshold and in memory block buffer target + test_harness.tree.advance_persistence().unwrap(); + let current_action = test_harness.tree.persistence_state.current_action().cloned(); + assert_eq!( + current_action, + Some(CurrentPersistenceAction::SavingBlocks { + highest: fork_block_4.recovered_block().num_hash() + }) + ); +} + +#[test] +fn test_tree_state_on_new_head_deep_fork() { + reth_tracing::init_test_tracing(); + + let chain_spec = MAINNET.clone(); + let mut test_harness = TestHarness::new(chain_spec); + let mut test_block_builder = TestBlockBuilder::eth(); + + let blocks: Vec<_> = test_block_builder.get_executed_blocks(0..5).collect(); + + for block in &blocks { + test_harness.tree.state.tree_state.insert_executed(block.clone()); + } + + // set last block as the current canonical head + let last_block = blocks.last().unwrap().recovered_block().clone(); + + test_harness.tree.state.tree_state.set_canonical_head(last_block.num_hash()); + + // create a fork chain from last_block + let chain_a = test_block_builder.create_fork(&last_block, 10); + let chain_b = test_block_builder.create_fork(&last_block, 10); + + for block in &chain_a { + test_harness.tree.state.tree_state.insert_executed(ExecutedBlock { + recovered_block: Arc::new(block.clone()), + execution_output: Arc::new(ExecutionOutcome::default()), + hashed_state: Arc::new(HashedPostState::default()), + trie_updates: Arc::new(TrieUpdates::default()), + }); + } + test_harness.tree.state.tree_state.set_canonical_head(chain_a.last().unwrap().num_hash()); + + for block in &chain_b { + test_harness.tree.state.tree_state.insert_executed(ExecutedBlock { + recovered_block: Arc::new(block.clone()), + execution_output: Arc::new(ExecutionOutcome::default()), + hashed_state: Arc::new(HashedPostState::default()), + trie_updates: Arc::new(TrieUpdates::default()), + }); + } + + // for each block in chain_b, reorg to it and then back to canonical + let mut expected_new = Vec::new(); + for block in &chain_b { + // reorg to chain from block b + let result = test_harness.tree.on_new_head(block.hash()).unwrap(); + assert_matches!(result, Some(NewCanonicalChain::Reorg { .. })); + + expected_new.push(block); + if let Some(NewCanonicalChain::Reorg { new, old }) = result { + assert_eq!(new.len(), expected_new.len()); + for (index, block) in expected_new.iter().enumerate() { + assert_eq!(new[index].recovered_block().hash(), block.hash()); + } + + assert_eq!(old.len(), chain_a.len()); + for (index, block) in chain_a.iter().enumerate() { + assert_eq!(old[index].recovered_block().hash(), block.hash()); + } + } + + // set last block of chain a as canonical head + test_harness.tree.on_new_head(chain_a.last().unwrap().hash()).unwrap(); + } +} + +#[tokio::test] +async fn test_get_canonical_blocks_to_persist() { + let chain_spec = MAINNET.clone(); + let mut test_harness = TestHarness::new(chain_spec); + let mut test_block_builder = TestBlockBuilder::eth(); + + let canonical_head_number = 9; + let blocks: Vec<_> = + test_block_builder.get_executed_blocks(0..canonical_head_number + 1).collect(); + test_harness = test_harness.with_blocks(blocks.clone()); + + let last_persisted_block_number = 3; + test_harness.tree.persistence_state.last_persisted_block = + blocks[last_persisted_block_number as usize].recovered_block.num_hash(); + + let persistence_threshold = 4; + let memory_block_buffer_target = 3; + test_harness.tree.config = TreeConfig::default() + .with_persistence_threshold(persistence_threshold) + .with_memory_block_buffer_target(memory_block_buffer_target); + + let blocks_to_persist = test_harness.tree.get_canonical_blocks_to_persist().unwrap(); + + let expected_blocks_to_persist_length: usize = + (canonical_head_number - memory_block_buffer_target - last_persisted_block_number) + .try_into() + .unwrap(); + + assert_eq!(blocks_to_persist.len(), expected_blocks_to_persist_length); + for (i, item) in blocks_to_persist.iter().enumerate().take(expected_blocks_to_persist_length) { + assert_eq!(item.recovered_block().number, last_persisted_block_number + i as u64 + 1); + } + + // make sure only canonical blocks are included + let fork_block = test_block_builder.get_executed_block_with_number(4, B256::random()); + let fork_block_hash = fork_block.recovered_block().hash(); + test_harness.tree.state.tree_state.insert_executed(fork_block); + + assert!(test_harness.tree.state.tree_state.sealed_header_by_hash(&fork_block_hash).is_some()); + + let blocks_to_persist = test_harness.tree.get_canonical_blocks_to_persist().unwrap(); + assert_eq!(blocks_to_persist.len(), expected_blocks_to_persist_length); + + // check that the fork block is not included in the blocks to persist + assert!(!blocks_to_persist.iter().any(|b| b.recovered_block().hash() == fork_block_hash)); + + // check that the original block 4 is still included + assert!(blocks_to_persist.iter().any(|b| b.recovered_block().number == 4 && + b.recovered_block().hash() == blocks[4].recovered_block().hash())); + + // check that if we advance persistence, the persistence action is the correct value + test_harness.tree.advance_persistence().expect("advancing persistence should succeed"); + assert_eq!( + test_harness.tree.persistence_state.current_action().cloned(), + Some(CurrentPersistenceAction::SavingBlocks { + highest: blocks_to_persist.last().unwrap().recovered_block().num_hash() + }) + ); +} + +#[tokio::test] +async fn test_engine_tree_fcu_missing_head() { + let chain_spec = MAINNET.clone(); + let mut test_harness = TestHarness::new(chain_spec.clone()); + + let mut test_block_builder = TestBlockBuilder::eth().with_chain_spec((*chain_spec).clone()); + + let blocks: Vec<_> = test_block_builder.get_executed_blocks(0..5).collect(); + test_harness = test_harness.with_blocks(blocks); + + let missing_block = test_block_builder + .generate_random_block(6, test_harness.blocks.last().unwrap().recovered_block().hash()); + + test_harness.fcu_to(missing_block.hash(), PayloadStatusEnum::Syncing).await; + + // after FCU we receive an EngineApiEvent::Download event to get the missing block. + let event = test_harness.from_tree_rx.recv().await.unwrap(); + match event { + EngineApiEvent::Download(DownloadRequest::BlockSet(actual_block_set)) => { + let expected_block_set = HashSet::from_iter([missing_block.hash()]); + assert_eq!(actual_block_set, expected_block_set); + } + _ => panic!("Unexpected event: {event:#?}"), + } +} + +#[tokio::test] +async fn test_engine_tree_live_sync_transition_required_blocks_requested() { + reth_tracing::init_test_tracing(); + + let chain_spec = MAINNET.clone(); + let mut test_harness = TestHarness::new(chain_spec.clone()); + + let base_chain: Vec<_> = test_harness.block_builder.get_executed_blocks(0..1).collect(); + test_harness = test_harness.with_blocks(base_chain.clone()); + + test_harness + .fcu_to(base_chain.last().unwrap().recovered_block().hash(), ForkchoiceStatus::Valid) + .await; + + // extend main chain with enough blocks to trigger pipeline run but don't insert them + let main_chain = test_harness + .block_builder + .create_fork(base_chain[0].recovered_block(), MIN_BLOCKS_FOR_PIPELINE_RUN + 10); + + let main_chain_last_hash = main_chain.last().unwrap().hash(); + test_harness.send_fcu(main_chain_last_hash, ForkchoiceStatus::Syncing).await; + + test_harness.check_fcu(main_chain_last_hash, ForkchoiceStatus::Syncing).await; + + // create event for backfill finished + let backfill_finished_block_number = MIN_BLOCKS_FOR_PIPELINE_RUN + 1; + let backfill_finished = FromOrchestrator::BackfillSyncFinished(ControlFlow::Continue { + block_number: backfill_finished_block_number, + }); + + let backfill_tip_block = main_chain[(backfill_finished_block_number - 1) as usize].clone(); + // add block to mock provider to enable persistence clean up. + test_harness.provider.add_block(backfill_tip_block.hash(), backfill_tip_block.into_block()); + test_harness.tree.on_engine_message(FromEngine::Event(backfill_finished)).unwrap(); + + let event = test_harness.from_tree_rx.recv().await.unwrap(); + match event { + EngineApiEvent::Download(DownloadRequest::BlockSet(hash_set)) => { + assert_eq!(hash_set, HashSet::from_iter([main_chain_last_hash])); + } + _ => panic!("Unexpected event: {event:#?}"), + } + + test_harness + .tree + .on_engine_message(FromEngine::DownloadedBlocks(vec![main_chain.last().unwrap().clone()])) + .unwrap(); + + let event = test_harness.from_tree_rx.recv().await.unwrap(); + match event { + EngineApiEvent::Download(DownloadRequest::BlockRange(initial_hash, total_blocks)) => { + assert_eq!( + total_blocks, + (main_chain.len() - backfill_finished_block_number as usize - 1) as u64 + ); + assert_eq!(initial_hash, main_chain.last().unwrap().parent_hash); + } + _ => panic!("Unexpected event: {event:#?}"), + } +} + +#[tokio::test] +async fn test_fcu_with_canonical_ancestor_updates_latest_block() { + // Test for issue where FCU with canonical ancestor doesn't update Latest block state + // This was causing "nonce too low" errors when discard_reorged_transactions is enabled + + reth_tracing::init_test_tracing(); + let chain_spec = MAINNET.clone(); + + // Create test harness + let mut test_harness = TestHarness::new(chain_spec.clone()); + + // Set engine kind to OpStack and enable unwind_canonical_header to ensure the fix is triggered + test_harness.tree.engine_kind = EngineApiKind::OpStack; + test_harness.tree.config = test_harness.tree.config.clone().with_unwind_canonical_header(true); + let mut test_block_builder = TestBlockBuilder::eth().with_chain_spec((*chain_spec).clone()); + + // Create a chain of blocks + let blocks: Vec<_> = test_block_builder.get_executed_blocks(1..5).collect(); + test_harness = test_harness.with_blocks(blocks.clone()); + + // Set block 4 as the current canonical head + let current_head = blocks[3].recovered_block().clone(); // Block 4 (0-indexed as blocks[3]) + let current_head_sealed = current_head.clone_sealed_header(); + test_harness.tree.state.tree_state.set_canonical_head(current_head.num_hash()); + test_harness.tree.canonical_in_memory_state.set_canonical_head(current_head_sealed); + + // Verify the current head is set correctly + assert_eq!(test_harness.tree.state.tree_state.canonical_block_number(), current_head.number()); + assert_eq!(test_harness.tree.state.tree_state.canonical_block_hash(), current_head.hash()); + + // Now perform FCU to a canonical ancestor (block 2) + let ancestor_block = blocks[1].recovered_block().clone(); // Block 2 (0-indexed as blocks[1]) + + // Send FCU to the canonical ancestor + let (tx, rx) = oneshot::channel(); + test_harness + .tree + .on_engine_message(FromEngine::Request( + BeaconEngineMessage::ForkchoiceUpdated { + state: ForkchoiceState { + head_block_hash: ancestor_block.hash(), + safe_block_hash: B256::ZERO, + finalized_block_hash: B256::ZERO, + }, + payload_attrs: None, + tx, + version: EngineApiMessageVersion::default(), + } + .into(), + )) + .unwrap(); + + // Verify FCU succeeds + let response = rx.await.unwrap().unwrap().await.unwrap(); + assert!(response.payload_status.is_valid()); + + // The critical test: verify that Latest block has been updated to the canonical ancestor + // Check tree state + assert_eq!( + test_harness.tree.state.tree_state.canonical_block_number(), + ancestor_block.number(), + "Tree state: Latest block number should be updated to canonical ancestor" + ); + assert_eq!( + test_harness.tree.state.tree_state.canonical_block_hash(), + ancestor_block.hash(), + "Tree state: Latest block hash should be updated to canonical ancestor" + ); + + // Also verify canonical in-memory state is synchronized + assert_eq!( + test_harness.tree.canonical_in_memory_state.get_canonical_head().number, + ancestor_block.number(), + "In-memory state: Latest block number should be updated to canonical ancestor" + ); + assert_eq!( + test_harness.tree.canonical_in_memory_state.get_canonical_head().hash(), + ancestor_block.hash(), + "In-memory state: Latest block hash should be updated to canonical ancestor" + ); +} + +/// Test that verifies the happy path where a new payload extends the canonical chain +#[test] +fn test_on_new_payload_canonical_insertion() { + reth_tracing::init_test_tracing(); + + // Use test data similar to test_disconnected_payload + let s = include_str!("../../test-data/holesky/1.rlp"); + let data = Bytes::from_str(s).unwrap(); + let block1 = Block::decode(&mut data.as_ref()).unwrap(); + let sealed1 = block1.seal_slow(); + let hash1 = sealed1.hash(); + let sealed1_clone = sealed1.clone(); + let block1 = sealed1.into_block(); + let payload1 = ExecutionPayloadV1::from_block_unchecked(hash1, &block1); + + let mut test_harness = TestHarness::new(HOLESKY.clone()); + + // Case 1: Submit payload when NOT sync target head - should be syncing (disconnected) + let outcome1 = test_harness + .tree + .on_new_payload(ExecutionData { + payload: payload1.into(), + sidecar: ExecutionPayloadSidecar::none(), + }) + .unwrap(); + + // Since this is disconnected from genesis, it should be syncing + assert!(outcome1.outcome.is_syncing(), "Disconnected payload should be syncing"); + + // Verify no canonicalization event + assert!(outcome1.event.is_none(), "Should not trigger canonicalization when syncing"); + + // Ensure block is buffered (like test_disconnected_payload) + let buffered = test_harness.tree.state.buffer.block(&hash1).unwrap(); + assert_eq!(buffered.clone_sealed_block(), sealed1_clone, "Block should be buffered"); +} + +/// Test that ensures payloads are rejected when linking to a known-invalid ancestor +#[test] +fn test_on_new_payload_invalid_ancestor() { + reth_tracing::init_test_tracing(); + + // Use Holesky test data + let mut test_harness = TestHarness::new(HOLESKY.clone()); + + // Read block 1 from test data + let s1 = include_str!("../../test-data/holesky/1.rlp"); + let data1 = Bytes::from_str(s1).unwrap(); + let block1 = Block::decode(&mut data1.as_ref()).unwrap(); + let sealed1 = block1.seal_slow(); + let hash1 = sealed1.hash(); + let parent1 = sealed1.parent_hash(); + + // Mark block 1 as invalid + test_harness + .tree + .state + .invalid_headers + .insert(BlockWithParent { block: sealed1.num_hash(), parent: parent1 }); + + // Read block 2 which has block 1 as parent + let s2 = include_str!("../../test-data/holesky/2.rlp"); + let data2 = Bytes::from_str(s2).unwrap(); + let block2 = Block::decode(&mut data2.as_ref()).unwrap(); + let sealed2 = block2.seal_slow(); + let hash2 = sealed2.hash(); + + // Verify block2's parent is block1 + assert_eq!(sealed2.parent_hash(), hash1, "Block 2 should have block 1 as parent"); + + let payload2 = ExecutionPayloadV1::from_block_unchecked(hash2, &sealed2.into_block()); + + // Submit payload 2 (child of invalid block 1) + let outcome = test_harness + .tree + .on_new_payload(ExecutionData { + payload: payload2.into(), + sidecar: ExecutionPayloadSidecar::none(), + }) + .unwrap(); + + // Verify response is INVALID + assert!( + outcome.outcome.is_invalid(), + "Payload should be invalid when parent is marked invalid" + ); + + // For invalid ancestors, the latest_valid_hash behavior varies + // We just verify it's marked as invalid + assert!( + outcome.outcome.latest_valid_hash.is_some() || outcome.outcome.latest_valid_hash.is_none(), + "Latest valid hash should be set appropriately for invalid ancestor" + ); + + // Verify block 2 is now also marked as invalid + assert!( + test_harness.tree.state.invalid_headers.get(&hash2).is_some(), + "Block should be added to invalid headers when parent is invalid" + ); +} + +/// Test that confirms payloads received during backfill sync are buffered and reported as syncing +#[test] +fn test_on_new_payload_backfill_buffering() { + reth_tracing::init_test_tracing(); + + // Use a test data file similar to test_holesky_payload + let s = include_str!("../../test-data/holesky/1.rlp"); + let data = Bytes::from_str(s).unwrap(); + let block = Block::decode(&mut data.as_ref()).unwrap(); + let sealed = block.seal_slow(); + let hash = sealed.hash(); + let block = sealed.clone().into_block(); + let payload = ExecutionPayloadV1::from_block_unchecked(hash, &block); + + // Initialize test harness with backfill sync active + let mut test_harness = + TestHarness::new(HOLESKY.clone()).with_backfill_state(BackfillSyncState::Active); + + // Submit payload during backfill + let outcome = test_harness + .tree + .on_new_payload(ExecutionData { + payload: payload.into(), + sidecar: ExecutionPayloadSidecar::none(), + }) + .unwrap(); + + // Verify response is SYNCING + assert!(outcome.outcome.is_syncing(), "Payload should be syncing during backfill"); + + // Verify the block is present in the buffer + let hash = sealed.hash(); + let buffered_block = test_harness + .tree + .state + .buffer + .block(&hash) + .expect("Block should be buffered during backfill sync"); + + // Verify the buffered block matches what we submitted + assert_eq!( + buffered_block.clone_sealed_block(), + sealed, + "Buffered block should match submitted payload" + ); +} + +/// Test that captures the Engine-API rule where malformed payloads report latestValidHash = None +#[test] +fn test_on_new_payload_malformed_payload() { + reth_tracing::init_test_tracing(); + + let mut test_harness = TestHarness::new(HOLESKY.clone()); + + // Use test data + let s = include_str!("../../test-data/holesky/1.rlp"); + let data = Bytes::from_str(s).unwrap(); + let block = Block::decode(&mut data.as_ref()).unwrap(); + let sealed = block.seal_slow(); + + // Create a payload with incorrect block hash to trigger malformed validation + let mut payload = ExecutionPayloadV1::from_block_unchecked(sealed.hash(), &sealed.into_block()); + + // Corrupt the block hash - this makes the computed hash not match the provided hash + // This will cause ensure_well_formed_payload to fail + let wrong_hash = B256::random(); + payload.block_hash = wrong_hash; + + // Submit the malformed payload + let outcome = test_harness + .tree + .on_new_payload(ExecutionData { + payload: payload.into(), + sidecar: ExecutionPayloadSidecar::none(), + }) + .unwrap(); + + // For malformed payloads with incorrect hash, the current implementation + // returns SYNCING since it doesn't match computed hash + // This test captures the current behavior to prevent regression + assert!( + outcome.outcome.is_syncing() || outcome.outcome.is_invalid(), + "Malformed payload should be either syncing or invalid" + ); + + // If invalid, latestValidHash should be None per Engine API spec + if outcome.outcome.is_invalid() { + assert_eq!( + outcome.outcome.latest_valid_hash, None, + "Malformed payload must have latestValidHash = None when invalid" + ); + } +} + +/// Test different `StateRootStrategy` paths: `StateRootTask` with empty/non-empty prefix sets, +/// `Parallel`, `Synchronous` +#[test] +fn test_state_root_strategy_paths() { + reth_tracing::init_test_tracing(); + + let mut test_harness = TestHarness::new(MAINNET.clone()); + + // Test multiple scenarios to ensure different StateRootStrategy paths are taken: + // 1. `StateRootTask` with empty prefix_sets → uses payload_processor.spawn() + // 2. `StateRootTask` with non-empty prefix_sets → switches to `Parallel`, uses + // spawn_cache_exclusive() + // 3. `Parallel` strategy → uses spawn_cache_exclusive() + // 4. `Synchronous` strategy → uses spawn_cache_exclusive() + + let s1 = include_str!("../../test-data/holesky/1.rlp"); + let data1 = Bytes::from_str(s1).unwrap(); + let block1 = Block::decode(&mut data1.as_ref()).unwrap(); + let sealed1 = block1.seal_slow(); + let hash1 = sealed1.hash(); + let block1 = sealed1.into_block(); + let payload1 = ExecutionPayloadV1::from_block_unchecked(hash1, &block1); + + // Scenario 1: Test one strategy path + let outcome1 = test_harness + .tree + .on_new_payload(ExecutionData { + payload: payload1.into(), + sidecar: ExecutionPayloadSidecar::none(), + }) + .unwrap(); + + assert!( + outcome1.outcome.is_valid() || outcome1.outcome.is_syncing(), + "First strategy path should work" + ); + + let s2 = include_str!("../../test-data/holesky/2.rlp"); + let data2 = Bytes::from_str(s2).unwrap(); + let block2 = Block::decode(&mut data2.as_ref()).unwrap(); + let sealed2 = block2.seal_slow(); + let hash2 = sealed2.hash(); + let block2 = sealed2.into_block(); + let payload2 = ExecutionPayloadV1::from_block_unchecked(hash2, &block2); + + // Scenario 2: Test different strategy path (disconnected) + let outcome2 = test_harness + .tree + .on_new_payload(ExecutionData { + payload: payload2.into(), + sidecar: ExecutionPayloadSidecar::none(), + }) + .unwrap(); + + assert!(outcome2.outcome.is_syncing(), "Second strategy path should work"); + + // This test passes if multiple StateRootStrategy scenarios work correctly, + // confirming that passing arguments directly doesn't break: + // - `StateRootTask` strategy with empty/non-empty prefix_sets + // - Dynamic strategy switching (StateRootTask → Parallel) + // - Parallel and Synchronous strategy paths + // - All parameter passing through the args struct +} + +// ================================================================================================ +// VALIDATE_BLOCK_WITH_STATE TEST SUITE +// ================================================================================================ +// +// This test suite exercises `validate_block_with_state` across different scenarios including: +// - Basic block validation with state root computation +// - Strategy selection based on conditions (`StateRootTask`, `Parallel`, `Synchronous`) +// - Trie update retention and discard logic +// - Error precedence handling (consensus vs execution errors) +// - Different validation scenarios (valid, invalid consensus, invalid execution blocks) + +/// Test `Synchronous` strategy when persistence is active +#[test] +fn test_validate_block_synchronous_strategy_during_persistence() { + reth_tracing::init_test_tracing(); + + let mut test_harness = ValidatorTestHarness::new(MAINNET.clone()); + + // Set up persistence action to force `Synchronous` strategy + use crate::tree::persistence_state::CurrentPersistenceAction; + let persistence_action = CurrentPersistenceAction::SavingBlocks { + highest: alloy_eips::NumHash::new(1, B256::random()), + }; + test_harness.start_persistence_operation(persistence_action); + + // Verify persistence is active + assert!(test_harness.is_persistence_in_progress()); + + // Create valid block + let mut block_factory = TestBlockFactory::new(MAINNET.as_ref().clone()); + let genesis_hash = MAINNET.genesis_hash(); + let valid_block = block_factory.create_valid_block(genesis_hash); + + // Test that Synchronous strategy executes during active persistence without panicking + let _result = test_harness.validate_block_direct(valid_block); +} + +/// Test multiple validation scenarios including valid, consensus-invalid, and execution-invalid +/// blocks with proper result validation +#[test] +fn test_validate_block_multiple_scenarios() { + reth_tracing::init_test_tracing(); + + // Test multiple scenarios to ensure comprehensive coverage + let mut test_harness = ValidatorTestHarness::new(MAINNET.clone()); + let mut block_factory = TestBlockFactory::new(MAINNET.as_ref().clone()); + let genesis_hash = MAINNET.genesis_hash(); + + // Scenario 1: Valid block validation (test execution, not result) + let valid_block = block_factory.create_valid_block(genesis_hash); + let _result1 = test_harness.validate_block_direct(valid_block); + + // Scenario 2: Block with consensus issues should be rejected + let consensus_invalid = block_factory.create_invalid_consensus_block(genesis_hash); + let result2 = test_harness.validate_block_direct(consensus_invalid); + assert!(result2.is_err(), "Consensus-invalid block (invalid state root) should be rejected"); + + // Scenario 3: Block with execution issues should be rejected + let execution_invalid = block_factory.create_invalid_execution_block(genesis_hash); + let result3 = test_harness.validate_block_direct(execution_invalid); + assert!(result3.is_err(), "Execution-invalid block (gas limit exceeded) should be rejected"); + + // Verify all validation scenarios executed without panics + let total_calls = test_harness.validation_call_count(); + assert!( + total_calls >= 2, + "At least invalid block validations should have executed (got {})", + total_calls + ); +} + +/// Test suite for the `check_invalid_ancestors` method +#[cfg(test)] +mod check_invalid_ancestors_tests { + use super::*; + + /// Test that `find_invalid_ancestor` returns None when no invalid ancestors exist + #[test] + fn test_find_invalid_ancestor_no_invalid() { + reth_tracing::init_test_tracing(); + + let mut test_harness = TestHarness::new(HOLESKY.clone()); + + // Create a valid block payload + let s = include_str!("../../test-data/holesky/1.rlp"); + let data = Bytes::from_str(s).unwrap(); + let block = Block::decode(&mut data.as_ref()).unwrap(); + let sealed = block.seal_slow(); + let payload = ExecutionData { + payload: ExecutionPayloadV1::from_block_unchecked(sealed.hash(), &sealed.into_block()) + .into(), + sidecar: ExecutionPayloadSidecar::none(), + }; + + // Check for invalid ancestors - should return None since none are marked invalid + let result = test_harness.tree.find_invalid_ancestor(&payload); + assert!(result.is_none(), "Should return None when no invalid ancestors exist"); + } + + /// Test that `find_invalid_ancestor` detects an invalid parent + #[test] + fn test_find_invalid_ancestor_with_invalid_parent() { + reth_tracing::init_test_tracing(); + + let mut test_harness = TestHarness::new(HOLESKY.clone()); + + // Read block 1 + let s1 = include_str!("../../test-data/holesky/1.rlp"); + let data1 = Bytes::from_str(s1).unwrap(); + let block1 = Block::decode(&mut data1.as_ref()).unwrap(); + let sealed1 = block1.seal_slow(); + let parent1 = sealed1.parent_hash(); + + // Mark block 1 as invalid + test_harness + .tree + .state + .invalid_headers + .insert(BlockWithParent { block: sealed1.num_hash(), parent: parent1 }); + + // Read block 2 which has block 1 as parent + let s2 = include_str!("../../test-data/holesky/2.rlp"); + let data2 = Bytes::from_str(s2).unwrap(); + let block2 = Block::decode(&mut data2.as_ref()).unwrap(); + let sealed2 = block2.seal_slow(); + + // Create payload for block 2 + let payload2 = ExecutionData { + payload: ExecutionPayloadV1::from_block_unchecked( + sealed2.hash(), + &sealed2.into_block(), + ) + .into(), + sidecar: ExecutionPayloadSidecar::none(), + }; + + // Check for invalid ancestors - should detect invalid parent + let invalid_ancestor = test_harness.tree.find_invalid_ancestor(&payload2); + assert!( + invalid_ancestor.is_some(), + "Should find invalid ancestor when parent is marked as invalid" + ); + + // Now test that handling the payload with invalid ancestor returns invalid status + let invalid = invalid_ancestor.unwrap(); + let status = test_harness.tree.handle_invalid_ancestor_payload(payload2, invalid).unwrap(); + assert!(status.is_invalid(), "Status should be invalid when parent is invalid"); + } + + /// Test genesis block handling (`parent_hash` = `B256::ZERO`) + #[test] + fn test_genesis_block_handling() { + reth_tracing::init_test_tracing(); + + let mut test_harness = TestHarness::new(HOLESKY.clone()); + + // Create a genesis-like payload with parent_hash = B256::ZERO + let mut test_block_builder = TestBlockBuilder::eth(); + let genesis_block = test_block_builder.generate_random_block(0, B256::ZERO); + let (sealed_genesis, _) = genesis_block.split_sealed(); + let genesis_payload = ExecutionData { + payload: ExecutionPayloadV1::from_block_unchecked( + sealed_genesis.hash(), + &sealed_genesis.into_block(), + ) + .into(), + sidecar: ExecutionPayloadSidecar::none(), + }; + + // Check for invalid ancestors - should return None for genesis block + let result = test_harness.tree.find_invalid_ancestor(&genesis_payload); + assert!(result.is_none(), "Genesis block should have no invalid ancestors"); + } + + /// Test malformed payload with invalid ancestor scenario + #[test] + fn test_malformed_payload_with_invalid_ancestor() { + reth_tracing::init_test_tracing(); + + let mut test_harness = TestHarness::new(HOLESKY.clone()); + + // Mark an ancestor as invalid + let invalid_block = Block::default().seal_slow(); + test_harness.tree.state.invalid_headers.insert(BlockWithParent { + block: invalid_block.num_hash(), + parent: invalid_block.parent_hash(), + }); + + // Create a payload that descends from the invalid ancestor but is malformed + let malformed_payload = create_malformed_payload_descending_from(invalid_block.hash()); + + // The function should handle the malformed payload gracefully + let invalid_ancestor = test_harness.tree.find_invalid_ancestor(&malformed_payload); + if let Some(invalid) = invalid_ancestor { + let status = test_harness + .tree + .handle_invalid_ancestor_payload(malformed_payload, invalid) + .unwrap(); + assert!( + status.is_invalid(), + "Should return invalid status for malformed payload with invalid ancestor" + ); + } + } + + /// Helper function to create a malformed payload that descends from a given parent + fn create_malformed_payload_descending_from(parent_hash: B256) -> ExecutionData { + // Create a block with invalid hash (mismatch between computed and provided hash) + let mut test_block_builder = TestBlockBuilder::eth(); + let block = test_block_builder.generate_random_block(1, parent_hash); + + // Intentionally corrupt the block to make it malformed + // Modify the block after creation to make validation fail + let (sealed_block, _senders) = block.split_sealed(); + let unsealed_block = sealed_block.unseal(); + + // Create payload with wrong hash (this makes it malformed) + let wrong_hash = B256::from([0xff; 32]); + + ExecutionData { + payload: ExecutionPayloadV1::from_block_unchecked(wrong_hash, &unsealed_block).into(), + sidecar: ExecutionPayloadSidecar::none(), + } + } +} + +/// Test suite for `try_insert_payload` and `try_buffer_payload` +/// methods +#[cfg(test)] +mod payload_execution_tests { + use super::*; + + /// Test `try_insert_payload` with different `InsertPayloadOk` variants + #[test] + fn test_try_insert_payload_variants() { + reth_tracing::init_test_tracing(); + + let mut test_harness = TestHarness::new(HOLESKY.clone()); + + // Create a valid payload + let mut test_block_builder = TestBlockBuilder::eth(); + let block = test_block_builder.generate_random_block(1, B256::ZERO); + let (sealed_block, _) = block.split_sealed(); + let payload = ExecutionData { + payload: ExecutionPayloadV1::from_block_unchecked( + sealed_block.hash(), + &sealed_block.into_block(), + ) + .into(), + sidecar: ExecutionPayloadSidecar::none(), + }; + + // Test the function directly + let result = test_harness.tree.try_insert_payload(payload); + // Should handle the payload gracefully + assert!(result.is_ok(), "Should handle valid payload without error"); + } + + /// Test `try_buffer_payload` with validation errors + #[test] + fn test_buffer_payload_validation_errors() { + reth_tracing::init_test_tracing(); + + let mut test_harness = TestHarness::new(HOLESKY.clone()); + + // Create a malformed payload that will fail validation + let malformed_payload = create_malformed_payload(); + + // Test buffering during backfill sync + let result = test_harness.tree.try_buffer_payload(malformed_payload); + assert!(result.is_ok(), "Should handle malformed payload gracefully"); + let status = result.unwrap(); + assert!( + status.is_invalid() || status.is_syncing(), + "Should return invalid or syncing status for malformed payload" + ); + } + + /// Test `try_buffer_payload` with valid payload + #[test] + fn test_buffer_payload_valid_payload() { + reth_tracing::init_test_tracing(); + + let mut test_harness = TestHarness::new(HOLESKY.clone()); + + // Create a valid payload + let mut test_block_builder = TestBlockBuilder::eth(); + let block = test_block_builder.generate_random_block(1, B256::ZERO); + let (sealed_block, _) = block.split_sealed(); + let payload = ExecutionData { + payload: ExecutionPayloadV1::from_block_unchecked( + sealed_block.hash(), + &sealed_block.into_block(), + ) + .into(), + sidecar: ExecutionPayloadSidecar::none(), + }; + + // Test buffering during backfill sync + let result = test_harness.tree.try_buffer_payload(payload); + assert!(result.is_ok(), "Should handle valid payload gracefully"); + let status = result.unwrap(); + // The payload may be invalid due to missing withdrawals root, so accept either status + assert!( + status.is_syncing() || status.is_invalid(), + "Should return syncing or invalid status for payload" + ); + } + + /// Helper function to create a malformed payload + fn create_malformed_payload() -> ExecutionData { + // Create a payload with invalid structure that will fail validation + let mut test_block_builder = TestBlockBuilder::eth(); + let block = test_block_builder.generate_random_block(1, B256::ZERO); + + // Modify the block to make it malformed + let (sealed_block, _senders) = block.split_sealed(); + let mut unsealed_block = sealed_block.unseal(); + + // Corrupt the block by setting an invalid gas limit + unsealed_block.header.gas_limit = 0; + + ExecutionData { + payload: ExecutionPayloadV1::from_block_unchecked( + unsealed_block.hash_slow(), + &unsealed_block, + ) + .into(), + sidecar: ExecutionPayloadSidecar::none(), + } + } +} + +/// Test suite for the refactored `on_forkchoice_updated` helper methods +#[cfg(test)] +mod forkchoice_updated_tests { + use super::*; + use alloy_primitives::Address; + + /// Test that validates the forkchoice state pre-validation logic + #[tokio::test] + async fn test_validate_forkchoice_state() { + let chain_spec = MAINNET.clone(); + let mut test_harness = TestHarness::new(chain_spec); + + // Test 1: Zero head block hash should return early with invalid state + let zero_state = ForkchoiceState { + head_block_hash: B256::ZERO, + safe_block_hash: B256::ZERO, + finalized_block_hash: B256::ZERO, + }; + + let result = test_harness.tree.validate_forkchoice_state(zero_state).unwrap(); + assert!(result.is_some(), "Zero head block hash should return early"); + let outcome = result.unwrap(); + // For invalid state, we expect an error response + assert!(matches!(outcome, OnForkChoiceUpdated { .. })); + + // Test 2: Valid state with backfill active should return syncing + test_harness.tree.backfill_sync_state = BackfillSyncState::Active; + let valid_state = ForkchoiceState { + head_block_hash: B256::random(), + safe_block_hash: B256::ZERO, + finalized_block_hash: B256::ZERO, + }; + + let result = test_harness.tree.validate_forkchoice_state(valid_state).unwrap(); + assert!(result.is_some(), "Backfill active should return early"); + let outcome = result.unwrap(); + // We need to await the outcome to check the payload status + let fcu_result = outcome.await.unwrap(); + assert!(fcu_result.payload_status.is_syncing()); + + // Test 3: Valid state with idle backfill should continue processing + test_harness.tree.backfill_sync_state = BackfillSyncState::Idle; + let valid_state = ForkchoiceState { + head_block_hash: B256::random(), + safe_block_hash: B256::ZERO, + finalized_block_hash: B256::ZERO, + }; + + let result = test_harness.tree.validate_forkchoice_state(valid_state).unwrap(); + assert!(result.is_none(), "Valid state should continue processing"); + } + + /// Test that verifies canonical head handling + #[tokio::test] + async fn test_handle_canonical_head() { + let chain_spec = MAINNET.clone(); + let mut test_harness = TestHarness::new(chain_spec); + + // Create test blocks + let blocks: Vec<_> = test_harness.block_builder.get_executed_blocks(0..3).collect(); + test_harness = test_harness.with_blocks(blocks); + + let canonical_head = test_harness.tree.state.tree_state.canonical_block_hash(); + + // Test 1: Head is already canonical, no payload attributes + let state = ForkchoiceState { + head_block_hash: canonical_head, + safe_block_hash: B256::ZERO, + finalized_block_hash: B256::ZERO, + }; + + let result = test_harness + .tree + .handle_canonical_head(state, &None, EngineApiMessageVersion::default()) + .unwrap(); + assert!(result.is_some(), "Should return outcome for canonical head"); + let outcome = result.unwrap(); + let fcu_result = outcome.outcome.await.unwrap(); + assert!(fcu_result.payload_status.is_valid()); + + // Test 2: Head is not canonical - should return None to continue processing + let non_canonical_state = ForkchoiceState { + head_block_hash: B256::random(), + safe_block_hash: B256::ZERO, + finalized_block_hash: B256::ZERO, + }; + + let result = test_harness + .tree + .handle_canonical_head(non_canonical_state, &None, EngineApiMessageVersion::default()) + .unwrap(); + assert!(result.is_none(), "Non-canonical head should return None"); + } + + /// Test that verifies chain update application + #[tokio::test] + async fn test_apply_chain_update() { + let chain_spec = MAINNET.clone(); + let mut test_harness = TestHarness::new(chain_spec); + + // Create a chain of blocks + let blocks: Vec<_> = test_harness.block_builder.get_executed_blocks(0..5).collect(); + test_harness = test_harness.with_blocks(blocks.clone()); + + let new_head = blocks[2].recovered_block().hash(); + + // Test 1: Apply chain update to a new head + let state = ForkchoiceState { + head_block_hash: new_head, + safe_block_hash: B256::ZERO, + finalized_block_hash: B256::ZERO, + }; + + let result = test_harness + .tree + .apply_chain_update(state, &None, EngineApiMessageVersion::default()) + .unwrap(); + assert!(result.is_some(), "Should apply chain update for new head"); + let outcome = result.unwrap(); + let fcu_result = outcome.outcome.await.unwrap(); + assert!(fcu_result.payload_status.is_valid()); + + // Test 2: Try to apply chain update to missing block + let missing_state = ForkchoiceState { + head_block_hash: B256::random(), + safe_block_hash: B256::ZERO, + finalized_block_hash: B256::ZERO, + }; + + let result = test_harness + .tree + .apply_chain_update(missing_state, &None, EngineApiMessageVersion::default()) + .unwrap(); + assert!(result.is_none(), "Missing block should return None"); + } + + /// Test that verifies missing block handling + #[tokio::test] + async fn test_handle_missing_block() { + let chain_spec = MAINNET.clone(); + let test_harness = TestHarness::new(chain_spec); + + let state = ForkchoiceState { + head_block_hash: B256::random(), + safe_block_hash: B256::ZERO, + finalized_block_hash: B256::ZERO, + }; + + let result = test_harness.tree.handle_missing_block(state).unwrap(); + + // Should return syncing status with download event + let fcu_result = result.outcome.await.unwrap(); + assert!(fcu_result.payload_status.is_syncing()); + assert!(result.event.is_some()); + + if let Some(TreeEvent::Download(download_request)) = result.event { + match download_request { + DownloadRequest::BlockSet(block_set) => { + assert_eq!(block_set.len(), 1); + } + _ => panic!("Expected single block download request"), + } + } + } + + /// Test the complete `on_forkchoice_updated` flow with all helper methods + #[tokio::test] + async fn test_on_forkchoice_updated_integration() { + reth_tracing::init_test_tracing(); + + let chain_spec = MAINNET.clone(); + let mut test_harness = TestHarness::new(chain_spec); + + // Create test blocks + let blocks: Vec<_> = test_harness.block_builder.get_executed_blocks(0..3).collect(); + test_harness = test_harness.with_blocks(blocks.clone()); + + let canonical_head = test_harness.tree.state.tree_state.canonical_block_hash(); + + // Test Case 1: FCU to existing canonical head + let state = ForkchoiceState { + head_block_hash: canonical_head, + safe_block_hash: canonical_head, + finalized_block_hash: canonical_head, + }; + + let result = test_harness + .tree + .on_forkchoice_updated(state, None, EngineApiMessageVersion::default()) + .unwrap(); + let fcu_result = result.outcome.await.unwrap(); + assert!(fcu_result.payload_status.is_valid()); + + // Test Case 2: FCU to missing block + let missing_state = ForkchoiceState { + head_block_hash: B256::random(), + safe_block_hash: B256::ZERO, + finalized_block_hash: B256::ZERO, + }; + + let result = test_harness + .tree + .on_forkchoice_updated(missing_state, None, EngineApiMessageVersion::default()) + .unwrap(); + let fcu_result = result.outcome.await.unwrap(); + assert!(fcu_result.payload_status.is_syncing()); + assert!(result.event.is_some(), "Should trigger download event for missing block"); + + // Test Case 3: FCU during backfill sync + test_harness.tree.backfill_sync_state = BackfillSyncState::Active; + let state = ForkchoiceState { + head_block_hash: canonical_head, + safe_block_hash: B256::ZERO, + finalized_block_hash: B256::ZERO, + }; + + let result = test_harness + .tree + .on_forkchoice_updated(state, None, EngineApiMessageVersion::default()) + .unwrap(); + let fcu_result = result.outcome.await.unwrap(); + assert!(fcu_result.payload_status.is_syncing(), "Should return syncing during backfill"); + } + + /// Test metrics recording in forkchoice updated + #[tokio::test] + async fn test_record_forkchoice_metrics() { + let chain_spec = MAINNET.clone(); + let test_harness = TestHarness::new(chain_spec); + + // Get initial metrics state by checking if metrics are recorded + // We can't directly get counter values, but we can verify the methods are called + + // Test without attributes + let attrs_none = None; + test_harness.tree.record_forkchoice_metrics(&attrs_none); + + // Test with attributes + let attrs_some = Some(alloy_rpc_types_engine::PayloadAttributes { + timestamp: 1000, + prev_randao: B256::random(), + suggested_fee_recipient: Address::random(), + withdrawals: None, + parent_beacon_block_root: None, + }); + test_harness.tree.record_forkchoice_metrics(&attrs_some); + + // We can't directly verify counter values since they're private metrics + // But we can verify the methods don't panic and execute successfully + } + + /// Test edge case: FCU with invalid ancestor + #[tokio::test] + async fn test_fcu_with_invalid_ancestor() { + let chain_spec = MAINNET.clone(); + let mut test_harness = TestHarness::new(chain_spec); + + // Mark a block as invalid + let invalid_block_hash = B256::random(); + test_harness.tree.state.invalid_headers.insert(BlockWithParent { + block: NumHash::new(1, invalid_block_hash), + parent: B256::ZERO, + }); + + // Test FCU that points to a descendant of the invalid block + // This is a bit tricky to test directly, but we can verify the check_invalid_ancestor + // method + let result = test_harness.tree.check_invalid_ancestor(invalid_block_hash).unwrap(); + assert!(result.is_some(), "Should detect invalid ancestor"); + } + + /// Test `OpStack` specific behavior with canonical head + #[tokio::test] + async fn test_opstack_canonical_head_behavior() { + let chain_spec = MAINNET.clone(); + let mut test_harness = TestHarness::new(chain_spec); + + // Set engine kind to OpStack + test_harness.tree.engine_kind = EngineApiKind::OpStack; + + // Create test blocks + let blocks: Vec<_> = test_harness.block_builder.get_executed_blocks(0..3).collect(); + test_harness = test_harness.with_blocks(blocks); + + let canonical_head = test_harness.tree.state.tree_state.canonical_block_hash(); + + // For OpStack, even if head is already canonical, we should still process payload + // attributes + let state = ForkchoiceState { + head_block_hash: canonical_head, + safe_block_hash: B256::ZERO, + finalized_block_hash: B256::ZERO, + }; + + let result = test_harness + .tree + .handle_canonical_head(state, &None, EngineApiMessageVersion::default()) + .unwrap(); + assert!(result.is_some(), "OpStack should handle canonical head"); + } +} diff --git a/crates/engine/tree/src/tree/trie_updates.rs b/crates/engine/tree/src/tree/trie_updates.rs index 4f2e3c40eb1..ba8f7fc16a9 100644 --- a/crates/engine/tree/src/tree/trie_updates.rs +++ b/crates/engine/tree/src/tree/trie_updates.rs @@ -114,11 +114,11 @@ pub(super) fn compare_trie_updates( .account_nodes .keys() .chain(regular.account_nodes.keys()) - .cloned() + .copied() .collect::>() { let (task, regular) = (task.account_nodes.remove(&key), regular.account_nodes.remove(&key)); - let database = account_trie_cursor.seek_exact(key.clone())?.map(|x| x.1); + let database = account_trie_cursor.seek_exact(key)?.map(|x| x.1); if !branch_nodes_equal(task.as_ref(), regular.as_ref(), database.as_ref())? { diff.account_nodes.insert(key, EntryDiff { task, regular, database }); @@ -131,12 +131,12 @@ pub(super) fn compare_trie_updates( .removed_nodes .iter() .chain(regular.removed_nodes.iter()) - .cloned() + .copied() .collect::>() { let (task_removed, regular_removed) = (task.removed_nodes.contains(&key), regular.removed_nodes.contains(&key)); - let database_not_exists = account_trie_cursor.seek_exact(key.clone())?.is_none(); + let database_not_exists = account_trie_cursor.seek_exact(key)?.is_none(); // If the deletion is a no-op, meaning that the entry is not in the // database, do not add it to the diff. if task_removed != regular_removed && !database_not_exists { @@ -206,11 +206,11 @@ fn compare_storage_trie_updates( .storage_nodes .keys() .chain(regular.storage_nodes.keys()) - .cloned() + .copied() .collect::>() { let (task, regular) = (task.storage_nodes.remove(&key), regular.storage_nodes.remove(&key)); - let database = storage_trie_cursor.seek_exact(key.clone())?.map(|x| x.1); + let database = storage_trie_cursor.seek_exact(key)?.map(|x| x.1); if !branch_nodes_equal(task.as_ref(), regular.as_ref(), database.as_ref())? { diff.storage_nodes.insert(key, EntryDiff { task, regular, database }); } @@ -218,22 +218,20 @@ fn compare_storage_trie_updates( // compare removed nodes let mut storage_trie_cursor = trie_cursor()?; - for key in task - .removed_nodes - .iter() - .chain(regular.removed_nodes.iter()) - .cloned() - .collect::>() + for key in + task.removed_nodes.iter().chain(regular.removed_nodes.iter()).collect::>() { let (task_removed, regular_removed) = - (task.removed_nodes.contains(&key), regular.removed_nodes.contains(&key)); - let database_not_exists = - storage_trie_cursor.seek_exact(key.clone())?.map(|x| x.1).is_none(); + (task.removed_nodes.contains(key), regular.removed_nodes.contains(key)); + if task_removed == regular_removed { + continue; + } + let database_not_exists = storage_trie_cursor.seek_exact(*key)?.map(|x| x.1).is_none(); // If the deletion is a no-op, meaning that the entry is not in the // database, do not add it to the diff. - if task_removed != regular_removed && !database_not_exists { + if !database_not_exists { diff.removed_nodes.insert( - key, + *key, EntryDiff { task: task_removed, regular: regular_removed, diff --git a/crates/engine/tree/tests/e2e-testsuite/fcu_finalized_blocks.rs b/crates/engine/tree/tests/e2e-testsuite/fcu_finalized_blocks.rs new file mode 100644 index 00000000000..e7ec9ee8e68 --- /dev/null +++ b/crates/engine/tree/tests/e2e-testsuite/fcu_finalized_blocks.rs @@ -0,0 +1,79 @@ +//! E2E test for forkchoice update with finalized blocks. +//! +//! This test verifies the behavior when attempting to reorg behind a finalized block. + +use eyre::Result; +use reth_chainspec::{ChainSpecBuilder, MAINNET}; +use reth_e2e_test_utils::testsuite::{ + actions::{ + BlockReference, CaptureBlock, CreateFork, FinalizeBlock, MakeCanonical, ProduceBlocks, + SendForkchoiceUpdate, + }, + setup::{NetworkSetup, Setup}, + TestBuilder, +}; +use reth_engine_tree::tree::TreeConfig; +use reth_ethereum_engine_primitives::EthEngineTypes; +use reth_node_ethereum::EthereumNode; +use std::sync::Arc; + +/// Creates the standard setup for engine tree e2e tests. +fn default_engine_tree_setup() -> Setup { + Setup::default() + .with_chain_spec(Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis( + serde_json::from_str(include_str!( + "../../../../e2e-test-utils/src/testsuite/assets/genesis.json" + )) + .unwrap(), + ) + .cancun_activated() + .build(), + )) + .with_network(NetworkSetup::single_node()) + .with_tree_config( + TreeConfig::default().with_legacy_state_root(false).with_has_enough_parallelism(true), + ) +} + +/// This test: +/// 1. Creates a main chain and finalizes a block +/// 2. Creates a fork that branches BEFORE the finalized block +/// 3. Attempts to switch to that fork (which would require changing history behind finalized) +#[tokio::test] +async fn test_reorg_to_fork_behind_finalized() -> Result<()> { + reth_tracing::init_test_tracing(); + + let test = TestBuilder::new() + .with_setup(default_engine_tree_setup()) + // Build main chain: blocks 1-10 + .with_action(ProduceBlocks::::new(10)) + .with_action(MakeCanonical::new()) + // Capture blocks for the test + .with_action(CaptureBlock::new("block_5")) // Will be fork point + .with_action(CaptureBlock::new("block_7")) // Will be finalized + .with_action(CaptureBlock::new("block_10")) // Current head + // Create a fork from block 5 (before block 7 which will be finalized) + .with_action(CreateFork::::new(5, 5)) // Fork from block 5, add 5 blocks + .with_action(CaptureBlock::new("fork_tip")) + // Step 1: Finalize block 7 with head at block 10 + .with_action( + FinalizeBlock::::new(BlockReference::Tag("block_7".to_string())) + .with_head(BlockReference::Tag("block_10".to_string())), + ) + // Step 2: Attempt to reorg to a fork that doesn't contain the finalized block + .with_action( + SendForkchoiceUpdate::::new( + BlockReference::Tag("block_7".to_string()), // Keep finalized + BlockReference::Tag("fork_tip".to_string()), // New safe + BlockReference::Tag("fork_tip".to_string()), // New head + ) + .with_expected_status(alloy_rpc_types_engine::PayloadStatusEnum::Valid), + ); + + test.run::().await?; + + Ok(()) +} diff --git a/crates/engine/tree/tests/e2e-testsuite/main.rs b/crates/engine/tree/tests/e2e-testsuite/main.rs new file mode 100644 index 00000000000..20d7e2d68e5 --- /dev/null +++ b/crates/engine/tree/tests/e2e-testsuite/main.rs @@ -0,0 +1,336 @@ +//! E2E test implementations using the e2e test framework for engine tree functionality. + +mod fcu_finalized_blocks; + +use eyre::Result; +use reth_chainspec::{ChainSpecBuilder, MAINNET}; +use reth_e2e_test_utils::testsuite::{ + actions::{ + CaptureBlock, CompareNodeChainTips, CreateFork, ExpectFcuStatus, MakeCanonical, + ProduceBlocks, ProduceBlocksLocally, ProduceInvalidBlocks, ReorgTo, SelectActiveNode, + SendNewPayloads, UpdateBlockInfo, ValidateCanonicalTag, WaitForSync, + }, + setup::{NetworkSetup, Setup}, + TestBuilder, +}; +use reth_engine_tree::tree::TreeConfig; +use reth_ethereum_engine_primitives::EthEngineTypes; +use reth_node_ethereum::EthereumNode; +use std::sync::Arc; + +/// Creates the standard setup for engine tree e2e tests. +fn default_engine_tree_setup() -> Setup { + Setup::default() + .with_chain_spec(Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis( + serde_json::from_str(include_str!( + "../../../../e2e-test-utils/src/testsuite/assets/genesis.json" + )) + .unwrap(), + ) + .cancun_activated() + .build(), + )) + .with_network(NetworkSetup::single_node()) + .with_tree_config( + TreeConfig::default().with_legacy_state_root(false).with_has_enough_parallelism(true), + ) +} + +/// Test that verifies forkchoice update and canonical chain insertion functionality. +#[tokio::test] +async fn test_engine_tree_fcu_canon_chain_insertion_e2e() -> Result<()> { + reth_tracing::init_test_tracing(); + + let test = TestBuilder::new() + .with_setup(default_engine_tree_setup()) + // produce one block + .with_action(ProduceBlocks::::new(1)) + // make it canonical via forkchoice update + .with_action(MakeCanonical::new()) + // extend with 3 more blocks + .with_action(ProduceBlocks::::new(3)) + // make the latest block canonical + .with_action(MakeCanonical::new()); + + test.run::().await?; + + Ok(()) +} + +/// Test that verifies forkchoice update with a reorg where all blocks are already available. +#[tokio::test] +async fn test_engine_tree_fcu_reorg_with_all_blocks_e2e() -> Result<()> { + reth_tracing::init_test_tracing(); + + let test = TestBuilder::new() + .with_setup(default_engine_tree_setup()) + // create a main chain with 5 blocks (blocks 0-4) + .with_action(ProduceBlocks::::new(5)) + .with_action(MakeCanonical::new()) + // create a fork from block 2 with 3 additional blocks + .with_action(CreateFork::::new(2, 3)) + .with_action(CaptureBlock::new("fork_tip")) + // perform FCU to the fork tip - this should make the fork canonical + .with_action(ReorgTo::::new_from_tag("fork_tip")); + + test.run::().await?; + + Ok(()) +} + +/// Test that verifies valid forks with an older canonical head. +/// +/// This test creates two competing fork chains starting from a common ancestor, +/// then switches between them using forkchoice updates, verifying that the engine +/// correctly handles chains where the canonical head is older than fork tips. +#[tokio::test] +async fn test_engine_tree_valid_forks_with_older_canonical_head_e2e() -> Result<()> { + reth_tracing::init_test_tracing(); + + let test = TestBuilder::new() + .with_setup(default_engine_tree_setup()) + // create base chain with 1 block (this will be our old head) + .with_action(ProduceBlocks::::new(1)) + .with_action(CaptureBlock::new("old_head")) + .with_action(MakeCanonical::new()) + // extend base chain with 5 more blocks to establish a fork point + .with_action(ProduceBlocks::::new(5)) + .with_action(CaptureBlock::new("fork_point")) + .with_action(MakeCanonical::new()) + // revert to old head to simulate scenario where canonical head is older + .with_action(ReorgTo::::new_from_tag("old_head")) + // create first competing chain (chain A) from fork point with 10 blocks + .with_action(CreateFork::::new_from_tag("fork_point", 10)) + .with_action(CaptureBlock::new("chain_a_tip")) + // create second competing chain (chain B) from same fork point with 10 blocks + .with_action(CreateFork::::new_from_tag("fork_point", 10)) + .with_action(CaptureBlock::new("chain_b_tip")) + // switch to chain B via forkchoice update - this should become canonical + .with_action(ReorgTo::::new_from_tag("chain_b_tip")); + + test.run::().await?; + + Ok(()) +} + +/// Test that verifies valid and invalid forks with an older canonical head. +#[tokio::test] +async fn test_engine_tree_valid_and_invalid_forks_with_older_canonical_head_e2e() -> Result<()> { + reth_tracing::init_test_tracing(); + + let test = TestBuilder::new() + .with_setup(default_engine_tree_setup()) + // create base chain with 1 block (old head) + .with_action(ProduceBlocks::::new(1)) + .with_action(CaptureBlock::new("old_head")) + .with_action(MakeCanonical::new()) + // extend base chain with 5 more blocks to establish fork point + .with_action(ProduceBlocks::::new(5)) + .with_action(CaptureBlock::new("fork_point")) + .with_action(MakeCanonical::new()) + // revert to old head to simulate older canonical head scenario + .with_action(ReorgTo::::new_from_tag("old_head")) + // create chain B (the valid chain) from fork point with 10 blocks + .with_action(CreateFork::::new_from_tag("fork_point", 10)) + .with_action(CaptureBlock::new("chain_b_tip")) + // make chain B canonical via FCU - this becomes the valid chain + .with_action(ReorgTo::::new_from_tag("chain_b_tip")) + // create chain A (competing chain) - first produce valid blocks, then test invalid + // scenario + .with_action(ReorgTo::::new_from_tag("fork_point")) + .with_action(ProduceBlocks::::new(10)) + .with_action(CaptureBlock::new("chain_a_tip")) + // test that FCU to chain A tip returns VALID status (it's a valid competing chain) + .with_action(ExpectFcuStatus::valid("chain_a_tip")) + // attempt to produce invalid blocks (which should be rejected) + .with_action(ProduceInvalidBlocks::::with_invalid_at(3, 2)) + // chain B remains the canonical chain + .with_action(ValidateCanonicalTag::new("chain_b_tip")); + + test.run::().await?; + + Ok(()) +} + +/// Test that verifies engine tree behavior when handling invalid blocks. +/// This test demonstrates that invalid blocks are correctly rejected and that +/// attempts to build on top of them fail appropriately. +#[tokio::test] +async fn test_engine_tree_reorg_with_missing_ancestor_expecting_valid_e2e() -> Result<()> { + reth_tracing::init_test_tracing(); + + let test = TestBuilder::new() + .with_setup(default_engine_tree_setup()) + // build main chain (blocks 1-6) + .with_action(ProduceBlocks::::new(6)) + .with_action(MakeCanonical::new()) + .with_action(CaptureBlock::new("main_chain_tip")) + // create a valid fork first + .with_action(CreateFork::::new_from_tag("main_chain_tip", 5)) + .with_action(CaptureBlock::new("valid_fork_tip")) + // FCU to the valid fork should work + .with_action(ExpectFcuStatus::valid("valid_fork_tip")); + + test.run::().await?; + + // attempting to build invalid chains fails properly + let invalid_test = TestBuilder::new() + .with_setup(default_engine_tree_setup()) + .with_action(ProduceBlocks::::new(3)) + .with_action(MakeCanonical::new()) + // This should fail when trying to build subsequent blocks on the invalid block + .with_action(ProduceInvalidBlocks::::with_invalid_at(2, 0)); + + assert!(invalid_test.run::().await.is_err()); + + Ok(()) +} + +/// Test that verifies buffered blocks are eventually connected when sent in reverse order. +#[tokio::test] +async fn test_engine_tree_buffered_blocks_are_eventually_connected_e2e() -> Result<()> { + reth_tracing::init_test_tracing(); + + let test = TestBuilder::new() + .with_setup( + Setup::default() + .with_chain_spec(Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis( + serde_json::from_str(include_str!( + "../../../../e2e-test-utils/src/testsuite/assets/genesis.json" + )) + .unwrap(), + ) + .cancun_activated() + .build(), + )) + .with_network(NetworkSetup::multi_node_unconnected(2)) // Need 2 disconnected nodes + .with_tree_config( + TreeConfig::default() + .with_legacy_state_root(false) + .with_has_enough_parallelism(true), + ), + ) + // node 0 produces blocks 1 and 2 locally without broadcasting + .with_action(SelectActiveNode::new(0)) + .with_action(ProduceBlocksLocally::::new(2)) + // make the blocks canonical on node 0 so they're available via RPC + .with_action(MakeCanonical::with_active_node()) + // send blocks in reverse order (2, then 1) from node 0 to node 1 + .with_action( + SendNewPayloads::::new() + .with_target_node(1) + .with_source_node(0) + .with_start_block(1) + .with_total_blocks(2) + .in_reverse_order(), + ) + // update node 1's view to recognize the new blocks + .with_action(SelectActiveNode::new(1)) + // get the latest block from node 1's RPC and update environment + .with_action(UpdateBlockInfo::default()) + // make block 2 canonical on node 1 with a forkchoice update + .with_action(MakeCanonical::with_active_node()) + // verify both nodes eventually have the same chain tip + .with_action(CompareNodeChainTips::expect_same(0, 1)); + + test.run::().await?; + + Ok(()) +} + +/// Test that verifies forkchoice updates can extend the canonical chain progressively. +/// +/// This test creates a longer chain of blocks, then uses forkchoice updates to make +/// different parts of the chain canonical in sequence, verifying that FCU properly +/// advances the canonical head when all blocks are already available. +#[tokio::test] +async fn test_engine_tree_fcu_extends_canon_chain_e2e() -> Result<()> { + reth_tracing::init_test_tracing(); + + let test = TestBuilder::new() + .with_setup(default_engine_tree_setup()) + // create and make canonical a base chain with 1 block + .with_action(ProduceBlocks::::new(1)) + .with_action(MakeCanonical::new()) + // extend the chain with 10 more blocks (total 11 blocks: 0-10) + .with_action(ProduceBlocks::::new(10)) + // capture block 6 as our intermediate target (from 0-indexed, this is block 6) + .with_action(CaptureBlock::new("target_block")) + // make the intermediate target canonical via FCU + .with_action(ReorgTo::::new_from_tag("target_block")) + // now make the chain tip canonical via FCU + .with_action(MakeCanonical::new()); + + test.run::().await?; + + Ok(()) +} + +/// Test that verifies live sync transition where a long chain eventually becomes canonical. +/// +/// This test simulates a scenario where: +/// 1. Both nodes start with the same short base chain +/// 2. Node 0 builds a long chain locally (no broadcast, becomes its canonical tip) +/// 3. Node 1 still has only the short base chain as its canonical tip +/// 4. Node 1 receives FCU pointing to Node 0's long chain tip and must sync +/// 5. Both nodes end up with the same canonical chain through real P2P sync +#[tokio::test] +async fn test_engine_tree_live_sync_transition_eventually_canonical_e2e() -> Result<()> { + reth_tracing::init_test_tracing(); + + const MIN_BLOCKS_FOR_PIPELINE_RUN: u64 = 32; // EPOCH_SLOTS from alloy-eips + + let test = TestBuilder::new() + .with_setup( + Setup::default() + .with_chain_spec(Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis( + serde_json::from_str(include_str!( + "../../../../e2e-test-utils/src/testsuite/assets/genesis.json" + )) + .unwrap(), + ) + .cancun_activated() + .build(), + )) + .with_network(NetworkSetup::multi_node(2)) // Two connected nodes + .with_tree_config( + TreeConfig::default() + .with_legacy_state_root(false) + .with_has_enough_parallelism(true), + ), + ) + // Both nodes start with the same base chain (1 block) + .with_action(SelectActiveNode::new(0)) + .with_action(ProduceBlocks::::new(1)) + .with_action(MakeCanonical::new()) // Both nodes have the same base chain + .with_action(CaptureBlock::new("base_chain_tip")) + // Node 0: Build a much longer chain but don't broadcast it yet + .with_action(ProduceBlocksLocally::::new(MIN_BLOCKS_FOR_PIPELINE_RUN + 10)) + .with_action(MakeCanonical::with_active_node()) // Only make it canonical on Node 0 + .with_action(CaptureBlock::new("long_chain_tip")) + // Verify Node 0's canonical tip is the long chain tip + .with_action(ValidateCanonicalTag::new("long_chain_tip")) + // Verify Node 1's canonical tip is still the base chain tip + .with_action(SelectActiveNode::new(1)) + .with_action(ValidateCanonicalTag::new("base_chain_tip")) + // Node 1: Send FCU pointing to Node 0's long chain tip + // This should trigger Node 1 to sync the missing blocks from Node 0 + .with_action(ReorgTo::::new_from_tag("long_chain_tip")) + // Wait for Node 1 to sync with Node 0 + .with_action(WaitForSync::new(0, 1).with_timeout(60)) + // Verify both nodes end up with the same canonical chain + .with_action(CompareNodeChainTips::expect_same(0, 1)); + + test.run::().await?; + + Ok(()) +} diff --git a/crates/engine/util/Cargo.toml b/crates/engine/util/Cargo.toml index c6a34bcc164..58ee6ac255c 100644 --- a/crates/engine/util/Cargo.toml +++ b/crates/engine/util/Cargo.toml @@ -17,6 +17,7 @@ reth-errors.workspace = true reth-chainspec.workspace = true reth-fs-util.workspace = true reth-engine-primitives.workspace = true +reth-engine-tree.workspace = true reth-evm.workspace = true reth-revm.workspace = true reth-storage-api.workspace = true diff --git a/crates/engine/util/src/engine_store.rs b/crates/engine/util/src/engine_store.rs index 4f9ccb586ea..a79504db30e 100644 --- a/crates/engine/util/src/engine_store.rs +++ b/crates/engine/util/src/engine_store.rs @@ -140,10 +140,10 @@ where fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { let mut this = self.project(); let next = ready!(this.stream.poll_next_unpin(cx)); - if let Some(msg) = &next { - if let Err(error) = this.store.on_message(msg, SystemTime::now()) { - error!(target: "engine::stream::store", ?msg, %error, "Error handling Engine API message"); - } + if let Some(msg) = &next && + let Err(error) = this.store.on_message(msg, SystemTime::now()) + { + error!(target: "engine::stream::store", ?msg, %error, "Error handling Engine API message"); } Poll::Ready(next) } diff --git a/crates/engine/util/src/lib.rs b/crates/engine/util/src/lib.rs index 9c2e9449bb3..03f81302c14 100644 --- a/crates/engine/util/src/lib.rs +++ b/crates/engine/util/src/lib.rs @@ -5,10 +5,10 @@ html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256", issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -use futures::Stream; +use futures::{Future, Stream}; use reth_engine_primitives::BeaconEngineMessage; use reth_payload_primitives::PayloadTypes; use std::path::PathBuf; @@ -26,6 +26,10 @@ use skip_new_payload::EngineSkipNewPayload; pub mod reorg; use reorg::EngineReorg; +/// The result type for `maybe_reorg` method. +type MaybeReorgResult = + Result, S>, E>; + /// The collection of stream extensions for engine API message stream. pub trait EngineMessageStreamExt: Stream> { /// Skips the specified number of [`BeaconEngineMessage::ForkchoiceUpdated`] messages from the @@ -123,28 +127,38 @@ pub trait EngineMessageStreamExt: Stream( + /// + /// The `payload_validator_fn` closure is only called if `frequency` is `Some`, + /// allowing for lazy initialization of the validator. + fn maybe_reorg( self, provider: Provider, evm_config: Evm, - payload_validator: Validator, + payload_validator_fn: F, frequency: Option, depth: Option, - ) -> Either, Self> + ) -> impl Future> + Send where - Self: Sized, + Self: Sized + Send, + Provider: Send, + Evm: Send, + F: FnOnce() -> Fut + Send, + Fut: Future> + Send, { - if let Some(frequency) = frequency { - Either::Left(reorg::EngineReorg::new( - self, - provider, - evm_config, - payload_validator, - frequency, - depth.unwrap_or_default(), - )) - } else { - Either::Right(self) + async move { + if let Some(frequency) = frequency { + let validator = payload_validator_fn().await?; + Ok(Either::Left(reorg::EngineReorg::new( + self, + provider, + evm_config, + validator, + frequency, + depth.unwrap_or_default(), + ))) + } else { + Ok(Either::Right(self)) + } } } } diff --git a/crates/engine/util/src/reorg.rs b/crates/engine/util/src/reorg.rs index 63770a7f371..7d84afc6d59 100644 --- a/crates/engine/util/src/reorg.rs +++ b/crates/engine/util/src/reorg.rs @@ -7,8 +7,8 @@ use itertools::Either; use reth_chainspec::{ChainSpecProvider, EthChainSpec}; use reth_engine_primitives::{ BeaconEngineMessage, BeaconOnNewPayloadError, ExecutionPayload as _, OnForkChoiceUpdated, - PayloadValidator, }; +use reth_engine_tree::tree::EngineValidator; use reth_errors::{BlockExecutionError, BlockValidationError, RethError, RethResult}; use reth_evm::{ execute::{BlockBuilder, BlockBuilderOutcome}, @@ -103,7 +103,7 @@ where + StateProviderFactory + ChainSpecProvider, Evm: ConfigureEvm, - Validator: PayloadValidator>, + Validator: EngineValidator, { type Item = S::Item; @@ -236,19 +236,20 @@ where } } -fn create_reorg_head( +fn create_reorg_head( provider: &Provider, evm_config: &Evm, payload_validator: &Validator, mut depth: usize, - next_payload: Validator::ExecutionData, + next_payload: T::ExecutionData, ) -> RethResult>> where Provider: BlockReader
, Block = BlockTy> + StateProviderFactory + ChainSpecProvider, Evm: ConfigureEvm, - Validator: PayloadValidator>, + T: PayloadTypes>, + Validator: EngineValidator, { // Ensure next payload is valid. let next_block = @@ -284,8 +285,8 @@ where .with_bundle_update() .build(); - let ctx = evm_config.context_for_block(&reorg_target); - let evm = evm_config.evm_for_block(&mut state, &reorg_target); + let ctx = evm_config.context_for_block(&reorg_target).map_err(RethError::other)?; + let evm = evm_config.evm_for_block(&mut state, &reorg_target).map_err(RethError::other)?; let mut builder = evm_config.create_block_builder(evm, &reorg_target_parent, ctx); builder.apply_pre_execution_changes()?; @@ -298,7 +299,7 @@ where } let tx_recovered = - tx.try_clone_into_recovered().map_err(|_| ProviderError::SenderRecoveryError)?; + tx.try_into_recovered().map_err(|_| ProviderError::SenderRecoveryError)?; let gas_used = match builder.execute_transaction(tx_recovered) { Ok(gas_used) => gas_used, Err(BlockExecutionError::Validation(BlockValidationError::InvalidTx { diff --git a/crates/era-downloader/Cargo.toml b/crates/era-downloader/Cargo.toml index 84a5187a70f..54ae581813a 100644 --- a/crates/era-downloader/Cargo.toml +++ b/crates/era-downloader/Cargo.toml @@ -35,8 +35,6 @@ sha2.workspace = true sha2.features = ["std"] [dev-dependencies] -tokio.workspace = true -tokio.features = ["fs", "io-util", "macros"] tempfile.workspace = true test-case.workspace = true futures.workspace = true diff --git a/crates/era-downloader/src/client.rs b/crates/era-downloader/src/client.rs index 1d67fc39c2b..36ed93e1e2f 100644 --- a/crates/era-downloader/src/client.rs +++ b/crates/era-downloader/src/client.rs @@ -7,7 +7,7 @@ use sha2::{Digest, Sha256}; use std::{future::Future, path::Path, str::FromStr}; use tokio::{ fs::{self, File}, - io::{self, AsyncBufReadExt, AsyncWriteExt}, + io::{self, AsyncBufReadExt, AsyncRead, AsyncReadExt, AsyncWriteExt}, join, try_join, }; @@ -47,8 +47,8 @@ impl EraClient { const CHECKSUMS: &'static str = "checksums.txt"; /// Constructs [`EraClient`] using `client` to download from `url` into `folder`. - pub const fn new(client: Http, url: Url, folder: Box) -> Self { - Self { client, url, folder } + pub fn new(client: Http, url: Url, folder: impl Into>) -> Self { + Self { client, url, folder: folder.into() } } /// Performs a GET request on `url` and stores the response body into a file located within @@ -65,62 +65,79 @@ impl EraClient { .ok_or_eyre("empty path segments")?; let path = path.join(file_name); - let number = - self.file_name_to_number(file_name).ok_or_eyre("Cannot parse number from file name")?; - let mut stream = client.get(url).await?; - let mut file = File::create(&path).await?; - let mut hasher = Sha256::new(); - - while let Some(item) = stream.next().await.transpose()? { - io::copy(&mut item.as_ref(), &mut file).await?; - hasher.update(item); - } + if !self.is_downloaded(file_name, &path).await? { + let number = self + .file_name_to_number(file_name) + .ok_or_eyre("Cannot parse number from file name")?; + + let mut tries = 1..3; + let mut actual_checksum: eyre::Result<_>; + loop { + actual_checksum = async { + let mut file = File::create(&path).await?; + let mut stream = client.get(url.clone()).await?; + let mut hasher = Sha256::new(); + + while let Some(item) = stream.next().await.transpose()? { + io::copy(&mut item.as_ref(), &mut file).await?; + hasher.update(item); + } - let actual_checksum = hasher.finalize().to_vec(); + Ok(hasher.finalize().to_vec()) + } + .await; - let file = File::open(self.folder.join(Self::CHECKSUMS)).await?; - let reader = io::BufReader::new(file); - let mut lines = reader.lines(); + if actual_checksum.is_ok() || tries.next().is_none() { + break; + } + } - for _ in 0..number { - lines.next_line().await?; - } - let expected_checksum = - lines.next_line().await?.ok_or_else(|| eyre!("Missing hash for number {number}"))?; - let expected_checksum = hex::decode(expected_checksum)?; - - if actual_checksum != expected_checksum { - return Err(eyre!( - "Checksum mismatch, got: {}, expected: {}", - actual_checksum.encode_hex(), - expected_checksum.encode_hex() - )); + self.assert_checksum(number, actual_checksum?) + .await + .map_err(|e| eyre!("{e} for {file_name} at {}", path.display()))?; } Ok(path.into_boxed_path()) } /// Recovers index of file following the latest downloaded file from a different run. - pub async fn recover_index(&self) -> u64 { + pub async fn recover_index(&self) -> Option { let mut max = None; if let Ok(mut dir) = fs::read_dir(&self.folder).await { while let Ok(Some(entry)) = dir.next_entry().await { - if let Some(name) = entry.file_name().to_str() { - if let Some(number) = self.file_name_to_number(name) { - if max.is_none() || matches!(max, Some(max) if number > max) { - max.replace(number); - } - } + if let Some(name) = entry.file_name().to_str() && + let Some(number) = self.file_name_to_number(name) && + (max.is_none() || matches!(max, Some(max) if number > max)) + { + max.replace(number + 1); } } } - max.map(|v| v + 1).unwrap_or(0) + max + } + + /// Deletes files that are outside-of the working range. + pub async fn delete_outside_range(&self, index: usize, max_files: usize) -> eyre::Result<()> { + let last = index + max_files; + + if let Ok(mut dir) = fs::read_dir(&self.folder).await { + while let Ok(Some(entry)) = dir.next_entry().await { + if let Some(name) = entry.file_name().to_str() && + let Some(number) = self.file_name_to_number(name) && + (number < index || number >= last) + { + reth_fs_util::remove_file(entry.path())?; + } + } + } + + Ok(()) } /// Returns a download URL for the file corresponding to `number`. - pub async fn url(&self, number: u64) -> eyre::Result> { + pub async fn url(&self, number: usize) -> eyre::Result> { Ok(self.number_to_file_name(number).await?.map(|name| self.url.join(&name)).transpose()?) } @@ -142,7 +159,7 @@ impl EraClient { /// Fetches the list of ERA1 files from `url` and stores it in a file located within `folder`. pub async fn fetch_file_list(&self) -> eyre::Result<()> { let (mut index, mut checksums) = try_join!( - self.client.get(self.url.clone().join("index.html")?), + self.client.get(self.url.clone()), self.client.get(self.url.clone().join(Self::CHECKSUMS)?), )?; @@ -187,12 +204,12 @@ impl EraClient { let mut writer = io::BufWriter::new(file); while let Some(line) = lines.next_line().await? { - if let Some(j) = line.find(".era1") { - if let Some(i) = line[..j].rfind(|c: char| !c.is_alphanumeric() && c != '-') { - let era = &line[i + 1..j + 5]; - writer.write_all(era.as_bytes()).await?; - writer.write_all(b"\n").await?; - } + if let Some(j) = line.find(".era1") && + let Some(i) = line[..j].rfind(|c: char| !c.is_alphanumeric() && c != '-') + { + let era = &line[i + 1..j + 5]; + writer.write_all(era.as_bytes()).await?; + writer.write_all(b"\n").await?; } } writer.flush().await?; @@ -201,7 +218,7 @@ impl EraClient { } /// Returns ERA1 file name that is ordered at `number`. - pub async fn number_to_file_name(&self, number: u64) -> eyre::Result> { + pub async fn number_to_file_name(&self, number: usize) -> eyre::Result> { let path = self.folder.to_path_buf().join("index"); let file = File::open(&path).await?; let reader = io::BufReader::new(file); @@ -213,11 +230,101 @@ impl EraClient { Ok(lines.next_line().await?) } - fn file_name_to_number(&self, file_name: &str) -> Option { - file_name.split('-').nth(1).and_then(|v| u64::from_str(v).ok()) + async fn is_downloaded(&self, name: &str, path: impl AsRef) -> eyre::Result { + let path = path.as_ref(); + + match File::open(path).await { + Ok(file) => { + let number = self + .file_name_to_number(name) + .ok_or_else(|| eyre!("Cannot parse ERA number from {name}"))?; + + let actual_checksum = checksum(file).await?; + let is_verified = self.verify_checksum(number, actual_checksum).await?; + + if !is_verified { + fs::remove_file(path).await?; + } + + Ok(is_verified) + } + Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(false), + Err(e) => Err(e)?, + } + } + + /// Returns `true` if `actual_checksum` matches expected checksum of the ERA1 file indexed by + /// `number` based on the [file list]. + /// + /// [file list]: Self::fetch_file_list + async fn verify_checksum(&self, number: usize, actual_checksum: Vec) -> eyre::Result { + Ok(actual_checksum == self.expected_checksum(number).await?) + } + + /// Returns `Ok` if `actual_checksum` matches expected checksum of the ERA1 file indexed by + /// `number` based on the [file list]. + /// + /// [file list]: Self::fetch_file_list + async fn assert_checksum(&self, number: usize, actual_checksum: Vec) -> eyre::Result<()> { + let expected_checksum = self.expected_checksum(number).await?; + + if actual_checksum == expected_checksum { + Ok(()) + } else { + Err(eyre!( + "Checksum mismatch, got: {}, expected: {}", + actual_checksum.encode_hex(), + expected_checksum.encode_hex() + )) + } + } + + /// Returns SHA-256 checksum for ERA1 file indexed by `number` based on the [file list]. + /// + /// [file list]: Self::fetch_file_list + async fn expected_checksum(&self, number: usize) -> eyre::Result> { + let file = File::open(self.folder.join(Self::CHECKSUMS)).await?; + let reader = io::BufReader::new(file); + let mut lines = reader.lines(); + + for _ in 0..number { + lines.next_line().await?; + } + let expected_checksum = + lines.next_line().await?.ok_or_else(|| eyre!("Missing hash for number {number}"))?; + let expected_checksum = hex::decode(expected_checksum)?; + + Ok(expected_checksum) + } + + fn file_name_to_number(&self, file_name: &str) -> Option { + file_name.split('-').nth(1).and_then(|v| usize::from_str(v).ok()) } } +async fn checksum(mut reader: impl AsyncRead + Unpin) -> eyre::Result> { + let mut hasher = Sha256::new(); + + // Create a buffer to read data into, sized for performance. + let mut data = vec![0; 64 * 1024]; + + loop { + // Read data from the reader into the buffer. + let len = reader.read(&mut data).await?; + if len == 0 { + break; + } // Exit loop if no more data. + + // Update the hash with the data read. + hasher.update(&data[..len]); + } + + // Finalize the hash after all data has been processed. + let hash = hasher.finalize().to_vec(); + + Ok(hash) +} + #[cfg(test)] mod tests { use super::*; @@ -226,11 +333,7 @@ mod tests { impl EraClient { fn empty() -> Self { - Self::new( - Client::new(), - Url::from_str("file:///").unwrap(), - PathBuf::new().into_boxed_path(), - ) + Self::new(Client::new(), Url::from_str("file:///").unwrap(), PathBuf::new()) } } @@ -238,7 +341,7 @@ mod tests { #[test_case("mainnet-00000-a81ae85f.era1", Some(0))] #[test_case("00000-a81ae85f.era1", None)] #[test_case("", None)] - fn test_file_name_to_number(file_name: &str, expected_number: Option) { + fn test_file_name_to_number(file_name: &str, expected_number: Option) { let client = EraClient::empty(); let actual_number = client.file_name_to_number(file_name); diff --git a/crates/era-downloader/src/fs.rs b/crates/era-downloader/src/fs.rs index 076c2f40f8d..19532f01cff 100644 --- a/crates/era-downloader/src/fs.rs +++ b/crates/era-downloader/src/fs.rs @@ -1,5 +1,5 @@ -use crate::EraMeta; -use alloy_primitives::{hex, hex::ToHexExt}; +use crate::{EraMeta, BLOCKS_PER_FILE}; +use alloy_primitives::{hex, hex::ToHexExt, BlockNumber}; use eyre::{eyre, OptionExt}; use futures_util::{stream, Stream}; use reth_fs_util as fs; @@ -9,6 +9,7 @@ use std::{fmt::Debug, io, io::BufRead, path::Path, str::FromStr}; /// Creates a new ordered asynchronous [`Stream`] of ERA1 files read from `dir`. pub fn read_dir( dir: impl AsRef + Send + Sync + 'static, + start_from: BlockNumber, ) -> eyre::Result> + Send + Sync + 'static + Unpin> { let mut checksums = None; let mut entries = fs::read_dir(dir)? @@ -16,16 +17,16 @@ pub fn read_dir( (|| { let path = entry?.path(); - if path.extension() == Some("era1".as_ref()) { - if let Some(last) = path.components().next_back() { - let str = last.as_os_str().to_string_lossy().to_string(); - let parts = str.split('-').collect::>(); + if path.extension() == Some("era1".as_ref()) && + let Some(last) = path.components().next_back() + { + let str = last.as_os_str().to_string_lossy().to_string(); + let parts = str.split('-').collect::>(); - if parts.len() == 3 { - let number = usize::from_str(parts[1])?; + if parts.len() == 3 { + let number = usize::from_str(parts[1])?; - return Ok(Some((number, path.into_boxed_path()))); - } + return Ok(Some((number, path.into_boxed_path()))); } } if path.file_name() == Some("checksums.txt".as_ref()) { @@ -44,27 +45,29 @@ pub fn read_dir( entries.sort_by(|(left, _), (right, _)| left.cmp(right)); - Ok(stream::iter(entries.into_iter().map(move |(_, path)| { - let expected_checksum = - checksums.next().transpose()?.ok_or_eyre("Got less checksums than ERA files")?; - let expected_checksum = hex::decode(expected_checksum)?; - - let mut hasher = Sha256::new(); - let mut reader = io::BufReader::new(fs::open(&path)?); - - io::copy(&mut reader, &mut hasher)?; - let actual_checksum = hasher.finalize().to_vec(); - - if actual_checksum != expected_checksum { - return Err(eyre!( - "Checksum mismatch, got: {}, expected: {}", - actual_checksum.encode_hex(), - expected_checksum.encode_hex() - )); - } - - Ok(EraLocalMeta::new(path)) - }))) + Ok(stream::iter(entries.into_iter().skip(start_from as usize / BLOCKS_PER_FILE).map( + move |(_, path)| { + let expected_checksum = + checksums.next().transpose()?.ok_or_eyre("Got less checksums than ERA files")?; + let expected_checksum = hex::decode(expected_checksum)?; + + let mut hasher = Sha256::new(); + let mut reader = io::BufReader::new(fs::open(&path)?); + + io::copy(&mut reader, &mut hasher)?; + let actual_checksum = hasher.finalize().to_vec(); + + if actual_checksum != expected_checksum { + return Err(eyre!( + "Checksum mismatch, got: {}, expected: {}", + actual_checksum.encode_hex(), + expected_checksum.encode_hex() + )); + } + + Ok(EraLocalMeta::new(path)) + }, + ))) } /// Contains information about an ERA file that is on the local file-system and is read-only. @@ -93,7 +96,11 @@ impl AsRef for EraLocalMeta { impl EraMeta for EraLocalMeta { /// A no-op. - fn mark_as_processed(self) -> eyre::Result<()> { + fn mark_as_processed(&self) -> eyre::Result<()> { Ok(()) } + + fn path(&self) -> &Path { + &self.path + } } diff --git a/crates/era-downloader/src/lib.rs b/crates/era-downloader/src/lib.rs index 6aaec5ba0a1..88afaa7af4e 100644 --- a/crates/era-downloader/src/lib.rs +++ b/crates/era-downloader/src/lib.rs @@ -12,7 +12,7 @@ //! let url = Url::from_str("file:///")?; //! //! // Directory where the ERA1 files will be downloaded to -//! let folder = PathBuf::new().into_boxed_path(); +//! let folder = PathBuf::new(); //! //! let client = EraClient::new(Client::new(), url, folder); //! @@ -41,3 +41,5 @@ mod stream; pub use client::{EraClient, HttpClient}; pub use fs::read_dir; pub use stream::{EraMeta, EraStream, EraStreamConfig}; + +pub(crate) const BLOCKS_PER_FILE: usize = 8192; diff --git a/crates/era-downloader/src/stream.rs b/crates/era-downloader/src/stream.rs index 51278aa3b61..4e8a178e577 100644 --- a/crates/era-downloader/src/stream.rs +++ b/crates/era-downloader/src/stream.rs @@ -1,4 +1,5 @@ -use crate::{client::HttpClient, EraClient}; +use crate::{client::HttpClient, EraClient, BLOCKS_PER_FILE}; +use alloy_primitives::BlockNumber; use futures_util::{stream::FuturesOrdered, FutureExt, Stream, StreamExt}; use reqwest::Url; use reth_fs_util as fs; @@ -23,11 +24,12 @@ use std::{ pub struct EraStreamConfig { max_files: usize, max_concurrent_downloads: usize, + start_from: Option, } impl Default for EraStreamConfig { fn default() -> Self { - Self { max_files: 5, max_concurrent_downloads: 3 } + Self { max_files: 5, max_concurrent_downloads: 3, start_from: None } } } @@ -43,6 +45,12 @@ impl EraStreamConfig { self.max_concurrent_downloads = max_concurrent_downloads; self } + + /// Overrides the starting ERA file index to be the first one that contains `block_number`. + pub const fn start_from(mut self, block_number: BlockNumber) -> Self { + self.start_from.replace(block_number as usize / BLOCKS_PER_FILE); + self + } } /// An asynchronous stream of ERA1 files. @@ -50,12 +58,13 @@ impl EraStreamConfig { /// # Examples /// ``` /// use futures_util::StreamExt; -/// use reth_era_downloader::{EraStream, HttpClient}; +/// use reth_era_downloader::{EraMeta, EraStream, HttpClient}; /// /// # async fn import(mut stream: EraStream) -> eyre::Result<()> { -/// while let Some(file) = stream.next().await { -/// let file = file?; -/// // Process `file: Box` +/// while let Some(meta) = stream.next().await { +/// let meta = meta?; +/// // Process file at `meta.path(): &Path` +/// meta.mark_as_processed()?; /// } /// # Ok(()) /// # } @@ -81,11 +90,13 @@ impl EraStream { client, files_count: Box::pin(async move { usize::MAX }), next_url: Box::pin(async move { Ok(None) }), - recover_index: Box::pin(async move { 0 }), + delete_outside_range: Box::pin(async move { Ok(()) }), + recover_index: Box::pin(async move { None }), fetch_file_list: Box::pin(async move { Ok(()) }), state: Default::default(), max_files: config.max_files, - index: 0, + index: config.start_from.unwrap_or_default(), + last: None, downloading: 0, }, } @@ -93,13 +104,28 @@ impl EraStream { } /// Contains information about an ERA file. -pub trait EraMeta: AsRef { +pub trait EraMeta: Debug { /// Marking this particular ERA file as "processed" lets the caller hint that it is no longer /// going to be using it. /// /// The meaning of that is up to the implementation. The caller should assume that after this /// point is no longer possible to safely read it. - fn mark_as_processed(self) -> eyre::Result<()>; + fn mark_as_processed(&self) -> eyre::Result<()>; + + /// A path to the era file. + /// + /// File should be openable and treated as read-only. + fn path(&self) -> &Path; +} + +impl EraMeta for Box { + fn mark_as_processed(&self) -> eyre::Result<()> { + T::mark_as_processed(self) + } + + fn path(&self) -> &Path { + T::path(self) + } } /// Contains information about ERA file that is hosted remotely and represented by a temporary @@ -123,8 +149,12 @@ impl AsRef for EraRemoteMeta { impl EraMeta for EraRemoteMeta { /// Removes a temporary local file representation of the remotely hosted original. - fn mark_as_processed(self) -> eyre::Result<()> { - Ok(fs::remove_file(self.path)?) + fn mark_as_processed(&self) -> eyre::Result<()> { + Ok(fs::remove_file(&self.path)?) + } + + fn path(&self) -> &Path { + &self.path } } @@ -192,11 +222,13 @@ struct StartingStream { client: EraClient, files_count: Pin + Send + Sync + 'static>>, next_url: Pin>> + Send + Sync + 'static>>, - recover_index: Pin + Send + Sync + 'static>>, + delete_outside_range: Pin> + Send + Sync + 'static>>, + recover_index: Pin> + Send + Sync + 'static>>, fetch_file_list: Pin> + Send + Sync + 'static>>, state: State, max_files: usize, - index: u64, + index: usize, + last: Option, downloading: usize, } @@ -215,6 +247,7 @@ enum State { #[default] Initial, FetchFileList, + DeleteOutsideRange, RecoverIndex, CountFiles, Missing(usize), @@ -229,27 +262,47 @@ impl Stream for Starti self.fetch_file_list(); } - if self.state == State::FetchFileList { - if let Poll::Ready(result) = self.fetch_file_list.poll_unpin(cx) { - match result { - Ok(_) => self.recover_index(), - Err(e) => return Poll::Ready(Some(Box::pin(async move { Err(e) }))), + if self.state == State::FetchFileList && + let Poll::Ready(result) = self.fetch_file_list.poll_unpin(cx) + { + match result { + Ok(_) => self.delete_outside_range(), + Err(e) => { + self.fetch_file_list(); + + return Poll::Ready(Some(Box::pin(async move { Err(e) }))); } } } - if self.state == State::RecoverIndex { - if let Poll::Ready(index) = self.recover_index.poll_unpin(cx) { - self.index = index; - self.count_files(); + if self.state == State::DeleteOutsideRange && + let Poll::Ready(result) = self.delete_outside_range.poll_unpin(cx) + { + match result { + Ok(_) => self.recover_index(), + Err(e) => { + self.delete_outside_range(); + + return Poll::Ready(Some(Box::pin(async move { Err(e) }))); + } } } - if self.state == State::CountFiles { - if let Poll::Ready(downloaded) = self.files_count.poll_unpin(cx) { - let max_missing = self.max_files.saturating_sub(downloaded + self.downloading); - self.state = State::Missing(max_missing); - } + if self.state == State::RecoverIndex && + let Poll::Ready(last) = self.recover_index.poll_unpin(cx) + { + self.last = last; + self.count_files(); + } + + if self.state == State::CountFiles && + let Poll::Ready(downloaded) = self.files_count.poll_unpin(cx) + { + let max_missing = self + .max_files + .saturating_sub(downloaded + self.downloading) + .max(self.last.unwrap_or_default().saturating_sub(self.index)); + self.state = State::Missing(max_missing); } if let State::Missing(max_missing) = self.state { @@ -263,18 +316,16 @@ impl Stream for Starti } } - if let State::NextUrl(max_missing) = self.state { - if let Poll::Ready(url) = self.next_url.poll_unpin(cx) { - self.state = State::Missing(max_missing - 1); + if let State::NextUrl(max_missing) = self.state && + let Poll::Ready(url) = self.next_url.poll_unpin(cx) + { + self.state = State::Missing(max_missing - 1); - return Poll::Ready(url.transpose().map(|url| -> DownloadFuture { - let mut client = self.client.clone(); + return Poll::Ready(url.transpose().map(|url| -> DownloadFuture { + let mut client = self.client.clone(); - Box::pin( - async move { client.download_to_file(url?).await.map(EraRemoteMeta::new) }, - ) - })); - } + Box::pin(async move { client.download_to_file(url?).await.map(EraRemoteMeta::new) }) + })); } Poll::Pending @@ -297,6 +348,17 @@ impl StartingStream { self.state = State::FetchFileList; } + fn delete_outside_range(&mut self) { + let index = self.index; + let max_files = self.max_files; + let client = self.client.clone(); + + Pin::new(&mut self.delete_outside_range) + .set(Box::pin(async move { client.delete_outside_range(index, max_files).await })); + + self.state = State::DeleteOutsideRange; + } + fn recover_index(&mut self) { let client = self.client.clone(); @@ -314,7 +376,7 @@ impl StartingStream { self.state = State::CountFiles; } - fn next_url(&mut self, index: u64, max_missing: usize) { + fn next_url(&mut self, index: usize, max_missing: usize) { let client = self.client.clone(); Pin::new(&mut self.next_url).set(Box::pin(async move { client.url(index).await })); diff --git a/crates/era-downloader/tests/it/checksums.rs b/crates/era-downloader/tests/it/checksums.rs index 511fbc6b65e..630cbece5d4 100644 --- a/crates/era-downloader/tests/it/checksums.rs +++ b/crates/era-downloader/tests/it/checksums.rs @@ -3,19 +3,19 @@ use futures::Stream; use futures_util::StreamExt; use reqwest::{IntoUrl, Url}; use reth_era_downloader::{EraClient, EraStream, EraStreamConfig, HttpClient}; -use std::{future::Future, str::FromStr}; +use std::str::FromStr; use tempfile::tempdir; use test_case::test_case; #[test_case("https://mainnet.era1.nimbus.team/"; "nimbus")] #[test_case("https://era1.ethportal.net/"; "ethportal")] -#[test_case("https://era.ithaca.xyz/era1/"; "ithaca")] +#[test_case("https://era.ithaca.xyz/era1/index.html"; "ithaca")] #[tokio::test] async fn test_invalid_checksum_returns_error(url: &str) { let base_url = Url::from_str(url).unwrap(); let folder = tempdir().unwrap(); - let folder = folder.path().to_owned().into_boxed_path(); - let client = EraClient::new(FailingClient, base_url, folder.clone()); + let folder = folder.path(); + let client = EraClient::new(FailingClient, base_url, folder); let mut stream = EraStream::new( client, @@ -23,16 +23,24 @@ async fn test_invalid_checksum_returns_error(url: &str) { ); let actual_err = stream.next().await.unwrap().unwrap_err().to_string(); - let expected_err = "Checksum mismatch, \ + let expected_err = format!( + "Checksum mismatch, \ got: 87428fc522803d31065e7bce3cf03fe475096631e5e07bbd7a0fde60c4cf25c7, \ -expected: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; +expected: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \ +for mainnet-00000-5ec1ffb8.era1 at {}/mainnet-00000-5ec1ffb8.era1", + folder.display() + ); assert_eq!(actual_err, expected_err); let actual_err = stream.next().await.unwrap().unwrap_err().to_string(); - let expected_err = "Checksum mismatch, \ + let expected_err = format!( + "Checksum mismatch, \ got: 0263829989b6fd954f72baaf2fc64bc2e2f01d692d4de72986ea808f6e99813f, \ -expected: bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; +expected: bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb \ +for mainnet-00001-a5364e9a.era1 at {}/mainnet-00001-a5364e9a.era1", + folder.display() + ); assert_eq!(actual_err, expected_err); } @@ -46,91 +54,30 @@ const CHECKSUMS: &[u8] = b"0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa struct FailingClient; impl HttpClient for FailingClient { - fn get( + async fn get( &self, url: U, - ) -> impl Future< - Output = eyre::Result> + Send + Sync + Unpin>, - > + Send - + Sync { + ) -> eyre::Result> + Send + Sync + Unpin> { let url = url.into_url().unwrap(); - async move { - match url.to_string().as_str() { - "https://mainnet.era1.nimbus.team/index.html" => { - Ok(Box::new(futures::stream::once(Box::pin(async move { - Ok(bytes::Bytes::from(crate::NIMBUS)) - }))) - as Box> + Send + Sync + Unpin>) - } - "https://era1.ethportal.net/index.html" => { - Ok(Box::new(futures::stream::once(Box::pin(async move { - Ok(bytes::Bytes::from(crate::ETH_PORTAL)) - }))) - as Box> + Send + Sync + Unpin>) - } - "https://era.ithaca.xyz/era1/index.html" => { - Ok(Box::new(futures::stream::once(Box::pin(async move { - Ok(bytes::Bytes::from(crate::ITHACA)) - }))) - as Box> + Send + Sync + Unpin>) - } - "https://mainnet.era1.nimbus.team/checksums.txt" => { - Ok(Box::new(futures::stream::once(Box::pin(async move { - Ok(bytes::Bytes::from(CHECKSUMS)) - }))) - as Box> + Send + Sync + Unpin>) - } - "https://era1.ethportal.net/checksums.txt" => { - Ok(Box::new(futures::stream::once(Box::pin(async move { - Ok(bytes::Bytes::from(CHECKSUMS)) - }))) - as Box> + Send + Sync + Unpin>) - } - "https://era.ithaca.xyz/era1/checksums.txt" => { - Ok(Box::new(futures::stream::once(Box::pin(async move { - Ok(bytes::Bytes::from(CHECKSUMS)) - }))) - as Box> + Send + Sync + Unpin>) - } - "https://era1.ethportal.net/mainnet-00000-5ec1ffb8.era1" => { - Ok(Box::new(futures::stream::once(Box::pin(async move { - Ok(bytes::Bytes::from(crate::MAINNET_0)) - }))) - as Box> + Send + Sync + Unpin>) - } - "https://mainnet.era1.nimbus.team/mainnet-00000-5ec1ffb8.era1" => { - Ok(Box::new(futures::stream::once(Box::pin(async move { - Ok(bytes::Bytes::from(crate::MAINNET_0)) - }))) - as Box> + Send + Sync + Unpin>) - } - "https://era.ithaca.xyz/era1/mainnet-00000-5ec1ffb8.era1" => { - Ok(Box::new(futures::stream::once(Box::pin(async move { - Ok(bytes::Bytes::from(crate::MAINNET_0)) - }))) - as Box> + Send + Sync + Unpin>) - } - "https://era1.ethportal.net/mainnet-00001-a5364e9a.era1" => { - Ok(Box::new(futures::stream::once(Box::pin(async move { - Ok(bytes::Bytes::from(crate::MAINNET_1)) - }))) - as Box> + Send + Sync + Unpin>) - } - "https://mainnet.era1.nimbus.team/mainnet-00001-a5364e9a.era1" => { - Ok(Box::new(futures::stream::once(Box::pin(async move { - Ok(bytes::Bytes::from(crate::MAINNET_1)) - }))) - as Box> + Send + Sync + Unpin>) - } - "https://era.ithaca.xyz/era1/mainnet-00001-a5364e9a.era1" => { - Ok(Box::new(futures::stream::once(Box::pin(async move { - Ok(bytes::Bytes::from(crate::MAINNET_1)) - }))) - as Box> + Send + Sync + Unpin>) - } - v => unimplemented!("Unexpected URL \"{v}\""), + Ok(futures::stream::iter(vec![Ok(match url.to_string().as_str() { + "https://mainnet.era1.nimbus.team/" => Bytes::from_static(crate::NIMBUS), + "https://era1.ethportal.net/" => Bytes::from_static(crate::ETH_PORTAL), + "https://era.ithaca.xyz/era1/index.html" => Bytes::from_static(crate::ITHACA), + "https://mainnet.era1.nimbus.team/checksums.txt" | + "https://era1.ethportal.net/checksums.txt" | + "https://era.ithaca.xyz/era1/checksums.txt" => Bytes::from_static(CHECKSUMS), + "https://era1.ethportal.net/mainnet-00000-5ec1ffb8.era1" | + "https://mainnet.era1.nimbus.team/mainnet-00000-5ec1ffb8.era1" | + "https://era.ithaca.xyz/era1/mainnet-00000-5ec1ffb8.era1" => { + Bytes::from_static(crate::MAINNET_0) + } + "https://era1.ethportal.net/mainnet-00001-a5364e9a.era1" | + "https://mainnet.era1.nimbus.team/mainnet-00001-a5364e9a.era1" | + "https://era.ithaca.xyz/era1/mainnet-00001-a5364e9a.era1" => { + Bytes::from_static(crate::MAINNET_1) } - } + v => unimplemented!("Unexpected URL \"{v}\""), + })])) } } diff --git a/crates/era-downloader/tests/it/download.rs b/crates/era-downloader/tests/it/download.rs index dba658b4eb1..e7756bfede9 100644 --- a/crates/era-downloader/tests/it/download.rs +++ b/crates/era-downloader/tests/it/download.rs @@ -8,17 +8,17 @@ use test_case::test_case; #[test_case("https://mainnet.era1.nimbus.team/"; "nimbus")] #[test_case("https://era1.ethportal.net/"; "ethportal")] -#[test_case("https://era.ithaca.xyz/era1/"; "ithaca")] +#[test_case("https://era.ithaca.xyz/era1/index.html"; "ithaca")] #[tokio::test] async fn test_getting_file_url_after_fetching_file_list(url: &str) { let base_url = Url::from_str(url).unwrap(); let folder = tempdir().unwrap(); - let folder = folder.path().to_owned().into_boxed_path(); - let client = EraClient::new(StubClient, base_url, folder); + let folder = folder.path(); + let client = EraClient::new(StubClient, base_url.clone(), folder); client.fetch_file_list().await.unwrap(); - let expected_url = Some(Url::from_str(&format!("{url}mainnet-00000-5ec1ffb8.era1")).unwrap()); + let expected_url = Some(base_url.join("mainnet-00000-5ec1ffb8.era1").unwrap()); let actual_url = client.url(0).await.unwrap(); assert_eq!(actual_url, expected_url); @@ -26,12 +26,12 @@ async fn test_getting_file_url_after_fetching_file_list(url: &str) { #[test_case("https://mainnet.era1.nimbus.team/"; "nimbus")] #[test_case("https://era1.ethportal.net/"; "ethportal")] -#[test_case("https://era.ithaca.xyz/era1/"; "ithaca")] +#[test_case("https://era.ithaca.xyz/era1/index.html"; "ithaca")] #[tokio::test] async fn test_getting_file_after_fetching_file_list(url: &str) { let base_url = Url::from_str(url).unwrap(); let folder = tempdir().unwrap(); - let folder = folder.path().to_owned().into_boxed_path(); + let folder = folder.path(); let mut client = EraClient::new(StubClient, base_url, folder); client.fetch_file_list().await.unwrap(); diff --git a/crates/era-downloader/tests/it/fs.rs b/crates/era-downloader/tests/it/fs.rs index 5ad7ba28007..00a36124745 100644 --- a/crates/era-downloader/tests/it/fs.rs +++ b/crates/era-downloader/tests/it/fs.rs @@ -70,7 +70,7 @@ async fn test_streaming_from_local_directory( fs::write(folder.join("mainnet-00001-a5364e9a.era1"), CONTENTS_1).await.unwrap(); let folder = folder.into_boxed_path(); - let actual = read_dir(folder.clone()); + let actual = read_dir(folder.clone(), 0); match checksums { Ok(_) => match actual { diff --git a/crates/era-downloader/tests/it/list.rs b/crates/era-downloader/tests/it/list.rs index cfb0e3b8b42..3940fa5d8be 100644 --- a/crates/era-downloader/tests/it/list.rs +++ b/crates/era-downloader/tests/it/list.rs @@ -8,12 +8,12 @@ use test_case::test_case; #[test_case("https://mainnet.era1.nimbus.team/"; "nimbus")] #[test_case("https://era1.ethportal.net/"; "ethportal")] -#[test_case("https://era.ithaca.xyz/era1/"; "ithaca")] +#[test_case("https://era.ithaca.xyz/era1/index.html"; "ithaca")] #[tokio::test] async fn test_getting_file_name_after_fetching_file_list(url: &str) { let url = Url::from_str(url).unwrap(); let folder = tempdir().unwrap(); - let folder = folder.path().to_owned().into_boxed_path(); + let folder = folder.path(); let client = EraClient::new(StubClient, url, folder); client.fetch_file_list().await.unwrap(); diff --git a/crates/era-downloader/tests/it/main.rs b/crates/era-downloader/tests/it/main.rs index f40e5ddb30a..526d3885bff 100644 --- a/crates/era-downloader/tests/it/main.rs +++ b/crates/era-downloader/tests/it/main.rs @@ -9,10 +9,9 @@ mod stream; const fn main() {} use bytes::Bytes; -use futures_util::Stream; +use futures::Stream; use reqwest::IntoUrl; use reth_era_downloader::HttpClient; -use std::future::Future; pub(crate) const NIMBUS: &[u8] = include_bytes!("../res/nimbus.html"); pub(crate) const ETH_PORTAL: &[u8] = include_bytes!("../res/ethportal.html"); @@ -27,91 +26,30 @@ pub(crate) const MAINNET_1: &[u8] = include_bytes!("../res/mainnet-00001-a5364e9 struct StubClient; impl HttpClient for StubClient { - fn get( + async fn get( &self, url: U, - ) -> impl Future< - Output = eyre::Result> + Send + Sync + Unpin>, - > + Send - + Sync { + ) -> eyre::Result> + Send + Sync + Unpin> { let url = url.into_url().unwrap(); - async move { - match url.to_string().as_str() { - "https://mainnet.era1.nimbus.team/index.html" => { - Ok(Box::new(futures::stream::once(Box::pin(async move { - Ok(bytes::Bytes::from(NIMBUS)) - }))) - as Box> + Send + Sync + Unpin>) - } - "https://era1.ethportal.net/index.html" => { - Ok(Box::new(futures::stream::once(Box::pin(async move { - Ok(bytes::Bytes::from(ETH_PORTAL)) - }))) - as Box> + Send + Sync + Unpin>) - } - "https://era.ithaca.xyz/era1/index.html" => { - Ok(Box::new(futures::stream::once(Box::pin(async move { - Ok(bytes::Bytes::from(ITHACA)) - }))) - as Box> + Send + Sync + Unpin>) - } - "https://mainnet.era1.nimbus.team/checksums.txt" => { - Ok(Box::new(futures::stream::once(Box::pin(async move { - Ok(bytes::Bytes::from(CHECKSUMS)) - }))) - as Box> + Send + Sync + Unpin>) - } - "https://era1.ethportal.net/checksums.txt" => { - Ok(Box::new(futures::stream::once(Box::pin(async move { - Ok(bytes::Bytes::from(CHECKSUMS)) - }))) - as Box> + Send + Sync + Unpin>) - } - "https://era.ithaca.xyz/era1/checksums.txt" => { - Ok(Box::new(futures::stream::once(Box::pin(async move { - Ok(bytes::Bytes::from(CHECKSUMS)) - }))) - as Box> + Send + Sync + Unpin>) - } - "https://era1.ethportal.net/mainnet-00000-5ec1ffb8.era1" => { - Ok(Box::new(futures::stream::once(Box::pin(async move { - Ok(bytes::Bytes::from(MAINNET_0)) - }))) - as Box> + Send + Sync + Unpin>) - } - "https://mainnet.era1.nimbus.team/mainnet-00000-5ec1ffb8.era1" => { - Ok(Box::new(futures::stream::once(Box::pin(async move { - Ok(bytes::Bytes::from(MAINNET_0)) - }))) - as Box> + Send + Sync + Unpin>) - } - "https://era.ithaca.xyz/era1/mainnet-00000-5ec1ffb8.era1" => { - Ok(Box::new(futures::stream::once(Box::pin(async move { - Ok(bytes::Bytes::from(MAINNET_0)) - }))) - as Box> + Send + Sync + Unpin>) - } - "https://era1.ethportal.net/mainnet-00001-a5364e9a.era1" => { - Ok(Box::new(futures::stream::once(Box::pin(async move { - Ok(bytes::Bytes::from(MAINNET_1)) - }))) - as Box> + Send + Sync + Unpin>) - } - "https://mainnet.era1.nimbus.team/mainnet-00001-a5364e9a.era1" => { - Ok(Box::new(futures::stream::once(Box::pin(async move { - Ok(bytes::Bytes::from(MAINNET_1)) - }))) - as Box> + Send + Sync + Unpin>) - } - "https://era.ithaca.xyz/era1/mainnet-00001-a5364e9a.era1" => { - Ok(Box::new(futures::stream::once(Box::pin(async move { - Ok(bytes::Bytes::from(MAINNET_1)) - }))) - as Box> + Send + Sync + Unpin>) - } - v => unimplemented!("Unexpected URL \"{v}\""), + Ok(futures::stream::iter(vec![Ok(match url.to_string().as_str() { + "https://mainnet.era1.nimbus.team/" => Bytes::from_static(NIMBUS), + "https://era1.ethportal.net/" => Bytes::from_static(ETH_PORTAL), + "https://era.ithaca.xyz/era1/index.html" => Bytes::from_static(ITHACA), + "https://mainnet.era1.nimbus.team/checksums.txt" | + "https://era1.ethportal.net/checksums.txt" | + "https://era.ithaca.xyz/era1/checksums.txt" => Bytes::from_static(CHECKSUMS), + "https://era1.ethportal.net/mainnet-00000-5ec1ffb8.era1" | + "https://mainnet.era1.nimbus.team/mainnet-00000-5ec1ffb8.era1" | + "https://era.ithaca.xyz/era1/mainnet-00000-5ec1ffb8.era1" => { + Bytes::from_static(MAINNET_0) } - } + "https://era1.ethportal.net/mainnet-00001-a5364e9a.era1" | + "https://mainnet.era1.nimbus.team/mainnet-00001-a5364e9a.era1" | + "https://era.ithaca.xyz/era1/mainnet-00001-a5364e9a.era1" => { + Bytes::from_static(MAINNET_1) + } + v => unimplemented!("Unexpected URL \"{v}\""), + })])) } } diff --git a/crates/era-downloader/tests/it/stream.rs b/crates/era-downloader/tests/it/stream.rs index 24fb43b7250..eb7dc2da727 100644 --- a/crates/era-downloader/tests/it/stream.rs +++ b/crates/era-downloader/tests/it/stream.rs @@ -9,13 +9,13 @@ use test_case::test_case; #[test_case("https://mainnet.era1.nimbus.team/"; "nimbus")] #[test_case("https://era1.ethportal.net/"; "ethportal")] -#[test_case("https://era.ithaca.xyz/era1/"; "ithaca")] +#[test_case("https://era.ithaca.xyz/era1/index.html"; "ithaca")] #[tokio::test] async fn test_streaming_files_after_fetching_file_list(url: &str) { let base_url = Url::from_str(url).unwrap(); let folder = tempdir().unwrap(); - let folder = folder.path().to_owned().into_boxed_path(); - let client = EraClient::new(StubClient, base_url, folder.clone()); + let folder = folder.path(); + let client = EraClient::new(StubClient, base_url, folder); let mut stream = EraStream::new( client, @@ -32,3 +32,20 @@ async fn test_streaming_files_after_fetching_file_list(url: &str) { assert_eq!(actual_file.as_ref(), expected_file.as_ref()); } + +#[tokio::test] +async fn test_streaming_files_after_fetching_file_list_into_missing_folder_fails() { + let base_url = Url::from_str("https://era.ithaca.xyz/era1/index.html").unwrap(); + let folder = tempdir().unwrap().path().to_owned(); + let client = EraClient::new(StubClient, base_url, folder); + + let mut stream = EraStream::new( + client, + EraStreamConfig::default().with_max_files(2).with_max_concurrent_downloads(1), + ); + + let actual_error = stream.next().await.unwrap().unwrap_err().to_string(); + let expected_error = "No such file or directory (os error 2)".to_owned(); + + assert_eq!(actual_error, expected_error); +} diff --git a/crates/era-utils/Cargo.toml b/crates/era-utils/Cargo.toml index 46764560a67..3363545faa0 100644 --- a/crates/era-utils/Cargo.toml +++ b/crates/era-utils/Cargo.toml @@ -11,6 +11,7 @@ exclude.workspace = true [dependencies] # alloy +alloy-consensus.workspace = true alloy-primitives.workspace = true # reth @@ -20,12 +21,12 @@ reth-era-downloader.workspace = true reth-etl.workspace = true reth-fs-util.workspace = true reth-provider.workspace = true +reth-stages-types.workspace = true reth-storage-api.workspace = true reth-primitives-traits.workspace = true # async -tokio.workspace = true -tokio.features = ["fs", "io-util"] +tokio = { workspace = true, features = ["fs", "io-util", "macros", "rt-multi-thread"] } futures-util.workspace = true # errors @@ -39,9 +40,7 @@ reth-provider.features = ["test-utils"] reth-db-common.workspace = true # async -tokio.workspace = true -tokio.features = ["fs", "io-util", "macros", "rt-multi-thread"] -futures.workspace = true +tokio-util.workspace = true bytes.workspace = true # http diff --git a/crates/era-utils/src/export.rs b/crates/era-utils/src/export.rs new file mode 100644 index 00000000000..6ccdba24262 --- /dev/null +++ b/crates/era-utils/src/export.rs @@ -0,0 +1,351 @@ +//! Logic to export from database era1 block history +//! and injecting them into era1 files with `Era1Writer`. + +use crate::calculate_td_by_number; +use alloy_consensus::BlockHeader; +use alloy_primitives::{BlockNumber, B256, U256}; +use eyre::{eyre, Result}; +use reth_era::{ + e2s_types::IndexEntry, + era1_file::Era1Writer, + era1_types::{BlockIndex, Era1Id}, + era_file_ops::{EraFileId, StreamWriter}, + execution_types::{ + Accumulator, BlockTuple, CompressedBody, CompressedHeader, CompressedReceipts, + TotalDifficulty, MAX_BLOCKS_PER_ERA1, + }, +}; +use reth_fs_util as fs; +use reth_storage_api::{BlockNumReader, BlockReader, HeaderProvider}; +use std::{ + path::PathBuf, + time::{Duration, Instant}, +}; +use tracing::{debug, info, warn}; + +const REPORT_INTERVAL_SECS: u64 = 10; +const ENTRY_HEADER_SIZE: usize = 8; +const VERSION_ENTRY_SIZE: usize = ENTRY_HEADER_SIZE; + +/// Configuration to export block history +/// to era1 files +#[derive(Clone, Debug)] +pub struct ExportConfig { + /// Directory to export era1 files to + pub dir: PathBuf, + /// First block to export + pub first_block_number: BlockNumber, + /// Last block to export + pub last_block_number: BlockNumber, + /// Number of blocks per era1 file + /// It can never be larger than `MAX_BLOCKS_PER_ERA1 = 8192` + /// See also <`https://github.com/eth-clients/e2store-format-specs/blob/main/formats/era1.md`> + pub max_blocks_per_file: u64, + /// Network name. + pub network: String, +} + +impl Default for ExportConfig { + fn default() -> Self { + Self { + dir: PathBuf::new(), + first_block_number: 0, + last_block_number: (MAX_BLOCKS_PER_ERA1 - 1) as u64, + max_blocks_per_file: MAX_BLOCKS_PER_ERA1 as u64, + network: "mainnet".to_string(), + } + } +} + +impl ExportConfig { + /// Validates the export configuration parameters + pub fn validate(&self) -> Result<()> { + if self.max_blocks_per_file > MAX_BLOCKS_PER_ERA1 as u64 { + return Err(eyre!( + "Max blocks per file ({}) exceeds ERA1 limit ({})", + self.max_blocks_per_file, + MAX_BLOCKS_PER_ERA1 + )); + } + + if self.max_blocks_per_file == 0 { + return Err(eyre!("Max blocks per file cannot be zero")); + } + + Ok(()) + } +} + +/// Fetches block history data from the provider +/// and prepares it for export to era1 files +/// for a given number of blocks then writes them to disk. +pub fn export

(provider: &P, config: &ExportConfig) -> Result> +where + P: BlockReader, +{ + config.validate()?; + info!( + "Exporting blockchain history from block {} to {} with this max of blocks per file of {}", + config.first_block_number, config.last_block_number, config.max_blocks_per_file + ); + + // Determine the actual last block to export + // best_block_number() might be outdated, so check actual block availability + let last_block_number = determine_export_range(provider, config)?; + + info!( + target: "era::history::export", + first = config.first_block_number, + last = last_block_number, + max_blocks_per_file = config.max_blocks_per_file, + "Preparing era1 export data" + ); + + if !config.dir.exists() { + fs::create_dir_all(&config.dir) + .map_err(|e| eyre!("Failed to create output directory: {}", e))?; + } + + let start_time = Instant::now(); + let mut last_report_time = Instant::now(); + let report_interval = Duration::from_secs(REPORT_INTERVAL_SECS); + + let mut created_files = Vec::new(); + let mut total_blocks_processed = 0; + + let mut total_difficulty = if config.first_block_number > 0 { + let prev_block_number = config.first_block_number - 1; + calculate_td_by_number(provider, prev_block_number)? + } else { + U256::ZERO + }; + + // Process blocks in chunks according to `max_blocks_per_file` + for start_block in + (config.first_block_number..=last_block_number).step_by(config.max_blocks_per_file as usize) + { + let end_block = (start_block + config.max_blocks_per_file - 1).min(last_block_number); + let block_count = (end_block - start_block + 1) as usize; + + info!( + target: "era::history::export", + "Processing blocks {start_block} to {end_block} ({block_count} blocks)" + ); + + let headers = provider.headers_range(start_block..=end_block)?; + + // Extract first 4 bytes of last block's state root as historical identifier + let historical_root = headers + .last() + .map(|header| { + let state_root = header.state_root(); + [state_root[0], state_root[1], state_root[2], state_root[3]] + }) + .unwrap_or([0u8; 4]); + + let era1_id = Era1Id::new(&config.network, start_block, block_count as u32) + .with_hash(historical_root); + + debug!("Final file name {}", era1_id.to_file_name()); + let file_path = config.dir.join(era1_id.to_file_name()); + let file = std::fs::File::create(&file_path)?; + let mut writer = Era1Writer::new(file); + writer.write_version()?; + + let mut offsets = Vec::::with_capacity(block_count); + let mut position = VERSION_ENTRY_SIZE as i64; + let mut blocks_written = 0; + let mut final_header_data = Vec::new(); + + for (i, header) in headers.into_iter().enumerate() { + let expected_block_number = start_block + i as u64; + + let (compressed_header, compressed_body, compressed_receipts) = compress_block_data( + provider, + header, + expected_block_number, + &mut total_difficulty, + )?; + + // Save last block's header data for accumulator + if expected_block_number == end_block { + final_header_data = compressed_header.data.clone(); + } + + let difficulty = TotalDifficulty::new(total_difficulty); + + let header_size = compressed_header.data.len() + ENTRY_HEADER_SIZE; + let body_size = compressed_body.data.len() + ENTRY_HEADER_SIZE; + let receipts_size = compressed_receipts.data.len() + ENTRY_HEADER_SIZE; + let difficulty_size = 32 + ENTRY_HEADER_SIZE; // U256 is 32 + 8 bytes header overhead + let total_size = (header_size + body_size + receipts_size + difficulty_size) as i64; + + let block_tuple = BlockTuple::new( + compressed_header, + compressed_body, + compressed_receipts, + difficulty, + ); + + offsets.push(position); + position += total_size; + + writer.write_block(&block_tuple)?; + blocks_written += 1; + total_blocks_processed += 1; + + if last_report_time.elapsed() >= report_interval { + info!( + target: "era::history::export", + "Export progress: block {expected_block_number}/{last_block_number} ({:.2}%) - elapsed: {:?}", + (total_blocks_processed as f64) / + ((last_block_number - config.first_block_number + 1) as f64) * + 100.0, + start_time.elapsed() + ); + last_report_time = Instant::now(); + } + } + if blocks_written > 0 { + let accumulator_hash = + B256::from_slice(&final_header_data[0..32.min(final_header_data.len())]); + let accumulator = Accumulator::new(accumulator_hash); + let block_index = BlockIndex::new(start_block, offsets); + + writer.write_accumulator(&accumulator)?; + writer.write_block_index(&block_index)?; + writer.flush()?; + created_files.push(file_path.clone()); + + info!( + target: "era::history::export", + "Wrote ERA1 file: {file_path:?} with {blocks_written} blocks" + ); + } + } + + info!( + target: "era::history::export", + "Successfully wrote {} ERA1 files in {:?}", + created_files.len(), + start_time.elapsed() + ); + + Ok(created_files) +} + +// Determines the actual last block number that can be exported, +// Uses `headers_range` fallback when `best_block_number` is stale due to static file storage. +fn determine_export_range

(provider: &P, config: &ExportConfig) -> Result +where + P: HeaderProvider + BlockNumReader, +{ + let best_block_number = provider.best_block_number()?; + + let last_block_number = if best_block_number < config.last_block_number { + warn!( + "Last block {} is beyond current head {}, setting last = head", + config.last_block_number, best_block_number + ); + + // Check if more blocks are actually available beyond what `best_block_number()` reports + if let Ok(headers) = provider.headers_range(best_block_number..=config.last_block_number) { + if let Some(last_header) = headers.last() { + let highest_block = last_header.number(); + info!("Found highest available block {} via headers_range", highest_block); + highest_block + } else { + warn!("No headers found in range, using best_block_number {}", best_block_number); + best_block_number + } + } else { + warn!("headers_range failed, using best_block_number {}", best_block_number); + best_block_number + } + } else { + config.last_block_number + }; + + Ok(last_block_number) +} + +// Compresses block data and returns compressed components with metadata +fn compress_block_data

( + provider: &P, + header: P::Header, + expected_block_number: BlockNumber, + total_difficulty: &mut U256, +) -> Result<(CompressedHeader, CompressedBody, CompressedReceipts)> +where + P: BlockReader, +{ + let actual_block_number = header.number(); + + if expected_block_number != actual_block_number { + return Err(eyre!("Expected block {expected_block_number}, got {actual_block_number}")); + } + + let body = provider + .block_by_number(actual_block_number)? + .ok_or_else(|| eyre!("Block body not found for block {}", actual_block_number))?; + + let receipts = provider + .receipts_by_block(actual_block_number.into())? + .ok_or_else(|| eyre!("Receipts not found for block {}", actual_block_number))?; + + *total_difficulty += header.difficulty(); + + let compressed_header = CompressedHeader::from_header(&header)?; + let compressed_body = CompressedBody::from_body(&body)?; + let compressed_receipts = CompressedReceipts::from_encodable_list(&receipts) + .map_err(|e| eyre!("Failed to compress receipts: {}", e))?; + + Ok((compressed_header, compressed_body, compressed_receipts)) +} + +#[cfg(test)] +mod tests { + use crate::ExportConfig; + use reth_era::execution_types::MAX_BLOCKS_PER_ERA1; + use tempfile::tempdir; + + #[test] + fn test_export_config_validation() { + let temp_dir = tempdir().unwrap(); + + // Default config should pass + let default_config = ExportConfig::default(); + assert!(default_config.validate().is_ok(), "Default config should be valid"); + + // Exactly at the limit should pass + let limit_config = + ExportConfig { max_blocks_per_file: MAX_BLOCKS_PER_ERA1 as u64, ..Default::default() }; + assert!(limit_config.validate().is_ok(), "Config at ERA1 limit should pass validation"); + + // Valid config should pass + let valid_config = ExportConfig { + dir: temp_dir.path().to_path_buf(), + max_blocks_per_file: 1000, + ..Default::default() + }; + assert!(valid_config.validate().is_ok(), "Valid config should pass validation"); + + // Zero blocks per file should fail + let zero_blocks_config = ExportConfig { + max_blocks_per_file: 0, // Invalid + ..Default::default() + }; + let result = zero_blocks_config.validate(); + assert!(result.is_err(), "Zero blocks per file should fail validation"); + assert!(result.unwrap_err().to_string().contains("cannot be zero")); + + // Exceeding era1 limit should fail + let oversized_config = ExportConfig { + max_blocks_per_file: MAX_BLOCKS_PER_ERA1 as u64 + 1, // Invalid + ..Default::default() + }; + let result = oversized_config.validate(); + assert!(result.is_err(), "Oversized blocks per file should fail validation"); + assert!(result.unwrap_err().to_string().contains("exceeds ERA1 limit")); + } +} diff --git a/crates/era-utils/src/history.rs b/crates/era-utils/src/history.rs index 58637286bf9..58d5e383c37 100644 --- a/crates/era-utils/src/history.rs +++ b/crates/era-utils/src/history.rs @@ -1,4 +1,5 @@ -use alloy_primitives::{BlockHash, BlockNumber}; +use alloy_consensus::BlockHeader; +use alloy_primitives::{BlockHash, BlockNumber, U256}; use futures_util::{Stream, StreamExt}; use reth_db_api::{ cursor::{DbCursorRO, DbCursorRW}, @@ -7,37 +8,62 @@ use reth_db_api::{ transaction::{DbTx, DbTxMut}, RawKey, RawTable, RawValue, }; -use reth_era::{era1_file::Era1Reader, execution_types::DecodeCompressed}; +use reth_era::{ + e2s_types::E2sError, + era1_file::{BlockTupleIterator, Era1Reader}, + era_file_ops::StreamReader, + execution_types::BlockTuple, + DecodeCompressed, +}; use reth_era_downloader::EraMeta; use reth_etl::Collector; use reth_fs_util as fs; use reth_primitives_traits::{Block, FullBlockBody, FullBlockHeader, NodePrimitives}; use reth_provider::{ - BlockWriter, ProviderError, StaticFileProviderFactory, StaticFileSegment, StaticFileWriter, + providers::StaticFileProviderRWRefMut, BlockReader, BlockWriter, StaticFileProviderFactory, + StaticFileSegment, StaticFileWriter, +}; +use reth_stages_types::{ + CheckpointBlockRange, EntitiesCheckpoint, HeadersCheckpoint, StageCheckpoint, StageId, +}; +use reth_storage_api::{ + errors::ProviderResult, DBProvider, DatabaseProviderFactory, NodePrimitivesProvider, + StageCheckpointWriter, +}; +use std::{ + collections::Bound, + error::Error, + fmt::{Display, Formatter}, + io::{Read, Seek}, + iter::Map, + ops::RangeBounds, + sync::mpsc, }; -use reth_storage_api::{DBProvider, HeaderProvider, NodePrimitivesProvider, StorageLocation}; -use std::sync::mpsc; use tracing::info; /// Imports blocks from `downloader` using `provider`. /// /// Returns current block height. -pub fn import( +pub fn import( mut downloader: Downloader, - provider: &P, - mut hash_collector: Collector, + provider_factory: &PF, + hash_collector: &mut Collector, ) -> eyre::Result where B: Block

, BH: FullBlockHeader + Value, BB: FullBlockBody< - Transaction = <

::Primitives as NodePrimitives>::SignedTx, + Transaction = <<::ProviderRW as NodePrimitivesProvider>::Primitives as NodePrimitives>::SignedTx, OmmerHeader = BH, >, Downloader: Stream> + Send + 'static + Unpin, Era: EraMeta + Send + 'static, - P: DBProvider + StaticFileProviderFactory + BlockWriter, -

::Primitives: NodePrimitives, + PF: DatabaseProviderFactory< + ProviderRW: BlockWriter + + DBProvider + + StaticFileProviderFactory> + + StageCheckpointWriter, + > + StaticFileProviderFactory::ProviderRW as NodePrimitivesProvider>::Primitives>, { let (tx, rx) = mpsc::channel(); @@ -49,89 +75,288 @@ where tx.send(None) }); - let static_file_provider = provider.static_file_provider(); + let static_file_provider = provider_factory.static_file_provider(); // Consistency check of expected headers in static files vs DB is done on provider::sync_gap // when poll_execute_ready is polled. - let mut last_header_number = static_file_provider + let mut height = static_file_provider .get_highest_static_file_block(StaticFileSegment::Headers) .unwrap_or_default(); - // Find the latest total difficulty - let mut td = static_file_provider - .header_td_by_number(last_header_number)? - .ok_or(ProviderError::TotalDifficultyNotFound(last_header_number))?; + while let Some(meta) = rx.recv()? { + let from = height; + let provider = provider_factory.database_provider_rw()?; - // Although headers were downloaded in reverse order, the collector iterates it in ascending - // order - let mut writer = static_file_provider.latest_writer(StaticFileSegment::Headers)?; + height = process( + &meta?, + &mut static_file_provider.latest_writer(StaticFileSegment::Headers)?, + &provider, + hash_collector, + height.., + )?; - while let Some(meta) = rx.recv()? { - let meta = meta?; - let file = fs::open(meta.as_ref())?; - let mut reader = Era1Reader::new(file); + save_stage_checkpoints(&provider, from, height, height, height)?; - for block in reader.iter() { - let block = block?; - let header: BH = block.header.decode()?; - let body: BB = block.body.decode()?; - let number = header.number(); + provider.commit()?; + } - if number == 0 { - continue; - } + let provider = provider_factory.database_provider_rw()?; - let hash = header.hash_slow(); - last_header_number = number; + build_index(&provider, hash_collector)?; - // Increase total difficulty - td += header.difficulty(); + provider.commit()?; - // Append to Headers segment - writer.append_header(&header, td, &hash)?; + Ok(height) +} - // Write bodies to database. - provider.append_block_bodies( - vec![(header.number(), Some(body))], - // We are writing transactions directly to static files. - StorageLocation::StaticFiles, - )?; +/// Saves progress of ERA import into stages sync. +/// +/// Since the ERA import does the same work as `HeaderStage` and `BodyStage`, it needs to inform +/// these stages that this work has already been done. Otherwise, there might be some conflict with +/// database integrity. +pub fn save_stage_checkpoints

( + provider: &P, + from: BlockNumber, + to: BlockNumber, + processed: u64, + total: u64, +) -> ProviderResult<()> +where + P: StageCheckpointWriter, +{ + provider.save_stage_checkpoint( + StageId::Headers, + StageCheckpoint::new(to).with_headers_stage_checkpoint(HeadersCheckpoint { + block_range: CheckpointBlockRange { from, to }, + progress: EntitiesCheckpoint { processed, total }, + }), + )?; + provider.save_stage_checkpoint( + StageId::Bodies, + StageCheckpoint::new(to) + .with_entities_stage_checkpoint(EntitiesCheckpoint { processed, total }), + )?; + Ok(()) +} - hash_collector.insert(hash, number)?; +/// Extracts block headers and bodies from `meta` and appends them using `writer` and `provider`. +/// +/// Collects hash to height using `hash_collector`. +/// +/// Skips all blocks below the [`start_bound`] of `block_numbers` and stops when reaching past the +/// [`end_bound`] or the end of the file. +/// +/// Returns last block height. +/// +/// [`start_bound`]: RangeBounds::start_bound +/// [`end_bound`]: RangeBounds::end_bound +pub fn process( + meta: &Era, + writer: &mut StaticFileProviderRWRefMut<'_,

::Primitives>, + provider: &P, + hash_collector: &mut Collector, + block_numbers: impl RangeBounds, +) -> eyre::Result +where + B: Block

, + BH: FullBlockHeader + Value, + BB: FullBlockBody< + Transaction = <

::Primitives as NodePrimitives>::SignedTx, + OmmerHeader = BH, + >, + Era: EraMeta + ?Sized, + P: DBProvider + NodePrimitivesProvider + BlockWriter, +

::Primitives: NodePrimitives, +{ + let reader = open(meta)?; + let iter = + reader + .iter() + .map(Box::new(decode) + as Box) -> eyre::Result<(BH, BB)>>); + let iter = ProcessIter { iter, era: meta }; + + process_iter(iter, writer, provider, hash_collector, block_numbers) +} + +type ProcessInnerIter = + Map, Box) -> eyre::Result<(BH, BB)>>>; + +/// An iterator that wraps era file extraction. After the final item [`EraMeta::mark_as_processed`] +/// is called to ensure proper cleanup. +#[derive(Debug)] +pub struct ProcessIter<'a, Era: ?Sized, R: Read, BH, BB> +where + BH: FullBlockHeader + Value, + BB: FullBlockBody, +{ + iter: ProcessInnerIter, + era: &'a Era, +} + +impl<'a, Era: EraMeta + ?Sized, R: Read, BH, BB> Display for ProcessIter<'a, Era, R, BH, BB> +where + BH: FullBlockHeader + Value, + BB: FullBlockBody, +{ + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + Display::fmt(&self.era.path().to_string_lossy(), f) + } +} + +impl<'a, Era, R, BH, BB> Iterator for ProcessIter<'a, Era, R, BH, BB> +where + R: Read + Seek, + Era: EraMeta + ?Sized, + BH: FullBlockHeader + Value, + BB: FullBlockBody, +{ + type Item = eyre::Result<(BH, BB)>; + + fn next(&mut self) -> Option { + match self.iter.next() { + Some(item) => Some(item), + None => match self.era.mark_as_processed() { + Ok(..) => None, + Err(e) => Some(Err(e)), + }, } + } +} - info!(target: "era::history::import", "Processed {}", meta.as_ref().to_string_lossy()); +/// Opens the era file described by `meta`. +pub fn open(meta: &Era) -> eyre::Result> +where + Era: EraMeta + ?Sized, +{ + let file = fs::open(meta.path())?; + let reader = Era1Reader::new(file); - meta.mark_as_processed()?; + Ok(reader) +} + +/// Extracts a pair of [`FullBlockHeader`] and [`FullBlockBody`] from [`BlockTuple`]. +pub fn decode(block: Result) -> eyre::Result<(BH, BB)> +where + BH: FullBlockHeader + Value, + BB: FullBlockBody, + E: From + Error + Send + Sync + 'static, +{ + let block = block?; + let header: BH = block.header.decode()?; + let body: BB = block.body.decode()?; + + Ok((header, body)) +} + +/// Extracts block headers and bodies from `iter` and appends them using `writer` and `provider`. +/// +/// Adds on to `total_difficulty` and collects hash to height using `hash_collector`. +/// +/// Skips all blocks below the [`start_bound`] of `block_numbers` and stops when reaching past the +/// [`end_bound`] or the end of the file. +/// +/// Returns last block height. +/// +/// [`start_bound`]: RangeBounds::start_bound +/// [`end_bound`]: RangeBounds::end_bound +pub fn process_iter( + mut iter: impl Iterator>, + writer: &mut StaticFileProviderRWRefMut<'_,

::Primitives>, + provider: &P, + hash_collector: &mut Collector, + block_numbers: impl RangeBounds, +) -> eyre::Result +where + B: Block

, + BH: FullBlockHeader + Value, + BB: FullBlockBody< + Transaction = <

::Primitives as NodePrimitives>::SignedTx, + OmmerHeader = BH, + >, + P: DBProvider + NodePrimitivesProvider + BlockWriter, +

::Primitives: NodePrimitives, +{ + let mut last_header_number = match block_numbers.start_bound() { + Bound::Included(&number) => number, + Bound::Excluded(&number) => number.saturating_add(1), + Bound::Unbounded => 0, + }; + let target = match block_numbers.end_bound() { + Bound::Included(&number) => Some(number), + Bound::Excluded(&number) => Some(number.saturating_sub(1)), + Bound::Unbounded => None, + }; + + for block in &mut iter { + let (header, body) = block?; + let number = header.number(); + + if number <= last_header_number { + continue; + } + if let Some(target) = target && + number > target + { + break; + } + + let hash = header.hash_slow(); + last_header_number = number; + + // Append to Headers segment + writer.append_header(&header, &hash)?; + + // Write bodies to database. + provider.append_block_bodies(vec![(header.number(), Some(body))])?; + + hash_collector.insert(hash, number)?; } + Ok(last_header_number) +} + +/// Dumps the contents of `hash_collector` into [`tables::HeaderNumbers`]. +pub fn build_index( + provider: &P, + hash_collector: &mut Collector, +) -> eyre::Result<()> +where + B: Block

, + BH: FullBlockHeader + Value, + BB: FullBlockBody< + Transaction = <

::Primitives as NodePrimitives>::SignedTx, + OmmerHeader = BH, + >, + P: DBProvider + NodePrimitivesProvider + BlockWriter, +

::Primitives: NodePrimitives, +{ let total_headers = hash_collector.len(); info!(target: "era::history::import", total = total_headers, "Writing headers hash index"); // Database cursor for hash to number index let mut cursor_header_numbers = provider.tx_ref().cursor_write::>()?; - let mut first_sync = false; - // If we only have the genesis block hash, then we are at first sync, and we can remove it, // add it to the collector and use tx.append on all hashes. - if provider.tx_ref().entries::>()? == 1 { - if let Some((hash, block_number)) = cursor_header_numbers.last()? { - if block_number.value()? == 0 { - hash_collector.insert(hash.key()?, 0)?; - cursor_header_numbers.delete_current()?; - first_sync = true; - } - } - } + let first_sync = if provider.tx_ref().entries::>()? == 1 && + let Some((hash, block_number)) = cursor_header_numbers.last()? && + block_number.value()? == 0 + { + hash_collector.insert(hash.key()?, 0)?; + cursor_header_numbers.delete_current()?; + true + } else { + false + }; - let interval = (total_headers / 10).max(1); + let interval = (total_headers / 10).max(8192); // Build block hash to block number index for (index, hash_to_number) in hash_collector.iter()?.enumerate() { let (hash, number) = hash_to_number?; - if index > 0 && index % interval == 0 && total_headers > 100 { + if index != 0 && index.is_multiple_of(interval) { info!(target: "era::history::import", progress = %format!("{:.2}%", (index as f64 / total_headers as f64) * 100.0), "Writing headers hash index"); } @@ -145,5 +370,30 @@ where } } - Ok(last_header_number) + Ok(()) +} + +/// Calculates the total difficulty for a given block number by summing the difficulty +/// of all blocks from genesis to the given block. +/// +/// Very expensive - iterates through all blocks in batches of 1000. +/// +/// Returns an error if any block is missing. +pub fn calculate_td_by_number

(provider: &P, num: BlockNumber) -> eyre::Result +where + P: BlockReader, +{ + let mut total_difficulty = U256::ZERO; + let mut start = 0; + + while start <= num { + let end = (start + 1000 - 1).min(num); + + total_difficulty += + provider.headers_range(start..=end)?.iter().map(|h| h.difficulty()).sum::(); + + start = end + 1; + } + + Ok(total_difficulty) } diff --git a/crates/era-utils/src/lib.rs b/crates/era-utils/src/lib.rs index b72f0eb0c0c..13a5ceefe92 100644 --- a/crates/era-utils/src/lib.rs +++ b/crates/era-utils/src/lib.rs @@ -1,8 +1,19 @@ //! Utilities to store history from downloaded ERA files with storage-api +//! and export it to recreate era1 files. //! //! The import is downloaded using [`reth_era_downloader`] and parsed using [`reth_era`]. mod history; +/// Export block history data from the database to recreate era1 files. +mod export; + +/// Export history from storage-api between 2 blocks +/// with parameters defined in [`ExportConfig`]. +pub use export::{export, ExportConfig}; + /// Imports history from ERA files. -pub use history::import; +pub use history::{ + build_index, calculate_td_by_number, decode, import, open, process, process_iter, + save_stage_checkpoints, ProcessIter, +}; diff --git a/crates/era-utils/tests/it/genesis.rs b/crates/era-utils/tests/it/genesis.rs new file mode 100644 index 00000000000..0c35c458aac --- /dev/null +++ b/crates/era-utils/tests/it/genesis.rs @@ -0,0 +1,33 @@ +use reth_db_common::init::init_genesis; +use reth_era_utils::{export, ExportConfig}; +use reth_fs_util as fs; +use reth_provider::{test_utils::create_test_provider_factory, BlockReader}; +use tempfile::tempdir; + +#[test] +fn test_export_with_genesis_only() { + let provider_factory = create_test_provider_factory(); + init_genesis(&provider_factory).unwrap(); + let provider = provider_factory.provider().unwrap(); + assert!(provider.block_by_number(0).unwrap().is_some(), "Genesis block should exist"); + assert!(provider.block_by_number(1).unwrap().is_none(), "Block 1 should not exist"); + + let export_dir = tempdir().unwrap(); + let export_config = ExportConfig { dir: export_dir.path().to_owned(), ..Default::default() }; + + let exported_files = + export(&provider_factory.provider_rw().unwrap().0, &export_config).unwrap(); + + assert_eq!(exported_files.len(), 1, "Should export exactly one file"); + + let file_path = &exported_files[0]; + assert!(file_path.exists(), "Exported file should exist on disk"); + let file_name = file_path.file_name().unwrap().to_str().unwrap(); + assert!( + file_name.starts_with("mainnet-00000-00001-"), + "File should have correct prefix with era format" + ); + assert!(file_name.ends_with(".era1"), "File should have correct extension"); + let metadata = fs::metadata(file_path).unwrap(); + assert!(metadata.len() > 0, "Exported file should not be empty"); +} diff --git a/crates/era-utils/tests/it/history.rs b/crates/era-utils/tests/it/history.rs index 5dcf91f1c6b..8e720f1001b 100644 --- a/crates/era-utils/tests/it/history.rs +++ b/crates/era-utils/tests/it/history.rs @@ -1,21 +1,28 @@ -use alloy_primitives::bytes::Bytes; -use futures_util::{Stream, TryStreamExt}; -use reqwest::{Client, IntoUrl, Url}; +use crate::{ClientWithFakeIndex, ITHACA_ERA_INDEX_URL}; +use reqwest::{Client, Url}; use reth_db_common::init::init_genesis; -use reth_era_downloader::{EraClient, EraStream, EraStreamConfig, HttpClient}; +use reth_era::execution_types::MAX_BLOCKS_PER_ERA1; +use reth_era_downloader::{EraClient, EraStream, EraStreamConfig}; +use reth_era_utils::{export, import, ExportConfig}; use reth_etl::Collector; -use reth_provider::test_utils::create_test_provider_factory; -use std::{future::Future, str::FromStr}; +use reth_fs_util as fs; +use reth_provider::{test_utils::create_test_provider_factory, BlockNumReader, BlockReader}; +use std::str::FromStr; use tempfile::tempdir; +const EXPORT_FIRST_BLOCK: u64 = 0; +const EXPORT_BLOCKS_PER_FILE: u64 = 250; +const EXPORT_TOTAL_BLOCKS: u64 = 900; +const EXPORT_LAST_BLOCK: u64 = EXPORT_FIRST_BLOCK + EXPORT_TOTAL_BLOCKS - 1; + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_history_imports_from_fresh_state_successfully() { // URL where the ERA1 files are hosted - let url = Url::from_str("https://era.ithaca.xyz/era1/").unwrap(); + let url = Url::from_str(ITHACA_ERA_INDEX_URL).unwrap(); // Directory where the ERA1 files will be downloaded to let folder = tempdir().unwrap(); - let folder = folder.path().to_owned().into_boxed_path(); + let folder = folder.path(); let client = EraClient::new(ClientWithFakeIndex(Client::new()), url, folder); @@ -28,46 +35,125 @@ async fn test_history_imports_from_fresh_state_successfully() { let folder = tempdir().unwrap(); let folder = Some(folder.path().to_owned()); - let hash_collector = Collector::new(4096, folder); + let mut hash_collector = Collector::new(4096, folder); let expected_block_number = 8191; - let actual_block_number = - reth_era_utils::import(stream, &pf.provider_rw().unwrap().0, hash_collector).unwrap(); + let actual_block_number = import(stream, &pf, &mut hash_collector).unwrap(); assert_eq!(actual_block_number, expected_block_number); } -/// An HTTP client pre-programmed with canned answer to index. -/// -/// Passes any other calls to a real HTTP client! -#[derive(Debug, Clone)] -struct ClientWithFakeIndex(Client); - -impl HttpClient for ClientWithFakeIndex { - fn get( - &self, - url: U, - ) -> impl Future< - Output = eyre::Result> + Send + Sync + Unpin>, - > + Send - + Sync { - let url = url.into_url().unwrap(); - - async move { - match url.to_string().as_str() { - "https://era.ithaca.xyz/era1/index.html" => { - Ok(Box::new(futures::stream::once(Box::pin(async move { - Ok(bytes::Bytes::from_static(b"mainnet-00000-5ec1ffb8.era1")) - }))) - as Box> + Send + Sync + Unpin>) - } - _ => { - let response = Client::get(&self.0, url).send().await?; - - Ok(Box::new(response.bytes_stream().map_err(|e| eyre::Error::new(e))) - as Box> + Send + Sync + Unpin>) - } - } - } +/// Test that verifies the complete roundtrip from importing to exporting era1 files. +/// It validates : +/// - Downloads the first era1 file from ithaca's url and import the file data, into the database +/// - Exports blocks from database back to era1 format +/// - Ensure exported files have correct structure and naming +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_roundtrip_export_after_import() { + // URL where the ERA1 files are hosted + let url = Url::from_str(ITHACA_ERA_INDEX_URL).unwrap(); + let download_folder = tempdir().unwrap(); + let download_folder = download_folder.path().to_owned().into_boxed_path(); + + let client = EraClient::new(ClientWithFakeIndex(Client::new()), url, download_folder); + let config = EraStreamConfig::default().with_max_files(1).with_max_concurrent_downloads(1); + + let stream = EraStream::new(client, config); + let pf = create_test_provider_factory(); + init_genesis(&pf).unwrap(); + + let folder = tempdir().unwrap(); + let folder = Some(folder.path().to_owned()); + let mut hash_collector = Collector::new(4096, folder); + + // Import blocks from one era1 file into database + let last_imported_block_height = import(stream, &pf, &mut hash_collector).unwrap(); + + assert_eq!(last_imported_block_height, 8191); + let provider_ref = pf.provider_rw().unwrap().0; + let best_block = provider_ref.best_block_number().unwrap(); + + assert!(best_block <= 8191, "Best block {best_block} should not exceed imported count"); + + // Verify some blocks exist in the database + for &block_num in &[0, 1, 2, 10, 50, 100, 5000, 8190, 8191] { + let block_exists = provider_ref.block_by_number(block_num).unwrap().is_some(); + assert!(block_exists, "Block {block_num} should exist after importing 8191 blocks"); + } + + // The import was verified let's start the export! + + // 900 blocks will be exported from 0 to 899 + // It should be split into 3 files of 250 blocks each, and the last file with 150 blocks + let export_folder = tempdir().unwrap(); + let export_config = ExportConfig { + dir: export_folder.path().to_path_buf(), + first_block_number: EXPORT_FIRST_BLOCK, // 0 + last_block_number: EXPORT_LAST_BLOCK, // 899 + max_blocks_per_file: EXPORT_BLOCKS_PER_FILE, // 250 blocks per file + network: "mainnet".to_string(), + }; + + // Export blocks from database to era1 files + let exported_files = export(&provider_ref, &export_config).expect("Export should succeed"); + + // Calculate how many files we expect based on the configuration + // We expect 4 files for 900 blocks: first 3 files with 250 blocks each, + // then 150 for the last file + let expected_files_number = EXPORT_TOTAL_BLOCKS.div_ceil(EXPORT_BLOCKS_PER_FILE); + + assert_eq!( + exported_files.len(), + expected_files_number as usize, + "Should create {expected_files_number} files for {EXPORT_TOTAL_BLOCKS} blocks with {EXPORT_BLOCKS_PER_FILE} blocks per file" + ); + + for (i, file_path) in exported_files.iter().enumerate() { + // Verify file exists and has content + assert!(file_path.exists(), "File {} should exist", i + 1); + let file_size = fs::metadata(file_path).unwrap().len(); + assert!(file_size > 0, "File {} should not be empty", i + 1); + + // Calculate expected file parameters + let file_start_block = EXPORT_FIRST_BLOCK + (i as u64 * EXPORT_BLOCKS_PER_FILE); + let remaining_blocks = EXPORT_TOTAL_BLOCKS - (i as u64 * EXPORT_BLOCKS_PER_FILE); + let blocks_numbers_per_file = std::cmp::min(EXPORT_BLOCKS_PER_FILE, remaining_blocks); + + // Verify chunking : first 3 files have 250 blocks, last file has 150 blocks - 900 total + let expected_blocks = if i < 3 { 250 } else { 150 }; + assert_eq!( + blocks_numbers_per_file, + expected_blocks, + "File {} should contain exactly {} blocks, got {}", + i + 1, + expected_blocks, + blocks_numbers_per_file + ); + + // Verify format: mainnet-{era_number:05}-{era_count:05}-{8hexchars}.era1 + let era_number = file_start_block / MAX_BLOCKS_PER_ERA1 as u64; + + // Era count is always 1 for this test, as we are only exporting one era + let expected_prefix = format!("mainnet-{:05}-{:05}-", era_number, 1); + + let file_name = file_path.file_name().unwrap().to_str().unwrap(); + assert!( + file_name.starts_with(&expected_prefix), + "File {} should start with '{expected_prefix}', got '{file_name}'", + i + 1 + ); + + // Verify the hash part is 8 characters + let hash_start = expected_prefix.len(); + let hash_end = file_name.len() - 5; // remove ".era1" + let hash_part = &file_name[hash_start..hash_end]; + assert_eq!( + hash_part.len(), + 8, + "File {} hash should be 8 characters, got {} in '{}'", + i + 1, + hash_part.len(), + file_name + ); } } diff --git a/crates/era-utils/tests/it/main.rs b/crates/era-utils/tests/it/main.rs index 9a035cdf7da..94805c5b356 100644 --- a/crates/era-utils/tests/it/main.rs +++ b/crates/era-utils/tests/it/main.rs @@ -1,5 +1,49 @@ //! Root module for test modules, so that the tests are built into a single binary. +use alloy_primitives::bytes::Bytes; +use futures_util::{stream, Stream, TryStreamExt}; +use reqwest::{Client, IntoUrl}; +use reth_era_downloader::HttpClient; +use tokio_util::either::Either; + +// Url where the ERA1 files are hosted +const ITHACA_ERA_INDEX_URL: &str = "https://era.ithaca.xyz/era1/index.html"; + +// The response containing one file that the fake client will return when the index Url is requested +const GENESIS_ITHACA_INDEX_RESPONSE: &[u8] = b"mainnet-00000-5ec1ffb8.era1"; + +mod genesis; mod history; const fn main() {} + +/// An HTTP client that fakes the file list to always show one known file +/// +/// but passes all other calls including actual downloads to a real HTTP client +/// +/// In that way, only one file is used but downloads are still performed from the original source. +#[derive(Debug, Clone)] +struct ClientWithFakeIndex(Client); + +impl HttpClient for ClientWithFakeIndex { + async fn get( + &self, + url: U, + ) -> eyre::Result> + Send + Sync + Unpin> { + let url = url.into_url()?; + + match url.to_string().as_str() { + ITHACA_ERA_INDEX_URL => { + // Create a static stream without boxing + let stream = + stream::iter(vec![Ok(Bytes::from_static(GENESIS_ITHACA_INDEX_RESPONSE))]); + Ok(Either::Left(stream)) + } + _ => { + let response = Client::get(&self.0, url).send().await?; + let stream = response.bytes_stream().map_err(|e| eyre::Error::new(e)); + Ok(Either::Right(stream)) + } + } + } +} diff --git a/crates/era/Cargo.toml b/crates/era/Cargo.toml index d8259ec813c..09d5b8b9180 100644 --- a/crates/era/Cargo.toml +++ b/crates/era/Cargo.toml @@ -1,5 +1,6 @@ [package] name = "reth-era" +description = "e2store and era1 files core logic" version.workspace = true edition.workspace = true rust-version.workspace = true diff --git a/crates/era/src/consensus_types.rs b/crates/era/src/consensus_types.rs new file mode 100644 index 00000000000..cdcc77ce57a --- /dev/null +++ b/crates/era/src/consensus_types.rs @@ -0,0 +1,235 @@ +//! Consensus types for Era post-merge history files + +use crate::{ + e2s_types::{E2sError, Entry}, + DecodeCompressedSsz, +}; +use snap::{read::FrameDecoder, write::FrameEncoder}; +use ssz::Decode; +use std::io::{Read, Write}; + +/// `CompressedSignedBeaconBlock` record type: [0x01, 0x00] +pub const COMPRESSED_SIGNED_BEACON_BLOCK: [u8; 2] = [0x01, 0x00]; + +/// `CompressedBeaconState` record type: [0x02, 0x00] +pub const COMPRESSED_BEACON_STATE: [u8; 2] = [0x02, 0x00]; + +/// Compressed signed beacon block +/// +/// See also . +#[derive(Debug, Clone)] +pub struct CompressedSignedBeaconBlock { + /// Snappy-compressed ssz-encoded `SignedBeaconBlock` + pub data: Vec, +} + +impl CompressedSignedBeaconBlock { + /// Create a new [`CompressedSignedBeaconBlock`] from compressed data + pub const fn new(data: Vec) -> Self { + Self { data } + } + + /// Create from ssz-encoded block by compressing it with snappy + pub fn from_ssz(ssz_data: &[u8]) -> Result { + let mut compressed = Vec::new(); + { + let mut encoder = FrameEncoder::new(&mut compressed); + + Write::write_all(&mut encoder, ssz_data).map_err(|e| { + E2sError::SnappyCompression(format!("Failed to compress signed beacon block: {e}")) + })?; + + encoder.flush().map_err(|e| { + E2sError::SnappyCompression(format!("Failed to flush encoder: {e}")) + })?; + } + Ok(Self { data: compressed }) + } + + /// Decompress to get the original ssz-encoded signed beacon block + pub fn decompress(&self) -> Result, E2sError> { + let mut decoder = FrameDecoder::new(self.data.as_slice()); + let mut decompressed = Vec::new(); + Read::read_to_end(&mut decoder, &mut decompressed).map_err(|e| { + E2sError::SnappyDecompression(format!("Failed to decompress signed beacon block: {e}")) + })?; + + Ok(decompressed) + } + + /// Convert to an [`Entry`] + pub fn to_entry(&self) -> Entry { + Entry::new(COMPRESSED_SIGNED_BEACON_BLOCK, self.data.clone()) + } + + /// Create from an [`Entry`] + pub fn from_entry(entry: &Entry) -> Result { + if entry.entry_type != COMPRESSED_SIGNED_BEACON_BLOCK { + return Err(E2sError::Ssz(format!( + "Invalid entry type for CompressedSignedBeaconBlock: expected {:02x}{:02x}, got {:02x}{:02x}", + COMPRESSED_SIGNED_BEACON_BLOCK[0], + COMPRESSED_SIGNED_BEACON_BLOCK[1], + entry.entry_type[0], + entry.entry_type[1] + ))); + } + + Ok(Self { data: entry.data.clone() }) + } + + /// Decode the compressed signed beacon block into ssz bytes + pub fn decode_to_ssz(&self) -> Result, E2sError> { + self.decompress() + } +} + +impl DecodeCompressedSsz for CompressedSignedBeaconBlock { + fn decode(&self) -> Result { + let ssz_bytes = self.decompress()?; + T::from_ssz_bytes(&ssz_bytes).map_err(|e| { + E2sError::Ssz(format!("Failed to decode SSZ data into target type: {e:?}")) + }) + } +} + +/// Compressed beacon state +/// +/// See also . +#[derive(Debug, Clone)] +pub struct CompressedBeaconState { + /// Snappy-compressed ssz-encoded `BeaconState` + pub data: Vec, +} + +impl CompressedBeaconState { + /// Create a new [`CompressedBeaconState`] from compressed data + pub const fn new(data: Vec) -> Self { + Self { data } + } + + /// Compress with snappy from ssz-encoded state + pub fn from_ssz(ssz_data: &[u8]) -> Result { + let mut compressed = Vec::new(); + { + let mut encoder = FrameEncoder::new(&mut compressed); + + Write::write_all(&mut encoder, ssz_data).map_err(|e| { + E2sError::SnappyCompression(format!("Failed to compress beacon state: {e}")) + })?; + + encoder.flush().map_err(|e| { + E2sError::SnappyCompression(format!("Failed to flush encoder: {e}")) + })?; + } + Ok(Self { data: compressed }) + } + + /// Decompress to get the original ssz-encoded beacon state + pub fn decompress(&self) -> Result, E2sError> { + let mut decoder = FrameDecoder::new(self.data.as_slice()); + let mut decompressed = Vec::new(); + Read::read_to_end(&mut decoder, &mut decompressed).map_err(|e| { + E2sError::SnappyDecompression(format!("Failed to decompress beacon state: {e}")) + })?; + + Ok(decompressed) + } + + /// Convert to an [`Entry`] + pub fn to_entry(&self) -> Entry { + Entry::new(COMPRESSED_BEACON_STATE, self.data.clone()) + } + + /// Create from an [`Entry`] + pub fn from_entry(entry: &Entry) -> Result { + if entry.entry_type != COMPRESSED_BEACON_STATE { + return Err(E2sError::Ssz(format!( + "Invalid entry type for CompressedBeaconState: expected {:02x}{:02x}, got {:02x}{:02x}", + COMPRESSED_BEACON_STATE[0], + COMPRESSED_BEACON_STATE[1], + entry.entry_type[0], + entry.entry_type[1] + ))); + } + + Ok(Self { data: entry.data.clone() }) + } + + /// Decode the compressed beacon state into ssz bytes + pub fn decode_to_ssz(&self) -> Result, E2sError> { + self.decompress() + } +} + +impl DecodeCompressedSsz for CompressedBeaconState { + fn decode(&self) -> Result { + let ssz_bytes = self.decompress()?; + T::from_ssz_bytes(&ssz_bytes).map_err(|e| { + E2sError::Ssz(format!("Failed to decode SSZ data into target type: {e:?}")) + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_signed_beacon_block_compression_roundtrip() { + let ssz_data = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + + let compressed_block = CompressedSignedBeaconBlock::from_ssz(&ssz_data).unwrap(); + let decompressed = compressed_block.decompress().unwrap(); + + assert_eq!(decompressed, ssz_data); + } + + #[test] + fn test_beacon_state_compression_roundtrip() { + let ssz_data = vec![10, 9, 8, 7, 6, 5, 4, 3, 2, 1]; + + let compressed_state = CompressedBeaconState::from_ssz(&ssz_data).unwrap(); + let decompressed = compressed_state.decompress().unwrap(); + + assert_eq!(decompressed, ssz_data); + } + + #[test] + fn test_entry_conversion_signed_beacon_block() { + let ssz_data = vec![1, 2, 3, 4, 5]; + let compressed_block = CompressedSignedBeaconBlock::from_ssz(&ssz_data).unwrap(); + + let entry = compressed_block.to_entry(); + assert_eq!(entry.entry_type, COMPRESSED_SIGNED_BEACON_BLOCK); + + let recovered = CompressedSignedBeaconBlock::from_entry(&entry).unwrap(); + let recovered_ssz = recovered.decode_to_ssz().unwrap(); + + assert_eq!(recovered_ssz, ssz_data); + } + + #[test] + fn test_entry_conversion_beacon_state() { + let ssz_data = vec![5, 4, 3, 2, 1]; + let compressed_state = CompressedBeaconState::from_ssz(&ssz_data).unwrap(); + + let entry = compressed_state.to_entry(); + assert_eq!(entry.entry_type, COMPRESSED_BEACON_STATE); + + let recovered = CompressedBeaconState::from_entry(&entry).unwrap(); + let recovered_ssz = recovered.decode_to_ssz().unwrap(); + + assert_eq!(recovered_ssz, ssz_data); + } + + #[test] + fn test_invalid_entry_type() { + let invalid_entry = Entry::new([0xFF, 0xFF], vec![1, 2, 3]); + + let result = CompressedSignedBeaconBlock::from_entry(&invalid_entry); + assert!(result.is_err()); + + let result = CompressedBeaconState::from_entry(&invalid_entry); + assert!(result.is_err()); + } +} diff --git a/crates/era/src/e2s_file.rs b/crates/era/src/e2s_file.rs index 201730adc60..e1b6989a0f3 100644 --- a/crates/era/src/e2s_file.rs +++ b/crates/era/src/e2s_file.rs @@ -36,7 +36,7 @@ impl E2StoreReader { Entry::read(&mut self.reader) } - /// Iterate through all entries, including the version entry + /// Read all entries from the file, including the version entry pub fn entries(&mut self) -> Result, E2sError> { // Reset reader to beginning self.reader.seek(SeekFrom::Start(0))?; @@ -74,7 +74,8 @@ impl E2StoreWriter { } /// Write the version entry as the first entry in the file. - /// This must be called before writing any other entries. + /// If not called explicitly, it will be written automatically before the first non-version + /// entry. pub fn write_version(&mut self) -> Result<(), E2sError> { if self.has_written_version { return Ok(()); diff --git a/crates/era/src/e2s_types.rs b/crates/era/src/e2s_types.rs index c2d4734c2e7..f14bfe56e86 100644 --- a/crates/era/src/e2s_types.rs +++ b/crates/era/src/e2s_types.rs @@ -165,3 +165,96 @@ impl Entry { self.entry_type == SLOT_INDEX } } + +/// Serialize and deserialize index entries with format: +/// `starting-number | offsets... | count` +pub trait IndexEntry: Sized { + /// Get the entry type identifier for this index + fn entry_type() -> [u8; 2]; + + /// Create a new instance with starting number and offsets + fn new(starting_number: u64, offsets: Vec) -> Self; + + /// Get the starting number - can be starting slot or block number for example + fn starting_number(&self) -> u64; + + /// Get the offsets vector + fn offsets(&self) -> &[i64]; + + /// Convert to an [`Entry`] for storage in an e2store file + /// Format: starting-number | offset1 | offset2 | ... | count + fn to_entry(&self) -> Entry { + let mut data = Vec::with_capacity(8 + self.offsets().len() * 8 + 8); + + // Add starting number + data.extend_from_slice(&self.starting_number().to_le_bytes()); + + // Add all offsets + data.extend(self.offsets().iter().flat_map(|offset| offset.to_le_bytes())); + + // Encode count - 8 bytes again + let count = self.offsets().len() as i64; + data.extend_from_slice(&count.to_le_bytes()); + + Entry::new(Self::entry_type(), data) + } + + /// Create from an [`Entry`] + fn from_entry(entry: &Entry) -> Result { + let expected_type = Self::entry_type(); + + if entry.entry_type != expected_type { + return Err(E2sError::Ssz(format!( + "Invalid entry type: expected {:02x}{:02x}, got {:02x}{:02x}", + expected_type[0], expected_type[1], entry.entry_type[0], entry.entry_type[1] + ))); + } + + if entry.data.len() < 16 { + return Err(E2sError::Ssz( + "Index entry too short: need at least 16 bytes for starting_number and count" + .to_string(), + )); + } + + // Extract count from last 8 bytes + let count_bytes = &entry.data[entry.data.len() - 8..]; + let count = i64::from_le_bytes( + count_bytes + .try_into() + .map_err(|_| E2sError::Ssz("Failed to read count bytes".to_string()))?, + ) as usize; + + // Verify entry has correct size + let expected_len = 8 + count * 8 + 8; + if entry.data.len() != expected_len { + return Err(E2sError::Ssz(format!( + "Index entry has incorrect length: expected {expected_len}, got {}", + entry.data.len() + ))); + } + + // Extract starting number from first 8 bytes + let starting_number = u64::from_le_bytes( + entry.data[0..8] + .try_into() + .map_err(|_| E2sError::Ssz("Failed to read starting_number bytes".to_string()))?, + ); + + // Extract all offsets + let mut offsets = Vec::with_capacity(count); + for i in 0..count { + let start = 8 + i * 8; + let end = start + 8; + let offset_bytes = &entry.data[start..end]; + let offset = i64::from_le_bytes( + offset_bytes + .try_into() + .map_err(|_| E2sError::Ssz(format!("Failed to read offset {i} bytes")))?, + ); + offsets.push(offset); + } + + Ok(Self::new(starting_number, offsets)) + } +} diff --git a/crates/era/src/era1_file.rs b/crates/era/src/era1_file.rs index 7f3b558ca8b..dc34ddef42b 100644 --- a/crates/era/src/era1_file.rs +++ b/crates/era/src/era1_file.rs @@ -3,12 +3,13 @@ //! The structure of an Era1 file follows the specification: //! `Version | block-tuple* | other-entries* | Accumulator | BlockIndex` //! -//! See also +//! See also . use crate::{ e2s_file::{E2StoreReader, E2StoreWriter}, - e2s_types::{E2sError, Entry, Version}, + e2s_types::{E2sError, Entry, IndexEntry, Version}, era1_types::{BlockIndex, Era1Group, Era1Id, BLOCK_INDEX}, + era_file_ops::{EraFileFormat, FileReader, StreamReader, StreamWriter}, execution_types::{ self, Accumulator, BlockTuple, CompressedBody, CompressedHeader, CompressedReceipts, TotalDifficulty, MAX_BLOCKS_PER_ERA1, @@ -19,7 +20,6 @@ use std::{ collections::VecDeque, fs::File, io::{Read, Seek, Write}, - path::Path, }; /// Era1 file interface @@ -35,21 +35,38 @@ pub struct Era1File { pub id: Era1Id, } -impl Era1File { +impl EraFileFormat for Era1File { + type EraGroup = Era1Group; + type Id = Era1Id; + /// Create a new [`Era1File`] - pub const fn new(group: Era1Group, id: Era1Id) -> Self { + fn new(group: Era1Group, id: Era1Id) -> Self { Self { version: Version, group, id } } + fn version(&self) -> &Version { + &self.version + } + + fn group(&self) -> &Self::EraGroup { + &self.group + } + + fn id(&self) -> &Self::Id { + &self.id + } +} + +impl Era1File { /// Get a block by its number, if present in this file pub fn get_block_by_number(&self, number: BlockNumber) -> Option<&BlockTuple> { - let index = (number - self.group.block_index.starting_number) as usize; + let index = (number - self.group.block_index.starting_number()) as usize; (index < self.group.blocks.len()).then(|| &self.group.blocks[index]) } /// Get the range of block numbers contained in this file pub fn block_range(&self) -> std::ops::RangeInclusive { - let start = self.group.block_index.starting_number; + let start = self.group.block_index.starting_number(); let end = start + (self.group.blocks.len() as u64) - 1; start..=end } @@ -59,6 +76,7 @@ impl Era1File { self.block_range().contains(&number) } } + /// Reader for Era1 files that builds on top of [`E2StoreReader`] #[derive(Debug)] pub struct Era1Reader { @@ -67,8 +85,8 @@ pub struct Era1Reader { /// An iterator of [`BlockTuple`] streaming from [`E2StoreReader`]. #[derive(Debug)] -pub struct BlockTupleIterator<'r, R: Read> { - reader: &'r mut E2StoreReader, +pub struct BlockTupleIterator { + reader: E2StoreReader, headers: VecDeque, bodies: VecDeque, receipts: VecDeque, @@ -78,8 +96,8 @@ pub struct BlockTupleIterator<'r, R: Read> { block_index: Option, } -impl<'r, R: Read> BlockTupleIterator<'r, R> { - fn new(reader: &'r mut E2StoreReader) -> Self { +impl BlockTupleIterator { + fn new(reader: E2StoreReader) -> Self { Self { reader, headers: Default::default(), @@ -93,7 +111,7 @@ impl<'r, R: Read> BlockTupleIterator<'r, R> { } } -impl<'r, R: Read + Seek> Iterator for BlockTupleIterator<'r, R> { +impl Iterator for BlockTupleIterator { type Item = Result; fn next(&mut self) -> Option { @@ -101,7 +119,7 @@ impl<'r, R: Read + Seek> Iterator for BlockTupleIterator<'r, R> { } } -impl<'r, R: Read + Seek> BlockTupleIterator<'r, R> { +impl BlockTupleIterator { fn next_result(&mut self) -> Result, E2sError> { loop { let Some(entry) = self.reader.read_next_entry()? else { @@ -154,20 +172,29 @@ impl<'r, R: Read + Seek> BlockTupleIterator<'r, R> { } } -impl Era1Reader { +impl StreamReader for Era1Reader { + type File = Era1File; + type Iterator = BlockTupleIterator; + /// Create a new [`Era1Reader`] - pub fn new(reader: R) -> Self { + fn new(reader: R) -> Self { Self { reader: E2StoreReader::new(reader) } } /// Returns an iterator of [`BlockTuple`] streaming from `reader`. - pub fn iter(&mut self) -> BlockTupleIterator<'_, R> { - BlockTupleIterator::new(&mut self.reader) + fn iter(self) -> BlockTupleIterator { + BlockTupleIterator::new(self.reader) } + fn read(self, network_name: String) -> Result { + self.read_and_assemble(network_name) + } +} + +impl Era1Reader { /// Reads and parses an Era1 file from the underlying reader, assembling all components /// into a complete [`Era1File`] with an [`Era1Id`] that includes the provided network name. - pub fn read(&mut self, network_name: String) -> Result { + pub fn read_and_assemble(mut self, network_name: String) -> Result { // Validate version entry let _version_entry = match self.reader.read_version()? { Some(entry) if entry.is_version() => entry, @@ -215,25 +242,15 @@ impl Era1Reader { let id = Era1Id::new( network_name, - block_index.starting_number, - block_index.offsets.len() as u32, + block_index.starting_number(), + block_index.offsets().len() as u32, ); Ok(Era1File::new(group, id)) } } -impl Era1Reader { - /// Opens and reads an Era1 file from the given path - pub fn open>( - path: P, - network_name: impl Into, - ) -> Result { - let file = File::open(path).map_err(E2sError::Io)?; - let mut reader = Self::new(file); - reader.read(network_name.into()) - } -} +impl FileReader for Era1Reader {} /// Writer for Era1 files that builds on top of [`E2StoreWriter`] #[derive(Debug)] @@ -245,9 +262,11 @@ pub struct Era1Writer { has_written_block_index: bool, } -impl Era1Writer { +impl StreamWriter for Era1Writer { + type File = Era1File; + /// Create a new [`Era1Writer`] - pub fn new(writer: W) -> Self { + fn new(writer: W) -> Self { Self { writer: E2StoreWriter::new(writer), has_written_version: false, @@ -258,7 +277,7 @@ impl Era1Writer { } /// Write the version entry - pub fn write_version(&mut self) -> Result<(), E2sError> { + fn write_version(&mut self) -> Result<(), E2sError> { if self.has_written_version { return Ok(()); } @@ -269,7 +288,7 @@ impl Era1Writer { } /// Write a complete [`Era1File`] to the underlying writer - pub fn write_era1_file(&mut self, era1_file: &Era1File) -> Result<(), E2sError> { + fn write_file(&mut self, era1_file: &Era1File) -> Result<(), E2sError> { // Write version self.write_version()?; @@ -300,6 +319,13 @@ impl Era1Writer { Ok(()) } + /// Flush any buffered data to the underlying writer + fn flush(&mut self) -> Result<(), E2sError> { + self.writer.flush() + } +} + +impl Era1Writer { /// Write a single block tuple pub fn write_block( &mut self, @@ -336,27 +362,6 @@ impl Era1Writer { Ok(()) } - /// Write the accumulator - pub fn write_accumulator(&mut self, accumulator: &Accumulator) -> Result<(), E2sError> { - if !self.has_written_version { - self.write_version()?; - } - - if self.has_written_accumulator { - return Err(E2sError::Ssz("Accumulator already written".to_string())); - } - - if self.has_written_block_index { - return Err(E2sError::Ssz("Cannot write accumulator after block index".to_string())); - } - - let accumulator_entry = accumulator.to_entry(); - self.writer.write_entry(&accumulator_entry)?; - self.has_written_accumulator = true; - - Ok(()) - } - /// Write the block index pub fn write_block_index(&mut self, block_index: &BlockIndex) -> Result<(), E2sError> { if !self.has_written_version { @@ -374,39 +379,36 @@ impl Era1Writer { Ok(()) } - /// Flush any buffered data to the underlying writer - pub fn flush(&mut self) -> Result<(), E2sError> { - self.writer.flush() - } -} + /// Write the accumulator + pub fn write_accumulator(&mut self, accumulator: &Accumulator) -> Result<(), E2sError> { + if !self.has_written_version { + self.write_version()?; + } -impl Era1Writer { - /// Creates a new file at the specified path and writes the [`Era1File`] to it - pub fn create>(path: P, era1_file: &Era1File) -> Result<(), E2sError> { - let file = File::create(path).map_err(E2sError::Io)?; - let mut writer = Self::new(file); - writer.write_era1_file(era1_file)?; - Ok(()) - } + if self.has_written_accumulator { + return Err(E2sError::Ssz("Accumulator already written".to_string())); + } - /// Creates a new file in the specified directory with a filename derived from the - /// [`Era1File`]'s ID using the standardized Era1 file naming convention - pub fn create_with_id>( - directory: P, - era1_file: &Era1File, - ) -> Result<(), E2sError> { - let filename = era1_file.id.to_file_name(); - let path = directory.as_ref().join(filename); - Self::create(path, era1_file) + if self.has_written_block_index { + return Err(E2sError::Ssz("Cannot write accumulator after block index".to_string())); + } + + let accumulator_entry = accumulator.to_entry(); + self.writer.write_entry(&accumulator_entry)?; + self.has_written_accumulator = true; + Ok(()) } } #[cfg(test)] mod tests { use super::*; - use crate::execution_types::{ - Accumulator, BlockTuple, CompressedBody, CompressedHeader, CompressedReceipts, - TotalDifficulty, + use crate::{ + era_file_ops::FileWriter, + execution_types::{ + Accumulator, BlockTuple, CompressedBody, CompressedHeader, CompressedReceipts, + TotalDifficulty, + }, }; use alloy_primitives::{B256, U256}; use std::io::Cursor; @@ -464,11 +466,11 @@ mod tests { let mut buffer = Vec::new(); { let mut writer = Era1Writer::new(&mut buffer); - writer.write_era1_file(&era1_file)?; + writer.write_file(&era1_file)?; } // Read back from memory buffer - let mut reader = Era1Reader::new(Cursor::new(&buffer)); + let reader = Era1Reader::new(Cursor::new(&buffer)); let read_era1 = reader.read("testnet".to_string())?; // Verify core properties diff --git a/crates/era/src/era1_types.rs b/crates/era/src/era1_types.rs index 135f7225f60..ef239f3e164 100644 --- a/crates/era/src/era1_types.rs +++ b/crates/era/src/era1_types.rs @@ -3,12 +3,13 @@ //! See also use crate::{ - e2s_types::{E2sError, Entry}, - execution_types::{Accumulator, BlockTuple}, + e2s_types::{Entry, IndexEntry}, + era_file_ops::EraFileId, + execution_types::{Accumulator, BlockTuple, MAX_BLOCKS_PER_ERA1}, }; use alloy_primitives::BlockNumber; -/// `BlockIndex` record: ['i', '2'] +/// `BlockIndex` record: ['f', '2'] pub const BLOCK_INDEX: [u8; 2] = [0x66, 0x32]; /// File content in an Era1 file @@ -25,7 +26,7 @@ pub struct Era1Group { /// Accumulator is hash tree root of block headers and difficulties pub accumulator: Accumulator, - /// Block index, optional, omitted for genesis era + /// Block index, required pub block_index: BlockIndex, } @@ -38,6 +39,7 @@ impl Era1Group { ) -> Self { Self { blocks, accumulator, block_index, other_entries: Vec::new() } } + /// Add another entry to this group pub fn add_entry(&mut self, entry: Entry) { self.other_entries.push(entry); @@ -52,18 +54,13 @@ impl Era1Group { #[derive(Debug, Clone)] pub struct BlockIndex { /// Starting block number - pub starting_number: BlockNumber, + starting_number: BlockNumber, /// Offsets to data at each block number - pub offsets: Vec, + offsets: Vec, } impl BlockIndex { - /// Create a new [`BlockIndex`] - pub const fn new(starting_number: BlockNumber, offsets: Vec) -> Self { - Self { starting_number, offsets } - } - /// Get the offset for a specific block number pub fn offset_for_block(&self, block_number: BlockNumber) -> Option { if block_number < self.starting_number { @@ -73,72 +70,23 @@ impl BlockIndex { let index = (block_number - self.starting_number) as usize; self.offsets.get(index).copied() } +} - /// Convert to an [`Entry`] for storage in an e2store file - pub fn to_entry(&self) -> Entry { - // Format: starting-(block)-number | index | index | index ... | count - let mut data = Vec::with_capacity(8 + self.offsets.len() * 8 + 8); - - // Add starting block number - data.extend_from_slice(&self.starting_number.to_le_bytes()); - - // Add all offsets - for offset in &self.offsets { - data.extend_from_slice(&offset.to_le_bytes()); - } - - // Add count - data.extend_from_slice(&(self.offsets.len() as i64).to_le_bytes()); - - Entry::new(BLOCK_INDEX, data) +impl IndexEntry for BlockIndex { + fn new(starting_number: u64, offsets: Vec) -> Self { + Self { starting_number, offsets } } - /// Create from an [`Entry`] - pub fn from_entry(entry: &Entry) -> Result { - if entry.entry_type != BLOCK_INDEX { - return Err(E2sError::Ssz(format!( - "Invalid entry type for BlockIndex: expected {:02x}{:02x}, got {:02x}{:02x}", - BLOCK_INDEX[0], BLOCK_INDEX[1], entry.entry_type[0], entry.entry_type[1] - ))); - } - - if entry.data.len() < 16 { - return Err(E2sError::Ssz(String::from( - "BlockIndex entry too short to contain starting block number and count", - ))); - } - - // Extract starting block number = first 8 bytes - let mut starting_number_bytes = [0u8; 8]; - starting_number_bytes.copy_from_slice(&entry.data[0..8]); - let starting_number = u64::from_le_bytes(starting_number_bytes); - - // Extract count = last 8 bytes - let mut count_bytes = [0u8; 8]; - count_bytes.copy_from_slice(&entry.data[entry.data.len() - 8..]); - let count = u64::from_le_bytes(count_bytes) as usize; - - // Verify that the entry has the correct size - let expected_size = 8 + count * 8 + 8; - if entry.data.len() != expected_size { - return Err(E2sError::Ssz(format!( - "BlockIndex entry has incorrect size: expected {}, got {}", - expected_size, - entry.data.len() - ))); - } + fn entry_type() -> [u8; 2] { + BLOCK_INDEX + } - // Extract all offsets - let mut offsets = Vec::with_capacity(count); - for i in 0..count { - let start = 8 + i * 8; - let end = start + 8; - let mut offset_bytes = [0u8; 8]; - offset_bytes.copy_from_slice(&entry.data[start..end]); - offsets.push(i64::from_le_bytes(offset_bytes)); - } + fn starting_number(&self) -> u64 { + self.starting_number + } - Ok(Self { starting_number, offsets }) + fn offsets(&self) -> &[i64] { + &self.offsets } } @@ -155,6 +103,7 @@ pub struct Era1Id { pub block_count: u32, /// Optional hash identifier for this file + /// First 4 bytes of the last historical root in the last state in the era file pub hash: Option<[u8; 4]>, } @@ -174,22 +123,49 @@ impl Era1Id { self } - /// Convert to file name following the era1 file naming: - /// `--.era1` - /// inspired from era file naming convention in + // Helper function to calculate the number of eras per era1 file, + // If the user can decide how many blocks per era1 file there are, we need to calculate it. + // Most of the time it should be 1, but it can never be more than 2 eras per file + // as there is a maximum of 8192 blocks per era1 file. + const fn calculate_era_count(&self, first_era: u64) -> u64 { + // Calculate the actual last block number in the range + let last_block = self.start_block + self.block_count as u64 - 1; + // Find which era the last block belongs to + let last_era = last_block / MAX_BLOCKS_PER_ERA1 as u64; + // Count how many eras we span + last_era - first_era + 1 + } +} + +impl EraFileId for Era1Id { + fn network_name(&self) -> &str { + &self.network_name + } + + fn start_number(&self) -> u64 { + self.start_block + } + + fn count(&self) -> u32 { + self.block_count + } + /// Convert to file name following the era file naming: + /// `---.era(1)` /// /// See also - pub fn to_file_name(&self) -> String { + fn to_file_name(&self) -> String { + // Find which era the first block belongs to + let era_number = self.start_block / MAX_BLOCKS_PER_ERA1 as u64; + let era_count = self.calculate_era_count(era_number); if let Some(hash) = self.hash { - // Format with zero-padded era number and hash: - // For example network-00000-5ec1ffb8.era1 format!( - "{}-{:05}-{:02x}{:02x}{:02x}{:02x}.era1", - self.network_name, self.start_block, hash[0], hash[1], hash[2], hash[3] + "{}-{:05}-{:05}-{:02x}{:02x}{:02x}{:02x}.era1", + self.network_name, era_number, era_count, hash[0], hash[1], hash[2], hash[3] ) } else { - // Original format without hash - format!("{}-{}-{}.era1", self.network_name, self.start_block, self.block_count) + // era spec format with placeholder hash when no hash available + // Format: `---00000000.era1` + format!("{}-{:05}-{:05}-00000000.era1", self.network_name, era_number, era_count) } } } @@ -197,29 +173,37 @@ impl Era1Id { #[cfg(test)] mod tests { use super::*; - use crate::execution_types::{ - CompressedBody, CompressedHeader, CompressedReceipts, TotalDifficulty, + use crate::{ + test_utils::{create_sample_block, create_test_block_with_compressed_data}, + DecodeCompressed, }; + use alloy_consensus::ReceiptWithBloom; use alloy_primitives::{B256, U256}; - /// Helper function to create a sample block tuple - fn create_sample_block(data_size: usize) -> BlockTuple { - // Create a compressed header with very sample data - let header_data = vec![0xAA; data_size]; - let header = CompressedHeader::new(header_data); - - // Create a compressed body - let body_data = vec![0xBB; data_size * 2]; - let body = CompressedBody::new(body_data); - - // Create compressed receipts - let receipts_data = vec![0xCC; data_size]; - let receipts = CompressedReceipts::new(receipts_data); - - let difficulty = TotalDifficulty::new(U256::from(data_size)); - - // Create and return the block tuple - BlockTuple::new(header, body, receipts, difficulty) + #[test] + fn test_alloy_components_decode_and_receipt_in_bloom() { + // Create a block tuple from compressed data + let block: BlockTuple = create_test_block_with_compressed_data(30); + + // Decode and decompress the block header + let header: alloy_consensus::Header = block.header.decode().unwrap(); + assert_eq!(header.number, 30, "Header block number should match"); + assert_eq!(header.difficulty, U256::from(30 * 1000), "Header difficulty should match"); + assert_eq!(header.gas_limit, 5000000, "Gas limit should match"); + assert_eq!(header.gas_used, 21000, "Gas used should match"); + assert_eq!(header.timestamp, 1609459200 + 30, "Timestamp should match"); + assert_eq!(header.base_fee_per_gas, Some(10), "Base fee per gas should match"); + assert!(header.withdrawals_root.is_some(), "Should have withdrawals root"); + assert!(header.blob_gas_used.is_none(), "Should not have blob gas used"); + assert!(header.excess_blob_gas.is_none(), "Should not have excess blob gas"); + + let body: alloy_consensus::BlockBody = + block.body.decode().unwrap(); + assert_eq!(body.ommers.len(), 0, "Should have no ommers"); + assert!(body.withdrawals.is_some(), "Should have withdrawals field"); + + let receipts: Vec = block.receipts.decode().unwrap(); + assert_eq!(receipts.len(), 1, "Should have exactly 1 receipt"); } #[test] @@ -330,33 +314,33 @@ mod tests { #[test_case::test_case( Era1Id::new("mainnet", 0, 8192).with_hash([0x5e, 0xc1, 0xff, 0xb8]), - "mainnet-00000-5ec1ffb8.era1"; - "Mainnet 00000" + "mainnet-00000-00001-5ec1ffb8.era1"; + "Mainnet era 0" )] #[test_case::test_case( - Era1Id::new("mainnet", 12, 8192).with_hash([0x5e, 0xcb, 0x9b, 0xf9]), - "mainnet-00012-5ecb9bf9.era1"; - "Mainnet 00012" + Era1Id::new("mainnet", 8192, 8192).with_hash([0x5e, 0xcb, 0x9b, 0xf9]), + "mainnet-00001-00001-5ecb9bf9.era1"; + "Mainnet era 1" )] #[test_case::test_case( - Era1Id::new("sepolia", 5, 8192).with_hash([0x90, 0x91, 0x84, 0x72]), - "sepolia-00005-90918472.era1"; - "Sepolia 00005" + Era1Id::new("sepolia", 0, 8192).with_hash([0x90, 0x91, 0x84, 0x72]), + "sepolia-00000-00001-90918472.era1"; + "Sepolia era 0" )] #[test_case::test_case( - Era1Id::new("sepolia", 19, 8192).with_hash([0xfa, 0x77, 0x00, 0x19]), - "sepolia-00019-fa770019.era1"; - "Sepolia 00019" + Era1Id::new("sepolia", 155648, 8192).with_hash([0xfa, 0x77, 0x00, 0x19]), + "sepolia-00019-00001-fa770019.era1"; + "Sepolia era 19" )] #[test_case::test_case( Era1Id::new("mainnet", 1000, 100), - "mainnet-1000-100.era1"; + "mainnet-00000-00001-00000000.era1"; "ID without hash" )] #[test_case::test_case( - Era1Id::new("sepolia", 12345, 8192).with_hash([0xab, 0xcd, 0xef, 0x12]), - "sepolia-12345-abcdef12.era1"; - "Large block number" + Era1Id::new("sepolia", 101130240, 8192).with_hash([0xab, 0xcd, 0xef, 0x12]), + "sepolia-12345-00001-abcdef12.era1"; + "Large block number era 12345" )] fn test_era1id_file_naming(id: Era1Id, expected_file_name: &str) { let actual_file_name = id.to_file_name(); diff --git a/crates/era/src/era_file_ops.rs b/crates/era/src/era_file_ops.rs new file mode 100644 index 00000000000..469d6b78351 --- /dev/null +++ b/crates/era/src/era_file_ops.rs @@ -0,0 +1,124 @@ +//! Represents reading and writing operations' era file + +use crate::{e2s_types::Version, E2sError}; +use std::{ + fs::File, + io::{Read, Seek, Write}, + path::Path, +}; + +/// Represents era file with generic content and identifier types +pub trait EraFileFormat: Sized { + /// Content group type + type EraGroup; + + /// The identifier type + type Id: EraFileId; + + /// Get the version + fn version(&self) -> &Version; + + /// Get the content group + fn group(&self) -> &Self::EraGroup; + + /// Get the file identifier + fn id(&self) -> &Self::Id; + + /// Create a new instance + fn new(group: Self::EraGroup, id: Self::Id) -> Self; +} + +/// Era file identifiers +pub trait EraFileId: Clone { + /// Convert to standardized file name + fn to_file_name(&self) -> String; + + /// Get the network name + fn network_name(&self) -> &str; + + /// Get the starting number (block or slot) + fn start_number(&self) -> u64; + + /// Get the count of items + fn count(&self) -> u32; +} + +/// [`StreamReader`] for reading era-format files +pub trait StreamReader: Sized { + /// The file type the reader produces + type File: EraFileFormat; + + /// The iterator type for streaming data + type Iterator; + + /// Create a new reader + fn new(reader: R) -> Self; + + /// Read and parse the complete file + fn read(self, network_name: String) -> Result; + + /// Get an iterator for streaming processing + fn iter(self) -> Self::Iterator; +} + +/// [`FileReader`] provides reading era file operations for era files +pub trait FileReader: StreamReader { + /// Opens and reads an era file from the given path + fn open>( + path: P, + network_name: impl Into, + ) -> Result { + let file = File::open(path).map_err(E2sError::Io)?; + let reader = Self::new(file); + reader.read(network_name.into()) + } +} + +/// [`StreamWriter`] for writing era-format files +pub trait StreamWriter: Sized { + /// The file type this writer handles + type File: EraFileFormat; + + /// Create a new writer + fn new(writer: W) -> Self; + + /// Writer version + fn write_version(&mut self) -> Result<(), E2sError>; + + /// Write a complete era file + fn write_file(&mut self, file: &Self::File) -> Result<(), E2sError>; + + /// Flush any buffered data + fn flush(&mut self) -> Result<(), E2sError>; +} + +/// [`StreamWriter`] provides writing file operations for era files +pub trait FileWriter { + /// Era file type the writer handles + type File: EraFileFormat; + + /// Creates a new file at the specified path and writes the era file to it + fn create>(path: P, file: &Self::File) -> Result<(), E2sError>; + + /// Creates a file in the directory using standardized era naming + fn create_with_id>(directory: P, file: &Self::File) -> Result<(), E2sError>; +} + +impl> FileWriter for T { + type File = T::File; + + /// Creates a new file at the specified path and writes the era file to it + fn create>(path: P, file: &Self::File) -> Result<(), E2sError> { + let file_handle = File::create(path).map_err(E2sError::Io)?; + let mut writer = Self::new(file_handle); + writer.write_file(file)?; + Ok(()) + } + + /// Creates a file in the directory using standardized era naming + fn create_with_id>(directory: P, file: &Self::File) -> Result<(), E2sError> { + let filename = file.id().to_file_name(); + let path = directory.as_ref().join(filename); + Self::create(path, file) + } +} diff --git a/crates/era/src/era_types.rs b/crates/era/src/era_types.rs new file mode 100644 index 00000000000..a50b6f19281 --- /dev/null +++ b/crates/era/src/era_types.rs @@ -0,0 +1,289 @@ +//! Era types for `.era` files +//! +//! See also + +use crate::{ + consensus_types::{CompressedBeaconState, CompressedSignedBeaconBlock}, + e2s_types::{Entry, IndexEntry, SLOT_INDEX}, +}; + +/// Era file content group +/// +/// Format: `Version | block* | era-state | other-entries* | slot-index(block)? | slot-index(state)` +/// See also +#[derive(Debug)] +pub struct EraGroup { + /// Group including all blocks leading up to the era transition in slot order + pub blocks: Vec, + + /// State in the era transition slot + pub era_state: CompressedBeaconState, + + /// Other entries that don't fit into standard categories + pub other_entries: Vec, + + /// Block slot index, omitted for genesis era + pub slot_index: Option, + + /// State slot index + pub state_slot_index: SlotIndex, +} + +impl EraGroup { + /// Create a new era group + pub const fn new( + blocks: Vec, + era_state: CompressedBeaconState, + state_slot_index: SlotIndex, + ) -> Self { + Self { blocks, era_state, other_entries: Vec::new(), slot_index: None, state_slot_index } + } + + /// Create a new era group with block slot index + pub const fn with_block_index( + blocks: Vec, + era_state: CompressedBeaconState, + slot_index: SlotIndex, + state_slot_index: SlotIndex, + ) -> Self { + Self { + blocks, + era_state, + other_entries: Vec::new(), + slot_index: Some(slot_index), + state_slot_index, + } + } + + /// Check if this is a genesis era - no blocks yet + pub const fn is_genesis(&self) -> bool { + self.blocks.is_empty() && self.slot_index.is_none() + } + + /// Add another entry to this group + pub fn add_entry(&mut self, entry: Entry) { + self.other_entries.push(entry); + } +} + +/// [`SlotIndex`] records store offsets to data at specific slots +/// from the beginning of the index record to the beginning of the corresponding data. +/// +/// Format: `starting-slot | index | index | index ... | count` +/// +/// See also . +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SlotIndex { + /// Starting slot number + pub starting_slot: u64, + + /// Offsets to data at each slot + /// 0 indicates no data for that slot + pub offsets: Vec, +} + +impl SlotIndex { + /// Create a new slot index + pub const fn new(starting_slot: u64, offsets: Vec) -> Self { + Self { starting_slot, offsets } + } + + /// Get the number of slots covered by this index + pub const fn slot_count(&self) -> usize { + self.offsets.len() + } + + /// Get the offset for a specific slot + pub fn get_offset(&self, slot_index: usize) -> Option { + self.offsets.get(slot_index).copied() + } + + /// Check if a slot has data - non-zero offset + pub fn has_data_at_slot(&self, slot_index: usize) -> bool { + self.get_offset(slot_index).is_some_and(|offset| offset != 0) + } +} + +impl IndexEntry for SlotIndex { + fn new(starting_number: u64, offsets: Vec) -> Self { + Self { starting_slot: starting_number, offsets } + } + + fn entry_type() -> [u8; 2] { + SLOT_INDEX + } + + fn starting_number(&self) -> u64 { + self.starting_slot + } + + fn offsets(&self) -> &[i64] { + &self.offsets + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + e2s_types::{Entry, IndexEntry}, + test_utils::{create_beacon_block, create_beacon_state}, + }; + + #[test] + fn test_slot_index_roundtrip() { + let starting_slot = 1000; + let offsets = vec![100, 200, 300, 400, 500]; + + let slot_index = SlotIndex::new(starting_slot, offsets.clone()); + + let entry = slot_index.to_entry(); + + // Validate entry type + assert_eq!(entry.entry_type, SLOT_INDEX); + + // Convert back to slot index + let recovered = SlotIndex::from_entry(&entry).unwrap(); + + // Verify fields match + assert_eq!(recovered.starting_slot, starting_slot); + assert_eq!(recovered.offsets, offsets); + } + #[test] + fn test_slot_index_basic_operations() { + let starting_slot = 2000; + let offsets = vec![100, 200, 300]; + + let slot_index = SlotIndex::new(starting_slot, offsets); + + assert_eq!(slot_index.slot_count(), 3); + assert_eq!(slot_index.starting_slot, 2000); + } + + #[test] + fn test_slot_index_empty_slots() { + let starting_slot = 1000; + let offsets = vec![100, 0, 300, 0, 500]; + + let slot_index = SlotIndex::new(starting_slot, offsets); + + // Test that empty slots return false for has_data_at_slot + // slot 1000: offset 100 + assert!(slot_index.has_data_at_slot(0)); + // slot 1001: offset 0 - empty + assert!(!slot_index.has_data_at_slot(1)); + // slot 1002: offset 300 + assert!(slot_index.has_data_at_slot(2)); + // slot 1003: offset 0 - empty + assert!(!slot_index.has_data_at_slot(3)); + // slot 1004: offset 500 + assert!(slot_index.has_data_at_slot(4)); + } + + #[test] + fn test_era_group_basic_construction() { + let blocks = + vec![create_beacon_block(10), create_beacon_block(15), create_beacon_block(20)]; + let era_state = create_beacon_state(50); + let state_slot_index = SlotIndex::new(1000, vec![100, 200, 300]); + + let era_group = EraGroup::new(blocks, era_state, state_slot_index); + + // Verify initial state + assert_eq!(era_group.blocks.len(), 3); + assert_eq!(era_group.other_entries.len(), 0); + assert_eq!(era_group.slot_index, None); + assert_eq!(era_group.state_slot_index.starting_slot, 1000); + assert_eq!(era_group.state_slot_index.offsets, vec![100, 200, 300]); + } + + #[test] + fn test_era_group_with_block_index() { + let blocks = vec![create_beacon_block(10), create_beacon_block(15)]; + let era_state = create_beacon_state(50); + let block_slot_index = SlotIndex::new(500, vec![50, 100]); + let state_slot_index = SlotIndex::new(1000, vec![200, 300]); + + let era_group = + EraGroup::with_block_index(blocks, era_state, block_slot_index, state_slot_index); + + // Verify state with block index + assert_eq!(era_group.blocks.len(), 2); + assert_eq!(era_group.other_entries.len(), 0); + assert!(era_group.slot_index.is_some()); + + let block_index = era_group.slot_index.as_ref().unwrap(); + assert_eq!(block_index.starting_slot, 500); + assert_eq!(block_index.offsets, vec![50, 100]); + + assert_eq!(era_group.state_slot_index.starting_slot, 1000); + assert_eq!(era_group.state_slot_index.offsets, vec![200, 300]); + } + + #[test] + fn test_era_group_genesis_check() { + // Genesis era - no blocks, no block slot index + let era_state = create_beacon_state(50); + let state_slot_index = SlotIndex::new(0, vec![100]); + + let genesis_era = EraGroup::new(vec![], era_state, state_slot_index); + assert!(genesis_era.is_genesis()); + + // Non-genesis era - has blocks + let blocks = vec![create_beacon_block(10)]; + let era_state = create_beacon_state(50); + let state_slot_index = SlotIndex::new(1000, vec![100]); + + let normal_era = EraGroup::new(blocks, era_state, state_slot_index); + assert!(!normal_era.is_genesis()); + + // Non-genesis era - has block slot index + let era_state = create_beacon_state(50); + let block_slot_index = SlotIndex::new(500, vec![50]); + let state_slot_index = SlotIndex::new(1000, vec![100]); + + let era_with_index = + EraGroup::with_block_index(vec![], era_state, block_slot_index, state_slot_index); + assert!(!era_with_index.is_genesis()); + } + + #[test] + fn test_era_group_add_entries() { + let blocks = vec![create_beacon_block(10)]; + let era_state = create_beacon_state(50); + let state_slot_index = SlotIndex::new(1000, vec![100]); + + // Create and verify group + let mut era_group = EraGroup::new(blocks, era_state, state_slot_index); + assert_eq!(era_group.other_entries.len(), 0); + + // Create custom entries with different types + let entry1 = Entry::new([0x01, 0x01], vec![1, 2, 3, 4]); + let entry2 = Entry::new([0x02, 0x02], vec![5, 6, 7, 8]); + + // Add those entries + era_group.add_entry(entry1); + era_group.add_entry(entry2); + + // Verify entries were added correctly + assert_eq!(era_group.other_entries.len(), 2); + assert_eq!(era_group.other_entries[0].entry_type, [0x01, 0x01]); + assert_eq!(era_group.other_entries[0].data, vec![1, 2, 3, 4]); + assert_eq!(era_group.other_entries[1].entry_type, [0x02, 0x02]); + assert_eq!(era_group.other_entries[1].data, vec![5, 6, 7, 8]); + } + + #[test] + fn test_index_with_negative_offset() { + let mut data = Vec::new(); + data.extend_from_slice(&0u64.to_le_bytes()); + data.extend_from_slice(&(-1024i64).to_le_bytes()); + data.extend_from_slice(&0i64.to_le_bytes()); + data.extend_from_slice(&2i64.to_le_bytes()); + + let entry = Entry::new(SLOT_INDEX, data); + let index = SlotIndex::from_entry(&entry).unwrap(); + let parsed_offset = index.offsets[0]; + assert_eq!(parsed_offset, -1024); + } +} diff --git a/crates/era/src/execution_types.rs b/crates/era/src/execution_types.rs index 4591abb281a..6feb2873fbd 100644 --- a/crates/era/src/execution_types.rs +++ b/crates/era/src/execution_types.rs @@ -1,4 +1,4 @@ -//! Execution layer specific types for era1 files +//! Execution layer specific types for `.era1` files //! //! Contains implementations for compressed execution layer data structures: //! - [`CompressedHeader`] - Block header @@ -9,8 +9,72 @@ //! These types use Snappy compression to match the specification. //! //! See also - -use crate::e2s_types::{E2sError, Entry}; +//! +//! # Examples +//! +//! ## [`CompressedHeader`] +//! +//! ```rust +//! use alloy_consensus::Header; +//! use reth_era::{execution_types::CompressedHeader, DecodeCompressed}; +//! +//! let header = Header { number: 100, ..Default::default() }; +//! // Compress the header: rlp encoding and Snappy compression +//! let compressed = CompressedHeader::from_header(&header)?; +//! // Decompressed and decode typed compressed header +//! let decoded_header: Header = compressed.decode_header()?; +//! assert_eq!(decoded_header.number, 100); +//! # Ok::<(), reth_era::e2s_types::E2sError>(()) +//! ``` +//! +//! ## [`CompressedBody`] +//! +//! ```rust +//! use alloy_consensus::{BlockBody, Header}; +//! use alloy_primitives::Bytes; +//! use reth_era::{execution_types::CompressedBody, DecodeCompressed}; +//! use reth_ethereum_primitives::TransactionSigned; +//! +//! let body: BlockBody = BlockBody { +//! transactions: vec![Bytes::from(vec![1, 2, 3])], +//! ommers: vec![], +//! withdrawals: None, +//! }; +//! // Compress the body: rlp encoding and snappy compression +//! let compressed_body = CompressedBody::from_body(&body)?; +//! // Decode back to typed body by decompressing and decoding +//! let decoded_body: alloy_consensus::BlockBody = +//! compressed_body.decode()?; +//! assert_eq!(decoded_body.transactions.len(), 1); +//! # Ok::<(), reth_era::e2s_types::E2sError>(()) +//! ``` +//! +//! ## [`CompressedReceipts`] +//! +//! ```rust +//! use alloy_consensus::ReceiptWithBloom; +//! use reth_era::{execution_types::CompressedReceipts, DecodeCompressed}; +//! use reth_ethereum_primitives::{Receipt, TxType}; +//! +//! let receipt = Receipt { +//! tx_type: TxType::Legacy, +//! success: true, +//! cumulative_gas_used: 21000, +//! logs: vec![], +//! }; +//! let receipt_with_bloom = ReceiptWithBloom { receipt, logs_bloom: Default::default() }; +//! // Compress the receipt: rlp encoding and snappy compression +//! let compressed_receipt_data = CompressedReceipts::from_encodable(&receipt_with_bloom)?; +//! // Get raw receipt by decoding and decompressing compressed and encoded receipt +//! let decompressed_receipt = compressed_receipt_data.decode::()?; +//! assert_eq!(decompressed_receipt.receipt.cumulative_gas_used, 21000); +//! # Ok::<(), reth_era::e2s_types::E2sError>(()) +//! `````` + +use crate::{ + e2s_types::{E2sError, Entry}, + DecodeCompressed, +}; use alloy_consensus::{Block, BlockBody, Header}; use alloy_primitives::{B256, U256}; use alloy_rlp::{Decodable, Encodable}; @@ -96,12 +160,6 @@ pub struct CompressedHeader { pub data: Vec, } -/// Extension trait for generic decoding from compressed data -pub trait DecodeCompressed { - /// Decompress and decode the data into the given type - fn decode(&self) -> Result; -} - impl CompressedHeader { /// Create a new [`CompressedHeader`] from compressed data pub const fn new(data: Vec) -> Self { @@ -161,9 +219,9 @@ impl CompressedHeader { self.decode() } - /// Create a [`CompressedHeader`] from an `alloy_consensus::Header` - pub fn from_header(header: &Header) -> Result { - let encoder = SnappyRlpCodec::

::new(); + /// Create a [`CompressedHeader`] from a header + pub fn from_header(header: &H) -> Result { + let encoder = SnappyRlpCodec::new(); let compressed = encoder.encode(header)?; Ok(Self::new(compressed)) } @@ -248,9 +306,9 @@ impl CompressedBody { .map_err(|e| E2sError::Rlp(format!("Failed to decode RLP data: {e}"))) } - /// Create a [`CompressedBody`] from an `alloy_consensus::BlockBody` - pub fn from_body(body: &BlockBody) -> Result { - let encoder = SnappyRlpCodec::>::new(); + /// Create a [`CompressedBody`] from a block body (e.g. `alloy_consensus::BlockBody`) + pub fn from_body(body: &B) -> Result { + let encoder = SnappyRlpCodec::new(); let compressed = encoder.encode(body)?; Ok(Self::new(compressed)) } @@ -333,6 +391,18 @@ impl CompressedReceipts { let compressed = encoder.encode(data)?; Ok(Self::new(compressed)) } + /// Encode a list of receipts to RLP format + pub fn encode_receipts_to_rlp(receipts: &[T]) -> Result, E2sError> { + let mut rlp_data = Vec::new(); + alloy_rlp::encode_list(receipts, &mut rlp_data); + Ok(rlp_data) + } + + /// Encode and compress a list of receipts + pub fn from_encodable_list(receipts: &[T]) -> Result { + let rlp_data = Self::encode_receipts_to_rlp(receipts)?; + Self::from_rlp(&rlp_data) + } } impl DecodeCompressed for CompressedReceipts { @@ -490,34 +560,14 @@ impl BlockTuple { #[cfg(test)] mod tests { use super::*; + use crate::test_utils::{create_header, create_test_receipt, create_test_receipts}; use alloy_eips::eip4895::Withdrawals; - use alloy_primitives::{Address, Bytes, B64}; + use alloy_primitives::{Bytes, U256}; + use reth_ethereum_primitives::{Receipt, TxType}; #[test] fn test_header_conversion_roundtrip() { - let header = Header { - parent_hash: B256::default(), - ommers_hash: B256::default(), - beneficiary: Address::default(), - state_root: B256::default(), - transactions_root: B256::default(), - receipts_root: B256::default(), - logs_bloom: Default::default(), - difficulty: U256::from(123456u64), - number: 100, - gas_limit: 5000000, - gas_used: 21000, - timestamp: 1609459200, - extra_data: Bytes::default(), - mix_hash: B256::default(), - nonce: B64::default(), - base_fee_per_gas: Some(10), - withdrawals_root: None, - blob_gas_used: None, - excess_blob_gas: None, - parent_beacon_block_root: None, - requests_hash: None, - }; + let header = create_header(); let compressed_header = CompressedHeader::from_header(&header).unwrap(); @@ -583,29 +633,7 @@ mod tests { #[test] fn test_block_tuple_with_data() { // Create block with transactions and withdrawals - let header = Header { - parent_hash: B256::default(), - ommers_hash: B256::default(), - beneficiary: Address::default(), - state_root: B256::default(), - transactions_root: B256::default(), - receipts_root: B256::default(), - logs_bloom: Default::default(), - difficulty: U256::from(123456u64), - number: 100, - gas_limit: 5000000, - gas_used: 21000, - timestamp: 1609459200, - extra_data: Bytes::default(), - mix_hash: B256::default(), - nonce: B64::default(), - base_fee_per_gas: Some(10), - withdrawals_root: Some(B256::default()), - blob_gas_used: None, - excess_blob_gas: None, - parent_beacon_block_root: None, - requests_hash: None, - }; + let header = create_header(); let transactions = vec![Bytes::from(vec![1, 2, 3, 4]), Bytes::from(vec![5, 6, 7, 8])]; @@ -630,4 +658,63 @@ mod tests { assert_eq!(decoded_block.body.transactions[1], Bytes::from(vec![5, 6, 7, 8])); assert!(decoded_block.body.withdrawals.is_some()); } + + #[test] + fn test_single_receipt_compression_roundtrip() { + let test_receipt = create_test_receipt(TxType::Eip1559, true, 21000, 2); + + // Compress the receipt + let compressed_receipts = + CompressedReceipts::from_encodable(&test_receipt).expect("Failed to compress receipt"); + + // Verify compression + assert!(!compressed_receipts.data.is_empty()); + + // Decode the compressed receipt back + let decoded_receipt: Receipt = + compressed_receipts.decode().expect("Failed to decode compressed receipt"); + + // Verify that the decoded receipt matches the original + assert_eq!(decoded_receipt.tx_type, test_receipt.tx_type); + assert_eq!(decoded_receipt.success, test_receipt.success); + assert_eq!(decoded_receipt.cumulative_gas_used, test_receipt.cumulative_gas_used); + assert_eq!(decoded_receipt.logs.len(), test_receipt.logs.len()); + + // Verify each log + for (original_log, decoded_log) in test_receipt.logs.iter().zip(decoded_receipt.logs.iter()) + { + assert_eq!(decoded_log.address, original_log.address); + assert_eq!(decoded_log.data.topics(), original_log.data.topics()); + } + } + + #[test] + fn test_receipt_list_compression() { + let receipts = create_test_receipts(); + + // Compress the list of receipts + let compressed_receipts = CompressedReceipts::from_encodable_list(&receipts) + .expect("Failed to compress receipt list"); + + // Decode the compressed receipts back + // Note: most likely the decoding for real era files will be done to reach + // `Vec`` + let decoded_receipts: Vec = + compressed_receipts.decode().expect("Failed to decode compressed receipt list"); + + // Verify that the decoded receipts match the original + assert_eq!(decoded_receipts.len(), receipts.len()); + + for (original, decoded) in receipts.iter().zip(decoded_receipts.iter()) { + assert_eq!(decoded.tx_type, original.tx_type); + assert_eq!(decoded.success, original.success); + assert_eq!(decoded.cumulative_gas_used, original.cumulative_gas_used); + assert_eq!(decoded.logs.len(), original.logs.len()); + + for (original_log, decoded_log) in original.logs.iter().zip(decoded.logs.iter()) { + assert_eq!(decoded_log.address, original_log.address); + assert_eq!(decoded_log.data.topics(), original_log.data.topics()); + } + } + } } diff --git a/crates/era/src/lib.rs b/crates/era/src/lib.rs index 6007da18738..fd0596e9dfc 100644 --- a/crates/era/src/lib.rs +++ b/crates/era/src/lib.rs @@ -1,19 +1,40 @@ //! Era and Era1 files support for Ethereum history expiry. //! -//! -//! Era files are special instances of .e2s files with a strict content format -//! optimized for reading and long-term storage and distribution. -//! //! Era1 files use the same e2store foundation but are specialized for //! execution layer block history, following the format: //! Version | block-tuple* | other-entries* | Accumulator | `BlockIndex` //! +//! Era files are special instances of `.e2s` files with a strict content format +//! optimized for reading and long-term storage and distribution. +//! //! See also: //! - E2store format: +//! - Era format: //! - Era1 format: +pub mod consensus_types; pub mod e2s_file; pub mod e2s_types; pub mod era1_file; pub mod era1_types; +pub mod era_file_ops; +pub mod era_types; pub mod execution_types; +#[cfg(test)] +pub(crate) mod test_utils; + +use crate::e2s_types::E2sError; +use alloy_rlp::Decodable; +use ssz::Decode; + +/// Extension trait for generic decoding from compressed data +pub trait DecodeCompressed { + /// Decompress and decode the data into the given type + fn decode(&self) -> Result; +} + +/// Extension trait for generic decoding from compressed ssz data +pub trait DecodeCompressedSsz { + /// Decompress and decode the SSZ data into the given type + fn decode(&self) -> Result; +} diff --git a/crates/era/src/test_utils.rs b/crates/era/src/test_utils.rs new file mode 100644 index 00000000000..96b2545be16 --- /dev/null +++ b/crates/era/src/test_utils.rs @@ -0,0 +1,177 @@ +//! Utilities helpers to create era data structures for testing purposes. + +use crate::{ + consensus_types::{CompressedBeaconState, CompressedSignedBeaconBlock}, + execution_types::{ + BlockTuple, CompressedBody, CompressedHeader, CompressedReceipts, TotalDifficulty, + }, +}; +use alloy_consensus::{Header, ReceiptWithBloom}; +use alloy_primitives::{Address, BlockNumber, Bytes, Log, LogData, B256, B64, U256}; +use reth_ethereum_primitives::{Receipt, TxType}; + +// Helper function to create a test header +pub(crate) fn create_header() -> Header { + Header { + parent_hash: B256::default(), + ommers_hash: B256::default(), + beneficiary: Address::default(), + state_root: B256::default(), + transactions_root: B256::default(), + receipts_root: B256::default(), + logs_bloom: Default::default(), + difficulty: U256::from(123456u64), + number: 100, + gas_limit: 5000000, + gas_used: 21000, + timestamp: 1609459200, + extra_data: Bytes::default(), + mix_hash: B256::default(), + nonce: B64::default(), + base_fee_per_gas: Some(10), + withdrawals_root: Some(B256::default()), + blob_gas_used: None, + excess_blob_gas: None, + parent_beacon_block_root: None, + requests_hash: None, + } +} + +// Helper function to create a test receipt with customizable parameters +pub(crate) fn create_test_receipt( + tx_type: TxType, + success: bool, + cumulative_gas_used: u64, + log_count: usize, +) -> Receipt { + let mut logs = Vec::new(); + + for i in 0..log_count { + let address_byte = (i + 1) as u8; + let topic_byte = (i + 10) as u8; + let data_byte = (i + 100) as u8; + + logs.push(Log { + address: Address::from([address_byte; 20]), + data: LogData::new_unchecked( + vec![B256::from([topic_byte; 32]), B256::from([topic_byte + 1; 32])], + alloy_primitives::Bytes::from(vec![data_byte, data_byte + 1, data_byte + 2]), + ), + }); + } + + Receipt { tx_type, success, cumulative_gas_used, logs } +} + +// Helper function to create a list of test receipts with different characteristics +pub(crate) fn create_test_receipts() -> Vec { + vec![ + // Legacy transaction, successful, no logs + create_test_receipt(TxType::Legacy, true, 21000, 0), + // EIP-2930 transaction, failed, one log + create_test_receipt(TxType::Eip2930, false, 42000, 1), + // EIP-1559 transaction, successful, multiple logs + create_test_receipt(TxType::Eip1559, true, 63000, 3), + // EIP-4844 transaction, successful, two logs + create_test_receipt(TxType::Eip4844, true, 84000, 2), + // EIP-7702 transaction, failed, no logs + create_test_receipt(TxType::Eip7702, false, 105000, 0), + ] +} + +pub(crate) fn create_test_receipt_with_bloom( + tx_type: TxType, + success: bool, + cumulative_gas_used: u64, + log_count: usize, +) -> ReceiptWithBloom { + let receipt = create_test_receipt(tx_type, success, cumulative_gas_used, log_count); + ReceiptWithBloom { receipt: receipt.into(), logs_bloom: Default::default() } +} + +// Helper function to create a sample block tuple +pub(crate) fn create_sample_block(data_size: usize) -> BlockTuple { + // Create a compressed header with very sample data - not compressed for simplicity + let header_data = vec![0xAA; data_size]; + let header = CompressedHeader::new(header_data); + + // Create a compressed body with very sample data - not compressed for simplicity + let body_data = vec![0xBB; data_size * 2]; + let body = CompressedBody::new(body_data); + + // Create compressed receipts with very sample data - not compressed for simplicity + let receipts_data = vec![0xCC; data_size]; + let receipts = CompressedReceipts::new(receipts_data); + + let difficulty = TotalDifficulty::new(U256::from(data_size)); + + // Create and return the block tuple + BlockTuple::new(header, body, receipts, difficulty) +} + +// Helper function to create a test block with compressed data +pub(crate) fn create_test_block_with_compressed_data(number: BlockNumber) -> BlockTuple { + use alloy_consensus::{BlockBody, Header}; + use alloy_eips::eip4895::Withdrawals; + use alloy_primitives::{Address, Bytes, B256, B64, U256}; + + // Create test header + let header = Header { + parent_hash: B256::default(), + ommers_hash: B256::default(), + beneficiary: Address::default(), + state_root: B256::default(), + transactions_root: B256::default(), + receipts_root: B256::default(), + logs_bloom: Default::default(), + difficulty: U256::from(number * 1000), + number, + gas_limit: 5000000, + gas_used: 21000, + timestamp: 1609459200 + number, + extra_data: Bytes::default(), + mix_hash: B256::default(), + nonce: B64::default(), + base_fee_per_gas: Some(10), + withdrawals_root: Some(B256::default()), + blob_gas_used: None, + excess_blob_gas: None, + parent_beacon_block_root: None, + requests_hash: None, + }; + + // Create test body + let body: BlockBody = BlockBody { + transactions: vec![Bytes::from(vec![(number % 256) as u8; 10])], + ommers: vec![], + withdrawals: Some(Withdrawals(vec![])), + }; + + // Create test receipt list with bloom + let receipts_list: Vec = vec![create_test_receipt_with_bloom( + reth_ethereum_primitives::TxType::Legacy, + true, + 21000, + 0, + )]; + + // Compressed test compressed + let compressed_header = CompressedHeader::from_header(&header).unwrap(); + let compressed_body = CompressedBody::from_body(&body).unwrap(); + let compressed_receipts = CompressedReceipts::from_encodable_list(&receipts_list).unwrap(); + let total_difficulty = TotalDifficulty::new(U256::from(number * 1000)); + + BlockTuple::new(compressed_header, compressed_body, compressed_receipts, total_difficulty) +} + +/// Helper function to create a simple beacon block +pub(crate) fn create_beacon_block(data_size: usize) -> CompressedSignedBeaconBlock { + let block_data = vec![0xAA; data_size]; + CompressedSignedBeaconBlock::new(block_data) +} + +/// Helper function to create a simple beacon state +pub(crate) fn create_beacon_state(data_size: usize) -> CompressedBeaconState { + let state_data = vec![0xBB; data_size]; + CompressedBeaconState::new(state_data) +} diff --git a/crates/era/tests/it/dd.rs b/crates/era/tests/it/dd.rs index 73d5c9e9b96..769a398d6ce 100644 --- a/crates/era/tests/it/dd.rs +++ b/crates/era/tests/it/dd.rs @@ -4,7 +4,9 @@ use alloy_consensus::{BlockBody, Header}; use alloy_primitives::U256; use reth_era::{ + e2s_types::IndexEntry, era1_file::{Era1Reader, Era1Writer}, + era_file_ops::{StreamReader, StreamWriter}, execution_types::CompressedBody, }; use reth_ethereum_primitives::TransactionSigned; @@ -30,7 +32,7 @@ async fn test_mainnet_era1_only_file_decompression_and_decoding() -> eyre::Resul for &block_idx in &test_block_indices { let block = &file.group.blocks[block_idx]; - let block_number = file.group.block_index.starting_number + block_idx as u64; + let block_number = file.group.block_index.starting_number() + block_idx as u64; println!( "\n Testing block {}, compressed body size: {} bytes", @@ -93,11 +95,11 @@ async fn test_mainnet_era1_only_file_decompression_and_decoding() -> eyre::Resul let mut buffer = Vec::new(); { let mut writer = Era1Writer::new(&mut buffer); - writer.write_era1_file(&file)?; + writer.write_file(&file)?; } // Read back from buffer - let mut reader = Era1Reader::new(Cursor::new(&buffer)); + let reader = Era1Reader::new(Cursor::new(&buffer)); let read_back_file = reader.read(file.id.network_name.clone())?; // Verify basic properties are preserved @@ -110,7 +112,7 @@ async fn test_mainnet_era1_only_file_decompression_and_decoding() -> eyre::Resul for &idx in &test_block_indices { let original_block = &file.group.blocks[idx]; let read_back_block = &read_back_file.group.blocks[idx]; - let block_number = file.group.block_index.starting_number + idx as u64; + let block_number = file.group.block_index.starting_number() + idx as u64; println!("Block {block_number} details:"); println!(" Header size: {} bytes", original_block.header.data.len()); diff --git a/crates/era/tests/it/genesis.rs b/crates/era/tests/it/genesis.rs index 1812a77798a..80869f97fa0 100644 --- a/crates/era/tests/it/genesis.rs +++ b/crates/era/tests/it/genesis.rs @@ -3,13 +3,12 @@ //! These tests verify proper decompression and decoding of genesis blocks //! from different networks. -use alloy_consensus::{BlockBody, Header}; -use reth_era::execution_types::CompressedBody; -use reth_ethereum_primitives::TransactionSigned; - use crate::{ Era1TestDownloader, ERA1_MAINNET_FILES_NAMES, ERA1_SEPOLIA_FILES_NAMES, MAINNET, SEPOLIA, }; +use alloy_consensus::{BlockBody, Header}; +use reth_era::{e2s_types::IndexEntry, execution_types::CompressedBody}; +use reth_ethereum_primitives::TransactionSigned; #[tokio::test(flavor = "multi_thread")] #[ignore = "download intensive"] @@ -23,7 +22,7 @@ async fn test_mainnet_genesis_block_decompression() -> eyre::Result<()> { for &block_idx in &test_blocks { let block = &file.group.blocks[block_idx]; - let block_number = file.group.block_index.starting_number + block_idx as u64; + let block_number = file.group.block_index.starting_number() + block_idx as u64; println!( "Testing block {}, compressed body size: {} bytes", @@ -75,7 +74,7 @@ async fn test_sepolia_genesis_block_decompression() -> eyre::Result<()> { for &block_idx in &test_blocks { let block = &file.group.blocks[block_idx]; - let block_number = file.group.block_index.starting_number + block_idx as u64; + let block_number = file.group.block_index.starting_number() + block_idx as u64; println!( "Testing block {}, compressed body size: {} bytes", diff --git a/crates/era/tests/it/main.rs b/crates/era/tests/it/main.rs index ca95b38b541..611862aa8ea 100644 --- a/crates/era/tests/it/main.rs +++ b/crates/era/tests/it/main.rs @@ -10,6 +10,7 @@ use reqwest::{Client, Url}; use reth_era::{ e2s_types::E2sError, era1_file::{Era1File, Era1Reader}, + era_file_ops::FileReader, }; use reth_era_downloader::EraClient; use std::{ @@ -37,27 +38,33 @@ const MAINNET_URL: &str = "https://era.ithaca.xyz/era1/"; /// Succinct list of mainnet files we want to download /// from /// for testing purposes -const ERA1_MAINNET_FILES_NAMES: [&str; 6] = [ +const ERA1_MAINNET_FILES_NAMES: [&str; 8] = [ "mainnet-00000-5ec1ffb8.era1", "mainnet-00003-d8b8a40b.era1", "mainnet-00151-e322efe1.era1", "mainnet-00293-0d6c5812.era1", "mainnet-00443-ea71b6f9.era1", "mainnet-01367-d7efc68f.era1", + "mainnet-01610-99fdde4b.era1", + "mainnet-01895-3f81607c.era1", ]; /// Sepolia network name const SEPOLIA: &str = "sepolia"; -/// Default sepolia mainnet url +/// Default sepolia url /// for downloading sepolia `.era1` files const SEPOLIA_URL: &str = "https://era.ithaca.xyz/sepolia-era1/"; /// Succinct list of sepolia files we want to download /// from /// for testing purposes -const ERA1_SEPOLIA_FILES_NAMES: [&str; 3] = - ["sepolia-00000-643a00f7.era1", "sepolia-00074-0e81003c.era1", "sepolia-00173-b6924da5.era1"]; +const ERA1_SEPOLIA_FILES_NAMES: [&str; 4] = [ + "sepolia-00000-643a00f7.era1", + "sepolia-00074-0e81003c.era1", + "sepolia-00173-b6924da5.era1", + "sepolia-00182-a4f0a8a1.era1 ", +]; /// Utility for downloading `.era1` files for tests /// in a temporary directory @@ -115,7 +122,7 @@ impl Era1TestDownloader { let final_url = Url::from_str(url).map_err(|e| eyre!("Failed to parse URL: {}", e))?; - let folder = self.temp_dir.path().to_owned().into_boxed_path(); + let folder = self.temp_dir.path(); // set up the client let client = EraClient::new(Client::new(), final_url, folder); diff --git a/crates/era/tests/it/roundtrip.rs b/crates/era/tests/it/roundtrip.rs index 3ff83001d9a..a78af341371 100644 --- a/crates/era/tests/it/roundtrip.rs +++ b/crates/era/tests/it/roundtrip.rs @@ -7,12 +7,16 @@ //! - Writing the data back to a new file //! - Confirming that all original data is preserved throughout the process -use alloy_consensus::{BlockBody, BlockHeader, Header}; +use alloy_consensus::{BlockBody, BlockHeader, Header, ReceiptWithBloom}; use rand::{prelude::IndexedRandom, rng}; use reth_era::{ + e2s_types::IndexEntry, era1_file::{Era1File, Era1Reader, Era1Writer}, era1_types::{Era1Group, Era1Id}, - execution_types::{BlockTuple, CompressedBody, CompressedHeader, TotalDifficulty}, + era_file_ops::{EraFileFormat, StreamReader, StreamWriter}, + execution_types::{ + BlockTuple, CompressedBody, CompressedHeader, CompressedReceipts, TotalDifficulty, + }, }; use reth_ethereum_primitives::TransactionSigned; use std::io::Cursor; @@ -42,11 +46,11 @@ async fn test_file_roundtrip( let mut buffer = Vec::new(); { let mut writer = Era1Writer::new(&mut buffer); - writer.write_era1_file(&original_file)?; + writer.write_file(&original_file)?; } // Read back from buffer - let mut reader = Era1Reader::new(Cursor::new(&buffer)); + let reader = Era1Reader::new(Cursor::new(&buffer)); let roundtrip_file = reader.read(network.to_string())?; assert_eq!( @@ -71,7 +75,7 @@ async fn test_file_roundtrip( for &block_id in &test_block_indices { let original_block = &original_file.group.blocks[block_id]; let roundtrip_block = &roundtrip_file.group.blocks[block_id]; - let block_number = original_file.group.block_index.starting_number + block_id as u64; + let block_number = original_file.group.block_index.starting_number() + block_id as u64; println!("Testing roundtrip for block {block_number}"); @@ -143,6 +147,21 @@ async fn test_file_roundtrip( "Ommers count should match after roundtrip" ); + // Decode receipts + let original_receipts_decoded = + original_block.receipts.decode::>()?; + let roundtrip_receipts_decoded = + roundtrip_block.receipts.decode::>()?; + + assert_eq!( + original_receipts_decoded, roundtrip_receipts_decoded, + "Block {block_number} decoded receipts should be identical after roundtrip" + ); + assert_eq!( + original_receipts_data, roundtrip_receipts_data, + "Block {block_number} receipts data should be identical after roundtrip" + ); + // Check withdrawals presence/absence matches assert_eq!( original_decoded_body.withdrawals.is_some(), @@ -178,11 +197,21 @@ async fn test_file_roundtrip( "Transaction count should match after re-compression" ); + // Re-encore and re-compress the receipts + let recompressed_receipts = + CompressedReceipts::from_encodable(&roundtrip_receipts_decoded)?; + let recompressed_receipts_data = recompressed_receipts.decompress()?; + + assert_eq!( + original_receipts_data.len(), + recompressed_receipts_data.len(), + "Receipts length should match after re-compression" + ); + let recompressed_block = BlockTuple::new( recompressed_header, recompressed_body, - original_block.receipts.clone(), /* reuse original receipts directly as it not - * possible to decode them */ + recompressed_receipts, TotalDifficulty::new(original_block.total_difficulty.value), ); @@ -200,10 +229,10 @@ async fn test_file_roundtrip( Era1File::new(new_group, Era1Id::new(network, original_file.id.start_block, 1)); let mut writer = Era1Writer::new(&mut recompressed_buffer); - writer.write_era1_file(&new_file)?; + writer.write_file(&new_file)?; } - let mut reader = Era1Reader::new(Cursor::new(&recompressed_buffer)); + let reader = Era1Reader::new(Cursor::new(&recompressed_buffer)); let recompressed_file = reader.read(network.to_string())?; let recompressed_first_block = &recompressed_file.group.blocks[0]; diff --git a/crates/errors/src/lib.rs b/crates/errors/src/lib.rs index 7deba98f8aa..7840a0d434d 100644 --- a/crates/errors/src/lib.rs +++ b/crates/errors/src/lib.rs @@ -6,7 +6,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![no_std] extern crate alloc; diff --git a/crates/ethereum/cli/Cargo.toml b/crates/ethereum/cli/Cargo.toml index 78080bcbc42..728a97bfe3d 100644 --- a/crates/ethereum/cli/Cargo.toml +++ b/crates/ethereum/cli/Cargo.toml @@ -17,102 +17,69 @@ reth-cli-commands.workspace = true reth-cli-runner.workspace = true reth-chainspec.workspace = true reth-db.workspace = true -reth-ethereum-primitives.workspace = true -reth-network.workspace = true reth-node-builder.workspace = true reth-node-core.workspace = true reth-node-ethereum.workspace = true reth-node-metrics.workspace = true +reth-rpc-server-types.workspace = true reth-tracing.workspace = true -reth-db-api.workspace = true -reth-consensus.workspace = true -reth-errors.workspace = true -reth-ethereum-payload-builder.workspace = true -reth-evm.workspace = true -reth-execution-types.workspace = true -reth-fs-util.workspace = true +reth-tracing-otlp.workspace = true reth-node-api.workspace = true -reth-basic-payload-builder.workspace = true -reth-primitives-traits.workspace = true -reth-provider.workspace = true -reth-revm.workspace = true -reth-stages.workspace = true -reth-transaction-pool.workspace = true -reth-trie.workspace = true -reth-trie-db.workspace = true -reth-cli-util.workspace = true -reth-config.workspace = true -reth-downloaders.workspace = true -reth-exex.workspace = true -reth-network-api.workspace = true -reth-network-p2p.workspace = true -reth-node-events.workspace = true -reth-prune.workspace = true -reth-static-file.workspace = true -reth-tasks.workspace = true -reth-payload-builder.workspace = true - -# serde -serde_json.workspace = true - -# backoff -backon.workspace = true - -# test -similar-asserts.workspace = true - -# async -tokio.workspace = true -futures.workspace = true - -# alloy -alloy-eips = { workspace = true, features = ["kzg"] } -alloy-rlp.workspace = true -alloy-rpc-types = { workspace = true, features = ["engine"] } -alloy-consensus.workspace = true -alloy-primitives.workspace = true # misc clap.workspace = true eyre.workspace = true +url.workspace = true tracing.workspace = true [dev-dependencies] -# reth -reth-cli-commands.workspace = true - # fs tempfile.workspace = true [features] -default = ["jemalloc", "reth-revm/portable"] +default = [] + +otlp = ["reth-tracing/otlp", "reth-node-core/otlp"] dev = ["reth-cli-commands/arbitrary"] asm-keccak = [ "reth-node-core/asm-keccak", - "alloy-primitives/asm-keccak", + "reth-node-ethereum/asm-keccak", ] jemalloc = [ - "reth-cli-util/jemalloc", "reth-node-core/jemalloc", "reth-node-metrics/jemalloc", ] jemalloc-prof = [ - "reth-cli-util/jemalloc", - "reth-cli-util/jemalloc-prof", + "reth-node-core/jemalloc", ] -tracy-allocator = ["reth-cli-util/tracy-allocator"] +tracy-allocator = [] # Because jemalloc is default and preferred over snmalloc when both features are # enabled, `--no-default-features` should be used when enabling snmalloc or # snmalloc-native. -snmalloc = ["reth-cli-util/snmalloc"] -snmalloc-native = ["reth-cli-util/snmalloc-native"] +snmalloc = [] +snmalloc-native = [] -min-error-logs = ["tracing/release_max_level_error"] -min-warn-logs = ["tracing/release_max_level_warn"] -min-info-logs = ["tracing/release_max_level_info"] -min-debug-logs = ["tracing/release_max_level_debug"] -min-trace-logs = ["tracing/release_max_level_trace"] +min-error-logs = [ + "tracing/release_max_level_error", + "reth-node-core/min-error-logs", +] +min-warn-logs = [ + "tracing/release_max_level_warn", + "reth-node-core/min-warn-logs", +] +min-info-logs = [ + "tracing/release_max_level_info", + "reth-node-core/min-info-logs", +] +min-debug-logs = [ + "tracing/release_max_level_debug", + "reth-node-core/min-debug-logs", +] +min-trace-logs = [ + "tracing/release_max_level_trace", + "reth-node-core/min-trace-logs", +] diff --git a/crates/ethereum/cli/src/app.rs b/crates/ethereum/cli/src/app.rs new file mode 100644 index 00000000000..b947d6df1db --- /dev/null +++ b/crates/ethereum/cli/src/app.rs @@ -0,0 +1,261 @@ +use crate::{interface::Commands, Cli}; +use eyre::{eyre, Result}; +use reth_chainspec::{ChainSpec, EthChainSpec, Hardforks}; +use reth_cli::chainspec::ChainSpecParser; +use reth_cli_commands::{ + common::{CliComponentsBuilder, CliHeader, CliNodeTypes}, + launcher::{FnLauncher, Launcher}, +}; +use reth_cli_runner::CliRunner; +use reth_db::DatabaseEnv; +use reth_node_api::NodePrimitives; +use reth_node_builder::{NodeBuilder, WithLaunchContext}; +use reth_node_ethereum::{consensus::EthBeaconConsensus, EthEvmConfig, EthereumNode}; +use reth_node_metrics::recorder::install_prometheus_recorder; +use reth_rpc_server_types::RpcModuleValidator; +use reth_tracing::{FileWorkerGuard, Layers}; +use reth_tracing_otlp::OtlpProtocol; +use std::{fmt, sync::Arc}; +use tracing::info; +use url::Url; + +/// A wrapper around a parsed CLI that handles command execution. +#[derive(Debug)] +pub struct CliApp { + cli: Cli, + runner: Option, + layers: Option, + guard: Option, +} + +impl CliApp +where + C: ChainSpecParser, + Ext: clap::Args + fmt::Debug, + Rpc: RpcModuleValidator, +{ + pub(crate) fn new(cli: Cli) -> Self { + Self { cli, runner: None, layers: Some(Layers::new()), guard: None } + } + + /// Sets the runner for the CLI commander. + /// + /// This replaces any existing runner with the provided one. + pub fn set_runner(&mut self, runner: CliRunner) { + self.runner = Some(runner); + } + + /// Access to tracing layers. + /// + /// Returns a mutable reference to the tracing layers, or error + /// if tracing initialized and layers have detached already. + pub fn access_tracing_layers(&mut self) -> Result<&mut Layers> { + self.layers.as_mut().ok_or_else(|| eyre!("Tracing already initialized")) + } + + /// Execute the configured cli command. + /// + /// This accepts a closure that is used to launch the node via the + /// [`NodeCommand`](reth_cli_commands::node::NodeCommand). + pub fn run(self, launcher: impl Launcher) -> Result<()> + where + C: ChainSpecParser, + { + let components = |spec: Arc| { + (EthEvmConfig::ethereum(spec.clone()), Arc::new(EthBeaconConsensus::new(spec))) + }; + + self.run_with_components::(components, |builder, ext| async move { + launcher.entrypoint(builder, ext).await + }) + } + + /// Execute the configured cli command with the provided [`CliComponentsBuilder`]. + /// + /// This accepts a closure that is used to launch the node via the + /// [`NodeCommand`](reth_cli_commands::node::NodeCommand) and allows providing custom + /// components. + pub fn run_with_components( + mut self, + components: impl CliComponentsBuilder, + launcher: impl AsyncFnOnce( + WithLaunchContext, C::ChainSpec>>, + Ext, + ) -> Result<()>, + ) -> Result<()> + where + N: CliNodeTypes, ChainSpec: Hardforks>, + C: ChainSpecParser, + { + let runner = match self.runner.take() { + Some(runner) => runner, + None => CliRunner::try_default_runtime()?, + }; + + // Add network name if available to the logs dir + if let Some(chain_spec) = self.cli.command.chain_spec() { + self.cli.logs.log_file_directory = + self.cli.logs.log_file_directory.join(chain_spec.chain().to_string()); + } + + self.init_tracing(&runner)?; + + // Install the prometheus recorder to be sure to record all metrics + let _ = install_prometheus_recorder(); + + run_commands_with::(self.cli, runner, components, launcher) + } + + /// Initializes tracing with the configured options. + /// + /// If file logging is enabled, this function stores guard to the struct. + /// For gRPC OTLP, it requires tokio runtime context. + pub fn init_tracing(&mut self, runner: &CliRunner) -> Result<()> { + if self.guard.is_none() { + let mut layers = self.layers.take().unwrap_or_default(); + + #[cfg(feature = "otlp")] + { + self.cli.traces.validate()?; + + if let Some(endpoint) = &self.cli.traces.otlp { + info!(target: "reth::cli", "Starting OTLP tracing export to {:?}", endpoint); + self.init_otlp_export(&mut layers, endpoint, runner)?; + } + } + + self.guard = self.cli.logs.init_tracing_with_layers(layers)?; + info!(target: "reth::cli", "Initialized tracing, debug log directory: {}", self.cli.logs.log_file_directory); + } + Ok(()) + } + + /// Initialize OTLP tracing export based on protocol type. + /// + /// For gRPC, `block_on` is required because tonic's channel initialization needs + /// a tokio runtime context, even though `with_span_layer` itself is not async. + #[cfg(feature = "otlp")] + fn init_otlp_export( + &self, + layers: &mut Layers, + endpoint: &Url, + runner: &CliRunner, + ) -> Result<()> { + let endpoint = endpoint.clone(); + let protocol = self.cli.traces.protocol; + let filter_level = self.cli.traces.otlp_filter.clone(); + + match protocol { + OtlpProtocol::Grpc => { + runner.block_on(async { + layers.with_span_layer("reth".to_string(), endpoint, filter_level, protocol) + })?; + } + OtlpProtocol::Http => { + layers.with_span_layer("reth".to_string(), endpoint, filter_level, protocol)?; + } + } + + Ok(()) + } +} + +/// Run CLI commands with the provided runner, components and launcher. +/// This is the shared implementation used by both `CliApp` and Cli methods. +pub(crate) fn run_commands_with( + cli: Cli, + runner: CliRunner, + components: impl CliComponentsBuilder, + launcher: impl AsyncFnOnce( + WithLaunchContext, C::ChainSpec>>, + Ext, + ) -> Result<()>, +) -> Result<()> +where + C: ChainSpecParser, + Ext: clap::Args + fmt::Debug, + Rpc: RpcModuleValidator, + N: CliNodeTypes, ChainSpec: Hardforks>, +{ + match cli.command { + Commands::Node(command) => { + // Validate RPC modules using the configured validator + if let Some(http_api) = &command.rpc.http_api { + Rpc::validate_selection(http_api, "http.api").map_err(|e| eyre!("{e}"))?; + } + if let Some(ws_api) = &command.rpc.ws_api { + Rpc::validate_selection(ws_api, "ws.api").map_err(|e| eyre!("{e}"))?; + } + + runner.run_command_until_exit(|ctx| { + command.execute(ctx, FnLauncher::new::(launcher)) + }) + } + Commands::Init(command) => runner.run_blocking_until_ctrl_c(command.execute::()), + Commands::InitState(command) => runner.run_blocking_until_ctrl_c(command.execute::()), + Commands::Import(command) => { + runner.run_blocking_until_ctrl_c(command.execute::(components)) + } + Commands::ImportEra(command) => runner.run_blocking_until_ctrl_c(command.execute::()), + Commands::ExportEra(command) => runner.run_blocking_until_ctrl_c(command.execute::()), + Commands::DumpGenesis(command) => runner.run_blocking_until_ctrl_c(command.execute()), + Commands::Db(command) => runner.run_blocking_until_ctrl_c(command.execute::()), + Commands::Download(command) => runner.run_blocking_until_ctrl_c(command.execute::()), + Commands::Stage(command) => { + runner.run_command_until_exit(|ctx| command.execute::(ctx, components)) + } + Commands::P2P(command) => runner.run_until_ctrl_c(command.execute::()), + Commands::Config(command) => runner.run_until_ctrl_c(command.execute()), + Commands::Prune(command) => runner.run_until_ctrl_c(command.execute::()), + #[cfg(feature = "dev")] + Commands::TestVectors(command) => runner.run_until_ctrl_c(command.execute()), + Commands::ReExecute(command) => runner.run_until_ctrl_c(command.execute::(components)), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::chainspec::EthereumChainSpecParser; + use clap::Parser; + use reth_cli_commands::node::NoArgs; + + #[test] + fn test_cli_app_creation() { + let args = vec!["reth", "config"]; + let cli = Cli::::try_parse_from(args).unwrap(); + let app = cli.configure(); + + // Verify app is created correctly + assert!(app.runner.is_none()); + assert!(app.layers.is_some()); + assert!(app.guard.is_none()); + } + + #[test] + fn test_set_runner() { + let args = vec!["reth", "config"]; + let cli = Cli::::try_parse_from(args).unwrap(); + let mut app = cli.configure(); + + // Create and set a runner + if let Ok(runner) = CliRunner::try_default_runtime() { + app.set_runner(runner); + assert!(app.runner.is_some()); + } + } + + #[test] + fn test_access_tracing_layers() { + let args = vec!["reth", "config"]; + let cli = Cli::::try_parse_from(args).unwrap(); + let mut app = cli.configure(); + + // Should be able to access layers before initialization + assert!(app.access_tracing_layers().is_ok()); + + // After taking layers (simulating initialization), access should error + app.layers = None; + assert!(app.access_tracing_layers().is_err()); + } +} diff --git a/crates/ethereum/cli/src/debug_cmd/build_block.rs b/crates/ethereum/cli/src/debug_cmd/build_block.rs deleted file mode 100644 index ed97d897c30..00000000000 --- a/crates/ethereum/cli/src/debug_cmd/build_block.rs +++ /dev/null @@ -1,272 +0,0 @@ -//! Command for debugging block building. -use alloy_consensus::BlockHeader; -use alloy_eips::{ - eip2718::Encodable2718, - eip4844::{env_settings::EnvKzgSettings, BlobTransactionSidecar}, -}; -use alloy_primitives::{Address, Bytes, B256}; -use alloy_rlp::Decodable; -use alloy_rpc_types::engine::{BlobsBundleV1, PayloadAttributes}; -use clap::Parser; -use eyre::Context; -use reth_basic_payload_builder::{BuildArguments, BuildOutcome, PayloadBuilder, PayloadConfig}; -use reth_chainspec::{ChainSpec, EthereumHardforks}; -use reth_cli::chainspec::ChainSpecParser; -use reth_cli_commands::common::{AccessRights, CliNodeTypes, Environment, EnvironmentArgs}; -use reth_cli_runner::CliContext; -use reth_consensus::{Consensus, FullConsensus}; -use reth_errors::{ConsensusError, RethResult}; -use reth_ethereum_payload_builder::EthereumBuilderConfig; -use reth_ethereum_primitives::{EthPrimitives, TransactionSigned}; -use reth_evm::{execute::Executor, ConfigureEvm}; -use reth_execution_types::ExecutionOutcome; -use reth_fs_util as fs; -use reth_node_api::{BlockTy, EngineApiMessageVersion, PayloadBuilderAttributes}; -use reth_node_ethereum::{consensus::EthBeaconConsensus, EthEvmConfig}; -use reth_primitives_traits::{Block as _, SealedBlock, SealedHeader, SignedTransaction}; -use reth_provider::{ - providers::{BlockchainProvider, ProviderNodeTypes}, - BlockHashReader, BlockReader, BlockWriter, ChainSpecProvider, ProviderFactory, - StageCheckpointReader, StateProviderFactory, -}; -use reth_revm::{cached::CachedReads, cancelled::CancelOnDrop, database::StateProviderDatabase}; -use reth_stages::StageId; -use reth_transaction_pool::{ - blobstore::InMemoryBlobStore, BlobStore, EthPooledTransaction, PoolConfig, TransactionOrigin, - TransactionPool, TransactionValidationTaskExecutor, -}; -use reth_trie::StateRoot; -use reth_trie_db::DatabaseStateRoot; -use std::{path::PathBuf, str::FromStr, sync::Arc}; -use tracing::*; - -/// `reth debug build-block` command -/// This debug routine requires that the node is positioned at the block before the target. -/// The script will then parse the block and attempt to build a similar one. -#[derive(Debug, Parser)] -pub struct Command { - #[command(flatten)] - env: EnvironmentArgs, - - #[arg(long)] - parent_beacon_block_root: Option, - - #[arg(long)] - prev_randao: B256, - - #[arg(long)] - timestamp: u64, - - #[arg(long)] - suggested_fee_recipient: Address, - - /// Array of transactions. - /// NOTE: 4844 transactions must be provided in the same order as they appear in the blobs - /// bundle. - #[arg(long, value_delimiter = ',')] - transactions: Vec, - - /// Path to the file that contains a corresponding blobs bundle. - #[arg(long)] - blobs_bundle_path: Option, -} - -impl> Command { - /// Fetches the best block from the database. - /// - /// If the database is empty, returns the genesis block. - fn lookup_best_block>( - &self, - factory: ProviderFactory, - ) -> RethResult>>> { - let provider = factory.provider()?; - - let best_number = - provider.get_stage_checkpoint(StageId::Finish)?.unwrap_or_default().block_number; - let best_hash = provider - .block_hash(best_number)? - .expect("the hash for the latest block is missing, database is corrupt"); - - Ok(Arc::new( - provider - .block(best_number.into())? - .expect("the header for the latest block is missing, database is corrupt") - .seal_unchecked(best_hash), - )) - } - - /// Returns the default KZG settings - const fn kzg_settings(&self) -> eyre::Result { - Ok(EnvKzgSettings::Default) - } - - /// Execute `debug in-memory-merkle` command - pub async fn execute>( - self, - ctx: CliContext, - ) -> eyre::Result<()> { - let Environment { provider_factory, .. } = self.env.init::(AccessRights::RW)?; - - let consensus: Arc> = - Arc::new(EthBeaconConsensus::new(provider_factory.chain_spec())); - - // fetch the best block from the database - let best_block = self - .lookup_best_block(provider_factory.clone()) - .wrap_err("the head block is missing")?; - - let blockchain_db = BlockchainProvider::new(provider_factory.clone())?; - let blob_store = InMemoryBlobStore::default(); - - let validator = TransactionValidationTaskExecutor::eth_builder(blockchain_db.clone()) - .with_head_timestamp(best_block.timestamp) - .kzg_settings(self.kzg_settings()?) - .with_additional_tasks(1) - .build_with_tasks(ctx.task_executor.clone(), blob_store.clone()); - - let transaction_pool = reth_transaction_pool::Pool::eth_pool( - validator, - blob_store.clone(), - PoolConfig::default(), - ); - info!(target: "reth::cli", "Transaction pool initialized"); - - let mut blobs_bundle = self - .blobs_bundle_path - .map(|path| -> eyre::Result { - let contents = fs::read_to_string(&path) - .wrap_err(format!("could not read {}", path.display()))?; - serde_json::from_str(&contents).wrap_err("failed to deserialize blobs bundle") - }) - .transpose()?; - - for tx_bytes in &self.transactions { - debug!(target: "reth::cli", bytes = ?tx_bytes, "Decoding transaction"); - let transaction = TransactionSigned::decode(&mut &Bytes::from_str(tx_bytes)?[..])? - .try_clone_into_recovered() - .map_err(|e| eyre::eyre!("failed to recover tx: {e}"))?; - - let encoded_length = match transaction.inner() { - TransactionSigned::Eip4844(tx) => { - let blobs_bundle = blobs_bundle.as_mut().ok_or_else(|| { - eyre::eyre!("encountered a blob tx. `--blobs-bundle-path` must be provided") - })?; - - let sidecar: BlobTransactionSidecar = - blobs_bundle.pop_sidecar(tx.tx().blob_versioned_hashes.len()); - - let pooled = transaction - .clone() - .into_inner() - .try_into_pooled_eip4844(sidecar.clone()) - .expect("should not fail to convert blob tx if it is already eip4844"); - let encoded_length = pooled.encode_2718_len(); - - // insert the blob into the store - blob_store.insert(*transaction.tx_hash(), sidecar)?; - - encoded_length - } - _ => transaction.encode_2718_len(), - }; - - debug!(target: "reth::cli", ?transaction, "Adding transaction to the pool"); - transaction_pool - .add_transaction( - TransactionOrigin::External, - EthPooledTransaction::new(transaction, encoded_length), - ) - .await?; - } - - let payload_attrs = PayloadAttributes { - parent_beacon_block_root: self.parent_beacon_block_root, - prev_randao: self.prev_randao, - timestamp: self.timestamp, - suggested_fee_recipient: self.suggested_fee_recipient, - // Set empty withdrawals vector if Shanghai is active, None otherwise - withdrawals: provider_factory - .chain_spec() - .is_shanghai_active_at_timestamp(self.timestamp) - .then(Vec::new), - }; - let payload_config = PayloadConfig::new( - Arc::new(SealedHeader::new(best_block.header().clone(), best_block.hash())), - reth_payload_builder::EthPayloadBuilderAttributes::try_new( - best_block.hash(), - payload_attrs, - EngineApiMessageVersion::default() as u8, - )?, - ); - - let args = BuildArguments::new( - CachedReads::default(), - payload_config, - CancelOnDrop::default(), - None, - ); - - let payload_builder = reth_ethereum_payload_builder::EthereumPayloadBuilder::new( - blockchain_db.clone(), - transaction_pool, - EthEvmConfig::new(provider_factory.chain_spec()), - EthereumBuilderConfig::new(), - ); - - match payload_builder.try_build(args)? { - BuildOutcome::Better { payload, .. } => { - let block = payload.block(); - debug!(target: "reth::cli", ?block, "Built new payload"); - - consensus.validate_header(block.sealed_header())?; - consensus.validate_block_pre_execution(block)?; - - let block_with_senders = block.clone().try_recover().unwrap(); - - let state_provider = blockchain_db.latest()?; - let db = StateProviderDatabase::new(&state_provider); - let evm_config = EthEvmConfig::ethereum(provider_factory.chain_spec()); - let executor = evm_config.batch_executor(db); - - let block_execution_output = executor.execute(&block_with_senders)?; - let execution_outcome = - ExecutionOutcome::from((block_execution_output, block.number)); - debug!(target: "reth::cli", ?execution_outcome, "Executed block"); - - let hashed_post_state = state_provider.hashed_post_state(execution_outcome.state()); - let (state_root, trie_updates) = StateRoot::overlay_root_with_updates( - provider_factory.provider()?.tx_ref(), - hashed_post_state.clone(), - )?; - - if state_root != block_with_senders.state_root() { - eyre::bail!( - "state root mismatch. expected: {}. got: {}", - block_with_senders.state_root, - state_root - ); - } - - // Attempt to insert new block without committing - let provider_rw = provider_factory.provider_rw()?; - provider_rw.append_blocks_with_state( - Vec::from([block_with_senders]), - &execution_outcome, - hashed_post_state.into_sorted(), - trie_updates, - )?; - info!(target: "reth::cli", "Successfully appended built block"); - } - _ => unreachable!("other outcomes are unreachable"), - }; - - Ok(()) - } -} - -impl Command { - /// Returns the underlying chain being used to run this command - pub const fn chain_spec(&self) -> Option<&Arc> { - Some(&self.env.chain) - } -} diff --git a/crates/ethereum/cli/src/debug_cmd/execution.rs b/crates/ethereum/cli/src/debug_cmd/execution.rs deleted file mode 100644 index 99b27137fa3..00000000000 --- a/crates/ethereum/cli/src/debug_cmd/execution.rs +++ /dev/null @@ -1,252 +0,0 @@ -//! Command for debugging execution. - -use alloy_eips::BlockHashOrNumber; -use alloy_primitives::{BlockNumber, B256}; -use clap::Parser; -use futures::StreamExt; -use reth_chainspec::ChainSpec; -use reth_cli::chainspec::ChainSpecParser; -use reth_cli_commands::common::{AccessRights, CliNodeTypes, Environment, EnvironmentArgs}; -use reth_cli_runner::CliContext; -use reth_cli_util::get_secret_key; -use reth_config::Config; -use reth_consensus::FullConsensus; -use reth_db::DatabaseEnv; -use reth_downloaders::{ - bodies::bodies::BodiesDownloaderBuilder, - headers::reverse_headers::ReverseHeadersDownloaderBuilder, -}; -use reth_errors::ConsensusError; -use reth_ethereum_primitives::EthPrimitives; -use reth_exex::ExExManagerHandle; -use reth_network::{BlockDownloaderProvider, NetworkHandle}; -use reth_network_api::NetworkInfo; -use reth_network_p2p::{headers::client::HeadersClient, EthBlockClient}; -use reth_node_api::NodeTypesWithDBAdapter; -use reth_node_core::{args::NetworkArgs, utils::get_single_header}; -use reth_node_ethereum::{consensus::EthBeaconConsensus, EthExecutorProvider}; -use reth_node_events::node::NodeEvent; -use reth_provider::{ - providers::ProviderNodeTypes, ChainSpecProvider, ProviderFactory, StageCheckpointReader, -}; -use reth_prune::PruneModes; -use reth_stages::{ - sets::DefaultStages, stages::ExecutionStage, ExecutionStageThresholds, Pipeline, StageId, - StageSet, -}; -use reth_static_file::StaticFileProducer; -use reth_tasks::TaskExecutor; -use std::{path::PathBuf, sync::Arc}; -use tokio::sync::watch; -use tracing::*; - -/// `reth debug execution` command -#[derive(Debug, Parser)] -pub struct Command { - #[command(flatten)] - env: EnvironmentArgs, - - #[command(flatten)] - network: NetworkArgs, - - /// The maximum block height. - #[arg(long)] - pub to: u64, - - /// The block interval for sync and unwind. - /// Defaults to `1000`. - #[arg(long, default_value = "1000")] - pub interval: u64, -} - -impl> Command { - fn build_pipeline( - &self, - config: &Config, - client: Client, - consensus: Arc>, - provider_factory: ProviderFactory, - task_executor: &TaskExecutor, - static_file_producer: StaticFileProducer>, - ) -> eyre::Result> - where - N: ProviderNodeTypes, - Client: EthBlockClient + 'static, - { - // building network downloaders using the fetch client - let header_downloader = ReverseHeadersDownloaderBuilder::new(config.stages.headers) - .build(client.clone(), consensus.clone()) - .into_task_with(task_executor); - - let body_downloader = BodiesDownloaderBuilder::new(config.stages.bodies) - .build(client, consensus.clone(), provider_factory.clone()) - .into_task_with(task_executor); - - let stage_conf = &config.stages; - let prune_modes = config.prune.clone().map(|prune| prune.segments).unwrap_or_default(); - - let (tip_tx, tip_rx) = watch::channel(B256::ZERO); - let executor = EthExecutorProvider::ethereum(provider_factory.chain_spec()); - - let pipeline = Pipeline::::builder() - .with_tip_sender(tip_tx) - .add_stages( - DefaultStages::new( - provider_factory.clone(), - tip_rx, - consensus.clone(), - header_downloader, - body_downloader, - executor.clone(), - stage_conf.clone(), - prune_modes, - ) - .set(ExecutionStage::new( - executor, - consensus.clone(), - ExecutionStageThresholds { - max_blocks: None, - max_changes: None, - max_cumulative_gas: None, - max_duration: None, - }, - stage_conf.execution_external_clean_threshold(), - ExExManagerHandle::empty(), - )), - ) - .build(provider_factory, static_file_producer); - - Ok(pipeline) - } - - async fn build_network< - N: CliNodeTypes, - >( - &self, - config: &Config, - task_executor: TaskExecutor, - provider_factory: ProviderFactory>>, - network_secret_path: PathBuf, - default_peers_path: PathBuf, - ) -> eyre::Result { - let secret_key = get_secret_key(&network_secret_path)?; - let network = self - .network - .network_config(config, provider_factory.chain_spec(), secret_key, default_peers_path) - .with_task_executor(Box::new(task_executor)) - .build(provider_factory) - .start_network() - .await?; - info!(target: "reth::cli", peer_id = %network.peer_id(), local_addr = %network.local_addr(), "Connected to P2P network"); - debug!(target: "reth::cli", peer_id = ?network.peer_id(), "Full peer ID"); - Ok(network) - } - - async fn fetch_block_hash( - &self, - client: Client, - block: BlockNumber, - ) -> eyre::Result - where - Client: HeadersClient, - { - info!(target: "reth::cli", ?block, "Fetching block from the network."); - loop { - match get_single_header(&client, BlockHashOrNumber::Number(block)).await { - Ok(tip_header) => { - info!(target: "reth::cli", ?block, "Successfully fetched block"); - return Ok(tip_header.hash()) - } - Err(error) => { - error!(target: "reth::cli", ?block, %error, "Failed to fetch the block. Retrying..."); - } - } - } - } - - /// Execute `execution-debug` command - pub async fn execute>( - self, - ctx: CliContext, - ) -> eyre::Result<()> { - let Environment { provider_factory, config, data_dir } = - self.env.init::(AccessRights::RW)?; - - let consensus: Arc> = - Arc::new(EthBeaconConsensus::new(provider_factory.chain_spec())); - - // Configure and build network - let network_secret_path = - self.network.p2p_secret_key.clone().unwrap_or_else(|| data_dir.p2p_secret()); - let network = self - .build_network( - &config, - ctx.task_executor.clone(), - provider_factory.clone(), - network_secret_path, - data_dir.known_peers(), - ) - .await?; - - let static_file_producer = - StaticFileProducer::new(provider_factory.clone(), PruneModes::default()); - - // Configure the pipeline - let fetch_client = network.fetch_client().await?; - let mut pipeline = self.build_pipeline( - &config, - fetch_client.clone(), - consensus.clone(), - provider_factory.clone(), - &ctx.task_executor, - static_file_producer, - )?; - - let provider = provider_factory.provider()?; - - let latest_block_number = - provider.get_stage_checkpoint(StageId::Finish)?.map(|ch| ch.block_number); - if latest_block_number.unwrap_or_default() >= self.to { - info!(target: "reth::cli", latest = latest_block_number, "Nothing to run"); - return Ok(()) - } - - ctx.task_executor.spawn_critical( - "events task", - reth_node_events::node::handle_events( - Some(Box::new(network)), - latest_block_number, - pipeline.events().map(Into::>::into), - ), - ); - - let mut current_max_block = latest_block_number.unwrap_or_default(); - while current_max_block < self.to { - let next_block = current_max_block + 1; - let target_block = self.to.min(current_max_block + self.interval); - let target_block_hash = - self.fetch_block_hash(fetch_client.clone(), target_block).await?; - - // Run the pipeline - info!(target: "reth::cli", from = next_block, to = target_block, tip = ?target_block_hash, "Starting pipeline"); - pipeline.set_tip(target_block_hash); - let result = pipeline.run_loop().await?; - trace!(target: "reth::cli", from = next_block, to = target_block, tip = ?target_block_hash, ?result, "Pipeline finished"); - - // Unwind the pipeline without committing. - provider_factory.provider_rw()?.unwind_trie_state_range(next_block..=target_block)?; - - // Update latest block - current_max_block = target_block; - } - - Ok(()) - } -} - -impl Command { - /// Returns the underlying chain being used to run this command - pub const fn chain_spec(&self) -> Option<&Arc> { - Some(&self.env.chain) - } -} diff --git a/crates/ethereum/cli/src/debug_cmd/in_memory_merkle.rs b/crates/ethereum/cli/src/debug_cmd/in_memory_merkle.rs deleted file mode 100644 index b45e712da29..00000000000 --- a/crates/ethereum/cli/src/debug_cmd/in_memory_merkle.rs +++ /dev/null @@ -1,243 +0,0 @@ -//! Command for debugging in-memory merkle trie calculation. - -use alloy_consensus::BlockHeader; -use alloy_eips::BlockHashOrNumber; -use backon::{ConstantBuilder, Retryable}; -use clap::Parser; -use reth_chainspec::ChainSpec; -use reth_cli::chainspec::ChainSpecParser; -use reth_cli_commands::common::{AccessRights, CliNodeTypes, Environment, EnvironmentArgs}; -use reth_cli_runner::CliContext; -use reth_cli_util::get_secret_key; -use reth_config::Config; -use reth_ethereum_primitives::EthPrimitives; -use reth_evm::{execute::Executor, ConfigureEvm}; -use reth_execution_types::ExecutionOutcome; -use reth_network::{BlockDownloaderProvider, NetworkHandle}; -use reth_network_api::NetworkInfo; -use reth_node_api::{BlockTy, NodePrimitives}; -use reth_node_core::{ - args::NetworkArgs, - utils::{get_single_body, get_single_header}, -}; -use reth_node_ethereum::{consensus::EthBeaconConsensus, EthEvmConfig}; -use reth_primitives_traits::SealedBlock; -use reth_provider::{ - providers::ProviderNodeTypes, AccountExtReader, ChainSpecProvider, DatabaseProviderFactory, - HashedPostStateProvider, HashingWriter, LatestStateProviderRef, OriginalValuesKnown, - ProviderFactory, StageCheckpointReader, StateWriter, StorageLocation, StorageReader, -}; -use reth_revm::database::StateProviderDatabase; -use reth_stages::StageId; -use reth_tasks::TaskExecutor; -use reth_trie::StateRoot; -use reth_trie_db::DatabaseStateRoot; -use std::{path::PathBuf, sync::Arc}; -use tracing::*; - -/// `reth debug in-memory-merkle` command -/// This debug routine requires that the node is positioned at the block before the target. -/// The script will then download the block from p2p network and attempt to calculate and verify -/// merkle root for it. -#[derive(Debug, Parser)] -pub struct Command { - #[command(flatten)] - env: EnvironmentArgs, - - #[command(flatten)] - network: NetworkArgs, - - /// The number of retries per request - #[arg(long, default_value = "5")] - retries: usize, - - /// The depth after which we should start comparing branch nodes - #[arg(long)] - skip_node_depth: Option, -} - -impl> Command { - async fn build_network< - N: ProviderNodeTypes< - ChainSpec = C::ChainSpec, - Primitives: NodePrimitives< - Block = reth_ethereum_primitives::Block, - Receipt = reth_ethereum_primitives::Receipt, - BlockHeader = alloy_consensus::Header, - >, - >, - >( - &self, - config: &Config, - task_executor: TaskExecutor, - provider_factory: ProviderFactory, - network_secret_path: PathBuf, - default_peers_path: PathBuf, - ) -> eyre::Result { - let secret_key = get_secret_key(&network_secret_path)?; - let network = self - .network - .network_config(config, provider_factory.chain_spec(), secret_key, default_peers_path) - .with_task_executor(Box::new(task_executor)) - .build(provider_factory) - .start_network() - .await?; - info!(target: "reth::cli", peer_id = %network.peer_id(), local_addr = %network.local_addr(), "Connected to P2P network"); - debug!(target: "reth::cli", peer_id = ?network.peer_id(), "Full peer ID"); - Ok(network) - } - - /// Execute `debug in-memory-merkle` command - pub async fn execute>( - self, - ctx: CliContext, - ) -> eyre::Result<()> { - let Environment { provider_factory, config, data_dir } = - self.env.init::(AccessRights::RW)?; - - let provider = provider_factory.provider()?; - - // Look up merkle checkpoint - let merkle_checkpoint = provider - .get_stage_checkpoint(StageId::MerkleExecute)? - .expect("merkle checkpoint exists"); - - let merkle_block_number = merkle_checkpoint.block_number; - - // Configure and build network - let network_secret_path = - self.network.p2p_secret_key.clone().unwrap_or_else(|| data_dir.p2p_secret()); - let network = self - .build_network( - &config, - ctx.task_executor.clone(), - provider_factory.clone(), - network_secret_path, - data_dir.known_peers(), - ) - .await?; - - let target_block_number = merkle_block_number + 1; - - info!(target: "reth::cli", target_block_number, "Downloading full block"); - let fetch_client = network.fetch_client().await?; - - let retries = self.retries.max(1); - let backoff = ConstantBuilder::default().with_max_times(retries); - - let client = fetch_client.clone(); - let header = (move || { - get_single_header(client.clone(), BlockHashOrNumber::Number(target_block_number)) - }) - .retry(backoff) - .notify(|err, _| warn!(target: "reth::cli", "Error requesting header: {err}. Retrying...")) - .await?; - - let client = fetch_client.clone(); - let chain = provider_factory.chain_spec(); - let consensus = Arc::new(EthBeaconConsensus::new(chain.clone())); - let block: SealedBlock> = (move || { - get_single_body(client.clone(), header.clone(), consensus.clone()) - }) - .retry(backoff) - .notify(|err, _| warn!(target: "reth::cli", "Error requesting body: {err}. Retrying...")) - .await?; - - let state_provider = LatestStateProviderRef::new(&provider); - let db = StateProviderDatabase::new(&state_provider); - - let evm_config = EthEvmConfig::ethereum(provider_factory.chain_spec()); - let executor = evm_config.batch_executor(db); - let block_execution_output = executor.execute(&block.clone().try_recover()?)?; - let execution_outcome = ExecutionOutcome::from((block_execution_output, block.number())); - - // Unpacked `BundleState::state_root_slow` function - let (in_memory_state_root, in_memory_updates) = StateRoot::overlay_root_with_updates( - provider.tx_ref(), - state_provider.hashed_post_state(execution_outcome.state()), - )?; - - if in_memory_state_root == block.state_root() { - info!(target: "reth::cli", state_root = ?in_memory_state_root, "Computed in-memory state root matches"); - return Ok(()) - } - - let provider_rw = provider_factory.database_provider_rw()?; - - // Insert block, state and hashes - provider_rw.insert_historical_block(block.clone().try_recover()?)?; - provider_rw.write_state( - &execution_outcome, - OriginalValuesKnown::No, - StorageLocation::Database, - )?; - let storage_lists = - provider_rw.changed_storages_with_range(block.number..=block.number())?; - let storages = provider_rw.plain_state_storages(storage_lists)?; - provider_rw.insert_storage_for_hashing(storages)?; - let account_lists = - provider_rw.changed_accounts_with_range(block.number..=block.number())?; - let accounts = provider_rw.basic_accounts(account_lists)?; - provider_rw.insert_account_for_hashing(accounts)?; - - let (state_root, incremental_trie_updates) = StateRoot::incremental_root_with_updates( - provider_rw.tx_ref(), - block.number..=block.number(), - )?; - if state_root != block.state_root() { - eyre::bail!( - "Computed incremental state root mismatch. Expected: {:?}. Got: {:?}", - block.state_root, - state_root - ); - } - - // Compare updates - let mut in_mem_mismatched = Vec::new(); - let mut incremental_mismatched = Vec::new(); - let mut in_mem_updates_iter = in_memory_updates.account_nodes_ref().iter().peekable(); - let mut incremental_updates_iter = - incremental_trie_updates.account_nodes_ref().iter().peekable(); - - while in_mem_updates_iter.peek().is_some() || incremental_updates_iter.peek().is_some() { - match (in_mem_updates_iter.next(), incremental_updates_iter.next()) { - (Some(in_mem), Some(incr)) => { - similar_asserts::assert_eq!(in_mem.0, incr.0, "Nibbles don't match"); - if in_mem.1 != incr.1 && - in_mem.0.len() > self.skip_node_depth.unwrap_or_default() - { - in_mem_mismatched.push(in_mem); - incremental_mismatched.push(incr); - } - } - (Some(in_mem), None) => { - warn!(target: "reth::cli", next = ?in_mem, "In-memory trie updates have more entries"); - } - (None, Some(incr)) => { - tracing::warn!(target: "reth::cli", next = ?incr, "Incremental trie updates have more entries"); - } - (None, None) => { - tracing::info!(target: "reth::cli", "Exhausted all trie updates entries"); - } - } - } - - similar_asserts::assert_eq!( - incremental_mismatched, - in_mem_mismatched, - "Mismatched trie updates" - ); - - // Drop without committing. - drop(provider_rw); - - Ok(()) - } -} - -impl Command { - /// Returns the underlying chain being used to run this command - pub const fn chain_spec(&self) -> Option<&Arc> { - Some(&self.env.chain) - } -} diff --git a/crates/ethereum/cli/src/debug_cmd/merkle.rs b/crates/ethereum/cli/src/debug_cmd/merkle.rs deleted file mode 100644 index acfbd20ef99..00000000000 --- a/crates/ethereum/cli/src/debug_cmd/merkle.rs +++ /dev/null @@ -1,314 +0,0 @@ -//! Command for debugging merkle tree calculation. -use alloy_eips::BlockHashOrNumber; -use backon::{ConstantBuilder, Retryable}; -use clap::Parser; -use reth_chainspec::ChainSpec; -use reth_cli::chainspec::ChainSpecParser; -use reth_cli_commands::common::{AccessRights, CliNodeTypes, Environment, EnvironmentArgs}; -use reth_cli_runner::CliContext; -use reth_cli_util::get_secret_key; -use reth_config::Config; -use reth_consensus::{Consensus, ConsensusError}; -use reth_db_api::{cursor::DbCursorRO, tables, transaction::DbTx}; -use reth_ethereum_primitives::EthPrimitives; -use reth_evm::{execute::Executor, ConfigureEvm}; -use reth_execution_types::ExecutionOutcome; -use reth_network::{BlockDownloaderProvider, NetworkHandle}; -use reth_network_api::NetworkInfo; -use reth_network_p2p::full_block::FullBlockClient; -use reth_node_api::{BlockTy, NodePrimitives}; -use reth_node_core::{args::NetworkArgs, utils::get_single_header}; -use reth_node_ethereum::{consensus::EthBeaconConsensus, EthExecutorProvider}; -use reth_provider::{ - providers::ProviderNodeTypes, BlockNumReader, BlockWriter, ChainSpecProvider, - DatabaseProviderFactory, LatestStateProviderRef, OriginalValuesKnown, ProviderFactory, - StateWriter, StorageLocation, -}; -use reth_revm::database::StateProviderDatabase; -use reth_stages::{ - stages::{AccountHashingStage, MerkleStage, StorageHashingStage}, - ExecInput, Stage, StageCheckpoint, -}; -use reth_tasks::TaskExecutor; -use std::{path::PathBuf, sync::Arc}; -use tracing::*; - -/// `reth debug merkle` command -#[derive(Debug, Parser)] -pub struct Command { - #[command(flatten)] - env: EnvironmentArgs, - - #[command(flatten)] - network: NetworkArgs, - - /// The number of retries per request - #[arg(long, default_value = "5")] - retries: usize, - - /// The height to finish at - #[arg(long)] - to: u64, - - /// The depth after which we should start comparing branch nodes - #[arg(long)] - skip_node_depth: Option, -} - -impl> Command { - async fn build_network< - N: ProviderNodeTypes< - ChainSpec = C::ChainSpec, - Primitives: NodePrimitives< - Block = reth_ethereum_primitives::Block, - Receipt = reth_ethereum_primitives::Receipt, - BlockHeader = alloy_consensus::Header, - >, - >, - >( - &self, - config: &Config, - task_executor: TaskExecutor, - provider_factory: ProviderFactory, - network_secret_path: PathBuf, - default_peers_path: PathBuf, - ) -> eyre::Result { - let secret_key = get_secret_key(&network_secret_path)?; - let network = self - .network - .network_config(config, provider_factory.chain_spec(), secret_key, default_peers_path) - .with_task_executor(Box::new(task_executor)) - .build(provider_factory) - .start_network() - .await?; - info!(target: "reth::cli", peer_id = %network.peer_id(), local_addr = %network.local_addr(), "Connected to P2P network"); - debug!(target: "reth::cli", peer_id = ?network.peer_id(), "Full peer ID"); - Ok(network) - } - - /// Execute `merkle-debug` command - pub async fn execute>( - self, - ctx: CliContext, - ) -> eyre::Result<()> { - let Environment { provider_factory, config, data_dir } = - self.env.init::(AccessRights::RW)?; - - let provider_rw = provider_factory.database_provider_rw()?; - - // Configure and build network - let network_secret_path = - self.network.p2p_secret_key.clone().unwrap_or_else(|| data_dir.p2p_secret()); - let network = self - .build_network( - &config, - ctx.task_executor.clone(), - provider_factory.clone(), - network_secret_path, - data_dir.known_peers(), - ) - .await?; - - let executor_provider = EthExecutorProvider::ethereum(provider_factory.chain_spec()); - - // Initialize the fetch client - info!(target: "reth::cli", target_block_number = self.to, "Downloading tip of block range"); - let fetch_client = network.fetch_client().await?; - - // fetch the header at `self.to` - let retries = self.retries.max(1); - let backoff = ConstantBuilder::default().with_max_times(retries); - let client = fetch_client.clone(); - let to_header = (move || { - get_single_header(client.clone(), BlockHashOrNumber::Number(self.to)) - }) - .retry(backoff) - .notify(|err, _| warn!(target: "reth::cli", "Error requesting header: {err}. Retrying...")) - .await?; - info!(target: "reth::cli", target_block_number=self.to, "Finished downloading tip of block range"); - - // build the full block client - let consensus: Arc, Error = ConsensusError>> = - Arc::new(EthBeaconConsensus::new(provider_factory.chain_spec())); - let block_range_client = FullBlockClient::new(fetch_client, consensus); - - // get best block number - let best_block_number = provider_rw.best_block_number()?; - assert!(best_block_number < self.to, "Nothing to run"); - - // get the block range from the network - let block_range = best_block_number + 1..=self.to; - info!(target: "reth::cli", ?block_range, "Downloading range of blocks"); - let blocks = block_range_client - .get_full_block_range(to_header.hash_slow(), self.to - best_block_number) - .await; - - let mut account_hashing_stage = AccountHashingStage::default(); - let mut storage_hashing_stage = StorageHashingStage::default(); - let mut merkle_stage = MerkleStage::default_execution(); - - for block in blocks.into_iter().rev() { - let block_number = block.number; - let sealed_block = - block.try_recover().map_err(|_| eyre::eyre!("Error sealing block with senders"))?; - trace!(target: "reth::cli", block_number, "Executing block"); - - provider_rw.insert_block(sealed_block.clone(), StorageLocation::Database)?; - - let executor = executor_provider.batch_executor(StateProviderDatabase::new( - LatestStateProviderRef::new(&provider_rw), - )); - let output = executor.execute(&sealed_block)?; - - provider_rw.write_state( - &ExecutionOutcome::single(block_number, output), - OriginalValuesKnown::Yes, - StorageLocation::Database, - )?; - - let checkpoint = Some(StageCheckpoint::new( - block_number - .checked_sub(1) - .ok_or_else(|| eyre::eyre!("GenesisBlockHasNoParent"))?, - )); - - let mut account_hashing_done = false; - while !account_hashing_done { - let output = account_hashing_stage - .execute(&provider_rw, ExecInput { target: Some(block_number), checkpoint })?; - account_hashing_done = output.done; - } - - let mut storage_hashing_done = false; - while !storage_hashing_done { - let output = storage_hashing_stage - .execute(&provider_rw, ExecInput { target: Some(block_number), checkpoint })?; - storage_hashing_done = output.done; - } - - let incremental_result = merkle_stage - .execute(&provider_rw, ExecInput { target: Some(block_number), checkpoint }); - - if incremental_result.is_ok() { - debug!(target: "reth::cli", block_number, "Successfully computed incremental root"); - continue - } - - warn!(target: "reth::cli", block_number, "Incremental calculation failed, retrying from scratch"); - let incremental_account_trie = provider_rw - .tx_ref() - .cursor_read::()? - .walk_range(..)? - .collect::, _>>()?; - let incremental_storage_trie = provider_rw - .tx_ref() - .cursor_dup_read::()? - .walk_range(..)? - .collect::, _>>()?; - - let clean_input = ExecInput { target: Some(sealed_block.number), checkpoint: None }; - loop { - let clean_result = merkle_stage - .execute(&provider_rw, clean_input) - .map_err(|e| eyre::eyre!("Clean state root calculation failed: {}", e))?; - if clean_result.done { - break; - } - } - - let clean_account_trie = provider_rw - .tx_ref() - .cursor_read::()? - .walk_range(..)? - .collect::, _>>()?; - let clean_storage_trie = provider_rw - .tx_ref() - .cursor_dup_read::()? - .walk_range(..)? - .collect::, _>>()?; - - info!(target: "reth::cli", block_number, "Comparing incremental trie vs clean trie"); - - // Account trie - let mut incremental_account_mismatched = Vec::new(); - let mut clean_account_mismatched = Vec::new(); - let mut incremental_account_trie_iter = incremental_account_trie.into_iter().peekable(); - let mut clean_account_trie_iter = clean_account_trie.into_iter().peekable(); - while incremental_account_trie_iter.peek().is_some() || - clean_account_trie_iter.peek().is_some() - { - match (incremental_account_trie_iter.next(), clean_account_trie_iter.next()) { - (Some(incremental), Some(clean)) => { - similar_asserts::assert_eq!(incremental.0, clean.0, "Nibbles don't match"); - if incremental.1 != clean.1 && - clean.0 .0.len() > self.skip_node_depth.unwrap_or_default() - { - incremental_account_mismatched.push(incremental); - clean_account_mismatched.push(clean); - } - } - (Some(incremental), None) => { - warn!(target: "reth::cli", next = ?incremental, "Incremental account trie has more entries"); - } - (None, Some(clean)) => { - warn!(target: "reth::cli", next = ?clean, "Clean account trie has more entries"); - } - (None, None) => { - info!(target: "reth::cli", "Exhausted all account trie entries"); - } - } - } - - // Storage trie - let mut first_mismatched_storage = None; - let mut incremental_storage_trie_iter = incremental_storage_trie.into_iter().peekable(); - let mut clean_storage_trie_iter = clean_storage_trie.into_iter().peekable(); - while incremental_storage_trie_iter.peek().is_some() || - clean_storage_trie_iter.peek().is_some() - { - match (incremental_storage_trie_iter.next(), clean_storage_trie_iter.next()) { - (Some(incremental), Some(clean)) => { - if incremental != clean && - clean.1.nibbles.len() > self.skip_node_depth.unwrap_or_default() - { - first_mismatched_storage = Some((incremental, clean)); - break - } - } - (Some(incremental), None) => { - warn!(target: "reth::cli", next = ?incremental, "Incremental storage trie has more entries"); - } - (None, Some(clean)) => { - warn!(target: "reth::cli", next = ?clean, "Clean storage trie has more entries") - } - (None, None) => { - info!(target: "reth::cli", "Exhausted all storage trie entries.") - } - } - } - - similar_asserts::assert_eq!( - ( - incremental_account_mismatched, - first_mismatched_storage.as_ref().map(|(incremental, _)| incremental) - ), - ( - clean_account_mismatched, - first_mismatched_storage.as_ref().map(|(_, clean)| clean) - ), - "Mismatched trie nodes" - ); - } - - info!(target: "reth::cli", ?block_range, "Successfully validated incremental roots"); - - Ok(()) - } -} - -impl Command { - /// Returns the underlying chain being used to run this command - pub const fn chain_spec(&self) -> Option<&Arc> { - Some(&self.env.chain) - } -} diff --git a/crates/ethereum/cli/src/debug_cmd/mod.rs b/crates/ethereum/cli/src/debug_cmd/mod.rs deleted file mode 100644 index 1a7bd5ed0cc..00000000000 --- a/crates/ethereum/cli/src/debug_cmd/mod.rs +++ /dev/null @@ -1,68 +0,0 @@ -//! `reth debug` command. Collection of various debugging routines. - -use clap::{Parser, Subcommand}; -use reth_chainspec::ChainSpec; -use reth_cli::chainspec::ChainSpecParser; -use reth_cli_commands::common::CliNodeTypes; -use reth_cli_runner::CliContext; -use reth_ethereum_primitives::EthPrimitives; -use reth_node_ethereum::EthEngineTypes; -use std::sync::Arc; - -mod build_block; -mod execution; -mod in_memory_merkle; -mod merkle; - -/// `reth debug` command -#[derive(Debug, Parser)] -pub struct Command { - #[command(subcommand)] - command: Subcommands, -} - -/// `reth debug` subcommands -#[derive(Subcommand, Debug)] -pub enum Subcommands { - /// Debug the roundtrip execution of blocks as well as the generated data. - Execution(execution::Command), - /// Debug the clean & incremental state root calculations. - Merkle(merkle::Command), - /// Debug in-memory state root calculation. - InMemoryMerkle(in_memory_merkle::Command), - /// Debug block building. - BuildBlock(build_block::Command), -} - -impl> Command { - /// Execute `debug` command - pub async fn execute< - N: CliNodeTypes< - Payload = EthEngineTypes, - Primitives = EthPrimitives, - ChainSpec = C::ChainSpec, - >, - >( - self, - ctx: CliContext, - ) -> eyre::Result<()> { - match self.command { - Subcommands::Execution(command) => command.execute::(ctx).await, - Subcommands::Merkle(command) => command.execute::(ctx).await, - Subcommands::InMemoryMerkle(command) => command.execute::(ctx).await, - Subcommands::BuildBlock(command) => command.execute::(ctx).await, - } - } -} - -impl Command { - /// Returns the underlying chain being used to run this command - pub const fn chain_spec(&self) -> Option<&Arc> { - match &self.command { - Subcommands::Execution(command) => command.chain_spec(), - Subcommands::Merkle(command) => command.chain_spec(), - Subcommands::InMemoryMerkle(command) => command.chain_spec(), - Subcommands::BuildBlock(command) => command.chain_spec(), - } - } -} diff --git a/crates/ethereum/cli/src/interface.rs b/crates/ethereum/cli/src/interface.rs index ebfe1bbb661..f41143bb4fa 100644 --- a/crates/ethereum/cli/src/interface.rs +++ b/crates/ethereum/cli/src/interface.rs @@ -1,35 +1,43 @@ //! CLI definition and entrypoint to executable -use crate::{chainspec::EthereumChainSpecParser, debug_cmd}; +use crate::{ + app::{run_commands_with, CliApp}, + chainspec::EthereumChainSpecParser, +}; use clap::{Parser, Subcommand}; -use reth_chainspec::ChainSpec; +use reth_chainspec::{ChainSpec, EthChainSpec, Hardforks}; use reth_cli::chainspec::ChainSpecParser; use reth_cli_commands::{ - config_cmd, db, download, dump_genesis, import, import_era, init_cmd, init_state, + common::{CliComponentsBuilder, CliHeader, CliNodeTypes}, + config_cmd, db, download, dump_genesis, export_era, import, import_era, init_cmd, init_state, + launcher::FnLauncher, node::{self, NoArgs}, - p2p, prune, recover, stage, + p2p, prune, re_execute, stage, }; use reth_cli_runner::CliRunner; use reth_db::DatabaseEnv; -use reth_network::EthNetworkPrimitives; +use reth_node_api::NodePrimitives; use reth_node_builder::{NodeBuilder, WithLaunchContext}; use reth_node_core::{ - args::LogArgs, - version::{LONG_VERSION, SHORT_VERSION}, + args::{LogArgs, TraceArgs}, + version::version_metadata, }; -use reth_node_ethereum::{consensus::EthBeaconConsensus, EthExecutorProvider, EthereumNode}; use reth_node_metrics::recorder::install_prometheus_recorder; +use reth_rpc_server_types::{DefaultRpcModuleValidator, RpcModuleValidator}; use reth_tracing::FileWorkerGuard; -use std::{ffi::OsString, fmt, future::Future, sync::Arc}; +use std::{ffi::OsString, fmt, future::Future, marker::PhantomData, sync::Arc}; use tracing::info; /// The main reth cli interface. /// /// This is the entrypoint to the executable. #[derive(Debug, Parser)] -#[command(author, version = SHORT_VERSION, long_version = LONG_VERSION, about = "Reth", long_about = None)] -pub struct Cli -{ +#[command(author, name = version_metadata().name_client.as_ref(), version = version_metadata().short_version.as_ref(), long_version = version_metadata().long_version.as_ref(), about = "Reth", long_about = None)] +pub struct Cli< + C: ChainSpecParser = EthereumChainSpecParser, + Ext: clap::Args + fmt::Debug = NoArgs, + Rpc: RpcModuleValidator = DefaultRpcModuleValidator, +> { /// The command to run #[command(subcommand)] pub command: Commands, @@ -37,6 +45,14 @@ pub struct Cli, } impl Cli { @@ -55,7 +71,18 @@ impl Cli { } } -impl, Ext: clap::Args + fmt::Debug> Cli { +impl Cli { + /// Configures the CLI and returns a [`CliApp`] instance. + /// + /// This method is used to prepare the CLI for execution by wrapping it in a + /// [`CliApp`] that can be further configured before running. + pub fn configure(self) -> CliApp + where + C: ChainSpecParser, + { + CliApp::new(self) + } + /// Execute the configured cli command. /// /// This accepts a closure that is used to launch the node via the @@ -102,10 +129,32 @@ impl, Ext: clap::Args + fmt::Debug> Cl where L: FnOnce(WithLaunchContext, C::ChainSpec>>, Ext) -> Fut, Fut: Future>, + C: ChainSpecParser, { self.with_runner(CliRunner::try_default_runtime()?, launcher) } + /// Execute the configured cli command with the provided [`CliComponentsBuilder`]. + /// + /// This accepts a closure that is used to launch the node via the + /// [`NodeCommand`](node::NodeCommand). + /// + /// This command will be run on the [default tokio runtime](reth_cli_runner::tokio_runtime). + pub fn run_with_components( + self, + components: impl CliComponentsBuilder, + launcher: impl AsyncFnOnce( + WithLaunchContext, C::ChainSpec>>, + Ext, + ) -> eyre::Result<()>, + ) -> eyre::Result<()> + where + N: CliNodeTypes, ChainSpec: Hardforks>, + C: ChainSpecParser, + { + self.with_runner_and_components(CliRunner::try_default_runtime()?, components, launcher) + } + /// Execute the configured cli command with the provided [`CliRunner`]. /// /// @@ -116,13 +165,7 @@ impl, Ext: clap::Args + fmt::Debug> Cl /// use reth_ethereum_cli::interface::Cli; /// use reth_node_ethereum::EthereumNode; /// - /// let runtime = tokio::runtime::Builder::new_multi_thread() - /// .worker_threads(4) - /// .max_blocking_threads(256) - /// .enable_all() - /// .build() - /// .unwrap(); - /// let runner = CliRunner::from_runtime(runtime); + /// let runner = CliRunner::try_default_runtime().unwrap(); /// /// Cli::parse_args() /// .with_runner(runner, |builder, _| async move { @@ -131,15 +174,36 @@ impl, Ext: clap::Args + fmt::Debug> Cl /// }) /// .unwrap(); /// ``` - pub fn with_runner(mut self, runner: CliRunner, launcher: L) -> eyre::Result<()> + pub fn with_runner(self, runner: CliRunner, launcher: L) -> eyre::Result<()> where L: FnOnce(WithLaunchContext, C::ChainSpec>>, Ext) -> Fut, Fut: Future>, + C: ChainSpecParser, + { + let mut app = self.configure(); + app.set_runner(runner); + app.run(FnLauncher::new::(async move |builder, ext| launcher(builder, ext).await)) + } + + /// Execute the configured cli command with the provided [`CliRunner`] and + /// [`CliComponentsBuilder`]. + pub fn with_runner_and_components( + mut self, + runner: CliRunner, + components: impl CliComponentsBuilder, + launcher: impl AsyncFnOnce( + WithLaunchContext, C::ChainSpec>>, + Ext, + ) -> eyre::Result<()>, + ) -> eyre::Result<()> + where + N: CliNodeTypes, ChainSpec: Hardforks>, + C: ChainSpecParser, { // Add network name if available to the logs dir if let Some(chain_spec) = self.command.chain_spec() { self.logs.log_file_directory = - self.logs.log_file_directory.join(chain_spec.chain.to_string()); + self.logs.log_file_directory.join(chain_spec.chain().to_string()); } let _guard = self.init_tracing()?; info!(target: "reth::cli", "Initialized tracing, debug log directory: {}", self.logs.log_file_directory); @@ -147,64 +211,25 @@ impl, Ext: clap::Args + fmt::Debug> Cl // Install the prometheus recorder to be sure to record all metrics let _ = install_prometheus_recorder(); - let components = |spec: Arc| { - (EthExecutorProvider::ethereum(spec.clone()), EthBeaconConsensus::new(spec)) - }; - match self.command { - Commands::Node(command) => { - runner.run_command_until_exit(|ctx| command.execute(ctx, launcher)) - } - Commands::Init(command) => { - runner.run_blocking_until_ctrl_c(command.execute::()) - } - Commands::InitState(command) => { - runner.run_blocking_until_ctrl_c(command.execute::()) - } - Commands::Import(command) => { - runner.run_blocking_until_ctrl_c(command.execute::(components)) - } - Commands::ImportEra(command) => { - runner.run_blocking_until_ctrl_c(command.execute::()) - } - Commands::DumpGenesis(command) => runner.run_blocking_until_ctrl_c(command.execute()), - Commands::Db(command) => { - runner.run_blocking_until_ctrl_c(command.execute::()) - } - Commands::Download(command) => { - runner.run_blocking_until_ctrl_c(command.execute::()) - } - Commands::Stage(command) => runner.run_command_until_exit(|ctx| { - command.execute::(ctx, components) - }), - Commands::P2P(command) => { - runner.run_until_ctrl_c(command.execute::()) - } - #[cfg(feature = "dev")] - Commands::TestVectors(command) => runner.run_until_ctrl_c(command.execute()), - Commands::Config(command) => runner.run_until_ctrl_c(command.execute()), - Commands::Debug(command) => { - runner.run_command_until_exit(|ctx| command.execute::(ctx)) - } - Commands::Recover(command) => { - runner.run_command_until_exit(|ctx| command.execute::(ctx)) - } - Commands::Prune(command) => runner.run_until_ctrl_c(command.execute::()), - } + // Use the shared standalone function to avoid duplication + run_commands_with::(self, runner, components, launcher) } /// Initializes tracing with the configured options. /// /// If file logging is enabled, this function returns a guard that must be kept alive to ensure /// that all logs are flushed to disk. + /// If an OTLP endpoint is specified, it will export metrics to the configured collector. pub fn init_tracing(&self) -> eyre::Result> { - let guard = self.logs.init_tracing()?; + let layers = reth_tracing::Layers::new(); + + let guard = self.logs.init_tracing_with_layers(layers)?; Ok(guard) } } /// Commands to be executed #[derive(Debug, Subcommand)] -#[expect(clippy::large_enum_variant)] pub enum Commands { /// Start the node #[command(name = "node")] @@ -215,17 +240,20 @@ pub enum Commands { /// Initialize the database from a state dump file. #[command(name = "init-state")] InitState(init_state::InitStateCommand), - /// This syncs RLP encoded blocks from a file. + /// This syncs RLP encoded blocks from a file or files. #[command(name = "import")] Import(import::ImportCommand), /// This syncs ERA encoded blocks from a directory. #[command(name = "import-era")] ImportEra(import_era::ImportEraCommand), + /// Exports block to era1 files in a specified directory. + #[command(name = "export-era")] + ExportEra(export_era::ExportEraCommand), /// Dumps genesis block JSON configuration to stdout. DumpGenesis(dump_genesis::DumpGenesisCommand), /// Database debugging utilities #[command(name = "db")] - Db(db::Command), + Db(Box>), /// Download public node snapshots #[command(name = "download")] Download(download::DownloadCommand), @@ -234,7 +262,7 @@ pub enum Commands { Stage(stage::Command), /// P2P Debugging utilities #[command(name = "p2p")] - P2P(p2p::Command), + P2P(Box>), /// Generate Test Vectors #[cfg(feature = "dev")] #[command(name = "test-vectors")] @@ -242,15 +270,12 @@ pub enum Commands { /// Write config to stdout #[command(name = "config")] Config(config_cmd::Command), - /// Various debug routines - #[command(name = "debug")] - Debug(Box>), - /// Scripts for node recovery - #[command(name = "recover")] - Recover(recover::Command), /// Prune according to the configuration without any limits #[command(name = "prune")] Prune(prune::PruneCommand), + /// Re-execute blocks in parallel to verify historical sync correctness. + #[command(name = "re-execute")] + ReExecute(re_execute::Command), } impl Commands { @@ -261,6 +286,7 @@ impl Commands { Self::Init(cmd) => cmd.chain_spec(), Self::InitState(cmd) => cmd.chain_spec(), Self::Import(cmd) => cmd.chain_spec(), + Self::ExportEra(cmd) => cmd.chain_spec(), Self::ImportEra(cmd) => cmd.chain_spec(), Self::DumpGenesis(cmd) => cmd.chain_spec(), Self::Db(cmd) => cmd.chain_spec(), @@ -270,9 +296,8 @@ impl Commands { #[cfg(feature = "dev")] Self::TestVectors(_) => None, Self::Config(_) => None, - Self::Debug(cmd) => cmd.chain_spec(), - Self::Recover(cmd) => cmd.chain_spec(), Self::Prune(cmd) => cmd.chain_spec(), + Self::ReExecute(cmd) => cmd.chain_spec(), } } } @@ -282,6 +307,7 @@ mod tests { use super::*; use crate::chainspec::SUPPORTED_CHAINS; use clap::CommandFactory; + use reth_chainspec::SEPOLIA; use reth_node_core::args::ColorMode; #[test] @@ -380,4 +406,119 @@ mod tests { .unwrap(); assert!(reth.run(async move |_, _| Ok(())).is_ok()); } + + #[test] + fn test_rpc_module_validation() { + use reth_rpc_server_types::RethRpcModule; + + // Test that standard modules are accepted + let cli = + Cli::try_parse_args_from(["reth", "node", "--http.api", "eth,admin,debug"]).unwrap(); + + if let Commands::Node(command) = &cli.command { + if let Some(http_api) = &command.rpc.http_api { + // Should contain the expected modules + let modules = http_api.to_selection(); + assert!(modules.contains(&RethRpcModule::Eth)); + assert!(modules.contains(&RethRpcModule::Admin)); + assert!(modules.contains(&RethRpcModule::Debug)); + } else { + panic!("Expected http.api to be set"); + } + } else { + panic!("Expected Node command"); + } + + // Test that unknown modules are parsed as Other variant + let cli = + Cli::try_parse_args_from(["reth", "node", "--http.api", "eth,customrpc"]).unwrap(); + + if let Commands::Node(command) = &cli.command { + if let Some(http_api) = &command.rpc.http_api { + let modules = http_api.to_selection(); + assert!(modules.contains(&RethRpcModule::Eth)); + assert!(modules.contains(&RethRpcModule::Other("customrpc".to_string()))); + } else { + panic!("Expected http.api to be set"); + } + } else { + panic!("Expected Node command"); + } + } + + #[test] + fn test_rpc_module_unknown_rejected() { + use reth_cli_runner::CliRunner; + + // Test that unknown module names are rejected during validation + let cli = + Cli::try_parse_args_from(["reth", "node", "--http.api", "unknownmodule"]).unwrap(); + + // When we try to run the CLI with validation, it should fail + let runner = CliRunner::try_default_runtime().unwrap(); + let result = cli.with_runner(runner, |_, _| async { Ok(()) }); + + assert!(result.is_err()); + let err = result.unwrap_err(); + let err_msg = err.to_string(); + + // The error should mention it's an unknown module + assert!( + err_msg.contains("Unknown RPC module"), + "Error should mention unknown module: {}", + err_msg + ); + assert!( + err_msg.contains("'unknownmodule'"), + "Error should mention the module name: {}", + err_msg + ); + } + + #[test] + fn parse_unwind_chain() { + let cli = Cli::try_parse_args_from([ + "reth", "stage", "unwind", "--chain", "sepolia", "to-block", "100", + ]) + .unwrap(); + match cli.command { + Commands::Stage(cmd) => match cmd.command { + stage::Subcommands::Unwind(cmd) => { + assert_eq!(cmd.chain_spec().unwrap().chain_id(), SEPOLIA.chain_id()); + } + _ => panic!("Expected Unwind command"), + }, + _ => panic!("Expected Stage command"), + }; + } + + #[test] + fn parse_empty_supported_chains() { + #[derive(Debug, Clone, Default)] + struct FileChainSpecParser; + + impl ChainSpecParser for FileChainSpecParser { + type ChainSpec = ChainSpec; + + const SUPPORTED_CHAINS: &'static [&'static str] = &[]; + + fn parse(s: &str) -> eyre::Result> { + EthereumChainSpecParser::parse(s) + } + } + + let cli = Cli::::try_parse_from([ + "reth", "stage", "unwind", "--chain", "sepolia", "to-block", "100", + ]) + .unwrap(); + match cli.command { + Commands::Stage(cmd) => match cmd.command { + stage::Subcommands::Unwind(cmd) => { + assert_eq!(cmd.chain_spec().unwrap().chain_id(), SEPOLIA.chain_id()); + } + _ => panic!("Expected Unwind command"), + }, + _ => panic!("Expected Stage command"), + }; + } } diff --git a/crates/ethereum/cli/src/lib.rs b/crates/ethereum/cli/src/lib.rs index a9d0e355bac..a9080030690 100644 --- a/crates/ethereum/cli/src/lib.rs +++ b/crates/ethereum/cli/src/lib.rs @@ -6,13 +6,16 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] +/// A configurable App on top of the cli parser. +pub mod app; /// Chain specification parser. pub mod chainspec; -pub mod debug_cmd; pub mod interface; -pub use interface::Cli; + +pub use app::CliApp; +pub use interface::{Cli, Commands}; #[cfg(test)] mod test { diff --git a/crates/ethereum/consensus/src/lib.rs b/crates/ethereum/consensus/src/lib.rs index 89ce9f72e0d..5aef1393032 100644 --- a/crates/ethereum/consensus/src/lib.rs +++ b/crates/ethereum/consensus/src/lib.rs @@ -6,7 +6,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(feature = "std"), no_std)] extern crate alloc; @@ -18,13 +18,13 @@ use reth_chainspec::{EthChainSpec, EthereumHardforks}; use reth_consensus::{Consensus, ConsensusError, FullConsensus, HeaderValidator}; use reth_consensus_common::validation::{ validate_4844_header_standalone, validate_against_parent_4844, - validate_against_parent_eip1559_base_fee, validate_against_parent_hash_number, - validate_against_parent_timestamp, validate_block_pre_execution, validate_body_against_header, - validate_header_base_fee, validate_header_extra_data, validate_header_gas, + validate_against_parent_eip1559_base_fee, validate_against_parent_gas_limit, + validate_against_parent_hash_number, validate_against_parent_timestamp, + validate_block_pre_execution, validate_body_against_header, validate_header_base_fee, + validate_header_extra_data, validate_header_gas, }; use reth_execution_types::BlockExecutionResult; use reth_primitives_traits::{ - constants::{GAS_LIMIT_BOUND_DIVISOR, MINIMUM_GAS_LIMIT}, Block, BlockHeader, NodePrimitives, RecoveredBlock, SealedBlock, SealedHeader, }; @@ -46,59 +46,15 @@ impl EthBeaconConsensus Self { chain_spec } } - /// Checks the gas limit for consistency between parent and self headers. - /// - /// The maximum allowable difference between self and parent gas limits is determined by the - /// parent's gas limit divided by the [`GAS_LIMIT_BOUND_DIVISOR`]. - fn validate_against_parent_gas_limit( - &self, - header: &SealedHeader, - parent: &SealedHeader, - ) -> Result<(), ConsensusError> { - // Determine the parent gas limit, considering elasticity multiplier on the London fork. - let parent_gas_limit = if !self.chain_spec.is_london_active_at_block(parent.number()) && - self.chain_spec.is_london_active_at_block(header.number()) - { - parent.gas_limit() * - self.chain_spec - .base_fee_params_at_timestamp(header.timestamp()) - .elasticity_multiplier as u64 - } else { - parent.gas_limit() - }; - - // Check for an increase in gas limit beyond the allowed threshold. - if header.gas_limit() > parent_gas_limit { - if header.gas_limit() - parent_gas_limit >= parent_gas_limit / GAS_LIMIT_BOUND_DIVISOR { - return Err(ConsensusError::GasLimitInvalidIncrease { - parent_gas_limit, - child_gas_limit: header.gas_limit(), - }) - } - } - // Check for a decrease in gas limit beyond the allowed threshold. - else if parent_gas_limit - header.gas_limit() >= - parent_gas_limit / GAS_LIMIT_BOUND_DIVISOR - { - return Err(ConsensusError::GasLimitInvalidDecrease { - parent_gas_limit, - child_gas_limit: header.gas_limit(), - }) - } - // Check if the self gas limit is below the minimum required limit. - else if header.gas_limit() < MINIMUM_GAS_LIMIT { - return Err(ConsensusError::GasLimitInvalidMinimum { - child_gas_limit: header.gas_limit(), - }) - } - - Ok(()) + /// Returns the chain spec associated with this consensus engine. + pub const fn chain_spec(&self) -> &Arc { + &self.chain_spec } } impl FullConsensus for EthBeaconConsensus where - ChainSpec: Send + Sync + EthChainSpec + EthereumHardforks + Debug, + ChainSpec: Send + Sync + EthChainSpec
+ EthereumHardforks + Debug, N: NodePrimitives, { fn validate_block_post_execution( @@ -110,10 +66,10 @@ where } } -impl Consensus - for EthBeaconConsensus +impl Consensus for EthBeaconConsensus where B: Block, + ChainSpec: EthChainSpec
+ EthereumHardforks + Debug + Send + Sync, { type Error = ConsensusError; @@ -130,10 +86,10 @@ where } } -impl HeaderValidator - for EthBeaconConsensus +impl HeaderValidator for EthBeaconConsensus where H: BlockHeader, + ChainSpec: EthChainSpec
+ EthereumHardforks + Debug + Send + Sync, { fn validate_header(&self, header: &SealedHeader) -> Result<(), ConsensusError> { let header = header.header(); @@ -220,9 +176,7 @@ where validate_against_parent_timestamp(header.header(), parent.header())?; - // TODO Check difficulty increment between parent and self - // Ace age did increment it by some formula that we need to follow. - self.validate_against_parent_gas_limit(header, parent)?; + validate_against_parent_gas_limit(header, parent, &self.chain_spec)?; validate_against_parent_eip1559_base_fee( header.header(), @@ -242,9 +196,14 @@ where #[cfg(test)] mod tests { use super::*; + use alloy_consensus::Header; use alloy_primitives::B256; use reth_chainspec::{ChainSpec, ChainSpecBuilder}; - use reth_primitives_traits::proofs; + use reth_consensus_common::validation::validate_against_parent_gas_limit; + use reth_primitives_traits::{ + constants::{GAS_LIMIT_BOUND_DIVISOR, MINIMUM_GAS_LIMIT}, + proofs, + }; fn header_with_gas_limit(gas_limit: u64) -> SealedHeader { let header = reth_primitives_traits::Header { gas_limit, ..Default::default() }; @@ -257,8 +216,7 @@ mod tests { let child = header_with_gas_limit((parent.gas_limit + 5) as u64); assert_eq!( - EthBeaconConsensus::new(Arc::new(ChainSpec::default())) - .validate_against_parent_gas_limit(&child, &parent), + validate_against_parent_gas_limit(&child, &parent, &ChainSpec::
::default()), Ok(()) ); } @@ -269,8 +227,7 @@ mod tests { let child = header_with_gas_limit(MINIMUM_GAS_LIMIT - 1); assert_eq!( - EthBeaconConsensus::new(Arc::new(ChainSpec::default())) - .validate_against_parent_gas_limit(&child, &parent), + validate_against_parent_gas_limit(&child, &parent, &ChainSpec::
::default()), Err(ConsensusError::GasLimitInvalidMinimum { child_gas_limit: child.gas_limit as u64 }) ); } @@ -283,8 +240,7 @@ mod tests { ); assert_eq!( - EthBeaconConsensus::new(Arc::new(ChainSpec::default())) - .validate_against_parent_gas_limit(&child, &parent), + validate_against_parent_gas_limit(&child, &parent, &ChainSpec::
::default()), Err(ConsensusError::GasLimitInvalidIncrease { parent_gas_limit: parent.gas_limit, child_gas_limit: child.gas_limit, @@ -298,8 +254,7 @@ mod tests { let child = header_with_gas_limit(parent.gas_limit - 5); assert_eq!( - EthBeaconConsensus::new(Arc::new(ChainSpec::default())) - .validate_against_parent_gas_limit(&child, &parent), + validate_against_parent_gas_limit(&child, &parent, &ChainSpec::
::default()), Ok(()) ); } @@ -312,8 +267,7 @@ mod tests { ); assert_eq!( - EthBeaconConsensus::new(Arc::new(ChainSpec::default())) - .validate_against_parent_gas_limit(&child, &parent), + validate_against_parent_gas_limit(&child, &parent, &ChainSpec::
::default()), Err(ConsensusError::GasLimitInvalidDecrease { parent_gas_limit: parent.gas_limit, child_gas_limit: child.gas_limit, diff --git a/crates/ethereum/consensus/src/validation.rs b/crates/ethereum/consensus/src/validation.rs index 5b243f92680..71affffeb0c 100644 --- a/crates/ethereum/consensus/src/validation.rs +++ b/crates/ethereum/consensus/src/validation.rs @@ -1,7 +1,7 @@ use alloc::vec::Vec; use alloy_consensus::{proofs::calculate_receipt_root, BlockHeader, TxReceipt}; -use alloy_eips::eip7685::Requests; -use alloy_primitives::{Bloom, B256}; +use alloy_eips::{eip7685::Requests, Encodable2718}; +use alloy_primitives::{Bloom, Bytes, B256}; use reth_chainspec::EthereumHardforks; use reth_consensus::ConsensusError; use reth_primitives_traits::{ @@ -37,13 +37,19 @@ where // operation as hashing that is required for state root got calculated in every // transaction This was replaced with is_success flag. // See more about EIP here: https://eips.ethereum.org/EIPS/eip-658 - if chain_spec.is_byzantium_active_at_block(block.header().number()) { - if let Err(error) = - verify_receipts(block.header().receipts_root(), block.header().logs_bloom(), receipts) - { - tracing::debug!(%error, ?receipts, "receipts verification failed"); - return Err(error) - } + if chain_spec.is_byzantium_active_at_block(block.header().number()) && + let Err(error) = verify_receipts( + block.header().receipts_root(), + block.header().logs_bloom(), + receipts, + ) + { + let receipts = receipts + .iter() + .map(|r| Bytes::from(r.with_bloom_ref().encoded_2718())) + .collect::>(); + tracing::debug!(%error, ?receipts, "receipts verification failed"); + return Err(error) } // Validate that the header requests hash matches the calculated requests hash @@ -118,7 +124,7 @@ mod tests { #[test] fn test_verify_receipts_success() { // Create a vector of 5 default Receipt instances - let receipts = vec![Receipt::default(); 5]; + let receipts: Vec = vec![Receipt::default(); 5]; // Compare against expected values assert!(verify_receipts( @@ -136,7 +142,7 @@ mod tests { let expected_logs_bloom = Bloom::random(); // Create a vector of 5 random Receipt instances - let receipts = vec![Receipt::default(); 5]; + let receipts: Vec = vec![Receipt::default(); 5]; assert!(verify_receipts(expected_receipts_root, expected_logs_bloom, &receipts).is_err()); } diff --git a/crates/ethereum/engine-primitives/Cargo.toml b/crates/ethereum/engine-primitives/Cargo.toml index 75e2af0237e..db42e79aa71 100644 --- a/crates/ethereum/engine-primitives/Cargo.toml +++ b/crates/ethereum/engine-primitives/Cargo.toml @@ -26,6 +26,7 @@ alloy-rlp.workspace = true # misc serde.workspace = true sha2.workspace = true +thiserror.workspace = true [dev-dependencies] serde_json.workspace = true @@ -41,6 +42,7 @@ std = [ "serde/std", "sha2/std", "serde_json/std", + "thiserror/std", "reth-engine-primitives/std", "reth-primitives-traits/std", ] diff --git a/crates/ethereum/engine-primitives/src/error.rs b/crates/ethereum/engine-primitives/src/error.rs new file mode 100644 index 00000000000..917e0a74b5e --- /dev/null +++ b/crates/ethereum/engine-primitives/src/error.rs @@ -0,0 +1,13 @@ +use thiserror::Error; + +/// Error during [`EthBuiltPayload`](crate::EthBuiltPayload) to execution payload envelope +/// conversion. +#[derive(Error, Debug)] +pub enum BuiltPayloadConversionError { + /// Unexpected EIP-4844 sidecars in the built payload. + #[error("unexpected EIP-4844 sidecars")] + UnexpectedEip4844Sidecars, + /// Unexpected EIP-7594 sidecars in the built payload. + #[error("unexpected EIP-7594 sidecars")] + UnexpectedEip7594Sidecars, +} diff --git a/crates/ethereum/engine-primitives/src/lib.rs b/crates/ethereum/engine-primitives/src/lib.rs index 80f0bf39374..95c317a8c0f 100644 --- a/crates/ethereum/engine-primitives/src/lib.rs +++ b/crates/ethereum/engine-primitives/src/lib.rs @@ -6,15 +6,18 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(feature = "std"), no_std)] extern crate alloc; mod payload; -pub use payload::{EthBuiltPayload, EthPayloadBuilderAttributes}; +pub use payload::{payload_id, BlobSidecars, EthBuiltPayload, EthPayloadBuilderAttributes}; -use alloy_rpc_types_engine::{ExecutionData, ExecutionPayload}; +mod error; +pub use error::*; + +use alloy_rpc_types_engine::{ExecutionData, ExecutionPayload, ExecutionPayloadEnvelopeV5}; pub use alloy_rpc_types_engine::{ ExecutionPayloadEnvelopeV2, ExecutionPayloadEnvelopeV3, ExecutionPayloadEnvelopeV4, ExecutionPayloadV1, PayloadAttributes as EthPayloadAttributes, @@ -62,12 +65,14 @@ where + TryInto + TryInto + TryInto - + TryInto, + + TryInto + + TryInto, { type ExecutionPayloadEnvelopeV1 = ExecutionPayloadV1; type ExecutionPayloadEnvelopeV2 = ExecutionPayloadEnvelopeV2; type ExecutionPayloadEnvelopeV3 = ExecutionPayloadEnvelopeV3; type ExecutionPayloadEnvelopeV4 = ExecutionPayloadEnvelopeV4; + type ExecutionPayloadEnvelopeV5 = ExecutionPayloadEnvelopeV5; } /// A default payload type for [`EthEngineTypes`] diff --git a/crates/ethereum/engine-primitives/src/payload.rs b/crates/ethereum/engine-primitives/src/payload.rs index d43c43abc1b..45c1f6a31fa 100644 --- a/crates/ethereum/engine-primitives/src/payload.rs +++ b/crates/ethereum/engine-primitives/src/payload.rs @@ -1,17 +1,25 @@ //! Contains types required for building a payload. use alloc::{sync::Arc, vec::Vec}; -use alloy_eips::{eip4844::BlobTransactionSidecar, eip4895::Withdrawals, eip7685::Requests}; +use alloy_eips::{ + eip4844::BlobTransactionSidecar, + eip4895::Withdrawals, + eip7594::{BlobTransactionSidecarEip7594, BlobTransactionSidecarVariant}, + eip7685::Requests, +}; use alloy_primitives::{Address, B256, U256}; use alloy_rlp::Encodable; use alloy_rpc_types_engine::{ - ExecutionPayloadEnvelopeV2, ExecutionPayloadEnvelopeV3, ExecutionPayloadEnvelopeV4, - ExecutionPayloadFieldV2, ExecutionPayloadV1, ExecutionPayloadV3, PayloadAttributes, PayloadId, + BlobsBundleV1, BlobsBundleV2, ExecutionPayloadEnvelopeV2, ExecutionPayloadEnvelopeV3, + ExecutionPayloadEnvelopeV4, ExecutionPayloadEnvelopeV5, ExecutionPayloadFieldV2, + ExecutionPayloadV1, ExecutionPayloadV3, PayloadAttributes, PayloadId, }; use core::convert::Infallible; -use reth_ethereum_primitives::{Block, EthPrimitives}; +use reth_ethereum_primitives::EthPrimitives; use reth_payload_primitives::{BuiltPayload, PayloadBuilderAttributes}; -use reth_primitives_traits::SealedBlock; +use reth_primitives_traits::{NodePrimitives, SealedBlock}; + +use crate::BuiltPayloadConversionError; /// Contains the built payload. /// @@ -19,33 +27,33 @@ use reth_primitives_traits::SealedBlock; /// Therefore, the empty-block here is always available and full-block will be set/updated /// afterward. #[derive(Debug, Clone)] -pub struct EthBuiltPayload { +pub struct EthBuiltPayload { /// Identifier of the payload pub(crate) id: PayloadId, /// The built block - pub(crate) block: Arc>, + pub(crate) block: Arc>, /// The fees of the block pub(crate) fees: U256, /// The blobs, proofs, and commitments in the block. If the block is pre-cancun, this will be /// empty. - pub(crate) sidecars: Vec, + pub(crate) sidecars: BlobSidecars, /// The requests of the payload pub(crate) requests: Option, } // === impl BuiltPayload === -impl EthBuiltPayload { +impl EthBuiltPayload { /// Initializes the payload with the given initial block /// - /// Caution: This does not set any [`BlobTransactionSidecar`]. + /// Caution: This does not set any [`BlobSidecars`]. pub const fn new( id: PayloadId, - block: Arc>, + block: Arc>, fees: U256, requests: Option, ) -> Self { - Self { id, block, fees, sidecars: Vec::new(), requests } + Self { id, block, fees, requests, sidecars: BlobSidecars::Empty } } /// Returns the identifier of the payload. @@ -54,7 +62,7 @@ impl EthBuiltPayload { } /// Returns the built block(sealed) - pub fn block(&self) -> &SealedBlock { + pub fn block(&self) -> &SealedBlock { &self.block } @@ -64,29 +72,98 @@ impl EthBuiltPayload { } /// Returns the blob sidecars. - pub fn sidecars(&self) -> &[BlobTransactionSidecar] { + pub const fn sidecars(&self) -> &BlobSidecars { &self.sidecars } - /// Adds sidecars to the payload. - pub fn extend_sidecars(&mut self, sidecars: impl IntoIterator) { - self.sidecars.extend(sidecars) + /// Sets blob transactions sidecars on the payload. + pub fn with_sidecars(mut self, sidecars: impl Into) -> Self { + self.sidecars = sidecars.into(); + self } +} - /// Same as [`Self::extend_sidecars`] but returns the type again. - pub fn with_sidecars( - mut self, - sidecars: impl IntoIterator, - ) -> Self { - self.extend_sidecars(sidecars); - self +impl EthBuiltPayload { + /// Try converting built payload into [`ExecutionPayloadEnvelopeV3`]. + /// + /// Returns an error if the payload contains non EIP-4844 sidecar. + pub fn try_into_v3(self) -> Result { + let Self { block, fees, sidecars, .. } = self; + + let blobs_bundle = match sidecars { + BlobSidecars::Empty => BlobsBundleV1::empty(), + BlobSidecars::Eip4844(sidecars) => BlobsBundleV1::from(sidecars), + BlobSidecars::Eip7594(_) => { + return Err(BuiltPayloadConversionError::UnexpectedEip7594Sidecars) + } + }; + + Ok(ExecutionPayloadEnvelopeV3 { + execution_payload: ExecutionPayloadV3::from_block_unchecked( + block.hash(), + &Arc::unwrap_or_clone(block).into_block(), + ), + block_value: fees, + // From the engine API spec: + // + // > Client software **MAY** use any heuristics to decide whether to set + // `shouldOverrideBuilder` flag or not. If client software does not implement any + // heuristic this flag **SHOULD** be set to `false`. + // + // Spec: + // + should_override_builder: false, + blobs_bundle, + }) + } + + /// Try converting built payload into [`ExecutionPayloadEnvelopeV4`]. + /// + /// Returns an error if the payload contains non EIP-4844 sidecar. + pub fn try_into_v4(self) -> Result { + Ok(ExecutionPayloadEnvelopeV4 { + execution_requests: self.requests.clone().unwrap_or_default(), + envelope_inner: self.try_into()?, + }) + } + + /// Try converting built payload into [`ExecutionPayloadEnvelopeV5`]. + pub fn try_into_v5(self) -> Result { + let Self { block, fees, sidecars, requests, .. } = self; + + let blobs_bundle = match sidecars { + BlobSidecars::Empty => BlobsBundleV2::empty(), + BlobSidecars::Eip7594(sidecars) => BlobsBundleV2::from(sidecars), + BlobSidecars::Eip4844(_) => { + return Err(BuiltPayloadConversionError::UnexpectedEip4844Sidecars) + } + }; + + Ok(ExecutionPayloadEnvelopeV5 { + execution_payload: ExecutionPayloadV3::from_block_unchecked( + block.hash(), + &Arc::unwrap_or_clone(block).into_block(), + ), + block_value: fees, + // From the engine API spec: + // + // > Client software **MAY** use any heuristics to decide whether to set + // `shouldOverrideBuilder` flag or not. If client software does not implement any + // heuristic this flag **SHOULD** be set to `false`. + // + // Spec: + // + should_override_builder: false, + blobs_bundle, + execution_requests: requests.unwrap_or_default(), + }) } } -impl BuiltPayload for EthBuiltPayload { - type Primitives = EthPrimitives; +impl BuiltPayload for EthBuiltPayload { + type Primitives = N; - fn block(&self) -> &SealedBlock { + fn block(&self) -> &SealedBlock { &self.block } @@ -124,39 +201,117 @@ impl From for ExecutionPayloadEnvelopeV2 { } } -impl From for ExecutionPayloadEnvelopeV3 { - fn from(value: EthBuiltPayload) -> Self { - let EthBuiltPayload { block, fees, sidecars, .. } = value; +impl TryFrom for ExecutionPayloadEnvelopeV3 { + type Error = BuiltPayloadConversionError; - Self { - execution_payload: ExecutionPayloadV3::from_block_unchecked( - block.hash(), - &Arc::unwrap_or_clone(block).into_block(), - ), - block_value: fees, - // From the engine API spec: - // - // > Client software **MAY** use any heuristics to decide whether to set - // `shouldOverrideBuilder` flag or not. If client software does not implement any - // heuristic this flag **SHOULD** be set to `false`. - // - // Spec: - // - should_override_builder: false, - blobs_bundle: sidecars.into(), - } + fn try_from(value: EthBuiltPayload) -> Result { + value.try_into_v3() } } -impl From for ExecutionPayloadEnvelopeV4 { - fn from(value: EthBuiltPayload) -> Self { - Self { - execution_requests: value.requests.clone().unwrap_or_default(), - envelope_inner: value.into(), +impl TryFrom for ExecutionPayloadEnvelopeV4 { + type Error = BuiltPayloadConversionError; + + fn try_from(value: EthBuiltPayload) -> Result { + value.try_into_v4() + } +} + +impl TryFrom for ExecutionPayloadEnvelopeV5 { + type Error = BuiltPayloadConversionError; + + fn try_from(value: EthBuiltPayload) -> Result { + value.try_into_v5() + } +} + +/// An enum representing blob transaction sidecars belonging to [`EthBuiltPayload`]. +#[derive(Clone, Default, Debug)] +pub enum BlobSidecars { + /// No sidecars (default). + #[default] + Empty, + /// EIP-4844 style sidecars. + Eip4844(Vec), + /// EIP-7594 style sidecars. + Eip7594(Vec), +} + +impl BlobSidecars { + /// Create new EIP-4844 style sidecars. + pub const fn eip4844(sidecars: Vec) -> Self { + Self::Eip4844(sidecars) + } + + /// Create new EIP-7594 style sidecars. + pub const fn eip7594(sidecars: Vec) -> Self { + Self::Eip7594(sidecars) + } + + /// Push EIP-4844 blob sidecar. Ignores the item if sidecars already contain EIP-7594 sidecars. + pub fn push_eip4844_sidecar(&mut self, sidecar: BlobTransactionSidecar) { + match self { + Self::Empty => { + *self = Self::Eip4844(Vec::from([sidecar])); + } + Self::Eip4844(sidecars) => { + sidecars.push(sidecar); + } + Self::Eip7594(_) => {} + } + } + + /// Push EIP-7594 blob sidecar. Ignores the item if sidecars already contain EIP-4844 sidecars. + pub fn push_eip7594_sidecar(&mut self, sidecar: BlobTransactionSidecarEip7594) { + match self { + Self::Empty => { + *self = Self::Eip7594(Vec::from([sidecar])); + } + Self::Eip7594(sidecars) => { + sidecars.push(sidecar); + } + Self::Eip4844(_) => {} + } + } + + /// Push a [`BlobTransactionSidecarVariant`]. Ignores the item if sidecars already contain the + /// opposite type. + pub fn push_sidecar_variant(&mut self, sidecar: BlobTransactionSidecarVariant) { + match sidecar { + BlobTransactionSidecarVariant::Eip4844(sidecar) => { + self.push_eip4844_sidecar(sidecar); + } + BlobTransactionSidecarVariant::Eip7594(sidecar) => { + self.push_eip7594_sidecar(sidecar); + } } } } +impl From> for BlobSidecars { + fn from(value: Vec) -> Self { + Self::eip4844(value) + } +} + +impl From> for BlobSidecars { + fn from(value: Vec) -> Self { + Self::eip7594(value) + } +} + +impl From> for BlobSidecars { + fn from(value: alloc::vec::IntoIter) -> Self { + value.collect::>().into() + } +} + +impl From> for BlobSidecars { + fn from(value: alloc::vec::IntoIter) -> Self { + value.collect::>().into() + } +} + /// Container type for all components required to build a payload. #[derive(Debug, Clone, PartialEq, Eq, Default)] pub struct EthPayloadBuilderAttributes { @@ -251,7 +406,7 @@ impl PayloadBuilderAttributes for EthPayloadBuilderAttributes { /// Generates the payload id for the configured payload from the [`PayloadAttributes`]. /// /// Returns an 8-byte identifier by hashing the payload components with sha256 hash. -pub(crate) fn payload_id(parent: &B256, attributes: &PayloadAttributes) -> PayloadId { +pub fn payload_id(parent: &B256, attributes: &PayloadAttributes) -> PayloadId { use sha2::Digest; let mut hasher = sha2::Sha256::new(); hasher.update(parent.as_slice()); diff --git a/crates/ethereum/evm/Cargo.toml b/crates/ethereum/evm/Cargo.toml index b0f75388ec2..fbbbeeed836 100644 --- a/crates/ethereum/evm/Cargo.toml +++ b/crates/ethereum/evm/Cargo.toml @@ -19,12 +19,14 @@ reth-primitives-traits.workspace = true reth-ethereum-primitives.workspace = true revm.workspace = true reth-evm.workspace = true +reth-storage-errors.workspace = true # Alloy alloy-primitives.workspace = true alloy-eips.workspace = true alloy-evm.workspace = true alloy-consensus.workspace = true +alloy-rpc-types-engine.workspace = true # Misc parking_lot = { workspace = true, optional = true } @@ -33,7 +35,6 @@ derive_more = { workspace = true, optional = true } [dev-dependencies] reth-testing-utils.workspace = true reth-evm = { workspace = true, features = ["test-utils"] } -reth-execution-types.workspace = true secp256k1.workspace = true alloy-genesis.workspace = true @@ -54,6 +55,8 @@ std = [ "revm/std", "reth-ethereum-primitives/std", "derive_more?/std", + "alloy-rpc-types-engine/std", + "reth-storage-errors/std", ] test-utils = [ "dep:parking_lot", diff --git a/crates/ethereum/evm/src/build.rs b/crates/ethereum/evm/src/build.rs index 1762e951cd1..85d4cae311b 100644 --- a/crates/ethereum/evm/src/build.rs +++ b/crates/ethereum/evm/src/build.rs @@ -1,15 +1,16 @@ -use alloc::sync::Arc; +use alloc::{sync::Arc, vec::Vec}; use alloy_consensus::{ - proofs, Block, BlockBody, BlockHeader, Header, Transaction, TxReceipt, EMPTY_OMMER_ROOT_HASH, + proofs::{self, calculate_receipt_root}, + Block, BlockBody, BlockHeader, Header, TxReceipt, EMPTY_OMMER_ROOT_HASH, }; use alloy_eips::merge::BEACON_NONCE; use alloy_evm::{block::BlockExecutorFactory, eth::EthBlockExecutionCtx}; use alloy_primitives::Bytes; use reth_chainspec::{EthChainSpec, EthereumHardforks}; -use reth_ethereum_primitives::{Receipt, TransactionSigned}; use reth_evm::execute::{BlockAssembler, BlockAssemblerInput, BlockExecutionError}; use reth_execution_types::BlockExecutionResult; -use reth_primitives_traits::logs_bloom; +use reth_primitives_traits::{logs_bloom, Receipt, SignedTransaction}; +use revm::context::Block as _; /// Block builder for Ethereum. #[derive(Debug, Clone)] @@ -31,31 +32,33 @@ impl BlockAssembler for EthBlockAssembler where F: for<'a> BlockExecutorFactory< ExecutionCtx<'a> = EthBlockExecutionCtx<'a>, - Transaction = TransactionSigned, - Receipt = Receipt, + Transaction: SignedTransaction, + Receipt: Receipt, >, ChainSpec: EthChainSpec + EthereumHardforks, { - type Block = Block; + type Block = Block; fn assemble_block( &self, input: BlockAssemblerInput<'_, '_, F>, - ) -> Result, BlockExecutionError> { + ) -> Result { let BlockAssemblerInput { evm_env, execution_ctx: ctx, parent, transactions, - output: BlockExecutionResult { receipts, requests, gas_used }, + output: BlockExecutionResult { receipts, requests, gas_used, blob_gas_used }, state_root, .. } = input; - let timestamp = evm_env.block_env.timestamp; + let timestamp = evm_env.block_env.timestamp().saturating_to(); let transactions_root = proofs::calculate_transaction_root(&transactions); - let receipts_root = Receipt::calculate_receipt_root_no_memo(receipts); + let receipts_root = calculate_receipt_root( + &receipts.iter().map(|r| r.with_bloom_ref()).collect::>(), + ); let logs_bloom = logs_bloom(receipts.iter().flat_map(|r| r.logs())); let withdrawals = self @@ -71,12 +74,11 @@ where .then(|| requests.requests_hash()); let mut excess_blob_gas = None; - let mut blob_gas_used = None; + let mut block_blob_gas_used = None; // only determine cancun fields when active if self.chain_spec.is_cancun_active_at_timestamp(timestamp) { - blob_gas_used = - Some(transactions.iter().map(|tx| tx.blob_gas_used().unwrap_or_default()).sum()); + block_blob_gas_used = Some(*blob_gas_used); excess_blob_gas = if self.chain_spec.is_cancun_active_at_timestamp(parent.timestamp) { parent.maybe_next_block_excess_blob_gas( self.chain_spec.blob_params_at_timestamp(timestamp), @@ -84,30 +86,33 @@ where } else { // for the first post-fork block, both parent.blob_gas_used and // parent.excess_blob_gas are evaluated as 0 - Some(alloy_eips::eip7840::BlobParams::cancun().next_block_excess_blob_gas(0, 0)) + Some( + alloy_eips::eip7840::BlobParams::cancun() + .next_block_excess_blob_gas_osaka(0, 0, 0), + ) }; } let header = Header { parent_hash: ctx.parent_hash, ommers_hash: EMPTY_OMMER_ROOT_HASH, - beneficiary: evm_env.block_env.beneficiary, + beneficiary: evm_env.block_env.beneficiary(), state_root, transactions_root, receipts_root, withdrawals_root, logs_bloom, timestamp, - mix_hash: evm_env.block_env.prevrandao.unwrap_or_default(), + mix_hash: evm_env.block_env.prevrandao().unwrap_or_default(), nonce: BEACON_NONCE.into(), - base_fee_per_gas: Some(evm_env.block_env.basefee), - number: evm_env.block_env.number, - gas_limit: evm_env.block_env.gas_limit, - difficulty: evm_env.block_env.difficulty, + base_fee_per_gas: Some(evm_env.block_env.basefee()), + number: evm_env.block_env.number().saturating_to(), + gas_limit: evm_env.block_env.gas_limit(), + difficulty: evm_env.block_env.difficulty(), gas_used: *gas_used, extra_data: self.extra_data.clone(), parent_beacon_block_root: ctx.parent_beacon_block_root, - blob_gas_used, + blob_gas_used: block_blob_gas_used, excess_blob_gas, requests_hash, }; diff --git a/crates/ethereum/evm/src/config.rs b/crates/ethereum/evm/src/config.rs index e94ca17fb37..f9c288f0674 100644 --- a/crates/ethereum/evm/src/config.rs +++ b/crates/ethereum/evm/src/config.rs @@ -1,254 +1,4 @@ -use alloy_consensus::Header; -use reth_chainspec::{ChainSpec, EthereumHardforks}; -use reth_ethereum_forks::EthereumHardfork; -use revm::primitives::hardfork::SpecId; - -/// Map the latest active hardfork at the given header to a revm [`SpecId`]. -pub fn revm_spec(chain_spec: &ChainSpec, header: &Header) -> SpecId { - revm_spec_by_timestamp_and_block_number(chain_spec, header.timestamp, header.number) -} - -/// Map the latest active hardfork at the given timestamp or block number to a revm [`SpecId`]. -pub fn revm_spec_by_timestamp_and_block_number( - chain_spec: &ChainSpec, - timestamp: u64, - block_number: u64, -) -> SpecId { - if chain_spec - .fork(EthereumHardfork::Osaka) - .active_at_timestamp_or_number(timestamp, block_number) - { - SpecId::OSAKA - } else if chain_spec - .fork(EthereumHardfork::Prague) - .active_at_timestamp_or_number(timestamp, block_number) - { - SpecId::PRAGUE - } else if chain_spec - .fork(EthereumHardfork::Cancun) - .active_at_timestamp_or_number(timestamp, block_number) - { - SpecId::CANCUN - } else if chain_spec - .fork(EthereumHardfork::Shanghai) - .active_at_timestamp_or_number(timestamp, block_number) - { - SpecId::SHANGHAI - } else if chain_spec.is_paris_active_at_block(block_number) { - SpecId::MERGE - } else if chain_spec - .fork(EthereumHardfork::London) - .active_at_timestamp_or_number(timestamp, block_number) - { - SpecId::LONDON - } else if chain_spec - .fork(EthereumHardfork::Berlin) - .active_at_timestamp_or_number(timestamp, block_number) - { - SpecId::BERLIN - } else if chain_spec - .fork(EthereumHardfork::Istanbul) - .active_at_timestamp_or_number(timestamp, block_number) - { - SpecId::ISTANBUL - } else if chain_spec - .fork(EthereumHardfork::Petersburg) - .active_at_timestamp_or_number(timestamp, block_number) - { - SpecId::PETERSBURG - } else if chain_spec - .fork(EthereumHardfork::Byzantium) - .active_at_timestamp_or_number(timestamp, block_number) - { - SpecId::BYZANTIUM - } else if chain_spec - .fork(EthereumHardfork::SpuriousDragon) - .active_at_timestamp_or_number(timestamp, block_number) - { - SpecId::SPURIOUS_DRAGON - } else if chain_spec - .fork(EthereumHardfork::Tangerine) - .active_at_timestamp_or_number(timestamp, block_number) - { - SpecId::TANGERINE - } else if chain_spec - .fork(EthereumHardfork::Homestead) - .active_at_timestamp_or_number(timestamp, block_number) - { - SpecId::HOMESTEAD - } else if chain_spec - .fork(EthereumHardfork::Frontier) - .active_at_timestamp_or_number(timestamp, block_number) - { - SpecId::FRONTIER - } else { - panic!( - "invalid hardfork chainspec: expected at least one hardfork, got {:?}", - chain_spec.hardforks - ) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::U256; - use reth_chainspec::{ChainSpecBuilder, MAINNET}; - - #[test] - fn test_revm_spec_by_timestamp() { - assert_eq!( - revm_spec_by_timestamp_and_block_number( - &ChainSpecBuilder::mainnet().cancun_activated().build(), - 0, - 0 - ), - SpecId::CANCUN - ); - assert_eq!( - revm_spec_by_timestamp_and_block_number( - &ChainSpecBuilder::mainnet().shanghai_activated().build(), - 0, - 0 - ), - SpecId::SHANGHAI - ); - let mainnet = ChainSpecBuilder::mainnet().build(); - assert_eq!( - revm_spec_by_timestamp_and_block_number(&mainnet, 0, mainnet.paris_block().unwrap()), - SpecId::MERGE - ); - } - - #[test] - fn test_to_revm_spec() { - assert_eq!( - revm_spec(&ChainSpecBuilder::mainnet().cancun_activated().build(), &Default::default()), - SpecId::CANCUN - ); - assert_eq!( - revm_spec( - &ChainSpecBuilder::mainnet().shanghai_activated().build(), - &Default::default() - ), - SpecId::SHANGHAI - ); - assert_eq!( - revm_spec(&ChainSpecBuilder::mainnet().paris_activated().build(), &Default::default()), - SpecId::MERGE - ); - assert_eq!( - revm_spec(&ChainSpecBuilder::mainnet().london_activated().build(), &Default::default()), - SpecId::LONDON - ); - assert_eq!( - revm_spec(&ChainSpecBuilder::mainnet().berlin_activated().build(), &Default::default()), - SpecId::BERLIN - ); - assert_eq!( - revm_spec( - &ChainSpecBuilder::mainnet().istanbul_activated().build(), - &Default::default() - ), - SpecId::ISTANBUL - ); - assert_eq!( - revm_spec( - &ChainSpecBuilder::mainnet().petersburg_activated().build(), - &Default::default() - ), - SpecId::PETERSBURG - ); - assert_eq!( - revm_spec( - &ChainSpecBuilder::mainnet().byzantium_activated().build(), - &Default::default() - ), - SpecId::BYZANTIUM - ); - assert_eq!( - revm_spec( - &ChainSpecBuilder::mainnet().spurious_dragon_activated().build(), - &Default::default() - ), - SpecId::SPURIOUS_DRAGON - ); - assert_eq!( - revm_spec( - &ChainSpecBuilder::mainnet().tangerine_whistle_activated().build(), - &Default::default() - ), - SpecId::TANGERINE - ); - assert_eq!( - revm_spec( - &ChainSpecBuilder::mainnet().homestead_activated().build(), - &Default::default() - ), - SpecId::HOMESTEAD - ); - assert_eq!( - revm_spec( - &ChainSpecBuilder::mainnet().frontier_activated().build(), - &Default::default() - ), - SpecId::FRONTIER - ); - } - - #[test] - fn test_eth_spec() { - assert_eq!( - revm_spec(&MAINNET, &Header { timestamp: 1710338135, ..Default::default() }), - SpecId::CANCUN - ); - assert_eq!( - revm_spec(&MAINNET, &Header { timestamp: 1681338455, ..Default::default() }), - SpecId::SHANGHAI - ); - - assert_eq!( - revm_spec( - &MAINNET, - &Header { difficulty: U256::from(10_u128), number: 15537394, ..Default::default() } - ), - SpecId::MERGE - ); - assert_eq!( - revm_spec(&MAINNET, &Header { number: 15537394 - 10, ..Default::default() }), - SpecId::LONDON - ); - assert_eq!( - revm_spec(&MAINNET, &Header { number: 12244000 + 10, ..Default::default() }), - SpecId::BERLIN - ); - assert_eq!( - revm_spec(&MAINNET, &Header { number: 12244000 - 10, ..Default::default() }), - SpecId::ISTANBUL - ); - assert_eq!( - revm_spec(&MAINNET, &Header { number: 7280000 + 10, ..Default::default() }), - SpecId::PETERSBURG - ); - assert_eq!( - revm_spec(&MAINNET, &Header { number: 7280000 - 10, ..Default::default() }), - SpecId::BYZANTIUM - ); - assert_eq!( - revm_spec(&MAINNET, &Header { number: 2675000 + 10, ..Default::default() }), - SpecId::SPURIOUS_DRAGON - ); - assert_eq!( - revm_spec(&MAINNET, &Header { number: 2675000 - 10, ..Default::default() }), - SpecId::TANGERINE - ); - assert_eq!( - revm_spec(&MAINNET, &Header { number: 1150000 + 10, ..Default::default() }), - SpecId::HOMESTEAD - ); - assert_eq!( - revm_spec(&MAINNET, &Header { number: 1150000 - 10, ..Default::default() }), - SpecId::FRONTIER - ); - } -} +pub use alloy_evm::{ + spec as revm_spec, + spec_by_timestamp_and_block_number as revm_spec_by_timestamp_and_block_number, +}; diff --git a/crates/ethereum/evm/src/execute.rs b/crates/ethereum/evm/src/execute.rs deleted file mode 100644 index 072f314ce7a..00000000000 --- a/crates/ethereum/evm/src/execute.rs +++ /dev/null @@ -1,863 +0,0 @@ -//! Ethereum block execution strategy. - -/// Helper type with backwards compatible methods to obtain Ethereum executor -/// providers. -pub type EthExecutorProvider = crate::EthEvmConfig; - -#[cfg(test)] -mod tests { - use crate::EthEvmConfig; - use alloy_consensus::{constants::ETH_TO_WEI, Header, TxLegacy}; - use alloy_eips::{ - eip2935::{HISTORY_SERVE_WINDOW, HISTORY_STORAGE_ADDRESS, HISTORY_STORAGE_CODE}, - eip4788::{BEACON_ROOTS_ADDRESS, BEACON_ROOTS_CODE, SYSTEM_ADDRESS}, - eip4895::Withdrawal, - eip7002::{WITHDRAWAL_REQUEST_PREDEPLOY_ADDRESS, WITHDRAWAL_REQUEST_PREDEPLOY_CODE}, - eip7685::EMPTY_REQUESTS_HASH, - }; - use alloy_evm::block::BlockValidationError; - use alloy_primitives::{b256, fixed_bytes, keccak256, Bytes, TxKind, B256, U256}; - use reth_chainspec::{ChainSpec, ChainSpecBuilder, EthereumHardfork, ForkCondition, MAINNET}; - use reth_ethereum_primitives::{Block, BlockBody, Transaction}; - use reth_evm::{execute::Executor, ConfigureEvm}; - use reth_execution_types::BlockExecutionResult; - use reth_primitives_traits::{ - crypto::secp256k1::public_key_to_address, Block as _, RecoveredBlock, - }; - use reth_testing_utils::generators::{self, sign_tx_with_key_pair}; - use revm::{ - database::{CacheDB, EmptyDB, TransitionState}, - primitives::address, - state::{AccountInfo, Bytecode, EvmState}, - Database, - }; - use std::sync::{mpsc, Arc}; - - fn create_database_with_beacon_root_contract() -> CacheDB { - let mut db = CacheDB::new(Default::default()); - - let beacon_root_contract_account = AccountInfo { - balance: U256::ZERO, - code_hash: keccak256(BEACON_ROOTS_CODE.clone()), - nonce: 1, - code: Some(Bytecode::new_raw(BEACON_ROOTS_CODE.clone())), - }; - - db.insert_account_info(BEACON_ROOTS_ADDRESS, beacon_root_contract_account); - - db - } - - fn create_database_with_withdrawal_requests_contract() -> CacheDB { - let mut db = CacheDB::new(Default::default()); - - let withdrawal_requests_contract_account = AccountInfo { - nonce: 1, - balance: U256::ZERO, - code_hash: keccak256(WITHDRAWAL_REQUEST_PREDEPLOY_CODE.clone()), - code: Some(Bytecode::new_raw(WITHDRAWAL_REQUEST_PREDEPLOY_CODE.clone())), - }; - - db.insert_account_info( - WITHDRAWAL_REQUEST_PREDEPLOY_ADDRESS, - withdrawal_requests_contract_account, - ); - - db - } - - fn evm_config(chain_spec: Arc) -> EthEvmConfig { - EthEvmConfig::new(chain_spec) - } - - #[test] - fn eip_4788_non_genesis_call() { - let mut header = - Header { timestamp: 1, number: 1, excess_blob_gas: Some(0), ..Header::default() }; - - let db = create_database_with_beacon_root_contract(); - - let chain_spec = Arc::new( - ChainSpecBuilder::from(&*MAINNET) - .shanghai_activated() - .with_fork(EthereumHardfork::Cancun, ForkCondition::Timestamp(1)) - .build(), - ); - - let provider = evm_config(chain_spec); - - let mut executor = provider.batch_executor(db); - - // attempt to execute a block without parent beacon block root, expect err - let err = executor - .execute_one(&RecoveredBlock::new_unhashed( - Block { - header: header.clone(), - body: BlockBody { transactions: vec![], ommers: vec![], withdrawals: None }, - }, - vec![], - )) - .expect_err( - "Executing cancun block without parent beacon block root field should fail", - ); - - assert!(matches!( - err.as_validation().unwrap(), - BlockValidationError::MissingParentBeaconBlockRoot - )); - - // fix header, set a gas limit - header.parent_beacon_block_root = Some(B256::with_last_byte(0x69)); - - // Now execute a block with the fixed header, ensure that it does not fail - executor - .execute_one(&RecoveredBlock::new_unhashed( - Block { - header: header.clone(), - body: BlockBody { transactions: vec![], ommers: vec![], withdrawals: None }, - }, - vec![], - )) - .unwrap(); - - // check the actual storage of the contract - it should be: - // * The storage value at header.timestamp % HISTORY_BUFFER_LENGTH should be - // header.timestamp - // * The storage value at header.timestamp % HISTORY_BUFFER_LENGTH + HISTORY_BUFFER_LENGTH - // // should be parent_beacon_block_root - let history_buffer_length = 8191u64; - let timestamp_index = header.timestamp % history_buffer_length; - let parent_beacon_block_root_index = - timestamp_index % history_buffer_length + history_buffer_length; - - let timestamp_storage = executor.with_state_mut(|state| { - state.storage(BEACON_ROOTS_ADDRESS, U256::from(timestamp_index)).unwrap() - }); - assert_eq!(timestamp_storage, U256::from(header.timestamp)); - - // get parent beacon block root storage and compare - let parent_beacon_block_root_storage = executor.with_state_mut(|state| { - state - .storage(BEACON_ROOTS_ADDRESS, U256::from(parent_beacon_block_root_index)) - .expect("storage value should exist") - }); - assert_eq!(parent_beacon_block_root_storage, U256::from(0x69)); - } - - #[test] - fn eip_4788_no_code_cancun() { - // This test ensures that we "silently fail" when cancun is active and there is no code at - // // BEACON_ROOTS_ADDRESS - let header = Header { - timestamp: 1, - number: 1, - parent_beacon_block_root: Some(B256::with_last_byte(0x69)), - excess_blob_gas: Some(0), - ..Header::default() - }; - - let db = CacheDB::new(EmptyDB::default()); - - // DON'T deploy the contract at genesis - let chain_spec = Arc::new( - ChainSpecBuilder::from(&*MAINNET) - .shanghai_activated() - .with_fork(EthereumHardfork::Cancun, ForkCondition::Timestamp(1)) - .build(), - ); - - let provider = evm_config(chain_spec); - - // attempt to execute an empty block with parent beacon block root, this should not fail - provider - .batch_executor(db) - .execute_one(&RecoveredBlock::new_unhashed( - Block { - header, - body: BlockBody { transactions: vec![], ommers: vec![], withdrawals: None }, - }, - vec![], - )) - .expect( - "Executing a block with no transactions while cancun is active should not fail", - ); - } - - #[test] - fn eip_4788_empty_account_call() { - // This test ensures that we do not increment the nonce of an empty SYSTEM_ADDRESS account - // // during the pre-block call - - let mut db = create_database_with_beacon_root_contract(); - - // insert an empty SYSTEM_ADDRESS - db.insert_account_info(SYSTEM_ADDRESS, Default::default()); - - let chain_spec = Arc::new( - ChainSpecBuilder::from(&*MAINNET) - .shanghai_activated() - .with_fork(EthereumHardfork::Cancun, ForkCondition::Timestamp(1)) - .build(), - ); - - let provider = evm_config(chain_spec); - - // construct the header for block one - let header = Header { - timestamp: 1, - number: 1, - parent_beacon_block_root: Some(B256::with_last_byte(0x69)), - excess_blob_gas: Some(0), - ..Header::default() - }; - - let mut executor = provider.batch_executor(db); - - // attempt to execute an empty block with parent beacon block root, this should not fail - executor - .execute_one(&RecoveredBlock::new_unhashed( - Block { - header, - body: BlockBody { transactions: vec![], ommers: vec![], withdrawals: None }, - }, - vec![], - )) - .expect( - "Executing a block with no transactions while cancun is active should not fail", - ); - - // ensure that the nonce of the system address account has not changed - let nonce = - executor.with_state_mut(|state| state.basic(SYSTEM_ADDRESS).unwrap().unwrap().nonce); - assert_eq!(nonce, 0); - } - - #[test] - fn eip_4788_genesis_call() { - let db = create_database_with_beacon_root_contract(); - - // activate cancun at genesis - let chain_spec = Arc::new( - ChainSpecBuilder::from(&*MAINNET) - .shanghai_activated() - .with_fork(EthereumHardfork::Cancun, ForkCondition::Timestamp(0)) - .build(), - ); - - let mut header = chain_spec.genesis_header().clone(); - let provider = evm_config(chain_spec); - let mut executor = provider.batch_executor(db); - - // attempt to execute the genesis block with non-zero parent beacon block root, expect err - header.parent_beacon_block_root = Some(B256::with_last_byte(0x69)); - let _err = executor - .execute_one(&RecoveredBlock::new_unhashed( - Block { header: header.clone(), body: Default::default() }, - vec![], - )) - .expect_err( - "Executing genesis cancun block with non-zero parent beacon block root field - should fail", - ); - - // fix header - header.parent_beacon_block_root = Some(B256::ZERO); - - // now try to process the genesis block again, this time ensuring that a system contract - // call does not occur - executor - .execute_one(&RecoveredBlock::new_unhashed( - Block { header, body: Default::default() }, - vec![], - )) - .unwrap(); - - // there is no system contract call so there should be NO STORAGE CHANGES - // this means we'll check the transition state - let transition_state = executor.with_state_mut(|state| { - state - .transition_state - .take() - .expect("the evm should be initialized with bundle updates") - }); - - // assert that it is the default (empty) transition state - assert_eq!(transition_state, TransitionState::default()); - } - - #[test] - fn eip_4788_high_base_fee() { - // This test ensures that if we have a base fee, then we don't return an error when the - // system contract is called, due to the gas price being less than the base fee. - let header = Header { - timestamp: 1, - number: 1, - parent_beacon_block_root: Some(B256::with_last_byte(0x69)), - base_fee_per_gas: Some(u64::MAX), - excess_blob_gas: Some(0), - ..Header::default() - }; - - let db = create_database_with_beacon_root_contract(); - - let chain_spec = Arc::new( - ChainSpecBuilder::from(&*MAINNET) - .shanghai_activated() - .with_fork(EthereumHardfork::Cancun, ForkCondition::Timestamp(1)) - .build(), - ); - - let provider = evm_config(chain_spec); - - // execute header - let mut executor = provider.batch_executor(db); - - // Now execute a block with the fixed header, ensure that it does not fail - executor - .execute_one(&RecoveredBlock::new_unhashed( - Block { header: header.clone(), body: Default::default() }, - vec![], - )) - .unwrap(); - - // check the actual storage of the contract - it should be: - // * The storage value at header.timestamp % HISTORY_BUFFER_LENGTH should be - // header.timestamp - // * The storage value at header.timestamp % HISTORY_BUFFER_LENGTH + HISTORY_BUFFER_LENGTH - // // should be parent_beacon_block_root - let history_buffer_length = 8191u64; - let timestamp_index = header.timestamp % history_buffer_length; - let parent_beacon_block_root_index = - timestamp_index % history_buffer_length + history_buffer_length; - - // get timestamp storage and compare - let timestamp_storage = executor.with_state_mut(|state| { - state.storage(BEACON_ROOTS_ADDRESS, U256::from(timestamp_index)).unwrap() - }); - assert_eq!(timestamp_storage, U256::from(header.timestamp)); - - // get parent beacon block root storage and compare - let parent_beacon_block_root_storage = executor.with_state_mut(|state| { - state.storage(BEACON_ROOTS_ADDRESS, U256::from(parent_beacon_block_root_index)).unwrap() - }); - assert_eq!(parent_beacon_block_root_storage, U256::from(0x69)); - } - - /// Create a state provider with blockhashes and the EIP-2935 system contract. - fn create_database_with_block_hashes(latest_block: u64) -> CacheDB { - let mut db = CacheDB::new(Default::default()); - for block_number in 0..=latest_block { - db.cache - .block_hashes - .insert(U256::from(block_number), keccak256(block_number.to_string())); - } - - let blockhashes_contract_account = AccountInfo { - balance: U256::ZERO, - code_hash: keccak256(HISTORY_STORAGE_CODE.clone()), - code: Some(Bytecode::new_raw(HISTORY_STORAGE_CODE.clone())), - nonce: 1, - }; - - db.insert_account_info(HISTORY_STORAGE_ADDRESS, blockhashes_contract_account); - - db - } - #[test] - fn eip_2935_pre_fork() { - let db = create_database_with_block_hashes(1); - - let chain_spec = Arc::new( - ChainSpecBuilder::from(&*MAINNET) - .shanghai_activated() - .with_fork(EthereumHardfork::Prague, ForkCondition::Never) - .build(), - ); - - let provider = evm_config(chain_spec); - let mut executor = provider.batch_executor(db); - - // construct the header for block one - let header = Header { timestamp: 1, number: 1, ..Header::default() }; - - // attempt to execute an empty block, this should not fail - executor - .execute_one(&RecoveredBlock::new_unhashed( - Block { header, body: Default::default() }, - vec![], - )) - .expect( - "Executing a block with no transactions while Prague is active should not fail", - ); - - // ensure that the block hash was *not* written to storage, since this is before the fork - // was activated - // - // we load the account first, because revm expects it to be - // loaded - executor.with_state_mut(|state| state.basic(HISTORY_STORAGE_ADDRESS).unwrap()); - assert!(executor.with_state_mut(|state| { - state.storage(HISTORY_STORAGE_ADDRESS, U256::ZERO).unwrap().is_zero() - })); - } - - #[test] - fn eip_2935_fork_activation_genesis() { - let db = create_database_with_block_hashes(0); - - let chain_spec = Arc::new( - ChainSpecBuilder::from(&*MAINNET) - .shanghai_activated() - .cancun_activated() - .prague_activated() - .build(), - ); - - let header = chain_spec.genesis_header().clone(); - let provider = evm_config(chain_spec); - let mut executor = provider.batch_executor(db); - - // attempt to execute genesis block, this should not fail - executor - .execute_one(&RecoveredBlock::new_unhashed( - Block { header, body: Default::default() }, - vec![], - )) - .expect( - "Executing a block with no transactions while Prague is active should not fail", - ); - - // ensure that the block hash was *not* written to storage, since there are no blocks - // preceding genesis - // - // we load the account first, because revm expects it to be - // loaded - executor.with_state_mut(|state| state.basic(HISTORY_STORAGE_ADDRESS).unwrap()); - assert!(executor.with_state_mut(|state| { - state.storage(HISTORY_STORAGE_ADDRESS, U256::ZERO).unwrap().is_zero() - })); - } - - #[test] - fn eip_2935_fork_activation_within_window_bounds() { - let fork_activation_block = (HISTORY_SERVE_WINDOW - 10) as u64; - let db = create_database_with_block_hashes(fork_activation_block); - - let chain_spec = Arc::new( - ChainSpecBuilder::from(&*MAINNET) - .shanghai_activated() - .cancun_activated() - .with_fork(EthereumHardfork::Prague, ForkCondition::Timestamp(1)) - .build(), - ); - - let header = Header { - parent_hash: B256::random(), - timestamp: 1, - number: fork_activation_block, - requests_hash: Some(EMPTY_REQUESTS_HASH), - excess_blob_gas: Some(0), - parent_beacon_block_root: Some(B256::random()), - ..Header::default() - }; - let provider = evm_config(chain_spec); - let mut executor = provider.batch_executor(db); - - // attempt to execute the fork activation block, this should not fail - executor - .execute_one(&RecoveredBlock::new_unhashed( - Block { header, body: Default::default() }, - vec![], - )) - .expect( - "Executing a block with no transactions while Prague is active should not fail", - ); - - // the hash for the ancestor of the fork activation block should be present - assert!(executor - .with_state_mut(|state| state.basic(HISTORY_STORAGE_ADDRESS).unwrap().is_some())); - assert_ne!( - executor.with_state_mut(|state| state - .storage(HISTORY_STORAGE_ADDRESS, U256::from(fork_activation_block - 1)) - .unwrap()), - U256::ZERO - ); - - // the hash of the block itself should not be in storage - assert!(executor.with_state_mut(|state| { - state - .storage(HISTORY_STORAGE_ADDRESS, U256::from(fork_activation_block)) - .unwrap() - .is_zero() - })); - } - - // - #[test] - fn eip_2935_fork_activation_outside_window_bounds() { - let fork_activation_block = (HISTORY_SERVE_WINDOW + 256) as u64; - let db = create_database_with_block_hashes(fork_activation_block); - - let chain_spec = Arc::new( - ChainSpecBuilder::from(&*MAINNET) - .shanghai_activated() - .cancun_activated() - .with_fork(EthereumHardfork::Prague, ForkCondition::Timestamp(1)) - .build(), - ); - - let provider = evm_config(chain_spec); - let mut executor = provider.batch_executor(db); - - let header = Header { - parent_hash: B256::random(), - timestamp: 1, - number: fork_activation_block, - requests_hash: Some(EMPTY_REQUESTS_HASH), - excess_blob_gas: Some(0), - parent_beacon_block_root: Some(B256::random()), - ..Header::default() - }; - - // attempt to execute the fork activation block, this should not fail - executor - .execute_one(&RecoveredBlock::new_unhashed( - Block { header, body: Default::default() }, - vec![], - )) - .expect( - "Executing a block with no transactions while Prague is active should not fail", - ); - - // the hash for the ancestor of the fork activation block should be present - assert!(executor - .with_state_mut(|state| state.basic(HISTORY_STORAGE_ADDRESS).unwrap().is_some())); - } - - #[test] - fn eip_2935_state_transition_inside_fork() { - let db = create_database_with_block_hashes(2); - - let chain_spec = Arc::new( - ChainSpecBuilder::from(&*MAINNET) - .shanghai_activated() - .cancun_activated() - .prague_activated() - .build(), - ); - - let header = chain_spec.genesis_header().clone(); - let header_hash = header.hash_slow(); - - let provider = evm_config(chain_spec); - let mut executor = provider.batch_executor(db); - - // attempt to execute the genesis block, this should not fail - executor - .execute_one(&RecoveredBlock::new_unhashed( - Block { header, body: Default::default() }, - vec![], - )) - .expect( - "Executing a block with no transactions while Prague is active should not fail", - ); - - // nothing should be written as the genesis has no ancestors - // - // we load the account first, because revm expects it to be - // loaded - executor.with_state_mut(|state| state.basic(HISTORY_STORAGE_ADDRESS).unwrap()); - assert!(executor.with_state_mut(|state| { - state.storage(HISTORY_STORAGE_ADDRESS, U256::ZERO).unwrap().is_zero() - })); - - // attempt to execute block 1, this should not fail - let header = Header { - parent_hash: header_hash, - timestamp: 1, - number: 1, - requests_hash: Some(EMPTY_REQUESTS_HASH), - excess_blob_gas: Some(0), - parent_beacon_block_root: Some(B256::random()), - ..Header::default() - }; - let header_hash = header.hash_slow(); - - executor - .execute_one(&RecoveredBlock::new_unhashed( - Block { header, body: Default::default() }, - vec![], - )) - .expect( - "Executing a block with no transactions while Prague is active should not fail", - ); - - // the block hash of genesis should now be in storage, but not block 1 - assert!(executor - .with_state_mut(|state| state.basic(HISTORY_STORAGE_ADDRESS).unwrap().is_some())); - assert_ne!( - executor.with_state_mut(|state| state - .storage(HISTORY_STORAGE_ADDRESS, U256::ZERO) - .unwrap()), - U256::ZERO - ); - assert!(executor.with_state_mut(|state| { - state.storage(HISTORY_STORAGE_ADDRESS, U256::from(1)).unwrap().is_zero() - })); - - // attempt to execute block 2, this should not fail - let header = Header { - parent_hash: header_hash, - timestamp: 1, - number: 2, - requests_hash: Some(EMPTY_REQUESTS_HASH), - excess_blob_gas: Some(0), - parent_beacon_block_root: Some(B256::random()), - ..Header::default() - }; - - executor - .execute_one(&RecoveredBlock::new_unhashed( - Block { header, body: Default::default() }, - vec![], - )) - .expect( - "Executing a block with no transactions while Prague is active should not fail", - ); - - // the block hash of genesis and block 1 should now be in storage, but not block 2 - assert!(executor - .with_state_mut(|state| state.basic(HISTORY_STORAGE_ADDRESS).unwrap().is_some())); - assert_ne!( - executor.with_state_mut(|state| state - .storage(HISTORY_STORAGE_ADDRESS, U256::ZERO) - .unwrap()), - U256::ZERO - ); - assert_ne!( - executor.with_state_mut(|state| state - .storage(HISTORY_STORAGE_ADDRESS, U256::from(1)) - .unwrap()), - U256::ZERO - ); - assert!(executor.with_state_mut(|state| { - state.storage(HISTORY_STORAGE_ADDRESS, U256::from(2)).unwrap().is_zero() - })); - } - - #[test] - fn eip_7002() { - let chain_spec = Arc::new( - ChainSpecBuilder::from(&*MAINNET) - .shanghai_activated() - .cancun_activated() - .prague_activated() - .build(), - ); - - let mut db = create_database_with_withdrawal_requests_contract(); - - let sender_key_pair = generators::generate_key(&mut generators::rng()); - let sender_address = public_key_to_address(sender_key_pair.public_key()); - - db.insert_account_info( - sender_address, - AccountInfo { nonce: 1, balance: U256::from(ETH_TO_WEI), ..Default::default() }, - ); - - // https://github.com/lightclient/sys-asm/blob/9282bdb9fd64e024e27f60f507486ffb2183cba2/test/Withdrawal.t.sol.in#L36 - let validator_public_key = fixed_bytes!( - "111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111" - ); - let withdrawal_amount = fixed_bytes!("0203040506070809"); - let input: Bytes = [&validator_public_key[..], &withdrawal_amount[..]].concat().into(); - assert_eq!(input.len(), 56); - - let mut header = chain_spec.genesis_header().clone(); - header.gas_limit = 1_500_000; - // measured - header.gas_used = 135_856; - header.receipts_root = - b256!("0xb31a3e47b902e9211c4d349af4e4c5604ce388471e79ca008907ae4616bb0ed3"); - - let tx = sign_tx_with_key_pair( - sender_key_pair, - Transaction::Legacy(TxLegacy { - chain_id: Some(chain_spec.chain.id()), - nonce: 1, - gas_price: header.base_fee_per_gas.unwrap().into(), - gas_limit: header.gas_used, - to: TxKind::Call(WITHDRAWAL_REQUEST_PREDEPLOY_ADDRESS), - // `MIN_WITHDRAWAL_REQUEST_FEE` - value: U256::from(2), - input, - }), - ); - - let provider = evm_config(chain_spec); - - let mut executor = provider.batch_executor(db); - - let BlockExecutionResult { receipts, requests, .. } = executor - .execute_one( - &Block { header, body: BlockBody { transactions: vec![tx], ..Default::default() } } - .try_into_recovered() - .unwrap(), - ) - .unwrap(); - - let receipt = receipts.first().unwrap(); - assert!(receipt.success); - - // There should be exactly one entry with withdrawal requests - assert_eq!(requests.len(), 1); - assert_eq!(requests[0][0], 1); - } - - #[test] - fn block_gas_limit_error() { - // Create a chain specification with fork conditions set for Prague - let chain_spec = Arc::new( - ChainSpecBuilder::from(&*MAINNET) - .shanghai_activated() - .with_fork(EthereumHardfork::Prague, ForkCondition::Timestamp(0)) - .build(), - ); - - // Create a state provider with the withdrawal requests contract pre-deployed - let mut db = create_database_with_withdrawal_requests_contract(); - - // Generate a new key pair for the sender - let sender_key_pair = generators::generate_key(&mut generators::rng()); - // Get the sender's address from the public key - let sender_address = public_key_to_address(sender_key_pair.public_key()); - - // Insert the sender account into the state with a nonce of 1 and a balance of 1 ETH in Wei - db.insert_account_info( - sender_address, - AccountInfo { nonce: 1, balance: U256::from(ETH_TO_WEI), ..Default::default() }, - ); - - // Define the validator public key and withdrawal amount as fixed bytes - let validator_public_key = fixed_bytes!( - "111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111" - ); - let withdrawal_amount = fixed_bytes!("2222222222222222"); - // Concatenate the validator public key and withdrawal amount into a single byte array - let input: Bytes = [&validator_public_key[..], &withdrawal_amount[..]].concat().into(); - // Ensure the input length is 56 bytes - assert_eq!(input.len(), 56); - - // Create a genesis block header with a specified gas limit and gas used - let mut header = chain_spec.genesis_header().clone(); - header.gas_limit = 1_500_000; - header.gas_used = 134_807; - header.receipts_root = - b256!("0xb31a3e47b902e9211c4d349af4e4c5604ce388471e79ca008907ae4616bb0ed3"); - - // Create a transaction with a gas limit higher than the block gas limit - let tx = sign_tx_with_key_pair( - sender_key_pair, - Transaction::Legacy(TxLegacy { - chain_id: Some(chain_spec.chain.id()), - nonce: 1, - gas_price: header.base_fee_per_gas.unwrap().into(), - gas_limit: 2_500_000, // higher than block gas limit - to: TxKind::Call(WITHDRAWAL_REQUEST_PREDEPLOY_ADDRESS), - value: U256::from(1), - input, - }), - ); - - // Create an executor from the state provider - let evm_config = evm_config(chain_spec); - let mut executor = evm_config.batch_executor(db); - - // Execute the block and capture the result - let exec_result = executor.execute_one( - &Block { header, body: BlockBody { transactions: vec![tx], ..Default::default() } } - .try_into_recovered() - .unwrap(), - ); - - // Check if the execution result is an error and assert the specific error type - match exec_result { - Ok(_) => panic!("Expected block gas limit error"), - Err(err) => assert!(matches!( - *err.as_validation().unwrap(), - BlockValidationError::TransactionGasLimitMoreThanAvailableBlockGas { - transaction_gas_limit: 2_500_000, - block_available_gas: 1_500_000, - } - )), - } - } - - #[test] - fn test_balance_increment_not_duplicated() { - let chain_spec = Arc::new( - ChainSpecBuilder::from(&*MAINNET) - .shanghai_activated() - .cancun_activated() - .prague_activated() - .build(), - ); - - let withdrawal_recipient = address!("0x1000000000000000000000000000000000000000"); - - let mut db = CacheDB::new(EmptyDB::default()); - let initial_balance = 100; - db.insert_account_info( - withdrawal_recipient, - AccountInfo { balance: U256::from(initial_balance), nonce: 1, ..Default::default() }, - ); - - let withdrawal = - Withdrawal { index: 0, validator_index: 0, address: withdrawal_recipient, amount: 1 }; - - let header = Header { - timestamp: 1, - number: 1, - excess_blob_gas: Some(0), - parent_beacon_block_root: Some(B256::random()), - ..Header::default() - }; - - let block = &RecoveredBlock::new_unhashed( - Block { - header, - body: BlockBody { - transactions: vec![], - ommers: vec![], - withdrawals: Some(vec![withdrawal].into()), - }, - }, - vec![], - ); - - let provider = evm_config(chain_spec); - let executor = provider.batch_executor(db); - - let (tx, rx) = mpsc::channel(); - let tx_clone = tx.clone(); - - let _output = executor - .execute_with_state_hook(block, move |_, state: &EvmState| { - if let Some(account) = state.get(&withdrawal_recipient) { - let _ = tx_clone.send(account.info.balance); - } - }) - .expect("Block execution should succeed"); - - drop(tx); - let balance_changes: Vec = rx.try_iter().collect(); - - if let Some(final_balance) = balance_changes.last() { - let expected_final_balance = U256::from(initial_balance) + U256::from(1_000_000_000); // initial + 1 Gwei in Wei - assert_eq!( - *final_balance, expected_final_balance, - "Final balance should match expected value after withdrawal" - ); - } - } -} diff --git a/crates/ethereum/evm/src/lib.rs b/crates/ethereum/evm/src/lib.rs index 1d4d4f111ea..c0f8adc9c54 100644 --- a/crates/ethereum/evm/src/lib.rs +++ b/crates/ethereum/evm/src/lib.rs @@ -12,24 +12,33 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(feature = "std"), no_std)] extern crate alloc; -use alloc::{borrow::Cow, sync::Arc, vec::Vec}; -use alloy_consensus::{BlockHeader, Header}; +use alloc::{borrow::Cow, sync::Arc}; +use alloy_consensus::Header; +use alloy_eips::Decodable2718; pub use alloy_evm::EthEvm; use alloy_evm::{ eth::{EthBlockExecutionCtx, EthBlockExecutorFactory}, EthEvmFactory, FromRecoveredTx, FromTxWithEncoded, }; use alloy_primitives::{Bytes, U256}; +use alloy_rpc_types_engine::ExecutionData; use core::{convert::Infallible, fmt::Debug}; -use reth_chainspec::{ChainSpec, EthChainSpec, MAINNET}; +use reth_chainspec::{ChainSpec, EthChainSpec, EthereumHardforks, MAINNET}; use reth_ethereum_primitives::{Block, EthPrimitives, TransactionSigned}; -use reth_evm::{ConfigureEvm, EvmEnv, EvmFactory, NextBlockEnvAttributes, TransactionEnv}; -use reth_primitives_traits::{SealedBlock, SealedHeader}; +use reth_evm::{ + eth::NextEvmEnvAttributes, precompiles::PrecompilesMap, ConfigureEngineEvm, ConfigureEvm, + EvmEnv, EvmEnvFor, EvmFactory, ExecutableTxIterator, ExecutionCtxFor, NextBlockEnvAttributes, + TransactionEnv, +}; +use reth_primitives_traits::{ + constants::MAX_TX_GAS_LIMIT_OSAKA, SealedBlock, SealedHeader, SignedTransaction, TxTy, +}; +use reth_storage_errors::any::AnyError; use revm::{ context::{BlockEnv, CfgEnv}, context_interface::block::BlobExcessGasAndPrice, @@ -37,11 +46,19 @@ use revm::{ }; mod config; -use alloy_eips::{eip1559::INITIAL_BASE_FEE, eip7840::BlobParams}; +use alloy_evm::eth::spec::EthExecutorSpec; pub use config::{revm_spec, revm_spec_by_timestamp_and_block_number}; -use reth_ethereum_forks::EthereumHardfork; +use reth_ethereum_forks::Hardforks; + +/// Helper type with backwards compatible methods to obtain Ethereum executor +/// providers. +#[doc(hidden)] +pub mod execute { + use crate::EthEvmConfig; -pub mod execute; + #[deprecated(note = "Use `EthEvmConfig` instead")] + pub type EthExecutorProvider = EthEvmConfig; +} mod build; pub use build::EthBlockAssembler; @@ -56,14 +73,21 @@ pub use test_utils::*; /// Ethereum-related EVM configuration. #[derive(Debug, Clone)] -pub struct EthEvmConfig { +pub struct EthEvmConfig { /// Inner [`EthBlockExecutorFactory`]. - pub executor_factory: EthBlockExecutorFactory, EvmFactory>, + pub executor_factory: EthBlockExecutorFactory, EvmFactory>, /// Ethereum block assembler. - pub block_assembler: EthBlockAssembler, + pub block_assembler: EthBlockAssembler, } impl EthEvmConfig { + /// Creates a new Ethereum EVM configuration for the ethereum mainnet. + pub fn mainnet() -> Self { + Self::ethereum(MAINNET.clone()) + } +} + +impl EthEvmConfig { /// Creates a new Ethereum EVM configuration with the given chain spec. pub fn new(chain_spec: Arc) -> Self { Self::ethereum(chain_spec) @@ -73,14 +97,9 @@ impl EthEvmConfig { pub fn ethereum(chain_spec: Arc) -> Self { Self::new_with_evm_factory(chain_spec, EthEvmFactory::default()) } - - /// Creates a new Ethereum EVM configuration for the ethereum mainnet. - pub fn mainnet() -> Self { - Self::ethereum(MAINNET.clone()) - } } -impl EthEvmConfig { +impl EthEvmConfig { /// Creates a new Ethereum EVM configuration with the given chain spec and EVM factory. pub fn new_with_evm_factory(chain_spec: Arc, evm_factory: EvmFactory) -> Self { Self { @@ -98,17 +117,6 @@ impl EthEvmConfig { self.executor_factory.spec() } - /// Returns blob params by hard fork as specified in chain spec. - /// Blob params are in format `(spec id, target blob count, max blob count)`. - pub fn blob_max_and_target_count_by_hardfork(&self) -> Vec<(SpecId, u64, u64)> { - let cancun = self.chain_spec().blob_params.cancun(); - let prague = self.chain_spec().blob_params.prague(); - Vec::from([ - (SpecId::CANCUN, cancun.target_blob_count, cancun.max_blob_count), - (SpecId::PRAGUE, prague.target_blob_count, prague.max_blob_count), - ]) - } - /// Sets the extra data for the block assembler. pub fn with_extra_data(mut self, extra_data: Bytes) -> Self { self.block_assembler.extra_data = extra_data; @@ -116,13 +124,16 @@ impl EthEvmConfig { } } -impl ConfigureEvm for EthEvmConfig +impl ConfigureEvm for EthEvmConfig where + ChainSpec: EthExecutorSpec + EthChainSpec
+ Hardforks + 'static, EvmF: EvmFactory< Tx: TransactionEnv + FromRecoveredTx + FromTxWithEncoded, Spec = SpecId, + BlockEnv = BlockEnv, + Precompiles = PrecompilesMap, > + Clone + Debug + Send @@ -144,37 +155,13 @@ where &self.block_assembler } - fn evm_env(&self, header: &Header) -> EvmEnv { - let spec = config::revm_spec(self.chain_spec(), header); - - // configure evm env based on parent block - let cfg_env = CfgEnv::new() - .with_chain_id(self.chain_spec().chain().id()) - .with_spec(spec) - .with_blob_max_and_target_count(self.blob_max_and_target_count_by_hardfork()); - - // derive the EIP-4844 blob fees from the header's `excess_blob_gas` and the current - // blobparams - let blob_excess_gas_and_price = header - .excess_blob_gas - .zip(self.chain_spec().blob_params_at_timestamp(header.timestamp)) - .map(|(excess_blob_gas, params)| { - let blob_gasprice = params.calc_blob_fee(excess_blob_gas); - BlobExcessGasAndPrice { excess_blob_gas, blob_gasprice } - }); - - let block_env = BlockEnv { - number: header.number(), - beneficiary: header.beneficiary(), - timestamp: header.timestamp(), - difficulty: if spec >= SpecId::MERGE { U256::ZERO } else { header.difficulty() }, - prevrandao: if spec >= SpecId::MERGE { header.mix_hash() } else { None }, - gas_limit: header.gas_limit(), - basefee: header.base_fee_per_gas().unwrap_or_default(), - blob_excess_gas_and_price, - }; - - EvmEnv { cfg_env, block_env } + fn evm_env(&self, header: &Header) -> Result, Self::Error> { + Ok(EvmEnv::for_eth_block( + header, + self.chain_spec(), + self.chain_spec().chain().id(), + self.chain_spec().blob_params_at_timestamp(header.timestamp), + )) } fn next_evm_env( @@ -182,89 +169,132 @@ where parent: &Header, attributes: &NextBlockEnvAttributes, ) -> Result { - // ensure we're not missing any timestamp based hardforks - let spec_id = revm_spec_by_timestamp_and_block_number( + Ok(EvmEnv::for_eth_next_block( + parent, + NextEvmEnvAttributes { + timestamp: attributes.timestamp, + suggested_fee_recipient: attributes.suggested_fee_recipient, + prev_randao: attributes.prev_randao, + gas_limit: attributes.gas_limit, + }, + self.chain_spec().next_block_base_fee(parent, attributes.timestamp).unwrap_or_default(), self.chain_spec(), - attributes.timestamp, - parent.number() + 1, - ); - - // configure evm env based on parent block - let cfg = CfgEnv::new() - .with_chain_id(self.chain_spec().chain().id()) - .with_spec(spec_id) - .with_blob_max_and_target_count(self.blob_max_and_target_count_by_hardfork()); - - let blob_params = self.chain_spec().blob_params_at_timestamp(attributes.timestamp); - // if the parent block did not have excess blob gas (i.e. it was pre-cancun), but it is - // cancun now, we need to set the excess blob gas to the default value(0) - let blob_excess_gas_and_price = parent - .maybe_next_block_excess_blob_gas(blob_params) - .or_else(|| (spec_id == SpecId::CANCUN).then_some(0)) - .map(|excess_blob_gas| { - let blob_gasprice = - blob_params.unwrap_or_else(BlobParams::cancun).calc_blob_fee(excess_blob_gas); - BlobExcessGasAndPrice { excess_blob_gas, blob_gasprice } - }); - - let mut basefee = parent.next_block_base_fee( - self.chain_spec().base_fee_params_at_timestamp(attributes.timestamp), - ); - - let mut gas_limit = attributes.gas_limit; - - // If we are on the London fork boundary, we need to multiply the parent's gas limit by the - // elasticity multiplier to get the new gas limit. - if self.chain_spec().fork(EthereumHardfork::London).transitions_at_block(parent.number + 1) - { - let elasticity_multiplier = self - .chain_spec() - .base_fee_params_at_timestamp(attributes.timestamp) - .elasticity_multiplier; - - // multiply the gas limit by the elasticity multiplier - gas_limit *= elasticity_multiplier as u64; - - // set the base fee to the initial base fee from the EIP-1559 spec - basefee = Some(INITIAL_BASE_FEE) - } - - let block_env = BlockEnv { - number: parent.number + 1, - beneficiary: attributes.suggested_fee_recipient, - timestamp: attributes.timestamp, - difficulty: U256::ZERO, - prevrandao: Some(attributes.prev_randao), - gas_limit, - // calculate basefee based on parent block's gas usage - basefee: basefee.unwrap_or_default(), - // calculate excess gas based on parent block's blob gas usage - blob_excess_gas_and_price, - }; - - Ok((cfg, block_env).into()) + self.chain_spec().chain().id(), + self.chain_spec().blob_params_at_timestamp(attributes.timestamp), + )) } - fn context_for_block<'a>(&self, block: &'a SealedBlock) -> EthBlockExecutionCtx<'a> { - EthBlockExecutionCtx { + fn context_for_block<'a>( + &self, + block: &'a SealedBlock, + ) -> Result, Self::Error> { + Ok(EthBlockExecutionCtx { parent_hash: block.header().parent_hash, parent_beacon_block_root: block.header().parent_beacon_block_root, ommers: &block.body().ommers, withdrawals: block.body().withdrawals.as_ref().map(Cow::Borrowed), - } + }) } fn context_for_next_block( &self, parent: &SealedHeader, attributes: Self::NextBlockEnvCtx, - ) -> EthBlockExecutionCtx<'_> { - EthBlockExecutionCtx { + ) -> Result, Self::Error> { + Ok(EthBlockExecutionCtx { parent_hash: parent.hash(), parent_beacon_block_root: attributes.parent_beacon_block_root, ommers: &[], withdrawals: attributes.withdrawals.map(Cow::Owned), + }) + } +} + +impl ConfigureEngineEvm for EthEvmConfig +where + ChainSpec: EthExecutorSpec + EthChainSpec
+ Hardforks + 'static, + EvmF: EvmFactory< + Tx: TransactionEnv + + FromRecoveredTx + + FromTxWithEncoded, + Spec = SpecId, + BlockEnv = BlockEnv, + Precompiles = PrecompilesMap, + > + Clone + + Debug + + Send + + Sync + + Unpin + + 'static, +{ + fn evm_env_for_payload(&self, payload: &ExecutionData) -> Result, Self::Error> { + let timestamp = payload.payload.timestamp(); + let block_number = payload.payload.block_number(); + + let blob_params = self.chain_spec().blob_params_at_timestamp(timestamp); + let spec = + revm_spec_by_timestamp_and_block_number(self.chain_spec(), timestamp, block_number); + + // configure evm env based on parent block + let mut cfg_env = + CfgEnv::new().with_chain_id(self.chain_spec().chain().id()).with_spec(spec); + + if let Some(blob_params) = &blob_params { + cfg_env.set_max_blobs_per_tx(blob_params.max_blobs_per_tx); + } + + if self.chain_spec().is_osaka_active_at_timestamp(timestamp) { + cfg_env.tx_gas_limit_cap = Some(MAX_TX_GAS_LIMIT_OSAKA); } + + // derive the EIP-4844 blob fees from the header's `excess_blob_gas` and the current + // blobparams + let blob_excess_gas_and_price = + payload.payload.excess_blob_gas().zip(blob_params).map(|(excess_blob_gas, params)| { + let blob_gasprice = params.calc_blob_fee(excess_blob_gas); + BlobExcessGasAndPrice { excess_blob_gas, blob_gasprice } + }); + + let block_env = BlockEnv { + number: U256::from(block_number), + beneficiary: payload.payload.fee_recipient(), + timestamp: U256::from(timestamp), + difficulty: if spec >= SpecId::MERGE { + U256::ZERO + } else { + payload.payload.as_v1().prev_randao.into() + }, + prevrandao: (spec >= SpecId::MERGE).then(|| payload.payload.as_v1().prev_randao), + gas_limit: payload.payload.gas_limit(), + basefee: payload.payload.saturated_base_fee_per_gas(), + blob_excess_gas_and_price, + }; + + Ok(EvmEnv { cfg_env, block_env }) + } + + fn context_for_payload<'a>( + &self, + payload: &'a ExecutionData, + ) -> Result, Self::Error> { + Ok(EthBlockExecutionCtx { + parent_hash: payload.parent_hash(), + parent_beacon_block_root: payload.sidecar.parent_beacon_block_root(), + ommers: &[], + withdrawals: payload.payload.withdrawals().map(|w| Cow::Owned(w.clone().into())), + }) + } + + fn tx_iterator_for_payload( + &self, + payload: &ExecutionData, + ) -> Result, Self::Error> { + Ok(payload.payload.transactions().clone().into_iter().map(|tx| { + let tx = + TxTy::::decode_2718_exact(tx.as_ref()).map_err(AnyError::new)?; + let signer = tx.try_recover().map_err(AnyError::new)?; + Ok::<_, AnyError>(tx.with_signer(signer)) + })) } } @@ -300,7 +330,7 @@ mod tests { // Use the `EthEvmConfig` to fill the `cfg_env` and `block_env` based on the ChainSpec, // Header, and total difficulty let EvmEnv { cfg_env, .. } = - EthEvmConfig::new(Arc::new(chain_spec.clone())).evm_env(&header); + EthEvmConfig::new(Arc::new(chain_spec.clone())).evm_env(&header).unwrap(); // Assert that the chain ID in the `cfg_env` is correctly set to the chain ID of the // ChainSpec @@ -346,8 +376,12 @@ mod tests { let db = CacheDB::>::default(); // Create customs block and tx env - let block = - BlockEnv { basefee: 1000, gas_limit: 10_000_000, number: 42, ..Default::default() }; + let block = BlockEnv { + basefee: 1000, + gas_limit: 10_000_000, + number: U256::from(42), + ..Default::default() + }; let evm_env = EvmEnv { block_env: block, ..Default::default() }; @@ -413,8 +447,12 @@ mod tests { let db = CacheDB::>::default(); // Create custom block and tx environment - let block = - BlockEnv { basefee: 1000, gas_limit: 10_000_000, number: 42, ..Default::default() }; + let block = BlockEnv { + basefee: 1000, + gas_limit: 10_000_000, + number: U256::from(42), + ..Default::default() + }; let evm_env = EvmEnv { block_env: block, ..Default::default() }; let evm = evm_config.evm_with_env_and_inspector(db, evm_env.clone(), NoOpInspector {}); diff --git a/crates/ethereum/evm/src/test_utils.rs b/crates/ethereum/evm/src/test_utils.rs index 9639c4f6751..fe791b9f5fd 100644 --- a/crates/ethereum/evm/src/test_utils.rs +++ b/crates/ethereum/evm/src/test_utils.rs @@ -1,7 +1,10 @@ use crate::EthEvmConfig; -use alloc::{boxed::Box, sync::Arc, vec::Vec}; +use alloc::{boxed::Box, sync::Arc, vec, vec::Vec}; use alloy_consensus::Header; use alloy_eips::eip7685::Requests; +use alloy_evm::precompiles::PrecompilesMap; +use alloy_primitives::Bytes; +use alloy_rpc_types_engine::ExecutionData; use parking_lot::Mutex; use reth_ethereum_primitives::{Receipt, TransactionSigned}; use reth_evm::{ @@ -9,12 +12,13 @@ use reth_evm::{ BlockExecutionError, BlockExecutor, BlockExecutorFactory, BlockExecutorFor, ExecutableTx, }, eth::{EthBlockExecutionCtx, EthEvmContext}, - ConfigureEvm, Database, EthEvm, EthEvmFactory, Evm, EvmEnvFor, EvmFactory, + ConfigureEngineEvm, ConfigureEvm, Database, EthEvm, EthEvmFactory, Evm, EvmEnvFor, EvmFactory, + ExecutableTxIterator, ExecutionCtxFor, }; use reth_execution_types::{BlockExecutionResult, ExecutionOutcome}; use reth_primitives_traits::{BlockTy, SealedBlock, SealedHeader}; use revm::{ - context::result::{ExecutionResult, HaltReason}, + context::result::{ExecutionResult, Output, ResultAndState, SuccessReason}, database::State, Inspector, }; @@ -54,7 +58,7 @@ impl BlockExecutorFactory for MockEvmConfig { fn create_executor<'a, DB, I>( &'a self, - evm: EthEvm<&'a mut State, I>, + evm: EthEvm<&'a mut State, I, PrecompilesMap>, _ctx: Self::ExecutionCtx<'a>, ) -> impl BlockExecutorFor<'a, Self, DB, I> where @@ -69,7 +73,7 @@ impl BlockExecutorFactory for MockEvmConfig { #[derive(derive_more::Debug)] pub struct MockExecutor<'a, DB: Database, I> { result: ExecutionOutcome, - evm: EthEvm<&'a mut State, I>, + evm: EthEvm<&'a mut State, I, PrecompilesMap>, #[debug(skip)] hook: Option>, } @@ -77,7 +81,7 @@ pub struct MockExecutor<'a, DB: Database, I> { impl<'a, DB: Database, I: Inspector>>> BlockExecutor for MockExecutor<'a, DB, I> { - type Evm = EthEvm<&'a mut State, I>; + type Evm = EthEvm<&'a mut State, I, PrecompilesMap>; type Transaction = TransactionSigned; type Receipt = Receipt; @@ -85,10 +89,26 @@ impl<'a, DB: Database, I: Inspector>>> BlockExec Ok(()) } - fn execute_transaction_with_result_closure( + fn execute_transaction_without_commit( &mut self, _tx: impl ExecutableTx, - _f: impl FnOnce(&ExecutionResult), + ) -> Result::HaltReason>, BlockExecutionError> { + Ok(ResultAndState::new( + ExecutionResult::Success { + reason: SuccessReason::Return, + gas_used: 0, + gas_refunded: 0, + logs: vec![], + output: Output::Call(Bytes::from(vec![])), + }, + Default::default(), + )) + } + + fn commit_transaction( + &mut self, + _output: ResultAndState<::HaltReason>, + _tx: impl ExecutableTx, ) -> Result { Ok(0) } @@ -105,6 +125,7 @@ impl<'a, DB: Database, I: Inspector>>> BlockExec reqs }), gas_used: 0, + blob_gas_used: 0, }; evm.db_mut().bundle_state = bundle; @@ -140,7 +161,7 @@ impl ConfigureEvm for MockEvmConfig { self.inner.block_assembler() } - fn evm_env(&self, header: &Header) -> EvmEnvFor { + fn evm_env(&self, header: &Header) -> Result, Self::Error> { self.inner.evm_env(header) } @@ -155,7 +176,7 @@ impl ConfigureEvm for MockEvmConfig { fn context_for_block<'a>( &self, block: &'a SealedBlock>, - ) -> reth_evm::ExecutionCtxFor<'a, Self> { + ) -> Result, Self::Error> { self.inner.context_for_block(block) } @@ -163,7 +184,27 @@ impl ConfigureEvm for MockEvmConfig { &self, parent: &SealedHeader, attributes: Self::NextBlockEnvCtx, - ) -> reth_evm::ExecutionCtxFor<'_, Self> { + ) -> Result, Self::Error> { self.inner.context_for_next_block(parent, attributes) } } + +impl ConfigureEngineEvm for MockEvmConfig { + fn evm_env_for_payload(&self, payload: &ExecutionData) -> Result, Self::Error> { + self.inner.evm_env_for_payload(payload) + } + + fn context_for_payload<'a>( + &self, + payload: &'a ExecutionData, + ) -> Result, Self::Error> { + self.inner.context_for_payload(payload) + } + + fn tx_iterator_for_payload( + &self, + payload: &ExecutionData, + ) -> Result, Self::Error> { + self.inner.tx_iterator_for_payload(payload) + } +} diff --git a/crates/ethereum/evm/tests/execute.rs b/crates/ethereum/evm/tests/execute.rs new file mode 100644 index 00000000000..61e0c1c4b66 --- /dev/null +++ b/crates/ethereum/evm/tests/execute.rs @@ -0,0 +1,828 @@ +//! Execution tests. + +use alloy_consensus::{constants::ETH_TO_WEI, Header, TxLegacy}; +use alloy_eips::{ + eip2935::{HISTORY_SERVE_WINDOW, HISTORY_STORAGE_ADDRESS, HISTORY_STORAGE_CODE}, + eip4788::{BEACON_ROOTS_ADDRESS, BEACON_ROOTS_CODE, SYSTEM_ADDRESS}, + eip4895::Withdrawal, + eip7002::{WITHDRAWAL_REQUEST_PREDEPLOY_ADDRESS, WITHDRAWAL_REQUEST_PREDEPLOY_CODE}, + eip7685::EMPTY_REQUESTS_HASH, +}; +use alloy_evm::block::BlockValidationError; +use alloy_primitives::{b256, fixed_bytes, keccak256, Bytes, TxKind, B256, U256}; +use reth_chainspec::{ChainSpecBuilder, EthereumHardfork, ForkCondition, MAINNET}; +use reth_ethereum_primitives::{Block, BlockBody, Transaction}; +use reth_evm::{ + execute::{BasicBlockExecutor, Executor}, + ConfigureEvm, +}; +use reth_evm_ethereum::EthEvmConfig; +use reth_execution_types::BlockExecutionResult; +use reth_primitives_traits::{ + crypto::secp256k1::public_key_to_address, Block as _, RecoveredBlock, +}; +use reth_testing_utils::generators::{self, sign_tx_with_key_pair}; +use revm::{ + database::{CacheDB, EmptyDB, TransitionState}, + primitives::address, + state::{AccountInfo, Bytecode, EvmState}, + Database, +}; +use std::sync::{mpsc, Arc}; + +fn create_database_with_beacon_root_contract() -> CacheDB { + let mut db = CacheDB::new(Default::default()); + + let beacon_root_contract_account = AccountInfo { + balance: U256::ZERO, + code_hash: keccak256(BEACON_ROOTS_CODE.clone()), + nonce: 1, + code: Some(Bytecode::new_raw(BEACON_ROOTS_CODE.clone())), + }; + + db.insert_account_info(BEACON_ROOTS_ADDRESS, beacon_root_contract_account); + + db +} + +fn create_database_with_withdrawal_requests_contract() -> CacheDB { + let mut db = CacheDB::new(Default::default()); + + let withdrawal_requests_contract_account = AccountInfo { + nonce: 1, + balance: U256::ZERO, + code_hash: keccak256(WITHDRAWAL_REQUEST_PREDEPLOY_CODE.clone()), + code: Some(Bytecode::new_raw(WITHDRAWAL_REQUEST_PREDEPLOY_CODE.clone())), + }; + + db.insert_account_info( + WITHDRAWAL_REQUEST_PREDEPLOY_ADDRESS, + withdrawal_requests_contract_account, + ); + + db +} + +#[test] +fn eip_4788_non_genesis_call() { + let mut header = + Header { timestamp: 1, number: 1, excess_blob_gas: Some(0), ..Header::default() }; + + let db = create_database_with_beacon_root_contract(); + + let chain_spec = Arc::new( + ChainSpecBuilder::from(&*MAINNET) + .shanghai_activated() + .with_fork(EthereumHardfork::Cancun, ForkCondition::Timestamp(1)) + .build(), + ); + + let provider = EthEvmConfig::new(chain_spec); + + let mut executor = BasicBlockExecutor::new(provider, db); + + // attempt to execute a block without parent beacon block root, expect err + let err = executor + .execute_one(&RecoveredBlock::new_unhashed( + Block { + header: header.clone(), + body: BlockBody { transactions: vec![], ommers: vec![], withdrawals: None }, + }, + vec![], + )) + .expect_err("Executing cancun block without parent beacon block root field should fail"); + + assert!(matches!( + err.as_validation().unwrap(), + BlockValidationError::MissingParentBeaconBlockRoot + )); + + // fix header, set a gas limit + header.parent_beacon_block_root = Some(B256::with_last_byte(0x69)); + + // Now execute a block with the fixed header, ensure that it does not fail + executor + .execute_one(&RecoveredBlock::new_unhashed( + Block { + header: header.clone(), + body: BlockBody { transactions: vec![], ommers: vec![], withdrawals: None }, + }, + vec![], + )) + .unwrap(); + + // check the actual storage of the contract - it should be: + // * The storage value at header.timestamp % HISTORY_BUFFER_LENGTH should be + // header.timestamp + // * The storage value at header.timestamp % HISTORY_BUFFER_LENGTH + HISTORY_BUFFER_LENGTH // + // should be parent_beacon_block_root + let history_buffer_length = 8191u64; + let timestamp_index = header.timestamp % history_buffer_length; + let parent_beacon_block_root_index = + timestamp_index % history_buffer_length + history_buffer_length; + + let timestamp_storage = executor.with_state_mut(|state| { + state.storage(BEACON_ROOTS_ADDRESS, U256::from(timestamp_index)).unwrap() + }); + assert_eq!(timestamp_storage, U256::from(header.timestamp)); + + // get parent beacon block root storage and compare + let parent_beacon_block_root_storage = executor.with_state_mut(|state| { + state + .storage(BEACON_ROOTS_ADDRESS, U256::from(parent_beacon_block_root_index)) + .expect("storage value should exist") + }); + assert_eq!(parent_beacon_block_root_storage, U256::from(0x69)); +} + +#[test] +fn eip_4788_no_code_cancun() { + // This test ensures that we "silently fail" when cancun is active and there is no code at + // // BEACON_ROOTS_ADDRESS + let header = Header { + timestamp: 1, + number: 1, + parent_beacon_block_root: Some(B256::with_last_byte(0x69)), + excess_blob_gas: Some(0), + ..Header::default() + }; + + let db = CacheDB::new(EmptyDB::default()); + + // DON'T deploy the contract at genesis + let chain_spec = Arc::new( + ChainSpecBuilder::from(&*MAINNET) + .shanghai_activated() + .with_fork(EthereumHardfork::Cancun, ForkCondition::Timestamp(1)) + .build(), + ); + + let provider = EthEvmConfig::new(chain_spec); + + // attempt to execute an empty block with parent beacon block root, this should not fail + provider + .batch_executor(db) + .execute_one(&RecoveredBlock::new_unhashed( + Block { + header, + body: BlockBody { transactions: vec![], ommers: vec![], withdrawals: None }, + }, + vec![], + )) + .expect("Executing a block with no transactions while cancun is active should not fail"); +} + +#[test] +fn eip_4788_empty_account_call() { + // This test ensures that we do not increment the nonce of an empty SYSTEM_ADDRESS account + // // during the pre-block call + + let mut db = create_database_with_beacon_root_contract(); + + // insert an empty SYSTEM_ADDRESS + db.insert_account_info(SYSTEM_ADDRESS, Default::default()); + + let chain_spec = Arc::new( + ChainSpecBuilder::from(&*MAINNET) + .shanghai_activated() + .with_fork(EthereumHardfork::Cancun, ForkCondition::Timestamp(1)) + .build(), + ); + + let provider = EthEvmConfig::new(chain_spec); + + // construct the header for block one + let header = Header { + timestamp: 1, + number: 1, + parent_beacon_block_root: Some(B256::with_last_byte(0x69)), + excess_blob_gas: Some(0), + ..Header::default() + }; + + let mut executor = BasicBlockExecutor::new(provider, db); + + // attempt to execute an empty block with parent beacon block root, this should not fail + executor + .execute_one(&RecoveredBlock::new_unhashed( + Block { + header, + body: BlockBody { transactions: vec![], ommers: vec![], withdrawals: None }, + }, + vec![], + )) + .expect("Executing a block with no transactions while cancun is active should not fail"); + + // ensure that the nonce of the system address account has not changed + let nonce = + executor.with_state_mut(|state| state.basic(SYSTEM_ADDRESS).unwrap().unwrap().nonce); + assert_eq!(nonce, 0); +} + +#[test] +fn eip_4788_genesis_call() { + let db = create_database_with_beacon_root_contract(); + + // activate cancun at genesis + let chain_spec = Arc::new( + ChainSpecBuilder::from(&*MAINNET) + .shanghai_activated() + .with_fork(EthereumHardfork::Cancun, ForkCondition::Timestamp(0)) + .build(), + ); + + let mut header = chain_spec.genesis_header().clone(); + let provider = EthEvmConfig::new(chain_spec); + let mut executor = BasicBlockExecutor::new(provider, db); + + // attempt to execute the genesis block with non-zero parent beacon block root, expect err + header.parent_beacon_block_root = Some(B256::with_last_byte(0x69)); + let _err = executor + .execute_one(&RecoveredBlock::new_unhashed( + Block { header: header.clone(), body: Default::default() }, + vec![], + )) + .expect_err( + "Executing genesis cancun block with non-zero parent beacon block root field + should fail", + ); + + // fix header + header.parent_beacon_block_root = Some(B256::ZERO); + + // now try to process the genesis block again, this time ensuring that a system contract + // call does not occur + executor + .execute_one(&RecoveredBlock::new_unhashed( + Block { header, body: Default::default() }, + vec![], + )) + .unwrap(); + + // there is no system contract call so there should be NO STORAGE CHANGES + // this means we'll check the transition state + let transition_state = executor.with_state_mut(|state| { + state.transition_state.take().expect("the evm should be initialized with bundle updates") + }); + + // assert that it is the default (empty) transition state + assert_eq!(transition_state, TransitionState::default()); +} + +#[test] +fn eip_4788_high_base_fee() { + // This test ensures that if we have a base fee, then we don't return an error when the + // system contract is called, due to the gas price being less than the base fee. + let header = Header { + timestamp: 1, + number: 1, + parent_beacon_block_root: Some(B256::with_last_byte(0x69)), + base_fee_per_gas: Some(u64::MAX), + excess_blob_gas: Some(0), + ..Header::default() + }; + + let db = create_database_with_beacon_root_contract(); + + let chain_spec = Arc::new( + ChainSpecBuilder::from(&*MAINNET) + .shanghai_activated() + .with_fork(EthereumHardfork::Cancun, ForkCondition::Timestamp(1)) + .build(), + ); + + let provider = EthEvmConfig::new(chain_spec); + + // execute header + let mut executor = BasicBlockExecutor::new(provider, db); + + // Now execute a block with the fixed header, ensure that it does not fail + executor + .execute_one(&RecoveredBlock::new_unhashed( + Block { header: header.clone(), body: Default::default() }, + vec![], + )) + .unwrap(); + + // check the actual storage of the contract - it should be: + // * The storage value at header.timestamp % HISTORY_BUFFER_LENGTH should be + // header.timestamp + // * The storage value at header.timestamp % HISTORY_BUFFER_LENGTH + HISTORY_BUFFER_LENGTH // + // should be parent_beacon_block_root + let history_buffer_length = 8191u64; + let timestamp_index = header.timestamp % history_buffer_length; + let parent_beacon_block_root_index = + timestamp_index % history_buffer_length + history_buffer_length; + + // get timestamp storage and compare + let timestamp_storage = executor.with_state_mut(|state| { + state.storage(BEACON_ROOTS_ADDRESS, U256::from(timestamp_index)).unwrap() + }); + assert_eq!(timestamp_storage, U256::from(header.timestamp)); + + // get parent beacon block root storage and compare + let parent_beacon_block_root_storage = executor.with_state_mut(|state| { + state.storage(BEACON_ROOTS_ADDRESS, U256::from(parent_beacon_block_root_index)).unwrap() + }); + assert_eq!(parent_beacon_block_root_storage, U256::from(0x69)); +} + +/// Create a state provider with blockhashes and the EIP-2935 system contract. +fn create_database_with_block_hashes(latest_block: u64) -> CacheDB { + let mut db = CacheDB::new(Default::default()); + for block_number in 0..=latest_block { + db.cache.block_hashes.insert(U256::from(block_number), keccak256(block_number.to_string())); + } + + let blockhashes_contract_account = AccountInfo { + balance: U256::ZERO, + code_hash: keccak256(HISTORY_STORAGE_CODE.clone()), + code: Some(Bytecode::new_raw(HISTORY_STORAGE_CODE.clone())), + nonce: 1, + }; + + db.insert_account_info(HISTORY_STORAGE_ADDRESS, blockhashes_contract_account); + + db +} +#[test] +fn eip_2935_pre_fork() { + let db = create_database_with_block_hashes(1); + + let chain_spec = Arc::new( + ChainSpecBuilder::from(&*MAINNET) + .shanghai_activated() + .with_fork(EthereumHardfork::Prague, ForkCondition::Never) + .build(), + ); + + let provider = EthEvmConfig::new(chain_spec); + let mut executor = BasicBlockExecutor::new(provider, db); + + // construct the header for block one + let header = Header { timestamp: 1, number: 1, ..Header::default() }; + + // attempt to execute an empty block, this should not fail + executor + .execute_one(&RecoveredBlock::new_unhashed( + Block { header, body: Default::default() }, + vec![], + )) + .expect("Executing a block with no transactions while Prague is active should not fail"); + + // ensure that the block hash was *not* written to storage, since this is before the fork + // was activated + // + // we load the account first, because revm expects it to be + // loaded + executor.with_state_mut(|state| state.basic(HISTORY_STORAGE_ADDRESS).unwrap()); + assert!(executor.with_state_mut(|state| { + state.storage(HISTORY_STORAGE_ADDRESS, U256::ZERO).unwrap().is_zero() + })); +} + +#[test] +fn eip_2935_fork_activation_genesis() { + let db = create_database_with_block_hashes(0); + + let chain_spec = Arc::new( + ChainSpecBuilder::from(&*MAINNET) + .shanghai_activated() + .cancun_activated() + .prague_activated() + .build(), + ); + + let header = chain_spec.genesis_header().clone(); + let provider = EthEvmConfig::new(chain_spec); + let mut executor = BasicBlockExecutor::new(provider, db); + + // attempt to execute genesis block, this should not fail + executor + .execute_one(&RecoveredBlock::new_unhashed( + Block { header, body: Default::default() }, + vec![], + )) + .expect("Executing a block with no transactions while Prague is active should not fail"); + + // ensure that the block hash was *not* written to storage, since there are no blocks + // preceding genesis + // + // we load the account first, because revm expects it to be + // loaded + executor.with_state_mut(|state| state.basic(HISTORY_STORAGE_ADDRESS).unwrap()); + assert!(executor.with_state_mut(|state| { + state.storage(HISTORY_STORAGE_ADDRESS, U256::ZERO).unwrap().is_zero() + })); +} + +#[test] +fn eip_2935_fork_activation_within_window_bounds() { + let fork_activation_block = (HISTORY_SERVE_WINDOW - 10) as u64; + let db = create_database_with_block_hashes(fork_activation_block); + + let chain_spec = Arc::new( + ChainSpecBuilder::from(&*MAINNET) + .shanghai_activated() + .cancun_activated() + .with_fork(EthereumHardfork::Prague, ForkCondition::Timestamp(1)) + .build(), + ); + + let header = Header { + parent_hash: B256::random(), + timestamp: 1, + number: fork_activation_block, + requests_hash: Some(EMPTY_REQUESTS_HASH), + excess_blob_gas: Some(0), + parent_beacon_block_root: Some(B256::random()), + ..Header::default() + }; + let provider = EthEvmConfig::new(chain_spec); + let mut executor = BasicBlockExecutor::new(provider, db); + + // attempt to execute the fork activation block, this should not fail + executor + .execute_one(&RecoveredBlock::new_unhashed( + Block { header, body: Default::default() }, + vec![], + )) + .expect("Executing a block with no transactions while Prague is active should not fail"); + + // the hash for the ancestor of the fork activation block should be present + assert!( + executor.with_state_mut(|state| state.basic(HISTORY_STORAGE_ADDRESS).unwrap().is_some()) + ); + assert_ne!( + executor.with_state_mut(|state| state + .storage(HISTORY_STORAGE_ADDRESS, U256::from(fork_activation_block - 1)) + .unwrap()), + U256::ZERO + ); + + // the hash of the block itself should not be in storage + assert!(executor.with_state_mut(|state| { + state.storage(HISTORY_STORAGE_ADDRESS, U256::from(fork_activation_block)).unwrap().is_zero() + })); +} + +// +#[test] +fn eip_2935_fork_activation_outside_window_bounds() { + let fork_activation_block = (HISTORY_SERVE_WINDOW + 256) as u64; + let db = create_database_with_block_hashes(fork_activation_block); + + let chain_spec = Arc::new( + ChainSpecBuilder::from(&*MAINNET) + .shanghai_activated() + .cancun_activated() + .with_fork(EthereumHardfork::Prague, ForkCondition::Timestamp(1)) + .build(), + ); + + let provider = EthEvmConfig::new(chain_spec); + let mut executor = BasicBlockExecutor::new(provider, db); + + let header = Header { + parent_hash: B256::random(), + timestamp: 1, + number: fork_activation_block, + requests_hash: Some(EMPTY_REQUESTS_HASH), + excess_blob_gas: Some(0), + parent_beacon_block_root: Some(B256::random()), + ..Header::default() + }; + + // attempt to execute the fork activation block, this should not fail + executor + .execute_one(&RecoveredBlock::new_unhashed( + Block { header, body: Default::default() }, + vec![], + )) + .expect("Executing a block with no transactions while Prague is active should not fail"); + + // the hash for the ancestor of the fork activation block should be present + assert!( + executor.with_state_mut(|state| state.basic(HISTORY_STORAGE_ADDRESS).unwrap().is_some()) + ); +} + +#[test] +fn eip_2935_state_transition_inside_fork() { + let db = create_database_with_block_hashes(2); + + let chain_spec = Arc::new( + ChainSpecBuilder::from(&*MAINNET) + .shanghai_activated() + .cancun_activated() + .prague_activated() + .build(), + ); + + let header = chain_spec.genesis_header().clone(); + let header_hash = header.hash_slow(); + + let provider = EthEvmConfig::new(chain_spec); + let mut executor = BasicBlockExecutor::new(provider, db); + + // attempt to execute the genesis block, this should not fail + executor + .execute_one(&RecoveredBlock::new_unhashed( + Block { header, body: Default::default() }, + vec![], + )) + .expect("Executing a block with no transactions while Prague is active should not fail"); + + // nothing should be written as the genesis has no ancestors + // + // we load the account first, because revm expects it to be + // loaded + executor.with_state_mut(|state| state.basic(HISTORY_STORAGE_ADDRESS).unwrap()); + assert!(executor.with_state_mut(|state| { + state.storage(HISTORY_STORAGE_ADDRESS, U256::ZERO).unwrap().is_zero() + })); + + // attempt to execute block 1, this should not fail + let header = Header { + parent_hash: header_hash, + timestamp: 1, + number: 1, + requests_hash: Some(EMPTY_REQUESTS_HASH), + excess_blob_gas: Some(0), + parent_beacon_block_root: Some(B256::random()), + ..Header::default() + }; + let header_hash = header.hash_slow(); + + executor + .execute_one(&RecoveredBlock::new_unhashed( + Block { header, body: Default::default() }, + vec![], + )) + .expect("Executing a block with no transactions while Prague is active should not fail"); + + // the block hash of genesis should now be in storage, but not block 1 + assert!( + executor.with_state_mut(|state| state.basic(HISTORY_STORAGE_ADDRESS).unwrap().is_some()) + ); + assert_ne!( + executor + .with_state_mut(|state| state.storage(HISTORY_STORAGE_ADDRESS, U256::ZERO).unwrap()), + U256::ZERO + ); + assert!(executor.with_state_mut(|state| { + state.storage(HISTORY_STORAGE_ADDRESS, U256::from(1)).unwrap().is_zero() + })); + + // attempt to execute block 2, this should not fail + let header = Header { + parent_hash: header_hash, + timestamp: 1, + number: 2, + requests_hash: Some(EMPTY_REQUESTS_HASH), + excess_blob_gas: Some(0), + parent_beacon_block_root: Some(B256::random()), + ..Header::default() + }; + + executor + .execute_one(&RecoveredBlock::new_unhashed( + Block { header, body: Default::default() }, + vec![], + )) + .expect("Executing a block with no transactions while Prague is active should not fail"); + + // the block hash of genesis and block 1 should now be in storage, but not block 2 + assert!( + executor.with_state_mut(|state| state.basic(HISTORY_STORAGE_ADDRESS).unwrap().is_some()) + ); + assert_ne!( + executor + .with_state_mut(|state| state.storage(HISTORY_STORAGE_ADDRESS, U256::ZERO).unwrap()), + U256::ZERO + ); + assert_ne!( + executor + .with_state_mut(|state| state.storage(HISTORY_STORAGE_ADDRESS, U256::from(1)).unwrap()), + U256::ZERO + ); + assert!(executor.with_state_mut(|state| { + state.storage(HISTORY_STORAGE_ADDRESS, U256::from(2)).unwrap().is_zero() + })); +} + +#[test] +fn eip_7002() { + let chain_spec = Arc::new( + ChainSpecBuilder::from(&*MAINNET) + .shanghai_activated() + .cancun_activated() + .prague_activated() + .build(), + ); + + let mut db = create_database_with_withdrawal_requests_contract(); + + let sender_key_pair = generators::generate_key(&mut generators::rng()); + let sender_address = public_key_to_address(sender_key_pair.public_key()); + + db.insert_account_info( + sender_address, + AccountInfo { nonce: 1, balance: U256::from(ETH_TO_WEI), ..Default::default() }, + ); + + // https://github.com/lightclient/sys-asm/blob/9282bdb9fd64e024e27f60f507486ffb2183cba2/test/Withdrawal.t.sol.in#L36 + let validator_public_key = fixed_bytes!( + "111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111" + ); + let withdrawal_amount = fixed_bytes!("0203040506070809"); + let input: Bytes = [&validator_public_key[..], &withdrawal_amount[..]].concat().into(); + assert_eq!(input.len(), 56); + + let mut header = chain_spec.genesis_header().clone(); + header.gas_limit = 1_500_000; + // measured + header.gas_used = 135_856; + header.receipts_root = + b256!("0xb31a3e47b902e9211c4d349af4e4c5604ce388471e79ca008907ae4616bb0ed3"); + + let tx = sign_tx_with_key_pair( + sender_key_pair, + Transaction::Legacy(TxLegacy { + chain_id: Some(chain_spec.chain.id()), + nonce: 1, + gas_price: header.base_fee_per_gas.unwrap().into(), + gas_limit: header.gas_used, + to: TxKind::Call(WITHDRAWAL_REQUEST_PREDEPLOY_ADDRESS), + // `MIN_WITHDRAWAL_REQUEST_FEE` + value: U256::from(2), + input, + }), + ); + + let provider = EthEvmConfig::new(chain_spec); + + let mut executor = provider.batch_executor(db); + + let BlockExecutionResult { receipts, requests, .. } = executor + .execute_one( + &Block { header, body: BlockBody { transactions: vec![tx], ..Default::default() } } + .try_into_recovered() + .unwrap(), + ) + .unwrap(); + + let receipt = receipts.first().unwrap(); + assert!(receipt.success); + + // There should be exactly one entry with withdrawal requests + assert_eq!(requests.len(), 1); + assert_eq!(requests[0][0], 1); +} + +#[test] +fn block_gas_limit_error() { + // Create a chain specification with fork conditions set for Prague + let chain_spec = Arc::new( + ChainSpecBuilder::from(&*MAINNET) + .shanghai_activated() + .with_fork(EthereumHardfork::Prague, ForkCondition::Timestamp(0)) + .build(), + ); + + // Create a state provider with the withdrawal requests contract pre-deployed + let mut db = create_database_with_withdrawal_requests_contract(); + + // Generate a new key pair for the sender + let sender_key_pair = generators::generate_key(&mut generators::rng()); + // Get the sender's address from the public key + let sender_address = public_key_to_address(sender_key_pair.public_key()); + + // Insert the sender account into the state with a nonce of 1 and a balance of 1 ETH in Wei + db.insert_account_info( + sender_address, + AccountInfo { nonce: 1, balance: U256::from(ETH_TO_WEI), ..Default::default() }, + ); + + // Define the validator public key and withdrawal amount as fixed bytes + let validator_public_key = fixed_bytes!( + "111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111" + ); + let withdrawal_amount = fixed_bytes!("2222222222222222"); + // Concatenate the validator public key and withdrawal amount into a single byte array + let input: Bytes = [&validator_public_key[..], &withdrawal_amount[..]].concat().into(); + // Ensure the input length is 56 bytes + assert_eq!(input.len(), 56); + + // Create a genesis block header with a specified gas limit and gas used + let mut header = chain_spec.genesis_header().clone(); + header.gas_limit = 1_500_000; + header.gas_used = 134_807; + header.receipts_root = + b256!("0xb31a3e47b902e9211c4d349af4e4c5604ce388471e79ca008907ae4616bb0ed3"); + + // Create a transaction with a gas limit higher than the block gas limit + let tx = sign_tx_with_key_pair( + sender_key_pair, + Transaction::Legacy(TxLegacy { + chain_id: Some(chain_spec.chain.id()), + nonce: 1, + gas_price: header.base_fee_per_gas.unwrap().into(), + gas_limit: 2_500_000, // higher than block gas limit + to: TxKind::Call(WITHDRAWAL_REQUEST_PREDEPLOY_ADDRESS), + value: U256::from(1), + input, + }), + ); + + // Create an executor from the state provider + let evm_config = EthEvmConfig::new(chain_spec); + let mut executor = evm_config.batch_executor(db); + + // Execute the block and capture the result + let exec_result = executor.execute_one( + &Block { header, body: BlockBody { transactions: vec![tx], ..Default::default() } } + .try_into_recovered() + .unwrap(), + ); + + // Check if the execution result is an error and assert the specific error type + match exec_result { + Ok(_) => panic!("Expected block gas limit error"), + Err(err) => assert!(matches!( + *err.as_validation().unwrap(), + BlockValidationError::TransactionGasLimitMoreThanAvailableBlockGas { + transaction_gas_limit: 2_500_000, + block_available_gas: 1_500_000, + } + )), + } +} + +#[test] +fn test_balance_increment_not_duplicated() { + let chain_spec = Arc::new( + ChainSpecBuilder::from(&*MAINNET) + .shanghai_activated() + .cancun_activated() + .prague_activated() + .build(), + ); + + let withdrawal_recipient = address!("0x1000000000000000000000000000000000000000"); + + let mut db = CacheDB::new(EmptyDB::default()); + let initial_balance = 100; + db.insert_account_info( + withdrawal_recipient, + AccountInfo { balance: U256::from(initial_balance), nonce: 1, ..Default::default() }, + ); + + let withdrawal = + Withdrawal { index: 0, validator_index: 0, address: withdrawal_recipient, amount: 1 }; + + let header = Header { + timestamp: 1, + number: 1, + excess_blob_gas: Some(0), + parent_beacon_block_root: Some(B256::random()), + ..Header::default() + }; + + let block = &RecoveredBlock::new_unhashed( + Block { + header, + body: BlockBody { + transactions: vec![], + ommers: vec![], + withdrawals: Some(vec![withdrawal].into()), + }, + }, + vec![], + ); + + let provider = EthEvmConfig::new(chain_spec); + let executor = provider.batch_executor(db); + + let (tx, rx) = mpsc::channel(); + let tx_clone = tx.clone(); + + let _output = executor + .execute_with_state_hook(block, move |_, state: &EvmState| { + if let Some(account) = state.get(&withdrawal_recipient) { + let _ = tx_clone.send(account.info.balance); + } + }) + .expect("Block execution should succeed"); + + drop(tx); + let balance_changes: Vec = rx.try_iter().collect(); + + if let Some(final_balance) = balance_changes.last() { + let expected_final_balance = U256::from(initial_balance) + U256::from(1_000_000_000); // initial + 1 Gwei in Wei + assert_eq!( + *final_balance, expected_final_balance, + "Final balance should match expected value after withdrawal" + ); + } +} diff --git a/crates/ethereum/hardforks/src/display.rs b/crates/ethereum/hardforks/src/display.rs index e40a117d26a..b01c478df80 100644 --- a/crates/ethereum/hardforks/src/display.rs +++ b/crates/ethereum/hardforks/src/display.rs @@ -25,6 +25,8 @@ struct DisplayFork { activated_at: ForkCondition, /// An optional EIP (e.g. `EIP-1559`). eip: Option, + /// Optional metadata to display alongside the fork (e.g. blob parameters) + metadata: Option, } impl core::fmt::Display for DisplayFork { @@ -38,6 +40,9 @@ impl core::fmt::Display for DisplayFork { match self.activated_at { ForkCondition::Block(at) | ForkCondition::Timestamp(at) => { write!(f, "{name_with_eip:32} @{at}")?; + if let Some(metadata) = &self.metadata { + write!(f, " {metadata}")?; + } } ForkCondition::TTD { total_difficulty, .. } => { // All networks that have merged are finalized. @@ -45,6 +50,9 @@ impl core::fmt::Display for DisplayFork { f, "{name_with_eip:32} @{total_difficulty} (network is known to be merged)", )?; + if let Some(metadata) = &self.metadata { + write!(f, " {metadata}")?; + } } ForkCondition::Never => unreachable!(), } @@ -145,14 +153,27 @@ impl DisplayHardforks { pub fn new<'a, I>(hardforks: I) -> Self where I: IntoIterator, + { + // Delegate to with_meta by mapping the iterator to include None for metadata + Self::with_meta(hardforks.into_iter().map(|(fork, condition)| (fork, condition, None))) + } + + /// Creates a new [`DisplayHardforks`] from an iterator of hardforks with optional metadata. + pub fn with_meta<'a, I>(hardforks: I) -> Self + where + I: IntoIterator)>, { let mut pre_merge = Vec::new(); let mut with_merge = Vec::new(); let mut post_merge = Vec::new(); - for (fork, condition) in hardforks { - let mut display_fork = - DisplayFork { name: fork.name().to_string(), activated_at: condition, eip: None }; + for (fork, condition, metadata) in hardforks { + let mut display_fork = DisplayFork { + name: fork.name().to_string(), + activated_at: condition, + eip: None, + metadata, + }; match condition { ForkCondition::Block(_) => { diff --git a/crates/ethereum/hardforks/src/hardforks/mod.rs b/crates/ethereum/hardforks/src/hardforks/mod.rs index 1c67c380d96..dad175e8f66 100644 --- a/crates/ethereum/hardforks/src/hardforks/mod.rs +++ b/crates/ethereum/hardforks/src/hardforks/mod.rs @@ -4,11 +4,7 @@ pub use dev::DEV_HARDFORKS; use crate::{ForkCondition, ForkFilter, ForkId, Hardfork, Head}; #[cfg(feature = "std")] use rustc_hash::FxHashMap; -#[cfg(feature = "std")] -use std::collections::hash_map::Entry; -#[cfg(not(feature = "std"))] -use alloc::collections::btree_map::Entry; use alloc::{boxed::Box, vec::Vec}; /// Generic trait over a set of ordered hardforks @@ -115,26 +111,74 @@ impl ChainHardforks { self.fork(fork).active_at_block(block_number) } - /// Inserts `fork` into list, updating with a new [`ForkCondition`] if it already exists. + /// Inserts a fork with the given [`ForkCondition`], maintaining forks in ascending order + /// based on the `Ord` implementation of [`ForkCondition`]. + /// + /// If the fork already exists (regardless of its current condition type), it will be removed + /// and re-inserted at the appropriate position based on the new condition. + /// + /// # Ordering Behavior + /// + /// Forks are ordered according to [`ForkCondition`]'s `Ord` implementation: + /// - [`ForkCondition::Never`] comes first + /// - [`ForkCondition::Block`] ordered by block number + /// - [`ForkCondition::Timestamp`] ordered by timestamp value + /// - [`ForkCondition::TTD`] ordered by total difficulty + /// + /// # Example + /// + /// ```ignore + /// let mut forks = ChainHardforks::default(); + /// forks.insert(Fork::Frontier, ForkCondition::Block(0)); + /// forks.insert(Fork::Homestead, ForkCondition::Block(1_150_000)); + /// forks.insert(Fork::Cancun, ForkCondition::Timestamp(1710338135)); + /// + /// // Forks are ordered: Frontier (Block 0), Homestead (Block 1150000), Cancun (Timestamp) + /// ``` pub fn insert(&mut self, fork: H, condition: ForkCondition) { - match self.map.entry(fork.name()) { - Entry::Occupied(mut entry) => { - *entry.get_mut() = condition; - if let Some((_, inner)) = - self.forks.iter_mut().find(|(inner, _)| inner.name() == fork.name()) - { - *inner = condition; - } - } - Entry::Vacant(entry) => { - entry.insert(condition); - self.forks.push((Box::new(fork), condition)); - } + // Remove existing fork if it exists + self.remove(&fork); + + // Find the correct position based on ForkCondition's Ord implementation + let pos = self + .forks + .iter() + .position(|(_, existing_condition)| *existing_condition > condition) + .unwrap_or(self.forks.len()); + + self.map.insert(fork.name(), condition); + self.forks.insert(pos, (Box::new(fork), condition)); + } + + /// Extends the list with multiple forks, updating existing entries with new + /// [`ForkCondition`]s if they already exist. + /// + /// Each fork is inserted using [`Self::insert`], maintaining proper ordering based on + /// [`ForkCondition`]'s `Ord` implementation. + /// + /// # Example + /// + /// ```ignore + /// let mut forks = ChainHardforks::default(); + /// forks.extend([ + /// (Fork::Homestead, ForkCondition::Block(1_150_000)), + /// (Fork::Frontier, ForkCondition::Block(0)), + /// (Fork::Cancun, ForkCondition::Timestamp(1710338135)), + /// ]); + /// + /// // Forks will be automatically ordered: Frontier, Homestead, Cancun + /// ``` + pub fn extend( + &mut self, + forks: impl IntoIterator, + ) { + for (fork, condition) in forks { + self.insert(fork, condition); } } /// Removes `fork` from list. - pub fn remove(&mut self, fork: H) { + pub fn remove(&mut self, fork: &H) { self.forks.retain(|(inner_fork, _)| inner_fork.name() != fork.name()); self.map.remove(fork.name()); } @@ -157,3 +201,122 @@ impl From<[(T, ForkCondition); N]> for ChainHardfor ) } } + +#[cfg(test)] +mod tests { + use super::*; + use alloy_hardforks::hardfork; + + hardfork!(AHardfork { A1, A2, A3 }); + hardfork!(BHardfork { B1, B2 }); + + #[test] + fn add_hardforks() { + let mut forks = ChainHardforks::default(); + forks.insert(AHardfork::A1, ForkCondition::Block(1)); + forks.insert(BHardfork::B1, ForkCondition::Block(1)); + assert_eq!(forks.len(), 2); + forks.is_fork_active_at_block(AHardfork::A1, 1); + forks.is_fork_active_at_block(BHardfork::B1, 1); + } + + #[test] + fn insert_maintains_fork_order() { + let mut forks = ChainHardforks::default(); + + // Insert forks in random order + forks.insert(BHardfork::B1, ForkCondition::Timestamp(2000)); + forks.insert(AHardfork::A1, ForkCondition::Block(100)); + forks.insert(AHardfork::A2, ForkCondition::Block(50)); + forks.insert(BHardfork::B2, ForkCondition::Timestamp(1000)); + + assert_eq!(forks.len(), 4); + + let fork_list: Vec<_> = forks.forks_iter().collect(); + + // Verify ordering: Block conditions come before Timestamp conditions + // and within each type, they're ordered by value + assert_eq!(fork_list[0].0.name(), "A2"); + assert_eq!(fork_list[0].1, ForkCondition::Block(50)); + assert_eq!(fork_list[1].0.name(), "A1"); + assert_eq!(fork_list[1].1, ForkCondition::Block(100)); + assert_eq!(fork_list[2].0.name(), "B2"); + assert_eq!(fork_list[2].1, ForkCondition::Timestamp(1000)); + assert_eq!(fork_list[3].0.name(), "B1"); + assert_eq!(fork_list[3].1, ForkCondition::Timestamp(2000)); + } + + #[test] + fn insert_replaces_and_reorders_existing_fork() { + let mut forks = ChainHardforks::default(); + + // Insert initial forks + forks.insert(AHardfork::A1, ForkCondition::Block(100)); + forks.insert(BHardfork::B1, ForkCondition::Block(200)); + forks.insert(AHardfork::A2, ForkCondition::Timestamp(1000)); + + assert_eq!(forks.len(), 3); + + // Update A1 from Block to Timestamp - should move it after B1 + forks.insert(AHardfork::A1, ForkCondition::Timestamp(500)); + assert_eq!(forks.len(), 3); + + let fork_list: Vec<_> = forks.forks_iter().collect(); + + // Verify new ordering + assert_eq!(fork_list[0].0.name(), "B1"); + assert_eq!(fork_list[0].1, ForkCondition::Block(200)); + assert_eq!(fork_list[1].0.name(), "A1"); + assert_eq!(fork_list[1].1, ForkCondition::Timestamp(500)); + assert_eq!(fork_list[2].0.name(), "A2"); + assert_eq!(fork_list[2].1, ForkCondition::Timestamp(1000)); + + // Update A1 timestamp to move it after A2 + forks.insert(AHardfork::A1, ForkCondition::Timestamp(2000)); + assert_eq!(forks.len(), 3); + + let fork_list: Vec<_> = forks.forks_iter().collect(); + + assert_eq!(fork_list[0].0.name(), "B1"); + assert_eq!(fork_list[0].1, ForkCondition::Block(200)); + assert_eq!(fork_list[1].0.name(), "A2"); + assert_eq!(fork_list[1].1, ForkCondition::Timestamp(1000)); + assert_eq!(fork_list[2].0.name(), "A1"); + assert_eq!(fork_list[2].1, ForkCondition::Timestamp(2000)); + } + + #[test] + fn extend_maintains_order() { + let mut forks = ChainHardforks::default(); + + // Use extend to insert multiple forks at once in random order + forks.extend([ + (AHardfork::A1, ForkCondition::Block(100)), + (AHardfork::A2, ForkCondition::Timestamp(1000)), + ]); + forks.extend([(BHardfork::B1, ForkCondition::Timestamp(2000))]); + + assert_eq!(forks.len(), 3); + + let fork_list: Vec<_> = forks.forks_iter().collect(); + + // Verify ordering is maintained + assert_eq!(fork_list[0].0.name(), "A1"); + assert_eq!(fork_list[0].1, ForkCondition::Block(100)); + assert_eq!(fork_list[1].0.name(), "A2"); + assert_eq!(fork_list[1].1, ForkCondition::Timestamp(1000)); + assert_eq!(fork_list[2].0.name(), "B1"); + assert_eq!(fork_list[2].1, ForkCondition::Timestamp(2000)); + + // Extend again with an update to A2 + forks.extend([(AHardfork::A2, ForkCondition::Timestamp(3000))]); + assert_eq!(forks.len(), 3); + + let fork_list: Vec<_> = forks.forks_iter().collect(); + + assert_eq!(fork_list[0].0.name(), "A1"); + assert_eq!(fork_list[1].0.name(), "B1"); + assert_eq!(fork_list[2].0.name(), "A2"); + assert_eq!(fork_list[2].1, ForkCondition::Timestamp(3000)); + } +} diff --git a/crates/ethereum/hardforks/src/lib.rs b/crates/ethereum/hardforks/src/lib.rs index 44c05e24a38..caff30cfcbe 100644 --- a/crates/ethereum/hardforks/src/lib.rs +++ b/crates/ethereum/hardforks/src/lib.rs @@ -12,7 +12,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(feature = "std"), no_std)] extern crate alloc; diff --git a/crates/ethereum/node/Cargo.toml b/crates/ethereum/node/Cargo.toml index 74b5867bca2..575934007f9 100644 --- a/crates/ethereum/node/Cargo.toml +++ b/crates/ethereum/node/Cargo.toml @@ -16,7 +16,8 @@ reth-ethereum-engine-primitives.workspace = true reth-ethereum-payload-builder.workspace = true reth-ethereum-consensus.workspace = true reth-ethereum-primitives.workspace = true -reth-primitives-traits.workspace = true +## ensure secp256k1 recovery with rayon support is activated +reth-primitives-traits = { workspace = true, features = ["secp256k1", "rayon"] } reth-node-builder.workspace = true reth-tracing.workspace = true reth-provider.workspace = true @@ -24,49 +25,49 @@ reth-transaction-pool.workspace = true reth-network.workspace = true reth-evm.workspace = true reth-evm-ethereum.workspace = true -reth-consensus.workspace = true reth-rpc.workspace = true -reth-rpc-builder.workspace = true reth-rpc-api.workspace = true +reth-rpc-eth-api.workspace = true +reth-rpc-builder.workspace = true reth-rpc-server-types.workspace = true reth-node-api.workspace = true reth-chainspec.workspace = true reth-revm = { workspace = true, features = ["std"] } -reth-trie-db.workspace = true reth-rpc-eth-types.workspace = true +reth-engine-local.workspace = true reth-engine-primitives.workspace = true reth-payload-primitives.workspace = true # ethereum alloy-eips.workspace = true +alloy-network.workspace = true alloy-rpc-types-eth.workspace = true alloy-rpc-types-engine.workspace = true + +# async +tokio.workspace = true + # revm with required ethereum features -revm = { workspace = true, features = ["secp256k1", "blst", "c-kzg"] } +# Note: this must be kept to ensure all features are properly enabled/forwarded +revm = { workspace = true, features = ["secp256k1", "blst", "c-kzg", "memory_limit"] } # misc eyre.workspace = true [dev-dependencies] -reth-chainspec.workspace = true reth-db.workspace = true reth-exex.workspace = true reth-node-core.workspace = true -reth-payload-primitives.workspace = true reth-e2e-test-utils.workspace = true -reth-rpc-eth-api.workspace = true reth-tasks.workspace = true alloy-primitives.workspace = true alloy-provider.workspace = true alloy-genesis.workspace = true alloy-signer.workspace = true -alloy-eips.workspace = true alloy-sol-types.workspace = true alloy-contract.workspace = true alloy-rpc-types-beacon = { workspace = true, features = ["ssz"] } -alloy-rpc-types-engine.workspace = true -alloy-rpc-types-eth.workspace = true alloy-consensus.workspace = true futures.workspace = true @@ -76,18 +77,26 @@ rand.workspace = true [features] default = [] -js-tracer = ["reth-node-builder/js-tracer"] +asm-keccak = [ + "alloy-primitives/asm-keccak", + "reth-node-core/asm-keccak", + "revm/asm-keccak", +] +js-tracer = [ + "reth-node-builder/js-tracer", + "reth-rpc/js-tracer", + "reth-rpc-eth-api/js-tracer", + "reth-rpc-eth-types/js-tracer", +] test-utils = [ "reth-node-builder/test-utils", "reth-chainspec/test-utils", - "reth-consensus/test-utils", "reth-network/test-utils", "reth-ethereum-primitives/test-utils", "reth-revm/test-utils", "reth-db/test-utils", "reth-provider/test-utils", "reth-transaction-pool/test-utils", - "reth-trie-db/test-utils", "reth-evm/test-utils", "reth-primitives-traits/test-utils", "reth-evm-ethereum/test-utils", diff --git a/crates/ethereum/node/src/engine.rs b/crates/ethereum/node/src/engine.rs index f6c26dbfb25..441e05d1cc7 100644 --- a/crates/ethereum/node/src/engine.rs +++ b/crates/ethereum/node/src/engine.rs @@ -5,8 +5,8 @@ pub use alloy_rpc_types_engine::{ ExecutionPayloadEnvelopeV2, ExecutionPayloadEnvelopeV3, ExecutionPayloadEnvelopeV4, ExecutionPayloadV1, PayloadAttributes as EthPayloadAttributes, }; -use reth_chainspec::ChainSpec; -use reth_engine_primitives::{EngineValidator, PayloadValidator}; +use reth_chainspec::{EthChainSpec, EthereumHardforks}; +use reth_engine_primitives::{EngineApiValidator, PayloadValidator}; use reth_ethereum_payload_builder::EthereumExecutionPayloadValidator; use reth_ethereum_primitives::Block; use reth_node_api::PayloadTypes; @@ -19,11 +19,11 @@ use std::sync::Arc; /// Validator for the ethereum engine API. #[derive(Debug, Clone)] -pub struct EthereumEngineValidator { +pub struct EthereumEngineValidator { inner: EthereumExecutionPayloadValidator, } -impl EthereumEngineValidator { +impl EthereumEngineValidator { /// Instantiates a new validator. pub const fn new(chain_spec: Arc) -> Self { Self { inner: EthereumExecutionPayloadValidator::new(chain_spec) } @@ -36,9 +36,12 @@ impl EthereumEngineValidator { } } -impl PayloadValidator for EthereumEngineValidator { +impl PayloadValidator for EthereumEngineValidator +where + ChainSpec: EthChainSpec + EthereumHardforks + 'static, + Types: PayloadTypes, +{ type Block = Block; - type ExecutionData = ExecutionData; fn ensure_well_formed_payload( &self, @@ -49,14 +52,15 @@ impl PayloadValidator for EthereumEngineValidator { } } -impl EngineValidator for EthereumEngineValidator +impl EngineApiValidator for EthereumEngineValidator where + ChainSpec: EthChainSpec + EthereumHardforks + 'static, Types: PayloadTypes, { fn validate_version_specific_fields( &self, version: EngineApiMessageVersion, - payload_or_attrs: PayloadOrAttributes<'_, Self::ExecutionData, EthPayloadAttributes>, + payload_or_attrs: PayloadOrAttributes<'_, Types::ExecutionData, EthPayloadAttributes>, ) -> Result<(), EngineObjectValidationError> { payload_or_attrs .execution_requests() @@ -74,7 +78,7 @@ where validate_version_specific_fields( self.chain_spec(), version, - PayloadOrAttributes::::PayloadAttributes( + PayloadOrAttributes::::PayloadAttributes( attributes, ), ) diff --git a/crates/ethereum/node/src/evm.rs b/crates/ethereum/node/src/evm.rs index 99fd6dfd691..4e8fd99f82b 100644 --- a/crates/ethereum/node/src/evm.rs +++ b/crates/ethereum/node/src/evm.rs @@ -1,6 +1,7 @@ //! Ethereum EVM support #[doc(inline)] +#[allow(deprecated)] pub use reth_evm_ethereum::execute::EthExecutorProvider; #[doc(inline)] pub use reth_evm_ethereum::{EthEvm, EthEvmConfig}; diff --git a/crates/ethereum/node/src/lib.rs b/crates/ethereum/node/src/lib.rs index 0f6d8ed312d..60d2c8ee2a7 100644 --- a/crates/ethereum/node/src/lib.rs +++ b/crates/ethereum/node/src/lib.rs @@ -9,7 +9,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] use reth_revm as _; use revm as _; @@ -17,11 +17,14 @@ use revm as _; pub use reth_ethereum_engine_primitives::EthEngineTypes; pub mod evm; -pub use evm::{EthEvmConfig, EthExecutorProvider}; +pub use evm::EthEvmConfig; + +#[allow(deprecated)] +pub use evm::EthExecutorProvider; pub use reth_ethereum_consensus as consensus; pub mod node; -pub use node::{EthereumEthApiBuilder, EthereumNode}; +pub use node::*; pub mod payload; diff --git a/crates/ethereum/node/src/node.rs b/crates/ethereum/node/src/node.rs index 037fe473516..fa81d70e61f 100644 --- a/crates/ethereum/node/src/node.rs +++ b/crates/ethereum/node/src/node.rs @@ -3,43 +3,61 @@ pub use crate::{payload::EthereumPayloadBuilder, EthereumEngineValidator}; use crate::{EthEngineTypes, EthEvmConfig}; use alloy_eips::{eip7840::BlobParams, merge::EPOCH_SLOTS}; -use reth_chainspec::{ChainSpec, EthChainSpec, EthereumHardforks}; -use reth_consensus::{ConsensusError, FullConsensus}; +use alloy_network::Ethereum; +use alloy_rpc_types_engine::ExecutionData; +use reth_chainspec::{ChainSpec, EthChainSpec, EthereumHardforks, Hardforks}; +use reth_engine_local::LocalPayloadAttributesBuilder; +use reth_engine_primitives::EngineTypes; use reth_ethereum_consensus::EthBeaconConsensus; use reth_ethereum_engine_primitives::{ EthBuiltPayload, EthPayloadAttributes, EthPayloadBuilderAttributes, }; -use reth_ethereum_primitives::{EthPrimitives, PooledTransaction, TransactionSigned}; -use reth_evm::{ConfigureEvm, EvmFactory, EvmFactoryFor, NextBlockEnvAttributes}; -use reth_network::{EthNetworkPrimitives, NetworkHandle, PeersInfo}; -use reth_node_api::{AddOnsContext, FullNodeComponents, NodeAddOns, NodePrimitives, TxTy}; +use reth_ethereum_primitives::{EthPrimitives, TransactionSigned}; +use reth_evm::{ + eth::spec::EthExecutorSpec, ConfigureEvm, EvmFactory, EvmFactoryFor, NextBlockEnvAttributes, +}; +use reth_network::{primitives::BasicNetworkPrimitives, NetworkHandle, PeersInfo}; +use reth_node_api::{ + AddOnsContext, FullNodeComponents, HeaderTy, NodeAddOns, NodePrimitives, + PayloadAttributesBuilder, PrimitivesTy, TxTy, +}; use reth_node_builder::{ components::{ BasicPayloadServiceBuilder, ComponentsBuilder, ConsensusBuilder, ExecutorBuilder, - NetworkBuilder, PoolBuilder, + NetworkBuilder, PoolBuilder, TxPoolBuilder, }, node::{FullNodeTypes, NodeTypes}, rpc::{ - EngineValidatorAddOn, EngineValidatorBuilder, EthApiBuilder, EthApiCtx, RethRpcAddOns, - RpcAddOns, RpcHandle, + BasicEngineApiBuilder, BasicEngineValidatorBuilder, EngineApiBuilder, EngineValidatorAddOn, + EngineValidatorBuilder, EthApiBuilder, EthApiCtx, Identity, PayloadValidatorBuilder, + RethRpcAddOns, RpcAddOns, RpcHandle, + }, + BuilderContext, DebugNode, Node, NodeAdapter, PayloadBuilderConfig, +}; +use reth_payload_primitives::PayloadTypes; +use reth_provider::{providers::ProviderFactoryBuilder, EthStorage}; +use reth_rpc::{ + eth::core::{EthApiFor, EthRpcConverterFor}, + ValidationApi, +}; +use reth_rpc_api::servers::BlockSubmissionValidationApiServer; +use reth_rpc_builder::{config::RethRpcServerConfig, middleware::RethRpcMiddleware}; +use reth_rpc_eth_api::{ + helpers::{ + config::{EthConfigApiServer, EthConfigHandler}, + pending_block::BuildPendingEnv, }, - BuilderContext, DebugNode, Node, NodeAdapter, NodeComponentsBuilder, PayloadBuilderConfig, - PayloadTypes, + RpcConvert, RpcTypes, SignableTxRequest, }; -use reth_provider::{providers::ProviderFactoryBuilder, CanonStateSubscriptions, EthStorage}; -use reth_rpc::{eth::core::EthApiFor, ValidationApi}; -use reth_rpc_api::{eth::FullEthApiServer, servers::BlockSubmissionValidationApiServer}; -use reth_rpc_builder::config::RethRpcServerConfig; use reth_rpc_eth_types::{error::FromEvmError, EthApiError}; use reth_rpc_server_types::RethRpcModule; use reth_tracing::tracing::{debug, info}; use reth_transaction_pool::{ - blobstore::{DiskFileBlobStore, DiskFileBlobStoreConfig}, - EthTransactionPool, PoolTransaction, TransactionPool, TransactionValidationTaskExecutor, + blobstore::DiskFileBlobStore, EthTransactionPool, PoolPooledTx, PoolTransaction, + TransactionPool, TransactionValidationTaskExecutor, }; -use reth_trie_db::MerklePatriciaTrie; use revm::context::TxEnv; -use std::{default::Default, sync::Arc, time::SystemTime}; +use std::{marker::PhantomData, sync::Arc, time::SystemTime}; /// Type configuration for a regular Ethereum node. #[derive(Debug, Default, Clone, Copy)] @@ -57,7 +75,12 @@ impl EthereumNode { EthereumConsensusBuilder, > where - Node: FullNodeTypes>, + Node: FullNodeTypes< + Types: NodeTypes< + ChainSpec: Hardforks + EthereumHardforks + EthExecutorSpec, + Primitives = EthPrimitives, + >, + >, ::Payload: PayloadTypes< BuiltPayload = EthBuiltPayload, PayloadAttributes = EthPayloadAttributes, @@ -112,80 +135,162 @@ impl EthereumNode { impl NodeTypes for EthereumNode { type Primitives = EthPrimitives; type ChainSpec = ChainSpec; - type StateCommitment = MerklePatriciaTrie; type Storage = EthStorage; type Payload = EthEngineTypes; } /// Builds [`EthApi`](reth_rpc::EthApi) for Ethereum. -#[derive(Debug, Default)] -pub struct EthereumEthApiBuilder; +#[derive(Debug)] +pub struct EthereumEthApiBuilder(PhantomData); -impl EthApiBuilder for EthereumEthApiBuilder +impl Default for EthereumEthApiBuilder { + fn default() -> Self { + Self(Default::default()) + } +} + +impl EthApiBuilder for EthereumEthApiBuilder where - N: FullNodeComponents, - EthApiFor: FullEthApiServer, + N: FullNodeComponents< + Types: NodeTypes, + Evm: ConfigureEvm>>, + >, + NetworkT: RpcTypes>>, + EthRpcConverterFor: RpcConvert< + Primitives = PrimitivesTy, + Error = EthApiError, + Network = NetworkT, + Evm = N::Evm, + >, + EthApiError: FromEvmError, { - type EthApi = EthApiFor; + type EthApi = EthApiFor; async fn build_eth_api(self, ctx: EthApiCtx<'_, N>) -> eyre::Result { - let api = reth_rpc::EthApiBuilder::new( - ctx.components.provider().clone(), - ctx.components.pool().clone(), - ctx.components.network().clone(), - ctx.components.evm_config().clone(), - ) - .eth_cache(ctx.cache) - .task_spawner(ctx.components.task_executor().clone()) - .gas_cap(ctx.config.rpc_gas_cap.into()) - .max_simulate_blocks(ctx.config.rpc_max_simulate_blocks) - .eth_proof_window(ctx.config.eth_proof_window) - .fee_history_cache_config(ctx.config.fee_history_cache) - .proof_permits(ctx.config.proof_permits) - .gas_oracle_config(ctx.config.gas_oracle) - .build(); - Ok(api) + Ok(ctx.eth_api_builder().map_converter(|r| r.with_network()).build()) } } /// Add-ons w.r.t. l1 ethereum. #[derive(Debug)] -pub struct EthereumAddOns +pub struct EthereumAddOns< + N: FullNodeComponents, + EthB: EthApiBuilder, + PVB, + EB = BasicEngineApiBuilder, + EVB = BasicEngineValidatorBuilder, + RpcMiddleware = Identity, +> { + inner: RpcAddOns, +} + +impl EthereumAddOns where - EthApiFor: FullEthApiServer, + N: FullNodeComponents, + EthB: EthApiBuilder, { - inner: RpcAddOns, + /// Creates a new instance from the inner `RpcAddOns`. + pub const fn new(inner: RpcAddOns) -> Self { + Self { inner } + } } -impl Default for EthereumAddOns +impl Default for EthereumAddOns where - EthApiFor: FullEthApiServer, + N: FullNodeComponents< + Types: NodeTypes< + ChainSpec: EthereumHardforks + Clone + 'static, + Payload: EngineTypes + + PayloadTypes, + Primitives = EthPrimitives, + >, + >, + EthereumEthApiBuilder: EthApiBuilder, { fn default() -> Self { - Self { inner: Default::default() } + Self::new(RpcAddOns::new( + EthereumEthApiBuilder::default(), + EthereumEngineValidatorBuilder::default(), + BasicEngineApiBuilder::default(), + BasicEngineValidatorBuilder::default(), + Default::default(), + )) } } -impl NodeAddOns for EthereumAddOns +impl EthereumAddOns +where + N: FullNodeComponents, + EthB: EthApiBuilder, +{ + /// Replace the engine API builder. + pub fn with_engine_api( + self, + engine_api_builder: T, + ) -> EthereumAddOns + where + T: Send, + { + let Self { inner } = self; + EthereumAddOns::new(inner.with_engine_api(engine_api_builder)) + } + + /// Replace the payload validator builder. + pub fn with_payload_validator( + self, + payload_validator_builder: T, + ) -> EthereumAddOns { + let Self { inner } = self; + EthereumAddOns::new(inner.with_payload_validator(payload_validator_builder)) + } + + /// Sets rpc middleware + pub fn with_rpc_middleware( + self, + rpc_middleware: T, + ) -> EthereumAddOns + where + T: Send, + { + let Self { inner } = self; + EthereumAddOns::new(inner.with_rpc_middleware(rpc_middleware)) + } + + /// Sets the tokio runtime for the RPC servers. + /// + /// Caution: This runtime must not be created from within asynchronous context. + pub fn with_tokio_runtime(self, tokio_runtime: Option) -> Self { + let Self { inner } = self; + Self { inner: inner.with_tokio_runtime(tokio_runtime) } + } +} + +impl NodeAddOns + for EthereumAddOns where N: FullNodeComponents< Types: NodeTypes< - ChainSpec = ChainSpec, + ChainSpec: Hardforks + EthereumHardforks, Primitives = EthPrimitives, - Payload = EthEngineTypes, + Payload: EngineTypes, >, Evm: ConfigureEvm, >, + EthB: EthApiBuilder, + PVB: Send, + EB: EngineApiBuilder, + EVB: EngineValidatorBuilder, EthApiError: FromEvmError, EvmFactoryFor: EvmFactory, + RpcMiddleware: RethRpcMiddleware, { - type Handle = RpcHandle>; + type Handle = RpcHandle; async fn launch_add_ons( self, ctx: reth_node_api::AddOnsContext<'_, N>, ) -> eyre::Result { - let validation_api = ValidationApi::new( + let validation_api = ValidationApi::<_, _, ::Payload>::new( ctx.node.provider().clone(), Arc::new(ctx.node.consensus().clone()), ctx.node.evm_config().clone(), @@ -194,54 +299,73 @@ where Arc::new(EthereumEngineValidator::new(ctx.config.chain.clone())), ); + let eth_config = + EthConfigHandler::new(ctx.node.provider().clone(), ctx.node.evm_config().clone()); + self.inner - .launch_add_ons_with(ctx, move |modules, _, _| { - modules.merge_if_module_configured( + .launch_add_ons_with(ctx, move |container| { + container.modules.merge_if_module_configured( RethRpcModule::Flashbots, validation_api.into_rpc(), )?; + container + .modules + .merge_if_module_configured(RethRpcModule::Eth, eth_config.into_rpc())?; + Ok(()) }) .await } } -impl RethRpcAddOns for EthereumAddOns +impl RethRpcAddOns for EthereumAddOns where N: FullNodeComponents< Types: NodeTypes< - ChainSpec = ChainSpec, + ChainSpec: Hardforks + EthereumHardforks, Primitives = EthPrimitives, - Payload = EthEngineTypes, + Payload: EngineTypes, >, Evm: ConfigureEvm, >, + EthB: EthApiBuilder, + PVB: PayloadValidatorBuilder, + EB: EngineApiBuilder, + EVB: EngineValidatorBuilder, EthApiError: FromEvmError, EvmFactoryFor: EvmFactory, { - type EthApi = EthApiFor; + type EthApi = EthB::EthApi; fn hooks_mut(&mut self) -> &mut reth_node_builder::rpc::RpcHooks { self.inner.hooks_mut() } } -impl EngineValidatorAddOn for EthereumAddOns +impl EngineValidatorAddOn + for EthereumAddOns where N: FullNodeComponents< Types: NodeTypes< - ChainSpec = ChainSpec, + ChainSpec: EthChainSpec + EthereumHardforks, Primitives = EthPrimitives, - Payload = EthEngineTypes, + Payload: EngineTypes, >, + Evm: ConfigureEvm, >, - EthApiFor: FullEthApiServer, + EthB: EthApiBuilder, + PVB: Send, + EB: EngineApiBuilder, + EVB: EngineValidatorBuilder, + EthApiError: FromEvmError, + EvmFactoryFor: EvmFactory, + RpcMiddleware: Send, { - type Validator = EthereumEngineValidator; + type ValidatorBuilder = EVB; - async fn engine_validator(&self, ctx: &AddOnsContext<'_, N>) -> eyre::Result { - EthereumEngineValidatorBuilder::default().build(ctx).await + fn engine_validator_builder(&self) -> Self::ValidatorBuilder { + self.inner.engine_validator_builder() } } @@ -258,9 +382,8 @@ where EthereumConsensusBuilder, >; - type AddOns = EthereumAddOns< - NodeAdapter>::Components>, - >; + type AddOns = + EthereumAddOns, EthereumEthApiBuilder, EthereumEngineValidatorBuilder>; fn components_builder(&self) -> Self::ComponentsBuilder { Self::components() @@ -275,18 +398,13 @@ impl> DebugNode for EthereumNode { type RpcBlock = alloy_rpc_types_eth::Block; fn rpc_to_primitive_block(rpc_block: Self::RpcBlock) -> reth_ethereum_primitives::Block { - let alloy_rpc_types_eth::Block { header, transactions, withdrawals, .. } = rpc_block; - reth_ethereum_primitives::Block { - header: header.inner, - body: reth_ethereum_primitives::BlockBody { - transactions: transactions - .into_transactions() - .map(|tx| tx.inner.into_inner().into()) - .collect(), - ommers: Default::default(), - withdrawals, - }, - } + rpc_block.into_consensus().convert_transactions() + } + + fn local_payload_attributes_builder( + chain_spec: &Self::ChainSpec, + ) -> impl PayloadAttributesBuilder<::PayloadAttributes> { + LocalPayloadAttributesBuilder::new(Arc::new(chain_spec.clone())) } } @@ -297,10 +415,13 @@ pub struct EthereumExecutorBuilder; impl ExecutorBuilder for EthereumExecutorBuilder where - Types: NodeTypes, + Types: NodeTypes< + ChainSpec: Hardforks + EthExecutorSpec + EthereumHardforks, + Primitives = EthPrimitives, + >, Node: FullNodeTypes, { - type EVM = EthEvmConfig; + type EVM = EthEvmConfig; async fn build_evm(self, ctx: &BuilderContext) -> eyre::Result { let evm_config = EthEvmConfig::new(ctx.chain_spec()) @@ -330,11 +451,10 @@ where type Pool = EthTransactionPool; async fn build_pool(self, ctx: &BuilderContext) -> eyre::Result { - let data_dir = ctx.config().datadir(); let pool_config = ctx.pool_config(); let blob_cache_size = if let Some(blob_cache_size) = pool_config.blob_cache_size { - blob_cache_size + Some(blob_cache_size) } else { // get the current blob params for the current timestamp, fallback to default Cancun // params @@ -347,76 +467,41 @@ where // Derive the blob cache size from the target blob count, to auto scale it by // multiplying it with the slot count for 2 epochs: 384 for pectra - (blob_params.target_blob_count * EPOCH_SLOTS * 2) as u32 + Some((blob_params.target_blob_count * EPOCH_SLOTS * 2) as u32) }; - let custom_config = - DiskFileBlobStoreConfig::default().with_max_cached_entries(blob_cache_size); + let blob_store = + reth_node_builder::components::create_blob_store_with_cache(ctx, blob_cache_size)?; - let blob_store = DiskFileBlobStore::open(data_dir.blobstore(), custom_config)?; let validator = TransactionValidationTaskExecutor::eth_builder(ctx.provider().clone()) .with_head_timestamp(ctx.head().timestamp) + .with_max_tx_input_bytes(ctx.config().txpool.max_tx_input_bytes) .kzg_settings(ctx.kzg_settings()?) .with_local_transactions_config(pool_config.local_transactions_config.clone()) .set_tx_fee_cap(ctx.config().rpc.rpc_tx_fee_cap) + .with_max_tx_gas_limit(ctx.config().txpool.max_tx_gas_limit) + .with_minimum_priority_fee(ctx.config().txpool.minimum_priority_fee) .with_additional_tasks(ctx.config().txpool.additional_validation_tasks) .build_with_tasks(ctx.task_executor().clone(), blob_store.clone()); - let transaction_pool = - reth_transaction_pool::Pool::eth_pool(validator, blob_store, pool_config); - info!(target: "reth::cli", "Transaction pool initialized"); - - // spawn txpool maintenance task - { - let pool = transaction_pool.clone(); - let chain_events = ctx.provider().canonical_state_stream(); - let client = ctx.provider().clone(); - // Only spawn backup task if not disabled - if !ctx.config().txpool.disable_transactions_backup { - // Use configured backup path or default to data dir - let transactions_path = ctx - .config() - .txpool - .transactions_backup_path - .clone() - .unwrap_or_else(|| data_dir.txpool_transactions()); - - let transactions_backup_config = - reth_transaction_pool::maintain::LocalTransactionBackupConfig::with_local_txs_backup(transactions_path); - - ctx.task_executor().spawn_critical_with_graceful_shutdown_signal( - "local transactions backup task", - |shutdown| { - reth_transaction_pool::maintain::backup_local_transactions_task( - shutdown, - pool.clone(), - transactions_backup_config, - ) - }, - ); - } - - // spawn the maintenance task - ctx.task_executor().spawn_critical( - "txpool maintenance task", - reth_transaction_pool::maintain::maintain_transaction_pool_future( - client, - pool, - chain_events, - ctx.task_executor().clone(), - reth_transaction_pool::maintain::MaintainPoolConfig { - max_tx_lifetime: transaction_pool.config().max_queued_lifetime, - no_local_exemptions: transaction_pool - .config() - .local_transactions_config - .no_exemptions, - ..Default::default() - }, - ), - ); - debug!(target: "reth::cli", "Spawned txpool maintenance task"); + if validator.validator().eip4844() { + // initializing the KZG settings can be expensive, this should be done upfront so that + // it doesn't impact the first block or the first gossiped blob transaction, so we + // initialize this in the background + let kzg_settings = validator.validator().kzg_settings().clone(); + ctx.task_executor().spawn_blocking(async move { + let _ = kzg_settings.get(); + debug!(target: "reth::cli", "Initialized KZG settings"); + }); } + let transaction_pool = TxPoolBuilder::new(ctx) + .with_validator(validator) + .build_and_spawn_maintenance_task(blob_store, pool_config)?; + + info!(target: "reth::cli", "Transaction pool initialized"); + debug!(target: "reth::cli", "Spawned txpool maintenance task"); + Ok(transaction_pool) } } @@ -429,13 +514,13 @@ pub struct EthereumNetworkBuilder { impl NetworkBuilder for EthereumNetworkBuilder where - Node: FullNodeTypes>, - Pool: TransactionPool< - Transaction: PoolTransaction, Pooled = PooledTransaction>, - > + Unpin + Node: FullNodeTypes>, + Pool: TransactionPool>> + + Unpin + 'static, { - type Network = NetworkHandle; + type Network = + NetworkHandle, PoolPooledTx>>; async fn build_network( self, @@ -457,9 +542,11 @@ pub struct EthereumConsensusBuilder { impl ConsensusBuilder for EthereumConsensusBuilder where - Node: FullNodeTypes>, + Node: FullNodeTypes< + Types: NodeTypes, + >, { - type Consensus = Arc>; + type Consensus = Arc::ChainSpec>>; async fn build_consensus(self, ctx: &BuilderContext) -> eyre::Result { Ok(Arc::new(EthBeaconConsensus::new(ctx.chain_spec()))) @@ -471,12 +558,17 @@ where #[non_exhaustive] pub struct EthereumEngineValidatorBuilder; -impl EngineValidatorBuilder for EthereumEngineValidatorBuilder +impl PayloadValidatorBuilder for EthereumEngineValidatorBuilder where - Types: NodeTypes, + Types: NodeTypes< + ChainSpec: Hardforks + EthereumHardforks + Clone + 'static, + Payload: EngineTypes + + PayloadTypes, + Primitives = EthPrimitives, + >, Node: FullNodeComponents, { - type Validator = EthereumEngineValidator; + type Validator = EthereumEngineValidator; async fn build(self, ctx: &AddOnsContext<'_, Node>) -> eyre::Result { Ok(EthereumEngineValidator::new(ctx.config.chain.clone())) diff --git a/crates/ethereum/node/src/payload.rs b/crates/ethereum/node/src/payload.rs index 3aebc39e76c..405bae94d2a 100644 --- a/crates/ethereum/node/src/payload.rs +++ b/crates/ethereum/node/src/payload.rs @@ -1,6 +1,6 @@ //! Payload component configuration for the Ethereum node. -use reth_chainspec::EthereumHardforks; +use reth_chainspec::{EthChainSpec, EthereumHardforks}; use reth_ethereum_engine_primitives::{ EthBuiltPayload, EthPayloadAttributes, EthPayloadBuilderAttributes, }; @@ -45,11 +45,14 @@ where evm_config: Evm, ) -> eyre::Result { let conf = ctx.payload_builder_config(); + let chain = ctx.chain_spec().chain(); + let gas_limit = conf.gas_limit_for(chain); + Ok(reth_ethereum_payload_builder::EthereumPayloadBuilder::new( ctx.provider().clone(), pool, evm_config, - EthereumBuilderConfig::new().with_gas_limit(conf.gas_limit()), + EthereumBuilderConfig::new().with_gas_limit(gas_limit), )) } } diff --git a/crates/ethereum/node/tests/assets/genesis.json b/crates/ethereum/node/tests/assets/genesis.json index 671bf85e423..76abaaaea5d 100644 --- a/crates/ethereum/node/tests/assets/genesis.json +++ b/crates/ethereum/node/tests/assets/genesis.json @@ -1 +1 @@ -{"config":{"chainId":1,"homesteadBlock":0,"daoForkSupport":true,"eip150Block":0,"eip155Block":0,"eip158Block":0,"byzantiumBlock":0,"constantinopleBlock":0,"petersburgBlock":0,"istanbulBlock":0,"muirGlacierBlock":0,"berlinBlock":0,"londonBlock":0,"arrowGlacierBlock":0,"grayGlacierBlock":0,"shanghaiTime":0,"cancunTime":0,"terminalTotalDifficulty":"0x0","terminalTotalDifficultyPassed":true},"nonce":"0x0","timestamp":"0x0","extraData":"0x00","gasLimit":"0x1c9c380","difficulty":"0x0","mixHash":"0x0000000000000000000000000000000000000000000000000000000000000000","coinbase":"0x0000000000000000000000000000000000000000","alloc":{"0x14dc79964da2c08b23698b3d3cc7ca32193d9955":{"balance":"0xd3c21bcecceda1000000"},"0x15d34aaf54267db7d7c367839aaf71a00a2c6a65":{"balance":"0xd3c21bcecceda1000000"},"0x1cbd3b2770909d4e10f157cabc84c7264073c9ec":{"balance":"0xd3c21bcecceda1000000"},"0x23618e81e3f5cdf7f54c3d65f7fbc0abf5b21e8f":{"balance":"0xd3c21bcecceda1000000"},"0x2546bcd3c84621e976d8185a91a922ae77ecec30":{"balance":"0xd3c21bcecceda1000000"},"0x3c44cdddb6a900fa2b585dd299e03d12fa4293bc":{"balance":"0xd3c21bcecceda1000000"},"0x70997970c51812dc3a010c7d01b50e0d17dc79c8":{"balance":"0xd3c21bcecceda1000000"},"0x71be63f3384f5fb98995898a86b02fb2426c5788":{"balance":"0xd3c21bcecceda1000000"},"0x8626f6940e2eb28930efb4cef49b2d1f2c9c1199":{"balance":"0xd3c21bcecceda1000000"},"0x90f79bf6eb2c4f870365e785982e1f101e93b906":{"balance":"0xd3c21bcecceda1000000"},"0x976ea74026e726554db657fa54763abd0c3a0aa9":{"balance":"0xd3c21bcecceda1000000"},"0x9965507d1a55bcc2695c58ba16fb37d819b0a4dc":{"balance":"0xd3c21bcecceda1000000"},"0x9c41de96b2088cdc640c6182dfcf5491dc574a57":{"balance":"0xd3c21bcecceda1000000"},"0xa0ee7a142d267c1f36714e4a8f75612f20a79720":{"balance":"0xd3c21bcecceda1000000"},"0xbcd4042de499d14e55001ccbb24a551f3b954096":{"balance":"0xd3c21bcecceda1000000"},"0xbda5747bfd65f08deb54cb465eb87d40e51b197e":{"balance":"0xd3c21bcecceda1000000"},"0xcd3b766ccdd6ae721141f452c550ca635964ce71":{"balance":"0xd3c21bcecceda1000000"},"0xdd2fd4581271e230360230f9337d5c0430bf44c0":{"balance":"0xd3c21bcecceda1000000"},"0xdf3e18d64bc6a983f673ab319ccae4f1a57c7097":{"balance":"0xd3c21bcecceda1000000"},"0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266":{"balance":"0xd3c21bcecceda1000000"},"0xfabb0ac9d68b0b445fb7357272ff202c5651694a":{"balance":"0xd3c21bcecceda1000000"}},"number":"0x0"} \ No newline at end of file +{"config":{"chainId":1,"homesteadBlock":0,"daoForkSupport":true,"eip150Block":0,"eip155Block":0,"eip158Block":0,"byzantiumBlock":0,"constantinopleBlock":0,"petersburgBlock":0,"istanbulBlock":0,"muirGlacierBlock":0,"berlinBlock":0,"londonBlock":0,"arrowGlacierBlock":0,"grayGlacierBlock":0,"shanghaiTime":0,"cancunTime":0,"terminalTotalDifficulty":"0x0","terminalTotalDifficultyPassed":true},"nonce":"0x0","timestamp":"0x0","extraData":"0x00","gasLimit":"0x1c9c380","difficulty":"0x0","mixHash":"0x0000000000000000000000000000000000000000000000000000000000000000","coinbase":"0x0000000000000000000000000000000000000000","alloc":{"0x14dc79964da2c08b23698b3d3cc7ca32193d9955":{"balance":"0xd3c21bcecceda1000000"},"0x15d34aaf54267db7d7c367839aaf71a00a2c6a65":{"balance":"0xd3c21bcecceda1000000"},"0x1cbd3b2770909d4e10f157cabc84c7264073c9ec":{"balance":"0xd3c21bcecceda1000000"},"0x23618e81e3f5cdf7f54c3d65f7fbc0abf5b21e8f":{"balance":"0xd3c21bcecceda1000000"},"0x2546bcd3c84621e976d8185a91a922ae77ecec30":{"balance":"0xd3c21bcecceda1000000"},"0x3c44cdddb6a900fa2b585dd299e03d12fa4293bc":{"balance":"0xd3c21bcecceda1000000"},"0x70997970c51812dc3a010c7d01b50e0d17dc79c8":{"balance":"0xd3c21bcecceda1000000"},"0x71be63f3384f5fb98995898a86b02fb2426c5788":{"balance":"0xd3c21bcecceda1000000"},"0x8626f6940e2eb28930efb4cef49b2d1f2c9c1199":{"balance":"0xd3c21bcecceda1000000"},"0x90f79bf6eb2c4f870365e785982e1f101e93b906":{"balance":"0xd3c21bcecceda1000000"},"0x976ea74026e726554db657fa54763abd0c3a0aa9":{"balance":"0xd3c21bcecceda1000000"},"0x9965507d1a55bcc2695c58ba16fb37d819b0a4dc":{"balance":"0xd3c21bcecceda1000000"},"0x9c41de96b2088cdc640c6182dfcf5491dc574a57":{"balance":"0xd3c21bcecceda1000000"},"0xa0ee7a142d267c1f36714e4a8f75612f20a79720":{"balance":"0xd3c21bcecceda1000000"},"0xbcd4042de499d14e55001ccbb24a551f3b954096":{"balance":"0xd3c21bcecceda1000000"},"0xbda5747bfd65f08deb54cb465eb87d40e51b197e":{"balance":"0xd3c21bcecceda1000000"},"0xcd3b766ccdd6ae721141f452c550ca635964ce71":{"balance":"0xd3c21bcecceda1000000"},"0xdd2fd4581271e230360230f9337d5c0430bf44c0":{"balance":"0xd3c21bcecceda1000000"},"0xdf3e18d64bc6a983f673ab319ccae4f1a57c7097":{"balance":"0xd3c21bcecceda1000000"},"0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266":{"balance":"0xd3c21bcecceda1000000"},"0xfabb0ac9d68b0b445fb7357272ff202c5651694a":{"balance":"0xd3c21bcecceda1000000"}},"number":"0x0"} diff --git a/crates/ethereum/node/tests/e2e/blobs.rs b/crates/ethereum/node/tests/e2e/blobs.rs index 8fd9d08d2dc..1c088e33da6 100644 --- a/crates/ethereum/node/tests/e2e/blobs.rs +++ b/crates/ethereum/node/tests/e2e/blobs.rs @@ -1,15 +1,21 @@ use crate::utils::eth_payload_attributes; +use alloy_eips::Decodable2718; use alloy_genesis::Genesis; use reth_chainspec::{ChainSpecBuilder, MAINNET}; use reth_e2e_test_utils::{ node::NodeTestContext, transaction::TransactionTestContext, wallet::Wallet, }; +use reth_ethereum_engine_primitives::BlobSidecars; +use reth_ethereum_primitives::PooledTransactionVariant; use reth_node_builder::{NodeBuilder, NodeHandle}; use reth_node_core::{args::RpcServerArgs, node_config::NodeConfig}; use reth_node_ethereum::EthereumNode; use reth_tasks::TaskManager; use reth_transaction_pool::TransactionPool; -use std::sync::Arc; +use std::{ + sync::Arc, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; #[tokio::test] async fn can_handle_blobs() -> eyre::Result<()> { @@ -82,3 +88,165 @@ async fn can_handle_blobs() -> eyre::Result<()> { Ok(()) } + +#[tokio::test] +async fn can_send_legacy_sidecar_post_activation() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + let tasks = TaskManager::current(); + let exec = tasks.executor(); + + let genesis: Genesis = serde_json::from_str(include_str!("../assets/genesis.json")).unwrap(); + let chain_spec = Arc::new( + ChainSpecBuilder::default().chain(MAINNET.chain).genesis(genesis).osaka_activated().build(), + ); + let genesis_hash = chain_spec.genesis_hash(); + let node_config = NodeConfig::test() + .with_chain(chain_spec) + .with_unused_ports() + .with_rpc(RpcServerArgs::default().with_unused_ports().with_http()); + let NodeHandle { node, node_exit_future: _ } = NodeBuilder::new(node_config.clone()) + .testing_node(exec.clone()) + .node(EthereumNode::default()) + .launch() + .await?; + + let mut node = NodeTestContext::new(node, eth_payload_attributes).await?; + + let wallets = Wallet::new(2).wallet_gen(); + let blob_wallet = wallets.first().unwrap(); + + // build blob tx + let blob_tx = TransactionTestContext::tx_with_blobs_bytes(1, blob_wallet.clone()).await?; + + let tx = PooledTransactionVariant::decode_2718_exact(&blob_tx).unwrap(); + assert!(tx.as_eip4844().unwrap().tx().sidecar.is_eip4844()); + + // inject blob tx to the pool + let blob_tx_hash = node.rpc.inject_tx(blob_tx).await?; + // fetch it from rpc + let envelope = node.rpc.envelope_by_hash(blob_tx_hash).await?; + // assert that sidecar was converted to eip7594 + assert!(envelope.as_eip4844().unwrap().tx().sidecar().unwrap().is_eip7594()); + // validate sidecar + TransactionTestContext::validate_sidecar(envelope); + + // build a payload + let blob_payload = node.new_payload().await?; + + // submit the blob payload + let blob_block_hash = node.submit_payload(blob_payload).await?; + + node.update_forkchoice(genesis_hash, blob_block_hash).await?; + + Ok(()) +} + +#[tokio::test] +async fn blob_conversion_at_osaka() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + let tasks = TaskManager::current(); + let exec = tasks.executor(); + + let current_timestamp = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs(); + // Osaka activates in 2 slots + let osaka_timestamp = current_timestamp + 24; + + let genesis: Genesis = serde_json::from_str(include_str!("../assets/genesis.json")).unwrap(); + let chain_spec = Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis(genesis) + .prague_activated() + .with_osaka_at(osaka_timestamp) + .build(), + ); + let genesis_hash = chain_spec.genesis_hash(); + let node_config = NodeConfig::test() + .with_chain(chain_spec) + .with_unused_ports() + .with_rpc(RpcServerArgs::default().with_unused_ports().with_http()); + let NodeHandle { node, node_exit_future: _ } = NodeBuilder::new(node_config.clone()) + .testing_node(exec.clone()) + .node(EthereumNode::default()) + .launch() + .await?; + + let mut node = NodeTestContext::new(node, eth_payload_attributes).await?; + + let mut wallets = Wallet::new(3).wallet_gen(); + let first = wallets.pop().unwrap(); + let second = wallets.pop().unwrap(); + + // build a dummy payload at `current_timestamp` + let raw_tx = TransactionTestContext::transfer_tx_bytes(1, wallets.pop().unwrap()).await; + node.rpc.inject_tx(raw_tx).await?; + node.payload.timestamp = current_timestamp - 1; + node.advance_block().await?; + + // build blob txs + let first_blob = TransactionTestContext::tx_with_blobs_bytes(1, first.clone()).await?; + let second_blob = TransactionTestContext::tx_with_blobs_bytes(1, second.clone()).await?; + + // assert both txs have legacy sidecars + assert!(PooledTransactionVariant::decode_2718_exact(&first_blob) + .unwrap() + .as_eip4844() + .unwrap() + .tx() + .sidecar + .is_eip4844()); + assert!(PooledTransactionVariant::decode_2718_exact(&second_blob) + .unwrap() + .as_eip4844() + .unwrap() + .tx() + .sidecar + .is_eip4844()); + + // inject first blob tx to the pool + let blob_tx_hash = node.rpc.inject_tx(first_blob).await?; + // fetch it from rpc + let envelope = node.rpc.envelope_by_hash(blob_tx_hash).await?; + // assert that it still has a legacy sidecar + assert!(envelope.as_eip4844().unwrap().tx().sidecar().unwrap().is_eip4844()); + // validate sidecar + TransactionTestContext::validate_sidecar(envelope); + + // build last Prague payload + node.payload.timestamp = current_timestamp + 11; + let prague_payload = node.new_payload().await?; + assert!(matches!(prague_payload.sidecars(), BlobSidecars::Eip4844(_))); + + // inject second blob tx to the pool + let blob_tx_hash = node.rpc.inject_tx(second_blob).await?; + // fetch it from rpc + let envelope = node.rpc.envelope_by_hash(blob_tx_hash).await?; + // assert that it still has a legacy sidecar + assert!(envelope.as_eip4844().unwrap().tx().sidecar().unwrap().is_eip4844()); + // validate sidecar + TransactionTestContext::validate_sidecar(envelope); + + tokio::time::sleep(Duration::from_secs(11)).await; + + // fetch second blob tx from rpc again + let envelope = node.rpc.envelope_by_hash(blob_tx_hash).await?; + // assert that it was converted to eip7594 + assert!(envelope.as_eip4844().unwrap().tx().sidecar().unwrap().is_eip7594()); + // validate sidecar + TransactionTestContext::validate_sidecar(envelope); + + // submit the Prague payload + node.update_forkchoice(genesis_hash, node.submit_payload(prague_payload).await?).await?; + + // Build first Osaka payload + node.payload.timestamp = osaka_timestamp - 1; + let osaka_payload = node.new_payload().await?; + + // Assert that it includes the second blob tx with eip7594 sidecar + assert!(osaka_payload.block().body().transactions().any(|tx| *tx.hash() == blob_tx_hash)); + assert!(matches!(osaka_payload.sidecars(), BlobSidecars::Eip7594(_))); + + node.update_forkchoice(genesis_hash, node.submit_payload(osaka_payload).await?).await?; + + Ok(()) +} diff --git a/crates/ethereum/node/tests/e2e/dev.rs b/crates/ethereum/node/tests/e2e/dev.rs index d4a24191dbd..bf022a514e8 100644 --- a/crates/ethereum/node/tests/e2e/dev.rs +++ b/crates/ethereum/node/tests/e2e/dev.rs @@ -1,16 +1,14 @@ use alloy_eips::eip2718::Encodable2718; use alloy_genesis::Genesis; -use alloy_primitives::{b256, hex}; +use alloy_primitives::{b256, hex, Address}; use futures::StreamExt; use reth_chainspec::ChainSpec; -use reth_node_api::{BlockBody, FullNodeComponents, FullNodePrimitives, NodeTypes}; -use reth_node_builder::{ - rpc::RethRpcAddOns, EngineNodeLauncher, FullNode, NodeBuilder, NodeConfig, NodeHandle, -}; +use reth_node_api::{BlockBody, FullNodeComponents}; +use reth_node_builder::{rpc::RethRpcAddOns, FullNode, NodeBuilder, NodeConfig, NodeHandle}; use reth_node_core::args::DevArgs; use reth_node_ethereum::{node::EthereumAddOns, EthereumNode}; use reth_provider::{providers::BlockchainProvider, CanonStateSubscriptions}; -use reth_rpc_eth_api::helpers::EthTransactions; +use reth_rpc_eth_api::{helpers::EthTransactions, EthApiServer}; use reth_tasks::TaskManager; use std::sync::Arc; @@ -28,26 +26,61 @@ async fn can_run_dev_node() -> eyre::Result<()> { .with_types_and_provider::>() .with_components(EthereumNode::components()) .with_add_ons(EthereumAddOns::default()) - .launch_with_fn(|builder| { - let launcher = EngineNodeLauncher::new( - builder.task_executor().clone(), - builder.config().datadir(), - Default::default(), - ); - builder.launch_with(launcher) + .launch_with_debug_capabilities() + .await?; + + assert_chain_advances(&node).await; + + Ok(()) +} + +#[tokio::test] +async fn can_run_dev_node_custom_attributes() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + let tasks = TaskManager::current(); + let exec = tasks.executor(); + + let node_config = NodeConfig::test() + .with_chain(custom_chain()) + .with_dev(DevArgs { dev: true, ..Default::default() }); + let fee_recipient = Address::random(); + let NodeHandle { node, .. } = NodeBuilder::new(node_config.clone()) + .testing_node(exec.clone()) + .with_types_and_provider::>() + .with_components(EthereumNode::components()) + .with_add_ons(EthereumAddOns::default()) + .launch_with_debug_capabilities() + .map_debug_payload_attributes(move |mut attributes| { + attributes.suggested_fee_recipient = fee_recipient; + attributes }) .await?; - assert_chain_advances(node).await; + assert_chain_advances(&node).await; + + assert!( + node.rpc_registry.eth_api().balance(fee_recipient, Default::default()).await.unwrap() > 0 + ); + + assert!( + node.rpc_registry + .eth_api() + .block_by_number(Default::default(), false) + .await + .unwrap() + .unwrap() + .header + .beneficiary == + fee_recipient + ); Ok(()) } -async fn assert_chain_advances(node: FullNode) +async fn assert_chain_advances(node: &FullNode) where N: FullNodeComponents, AddOns: RethRpcAddOns, - N::Types: NodeTypes, { let mut notifications = node.provider.canonical_state_stream(); diff --git a/crates/ethereum/node/tests/e2e/p2p.rs b/crates/ethereum/node/tests/e2e/p2p.rs index fc00df9e719..34a42105381 100644 --- a/crates/ethereum/node/tests/e2e/p2p.rs +++ b/crates/ethereum/node/tests/e2e/p2p.rs @@ -66,8 +66,14 @@ async fn e2e_test_send_transactions() -> eyre::Result<()> { .build(), ); - let (mut nodes, _tasks, _) = - setup_engine::(2, chain_spec.clone(), false, eth_payload_attributes).await?; + let (mut nodes, _tasks, _) = setup_engine::( + 2, + chain_spec.clone(), + false, + Default::default(), + eth_payload_attributes, + ) + .await?; let mut node = nodes.pop().unwrap(); let provider = ProviderBuilder::new().connect_http(node.rpc_url()); @@ -102,8 +108,14 @@ async fn test_long_reorg() -> eyre::Result<()> { .build(), ); - let (mut nodes, _tasks, _) = - setup_engine::(2, chain_spec.clone(), false, eth_payload_attributes).await?; + let (mut nodes, _tasks, _) = setup_engine::( + 2, + chain_spec.clone(), + false, + Default::default(), + eth_payload_attributes, + ) + .await?; let mut first_node = nodes.pop().unwrap(); let mut second_node = nodes.pop().unwrap(); @@ -152,8 +164,14 @@ async fn test_reorg_through_backfill() -> eyre::Result<()> { .build(), ); - let (mut nodes, _tasks, _) = - setup_engine::(2, chain_spec.clone(), false, eth_payload_attributes).await?; + let (mut nodes, _tasks, _) = setup_engine::( + 2, + chain_spec.clone(), + false, + Default::default(), + eth_payload_attributes, + ) + .await?; let mut first_node = nodes.pop().unwrap(); let mut second_node = nodes.pop().unwrap(); @@ -167,9 +185,9 @@ async fn test_reorg_through_backfill() -> eyre::Result<()> { let head = first_provider.get_block_by_number(20.into()).await?.unwrap(); second_node.sync_to(head.header.hash).await?; - // Produce an unfinalized fork chain with 5 blocks + // Produce an unfinalized fork chain with 30 blocks second_node.payload.timestamp = head.header.timestamp; - advance_with_random_transactions(&mut second_node, 5, &mut rng, false).await?; + advance_with_random_transactions(&mut second_node, 30, &mut rng, false).await?; // Now reorg second node to the finalized canonical head let head = first_provider.get_block_by_number(100.into()).await?.unwrap(); diff --git a/crates/ethereum/node/tests/e2e/pool.rs b/crates/ethereum/node/tests/e2e/pool.rs index 6f7174b348e..9187cb61405 100644 --- a/crates/ethereum/node/tests/e2e/pool.rs +++ b/crates/ethereum/node/tests/e2e/pool.rs @@ -15,9 +15,216 @@ use reth_provider::CanonStateSubscriptions; use reth_tasks::TaskManager; use reth_transaction_pool::{ blobstore::InMemoryBlobStore, test_utils::OkValidator, BlockInfo, CoinbaseTipOrdering, - EthPooledTransaction, Pool, TransactionOrigin, TransactionPool, TransactionPoolExt, + EthPooledTransaction, Pool, PoolTransaction, TransactionOrigin, TransactionPool, + TransactionPoolExt, }; -use std::sync::Arc; +use std::{sync::Arc, time::Duration}; + +// Test that stale transactions could be correctly evicted. +#[tokio::test] +async fn maintain_txpool_stale_eviction() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + let tasks = TaskManager::current(); + let executor = tasks.executor(); + + let txpool = Pool::new( + OkValidator::default(), + CoinbaseTipOrdering::default(), + InMemoryBlobStore::default(), + Default::default(), + ); + + // Directly generate a node to simulate various traits such as `StateProviderFactory` required + // by the pool maintenance task + let genesis: Genesis = serde_json::from_str(include_str!("../assets/genesis.json")).unwrap(); + let chain_spec = Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis(genesis) + .cancun_activated() + .build(), + ); + let node_config = NodeConfig::test() + .with_chain(chain_spec) + .with_unused_ports() + .with_rpc(RpcServerArgs::default().with_unused_ports().with_http()); + let NodeHandle { node, node_exit_future: _ } = NodeBuilder::new(node_config.clone()) + .testing_node(executor.clone()) + .node(EthereumNode::default()) + .launch() + .await?; + + let node = NodeTestContext::new(node, eth_payload_attributes).await?; + + let wallet = Wallet::default(); + + let config = reth_transaction_pool::maintain::MaintainPoolConfig { + max_tx_lifetime: Duration::from_secs(1), + ..Default::default() + }; + + executor.spawn_critical( + "txpool maintenance task", + reth_transaction_pool::maintain::maintain_transaction_pool_future( + node.inner.provider.clone(), + txpool.clone(), + node.inner.provider.clone().canonical_state_stream(), + executor.clone(), + config, + ), + ); + + // create a tx with insufficient gas fee and it will be parked + let envelop = + TransactionTestContext::transfer_tx_with_gas_fee(1, Some(8_u128), wallet.inner).await; + let tx = Recovered::new_unchecked( + EthereumTxEnvelope::::from(envelop.clone()), + Default::default(), + ); + let pooled_tx = EthPooledTransaction::new(tx.clone(), 200); + + txpool.add_transaction(TransactionOrigin::External, pooled_tx).await.unwrap(); + assert_eq!(txpool.len(), 1); + + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + + // stale tx should be evicted + assert_eq!(txpool.len(), 0); + + Ok(()) +} + +// Test that the pool's maintenance task can correctly handle `CanonStateNotification::Reorg` events +#[tokio::test] +async fn maintain_txpool_reorg() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + let tasks = TaskManager::current(); + let executor = tasks.executor(); + + let txpool = Pool::new( + OkValidator::default(), + CoinbaseTipOrdering::default(), + InMemoryBlobStore::default(), + Default::default(), + ); + + // Directly generate a node to simulate various traits such as `StateProviderFactory` required + // by the pool maintenance task + let genesis: Genesis = serde_json::from_str(include_str!("../assets/genesis.json")).unwrap(); + let chain_spec = Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis(genesis) + .cancun_activated() + .build(), + ); + let genesis_hash = chain_spec.genesis_hash(); + let node_config = NodeConfig::test() + .with_chain(chain_spec) + .with_unused_ports() + .with_rpc(RpcServerArgs::default().with_unused_ports().with_http()); + let NodeHandle { node, node_exit_future: _ } = NodeBuilder::new(node_config.clone()) + .testing_node(executor.clone()) + .node(EthereumNode::default()) + .launch() + .await?; + + let mut node = NodeTestContext::new(node, eth_payload_attributes).await?; + + let wallets = Wallet::new(2).wallet_gen(); + let w1 = wallets.first().unwrap(); + let w2 = wallets.last().unwrap(); + + executor.spawn_critical( + "txpool maintenance task", + reth_transaction_pool::maintain::maintain_transaction_pool_future( + node.inner.provider.clone(), + txpool.clone(), + node.inner.provider.clone().canonical_state_stream(), + executor.clone(), + reth_transaction_pool::maintain::MaintainPoolConfig::default(), + ), + ); + + // build tx1 from wallet1 + let envelop1 = TransactionTestContext::transfer_tx(1, w1.clone()).await; + let tx1 = Recovered::new_unchecked( + EthereumTxEnvelope::::from(envelop1.clone()), + w1.address(), + ); + let pooled_tx1 = EthPooledTransaction::new(tx1.clone(), 200); + let tx_hash1 = *pooled_tx1.clone().hash(); + + // build tx2 from wallet2 + let envelop2 = TransactionTestContext::transfer_tx(1, w2.clone()).await; + let tx2 = Recovered::new_unchecked( + EthereumTxEnvelope::::from(envelop2.clone()), + w2.address(), + ); + let pooled_tx2 = EthPooledTransaction::new(tx2.clone(), 200); + let tx_hash2 = *pooled_tx2.clone().hash(); + + let block_info = BlockInfo { + block_gas_limit: ETHEREUM_BLOCK_GAS_LIMIT_30M, + last_seen_block_hash: B256::ZERO, + last_seen_block_number: 0, + pending_basefee: 10, + pending_blob_fee: Some(10), + }; + + txpool.set_block_info(block_info); + + // add two txs to the pool + txpool.add_transaction(TransactionOrigin::External, pooled_tx1).await.unwrap(); + txpool.add_transaction(TransactionOrigin::External, pooled_tx2).await.unwrap(); + + // inject tx1, make the node advance and eventually generate `CanonStateNotification::Commit` + // event to propagate to the pool + let _ = node.rpc.inject_tx(envelop1.encoded_2718().into()).await.unwrap(); + + // build a payload based on tx1 + let payload1 = node.new_payload().await?; + + // clean up the internal pool of the provider node + node.inner.pool.remove_transactions(vec![tx_hash1]); + + // inject tx2, make the node reorg and eventually generate `CanonStateNotification::Reorg` event + // to propagate to the pool + let _ = node.rpc.inject_tx(envelop2.encoded_2718().into()).await.unwrap(); + + // build a payload based on tx2 + let payload2 = node.new_payload().await?; + + // submit payload1 + let block_hash1 = node.submit_payload(payload1).await?; + + node.update_forkchoice(genesis_hash, block_hash1).await?; + + loop { + // wait for pool to process `CanonStateNotification::Commit` event correctly, and finally + // tx1 will be removed and tx2 is still in the pool + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + if txpool.get(&tx_hash1).is_none() && txpool.get(&tx_hash2).is_some() { + break; + } + } + + // submit payload2 + let block_hash2 = node.submit_payload(payload2).await?; + + node.update_forkchoice(genesis_hash, block_hash2).await?; + + loop { + // wait for pool to process `CanonStateNotification::Reorg` event properly, and finally tx1 + // will be added back to the pool and tx2 will be removed. + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + if txpool.get(&tx_hash1).is_some() && txpool.get(&tx_hash2).is_none() { + break; + } + } + + Ok(()) +} // Test that the pool's maintenance task can correctly handle `CanonStateNotification::Commit` // events diff --git a/crates/ethereum/node/tests/e2e/rpc.rs b/crates/ethereum/node/tests/e2e/rpc.rs index b8eefea3d85..f040f44dfd8 100644 --- a/crates/ethereum/node/tests/e2e/rpc.rs +++ b/crates/ethereum/node/tests/e2e/rpc.rs @@ -1,5 +1,5 @@ use crate::utils::eth_payload_attributes; -use alloy_eips::{calc_next_block_base_fee, eip2718::Encodable2718}; +use alloy_eips::{eip2718::Encodable2718, eip7910::EthConfig}; use alloy_primitives::{Address, B256, U256}; use alloy_provider::{network::EthereumWallet, Provider, ProviderBuilder, SendableTx}; use alloy_rpc_types_beacon::relay::{ @@ -9,11 +9,14 @@ use alloy_rpc_types_beacon::relay::{ use alloy_rpc_types_engine::{BlobsBundleV1, ExecutionPayloadV3}; use alloy_rpc_types_eth::TransactionRequest; use rand::{rngs::StdRng, Rng, SeedableRng}; -use reth_chainspec::{ChainSpecBuilder, MAINNET}; +use reth_chainspec::{ChainSpecBuilder, EthChainSpec, MAINNET}; use reth_e2e_test_utils::setup_engine; use reth_node_ethereum::EthereumNode; use reth_payload_primitives::BuiltPayload; -use std::sync::Arc; +use std::{ + sync::Arc, + time::{SystemTime, UNIX_EPOCH}, +}; alloy_sol_types::sol! { #[sol(rpc, bytecode = "6080604052348015600f57600080fd5b5060405160db38038060db833981016040819052602a91607a565b60005b818110156074576040805143602082015290810182905260009060600160408051601f19818403018152919052805160209091012080555080606d816092565b915050602d565b505060b8565b600060208284031215608b57600080fd5b5051919050565b60006001820160b157634e487b7160e01b600052601160045260246000fd5b5060010190565b60168060c56000396000f3fe6080604052600080fdfea164736f6c6343000810000a")] @@ -45,8 +48,14 @@ async fn test_fee_history() -> eyre::Result<()> { .build(), ); - let (mut nodes, _tasks, wallet) = - setup_engine::(1, chain_spec.clone(), false, eth_payload_attributes).await?; + let (mut nodes, _tasks, wallet) = setup_engine::( + 1, + chain_spec.clone(), + false, + Default::default(), + eth_payload_attributes, + ) + .await?; let mut node = nodes.pop().unwrap(); let provider = ProviderBuilder::new() .wallet(EthereumWallet::new(wallet.wallet_gen().swap_remove(0))) @@ -56,10 +65,12 @@ async fn test_fee_history() -> eyre::Result<()> { let genesis_base_fee = chain_spec.initial_base_fee().unwrap() as u128; let expected_first_base_fee = genesis_base_fee - - genesis_base_fee / chain_spec.base_fee_params_at_block(0).max_change_denominator; + genesis_base_fee / + chain_spec + .base_fee_params_at_timestamp(chain_spec.genesis_timestamp()) + .max_change_denominator; assert_eq!(fee_history.base_fee_per_gas[0], genesis_base_fee); assert_eq!(fee_history.base_fee_per_gas[1], expected_first_base_fee,); - // Spend some gas let builder = GasWaster::deploy_builder(&provider, U256::from(500)).send().await?; node.advance_block().await?; @@ -92,14 +103,9 @@ async fn test_fee_history() -> eyre::Result<()> { .unwrap() .header; for block in (latest_block + 2 - block_count)..=latest_block { - let expected_base_fee = calc_next_block_base_fee( - prev_header.gas_used, - prev_header.gas_limit, - prev_header.base_fee_per_gas.unwrap(), - chain_spec.base_fee_params_at_block(block), - ); - let header = provider.get_block_by_number(block.into()).await?.unwrap().header; + let expected_base_fee = + chain_spec.next_block_base_fee(&prev_header, header.timestamp).unwrap(); assert_eq!(header.base_fee_per_gas.unwrap(), expected_base_fee); assert_eq!( @@ -127,8 +133,14 @@ async fn test_flashbots_validate_v3() -> eyre::Result<()> { .build(), ); - let (mut nodes, _tasks, wallet) = - setup_engine::(1, chain_spec.clone(), false, eth_payload_attributes).await?; + let (mut nodes, _tasks, wallet) = setup_engine::( + 1, + chain_spec.clone(), + false, + Default::default(), + eth_payload_attributes, + ) + .await?; let mut node = nodes.pop().unwrap(); let provider = ProviderBuilder::new() .wallet(EthereumWallet::new(wallet.wallet_gen().swap_remove(0))) @@ -203,8 +215,14 @@ async fn test_flashbots_validate_v4() -> eyre::Result<()> { .build(), ); - let (mut nodes, _tasks, wallet) = - setup_engine::(1, chain_spec.clone(), false, eth_payload_attributes).await?; + let (mut nodes, _tasks, wallet) = setup_engine::( + 1, + chain_spec.clone(), + false, + Default::default(), + eth_payload_attributes, + ) + .await?; let mut node = nodes.pop().unwrap(); let provider = ProviderBuilder::new() .wallet(EthereumWallet::new(wallet.wallet_gen().swap_remove(0))) @@ -267,3 +285,47 @@ async fn test_flashbots_validate_v4() -> eyre::Result<()> { .is_err()); Ok(()) } + +#[tokio::test] +async fn test_eth_config() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + + let timestamp = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs(); + + let prague_timestamp = 10; + let osaka_timestamp = timestamp + 10000000; + + let chain_spec = Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) + .cancun_activated() + .with_prague_at(prague_timestamp) + .with_osaka_at(osaka_timestamp) + .build(), + ); + + let (mut nodes, _tasks, wallet) = setup_engine::( + 1, + chain_spec.clone(), + false, + Default::default(), + eth_payload_attributes, + ) + .await?; + let mut node = nodes.pop().unwrap(); + let provider = ProviderBuilder::new() + .wallet(EthereumWallet::new(wallet.wallet_gen().swap_remove(0))) + .connect_http(node.rpc_url()); + + let _ = provider.send_transaction(TransactionRequest::default().to(Address::ZERO)).await?; + node.advance_block().await?; + + let config = provider.client().request_noparams::("eth_config").await?; + + assert_eq!(config.last.unwrap().activation_time, osaka_timestamp); + assert_eq!(config.current.activation_time, prague_timestamp); + assert_eq!(config.next.unwrap().activation_time, osaka_timestamp); + + Ok(()) +} diff --git a/crates/ethereum/node/tests/it/builder.rs b/crates/ethereum/node/tests/it/builder.rs index e3d78182ed5..48f1e0da2fb 100644 --- a/crates/ethereum/node/tests/it/builder.rs +++ b/crates/ethereum/node/tests/it/builder.rs @@ -10,6 +10,7 @@ use reth_node_api::NodeTypesWithDBAdapter; use reth_node_builder::{EngineNodeLauncher, FullNodeComponents, NodeBuilder, NodeConfig}; use reth_node_ethereum::node::{EthereumAddOns, EthereumNode}; use reth_provider::providers::BlockchainProvider; +use reth_rpc_builder::Identity; use reth_tasks::TaskManager; #[test] @@ -33,6 +34,7 @@ fn test_basic_setup() { let _client = handles.rpc.http_client(); Ok(()) }) + .map_add_ons(|addons| addons.with_rpc_middleware(Identity::default())) .extend_rpc_modules(|ctx| { let _ = ctx.config(); let _ = ctx.node().provider(); @@ -50,21 +52,64 @@ async fn test_eth_launcher() { let _builder = NodeBuilder::new(config) .with_database(db) + .with_launch_context(tasks.executor()) .with_types_and_provider::>>, >>() .with_components(EthereumNode::components()) .with_add_ons(EthereumAddOns::default()) + .apply(|builder| { + let _ = builder.db(); + builder + }) .launch_with_fn(|builder| { let launcher = EngineNodeLauncher::new( tasks.executor(), - builder.config.datadir(), + builder.config().datadir(), Default::default(), ); builder.launch_with(launcher) }); } +#[test] +fn test_eth_launcher_with_tokio_runtime() { + // #[tokio::test] can not be used here because we need to create a custom tokio runtime + // and it would be dropped before the test is finished, resulting in a panic. + let main_rt = tokio::runtime::Runtime::new().expect("Failed to create tokio runtime"); + + let custom_rt = tokio::runtime::Runtime::new().expect("Failed to create tokio runtime"); + + main_rt.block_on(async { + let tasks = TaskManager::current(); + let config = NodeConfig::test(); + let db = create_test_rw_db(); + let _builder = + NodeBuilder::new(config) + .with_database(db) + .with_launch_context(tasks.executor()) + .with_types_and_provider::>>, + >>() + .with_components(EthereumNode::components()) + .with_add_ons( + EthereumAddOns::default().with_tokio_runtime(Some(custom_rt.handle().clone())), + ) + .apply(|builder| { + let _ = builder.db(); + builder + }) + .launch_with_fn(|builder| { + let launcher = EngineNodeLauncher::new( + tasks.executor(), + builder.config().datadir(), + Default::default(), + ); + builder.launch_with(launcher) + }); + }); +} + #[test] fn test_node_setup() { let config = NodeConfig::test(); diff --git a/crates/ethereum/payload/Cargo.toml b/crates/ethereum/payload/Cargo.toml index 0907147ca4e..42d159fb844 100644 --- a/crates/ethereum/payload/Cargo.toml +++ b/crates/ethereum/payload/Cargo.toml @@ -13,6 +13,7 @@ workspace = true [dependencies] # reth +reth-consensus-common.workspace = true reth-ethereum-primitives.workspace = true reth-primitives-traits.workspace = true reth-revm.workspace = true @@ -29,6 +30,7 @@ reth-chainspec.workspace = true reth-payload-validator.workspace = true # ethereum +alloy-rlp.workspace = true revm.workspace = true alloy-rpc-types-engine.workspace = true diff --git a/crates/ethereum/payload/src/lib.rs b/crates/ethereum/payload/src/lib.rs index cf5ec600685..5b3eb9cfcbd 100644 --- a/crates/ethereum/payload/src/lib.rs +++ b/crates/ethereum/payload/src/lib.rs @@ -6,34 +6,35 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![allow(clippy::useless_let_if_seq)] -pub mod validator; -pub use validator::EthereumExecutionPayloadValidator; - -use alloy_consensus::{Transaction, Typed2718}; +use alloy_consensus::Transaction; use alloy_primitives::U256; +use alloy_rlp::Encodable; use reth_basic_payload_builder::{ is_better_payload, BuildArguments, BuildOutcome, MissingPayloadBehaviour, PayloadBuilder, PayloadConfig, }; use reth_chainspec::{ChainSpecProvider, EthChainSpec, EthereumHardforks}; -use reth_errors::{BlockExecutionError, BlockValidationError}; +use reth_consensus_common::validation::MAX_RLP_BLOCK_SIZE; +use reth_errors::{BlockExecutionError, BlockValidationError, ConsensusError}; use reth_ethereum_primitives::{EthPrimitives, TransactionSigned}; use reth_evm::{ execute::{BlockBuilder, BlockBuilderOutcome}, ConfigureEvm, Evm, NextBlockEnvAttributes, }; use reth_evm_ethereum::EthEvmConfig; -use reth_payload_builder::{EthBuiltPayload, EthPayloadBuilderAttributes}; +use reth_payload_builder::{BlobSidecars, EthBuiltPayload, EthPayloadBuilderAttributes}; use reth_payload_builder_primitives::PayloadBuilderError; use reth_payload_primitives::PayloadBuilderAttributes; +use reth_primitives_traits::transaction::error::InvalidTransactionError; use reth_revm::{database::StateProviderDatabase, db::State}; use reth_storage_api::StateProviderFactory; use reth_transaction_pool::{ - error::InvalidPoolTransactionError, BestTransactions, BestTransactionsAttributes, - PoolTransaction, TransactionPool, ValidPoolTransaction, + error::{Eip4844PoolTransactionError, InvalidPoolTransactionError}, + BestTransactions, BestTransactionsAttributes, PoolTransaction, TransactionPool, + ValidPoolTransaction, }; use revm::context_interface::Block as _; use std::sync::Arc; @@ -41,8 +42,9 @@ use tracing::{debug, trace, warn}; mod config; pub use config::*; -use reth_primitives_traits::transaction::error::InvalidTransactionError; -use reth_transaction_pool::error::Eip4844PoolTransactionError; + +pub mod validator; +pub use validator::EthereumExecutionPayloadValidator; type BestTransactionsIter = Box< dyn BestTransactions::Transaction>>>, @@ -174,8 +176,8 @@ where debug!(target: "payload_builder", id=%attributes.id, parent_header = ?parent_header.hash(), parent_number = parent_header.number, "building new payload"); let mut cumulative_gas_used = 0; - let block_gas_limit: u64 = builder.evm_mut().block().gas_limit; - let base_fee = builder.evm_mut().block().basefee; + let block_gas_limit: u64 = builder.evm_mut().block().gas_limit(); + let base_fee = builder.evm_mut().block().basefee(); let mut best_txs = best_txs(BestTransactionsAttributes::new( base_fee, @@ -188,11 +190,19 @@ where PayloadBuilderError::Internal(err.into()) })?; + // initialize empty blob sidecars at first. If cancun is active then this will be populated by + // blob sidecars if any. + let mut blob_sidecars = BlobSidecars::Empty; + let mut block_blob_count = 0; + let mut block_transactions_rlp_length = 0; + let blob_params = chain_spec.blob_params_at_timestamp(attributes.timestamp); let max_blob_count = blob_params.as_ref().map(|params| params.max_blob_count).unwrap_or_default(); + let is_osaka = chain_spec.is_osaka_active_at_timestamp(attributes.timestamp); + while let Some(pool_tx) = best_txs.next() { // ensure we still have capacity for this transaction if cumulative_gas_used + pool_tx.gas_limit() > block_gas_limit { @@ -214,8 +224,25 @@ where // convert tx to a signed transaction let tx = pool_tx.to_consensus(); + let estimated_block_size_with_tx = block_transactions_rlp_length + + tx.inner().length() + + attributes.withdrawals().length() + + 1024; // 1Kb of overhead for the block header + + if is_osaka && estimated_block_size_with_tx > MAX_RLP_BLOCK_SIZE { + best_txs.mark_invalid( + &pool_tx, + InvalidPoolTransactionError::OversizedData { + size: estimated_block_size_with_tx, + limit: MAX_RLP_BLOCK_SIZE, + }, + ); + continue; + } + // There's only limited amount of blob space available per block, so we need to check if // the EIP-4844 can still fit in the block + let mut blob_tx_sidecar = None; if let Some(blob_tx) = tx.as_eip4844() { let tx_blob_count = blob_tx.tx().blob_versioned_hashes.len() as u64; @@ -236,6 +263,34 @@ where ); continue } + + let blob_sidecar_result = 'sidecar: { + let Some(sidecar) = + pool.get_blob(*tx.hash()).map_err(PayloadBuilderError::other)? + else { + break 'sidecar Err(Eip4844PoolTransactionError::MissingEip4844BlobSidecar) + }; + + if is_osaka { + if sidecar.is_eip7594() { + Ok(sidecar) + } else { + Err(Eip4844PoolTransactionError::UnexpectedEip4844SidecarAfterOsaka) + } + } else if sidecar.is_eip4844() { + Ok(sidecar) + } else { + Err(Eip4844PoolTransactionError::UnexpectedEip7594SidecarBeforeOsaka) + } + }; + + blob_tx_sidecar = match blob_sidecar_result { + Ok(sidecar) => Some(sidecar), + Err(error) => { + best_txs.mark_invalid(&pool_tx, InvalidPoolTransactionError::Eip4844(error)); + continue + } + }; } let gas_used = match builder.execute_transaction(tx.clone()) { @@ -273,11 +328,18 @@ where } } - // update add to total fees + block_transactions_rlp_length += tx.inner().length(); + + // update and add to total fees let miner_fee = tx.effective_tip_per_gas(base_fee).expect("fee is always valid; execution succeeded"); total_fees += U256::from(miner_fee) * U256::from(gas_used); cumulative_gas_used += gas_used; + + // Add blob tx sidecar to the payload. + if let Some(sidecar) = blob_tx_sidecar { + blob_sidecars.push_sidecar_variant(sidecar.as_ref().clone()); + } } // check if we have a better block @@ -294,31 +356,19 @@ where .is_prague_active_at_timestamp(attributes.timestamp) .then_some(execution_result.requests); - // initialize empty blob sidecars at first. If cancun is active then this will - let mut blob_sidecars = Vec::new(); - - // only determine cancun fields when active - if chain_spec.is_cancun_active_at_timestamp(attributes.timestamp) { - // grab the blob sidecars from the executed txs - blob_sidecars = pool - .get_all_blobs_exact( - block - .body() - .transactions() - .filter(|tx| tx.is_eip4844()) - .map(|tx| *tx.tx_hash()) - .collect(), - ) - .map_err(PayloadBuilderError::other)?; - } - let sealed_block = Arc::new(block.sealed_block().clone()); debug!(target: "payload_builder", id=%attributes.id, sealed_block_header = ?sealed_block.sealed_header(), "sealed built block"); - let mut payload = EthBuiltPayload::new(attributes.id, sealed_block, total_fees, requests); + if is_osaka && sealed_block.rlp_length() > MAX_RLP_BLOCK_SIZE { + return Err(PayloadBuilderError::other(ConsensusError::BlockTooLarge { + rlp_length: sealed_block.rlp_length(), + max_rlp_length: MAX_RLP_BLOCK_SIZE, + })); + } - // extend the payload with the blob sidecars from the executed txs - payload.extend_sidecars(blob_sidecars.into_iter().map(Arc::unwrap_or_clone)); + let payload = EthBuiltPayload::new(attributes.id, sealed_block, total_fees, requests) + // add blob sidecars from the executed txs + .with_sidecars(blob_sidecars); Ok(BuildOutcome::Better { payload, cached_reads }) } diff --git a/crates/ethereum/payload/src/validator.rs b/crates/ethereum/payload/src/validator.rs index 75f4b1f474c..ccace26ef80 100644 --- a/crates/ethereum/payload/src/validator.rs +++ b/crates/ethereum/payload/src/validator.rs @@ -28,83 +28,80 @@ impl EthereumExecutionPayloadValidator { } impl EthereumExecutionPayloadValidator { - /// Returns true if the Cancun hardfork is active at the given timestamp. - #[inline] - fn is_cancun_active_at_timestamp(&self, timestamp: u64) -> bool { - self.chain_spec().is_cancun_active_at_timestamp(timestamp) - } - - /// Returns true if the Shanghai hardfork is active at the given timestamp. - #[inline] - fn is_shanghai_active_at_timestamp(&self, timestamp: u64) -> bool { - self.chain_spec().is_shanghai_active_at_timestamp(timestamp) - } - - /// Returns true if the Prague hardfork is active at the given timestamp. - #[inline] - fn is_prague_active_at_timestamp(&self, timestamp: u64) -> bool { - self.chain_spec().is_prague_active_at_timestamp(timestamp) - } - /// Ensures that the given payload does not violate any consensus rules that concern the block's - /// layout, like: - /// - missing or invalid base fee - /// - invalid extra data - /// - invalid transactions - /// - incorrect hash - /// - the versioned hashes passed with the payload do not exactly match transaction versioned - /// hashes - /// - the block does not contain blob transactions if it is pre-cancun + /// layout, /// - /// The checks are done in the order that conforms with the engine-API specification. - /// - /// This is intended to be invoked after receiving the payload from the CLI. - /// The additional [`MaybeCancunPayloadFields`](alloy_rpc_types_engine::MaybeCancunPayloadFields) are not part of the payload, but are additional fields in the `engine_newPayloadV3` RPC call, See also - /// - /// If the cancun fields are provided this also validates that the versioned hashes in the block - /// match the versioned hashes passed in the - /// [`CancunPayloadFields`](alloy_rpc_types_engine::CancunPayloadFields), if the cancun payload - /// fields are provided. If the payload fields are not provided, but versioned hashes exist - /// in the block, this is considered an error: [`PayloadError::InvalidVersionedHashes`]. - /// - /// This validates versioned hashes according to the Engine API Cancun spec: - /// + /// See also [`ensure_well_formed_payload`] pub fn ensure_well_formed_payload( &self, payload: ExecutionData, ) -> Result>, PayloadError> { - let ExecutionData { payload, sidecar } = payload; + ensure_well_formed_payload(&self.chain_spec, payload) + } +} + +/// Ensures that the given payload does not violate any consensus rules that concern the block's +/// layout, like: +/// - missing or invalid base fee +/// - invalid extra data +/// - invalid transactions +/// - incorrect hash +/// - the versioned hashes passed with the payload do not exactly match transaction versioned +/// hashes +/// - the block does not contain blob transactions if it is pre-cancun +/// +/// The checks are done in the order that conforms with the engine-API specification. +/// +/// This is intended to be invoked after receiving the payload from the CLI. +/// The additional [`MaybeCancunPayloadFields`](alloy_rpc_types_engine::MaybeCancunPayloadFields) are not part of the payload, but are additional fields in the `engine_newPayloadV3` RPC call, See also +/// +/// If the cancun fields are provided this also validates that the versioned hashes in the block +/// match the versioned hashes passed in the +/// [`CancunPayloadFields`](alloy_rpc_types_engine::CancunPayloadFields), if the cancun payload +/// fields are provided. If the payload fields are not provided, but versioned hashes exist +/// in the block, this is considered an error: [`PayloadError::InvalidVersionedHashes`]. +/// +/// This validates versioned hashes according to the Engine API Cancun spec: +/// +pub fn ensure_well_formed_payload( + chain_spec: ChainSpec, + payload: ExecutionData, +) -> Result>, PayloadError> +where + ChainSpec: EthereumHardforks, + T: SignedTransaction, +{ + let ExecutionData { payload, sidecar } = payload; - let expected_hash = payload.block_hash(); + let expected_hash = payload.block_hash(); - // First parse the block - let sealed_block = payload.try_into_block_with_sidecar(&sidecar)?.seal_slow(); + // First parse the block + let sealed_block = payload.try_into_block_with_sidecar(&sidecar)?.seal_slow(); - // Ensure the hash included in the payload matches the block hash - if expected_hash != sealed_block.hash() { - return Err(PayloadError::BlockHash { - execution: sealed_block.hash(), - consensus: expected_hash, - }) - } + // Ensure the hash included in the payload matches the block hash + if expected_hash != sealed_block.hash() { + return Err(PayloadError::BlockHash { + execution: sealed_block.hash(), + consensus: expected_hash, + }) + } - shanghai::ensure_well_formed_fields( - sealed_block.body(), - self.is_shanghai_active_at_timestamp(sealed_block.timestamp), - )?; + shanghai::ensure_well_formed_fields( + sealed_block.body(), + chain_spec.is_shanghai_active_at_timestamp(sealed_block.timestamp), + )?; - cancun::ensure_well_formed_fields( - &sealed_block, - sidecar.cancun(), - self.is_cancun_active_at_timestamp(sealed_block.timestamp), - )?; + cancun::ensure_well_formed_fields( + &sealed_block, + sidecar.cancun(), + chain_spec.is_cancun_active_at_timestamp(sealed_block.timestamp), + )?; - prague::ensure_well_formed_fields( - sealed_block.body(), - sidecar.prague(), - self.is_prague_active_at_timestamp(sealed_block.timestamp), - )?; + prague::ensure_well_formed_fields( + sealed_block.body(), + sidecar.prague(), + chain_spec.is_prague_active_at_timestamp(sealed_block.timestamp), + )?; - Ok(sealed_block) - } + Ok(sealed_block) } diff --git a/crates/ethereum/primitives/Cargo.toml b/crates/ethereum/primitives/Cargo.toml index 5a5f0ee101c..3bf9e8f3a48 100644 --- a/crates/ethereum/primitives/Cargo.toml +++ b/crates/ethereum/primitives/Cargo.toml @@ -21,6 +21,8 @@ reth-zstd-compressors = { workspace = true, optional = true } alloy-eips = { workspace = true, features = ["k256"] } alloy-primitives.workspace = true alloy-consensus = { workspace = true, features = ["serde"] } +alloy-serde = { workspace = true, optional = true } +alloy-rpc-types-eth = { workspace = true, optional = true } alloy-rlp.workspace = true # misc @@ -40,8 +42,8 @@ rand.workspace = true reth-codecs = { workspace = true, features = ["test-utils"] } reth-zstd-compressors.workspace = true secp256k1 = { workspace = true, features = ["rand"] } -test-fuzz.workspace = true alloy-consensus = { workspace = true, features = ["serde", "arbitrary"] } +serde_json.workspace = true [features] default = ["std"] @@ -60,6 +62,9 @@ std = [ "derive_more/std", "serde_with?/std", "secp256k1/std", + "alloy-rpc-types-eth?/std", + "alloy-serde?/std", + "serde_json/std", ] reth-codec = [ "std", @@ -68,6 +73,7 @@ reth-codec = [ "dep:reth-zstd-compressors", ] arbitrary = [ + "std", "dep:arbitrary", "alloy-consensus/arbitrary", "alloy-consensus/k256", @@ -75,15 +81,19 @@ arbitrary = [ "reth-codecs?/arbitrary", "reth-primitives-traits/arbitrary", "alloy-eips/arbitrary", + "alloy-rpc-types-eth?/arbitrary", + "alloy-serde?/arbitrary", ] serde-bincode-compat = [ "dep:serde_with", "alloy-consensus/serde-bincode-compat", "alloy-eips/serde-bincode-compat", "reth-primitives-traits/serde-bincode-compat", + "alloy-rpc-types-eth?/serde-bincode-compat", ] serde = [ "dep:serde", + "dep:alloy-serde", "alloy-consensus/serde", "alloy-eips/serde", "alloy-primitives/serde", @@ -92,4 +102,6 @@ serde = [ "rand/serde", "rand_08/serde", "secp256k1/serde", + "alloy-rpc-types-eth?/serde", ] +rpc = ["dep:alloy-rpc-types-eth"] diff --git a/crates/ethereum/primitives/src/lib.rs b/crates/ethereum/primitives/src/lib.rs index aa9086b0213..09e7ef7add9 100644 --- a/crates/ethereum/primitives/src/lib.rs +++ b/crates/ethereum/primitives/src/lib.rs @@ -5,7 +5,7 @@ html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256", issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(test), warn(unused_crate_dependencies))] #![cfg_attr(not(feature = "std"), no_std)] @@ -14,12 +14,13 @@ extern crate alloc; mod receipt; pub use receipt::*; -/// Kept for concistency tests +/// Kept for consistency tests #[cfg(test)] mod transaction; -use alloy_consensus::TxEip4844; pub use alloy_consensus::{transaction::PooledTransaction, TxType}; +use alloy_consensus::{TxEip4844, TxEip4844WithSidecar}; +use alloy_eips::eip7594::BlobTransactionSidecarVariant; /// Typed Transaction type without a signature pub type Transaction = alloy_consensus::EthereumTypedTransaction; @@ -27,6 +28,10 @@ pub type Transaction = alloy_consensus::EthereumTypedTransaction; /// Signed transaction. pub type TransactionSigned = alloy_consensus::EthereumTxEnvelope; +/// A type alias for [`PooledTransaction`] that's also generic over blob sidecar. +pub type PooledTransactionVariant = + alloy_consensus::EthereumTxEnvelope>; + /// Bincode-compatible serde implementations. #[cfg(all(feature = "serde", feature = "serde-bincode-compat"))] pub mod serde_bincode_compat { diff --git a/crates/ethereum/primitives/src/receipt.rs b/crates/ethereum/primitives/src/receipt.rs index 1b03fa9dde4..cbe8b5b806d 100644 --- a/crates/ethereum/primitives/src/receipt.rs +++ b/crates/ethereum/primitives/src/receipt.rs @@ -1,41 +1,91 @@ +use core::fmt::Debug; + use alloc::vec::Vec; use alloy_consensus::{ - Eip2718EncodableReceipt, Eip658Value, ReceiptWithBloom, RlpDecodableReceipt, + Eip2718EncodableReceipt, Eip658Value, ReceiptEnvelope, ReceiptWithBloom, RlpDecodableReceipt, RlpEncodableReceipt, TxReceipt, TxType, Typed2718, }; use alloy_eips::{ - eip2718::{Eip2718Result, Encodable2718}, + eip2718::{Eip2718Error, Eip2718Result, Encodable2718, IsTyped2718}, Decodable2718, }; use alloy_primitives::{Bloom, Log, B256}; use alloy_rlp::{BufMut, Decodable, Encodable, Header}; use reth_primitives_traits::{proofs::ordered_trie_root_with_encoder, InMemorySize}; +/// Helper trait alias with requirements for transaction type generic to be used within [`Receipt`]. +pub trait TxTy: + Debug + + Copy + + Eq + + Send + + Sync + + InMemorySize + + Typed2718 + + TryFrom + + Decodable + + 'static +{ +} +impl TxTy for T where + T: Debug + + Copy + + Eq + + Send + + Sync + + InMemorySize + + Typed2718 + + TryFrom + + Decodable + + 'static +{ +} + +/// Raw ethereum receipt. +pub type Receipt = EthereumReceipt; + +#[cfg(feature = "rpc")] +/// Receipt representation for RPC. +pub type RpcReceipt = EthereumReceipt; + /// Typed ethereum transaction receipt. /// Receipt containing result of transaction execution. #[derive(Clone, Debug, PartialEq, Eq, Default)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[cfg_attr(feature = "reth-codec", derive(reth_codecs::CompactZstd))] #[cfg_attr(feature = "reth-codec", reth_codecs::add_arbitrary_tests(compact, rlp))] -#[cfg_attr(feature = "reth-codec", reth_zstd( - compressor = reth_zstd_compressors::RECEIPT_COMPRESSOR, - decompressor = reth_zstd_compressors::RECEIPT_DECOMPRESSOR -))] -pub struct Receipt { +#[cfg_attr(feature = "serde", serde(rename_all = "camelCase"))] +pub struct EthereumReceipt { /// Receipt type. - pub tx_type: TxType, + #[cfg_attr(feature = "serde", serde(rename = "type"))] + pub tx_type: T, /// If transaction is executed successfully. /// /// This is the `statusCode` + #[cfg_attr(feature = "serde", serde(with = "alloy_serde::quantity", rename = "status"))] pub success: bool, /// Gas used + #[cfg_attr(feature = "serde", serde(with = "alloy_serde::quantity"))] pub cumulative_gas_used: u64, /// Log send from contracts. - pub logs: Vec, + pub logs: Vec, } -impl Receipt { +#[cfg(feature = "rpc")] +impl Receipt { + /// Converts the logs of the receipt to RPC logs. + pub fn into_rpc( + self, + next_log_index: usize, + meta: alloy_consensus::transaction::TransactionMeta, + ) -> RpcReceipt { + let Self { tx_type, success, cumulative_gas_used, logs } = self; + let logs = alloy_rpc_types_eth::Log::collect_for_receipt(next_log_index, meta, logs); + RpcReceipt { tx_type, success, cumulative_gas_used, logs } + } +} + +impl Receipt { /// Returns length of RLP-encoded receipt fields with the given [`Bloom`] without an RLP header. pub fn rlp_encoded_fields_length(&self, bloom: &Bloom) -> usize { self.success.length() + @@ -61,7 +111,7 @@ impl Receipt { /// network header. pub fn rlp_decode_inner( buf: &mut &[u8], - tx_type: TxType, + tx_type: T, ) -> alloy_rlp::Result> { let header = Header::decode(buf)?; if !header.list { @@ -112,10 +162,7 @@ impl Receipt { /// RLP-decodes the receipt from the provided buffer. This does not expect a type byte or /// network header. - pub fn rlp_decode_inner_without_bloom( - buf: &mut &[u8], - tx_type: TxType, - ) -> alloy_rlp::Result { + pub fn rlp_decode_inner_without_bloom(buf: &mut &[u8], tx_type: T) -> alloy_rlp::Result { let header = Header::decode(buf)?; if !header.list { return Err(alloy_rlp::Error::UnexpectedString); @@ -134,21 +181,21 @@ impl Receipt { } } -impl Eip2718EncodableReceipt for Receipt { +impl Eip2718EncodableReceipt for Receipt { fn eip2718_encoded_length_with_bloom(&self, bloom: &Bloom) -> usize { !self.tx_type.is_legacy() as usize + self.rlp_header_inner(bloom).length_with_payload() } fn eip2718_encode_with_bloom(&self, bloom: &Bloom, out: &mut dyn BufMut) { if !self.tx_type.is_legacy() { - out.put_u8(self.tx_type as u8); + out.put_u8(self.tx_type.ty()); } self.rlp_header_inner(bloom).encode(out); self.rlp_encode_fields(bloom, out); } } -impl RlpEncodableReceipt for Receipt { +impl RlpEncodableReceipt for Receipt { fn rlp_encoded_length_with_bloom(&self, bloom: &Bloom) -> usize { let mut len = self.eip2718_encoded_length_with_bloom(bloom); if !self.tx_type.is_legacy() { @@ -171,21 +218,21 @@ impl RlpEncodableReceipt for Receipt { } } -impl RlpDecodableReceipt for Receipt { +impl RlpDecodableReceipt for Receipt { fn rlp_decode_with_bloom(buf: &mut &[u8]) -> alloy_rlp::Result> { let header_buf = &mut &**buf; let header = Header::decode(header_buf)?; // Legacy receipt, reuse initial buffer without advancing if header.list { - return Self::rlp_decode_inner(buf, TxType::Legacy) + return Self::rlp_decode_inner(buf, T::try_from(0)?) } // Otherwise, advance the buffer and try decoding type flag followed by receipt *buf = *header_buf; let remaining = buf.len(); - let tx_type = TxType::decode(buf)?; + let tx_type = T::decode(buf)?; let this = Self::rlp_decode_inner(buf, tx_type)?; if buf.len() + header.payload_length != remaining { @@ -196,7 +243,7 @@ impl RlpDecodableReceipt for Receipt { } } -impl Encodable2718 for Receipt { +impl Encodable2718 for Receipt { fn encode_2718_len(&self) -> usize { (!self.tx_type.is_legacy() as usize) + self.rlp_header_inner_without_bloom().length_with_payload() @@ -205,24 +252,24 @@ impl Encodable2718 for Receipt { // encode the header fn encode_2718(&self, out: &mut dyn BufMut) { if !self.tx_type.is_legacy() { - out.put_u8(self.tx_type as u8); + out.put_u8(self.tx_type.ty()); } self.rlp_header_inner_without_bloom().encode(out); self.rlp_encode_fields_without_bloom(out); } } -impl Decodable2718 for Receipt { +impl Decodable2718 for Receipt { fn typed_decode(ty: u8, buf: &mut &[u8]) -> Eip2718Result { - Ok(Self::rlp_decode_inner_without_bloom(buf, TxType::try_from(ty)?)?) + Ok(Self::rlp_decode_inner_without_bloom(buf, T::try_from(ty)?)?) } fn fallback_decode(buf: &mut &[u8]) -> Eip2718Result { - Ok(Self::rlp_decode_inner_without_bloom(buf, TxType::Legacy)?) + Ok(Self::rlp_decode_inner_without_bloom(buf, T::try_from(0)?)?) } } -impl Encodable for Receipt { +impl Encodable for Receipt { fn encode(&self, out: &mut dyn BufMut) { self.network_encode(out); } @@ -232,14 +279,18 @@ impl Encodable for Receipt { } } -impl Decodable for Receipt { +impl Decodable for Receipt { fn decode(buf: &mut &[u8]) -> alloy_rlp::Result { Ok(Self::network_decode(buf)?) } } -impl TxReceipt for Receipt { - type Log = Log; +impl TxReceipt for EthereumReceipt +where + T: TxTy, + L: Send + Sync + Clone + Debug + Eq + AsRef, +{ + type Log = L; fn status_or_post_state(&self) -> Eip658Value { self.success.into() @@ -250,40 +301,92 @@ impl TxReceipt for Receipt { } fn bloom(&self) -> Bloom { - alloy_primitives::logs_bloom(self.logs()) + alloy_primitives::logs_bloom(self.logs.iter().map(|l| l.as_ref())) } fn cumulative_gas_used(&self) -> u64 { self.cumulative_gas_used } - fn logs(&self) -> &[Log] { + fn logs(&self) -> &[L] { &self.logs } + + fn into_logs(self) -> Vec { + self.logs + } } -impl Typed2718 for Receipt { +impl Typed2718 for Receipt { fn ty(&self) -> u8 { - self.tx_type as u8 + self.tx_type.ty() + } +} + +impl IsTyped2718 for Receipt { + fn is_type(type_id: u8) -> bool { + ::is_type(type_id) } } -impl InMemorySize for Receipt { +impl InMemorySize for Receipt { fn size(&self) -> usize { self.tx_type.size() + core::mem::size_of::() + core::mem::size_of::() + - self.logs.capacity() * core::mem::size_of::() + self.logs.iter().map(|log| log.size()).sum::() } } -impl reth_primitives_traits::Receipt for Receipt {} +impl From> for Receipt +where + T: Into, +{ + fn from(value: ReceiptEnvelope) -> Self { + let value = value.into_primitives_receipt(); + Self { + tx_type: value.tx_type(), + success: value.is_success(), + cumulative_gas_used: value.cumulative_gas_used(), + logs: value.into_logs(), + } + } +} + +impl From> for alloy_consensus::Receipt { + fn from(value: EthereumReceipt) -> Self { + Self { + status: value.success.into(), + cumulative_gas_used: value.cumulative_gas_used, + logs: value.logs, + } + } +} + +impl From> for ReceiptEnvelope +where + L: Send + Sync + Clone + Debug + Eq + AsRef, +{ + fn from(value: EthereumReceipt) -> Self { + let tx_type = value.tx_type; + let receipt = value.into_with_bloom().map_receipt(Into::into); + match tx_type { + TxType::Legacy => Self::Legacy(receipt), + TxType::Eip2930 => Self::Eip2930(receipt), + TxType::Eip1559 => Self::Eip1559(receipt), + TxType::Eip4844 => Self::Eip4844(receipt), + TxType::Eip7702 => Self::Eip7702(receipt), + } + } +} #[cfg(all(feature = "serde", feature = "serde-bincode-compat"))] pub(super) mod serde_bincode_compat { use alloc::{borrow::Cow, vec::Vec}; use alloy_consensus::TxType; + use alloy_eips::eip2718::Eip2718Error; use alloy_primitives::{Log, U8}; + use core::fmt::Debug; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use serde_with::{DeserializeAs, SerializeAs}; @@ -291,6 +394,7 @@ pub(super) mod serde_bincode_compat { /// /// Intended to use with the [`serde_with::serde_as`] macro in the following way: /// ```rust + /// use alloy_consensus::TxType; /// use reth_ethereum_primitives::{serde_bincode_compat, Receipt}; /// use serde::{de::DeserializeOwned, Deserialize, Serialize}; /// use serde_with::serde_as; @@ -299,14 +403,15 @@ pub(super) mod serde_bincode_compat { /// #[derive(Serialize, Deserialize)] /// struct Data { /// #[serde_as(as = "serde_bincode_compat::Receipt<'_>")] - /// receipt: Receipt, + /// receipt: Receipt, /// } /// ``` #[derive(Debug, Serialize, Deserialize)] - pub struct Receipt<'a> { + #[serde(bound(deserialize = "T: TryFrom"))] + pub struct Receipt<'a, T = TxType> { /// Receipt type. #[serde(deserialize_with = "deserde_txtype")] - pub tx_type: TxType, + pub tx_type: T, /// If transaction is executed successfully. /// /// This is the `statusCode` @@ -318,16 +423,16 @@ pub(super) mod serde_bincode_compat { } /// Ensures that txtype is deserialized symmetrically as U8 - fn deserde_txtype<'de, D>(deserializer: D) -> Result + fn deserde_txtype<'de, D, T>(deserializer: D) -> Result where D: Deserializer<'de>, + T: TryFrom, { - let value = U8::deserialize(deserializer)?; - value.to::().try_into().map_err(serde::de::Error::custom) + U8::deserialize(deserializer)?.to::().try_into().map_err(serde::de::Error::custom) } - impl<'a> From<&'a super::Receipt> for Receipt<'a> { - fn from(value: &'a super::Receipt) -> Self { + impl<'a, T: Copy> From<&'a super::Receipt> for Receipt<'a, T> { + fn from(value: &'a super::Receipt) -> Self { Self { tx_type: value.tx_type, success: value.success, @@ -337,8 +442,8 @@ pub(super) mod serde_bincode_compat { } } - impl<'a> From> for super::Receipt { - fn from(value: Receipt<'a>) -> Self { + impl<'a, T> From> for super::Receipt { + fn from(value: Receipt<'a, T>) -> Self { Self { tx_type: value.tx_type, success: value.success, @@ -348,8 +453,8 @@ pub(super) mod serde_bincode_compat { } } - impl SerializeAs for Receipt<'_> { - fn serialize_as(source: &super::Receipt, serializer: S) -> Result + impl SerializeAs> for Receipt<'_, T> { + fn serialize_as(source: &super::Receipt, serializer: S) -> Result where S: Serializer, { @@ -357,17 +462,22 @@ pub(super) mod serde_bincode_compat { } } - impl<'de> DeserializeAs<'de, super::Receipt> for Receipt<'de> { - fn deserialize_as(deserializer: D) -> Result + impl<'de, T: TryFrom> DeserializeAs<'de, super::Receipt> + for Receipt<'de, T> + { + fn deserialize_as(deserializer: D) -> Result, D::Error> where D: Deserializer<'de>, { - Receipt::<'_>::deserialize(deserializer).map(Into::into) + Receipt::<'_, T>::deserialize(deserializer).map(Into::into) } } - impl reth_primitives_traits::serde_bincode_compat::SerdeBincodeCompat for super::Receipt { - type BincodeRepr<'a> = Receipt<'a>; + impl reth_primitives_traits::serde_bincode_compat::SerdeBincodeCompat for super::Receipt + where + T: Copy + Serialize + TryFrom + Debug + 'static, + { + type BincodeRepr<'a> = Receipt<'a, T>; fn as_repr(&self) -> Self::BincodeRepr<'_> { self.into() @@ -381,6 +491,7 @@ pub(super) mod serde_bincode_compat { #[cfg(test)] mod tests { use crate::{receipt::serde_bincode_compat, Receipt}; + use alloy_consensus::TxType; use arbitrary::Arbitrary; use rand::Rng; use serde_with::serde_as; @@ -391,14 +502,14 @@ pub(super) mod serde_bincode_compat { #[derive(Debug, PartialEq, Eq)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] struct Data { - #[serde_as(as = "serde_bincode_compat::Receipt<'_>")] - reseipt: Receipt, + #[serde_as(as = "serde_bincode_compat::Receipt<'_, TxType>")] + receipt: Receipt, } let mut bytes = [0u8; 1024]; rand::rng().fill(bytes.as_mut_slice()); let data = Data { - reseipt: Receipt::arbitrary(&mut arbitrary::Unstructured::new(&bytes)).unwrap(), + receipt: Receipt::arbitrary(&mut arbitrary::Unstructured::new(&bytes)).unwrap(), }; let encoded = bincode::serialize(&data).unwrap(); let decoded: Data = bincode::deserialize(&encoded).unwrap(); @@ -407,6 +518,124 @@ pub(super) mod serde_bincode_compat { } } +#[cfg(feature = "reth-codec")] +mod compact { + use super::*; + use reth_codecs::{ + Compact, + __private::{modular_bitfield::prelude::*, Buf}, + }; + + impl Receipt { + #[doc = "Used bytes by [`ReceiptFlags`]"] + pub const fn bitflag_encoded_bytes() -> usize { + 1u8 as usize + } + #[doc = "Unused bits for new fields by [`ReceiptFlags`]"] + pub const fn bitflag_unused_bits() -> usize { + 0u8 as usize + } + } + + #[allow(non_snake_case, unused_parens)] + mod flags { + use super::*; + + #[doc = "Fieldset that facilitates compacting the parent type. Used bytes: 1 | Unused bits: 0"] + #[bitfield] + #[derive(Clone, Copy, Debug, Default)] + pub struct ReceiptFlags { + pub tx_type_len: B2, + pub success_len: B1, + pub cumulative_gas_used_len: B4, + pub __zstd: B1, + } + + impl ReceiptFlags { + #[doc = r" Deserializes this fieldset and returns it, alongside the original slice in an advanced position."] + pub fn from(mut buf: &[u8]) -> (Self, &[u8]) { + (Self::from_bytes([buf.get_u8()]), buf) + } + } + } + + pub use flags::ReceiptFlags; + + impl Compact for Receipt { + fn to_compact(&self, buf: &mut B) -> usize + where + B: reth_codecs::__private::bytes::BufMut + AsMut<[u8]>, + { + let mut flags = ReceiptFlags::default(); + let mut total_length = 0; + let mut buffer = reth_codecs::__private::bytes::BytesMut::new(); + + let tx_type_len = self.tx_type.to_compact(&mut buffer); + flags.set_tx_type_len(tx_type_len as u8); + let success_len = self.success.to_compact(&mut buffer); + flags.set_success_len(success_len as u8); + let cumulative_gas_used_len = self.cumulative_gas_used.to_compact(&mut buffer); + flags.set_cumulative_gas_used_len(cumulative_gas_used_len as u8); + self.logs.to_compact(&mut buffer); + + let zstd = buffer.len() > 7; + if zstd { + flags.set___zstd(1); + } + + let flags = flags.into_bytes(); + total_length += flags.len() + buffer.len(); + buf.put_slice(&flags); + if zstd { + reth_zstd_compressors::RECEIPT_COMPRESSOR.with(|compressor| { + let compressed = + compressor.borrow_mut().compress(&buffer).expect("Failed to compress."); + buf.put(compressed.as_slice()); + }); + } else { + buf.put(buffer); + } + total_length + } + + fn from_compact(buf: &[u8], _len: usize) -> (Self, &[u8]) { + let (flags, mut buf) = ReceiptFlags::from(buf); + if flags.__zstd() != 0 { + reth_zstd_compressors::RECEIPT_DECOMPRESSOR.with(|decompressor| { + let decompressor = &mut decompressor.borrow_mut(); + let decompressed = decompressor.decompress(buf); + let original_buf = buf; + let mut buf: &[u8] = decompressed; + let (tx_type, new_buf) = T::from_compact(buf, flags.tx_type_len() as usize); + buf = new_buf; + let (success, new_buf) = bool::from_compact(buf, flags.success_len() as usize); + buf = new_buf; + let (cumulative_gas_used, new_buf) = + u64::from_compact(buf, flags.cumulative_gas_used_len() as usize); + buf = new_buf; + let (logs, _) = Vec::from_compact(buf, buf.len()); + (Self { tx_type, success, cumulative_gas_used, logs }, original_buf) + }) + } else { + let (tx_type, new_buf) = T::from_compact(buf, flags.tx_type_len() as usize); + buf = new_buf; + let (success, new_buf) = bool::from_compact(buf, flags.success_len() as usize); + buf = new_buf; + let (cumulative_gas_used, new_buf) = + u64::from_compact(buf, flags.cumulative_gas_used_len() as usize); + buf = new_buf; + let (logs, new_buf) = Vec::from_compact(buf, buf.len()); + buf = new_buf; + let obj = Self { tx_type, success, cumulative_gas_used, logs }; + (obj, buf) + } + } + } +} + +#[cfg(feature = "reth-codec")] +pub use compact::*; + #[cfg(test)] mod tests { use super::*; @@ -427,8 +656,9 @@ mod tests { pub(crate) type Block = alloy_consensus::Block; #[test] + #[cfg(feature = "reth-codec")] fn test_decode_receipt() { - reth_codecs::test_utils::test_decode::(&hex!( + reth_codecs::test_utils::test_decode::>(&hex!( "c428b52ffd23fc42696156b10200f034792b6a94c3850215c2fef7aea361a0c31b79d9a32652eefc0d4e2e730036061cff7344b6fc6132b50cda0ed810a991ae58ef013150c12b2522533cb3b3a8b19b7786a8b5ff1d3cdc84225e22b02def168c8858df" )); } @@ -520,7 +750,7 @@ mod tests { let mut data = vec![]; receipt.to_compact(&mut data); - let (decoded, _) = Receipt::from_compact(&data[..], data.len()); + let (decoded, _) = Receipt::::from_compact(&data[..], data.len()); assert_eq!(decoded, receipt); } @@ -627,4 +857,20 @@ mod tests { b256!("0xfe70ae4a136d98944951b2123859698d59ad251a381abc9960fa81cae3d0d4a0") ); } + + // Ensures that reth and alloy receipts encode to the same JSON + #[test] + #[cfg(feature = "rpc")] + fn test_receipt_serde() { + let input = r#"{"status":"0x1","cumulativeGasUsed":"0x175cc0e","logs":[{"address":"0xa18b9ca2a78660d44ab38ae72e72b18792ffe413","topics":["0x8c5be1e5ebec7d5bd14f71427d1e84f3dd0314c0f7b2291e5b200ac8c7c3b925","0x000000000000000000000000e7e7d8006cbff47bc6ac2dabf592c98e97502708","0x0000000000000000000000007a250d5630b4cf539739df2c5dacb4c659f2488d"],"data":"0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff","blockHash":"0xbf9e6a368a399f996a0f0b27cab4191c028c3c99f5f76ea08a5b70b961475fcb","blockNumber":"0x164b59f","blockTimestamp":"0x68c9a713","transactionHash":"0x533aa9e57865675bb94f41aa2895c0ac81eee69686c77af16149c301e19805f1","transactionIndex":"0x14d","logIndex":"0x238","removed":false}],"logsBloom":"0x00000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000400000040000000000000004000000000000000000000000000000000000000000000020000000000000000000000000080000000000000000000000000200000020000000000000000000000000000000000000000000000000000000000000020000010000000000000000000000000000000000000000000000000000000000000","type":"0x2","transactionHash":"0x533aa9e57865675bb94f41aa2895c0ac81eee69686c77af16149c301e19805f1","transactionIndex":"0x14d","blockHash":"0xbf9e6a368a399f996a0f0b27cab4191c028c3c99f5f76ea08a5b70b961475fcb","blockNumber":"0x164b59f","gasUsed":"0xb607","effectiveGasPrice":"0x4a3ee768","from":"0xe7e7d8006cbff47bc6ac2dabf592c98e97502708","to":"0xa18b9ca2a78660d44ab38ae72e72b18792ffe413","contractAddress":null}"#; + let receipt: RpcReceipt = serde_json::from_str(input).unwrap(); + let envelope: ReceiptEnvelope = + serde_json::from_str(input).unwrap(); + + assert_eq!(envelope, receipt.clone().into()); + + let json_envelope = serde_json::to_value(&envelope).unwrap(); + let json_receipt = serde_json::to_value(receipt.into_with_bloom()).unwrap(); + assert_eq!(json_envelope, json_receipt); + } } diff --git a/crates/ethereum/primitives/src/transaction.rs b/crates/ethereum/primitives/src/transaction.rs index dfd3a0085b4..f2ec4ad9cdf 100644 --- a/crates/ethereum/primitives/src/transaction.rs +++ b/crates/ethereum/primitives/src/transaction.rs @@ -3,7 +3,7 @@ use alloc::vec::Vec; use alloy_consensus::{ - transaction::{RlpEcdsaDecodableTx, RlpEcdsaEncodableTx}, + transaction::{RlpEcdsaDecodableTx, RlpEcdsaEncodableTx, SignerRecoverable, TxHashRef}, EthereumTxEnvelope, SignableTransaction, Signed, TxEip1559, TxEip2930, TxEip4844, TxEip7702, TxLegacy, TxType, Typed2718, }; @@ -640,26 +640,32 @@ impl reth_codecs::Compact for TransactionSigned { } } -impl SignedTransaction for TransactionSigned { - fn tx_hash(&self) -> &TxHash { - self.hash.get_or_init(|| self.recalculate_hash()) - } - +impl SignerRecoverable for TransactionSigned { fn recover_signer(&self) -> Result { let signature_hash = self.signature_hash(); recover_signer(&self.signature, signature_hash) } - fn recover_signer_unchecked_with_buf( - &self, - buf: &mut Vec, - ) -> Result { + fn recover_signer_unchecked(&self) -> Result { + let signature_hash = self.signature_hash(); + recover_signer_unchecked(&self.signature, signature_hash) + } + + fn recover_unchecked_with_buf(&self, buf: &mut Vec) -> Result { self.encode_for_signing(buf); let signature_hash = keccak256(buf); recover_signer_unchecked(&self.signature, signature_hash) } } +impl TxHashRef for TransactionSigned { + fn tx_hash(&self) -> &TxHash { + self.hash.get_or_init(|| self.recalculate_hash()) + } +} + +impl SignedTransaction for TransactionSigned {} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/ethereum/reth/Cargo.toml b/crates/ethereum/reth/Cargo.toml index 593bcd6cec1..0d57abf6f20 100644 --- a/crates/ethereum/reth/Cargo.toml +++ b/crates/ethereum/reth/Cargo.toml @@ -19,6 +19,7 @@ reth-network-api = { workspace = true, optional = true } reth-eth-wire = { workspace = true, optional = true } reth-provider = { workspace = true, optional = true } reth-db = { workspace = true, optional = true, features = ["mdbx"] } +reth-codecs = { workspace = true, optional = true } reth-storage-api = { workspace = true, optional = true } reth-node-api = { workspace = true, optional = true } reth-node-core = { workspace = true, optional = true } @@ -33,6 +34,11 @@ reth-rpc-eth-types = { workspace = true, optional = true } reth-rpc-builder = { workspace = true, optional = true } reth-exex = { workspace = true, optional = true } reth-trie = { workspace = true, optional = true } +reth-trie-db = { workspace = true, optional = true } +reth-node-builder = { workspace = true, optional = true } +reth-tasks = { workspace = true, optional = true } +reth-cli-util = { workspace = true, optional = true } +reth-engine-local = { workspace = true, optional = true } # reth-ethereum reth-ethereum-primitives.workspace = true @@ -43,13 +49,14 @@ reth-node-ethereum = { workspace = true, optional = true } # alloy alloy-rpc-types-eth = { workspace = true, optional = true } +alloy-rpc-types-engine = { workspace = true, optional = true } [features] default = ["std"] std = [ "reth-chainspec/std", "reth-ethereum-primitives/std", - "reth-ethereum-consensus/std", + "reth-ethereum-consensus?/std", "reth-primitives-traits/std", "reth-consensus?/std", "reth-consensus-common?/std", @@ -58,6 +65,7 @@ std = [ "reth-evm?/std", "reth-evm-ethereum?/std", "reth-revm?/std", + "alloy-rpc-types-engine?/std", ] arbitrary = [ "std", @@ -68,6 +76,8 @@ arbitrary = [ "alloy-rpc-types-eth?/arbitrary", "reth-transaction-pool?/arbitrary", "reth-eth-wire?/arbitrary", + "alloy-rpc-types-engine?/arbitrary", + "reth-codecs?/arbitrary", ] test-utils = [ @@ -84,6 +94,9 @@ test-utils = [ "reth-trie?/test-utils", "reth-transaction-pool?/test-utils", "reth-evm-ethereum?/test-utils", + "reth-node-builder?/test-utils", + "reth-trie-db?/test-utils", + "reth-codecs?/test-utils", ] full = [ @@ -98,7 +111,7 @@ full = [ "network", ] -cli = ["dep:reth-ethereum-cli"] +cli = ["dep:reth-ethereum-cli", "dep:reth-cli-util"] consensus = [ "dep:reth-consensus", "dep:reth-consensus-common", @@ -111,21 +124,35 @@ node = [ "provider", "consensus", "evm", + "network", "node-api", "dep:reth-node-ethereum", + "dep:reth-node-builder", + "dep:reth-engine-local", "rpc", - "trie", + "trie-db", + "pool", ] pool = ["dep:reth-transaction-pool"] rpc = [ + "tasks", "dep:reth-rpc", "dep:reth-rpc-builder", "dep:reth-rpc-api", "dep:reth-rpc-eth-types", "dep:alloy-rpc-types-eth", + "dep:alloy-rpc-types-engine", +] +tasks = ["dep:reth-tasks"] +js-tracer = [ + "rpc", + "reth-rpc/js-tracer", + "reth-node-builder?/js-tracer", + "reth-node-ethereum?/js-tracer", + "reth-rpc-eth-types?/js-tracer", ] -js-tracer = ["rpc", "reth-rpc/js-tracer"] -network = ["dep:reth-network", "dep:reth-network-api", "dep:reth-eth-wire"] -provider = ["storage-api", "dep:reth-provider", "dep:reth-db"] +network = ["dep:reth-network", "tasks", "dep:reth-network-api", "dep:reth-eth-wire"] +provider = ["storage-api", "tasks", "dep:reth-provider", "dep:reth-db", "dep:reth-codecs"] storage-api = ["dep:reth-storage-api"] trie = ["dep:reth-trie"] +trie-db = ["trie", "dep:reth-trie-db"] diff --git a/crates/ethereum/reth/src/lib.rs b/crates/ethereum/reth/src/lib.rs index 26b0f34d277..1a1962ba9c6 100644 --- a/crates/ethereum/reth/src/lib.rs +++ b/crates/ethereum/reth/src/lib.rs @@ -6,7 +6,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(feature = "std"), no_std)] /// Re-exported ethereum types @@ -21,7 +21,12 @@ pub mod primitives { /// Re-exported cli types #[cfg(feature = "cli")] -pub use reth_ethereum_cli as cli; +pub mod cli { + #[doc(inline)] + pub use reth_cli_util::*; + #[doc(inline)] + pub use reth_ethereum_cli::*; +} /// Re-exported pool types #[cfg(feature = "pool")] @@ -59,6 +64,12 @@ pub mod evm { #[cfg(feature = "exex")] pub use reth_exex as exex; +/// Re-exported from `tasks`. +#[cfg(feature = "tasks")] +pub mod tasks { + pub use reth_tasks::*; +} + /// Re-exported reth network types #[cfg(feature = "network")] pub mod network { @@ -80,6 +91,10 @@ pub mod provider { pub use reth_db as db; } +/// Re-exported codec crate +#[cfg(feature = "provider")] +pub use reth_codecs as codec; + /// Re-exported reth storage api types #[cfg(feature = "storage-api")] pub mod storage { @@ -92,17 +107,32 @@ pub mod storage { pub mod node { #[doc(inline)] pub use reth_node_api as api; + #[cfg(feature = "node")] + pub use reth_node_builder as builder; #[doc(inline)] pub use reth_node_core as core; #[cfg(feature = "node")] pub use reth_node_ethereum::*; } +/// Re-exported ethereum engine types +#[cfg(feature = "node")] +pub mod engine { + #[doc(inline)] + pub use reth_engine_local as local; + #[doc(inline)] + pub use reth_node_ethereum::engine::*; +} + /// Re-exported reth trie types #[cfg(feature = "trie")] pub mod trie { #[doc(inline)] pub use reth_trie::*; + + #[cfg(feature = "trie-db")] + #[doc(inline)] + pub use reth_trie_db::*; } /// Re-exported rpc types @@ -117,10 +147,19 @@ pub mod rpc { pub use reth_rpc_builder as builder; /// Re-exported eth types + #[allow(ambiguous_glob_reexports)] pub mod eth { #[doc(inline)] pub use alloy_rpc_types_eth as primitives; #[doc(inline)] pub use reth_rpc_eth_types::*; + + pub use reth_rpc::eth::*; + } + + /// Re-exported types + pub mod types { + #[doc(inline)] + pub use alloy_rpc_types_engine as engine; } } diff --git a/crates/etl/src/lib.rs b/crates/etl/src/lib.rs index 46d41d704d0..d32905ea043 100644 --- a/crates/etl/src/lib.rs +++ b/crates/etl/src/lib.rs @@ -12,7 +12,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] use std::{ cmp::Reverse, diff --git a/crates/evm/evm/Cargo.toml b/crates/evm/evm/Cargo.toml index d3389b39851..4bc8ef06dbb 100644 --- a/crates/evm/evm/Cargo.toml +++ b/crates/evm/evm/Cargo.toml @@ -21,7 +21,6 @@ reth-storage-errors.workspace = true reth-trie-common.workspace = true revm.workspace = true -op-revm = { workspace = true, optional = true } # alloy alloy-primitives.workspace = true @@ -37,8 +36,6 @@ metrics = { workspace = true, optional = true } [dev-dependencies] reth-ethereum-primitives.workspace = true reth-ethereum-forks.workspace = true -alloy-consensus.workspace = true -metrics-util = { workspace = true, features = ["debugging"] } [features] default = ["std"] @@ -50,7 +47,6 @@ std = [ "revm/std", "reth-ethereum-forks/std", "alloy-evm/std", - "op-revm?/std", "reth-execution-errors/std", "reth-execution-types/std", "reth-storage-errors/std", @@ -66,4 +62,4 @@ test-utils = [ "reth-trie-common/test-utils", "reth-ethereum-primitives/test-utils", ] -op = ["op-revm", "alloy-evm/op", "reth-primitives-traits/op"] +op = ["alloy-evm/op", "reth-primitives-traits/op"] diff --git a/crates/evm/evm/src/aliases.rs b/crates/evm/evm/src/aliases.rs index 6bb1ab1c35a..7758f0aea17 100644 --- a/crates/evm/evm/src/aliases.rs +++ b/crates/evm/evm/src/aliases.rs @@ -11,6 +11,9 @@ pub type EvmFactoryFor = /// Helper to access [`EvmFactory::Spec`] for a given [`ConfigureEvm`]. pub type SpecFor = as EvmFactory>::Spec; +/// Helper to access [`EvmFactory::BlockEnv`] for a given [`ConfigureEvm`]. +pub type BlockEnvFor = as EvmFactory>::BlockEnv; + /// Helper to access [`EvmFactory::Evm`] for a given [`ConfigureEvm`]. pub type EvmFor = as EvmFactory>::Evm; @@ -31,7 +34,7 @@ pub type ExecutionCtxFor<'a, Evm> = <::BlockExecutorFactory as BlockExecutorFactory>::ExecutionCtx<'a>; /// Type alias for [`EvmEnv`] for a given [`ConfigureEvm`]. -pub type EvmEnvFor = EvmEnv>; +pub type EvmEnvFor = EvmEnv, BlockEnvFor>; /// Helper trait to bound [`Inspector`] for a [`ConfigureEvm`]. pub trait InspectorFor: Inspector> {} diff --git a/crates/evm/evm/src/engine.rs b/crates/evm/evm/src/engine.rs new file mode 100644 index 00000000000..e8316426079 --- /dev/null +++ b/crates/evm/evm/src/engine.rs @@ -0,0 +1,39 @@ +use crate::{execute::ExecutableTxFor, ConfigureEvm, EvmEnvFor, ExecutionCtxFor}; + +/// [`ConfigureEvm`] extension providing methods for executing payloads. +pub trait ConfigureEngineEvm: ConfigureEvm { + /// Returns an [`crate::EvmEnv`] for the given payload. + fn evm_env_for_payload(&self, payload: &ExecutionData) -> Result, Self::Error>; + + /// Returns an [`ExecutionCtxFor`] for the given payload. + fn context_for_payload<'a>( + &self, + payload: &'a ExecutionData, + ) -> Result, Self::Error>; + + /// Returns an [`ExecutableTxIterator`] for the given payload. + fn tx_iterator_for_payload( + &self, + payload: &ExecutionData, + ) -> Result, Self::Error>; +} + +/// Iterator over executable transactions. +pub trait ExecutableTxIterator: + Iterator> + Send + 'static +{ + /// The executable transaction type iterator yields. + type Tx: ExecutableTxFor + Clone + Send + Sync + 'static; + /// Errors that may occur while recovering or decoding transactions. + type Error: core::error::Error + Send + Sync + 'static; +} + +impl ExecutableTxIterator for T +where + Tx: ExecutableTxFor + Clone + Send + Sync + 'static, + Err: core::error::Error + Send + Sync + 'static, + T: Iterator> + Send + 'static, +{ + type Tx = Tx; + type Error = Err; +} diff --git a/crates/evm/evm/src/execute.rs b/crates/evm/evm/src/execute.rs index 4b26c7067cc..fca8f6241d5 100644 --- a/crates/evm/evm/src/execute.rs +++ b/crates/evm/evm/src/execute.rs @@ -1,13 +1,15 @@ //! Traits for execution. -use crate::{ConfigureEvm, Database, OnStateHook}; -use alloc::{boxed::Box, vec::Vec}; +use crate::{ConfigureEvm, Database, OnStateHook, TxEnvFor}; +use alloc::{boxed::Box, sync::Arc, vec::Vec}; use alloy_consensus::{BlockHeader, Header}; use alloy_eips::eip2718::WithEncoded; pub use alloy_evm::block::{BlockExecutor, BlockExecutorFactory}; -use alloy_evm::{block::ExecutableTx, Evm, EvmEnv, EvmFactory}; -use alloy_primitives::B256; -use core::fmt::Debug; +use alloy_evm::{ + block::{CommitChanges, ExecutableTx}, + Evm, EvmEnv, EvmFactory, RecoveredTx, ToTxEnv, +}; +use alloy_primitives::{Address, B256}; pub use reth_execution_errors::{ BlockExecutionError, BlockValidationError, InternalBlockExecutionError, }; @@ -105,6 +107,23 @@ pub trait Executor: Sized { Ok(BlockExecutionOutput { state: state.take_bundle(), result }) } + /// Executes the EVM with the given input and accepts a state closure that is always invoked + /// with the EVM state after execution, even after failure. + fn execute_with_state_closure_always( + mut self, + block: &RecoveredBlock<::Block>, + mut f: F, + ) -> Result::Receipt>, Self::Error> + where + F: FnMut(&State), + { + let result = self.execute_one(block); + let mut state = self.into_state(); + f(&state); + + Ok(BlockExecutionOutput { state: state.take_bundle(), result: result? }) + } + /// Executes the EVM with the given input and accepts a state hook closure that is invoked with /// the EVM state after execution. fn execute_with_state_hook( @@ -130,6 +149,11 @@ pub trait Executor: Sized { } /// Helper type for the output of executing a block. +/// +/// Deprecated: this type is unused within reth and will be removed in the next +/// major release. Use `reth_execution_types::BlockExecutionResult` or +/// `reth_execution_types::BlockExecutionOutput`. +#[deprecated(note = "Use reth_execution_types::BlockExecutionResult or BlockExecutionOutput")] #[derive(Debug, Clone)] pub struct ExecuteOutput { /// Receipts obtained after executing a block. @@ -139,13 +163,48 @@ pub struct ExecuteOutput { } /// Input for block building. Consumed by [`BlockAssembler`]. +/// +/// This struct contains all the data needed by the [`BlockAssembler`] to create +/// a complete block after transaction execution. +/// +/// # Fields Overview +/// +/// - `evm_env`: The EVM configuration used during execution (spec ID, block env, etc.) +/// - `execution_ctx`: Additional context like withdrawals and ommers +/// - `parent`: The parent block header this block builds on +/// - `transactions`: All transactions that were successfully executed +/// - `output`: Execution results including receipts and gas used +/// - `bundle_state`: Accumulated state changes from all transactions +/// - `state_provider`: Access to the current state for additional lookups +/// - `state_root`: The calculated state root after all changes +/// +/// # Usage +/// +/// This is typically created internally by [`BlockBuilder::finish`] after all +/// transactions have been executed: +/// +/// ```rust,ignore +/// let input = BlockAssemblerInput { +/// evm_env: builder.evm_env(), +/// execution_ctx: builder.context(), +/// parent: &parent_header, +/// transactions: executed_transactions, +/// output: &execution_result, +/// bundle_state: &state_changes, +/// state_provider: &state, +/// state_root: calculated_root, +/// }; +/// +/// let block = assembler.assemble_block(input)?; +/// ``` #[derive(derive_more::Debug)] #[non_exhaustive] pub struct BlockAssemblerInput<'a, 'b, F: BlockExecutorFactory, H = Header> { /// Configuration of EVM used when executing the block. /// /// Contains context relevant to EVM such as [`revm::context::BlockEnv`]. - pub evm_env: EvmEnv<::Spec>, + pub evm_env: + EvmEnv<::Spec, ::BlockEnv>, /// [`BlockExecutorFactory::ExecutionCtx`] used to execute the block. pub execution_ctx: F::ExecutionCtx<'a>, /// Parent block header. @@ -163,7 +222,77 @@ pub struct BlockAssemblerInput<'a, 'b, F: BlockExecutorFactory, H = Header> { pub state_root: B256, } -/// A type that knows how to assemble a block. +impl<'a, 'b, F: BlockExecutorFactory, H> BlockAssemblerInput<'a, 'b, F, H> { + /// Creates a new [`BlockAssemblerInput`]. + #[expect(clippy::too_many_arguments)] + pub fn new( + evm_env: EvmEnv< + ::Spec, + ::BlockEnv, + >, + execution_ctx: F::ExecutionCtx<'a>, + parent: &'a SealedHeader, + transactions: Vec, + output: &'b BlockExecutionResult, + bundle_state: &'a BundleState, + state_provider: &'b dyn StateProvider, + state_root: B256, + ) -> Self { + Self { + evm_env, + execution_ctx, + parent, + transactions, + output, + bundle_state, + state_provider, + state_root, + } + } +} + +/// A type that knows how to assemble a block from execution results. +/// +/// The [`BlockAssembler`] is the final step in block production. After transactions +/// have been executed by the [`BlockExecutor`], the assembler takes all the execution +/// outputs and creates a properly formatted block. +/// +/// # Responsibilities +/// +/// The assembler is responsible for: +/// - Setting the correct block header fields (gas used, receipts root, logs bloom, etc.) +/// - Including the executed transactions in the correct order +/// - Setting the state root from the post-execution state +/// - Applying any chain-specific rules or adjustments +/// +/// # Example Flow +/// +/// ```rust,ignore +/// // 1. Execute transactions and get results +/// let execution_result = block_executor.finish()?; +/// +/// // 2. Calculate state root from changes +/// let state_root = state_provider.state_root(&bundle_state)?; +/// +/// // 3. Assemble the final block +/// let block = assembler.assemble_block(BlockAssemblerInput { +/// evm_env, // Environment used during execution +/// execution_ctx, // Context like withdrawals, ommers +/// parent, // Parent block header +/// transactions, // Executed transactions +/// output, // Execution results (receipts, gas) +/// bundle_state, // All state changes +/// state_provider, // For additional lookups if needed +/// state_root, // Computed state root +/// })?; +/// ``` +/// +/// # Relationship with Block Building +/// +/// The assembler works together with: +/// - `NextBlockEnvAttributes`: Provides the configuration for the new block +/// - [`BlockExecutor`]: Executes transactions and produces results +/// - [`BlockBuilder`]: Orchestrates the entire process and calls the assembler #[auto_impl::auto_impl(&, Arc)] pub trait BlockAssembler { /// The block type produced by the assembler. @@ -207,19 +336,35 @@ pub trait BlockBuilder { /// Invokes [`BlockExecutor::apply_pre_execution_changes`]. fn apply_pre_execution_changes(&mut self) -> Result<(), BlockExecutionError>; + /// Invokes [`BlockExecutor::execute_transaction_with_commit_condition`] and saves the + /// transaction in internal state only if the transaction was committed. + fn execute_transaction_with_commit_condition( + &mut self, + tx: impl ExecutorTx, + f: impl FnOnce( + &ExecutionResult<<::Evm as Evm>::HaltReason>, + ) -> CommitChanges, + ) -> Result, BlockExecutionError>; + /// Invokes [`BlockExecutor::execute_transaction_with_result_closure`] and saves the /// transaction in internal state. fn execute_transaction_with_result_closure( &mut self, tx: impl ExecutorTx, f: impl FnOnce(&ExecutionResult<<::Evm as Evm>::HaltReason>), - ) -> Result; + ) -> Result { + self.execute_transaction_with_commit_condition(tx, |res| { + f(res); + CommitChanges::Yes + }) + .map(Option::unwrap_or_default) + } /// Invokes [`BlockExecutor::execute_transaction`] and saves the transaction in /// internal state. fn execute_transaction( &mut self, - tx: Recovered>, + tx: impl ExecutorTx, ) -> Result { self.execute_transaction_with_result_closure(tx, |_| ()) } @@ -250,15 +395,22 @@ pub trait BlockBuilder { fn into_executor(self) -> Self::Executor; } -pub(crate) struct BasicBlockBuilder<'a, F, Executor, Builder, N: NodePrimitives> +/// A type that constructs a block from transactions and execution results. +#[derive(Debug)] +pub struct BasicBlockBuilder<'a, F, Executor, Builder, N: NodePrimitives> where F: BlockExecutorFactory, { - pub(crate) executor: Executor, - pub(crate) transactions: Vec>>, - pub(crate) ctx: F::ExecutionCtx<'a>, - pub(crate) parent: &'a SealedHeader>, - pub(crate) assembler: Builder, + /// The block executor used to execute transactions. + pub executor: Executor, + /// The transactions executed in this block. + pub transactions: Vec>>, + /// The parent block execution context. + pub ctx: F::ExecutionCtx<'a>, + /// The sealed parent block header. + pub parent: &'a SealedHeader>, + /// The assembler used to build the block. + pub assembler: Builder, } /// Conversions for executable transactions. @@ -292,6 +444,23 @@ impl ExecutorTx for Recovered ExecutorTx + for WithTxEnv<<::Evm as Evm>::Tx, T> +where + T: ExecutorTx + Clone, + Executor: BlockExecutor, + <::Evm as Evm>::Tx: Clone, + Self: RecoveredTx, +{ + fn as_executable(&self) -> impl ExecutableTx { + self + } + + fn into_recovered(self) -> Recovered { + Arc::unwrap_or_clone(self.tx).into_recovered() + } +} + impl<'a, F, DB, Executor, Builder, N> BlockBuilder for BasicBlockBuilder<'a, F, Executor, Builder, N> where @@ -300,6 +469,7 @@ where Evm: Evm< Spec = ::Spec, HaltReason = ::HaltReason, + BlockEnv = ::BlockEnv, DB = &'a mut State, >, Transaction = N::SignedTx, @@ -316,15 +486,21 @@ where self.executor.apply_pre_execution_changes() } - fn execute_transaction_with_result_closure( + fn execute_transaction_with_commit_condition( &mut self, tx: impl ExecutorTx, - f: impl FnOnce(&ExecutionResult<::HaltReason>), - ) -> Result { - let gas_used = - self.executor.execute_transaction_with_result_closure(tx.as_executable(), f)?; - self.transactions.push(tx.into_recovered()); - Ok(gas_used) + f: impl FnOnce( + &ExecutionResult<<::Evm as Evm>::HaltReason>, + ) -> CommitChanges, + ) -> Result, BlockExecutionError> { + if let Some(gas_used) = + self.executor.execute_transaction_with_commit_condition(tx.as_executable(), f)? + { + self.transactions.push(tx.into_recovered()); + Ok(Some(gas_used)) + } else { + Ok(None) + } } fn finish( @@ -410,6 +586,7 @@ where let result = self .strategy_factory .executor_for_block(&mut self.db, block) + .map_err(BlockExecutionError::other)? .execute_block(block.transactions_recovered())?; self.db.merge_transitions(BundleRetention::Reverts); @@ -428,6 +605,7 @@ where let result = self .strategy_factory .executor_for_block(&mut self.db, block) + .map_err(BlockExecutionError::other)? .with_state_hook(Some(Box::new(state_hook))) .execute_block(block.transactions_recovered())?; @@ -445,6 +623,43 @@ where } } +/// A helper trait marking a 'static type that can be converted into an [`ExecutableTx`] for block +/// executor. +pub trait ExecutableTxFor: + ToTxEnv> + RecoveredTx> +{ +} + +impl ExecutableTxFor for T where + T: ToTxEnv> + RecoveredTx> +{ +} + +/// A container for a transaction and a transaction environment. +#[derive(Debug, Clone)] +pub struct WithTxEnv { + /// The transaction environment for EVM. + pub tx_env: TxEnv, + /// The recovered transaction. + pub tx: Arc, +} + +impl> RecoveredTx for WithTxEnv { + fn tx(&self) -> &Tx { + self.tx.tx() + } + + fn signer(&self) -> &Address { + self.tx.signer() + } +} + +impl ToTxEnv for WithTxEnv { + fn to_tx_env(&self) -> TxEnv { + self.tx_env.clone() + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/evm/evm/src/lib.rs b/crates/evm/evm/src/lib.rs index b9fcfa3eb62..e2101fd915b 100644 --- a/crates/evm/evm/src/lib.rs +++ b/crates/evm/evm/src/lib.rs @@ -12,18 +12,26 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(feature = "std"), no_std)] extern crate alloc; -use crate::execute::BasicBlockBuilder; +use crate::execute::{BasicBlockBuilder, Executor}; use alloc::vec::Vec; -use alloy_eips::{eip2930::AccessList, eip4895::Withdrawals}; -use alloy_evm::block::{BlockExecutorFactory, BlockExecutorFor}; +use alloy_eips::{ + eip2718::{EIP2930_TX_TYPE_ID, LEGACY_TX_TYPE_ID}, + eip2930::AccessList, + eip4895::Withdrawals, +}; +use alloy_evm::{ + block::{BlockExecutorFactory, BlockExecutorFor}, + precompiles::PrecompilesMap, +}; use alloy_primitives::{Address, B256}; use core::{error::Error, fmt::Debug}; use execute::{BasicBlockExecutor, BlockAssembler, BlockBuilder}; +use reth_execution_errors::BlockExecutionError; use reth_primitives_traits::{ BlockTy, HeaderTy, NodePrimitives, ReceiptTy, SealedBlock, SealedHeader, TxTy, }; @@ -36,6 +44,9 @@ pub mod execute; mod aliases; pub use aliases::*; +mod engine; +pub use engine::{ConfigureEngineEvm, ExecutableTxIterator}; + #[cfg(feature = "metrics")] pub mod metrics; pub mod noop; @@ -53,37 +64,118 @@ pub use alloy_evm::block::state_changes as state_change; /// A complete configuration of EVM for Reth. /// /// This trait encapsulates complete configuration required for transaction execution and block -/// execution/building. +/// execution/building, providing a unified interface for EVM operations. +/// +/// # Architecture Overview /// /// The EVM abstraction consists of the following layers: -/// - [`Evm`] produced by [`EvmFactory`]: The EVM implementation responsilble for executing -/// individual transactions and producing output for them including state changes, logs, gas -/// usage, etc. -/// - [`BlockExecutor`] produced by [`BlockExecutorFactory`]: Executor operates on top of -/// [`Evm`] and is responsible for executing entire blocks. This is different from simply -/// aggregating outputs of transactions execution as it also involves higher level state -/// changes such as receipt building, applying block rewards, system calls, etc. -/// - [`BlockAssembler`]: Encapsulates logic for assembling blocks. It operates on context and -/// output of [`BlockExecutor`], and is required to know how to assemble a next block to -/// include in the chain. -/// -/// All of the above components need configuration environment which we are abstracting over to -/// allow plugging EVM implementation into Reth SDK. -/// -/// The abstraction is designed to serve 2 codepaths: -/// 1. Externally provided complete block (e.g received while syncing). -/// 2. Block building when we know parent block and some additional context obtained from -/// payload attributes or alike. -/// -/// First case is handled by [`ConfigureEvm::evm_env`] and [`ConfigureEvm::context_for_block`] -/// which implement a conversion from [`NodePrimitives::Block`] to [`EvmEnv`] and [`ExecutionCtx`], -/// and allow configuring EVM and block execution environment at a given block. -/// -/// Second case is handled by similar [`ConfigureEvm::next_evm_env`] and -/// [`ConfigureEvm::context_for_next_block`] which take parent [`NodePrimitives::BlockHeader`] -/// along with [`NextBlockEnvCtx`]. [`NextBlockEnvCtx`] is very similar to payload attributes and -/// simply contains context for next block that is generally received from a CL node (timestamp, -/// beneficiary, withdrawals, etc.). +/// +/// 1. **[`Evm`] (produced by [`EvmFactory`])**: The core EVM implementation responsible for +/// executing individual transactions and producing outputs including state changes, logs, gas +/// usage, etc. +/// +/// 2. **[`BlockExecutor`] (produced by [`BlockExecutorFactory`])**: A higher-level component that +/// operates on top of [`Evm`] to execute entire blocks. This involves: +/// - Executing all transactions in sequence +/// - Building receipts from transaction outputs +/// - Applying block rewards to the beneficiary +/// - Executing system calls (e.g., EIP-4788 beacon root updates) +/// - Managing state changes and bundle accumulation +/// +/// 3. **[`BlockAssembler`]**: Responsible for assembling valid blocks from executed transactions. +/// It takes the output from [`BlockExecutor`] along with execution context and produces a +/// complete block ready for inclusion in the chain. +/// +/// # Usage Patterns +/// +/// The abstraction supports two primary use cases: +/// +/// ## 1. Executing Externally Provided Blocks (e.g., during sync) +/// +/// ```rust,ignore +/// use reth_evm::ConfigureEvm; +/// +/// // Execute a received block +/// let mut executor = evm_config.executor(state_db); +/// let output = executor.execute(&block)?; +/// +/// // Access the execution results +/// println!("Gas used: {}", output.result.gas_used); +/// println!("Receipts: {:?}", output.result.receipts); +/// ``` +/// +/// ## 2. Building New Blocks (e.g., payload building) +/// +/// Payload building is slightly different as it doesn't have the block's header yet, but rather +/// attributes for the block's environment, such as timestamp, fee recipient, and randomness value. +/// The block's header will be the outcome of the block building process. +/// +/// ```rust,ignore +/// use reth_evm::{ConfigureEvm, NextBlockEnvAttributes}; +/// +/// // Create attributes for the next block +/// let attributes = NextBlockEnvAttributes { +/// timestamp: current_time + 12, +/// suggested_fee_recipient: beneficiary_address, +/// prev_randao: randomness_value, +/// gas_limit: 30_000_000, +/// withdrawals: Some(withdrawals), +/// parent_beacon_block_root: Some(beacon_root), +/// }; +/// +/// // Build a new block on top of parent +/// let mut builder = evm_config.builder_for_next_block( +/// &mut state_db, +/// &parent_header, +/// attributes +/// )?; +/// +/// // Apply pre-execution changes (e.g., beacon root update) +/// builder.apply_pre_execution_changes()?; +/// +/// // Execute transactions +/// for tx in pending_transactions { +/// match builder.execute_transaction(tx) { +/// Ok(gas_used) => { +/// println!("Transaction executed, gas used: {}", gas_used); +/// } +/// Err(e) => { +/// println!("Transaction failed: {:?}", e); +/// } +/// } +/// } +/// +/// // Finish block building and get the outcome (block) +/// let outcome = builder.finish(state_provider)?; +/// let block = outcome.block; +/// ``` +/// +/// # Key Components +/// +/// ## [`NextBlockEnvCtx`] +/// +/// Contains attributes needed to configure the next block that cannot be derived from the +/// parent block alone. This includes data typically provided by the consensus layer: +/// - `timestamp`: Block timestamp +/// - `suggested_fee_recipient`: Beneficiary address +/// - `prev_randao`: Randomness value +/// - `gas_limit`: Block gas limit +/// - `withdrawals`: Consensus layer withdrawals +/// - `parent_beacon_block_root`: EIP-4788 beacon root +/// +/// ## [`BlockAssembler`] +/// +/// Takes the execution output and produces a complete block. It receives: +/// - Transaction execution results (receipts, gas used) +/// - Final state root after all executions +/// - Bundle state with all changes +/// - Execution context and environment +/// +/// The assembler is responsible for: +/// - Setting the correct block header fields +/// - Including executed transactions +/// - Setting gas used and receipts root +/// - Applying any chain-specific rules /// /// [`ExecutionCtx`]: BlockExecutorFactory::ExecutionCtx /// [`NextBlockEnvCtx`]: ConfigureEvm::NextBlockEnvCtx @@ -102,13 +194,15 @@ pub trait ConfigureEvm: Clone + Debug + Send + Sync + Unpin { type NextBlockEnvCtx: Debug + Clone; /// Configured [`BlockExecutorFactory`], contains [`EvmFactory`] internally. - type BlockExecutorFactory: BlockExecutorFactory< + type BlockExecutorFactory: for<'a> BlockExecutorFactory< Transaction = TxTy, Receipt = ReceiptTy, + ExecutionCtx<'a>: Debug + Send, EvmFactory: EvmFactory< Tx: TransactionEnv + FromRecoveredTx> + FromTxWithEncoded>, + Precompiles = PrecompilesMap, >, >; @@ -125,13 +219,23 @@ pub trait ConfigureEvm: Clone + Debug + Send + Sync + Unpin { fn block_assembler(&self) -> &Self::BlockAssembler; /// Creates a new [`EvmEnv`] for the given header. - fn evm_env(&self, header: &HeaderTy) -> EvmEnvFor; + fn evm_env(&self, header: &HeaderTy) -> Result, Self::Error>; /// Returns the configured [`EvmEnv`] for `parent + 1` block. /// /// This is intended for usage in block building after the merge and requires additional /// attributes that can't be derived from the parent block: attributes that are determined by /// the CL, such as the timestamp, suggested fee recipient, and randomness value. + /// + /// # Example + /// + /// ```rust,ignore + /// let evm_env = evm_config.next_evm_env(&parent_header, &attributes)?; + /// // evm_env now contains: + /// // - Correct spec ID based on timestamp and block number + /// // - Block environment with next block's parameters + /// // - Configuration like chain ID and blob parameters + /// ``` fn next_evm_env( &self, parent: &HeaderTy, @@ -142,7 +246,7 @@ pub trait ConfigureEvm: Clone + Debug + Send + Sync + Unpin { fn context_for_block<'a>( &self, block: &'a SealedBlock>, - ) -> ExecutionCtxFor<'a, Self>; + ) -> Result, Self::Error>; /// Returns the configured [`BlockExecutorFactory::ExecutionCtx`] for `parent + 1` /// block. @@ -150,7 +254,7 @@ pub trait ConfigureEvm: Clone + Debug + Send + Sync + Unpin { &self, parent: &SealedHeader>, attributes: Self::NextBlockEnvCtx, - ) -> ExecutionCtxFor<'_, Self>; + ) -> Result, Self::Error>; /// Returns a [`TxEnv`] from a transaction and [`Address`]. fn tx_env(&self, transaction: impl IntoTxEnv>) -> TxEnvFor { @@ -181,9 +285,9 @@ pub trait ConfigureEvm: Clone + Debug + Send + Sync + Unpin { &self, db: DB, header: &HeaderTy, - ) -> EvmFor { - let evm_env = self.evm_env(header); - self.evm_with_env(db, evm_env) + ) -> Result, Self::Error> { + let evm_env = self.evm_env(header)?; + Ok(self.evm_with_env(db, evm_env)) } /// Returns a new EVM with the given database configured with the given environment settings, @@ -223,10 +327,10 @@ pub trait ConfigureEvm: Clone + Debug + Send + Sync + Unpin { &'a self, db: &'a mut State, block: &'a SealedBlock<::Block>, - ) -> impl BlockExecutorFor<'a, Self::BlockExecutorFactory, DB> { - let evm = self.evm_for_block(db, block.header()); - let ctx = self.context_for_block(block); - self.create_executor(evm, ctx) + ) -> Result, Self::Error> { + let evm = self.evm_for_block(db, block.header())?; + let ctx = self.context_for_block(block)?; + Ok(self.create_executor(evm, ctx)) } /// Creates a [`BlockBuilder`]. Should be used when building a new block. @@ -235,6 +339,15 @@ pub trait ConfigureEvm: Clone + Debug + Send + Sync + Unpin { /// interface. Builder collects all of the executed transactions, and once /// [`BlockBuilder::finish`] is called, it invokes the configured [`BlockAssembler`] to /// create a block. + /// + /// # Example + /// + /// ```rust,ignore + /// // Create a builder with specific EVM configuration + /// let evm = evm_config.evm_with_env(&mut state_db, evm_env); + /// let ctx = evm_config.context_for_next_block(&parent, attributes); + /// let builder = evm_config.create_block_builder(evm, &parent, ctx); + /// ``` fn create_block_builder<'a, DB, I>( &'a self, evm: EvmFor, I>, @@ -259,35 +372,121 @@ pub trait ConfigureEvm: Clone + Debug + Send + Sync + Unpin { /// Creates a [`BlockBuilder`] for building of a new block. This is a helper to invoke /// [`ConfigureEvm::create_block_builder`]. + /// + /// This is the primary method for building new blocks. It combines: + /// 1. Creating the EVM environment for the next block + /// 2. Setting up the execution context from attributes + /// 3. Initializing the block builder with proper configuration + /// + /// # Example + /// + /// ```rust,ignore + /// // Build a block with specific attributes + /// let mut builder = evm_config.builder_for_next_block( + /// &mut state_db, + /// &parent_header, + /// attributes + /// )?; + /// + /// // Execute system calls (e.g., beacon root update) + /// builder.apply_pre_execution_changes()?; + /// + /// // Execute transactions + /// for tx in transactions { + /// builder.execute_transaction(tx)?; + /// } + /// + /// // Complete block building + /// let outcome = builder.finish(state_provider)?; + /// ``` fn builder_for_next_block<'a, DB: Database>( &'a self, db: &'a mut State, parent: &'a SealedHeader<::BlockHeader>, attributes: Self::NextBlockEnvCtx, - ) -> Result, Self::Error> { + ) -> Result< + impl BlockBuilder< + Primitives = Self::Primitives, + Executor: BlockExecutorFor<'a, Self::BlockExecutorFactory, DB>, + >, + Self::Error, + > { let evm_env = self.next_evm_env(parent, &attributes)?; let evm = self.evm_with_env(db, evm_env); - let ctx = self.context_for_next_block(parent, attributes); + let ctx = self.context_for_next_block(parent, attributes)?; Ok(self.create_block_builder(evm, parent, ctx)) } - /// Returns a new [`BasicBlockExecutor`]. + /// Returns a new [`Executor`] for executing blocks. + /// + /// The executor processes complete blocks including: + /// - All transactions in order + /// - Block rewards and fees + /// - Block level system calls + /// - State transitions + /// + /// # Example + /// + /// ```rust,ignore + /// // Create an executor + /// let mut executor = evm_config.executor(state_db); + /// + /// // Execute a single block + /// let output = executor.execute(&block)?; + /// + /// // Execute multiple blocks + /// let batch_output = executor.execute_batch(&blocks)?; + /// ``` #[auto_impl(keep_default_for(&, Arc))] - fn executor(&self, db: DB) -> BasicBlockExecutor<&Self, DB> { + fn executor( + &self, + db: DB, + ) -> impl Executor { BasicBlockExecutor::new(self, db) } /// Returns a new [`BasicBlockExecutor`]. #[auto_impl(keep_default_for(&, Arc))] - fn batch_executor(&self, db: DB) -> BasicBlockExecutor<&Self, DB> { + fn batch_executor( + &self, + db: DB, + ) -> impl Executor { BasicBlockExecutor::new(self, db) } } /// Represents additional attributes required to configure the next block. -/// This is used to configure the next block's environment -/// [`ConfigureEvm::next_evm_env`] and contains fields that can't be derived from the -/// parent header alone (attributes that are determined by the CL.) +/// +/// This struct contains all the information needed to build a new block that cannot be +/// derived from the parent block header alone. These attributes are typically provided +/// by the consensus layer (CL) through the Engine API during payload building. +/// +/// # Relationship with [`ConfigureEvm`] and [`BlockAssembler`] +/// +/// The flow for building a new block involves: +/// +/// 1. **Receive attributes** from the consensus layer containing: +/// - Timestamp for the new block +/// - Fee recipient (coinbase/beneficiary) +/// - Randomness value (prevRandao) +/// - Withdrawals to process +/// - Parent beacon block root for EIP-4788 +/// +/// 2. **Configure EVM environment** using these attributes: ```rust,ignore let evm_env = +/// evm_config.next_evm_env(&parent, &attributes)?; ``` +/// +/// 3. **Build the block** with transactions: ```rust,ignore let mut builder = +/// evm_config.builder_for_next_block( &mut state, &parent, attributes )?; ``` +/// +/// 4. **Assemble the final block** using [`BlockAssembler`] which takes: +/// - Execution results from all transactions +/// - The attributes used during execution +/// - Final state root after all changes +/// +/// This design cleanly separates: +/// - **Configuration** (what parameters to use) - handled by `NextBlockEnvAttributes` +/// - **Execution** (running transactions) - handled by `BlockExecutor` +/// - **Assembly** (creating the final block) - handled by `BlockAssembler` #[derive(Debug, Clone, PartialEq, Eq)] pub struct NextBlockEnvAttributes { /// The timestamp of the next block. @@ -354,6 +553,12 @@ impl TransactionEnv for TxEnv { fn set_access_list(&mut self, access_list: AccessList) { self.access_list = access_list; + + if self.tx_type == LEGACY_TX_TYPE_ID { + // if this was previously marked as legacy tx, this must be upgraded to eip2930 with an + // accesslist + self.tx_type = EIP2930_TX_TYPE_ID; + } } } diff --git a/crates/evm/evm/src/metrics.rs b/crates/evm/evm/src/metrics.rs index 56d3af89707..3fa02c32654 100644 --- a/crates/evm/evm/src/metrics.rs +++ b/crates/evm/evm/src/metrics.rs @@ -1,51 +1,10 @@ //! Executor metrics. -//! -//! Block processing related to syncing should take care to update the metrics by using either -//! [`ExecutorMetrics::execute_metered`] or [`ExecutorMetrics::metered_one`]. -use crate::{Database, OnStateHook}; use alloy_consensus::BlockHeader; -use alloy_evm::{ - block::{BlockExecutor, StateChangeSource}, - Evm, -}; -use core::borrow::BorrowMut; use metrics::{Counter, Gauge, Histogram}; -use reth_execution_errors::BlockExecutionError; -use reth_execution_types::BlockExecutionOutput; use reth_metrics::Metrics; -use reth_primitives_traits::{Block, BlockBody, RecoveredBlock}; -use revm::{ - database::{states::bundle_state::BundleRetention, State}, - state::EvmState, -}; +use reth_primitives_traits::{Block, RecoveredBlock}; use std::time::Instant; -/// Wrapper struct that combines metrics and state hook -struct MeteredStateHook { - metrics: ExecutorMetrics, - inner_hook: Box, -} - -impl OnStateHook for MeteredStateHook { - fn on_state(&mut self, source: StateChangeSource, state: &EvmState) { - // Update the metrics for the number of accounts, storage slots and bytecodes loaded - let accounts = state.keys().len(); - let storage_slots = state.values().map(|account| account.storage.len()).sum::(); - let bytecodes = state - .values() - .filter(|account| !account.info.is_empty_code_hash()) - .collect::>() - .len(); - - self.metrics.accounts_loaded_histogram.record(accounts as f64); - self.metrics.storage_slots_loaded_histogram.record(storage_slots as f64); - self.metrics.bytecodes_loaded_histogram.record(bytecodes as f64); - - // Call the original state hook - self.inner_hook.on_state(source, state); - } -} - /// Executor metrics. // TODO(onbjerg): add sload/sstore #[derive(Metrics, Clone)] @@ -79,259 +38,84 @@ pub struct ExecutorMetrics { } impl ExecutorMetrics { - fn metered(&self, block: &RecoveredBlock, f: F) -> R + /// Helper function for metered execution + fn metered(&self, f: F) -> R where - F: FnOnce() -> R, - B: reth_primitives_traits::Block, + F: FnOnce() -> (u64, R), { // Execute the block and record the elapsed time. let execute_start = Instant::now(); - let output = f(); + let (gas_used, output) = f(); let execution_duration = execute_start.elapsed().as_secs_f64(); // Update gas metrics. - self.gas_processed_total.increment(block.header().gas_used()); - self.gas_per_second.set(block.header().gas_used() as f64 / execution_duration); - self.gas_used_histogram.record(block.header().gas_used() as f64); + self.gas_processed_total.increment(gas_used); + self.gas_per_second.set(gas_used as f64 / execution_duration); + self.gas_used_histogram.record(gas_used as f64); self.execution_histogram.record(execution_duration); self.execution_duration.set(execution_duration); output } - /// Execute the given block using the provided [`BlockExecutor`] and update metrics for the - /// execution. + /// Execute a block and update basic gas/timing metrics. /// - /// Compared to [`Self::metered_one`], this method additionally updates metrics for the number - /// of accounts, storage slots and bytecodes loaded and updated. - /// Execute the given block using the provided [`BlockExecutor`] and update metrics for the - /// execution. - pub fn execute_metered( - &self, - executor: E, - input: &RecoveredBlock>>, - state_hook: Box, - ) -> Result, BlockExecutionError> - where - DB: Database, - E: BlockExecutor>>>, - { - // clone here is cheap, all the metrics are Option>. additionally - // they are globally registered so that the data recorded in the hook will - // be accessible. - let wrapper = MeteredStateHook { metrics: self.clone(), inner_hook: state_hook }; - - let mut executor = executor.with_state_hook(Some(Box::new(wrapper))); - - // Use metered to execute and track timing/gas metrics - let (mut db, result) = self.metered(input, || { - executor.apply_pre_execution_changes()?; - for tx in input.transactions_recovered() { - executor.execute_transaction(tx)?; - } - executor.finish().map(|(evm, result)| (evm.into_db(), result)) - })?; - - // merge transactions into bundle state - db.borrow_mut().merge_transitions(BundleRetention::Reverts); - let output = BlockExecutionOutput { result, state: db.borrow_mut().take_bundle() }; - - // Update the metrics for the number of accounts, storage slots and bytecodes updated - let accounts = output.state.state.len(); - let storage_slots = - output.state.state.values().map(|account| account.storage.len()).sum::(); - let bytecodes = output.state.contracts.len(); - - self.accounts_updated_histogram.record(accounts as f64); - self.storage_slots_updated_histogram.record(storage_slots as f64); - self.bytecodes_updated_histogram.record(bytecodes as f64); - - Ok(output) - } - - /// Execute the given block and update metrics for the execution. - pub fn metered_one(&self, input: &RecoveredBlock, f: F) -> R + /// This is a simple helper that tracks execution time and gas usage. + /// For more complex metrics tracking (like state changes), use the + /// metered execution functions in the engine/tree module. + pub fn metered_one(&self, block: &RecoveredBlock, f: F) -> R where F: FnOnce(&RecoveredBlock) -> R, - B: reth_primitives_traits::Block, + B: Block, + B::Header: BlockHeader, { - self.metered(input, || f(input)) + self.metered(|| (block.header().gas_used(), f(block))) } } #[cfg(test)] mod tests { use super::*; - use alloy_eips::eip7685::Requests; - use alloy_evm::EthEvm; - use alloy_primitives::{B256, U256}; - use metrics_util::debugging::{DebugValue, DebuggingRecorder, Snapshotter}; - use reth_ethereum_primitives::{Receipt, TransactionSigned}; - use reth_execution_types::BlockExecutionResult; - use revm::{ - database::State, - database_interface::EmptyDB, - inspector::NoOpInspector, - state::{Account, AccountInfo, AccountStatus, EvmStorage, EvmStorageSlot}, - Context, MainBuilder, MainContext, - }; - use std::sync::mpsc; - - /// A mock executor that simulates state changes - struct MockExecutor { - state: EvmState, - hook: Option>, - evm: EthEvm, NoOpInspector>, - } - - impl MockExecutor { - fn new(state: EvmState) -> Self { - let db = State::builder() - .with_database(EmptyDB::default()) - .with_bundle_update() - .without_state_clear() - .build(); - let evm = EthEvm::new( - Context::mainnet().with_db(db).build_mainnet_with_inspector(NoOpInspector {}), - false, - ); - Self { state, hook: None, evm } - } - } - - impl BlockExecutor for MockExecutor { - type Transaction = TransactionSigned; - type Receipt = Receipt; - type Evm = EthEvm, NoOpInspector>; - - fn apply_pre_execution_changes(&mut self) -> Result<(), BlockExecutionError> { - Ok(()) - } - - fn execute_transaction_with_result_closure( - &mut self, - _tx: impl alloy_evm::block::ExecutableTx, - _f: impl FnOnce(&revm::context::result::ExecutionResult<::HaltReason>), - ) -> Result { - Ok(0) - } - - fn finish( - self, - ) -> Result<(Self::Evm, BlockExecutionResult), BlockExecutionError> { - let Self { evm, hook, .. } = self; - - // Call hook with our mock state - if let Some(mut hook) = hook { - hook.on_state(StateChangeSource::Transaction(0), &self.state); - } - - Ok(( - evm, - BlockExecutionResult { - receipts: vec![], - requests: Requests::default(), - gas_used: 0, - }, - )) - } - - fn set_state_hook(&mut self, hook: Option>) { - self.hook = hook; - } - - fn evm(&self) -> &Self::Evm { - &self.evm - } - - fn evm_mut(&mut self) -> &mut Self::Evm { - &mut self.evm - } - } - - struct ChannelStateHook { - output: i32, - sender: mpsc::Sender, - } - - impl OnStateHook for ChannelStateHook { - fn on_state(&mut self, _source: StateChangeSource, _state: &EvmState) { - let _ = self.sender.send(self.output); - } - } - - fn setup_test_recorder() -> Snapshotter { - let recorder = DebuggingRecorder::new(); - let snapshotter = recorder.snapshotter(); - recorder.install().unwrap(); - snapshotter + use alloy_consensus::Header; + use alloy_primitives::B256; + use reth_ethereum_primitives::Block; + use reth_primitives_traits::Block as BlockTrait; + + fn create_test_block_with_gas(gas_used: u64) -> RecoveredBlock { + let header = Header { gas_used, ..Default::default() }; + let block = Block { header, body: Default::default() }; + // Use a dummy hash for testing + let hash = B256::default(); + let sealed = block.seal_unchecked(hash); + RecoveredBlock::new_sealed(sealed, Default::default()) } #[test] - fn test_executor_metrics_hook_metrics_recorded() { - let snapshotter = setup_test_recorder(); + fn test_metered_one_updates_metrics() { let metrics = ExecutorMetrics::default(); - let input = RecoveredBlock::::default(); - - let (tx, _rx) = mpsc::channel(); - let expected_output = 42; - let state_hook = Box::new(ChannelStateHook { sender: tx, output: expected_output }); - - let state = { - let mut state = EvmState::default(); - let storage = - EvmStorage::from_iter([(U256::from(1), EvmStorageSlot::new(U256::from(2)))]); - state.insert( - Default::default(), - Account { - info: AccountInfo { - balance: U256::from(100), - nonce: 10, - code_hash: B256::random(), - code: Default::default(), - }, - storage, - status: AccountStatus::Loaded, - }, - ); - state - }; - let executor = MockExecutor::new(state); - let _result = metrics.execute_metered::<_, EmptyDB>(executor, &input, state_hook).unwrap(); + let block = create_test_block_with_gas(1000); - let snapshot = snapshotter.snapshot().into_vec(); + // Execute with metered_one + let result = metrics.metered_one(&block, |b| { + // Simulate some work + std::thread::sleep(std::time::Duration::from_millis(10)); + b.header().gas_used() + }); - for metric in snapshot { - let metric_name = metric.0.key().name(); - if metric_name == "sync.execution.accounts_loaded_histogram" || - metric_name == "sync.execution.storage_slots_loaded_histogram" || - metric_name == "sync.execution.bytecodes_loaded_histogram" - { - if let DebugValue::Histogram(vs) = metric.3 { - assert!( - vs.iter().any(|v| v.into_inner() > 0.0), - "metric {metric_name} not recorded" - ); - } - } - } + // Verify result + assert_eq!(result, 1000); } #[test] - fn test_executor_metrics_hook_called() { + fn test_metered_helper_tracks_timing() { let metrics = ExecutorMetrics::default(); - let input = RecoveredBlock::::default(); - - let (tx, rx) = mpsc::channel(); - let expected_output = 42; - let state_hook = Box::new(ChannelStateHook { sender: tx, output: expected_output }); - - let state = EvmState::default(); - let executor = MockExecutor::new(state); - let _result = metrics.execute_metered::<_, EmptyDB>(executor, &input, state_hook).unwrap(); + let result = metrics.metered(|| { + // Simulate some work + std::thread::sleep(std::time::Duration::from_millis(10)); + (500, "test_result") + }); - let actual_output = rx.try_recv().unwrap(); - assert_eq!(actual_output, expected_output); + assert_eq!(result, "test_result"); } } diff --git a/crates/evm/evm/src/noop.rs b/crates/evm/evm/src/noop.rs index 64cc403819b..1125650a9cb 100644 --- a/crates/evm/evm/src/noop.rs +++ b/crates/evm/evm/src/noop.rs @@ -43,7 +43,7 @@ where self.inner().block_assembler() } - fn evm_env(&self, header: &HeaderTy) -> EvmEnvFor { + fn evm_env(&self, header: &HeaderTy) -> Result, Self::Error> { self.inner().evm_env(header) } @@ -58,7 +58,7 @@ where fn context_for_block<'a>( &self, block: &'a SealedBlock>, - ) -> crate::ExecutionCtxFor<'a, Self> { + ) -> Result, Self::Error> { self.inner().context_for_block(block) } @@ -66,7 +66,7 @@ where &self, parent: &SealedHeader>, attributes: Self::NextBlockEnvCtx, - ) -> crate::ExecutionCtxFor<'_, Self> { + ) -> Result, Self::Error> { self.inner().context_for_next_block(parent, attributes) } } diff --git a/crates/evm/execution-errors/src/lib.rs b/crates/evm/execution-errors/src/lib.rs index b8ddd1b4469..30ab734fc92 100644 --- a/crates/evm/execution-errors/src/lib.rs +++ b/crates/evm/execution-errors/src/lib.rs @@ -6,7 +6,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(feature = "std"), no_std)] extern crate alloc; diff --git a/crates/evm/execution-errors/src/trie.rs b/crates/evm/execution-errors/src/trie.rs index b8a1a3e9bd3..7dd749f0c11 100644 --- a/crates/evm/execution-errors/src/trie.rs +++ b/crates/evm/execution-errors/src/trie.rs @@ -170,6 +170,12 @@ pub enum SparseTrieErrorKind { /// RLP error. #[error(transparent)] Rlp(#[from] alloy_rlp::Error), + /// Node not found in provider during revealing. + #[error("node {path:?} not found in provider during revealing")] + NodeNotFoundInProvider { + /// Path to the missing node. + path: Nibbles, + }, /// Other. #[error(transparent)] Other(#[from] Box), diff --git a/crates/evm/execution-types/src/chain.rs b/crates/evm/execution-types/src/chain.rs index facbe115c78..dc7218631f9 100644 --- a/crates/evm/execution-types/src/chain.rs +++ b/crates/evm/execution-types/src/chain.rs @@ -242,16 +242,25 @@ impl Chain { N::SignedTx: Encodable2718, { let mut receipt_attach = Vec::with_capacity(self.blocks().len()); - for ((block_num, block), receipts) in - self.blocks().iter().zip(self.execution_outcome.receipts().iter()) - { - let mut tx_receipts = Vec::with_capacity(receipts.len()); - for (tx, receipt) in block.body().transactions().iter().zip(receipts.iter()) { - tx_receipts.push((tx.trie_hash(), receipt.clone())); - } - let block_num_hash = BlockNumHash::new(*block_num, block.hash()); - receipt_attach.push(BlockReceipts { block: block_num_hash, tx_receipts }); - } + + self.blocks_and_receipts().for_each(|(block, receipts)| { + let block_num_hash = BlockNumHash::new(block.number(), block.hash()); + + let tx_receipts = block + .body() + .transactions() + .iter() + .zip(receipts) + .map(|(tx, receipt)| (tx.trie_hash(), receipt.clone())) + .collect(); + + receipt_attach.push(BlockReceipts { + block: block_num_hash, + tx_receipts, + timestamp: block.timestamp(), + }); + }); + receipt_attach } @@ -400,6 +409,8 @@ pub struct BlockReceipts { pub block: BlockNumHash, /// Transaction identifier and receipt. pub tx_receipts: Vec<(TxHash, T)>, + /// Block timestamp + pub timestamp: u64, } /// Bincode-compatible [`Chain`] serde implementation. diff --git a/crates/evm/execution-types/src/execution_outcome.rs b/crates/evm/execution-types/src/execution_outcome.rs index b198713a2e0..49c35247297 100644 --- a/crates/evm/execution-types/src/execution_outcome.rs +++ b/crates/evm/execution-types/src/execution_outcome.rs @@ -113,7 +113,7 @@ impl ExecutionOutcome { ) }), reverts.into_iter().map(|(_, reverts)| { - // does not needs to be sorted, it is done when taking reverts. + // does not need to be sorted, it is done when taking reverts. reverts.into_iter().map(|(address, (original, storage))| { ( address, @@ -201,7 +201,7 @@ impl ExecutionOutcome { } /// Transform block number to the index of block. - pub fn block_number_to_index(&self, block_number: BlockNumber) -> Option { + pub const fn block_number_to_index(&self, block_number: BlockNumber) -> Option { if self.first_block > block_number { return None } @@ -214,7 +214,7 @@ impl ExecutionOutcome { /// Returns the receipt root for all recorded receipts. /// Note: this function calculated Bloom filters for every receipt and created merkle trees - /// of receipt. This is a expensive operation. + /// of receipt. This is an expensive operation. pub fn generic_receipts_root_slow( &self, block_number: BlockNumber, @@ -240,12 +240,12 @@ impl ExecutionOutcome { } /// Is execution outcome empty. - pub fn is_empty(&self) -> bool { + pub const fn is_empty(&self) -> bool { self.len() == 0 } /// Number of blocks in the execution outcome. - pub fn len(&self) -> usize { + pub const fn len(&self) -> usize { self.receipts.len() } @@ -255,7 +255,7 @@ impl ExecutionOutcome { } /// Return last block of the execution outcome - pub fn last_block(&self) -> BlockNumber { + pub const fn last_block(&self) -> BlockNumber { (self.first_block + self.len() as u64).saturating_sub(1) } @@ -558,7 +558,7 @@ mod tests { use alloy_primitives::{bytes, Address, LogData, B256}; #[test] - fn test_initialisation() { + fn test_initialization() { // Create a new BundleState object with initial data let bundle = BundleState::new( vec![(Address::new([2; 20]), None, Some(AccountInfo::default()), HashMap::default())], diff --git a/crates/evm/execution-types/src/lib.rs b/crates/evm/execution-types/src/lib.rs index 04dd8473134..8b795981fb5 100644 --- a/crates/evm/execution-types/src/lib.rs +++ b/crates/evm/execution-types/src/lib.rs @@ -6,7 +6,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(feature = "std"), no_std)] extern crate alloc; diff --git a/crates/exex/exex/Cargo.toml b/crates/exex/exex/Cargo.toml index a98eac62783..0d09f0a8c68 100644 --- a/crates/exex/exex/Cargo.toml +++ b/crates/exex/exex/Cargo.toml @@ -54,13 +54,11 @@ tracing.workspace = true [dev-dependencies] reth-db-common.workspace = true reth-evm-ethereum.workspace = true -reth-node-api.workspace = true reth-primitives-traits = { workspace = true, features = ["test-utils"] } reth-provider = { workspace = true, features = ["test-utils"] } reth-testing-utils.workspace = true alloy-genesis.workspace = true -alloy-consensus.workspace = true rand.workspace = true secp256k1.workspace = true @@ -81,4 +79,5 @@ serde = [ "reth-prune-types/serde", "reth-config/serde", "reth-ethereum-primitives/serde", + "reth-chain-state/serde", ] diff --git a/crates/exex/exex/src/backfill/factory.rs b/crates/exex/exex/src/backfill/factory.rs index 789d63f84e2..d9a51bc47a7 100644 --- a/crates/exex/exex/src/backfill/factory.rs +++ b/crates/exex/exex/src/backfill/factory.rs @@ -24,7 +24,7 @@ impl BackfillJobFactory { Self { evm_config, provider, - prune_modes: PruneModes::none(), + prune_modes: PruneModes::default(), thresholds: ExecutionStageThresholds { // Default duration for a database transaction to be considered long-lived is // 60 seconds, so we limit the backfill job to the half of it to be sure we finish diff --git a/crates/exex/exex/src/backfill/job.rs b/crates/exex/exex/src/backfill/job.rs index 393d00c62ee..1a294e50659 100644 --- a/crates/exex/exex/src/backfill/job.rs +++ b/crates/exex/exex/src/backfill/job.rs @@ -124,7 +124,7 @@ where blocks.push(block); // Check if we should commit now if self.thresholds.is_end_of_batch( - block_number - *self.range.start(), + block_number - *self.range.start() + 1, executor.size_hint() as u64, cumulative_gas, batch_start.elapsed(), @@ -243,11 +243,14 @@ impl From> for SingleBlockBackfillJob { #[cfg(test)] mod tests { use crate::{ - backfill::test_utils::{blocks_and_execution_outputs, chain_spec, to_execution_outcome}, + backfill::{ + job::ExecutionStageThresholds, + test_utils::{blocks_and_execution_outputs, chain_spec, to_execution_outcome}, + }, BackfillJobFactory, }; use reth_db_common::init::init_genesis; - use reth_evm_ethereum::execute::EthExecutorProvider; + use reth_evm_ethereum::EthEvmConfig; use reth_primitives_traits::crypto::secp256k1::public_key_to_address; use reth_provider::{ providers::BlockchainProvider, test_utils::create_test_provider_factory_with_chain_spec, @@ -264,7 +267,7 @@ mod tests { let chain_spec = chain_spec(address); - let executor = EthExecutorProvider::ethereum(chain_spec.clone()); + let executor = EthEvmConfig::ethereum(chain_spec.clone()); let provider_factory = create_test_provider_factory_with_chain_spec(chain_spec.clone()); init_genesis(&provider_factory)?; let blockchain_db = BlockchainProvider::new(provider_factory.clone())?; @@ -300,7 +303,7 @@ mod tests { let chain_spec = chain_spec(address); - let executor = EthExecutorProvider::ethereum(chain_spec.clone()); + let executor = EthEvmConfig::ethereum(chain_spec.clone()); let provider_factory = create_test_provider_factory_with_chain_spec(chain_spec.clone()); init_genesis(&provider_factory)?; let blockchain_db = BlockchainProvider::new(provider_factory.clone())?; @@ -333,4 +336,47 @@ mod tests { Ok(()) } + + #[test] + fn test_backfill_with_batch_threshold() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + + // Create a key pair for the sender + let key_pair = generators::generate_key(&mut generators::rng()); + let address = public_key_to_address(key_pair.public_key()); + + let chain_spec = chain_spec(address); + + let executor = EthEvmConfig::ethereum(chain_spec.clone()); + let provider_factory = create_test_provider_factory_with_chain_spec(chain_spec.clone()); + init_genesis(&provider_factory)?; + let blockchain_db = BlockchainProvider::new(provider_factory.clone())?; + + let blocks_and_execution_outputs = + blocks_and_execution_outputs(provider_factory, chain_spec, key_pair)?; + let (block1, output1) = blocks_and_execution_outputs[0].clone(); + let (block2, output2) = blocks_and_execution_outputs[1].clone(); + + // Backfill with max_blocks=1, expect two separate chains + let factory = BackfillJobFactory::new(executor, blockchain_db).with_thresholds( + ExecutionStageThresholds { max_blocks: Some(1), ..Default::default() }, + ); + let job = factory.backfill(1..=2); + let chains = job.collect::, _>>()?; + + // Assert two chains, each with one block + assert_eq!(chains.len(), 2); + + let mut chain1 = chains[0].clone(); + chain1.execution_outcome_mut().bundle.reverts.sort(); + assert_eq!(chain1.blocks(), &[(1, block1)].into()); + assert_eq!(chain1.execution_outcome(), &to_execution_outcome(1, &output1)); + + let mut chain2 = chains[1].clone(); + chain2.execution_outcome_mut().bundle.reverts.sort(); + assert_eq!(chain2.blocks(), &[(2, block2)].into()); + assert_eq!(chain2.execution_outcome(), &to_execution_outcome(2, &output2)); + + Ok(()) + } } diff --git a/crates/exex/exex/src/backfill/stream.rs b/crates/exex/exex/src/backfill/stream.rs index d9328db1833..9d50737f5aa 100644 --- a/crates/exex/exex/src/backfill/stream.rs +++ b/crates/exex/exex/src/backfill/stream.rs @@ -239,21 +239,34 @@ where #[cfg(test)] mod tests { + use super::*; use crate::{ backfill::test_utils::{ blocks_and_execution_outcome, blocks_and_execution_outputs, chain_spec, + execute_block_and_commit_to_database, }, BackfillJobFactory, }; + use alloy_consensus::{constants::ETH_TO_WEI, Header, TxEip2930}; + use alloy_primitives::{b256, Address, TxKind, U256}; + use eyre::Result; use futures::StreamExt; + use reth_chainspec::{ChainSpec, EthereumHardfork, MIN_TRANSACTION_GAS}; use reth_db_common::init::init_genesis; - use reth_evm_ethereum::execute::EthExecutorProvider; - use reth_primitives_traits::crypto::secp256k1::public_key_to_address; + use reth_ethereum_primitives::{Block, BlockBody, Transaction}; + use reth_evm_ethereum::EthEvmConfig; + use reth_primitives_traits::{ + crypto::secp256k1::public_key_to_address, Block as _, NodePrimitives, + }; use reth_provider::{ - providers::BlockchainProvider, test_utils::create_test_provider_factory_with_chain_spec, + providers::{BlockchainProvider, ProviderNodeTypes}, + test_utils::create_test_provider_factory_with_chain_spec, + ProviderFactory, }; use reth_stages_api::ExecutionStageThresholds; - use reth_testing_utils::generators; + use reth_testing_utils::{generators, generators::sign_tx_with_key_pair}; + use secp256k1::Keypair; + use std::sync::Arc; #[tokio::test] async fn test_single_blocks() -> eyre::Result<()> { @@ -265,7 +278,7 @@ mod tests { let chain_spec = chain_spec(address); - let executor = EthExecutorProvider::ethereum(chain_spec.clone()); + let executor = EthEvmConfig::ethereum(chain_spec.clone()); let provider_factory = create_test_provider_factory_with_chain_spec(chain_spec.clone()); init_genesis(&provider_factory)?; let blockchain_db = BlockchainProvider::new(provider_factory.clone())?; @@ -302,7 +315,7 @@ mod tests { let chain_spec = chain_spec(address); - let executor = EthExecutorProvider::ethereum(chain_spec.clone()); + let executor = EthEvmConfig::ethereum(chain_spec.clone()); let provider_factory = create_test_provider_factory_with_chain_spec(chain_spec.clone()); init_genesis(&provider_factory)?; let blockchain_db = BlockchainProvider::new(provider_factory.clone())?; @@ -327,4 +340,131 @@ mod tests { Ok(()) } + + fn create_blocks( + chain_spec: &Arc, + key_pair: Keypair, + n: u64, + ) -> Result>> { + let mut blocks = Vec::with_capacity(n as usize); + let mut parent_hash = chain_spec.genesis_hash(); + + for (i, nonce) in (1..=n).zip(0..n) { + let block = Block { + header: Header { + parent_hash, + // Hardcoded receipts_root matching the original test (same tx in each block) + receipts_root: b256!( + "0xd3a6acf9a244d78b33831df95d472c4128ea85bf079a1d41e32ed0b7d2244c9e" + ), + difficulty: chain_spec.fork(EthereumHardfork::Paris).ttd().expect("Paris TTD"), + number: i, + gas_limit: MIN_TRANSACTION_GAS, + gas_used: MIN_TRANSACTION_GAS, + ..Default::default() + }, + body: BlockBody { + transactions: vec![sign_tx_with_key_pair( + key_pair, + Transaction::Eip2930(TxEip2930 { + chain_id: chain_spec.chain.id(), + nonce, + gas_limit: MIN_TRANSACTION_GAS, + gas_price: 1_500_000_000, + to: TxKind::Call(Address::ZERO), + value: U256::from(0.1 * ETH_TO_WEI as f64), + ..Default::default() + }), + )], + ..Default::default() + }, + } + .try_into_recovered()?; + + parent_hash = block.hash(); + blocks.push(block); + } + + Ok(blocks) + } + + fn execute_and_commit_blocks( + provider_factory: &ProviderFactory, + chain_spec: &Arc, + blocks: &[RecoveredBlock], + ) -> Result<()> + where + N: ProviderNodeTypes< + Primitives: NodePrimitives< + Block = reth_ethereum_primitives::Block, + BlockBody = reth_ethereum_primitives::BlockBody, + Receipt = reth_ethereum_primitives::Receipt, + >, + >, + { + for block in blocks { + execute_block_and_commit_to_database(provider_factory, chain_spec.clone(), block)?; + } + Ok(()) + } + + #[tokio::test] + async fn test_batch_parallel_range_advance() -> Result<()> { + reth_tracing::init_test_tracing(); + + // Create a key pair for the sender + let key_pair = generators::generate_key(&mut generators::rng()); + let address = public_key_to_address(key_pair.public_key()); + + let chain_spec = chain_spec(address); + + let executor = EthEvmConfig::ethereum(chain_spec.clone()); + let provider_factory = create_test_provider_factory_with_chain_spec(chain_spec.clone()); + init_genesis(&provider_factory)?; + let blockchain_db = BlockchainProvider::new(provider_factory.clone())?; + + // Create and commit 4 blocks + let blocks = create_blocks(&chain_spec, key_pair, 4)?; + execute_and_commit_blocks(&provider_factory, &chain_spec, &blocks)?; + + // Create factory with batch size 2 (via thresholds max_blocks=2) and parallelism=2 + let factory = BackfillJobFactory::new(executor.clone(), blockchain_db.clone()) + .with_thresholds(ExecutionStageThresholds { max_blocks: Some(2), ..Default::default() }) + .with_stream_parallelism(2); + + // Stream backfill for range 1..=4 + let mut backfill_stream = factory.backfill(1..=4).into_stream(); + + // Collect the two expected chains from the stream + let mut chain1 = backfill_stream.next().await.unwrap()?; + let mut chain2 = backfill_stream.next().await.unwrap()?; + assert!(backfill_stream.next().await.is_none()); + + // Sort reverts for comparison + chain1.execution_outcome_mut().state_mut().reverts.sort(); + chain2.execution_outcome_mut().state_mut().reverts.sort(); + + // Compute expected chains using non-stream BackfillJob (sequential) + let factory_seq = + BackfillJobFactory::new(executor.clone(), blockchain_db.clone()).with_thresholds( + ExecutionStageThresholds { max_blocks: Some(2), ..Default::default() }, + ); + + let mut expected_chain1 = + factory_seq.backfill(1..=2).collect::, _>>()?.into_iter().next().unwrap(); + let mut expected_chain2 = + factory_seq.backfill(3..=4).collect::, _>>()?.into_iter().next().unwrap(); + + // Sort reverts for expected + expected_chain1.execution_outcome_mut().state_mut().reverts.sort(); + expected_chain2.execution_outcome_mut().state_mut().reverts.sort(); + + // Assert the streamed chains match the expected sequential ones + assert_eq!(chain1.blocks(), expected_chain1.blocks()); + assert_eq!(chain1.execution_outcome(), expected_chain1.execution_outcome()); + assert_eq!(chain2.blocks(), expected_chain2.blocks()); + assert_eq!(chain2.execution_outcome(), expected_chain2.execution_outcome()); + + Ok(()) + } } diff --git a/crates/exex/exex/src/backfill/test_utils.rs b/crates/exex/exex/src/backfill/test_utils.rs index 00bfbd94ee4..e489a98abf7 100644 --- a/crates/exex/exex/src/backfill/test_utils.rs +++ b/crates/exex/exex/src/backfill/test_utils.rs @@ -9,8 +9,8 @@ use reth_evm::{ execute::{BlockExecutionOutput, Executor}, ConfigureEvm, }; -use reth_evm_ethereum::{execute::EthExecutorProvider, EthEvmConfig}; -use reth_node_api::FullNodePrimitives; +use reth_evm_ethereum::EthEvmConfig; +use reth_node_api::NodePrimitives; use reth_primitives_traits::{Block as _, RecoveredBlock}; use reth_provider::{ providers::ProviderNodeTypes, BlockWriter as _, ExecutionOutcome, LatestStateProviderRef, @@ -58,7 +58,7 @@ pub(crate) fn execute_block_and_commit_to_database( ) -> eyre::Result> where N: ProviderNodeTypes< - Primitives: FullNodePrimitives< + Primitives: NodePrimitives< Block = reth_ethereum_primitives::Block, BlockBody = reth_ethereum_primitives::BlockBody, Receipt = reth_ethereum_primitives::Receipt, @@ -68,7 +68,7 @@ where let provider = provider_factory.provider()?; // Execute the block to produce a block execution output - let mut block_execution_output = EthExecutorProvider::ethereum(chain_spec) + let mut block_execution_output = EthEvmConfig::ethereum(chain_spec) .batch_executor(StateProviderDatabase::new(LatestStateProviderRef::new(&provider))) .execute(block)?; block_execution_output.state.reverts.sort(); @@ -82,7 +82,6 @@ where vec![block.clone()], &execution_outcome, Default::default(), - Default::default(), )?; provider_rw.commit()?; @@ -170,7 +169,7 @@ pub(crate) fn blocks_and_execution_outputs( > where N: ProviderNodeTypes< - Primitives: FullNodePrimitives< + Primitives: NodePrimitives< Block = reth_ethereum_primitives::Block, BlockBody = reth_ethereum_primitives::BlockBody, Receipt = reth_ethereum_primitives::Receipt, @@ -194,7 +193,7 @@ pub(crate) fn blocks_and_execution_outcome( ) -> eyre::Result<(Vec>, ExecutionOutcome)> where N: ProviderNodeTypes, - N::Primitives: FullNodePrimitives< + N::Primitives: NodePrimitives< Block = reth_ethereum_primitives::Block, Receipt = reth_ethereum_primitives::Receipt, >, @@ -216,7 +215,6 @@ where vec![block1.clone(), block2.clone()], &execution_outcome, Default::default(), - Default::default(), )?; provider_rw.commit()?; diff --git a/crates/exex/exex/src/lib.rs b/crates/exex/exex/src/lib.rs index d5da6a18faa..71e1269862f 100644 --- a/crates/exex/exex/src/lib.rs +++ b/crates/exex/exex/src/lib.rs @@ -85,7 +85,7 @@ html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256", issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(test), warn(unused_crate_dependencies))] mod backfill; diff --git a/crates/exex/exex/src/manager.rs b/crates/exex/exex/src/manager.rs index 921ed696fa2..99694f0a51b 100644 --- a/crates/exex/exex/src/manager.rs +++ b/crates/exex/exex/src/manager.rs @@ -370,7 +370,7 @@ where .map(|(exex_id, num_hash)| { num_hash.map_or(Ok((exex_id, num_hash, false)), |num_hash| { self.provider - .is_known(&num_hash.hash) + .is_known(num_hash.hash) // Save the ExEx ID, finished height, and whether the hash is canonical .map(|is_canonical| (exex_id, Some(num_hash), is_canonical)) }) @@ -501,11 +501,11 @@ where .next_notification_id .checked_sub(this.min_id) .expect("exex expected notification ID outside the manager's range"); - if let Some(notification) = this.buffer.get(notification_index) { - if let Poll::Ready(Err(err)) = exex.send(cx, notification) { - // The channel was closed, which is irrecoverable for the manager - return Poll::Ready(Err(err.into())) - } + if let Some(notification) = this.buffer.get(notification_index) && + let Poll::Ready(Err(err)) = exex.send(cx, notification) + { + // The channel was closed, which is irrecoverable for the manager + return Poll::Ready(Err(err.into())) } min_id = min_id.min(exex.next_notification_id); this.exex_handles.push(exex); @@ -663,11 +663,11 @@ mod tests { use futures::{StreamExt, TryStreamExt}; use rand::Rng; use reth_db_common::init::init_genesis; - use reth_evm_ethereum::{execute::EthExecutorProvider, EthEvmConfig}; + use reth_evm_ethereum::EthEvmConfig; use reth_primitives_traits::RecoveredBlock; use reth_provider::{ providers::BlockchainProvider, test_utils::create_test_provider_factory, BlockReader, - BlockWriter, Chain, DatabaseProviderFactory, StorageLocation, TransactionVariant, + BlockWriter, Chain, DBProvider, DatabaseProviderFactory, TransactionVariant, }; use reth_testing_utils::generators::{self, random_block, BlockParams}; @@ -1107,7 +1107,7 @@ mod tests { "test_exex".to_string(), Default::default(), provider, - EthExecutorProvider::mainnet(), + EthEvmConfig::mainnet(), wal.handle(), ); @@ -1162,7 +1162,7 @@ mod tests { "test_exex".to_string(), Default::default(), provider, - EthExecutorProvider::mainnet(), + EthEvmConfig::mainnet(), wal.handle(), ); @@ -1212,7 +1212,7 @@ mod tests { "test_exex".to_string(), Default::default(), provider, - EthExecutorProvider::mainnet(), + EthEvmConfig::mainnet(), wal.handle(), ); @@ -1255,7 +1255,7 @@ mod tests { "test_exex".to_string(), Default::default(), provider, - EthExecutorProvider::mainnet(), + EthEvmConfig::mainnet(), wal.handle(), ); @@ -1303,7 +1303,7 @@ mod tests { .try_recover() .unwrap(); let provider_rw = provider_factory.database_provider_rw().unwrap(); - provider_rw.insert_block(block.clone(), StorageLocation::Database).unwrap(); + provider_rw.insert_block(block.clone()).unwrap(); provider_rw.commit().unwrap(); let provider = BlockchainProvider::new(provider_factory).unwrap(); @@ -1315,7 +1315,7 @@ mod tests { "test_exex".to_string(), Default::default(), provider.clone(), - EthExecutorProvider::mainnet(), + EthEvmConfig::mainnet(), wal.handle(), ); @@ -1358,7 +1358,7 @@ mod tests { // WAL shouldn't contain the genesis notification, because it's finalized assert_eq!( exex_manager.wal.iter_notifications()?.collect::>>()?, - [notification.clone()] + std::slice::from_ref(¬ification) ); finalized_headers_tx.send(Some(block.clone_sealed_header()))?; @@ -1366,7 +1366,7 @@ mod tests { // WAL isn't finalized because the ExEx didn't emit the `FinishedHeight` event assert_eq!( exex_manager.wal.iter_notifications()?.collect::>>()?, - [notification.clone()] + std::slice::from_ref(¬ification) ); // Send a `FinishedHeight` event with a non-canonical block @@ -1380,7 +1380,7 @@ mod tests { // non-canonical block assert_eq!( exex_manager.wal.iter_notifications()?.collect::>>()?, - [notification] + std::slice::from_ref(¬ification) ); // Send a `FinishedHeight` event with a canonical block diff --git a/crates/exex/exex/src/notifications.rs b/crates/exex/exex/src/notifications.rs index eac5208f2fb..c6a54e647cf 100644 --- a/crates/exex/exex/src/notifications.rs +++ b/crates/exex/exex/src/notifications.rs @@ -308,7 +308,7 @@ where /// we're not on the canonical chain and we need to revert the notification with the ExEx /// head block. fn check_canonical(&mut self) -> eyre::Result>> { - if self.provider.is_known(&self.initial_exex_head.block.hash)? && + if self.provider.is_known(self.initial_exex_head.block.hash)? && self.initial_exex_head.block.number <= self.initial_local_head.number { // we have the targeted block and that block is below the current head @@ -350,7 +350,7 @@ where /// Compares the node head against the ExEx head, and backfills if needed. /// - /// CAUTON: This method assumes that the ExEx head is <= the node head, and that it's on the + /// CAUTION: This method assumes that the ExEx head is <= the node head, and that it's on the /// canonical chain. /// /// Possible situations are: @@ -453,11 +453,11 @@ mod tests { use futures::StreamExt; use reth_db_common::init::init_genesis; use reth_ethereum_primitives::Block; - use reth_evm_ethereum::execute::EthExecutorProvider; + use reth_evm_ethereum::EthEvmConfig; use reth_primitives_traits::Block as _; use reth_provider::{ providers::BlockchainProvider, test_utils::create_test_provider_factory, BlockWriter, - Chain, DatabaseProviderFactory, StorageLocation, + Chain, DBProvider, DatabaseProviderFactory, }; use reth_testing_utils::generators::{self, random_block, BlockParams}; use tokio::sync::mpsc; @@ -483,8 +483,7 @@ mod tests { BlockParams { parent: Some(genesis_hash), tx_count: Some(0), ..Default::default() }, ); let provider_rw = provider_factory.provider_rw()?; - provider_rw - .insert_block(node_head_block.clone().try_recover()?, StorageLocation::Database)?; + provider_rw.insert_block(node_head_block.clone().try_recover()?)?; provider_rw.commit()?; let node_head = node_head_block.num_hash(); @@ -511,7 +510,7 @@ mod tests { let mut notifications = ExExNotificationsWithoutHead::new( node_head, provider, - EthExecutorProvider::mainnet(), + EthEvmConfig::mainnet(), notifications_rx, wal.handle(), ) @@ -579,7 +578,7 @@ mod tests { let mut notifications = ExExNotificationsWithoutHead::new( node_head, provider, - EthExecutorProvider::mainnet(), + EthEvmConfig::mainnet(), notifications_rx, wal.handle(), ) @@ -614,11 +613,11 @@ mod tests { .try_recover()?; let node_head = node_head_block.num_hash(); let provider_rw = provider.database_provider_rw()?; - provider_rw.insert_block(node_head_block, StorageLocation::Database)?; + provider_rw.insert_block(node_head_block)?; provider_rw.commit()?; let node_head_notification = ExExNotification::ChainCommitted { new: Arc::new( - BackfillJobFactory::new(EthExecutorProvider::mainnet(), provider.clone()) + BackfillJobFactory::new(EthEvmConfig::mainnet(), provider.clone()) .backfill(node_head.number..=node_head.number) .next() .ok_or_else(|| eyre::eyre!("failed to backfill"))??, @@ -660,7 +659,7 @@ mod tests { let mut notifications = ExExNotificationsWithoutHead::new( node_head, provider, - EthExecutorProvider::mainnet(), + EthEvmConfig::mainnet(), notifications_rx, wal.handle(), ) @@ -736,7 +735,7 @@ mod tests { let mut notifications = ExExNotificationsWithoutHead::new( node_head, provider, - EthExecutorProvider::mainnet(), + EthEvmConfig::mainnet(), notifications_rx, wal.handle(), ) diff --git a/crates/exex/exex/src/wal/mod.rs b/crates/exex/exex/src/wal/mod.rs index 66c528c14fa..b5537aa88fc 100644 --- a/crates/exex/exex/src/wal/mod.rs +++ b/crates/exex/exex/src/wal/mod.rs @@ -96,7 +96,7 @@ where N: NodePrimitives, { fn new(directory: impl AsRef) -> WalResult { - let mut wal = Self { + let wal = Self { next_file_id: AtomicU32::new(0), storage: Storage::new(directory)?, block_cache: RwLock::new(BlockCache::default()), @@ -112,7 +112,7 @@ where /// Fills the block cache with the notifications from the storage. #[instrument(skip(self))] - fn fill_block_cache(&mut self) -> WalResult<()> { + fn fill_block_cache(&self) -> WalResult<()> { let Some(files_range) = self.storage.files_range()? else { return Ok(()) }; self.next_file_id.store(files_range.end() + 1, Ordering::Relaxed); diff --git a/crates/exex/test-utils/Cargo.toml b/crates/exex/test-utils/Cargo.toml index 4081d1eda16..80ce4167e46 100644 --- a/crates/exex/test-utils/Cargo.toml +++ b/crates/exex/test-utils/Cargo.toml @@ -31,7 +31,6 @@ reth-ethereum-primitives.workspace = true reth-provider = { workspace = true, features = ["test-utils"] } reth-tasks.workspace = true reth-transaction-pool = { workspace = true, features = ["test-utils"] } -reth-trie-db.workspace = true ## alloy alloy-eips.workspace = true diff --git a/crates/exex/test-utils/src/lib.rs b/crates/exex/test-utils/src/lib.rs index 7971912da08..0305da323d0 100644 --- a/crates/exex/test-utils/src/lib.rs +++ b/crates/exex/test-utils/src/lib.rs @@ -5,7 +5,7 @@ html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256", issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(test), warn(unused_crate_dependencies))] use std::{ @@ -35,13 +35,16 @@ use reth_node_api::{ use reth_node_builder::{ components::{ BasicPayloadServiceBuilder, Components, ComponentsBuilder, ConsensusBuilder, - ExecutorBuilder, NodeComponentsBuilder, PoolBuilder, + ExecutorBuilder, PoolBuilder, }, BuilderContext, Node, NodeAdapter, RethFullAdapter, }; use reth_node_core::node_config::NodeConfig; use reth_node_ethereum::{ - node::{EthereumAddOns, EthereumNetworkBuilder, EthereumPayloadBuilder}, + node::{ + EthereumAddOns, EthereumEngineValidatorBuilder, EthereumEthApiBuilder, + EthereumNetworkBuilder, EthereumPayloadBuilder, + }, EthEngineTypes, }; use reth_payload_builder::noop::NoopPayloadBuilderService; @@ -113,21 +116,13 @@ pub struct TestNode; impl NodeTypes for TestNode { type Primitives = EthPrimitives; type ChainSpec = ChainSpec; - type StateCommitment = reth_trie_db::MerklePatriciaTrie; type Storage = EthStorage; type Payload = EthEngineTypes; } impl Node for TestNode where - N: FullNodeTypes< - Types: NodeTypes< - Payload = EthEngineTypes, - ChainSpec = ChainSpec, - Primitives = EthPrimitives, - Storage = EthStorage, - >, - >, + N: FullNodeTypes, { type ComponentsBuilder = ComponentsBuilder< N, @@ -137,9 +132,8 @@ where TestExecutorBuilder, TestConsensusBuilder, >; - type AddOns = EthereumAddOns< - NodeAdapter>::Components>, - >; + type AddOns = + EthereumAddOns, EthereumEthApiBuilder, EthereumEngineValidatorBuilder>; fn components_builder(&self) -> Self::ComponentsBuilder { ComponentsBuilder::default() @@ -160,16 +154,7 @@ where pub type TmpDB = Arc>; /// The [`NodeAdapter`] for the [`TestExExContext`]. Contains type necessary to /// boot the testing environment -pub type Adapter = NodeAdapter< - RethFullAdapter, - <>, - >, - >>::ComponentsBuilder as NodeComponentsBuilder>>::Components, ->; +pub type Adapter = NodeAdapter>; /// An [`ExExContext`] using the [`Adapter`] type. pub type TestExExContext = ExExContext; @@ -258,7 +243,7 @@ pub async fn test_exex_context_with_chain_spec( let provider_factory = ProviderFactory::>::new( db, chain_spec.clone(), - StaticFileProvider::read_write(static_dir.into_path()).expect("static file provider"), + StaticFileProvider::read_write(static_dir.keep()).expect("static file provider"), ); let genesis_hash = init_genesis(&provider_factory)?; diff --git a/crates/exex/types/Cargo.toml b/crates/exex/types/Cargo.toml index 9fe58fb8690..11dec0246fe 100644 --- a/crates/exex/types/Cargo.toml +++ b/crates/exex/types/Cargo.toml @@ -42,6 +42,7 @@ serde = [ "rand/serde", "reth-primitives-traits/serde", "reth-ethereum-primitives/serde", + "reth-chain-state/serde", ] serde-bincode-compat = [ "reth-execution-types/serde-bincode-compat", diff --git a/crates/exex/types/src/lib.rs b/crates/exex/types/src/lib.rs index ffed819d6ec..3c5fb61b42e 100644 --- a/crates/exex/types/src/lib.rs +++ b/crates/exex/types/src/lib.rs @@ -6,7 +6,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] mod finished_height; mod head; diff --git a/crates/fs-util/src/lib.rs b/crates/fs-util/src/lib.rs index 922bec6bf67..54a22875d94 100644 --- a/crates/fs-util/src/lib.rs +++ b/crates/fs-util/src/lib.rs @@ -39,7 +39,7 @@ pub enum FsPathError { }, /// Error variant for failed read link operation with additional path context. - #[error("failed to read from {path:?}: {source}")] + #[error("failed to read link {path:?}: {source}")] ReadLink { /// The source `io::Error`. source: io::Error, @@ -230,6 +230,12 @@ pub fn read(path: impl AsRef) -> Result> { fs::read(path).map_err(|err| FsPathError::read(err, path)) } +/// Wrapper for `std::fs::read_link` +pub fn read_link(path: impl AsRef) -> Result { + let path = path.as_ref(); + fs::read_link(path).map_err(|err| FsPathError::read_link(err, path)) +} + /// Wrapper for `std::fs::write` pub fn write(path: impl AsRef, contents: impl AsRef<[u8]>) -> Result<()> { let path = path.as_ref(); @@ -323,10 +329,18 @@ where let mut file = File::create(&tmp_path).map_err(|err| FsPathError::create_file(err, &tmp_path))?; - write_fn(&mut file).map_err(|err| FsPathError::Write { - source: Error::other(err.into()), - path: tmp_path.clone(), - })?; + // Execute the write function and handle errors properly + // If write_fn fails, we need to clean up the temporary file before returning + match write_fn(&mut file) { + Ok(()) => { + // Success - continue with the atomic operation + } + Err(err) => { + // Clean up the temporary file before returning the error + let _ = fs::remove_file(&tmp_path); + return Err(FsPathError::Write { source: Error::other(err.into()), path: tmp_path }); + } + } // fsync() file file.sync_all().map_err(|err| FsPathError::fsync(err, &tmp_path))?; diff --git a/crates/mantle-hardforks/Cargo.toml b/crates/mantle-hardforks/Cargo.toml index 3bc97070bb2..9c31b2db071 100644 --- a/crates/mantle-hardforks/Cargo.toml +++ b/crates/mantle-hardforks/Cargo.toml @@ -18,18 +18,30 @@ workspace = true # Core dependencies alloy-chains = { workspace = true } alloy-hardforks = { workspace = true } -# alloy-primitives = { workspace = true } +alloy-op-evm = { workspace = true } +alloy-primitives = { workspace = true } # Reth dependencies # reth-chainspec = { workspace = true } # reth-ethereum-forks = { workspace = true } reth-optimism-forks = { workspace = true } +reth-db = { workspace = true } +reth-db-api = { workspace = true } +reth-provider = { workspace = true } +reth-trie = { workspace = true } +reth-trie-db = { workspace = true } +reth-primitives-traits = { workspace = true } +op-revm = { workspace = true } +revm = { workspace = true } auto_impl.workspace = true -serde = { workspace = true, optional = true } +serde = { workspace = true } +eyre = { workspace = true } +tracing = { workspace = true } [features] +default = ["std", "serde"] +std = [] serde = [ - "dep:serde", "alloy-hardforks/serde" ] diff --git a/crates/mantle-hardforks/README.md b/crates/mantle-hardforks/README.md deleted file mode 100644 index 6a3071b8346..00000000000 --- a/crates/mantle-hardforks/README.md +++ /dev/null @@ -1,59 +0,0 @@ -# Mantle Hardforks - -This crate provides hardfork definitions and utilities specific to the Mantle network, which is built on top of the OP Stack. - -## Overview - -Mantle is a Layer 2 scaling solution that extends the OP Stack with network-specific optimizations and features. This crate defines the hardfork progression for Mantle networks, including both standard OP Stack hardforks and Mantle-specific upgrades. - -## Features - -- **Mantle-specific hardforks**: Defines hardforks unique to the Mantle network -- **OP Stack compatibility**: Maintains full compatibility with the OP Stack hardfork system -- **Chain-specific configurations**: Separate configurations for Mantle mainnet and testnet -- **Timestamp-based activation**: Uses timestamp-based activation for network upgrades - -## Hardforks - -### Mantle-Specific Hardforks - -- **Skadi**: Mantle's Prague-equivalent upgrade that introduces network-specific features and optimizations - -### OP Stack Hardforks - -Mantle inherits all OP Stack hardforks: -- Bedrock -- Regolith -- Canyon -- Ecotone -- Fjord -- Granite -- Holocene -- Isthmus -- Interop - -## Usage - -```rust -use mantle_hardforks::{MantleHardfork, MantleChainHardforks, MANTLE_MAINNET_HARDFORKS}; - -// Check if Skadi is active at a given timestamp -let mantle_forks = MantleChainHardforks::mantle_mainnet(); -let is_skadi_active = mantle_forks.is_skadi_active_at_timestamp(1_756_278_000); - -// Get the full Mantle network hardforks configuration -let network_hardforks = MANTLE_MAINNET_HARDFORKS.clone(); -``` - -## Chain IDs - -- **Mantle Mainnet**: 5000 -- **Mantle Sepolia**: 5001 - -## Activation Timestamps - -- **Skadi**: 1756278000 (August 15, 2025 12:00:00 UTC) - -## License - -This project is licensed under the same terms as the main Reth project. diff --git a/crates/mantle-hardforks/src/debug/mod.rs b/crates/mantle-hardforks/src/debug/mod.rs new file mode 100644 index 00000000000..3f1fb6dcb1b --- /dev/null +++ b/crates/mantle-hardforks/src/debug/mod.rs @@ -0,0 +1,3 @@ +//! Debug utilities for Mantle hardfork development and troubleshooting + +pub mod state_export; diff --git a/crates/mantle-hardforks/src/debug/state_export.rs b/crates/mantle-hardforks/src/debug/state_export.rs new file mode 100644 index 00000000000..bfea78a5e86 --- /dev/null +++ b/crates/mantle-hardforks/src/debug/state_export.rs @@ -0,0 +1,441 @@ +//! State export utilities for debugging Mantle hardfork issues +//! +//! This module provides functionality to export complete blockchain state to JSON files, +//! extracting original keys from `BundleState` for debugging purposes. + +use alloy_primitives::{hex, keccak256, Address, B256, U256}; +use eyre::Result; +use reth_db::tables; +use reth_db_api::cursor::{DbCursorRO, DbDupCursorRO}; +use reth_db_api::transaction::DbTx; +use reth_provider::DBProvider; +use reth_primitives_traits::Account; +use reth_trie::HashedStorage; +use reth_trie_db::{DatabaseHashedCursorFactory, DatabaseStorageRoot, DatabaseTrieCursorFactory}; +use revm::database::BundleState; +use std::collections::BTreeMap; +use std::fs::File; +use std::io::{BufWriter, Write}; + +/// Storage entry for JSON export +#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)] +pub struct StorageEntryExport { + /// Original storage key before hashing + pub original_key: String, + /// Keccak256 hash of the storage key + pub hashed_key: String, + /// Storage slot value + pub value: String, +} + +/// Account data for JSON export +#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)] +pub struct AccountExport { + /// Keccak256 hash of the account address + pub hashed_address: String, + /// Account balance in wei + pub balance: String, + /// Account nonce (transaction count) + pub nonce: u64, + /// Contract bytecode (hex encoded) + pub code: String, + /// Keccak256 hash of the contract code + pub code_hash: String, + /// Root hash of the account's storage trie + pub storage_hash: String, + /// Map of storage entries for this account + pub storage: BTreeMap, +} + +/// Extract storage key mapping from `BundleState` (`hashed_key` -> `original_key`) +fn extract_storage_key_mapping_from_bundle(bundle_state: &BundleState) -> BTreeMap> { + let mut mapping: BTreeMap> = BTreeMap::new(); + + for (address, account) in &bundle_state.state { + let hashed_address = keccak256(address.as_slice()); + + for slot in account.storage.keys() { + let original_key = B256::from(*slot); + let hashed_key = keccak256(original_key.as_slice()); + + mapping.entry(hashed_address) + .or_default() + .insert(hashed_key, original_key); + } + } + + mapping +} + +/// Extract address mapping from `BundleState` (`hashed_address` -> `original_address`) +fn extract_address_mapping_from_bundle(bundle_state: &BundleState) -> BTreeMap { + let mut mapping: BTreeMap = BTreeMap::new(); + + for address in bundle_state.state.keys() { + let hashed_address = keccak256(address.as_slice()); + mapping.insert(hashed_address, *address); + } + + mapping +} + +/// Export full state using `BundleState` +/// +/// This function exports the complete blockchain state by: +/// 1. Reading all accounts from the database +/// 2. Applying `BundleState` changes on top of database state +/// 3. Extracting original keys from `BundleState` for accurate debugging +/// 4. Writing the result to a JSON file in a standardized format +/// +/// # Arguments +/// * `provider` - Database provider for reading state +/// * `bundle_state` - `BundleState` containing execution changes (with original keys) +/// * `filename` - Output filename +/// * `state_root` - Optional state root hash to include in the export. If None, uses zero hash. +/// * `export_bundle_storage_only` - If true, only export storage details for accounts in `bundle_state` +/// +/// # Format +/// The output JSON has the following structure: +/// ```json +/// { +/// "state_root": "0x...", +/// "accounts": { +/// "0xAddress": { +/// "hashed_address": "0x...", +/// "balance": "123", +/// "nonce": 0, +/// "code": "0x...", +/// "code_hash": "0x...", +/// "storage_hash": "0x...", +/// "storage": { +/// "0xKey": { +/// "original_key": "0x...", +/// "hashed_key": "0x...", +/// "value": "0x..." +/// } +/// } +/// } +/// } +/// } +/// ``` +pub fn export_full_state_with_bundle( + provider: &Provider, + bundle_state: &BundleState, + filename: &str, + state_root: Option, + export_bundle_storage_only: bool, +) -> Result<()> +where + Provider: DBProvider, +{ + tracing::info!( + target: "mantle_hardfork::debug", + filename = %filename, + export_bundle_storage_only = export_bundle_storage_only, + "Starting full state export from bundle_state" + ); + + // Step 1: Collect all accounts (database + bundle) + let mut all_accounts: BTreeMap> = BTreeMap::new(); + + // Read from database + let mut hashed_account_cursor = provider.tx_ref().cursor_read::()?; + let mut db_account_count = 0; + + if let Some((hashed_address, account_info)) = hashed_account_cursor.first()? { + all_accounts.insert(hashed_address, Some(account_info)); + db_account_count += 1; + while let Some((hashed_address, account_info)) = hashed_account_cursor.next()? { + all_accounts.insert(hashed_address, Some(account_info)); + db_account_count += 1; + } + } + + tracing::info!(target: "mantle_hardfork::debug", "Loaded {} accounts from database", db_account_count); + + // Apply bundle state changes + for (address, account) in &bundle_state.state { + let hashed_address = keccak256(address.as_slice()); + if let Some(info) = &account.info { + all_accounts.insert(hashed_address, Some(Account::from(info.clone()))); + } else { + all_accounts.insert(hashed_address, None); + } + } + + tracing::info!(target: "mantle_hardfork::debug", "Applied {} account changes from bundle_state", bundle_state.state.len()); + + // Remove deleted accounts + all_accounts.retain(|_, account| account.is_some()); + + tracing::info!(target: "mantle_hardfork::debug", "Total accounts to export: {}", all_accounts.len()); + + // Step 2: Collect address mapping (hashed -> original) + let mut address_mapping: BTreeMap = BTreeMap::new(); + + // First from database + let mut plain_account_cursor = provider.tx_ref().cursor_read::()?; + if let Some((address, _)) = plain_account_cursor.first()? { + let hashed = keccak256(address.as_slice()); + address_mapping.insert(hashed, address); + while let Some((address, _)) = plain_account_cursor.next()? { + let hashed = keccak256(address.as_slice()); + address_mapping.insert(hashed, address); + } + } + + // Then from bundle_state (this will override database entries with correct values) + let bundle_address_mapping = extract_address_mapping_from_bundle(bundle_state); + address_mapping.extend(bundle_address_mapping); + + // Step 3: Collect storage key mapping (for original keys) + let mut storage_key_mapping: BTreeMap> = BTreeMap::new(); + + // First from database + let mut plain_storage_cursor = provider.tx_ref().cursor_dup_read::()?; + let mut current_address: Option
= None; + + while let Some((address, storage_entry)) = + if current_address.is_none() { + plain_storage_cursor.first()? + } else { + plain_storage_cursor.next_no_dup()? + } + { + current_address = Some(address); + let hashed_address = keccak256(address.as_slice()); + let hashed_key = keccak256(storage_entry.key.as_slice()); + + storage_key_mapping.entry(hashed_address) + .or_default() + .insert(hashed_key, storage_entry.key); + + while let Some((_, storage_entry)) = plain_storage_cursor.next_dup()? { + let hashed_key = keccak256(storage_entry.key.as_slice()); + storage_key_mapping.entry(hashed_address) + .or_default() + .insert(hashed_key, storage_entry.key); + } + } + + // Then from bundle_state (this will add new keys that aren't in the database yet) + let bundle_storage_mapping = extract_storage_key_mapping_from_bundle(bundle_state); + for (hashed_addr, keys) in bundle_storage_mapping { + storage_key_mapping.entry(hashed_addr) + .or_default() + .extend(keys); + } + + tracing::info!(target: "mantle_hardfork::debug", "Collected storage key mappings for {} addresses from bundle_state", storage_key_mapping.len()); + + // Step 4: Use provided state root or default to zero + let state_root = state_root.unwrap_or_default(); + tracing::info!(target: "mantle_hardfork::debug", state_root = ?state_root, "Using state root"); + + // Step 5: Stream write to file + let file = File::create(filename)?; + let mut writer = BufWriter::with_capacity(8 * 1024 * 1024, file); + + // Write JSON header with actual state root + write!(writer, "{{\n \"state_root\": \"0x{}\",\n \"accounts\": {{\n", hex::encode(state_root.as_slice()))?; + + let total_accounts = all_accounts.len(); + let mut processed_count = 0; + let mut first = true; + + // Process each account + for (hashed_address, account_opt) in all_accounts { + if let Some(account_info) = account_opt { + processed_count += 1; + + // Get original address first (needed for storage operations) + let original_address = address_mapping.get(&hashed_address).copied().unwrap_or(Address::ZERO); + + // Check if this account is in bundle_state + let is_bundle_account = bundle_state.state.contains_key(&original_address); + + // Determine if we should export storage details for this account + let should_export_storage = !export_bundle_storage_only || is_bundle_account; + + // Collect storage sorted by original_key (not hashed_key) + // We need two maps: one for sorting by original_key, one for calculating storage root + let mut storage_by_original_key: BTreeMap = BTreeMap::new(); + let mut hashed_storage_for_root: BTreeMap = BTreeMap::new(); + + // Read from database and convert hashed_key back to original_key + let mut hashed_storage_cursor = provider.tx_ref().cursor_dup_read::()?; + match hashed_storage_cursor.seek_exact(hashed_address)? { + Some((found_addr, storage_entry)) if found_addr == hashed_address => { + let hashed_key = storage_entry.key; + let value = storage_entry.value; + + // Find original key from mapping + let original_key = storage_key_mapping.get(&hashed_address) + .and_then(|m| m.get(&hashed_key)) + .copied() + .unwrap_or(hashed_key); + + if should_export_storage { + storage_by_original_key.insert(original_key, value); + } + hashed_storage_for_root.insert(hashed_key, value); + + while let Some((_, storage_entry)) = hashed_storage_cursor.next_dup()? { + let hashed_key = storage_entry.key; + let value = storage_entry.value; + + let original_key = storage_key_mapping.get(&hashed_address) + .and_then(|m| m.get(&hashed_key)) + .copied() + .unwrap_or(hashed_key); + + if should_export_storage { + storage_by_original_key.insert(original_key, value); + } + hashed_storage_for_root.insert(hashed_key, value); + } + } + _ => {} + } + + // Apply bundle state changes + if let Some(bundle_account) = address_mapping.get(&hashed_address) + .and_then(|addr| bundle_state.state.get(addr)) + { + // Apply storage changes from bundle + for (slot, slot_value) in &bundle_account.storage { + let original_key = B256::from(*slot); + let hashed_key = keccak256(original_key.as_slice()); + + if slot_value.present_value == U256::ZERO { + if should_export_storage { + storage_by_original_key.remove(&original_key); + } + hashed_storage_for_root.remove(&hashed_key); + } else { + if should_export_storage { + storage_by_original_key.insert(original_key, slot_value.present_value); + } + hashed_storage_for_root.insert(hashed_key, slot_value.present_value); + } + } + } + + // Calculate storage root using the merged storage + // Build HashedStorage from hashed_storage_for_root + let mut hashed_storage = HashedStorage::new(false); + for (hashed_key, value) in &hashed_storage_for_root { + hashed_storage.storage.insert(*hashed_key, *value); + } + + // Calculate storage root with bundle changes included + let storage_hash = if original_address != Address::ZERO && !hashed_storage_for_root.is_empty() { + // Use the trait method through fully qualified syntax + use reth_trie::StorageRoot; + type StorageRootImpl<'a, TX> = StorageRoot, DatabaseHashedCursorFactory<&'a TX>>; + + match as DatabaseStorageRoot<'_, _>>::overlay_root( + provider.tx_ref(), + original_address, + hashed_storage, + ) { + Ok(root) => format!("0x{}", hex::encode(root.as_slice())), + Err(_) => "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421".to_string(), + } + } else if hashed_storage_for_root.is_empty() { + // If storage is empty, use empty root + "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421".to_string() + } else { + // If we don't have the original address, use database calculation + match reth_trie::StorageRoot::from_tx_hashed(provider.tx_ref(), hashed_address).root() { + Ok(root) => format!("0x{}", hex::encode(root.as_slice())), + Err(_) => "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421".to_string(), + } + }; + + // Get bytecode + let (code, code_hash_str) = if let Some(code_hash) = account_info.bytecode_hash { + if code_hash == B256::ZERO { + ("0x".to_string(), format!("0x{}", hex::encode(B256::ZERO.as_slice()))) + } else { + match provider.tx_ref().get_by_encoded_key::(&code_hash) { + Ok(Some(bytecode)) => { + (format!("0x{}", hex::encode(bytecode.original_bytes())), + format!("0x{}", hex::encode(code_hash.as_slice()))) + }, + _ => ("0x".to_string(), format!("0x{}", hex::encode(code_hash.as_slice()))) + } + } + } else { + ("0x".to_string(), format!("0x{}", hex::encode(alloy_primitives::KECCAK256_EMPTY.as_slice()))) + }; + + // Format address string (checksum format) + let address_str = if original_address == Address::ZERO { + format!("hashed:0x{}", hex::encode(hashed_address.as_slice())) + } else { + format!("{:?}", original_address) + }; + + // Write account + if !first { + writer.write_all(b",\n")?; + } + first = false; + + writeln!(writer, " \"{}\": {{", address_str)?; + writeln!(writer, " \"hashed_address\": \"0x{}\",", hex::encode(hashed_address.as_slice()))?; + writeln!(writer, " \"balance\": \"{}\",", account_info.balance)?; + writeln!(writer, " \"nonce\": {},", account_info.nonce)?; + writeln!(writer, " \"code\": \"{}\",", code)?; + writeln!(writer, " \"code_hash\": \"{}\",", code_hash_str)?; + writeln!(writer, " \"storage_hash\": \"{}\",", storage_hash)?; + write!(writer, " \"storage\": {{")?; + + // Write storage (now sorted by original_key) + if should_export_storage { + let mut first_storage = true; + let storage_is_empty = storage_by_original_key.is_empty(); + for (original_key, value) in storage_by_original_key { + let hashed_key = keccak256(original_key.as_slice()); + + if !first_storage { + writer.write_all(b",")?; + } + first_storage = false; + + writeln!(writer, "\n \"0x{}\": {{", hex::encode(original_key.as_slice()))?; + writeln!(writer, " \"original_key\": \"0x{}\",", hex::encode(original_key.as_slice()))?; + writeln!(writer, " \"hashed_key\": \"0x{}\",", hex::encode(hashed_key.as_slice()))?; + writeln!(writer, " \"value\": \"0x{}\"", hex::encode(value.to_be_bytes::<32>()))?; + write!(writer, " }}")?; + } + + if !storage_is_empty { + writer.write_all(b"\n ")?; + } + writer.write_all(b"}\n }}")?; + } else { + // For accounts not in bundle, omit storage details + writer.write_all(b"\"omitted\"}\n }}")?; + } + + if processed_count % 1000 == 0 { + tracing::info!(target: "mantle_hardfork::debug", + "Processed {}/{} accounts ({:.1}%)", + processed_count, total_accounts, + (processed_count as f64 / total_accounts as f64) * 100.0); + } + } + } + + writer.write_all(b"\n }\n}\n")?; + writer.flush()?; + + tracing::info!(target: "mantle_hardfork::debug", + filename = %filename, + total_accounts = total_accounts, + "State export completed successfully"); + + Ok(()) +} diff --git a/crates/mantle-hardforks/src/lib.rs b/crates/mantle-hardforks/src/lib.rs index 245766008a3..2b6d1b29d0e 100644 --- a/crates/mantle-hardforks/src/lib.rs +++ b/crates/mantle-hardforks/src/lib.rs @@ -11,11 +11,19 @@ )] #![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![no_std] +#![cfg_attr(not(feature = "std"), no_std)] +#[cfg(not(feature = "std"))] extern crate alloc; +// Debug utilities (requires std) +#[cfg(feature = "std")] +pub mod debug; + +#[cfg(not(feature = "std"))] use alloc::{vec::Vec}; +#[cfg(feature = "std")] +use std::vec::Vec; use alloy_chains::{Chain, NamedChain}; use alloy_hardforks::{EthereumHardfork, hardfork}; pub use alloy_hardforks::{EthereumHardforks, ForkCondition}; @@ -35,6 +43,9 @@ pub const MANTLE_MAINNET_SKADI_TIMESTAMP: u64 = 1_756_278_000; // Wed Aug 27 202 /// Skadi upgrade timestamp for Mantle Sepolia testnet pub const MANTLE_SEPOLIA_SKADI_TIMESTAMP: u64 = 1_752_649_200; // Wed Jul 16 2025 15:00:00 GMT+0800 +/// Limb upgrade timestamp for Mantle Sepolia testnet +pub const MANTLE_SEPOLIA_LIMB_TIMESTAMP: u64 = 1_764_745_200; // Wed Dec 03 2025 15:00:00 GMT+0800 + hardfork!( /// Mantle-specific hardforks that extend the OP Stack hardfork set. /// @@ -49,6 +60,12 @@ hardfork!( /// while maintaining compatibility with the OP Stack ecosystem. #[default] Skadi, + + /// Limb: Mantle's Osaka-equivalent upgrade + /// + /// This hardfork introduces Mantle-specific features and optimizations + /// while maintaining compatibility with the OP Stack ecosystem. + Limb, } ); @@ -65,7 +82,8 @@ impl MantleHardfork { }), NamedChain::MantleSepolia => Some(match timestamp { _i if timestamp < MANTLE_SEPOLIA_SKADI_TIMESTAMP => Self::Skadi, - _ => Self::Skadi, + _i if timestamp < MANTLE_SEPOLIA_LIMB_TIMESTAMP => Self::Limb, + _ => Self::Limb, }), _ => None, } @@ -79,9 +97,10 @@ impl MantleHardfork { } /// Mantle Sepolia list of hardforks. - pub const fn mantle_sepolia() -> [(Self, ForkCondition); 1] { + pub const fn mantle_sepolia() -> [(Self, ForkCondition); 2] { [ (Self::Skadi, ForkCondition::Timestamp(MANTLE_SEPOLIA_SKADI_TIMESTAMP)), + (Self::Limb, ForkCondition::Timestamp(MANTLE_SEPOLIA_LIMB_TIMESTAMP)), ] } @@ -102,6 +121,31 @@ pub trait MantleHardforks: OpHardforks { fn is_skadi_active_at_timestamp(&self, timestamp: u64) -> bool { self.mantle_fork_activation(MantleHardfork::Skadi).active_at_timestamp(timestamp) } + + /// Returns `true` if [`Limb`](MantleHardfork::Limb) is active at given block timestamp. + fn is_limb_active_at_timestamp(&self, timestamp: u64) -> bool { + self.mantle_fork_activation(MantleHardfork::Limb).active_at_timestamp(timestamp) + } + + /// Returns the revm spec ID for Mantle chains at the given timestamp. + /// + /// This checks Mantle-specific hardforks (like Skadi) first, then falls back + /// to standard OP Stack hardfork detection. + /// + /// # Note + /// + /// This is only intended to be used after Bedrock, when hardforks are activated by timestamp. + fn revm_spec_at_timestamp(&self, timestamp: u64) -> op_revm::OpSpecId { + // Check Mantle Skadi first + if self.is_limb_active_at_timestamp(timestamp) { + op_revm::OpSpecId::OSAKA + } else if self.is_skadi_active_at_timestamp(timestamp) { + op_revm::OpSpecId::ISTHMUS + } else { + // Fall back to OP Stack hardforks + alloy_op_evm::spec_by_timestamp_after_bedrock(self, timestamp) + } + } } /// A type allowing to configure activation [`ForkCondition`]s for a given list of @@ -142,8 +186,20 @@ impl MantleChainHardforks { } impl EthereumHardforks for MantleChainHardforks { - fn ethereum_fork_activation(&self, _fork: EthereumHardfork) -> ForkCondition { - todo!() + fn ethereum_fork_activation(&self, fork: EthereumHardfork) -> ForkCondition { + use EthereumHardfork::{Cancun, Prague, Shanghai}; + use MantleHardfork::Skadi; + + if self.forks.is_empty() { + return ForkCondition::Never; + } + + let forks_len = self.forks.len(); + // check index out of bounds + match fork { + Shanghai|Cancun|Prague if forks_len <= Skadi.idx() => ForkCondition::Never, + _ => self[fork], + } } } @@ -185,120 +241,15 @@ impl Index for MantleChainHardforks { type Output = ForkCondition; fn index(&self, hf: MantleHardfork) -> &Self::Output { - use MantleHardfork::Skadi; + use MantleHardfork::{Skadi, Limb}; match hf { Skadi => &self.forks[Skadi.idx()].1, + Limb => &self.forks[Limb.idx()].1, } } } -// /// Combined hardforks for Mantle networks that include both OP Stack and Mantle-specific hardforks. -// pub struct MantleNetworkHardforks { -// /// OP Stack hardforks -// pub op_hardforks: ChainHardforks, -// /// Mantle-specific hardforks -// pub mantle_hardforks: MantleChainHardforks, -// } - -// impl MantleNetworkHardforks { -// /// Creates a new [`MantleNetworkHardforks`] with Mantle mainnet configuration. -// pub fn mantle_mainnet() -> Self { -// Self { -// op_hardforks: ChainHardforks::new(vec![ -// // Standard Ethereum hardforks -// (EthereumHardfork::Frontier.boxed(), ForkCondition::Block(0)), -// (EthereumHardfork::Homestead.boxed(), ForkCondition::Block(0)), -// (EthereumHardfork::Tangerine.boxed(), ForkCondition::Block(0)), -// (EthereumHardfork::SpuriousDragon.boxed(), ForkCondition::Block(0)), -// (EthereumHardfork::Byzantium.boxed(), ForkCondition::Block(0)), -// (EthereumHardfork::Constantinople.boxed(), ForkCondition::Block(0)), -// (EthereumHardfork::Petersburg.boxed(), ForkCondition::Block(0)), -// (EthereumHardfork::Istanbul.boxed(), ForkCondition::Block(0)), -// (EthereumHardfork::MuirGlacier.boxed(), ForkCondition::Block(0)), -// (EthereumHardfork::Berlin.boxed(), ForkCondition::Block(0)), -// (EthereumHardfork::London.boxed(), ForkCondition::Block(0)), -// (EthereumHardfork::ArrowGlacier.boxed(), ForkCondition::Block(0)), -// (EthereumHardfork::GrayGlacier.boxed(), ForkCondition::Block(0)), -// ( -// EthereumHardfork::Paris.boxed(), -// ForkCondition::TTD { -// activation_block_number: 0, -// total_difficulty: alloy_primitives::U256::ZERO, -// fork_block: Some(0), -// }, -// ), -// // OP Stack hardforks -// (OpHardfork::Bedrock.boxed(), ForkCondition::Block(0)), -// (OpHardfork::Regolith.boxed(), ForkCondition::Timestamp(0)), -// (EthereumHardfork::Shanghai.boxed(), ForkCondition::Timestamp(0)), -// (OpHardfork::Canyon.boxed(), ForkCondition::Timestamp(0)), -// (EthereumHardfork::Cancun.boxed(), ForkCondition::Timestamp(0)), -// (OpHardfork::Ecotone.boxed(), ForkCondition::Timestamp(0)), -// (OpHardfork::Fjord.boxed(), ForkCondition::Timestamp(0)), -// (OpHardfork::Granite.boxed(), ForkCondition::Timestamp(0)), -// (OpHardfork::Holocene.boxed(), ForkCondition::Timestamp(0)), -// (EthereumHardfork::Prague.boxed(), ForkCondition::Timestamp(MANTLE_MAINNET_SKADI_TIMESTAMP)), -// (OpHardfork::Isthmus.boxed(), ForkCondition::Timestamp(MANTLE_MAINNET_SKADI_TIMESTAMP)), -// ]), -// mantle_hardforks: MantleChainHardforks::mantle_mainnet(), -// } -// } - -// /// Creates a new [`MantleNetworkHardforks`] with Mantle Sepolia configuration. -// pub fn mantle_sepolia() -> Self { -// Self { -// op_hardforks: ChainHardforks::new(vec![ -// // Standard Ethereum hardforks -// (EthereumHardfork::Frontier.boxed(), ForkCondition::Block(0)), -// (EthereumHardfork::Homestead.boxed(), ForkCondition::Block(0)), -// (EthereumHardfork::Tangerine.boxed(), ForkCondition::Block(0)), -// (EthereumHardfork::SpuriousDragon.boxed(), ForkCondition::Block(0)), -// (EthereumHardfork::Byzantium.boxed(), ForkCondition::Block(0)), -// (EthereumHardfork::Constantinople.boxed(), ForkCondition::Block(0)), -// (EthereumHardfork::Petersburg.boxed(), ForkCondition::Block(0)), -// (EthereumHardfork::Istanbul.boxed(), ForkCondition::Block(0)), -// (EthereumHardfork::MuirGlacier.boxed(), ForkCondition::Block(0)), -// (EthereumHardfork::Berlin.boxed(), ForkCondition::Block(0)), -// (EthereumHardfork::London.boxed(), ForkCondition::Block(0)), -// (EthereumHardfork::ArrowGlacier.boxed(), ForkCondition::Block(0)), -// (EthereumHardfork::GrayGlacier.boxed(), ForkCondition::Block(0)), -// ( -// EthereumHardfork::Paris.boxed(), -// ForkCondition::TTD { -// activation_block_number: 0, -// total_difficulty: alloy_primitives::U256::ZERO, -// fork_block: Some(0), -// }, -// ), -// // OP Stack hardforks -// (OpHardfork::Bedrock.boxed(), ForkCondition::Block(0)), -// (OpHardfork::Regolith.boxed(), ForkCondition::Timestamp(0)), -// (EthereumHardfork::Shanghai.boxed(), ForkCondition::Timestamp(0)), -// (OpHardfork::Canyon.boxed(), ForkCondition::Timestamp(0)), -// (EthereumHardfork::Cancun.boxed(), ForkCondition::Timestamp(0)), -// (OpHardfork::Ecotone.boxed(), ForkCondition::Timestamp(0)), -// (OpHardfork::Fjord.boxed(), ForkCondition::Timestamp(0)), -// (OpHardfork::Granite.boxed(), ForkCondition::Timestamp(0)), -// (OpHardfork::Holocene.boxed(), ForkCondition::Timestamp(0)), -// (EthereumHardfork::Prague.boxed(), ForkCondition::Timestamp(MANTLE_SEPOLIA_SKADI_TIMESTAMP)), -// (OpHardfork::Isthmus.boxed(), ForkCondition::Timestamp(MANTLE_SEPOLIA_SKADI_TIMESTAMP)), -// ]), -// mantle_hardforks: MantleChainHardforks::mantle_sepolia(), -// } -// } -// } - -// /// Mantle mainnet hardforks configuration. -// pub static MANTLE_MAINNET_HARDFORKS: LazyLock = LazyLock::new(|| { -// MantleNetworkHardforks::mantle_mainnet() -// }); - -// /// Mantle Sepolia hardforks configuration. -// pub static MANTLE_SEPOLIA_HARDFORKS: LazyLock = LazyLock::new(|| { -// MantleNetworkHardforks::mantle_sepolia() -// }); - #[cfg(test)] mod tests { use super::*; @@ -374,16 +325,29 @@ mod tests { ); } - // #[test] - // fn test_mantle_network_hardforks() { - // let mantle_mainnet = MantleNetworkHardforks::mantle_mainnet(); + #[test] + fn test_reverse_lookup_limb_hardfork() { + let mantle_sepolia_chain = Chain::from_id(MANTLE_SEPOLIA_CHAIN_ID); - // // Test OP hardforks are present - // assert!(mantle_mainnet.op_hardforks.fork(EthereumHardfork::Prague).active_at_timestamp(MANTLE_MAINNET_SKADI_TIMESTAMP)); - // assert!(mantle_mainnet.op_hardforks.fork(OpHardfork::Isthmus).active_at_timestamp(MANTLE_MAINNET_SKADI_TIMESTAMP)); + // Test Limb activation + assert_eq!( + MantleHardfork::from_chain_and_timestamp(mantle_sepolia_chain, MANTLE_SEPOLIA_LIMB_TIMESTAMP), + Some(MantleHardfork::Limb) + ); - // // Test Mantle hardforks are present - // assert!(mantle_mainnet.mantle_hardforks.is_skadi_active_at_timestamp(MANTLE_MAINNET_SKADI_TIMESTAMP)); - // assert!(!mantle_mainnet.mantle_hardforks.is_skadi_active_at_timestamp(MANTLE_MAINNET_SKADI_TIMESTAMP - 1)); - // } + // Test before Limb but after Skadi + assert_eq!( + MantleHardfork::from_chain_and_timestamp(mantle_sepolia_chain, MANTLE_SEPOLIA_LIMB_TIMESTAMP - 1), + Some(MantleHardfork::Skadi) + ); + } + + #[test] + fn test_limb_fork_condition() { + let mantle_sepolia_forks = MantleChainHardforks::mantle_sepolia(); + assert_eq!( + mantle_sepolia_forks[MantleHardfork::Limb], + ForkCondition::Timestamp(MANTLE_SEPOLIA_LIMB_TIMESTAMP) + ); + } } diff --git a/crates/metrics/src/common/mpsc.rs b/crates/metrics/src/common/mpsc.rs index 2de8ddf9d53..0b3d66ecb12 100644 --- a/crates/metrics/src/common/mpsc.rs +++ b/crates/metrics/src/common/mpsc.rs @@ -11,7 +11,6 @@ use std::{ use tokio::sync::mpsc::{ self, error::{SendError, TryRecvError, TrySendError}, - OwnedPermit, }; use tokio_util::sync::{PollSendError, PollSender}; @@ -144,11 +143,38 @@ impl MeteredSender { Self { sender, metrics: MeteredSenderMetrics::new(scope) } } - /// Tries to acquire a permit to send a message. + /// Tries to acquire a permit to send a message without waiting. /// /// See also [Sender](mpsc::Sender)'s `try_reserve_owned`. - pub fn try_reserve_owned(&self) -> Result, TrySendError>> { - self.sender.clone().try_reserve_owned() + pub fn try_reserve_owned(self) -> Result, TrySendError> { + let Self { sender, metrics } = self; + sender.try_reserve_owned().map(|permit| OwnedPermit::new(permit, metrics.clone())).map_err( + |err| match err { + TrySendError::Full(sender) => TrySendError::Full(Self { sender, metrics }), + TrySendError::Closed(sender) => TrySendError::Closed(Self { sender, metrics }), + }, + ) + } + + /// Waits to acquire a permit to send a message and return owned permit. + /// + /// See also [Sender](mpsc::Sender)'s `reserve_owned`. + pub async fn reserve_owned(self) -> Result, SendError<()>> { + self.sender.reserve_owned().await.map(|permit| OwnedPermit::new(permit, self.metrics)) + } + + /// Waits to acquire a permit to send a message. + /// + /// See also [Sender](mpsc::Sender)'s `reserve`. + pub async fn reserve(&self) -> Result, SendError<()>> { + self.sender.reserve().await.map(|permit| Permit::new(permit, &self.metrics)) + } + + /// Tries to acquire a permit to send a message without waiting. + /// + /// See also [Sender](mpsc::Sender)'s `try_reserve`. + pub fn try_reserve(&self) -> Result, TrySendError<()>> { + self.sender.try_reserve().map(|permit| Permit::new(permit, &self.metrics)) } /// Returns the underlying [Sender](mpsc::Sender). @@ -193,10 +219,55 @@ impl Clone for MeteredSender { } } +/// A wrapper type around [`OwnedPermit`](mpsc::OwnedPermit) that updates metrics accounting +/// when sending +#[derive(Debug)] +pub struct OwnedPermit { + permit: mpsc::OwnedPermit, + /// Holds metrics for this type + metrics: MeteredSenderMetrics, +} + +impl OwnedPermit { + /// Creates a new [`OwnedPermit`] wrapping the provided [`mpsc::OwnedPermit`] with given metrics + /// handle. + pub const fn new(permit: mpsc::OwnedPermit, metrics: MeteredSenderMetrics) -> Self { + Self { permit, metrics } + } + + /// Sends a value using the reserved capacity and update metrics accordingly. + pub fn send(self, value: T) -> MeteredSender { + let Self { permit, metrics } = self; + metrics.messages_sent_total.increment(1); + MeteredSender { sender: permit.send(value), metrics } + } +} + +/// A wrapper type around [Permit](mpsc::Permit) that updates metrics accounting +/// when sending +#[derive(Debug)] +pub struct Permit<'a, T> { + permit: mpsc::Permit<'a, T>, + metrics_ref: &'a MeteredSenderMetrics, +} + +impl<'a, T> Permit<'a, T> { + /// Creates a new [`Permit`] wrapping the provided [`mpsc::Permit`] with given metrics ref. + pub const fn new(permit: mpsc::Permit<'a, T>, metrics_ref: &'a MeteredSenderMetrics) -> Self { + Self { permit, metrics_ref } + } + + /// Sends a value using the reserved capacity and updates metrics accordingly. + pub fn send(self, value: T) { + self.metrics_ref.messages_sent_total.increment(1); + self.permit.send(value); + } +} + /// A wrapper type around [Receiver](mpsc::Receiver) that updates metrics on receive. #[derive(Debug)] pub struct MeteredReceiver { - /// The [Sender](mpsc::Sender) that this wraps around + /// The [Receiver](mpsc::Receiver) that this wraps around receiver: mpsc::Receiver, /// Holds metrics for this type metrics: MeteredReceiverMetrics, @@ -252,7 +323,7 @@ impl Stream for MeteredReceiver { /// Throughput metrics for [`MeteredSender`] #[derive(Clone, Metrics)] #[metrics(dynamic = true)] -struct MeteredSenderMetrics { +pub struct MeteredSenderMetrics { /// Number of messages sent messages_sent_total: Counter, /// Number of failed message deliveries diff --git a/crates/metrics/src/lib.rs b/crates/metrics/src/lib.rs index a5411b617c9..b647d85e745 100644 --- a/crates/metrics/src/lib.rs +++ b/crates/metrics/src/lib.rs @@ -11,7 +11,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] /// Metrics derive macro. pub use metrics_derive::Metrics; diff --git a/crates/net/banlist/src/lib.rs b/crates/net/banlist/src/lib.rs index 29cf8eb76a4..31b779bc8d5 100644 --- a/crates/net/banlist/src/lib.rs +++ b/crates/net/banlist/src/lib.rs @@ -6,7 +6,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] type PeerId = alloy_primitives::B512; @@ -59,11 +59,11 @@ impl BanList { pub fn evict_peers(&mut self, now: Instant) -> Vec { let mut evicted = Vec::new(); self.banned_peers.retain(|peer, until| { - if let Some(until) = until { - if now > *until { - evicted.push(*peer); - return false - } + if let Some(until) = until && + now > *until + { + evicted.push(*peer); + return false } true }); @@ -74,11 +74,11 @@ impl BanList { pub fn evict_ips(&mut self, now: Instant) -> Vec { let mut evicted = Vec::new(); self.banned_ips.retain(|peer, until| { - if let Some(until) = until { - if now > *until { - evicted.push(*peer); - return false - } + if let Some(until) = until && + now > *until + { + evicted.push(*peer); + return false } true }); @@ -125,11 +125,14 @@ impl BanList { /// Bans the IP until the timestamp. /// /// This does not ban non-global IPs. + /// If the IP is already banned, the timeout will be updated to the new value. pub fn ban_ip_until(&mut self, ip: IpAddr, until: Instant) { self.ban_ip_with(ip, Some(until)); } - /// Bans the peer until the timestamp + /// Bans the peer until the timestamp. + /// + /// If the peer is already banned, the timeout will be updated to the new value. pub fn ban_peer_until(&mut self, node_id: PeerId, until: Instant) { self.ban_peer_with(node_id, Some(until)); } @@ -147,6 +150,8 @@ impl BanList { } /// Bans the peer indefinitely or until the given timeout. + /// + /// If the peer is already banned, the timeout will be updated to the new value. pub fn ban_peer_with(&mut self, node_id: PeerId, until: Option) { self.banned_peers.insert(node_id, until); } @@ -154,6 +159,7 @@ impl BanList { /// Bans the ip indefinitely or until the given timeout. /// /// This does not ban non-global IPs. + /// If the IP is already banned, the timeout will be updated to the new value. pub fn ban_ip_with(&mut self, ip: IpAddr, until: Option) { if is_global(&ip) { self.banned_ips.insert(ip, until); @@ -167,7 +173,7 @@ mod tests { #[test] fn can_ban_unban_peer() { - let peer = PeerId::random(); + let peer = PeerId::new([1; 64]); let mut banlist = BanList::default(); banlist.ban_peer(peer); assert!(banlist.is_banned_peer(&peer)); diff --git a/crates/net/discv4/Cargo.toml b/crates/net/discv4/Cargo.toml index 20691a6d929..fadda2b6348 100644 --- a/crates/net/discv4/Cargo.toml +++ b/crates/net/discv4/Cargo.toml @@ -35,7 +35,6 @@ tracing.workspace = true thiserror.workspace = true parking_lot.workspace = true rand_08 = { workspace = true, optional = true } -generic-array.workspace = true serde = { workspace = true, optional = true } itertools.workspace = true @@ -53,7 +52,6 @@ serde = [ "alloy-primitives/serde", "discv5/serde", "enr/serde", - "generic-array/serde", "parking_lot/serde", "rand_08?/serde", "secp256k1/serde", diff --git a/crates/net/discv4/README.md b/crates/net/discv4/README.md index d5caa0ab429..e8ca01dc6dc 100644 --- a/crates/net/discv4/README.md +++ b/crates/net/discv4/README.md @@ -1,7 +1,7 @@ #

discv4

This is a rust implementation of -the [Discovery v4](https://github.com/ethereum/devp2p/blob/40ab248bf7e017e83cc9812a4e048446709623e8/discv4.md) +the [Discovery v4](https://github.com/ethereum/devp2p/blob/0b3b679be294324eb893340461c7c51fb4c15864/discv4.md) peer discovery protocol. For comparison to Discovery v5, @@ -14,7 +14,7 @@ This is inspired by the [discv5](https://github.com/sigp/discv5) crate and reuse The discovery service continuously attempts to connect to other nodes on the network until it has found enough peers. If UPnP (Universal Plug and Play) is supported by the router the service is running on, it will also accept connections from external nodes. In the discovery protocol, nodes exchange information about where the node can be reached to -eventually establish ``RLPx`` sessions. +eventually establish `RLPx` sessions. ## Trouble Shooting diff --git a/crates/net/discv4/src/lib.rs b/crates/net/discv4/src/lib.rs index c433782ed08..47cdf5f6a44 100644 --- a/crates/net/discv4/src/lib.rs +++ b/crates/net/discv4/src/lib.rs @@ -22,7 +22,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] use crate::{ error::{DecodePacketError, Discv4Error}, @@ -213,12 +213,12 @@ impl Discv4 { /// Binds a new `UdpSocket` and creates the service /// /// ``` - /// # use std::io; /// use reth_discv4::{Discv4, Discv4Config}; /// use reth_network_peers::{pk2id, NodeRecord, PeerId}; /// use secp256k1::SECP256K1; /// use std::{net::SocketAddr, str::FromStr}; - /// # async fn t() -> io::Result<()> { + /// # async fn t() -> std:: io::Result<()> { + /// /// // generate a (random) keypair /// let (secret_key, pk) = SECP256K1.generate_keypair(&mut rand_08::thread_rng()); /// let id = pk2id(&pk); @@ -252,7 +252,12 @@ impl Discv4 { local_node_record.udp_port = local_addr.port(); trace!(target: "discv4", ?local_addr,"opened UDP socket"); - let service = Discv4Service::new(socket, local_addr, local_node_record, secret_key, config); + let mut service = + Discv4Service::new(socket, local_addr, local_node_record, secret_key, config); + + // resolve the external address immediately + service.resolve_external_ip(); + let discv4 = service.handle(); Ok((discv4, service)) } @@ -620,6 +625,15 @@ impl Discv4Service { self.lookup_interval = tokio::time::interval(duration); } + /// Sets the external Ip to the configured external IP if [`NatResolver::ExternalIp`]. + fn resolve_external_ip(&mut self) { + if let Some(r) = &self.resolve_external_ip_interval && + let Some(external_ip) = r.resolver().as_external_ip() + { + self.set_external_ip_addr(external_ip); + } + } + /// Sets the given ip address as the node's external IP in the node record announced in /// discovery pub fn set_external_ip_addr(&mut self, external_ip: IpAddr) { @@ -890,10 +904,10 @@ impl Discv4Service { /// Check if the peer has an active bond. fn has_bond(&self, remote_id: PeerId, remote_ip: IpAddr) -> bool { - if let Some(timestamp) = self.received_pongs.last_pong(remote_id, remote_ip) { - if timestamp.elapsed() < self.config.bond_expiration { - return true - } + if let Some(timestamp) = self.received_pongs.last_pong(remote_id, remote_ip) && + timestamp.elapsed() < self.config.bond_expiration + { + return true } false } @@ -2095,7 +2109,7 @@ impl Default for LookupTargetRotator { } impl LookupTargetRotator { - /// this will return the next node id to lookup + /// This will return the next node id to lookup fn next(&mut self, local: &PeerId) -> PeerId { self.counter += 1; self.counter %= self.interval; @@ -2388,7 +2402,7 @@ pub enum DiscoveryUpdate { /// Node that was removed from the table Removed(PeerId), /// A series of updates - Batch(Vec), + Batch(Vec), } #[cfg(test)] @@ -3035,12 +3049,11 @@ mod tests { loop { tokio::select! { Some(update) = updates.next() => { - if let DiscoveryUpdate::Added(record) = update { - if record.id == peerid_1 { + if let DiscoveryUpdate::Added(record) = update + && record.id == peerid_1 { bootnode_appeared = true; break; } - } } _ = &mut timeout => break, } diff --git a/crates/net/discv4/src/node.rs b/crates/net/discv4/src/node.rs index 242c3883228..7e993ff8333 100644 --- a/crates/net/discv4/src/node.rs +++ b/crates/net/discv4/src/node.rs @@ -1,5 +1,4 @@ use alloy_primitives::keccak256; -use generic_array::GenericArray; use reth_network_peers::{NodeRecord, PeerId}; /// The key type for the table. @@ -15,8 +14,7 @@ impl From for NodeKey { impl From for discv5::Key { fn from(value: NodeKey) -> Self { let hash = keccak256(value.0.as_slice()); - let hash = *GenericArray::from_slice(hash.as_slice()); - Self::new_raw(value, hash) + Self::new_raw(value, hash.0.into()) } } diff --git a/crates/net/discv5/src/config.rs b/crates/net/discv5/src/config.rs index e2a93a2a647..c5677544416 100644 --- a/crates/net/discv5/src/config.rs +++ b/crates/net/discv5/src/config.rs @@ -14,7 +14,7 @@ use discv5::{ }; use reth_ethereum_forks::{EnrForkIdEntry, ForkId}; use reth_network_peers::NodeRecord; -use tracing::warn; +use tracing::debug; use crate::{enr::discv4_id_to_multiaddr_id, filter::MustNotIncludeKeys, NetworkStackId}; @@ -152,10 +152,10 @@ impl ConfigBuilder { /// Adds a comma-separated list of enodes, serialized unsigned node records, to boot nodes. pub fn add_serialized_unsigned_boot_nodes(mut self, enodes: &[&str]) -> Self { for node in enodes { - if let Ok(node) = node.parse() { - if let Ok(node) = BootNode::from_unsigned(node) { - self.bootstrap_nodes.insert(node); - } + if let Ok(node) = node.parse() && + let Ok(node) = BootNode::from_unsigned(node) + { + self.bootstrap_nodes.insert(node); } } @@ -411,14 +411,14 @@ pub fn discv5_sockets_wrt_rlpx_addr( let discv5_socket_ipv6 = discv5_addr_ipv6.map(|ip| SocketAddrV6::new(ip, discv5_port_ipv6, 0, 0)); - if let Some(discv5_addr) = discv5_addr_ipv4 { - if discv5_addr != rlpx_addr { - warn!(target: "net::discv5", - %discv5_addr, - %rlpx_addr, - "Overwriting discv5 IPv4 address with RLPx IPv4 address, limited to one advertised IP address per IP version" - ); - } + if let Some(discv5_addr) = discv5_addr_ipv4 && + discv5_addr != rlpx_addr + { + debug!(target: "net::discv5", + %discv5_addr, + %rlpx_addr, + "Overwriting discv5 IPv4 address with RLPx IPv4 address, limited to one advertised IP address per IP version" + ); } // overwrite discv5 ipv4 addr with RLPx address. this is since there is no @@ -430,14 +430,14 @@ pub fn discv5_sockets_wrt_rlpx_addr( let discv5_socket_ipv4 = discv5_addr_ipv4.map(|ip| SocketAddrV4::new(ip, discv5_port_ipv4)); - if let Some(discv5_addr) = discv5_addr_ipv6 { - if discv5_addr != rlpx_addr { - warn!(target: "net::discv5", - %discv5_addr, - %rlpx_addr, - "Overwriting discv5 IPv6 address with RLPx IPv6 address, limited to one advertised IP address per IP version" - ); - } + if let Some(discv5_addr) = discv5_addr_ipv6 && + discv5_addr != rlpx_addr + { + debug!(target: "net::discv5", + %discv5_addr, + %rlpx_addr, + "Overwriting discv5 IPv6 address with RLPx IPv6 address, limited to one advertised IP address per IP version" + ); } // overwrite discv5 ipv6 addr with RLPx address. this is since there is no diff --git a/crates/net/discv5/src/error.rs b/crates/net/discv5/src/error.rs index c373a17194c..64b2cd73af8 100644 --- a/crates/net/discv5/src/error.rs +++ b/crates/net/discv5/src/error.rs @@ -13,7 +13,7 @@ pub enum Error { #[error("network stack identifier is not configured")] NetworkStackIdNotConfigured, /// Missing key used to identify rlpx network. - #[error("fork missing on enr, key missing")] + #[error("fork missing on enr, key {0:?} and key 'eth' missing")] ForkMissing(&'static [u8]), /// Failed to decode [`ForkId`](reth_ethereum_forks::ForkId) rlp value. #[error("failed to decode fork id, 'eth': {0:?}")] diff --git a/crates/net/discv5/src/filter.rs b/crates/net/discv5/src/filter.rs index a83345a9a5e..def00f54dc3 100644 --- a/crates/net/discv5/src/filter.rs +++ b/crates/net/discv5/src/filter.rs @@ -1,4 +1,4 @@ -//! Predicates to constraint peer lookups. +//! Predicates to constrain peer lookups. use std::collections::HashSet; diff --git a/crates/net/discv5/src/lib.rs b/crates/net/discv5/src/lib.rs index 17fa68754e5..92c7c543a3a 100644 --- a/crates/net/discv5/src/lib.rs +++ b/crates/net/discv5/src/lib.rs @@ -6,7 +6,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] use std::{ collections::HashSet, @@ -83,7 +83,6 @@ impl Discv5 { //////////////////////////////////////////////////////////////////////////////////////////////// /// Adds the node to the table, if it is not already present. - #[expect(clippy::result_large_err)] pub fn add_node(&self, node_record: Enr) -> Result<(), Error> { let EnrCombinedKeyWrapper(enr) = node_record.into(); self.discv5.add_enr(enr).map_err(Error::AddNodeFailed) @@ -320,10 +319,7 @@ impl Discv5 { return None } - // todo: extend for all network stacks in reth-network rlpx logic - let fork_id = (self.fork_key == Some(NetworkStackId::ETH)) - .then(|| self.get_fork_id(enr).ok()) - .flatten(); + let fork_id = self.get_fork_id(enr).ok(); trace!(target: "net::discv5", ?fork_id, @@ -379,7 +375,6 @@ impl Discv5 { /// Returns the [`ForkId`] of the given [`Enr`](discv5::Enr) w.r.t. the local node's network /// stack, if field is set. - #[expect(clippy::result_large_err)] pub fn get_fork_id( &self, enr: &discv5::enr::Enr, @@ -387,7 +382,22 @@ impl Discv5 { let Some(key) = self.fork_key else { return Err(Error::NetworkStackIdNotConfigured) }; let fork_id = enr .get_decodable::(key) - .ok_or(Error::ForkMissing(key))? + .or_else(|| { + (key != NetworkStackId::ETH) + .then(|| { + // Fallback: trying to get fork id from Enr with 'eth' as network stack id + trace!(target: "net::discv5", + key = %String::from_utf8_lossy(key), + "Fork id not found for key, trying 'eth'..." + ); + enr.get_decodable::(NetworkStackId::ETH) + }) + .flatten() + }) + .ok_or({ + trace!(target: "net::discv5", "Fork id not found for 'eth' network stack id"); + Error::ForkMissing(key) + })? .map(Into::into)?; Ok(fork_id) @@ -547,58 +557,54 @@ pub fn spawn_populate_kbuckets_bg( metrics: Discv5Metrics, discv5: Arc, ) { - task::spawn({ - let local_node_id = discv5.local_enr().node_id(); - let lookup_interval = Duration::from_secs(lookup_interval); - let metrics = metrics.discovered_peers; - let mut kbucket_index = MAX_KBUCKET_INDEX; - let pulse_lookup_interval = Duration::from_secs(bootstrap_lookup_interval); - // todo: graceful shutdown - - async move { - // make many fast lookup queries at bootstrap, trying to fill kbuckets at furthest - // log2distance from local node - for i in (0..bootstrap_lookup_countdown).rev() { - let target = discv5::enr::NodeId::random(); + let local_node_id = discv5.local_enr().node_id(); + let lookup_interval = Duration::from_secs(lookup_interval); + let metrics = metrics.discovered_peers; + let mut kbucket_index = MAX_KBUCKET_INDEX; + let pulse_lookup_interval = Duration::from_secs(bootstrap_lookup_interval); + task::spawn(Box::pin(async move { + // make many fast lookup queries at bootstrap, trying to fill kbuckets at furthest + // log2distance from local node + for i in (0..bootstrap_lookup_countdown).rev() { + let target = discv5::enr::NodeId::random(); - trace!(target: "net::discv5", - %target, - bootstrap_boost_runs_countdown=i, - lookup_interval=format!("{:#?}", pulse_lookup_interval), - "starting bootstrap boost lookup query" - ); + trace!(target: "net::discv5", + %target, + bootstrap_boost_runs_countdown=i, + lookup_interval=format!("{:#?}", pulse_lookup_interval), + "starting bootstrap boost lookup query" + ); - lookup(target, &discv5, &metrics).await; + lookup(target, &discv5, &metrics).await; - tokio::time::sleep(pulse_lookup_interval).await; - } + tokio::time::sleep(pulse_lookup_interval).await; + } - // initiate regular lookups to populate kbuckets - loop { - // make sure node is connected to each subtree in the network by target - // selection (ref kademlia) - let target = get_lookup_target(kbucket_index, local_node_id); + // initiate regular lookups to populate kbuckets + loop { + // make sure node is connected to each subtree in the network by target + // selection (ref kademlia) + let target = get_lookup_target(kbucket_index, local_node_id); - trace!(target: "net::discv5", - %target, - lookup_interval=format!("{:#?}", lookup_interval), - "starting periodic lookup query" - ); - - lookup(target, &discv5, &metrics).await; + trace!(target: "net::discv5", + %target, + lookup_interval=format!("{:#?}", lookup_interval), + "starting periodic lookup query" + ); - if kbucket_index > DEFAULT_MIN_TARGET_KBUCKET_INDEX { - // try to populate bucket one step closer - kbucket_index -= 1 - } else { - // start over with bucket furthest away - kbucket_index = MAX_KBUCKET_INDEX - } + lookup(target, &discv5, &metrics).await; - tokio::time::sleep(lookup_interval).await; + if kbucket_index > DEFAULT_MIN_TARGET_KBUCKET_INDEX { + // try to populate bucket one step closer + kbucket_index -= 1 + } else { + // start over with bucket furthest away + kbucket_index = MAX_KBUCKET_INDEX } + + tokio::time::sleep(lookup_interval).await; } - }); + })); } /// Gets the next lookup target, based on which bucket is currently being targeted. @@ -673,6 +679,8 @@ mod test { use ::enr::{CombinedKey, EnrKey}; use rand_08::thread_rng; use reth_chainspec::MAINNET; + use reth_tracing::init_test_tracing; + use std::env; use tracing::trace; fn discv5_noop() -> Discv5 { @@ -905,4 +913,55 @@ mod test { assert_eq!(fork_id, decoded_fork_id); assert_eq!(TCP_PORT, enr.tcp4().unwrap()); // listen config is defaulting to ip mode ipv4 } + + #[test] + fn get_fork_id_with_different_network_stack_ids() { + unsafe { + env::set_var("RUST_LOG", "net::discv5=trace"); + } + init_test_tracing(); + + let fork_id = MAINNET.latest_fork_id(); + let sk = SecretKey::new(&mut thread_rng()); + + // Test 1: ENR with OPEL fork ID, Discv5 configured for OPEL + let enr_with_opel = Enr::builder() + .add_value_rlp( + NetworkStackId::OPEL, + alloy_rlp::encode(EnrForkIdEntry::from(fork_id)).into(), + ) + .build(&sk) + .unwrap(); + + let mut discv5 = discv5_noop(); + discv5.fork_key = Some(NetworkStackId::OPEL); + assert_eq!(discv5.get_fork_id(&enr_with_opel).unwrap(), fork_id); + + // Test 2: ENR with ETH fork ID, Discv5 configured for OPEL (fallback to ETH) + let enr_with_eth = Enr::builder() + .add_value_rlp( + NetworkStackId::ETH, + alloy_rlp::encode(EnrForkIdEntry::from(fork_id)).into(), + ) + .build(&sk) + .unwrap(); + + discv5.fork_key = Some(NetworkStackId::OPEL); + assert_eq!(discv5.get_fork_id(&enr_with_eth).unwrap(), fork_id); + + // Test 3: ENR with neither OPEL nor ETH fork ID (should fail) + let enr_without_network_stack_id = Enr::empty(&sk).unwrap(); + discv5.fork_key = Some(NetworkStackId::OPEL); + assert!(matches!( + discv5.get_fork_id(&enr_without_network_stack_id), + Err(Error::ForkMissing(NetworkStackId::OPEL)) + )); + + // Test 4: discv5 without network stack id configured (should fail) + let discv5 = discv5_noop(); + assert!(matches!( + discv5.get_fork_id(&enr_without_network_stack_id), + Err(Error::NetworkStackIdNotConfigured) + )); + } } diff --git a/crates/net/dns/Cargo.toml b/crates/net/dns/Cargo.toml index ee827cef398..8a04d1517d0 100644 --- a/crates/net/dns/Cargo.toml +++ b/crates/net/dns/Cargo.toml @@ -43,6 +43,7 @@ serde_with = { workspace = true, optional = true } reth-chainspec.workspace = true alloy-rlp.workspace = true alloy-chains.workspace = true +secp256k1 = { workspace = true, features = ["rand"] } tokio = { workspace = true, features = ["sync", "rt", "rt-multi-thread"] } reth-tracing.workspace = true rand.workspace = true diff --git a/crates/net/dns/src/lib.rs b/crates/net/dns/src/lib.rs index 14a78ab7cc6..df597a755e2 100644 --- a/crates/net/dns/src/lib.rs +++ b/crates/net/dns/src/lib.rs @@ -11,7 +11,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] pub use crate::resolver::{DnsResolver, MapResolver, Resolver}; use crate::{ diff --git a/crates/net/dns/src/query.rs b/crates/net/dns/src/query.rs index edf387ec5c6..f64551f42f1 100644 --- a/crates/net/dns/src/query.rs +++ b/crates/net/dns/src/query.rs @@ -80,12 +80,12 @@ impl QueryPool { // queue in new queries if we have capacity 'queries: while self.active_queries.len() < self.rate_limit.limit() as usize { - if self.rate_limit.poll_ready(cx).is_ready() { - if let Some(query) = self.queued_queries.pop_front() { - self.rate_limit.tick(); - self.active_queries.push(query); - continue 'queries - } + if self.rate_limit.poll_ready(cx).is_ready() && + let Some(query) = self.queued_queries.pop_front() + { + self.rate_limit.tick(); + self.active_queries.push(query); + continue 'queries } break } diff --git a/crates/net/downloaders/Cargo.toml b/crates/net/downloaders/Cargo.toml index 9c833e17047..056d809d02f 100644 --- a/crates/net/downloaders/Cargo.toml +++ b/crates/net/downloaders/Cargo.toml @@ -22,16 +22,15 @@ reth-storage-api.workspace = true reth-tasks.workspace = true # optional deps for the test-utils feature -reth-db = { workspace = true, optional = true } -reth-db-api = { workspace = true, optional = true } reth-ethereum-primitives = { workspace = true, optional = true } +reth-provider = { workspace = true, optional = true } reth-testing-utils = { workspace = true, optional = true } # ethereum alloy-consensus.workspace = true alloy-eips.workspace = true alloy-primitives.workspace = true -alloy-rlp.workspace = true +alloy-rlp = { workspace = true, optional = true } # async futures.workspace = true @@ -40,6 +39,7 @@ pin-project.workspace = true tokio = { workspace = true, features = ["sync", "fs", "io-util"] } tokio-stream.workspace = true tokio-util = { workspace = true, features = ["codec"] } +async-compression = { workspace = true, features = ["gzip", "tokio"], optional = true } # metrics reth-metrics.workspace = true @@ -51,13 +51,12 @@ thiserror.workspace = true tracing.workspace = true tempfile = { workspace = true, optional = true } -itertools.workspace = true +itertools = { workspace = true, optional = true } [dev-dependencies] +async-compression = { workspace = true, features = ["gzip", "tokio"] } reth-ethereum-primitives.workspace = true reth-chainspec.workspace = true -reth-db = { workspace = true, features = ["test-utils"] } -reth-db-api.workspace = true reth-consensus = { workspace = true, features = ["test-utils"] } reth-network-p2p = { workspace = true, features = ["test-utils"] } reth-provider = { workspace = true, features = ["test-utils"] } @@ -66,22 +65,18 @@ reth-tracing.workspace = true assert_matches.workspace = true tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } -alloy-rlp.workspace = true -itertools.workspace = true rand.workspace = true - tempfile.workspace = true [features] +default = [] +file-client = ["dep:async-compression", "dep:alloy-rlp", "dep:itertools"] test-utils = [ "tempfile", - "reth-db-api", - "reth-db/test-utils", "reth-consensus/test-utils", "reth-network-p2p/test-utils", "reth-testing-utils", "reth-chainspec/test-utils", - "reth-db-api?/test-utils", "reth-provider/test-utils", "reth-primitives-traits/test-utils", "dep:reth-ethereum-primitives", diff --git a/crates/net/downloaders/src/bodies/bodies.rs b/crates/net/downloaders/src/bodies/bodies.rs index a6e454b0414..5d6bd3cf7f8 100644 --- a/crates/net/downloaders/src/bodies/bodies.rs +++ b/crates/net/downloaders/src/bodies/bodies.rs @@ -21,7 +21,6 @@ use std::{ cmp::Ordering, collections::BinaryHeap, fmt::Debug, - mem, ops::RangeInclusive, pin::Pin, sync::Arc, @@ -143,7 +142,7 @@ where /// Max requests to handle at the same time /// /// This depends on the number of active peers but will always be - /// [`min_concurrent_requests`..`max_concurrent_requests`] + /// `min_concurrent_requests..max_concurrent_requests` #[inline] fn concurrent_request_limit(&self) -> usize { let num_peers = self.client.num_connected_peers(); @@ -215,9 +214,7 @@ where /// Adds a new response to the internal buffer fn buffer_bodies_response(&mut self, response: Vec>) { - // take into account capacity - let size = response.iter().map(BlockResponse::size).sum::() + - response.capacity() * mem::size_of::>(); + let size = response.iter().map(BlockResponse::size).sum::(); let response = OrderedBodiesResponse { resp: response, size }; let response_len = response.len(); @@ -230,7 +227,7 @@ where self.metrics.buffered_responses.set(self.buffered_responses.len() as f64); } - /// Returns a response if it's first block number matches the next expected. + /// Returns a response if its first block number matches the next expected. fn try_next_buffered(&mut self) -> Option>> { if let Some(next) = self.buffered_responses.peek() { let expected = self.next_expected_block_number(); @@ -347,6 +344,12 @@ where // written by external services (e.g. BlockchainTree). tracing::trace!(target: "downloaders::bodies", ?range, prev_range = ?self.download_range, "Download range reset"); info!(target: "downloaders::bodies", count, ?range, "Downloading bodies"); + // Increment out-of-order requests metric if the new start is below the last returned block + if let Some(last_returned) = self.latest_queued_block_number && + *range.start() < last_returned + { + self.metrics.out_of_order_requests.increment(1); + } self.clear(); self.download_range = range; Ok(()) @@ -449,7 +452,7 @@ struct OrderedBodiesResponse { impl OrderedBodiesResponse { #[inline] - fn len(&self) -> usize { + const fn len(&self) -> usize { self.resp.len() } @@ -618,12 +621,8 @@ mod tests { }; use alloy_primitives::B256; use assert_matches::assert_matches; - use reth_chainspec::MAINNET; use reth_consensus::test_utils::TestConsensus; - use reth_db::test_utils::{create_test_rw_db, create_test_static_files_dir}; - use reth_provider::{ - providers::StaticFileProvider, test_utils::MockNodeTypesWithDB, ProviderFactory, - }; + use reth_provider::test_utils::create_test_provider_factory; use reth_testing_utils::generators::{self, random_block_range, BlockRangeParams}; use std::collections::HashMap; @@ -632,25 +631,20 @@ mod tests { #[tokio::test] async fn streams_bodies_in_order() { // Generate some random blocks - let db = create_test_rw_db(); + let factory = create_test_provider_factory(); let (headers, mut bodies) = generate_bodies(0..=19); - insert_headers(db.db(), &headers); + insert_headers(&factory, &headers); let client = Arc::new( TestBodiesClient::default().with_bodies(bodies.clone()).with_should_delay(true), ); - let (_static_dir, static_dir_path) = create_test_static_files_dir(); let mut downloader = BodiesDownloaderBuilder::default() .build::( client.clone(), Arc::new(TestConsensus::default()), - ProviderFactory::::new( - db, - MAINNET.clone(), - StaticFileProvider::read_write(static_dir_path).unwrap(), - ), + factory, ); downloader.set_download_range(0..=19).expect("failed to set download range"); @@ -666,7 +660,7 @@ mod tests { #[tokio::test] async fn requests_correct_number_of_times() { // Generate some random blocks - let db = create_test_rw_db(); + let factory = create_test_provider_factory(); let mut rng = generators::rng(); let blocks = random_block_range( &mut rng, @@ -680,22 +674,17 @@ mod tests { .map(|block| (block.hash(), block.into_body())) .collect::>(); - insert_headers(db.db(), &headers); + insert_headers(&factory, &headers); let request_limit = 10; let client = Arc::new(TestBodiesClient::default().with_bodies(bodies.clone())); - let (_static_dir, static_dir_path) = create_test_static_files_dir(); let mut downloader = BodiesDownloaderBuilder::default() .with_request_limit(request_limit) .build::( client.clone(), Arc::new(TestConsensus::default()), - ProviderFactory::::new( - db, - MAINNET.clone(), - StaticFileProvider::read_write(static_dir_path).unwrap(), - ), + factory, ); downloader.set_download_range(0..=199).expect("failed to set download range"); @@ -708,28 +697,23 @@ mod tests { #[tokio::test] async fn streams_bodies_in_order_after_range_reset() { // Generate some random blocks - let db = create_test_rw_db(); + let factory = create_test_provider_factory(); let (headers, mut bodies) = generate_bodies(0..=99); - insert_headers(db.db(), &headers); + insert_headers(&factory, &headers); let stream_batch_size = 20; let request_limit = 10; let client = Arc::new( TestBodiesClient::default().with_bodies(bodies.clone()).with_should_delay(true), ); - let (_static_dir, static_dir_path) = create_test_static_files_dir(); let mut downloader = BodiesDownloaderBuilder::default() .with_stream_batch_size(stream_batch_size) .with_request_limit(request_limit) .build::( client.clone(), Arc::new(TestConsensus::default()), - ProviderFactory::::new( - db, - MAINNET.clone(), - StaticFileProvider::read_write(static_dir_path).unwrap(), - ), + factory, ); let mut range_start = 0; @@ -750,24 +734,19 @@ mod tests { #[tokio::test] async fn can_download_new_range_after_termination() { // Generate some random blocks - let db = create_test_rw_db(); + let factory = create_test_provider_factory(); let (headers, mut bodies) = generate_bodies(0..=199); - insert_headers(db.db(), &headers); + insert_headers(&factory, &headers); let client = Arc::new(TestBodiesClient::default().with_bodies(bodies.clone())); - let (_static_dir, static_dir_path) = create_test_static_files_dir(); let mut downloader = BodiesDownloaderBuilder::default() .with_stream_batch_size(100) .build::( client.clone(), Arc::new(TestConsensus::default()), - ProviderFactory::::new( - db, - MAINNET.clone(), - StaticFileProvider::read_write(static_dir_path).unwrap(), - ), + factory, ); // Set and download the first range @@ -792,14 +771,13 @@ mod tests { #[tokio::test] async fn can_download_after_exceeding_limit() { // Generate some random blocks - let db = create_test_rw_db(); + let factory = create_test_provider_factory(); let (headers, mut bodies) = generate_bodies(0..=199); - insert_headers(db.db(), &headers); + insert_headers(&factory, &headers); let client = Arc::new(TestBodiesClient::default().with_bodies(bodies.clone())); - let (_static_dir, static_dir_path) = create_test_static_files_dir(); // Set the max buffered block size to 1 byte, to make sure that every response exceeds the // limit let mut downloader = BodiesDownloaderBuilder::default() @@ -809,11 +787,7 @@ mod tests { .build::( client.clone(), Arc::new(TestConsensus::default()), - ProviderFactory::::new( - db, - MAINNET.clone(), - StaticFileProvider::read_write(static_dir_path).unwrap(), - ), + factory, ); // Set and download the entire range @@ -829,16 +803,15 @@ mod tests { #[tokio::test] async fn can_tolerate_empty_responses() { // Generate some random blocks - let db = create_test_rw_db(); + let factory = create_test_provider_factory(); let (headers, mut bodies) = generate_bodies(0..=99); - insert_headers(db.db(), &headers); + insert_headers(&factory, &headers); // respond with empty bodies for every other request. let client = Arc::new( TestBodiesClient::default().with_bodies(bodies.clone()).with_empty_responses(2), ); - let (_static_dir, static_dir_path) = create_test_static_files_dir(); let mut downloader = BodiesDownloaderBuilder::default() .with_request_limit(3) @@ -846,11 +819,7 @@ mod tests { .build::( client.clone(), Arc::new(TestConsensus::default()), - ProviderFactory::::new( - db, - MAINNET.clone(), - StaticFileProvider::read_write(static_dir_path).unwrap(), - ), + factory, ); // Download the requested range diff --git a/crates/net/downloaders/src/bodies/request.rs b/crates/net/downloaders/src/bodies/request.rs index aa10db382a7..2adb8a585c5 100644 --- a/crates/net/downloaders/src/bodies/request.rs +++ b/crates/net/downloaders/src/bodies/request.rs @@ -12,7 +12,6 @@ use reth_network_peers::{PeerId, WithPeerId}; use reth_primitives_traits::{Block, GotExpected, InMemorySize, SealedBlock, SealedHeader}; use std::{ collections::VecDeque, - mem, pin::Pin, sync::Arc, task::{ready, Context, Poll}, @@ -166,11 +165,10 @@ where where C::Body: InMemorySize, { - let bodies_capacity = bodies.capacity(); let bodies_len = bodies.len(); let mut bodies = bodies.into_iter().peekable(); - let mut total_size = bodies_capacity * mem::size_of::(); + let mut total_size = 0; while bodies.peek().is_some() { let next_header = match self.pending_headers.pop_front() { Some(header) => header, @@ -178,8 +176,6 @@ where }; if next_header.is_empty() { - // increment empty block body metric - total_size += mem::size_of::(); self.buffer.push(BlockResponse::Empty(next_header)); } else { let next_body = bodies.next().unwrap(); diff --git a/crates/net/downloaders/src/bodies/task.rs b/crates/net/downloaders/src/bodies/task.rs index df1d5540db3..4da5946fffb 100644 --- a/crates/net/downloaders/src/bodies/task.rs +++ b/crates/net/downloaders/src/bodies/task.rs @@ -190,7 +190,7 @@ mod tests { let factory = create_test_provider_factory(); let (headers, mut bodies) = generate_bodies(0..=19); - insert_headers(factory.db_ref().db(), &headers); + insert_headers(&factory, &headers); let client = Arc::new( TestBodiesClient::default().with_bodies(bodies.clone()).with_should_delay(true), diff --git a/crates/net/downloaders/src/bodies/test_utils.rs b/crates/net/downloaders/src/bodies/test_utils.rs index aeb4488eb0d..513226a2c91 100644 --- a/crates/net/downloaders/src/bodies/test_utils.rs +++ b/crates/net/downloaders/src/bodies/test_utils.rs @@ -4,11 +4,13 @@ use alloy_consensus::BlockHeader; use alloy_primitives::B256; -use reth_db::DatabaseEnv; -use reth_db_api::{database::Database, tables, transaction::DbTxMut}; use reth_ethereum_primitives::BlockBody; use reth_network_p2p::bodies::response::BlockResponse; use reth_primitives_traits::{Block, SealedBlock, SealedHeader}; +use reth_provider::{ + test_utils::MockNodeTypesWithDB, ProviderFactory, StaticFileProviderFactory, StaticFileSegment, + StaticFileWriter, +}; use std::collections::HashMap; pub(crate) fn zip_blocks<'a, B: Block>( @@ -42,12 +44,19 @@ pub(crate) fn create_raw_bodies( } #[inline] -pub(crate) fn insert_headers(db: &DatabaseEnv, headers: &[SealedHeader]) { - db.update(|tx| { - for header in headers { - tx.put::(header.number, header.hash()).unwrap(); - tx.put::(header.number, header.clone_header()).unwrap(); - } - }) - .expect("failed to commit") +pub(crate) fn insert_headers( + factory: &ProviderFactory, + headers: &[SealedHeader], +) { + let provider_rw = factory.provider_rw().expect("failed to create provider"); + let static_file_provider = provider_rw.static_file_provider(); + let mut writer = static_file_provider + .latest_writer(StaticFileSegment::Headers) + .expect("failed to create writer"); + + for header in headers { + writer.append_header(header.header(), &header.hash()).expect("failed to append header"); + } + drop(writer); + provider_rw.commit().expect("failed to commit"); } diff --git a/crates/net/downloaders/src/file_client.rs b/crates/net/downloaders/src/file_client.rs index d1026954e90..4d545aec178 100644 --- a/crates/net/downloaders/src/file_client.rs +++ b/crates/net/downloaders/src/file_client.rs @@ -1,6 +1,7 @@ use alloy_consensus::BlockHeader; use alloy_eips::BlockHashOrNumber; use alloy_primitives::{BlockHash, BlockNumber, Sealable, B256}; +use async_compression::tokio::bufread::GzipDecoder; use futures::Future; use itertools::Either; use reth_consensus::{Consensus, ConsensusError}; @@ -14,9 +15,12 @@ use reth_network_p2p::{ }; use reth_network_peers::PeerId; use reth_primitives_traits::{Block, BlockBody, FullBlock, SealedBlock, SealedHeader}; -use std::{collections::HashMap, io, path::Path, sync::Arc}; +use std::{collections::HashMap, io, ops::RangeInclusive, path::Path, sync::Arc}; use thiserror::Error; -use tokio::{fs::File, io::AsyncReadExt}; +use tokio::{ + fs::File, + io::{AsyncReadExt, BufReader}, +}; use tokio_stream::StreamExt; use tokio_util::codec::FramedRead; use tracing::{debug, trace, warn}; @@ -354,10 +358,11 @@ impl BodiesClient for FileClient { type Body = B::Body; type Output = BodiesFut; - fn get_block_bodies_with_priority( + fn get_block_bodies_with_priority_and_range_hint( &self, hashes: Vec, _priority: Priority, + _range_hint: Option>, ) -> Self::Output { // this just searches the buffer, and fails if it can't find the block let mut bodies = Vec::new(); @@ -391,114 +396,181 @@ impl BlockClient for FileClient { type Block = B; } -/// Chunks file into several [`FileClient`]s. +/// File reader type for handling different compression formats. #[derive(Debug)] -pub struct ChunkedFileReader { - /// File to read from. - file: File, - /// Current file byte length. - file_byte_len: u64, - /// Bytes that have been read. - chunk: Vec, - /// Max bytes per chunk. - chunk_byte_len: u64, - /// Optionally, tracks highest decoded block number. Needed when decoding data that maps * to 1 - /// with block number - highest_block: Option, +enum FileReader { + /// Regular uncompressed file with remaining byte tracking. + Plain { file: File, remaining_bytes: u64 }, + /// Gzip compressed file. + Gzip(GzipDecoder>), } -impl ChunkedFileReader { - /// Returns the remaining file length. - pub const fn file_len(&self) -> u64 { - self.file_byte_len - } - - /// Opens the file to import from given path. Returns a new instance. If no chunk byte length - /// is passed, chunks have [`DEFAULT_BYTE_LEN_CHUNK_CHAIN_FILE`] (one static file). - pub async fn new>( - path: P, - chunk_byte_len: Option, - ) -> Result { - let file = File::open(path).await?; - let chunk_byte_len = chunk_byte_len.unwrap_or(DEFAULT_BYTE_LEN_CHUNK_CHAIN_FILE); - - Self::from_file(file, chunk_byte_len).await - } - - /// Opens the file to import from given path. Returns a new instance. - pub async fn from_file(file: File, chunk_byte_len: u64) -> Result { - // get file len from metadata before reading - let metadata = file.metadata().await?; - let file_byte_len = metadata.len(); - - Ok(Self { file, file_byte_len, chunk: vec![], chunk_byte_len, highest_block: None }) +impl FileReader { + /// Read some data into the provided buffer, returning the number of bytes read. + async fn read(&mut self, buf: &mut [u8]) -> Result { + match self { + Self::Plain { file, .. } => file.read(buf).await, + Self::Gzip(decoder) => decoder.read(buf).await, + } } - /// Calculates the number of bytes to read from the chain file. Returns a tuple of the chunk - /// length and the remaining file length. - fn chunk_len(&self) -> u64 { - let Self { chunk_byte_len, file_byte_len, .. } = *self; - let file_byte_len = file_byte_len + self.chunk.len() as u64; - - if chunk_byte_len > file_byte_len { - // last chunk - file_byte_len - } else { - chunk_byte_len + /// Read next chunk from file. Returns the number of bytes read for plain files, + /// or a boolean indicating if data is available for gzip files. + async fn read_next_chunk( + &mut self, + chunk: &mut Vec, + chunk_byte_len: u64, + ) -> Result, FileClientError> { + match self { + Self::Plain { .. } => self.read_plain_chunk(chunk, chunk_byte_len).await, + Self::Gzip(_) => { + Ok((self.read_gzip_chunk(chunk, chunk_byte_len).await?) + .then_some(chunk.len() as u64)) + } } } - /// Reads bytes from file and buffers as next chunk to decode. Returns byte length of next - /// chunk to read. - async fn read_next_chunk(&mut self) -> Result, io::Error> { - if self.file_byte_len == 0 && self.chunk.is_empty() { + async fn read_plain_chunk( + &mut self, + chunk: &mut Vec, + chunk_byte_len: u64, + ) -> Result, FileClientError> { + let Self::Plain { file, remaining_bytes } = self else { + unreachable!("read_plain_chunk should only be called on Plain variant") + }; + + if *remaining_bytes == 0 && chunk.is_empty() { // eof return Ok(None) } - let chunk_target_len = self.chunk_len(); - let old_bytes_len = self.chunk.len() as u64; + let chunk_target_len = chunk_byte_len.min(*remaining_bytes + chunk.len() as u64); + let old_bytes_len = chunk.len() as u64; // calculate reserved space in chunk let new_read_bytes_target_len = chunk_target_len - old_bytes_len; // read new bytes from file - let prev_read_bytes_len = self.chunk.len(); - self.chunk.extend(std::iter::repeat_n(0, new_read_bytes_target_len as usize)); - let reader = &mut self.chunk[prev_read_bytes_len..]; + let prev_read_bytes_len = chunk.len(); + chunk.extend(std::iter::repeat_n(0, new_read_bytes_target_len as usize)); + let reader = &mut chunk[prev_read_bytes_len..]; // actual bytes that have been read - let new_read_bytes_len = self.file.read_exact(reader).await? as u64; - let next_chunk_byte_len = self.chunk.len(); + let new_read_bytes_len = file.read_exact(reader).await? as u64; + let next_chunk_byte_len = chunk.len(); // update remaining file length - self.file_byte_len -= new_read_bytes_len; + *remaining_bytes -= new_read_bytes_len; debug!(target: "downloaders::file", - max_chunk_byte_len=self.chunk_byte_len, + max_chunk_byte_len=chunk_byte_len, prev_read_bytes_len, new_read_bytes_target_len, new_read_bytes_len, next_chunk_byte_len, - remaining_file_byte_len=self.file_byte_len, + remaining_file_byte_len=*remaining_bytes, "new bytes were read from file" ); Ok(Some(next_chunk_byte_len as u64)) } + /// Read next chunk from gzipped file. + async fn read_gzip_chunk( + &mut self, + chunk: &mut Vec, + chunk_byte_len: u64, + ) -> Result { + let mut buffer = vec![0u8; 64 * 1024]; + loop { + if chunk.len() >= chunk_byte_len as usize { + return Ok(true) + } + + match self.read(&mut buffer).await { + Ok(0) => return Ok(!chunk.is_empty()), + Ok(n) => { + chunk.extend_from_slice(&buffer[..n]); + } + Err(e) => return Err(e.into()), + } + } + } +} + +/// Chunks file into several [`FileClient`]s. +#[derive(Debug)] +pub struct ChunkedFileReader { + /// File reader (either plain or gzip). + file: FileReader, + /// Bytes that have been read. + chunk: Vec, + /// Max bytes per chunk. + chunk_byte_len: u64, + /// Optionally, tracks highest decoded block number. Needed when decoding data that maps * to 1 + /// with block number + highest_block: Option, +} + +impl ChunkedFileReader { + /// Opens the file to import from given path. Returns a new instance. If no chunk byte length + /// is passed, chunks have [`DEFAULT_BYTE_LEN_CHUNK_CHAIN_FILE`] (one static file). + /// Automatically detects gzip files by extension (.gz, .gzip). + pub async fn new>( + path: P, + chunk_byte_len: Option, + ) -> Result { + let path = path.as_ref(); + let file = File::open(path).await?; + let chunk_byte_len = chunk_byte_len.unwrap_or(DEFAULT_BYTE_LEN_CHUNK_CHAIN_FILE); + + Self::from_file( + file, + chunk_byte_len, + path.extension() + .and_then(|ext| ext.to_str()) + .is_some_and(|ext| ["gz", "gzip"].contains(&ext)), + ) + .await + } + + /// Opens the file to import from given path. Returns a new instance. + pub async fn from_file( + file: File, + chunk_byte_len: u64, + is_gzip: bool, + ) -> Result { + let file_reader = if is_gzip { + FileReader::Gzip(GzipDecoder::new(BufReader::new(file))) + } else { + let remaining_bytes = file.metadata().await?.len(); + FileReader::Plain { file, remaining_bytes } + }; + + Ok(Self { file: file_reader, chunk: vec![], chunk_byte_len, highest_block: None }) + } + + /// Reads bytes from file and buffers as next chunk to decode. Returns byte length of next + /// chunk to read. + async fn read_next_chunk(&mut self) -> Result, FileClientError> { + self.file.read_next_chunk(&mut self.chunk, self.chunk_byte_len).await + } + /// Read next chunk from file. Returns [`FileClient`] containing decoded chunk. + /// + /// For gzipped files, this method accumulates data until at least `chunk_byte_len` bytes + /// are available before processing. For plain files, it uses the original chunking logic. pub async fn next_chunk( &mut self, consensus: Arc>, parent_header: Option>, ) -> Result>, FileClientError> { - let Some(next_chunk_byte_len) = self.read_next_chunk().await? else { return Ok(None) }; + let Some(chunk_len) = self.read_next_chunk().await? else { return Ok(None) }; // make new file client from chunk let DecodedFileChunk { file_client, remaining_bytes, .. } = FileClientBuilder { consensus, parent_header } - .build(&self.chunk[..], next_chunk_byte_len) + .build(&self.chunk[..], chunk_len) .await?; // save left over bytes @@ -512,7 +584,15 @@ impl ChunkedFileReader { where T: FromReceiptReader, { - let Some(next_chunk_byte_len) = self.read_next_chunk().await? else { return Ok(None) }; + let Some(next_chunk_byte_len) = self.read_next_chunk().await.map_err(|e| { + T::Error::from(match e { + FileClientError::Io(io_err) => io_err, + _ => io::Error::other(e.to_string()), + }) + })? + else { + return Ok(None) + }; // make new file client from chunk let DecodedFileChunk { file_client, remaining_bytes, highest_block } = @@ -571,6 +651,7 @@ mod tests { test_utils::{generate_bodies, generate_bodies_file}, }; use assert_matches::assert_matches; + use async_compression::tokio::write::GzipEncoder; use futures_util::stream::StreamExt; use rand::Rng; use reth_consensus::{noop::NoopConsensus, test_utils::TestConsensus}; @@ -581,6 +662,10 @@ mod tests { }; use reth_provider::test_utils::create_test_provider_factory; use std::sync::Arc; + use tokio::{ + fs::File, + io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt, SeekFrom}, + }; #[tokio::test] async fn streams_bodies_from_buffer() { @@ -588,7 +673,7 @@ mod tests { let factory = create_test_provider_factory(); let (headers, mut bodies) = generate_bodies(0..=19); - insert_headers(factory.db_ref().db(), &headers); + insert_headers(&factory, &headers); // create an empty file let file = tempfile::tempfile().unwrap(); @@ -683,7 +768,7 @@ mod tests { Arc::new(FileClient::from_file(file, NoopConsensus::arc()).await.unwrap()); // insert headers in db for the bodies downloader - insert_headers(factory.db_ref().db(), &headers); + insert_headers(&factory, &headers); let mut downloader = BodiesDownloaderBuilder::default().build::( client.clone(), @@ -711,7 +796,8 @@ mod tests { trace!(target: "downloaders::file::test", chunk_byte_len); // init reader - let mut reader = ChunkedFileReader::from_file(file, chunk_byte_len as u64).await.unwrap(); + let mut reader = + ChunkedFileReader::from_file(file, chunk_byte_len as u64, false).await.unwrap(); let mut downloaded_headers: Vec = vec![]; @@ -727,7 +813,82 @@ mod tests { // construct headers downloader and use first header let mut header_downloader = ReverseHeadersDownloaderBuilder::default() - .build(Arc::clone(&Arc::new(client)), Arc::new(TestConsensus::default())); + .build(Arc::new(client), Arc::new(TestConsensus::default())); + header_downloader.update_local_head(local_header.clone()); + header_downloader.update_sync_target(SyncTarget::Tip(sync_target_hash)); + + // get headers first + let mut downloaded_headers_chunk = header_downloader.next().await.unwrap().unwrap(); + + // export new local header to outer scope + local_header = sync_target; + + // reverse to make sure it's in the right order before comparing + downloaded_headers_chunk.reverse(); + downloaded_headers.extend_from_slice(&downloaded_headers_chunk); + } + + // the first header is not included in the response + assert_eq!(headers[1..], downloaded_headers); + } + + #[tokio::test] + async fn test_chunk_download_headers_from_gzip_file() { + reth_tracing::init_test_tracing(); + + // Generate some random blocks + let (file, headers, _) = generate_bodies_file(0..=14).await; + + // Create a gzipped version of the file + let gzip_temp_file = tempfile::NamedTempFile::new().unwrap(); + let gzip_path = gzip_temp_file.path().to_owned(); + drop(gzip_temp_file); // Close the file so we can write to it + + // Read original file content first + let mut original_file = file; + original_file.seek(SeekFrom::Start(0)).await.unwrap(); + let mut original_content = Vec::new(); + original_file.read_to_end(&mut original_content).await.unwrap(); + + let mut gzip_file = File::create(&gzip_path).await.unwrap(); + let mut encoder = GzipEncoder::new(&mut gzip_file); + + // Write the original content through the gzip encoder + encoder.write_all(&original_content).await.unwrap(); + encoder.shutdown().await.unwrap(); + drop(gzip_file); + + // Reopen the gzipped file for reading + let gzip_file = File::open(&gzip_path).await.unwrap(); + + // calculate min for chunk byte length range, pick a lower bound that guarantees at least + // one block will be read + let chunk_byte_len = rand::rng().random_range(2000..=10_000); + trace!(target: "downloaders::file::test", chunk_byte_len); + + // init reader with gzip=true + let mut reader = + ChunkedFileReader::from_file(gzip_file, chunk_byte_len as u64, true).await.unwrap(); + + let mut downloaded_headers: Vec = vec![]; + + let mut local_header = headers.first().unwrap().clone(); + + // test + while let Some(client) = + reader.next_chunk::(NoopConsensus::arc(), None).await.unwrap() + { + if client.headers_len() == 0 { + continue; + } + + let sync_target = client.tip_header().expect("tip_header should not be None"); + + let sync_target_hash = sync_target.hash(); + + // construct headers downloader and use first header + let mut header_downloader = ReverseHeadersDownloaderBuilder::default() + .build(Arc::new(client), Arc::new(TestConsensus::default())); header_downloader.update_local_head(local_header.clone()); header_downloader.update_sync_target(SyncTarget::Tip(sync_target_hash)); diff --git a/crates/net/downloaders/src/headers/reverse_headers.rs b/crates/net/downloaders/src/headers/reverse_headers.rs index 5f27467cf83..cb6b36c9ff9 100644 --- a/crates/net/downloaders/src/headers/reverse_headers.rs +++ b/crates/net/downloaders/src/headers/reverse_headers.rs @@ -148,7 +148,7 @@ where /// Max requests to handle at the same time /// /// This depends on the number of active peers but will always be - /// [`min_concurrent_requests`..`max_concurrent_requests`] + /// `min_concurrent_requests..max_concurrent_requests` #[inline] fn concurrent_request_limit(&self) -> usize { let num_peers = self.client.num_connected_peers(); @@ -172,19 +172,16 @@ where /// /// Returns `None` if no more requests are required. fn next_request(&mut self) -> Option { - if let Some(local_head) = self.local_block_number() { - if self.next_request_block_number > local_head { - let request = calc_next_request( - local_head, - self.next_request_block_number, - self.request_limit, - ); - // need to shift the tracked request block number based on the number of requested - // headers so follow-up requests will use that as start. - self.next_request_block_number -= request.limit; - - return Some(request) - } + if let Some(local_head) = self.local_block_number() && + self.next_request_block_number > local_head + { + let request = + calc_next_request(local_head, self.next_request_block_number, self.request_limit); + // need to shift the tracked request block number based on the number of requested + // headers so follow-up requests will use that as start. + self.next_request_block_number -= request.limit; + + return Some(request) } None diff --git a/crates/net/downloaders/src/lib.rs b/crates/net/downloaders/src/lib.rs index 8d50b6fbc03..90d9709ebe0 100644 --- a/crates/net/downloaders/src/lib.rs +++ b/crates/net/downloaders/src/lib.rs @@ -3,6 +3,7 @@ //! ## Feature Flags //! //! - `test-utils`: Export utilities for testing +//! - `file-client`: Enables the file-based clients for reading blocks and receipts from files. #![doc( html_logo_url = "https://raw.githubusercontent.com/paradigmxyz/reth/main/assets/reth-docs.png", @@ -10,7 +11,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] /// The collection of algorithms for downloading block bodies. pub mod bodies; @@ -25,20 +26,24 @@ pub mod metrics; /// /// Contains [`FileClient`](file_client::FileClient) to read block data from files, /// efficiently buffering headers and bodies for retrieval. +#[cfg(any(test, feature = "file-client"))] pub mod file_client; /// Module managing file-based data retrieval and buffering of receipts. /// /// Contains [`ReceiptFileClient`](receipt_file_client::ReceiptFileClient) to read receipt data from /// files, efficiently buffering receipts for retrieval. +#[cfg(any(test, feature = "file-client"))] pub mod receipt_file_client; /// Module with a codec for reading and encoding block bodies in files. /// /// Enables decoding and encoding `Block` types within file contexts. +#[cfg(any(test, feature = "file-client"))] pub mod file_codec; #[cfg(any(test, feature = "test-utils"))] pub mod test_utils; +#[cfg(any(test, feature = "file-client"))] pub use file_client::{DecodedFileChunk, FileClientError}; diff --git a/crates/net/downloaders/src/test_utils/bodies_client.rs b/crates/net/downloaders/src/test_utils/bodies_client.rs index fed86b989d1..103557a6162 100644 --- a/crates/net/downloaders/src/test_utils/bodies_client.rs +++ b/crates/net/downloaders/src/test_utils/bodies_client.rs @@ -9,6 +9,7 @@ use reth_network_peers::PeerId; use std::{ collections::HashMap, fmt::Debug, + ops::RangeInclusive, sync::{ atomic::{AtomicU64, Ordering}, Arc, @@ -60,7 +61,7 @@ impl TestBodiesClient { /// empty_response_mod == 0`. pub(crate) fn should_respond_empty(&self) -> bool { if let Some(empty_response_mod) = self.empty_response_mod { - self.times_requested.load(Ordering::Relaxed) % empty_response_mod == 0 + self.times_requested.load(Ordering::Relaxed).is_multiple_of(empty_response_mod) } else { false } @@ -81,10 +82,11 @@ impl BodiesClient for TestBodiesClient { type Body = BlockBody; type Output = BodiesFut; - fn get_block_bodies_with_priority( + fn get_block_bodies_with_priority_and_range_hint( &self, hashes: Vec, _priority: Priority, + _range_hint: Option>, ) -> Self::Output { let should_delay = self.should_delay; let bodies = self.bodies.clone(); diff --git a/crates/net/downloaders/src/test_utils/mod.rs b/crates/net/downloaders/src/test_utils/mod.rs index 159859779e0..d945573b93d 100644 --- a/crates/net/downloaders/src/test_utils/mod.rs +++ b/crates/net/downloaders/src/test_utils/mod.rs @@ -2,6 +2,7 @@ #![allow(dead_code)] +#[cfg(any(test, feature = "file-client"))] use crate::{bodies::test_utils::create_raw_bodies, file_codec::BlockFileCodec}; use alloy_primitives::B256; use futures::SinkExt; @@ -37,6 +38,7 @@ pub(crate) fn generate_bodies( /// Generate a set of bodies, write them to a temporary file, and return the file along with the /// bodies and corresponding block hashes +#[cfg(any(test, feature = "file-client"))] pub(crate) async fn generate_bodies_file( range: RangeInclusive, ) -> (tokio::fs::File, Vec, HashMap) { diff --git a/crates/net/ecies/src/algorithm.rs b/crates/net/ecies/src/algorithm.rs index 350cd3f7ed4..dae5e501695 100644 --- a/crates/net/ecies/src/algorithm.rs +++ b/crates/net/ecies/src/algorithm.rs @@ -499,7 +499,7 @@ impl ECIES { } /// Read and verify an auth message from the input data. - #[tracing::instrument(skip_all)] + #[tracing::instrument(level = "trace", skip_all)] pub fn read_auth(&mut self, data: &mut [u8]) -> Result<(), ECIESError> { self.remote_init_msg = Some(Bytes::copy_from_slice(data)); let unencrypted = self.decrypt_message(data)?; @@ -571,7 +571,7 @@ impl ECIES { } /// Read and verify an ack message from the input data. - #[tracing::instrument(skip_all)] + #[tracing::instrument(level = "trace", skip_all)] pub fn read_ack(&mut self, data: &mut [u8]) -> Result<(), ECIESError> { self.remote_init_msg = Some(Bytes::copy_from_slice(data)); let unencrypted = self.decrypt_message(data)?; diff --git a/crates/net/ecies/src/codec.rs b/crates/net/ecies/src/codec.rs index b5a10284cf2..73c3469cd2f 100644 --- a/crates/net/ecies/src/codec.rs +++ b/crates/net/ecies/src/codec.rs @@ -58,7 +58,7 @@ impl Decoder for ECIESCodec { type Item = IngressECIESValue; type Error = ECIESError; - #[instrument(level = "trace", skip_all, fields(peer=?self.ecies.remote_id, state=?self.state))] + #[instrument(level = "trace", target = "net::ecies", skip_all, fields(peer=?self.ecies.remote_id, state=?self.state))] fn decode(&mut self, buf: &mut BytesMut) -> Result, Self::Error> { loop { match self.state { @@ -110,7 +110,7 @@ impl Decoder for ECIESCodec { self.ecies.read_header(&mut buf.split_to(ECIES::header_len()))?; if body_size > MAX_INITIAL_HANDSHAKE_SIZE { - trace!(?body_size, max=?MAX_INITIAL_HANDSHAKE_SIZE, "Header exceeds max initial handshake size"); + trace!(?body_size, max=?MAX_INITIAL_HANDSHAKE_SIZE, "Body exceeds max initial handshake size"); return Err(ECIESErrorImpl::InitialHeaderBodyTooLarge { body_size, max_body_size: MAX_INITIAL_HANDSHAKE_SIZE, @@ -150,7 +150,7 @@ impl Decoder for ECIESCodec { impl Encoder for ECIESCodec { type Error = io::Error; - #[instrument(level = "trace", skip(self, buf), fields(peer=?self.ecies.remote_id, state=?self.state))] + #[instrument(level = "trace", target = "net::ecies", skip(self, buf), fields(peer=?self.ecies.remote_id, state=?self.state))] fn encode(&mut self, item: EgressECIESValue, buf: &mut BytesMut) -> Result<(), Self::Error> { match item { EgressECIESValue::Auth => { diff --git a/crates/net/ecies/src/error.rs b/crates/net/ecies/src/error.rs index 9dabfc16183..a93b731fee6 100644 --- a/crates/net/ecies/src/error.rs +++ b/crates/net/ecies/src/error.rs @@ -33,7 +33,7 @@ pub enum ECIESErrorImpl { #[error(transparent)] IO(std::io::Error), /// Error when checking the HMAC tag against the tag on the message being decrypted - #[error("tag check failure in read_header")] + #[error("tag check failure in decrypt_message")] TagCheckDecryptFailed, /// Error when checking the HMAC tag against the tag on the header #[error("tag check failure in read_header")] @@ -47,8 +47,8 @@ pub enum ECIESErrorImpl { /// Error when parsing ACK data #[error("invalid ack data")] InvalidAckData, - /// Error when reading the header if its length is <3 - #[error("invalid body data")] + /// Error when reading/parsing the `RLPx` header + #[error("invalid header")] InvalidHeader, /// Error when interacting with secp256k1 #[error(transparent)] diff --git a/crates/net/ecies/src/lib.rs b/crates/net/ecies/src/lib.rs index b2dcdac6709..0876356b19c 100644 --- a/crates/net/ecies/src/lib.rs +++ b/crates/net/ecies/src/lib.rs @@ -6,7 +6,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] pub mod algorithm; pub mod mac; diff --git a/crates/net/ecies/src/stream.rs b/crates/net/ecies/src/stream.rs index 9915fc42e6a..d99422f512f 100644 --- a/crates/net/ecies/src/stream.rs +++ b/crates/net/ecies/src/stream.rs @@ -40,7 +40,7 @@ where Io: AsyncRead + AsyncWrite + Unpin, { /// Connect to an `ECIES` server - #[instrument(skip(transport, secret_key))] + #[instrument(level = "trace", target = "net::ecies", skip(transport, secret_key))] pub async fn connect( transport: Io, secret_key: SecretKey, diff --git a/crates/net/eth-wire-types/src/broadcast.rs b/crates/net/eth-wire-types/src/broadcast.rs index b952f329593..1900cf004aa 100644 --- a/crates/net/eth-wire-types/src/broadcast.rs +++ b/crates/net/eth-wire-types/src/broadcast.rs @@ -9,11 +9,11 @@ use alloy_primitives::{ use alloy_rlp::{ Decodable, Encodable, RlpDecodable, RlpDecodableWrapper, RlpEncodable, RlpEncodableWrapper, }; -use core::mem; +use core::{fmt::Debug, mem}; use derive_more::{Constructor, Deref, DerefMut, From, IntoIterator}; use reth_codecs_derive::{add_arbitrary_tests, generate_tests}; use reth_ethereum_primitives::TransactionSigned; -use reth_primitives_traits::SignedTransaction; +use reth_primitives_traits::{Block, SignedTransaction}; /// This informs peers of new blocks that have appeared on the network. #[derive(Clone, Debug, PartialEq, Eq, RlpEncodableWrapper, RlpDecodableWrapper, Default)] @@ -64,6 +64,17 @@ impl From for Vec { } } +/// A trait for block payloads transmitted through p2p. +pub trait NewBlockPayload: + Encodable + Decodable + Clone + Eq + Debug + Send + Sync + Unpin + 'static +{ + /// The block type. + type Block: Block; + + /// Returns a reference to the block. + fn block(&self) -> &Self::Block; +} + /// A new block with the current total difficulty, which includes the difficulty of the returned /// block. #[derive(Clone, Debug, PartialEq, Eq, RlpEncodable, RlpDecodable, Default)] @@ -76,6 +87,14 @@ pub struct NewBlock { pub td: U128, } +impl NewBlockPayload for NewBlock { + type Block = B; + + fn block(&self) -> &Self::Block { + &self.block + } +} + generate_tests!(#[rlp, 25] NewBlock, EthNewBlockTests); /// This informs peers of transactions that have appeared on the network and are not yet included @@ -150,7 +169,7 @@ impl NewPooledTransactionHashes { matches!(version, EthVersion::Eth67 | EthVersion::Eth66) } Self::Eth68(_) => { - matches!(version, EthVersion::Eth68) + matches!(version, EthVersion::Eth68 | EthVersion::Eth69) } } } @@ -209,7 +228,7 @@ impl NewPooledTransactionHashes { } /// Returns true if the message is empty - pub fn is_empty(&self) -> bool { + pub const fn is_empty(&self) -> bool { match self { Self::Eth66(msg) => msg.0.is_empty(), Self::Eth68(msg) => msg.hashes.is_empty(), @@ -217,7 +236,7 @@ impl NewPooledTransactionHashes { } /// Returns the number of hashes in the message - pub fn len(&self) -> usize { + pub const fn len(&self) -> usize { match self { Self::Eth66(msg) => msg.0.len(), Self::Eth68(msg) => msg.hashes.len(), @@ -765,10 +784,24 @@ impl FromIterator<(TxHash, Eth68TxMetadata)> for RequestTxHashes { } } +/// The earliest block, the latest block and hash of the latest block which can be provided. +/// See [BlockRangeUpdate](https://github.com/ethereum/devp2p/blob/master/caps/eth.md#blockrangeupdate-0x11). +#[derive(Clone, Debug, PartialEq, Eq, Default, RlpEncodable, RlpDecodable)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(rename_all = "camelCase"))] +pub struct BlockRangeUpdate { + /// The earliest block which is available. + pub earliest: u64, + /// The latest block which is available. + pub latest: u64, + /// Latest available block's hash. + pub latest_hash: B256, +} + #[cfg(test)] mod tests { use super::*; - use alloy_consensus::Typed2718; + use alloy_consensus::{transaction::TxHashRef, Typed2718}; use alloy_eips::eip2718::Encodable2718; use alloy_primitives::{b256, hex, Signature, U256}; use reth_ethereum_primitives::{Transaction, TransactionSigned}; diff --git a/crates/net/eth-wire-types/src/disconnect_reason.rs b/crates/net/eth-wire-types/src/disconnect_reason.rs index e6efa0fca80..5efa9e571ca 100644 --- a/crates/net/eth-wire-types/src/disconnect_reason.rs +++ b/crates/net/eth-wire-types/src/disconnect_reason.rs @@ -7,7 +7,7 @@ use derive_more::Display; use reth_codecs_derive::add_arbitrary_tests; use thiserror::Error; -/// RLPx disconnect reason. +/// `RLPx` disconnect reason. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Display)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[cfg_attr(any(test, feature = "arbitrary"), derive(arbitrary::Arbitrary))] diff --git a/crates/net/eth-wire-types/src/header.rs b/crates/net/eth-wire-types/src/header.rs index 402212fda8c..986fbb006df 100644 --- a/crates/net/eth-wire-types/src/header.rs +++ b/crates/net/eth-wire-types/src/header.rs @@ -88,7 +88,7 @@ impl From for bool { mod tests { use super::*; use alloy_consensus::{Header, EMPTY_OMMER_ROOT_HASH, EMPTY_ROOT_HASH}; - use alloy_primitives::{address, b256, bloom, bytes, hex, Address, Bytes, B256, U256}; + use alloy_primitives::{address, b256, bloom, bytes, hex, Bytes, B256, U256}; use alloy_rlp::{Decodable, Encodable}; use std::str::FromStr; @@ -121,8 +121,7 @@ mod tests { #[test] fn test_eip1559_block_header_hash() { let expected_hash = - B256::from_str("6a251c7c3c5dca7b42407a3752ff48f3bbca1fab7f9868371d9918daf1988d1f") - .unwrap(); + b256!("0x6a251c7c3c5dca7b42407a3752ff48f3bbca1fab7f9868371d9918daf1988d1f"); let header = Header { parent_hash: b256!("0xe0a94a7a3c9617401586b1a27025d2d9671332d22d540e0af72b069170380f2a"), ommers_hash: EMPTY_OMMER_ROOT_HASH, @@ -181,8 +180,7 @@ mod tests { // make sure the hash matches let expected_hash = - B256::from_str("8c2f2af15b7b563b6ab1e09bed0e9caade7ed730aec98b70a993597a797579a9") - .unwrap(); + b256!("0x8c2f2af15b7b563b6ab1e09bed0e9caade7ed730aec98b70a993597a797579a9"); assert_eq!(header.hash_slow(), expected_hash); } @@ -197,7 +195,7 @@ mod tests { "18db39e19931515b30b16b3a92c292398039e31d6c267111529c3f2ba0a26c17", ) .unwrap(), - beneficiary: Address::from_str("2adc25665018aa1fe0e6bc666dac8fc2697ff9ba").unwrap(), + beneficiary: address!("0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba"), state_root: B256::from_str( "95efce3d6972874ca8b531b233b7a1d1ff0a56f08b20c8f1b89bef1b001194a5", ) @@ -217,18 +215,16 @@ mod tests { extra_data: Bytes::from_str("42").unwrap(), mix_hash: EMPTY_ROOT_HASH, base_fee_per_gas: Some(0x09), - withdrawals_root: Some( - B256::from_str("27f166f1d7c789251299535cb176ba34116e44894476a7886fe5d73d9be5c973") - .unwrap(), - ), + withdrawals_root: Some(b256!( + "0x27f166f1d7c789251299535cb176ba34116e44894476a7886fe5d73d9be5c973" + )), ..Default::default() }; let header =
::decode(&mut data.as_slice()).unwrap(); assert_eq!(header, expected); let expected_hash = - B256::from_str("85fdec94c534fa0a1534720f167b899d1fc268925c71c0cbf5aaa213483f5a69") - .unwrap(); + b256!("0x85fdec94c534fa0a1534720f167b899d1fc268925c71c0cbf5aaa213483f5a69"); assert_eq!(header.hash_slow(), expected_hash); } @@ -244,7 +240,7 @@ mod tests { ) .unwrap(), ommers_hash: EMPTY_OMMER_ROOT_HASH, - beneficiary: Address::from_str("2adc25665018aa1fe0e6bc666dac8fc2697ff9ba").unwrap(), + beneficiary: address!("0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba"), state_root: B256::from_str( "3c837fc158e3e93eafcaf2e658a02f5d8f99abc9f1c4c66cdea96c0ca26406ae", ) diff --git a/crates/net/eth-wire-types/src/lib.rs b/crates/net/eth-wire-types/src/lib.rs index cabbc108287..b7d27227846 100644 --- a/crates/net/eth-wire-types/src/lib.rs +++ b/crates/net/eth-wire-types/src/lib.rs @@ -6,13 +6,13 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(feature = "std"), no_std)] extern crate alloc; mod status; -pub use status::{Status, StatusBuilder, StatusEth69, StatusMessage}; +pub use status::{Status, StatusBuilder, StatusEth69, StatusMessage, UnifiedStatus}; pub mod version; pub use version::{EthVersion, ProtocolVersion}; diff --git a/crates/net/eth-wire-types/src/message.rs b/crates/net/eth-wire-types/src/message.rs index 601cb2b43c1..5f36115204b 100644 --- a/crates/net/eth-wire-types/src/message.rs +++ b/crates/net/eth-wire-types/src/message.rs @@ -4,19 +4,19 @@ //! //! Examples include creating, encoding, and decoding protocol messages. //! -//! Reference: [Ethereum Wire Protocol](https://github.com/ethereum/wiki/wiki/Ethereum-Wire-Protocol). +//! Reference: [Ethereum Wire Protocol](https://github.com/ethereum/devp2p/blob/master/caps/eth.md). use super::{ broadcast::NewBlockHashes, BlockBodies, BlockHeaders, GetBlockBodies, GetBlockHeaders, - GetNodeData, GetPooledTransactions, GetReceipts, NewBlock, NewPooledTransactionHashes66, + GetNodeData, GetPooledTransactions, GetReceipts, NewPooledTransactionHashes66, NewPooledTransactionHashes68, NodeData, PooledTransactions, Receipts, Status, StatusEth69, Transactions, }; use crate::{ - status::StatusMessage, EthNetworkPrimitives, EthVersion, NetworkPrimitives, - RawCapabilityMessage, SharedTransactions, + status::StatusMessage, BlockRangeUpdate, EthNetworkPrimitives, EthVersion, NetworkPrimitives, + RawCapabilityMessage, Receipts69, SharedTransactions, }; -use alloc::{boxed::Box, sync::Arc}; +use alloc::{boxed::Box, string::String, sync::Arc}; use alloy_primitives::{ bytes::{Buf, BufMut}, Bytes, @@ -37,6 +37,9 @@ pub enum MessageError { /// Thrown when rlp decoding a message failed. #[error("RLP error: {0}")] RlpError(#[from] alloy_rlp::Error), + /// Other message error with custom message + #[error("{0}")] + Other(String), } /// An `eth` protocol message, containing a message ID and payload. @@ -55,6 +58,8 @@ pub struct ProtocolMessage { impl ProtocolMessage { /// Create a new `ProtocolMessage` from a message type and message rlp bytes. + /// + /// This will enforce decoding according to the given [`EthVersion`] of the connection. pub fn decode_message(version: EthVersion, buf: &mut &[u8]) -> Result { let message_type = EthMessageID::decode(buf)?; @@ -67,16 +72,10 @@ impl ProtocolMessage { StatusMessage::Eth69(StatusEth69::decode(buf)?) }), EthMessageID::NewBlockHashes => { - if version.is_eth69() { - return Err(MessageError::Invalid(version, EthMessageID::NewBlockHashes)); - } EthMessage::NewBlockHashes(NewBlockHashes::decode(buf)?) } EthMessageID::NewBlock => { - if version.is_eth69() { - return Err(MessageError::Invalid(version, EthMessageID::NewBlock)); - } - EthMessage::NewBlock(Box::new(NewBlock::decode(buf)?)) + EthMessage::NewBlock(Box::new(N::NewBlockPayload::decode(buf)?)) } EthMessageID::Transactions => EthMessage::Transactions(Transactions::decode(buf)?), EthMessageID::NewPooledTransactionHashes => { @@ -113,7 +112,20 @@ impl ProtocolMessage { EthMessage::NodeData(RequestPair::decode(buf)?) } EthMessageID::GetReceipts => EthMessage::GetReceipts(RequestPair::decode(buf)?), - EthMessageID::Receipts => EthMessage::Receipts(RequestPair::decode(buf)?), + EthMessageID::Receipts => { + if version < EthVersion::Eth69 { + EthMessage::Receipts(RequestPair::decode(buf)?) + } else { + // with eth69, receipts no longer include the bloom + EthMessage::Receipts69(RequestPair::decode(buf)?) + } + } + EthMessageID::BlockRangeUpdate => { + if version < EthVersion::Eth69 { + return Err(MessageError::Invalid(version, EthMessageID::BlockRangeUpdate)) + } + EthMessage::BlockRangeUpdate(BlockRangeUpdate::decode(buf)?) + } EthMessageID::Other(_) => { let raw_payload = Bytes::copy_from_slice(buf); buf.advance(raw_payload.len()); @@ -173,7 +185,7 @@ impl From> for ProtocolBroadcastMes } } -/// Represents a message in the eth wire protocol, versions 66, 67 and 68. +/// Represents a message in the eth wire protocol, versions 66, 67, 68 and 69. /// /// The ethereum wire protocol is a set of messages that are broadcast to the network in two /// styles: @@ -190,6 +202,9 @@ impl From> for ProtocolBroadcastMes /// The `eth/68` changes only `NewPooledTransactionHashes` to include `types` and `sized`. For /// it, `NewPooledTransactionHashes` is renamed as [`NewPooledTransactionHashes66`] and /// [`NewPooledTransactionHashes68`] is defined. +/// +/// The `eth/69` announces the historical block range served by the node. Removes total difficulty +/// information. And removes the Bloom field from receipts transferred over the protocol. #[derive(Clone, Debug, PartialEq, Eq)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] pub enum EthMessage { @@ -200,9 +215,9 @@ pub enum EthMessage { /// Represents a `NewBlock` message broadcast to the network. #[cfg_attr( feature = "serde", - serde(bound = "N::Block: serde::Serialize + serde::de::DeserializeOwned") + serde(bound = "N::NewBlockPayload: serde::Serialize + serde::de::DeserializeOwned") )] - NewBlock(Box>), + NewBlock(Box), /// Represents a Transactions message broadcast to the network. #[cfg_attr( feature = "serde", @@ -250,6 +265,18 @@ pub enum EthMessage { serde(bound = "N::Receipt: serde::Serialize + serde::de::DeserializeOwned") )] Receipts(RequestPair>), + /// Represents a Receipts request-response pair for eth/69. + #[cfg_attr( + feature = "serde", + serde(bound = "N::Receipt: serde::Serialize + serde::de::DeserializeOwned") + )] + Receipts69(RequestPair>), + /// Represents a `BlockRangeUpdate` message broadcast to the network. + #[cfg_attr( + feature = "serde", + serde(bound = "N::BroadcastedTransaction: serde::Serialize + serde::de::DeserializeOwned") + )] + BlockRangeUpdate(BlockRangeUpdate), /// Represents an encoded message that doesn't match any other variant Other(RawCapabilityMessage), } @@ -274,7 +301,8 @@ impl EthMessage { Self::GetNodeData(_) => EthMessageID::GetNodeData, Self::NodeData(_) => EthMessageID::NodeData, Self::GetReceipts(_) => EthMessageID::GetReceipts, - Self::Receipts(_) => EthMessageID::Receipts, + Self::Receipts(_) | Self::Receipts69(_) => EthMessageID::Receipts, + Self::BlockRangeUpdate(_) => EthMessageID::BlockRangeUpdate, Self::Other(msg) => EthMessageID::Other(msg.id as u8), } } @@ -297,6 +325,7 @@ impl EthMessage { self, Self::PooledTransactions(_) | Self::Receipts(_) | + Self::Receipts69(_) | Self::BlockHeaders(_) | Self::BlockBodies(_) | Self::NodeData(_) @@ -323,6 +352,8 @@ impl Encodable for EthMessage { Self::NodeData(data) => data.encode(out), Self::GetReceipts(request) => request.encode(out), Self::Receipts(receipts) => receipts.encode(out), + Self::Receipts69(receipt69) => receipt69.encode(out), + Self::BlockRangeUpdate(block_range_update) => block_range_update.encode(out), Self::Other(unknown) => out.put_slice(&unknown.payload), } } @@ -344,6 +375,8 @@ impl Encodable for EthMessage { Self::NodeData(data) => data.length(), Self::GetReceipts(request) => request.length(), Self::Receipts(receipts) => receipts.length(), + Self::Receipts69(receipt69) => receipt69.length(), + Self::BlockRangeUpdate(block_range_update) => block_range_update.length(), Self::Other(unknown) => unknown.length(), } } @@ -359,7 +392,7 @@ impl Encodable for EthMessage { #[derive(Clone, Debug, PartialEq, Eq)] pub enum EthBroadcastMessage { /// Represents a new block broadcast message. - NewBlock(Arc>), + NewBlock(Arc), /// Represents a transactions broadcast message. Transactions(SharedTransactions), } @@ -427,6 +460,10 @@ pub enum EthMessageID { GetReceipts = 0x0f, /// Represents receipts. Receipts = 0x10, + /// Block range update. + /// + /// Introduced in Eth69 + BlockRangeUpdate = 0x11, /// Represents unknown message types. Other(u8), } @@ -450,13 +487,27 @@ impl EthMessageID { Self::NodeData => 0x0e, Self::GetReceipts => 0x0f, Self::Receipts => 0x10, + Self::BlockRangeUpdate => 0x11, Self::Other(value) => *value, // Return the stored `u8` } } - /// Returns the max value. - pub const fn max() -> u8 { - Self::Receipts.to_u8() + /// Returns the max value for the given version. + pub const fn max(version: EthVersion) -> u8 { + if version.is_eth69() { + Self::BlockRangeUpdate.to_u8() + } else { + Self::Receipts.to_u8() + } + } + + /// Returns the total number of message types for the given version. + /// + /// This is used for message ID multiplexing. + /// + /// + pub const fn message_count(version: EthVersion) -> u8 { + Self::max(version) + 1 } } @@ -487,6 +538,7 @@ impl Decodable for EthMessageID { 0x0e => Self::NodeData, 0x0f => Self::GetReceipts, 0x10 => Self::Receipts, + 0x11 => Self::BlockRangeUpdate, unknown => Self::Other(*unknown), }; buf.advance(1); @@ -514,6 +566,7 @@ impl TryFrom for EthMessageID { 0x0e => Ok(Self::NodeData), 0x0f => Ok(Self::GetReceipts), 0x10 => Ok(Self::Receipts), + 0x11 => Ok(Self::BlockRangeUpdate), _ => Err("Invalid message ID"), } } @@ -533,6 +586,17 @@ pub struct RequestPair { pub message: T, } +impl RequestPair { + /// Converts the message type with the given closure. + pub fn map(self, f: F) -> RequestPair + where + F: FnOnce(T) -> R, + { + let Self { request_id, message } = self; + RequestPair { request_id, message: f(message) } + } +} + /// Allows messages with request ids to be serialized into RLP bytes. impl Encodable for RequestPair where diff --git a/crates/net/eth-wire-types/src/primitives.rs b/crates/net/eth-wire-types/src/primitives.rs index ea7813422e5..25f08f35efc 100644 --- a/crates/net/eth-wire-types/src/primitives.rs +++ b/crates/net/eth-wire-types/src/primitives.rs @@ -1,12 +1,31 @@ //! Abstraction over primitive types in network messages. +use crate::NewBlockPayload; use alloy_consensus::{RlpDecodableReceipt, RlpEncodableReceipt, TxReceipt}; use alloy_rlp::{Decodable, Encodable}; use core::fmt::Debug; -use reth_primitives_traits::{Block, BlockBody, BlockHeader, NodePrimitives, SignedTransaction}; +use reth_ethereum_primitives::{EthPrimitives, PooledTransactionVariant}; +use reth_primitives_traits::{ + Block, BlockBody, BlockHeader, BlockTy, NodePrimitives, SignedTransaction, +}; -/// Abstraction over primitive types which might appear in network messages. See -/// [`crate::EthMessage`] for more context. +/// Abstraction over primitive types which might appear in network messages. +/// +/// This trait defines the types used in the Ethereum Wire Protocol (devp2p) for +/// peer-to-peer communication. While [`NodePrimitives`] defines the core types +/// used throughout the node (consensus format), `NetworkPrimitives` defines how +/// these types are represented when transmitted over the network. +/// +/// The key distinction is in transaction handling: +/// - [`NodePrimitives`] defines `SignedTx` - the consensus format stored in blocks +/// - `NetworkPrimitives` defines `BroadcastedTransaction` and `PooledTransaction` - the formats +/// used for network propagation with additional data like blob sidecars +/// +/// These traits work together through implementations like [`NetPrimitivesFor`], +/// which ensures type compatibility between a node's internal representation and +/// its network representation. +/// +/// See [`crate::EthMessage`] for more context. pub trait NetworkPrimitives: Send + Sync + Unpin + Clone + Debug + 'static { /// The block header type. type BlockHeader: BlockHeader + 'static; @@ -20,16 +39,33 @@ pub trait NetworkPrimitives: Send + Sync + Unpin + Clone + Debug + 'static { + Decodable + 'static; - /// The transaction type which peers announce in `Transactions` messages. It is different from - /// `PooledTransactions` to account for Ethereum case where EIP-4844 transactions are not being - /// announced and can only be explicitly requested from peers. + /// The transaction type which peers announce in `Transactions` messages. + /// + /// This is different from `PooledTransactions` to account for the Ethereum case where + /// EIP-4844 blob transactions are not announced over the network and can only be + /// explicitly requested from peers. This is because blob transactions can be quite + /// large and broadcasting them to all peers would cause + /// significant bandwidth usage. type BroadcastedTransaction: SignedTransaction + 'static; /// The transaction type which peers return in `PooledTransactions` messages. + /// + /// For EIP-4844 blob transactions, this includes the full blob sidecar with + /// KZG commitments and proofs that are needed for validation but are not + /// included in the consensus block format. type PooledTransaction: SignedTransaction + TryFrom + 'static; /// The transaction type which peers return in `GetReceipts` messages. - type Receipt: TxReceipt + RlpEncodableReceipt + RlpDecodableReceipt + Unpin + 'static; + type Receipt: TxReceipt + + RlpEncodableReceipt + + RlpDecodableReceipt + + Encodable + + Decodable + + Unpin + + 'static; + + /// The payload type for the `NewBlock` message. + type NewBlockPayload: NewBlockPayload; } /// This is a helper trait for use in bounds, where some of the [`NetworkPrimitives`] associated @@ -56,16 +92,27 @@ where { } -/// Network primitive types used by Ethereum networks. +/// Basic implementation of [`NetworkPrimitives`] combining [`NodePrimitives`] and a pooled +/// transaction. #[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Hash)] -#[non_exhaustive] -pub struct EthNetworkPrimitives; +pub struct BasicNetworkPrimitives>>( + core::marker::PhantomData<(N, Pooled, NewBlock)>, +); -impl NetworkPrimitives for EthNetworkPrimitives { - type BlockHeader = alloy_consensus::Header; - type BlockBody = reth_ethereum_primitives::BlockBody; - type Block = reth_ethereum_primitives::Block; - type BroadcastedTransaction = reth_ethereum_primitives::TransactionSigned; - type PooledTransaction = reth_ethereum_primitives::PooledTransaction; - type Receipt = reth_ethereum_primitives::Receipt; +impl NetworkPrimitives for BasicNetworkPrimitives +where + N: NodePrimitives, + Pooled: SignedTransaction + TryFrom + 'static, + NewBlock: NewBlockPayload, +{ + type BlockHeader = N::BlockHeader; + type BlockBody = N::BlockBody; + type Block = N::Block; + type BroadcastedTransaction = N::SignedTx; + type PooledTransaction = Pooled; + type Receipt = N::Receipt; + type NewBlockPayload = NewBlock; } + +/// Network primitive types used by Ethereum networks. +pub type EthNetworkPrimitives = BasicNetworkPrimitives; diff --git a/crates/net/eth-wire-types/src/receipts.rs b/crates/net/eth-wire-types/src/receipts.rs index 07ce7fbba03..416797c50ee 100644 --- a/crates/net/eth-wire-types/src/receipts.rs +++ b/crates/net/eth-wire-types/src/receipts.rs @@ -1,7 +1,7 @@ //! Implements the `GetReceipts` and `Receipts` message types. use alloc::vec::Vec; -use alloy_consensus::{ReceiptWithBloom, RlpDecodableReceipt, RlpEncodableReceipt}; +use alloy_consensus::{ReceiptWithBloom, RlpDecodableReceipt, RlpEncodableReceipt, TxReceipt}; use alloy_primitives::B256; use alloy_rlp::{RlpDecodableWrapper, RlpEncodableWrapper}; use reth_codecs_derive::add_arbitrary_tests; @@ -46,6 +46,53 @@ impl alloy_rlp::Decodable for Receipts { } } +/// Eth/69 receipt response type that removes bloom filters from the protocol. +/// +/// This is effectively a subset of [`Receipts`]. +#[derive(Clone, Debug, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(any(test, feature = "arbitrary"), derive(arbitrary::Arbitrary))] +#[add_arbitrary_tests(rlp)] +pub struct Receipts69(pub Vec>); + +impl alloy_rlp::Encodable for Receipts69 { + #[inline] + fn encode(&self, out: &mut dyn alloy_rlp::BufMut) { + self.0.encode(out) + } + #[inline] + fn length(&self) -> usize { + self.0.length() + } +} + +impl alloy_rlp::Decodable for Receipts69 { + #[inline] + fn decode(buf: &mut &[u8]) -> alloy_rlp::Result { + alloy_rlp::Decodable::decode(buf).map(Self) + } +} + +impl Receipts69 { + /// Encodes all receipts with the bloom filter. + /// + /// Note: This is an expensive operation that recalculates the bloom for each receipt. + pub fn into_with_bloom(self) -> Receipts { + Receipts( + self.0 + .into_iter() + .map(|receipts| receipts.into_iter().map(|r| r.into_with_bloom()).collect()) + .collect(), + ) + } +} + +impl From> for Receipts { + fn from(receipts: Receipts69) -> Self { + receipts.into_with_bloom() + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/net/eth-wire-types/src/status.rs b/crates/net/eth-wire-types/src/status.rs index aa646ef61bb..db363695c32 100644 --- a/crates/net/eth-wire-types/src/status.rs +++ b/crates/net/eth-wire-types/src/status.rs @@ -7,6 +7,205 @@ use core::fmt::{Debug, Display}; use reth_chainspec::{EthChainSpec, Hardforks, MAINNET}; use reth_codecs_derive::add_arbitrary_tests; +/// `UnifiedStatus` is an internal superset of all ETH status fields for all `eth/` versions. +/// +/// This type can be converted into [`Status`] or [`StatusEth69`] depending on the version and +/// unsupported fields are stripped out. +#[derive(Clone, Debug, PartialEq, Eq, Copy)] +pub struct UnifiedStatus { + /// The eth protocol version (e.g. eth/66 to eth/69). + pub version: EthVersion, + /// The chain ID identifying the peer’s network. + pub chain: Chain, + /// The genesis block hash of the peer’s chain. + pub genesis: B256, + /// The fork ID as defined by EIP-2124. + pub forkid: ForkId, + /// The latest block hash known to the peer. + pub blockhash: B256, + /// The total difficulty of the peer’s best chain (eth/66–68 only). + pub total_difficulty: Option, + /// The earliest block this node can serve (eth/69 only). + pub earliest_block: Option, + /// The latest block number this node has (eth/69 only). + pub latest_block: Option, +} + +impl Default for UnifiedStatus { + fn default() -> Self { + let mainnet_genesis = MAINNET.genesis_hash(); + Self { + version: EthVersion::Eth68, + chain: Chain::from_named(NamedChain::Mainnet), + genesis: mainnet_genesis, + forkid: MAINNET + .hardfork_fork_id(EthereumHardfork::Frontier) + .expect("Frontier must exist"), + blockhash: mainnet_genesis, + total_difficulty: Some(U256::from(17_179_869_184u64)), + earliest_block: Some(0), + latest_block: Some(0), + } + } +} + +impl UnifiedStatus { + /// Helper for creating the `UnifiedStatus` builder + pub fn builder() -> StatusBuilder { + Default::default() + } + + /// Build from chain‑spec + head. Earliest/latest default to full history. + pub fn spec_builder(spec: &Spec, head: &Head) -> Self + where + Spec: EthChainSpec + Hardforks, + { + Self::builder() + .chain(spec.chain()) + .genesis(spec.genesis_hash()) + .forkid(spec.fork_id(head)) + .blockhash(head.hash) + .total_difficulty(Some(head.total_difficulty)) + .earliest_block(Some(0)) + .latest_block(Some(head.number)) + .build() + } + + /// Override the `(earliest, latest)` history range we’ll advertise to + /// eth/69 peers. + pub const fn set_history_range(&mut self, earliest: u64, latest: u64) { + self.earliest_block = Some(earliest); + self.latest_block = Some(latest); + } + + /// Sets the [`EthVersion`] for the status. + pub const fn set_eth_version(&mut self, v: EthVersion) { + self.version = v; + } + + /// Consume this `UnifiedStatus` and produce the legacy [`Status`] message used by all + /// `eth/66`–`eth/68`. + pub fn into_legacy(self) -> Status { + Status { + version: self.version, + chain: self.chain, + genesis: self.genesis, + forkid: self.forkid, + blockhash: self.blockhash, + total_difficulty: self.total_difficulty.unwrap_or(U256::ZERO), + } + } + + /// Consume this `UnifiedStatus` and produce the [`StatusEth69`] message used by `eth/69`. + pub fn into_eth69(self) -> StatusEth69 { + StatusEth69 { + version: self.version, + chain: self.chain, + genesis: self.genesis, + forkid: self.forkid, + earliest: self.earliest_block.unwrap_or(0), + latest: self.latest_block.unwrap_or(0), + blockhash: self.blockhash, + } + } + + /// Convert this `UnifiedStatus` into the appropriate `StatusMessage` variant based on version. + pub fn into_message(self) -> StatusMessage { + if self.version >= EthVersion::Eth69 { + StatusMessage::Eth69(self.into_eth69()) + } else { + StatusMessage::Legacy(self.into_legacy()) + } + } + + /// Build a `UnifiedStatus` from a received `StatusMessage`. + pub const fn from_message(msg: StatusMessage) -> Self { + match msg { + StatusMessage::Legacy(s) => Self { + version: s.version, + chain: s.chain, + genesis: s.genesis, + forkid: s.forkid, + blockhash: s.blockhash, + total_difficulty: Some(s.total_difficulty), + earliest_block: None, + latest_block: None, + }, + StatusMessage::Eth69(e) => Self { + version: e.version, + chain: e.chain, + genesis: e.genesis, + forkid: e.forkid, + blockhash: e.blockhash, + total_difficulty: None, + earliest_block: Some(e.earliest), + latest_block: Some(e.latest), + }, + } + } +} + +/// Builder type for constructing a [`UnifiedStatus`] message. +#[derive(Debug, Default)] +pub struct StatusBuilder { + status: UnifiedStatus, +} + +impl StatusBuilder { + /// Consumes the builder and returns the constructed [`UnifiedStatus`]. + pub const fn build(self) -> UnifiedStatus { + self.status + } + + /// Sets the eth protocol version (e.g., eth/66, eth/69). + pub const fn version(mut self, version: EthVersion) -> Self { + self.status.version = version; + self + } + + /// Sets the chain ID + pub const fn chain(mut self, chain: Chain) -> Self { + self.status.chain = chain; + self + } + + /// Sets the genesis block hash of the chain. + pub const fn genesis(mut self, genesis: B256) -> Self { + self.status.genesis = genesis; + self + } + + /// Sets the fork ID, used for fork compatibility checks. + pub const fn forkid(mut self, forkid: ForkId) -> Self { + self.status.forkid = forkid; + self + } + + /// Sets the block hash of the current head. + pub const fn blockhash(mut self, blockhash: B256) -> Self { + self.status.blockhash = blockhash; + self + } + + /// Sets the total difficulty, if relevant (Some for eth/66–68). + pub const fn total_difficulty(mut self, td: Option) -> Self { + self.status.total_difficulty = td; + self + } + + /// Sets the earliest available block, if known (Some for eth/69). + pub const fn earliest_block(mut self, earliest: Option) -> Self { + self.status.earliest_block = earliest; + self + } + + /// Sets the latest known block, if known (Some for eth/69). + pub const fn latest_block(mut self, latest: Option) -> Self { + self.status.latest_block = latest; + self + } +} + /// The status message is used in the eth protocol handshake to ensure that peers are on the same /// network and are following the same fork. /// @@ -42,41 +241,19 @@ pub struct Status { pub forkid: ForkId, } -impl Status { - /// Helper for returning a builder for the status message. - pub fn builder() -> StatusBuilder { - Default::default() - } - - /// Sets the [`EthVersion`] for the status. - pub const fn set_eth_version(&mut self, version: EthVersion) { - self.version = version; - } - - /// Create a [`StatusBuilder`] from the given [`EthChainSpec`] and head block. - /// - /// Sets the `chain` and `genesis`, `blockhash`, and `forkid` fields based on the - /// [`EthChainSpec`] and head. - pub fn spec_builder(spec: Spec, head: &Head) -> StatusBuilder - where - Spec: EthChainSpec + Hardforks, - { - Self::builder() - .chain(spec.chain()) - .genesis(spec.genesis_hash()) - .blockhash(head.hash) - .total_difficulty(head.total_difficulty) - .forkid(spec.fork_id(head)) - } - - /// Converts this [`Status`] into the [Eth69](https://github.com/ethereum/EIPs/blob/master/EIPS/eip-7642.md) variant that excludes the total difficulty field. - pub const fn into_eth69(self) -> StatusEth69 { - StatusEth69 { - version: EthVersion::Eth69, - chain: self.chain, - blockhash: self.blockhash, - genesis: self.genesis, - forkid: self.forkid, +// +impl Default for Status { + fn default() -> Self { + let mainnet_genesis = MAINNET.genesis_hash(); + Self { + version: EthVersion::Eth68, + chain: Chain::from_named(NamedChain::Mainnet), + total_difficulty: U256::from(17_179_869_184u64), + blockhash: mainnet_genesis, + genesis: mainnet_genesis, + forkid: MAINNET + .hardfork_fork_id(EthereumHardfork::Frontier) + .expect("The Frontier hardfork should always exist"), } } } @@ -128,102 +305,6 @@ impl Debug for Status { } } -// -impl Default for Status { - fn default() -> Self { - let mainnet_genesis = MAINNET.genesis_hash(); - Self { - version: EthVersion::Eth68, - chain: Chain::from_named(NamedChain::Mainnet), - total_difficulty: U256::from(17_179_869_184u64), - blockhash: mainnet_genesis, - genesis: mainnet_genesis, - forkid: MAINNET - .hardfork_fork_id(EthereumHardfork::Frontier) - .expect("The Frontier hardfork should always exist"), - } - } -} - -/// Builder for [`Status`] messages. -/// -/// # Example -/// ``` -/// use alloy_consensus::constants::MAINNET_GENESIS_HASH; -/// use alloy_primitives::{B256, U256}; -/// use reth_chainspec::{Chain, EthereumHardfork, MAINNET}; -/// use reth_eth_wire_types::{EthVersion, Status}; -/// -/// // this is just an example status message! -/// let status = Status::builder() -/// .version(EthVersion::Eth66) -/// .chain(Chain::mainnet()) -/// .total_difficulty(U256::from(100)) -/// .blockhash(B256::from(MAINNET_GENESIS_HASH)) -/// .genesis(B256::from(MAINNET_GENESIS_HASH)) -/// .forkid(MAINNET.hardfork_fork_id(EthereumHardfork::Paris).unwrap()) -/// .build(); -/// -/// assert_eq!( -/// status, -/// Status { -/// version: EthVersion::Eth66, -/// chain: Chain::mainnet(), -/// total_difficulty: U256::from(100), -/// blockhash: B256::from(MAINNET_GENESIS_HASH), -/// genesis: B256::from(MAINNET_GENESIS_HASH), -/// forkid: MAINNET.hardfork_fork_id(EthereumHardfork::Paris).unwrap(), -/// } -/// ); -/// ``` -#[derive(Debug, Default)] -pub struct StatusBuilder { - status: Status, -} - -impl StatusBuilder { - /// Consumes the type and creates the actual [`Status`] message. - pub const fn build(self) -> Status { - self.status - } - - /// Sets the protocol version. - pub const fn version(mut self, version: EthVersion) -> Self { - self.status.version = version; - self - } - - /// Sets the chain id. - pub const fn chain(mut self, chain: Chain) -> Self { - self.status.chain = chain; - self - } - - /// Sets the total difficulty. - pub const fn total_difficulty(mut self, total_difficulty: U256) -> Self { - self.status.total_difficulty = total_difficulty; - self - } - - /// Sets the block hash. - pub const fn blockhash(mut self, blockhash: B256) -> Self { - self.status.blockhash = blockhash; - self - } - - /// Sets the genesis hash. - pub const fn genesis(mut self, genesis: B256) -> Self { - self.status.genesis = genesis; - self - } - - /// Sets the fork id. - pub const fn forkid(mut self, forkid: ForkId) -> Self { - self.status.forkid = forkid; - self - } -} - /// Similar to [`Status`], but for `eth/69` version, which does not contain /// the `total_difficulty` field. #[derive(Copy, Clone, PartialEq, Eq, RlpEncodable, RlpDecodable)] @@ -239,9 +320,6 @@ pub struct StatusEth69 { /// [EIP155](https://eips.ethereum.org/EIPS/eip-155#list-of-chain-ids). pub chain: Chain, - /// The highest difficulty block hash the peer has seen - pub blockhash: B256, - /// The genesis hash of the peer's chain. pub genesis: B256, @@ -251,6 +329,15 @@ pub struct StatusEth69 { /// [EIP-2124](https://github.com/ethereum/EIPs/blob/master/EIPS/eip-2124.md). /// This was added in [`eth/64`](https://eips.ethereum.org/EIPS/eip-2364) pub forkid: ForkId, + + /// Earliest block number this node can serve + pub earliest: u64, + + /// Latest block number this node has (current head) + pub latest: u64, + + /// Hash of the latest block this node has (current head) + pub blockhash: B256, } impl Display for StatusEth69 { @@ -259,8 +346,14 @@ impl Display for StatusEth69 { let hexed_genesis = hex::encode(self.genesis); write!( f, - "Status {{ version: {}, chain: {}, blockhash: {}, genesis: {}, forkid: {:X?} }}", - self.version, self.chain, hexed_blockhash, hexed_genesis, self.forkid + "StatusEth69 {{ version: {}, chain: {}, genesis: {}, forkid: {:X?}, earliest: {}, latest: {}, blockhash: {} }}", + self.version, + self.chain, + hexed_genesis, + self.forkid, + self.earliest, + self.latest, + hexed_blockhash, ) } } @@ -285,19 +378,6 @@ impl Debug for StatusEth69 { } } -// -impl Default for StatusEth69 { - fn default() -> Self { - Status::default().into() - } -} - -impl From for StatusEth69 { - fn from(status: Status) -> Self { - status.into_eth69() - } -} - /// `StatusMessage` can store either the Legacy version (with TD) or the /// eth/69 version (omits TD). #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] @@ -342,21 +422,11 @@ impl StatusMessage { } } - /// Converts to legacy Status since full support for EIP-7642 - /// is not fully implemented - /// `` - pub fn to_legacy(self) -> Status { + /// Returns the latest block hash + pub const fn blockhash(&self) -> B256 { match self { - Self::Legacy(legacy_status) => legacy_status, - Self::Eth69(status_69) => Status { - version: status_69.version, - chain: status_69.chain, - // total_difficulty is omitted in Eth69. - total_difficulty: U256::default(), - blockhash: status_69.blockhash, - genesis: status_69.genesis, - forkid: status_69.forkid, - }, + Self::Legacy(legacy_status) => legacy_status.blockhash, + Self::Eth69(status_69) => status_69.blockhash, } } } @@ -377,13 +447,21 @@ impl Encodable for StatusMessage { } } +impl Display for StatusMessage { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Self::Legacy(s) => Display::fmt(s, f), + Self::Eth69(s69) => Display::fmt(s69, f), + } + } +} #[cfg(test)] mod tests { - use crate::{EthVersion, Status, StatusEth69}; + use crate::{EthVersion, Status, StatusEth69, StatusMessage, UnifiedStatus}; use alloy_consensus::constants::MAINNET_GENESIS_HASH; use alloy_genesis::Genesis; use alloy_hardforks::{EthereumHardfork, ForkHash, ForkId, Head}; - use alloy_primitives::{hex, B256, U256}; + use alloy_primitives::{b256, hex, B256, U256}; use alloy_rlp::{Decodable, Encodable}; use rand::Rng; use reth_chainspec::{Chain, ChainSpec, ForkCondition, NamedChain}; @@ -432,62 +510,74 @@ mod tests { } #[test] - fn test_status_to_statuseth69_conversion() { - let status = StatusEth69 { - version: EthVersion::Eth69, - chain: Chain::from_named(NamedChain::Mainnet), - blockhash: B256::from_str( - "feb27336ca7923f8fab3bd617fcb6e75841538f71c1bcfc267d7838489d9e13d", - ) - .unwrap(), - genesis: MAINNET_GENESIS_HASH, - forkid: ForkId { hash: ForkHash([0xb7, 0x15, 0x07, 0x7d]), next: 0 }, - }; - let status_converted: StatusEth69 = Status { - version: EthVersion::Eth69, - chain: Chain::from_named(NamedChain::Mainnet), - total_difficulty: U256::from(36206751599115524359527u128), - blockhash: B256::from_str( - "feb27336ca7923f8fab3bd617fcb6e75841538f71c1bcfc267d7838489d9e13d", - ) - .unwrap(), - genesis: MAINNET_GENESIS_HASH, - forkid: ForkId { hash: ForkHash([0xb7, 0x15, 0x07, 0x7d]), next: 0 }, - } - .into(); - assert_eq!(status, status_converted); + fn roundtrip_eth69() { + let unified_status = UnifiedStatus::builder() + .version(EthVersion::Eth69) + .chain(Chain::mainnet()) + .genesis(MAINNET_GENESIS_HASH) + .forkid(ForkId { hash: ForkHash([0xb7, 0x15, 0x07, 0x7d]), next: 0 }) + .blockhash(b256!("0xfeb27336ca7923f8fab3bd617fcb6e75841538f71c1bcfc267d7838489d9e13d")) + .earliest_block(Some(1)) + .latest_block(Some(2)) + .total_difficulty(None) + .build(); + + let status_message = unified_status.into_message(); + let roundtripped_unified_status = UnifiedStatus::from_message(status_message); + + assert_eq!(unified_status, roundtripped_unified_status); + } + + #[test] + fn roundtrip_legacy() { + let unified_status = UnifiedStatus::builder() + .version(EthVersion::Eth68) + .chain(Chain::sepolia()) + .genesis(MAINNET_GENESIS_HASH) + .forkid(ForkId { hash: ForkHash([0xaa, 0xbb, 0xcc, 0xdd]), next: 0 }) + .blockhash(b256!("0xfeb27336ca7923f8fab3bd617fcb6e75841538f71c1bcfc267d7838489d9e13d")) + .total_difficulty(Some(U256::from(42u64))) + .earliest_block(None) + .latest_block(None) + .build(); + + let status_message = unified_status.into_message(); + let roundtripped_unified_status = UnifiedStatus::from_message(status_message); + assert_eq!(unified_status, roundtripped_unified_status); } #[test] fn encode_eth69_status_message() { - let expected = hex!( - "f84b4501a0feb27336ca7923f8fab3bd617fcb6e75841538f71c1bcfc267d7838489d9e13da0d4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3c684b715077d80" - ); + let expected = hex!("f8544501a0d4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3c684b715077d8083ed14f2840112a880a0feb27336ca7923f8fab3bd617fcb6e75841538f71c1bcfc267d7838489d9e13d"); let status = StatusEth69 { version: EthVersion::Eth69, chain: Chain::from_named(NamedChain::Mainnet), + + genesis: MAINNET_GENESIS_HASH, + forkid: ForkId { hash: ForkHash([0xb7, 0x15, 0x07, 0x7d]), next: 0 }, + earliest: 15_537_394, + latest: 18_000_000, blockhash: B256::from_str( "feb27336ca7923f8fab3bd617fcb6e75841538f71c1bcfc267d7838489d9e13d", ) .unwrap(), - genesis: MAINNET_GENESIS_HASH, - forkid: ForkId { hash: ForkHash([0xb7, 0x15, 0x07, 0x7d]), next: 0 }, }; let mut rlp_status = vec![]; status.encode(&mut rlp_status); assert_eq!(rlp_status, expected); - let status: StatusEth69 = Status::builder() + let status = UnifiedStatus::builder() + .version(EthVersion::Eth69) .chain(Chain::from_named(NamedChain::Mainnet)) - .blockhash( - B256::from_str("feb27336ca7923f8fab3bd617fcb6e75841538f71c1bcfc267d7838489d9e13d") - .unwrap(), - ) .genesis(MAINNET_GENESIS_HASH) .forkid(ForkId { hash: ForkHash([0xb7, 0x15, 0x07, 0x7d]), next: 0 }) + .blockhash(b256!("0xfeb27336ca7923f8fab3bd617fcb6e75841538f71c1bcfc267d7838489d9e13d")) + .earliest_block(Some(15_537_394)) + .latest_block(Some(18_000_000)) .build() - .into(); + .into_message(); + let mut rlp_status = vec![]; status.encode(&mut rlp_status); assert_eq!(rlp_status, expected); @@ -495,21 +585,40 @@ mod tests { #[test] fn decode_eth69_status_message() { - let data = hex!( - "0xf84b4501a0feb27336ca7923f8fab3bd617fcb6e75841538f71c1bcfc267d7838489d9e13da0d4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3c684b715077d80" - ); + let data = hex!("f8544501a0d4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3c684b715077d8083ed14f2840112a880a0feb27336ca7923f8fab3bd617fcb6e75841538f71c1bcfc267d7838489d9e13d"); let expected = StatusEth69 { version: EthVersion::Eth69, chain: Chain::from_named(NamedChain::Mainnet), + genesis: MAINNET_GENESIS_HASH, + forkid: ForkId { hash: ForkHash([0xb7, 0x15, 0x07, 0x7d]), next: 0 }, + earliest: 15_537_394, + latest: 18_000_000, blockhash: B256::from_str( "feb27336ca7923f8fab3bd617fcb6e75841538f71c1bcfc267d7838489d9e13d", ) .unwrap(), - genesis: MAINNET_GENESIS_HASH, - forkid: ForkId { hash: ForkHash([0xb7, 0x15, 0x07, 0x7d]), next: 0 }, }; let status = StatusEth69::decode(&mut &data[..]).unwrap(); assert_eq!(status, expected); + + let expected_message = UnifiedStatus::builder() + .version(EthVersion::Eth69) + .chain(Chain::from_named(NamedChain::Mainnet)) + .genesis(MAINNET_GENESIS_HASH) + .forkid(ForkId { hash: ForkHash([0xb7, 0x15, 0x07, 0x7d]), next: 0 }) + .earliest_block(Some(15_537_394)) + .latest_block(Some(18_000_000)) + .blockhash(b256!("0xfeb27336ca7923f8fab3bd617fcb6e75841538f71c1bcfc267d7838489d9e13d")) + .build() + .into_message(); + + let expected_status = if let StatusMessage::Eth69(status69) = expected_message { + status69 + } else { + panic!("expected StatusMessage::Eth69 variant"); + }; + + assert_eq!(status, expected_status); } #[test] @@ -634,11 +743,11 @@ mod tests { let forkid = ForkId { hash: forkhash, next: 0 }; - let status = Status::spec_builder(&spec, &head).build(); + let status = UnifiedStatus::spec_builder(&spec, &head); assert_eq!(status.chain, Chain::from_id(1337)); assert_eq!(status.forkid, forkid); - assert_eq!(status.total_difficulty, total_difficulty); + assert_eq!(status.total_difficulty.unwrap(), total_difficulty); assert_eq!(status.blockhash, head_hash); assert_eq!(status.genesis, genesis_hash); } diff --git a/crates/net/eth-wire-types/src/version.rs b/crates/net/eth-wire-types/src/version.rs index 93ad8f7e5c9..8b2e3a424d9 100644 --- a/crates/net/eth-wire-types/src/version.rs +++ b/crates/net/eth-wire-types/src/version.rs @@ -31,20 +31,10 @@ pub enum EthVersion { impl EthVersion { /// The latest known eth version - pub const LATEST: Self = Self::Eth68; - - /// Returns the total number of messages the protocol version supports. - pub const fn total_messages(&self) -> u8 { - match self { - Self::Eth66 => 15, - Self::Eth67 | Self::Eth68 => { - // eth/67,68 are eth/66 minus GetNodeData and NodeData messages - 13 - } - // eth69 is both eth67 and eth68 minus NewBlockHashes and NewBlock - Self::Eth69 => 11, - } - } + pub const LATEST: Self = Self::Eth69; + + /// All known eth versions + pub const ALL_VERSIONS: &'static [Self] = &[Self::Eth69, Self::Eth68, Self::Eth67, Self::Eth66]; /// Returns true if the version is eth/66 pub const fn is_eth66(&self) -> bool { @@ -163,7 +153,7 @@ impl From for &'static str { } } -/// RLPx `p2p` protocol version +/// `RLPx` `p2p` protocol version #[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[cfg_attr(any(test, feature = "arbitrary"), derive(arbitrary::Arbitrary))] @@ -259,12 +249,4 @@ mod tests { assert_eq!(result, expected); } } - - #[test] - fn test_eth_version_total_messages() { - assert_eq!(EthVersion::Eth66.total_messages(), 15); - assert_eq!(EthVersion::Eth67.total_messages(), 13); - assert_eq!(EthVersion::Eth68.total_messages(), 13); - assert_eq!(EthVersion::Eth69.total_messages(), 11); - } } diff --git a/crates/net/eth-wire/Cargo.toml b/crates/net/eth-wire/Cargo.toml index 93a3ed4aa77..47c372dba24 100644 --- a/crates/net/eth-wire/Cargo.toml +++ b/crates/net/eth-wire/Cargo.toml @@ -51,7 +51,6 @@ reth-tracing.workspace = true alloy-consensus.workspace = true test-fuzz.workspace = true tokio = { workspace = true, features = ["rt", "rt-multi-thread"] } -tokio-util = { workspace = true, features = ["io", "codec"] } rand.workspace = true secp256k1 = { workspace = true, features = ["global-context", "std", "recovery"] } rand_08.workspace = true diff --git a/crates/net/eth-wire/src/capability.rs b/crates/net/eth-wire/src/capability.rs index 69ce6a43d3f..9b706a02cf9 100644 --- a/crates/net/eth-wire/src/capability.rs +++ b/crates/net/eth-wire/src/capability.rs @@ -134,7 +134,7 @@ impl SharedCapability { /// Returns the number of protocol messages supported by this capability. pub const fn num_messages(&self) -> u8 { match self { - Self::Eth { version: _version, .. } => EthMessageID::max() + 1, + Self::Eth { version, .. } => EthMessageID::message_count(*version), Self::UnknownCapability { messages, .. } => *messages, } } @@ -238,13 +238,13 @@ impl SharedCapabilities { /// Returns the number of shared capabilities. #[inline] - pub fn len(&self) -> usize { + pub const fn len(&self) -> usize { self.0.len() } /// Returns true if there are no shared capabilities. #[inline] - pub fn is_empty(&self) -> bool { + pub const fn is_empty(&self) -> bool { self.0.is_empty() } } @@ -534,7 +534,7 @@ mod tests { let mut encoded = Vec::new(); msg.encode(&mut encoded); - // Decode the bytes back into RawCapbailitMessage + // Decode the bytes back into RawCapabilityMessage let decoded = RawCapabilityMessage::decode(&mut &encoded[..]).unwrap(); // Verify that the decoded message matches the original diff --git a/crates/net/eth-wire/src/errors/eth.rs b/crates/net/eth-wire/src/errors/eth.rs index 5e3cbbdb9af..a1624113826 100644 --- a/crates/net/eth-wire/src/errors/eth.rs +++ b/crates/net/eth-wire/src/errors/eth.rs @@ -110,4 +110,15 @@ pub enum EthHandshakeError { /// The maximum allowed bit length for the total difficulty. maximum: usize, }, + #[error("earliest block > latest block: got {got}, latest {latest}")] + /// Earliest block > latest block. + EarliestBlockGreaterThanLatestBlock { + /// The earliest block. + got: u64, + /// The latest block. + latest: u64, + }, + #[error("blockhash is zero")] + /// Blockhash is zero. + BlockhashZero, } diff --git a/crates/net/eth-wire/src/eth_snap_stream.rs b/crates/net/eth-wire/src/eth_snap_stream.rs index 5691184c670..43b91a7fd50 100644 --- a/crates/net/eth-wire/src/eth_snap_stream.rs +++ b/crates/net/eth-wire/src/eth_snap_stream.rs @@ -44,7 +44,7 @@ pub enum EthSnapStreamError { StatusNotInHandshake, } -/// Combined message type that include either eth or snao protocol messages +/// Combined message type that include either eth or snap protocol messages #[derive(Debug)] pub enum EthSnapMessage { /// An Ethereum protocol message @@ -223,7 +223,7 @@ where // and eth message IDs are <= [`EthMessageID::max()`], // snap message IDs are > [`EthMessageID::max()`]. // See also . - if message_id <= EthMessageID::max() { + if message_id <= EthMessageID::max(self.eth_version) { let mut buf = bytes.as_ref(); match ProtocolMessage::decode_message(self.eth_version, &mut buf) { Ok(protocol_msg) => { @@ -236,16 +236,17 @@ where Err(EthSnapStreamError::InvalidMessage(self.eth_version, err.to_string())) } } - } else if message_id > EthMessageID::max() && - message_id <= EthMessageID::max() + 1 + SnapMessageId::TrieNodes as u8 + } else if message_id > EthMessageID::max(self.eth_version) && + message_id <= + EthMessageID::message_count(self.eth_version) + SnapMessageId::TrieNodes as u8 { // Checks for multiplexed snap message IDs : // - message_id > EthMessageID::max() : ensures it's not an eth message - // - message_id <= EthMessageID::max() + 1 + snap_max : ensures it's within valid snap - // range + // - message_id <= EthMessageID::message_count() + snap_max : ensures it's within valid + // snap range // Message IDs are assigned lexicographically during capability negotiation // So real_snap_id = multiplexed_id - num_eth_messages - let adjusted_message_id = message_id - (EthMessageID::max() + 1); + let adjusted_message_id = message_id - EthMessageID::message_count(self.eth_version); let mut buf = &bytes[1..]; match SnapProtocolMessage::decode(adjusted_message_id, &mut buf) { @@ -275,7 +276,7 @@ where let encoded = message.encode(); let message_id = encoded[0]; - let adjusted_id = message_id + EthMessageID::max() + 1; + let adjusted_id = message_id + EthMessageID::message_count(self.eth_version); let mut adjusted = Vec::with_capacity(encoded.len()); adjusted.push(adjusted_id); @@ -396,7 +397,7 @@ mod tests { let inner = EthSnapStreamInner::::new(EthVersion::Eth67); // Create a bytes buffer with eth message ID at the max boundary with minimal content - let eth_max_id = EthMessageID::max(); + let eth_max_id = EthMessageID::max(EthVersion::Eth67); let mut eth_boundary_bytes = BytesMut::new(); eth_boundary_bytes.extend_from_slice(&[eth_max_id]); eth_boundary_bytes.extend_from_slice(&[0, 0]); diff --git a/crates/net/eth-wire/src/ethstream.rs b/crates/net/eth-wire/src/ethstream.rs index a9afd29a20d..e2c041bd1a8 100644 --- a/crates/net/eth-wire/src/ethstream.rs +++ b/crates/net/eth-wire/src/ethstream.rs @@ -10,7 +10,7 @@ use crate::{ message::{EthBroadcastMessage, ProtocolBroadcastMessage}, p2pstream::HANDSHAKE_TIMEOUT, CanDisconnect, DisconnectReason, EthMessage, EthNetworkPrimitives, EthVersion, ProtocolMessage, - Status, + UnifiedStatus, }; use alloy_primitives::bytes::{Bytes, BytesMut}; use alloy_rlp::Encodable; @@ -32,9 +32,6 @@ use tracing::{debug, trace}; // https://github.com/ethereum/go-ethereum/blob/30602163d5d8321fbc68afdcbbaf2362b2641bde/eth/protocols/eth/protocol.go#L50 pub const MAX_MESSAGE_SIZE: usize = 10 * 1024 * 1024; -/// [`MAX_STATUS_SIZE`] is the maximum cap on the size of the initial status message -pub(crate) const MAX_STATUS_SIZE: usize = 500 * 1024; - /// An un-authenticated [`EthStream`]. This is consumed and returns a [`EthStream`] after the /// `Status` handshake is completed. #[pin_project] @@ -64,21 +61,24 @@ where /// Consumes the [`UnauthedEthStream`] and returns an [`EthStream`] after the `Status` /// handshake is completed successfully. This also returns the `Status` message sent by the /// remote peer. + /// + /// Caution: This expects that the [`UnifiedStatus`] has the proper eth version configured, with + /// ETH69 the initial status message changed. pub async fn handshake( self, - status: Status, + status: UnifiedStatus, fork_filter: ForkFilter, - ) -> Result<(EthStream, Status), EthStreamError> { + ) -> Result<(EthStream, UnifiedStatus), EthStreamError> { self.handshake_with_timeout(status, fork_filter, HANDSHAKE_TIMEOUT).await } /// Wrapper around handshake which enforces a timeout. pub async fn handshake_with_timeout( self, - status: Status, + status: UnifiedStatus, fork_filter: ForkFilter, timeout_limit: Duration, - ) -> Result<(EthStream, Status), EthStreamError> { + ) -> Result<(EthStream, UnifiedStatus), EthStreamError> { timeout(timeout_limit, Self::handshake_without_timeout(self, status, fork_filter)) .await .map_err(|_| EthStreamError::StreamTimeout)? @@ -87,20 +87,21 @@ where /// Handshake with no timeout pub async fn handshake_without_timeout( mut self, - status: Status, + status: UnifiedStatus, fork_filter: ForkFilter, - ) -> Result<(EthStream, Status), EthStreamError> { + ) -> Result<(EthStream, UnifiedStatus), EthStreamError> { trace!( - %status, + status = %status.into_message(), "sending eth status to peer" ); - EthereumEthHandshake(&mut self.inner).eth_handshake(status, fork_filter).await?; + let their_status = + EthereumEthHandshake(&mut self.inner).eth_handshake(status, fork_filter).await?; // now we can create the `EthStream` because the peer has successfully completed // the handshake let stream = EthStream::new(status.version, self.inner); - Ok((stream, status)) + Ok((stream, their_status)) } } @@ -276,15 +277,13 @@ where fn start_send(self: Pin<&mut Self>, item: EthMessage) -> Result<(), Self::Error> { if matches!(item, EthMessage::Status(_)) { - // TODO: to disconnect here we would need to do something similar to P2PStream's - // start_disconnect, which would ideally be a part of the CanDisconnect trait, or at - // least similar. - // - // Other parts of reth do not yet need traits like CanDisconnect because atm they work - // exclusively with EthStream>, where the inner P2PStream is accessible, - // allowing for its start_disconnect method to be called. - // - // self.project().inner.start_disconnect(DisconnectReason::ProtocolBreach); + // Attempt to disconnect the peer for protocol breach when trying to send Status + // message after handshake is complete + let mut this = self.project(); + // We can't await the disconnect future here since this is a synchronous method, + // but we can start the disconnect process. The actual disconnect will be handled + // asynchronously by the caller or the stream's poll methods. + let _disconnect_future = this.inner.disconnect(DisconnectReason::ProtocolBreach); return Err(EthStreamError::EthHandshakeError(EthHandshakeError::StatusNotInHandshake)) } @@ -328,14 +327,14 @@ mod tests { hello::DEFAULT_TCP_PORT, p2pstream::UnauthedP2PStream, EthMessage, EthStream, EthVersion, HelloMessageWithProtocols, PassthroughCodec, - ProtocolVersion, Status, + ProtocolVersion, Status, StatusMessage, }; use alloy_chains::NamedChain; use alloy_primitives::{bytes::Bytes, B256, U256}; use alloy_rlp::Decodable; use futures::{SinkExt, StreamExt}; use reth_ecies::stream::ECIESStream; - use reth_eth_wire_types::EthNetworkPrimitives; + use reth_eth_wire_types::{EthNetworkPrimitives, UnifiedStatus}; use reth_ethereum_forks::{ForkFilter, Head}; use reth_network_peers::pk2id; use secp256k1::{SecretKey, SECP256K1}; @@ -357,11 +356,12 @@ mod tests { // Pass the current fork id. forkid: fork_filter.current(), }; + let unified_status = UnifiedStatus::from_message(StatusMessage::Legacy(status)); let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let local_addr = listener.local_addr().unwrap(); - let status_clone = status; + let status_clone = unified_status; let fork_filter_clone = fork_filter.clone(); let handle = tokio::spawn(async move { // roughly based off of the design of tokio::net::TcpListener @@ -381,12 +381,12 @@ mod tests { // try to connect let (_, their_status) = UnauthedEthStream::new(sink) - .handshake::(status, fork_filter) + .handshake::(unified_status, fork_filter) .await .unwrap(); // their status is a clone of our status, these should be equal - assert_eq!(their_status, status); + assert_eq!(their_status, unified_status); // wait for it to finish handle.await.unwrap(); @@ -406,11 +406,12 @@ mod tests { // Pass the current fork id. forkid: fork_filter.current(), }; + let unified_status = UnifiedStatus::from_message(StatusMessage::Legacy(status)); let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let local_addr = listener.local_addr().unwrap(); - let status_clone = status; + let status_clone = unified_status; let fork_filter_clone = fork_filter.clone(); let handle = tokio::spawn(async move { // roughly based off of the design of tokio::net::TcpListener @@ -430,12 +431,12 @@ mod tests { // try to connect let (_, their_status) = UnauthedEthStream::new(sink) - .handshake::(status, fork_filter) + .handshake::(unified_status, fork_filter) .await .unwrap(); // their status is a clone of our status, these should be equal - assert_eq!(their_status, status); + assert_eq!(their_status, unified_status); // await the other handshake handle.await.unwrap(); @@ -455,11 +456,12 @@ mod tests { // Pass the current fork id. forkid: fork_filter.current(), }; + let unified_status = UnifiedStatus::from_message(StatusMessage::Legacy(status)); let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let local_addr = listener.local_addr().unwrap(); - let status_clone = status; + let status_clone = unified_status; let fork_filter_clone = fork_filter.clone(); let handle = tokio::spawn(async move { // roughly based off of the design of tokio::net::TcpListener @@ -483,7 +485,7 @@ mod tests { // try to connect let handshake_res = UnauthedEthStream::new(sink) - .handshake::(status, fork_filter) + .handshake::(unified_status, fork_filter) .await; // this handshake should also fail due to td too high @@ -599,8 +601,9 @@ mod tests { // Pass the current fork id. forkid: fork_filter.current(), }; + let unified_status = UnifiedStatus::from_message(StatusMessage::Legacy(status)); - let status_copy = status; + let status_copy = unified_status; let fork_filter_clone = fork_filter.clone(); let test_msg_clone = test_msg.clone(); let handle = tokio::spawn(async move { @@ -647,8 +650,10 @@ mod tests { let unauthed_stream = UnauthedP2PStream::new(sink); let (p2p_stream, _) = unauthed_stream.handshake(client_hello).await.unwrap(); - let (mut client_stream, _) = - UnauthedEthStream::new(p2p_stream).handshake(status, fork_filter).await.unwrap(); + let (mut client_stream, _) = UnauthedEthStream::new(p2p_stream) + .handshake(unified_status, fork_filter) + .await + .unwrap(); client_stream.send(test_msg).await.unwrap(); @@ -670,11 +675,12 @@ mod tests { // Pass the current fork id. forkid: fork_filter.current(), }; + let unified_status = UnifiedStatus::from_message(StatusMessage::Legacy(status)); let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let local_addr = listener.local_addr().unwrap(); - let status_clone = status; + let status_clone = unified_status; let fork_filter_clone = fork_filter.clone(); let _handle = tokio::spawn(async move { // Delay accepting the connection for longer than the client's timeout period @@ -697,7 +703,7 @@ mod tests { // try to connect let handshake_result = UnauthedEthStream::new(sink) .handshake_with_timeout::( - status, + unified_status, fork_filter, Duration::from_secs(1), ) @@ -743,4 +749,48 @@ mod tests { handle.await.unwrap(); } + + #[tokio::test] + async fn status_message_after_handshake_triggers_disconnect() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let local_addr = listener.local_addr().unwrap(); + + let handle = tokio::spawn(async move { + let (incoming, _) = listener.accept().await.unwrap(); + let stream = PassthroughCodec::default().framed(incoming); + let mut stream = EthStream::<_, EthNetworkPrimitives>::new(EthVersion::Eth67, stream); + + // Try to send a Status message after handshake - this should trigger disconnect + let status = Status { + version: EthVersion::Eth67, + chain: NamedChain::Mainnet.into(), + total_difficulty: U256::ZERO, + blockhash: B256::random(), + genesis: B256::random(), + forkid: ForkFilter::new(Head::default(), B256::random(), 0, Vec::new()).current(), + }; + let status_message = + EthMessage::::Status(StatusMessage::Legacy(status)); + + // This should return an error and trigger disconnect + let result = stream.send(status_message).await; + assert!(result.is_err()); + assert!(matches!( + result.unwrap_err(), + EthStreamError::EthHandshakeError(EthHandshakeError::StatusNotInHandshake) + )); + }); + + let outgoing = TcpStream::connect(local_addr).await.unwrap(); + let sink = PassthroughCodec::default().framed(outgoing); + let mut client_stream = EthStream::<_, EthNetworkPrimitives>::new(EthVersion::Eth67, sink); + + // Send a valid message to keep the connection alive + let test_msg = EthMessage::::NewBlockHashes( + vec![BlockHashNumber { hash: B256::random(), number: 5 }].into(), + ); + client_stream.send(test_msg).await.unwrap(); + + handle.await.unwrap(); + } } diff --git a/crates/net/eth-wire/src/handshake.rs b/crates/net/eth-wire/src/handshake.rs index a7c9384c1ba..f604f1fca11 100644 --- a/crates/net/eth-wire/src/handshake.rs +++ b/crates/net/eth-wire/src/handshake.rs @@ -1,12 +1,13 @@ use crate::{ errors::{EthHandshakeError, EthStreamError, P2PStreamError}, - ethstream::MAX_STATUS_SIZE, + ethstream::MAX_MESSAGE_SIZE, CanDisconnect, }; use bytes::{Bytes, BytesMut}; use futures::{Sink, SinkExt, Stream}; use reth_eth_wire_types::{ - DisconnectReason, EthMessage, EthNetworkPrimitives, ProtocolMessage, Status, StatusMessage, + DisconnectReason, EthMessage, EthNetworkPrimitives, ProtocolMessage, StatusMessage, + UnifiedStatus, }; use reth_ethereum_forks::ForkFilter; use reth_primitives_traits::GotExpected; @@ -21,10 +22,10 @@ pub trait EthRlpxHandshake: Debug + Send + Sync + 'static { fn handshake<'a>( &'a self, unauth: &'a mut dyn UnauthEth, - status: Status, + status: UnifiedStatus, fork_filter: ForkFilter, timeout_limit: Duration, - ) -> Pin> + 'a + Send>>; + ) -> Pin> + 'a + Send>>; } /// An unauthenticated stream that can send and receive messages. @@ -57,10 +58,10 @@ impl EthRlpxHandshake for EthHandshake { fn handshake<'a>( &'a self, unauth: &'a mut dyn UnauthEth, - status: Status, + status: UnifiedStatus, fork_filter: ForkFilter, timeout_limit: Duration, - ) -> Pin> + 'a + Send>> { + ) -> Pin> + 'a + Send>> { Box::pin(async move { timeout(timeout_limit, EthereumEthHandshake(unauth).eth_handshake(status, fork_filter)) .await @@ -81,18 +82,18 @@ where /// Performs the `eth` rlpx protocol handshake using the given input stream. pub async fn eth_handshake( self, - status: Status, + unified_status: UnifiedStatus, fork_filter: ForkFilter, - ) -> Result { + ) -> Result { let unauth = self.0; + + let status = unified_status.into_message(); + // Send our status message - let status_msg = - alloy_rlp::encode(ProtocolMessage::::from(EthMessage::< - EthNetworkPrimitives, - >::Status( - StatusMessage::Legacy(status), - ))) - .into(); + let status_msg = alloy_rlp::encode(ProtocolMessage::::from( + EthMessage::Status(status), + )) + .into(); unauth.send(status_msg).await.map_err(EthStreamError::from)?; // Receive peer's response @@ -109,7 +110,7 @@ where } }; - if their_msg.len() > MAX_STATUS_SIZE { + if their_msg.len() > MAX_MESSAGE_SIZE { unauth .disconnect(DisconnectReason::ProtocolBreach) .await @@ -117,7 +118,7 @@ where return Err(EthStreamError::MessageTooBig(their_msg.len())); } - let version = status.version; + let version = status.version(); let msg = match ProtocolMessage::::decode_message( version, &mut their_msg.as_ref(), @@ -138,14 +139,14 @@ where EthMessage::Status(their_status_message) => { trace!("Validating incoming ETH status from peer"); - if status.genesis != their_status_message.genesis() { + if status.genesis() != their_status_message.genesis() { unauth .disconnect(DisconnectReason::ProtocolBreach) .await .map_err(EthStreamError::from)?; return Err(EthHandshakeError::MismatchedGenesis( GotExpected { - expected: status.genesis, + expected: status.genesis(), got: their_status_message.genesis(), } .into(), @@ -153,38 +154,40 @@ where .into()); } - if status.version != their_status_message.version() { + if status.version() != their_status_message.version() { unauth .disconnect(DisconnectReason::ProtocolBreach) .await .map_err(EthStreamError::from)?; return Err(EthHandshakeError::MismatchedProtocolVersion(GotExpected { got: their_status_message.version(), - expected: status.version, + expected: status.version(), }) .into()); } - if status.chain != *their_status_message.chain() { + if *status.chain() != *their_status_message.chain() { unauth .disconnect(DisconnectReason::ProtocolBreach) .await .map_err(EthStreamError::from)?; return Err(EthHandshakeError::MismatchedChain(GotExpected { got: *their_status_message.chain(), - expected: status.chain, + expected: *status.chain(), }) .into()); } - // Ensure total difficulty is reasonable - if status.total_difficulty.bit_len() > 160 { + // Ensure peer's total difficulty is reasonable + if let StatusMessage::Legacy(s) = their_status_message && + s.total_difficulty.bit_len() > 160 + { unauth .disconnect(DisconnectReason::ProtocolBreach) .await .map_err(EthStreamError::from)?; return Err(EthHandshakeError::TotalDifficultyBitLenTooLarge { - got: status.total_difficulty.bit_len(), + got: s.total_difficulty.bit_len(), maximum: 160, } .into()); @@ -202,7 +205,21 @@ where return Err(err.into()); } - Ok(their_status_message.to_legacy()) + if let StatusMessage::Eth69(s) = their_status_message { + if s.earliest > s.latest { + return Err(EthHandshakeError::EarliestBlockGreaterThanLatestBlock { + got: s.earliest, + latest: s.latest, + } + .into()); + } + + if s.blockhash.is_zero() { + return Err(EthHandshakeError::BlockhashZero.into()); + } + } + + Ok(UnifiedStatus::from_message(their_status_message)) } _ => { unauth diff --git a/crates/net/eth-wire/src/hello.rs b/crates/net/eth-wire/src/hello.rs index 58432520a2a..40deebb6310 100644 --- a/crates/net/eth-wire/src/hello.rs +++ b/crates/net/eth-wire/src/hello.rs @@ -100,7 +100,7 @@ impl HelloMessageWithProtocols { // TODO: determine if we should allow for the extra fields at the end like EIP-706 suggests /// Raw rlpx protocol message used in the `p2p` handshake, containing information about the -/// supported RLPx protocol version and capabilities. +/// supported `RLPx` protocol version and capabilities. /// /// See also #[derive(Clone, Debug, PartialEq, Eq, RlpEncodable, RlpDecodable)] @@ -205,7 +205,7 @@ impl HelloMessageBuilder { protocol_version: protocol_version.unwrap_or_default(), client_version: client_version.unwrap_or_else(|| RETH_CLIENT_VERSION.to_string()), protocols: protocols.unwrap_or_else(|| { - vec![EthVersion::Eth68.into(), EthVersion::Eth67.into(), EthVersion::Eth66.into()] + EthVersion::ALL_VERSIONS.iter().copied().map(Into::into).collect() }), port: port.unwrap_or(DEFAULT_TCP_PORT), id, @@ -215,7 +215,10 @@ impl HelloMessageBuilder { #[cfg(test)] mod tests { - use crate::{p2pstream::P2PMessage, Capability, EthVersion, HelloMessage, ProtocolVersion}; + use crate::{ + p2pstream::P2PMessage, Capability, EthVersion, HelloMessage, HelloMessageWithProtocols, + ProtocolVersion, + }; use alloy_rlp::{Decodable, Encodable, EMPTY_STRING_CODE}; use reth_network_peers::pk2id; use secp256k1::{SecretKey, SECP256K1}; @@ -258,6 +261,20 @@ mod tests { assert_eq!(hello_encoded.len(), hello.length()); } + #[test] + fn test_default_protocols_include_eth69() { + // ensure that the default protocol list includes Eth69 as the latest version + let secret_key = SecretKey::new(&mut rand_08::thread_rng()); + let id = pk2id(&secret_key.public_key(SECP256K1)); + let hello = HelloMessageWithProtocols::builder(id).build(); + + let has_eth69 = hello + .protocols + .iter() + .any(|p| p.cap.name == "eth" && p.cap.version == EthVersion::Eth69 as usize); + assert!(has_eth69, "Default protocols should include Eth69"); + } + #[test] fn hello_message_id_prefix() { // ensure that the hello message id is prefixed diff --git a/crates/net/eth-wire/src/lib.rs b/crates/net/eth-wire/src/lib.rs index a2cb35ae7fd..0248378a0ac 100644 --- a/crates/net/eth-wire/src/lib.rs +++ b/crates/net/eth-wire/src/lib.rs @@ -11,7 +11,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] pub mod capability; mod disconnect; diff --git a/crates/net/eth-wire/src/multiplex.rs b/crates/net/eth-wire/src/multiplex.rs index 184080cfcb2..489fd86e7dc 100644 --- a/crates/net/eth-wire/src/multiplex.rs +++ b/crates/net/eth-wire/src/multiplex.rs @@ -13,14 +13,17 @@ use std::{ future::Future, io, pin::{pin, Pin}, + sync::Arc, task::{ready, Context, Poll}, }; use crate::{ capability::{SharedCapabilities, SharedCapability, UnsupportedCapabilityError}, errors::{EthStreamError, P2PStreamError}, + handshake::EthRlpxHandshake, p2pstream::DisconnectP2P, - CanDisconnect, Capability, DisconnectReason, EthStream, P2PStream, Status, UnauthedEthStream, + CanDisconnect, Capability, DisconnectReason, EthStream, P2PStream, UnifiedStatus, + HANDSHAKE_TIMEOUT, }; use bytes::{Bytes, BytesMut}; use futures::{Sink, SinkExt, Stream, StreamExt, TryStream, TryStreamExt}; @@ -134,7 +137,7 @@ impl RlpxProtocolMultiplexer { /// This accepts a closure that does a handshake with the remote peer and returns a tuple of the /// primary stream and extra data. /// - /// See also [`UnauthedEthStream::handshake`] + /// See also [`UnauthedEthStream::handshake`](crate::UnauthedEthStream) pub async fn into_satellite_stream_with_tuple_handshake( mut self, cap: &Capability, @@ -166,6 +169,7 @@ impl RlpxProtocolMultiplexer { // complete loop { tokio::select! { + biased; Some(Ok(msg)) = self.inner.conn.next() => { // Ensure the message belongs to the primary protocol let Some(offset) = msg.first().copied() @@ -187,6 +191,10 @@ impl RlpxProtocolMultiplexer { Some(msg) = from_primary.recv() => { self.inner.conn.send(msg).await.map_err(Into::into)?; } + // Poll all subprotocols for new messages + msg = ProtocolsPoller::new(&mut self.inner.protocols) => { + self.inner.conn.send(msg.map_err(Into::into)?).await.map_err(Into::into)?; + } res = &mut f => { let (st, extra) = res?; return Ok((RlpxSatelliteStream { @@ -204,22 +212,28 @@ impl RlpxProtocolMultiplexer { } /// Converts this multiplexer into a [`RlpxSatelliteStream`] with eth protocol as the given - /// primary protocol. + /// primary protocol and the handshake implementation. pub async fn into_eth_satellite_stream( self, - status: Status, + status: UnifiedStatus, fork_filter: ForkFilter, - ) -> Result<(RlpxSatelliteStream>, Status), EthStreamError> + handshake: Arc, + ) -> Result<(RlpxSatelliteStream>, UnifiedStatus), EthStreamError> where St: Stream> + Sink + Unpin, { let eth_cap = self.inner.conn.shared_capabilities().eth_version()?; - self.into_satellite_stream_with_tuple_handshake( - &Capability::eth(eth_cap), - move |proxy| async move { - UnauthedEthStream::new(proxy).handshake(status, fork_filter).await - }, - ) + self.into_satellite_stream_with_tuple_handshake(&Capability::eth(eth_cap), move |proxy| { + let handshake = handshake.clone(); + async move { + let mut unauth = UnauthProxy { inner: proxy }; + let their_status = handshake + .handshake(&mut unauth, status, fork_filter, HANDSHAKE_TIMEOUT) + .await?; + let eth_stream = EthStream::new(eth_cap, unauth.into_inner()); + Ok((eth_stream, their_status)) + } + }) .await } } @@ -318,9 +332,9 @@ impl ProtocolProxy { return Ok(msg); } - let mut masked = Vec::from(msg); + let mut masked: BytesMut = msg.into(); masked[0] = masked[0].checked_add(offset).ok_or(io::ErrorKind::InvalidInput)?; - Ok(masked.into()) + Ok(masked.freeze()) } /// Unmasks the message ID of a message received from the wire. @@ -371,11 +385,61 @@ impl CanDisconnect for ProtocolProxy { &mut self, _reason: DisconnectReason, ) -> Pin>::Error>> + Send + '_>> { - // TODO handle disconnects Box::pin(async move { Ok(()) }) } } +/// Adapter so the injected `EthRlpxHandshake` can run over a multiplexed `ProtocolProxy` +/// using the same error type expectations (`P2PStreamError`). +#[derive(Debug)] +struct UnauthProxy { + inner: ProtocolProxy, +} + +impl UnauthProxy { + fn into_inner(self) -> ProtocolProxy { + self.inner + } +} + +impl Stream for UnauthProxy { + type Item = Result; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + self.inner.poll_next_unpin(cx).map(|opt| opt.map(|res| res.map_err(P2PStreamError::from))) + } +} + +impl Sink for UnauthProxy { + type Error = P2PStreamError; + + fn poll_ready(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + self.inner.poll_ready_unpin(cx).map_err(P2PStreamError::from) + } + + fn start_send(mut self: Pin<&mut Self>, item: Bytes) -> Result<(), Self::Error> { + self.inner.start_send_unpin(item).map_err(P2PStreamError::from) + } + + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + self.inner.poll_flush_unpin(cx).map_err(P2PStreamError::from) + } + + fn poll_close(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + self.inner.poll_close_unpin(cx).map_err(P2PStreamError::from) + } +} + +impl CanDisconnect for UnauthProxy { + fn disconnect( + &mut self, + reason: DisconnectReason, + ) -> Pin>::Error>> + Send + '_>> { + let fut = self.inner.disconnect(reason); + Box::pin(async move { fut.await.map_err(P2PStreamError::from) }) + } +} + /// A connection channel to receive _`non_empty`_ messages for the negotiated protocol. /// /// This is a [Stream] that returns raw bytes of the received messages for this protocol. @@ -665,15 +729,56 @@ impl fmt::Debug for ProtocolStream { } } +/// Helper to poll multiple protocol streams in a `tokio::select`! branch +struct ProtocolsPoller<'a> { + protocols: &'a mut Vec, +} + +impl<'a> ProtocolsPoller<'a> { + const fn new(protocols: &'a mut Vec) -> Self { + Self { protocols } + } +} + +impl<'a> Future for ProtocolsPoller<'a> { + type Output = Result; + + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + // Process protocols in reverse order, like the existing pattern + for idx in (0..self.protocols.len()).rev() { + let mut proto = self.protocols.swap_remove(idx); + match proto.poll_next_unpin(cx) { + Poll::Ready(Some(Err(err))) => { + self.protocols.push(proto); + return Poll::Ready(Err(P2PStreamError::from(err))) + } + Poll::Ready(Some(Ok(msg))) => { + // Got a message, put protocol back and return the message + self.protocols.push(proto); + return Poll::Ready(Ok(msg)); + } + _ => { + // push it back because we still want to complete the handshake first + self.protocols.push(proto); + } + } + } + + // All protocols processed, nothing ready + Poll::Pending + } +} + #[cfg(test)] mod tests { use super::*; use crate::{ + handshake::EthHandshake, test_utils::{ connect_passthrough, eth_handshake, eth_hello, proto::{test_hello, TestProtoMessage}, }, - UnauthedP2PStream, + UnauthedEthStream, UnauthedP2PStream, }; use reth_eth_wire_types::EthNetworkPrimitives; use tokio::{net::TcpListener, sync::oneshot}; @@ -735,7 +840,11 @@ mod tests { let (conn, _) = UnauthedP2PStream::new(stream).handshake(server_hello).await.unwrap(); let (mut st, _their_status) = RlpxProtocolMultiplexer::new(conn) - .into_eth_satellite_stream::(other_status, other_fork_filter) + .into_eth_satellite_stream::( + other_status, + other_fork_filter, + Arc::new(EthHandshake::default()), + ) .await .unwrap(); @@ -766,7 +875,11 @@ mod tests { let conn = connect_passthrough(local_addr, test_hello().0).await; let (mut st, _their_status) = RlpxProtocolMultiplexer::new(conn) - .into_eth_satellite_stream::(status, fork_filter) + .into_eth_satellite_stream::( + status, + fork_filter, + Arc::new(EthHandshake::default()), + ) .await .unwrap(); diff --git a/crates/net/eth-wire/src/p2pstream.rs b/crates/net/eth-wire/src/p2pstream.rs index 0d0747d4358..e794795b1c4 100644 --- a/crates/net/eth-wire/src/p2pstream.rs +++ b/crates/net/eth-wire/src/p2pstream.rs @@ -203,7 +203,7 @@ where } } -/// A P2PStream wraps over any `Stream` that yields bytes and makes it compatible with `p2p` +/// A `P2PStream` wraps over any `Stream` that yields bytes and makes it compatible with `p2p` /// protocol messages. /// /// This stream supports multiple shared capabilities, that were negotiated during the handshake. diff --git a/crates/net/eth-wire/src/pinger.rs b/crates/net/eth-wire/src/pinger.rs index d93404c5f97..d488de20f53 100644 --- a/crates/net/eth-wire/src/pinger.rs +++ b/crates/net/eth-wire/src/pinger.rs @@ -1,5 +1,6 @@ use crate::errors::PingerError; use std::{ + future::Future, pin::Pin, task::{Context, Poll}, time::Duration, @@ -7,13 +8,13 @@ use std::{ use tokio::time::{Instant, Interval, Sleep}; use tokio_stream::Stream; -/// The pinger is a state machine that is created with a maximum number of pongs that can be -/// missed. +/// The pinger is a simple state machine that sends a ping, waits for a pong, +/// and transitions to timeout if the pong is not received within the timeout. #[derive(Debug)] pub(crate) struct Pinger { /// The timer used for the next ping. ping_interval: Interval, - /// The timer used for the next ping. + /// The timer used to detect a ping timeout. timeout_timer: Pin>, /// The timeout duration for each ping. timeout: Duration, @@ -38,7 +39,7 @@ impl Pinger { } /// Mark a pong as received, and transition the pinger to the `Ready` state if it was in the - /// `WaitingForPong` state. Unsets the sleep timer. + /// `WaitingForPong` state. Resets readiness by resetting the ping interval. pub(crate) fn on_pong(&mut self) -> Result<(), PingerError> { match self.state { PingState::Ready => Err(PingerError::UnexpectedPong), @@ -77,7 +78,7 @@ impl Pinger { } } PingState::WaitingForPong => { - if self.timeout_timer.is_elapsed() { + if self.timeout_timer.as_mut().poll(cx).is_ready() { self.state = PingState::TimedOut; return Poll::Ready(Ok(PingerEvent::Timeout)) } diff --git a/crates/net/eth-wire/src/protocol.rs b/crates/net/eth-wire/src/protocol.rs index 13c39d46e1f..16ec62b7cd7 100644 --- a/crates/net/eth-wire/src/protocol.rs +++ b/crates/net/eth-wire/src/protocol.rs @@ -26,7 +26,7 @@ impl Protocol { /// Returns the corresponding eth capability for the given version. pub const fn eth(version: EthVersion) -> Self { let cap = Capability::eth(version); - let messages = version.total_messages(); + let messages = EthMessageID::message_count(version); Self::new(cap, messages) } @@ -52,10 +52,7 @@ impl Protocol { } /// The number of values needed to represent all message IDs of capability. - pub fn messages(&self) -> u8 { - if self.cap.is_eth() { - return EthMessageID::max() + 1 - } + pub const fn messages(&self) -> u8 { self.messages } } @@ -74,3 +71,18 @@ pub(crate) struct ProtoVersion { /// Version of the protocol pub(crate) version: usize, } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_protocol_eth_message_count() { + // Test that Protocol::eth() returns correct message counts for each version + // This ensures that EthMessageID::message_count() produces the expected results + assert_eq!(Protocol::eth(EthVersion::Eth66).messages(), 17); + assert_eq!(Protocol::eth(EthVersion::Eth67).messages(), 17); + assert_eq!(Protocol::eth(EthVersion::Eth68).messages(), 17); + assert_eq!(Protocol::eth(EthVersion::Eth69).messages(), 18); + } +} diff --git a/crates/net/eth-wire/src/test_utils.rs b/crates/net/eth-wire/src/test_utils.rs index ee989d00d54..5e90d864439 100644 --- a/crates/net/eth-wire/src/test_utils.rs +++ b/crates/net/eth-wire/src/test_utils.rs @@ -4,7 +4,7 @@ use crate::{ hello::DEFAULT_TCP_PORT, EthVersion, HelloMessageWithProtocols, P2PStream, ProtocolVersion, - Status, UnauthedP2PStream, + Status, StatusMessage, UnauthedP2PStream, UnifiedStatus, }; use alloy_chains::Chain; use alloy_primitives::{B256, U256}; @@ -32,7 +32,7 @@ pub fn eth_hello() -> (HelloMessageWithProtocols, SecretKey) { } /// Returns testing eth handshake status and fork filter. -pub fn eth_handshake() -> (Status, ForkFilter) { +pub fn eth_handshake() -> (UnifiedStatus, ForkFilter) { let genesis = B256::random(); let fork_filter = ForkFilter::new(Head::default(), genesis, 0, Vec::new()); @@ -45,7 +45,9 @@ pub fn eth_handshake() -> (Status, ForkFilter) { // Pass the current fork id. forkid: fork_filter.current(), }; - (status, fork_filter) + let unified_status = UnifiedStatus::from_message(StatusMessage::Legacy(status)); + + (unified_status, fork_filter) } /// Connects to a remote node and returns an authenticated `P2PStream` with the remote node. @@ -60,7 +62,7 @@ pub async fn connect_passthrough( p2p_stream } -/// An Rplx subprotocol for testing +/// An Rlpx subprotocol for testing pub mod proto { use super::*; use crate::{protocol::Protocol, Capability}; diff --git a/crates/net/eth-wire/tests/fuzz_roundtrip.rs b/crates/net/eth-wire/tests/fuzz_roundtrip.rs index f09035f45de..9cd9194ab1f 100644 --- a/crates/net/eth-wire/tests/fuzz_roundtrip.rs +++ b/crates/net/eth-wire/tests/fuzz_roundtrip.rs @@ -19,8 +19,8 @@ where } /// This method delegates to `roundtrip_encoding`, but is used to enforce that each type input to -/// the macro has a proper Default, Clone, and Serialize impl. These trait implementations are -/// necessary for test-fuzz to autogenerate a corpus. +/// the macro has proper `Clone` and `Serialize` impls. These trait implementations are necessary +/// for test-fuzz to autogenerate a corpus. /// /// If it makes sense to remove a Default impl from a type that we fuzz, this should prevent the /// fuzz test from compiling, rather than failing at runtime. diff --git a/crates/net/nat/src/lib.rs b/crates/net/nat/src/lib.rs index 3bdb3afc902..e39889ae16c 100644 --- a/crates/net/nat/src/lib.rs +++ b/crates/net/nat/src/lib.rs @@ -10,7 +10,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] pub mod net_if; @@ -25,7 +25,7 @@ use std::{ task::{Context, Poll}, time::Duration, }; -use tracing::{debug, error}; +use tracing::debug; use crate::net_if::resolve_net_if_ip; #[cfg(feature = "serde")] @@ -161,6 +161,11 @@ impl ResolveNatInterval { Self::with_interval(resolver, interval) } + /// Returns the resolver used by this interval + pub const fn resolver(&self) -> &NatResolver { + &self.resolver + } + /// Completes when the next [`IpAddr`] in the interval has been reached. pub async fn tick(&mut self) -> Option { poll_fn(|cx| self.poll_tick(cx)).await @@ -230,7 +235,8 @@ async fn resolve_external_ip_url_res(url: &str) -> Result { } async fn resolve_external_ip_url(url: &str) -> Option { - let response = reqwest::get(url).await.ok()?; + let client = reqwest::Client::builder().timeout(Duration::from_secs(10)).build().ok()?; + let response = client.get(url).send().await.ok()?; let response = response.error_for_status().ok()?; let text = response.text().await.ok()?; text.trim().parse().ok() diff --git a/crates/net/network-api/Cargo.toml b/crates/net/network-api/Cargo.toml index 4ecfa1f593e..b0ebed8bcfb 100644 --- a/crates/net/network-api/Cargo.toml +++ b/crates/net/network-api/Cargo.toml @@ -21,6 +21,8 @@ reth-tokio-util.workspace = true reth-ethereum-forks.workspace = true # ethereum +alloy-consensus.workspace = true +alloy-rpc-types-eth.workspace = true alloy-primitives = { workspace = true, features = ["getrandom"] } alloy-rpc-types-admin.workspace = true enr = { workspace = true, default-features = false, features = ["rust-secp256k1"] } @@ -44,4 +46,6 @@ serde = [ "alloy-primitives/serde", "enr/serde", "reth-ethereum-forks/serde", + "alloy-consensus/serde", + "alloy-rpc-types-eth/serde", ] diff --git a/crates/net/network-api/src/events.rs b/crates/net/network-api/src/events.rs index d71bd016173..8a5c7541490 100644 --- a/crates/net/network-api/src/events.rs +++ b/crates/net/network-api/src/events.rs @@ -4,7 +4,7 @@ use reth_eth_wire_types::{ message::RequestPair, BlockBodies, BlockHeaders, Capabilities, DisconnectReason, EthMessage, EthNetworkPrimitives, EthVersion, GetBlockBodies, GetBlockHeaders, GetNodeData, GetPooledTransactions, GetReceipts, NetworkPrimitives, NodeData, PooledTransactions, Receipts, - Status, + Receipts69, UnifiedStatus, }; use reth_ethereum_forks::ForkId; use reth_network_p2p::error::{RequestError, RequestResult}; @@ -63,7 +63,7 @@ pub struct SessionInfo { /// Capabilities the peer announced. pub capabilities: Arc, /// The status of the peer to which a session was established. - pub status: Arc, + pub status: Arc, /// Negotiated eth version of the session. pub version: EthVersion, /// The kind of peer this session represents @@ -229,6 +229,15 @@ pub enum PeerRequest { /// The channel to send the response for receipts. response: oneshot::Sender>>, }, + /// Requests receipts from the peer without bloom filter. + /// + /// The response should be sent through the channel. + GetReceipts69 { + /// The request for receipts. + request: GetReceipts, + /// The channel to send the response for receipts. + response: oneshot::Sender>>, + }, } // === impl PeerRequest === @@ -247,6 +256,7 @@ impl PeerRequest { Self::GetPooledTransactions { response, .. } => response.send(Err(err)).ok(), Self::GetNodeData { response, .. } => response.send(Err(err)).ok(), Self::GetReceipts { response, .. } => response.send(Err(err)).ok(), + Self::GetReceipts69 { response, .. } => response.send(Err(err)).ok(), }; } @@ -268,7 +278,7 @@ impl PeerRequest { Self::GetNodeData { request, .. } => { EthMessage::GetNodeData(RequestPair { request_id, message: request.clone() }) } - Self::GetReceipts { request, .. } => { + Self::GetReceipts { request, .. } | Self::GetReceipts69 { request, .. } => { EthMessage::GetReceipts(RequestPair { request_id, message: request.clone() }) } } diff --git a/crates/net/network-api/src/lib.rs b/crates/net/network-api/src/lib.rs index 662abc164c8..754463cb34f 100644 --- a/crates/net/network-api/src/lib.rs +++ b/crates/net/network-api/src/lib.rs @@ -11,7 +11,7 @@ html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256", issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] pub mod downloaders; /// Network Error @@ -35,7 +35,8 @@ pub use events::{ }; use reth_eth_wire_types::{ - capability::Capabilities, DisconnectReason, EthVersion, NetworkPrimitives, Status, + capability::Capabilities, Capability, DisconnectReason, EthVersion, NetworkPrimitives, + UnifiedStatus, }; use reth_network_p2p::sync::NetworkSyncUpdater; use reth_network_peers::NodeRecord; @@ -191,7 +192,7 @@ pub trait Peers: PeersInfo { /// Disconnect an existing connection to the given peer using the provided reason fn disconnect_peer_with_reason(&self, peer: PeerId, reason: DisconnectReason); - /// Connect to the given peer. NOTE: if the maximum number out outbound sessions is reached, + /// Connect to the given peer. NOTE: if the maximum number of outbound sessions is reached, /// this won't do anything. See `reth_network::SessionManager::dial_outbound`. fn connect_peer(&self, peer: PeerId, tcp_addr: SocketAddr) { self.connect_peer_kind(peer, PeerKind::Static, tcp_addr, None) @@ -238,7 +239,7 @@ pub struct PeerInfo { /// The negotiated eth version. pub eth_version: EthVersion, /// The Status message the peer sent for the `eth` handshake - pub status: Arc, + pub status: Arc, /// The timestamp when the session to that peer has been established. pub session_established: Instant, /// The peer's connection kind @@ -285,4 +286,6 @@ pub struct NetworkStatus { pub protocol_version: u64, /// Information about the Ethereum Wire Protocol. pub eth_protocol_info: EthProtocolInfo, + /// The list of supported capabilities and their versions. + pub capabilities: Vec, } diff --git a/crates/net/network-api/src/noop.rs b/crates/net/network-api/src/noop.rs index bd5d08f8f13..2aaa0093568 100644 --- a/crates/net/network-api/src/noop.rs +++ b/crates/net/network-api/src/noop.rs @@ -6,6 +6,13 @@ use core::{fmt, marker::PhantomData}; use std::net::{IpAddr, SocketAddr}; +use crate::{ + events::{NetworkPeersEvents, PeerEventStream}, + test_utils::{PeersHandle, PeersHandleProvider}, + BlockDownloaderProvider, DiscoveryEvent, NetworkError, NetworkEvent, + NetworkEventListenerProvider, NetworkInfo, NetworkStatus, PeerId, PeerInfo, PeerRequest, Peers, + PeersInfo, +}; use alloy_rpc_types_admin::EthProtocolInfo; use enr::{secp256k1::SecretKey, Enr}; use reth_eth_wire_types::{ @@ -18,20 +25,13 @@ use reth_tokio_util::{EventSender, EventStream}; use tokio::sync::{mpsc, oneshot}; use tokio_stream::wrappers::UnboundedReceiverStream; -use crate::{ - events::{NetworkPeersEvents, PeerEventStream}, - test_utils::{PeersHandle, PeersHandleProvider}, - BlockDownloaderProvider, DiscoveryEvent, NetworkError, NetworkEvent, - NetworkEventListenerProvider, NetworkInfo, NetworkStatus, PeerId, PeerInfo, PeerRequest, Peers, - PeersInfo, -}; - /// A type that implements all network trait that does nothing. /// /// Intended for testing purposes where network is not used. #[derive(Debug, Clone)] #[non_exhaustive] pub struct NoopNetwork { + chain_id: u64, peers_handle: PeersHandle, _marker: PhantomData, } @@ -41,15 +41,23 @@ impl NoopNetwork { pub fn new() -> Self { let (tx, _) = mpsc::unbounded_channel(); - Self { peers_handle: PeersHandle::new(tx), _marker: PhantomData } + Self { + chain_id: 1, // mainnet + peers_handle: PeersHandle::new(tx), + _marker: PhantomData, + } + } + + /// Creates a new [`NoopNetwork`] from an existing one but with a new chain id. + pub const fn with_chain_id(mut self, chain_id: u64) -> Self { + self.chain_id = chain_id; + self } } impl Default for NoopNetwork { fn default() -> Self { - let (tx, _) = mpsc::unbounded_channel(); - - Self { peers_handle: PeersHandle::new(tx), _marker: PhantomData } + Self::new() } } @@ -73,12 +81,12 @@ where config: Default::default(), head: Default::default(), }, + capabilities: vec![], }) } fn chain_id(&self) -> u64 { - // mainnet - 1 + self.chain_id } fn is_syncing(&self) -> bool { @@ -163,7 +171,7 @@ where impl BlockDownloaderProvider for NoopNetwork where - Net: NetworkPrimitives + Default, + Net: NetworkPrimitives, { type Client = NoopFullBlockClient; @@ -179,6 +187,8 @@ where fn update_status(&self, _head: reth_ethereum_forks::Head) {} fn update_sync_state(&self, _state: reth_network_p2p::sync::SyncState) {} + + fn update_block_range(&self, _: reth_eth_wire_types::BlockRangeUpdate) {} } impl NetworkEventListenerProvider for NoopNetwork diff --git a/crates/net/network-types/src/lib.rs b/crates/net/network-types/src/lib.rs index 1e8ad581d28..d4215c9c42d 100644 --- a/crates/net/network-types/src/lib.rs +++ b/crates/net/network-types/src/lib.rs @@ -10,7 +10,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] /// Types related to peering. pub mod peers; @@ -25,7 +25,10 @@ pub use backoff::BackoffKind; pub use peers::{ addr::PeerAddr, kind::PeerKind, - reputation::{is_banned_reputation, ReputationChangeOutcome, DEFAULT_REPUTATION}, + reputation::{ + is_banned_reputation, is_connection_failed_reputation, ReputationChangeOutcome, + DEFAULT_REPUTATION, + }, state::PeerConnectionState, ConnectionsConfig, Peer, PeersConfig, }; diff --git a/crates/net/network-types/src/peers/config.rs b/crates/net/network-types/src/peers/config.rs index 221705c3846..1fe685b0e81 100644 --- a/crates/net/network-types/src/peers/config.rs +++ b/crates/net/network-types/src/peers/config.rs @@ -131,6 +131,9 @@ pub struct PeersConfig { /// Connect to or accept from trusted nodes only? #[cfg_attr(feature = "serde", serde(alias = "connect_trusted_nodes_only"))] pub trusted_nodes_only: bool, + /// Interval to update trusted nodes DNS resolution + #[cfg_attr(feature = "serde", serde(with = "humantime_serde"))] + pub trusted_nodes_resolution_interval: Duration, /// Maximum number of backoff attempts before we give up on a peer and dropping. /// /// The max time spent of a peer before it's removed from the set is determined by the @@ -177,6 +180,7 @@ impl Default for PeersConfig { backoff_durations: Default::default(), trusted_nodes: Default::default(), trusted_nodes_only: false, + trusted_nodes_resolution_interval: Duration::from_secs(60 * 60), basic_nodes: Default::default(), max_backoff_count: 5, incoming_ip_throttle_duration: INBOUND_IP_THROTTLE_DURATION, diff --git a/crates/net/network-types/src/peers/mod.rs b/crates/net/network-types/src/peers/mod.rs index 2f0bd6141b8..d41882d494c 100644 --- a/crates/net/network-types/src/peers/mod.rs +++ b/crates/net/network-types/src/peers/mod.rs @@ -25,7 +25,7 @@ pub struct Peer { /// The state of the connection, if any. pub state: PeerConnectionState, /// The [`ForkId`] that the peer announced via discovery. - pub fork_id: Option, + pub fork_id: Option>, /// Whether the entry should be removed after an existing session was terminated. pub remove_after_disconnect: bool, /// The kind of peer @@ -83,12 +83,16 @@ impl Peer { } /// Applies a reputation change to the peer and returns what action should be taken. - pub fn apply_reputation(&mut self, reputation: i32) -> ReputationChangeOutcome { + pub fn apply_reputation( + &mut self, + reputation: i32, + kind: ReputationChangeKind, + ) -> ReputationChangeOutcome { let previous = self.reputation; // we add reputation since negative reputation change decrease total reputation self.reputation = previous.saturating_add(reputation); - trace!(target: "net::peers", reputation=%self.reputation, banned=%self.is_banned(), "applied reputation change"); + trace!(target: "net::peers", reputation=%self.reputation, banned=%self.is_banned(), ?kind, "applied reputation change"); if self.state.is_connected() && self.is_banned() { self.state.disconnect(); diff --git a/crates/net/network-types/src/peers/reputation.rs b/crates/net/network-types/src/peers/reputation.rs index 91035d8d45a..cf4b555b23c 100644 --- a/crates/net/network-types/src/peers/reputation.rs +++ b/crates/net/network-types/src/peers/reputation.rs @@ -13,7 +13,7 @@ pub const BANNED_REPUTATION: i32 = 50 * REPUTATION_UNIT; const REMOTE_DISCONNECT_REPUTATION_CHANGE: i32 = 4 * REPUTATION_UNIT; /// The reputation change to apply to a peer that we failed to connect to. -const FAILED_TO_CONNECT_REPUTATION_CHANGE: i32 = 25 * REPUTATION_UNIT; +pub const FAILED_TO_CONNECT_REPUTATION_CHANGE: i32 = 25 * REPUTATION_UNIT; /// The reputation change to apply to a peer that failed to respond in time. const TIMEOUT_REPUTATION_CHANGE: i32 = 4 * REPUTATION_UNIT; @@ -48,6 +48,13 @@ pub const fn is_banned_reputation(reputation: i32) -> bool { reputation < BANNED_REPUTATION } +/// Returns `true` if the given reputation is below the [`FAILED_TO_CONNECT_REPUTATION_CHANGE`] +/// threshold +#[inline] +pub const fn is_connection_failed_reputation(reputation: i32) -> bool { + reputation < FAILED_TO_CONNECT_REPUTATION_CHANGE +} + /// The type that tracks the reputation score. pub type Reputation = i32; diff --git a/crates/net/network/Cargo.toml b/crates/net/network/Cargo.toml index 2aa2e627739..54902ef4788 100644 --- a/crates/net/network/Cargo.toml +++ b/crates/net/network/Cargo.toml @@ -89,11 +89,9 @@ reth-tracing.workspace = true reth-transaction-pool = { workspace = true, features = ["test-utils"] } # alloy deps for testing against nodes -alloy-consensus.workspace = true alloy-genesis.workspace = true # misc -tempfile.workspace = true url.workspace = true secp256k1 = { workspace = true, features = ["rand"] } @@ -124,6 +122,7 @@ serde = [ "reth-ethereum-primitives/serde", "reth-network-api/serde", "rand_08/serde", + "reth-storage-api/serde", ] test-utils = [ "reth-transaction-pool/test-utils", diff --git a/crates/net/network/docs/mermaid/network-manager.mmd b/crates/net/network/docs/mermaid/network-manager.mmd index e34dbb17777..aa2514a54d5 100644 --- a/crates/net/network/docs/mermaid/network-manager.mmd +++ b/crates/net/network/docs/mermaid/network-manager.mmd @@ -9,7 +9,7 @@ graph TB subgraph Swarm direction TB B1[(Session Manager)] - B2[(Connection Lister)] + B2[(Connection Listener)] B3[(Network State)] end end diff --git a/crates/net/network/src/budget.rs b/crates/net/network/src/budget.rs index 824148387b4..f1d9ca87469 100644 --- a/crates/net/network/src/budget.rs +++ b/crates/net/network/src/budget.rs @@ -35,13 +35,6 @@ pub const DEFAULT_BUDGET_TRY_DRAIN_NETWORK_TRANSACTION_EVENTS: u32 = DEFAULT_BUD // Default is 40 pending pool imports. pub const DEFAULT_BUDGET_TRY_DRAIN_PENDING_POOL_IMPORTS: u32 = 4 * DEFAULT_BUDGET_TRY_DRAIN_STREAM; -/// Default budget to try and stream hashes of successfully imported transactions from the pool. -/// -/// Default is naturally same as the number of transactions to attempt importing, -/// [`DEFAULT_BUDGET_TRY_DRAIN_PENDING_POOL_IMPORTS`], so 40 pool imports. -pub const DEFAULT_BUDGET_TRY_DRAIN_POOL_IMPORTS: u32 = - DEFAULT_BUDGET_TRY_DRAIN_PENDING_POOL_IMPORTS; - /// Polls the given stream. Breaks with `true` if there maybe is more work. #[macro_export] macro_rules! poll_nested_stream_with_budget { diff --git a/crates/net/network/src/builder.rs b/crates/net/network/src/builder.rs index 65775342a26..3f36b1bdc80 100644 --- a/crates/net/network/src/builder.rs +++ b/crates/net/network/src/builder.rs @@ -1,8 +1,14 @@ //! Builder support for configuring the entire setup. +use std::fmt::Debug; + use crate::{ eth_requests::EthRequestHandler, - transactions::{TransactionPropagationPolicy, TransactionsManager, TransactionsManagerConfig}, + transactions::{ + config::{StrictEthAnnouncementFilter, TransactionPropagationKind}, + policy::NetworkPolicies, + TransactionPropagationPolicy, TransactionsManager, TransactionsManagerConfig, + }, NetworkHandle, NetworkManager, }; use reth_eth_wire::{EthNetworkPrimitives, NetworkPrimitives}; @@ -71,27 +77,49 @@ impl NetworkBuilder { self, pool: Pool, transactions_manager_config: TransactionsManagerConfig, - ) -> NetworkBuilder, Eth, N> { - self.transactions_with_policy(pool, transactions_manager_config, Default::default()) + ) -> NetworkBuilder< + TransactionsManager< + Pool, + N, + NetworkPolicies, + >, + Eth, + N, + > { + self.transactions_with_policy( + pool, + transactions_manager_config, + TransactionPropagationKind::default(), + ) } /// Creates a new [`TransactionsManager`] and wires it to the network. - pub fn transactions_with_policy( + pub fn transactions_with_policy< + Pool: TransactionPool, + P: TransactionPropagationPolicy + Debug, + >( self, pool: Pool, transactions_manager_config: TransactionsManagerConfig, propagation_policy: P, - ) -> NetworkBuilder, Eth, N> { + ) -> NetworkBuilder< + TransactionsManager>, + Eth, + N, + > { let Self { mut network, request_handler, .. } = self; let (tx, rx) = mpsc::unbounded_channel(); network.set_transactions(tx); let handle = network.handle().clone(); + let announcement_policy = StrictEthAnnouncementFilter::default(); + let policies = NetworkPolicies::new(propagation_policy, announcement_policy); + let transactions = TransactionsManager::with_policy( handle, pool, rx, transactions_manager_config, - propagation_policy, + policies, ); NetworkBuilder { network, request_handler, transactions } } diff --git a/crates/net/network/src/cache.rs b/crates/net/network/src/cache.rs index a06d7dcd69f..2c1ea15792c 100644 --- a/crates/net/network/src/cache.rs +++ b/crates/net/network/src/cache.rs @@ -1,9 +1,10 @@ //! Network cache support +use alloy_primitives::map::DefaultHashBuilder; use core::hash::BuildHasher; use derive_more::{Deref, DerefMut}; use itertools::Itertools; -use schnellru::{ByLength, Limiter, RandomState, Unlimited}; +use schnellru::{ByLength, Limiter, Unlimited}; use std::{fmt, hash::Hash}; /// A minimal LRU cache based on a [`LruMap`](schnellru::LruMap) with limited capacity. @@ -133,9 +134,10 @@ where } } -/// Wrapper of [`schnellru::LruMap`] that implements [`fmt::Debug`]. +/// Wrapper of [`schnellru::LruMap`] that implements [`fmt::Debug`] and with the common hash +/// builder. #[derive(Deref, DerefMut, Default)] -pub struct LruMap(schnellru::LruMap) +pub struct LruMap(schnellru::LruMap) where K: Hash + PartialEq, L: Limiter, @@ -171,7 +173,7 @@ where { /// Returns a new cache with default limiter and hash builder. pub fn new(max_length: u32) -> Self { - Self(schnellru::LruMap::new(ByLength::new(max_length))) + Self(schnellru::LruMap::with_hasher(ByLength::new(max_length), Default::default())) } } @@ -181,7 +183,7 @@ where { /// Returns a new cache with [`Unlimited`] limiter and default hash builder. pub fn new_unlimited() -> Self { - Self(schnellru::LruMap::new(Unlimited)) + Self(schnellru::LruMap::with_hasher(Unlimited, Default::default())) } } diff --git a/crates/net/network/src/config.rs b/crates/net/network/src/config.rs index 54f19868970..c403bdcb557 100644 --- a/crates/net/network/src/config.rs +++ b/crates/net/network/src/config.rs @@ -6,13 +6,15 @@ use crate::{ transactions::TransactionsManagerConfig, NetworkHandle, NetworkManager, }; +use alloy_primitives::B256; use reth_chainspec::{ChainSpecProvider, EthChainSpec, Hardforks}; use reth_discv4::{Discv4Config, Discv4ConfigBuilder, NatResolver, DEFAULT_DISCOVERY_ADDRESS}; use reth_discv5::NetworkStackId; use reth_dns_discovery::DnsDiscoveryConfig; use reth_eth_wire::{ handshake::{EthHandshake, EthRlpxHandshake}, - EthNetworkPrimitives, HelloMessage, HelloMessageWithProtocols, NetworkPrimitives, Status, + EthNetworkPrimitives, HelloMessage, HelloMessageWithProtocols, NetworkPrimitives, + UnifiedStatus, }; use reth_ethereum_forks::{ForkFilter, Head}; use reth_network_peers::{mainnet_nodes, pk2id, sepolia_nodes, PeerId, TrustedPeer}; @@ -23,7 +25,10 @@ use secp256k1::SECP256K1; use std::{collections::HashSet, net::SocketAddr, sync::Arc}; // re-export for convenience -use crate::protocol::{IntoRlpxSubProtocol, RlpxSubProtocols}; +use crate::{ + protocol::{IntoRlpxSubProtocol, RlpxSubProtocols}, + transactions::TransactionPropagationMode, +}; pub use secp256k1::SecretKey; /// Convenience function to create a new random [`SecretKey`] @@ -37,7 +42,7 @@ pub struct NetworkConfig { /// The client type that can interact with the chain. /// /// This type is used to fetch the block number after we established a session and received the - /// [Status] block hash. + /// [`UnifiedStatus`] block hash. pub client: C, /// The node's secret key, from which the node's identity is derived. pub secret_key: SecretKey, @@ -67,13 +72,13 @@ pub struct NetworkConfig { /// first hardfork, `Frontier` for mainnet. pub fork_filter: ForkFilter, /// The block importer type. - pub block_import: Box>, + pub block_import: Box>, /// The default mode of the network. pub network_mode: NetworkMode, /// The executor to use for spawning tasks. pub executor: Box, /// The `Status` message to send to peers at the beginning. - pub status: Status, + pub status: UnifiedStatus, /// Sets the hello message for the p2p handshake in `RLPx` pub hello_message: HelloMessageWithProtocols, /// Additional protocols to announce and handle in `RLPx` @@ -89,6 +94,9 @@ pub struct NetworkConfig { /// This can be overridden to support custom handshake logic via the /// [`NetworkConfigBuilder`]. pub handshake: Arc, + /// List of block hashes to check for required blocks. + /// If non-empty, peers that don't have these blocks will be filtered out. + pub required_block_hashes: Vec, } // === impl NetworkConfig === @@ -208,7 +216,7 @@ pub struct NetworkConfigBuilder { /// Whether tx gossip is disabled tx_gossip_disabled: bool, /// The block importer type - block_import: Option>>, + block_import: Option>>, /// How to instantiate transactions manager. transactions_manager_config: TransactionsManagerConfig, /// The NAT resolver for external IP @@ -216,6 +224,10 @@ pub struct NetworkConfigBuilder { /// The Ethereum P2P handshake, see also: /// . handshake: Arc, + /// List of block hashes to check for required blocks. + required_block_hashes: Vec, + /// Optional network id + network_id: Option, } impl NetworkConfigBuilder { @@ -256,6 +268,8 @@ impl NetworkConfigBuilder { transactions_manager_config: Default::default(), nat: None, handshake: Arc::new(EthHandshake::default()), + required_block_hashes: Vec::new(), + network_id: None, } } @@ -296,7 +310,7 @@ impl NetworkConfigBuilder { /// Sets the highest synced block. /// - /// This is used to construct the appropriate [`ForkFilter`] and [`Status`] message. + /// This is used to construct the appropriate [`ForkFilter`] and [`UnifiedStatus`] message. /// /// If not set, this defaults to the genesis specified by the current chain specification. pub const fn set_head(mut self, head: Head) -> Self { @@ -345,6 +359,12 @@ impl NetworkConfigBuilder { self } + /// Configures the propagation mode for the transaction manager. + pub const fn transaction_propagation_mode(mut self, mode: TransactionPropagationMode) -> Self { + self.transactions_manager_config.propagation_mode = mode; + self + } + /// Sets the discovery and listener address /// /// This is a convenience function for both [`NetworkConfigBuilder::listener_addr`] and @@ -534,8 +554,14 @@ impl NetworkConfigBuilder { self } + /// Sets the required block hashes for peer filtering. + pub fn required_block_hashes(mut self, hashes: Vec) -> Self { + self.required_block_hashes = hashes; + self + } + /// Sets the block import type. - pub fn block_import(mut self, block_import: Box>) -> Self { + pub fn block_import(mut self, block_import: Box>) -> Self { self.block_import = Some(block_import); self } @@ -564,6 +590,12 @@ impl NetworkConfigBuilder { self } + /// Set the optional network id. + pub const fn network_id(mut self, network_id: Option) -> Self { + self.network_id = network_id; + self + } + /// Consumes the type and creates the actual [`NetworkConfig`] /// for the given client type that can interact with the chain. /// @@ -596,6 +628,8 @@ impl NetworkConfigBuilder { transactions_manager_config, nat, handshake, + required_block_hashes, + network_id, } = self; let head = head.unwrap_or_else(|| Head { @@ -622,7 +656,11 @@ impl NetworkConfigBuilder { hello_message.port = listener_addr.port(); // set the status - let status = Status::spec_builder(&chain_spec, &head).build(); + let mut status = UnifiedStatus::spec_builder(&chain_spec, &head); + + if let Some(id) = network_id { + status.chain = id.into(); + } // set a fork filter based on the chain spec and head let fork_filter = chain_spec.fork_filter(head); @@ -632,13 +670,11 @@ impl NetworkConfigBuilder { // If default DNS config is used then we add the known dns network to bootstrap from if let Some(dns_networks) = - dns_discovery_config.as_mut().and_then(|c| c.bootstrap_dns_networks.as_mut()) + dns_discovery_config.as_mut().and_then(|c| c.bootstrap_dns_networks.as_mut()) && + dns_networks.is_empty() && + let Some(link) = chain_spec.chain().public_dns_network_protocol() { - if dns_networks.is_empty() { - if let Some(link) = chain_spec.chain().public_dns_network_protocol() { - dns_networks.insert(link.parse().expect("is valid DNS link entry")); - } - } + dns_networks.insert(link.parse().expect("is valid DNS link entry")); } NetworkConfig { @@ -664,6 +700,7 @@ impl NetworkConfigBuilder { transactions_manager_config, nat, handshake, + required_block_hashes, } } } diff --git a/crates/net/network/src/discovery.rs b/crates/net/network/src/discovery.rs index 5809380aa8a..9cc3a6249a8 100644 --- a/crates/net/network/src/discovery.rs +++ b/crates/net/network/src/discovery.rs @@ -200,7 +200,6 @@ impl Discovery { } /// Add a node to the discv4 table. - #[expect(clippy::result_large_err)] pub(crate) fn add_discv5_node(&self, enr: Enr) -> Result<(), NetworkError> { if let Some(discv5) = &self.discv5 { discv5.add_node(enr).map_err(NetworkError::Discv5Error)?; @@ -267,12 +266,11 @@ impl Discovery { while let Some(Poll::Ready(Some(update))) = self.discv5_updates.as_mut().map(|updates| updates.poll_next_unpin(cx)) { - if let Some(discv5) = self.discv5.as_mut() { - if let Some(DiscoveredPeer { node_record, fork_id }) = + if let Some(discv5) = self.discv5.as_mut() && + let Some(DiscoveredPeer { node_record, fork_id }) = discv5.on_discv5_update(update) - { - self.on_node_record_update(node_record, fork_id); - } + { + self.on_node_record_update(node_record, fork_id); } } diff --git a/crates/net/network/src/error.rs b/crates/net/network/src/error.rs index f88d8bd8158..96ba2ff85ec 100644 --- a/crates/net/network/src/error.rs +++ b/crates/net/network/src/error.rs @@ -53,8 +53,8 @@ pub enum NetworkError { error: io::Error, }, /// IO error when creating the discovery service - #[error("failed to launch discovery service: {0}")] - Discovery(io::Error), + #[error("failed to launch discovery service on {0}: {1}")] + Discovery(SocketAddr, io::Error), /// An error occurred with discovery v5 node. #[error("discv5 error, {0}")] Discv5Error(#[from] reth_discv5::Error), @@ -71,8 +71,8 @@ impl NetworkError { match err.kind() { ErrorKind::AddrInUse => Self::AddressAlreadyInUse { kind, error: err }, _ => { - if let ServiceKind::Discovery(_) = kind { - return Self::Discovery(err) + if let ServiceKind::Discovery(address) = kind { + return Self::Discovery(address, err) } Self::Io(err) } diff --git a/crates/net/network/src/eth_requests.rs b/crates/net/network/src/eth_requests.rs index 408937e4533..492bf8bd55e 100644 --- a/crates/net/network/src/eth_requests.rs +++ b/crates/net/network/src/eth_requests.rs @@ -10,7 +10,7 @@ use alloy_rlp::Encodable; use futures::StreamExt; use reth_eth_wire::{ BlockBodies, BlockHeaders, EthNetworkPrimitives, GetBlockBodies, GetBlockHeaders, GetNodeData, - GetReceipts, HeadersDirection, NetworkPrimitives, NodeData, Receipts, + GetReceipts, HeadersDirection, NetworkPrimitives, NodeData, Receipts, Receipts69, }; use reth_network_api::test_utils::PeersHandle; use reth_network_p2p::error::RequestResult; @@ -104,9 +104,20 @@ where for _ in 0..limit { if let Some(header) = self.client.header_by_hash_or_number(block).unwrap_or_default() { + let number = header.number(); + let parent_hash = header.parent_hash(); + + total_bytes += header.length(); + headers.push(header); + + if headers.len() >= MAX_HEADERS_SERVE || total_bytes > SOFT_RESPONSE_LIMIT { + break + } + match direction { HeadersDirection::Rising => { - if let Some(next) = (header.number() + 1).checked_add(skip) { + if let Some(next) = number.checked_add(1).and_then(|n| n.checked_add(skip)) + { block = next.into() } else { break @@ -117,24 +128,17 @@ where // prevent under flows for block.number == 0 and `block.number - skip < // 0` if let Some(next) = - header.number().checked_sub(1).and_then(|num| num.checked_sub(skip)) + number.checked_sub(1).and_then(|num| num.checked_sub(skip)) { block = next.into() } else { break } } else { - block = header.parent_hash().into() + block = parent_hash.into() } } } - - total_bytes += header.length(); - headers.push(header); - - if headers.len() >= MAX_HEADERS_SERVE || total_bytes > SOFT_RESPONSE_LIMIT { - break - } } else { break } @@ -190,19 +194,45 @@ where ) { self.metrics.eth_receipts_requests_received_total.increment(1); - let mut receipts = Vec::new(); + let receipts = self.get_receipts_response(request, |receipts_by_block| { + receipts_by_block.into_iter().map(ReceiptWithBloom::from).collect::>() + }); + let _ = response.send(Ok(Receipts(receipts))); + } + + fn on_receipts69_request( + &self, + _peer_id: PeerId, + request: GetReceipts, + response: oneshot::Sender>>, + ) { + self.metrics.eth_receipts_requests_received_total.increment(1); + + let receipts = self.get_receipts_response(request, |receipts_by_block| { + // skip bloom filter for eth69 + receipts_by_block + }); + + let _ = response.send(Ok(Receipts69(receipts))); + } + + #[inline] + fn get_receipts_response(&self, request: GetReceipts, transform_fn: F) -> Vec> + where + F: Fn(Vec) -> Vec, + T: Encodable, + { + let mut receipts = Vec::new(); let mut total_bytes = 0; for hash in request.0 { if let Some(receipts_by_block) = self.client.receipts_by_block(BlockHashOrNumber::Hash(hash)).unwrap_or_default() { - let receipt = - receipts_by_block.into_iter().map(ReceiptWithBloom::from).collect::>(); - - total_bytes += receipt.length(); - receipts.push(receipt); + let transformed_receipts = transform_fn(receipts_by_block); + total_bytes += transformed_receipts.length(); + receipts.push(transformed_receipts); if receipts.len() >= MAX_RECEIPTS_SERVE || total_bytes > SOFT_RESPONSE_LIMIT { break @@ -212,7 +242,7 @@ where } } - let _ = response.send(Ok(Receipts(receipts))); + receipts } } @@ -252,6 +282,9 @@ where IncomingEthRequest::GetReceipts { peer_id, request, response } => { this.on_receipts_request(peer_id, request, response) } + IncomingEthRequest::GetReceipts69 { peer_id, request, response } => { + this.on_receipts69_request(peer_id, request, response) + } } }, ); @@ -315,4 +348,15 @@ pub enum IncomingEthRequest { /// The channel sender for the response containing receipts. response: oneshot::Sender>>, }, + /// Request Receipts from the peer without bloom filter. + /// + /// The response should be sent through the channel. + GetReceipts69 { + /// The ID of the peer to request receipts from. + peer_id: PeerId, + /// The specific receipts requested. + request: GetReceipts, + /// The channel sender for the response containing Receipts69. + response: oneshot::Sender>>, + }, } diff --git a/crates/net/network/src/fetch/client.rs b/crates/net/network/src/fetch/client.rs index c043692d26d..fdfd051a8a6 100644 --- a/crates/net/network/src/fetch/client.rs +++ b/crates/net/network/src/fetch/client.rs @@ -15,9 +15,12 @@ use reth_network_p2p::{ }; use reth_network_peers::PeerId; use reth_network_types::ReputationChangeKind; -use std::sync::{ - atomic::{AtomicUsize, Ordering}, - Arc, +use std::{ + ops::RangeInclusive, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, }; use tokio::sync::{mpsc::UnboundedSender, oneshot}; @@ -80,15 +83,16 @@ impl BodiesClient for FetchClient { type Output = BodiesFut; /// Sends a `GetBlockBodies` request to an available peer. - fn get_block_bodies_with_priority( + fn get_block_bodies_with_priority_and_range_hint( &self, request: Vec, priority: Priority, + range_hint: Option>, ) -> Self::Output { let (response, rx) = oneshot::channel(); if self .request_tx - .send(DownloadRequest::GetBlockBodies { request, response, priority }) + .send(DownloadRequest::GetBlockBodies { request, response, priority, range_hint }) .is_ok() { Box::pin(FlattenedResponse::from(rx)) diff --git a/crates/net/network/src/fetch/mod.rs b/crates/net/network/src/fetch/mod.rs index c64d83757b3..55bde002b3e 100644 --- a/crates/net/network/src/fetch/mod.rs +++ b/crates/net/network/src/fetch/mod.rs @@ -4,7 +4,7 @@ mod client; pub use client::FetchClient; -use crate::message::BlockRequest; +use crate::{message::BlockRequest, session::BlockRangeInfo}; use alloy_primitives::B256; use futures::StreamExt; use reth_eth_wire::{EthNetworkPrimitives, GetBlockBodies, GetBlockHeaders, NetworkPrimitives}; @@ -18,6 +18,7 @@ use reth_network_peers::PeerId; use reth_network_types::ReputationChangeKind; use std::{ collections::{HashMap, VecDeque}, + ops::RangeInclusive, sync::{ atomic::{AtomicU64, AtomicUsize, Ordering}, Arc, @@ -28,7 +29,7 @@ use tokio::sync::{mpsc, mpsc::UnboundedSender, oneshot}; use tokio_stream::wrappers::UnboundedReceiverStream; type InflightHeadersRequest = Request>>; -type InflightBodiesRequest = Request, PeerRequestResult>>; +type InflightBodiesRequest = Request<(), PeerRequestResult>>; /// Manages data fetching operations. /// @@ -80,6 +81,7 @@ impl StateFetcher { best_hash: B256, best_number: u64, timeout: Arc, + range_info: Option, ) { self.peers.insert( peer_id, @@ -89,6 +91,7 @@ impl StateFetcher { best_number, timeout, last_response_likely_bad: false, + range_info, }, ); } @@ -113,12 +116,12 @@ impl StateFetcher { /// /// Returns `true` if this a newer block pub(crate) fn update_peer_block(&mut self, peer_id: &PeerId, hash: B256, number: u64) -> bool { - if let Some(peer) = self.peers.get_mut(peer_id) { - if number > peer.best_number { - peer.best_hash = hash; - peer.best_number = number; - return true - } + if let Some(peer) = self.peers.get_mut(peer_id) && + number > peer.best_number + { + peer.best_hash = hash; + peer.best_number = number; + return true } false } @@ -234,7 +237,7 @@ impl StateFetcher { }) } DownloadRequest::GetBlockBodies { request, response, .. } => { - let inflight = Request { request: request.clone(), response }; + let inflight = Request { request: (), response }; self.inflight_bodies_requests.insert(peer_id, inflight); BlockRequest::GetBlockBodies(GetBlockBodies(request)) } @@ -347,6 +350,9 @@ struct Peer { /// downloaded), but we still want to avoid requesting from the same peer again if it has the /// lowest timeout. last_response_likely_bad: bool, + /// Tracks the range info for the peer. + #[allow(dead_code)] + range_info: Option, } impl Peer { @@ -414,6 +420,8 @@ pub(crate) enum DownloadRequest { request: Vec, response: oneshot::Sender>>, priority: Priority, + #[allow(dead_code)] + range_hint: Option>, }, } @@ -486,6 +494,7 @@ mod tests { request: vec![], response: tx, priority: Priority::default(), + range_hint: None, }); assert!(fetcher.poll(cx).is_pending()); @@ -502,8 +511,8 @@ mod tests { // Add a few random peers let peer1 = B512::random(); let peer2 = B512::random(); - fetcher.new_active_peer(peer1, B256::random(), 1, Arc::new(AtomicU64::new(1))); - fetcher.new_active_peer(peer2, B256::random(), 2, Arc::new(AtomicU64::new(1))); + fetcher.new_active_peer(peer1, B256::random(), 1, Arc::new(AtomicU64::new(1)), None); + fetcher.new_active_peer(peer2, B256::random(), 2, Arc::new(AtomicU64::new(1)), None); let first_peer = fetcher.next_best_peer().unwrap(); assert!(first_peer == peer1 || first_peer == peer2); @@ -530,9 +539,9 @@ mod tests { let peer2_timeout = Arc::new(AtomicU64::new(300)); - fetcher.new_active_peer(peer1, B256::random(), 1, Arc::new(AtomicU64::new(30))); - fetcher.new_active_peer(peer2, B256::random(), 2, Arc::clone(&peer2_timeout)); - fetcher.new_active_peer(peer3, B256::random(), 3, Arc::new(AtomicU64::new(50))); + fetcher.new_active_peer(peer1, B256::random(), 1, Arc::new(AtomicU64::new(30)), None); + fetcher.new_active_peer(peer2, B256::random(), 2, Arc::clone(&peer2_timeout), None); + fetcher.new_active_peer(peer3, B256::random(), 3, Arc::new(AtomicU64::new(50)), None); // Must always get peer1 (lowest timeout) assert_eq!(fetcher.next_best_peer(), Some(peer1)); @@ -601,6 +610,7 @@ mod tests { Default::default(), Default::default(), Default::default(), + None, ); let (req, header) = request_pair(); diff --git a/crates/net/network/src/flattened_response.rs b/crates/net/network/src/flattened_response.rs index 61dae9c7c72..2827a015eca 100644 --- a/crates/net/network/src/flattened_response.rs +++ b/crates/net/network/src/flattened_response.rs @@ -6,7 +6,7 @@ use std::{ }; use tokio::sync::oneshot::{error::RecvError, Receiver}; -/// Flatten a [Receiver] message in order to get rid of the [RecvError] result +/// Flatten a [Receiver] message in order to get rid of the [`RecvError`] result #[derive(Debug)] #[pin_project] pub struct FlattenedResponse { diff --git a/crates/net/network/src/import.rs b/crates/net/network/src/import.rs index 491fabba9a6..52187e5b2f1 100644 --- a/crates/net/network/src/import.rs +++ b/crates/net/network/src/import.rs @@ -1,6 +1,7 @@ //! This module provides an abstraction over block import in the form of the `BlockImport` trait. use crate::message::NewBlockMessage; +use reth_eth_wire::NewBlock; use reth_eth_wire_types::broadcast::NewBlockHashes; use reth_network_peers::PeerId; use std::{ @@ -9,7 +10,7 @@ use std::{ }; /// Abstraction over block import. -pub trait BlockImport: std::fmt::Debug + Send + Sync { +pub trait BlockImport: std::fmt::Debug + Send + Sync { /// Invoked for a received block announcement from the peer. /// /// For a `NewBlock` message: @@ -27,7 +28,7 @@ pub trait BlockImport: std::fmt::Debug + Se /// Represents different types of block announcement events from the network. #[derive(Debug, Clone)] -pub enum NewBlockEvent { +pub enum NewBlockEvent { /// A new full block announcement Block(NewBlockMessage), /// Only the hashes of new blocks diff --git a/crates/net/network/src/lib.rs b/crates/net/network/src/lib.rs index 13352893a10..a84168d3846 100644 --- a/crates/net/network/src/lib.rs +++ b/crates/net/network/src/lib.rs @@ -115,7 +115,7 @@ )] #![allow(unreachable_pub)] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #[cfg(any(test, feature = "test-utils"))] /// Common helpers for network testing. @@ -140,6 +140,7 @@ mod listener; mod manager; mod metrics; mod network; +mod required_block_filter; mod session; mod state; mod swarm; @@ -168,13 +169,15 @@ pub use manager::NetworkManager; pub use metrics::TxTypesCounter; pub use network::{NetworkHandle, NetworkProtocols}; pub use swarm::NetworkConnectionState; -pub use transactions::{FilterAnnouncement, MessageFilter}; /// re-export p2p interfaces pub use reth_network_p2p as p2p; -/// re-export types crate -pub use reth_eth_wire_types as types; +/// re-export types crates +pub mod types { + pub use reth_eth_wire_types::*; + pub use reth_network_types::*; +} use aquamarine as _; diff --git a/crates/net/network/src/manager.rs b/crates/net/network/src/manager.rs index b76fac11a22..c0a2934df75 100644 --- a/crates/net/network/src/manager.rs +++ b/crates/net/network/src/manager.rs @@ -29,6 +29,7 @@ use crate::{ peers::PeersManager, poll_nested_stream_with_budget, protocol::IntoRlpxSubProtocol, + required_block_filter::RequiredBlockFilter, session::SessionManager, state::NetworkState, swarm::{Swarm, SwarmEvent}, @@ -37,6 +38,7 @@ use crate::{ }; use futures::{Future, StreamExt}; use parking_lot::Mutex; +use reth_chainspec::EnrForkIdEntry; use reth_eth_wire::{DisconnectReason, EthNetworkPrimitives, NetworkPrimitives}; use reth_fs_util::{self as fs, FsPathError}; use reth_metrics::common::mpsc::UnboundedMeteredSender; @@ -88,7 +90,7 @@ use tracing::{debug, error, trace, warn}; /// subgraph Swarm /// direction TB /// B1[(Session Manager)] -/// B2[(Connection Lister)] +/// B2[(Connection Listener)] /// B3[(Network State)] /// end /// end @@ -108,7 +110,7 @@ pub struct NetworkManager { /// Receiver half of the command channel set up between this type and the [`NetworkHandle`] from_handle_rx: UnboundedReceiverStream>, /// Handles block imports according to the `eth` protocol. - block_import: Box>, + block_import: Box>, /// Sender for high level network events. event_sender: EventSender>>, /// Sender half to send events to the @@ -249,6 +251,7 @@ impl NetworkManager { transactions_manager_config: _, nat, handshake, + required_block_hashes, } = config; let peers_manager = PeersManager::new(peers_config); @@ -268,7 +271,9 @@ impl NetworkManager { if let Some(disc_config) = discovery_v4_config.as_mut() { // merge configured boot nodes disc_config.bootstrap_nodes.extend(resolved_boot_nodes.clone()); - disc_config.add_eip868_pair("eth", status.forkid); + // add the forkid entry for EIP-868, but wrap it in an `EnrForkIdEntry` for proper + // encoding + disc_config.add_eip868_pair("eth", EnrForkIdEntry::from(status.forkid)); } if let Some(discv5) = discovery_v5_config.as_mut() { @@ -332,6 +337,12 @@ impl NetworkManager { nat, ); + // Spawn required block peer filter if configured + if !required_block_hashes.is_empty() { + let filter = RequiredBlockFilter::new(handle.clone(), required_block_hashes); + filter.spawn(); + } + Ok(Self { swarm, handle, @@ -454,6 +465,11 @@ impl NetworkManager { genesis: status.genesis, config: Default::default(), }, + capabilities: hello_message + .protocols + .into_iter() + .map(|protocol| protocol.cap) + .collect(), } } @@ -509,6 +525,13 @@ impl NetworkManager { response, }) } + PeerRequest::GetReceipts69 { request, response } => { + self.delegate_eth_request(IncomingEthRequest::GetReceipts69 { + peer_id, + request, + response, + }) + } PeerRequest::GetPooledTransactions { request, response } => { self.notify_tx_manager(NetworkTransactionEvent::GetPooledTransactions { peer_id, @@ -520,7 +543,7 @@ impl NetworkManager { } /// Invoked after a `NewBlock` message from the peer was validated - fn on_block_import_result(&mut self, event: BlockImportEvent) { + fn on_block_import_result(&mut self, event: BlockImportEvent) { match event { BlockImportEvent::Announcement(validation) => match validation { BlockValidation::ValidHeader { block } => { @@ -613,6 +636,7 @@ impl NetworkManager { PeerMessage::SendTransactions(_) => { unreachable!("Not emitted by session") } + PeerMessage::BlockRangeUpdated(_) => {} PeerMessage::Other(other) => { debug!(target: "net", message_id=%other.id, "Ignoring unsupported message"); } @@ -712,6 +736,9 @@ impl NetworkManager { let _ = tx.send(None); } } + NetworkHandleMessage::InternalBlockRangeUpdate(block_range_update) => { + self.swarm.sessions_mut().update_advertised_block_range(block_range_update); + } NetworkHandleMessage::EthMessage { peer_id, message } => { self.swarm.sessions_mut().send_message(&peer_id, message) } @@ -818,8 +845,7 @@ impl NetworkManager { "Session disconnected" ); - let mut reason = None; - if let Some(ref err) = error { + let reason = if let Some(ref err) = error { // If the connection was closed due to an error, we report // the peer self.swarm.state_mut().peers_mut().on_active_session_dropped( @@ -827,11 +853,12 @@ impl NetworkManager { &peer_id, err, ); - reason = err.as_disconnected(); + err.as_disconnected() } else { // Gracefully disconnected self.swarm.state_mut().peers_mut().on_active_session_gracefully_closed(peer_id); - } + None + }; self.metrics.closed_sessions.increment(1); self.update_active_connection_metrics(); diff --git a/crates/net/network/src/message.rs b/crates/net/network/src/message.rs index 8ef7f4537d2..115939b1616 100644 --- a/crates/net/network/src/message.rs +++ b/crates/net/network/src/message.rs @@ -3,18 +3,20 @@ //! An `RLPx` stream is multiplexed via the prepended message-id of a framed message. //! Capabilities are exchanged via the `RLPx` `Hello` message as pairs of `(id, version)`, +use crate::types::Receipts69; use alloy_consensus::{BlockHeader, ReceiptWithBloom}; use alloy_primitives::{Bytes, B256}; use futures::FutureExt; use reth_eth_wire::{ - message::RequestPair, BlockBodies, BlockHeaders, EthMessage, EthNetworkPrimitives, - GetBlockBodies, GetBlockHeaders, NetworkPrimitives, NewBlock, NewBlockHashes, - NewPooledTransactionHashes, NodeData, PooledTransactions, Receipts, SharedTransactions, - Transactions, + message::RequestPair, BlockBodies, BlockHeaders, BlockRangeUpdate, EthMessage, + EthNetworkPrimitives, GetBlockBodies, GetBlockHeaders, NetworkPrimitives, NewBlock, + NewBlockHashes, NewBlockPayload, NewPooledTransactionHashes, NodeData, PooledTransactions, + Receipts, SharedTransactions, Transactions, }; use reth_eth_wire_types::RawCapabilityMessage; use reth_network_api::PeerRequest; use reth_network_p2p::error::{RequestError, RequestResult}; +use reth_primitives_traits::Block; use std::{ sync::Arc, task::{ready, Context, Poll}, @@ -23,19 +25,19 @@ use tokio::sync::oneshot; /// Internal form of a `NewBlock` message #[derive(Debug, Clone)] -pub struct NewBlockMessage { +pub struct NewBlockMessage

> { /// Hash of the block pub hash: B256, /// Raw received message - pub block: Arc>, + pub block: Arc

, } // === impl NewBlockMessage === -impl NewBlockMessage { +impl NewBlockMessage

{ /// Returns the block number of the block pub fn number(&self) -> u64 { - self.block.block.header().number() + self.block.block().header().number() } } @@ -46,7 +48,7 @@ pub enum PeerMessage { /// Announce new block hashes NewBlockHashes(NewBlockHashes), /// Broadcast new block. - NewBlock(NewBlockMessage), + NewBlock(NewBlockMessage), /// Received transactions _from_ the peer ReceivedTransaction(Transactions), /// Broadcast transactions _from_ local _to_ a peer. @@ -55,6 +57,8 @@ pub enum PeerMessage { PooledTransactions(NewPooledTransactionHashes), /// All `eth` request variants. EthRequest(PeerRequest), + /// Announces when `BlockRange` is updated. + BlockRangeUpdated(BlockRangeUpdate), /// Any other or manually crafted eth message. /// /// Caution: It is expected that this is a valid `eth_` capability message. @@ -103,6 +107,15 @@ pub enum PeerResponse { /// The receiver channel for the response to a receipts request. response: oneshot::Receiver>>, }, + /// Represents a response to a request for receipts. + /// + /// This is a variant of `Receipts` that was introduced in `eth/69`. + /// The difference is that this variant does not require the inclusion of bloom filters in the + /// response, making it more lightweight. + Receipts69 { + /// The receiver channel for the response to a receipts request. + response: oneshot::Receiver>>, + }, } // === impl PeerResponse === @@ -135,6 +148,9 @@ impl PeerResponse { Self::Receipts { response } => { poll_request!(response, Receipts, cx) } + Self::Receipts69 { response } => { + poll_request!(response, Receipts69, cx) + } }; Poll::Ready(res) } @@ -153,6 +169,8 @@ pub enum PeerResponseResult { NodeData(RequestResult>), /// Represents a result containing receipts or an error. Receipts(RequestResult>>>), + /// Represents a result containing receipts or an error for eth/69. + Receipts69(RequestResult>>), } // === impl PeerResponseResult === @@ -187,6 +205,9 @@ impl PeerResponseResult { Self::Receipts(resp) => { to_message!(resp, Receipts, id) } + Self::Receipts69(resp) => { + to_message!(resp, Receipts69, id) + } } } @@ -198,6 +219,7 @@ impl PeerResponseResult { Self::PooledTransactions(res) => res.as_ref().err(), Self::NodeData(res) => res.as_ref().err(), Self::Receipts(res) => res.as_ref().err(), + Self::Receipts69(res) => res.as_ref().err(), } } diff --git a/crates/net/network/src/network.rs b/crates/net/network/src/network.rs index 16fda0dc4b8..cfc3d56cb28 100644 --- a/crates/net/network/src/network.rs +++ b/crates/net/network/src/network.rs @@ -9,7 +9,7 @@ use parking_lot::Mutex; use reth_discv4::{Discv4, NatResolver}; use reth_discv5::Discv5; use reth_eth_wire::{ - DisconnectReason, EthNetworkPrimitives, NetworkPrimitives, NewBlock, + BlockRangeUpdate, DisconnectReason, EthNetworkPrimitives, NetworkPrimitives, NewPooledTransactionHashes, SharedTransactions, }; use reth_ethereum_forks::Head; @@ -116,7 +116,7 @@ impl NetworkHandle { /// Caution: in `PoS` this is a noop because new blocks are no longer announced over devp2p. /// Instead they are sent to the node by CL and can be requested over devp2p. /// Broadcasting new blocks is considered a protocol violation. - pub fn announce_block(&self, block: NewBlock, hash: B256) { + pub fn announce_block(&self, block: N::NewBlockPayload, hash: B256) { self.send_message(NetworkHandleMessage::AnnounceBlock(block, hash)) } @@ -415,6 +415,11 @@ impl NetworkSyncUpdater for NetworkHandle { fn update_status(&self, head: Head) { self.send_message(NetworkHandleMessage::StatusUpdate { head }); } + + /// Updates the advertised block range. + fn update_block_range(&self, update: reth_eth_wire::BlockRangeUpdate) { + self.send_message(NetworkHandleMessage::InternalBlockRangeUpdate(update)); + } } impl BlockDownloaderProvider for NetworkHandle { @@ -479,7 +484,7 @@ pub(crate) enum NetworkHandleMessage), /// Broadcasts an event to announce a new block to all nodes. - AnnounceBlock(NewBlock, B256), + AnnounceBlock(N::NewBlockPayload, B256), /// Sends a list of transactions to the given peer. SendTransaction { /// The ID of the peer to which the transactions are sent. @@ -541,4 +546,6 @@ pub(crate) enum NetworkHandleMessage { let peer = entry.get_mut(); peer.kind = kind; - peer.fork_id = fork_id; + peer.fork_id = fork_id.map(Box::new); peer.addr = addr; if peer.state.is_incoming() { @@ -758,7 +770,7 @@ impl PeersManager { Entry::Vacant(entry) => { trace!(target: "net::peers", ?peer_id, addr=?addr.tcp(), "discovered new node"); let mut peer = Peer::with_kind(addr, kind); - peer.fork_id = fork_id; + peer.fork_id = fork_id.map(Box::new); entry.insert(peer); self.queued_actions.push_back(PeerAction::PeerAdded(peer_id)); } @@ -796,7 +808,7 @@ impl PeersManager { } } - /// Connect to the given peer. NOTE: if the maximum number out outbound sessions is reached, + /// Connect to the given peer. NOTE: if the maximum number of outbound sessions is reached, /// this won't do anything. See `reth_network::SessionManager::dial_outbound`. #[cfg_attr(not(test), expect(dead_code))] pub(crate) fn add_and_connect( @@ -826,7 +838,7 @@ impl PeersManager { Entry::Occupied(mut entry) => { let peer = entry.get_mut(); peer.kind = kind; - peer.fork_id = fork_id; + peer.fork_id = fork_id.map(Box::new); peer.addr = addr; if peer.state == PeerConnectionState::Idle { @@ -841,7 +853,7 @@ impl PeersManager { trace!(target: "net::peers", ?peer_id, addr=?addr.tcp(), "connects new node"); let mut peer = Peer::with_kind(addr, kind); peer.state = PeerConnectionState::PendingOut; - peer.fork_id = fork_id; + peer.fork_id = fork_id.map(Box::new); entry.insert(peer); self.connection_info.inc_pending_out(); self.queued_actions @@ -948,7 +960,7 @@ impl PeersManager { if peer.addr != new_addr { peer.addr = new_addr; - trace!(target: "net::peers", ?peer_id, addre=?peer.addr, "Updated resolved trusted peer address"); + trace!(target: "net::peers", ?peer_id, addr=?peer.addr, "Updated resolved trusted peer address"); } } } diff --git a/crates/net/network/src/required_block_filter.rs b/crates/net/network/src/required_block_filter.rs new file mode 100644 index 00000000000..9c831e2f5d2 --- /dev/null +++ b/crates/net/network/src/required_block_filter.rs @@ -0,0 +1,179 @@ +//! Required block peer filtering implementation. +//! +//! This module provides functionality to filter out peers that don't have +//! specific required blocks (primarily used for shadowfork testing). + +use alloy_primitives::B256; +use futures::StreamExt; +use reth_eth_wire_types::{GetBlockHeaders, HeadersDirection}; +use reth_network_api::{ + NetworkEvent, NetworkEventListenerProvider, PeerRequest, Peers, ReputationChangeKind, +}; +use tokio::sync::oneshot; +use tracing::{debug, info, trace}; + +/// Task that filters peers based on required block hashes. +/// +/// This task listens for new peer sessions and checks if they have the required +/// block hashes. Peers that don't have these blocks are banned. +pub struct RequiredBlockFilter { + /// Network handle for listening to events and managing peer reputation. + network: N, + /// List of block hashes that peers must have to be considered valid. + block_hashes: Vec, +} + +impl RequiredBlockFilter +where + N: NetworkEventListenerProvider + Peers + Clone + Send + Sync + 'static, +{ + /// Creates a new required block peer filter. + pub const fn new(network: N, block_hashes: Vec) -> Self { + Self { network, block_hashes } + } + + /// Spawns the required block peer filter task. + /// + /// This task will run indefinitely, monitoring new peer sessions and filtering + /// out peers that don't have the required blocks. + pub fn spawn(self) { + if self.block_hashes.is_empty() { + debug!(target: "net::filter", "No required block hashes configured, skipping peer filtering"); + return; + } + + info!(target: "net::filter", "Starting required block peer filter with {} block hashes", self.block_hashes.len()); + + tokio::spawn(async move { + self.run().await; + }); + } + + /// Main loop for the required block peer filter. + async fn run(self) { + let mut event_stream = self.network.event_listener(); + + while let Some(event) = event_stream.next().await { + if let NetworkEvent::ActivePeerSession { info, messages } = event { + let peer_id = info.peer_id; + debug!(target: "net::filter", "New peer session established: {}", peer_id); + + // Spawn a task to check this peer's blocks + let network = self.network.clone(); + let block_hashes = self.block_hashes.clone(); + + tokio::spawn(async move { + Self::check_peer_blocks(network, peer_id, messages, block_hashes).await; + }); + } + } + } + + /// Checks if a peer has the required blocks and bans them if not. + async fn check_peer_blocks( + network: N, + peer_id: reth_network_api::PeerId, + messages: reth_network_api::PeerRequestSender>, + block_hashes: Vec, + ) { + for block_hash in block_hashes { + trace!(target: "net::filter", "Checking if peer {} has block {}", peer_id, block_hash); + + // Create a request for block headers + let request = GetBlockHeaders { + start_block: block_hash.into(), + limit: 1, + skip: 0, + direction: HeadersDirection::Rising, + }; + + let (tx, rx) = oneshot::channel(); + let peer_request = PeerRequest::GetBlockHeaders { request, response: tx }; + + // Send the request to the peer + if let Err(e) = messages.try_send(peer_request) { + debug!(target: "net::filter", "Failed to send block header request to peer {}: {:?}", peer_id, e); + continue; + } + + // Wait for the response + let response = match rx.await { + Ok(response) => response, + Err(e) => { + debug!( + target: "net::filter", + "Channel error getting block {} from peer {}: {:?}", + block_hash, peer_id, e + ); + continue; + } + }; + + let headers = match response { + Ok(headers) => headers, + Err(e) => { + debug!(target: "net::filter", "Error getting block {} from peer {}: {:?}", block_hash, peer_id, e); + // Ban the peer if they fail to respond properly + network.reputation_change(peer_id, ReputationChangeKind::BadProtocol); + return; + } + }; + + if headers.0.is_empty() { + info!( + target: "net::filter", + "Peer {} does not have required block {}, banning", + peer_id, block_hash + ); + network.reputation_change(peer_id, ReputationChangeKind::BadProtocol); + return; // No need to check more blocks if one is missing + } + + trace!(target: "net::filter", "Peer {} has required block {}", peer_id, block_hash); + } + + debug!(target: "net::filter", "Peer {} has all required blocks", peer_id); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use alloy_primitives::{b256, B256}; + use reth_network_api::noop::NoopNetwork; + + #[test] + fn test_required_block_filter_creation() { + let network = NoopNetwork::default(); + let block_hashes = vec![ + b256!("0x1111111111111111111111111111111111111111111111111111111111111111"), + b256!("0x2222222222222222222222222222222222222222222222222222222222222222"), + ]; + + let filter = RequiredBlockFilter::new(network, block_hashes.clone()); + assert_eq!(filter.block_hashes.len(), 2); + assert_eq!(filter.block_hashes, block_hashes); + } + + #[test] + fn test_required_block_filter_empty_hashes_does_not_spawn() { + let network = NoopNetwork::default(); + let block_hashes = vec![]; + + let filter = RequiredBlockFilter::new(network, block_hashes); + // This should not panic and should exit early when spawn is called + filter.spawn(); + } + + #[tokio::test] + async fn test_required_block_filter_with_mock_peer() { + // This test would require a more complex setup with mock network components + // For now, we ensure the basic structure is correct + let network = NoopNetwork::default(); + let block_hashes = vec![B256::default()]; + + let filter = RequiredBlockFilter::new(network, block_hashes); + // Verify the filter can be created and basic properties are set + assert_eq!(filter.block_hashes.len(), 1); + } +} diff --git a/crates/net/network/src/session/active.rs b/crates/net/network/src/session/active.rs index 77e6da94ce1..0044c1f92e1 100644 --- a/crates/net/network/src/session/active.rs +++ b/crates/net/network/src/session/active.rs @@ -16,16 +16,17 @@ use crate::{ session::{ conn::EthRlpxConnection, handle::{ActiveSessionMessage, SessionCommand}, - SessionId, + BlockRangeInfo, EthVersion, SessionId, }, }; +use alloy_eips::merge::EPOCH_SLOTS; use alloy_primitives::Sealable; use futures::{stream::Fuse, SinkExt, StreamExt}; use metrics::Gauge; use reth_eth_wire::{ errors::{EthHandshakeError, EthStreamError}, - message::{EthBroadcastMessage, RequestPair}, - Capabilities, DisconnectP2P, DisconnectReason, EthMessage, NetworkPrimitives, + message::{EthBroadcastMessage, MessageError, RequestPair}, + Capabilities, DisconnectP2P, DisconnectReason, EthMessage, NetworkPrimitives, NewBlockPayload, }; use reth_eth_wire_types::RawCapabilityMessage; use reth_metrics::common::mpsc::MeteredPollSender; @@ -43,10 +44,18 @@ use tokio_stream::wrappers::ReceiverStream; use tokio_util::sync::PollSender; use tracing::{debug, trace}; +/// The recommended interval at which to check if a new range update should be sent to the remote +/// peer. +/// +/// Updates are only sent when the block height has advanced by at least one epoch (32 blocks) +/// since the last update. The interval is set to one epoch duration in seconds. +pub(super) const RANGE_UPDATE_INTERVAL: Duration = Duration::from_secs(EPOCH_SLOTS * 12); + // Constants for timeout updating. /// Minimum timeout value const MINIMUM_TIMEOUT: Duration = Duration::from_secs(2); + /// Maximum timeout value const MAXIMUM_TIMEOUT: Duration = INITIAL_REQUEST_TIMEOUT; /// How much the new measurements affect the current timeout (X percent) @@ -114,6 +123,18 @@ pub(crate) struct ActiveSession { /// Used to reserve a slot to guarantee that the termination message is delivered pub(crate) terminate_message: Option<(PollSender>, ActiveSessionMessage)>, + /// The eth69 range info for the remote peer. + pub(crate) range_info: Option, + /// The eth69 range info for the local node (this node). + /// This represents the range of blocks that this node can serve to other peers. + pub(crate) local_range_info: BlockRangeInfo, + /// Optional interval for sending periodic range updates to the remote peer (eth69+) + /// The interval is set to one epoch duration (~6.4 minutes), but updates are only sent when + /// the block height has advanced by at least one epoch (32 blocks) since the last update + pub(crate) range_update_interval: Option, + /// The last latest block number we sent in a range update + /// Used to avoid sending unnecessary updates when block height hasn't changed significantly + pub(crate) last_sent_latest_block: Option, } impl ActiveSession { @@ -172,6 +193,7 @@ impl ActiveSession { if let Some(req) = self.inflight_requests.remove(&request_id) { match req.request { RequestState::Waiting(PeerRequest::$item { response, .. }) => { + trace!(peer_id=?self.remote_peer_id, ?request_id, "received response from peer"); let _ = response.send(Ok(message)); self.update_request_timeout(req.timestamp, Instant::now()); } @@ -184,6 +206,7 @@ impl ActiveSession { } } } else { + trace!(peer_id=?self.remote_peer_id, ?request_id, "received response to unknown request"); // we received a response to a request we never sent self.on_bad_message(); } @@ -201,8 +224,10 @@ impl ActiveSession { self.try_emit_broadcast(PeerMessage::NewBlockHashes(msg)).into() } EthMessage::NewBlock(msg) => { - let block = - NewBlockMessage { hash: msg.block.header().hash_slow(), block: Arc::new(*msg) }; + let block = NewBlockMessage { + hash: msg.block().header().hash_slow(), + block: Arc::new(*msg), + }; self.try_emit_broadcast(PeerMessage::NewBlock(block)).into() } EthMessage::Transactions(msg) => { @@ -249,11 +274,46 @@ impl ActiveSession { on_response!(resp, GetNodeData) } EthMessage::GetReceipts(req) => { - on_request!(req, Receipts, GetReceipts) + if self.conn.version() >= EthVersion::Eth69 { + on_request!(req, Receipts69, GetReceipts69) + } else { + on_request!(req, Receipts, GetReceipts) + } } EthMessage::Receipts(resp) => { on_response!(resp, GetReceipts) } + EthMessage::Receipts69(resp) => { + on_response!(resp, GetReceipts69) + } + EthMessage::BlockRangeUpdate(msg) => { + // Validate that earliest <= latest according to the spec + if msg.earliest > msg.latest { + return OnIncomingMessageOutcome::BadMessage { + error: EthStreamError::InvalidMessage(MessageError::Other(format!( + "invalid block range: earliest ({}) > latest ({})", + msg.earliest, msg.latest + ))), + message: EthMessage::BlockRangeUpdate(msg), + }; + } + + // Validate that the latest hash is not zero + if msg.latest_hash.is_zero() { + return OnIncomingMessageOutcome::BadMessage { + error: EthStreamError::InvalidMessage(MessageError::Other( + "invalid block range: latest_hash cannot be zero".to_string(), + )), + message: EthMessage::BlockRangeUpdate(msg), + }; + } + + if let Some(range_info) = self.range_info.as_ref() { + range_info.update(msg.earliest, msg.latest, msg.latest_hash); + } + + OnIncomingMessageOutcome::Ok + } EthMessage::Other(bytes) => self.try_emit_broadcast(PeerMessage::Other(bytes)).into(), } } @@ -261,6 +321,8 @@ impl ActiveSession { /// Handle an internal peer request that will be sent to the remote. fn on_internal_peer_request(&mut self, request: PeerRequest, deadline: Instant) { let request_id = self.next_id(); + + trace!(?request, peer_id=?self.remote_peer_id, ?request_id, "sending request to peer"); let msg = request.create_request_message(request_id); self.queued_outgoing.push_back(msg.into()); let req = InflightRequest { @@ -283,6 +345,8 @@ impl ActiveSession { PeerMessage::PooledTransactions(msg) => { if msg.is_valid_for_version(self.conn.version()) { self.queued_outgoing.push_back(EthMessage::from(msg).into()); + } else { + debug!(target: "net", ?msg, version=?self.conn.version(), "Message is invalid for connection version, skipping"); } } PeerMessage::EthRequest(req) => { @@ -292,6 +356,7 @@ impl ActiveSession { PeerMessage::SendTransactions(msg) => { self.queued_outgoing.push_back(EthBroadcastMessage::Transactions(msg).into()); } + PeerMessage::BlockRangeUpdated(_) => {} PeerMessage::ReceivedTransaction(_) => { unreachable!("Not emitted by network") } @@ -679,13 +744,33 @@ impl Future for ActiveSession { } } + if let Some(interval) = &mut this.range_update_interval { + // Check if we should send a range update based on block height changes + while interval.poll_tick(cx).is_ready() { + let current_latest = this.local_range_info.latest(); + let should_send = if let Some(last_sent) = this.last_sent_latest_block { + // Only send if block height has advanced by at least one epoch (32 blocks) + current_latest.saturating_sub(last_sent) >= EPOCH_SLOTS + } else { + true // First update, always send + }; + + if should_send { + this.queued_outgoing.push_back( + EthMessage::BlockRangeUpdate(this.local_range_info.to_message()).into(), + ); + this.last_sent_latest_block = Some(current_latest); + } + } + } + while this.internal_request_timeout_interval.poll_tick(cx).is_ready() { // check for timed out requests - if this.check_timed_out_requests(Instant::now()) { - if let Poll::Ready(Ok(_)) = this.to_session_manager.poll_reserve(cx) { - let msg = ActiveSessionMessage::ProtocolBreach { peer_id: this.remote_peer_id }; - this.pending_message_to_session = Some(msg); - } + if this.check_timed_out_requests(Instant::now()) && + let Poll::Ready(Ok(_)) = this.to_session_manager.poll_reserve(cx) + { + let msg = ActiveSessionMessage::ProtocolBreach { peer_id: this.remote_peer_id }; + this.pending_message_to_session = Some(msg); } } @@ -771,6 +856,7 @@ enum RequestState { } /// Outgoing messages that can be sent over the wire. +#[derive(Debug)] pub(crate) enum OutgoingMessage { /// A message that is owned. Eth(EthMessage), @@ -838,6 +924,16 @@ impl QueuedOutgoingMessages { } } +impl Drop for QueuedOutgoingMessages { + fn drop(&mut self) { + // Ensure gauge is decremented for any remaining items to avoid metric leak on teardown. + let remaining = self.messages.len(); + if remaining > 0 { + self.count.decrement(remaining as f64); + } + } +} + #[cfg(test)] mod tests { use super::*; @@ -847,8 +943,8 @@ mod tests { use reth_ecies::stream::ECIESStream; use reth_eth_wire::{ handshake::EthHandshake, EthNetworkPrimitives, EthStream, GetBlockBodies, - HelloMessageWithProtocols, P2PStream, Status, StatusBuilder, UnauthedEthStream, - UnauthedP2PStream, + HelloMessageWithProtocols, P2PStream, StatusBuilder, UnauthedEthStream, UnauthedP2PStream, + UnifiedStatus, }; use reth_ethereum_forks::EthereumHardfork; use reth_network_peers::pk2id; @@ -872,7 +968,7 @@ mod tests { secret_key: SecretKey, local_peer_id: PeerId, hello: HelloMessageWithProtocols, - status: Status, + status: UnifiedStatus, fork_filter: ForkFilter, next_id: usize, } @@ -894,7 +990,7 @@ mod tests { F: FnOnce(EthStream>, N>) -> O + Send + 'static, O: Future + Send + Sync, { - let status = self.status; + let mut status = self.status; let fork_filter = self.fork_filter.clone(); let local_peer_id = self.local_peer_id; let mut hello = self.hello.clone(); @@ -906,6 +1002,9 @@ mod tests { let (p2p_stream, _) = UnauthedP2PStream::new(sink).handshake(hello).await.unwrap(); + let eth_version = p2p_stream.shared_capabilities().eth_version().unwrap(); + status.set_eth_version(eth_version); + let (client_stream, _) = UnauthedEthStream::new(p2p_stream) .handshake(status, fork_filter) .await @@ -976,6 +1075,14 @@ mod tests { )), protocol_breach_request_timeout: PROTOCOL_BREACH_REQUEST_TIMEOUT, terminate_message: None, + range_info: None, + local_range_info: BlockRangeInfo::new( + 0, + 1000, + alloy_primitives::B256::ZERO, + ), + range_update_interval: None, + last_sent_latest_block: None, } } ev => { diff --git a/crates/net/network/src/session/conn.rs b/crates/net/network/src/session/conn.rs index 1b262430f14..ea13cef4f01 100644 --- a/crates/net/network/src/session/conn.rs +++ b/crates/net/network/src/session/conn.rs @@ -65,7 +65,7 @@ impl EthRlpxConnection { } } - /// Returns access to the underlying stream. + /// Returns access to the underlying stream. #[inline] pub(crate) const fn inner(&self) -> &P2PStream> { match self { diff --git a/crates/net/network/src/session/counter.rs b/crates/net/network/src/session/counter.rs index 215c7279d1b..a3318ea05c5 100644 --- a/crates/net/network/src/session/counter.rs +++ b/crates/net/network/src/session/counter.rs @@ -3,7 +3,7 @@ use reth_network_api::Direction; use reth_network_types::SessionLimits; /// Keeps track of all sessions. -#[derive(Debug, Clone)] +#[derive(Debug)] pub struct SessionCounter { /// Limits to enforce. limits: SessionLimits, @@ -80,10 +80,10 @@ impl SessionCounter { } const fn ensure(current: u32, limit: Option) -> Result<(), ExceedsSessionLimit> { - if let Some(limit) = limit { - if current >= limit { - return Err(ExceedsSessionLimit(limit)) - } + if let Some(limit) = limit && + current >= limit + { + return Err(ExceedsSessionLimit(limit)) } Ok(()) } diff --git a/crates/net/network/src/session/handle.rs b/crates/net/network/src/session/handle.rs index ed465d33ec2..a023d30fa1b 100644 --- a/crates/net/network/src/session/handle.rs +++ b/crates/net/network/src/session/handle.rs @@ -7,7 +7,8 @@ use crate::{ }; use reth_ecies::ECIESError; use reth_eth_wire::{ - errors::EthStreamError, Capabilities, DisconnectReason, EthVersion, NetworkPrimitives, Status, + errors::EthStreamError, Capabilities, DisconnectReason, EthVersion, NetworkPrimitives, + UnifiedStatus, }; use reth_network_api::PeerInfo; use reth_network_peers::{NodeRecord, PeerId}; @@ -73,7 +74,7 @@ pub struct ActiveSessionHandle { /// The local address of the connection. pub(crate) local_addr: Option, /// The Status message the peer sent for the `eth` handshake - pub(crate) status: Arc, + pub(crate) status: Arc, } // === impl ActiveSessionHandle === @@ -173,7 +174,7 @@ pub enum PendingSessionEvent { /// All capabilities the peer announced capabilities: Arc, /// The Status message the peer sent for the `eth` handshake - status: Arc, + status: Arc, /// The actual connection stream which can be used to send and receive `eth` protocol /// messages conn: EthRlpxConnection, diff --git a/crates/net/network/src/session/mod.rs b/crates/net/network/src/session/mod.rs index f9692f12135..17528e2fcfa 100644 --- a/crates/net/network/src/session/mod.rs +++ b/crates/net/network/src/session/mod.rs @@ -4,24 +4,8 @@ mod active; mod conn; mod counter; mod handle; - -use active::QueuedOutgoingMessages; -pub use conn::EthRlpxConnection; -pub use handle::{ - ActiveSessionHandle, ActiveSessionMessage, PendingSessionEvent, PendingSessionHandle, - SessionCommand, -}; - -pub use reth_network_api::{Direction, PeerInfo}; - -use std::{ - collections::HashMap, - future::Future, - net::SocketAddr, - sync::{atomic::AtomicU64, Arc}, - task::{Context, Poll}, - time::{Duration, Instant}, -}; +mod types; +pub use types::BlockRangeInfo; use crate::{ message::PeerMessage, @@ -29,13 +13,15 @@ use crate::{ protocol::{IntoRlpxSubProtocol, OnNotSupported, RlpxSubProtocolHandlers, RlpxSubProtocols}, session::active::ActiveSession, }; +use active::QueuedOutgoingMessages; use counter::SessionCounter; use futures::{future::Either, io, FutureExt, StreamExt}; use reth_ecies::{stream::ECIESStream, ECIESError}; use reth_eth_wire::{ errors::EthStreamError, handshake::EthRlpxHandshake, multiplex::RlpxProtocolMultiplexer, - Capabilities, DisconnectReason, EthStream, EthVersion, HelloMessageWithProtocols, - NetworkPrimitives, Status, UnauthedP2PStream, HANDSHAKE_TIMEOUT, + BlockRangeUpdate, Capabilities, DisconnectReason, EthStream, EthVersion, + HelloMessageWithProtocols, NetworkPrimitives, UnauthedP2PStream, UnifiedStatus, + HANDSHAKE_TIMEOUT, }; use reth_ethereum_forks::{ForkFilter, ForkId, ForkTransition, Head}; use reth_metrics::common::mpsc::MeteredPollSender; @@ -45,6 +31,14 @@ use reth_network_types::SessionsConfig; use reth_tasks::TaskSpawner; use rustc_hash::FxHashMap; use secp256k1::SecretKey; +use std::{ + collections::HashMap, + future::Future, + net::SocketAddr, + sync::{atomic::AtomicU64, Arc}, + task::{Context, Poll}, + time::{Duration, Instant}, +}; use tokio::{ io::{AsyncRead, AsyncWrite}, net::TcpStream, @@ -54,6 +48,14 @@ use tokio_stream::wrappers::ReceiverStream; use tokio_util::sync::PollSender; use tracing::{debug, instrument, trace}; +use crate::session::active::RANGE_UPDATE_INTERVAL; +pub use conn::EthRlpxConnection; +pub use handle::{ + ActiveSessionHandle, ActiveSessionMessage, PendingSessionEvent, PendingSessionHandle, + SessionCommand, +}; +pub use reth_network_api::{Direction, PeerInfo}; + /// Internal identifier for active sessions. #[derive(Debug, Clone, Copy, PartialOrd, PartialEq, Eq, Hash)] pub struct SessionId(usize); @@ -77,7 +79,7 @@ pub struct SessionManager { /// The secret key used for authenticating sessions. secret_key: SecretKey, /// The `Status` message to send to peers. - status: Status, + status: UnifiedStatus, /// The `HelloMessage` message to send to peers. hello_message: HelloMessageWithProtocols, /// The [`ForkFilter`] used to validate the peer's `Status` message. @@ -115,6 +117,9 @@ pub struct SessionManager { metrics: SessionManagerMetrics, /// The [`EthRlpxHandshake`] is used to perform the initial handshake with the peer. handshake: Arc, + /// Shared local range information that gets propagated to active sessions. + /// This represents the range of blocks that this node can serve to other peers. + local_range_info: BlockRangeInfo, } // === impl SessionManager === @@ -126,7 +131,7 @@ impl SessionManager { secret_key: SecretKey, config: SessionsConfig, executor: Box, - status: Status, + status: UnifiedStatus, hello_message: HelloMessageWithProtocols, fork_filter: ForkFilter, extra_protocols: RlpxSubProtocols, @@ -136,6 +141,13 @@ impl SessionManager { let (active_session_tx, active_session_rx) = mpsc::channel(config.session_event_buffer); let active_session_tx = PollSender::new(active_session_tx); + // Initialize local range info from the status + let local_range_info = BlockRangeInfo::new( + status.earliest_block.unwrap_or_default(), + status.latest_block.unwrap_or_default(), + status.blockhash, + ); + Self { next_id: 0, counter: SessionCounter::new(config.limits), @@ -158,9 +170,15 @@ impl SessionManager { disconnections_counter: Default::default(), metrics: Default::default(), handshake, + local_range_info, } } + /// Returns the currently tracked [`ForkId`]. + pub(crate) const fn fork_id(&self) -> ForkId { + self.fork_filter.current() + } + /// Check whether the provided [`ForkId`] is compatible based on the validation rules in /// `EIP-2124`. pub fn is_valid_fork_id(&self, fork_id: ForkId) -> bool { @@ -175,7 +193,7 @@ impl SessionManager { } /// Returns the current status of the session. - pub const fn status(&self) -> Status { + pub const fn status(&self) -> UnifiedStatus { self.status } @@ -220,9 +238,11 @@ impl SessionManager { /// active [`ForkId`]. See also [`ForkFilter::set_head`]. pub(crate) fn on_status_update(&mut self, head: Head) -> Option { self.status.blockhash = head.hash; - self.status.total_difficulty = head.total_difficulty; + self.status.total_difficulty = Some(head.total_difficulty); let transition = self.fork_filter.set_head(head); self.status.forkid = self.fork_filter.current(); + self.status.latest_block = Some(head.number); + transition } @@ -518,6 +538,14 @@ impl SessionManager { // negotiated version let version = conn.version(); + // Configure the interval at which the range information is updated, starting with + // ETH69 + let range_update_interval = (conn.version() >= EthVersion::Eth69).then(|| { + let mut interval = tokio::time::interval(RANGE_UPDATE_INTERVAL); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + interval + }); + let session = ActiveSession { next_id: 0, remote_peer_id: peer_id, @@ -540,6 +568,10 @@ impl SessionManager { internal_request_timeout: Arc::clone(&timeout), protocol_breach_request_timeout: self.protocol_breach_request_timeout, terminate_message: None, + range_info: None, + local_range_info: self.local_range_info.clone(), + range_update_interval, + last_sent_latest_block: None, }; self.spawn(session); @@ -576,6 +608,7 @@ impl SessionManager { messages, direction, timeout, + range_info: None, }) } PendingSessionEvent::Disconnected { remote_addr, session_id, direction, error } => { @@ -647,6 +680,26 @@ impl SessionManager { } } } + + /// Updates the advertised block range that this node can serve to other peers starting with + /// Eth69. + /// + /// This method updates both the local status message that gets sent to peers during handshake + /// and the shared local range information that gets propagated to active sessions (Eth69). + /// The range information is used in ETH69 protocol where peers announce the range of blocks + /// they can serve to optimize data synchronization. + pub(crate) fn update_advertised_block_range(&mut self, block_range_update: BlockRangeUpdate) { + self.status.earliest_block = Some(block_range_update.earliest); + self.status.latest_block = Some(block_range_update.latest); + self.status.blockhash = block_range_update.latest_hash; + + // Update the shared local range info that gets propagated to active sessions + self.local_range_info.update( + block_range_update.earliest, + block_range_update.latest, + block_range_update.latest_hash, + ); + } } /// A counter for ongoing graceful disconnections attempts. @@ -681,7 +734,7 @@ pub enum SessionEvent { /// negotiated eth version version: EthVersion, /// The Status message the peer sent during the `eth` handshake - status: Arc, + status: Arc, /// The channel for sending messages to the peer with the session messages: PeerRequestSender>, /// The direction of the session, either `Inbound` or `Outgoing` @@ -689,6 +742,8 @@ pub enum SessionEvent { /// The maximum time that the session waits for a response from the peer before timing out /// the connection timeout: Arc, + /// The range info for the peer. + range_info: Option, }, /// The peer was already connected with another session. AlreadyConnected { @@ -828,7 +883,7 @@ pub(crate) async fn start_pending_incoming_session( remote_addr: SocketAddr, secret_key: SecretKey, hello: HelloMessageWithProtocols, - status: Status, + status: UnifiedStatus, fork_filter: ForkFilter, extra_handlers: RlpxSubProtocolHandlers, ) { @@ -850,7 +905,7 @@ pub(crate) async fn start_pending_incoming_session( } /// Starts the authentication process for a connection initiated by a remote peer. -#[instrument(skip_all, fields(%remote_addr, peer_id), target = "net")] +#[instrument(level = "trace", target = "net::network", skip_all, fields(%remote_addr, peer_id))] #[expect(clippy::too_many_arguments)] async fn start_pending_outbound_session( handshake: Arc, @@ -861,7 +916,7 @@ async fn start_pending_outbound_session( remote_peer_id: PeerId, secret_key: SecretKey, hello: HelloMessageWithProtocols, - status: Status, + status: UnifiedStatus, fork_filter: ForkFilter, extra_handlers: RlpxSubProtocolHandlers, ) { @@ -913,7 +968,7 @@ async fn authenticate( secret_key: SecretKey, direction: Direction, hello: HelloMessageWithProtocols, - status: Status, + status: UnifiedStatus, fork_filter: ForkFilter, extra_handlers: RlpxSubProtocolHandlers, ) { @@ -996,7 +1051,7 @@ async fn authenticate_stream( local_addr: Option, direction: Direction, mut hello: HelloMessageWithProtocols, - mut status: Status, + mut status: UnifiedStatus, fork_filter: ForkFilter, mut extra_handlers: RlpxSubProtocolHandlers, ) -> PendingSessionEvent { @@ -1055,12 +1110,12 @@ async fn authenticate_stream( } }; + // Before trying status handshake, set up the version to negotiated shared version + status.set_eth_version(eth_version); + let (conn, their_status) = if p2p_stream.shared_capabilities().len() == 1 { // if the shared caps are 1, we know both support the eth version // if the hello handshake was successful we can try status handshake - // - // Before trying status handshake, set up the version to negotiated shared version - status.set_eth_version(eth_version); // perform the eth protocol handshake match handshake @@ -1068,7 +1123,7 @@ async fn authenticate_stream( .await { Ok(their_status) => { - let eth_stream = EthStream::new(status.version, p2p_stream); + let eth_stream = EthStream::new(eth_version, p2p_stream); (eth_stream.into(), their_status) } Err(err) => { @@ -1096,18 +1151,20 @@ async fn authenticate_stream( .ok(); } - let (multiplex_stream, their_status) = - match multiplex_stream.into_eth_satellite_stream(status, fork_filter).await { - Ok((multiplex_stream, their_status)) => (multiplex_stream, their_status), - Err(err) => { - return PendingSessionEvent::Disconnected { - remote_addr, - session_id, - direction, - error: Some(PendingSessionHandshakeError::Eth(err)), - } + let (multiplex_stream, their_status) = match multiplex_stream + .into_eth_satellite_stream(status, fork_filter, handshake) + .await + { + Ok((multiplex_stream, their_status)) => (multiplex_stream, their_status), + Err(err) => { + return PendingSessionEvent::Disconnected { + remote_addr, + session_id, + direction, + error: Some(PendingSessionHandshakeError::Eth(err)), } - }; + } + }; (multiplex_stream.into(), their_status) }; diff --git a/crates/net/network/src/session/types.rs b/crates/net/network/src/session/types.rs new file mode 100644 index 00000000000..b73bfe3b992 --- /dev/null +++ b/crates/net/network/src/session/types.rs @@ -0,0 +1,89 @@ +//! Shared types for network sessions. + +use alloy_primitives::B256; +use parking_lot::RwLock; +use reth_eth_wire::BlockRangeUpdate; +use std::{ + ops::RangeInclusive, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, + }, +}; + +/// Information about the range of blocks available from a peer. +/// +/// This represents the announced `eth69` +/// [`BlockRangeUpdate`] of a peer. +#[derive(Debug, Clone)] +pub struct BlockRangeInfo { + /// The inner range information. + inner: Arc, +} + +impl BlockRangeInfo { + /// Creates a new range information. + pub fn new(earliest: u64, latest: u64, latest_hash: B256) -> Self { + Self { + inner: Arc::new(BlockRangeInfoInner { + earliest: AtomicU64::new(earliest), + latest: AtomicU64::new(latest), + latest_hash: RwLock::new(latest_hash), + }), + } + } + + /// Returns true if the block number is within the range of blocks available from the peer. + pub fn contains(&self, block_number: u64) -> bool { + self.range().contains(&block_number) + } + + /// Returns the range of blocks available from the peer. + pub fn range(&self) -> RangeInclusive { + let earliest = self.earliest(); + let latest = self.latest(); + RangeInclusive::new(earliest, latest) + } + + /// Returns the earliest block number available from the peer. + pub fn earliest(&self) -> u64 { + self.inner.earliest.load(Ordering::Relaxed) + } + + /// Returns the latest block number available from the peer. + pub fn latest(&self) -> u64 { + self.inner.latest.load(Ordering::Relaxed) + } + + /// Returns the latest block hash available from the peer. + pub fn latest_hash(&self) -> B256 { + *self.inner.latest_hash.read() + } + + /// Updates the range information. + pub fn update(&self, earliest: u64, latest: u64, latest_hash: B256) { + self.inner.earliest.store(earliest, Ordering::Relaxed); + self.inner.latest.store(latest, Ordering::Relaxed); + *self.inner.latest_hash.write() = latest_hash; + } + + /// Converts the current range information to an Eth69 [`BlockRangeUpdate`] message. + pub fn to_message(&self) -> BlockRangeUpdate { + BlockRangeUpdate { + earliest: self.earliest(), + latest: self.latest(), + latest_hash: self.latest_hash(), + } + } +} + +/// Inner structure containing the range information with atomic and thread-safe fields. +#[derive(Debug)] +pub(crate) struct BlockRangeInfoInner { + /// The earliest block which is available. + earliest: AtomicU64, + /// The latest block which is available. + latest: AtomicU64, + /// Latest available block's hash. + latest_hash: RwLock, +} diff --git a/crates/net/network/src/state.rs b/crates/net/network/src/state.rs index 02df5a7fe04..57d1a73198e 100644 --- a/crates/net/network/src/state.rs +++ b/crates/net/network/src/state.rs @@ -6,6 +6,7 @@ use crate::{ fetch::{BlockResponseOutcome, FetchAction, StateFetcher}, message::{BlockRequest, NewBlockMessage, PeerResponse, PeerResponseResult}, peers::{PeerAction, PeersManager}, + session::BlockRangeInfo, FetchClient, }; use alloy_consensus::BlockHeader; @@ -13,7 +14,7 @@ use alloy_primitives::B256; use rand::seq::SliceRandom; use reth_eth_wire::{ BlockHashNumber, Capabilities, DisconnectReason, EthNetworkPrimitives, NetworkPrimitives, - NewBlockHashes, Status, + NewBlockHashes, NewBlockPayload, UnifiedStatus, }; use reth_ethereum_forks::ForkId; use reth_network_api::{DiscoveredEvent, DiscoveryEvent, PeerRequest, PeerRequestSender}; @@ -82,7 +83,7 @@ pub struct NetworkState { /// The client type that can interact with the chain. /// /// This type is used to fetch the block number after we established a session and received the - /// [Status] block hash. + /// [`UnifiedStatus`] block hash. client: BlockNumReader, /// Network discovery. discovery: Discovery, @@ -146,16 +147,23 @@ impl NetworkState { &mut self, peer: PeerId, capabilities: Arc, - status: Arc, + status: Arc, request_tx: PeerRequestSender>, timeout: Arc, + range_info: Option, ) { debug_assert!(!self.active_peers.contains_key(&peer), "Already connected; not possible"); // find the corresponding block number let block_number = self.client.block_number(status.blockhash).ok().flatten().unwrap_or_default(); - self.state_fetcher.new_active_peer(peer, status.blockhash, block_number, timeout); + self.state_fetcher.new_active_peer( + peer, + status.blockhash, + block_number, + timeout, + range_info, + ); self.active_peers.insert( peer, @@ -185,12 +193,12 @@ impl NetworkState { /// > the total number of peers) using the `NewBlock` message. /// /// See also - pub(crate) fn announce_new_block(&mut self, msg: NewBlockMessage) { + pub(crate) fn announce_new_block(&mut self, msg: NewBlockMessage) { // send a `NewBlock` message to a fraction of the connected peers (square root of the total // number of peers) let num_propagate = (self.active_peers.len() as f64).sqrt() as u64 + 1; - let number = msg.block.block.header().number(); + let number = msg.block.block().header().number(); let mut count = 0; // Shuffle to propagate to a random sample of peers on every block announcement @@ -227,8 +235,8 @@ impl NetworkState { /// Completes the block propagation process started in [`NetworkState::announce_new_block()`] /// but sending `NewBlockHash` broadcast to all peers that haven't seen it yet. - pub(crate) fn announce_new_block_hash(&mut self, msg: NewBlockMessage) { - let number = msg.block.block.header().number(); + pub(crate) fn announce_new_block_hash(&mut self, msg: NewBlockMessage) { + let number = msg.block.block().header().number(); let hashes = NewBlockHashes(vec![BlockHashNumber { hash: msg.hash, number }]); for (peer_id, peer) in &mut self.active_peers { if peer.blocks.contains(&msg.hash) { @@ -489,7 +497,7 @@ impl NetworkState { self.on_peer_action(action); } - // We need to poll again tn case we have received any responses because they may have + // We need to poll again in case we have received any responses because they may have // triggered follow-up requests. if self.queued_messages.is_empty() { return Poll::Pending @@ -524,7 +532,7 @@ pub(crate) enum StateAction { /// Target of the message peer_id: PeerId, /// The `NewBlock` message - block: NewBlockMessage, + block: NewBlockMessage, }, NewBlockHashes { /// Target of the message @@ -613,6 +621,7 @@ mod tests { Arc::default(), peer_tx, Arc::new(AtomicU64::new(1)), + None, ); assert!(state.active_peers.contains_key(&peer_id)); diff --git a/crates/net/network/src/swarm.rs b/crates/net/network/src/swarm.rs index 504e3115065..229d149a2f9 100644 --- a/crates/net/network/src/swarm.rs +++ b/crates/net/network/src/swarm.rs @@ -9,7 +9,7 @@ use crate::{ use futures::Stream; use reth_eth_wire::{ errors::EthStreamError, Capabilities, DisconnectReason, EthNetworkPrimitives, EthVersion, - NetworkPrimitives, Status, + NetworkPrimitives, UnifiedStatus, }; use reth_network_api::{PeerRequest, PeerRequestSender}; use reth_network_peers::PeerId; @@ -20,7 +20,7 @@ use std::{ sync::Arc, task::{Context, Poll}, }; -use tracing::trace; +use tracing::{debug, trace}; #[cfg_attr(doc, aquamarine::aquamarine)] /// Contains the connectivity related state of the network. @@ -122,6 +122,7 @@ impl Swarm { messages, direction, timeout, + range_info, } => { self.state.on_session_activated( peer_id, @@ -129,6 +130,7 @@ impl Swarm { status.clone(), messages.clone(), timeout, + range_info, ); Some(SwarmEvent::SessionEstablished { peer_id, @@ -257,6 +259,7 @@ impl Swarm { if self.sessions.is_valid_fork_id(fork_id) { self.state_mut().peers_mut().set_discovered_fork_id(peer_id, fork_id); } else { + debug!(target: "net", ?peer_id, remote_fork_id=?fork_id, our_fork_id=?self.sessions.fork_id(), "fork id mismatch, removing peer"); self.state_mut().peers_mut().remove_peer(peer_id); } } @@ -382,7 +385,7 @@ pub(crate) enum SwarmEvent { /// negotiated eth version version: EthVersion, messages: PeerRequestSender>, - status: Arc, + status: Arc, direction: Direction, }, SessionClosed { diff --git a/crates/net/network/src/test_utils/init.rs b/crates/net/network/src/test_utils/init.rs index 51537f37d87..db61931dd47 100644 --- a/crates/net/network/src/test_utils/init.rs +++ b/crates/net/network/src/test_utils/init.rs @@ -13,7 +13,7 @@ pub fn enr_to_peer_id(enr: Enr) -> PeerId { // copied from ethers-rs /// A bit of hack to find an unused TCP port. /// -/// Does not guarantee that the given port is unused after the function exists, just that it was +/// Does not guarantee that the given port is unused after the function exits, just that it was /// unused before the function started (i.e., it does not reserve a port). pub fn unused_port() -> u16 { unused_tcp_addr().port() diff --git a/crates/net/network/src/test_utils/testnet.rs b/crates/net/network/src/test_utils/testnet.rs index b29341a9405..d2466899543 100644 --- a/crates/net/network/src/test_utils/testnet.rs +++ b/crates/net/network/src/test_utils/testnet.rs @@ -6,19 +6,19 @@ use crate::{ eth_requests::EthRequestHandler, protocol::IntoRlpxSubProtocol, transactions::{ - config::TransactionPropagationKind, TransactionsHandle, TransactionsManager, - TransactionsManagerConfig, + config::{StrictEthAnnouncementFilter, TransactionPropagationKind}, + policy::NetworkPolicies, + TransactionsHandle, TransactionsManager, TransactionsManagerConfig, }, NetworkConfig, NetworkConfigBuilder, NetworkHandle, NetworkManager, }; -use alloy_consensus::transaction::PooledTransaction; use futures::{FutureExt, StreamExt}; use pin_project::pin_project; use reth_chainspec::{ChainSpecProvider, EthereumHardforks, Hardforks}; use reth_eth_wire::{ protocol::Protocol, DisconnectReason, EthNetworkPrimitives, HelloMessageWithProtocols, }; -use reth_ethereum_primitives::TransactionSigned; +use reth_ethereum_primitives::{PooledTransactionVariant, TransactionSigned}; use reth_network_api::{ events::{PeerEvent, SessionInfo}, test_utils::{PeersHandle, PeersHandleProvider}, @@ -246,7 +246,10 @@ where + Unpin + 'static, Pool: TransactionPool< - Transaction: PoolTransaction, + Transaction: PoolTransaction< + Consensus = TransactionSigned, + Pooled = PooledTransactionVariant, + >, > + Unpin + 'static, { @@ -314,7 +317,10 @@ where + Unpin + 'static, Pool: TransactionPool< - Transaction: PoolTransaction, + Transaction: PoolTransaction< + Consensus = TransactionSigned, + Pooled = PooledTransactionVariant, + >, > + Unpin + 'static, { @@ -393,7 +399,13 @@ pub struct Peer { #[pin] request_handler: Option>, #[pin] - transactions_manager: Option>, + transactions_manager: Option< + TransactionsManager< + Pool, + EthNetworkPrimitives, + NetworkPolicies, + >, + >, pool: Option, client: C, secret_key: SecretKey, @@ -523,12 +535,15 @@ where let (tx, rx) = unbounded_channel(); network.set_transactions(tx); + let announcement_policy = StrictEthAnnouncementFilter::default(); + let policies = NetworkPolicies::new(policy, announcement_policy); + let transactions_manager = TransactionsManager::with_policy( network.handle().clone(), pool.clone(), rx, config, - policy, + policies, ); Peer { @@ -562,7 +577,10 @@ where + Unpin + 'static, Pool: TransactionPool< - Transaction: PoolTransaction, + Transaction: PoolTransaction< + Consensus = TransactionSigned, + Pooled = PooledTransactionVariant, + >, > + Unpin + 'static, { @@ -768,9 +786,9 @@ impl NetworkEventStream { peers } - /// Ensures that the first two events are a [`NetworkEvent::Peer(PeerEvent::PeerAdded`] and - /// [`NetworkEvent::ActivePeerSession`], returning the [`PeerId`] of the established - /// session. + /// Ensures that the first two events are a [`NetworkEvent::Peer`] and + /// [`PeerEvent::PeerAdded`][`NetworkEvent::ActivePeerSession`], returning the [`PeerId`] of the + /// established session. pub async fn peer_added_and_established(&mut self) -> Option { let peer_id = match self.inner.next().await { Some(NetworkEvent::Peer(PeerEvent::PeerAdded(peer_id))) => peer_id, diff --git a/crates/net/network/src/test_utils/transactions.rs b/crates/net/network/src/test_utils/transactions.rs index c3c38e3f1c7..467f146b059 100644 --- a/crates/net/network/src/test_utils/transactions.rs +++ b/crates/net/network/src/test_utils/transactions.rs @@ -51,7 +51,7 @@ pub async fn new_tx_manager( (transactions, network) } -/// Directly buffer hahs into tx fetcher for testing. +/// Directly buffer hash into tx fetcher for testing. pub fn buffer_hash_to_tx_fetcher( tx_fetcher: &mut TransactionFetcher, hash: TxHash, diff --git a/crates/net/network/src/transactions/config.rs b/crates/net/network/src/transactions/config.rs index e799d14c766..c34bbecd77b 100644 --- a/crates/net/network/src/transactions/config.rs +++ b/crates/net/network/src/transactions/config.rs @@ -1,4 +1,4 @@ -use std::str::FromStr; +use std::{fmt::Debug, marker::PhantomData, str::FromStr}; use super::{ PeerMetadata, DEFAULT_MAX_COUNT_TRANSACTIONS_SEEN_BY_PEER, @@ -9,8 +9,11 @@ use crate::transactions::constants::tx_fetcher::{ DEFAULT_MAX_CAPACITY_CACHE_PENDING_FETCH, DEFAULT_MAX_COUNT_CONCURRENT_REQUESTS, DEFAULT_MAX_COUNT_CONCURRENT_REQUESTS_PER_PEER, }; +use alloy_primitives::B256; use derive_more::{Constructor, Display}; + use reth_eth_wire::NetworkPrimitives; +use reth_ethereum_primitives::TxType; /// Configuration for managing transactions within the network. #[derive(Debug, Clone)] @@ -36,7 +39,7 @@ impl Default for TransactionsManagerConfig { } /// Determines how new pending transactions are propagated to other peers in full. -#[derive(Debug, Clone, Default)] +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] pub enum TransactionPropagationMode { /// Send full transactions to sqrt of current peers. @@ -58,6 +61,26 @@ impl TransactionPropagationMode { } } } +impl FromStr for TransactionPropagationMode { + type Err = String; + + fn from_str(s: &str) -> Result { + let s = s.to_lowercase(); + match s.as_str() { + "sqrt" => Ok(Self::Sqrt), + "all" => Ok(Self::All), + s => { + if let Some(num) = s.strip_prefix("max:") { + num.parse::() + .map(TransactionPropagationMode::Max) + .map_err(|_| format!("Invalid number for Max variant: {num}")) + } else { + Err(format!("Invalid transaction propagation mode: {s}")) + } + } + } + } +} /// Configuration for fetching transactions. #[derive(Debug, Constructor, Clone)] @@ -118,11 +141,13 @@ pub trait TransactionPropagationPolicy: Send + Sync + Unpin + 'static { pub enum TransactionPropagationKind { /// Propagate transactions to all peers. /// - /// No restructions + /// No restrictions #[default] All, /// Propagate transactions to only trusted peers. Trusted, + /// Do not propagate transactions + None, } impl TransactionPropagationPolicy for TransactionPropagationKind { @@ -130,6 +155,7 @@ impl TransactionPropagationPolicy for TransactionPropagationKind { match self { Self::All => true, Self::Trusted => peer.peer_kind.is_trusted(), + Self::None => false, } } @@ -145,7 +171,165 @@ impl FromStr for TransactionPropagationKind { match s { "All" | "all" => Ok(Self::All), "Trusted" | "trusted" => Ok(Self::Trusted), + "None" | "none" => Ok(Self::None), _ => Err(format!("Invalid transaction propagation policy: {s}")), } } } + +/// Defines the outcome of evaluating a transaction against an `AnnouncementFilteringPolicy`. +/// +/// Dictates how the `TransactionManager` should proceed on an announced transaction. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AnnouncementAcceptance { + /// Accept the transaction announcement. + Accept, + /// Log the transaction but not fetching the transaction or penalizing the peer. + Ignore, + /// Reject + Reject { + /// If true, the peer sending this announcement should be penalized. + penalize_peer: bool, + }, +} + +/// A policy that defines how to handle incoming transaction announcements, +/// particularly concerning transaction types and other announcement metadata. +pub trait AnnouncementFilteringPolicy: Send + Sync + Unpin + 'static { + /// Decides how to handle a transaction announcement based on its type, hash, and size. + fn decide_on_announcement(&self, ty: u8, hash: &B256, size: usize) -> AnnouncementAcceptance; +} + +/// A generic `AnnouncementFilteringPolicy` that enforces strict validation +/// of transaction type based on a generic type `T`. +#[derive(Debug, Clone)] +pub struct TypedStrictFilter + Debug + Send + Sync + 'static>(PhantomData); + +impl + Debug + Send + Sync + 'static> Default for TypedStrictFilter { + fn default() -> Self { + Self(PhantomData) + } +} + +impl AnnouncementFilteringPolicy for TypedStrictFilter +where + T: TryFrom + Debug + Send + Sync + Unpin + 'static, + >::Error: Debug, +{ + fn decide_on_announcement(&self, ty: u8, hash: &B256, size: usize) -> AnnouncementAcceptance { + match T::try_from(ty) { + Ok(_valid_type) => AnnouncementAcceptance::Accept, + Err(e) => { + tracing::trace!(target: "net::tx::policy::strict_typed", + type_param = %std::any::type_name::(), + %ty, + %size, + %hash, + error = ?e, + "Invalid or unrecognized transaction type byte. Rejecting entry and recommending peer penalization." + ); + AnnouncementAcceptance::Reject { penalize_peer: true } + } + } + } +} + +/// Type alias for a `TypedStrictFilter`. This is the default strict announcement filter. +pub type StrictEthAnnouncementFilter = TypedStrictFilter; + +/// An [`AnnouncementFilteringPolicy`] that permissively handles unknown type bytes +/// based on a given type `T` using `T::try_from(u8)`. +/// +/// If `T::try_from(ty)` succeeds, the announcement is accepted. Otherwise, it's ignored. +#[derive(Debug, Clone)] +pub struct TypedRelaxedFilter + Debug + Send + Sync + 'static>(PhantomData); + +impl + Debug + Send + Sync + 'static> Default for TypedRelaxedFilter { + fn default() -> Self { + Self(PhantomData) + } +} + +impl AnnouncementFilteringPolicy for TypedRelaxedFilter +where + T: TryFrom + Debug + Send + Sync + Unpin + 'static, + >::Error: Debug, +{ + fn decide_on_announcement(&self, ty: u8, hash: &B256, size: usize) -> AnnouncementAcceptance { + match T::try_from(ty) { + Ok(_valid_type) => AnnouncementAcceptance::Accept, + Err(e) => { + tracing::trace!(target: "net::tx::policy::relaxed_typed", + type_param = %std::any::type_name::(), + %ty, + %size, + %hash, + error = ?e, + "Unknown transaction type byte. Ignoring entry." + ); + AnnouncementAcceptance::Ignore + } + } + } +} + +/// Type alias for `TypedRelaxedFilter`. This filter accepts known Ethereum transaction types and +/// ignores unknown ones without penalizing the peer. +pub type RelaxedEthAnnouncementFilter = TypedRelaxedFilter; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_transaction_propagation_mode_from_str() { + // Test "sqrt" variant + assert_eq!( + TransactionPropagationMode::from_str("sqrt").unwrap(), + TransactionPropagationMode::Sqrt + ); + assert_eq!( + TransactionPropagationMode::from_str("SQRT").unwrap(), + TransactionPropagationMode::Sqrt + ); + assert_eq!( + TransactionPropagationMode::from_str("Sqrt").unwrap(), + TransactionPropagationMode::Sqrt + ); + + // Test "all" variant + assert_eq!( + TransactionPropagationMode::from_str("all").unwrap(), + TransactionPropagationMode::All + ); + assert_eq!( + TransactionPropagationMode::from_str("ALL").unwrap(), + TransactionPropagationMode::All + ); + assert_eq!( + TransactionPropagationMode::from_str("All").unwrap(), + TransactionPropagationMode::All + ); + + // Test "max:N" variant + assert_eq!( + TransactionPropagationMode::from_str("max:10").unwrap(), + TransactionPropagationMode::Max(10) + ); + assert_eq!( + TransactionPropagationMode::from_str("MAX:42").unwrap(), + TransactionPropagationMode::Max(42) + ); + assert_eq!( + TransactionPropagationMode::from_str("Max:100").unwrap(), + TransactionPropagationMode::Max(100) + ); + + // Test invalid inputs + assert!(TransactionPropagationMode::from_str("invalid").is_err()); + assert!(TransactionPropagationMode::from_str("max:not_a_number").is_err()); + assert!(TransactionPropagationMode::from_str("max:").is_err()); + assert!(TransactionPropagationMode::from_str("max").is_err()); + assert!(TransactionPropagationMode::from_str("").is_err()); + } +} diff --git a/crates/net/network/src/transactions/constants.rs b/crates/net/network/src/transactions/constants.rs index 4213c171e05..905c5931e9e 100644 --- a/crates/net/network/src/transactions/constants.rs +++ b/crates/net/network/src/transactions/constants.rs @@ -57,7 +57,6 @@ pub mod tx_manager { /// Constants used by [`TransactionFetcher`](super::TransactionFetcher). pub mod tx_fetcher { - use crate::transactions::fetcher::TransactionFetcherInfo; use reth_network_types::peers::config::{ DEFAULT_MAX_COUNT_PEERS_INBOUND, DEFAULT_MAX_COUNT_PEERS_OUTBOUND, }; @@ -202,14 +201,16 @@ pub mod tx_fetcher { /// Default divisor of the max inflight request when calculating search breadth of the search /// for any idle peer to which to send a request filled with hashes pending fetch. The max - /// inflight requests is configured in [`TransactionFetcherInfo`]. + /// inflight requests is configured in + /// [`TransactionFetcherInfo`](crate::transactions::fetcher::TransactionFetcherInfo). /// /// Default is 3 requests. pub const DEFAULT_DIVISOR_MAX_COUNT_INFLIGHT_REQUESTS_ON_FIND_IDLE_PEER: usize = 3; /// Default divisor of the max inflight request when calculating search breadth of the search /// for the intersection of hashes announced by a peer and hashes pending fetch. The max - /// inflight requests is configured in [`TransactionFetcherInfo`]. + /// inflight requests is configured in + /// [`TransactionFetcherInfo`](crate::transactions::fetcher::TransactionFetcherInfo). /// /// Default is 3 requests. pub const DEFAULT_DIVISOR_MAX_COUNT_INFLIGHT_REQUESTS_ON_FIND_INTERSECTION: usize = 3; @@ -256,26 +257,4 @@ pub mod tx_fetcher { /// /// Default is 8 hashes. pub const DEFAULT_MARGINAL_COUNT_HASHES_GET_POOLED_TRANSACTIONS_REQUEST: usize = 8; - - /// Returns the approx number of transaction hashes that a - /// [`GetPooledTransactions`](reth_eth_wire::GetPooledTransactions) request will have capacity - /// for w.r.t. the [`Eth68`](reth_eth_wire::EthVersion::Eth68) protocol version. This is useful - /// for preallocating memory. - pub const fn approx_capacity_get_pooled_transactions_req_eth68( - info: &TransactionFetcherInfo, - ) -> usize { - let max_size_expected_response = - info.soft_limit_byte_size_pooled_transactions_response_on_pack_request; - - max_size_expected_response / MEDIAN_BYTE_SIZE_SMALL_LEGACY_TX_ENCODED + - DEFAULT_MARGINAL_COUNT_HASHES_GET_POOLED_TRANSACTIONS_REQUEST - } - - /// Returns the approx number of transactions that a - /// [`GetPooledTransactions`](reth_eth_wire::GetPooledTransactions) request will - /// have capacity for w.r.t. the [`Eth66`](reth_eth_wire::EthVersion::Eth66) protocol version. - /// This is useful for preallocating memory. - pub const fn approx_capacity_get_pooled_transactions_req_eth66() -> usize { - SOFT_LIMIT_COUNT_HASHES_IN_GET_POOLED_TRANSACTIONS_REQUEST - } } diff --git a/crates/net/network/src/transactions/fetcher.rs b/crates/net/network/src/transactions/fetcher.rs index 43dc1715fb5..a112e8cac89 100644 --- a/crates/net/network/src/transactions/fetcher.rs +++ b/crates/net/network/src/transactions/fetcher.rs @@ -28,14 +28,12 @@ use super::{ config::TransactionFetcherConfig, constants::{tx_fetcher::*, SOFT_LIMIT_COUNT_HASHES_IN_GET_POOLED_TRANSACTIONS_REQUEST}, - MessageFilter, PeerMetadata, PooledTransactions, - SOFT_LIMIT_BYTE_SIZE_POOLED_TRANSACTIONS_RESPONSE, + PeerMetadata, PooledTransactions, SOFT_LIMIT_BYTE_SIZE_POOLED_TRANSACTIONS_RESPONSE, }; use crate::{ cache::{LruCache, LruMap}, duration_metered_exec, metrics::TransactionFetcherMetrics, - transactions::{validation, PartiallyFilterMessage}, }; use alloy_consensus::transaction::PooledTransaction; use alloy_primitives::TxHash; @@ -43,7 +41,7 @@ use derive_more::{Constructor, Deref}; use futures::{stream::FuturesUnordered, Future, FutureExt, Stream, StreamExt}; use pin_project::pin_project; use reth_eth_wire::{ - DedupPayload, EthVersion, GetPooledTransactions, HandleMempoolData, HandleVersionedMempoolData, + DedupPayload, GetPooledTransactions, HandleMempoolData, HandleVersionedMempoolData, PartiallyValidData, RequestTxHashes, ValidAnnouncementData, }; use reth_eth_wire_types::{EthNetworkPrimitives, NetworkPrimitives}; @@ -60,7 +58,6 @@ use std::{ }; use tokio::sync::{mpsc::error::TrySendError, oneshot, oneshot::error::RecvError}; use tracing::trace; -use validation::FilterOutcome; /// The type responsible for fetching missing transactions from peers. /// @@ -85,8 +82,6 @@ pub struct TransactionFetcher { pub hashes_pending_fetch: LruCache, /// Tracks all hashes in the transaction fetcher. pub hashes_fetch_inflight_and_pending_fetch: LruMap, - /// Filter for valid announcement and response data. - pub(super) filter_valid_message: MessageFilter, /// Info on capacity of the transaction fetcher. pub info: TransactionFetcherInfo, #[doc(hidden)] @@ -151,13 +146,13 @@ impl TransactionFetcher { /// Removes the specified hashes from inflight tracking. #[inline] - pub fn remove_hashes_from_transaction_fetcher(&mut self, hashes: I) + pub fn remove_hashes_from_transaction_fetcher<'a, I>(&mut self, hashes: I) where - I: IntoIterator, + I: IntoIterator, { for hash in hashes { - self.hashes_fetch_inflight_and_pending_fetch.remove(&hash); - self.hashes_pending_fetch.remove(&hash); + self.hashes_fetch_inflight_and_pending_fetch.remove(hash); + self.hashes_pending_fetch.remove(hash); } } @@ -289,9 +284,7 @@ impl TransactionFetcher { // folds size based on expected response size and adds selected hashes to the request // list and the other hashes to the surplus list - loop { - let Some((hash, metadata)) = hashes_from_announcement_iter.next() else { break }; - + for (hash, metadata) in hashes_from_announcement_iter.by_ref() { let Some((_ty, size)) = metadata else { unreachable!("this method is called upon reception of an eth68 announcement") }; @@ -418,7 +411,6 @@ impl TransactionFetcher { if let (_, Some(evicted_hash)) = self.hashes_pending_fetch.insert_and_get_evicted(hash) { self.hashes_fetch_inflight_and_pending_fetch.remove(&evicted_hash); - self.hashes_pending_fetch.remove(&evicted_hash); } } } @@ -845,19 +837,6 @@ impl TransactionFetcher { } } - /// Returns the approx number of transactions that a [`GetPooledTransactions`] request will - /// have capacity for w.r.t. the given version of the protocol. - pub const fn approx_capacity_get_pooled_transactions_req( - &self, - announcement_version: EthVersion, - ) -> usize { - if announcement_version.is_eth68() { - approx_capacity_get_pooled_transactions_req_eth68(&self.info) - } else { - approx_capacity_get_pooled_transactions_req_eth66() - } - } - /// Processes a resolved [`GetPooledTransactions`] request. Queues the outcome as a /// [`FetchEvent`], which will then be streamed by /// [`TransactionsManager`](super::TransactionsManager). @@ -900,15 +879,19 @@ impl TransactionFetcher { if unsolicited > 0 { self.metrics.unsolicited_transactions.increment(unsolicited as u64); } - if verification_outcome == VerificationOutcome::ReportPeer { - // todo: report peer for sending hashes that weren't requested + + let report_peer = if verification_outcome == VerificationOutcome::ReportPeer { trace!(target: "net::tx", peer_id=format!("{peer_id:#}"), unverified_len, verified_payload_len=verified_payload.len(), "received `PooledTransactions` response from peer with entries that didn't verify against request, filtered out transactions" ); - } + true + } else { + false + }; + // peer has only sent hashes that we didn't request if verified_payload.is_empty() { return FetchEvent::FetchError { peer_id, error: RequestError::BadResponse } @@ -919,20 +902,19 @@ impl TransactionFetcher { // let unvalidated_payload_len = verified_payload.len(); - let (validation_outcome, valid_payload) = - self.filter_valid_message.partially_filter_valid_entries(verified_payload); + let valid_payload = verified_payload.dedup(); // todo: validate based on announced tx size/type and report peer for sending // invalid response . requires // passing the rlp encoded length down from active session along with the decoded // tx. - if validation_outcome == FilterOutcome::ReportPeer { + if valid_payload.len() != unvalidated_payload_len { trace!(target: "net::tx", - peer_id=format!("{peer_id:#}"), - unvalidated_payload_len, - valid_payload_len=valid_payload.len(), - "received invalid `PooledTransactions` response from peer, filtered out duplicate entries" + peer_id=format!("{peer_id:#}"), + unvalidated_payload_len, + valid_payload_len=valid_payload.len(), + "received `PooledTransactions` response from peer with duplicate entries, filtered them out" ); } // valid payload will have at least one transaction at this point. even if the tx @@ -971,7 +953,7 @@ impl TransactionFetcher { let transactions = valid_payload.into_data().into_values().collect(); - FetchEvent::TransactionsFetched { peer_id, transactions } + FetchEvent::TransactionsFetched { peer_id, transactions, report_peer } } Ok(Err(req_err)) => { self.try_buffer_hashes_for_retry(requested_hashes, &peer_id); @@ -1014,7 +996,6 @@ impl Default for TransactionFetcher { hashes_fetch_inflight_and_pending_fetch: LruMap::new( DEFAULT_MAX_CAPACITY_CACHE_INFLIGHT_AND_PENDING_FETCH, ), - filter_valid_message: Default::default(), info: TransactionFetcherInfo::default(), metrics: Default::default(), } @@ -1059,6 +1040,9 @@ pub enum FetchEvent { peer_id: PeerId, /// The transactions that were fetched, if available. transactions: PooledTransactions, + /// Whether the peer should be penalized for sending unsolicited transactions or for + /// misbehavior. + report_peer: bool, }, /// Triggered when there is an error in fetching transactions. FetchError { @@ -1305,6 +1289,7 @@ mod test { use alloy_primitives::{hex, B256}; use alloy_rlp::Decodable; use derive_more::IntoIterator; + use reth_eth_wire_types::EthVersion; use reth_ethereum_primitives::TransactionSigned; use std::{collections::HashSet, str::FromStr}; diff --git a/crates/net/network/src/transactions/mod.rs b/crates/net/network/src/transactions/mod.rs index cd741dce405..f4ef42523d5 100644 --- a/crates/net/network/src/transactions/mod.rs +++ b/crates/net/network/src/transactions/mod.rs @@ -1,23 +1,26 @@ //! Transactions management for the p2p network. +use alloy_consensus::transaction::TxHashRef; + /// Aggregation on configurable parameters for [`TransactionsManager`]. pub mod config; /// Default and spec'd bounds. pub mod constants; /// Component responsible for fetching transactions from [`NewPooledTransactionHashes`]. pub mod fetcher; -pub mod validation; +/// Defines the [`TransactionPolicies`] trait for aggregating transaction-related policies. +pub mod policy; pub use self::constants::{ tx_fetcher::DEFAULT_SOFT_LIMIT_BYTE_SIZE_POOLED_TRANSACTIONS_RESP_ON_PACK_GET_POOLED_TRANSACTIONS_REQ, SOFT_LIMIT_BYTE_SIZE_POOLED_TRANSACTIONS_RESPONSE, }; -use config::TransactionPropagationKind; +use config::{AnnouncementAcceptance, StrictEthAnnouncementFilter, TransactionPropagationKind}; pub use config::{ - TransactionFetcherConfig, TransactionPropagationMode, TransactionPropagationPolicy, - TransactionsManagerConfig, + AnnouncementFilteringPolicy, TransactionFetcherConfig, TransactionPropagationMode, + TransactionPropagationPolicy, TransactionsManagerConfig, }; -pub use validation::*; +use policy::{NetworkPolicies, TransactionPolicies}; pub(crate) use fetcher::{FetchEvent, TransactionFetcher}; @@ -25,13 +28,14 @@ use self::constants::{tx_manager::*, DEFAULT_SOFT_LIMIT_BYTE_SIZE_TRANSACTIONS_B use crate::{ budget::{ DEFAULT_BUDGET_TRY_DRAIN_NETWORK_TRANSACTION_EVENTS, - DEFAULT_BUDGET_TRY_DRAIN_PENDING_POOL_IMPORTS, DEFAULT_BUDGET_TRY_DRAIN_POOL_IMPORTS, - DEFAULT_BUDGET_TRY_DRAIN_STREAM, + DEFAULT_BUDGET_TRY_DRAIN_PENDING_POOL_IMPORTS, DEFAULT_BUDGET_TRY_DRAIN_STREAM, }, cache::LruCache, duration_metered_exec, metered_poll_nested_stream_with_budget, - metrics::{TransactionsManagerMetrics, NETWORK_POOL_TRANSACTIONS_SCOPE}, - NetworkHandle, + metrics::{ + AnnouncedTxTypesMetrics, TransactionsManagerMetrics, NETWORK_POOL_TRANSACTIONS_SCOPE, + }, + NetworkHandle, TxTypesCounter, }; use alloy_primitives::{TxHash, B256}; use constants::SOFT_LIMIT_COUNT_HASHES_IN_NEW_POOLED_TRANSACTIONS_BROADCAST_MESSAGE; @@ -40,9 +44,9 @@ use reth_eth_wire::{ DedupPayload, EthNetworkPrimitives, EthVersion, GetPooledTransactions, HandleMempoolData, HandleVersionedMempoolData, NetworkPrimitives, NewPooledTransactionHashes, NewPooledTransactionHashes66, NewPooledTransactionHashes68, PooledTransactions, - RequestTxHashes, Transactions, + RequestTxHashes, Transactions, ValidAnnouncementData, }; -use reth_ethereum_primitives::TransactionSigned; +use reth_ethereum_primitives::{TransactionSigned, TxType}; use reth_metrics::common::mpsc::UnboundedMeteredReceiver; use reth_network_api::{ events::{PeerEvent, SessionInfo}, @@ -58,8 +62,8 @@ use reth_primitives_traits::SignedTransaction; use reth_tokio_util::EventStream; use reth_transaction_pool::{ error::{PoolError, PoolResult}, - GetPooledTransactionLimit, PoolTransaction, PropagateKind, PropagatedTransactions, - TransactionPool, ValidPoolTransaction, + AddedTransactionOutcome, GetPooledTransactionLimit, PoolTransaction, PropagateKind, + PropagatedTransactions, TransactionPool, ValidPoolTransaction, }; use std::{ collections::{hash_map::Entry, HashMap, HashSet}, @@ -72,13 +76,14 @@ use std::{ time::{Duration, Instant}, }; use tokio::sync::{mpsc, oneshot, oneshot::error::RecvError}; -use tokio_stream::wrappers::{ReceiverStream, UnboundedReceiverStream}; +use tokio_stream::wrappers::UnboundedReceiverStream; use tracing::{debug, trace}; /// The future for importing transactions into the pool. /// /// Resolves with the result of each transaction import. -pub type PoolImportFuture = Pin>> + Send + 'static>>; +pub type PoolImportFuture = + Pin>> + Send + 'static>>; /// Api to interact with [`TransactionsManager`] task. /// @@ -236,12 +241,52 @@ impl TransactionsHandle { /// /// It is directly connected to the [`TransactionPool`] to retrieve requested transactions and /// propagate new transactions over the network. +/// +/// It can be configured with different policies for transaction propagation and announcement +/// filtering. See [`NetworkPolicies`] and [`TransactionPolicies`] for more details. +/// +/// ## Network Transaction Processing +/// +/// ### Message Types +/// +/// - **`Transactions`**: Full transaction broadcasts (rejects blob transactions) +/// - **`NewPooledTransactionHashes`**: Hash announcements +/// +/// ### Peer Tracking +/// +/// - Maintains per-peer transaction cache (default: 10,240 entries) +/// - Prevents duplicate imports and enables efficient propagation +/// +/// ### Bad Transaction Handling +/// +/// Caches and rejects transactions with consensus violations (gas, signature, chain ID). +/// Penalizes peers sending invalid transactions. +/// +/// ### Import Management +/// +/// Limits concurrent pool imports and backs off when approaching capacity. +/// +/// ### Transaction Fetching +/// +/// For announced transactions: filters known → queues unknown → fetches → imports +/// +/// ### Propagation Rules +/// +/// Based on: origin (Local/External/Private), peer capabilities, and network state. +/// Disabled during initial sync. +/// +/// ### Security +/// +/// Rate limiting via reputation, bad transaction isolation, peer scoring. #[derive(Debug)] #[must_use = "Manager does nothing unless polled."] pub struct TransactionsManager< Pool, N: NetworkPrimitives = EthNetworkPrimitives, - P: TransactionPropagationPolicy = TransactionPropagationKind, + PBundle: TransactionPolicies = NetworkPolicies< + TransactionPropagationKind, + StrictEthAnnouncementFilter, + >, > { /// Access to the transaction pool. pool: Pool, @@ -293,18 +338,26 @@ pub struct TransactionsManager< /// - no nonce gaps /// - all dynamic fee requirements are (currently) met /// - account has enough balance to cover the transaction's gas - pending_transactions: ReceiverStream, + pending_transactions: mpsc::Receiver, /// Incoming events from the [`NetworkManager`](crate::NetworkManager). transaction_events: UnboundedMeteredReceiver>, /// How the `TransactionsManager` is configured. config: TransactionsManagerConfig, - /// The policy to use when propagating transactions. - propagation_policy: P, + /// Network Policies + policies: PBundle, /// `TransactionsManager` metrics metrics: TransactionsManagerMetrics, + /// `AnnouncedTxTypes` metrics + announced_tx_types_metrics: AnnouncedTxTypesMetrics, } -impl TransactionsManager { +impl + TransactionsManager< + Pool, + N, + NetworkPolicies, + > +{ /// Sets up a new instance. /// /// Note: This expects an existing [`NetworkManager`](crate::NetworkManager) instance. @@ -319,13 +372,13 @@ impl TransactionsManager { pool, from_network, transactions_manager_config, - TransactionPropagationKind::default(), + NetworkPolicies::default(), ) } } -impl - TransactionsManager +impl + TransactionsManager { /// Sets up a new instance with given the settings. /// @@ -335,7 +388,7 @@ impl>, transactions_manager_config: TransactionsManagerConfig, - propagation_policy: P, + policies: PBundle, ) -> Self { let network_events = network.event_listener(); @@ -368,14 +421,15 @@ impl TransactionsManager -where - Pool: TransactionPool, - N: NetworkPrimitives, - Policy: TransactionPropagationPolicy, +impl + TransactionsManager { /// Processes a batch import results. - fn on_batch_import_result(&mut self, batch_results: Vec>) { + fn on_batch_import_result(&mut self, batch_results: Vec>) { for res in batch_results { match res { - Ok(hash) => { + Ok(AddedTransactionOutcome { hash, .. }) => { self.on_good_import(hash); } Err(err) => { @@ -579,10 +630,15 @@ where } // 1. filter out spam - let (validation_outcome, mut partially_valid_msg) = - self.transaction_fetcher.filter_valid_message.partially_filter_valid_entries(msg); + if msg.is_empty() { + self.report_peer(peer_id, ReputationChangeKind::BadAnnouncement); + return; + } - if validation_outcome == FilterOutcome::ReportPeer { + let original_len = msg.len(); + let mut partially_valid_msg = msg.dedup(); + + if partially_valid_msg.len() != original_len { self.report_peer(peer_id, ReputationChangeKind::BadAnnouncement); } @@ -615,26 +671,66 @@ where // // validates messages with respect to the given network, e.g. allowed tx types // - let (validation_outcome, mut valid_announcement_data) = if partially_valid_msg + let mut should_report_peer = false; + let mut tx_types_counter = TxTypesCounter::default(); + + let is_eth68_message = partially_valid_msg .msg_version() - .expect("partially valid announcement should have version") - .is_eth68() - { - // validate eth68 announcement data - self.transaction_fetcher - .filter_valid_message - .filter_valid_entries_68(partially_valid_msg) - } else { - // validate eth66 announcement data - self.transaction_fetcher - .filter_valid_message - .filter_valid_entries_66(partially_valid_msg) - }; + .expect("partially valid announcement should have a version") + .is_eth68(); + + partially_valid_msg.retain(|tx_hash, metadata_ref_mut| { + let (ty_byte, size_val) = match *metadata_ref_mut { + Some((ty, size)) => { + if !is_eth68_message { + should_report_peer = true; + } + (ty, size) + } + None => { + if is_eth68_message { + should_report_peer = true; + return false; + } + (0u8, 0) + } + }; + + if is_eth68_message && + let Some((actual_ty_byte, _)) = *metadata_ref_mut && + let Ok(parsed_tx_type) = TxType::try_from(actual_ty_byte) + { + tx_types_counter.increase_by_tx_type(parsed_tx_type); + } + + let decision = self + .policies + .announcement_filter() + .decide_on_announcement(ty_byte, tx_hash, size_val); + + match decision { + AnnouncementAcceptance::Accept => true, + AnnouncementAcceptance::Ignore => false, + AnnouncementAcceptance::Reject { penalize_peer } => { + if penalize_peer { + should_report_peer = true; + } + false + } + } + }); + + if is_eth68_message { + self.announced_tx_types_metrics.update_eth68_announcement_metrics(tx_types_counter); + } - if validation_outcome == FilterOutcome::ReportPeer { + if should_report_peer { self.report_peer(peer_id, ReputationChangeKind::BadAnnouncement); } + let mut valid_announcement_data = + ValidAnnouncementData::from_partially_valid_data(partially_valid_msg); + if valid_announcement_data.is_empty() { // no valid announcement data return @@ -661,7 +757,7 @@ where trace!(target: "net::tx::propagation", peer_id=format!("{peer_id:#}"), - hashes_len=valid_announcement_data.iter().count(), + hashes_len=valid_announcement_data.len(), hashes=?valid_announcement_data.keys().collect::>(), msg_version=%valid_announcement_data.msg_version(), client_version=%client, @@ -732,16 +828,18 @@ where } } -impl TransactionsManager +impl TransactionsManager where - Pool: TransactionPool + 'static, + Pool: TransactionPool + Unpin + 'static, + N: NetworkPrimitives< - BroadcastedTransaction: SignedTransaction, - PooledTransaction: SignedTransaction, - >, + BroadcastedTransaction: SignedTransaction, + PooledTransaction: SignedTransaction, + > + Unpin, + + PBundle: TransactionPolicies, Pool::Transaction: PoolTransaction, - Policy: TransactionPropagationPolicy, { /// Invoked when transactions in the local mempool are considered __pending__. /// @@ -927,7 +1025,7 @@ where // Note: Assuming ~random~ order due to random state of the peers map hasher for (peer_idx, (peer_id, peer)) in self.peers.iter_mut().enumerate() { - if !self.propagation_policy.can_propagate(peer) { + if !self.policies.propagation_policy().can_propagate(peer) { // skip peers we should not propagate to continue } @@ -1009,6 +1107,10 @@ where /// This fetches all transaction from the pool, including the 4844 blob transactions but /// __without__ their sidecar, because 4844 transactions are only ever announced as hashes. fn propagate_all(&mut self, hashes: Vec) { + if self.peers.is_empty() { + // nothing to propagate + return + } let propagated = self.propagate_transactions( self.pool.get_all(hashes).into_iter().map(PropagateTransaction::pool_tx).collect(), PropagationMode::Basic, @@ -1069,7 +1171,8 @@ where } TransactionsCommand::PropagateTransactions(txs) => self.propagate_all(txs), TransactionsCommand::BroadcastTransactions(txs) => { - self.propagate_transactions(txs, PropagationMode::Forced); + let propagated = self.propagate_transactions(txs, PropagationMode::Forced); + self.pool.on_propagated(propagated); } TransactionsCommand::GetTransactionHashes { peers, tx } => { let mut res = HashMap::with_capacity(peers.len()); @@ -1116,7 +1219,7 @@ where Entry::Vacant(entry) => entry.insert(peer), }; - self.propagation_policy.on_session_established(peer); + self.policies.propagation_policy_mut().on_session_established(peer); // Send a `NewPooledTransactionHashes` to the peer with up to // `SOFT_LIMIT_COUNT_HASHES_IN_NEW_POOLED_TRANSACTIONS_BROADCAST_MESSAGE` @@ -1155,7 +1258,7 @@ where let peer = self.peers.remove(&peer_id); if let Some(mut peer) = peer { - self.propagation_policy.on_session_closed(&mut peer); + self.policies.propagation_policy_mut().on_session_closed(&mut peer); } self.transaction_fetcher.remove_peer(&peer_id); } @@ -1234,7 +1337,7 @@ where // mark the transactions as received self.transaction_fetcher - .remove_hashes_from_transaction_fetcher(transactions.iter().map(|tx| *tx.tx_hash())); + .remove_hashes_from_transaction_fetcher(transactions.iter().map(|tx| tx.tx_hash())); // track that the peer knows these transaction, but only if this is a new broadcast. // If we received the transactions as the response to our `GetPooledTransactions`` @@ -1260,91 +1363,89 @@ where // tracks the quality of the given transactions let mut has_bad_transactions = false; - // 2. filter out transactions that are invalid or already pending import - if let Some(peer) = self.peers.get_mut(&peer_id) { - // pre-size to avoid reallocations - let mut new_txs = Vec::with_capacity(transactions.len()); - for tx in transactions { - // recover transaction - let tx = match tx.try_into_recovered() { - Ok(tx) => tx, - Err(badtx) => { + // 2. filter out transactions that are invalid or already pending import pre-size to avoid + // reallocations + let mut new_txs = Vec::with_capacity(transactions.len()); + for tx in transactions { + // recover transaction + let tx = match tx.try_into_recovered() { + Ok(tx) => tx, + Err(badtx) => { + trace!(target: "net::tx", + peer_id=format!("{peer_id:#}"), + hash=%badtx.tx_hash(), + client_version=%peer.client_version, + "failed ecrecovery for transaction" + ); + has_bad_transactions = true; + continue + } + }; + + match self.transactions_by_peers.entry(*tx.tx_hash()) { + Entry::Occupied(mut entry) => { + // transaction was already inserted + entry.get_mut().insert(peer_id); + } + Entry::Vacant(entry) => { + if self.bad_imports.contains(tx.tx_hash()) { trace!(target: "net::tx", peer_id=format!("{peer_id:#}"), - hash=%badtx.tx_hash(), + hash=%tx.tx_hash(), client_version=%peer.client_version, - "failed ecrecovery for transaction" + "received a known bad transaction from peer" ); has_bad_transactions = true; - continue - } - }; + } else { + // this is a new transaction that should be imported into the pool - match self.transactions_by_peers.entry(*tx.tx_hash()) { - Entry::Occupied(mut entry) => { - // transaction was already inserted - entry.get_mut().insert(peer_id); - } - Entry::Vacant(entry) => { - if self.bad_imports.contains(tx.tx_hash()) { - trace!(target: "net::tx", - peer_id=format!("{peer_id:#}"), - hash=%tx.tx_hash(), - client_version=%peer.client_version, - "received a known bad transaction from peer" - ); - has_bad_transactions = true; - } else { - // this is a new transaction that should be imported into the pool - - let pool_transaction = Pool::Transaction::from_pooled(tx); - new_txs.push(pool_transaction); - - entry.insert(HashSet::from([peer_id])); - } + let pool_transaction = Pool::Transaction::from_pooled(tx); + new_txs.push(pool_transaction); + + entry.insert(HashSet::from([peer_id])); } } } - new_txs.shrink_to_fit(); + } + new_txs.shrink_to_fit(); - // 3. import new transactions as a batch to minimize lock contention on the underlying - // pool - if !new_txs.is_empty() { - let pool = self.pool.clone(); - // update metrics - let metric_pending_pool_imports = self.metrics.pending_pool_imports.clone(); - metric_pending_pool_imports.increment(new_txs.len() as f64); + // 3. import new transactions as a batch to minimize lock contention on the underlying + // pool + if !new_txs.is_empty() { + let pool = self.pool.clone(); + // update metrics + let metric_pending_pool_imports = self.metrics.pending_pool_imports.clone(); + metric_pending_pool_imports.increment(new_txs.len() as f64); + + // update self-monitoring info + self.pending_pool_imports_info + .pending_pool_imports + .fetch_add(new_txs.len(), Ordering::Relaxed); + let tx_manager_info_pending_pool_imports = + self.pending_pool_imports_info.pending_pool_imports.clone(); + + trace!(target: "net::tx::propagation", new_txs_len=?new_txs.len(), "Importing new transactions"); + let import = Box::pin(async move { + let added = new_txs.len(); + let res = pool.add_external_transactions(new_txs).await; + // update metrics + metric_pending_pool_imports.decrement(added as f64); // update self-monitoring info - self.pending_pool_imports_info - .pending_pool_imports - .fetch_add(new_txs.len(), Ordering::Relaxed); - let tx_manager_info_pending_pool_imports = - self.pending_pool_imports_info.pending_pool_imports.clone(); - - trace!(target: "net::tx::propagation", new_txs_len=?new_txs.len(), "Importing new transactions"); - let import = Box::pin(async move { - let added = new_txs.len(); - let res = pool.add_external_transactions(new_txs).await; - - // update metrics - metric_pending_pool_imports.decrement(added as f64); - // update self-monitoring info - tx_manager_info_pending_pool_imports.fetch_sub(added, Ordering::Relaxed); - - res - }); - - self.pool_imports.push(import); - } + tx_manager_info_pending_pool_imports.fetch_sub(added, Ordering::Relaxed); - if num_already_seen_by_peer > 0 { - self.metrics.messages_with_transactions_already_seen_by_peer.increment(1); - self.metrics - .occurrences_of_transaction_already_seen_by_peer - .increment(num_already_seen_by_peer); - trace!(target: "net::tx", num_txs=%num_already_seen_by_peer, ?peer_id, client=?peer.client_version, "Peer sent already seen transactions"); - } + res + }); + + self.pool_imports.push(import); + } + + if num_already_seen_by_peer > 0 { + self.metrics.messages_with_transactions_already_seen_by_peer.increment(1); + self.metrics + .occurrences_of_transaction_already_seen_by_peer + .increment(num_already_seen_by_peer); + trace!(target: "net::tx", num_txs=%num_already_seen_by_peer, ?peer_id, client=?peer.client_version, "Peer sent already seen transactions"); } if has_bad_transactions { @@ -1360,8 +1461,11 @@ where /// Processes a [`FetchEvent`]. fn on_fetch_event(&mut self, fetch_event: FetchEvent) { match fetch_event { - FetchEvent::TransactionsFetched { peer_id, transactions } => { + FetchEvent::TransactionsFetched { peer_id, transactions, report_peer } => { self.import_transactions(peer_id, transactions, TransactionSource::Response); + if report_peer { + self.report_peer(peer_id, ReputationChangeKind::BadTransactions); + } } FetchEvent::FetchError { peer_id, error } => { trace!(target: "net::tx", ?peer_id, %error, "requesting transactions from peer failed"); @@ -1381,16 +1485,17 @@ where // // spawned in `NodeConfig::start_network`(reth_node_core::NodeConfig) and // `NetworkConfig::start_network`(reth_network::NetworkConfig) -impl Future for TransactionsManager +impl< + Pool: TransactionPool + Unpin + 'static, + N: NetworkPrimitives< + BroadcastedTransaction: SignedTransaction, + PooledTransaction: SignedTransaction, + > + Unpin, + PBundle: TransactionPolicies + Unpin, + > Future for TransactionsManager where - Pool: TransactionPool + Unpin + 'static, - N: NetworkPrimitives< - BroadcastedTransaction: SignedTransaction, - PooledTransaction: SignedTransaction, - >, Pool::Transaction: PoolTransaction, - Policy: TransactionPropagationPolicy, { type Output = (); @@ -1423,14 +1528,16 @@ where // We don't expect this buffer to be large, since only pending transactions are // emitted here. let mut new_txs = Vec::new(); - let maybe_more_pending_txns = metered_poll_nested_stream_with_budget!( - poll_durations.acc_imported_txns, - "net::tx", - "Pending transactions stream", - DEFAULT_BUDGET_TRY_DRAIN_POOL_IMPORTS, - this.pending_transactions.poll_next_unpin(cx), - |hash| new_txs.push(hash) - ); + let maybe_more_pending_txns = match this.pending_transactions.poll_recv_many( + cx, + &mut new_txs, + SOFT_LIMIT_COUNT_HASHES_IN_NEW_POOLED_TRANSACTIONS_BROADCAST_MESSAGE, + ) { + Poll::Ready(count) => { + count == SOFT_LIMIT_COUNT_HASHES_IN_NEW_POOLED_TRANSACTIONS_BROADCAST_MESSAGE + } + Poll::Pending => false, + }; if !new_txs.is_empty() { this.on_new_pending_transactions(new_txs); } @@ -1993,14 +2100,15 @@ mod tests { transactions::{buffer_hash_to_tx_fetcher, new_mock_session, new_tx_manager}, Testnet, }, + transactions::config::RelaxedEthAnnouncementFilter, NetworkConfigBuilder, NetworkManager, }; - use alloy_consensus::{transaction::PooledTransaction, TxEip1559, TxLegacy}; + use alloy_consensus::{TxEip1559, TxLegacy}; use alloy_primitives::{hex, Signature, TxKind, U256}; use alloy_rlp::Decodable; use futures::FutureExt; use reth_chainspec::MIN_TRANSACTION_GAS; - use reth_ethereum_primitives::{Transaction, TransactionSigned}; + use reth_ethereum_primitives::{PooledTransactionVariant, Transaction, TransactionSigned}; use reth_network_api::{NetworkInfo, PeerKind}; use reth_network_p2p::{ error::{RequestError, RequestResult}, @@ -2008,7 +2116,7 @@ mod tests { }; use reth_storage_api::noop::NoopProvider; use reth_transaction_pool::test_utils::{ - testing_pool, MockTransaction, MockTransactionFactory, + testing_pool, MockTransaction, MockTransactionFactory, TestPool, }; use secp256k1::SecretKey; use std::{ @@ -2236,10 +2344,10 @@ mod tests { let PeerRequest::GetPooledTransactions { request, response } = req else { unreachable!() }; assert_eq!(request, GetPooledTransactions::from(txs_hashes.clone())); - let message: Vec = txs + let message: Vec = txs .into_iter() .map(|tx| { - PooledTransaction::try_from(tx) + PooledTransactionVariant::try_from(tx) .expect("Failed to convert MockTransaction to PooledTransaction") }) .collect(); @@ -2399,7 +2507,8 @@ mod tests { let request = GetPooledTransactions(vec![*tx.get_hash()]); - let (send, receive) = oneshot::channel::>(); + let (send, receive) = + oneshot::channel::>>(); transactions.on_network_tx_event(NetworkTransactionEvent::GetPooledTransactions { peer_id: *handle1.peer_id(), @@ -2510,11 +2619,11 @@ mod tests { .expect("peer_1 session should receive request with buffered hashes"); let PeerRequest::GetPooledTransactions { response, .. } = req else { unreachable!() }; - let message: Vec = txs + let message: Vec = txs .into_iter() .take(1) .map(|tx| { - PooledTransaction::try_from(tx) + PooledTransactionVariant::try_from(tx) .expect("Failed to convert MockTransaction to PooledTransaction") }) .collect(); @@ -2759,4 +2868,114 @@ mod tests { let propagated = tx_manager.propagate_transactions(propagate, PropagationMode::Basic); assert!(propagated.0.is_empty()); } + + #[tokio::test] + async fn test_relaxed_filter_ignores_unknown_tx_types() { + reth_tracing::init_test_tracing(); + + let transactions_manager_config = TransactionsManagerConfig::default(); + + let propagation_policy = TransactionPropagationKind::default(); + let announcement_policy = RelaxedEthAnnouncementFilter::default(); + + let policy_bundle = NetworkPolicies::new(propagation_policy, announcement_policy); + + let pool = testing_pool(); + let secret_key = SecretKey::new(&mut rand_08::thread_rng()); + let client = NoopProvider::default(); + + let network_config = NetworkConfigBuilder::new(secret_key) + .listener_port(0) + .disable_discovery() + .build(client.clone()); + + let mut network_manager = NetworkManager::new(network_config).await.unwrap(); + let (to_tx_manager_tx, from_network_rx) = + mpsc::unbounded_channel::>(); + network_manager.set_transactions(to_tx_manager_tx); + let network_handle = network_manager.handle().clone(); + let network_service_handle = tokio::spawn(network_manager); + + let mut tx_manager = TransactionsManager::< + TestPool, + EthNetworkPrimitives, + NetworkPolicies, + >::with_policy( + network_handle.clone(), + pool.clone(), + from_network_rx, + transactions_manager_config, + policy_bundle, + ); + + let peer_id = PeerId::random(); + let eth_version = EthVersion::Eth68; + let (mock_peer_metadata, mut mock_session_rx) = new_mock_session(peer_id, eth_version); + tx_manager.peers.insert(peer_id, mock_peer_metadata); + + let mut tx_factory = MockTransactionFactory::default(); + + let valid_known_tx = tx_factory.create_eip1559(); + let known_tx_signed: Arc> = Arc::new(valid_known_tx); + + let known_tx_hash = *known_tx_signed.hash(); + let known_tx_type_byte = known_tx_signed.transaction.tx_type(); + let known_tx_size = known_tx_signed.encoded_length(); + + let unknown_tx_hash = B256::random(); + let unknown_tx_type_byte = 0xff_u8; + let unknown_tx_size = 150; + + let announcement_msg = NewPooledTransactionHashes::Eth68(NewPooledTransactionHashes68 { + types: vec![known_tx_type_byte, unknown_tx_type_byte], + sizes: vec![known_tx_size, unknown_tx_size], + hashes: vec![known_tx_hash, unknown_tx_hash], + }); + + tx_manager.on_new_pooled_transaction_hashes(peer_id, announcement_msg); + + poll_fn(|cx| { + let _ = tx_manager.poll_unpin(cx); + Poll::Ready(()) + }) + .await; + + let mut requested_hashes_in_getpooled = HashSet::new(); + let mut unexpected_request_received = false; + + match tokio::time::timeout(std::time::Duration::from_millis(200), mock_session_rx.recv()) + .await + { + Ok(Some(PeerRequest::GetPooledTransactions { request, response: tx_response_ch })) => { + let GetPooledTransactions(hashes) = request; + for hash in hashes { + requested_hashes_in_getpooled.insert(hash); + } + let _ = tx_response_ch.send(Ok(PooledTransactions(vec![]))); + } + Ok(Some(other_request)) => { + tracing::error!(?other_request, "Received unexpected PeerRequest type"); + unexpected_request_received = true; + } + Ok(None) => tracing::info!("Mock session channel closed or no request received."), + Err(_timeout_err) => { + tracing::info!("Timeout: No GetPooledTransactions request received.") + } + } + + assert!( + requested_hashes_in_getpooled.contains(&known_tx_hash), + "Should have requested the known EIP-1559 transaction. Requested: {requested_hashes_in_getpooled:?}" + ); + assert!( + !requested_hashes_in_getpooled.contains(&unknown_tx_hash), + "Should NOT have requested the unknown transaction type. Requested: {requested_hashes_in_getpooled:?}" + ); + assert!( + !unexpected_request_received, + "An unexpected P2P request was received by the mock peer." + ); + + network_service_handle.abort(); + } } diff --git a/crates/net/network/src/transactions/policy.rs b/crates/net/network/src/transactions/policy.rs new file mode 100644 index 00000000000..c25b9d9b414 --- /dev/null +++ b/crates/net/network/src/transactions/policy.rs @@ -0,0 +1,78 @@ +use crate::transactions::config::{AnnouncementFilteringPolicy, TransactionPropagationPolicy}; +use std::fmt::Debug; + +/// A bundle of policies that control the behavior of network components like +/// the [`TransactionsManager`](super::TransactionsManager). +/// +/// This trait allows for different collections of policies to be used interchangeably. +pub trait TransactionPolicies: Send + Sync + Debug + 'static { + /// The type of the policy used for transaction propagation. + type Propagation: TransactionPropagationPolicy; + /// The type of the policy used for filtering transaction announcements. + type Announcement: AnnouncementFilteringPolicy; + + /// Returns a reference to the transaction propagation policy. + fn propagation_policy(&self) -> &Self::Propagation; + + /// Returns a mutable reference to the transaction propagation policy. + fn propagation_policy_mut(&mut self) -> &mut Self::Propagation; + + /// Returns a reference to the announcement filtering policy. + fn announcement_filter(&self) -> &Self::Announcement; +} + +/// A container that bundles specific implementations of transaction-related policies, +/// +/// This struct implements the [`TransactionPolicies`] trait, providing a complete set of +/// policies required by components like the [`TransactionsManager`](super::TransactionsManager). +/// It holds a specific [`TransactionPropagationPolicy`] and an +/// [`AnnouncementFilteringPolicy`]. +#[derive(Debug, Clone, Default)] +pub struct NetworkPolicies { + propagation: P, + announcement: A, +} + +impl NetworkPolicies { + /// Creates a new bundle of network policies. + pub const fn new(propagation: P, announcement: A) -> Self { + Self { propagation, announcement } + } + + /// Returns a new `NetworkPolicies` bundle with the `TransactionPropagationPolicy` replaced. + pub fn with_propagation(self, new_propagation: NewP) -> NetworkPolicies + where + NewP: TransactionPropagationPolicy, + { + NetworkPolicies::new(new_propagation, self.announcement) + } + + /// Returns a new `NetworkPolicies` bundle with the `AnnouncementFilteringPolicy` replaced. + pub fn with_announcement(self, new_announcement: NewA) -> NetworkPolicies + where + NewA: AnnouncementFilteringPolicy, + { + NetworkPolicies::new(self.propagation, new_announcement) + } +} + +impl TransactionPolicies for NetworkPolicies +where + P: TransactionPropagationPolicy + Debug, + A: AnnouncementFilteringPolicy + Debug, +{ + type Propagation = P; + type Announcement = A; + + fn propagation_policy(&self) -> &Self::Propagation { + &self.propagation + } + + fn propagation_policy_mut(&mut self) -> &mut Self::Propagation { + &mut self.propagation + } + + fn announcement_filter(&self) -> &Self::Announcement { + &self.announcement + } +} diff --git a/crates/net/network/src/transactions/validation.rs b/crates/net/network/src/transactions/validation.rs deleted file mode 100644 index cf91ce69a7e..00000000000 --- a/crates/net/network/src/transactions/validation.rs +++ /dev/null @@ -1,461 +0,0 @@ -//! Validation of [`NewPooledTransactionHashes66`](reth_eth_wire::NewPooledTransactionHashes66) -//! and [`NewPooledTransactionHashes68`](reth_eth_wire::NewPooledTransactionHashes68) -//! announcements. Validation and filtering of announcements is network dependent. - -use crate::metrics::{AnnouncedTxTypesMetrics, TxTypesCounter}; -use alloy_primitives::Signature; -use derive_more::{Deref, DerefMut}; -use reth_eth_wire::{ - DedupPayload, Eth68TxMetadata, HandleMempoolData, PartiallyValidData, ValidAnnouncementData, -}; -use reth_ethereum_primitives::TxType; -use std::{fmt, fmt::Display, mem}; -use tracing::trace; - -/// The size of a decoded signature in bytes. -pub const SIGNATURE_DECODED_SIZE_BYTES: usize = mem::size_of::(); - -/// Outcomes from validating a `(ty, hash, size)` entry from a -/// [`NewPooledTransactionHashes68`](reth_eth_wire::NewPooledTransactionHashes68). Signals to the -/// caller how to deal with an announcement entry and the peer who sent the announcement. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum ValidationOutcome { - /// Tells the caller to keep the entry in the announcement for fetch. - Fetch, - /// Tells the caller to filter out the entry from the announcement. - Ignore, - /// Tells the caller to filter out the entry from the announcement and penalize the peer. On - /// this outcome, caller can drop the announcement, that is up to each implementation. - ReportPeer, -} - -/// Generic filter for announcements and responses. Checks for empty message and unique hashes/ -/// transactions in message. -pub trait PartiallyFilterMessage { - /// Removes duplicate entries from a mempool message. Returns [`FilterOutcome::ReportPeer`] if - /// the caller should penalize the peer, otherwise [`FilterOutcome::Ok`]. - fn partially_filter_valid_entries( - &self, - msg: impl DedupPayload + fmt::Debug, - ) -> (FilterOutcome, PartiallyValidData) { - // 1. checks if the announcement is empty - if msg.is_empty() { - trace!(target: "net::tx", - msg=?msg, - "empty payload" - ); - return (FilterOutcome::ReportPeer, PartiallyValidData::empty_eth66()) - } - - // 2. checks if announcement is spam packed with duplicate hashes - let original_len = msg.len(); - let partially_valid_data = msg.dedup(); - - ( - if partially_valid_data.len() == original_len { - FilterOutcome::Ok - } else { - FilterOutcome::ReportPeer - }, - partially_valid_data, - ) - } -} - -/// Filters valid entries in -/// [`NewPooledTransactionHashes68`](reth_eth_wire::NewPooledTransactionHashes68) and -/// [`NewPooledTransactionHashes66`](reth_eth_wire::NewPooledTransactionHashes66) in place, and -/// flags misbehaving peers. -pub trait FilterAnnouncement { - /// Removes invalid entries from a - /// [`NewPooledTransactionHashes68`](reth_eth_wire::NewPooledTransactionHashes68) announcement. - /// Returns [`FilterOutcome::ReportPeer`] if the caller should penalize the peer, otherwise - /// [`FilterOutcome::Ok`]. - fn filter_valid_entries_68( - &self, - msg: PartiallyValidData, - ) -> (FilterOutcome, ValidAnnouncementData); - - /// Removes invalid entries from a - /// [`NewPooledTransactionHashes66`](reth_eth_wire::NewPooledTransactionHashes66) announcement. - /// Returns [`FilterOutcome::ReportPeer`] if the caller should penalize the peer, otherwise - /// [`FilterOutcome::Ok`]. - fn filter_valid_entries_66( - &self, - msg: PartiallyValidData, - ) -> (FilterOutcome, ValidAnnouncementData); -} - -/// Outcome from filtering -/// [`NewPooledTransactionHashes68`](reth_eth_wire::NewPooledTransactionHashes68). Signals to caller -/// whether to penalize the sender of the announcement or not. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum FilterOutcome { - /// Peer behaves appropriately. - Ok, - /// A penalty should be flagged for the peer. Peer sent an announcement with unacceptably - /// invalid entries. - ReportPeer, -} - -/// Wrapper for types that implement [`FilterAnnouncement`]. The definition of a valid -/// announcement is network dependent. For example, different networks support different -/// [`TxType`]s, and different [`TxType`]s have different transaction size constraints. Defaults to -/// [`EthMessageFilter`]. -#[derive(Debug, Default, Deref, DerefMut)] -pub struct MessageFilter(N); - -/// Filter for announcements containing EIP [`TxType`]s. -#[derive(Debug, Default)] -pub struct EthMessageFilter { - announced_tx_types_metrics: AnnouncedTxTypesMetrics, -} - -impl Display for EthMessageFilter { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "EthMessageFilter") - } -} - -impl PartiallyFilterMessage for EthMessageFilter {} - -impl FilterAnnouncement for EthMessageFilter { - fn filter_valid_entries_68( - &self, - mut msg: PartiallyValidData, - ) -> (FilterOutcome, ValidAnnouncementData) { - trace!(target: "net::tx::validation", - msg=?*msg, - network=%self, - "validating eth68 announcement data.." - ); - - let mut should_report_peer = false; - let mut tx_types_counter = TxTypesCounter::default(); - - // checks if eth68 announcement metadata is valid - // - // transactions that are filtered out here, may not be spam, rather from benevolent peers - // that are unknowingly sending announcements with invalid data. - // - msg.retain(|hash, metadata| { - debug_assert!( - metadata.is_some(), - "metadata should exist for `%hash` in eth68 announcement passed to `%filter_valid_entries_68`, -`%hash`: {hash}" - ); - - let Some((ty, size)) = metadata else { - return false - }; - - // - // checks if tx type is valid value for this network - // - let tx_type = match TxType::try_from(*ty) { - Ok(ty) => ty, - Err(_) => { - trace!(target: "net::eth-wire", - ty=ty, - size=size, - hash=%hash, - network=%self, - "invalid tx type in eth68 announcement" - ); - - should_report_peer = true; - return false; - } - - }; - tx_types_counter.increase_by_tx_type(tx_type); - - true - }); - self.announced_tx_types_metrics.update_eth68_announcement_metrics(tx_types_counter); - ( - if should_report_peer { FilterOutcome::ReportPeer } else { FilterOutcome::Ok }, - ValidAnnouncementData::from_partially_valid_data(msg), - ) - } - - fn filter_valid_entries_66( - &self, - partially_valid_data: PartiallyValidData>, - ) -> (FilterOutcome, ValidAnnouncementData) { - trace!(target: "net::tx::validation", - hashes=?*partially_valid_data, - network=%self, - "validating eth66 announcement data.." - ); - - (FilterOutcome::Ok, ValidAnnouncementData::from_partially_valid_data(partially_valid_data)) - } -} - -#[cfg(test)] -mod test { - use super::*; - use alloy_primitives::B256; - use reth_eth_wire::{ - NewPooledTransactionHashes66, NewPooledTransactionHashes68, MAX_MESSAGE_SIZE, - }; - use std::{collections::HashMap, str::FromStr}; - - #[test] - fn eth68_empty_announcement() { - let types = vec![]; - let sizes = vec![]; - let hashes = vec![]; - - let announcement = NewPooledTransactionHashes68 { types, sizes, hashes }; - - let filter = EthMessageFilter::default(); - - let (outcome, _partially_valid_data) = filter.partially_filter_valid_entries(announcement); - - assert_eq!(outcome, FilterOutcome::ReportPeer); - } - - #[test] - fn eth68_announcement_unrecognized_tx_type() { - let types = vec![ - TxType::Eip7702 as u8 + 1, // the first type isn't valid - TxType::Legacy as u8, - ]; - let sizes = vec![MAX_MESSAGE_SIZE, MAX_MESSAGE_SIZE]; - let hashes = vec![ - B256::from_str("0xbeefcafebeefcafebeefcafebeefcafebeefcafebeefcafebeefcafebeefcafa") - .unwrap(), - B256::from_str("0xbeefcafebeefcafebeefcafebeefcafebeefcafebeefcafebeefcafebeefbbbb") - .unwrap(), - ]; - - let announcement = NewPooledTransactionHashes68 { - types: types.clone(), - sizes: sizes.clone(), - hashes: hashes.clone(), - }; - - let filter = EthMessageFilter::default(); - - let (outcome, partially_valid_data) = filter.partially_filter_valid_entries(announcement); - - assert_eq!(outcome, FilterOutcome::Ok); - - let (outcome, valid_data) = filter.filter_valid_entries_68(partially_valid_data); - - assert_eq!(outcome, FilterOutcome::ReportPeer); - - let mut expected_data = HashMap::default(); - expected_data.insert(hashes[1], Some((types[1], sizes[1]))); - - assert_eq!(expected_data, valid_data.into_data()) - } - - #[test] - fn eth68_announcement_duplicate_tx_hash() { - let types = vec![ - TxType::Eip1559 as u8, - TxType::Eip4844 as u8, - TxType::Eip1559 as u8, - TxType::Eip4844 as u8, - ]; - let sizes = vec![1, 1, 1, MAX_MESSAGE_SIZE]; - // first three or the same - let hashes = vec![ - B256::from_str("0xbeefcafebeefcafebeefcafebeefcafebeefcafebeefcafebeefcafebeefcafa") // dup - .unwrap(), - B256::from_str("0xbeefcafebeefcafebeefcafebeefcafebeefcafebeefcafebeefcafebeefcafa") // removed dup - .unwrap(), - B256::from_str("0xbeefcafebeefcafebeefcafebeefcafebeefcafebeefcafebeefcafebeefcafa") // removed dup - .unwrap(), - B256::from_str("0xbeefcafebeefcafebeefcafebeefcafebeefcafebeefcafebeefcafebeefbbbb") - .unwrap(), - ]; - - let announcement = NewPooledTransactionHashes68 { - types: types.clone(), - sizes: sizes.clone(), - hashes: hashes.clone(), - }; - - let filter = EthMessageFilter::default(); - - let (outcome, partially_valid_data) = filter.partially_filter_valid_entries(announcement); - - assert_eq!(outcome, FilterOutcome::ReportPeer); - - let mut expected_data = HashMap::default(); - expected_data.insert(hashes[3], Some((types[3], sizes[3]))); - expected_data.insert(hashes[0], Some((types[0], sizes[0]))); - - assert_eq!(expected_data, partially_valid_data.into_data()) - } - - #[test] - fn eth66_empty_announcement() { - let hashes = vec![]; - - let announcement = NewPooledTransactionHashes66(hashes); - - let filter: MessageFilter = MessageFilter::default(); - - let (outcome, _partially_valid_data) = filter.partially_filter_valid_entries(announcement); - - assert_eq!(outcome, FilterOutcome::ReportPeer); - } - - #[test] - fn eth66_announcement_duplicate_tx_hash() { - // first three or the same - let hashes = vec![ - B256::from_str("0xbeefcafebeefcafebeefcafebeefcafebeefcafebeefcafebeefcafebeefbbbb") // dup1 - .unwrap(), - B256::from_str("0xbeefcafebeefcafebeefcafebeefcafebeefcafebeefcafebeefcafebeefcafa") // dup2 - .unwrap(), - B256::from_str("0xbeefcafebeefcafebeefcafebeefcafebeefcafebeefcafebeefcafebeefcafa") // removed dup2 - .unwrap(), - B256::from_str("0xbeefcafebeefcafebeefcafebeefcafebeefcafebeefcafebeefcafebeefcafa") // removed dup2 - .unwrap(), - B256::from_str("0xbeefcafebeefcafebeefcafebeefcafebeefcafebeefcafebeefcafebeefbbbb") // removed dup1 - .unwrap(), - ]; - - let announcement = NewPooledTransactionHashes66(hashes.clone()); - - let filter: MessageFilter = MessageFilter::default(); - - let (outcome, partially_valid_data) = filter.partially_filter_valid_entries(announcement); - - assert_eq!(outcome, FilterOutcome::ReportPeer); - - let mut expected_data = HashMap::default(); - expected_data.insert(hashes[1], None); - expected_data.insert(hashes[0], None); - - assert_eq!(expected_data, partially_valid_data.into_data()) - } - - #[test] - fn eth68_announcement_eip7702_tx() { - let types = vec![TxType::Eip7702 as u8, TxType::Legacy as u8]; - let sizes = vec![MAX_MESSAGE_SIZE, MAX_MESSAGE_SIZE]; - let hashes = vec![ - B256::from_str("0xbeefcafebeefcafebeefcafebeefcafebeefcafebeefcafebeefcafebeefcafa") - .unwrap(), - B256::from_str("0xbeefcafebeefcafebeefcafebeefcafebeefcafebeefcafebeefcafebeefbbbb") - .unwrap(), - ]; - - let announcement = NewPooledTransactionHashes68 { - types: types.clone(), - sizes: sizes.clone(), - hashes: hashes.clone(), - }; - - let filter = EthMessageFilter::default(); - - let (outcome, partially_valid_data) = filter.partially_filter_valid_entries(announcement); - assert_eq!(outcome, FilterOutcome::Ok); - - let (outcome, valid_data) = filter.filter_valid_entries_68(partially_valid_data); - assert_eq!(outcome, FilterOutcome::Ok); - - let mut expected_data = HashMap::default(); - expected_data.insert(hashes[0], Some((types[0], sizes[0]))); - expected_data.insert(hashes[1], Some((types[1], sizes[1]))); - - assert_eq!(expected_data, valid_data.into_data()); - } - - #[test] - fn eth68_announcement_eip7702_tx_size_validation() { - let types = vec![TxType::Eip7702 as u8, TxType::Eip7702 as u8, TxType::Eip7702 as u8]; - // Test with different sizes: too small, reasonable, too large - let sizes = vec![ - 1, // too small - MAX_MESSAGE_SIZE / 2, // reasonable size - MAX_MESSAGE_SIZE + 1, // too large - ]; - let hashes = vec![ - B256::from_str("0xbeefcafebeefcafebeefcafebeefcafebeefcafebeefcafebeefcafebeefcafa") - .unwrap(), - B256::from_str("0xbeefcafebeefcafebeefcafebeefcafebeefcafebeefcafebeefcafebeefbbbb") - .unwrap(), - B256::from_str("0xbeefcafebeefcafebeefcafebeefcafebeefcafebeefcafebeefcafebeefcccc") - .unwrap(), - ]; - - let announcement = NewPooledTransactionHashes68 { - types: types.clone(), - sizes: sizes.clone(), - hashes: hashes.clone(), - }; - - let filter = EthMessageFilter::default(); - - let (outcome, partially_valid_data) = filter.partially_filter_valid_entries(announcement); - assert_eq!(outcome, FilterOutcome::Ok); - - let (outcome, valid_data) = filter.filter_valid_entries_68(partially_valid_data); - assert_eq!(outcome, FilterOutcome::Ok); - - let mut expected_data = HashMap::default(); - - for i in 0..3 { - expected_data.insert(hashes[i], Some((types[i], sizes[i]))); - } - - assert_eq!(expected_data, valid_data.into_data()); - } - - #[test] - fn eth68_announcement_mixed_tx_types() { - let types = vec![ - TxType::Legacy as u8, - TxType::Eip7702 as u8, - TxType::Eip1559 as u8, - TxType::Eip4844 as u8, - ]; - let sizes = vec![MAX_MESSAGE_SIZE, MAX_MESSAGE_SIZE, MAX_MESSAGE_SIZE, MAX_MESSAGE_SIZE]; - let hashes = vec![ - B256::from_str("0xbeefcafebeefcafebeefcafebeefcafebeefcafebeefcafebeefcafebeefcafa") - .unwrap(), - B256::from_str("0xbeefcafebeefcafebeefcafebeefcafebeefcafebeefcafebeefcafebeefbbbb") - .unwrap(), - B256::from_str("0xbeefcafebeefcafebeefcafebeefcafebeefcafebeefcafebeefcafebeefcccc") - .unwrap(), - B256::from_str("0xbeefcafebeefcafebeefcafebeefcafebeefcafebeefcafebeefcafebeefdddd") - .unwrap(), - ]; - - let announcement = NewPooledTransactionHashes68 { - types: types.clone(), - sizes: sizes.clone(), - hashes: hashes.clone(), - }; - - let filter = EthMessageFilter::default(); - - let (outcome, partially_valid_data) = filter.partially_filter_valid_entries(announcement); - assert_eq!(outcome, FilterOutcome::Ok); - - let (outcome, valid_data) = filter.filter_valid_entries_68(partially_valid_data); - assert_eq!(outcome, FilterOutcome::Ok); - - let mut expected_data = HashMap::default(); - // All transaction types should be included as they are valid - for i in 0..4 { - expected_data.insert(hashes[i], Some((types[i], sizes[i]))); - } - - assert_eq!(expected_data, valid_data.into_data()); - } - - #[test] - fn test_display_for_zst() { - let filter = EthMessageFilter::default(); - assert_eq!("EthMessageFilter", &filter.to_string()); - } -} diff --git a/crates/net/network/src/trusted_peers_resolver.rs b/crates/net/network/src/trusted_peers_resolver.rs index 04fc7b6b5fd..29940f415ec 100644 --- a/crates/net/network/src/trusted_peers_resolver.rs +++ b/crates/net/network/src/trusted_peers_resolver.rs @@ -13,7 +13,7 @@ use tracing::warn; /// It returns a resolved (`PeerId`, `NodeRecord`) update when one of its in‑flight tasks completes. #[derive(Debug)] pub struct TrustedPeersResolver { - /// The timer that triggers a new resolution cycle. + /// The list of trusted peers to resolve. pub trusted_peers: Vec, /// The timer that triggers a new resolution cycle. pub interval: Interval, diff --git a/crates/net/network/tests/it/big_pooled_txs_req.rs b/crates/net/network/tests/it/big_pooled_txs_req.rs index 1db378606ba..691ef5d379d 100644 --- a/crates/net/network/tests/it/big_pooled_txs_req.rs +++ b/crates/net/network/tests/it/big_pooled_txs_req.rs @@ -78,7 +78,9 @@ async fn test_large_tx_req() { // check all txs have been received match receive.await.unwrap() { Ok(PooledTransactions(txs)) => { - txs.into_iter().for_each(|tx| assert!(txs_hashes.contains(tx.hash()))); + for tx in txs { + assert!(txs_hashes.contains(tx.hash())); + } } Err(e) => { panic!("error: {e:?}"); diff --git a/crates/net/network/tests/it/connect.rs b/crates/net/network/tests/it/connect.rs index ab6ddac7345..1a3371a9073 100644 --- a/crates/net/network/tests/it/connect.rs +++ b/crates/net/network/tests/it/connect.rs @@ -19,7 +19,9 @@ use reth_network_p2p::{ sync::{NetworkSyncUpdater, SyncState}, }; use reth_network_peers::{mainnet_nodes, NodeRecord, TrustedPeer}; +use reth_network_types::peers::config::PeerBackoffDurations; use reth_storage_api::noop::NoopProvider; +use reth_tracing::init_test_tracing; use reth_transaction_pool::test_utils::testing_pool; use secp256k1::SecretKey; use std::time::Duration; @@ -359,16 +361,28 @@ async fn test_shutdown() { #[tokio::test(flavor = "multi_thread")] async fn test_trusted_peer_only() { + init_test_tracing(); let net = Testnet::create(2).await; let mut handles = net.handles(); + + // handle0 is used to test that: + // * outgoing connections to untrusted peers are not allowed + // * outgoing connections to trusted peers are allowed and succeed let handle0 = handles.next().unwrap(); + + // handle1 is used to test that: + // * incoming connections from untrusted peers are not allowed + // * incoming connections from trusted peers are allowed and succeed let handle1 = handles.next().unwrap(); drop(handles); let _handle = net.spawn(); let secret_key = SecretKey::new(&mut rand_08::thread_rng()); - let peers_config = PeersConfig::default().with_trusted_nodes_only(true); + let peers_config = PeersConfig::default() + .with_backoff_durations(PeerBackoffDurations::test()) + .with_ban_duration(Duration::from_millis(200)) + .with_trusted_nodes_only(true); let config = NetworkConfigBuilder::eth(secret_key) .listener_port(0) @@ -390,8 +404,8 @@ async fn test_trusted_peer_only() { // connect to an untrusted peer should fail. handle.add_peer(*handle0.peer_id(), handle0.local_addr()); - // wait 2 seconds, the number of connection is still 0. - tokio::time::sleep(Duration::from_secs(2)).await; + // wait 1 second, the number of connection is still 0. + tokio::time::sleep(Duration::from_secs(1)).await; assert_eq!(handle.num_connected_peers(), 0); // add to trusted peer. @@ -402,18 +416,24 @@ async fn test_trusted_peer_only() { assert_eq!(handle.num_connected_peers(), 1); // only receive connections from trusted peers. + handle1.add_peer(*handle.peer_id(), handle.local_addr()); - handle1.add_peer(*handle.peer_id(), handle0.local_addr()); - - // wait 2 seconds, the number of connections is still 1, because peer1 is untrusted. - tokio::time::sleep(Duration::from_secs(2)).await; + // wait 1 second, the number of connections is still 1, because peer1 is untrusted. + tokio::time::sleep(Duration::from_secs(1)).await; assert_eq!(handle.num_connected_peers(), 1); - handle1.add_trusted_peer(*handle.peer_id(), handle.local_addr()); + handle.add_trusted_peer(*handle1.peer_id(), handle1.local_addr()); + // wait for the next session established event to check the handle1 incoming connection let outgoing_peer_id1 = event_stream.next_session_established().await.unwrap(); assert_eq!(outgoing_peer_id1, *handle1.peer_id()); + + tokio::time::sleep(Duration::from_secs(2)).await; assert_eq!(handle.num_connected_peers(), 2); + + // check that handle0 and handle1 both have peers. + assert_eq!(handle0.num_connected_peers(), 1); + assert_eq!(handle1.num_connected_peers(), 1); } #[tokio::test(flavor = "multi_thread")] diff --git a/crates/net/network/tests/it/requests.rs b/crates/net/network/tests/it/requests.rs index 3a9dcf6308a..ac9b1a6dcac 100644 --- a/crates/net/network/tests/it/requests.rs +++ b/crates/net/network/tests/it/requests.rs @@ -1,14 +1,12 @@ #![allow(unreachable_pub)] //! Tests for eth related requests -use std::sync::Arc; - use alloy_consensus::Header; use rand::Rng; -use reth_eth_wire::HeadersDirection; +use reth_eth_wire::{EthVersion, HeadersDirection}; use reth_ethereum_primitives::Block; use reth_network::{ - test_utils::{NetworkEventStream, Testnet}, + test_utils::{NetworkEventStream, PeerConfig, Testnet}, BlockDownloaderProvider, NetworkEventListenerProvider, }; use reth_network_api::{NetworkInfo, Peers}; @@ -17,7 +15,9 @@ use reth_network_p2p::{ headers::client::{HeadersClient, HeadersRequest}, }; use reth_provider::test_utils::MockEthProvider; -use reth_transaction_pool::test_utils::TransactionGenerator; +use reth_transaction_pool::test_utils::{TestPool, TransactionGenerator}; +use std::sync::Arc; +use tokio::sync::oneshot; #[tokio::test(flavor = "multi_thread")] async fn test_get_body() { @@ -62,6 +62,60 @@ async fn test_get_body() { } } +#[tokio::test(flavor = "multi_thread")] +async fn test_get_body_range() { + reth_tracing::init_test_tracing(); + let mut rng = rand::rng(); + let mock_provider = Arc::new(MockEthProvider::default()); + let mut tx_gen = TransactionGenerator::new(rand::rng()); + + let mut net = Testnet::create_with(2, mock_provider.clone()).await; + + // install request handlers + net.for_each_mut(|peer| peer.install_request_handler()); + + let handle0 = net.peers()[0].handle(); + let mut events0 = NetworkEventStream::new(handle0.event_listener()); + + let handle1 = net.peers()[1].handle(); + + let _handle = net.spawn(); + + let fetch0 = handle0.fetch_client().await.unwrap(); + + handle0.add_peer(*handle1.peer_id(), handle1.local_addr()); + let connected = events0.next_session_established().await.unwrap(); + assert_eq!(connected, *handle1.peer_id()); + + let mut all_blocks = Vec::new(); + let mut block_hashes = Vec::new(); + // add some blocks + for _ in 0..100 { + let block_hash = rng.random(); + let mut block: Block = Block::default(); + block.body.transactions.push(tx_gen.gen_eip4844()); + + mock_provider.add_block(block_hash, block.clone()); + all_blocks.push(block); + block_hashes.push(block_hash); + } + + // ensure we can fetch the correct bodies + for idx in 0..100 { + let count = std::cmp::min(100 - idx, 10); // Limit to 10 bodies per request + let hashes_to_fetch = &block_hashes[idx..idx + count]; + + let res = fetch0.get_block_bodies(hashes_to_fetch.to_vec()).await; + assert!(res.is_ok(), "{res:?}"); + + let bodies = res.unwrap().1; + assert_eq!(bodies.len(), count); + for i in 0..bodies.len() { + assert_eq!(bodies[i], all_blocks[idx + i].body); + } + } +} + #[tokio::test(flavor = "multi_thread")] async fn test_get_header() { reth_tracing::init_test_tracing(); @@ -107,3 +161,368 @@ async fn test_get_header() { assert_eq!(headers[0], header); } } + +#[tokio::test(flavor = "multi_thread")] +async fn test_get_header_range() { + reth_tracing::init_test_tracing(); + let mut rng = rand::rng(); + let mock_provider = Arc::new(MockEthProvider::default()); + + let mut net = Testnet::create_with(2, mock_provider.clone()).await; + + // install request handlers + net.for_each_mut(|peer| peer.install_request_handler()); + + let handle0 = net.peers()[0].handle(); + let mut events0 = NetworkEventStream::new(handle0.event_listener()); + + let handle1 = net.peers()[1].handle(); + + let _handle = net.spawn(); + + let fetch0 = handle0.fetch_client().await.unwrap(); + + handle0.add_peer(*handle1.peer_id(), handle1.local_addr()); + let connected = events0.next_session_established().await.unwrap(); + assert_eq!(connected, *handle1.peer_id()); + + let start: u64 = rng.random(); + let mut hash = rng.random(); + let mut all_headers = Vec::new(); + // add some headers + for idx in 0..100 { + // Set a new random header to the mock storage and request it via the network + let header = Header { number: start + idx, parent_hash: hash, ..Default::default() }; + hash = rng.random(); + mock_provider.add_header(hash, header.clone()); + all_headers.push(header.seal(hash)); + } + + // ensure we can fetch the correct headers + for idx in 0..100 { + let count = 100 - idx; + let header = &all_headers[idx]; + let req = HeadersRequest { + start: header.hash().into(), + limit: count as u64, + direction: HeadersDirection::Rising, + }; + + let res = fetch0.get_headers(req).await; + assert!(res.is_ok(), "{res:?}"); + + let headers = res.unwrap().1; + assert_eq!(headers.len(), count); + assert_eq!(headers[0].number, start + idx as u64); + for i in 0..headers.len() { + assert_eq!(&headers[i], all_headers[idx + i].inner()); + } + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_get_header_range_falling() { + reth_tracing::init_test_tracing(); + let mut rng = rand::rng(); + let mock_provider = Arc::new(MockEthProvider::default()); + + let mut net = Testnet::create_with(2, mock_provider.clone()).await; + + // install request handlers + net.for_each_mut(|peer| peer.install_request_handler()); + + let handle0 = net.peers()[0].handle(); + let mut events0 = NetworkEventStream::new(handle0.event_listener()); + + let handle1 = net.peers()[1].handle(); + + let _handle = net.spawn(); + + let fetch0 = handle0.fetch_client().await.unwrap(); + + handle0.add_peer(*handle1.peer_id(), handle1.local_addr()); + let connected = events0.next_session_established().await.unwrap(); + assert_eq!(connected, *handle1.peer_id()); + + let start: u64 = rng.random(); + let mut hash = rng.random(); + let mut all_headers = Vec::new(); + // add some headers + for idx in 0..100 { + // Set a new random header to the mock storage + let header = Header { number: start + idx, parent_hash: hash, ..Default::default() }; + hash = rng.random(); + mock_provider.add_header(hash, header.clone()); + all_headers.push(header.seal(hash)); + } + + // ensure we can fetch the correct headers in falling direction + // start from the last header and work backwards + for idx in (0..100).rev() { + let count = std::cmp::min(idx + 1, 100); // Can't fetch more than idx+1 headers when going backwards + let header = &all_headers[idx]; + let req = HeadersRequest { + start: header.hash().into(), + limit: count as u64, + direction: HeadersDirection::Falling, + }; + + let res = fetch0.get_headers(req).await; + assert!(res.is_ok(), "{res:?}"); + + let headers = res.unwrap().1; + assert_eq!(headers.len(), count); + assert_eq!(headers[0].number, start + idx as u64); + // When fetching in Falling direction, headers come in reverse order + for i in 0..headers.len() { + assert_eq!(&headers[i], all_headers[idx - i].inner()); + } + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_eth68_get_receipts() { + reth_tracing::init_test_tracing(); + let mut rng = rand::rng(); + let mock_provider = Arc::new(MockEthProvider::default()); + + let mut net: Testnet, TestPool> = Testnet::default(); + + // Create peers with ETH68 protocol explicitly + let p0 = PeerConfig::with_protocols(mock_provider.clone(), Some(EthVersion::Eth68.into())); + net.add_peer_with_config(p0).await.unwrap(); + + let p1 = PeerConfig::with_protocols(mock_provider.clone(), Some(EthVersion::Eth68.into())); + net.add_peer_with_config(p1).await.unwrap(); + + // install request handlers + net.for_each_mut(|peer| peer.install_request_handler()); + + let handle0 = net.peers()[0].handle(); + let mut events0 = NetworkEventStream::new(handle0.event_listener()); + + let handle1 = net.peers()[1].handle(); + + let _handle = net.spawn(); + + handle0.add_peer(*handle1.peer_id(), handle1.local_addr()); + let connected = events0.next_session_established().await.unwrap(); + assert_eq!(connected, *handle1.peer_id()); + + // Create test receipts and add them to the mock provider + for block_num in 1..=10 { + let block_hash = rng.random(); + let header = Header { number: block_num, ..Default::default() }; + + // Create some test receipts + let receipts = vec![ + reth_ethereum_primitives::Receipt { + cumulative_gas_used: 21000, + success: true, + ..Default::default() + }, + reth_ethereum_primitives::Receipt { + cumulative_gas_used: 42000, + success: false, + ..Default::default() + }, + ]; + + mock_provider.add_header(block_hash, header.clone()); + mock_provider.add_receipts(header.number, receipts); + + // Test receipt request via low-level peer request + let (tx, rx) = oneshot::channel(); + handle0.send_request( + *handle1.peer_id(), + reth_network::PeerRequest::GetReceipts { + request: reth_eth_wire::GetReceipts(vec![block_hash]), + response: tx, + }, + ); + + let result = rx.await.unwrap(); + let receipts_response = result.unwrap(); + assert_eq!(receipts_response.0.len(), 1); + assert_eq!(receipts_response.0[0].len(), 2); + // Eth68 receipts should have bloom filters - verify the structure + assert_eq!(receipts_response.0[0][0].receipt.cumulative_gas_used, 21000); + assert_eq!(receipts_response.0[0][1].receipt.cumulative_gas_used, 42000); + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_eth69_get_headers() { + reth_tracing::init_test_tracing(); + let mut rng = rand::rng(); + let mock_provider = Arc::new(MockEthProvider::default()); + + let mut net: Testnet, TestPool> = Testnet::default(); + + // Create peers with ETH69 protocol + let p0 = PeerConfig::with_protocols(mock_provider.clone(), Some(EthVersion::Eth69.into())); + net.add_peer_with_config(p0).await.unwrap(); + + let p1 = PeerConfig::with_protocols(mock_provider.clone(), Some(EthVersion::Eth69.into())); + net.add_peer_with_config(p1).await.unwrap(); + + // install request handlers + net.for_each_mut(|peer| peer.install_request_handler()); + + let handle0 = net.peers()[0].handle(); + let mut events0 = NetworkEventStream::new(handle0.event_listener()); + + let handle1 = net.peers()[1].handle(); + + let _handle = net.spawn(); + + let fetch0 = handle0.fetch_client().await.unwrap(); + + handle0.add_peer(*handle1.peer_id(), handle1.local_addr()); + let connected = events0.next_session_established().await.unwrap(); + assert_eq!(connected, *handle1.peer_id()); + + let start: u64 = rng.random(); + let mut hash = rng.random(); + // request some headers via eth69 connection + for idx in 0..50 { + let header = Header { number: start + idx, parent_hash: hash, ..Default::default() }; + hash = rng.random(); + + mock_provider.add_header(hash, header.clone()); + + let req = + HeadersRequest { start: hash.into(), limit: 1, direction: HeadersDirection::Falling }; + + let res = fetch0.get_headers(req).await; + assert!(res.is_ok(), "{res:?}"); + + let headers = res.unwrap().1; + assert_eq!(headers.len(), 1); + assert_eq!(headers[0], header); + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_eth69_get_bodies() { + reth_tracing::init_test_tracing(); + let mut rng = rand::rng(); + let mock_provider = Arc::new(MockEthProvider::default()); + let mut tx_gen = TransactionGenerator::new(rand::rng()); + + let mut net: Testnet, TestPool> = Testnet::default(); + + // Create peers with ETH69 protocol + let p0 = PeerConfig::with_protocols(mock_provider.clone(), Some(EthVersion::Eth69.into())); + net.add_peer_with_config(p0).await.unwrap(); + + let p1 = PeerConfig::with_protocols(mock_provider.clone(), Some(EthVersion::Eth69.into())); + net.add_peer_with_config(p1).await.unwrap(); + + // install request handlers + net.for_each_mut(|peer| peer.install_request_handler()); + + let handle0 = net.peers()[0].handle(); + let mut events0 = NetworkEventStream::new(handle0.event_listener()); + + let handle1 = net.peers()[1].handle(); + + let _handle = net.spawn(); + + let fetch0 = handle0.fetch_client().await.unwrap(); + + handle0.add_peer(*handle1.peer_id(), handle1.local_addr()); + let connected = events0.next_session_established().await.unwrap(); + assert_eq!(connected, *handle1.peer_id()); + + // request some blocks via eth69 connection + for _ in 0..50 { + let block_hash = rng.random(); + let mut block: Block = Block::default(); + block.body.transactions.push(tx_gen.gen_eip4844()); + + mock_provider.add_block(block_hash, block.clone()); + + let res = fetch0.get_block_bodies(vec![block_hash]).await; + assert!(res.is_ok(), "{res:?}"); + + let blocks = res.unwrap().1; + assert_eq!(blocks.len(), 1); + assert_eq!(blocks[0], block.body); + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_eth69_get_receipts() { + reth_tracing::init_test_tracing(); + let mut rng = rand::rng(); + let mock_provider = Arc::new(MockEthProvider::default()); + + let mut net: Testnet, TestPool> = Testnet::default(); + + // Create peers with ETH69 protocol + let p0 = PeerConfig::with_protocols(mock_provider.clone(), Some(EthVersion::Eth69.into())); + net.add_peer_with_config(p0).await.unwrap(); + + let p1 = PeerConfig::with_protocols(mock_provider.clone(), Some(EthVersion::Eth69.into())); + net.add_peer_with_config(p1).await.unwrap(); + + // install request handlers + net.for_each_mut(|peer| peer.install_request_handler()); + + let handle0 = net.peers()[0].handle(); + let mut events0 = NetworkEventStream::new(handle0.event_listener()); + + let handle1 = net.peers()[1].handle(); + + let _handle = net.spawn(); + + handle0.add_peer(*handle1.peer_id(), handle1.local_addr()); + + // Wait for the session to be established + let connected = events0.next_session_established().await.unwrap(); + assert_eq!(connected, *handle1.peer_id()); + + // Create test receipts and add them to the mock provider + for block_num in 1..=10 { + let block_hash = rng.random(); + let header = Header { number: block_num, ..Default::default() }; + + // Create some test receipts + let receipts = vec![ + reth_ethereum_primitives::Receipt { + cumulative_gas_used: 21000, + success: true, + ..Default::default() + }, + reth_ethereum_primitives::Receipt { + cumulative_gas_used: 42000, + success: false, + ..Default::default() + }, + ]; + + mock_provider.add_header(block_hash, header.clone()); + mock_provider.add_receipts(header.number, receipts); + + let (tx, rx) = oneshot::channel(); + handle0.send_request( + *handle1.peer_id(), + reth_network::PeerRequest::GetReceipts69 { + request: reth_eth_wire::GetReceipts(vec![block_hash]), + response: tx, + }, + ); + + let result = rx.await.unwrap(); + let receipts_response = match result { + Ok(resp) => resp, + Err(e) => panic!("Failed to get receipts response: {e:?}"), + }; + assert_eq!(receipts_response.0.len(), 1); + assert_eq!(receipts_response.0[0].len(), 2); + // ETH69 receipts do not include bloom filters - verify the structure + assert_eq!(receipts_response.0[0][0].cumulative_gas_used, 21000); + assert_eq!(receipts_response.0[0][1].cumulative_gas_used, 42000); + } +} diff --git a/crates/net/network/tests/it/session.rs b/crates/net/network/tests/it/session.rs index 5ab305e5746..24875a0f410 100644 --- a/crates/net/network/tests/it/session.rs +++ b/crates/net/network/tests/it/session.rs @@ -37,7 +37,7 @@ async fn test_session_established_with_highest_version() { NetworkEvent::ActivePeerSession { info, .. } => { let SessionInfo { peer_id, status, .. } = info; assert_eq!(handle1.peer_id(), &peer_id); - assert_eq!(status.version, EthVersion::Eth68); + assert_eq!(status.version, EthVersion::LATEST); } ev => { panic!("unexpected event {ev:?}") @@ -123,3 +123,365 @@ async fn test_capability_version_mismatch() { handle.terminate().await; } + +#[tokio::test(flavor = "multi_thread")] +async fn test_eth69_peers_can_connect() { + reth_tracing::init_test_tracing(); + + let mut net = Testnet::create(0).await; + + // Create two peers that only support ETH69 + let p0 = PeerConfig::with_protocols(NoopProvider::default(), Some(EthVersion::Eth69.into())); + net.add_peer_with_config(p0).await.unwrap(); + + let p1 = PeerConfig::with_protocols(NoopProvider::default(), Some(EthVersion::Eth69.into())); + net.add_peer_with_config(p1).await.unwrap(); + + net.for_each(|peer| assert_eq!(0, peer.num_peers())); + + let mut handles = net.handles(); + let handle0 = handles.next().unwrap(); + let handle1 = handles.next().unwrap(); + drop(handles); + + let handle = net.spawn(); + + let mut events = handle0.event_listener().take(2); + handle0.add_peer(*handle1.peer_id(), handle1.local_addr()); + + while let Some(event) = events.next().await { + match event { + NetworkEvent::Peer(PeerEvent::PeerAdded(peer_id)) => { + assert_eq!(handle1.peer_id(), &peer_id); + } + NetworkEvent::ActivePeerSession { info, .. } => { + let SessionInfo { peer_id, status, .. } = info; + assert_eq!(handle1.peer_id(), &peer_id); + // Both peers support only ETH69, so they should connect with ETH69 + assert_eq!(status.version, EthVersion::Eth69); + } + ev => { + panic!("unexpected event: {ev:?}") + } + } + } + + handle.terminate().await; +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_peers_negotiate_highest_version_eth69() { + reth_tracing::init_test_tracing(); + + let mut net = Testnet::create(0).await; + + // Create one peer with multiple ETH versions including ETH69 + let p0 = PeerConfig::with_protocols( + NoopProvider::default(), + vec![ + EthVersion::Eth69.into(), + EthVersion::Eth68.into(), + EthVersion::Eth67.into(), + EthVersion::Eth66.into(), + ], + ); + net.add_peer_with_config(p0).await.unwrap(); + + // Create another peer with multiple ETH versions including ETH69 + let p1 = PeerConfig::with_protocols( + NoopProvider::default(), + vec![EthVersion::Eth69.into(), EthVersion::Eth68.into(), EthVersion::Eth67.into()], + ); + net.add_peer_with_config(p1).await.unwrap(); + + net.for_each(|peer| assert_eq!(0, peer.num_peers())); + + let mut handles = net.handles(); + let handle0 = handles.next().unwrap(); + let handle1 = handles.next().unwrap(); + drop(handles); + + let handle = net.spawn(); + + let mut events = handle0.event_listener().take(2); + handle0.add_peer(*handle1.peer_id(), handle1.local_addr()); + + while let Some(event) = events.next().await { + match event { + NetworkEvent::Peer(PeerEvent::PeerAdded(peer_id)) => { + assert_eq!(handle1.peer_id(), &peer_id); + } + NetworkEvent::ActivePeerSession { info, .. } => { + let SessionInfo { peer_id, status, .. } = info; + assert_eq!(handle1.peer_id(), &peer_id); + // Both peers support ETH69, so they should negotiate to the highest version: ETH69 + assert_eq!(status.version, EthVersion::Eth69); + } + ev => { + panic!("unexpected event: {ev:?}") + } + } + } + + handle.terminate().await; +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_eth69_vs_eth68_incompatible() { + reth_tracing::init_test_tracing(); + + let mut net = Testnet::create(0).await; + + // Create one peer that only supports ETH69 + let p0 = PeerConfig::with_protocols(NoopProvider::default(), Some(EthVersion::Eth69.into())); + net.add_peer_with_config(p0).await.unwrap(); + + // Create another peer that only supports ETH68 + let p1 = PeerConfig::with_protocols(NoopProvider::default(), Some(EthVersion::Eth68.into())); + net.add_peer_with_config(p1).await.unwrap(); + + net.for_each(|peer| assert_eq!(0, peer.num_peers())); + + let mut handles = net.handles(); + let handle0 = handles.next().unwrap(); + let handle1 = handles.next().unwrap(); + drop(handles); + + let handle = net.spawn(); + + let events = handle0.event_listener(); + let mut event_stream = NetworkEventStream::new(events); + + handle0.add_peer(*handle1.peer_id(), handle1.local_addr()); + + let added_peer_id = event_stream.peer_added().await.unwrap(); + assert_eq!(added_peer_id, *handle1.peer_id()); + + // Peers with no shared ETH version should fail to connect and be removed. + let removed_peer_id = event_stream.peer_removed().await.unwrap(); + assert_eq!(removed_peer_id, *handle1.peer_id()); + + handle.terminate().await; +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_eth69_mixed_version_negotiation() { + reth_tracing::init_test_tracing(); + + let mut net = Testnet::create(0).await; + + // Create one peer that supports ETH69 + ETH68 + let p0 = PeerConfig::with_protocols( + NoopProvider::default(), + vec![EthVersion::Eth69.into(), EthVersion::Eth68.into()], + ); + net.add_peer_with_config(p0).await.unwrap(); + + // Create another peer that only supports ETH68 + let p1 = PeerConfig::with_protocols(NoopProvider::default(), Some(EthVersion::Eth68.into())); + net.add_peer_with_config(p1).await.unwrap(); + + net.for_each(|peer| assert_eq!(0, peer.num_peers())); + + let mut handles = net.handles(); + let handle0 = handles.next().unwrap(); + let handle1 = handles.next().unwrap(); + drop(handles); + + let handle = net.spawn(); + + let mut events = handle0.event_listener().take(2); + handle0.add_peer(*handle1.peer_id(), handle1.local_addr()); + + while let Some(event) = events.next().await { + match event { + NetworkEvent::Peer(PeerEvent::PeerAdded(peer_id)) => { + assert_eq!(handle1.peer_id(), &peer_id); + } + NetworkEvent::ActivePeerSession { info, .. } => { + let SessionInfo { peer_id, status, .. } = info; + assert_eq!(handle1.peer_id(), &peer_id); + // Should negotiate to ETH68 (highest common version) + assert_eq!(status.version, EthVersion::Eth68); + } + ev => { + panic!("unexpected event: {ev:?}") + } + } + } + + handle.terminate().await; +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_multiple_peers_different_eth_versions() { + reth_tracing::init_test_tracing(); + + let mut net = Testnet::create(0).await; + + // Create a peer that supports all versions (ETH66-ETH69) + let p0 = PeerConfig::with_protocols( + NoopProvider::default(), + vec![ + EthVersion::Eth69.into(), + EthVersion::Eth68.into(), + EthVersion::Eth67.into(), + EthVersion::Eth66.into(), + ], + ); + net.add_peer_with_config(p0).await.unwrap(); + + // Create a peer that only supports newer versions (ETH68-ETH69) + let p1 = PeerConfig::with_protocols( + NoopProvider::default(), + vec![EthVersion::Eth69.into(), EthVersion::Eth68.into()], + ); + net.add_peer_with_config(p1).await.unwrap(); + + // Create a peer that only supports older versions (ETH66-ETH67) + let p2 = PeerConfig::with_protocols( + NoopProvider::default(), + vec![EthVersion::Eth67.into(), EthVersion::Eth66.into()], + ); + net.add_peer_with_config(p2).await.unwrap(); + + net.for_each(|peer| assert_eq!(0, peer.num_peers())); + + let mut handles = net.handles(); + let handle0 = handles.next().unwrap(); // All versions peer + let handle1 = handles.next().unwrap(); // Newer versions peer + let handle2 = handles.next().unwrap(); // Older versions peer + drop(handles); + + let handle = net.spawn(); + + let events = handle0.event_listener(); + let mut event_stream = NetworkEventStream::new(events); + + // Connect peer0 (all versions) to peer1 (newer versions) - should negotiate ETH69 + handle0.add_peer(*handle1.peer_id(), handle1.local_addr()); + + let added_peer_id = event_stream.peer_added().await.unwrap(); + assert_eq!(added_peer_id, *handle1.peer_id()); + + let established_peer_id = event_stream.next_session_established().await.unwrap(); + assert_eq!(established_peer_id, *handle1.peer_id()); + + // Connect peer0 (all versions) to peer2 (older versions) - should negotiate ETH67 + handle0.add_peer(*handle2.peer_id(), handle2.local_addr()); + + let added_peer_id = event_stream.peer_added().await.unwrap(); + assert_eq!(added_peer_id, *handle2.peer_id()); + + let established_peer_id = event_stream.next_session_established().await.unwrap(); + assert_eq!(established_peer_id, *handle2.peer_id()); + + // Both connections should be established successfully + + handle.terminate().await; +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_eth69_capability_negotiation_fallback() { + reth_tracing::init_test_tracing(); + + let mut net = Testnet::create(0).await; + + // Create a peer that prefers ETH69 but supports fallback to ETH67 + let p0 = PeerConfig::with_protocols( + NoopProvider::default(), + vec![EthVersion::Eth69.into(), EthVersion::Eth67.into()], + ); + net.add_peer_with_config(p0).await.unwrap(); + + // Create a peer that skips ETH68 and only supports ETH67/ETH66 + let p1 = PeerConfig::with_protocols( + NoopProvider::default(), + vec![EthVersion::Eth67.into(), EthVersion::Eth66.into()], + ); + net.add_peer_with_config(p1).await.unwrap(); + + net.for_each(|peer| assert_eq!(0, peer.num_peers())); + + let mut handles = net.handles(); + let handle0 = handles.next().unwrap(); + let handle1 = handles.next().unwrap(); + drop(handles); + + let handle = net.spawn(); + + let mut events = handle0.event_listener().take(2); + handle0.add_peer(*handle1.peer_id(), handle1.local_addr()); + + while let Some(event) = events.next().await { + match event { + NetworkEvent::Peer(PeerEvent::PeerAdded(peer_id)) => { + assert_eq!(handle1.peer_id(), &peer_id); + } + NetworkEvent::ActivePeerSession { info, .. } => { + let SessionInfo { peer_id, status, .. } = info; + assert_eq!(handle1.peer_id(), &peer_id); + // Should fallback to ETH67 (skipping ETH68 which neither supports) + assert_eq!(status.version, EthVersion::Eth67); + } + ev => { + panic!("unexpected event: {ev:?}") + } + } + } + + handle.terminate().await; +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_overlapping_version_sets_negotiation() { + reth_tracing::init_test_tracing(); + + let mut net = Testnet::create(0).await; + + // Peer 0: supports ETH69, ETH67, ETH66 (skips ETH68) + let p0 = PeerConfig::with_protocols( + NoopProvider::default(), + vec![EthVersion::Eth69.into(), EthVersion::Eth67.into(), EthVersion::Eth66.into()], + ); + net.add_peer_with_config(p0).await.unwrap(); + + // Peer 1: supports ETH68, ETH67, ETH66 (skips ETH69) + let p1 = PeerConfig::with_protocols( + NoopProvider::default(), + vec![EthVersion::Eth68.into(), EthVersion::Eth67.into(), EthVersion::Eth66.into()], + ); + net.add_peer_with_config(p1).await.unwrap(); + + net.for_each(|peer| assert_eq!(0, peer.num_peers())); + + let mut handles = net.handles(); + let handle0 = handles.next().unwrap(); + let handle1 = handles.next().unwrap(); + drop(handles); + + let handle = net.spawn(); + + let mut events = handle0.event_listener().take(2); + handle0.add_peer(*handle1.peer_id(), handle1.local_addr()); + + while let Some(event) = events.next().await { + match event { + NetworkEvent::Peer(PeerEvent::PeerAdded(peer_id)) => { + assert_eq!(handle1.peer_id(), &peer_id); + } + NetworkEvent::ActivePeerSession { info, .. } => { + let SessionInfo { peer_id, status, .. } = info; + assert_eq!(handle1.peer_id(), &peer_id); + // Should negotiate to ETH67 (highest common version between ETH69,67,66 and + // ETH68,67,66) + assert_eq!(status.version, EthVersion::Eth67); + } + ev => { + panic!("unexpected event: {ev:?}") + } + } + } + + handle.terminate().await; +} diff --git a/crates/net/network/tests/it/startup.rs b/crates/net/network/tests/it/startup.rs index a40e72b1186..3e05409961c 100644 --- a/crates/net/network/tests/it/startup.rs +++ b/crates/net/network/tests/it/startup.rs @@ -97,19 +97,25 @@ async fn test_discv5_and_discv4_same_socket_fails() { #[tokio::test(flavor = "multi_thread")] async fn test_discv5_and_rlpx_same_socket_ok_without_discv4() { + let test_port: u16 = TcpListener::bind("127.0.0.1:0") // 0 means OS assigns a free port + .await + .expect("Failed to bind to a port") + .local_addr() + .unwrap() + .port(); + let secret_key = SecretKey::new(&mut rand_08::thread_rng()); let config = NetworkConfigBuilder::eth(secret_key) - .listener_port(DEFAULT_DISCOVERY_PORT) + .listener_port(test_port) .disable_discv4_discovery() .discovery_v5( - reth_discv5::Config::builder((DEFAULT_DISCOVERY_ADDR, DEFAULT_DISCOVERY_PORT).into()) - .discv5_config( - discv5::ConfigBuilder::new(discv5::ListenConfig::from_ip( - DEFAULT_DISCOVERY_ADDR, - DEFAULT_DISCOVERY_PORT, - )) - .build(), - ), + reth_discv5::Config::builder((DEFAULT_DISCOVERY_ADDR, test_port).into()).discv5_config( + discv5::ConfigBuilder::new(discv5::ListenConfig::from_ip( + DEFAULT_DISCOVERY_ADDR, + test_port, + )) + .build(), + ), ) .disable_dns_discovery() .build(NoopProvider::default()); diff --git a/crates/net/network/tests/it/txgossip.rs b/crates/net/network/tests/it/txgossip.rs index 4014f41bfcb..ed1c2f925dd 100644 --- a/crates/net/network/tests/it/txgossip.rs +++ b/crates/net/network/tests/it/txgossip.rs @@ -10,7 +10,9 @@ use reth_network::{ }; use reth_network_api::{events::PeerEvent, PeerKind, PeersInfo}; use reth_provider::test_utils::{ExtendedAccount, MockEthProvider}; -use reth_transaction_pool::{test_utils::TransactionGenerator, PoolTransaction, TransactionPool}; +use reth_transaction_pool::{ + test_utils::TransactionGenerator, AddedTransactionOutcome, PoolTransaction, TransactionPool, +}; use std::sync::Arc; use tokio::join; @@ -42,7 +44,8 @@ async fn test_tx_gossip() { provider.add_account(sender, ExtendedAccount::new(0, U256::from(100_000_000))); // insert pending tx in peer0's pool - let hash = peer0_pool.add_external_transaction(tx).await.unwrap(); + let AddedTransactionOutcome { hash, .. } = + peer0_pool.add_external_transaction(tx).await.unwrap(); let inserted = peer0_tx_listener.recv().await.unwrap(); assert_eq!(inserted, hash); @@ -81,10 +84,10 @@ async fn test_tx_propagation_policy_trusted_only() { provider.add_account(sender, ExtendedAccount::new(0, U256::from(100_000_000))); // insert the tx in peer0's pool - let hash_0 = peer_0_handle.pool().unwrap().add_external_transaction(tx).await.unwrap(); + let outcome_0 = peer_0_handle.pool().unwrap().add_external_transaction(tx).await.unwrap(); let inserted = peer0_tx_listener.recv().await.unwrap(); - assert_eq!(inserted, hash_0); + assert_eq!(inserted, outcome_0.hash); // ensure tx is not gossiped to peer1 peer1_tx_listener.try_recv().expect_err("Empty"); @@ -108,16 +111,16 @@ async fn test_tx_propagation_policy_trusted_only() { provider.add_account(sender, ExtendedAccount::new(0, U256::from(100_000_000))); // insert pending tx in peer0's pool - let hash_1 = peer_0_handle.pool().unwrap().add_external_transaction(tx).await.unwrap(); + let outcome_1 = peer_0_handle.pool().unwrap().add_external_transaction(tx).await.unwrap(); let inserted = peer0_tx_listener.recv().await.unwrap(); - assert_eq!(inserted, hash_1); + assert_eq!(inserted, outcome_1.hash); // ensure peer1 now receives the pending txs from peer0 let mut buff = Vec::with_capacity(2); buff.push(peer1_tx_listener.recv().await.unwrap()); buff.push(peer1_tx_listener.recv().await.unwrap()); - assert!(buff.contains(&hash_1)); + assert!(buff.contains(&outcome_1.hash)); } #[tokio::test(flavor = "multi_thread")] diff --git a/crates/net/p2p/src/bodies/client.rs b/crates/net/p2p/src/bodies/client.rs index c97b9ab5385..90862c5144e 100644 --- a/crates/net/p2p/src/bodies/client.rs +++ b/crates/net/p2p/src/bodies/client.rs @@ -1,4 +1,5 @@ use std::{ + ops::RangeInclusive, pin::Pin, task::{ready, Context, Poll}, }; @@ -26,8 +27,26 @@ pub trait BodiesClient: DownloadClient { } /// Fetches the block body for the requested block with priority - fn get_block_bodies_with_priority(&self, hashes: Vec, priority: Priority) - -> Self::Output; + fn get_block_bodies_with_priority( + &self, + hashes: Vec, + priority: Priority, + ) -> Self::Output { + self.get_block_bodies_with_priority_and_range_hint(hashes, priority, None) + } + + /// Fetches the block body for the requested block with priority and a range hint for the + /// requested blocks. + /// + /// The range hint is not required, but can be used to optimize the routing of the request if + /// the hashes are continuous or close together and the range hint is `[earliest, latest]` for + /// the requested blocks. + fn get_block_bodies_with_priority_and_range_hint( + &self, + hashes: Vec, + priority: Priority, + range_hint: Option>, + ) -> Self::Output; /// Fetches a single block body for the requested hash. fn get_block_body(&self, hash: B256) -> SingleBodyRequest { diff --git a/crates/net/p2p/src/bodies/response.rs b/crates/net/p2p/src/bodies/response.rs index 20287a4b450..772fe6cbbd3 100644 --- a/crates/net/p2p/src/bodies/response.rs +++ b/crates/net/p2p/src/bodies/response.rs @@ -22,7 +22,7 @@ where } } - /// Return the reference to the response header + /// Return the difficulty of the response header pub fn difficulty(&self) -> U256 { match self { Self::Full(block) => block.difficulty(), diff --git a/crates/net/p2p/src/either.rs b/crates/net/p2p/src/either.rs index 3f1182bd482..a53592d8f9f 100644 --- a/crates/net/p2p/src/either.rs +++ b/crates/net/p2p/src/either.rs @@ -1,5 +1,7 @@ //! Support for different download types. +use std::ops::RangeInclusive; + use crate::{ bodies::client::BodiesClient, download::DownloadClient, @@ -37,14 +39,19 @@ where type Body = A::Body; type Output = Either; - fn get_block_bodies_with_priority( + fn get_block_bodies_with_priority_and_range_hint( &self, hashes: Vec, priority: Priority, + range_hint: Option>, ) -> Self::Output { match self { - Self::Left(a) => Either::Left(a.get_block_bodies_with_priority(hashes, priority)), - Self::Right(b) => Either::Right(b.get_block_bodies_with_priority(hashes, priority)), + Self::Left(a) => Either::Left( + a.get_block_bodies_with_priority_and_range_hint(hashes, priority, range_hint), + ), + Self::Right(b) => Either::Right( + b.get_block_bodies_with_priority_and_range_hint(hashes, priority, range_hint), + ), } } } diff --git a/crates/net/p2p/src/full_block.rs b/crates/net/p2p/src/full_block.rs index 9f542fc3c9b..06128c6b542 100644 --- a/crates/net/p2p/src/full_block.rs +++ b/crates/net/p2p/src/full_block.rs @@ -20,6 +20,7 @@ use std::{ fmt::Debug, future::Future, hash::Hash, + ops::RangeInclusive, pin::Pin, sync::Arc, task::{ready, Context, Poll}, @@ -279,18 +280,18 @@ where Client: BlockClient, { fn poll(&mut self, cx: &mut Context<'_>) -> Poll> { - if let Some(fut) = Pin::new(&mut self.header).as_pin_mut() { - if let Poll::Ready(res) = fut.poll(cx) { - self.header = None; - return Poll::Ready(ResponseResult::Header(res)) - } + if let Some(fut) = Pin::new(&mut self.header).as_pin_mut() && + let Poll::Ready(res) = fut.poll(cx) + { + self.header = None; + return Poll::Ready(ResponseResult::Header(res)) } - if let Some(fut) = Pin::new(&mut self.body).as_pin_mut() { - if let Poll::Ready(res) = fut.poll(cx) { - self.body = None; - return Poll::Ready(ResponseResult::Body(res)) - } + if let Some(fut) = Pin::new(&mut self.body).as_pin_mut() && + let Poll::Ready(res) = fut.poll(cx) + { + self.body = None; + return Poll::Ready(ResponseResult::Body(res)) } Poll::Pending @@ -620,18 +621,18 @@ where &mut self, cx: &mut Context<'_>, ) -> Poll> { - if let Some(fut) = Pin::new(&mut self.headers).as_pin_mut() { - if let Poll::Ready(res) = fut.poll(cx) { - self.headers = None; - return Poll::Ready(RangeResponseResult::Header(res)) - } + if let Some(fut) = Pin::new(&mut self.headers).as_pin_mut() && + let Poll::Ready(res) = fut.poll(cx) + { + self.headers = None; + return Poll::Ready(RangeResponseResult::Header(res)) } - if let Some(fut) = Pin::new(&mut self.bodies).as_pin_mut() { - if let Poll::Ready(res) = fut.poll(cx) { - self.bodies = None; - return Poll::Ready(RangeResponseResult::Body(res)) - } + if let Some(fut) = Pin::new(&mut self.bodies).as_pin_mut() && + let Poll::Ready(res) = fut.poll(cx) + { + self.bodies = None; + return Poll::Ready(RangeResponseResult::Body(res)) } Poll::Pending @@ -646,7 +647,7 @@ enum RangeResponseResult { } /// A headers+bodies client implementation that does nothing. -#[derive(Debug, Default, Clone)] +#[derive(Debug, Clone)] #[non_exhaustive] pub struct NoopFullBlockClient(PhantomData); @@ -692,10 +693,11 @@ where /// # Returns /// /// A future containing an empty vector of block bodies and a randomly generated `PeerId`. - fn get_block_bodies_with_priority( + fn get_block_bodies_with_priority_and_range_hint( &self, _hashes: Vec, _priority: Priority, + _range_hint: Option>, ) -> Self::Output { // Create a future that immediately returns an empty vector of block bodies and a random // PeerId. @@ -741,6 +743,12 @@ where type Block = Net::Block; } +impl Default for NoopFullBlockClient { + fn default() -> Self { + Self(PhantomData::) + } +} + #[cfg(test)] mod tests { use reth_ethereum_primitives::BlockBody; diff --git a/crates/net/p2p/src/headers/downloader.rs b/crates/net/p2p/src/headers/downloader.rs index 9ceb223e887..d0cf6550ea1 100644 --- a/crates/net/p2p/src/headers/downloader.rs +++ b/crates/net/p2p/src/headers/downloader.rs @@ -80,8 +80,8 @@ impl SyncTarget { } /// Represents a gap to sync: from `local_head` to `target` -#[derive(Clone, Debug)] -pub struct HeaderSyncGap { +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct HeaderSyncGap { /// The local head block. Represents lower bound of sync range. pub local_head: SealedHeader, diff --git a/crates/net/p2p/src/lib.rs b/crates/net/p2p/src/lib.rs index dead0f43bae..cb2b5d49721 100644 --- a/crates/net/p2p/src/lib.rs +++ b/crates/net/p2p/src/lib.rs @@ -9,7 +9,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] /// Shared abstractions for downloader implementations. pub mod download; diff --git a/crates/net/p2p/src/snap/client.rs b/crates/net/p2p/src/snap/client.rs index 7f08da31e27..c8003c38f8e 100644 --- a/crates/net/p2p/src/snap/client.rs +++ b/crates/net/p2p/src/snap/client.rs @@ -1,12 +1,28 @@ use crate::{download::DownloadClient, error::PeerRequestResult, priority::Priority}; use futures::Future; -use reth_eth_wire_types::snap::{AccountRangeMessage, GetAccountRangeMessage}; +use reth_eth_wire_types::snap::{ + AccountRangeMessage, ByteCodesMessage, GetAccountRangeMessage, GetByteCodesMessage, + GetStorageRangesMessage, GetTrieNodesMessage, StorageRangesMessage, TrieNodesMessage, +}; + +/// Response types for snap sync requests +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SnapResponse { + /// Response containing account range data + AccountRange(AccountRangeMessage), + /// Response containing storage ranges data + StorageRanges(StorageRangesMessage), + /// Response containing bytecode data + ByteCodes(ByteCodesMessage), + /// Response containing trie node data + TrieNodes(TrieNodesMessage), +} /// The snap sync downloader client #[auto_impl::auto_impl(&, Arc, Box)] pub trait SnapClient: DownloadClient { - /// The output future type for account range requests - type Output: Future> + Send + Sync + Unpin; + /// The output future type for snap requests + type Output: Future> + Send + Sync + Unpin; /// Sends the account range request to the p2p network and returns the account range /// response received from a peer. @@ -21,4 +37,40 @@ pub trait SnapClient: DownloadClient { request: GetAccountRangeMessage, priority: Priority, ) -> Self::Output; + + /// Sends the storage ranges request to the p2p network and returns the storage ranges + /// response received from a peer. + fn get_storage_ranges(&self, request: GetStorageRangesMessage) -> Self::Output; + + /// Sends the storage ranges request to the p2p network with priority set and returns + /// the storage ranges response received from a peer. + fn get_storage_ranges_with_priority( + &self, + request: GetStorageRangesMessage, + priority: Priority, + ) -> Self::Output; + + /// Sends the byte codes request to the p2p network and returns the byte codes + /// response received from a peer. + fn get_byte_codes(&self, request: GetByteCodesMessage) -> Self::Output; + + /// Sends the byte codes request to the p2p network with priority set and returns + /// the byte codes response received from a peer. + fn get_byte_codes_with_priority( + &self, + request: GetByteCodesMessage, + priority: Priority, + ) -> Self::Output; + + /// Sends the trie nodes request to the p2p network and returns the trie nodes + /// response received from a peer. + fn get_trie_nodes(&self, request: GetTrieNodesMessage) -> Self::Output; + + /// Sends the trie nodes request to the p2p network with priority set and returns + /// the trie nodes response received from a peer. + fn get_trie_nodes_with_priority( + &self, + request: GetTrieNodesMessage, + priority: Priority, + ) -> Self::Output; } diff --git a/crates/net/p2p/src/sync.rs b/crates/net/p2p/src/sync.rs index c7c43befc2a..77b97116d19 100644 --- a/crates/net/p2p/src/sync.rs +++ b/crates/net/p2p/src/sync.rs @@ -1,6 +1,7 @@ //! Traits used when interacting with the sync status of the network. use alloy_eips::eip2124::Head; +use reth_eth_wire_types::BlockRangeUpdate; /// A type that provides information about whether the node is currently syncing and the network is /// currently serving syncing related requests. @@ -22,11 +23,14 @@ pub trait SyncStateProvider: Send + Sync { /// which point the node is considered fully synced. #[auto_impl::auto_impl(&, Arc, Box)] pub trait NetworkSyncUpdater: std::fmt::Debug + Send + Sync + 'static { - /// Notifies about a [SyncState] update. + /// Notifies about a [`SyncState`] update. fn update_sync_state(&self, state: SyncState); - /// Updates the status of the p2p node + /// Updates the status of the p2p node. fn update_status(&self, head: Head); + + /// Updates the advertised block range. + fn update_block_range(&self, update: BlockRangeUpdate); } /// The state the network is currently in when it comes to synchronization. @@ -66,4 +70,5 @@ impl SyncStateProvider for NoopSyncStateUpdater { impl NetworkSyncUpdater for NoopSyncStateUpdater { fn update_sync_state(&self, _state: SyncState) {} fn update_status(&self, _: Head) {} + fn update_block_range(&self, _update: BlockRangeUpdate) {} } diff --git a/crates/net/p2p/src/test_utils/bodies.rs b/crates/net/p2p/src/test_utils/bodies.rs index 7570756d0fd..63f5656538d 100644 --- a/crates/net/p2p/src/test_utils/bodies.rs +++ b/crates/net/p2p/src/test_utils/bodies.rs @@ -8,7 +8,10 @@ use alloy_primitives::B256; use futures::FutureExt; use reth_ethereum_primitives::BlockBody; use reth_network_peers::PeerId; -use std::fmt::{Debug, Formatter}; +use std::{ + fmt::{Debug, Formatter}, + ops::RangeInclusive, +}; use tokio::sync::oneshot; /// A test client for fetching bodies @@ -40,10 +43,11 @@ where type Body = BlockBody; type Output = BodiesFut; - fn get_block_bodies_with_priority( + fn get_block_bodies_with_priority_and_range_hint( &self, hashes: Vec, _priority: Priority, + _range_hint: Option>, ) -> Self::Output { let (tx, rx) = oneshot::channel(); let _ = tx.send((self.responder)(hashes)); diff --git a/crates/net/p2p/src/test_utils/full_block.rs b/crates/net/p2p/src/test_utils/full_block.rs index 0ef329ef7db..dce6a3f9f45 100644 --- a/crates/net/p2p/src/test_utils/full_block.rs +++ b/crates/net/p2p/src/test_utils/full_block.rs @@ -14,7 +14,7 @@ use reth_eth_wire_types::HeadersDirection; use reth_ethereum_primitives::{Block, BlockBody}; use reth_network_peers::{PeerId, WithPeerId}; use reth_primitives_traits::{SealedBlock, SealedHeader}; -use std::{collections::HashMap, sync::Arc}; +use std::{collections::HashMap, ops::RangeInclusive, sync::Arc}; /// A headers+bodies client that stores the headers and bodies in memory, with an artificial soft /// bodies response limit that is set to 20 by default. @@ -145,10 +145,11 @@ impl BodiesClient for TestFullBlockClient { /// # Returns /// /// A future containing the result of the block body retrieval operation. - fn get_block_bodies_with_priority( + fn get_block_bodies_with_priority_and_range_hint( &self, hashes: Vec, _priority: Priority, + _range_hint: Option>, ) -> Self::Output { // Acquire a lock on the bodies. let bodies = self.bodies.lock(); diff --git a/crates/net/p2p/src/test_utils/headers.rs b/crates/net/p2p/src/test_utils/headers.rs index b1b34773388..b19b29d58dc 100644 --- a/crates/net/p2p/src/test_utils/headers.rs +++ b/crates/net/p2p/src/test_utils/headers.rs @@ -141,7 +141,9 @@ impl Stream for TestDownload { let mut headers = resp.1.into_iter().skip(1).map(SealedHeader::seal_slow).collect::>(); headers.sort_unstable_by_key(|h| h.number); - headers.into_iter().for_each(|h| this.buffer.push(h)); + for h in headers { + this.buffer.push(h); + } this.done = true; } Err(err) => { diff --git a/crates/net/peers/src/lib.rs b/crates/net/peers/src/lib.rs index bfca16a82fc..a2b9d9efb00 100644 --- a/crates/net/peers/src/lib.rs +++ b/crates/net/peers/src/lib.rs @@ -51,7 +51,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(feature = "std"), no_std)] extern crate alloc; diff --git a/crates/net/peers/src/node_record.rs b/crates/net/peers/src/node_record.rs index d9f10ebdbe7..641f2d274dc 100644 --- a/crates/net/peers/src/node_record.rs +++ b/crates/net/peers/src/node_record.rs @@ -63,11 +63,11 @@ impl NodeRecord { /// See also [`std::net::Ipv6Addr::to_ipv4_mapped`] pub fn convert_ipv4_mapped(&mut self) -> bool { // convert IPv4 mapped IPv6 address - if let IpAddr::V6(v6) = self.address { - if let Some(v4) = v6.to_ipv4_mapped() { - self.address = v4.into(); - return true - } + if let IpAddr::V6(v6) = self.address && + let Some(v4) = v6.to_ipv4_mapped() + { + self.address = v4.into(); + return true } false } @@ -309,6 +309,18 @@ mod tests { } } + #[test] + fn test_node_record() { + let url = "enode://fc8a2ff614e848c0af4c99372a81b8655edb8e11b617cffd0aab1a0691bcca66ca533626a528ee567f05f70c8cb529bda2c0a864cc0aec638a367fd2bb8e49fb@127.0.0.1:35481?discport=0"; + let node: NodeRecord = url.parse().unwrap(); + assert_eq!(node, NodeRecord { + address: IpAddr::V4([127,0,0, 1].into()), + tcp_port: 35481, + udp_port: 0, + id: "0xfc8a2ff614e848c0af4c99372a81b8655edb8e11b617cffd0aab1a0691bcca66ca533626a528ee567f05f70c8cb529bda2c0a864cc0aec638a367fd2bb8e49fb".parse().unwrap(), + }) + } + #[test] fn test_url_parse() { let url = "enode://6f8a80d14311c39f35f516fa664deaaaa13e85b2f7493f37f6144d86991ec012937307647bd3b9a82abe2974e1407241d54947bbb39763a4cac9f77166ad92a0@10.3.58.6:30303?discport=30301"; diff --git a/crates/node/api/src/lib.rs b/crates/node/api/src/lib.rs index b7cd087be3d..e8d6b697271 100644 --- a/crates/node/api/src/lib.rs +++ b/crates/node/api/src/lib.rs @@ -6,7 +6,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] /// Traits, validation methods, and helper types used to abstract over engine types. pub use reth_engine_primitives as engine; diff --git a/crates/node/api/src/node.rs b/crates/node/api/src/node.rs index d27b908f87e..a4a46d2e0ac 100644 --- a/crates/node/api/src/node.rs +++ b/crates/node/api/src/node.rs @@ -5,7 +5,7 @@ use alloy_rpc_types_engine::JwtSecret; use reth_basic_payload_builder::PayloadBuilder; use reth_consensus::{ConsensusError, FullConsensus}; use reth_db_api::{database_metrics::DatabaseMetrics, Database}; -use reth_engine_primitives::{BeaconConsensusEngineEvent, BeaconConsensusEngineHandle}; +use reth_engine_primitives::{ConsensusEngineEvent, ConsensusEngineHandle}; use reth_evm::ConfigureEvm; use reth_network_api::FullNetwork; use reth_node_core::node_config::NodeConfig; @@ -98,7 +98,10 @@ pub trait FullNodeComponents: FullNodeTypes + Clone + 'static { /// Returns the provider of the node. fn provider(&self) -> &Self::Provider; - /// Returns handle to runtime. + /// Returns an executor handle to spawn tasks. + /// + /// This can be used to spawn critical, blocking tasks or register tasks that should be + /// terminated gracefully. See also [`TaskSpawner`](reth_tasks::TaskSpawner). fn task_executor(&self) -> &TaskExecutor; } @@ -110,19 +113,92 @@ pub struct AddOnsContext<'a, N: FullNodeComponents> { /// Node configuration. pub config: &'a NodeConfig<::ChainSpec>, /// Handle to the beacon consensus engine. - pub beacon_engine_handle: BeaconConsensusEngineHandle<::Payload>, + pub beacon_engine_handle: ConsensusEngineHandle<::Payload>, /// Notification channel for engine API events - pub engine_events: EventSender::Primitives>>, + pub engine_events: EventSender::Primitives>>, /// JWT secret for the node. pub jwt_secret: JwtSecret, } /// Customizable node add-on types. +/// +/// This trait defines the interface for extending a node with additional functionality beyond +/// the core [`FullNodeComponents`]. It provides a way to launch supplementary services such as +/// RPC servers, monitoring, external integrations, or any custom functionality that builds on +/// top of the core node components. +/// +/// ## Purpose +/// +/// The `NodeAddOns` trait serves as an extension point in the node builder architecture, +/// allowing developers to: +/// - Define custom services that run alongside the main node +/// - Access all node components and configuration during initialization +/// - Return a handle for managing the launched services (e.g. handle to rpc server) +/// +/// ## How it fits into `NodeBuilder` +/// +/// In the node builder pattern, add-ons are the final layer that gets applied after all core +/// components are configured and started. The builder flow typically follows: +/// +/// 1. Configure [`NodeTypes`] (chain spec, database types, etc.) +/// 2. Build [`FullNodeComponents`] (consensus, networking, transaction pool, etc.) +/// 3. Launch [`NodeAddOns`] with access to all components via [`AddOnsContext`] +/// +/// ## Primary Use Case +/// +/// The primary use of this trait is to launch RPC servers that provide external API access to +/// the node. For Ethereum nodes, this typically includes two main servers: the regular RPC +/// server (HTTP/WS/IPC) that handles user requests and the authenticated Engine API server +/// that communicates with the consensus layer. The returned handle contains the necessary +/// endpoints and control mechanisms for these servers, allowing the node to serve JSON-RPC +/// requests and participate in consensus. While RPC is the main use case, the trait is +/// intentionally flexible to support other kinds of add-ons such as monitoring, indexing, or +/// custom protocol extensions. +/// +/// ## Context Access +/// +/// The [`AddOnsContext`] provides access to: +/// - All node components via the `node` field +/// - Node configuration +/// - Engine API handles for consensus layer communication +/// - JWT secrets for authenticated endpoints +/// +/// This ensures add-ons can integrate deeply with the node while maintaining clean separation +/// of concerns. pub trait NodeAddOns: Send { /// Handle to add-ons. + /// + /// This type is returned by [`launch_add_ons`](Self::launch_add_ons) and represents a + /// handle to the launched services. It must be `Clone` to allow multiple components to + /// hold references and should provide methods to interact with the running services. + /// + /// For RPC add-ons, this typically includes: + /// - Server handles to access local addresses and shutdown methods + /// - RPC module registry for runtime inspection of available methods + /// - Configured middleware and transport-specific settings + /// - For Engine API implementations, this also includes handles for consensus layer + /// communication type Handle: Send + Sync + Clone; /// Configures and launches the add-ons. + /// + /// This method is called once during node startup after all core components are initialized. + /// It receives an [`AddOnsContext`] that provides access to: + /// + /// - The fully configured node with all its components + /// - Node configuration for reading settings + /// - Engine API handles for consensus layer communication + /// - JWT secrets for setting up authenticated endpoints (if any). + /// + /// The implementation should: + /// 1. Use the context to configure the add-on services + /// 2. Launch any background tasks using the node's task executor + /// 3. Return a handle that allows interaction with the launched services + /// + /// # Errors + /// + /// This method may fail if the add-ons cannot be properly configured or launched, + /// for example due to port binding issues or invalid configuration. fn launch_add_ons( self, ctx: AddOnsContext<'_, N>, diff --git a/crates/node/builder/Cargo.toml b/crates/node/builder/Cargo.toml index d08c62d38ce..8e8774e86c8 100644 --- a/crates/node/builder/Cargo.toml +++ b/crates/node/builder/Cargo.toml @@ -19,11 +19,12 @@ reth-cli-util.workspace = true reth-config.workspace = true reth-consensus-debug-client.workspace = true reth-consensus.workspace = true -reth-db = { workspace = true, features = ["mdbx"], optional = true } +reth-db = { workspace = true, features = ["mdbx"] } reth-db-api.workspace = true reth-db-common.workspace = true reth-downloaders.workspace = true reth-engine-local.workspace = true +reth-engine-primitives.workspace = true reth-engine-service.workspace = true reth-engine-tree.workspace = true reth-engine-util.workspace = true @@ -39,6 +40,7 @@ reth-node-core.workspace = true reth-node-events.workspace = true reth-node-metrics.workspace = true reth-payload-builder.workspace = true +reth-primitives-traits.workspace = true reth-provider.workspace = true reth-prune.workspace = true reth-rpc.workspace = true @@ -54,6 +56,7 @@ reth-tokio-util.workspace = true reth-tracing.workspace = true reth-transaction-pool.workspace = true reth-basic-payload-builder.workspace = true +reth-node-ethstats.workspace = true ## ethereum alloy-consensus.workspace = true @@ -74,8 +77,8 @@ secp256k1 = { workspace = true, features = ["global-context", "std", "recovery"] ## misc aquamarine.workspace = true eyre.workspace = true -fdlimit.workspace = true jsonrpsee.workspace = true +fdlimit.workspace = true rayon.workspace = true serde_json.workspace = true @@ -84,12 +87,20 @@ tracing.workspace = true [dev-dependencies] tempfile.workspace = true +reth-ethereum-engine-primitives.workspace = true +reth-payload-builder = { workspace = true, features = ["test-utils"] } +reth-node-ethereum.workspace = true +reth-provider = { workspace = true, features = ["test-utils"] } +reth-evm-ethereum = { workspace = true, features = ["test-utils"] } [features] default = [] -js-tracer = ["reth-rpc/js-tracer"] +js-tracer = [ + "reth-rpc/js-tracer", + "reth-node-ethereum/js-tracer", + "reth-rpc-eth-types/js-tracer", +] test-utils = [ - "dep:reth-db", "reth-db/test-utils", "reth-chain-state/test-utils", "reth-chainspec/test-utils", @@ -104,10 +115,14 @@ test-utils = [ "reth-db-api/test-utils", "reth-provider/test-utils", "reth-transaction-pool/test-utils", + "reth-evm-ethereum/test-utils", + "reth-node-ethereum/test-utils", + "reth-primitives-traits/test-utils", ] op = [ - "reth-db?/op", + "reth-db/op", "reth-db-api/op", "reth-engine-local/op", "reth-evm/op", + "reth-primitives-traits/op", ] diff --git a/crates/node/builder/src/builder/mod.rs b/crates/node/builder/src/builder/mod.rs index 829ec88ca4d..f2886f47567 100644 --- a/crates/node/builder/src/builder/mod.rs +++ b/crates/node/builder/src/builder/mod.rs @@ -21,8 +21,7 @@ use reth_network::{ NetworkPrimitives, }; use reth_node_api::{ - FullNodePrimitives, FullNodeTypes, FullNodeTypesAdapter, NodeAddOns, NodeTypes, - NodeTypesWithDBAdapter, + FullNodeTypes, FullNodeTypesAdapter, NodeAddOns, NodeTypes, NodeTypesWithDBAdapter, }; use reth_node_core::{ cli::config::{PayloadBuilderConfig, RethTransactionPoolConfig}, @@ -37,7 +36,7 @@ use reth_provider::{ use reth_tasks::TaskExecutor; use reth_transaction_pool::{PoolConfig, PoolTransaction, TransactionPool}; use secp256k1::SecretKey; -use std::sync::Arc; +use std::{fmt::Debug, sync::Arc}; use tracing::{info, trace, warn}; pub mod add_ons; @@ -160,6 +159,48 @@ impl NodeBuilder<(), ChainSpec> { pub const fn new(config: NodeConfig) -> Self { Self { config, database: () } } +} + +impl NodeBuilder { + /// Returns a reference to the node builder's config. + pub const fn config(&self) -> &NodeConfig { + &self.config + } + + /// Returns a mutable reference to the node builder's config. + pub const fn config_mut(&mut self) -> &mut NodeConfig { + &mut self.config + } + + /// Returns a reference to the node's database + pub const fn db(&self) -> &DB { + &self.database + } + + /// Returns a mutable reference to the node's database + pub const fn db_mut(&mut self) -> &mut DB { + &mut self.database + } + + /// Applies a fallible function to the builder. + pub fn try_apply(self, f: F) -> Result + where + F: FnOnce(Self) -> Result, + { + f(self) + } + + /// Applies a fallible function to the builder, if the condition is `true`. + pub fn try_apply_if(self, cond: bool, f: F) -> Result + where + F: FnOnce(Self) -> Result, + { + if cond { + f(self) + } else { + Ok(self) + } + } /// Apply a function to the builder pub fn apply(self, f: F) -> Self @@ -182,18 +223,6 @@ impl NodeBuilder<(), ChainSpec> { } } -impl NodeBuilder { - /// Returns a reference to the node builder's config. - pub const fn config(&self) -> &NodeConfig { - &self.config - } - - /// Returns a mutable reference to the node builder's config. - pub const fn config_mut(&mut self) -> &mut NodeConfig { - &mut self.config - } -} - impl NodeBuilder { /// Configures the underlying database that the node will use. pub fn with_database(self, database: D) -> NodeBuilder { @@ -210,14 +239,25 @@ impl NodeBuilder { /// Creates an _ephemeral_ preconfigured node for testing purposes. #[cfg(feature = "test-utils")] pub fn testing_node( + self, + task_executor: TaskExecutor, + ) -> WithLaunchContext< + NodeBuilder>, ChainSpec>, + > { + let path = reth_db::test_utils::tempdir_path(); + self.testing_node_with_datadir(task_executor, path) + } + + /// Creates a preconfigured node for testing purposes with a specific datadir. + #[cfg(feature = "test-utils")] + pub fn testing_node_with_datadir( mut self, task_executor: TaskExecutor, + datadir: impl Into, ) -> WithLaunchContext< NodeBuilder>, ChainSpec>, > { - let path = reth_node_core::dirs::MaybePlatformPath::::from( - reth_db::test_utils::tempdir_path(), - ); + let path = reth_node_core::dirs::MaybePlatformPath::::from(datadir.into()); self.config = self.config.with_datadir_args(reth_node_core::args::DatadirArgs { datadir: path.clone(), ..Default::default() @@ -240,7 +280,7 @@ where /// Configures the types of the node. pub fn with_types(self) -> NodeBuilderWithTypes> where - T: NodeTypes + NodeTypesForProvider, + T: NodeTypesForProvider, { self.with_types_and_provider() } @@ -250,7 +290,7 @@ where self, ) -> NodeBuilderWithTypes> where - T: NodeTypes + NodeTypesForProvider, + T: NodeTypesForProvider, P: FullProvider>, { NodeBuilderWithTypes::new(self.config, self.database) @@ -270,7 +310,7 @@ where } } -/// A [`NodeBuilder`] with it's launch context already configured. +/// A [`NodeBuilder`] with its launch context already configured. /// /// This exposes the same methods as [`NodeBuilder`] but with the launch context already configured, /// See [`WithLaunchContext::launch`] @@ -291,6 +331,11 @@ impl WithLaunchContext> { pub const fn config(&self) -> &NodeConfig { self.builder.config() } + + /// Returns a mutable reference to the node builder's config. + pub const fn config_mut(&mut self) -> &mut NodeConfig { + self.builder.config_mut() + } } impl WithLaunchContext> @@ -301,7 +346,7 @@ where /// Configures the types of the node. pub fn with_types(self) -> WithLaunchContext>> where - T: NodeTypes + NodeTypesForProvider, + T: NodeTypesForProvider, { WithLaunchContext { builder: self.builder.with_types(), task_executor: self.task_executor } } @@ -311,7 +356,7 @@ where self, ) -> WithLaunchContext>> where - T: NodeTypes + NodeTypesForProvider, + T: NodeTypesForProvider, P: FullProvider>, { WithLaunchContext { @@ -356,7 +401,6 @@ where >>::Components, >, >, - N::Primitives: FullNodePrimitives, EngineNodeLauncher: LaunchNode< NodeBuilderWithComponents, N::ComponentsBuilder, N::AddOns>, >, @@ -413,6 +457,41 @@ where &self.builder.config } + /// Returns a mutable reference to the node builder's config. + pub const fn config_mut(&mut self) -> &mut NodeConfig<::ChainSpec> { + &mut self.builder.config + } + + /// Returns a reference to node's database. + pub const fn db(&self) -> &T::DB { + &self.builder.adapter.database + } + + /// Returns a mutable reference to node's database. + pub const fn db_mut(&mut self) -> &mut T::DB { + &mut self.builder.adapter.database + } + + /// Applies a fallible function to the builder. + pub fn try_apply(self, f: F) -> Result + where + F: FnOnce(Self) -> Result, + { + f(self) + } + + /// Applies a fallible function to the builder, if the condition is `true`. + pub fn try_apply_if(self, cond: bool, f: F) -> Result + where + F: FnOnce(Self) -> Result, + { + if cond { + f(self) + } else { + Ok(self) + } + } + /// Apply a function to the builder pub fn apply(self, f: F) -> Self where @@ -476,6 +555,39 @@ where } /// Sets the hook that is run to configure the rpc modules. + /// + /// This hook can obtain the node's components (txpool, provider, etc.) and can modify the + /// modules that the RPC server installs. + /// + /// # Examples + /// + /// ```rust,ignore + /// use jsonrpsee::{core::RpcResult, proc_macros::rpc}; + /// + /// #[derive(Clone)] + /// struct CustomApi { pool: Pool } + /// + /// #[rpc(server, namespace = "custom")] + /// impl CustomApi { + /// #[method(name = "hello")] + /// async fn hello(&self) -> RpcResult { + /// Ok("World".to_string()) + /// } + /// } + /// + /// let node = NodeBuilder::new(config) + /// .node(EthereumNode::default()) + /// .extend_rpc_modules(|ctx| { + /// // Access node components, so they can used by the CustomApi + /// let pool = ctx.pool().clone(); + /// + /// // Add custom RPC namespace + /// ctx.modules.merge_configured(CustomApi { pool }.into_rpc())?; + /// + /// Ok(()) + /// }) + /// .build()?; + /// ``` pub fn extend_rpc_modules(self, hook: F) -> Self where F: FnOnce(RpcContext<'_, NodeAdapter, AO::EthApi>) -> eyre::Result<()> @@ -550,22 +662,17 @@ where where EngineNodeLauncher: LaunchNode>, { - let Self { builder, task_executor } = self; - - let engine_tree_config = builder.config.engine.tree_config(); - - let launcher = - EngineNodeLauncher::new(task_executor, builder.config.datadir(), engine_tree_config); - builder.launch_with(launcher).await + let launcher = self.engine_api_launcher(); + self.builder.launch_with(launcher).await } /// Launches the node with the [`DebugNodeLauncher`]. /// /// This is equivalent to [`WithLaunchContext::launch`], but will enable the debugging features, /// if they are configured. - pub async fn launch_with_debug_capabilities( + pub fn launch_with_debug_capabilities( self, - ) -> eyre::Result<>>::Node> + ) -> >>::Future where T::Types: DebugNode>, DebugNodeLauncher: LaunchNode>, @@ -579,7 +686,18 @@ where builder.config.datadir(), engine_tree_config, )); - builder.launch_with(launcher).await + builder.launch_with(launcher) + } + + /// Returns an [`EngineNodeLauncher`] that can be used to launch the node with engine API + /// support. + pub fn engine_api_launcher(&self) -> EngineNodeLauncher { + let engine_tree_config = self.builder.config.engine.tree_config(); + EngineNodeLauncher::new( + self.task_executor.clone(), + self.builder.config.datadir(), + engine_tree_config, + ) } } @@ -621,6 +739,11 @@ impl BuilderContext { &self.config_container.config } + /// Returns a mutable reference to the config of the node. + pub const fn config_mut(&mut self) -> &mut NodeConfig<::ChainSpec> { + &mut self.config_container.config + } + /// Returns the loaded reh.toml config. pub const fn reth_config(&self) -> &reth_config::Config { &self.config_container.toml_config @@ -709,22 +832,22 @@ impl BuilderContext { > + Unpin + 'static, Node::Provider: BlockReaderFor, - Policy: TransactionPropagationPolicy, + Policy: TransactionPropagationPolicy + Debug, { let (handle, network, txpool, eth) = builder .transactions_with_policy(pool, tx_config, propagation_policy) .request_handler(self.provider().clone()) .split_with_handle(); - self.executor.spawn_critical("p2p txpool", txpool); - self.executor.spawn_critical("p2p eth request handler", eth); + self.executor.spawn_critical("p2p txpool", Box::pin(txpool)); + self.executor.spawn_critical("p2p eth request handler", Box::pin(eth)); let default_peers_path = self.config().datadir().known_peers(); let known_peers_file = self.config().network.persistent_peers_file(default_peers_path); self.executor.spawn_critical_with_graceful_shutdown_signal( "p2p network task", |shutdown| { - network.run_until_graceful_shutdown(shutdown, |network| { + Box::pin(network.run_until_graceful_shutdown(shutdown, |network| { if let Some(peers_file) = known_peers_file { let num_known_peers = network.num_known_peers(); trace!(target: "reth::cli", peers_file=?peers_file, num_peers=%num_known_peers, "Saving current peers"); @@ -737,7 +860,7 @@ impl BuilderContext { } } } - }) + })) }, ); diff --git a/crates/node/builder/src/builder/states.rs b/crates/node/builder/src/builder/states.rs index c4ced59d493..f60b56d57e7 100644 --- a/crates/node/builder/src/builder/states.rs +++ b/crates/node/builder/src/builder/states.rs @@ -10,7 +10,7 @@ use crate::{ hooks::NodeHooks, launch::LaunchNode, rpc::{RethRpcAddOns, RethRpcServerHandles, RpcContext}, - AddOns, FullNode, + AddOns, ComponentsFor, FullNode, }; use reth_exex::ExExContext; @@ -74,7 +74,7 @@ impl fmt::Debug for NodeTypesAdapter { /// Container for the node's types and the components and other internals that can be used by /// addons of the node. #[derive(Debug)] -pub struct NodeAdapter> { +pub struct NodeAdapter = ComponentsFor> { /// The components of the node. pub components: C, /// The task executor for the node. @@ -251,11 +251,11 @@ where AO: RethRpcAddOns>, { /// Launches the node with the given launcher. - pub async fn launch_with(self, launcher: L) -> eyre::Result + pub fn launch_with(self, launcher: L) -> L::Future where L: LaunchNode, { - launcher.launch_node(self).await + launcher.launch_node(self) } /// Sets the hook that is run once the rpc server is started. @@ -287,3 +287,51 @@ where }) } } + +#[cfg(test)] +mod test { + use super::*; + use crate::components::Components; + use reth_consensus::noop::NoopConsensus; + use reth_db_api::mock::DatabaseMock; + use reth_ethereum_engine_primitives::EthEngineTypes; + use reth_evm::noop::NoopEvmConfig; + use reth_evm_ethereum::MockEvmConfig; + use reth_network::EthNetworkPrimitives; + use reth_network_api::noop::NoopNetwork; + use reth_node_api::FullNodeTypesAdapter; + use reth_node_ethereum::EthereumNode; + use reth_payload_builder::PayloadBuilderHandle; + use reth_provider::noop::NoopProvider; + use reth_tasks::TaskManager; + use reth_transaction_pool::noop::NoopTransactionPool; + + #[test] + fn test_noop_components() { + let components = Components::< + FullNodeTypesAdapter, + NoopNetwork, + _, + NoopEvmConfig, + _, + > { + transaction_pool: NoopTransactionPool::default(), + evm_config: NoopEvmConfig::default(), + consensus: NoopConsensus::default(), + network: NoopNetwork::default(), + payload_builder_handle: PayloadBuilderHandle::::noop(), + }; + + let task_executor = { + let runtime = tokio::runtime::Runtime::new().unwrap(); + let handle = runtime.handle().clone(); + let manager = TaskManager::new(handle); + manager.executor() + }; + + let node = NodeAdapter { components, task_executor, provider: NoopProvider::default() }; + + // test that node implements `FullNodeComponents`` + as FullNodeComponents>::pool(&node); + } +} diff --git a/crates/node/builder/src/components/builder.rs b/crates/node/builder/src/components/builder.rs index 57a200617da..c54cc0e37f1 100644 --- a/crates/node/builder/src/components/builder.rs +++ b/crates/node/builder/src/components/builder.rs @@ -7,11 +7,16 @@ use crate::{ }, BuilderContext, ConfigureEvm, FullNodeTypes, }; -use reth_consensus::{ConsensusError, FullConsensus}; -use reth_network::types::NetPrimitivesFor; -use reth_network_api::FullNetwork; -use reth_node_api::{PrimitivesTy, TxTy}; -use reth_transaction_pool::{PoolPooledTx, PoolTransaction, TransactionPool}; +use reth_chainspec::EthChainSpec; +use reth_consensus::{noop::NoopConsensus, ConsensusError, FullConsensus}; +use reth_network::{types::NetPrimitivesFor, EthNetworkPrimitives, NetworkPrimitives}; +use reth_network_api::{noop::NoopNetwork, FullNetwork}; +use reth_node_api::{BlockTy, BodyTy, HeaderTy, NodeTypes, PrimitivesTy, ReceiptTy, TxTy}; +use reth_payload_builder::PayloadBuilderHandle; +use reth_transaction_pool::{ + noop::NoopTransactionPool, EthPoolTransaction, EthPooledTransaction, PoolPooledTx, + PoolTransaction, TransactionPool, +}; use std::{future::Future, marker::PhantomData}; /// A generic, general purpose and customizable [`NodeComponentsBuilder`] implementation. @@ -165,6 +170,21 @@ where _marker, } } + + /// Sets [`NoopTransactionPoolBuilder`]. + pub fn noop_pool( + self, + ) -> ComponentsBuilder, PayloadB, NetworkB, ExecB, ConsB> + { + ComponentsBuilder { + pool_builder: NoopTransactionPoolBuilder::::default(), + payload_builder: self.payload_builder, + network_builder: self.network_builder, + executor_builder: self.executor_builder, + consensus_builder: self.consensus_builder, + _marker: self._marker, + } + } } impl @@ -290,6 +310,48 @@ where _marker, } } + + /// Sets [`NoopNetworkBuilder`]. + pub fn noop_network( + self, + ) -> ComponentsBuilder, ExecB, ConsB> { + ComponentsBuilder { + pool_builder: self.pool_builder, + payload_builder: self.payload_builder, + network_builder: NoopNetworkBuilder::::default(), + executor_builder: self.executor_builder, + consensus_builder: self.consensus_builder, + _marker: self._marker, + } + } + + /// Sets [`NoopPayloadBuilder`]. + pub fn noop_payload( + self, + ) -> ComponentsBuilder { + ComponentsBuilder { + pool_builder: self.pool_builder, + payload_builder: NoopPayloadBuilder, + network_builder: self.network_builder, + executor_builder: self.executor_builder, + consensus_builder: self.consensus_builder, + _marker: self._marker, + } + } + + /// Sets [`NoopConsensusBuilder`]. + pub fn noop_consensus( + self, + ) -> ComponentsBuilder { + ComponentsBuilder { + pool_builder: self.pool_builder, + payload_builder: self.payload_builder, + network_builder: self.network_builder, + executor_builder: self.executor_builder, + consensus_builder: NoopConsensusBuilder, + _marker: self._marker, + } + } } impl NodeComponentsBuilder @@ -405,3 +467,99 @@ where self(ctx) } } + +/// Builds [`NoopTransactionPool`]. +#[derive(Debug, Clone)] +pub struct NoopTransactionPoolBuilder(PhantomData); + +impl PoolBuilder for NoopTransactionPoolBuilder +where + N: FullNodeTypes, + Tx: EthPoolTransaction> + Unpin, +{ + type Pool = NoopTransactionPool; + + async fn build_pool(self, _ctx: &BuilderContext) -> eyre::Result { + Ok(NoopTransactionPool::::new()) + } +} + +impl Default for NoopTransactionPoolBuilder { + fn default() -> Self { + Self(PhantomData) + } +} + +/// Builds [`NoopNetwork`]. +#[derive(Debug, Clone)] +pub struct NoopNetworkBuilder(PhantomData); + +impl NoopNetworkBuilder { + /// Returns the instance with ethereum types. + pub fn eth() -> Self { + Self::default() + } +} + +impl NetworkBuilder for NoopNetworkBuilder +where + N: FullNodeTypes, + Pool: TransactionPool, + Net: NetworkPrimitives< + BlockHeader = HeaderTy, + BlockBody = BodyTy, + Block = BlockTy, + Receipt = ReceiptTy, + >, +{ + type Network = NoopNetwork; + + async fn build_network( + self, + ctx: &BuilderContext, + _pool: Pool, + ) -> eyre::Result { + Ok(NoopNetwork::new().with_chain_id(ctx.chain_spec().chain_id())) + } +} + +impl Default for NoopNetworkBuilder { + fn default() -> Self { + Self(PhantomData) + } +} + +/// Builds [`NoopConsensus`]. +#[derive(Debug, Clone, Default)] +pub struct NoopConsensusBuilder; + +impl ConsensusBuilder for NoopConsensusBuilder +where + N: FullNodeTypes, +{ + type Consensus = NoopConsensus; + + async fn build_consensus(self, _ctx: &BuilderContext) -> eyre::Result { + Ok(NoopConsensus::default()) + } +} + +/// Builds [`PayloadBuilderHandle::noop`]. +#[derive(Debug, Clone, Default)] +pub struct NoopPayloadBuilder; + +impl PayloadServiceBuilder for NoopPayloadBuilder +where + N: FullNodeTypes, + Pool: TransactionPool, + EVM: ConfigureEvm> + 'static, +{ + async fn spawn_payload_builder_service( + self, + _ctx: &BuilderContext, + _pool: Pool, + _evm_config: EVM, + ) -> eyre::Result::Payload>> { + Ok(PayloadBuilderHandle::<::Payload>::noop()) + } +} diff --git a/crates/node/builder/src/components/payload.rs b/crates/node/builder/src/components/payload.rs index 2edc3b5e822..b587889e86f 100644 --- a/crates/node/builder/src/components/payload.rs +++ b/crates/node/builder/src/components/payload.rs @@ -4,9 +4,11 @@ use crate::{BuilderContext, FullNodeTypes}; use reth_basic_payload_builder::{BasicPayloadJobGenerator, BasicPayloadJobGeneratorConfig}; use reth_chain_state::CanonStateSubscriptions; use reth_node_api::{NodeTypes, PayloadBuilderFor}; -use reth_payload_builder::{PayloadBuilderHandle, PayloadBuilderService}; +use reth_payload_builder::{PayloadBuilderHandle, PayloadBuilderService, PayloadServiceCommand}; use reth_transaction_pool::TransactionPool; use std::future::Future; +use tokio::sync::{broadcast, mpsc}; +use tracing::warn; /// A type that knows how to spawn the payload service. pub trait PayloadServiceBuilder: @@ -110,3 +112,44 @@ where Ok(payload_service_handle) } } + +/// A `NoopPayloadServiceBuilder` useful for node implementations that are not implementing +/// validating/sequencing logic. +#[derive(Debug, Clone, Copy, Default)] +#[non_exhaustive] +pub struct NoopPayloadServiceBuilder; + +impl PayloadServiceBuilder for NoopPayloadServiceBuilder +where + Node: FullNodeTypes, + Pool: TransactionPool, + Evm: Send, +{ + async fn spawn_payload_builder_service( + self, + ctx: &BuilderContext, + _pool: Pool, + _evm_config: Evm, + ) -> eyre::Result::Payload>> { + let (tx, mut rx) = mpsc::unbounded_channel(); + + ctx.task_executor().spawn_critical("payload builder", async move { + #[allow(clippy::collection_is_never_read)] + let mut subscriptions = Vec::new(); + + while let Some(message) = rx.recv().await { + match message { + PayloadServiceCommand::Subscribe(tx) => { + let (events_tx, events_rx) = broadcast::channel(100); + // Retain senders to make sure that channels are not getting closed + subscriptions.push(events_tx); + let _ = tx.send(events_rx); + } + message => warn!(?message, "Noop payload service received a message"), + } + } + }); + + Ok(PayloadBuilderHandle::new(tx)) + } +} diff --git a/crates/node/builder/src/components/pool.rs b/crates/node/builder/src/components/pool.rs index 5b08e0a7739..a261f02c756 100644 --- a/crates/node/builder/src/components/pool.rs +++ b/crates/node/builder/src/components/pool.rs @@ -1,12 +1,16 @@ //! Pool component for the node builder. +use crate::{BuilderContext, FullNodeTypes}; use alloy_primitives::Address; -use reth_node_api::TxTy; -use reth_transaction_pool::{PoolConfig, PoolTransaction, SubPoolLimit, TransactionPool}; +use reth_chain_state::CanonStateSubscriptions; +use reth_chainspec::EthereumHardforks; +use reth_node_api::{NodeTypes, TxTy}; +use reth_transaction_pool::{ + blobstore::DiskFileBlobStore, CoinbaseTipOrdering, PoolConfig, PoolTransaction, SubPoolLimit, + TransactionPool, TransactionValidationTaskExecutor, TransactionValidator, +}; use std::{collections::HashSet, future::Future}; -use crate::{BuilderContext, FullNodeTypes}; - /// A type that knows how to build the transaction pool. pub trait PoolBuilder: Send { /// The transaction pool to build. @@ -98,3 +102,205 @@ impl PoolBuilderConfigOverrides { config } } + +/// A builder for creating transaction pools with common configuration options. +/// +/// This builder provides a fluent API for setting up transaction pools with various +/// configurations like blob stores, validators, and maintenance tasks. +pub struct TxPoolBuilder<'a, Node: FullNodeTypes, V = ()> { + ctx: &'a BuilderContext, + validator: V, +} + +impl<'a, Node: FullNodeTypes> TxPoolBuilder<'a, Node> { + /// Creates a new `TxPoolBuilder` with the given context. + pub const fn new(ctx: &'a BuilderContext) -> Self { + Self { ctx, validator: () } + } +} + +impl<'a, Node: FullNodeTypes, V> TxPoolBuilder<'a, Node, V> { + /// Configure the validator for the transaction pool. + pub fn with_validator(self, validator: NewV) -> TxPoolBuilder<'a, Node, NewV> { + TxPoolBuilder { ctx: self.ctx, validator } + } +} + +impl<'a, Node, V> TxPoolBuilder<'a, Node, TransactionValidationTaskExecutor> +where + Node: FullNodeTypes>, + V: TransactionValidator + 'static, + V::Transaction: + PoolTransaction> + reth_transaction_pool::EthPoolTransaction, +{ + /// Build the transaction pool and spawn its maintenance tasks. + /// This method creates the blob store, builds the pool, and spawns maintenance tasks. + pub fn build_and_spawn_maintenance_task( + self, + blob_store: DiskFileBlobStore, + pool_config: PoolConfig, + ) -> eyre::Result< + reth_transaction_pool::Pool< + TransactionValidationTaskExecutor, + CoinbaseTipOrdering, + DiskFileBlobStore, + >, + > { + // Destructure self to avoid partial move issues + let TxPoolBuilder { ctx, validator, .. } = self; + + let transaction_pool = reth_transaction_pool::Pool::new( + validator, + CoinbaseTipOrdering::default(), + blob_store, + pool_config.clone(), + ); + + // Spawn maintenance tasks using standalone functions + spawn_maintenance_tasks(ctx, transaction_pool.clone(), &pool_config)?; + + Ok(transaction_pool) + } +} + +/// Create blob store with default configuration. +pub fn create_blob_store( + ctx: &BuilderContext, +) -> eyre::Result { + let cache_size = Some(ctx.config().txpool.max_cached_entries); + create_blob_store_with_cache(ctx, cache_size) +} + +/// Create blob store with custom cache size configuration for how many blobs should be cached in +/// memory. +pub fn create_blob_store_with_cache( + ctx: &BuilderContext, + cache_size: Option, +) -> eyre::Result { + let data_dir = ctx.config().datadir(); + let config = if let Some(cache_size) = cache_size { + reth_transaction_pool::blobstore::DiskFileBlobStoreConfig::default() + .with_max_cached_entries(cache_size) + } else { + Default::default() + }; + + Ok(reth_transaction_pool::blobstore::DiskFileBlobStore::open(data_dir.blobstore(), config)?) +} + +/// Spawn local transaction backup task if enabled. +fn spawn_local_backup_task(ctx: &BuilderContext, pool: Pool) -> eyre::Result<()> +where + Node: FullNodeTypes, + Pool: TransactionPool + Clone + 'static, +{ + if !ctx.config().txpool.disable_transactions_backup { + let data_dir = ctx.config().datadir(); + let transactions_path = ctx + .config() + .txpool + .transactions_backup_path + .clone() + .unwrap_or_else(|| data_dir.txpool_transactions()); + + let transactions_backup_config = + reth_transaction_pool::maintain::LocalTransactionBackupConfig::with_local_txs_backup( + transactions_path, + ); + + ctx.task_executor().spawn_critical_with_graceful_shutdown_signal( + "local transactions backup task", + |shutdown| { + reth_transaction_pool::maintain::backup_local_transactions_task( + shutdown, + pool, + transactions_backup_config, + ) + }, + ); + } + Ok(()) +} + +/// Spawn the main maintenance task for transaction pool. +fn spawn_pool_maintenance_task( + ctx: &BuilderContext, + pool: Pool, + pool_config: &PoolConfig, +) -> eyre::Result<()> +where + Node: FullNodeTypes>, + Pool: reth_transaction_pool::TransactionPoolExt + Clone + 'static, + Pool::Transaction: PoolTransaction>, +{ + let chain_events = ctx.provider().canonical_state_stream(); + let client = ctx.provider().clone(); + + ctx.task_executor().spawn_critical( + "txpool maintenance task", + reth_transaction_pool::maintain::maintain_transaction_pool_future( + client, + pool, + chain_events, + ctx.task_executor().clone(), + reth_transaction_pool::maintain::MaintainPoolConfig { + max_tx_lifetime: pool_config.max_queued_lifetime, + no_local_exemptions: pool_config.local_transactions_config.no_exemptions, + ..Default::default() + }, + ), + ); + + Ok(()) +} + +/// Spawn all maintenance tasks for a transaction pool (backup + main maintenance). +pub fn spawn_maintenance_tasks( + ctx: &BuilderContext, + pool: Pool, + pool_config: &PoolConfig, +) -> eyre::Result<()> +where + Node: FullNodeTypes>, + Pool: reth_transaction_pool::TransactionPoolExt + Clone + 'static, + Pool::Transaction: PoolTransaction>, +{ + spawn_local_backup_task(ctx, pool.clone())?; + spawn_pool_maintenance_task(ctx, pool, pool_config)?; + Ok(()) +} + +impl std::fmt::Debug for TxPoolBuilder<'_, Node, V> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TxPoolBuilder").field("validator", &self.validator).finish() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use reth_transaction_pool::PoolConfig; + + #[test] + fn test_pool_builder_config_overrides_apply() { + let base_config = PoolConfig::default(); + let overrides = PoolBuilderConfigOverrides { + pending_limit: Some(SubPoolLimit::default()), + max_account_slots: Some(100), + minimal_protocol_basefee: Some(1000), + ..Default::default() + }; + + let updated_config = overrides.apply(base_config); + assert_eq!(updated_config.max_account_slots, 100); + assert_eq!(updated_config.minimal_protocol_basefee, 1000); + } + + #[test] + fn test_pool_builder_config_overrides_default() { + let overrides = PoolBuilderConfigOverrides::default(); + assert!(overrides.pending_limit.is_none()); + assert!(overrides.max_account_slots.is_none()); + assert!(overrides.local_addresses.is_empty()); + } +} diff --git a/crates/node/builder/src/engine_api_ext.rs b/crates/node/builder/src/engine_api_ext.rs new file mode 100644 index 00000000000..33d1d3e63ad --- /dev/null +++ b/crates/node/builder/src/engine_api_ext.rs @@ -0,0 +1,44 @@ +//! `EngineApiBuilder` callback wrapper +//! +//! Wraps an `EngineApiBuilder` to provide access to the built Engine API instance. + +use crate::rpc::EngineApiBuilder; +use eyre::Result; +use reth_node_api::{AddOnsContext, FullNodeComponents}; + +/// Provides access to an `EngineApi` instance with a callback +#[derive(Debug)] +pub struct EngineApiExt { + /// The inner builder that constructs the actual `EngineApi` + inner: B, + /// Optional callback function to execute with the built API + callback: Option, +} + +impl EngineApiExt { + /// Creates a new wrapper that calls `callback` when the API is built. + pub const fn new(inner: B, callback: F) -> Self { + Self { inner, callback: Some(callback) } + } +} + +impl EngineApiBuilder for EngineApiExt +where + B: EngineApiBuilder, + N: FullNodeComponents, + B::EngineApi: Clone, + F: FnOnce(B::EngineApi) + Send + Sync + 'static, +{ + type EngineApi = B::EngineApi; + + /// Builds the `EngineApi` and executes the callback if present. + async fn build_engine_api(mut self, ctx: &AddOnsContext<'_, N>) -> Result { + let api = self.inner.build_engine_api(ctx).await?; + + if let Some(callback) = self.callback.take() { + callback(api.clone()); + } + + Ok(api) + } +} diff --git a/crates/node/builder/src/hooks.rs b/crates/node/builder/src/hooks.rs index dda976599ed..71f0f3b4d2c 100644 --- a/crates/node/builder/src/hooks.rs +++ b/crates/node/builder/src/hooks.rs @@ -10,7 +10,6 @@ pub struct NodeHooks> { pub on_component_initialized: Box>, /// Hook to run once the node is started. pub on_node_started: Box>, - _marker: std::marker::PhantomData, } impl NodeHooks @@ -23,7 +22,6 @@ where Self { on_component_initialized: Box::<()>::default(), on_node_started: Box::<()>::default(), - _marker: Default::default(), } } diff --git a/crates/node/builder/src/launch/common.rs b/crates/node/builder/src/launch/common.rs index c26aa01cf7a..95909e34710 100644 --- a/crates/node/builder/src/launch/common.rs +++ b/crates/node/builder/src/launch/common.rs @@ -1,13 +1,42 @@ //! Helper types that can be used by launchers. +//! +//! ## Launch Context Type System +//! +//! The node launch process uses a type-state pattern to ensure correct initialization +//! order at compile time. Methods are only available when their prerequisites are met. +//! +//! ### Core Types +//! +//! - [`LaunchContext`]: Base context with executor and data directory +//! - [`LaunchContextWith`]: Context with an attached value of type `T` +//! - [`Attached`]: Pairs values, preserving both previous (L) and new (R) state +//! +//! ### Helper Attachments +//! +//! - [`WithConfigs`]: Node config + TOML config +//! - [`WithMeteredProvider`]: Provider factory with metrics +//! - [`WithMeteredProviders`]: Provider factory + blockchain provider +//! - [`WithComponents`]: Final form with all components +//! +//! ### Method Availability +//! +//! Methods are implemented on specific type combinations: +//! - `impl LaunchContextWith`: Generic methods available for any attachment +//! - `impl LaunchContextWith`: Config-specific methods +//! - `impl LaunchContextWith>`: Database operations +//! - `impl LaunchContextWith>`: Provider operations +//! - etc. +//! +//! This ensures correct initialization order without runtime checks. use crate::{ components::{NodeComponents, NodeComponentsBuilder}, hooks::OnComponentInitializedHook, - BuilderContext, NodeAdapter, + BuilderContext, ExExLauncher, NodeAdapter, PrimitivesTy, }; use alloy_eips::eip2124::Head; use alloy_primitives::{BlockNumber, B256}; -use eyre::{Context, OptionExt}; +use eyre::Context; use rayon::ThreadPoolBuilder; use reth_chainspec::{Chain, EthChainSpec, EthereumHardforks}; use reth_config::{config::EtlConfig, PruneConfig}; @@ -16,21 +45,17 @@ use reth_db_api::{database::Database, database_metrics::DatabaseMetrics}; use reth_db_common::init::{init_genesis, InitStorageError}; use reth_downloaders::{bodies::noop::NoopBodiesDownloader, headers::noop::NoopHeaderDownloader}; use reth_engine_local::MiningMode; -use reth_engine_tree::tree::{InvalidBlockHook, InvalidBlockHooks, NoopInvalidBlockHook}; use reth_evm::{noop::NoopEvmConfig, ConfigureEvm}; +use reth_exex::ExExManagerHandle; use reth_fs_util as fs; -use reth_invalid_block_hooks::InvalidBlockWitnessHook; use reth_network_p2p::headers::client::HeadersClient; use reth_node_api::{FullNodeTypes, NodeTypes, NodeTypesWithDB, NodeTypesWithDBAdapter}; use reth_node_core::{ - args::InvalidBlockHookType, + args::DefaultEraHost, dirs::{ChainPath, DataDirPath}, node_config::NodeConfig, primitives::BlockHeader, - version::{ - BUILD_PROFILE_NAME, CARGO_PKG_VERSION, VERGEN_BUILD_TIMESTAMP, VERGEN_CARGO_FEATURES, - VERGEN_CARGO_TARGET_TRIPLE, VERGEN_GIT_SHA, - }, + version::version_metadata, }; use reth_node_metrics::{ chain::ChainSpecInfo, @@ -41,14 +66,16 @@ use reth_node_metrics::{ }; use reth_provider::{ providers::{NodeTypesForProvider, ProviderNodeTypes, StaticFileProvider}, - BlockHashReader, BlockNumReader, ChainSpecProvider, ProviderError, ProviderFactory, - ProviderResult, StageCheckpointReader, StateProviderFactory, StaticFileProviderFactory, + BlockHashReader, BlockNumReader, ProviderError, ProviderFactory, ProviderResult, + StageCheckpointReader, StaticFileProviderFactory, }; use reth_prune::{PruneModes, PrunerBuilder}; -use reth_rpc_api::clients::EthApiClient; use reth_rpc_builder::config::RethRpcServerConfig; use reth_rpc_layer::JwtSecret; -use reth_stages::{sets::DefaultStages, MetricEvent, PipelineBuilder, PipelineTarget, StageId}; +use reth_stages::{ + sets::DefaultStages, stages::EraImportSource, MetricEvent, PipelineBuilder, PipelineTarget, + StageId, +}; use reth_static_file::StaticFileProducer; use reth_tasks::TaskExecutor; use reth_tracing::tracing::{debug, error, info, warn}; @@ -59,9 +86,28 @@ use tokio::sync::{ oneshot, watch, }; +use futures::{future::Either, stream, Stream, StreamExt}; +use reth_node_ethstats::EthStatsService; +use reth_node_events::{cl::ConsensusLayerHealthEvents, node::NodeEvent}; + /// Reusable setup for launching a node. /// -/// This provides commonly used boilerplate for launching a node. +/// This is the entry point for the node launch process. It implements a builder +/// pattern using type-state programming to enforce correct initialization order. +/// +/// ## Type Evolution +/// +/// Starting from `LaunchContext`, each method transforms the type to reflect +/// accumulated state: +/// +/// ```text +/// LaunchContext +/// └─> LaunchContextWith +/// └─> LaunchContextWith> +/// └─> LaunchContextWith> +/// └─> LaunchContextWith> +/// └─> LaunchContextWith> +/// ``` #[derive(Debug, Clone)] pub struct LaunchContext { /// The task executor for the node. @@ -85,10 +131,13 @@ impl LaunchContext { /// `config`. /// /// Attaches both the `NodeConfig` and the loaded `reth.toml` config to the launch context. - pub fn with_loaded_toml_config( + pub fn with_loaded_toml_config( self, config: NodeConfig, - ) -> eyre::Result>> { + ) -> eyre::Result>> + where + ChainSpec: EthChainSpec + reth_chainspec::EthereumHardforks, + { let toml_config = self.load_toml_config(&config)?; Ok(self.with(WithConfigs { config, toml_config })) } @@ -97,16 +146,19 @@ impl LaunchContext { /// `config`. /// /// This is async because the trusted peers may have to be resolved. - pub fn load_toml_config( + pub fn load_toml_config( &self, config: &NodeConfig, - ) -> eyre::Result { + ) -> eyre::Result + where + ChainSpec: EthChainSpec + reth_chainspec::EthereumHardforks, + { let config_path = config.config.clone().unwrap_or_else(|| self.data_dir.config()); let mut toml_config = reth_config::Config::from_path(&config_path) .wrap_err_with(|| format!("Could not load config file {config_path:?}"))?; - Self::save_pruning_config_if_full_node(&mut toml_config, config, &config_path)?; + Self::save_pruning_config(&mut toml_config, config, &config_path)?; info!(target: "reth::cli", path = ?config_path, "Configuration loaded"); @@ -116,20 +168,24 @@ impl LaunchContext { Ok(toml_config) } - /// Save prune config to the toml file if node is a full node. - fn save_pruning_config_if_full_node( + /// Save prune config to the toml file if node is a full node or has custom pruning CLI + /// arguments. + fn save_pruning_config( reth_config: &mut reth_config::Config, config: &NodeConfig, config_path: impl AsRef, - ) -> eyre::Result<()> { - if reth_config.prune.is_none() { - if let Some(prune_config) = config.prune_config() { - reth_config.update_prune_config(prune_config); + ) -> eyre::Result<()> + where + ChainSpec: EthChainSpec + reth_chainspec::EthereumHardforks, + { + if let Some(prune_config) = config.prune_config() { + if reth_config.prune != prune_config { + reth_config.set_prune_config(prune_config); info!(target: "reth::cli", "Saving prune config to toml file"); reth_config.save(config_path.as_ref())?; } - } else if config.prune_config().is_none() { - warn!(target: "reth::cli", "Prune configs present in config file but --full not provided. Running as a Full node"); + } else if !reth_config.prune.is_default() { + warn!(target: "reth::cli", "Pruning configuration is present in the config file, but no CLI arguments are provided. Using config from file."); } Ok(()) } @@ -167,16 +223,21 @@ impl LaunchContext { .thread_name(|i| format!("reth-rayon-{i}")) .build_global() { - error!(%err, "Failed to build global thread pool") + warn!(%err, "Failed to build global thread pool") } } } /// A [`LaunchContext`] along with an additional value. /// -/// This can be used to sequentially attach additional values to the type during the launch process. +/// The type parameter `T` represents the current state of the launch process. +/// Methods are conditionally implemented based on `T`, ensuring operations +/// are only available when their prerequisites are met. /// -/// The type provides common boilerplate for launching a node depending on the additional value. +/// For example: +/// - Config methods when `T = WithConfigs` +/// - Database operations when `T = Attached, DB>` +/// - Provider operations when `T = Attached, ProviderFactory>` #[derive(Debug, Clone)] pub struct LaunchContextWith { /// The wrapped launch context. @@ -250,7 +311,7 @@ impl LaunchContextWith> { &self.attachment.right } - /// Get a mutable reference to the right value. + /// Get a mutable reference to the left value. pub const fn left_mut(&mut self) -> &mut L { &mut self.attachment.left } @@ -273,8 +334,14 @@ impl LaunchContextWith Self { if self.toml_config_mut().stages.etl.dir.is_none() { - self.toml_config_mut().stages.etl.dir = - Some(EtlConfig::from_datadir(self.data_dir().data_dir())) + let etl_path = EtlConfig::from_datadir(self.data_dir().data_dir()); + if etl_path.exists() { + // Remove etl-path files on launch + if let Err(err) = fs::remove_dir_all(&etl_path) { + warn!(target: "reth::cli", ?etl_path, %err, "Failed to remove ETL path on launch"); + } + } + self.toml_config_mut().stages.etl.dir = Some(etl_path); } self @@ -332,8 +399,12 @@ impl LaunchContextWith Option { + pub fn prune_config(&self) -> PruneConfig + where + ChainSpec: reth_chainspec::EthereumHardforks, + { let Some(mut node_prune_config) = self.node_config().prune_config() else { // No CLI config is set, use the toml config. return self.toml_config().prune.clone(); @@ -341,19 +412,23 @@ impl LaunchContextWith PruneModes { - self.prune_config().map(|config| config.segments).unwrap_or_default() + pub fn prune_modes(&self) -> PruneModes + where + ChainSpec: reth_chainspec::EthereumHardforks, + { + self.prune_config().segments } /// Returns an initialized [`PrunerBuilder`] based on the configured [`PruneConfig`] - pub fn pruner_builder(&self) -> PrunerBuilder { - PrunerBuilder::new(self.prune_config().unwrap_or_default()) - .delete_limit(self.chain_spec().prune_delete_limit()) - .timeout(PrunerBuilder::DEFAULT_TIMEOUT) + pub fn pruner_builder(&self) -> PrunerBuilder + where + ChainSpec: reth_chainspec::EthereumHardforks, + { + PrunerBuilder::new(self.prune_config()) } /// Loads the JWT secret for the engine API @@ -364,11 +439,14 @@ impl LaunchContextWith MiningMode { + pub fn dev_mining_mode(&self, pool: Pool) -> MiningMode + where + Pool: TransactionPool + Unpin, + { if let Some(interval) = self.node_config().dev.block_time { MiningMode::interval(interval) } else { - MiningMode::instant(pool) + MiningMode::instant(pool, self.node_config().dev.block_max_transactions) } } } @@ -394,8 +472,7 @@ where .with_prune_modes(self.prune_modes()) .with_static_files_metrics(); - let has_receipt_pruning = - self.toml_config().prune.as_ref().is_some_and(|a| a.has_receipts_pruning()); + let has_receipt_pruning = self.toml_config().prune.has_receipts_pruning(); // Check for consistency between database and static files. If it fails, it unwinds to // the first block that's consistent between database and static files. @@ -426,6 +503,7 @@ where NoopEvmConfig::::default(), self.toml_config().stages.clone(), self.prune_modes(), + None, )) .build( factory.clone(), @@ -443,7 +521,9 @@ where let _ = tx.send(result); }), ); - rx.await??; + rx.await?.inspect_err(|err| { + error!(target: "reth::cli", unwind_target = %unwind_target, %err, "failed to run unwind") + })?; } Ok(factory) @@ -499,18 +579,17 @@ where // ensure recorder runs upkeep periodically install_prometheus_recorder().spawn_upkeep(); - let listen_addr = self.node_config().metrics; + let listen_addr = self.node_config().metrics.prometheus; if let Some(addr) = listen_addr { - info!(target: "reth::cli", "Starting metrics endpoint at {}", addr); let config = MetricServerConfig::new( addr, VersionInfo { - version: CARGO_PKG_VERSION, - build_timestamp: VERGEN_BUILD_TIMESTAMP, - cargo_features: VERGEN_CARGO_FEATURES, - git_sha: VERGEN_GIT_SHA, - target_triple: VERGEN_CARGO_TARGET_TRIPLE, - build_profile: BUILD_PROFILE_NAME, + version: version_metadata().cargo_pkg_version.as_ref(), + build_timestamp: version_metadata().vergen_build_timestamp.as_ref(), + cargo_features: version_metadata().vergen_cargo_features.as_ref(), + git_sha: version_metadata().vergen_git_sha.as_ref(), + target_triple: version_metadata().vergen_cargo_target_triple.as_ref(), + build_profile: version_metadata().build_profile_name.as_ref(), }, ChainSpecInfo { name: self.left().config.chain.chain().to_string() }, self.task_executor().clone(), @@ -528,7 +607,7 @@ where } }) .build(), - ); + ).with_push_gateway(self.node_config().metrics.push_gateway_url.clone(), self.node_config().metrics.push_gateway_interval); MetricServer::new(config).serve().await?; } @@ -872,84 +951,88 @@ where pub const fn components(&self) -> &CB::Components { &self.node_adapter().components } -} -impl - LaunchContextWith< - Attached::ChainSpec>, WithComponents>, - > -where - T: FullNodeTypes< - Provider: StateProviderFactory + ChainSpecProvider, - Types: NodeTypesForProvider, - >, - CB: NodeComponentsBuilder, -{ - /// Returns the [`InvalidBlockHook`] to use for the node. - pub fn invalid_block_hook( + /// Launches ExEx (Execution Extensions) and returns the ExEx manager handle. + #[allow(clippy::type_complexity)] + pub async fn launch_exex( &self, - ) -> eyre::Result::Primitives>>> { - let Some(ref hook) = self.node_config().debug.invalid_block_hook else { - return Ok(Box::new(NoopInvalidBlockHook::default())) - }; - let healthy_node_rpc_client = self.get_healthy_node_client()?; - - let output_directory = self.data_dir().invalid_block_hooks(); - let hooks = hook - .iter() - .copied() - .map(|hook| { - let output_directory = output_directory.join(hook.to_string()); - fs::create_dir_all(&output_directory)?; - - Ok(match hook { - InvalidBlockHookType::Witness => Box::new(InvalidBlockWitnessHook::new( - self.blockchain_db().clone(), - self.components().evm_config().clone(), - output_directory, - healthy_node_rpc_client.clone(), - )), - InvalidBlockHookType::PreState | InvalidBlockHookType::Opcode => { - eyre::bail!("invalid block hook {hook:?} is not implemented yet") - } - } as Box>) - }) - .collect::>()?; - - Ok(Box::new(InvalidBlockHooks(hooks))) - } - - /// Returns an RPC client for the healthy node, if configured in the node config. - fn get_healthy_node_client(&self) -> eyre::Result> { - self.node_config() - .debug - .healthy_node_rpc_url - .as_ref() - .map(|url| { - let client = jsonrpsee::http_client::HttpClientBuilder::default().build(url)?; - - // Verify that the healthy node is running the same chain as the current node. - let chain_id = futures::executor::block_on(async { - EthApiClient::< - alloy_rpc_types::Transaction, - alloy_rpc_types::Block, - alloy_rpc_types::Receipt, - alloy_rpc_types::Header, - >::chain_id(&client) - .await - })? - .ok_or_eyre("healthy node rpc client didn't return a chain id")?; - if chain_id.to::() != self.chain_id().id() { - eyre::bail!("invalid chain id for healthy node: {chain_id}") - } + installed_exex: Vec<( + String, + Box>>, + )>, + ) -> eyre::Result>>> { + ExExLauncher::new( + self.head(), + self.node_adapter().clone(), + installed_exex, + self.configs().clone(), + ) + .launch() + .await + } + + /// Creates the ERA import source based on node configuration. + /// + /// Returns `Some(EraImportSource)` if ERA is enabled in the node config, otherwise `None`. + pub fn era_import_source(&self) -> Option { + let node_config = self.node_config(); + if !node_config.era.enabled { + return None; + } + + EraImportSource::maybe_new( + node_config.era.source.path.clone(), + node_config.era.source.url.clone(), + || node_config.chain.chain().kind().default_era_host(), + || node_config.datadir().data_dir().join("era").into(), + ) + } + + /// Creates consensus layer health events stream based on node configuration. + /// + /// Returns a stream that monitors consensus layer health if: + /// - No debug tip is configured + /// - Not running in dev mode + /// + /// Otherwise returns an empty stream. + pub fn consensus_layer_events( + &self, + ) -> impl Stream>> + 'static + where + T::Provider: reth_provider::CanonChainTracker, + { + if self.node_config().debug.tip.is_none() && !self.is_dev() { + Either::Left( + ConsensusLayerHealthEvents::new(Box::new(self.blockchain_db().clone())) + .map(Into::into), + ) + } else { + Either::Right(stream::empty()) + } + } + + /// Spawns the [`EthStatsService`] service if configured. + pub async fn spawn_ethstats(&self) -> eyre::Result<()> { + let Some(url) = self.node_config().debug.ethstats.as_ref() else { return Ok(()) }; + + let network = self.components().network().clone(); + let pool = self.components().pool().clone(); + let provider = self.node_adapter().provider.clone(); - Ok(client) - }) - .transpose() + info!(target: "reth::cli", "Starting EthStats service at {}", url); + + let ethstats = EthStatsService::new(url, network, provider, pool).await?; + tokio::spawn(async move { ethstats.run().await }); + + Ok(()) } } -/// Joins two attachments together. +/// Joins two attachments together, preserving access to both values. +/// +/// This type enables the launch process to accumulate state while maintaining +/// access to all previously attached components. The `left` field holds the +/// previous state, while `right` holds the newly attached component. #[derive(Clone, Copy, Debug)] pub struct Attached { left: L, @@ -988,9 +1071,9 @@ impl Attached { &self.right } - /// Get a mutable reference to the right value. - pub const fn left_mut(&mut self) -> &mut R { - &mut self.right + /// Get a mutable reference to the left value. + pub const fn left_mut(&mut self) -> &mut L { + &mut self.left } /// Get a mutable reference to the right value. @@ -1076,6 +1159,7 @@ mod tests { transaction_lookup_distance: None, transaction_lookup_before: None, receipts_full: false, + receipts_pre_merge: false, receipts_distance: None, receipts_before: None, account_history_full: false, @@ -1084,16 +1168,15 @@ mod tests { storage_history_full: false, storage_history_distance: None, storage_history_before: None, - receipts_log_filter: vec![], + bodies_pre_merge: false, + bodies_distance: None, + receipts_log_filter: None, + bodies_before: None, }, ..NodeConfig::test() }; - LaunchContext::save_pruning_config_if_full_node( - &mut reth_config, - &node_config, - config_path, - ) - .unwrap(); + LaunchContext::save_pruning_config(&mut reth_config, &node_config, config_path) + .unwrap(); let loaded_config = Config::from_path(config_path).unwrap(); diff --git a/crates/node/builder/src/launch/debug.rs b/crates/node/builder/src/launch/debug.rs index 7609616b031..f5e9745cddc 100644 --- a/crates/node/builder/src/launch/debug.rs +++ b/crates/node/builder/src/launch/debug.rs @@ -1,24 +1,104 @@ use super::LaunchNode; use crate::{rpc::RethRpcAddOns, EngineNodeLauncher, Node, NodeHandle}; +use alloy_consensus::transaction::Either; use alloy_provider::network::AnyNetwork; use jsonrpsee::core::{DeserializeOwned, Serialize}; use reth_chainspec::EthChainSpec; use reth_consensus_debug_client::{DebugConsensusClient, EtherscanBlockProvider, RpcBlockProvider}; -use reth_node_api::{BlockTy, FullNodeComponents}; -use std::sync::Arc; +use reth_engine_local::LocalMiner; +use reth_node_api::{ + BlockTy, FullNodeComponents, PayloadAttrTy, PayloadAttributesBuilder, PayloadTypes, +}; +use std::{ + future::{Future, IntoFuture}, + pin::Pin, + sync::Arc, +}; use tracing::info; -/// [`Node`] extension with support for debugging utilities, see [`DebugNodeLauncher`] for more -/// context. + +/// [`Node`] extension with support for debugging utilities. +/// +/// This trait provides additional necessary conversion from RPC block type to the node's +/// primitive block type, e.g. `alloy_rpc_types_eth::Block` to the node's internal block +/// representation. +/// +/// This is used in conjunction with the [`DebugNodeLauncher`] to enable debugging features such as: +/// +/// - **Etherscan Integration**: Use Etherscan as a consensus client to follow the chain and submit +/// blocks to the local engine. +/// - **RPC Consensus Client**: Connect to an external RPC endpoint to fetch blocks and submit them +/// to the local engine to follow the chain. +/// +/// See [`DebugNodeLauncher`] for the launcher that enables these features. +/// +/// # Implementation +/// +/// To implement this trait, you need to: +/// 1. Define the RPC block type (typically `alloy_rpc_types_eth::Block`) +/// 2. Implement the conversion from RPC format to your primitive block type +/// +/// # Example +/// +/// ```ignore +/// impl> DebugNode for MyNode { +/// type RpcBlock = alloy_rpc_types_eth::Block; +/// +/// fn rpc_to_primitive_block(rpc_block: Self::RpcBlock) -> BlockTy { +/// // Convert from RPC format to primitive format by converting the transactions +/// rpc_block.into_consensus().convert_transactions() +/// } +/// } +/// ``` pub trait DebugNode: Node { /// RPC block type. Used by [`DebugConsensusClient`] to fetch blocks and submit them to the - /// engine. + /// engine. This is intended to match the block format returned by the external RPC endpoint. type RpcBlock: Serialize + DeserializeOwned + 'static; /// Converts an RPC block to a primitive block. + /// + /// This method handles the conversion between the RPC block format and the internal primitive + /// block format used by the node's consensus engine. + /// + /// # Example + /// + /// For Ethereum nodes, this typically converts from `alloy_rpc_types_eth::Block` + /// to the node's internal block representation. fn rpc_to_primitive_block(rpc_block: Self::RpcBlock) -> BlockTy; + + /// Creates a payload attributes builder for local mining in dev mode. + /// + /// It will be used by the `LocalMiner` when dev mode is enabled. + /// + /// The builder is responsible for creating the payload attributes that define how blocks should + /// be constructed during local mining. + fn local_payload_attributes_builder( + chain_spec: &Self::ChainSpec, + ) -> impl PayloadAttributesBuilder< + <::Payload as PayloadTypes>::PayloadAttributes, + >; } /// Node launcher with support for launching various debugging utilities. +/// +/// This launcher wraps an existing launcher and adds debugging capabilities when +/// certain debug flags are enabled. It provides two main debugging features: +/// +/// ## RPC Consensus Client +/// +/// When `--debug.rpc-consensus-ws ` is provided, the launcher will: +/// - Connect to an external RPC endpoint (`WebSocket` or HTTP) +/// - Fetch blocks from that endpoint (using subscriptions for `WebSocket`, polling for HTTP) +/// - Submit them to the local engine for execution +/// - Useful for testing engine behavior with real network data +/// +/// ## Etherscan Consensus Client +/// +/// When `--debug.etherscan [URL]` is provided, the launcher will: +/// - Use Etherscan API as a consensus client +/// - Fetch recent blocks from Etherscan +/// - Submit them to the local engine +/// - Requires `ETHERSCAN_API_KEY` environment variable +/// - Falls back to default Etherscan URL for the chain if URL not provided #[derive(Debug, Clone)] pub struct DebugNodeLauncher { inner: L, @@ -31,23 +111,61 @@ impl DebugNodeLauncher { } } -impl LaunchNode for DebugNodeLauncher +/// Future for the [`DebugNodeLauncher`]. +#[expect(missing_debug_implementations, clippy::type_complexity)] +pub struct DebugNodeLauncherFuture +where + N: FullNodeComponents>, +{ + inner: L, + target: Target, + local_payload_attributes_builder: + Option>>>, + map_attributes: + Option) -> PayloadAttrTy + Send + Sync>>, +} + +impl DebugNodeLauncherFuture where N: FullNodeComponents>, AddOns: RethRpcAddOns, L: LaunchNode>, { - type Node = NodeHandle; + pub fn with_payload_attributes_builder( + self, + builder: impl PayloadAttributesBuilder>, + ) -> Self { + Self { + inner: self.inner, + target: self.target, + local_payload_attributes_builder: Some(Box::new(builder)), + map_attributes: None, + } + } + + pub fn map_debug_payload_attributes( + self, + f: impl Fn(PayloadAttrTy) -> PayloadAttrTy + Send + Sync + 'static, + ) -> Self { + Self { + inner: self.inner, + target: self.target, + local_payload_attributes_builder: None, + map_attributes: Some(Box::new(f)), + } + } - async fn launch_node(self, target: Target) -> eyre::Result { - let handle = self.inner.launch_node(target).await?; + async fn launch_node(self) -> eyre::Result> { + let Self { inner, target, local_payload_attributes_builder, map_attributes } = self; + + let handle = inner.launch_node(target).await?; let config = &handle.node.config; - if let Some(ws_url) = config.debug.rpc_consensus_ws.clone() { - info!(target: "reth::cli", "Using RPC WebSocket consensus client: {}", ws_url); + if let Some(url) = config.debug.rpc_consensus_url.clone() { + info!(target: "reth::cli", "Using RPC consensus client: {}", url); let block_provider = - RpcBlockProvider::::new(ws_url.as_str(), |block_response| { + RpcBlockProvider::::new(url.as_str(), |block_response| { let json = serde_json::to_value(block_response) .expect("Block serialization cannot fail"); let rpc_block = @@ -66,7 +184,6 @@ where }); } - // TODO: migrate to devmode with https://github.com/paradigmxyz/reth/issues/10104 if let Some(maybe_custom_etherscan_url) = config.debug.etherscan.clone() { info!(target: "reth::cli", "Using etherscan as consensus client"); @@ -86,6 +203,7 @@ where "etherscan api key not found for rpc consensus client for chain: {chain}" ) })?, + chain.id(), N::Types::rpc_to_primitive_block, ); let rpc_consensus_client = DebugConsensusClient::new( @@ -97,6 +215,76 @@ where }); } + if config.dev.dev { + info!(target: "reth::cli", "Using local payload attributes builder for dev mode"); + + let blockchain_db = handle.node.provider.clone(); + let chain_spec = config.chain.clone(); + let beacon_engine_handle = handle.node.add_ons_handle.beacon_engine_handle.clone(); + let pool = handle.node.pool.clone(); + let payload_builder_handle = handle.node.payload_builder_handle.clone(); + + let builder = if let Some(builder) = local_payload_attributes_builder { + Either::Left(builder) + } else { + let local = N::Types::local_payload_attributes_builder(&chain_spec); + let builder = if let Some(f) = map_attributes { + Either::Left(move |block_number| f(local.build(block_number))) + } else { + Either::Right(local) + }; + Either::Right(builder) + }; + + let dev_mining_mode = handle.node.config.dev_mining_mode(pool); + handle.node.task_executor.spawn_critical("local engine", async move { + LocalMiner::new( + blockchain_db, + builder, + beacon_engine_handle, + dev_mining_mode, + payload_builder_handle, + ) + .run() + .await + }); + } + Ok(handle) } } + +impl IntoFuture for DebugNodeLauncherFuture +where + Target: Send + 'static, + N: FullNodeComponents>, + AddOns: RethRpcAddOns + 'static, + L: LaunchNode> + 'static, +{ + type Output = eyre::Result>; + type IntoFuture = Pin>> + Send>>; + + fn into_future(self) -> Self::IntoFuture { + Box::pin(self.launch_node()) + } +} + +impl LaunchNode for DebugNodeLauncher +where + Target: Send + 'static, + N: FullNodeComponents>, + AddOns: RethRpcAddOns + 'static, + L: LaunchNode> + 'static, +{ + type Node = NodeHandle; + type Future = DebugNodeLauncherFuture; + + fn launch_node(self, target: Target) -> Self::Future { + DebugNodeLauncherFuture { + inner: self.inner, + target, + local_payload_attributes_builder: None, + map_attributes: None, + } + } +} diff --git a/crates/node/builder/src/launch/engine.rs b/crates/node/builder/src/launch/engine.rs index 09de91a1eff..ffe07aaac88 100644 --- a/crates/node/builder/src/launch/engine.rs +++ b/crates/node/builder/src/launch/engine.rs @@ -1,10 +1,16 @@ //! Engine node related functionality. +use crate::{ + common::{Attached, LaunchContextWith, WithConfigs}, + hooks::NodeHooks, + rpc::{EngineValidatorAddOn, EngineValidatorBuilder, RethRpcAddOns, RpcHandle}, + setup::build_networked_pipeline, + AddOns, AddOnsContext, FullNode, LaunchContext, LaunchNode, NodeAdapter, + NodeBuilderWithComponents, NodeComponents, NodeComponentsBuilder, NodeHandle, NodeTypesAdapter, +}; use alloy_consensus::BlockHeader; -use futures::{future::Either, stream, stream_select, StreamExt}; +use futures::{stream_select, StreamExt}; use reth_chainspec::{EthChainSpec, EthereumHardforks}; -use reth_db_api::{database_metrics::DatabaseMetrics, Database}; -use reth_engine_local::{LocalEngineService, LocalPayloadAttributesBuilder}; use reth_engine_service::service::{ChainEvent, EngineService}; use reth_engine_tree::{ engine::{EngineApiRequest, EngineRequestHandler}, @@ -12,35 +18,28 @@ use reth_engine_tree::{ }; use reth_engine_util::EngineMessageStreamExt; use reth_exex::ExExManagerHandle; -use reth_network::{NetworkSyncUpdater, SyncState}; +use reth_network::{types::BlockRangeUpdate, NetworkSyncUpdater, SyncState}; use reth_network_api::BlockDownloaderProvider; use reth_node_api::{ - BeaconConsensusEngineHandle, BuiltPayload, FullNodeTypes, NodeTypes, NodeTypesWithDBAdapter, - PayloadAttributesBuilder, PayloadTypes, + BuiltPayload, ConsensusEngineHandle, FullNodeTypes, NodeTypes, NodeTypesWithDBAdapter, }; use reth_node_core::{ dirs::{ChainPath, DataDirPath}, exit::NodeExitFuture, primitives::Head, }; -use reth_node_events::{cl::ConsensusLayerHealthEvents, node}; -use reth_provider::providers::{BlockchainProvider, NodeTypesForProvider}; +use reth_node_events::node; +use reth_provider::{ + providers::{BlockchainProvider, NodeTypesForProvider}, + BlockNumReader, +}; use reth_tasks::TaskExecutor; use reth_tokio_util::EventSender; use reth_tracing::tracing::{debug, error, info}; -use std::sync::Arc; +use std::{future::Future, pin::Pin, sync::Arc}; use tokio::sync::{mpsc::unbounded_channel, oneshot}; use tokio_stream::wrappers::UnboundedReceiverStream; -use crate::{ - common::{Attached, LaunchContextWith, WithConfigs}, - hooks::NodeHooks, - rpc::{EngineValidatorAddOn, RethRpcAddOns, RpcHandle}, - setup::build_networked_pipeline, - AddOns, AddOnsContext, ExExLauncher, FullNode, LaunchContext, LaunchNode, NodeAdapter, - NodeBuilderWithComponents, NodeComponents, NodeComponentsBuilder, NodeHandle, NodeTypesAdapter, -}; - /// The engine node launcher. #[derive(Debug)] pub struct EngineNodeLauncher { @@ -61,30 +60,22 @@ impl EngineNodeLauncher { ) -> Self { Self { ctx: LaunchContext::new(task_executor, data_dir), engine_tree_config } } -} -impl LaunchNode> for EngineNodeLauncher -where - Types: NodeTypesForProvider + NodeTypes, - DB: Database + DatabaseMetrics + Clone + Unpin + 'static, - T: FullNodeTypes< - Types = Types, - DB = DB, - Provider = BlockchainProvider>, - >, - CB: NodeComponentsBuilder, - AO: RethRpcAddOns> - + EngineValidatorAddOn>, - LocalPayloadAttributesBuilder: PayloadAttributesBuilder< - <::Payload as PayloadTypes>::PayloadAttributes, - >, -{ - type Node = NodeHandle, AO>; - - async fn launch_node( + async fn launch_node( self, target: NodeBuilderWithComponents, - ) -> eyre::Result { + ) -> eyre::Result, AO>> + where + T: FullNodeTypes< + Types: NodeTypesForProvider, + Provider = BlockchainProvider< + NodeTypesWithDBAdapter<::Types, ::DB>, + >, + >, + CB: NodeComponentsBuilder, + AO: RethRpcAddOns> + + EngineValidatorAddOn>, + { let Self { ctx, engine_tree_config } = self; let NodeBuilderWithComponents { adapter: NodeTypesAdapter { database }, @@ -115,7 +106,7 @@ where debug!(target: "reth::cli", chain=%this.chain_id(), genesis=?this.genesis_hash(), "Initializing genesis"); }) .with_genesis()? - .inspect(|this: &LaunchContextWith, _>>| { + .inspect(|this: &LaunchContextWith::ChainSpec>, _>>| { info!(target: "reth::cli", "\n{}", this.chain_spec().display_hardforks()); }) .with_metrics_task() @@ -126,22 +117,19 @@ where })? .with_components(components_builder, on_component_initialized).await?; - // spawn exexs - let exex_manager_handle = ExExLauncher::new( - ctx.head(), - ctx.node_adapter().clone(), - installed_exex, - ctx.configs().clone(), - ) - .launch() - .await?; + // spawn exexs if any + let maybe_exex_manager_handle = ctx.launch_exex(installed_exex).await?; // create pipeline - let network_client = ctx.components().network().fetch_client().await?; + let network_handle = ctx.components().network().clone(); + let network_client = network_handle.fetch_client().await?; let (consensus_engine_tx, consensus_engine_rx) = unbounded_channel(); let node_config = ctx.node_config(); + // We always assume that node is syncing after a restart + network_handle.update_sync_state(SyncState::Syncing); + let max_block = ctx.max_block(network_client.clone()).await?; let static_file_producer = ctx.static_file_producer(); @@ -150,9 +138,6 @@ where let consensus = Arc::new(ctx.components().consensus().clone()); - // Configure the pipeline - let pipeline_exex_handle = - exex_manager_handle.clone().unwrap_or_else(ExExManagerHandle::empty); let pipeline = build_networked_pipeline( &ctx.toml_config().stages, network_client.clone(), @@ -164,7 +149,8 @@ where max_block, static_file_producer, ctx.components().evm_config().clone(), - pipeline_exex_handle, + maybe_exex_manager_handle.clone().unwrap_or_else(ExExManagerHandle::empty), + ctx.era_import_source(), )?; // The new engine writes directly to static files. This ensures that they're up to the tip. @@ -173,17 +159,17 @@ where let pipeline_events = pipeline.events(); let mut pruner_builder = ctx.pruner_builder(); - if let Some(exex_manager_handle) = &exex_manager_handle { + if let Some(exex_manager_handle) = &maybe_exex_manager_handle { pruner_builder = pruner_builder.finished_exex_height(exex_manager_handle.finished_height()); } let pruner = pruner_builder.build_with_provider_factory(ctx.provider_factory().clone()); let pruner_events = pruner.events(); - info!(target: "reth::cli", prune_config=?ctx.prune_config().unwrap_or_default(), "Pruner initialized"); + info!(target: "reth::cli", prune_config=?ctx.prune_config(), "Pruner initialized"); let event_sender = EventSender::default(); - let beacon_engine_handle = BeaconConsensusEngineHandle::new(consensus_engine_tx.clone()); + let beacon_engine_handle = ConsensusEngineHandle::new(consensus_engine_tx.clone()); // extract the jwt secret from the args if possible let jwt_secret = ctx.auth_jwt_secret()?; @@ -195,88 +181,66 @@ where jwt_secret, engine_events: event_sender.clone(), }; - let engine_payload_validator = add_ons.engine_validator(&add_ons_ctx).await?; + let validator_builder = add_ons.engine_validator_builder(); + // Build the engine validator with all required components + let engine_validator = validator_builder + .clone() + .build_tree_validator(&add_ons_ctx, engine_tree_config.clone()) + .await?; + + // Create the consensus engine stream with optional reorg let consensus_engine_stream = UnboundedReceiverStream::from(consensus_engine_rx) .maybe_skip_fcu(node_config.debug.skip_fcu) .maybe_skip_new_payload(node_config.debug.skip_new_payload) .maybe_reorg( ctx.blockchain_db().clone(), ctx.components().evm_config().clone(), - engine_payload_validator.clone(), + || validator_builder.build_tree_validator(&add_ons_ctx, engine_tree_config.clone()), node_config.debug.reorg_frequency, node_config.debug.reorg_depth, ) + .await? // Store messages _after_ skipping so that `replay-engine` command // would replay only the messages that were observed by the engine // during this run. .maybe_store_messages(node_config.debug.engine_api_store.clone()); - let mut engine_service = if ctx.is_dev() { - let eth_service = LocalEngineService::new( - consensus.clone(), - ctx.provider_factory().clone(), - ctx.blockchain_db().clone(), - pruner, - ctx.components().payload_builder_handle().clone(), - engine_payload_validator, - engine_tree_config, - ctx.invalid_block_hook()?, - ctx.sync_metrics_tx(), - consensus_engine_tx.clone(), - Box::pin(consensus_engine_stream), - ctx.dev_mining_mode(ctx.components().pool()), - LocalPayloadAttributesBuilder::new(ctx.chain_spec()), - ctx.components().evm_config().clone(), - ); - - Either::Left(eth_service) - } else { - let eth_service = EngineService::new( - consensus.clone(), - ctx.chain_spec(), - network_client.clone(), - Box::pin(consensus_engine_stream), - pipeline, - Box::new(ctx.task_executor().clone()), - ctx.provider_factory().clone(), - ctx.blockchain_db().clone(), - pruner, - ctx.components().payload_builder_handle().clone(), - engine_payload_validator, - engine_tree_config, - ctx.invalid_block_hook()?, - ctx.sync_metrics_tx(), - ctx.components().evm_config().clone(), - ); - - Either::Right(eth_service) - }; + let mut engine_service = EngineService::new( + consensus.clone(), + ctx.chain_spec(), + network_client.clone(), + Box::pin(consensus_engine_stream), + pipeline, + Box::new(ctx.task_executor().clone()), + ctx.provider_factory().clone(), + ctx.blockchain_db().clone(), + pruner, + ctx.components().payload_builder_handle().clone(), + engine_validator, + engine_tree_config, + ctx.sync_metrics_tx(), + ctx.components().evm_config().clone(), + ); info!(target: "reth::cli", "Consensus engine initialized"); + #[allow(clippy::needless_continue)] let events = stream_select!( event_sender.new_listener().map(Into::into), pipeline_events.map(Into::into), - if ctx.node_config().debug.tip.is_none() && !ctx.is_dev() { - Either::Left( - ConsensusLayerHealthEvents::new(Box::new(ctx.blockchain_db().clone())) - .map(Into::into), - ) - } else { - Either::Right(stream::empty()) - }, + ctx.consensus_layer_events(), pruner_events.map(Into::into), static_file_producer_events.map(Into::into), ); ctx.task_executor().spawn_critical( "events task", - node::handle_events( + Box::pin(node::handle_events( Some(Box::new(ctx.components().network().clone())), Some(ctx.head().number), events, - ), + )), ); let RpcHandle { rpc_server_handles, rpc_registry, engine_events, beacon_engine_handle } = @@ -284,7 +248,6 @@ where // Run consensus engine to completion let initial_target = ctx.initial_backfill_target()?; - let network_handle = ctx.components().network().clone(); let mut built_payloads = ctx .components() .payload_builder_handle() @@ -293,17 +256,21 @@ where .map_err(|e| eyre::eyre!("Failed to subscribe to payload builder events: {:?}", e))? .into_built_payload_stream() .fuse(); + let chainspec = ctx.chain_spec(); + let provider = ctx.blockchain_db().clone(); let (exit, rx) = oneshot::channel(); let terminate_after_backfill = ctx.terminate_after_initial_backfill(); + let startup_sync_state_idle = ctx.node_config().debug.startup_sync_state_idle; info!(target: "reth::cli", "Starting consensus engine"); - ctx.task_executor().spawn_critical("consensus engine", async move { + ctx.task_executor().spawn_critical("consensus engine", Box::pin(async move { if let Some(initial_target) = initial_target { debug!(target: "reth::cli", %initial_target, "start backfill sync"); - if let Either::Right(eth_service) = &mut engine_service { - eth_service.orchestrator_mut().start_backfill_sync(initial_target); - } + // network_handle's sync state is already initialized at Syncing + engine_service.orchestrator_mut().start_backfill_sync(initial_target); + } else if startup_sync_state_idle { + network_handle.update_sync_state(SyncState::Idle); } let mut res = Ok(()); @@ -314,9 +281,7 @@ where payload = built_payloads.select_next_some() => { if let Some(executed_block) = payload.executed_block() { debug!(target: "reth::cli", block=?executed_block.recovered_block().num_hash(), "inserting built payload"); - if let Either::Right(eth_service) = &mut engine_service { - eth_service.orchestrator_mut().handler_mut().handler_mut().on_event(EngineApiRequest::InsertExecutedBlock(executed_block).into()); - } + engine_service.orchestrator_mut().handler_mut().handler_mut().on_event(EngineApiRequest::InsertExecutedBlock(executed_block).into()); } } event = engine_service.next() => { @@ -328,8 +293,9 @@ where debug!(target: "reth::cli", "Terminating after initial backfill"); break } - - network_handle.update_sync_state(SyncState::Idle); + if startup_sync_state_idle { + network_handle.update_sync_state(SyncState::Idle); + } } ChainEvent::BackfillSyncStarted => { network_handle.update_sync_state(SyncState::Syncing); @@ -341,7 +307,9 @@ where } ChainEvent::Handler(ev) => { if let Some(head) = ev.canonical_header() { - let head_block = Head { + // Once we're progressing via live sync, we can consider the node is not syncing anymore + network_handle.update_sync_state(SyncState::Idle); + let head_block = Head { number: head.number(), hash: head.hash(), difficulty: head.difficulty(), @@ -349,6 +317,13 @@ where total_difficulty: chainspec.final_paris_total_difficulty().filter(|_| chainspec.is_paris_active_at_block(head.number())).unwrap_or_default(), }; network_handle.update_status(head_block); + + let updated = BlockRangeUpdate { + earliest: provider.earliest_block_number().unwrap_or_default(), + latest:head.number(), + latest_hash:head.hash() + }; + network_handle.update_block_range(updated); } event_sender.notify(ev); } @@ -358,7 +333,7 @@ where } let _ = exit.send(res); - }); + })); let full_node = FullNode { evm_config: ctx.components().evm_config().clone(), @@ -379,6 +354,8 @@ where // Notify on node started on_node_started.on_event(FullNode::clone(&full_node))?; + ctx.spawn_ethstats().await?; + let handle = NodeHandle { node_exit_future: NodeExitFuture::new( async { rx.await? }, @@ -390,3 +367,24 @@ where Ok(handle) } } + +impl LaunchNode> for EngineNodeLauncher +where + T: FullNodeTypes< + Types: NodeTypesForProvider, + Provider = BlockchainProvider< + NodeTypesWithDBAdapter<::Types, ::DB>, + >, + >, + CB: NodeComponentsBuilder + 'static, + AO: RethRpcAddOns> + + EngineValidatorAddOn> + + 'static, +{ + type Node = NodeHandle, AO>; + type Future = Pin> + Send>>; + + fn launch_node(self, target: NodeBuilderWithComponents) -> Self::Future { + Box::pin(self.launch_node(target)) + } +} diff --git a/crates/node/builder/src/launch/exex.rs b/crates/node/builder/src/launch/exex.rs index b40c9c8b612..e757cda8770 100644 --- a/crates/node/builder/src/launch/exex.rs +++ b/crates/node/builder/src/launch/exex.rs @@ -90,7 +90,7 @@ impl ExExLauncher { let span = reth_tracing::tracing::info_span!("exex", id); // init the exex - let exex = exex.launch(context).instrument(span.clone()).await.unwrap(); + let exex = exex.launch(context).instrument(span.clone()).await?; // spawn it as a crit task executor.spawn_critical( @@ -104,10 +104,12 @@ impl ExExLauncher { } .instrument(span), ); + + Ok::<(), eyre::Error>(()) }); } - future::join_all(exexes).await; + future::try_join_all(exexes).await?; // spawn exex manager debug!(target: "reth::cli", "spawning exex manager"); diff --git a/crates/node/builder/src/launch/invalid_block_hook.rs b/crates/node/builder/src/launch/invalid_block_hook.rs new file mode 100644 index 00000000000..3c1848dceb4 --- /dev/null +++ b/crates/node/builder/src/launch/invalid_block_hook.rs @@ -0,0 +1,148 @@ +//! Invalid block hook helpers for the node builder. + +use crate::AddOnsContext; +use alloy_consensus::TxEnvelope; +use alloy_rpc_types::{Block, Header, Receipt, Transaction, TransactionRequest}; +use eyre::OptionExt; +use reth_chainspec::EthChainSpec; +use reth_engine_primitives::InvalidBlockHook; +use reth_node_api::{FullNodeComponents, NodeTypes}; +use reth_node_core::{ + args::InvalidBlockHookType, + dirs::{ChainPath, DataDirPath}, + node_config::NodeConfig, +}; +use reth_primitives_traits::NodePrimitives; +use reth_provider::ChainSpecProvider; +use reth_rpc_api::EthApiClient; + +/// Extension trait for [`AddOnsContext`] to create invalid block hooks. +pub trait InvalidBlockHookExt { + /// Node primitives type. + type Primitives: NodePrimitives; + + /// Creates an invalid block hook based on the node configuration. + fn create_invalid_block_hook( + &self, + data_dir: &ChainPath, + ) -> impl std::future::Future>>> + + Send; +} + +impl InvalidBlockHookExt for AddOnsContext<'_, N> +where + N: FullNodeComponents, +{ + type Primitives = ::Primitives; + + async fn create_invalid_block_hook( + &self, + data_dir: &ChainPath, + ) -> eyre::Result>> { + create_invalid_block_hook( + self.config, + data_dir, + self.node.provider().clone(), + self.node.evm_config().clone(), + self.node.provider().chain_spec().chain().id(), + ) + .await + } +} + +/// Creates an invalid block hook based on the node configuration. +/// +/// This function constructs the appropriate [`InvalidBlockHook`] based on the debug +/// configuration in the node config. It supports: +/// - Witness hooks for capturing block witness data +/// - Healthy node verification via RPC +/// +/// # Arguments +/// * `config` - The node configuration containing debug settings +/// * `data_dir` - The data directory for storing hook outputs +/// * `provider` - The blockchain database provider +/// * `evm_config` - The EVM configuration +/// * `chain_id` - The chain ID for verification +pub async fn create_invalid_block_hook( + config: &NodeConfig, + data_dir: &ChainPath, + provider: P, + evm_config: E, + chain_id: u64, +) -> eyre::Result>> +where + N: NodePrimitives, + P: reth_provider::StateProviderFactory + + reth_provider::ChainSpecProvider + + Clone + + Send + + Sync + + 'static, + E: reth_evm::ConfigureEvm + Clone + 'static, +{ + use reth_engine_primitives::{InvalidBlockHooks, NoopInvalidBlockHook}; + use reth_invalid_block_hooks::InvalidBlockWitnessHook; + + let Some(ref hook) = config.debug.invalid_block_hook else { + return Ok(Box::new(NoopInvalidBlockHook::default())) + }; + + let healthy_node_rpc_client = get_healthy_node_client(config, chain_id).await?; + + let output_directory = data_dir.invalid_block_hooks(); + let hooks = hook + .iter() + .copied() + .map(|hook| { + let output_directory = output_directory.join(hook.to_string()); + std::fs::create_dir_all(&output_directory)?; + + Ok(match hook { + InvalidBlockHookType::Witness => Box::new(InvalidBlockWitnessHook::new( + provider.clone(), + evm_config.clone(), + output_directory, + healthy_node_rpc_client.clone(), + )), + InvalidBlockHookType::PreState | InvalidBlockHookType::Opcode => { + eyre::bail!("invalid block hook {hook:?} is not implemented yet") + } + } as Box>) + }) + .collect::>()?; + + Ok(Box::new(InvalidBlockHooks(hooks))) +} + +/// Returns an RPC client for the healthy node, if configured in the node config. +async fn get_healthy_node_client( + config: &NodeConfig, + chain_id: u64, +) -> eyre::Result> +where + C: EthChainSpec, +{ + let Some(url) = config.debug.healthy_node_rpc_url.as_ref() else { + return Ok(None); + }; + + let client = jsonrpsee::http_client::HttpClientBuilder::default().build(url)?; + + // Verify that the healthy node is running the same chain as the current node. + let healthy_chain_id = EthApiClient::< + TransactionRequest, + Transaction, + Block, + Receipt, + Header, + TxEnvelope, + >::chain_id(&client) + .await? + .ok_or_eyre("healthy node rpc client didn't return a chain id")?; + + if healthy_chain_id.to::() != chain_id { + eyre::bail!("Invalid chain ID. Expected {}, got {}", chain_id, healthy_chain_id); + } + + Ok(Some(client)) +} diff --git a/crates/node/builder/src/launch/mod.rs b/crates/node/builder/src/launch/mod.rs index 2f770e69564..cc6b1927d82 100644 --- a/crates/node/builder/src/launch/mod.rs +++ b/crates/node/builder/src/launch/mod.rs @@ -2,6 +2,7 @@ pub mod common; mod exex; +pub mod invalid_block_hook; pub(crate) mod debug; pub(crate) mod engine; @@ -9,7 +10,7 @@ pub(crate) mod engine; pub use common::LaunchContext; pub use exex::ExExLauncher; -use std::future::Future; +use std::future::IntoFuture; /// A general purpose trait that launches a new node of any kind. /// @@ -20,22 +21,26 @@ use std::future::Future; /// /// See also [`EngineNodeLauncher`](crate::EngineNodeLauncher) and /// [`NodeBuilderWithComponents::launch_with`](crate::NodeBuilderWithComponents) -pub trait LaunchNode { +pub trait LaunchNode: Send { /// The node type that is created. type Node; + /// The future type that is returned. + type Future: IntoFuture, IntoFuture: Send>; + /// Create and return a new node asynchronously. - fn launch_node(self, target: Target) -> impl Future>; + fn launch_node(self, target: Target) -> Self::Future; } impl LaunchNode for F where F: FnOnce(Target) -> Fut + Send, - Fut: Future> + Send, + Fut: IntoFuture, IntoFuture: Send> + Send, { type Node = Node; + type Future = Fut; - fn launch_node(self, target: Target) -> impl Future> { + fn launch_node(self, target: Target) -> Self::Future { self(target) } } diff --git a/crates/node/builder/src/lib.rs b/crates/node/builder/src/lib.rs index d2e630231b3..1218465e95e 100644 --- a/crates/node/builder/src/lib.rs +++ b/crates/node/builder/src/lib.rs @@ -9,7 +9,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] /// Node event hooks. pub mod hooks; @@ -18,6 +18,10 @@ pub mod hooks; pub mod node; pub use node::*; +/// Support for accessing the EngineApi outside the RPC server context. +mod engine_api_ext; +pub use engine_api_ext::EngineApiExt; + /// Support for configuring the components of a node. pub mod components; pub use components::{NodeComponents, NodeComponentsBuilder}; diff --git a/crates/node/builder/src/node.rs b/crates/node/builder/src/node.rs index 966b1227629..1cc50c4ba6f 100644 --- a/crates/node/builder/src/node.rs +++ b/crates/node/builder/src/node.rs @@ -1,7 +1,11 @@ +use reth_db::DatabaseEnv; // re-export the node api types pub use reth_node_api::{FullNodeTypes, NodeTypes}; -use crate::{components::NodeComponentsBuilder, rpc::RethRpcAddOns, NodeAdapter, NodeAddOns}; +use crate::{ + components::NodeComponentsBuilder, rpc::RethRpcAddOns, NodeAdapter, NodeAddOns, NodeHandle, + RethFullAdapter, +}; use reth_node_api::{EngineTypes, FullNodeComponents, PayloadTypes}; use reth_node_core::{ dirs::{ChainPath, DataDirPath}, @@ -19,6 +23,10 @@ use std::{ sync::Arc, }; +/// A helper type to obtain components for a given node when [`FullNodeTypes::Types`] is a [`Node`] +/// implementation. +pub type ComponentsFor = <<::Types as Node>::ComponentsBuilder as NodeComponentsBuilder>::Components; + /// A [`crate::Node`] is a [`NodeTypes`] that comes with preconfigured components. /// /// This can be used to configure the builder with a preset of components. @@ -69,8 +77,6 @@ where type ChainSpec = ::ChainSpec; - type StateCommitment = ::StateCommitment; - type Storage = ::Storage; type Payload = ::Payload; @@ -173,14 +179,16 @@ where /// Returns the [`EngineApiClient`] interface for the authenticated engine API. /// /// This will send authenticated http requests to the node's auth server. - pub fn engine_http_client(&self) -> impl EngineApiClient { + pub fn engine_http_client(&self) -> impl EngineApiClient + use { self.auth_server_handle().http_client() } /// Returns the [`EngineApiClient`] interface for the authenticated engine API. /// /// This will send authenticated ws requests to the node's auth server. - pub async fn engine_ws_client(&self) -> impl EngineApiClient { + pub async fn engine_ws_client( + &self, + ) -> impl EngineApiClient + use { self.auth_server_handle().ws_client().await } @@ -188,7 +196,9 @@ where /// /// This will send not authenticated IPC requests to the node's auth server. #[cfg(unix)] - pub async fn engine_ipc_client(&self) -> Option> { + pub async fn engine_ipc_client( + &self, + ) -> Option + use> { self.auth_server_handle().ipc_client().await } } @@ -206,3 +216,11 @@ impl> DerefMut for FullNode> = + FullNode>, >>::AddOns>; + +/// Helper type alias to define [`NodeHandle`] for a given [`Node`]. +pub type NodeHandleFor> = + NodeHandle>, >>::AddOns>; diff --git a/crates/node/builder/src/rpc.rs b/crates/node/builder/src/rpc.rs index 2fe2b63e554..a66d7b222e4 100644 --- a/crates/node/builder/src/rpc.rs +++ b/crates/node/builder/src/rpc.rs @@ -1,31 +1,37 @@ //! Builder support for rpc components. -use crate::{BeaconConsensusEngineEvent, BeaconConsensusEngineHandle}; +pub use jsonrpsee::server::middleware::rpc::{RpcService, RpcServiceBuilder}; +pub use reth_engine_tree::tree::{BasicEngineValidator, EngineValidator}; +pub use reth_rpc_builder::{middleware::RethRpcMiddleware, Identity, Stack}; + +use crate::{ + invalid_block_hook::InvalidBlockHookExt, ConfigureEngineEvm, ConsensusEngineEvent, + ConsensusEngineHandle, +}; use alloy_rpc_types::engine::ClientVersionV1; use alloy_rpc_types_engine::ExecutionData; -use futures::TryFutureExt; -use jsonrpsee::RpcModule; +use jsonrpsee::{core::middleware::layer::Either, RpcModule}; use reth_chain_state::CanonStateSubscriptions; -use reth_chainspec::{ChainSpecProvider, EthereumHardforks}; +use reth_chainspec::{ChainSpecProvider, EthChainSpec, EthereumHardforks, Hardforks}; use reth_node_api::{ - AddOnsContext, BlockTy, EngineTypes, EngineValidator, FullNodeComponents, FullNodeTypes, - NodeAddOns, NodeTypes, PayloadTypes, ReceiptTy, + AddOnsContext, BlockTy, EngineApiValidator, EngineTypes, FullNodeComponents, FullNodeTypes, + NodeAddOns, NodeTypes, PayloadTypes, PayloadValidator, PrimitivesTy, TreeConfig, }; use reth_node_core::{ + cli::config::RethTransactionPoolConfig, node_config::NodeConfig, - version::{CARGO_PKG_VERSION, CLIENT_CODE, NAME_CLIENT, VERGEN_GIT_SHA}, + version::{version_metadata, CLIENT_CODE}, }; use reth_payload_builder::{PayloadBuilderHandle, PayloadStore}; -use reth_rpc::eth::{EthApiTypes, FullEthApiServer}; -use reth_rpc_api::{eth::helpers::AddDevSigners, IntoEngineApiRpcModule}; +use reth_rpc::eth::{core::EthRpcConverterFor, DevSigner, EthApiTypes, FullEthApiServer}; +use reth_rpc_api::{eth::helpers::EthTransactions, IntoEngineApiRpcModule}; use reth_rpc_builder::{ auth::{AuthRpcModule, AuthServerHandle}, config::RethRpcServerConfig, - RpcModuleBuilder, RpcRegistryInner, RpcServerHandle, TransportRpcModules, + RpcModuleBuilder, RpcRegistryInner, RpcServerConfig, RpcServerHandle, TransportRpcModules, }; use reth_rpc_engine_api::{capabilities::EngineCapabilities, EngineApi}; use reth_rpc_eth_types::{cache::cache_new_blocks_task, EthConfig, EthStateCache}; -use reth_tasks::TaskExecutor; use reth_tokio_util::EventSender; use reth_tracing::tracing::{debug, info}; use std::{ @@ -194,7 +200,6 @@ pub struct RpcRegistry { Node::Provider, Node::Pool, Node::Network, - TaskExecutor, EthApi, Node::Evm, Node::Consensus, @@ -210,7 +215,6 @@ where Node::Provider, Node::Pool, Node::Network, - TaskExecutor, EthApi, Node::Evm, Node::Consensus, @@ -231,6 +235,17 @@ where } } +/// Helper container for the parameters commonly passed to RPC module extension functions. +#[expect(missing_debug_implementations)] +pub struct RpcModuleContainer<'a, Node: FullNodeComponents, EthApi: EthApiTypes> { + /// Holds installed modules per transport type. + pub modules: &'a mut TransportRpcModules, + /// Holds jwt authenticated rpc module. + pub auth_module: &'a mut AuthRpcModule, + /// A Helper type the holds instances of the configured modules. + pub registry: &'a mut RpcRegistry, +} + /// Helper container to encapsulate [`RpcRegistryInner`], [`TransportRpcModules`] and /// [`AuthRpcModule`]. /// @@ -272,6 +287,8 @@ where } /// Returns a reference to the configured node. + /// + /// This gives access to the node's components. pub const fn node(&self) -> &Node { &self.node } @@ -309,10 +326,9 @@ pub struct RpcHandle { /// /// Caution: This is a multi-producer, multi-consumer broadcast and allows grants access to /// dispatch events - pub engine_events: - EventSender::Primitives>>, + pub engine_events: EventSender::Primitives>>, /// Handle to the beacon consensus engine. - pub beacon_engine_handle: BeaconConsensusEngineHandle<::Payload>, + pub beacon_engine_handle: ConsensusEngineHandle<::Payload>, } impl Clone for RpcHandle { @@ -346,6 +362,117 @@ where } } +impl RpcHandle { + /// Returns the RPC server handles. + pub const fn rpc_server_handles(&self) -> &RethRpcServerHandles { + &self.rpc_server_handles + } + + /// Returns the consensus engine handle. + /// + /// This handle can be used to interact with the engine service directly. + pub const fn consensus_engine_handle( + &self, + ) -> &ConsensusEngineHandle<::Payload> { + &self.beacon_engine_handle + } + + /// Returns the consensus engine events sender. + pub const fn consensus_engine_events( + &self, + ) -> &EventSender::Primitives>> { + &self.engine_events + } +} + +/// Handle returned when only the regular RPC server (HTTP/WS/IPC) is launched. +/// +/// This handle provides access to the RPC server endpoints and registry, but does not +/// include an authenticated Engine API server. Use this when you only need regular +/// RPC functionality. +#[derive(Debug, Clone)] +pub struct RpcServerOnlyHandle { + /// Handle to the RPC server + pub rpc_server_handle: RpcServerHandle, + /// Configured RPC modules. + pub rpc_registry: RpcRegistry, + /// Notification channel for engine API events + pub engine_events: EventSender::Primitives>>, + /// Handle to the consensus engine. + pub engine_handle: ConsensusEngineHandle<::Payload>, +} + +impl RpcServerOnlyHandle { + /// Returns the RPC server handle. + pub const fn rpc_server_handle(&self) -> &RpcServerHandle { + &self.rpc_server_handle + } + + /// Returns the consensus engine handle. + /// + /// This handle can be used to interact with the engine service directly. + pub const fn consensus_engine_handle( + &self, + ) -> &ConsensusEngineHandle<::Payload> { + &self.engine_handle + } + + /// Returns the consensus engine events sender. + pub const fn consensus_engine_events( + &self, + ) -> &EventSender::Primitives>> { + &self.engine_events + } +} + +/// Handle returned when only the authenticated Engine API server is launched. +/// +/// This handle provides access to the Engine API server and registry, but does not +/// include the regular RPC servers (HTTP/WS/IPC). Use this for specialized setups +/// that only need Engine API functionality. +#[derive(Debug, Clone)] +pub struct AuthServerOnlyHandle { + /// Handle to the auth server (engine API) + pub auth_server_handle: AuthServerHandle, + /// Configured RPC modules. + pub rpc_registry: RpcRegistry, + /// Notification channel for engine API events + pub engine_events: EventSender::Primitives>>, + /// Handle to the consensus engine. + pub engine_handle: ConsensusEngineHandle<::Payload>, +} + +impl AuthServerOnlyHandle { + /// Returns the consensus engine handle. + /// + /// This handle can be used to interact with the engine service directly. + pub const fn consensus_engine_handle( + &self, + ) -> &ConsensusEngineHandle<::Payload> { + &self.engine_handle + } + + /// Returns the consensus engine events sender. + pub const fn consensus_engine_events( + &self, + ) -> &EventSender::Primitives>> { + &self.engine_events + } +} + +/// Internal context struct for RPC setup shared between different launch methods +struct RpcSetupContext<'a, Node: FullNodeComponents, EthApi: EthApiTypes> { + node: Node, + config: &'a NodeConfig<::ChainSpec>, + modules: TransportRpcModules, + auth_module: AuthRpcModule, + auth_config: reth_rpc_builder::auth::AuthServerConfig, + registry: RpcRegistry, + on_rpc_started: Box>, + engine_events: EventSender::Primitives>>, + engine_handle: ConsensusEngineHandle<::Payload>, +} + /// Node add-ons containing RPC server configuration, with customizable eth API handler. /// /// This struct can be used to provide the RPC server functionality. It is responsible for launching @@ -359,37 +486,52 @@ where pub struct RpcAddOns< Node: FullNodeComponents, EthB: EthApiBuilder, - EV, - EB = BasicEngineApiBuilder, + PVB, + EB = BasicEngineApiBuilder, + EVB = BasicEngineValidatorBuilder, + RpcMiddleware = Identity, > { /// Additional RPC add-ons. pub hooks: RpcHooks, /// Builder for `EthApi` eth_api_builder: EthB, - /// Engine validator - engine_validator_builder: EV, + /// Payload validator builder + payload_validator_builder: PVB, /// Builder for `EngineApi` engine_api_builder: EB, + /// Builder for tree validator + engine_validator_builder: EVB, + /// Configurable RPC middleware stack. + /// + /// This middleware is applied to all RPC requests across all transports (HTTP, WS, IPC). + /// See [`RpcAddOns::with_rpc_middleware`] for more details. + rpc_middleware: RpcMiddleware, + /// Optional custom tokio runtime for the RPC server. + tokio_runtime: Option, } -impl Debug for RpcAddOns +impl Debug + for RpcAddOns where Node: FullNodeComponents, EthB: EthApiBuilder, - EV: Debug, + PVB: Debug, EB: Debug, + EVB: Debug, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("RpcAddOns") .field("hooks", &self.hooks) .field("eth_api_builder", &"...") - .field("engine_validator_builder", &self.engine_validator_builder) + .field("payload_validator_builder", &self.payload_validator_builder) .field("engine_api_builder", &self.engine_api_builder) + .field("engine_validator_builder", &self.engine_validator_builder) + .field("rpc_middleware", &"...") .finish() } } -impl RpcAddOns +impl RpcAddOns where Node: FullNodeComponents, EthB: EthApiBuilder, @@ -397,17 +539,219 @@ where /// Creates a new instance of the RPC add-ons. pub fn new( eth_api_builder: EthB, - engine_validator_builder: EV, + payload_validator_builder: PVB, engine_api_builder: EB, + engine_validator_builder: EVB, + rpc_middleware: RpcMiddleware, ) -> Self { Self { hooks: RpcHooks::default(), eth_api_builder, + payload_validator_builder, + engine_api_builder, engine_validator_builder, + rpc_middleware, + tokio_runtime: None, + } + } + + /// Maps the [`EngineApiBuilder`] builder type. + pub fn with_engine_api( + self, + engine_api_builder: T, + ) -> RpcAddOns { + let Self { + hooks, + eth_api_builder, + payload_validator_builder, + engine_validator_builder, + rpc_middleware, + tokio_runtime, + .. + } = self; + RpcAddOns { + hooks, + eth_api_builder, + payload_validator_builder, engine_api_builder, + engine_validator_builder, + rpc_middleware, + tokio_runtime, } } + /// Maps the [`PayloadValidatorBuilder`] builder type. + pub fn with_payload_validator( + self, + payload_validator_builder: T, + ) -> RpcAddOns { + let Self { + hooks, + eth_api_builder, + engine_api_builder, + engine_validator_builder, + rpc_middleware, + tokio_runtime, + .. + } = self; + RpcAddOns { + hooks, + eth_api_builder, + payload_validator_builder, + engine_api_builder, + engine_validator_builder, + rpc_middleware, + tokio_runtime, + } + } + + /// Maps the [`EngineValidatorBuilder`] builder type. + pub fn with_engine_validator( + self, + engine_validator_builder: T, + ) -> RpcAddOns { + let Self { + hooks, + eth_api_builder, + payload_validator_builder, + engine_api_builder, + rpc_middleware, + tokio_runtime, + .. + } = self; + RpcAddOns { + hooks, + eth_api_builder, + payload_validator_builder, + engine_api_builder, + engine_validator_builder, + rpc_middleware, + tokio_runtime, + } + } + + /// Sets the RPC middleware stack for processing RPC requests. + /// + /// This method configures a custom middleware stack that will be applied to all RPC requests + /// across HTTP, `WebSocket`, and IPC transports. The middleware is applied to the RPC service + /// layer, allowing you to intercept, modify, or enhance RPC request processing. + /// + /// + /// # How It Works + /// + /// The middleware uses the Tower ecosystem's `Layer` pattern. When an RPC server is started, + /// the configured middleware stack is applied to create a layered service that processes + /// requests in the order the layers were added. + /// + /// # Examples + /// + /// ```ignore + /// use reth_rpc_builder::{RpcServiceBuilder, RpcRequestMetrics}; + /// use tower::Layer; + /// + /// // Simple example with metrics + /// let metrics_layer = RpcRequestMetrics::new(metrics_recorder); + /// let with_metrics = rpc_addons.with_rpc_middleware( + /// RpcServiceBuilder::new().layer(metrics_layer) + /// ); + /// + /// // Composing multiple middleware layers + /// let middleware_stack = RpcServiceBuilder::new() + /// .layer(rate_limit_layer) + /// .layer(logging_layer) + /// .layer(metrics_layer); + /// let with_full_stack = rpc_addons.with_rpc_middleware(middleware_stack); + /// ``` + /// + /// # Notes + /// + /// - Middleware is applied to the RPC service layer, not the HTTP transport layer + /// - The default middleware is `Identity` (no-op), which passes through requests unchanged + /// - Middleware layers are applied in the order they are added via `.layer()` + pub fn with_rpc_middleware( + self, + rpc_middleware: T, + ) -> RpcAddOns { + let Self { + hooks, + eth_api_builder, + payload_validator_builder, + engine_api_builder, + engine_validator_builder, + tokio_runtime, + .. + } = self; + RpcAddOns { + hooks, + eth_api_builder, + payload_validator_builder, + engine_api_builder, + engine_validator_builder, + rpc_middleware, + tokio_runtime, + } + } + + /// Sets the tokio runtime for the RPC servers. + /// + /// Caution: This runtime must not be created from within asynchronous context. + pub fn with_tokio_runtime(self, tokio_runtime: Option) -> Self { + let Self { + hooks, + eth_api_builder, + payload_validator_builder, + engine_validator_builder, + engine_api_builder, + rpc_middleware, + .. + } = self; + Self { + hooks, + eth_api_builder, + payload_validator_builder, + engine_validator_builder, + engine_api_builder, + rpc_middleware, + tokio_runtime, + } + } + + /// Add a new layer `T` to the configured [`RpcServiceBuilder`]. + pub fn layer_rpc_middleware( + self, + layer: T, + ) -> RpcAddOns> { + let Self { + hooks, + eth_api_builder, + payload_validator_builder, + engine_api_builder, + engine_validator_builder, + rpc_middleware, + tokio_runtime, + } = self; + let rpc_middleware = Stack::new(rpc_middleware, layer); + RpcAddOns { + hooks, + eth_api_builder, + payload_validator_builder, + engine_api_builder, + engine_validator_builder, + rpc_middleware, + tokio_runtime, + } + } + + /// Optionally adds a new layer `T` to the configured [`RpcServiceBuilder`]. + #[expect(clippy::type_complexity)] + pub fn option_layer_rpc_middleware( + self, + layer: Option, + ) -> RpcAddOns>> { + let layer = layer.map(Either::Left).unwrap_or(Either::Right(Identity::new())); + self.layer_rpc_middleware(layer) + } + /// Sets the hook that is run once the rpc server is started. pub fn on_rpc_started(mut self, hook: F) -> Self where @@ -429,39 +773,182 @@ where } } -impl Default for RpcAddOns +impl Default for RpcAddOns where Node: FullNodeComponents, EthB: EthApiBuilder, EV: Default, EB: Default, + Engine: Default, { fn default() -> Self { - Self::new(EthB::default(), EV::default(), EB::default()) + Self::new( + EthB::default(), + EV::default(), + EB::default(), + Engine::default(), + Default::default(), + ) } } -impl RpcAddOns +impl RpcAddOns where N: FullNodeComponents, N::Provider: ChainSpecProvider, EthB: EthApiBuilder, - EV: EngineValidatorBuilder, EB: EngineApiBuilder, + EVB: EngineValidatorBuilder, + RpcMiddleware: RethRpcMiddleware, { + /// Launches only the regular RPC server (HTTP/WS/IPC), without the authenticated Engine API + /// server. + /// + /// This is useful when you only need the regular RPC functionality and want to avoid + /// starting the auth server. + pub async fn launch_rpc_server( + self, + ctx: AddOnsContext<'_, N>, + ext: F, + ) -> eyre::Result> + where + F: FnOnce(RpcModuleContainer<'_, N, EthB::EthApi>) -> eyre::Result<()>, + { + let rpc_middleware = self.rpc_middleware.clone(); + let tokio_runtime = self.tokio_runtime.clone(); + let setup_ctx = self.setup_rpc_components(ctx, ext).await?; + let RpcSetupContext { + node, + config, + mut modules, + mut auth_module, + auth_config: _, + mut registry, + on_rpc_started, + engine_events, + engine_handle, + } = setup_ctx; + + let server_config = config + .rpc + .rpc_server_config() + .set_rpc_middleware(rpc_middleware) + .with_tokio_runtime(tokio_runtime); + let rpc_server_handle = Self::launch_rpc_server_internal(server_config, &modules).await?; + + let handles = + RethRpcServerHandles { rpc: rpc_server_handle.clone(), auth: AuthServerHandle::noop() }; + Self::finalize_rpc_setup( + &mut registry, + &mut modules, + &mut auth_module, + &node, + config, + on_rpc_started, + handles, + )?; + + Ok(RpcServerOnlyHandle { + rpc_server_handle, + rpc_registry: registry, + engine_events, + engine_handle, + }) + } + /// Launches the RPC servers with the given context and an additional hook for extending - /// modules. + /// modules. Whether the auth server is launched depends on the CLI configuration. pub async fn launch_add_ons_with( self, ctx: AddOnsContext<'_, N>, ext: F, ) -> eyre::Result> where - F: FnOnce( - &mut TransportRpcModules, - &mut AuthRpcModule, - &mut RpcRegistry, - ) -> eyre::Result<()>, + F: FnOnce(RpcModuleContainer<'_, N, EthB::EthApi>) -> eyre::Result<()>, + { + // Check CLI config to determine if auth server should be disabled + let disable_auth = ctx.config.rpc.disable_auth_server; + self.launch_add_ons_with_opt_engine(ctx, ext, disable_auth).await + } + + /// Launches the RPC servers with the given context and an additional hook for extending + /// modules. Optionally disables the auth server based on the `disable_auth` parameter. + /// + /// When `disable_auth` is true, the auth server will not be started and a noop handle + /// will be used instead. + pub async fn launch_add_ons_with_opt_engine( + self, + ctx: AddOnsContext<'_, N>, + ext: F, + disable_auth: bool, + ) -> eyre::Result> + where + F: FnOnce(RpcModuleContainer<'_, N, EthB::EthApi>) -> eyre::Result<()>, + { + let rpc_middleware = self.rpc_middleware.clone(); + let tokio_runtime = self.tokio_runtime.clone(); + let setup_ctx = self.setup_rpc_components(ctx, ext).await?; + let RpcSetupContext { + node, + config, + mut modules, + mut auth_module, + auth_config, + mut registry, + on_rpc_started, + engine_events, + engine_handle, + } = setup_ctx; + + let server_config = config + .rpc + .rpc_server_config() + .set_rpc_middleware(rpc_middleware) + .with_tokio_runtime(tokio_runtime); + + let (rpc, auth) = if disable_auth { + // Only launch the RPC server, use a noop auth handle + let rpc = Self::launch_rpc_server_internal(server_config, &modules).await?; + (rpc, AuthServerHandle::noop()) + } else { + let auth_module_clone = auth_module.clone(); + // launch servers concurrently + let (rpc, auth) = futures::future::try_join( + Self::launch_rpc_server_internal(server_config, &modules), + Self::launch_auth_server_internal(auth_module_clone, auth_config), + ) + .await?; + (rpc, auth) + }; + + let handles = RethRpcServerHandles { rpc, auth }; + + Self::finalize_rpc_setup( + &mut registry, + &mut modules, + &mut auth_module, + &node, + config, + on_rpc_started, + handles.clone(), + )?; + + Ok(RpcHandle { + rpc_server_handles: handles, + rpc_registry: registry, + engine_events, + beacon_engine_handle: engine_handle, + }) + } + + /// Common setup for RPC server initialization + async fn setup_rpc_components<'a, F>( + self, + ctx: AddOnsContext<'a, N>, + ext: F, + ) -> eyre::Result> + where + F: FnOnce(RpcModuleContainer<'_, N, EthB::EthApi>) -> eyre::Result<()>, { let Self { eth_api_builder, engine_api_builder, hooks, .. } = self; @@ -485,7 +972,8 @@ where }), ); - let ctx = EthApiCtx { components: &node, config: config.rpc.eth_config(), cache }; + let eth_config = config.rpc.eth_config().max_batch_size(config.txpool.max_batch_size()); + let ctx = EthApiCtx { components: &node, config: eth_config, cache }; let eth_api = eth_api_builder.build_eth_api(ctx).await?; let auth_config = config.rpc.auth_server_config(jwt_secret)?; @@ -496,14 +984,15 @@ where .with_provider(node.provider().clone()) .with_pool(node.pool().clone()) .with_network(node.network().clone()) - .with_executor(node.task_executor().clone()) + .with_executor(Box::new(node.task_executor().clone())) .with_evm_config(node.evm_config().clone()) .with_consensus(node.consensus().clone()) .build_with_auth_server(module_config, engine_api, eth_api); // in dev mode we generate 20 random dev-signer accounts if config.dev.dev { - registry.eth_api().with_dev_accounts(); + let signers = DevSigner::from_mnemonic(config.dev.dev_mnemonic.as_str(), 20); + registry.eth_api().signers().write().extend(signers); } let mut registry = RpcRegistry { registry }; @@ -517,70 +1006,99 @@ where let RpcHooks { on_rpc_started, extend_rpc_modules } = hooks; - ext(ctx.modules, ctx.auth_module, ctx.registry)?; + ext(RpcModuleContainer { + modules: ctx.modules, + auth_module: ctx.auth_module, + registry: ctx.registry, + })?; extend_rpc_modules.extend_rpc_modules(ctx)?; - let server_config = config.rpc.rpc_server_config(); - let cloned_modules = modules.clone(); - let launch_rpc = server_config.start(&cloned_modules).map_ok(|handle| { - if let Some(path) = handle.ipc_endpoint() { - info!(target: "reth::cli", %path, "RPC IPC server started"); - } - if let Some(addr) = handle.http_local_addr() { - info!(target: "reth::cli", url=%addr, "RPC HTTP server started"); - } - if let Some(addr) = handle.ws_local_addr() { - info!(target: "reth::cli", url=%addr, "RPC WS server started"); - } - handle - }); - - let launch_auth = auth_module.clone().start_server(auth_config).map_ok(|handle| { - let addr = handle.local_addr(); - if let Some(ipc_endpoint) = handle.ipc_endpoint() { - info!(target: "reth::cli", url=%addr, ipc_endpoint=%ipc_endpoint, "RPC auth server started"); - } else { - info!(target: "reth::cli", url=%addr, "RPC auth server started"); - } - handle - }); - - // launch servers concurrently - let (rpc, auth) = futures::future::try_join(launch_rpc, launch_auth).await?; + Ok(RpcSetupContext { + node, + config, + modules, + auth_module, + auth_config, + registry, + on_rpc_started, + engine_events, + engine_handle: beacon_engine_handle, + }) + } - let handles = RethRpcServerHandles { rpc, auth }; + /// Helper to launch the RPC server + async fn launch_rpc_server_internal( + server_config: RpcServerConfig, + modules: &TransportRpcModules, + ) -> eyre::Result + where + M: RethRpcMiddleware, + { + let handle = server_config.start(modules).await?; - let ctx = RpcContext { - node: node.clone(), - config, - registry: &mut registry, - modules: &mut modules, - auth_module: &mut auth_module, - }; + if let Some(path) = handle.ipc_endpoint() { + info!(target: "reth::cli", %path, "RPC IPC server started"); + } + if let Some(addr) = handle.http_local_addr() { + info!(target: "reth::cli", url=%addr, "RPC HTTP server started"); + } + if let Some(addr) = handle.ws_local_addr() { + info!(target: "reth::cli", url=%addr, "RPC WS server started"); + } - on_rpc_started.on_rpc_started(ctx, handles.clone())?; + Ok(handle) + } - Ok(RpcHandle { - rpc_server_handles: handles, - rpc_registry: registry, - engine_events, - beacon_engine_handle, - }) + /// Helper to launch the auth server + async fn launch_auth_server_internal( + auth_module: AuthRpcModule, + auth_config: reth_rpc_builder::auth::AuthServerConfig, + ) -> eyre::Result { + auth_module.start_server(auth_config) + .await + .map_err(Into::into) + .inspect(|handle| { + let addr = handle.local_addr(); + if let Some(ipc_endpoint) = handle.ipc_endpoint() { + info!(target: "reth::cli", url=%addr, ipc_endpoint=%ipc_endpoint, "RPC auth server started"); + } else { + info!(target: "reth::cli", url=%addr, "RPC auth server started"); + } + }) + } + + /// Helper to finalize RPC setup by creating context and calling hooks + fn finalize_rpc_setup( + registry: &mut RpcRegistry, + modules: &mut TransportRpcModules, + auth_module: &mut AuthRpcModule, + node: &N, + config: &NodeConfig<::ChainSpec>, + on_rpc_started: Box>, + handles: RethRpcServerHandles, + ) -> eyre::Result<()> { + let ctx = RpcContext { node: node.clone(), config, registry, modules, auth_module }; + + on_rpc_started.on_rpc_started(ctx, handles)?; + Ok(()) } } -impl NodeAddOns for RpcAddOns +impl NodeAddOns + for RpcAddOns where N: FullNodeComponents, ::Provider: ChainSpecProvider, EthB: EthApiBuilder, - EV: EngineValidatorBuilder, + PVB: PayloadValidatorBuilder, EB: EngineApiBuilder, + EVB: EngineValidatorBuilder, + RpcMiddleware: RethRpcMiddleware, { type Handle = RpcHandle; async fn launch_add_ons(self, ctx: AddOnsContext<'_, N>) -> eyre::Result { - self.launch_add_ons_with(ctx, |_, _, _| Ok(())).await + self.launch_add_ons_with(ctx, |_| Ok(())).await } } @@ -596,7 +1114,8 @@ pub trait RethRpcAddOns: fn hooks_mut(&mut self) -> &mut RpcHooks; } -impl RethRpcAddOns for RpcAddOns +impl RethRpcAddOns + for RpcAddOns where Self: NodeAddOns>, EthB: EthApiBuilder, @@ -617,7 +1136,28 @@ pub struct EthApiCtx<'a, N: FullNodeTypes> { /// Eth API configuration pub config: EthConfig, /// Cache for eth state - pub cache: EthStateCache, ReceiptTy>, + pub cache: EthStateCache>, +} + +impl<'a, N: FullNodeComponents>> + EthApiCtx<'a, N> +{ + /// Provides a [`EthApiBuilder`] with preconfigured config and components. + pub fn eth_api_builder(self) -> reth_rpc::EthApiBuilder> { + reth_rpc::EthApiBuilder::new_with_components(self.components.clone()) + .eth_cache(self.cache) + .task_spawner(self.components.task_executor().clone()) + .gas_cap(self.config.rpc_gas_cap.into()) + .max_simulate_blocks(self.config.rpc_max_simulate_blocks) + .eth_proof_window(self.config.eth_proof_window) + .fee_history_cache_config(self.config.fee_history_cache) + .proof_permits(self.config.proof_permits) + .gas_oracle_config(self.config.gas_oracle) + .max_batch_size(self.config.max_batch_size) + .pending_block_kind(self.config.pending_block_kind) + .raw_tx_forwarder(self.config.raw_tx_forwarder) + .evm_memory_limit(self.config.rpc_evm_memory_limit) + } } /// A `EthApi` that knows how to build `eth` namespace API from [`FullNodeComponents`]. @@ -625,7 +1165,6 @@ pub trait EthApiBuilder: Default + Send + 'static { /// The Ethapi implementation this builder will build. type EthApi: EthApiTypes + FullEthApiServer - + AddDevSigners + Unpin + 'static; @@ -636,63 +1175,29 @@ pub trait EthApiBuilder: Default + Send + 'static { ) -> impl Future> + Send; } -/// Helper trait that provides the validator for the engine API +/// Helper trait that provides the validator builder for the engine API pub trait EngineValidatorAddOn: Send { - /// The Validator type to use for the engine API. - type Validator: EngineValidator<::Payload, Block = BlockTy> - + Clone; + /// The validator builder type to use. + type ValidatorBuilder: EngineValidatorBuilder; - /// Creates the engine validator for an engine API based node. - fn engine_validator( - &self, - ctx: &AddOnsContext<'_, Node>, - ) -> impl Future>; + /// Returns the validator builder. + fn engine_validator_builder(&self) -> Self::ValidatorBuilder; } -impl EngineValidatorAddOn for RpcAddOns +impl EngineValidatorAddOn + for RpcAddOns where N: FullNodeComponents, EthB: EthApiBuilder, - EV: EngineValidatorBuilder, + PVB: Send, EB: EngineApiBuilder, + EVB: EngineValidatorBuilder, + RpcMiddleware: Send, { - type Validator = EV::Validator; - - async fn engine_validator(&self, ctx: &AddOnsContext<'_, N>) -> eyre::Result { - self.engine_validator_builder.clone().build(ctx).await - } -} - -/// A type that knows how to build the engine validator. -pub trait EngineValidatorBuilder: Send + Sync + Clone { - /// The consensus implementation to build. - type Validator: EngineValidator<::Payload, Block = BlockTy> - + Clone; - - /// Creates the engine validator. - fn build( - self, - ctx: &AddOnsContext<'_, Node>, - ) -> impl Future> + Send; -} - -impl EngineValidatorBuilder for F -where - Node: FullNodeComponents, - Validator: EngineValidator<::Payload, Block = BlockTy> - + Clone - + Unpin - + 'static, - F: FnOnce(&AddOnsContext<'_, Node>) -> Fut + Send + Sync + Clone, - Fut: Future> + Send, -{ - type Validator = Validator; + type ValidatorBuilder = EVB; - fn build( - self, - ctx: &AddOnsContext<'_, Node>, - ) -> impl Future> { - self(ctx) + fn engine_validator_builder(&self) -> Self::ValidatorBuilder { + self.engine_validator_builder.clone() } } @@ -716,17 +1221,115 @@ pub trait EngineApiBuilder: Send + Sync { ) -> impl Future> + Send; } +/// Builder trait for creating payload validators specifically for the Engine API. +/// +/// This trait is responsible for building validators that the Engine API will use +/// to validate payloads. +pub trait PayloadValidatorBuilder: Send + Sync + Clone { + /// The validator type that will be used by the Engine API. + type Validator: PayloadValidator<::Payload>; + + /// Builds the engine API validator. + /// + /// Returns a validator that validates engine API version-specific fields and payload + /// attributes. + fn build( + self, + ctx: &AddOnsContext<'_, Node>, + ) -> impl Future> + Send; +} + +/// Builder trait for creating engine validators for the consensus engine. +/// +/// This trait is responsible for building validators that the consensus engine will use +/// for block execution, state validation, and fork handling. +pub trait EngineValidatorBuilder: Send + Sync + Clone { + /// The tree validator type that will be used by the consensus engine. + type EngineValidator: EngineValidator< + ::Payload, + ::Primitives, + >; + + /// Builds the tree validator for the consensus engine. + /// + /// Returns a validator that handles block execution, state validation, and fork handling. + fn build_tree_validator( + self, + ctx: &AddOnsContext<'_, Node>, + tree_config: TreeConfig, + ) -> impl Future> + Send; +} + +/// Basic implementation of [`EngineValidatorBuilder`]. +/// +/// This builder creates a [`BasicEngineValidator`] using the provided payload validator builder. +#[derive(Debug, Clone)] +pub struct BasicEngineValidatorBuilder { + /// The payload validator builder used to create the engine validator. + payload_validator_builder: EV, +} + +impl BasicEngineValidatorBuilder { + /// Creates a new instance with the given payload validator builder. + pub const fn new(payload_validator_builder: EV) -> Self { + Self { payload_validator_builder } + } +} + +impl Default for BasicEngineValidatorBuilder +where + EV: Default, +{ + fn default() -> Self { + Self::new(EV::default()) + } +} + +impl EngineValidatorBuilder for BasicEngineValidatorBuilder +where + Node: FullNodeComponents< + Evm: ConfigureEngineEvm< + <::Payload as PayloadTypes>::ExecutionData, + >, + >, + EV: PayloadValidatorBuilder, + EV::Validator: reth_engine_primitives::PayloadValidator< + ::Payload, + Block = BlockTy, + >, +{ + type EngineValidator = BasicEngineValidator; + + async fn build_tree_validator( + self, + ctx: &AddOnsContext<'_, Node>, + tree_config: TreeConfig, + ) -> eyre::Result { + let validator = self.payload_validator_builder.build(ctx).await?; + let data_dir = ctx.config.datadir.clone().resolve_datadir(ctx.config.chain.chain()); + let invalid_block_hook = ctx.create_invalid_block_hook(&data_dir).await?; + Ok(BasicEngineValidator::new( + ctx.node.provider().clone(), + std::sync::Arc::new(ctx.node.consensus().clone()), + ctx.node.evm_config().clone(), + validator, + tree_config, + invalid_block_hook, + )) + } +} + /// Builder for basic [`EngineApi`] implementation. /// /// This provides a basic default implementation for opstack and ethereum engine API via /// [`EngineTypes`] and uses the general purpose [`EngineApi`] implementation as the builder's /// output. #[derive(Debug, Default)] -pub struct BasicEngineApiBuilder { - engine_validator_builder: EV, +pub struct BasicEngineApiBuilder { + payload_validator_builder: PVB, } -impl EngineApiBuilder for BasicEngineApiBuilder +impl EngineApiBuilder for BasicEngineApiBuilder where N: FullNodeComponents< Types: NodeTypes< @@ -734,25 +1337,26 @@ where Payload: PayloadTypes + EngineTypes, >, >, - EV: EngineValidatorBuilder, + PVB: PayloadValidatorBuilder, + PVB::Validator: EngineApiValidator<::Payload>, { type EngineApi = EngineApi< N::Provider, ::Payload, N::Pool, - EV::Validator, + PVB::Validator, ::ChainSpec, >; async fn build_engine_api(self, ctx: &AddOnsContext<'_, N>) -> eyre::Result { - let Self { engine_validator_builder } = self; + let Self { payload_validator_builder } = self; - let engine_validator = engine_validator_builder.build(ctx).await?; + let engine_validator = payload_validator_builder.build(ctx).await?; let client = ClientVersionV1 { code: CLIENT_CODE, - name: NAME_CLIENT.to_string(), - version: CARGO_PKG_VERSION.to_string(), - commit: VERGEN_GIT_SHA.to_string(), + name: version_metadata().name_client.to_string(), + version: version_metadata().cargo_pkg_version.to_string(), + commit: version_metadata().vergen_git_sha.to_string(), }; Ok(EngineApi::new( ctx.node.provider().clone(), @@ -772,7 +1376,7 @@ where /// A noop Builder that satisfies the [`EngineApiBuilder`] trait without actually configuring an /// engine API module /// -/// This is intended to be used as a workaround for re-using all the existing ethereum node launch +/// This is intended to be used as a workaround for reusing all the existing ethereum node launch /// utilities which require an engine API. #[derive(Debug, Clone, Default)] #[non_exhaustive] diff --git a/crates/node/builder/src/setup.rs b/crates/node/builder/src/setup.rs index 255a844c93f..ad78ffb59a2 100644 --- a/crates/node/builder/src/setup.rs +++ b/crates/node/builder/src/setup.rs @@ -17,7 +17,11 @@ use reth_network_p2p::{ }; use reth_node_api::HeaderTy; use reth_provider::{providers::ProviderNodeTypes, ProviderFactory}; -use reth_stages::{prelude::DefaultStages, stages::ExecutionStage, Pipeline, StageSet}; +use reth_stages::{ + prelude::DefaultStages, + stages::{EraImportSource, ExecutionStage}, + Pipeline, StageSet, +}; use reth_static_file::StaticFileProducer; use reth_tasks::TaskExecutor; use reth_tracing::tracing::debug; @@ -32,11 +36,12 @@ pub fn build_networked_pipeline( provider_factory: ProviderFactory, task_executor: &TaskExecutor, metrics_tx: reth_stages::MetricEventsSender, - prune_config: Option, + prune_config: PruneConfig, max_block: Option, static_file_producer: StaticFileProducer>, evm_config: Evm, exex_manager_handle: ExExManagerHandle, + era_import_source: Option, ) -> eyre::Result> where N: ProviderNodeTypes, @@ -64,6 +69,7 @@ where static_file_producer, evm_config, exex_manager_handle, + era_import_source, )?; Ok(pipeline) @@ -79,10 +85,11 @@ pub fn build_pipeline( consensus: Arc>, max_block: Option, metrics_tx: reth_stages::MetricEventsSender, - prune_config: Option, + prune_config: PruneConfig, static_file_producer: StaticFileProducer>, evm_config: Evm, exex_manager_handle: ExExManagerHandle, + era_import_source: Option, ) -> eyre::Result> where N: ProviderNodeTypes, @@ -99,8 +106,6 @@ where let (tip_tx, tip_rx) = watch::channel(B256::ZERO); - let prune_modes = prune_config.map(|prune| prune.segments).unwrap_or_default(); - let pipeline = builder .with_tip_sender(tip_tx) .with_metrics_tx(metrics_tx) @@ -113,7 +118,8 @@ where body_downloader, evm_config.clone(), stage_config.clone(), - prune_modes, + prune_config.segments, + era_import_source, ) .set(ExecutionStage::new( evm_config, diff --git a/crates/node/core/Cargo.toml b/crates/node/core/Cargo.toml index 677fdb7980e..1d767865793 100644 --- a/crates/node/core/Cargo.toml +++ b/crates/node/core/Cargo.toml @@ -23,7 +23,7 @@ reth-network = { workspace = true, features = ["serde"] } reth-network-p2p.workspace = true reth-rpc-eth-types.workspace = true reth-rpc-server-types.workspace = true -reth-rpc-types-compat.workspace = true +reth-rpc-convert.workspace = true reth-transaction-pool.workspace = true reth-tracing.workspace = true reth-config = { workspace = true, features = ["serde"] } @@ -34,6 +34,7 @@ reth-network-peers.workspace = true reth-prune-types.workspace = true reth-stages-types.workspace = true reth-ethereum-forks.workspace = true +reth-engine-local.workspace = true reth-engine-primitives.workspace = true # ethereum @@ -44,7 +45,7 @@ alloy-eips.workspace = true # misc eyre.workspace = true -clap = { workspace = true, features = ["derive"] } +clap = { workspace = true, features = ["derive", "env"] } humantime.workspace = true rand.workspace = true derive_more.workspace = true @@ -52,13 +53,15 @@ toml.workspace = true serde.workspace = true strum = { workspace = true, features = ["derive"] } thiserror.workspace = true +url.workspace = true # io dirs-next.workspace = true shellexpand.workspace = true -# tracing +# obs tracing.workspace = true +reth-tracing-otlp.workspace = true # crypto secp256k1 = { workspace = true, features = ["global-context", "std", "recovery"] } @@ -75,6 +78,14 @@ tokio.workspace = true # Features for vergen to generate correct env vars jemalloc = ["reth-cli-util/jemalloc"] asm-keccak = ["alloy-primitives/asm-keccak"] +# Feature to enable opentelemetry export +otlp = ["reth-tracing/otlp"] + +min-error-logs = ["tracing/release_max_level_error"] +min-warn-logs = ["tracing/release_max_level_warn"] +min-info-logs = ["tracing/release_max_level_info"] +min-debug-logs = ["tracing/release_max_level_debug"] +min-trace-logs = ["tracing/release_max_level_trace"] [build-dependencies] vergen = { workspace = true, features = ["build", "cargo", "emit_and_set"] } diff --git a/crates/node/core/src/args/benchmark_args.rs b/crates/node/core/src/args/benchmark_args.rs index 1ff49c9c84d..2865054ded1 100644 --- a/crates/node/core/src/args/benchmark_args.rs +++ b/crates/node/core/src/args/benchmark_args.rs @@ -15,13 +15,25 @@ pub struct BenchmarkArgs { #[arg(long, verbatim_doc_comment)] pub to: Option, + /// Number of blocks to advance from the current head block. + /// When specified, automatically sets --from to current head + 1 and --to to current head + + /// advance. Cannot be used together with explicit --from and --to arguments. + #[arg(long, conflicts_with_all = &["from", "to"], verbatim_doc_comment)] + pub advance: Option, + /// Path to a JWT secret to use for the authenticated engine-API RPC server. /// /// This will perform JWT authentication for all requests to the given engine RPC url. /// /// If no path is provided, a secret will be generated and stored in the datadir under /// `

//jwt.hex`. For mainnet this would be `~/.reth/mainnet/jwt.hex` by default. - #[arg(long = "jwtsecret", value_name = "PATH", global = true, required = false)] + #[arg( + long = "jwt-secret", + alias = "jwtsecret", + value_name = "PATH", + global = true, + required = false + )] pub auth_jwtsecret: Option, /// The RPC url to use for sending engine requests. diff --git a/crates/node/core/src/args/database.rs b/crates/node/core/src/args/database.rs index 1a490bc2722..6f1d3bfc711 100644 --- a/crates/node/core/src/args/database.rs +++ b/crates/node/core/src/args/database.rs @@ -6,9 +6,12 @@ use crate::version::default_client_version; use clap::{ builder::{PossibleValue, TypedValueParser}, error::ErrorKind, - Arg, Args, Command, Error, + value_parser, Arg, Args, Command, Error, +}; +use reth_db::{ + mdbx::{MaxReadTransactionDuration, SyncMode}, + ClientVersion, }; -use reth_db::{mdbx::MaxReadTransactionDuration, ClientVersion}; use reth_storage_errors::db::LogLevel; /// Parameters for database configuration @@ -31,6 +34,15 @@ pub struct DatabaseArgs { /// Read transaction timeout in seconds, 0 means no timeout. #[arg(long = "db.read-transaction-timeout")] pub read_transaction_timeout: Option, + /// Maximum number of readers allowed to access the database concurrently. + #[arg(long = "db.max-readers")] + pub max_readers: Option, + /// Controls how aggressively the database synchronizes data to disk. + #[arg( + long = "db.sync-mode", + value_parser = value_parser!(SyncMode), + )] + pub sync_mode: Option, } impl DatabaseArgs { @@ -57,6 +69,8 @@ impl DatabaseArgs { .with_max_read_transaction_duration(max_read_transaction_duration) .with_geometry_max_size(self.max_size) .with_growth_step(self.growth_step) + .with_max_readers(self.max_readers) + .with_sync_mode(self.sync_mode) } } @@ -336,4 +350,36 @@ mod tests { let cmd = CommandParser::::try_parse_from(["reth"]).unwrap(); assert_eq!(cmd.args.log_level, None); } + + #[test] + fn test_command_parser_with_valid_default_sync_mode() { + let cmd = CommandParser::::try_parse_from(["reth"]).unwrap(); + assert!(cmd.args.sync_mode.is_none()); + } + + #[test] + fn test_command_parser_with_valid_sync_mode_durable() { + let cmd = + CommandParser::::try_parse_from(["reth", "--db.sync-mode", "durable"]) + .unwrap(); + assert!(matches!(cmd.args.sync_mode, Some(SyncMode::Durable))); + } + + #[test] + fn test_command_parser_with_valid_sync_mode_safe_no_sync() { + let cmd = CommandParser::::try_parse_from([ + "reth", + "--db.sync-mode", + "safe-no-sync", + ]) + .unwrap(); + assert!(matches!(cmd.args.sync_mode, Some(SyncMode::SafeNoSync))); + } + + #[test] + fn test_command_parser_with_invalid_sync_mode() { + let result = + CommandParser::::try_parse_from(["reth", "--db.sync-mode", "ultra-fast"]); + assert!(result.is_err()); + } } diff --git a/crates/node/core/src/args/debug.rs b/crates/node/core/src/args/debug.rs index 83c5c268d7d..b5d1fb3f7d8 100644 --- a/crates/node/core/src/args/debug.rs +++ b/crates/node/core/src/args/debug.rs @@ -32,19 +32,23 @@ pub struct DebugArgs { long = "debug.etherscan", help_heading = "Debug", conflicts_with = "tip", - conflicts_with = "rpc_consensus_ws", + conflicts_with = "rpc_consensus_url", value_name = "ETHERSCAN_API_URL" )] pub etherscan: Option>, - /// Runs a fake consensus client using blocks fetched from an RPC `WebSocket` endpoint. + /// Runs a fake consensus client using blocks fetched from an RPC endpoint. + /// Supports both HTTP and `WebSocket` endpoints - `WebSocket` endpoints will use + /// subscriptions, while HTTP endpoints will poll for new blocks. #[arg( - long = "debug.rpc-consensus-ws", + long = "debug.rpc-consensus-url", + alias = "debug.rpc-consensus-ws", help_heading = "Debug", conflicts_with = "tip", - conflicts_with = "etherscan" + conflicts_with = "etherscan", + value_name = "RPC_URL" )] - pub rpc_consensus_ws: Option, + pub rpc_consensus_url: Option, /// If provided, the engine will skip `n` consecutive FCUs. #[arg(long = "debug.skip-fcu", help_heading = "Debug")] @@ -80,6 +84,11 @@ pub struct DebugArgs { pub invalid_block_hook: Option, /// The RPC URL of a healthy node to use for comparing invalid block hook results against. + /// + ///Debug setting that enables execution witness comparison for troubleshooting bad blocks. + /// When enabled, the node will collect execution witnesses from the specified source and + /// compare them against local execution when a bad block is encountered, helping identify + /// discrepancies in state execution. #[arg( long = "debug.healthy-node-rpc-url", help_heading = "Debug", @@ -87,6 +96,18 @@ pub struct DebugArgs { verbatim_doc_comment )] pub healthy_node_rpc_url: Option, + + /// The URL of the ethstats server to connect to. + /// Example: `nodename:secret@host:port` + #[arg(long = "ethstats", help_heading = "Debug")] + pub ethstats: Option, + + /// Set the node to idle state when the backfill is not running. + /// + /// This makes the `eth_syncing` RPC return "Idle" when the node has just started or finished + /// the backfill, but did not yet receive any new blocks. + #[arg(long = "debug.startup-sync-state-idle", help_heading = "Debug")] + pub startup_sync_state_idle: bool, } impl Default for DebugArgs { @@ -96,7 +117,7 @@ impl Default for DebugArgs { tip: None, max_block: None, etherscan: None, - rpc_consensus_ws: None, + rpc_consensus_url: None, skip_fcu: None, skip_new_payload: None, reorg_frequency: None, @@ -104,6 +125,8 @@ impl Default for DebugArgs { engine_api_store: None, invalid_block_hook: Some(InvalidBlockSelection::default()), healthy_node_rpc_url: None, + ethstats: None, + startup_sync_state_idle: false, } } } diff --git a/crates/node/core/src/args/dev.rs b/crates/node/core/src/args/dev.rs index b6a01745257..d62ff1c5dce 100644 --- a/crates/node/core/src/args/dev.rs +++ b/crates/node/core/src/args/dev.rs @@ -5,8 +5,10 @@ use std::time::Duration; use clap::Args; use humantime::parse_duration; +const DEFAULT_MNEMONIC: &str = "test test test test test test test test test test test junk"; + /// Parameters for Dev testnet configuration -#[derive(Debug, Args, PartialEq, Eq, Default, Clone, Copy)] +#[derive(Debug, Args, PartialEq, Eq, Clone)] #[command(next_help_heading = "Dev testnet")] pub struct DevArgs { /// Start the node in dev mode @@ -39,6 +41,28 @@ pub struct DevArgs { verbatim_doc_comment )] pub block_time: Option, + + /// Derive dev accounts from a fixed mnemonic instead of random ones. + #[arg( + long = "dev.mnemonic", + help_heading = "Dev testnet", + value_name = "MNEMONIC", + requires = "dev", + verbatim_doc_comment, + default_value = DEFAULT_MNEMONIC + )] + pub dev_mnemonic: String, +} + +impl Default for DevArgs { + fn default() -> Self { + Self { + dev: false, + block_max_transactions: None, + block_time: None, + dev_mnemonic: DEFAULT_MNEMONIC.to_string(), + } + } } #[cfg(test)] @@ -56,13 +80,37 @@ mod tests { #[test] fn test_parse_dev_args() { let args = CommandParser::::parse_from(["reth"]).args; - assert_eq!(args, DevArgs { dev: false, block_max_transactions: None, block_time: None }); + assert_eq!( + args, + DevArgs { + dev: false, + block_max_transactions: None, + block_time: None, + dev_mnemonic: DEFAULT_MNEMONIC.to_string(), + } + ); let args = CommandParser::::parse_from(["reth", "--dev"]).args; - assert_eq!(args, DevArgs { dev: true, block_max_transactions: None, block_time: None }); + assert_eq!( + args, + DevArgs { + dev: true, + block_max_transactions: None, + block_time: None, + dev_mnemonic: DEFAULT_MNEMONIC.to_string(), + } + ); let args = CommandParser::::parse_from(["reth", "--auto-mine"]).args; - assert_eq!(args, DevArgs { dev: true, block_max_transactions: None, block_time: None }); + assert_eq!( + args, + DevArgs { + dev: true, + block_max_transactions: None, + block_time: None, + dev_mnemonic: DEFAULT_MNEMONIC.to_string(), + } + ); let args = CommandParser::::parse_from([ "reth", @@ -71,7 +119,15 @@ mod tests { "2", ]) .args; - assert_eq!(args, DevArgs { dev: true, block_max_transactions: Some(2), block_time: None }); + assert_eq!( + args, + DevArgs { + dev: true, + block_max_transactions: Some(2), + block_time: None, + dev_mnemonic: DEFAULT_MNEMONIC.to_string(), + } + ); let args = CommandParser::::parse_from(["reth", "--dev", "--dev.block-time", "1s"]).args; @@ -80,7 +136,8 @@ mod tests { DevArgs { dev: true, block_max_transactions: None, - block_time: Some(std::time::Duration::from_secs(1)) + block_time: Some(std::time::Duration::from_secs(1)), + dev_mnemonic: DEFAULT_MNEMONIC.to_string(), } ); } diff --git a/crates/node/core/src/args/engine.rs b/crates/node/core/src/args/engine.rs index cd2c1743d9a..29535f2c1df 100644 --- a/crates/node/core/src/args/engine.rs +++ b/crates/node/core/src/args/engine.rs @@ -1,11 +1,11 @@ //! clap [Args](clap::Args) for engine purposes use clap::Args; -use reth_engine_primitives::TreeConfig; +use reth_engine_primitives::{TreeConfig, DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE}; use crate::node_config::{ - DEFAULT_CROSS_BLOCK_CACHE_SIZE_MB, DEFAULT_MAX_PROOF_TASK_CONCURRENCY, - DEFAULT_MEMORY_BLOCK_BUFFER_TARGET, DEFAULT_PERSISTENCE_THRESHOLD, DEFAULT_RESERVED_CPU_CORES, + DEFAULT_CROSS_BLOCK_CACHE_SIZE_MB, DEFAULT_MEMORY_BLOCK_BUFFER_TARGET, + DEFAULT_PERSISTENCE_THRESHOLD, DEFAULT_RESERVED_CPU_CORES, }; /// Parameters for configuring the engine driver. @@ -25,13 +25,24 @@ pub struct EngineArgs { pub legacy_state_root_task_enabled: bool, /// CAUTION: This CLI flag has no effect anymore, use --engine.disable-caching-and-prewarming - /// if you want to disable caching and prewarming. - #[arg(long = "engine.caching-and-prewarming", default_value = "true")] + /// if you want to disable caching and prewarming + #[arg(long = "engine.caching-and-prewarming", default_value = "true", hide = true)] + #[deprecated] pub caching_and_prewarming_enabled: bool, - /// Disable cross-block caching and parallel prewarming - #[arg(long = "engine.disable-caching-and-prewarming")] - pub caching_and_prewarming_disabled: bool, + /// Disable parallel prewarming + #[arg(long = "engine.disable-prewarming", alias = "engine.disable-caching-and-prewarming")] + pub prewarming_disabled: bool, + + /// CAUTION: This CLI flag has no effect anymore, use --engine.disable-parallel-sparse-trie + /// if you want to disable usage of the `ParallelSparseTrie`. + #[deprecated] + #[arg(long = "engine.parallel-sparse-trie", default_value = "true", hide = true)] + pub parallel_sparse_trie_enabled: bool, + + /// Disable the parallel sparse trie in the engine. + #[arg(long = "engine.disable-parallel-sparse-trie", default_value = "false")] + pub parallel_sparse_trie_disabled: bool, /// Enable state provider latency metrics. This allows the engine to collect and report stats /// about how long state provider calls took during execution, but this does introduce slight @@ -52,15 +63,60 @@ pub struct EngineArgs { #[arg(long = "engine.accept-execution-requests-hash")] pub accept_execution_requests_hash: bool, - /// Configure the maximum number of concurrent proof tasks - #[arg(long = "engine.max-proof-task-concurrency", default_value_t = DEFAULT_MAX_PROOF_TASK_CONCURRENCY)] - pub max_proof_task_concurrency: u64, + /// Whether multiproof task should chunk proof targets. + #[arg(long = "engine.multiproof-chunking", default_value = "true")] + pub multiproof_chunking_enabled: bool, + + /// Multiproof task chunk size for proof targets. + #[arg(long = "engine.multiproof-chunk-size", default_value_t = DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE)] + pub multiproof_chunk_size: usize, /// Configure the number of reserved CPU cores for non-reth processes #[arg(long = "engine.reserved-cpu-cores", default_value_t = DEFAULT_RESERVED_CPU_CORES)] pub reserved_cpu_cores: usize, + + /// CAUTION: This CLI flag has no effect anymore, use --engine.disable-precompile-cache + /// if you want to disable precompile cache + #[arg(long = "engine.precompile-cache", default_value = "true", hide = true)] + #[deprecated] + pub precompile_cache_enabled: bool, + + /// Disable precompile cache + #[arg(long = "engine.disable-precompile-cache", default_value = "false")] + pub precompile_cache_disabled: bool, + + /// Enable state root fallback, useful for testing + #[arg(long = "engine.state-root-fallback", default_value = "false")] + pub state_root_fallback: bool, + + /// Always process payload attributes and begin a payload build process even if + /// `forkchoiceState.headBlockHash` is already the canonical head or an ancestor. See + /// `TreeConfig::always_process_payload_attributes_on_canonical_head` for more details. + /// + /// Note: This is a no-op on OP Stack. + #[arg( + long = "engine.always-process-payload-attributes-on-canonical-head", + default_value = "false" + )] + pub always_process_payload_attributes_on_canonical_head: bool, + + /// Allow unwinding canonical header to ancestor during forkchoice updates. + /// See `TreeConfig::unwind_canonical_header` for more details. + #[arg(long = "engine.allow-unwind-canonical-header", default_value = "false")] + pub allow_unwind_canonical_header: bool, + + /// Configure the number of storage proof workers in the Tokio blocking pool. + /// If not specified, defaults to 2x available parallelism, clamped between 2 and 64. + #[arg(long = "engine.storage-worker-count")] + pub storage_worker_count: Option, + + /// Configure the number of account proof workers in the Tokio blocking pool. + /// If not specified, defaults to the same count as storage workers. + #[arg(long = "engine.account-worker-count")] + pub account_worker_count: Option, } +#[allow(deprecated)] impl Default for EngineArgs { fn default() -> Self { Self { @@ -69,12 +125,22 @@ impl Default for EngineArgs { legacy_state_root_task_enabled: false, state_root_task_compare_updates: false, caching_and_prewarming_enabled: true, - caching_and_prewarming_disabled: false, + prewarming_disabled: false, + parallel_sparse_trie_enabled: true, + parallel_sparse_trie_disabled: false, state_provider_metrics: false, cross_block_cache_size: DEFAULT_CROSS_BLOCK_CACHE_SIZE_MB, accept_execution_requests_hash: false, - max_proof_task_concurrency: DEFAULT_MAX_PROOF_TASK_CONCURRENCY, + multiproof_chunking_enabled: true, + multiproof_chunk_size: DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE, reserved_cpu_cores: DEFAULT_RESERVED_CPU_CORES, + precompile_cache_enabled: true, + precompile_cache_disabled: false, + state_root_fallback: false, + always_process_payload_attributes_on_canonical_head: false, + allow_unwind_canonical_header: false, + storage_worker_count: None, + account_worker_count: None, } } } @@ -82,16 +148,34 @@ impl Default for EngineArgs { impl EngineArgs { /// Creates a [`TreeConfig`] from the engine arguments. pub fn tree_config(&self) -> TreeConfig { - TreeConfig::default() + let mut config = TreeConfig::default() .with_persistence_threshold(self.persistence_threshold) .with_memory_block_buffer_target(self.memory_block_buffer_target) .with_legacy_state_root(self.legacy_state_root_task_enabled) - .without_caching_and_prewarming(self.caching_and_prewarming_disabled) + .without_prewarming(self.prewarming_disabled) + .with_disable_parallel_sparse_trie(self.parallel_sparse_trie_disabled) .with_state_provider_metrics(self.state_provider_metrics) .with_always_compare_trie_updates(self.state_root_task_compare_updates) .with_cross_block_cache_size(self.cross_block_cache_size * 1024 * 1024) - .with_max_proof_task_concurrency(self.max_proof_task_concurrency) + .with_multiproof_chunking_enabled(self.multiproof_chunking_enabled) + .with_multiproof_chunk_size(self.multiproof_chunk_size) .with_reserved_cpu_cores(self.reserved_cpu_cores) + .without_precompile_cache(self.precompile_cache_disabled) + .with_state_root_fallback(self.state_root_fallback) + .with_always_process_payload_attributes_on_canonical_head( + self.always_process_payload_attributes_on_canonical_head, + ) + .with_unwind_canonical_header(self.allow_unwind_canonical_header); + + if let Some(count) = self.storage_worker_count { + config = config.with_storage_worker_count(count); + } + + if let Some(count) = self.account_worker_count { + config = config.with_account_worker_count(count); + } + + config } } diff --git a/crates/node/core/src/args/era.rs b/crates/node/core/src/args/era.rs new file mode 100644 index 00000000000..84e835c370a --- /dev/null +++ b/crates/node/core/src/args/era.rs @@ -0,0 +1,62 @@ +use clap::Args; +use reth_chainspec::{ChainKind, NamedChain}; +use std::path::Path; +use url::Url; + +/// Syncs ERA1 encoded blocks from a local or remote source. +#[derive(Clone, Debug, Default, Args)] +pub struct EraArgs { + /// Enable import from ERA1 files. + #[arg( + id = "era.enable", + long = "era.enable", + value_name = "ERA_ENABLE", + default_value_t = false + )] + pub enabled: bool, + + /// Describes where to get the ERA files to import from. + #[clap(flatten)] + pub source: EraSourceArgs, +} + +/// Arguments for the block history import based on ERA1 encoded files. +#[derive(Clone, Debug, Default, Args)] +#[group(required = false, multiple = false)] +pub struct EraSourceArgs { + /// The path to a directory for import. + /// + /// The ERA1 files are read from the local directory parsing headers and bodies. + #[arg(long = "era.path", value_name = "ERA_PATH", verbatim_doc_comment)] + pub path: Option>, + + /// The URL to a remote host where the ERA1 files are hosted. + /// + /// The ERA1 files are read from the remote host using HTTP GET requests parsing headers + /// and bodies. + #[arg(long = "era.url", value_name = "ERA_URL", verbatim_doc_comment)] + pub url: Option, +} + +/// The `ExtractEraHost` trait allows to derive a default URL host for ERA files. +pub trait DefaultEraHost { + /// Converts `self` into [`Url`] index page of the ERA host. + /// + /// Returns `Err` if the conversion is not possible. + fn default_era_host(&self) -> Option; +} + +impl DefaultEraHost for ChainKind { + fn default_era_host(&self) -> Option { + Some(match self { + Self::Named(NamedChain::Mainnet) => { + Url::parse("https://era.ithaca.xyz/era1/index.html").expect("URL should be valid") + } + Self::Named(NamedChain::Sepolia) => { + Url::parse("https://era.ithaca.xyz/sepolia-era1/index.html") + .expect("URL should be valid") + } + _ => return None, + }) + } +} diff --git a/crates/node/core/src/args/gas_price_oracle.rs b/crates/node/core/src/args/gas_price_oracle.rs index 160c667c702..8f4f7d2aa4a 100644 --- a/crates/node/core/src/args/gas_price_oracle.rs +++ b/crates/node/core/src/args/gas_price_oracle.rs @@ -3,7 +3,7 @@ use clap::Args; use reth_rpc_eth_types::GasPriceOracleConfig; use reth_rpc_server_types::constants::gas_oracle::{ DEFAULT_GAS_PRICE_BLOCKS, DEFAULT_GAS_PRICE_PERCENTILE, DEFAULT_IGNORE_GAS_PRICE, - DEFAULT_MAX_GAS_PRICE, DEFAULT_MIN_SUGGESTED_PRIORITY_FEE, + DEFAULT_MAX_GAS_PRICE, }; /// Parameters to configure Gas Price Oracle @@ -25,22 +25,22 @@ pub struct GasPriceOracleArgs { /// The percentile of gas prices to use for the estimate #[arg(long = "gpo.percentile", default_value_t = DEFAULT_GAS_PRICE_PERCENTILE)] pub percentile: u32, - - /// Minimum transaction priority fee to suggest. Used on OP chains when blocks are not full. - #[arg(long = "gpo.minsuggestedpriorityfee", default_value_t = DEFAULT_MIN_SUGGESTED_PRIORITY_FEE.to())] - pub min_suggested_priority_fee: u64, + + /// The default gas price to use if there are no blocks to use + #[arg(long = "gpo.default-suggested-fee")] + pub default_suggested_fee: Option, } impl GasPriceOracleArgs { /// Returns a [`GasPriceOracleConfig`] from the arguments. pub fn gas_price_oracle_config(&self) -> GasPriceOracleConfig { - let Self { blocks, ignore_price, max_price, percentile, min_suggested_priority_fee } = self; + let Self { blocks, ignore_price, max_price, percentile, default_suggested_fee } = self; GasPriceOracleConfig { max_price: Some(U256::from(*max_price)), ignore_price: Some(U256::from(*ignore_price)), percentile: *percentile, blocks: *blocks, - min_suggested_priority_fee: Some(U256::from(*min_suggested_priority_fee)), + default_suggested_fee: *default_suggested_fee, ..Default::default() } } @@ -53,7 +53,7 @@ impl Default for GasPriceOracleArgs { ignore_price: DEFAULT_IGNORE_GAS_PRICE.to(), max_price: DEFAULT_MAX_GAS_PRICE.to(), percentile: DEFAULT_GAS_PRICE_PERCENTILE, - min_suggested_priority_fee: DEFAULT_MIN_SUGGESTED_PRIORITY_FEE.to(), + default_suggested_fee: None, } } } @@ -79,7 +79,7 @@ mod tests { ignore_price: DEFAULT_IGNORE_GAS_PRICE.to(), max_price: DEFAULT_MAX_GAS_PRICE.to(), percentile: DEFAULT_GAS_PRICE_PERCENTILE, - min_suggested_priority_fee: DEFAULT_MIN_SUGGESTED_PRIORITY_FEE.to(), + default_suggested_fee: None, } ); } diff --git a/crates/node/core/src/args/log.rs b/crates/node/core/src/args/log.rs index 3d124fba229..20c60362d7b 100644 --- a/crates/node/core/src/args/log.rs +++ b/crates/node/core/src/args/log.rs @@ -3,7 +3,7 @@ use crate::dirs::{LogsDir, PlatformPath}; use clap::{ArgAction, Args, ValueEnum}; use reth_tracing::{ - tracing_subscriber::filter::Directive, FileInfo, FileWorkerGuard, LayerInfo, LogFormat, + tracing_subscriber::filter::Directive, FileInfo, FileWorkerGuard, LayerInfo, Layers, LogFormat, RethTracer, Tracer, }; use std::{fmt, fmt::Display}; @@ -35,6 +35,10 @@ pub struct LogArgs { #[arg(long = "log.file.directory", value_name = "PATH", global = true, default_value_t)] pub log_file_directory: PlatformPath, + /// The prefix name of the log files. + #[arg(long = "log.file.name", value_name = "NAME", global = true, default_value = "reth.log")] + pub log_file_name: String, + /// The maximum size (in MB) of one log file. #[arg(long = "log.file.max-size", value_name = "SIZE", global = true, default_value_t = 200)] pub log_file_max_size: u64, @@ -66,6 +70,7 @@ pub struct LogArgs { default_value_t = ColorMode::Always )] pub color: ColorMode, + /// The verbosity settings for the tracer. #[command(flatten)] pub verbosity: Verbosity, @@ -73,7 +78,7 @@ pub struct LogArgs { impl LogArgs { /// Creates a [`LayerInfo`] instance. - fn layer(&self, format: LogFormat, filter: String, use_color: bool) -> LayerInfo { + fn layer_info(&self, format: LogFormat, filter: String, use_color: bool) -> LayerInfo { LayerInfo::new( format, self.verbosity.directive().to_string(), @@ -86,6 +91,7 @@ impl LogArgs { fn file_info(&self) -> FileInfo { FileInfo::new( self.log_file_directory.clone().into(), + self.log_file_name.clone(), self.log_file_max_size * MB_TO_BYTES, self.log_file_max_files, ) @@ -93,11 +99,24 @@ impl LogArgs { /// Initializes tracing with the configured options from cli args. /// - /// Returns the file worker guard, and the file name, if a file worker was configured. + /// Uses default layers for tracing. If you need to include custom layers, + /// use `init_tracing_with_layers` instead. + /// + /// Returns the file worker guard if a file worker was configured. pub fn init_tracing(&self) -> eyre::Result> { + self.init_tracing_with_layers(Layers::new()) + } + + /// Initializes tracing with the configured options from cli args. + /// + /// Returns the file worker guard, and the file name, if a file worker was configured. + pub fn init_tracing_with_layers( + &self, + layers: Layers, + ) -> eyre::Result> { let mut tracer = RethTracer::new(); - let stdout = self.layer(self.log_stdout_format, self.log_stdout_filter.clone(), true); + let stdout = self.layer_info(self.log_stdout_format, self.log_stdout_filter.clone(), true); tracer = tracer.with_stdout(stdout); if self.journald { @@ -106,11 +125,11 @@ impl LogArgs { if self.log_file_max_files > 0 { let info = self.file_info(); - let file = self.layer(self.log_file_format, self.log_file_filter.clone(), false); + let file = self.layer_info(self.log_file_format, self.log_file_filter.clone(), false); tracer = tracer.with_file(file, info); } - let guard = tracer.init()?; + let guard = tracer.init_with_layers(layers)?; Ok(guard) } } @@ -120,7 +139,7 @@ impl LogArgs { pub enum ColorMode { /// Colors on Always, - /// Colors on + /// Auto-detect Auto, /// Colors off Never, diff --git a/crates/node/core/src/args/metric.rs b/crates/node/core/src/args/metric.rs new file mode 100644 index 00000000000..5ef18787a81 --- /dev/null +++ b/crates/node/core/src/args/metric.rs @@ -0,0 +1,35 @@ +use clap::Parser; +use reth_cli_util::{parse_duration_from_secs, parse_socket_address}; +use std::{net::SocketAddr, time::Duration}; + +/// Metrics configuration. +#[derive(Debug, Clone, Default, Parser)] +pub struct MetricArgs { + /// Enable Prometheus metrics. + /// + /// The metrics will be served at the given interface and port. + #[arg(long="metrics", alias = "metrics.prometheus", value_name = "PROMETHEUS", value_parser = parse_socket_address, help_heading = "Metrics")] + pub prometheus: Option, + + /// URL for pushing Prometheus metrics to a push gateway. + /// + /// If set, the node will periodically push metrics to the specified push gateway URL. + #[arg( + long = "metrics.prometheus.push.url", + value_name = "PUSH_GATEWAY_URL", + help_heading = "Metrics" + )] + pub push_gateway_url: Option, + + /// Interval in seconds for pushing metrics to push gateway. + /// + /// Default: 5 seconds + #[arg( + long = "metrics.prometheus.push.interval", + default_value = "5", + value_parser = parse_duration_from_secs, + value_name = "SECONDS", + help_heading = "Metrics" + )] + pub push_gateway_interval: Duration, +} diff --git a/crates/node/core/src/args/mod.rs b/crates/node/core/src/args/mod.rs index 3a5e55ce292..54e77740146 100644 --- a/crates/node/core/src/args/mod.rs +++ b/crates/node/core/src/args/mod.rs @@ -24,6 +24,14 @@ pub use database::DatabaseArgs; mod log; pub use log::{ColorMode, LogArgs, Verbosity}; +/// `TraceArgs` for tracing and spans support +mod trace; +pub use trace::TraceArgs; + +/// `MetricArgs` to configure metrics. +mod metric; +pub use metric::MetricArgs; + /// `PayloadBuilderArgs` struct for configuring the payload builder mod payload_builder; pub use payload_builder::PayloadBuilderArgs; @@ -64,5 +72,9 @@ pub use engine::EngineArgs; mod ress_args; pub use ress_args::RessArgs; +/// `EraArgs` for configuring ERA files import. +mod era; +pub use era::{DefaultEraHost, EraArgs, EraSourceArgs}; + mod error; pub mod types; diff --git a/crates/node/core/src/args/network.rs b/crates/node/core/src/args/network.rs index c11e927677f..52ff52b1cee 100644 --- a/crates/node/core/src/args/network.rs +++ b/crates/node/core/src/args/network.rs @@ -1,11 +1,13 @@ //! clap [Args](clap::Args) for network related arguments. +use alloy_primitives::B256; use std::{ net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV4, SocketAddrV6}, ops::Not, path::PathBuf, }; +use crate::version::version_metadata; use clap::Args; use reth_chainspec::EthChainSpec; use reth_config::Config; @@ -27,7 +29,7 @@ use reth_network::{ DEFAULT_MAX_COUNT_PENDING_POOL_IMPORTS, DEFAULT_MAX_COUNT_TRANSACTIONS_SEEN_BY_PEER, }, }, - TransactionFetcherConfig, TransactionsManagerConfig, + TransactionFetcherConfig, TransactionPropagationMode, TransactionsManagerConfig, DEFAULT_SOFT_LIMIT_BYTE_SIZE_POOLED_TRANSACTIONS_RESP_ON_PACK_GET_POOLED_TRANSACTIONS_REQ, SOFT_LIMIT_BYTE_SIZE_POOLED_TRANSACTIONS_RESPONSE, }, @@ -37,8 +39,6 @@ use reth_network_peers::{mainnet_nodes, TrustedPeer}; use secp256k1::SecretKey; use tracing::error; -use crate::version::P2P_CLIENT_VERSION; - /// Parameters for configuring the network more granularity via CLI #[derive(Debug, Clone, Args, PartialEq, Eq)] #[command(next_help_heading = "Networking")] @@ -74,7 +74,7 @@ pub struct NetworkArgs { pub peers_file: Option, /// Custom node identity - #[arg(long, value_name = "IDENTITY", default_value = P2P_CLIENT_VERSION)] + #[arg(long, value_name = "IDENTITY", default_value = version_metadata().p2p_client_version.as_ref())] pub identity: String, /// Secret key to use for this node. @@ -161,6 +161,33 @@ pub struct NetworkArgs { /// The policy determines which peers transactions are gossiped to. #[arg(long = "tx-propagation-policy", default_value_t = TransactionPropagationKind::All)] pub tx_propagation_policy: TransactionPropagationKind, + + /// Disable transaction pool gossip + /// + /// Disables gossiping of transactions in the mempool to peers. This can be omitted for + /// personal nodes, though providers should always opt to enable this flag. + #[arg(long = "disable-tx-gossip")] + pub disable_tx_gossip: bool, + + /// Sets the transaction propagation mode by determining how new pending transactions are + /// propagated to other peers in full. + /// + /// Examples: sqrt, all, max:10 + #[arg( + long = "tx-propagation-mode", + default_value = "sqrt", + help = "Transaction propagation mode (sqrt, all, max:)" + )] + pub propagation_mode: TransactionPropagationMode, + + /// Comma separated list of required block hashes. + /// Peers that don't have these blocks will be filtered out. + #[arg(long = "required-block-hashes", value_delimiter = ',')] + pub required_block_hashes: Vec, + + /// Optional network ID to override the chain specification's network ID for P2P connections + #[arg(long)] + pub network_id: Option, } impl NetworkArgs { @@ -192,7 +219,7 @@ impl NetworkArgs { }) } /// Configures and returns a `TransactionsManagerConfig` based on the current settings. - pub fn transactions_manager_config(&self) -> TransactionsManagerConfig { + pub const fn transactions_manager_config(&self) -> TransactionsManagerConfig { TransactionsManagerConfig { transaction_fetcher_config: TransactionFetcherConfig::new( self.max_concurrent_tx_requests, @@ -202,7 +229,7 @@ impl NetworkArgs { self.max_capacity_cache_txns_pending_fetch, ), max_transactions_seen_by_peer_history: self.max_seen_tx_history, - propagation_mode: Default::default(), + propagation_mode: self.propagation_mode, } } @@ -272,6 +299,9 @@ impl NetworkArgs { // set discovery port based on instance number self.discovery.port, )) + .disable_tx_gossip(self.disable_tx_gossip) + .required_block_hashes(self.required_block_hashes.clone()) + .network_id(self.network_id) } /// If `no_persist_peers` is false then this returns the path to the persistent peers file path. @@ -325,7 +355,7 @@ impl Default for NetworkArgs { bootnodes: None, dns_retries: 0, peers_file: None, - identity: P2P_CLIENT_VERSION.to_string(), + identity: version_metadata().p2p_client_version.to_string(), p2p_secret_key: None, no_persist_peers: false, nat: NatResolver::Any, @@ -342,7 +372,11 @@ impl Default for NetworkArgs { max_seen_tx_history: DEFAULT_MAX_COUNT_TRANSACTIONS_SEEN_BY_PEER, max_capacity_cache_txns_pending_fetch: DEFAULT_MAX_CAPACITY_CACHE_PENDING_FETCH, net_if: None, - tx_propagation_policy: TransactionPropagationKind::default() + tx_propagation_policy: TransactionPropagationKind::default(), + disable_tx_gossip: false, + propagation_mode: TransactionPropagationMode::Sqrt, + required_block_hashes: vec![], + network_id: None, } } } @@ -441,7 +475,7 @@ impl DiscoveryArgs { network_config_builder = network_config_builder.disable_nat(); } - if !self.disable_discovery && self.enable_discv5_discovery { + if self.should_enable_discv5() { network_config_builder = network_config_builder .discovery_v5(self.discovery_v5_builder(rlpx_tcp_socket, boot_nodes)); } @@ -490,6 +524,17 @@ impl DiscoveryArgs { .bootstrap_lookup_countdown(*discv5_bootstrap_lookup_countdown) } + /// Returns true if discv5 discovery should be configured + const fn should_enable_discv5(&self) -> bool { + if self.disable_discovery { + return false; + } + + self.enable_discv5_discovery || + self.discv5_addr.is_some() || + self.discv5_addr_ipv6.is_some() + } + /// Set the discovery port to zero, to allow the OS to assign a random unused port when /// discovery binds to the socket. pub const fn with_unused_discovery_port(mut self) -> Self { @@ -606,6 +651,12 @@ mod tests { } } + #[test] + fn parse_disable_tx_gossip_args() { + let args = CommandParser::::parse_from(["reth", "--disable-tx-gossip"]).args; + assert!(args.disable_tx_gossip); + } + #[test] fn network_args_default_sanity_test() { let default_args = NetworkArgs::default(); @@ -613,4 +664,30 @@ mod tests { assert_eq!(args, default_args); } + + #[test] + fn parse_required_block_hashes() { + let args = CommandParser::::parse_from([ + "reth", + "--required-block-hashes", + "0x1111111111111111111111111111111111111111111111111111111111111111,0x2222222222222222222222222222222222222222222222222222222222222222", + ]) + .args; + + assert_eq!(args.required_block_hashes.len(), 2); + assert_eq!( + args.required_block_hashes[0].to_string(), + "0x1111111111111111111111111111111111111111111111111111111111111111" + ); + assert_eq!( + args.required_block_hashes[1].to_string(), + "0x2222222222222222222222222222222222222222222222222222222222222222" + ); + } + + #[test] + fn parse_empty_required_block_hashes() { + let args = CommandParser::::parse_from(["reth"]).args; + assert!(args.required_block_hashes.is_empty()); + } } diff --git a/crates/node/core/src/args/payload_builder.rs b/crates/node/core/src/args/payload_builder.rs index 83bbb6cce82..ca7befc0f08 100644 --- a/crates/node/core/src/args/payload_builder.rs +++ b/crates/node/core/src/args/payload_builder.rs @@ -1,6 +1,6 @@ use crate::{cli::config::PayloadBuilderConfig, version::default_extra_data}; use alloy_consensus::constants::MAXIMUM_EXTRA_DATA_SIZE; -use alloy_eips::{eip1559::ETHEREUM_BLOCK_GAS_LIMIT_36M, merge::SLOT_DURATION}; +use alloy_eips::merge::SLOT_DURATION; use clap::{ builder::{RangedU64ValueParser, TypedValueParser}, Arg, Args, Command, @@ -17,8 +17,8 @@ pub struct PayloadBuilderArgs { pub extra_data: String, /// Target gas limit for built blocks. - #[arg(long = "builder.gaslimit", default_value_t = ETHEREUM_BLOCK_GAS_LIMIT_36M, value_name = "GAS_LIMIT")] - pub gas_limit: u64, + #[arg(long = "builder.gaslimit", alias = "miner.gaslimit", value_name = "GAS_LIMIT")] + pub gas_limit: Option, /// The interval at which the job should build a new payload after the last. /// @@ -41,8 +41,8 @@ impl Default for PayloadBuilderArgs { fn default() -> Self { Self { extra_data: default_extra_data(), - gas_limit: ETHEREUM_BLOCK_GAS_LIMIT_36M, interval: Duration::from_secs(1), + gas_limit: None, deadline: SLOT_DURATION, max_payload_tasks: 3, } @@ -62,7 +62,7 @@ impl PayloadBuilderConfig for PayloadBuilderArgs { self.deadline } - fn gas_limit(&self) -> u64 { + fn gas_limit(&self) -> Option { self.gas_limit } diff --git a/crates/node/core/src/args/pruning.rs b/crates/node/core/src/args/pruning.rs index cd852b9e168..2385911ee97 100644 --- a/crates/node/core/src/args/pruning.rs +++ b/crates/node/core/src/args/pruning.rs @@ -1,12 +1,12 @@ //! Pruning and full node arguments -use crate::args::error::ReceiptsLogError; +use crate::{args::error::ReceiptsLogError, primitives::EthereumHardfork}; use alloy_primitives::{Address, BlockNumber}; use clap::{builder::RangedU64ValueParser, Args}; -use reth_chainspec::EthChainSpec; +use reth_chainspec::EthereumHardforks; use reth_config::config::PruneConfig; use reth_prune_types::{PruneMode, PruneModes, ReceiptsLogPruneConfig, MINIMUM_PRUNING_DISTANCE}; -use std::collections::BTreeMap; +use std::{collections::BTreeMap, ops::Not}; /// Parameters for pruning and full node #[derive(Debug, Clone, Args, PartialEq, Eq, Default)] @@ -17,83 +17,104 @@ pub struct PruningArgs { pub full: bool, /// Minimum pruning interval measured in blocks. - #[arg(long, value_parser = RangedU64ValueParser::::new().range(1..),)] + #[arg(long = "prune.block-interval", alias = "block-interval", value_parser = RangedU64ValueParser::::new().range(1..))] pub block_interval: Option, // Sender Recovery /// Prunes all sender recovery data. - #[arg(long = "prune.senderrecovery.full", conflicts_with_all = &["sender_recovery_distance", "sender_recovery_before"])] + #[arg(long = "prune.sender-recovery.full", alias = "prune.senderrecovery.full", conflicts_with_all = &["sender_recovery_distance", "sender_recovery_before"])] pub sender_recovery_full: bool, /// Prune sender recovery data before the `head-N` block number. In other words, keep last N + /// 1 blocks. - #[arg(long = "prune.senderrecovery.distance", value_name = "BLOCKS", conflicts_with_all = &["sender_recovery_full", "sender_recovery_before"])] + #[arg(long = "prune.sender-recovery.distance", alias = "prune.senderrecovery.distance", value_name = "BLOCKS", conflicts_with_all = &["sender_recovery_full", "sender_recovery_before"])] pub sender_recovery_distance: Option, /// Prune sender recovery data before the specified block number. The specified block number is /// not pruned. - #[arg(long = "prune.senderrecovery.before", value_name = "BLOCK_NUMBER", conflicts_with_all = &["sender_recovery_full", "sender_recovery_distance"])] + #[arg(long = "prune.sender-recovery.before", alias = "prune.senderrecovery.before", value_name = "BLOCK_NUMBER", conflicts_with_all = &["sender_recovery_full", "sender_recovery_distance"])] pub sender_recovery_before: Option, // Transaction Lookup /// Prunes all transaction lookup data. - #[arg(long = "prune.transactionlookup.full", conflicts_with_all = &["transaction_lookup_distance", "transaction_lookup_before"])] + #[arg(long = "prune.transaction-lookup.full", alias = "prune.transactionlookup.full", conflicts_with_all = &["transaction_lookup_distance", "transaction_lookup_before"])] pub transaction_lookup_full: bool, /// Prune transaction lookup data before the `head-N` block number. In other words, keep last N /// + 1 blocks. - #[arg(long = "prune.transactionlookup.distance", value_name = "BLOCKS", conflicts_with_all = &["transaction_lookup_full", "transaction_lookup_before"])] + #[arg(long = "prune.transaction-lookup.distance", alias = "prune.transactionlookup.distance", value_name = "BLOCKS", conflicts_with_all = &["transaction_lookup_full", "transaction_lookup_before"])] pub transaction_lookup_distance: Option, /// Prune transaction lookup data before the specified block number. The specified block number /// is not pruned. - #[arg(long = "prune.transactionlookup.before", value_name = "BLOCK_NUMBER", conflicts_with_all = &["transaction_lookup_full", "transaction_lookup_distance"])] + #[arg(long = "prune.transaction-lookup.before", alias = "prune.transactionlookup.before", value_name = "BLOCK_NUMBER", conflicts_with_all = &["transaction_lookup_full", "transaction_lookup_distance"])] pub transaction_lookup_before: Option, // Receipts /// Prunes all receipt data. - #[arg(long = "prune.receipts.full", conflicts_with_all = &["receipts_distance", "receipts_before"])] + #[arg(long = "prune.receipts.full", conflicts_with_all = &["receipts_pre_merge", "receipts_distance", "receipts_before"])] pub receipts_full: bool, + /// Prune receipts before the merge block. + #[arg(long = "prune.receipts.pre-merge", conflicts_with_all = &["receipts_full", "receipts_distance", "receipts_before"])] + pub receipts_pre_merge: bool, /// Prune receipts before the `head-N` block number. In other words, keep last N + 1 blocks. - #[arg(long = "prune.receipts.distance", value_name = "BLOCKS", conflicts_with_all = &["receipts_full", "receipts_before"])] + #[arg(long = "prune.receipts.distance", value_name = "BLOCKS", conflicts_with_all = &["receipts_full", "receipts_pre_merge", "receipts_before"])] pub receipts_distance: Option, /// Prune receipts before the specified block number. The specified block number is not pruned. - #[arg(long = "prune.receipts.before", value_name = "BLOCK_NUMBER", conflicts_with_all = &["receipts_full", "receipts_distance"])] + #[arg(long = "prune.receipts.before", value_name = "BLOCK_NUMBER", conflicts_with_all = &["receipts_full", "receipts_pre_merge", "receipts_distance"])] pub receipts_before: Option, + // Receipts Log Filter + /// Configure receipts log filter. Format: + /// <`address`>:<`prune_mode`>... where <`prune_mode`> can be 'full', 'distance:<`blocks`>', or + /// 'before:<`block_number`>' + #[arg(long = "prune.receiptslogfilter", value_name = "FILTER_CONFIG", conflicts_with_all = &["receipts_full", "receipts_pre_merge", "receipts_distance", "receipts_before"], value_parser = parse_receipts_log_filter)] + pub receipts_log_filter: Option, // Account History /// Prunes all account history. - #[arg(long = "prune.accounthistory.full", conflicts_with_all = &["account_history_distance", "account_history_before"])] + #[arg(long = "prune.account-history.full", alias = "prune.accounthistory.full", conflicts_with_all = &["account_history_distance", "account_history_before"])] pub account_history_full: bool, /// Prune account before the `head-N` block number. In other words, keep last N + 1 blocks. - #[arg(long = "prune.accounthistory.distance", value_name = "BLOCKS", conflicts_with_all = &["account_history_full", "account_history_before"])] + #[arg(long = "prune.account-history.distance", alias = "prune.accounthistory.distance", value_name = "BLOCKS", conflicts_with_all = &["account_history_full", "account_history_before"])] pub account_history_distance: Option, /// Prune account history before the specified block number. The specified block number is not /// pruned. - #[arg(long = "prune.accounthistory.before", value_name = "BLOCK_NUMBER", conflicts_with_all = &["account_history_full", "account_history_distance"])] + #[arg(long = "prune.account-history.before", alias = "prune.accounthistory.before", value_name = "BLOCK_NUMBER", conflicts_with_all = &["account_history_full", "account_history_distance"])] pub account_history_before: Option, // Storage History /// Prunes all storage history data. - #[arg(long = "prune.storagehistory.full", conflicts_with_all = &["storage_history_distance", "storage_history_before"])] + #[arg(long = "prune.storage-history.full", alias = "prune.storagehistory.full", conflicts_with_all = &["storage_history_distance", "storage_history_before"])] pub storage_history_full: bool, /// Prune storage history before the `head-N` block number. In other words, keep last N + 1 /// blocks. - #[arg(long = "prune.storagehistory.distance", value_name = "BLOCKS", conflicts_with_all = &["storage_history_full", "storage_history_before"])] + #[arg(long = "prune.storage-history.distance", alias = "prune.storagehistory.distance", value_name = "BLOCKS", conflicts_with_all = &["storage_history_full", "storage_history_before"])] pub storage_history_distance: Option, /// Prune storage history before the specified block number. The specified block number is not /// pruned. - #[arg(long = "prune.storagehistory.before", value_name = "BLOCK_NUMBER", conflicts_with_all = &["storage_history_full", "storage_history_distance"])] + #[arg(long = "prune.storage-history.before", alias = "prune.storagehistory.before", value_name = "BLOCK_NUMBER", conflicts_with_all = &["storage_history_full", "storage_history_distance"])] pub storage_history_before: Option, - // Receipts Log Filter - /// Configure receipts log filter. Format: - /// <`address`>:<`prune_mode`>[,<`address`>:<`prune_mode`>...] Where <`prune_mode`> can be - /// 'full', 'distance:<`blocks`>', or 'before:<`block_number`>' - #[arg(long = "prune.receiptslogfilter", value_name = "FILTER_CONFIG", value_delimiter = ',', value_parser = parse_receipts_log_filter)] - pub receipts_log_filter: Vec, + // Bodies + /// Prune bodies before the merge block. + #[arg(long = "prune.bodies.pre-merge", value_name = "BLOCKS", conflicts_with_all = &["bodies_distance", "bodies_before"])] + pub bodies_pre_merge: bool, + /// Prune bodies before the `head-N` block number. In other words, keep last N + 1 + /// blocks. + #[arg(long = "prune.bodies.distance", value_name = "BLOCKS", conflicts_with_all = &["bodies_pre_merge", "bodies_before"])] + pub bodies_distance: Option, + /// Prune storage history before the specified block number. The specified block number is not + /// pruned. + #[arg(long = "prune.bodies.before", value_name = "BLOCK_NUMBER", conflicts_with_all = &["bodies_distance", "bodies_pre_merge"])] + pub bodies_before: Option, } impl PruningArgs { /// Returns pruning configuration. - pub fn prune_config(&self, chain_spec: &impl EthChainSpec) -> Option { - // Initialise with a default prune configuration. + /// + /// Returns [`None`] if no parameters are specified and default pruning configuration should be + /// used. + pub fn prune_config(&self, chain_spec: &ChainSpec) -> Option + where + ChainSpec: EthereumHardforks, + { + // Initialize with a default prune configuration. let mut config = PruneConfig::default(); // If --full is set, use full node defaults. @@ -103,21 +124,15 @@ impl PruningArgs { segments: PruneModes { sender_recovery: Some(PruneMode::Full), transaction_lookup: None, - // prune all receipts if chain doesn't have deposit contract specified in chain - // spec - receipts: chain_spec - .deposit_contract() - .map(|contract| PruneMode::Before(contract.block)) - .or(Some(PruneMode::Distance(MINIMUM_PRUNING_DISTANCE))), + receipts: Some(PruneMode::Distance(MINIMUM_PRUNING_DISTANCE)), account_history: Some(PruneMode::Distance(MINIMUM_PRUNING_DISTANCE)), storage_history: Some(PruneMode::Distance(MINIMUM_PRUNING_DISTANCE)), - receipts_log_filter: ReceiptsLogPruneConfig( - chain_spec - .deposit_contract() - .map(|contract| (contract.address, PruneMode::Before(contract.block))) - .into_iter() - .collect(), - ), + bodies_history: chain_spec + .ethereum_fork_activation(EthereumHardfork::Paris) + .block_number() + .map(PruneMode::Before), + merkle_changesets: PruneMode::Distance(MINIMUM_PRUNING_DISTANCE), + receipts_log_filter: Default::default(), }, } } @@ -132,18 +147,46 @@ impl PruningArgs { if let Some(mode) = self.transaction_lookup_prune_mode() { config.segments.transaction_lookup = Some(mode); } - if let Some(mode) = self.receipts_prune_mode() { + if let Some(mode) = self.receipts_prune_mode(chain_spec) { config.segments.receipts = Some(mode); } if let Some(mode) = self.account_history_prune_mode() { config.segments.account_history = Some(mode); } + if let Some(mode) = self.bodies_prune_mode(chain_spec) { + config.segments.bodies_history = Some(mode); + } if let Some(mode) = self.storage_history_prune_mode() { config.segments.storage_history = Some(mode); } + if let Some(receipt_logs) = + self.receipts_log_filter.as_ref().filter(|c| !c.is_empty()).cloned() + { + config.segments.receipts_log_filter = receipt_logs; + // need to remove the receipts segment filter entirely because that takes precedence + // over the logs filter + config.segments.receipts.take(); + } - Some(config) + config.is_default().not().then_some(config) } + + fn bodies_prune_mode(&self, chain_spec: &ChainSpec) -> Option + where + ChainSpec: EthereumHardforks, + { + if self.bodies_pre_merge { + chain_spec + .ethereum_fork_activation(EthereumHardfork::Paris) + .block_number() + .map(PruneMode::Before) + } else if let Some(distance) = self.bodies_distance { + Some(PruneMode::Distance(distance)) + } else { + self.bodies_before.map(PruneMode::Before) + } + } + const fn sender_recovery_prune_mode(&self) -> Option { if self.sender_recovery_full { Some(PruneMode::Full) @@ -168,15 +211,21 @@ impl PruningArgs { } } - const fn receipts_prune_mode(&self) -> Option { - if self.receipts_full { + fn receipts_prune_mode(&self, chain_spec: &ChainSpec) -> Option + where + ChainSpec: EthereumHardforks, + { + if self.receipts_pre_merge { + chain_spec + .ethereum_fork_activation(EthereumHardfork::Paris) + .block_number() + .map(PruneMode::Before) + } else if self.receipts_full { Some(PruneMode::Full) } else if let Some(distance) = self.receipts_distance { Some(PruneMode::Distance(distance)) - } else if let Some(block_number) = self.receipts_before { - Some(PruneMode::Before(block_number)) } else { - None + self.receipts_before.map(PruneMode::Before) } } @@ -205,6 +254,7 @@ impl PruningArgs { } } +/// Parses `,` separated pruning info into [`ReceiptsLogPruneConfig`]. pub(crate) fn parse_receipts_log_filter( value: &str, ) -> Result { @@ -236,9 +286,8 @@ pub(crate) fn parse_receipts_log_filter( if parts.len() < 3 { return Err(ReceiptsLogError::InvalidFilterFormat(filter.to_string())); } - let block_number = parts[2] - .parse::() - .map_err(ReceiptsLogError::InvalidBlockNumber)?; + let block_number = + parts[2].parse::().map_err(ReceiptsLogError::InvalidBlockNumber)?; PruneMode::Before(block_number) } _ => return Err(ReceiptsLogError::InvalidPruneMode(parts[1].to_string())), @@ -251,6 +300,7 @@ pub(crate) fn parse_receipts_log_filter( #[cfg(test)] mod tests { use super::*; + use alloy_primitives::address; use clap::Parser; /// A helper type to parse Args more easily @@ -262,6 +312,22 @@ mod tests { #[test] fn pruning_args_sanity_check() { + let args = CommandParser::::parse_from([ + "reth", + "--prune.receiptslogfilter", + "0x0000000000000000000000000000000000000003:before:5000000", + ]) + .args; + let mut config = ReceiptsLogPruneConfig::default(); + config.0.insert( + address!("0x0000000000000000000000000000000000000003"), + PruneMode::Before(5000000), + ); + assert_eq!(args.receipts_log_filter, Some(config)); + } + + #[test] + fn parse_receiptslogfilter() { let default_args = PruningArgs::default(); let args = CommandParser::::parse_from(["reth"]).args; assert_eq!(args, default_args); diff --git a/crates/node/core/src/args/rpc_server.rs b/crates/node/core/src/args/rpc_server.rs index 1835c4910a2..f4930db9f9b 100644 --- a/crates/node/core/src/args/rpc_server.rs +++ b/crates/node/core/src/args/rpc_server.rs @@ -1,12 +1,9 @@ //! clap [Args](clap::Args) for RPC related arguments. -use std::{ - collections::HashSet, - ffi::OsStr, - net::{IpAddr, Ipv4Addr}, - path::PathBuf, +use crate::args::{ + types::{MaxU32, ZeroAsNoneU64}, + GasPriceOracleArgs, RpcStateCacheArgs, }; - use alloy_primitives::Address; use alloy_rpc_types_engine::JwtSecret; use clap::{ @@ -14,13 +11,17 @@ use clap::{ Arg, Args, Command, }; use rand::Rng; -use reth_cli_util::parse_ether_value; +use reth_cli_util::{parse_duration_from_secs_or_ms, parse_ether_value}; +use reth_rpc_eth_types::builder::config::PendingBlockKind; use reth_rpc_server_types::{constants, RethRpcModule, RpcModuleSelection}; - -use crate::args::{ - types::{MaxU32, ZeroAsNoneU64}, - GasPriceOracleArgs, RpcStateCacheArgs, +use std::{ + collections::HashSet, + ffi::OsStr, + net::{IpAddr, Ipv4Addr}, + path::PathBuf, + time::Duration, }; +use url::Url; use super::types::MaxOr; @@ -54,6 +55,10 @@ pub struct RpcServerArgs { #[arg(long = "http.port", default_value_t = constants::DEFAULT_HTTP_RPC_PORT)] pub http_port: u16, + /// Disable compression for HTTP responses + #[arg(long = "http.disable-compression", default_value_t = false)] + pub http_disable_compression: bool, + /// Rpc Modules to be configured for the HTTP server #[arg(long = "http.api", value_parser = RpcModuleSelectionValueParser::default())] pub http_api: Option, @@ -90,6 +95,12 @@ pub struct RpcServerArgs { #[arg(long, default_value_t = constants::DEFAULT_IPC_ENDPOINT.to_string())] pub ipcpath: String, + /// Set the permissions for the IPC socket file, in octal format. + /// + /// If not specified, the permissions will be set by the system's umask. + #[arg(long = "ipc.permissions")] + pub ipc_socket_permissions: Option, + /// Auth server address to listen on #[arg(long = "authrpc.addr", default_value_t = IpAddr::V4(Ipv4Addr::LOCALHOST))] pub auth_addr: IpAddr, @@ -115,6 +126,13 @@ pub struct RpcServerArgs { #[arg(long = "auth-ipc.path", default_value_t = constants::DEFAULT_ENGINE_API_IPC_ENDPOINT.to_string())] pub auth_ipc_path: String, + /// Disable the auth/engine API server. + /// + /// This will prevent the authenticated engine-API server from starting. Use this if you're + /// running a node that doesn't need to serve engine API requests. + #[arg(long = "disable-auth-server", alias = "disable-engine-api")] + pub disable_auth_server: bool, + /// Hex encoded JWT secret to authenticate the regular RPC server(s), see `--http.api` and /// `--ws.api`. /// @@ -170,7 +188,17 @@ pub struct RpcServerArgs { )] pub rpc_gas_cap: u64, - /// Maximum eth transaction fee that can be sent via the RPC APIs (0 = no cap) + /// Maximum memory the EVM can allocate per RPC request. + #[arg( + long = "rpc.evm-memory-limit", + alias = "rpc-evm-memory-limit", + value_name = "MEMORY_LIMIT", + value_parser = MaxOr::new(RangedU64ValueParser::::new().range(1..)), + default_value_t = (1 << 32) - 1 + )] + pub rpc_evm_memory_limit: u64, + + /// Maximum eth transaction fee (in ether) that can be sent via the RPC APIs (0 = no cap) #[arg( long = "rpc.txfeecap", alias = "rpc-txfeecap", @@ -202,6 +230,17 @@ pub struct RpcServerArgs { #[arg(long = "rpc.proof-permits", alias = "rpc-proof-permits", value_name = "COUNT", default_value_t = constants::DEFAULT_PROOF_PERMITS)] pub rpc_proof_permits: usize, + /// Configures the pending block behavior for RPC responses. + /// + /// Options: full (include all transactions), empty (header only), none (disable pending + /// blocks). + #[arg(long = "rpc.pending-block", default_value = "full", value_name = "KIND")] + pub rpc_pending_block: PendingBlockKind, + + /// Endpoint to forward transactions to. + #[arg(long = "rpc.forwarder", alias = "rpc-forwarder", value_name = "FORWARDER")] + pub rpc_forwarder: Option, + /// Path to file containing disallowed addresses, json-encoded list of strings. Block /// validation API will reject blocks containing transactions from these addresses. #[arg(long = "builder.disallow", value_name = "PATH", value_parser = reth_cli_util::parsers::read_json_from_file::>)] @@ -214,6 +253,15 @@ pub struct RpcServerArgs { /// Gas price oracle configuration. #[command(flatten)] pub gas_price_oracle: GasPriceOracleArgs, + + /// Timeout for `send_raw_transaction_sync` RPC method. + #[arg( + long = "rpc.send-raw-transaction-sync-timeout", + value_name = "SECONDS", + default_value = "30s", + value_parser = parse_duration_from_secs_or_ms, + )] + pub rpc_send_raw_transaction_sync_timeout: Duration, } impl RpcServerArgs { @@ -235,12 +283,25 @@ impl RpcServerArgs { self } + /// Configures modules for WS-RPC server. + pub fn with_ws_api(mut self, ws_api: RpcModuleSelection) -> Self { + self.ws_api = Some(ws_api); + self + } + /// Enables the Auth IPC pub const fn with_auth_ipc(mut self) -> Self { self.auth_ipc = true; self } + /// Configures modules for both the HTTP-RPC server and WS-RPC server. + /// + /// This is the same as calling both [`Self::with_http_api`] and [`Self::with_ws_api`]. + pub fn with_api(self, api: RpcModuleSelection) -> Self { + self.with_http_api(api.clone()).with_ws_api(api) + } + /// Change rpc port numbers based on the instance number, if provided. /// * The `auth_port` is scaled by a factor of `instance * 100` /// * The `http_port` is scaled by a factor of `-instance` @@ -308,6 +369,20 @@ impl RpcServerArgs { self = self.with_ipc_random_path(); self } + + /// Apply a function to the args. + pub fn apply(self, f: F) -> Self + where + F: FnOnce(Self) -> Self, + { + f(self) + } + + /// Configures the timeout for send raw transaction sync. + pub const fn with_send_raw_transaction_sync_timeout(mut self, timeout: Duration) -> Self { + self.rpc_send_raw_transaction_sync_timeout = timeout; + self + } } impl Default for RpcServerArgs { @@ -316,6 +391,7 @@ impl Default for RpcServerArgs { http: false, http_addr: Ipv4Addr::LOCALHOST.into(), http_port: constants::DEFAULT_HTTP_RPC_PORT, + http_disable_compression: false, http_api: None, http_corsdomain: None, ws: false, @@ -325,11 +401,13 @@ impl Default for RpcServerArgs { ws_api: None, ipcdisable: false, ipcpath: constants::DEFAULT_IPC_ENDPOINT.to_string(), + ipc_socket_permissions: None, auth_addr: Ipv4Addr::LOCALHOST.into(), auth_port: constants::DEFAULT_AUTH_PORT, auth_jwtsecret: None, auth_ipc: false, auth_ipc_path: constants::DEFAULT_ENGINE_API_IPC_ENDPOINT.to_string(), + disable_auth_server: false, rpc_jwtsecret: None, rpc_max_request_size: RPC_DEFAULT_MAX_REQUEST_SIZE_MB.into(), rpc_max_response_size: RPC_DEFAULT_MAX_RESPONSE_SIZE_MB.into(), @@ -340,18 +418,23 @@ impl Default for RpcServerArgs { rpc_max_blocks_per_filter: constants::DEFAULT_MAX_BLOCKS_PER_FILTER.into(), rpc_max_logs_per_response: (constants::DEFAULT_MAX_LOGS_PER_RESPONSE as u64).into(), rpc_gas_cap: constants::gas_oracle::RPC_DEFAULT_GAS_CAP, + rpc_evm_memory_limit: (1 << 32) - 1, rpc_tx_fee_cap: constants::DEFAULT_TX_FEE_CAP_WEI, rpc_max_simulate_blocks: constants::DEFAULT_MAX_SIMULATE_BLOCKS, rpc_eth_proof_window: constants::DEFAULT_ETH_PROOF_WINDOW, + rpc_pending_block: PendingBlockKind::Full, gas_price_oracle: GasPriceOracleArgs::default(), rpc_state_cache: RpcStateCacheArgs::default(), rpc_proof_permits: constants::DEFAULT_PROOF_PERMITS, + rpc_forwarder: None, builder_disallow: Default::default(), + rpc_send_raw_transaction_sync_timeout: + constants::RPC_DEFAULT_SEND_RAW_TX_SYNC_TIMEOUT_SECS, } } } -/// clap value parser for [`RpcModuleSelection`]. +/// clap value parser for [`RpcModuleSelection`] with configurable validation. #[derive(Clone, Debug, Default)] #[non_exhaustive] struct RpcModuleSelectionValueParser; @@ -362,23 +445,20 @@ impl TypedValueParser for RpcModuleSelectionValueParser { fn parse_ref( &self, _cmd: &Command, - arg: Option<&Arg>, + _arg: Option<&Arg>, value: &OsStr, ) -> Result { let val = value.to_str().ok_or_else(|| clap::Error::new(clap::error::ErrorKind::InvalidUtf8))?; - val.parse::().map_err(|err| { - let arg = arg.map(|a| a.to_string()).unwrap_or_else(|| "...".to_owned()); - let possible_values = RethRpcModule::all_variant_names().to_vec().join(","); - let msg = format!( - "Invalid value '{val}' for {arg}: {err}.\n [possible values: {possible_values}]" - ); - clap::Error::raw(clap::error::ErrorKind::InvalidValue, msg) - }) + // This will now accept any module name, creating Other(name) for unknowns + Ok(val + .parse::() + .expect("RpcModuleSelection parsing cannot fail with Other variant")) } fn possible_values(&self) -> Option + '_>> { - let values = RethRpcModule::all_variant_names().iter().map(PossibleValue::new); + // Only show standard modules in help text (excludes "other") + let values = RethRpcModule::standard_variant_names().map(PossibleValue::new); Some(Box::new(values)) } } diff --git a/crates/node/core/src/args/stage.rs b/crates/node/core/src/args/stage.rs index 337f5a4a60b..7718fb85605 100644 --- a/crates/node/core/src/args/stage.rs +++ b/crates/node/core/src/args/stage.rs @@ -38,6 +38,11 @@ pub enum StageEnum { /// /// Handles Merkle tree-related computations and data processing. Merkle, + /// The merkle changesets stage within the pipeline. + /// + /// Handles Merkle trie changesets for storage and accounts. + #[value(name = "merkle-changesets")] + MerkleChangeSets, /// The transaction lookup stage within the pipeline. /// /// Deals with the retrieval and processing of transactions. diff --git a/crates/node/core/src/args/trace.rs b/crates/node/core/src/args/trace.rs new file mode 100644 index 00000000000..5b5e21502d1 --- /dev/null +++ b/crates/node/core/src/args/trace.rs @@ -0,0 +1,89 @@ +//! Opentelemetry tracing configuration through CLI args. + +use clap::Parser; +use eyre::WrapErr; +use reth_tracing::tracing_subscriber::EnvFilter; +use reth_tracing_otlp::OtlpProtocol; +use url::Url; + +/// CLI arguments for configuring `Opentelemetry` trace and span export. +#[derive(Debug, Clone, Parser)] +pub struct TraceArgs { + /// Enable `Opentelemetry` tracing export to an OTLP endpoint. + /// + /// If no value provided, defaults based on protocol: + /// - HTTP: `http://localhost:4318/v1/traces` + /// - gRPC: `http://localhost:4317` + /// + /// Example: --tracing-otlp=http://collector:4318/v1/traces + #[arg( + long = "tracing-otlp", + // Per specification. + env = "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", + global = true, + value_name = "URL", + num_args = 0..=1, + default_missing_value = "http://localhost:4318/v1/traces", + require_equals = true, + value_parser = parse_otlp_endpoint, + help_heading = "Tracing" + )] + pub otlp: Option, + + /// OTLP transport protocol to use for exporting traces. + /// + /// - `http`: expects endpoint path to end with `/v1/traces` + /// - `grpc`: expects endpoint without a path + /// + /// Defaults to HTTP if not specified. + #[arg( + long = "tracing-otlp-protocol", + env = "OTEL_EXPORTER_OTLP_PROTOCOL", + global = true, + value_name = "PROTOCOL", + default_value = "http", + help_heading = "Tracing" + )] + pub protocol: OtlpProtocol, + + /// Set a filter directive for the OTLP tracer. This controls the verbosity + /// of spans and events sent to the OTLP endpoint. It follows the same + /// syntax as the `RUST_LOG` environment variable. + /// + /// Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off + /// + /// Defaults to TRACE if not specified. + #[arg( + long = "tracing-otlp.filter", + global = true, + value_name = "FILTER", + default_value = "debug", + help_heading = "Tracing" + )] + pub otlp_filter: EnvFilter, +} + +impl Default for TraceArgs { + fn default() -> Self { + Self { + otlp: None, + protocol: OtlpProtocol::Http, + otlp_filter: EnvFilter::from_default_env(), + } + } +} + +impl TraceArgs { + /// Validate the configuration + pub fn validate(&mut self) -> eyre::Result<()> { + if let Some(url) = &mut self.otlp { + self.protocol.validate_endpoint(url)?; + } + Ok(()) + } +} + +// Parses an OTLP endpoint url. +fn parse_otlp_endpoint(arg: &str) -> eyre::Result { + Url::parse(arg).wrap_err("Invalid URL for OTLP trace output") +} diff --git a/crates/node/core/src/args/txpool.rs b/crates/node/core/src/args/txpool.rs index 59b920cc604..2ab604be168 100644 --- a/crates/node/core/src/args/txpool.rs +++ b/crates/node/core/src/args/txpool.rs @@ -65,10 +65,20 @@ pub struct TxPoolArgs { #[arg(long = "txpool.minimal-protocol-fee", default_value_t = MIN_PROTOCOL_BASE_FEE)] pub minimal_protocol_basefee: u64, + /// Minimum priority fee required for transaction acceptance into the pool. + /// Transactions with priority fee below this value will be rejected. + #[arg(long = "txpool.minimum-priority-fee")] + pub minimum_priority_fee: Option, + /// The default enforced gas limit for transactions entering the pool #[arg(long = "txpool.gas-limit", default_value_t = ETHEREUM_BLOCK_GAS_LIMIT_30M)] pub enforced_gas_limit: u64, + /// Maximum gas limit for individual transactions. Transactions exceeding this limit will be + /// rejected by the transaction pool + #[arg(long = "txpool.max-tx-gas")] + pub max_tx_gas_limit: Option, + /// Price bump percentage to replace an already existing blob transaction #[arg(long = "blobpool.pricebump", default_value_t = REPLACE_BLOB_PRICE_BUMP)] pub blob_transaction_price_bump: u128, @@ -122,6 +132,28 @@ pub struct TxPoolArgs { conflicts_with = "transactions_backup_path" )] pub disable_transactions_backup: bool, + + /// Max batch size for transaction pool insertions + #[arg(long = "txpool.max-batch-size", default_value_t = 1)] + pub max_batch_size: usize, +} + +impl TxPoolArgs { + /// Sets the minimal protocol base fee to 0, effectively disabling checks that enforce that a + /// transaction's fee must be higher than the [`MIN_PROTOCOL_BASE_FEE`] which is the lowest + /// value the ethereum EIP-1559 base fee can reach. + pub const fn with_disabled_protocol_base_fee(self) -> Self { + self.with_protocol_base_fee(0) + } + + /// Configures the minimal protocol base fee that should be enforced. + /// + /// Ethereum's EIP-1559 base fee can't drop below [`MIN_PROTOCOL_BASE_FEE`] hence this is + /// enforced by default in the pool. + pub const fn with_protocol_base_fee(mut self, protocol_base_fee: u64) -> Self { + self.minimal_protocol_basefee = protocol_base_fee; + self + } } impl Default for TxPoolArgs { @@ -139,7 +171,9 @@ impl Default for TxPoolArgs { max_account_slots: TXPOOL_MAX_ACCOUNT_SLOTS_PER_SENDER, price_bump: DEFAULT_PRICE_BUMP, minimal_protocol_basefee: MIN_PROTOCOL_BASE_FEE, + minimum_priority_fee: None, enforced_gas_limit: ETHEREUM_BLOCK_GAS_LIMIT_30M, + max_tx_gas_limit: None, blob_transaction_price_bump: REPLACE_BLOB_PRICE_BUMP, max_tx_input_bytes: DEFAULT_MAX_TX_INPUT_BYTES, max_cached_entries: DEFAULT_MAX_CACHED_BLOBS, @@ -153,6 +187,7 @@ impl Default for TxPoolArgs { max_queued_lifetime: MAX_QUEUED_TRANSACTION_LIFETIME, transactions_backup_path: None, disable_transactions_backup: false, + max_batch_size: 1, } } } @@ -189,13 +224,20 @@ impl RethTransactionPoolConfig for TxPoolArgs { replace_blob_tx_price_bump: self.blob_transaction_price_bump, }, minimal_protocol_basefee: self.minimal_protocol_basefee, + minimum_priority_fee: self.minimum_priority_fee, gas_limit: self.enforced_gas_limit, pending_tx_listener_buffer_size: self.pending_tx_listener_buffer_size, new_tx_listener_buffer_size: self.new_tx_listener_buffer_size, max_new_pending_txs_notifications: self.max_new_pending_txs_notifications, max_queued_lifetime: self.max_queued_lifetime, + ..Default::default() } } + + /// Returns max batch size for transaction batch insertion. + fn max_batch_size(&self) -> usize { + self.max_batch_size + } } #[cfg(test)] diff --git a/crates/node/core/src/cli/config.rs b/crates/node/core/src/cli/config.rs index 6c34defb439..8c29c4745e9 100644 --- a/crates/node/core/src/cli/config.rs +++ b/crates/node/core/src/cli/config.rs @@ -1,10 +1,15 @@ //! Config traits for various node components. +use alloy_eips::eip1559::ETHEREUM_BLOCK_GAS_LIMIT_36M; use alloy_primitives::Bytes; +use reth_chainspec::{Chain, ChainKind, NamedChain}; use reth_network::{protocol::IntoRlpxSubProtocol, NetworkPrimitives}; use reth_transaction_pool::PoolConfig; use std::{borrow::Cow, time::Duration}; +/// 60M gas limit +const ETHEREUM_BLOCK_GAS_LIMIT_60M: u64 = 60_000_000; + /// A trait that provides payload builder settings. /// /// This provides all basic payload builder settings and is implemented by the @@ -25,10 +30,24 @@ pub trait PayloadBuilderConfig { fn deadline(&self) -> Duration; /// Target gas limit for built blocks. - fn gas_limit(&self) -> u64; + fn gas_limit(&self) -> Option; /// Maximum number of tasks to spawn for building a payload. fn max_payload_tasks(&self) -> usize; + + /// Returns the configured gas limit if set, or a chain-specific default. + fn gas_limit_for(&self, chain: Chain) -> u64 { + if let Some(limit) = self.gas_limit() { + return limit; + } + + match chain.kind() { + ChainKind::Named( + NamedChain::Mainnet | NamedChain::Sepolia | NamedChain::Holesky | NamedChain::Hoodi, + ) => ETHEREUM_BLOCK_GAS_LIMIT_60M, + _ => ETHEREUM_BLOCK_GAS_LIMIT_36M, + } + } } /// A trait that represents the configured network and can be used to apply additional configuration @@ -64,4 +83,7 @@ impl RethNetworkConfig for reth_network::NetworkManager pub trait RethTransactionPoolConfig { /// Returns transaction pool configuration. fn pool_config(&self) -> PoolConfig; + + /// Returns max batch size for transaction batch insertion. + fn max_batch_size(&self) -> usize; } diff --git a/crates/node/core/src/lib.rs b/crates/node/core/src/lib.rs index aa4f72bd6a4..924bf797825 100644 --- a/crates/node/core/src/lib.rs +++ b/crates/node/core/src/lib.rs @@ -6,7 +6,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] pub mod args; pub mod cli; @@ -31,6 +31,6 @@ pub mod rpc { /// Re-exported from `reth_rpc::eth`. pub mod compat { - pub use reth_rpc_types_compat::*; + pub use reth_rpc_convert::*; } } diff --git a/crates/node/core/src/node_config.rs b/crates/node/core/src/node_config.rs index 1c76de3fe20..64b469086e7 100644 --- a/crates/node/core/src/node_config.rs +++ b/crates/node/core/src/node_config.rs @@ -10,11 +10,12 @@ use crate::{ }; use alloy_consensus::BlockHeader; use alloy_eips::BlockHashOrNumber; -use alloy_primitives::{BlockNumber, B256}; +use alloy_primitives::{BlockNumber, B256, U256}; use eyre::eyre; use reth_chainspec::{ChainSpec, EthChainSpec, MAINNET}; use reth_config::config::PruneConfig; -use reth_ethereum_forks::Head; +use reth_engine_local::MiningMode; +use reth_ethereum_forks::{EthereumHardforks, Head}; use reth_network_p2p::headers::client::HeadersClient; use reth_primitives_traits::SealedHeader; use reth_stages_types::StageId; @@ -22,23 +23,20 @@ use reth_storage_api::{ BlockHashReader, DatabaseProviderFactory, HeaderProvider, StageCheckpointReader, }; use reth_storage_errors::provider::ProviderResult; +use reth_transaction_pool::TransactionPool; use serde::{de::DeserializeOwned, Serialize}; use std::{ fs, - net::SocketAddr, path::{Path, PathBuf}, sync::Arc, }; use tracing::*; +use crate::args::{EraArgs, MetricArgs}; pub use reth_engine_primitives::{ - DEFAULT_MAX_PROOF_TASK_CONCURRENCY, DEFAULT_MEMORY_BLOCK_BUFFER_TARGET, - DEFAULT_RESERVED_CPU_CORES, + DEFAULT_MEMORY_BLOCK_BUFFER_TARGET, DEFAULT_PERSISTENCE_THRESHOLD, DEFAULT_RESERVED_CPU_CORES, }; -/// Triggers persistence when the number of canonical blocks in memory exceeds this threshold. -pub const DEFAULT_PERSISTENCE_THRESHOLD: u64 = 2; - /// Default size of cross-block cache in megabytes. pub const DEFAULT_CROSS_BLOCK_CACHE_SIZE_MB: u64 = 4 * 1024; @@ -100,10 +98,8 @@ pub struct NodeConfig { /// Possible values are either a built-in chain or the path to a chain specification file. pub chain: Arc, - /// Enable Prometheus metrics. - /// - /// The metrics will be served at the given interface and port. - pub metrics: Option, + /// Enable to configure metrics export to endpoints + pub metrics: MetricArgs, /// Add a new instance of a node. /// @@ -148,6 +144,9 @@ pub struct NodeConfig { /// All engine related arguments pub engine: EngineArgs, + + /// All ERA import related arguments with --era prefix + pub era: EraArgs, } impl NodeConfig { @@ -165,7 +164,7 @@ impl NodeConfig { Self { config: None, chain, - metrics: None, + metrics: MetricArgs::default(), instance: None, network: NetworkArgs::default(), rpc: RpcServerArgs::default(), @@ -177,6 +176,7 @@ impl NodeConfig { pruning: PruningArgs::default(), datadir: DatadirArgs::default(), engine: EngineArgs::default(), + era: EraArgs::default(), } } @@ -190,6 +190,22 @@ impl NodeConfig { self } + /// Apply a function to the config. + pub fn apply(self, f: F) -> Self + where + F: FnOnce(Self) -> Self, + { + f(self) + } + + /// Applies a fallible function to the config. + pub fn try_apply(self, f: F) -> Result + where + F: FnOnce(Self) -> Result, + { + f(self) + } + /// Sets --dev mode for the node [`NodeConfig::dev`], if `dev` is true. pub const fn set_dev(self, dev: bool) -> Self { if dev { @@ -218,8 +234,8 @@ impl NodeConfig { } /// Set the metrics address for the node - pub const fn with_metrics(mut self, metrics: SocketAddr) -> Self { - self.metrics = Some(metrics); + pub fn with_metrics(mut self, metrics: MetricArgs) -> Self { + self.metrics = metrics; self } @@ -271,7 +287,7 @@ impl NodeConfig { } /// Set the dev args for the node - pub const fn with_dev(mut self, dev: DevArgs) -> Self { + pub fn with_dev(mut self, dev: DevArgs) -> Self { self.dev = dev; self } @@ -285,7 +301,7 @@ impl NodeConfig { /// Returns pruning configuration. pub fn prune_config(&self) -> Option where - ChainSpec: EthChainSpec, + ChainSpec: EthereumHardforks, { self.pruning.prune_config(&self.chain) } @@ -329,12 +345,6 @@ impl NodeConfig { .header_by_number(head)? .expect("the header for the latest block is missing, database is corrupt"); - let total_difficulty = provider - .header_td_by_number(head)? - // total difficulty is effectively deprecated, but still required in some places, e.g. - // p2p - .unwrap_or_default(); - let hash = provider .block_hash(head)? .expect("the hash for the latest block is missing, database is corrupt"); @@ -343,7 +353,7 @@ impl NodeConfig { number: head, hash, difficulty: header.difficulty(), - total_difficulty, + total_difficulty: U256::ZERO, timestamp: header.timestamp(), }) } @@ -417,6 +427,12 @@ impl NodeConfig { self } + /// Disables all discovery services for the node. + pub const fn with_disabled_discovery(mut self) -> Self { + self.network.discovery.disable_discovery = true; + self + } + /// Effectively disables the RPC state cache by setting the cache sizes to `0`. /// /// By setting the cache sizes to 0, caching of newly executed or fetched blocks will be @@ -482,6 +498,19 @@ impl NodeConfig { dev: self.dev, pruning: self.pruning, engine: self.engine, + era: self.era, + } + } + + /// Returns the [`MiningMode`] intended for --dev mode. + pub fn dev_mining_mode(&self, pool: Pool) -> MiningMode + where + Pool: TransactionPool + Unpin, + { + if let Some(interval) = self.dev.block_time { + MiningMode::interval(interval) + } else { + MiningMode::instant(pool, self.dev.block_max_transactions) } } } @@ -497,7 +526,7 @@ impl Clone for NodeConfig { Self { chain: self.chain.clone(), config: self.config.clone(), - metrics: self.metrics, + metrics: self.metrics.clone(), instance: self.instance, network: self.network.clone(), rpc: self.rpc.clone(), @@ -505,10 +534,11 @@ impl Clone for NodeConfig { builder: self.builder.clone(), debug: self.debug.clone(), db: self.db, - dev: self.dev, + dev: self.dev.clone(), pruning: self.pruning.clone(), datadir: self.datadir.clone(), engine: self.engine.clone(), + era: self.era.clone(), } } } diff --git a/crates/node/core/src/version.rs b/crates/node/core/src/version.rs index a526301a224..9953aea2390 100644 --- a/crates/node/core/src/version.rs +++ b/crates/node/core/src/version.rs @@ -1,4 +1,6 @@ //! Version information for reth. +use std::{borrow::Cow, sync::OnceLock}; + use alloy_primitives::Bytes; use alloy_rpc_types_engine::ClientCode; use reth_db::ClientVersion; @@ -6,58 +8,66 @@ use reth_db::ClientVersion; /// The client code for Reth pub const CLIENT_CODE: ClientCode = ClientCode::RH; -/// The human readable name of the client -pub const NAME_CLIENT: &str = "Reth"; - -/// The latest version from Cargo.toml. -pub const CARGO_PKG_VERSION: &str = env!("CARGO_PKG_VERSION"); - -/// The full SHA of the latest commit. -pub const VERGEN_GIT_SHA_LONG: &str = env!("VERGEN_GIT_SHA"); - -/// The 8 character short SHA of the latest commit. -pub const VERGEN_GIT_SHA: &str = env!("VERGEN_GIT_SHA_SHORT"); - -/// The build timestamp. -pub const VERGEN_BUILD_TIMESTAMP: &str = env!("VERGEN_BUILD_TIMESTAMP"); - -/// The target triple. -pub const VERGEN_CARGO_TARGET_TRIPLE: &str = env!("VERGEN_CARGO_TARGET_TRIPLE"); +/// Global static version metadata +static VERSION_METADATA: OnceLock = OnceLock::new(); -/// The build features. -pub const VERGEN_CARGO_FEATURES: &str = env!("VERGEN_CARGO_FEATURES"); - -/// The short version information for reth. -pub const SHORT_VERSION: &str = env!("RETH_SHORT_VERSION"); - -/// The long version information for reth. -pub const LONG_VERSION: &str = concat!( - env!("RETH_LONG_VERSION_0"), - "\n", - env!("RETH_LONG_VERSION_1"), - "\n", - env!("RETH_LONG_VERSION_2"), - "\n", - env!("RETH_LONG_VERSION_3"), - "\n", - env!("RETH_LONG_VERSION_4") -); - -/// The build profile name. -pub const BUILD_PROFILE_NAME: &str = env!("RETH_BUILD_PROFILE"); +/// Initialize the global version metadata. +pub fn try_init_version_metadata( + metadata: RethCliVersionConsts, +) -> Result<(), RethCliVersionConsts> { + VERSION_METADATA.set(metadata) +} -/// The version information for reth formatted for P2P (devp2p). -/// -/// - The latest version from Cargo.toml -/// - The target triple -/// -/// # Example +/// Constants for reth-cli /// -/// ```text -/// reth/v{major}.{minor}.{patch}-{sha1}/{target} -/// ``` -/// e.g.: `reth/v0.1.0-alpha.1-428a6dc2f/aarch64-apple-darwin` -pub(crate) const P2P_CLIENT_VERSION: &str = env!("RETH_P2P_CLIENT_VERSION"); +/// Global defaults can be set via [`try_init_version_metadata`]. +#[derive(Debug, Default)] +pub struct RethCliVersionConsts { + /// The human readable name of the client + pub name_client: Cow<'static, str>, + + /// The latest version from Cargo.toml. + pub cargo_pkg_version: Cow<'static, str>, + + /// The full SHA of the latest commit. + pub vergen_git_sha_long: Cow<'static, str>, + + /// The 8 character short SHA of the latest commit. + pub vergen_git_sha: Cow<'static, str>, + + /// The build timestamp. + pub vergen_build_timestamp: Cow<'static, str>, + + /// The target triple. + pub vergen_cargo_target_triple: Cow<'static, str>, + + /// The build features. + pub vergen_cargo_features: Cow<'static, str>, + + /// The short version information for reth. + pub short_version: Cow<'static, str>, + + /// The long version information for reth. + pub long_version: Cow<'static, str>, + /// The build profile name. + pub build_profile_name: Cow<'static, str>, + + /// The version information for reth formatted for P2P (devp2p). + /// + /// - The latest version from Cargo.toml + /// - The target triple + /// + /// # Example + /// + /// ```text + /// reth/v{major}.{minor}.{patch}-{sha1}/{target} + /// ``` + /// e.g.: `reth/v0.1.0-alpha.1-428a6dc2f/aarch64-apple-darwin` + pub p2p_client_version: Cow<'static, str>, + + /// extra data used for payload building + pub extra_data: Cow<'static, str>, +} /// The default extra data used for payload building. /// @@ -81,10 +91,42 @@ pub fn default_extra_data_bytes() -> Bytes { /// The default client version accessing the database. pub fn default_client_version() -> ClientVersion { + let meta = version_metadata(); ClientVersion { - version: CARGO_PKG_VERSION.to_string(), - git_sha: VERGEN_GIT_SHA.to_string(), - build_timestamp: VERGEN_BUILD_TIMESTAMP.to_string(), + version: meta.cargo_pkg_version.to_string(), + git_sha: meta.vergen_git_sha.to_string(), + build_timestamp: meta.vergen_build_timestamp.to_string(), + } +} + +/// Get a reference to the global version metadata +pub fn version_metadata() -> &'static RethCliVersionConsts { + VERSION_METADATA.get_or_init(default_reth_version_metadata) +} + +/// default reth version metadata using compile-time env! macros. +pub fn default_reth_version_metadata() -> RethCliVersionConsts { + RethCliVersionConsts { + name_client: Cow::Borrowed("Reth"), + cargo_pkg_version: Cow::Borrowed(env!("CARGO_PKG_VERSION")), + vergen_git_sha_long: Cow::Borrowed(env!("VERGEN_GIT_SHA")), + vergen_git_sha: Cow::Borrowed(env!("VERGEN_GIT_SHA_SHORT")), + vergen_build_timestamp: Cow::Borrowed(env!("VERGEN_BUILD_TIMESTAMP")), + vergen_cargo_target_triple: Cow::Borrowed(env!("VERGEN_CARGO_TARGET_TRIPLE")), + vergen_cargo_features: Cow::Borrowed(env!("VERGEN_CARGO_FEATURES")), + short_version: Cow::Borrowed(env!("RETH_SHORT_VERSION")), + long_version: Cow::Owned(format!( + "{}\n{}\n{}\n{}\n{}", + env!("RETH_LONG_VERSION_0"), + env!("RETH_LONG_VERSION_1"), + env!("RETH_LONG_VERSION_2"), + env!("RETH_LONG_VERSION_3"), + env!("RETH_LONG_VERSION_4"), + )), + + build_profile_name: Cow::Borrowed(env!("RETH_BUILD_PROFILE")), + p2p_client_version: Cow::Borrowed(env!("RETH_P2P_CLIENT_VERSION")), + extra_data: Cow::Owned(default_extra_data()), } } diff --git a/crates/node/ethstats/Cargo.toml b/crates/node/ethstats/Cargo.toml new file mode 100644 index 00000000000..6ffad317702 --- /dev/null +++ b/crates/node/ethstats/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "reth-node-ethstats" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[dependencies] +reth-network-api.workspace = true +reth-transaction-pool.workspace = true +reth-primitives-traits.workspace = true +reth-storage-api.workspace = true +reth-chain-state.workspace = true + +alloy-primitives.workspace = true +alloy-consensus.workspace = true + +tokio.workspace = true +tokio-tungstenite = { workspace = true, features = ["rustls-tls-native-roots"] } +futures-util.workspace = true +tokio-stream.workspace = true + +serde.workspace = true +serde_json.workspace = true + +tracing.workspace = true +url.workspace = true +chrono.workspace = true +thiserror = { workspace = true, features = ["std"] } diff --git a/crates/node/ethstats/src/connection.rs b/crates/node/ethstats/src/connection.rs new file mode 100644 index 00000000000..049788dccc3 --- /dev/null +++ b/crates/node/ethstats/src/connection.rs @@ -0,0 +1,67 @@ +/// Abstractions for managing `WebSocket` connections in the ethstats service. +use crate::error::ConnectionError; +use futures_util::{ + stream::{SplitSink, SplitStream}, + SinkExt, StreamExt, +}; +use serde_json::Value; +use std::sync::Arc; +use tokio::{net::TcpStream, sync::Mutex}; +use tokio_tungstenite::{ + tungstenite::protocol::{frame::Utf8Bytes, Message}, + MaybeTlsStream, WebSocketStream, +}; + +/// Type alias for a `WebSocket` stream that may be TLS or plain TCP +pub(crate) type WsStream = WebSocketStream>; + +/// Wrapper for a thread-safe, asynchronously accessible `WebSocket` connection +#[derive(Debug, Clone)] +pub(crate) struct ConnWrapper { + /// Write-only part of the `WebSocket` stream + writer: Arc>>, + /// Read-only part of the `WebSocket` stream + reader: Arc>>, +} + +impl ConnWrapper { + /// Create a new connection wrapper from a `WebSocket` stream + pub(crate) fn new(stream: WsStream) -> Self { + let (writer, reader) = stream.split(); + + Self { writer: Arc::new(Mutex::new(writer)), reader: Arc::new(Mutex::new(reader)) } + } + + /// Write a JSON string as a text message to the `WebSocket` + pub(crate) async fn write_json(&self, value: &str) -> Result<(), ConnectionError> { + let mut writer = self.writer.lock().await; + writer.send(Message::Text(Utf8Bytes::from(value))).await?; + + Ok(()) + } + + /// Read the next JSON text message from the `WebSocket` + /// + /// Waits for the next text message, parses it as JSON, and returns the value. + /// Ignores non-text messages. Returns an error if the connection is closed or if parsing fails. + pub(crate) async fn read_json(&self) -> Result { + let mut reader = self.reader.lock().await; + while let Some(msg) = reader.next().await { + match msg? { + Message::Text(text) => return Ok(serde_json::from_str(&text)?), + Message::Close(_) => return Err(ConnectionError::ConnectionClosed), + _ => {} // Ignore non-text messages + } + } + + Err(ConnectionError::ConnectionClosed) + } + + /// Close the `WebSocket` connection gracefully + pub(crate) async fn close(&self) -> Result<(), ConnectionError> { + let mut writer = self.writer.lock().await; + writer.close().await?; + + Ok(()) + } +} diff --git a/crates/node/ethstats/src/credentials.rs b/crates/node/ethstats/src/credentials.rs new file mode 100644 index 00000000000..cf2adb785e8 --- /dev/null +++ b/crates/node/ethstats/src/credentials.rs @@ -0,0 +1,47 @@ +use crate::error::EthStatsError; +use std::str::FromStr; + +/// Credentials for connecting to an `EthStats` server +/// +/// Contains the node identifier, authentication secret, and server host +/// information needed to establish a connection with the `EthStats` service. +#[derive(Debug, Clone)] +pub(crate) struct EthstatsCredentials { + /// Unique identifier for this node in the `EthStats` network + pub node_id: String, + /// Authentication secret for the `EthStats` server + pub secret: String, + /// Host address of the `EthStats` server + pub host: String, +} + +impl FromStr for EthstatsCredentials { + type Err = EthStatsError; + + /// Parse credentials from a string in the format "`node_id:secret@host`" + /// + /// # Arguments + /// * `s` - String containing credentials in the format "`node_id:secret@host`" + /// + /// # Returns + /// * `Ok(EthstatsCredentials)` - Successfully parsed credentials + /// * `Err(EthStatsError::InvalidUrl)` - Invalid format or missing separators + fn from_str(s: &str) -> Result { + let parts: Vec<&str> = s.split('@').collect(); + if parts.len() != 2 { + return Err(EthStatsError::InvalidUrl("Missing '@' separator".to_string())); + } + let creds = parts[0]; + let host = parts[1].to_string(); + let creds_parts: Vec<&str> = creds.split(':').collect(); + if creds_parts.len() != 2 { + return Err(EthStatsError::InvalidUrl( + "Missing ':' separator in credentials".to_string(), + )); + } + let node_id = creds_parts[0].to_string(); + let secret = creds_parts[1].to_string(); + + Ok(Self { node_id, secret, host }) + } +} diff --git a/crates/node/ethstats/src/error.rs b/crates/node/ethstats/src/error.rs new file mode 100644 index 00000000000..fff9bf5306a --- /dev/null +++ b/crates/node/ethstats/src/error.rs @@ -0,0 +1,69 @@ +use thiserror::Error; + +/// Errors that can occur during `WebSocket` connection handling +#[derive(Debug, Error)] +pub enum ConnectionError { + /// The `WebSocket` connection was closed unexpectedly + #[error("Connection closed")] + ConnectionClosed, + + /// Error occurred during JSON serialization/deserialization + #[error("Serialization error: {0}")] + Serialization(#[from] serde_json::Error), + + /// Error occurred during `WebSocket` communication + #[error("WebSocket error: {0}")] + WebSocket(#[from] tokio_tungstenite::tungstenite::Error), +} + +/// Main error type for the `EthStats` client +/// +/// This enum covers all possible errors that can occur when interacting +/// with an `EthStats` server, including connection issues, authentication +/// problems, data fetching errors, and various I/O operations. +#[derive(Debug, Error)] +pub enum EthStatsError { + /// The provided URL is invalid or malformed + #[error("Invalid URL: {0}")] + InvalidUrl(String), + + /// Error occurred during connection establishment or maintenance + #[error("Connection error: {0}")] + ConnectionError(#[from] ConnectionError), + + /// Authentication failed with the `EthStats` server + #[error("Authentication error: {0}")] + AuthError(String), + + /// Attempted to perform an operation while not connected to the server + #[error("Not connected to server")] + NotConnected, + + /// Error occurred during JSON serialization or deserialization + #[error("Serialization error: {0}")] + Serialization(#[from] serde_json::Error), + + /// Error occurred during `WebSocket` communication + #[error("WebSocket error: {0}")] + WebSocket(#[from] tokio_tungstenite::tungstenite::Error), + + /// Operation timed out + #[error("Timeout error")] + Timeout, + + /// Error occurred while parsing a URL + #[error("URL parsing error: {0}")] + Url(#[from] url::ParseError), + + /// Requested block was not found in the blockchain + #[error("Block not found: {0}")] + BlockNotFound(u64), + + /// Error occurred while fetching data from the blockchain or server + #[error("Data fetch error: {0}")] + DataFetchError(String), + + /// The request sent to the server was invalid or malformed + #[error("Inivalid request")] + InvalidRequest, +} diff --git a/crates/node/ethstats/src/ethstats.rs b/crates/node/ethstats/src/ethstats.rs new file mode 100644 index 00000000000..7592e93ae9d --- /dev/null +++ b/crates/node/ethstats/src/ethstats.rs @@ -0,0 +1,819 @@ +use crate::{ + connection::ConnWrapper, + credentials::EthstatsCredentials, + error::EthStatsError, + events::{ + AuthMsg, BlockMsg, BlockStats, HistoryMsg, LatencyMsg, NodeInfo, NodeStats, PendingMsg, + PendingStats, PingMsg, StatsMsg, TxStats, UncleStats, + }, +}; +use alloy_consensus::{BlockHeader, Sealable}; +use alloy_primitives::U256; +use reth_chain_state::{CanonStateNotification, CanonStateSubscriptions}; +use reth_network_api::{NetworkInfo, Peers}; +use reth_primitives_traits::{Block, BlockBody}; +use reth_storage_api::{BlockReader, BlockReaderIdExt, NodePrimitivesProvider}; +use reth_transaction_pool::TransactionPool; + +use chrono::Local; +use serde_json::Value; +use std::{ + str::FromStr, + sync::Arc, + time::{Duration, Instant}, +}; +use tokio::{ + sync::{mpsc, Mutex, RwLock}, + time::{interval, sleep, timeout}, +}; +use tokio_stream::StreamExt; +use tokio_tungstenite::connect_async; +use tracing::{debug, info}; +use url::Url; + +/// Number of historical blocks to include in a history update sent to the `EthStats` server +const HISTORY_UPDATE_RANGE: u64 = 50; +/// Duration to wait before attempting to reconnect to the `EthStats` server +const RECONNECT_INTERVAL: Duration = Duration::from_secs(5); +/// Maximum time to wait for a ping response from the server +const PING_TIMEOUT: Duration = Duration::from_secs(5); +/// Interval between regular stats reports to the server +const REPORT_INTERVAL: Duration = Duration::from_secs(15); +/// Maximum time to wait for initial connection establishment +const CONNECT_TIMEOUT: Duration = Duration::from_secs(10); +/// Maximum time to wait for reading messages from the server +const READ_TIMEOUT: Duration = Duration::from_secs(30); + +/// Main service for interacting with an `EthStats` server +/// +/// This service handles all communication with the `EthStats` server including +/// authentication, stats reporting, block notifications, and connection management. +/// It maintains a persistent `WebSocket` connection and automatically reconnects +/// when the connection is lost. +#[derive(Debug)] +pub struct EthStatsService { + /// Authentication credentials for the `EthStats` server + credentials: EthstatsCredentials, + /// `WebSocket` connection wrapper, wrapped in `Arc` for shared access + conn: Arc>>, + /// Timestamp of the last ping sent to the server + last_ping: Arc>>, + /// Network interface for getting peer and sync information + network: Network, + /// Blockchain provider for reading block data and state + provider: Provider, + /// Transaction pool for getting pending transaction statistics + pool: Pool, +} + +impl EthStatsService +where + Network: NetworkInfo + Peers, + Provider: BlockReaderIdExt + CanonStateSubscriptions, + Pool: TransactionPool, +{ + /// Create a new `EthStats` service and establish initial connection + /// + /// # Arguments + /// * `url` - Connection string in format "`node_id:secret@host`" + /// * `network` - Network interface implementation + /// * `provider` - Blockchain provider implementation + /// * `pool` - Transaction pool implementation + pub async fn new( + url: &str, + network: Network, + provider: Provider, + pool: Pool, + ) -> Result { + let credentials = EthstatsCredentials::from_str(url)?; + let service = Self { + credentials, + conn: Arc::new(RwLock::new(None)), + last_ping: Arc::new(Mutex::new(None)), + network, + provider, + pool, + }; + service.connect().await?; + + Ok(service) + } + + /// Establish `WebSocket` connection to the `EthStats` server + /// + /// Attempts to connect to the server using the credentials and handles + /// connection timeouts and errors. + async fn connect(&self) -> Result<(), EthStatsError> { + debug!( + target: "ethstats", + "Attempting to connect to EthStats server at {}", self.credentials.host + ); + let full_url = format!("ws://{}/api", self.credentials.host); + let url = Url::parse(&full_url).map_err(EthStatsError::Url)?; + + match timeout(CONNECT_TIMEOUT, connect_async(url.as_str())).await { + Ok(Ok((ws_stream, _))) => { + debug!( + target: "ethstats", + "Successfully connected to EthStats server at {}", self.credentials.host + ); + let conn: ConnWrapper = ConnWrapper::new(ws_stream); + *self.conn.write().await = Some(conn.clone()); + self.login().await?; + Ok(()) + } + Ok(Err(e)) => Err(EthStatsError::WebSocket(e)), + Err(_) => { + debug!(target: "ethstats", "Connection to EthStats server timed out"); + Err(EthStatsError::Timeout) + } + } + } + + /// Authenticate with the `EthStats` server + /// + /// Sends authentication credentials and node information to the server + /// and waits for a successful acknowledgment. + async fn login(&self) -> Result<(), EthStatsError> { + debug!( + target: "ethstats", + "Attempting to login to EthStats server as node_id {}", self.credentials.node_id + ); + let conn = self.conn.read().await; + let conn = conn.as_ref().ok_or(EthStatsError::NotConnected)?; + + let network_status = self + .network + .network_status() + .await + .map_err(|e| EthStatsError::AuthError(e.to_string()))?; + let id = &self.credentials.node_id; + let secret = &self.credentials.secret; + let protocol = network_status + .capabilities + .iter() + .map(|cap| format!("{}/{}", cap.name, cap.version)) + .collect::>() + .join(", "); + let port = self.network.local_addr().port() as u64; + + let auth = AuthMsg { + id: id.clone(), + secret: secret.clone(), + info: NodeInfo { + name: id.clone(), + node: network_status.client_version.clone(), + port, + network: self.network.chain_id().to_string(), + protocol, + api: "No".to_string(), + os: std::env::consts::OS.into(), + os_ver: std::env::consts::ARCH.into(), + client: "0.1.1".to_string(), + history: true, + }, + }; + + let message = auth.generate_login_message(); + conn.write_json(&message).await?; + + let response = + timeout(READ_TIMEOUT, conn.read_json()).await.map_err(|_| EthStatsError::Timeout)??; + + if let Some(ack) = response.get("emit") && + ack.get(0) == Some(&Value::String("ready".to_string())) + { + info!( + target: "ethstats", + "Login successful to EthStats server as node_id {}", self.credentials.node_id + ); + return Ok(()); + } + + debug!(target: "ethstats", "Login failed: Unauthorized or unexpected login response"); + Err(EthStatsError::AuthError("Unauthorized or unexpected login response".into())) + } + + /// Report current node statistics to the `EthStats` server + /// + /// Sends information about the node's current state including sync status, + /// peer count, and uptime. + async fn report_stats(&self) -> Result<(), EthStatsError> { + let conn = self.conn.read().await; + let conn = conn.as_ref().ok_or(EthStatsError::NotConnected)?; + + let stats_msg = StatsMsg { + id: self.credentials.node_id.clone(), + stats: NodeStats { + active: true, + syncing: self.network.is_syncing(), + peers: self.network.num_connected_peers() as u64, + gas_price: 0, // TODO + uptime: 100, + }, + }; + + let message = stats_msg.generate_stats_message(); + conn.write_json(&message).await?; + + Ok(()) + } + + /// Send a ping message to the `EthStats` server + /// + /// Records the ping time and starts a timeout task to detect if the server + /// doesn't respond within the expected timeframe. + async fn send_ping(&self) -> Result<(), EthStatsError> { + let conn = self.conn.read().await; + let conn = conn.as_ref().ok_or(EthStatsError::NotConnected)?; + + let ping_time = Instant::now(); + *self.last_ping.lock().await = Some(ping_time); + + let client_time = Local::now().format("%Y-%m-%d %H:%M:%S%.f %:z %Z").to_string(); + let ping_msg = PingMsg { id: self.credentials.node_id.clone(), client_time }; + + let message = ping_msg.generate_ping_message(); + conn.write_json(&message).await?; + + // Start ping timeout + let active_ping = self.last_ping.clone(); + let conn_ref = self.conn.clone(); + tokio::spawn(async move { + sleep(PING_TIMEOUT).await; + let mut active = active_ping.lock().await; + if active.is_some() { + debug!(target: "ethstats", "Ping timeout"); + *active = None; + // Clear connection to trigger reconnect + if let Some(conn) = conn_ref.write().await.take() { + let _ = conn.close().await; + } + } + }); + + Ok(()) + } + + /// Report latency measurement to the `EthStats` server + /// + /// Calculates the round-trip time from the last ping and sends it to + /// the server. This is called when a pong response is received. + async fn report_latency(&self) -> Result<(), EthStatsError> { + let conn = self.conn.read().await; + let conn = conn.as_ref().ok_or(EthStatsError::NotConnected)?; + + let mut active = self.last_ping.lock().await; + if let Some(start) = active.take() { + let latency = start.elapsed().as_millis() as u64 / 2; + + debug!(target: "ethstats", "Reporting latency: {}ms", latency); + + let latency_msg = LatencyMsg { id: self.credentials.node_id.clone(), latency }; + + let message = latency_msg.generate_latency_message(); + conn.write_json(&message).await? + } + + Ok(()) + } + + /// Report pending transaction count to the `EthStats` server + /// + /// Gets the current number of pending transactions from the pool and + /// sends this information to the server. + async fn report_pending(&self) -> Result<(), EthStatsError> { + let conn = self.conn.read().await; + let conn = conn.as_ref().ok_or(EthStatsError::NotConnected)?; + let pending = self.pool.pool_size().pending as u64; + + debug!(target: "ethstats", "Reporting pending txs: {}", pending); + + let pending_msg = + PendingMsg { id: self.credentials.node_id.clone(), stats: PendingStats { pending } }; + + let message = pending_msg.generate_pending_message(); + conn.write_json(&message).await?; + + Ok(()) + } + + /// Report block information to the `EthStats` server + /// + /// Fetches block data either from a canonical state notification or + /// the current best block, converts it to stats format, and sends + /// it to the server. + /// + /// # Arguments + /// * `head` - Optional canonical state notification containing new block info + async fn report_block( + &self, + head: Option::Primitives>>, + ) -> Result<(), EthStatsError> { + let conn = self.conn.read().await; + let conn = conn.as_ref().ok_or(EthStatsError::NotConnected)?; + + let block_number = if let Some(head) = head { + head.tip().header().number() + } else { + self.provider + .best_block_number() + .map_err(|e| EthStatsError::DataFetchError(e.to_string()))? + }; + + match self.provider.block_by_id(block_number.into()) { + Ok(Some(block)) => { + let block_msg = BlockMsg { + id: self.credentials.node_id.clone(), + block: self.block_to_stats(&block)?, + }; + + debug!(target: "ethstats", "Reporting block: {}", block_number); + + let message = block_msg.generate_block_message(); + conn.write_json(&message).await?; + } + Ok(None) => { + // Block not found, stop fetching + debug!(target: "ethstats", "Block {} not found", block_number); + return Err(EthStatsError::BlockNotFound(block_number)); + } + Err(e) => { + debug!(target: "ethstats", "Error fetching block {}: {}", block_number, e); + return Err(EthStatsError::DataFetchError(e.to_string())); + } + }; + + Ok(()) + } + + /// Convert a block to `EthStats` block statistics format + /// + /// Extracts relevant information from a block and formats it according + /// to the `EthStats` protocol specification. + /// + /// # Arguments + /// * `block` - The block to convert + fn block_to_stats( + &self, + block: &::Block, + ) -> Result { + let body = block.body(); + let header = block.header(); + + let txs = body.transaction_hashes_iter().copied().map(|hash| TxStats { hash }).collect(); + + Ok(BlockStats { + number: U256::from(header.number()), + hash: header.hash_slow(), + parent_hash: header.parent_hash(), + timestamp: U256::from(header.timestamp()), + miner: header.beneficiary(), + gas_used: header.gas_used(), + gas_limit: header.gas_limit(), + diff: header.difficulty().to_string(), + total_diff: "0".into(), + txs, + tx_root: header.transactions_root(), + root: header.state_root(), + uncles: UncleStats(vec![]), + }) + } + + /// Report historical block data to the `EthStats` server + /// + /// Fetches multiple blocks by their numbers and sends their statistics + /// to the server. This is typically called in response to a history + /// request from the server. + /// + /// # Arguments + /// * `list` - Vector of block numbers to fetch and report + async fn report_history(&self, list: Option<&Vec>) -> Result<(), EthStatsError> { + let conn = self.conn.read().await; + let conn = conn.as_ref().ok_or(EthStatsError::NotConnected)?; + + let indexes = if let Some(list) = list { + list + } else { + let best_block_number = self + .provider + .best_block_number() + .map_err(|e| EthStatsError::DataFetchError(e.to_string()))?; + + let start = best_block_number.saturating_sub(HISTORY_UPDATE_RANGE); + + &(start..=best_block_number).collect() + }; + + let mut blocks = Vec::with_capacity(indexes.len()); + for &block_number in indexes { + match self.provider.block_by_id(block_number.into()) { + Ok(Some(block)) => { + blocks.push(block); + } + Ok(None) => { + // Block not found, stop fetching + debug!(target: "ethstats", "Block {} not found", block_number); + break; + } + Err(e) => { + debug!(target: "ethstats", "Error fetching block {}: {}", block_number, e); + break; + } + } + } + + let history: Vec = + blocks.iter().map(|block| self.block_to_stats(block)).collect::>()?; + + if history.is_empty() { + debug!(target: "ethstats", "No history to send to stats server"); + } else { + debug!( + target: "ethstats", + "Sending historical blocks to ethstats, first: {}, last: {}", + history.first().unwrap().number, + history.last().unwrap().number + ); + } + + let history_msg = HistoryMsg { id: self.credentials.node_id.clone(), history }; + + let message = history_msg.generate_history_message(); + conn.write_json(&message).await?; + + Ok(()) + } + + /// Send a complete status report to the `EthStats` server + /// + /// Performs all regular reporting tasks: ping, block info, pending + /// transactions, and general statistics. + async fn report(&self) -> Result<(), EthStatsError> { + self.send_ping().await?; + self.report_block(None).await?; + self.report_pending().await?; + self.report_stats().await?; + + Ok(()) + } + + /// Handle incoming messages from the `EthStats` server + /// + /// # Expected Message Variants + /// + /// This function expects messages in the following format: + /// + /// ```json + /// { "emit": [, ] } + /// ``` + /// + /// ## Supported Commands: + /// + /// - `"node-pong"`: Indicates a pong response to a previously sent ping. The payload is + /// ignored. Triggers a latency report to the server. + /// - Example: ```json { "emit": [ "node-pong", { "clientTime": "2025-07-10 12:00:00.123 + /// +00:00 UTC", "serverTime": "2025-07-10 12:00:01.456 +00:00 UTC" } ] } ``` + /// + /// - `"history"`: Requests historical block data. The payload may contain a `list` field with + /// block numbers to fetch. If `list` is not present, the default range is used. + /// - Example with list: `{ "emit": ["history", {"list": [1, 2, 3], "min": 1, "max": 3}] }` + /// - Example without list: `{ "emit": ["history", {}] }` + /// + /// ## Other Commands: + /// + /// Any other command is logged as unhandled and ignored. + async fn handle_message(&self, msg: Value) -> Result<(), EthStatsError> { + let emit = match msg.get("emit") { + Some(emit) => emit, + None => { + debug!(target: "ethstats", "Stats server sent non-broadcast, msg {}", msg); + return Err(EthStatsError::InvalidRequest); + } + }; + + let command = match emit.get(0) { + Some(Value::String(command)) => command.as_str(), + _ => { + debug!(target: "ethstats", "Invalid stats server message type, msg {}", msg); + return Err(EthStatsError::InvalidRequest); + } + }; + + match command { + "node-pong" => { + self.report_latency().await?; + } + "history" => { + let block_numbers = emit + .get(1) + .and_then(|v| v.as_object()) + .and_then(|obj| obj.get("list")) + .and_then(|v| v.as_array()); + + if block_numbers.is_none() { + self.report_history(None).await?; + + return Ok(()); + } + + let block_numbers = block_numbers + .unwrap() + .iter() + .map(|val| { + val.as_u64().ok_or_else(|| { + debug!( + target: "ethstats", + "Invalid stats history block number, msg {}", msg + ); + EthStatsError::InvalidRequest + }) + }) + .collect::>()?; + + self.report_history(Some(&block_numbers)).await?; + } + other => debug!(target: "ethstats", "Unhandled command: {}", other), + } + + Ok(()) + } + + /// Main service loop that handles all `EthStats` communication + /// + /// This method runs the main event loop that: + /// - Maintains the `WebSocket` connection + /// - Handles incoming messages from the server + /// - Reports statistics at regular intervals + /// - Processes new block notifications + /// - Automatically reconnects when the connection is lost + /// + /// The service runs until explicitly shut down or an unrecoverable + /// error occurs. + pub async fn run(self) { + // Create channels for internal communication + let (shutdown_tx, mut shutdown_rx) = mpsc::channel(1); + let (message_tx, mut message_rx) = mpsc::channel(32); + let (head_tx, mut head_rx) = mpsc::channel(10); + + // Start the read loop in a separate task + let read_handle = { + let conn = self.conn.clone(); + let message_tx = message_tx.clone(); + let shutdown_tx = shutdown_tx.clone(); + + tokio::spawn(async move { + loop { + let conn = conn.read().await; + if let Some(conn) = conn.as_ref() { + match conn.read_json().await { + Ok(msg) => { + if message_tx.send(msg).await.is_err() { + break; + } + } + Err(e) => { + debug!(target: "ethstats", "Read error: {}", e); + break; + } + } + } else { + sleep(RECONNECT_INTERVAL).await; + } + } + + let _ = shutdown_tx.send(()).await; + }) + }; + + let canonical_stream_handle = { + let mut canonical_stream = self.provider.canonical_state_stream(); + let head_tx = head_tx.clone(); + let shutdown_tx = shutdown_tx.clone(); + + tokio::spawn(async move { + loop { + let head = canonical_stream.next().await; + if let Some(head) = head && + head_tx.send(head).await.is_err() + { + break; + } + } + + let _ = shutdown_tx.send(()).await; + }) + }; + + let mut pending_tx_receiver = self.pool.pending_transactions_listener(); + + // Set up intervals + let mut report_interval = interval(REPORT_INTERVAL); + let mut reconnect_interval = interval(RECONNECT_INTERVAL); + + // Main event loop using select! + loop { + tokio::select! { + // Handle shutdown signal + _ = shutdown_rx.recv() => { + info!(target: "ethstats", "Shutting down ethstats service"); + break; + } + + // Handle messages from the read loop + Some(msg) = message_rx.recv() => { + if let Err(e) = self.handle_message(msg).await { + debug!(target: "ethstats", "Error handling message: {}", e); + self.disconnect().await; + } + } + + // Handle new block + Some(head) = head_rx.recv() => { + if let Err(e) = self.report_block(Some(head)).await { + debug!(target: "ethstats", "Failed to report block: {}", e); + self.disconnect().await; + } + + if let Err(e) = self.report_pending().await { + debug!(target: "ethstats", "Failed to report pending: {}", e); + self.disconnect().await; + } + } + + // Handle new pending tx + _= pending_tx_receiver.recv() => { + if let Err(e) = self.report_pending().await { + debug!(target: "ethstats", "Failed to report pending: {}", e); + self.disconnect().await; + } + } + + // Handle stats reporting + _ = report_interval.tick() => { + if let Err(e) = self.report().await { + debug!(target: "ethstats", "Failed to report: {}", e); + self.disconnect().await; + } + } + + // Handle reconnection + _ = reconnect_interval.tick(), if self.conn.read().await.is_none() => { + match self.connect().await { + Ok(_) => info!(target: "ethstats", "Reconnected successfully"), + Err(e) => debug!(target: "ethstats", "Reconnect failed: {}", e), + } + } + } + } + + // Cleanup + self.disconnect().await; + + // Cancel background tasks + read_handle.abort(); + canonical_stream_handle.abort(); + } + + /// Gracefully close the `WebSocket` connection + /// + /// Attempts to close the connection cleanly and logs any errors + /// that occur during the process. + async fn disconnect(&self) { + if let Some(conn) = self.conn.write().await.take() && + let Err(e) = conn.close().await + { + debug!(target: "ethstats", "Error closing connection: {}", e); + } + } + + /// Test helper to check connection status + #[cfg(test)] + pub async fn is_connected(&self) -> bool { + self.conn.read().await.is_some() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use futures_util::{SinkExt, StreamExt}; + use reth_network_api::noop::NoopNetwork; + use reth_storage_api::noop::NoopProvider; + use reth_transaction_pool::noop::NoopTransactionPool; + use serde_json::json; + use tokio::net::TcpListener; + use tokio_tungstenite::tungstenite::protocol::{frame::Utf8Bytes, Message}; + + const TEST_HOST: &str = "127.0.0.1"; + const TEST_PORT: u16 = 0; // Let OS choose port + + async fn setup_mock_server() -> (String, tokio::task::JoinHandle<()>) { + let listener = TcpListener::bind((TEST_HOST, TEST_PORT)).await.unwrap(); + let addr = listener.local_addr().unwrap(); + + let handle = tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + let mut ws_stream = tokio_tungstenite::accept_async(stream).await.unwrap(); + + // Handle login + if let Some(Ok(Message::Text(text))) = ws_stream.next().await { + let value: serde_json::Value = serde_json::from_str(&text).unwrap(); + if value["emit"][0] == "hello" { + let response = json!({ + "emit": ["ready", []] + }); + ws_stream + .send(Message::Text(Utf8Bytes::from(response.to_string()))) + .await + .unwrap(); + } + } + + // Handle ping + while let Some(Ok(msg)) = ws_stream.next().await { + if let Message::Text(text) = msg && + text.contains("node-ping") + { + let pong = json!({ + "emit": ["node-pong", {"id": "test-node"}] + }); + ws_stream.send(Message::Text(Utf8Bytes::from(pong.to_string()))).await.unwrap(); + } + } + }); + + (addr.to_string(), handle) + } + + #[tokio::test] + async fn test_connection_and_login() { + let (server_url, server_handle) = setup_mock_server().await; + let ethstats_url = format!("test-node:test-secret@{server_url}"); + + let network = NoopNetwork::default(); + let provider = NoopProvider::default(); + let pool = NoopTransactionPool::default(); + + let service = EthStatsService::new(ðstats_url, network, provider, pool) + .await + .expect("Service should connect"); + + // Verify connection was established + assert!(service.is_connected().await, "Service should be connected"); + + // Clean up server + server_handle.abort(); + } + + #[tokio::test] + async fn test_history_command_handling() { + let (server_url, server_handle) = setup_mock_server().await; + let ethstats_url = format!("test-node:test-secret@{server_url}"); + + let network = NoopNetwork::default(); + let provider = NoopProvider::default(); + let pool = NoopTransactionPool::default(); + + let service = EthStatsService::new(ðstats_url, network, provider, pool) + .await + .expect("Service should connect"); + + // Simulate receiving a history command + let history_cmd = json!({ + "emit": ["history", {"list": [1, 2, 3]}] + }); + + service.handle_message(history_cmd).await.expect("History command should be handled"); + + // Clean up server + server_handle.abort(); + } + + #[tokio::test] + async fn test_invalid_url_handling() { + let network = NoopNetwork::default(); + let provider = NoopProvider::default(); + let pool = NoopTransactionPool::default(); + + // Test missing secret + let result = EthStatsService::new( + "test-node@localhost", + network.clone(), + provider.clone(), + pool.clone(), + ) + .await; + assert!( + matches!(result, Err(EthStatsError::InvalidUrl(_))), + "Should detect invalid URL format" + ); + + // Test invalid URL format + let result = EthStatsService::new("invalid-url", network, provider, pool).await; + assert!( + matches!(result, Err(EthStatsError::InvalidUrl(_))), + "Should detect invalid URL format" + ); + } +} diff --git a/crates/node/ethstats/src/events.rs b/crates/node/ethstats/src/events.rs new file mode 100644 index 00000000000..08d0c90feb6 --- /dev/null +++ b/crates/node/ethstats/src/events.rs @@ -0,0 +1,283 @@ +//! Types for ethstats event reporting. +//! These structures define the data format used to report blockchain events to ethstats servers. + +use alloy_consensus::Header; +use alloy_primitives::{Address, B256, U256}; +use serde::{Deserialize, Serialize}; + +/// Collection of meta information about a node that is displayed on the monitoring page. +/// This information is used to identify and display node details in the ethstats monitoring +/// interface. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct NodeInfo { + /// The display name of the node in the monitoring interface + pub name: String, + + /// The node's unique identifier + pub node: String, + + /// The port number the node is listening on for P2P connections + pub port: u64, + + /// The network ID the node is connected to (e.g. "1" for mainnet) + #[serde(rename = "net")] + pub network: String, + + /// Comma-separated list of supported protocols and their versions + pub protocol: String, + + /// API availability indicator ("Yes" or "No") + pub api: String, + + /// Operating system the node is running on + pub os: String, + + /// Operating system version/architecture + #[serde(rename = "os_v")] + pub os_ver: String, + + /// Client software version + pub client: String, + + /// Whether the node can provide historical block data + #[serde(rename = "canUpdateHistory")] + pub history: bool, +} + +/// Authentication message used to login to the ethstats monitoring server. +/// Contains node identification and authentication information. +#[derive(Debug, Serialize, Deserialize)] +pub struct AuthMsg { + /// The node's unique identifier + pub id: String, + + /// Detailed information about the node + pub info: NodeInfo, + + /// Secret password for authentication with the monitoring server + pub secret: String, +} + +impl AuthMsg { + /// Generate a login message for the ethstats monitoring server. + pub fn generate_login_message(&self) -> String { + serde_json::json!({ + "emit": ["hello", self] + }) + .to_string() + } +} + +/// Simplified transaction info, containing only the hash. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TxStats { + /// Transaction hash + pub hash: B256, +} + +/// Wrapper for uncle block headers. +/// This ensures empty lists serialize as `[]` instead of `null`. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(transparent)] +pub struct UncleStats(pub Vec
); + +/// Information to report about individual blocks. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BlockStats { + /// Block number (height in the chain). + pub number: U256, + + /// Hash of this block. + pub hash: B256, + + /// Hash of the parent block. + #[serde(rename = "parentHash")] + pub parent_hash: B256, + + /// Timestamp of the block (Unix time). + pub timestamp: U256, + + /// Address of the miner who produced this block. + pub miner: Address, + + /// Total gas used by all transactions in the block. + #[serde(rename = "gasUsed")] + pub gas_used: u64, + + /// Maximum gas allowed for this block. + #[serde(rename = "gasLimit")] + pub gas_limit: u64, + + /// Difficulty for mining this block (as a decimal string). + #[serde(rename = "difficulty")] + pub diff: String, + + /// Cumulative difficulty up to this block (as a decimal string). + #[serde(rename = "totalDifficulty")] + pub total_diff: String, + + /// Simplified list of transactions in the block. + #[serde(rename = "transactions")] + pub txs: Vec, + + /// Root hash of all transactions (Merkle root). + #[serde(rename = "transactionsRoot")] + pub tx_root: B256, + + /// State root after applying this block. + #[serde(rename = "stateRoot")] + pub root: B256, + + /// List of uncle block headers. + pub uncles: UncleStats, +} + +/// Message containing a block to be reported to the ethstats monitoring server. +#[derive(Debug, Serialize, Deserialize)] +pub struct BlockMsg { + /// The node's unique identifier + pub id: String, + + /// The block to report + pub block: BlockStats, +} + +impl BlockMsg { + /// Generate a block message for the ethstats monitoring server. + pub fn generate_block_message(&self) -> String { + serde_json::json!({ + "emit": ["block", self] + }) + .to_string() + } +} + +/// Message containing historical block data to be reported to the ethstats monitoring server. +#[derive(Debug, Serialize, Deserialize)] +pub struct HistoryMsg { + /// The node's unique identifier + pub id: String, + + /// The historical block data to report + pub history: Vec, +} + +impl HistoryMsg { + /// Generate a history message for the ethstats monitoring server. + pub fn generate_history_message(&self) -> String { + serde_json::json!({ + "emit": ["history", self] + }) + .to_string() + } +} + +/// Message containing pending transaction statistics to be reported to the ethstats monitoring +/// server. +#[derive(Debug, Serialize, Deserialize)] +pub struct PendingStats { + /// Number of pending transactions + pub pending: u64, +} + +/// Message containing pending transaction statistics to be reported to the ethstats monitoring +/// server. +#[derive(Debug, Serialize, Deserialize)] +pub struct PendingMsg { + /// The node's unique identifier + pub id: String, + + /// The pending transaction statistics to report + pub stats: PendingStats, +} + +impl PendingMsg { + /// Generate a pending message for the ethstats monitoring server. + pub fn generate_pending_message(&self) -> String { + serde_json::json!({ + "emit": ["pending", self] + }) + .to_string() + } +} + +/// Information reported about the local node. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct NodeStats { + /// Whether the node is active + pub active: bool, + + /// Whether the node is currently syncing + pub syncing: bool, + + /// Number of connected peers + pub peers: u64, + + /// Current gas price in wei + #[serde(rename = "gasPrice")] + pub gas_price: u64, + + /// Node uptime percentage + pub uptime: u64, +} + +/// Message containing node statistics to be reported to the ethstats monitoring server. +#[derive(Debug, Serialize, Deserialize)] +pub struct StatsMsg { + /// The node's unique identifier + pub id: String, + + /// The stats to report + pub stats: NodeStats, +} + +impl StatsMsg { + /// Generate a stats message for the ethstats monitoring server. + pub fn generate_stats_message(&self) -> String { + serde_json::json!({ + "emit": ["stats", self] + }) + .to_string() + } +} + +/// Latency report message used to report network latency to the ethstats monitoring server. +#[derive(Serialize, Deserialize, Debug)] +pub struct LatencyMsg { + /// The node's unique identifier + pub id: String, + + /// The latency to report in milliseconds + pub latency: u64, +} + +impl LatencyMsg { + /// Generate a latency message for the ethstats monitoring server. + pub fn generate_latency_message(&self) -> String { + serde_json::json!({ + "emit": ["latency", self] + }) + .to_string() + } +} + +/// Ping message sent to the ethstats monitoring server to initiate latency measurement. +#[derive(Serialize, Deserialize, Debug)] +pub struct PingMsg { + /// The node's unique identifier + pub id: String, + + /// Client timestamp when the ping was sent + #[serde(rename = "clientTime")] + pub client_time: String, +} + +impl PingMsg { + /// Generate a ping message for the ethstats monitoring server. + pub fn generate_ping_message(&self) -> String { + serde_json::json!({ + "emit": ["node-ping", self] + }) + .to_string() + } +} diff --git a/crates/node/ethstats/src/lib.rs b/crates/node/ethstats/src/lib.rs new file mode 100644 index 00000000000..48d02a9f9bd --- /dev/null +++ b/crates/node/ethstats/src/lib.rs @@ -0,0 +1,30 @@ +//! +//! `EthStats` client support for Reth. +//! +//! This crate provides the necessary components to connect to, authenticate with, and report +//! node and network statistics to an `EthStats` server. It includes abstractions for `WebSocket` +//! connections, error handling, event/message types, and the main `EthStats` service logic. +//! +//! - `connection`: `WebSocket` connection management and utilities +//! - `error`: Error types for connection and `EthStats` operations +//! - `ethstats`: Main service logic for `EthStats` client +//! - `events`: Data structures for `EthStats` protocol messages + +#![doc( + html_logo_url = "https://raw.githubusercontent.com/paradigmxyz/reth/main/assets/reth-docs.png", + html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256", + issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" +)] +#![cfg_attr(not(test), warn(unused_crate_dependencies))] +#![cfg_attr(docsrs, feature(doc_cfg))] + +mod connection; +mod credentials; + +mod error; + +mod ethstats; +pub use ethstats::*; + +mod events; +pub use events::*; diff --git a/crates/node/events/src/cl.rs b/crates/node/events/src/cl.rs index bdced7c97d6..99cdc1c245f 100644 --- a/crates/node/events/src/cl.rs +++ b/crates/node/events/src/cl.rs @@ -61,7 +61,7 @@ impl Stream for ConsensusLayerHealthEvents { )) } - // We never had both FCU and transition config exchange. + // We never received any forkchoice updates. return Poll::Ready(Some(ConsensusLayerHealthEvent::NeverSeen)) } } @@ -71,12 +71,8 @@ impl Stream for ConsensusLayerHealthEvents { /// Execution Layer point of view. #[derive(Clone, Copy, Debug)] pub enum ConsensusLayerHealthEvent { - /// Consensus Layer client was never seen. + /// Consensus Layer client was never seen (no forkchoice updates received). NeverSeen, - /// Consensus Layer client has not been seen for a while. - HasNotBeenSeenForAWhile(Duration), - /// Updates from the Consensus Layer client were never received. - NeverReceivedUpdates, - /// Updates from the Consensus Layer client have not been received for a while. + /// Forkchoice updates from the Consensus Layer client have not been received for a while. HaveNotReceivedUpdatesForAWhile(Duration), } diff --git a/crates/node/events/src/lib.rs b/crates/node/events/src/lib.rs index e4665066c70..3647fbd1eec 100644 --- a/crates/node/events/src/lib.rs +++ b/crates/node/events/src/lib.rs @@ -6,7 +6,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] pub mod cl; pub mod node; diff --git a/crates/node/events/src/node.rs b/crates/node/events/src/node.rs index cb54bb70396..20ac4394b4f 100644 --- a/crates/node/events/src/node.rs +++ b/crates/node/events/src/node.rs @@ -6,7 +6,7 @@ use alloy_primitives::{BlockNumber, B256}; use alloy_rpc_types_engine::ForkchoiceState; use futures::Stream; use reth_engine_primitives::{ - BeaconConsensusEngineEvent, ConsensusEngineLiveSyncProgress, ForkchoiceStatus, + ConsensusEngineEvent, ConsensusEngineLiveSyncProgress, ForkchoiceStatus, }; use reth_network_api::PeersInfo; use reth_primitives_traits::{format_gas, format_gas_throughput, BlockBody, NodePrimitives}; @@ -37,14 +37,14 @@ struct NodeState { current_stage: Option, /// The latest block reached by either pipeline or consensus engine. latest_block: Option, - /// The time of the latest block seen by the pipeline - latest_block_time: Option, /// Hash of the head block last set by fork choice update head_block_hash: Option, /// Hash of the safe block last set by fork choice update safe_block_hash: Option, /// Hash of finalized block last set by fork choice update finalized_block_hash: Option, + /// The time when we last logged a status message + last_status_log_time: Option, } impl NodeState { @@ -56,10 +56,10 @@ impl NodeState { peers_info, current_stage: None, latest_block, - latest_block_time: None, head_block_hash: None, safe_block_hash: None, finalized_block_hash: None, + last_status_log_time: None, } } @@ -212,12 +212,9 @@ impl NodeState { } } - fn handle_consensus_engine_event( - &mut self, - event: BeaconConsensusEngineEvent, - ) { + fn handle_consensus_engine_event(&mut self, event: ConsensusEngineEvent) { match event { - BeaconConsensusEngineEvent::ForkchoiceUpdated(state, status) => { + ConsensusEngineEvent::ForkchoiceUpdated(state, status) => { let ForkchoiceState { head_block_hash, safe_block_hash, finalized_block_hash } = state; if self.safe_block_hash != Some(safe_block_hash) && @@ -236,7 +233,7 @@ impl NodeState { self.safe_block_hash = Some(safe_block_hash); self.finalized_block_hash = Some(finalized_block_hash); } - BeaconConsensusEngineEvent::LiveSyncProgress(live_sync_progress) => { + ConsensusEngineEvent::LiveSyncProgress(live_sync_progress) => { match live_sync_progress { ConsensusEngineLiveSyncProgress::DownloadingBlocks { remaining_blocks, @@ -250,36 +247,42 @@ impl NodeState { } } } - BeaconConsensusEngineEvent::CanonicalBlockAdded(executed, elapsed) => { + ConsensusEngineEvent::CanonicalBlockAdded(executed, elapsed) => { let block = executed.sealed_block(); + let mut full = block.gas_used() as f64 * 100.0 / block.gas_limit() as f64; + if full.is_nan() { + full = 0.0; + } info!( number=block.number(), hash=?block.hash(), peers=self.num_connected_peers(), txs=block.body().transactions().len(), - gas=%format_gas(block.gas_used()), + gas_used=%format_gas(block.gas_used()), gas_throughput=%format_gas_throughput(block.gas_used(), elapsed), - full=%format!("{:.1}%", block.gas_used() as f64 * 100.0 / block.gas_limit() as f64), - base_fee=%format!("{:.2}gwei", block.base_fee_per_gas().unwrap_or(0) as f64 / GWEI_TO_WEI as f64), + gas_limit=%format_gas(block.gas_limit()), + full=%format!("{:.1}%", full), + base_fee=%format!("{:.2}Gwei", block.base_fee_per_gas().unwrap_or(0) as f64 / GWEI_TO_WEI as f64), blobs=block.blob_gas_used().unwrap_or(0) / alloy_eips::eip4844::DATA_GAS_PER_BLOB, excess_blobs=block.excess_blob_gas().unwrap_or(0) / alloy_eips::eip4844::DATA_GAS_PER_BLOB, ?elapsed, "Block added to canonical chain" ); } - BeaconConsensusEngineEvent::CanonicalChainCommitted(head, elapsed) => { + ConsensusEngineEvent::CanonicalChainCommitted(head, elapsed) => { self.latest_block = Some(head.number()); - self.latest_block_time = Some(head.timestamp()); - info!(number=head.number(), hash=?head.hash(), ?elapsed, "Canonical chain committed"); } - BeaconConsensusEngineEvent::ForkBlockAdded(executed, elapsed) => { + ConsensusEngineEvent::ForkBlockAdded(executed, elapsed) => { let block = executed.sealed_block(); info!(number=block.number(), hash=?block.hash(), ?elapsed, "Block added to fork chain"); } - BeaconConsensusEngineEvent::InvalidBlock(block) => { + ConsensusEngineEvent::InvalidBlock(block) => { warn!(number=block.number(), hash=?block.hash(), "Encountered invalid block"); } + ConsensusEngineEvent::BlockReceived(num_hash) => { + info!(number=num_hash.number, hash=?num_hash.hash, "Received block from consensus engine"); + } } } @@ -293,17 +296,6 @@ impl NodeState { "Post-merge network, but never seen beacon client. Please launch one to follow the chain!" ) } - ConsensusLayerHealthEvent::HasNotBeenSeenForAWhile(period) => { - warn!( - ?period, - "Post-merge network, but no beacon client seen for a while. Please launch one to follow the chain!" - ) - } - ConsensusLayerHealthEvent::NeverReceivedUpdates => { - warn!( - "Beacon client online, but never received consensus updates. Please ensure your beacon client is operational to follow the chain!" - ) - } ConsensusLayerHealthEvent::HaveNotReceivedUpdatesForAWhile(period) => { warn!( ?period, @@ -374,7 +366,7 @@ pub enum NodeEvent { /// A sync pipeline event. Pipeline(PipelineEvent), /// A consensus engine event. - ConsensusEngine(BeaconConsensusEngineEvent), + ConsensusEngine(ConsensusEngineEvent), /// A Consensus Layer health event. ConsensusLayerHealth(ConsensusLayerHealthEvent), /// A pruner event @@ -478,25 +470,28 @@ where ) } } - } else if let Some(latest_block) = this.state.latest_block { + } else { let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs(); - if now.saturating_sub(this.state.latest_block_time.unwrap_or(0)) > 60 { - // Once we start receiving consensus nodes, don't emit status unless stalled for - // 1 minute - info!( - target: "reth::cli", - connected_peers = this.state.num_connected_peers(), - %latest_block, - "Status" - ); + + // Only log status if we haven't logged recently + if now.saturating_sub(this.state.last_status_log_time.unwrap_or(0)) > 60 { + if let Some(latest_block) = this.state.latest_block { + info!( + target: "reth::cli", + connected_peers = this.state.num_connected_peers(), + %latest_block, + "Status" + ); + } else { + info!( + target: "reth::cli", + connected_peers = this.state.num_connected_peers(), + "Status" + ); + } + this.state.last_status_log_time = Some(now); } - } else { - info!( - target: "reth::cli", - connected_peers = this.state.num_connected_peers(), - "Status" - ); } } @@ -601,6 +596,8 @@ impl Display for Eta { f, "{}", humantime::format_duration(Duration::from_secs(remaining.as_secs())) + .to_string() + .replace(' ', "") ) } } @@ -626,6 +623,6 @@ mod tests { } .to_string(); - assert_eq!(eta, "13m 37s"); + assert_eq!(eta, "13m37s"); } } diff --git a/crates/node/metrics/Cargo.toml b/crates/node/metrics/Cargo.toml index 39884fa73ef..9687c9c20ac 100644 --- a/crates/node/metrics/Cargo.toml +++ b/crates/node/metrics/Cargo.toml @@ -21,6 +21,7 @@ tokio.workspace = true jsonrpsee-server.workspace = true http.workspace = true tower.workspace = true +reqwest.workspace = true tracing.workspace = true eyre.workspace = true diff --git a/crates/node/metrics/src/lib.rs b/crates/node/metrics/src/lib.rs index d74a8aeffba..0f6525c873d 100644 --- a/crates/node/metrics/src/lib.rs +++ b/crates/node/metrics/src/lib.rs @@ -5,7 +5,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] pub mod chain; /// The metrics hooks for prometheus. diff --git a/crates/node/metrics/src/server.rs b/crates/node/metrics/src/server.rs index 96918f90f1a..26e9a918faa 100644 --- a/crates/node/metrics/src/server.rs +++ b/crates/node/metrics/src/server.rs @@ -8,9 +8,10 @@ use eyre::WrapErr; use http::{header::CONTENT_TYPE, HeaderValue, Response}; use metrics::describe_gauge; use metrics_process::Collector; +use reqwest::Client; use reth_metrics::metrics::Unit; use reth_tasks::TaskExecutor; -use std::{convert::Infallible, net::SocketAddr, sync::Arc}; +use std::{convert::Infallible, net::SocketAddr, sync::Arc, time::Duration}; /// Configuration for the [`MetricServer`] #[derive(Debug)] @@ -20,6 +21,8 @@ pub struct MetricServerConfig { chain_spec_info: ChainSpecInfo, task_executor: TaskExecutor, hooks: Hooks, + push_gateway_url: Option, + push_gateway_interval: Duration, } impl MetricServerConfig { @@ -31,7 +34,22 @@ impl MetricServerConfig { task_executor: TaskExecutor, hooks: Hooks, ) -> Self { - Self { listen_addr, hooks, task_executor, version_info, chain_spec_info } + Self { + listen_addr, + hooks, + task_executor, + version_info, + chain_spec_info, + push_gateway_url: None, + push_gateway_interval: Duration::from_secs(5), + } + } + + /// Set the gateway URL and interval for pushing metrics + pub fn with_push_gateway(mut self, url: Option, interval: Duration) -> Self { + self.push_gateway_url = url; + self.push_gateway_interval = interval; + self } } @@ -49,17 +67,34 @@ impl MetricServer { /// Spawns the metrics server pub async fn serve(&self) -> eyre::Result<()> { - let MetricServerConfig { listen_addr, hooks, task_executor, version_info, chain_spec_info } = - &self.config; + let MetricServerConfig { + listen_addr, + hooks, + task_executor, + version_info, + chain_spec_info, + push_gateway_url, + push_gateway_interval, + } = &self.config; - let hooks = hooks.clone(); + let hooks_for_endpoint = hooks.clone(); self.start_endpoint( *listen_addr, - Arc::new(move || hooks.iter().for_each(|hook| hook())), + Arc::new(move || hooks_for_endpoint.iter().for_each(|hook| hook())), task_executor.clone(), ) .await - .wrap_err("Could not start Prometheus endpoint")?; + .wrap_err_with(|| format!("Could not start Prometheus endpoint at {listen_addr}"))?; + + // Start push-gateway task if configured + if let Some(url) = push_gateway_url { + self.start_push_gateway_task( + url.clone(), + *push_gateway_interval, + hooks.clone(), + task_executor.clone(), + )?; + } // Describe metrics after recorder installation describe_db_metrics(); @@ -84,47 +119,97 @@ impl MetricServer { .await .wrap_err("Could not bind to address")?; - task_executor.spawn_with_graceful_shutdown_signal(|mut signal| async move { - loop { - let io = tokio::select! { - _ = &mut signal => break, - io = listener.accept() => { - match io { - Ok((stream, _remote_addr)) => stream, - Err(err) => { - tracing::error!(%err, "failed to accept connection"); - continue; + tracing::info!(target: "reth::cli", "Starting metrics endpoint at {}", listener.local_addr().unwrap()); + + task_executor.spawn_with_graceful_shutdown_signal(|mut signal| { + Box::pin(async move { + loop { + let io = tokio::select! { + _ = &mut signal => break, + io = listener.accept() => { + match io { + Ok((stream, _remote_addr)) => stream, + Err(err) => { + tracing::error!(%err, "failed to accept connection"); + continue; + } } } - } - }; + }; - let handle = install_prometheus_recorder(); - let hook = hook.clone(); - let service = tower::service_fn(move |_| { - (hook)(); - let metrics = handle.handle().render(); - let mut response = Response::new(metrics); - response - .headers_mut() - .insert(CONTENT_TYPE, HeaderValue::from_static("text/plain")); - async move { Ok::<_, Infallible>(response) } - }); - - let mut shutdown = signal.clone().ignore_guard(); - tokio::task::spawn(async move { - let _ = - jsonrpsee_server::serve_with_graceful_shutdown(io, service, &mut shutdown) - .await - .inspect_err( - |error| tracing::debug!(%error, "failed to serve request"), - ); - }); - } + let handle = install_prometheus_recorder(); + let hook = hook.clone(); + let service = tower::service_fn(move |_| { + (hook)(); + let metrics = handle.handle().render(); + let mut response = Response::new(metrics); + response + .headers_mut() + .insert(CONTENT_TYPE, HeaderValue::from_static("text/plain")); + async move { Ok::<_, Infallible>(response) } + }); + + let mut shutdown = signal.clone().ignore_guard(); + tokio::task::spawn(async move { + let _ = jsonrpsee_server::serve_with_graceful_shutdown( + io, + service, + &mut shutdown, + ) + .await + .inspect_err(|error| tracing::debug!(%error, "failed to serve request")); + }); + } + }) }); Ok(()) } + + /// Starts a background task to push metrics to a metrics gateway + fn start_push_gateway_task( + &self, + url: String, + interval: Duration, + hooks: Hooks, + task_executor: TaskExecutor, + ) -> eyre::Result<()> { + let client = Client::builder() + .build() + .wrap_err("Could not create HTTP client to push metrics to gateway")?; + task_executor.spawn_with_graceful_shutdown_signal(move |mut signal| { + Box::pin(async move { + tracing::info!(url = %url, interval = ?interval, "Starting task to push metrics to gateway"); + let handle = install_prometheus_recorder(); + loop { + tokio::select! { + _ = &mut signal => { + tracing::info!("Shutting down task to push metrics to gateway"); + break; + } + _ = tokio::time::sleep(interval) => { + hooks.iter().for_each(|hook| hook()); + let metrics = handle.handle().render(); + match client.put(&url).header("Content-Type", "text/plain").body(metrics).send().await { + Ok(response) => { + if !response.status().is_success() { + tracing::warn!( + status = %response.status(), + "Failed to push metrics to gateway" + ); + } + } + Err(err) => { + tracing::warn!(%err, "Failed to push metrics to gateway"); + } + } + } + } + } + }) + }); + Ok(()) + } } fn describe_db_metrics() { diff --git a/crates/node/types/Cargo.toml b/crates/node/types/Cargo.toml index 26296adf7d6..a6c0e80d2c8 100644 --- a/crates/node/types/Cargo.toml +++ b/crates/node/types/Cargo.toml @@ -16,7 +16,6 @@ reth-chainspec.workspace = true reth-db-api.workspace = true reth-engine-primitives.workspace = true reth-primitives-traits.workspace = true -reth-trie-db.workspace = true reth-payload-primitives.workspace = true [features] diff --git a/crates/node/types/src/lib.rs b/crates/node/types/src/lib.rs index 13245a18b9b..b5b38f48c7d 100644 --- a/crates/node/types/src/lib.rs +++ b/crates/node/types/src/lib.rs @@ -6,19 +6,18 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(feature = "std"), no_std)] use core::{fmt::Debug, marker::PhantomData}; pub use reth_primitives_traits::{ - Block, BlockBody, FullBlock, FullNodePrimitives, FullReceipt, FullSignedTx, NodePrimitives, + Block, BlockBody, FullBlock, FullReceipt, FullSignedTx, NodePrimitives, }; use reth_chainspec::EthChainSpec; use reth_db_api::{database_metrics::DatabaseMetrics, Database}; use reth_engine_primitives::EngineTypes; use reth_payload_primitives::{BuiltPayload, PayloadTypes}; -use reth_trie_db::StateCommitment; /// The type that configures the essential types of an Ethereum-like node. /// @@ -30,8 +29,6 @@ pub trait NodeTypes: Clone + Debug + Send + Sync + Unpin + 'static { type Primitives: NodePrimitives; /// The type used for configuration of the EVM. type ChainSpec: EthChainSpec
::BlockHeader>; - /// The type used to perform state commitment operations. - type StateCommitment: StateCommitment; /// The type responsible for writing chain primitives to storage. type Storage: Default + Send + Sync + Unpin + Debug + 'static; /// The node's engine types, defining the interaction with the consensus engine. @@ -68,7 +65,6 @@ where { type Primitives = Types::Primitives; type ChainSpec = Types::ChainSpec; - type StateCommitment = Types::StateCommitment; type Storage = Types::Storage; type Payload = Types::Payload; } @@ -83,119 +79,104 @@ where /// A [`NodeTypes`] type builder. #[derive(Clone, Debug, Default)] -pub struct AnyNodeTypes

( +pub struct AnyNodeTypes

( PhantomData

, PhantomData, - PhantomData, PhantomData, PhantomData, ); -impl AnyNodeTypes { +impl AnyNodeTypes { /// Creates a new instance of [`AnyNodeTypes`]. pub const fn new() -> Self { - Self(PhantomData, PhantomData, PhantomData, PhantomData, PhantomData) + Self(PhantomData, PhantomData, PhantomData, PhantomData) } /// Sets the `Primitives` associated type. - pub const fn primitives(self) -> AnyNodeTypes { + pub const fn primitives(self) -> AnyNodeTypes { AnyNodeTypes::new() } /// Sets the `ChainSpec` associated type. - pub const fn chain_spec(self) -> AnyNodeTypes { - AnyNodeTypes::new() - } - - /// Sets the `StateCommitment` associated type. - pub const fn state_commitment(self) -> AnyNodeTypes { + pub const fn chain_spec(self) -> AnyNodeTypes { AnyNodeTypes::new() } /// Sets the `Storage` associated type. - pub const fn storage(self) -> AnyNodeTypes { + pub const fn storage(self) -> AnyNodeTypes { AnyNodeTypes::new() } /// Sets the `Payload` associated type. - pub const fn payload(self) -> AnyNodeTypes { + pub const fn payload(self) -> AnyNodeTypes { AnyNodeTypes::new() } } -impl NodeTypes for AnyNodeTypes +impl NodeTypes for AnyNodeTypes where P: NodePrimitives + Send + Sync + Unpin + 'static, C: EthChainSpec

+ Clone + 'static, - SC: StateCommitment, S: Default + Clone + Send + Sync + Unpin + Debug + 'static, PL: PayloadTypes> + Send + Sync + Unpin + 'static, { type Primitives = P; type ChainSpec = C; - type StateCommitment = SC; type Storage = S; type Payload = PL; } /// A [`NodeTypes`] type builder. #[derive(Clone, Debug, Default)] -pub struct AnyNodeTypesWithEngine

{ +pub struct AnyNodeTypesWithEngine

{ /// Embedding the basic node types. - _base: AnyNodeTypes, + _base: AnyNodeTypes, /// Phantom data for the engine. _engine: PhantomData, } -impl AnyNodeTypesWithEngine { +impl AnyNodeTypesWithEngine { /// Creates a new instance of [`AnyNodeTypesWithEngine`]. pub const fn new() -> Self { Self { _base: AnyNodeTypes::new(), _engine: PhantomData } } /// Sets the `Primitives` associated type. - pub const fn primitives(self) -> AnyNodeTypesWithEngine { + pub const fn primitives(self) -> AnyNodeTypesWithEngine { AnyNodeTypesWithEngine::new() } /// Sets the `Engine` associated type. - pub const fn engine(self) -> AnyNodeTypesWithEngine { + pub const fn engine(self) -> AnyNodeTypesWithEngine { AnyNodeTypesWithEngine::new() } /// Sets the `ChainSpec` associated type. - pub const fn chain_spec(self) -> AnyNodeTypesWithEngine { - AnyNodeTypesWithEngine::new() - } - - /// Sets the `StateCommitment` associated type. - pub const fn state_commitment(self) -> AnyNodeTypesWithEngine { + pub const fn chain_spec(self) -> AnyNodeTypesWithEngine { AnyNodeTypesWithEngine::new() } /// Sets the `Storage` associated type. - pub const fn storage(self) -> AnyNodeTypesWithEngine { + pub const fn storage(self) -> AnyNodeTypesWithEngine { AnyNodeTypesWithEngine::new() } /// Sets the `Payload` associated type. - pub const fn payload(self) -> AnyNodeTypesWithEngine { + pub const fn payload(self) -> AnyNodeTypesWithEngine { AnyNodeTypesWithEngine::new() } } -impl NodeTypes for AnyNodeTypesWithEngine +impl NodeTypes for AnyNodeTypesWithEngine where P: NodePrimitives + Send + Sync + Unpin + 'static, E: EngineTypes + Send + Sync + Unpin, C: EthChainSpec

+ Clone + 'static, - SC: StateCommitment, S: Default + Clone + Send + Sync + Unpin + Debug + 'static, PL: PayloadTypes> + Send + Sync + Unpin + 'static, { type Primitives = P; type ChainSpec = C; - type StateCommitment = SC; type Storage = S; type Payload = PL; } @@ -218,5 +199,5 @@ pub type ReceiptTy = as NodePrimitives>::Receipt; /// Helper type for getting the `Primitives` associated type from a [`NodeTypes`]. pub type PrimitivesTy = ::Primitives; -/// Helper type for getting the `Primitives` associated type from a [`NodeTypes`]. -pub type KeyHasherTy = <::StateCommitment as StateCommitment>::KeyHasher; +/// Helper adapter type for accessing [`PayloadTypes::PayloadAttributes`] on [`NodeTypes`]. +pub type PayloadAttrTy = <::Payload as PayloadTypes>::PayloadAttributes; diff --git a/crates/optimism/bin/Cargo.toml b/crates/optimism/bin/Cargo.toml index 3733227a3aa..ef203df0fc0 100644 --- a/crates/optimism/bin/Cargo.toml +++ b/crates/optimism/bin/Cargo.toml @@ -12,7 +12,7 @@ exclude.workspace = true reth-cli-util.workspace = true reth-optimism-cli.workspace = true reth-optimism-rpc.workspace = true -reth-optimism-node = { workspace = true, features = ["js-tracer"] } +reth-optimism-node.workspace = true reth-optimism-chainspec.workspace = true reth-optimism-consensus.workspace = true reth-optimism-evm.workspace = true @@ -27,7 +27,13 @@ tracing.workspace = true workspace = true [features] -default = ["jemalloc", "reth-optimism-evm/portable"] +default = ["jemalloc", "otlp", "reth-optimism-evm/portable", "js-tracer"] + +otlp = ["reth-optimism-cli/otlp"] + +js-tracer = [ + "reth-optimism-node/js-tracer", +] jemalloc = ["reth-cli-util/jemalloc", "reth-optimism-cli/jemalloc"] jemalloc-prof = ["reth-cli-util/jemalloc-prof"] diff --git a/crates/optimism/bin/src/lib.rs b/crates/optimism/bin/src/lib.rs index 2a452c016e2..f518f2cef03 100644 --- a/crates/optimism/bin/src/lib.rs +++ b/crates/optimism/bin/src/lib.rs @@ -23,7 +23,7 @@ html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256", issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] /// Re-exported from `reth_optimism_cli`. pub mod cli { diff --git a/crates/optimism/bin/src/main.rs b/crates/optimism/bin/src/main.rs index e1567ef1c9b..b8f87ac77ef 100644 --- a/crates/optimism/bin/src/main.rs +++ b/crates/optimism/bin/src/main.rs @@ -13,7 +13,9 @@ fn main() { // Enable backtraces unless a RUST_BACKTRACE value has already been explicitly provided. if std::env::var_os("RUST_BACKTRACE").is_none() { - std::env::set_var("RUST_BACKTRACE", "1"); + unsafe { + std::env::set_var("RUST_BACKTRACE", "1"); + } } if let Err(err) = diff --git a/crates/optimism/chainspec/Cargo.toml b/crates/optimism/chainspec/Cargo.toml index 12ca67199c5..69b3d68e42c 100644 --- a/crates/optimism/chainspec/Cargo.toml +++ b/crates/optimism/chainspec/Cargo.toml @@ -48,11 +48,11 @@ miniz_oxide = { workspace = true, features = ["with-alloc"], optional = true } derive_more.workspace = true paste = { workspace = true, optional = true } thiserror = { workspace = true, optional = true } +op-alloy-consensus.workspace = true [dev-dependencies] reth-chainspec = { workspace = true, features = ["test-utils"] } -alloy-genesis.workspace = true -op-alloy-rpc-types.workspace = true +alloy-op-hardforks.workspace = true [features] default = ["std"] @@ -75,6 +75,7 @@ std = [ "serde?/std", "miniz_oxide?/std", "thiserror?/std", + "op-alloy-consensus/std", ] serde = [ "alloy-chains/serde", @@ -88,4 +89,6 @@ serde = [ "reth-optimism-forks/serde", "reth-optimism-primitives/serde", "reth-primitives-traits/serde", + "op-alloy-consensus/serde", + "alloy-op-hardforks/serde", ] diff --git a/crates/optimism/chainspec/res/superchain-configs.tar b/crates/optimism/chainspec/res/superchain-configs.tar index acfa1b81c91..2ed30f474b8 100644 Binary files a/crates/optimism/chainspec/res/superchain-configs.tar and b/crates/optimism/chainspec/res/superchain-configs.tar differ diff --git a/crates/optimism/chainspec/res/superchain_registry_commit b/crates/optimism/chainspec/res/superchain_registry_commit index 939855efb4f..239646ec046 100644 --- a/crates/optimism/chainspec/res/superchain_registry_commit +++ b/crates/optimism/chainspec/res/superchain_registry_commit @@ -1 +1 @@ -a9b57281842bf5742cf9e69114c6b81c622ca186 +59e22d265b7a423b7f51a67a722471a6f3c3cc39 diff --git a/crates/optimism/chainspec/src/basefee.rs b/crates/optimism/chainspec/src/basefee.rs new file mode 100644 index 00000000000..3c0dcdfd88d --- /dev/null +++ b/crates/optimism/chainspec/src/basefee.rs @@ -0,0 +1,199 @@ +//! Base fee related utilities for Optimism chains. + +use core::cmp::max; + +use alloy_consensus::BlockHeader; +use alloy_eips::calc_next_block_base_fee; +use op_alloy_consensus::{decode_holocene_extra_data, decode_jovian_extra_data, EIP1559ParamError}; +use reth_chainspec::{BaseFeeParams, EthChainSpec}; +use reth_optimism_forks::OpHardforks; + +/// Extracts the Holocene 1599 parameters from the encoded extra data from the parent header. +/// +/// Caution: Caller must ensure that holocene is active in the parent header. +/// +/// See also [Base fee computation](https://github.com/ethereum-optimism/specs/blob/main/specs/protocol/holocene/exec-engine.md#base-fee-computation) +pub fn decode_holocene_base_fee( + chain_spec: impl EthChainSpec + OpHardforks, + parent: &H, + timestamp: u64, +) -> Result +where + H: BlockHeader, +{ + let (elasticity, denominator) = decode_holocene_extra_data(parent.extra_data())?; + + let base_fee_params = if elasticity == 0 && denominator == 0 { + chain_spec.base_fee_params_at_timestamp(timestamp) + } else { + BaseFeeParams::new(denominator as u128, elasticity as u128) + }; + + Ok(parent.next_block_base_fee(base_fee_params).unwrap_or_default()) +} + +/// Extracts the Jovian 1599 parameters from the encoded extra data from the parent header. +/// Additionally to [`decode_holocene_base_fee`], checks if the next block base fee is less than the +/// minimum base fee, then the minimum base fee is returned. +/// +/// Caution: Caller must ensure that jovian is active in the parent header. +/// +/// See also [Base fee computation](https://github.com/ethereum-optimism/specs/blob/main/specs/protocol/jovian/exec-engine.md#base-fee-computation) +/// and [Minimum base fee in block header](https://github.com/ethereum-optimism/specs/blob/main/specs/protocol/jovian/exec-engine.md#minimum-base-fee-in-block-header) +pub fn compute_jovian_base_fee( + chain_spec: impl EthChainSpec + OpHardforks, + parent: &H, + timestamp: u64, +) -> Result +where + H: BlockHeader, +{ + let (elasticity, denominator, min_base_fee) = decode_jovian_extra_data(parent.extra_data())?; + + let base_fee_params = if elasticity == 0 && denominator == 0 { + chain_spec.base_fee_params_at_timestamp(timestamp) + } else { + BaseFeeParams::new(denominator as u128, elasticity as u128) + }; + + // Starting from Jovian, we use the maximum of the gas used and the blob gas used to calculate + // the next base fee. + let gas_used = max(parent.gas_used(), parent.blob_gas_used().unwrap_or_default()); + + let next_base_fee = calc_next_block_base_fee( + gas_used, + parent.gas_limit(), + parent.base_fee_per_gas().unwrap_or_default(), + base_fee_params, + ); + + if next_base_fee < min_base_fee { + return Ok(min_base_fee); + } + + Ok(next_base_fee) +} + +#[cfg(test)] +mod tests { + use alloc::sync::Arc; + + use op_alloy_consensus::encode_jovian_extra_data; + use reth_chainspec::{ChainSpec, ForkCondition, Hardfork}; + use reth_optimism_forks::OpHardfork; + + use crate::{OpChainSpec, BASE_SEPOLIA}; + + use super::*; + + const JOVIAN_TIMESTAMP: u64 = 1900000000; + + fn get_chainspec() -> Arc { + let mut base_sepolia_spec = BASE_SEPOLIA.inner.clone(); + base_sepolia_spec + .hardforks + .insert(OpHardfork::Jovian.boxed(), ForkCondition::Timestamp(JOVIAN_TIMESTAMP)); + Arc::new(OpChainSpec { + inner: ChainSpec { + chain: base_sepolia_spec.chain, + genesis: base_sepolia_spec.genesis, + genesis_header: base_sepolia_spec.genesis_header, + ..Default::default() + }, + }) + } + + #[test] + fn test_next_base_fee_jovian_blob_gas_used_greater_than_gas_used() { + let chain_spec = get_chainspec(); + let mut parent = chain_spec.genesis_header().clone(); + let timestamp = JOVIAN_TIMESTAMP; + + const GAS_LIMIT: u64 = 10_000_000_000; + const BLOB_GAS_USED: u64 = 5_000_000_000; + const GAS_USED: u64 = 1_000_000_000; + const MIN_BASE_FEE: u64 = 100_000_000; + + parent.extra_data = + encode_jovian_extra_data([0; 8].into(), BaseFeeParams::base_sepolia(), MIN_BASE_FEE) + .unwrap(); + parent.blob_gas_used = Some(BLOB_GAS_USED); + parent.gas_used = GAS_USED; + parent.gas_limit = GAS_LIMIT; + + let expected_base_fee = calc_next_block_base_fee( + BLOB_GAS_USED, + parent.gas_limit(), + parent.base_fee_per_gas().unwrap_or_default(), + BaseFeeParams::base_sepolia(), + ); + assert_eq!( + expected_base_fee, + compute_jovian_base_fee(chain_spec, &parent, timestamp).unwrap() + ); + assert_ne!( + expected_base_fee, + calc_next_block_base_fee( + GAS_USED, + parent.gas_limit(), + parent.base_fee_per_gas().unwrap_or_default(), + BaseFeeParams::base_sepolia(), + ) + ) + } + + #[test] + fn test_next_base_fee_jovian_blob_gas_used_less_than_gas_used() { + let chain_spec = get_chainspec(); + let mut parent = chain_spec.genesis_header().clone(); + let timestamp = JOVIAN_TIMESTAMP; + + const GAS_LIMIT: u64 = 10_000_000_000; + const BLOB_GAS_USED: u64 = 100_000_000; + const GAS_USED: u64 = 1_000_000_000; + const MIN_BASE_FEE: u64 = 100_000_000; + + parent.extra_data = + encode_jovian_extra_data([0; 8].into(), BaseFeeParams::base_sepolia(), MIN_BASE_FEE) + .unwrap(); + parent.blob_gas_used = Some(BLOB_GAS_USED); + parent.gas_used = GAS_USED; + parent.gas_limit = GAS_LIMIT; + + let expected_base_fee = calc_next_block_base_fee( + GAS_USED, + parent.gas_limit(), + parent.base_fee_per_gas().unwrap_or_default(), + BaseFeeParams::base_sepolia(), + ); + assert_eq!( + expected_base_fee, + compute_jovian_base_fee(chain_spec, &parent, timestamp).unwrap() + ); + } + + #[test] + fn test_next_base_fee_jovian_min_base_fee() { + let chain_spec = get_chainspec(); + let mut parent = chain_spec.genesis_header().clone(); + let timestamp = JOVIAN_TIMESTAMP; + + const GAS_LIMIT: u64 = 10_000_000_000; + const BLOB_GAS_USED: u64 = 100_000_000; + const GAS_USED: u64 = 1_000_000_000; + const MIN_BASE_FEE: u64 = 5_000_000_000; + + parent.extra_data = + encode_jovian_extra_data([0; 8].into(), BaseFeeParams::base_sepolia(), MIN_BASE_FEE) + .unwrap(); + parent.blob_gas_used = Some(BLOB_GAS_USED); + parent.gas_used = GAS_USED; + parent.gas_limit = GAS_LIMIT; + + let expected_base_fee = MIN_BASE_FEE; + assert_eq!( + expected_base_fee, + compute_jovian_base_fee(chain_spec, &parent, timestamp).unwrap() + ); + } +} diff --git a/crates/optimism/chainspec/src/dev.rs b/crates/optimism/chainspec/src/dev.rs index 3778cd712a3..ac8eaad24a8 100644 --- a/crates/optimism/chainspec/src/dev.rs +++ b/crates/optimism/chainspec/src/dev.rs @@ -27,7 +27,6 @@ pub static OP_DEV: LazyLock> = LazyLock::new(|| { paris_block_and_final_difficulty: Some((0, U256::from(0))), hardforks, base_fee_params: BaseFeeParamsKind::Constant(BaseFeeParams::ethereum()), - deposit_contract: None, // TODO: do we even have? ..Default::default() }, } diff --git a/crates/optimism/chainspec/src/lib.rs b/crates/optimism/chainspec/src/lib.rs index 7c5a637cf9a..2d5d7308d9a 100644 --- a/crates/optimism/chainspec/src/lib.rs +++ b/crates/optimism/chainspec/src/lib.rs @@ -5,7 +5,7 @@ html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256", issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(test), warn(unused_crate_dependencies))] #![cfg_attr(not(feature = "std"), no_std)] @@ -34,6 +34,7 @@ extern crate alloc; mod base; mod base_sepolia; +mod basefee; pub mod constants; mod dev; @@ -47,21 +48,25 @@ mod superchain; #[cfg(feature = "superchain-configs")] pub use superchain::*; +pub use base::BASE_MAINNET; +pub use base_sepolia::BASE_SEPOLIA; +pub use basefee::*; pub use dev::OP_DEV; pub use mantle_mainnet::MANTLE_MAINNET; pub use op::OP_MAINNET; pub use op_sepolia::OP_SEPOLIA; use crate::mantle::MantleChainInfo; +/// Re-export for convenience +pub use reth_optimism_forks::*; + use alloc::{boxed::Box, vec, vec::Vec}; use alloy_chains::Chain; -use alloy_consensus::{proofs::storage_root_unhashed, Header}; +use alloy_consensus::{proofs::storage_root_unhashed, BlockHeader, Header}; use alloy_eips::eip7840::BlobParams; use alloy_genesis::Genesis; use alloy_hardforks::Hardfork; use alloy_primitives::{B256, U256}; -pub use base::BASE_MAINNET; -pub use base_sepolia::BASE_SEPOLIA; use derive_more::{Constructor, Deref, From, Into}; use reth_chainspec::{ BaseFeeParams, BaseFeeParamsKind, ChainSpec, ChainSpecBuilder, DepositContract, @@ -70,7 +75,6 @@ use reth_chainspec::{ use reth_ethereum_forks::{ChainHardforks, EthereumHardfork, ForkCondition}; use reth_mantle_forks::{MantleHardfork, MantleHardforks}; use reth_network_peers::NodeRecord; -use reth_optimism_forks::{OpHardfork, OpHardforks, OP_MAINNET_HARDFORKS}; use reth_optimism_primitives::ADDRESS_L2_TO_L1_MESSAGE_PASSER; use reth_primitives_traits::{sync::LazyLock, SealedHeader}; @@ -194,9 +198,16 @@ impl OpChainSpecBuilder { self } + /// Enable Jovian at genesis + pub fn jovian_activated(mut self) -> Self { + self = self.isthmus_activated(); + self.inner = self.inner.with_fork(OpHardfork::Jovian, ForkCondition::Timestamp(0)); + self + } + /// Enable Interop at genesis pub fn interop_activated(mut self) -> Self { - self = self.isthmus_activated(); + self = self.jovian_activated(); self.inner = self.inner.with_fork(OpHardfork::Interop, ForkCondition::Timestamp(0)); self } @@ -237,10 +248,6 @@ impl EthChainSpec for OpChainSpec { self.inner.chain() } - fn base_fee_params_at_block(&self, block_number: u64) -> BaseFeeParams { - self.inner.base_fee_params_at_block(block_number) - } - fn base_fee_params_at_timestamp(&self, timestamp: u64) -> BaseFeeParams { self.inner.base_fee_params_at_timestamp(timestamp) } @@ -289,6 +296,21 @@ impl EthChainSpec for OpChainSpec { fn final_paris_total_difficulty(&self) -> Option { self.inner.final_paris_total_difficulty() } + + fn next_block_base_fee(&self, parent: &Header, target_timestamp: u64) -> Option { + if self.is_jovian_active_at_timestamp(parent.timestamp()) { + compute_jovian_base_fee(self, parent, target_timestamp).ok() + } else if self.is_holocene_active_at_timestamp(parent.timestamp()) { + decode_holocene_base_fee(self, parent, target_timestamp).ok() + } else { + // For chains with Constant basefee (e.g., Mantle), keep basefee unchanged + // to avoid dynamic calculation that would cause basefee variation + match &self.inner.base_fee_params { + BaseFeeParamsKind::Constant(_) => parent.base_fee_per_gas(), + BaseFeeParamsKind::Variable(_) => self.inner.next_block_base_fee(parent, target_timestamp), + } + } + } } impl Hardforks for OpChainSpec { @@ -383,6 +405,7 @@ impl From for OpChainSpec { (EthereumHardfork::Shanghai.boxed(), mantle_genesis_info.mantle_skadi_time), (EthereumHardfork::Cancun.boxed(), mantle_genesis_info.mantle_skadi_time), (EthereumHardfork::Prague.boxed(), mantle_genesis_info.mantle_skadi_time), + (EthereumHardfork::Osaka.boxed(), mantle_genesis_info.mantle_limb_time), // OP (OpHardfork::Regolith.boxed(), genesis_info.regolith_time), (OpHardfork::Canyon.boxed(), genesis_info.canyon_time), @@ -391,9 +414,11 @@ impl From for OpChainSpec { (OpHardfork::Granite.boxed(), genesis_info.granite_time), (OpHardfork::Holocene.boxed(), genesis_info.holocene_time), (OpHardfork::Isthmus.boxed(), genesis_info.isthmus_time), + (OpHardfork::Jovian.boxed(), genesis_info.jovian_time), (OpHardfork::Interop.boxed(), genesis_info.interop_time), // Mantle (MantleHardfork::Skadi.boxed(), mantle_genesis_info.mantle_skadi_time), + (MantleHardfork::Limb.boxed(), mantle_genesis_info.mantle_limb_time), ]; let mut time_hardforks = time_hardfork_opts @@ -431,7 +456,7 @@ impl From for OpChainSpec { // We assume no OP network merges, and set the paris block and total difficulty to // zero paris_block_and_final_difficulty: Some((0, U256::ZERO)), - base_fee_params: optimism_genesis_info.base_fee_params, + base_fee_params: BaseFeeParamsKind::Constant(BaseFeeParams::ethereum()), ..Default::default() }, } @@ -461,33 +486,33 @@ impl OpGenesisInfo { .unwrap_or_default(), ..Default::default() }; - if let Some(optimism_base_fee_info) = &info.optimism_chain_info.base_fee_info { - if let (Some(elasticity), Some(denominator)) = ( + if let Some(optimism_base_fee_info) = &info.optimism_chain_info.base_fee_info && + let (Some(elasticity), Some(denominator)) = ( optimism_base_fee_info.eip1559_elasticity, optimism_base_fee_info.eip1559_denominator, - ) { - let base_fee_params = if let Some(canyon_denominator) = - optimism_base_fee_info.eip1559_denominator_canyon - { - BaseFeeParamsKind::Variable( - vec![ - ( - EthereumHardfork::London.boxed(), - BaseFeeParams::new(denominator as u128, elasticity as u128), - ), - ( - OpHardfork::Canyon.boxed(), - BaseFeeParams::new(canyon_denominator as u128, elasticity as u128), - ), - ] - .into(), - ) - } else { - BaseFeeParams::new(denominator as u128, elasticity as u128).into() - }; - - info.base_fee_params = base_fee_params; - } + ) + { + let base_fee_params = if let Some(canyon_denominator) = + optimism_base_fee_info.eip1559_denominator_canyon + { + BaseFeeParamsKind::Variable( + vec![ + ( + EthereumHardfork::London.boxed(), + BaseFeeParams::new(denominator as u128, elasticity as u128), + ), + ( + OpHardfork::Canyon.boxed(), + BaseFeeParams::new(canyon_denominator as u128, elasticity as u128), + ), + ] + .into(), + ) + } else { + BaseFeeParams::new(denominator as u128, elasticity as u128).into() + }; + + info.base_fee_params = base_fee_params; } info @@ -498,11 +523,11 @@ impl OpGenesisInfo { pub fn make_op_genesis_header(genesis: &Genesis, hardforks: &ChainHardforks) -> Header { let mut header = reth_chainspec::make_genesis_header(genesis, hardforks); - // If Isthmus is active, overwrite the withdrawals root with the storage root of predeploy + // If Skadi(Isthmus) is active, overwrite the withdrawals root with the storage root of predeploy // `L2ToL1MessagePasser.sol` - if hardforks.fork(MantleHardfork::Skadi).active_at_timestamp(header.timestamp) { - if let Some(predeploy) = genesis.alloc.get(&ADDRESS_L2_TO_L1_MESSAGE_PASSER) { - if let Some(storage) = &predeploy.storage { + if hardforks.fork(MantleHardfork::Skadi).active_at_timestamp(header.timestamp) + && let Some(predeploy) = genesis.alloc.get(&ADDRESS_L2_TO_L1_MESSAGE_PASSER) + && let Some(storage) = &predeploy.storage { header.withdrawals_root = Some(storage_root_unhashed(storage.iter().filter_map(|(k, v)| { if v.is_zero() { @@ -512,17 +537,17 @@ pub fn make_op_genesis_header(genesis: &Genesis, hardforks: &ChainHardforks) -> } }))); } - } - } - header } #[cfg(test)] mod tests { - use alloc::string::String; + use alloc::string::{String, ToString}; use alloy_genesis::{ChainConfig, Genesis}; - use alloy_primitives::b256; + use alloy_op_hardforks::{ + BASE_MAINNET_JOVIAN_TIMESTAMP, OP_MAINNET_JOVIAN_TIMESTAMP, OP_SEPOLIA_JOVIAN_TIMESTAMP, + }; + use alloy_primitives::{b256, hex}; use reth_chainspec::{test_fork_ids, BaseFeeParams, BaseFeeParamsKind}; use reth_ethereum_forks::{EthereumHardfork, ForkCondition, ForkHash, ForkId, Head}; use reth_optimism_forks::{OpHardfork, OpHardforks}; @@ -532,7 +557,7 @@ mod tests { #[test] fn test_storage_root_consistency() { use alloy_primitives::{B256, U256}; - use std::str::FromStr; + use core::str::FromStr; let k1 = B256::from_str("0x0000000000000000000000000000000000000000000000000000000000000001") @@ -614,7 +639,19 @@ mod tests { // Isthmus ( Head { number: 0, timestamp: 1746806401, ..Default::default() }, - ForkId { hash: ForkHash([0x86, 0x72, 0x8b, 0x4e]), next: 0 }, + ForkId { + hash: ForkHash([0x86, 0x72, 0x8b, 0x4e]), + next: BASE_MAINNET_JOVIAN_TIMESTAMP, + }, + ), + // Jovian + ( + Head { + number: 0, + timestamp: BASE_MAINNET_JOVIAN_TIMESTAMP, + ..Default::default() + }, + BASE_MAINNET.hardfork_fork_id(OpHardfork::Jovian).unwrap(), ), ], ); @@ -665,10 +702,22 @@ mod tests { Head { number: 0, timestamp: 1732633200, ..Default::default() }, ForkId { hash: ForkHash([0x4a, 0x1c, 0x79, 0x2e]), next: 1744905600 }, ), - // isthmus + // Isthmus ( Head { number: 0, timestamp: 1744905600, ..Default::default() }, - ForkId { hash: ForkHash([0x6c, 0x62, 0x5e, 0xe1]), next: 0 }, + ForkId { + hash: ForkHash([0x6c, 0x62, 0x5e, 0xe1]), + next: OP_SEPOLIA_JOVIAN_TIMESTAMP, + }, + ), + // Jovian + ( + Head { + number: 0, + timestamp: OP_SEPOLIA_JOVIAN_TIMESTAMP, + ..Default::default() + }, + OP_SEPOLIA.hardfork_fork_id(OpHardfork::Jovian).unwrap(), ), ], ); @@ -732,7 +781,19 @@ mod tests { // Isthmus ( Head { number: 105235063, timestamp: 1746806401, ..Default::default() }, - ForkId { hash: ForkHash([0x37, 0xbe, 0x75, 0x8f]), next: 0 }, + ForkId { + hash: ForkHash([0x37, 0xbe, 0x75, 0x8f]), + next: OP_MAINNET_JOVIAN_TIMESTAMP, + }, + ), + // Jovian + ( + Head { + number: 105235063, + timestamp: OP_MAINNET_JOVIAN_TIMESTAMP, + ..Default::default() + }, + OP_MAINNET.hardfork_fork_id(OpHardfork::Jovian).unwrap(), ), ], ); @@ -783,10 +844,22 @@ mod tests { Head { number: 0, timestamp: 1732633200, ..Default::default() }, ForkId { hash: ForkHash([0x8b, 0x5e, 0x76, 0x29]), next: 1744905600 }, ), - // isthmus + // Isthmus ( Head { number: 0, timestamp: 1744905600, ..Default::default() }, - ForkId { hash: ForkHash([0x06, 0x0a, 0x4d, 0x1d]), next: 0 }, + ForkId { + hash: ForkHash([0x06, 0x0a, 0x4d, 0x1d]), + next: OP_SEPOLIA_JOVIAN_TIMESTAMP, + }, /* TODO: update timestamp when Jovian is planned */ + ), + // Jovian + ( + Head { + number: 0, + timestamp: OP_SEPOLIA_JOVIAN_TIMESTAMP, + ..Default::default() + }, + BASE_SEPOLIA.hardfork_fork_id(OpHardfork::Jovian).unwrap(), ), ], ); @@ -799,9 +872,7 @@ mod tests { genesis.hash_slow(), b256!("0xf712aa9241cc24369b143cf6dce85f0902a9731e70d66818a3a5845b296c73dd") ); - let base_fee = genesis - .next_block_base_fee(BASE_MAINNET.base_fee_params_at_timestamp(genesis.timestamp)) - .unwrap(); + let base_fee = BASE_MAINNET.next_block_base_fee(genesis, genesis.timestamp).unwrap(); // assert_eq!(base_fee, 980000000); } @@ -813,9 +884,7 @@ mod tests { genesis.hash_slow(), b256!("0x0dcc9e089e30b90ddfc55be9a37dd15bc551aeee999d2e2b51414c54eaf934e4") ); - let base_fee = genesis - .next_block_base_fee(BASE_SEPOLIA.base_fee_params_at_timestamp(genesis.timestamp)) - .unwrap(); + let base_fee = BASE_SEPOLIA.next_block_base_fee(genesis, genesis.timestamp).unwrap(); // assert_eq!(base_fee, 980000000); } @@ -827,9 +896,7 @@ mod tests { genesis.hash_slow(), b256!("0x102de6ffb001480cc9b8b548fd05c34cd4f46ae4aa91759393db90ea0409887d") ); - let base_fee = genesis - .next_block_base_fee(OP_SEPOLIA.base_fee_params_at_timestamp(genesis.timestamp)) - .unwrap(); + let base_fee = OP_SEPOLIA.next_block_base_fee(genesis, genesis.timestamp).unwrap(); // assert_eq!(base_fee, 980000000); } @@ -837,7 +904,7 @@ mod tests { #[test] fn latest_base_mainnet_fork_id() { assert_eq!( - ForkId { hash: ForkHash([0x86, 0x72, 0x8b, 0x4e]), next: 0 }, + ForkId { hash: ForkHash(hex!("1cfeafc9")), next: 0 }, BASE_MAINNET.latest_fork_id() ) } @@ -846,7 +913,7 @@ mod tests { fn latest_base_mainnet_fork_id_with_builder() { let base_mainnet = OpChainSpecBuilder::base_mainnet().build(); assert_eq!( - ForkId { hash: ForkHash([0x86, 0x72, 0x8b, 0x4e]), next: 0 }, + ForkId { hash: ForkHash(hex!("1cfeafc9")), next: 0 }, base_mainnet.latest_fork_id() ) } @@ -869,6 +936,7 @@ mod tests { "fjordTime": 50, "graniteTime": 51, "holoceneTime": 52, + "isthmusTime": 53, "optimism": { "eip1559Elasticity": 60, "eip1559Denominator": 70 @@ -892,6 +960,8 @@ mod tests { assert_eq!(actual_granite_timestamp, Some(serde_json::Value::from(51)).as_ref()); let actual_holocene_timestamp = genesis.config.extra_fields.get("holoceneTime"); assert_eq!(actual_holocene_timestamp, Some(serde_json::Value::from(52)).as_ref()); + let actual_isthmus_timestamp = genesis.config.extra_fields.get("isthmusTime"); + assert_eq!(actual_isthmus_timestamp, Some(serde_json::Value::from(53)).as_ref()); let optimism_object = genesis.config.extra_fields.get("optimism").unwrap(); assert_eq!( @@ -938,6 +1008,7 @@ mod tests { "fjordTime": 50, "graniteTime": 51, "holoceneTime": 52, + "isthmusTime": 53, "optimism": { "eip1559Elasticity": 60, "eip1559Denominator": 70, @@ -962,6 +1033,8 @@ mod tests { assert_eq!(actual_granite_timestamp, Some(serde_json::Value::from(51)).as_ref()); let actual_holocene_timestamp = genesis.config.extra_fields.get("holoceneTime"); assert_eq!(actual_holocene_timestamp, Some(serde_json::Value::from(52)).as_ref()); + let actual_isthmus_timestamp = genesis.config.extra_fields.get("isthmusTime"); + assert_eq!(actual_isthmus_timestamp, Some(serde_json::Value::from(53)).as_ref()); let optimism_object = genesis.config.extra_fields.get("optimism").unwrap(); assert_eq!( @@ -1115,6 +1188,7 @@ mod tests { (String::from("graniteTime"), 0.into()), (String::from("holoceneTime"), 0.into()), (String::from("isthmusTime"), 0.into()), + (String::from("jovianTime"), 0.into()), ] .into_iter() .collect(), @@ -1152,6 +1226,7 @@ mod tests { OpHardfork::Holocene.boxed(), EthereumHardfork::Prague.boxed(), OpHardfork::Isthmus.boxed(), + OpHardfork::Jovian.boxed(), // OpHardfork::Interop.boxed(), ]; diff --git a/crates/optimism/chainspec/src/mantle.rs b/crates/optimism/chainspec/src/mantle.rs index 46a0b49072f..8277cc43979 100644 --- a/crates/optimism/chainspec/src/mantle.rs +++ b/crates/optimism/chainspec/src/mantle.rs @@ -32,6 +32,8 @@ impl TryFrom<&OtherFields> for MantleChainInfo { pub(crate) struct MantleGenesisInfo { /// Mantle Skadi upgrade timestamp pub mantle_skadi_time: Option, + /// Mantle Limb upgrade timestamp + pub mantle_limb_time: Option, } impl MantleGenesisInfo { @@ -59,6 +61,10 @@ impl TryFrom<&OtherFields> for MantleGenesisInfo { .get_deserialized("mantleSkadiTime") .transpose()?; - Ok(Self { mantle_skadi_time }) + let mantle_limb_time = others + .get_deserialized("mantleLimbTime") + .transpose()?; + + Ok(Self { mantle_skadi_time, mantle_limb_time }) } } diff --git a/crates/optimism/chainspec/src/superchain/chain_metadata.rs b/crates/optimism/chainspec/src/superchain/chain_metadata.rs index 90330817b70..bf6228c099a 100644 --- a/crates/optimism/chainspec/src/superchain/chain_metadata.rs +++ b/crates/optimism/chainspec/src/superchain/chain_metadata.rs @@ -26,6 +26,7 @@ pub(crate) struct HardforkConfig { pub granite_time: Option, pub holocene_time: Option, pub isthmus_time: Option, + pub jovian_time: Option, } #[derive(Clone, Debug, Deserialize)] @@ -58,6 +59,8 @@ pub(crate) struct ChainConfigExtraFields { #[serde(skip_serializing_if = "Option::is_none")] pub isthmus_time: Option, #[serde(skip_serializing_if = "Option::is_none")] + pub jovian_time: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub optimism: Option, } @@ -137,6 +140,7 @@ pub(crate) fn to_genesis_chain_config(chain_config: &ChainMetadata) -> ChainConf granite_time: chain_config.hardforks.granite_time, holocene_time: chain_config.hardforks.holocene_time, isthmus_time: chain_config.hardforks.isthmus_time, + jovian_time: chain_config.hardforks.jovian_time, optimism: chain_config.optimism.as_ref().map(|o| o.into()), }; res.extra_fields = @@ -158,7 +162,8 @@ mod tests { "ecotone_time": 1710374401, "fjord_time": 1720627201, "granite_time": 1726070401, - "holocene_time": 1736445601 + "holocene_time": 1736445601, + "isthmus_time": 1746806401 }, "optimism": { "eip1559_elasticity": 6, @@ -179,6 +184,7 @@ mod tests { assert_eq!(config.hardforks.fjord_time, Some(1720627201)); assert_eq!(config.hardforks.granite_time, Some(1726070401)); assert_eq!(config.hardforks.holocene_time, Some(1736445601)); + assert_eq!(config.hardforks.isthmus_time, Some(1746806401)); // optimism assert_eq!(config.optimism.as_ref().unwrap().eip1559_elasticity, 6); assert_eq!(config.optimism.as_ref().unwrap().eip1559_denominator, 50); @@ -196,7 +202,8 @@ mod tests { fjord_time: Some(1720627201), granite_time: Some(1726070401), holocene_time: Some(1736445601), - isthmus_time: None, + isthmus_time: Some(1746806401), + jovian_time: None, optimism: Option::from(ChainConfigExtraFieldsOptimism { eip1559_elasticity: 6, eip1559_denominator: 50, @@ -212,7 +219,8 @@ mod tests { assert_eq!(value.get("fjordTime").unwrap(), 1720627201); assert_eq!(value.get("graniteTime").unwrap(), 1726070401); assert_eq!(value.get("holoceneTime").unwrap(), 1736445601); - assert_eq!(value.get("isthmusTime"), None); + assert_eq!(value.get("isthmusTime").unwrap(), 1746806401); + assert_eq!(value.get("jovianTime"), None); let optimism = value.get("optimism").unwrap(); assert_eq!(optimism.get("eip1559Elasticity").unwrap(), 6); assert_eq!(optimism.get("eip1559Denominator").unwrap(), 50); @@ -242,7 +250,7 @@ mod tests { assert_eq!(chain_config.merge_netsplit_block, Some(0)); assert_eq!(chain_config.shanghai_time, Some(1704992401)); assert_eq!(chain_config.cancun_time, Some(1710374401)); - assert_eq!(chain_config.prague_time, None); + assert_eq!(chain_config.prague_time, Some(1746806401)); assert_eq!(chain_config.osaka_time, None); assert_eq!(chain_config.terminal_total_difficulty, Some(U256::ZERO)); assert!(chain_config.terminal_total_difficulty_passed); @@ -256,7 +264,8 @@ mod tests { assert_eq!(chain_config.extra_fields.get("fjordTime").unwrap(), 1720627201); assert_eq!(chain_config.extra_fields.get("graniteTime").unwrap(), 1726070401); assert_eq!(chain_config.extra_fields.get("holoceneTime").unwrap(), 1736445601); - assert_eq!(chain_config.extra_fields.get("isthmusTime"), None); + assert_eq!(chain_config.extra_fields.get("isthmusTime").unwrap(), 1746806401); + assert_eq!(chain_config.extra_fields.get("jovianTime"), None); let optimism = chain_config.extra_fields.get("optimism").unwrap(); assert_eq!(optimism.get("eip1559Elasticity").unwrap(), 6); assert_eq!(optimism.get("eip1559Denominator").unwrap(), 50); @@ -274,7 +283,8 @@ mod tests { "ecotone_time": 1710374401, "fjord_time": 1720627201, "granite_time": 1726070401, - "holocene_time": 1736445601 + "holocene_time": 1736445601, + "isthmus_time": 1746806401 }, "optimism": { "eip1559_elasticity": 6, @@ -289,7 +299,7 @@ mod tests { assert_eq!(chain_config.chain_id, 10); assert_eq!(chain_config.shanghai_time, Some(1704992401)); assert_eq!(chain_config.cancun_time, Some(1710374401)); - assert_eq!(chain_config.prague_time, None); + assert_eq!(chain_config.prague_time, Some(1746806401)); assert_eq!(chain_config.berlin_block, Some(3950000)); assert_eq!(chain_config.london_block, Some(105235063)); assert_eq!(chain_config.arrow_glacier_block, Some(105235063)); @@ -303,7 +313,9 @@ mod tests { assert_eq!(chain_config.extra_fields.get("fjordTime").unwrap(), 1720627201); assert_eq!(chain_config.extra_fields.get("graniteTime").unwrap(), 1726070401); assert_eq!(chain_config.extra_fields.get("holoceneTime").unwrap(), 1736445601); - assert_eq!(chain_config.extra_fields.get("isthmusTime"), None); + assert_eq!(chain_config.extra_fields.get("isthmusTime").unwrap(), 1746806401); + assert_eq!(chain_config.extra_fields.get("jovianTime"), None); + let optimism = chain_config.extra_fields.get("optimism").unwrap(); assert_eq!(optimism.get("eip1559Elasticity").unwrap(), 6); assert_eq!(optimism.get("eip1559Denominator").unwrap(), 50); diff --git a/crates/optimism/chainspec/src/superchain/chain_spec_macro.rs b/crates/optimism/chainspec/src/superchain/chain_spec_macro.rs index 91a886d9b90..2ab75beba99 100644 --- a/crates/optimism/chainspec/src/superchain/chain_spec_macro.rs +++ b/crates/optimism/chainspec/src/superchain/chain_spec_macro.rs @@ -73,7 +73,6 @@ macro_rules! create_superchain_specs { /// All supported superchains, including both older and newer naming, /// for backwards compatibility pub const SUPPORTED_CHAINS: &'static [&'static str] = &[ - "dev", "optimism", "optimism_sepolia", "optimism-sepolia", @@ -83,6 +82,7 @@ macro_rules! create_superchain_specs { $( $crate::key_for!($name, $env), )+ + "dev", ]; /// Parses the chain into an [`$crate::OpChainSpec`], if recognized. diff --git a/crates/optimism/chainspec/src/superchain/chain_specs.rs b/crates/optimism/chainspec/src/superchain/chain_specs.rs index 2dd048771ad..8a794221ea6 100644 --- a/crates/optimism/chainspec/src/superchain/chain_specs.rs +++ b/crates/optimism/chainspec/src/superchain/chain_specs.rs @@ -3,16 +3,20 @@ use crate::create_superchain_specs; create_superchain_specs!( ("arena-z", "mainnet"), - ("arena-z-testnet", "sepolia"), + ("arena-z", "sepolia"), ("automata", "mainnet"), ("base-devnet-0", "sepolia-dev-0"), ("bob", "mainnet"), ("boba", "sepolia"), + ("boba", "mainnet"), + ("camp", "sepolia"), + ("celo", "mainnet"), ("creator-chain-testnet", "sepolia"), ("cyber", "mainnet"), ("cyber", "sepolia"), ("ethernity", "mainnet"), ("ethernity", "sepolia"), + ("fraxtal", "mainnet"), ("funki", "mainnet"), ("funki", "sepolia"), ("hashkeychain", "mainnet"), @@ -28,15 +32,20 @@ create_superchain_specs!( ("mode", "sepolia"), ("oplabs-devnet-0", "sepolia-dev-0"), ("orderly", "mainnet"), + ("ozean", "sepolia"), ("pivotal", "sepolia"), ("polynomial", "mainnet"), ("race", "mainnet"), ("race", "sepolia"), + ("radius_testnet", "sepolia"), ("redstone", "mainnet"), + ("rehearsal-0-bn-0", "rehearsal-0-bn"), + ("rehearsal-0-bn-1", "rehearsal-0-bn"), ("settlus-mainnet", "mainnet"), ("settlus-sepolia", "sepolia"), ("shape", "mainnet"), ("shape", "sepolia"), + ("silent-data-mainnet", "mainnet"), ("snax", "mainnet"), ("soneium", "mainnet"), ("soneium-minato", "sepolia"), diff --git a/crates/optimism/chainspec/src/superchain/configs.rs b/crates/optimism/chainspec/src/superchain/configs.rs index 428f197a049..bb1929646a0 100644 --- a/crates/optimism/chainspec/src/superchain/configs.rs +++ b/crates/optimism/chainspec/src/superchain/configs.rs @@ -8,8 +8,8 @@ use alloy_genesis::Genesis; use miniz_oxide::inflate::decompress_to_vec_zlib_with_limit; use tar_no_std::{CorruptDataError, TarArchiveRef}; -/// A genesis file can be up to 10MiB. This is a reasonable limit for the genesis file size. -const MAX_GENESIS_SIZE: usize = 16 * 1024 * 1024; // 16MiB +/// A genesis file can be up to 100MiB. This is a reasonable limit for the genesis file size. +const MAX_GENESIS_SIZE: usize = 100 * 1024 * 1024; // 100MiB /// The tar file contains the chain configs and genesis files for all chains. const SUPER_CHAIN_CONFIGS_TAR_BYTES: &[u8] = include_bytes!("../../res/superchain-configs.tar"); @@ -87,7 +87,17 @@ fn read_file( #[cfg(test)] mod tests { use super::*; - use crate::superchain::Superchain; + use crate::{generated_chain_value_parser, superchain::Superchain, SUPPORTED_CHAINS}; + use alloy_chains::NamedChain; + use alloy_op_hardforks::{ + OpHardfork, BASE_MAINNET_CANYON_TIMESTAMP, BASE_MAINNET_ECOTONE_TIMESTAMP, + BASE_MAINNET_ISTHMUS_TIMESTAMP, BASE_MAINNET_JOVIAN_TIMESTAMP, + BASE_SEPOLIA_CANYON_TIMESTAMP, BASE_SEPOLIA_ECOTONE_TIMESTAMP, + BASE_SEPOLIA_ISTHMUS_TIMESTAMP, BASE_SEPOLIA_JOVIAN_TIMESTAMP, OP_MAINNET_CANYON_TIMESTAMP, + OP_MAINNET_ECOTONE_TIMESTAMP, OP_MAINNET_ISTHMUS_TIMESTAMP, OP_MAINNET_JOVIAN_TIMESTAMP, + OP_SEPOLIA_CANYON_TIMESTAMP, OP_SEPOLIA_ECOTONE_TIMESTAMP, OP_SEPOLIA_ISTHMUS_TIMESTAMP, + OP_SEPOLIA_JOVIAN_TIMESTAMP, + }; use reth_optimism_primitives::ADDRESS_L2_TO_L1_MESSAGE_PASSER; use tar_no_std::TarArchiveRef; @@ -150,4 +160,139 @@ mod tests { ); } } + + #[test] + fn test_hardfork_timestamps() { + for &chain in SUPPORTED_CHAINS { + let metadata = generated_chain_value_parser(chain).unwrap(); + + match metadata.chain().named() { + Some(NamedChain::Optimism) => { + assert_eq!( + metadata.hardforks.get(OpHardfork::Jovian).unwrap().as_timestamp().unwrap(), + OP_MAINNET_JOVIAN_TIMESTAMP + ); + + assert_eq!( + metadata + .hardforks + .get(OpHardfork::Isthmus) + .unwrap() + .as_timestamp() + .unwrap(), + OP_MAINNET_ISTHMUS_TIMESTAMP + ); + + assert_eq!( + metadata.hardforks.get(OpHardfork::Canyon).unwrap().as_timestamp().unwrap(), + OP_MAINNET_CANYON_TIMESTAMP + ); + + assert_eq!( + metadata + .hardforks + .get(OpHardfork::Ecotone) + .unwrap() + .as_timestamp() + .unwrap(), + OP_MAINNET_ECOTONE_TIMESTAMP + ); + } + Some(NamedChain::OptimismSepolia) => { + assert_eq!( + metadata.hardforks.get(OpHardfork::Jovian).unwrap().as_timestamp().unwrap(), + OP_SEPOLIA_JOVIAN_TIMESTAMP + ); + + assert_eq!( + metadata + .hardforks + .get(OpHardfork::Isthmus) + .unwrap() + .as_timestamp() + .unwrap(), + OP_SEPOLIA_ISTHMUS_TIMESTAMP + ); + + assert_eq!( + metadata.hardforks.get(OpHardfork::Canyon).unwrap().as_timestamp().unwrap(), + OP_SEPOLIA_CANYON_TIMESTAMP + ); + + assert_eq!( + metadata + .hardforks + .get(OpHardfork::Ecotone) + .unwrap() + .as_timestamp() + .unwrap(), + OP_SEPOLIA_ECOTONE_TIMESTAMP + ); + } + Some(NamedChain::Base) => { + assert_eq!( + metadata.hardforks.get(OpHardfork::Jovian).unwrap().as_timestamp().unwrap(), + BASE_MAINNET_JOVIAN_TIMESTAMP + ); + + assert_eq!( + metadata + .hardforks + .get(OpHardfork::Isthmus) + .unwrap() + .as_timestamp() + .unwrap(), + BASE_MAINNET_ISTHMUS_TIMESTAMP + ); + + assert_eq!( + metadata.hardforks.get(OpHardfork::Canyon).unwrap().as_timestamp().unwrap(), + BASE_MAINNET_CANYON_TIMESTAMP + ); + + assert_eq!( + metadata + .hardforks + .get(OpHardfork::Ecotone) + .unwrap() + .as_timestamp() + .unwrap(), + BASE_MAINNET_ECOTONE_TIMESTAMP + ); + } + Some(NamedChain::BaseSepolia) => { + assert_eq!( + metadata.hardforks.get(OpHardfork::Jovian).unwrap().as_timestamp().unwrap(), + BASE_SEPOLIA_JOVIAN_TIMESTAMP + ); + + assert_eq!( + metadata + .hardforks + .get(OpHardfork::Isthmus) + .unwrap() + .as_timestamp() + .unwrap(), + BASE_SEPOLIA_ISTHMUS_TIMESTAMP + ); + + assert_eq!( + metadata.hardforks.get(OpHardfork::Canyon).unwrap().as_timestamp().unwrap(), + BASE_SEPOLIA_CANYON_TIMESTAMP + ); + + assert_eq!( + metadata + .hardforks + .get(OpHardfork::Ecotone) + .unwrap() + .as_timestamp() + .unwrap(), + BASE_SEPOLIA_ECOTONE_TIMESTAMP + ); + } + _ => {} + } + } + } } diff --git a/crates/optimism/cli/Cargo.toml b/crates/optimism/cli/Cargo.toml index 1661c3be476..aee7566de22 100644 --- a/crates/optimism/cli/Cargo.toml +++ b/crates/optimism/cli/Cargo.toml @@ -12,8 +12,10 @@ workspace = true [dependencies] reth-static-file-types = { workspace = true, features = ["clap"] } +reth-cli.workspace = true reth-cli-commands.workspace = true reth-consensus.workspace = true +reth-rpc-server-types.workspace = true reth-primitives-traits.workspace = true reth-db = { workspace = true, features = ["mdbx", "op"] } reth-db-api.workspace = true @@ -39,10 +41,10 @@ reth-optimism-consensus.workspace = true reth-chainspec.workspace = true reth-node-events.workspace = true reth-optimism-evm.workspace = true -reth-cli.workspace = true reth-cli-runner.workspace = true reth-node-builder = { workspace = true, features = ["op"] } reth-tracing.workspace = true +reth-tracing-otlp.workspace = true # eth alloy-eips.workspace = true @@ -54,6 +56,7 @@ alloy-rlp.workspace = true futures-util.workspace = true derive_more.workspace = true serde.workspace = true +url.workspace = true clap = { workspace = true, features = ["derive", "env"] } tokio = { workspace = true, features = ["sync", "macros", "time", "rt-multi-thread"] } @@ -68,13 +71,16 @@ op-alloy-consensus.workspace = true [dev-dependencies] tempfile.workspace = true reth-stages = { workspace = true, features = ["test-utils"] } -reth-db-common.workspace = true -reth-cli-commands.workspace = true [build-dependencies] reth-optimism-chainspec = { workspace = true, features = ["std", "superchain-configs"] } [features] +default = [] + +# Opentelemtry feature to activate metrics export +otlp = ["reth-tracing/otlp", "reth-node-core/otlp"] + asm-keccak = [ "alloy-primitives/asm-keccak", "reth-node-core/asm-keccak", @@ -101,4 +107,5 @@ serde = [ "reth-optimism-primitives/serde", "reth-primitives-traits/serde", "reth-optimism-chainspec/serde", + "url/serde", ] diff --git a/crates/optimism/cli/src/app.rs b/crates/optimism/cli/src/app.rs new file mode 100644 index 00000000000..8567c2b7e5a --- /dev/null +++ b/crates/optimism/cli/src/app.rs @@ -0,0 +1,168 @@ +use crate::{Cli, Commands}; +use eyre::{eyre, Result}; +use reth_cli::chainspec::ChainSpecParser; +use reth_cli_commands::launcher::Launcher; +use reth_cli_runner::CliRunner; +use reth_node_metrics::recorder::install_prometheus_recorder; +use reth_optimism_chainspec::OpChainSpec; +use reth_optimism_consensus::OpBeaconConsensus; +use reth_optimism_node::{OpExecutorProvider, OpNode}; +use reth_rpc_server_types::RpcModuleValidator; +use reth_tracing::{FileWorkerGuard, Layers}; +use reth_tracing_otlp::OtlpProtocol; +use std::{fmt, sync::Arc}; +use tracing::info; +use url::Url; + +/// A wrapper around a parsed CLI that handles command execution. +#[derive(Debug)] +pub struct CliApp { + cli: Cli, + runner: Option, + layers: Option, + guard: Option, +} + +impl CliApp +where + C: ChainSpecParser, + Ext: clap::Args + fmt::Debug, + Rpc: RpcModuleValidator, +{ + pub(crate) fn new(cli: Cli) -> Self { + Self { cli, runner: None, layers: Some(Layers::new()), guard: None } + } + + /// Sets the runner for the CLI commander. + /// + /// This replaces any existing runner with the provided one. + pub fn set_runner(&mut self, runner: CliRunner) { + self.runner = Some(runner); + } + + /// Access to tracing layers. + /// + /// Returns a mutable reference to the tracing layers, or error + /// if tracing initialized and layers have detached already. + pub fn access_tracing_layers(&mut self) -> Result<&mut Layers> { + self.layers.as_mut().ok_or_else(|| eyre!("Tracing already initialized")) + } + + /// Execute the configured cli command. + /// + /// This accepts a closure that is used to launch the node via the + /// [`NodeCommand`](reth_cli_commands::node::NodeCommand). + pub fn run(mut self, launcher: impl Launcher) -> Result<()> { + let runner = match self.runner.take() { + Some(runner) => runner, + None => CliRunner::try_default_runtime()?, + }; + + // add network name to logs dir + // Add network name if available to the logs dir + if let Some(chain_spec) = self.cli.command.chain_spec() { + self.cli.logs.log_file_directory = + self.cli.logs.log_file_directory.join(chain_spec.chain.to_string()); + } + + self.init_tracing(&runner)?; + + // Install the prometheus recorder to be sure to record all metrics + let _ = install_prometheus_recorder(); + + let components = |spec: Arc| { + (OpExecutorProvider::optimism(spec.clone()), Arc::new(OpBeaconConsensus::new(spec))) + }; + + match self.cli.command { + Commands::Node(command) => { + // Validate RPC modules using the configured validator + if let Some(http_api) = &command.rpc.http_api { + Rpc::validate_selection(http_api, "http.api").map_err(|e| eyre!("{e}"))?; + } + if let Some(ws_api) = &command.rpc.ws_api { + Rpc::validate_selection(ws_api, "ws.api").map_err(|e| eyre!("{e}"))?; + } + + runner.run_command_until_exit(|ctx| command.execute(ctx, launcher)) + } + Commands::Init(command) => { + runner.run_blocking_until_ctrl_c(command.execute::()) + } + Commands::InitState(command) => { + runner.run_blocking_until_ctrl_c(command.execute::()) + } + Commands::ImportOp(command) => { + runner.run_blocking_until_ctrl_c(command.execute::()) + } + Commands::ImportReceiptsOp(command) => { + runner.run_blocking_until_ctrl_c(command.execute::()) + } + Commands::DumpGenesis(command) => runner.run_blocking_until_ctrl_c(command.execute()), + Commands::Db(command) => runner.run_blocking_until_ctrl_c(command.execute::()), + Commands::Stage(command) => { + runner.run_command_until_exit(|ctx| command.execute::(ctx, components)) + } + Commands::P2P(command) => runner.run_until_ctrl_c(command.execute::()), + Commands::Config(command) => runner.run_until_ctrl_c(command.execute()), + Commands::Prune(command) => runner.run_until_ctrl_c(command.execute::()), + #[cfg(feature = "dev")] + Commands::TestVectors(command) => runner.run_until_ctrl_c(command.execute()), + Commands::ReExecute(command) => { + runner.run_until_ctrl_c(command.execute::(components)) + } + } + } + + /// Initializes tracing with the configured options. + /// + /// If file logging is enabled, this function stores guard to the struct. + /// For gRPC OTLP, it requires tokio runtime context. + pub fn init_tracing(&mut self, runner: &CliRunner) -> Result<()> { + if self.guard.is_none() { + let mut layers = self.layers.take().unwrap_or_default(); + + #[cfg(feature = "otlp")] + { + self.cli.traces.validate()?; + if let Some(endpoint) = &self.cli.traces.otlp { + info!(target: "reth::cli", "Starting OTLP tracing export to {:?}", endpoint); + self.init_otlp_export(&mut layers, endpoint, runner)?; + } + } + + self.guard = self.cli.logs.init_tracing_with_layers(layers)?; + info!(target: "reth::cli", "Initialized tracing, debug log directory: {}", self.cli.logs.log_file_directory); + } + Ok(()) + } + + /// Initialize OTLP tracing export based on protocol type. + /// + /// For gRPC, `block_on` is required because tonic's channel initialization needs + /// a tokio runtime context, even though `with_span_layer` itself is not async. + #[cfg(feature = "otlp")] + fn init_otlp_export( + &self, + layers: &mut Layers, + endpoint: &Url, + runner: &CliRunner, + ) -> Result<()> { + let endpoint = endpoint.clone(); + let protocol = self.cli.traces.protocol; + let level_filter = self.cli.traces.otlp_filter.clone(); + + match protocol { + OtlpProtocol::Grpc => { + runner.block_on(async { + layers.with_span_layer("reth".to_string(), endpoint, level_filter, protocol) + })?; + } + OtlpProtocol::Http => { + layers.with_span_layer("reth".to_string(), endpoint, level_filter, protocol)?; + } + } + + Ok(()) + } +} diff --git a/crates/optimism/cli/src/commands/import.rs b/crates/optimism/cli/src/commands/import.rs index 52b6a27c9d5..7adae4f2ef7 100644 --- a/crates/optimism/cli/src/commands/import.rs +++ b/crates/optimism/cli/src/commands/import.rs @@ -10,7 +10,7 @@ use reth_consensus::noop::NoopConsensus; use reth_db_api::{tables, transaction::DbTx}; use reth_downloaders::file_client::{ChunkedFileReader, DEFAULT_BYTE_LEN_CHUNK_CHAIN_FILE}; use reth_node_builder::BlockTy; -use reth_node_core::version::SHORT_VERSION; +use reth_node_core::version::version_metadata; use reth_optimism_chainspec::OpChainSpec; use reth_optimism_evm::OpExecutorProvider; use reth_optimism_primitives::{bedrock::is_dup_tx, OpPrimitives}; @@ -44,7 +44,7 @@ impl> ImportOpCommand { pub async fn execute>( self, ) -> eyre::Result<()> { - info!(target: "reth::cli", "reth {} starting", SHORT_VERSION); + info!(target: "reth::cli", "reth {} starting", version_metadata().short_version); info!(target: "reth::cli", "Disabled stages requiring state, since cannot execute OVM state changes" @@ -71,6 +71,7 @@ impl> ImportOpCommand { .sealed_header(provider_factory.last_block_number()?)? .expect("should have genesis"); + while let Some(mut file_client) = reader.next_chunk::>(consensus.clone(), Some(sealed_header)).await? { diff --git a/crates/optimism/cli/src/commands/import_receipts.rs b/crates/optimism/cli/src/commands/import_receipts.rs index 38503bc2d33..db25afe9099 100644 --- a/crates/optimism/cli/src/commands/import_receipts.rs +++ b/crates/optimism/cli/src/commands/import_receipts.rs @@ -12,14 +12,14 @@ use reth_downloaders::{ }; use reth_execution_types::ExecutionOutcome; use reth_node_builder::ReceiptTy; -use reth_node_core::version::SHORT_VERSION; +use reth_node_core::version::version_metadata; use reth_optimism_chainspec::OpChainSpec; use reth_optimism_primitives::{bedrock::is_dup_tx, OpPrimitives, OpReceipt}; use reth_primitives_traits::NodePrimitives; use reth_provider::{ - providers::ProviderNodeTypes, writer::UnifiedStorageWriter, DatabaseProviderFactory, - OriginalValuesKnown, ProviderFactory, StageCheckpointReader, StageCheckpointWriter, - StateWriter, StaticFileProviderFactory, StatsReader, StorageLocation, + providers::ProviderNodeTypes, DBProvider, DatabaseProviderFactory, OriginalValuesKnown, + ProviderFactory, StageCheckpointReader, StageCheckpointWriter, StateWriter, + StaticFileProviderFactory, StatsReader, }; use reth_stages::{StageCheckpoint, StageId}; use reth_static_file_types::StaticFileSegment; @@ -52,7 +52,7 @@ impl> ImportReceiptsOpCommand { pub async fn execute>( self, ) -> eyre::Result<()> { - info!(target: "reth::cli", "reth {} starting", SHORT_VERSION); + info!(target: "reth::cli", "reth {} starting", version_metadata().short_version); debug!(target: "reth::cli", chunk_byte_len=self.chunk_len.unwrap_or(DEFAULT_BYTE_LEN_CHUNK_CHAIN_FILE), @@ -141,11 +141,10 @@ where // Ensure that receipts hasn't been initialized apart from `init_genesis`. if let Some(num_receipts) = - static_file_provider.get_highest_static_file_tx(StaticFileSegment::Receipts) + static_file_provider.get_highest_static_file_tx(StaticFileSegment::Receipts) && + num_receipts > 0 { - if num_receipts > 0 { - eyre::bail!("Expected no receipts in storage, but found {num_receipts}."); - } + eyre::bail!("Expected no receipts in storage, but found {num_receipts}."); } match static_file_provider.get_highest_static_file_block(StaticFileSegment::Receipts) { Some(receipts_block) => { @@ -225,18 +224,11 @@ where // Update total_receipts after all filtering total_receipts += receipts.iter().map(|v| v.len()).sum::(); - // We're reusing receipt writing code internal to - // `UnifiedStorageWriter::append_receipts_from_blocks`, so we just use a default empty - // `BundleState`. let execution_outcome = ExecutionOutcome::new(Default::default(), receipts, first_block, Default::default()); // finally, write the receipts - provider.write_state( - &execution_outcome, - OriginalValuesKnown::Yes, - StorageLocation::StaticFiles, - )?; + provider.write_state(&execution_outcome, OriginalValuesKnown::Yes)?; } // Only commit if we have imported as many receipts as the number of transactions. @@ -261,7 +253,7 @@ where provider .save_stage_checkpoint(StageId::Execution, StageCheckpoint::new(highest_block_receipts))?; - UnifiedStorageWriter::commit(provider)?; + provider.commit()?; Ok(ImportReceiptsResult { total_decoded_receipts, total_filtered_out_dup_txns }) } @@ -309,13 +301,13 @@ mod test { f.flush().await.unwrap(); f.seek(SeekFrom::Start(0)).await.unwrap(); - let reader = - ChunkedFileReader::from_file(f, DEFAULT_BYTE_LEN_CHUNK_CHAIN_FILE).await.unwrap(); + let reader = ChunkedFileReader::from_file(f, DEFAULT_BYTE_LEN_CHUNK_CHAIN_FILE, false) + .await + .unwrap(); let db = TestStageDB::default(); init_genesis(&db.factory).unwrap(); - // todo: where does import command init receipts ? probably somewhere in pipeline let provider_factory = create_test_provider_factory_with_node_types::(OP_MAINNET.clone()); let ImportReceiptsResult { total_decoded_receipts, total_filtered_out_dup_txns } = diff --git a/crates/optimism/cli/src/commands/init_state.rs b/crates/optimism/cli/src/commands/init_state.rs index 6816e283f77..69858fe1bdb 100644 --- a/crates/optimism/cli/src/commands/init_state.rs +++ b/crates/optimism/cli/src/commands/init_state.rs @@ -1,6 +1,5 @@ //! Command that initializes the node from a genesis file. -use alloy_primitives::{B256, U256}; use clap::Parser; use reth_cli::chainspec::ChainSpecParser; use reth_cli_commands::{ @@ -11,10 +10,11 @@ use reth_db_common::init::init_from_state_dump; use reth_optimism_chainspec::OpChainSpec; use reth_optimism_primitives::OpPrimitives; use reth_primitives_traits::SealedHeader; +use reth_provider::DBProvider; use reth_provider::{ - BlockNumReader, DatabaseProviderFactory, StaticFileProviderFactory, StaticFileWriter + BlockNumReader, DatabaseProviderFactory, StaticFileProviderFactory, StaticFileWriter, }; -use std::{io::BufReader, str::FromStr, sync::Arc}; +use std::{io::BufReader, sync::Arc}; use tracing::info; /// Initializes the database with the genesis block. @@ -23,12 +23,11 @@ pub struct InitStateCommandOp { #[command(flatten)] init_state: reth_cli_commands::init_state::InitStateCommand, - /// **Optimism Mainnet Only** - /// - /// Specifies whether to initialize the state without relying on OVM historical data. + /// Specifies whether to initialize the state without relying on OVM or EVM historical data. /// /// When enabled, and before inserting the state, it creates a dummy chain up to the last OVM - /// block (#105235062) (14GB / 90 seconds). It then, appends the Bedrock block. + /// block (#105235062) (14GB / 90 seconds). It then, appends the Bedrock block. This is + /// hardcoded for OP mainnet, for other OP chains you will need to pass in a header. /// /// - **Note**: **Do not** import receipts and blocks beforehand, or this will fail or be /// ignored. @@ -39,13 +38,33 @@ pub struct InitStateCommandOp { impl> InitStateCommandOp { /// Execute the `init` command pub async fn execute>( - self, + mut self, ) -> eyre::Result<()> { - info!(target: "reth::cli", "Reth init-state starting"); + // If using --without-ovm for OP mainnet, handle the special case with hardcoded Bedrock + // header. Otherwise delegate to the base InitStateCommand implementation. + if self.without_ovm { + if self.init_state.env.chain.is_optimism_mainnet() { + return self.execute_with_bedrock_header::(); + } - let Environment { config, provider_factory, .. } = - self.init_state.env.init::(AccessRights::RW)?; + // For non-mainnet OP chains with --without-ovm, use the base implementation + // by setting the without_evm flag + self.init_state.without_evm = true; + } + + self.init_state.execute::().await + } + /// Execute init-state with hardcoded Bedrock header for OP mainnet. + fn execute_with_bedrock_header< + N: CliNodeTypes, + >( + self, + ) -> eyre::Result<()> { + info!(target: "reth::cli", "Reth init-state starting for OP mainnet"); + let env = self.init_state.env.init::(AccessRights::RW)?; + + let Environment { config, provider_factory, .. } = env; let static_file_provider = provider_factory.static_file_provider(); let provider_rw = provider_factory.database_provider_rw()?; @@ -56,30 +75,22 @@ impl> InitStateCommandOp { .init_state .header .ok_or_else(|| eyre::eyre!("Header file must be provided"))?; - let header = without_evm::read_header_from_file(header)?; + let header: alloy_consensus::Header = without_evm::read_header_from_file(&header)?; let header_hash = self .init_state .header_hash - .or_else(|| Some(header.hash_slow().to_string())) + .or_else(|| Some(header.hash_slow())) .ok_or_else(|| eyre::eyre!("Header hash must be provided"))?; - let header_hash = B256::from_str(&header_hash)?; - - let total_difficulty = self - .init_state - .total_difficulty - .or_else(|| Some(header.difficulty.to_string())) - .ok_or_else(|| eyre::eyre!("Total difficulty must be provided"))?; - let total_difficulty = U256::from_str(&total_difficulty)?; let last_block_number = provider_rw.last_block_number()?; if last_block_number == 0 { - info!(target: "reth::cli", "header: {:?}, header_hash: {:?}, total_difficulty: {:?}", header, header_hash, total_difficulty); + info!(target: "reth::cli", "header: {:?}, header_hash: {:?}", header, header_hash); without_evm::setup_without_evm( &provider_rw, SealedHeader::new(header, header_hash), - total_difficulty, + |_| alloy_consensus::Header::default(), )?; // SAFETY: it's safe to commit static files, since in the event of a crash, they diff --git a/crates/optimism/cli/src/commands/mod.rs b/crates/optimism/cli/src/commands/mod.rs index 515307c9ddb..5edd55b0ccb 100644 --- a/crates/optimism/cli/src/commands/mod.rs +++ b/crates/optimism/cli/src/commands/mod.rs @@ -7,7 +7,7 @@ use reth_cli::chainspec::ChainSpecParser; use reth_cli_commands::{ config_cmd, db, dump_genesis, init_cmd, node::{self, NoArgs}, - p2p, prune, recover, stage, + p2p, prune, re_execute, stage, }; use std::{fmt, sync::Arc}; @@ -20,7 +20,6 @@ pub mod test_vectors; /// Commands to be executed #[derive(Debug, Subcommand)] -#[expect(clippy::large_enum_variant)] pub enum Commands { /// Start the node @@ -48,13 +47,10 @@ pub enum Commands>), /// P2P Debugging utilities #[command(name = "p2p")] - P2P(p2p::Command), + P2P(Box>), /// Write config to stdout #[command(name = "config")] Config(config_cmd::Command), - /// Scripts for node recovery - #[command(name = "recover")] - Recover(recover::Command), /// Prune according to the configuration without any limits #[command(name = "prune")] Prune(prune::PruneCommand), @@ -62,6 +58,9 @@ pub enum Commands), } impl< @@ -80,12 +79,12 @@ impl< Self::Stage(cmd) => cmd.chain_spec(), Self::P2P(cmd) => cmd.chain_spec(), Self::Config(_) => None, - Self::Recover(cmd) => cmd.chain_spec(), Self::Prune(cmd) => cmd.chain_spec(), Self::ImportOp(cmd) => cmd.chain_spec(), Self::ImportReceiptsOp(cmd) => cmd.chain_spec(), #[cfg(feature = "dev")] Self::TestVectors(_) => None, + Self::ReExecute(cmd) => cmd.chain_spec(), } } } diff --git a/crates/optimism/cli/src/lib.rs b/crates/optimism/cli/src/lib.rs index 17e0869404e..52fdcc2ddd5 100644 --- a/crates/optimism/cli/src/lib.rs +++ b/crates/optimism/cli/src/lib.rs @@ -6,8 +6,10 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] +/// A configurable App on top of the cli parser. +pub mod app; /// Optimism chain specification parser. pub mod chainspec; /// Optimism CLI commands. @@ -30,41 +32,42 @@ pub mod receipt_file_codec; /// Enables decoding and encoding `Block` types within file contexts. pub mod ovm_file_codec; +pub use app::CliApp; pub use commands::{import::ImportOpCommand, import_receipts::ImportReceiptsOpCommand}; use reth_optimism_chainspec::OpChainSpec; +use reth_rpc_server_types::{DefaultRpcModuleValidator, RpcModuleValidator}; -use std::{ffi::OsString, fmt, sync::Arc}; +use std::{ffi::OsString, fmt, marker::PhantomData, sync::Arc}; use chainspec::OpChainSpecParser; -use clap::{command, Parser}; +use clap::Parser; use commands::Commands; use futures_util::Future; use reth_cli::chainspec::ChainSpecParser; +use reth_cli_commands::launcher::FnLauncher; use reth_cli_runner::CliRunner; use reth_db::DatabaseEnv; use reth_node_builder::{NodeBuilder, WithLaunchContext}; use reth_node_core::{ - args::LogArgs, - version::{LONG_VERSION, SHORT_VERSION}, + args::{LogArgs, TraceArgs}, + version::version_metadata, }; -use reth_optimism_consensus::OpBeaconConsensus; -use reth_optimism_evm::OpExecutorProvider; -use reth_optimism_node::{args::RollupArgs, OpNetworkPrimitives, OpNode}; -use reth_tracing::FileWorkerGuard; -use tracing::info; +use reth_optimism_node::args::RollupArgs; // This allows us to manually enable node metrics features, required for proper jemalloc metric // reporting use reth_node_metrics as _; -use reth_node_metrics::recorder::install_prometheus_recorder; /// The main op-reth cli interface. /// /// This is the entrypoint to the executable. #[derive(Debug, Parser)] -#[command(author, version = SHORT_VERSION, long_version = LONG_VERSION, about = "Reth", long_about = None)] -pub struct Cli -{ +#[command(author, name = version_metadata().name_client.as_ref(), version = version_metadata().short_version.as_ref(), long_version = version_metadata().long_version.as_ref(), about = "Reth", long_about = None)] +pub struct Cli< + Spec: ChainSpecParser = OpChainSpecParser, + Ext: clap::Args + fmt::Debug = RollupArgs, + Rpc: RpcModuleValidator = DefaultRpcModuleValidator, +> { /// The command to run #[command(subcommand)] pub command: Commands, @@ -72,6 +75,14 @@ pub struct Cli, } impl Cli { @@ -90,11 +101,20 @@ impl Cli { } } -impl Cli +impl Cli where C: ChainSpecParser, Ext: clap::Args + fmt::Debug, + Rpc: RpcModuleValidator, { + /// Configures the CLI and returns a [`CliApp`] instance. + /// + /// This method is used to prepare the CLI for execution by wrapping it in a + /// [`CliApp`] that can be further configured before running. + pub fn configure(self) -> CliApp { + CliApp::new(self) + } + /// Execute the configured cli command. /// /// This accepts a closure that is used to launch the node via the @@ -108,66 +128,16 @@ where } /// Execute the configured cli command with the provided [`CliRunner`]. - pub fn with_runner(mut self, runner: CliRunner, launcher: L) -> eyre::Result<()> + pub fn with_runner(self, runner: CliRunner, launcher: L) -> eyre::Result<()> where L: FnOnce(WithLaunchContext, C::ChainSpec>>, Ext) -> Fut, Fut: Future>, { - // add network name to logs dir - // Add network name if available to the logs dir - if let Some(chain_spec) = self.command.chain_spec() { - self.logs.log_file_directory = - self.logs.log_file_directory.join(chain_spec.chain.to_string()); - } - let _guard = self.init_tracing()?; - info!(target: "reth::cli", "Initialized tracing, debug log directory: {}", self.logs.log_file_directory); - - // Install the prometheus recorder to be sure to record all metrics - let _ = install_prometheus_recorder(); - - match self.command { - Commands::Node(command) => { - runner.run_command_until_exit(|ctx| command.execute(ctx, launcher)) - } - Commands::Init(command) => { - runner.run_blocking_until_ctrl_c(command.execute::()) - } - Commands::InitState(command) => { - runner.run_blocking_until_ctrl_c(command.execute::()) - } - Commands::ImportOp(command) => { - runner.run_blocking_until_ctrl_c(command.execute::()) - } - Commands::ImportReceiptsOp(command) => { - runner.run_blocking_until_ctrl_c(command.execute::()) - } - Commands::DumpGenesis(command) => runner.run_blocking_until_ctrl_c(command.execute()), - Commands::Db(command) => runner.run_blocking_until_ctrl_c(command.execute::()), - Commands::Stage(command) => runner.run_command_until_exit(|ctx| { - command.execute::(ctx, |spec| { - (OpExecutorProvider::optimism(spec.clone()), OpBeaconConsensus::new(spec)) - }) - }), - Commands::P2P(command) => { - runner.run_until_ctrl_c(command.execute::()) - } - Commands::Config(command) => runner.run_until_ctrl_c(command.execute()), - Commands::Recover(command) => { - runner.run_command_until_exit(|ctx| command.execute::(ctx)) - } - Commands::Prune(command) => runner.run_until_ctrl_c(command.execute::()), - #[cfg(feature = "dev")] - Commands::TestVectors(command) => runner.run_until_ctrl_c(command.execute()), - } - } - - /// Initializes tracing with the configured options. - /// - /// If file logging is enabled, this function returns a guard that must be kept alive to ensure - /// that all logs are flushed to disk. - pub fn init_tracing(&self) -> eyre::Result> { - let guard = self.logs.init_tracing()?; - Ok(guard) + let mut this = self.configure(); + this.set_runner(runner); + this.run(FnLauncher::new::(async move |builder, chain_spec| { + launcher(builder, chain_spec).await + })) } } @@ -230,6 +200,7 @@ mod test { "10000", "--metrics", "9003", + "--tracing-otlp=http://localhost:4318/v1/traces", "--log.file.max-size", "100", ]); diff --git a/crates/optimism/cli/src/ovm_file_codec.rs b/crates/optimism/cli/src/ovm_file_codec.rs index efd493b5855..83f3e487282 100644 --- a/crates/optimism/cli/src/ovm_file_codec.rs +++ b/crates/optimism/cli/src/ovm_file_codec.rs @@ -251,13 +251,7 @@ impl Encodable2718 for OvmTransactionSigned { } fn encode_2718(&self, out: &mut dyn alloy_rlp::BufMut) { - match &self.transaction { - OpTypedTransaction::Legacy(tx) => tx.eip2718_encode(&self.signature, out), - OpTypedTransaction::Eip2930(tx) => tx.eip2718_encode(&self.signature, out), - OpTypedTransaction::Eip1559(tx) => tx.eip2718_encode(&self.signature, out), - OpTypedTransaction::Eip7702(tx) => tx.eip2718_encode(&self.signature, out), - OpTypedTransaction::Deposit(tx) => tx.encode_2718(out), - } + self.transaction.eip2718_encode(&self.signature, out) } } @@ -309,7 +303,7 @@ mod tests { // Verify deposit transaction let deposit_tx = match &deposit_decoded.transaction { - OpTypedTransaction::Legacy(ref tx) => tx, + OpTypedTransaction::Legacy(tx) => tx, _ => panic!("Expected legacy transaction for NFT deposit"), }; @@ -351,7 +345,7 @@ mod tests { assert!(system_decoded.is_legacy()); let system_tx = match &system_decoded.transaction { - OpTypedTransaction::Legacy(ref tx) => tx, + OpTypedTransaction::Legacy(tx) => tx, _ => panic!("Expected Legacy transaction"), }; diff --git a/crates/optimism/cli/src/receipt_file_codec.rs b/crates/optimism/cli/src/receipt_file_codec.rs index f5b6b48b268..e12af039eac 100644 --- a/crates/optimism/cli/src/receipt_file_codec.rs +++ b/crates/optimism/cli/src/receipt_file_codec.rs @@ -148,8 +148,8 @@ pub(crate) mod test { bloom: Bloom::from(hex!( "00000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000400000000000100000000000000200000000002000000000000001000000000000000000004000000000000000000000000000040000400000100400000000000000100000000000000000000000000000020000000000000000000000000000000000000000000000001000000000000000000000100000000000000000000000000000000000000000000000000000000000000088000000080000000000010000000000000000000000000000800008000120000000000000000000000000000000002000" )), - logs: receipt.receipt.logs().to_vec(), - tx_hash: b256!("0x5e77a04531c7c107af1882d76cbff9486d0a9aa53701c30888509d4f5f2b003a"), contract_address: address!("0x0000000000000000000000000000000000000000"), gas_used: 202813, + logs: receipt.receipt.into_logs(), + tx_hash: b256!("0x5e77a04531c7c107af1882d76cbff9486d0a9aa53701c30888509d4f5f2b003a"), contract_address: Address::ZERO, gas_used: 202813, block_hash: b256!("0xbee7192e575af30420cae0c7776304ac196077ee72b048970549e4f08e875453"), block_number: receipt.number, transaction_index: 0, diff --git a/crates/optimism/consensus/Cargo.toml b/crates/optimism/consensus/Cargo.toml index 92e1642b5ba..54df0af80d2 100644 --- a/crates/optimism/consensus/Cargo.toml +++ b/crates/optimism/consensus/Cargo.toml @@ -32,23 +32,21 @@ alloy-primitives.workspace = true alloy-consensus.workspace = true alloy-trie.workspace = true revm.workspace = true -op-alloy-consensus.workspace = true # misc tracing.workspace = true thiserror.workspace = true +reth-optimism-chainspec.workspace = true [dev-dependencies] reth-provider = { workspace = true, features = ["test-utils"] } reth-db-common.workspace = true reth-revm.workspace = true reth-trie.workspace = true -reth-optimism-chainspec.workspace = true reth-optimism-node.workspace = true -reth-db-api = { workspace = true, features = ["op"] } alloy-chains.workspace = true -alloy-primitives.workspace = true + op-alloy-consensus.workspace = true [features] @@ -69,10 +67,10 @@ std = [ "alloy-primitives/std", "alloy-consensus/std", "alloy-trie/std", - "op-alloy-consensus/std", "reth-revm/std", "revm/std", "tracing/std", "thiserror/std", "reth-execution-types/std", + "op-alloy-consensus/std", ] diff --git a/crates/optimism/consensus/src/lib.rs b/crates/optimism/consensus/src/lib.rs index 164af8ab923..34a003bad32 100644 --- a/crates/optimism/consensus/src/lib.rs +++ b/crates/optimism/consensus/src/lib.rs @@ -5,7 +5,7 @@ html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256", issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(feature = "std"), no_std)] #![cfg_attr(not(test), warn(unused_crate_dependencies))] @@ -18,9 +18,9 @@ use core::fmt::Debug; use reth_chainspec::EthChainSpec; use reth_consensus::{Consensus, ConsensusError, FullConsensus, HeaderValidator}; use reth_consensus_common::validation::{ - validate_against_parent_4844, validate_against_parent_eip1559_base_fee, - validate_against_parent_hash_number, validate_against_parent_timestamp, validate_cancun_gas, - validate_header_base_fee, validate_header_extra_data, validate_header_gas, + validate_against_parent_eip1559_base_fee, validate_against_parent_hash_number, + validate_against_parent_timestamp, validate_cancun_gas, validate_header_base_fee, + validate_header_extra_data, validate_header_gas, }; use reth_execution_types::BlockExecutionResult; use reth_optimism_forks::OpHardforks; @@ -34,9 +34,7 @@ mod proof; pub use proof::calculate_receipt_root_no_memo_optimism; pub mod validation; -pub use validation::{ - canyon, decode_holocene_base_fee, isthmus, next_block_base_fee, validate_block_post_execution, -}; +pub use validation::{canyon, isthmus, validate_block_post_execution}; pub mod error; pub use error::OpConsensusError; @@ -57,20 +55,24 @@ impl OpBeaconConsensus { } } -impl> - FullConsensus for OpBeaconConsensus +impl FullConsensus for OpBeaconConsensus +where + N: NodePrimitives, + ChainSpec: EthChainSpec
+ OpHardforks + Debug + Send + Sync, { fn validate_block_post_execution( &self, block: &RecoveredBlock, result: &BlockExecutionResult, ) -> Result<(), ConsensusError> { - validate_block_post_execution(block.header(), &self.chain_spec, &result.receipts) + validate_block_post_execution(block.header(), &self.chain_spec, result) } } -impl Consensus - for OpBeaconConsensus +impl Consensus for OpBeaconConsensus +where + B: Block, + ChainSpec: EthChainSpec
+ OpHardforks + Debug + Send + Sync, { type Error = ConsensusError; @@ -109,7 +111,13 @@ impl Consensus return Ok(()) } - if self.chain_spec.is_ecotone_active_at_timestamp(block.timestamp()) { + // Blob gas used validation + // In Jovian, the blob gas used computation has changed. We are moving the blob base fee + // validation to post-execution since the DA footprint calculation is stateful. + // Pre-execution we only validate that the blob gas used is present in the header. + if self.chain_spec.is_jovian_active_at_timestamp(block.timestamp()) { + block.blob_gas_used().ok_or(ConsensusError::BlobGasUsedMissing)?; + } else if self.chain_spec.is_ecotone_active_at_timestamp(block.timestamp()) { validate_cancun_gas(block)?; } @@ -128,8 +136,10 @@ impl Consensus } } -impl HeaderValidator - for OpBeaconConsensus +impl HeaderValidator for OpBeaconConsensus +where + H: BlockHeader, + ChainSpec: EthChainSpec
+ OpHardforks + Debug + Send + Sync, { fn validate_header(&self, header: &SealedHeader) -> Result<(), ConsensusError> { let header = header.header(); @@ -172,35 +182,581 @@ impl HeaderValidator validate_against_parent_timestamp(header.header(), parent.header())?; } - // EIP1559 base fee validation - // - // > if Holocene is active in parent_header.timestamp, then the parameters from - // > parent_header.extraData are used. - if self.chain_spec.is_holocene_active_at_timestamp(parent.timestamp()) { - let header_base_fee = - header.base_fee_per_gas().ok_or(ConsensusError::BaseFeeMissing)?; - let expected_base_fee = - decode_holocene_base_fee(&self.chain_spec, parent.header(), header.timestamp()) - .map_err(|_| ConsensusError::BaseFeeMissing)?; - if expected_base_fee != header_base_fee { - return Err(ConsensusError::BaseFeeDiff(GotExpected { - expected: expected_base_fee, - got: header_base_fee, - })) + validate_against_parent_eip1559_base_fee( + header.header(), + parent.header(), + &self.chain_spec, + )?; + + // Ensure that the blob gas fields for this block are correctly set. + // In the op-stack, the excess blob gas is always 0 for all blocks after ecotone. + // The blob gas used and the excess blob gas should both be set after ecotone. + // After Jovian, the blob gas used contains the current DA footprint. + if self.chain_spec.is_ecotone_active_at_timestamp(header.timestamp()) { + let blob_gas_used = header.blob_gas_used().ok_or(ConsensusError::BlobGasUsedMissing)?; + + // Before Jovian and after ecotone, the blob gas used should be 0. + if !self.chain_spec.is_jovian_active_at_timestamp(header.timestamp()) && + blob_gas_used != 0 + { + return Err(ConsensusError::BlobGasUsedDiff(GotExpected { + got: blob_gas_used, + expected: 0, + })); } - } else { - validate_against_parent_eip1559_base_fee( - header.header(), - parent.header(), - &self.chain_spec, - )?; - } - // ensure that the blob gas fields for this block - if let Some(blob_params) = self.chain_spec.blob_params_at_timestamp(header.timestamp()) { - validate_against_parent_4844(header.header(), parent.header(), blob_params)?; + let excess_blob_gas = + header.excess_blob_gas().ok_or(ConsensusError::ExcessBlobGasMissing)?; + if excess_blob_gas != 0 { + return Err(ConsensusError::ExcessBlobGasDiff { + diff: GotExpected { got: excess_blob_gas, expected: 0 }, + parent_excess_blob_gas: parent.excess_blob_gas().unwrap_or(0), + parent_blob_gas_used: parent.blob_gas_used().unwrap_or(0), + }) + } } Ok(()) } } + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use alloy_consensus::{BlockBody, Eip658Value, Header, Receipt, TxEip7702, TxReceipt}; + use alloy_eips::{eip4895::Withdrawals, eip7685::Requests}; + use alloy_primitives::{Address, Bytes, Signature, U256}; + use op_alloy_consensus::{ + encode_holocene_extra_data, encode_jovian_extra_data, OpTypedTransaction, + }; + use reth_chainspec::BaseFeeParams; + use reth_consensus::{Consensus, ConsensusError, FullConsensus, HeaderValidator}; + use reth_optimism_chainspec::{OpChainSpec, OpChainSpecBuilder, OP_MAINNET}; + use reth_optimism_primitives::{OpPrimitives, OpReceipt, OpTransactionSigned}; + use reth_primitives_traits::{proofs, GotExpected, RecoveredBlock, SealedBlock, SealedHeader}; + use reth_provider::BlockExecutionResult; + + use crate::OpBeaconConsensus; + + fn mock_tx(nonce: u64) -> OpTransactionSigned { + let tx = TxEip7702 { + chain_id: 1u64, + nonce, + max_fee_per_gas: 0x28f000fff, + max_priority_fee_per_gas: 0x28f000fff, + gas_limit: 10, + to: Address::default(), + value: U256::from(3_u64), + input: Bytes::from(vec![1, 2]), + access_list: Default::default(), + authorization_list: Default::default(), + }; + + let signature = Signature::new(U256::default(), U256::default(), true); + + OpTransactionSigned::new_unhashed(OpTypedTransaction::Eip7702(tx), signature) + } + + #[test] + fn test_block_blob_gas_used_validation_isthmus() { + let chain_spec = OpChainSpecBuilder::default() + .isthmus_activated() + .genesis(OP_MAINNET.genesis.clone()) + .chain(OP_MAINNET.chain) + .build(); + + // create a tx + let transaction = mock_tx(0); + + let beacon_consensus = OpBeaconConsensus::new(Arc::new(chain_spec)); + + let header = Header { + base_fee_per_gas: Some(1337), + withdrawals_root: Some(proofs::calculate_withdrawals_root(&[])), + blob_gas_used: Some(0), + transactions_root: proofs::calculate_transaction_root(std::slice::from_ref( + &transaction, + )), + timestamp: u64::MAX, + ..Default::default() + }; + let body = BlockBody { + transactions: vec![transaction], + ommers: vec![], + withdrawals: Some(Withdrawals::default()), + }; + + let block = SealedBlock::seal_slow(alloy_consensus::Block { header, body }); + + // validate blob, it should pass blob gas used validation + let pre_execution = beacon_consensus.validate_block_pre_execution(&block); + + assert!(pre_execution.is_ok()); + } + + #[test] + fn test_block_blob_gas_used_validation_failure_isthmus() { + let chain_spec = OpChainSpecBuilder::default() + .isthmus_activated() + .genesis(OP_MAINNET.genesis.clone()) + .chain(OP_MAINNET.chain) + .build(); + + // create a tx + let transaction = mock_tx(0); + + let beacon_consensus = OpBeaconConsensus::new(Arc::new(chain_spec)); + + let header = Header { + base_fee_per_gas: Some(1337), + withdrawals_root: Some(proofs::calculate_withdrawals_root(&[])), + blob_gas_used: Some(10), + transactions_root: proofs::calculate_transaction_root(std::slice::from_ref( + &transaction, + )), + timestamp: u64::MAX, + ..Default::default() + }; + let body = BlockBody { + transactions: vec![transaction], + ommers: vec![], + withdrawals: Some(Withdrawals::default()), + }; + + let block = SealedBlock::seal_slow(alloy_consensus::Block { header, body }); + + // validate blob, it should fail blob gas used validation + let pre_execution = beacon_consensus.validate_block_pre_execution(&block); + + assert!(pre_execution.is_err()); + assert_eq!( + pre_execution.unwrap_err(), + ConsensusError::BlobGasUsedDiff(GotExpected { got: 10, expected: 0 }) + ); + } + + #[test] + fn test_block_blob_gas_used_validation_jovian() { + const BLOB_GAS_USED: u64 = 1000; + const GAS_USED: u64 = 10; + + let chain_spec = OpChainSpecBuilder::default() + .jovian_activated() + .genesis(OP_MAINNET.genesis.clone()) + .chain(OP_MAINNET.chain) + .build(); + + // create a tx + let transaction = mock_tx(0); + + let beacon_consensus = OpBeaconConsensus::new(Arc::new(chain_spec)); + + let receipt = OpReceipt::Eip7702(Receipt { + status: Eip658Value::success(), + cumulative_gas_used: GAS_USED, + logs: vec![], + }); + + let header = Header { + base_fee_per_gas: Some(1337), + withdrawals_root: Some(proofs::calculate_withdrawals_root(&[])), + blob_gas_used: Some(BLOB_GAS_USED), + transactions_root: proofs::calculate_transaction_root(std::slice::from_ref( + &transaction, + )), + timestamp: u64::MAX, + gas_used: GAS_USED, + receipts_root: proofs::calculate_receipt_root(std::slice::from_ref( + &receipt.with_bloom_ref(), + )), + logs_bloom: receipt.bloom(), + ..Default::default() + }; + let body = BlockBody { + transactions: vec![transaction], + ommers: vec![], + withdrawals: Some(Withdrawals::default()), + }; + + let block = SealedBlock::seal_slow(alloy_consensus::Block { header, body }); + + let result = BlockExecutionResult:: { + blob_gas_used: BLOB_GAS_USED, + receipts: vec![receipt], + requests: Requests::default(), + gas_used: GAS_USED, + }; + + // validate blob, it should pass blob gas used validation + let pre_execution = beacon_consensus.validate_block_pre_execution(&block); + + assert!(pre_execution.is_ok()); + + let block = RecoveredBlock::new_sealed(block, vec![Address::default()]); + + let post_execution = as FullConsensus>::validate_block_post_execution( + &beacon_consensus, + &block, + &result + ); + + // validate blob, it should pass blob gas used validation + assert!(post_execution.is_ok()); + } + + #[test] + fn test_block_blob_gas_used_validation_failure_jovian() { + const BLOB_GAS_USED: u64 = 1000; + const GAS_USED: u64 = 10; + + let chain_spec = OpChainSpecBuilder::default() + .jovian_activated() + .genesis(OP_MAINNET.genesis.clone()) + .chain(OP_MAINNET.chain) + .build(); + + // create a tx + let transaction = mock_tx(0); + + let beacon_consensus = OpBeaconConsensus::new(Arc::new(chain_spec)); + + let receipt = OpReceipt::Eip7702(Receipt { + status: Eip658Value::success(), + cumulative_gas_used: GAS_USED, + logs: vec![], + }); + + let header = Header { + base_fee_per_gas: Some(1337), + withdrawals_root: Some(proofs::calculate_withdrawals_root(&[])), + blob_gas_used: Some(BLOB_GAS_USED), + transactions_root: proofs::calculate_transaction_root(std::slice::from_ref( + &transaction, + )), + gas_used: GAS_USED, + timestamp: u64::MAX, + receipts_root: proofs::calculate_receipt_root(std::slice::from_ref(&receipt)), + logs_bloom: receipt.bloom(), + ..Default::default() + }; + let body = BlockBody { + transactions: vec![transaction], + ommers: vec![], + withdrawals: Some(Withdrawals::default()), + }; + + let block = SealedBlock::seal_slow(alloy_consensus::Block { header, body }); + + let result = BlockExecutionResult:: { + blob_gas_used: BLOB_GAS_USED + 1, + receipts: vec![receipt], + requests: Requests::default(), + gas_used: GAS_USED, + }; + + // validate blob, it should pass blob gas used validation + let pre_execution = beacon_consensus.validate_block_pre_execution(&block); + + assert!(pre_execution.is_ok()); + + let block = RecoveredBlock::new_sealed(block, vec![Address::default()]); + + let post_execution = as FullConsensus>::validate_block_post_execution( + &beacon_consensus, + &block, + &result + ); + + // validate blob, it should fail blob gas used validation post execution. + assert!(post_execution.is_err()); + assert_eq!( + post_execution.unwrap_err(), + ConsensusError::BlobGasUsedDiff(GotExpected { + got: BLOB_GAS_USED + 1, + expected: BLOB_GAS_USED, + }) + ); + } + + #[test] + fn test_header_min_base_fee_validation() { + const MIN_BASE_FEE: u64 = 1000; + + let chain_spec = OpChainSpecBuilder::default() + .jovian_activated() + .genesis(OP_MAINNET.genesis.clone()) + .chain(OP_MAINNET.chain) + .build(); + + // create a tx + let transaction = mock_tx(0); + + let beacon_consensus = OpBeaconConsensus::new(Arc::new(chain_spec)); + + let receipt = OpReceipt::Eip7702(Receipt { + status: Eip658Value::success(), + cumulative_gas_used: 0, + logs: vec![], + }); + + let parent = Header { + number: 0, + base_fee_per_gas: Some(MIN_BASE_FEE / 10), + withdrawals_root: Some(proofs::calculate_withdrawals_root(&[])), + blob_gas_used: Some(0), + excess_blob_gas: Some(0), + transactions_root: proofs::calculate_transaction_root(std::slice::from_ref( + &transaction, + )), + gas_used: 0, + timestamp: u64::MAX - 1, + receipts_root: proofs::calculate_receipt_root(std::slice::from_ref(&receipt)), + logs_bloom: receipt.bloom(), + extra_data: encode_jovian_extra_data( + Default::default(), + BaseFeeParams::optimism(), + MIN_BASE_FEE, + ) + .unwrap(), + ..Default::default() + }; + let parent = SealedHeader::seal_slow(parent); + + let header = Header { + number: 1, + base_fee_per_gas: Some(MIN_BASE_FEE), + withdrawals_root: Some(proofs::calculate_withdrawals_root(&[])), + blob_gas_used: Some(0), + excess_blob_gas: Some(0), + transactions_root: proofs::calculate_transaction_root(std::slice::from_ref( + &transaction, + )), + gas_used: 0, + timestamp: u64::MAX, + receipts_root: proofs::calculate_receipt_root(std::slice::from_ref(&receipt)), + logs_bloom: receipt.bloom(), + parent_hash: parent.hash(), + ..Default::default() + }; + let header = SealedHeader::seal_slow(header); + + let result = beacon_consensus.validate_header_against_parent(&header, &parent); + + assert!(result.is_ok()); + } + + #[test] + fn test_header_min_base_fee_validation_failure() { + const MIN_BASE_FEE: u64 = 1000; + + let chain_spec = OpChainSpecBuilder::default() + .jovian_activated() + .genesis(OP_MAINNET.genesis.clone()) + .chain(OP_MAINNET.chain) + .build(); + + // create a tx + let transaction = mock_tx(0); + + let beacon_consensus = OpBeaconConsensus::new(Arc::new(chain_spec)); + + let receipt = OpReceipt::Eip7702(Receipt { + status: Eip658Value::success(), + cumulative_gas_used: 0, + logs: vec![], + }); + + let parent = Header { + number: 0, + base_fee_per_gas: Some(MIN_BASE_FEE / 10), + withdrawals_root: Some(proofs::calculate_withdrawals_root(&[])), + blob_gas_used: Some(0), + excess_blob_gas: Some(0), + transactions_root: proofs::calculate_transaction_root(std::slice::from_ref( + &transaction, + )), + gas_used: 0, + timestamp: u64::MAX - 1, + receipts_root: proofs::calculate_receipt_root(std::slice::from_ref(&receipt)), + logs_bloom: receipt.bloom(), + extra_data: encode_jovian_extra_data( + Default::default(), + BaseFeeParams::optimism(), + MIN_BASE_FEE, + ) + .unwrap(), + ..Default::default() + }; + let parent = SealedHeader::seal_slow(parent); + + let header = Header { + number: 1, + base_fee_per_gas: Some(MIN_BASE_FEE - 1), + withdrawals_root: Some(proofs::calculate_withdrawals_root(&[])), + blob_gas_used: Some(0), + excess_blob_gas: Some(0), + transactions_root: proofs::calculate_transaction_root(std::slice::from_ref( + &transaction, + )), + gas_used: 0, + timestamp: u64::MAX, + receipts_root: proofs::calculate_receipt_root(std::slice::from_ref(&receipt)), + logs_bloom: receipt.bloom(), + parent_hash: parent.hash(), + ..Default::default() + }; + let header = SealedHeader::seal_slow(header); + + let result = beacon_consensus.validate_header_against_parent(&header, &parent); + + assert!(result.is_err()); + assert_eq!( + result.unwrap_err(), + ConsensusError::BaseFeeDiff(GotExpected { + got: MIN_BASE_FEE - 1, + expected: MIN_BASE_FEE, + }) + ); + } + + #[test] + fn test_header_da_footprint_validation() { + const MIN_BASE_FEE: u64 = 100_000; + const DA_FOOTPRINT: u64 = GAS_LIMIT - 1; + const GAS_LIMIT: u64 = 100_000_000; + + let chain_spec = OpChainSpecBuilder::default() + .jovian_activated() + .genesis(OP_MAINNET.genesis.clone()) + .chain(OP_MAINNET.chain) + .build(); + + // create a tx + let transaction = mock_tx(0); + + let beacon_consensus = OpBeaconConsensus::new(Arc::new(chain_spec)); + + let receipt = OpReceipt::Eip7702(Receipt { + status: Eip658Value::success(), + cumulative_gas_used: 0, + logs: vec![], + }); + + let parent = Header { + number: 0, + base_fee_per_gas: Some(MIN_BASE_FEE), + withdrawals_root: Some(proofs::calculate_withdrawals_root(&[])), + blob_gas_used: Some(DA_FOOTPRINT), + excess_blob_gas: Some(0), + transactions_root: proofs::calculate_transaction_root(std::slice::from_ref( + &transaction, + )), + gas_used: 0, + timestamp: u64::MAX - 1, + receipts_root: proofs::calculate_receipt_root(std::slice::from_ref(&receipt)), + logs_bloom: receipt.bloom(), + extra_data: encode_jovian_extra_data( + Default::default(), + BaseFeeParams::optimism(), + MIN_BASE_FEE, + ) + .unwrap(), + gas_limit: GAS_LIMIT, + ..Default::default() + }; + let parent = SealedHeader::seal_slow(parent); + + let header = Header { + number: 1, + base_fee_per_gas: Some(MIN_BASE_FEE + MIN_BASE_FEE / 10), + withdrawals_root: Some(proofs::calculate_withdrawals_root(&[])), + blob_gas_used: Some(DA_FOOTPRINT), + excess_blob_gas: Some(0), + transactions_root: proofs::calculate_transaction_root(std::slice::from_ref( + &transaction, + )), + gas_used: 0, + timestamp: u64::MAX, + receipts_root: proofs::calculate_receipt_root(std::slice::from_ref(&receipt)), + logs_bloom: receipt.bloom(), + parent_hash: parent.hash(), + ..Default::default() + }; + let header = SealedHeader::seal_slow(header); + + let result = beacon_consensus.validate_header_against_parent(&header, &parent); + + assert!(result.is_ok()); + } + + #[test] + fn test_header_isthmus_validation() { + const MIN_BASE_FEE: u64 = 100_000; + const DA_FOOTPRINT: u64 = GAS_LIMIT - 1; + const GAS_LIMIT: u64 = 100_000_000; + + let chain_spec = OpChainSpecBuilder::default() + .isthmus_activated() + .genesis(OP_MAINNET.genesis.clone()) + .chain(OP_MAINNET.chain) + .build(); + + // create a tx + let transaction = mock_tx(0); + + let beacon_consensus = OpBeaconConsensus::new(Arc::new(chain_spec)); + + let receipt = OpReceipt::Eip7702(Receipt { + status: Eip658Value::success(), + cumulative_gas_used: 0, + logs: vec![], + }); + + let parent = Header { + number: 0, + base_fee_per_gas: Some(MIN_BASE_FEE), + withdrawals_root: Some(proofs::calculate_withdrawals_root(&[])), + blob_gas_used: Some(DA_FOOTPRINT), + excess_blob_gas: Some(0), + transactions_root: proofs::calculate_transaction_root(std::slice::from_ref( + &transaction, + )), + gas_used: 0, + timestamp: u64::MAX - 1, + receipts_root: proofs::calculate_receipt_root(std::slice::from_ref(&receipt)), + logs_bloom: receipt.bloom(), + extra_data: encode_holocene_extra_data(Default::default(), BaseFeeParams::optimism()) + .unwrap(), + gas_limit: GAS_LIMIT, + ..Default::default() + }; + let parent = SealedHeader::seal_slow(parent); + + let header = Header { + number: 1, + base_fee_per_gas: Some(MIN_BASE_FEE - 2 * MIN_BASE_FEE / 100), + withdrawals_root: Some(proofs::calculate_withdrawals_root(&[])), + blob_gas_used: Some(DA_FOOTPRINT), + excess_blob_gas: Some(0), + transactions_root: proofs::calculate_transaction_root(std::slice::from_ref( + &transaction, + )), + gas_used: 0, + timestamp: u64::MAX, + receipts_root: proofs::calculate_receipt_root(std::slice::from_ref(&receipt)), + logs_bloom: receipt.bloom(), + parent_hash: parent.hash(), + ..Default::default() + }; + let header = SealedHeader::seal_slow(header); + + let result = beacon_consensus.validate_header_against_parent(&header, &parent); + + assert!(result.is_err()); + assert_eq!( + result.unwrap_err(), + ConsensusError::BlobGasUsedDiff(GotExpected { got: DA_FOOTPRINT, expected: 0 }) + ); + } +} diff --git a/crates/optimism/consensus/src/proof.rs b/crates/optimism/consensus/src/proof.rs index 86f7b2ecbeb..8c601942ece 100644 --- a/crates/optimism/consensus/src/proof.rs +++ b/crates/optimism/consensus/src/proof.rs @@ -118,7 +118,7 @@ mod tests { ]; for case in cases { - let receipts = vec![ + let receipts = [ // 0xb0d6ee650637911394396d81172bd1c637d568ed1fbddab0daddfca399c58b53 OpReceipt::Deposit(OpDepositReceipt { inner: Receipt { diff --git a/crates/optimism/consensus/src/validation/isthmus.rs b/crates/optimism/consensus/src/validation/isthmus.rs index 64d45eae5c8..4703e10869e 100644 --- a/crates/optimism/consensus/src/validation/isthmus.rs +++ b/crates/optimism/consensus/src/validation/isthmus.rs @@ -4,7 +4,6 @@ use crate::OpConsensusError; use alloy_consensus::BlockHeader; use alloy_primitives::{address, Address, B256}; use alloy_trie::EMPTY_ROOT_HASH; -use core::fmt::Debug; use reth_storage_api::{errors::ProviderResult, StorageRootProvider}; use reth_trie_common::HashedStorage; use revm::database::BundleState; @@ -72,7 +71,7 @@ pub fn verify_withdrawals_root( ) -> Result<(), OpConsensusError> where DB: StorageRootProvider, - H: BlockHeader + Debug, + H: BlockHeader, { let header_storage_root = header.withdrawals_root().ok_or(OpConsensusError::L2WithdrawalsRootMissing)?; @@ -110,7 +109,7 @@ pub fn verify_withdrawals_root_prehashed( ) -> Result<(), OpConsensusError> where DB: StorageRootProvider, - H: BlockHeader + core::fmt::Debug, + H: BlockHeader, { let header_storage_root = header.withdrawals_root().ok_or(OpConsensusError::L2WithdrawalsRootMissing)?; diff --git a/crates/optimism/consensus/src/validation/mod.rs b/crates/optimism/consensus/src/validation/mod.rs index dfb080f64f7..c17e8429c81 100644 --- a/crates/optimism/consensus/src/validation/mod.rs +++ b/crates/optimism/consensus/src/validation/mod.rs @@ -3,13 +3,16 @@ pub mod canyon; pub mod isthmus; +// Re-export the decode_holocene_base_fee function for compatibility +use reth_execution_types::BlockExecutionResult; +pub use reth_optimism_chainspec::decode_holocene_base_fee; + use crate::proof::calculate_receipt_root_optimism; use alloc::vec::Vec; use alloy_consensus::{BlockHeader, TxReceipt, EMPTY_OMMER_ROOT_HASH}; -use alloy_primitives::{Bloom, B256}; +use alloy_eips::Encodable2718; +use alloy_primitives::{Bloom, Bytes, B256}; use alloy_trie::EMPTY_ROOT_HASH; -use op_alloy_consensus::{decode_holocene_extra_data, EIP1559ParamError}; -use reth_chainspec::{BaseFeeParams, EthChainSpec}; use reth_consensus::ConsensusError; use reth_optimism_forks::OpHardforks; use reth_optimism_primitives::DepositReceipt; @@ -85,23 +88,43 @@ where pub fn validate_block_post_execution( header: impl BlockHeader, chain_spec: impl OpHardforks, - receipts: &[R], + result: &BlockExecutionResult, ) -> Result<(), ConsensusError> { + // Validate that the blob gas used is present and correctly computed if Jovian is active. + if chain_spec.is_jovian_active_at_timestamp(header.timestamp()) { + let computed_blob_gas_used = result.blob_gas_used; + let header_blob_gas_used = + header.blob_gas_used().ok_or(ConsensusError::BlobGasUsedMissing)?; + + if computed_blob_gas_used != header_blob_gas_used { + return Err(ConsensusError::BlobGasUsedDiff(GotExpected { + got: computed_blob_gas_used, + expected: header_blob_gas_used, + })); + } + } + + let receipts = &result.receipts; + // Before Byzantium, receipts contained state root that would mean that expensive // operation as hashing that is required for state root got calculated in every // transaction This was replaced with is_success flag. // See more about EIP here: https://eips.ethereum.org/EIPS/eip-658 - if chain_spec.is_byzantium_active_at_block(header.number()) { - if let Err(error) = verify_receipts_optimism( + if chain_spec.is_byzantium_active_at_block(header.number()) && + let Err(error) = verify_receipts_optimism( header.receipts_root(), header.logs_bloom(), receipts, chain_spec, header.timestamp(), - ) { - tracing::debug!(%error, ?receipts, "receipts verification failed"); - return Err(error) - } + ) + { + let receipts = receipts + .iter() + .map(|r| Bytes::from(r.with_bloom_ref().encoded_2718())) + .collect::>(); + tracing::debug!(%error, ?receipts, "receipts verification failed"); + return Err(error) } // Check if gas used matches the value set in header. @@ -166,58 +189,28 @@ fn compare_receipts_root_and_logs_bloom( Ok(()) } -/// Extracts the Holocene 1599 parameters from the encoded extra data from the parent header. -/// -/// Caution: Caller must ensure that holocene is active in the parent header. -/// -/// See also [Base fee computation](https://github.com/ethereum-optimism/specs/blob/main/specs/protocol/holocene/exec-engine.md#base-fee-computation) -pub fn decode_holocene_base_fee( - chain_spec: impl EthChainSpec + OpHardforks, - parent: impl BlockHeader, - timestamp: u64, -) -> Result { - let (elasticity, denominator) = decode_holocene_extra_data(parent.extra_data())?; - let base_fee_params = if elasticity == 0 && denominator == 0 { - chain_spec.base_fee_params_at_timestamp(timestamp) - } else { - BaseFeeParams::new(denominator as u128, elasticity as u128) - }; - - Ok(parent.next_block_base_fee(base_fee_params).unwrap_or_default()) -} - -/// Read from parent to determine the base fee for the next block -/// -/// See also [Base fee computation](https://github.com/ethereum-optimism/specs/blob/main/specs/protocol/holocene/exec-engine.md#base-fee-computation) -pub fn next_block_base_fee( - chain_spec: impl EthChainSpec + OpHardforks, - parent: impl BlockHeader, - timestamp: u64, -) -> Result { - // If we are in the Holocene, we need to use the base fee params - // from the parent block's extra data. - // Else, use the base fee params (default values) from chainspec - if chain_spec.is_holocene_active_at_timestamp(parent.timestamp()) { - Ok(decode_holocene_base_fee(chain_spec, parent, timestamp)?) - } else { - Ok(parent.base_fee_per_gas().unwrap_or_default()) - } -} - #[cfg(test)] mod tests { use super::*; use alloy_consensus::Header; + use alloy_eips::eip7685::Requests; use alloy_primitives::{b256, hex, Bytes, U256}; use op_alloy_consensus::OpTxEnvelope; - use reth_chainspec::{ChainSpec, ForkCondition, Hardfork}; + use reth_chainspec::{BaseFeeParams, ChainSpec, EthChainSpec, ForkCondition, Hardfork}; use reth_optimism_chainspec::{OpChainSpec, BASE_SEPOLIA}; use reth_optimism_forks::{OpHardfork, BASE_SEPOLIA_HARDFORKS}; + use reth_optimism_primitives::OpReceipt; use std::sync::Arc; + const HOLOCENE_TIMESTAMP: u64 = 1700000000; + const ISTHMUS_TIMESTAMP: u64 = 1750000000; + const JOVIAN_TIMESTAMP: u64 = 1800000000; + const BLOCK_TIME_SECONDS: u64 = 2; + fn holocene_chainspec() -> Arc { let mut hardforks = BASE_SEPOLIA_HARDFORKS.clone(); - hardforks.insert(OpHardfork::Holocene.boxed(), ForkCondition::Timestamp(1800000000)); + hardforks + .insert(OpHardfork::Holocene.boxed(), ForkCondition::Timestamp(HOLOCENE_TIMESTAMP)); Arc::new(OpChainSpec { inner: ChainSpec { chain: BASE_SEPOLIA.inner.chain, @@ -237,7 +230,16 @@ mod tests { chainspec .inner .hardforks - .insert(OpHardfork::Isthmus.boxed(), ForkCondition::Timestamp(1800000000)); + .insert(OpHardfork::Isthmus.boxed(), ForkCondition::Timestamp(ISTHMUS_TIMESTAMP)); + chainspec + } + + fn jovian_chainspec() -> OpChainSpec { + let mut chainspec = BASE_SEPOLIA.as_ref().clone(); + chainspec + .inner + .hardforks + .insert(OpHardfork::Jovian.boxed(), ForkCondition::Timestamp(JOVIAN_TIMESTAMP)); chainspec } @@ -250,12 +252,11 @@ mod tests { gas_limit: 144000000, ..Default::default() }; - let base_fee = next_block_base_fee(&op_chain_spec, &parent, 0); + let base_fee = + reth_optimism_chainspec::OpChainSpec::next_block_base_fee(&op_chain_spec, &parent, 0); assert_eq!( base_fee.unwrap(), - parent - .next_block_base_fee(op_chain_spec.base_fee_params_at_timestamp(0)) - .unwrap_or_default() + op_chain_spec.next_block_base_fee(&parent, 0).unwrap_or_default() ); } @@ -266,16 +267,18 @@ mod tests { base_fee_per_gas: Some(1), gas_used: 15763614, gas_limit: 144000000, - timestamp: 1800000003, + timestamp: HOLOCENE_TIMESTAMP + 3, extra_data: Bytes::from_static(&[0, 0, 0, 0, 0, 0, 0, 0, 0]), ..Default::default() }; - let base_fee = next_block_base_fee(&op_chain_spec, &parent, 1800000005); + let base_fee = reth_optimism_chainspec::OpChainSpec::next_block_base_fee( + &op_chain_spec, + &parent, + HOLOCENE_TIMESTAMP + 5, + ); assert_eq!( base_fee.unwrap(), - parent - .next_block_base_fee(op_chain_spec.base_fee_params_at_timestamp(0)) - .unwrap_or_default() + op_chain_spec.next_block_base_fee(&parent, 0).unwrap_or_default() ); } @@ -286,11 +289,15 @@ mod tests { gas_used: 15763614, gas_limit: 144000000, extra_data: Bytes::from_static(&[0, 0, 0, 0, 8, 0, 0, 0, 8]), - timestamp: 1800000003, + timestamp: HOLOCENE_TIMESTAMP + 3, ..Default::default() }; - let base_fee = next_block_base_fee(holocene_chainspec(), &parent, 1800000005); + let base_fee = reth_optimism_chainspec::OpChainSpec::next_block_base_fee( + &holocene_chainspec(), + &parent, + HOLOCENE_TIMESTAMP + 5, + ); assert_eq!( base_fee.unwrap(), parent @@ -311,10 +318,188 @@ mod tests { ..Default::default() }; - let base_fee = next_block_base_fee(&*BASE_SEPOLIA, &parent, 1735315546).unwrap(); + let base_fee = reth_optimism_chainspec::OpChainSpec::next_block_base_fee( + &*BASE_SEPOLIA, + &parent, + 1735315546, + ) + .unwrap(); assert_eq!(base_fee, 507); } + #[test] + fn test_get_base_fee_holocene_extra_data_set_and_min_base_fee_set() { + const MIN_BASE_FEE: u64 = 10; + + let mut extra_data = Vec::new(); + // eip1559 params + extra_data.append(&mut hex!("00000000fa0000000a").to_vec()); + // min base fee + extra_data.append(&mut MIN_BASE_FEE.to_be_bytes().to_vec()); + let extra_data = Bytes::from(extra_data); + + let parent = Header { + base_fee_per_gas: Some(507), + gas_used: 4847634, + gas_limit: 60000000, + extra_data, + timestamp: 1735315544, + ..Default::default() + }; + + let base_fee = reth_optimism_chainspec::OpChainSpec::next_block_base_fee( + &*BASE_SEPOLIA, + &parent, + 1735315546, + ); + assert_eq!(base_fee, None); + } + + /// The version byte for Jovian is 1. + const JOVIAN_EXTRA_DATA_VERSION_BYTE: u8 = 1; + + #[test] + fn test_get_base_fee_jovian_extra_data_and_min_base_fee_not_set() { + let op_chain_spec = jovian_chainspec(); + + let mut extra_data = Vec::new(); + extra_data.push(JOVIAN_EXTRA_DATA_VERSION_BYTE); + // eip1559 params + extra_data.append(&mut [0_u8; 8].to_vec()); + let extra_data = Bytes::from(extra_data); + + let parent = Header { + base_fee_per_gas: Some(1), + gas_used: 15763614, + gas_limit: 144000000, + timestamp: JOVIAN_TIMESTAMP, + extra_data, + ..Default::default() + }; + let base_fee = reth_optimism_chainspec::OpChainSpec::next_block_base_fee( + &op_chain_spec, + &parent, + JOVIAN_TIMESTAMP + BLOCK_TIME_SECONDS, + ); + assert_eq!(base_fee, None); + } + + /// After Jovian, the next block base fee cannot be less than the minimum base fee. + #[test] + fn test_get_base_fee_jovian_default_extra_data_and_min_base_fee() { + const CURR_BASE_FEE: u64 = 1; + const MIN_BASE_FEE: u64 = 10; + + let mut extra_data = Vec::new(); + extra_data.push(JOVIAN_EXTRA_DATA_VERSION_BYTE); + // eip1559 params + extra_data.append(&mut [0_u8; 8].to_vec()); + // min base fee + extra_data.append(&mut MIN_BASE_FEE.to_be_bytes().to_vec()); + let extra_data = Bytes::from(extra_data); + + let op_chain_spec = jovian_chainspec(); + let parent = Header { + base_fee_per_gas: Some(CURR_BASE_FEE), + gas_used: 15763614, + gas_limit: 144000000, + timestamp: JOVIAN_TIMESTAMP, + extra_data, + ..Default::default() + }; + let base_fee = reth_optimism_chainspec::OpChainSpec::next_block_base_fee( + &op_chain_spec, + &parent, + JOVIAN_TIMESTAMP + BLOCK_TIME_SECONDS, + ); + assert_eq!(base_fee, Some(MIN_BASE_FEE)); + } + + /// After Jovian, the next block base fee cannot be less than the minimum base fee. + #[test] + fn test_jovian_min_base_fee_cannot_decrease() { + const MIN_BASE_FEE: u64 = 10; + + let mut extra_data = Vec::new(); + extra_data.push(JOVIAN_EXTRA_DATA_VERSION_BYTE); + // eip1559 params + extra_data.append(&mut [0_u8; 8].to_vec()); + // min base fee + extra_data.append(&mut MIN_BASE_FEE.to_be_bytes().to_vec()); + let extra_data = Bytes::from(extra_data); + + let op_chain_spec = jovian_chainspec(); + + // If we're currently at the minimum base fee, the next block base fee cannot decrease. + let parent = Header { + base_fee_per_gas: Some(MIN_BASE_FEE), + gas_used: 10, + gas_limit: 144000000, + timestamp: JOVIAN_TIMESTAMP, + extra_data: extra_data.clone(), + ..Default::default() + }; + let base_fee = reth_optimism_chainspec::OpChainSpec::next_block_base_fee( + &op_chain_spec, + &parent, + JOVIAN_TIMESTAMP + BLOCK_TIME_SECONDS, + ); + assert_eq!(base_fee, Some(MIN_BASE_FEE)); + + // The next block can increase the base fee + let parent = Header { + base_fee_per_gas: Some(MIN_BASE_FEE), + gas_used: 144000000, + gas_limit: 144000000, + timestamp: JOVIAN_TIMESTAMP, + extra_data, + ..Default::default() + }; + let base_fee = reth_optimism_chainspec::OpChainSpec::next_block_base_fee( + &op_chain_spec, + &parent, + JOVIAN_TIMESTAMP + 2 * BLOCK_TIME_SECONDS, + ); + assert_eq!(base_fee, Some(MIN_BASE_FEE + 1)); + } + + #[test] + fn test_jovian_base_fee_can_decrease_if_above_min_base_fee() { + const MIN_BASE_FEE: u64 = 10; + + let mut extra_data = Vec::new(); + extra_data.push(JOVIAN_EXTRA_DATA_VERSION_BYTE); + // eip1559 params + extra_data.append(&mut [0_u8; 8].to_vec()); + // min base fee + extra_data.append(&mut MIN_BASE_FEE.to_be_bytes().to_vec()); + let extra_data = Bytes::from(extra_data); + + let op_chain_spec = jovian_chainspec(); + + let parent = Header { + base_fee_per_gas: Some(100 * MIN_BASE_FEE), + gas_used: 10, + gas_limit: 144000000, + timestamp: JOVIAN_TIMESTAMP, + extra_data, + ..Default::default() + }; + let base_fee = reth_optimism_chainspec::OpChainSpec::next_block_base_fee( + &op_chain_spec, + &parent, + JOVIAN_TIMESTAMP + BLOCK_TIME_SECONDS, + ) + .unwrap(); + assert_eq!( + base_fee, + op_chain_spec + .inner + .next_block_base_fee(&parent, JOVIAN_TIMESTAMP + BLOCK_TIME_SECONDS) + .unwrap() + ); + } + #[test] fn body_against_header_isthmus() { let chainspec = isthmus_chainspec(); @@ -339,4 +524,52 @@ mod tests { body.withdrawals.take(); validate_body_against_header_op(&chainspec, &body, &header).unwrap_err(); } + + #[test] + fn test_jovian_blob_gas_used_validation() { + const BLOB_GAS_USED: u64 = 1000; + const GAS_USED: u64 = 5000; + + let chainspec = jovian_chainspec(); + let header = Header { + timestamp: JOVIAN_TIMESTAMP, + blob_gas_used: Some(BLOB_GAS_USED), + ..Default::default() + }; + + let result = BlockExecutionResult:: { + blob_gas_used: BLOB_GAS_USED, + receipts: vec![], + requests: Requests::default(), + gas_used: GAS_USED, + }; + validate_block_post_execution(&header, &chainspec, &result).unwrap(); + } + + #[test] + fn test_jovian_blob_gas_used_validation_mismatched() { + const BLOB_GAS_USED: u64 = 1000; + const GAS_USED: u64 = 5000; + + let chainspec = jovian_chainspec(); + let header = Header { + timestamp: JOVIAN_TIMESTAMP, + blob_gas_used: Some(BLOB_GAS_USED + 1), + ..Default::default() + }; + + let result = BlockExecutionResult:: { + blob_gas_used: BLOB_GAS_USED, + receipts: vec![], + requests: Requests::default(), + gas_used: GAS_USED, + }; + assert_eq!( + validate_block_post_execution(&header, &chainspec, &result), + Err(ConsensusError::BlobGasUsedDiff(GotExpected { + got: BLOB_GAS_USED, + expected: BLOB_GAS_USED + 1, + })) + ); + } } diff --git a/crates/optimism/evm/Cargo.toml b/crates/optimism/evm/Cargo.toml index cf099bc3bf9..1232384547e 100644 --- a/crates/optimism/evm/Cargo.toml +++ b/crates/optimism/evm/Cargo.toml @@ -17,6 +17,9 @@ reth-evm = { workspace = true, features = ["op"] } reth-primitives-traits.workspace = true reth-execution-errors.workspace = true reth-execution-types.workspace = true +reth-storage-errors.workspace = true + +reth-rpc-eth-api = { workspace = true, optional = true } # ethereum alloy-eips.workspace = true @@ -24,16 +27,18 @@ alloy-evm.workspace = true alloy-primitives.workspace = true alloy-op-evm.workspace = true op-alloy-consensus.workspace = true +op-alloy-rpc-types-engine.workspace = true alloy-consensus.workspace = true # Optimism -reth-optimism-consensus.workspace = true reth-optimism-chainspec.workspace = true +reth-optimism-consensus.workspace = true reth-optimism-forks.workspace = true reth-optimism-primitives.workspace = true # Mantle reth-mantle-forks.workspace = true +tracing.workspace = true # revm revm.workspace = true @@ -45,9 +50,7 @@ thiserror.workspace = true [dev-dependencies] reth-evm = { workspace = true, features = ["test-utils"] } reth-revm = { workspace = true, features = ["test-utils"] } -reth-optimism-chainspec.workspace = true alloy-genesis.workspace = true -alloy-consensus.workspace = true reth-optimism-primitives = { workspace = true, features = ["arbitrary"] } [features] @@ -73,5 +76,8 @@ std = [ "alloy-op-evm/std", "op-revm/std", "reth-evm/std", + "op-alloy-rpc-types-engine/std", + "reth-storage-errors/std", ] portable = ["reth-revm/portable"] +rpc = ["reth-rpc-eth-api", "reth-optimism-primitives/serde", "reth-optimism-primitives/reth-codec"] diff --git a/crates/optimism/evm/src/build.rs b/crates/optimism/evm/src/build.rs index 8bbefb694c8..e7ea7c29a1d 100644 --- a/crates/optimism/evm/src/build.rs +++ b/crates/optimism/evm/src/build.rs @@ -15,6 +15,7 @@ use reth_optimism_forks::OpHardforks; use reth_optimism_primitives::DepositReceipt; use reth_primitives_traits::{Receipt, SignedTransaction}; use reth_mantle_forks::MantleHardforks; +use revm::context::Block as _; /// Block builder for Optimism. #[derive(Debug)] @@ -29,39 +30,32 @@ impl OpBlockAssembler { } } -impl Clone for OpBlockAssembler { - fn clone(&self) -> Self { - Self { chain_spec: self.chain_spec.clone() } - } -} - -impl BlockAssembler for OpBlockAssembler -where - ChainSpec: OpHardforks + MantleHardforks, - F: for<'a> BlockExecutorFactory< - ExecutionCtx<'a> = OpBlockExecutionCtx, - Transaction: SignedTransaction, - Receipt: Receipt + DepositReceipt, - >, -{ - type Block = alloy_consensus::Block; - - fn assemble_block( +impl OpBlockAssembler { + /// Builds a block for `input` without any bounds on header `H`. + pub fn assemble_block< + F: for<'a> BlockExecutorFactory< + ExecutionCtx<'a>: Into, + Transaction: SignedTransaction, + Receipt: Receipt + DepositReceipt, + >, + H, + >( &self, - input: BlockAssemblerInput<'_, '_, F>, - ) -> Result { + input: BlockAssemblerInput<'_, '_, F, H>, + ) -> Result, BlockExecutionError> { let BlockAssemblerInput { evm_env, execution_ctx: ctx, transactions, - output: BlockExecutionResult { receipts, gas_used, .. }, + output: BlockExecutionResult { receipts, gas_used, blob_gas_used, requests: _ }, bundle_state, state_root, state_provider, .. } = input; + let ctx = ctx.into(); - let timestamp = evm_env.block_env.timestamp; + let timestamp = evm_env.block_env.timestamp().saturating_to(); let transactions_root = proofs::calculate_transaction_root(&transactions); let receipts_root = @@ -89,6 +83,12 @@ where let (excess_blob_gas, blob_gas_used) = if self.chain_spec.is_skadi_active_at_timestamp(timestamp) { (Some(0), Some(0)) + } else if self.chain_spec.is_jovian_active_at_timestamp(timestamp) { + // In jovian, we're using the blob gas used field to store the current da + // footprint's value. + (Some(0), Some(*blob_gas_used)) + } else if self.chain_spec.is_ecotone_active_at_timestamp(timestamp) { + (Some(0), Some(0)) } else { (None, None) }; @@ -96,19 +96,19 @@ where let header = Header { parent_hash: ctx.parent_hash, ommers_hash: EMPTY_OMMER_ROOT_HASH, - beneficiary: evm_env.block_env.beneficiary, + beneficiary: evm_env.block_env.beneficiary(), state_root, transactions_root, receipts_root, withdrawals_root, logs_bloom, timestamp, - mix_hash: evm_env.block_env.prevrandao.unwrap_or_default(), + mix_hash: evm_env.block_env.prevrandao().unwrap_or_default(), nonce: BEACON_NONCE.into(), - base_fee_per_gas: Some(evm_env.block_env.basefee), - number: evm_env.block_env.number, - gas_limit: evm_env.block_env.gas_limit, - difficulty: evm_env.block_env.difficulty, + base_fee_per_gas: Some(evm_env.block_env.basefee()), + number: evm_env.block_env.number().saturating_to(), + gas_limit: evm_env.block_env.gas_limit(), + difficulty: evm_env.block_env.difficulty(), gas_used: *gas_used, extra_data: ctx.extra_data, parent_beacon_block_root: ctx.parent_beacon_block_root, @@ -130,3 +130,28 @@ where )) } } + +impl Clone for OpBlockAssembler { + fn clone(&self) -> Self { + Self { chain_spec: self.chain_spec.clone() } + } +} + +impl BlockAssembler for OpBlockAssembler +where + ChainSpec: OpHardforks, + F: for<'a> BlockExecutorFactory< + ExecutionCtx<'a> = OpBlockExecutionCtx, + Transaction: SignedTransaction, + Receipt: Receipt + DepositReceipt, + >, +{ + type Block = Block; + + fn assemble_block( + &self, + input: BlockAssemblerInput<'_, '_, F>, + ) -> Result { + self.assemble_block(input) + } +} diff --git a/crates/optimism/evm/src/config.rs b/crates/optimism/evm/src/config.rs index c5acdeeff6a..851da82f949 100644 --- a/crates/optimism/evm/src/config.rs +++ b/crates/optimism/evm/src/config.rs @@ -1,7 +1,7 @@ -use alloy_consensus::BlockHeader; -use op_revm::OpSpecId; +pub use alloy_op_evm::{ + spec as revm_spec, spec_by_timestamp_after_bedrock as revm_spec_by_timestamp_after_bedrock, +}; use revm::primitives::{Address, Bytes, B256}; -use reth_mantle_forks::MantleHardforks; /// Context relevant for execution of a next block w.r.t OP. #[derive(Debug, Clone, PartialEq, Eq)] @@ -20,131 +20,18 @@ pub struct OpNextBlockEnvAttributes { pub extra_data: Bytes, } -/// Map the latest active hardfork at the given header to a revm [`OpSpecId`]. -pub fn revm_spec(chain_spec: impl MantleHardforks, header: impl BlockHeader) -> OpSpecId { - revm_spec_by_timestamp_after_bedrock(chain_spec, header.timestamp()) -} - -/// Returns the revm [`OpSpecId`] at the given timestamp. -/// -/// # Note -/// -/// This is only intended to be used after the Bedrock, when hardforks are activated by -/// timestamp. -pub fn revm_spec_by_timestamp_after_bedrock( - chain_spec: impl MantleHardforks, - timestamp: u64, -) -> OpSpecId { - if chain_spec.is_skadi_active_at_timestamp(timestamp) { - OpSpecId::OSAKA - } else if chain_spec.is_interop_active_at_timestamp(timestamp) { - OpSpecId::INTEROP - } else if chain_spec.is_isthmus_active_at_timestamp(timestamp) { - OpSpecId::ISTHMUS - } else if chain_spec.is_holocene_active_at_timestamp(timestamp) { - OpSpecId::HOLOCENE - } else if chain_spec.is_granite_active_at_timestamp(timestamp) { - OpSpecId::GRANITE - } else if chain_spec.is_fjord_active_at_timestamp(timestamp) { - OpSpecId::FJORD - } else if chain_spec.is_ecotone_active_at_timestamp(timestamp) { - OpSpecId::ECOTONE - } else if chain_spec.is_canyon_active_at_timestamp(timestamp) { - OpSpecId::CANYON - } else if chain_spec.is_regolith_active_at_timestamp(timestamp) { - OpSpecId::REGOLITH - } else { - OpSpecId::BEDROCK - } -} - -#[cfg(test)] -mod tests { - use super::*; - use alloy_consensus::Header; - use reth_chainspec::ChainSpecBuilder; - use reth_optimism_chainspec::{OpChainSpec, OpChainSpecBuilder}; - - #[test] - fn test_revm_spec_by_timestamp_after_merge() { - #[inline(always)] - fn op_cs(f: impl FnOnce(OpChainSpecBuilder) -> OpChainSpecBuilder) -> OpChainSpec { - let cs = ChainSpecBuilder::mainnet().chain(reth_chainspec::Chain::from_id(10)).into(); - f(cs).build() +#[cfg(feature = "rpc")] +impl reth_rpc_eth_api::helpers::pending_block::BuildPendingEnv + for OpNextBlockEnvAttributes +{ + fn build_pending_env(parent: &crate::SealedHeader) -> Self { + Self { + timestamp: parent.timestamp().saturating_add(12), + suggested_fee_recipient: parent.beneficiary(), + prev_randao: B256::random(), + gas_limit: parent.gas_limit(), + parent_beacon_block_root: parent.parent_beacon_block_root(), + extra_data: parent.extra_data().clone(), } - assert_eq!( - revm_spec_by_timestamp_after_bedrock(op_cs(|cs| cs.interop_activated()), 0), - OpSpecId::INTEROP - ); - assert_eq!( - revm_spec_by_timestamp_after_bedrock(op_cs(|cs| cs.isthmus_activated()), 0), - OpSpecId::ISTHMUS - ); - assert_eq!( - revm_spec_by_timestamp_after_bedrock(op_cs(|cs| cs.holocene_activated()), 0), - OpSpecId::HOLOCENE - ); - assert_eq!( - revm_spec_by_timestamp_after_bedrock(op_cs(|cs| cs.granite_activated()), 0), - OpSpecId::GRANITE - ); - assert_eq!( - revm_spec_by_timestamp_after_bedrock(op_cs(|cs| cs.fjord_activated()), 0), - OpSpecId::FJORD - ); - assert_eq!( - revm_spec_by_timestamp_after_bedrock(op_cs(|cs| cs.ecotone_activated()), 0), - OpSpecId::ECOTONE - ); - assert_eq!( - revm_spec_by_timestamp_after_bedrock(op_cs(|cs| cs.canyon_activated()), 0), - OpSpecId::CANYON - ); - assert_eq!( - revm_spec_by_timestamp_after_bedrock(op_cs(|cs| cs.bedrock_activated()), 0), - OpSpecId::BEDROCK - ); - assert_eq!( - revm_spec_by_timestamp_after_bedrock(op_cs(|cs| cs.regolith_activated()), 0), - OpSpecId::REGOLITH - ); } - - #[test] - fn test_to_revm_spec() { - #[inline(always)] - fn op_cs(f: impl FnOnce(OpChainSpecBuilder) -> OpChainSpecBuilder) -> OpChainSpec { - let cs = ChainSpecBuilder::mainnet().chain(reth_chainspec::Chain::from_id(10)).into(); - f(cs).build() - } - assert_eq!( - revm_spec(op_cs(|cs| cs.isthmus_activated()), Header::default()), - OpSpecId::ISTHMUS - ); - assert_eq!( - revm_spec(op_cs(|cs| cs.holocene_activated()), Header::default()), - OpSpecId::HOLOCENE - ); - assert_eq!( - revm_spec(op_cs(|cs| cs.granite_activated()), Header::default()), - OpSpecId::GRANITE - ); - assert_eq!(revm_spec(op_cs(|cs| cs.fjord_activated()), Header::default()), OpSpecId::FJORD); - assert_eq!( - revm_spec(op_cs(|cs| cs.ecotone_activated()), Header::default()), - OpSpecId::ECOTONE - ); - assert_eq!( - revm_spec(op_cs(|cs| cs.canyon_activated()), Header::default()), - OpSpecId::CANYON - ); - assert_eq!( - revm_spec(op_cs(|cs| cs.bedrock_activated()), Header::default()), - OpSpecId::BEDROCK - ); - assert_eq!( - revm_spec(op_cs(|cs| cs.regolith_activated()), Header::default()), - OpSpecId::REGOLITH - ); - } -} +} \ No newline at end of file diff --git a/crates/optimism/evm/src/error.rs b/crates/optimism/evm/src/error.rs index 9b694243fac..1a8e76c1490 100644 --- a/crates/optimism/evm/src/error.rs +++ b/crates/optimism/evm/src/error.rs @@ -38,6 +38,9 @@ pub enum L1BlockInfoError { /// Operator fee constant conversion error #[error("could not convert operator fee constant")] OperatorFeeConstantConversion, + /// DA foootprint gas scalar constant conversion error + #[error("could not convert DA footprint gas scalar constant")] + DaFootprintGasScalarConversion, /// Optimism hardforks not active #[error("Optimism hardforks are not active")] HardforksNotActive, diff --git a/crates/optimism/evm/src/execute.rs b/crates/optimism/evm/src/execute.rs index 43ffae09125..ff8a72dc82a 100644 --- a/crates/optimism/evm/src/execute.rs +++ b/crates/optimism/evm/src/execute.rs @@ -5,15 +5,15 @@ pub type OpExecutorProvider = crate::OpEvmConfig; #[cfg(test)] mod tests { - use crate::{OpChainSpec, OpEvmConfig, OpRethReceiptBuilder}; + use crate::{OpEvmConfig, OpRethReceiptBuilder}; use alloc::sync::Arc; use alloy_consensus::{Block, BlockBody, Header, SignableTransaction, TxEip1559}; use alloy_primitives::{b256, Address, Signature, StorageKey, StorageValue, U256}; use op_alloy_consensus::TxDeposit; use op_revm::constants::L1_BLOCK_CONTRACT; use reth_chainspec::MIN_TRANSACTION_GAS; - use reth_evm::{execute::Executor, ConfigureEvm}; - use reth_optimism_chainspec::OpChainSpecBuilder; + use reth_evm::execute::{BasicBlockExecutor, Executor}; + use reth_optimism_chainspec::{OpChainSpec, OpChainSpecBuilder}; use reth_optimism_primitives::{OpReceipt, OpTransactionSigned}; use reth_primitives_traits::{Account, RecoveredBlock}; use reth_revm::{database::StateProviderDatabase, test_utils::StateProviderTest}; @@ -90,7 +90,7 @@ mod tests { .into(); let provider = evm_config(chain_spec); - let mut executor = provider.batch_executor(StateProviderDatabase::new(&db)); + let mut executor = BasicBlockExecutor::new(provider, StateProviderDatabase::new(&db)); // make sure the L1 block contract state is preloaded. executor.with_state_mut(|state| { @@ -163,7 +163,7 @@ mod tests { .into(); let provider = evm_config(chain_spec); - let mut executor = provider.batch_executor(StateProviderDatabase::new(&db)); + let mut executor = BasicBlockExecutor::new(provider, StateProviderDatabase::new(&db)); // make sure the L1 block contract state is preloaded. executor.with_state_mut(|state| { diff --git a/crates/optimism/evm/src/l1.rs b/crates/optimism/evm/src/l1.rs index 185186a8fb5..5557b18402a 100644 --- a/crates/optimism/evm/src/l1.rs +++ b/crates/optimism/evm/src/l1.rs @@ -2,7 +2,7 @@ use crate::{error::L1BlockInfoError, revm_spec_by_timestamp_after_bedrock, OpBlockExecutionError}; use alloy_consensus::Transaction; -use alloy_primitives::{hex, U256}; +use alloy_primitives::{hex, U16, U256}; use op_revm::L1BlockInfo; use reth_execution_errors::BlockExecutionError; use reth_mantle_forks::MantleHardforks; @@ -14,6 +14,10 @@ const L1_BLOCK_ECOTONE_SELECTOR: [u8; 4] = hex!("440a5e20"); /// The function selector of the "setL1BlockValuesIsthmus" function in the `L1Block` contract. const L1_BLOCK_ISTHMUS_SELECTOR: [u8; 4] = hex!("098999be"); +/// The function selector of the "setL1BlockValuesJovian" function in the `L1Block` contract. +/// This is the first 4 bytes of `keccak256("setL1BlockValuesJovian()")`. +const L1_BLOCK_JOVIAN_SELECTOR: [u8; 4] = hex!("3db6be2b"); + /// Extracts the [`L1BlockInfo`] from the L2 block. The L1 info transaction is always the first /// transaction in the L2 block. /// @@ -52,11 +56,14 @@ pub fn extract_l1_info_from_tx( /// If the input is shorter than 4 bytes. pub fn parse_l1_info(input: &[u8]) -> Result { // Parse the L1 info transaction into an L1BlockInfo struct, depending on the function selector. - // There are currently 3 variants: + // There are currently 4 variants: + // - Jovian // - Isthmus // - Ecotone // - Bedrock - if input[0..4] == L1_BLOCK_ISTHMUS_SELECTOR { + if input[0..4] == L1_BLOCK_JOVIAN_SELECTOR { + parse_l1_info_tx_jovian(input[4..].as_ref()) + } else if input[0..4] == L1_BLOCK_ISTHMUS_SELECTOR { parse_l1_info_tx_isthmus(input[4..].as_ref()) } else if input[0..4] == L1_BLOCK_ECOTONE_SELECTOR { parse_l1_info_tx_ecotone(input[4..].as_ref()) @@ -88,12 +95,12 @@ pub fn parse_l1_info_tx_bedrock(data: &[u8]) -> Result Result Result Result { + if data.len() != 174 { + return Err(OpBlockExecutionError::L1BlockInfo(L1BlockInfoError::UnexpectedCalldataLength)); + } + + // https://github.com/ethereum-optimism/op-geth/blob/60038121c7571a59875ff9ed7679c48c9f73405d/core/types/rollup_cost.go#L317-L328 + // + // data layout assumed for Ecotone: + // offset type varname + // 0 + // 4 uint32 _basefeeScalar (start offset in this scope) + // 8 uint32 _blobBaseFeeScalar + // 12 uint64 _sequenceNumber, + // 20 uint64 _timestamp, + // 28 uint64 _l1BlockNumber + // 36 uint256 _basefee, + // 68 uint256 _blobBaseFee, + // 100 bytes32 _hash, + // 132 bytes32 _batcherHash, + // 164 uint32 _operatorFeeScalar + // 168 uint64 _operatorFeeConstant + // 176 uint16 _daFootprintGasScalar + + let l1_base_fee_scalar = U256::try_from_be_slice(&data[..4]) + .ok_or(OpBlockExecutionError::L1BlockInfo(L1BlockInfoError::BaseFeeScalarConversion))?; + let l1_blob_base_fee_scalar = U256::try_from_be_slice(&data[4..8]).ok_or({ + OpBlockExecutionError::L1BlockInfo(L1BlockInfoError::BlobBaseFeeScalarConversion) + })?; + let l1_base_fee = U256::try_from_be_slice(&data[32..64]) + .ok_or(OpBlockExecutionError::L1BlockInfo(L1BlockInfoError::BaseFeeConversion))?; + let l1_blob_base_fee = U256::try_from_be_slice(&data[64..96]) + .ok_or(OpBlockExecutionError::L1BlockInfo(L1BlockInfoError::BlobBaseFeeConversion))?; + let operator_fee_scalar = U256::try_from_be_slice(&data[160..164]).ok_or({ + OpBlockExecutionError::L1BlockInfo(L1BlockInfoError::OperatorFeeScalarConversion) + })?; + let operator_fee_constant = U256::try_from_be_slice(&data[164..172]).ok_or({ + OpBlockExecutionError::L1BlockInfo(L1BlockInfoError::OperatorFeeConstantConversion) + })?; + let da_footprint_gas_scalar: u16 = U16::try_from_be_slice(&data[172..174]) + .ok_or({ + OpBlockExecutionError::L1BlockInfo(L1BlockInfoError::DaFootprintGasScalarConversion) + })? + .to(); - Ok(l1block) + Ok(L1BlockInfo { + l1_base_fee, + l1_base_fee_scalar, + l1_blob_base_fee: Some(l1_blob_base_fee), + l1_blob_base_fee_scalar: Some(l1_blob_base_fee_scalar), + operator_fee_scalar: Some(operator_fee_scalar), + operator_fee_constant: Some(operator_fee_constant), + da_footprint_gas_scalar: Some(da_footprint_gas_scalar), + ..Default::default() + }) } /// An extension trait for [`L1BlockInfo`] that allows us to calculate the L1 cost of a transaction @@ -276,6 +352,7 @@ mod tests { use super::*; use alloy_consensus::{Block, BlockBody}; use alloy_eips::eip2718::Decodable2718; + use alloy_primitives::keccak256; use reth_optimism_chainspec::OP_MAINNET; use reth_optimism_forks::OpHardforks; use reth_optimism_primitives::OpTransactionSigned; @@ -298,5 +375,143 @@ mod tests { assert_eq!(l1_info.l1_base_fee, U256::from(652_114)); assert_eq!(l1_info.l1_fee_overhead, Some(U256::from(2100))); assert_eq!(l1_info.l1_base_fee_scalar, U256::from(1_000_000)); + assert_eq!(l1_info.l1_blob_base_fee, None); + assert_eq!(l1_info.l1_blob_base_fee_scalar, None); + } + + #[test] + fn test_verify_set_jovian() { + let hash = &keccak256("setL1BlockValuesJovian()")[..4]; + assert_eq!(hash, L1_BLOCK_JOVIAN_SELECTOR) + } + + #[test] + fn sanity_l1_block_ecotone() { + // rig + + // OP mainnet ecotone block 118024092 + // + const TIMESTAMP: u64 = 1711603765; + assert!(OP_MAINNET.is_ecotone_active_at_timestamp(TIMESTAMP)); + + // First transaction in OP mainnet block 118024092 + // + // https://optimistic.etherscan.io/getRawTx?tx=0x88501da5d5ca990347c2193be90a07037af1e3820bb40774c8154871c7669150 + const TX: [u8; 251] = hex!( + "7ef8f8a0a539eb753df3b13b7e386e147d45822b67cb908c9ddc5618e3dbaa22ed00850b94deaddeaddeaddeaddeaddeaddeaddeaddead00019442000000000000000000000000000000000000158080830f424080b8a4440a5e2000000558000c5fc50000000000000000000000006605a89f00000000012a10d90000000000000000000000000000000000000000000000000000000af39ac3270000000000000000000000000000000000000000000000000000000d5ea528d24e582fa68786f080069bdbfe06a43f8e67bfd31b8e4d8a8837ba41da9a82a54a0000000000000000000000006887246668a3b87f54deb3b94ba47a6f63f32985" + ); + + let tx = OpTransactionSigned::decode_2718(&mut TX.as_slice()).unwrap(); + let block: Block = Block { + body: BlockBody { transactions: vec![tx], ..Default::default() }, + ..Default::default() + }; + + // expected l1 block info + let expected_l1_base_fee = U256::from_be_bytes(hex!( + "0000000000000000000000000000000000000000000000000000000af39ac327" // 47036678951 + )); + let expected_l1_base_fee_scalar = U256::from(1368); + let expected_l1_blob_base_fee = U256::from_be_bytes(hex!( + "0000000000000000000000000000000000000000000000000000000d5ea528d2" // 57422457042 + )); + let expected_l1_blob_base_fee_scalar = U256::from(810949); + + // test + + let l1_block_info: L1BlockInfo = extract_l1_info(&block.body).unwrap(); + + assert_eq!(l1_block_info.l1_base_fee, expected_l1_base_fee); + assert_eq!(l1_block_info.l1_base_fee_scalar, expected_l1_base_fee_scalar); + assert_eq!(l1_block_info.l1_blob_base_fee, Some(expected_l1_blob_base_fee)); + assert_eq!(l1_block_info.l1_blob_base_fee_scalar, Some(expected_l1_blob_base_fee_scalar)); + } + + #[test] + fn parse_l1_info_fjord() { + // rig + + // L1 block info for OP mainnet block 124665056 (stored in input of tx at index 0) + // + // https://optimistic.etherscan.io/tx/0x312e290cf36df704a2217b015d6455396830b0ce678b860ebfcc30f41403d7b1 + const DATA: &[u8] = &hex!( + "440a5e200000146b000f79c500000000000000040000000066d052e700000000013ad8a3000000000000000000000000000000000000000000000000000000003ef1278700000000000000000000000000000000000000000000000000000000000000012fdf87b89884a61e74b322bbcf60386f543bfae7827725efaaf0ab1de2294a590000000000000000000000006887246668a3b87f54deb3b94ba47a6f63f32985" + ); + + // expected l1 block info verified against expected l1 fee for tx. l1 tx fee listed on OP + // mainnet block scanner + // + // https://github.com/bluealloy/revm/blob/fa5650ee8a4d802f4f3557014dd157adfb074460/crates/revm/src/optimism/l1block.rs#L414-L443 + let l1_base_fee = U256::from(1055991687); + let l1_base_fee_scalar = U256::from(5227); + let l1_blob_base_fee = Some(U256::from(1)); + let l1_blob_base_fee_scalar = Some(U256::from(1014213)); + + // test + + let l1_block_info = parse_l1_info(DATA).unwrap(); + + assert_eq!(l1_block_info.l1_base_fee, l1_base_fee); + assert_eq!(l1_block_info.l1_base_fee_scalar, l1_base_fee_scalar); + assert_eq!(l1_block_info.l1_blob_base_fee, l1_blob_base_fee); + assert_eq!(l1_block_info.l1_blob_base_fee_scalar, l1_blob_base_fee_scalar); + } + + #[test] + fn parse_l1_info_isthmus() { + // rig + + // L1 block info from a devnet with Isthmus activated + const DATA: &[u8] = &hex!( + "098999be00000558000c5fc500000000000000030000000067a9f765000000000000002900000000000000000000000000000000000000000000000000000000006a6d09000000000000000000000000000000000000000000000000000000000000000172fcc8e8886636bdbe96ba0e4baab67ea7e7811633f52b52e8cf7a5123213b6f000000000000000000000000d3f2c5afb2d76f5579f326b0cd7da5f5a4126c3500004e2000000000000001f4" + ); + + // expected l1 block info verified against expected l1 fee and operator fee for tx. + let l1_base_fee = U256::from(6974729); + let l1_base_fee_scalar = U256::from(1368); + let l1_blob_base_fee = Some(U256::from(1)); + let l1_blob_base_fee_scalar = Some(U256::from(810949)); + let operator_fee_scalar = Some(U256::from(20000)); + let operator_fee_constant = Some(U256::from(500)); + + // test + + let l1_block_info = parse_l1_info(DATA).unwrap(); + + assert_eq!(l1_block_info.l1_base_fee, l1_base_fee); + assert_eq!(l1_block_info.l1_base_fee_scalar, l1_base_fee_scalar); + assert_eq!(l1_block_info.l1_blob_base_fee, l1_blob_base_fee); + assert_eq!(l1_block_info.l1_blob_base_fee_scalar, l1_blob_base_fee_scalar); + assert_eq!(l1_block_info.operator_fee_scalar, operator_fee_scalar); + assert_eq!(l1_block_info.operator_fee_constant, operator_fee_constant); + } + + #[test] + fn parse_l1_info_jovian() { + // L1 block info from a devnet with Isthmus activated + const DATA: &[u8] = &hex!( + "3db6be2b00000558000c5fc500000000000000030000000067a9f765000000000000002900000000000000000000000000000000000000000000000000000000006a6d09000000000000000000000000000000000000000000000000000000000000000172fcc8e8886636bdbe96ba0e4baab67ea7e7811633f52b52e8cf7a5123213b6f000000000000000000000000d3f2c5afb2d76f5579f326b0cd7da5f5a4126c3500004e2000000000000001f4dead" + ); + + // expected l1 block info verified against expected l1 fee and operator fee for tx. + let l1_base_fee = U256::from(6974729); + let l1_base_fee_scalar = U256::from(1368); + let l1_blob_base_fee = Some(U256::from(1)); + let l1_blob_base_fee_scalar = Some(U256::from(810949)); + let operator_fee_scalar = Some(U256::from(20000)); + let operator_fee_constant = Some(U256::from(500)); + let da_footprint_gas_scalar: Option = Some(U16::from(0xdead).to()); + + // test + + let l1_block_info = parse_l1_info(DATA).unwrap(); + + assert_eq!(l1_block_info.l1_base_fee, l1_base_fee); + assert_eq!(l1_block_info.l1_base_fee_scalar, l1_base_fee_scalar); + assert_eq!(l1_block_info.l1_blob_base_fee, l1_blob_base_fee); + assert_eq!(l1_block_info.l1_blob_base_fee_scalar, l1_blob_base_fee_scalar); + assert_eq!(l1_block_info.operator_fee_scalar, operator_fee_scalar); + assert_eq!(l1_block_info.operator_fee_constant, operator_fee_constant); + assert_eq!(l1_block_info.da_footprint_gas_scalar, da_footprint_gas_scalar); } } diff --git a/crates/optimism/evm/src/lib.rs b/crates/optimism/evm/src/lib.rs index 18bca42e1dc..4de10cc07c8 100644 --- a/crates/optimism/evm/src/lib.rs +++ b/crates/optimism/evm/src/lib.rs @@ -5,7 +5,7 @@ html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256", issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(feature = "std"), no_std)] #![cfg_attr(not(test), warn(unused_crate_dependencies))] @@ -13,25 +13,27 @@ extern crate alloc; use alloc::sync::Arc; use alloy_consensus::{BlockHeader, Header}; -use alloy_evm::{FromRecoveredTx, FromTxWithEncoded}; -use alloy_op_evm::{block::receipt_builder::OpReceiptBuilder, OpBlockExecutionCtx}; -use alloy_primitives::U256; +use alloy_eips::Decodable2718; +use alloy_evm::{EvmFactory, FromRecoveredTx, FromTxWithEncoded}; +use alloy_op_evm::block::{receipt_builder::OpReceiptBuilder, OpTxEnv}; use core::fmt::Debug; use op_alloy_consensus::EIP1559ParamError; +use op_alloy_rpc_types_engine::OpExecutionData; use op_revm::{OpSpecId, OpTransaction}; use reth_chainspec::EthChainSpec; -use reth_evm::{ConfigureEvm, EvmEnv}; +use reth_evm::{ + precompiles::PrecompilesMap, ConfigureEngineEvm, ConfigureEvm, EvmEnv, EvmEnvFor, + ExecutableTxIterator, ExecutionCtxFor, TransactionEnv, +}; +use reth_mantle_forks::MantleHardforks; use reth_optimism_chainspec::OpChainSpec; -use reth_optimism_consensus::next_block_base_fee; use reth_optimism_forks::OpHardforks; use reth_optimism_primitives::{DepositReceipt, OpPrimitives}; -use reth_mantle_forks::MantleHardforks; -use reth_primitives_traits::{NodePrimitives, SealedBlock, SealedHeader, SignedTransaction}; -use revm::{ - context::{BlockEnv, CfgEnv, TxEnv}, - context_interface::block::BlobExcessGasAndPrice, - primitives::hardfork::SpecId, +use reth_primitives_traits::{ + NodePrimitives, SealedBlock, SealedHeader, SignedTransaction, TxTy, WithEncoded, }; +use reth_storage_errors::any::AnyError; +use revm::context::{BlockEnv, TxEnv}; mod config; pub use config::{revm_spec, revm_spec_by_timestamp_after_bedrock, OpNextBlockEnvAttributes}; @@ -47,7 +49,10 @@ pub use build::OpBlockAssembler; mod error; pub use error::OpBlockExecutionError; -pub use alloy_op_evm::{OpBlockExecutorFactory, OpEvm, OpEvmFactory}; +mod mantle; +pub(crate) use mantle::MantleEvmEnvInput; + +pub use alloy_op_evm::{OpBlockExecutionCtx, OpBlockExecutorFactory, OpEvm, OpEvmFactory}; /// Optimism-related EVM configuration. #[derive(Debug)] @@ -55,15 +60,19 @@ pub struct OpEvmConfig< ChainSpec = OpChainSpec, N: NodePrimitives = OpPrimitives, R = OpRethReceiptBuilder, + EvmFactory = OpEvmFactory, > { /// Inner [`OpBlockExecutorFactory`]. - pub executor_factory: OpBlockExecutorFactory>, + pub executor_factory: OpBlockExecutorFactory, EvmFactory>, /// Optimism block assembler. pub block_assembler: OpBlockAssembler, - _pd: core::marker::PhantomData, + #[doc(hidden)] + pub _pd: core::marker::PhantomData, } -impl Clone for OpEvmConfig { +impl Clone + for OpEvmConfig +{ fn clone(&self) -> Self { Self { executor_factory: self.executor_factory.clone(), @@ -73,14 +82,14 @@ impl Clone for OpEvmConfig OpEvmConfig { +impl OpEvmConfig { /// Creates a new [`OpEvmConfig`] with the given chain spec for OP chains. pub fn optimism(chain_spec: Arc) -> Self { Self::new(chain_spec, OpRethReceiptBuilder::default()) } } -impl OpEvmConfig { +impl OpEvmConfig { /// Creates a new [`OpEvmConfig`] with the given chain spec. pub fn new(chain_spec: Arc, receipt_builder: R) -> Self { Self { @@ -93,16 +102,22 @@ impl OpEvmConfig { _pd: core::marker::PhantomData, } } +} +impl OpEvmConfig +where + ChainSpec: OpHardforks, + N: NodePrimitives, +{ /// Returns the chain spec associated with this configuration. pub const fn chain_spec(&self) -> &Arc { self.executor_factory.spec() } } -impl ConfigureEvm for OpEvmConfig +impl ConfigureEvm for OpEvmConfig where - ChainSpec: EthChainSpec + OpHardforks + MantleHardforks, + ChainSpec: EthChainSpec
+ OpHardforks + MantleHardforks, N: NodePrimitives< Receipt = R::Receipt, SignedTx = R::Transaction, @@ -112,12 +127,21 @@ where >, OpTransaction: FromRecoveredTx + FromTxWithEncoded, R: OpReceiptBuilder, + EvmF: EvmFactory< + Tx: FromRecoveredTx + + FromTxWithEncoded + + TransactionEnv + + OpTxEnv, + Precompiles = PrecompilesMap, + Spec = OpSpecId, + BlockEnv = BlockEnv, + > + Debug, Self: Send + Sync + Unpin + Clone + 'static, { type Primitives = N; type Error = EIP1559ParamError; type NextBlockEnvCtx = OpNextBlockEnvAttributes; - type BlockExecutorFactory = OpBlockExecutorFactory>; + type BlockExecutorFactory = OpBlockExecutorFactory, EvmF>; type BlockAssembler = OpBlockAssembler; fn block_executor_factory(&self) -> &Self::BlockExecutorFactory { @@ -128,34 +152,8 @@ where &self.block_assembler } - fn evm_env(&self, header: &Header) -> EvmEnv { - let spec = config::revm_spec(self.chain_spec(), header); - - let cfg_env = CfgEnv::new().with_chain_id(self.chain_spec().chain().id()).with_spec(spec); - - let block_env = BlockEnv { - number: header.number(), - beneficiary: header.beneficiary(), - timestamp: header.timestamp(), - difficulty: if spec.into_eth_spec() >= SpecId::MERGE { - U256::ZERO - } else { - header.difficulty() - }, - prevrandao: if spec.into_eth_spec() >= SpecId::MERGE { - header.mix_hash() - } else { - None - }, - gas_limit: header.gas_limit(), - basefee: header.base_fee_per_gas().unwrap_or_default(), - // EIP-4844 excess blob gas of this block, introduced in Cancun - blob_excess_gas_and_price: header.excess_blob_gas().map(|excess_blob_gas| { - BlobExcessGasAndPrice::new(excess_blob_gas, spec.into_eth_spec() >= SpecId::PRAGUE) - }), - }; - - EvmEnv { cfg_env, block_env } + fn evm_env(&self, header: &Header) -> Result, Self::Error> { + Ok(self.for_mantle(MantleEvmEnvInput::from_block_header(header))) } fn next_evm_env( @@ -163,75 +161,99 @@ where parent: &Header, attributes: &Self::NextBlockEnvCtx, ) -> Result, Self::Error> { - // ensure we're not missing any timestamp based hardforks - let spec_id = revm_spec_by_timestamp_after_bedrock(self.chain_spec(), attributes.timestamp); - - // configure evm env based on parent block - let cfg_env = - CfgEnv::new().with_chain_id(self.chain_spec().chain().id()).with_spec(spec_id); - - // if the parent block did not have excess blob gas (i.e. it was pre-cancun), but it is - // cancun now, we need to set the excess blob gas to the default value(0) - let blob_excess_gas_and_price = parent - .maybe_next_block_excess_blob_gas( - self.chain_spec().blob_params_at_timestamp(attributes.timestamp), - ) - .or_else(|| (spec_id.into_eth_spec().is_enabled_in(SpecId::CANCUN)).then_some(0)) - .map(|gas| BlobExcessGasAndPrice::new(gas, false)); - - let block_env = BlockEnv { - number: parent.number() + 1, - beneficiary: attributes.suggested_fee_recipient, - timestamp: attributes.timestamp, - difficulty: U256::ZERO, - prevrandao: Some(attributes.prev_randao), - gas_limit: attributes.gas_limit, - // calculate basefee based on parent block's gas usage - basefee: next_block_base_fee(self.chain_spec(), parent, attributes.timestamp)?, - // calculate excess gas based on parent block's blob gas usage - blob_excess_gas_and_price, - }; + let base_fee = + self.chain_spec().next_block_base_fee(parent, attributes.timestamp).unwrap_or_default(); - Ok(EvmEnv { cfg_env, block_env }) + Ok(self.for_mantle(MantleEvmEnvInput::for_next(parent, attributes, base_fee))) } - fn context_for_block(&self, block: &'_ SealedBlock) -> OpBlockExecutionCtx { - OpBlockExecutionCtx { + fn context_for_block( + &self, + block: &'_ SealedBlock, + ) -> Result { + Ok(OpBlockExecutionCtx { parent_hash: block.header().parent_hash(), parent_beacon_block_root: block.header().parent_beacon_block_root(), extra_data: block.header().extra_data().clone(), - } + }) } fn context_for_next_block( &self, parent: &SealedHeader, attributes: Self::NextBlockEnvCtx, - ) -> OpBlockExecutionCtx { - OpBlockExecutionCtx { + ) -> Result { + Ok(OpBlockExecutionCtx { parent_hash: parent.hash(), parent_beacon_block_root: attributes.parent_beacon_block_root, extra_data: attributes.extra_data, - } + }) } } + +impl ConfigureEngineEvm for OpEvmConfig +where + ChainSpec: EthChainSpec
+ OpHardforks + MantleHardforks, + N: NodePrimitives< + Receipt = R::Receipt, + SignedTx = R::Transaction, + BlockHeader = Header, + BlockBody = alloy_consensus::BlockBody, + Block = alloy_consensus::Block, + >, + OpTransaction: FromRecoveredTx + FromTxWithEncoded, + R: OpReceiptBuilder, + Self: Send + Sync + Unpin + Clone + 'static, +{ + fn evm_env_for_payload( + &self, + payload: &OpExecutionData, + ) -> Result, Self::Error> { + Ok(self.for_mantle(MantleEvmEnvInput::from_op_payload(payload))) + } + + fn context_for_payload<'a>( + &self, + payload: &'a OpExecutionData, + ) -> Result, Self::Error> { + Ok(OpBlockExecutionCtx { + parent_hash: payload.parent_hash(), + parent_beacon_block_root: payload.sidecar.parent_beacon_block_root(), + extra_data: payload.payload.as_v1().extra_data.clone(), + }) + } + + fn tx_iterator_for_payload( + &self, + payload: &OpExecutionData, + ) -> Result, Self::Error> { + Ok(payload.payload.transactions().clone().into_iter().map(|encoded| { + let tx = TxTy::::decode_2718_exact(encoded.as_ref()) + .map_err(AnyError::new)?; + let signer = tx.try_recover().map_err(AnyError::new)?; + Ok::<_, AnyError>(WithEncoded::new(encoded, tx.with_signer(signer))) + })) + } +} + #[cfg(test)] mod tests { use super::*; use alloy_consensus::{Header, Receipt}; use alloy_eips::eip7685::Requests; use alloy_genesis::Genesis; - use alloy_primitives::{bytes, map::HashMap, Address, LogData, B256}; + use alloy_primitives::{bytes, map::HashMap, Address, LogData, B256, U256}; use op_revm::OpSpecId; use reth_chainspec::ChainSpec; use reth_evm::execute::ProviderError; use reth_execution_types::{ AccountRevertInit, BundleStateInit, Chain, ExecutionOutcome, RevertsInit, }; - use reth_optimism_chainspec::BASE_MAINNET; + use reth_optimism_chainspec::{OpChainSpec, BASE_MAINNET}; use reth_optimism_primitives::{OpBlock, OpPrimitives, OpReceipt}; use reth_primitives_traits::{Account, RecoveredBlock}; use revm::{ + context::CfgEnv, database::{BundleState, CacheDB}, database_interface::EmptyDBTyped, inspector::NoOpInspector, @@ -263,7 +285,8 @@ mod tests { // Header, and total difficulty let EvmEnv { cfg_env, .. } = OpEvmConfig::optimism(Arc::new(OpChainSpec { inner: chain_spec.clone() })) - .evm_env(&header); + .evm_env(&header) + .unwrap(); // Assert that the chain ID in the `cfg_env` is correctly set to the chain ID of the // ChainSpec @@ -308,8 +331,12 @@ mod tests { let db = CacheDB::>::default(); // Create customs block and tx env - let block = - BlockEnv { basefee: 1000, gas_limit: 10_000_000, number: 42, ..Default::default() }; + let block = BlockEnv { + basefee: 1000, + gas_limit: 10_000_000, + number: U256::from(42), + ..Default::default() + }; let evm_env = EvmEnv { block_env: block, ..Default::default() }; @@ -369,8 +396,12 @@ mod tests { let db = CacheDB::>::default(); // Create custom block and tx environment - let block = - BlockEnv { basefee: 1000, gas_limit: 10_000_000, number: 42, ..Default::default() }; + let block = BlockEnv { + basefee: 1000, + gas_limit: 10_000_000, + number: U256::from(42), + ..Default::default() + }; let evm_env = EvmEnv { block_env: block, ..Default::default() }; let evm = evm_config.evm_with_env_and_inspector(db, evm_env.clone(), NoOpInspector {}); @@ -464,7 +495,7 @@ mod tests { } #[test] - fn test_initialisation() { + fn test_initialization() { // Create a new BundleState object with initial data let bundle = BundleState::new( vec![(Address::new([2; 20]), None, Some(AccountInfo::default()), HashMap::default())], diff --git a/crates/optimism/evm/src/mantle.rs b/crates/optimism/evm/src/mantle.rs new file mode 100644 index 00000000000..8babd7eac3b --- /dev/null +++ b/crates/optimism/evm/src/mantle.rs @@ -0,0 +1,465 @@ +//! Mantle-specific EVM environment configuration. +//! +//! This module provides Mantle chain support with hardfork awareness, +//! extending the base OP Stack configuration with Mantle-specific features +//! such as the Skadi hardfork. +//! +//! # Design +//! +//! The implementation follows the same pattern as `alloy-evm`: +//! - [`MantleEvmEnvInput`]: Encapsulates block environment parameters +//! - [`OpEvmConfig::for_mantle`]: Core method for EVM environment creation +//! +//! This ensures that Mantle-specific hardforks (like Skadi) are correctly +//! detected in all code paths (Engine API, RPC validation, block execution). + +use alloy_consensus::{BlockHeader, Header}; +use alloy_primitives::U256; +use op_alloy_rpc_types_engine::OpExecutionData; +use op_revm::OpSpecId; +use reth_chainspec::EthChainSpec; +use reth_evm::EvmEnv; +use reth_mantle_forks::MantleHardforks; +use reth_optimism_forks::OpHardforks; +use reth_primitives_traits::NodePrimitives; +use revm::{ + context::{BlockEnv, CfgEnv}, + context_interface::block::BlobExcessGasAndPrice, + primitives::hardfork::SpecId, +}; +use tracing::debug; + +use crate::{config::OpNextBlockEnvAttributes, OpEvmConfig}; + +/// Input parameters for constructing EVM environment on Mantle chains. +/// +/// This structure follows the same design pattern as `EvmEnvInput` from `alloy-evm`, +/// encapsulating block environment parameters with Mantle hardfork awareness. +/// +/// The input can be constructed from various sources: +/// - [`MantleEvmEnvInput::from_block_header`]: From a block header +/// - [`MantleEvmEnvInput::for_next`]: For the next block (from parent + attributes) +/// - [`MantleEvmEnvInput::from_op_payload`]: From an execution payload +/// +/// # Reference +/// Similar to `EvmEnvInput` in `alloy-evm/src/eth/env.rs` +pub(crate) struct MantleEvmEnvInput { + pub(crate) timestamp: u64, + pub(crate) number: u64, + pub(crate) beneficiary: alloy_primitives::Address, + pub(crate) gas_limit: u64, + pub(crate) base_fee_per_gas: u64, + pub(crate) difficulty: U256, + pub(crate) mix_hash: Option, + pub(crate) excess_blob_gas: Option, +} + +impl MantleEvmEnvInput { + /// Create input from a block header. + /// + /// Extracts all necessary EVM environment parameters from the given header. + /// + /// # Example + /// + /// ```rust,ignore + /// let input = MantleEvmEnvInput::from_block_header(&header); + /// let evm_env = config.for_mantle(input); + /// ``` + pub(crate) fn from_block_header(header: &Header) -> Self { + Self { + timestamp: header.timestamp(), + number: header.number(), + beneficiary: header.beneficiary(), + gas_limit: header.gas_limit(), + base_fee_per_gas: header.base_fee_per_gas().unwrap_or_default(), + difficulty: header.difficulty(), + mix_hash: header.mix_hash(), + excess_blob_gas: header.excess_blob_gas(), + } + } + + /// Create input for the next block from parent header and attributes. + /// + /// This is typically used when building a new block. The block number is + /// automatically incremented from the parent, and difficulty is set to zero + /// (as all OP Stack chains are post-merge). + /// + /// # Arguments + /// + /// * `parent` - The parent block header + /// * `attributes` - Next block attributes (timestamp, gas limit, etc.) + /// * `base_fee_per_gas` - Calculated base fee for the next block + /// + /// # Example + /// + /// ```rust,ignore + /// let base_fee = chain_spec.next_block_base_fee(parent, attributes.timestamp)?; + /// let input = MantleEvmEnvInput::for_next(parent, &attributes, base_fee); + /// let evm_env = config.for_mantle(input); + /// ``` + pub(crate) fn for_next( + parent: &Header, + attributes: &OpNextBlockEnvAttributes, + base_fee_per_gas: u64, + ) -> Self { + Self { + timestamp: attributes.timestamp, + number: parent.number() + 1, + beneficiary: attributes.suggested_fee_recipient, + gas_limit: attributes.gas_limit, + base_fee_per_gas, + difficulty: U256::ZERO, + mix_hash: Some(attributes.prev_randao), + excess_blob_gas: None, + } + } + + /// Create input from an OP execution payload. + /// + /// This is used when processing execution payloads from the Engine API. + /// + /// # Example + /// + /// ```rust,ignore + /// let input = MantleEvmEnvInput::from_op_payload(&payload); + /// let evm_env = config.for_mantle(input); + /// ``` + pub(crate) fn from_op_payload(payload: &OpExecutionData) -> Self { + Self { + timestamp: payload.payload.timestamp(), + number: payload.payload.block_number(), + beneficiary: payload.payload.as_v1().fee_recipient, + gas_limit: payload.payload.as_v1().gas_limit, + base_fee_per_gas: payload.payload.as_v1().base_fee_per_gas.to::(), + difficulty: payload.payload.as_v1().prev_randao.into(), + mix_hash: Some(payload.payload.as_v1().prev_randao), + excess_blob_gas: payload.payload.as_v3().map(|v| v.excess_blob_gas), + } + } +} + +impl OpEvmConfig +where + ChainSpec: EthChainSpec
+ OpHardforks + MantleHardforks, + N: NodePrimitives, +{ + /// Creates [`EvmEnv`] for Mantle chains with unified spec determination logic. + /// + /// This is the core method for EVM environment creation, ensuring all code paths + /// use the same spec determination logic via [`MantleHardforks::revm_spec_at_timestamp`]. + /// + /// # Mantle Hardfork Support + /// + /// Unlike the standard OP Stack implementation, this method: + /// - Checks Mantle-specific hardforks (e.g., Skadi) first via the trait method + /// - Falls back to standard OP Stack hardforks if no Mantle fork is active + /// - Returns `OpSpecId::OSAKA` when Skadi is active + /// + /// # Code Paths Using This Method + /// + /// All EVM environment creation paths now use this method: + /// - Engine API payloads: `evm_env_for_payload` → `for_mantle` + /// - RPC validation: `evm_env` → `for_mantle` + /// - Block execution: `evm_env` → `for_mantle` + /// - Next block building: `next_evm_env` → `for_mantle` + /// + /// This ensures consistent hardfork detection across all scenarios. + /// + /// # Arguments + /// + /// * `input` - Block environment parameters encapsulated in [`MantleEvmEnvInput`] + /// + /// # Returns + /// + /// An [`EvmEnv`] configured with the appropriate [`OpSpecId`] for the given timestamp. + /// + /// # Design + /// + /// This follows the same pattern as `for_op` in `alloy-op-evm`, but with Mantle + /// hardfork awareness. See [`MantleHardforks::revm_spec_at_timestamp`] for details + /// on the spec determination logic. + /// + /// # Example + /// + /// ```rust,ignore + /// let input = MantleEvmEnvInput::from_block_header(&header); + /// let evm_env = config.for_mantle(input); + /// // evm_env.cfg_env.spec == OpSpecId::OSAKA if Skadi is active + /// ``` + pub(crate) fn for_mantle(&self, input: MantleEvmEnvInput) -> EvmEnv { + // Use Mantle-aware trait method to determine spec + // This automatically handles both Mantle-specific and OP Stack hardforks + let spec = self.chain_spec().revm_spec_at_timestamp(input.timestamp); + + debug!(spec = ?spec, timestamp = input.timestamp, "Computed Mantle EVM spec"); + + let cfg_env = CfgEnv::new().with_chain_id(self.chain_spec().chain().id()).with_spec(spec); + + // Calculate blob excess gas and price for EIP-4844 (Cancun+) + let blob_excess_gas_and_price = + spec.into_eth_spec().is_enabled_in(SpecId::CANCUN).then(|| { + let excess_blob_gas = input.excess_blob_gas.unwrap_or(0); + let blob_gasprice = alloy_eips::eip4844::calc_blob_gasprice(excess_blob_gas); + BlobExcessGasAndPrice { excess_blob_gas, blob_gasprice } + }); + + let is_merge_active = spec.into_eth_spec() >= SpecId::MERGE; + + let block_env = BlockEnv { + number: U256::from(input.number), + beneficiary: input.beneficiary, + timestamp: U256::from(input.timestamp), + difficulty: if is_merge_active { U256::ZERO } else { input.difficulty }, + prevrandao: if is_merge_active { input.mix_hash } else { None }, + gas_limit: input.gas_limit, + basefee: input.base_fee_per_gas, + blob_excess_gas_and_price, + }; + + EvmEnv { cfg_env, block_env } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use alloy_consensus::Header; + use alloy_primitives::{address, b256}; + use reth_optimism_chainspec::{OpChainSpec, MANTLE_MAINNET}; + + #[test] + fn test_mantle_evm_env_input_from_block_header() { + let header = Header { + number: 1000, + timestamp: 1700000000, + beneficiary: address!("0000000000000000000000000000000000000001"), + gas_limit: 30_000_000, + base_fee_per_gas: Some(1000), + difficulty: U256::from(12345), + excess_blob_gas: Some(131072), + ..Default::default() + }; + + let input = MantleEvmEnvInput::from_block_header(&header); + + assert_eq!(input.timestamp, 1700000000); + assert_eq!(input.number, 1000); + assert_eq!(input.beneficiary, address!("0000000000000000000000000000000000000001")); + assert_eq!(input.gas_limit, 30_000_000); + assert_eq!(input.base_fee_per_gas, 1000); + assert_eq!(input.difficulty, U256::from(12345)); + assert_eq!(input.mix_hash, header.mix_hash()); + assert_eq!(input.excess_blob_gas, Some(131072)); + } + + #[test] + fn test_mantle_evm_env_input_for_next() { + let parent = Header { + number: 999, + timestamp: 1699999998, + beneficiary: address!("0000000000000000000000000000000000000001"), + gas_limit: 30_000_000, + base_fee_per_gas: Some(900), + ..Default::default() + }; + + let attributes = OpNextBlockEnvAttributes { + timestamp: 1700000000, + suggested_fee_recipient: address!("0000000000000000000000000000000000000002"), + prev_randao: b256!("2222222222222222222222222222222222222222222222222222222222222222"), + gas_limit: 30_000_000, + parent_beacon_block_root: None, + extra_data: Default::default(), + }; + + let base_fee = 1000u64; + let input = MantleEvmEnvInput::for_next(&parent, &attributes, base_fee); + + assert_eq!(input.timestamp, 1700000000); + assert_eq!(input.number, 1000); // parent.number + 1 + assert_eq!(input.beneficiary, address!("0000000000000000000000000000000000000002")); + assert_eq!(input.gas_limit, 30_000_000); + assert_eq!(input.base_fee_per_gas, 1000); + assert_eq!(input.difficulty, U256::ZERO); // Always zero for OP Stack chains + assert_eq!( + input.mix_hash, + Some(b256!("2222222222222222222222222222222222222222222222222222222222222222")) + ); + assert_eq!(input.excess_blob_gas, None); + } + + #[test] + fn test_for_mantle_uses_mantle_hardfork_trait() { + use reth_optimism_primitives::OpPrimitives; + + // Create a config with Mantle mainnet spec + let config = OpEvmConfig::::optimism(MANTLE_MAINNET.clone()); + + let header = Header { + number: 1000, + timestamp: 1700000000, // Before Skadi + beneficiary: address!("0000000000000000000000000000000000000001"), + gas_limit: 30_000_000, + base_fee_per_gas: Some(1000), + ..Default::default() + }; + + let input = MantleEvmEnvInput::from_block_header(&header); + let evm_env = config.for_mantle(input); + + // Verify that the spec is determined correctly + assert_eq!(evm_env.cfg_env.chain_id, MANTLE_MAINNET.chain().id()); + + // The spec should be determined by the timestamp + // This test verifies that for_mantle is being called correctly + assert!(evm_env.cfg_env.spec != OpSpecId::OSAKA); // Not Skadi yet at this timestamp + } + + #[test] + fn test_for_mantle_skadi_hardfork_detection() { + use reth_optimism_primitives::OpPrimitives; + + // Create config + let config = OpEvmConfig::::optimism(MANTLE_MAINNET.clone()); + + // Test before Skadi activation + let header_before = Header { + number: 1000, + timestamp: 1600000000, // Before Skadi + beneficiary: address!("0000000000000000000000000000000000000001"), + gas_limit: 30_000_000, + base_fee_per_gas: Some(1000), + ..Default::default() + }; + + let input_before = MantleEvmEnvInput::from_block_header(&header_before); + let evm_env_before = config.for_mantle(input_before); + + // Should not be OSAKA before Skadi + let spec_before = MANTLE_MAINNET.revm_spec_at_timestamp(1600000000); + assert_eq!(evm_env_before.cfg_env.spec, spec_before); + + // Test after Skadi activation (if configured in MANTLE_MAINNET) + // Note: This test demonstrates the pattern; actual activation time depends on chain config + let timestamp_future = 2000000000u64; + if MANTLE_MAINNET.is_skadi_active_at_timestamp(timestamp_future) { + let header_after = Header { + number: 2000, + timestamp: timestamp_future, + beneficiary: address!("0000000000000000000000000000000000000001"), + gas_limit: 30_000_000, + base_fee_per_gas: Some(1000), + ..Default::default() + }; + + let input_after = MantleEvmEnvInput::from_block_header(&header_after); + let evm_env_after = config.for_mantle(input_after); + + // Should be OSAKA when Skadi is active + assert_eq!(evm_env_after.cfg_env.spec, OpSpecId::OSAKA); + } + } + + #[test] + fn test_for_mantle_blob_gas_calculation() { + use reth_optimism_primitives::OpPrimitives; + + let config = OpEvmConfig::::optimism(MANTLE_MAINNET.clone()); + + // Create header with blob gas (post-Cancun scenario) + let excess_blob_gas = 131072u64; + let header = Header { + number: 1000, + timestamp: 1700000000, + beneficiary: address!("0000000000000000000000000000000000000001"), + gas_limit: 30_000_000, + base_fee_per_gas: Some(1000), + excess_blob_gas: Some(excess_blob_gas), + ..Default::default() + }; + + let input = MantleEvmEnvInput::from_block_header(&header); + let evm_env = config.for_mantle(input); + + // If Cancun is active, blob_excess_gas_and_price should be set + let spec = MANTLE_MAINNET.revm_spec_at_timestamp(1700000000); + if spec.into_eth_spec().is_enabled_in(SpecId::CANCUN) { + assert!(evm_env.block_env.blob_excess_gas_and_price.is_some()); + let blob_data = evm_env.block_env.blob_excess_gas_and_price.unwrap(); + assert_eq!(blob_data.excess_blob_gas, excess_blob_gas); + + // Verify blob gas price calculation + let expected_price = alloy_eips::eip4844::calc_blob_gasprice(excess_blob_gas); + assert_eq!(blob_data.blob_gasprice, expected_price); + } + } + + #[test] + fn test_for_mantle_merge_behavior() { + use reth_optimism_primitives::OpPrimitives; + + let config = OpEvmConfig::::optimism(MANTLE_MAINNET.clone()); + + let difficulty = U256::from(12345); + + let header = Header { + number: 1000, + timestamp: 1700000000, + beneficiary: address!("0000000000000000000000000000000000000001"), + gas_limit: 30_000_000, + base_fee_per_gas: Some(1000), + difficulty, + extra_data: Default::default(), + ..Default::default() + }; + + let input = MantleEvmEnvInput::from_block_header(&header); + let evm_env = config.for_mantle(input); + + let spec = MANTLE_MAINNET.revm_spec_at_timestamp(1700000000); + let is_merge_active = spec.into_eth_spec() >= SpecId::MERGE; + + // All OP Stack chains (including Mantle) are post-merge + // So we expect merge behavior + assert!(is_merge_active); + assert_eq!(evm_env.block_env.difficulty, U256::ZERO); + + // prevrandao will be set from header's mix_hash + // For OP Stack, this is typically derived from the beacon chain + } + + #[test] + fn test_mantle_evm_env_input_constructors_consistency() { + // Verify that all three constructors produce valid inputs + + // 1. from_block_header + let header = Header { + number: 1000, + timestamp: 1700000000, + beneficiary: address!("0000000000000000000000000000000000000001"), + gas_limit: 30_000_000, + base_fee_per_gas: Some(1000), + ..Default::default() + }; + let input1 = MantleEvmEnvInput::from_block_header(&header); + assert_eq!(input1.number, 1000); + assert_eq!(input1.timestamp, 1700000000); + + // 2. for_next + let prev_randao = b256!("4444444444444444444444444444444444444444444444444444444444444444"); + let attributes = OpNextBlockEnvAttributes { + timestamp: 1700000002, + suggested_fee_recipient: address!("0000000000000000000000000000000000000002"), + prev_randao, + gas_limit: 30_000_000, + parent_beacon_block_root: None, + extra_data: Default::default(), + }; + let input2 = MantleEvmEnvInput::for_next(&header, &attributes, 1100); + assert_eq!(input2.number, 1001); // parent.number + 1 + assert_eq!(input2.timestamp, 1700000002); + assert_eq!(input2.base_fee_per_gas, 1100); + assert_eq!(input2.difficulty, U256::ZERO); // Always zero for next block + assert_eq!(input2.mix_hash, Some(prev_randao)); + } +} diff --git a/crates/optimism/flashblocks/Cargo.toml b/crates/optimism/flashblocks/Cargo.toml new file mode 100644 index 00000000000..977e28d37e1 --- /dev/null +++ b/crates/optimism/flashblocks/Cargo.toml @@ -0,0 +1,59 @@ +[package] +name = "reth-optimism-flashblocks" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[dependencies] +# reth +reth-optimism-primitives = { workspace = true, features = ["serde"] } +reth-optimism-evm.workspace = true +reth-chain-state = { workspace = true, features = ["serde"] } +reth-primitives-traits = { workspace = true, features = ["serde"] } +reth-engine-primitives = { workspace = true, features = ["std"] } +reth-execution-types = { workspace = true, features = ["serde"] } +reth-evm.workspace = true +reth-revm.workspace = true +reth-optimism-payload-builder.workspace = true +reth-rpc-eth-types.workspace = true +reth-errors.workspace = true +reth-payload-primitives.workspace = true +reth-storage-api.workspace = true +reth-tasks.workspace = true +reth-metrics.workspace = true + +# alloy +alloy-eips = { workspace = true, features = ["serde"] } +alloy-serde.workspace = true +alloy-primitives = { workspace = true, features = ["serde"] } +alloy-rpc-types-engine = { workspace = true, features = ["serde"] } +alloy-consensus.workspace = true + +# io +tokio.workspace = true +tokio-tungstenite = { workspace = true, features = ["rustls-tls-native-roots"] } +serde.workspace = true +serde_json.workspace = true +url.workspace = true +futures-util.workspace = true +brotli.workspace = true + +# debug +tracing.workspace = true +metrics.workspace = true + +# errors +eyre.workspace = true + +ringbuffer.workspace = true +derive_more.workspace = true + +[dev-dependencies] +test-case.workspace = true +alloy-consensus.workspace = true diff --git a/crates/optimism/flashblocks/src/consensus.rs b/crates/optimism/flashblocks/src/consensus.rs new file mode 100644 index 00000000000..60314d2f6c8 --- /dev/null +++ b/crates/optimism/flashblocks/src/consensus.rs @@ -0,0 +1,86 @@ +use crate::FlashBlockCompleteSequenceRx; +use alloy_primitives::B256; +use reth_engine_primitives::ConsensusEngineHandle; +use reth_optimism_payload_builder::OpPayloadTypes; +use reth_payload_primitives::EngineApiMessageVersion; +use ringbuffer::{AllocRingBuffer, RingBuffer}; +use tracing::warn; + +/// Consensus client that sends FCUs and new payloads using blocks from a [`FlashBlockService`] +/// +/// [`FlashBlockService`]: crate::FlashBlockService +#[derive(Debug)] +pub struct FlashBlockConsensusClient { + /// Handle to execution client. + engine_handle: ConsensusEngineHandle, + sequence_receiver: FlashBlockCompleteSequenceRx, +} + +impl FlashBlockConsensusClient { + /// Create a new `FlashBlockConsensusClient` with the given Op engine and sequence receiver. + pub const fn new( + engine_handle: ConsensusEngineHandle, + sequence_receiver: FlashBlockCompleteSequenceRx, + ) -> eyre::Result { + Ok(Self { engine_handle, sequence_receiver }) + } + + /// Get previous block hash using previous block hash buffer. If it isn't available (buffer + /// started more recently than `offset`), return default zero hash + fn get_previous_block_hash( + &self, + previous_block_hashes: &AllocRingBuffer, + offset: usize, + ) -> B256 { + *previous_block_hashes + .len() + .checked_sub(offset) + .and_then(|index| previous_block_hashes.get(index)) + .unwrap_or_default() + } + + /// Spawn the client to start sending FCUs and new payloads by periodically fetching recent + /// blocks. + pub async fn run(mut self) { + let mut previous_block_hashes = AllocRingBuffer::new(64); + + loop { + match self.sequence_receiver.recv().await { + Ok(sequence) => { + let block_hash = sequence.payload_base().parent_hash; + previous_block_hashes.push(block_hash); + + if sequence.state_root().is_none() { + warn!("Missing state root for the complete sequence") + } + + // Load previous block hashes. We're using (head - 32) and (head - 64) as the + // safe and finalized block hashes. + let safe_block_hash = self.get_previous_block_hash(&previous_block_hashes, 32); + let finalized_block_hash = + self.get_previous_block_hash(&previous_block_hashes, 64); + + let state = alloy_rpc_types_engine::ForkchoiceState { + head_block_hash: block_hash, + safe_block_hash, + finalized_block_hash, + }; + + // Send FCU + let _ = self + .engine_handle + .fork_choice_updated(state, None, EngineApiMessageVersion::V3) + .await; + } + Err(err) => { + warn!( + target: "consensus::flashblock-client", + %err, + "error while fetching flashblock completed sequence" + ); + break; + } + } + } + } +} diff --git a/crates/optimism/flashblocks/src/lib.rs b/crates/optimism/flashblocks/src/lib.rs new file mode 100644 index 00000000000..7220f443cc1 --- /dev/null +++ b/crates/optimism/flashblocks/src/lib.rs @@ -0,0 +1,76 @@ +//! A downstream integration of Flashblocks. + +#![doc( + html_logo_url = "https://raw.githubusercontent.com/paradigmxyz/reth/main/assets/reth-docs.png", + html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256", + issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" +)] +#![cfg_attr(docsrs, feature(doc_cfg))] +#![cfg_attr(not(test), warn(unused_crate_dependencies))] + +use reth_primitives_traits::NodePrimitives; +use std::sync::Arc; + +pub use payload::{ + ExecutionPayloadBaseV1, ExecutionPayloadFlashblockDeltaV1, FlashBlock, FlashBlockDecoder, + Metadata, +}; +pub use service::{FlashBlockBuildInfo, FlashBlockService}; +pub use ws::{WsConnect, WsFlashBlockStream}; + +mod consensus; +pub use consensus::FlashBlockConsensusClient; +mod payload; +pub use payload::PendingFlashBlock; +mod sequence; +pub use sequence::{FlashBlockCompleteSequence, FlashBlockPendingSequence}; + +mod service; +mod worker; +mod ws; + +/// Receiver of the most recent [`PendingFlashBlock`] built out of [`FlashBlock`]s. +/// +/// [`FlashBlock`]: crate::FlashBlock +pub type PendingBlockRx = tokio::sync::watch::Receiver>>; + +/// Receiver of the sequences of [`FlashBlock`]s built. +/// +/// [`FlashBlock`]: crate::FlashBlock +pub type FlashBlockCompleteSequenceRx = + tokio::sync::broadcast::Receiver; + +/// Receiver of received [`FlashBlock`]s from the (websocket) subscription. +/// +/// [`FlashBlock`]: crate::FlashBlock +pub type FlashBlockRx = tokio::sync::broadcast::Receiver>; + +/// Receiver that signals whether a [`FlashBlock`] is currently being built. +pub type InProgressFlashBlockRx = tokio::sync::watch::Receiver>; + +/// Container for all flashblocks-related listeners. +/// +/// Groups together the channels for flashblock-related updates. +#[derive(Debug)] +pub struct FlashblocksListeners { + /// Receiver of the most recent executed [`PendingFlashBlock`] built out of [`FlashBlock`]s. + pub pending_block_rx: PendingBlockRx, + /// Subscription channel of the complete sequences of [`FlashBlock`]s built. + pub flashblocks_sequence: tokio::sync::broadcast::Sender, + /// Receiver that signals whether a [`FlashBlock`] is currently being built. + pub in_progress_rx: InProgressFlashBlockRx, + /// Subscription channel for received flashblocks from the (websocket) connection. + pub received_flashblocks: tokio::sync::broadcast::Sender>, +} + +impl FlashblocksListeners { + /// Creates a new [`FlashblocksListeners`] with the given channels. + pub const fn new( + pending_block_rx: PendingBlockRx, + flashblocks_sequence: tokio::sync::broadcast::Sender, + in_progress_rx: InProgressFlashBlockRx, + received_flashblocks: tokio::sync::broadcast::Sender>, + ) -> Self { + Self { pending_block_rx, flashblocks_sequence, in_progress_rx, received_flashblocks } + } +} diff --git a/crates/optimism/flashblocks/src/payload.rs b/crates/optimism/flashblocks/src/payload.rs new file mode 100644 index 00000000000..7469538ee3b --- /dev/null +++ b/crates/optimism/flashblocks/src/payload.rs @@ -0,0 +1,386 @@ +use alloy_consensus::BlockHeader; +use alloy_eips::eip4895::Withdrawal; +use alloy_primitives::{bytes, Address, Bloom, Bytes, B256, U256}; +use alloy_rpc_types_engine::PayloadId; +use derive_more::Deref; +use reth_optimism_evm::OpNextBlockEnvAttributes; +use reth_optimism_primitives::OpReceipt; +use reth_primitives_traits::NodePrimitives; +use reth_rpc_eth_types::PendingBlock; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +/// Represents a Flashblock, a real-time block-like structure emitted by the Base L2 chain. +/// +/// A Flashblock provides a snapshot of a block’s effects before finalization, +/// allowing faster insight into state transitions, balance changes, and logs. +/// It includes a diff of the block’s execution and associated metadata. +/// +/// See: [Base Flashblocks Documentation](https://docs.base.org/chain/flashblocks) +#[derive(Default, Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] +pub struct FlashBlock { + /// The unique payload ID as assigned by the execution engine for this block. + pub payload_id: PayloadId, + /// A sequential index that identifies the order of this Flashblock. + pub index: u64, + /// A subset of block header fields. + pub base: Option, + /// The execution diff representing state transitions and transactions. + pub diff: ExecutionPayloadFlashblockDeltaV1, + /// Additional metadata about the block such as receipts and balances. + pub metadata: Metadata, +} + +impl FlashBlock { + /// Returns the block number of this flashblock. + pub const fn block_number(&self) -> u64 { + self.metadata.block_number + } + + /// Returns the first parent hash of this flashblock. + pub fn parent_hash(&self) -> Option { + Some(self.base.as_ref()?.parent_hash) + } + + /// Returns the receipt for the given transaction hash. + pub fn receipt_by_hash(&self, hash: &B256) -> Option<&OpReceipt> { + self.metadata.receipt_by_hash(hash) + } +} + +/// A trait for decoding flashblocks from bytes. +pub trait FlashBlockDecoder: Send + 'static { + /// Decodes `bytes` into a [`FlashBlock`]. + fn decode(&self, bytes: bytes::Bytes) -> eyre::Result; +} + +/// Default implementation of the decoder. +impl FlashBlockDecoder for () { + fn decode(&self, bytes: bytes::Bytes) -> eyre::Result { + FlashBlock::decode(bytes) + } +} + +/// Provides metadata about the block that may be useful for indexing or analysis. +// Note: this uses mixed camel, snake case: +#[derive(Default, Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] +pub struct Metadata { + /// The number of the block in the L2 chain. + pub block_number: u64, + /// A map of addresses to their updated balances after the block execution. + /// This represents balance changes due to transactions, rewards, or system transfers. + pub new_account_balances: BTreeMap, + /// Execution receipts for all transactions in the block. + /// Contains logs, gas usage, and other EVM-level metadata. + pub receipts: BTreeMap, +} + +impl Metadata { + /// Returns the receipt for the given transaction hash. + pub fn receipt_by_hash(&self, hash: &B256) -> Option<&OpReceipt> { + self.receipts.get(hash) + } +} + +/// Represents the base configuration of an execution payload that remains constant +/// throughout block construction. This includes fundamental block properties like +/// parent hash, block number, and other header fields that are determined at +/// block creation and cannot be modified. +#[derive(Clone, Debug, Eq, PartialEq, Default, Deserialize, Serialize)] +pub struct ExecutionPayloadBaseV1 { + /// Ecotone parent beacon block root + pub parent_beacon_block_root: B256, + /// The parent hash of the block. + pub parent_hash: B256, + /// The fee recipient of the block. + pub fee_recipient: Address, + /// The previous randao of the block. + pub prev_randao: B256, + /// The block number. + #[serde(with = "alloy_serde::quantity")] + pub block_number: u64, + /// The gas limit of the block. + #[serde(with = "alloy_serde::quantity")] + pub gas_limit: u64, + /// The timestamp of the block. + #[serde(with = "alloy_serde::quantity")] + pub timestamp: u64, + /// The extra data of the block. + pub extra_data: Bytes, + /// The base fee per gas of the block. + pub base_fee_per_gas: U256, +} + +/// Represents the modified portions of an execution payload within a flashblock. +/// This structure contains only the fields that can be updated during block construction, +/// such as state root, receipts, logs, and new transactions. Other immutable block fields +/// like parent hash and block number are excluded since they remain constant throughout +/// the block's construction. +#[derive(Clone, Debug, Eq, PartialEq, Default, Deserialize, Serialize)] +pub struct ExecutionPayloadFlashblockDeltaV1 { + /// The state root of the block. + pub state_root: B256, + /// The receipts root of the block. + pub receipts_root: B256, + /// The logs bloom of the block. + pub logs_bloom: Bloom, + /// The gas used of the block. + #[serde(with = "alloy_serde::quantity")] + pub gas_used: u64, + /// The block hash of the block. + pub block_hash: B256, + /// The transactions of the block. + pub transactions: Vec, + /// Array of [`Withdrawal`] enabled with V2 + pub withdrawals: Vec, + /// The withdrawals root of the block. + pub withdrawals_root: B256, +} + +impl From for OpNextBlockEnvAttributes { + fn from(value: ExecutionPayloadBaseV1) -> Self { + Self { + timestamp: value.timestamp, + suggested_fee_recipient: value.fee_recipient, + prev_randao: value.prev_randao, + gas_limit: value.gas_limit, + parent_beacon_block_root: Some(value.parent_beacon_block_root), + extra_data: value.extra_data, + } + } +} + +/// The pending block built with all received Flashblocks alongside the metadata for the last added +/// Flashblock. +#[derive(Debug, Clone, Deref)] +pub struct PendingFlashBlock { + /// The complete pending block built out of all received Flashblocks. + #[deref] + pub pending: PendingBlock, + /// A sequential index that identifies the last Flashblock added to this block. + pub last_flashblock_index: u64, + /// The last Flashblock block hash, + pub last_flashblock_hash: B256, + /// Whether the [`PendingBlock`] has a properly computed stateroot. + pub has_computed_state_root: bool, +} + +impl PendingFlashBlock { + /// Create new pending flashblock. + pub const fn new( + pending: PendingBlock, + last_flashblock_index: u64, + last_flashblock_hash: B256, + has_computed_state_root: bool, + ) -> Self { + Self { pending, last_flashblock_index, last_flashblock_hash, has_computed_state_root } + } + + /// Returns the properly calculated state root for that block if it was computed. + pub fn computed_state_root(&self) -> Option { + self.has_computed_state_root.then_some(self.pending.block().state_root()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_flashblock_serde_roundtrip() { + let raw = r#"{ + "diff": { + "block_hash": "0x2d902e3fcb5bd57e0bf878cbbda1386e7fb8968d518912d58678a35e58261c46", + "gas_used": "0x2907796", + "logs_bloom": "0x5c21065292452cfcd5175abfee20e796773da578307356043ba4f62692aca01204e8908f97ab9df43f1e9c57f586b1c9a7df8b66ffa7746dfeeb538617fea5eb75ad87f8b6653f597d86814dc5ad6de404e5a48aeffcc4b1e170c2bdbc7a334936c66166ba0faa6517597b676ef65c588342756f280f7d610aa3ed35c5d877449bfacbdb9b40d98c457f974ab264ec40e4edd6e9fab4c0cb794bf75f10ea20dab75a1f9fd1c441d4c365d1476841e8593f1d1b9a1c52919a0fcf9fc5eef2ef82fe80971a72d1cde1cb195db4806058a229e88acfddfe1a1308adb6f69afa3aaf67f4bd49e93e9f9532ea30bd891a8ff08de61fb645bec678db816950b47fcef0", + "receipts_root": "0x2c4203e9aa87258627bf23ab4d5f9d92da30285ea11dc0b3e140a5a8d4b63e26", + "state_root": "0x0000000000000000000000000000000000000000000000000000000000000000", + "transactions": [ + "0x02f8c2822105830b0c58840b677c0f840c93fb5a834c4b4094d599955d17a1378651e76557ffc406c71300fcb080b851020026000100271000c8e9d514f85b57b70de033e841d788ab4df1acd691802acc26dcd13fb9e38fa8e10001004e2000c8e9d55bd42770e29cb76904377ffdb22737fc9f5eb36fde875fcbfa687b1c3023c080a07e8486ab3db9f07588a3f37bd8ffb9b349ba9bb738a2500d78a4583e1e54a6f9a068d0b3c729a6777c81dd49bd0c2dc3a079f0ceed4e778fbfe79176e8b70d68d8", + "0xf90fae820248840158a3c58307291a94bbbfd134e9b44bfb5123898ba36b01de7ab93d9880b90f443087505600000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000000000000000e0000000000000000000000000000000000000000000000000000000000000012000000000000000000000000001c2c79343de52f99538cd2cbbd67ba0813f403000000000000000000000000001c2c79343de52f99538cd2cbbd67ba0813f40300000000000000000000000000000000000000000000000000000000000000001000000000000000000000000833589fcd6edb6e08f4c7c32d4f71b54bda0291300000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000004b2ee6f00000000000000000000000000000000000000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000000000000000000000000018000000000000000000000000000000000000000000000000000000000000003600000000000000000000000000000000000000000000000000000000000000c00000000000000000000000000833589fcd6edb6e08f4c7c32d4f71b54bda029130000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000044095ea7b30000000000000000000000000000000000001ff3684f28c67538d4d072c227340000000000000000000000000000000000000000000000000000000004b2ee6f00000000000000000000000000000000000000000000000000000000000000000000000000000000f5042e6ffac5a625d4e7848e0b01373d8eb9e22200000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000001243b2253c8000000000000000000000000000000000000000000000000000000000000006000000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000000000000000e00000000000000000000000000000000000000000000000000000000000000001000000000000000000000000833589fcd6edb6e08f4c7c32d4f71b54bda029130000000000000000000000000000000000000000000000000000000000000001000000000000000000000000f70da97812cb96acdf810712aa562db8dfa3dbef000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000133f4000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001ff3684f28c67538d4d072c2273400000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000007e42213bc0b000000000000000000000000ea758cac6115309b325c582fd0782d79e3502177000000000000000000000000833589fcd6edb6e08f4c7c32d4f71b54bda029130000000000000000000000000000000000000000000000000000000004b1ba7b000000000000000000000000ea758cac6115309b325c582fd0782d79e350217700000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000000000000007041fff991f000000000000000000000000f5042e6ffac5a625d4e7848e0b01373d8eb9e222000000000000000000000000d9aaec86b65d86f6a7b5b1b0c42ffa531710b6ca0000000000000000000000000000000000000000000000000000000004b06d9200000000000000000000000000000000000000000000000000000000000000a0d311e79cd2099f6f1f0607040000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000000000000000000000000000000000000600000000000000000000000000000000000000000000000000000000000000180000000000000000000000000000000000000000000000000000000000000058000000000000000000000000000000000000000000000000000000000000000e4c1fb425e000000000000000000000000ea758cac6115309b325c582fd0782d79e3502177000000000000000000000000833589fcd6edb6e08f4c7c32d4f71b54bda029130000000000000000000000000000000000000000000000000000000004b1ba7b00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000069073bb900000000000000000000000000000000000000000000000000000000000000c000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003c438c9c147000000000000000000000000833589fcd6edb6e08f4c7c32d4f71b54bda029130000000000000000000000000000000000000000000000000000000000002710000000000000000000000000ba12222222228d8ba445958a75a0704d566bf2c800000000000000000000000000000000000000000000000000000000000001c400000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000000000000002e4945bcec9000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001200000000000000000000000000000000000000000000000000000000000000220000000000000000000000000ea758cac6115309b325c582fd0782d79e35021770000000000000000000000000000000000000000000000000000000000000000000000000000000000000000ea758cac6115309b325c582fd0782d79e3502177000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002800000000000000000000000000000000000000000000000000000000069073bb9000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000208f360baf899845441eccdc46525e26bb8860752a0002000000000000000001cd000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000004b1ba7b00000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000833589fcd6edb6e08f4c7c32d4f71b54bda02913000000000000000000000000d9aaec86b65d86f6a7b5b1b0c42ffa531710b6ca00000000000000000000000000000000000000000000000000000000000000027fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008434ee90ca000000000000000000000000f5c4f3dc02c3fb9279495a8fef7b0741da956157000000000000000000000000d9aaec86b65d86f6a7b5b1b0c42ffa531710b6ca0000000000000000000000000000000000000000000000000000000004b1a7880000000000000000000000000000000000000000000000000000000000002710000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f5042e6ffac5a625d4e7848e0b01373d8eb9e22200000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000001243b2253c8000000000000000000000000000000000000000000000000000000000000006000000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000000000000000e00000000000000000000000000000000000000000000000000000000000000001000000000000000000000000d9aaec86b65d86f6a7b5b1b0c42ffa531710b6ca000000000000000000000000000000000000000000000000000000000000000100000000000000000000000001c2c79343de52f99538cd2cbbd67ba0813f403000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002887696e8edbbcbd7306955512ff6f2d8426403eef4762157da3e9c5a89d78f682422da0c8d8b1aa1c9bfd1fe1e4a10c6123caa2fe582294aa5798c54546faa4c09590a9a012a1c78fca9cfefd281c1e44682de3c4420299da5cf2ae498f67d7de7dcf166c", + "0x02f8f582210582a649831db02984026c1a34833d090094f2cb4e685946beecbc9ce5f318b68edc583bcfa080b88600000000000069073af31c4289d066d04f33681f6686155c8243dff963557765630a39bdd8c54e6b7dbe5d4b689e9d536608db03163882cf005f7b5813e41d2fdec75161c8470a410c4c9201000202b6e39c63c7e4ebc01d51f845dfc9cff3f5adf9ef2710000000000103cd1f9777571493aeacb7eae45cd30a226d3e612d4e200000000000c080a088fd1a2b2e5891109afc3845b2c8b0ca76ea8306190dcb80a703a2451f7bab25a0718ae373e36c8ddb2b934ca936ed824db22c0625cfea29be3d408ff41787fc8c", + "0x02f9030b822105830536f9830f58ab84025c6b93833d090094c90d989d809e26b2d95fb72eb3288fef72af8c2f80b9029a00000000000069073af31c3d4d0646e102b6f958428cd8ed562efa6efb234f629b5f6ca52a15fd2e33aea76eb64fb04cae81b3e5b769dbdc681dcfd4b7a802a2cacdf1ccb65276a722c67607000202b6e39c63c7e4ebc01d51f845dfc9cff3f5adf9ef2710000000000103cd1f9777571493aeacb7eae45cd30a226d3e612d4e200000000000010206777762d3eb91810b15526c2c9102864d722ef7a9ed24e77271c1dcbf0fdcba68138800000000010698c8f03094a9e65ccedc14c40130e4a5dd0ce14fb12ea58cbeac11f662b458b9271000000000000003045a9ad2bb92b0b3e5c571fdd5125114e04e02be1a0bb80000000001036e55486ea6b8691ba58224f3cae35505add86c372710000000000003681d6e4b0b020656ca04956ddaf76add7ef022f60dac00000000010003028be0fcdd7cf0b53b7b82b8f6ea8586d07c53359f2710000000000006c30e25679d5c77b257ac3a61ad08603b11e7afe77ac9222a5386c27d08b6b6c3ea6000000000010696d4b53a38337a5733179751781178a2613306063c511b78cd02684739288c0a01f400000000000002020d028b2d7a29d2e57efc6405a1dce1437180e3ce27100000000001068a71465e76d736564b0c90f5cf3d0d7b69c461c36f69250ae27dbead147cc8f80bb80000000000000206354def8b7e6b2ee04bf85c00f5e79f173d0b76d5017bab3a90c7ba62e1722699000000000000010245f3ad9e63f629be6e278cc4cf34d3b0a79a4a0b27100000000000010404b154dbcd3c75580382c2353082df4390613d93c627120000000001011500cc7d9c2b460720a48cc7444d7e7dfe43f6050bb80a03000000015c8dec5f0eedf1f8934815ef8fb8cb8198eac6520bb80a030000010286f3dd3b4d08de718d7909b0fdc16f4cbdf94ef527100000000000c001a0d4c12f6433ff6ea0573633364c030d8b46ed5764494f80eb434f27060c39f315a034df82c4ac185a666280d578992feee0c05fc75d93e3e2286726c85fba1bb0a0", + "0x02f8f68221058305c7b3830f4ef58401a5485d832dc6c094f2cb4e685946beecbc9ce5f318b68edc583bcfa080b88600000000000069073af31b777ac6b2082fc399fde92a814114b7896ca0b0503106910ea099d5e32c93bfc0013ed2850534c3f8583ab7276414416c0d15ac021126f6cb6ca1ed091ddc01eb01000202b6e39c63c7e4ebc01d51f845dfc9cff3f5adf9ef2710000000000103cd1f9777571493aeacb7eae45cd30a226d3e612d4e200000000000c080a09694b95dc893bed698ede415c188db3530ccc98a01d79bb9f11d783de7dddde9a0275b0165ab21ea0e6f721c624aa2270a3f98276ca0c95381d90e3f9d434b4881", + "0x02f8f682210583034573830f4ef58401a5485d832dc6c094f2cb4e685946beecbc9ce5f318b68edc583bcfa080b88600000000000069073af31c970da8f2adb8bafe6d254ec4428f8342508e169f75e8450f6ff8488813dfa638395e16787966f01731fddffd0e7352cde07fd24bba283bd27f1828fb2a0c700701000202b6e39c63c7e4ebc01d51f845dfc9cff3f5adf9ef2710000000000103cd1f9777571493aeacb7eae45cd30a226d3e612d4e200000000000c080a00181afe4bedab67692a9c1ff30a89fde6b3d3c8407a47a2777efcd6bdc0c39d2a022d6a4219e72eebdbc5d31ae998243ccec1b192c5c7c586308ccddb4838cd631", + "0x02f8c1822105830b0cfd830f4ed084013bce1b834c4b4094d599955d17a1378651e76557ffc406c71300fcb080b851020026000100271000c8e9d514f85b57b70de033e841d788ab4df1acd691802acc26dcd13fb9e38fa8e10001004e2000c8e9d55bd42770e29cb76904377ffdb22737fc9f5eb36fde875fcbfa687b1c3023c001a0d87c4e16986db55b8846bccfe7bca824b75216e72d8f92369c46681800285cb2a00ec53251be3c2a0d19884747d123ddb0ada3c0a917b21882e297e95c2294d52a", + "0x02f901d58221058306361d830f4240840163efbc8301546194833589fcd6edb6e08f4c7c32d4f71b54bda0291380b90164cf092995000000000000000000000000d723d9f752c19faf88a5fd2111a38d0cc5d395b00000000000000000000000000b55712de2ce8f93be30d53c03d48ea275cd14d000000000000000000000000000000000000000000000000000000000000003e8000000000000000000000000000000000000000000000000000000006907385e0000000000000000000000000000000000000000000000000000000069073be2bef9866b70d0bb74d8763996eb5967b1b24cd48f7801f94ad80cb49431df6b1d00000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000000417c9c2382c6c3f029aa3dcbf1df075366fae7bc9fba7f3729713e0bf4d518951f5340350208db96af23686d9985ce552e3588244456a23ca99ecbcae779ea11e71c00000000000000000000000000000000000000000000000000000000000000c080a0b1090c8c67ca9a49ba3591c72c8851f187bbfc39b1920dff2f6c0157ed1ada39a0265b7f704f4c1b5c2c5ca57f1a4040e1e48878c9ad5f2cca9c4e6669d12989f2", + "0x02f8c1822105830b0c98830f424084013bc18b834c4b4094d599955d17a1378651e76557ffc406c71300fcb080b851020026000100271000c8e9d514f85b57b70de033e841d788ab4df1acd691802acc26dcd13fb9e38fa8e10001004e2000c8e9d55bd42770e29cb76904377ffdb22737fc9f5eb36fde875fcbfa687b1c3023c001a080a96d18ae46b58d9a470846a05b394ab4a49a2e379de1941205684e1ac291f9a01e6d4d2c6bab5bf8b89f1df2d6beb85d9f1b3f3be73ca2b72e4ad2d9da0d12d2", + "0x02f901d48221058231e0830f4240840163efbc8301544d94833589fcd6edb6e08f4c7c32d4f71b54bda0291380b90164cf0929950000000000000000000000001de8dbc2409c4bbf14445b0d404bb894f0c6cff70000000000000000000000008d8fa42584a727488eeb0e29405ad794a105bb9b0000000000000000000000000000000000000000000000000000000000002710000000000000000000000000000000000000000000000000000000006907385d0000000000000000000000000000000000000000000000000000000069073af16b129c414484e011621c44e0b32451fdbd69e63ef4919f427dde08c16cb199b100000000000000000000000000000000000000000000000000000000000000e00000000000000000000000000000000000000000000000000000000000000041ae0a4b618c30f0e5d92d7fe99bb435413b2201711427699fd285f69666396cee76199d4e901cfb298612cb3b8ad06178cefb4136a8bc1be07c01b5fea80e5ec11b00000000000000000000000000000000000000000000000000000000000000c080a0af315068084aae367f00263dbd872908bbb9ceaefd6b792fc48dd357e6bdf8afa01e7f0e5913570394b9648939ef71fc5ac34fe320a2757ec388316731a335e69f", + "0x02f9022f82210583052d0b830f423f84025c5527833d090094c90d989d809e26b2d95fb72eb3288fef72af8c2f80b901be00000000000069073af31cf0f932cecc8c4c6ffffa554a63e8fba251434483ed3903966d2ba5a70121618a1c45bd9ee158192ab8d7e12ce0f447f2848a48aedaa89e0efa8637bb931745de05000202b6e39c63c7e4ebc01d51f845dfc9cff3f5adf9ef2710000000000103cd1f9777571493aeacb7eae45cd30a226d3e612d4e2000000000000003045a9ad2bb92b0b3e5c571fdd5125114e04e02be1a0bb80000000001036e55486ea6b8691ba58224f3cae35505add86c372710000000000003681d6e4b0b020656ca04956ddaf76add7ef022f60dac0000000001010206777762d3eb91810b15526c2c9102864d722ef7a9ed24e77271c1dcbf0fdcba68138800000000010698c8f03094a9e65ccedc14c40130e4a5dd0ce14fb12ea58cbeac11f662b458b9271000000000000002005554419ccd0293d9383901f461c7c3e0c66e925f0bb80000000001028eb9437532fac8d6a7870f3f887b7978d20355fc271000000000000003035d28f920c9d23100e4a38b2ba2d8ae617c3b261501f4000000000102bc51db8aec659027ae0b0e468c0735418161a7800bb8000000000003dbc6998296caa1652a810dc8d3baf4a8294330f100500000000000c080a040000b130b1759df897a9573691a3d1cafacc6d95d0db1826f275afc30e2ff63a0400a7514f8d5383970c4412205ec8e9c6ca06acea504acabd2d3c36e9cb5003d" + ], + "withdrawals": [], + "withdrawals_root": "0x81864c23f426ad807d66c9fdde33213e1fdbac06c1b751d279901d1ce13670ac" + }, + "index": 10, + "metadata": { + "block_number": 37646058, + "new_account_balances": { + "0x000000000022d473030f116ddee9f6b43ac78ba3": "0x0", + "0x0000000071727de22e5e9d8baf0edac6f37da032": "0x23281e39594556899", + "0x0000f90827f1c53a10cb7a02335b175320002935": "0x0", + "0x000f3df6d732807ef1319fb7b8bb8522d0beac02": "0x0" + }, + "receipts": { + "0x1a766690fd6d0febffc488f12fbd7385c43fbe1e07113a1316f22f176355297e": { + "Legacy": { + "cumulativeGasUsed": "0x2868d76", + "logs": [ + { + "address": "0x833589fcd6edb6e08f4c7c32d4f71b54bda02913", + "data": "0x0000000000000000000000000000000000000000000000000000000004b2ee6f", + "topics": [ + "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", + "0x00000000000000000000000001c2c79343de52f99538cd2cbbd67ba0813f4030", + "0x000000000000000000000000f5042e6ffac5a625d4e7848e0b01373d8eb9e222" + ] + }, + { + "address": "0x833589fcd6edb6e08f4c7c32d4f71b54bda02913", + "data": "0x0000000000000000000000000000000000000000000000000000000004b2ee6f", + "topics": [ + "0x8c5be1e5ebec7d5bd14f71427d1e84f3dd0314c0f7b2291e5b200ac8c7c3b925", + "0x000000000000000000000000f5042e6ffac5a625d4e7848e0b01373d8eb9e222", + "0x0000000000000000000000000000000000001ff3684f28c67538d4d072c22734" + ] + }, + { + "address": "0xf5042e6ffac5a625d4e7848e0b01373d8eb9e222", + "data": "0x000000000000000000000000833589fcd6edb6e08f4c7c32d4f71b54bda02913000000000000000000000000000000000000000000000000000000000000006000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000044095ea7b30000000000000000000000000000000000001ff3684f28c67538d4d072c227340000000000000000000000000000000000000000000000000000000004b2ee6f00000000000000000000000000000000000000000000000000000000", + "topics": [ + "0x93485dcd31a905e3ffd7b012abe3438fa8fa77f98ddc9f50e879d3fa7ccdc324" + ] + }, + { + "address": "0x833589fcd6edb6e08f4c7c32d4f71b54bda02913", + "data": "0x00000000000000000000000000000000000000000000000000000000000133f4", + "topics": [ + "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", + "0x000000000000000000000000f5042e6ffac5a625d4e7848e0b01373d8eb9e222", + "0x000000000000000000000000f70da97812cb96acdf810712aa562db8dfa3dbef" + ] + }, + { + "address": "0xf5042e6ffac5a625d4e7848e0b01373d8eb9e222", + "data": "0x000000000000000000000000f5042e6ffac5a625d4e7848e0b01373d8eb9e2220000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001243b2253c8000000000000000000000000000000000000000000000000000000000000006000000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000000000000000e00000000000000000000000000000000000000000000000000000000000000001000000000000000000000000833589fcd6edb6e08f4c7c32d4f71b54bda029130000000000000000000000000000000000000000000000000000000000000001000000000000000000000000f70da97812cb96acdf810712aa562db8dfa3dbef000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000133f400000000000000000000000000000000000000000000000000000000", + "topics": [ + "0x93485dcd31a905e3ffd7b012abe3438fa8fa77f98ddc9f50e879d3fa7ccdc324" + ] + }, + { + "address": "0x833589fcd6edb6e08f4c7c32d4f71b54bda02913", + "data": "0x0000000000000000000000000000000000000000000000000000000004b1ba7b", + "topics": [ + "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", + "0x000000000000000000000000f5042e6ffac5a625d4e7848e0b01373d8eb9e222", + "0x000000000000000000000000ea758cac6115309b325c582fd0782d79e3502177" + ] + }, + { + "address": "0x8f360baf899845441eccdc46525e26bb8860752a", + "data": "0x00000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000001957cc57b7a9959c0000000000000000000000000000000000000000000000001957cc57b7a9959800000000000000000000000000000000000000000000000444e308096a22c339000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000092458cc3a866f04600000000000000000000000000000000000000000000000025f3e27916e84b59000", + "topics": [ + "0x4e1d56f7310a8c32b2267f756b19ba65019b4890068ce114a25009abe54de5ba" + ] + }, + { + "address": "0xba12222222228d8ba445958a75a0704d566bf2c8", + "data": "0x0000000000000000000000000000000000000000000000000000000004b1ba7b0000000000000000000000000000000000000000000000000000000004b1a44c", + "topics": [ + "0x2170c741c41531aec20e7c107c24eecfdd15e69c9bb0a8dd37b1840b9e0b207b", + "0x8f360baf899845441eccdc46525e26bb8860752a0002000000000000000001cd", + "0x000000000000000000000000833589fcd6edb6e08f4c7c32d4f71b54bda02913", + "0x000000000000000000000000d9aaec86b65d86f6a7b5b1b0c42ffa531710b6ca" + ] + }, + { + "address": "0x833589fcd6edb6e08f4c7c32d4f71b54bda02913", + "data": "0x0000000000000000000000000000000000000000000000000000000004b1ba7b", + "topics": [ + "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", + "0x000000000000000000000000ea758cac6115309b325c582fd0782d79e3502177", + "0x000000000000000000000000ba12222222228d8ba445958a75a0704d566bf2c8" + ] + }, + { + "address": "0xd9aaec86b65d86f6a7b5b1b0c42ffa531710b6ca", + "data": "0x0000000000000000000000000000000000000000000000000000000004b1a44c", + "topics": [ + "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", + "0x000000000000000000000000ba12222222228d8ba445958a75a0704d566bf2c8", + "0x000000000000000000000000ea758cac6115309b325c582fd0782d79e3502177" + ] + }, + { + "address": "0xd9aaec86b65d86f6a7b5b1b0c42ffa531710b6ca", + "data": "0x0000000000000000000000000000000000000000000000000000000004b1a44c", + "topics": [ + "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", + "0x000000000000000000000000ea758cac6115309b325c582fd0782d79e3502177", + "0x000000000000000000000000f5042e6ffac5a625d4e7848e0b01373d8eb9e222" + ] + }, + { + "address": "0xf5042e6ffac5a625d4e7848e0b01373d8eb9e222", + "data": "0x0000000000000000000000000000000000001ff3684f28c67538d4d072c227340000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000007e42213bc0b000000000000000000000000ea758cac6115309b325c582fd0782d79e3502177000000000000000000000000833589fcd6edb6e08f4c7c32d4f71b54bda029130000000000000000000000000000000000000000000000000000000004b1ba7b000000000000000000000000ea758cac6115309b325c582fd0782d79e350217700000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000000000000007041fff991f000000000000000000000000f5042e6ffac5a625d4e7848e0b01373d8eb9e222000000000000000000000000d9aaec86b65d86f6a7b5b1b0c42ffa531710b6ca0000000000000000000000000000000000000000000000000000000004b06d9200000000000000000000000000000000000000000000000000000000000000a0d311e79cd2099f6f1f0607040000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000000000000000000000000000000000000600000000000000000000000000000000000000000000000000000000000000180000000000000000000000000000000000000000000000000000000000000058000000000000000000000000000000000000000000000000000000000000000e4c1fb425e000000000000000000000000ea758cac6115309b325c582fd0782d79e3502177000000000000000000000000833589fcd6edb6e08f4c7c32d4f71b54bda029130000000000000000000000000000000000000000000000000000000004b1ba7b00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000069073bb900000000000000000000000000000000000000000000000000000000000000c000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003c438c9c147000000000000000000000000833589fcd6edb6e08f4c7c32d4f71b54bda029130000000000000000000000000000000000000000000000000000000000002710000000000000000000000000ba12222222228d8ba445958a75a0704d566bf2c800000000000000000000000000000000000000000000000000000000000001c400000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000000000000002e4945bcec9000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001200000000000000000000000000000000000000000000000000000000000000220000000000000000000000000ea758cac6115309b325c582fd0782d79e35021770000000000000000000000000000000000000000000000000000000000000000000000000000000000000000ea758cac6115309b325c582fd0782d79e3502177000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002800000000000000000000000000000000000000000000000000000000069073bb9000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000208f360baf899845441eccdc46525e26bb8860752a0002000000000000000001cd000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000004b1ba7b00000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000833589fcd6edb6e08f4c7c32d4f71b54bda02913000000000000000000000000d9aaec86b65d86f6a7b5b1b0c42ffa531710b6ca00000000000000000000000000000000000000000000000000000000000000027fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008434ee90ca000000000000000000000000f5c4f3dc02c3fb9279495a8fef7b0741da956157000000000000000000000000d9aaec86b65d86f6a7b5b1b0c42ffa531710b6ca0000000000000000000000000000000000000000000000000000000004b1a7880000000000000000000000000000000000000000000000000000000000002710000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "topics": [ + "0x93485dcd31a905e3ffd7b012abe3438fa8fa77f98ddc9f50e879d3fa7ccdc324" + ] + }, + { + "address": "0xd9aaec86b65d86f6a7b5b1b0c42ffa531710b6ca", + "data": "0x0000000000000000000000000000000000000000000000000000000004b1a44c", + "topics": [ + "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", + "0x000000000000000000000000f5042e6ffac5a625d4e7848e0b01373d8eb9e222", + "0x00000000000000000000000001c2c79343de52f99538cd2cbbd67ba0813f4030" + ] + }, + { + "address": "0xf5042e6ffac5a625d4e7848e0b01373d8eb9e222", + "data": "0x000000000000000000000000f5042e6ffac5a625d4e7848e0b01373d8eb9e2220000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001243b2253c8000000000000000000000000000000000000000000000000000000000000006000000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000000000000000e00000000000000000000000000000000000000000000000000000000000000001000000000000000000000000d9aaec86b65d86f6a7b5b1b0c42ffa531710b6ca000000000000000000000000000000000000000000000000000000000000000100000000000000000000000001c2c79343de52f99538cd2cbbd67ba0813f40300000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "topics": [ + "0x93485dcd31a905e3ffd7b012abe3438fa8fa77f98ddc9f50e879d3fa7ccdc324" + ] + } + ], + "status": "0x1" + } + }, + "0x2cd6b4825b5ee40b703c947e15630336dceda97825b70412da54ccc27f484496": { + "Eip1559": { + "cumulativeGasUsed": "0x28cca69", + "logs": [ + { + "address": "0x833589fcd6edb6e08f4c7c32d4f71b54bda02913", + "data": "0x", + "topics": [ + "0x98de503528ee59b575ef0c0a2576a82497bfc029a5685b209e9ec333479b10a5", + "0x000000000000000000000000d723d9f752c19faf88a5fd2111a38d0cc5d395b0", + "0xbef9866b70d0bb74d8763996eb5967b1b24cd48f7801f94ad80cb49431df6b1d" + ] + }, + { + "address": "0x833589fcd6edb6e08f4c7c32d4f71b54bda02913", + "data": "0x00000000000000000000000000000000000000000000000000000000000003e8", + "topics": [ + "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", + "0x000000000000000000000000d723d9f752c19faf88a5fd2111a38d0cc5d395b0", + "0x0000000000000000000000000b55712de2ce8f93be30d53c03d48ea275cd14d0" + ] + } + ], + "status": "0x1" + } + } + } + }, + "payload_id": "0x0316ecb1aa1671b5" +}"#; + + let flashblock: FlashBlock = serde_json::from_str(raw).expect("deserialize"); + let serialized = serde_json::to_string(&flashblock).expect("serialize"); + let roundtrip: FlashBlock = serde_json::from_str(&serialized).expect("roundtrip"); + + assert_eq!(flashblock, roundtrip); + } +} diff --git a/crates/optimism/flashblocks/src/sequence.rs b/crates/optimism/flashblocks/src/sequence.rs new file mode 100644 index 00000000000..f2363207e38 --- /dev/null +++ b/crates/optimism/flashblocks/src/sequence.rs @@ -0,0 +1,404 @@ +use crate::{ExecutionPayloadBaseV1, FlashBlock, FlashBlockCompleteSequenceRx}; +use alloy_eips::eip2718::WithEncoded; +use alloy_primitives::B256; +use alloy_rpc_types_engine::PayloadId; +use core::mem; +use eyre::{bail, OptionExt}; +use reth_primitives_traits::{Recovered, SignedTransaction}; +use std::{collections::BTreeMap, ops::Deref}; +use tokio::sync::broadcast; +use tracing::{debug, trace, warn}; + +/// The size of the broadcast channel for completed flashblock sequences. +const FLASHBLOCK_SEQUENCE_CHANNEL_SIZE: usize = 128; + +/// An ordered B-tree keeping the track of a sequence of [`FlashBlock`]s by their indices. +#[derive(Debug)] +pub struct FlashBlockPendingSequence { + /// tracks the individual flashblocks in order + /// + /// With a blocktime of 2s and flashblock tick-rate of 200ms plus one extra flashblock per new + /// pending block, we expect 11 flashblocks per slot. + inner: BTreeMap>, + /// Broadcasts flashblocks to subscribers. + block_broadcaster: broadcast::Sender, + /// Optional properly computed state root for the current sequence. + state_root: Option, +} + +impl FlashBlockPendingSequence +where + T: SignedTransaction, +{ + /// Create a new pending sequence. + pub fn new() -> Self { + // Note: if the channel is full, send will not block but rather overwrite the oldest + // messages. Order is preserved. + let (tx, _) = broadcast::channel(FLASHBLOCK_SEQUENCE_CHANNEL_SIZE); + Self { inner: BTreeMap::new(), block_broadcaster: tx, state_root: None } + } + + /// Returns the sender half of the [`FlashBlockCompleteSequence`] channel. + pub const fn block_sequence_broadcaster( + &self, + ) -> &broadcast::Sender { + &self.block_broadcaster + } + + /// Gets a subscriber to the flashblock sequences produced. + pub fn subscribe_block_sequence(&self) -> FlashBlockCompleteSequenceRx { + self.block_broadcaster.subscribe() + } + + // Clears the state and broadcasts the blocks produced to subscribers. + fn clear_and_broadcast_blocks(&mut self) { + let flashblocks = mem::take(&mut self.inner); + + // If there are any subscribers, send the flashblocks to them. + if self.block_broadcaster.receiver_count() > 0 { + let flashblocks = match FlashBlockCompleteSequence::new( + flashblocks.into_iter().map(|block| block.1.into()).collect(), + self.state_root, + ) { + Ok(flashblocks) => flashblocks, + Err(err) => { + debug!(target: "flashblocks", error = ?err, "Failed to create full flashblock complete sequence"); + return; + } + }; + + // Note: this should only ever fail if there are no receivers. This can happen if + // there is a race condition between the clause right above and this + // one. We can simply warn the user and continue. + if let Err(err) = self.block_broadcaster.send(flashblocks) { + warn!(target: "flashblocks", error = ?err, "Failed to send flashblocks to subscribers"); + } + } + } + + /// Inserts a new block into the sequence. + /// + /// A [`FlashBlock`] with index 0 resets the set. + pub fn insert(&mut self, flashblock: FlashBlock) -> eyre::Result<()> { + if flashblock.index == 0 { + trace!(number=%flashblock.block_number(), "Tracking new flashblock sequence"); + + // Flash block at index zero resets the whole state. + self.clear_and_broadcast_blocks(); + + self.inner.insert(flashblock.index, PreparedFlashBlock::new(flashblock)?); + return Ok(()) + } + + // only insert if we previously received the same block and payload, assume we received + // index 0 + let same_block = self.block_number() == Some(flashblock.metadata.block_number); + let same_payload = self.payload_id() == Some(flashblock.payload_id); + + if same_block && same_payload { + trace!(number=%flashblock.block_number(), index = %flashblock.index, block_count = self.inner.len() ,"Received followup flashblock"); + self.inner.insert(flashblock.index, PreparedFlashBlock::new(flashblock)?); + } else { + trace!(number=%flashblock.block_number(), index = %flashblock.index, current=?self.block_number() ,"Ignoring untracked flashblock following"); + } + + Ok(()) + } + + /// Set state root + pub const fn set_state_root(&mut self, state_root: Option) { + self.state_root = state_root; + } + + /// Iterator over sequence of executable transactions. + /// + /// A flashblocks is not ready if there's missing previous flashblocks, i.e. there's a gap in + /// the sequence + /// + /// Note: flashblocks start at `index 0`. + pub fn ready_transactions(&self) -> impl Iterator>> + '_ { + self.inner + .values() + .enumerate() + .take_while(|(idx, block)| { + // flashblock index 0 is the first flashblock + block.block().index == *idx as u64 + }) + .flat_map(|(_, block)| block.txs.clone()) + } + + /// Returns the first block number + pub fn block_number(&self) -> Option { + Some(self.inner.values().next()?.block().metadata.block_number) + } + + /// Returns the payload base of the first tracked flashblock. + pub fn payload_base(&self) -> Option { + self.inner.values().next()?.block().base.clone() + } + + /// Returns the number of tracked flashblocks. + pub fn count(&self) -> usize { + self.inner.len() + } + + /// Returns the reference to the last flashblock. + pub fn last_flashblock(&self) -> Option<&FlashBlock> { + self.inner.last_key_value().map(|(_, b)| &b.block) + } + + /// Returns the current/latest flashblock index in the sequence + pub fn index(&self) -> Option { + Some(self.inner.values().last()?.block().index) + } + /// Returns the payload id of the first tracked flashblock in the current sequence. + pub fn payload_id(&self) -> Option { + Some(self.inner.values().next()?.block().payload_id) + } +} + +impl Default for FlashBlockPendingSequence +where + T: SignedTransaction, +{ + fn default() -> Self { + Self::new() + } +} + +/// A complete sequence of flashblocks, often corresponding to a full block. +/// +/// Ensures invariants of a complete flashblocks sequence. +/// If this entire sequence of flashblocks was executed on top of latest block, this also includes +/// the computed state root. +#[derive(Debug, Clone)] +pub struct FlashBlockCompleteSequence { + inner: Vec, + /// Optional state root for the current sequence + state_root: Option, +} + +impl FlashBlockCompleteSequence { + /// Create a complete sequence from a vector of flashblocks. + /// Ensure that: + /// * vector is not empty + /// * first flashblock have the base payload + /// * sequence of flashblocks is sound (successive index from 0, same payload id, ...) + pub fn new(blocks: Vec, state_root: Option) -> eyre::Result { + let first_block = blocks.first().ok_or_eyre("No flashblocks in sequence")?; + + // Ensure that first flashblock have base + first_block.base.as_ref().ok_or_eyre("Flashblock at index 0 has no base")?; + + // Ensure that index are successive from 0, have same block number and payload id + if !blocks.iter().enumerate().all(|(idx, block)| { + idx == block.index as usize && + block.payload_id == first_block.payload_id && + block.metadata.block_number == first_block.metadata.block_number + }) { + bail!("Flashblock inconsistencies detected in sequence"); + } + + Ok(Self { inner: blocks, state_root }) + } + + /// Returns the block number + pub fn block_number(&self) -> u64 { + self.inner.first().unwrap().metadata.block_number + } + + /// Returns the payload base of the first flashblock. + pub fn payload_base(&self) -> &ExecutionPayloadBaseV1 { + self.inner.first().unwrap().base.as_ref().unwrap() + } + + /// Returns the number of flashblocks in the sequence. + pub const fn count(&self) -> usize { + self.inner.len() + } + + /// Returns the last flashblock in the sequence. + pub fn last(&self) -> &FlashBlock { + self.inner.last().unwrap() + } + + /// Returns the state root for the current sequence + pub const fn state_root(&self) -> Option { + self.state_root + } +} + +impl Deref for FlashBlockCompleteSequence { + type Target = Vec; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl TryFrom> for FlashBlockCompleteSequence { + type Error = eyre::Error; + fn try_from(sequence: FlashBlockPendingSequence) -> Result { + Self::new( + sequence.inner.into_values().map(|block| block.block().clone()).collect::>(), + sequence.state_root, + ) + } +} + +#[derive(Debug)] +struct PreparedFlashBlock { + /// The prepared transactions, ready for execution + txs: Vec>>, + /// The tracked flashblock + block: FlashBlock, +} + +impl PreparedFlashBlock { + const fn block(&self) -> &FlashBlock { + &self.block + } +} + +impl From> for FlashBlock { + fn from(val: PreparedFlashBlock) -> Self { + val.block + } +} + +impl PreparedFlashBlock +where + T: SignedTransaction, +{ + /// Creates a flashblock that is ready for execution by preparing all transactions + /// + /// Returns an error if decoding or signer recovery fails. + fn new(block: FlashBlock) -> eyre::Result { + let mut txs = Vec::with_capacity(block.diff.transactions.len()); + for encoded in block.diff.transactions.iter().cloned() { + let tx = T::decode_2718_exact(encoded.as_ref())?; + let signer = tx.try_recover()?; + let tx = WithEncoded::new(encoded, tx.with_signer(signer)); + txs.push(tx); + } + + Ok(Self { txs, block }) + } +} + +impl Deref for PreparedFlashBlock { + type Target = FlashBlock; + + fn deref(&self) -> &Self::Target { + &self.block + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ExecutionPayloadFlashblockDeltaV1; + use alloy_consensus::{ + transaction::SignerRecoverable, EthereumTxEnvelope, EthereumTypedTransaction, TxEip1559, + }; + use alloy_eips::Encodable2718; + use alloy_primitives::{hex, Signature, TxKind, U256}; + + #[test] + fn test_sequence_stops_before_gap() { + let mut sequence = FlashBlockPendingSequence::new(); + let tx = EthereumTxEnvelope::new_unhashed( + EthereumTypedTransaction::::Eip1559(TxEip1559 { + chain_id: 4, + nonce: 26u64, + max_priority_fee_per_gas: 1500000000, + max_fee_per_gas: 1500000013, + gas_limit: 21_000u64, + to: TxKind::Call(hex!("61815774383099e24810ab832a5b2a5425c154d5").into()), + value: U256::from(3000000000000000000u64), + input: Default::default(), + access_list: Default::default(), + }), + Signature::new( + U256::from_be_bytes(hex!( + "59e6b67f48fb32e7e570dfb11e042b5ad2e55e3ce3ce9cd989c7e06e07feeafd" + )), + U256::from_be_bytes(hex!( + "016b83f4f980694ed2eee4d10667242b1f40dc406901b34125b008d334d47469" + )), + true, + ), + ); + let tx = Recovered::new_unchecked(tx.clone(), tx.recover_signer_unchecked().unwrap()); + + sequence + .insert(FlashBlock { + payload_id: Default::default(), + index: 0, + base: None, + diff: ExecutionPayloadFlashblockDeltaV1 { + transactions: vec![tx.encoded_2718().into()], + ..Default::default() + }, + metadata: Default::default(), + }) + .unwrap(); + + sequence + .insert(FlashBlock { + payload_id: Default::default(), + index: 2, + base: None, + diff: Default::default(), + metadata: Default::default(), + }) + .unwrap(); + + let actual_txs: Vec<_> = sequence.ready_transactions().collect(); + let expected_txs = vec![WithEncoded::new(tx.encoded_2718().into(), tx)]; + + assert_eq!(actual_txs, expected_txs); + } + + #[test] + fn test_sequence_sends_flashblocks_to_subscribers() { + let mut sequence = FlashBlockPendingSequence::>::new(); + let mut subscriber = sequence.subscribe_block_sequence(); + + for idx in 0..10 { + sequence + .insert(FlashBlock { + payload_id: Default::default(), + index: idx, + base: Some(ExecutionPayloadBaseV1::default()), + diff: Default::default(), + metadata: Default::default(), + }) + .unwrap(); + } + + assert_eq!(sequence.count(), 10); + + // Then we don't receive anything until we insert a new flashblock + let no_flashblock = subscriber.try_recv(); + assert!(no_flashblock.is_err()); + + // Let's insert a new flashblock with index 0 + sequence + .insert(FlashBlock { + payload_id: Default::default(), + index: 0, + base: Some(ExecutionPayloadBaseV1::default()), + diff: Default::default(), + metadata: Default::default(), + }) + .unwrap(); + + let flashblocks = subscriber.try_recv().unwrap(); + assert_eq!(flashblocks.count(), 10); + + for (idx, block) in flashblocks.iter().enumerate() { + assert_eq!(block.index, idx as u64); + } + } +} diff --git a/crates/optimism/flashblocks/src/service.rs b/crates/optimism/flashblocks/src/service.rs new file mode 100644 index 00000000000..f5d4a4a810d --- /dev/null +++ b/crates/optimism/flashblocks/src/service.rs @@ -0,0 +1,384 @@ +use crate::{ + sequence::FlashBlockPendingSequence, + worker::{BuildArgs, FlashBlockBuilder}, + ExecutionPayloadBaseV1, FlashBlock, FlashBlockCompleteSequence, FlashBlockCompleteSequenceRx, + InProgressFlashBlockRx, PendingFlashBlock, +}; +use alloy_eips::eip2718::WithEncoded; +use alloy_primitives::B256; +use futures_util::{FutureExt, Stream, StreamExt}; +use metrics::Histogram; +use reth_chain_state::{CanonStateNotification, CanonStateNotifications, CanonStateSubscriptions}; +use reth_evm::ConfigureEvm; +use reth_metrics::Metrics; +use reth_primitives_traits::{ + AlloyBlockHeader, BlockTy, HeaderTy, NodePrimitives, ReceiptTy, Recovered, +}; +use reth_revm::cached::CachedReads; +use reth_storage_api::{BlockReaderIdExt, StateProviderFactory}; +use reth_tasks::TaskExecutor; +use std::{ + pin::Pin, + sync::Arc, + task::{ready, Context, Poll}, + time::Instant, +}; +use tokio::{ + pin, + sync::{oneshot, watch}, +}; +use tracing::{debug, trace, warn}; + +pub(crate) const FB_STATE_ROOT_FROM_INDEX: usize = 9; + +/// The `FlashBlockService` maintains an in-memory [`PendingFlashBlock`] built out of a sequence of +/// [`FlashBlock`]s. +#[derive(Debug)] +pub struct FlashBlockService< + N: NodePrimitives, + S, + EvmConfig: ConfigureEvm, + Provider, +> { + rx: S, + current: Option>, + blocks: FlashBlockPendingSequence, + /// Broadcast channel to forward received flashblocks from the subscription. + received_flashblocks_tx: tokio::sync::broadcast::Sender>, + rebuild: bool, + builder: FlashBlockBuilder, + canon_receiver: CanonStateNotifications, + spawner: TaskExecutor, + job: Option>, + /// Cached state reads for the current block. + /// Current `PendingFlashBlock` is built out of a sequence of `FlashBlocks`, and executed again + /// when fb received on top of the same block. Avoid redundant I/O across multiple + /// executions within the same block. + cached_state: Option<(B256, CachedReads)>, + /// Signals when a block build is in progress + in_progress_tx: watch::Sender>, + /// `FlashBlock` service's metrics + metrics: FlashBlockServiceMetrics, + /// Enable state root calculation from flashblock with index [`FB_STATE_ROOT_FROM_INDEX`] + compute_state_root: bool, +} + +impl FlashBlockService +where + N: NodePrimitives, + S: Stream> + Unpin + 'static, + EvmConfig: ConfigureEvm + Unpin> + + Clone + + 'static, + Provider: StateProviderFactory + + CanonStateSubscriptions + + BlockReaderIdExt< + Header = HeaderTy, + Block = BlockTy, + Transaction = N::SignedTx, + Receipt = ReceiptTy, + > + Unpin + + Clone + + 'static, +{ + /// Constructs a new `FlashBlockService` that receives [`FlashBlock`]s from `rx` stream. + pub fn new(rx: S, evm_config: EvmConfig, provider: Provider, spawner: TaskExecutor) -> Self { + let (in_progress_tx, _) = watch::channel(None); + let (received_flashblocks_tx, _) = tokio::sync::broadcast::channel(128); + Self { + rx, + current: None, + blocks: FlashBlockPendingSequence::new(), + received_flashblocks_tx, + canon_receiver: provider.subscribe_to_canonical_state(), + builder: FlashBlockBuilder::new(evm_config, provider), + rebuild: false, + spawner, + job: None, + cached_state: None, + in_progress_tx, + metrics: FlashBlockServiceMetrics::default(), + compute_state_root: false, + } + } + + /// Enable state root calculation from flashblock + pub const fn compute_state_root(mut self, enable_state_root: bool) -> Self { + self.compute_state_root = enable_state_root; + self + } + + /// Returns the sender half to the received flashblocks. + pub const fn flashblocks_broadcaster( + &self, + ) -> &tokio::sync::broadcast::Sender> { + &self.received_flashblocks_tx + } + + /// Returns the sender half to the flashblock sequence. + pub const fn block_sequence_broadcaster( + &self, + ) -> &tokio::sync::broadcast::Sender { + self.blocks.block_sequence_broadcaster() + } + + /// Returns a subscriber to the flashblock sequence. + pub fn subscribe_block_sequence(&self) -> FlashBlockCompleteSequenceRx { + self.blocks.subscribe_block_sequence() + } + + /// Returns a receiver that signals when a flashblock is being built. + pub fn subscribe_in_progress(&self) -> InProgressFlashBlockRx { + self.in_progress_tx.subscribe() + } + + /// Drives the services and sends new blocks to the receiver + /// + /// Note: this should be spawned + pub async fn run(mut self, tx: tokio::sync::watch::Sender>>) { + while let Some(block) = self.next().await { + if let Ok(block) = block.inspect_err(|e| tracing::error!("{e}")) { + let _ = tx.send(block).inspect_err(|e| tracing::error!("{e}")); + } + } + + warn!("Flashblock service has stopped"); + } + + /// Notifies all subscribers about the received flashblock + fn notify_received_flashblock(&self, flashblock: &FlashBlock) { + if self.received_flashblocks_tx.receiver_count() > 0 { + let _ = self.received_flashblocks_tx.send(Arc::new(flashblock.clone())); + } + } + + /// Returns the [`BuildArgs`] made purely out of [`FlashBlock`]s that were received earlier. + /// + /// Returns `None` if the flashblock have no `base` or the base is not a child block of latest. + fn build_args( + &mut self, + ) -> Option< + BuildArgs< + impl IntoIterator>> + + use, + >, + > { + let Some(base) = self.blocks.payload_base() else { + trace!( + flashblock_number = ?self.blocks.block_number(), + count = %self.blocks.count(), + "Missing flashblock payload base" + ); + + return None + }; + + // attempt an initial consecutive check + if let Some(latest) = self.builder.provider().latest_header().ok().flatten() && + latest.hash() != base.parent_hash + { + trace!(flashblock_parent=?base.parent_hash, flashblock_number=base.block_number, local_latest=?latest.num_hash(), "Skipping non consecutive build attempt"); + return None + } + + let Some(last_flashblock) = self.blocks.last_flashblock() else { + trace!(flashblock_number = ?self.blocks.block_number(), count = %self.blocks.count(), "Missing last flashblock"); + return None + }; + + // Check if state root must be computed + let compute_state_root = + self.compute_state_root && self.blocks.index() >= Some(FB_STATE_ROOT_FROM_INDEX as u64); + + Some(BuildArgs { + base, + transactions: self.blocks.ready_transactions().collect::>(), + cached_state: self.cached_state.take(), + last_flashblock_index: last_flashblock.index, + last_flashblock_hash: last_flashblock.diff.block_hash, + compute_state_root, + }) + } + + /// Takes out `current` [`PendingFlashBlock`] if `state` is not preceding it. + fn on_new_tip(&mut self, state: CanonStateNotification) -> Option> { + let tip = state.tip_checked()?; + let tip_hash = tip.hash(); + let current = self.current.take_if(|current| current.parent_hash() != tip_hash); + + // Prefill the cache with state from the new canonical tip, similar to payload/basic + let mut cached = CachedReads::default(); + let committed = state.committed(); + let new_execution_outcome = committed.execution_outcome(); + for (addr, acc) in new_execution_outcome.bundle_accounts_iter() { + if let Some(info) = acc.info.clone() { + // Pre-cache existing accounts and their storage (only changed accounts/storage) + let storage = + acc.storage.iter().map(|(key, slot)| (*key, slot.present_value)).collect(); + cached.insert_account(addr, info, storage); + } + } + self.cached_state = Some((tip_hash, cached)); + + current + } +} + +impl Stream for FlashBlockService +where + N: NodePrimitives, + S: Stream> + Unpin + 'static, + EvmConfig: ConfigureEvm + Unpin> + + Clone + + 'static, + Provider: StateProviderFactory + + CanonStateSubscriptions + + BlockReaderIdExt< + Header = HeaderTy, + Block = BlockTy, + Transaction = N::SignedTx, + Receipt = ReceiptTy, + > + Unpin + + Clone + + 'static, +{ + type Item = eyre::Result>>; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = self.get_mut(); + + loop { + // drive pending build job to completion + let result = match this.job.as_mut() { + Some((now, rx)) => { + let result = ready!(rx.poll_unpin(cx)); + result.ok().map(|res| (*now, res)) + } + None => None, + }; + // reset job + this.job.take(); + // No build in progress + let _ = this.in_progress_tx.send(None); + + if let Some((now, result)) = result { + match result { + Ok(Some((new_pending, cached_reads))) => { + // update state root of the current sequence + this.blocks.set_state_root(new_pending.computed_state_root()); + + // built a new pending block + this.current = Some(new_pending.clone()); + // cache reads + this.cached_state = Some((new_pending.parent_hash(), cached_reads)); + this.rebuild = false; + + let elapsed = now.elapsed(); + this.metrics.execution_duration.record(elapsed.as_secs_f64()); + trace!( + parent_hash = %new_pending.block().parent_hash(), + block_number = new_pending.block().number(), + flash_blocks = this.blocks.count(), + ?elapsed, + "Built new block with flashblocks" + ); + + return Poll::Ready(Some(Ok(Some(new_pending)))); + } + Ok(None) => { + // nothing to do because tracked flashblock doesn't attach to latest + } + Err(err) => { + // we can ignore this error + debug!(%err, "failed to execute flashblock"); + } + } + } + + // consume new flashblocks while they're ready + while let Poll::Ready(Some(result)) = this.rx.poll_next_unpin(cx) { + match result { + Ok(flashblock) => { + this.notify_received_flashblock(&flashblock); + if flashblock.index == 0 { + this.metrics.last_flashblock_length.record(this.blocks.count() as f64); + } + match this.blocks.insert(flashblock) { + Ok(_) => this.rebuild = true, + Err(err) => debug!(%err, "Failed to prepare flashblock"), + } + } + Err(err) => return Poll::Ready(Some(Err(err))), + } + } + + // update on new head block + if let Poll::Ready(Ok(state)) = { + let fut = this.canon_receiver.recv(); + pin!(fut); + fut.poll_unpin(cx) + } && let Some(current) = this.on_new_tip(state) + { + trace!( + parent_hash = %current.block().parent_hash(), + block_number = current.block().number(), + "Clearing current flashblock on new canonical block" + ); + + return Poll::Ready(Some(Ok(None))) + } + + if !this.rebuild && this.current.is_some() { + return Poll::Pending + } + + // try to build a block on top of latest + if let Some(args) = this.build_args() { + let now = Instant::now(); + + let fb_info = FlashBlockBuildInfo { + parent_hash: args.base.parent_hash, + index: args.last_flashblock_index, + block_number: args.base.block_number, + }; + // Signal that a flashblock build has started with build metadata + let _ = this.in_progress_tx.send(Some(fb_info)); + let (tx, rx) = oneshot::channel(); + let builder = this.builder.clone(); + + this.spawner.spawn_blocking(async move { + let _ = tx.send(builder.execute(args)); + }); + this.job.replace((now, rx)); + + // continue and poll the spawned job + continue + } + + return Poll::Pending + } + } +} + +/// Information for a flashblock currently built +#[derive(Debug, Clone, Copy)] +pub struct FlashBlockBuildInfo { + /// Parent block hash + pub parent_hash: B256, + /// Flashblock index within the current block's sequence + pub index: u64, + /// Block number of the flashblock being built. + pub block_number: u64, +} + +type BuildJob = + (Instant, oneshot::Receiver, CachedReads)>>>); + +#[derive(Metrics)] +#[metrics(scope = "flashblock_service")] +struct FlashBlockServiceMetrics { + /// The last complete length of flashblocks per block. + last_flashblock_length: Histogram, + /// The duration applying flashblock state changes in seconds. + execution_duration: Histogram, +} diff --git a/crates/optimism/flashblocks/src/worker.rs b/crates/optimism/flashblocks/src/worker.rs new file mode 100644 index 00000000000..8cf7777f6a6 --- /dev/null +++ b/crates/optimism/flashblocks/src/worker.rs @@ -0,0 +1,145 @@ +use crate::{ExecutionPayloadBaseV1, PendingFlashBlock}; +use alloy_eips::{eip2718::WithEncoded, BlockNumberOrTag}; +use alloy_primitives::B256; +use reth_chain_state::{CanonStateSubscriptions, ExecutedBlock}; +use reth_errors::RethError; +use reth_evm::{ + execute::{BlockBuilder, BlockBuilderOutcome}, + ConfigureEvm, +}; +use reth_execution_types::ExecutionOutcome; +use reth_primitives_traits::{ + AlloyBlockHeader, BlockTy, HeaderTy, NodePrimitives, ReceiptTy, Recovered, +}; +use reth_revm::{cached::CachedReads, database::StateProviderDatabase, db::State}; +use reth_rpc_eth_types::{EthApiError, PendingBlock}; +use reth_storage_api::{noop::NoopProvider, BlockReaderIdExt, StateProviderFactory}; +use std::{ + sync::Arc, + time::{Duration, Instant}, +}; +use tracing::trace; + +/// The `FlashBlockBuilder` builds [`PendingBlock`] out of a sequence of transactions. +#[derive(Debug)] +pub(crate) struct FlashBlockBuilder { + evm_config: EvmConfig, + provider: Provider, +} + +impl FlashBlockBuilder { + pub(crate) const fn new(evm_config: EvmConfig, provider: Provider) -> Self { + Self { evm_config, provider } + } + + pub(crate) const fn provider(&self) -> &Provider { + &self.provider + } +} + +pub(crate) struct BuildArgs { + pub(crate) base: ExecutionPayloadBaseV1, + pub(crate) transactions: I, + pub(crate) cached_state: Option<(B256, CachedReads)>, + pub(crate) last_flashblock_index: u64, + pub(crate) last_flashblock_hash: B256, + pub(crate) compute_state_root: bool, +} + +impl FlashBlockBuilder +where + N: NodePrimitives, + EvmConfig: ConfigureEvm + Unpin>, + Provider: StateProviderFactory + + CanonStateSubscriptions + + BlockReaderIdExt< + Header = HeaderTy, + Block = BlockTy, + Transaction = N::SignedTx, + Receipt = ReceiptTy, + > + Unpin, +{ + /// Returns the [`PendingFlashBlock`] made purely out of transactions and + /// [`ExecutionPayloadBaseV1`] in `args`. + /// + /// Returns `None` if the flashblock doesn't attach to the latest header. + pub(crate) fn execute>>>( + &self, + mut args: BuildArgs, + ) -> eyre::Result, CachedReads)>> { + trace!("Attempting new pending block from flashblocks"); + + let latest = self + .provider + .latest_header()? + .ok_or(EthApiError::HeaderNotFound(BlockNumberOrTag::Latest.into()))?; + let latest_hash = latest.hash(); + + if args.base.parent_hash != latest_hash { + trace!(flashblock_parent = ?args.base.parent_hash, local_latest=?latest.num_hash(),"Skipping non consecutive flashblock"); + // doesn't attach to the latest block + return Ok(None) + } + + let state_provider = self.provider.history_by_block_hash(latest.hash())?; + + let mut request_cache = args + .cached_state + .take() + .filter(|(hash, _)| hash == &latest_hash) + .map(|(_, state)| state) + .unwrap_or_default(); + let cached_db = request_cache.as_db_mut(StateProviderDatabase::new(&state_provider)); + let mut state = State::builder().with_database(cached_db).with_bundle_update().build(); + + let mut builder = self + .evm_config + .builder_for_next_block(&mut state, &latest, args.base.into()) + .map_err(RethError::other)?; + + builder.apply_pre_execution_changes()?; + + for tx in args.transactions { + let _gas_used = builder.execute_transaction(tx)?; + } + + // if the real state root should be computed + let BlockBuilderOutcome { execution_result, block, hashed_state, .. } = + if args.compute_state_root { + builder.finish(&state_provider)? + } else { + builder.finish(NoopProvider::default())? + }; + + let execution_outcome = ExecutionOutcome::new( + state.take_bundle(), + vec![execution_result.receipts], + block.number(), + vec![execution_result.requests], + ); + + let pending_block = PendingBlock::with_executed_block( + Instant::now() + Duration::from_secs(1), + ExecutedBlock { + recovered_block: block.into(), + execution_output: Arc::new(execution_outcome), + hashed_state: Arc::new(hashed_state), + trie_updates: Arc::default(), + }, + ); + let pending_flashblock = PendingFlashBlock::new( + pending_block, + args.last_flashblock_index, + args.last_flashblock_hash, + args.compute_state_root, + ); + + Ok(Some((pending_flashblock, request_cache))) + } +} + +impl Clone for FlashBlockBuilder { + fn clone(&self) -> Self { + Self { evm_config: self.evm_config.clone(), provider: self.provider.clone() } + } +} diff --git a/crates/optimism/flashblocks/src/ws/decoding.rs b/crates/optimism/flashblocks/src/ws/decoding.rs new file mode 100644 index 00000000000..267f79cf19a --- /dev/null +++ b/crates/optimism/flashblocks/src/ws/decoding.rs @@ -0,0 +1,66 @@ +use crate::{ExecutionPayloadBaseV1, ExecutionPayloadFlashblockDeltaV1, FlashBlock, Metadata}; +use alloy_primitives::bytes::Bytes; +use alloy_rpc_types_engine::PayloadId; +use serde::{Deserialize, Serialize}; +use std::{fmt::Debug, io}; + +/// Internal helper for decoding +#[derive(Clone, Debug, PartialEq, Default, Deserialize, Serialize)] +struct FlashblocksPayloadV1 { + /// The payload id of the flashblock + pub payload_id: PayloadId, + /// The index of the flashblock in the block + pub index: u64, + /// The base execution payload configuration + #[serde(skip_serializing_if = "Option::is_none")] + pub base: Option, + /// The delta/diff containing modified portions of the execution payload + pub diff: ExecutionPayloadFlashblockDeltaV1, + /// Additional metadata associated with the flashblock + pub metadata: serde_json::Value, +} + +impl FlashBlock { + /// Decodes `bytes` into [`FlashBlock`]. + /// + /// This function is specific to the Base Optimism websocket encoding. + /// + /// It is assumed that the `bytes` are encoded in JSON and optionally compressed using brotli. + /// Whether the `bytes` is compressed or not is determined by looking at the first + /// non ascii-whitespace character. + pub(crate) fn decode(bytes: Bytes) -> eyre::Result { + let bytes = try_parse_message(bytes)?; + + let payload: FlashblocksPayloadV1 = serde_json::from_slice(&bytes) + .map_err(|e| eyre::eyre!("failed to parse message: {e}"))?; + + let metadata: Metadata = serde_json::from_value(payload.metadata) + .map_err(|e| eyre::eyre!("failed to parse message metadata: {e}"))?; + + Ok(Self { + payload_id: payload.payload_id, + index: payload.index, + base: payload.base, + diff: payload.diff, + metadata, + }) + } +} + +/// Maps `bytes` into a potentially different [`Bytes`]. +/// +/// If the bytes start with a "{" character, prepended by any number of ASCII-whitespaces, +/// then it assumes that it is JSON-encoded and returns it as-is. +/// +/// Otherwise, the `bytes` are passed through a brotli decompressor and returned. +fn try_parse_message(bytes: Bytes) -> eyre::Result { + if bytes.trim_ascii_start().starts_with(b"{") { + return Ok(bytes); + } + + let mut decompressor = brotli::Decompressor::new(bytes.as_ref(), 4096); + let mut decompressed = Vec::new(); + io::copy(&mut decompressor, &mut decompressed)?; + + Ok(decompressed.into()) +} diff --git a/crates/optimism/flashblocks/src/ws/mod.rs b/crates/optimism/flashblocks/src/ws/mod.rs new file mode 100644 index 00000000000..2b820899312 --- /dev/null +++ b/crates/optimism/flashblocks/src/ws/mod.rs @@ -0,0 +1,4 @@ +pub use stream::{WsConnect, WsFlashBlockStream}; + +mod decoding; +mod stream; diff --git a/crates/optimism/flashblocks/src/ws/stream.rs b/crates/optimism/flashblocks/src/ws/stream.rs new file mode 100644 index 00000000000..64cf6f718e2 --- /dev/null +++ b/crates/optimism/flashblocks/src/ws/stream.rs @@ -0,0 +1,559 @@ +use crate::{FlashBlock, FlashBlockDecoder}; +use futures_util::{ + stream::{SplitSink, SplitStream}, + FutureExt, Sink, Stream, StreamExt, +}; +use std::{ + fmt::{Debug, Formatter}, + future::Future, + pin::Pin, + task::{ready, Context, Poll}, +}; +use tokio::net::TcpStream; +use tokio_tungstenite::{ + connect_async, + tungstenite::{protocol::CloseFrame, Bytes, Error, Message}, + MaybeTlsStream, WebSocketStream, +}; +use tracing::debug; +use url::Url; + +/// An asynchronous stream of [`FlashBlock`] from a websocket connection. +/// +/// The stream attempts to connect to a websocket URL and then decode each received item. +/// +/// If the connection fails, the error is returned and connection retried. The number of retries is +/// unbounded. +pub struct WsFlashBlockStream { + ws_url: Url, + state: State, + connector: Connector, + decoder: Box, + connect: ConnectFuture, + stream: Option, + sink: Option, +} + +impl WsFlashBlockStream { + /// Creates a new websocket stream over `ws_url`. + pub fn new(ws_url: Url) -> Self { + Self { + ws_url, + state: State::default(), + connector: WsConnector, + decoder: Box::new(()), + connect: Box::pin(async move { Err(Error::ConnectionClosed)? }), + stream: None, + sink: None, + } + } + + /// Sets the [`FlashBlock`] decoder for the websocket stream. + pub fn with_decoder(self, decoder: Box) -> Self { + Self { decoder, ..self } + } +} + +impl WsFlashBlockStream { + /// Creates a new websocket stream over `ws_url`. + pub fn with_connector(ws_url: Url, connector: C) -> Self { + Self { + ws_url, + state: State::default(), + decoder: Box::new(()), + connector, + connect: Box::pin(async move { Err(Error::ConnectionClosed)? }), + stream: None, + sink: None, + } + } +} + +impl Stream for WsFlashBlockStream +where + Str: Stream> + Unpin, + S: Sink + Send + Unpin, + C: WsConnect + Clone + Send + 'static + Unpin, +{ + type Item = eyre::Result; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = self.get_mut(); + + 'start: loop { + if this.state == State::Initial { + this.connect(); + } + + if this.state == State::Connect { + match ready!(this.connect.poll_unpin(cx)) { + Ok((sink, stream)) => this.stream(sink, stream), + Err(err) => { + this.state = State::Initial; + + return Poll::Ready(Some(Err(err))); + } + } + } + + while let State::Stream(msg) = &mut this.state { + if msg.is_some() { + let mut sink = Pin::new(this.sink.as_mut().unwrap()); + let _ = ready!(sink.as_mut().poll_ready(cx)); + if let Some(pong) = msg.take() { + let _ = sink.as_mut().start_send(pong); + } + let _ = ready!(sink.as_mut().poll_flush(cx)); + } + + let Some(msg) = ready!(this + .stream + .as_mut() + .expect("Stream state should be unreachable without stream") + .poll_next_unpin(cx)) + else { + this.state = State::Initial; + + continue 'start; + }; + + match msg { + Ok(Message::Binary(bytes)) => { + return Poll::Ready(Some(this.decoder.decode(bytes))) + } + Ok(Message::Text(bytes)) => { + return Poll::Ready(Some(this.decoder.decode(bytes.into()))) + } + Ok(Message::Ping(bytes)) => this.ping(bytes), + Ok(Message::Close(frame)) => this.close(frame), + Ok(msg) => debug!("Received unexpected message: {:?}", msg), + Err(err) => return Poll::Ready(Some(Err(err.into()))), + } + } + } + } +} + +impl WsFlashBlockStream +where + C: WsConnect + Clone + Send + 'static, +{ + fn connect(&mut self) { + let ws_url = self.ws_url.clone(); + let mut connector = self.connector.clone(); + + Pin::new(&mut self.connect).set(Box::pin(async move { connector.connect(ws_url).await })); + + self.state = State::Connect; + } + + fn stream(&mut self, sink: S, stream: Stream) { + self.sink.replace(sink); + self.stream.replace(stream); + + self.state = State::Stream(None); + } + + fn ping(&mut self, pong: Bytes) { + if let State::Stream(current) = &mut self.state { + current.replace(Message::Pong(pong)); + } + } + + fn close(&mut self, frame: Option) { + if let State::Stream(current) = &mut self.state { + current.replace(Message::Close(frame)); + } + } +} + +impl Debug for WsFlashBlockStream { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("FlashBlockStream") + .field("ws_url", &self.ws_url) + .field("state", &self.state) + .field("connector", &self.connector) + .field("connect", &"Pin>>") + .field("stream", &self.stream) + .finish() + } +} + +#[derive(Default, Debug, Eq, PartialEq)] +enum State { + #[default] + Initial, + Connect, + Stream(Option), +} + +type Ws = WebSocketStream>; +type WsStream = SplitStream; +type WsSink = SplitSink; +type ConnectFuture = + Pin> + Send + 'static>>; + +/// The `WsConnect` trait allows for connecting to a websocket. +/// +/// Implementors of the `WsConnect` trait are called 'connectors'. +/// +/// Connectors are defined by one method, [`connect()`]. A call to [`connect()`] attempts to +/// establish a secure websocket connection and return an asynchronous stream of [`Message`]s +/// wrapped in a [`Result`]. +/// +/// [`connect()`]: Self::connect +pub trait WsConnect { + /// An associated `Stream` of [`Message`]s wrapped in a [`Result`] that this connection returns. + type Stream; + + /// An associated `Sink` of [`Message`]s that this connection sends. + type Sink; + + /// Asynchronously connects to a websocket hosted on `ws_url`. + /// + /// See the [`WsConnect`] documentation for details. + fn connect( + &mut self, + ws_url: Url, + ) -> impl Future> + Send; +} + +/// Establishes a secure websocket subscription. +/// +/// See the [`WsConnect`] documentation for details. +#[derive(Debug, Clone)] +pub struct WsConnector; + +impl WsConnect for WsConnector { + type Stream = WsStream; + type Sink = WsSink; + + async fn connect(&mut self, ws_url: Url) -> eyre::Result<(WsSink, WsStream)> { + let (stream, _response) = connect_async(ws_url.as_str()).await?; + + Ok(stream.split()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ExecutionPayloadBaseV1; + use alloy_primitives::bytes::Bytes; + use brotli::enc::BrotliEncoderParams; + use std::{future, iter}; + use tokio_tungstenite::tungstenite::{ + protocol::frame::{coding::CloseCode, Frame}, + Error, + }; + + /// A `FakeConnector` creates [`FakeStream`]. + /// + /// It simulates the websocket stream instead of connecting to a real websocket. + #[derive(Clone)] + struct FakeConnector(FakeStream); + + /// A `FakeConnectorWithSink` creates [`FakeStream`] and [`FakeSink`]. + /// + /// It simulates the websocket stream instead of connecting to a real websocket. It also accepts + /// messages into an in-memory buffer. + #[derive(Clone)] + struct FakeConnectorWithSink(FakeStream); + + /// Simulates a websocket stream while using a preprogrammed set of messages instead. + #[derive(Default)] + struct FakeStream(Vec>); + + impl FakeStream { + fn new(mut messages: Vec>) -> Self { + messages.reverse(); + + Self(messages) + } + } + + impl Clone for FakeStream { + fn clone(&self) -> Self { + Self( + self.0 + .iter() + .map(|v| match v { + Ok(msg) => Ok(msg.clone()), + Err(err) => Err(match err { + Error::AttackAttempt => Error::AttackAttempt, + err => unimplemented!("Cannot clone this error: {err}"), + }), + }) + .collect(), + ) + } + } + + impl Stream for FakeStream { + type Item = Result; + + fn poll_next(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + let this = self.get_mut(); + + Poll::Ready(this.0.pop()) + } + } + + #[derive(Clone)] + struct NoopSink; + + impl Sink for NoopSink { + type Error = (); + + fn poll_ready( + self: Pin<&mut Self>, + _cx: &mut Context<'_>, + ) -> Poll> { + unimplemented!() + } + + fn start_send(self: Pin<&mut Self>, _item: T) -> Result<(), Self::Error> { + unimplemented!() + } + + fn poll_flush( + self: Pin<&mut Self>, + _cx: &mut Context<'_>, + ) -> Poll> { + unimplemented!() + } + + fn poll_close( + self: Pin<&mut Self>, + _cx: &mut Context<'_>, + ) -> Poll> { + unimplemented!() + } + } + + /// Receives [`Message`]s and stores them. A call to `start_send` first buffers the message + /// to simulate flushing behavior. + #[derive(Clone, Default)] + struct FakeSink(Option, Vec); + + impl Sink for FakeSink { + type Error = (); + + fn poll_ready(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + self.poll_flush(cx) + } + + fn start_send(self: Pin<&mut Self>, item: Message) -> Result<(), Self::Error> { + self.get_mut().0.replace(item); + Ok(()) + } + + fn poll_flush( + self: Pin<&mut Self>, + _cx: &mut Context<'_>, + ) -> Poll> { + let this = self.get_mut(); + if let Some(item) = this.0.take() { + this.1.push(item); + } + Poll::Ready(Ok(())) + } + + fn poll_close( + self: Pin<&mut Self>, + _cx: &mut Context<'_>, + ) -> Poll> { + Poll::Ready(Ok(())) + } + } + + impl WsConnect for FakeConnector { + type Stream = FakeStream; + type Sink = NoopSink; + + fn connect( + &mut self, + _ws_url: Url, + ) -> impl Future> + Send { + future::ready(Ok((NoopSink, self.0.clone()))) + } + } + + impl>> From for FakeConnector { + fn from(value: T) -> Self { + Self(FakeStream::new(value.into_iter().collect())) + } + } + + impl WsConnect for FakeConnectorWithSink { + type Stream = FakeStream; + type Sink = FakeSink; + + fn connect( + &mut self, + _ws_url: Url, + ) -> impl Future> + Send { + future::ready(Ok((FakeSink::default(), self.0.clone()))) + } + } + + impl>> From for FakeConnectorWithSink { + fn from(value: T) -> Self { + Self(FakeStream::new(value.into_iter().collect())) + } + } + + /// Repeatedly fails to connect with the given error message. + #[derive(Clone)] + struct FailingConnector(String); + + impl WsConnect for FailingConnector { + type Stream = FakeStream; + type Sink = NoopSink; + + fn connect( + &mut self, + _ws_url: Url, + ) -> impl Future> + Send { + future::ready(Err(eyre::eyre!("{}", &self.0))) + } + } + + fn to_json_message, F: Fn(B) -> Message>( + wrapper_f: F, + ) -> impl Fn(&FlashBlock) -> Result + use { + move |block| to_json_message_using(block, &wrapper_f) + } + + fn to_json_binary_message(block: &FlashBlock) -> Result { + to_json_message_using(block, Message::Binary) + } + + fn to_json_message_using, F: Fn(B) -> Message>( + block: &FlashBlock, + wrapper_f: F, + ) -> Result { + Ok(wrapper_f(B::try_from(Bytes::from(serde_json::to_vec(block).unwrap())).unwrap())) + } + + fn to_brotli_message(block: &FlashBlock) -> Result { + let json = serde_json::to_vec(block).unwrap(); + let mut compressed = Vec::new(); + brotli::BrotliCompress( + &mut json.as_slice(), + &mut compressed, + &BrotliEncoderParams::default(), + )?; + + Ok(Message::Binary(Bytes::from(compressed))) + } + + fn flashblock() -> FlashBlock { + FlashBlock { + payload_id: Default::default(), + index: 0, + base: Some(ExecutionPayloadBaseV1 { + parent_beacon_block_root: Default::default(), + parent_hash: Default::default(), + fee_recipient: Default::default(), + prev_randao: Default::default(), + block_number: 0, + gas_limit: 0, + timestamp: 0, + extra_data: Default::default(), + base_fee_per_gas: Default::default(), + }), + diff: Default::default(), + metadata: Default::default(), + } + } + + #[test_case::test_case(to_json_message(Message::Binary); "json binary")] + #[test_case::test_case(to_json_message(Message::Text); "json UTF-8")] + #[test_case::test_case(to_brotli_message; "brotli")] + #[tokio::test] + async fn test_stream_decodes_messages_successfully( + to_message: impl Fn(&FlashBlock) -> Result, + ) { + let flashblocks = [flashblock()]; + let connector = FakeConnector::from(flashblocks.iter().map(to_message)); + let ws_url = "http://localhost".parse().unwrap(); + let stream = WsFlashBlockStream::with_connector(ws_url, connector); + + let actual_messages: Vec<_> = stream.take(1).map(Result::unwrap).collect().await; + let expected_messages = flashblocks.to_vec(); + + assert_eq!(actual_messages, expected_messages); + } + + #[test_case::test_case(Message::Pong(Bytes::from(b"test".as_slice())); "pong")] + #[test_case::test_case(Message::Frame(Frame::pong(b"test".as_slice())); "frame")] + #[tokio::test] + async fn test_stream_ignores_unexpected_message(message: Message) { + let flashblock = flashblock(); + let connector = FakeConnector::from([Ok(message), to_json_binary_message(&flashblock)]); + let ws_url = "http://localhost".parse().unwrap(); + let mut stream = WsFlashBlockStream::with_connector(ws_url, connector); + + let expected_message = flashblock; + let actual_message = + stream.next().await.expect("Binary message should not be ignored").unwrap(); + + assert_eq!(actual_message, expected_message) + } + + #[tokio::test] + async fn test_stream_passes_errors_through() { + let connector = FakeConnector::from([Err(Error::AttackAttempt)]); + let ws_url = "http://localhost".parse().unwrap(); + let stream = WsFlashBlockStream::with_connector(ws_url, connector); + + let actual_messages: Vec<_> = + stream.take(1).map(Result::unwrap_err).map(|e| format!("{e}")).collect().await; + let expected_messages = vec!["Attack attempt detected".to_owned()]; + + assert_eq!(actual_messages, expected_messages); + } + + #[tokio::test] + async fn test_connect_error_causes_retries() { + let tries = 3; + let error_msg = "test".to_owned(); + let connector = FailingConnector(error_msg.clone()); + let ws_url = "http://localhost".parse().unwrap(); + let stream = WsFlashBlockStream::with_connector(ws_url, connector); + + let actual_errors: Vec<_> = + stream.take(tries).map(Result::unwrap_err).map(|e| format!("{e}")).collect().await; + let expected_errors: Vec<_> = iter::repeat_n(error_msg, tries).collect(); + + assert_eq!(actual_errors, expected_errors); + } + + #[test_case::test_case( + Message::Close(Some(CloseFrame { code: CloseCode::Normal, reason: "test".into() })), + Message::Close(Some(CloseFrame { code: CloseCode::Normal, reason: "test".into() })); + "close" + )] + #[test_case::test_case( + Message::Ping(Bytes::from_static(&[1u8, 2, 3])), + Message::Pong(Bytes::from_static(&[1u8, 2, 3])); + "ping" + )] + #[tokio::test] + async fn test_stream_responds_to_messages(msg: Message, expected_response: Message) { + let flashblock = flashblock(); + let messages = [Ok(msg), to_json_binary_message(&flashblock)]; + let connector = FakeConnectorWithSink::from(messages); + let ws_url = "http://localhost".parse().unwrap(); + let mut stream = WsFlashBlockStream::with_connector(ws_url, connector); + + let _ = stream.next().await; + + let expected_response = vec![expected_response]; + let FakeSink(actual_buffer, actual_response) = stream.sink.unwrap(); + + assert!(actual_buffer.is_none(), "buffer not flushed: {actual_buffer:#?}"); + assert_eq!(actual_response, expected_response); + } +} diff --git a/crates/optimism/flashblocks/tests/it/main.rs b/crates/optimism/flashblocks/tests/it/main.rs new file mode 100644 index 00000000000..bfe1f9695a9 --- /dev/null +++ b/crates/optimism/flashblocks/tests/it/main.rs @@ -0,0 +1,5 @@ +//! Integration tests. +//! +//! All the individual modules are rooted here to produce a single binary. + +mod stream; diff --git a/crates/optimism/flashblocks/tests/it/stream.rs b/crates/optimism/flashblocks/tests/it/stream.rs new file mode 100644 index 00000000000..99e78fee23a --- /dev/null +++ b/crates/optimism/flashblocks/tests/it/stream.rs @@ -0,0 +1,15 @@ +use futures_util::stream::StreamExt; +use reth_optimism_flashblocks::WsFlashBlockStream; + +#[tokio::test] +async fn test_streaming_flashblocks_from_remote_source_is_successful() { + let items = 3; + let ws_url = "wss://sepolia.flashblocks.base.org/ws".parse().unwrap(); + let stream = WsFlashBlockStream::new(ws_url); + + let blocks: Vec<_> = stream.take(items).collect().await; + + for block in blocks { + assert!(block.is_ok()); + } +} diff --git a/crates/optimism/hardforks/src/lib.rs b/crates/optimism/hardforks/src/lib.rs index aad509a11bf..202194c63a4 100644 --- a/crates/optimism/hardforks/src/lib.rs +++ b/crates/optimism/hardforks/src/lib.rs @@ -12,12 +12,16 @@ html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256", issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(feature = "std"), no_std)] #![cfg_attr(not(test), warn(unused_crate_dependencies))] extern crate alloc; +use alloy_op_hardforks::{ + BASE_MAINNET_JOVIAN_TIMESTAMP, BASE_SEPOLIA_JOVIAN_TIMESTAMP, OP_MAINNET_JOVIAN_TIMESTAMP, + OP_SEPOLIA_JOVIAN_TIMESTAMP, +}; // Re-export alloy-op-hardforks types. pub use alloy_op_hardforks::{OpHardfork, OpHardforks}; @@ -28,6 +32,7 @@ use reth_ethereum_forks::{ChainHardforks, EthereumHardfork, ForkCondition, Hardf /// Dev hardforks pub static DEV_HARDFORKS: LazyLock = LazyLock::new(|| { + const JOVIAN_TIMESTAMP: ForkCondition = ForkCondition::Timestamp(1761840000); ChainHardforks::new(vec![ (EthereumHardfork::Frontier.boxed(), ForkCondition::Block(0)), (EthereumHardfork::Homestead.boxed(), ForkCondition::Block(0)), @@ -58,6 +63,7 @@ pub static DEV_HARDFORKS: LazyLock = LazyLock::new(|| { (OpHardfork::Granite.boxed(), ForkCondition::Timestamp(0)), (EthereumHardfork::Prague.boxed(), ForkCondition::Timestamp(0)), (OpHardfork::Isthmus.boxed(), ForkCondition::Timestamp(0)), + (OpHardfork::Jovian.boxed(), JOVIAN_TIMESTAMP), ]) }); @@ -96,6 +102,7 @@ pub static OP_MAINNET_HARDFORKS: LazyLock = LazyLock::new(|| { (OpHardfork::Holocene.boxed(), ForkCondition::Timestamp(1736445601)), (EthereumHardfork::Prague.boxed(), ForkCondition::Timestamp(1746806401)), (OpHardfork::Isthmus.boxed(), ForkCondition::Timestamp(1746806401)), + (OpHardfork::Jovian.boxed(), ForkCondition::Timestamp(OP_MAINNET_JOVIAN_TIMESTAMP)), ]) }); /// Optimism Sepolia list of hardforks. @@ -133,6 +140,7 @@ pub static OP_SEPOLIA_HARDFORKS: LazyLock = LazyLock::new(|| { (OpHardfork::Holocene.boxed(), ForkCondition::Timestamp(1732633200)), (EthereumHardfork::Prague.boxed(), ForkCondition::Timestamp(1744905600)), (OpHardfork::Isthmus.boxed(), ForkCondition::Timestamp(1744905600)), + (OpHardfork::Jovian.boxed(), ForkCondition::Timestamp(OP_SEPOLIA_JOVIAN_TIMESTAMP)), ]) }); @@ -171,6 +179,7 @@ pub static BASE_SEPOLIA_HARDFORKS: LazyLock = LazyLock::new(|| { (OpHardfork::Holocene.boxed(), ForkCondition::Timestamp(1732633200)), (EthereumHardfork::Prague.boxed(), ForkCondition::Timestamp(1744905600)), (OpHardfork::Isthmus.boxed(), ForkCondition::Timestamp(1744905600)), + (OpHardfork::Jovian.boxed(), ForkCondition::Timestamp(BASE_SEPOLIA_JOVIAN_TIMESTAMP)), ]) }); @@ -209,5 +218,6 @@ pub static BASE_MAINNET_HARDFORKS: LazyLock = LazyLock::new(|| { (OpHardfork::Holocene.boxed(), ForkCondition::Timestamp(1736445601)), (EthereumHardfork::Prague.boxed(), ForkCondition::Timestamp(1746806401)), (OpHardfork::Isthmus.boxed(), ForkCondition::Timestamp(1746806401)), + (OpHardfork::Jovian.boxed(), ForkCondition::Timestamp(BASE_MAINNET_JOVIAN_TIMESTAMP)), ]) }); diff --git a/crates/optimism/node/Cargo.toml b/crates/optimism/node/Cargo.toml index 84942f21104..f2dc8cee73f 100644 --- a/crates/optimism/node/Cargo.toml +++ b/crates/optimism/node/Cargo.toml @@ -13,7 +13,8 @@ workspace = true [dependencies] # reth reth-chainspec.workspace = true -reth-primitives-traits.workspace = true +## ensure secp256k1 recovery with rayon support is activated +reth-primitives-traits = { workspace = true, features = ["secp256k1", "rayon"] } reth-payload-builder.workspace = true reth-consensus.workspace = true reth-node-api.workspace = true @@ -23,28 +24,32 @@ reth-provider.workspace = true reth-transaction-pool.workspace = true reth-network.workspace = true reth-evm.workspace = true -reth-trie-db.workspace = true reth-rpc-server-types.workspace = true -reth-rpc-eth-api.workspace = true -reth-rpc-eth-types.workspace = true reth-tasks = { workspace = true, optional = true } reth-trie-common.workspace = true reth-node-core.workspace = true reth-rpc-engine-api.workspace = true +reth-engine-local = { workspace = true, features = ["op"] } reth-rpc-api.workspace = true # op-reth reth-optimism-payload-builder.workspace = true -reth-optimism-evm.workspace = true +reth-optimism-evm = { workspace = true, features = ["rpc"] } reth-optimism-rpc.workspace = true +reth-optimism-storage.workspace = true reth-optimism-txpool.workspace = true reth-optimism-chainspec.workspace = true reth-optimism-consensus = { workspace = true, features = ["std"] } reth-optimism-forks.workspace = true -reth-optimism-primitives = { workspace = true, features = ["serde", "serde-bincode-compat", "reth-codec"] } +reth-optimism-primitives = { workspace = true, features = [ + "serde", + "serde-bincode-compat", + "reth-codec", +] } # revm with required optimism features -revm = { workspace = true, features = ["secp256k1", "blst", "c-kzg"] } +# Note: this must be kept to ensure all features are properly enabled/forwarded +revm = { workspace = true, features = ["secp256k1", "blst", "c-kzg", "memory_limit"] } op-revm.workspace = true # ethereum @@ -58,52 +63,53 @@ alloy-consensus.workspace = true # Mantle reth-mantle-forks.workspace = true +# async +tokio.workspace = true + # misc clap.workspace = true serde.workspace = true eyre.workspace = true +url.workspace = true # test-utils dependencies reth-e2e-test-utils = { workspace = true, optional = true } alloy-genesis = { workspace = true, optional = true } -tokio = { workspace = true, optional = true } serde_json = { workspace = true, optional = true } [dev-dependencies] reth-optimism-node = { workspace = true, features = ["test-utils"] } -reth-db = { workspace = true, features = ["op"] } -reth-node-core.workspace = true +reth-db = { workspace = true, features = ["op", "test-utils"] } reth-node-builder = { workspace = true, features = ["test-utils"] } reth-provider = { workspace = true, features = ["test-utils"] } reth-tasks.workspace = true reth-payload-util.workspace = true -reth-payload-validator.workspace = true reth-revm = { workspace = true, features = ["std"] } -reth-engine-local = { workspace = true, features = ["op"] } +reth-rpc.workspace = true +reth-rpc-eth-types.workspace = true -alloy-primitives.workspace = true -op-alloy-consensus.workspace = true alloy-network.workspace = true -alloy-consensus.workspace = true futures.workspace = true -alloy-eips.workspace = true +op-alloy-network.workspace = true [features] default = ["reth-codec"] asm-keccak = [ "alloy-primitives/asm-keccak", - "revm/asm-keccak", "reth-optimism-node/asm-keccak", "reth-node-core/asm-keccak", + "revm/asm-keccak", ] js-tracer = [ "reth-node-builder/js-tracer", + "reth-optimism-node/js-tracer", + "reth-rpc/js-tracer", + "reth-rpc-eth-types/js-tracer", ] test-utils = [ "reth-tasks", "reth-e2e-test-utils", "alloy-genesis", - "tokio", "serde_json", "reth-node-builder/test-utils", "reth-chainspec/test-utils", @@ -115,12 +121,13 @@ test-utils = [ "reth-db/test-utils", "reth-provider/test-utils", "reth-transaction-pool/test-utils", - "reth-trie-db/test-utils", "reth-optimism-node/test-utils", "reth-optimism-primitives/arbitrary", "reth-primitives-traits/test-utils", "reth-trie-common/test-utils", ] -reth-codec = [ - "reth-optimism-primitives/reth-codec", -] +reth-codec = ["reth-optimism-primitives/reth-codec"] + +[[test]] +name = "e2e_testsuite" +path = "tests/e2e-testsuite/main.rs" diff --git a/crates/optimism/node/src/args.rs b/crates/optimism/node/src/args.rs index 703313aabcd..4e9bb2ce7c3 100644 --- a/crates/optimism/node/src/args.rs +++ b/crates/optimism/node/src/args.rs @@ -4,6 +4,7 @@ use op_alloy_consensus::interop::SafetyLevel; use reth_optimism_txpool::supervisor::DEFAULT_SUPERVISOR_URL; +use url::Url; /// Parameters for rollup configuration #[derive(Debug, Clone, PartialEq, Eq, clap::Args)] @@ -17,11 +18,6 @@ pub struct RollupArgs { #[arg(long = "rollup.disable-tx-pool-gossip")] pub disable_txpool_gossip: bool, - /// Enable walkback to genesis on startup. This is useful for re-validating the existing DB - /// prior to beginning normal syncing. - #[arg(long = "rollup.enable-genesis-walkback")] - pub enable_genesis_walkback: bool, - /// By default the pending block equals the latest block /// to save resources and not leak txs from the tx-pool, /// this flag enables computing of the pending block @@ -59,6 +55,25 @@ pub struct RollupArgs { /// Optional headers to use when connecting to the sequencer. #[arg(long = "rollup.sequencer-headers", requires = "sequencer")] pub sequencer_headers: Vec, + + /// RPC endpoint for historical data. + #[arg( + long = "rollup.historicalrpc", + alias = "rollup.historical-rpc", + value_name = "HISTORICAL_HTTP_URL" + )] + pub historical_rpc: Option, + + /// Minimum suggested priority fee (tip) in wei, default `1_000_000` + #[arg(long, default_value_t = 1_000_000)] + pub min_suggested_priority_fee: u64, + + /// A URL pointing to a secure websocket subscription that streams out flashblocks. + /// + /// If given, the flashblocks are received to build pending block. All request with "pending" + /// block tag will use the pending state based on flashblocks. + #[arg(long)] + pub flashblocks_url: Option, } impl Default for RollupArgs { @@ -66,13 +81,15 @@ impl Default for RollupArgs { Self { sequencer: None, disable_txpool_gossip: false, - enable_genesis_walkback: false, compute_pending_block: false, discovery_v4: false, enable_tx_conditional: false, supervisor_http: DEFAULT_SUPERVISOR_URL.to_string(), supervisor_safety_level: SafetyLevel::CrossUnsafe, sequencer_headers: Vec::new(), + historical_rpc: None, + min_suggested_priority_fee: 1_000_000, + flashblocks_url: None, } } } @@ -96,15 +113,6 @@ mod tests { assert_eq!(args, default_args); } - #[test] - fn test_parse_optimism_walkback_args() { - let expected_args = RollupArgs { enable_genesis_walkback: true, ..Default::default() }; - let args = - CommandParser::::parse_from(["reth", "--rollup.enable-genesis-walkback"]) - .args; - assert_eq!(args, expected_args); - } - #[test] fn test_parse_optimism_compute_pending_block_args() { let expected_args = RollupArgs { compute_pending_block: true, ..Default::default() }; @@ -157,7 +165,6 @@ mod tests { let expected_args = RollupArgs { disable_txpool_gossip: true, compute_pending_block: true, - enable_genesis_walkback: true, enable_tx_conditional: true, sequencer: Some("http://host:port".into()), ..Default::default() @@ -166,7 +173,6 @@ mod tests { "reth", "--rollup.disable-tx-pool-gossip", "--rollup.compute-pending-block", - "--rollup.enable-genesis-walkback", "--rollup.enable-tx-conditional", "--rollup.sequencer-http", "http://host:port", diff --git a/crates/optimism/node/src/engine.rs b/crates/optimism/node/src/engine.rs index 1a784f36bda..9f4aa327191 100644 --- a/crates/optimism/node/src/engine.rs +++ b/crates/optimism/node/src/engine.rs @@ -5,7 +5,6 @@ use op_alloy_rpc_types_engine::{ OpExecutionData, OpExecutionPayloadEnvelopeV3, OpExecutionPayloadEnvelopeV4, OpPayloadAttributes, }; -use reth_chainspec::{ChainSpec, EthereumHardfork}; use reth_consensus::ConsensusError; use reth_node_api::{ payload::{ @@ -13,33 +12,26 @@ use reth_node_api::{ EngineObjectValidationError, MessageValidationKind, NewPayloadError, PayloadOrAttributes, PayloadTypes, VersionSpecificValidationError, }, - validate_version_specific_fields, BuiltPayload, EngineTypes, EngineValidator, NodePrimitives, - PayloadValidator, + validate_version_specific_fields, BuiltPayload, EngineApiValidator, EngineTypes, + NodePrimitives, PayloadValidator, }; -use reth_optimism_chainspec::OpChainSpec; use reth_optimism_consensus::isthmus; use reth_optimism_forks::OpHardforks; use reth_optimism_payload_builder::{OpExecutionPayloadValidator, OpPayloadTypes}; use reth_optimism_primitives::{OpBlock, ADDRESS_L2_TO_L1_MESSAGE_PASSER}; -use reth_primitives_traits::{RecoveredBlock, SealedBlock}; +use reth_primitives_traits::{Block, RecoveredBlock, SealedBlock, SignedTransaction}; use reth_provider::StateProviderFactory; use reth_trie_common::{HashedPostState, KeyHasher}; -use std::sync::Arc; +use std::{marker::PhantomData, sync::Arc}; /// The types used in the optimism beacon consensus engine. #[derive(Debug, Default, Clone, serde::Deserialize, serde::Serialize)] #[non_exhaustive] pub struct OpEngineTypes { - _marker: std::marker::PhantomData, + _marker: PhantomData, } -impl< - T: PayloadTypes< - ExecutionData = OpExecutionData, - BuiltPayload: BuiltPayload>, - >, - > PayloadTypes for OpEngineTypes -{ +impl> PayloadTypes for OpEngineTypes { type ExecutionData = T::ExecutionData; type BuiltPayload = T::BuiltPayload; type PayloadAttributes = T::PayloadAttributes; @@ -50,7 +42,10 @@ impl< <::Primitives as NodePrimitives>::Block, >, ) -> ::ExecutionData { - OpExecutionData::from_block_unchecked(block.hash(), &block.into_block()) + OpExecutionData::from_block_unchecked( + block.hash(), + &block.into_block().into_ethereum_block(), + ) } } @@ -66,44 +61,69 @@ where type ExecutionPayloadEnvelopeV2 = ExecutionPayloadEnvelopeV2; type ExecutionPayloadEnvelopeV3 = OpExecutionPayloadEnvelopeV3; type ExecutionPayloadEnvelopeV4 = OpExecutionPayloadEnvelopeV4; + type ExecutionPayloadEnvelopeV5 = OpExecutionPayloadEnvelopeV4; } /// Validator for Optimism engine API. -#[derive(Debug, Clone)] -pub struct OpEngineValidator

{ - inner: OpExecutionPayloadValidator, +#[derive(Debug)] +pub struct OpEngineValidator { + inner: OpExecutionPayloadValidator, provider: P, hashed_addr_l2tol1_msg_passer: B256, + phantom: PhantomData, } -impl

OpEngineValidator

{ +impl OpEngineValidator { /// Instantiates a new validator. - pub fn new(chain_spec: Arc, provider: P) -> Self { + pub fn new(chain_spec: Arc, provider: P) -> Self { let hashed_addr_l2tol1_msg_passer = KH::hash_key(ADDRESS_L2_TO_L1_MESSAGE_PASSER); Self { inner: OpExecutionPayloadValidator::new(chain_spec), provider, hashed_addr_l2tol1_msg_passer, + phantom: PhantomData, + } + } +} + +impl Clone for OpEngineValidator +where + P: Clone, + ChainSpec: OpHardforks, +{ + fn clone(&self) -> Self { + Self { + inner: OpExecutionPayloadValidator::new(self.inner.clone()), + provider: self.provider.clone(), + hashed_addr_l2tol1_msg_passer: self.hashed_addr_l2tol1_msg_passer, + phantom: Default::default(), } } +} +impl OpEngineValidator +where + ChainSpec: OpHardforks, +{ /// Returns the chain spec used by the validator. #[inline] - fn chain_spec(&self) -> &OpChainSpec { + pub fn chain_spec(&self) -> &ChainSpec { self.inner.chain_spec() } } -impl

PayloadValidator for OpEngineValidator

+impl PayloadValidator for OpEngineValidator where P: StateProviderFactory + Unpin + 'static, + Tx: SignedTransaction + Unpin + 'static, + ChainSpec: OpHardforks + Send + Sync + 'static, + Types: PayloadTypes, { - type Block = OpBlock; - type ExecutionData = OpExecutionData; + type Block = alloy_consensus::Block; fn ensure_well_formed_payload( &self, - payload: Self::ExecutionData, + payload: OpExecutionData, ) -> Result, NewPayloadError> { let sealed_block = self.inner.ensure_well_formed_payload(payload).map_err(NewPayloadError::other)?; @@ -120,7 +140,7 @@ where // FIXME: we don't necessarily have access to the parent block here because the // parent block isn't necessarily part of the canonical chain yet. Instead this // function should receive the list of in memory blocks as input - return Ok(()) + return Ok(()); }; let predeploy_storage_updates = state_updates .storages @@ -141,15 +161,25 @@ where } } -impl EngineValidator for OpEngineValidator

+impl EngineApiValidator for OpEngineValidator where - Types: PayloadTypes, + Types: PayloadTypes< + PayloadAttributes = OpPayloadAttributes, + ExecutionData = OpExecutionData, + BuiltPayload: BuiltPayload>, + >, P: StateProviderFactory + Unpin + 'static, + Tx: SignedTransaction + Unpin + 'static, + ChainSpec: OpHardforks + Send + Sync + 'static, { fn validate_version_specific_fields( &self, version: EngineApiMessageVersion, - payload_or_attrs: PayloadOrAttributes<'_, Self::ExecutionData, OpPayloadAttributes>, + payload_or_attrs: PayloadOrAttributes< + '_, + Types::ExecutionData, + ::PayloadAttributes, + >, ) -> Result<(), EngineObjectValidationError> { validate_withdrawals_presence( self.chain_spec(), @@ -170,12 +200,12 @@ where fn ensure_well_formed_attributes( &self, version: EngineApiMessageVersion, - attributes: &OpPayloadAttributes, + attributes: &::PayloadAttributes, ) -> Result<(), EngineObjectValidationError> { validate_version_specific_fields( self.chain_spec(), version, - PayloadOrAttributes::::PayloadAttributes( + PayloadOrAttributes::::PayloadAttributes( attributes, ), )?; @@ -183,7 +213,7 @@ where if attributes.gas_limit.is_none() { return Err(EngineObjectValidationError::InvalidParams( "MissingGasLimitInPayloadAttributes".to_string().into(), - )) + )); } if self @@ -196,11 +226,25 @@ where "MissingEip1559ParamsInPayloadAttributes".to_string().into(), ) })?; + if elasticity != 0 && denominator == 0 { return Err(EngineObjectValidationError::InvalidParams( "Eip1559ParamsDenominatorZero".to_string().into(), - )) + )); + } + } + + if self.chain_spec().is_jovian_active_at_timestamp(attributes.payload_attributes.timestamp) + { + if attributes.min_base_fee.is_none() { + return Err(EngineObjectValidationError::InvalidParams( + "MissingMinBaseFeeInPayloadAttributes".to_string().into(), + )); } + } else if attributes.min_base_fee.is_some() { + return Err(EngineObjectValidationError::InvalidParams( + "MinBaseFeeNotAllowedBeforeJovian".to_string().into(), + )); } Ok(()) @@ -215,33 +259,36 @@ where /// Canyon activates the Shanghai EIPs, see the Canyon specs for more details: /// pub fn validate_withdrawals_presence( - chain_spec: &ChainSpec, + chain_spec: impl OpHardforks, version: EngineApiMessageVersion, message_validation_kind: MessageValidationKind, timestamp: u64, has_withdrawals: bool, ) -> Result<(), EngineObjectValidationError> { - let is_shanghai = chain_spec.fork(EthereumHardfork::Shanghai).active_at_timestamp(timestamp); + let is_shanghai = chain_spec.is_shanghai_active_at_timestamp(timestamp); match version { EngineApiMessageVersion::V1 => { if has_withdrawals { return Err(message_validation_kind - .to_error(VersionSpecificValidationError::WithdrawalsNotSupportedInV1)) + .to_error(VersionSpecificValidationError::WithdrawalsNotSupportedInV1)); } if is_shanghai { return Err(message_validation_kind - .to_error(VersionSpecificValidationError::NoWithdrawalsPostShanghai)) + .to_error(VersionSpecificValidationError::NoWithdrawalsPostShanghai)); } } - EngineApiMessageVersion::V2 | EngineApiMessageVersion::V3 | EngineApiMessageVersion::V4 => { + EngineApiMessageVersion::V2 + | EngineApiMessageVersion::V3 + | EngineApiMessageVersion::V4 + | EngineApiMessageVersion::V5 => { if is_shanghai && !has_withdrawals { return Err(message_validation_kind - .to_error(VersionSpecificValidationError::NoWithdrawalsPostShanghai)) + .to_error(VersionSpecificValidationError::NoWithdrawalsPostShanghai)); } if !is_shanghai && has_withdrawals { return Err(message_validation_kind - .to_error(VersionSpecificValidationError::HasWithdrawalsPreShanghai)) + .to_error(VersionSpecificValidationError::HasWithdrawalsPreShanghai)); } } }; @@ -256,32 +303,46 @@ mod test { use crate::engine; use alloy_primitives::{b64, Address, B256, B64}; use alloy_rpc_types_engine::PayloadAttributes; - use reth_node_builder::EngineValidator; - use reth_optimism_chainspec::BASE_SEPOLIA; + use reth_chainspec::{ChainSpec, ForkCondition, Hardfork}; + use reth_optimism_chainspec::{OpChainSpec, BASE_SEPOLIA}; + use reth_optimism_forks::OpHardfork; use reth_provider::noop::NoopProvider; use reth_trie_common::KeccakKeyHasher; + const JOVIAN_TIMESTAMP: u64 = 1744909000; + fn get_chainspec() -> Arc { + let mut base_sepolia_spec = BASE_SEPOLIA.inner.clone(); + + // TODO: Remove this once we know the Jovian timestamp + base_sepolia_spec + .hardforks + .insert(OpHardfork::Jovian.boxed(), ForkCondition::Timestamp(JOVIAN_TIMESTAMP)); + Arc::new(OpChainSpec { inner: ChainSpec { - chain: BASE_SEPOLIA.inner.chain, - genesis: BASE_SEPOLIA.inner.genesis.clone(), - genesis_header: BASE_SEPOLIA.inner.genesis_header.clone(), - paris_block_and_final_difficulty: BASE_SEPOLIA - .inner + chain: base_sepolia_spec.chain, + genesis: base_sepolia_spec.genesis, + genesis_header: base_sepolia_spec.genesis_header, + paris_block_and_final_difficulty: base_sepolia_spec .paris_block_and_final_difficulty, - hardforks: BASE_SEPOLIA.inner.hardforks.clone(), - base_fee_params: BASE_SEPOLIA.inner.base_fee_params.clone(), + hardforks: base_sepolia_spec.hardforks, + base_fee_params: base_sepolia_spec.base_fee_params, prune_delete_limit: 10000, ..Default::default() }, }) } - const fn get_attributes(eip_1559_params: Option, timestamp: u64) -> OpPayloadAttributes { + const fn get_attributes( + eip_1559_params: Option, + min_base_fee: Option, + timestamp: u64, + ) -> OpPayloadAttributes { OpPayloadAttributes { gas_limit: Some(1000), eip_1559_params, + min_base_fee, transactions: None, no_tx_pool: None, payload_attributes: PayloadAttributes { @@ -298,12 +359,12 @@ mod test { fn test_well_formed_attributes_pre_holocene() { let validator = OpEngineValidator::new::(get_chainspec(), NoopProvider::default()); - let attributes = get_attributes(None, 1732633199); + let attributes = get_attributes(None, None, 1732633199); - let result = as EngineValidator< + let result = as EngineApiValidator< OpEngineTypes, >>::ensure_well_formed_attributes( - &validator, EngineApiMessageVersion::V3, &attributes + &validator, EngineApiMessageVersion::V3, &attributes, ); assert!(result.is_ok()); } @@ -312,12 +373,12 @@ mod test { fn test_well_formed_attributes_holocene_no_eip1559_params() { let validator = OpEngineValidator::new::(get_chainspec(), NoopProvider::default()); - let attributes = get_attributes(None, 1732633200); + let attributes = get_attributes(None, None, 1732633200); - let result = as EngineValidator< + let result = as EngineApiValidator< OpEngineTypes, >>::ensure_well_formed_attributes( - &validator, EngineApiMessageVersion::V3, &attributes + &validator, EngineApiMessageVersion::V3, &attributes, ); assert!(matches!(result, Err(EngineObjectValidationError::InvalidParams(_)))); } @@ -326,12 +387,12 @@ mod test { fn test_well_formed_attributes_holocene_eip1559_params_zero_denominator() { let validator = OpEngineValidator::new::(get_chainspec(), NoopProvider::default()); - let attributes = get_attributes(Some(b64!("0000000000000008")), 1732633200); + let attributes = get_attributes(Some(b64!("0000000000000008")), None, 1732633200); - let result = as EngineValidator< + let result = as EngineApiValidator< OpEngineTypes, >>::ensure_well_formed_attributes( - &validator, EngineApiMessageVersion::V3, &attributes + &validator, EngineApiMessageVersion::V3, &attributes, ); assert!(matches!(result, Err(EngineObjectValidationError::InvalidParams(_)))); } @@ -340,12 +401,12 @@ mod test { fn test_well_formed_attributes_holocene_valid() { let validator = OpEngineValidator::new::(get_chainspec(), NoopProvider::default()); - let attributes = get_attributes(Some(b64!("0000000800000008")), 1732633200); + let attributes = get_attributes(Some(b64!("0000000800000008")), None, 1732633200); - let result = as EngineValidator< + let result = as EngineApiValidator< OpEngineTypes, >>::ensure_well_formed_attributes( - &validator, EngineApiMessageVersion::V3, &attributes + &validator, EngineApiMessageVersion::V3, &attributes, ); assert!(result.is_ok()); } @@ -354,13 +415,72 @@ mod test { fn test_well_formed_attributes_holocene_valid_all_zero() { let validator = OpEngineValidator::new::(get_chainspec(), NoopProvider::default()); - let attributes = get_attributes(Some(b64!("0000000000000000")), 1732633200); + let attributes = get_attributes(Some(b64!("0000000000000000")), None, 1732633200); + + let result = as EngineApiValidator< + OpEngineTypes, + >>::ensure_well_formed_attributes( + &validator, EngineApiMessageVersion::V3, &attributes, + ); + assert!(result.is_ok()); + } + + #[test] + fn test_well_formed_attributes_jovian_valid() { + let validator = + OpEngineValidator::new::(get_chainspec(), NoopProvider::default()); + let attributes = get_attributes(Some(b64!("0000000000000000")), Some(1), JOVIAN_TIMESTAMP); - let result = as EngineValidator< + let result = as EngineApiValidator< OpEngineTypes, >>::ensure_well_formed_attributes( - &validator, EngineApiMessageVersion::V3, &attributes + &validator, EngineApiMessageVersion::V3, &attributes, ); assert!(result.is_ok()); } + + /// After Jovian (and holocene), eip1559 params must be Some + #[test] + fn test_malformed_attributes_jovian_with_eip_1559_params_none() { + let validator = + OpEngineValidator::new::(get_chainspec(), NoopProvider::default()); + let attributes = get_attributes(None, Some(1), JOVIAN_TIMESTAMP); + + let result = as EngineApiValidator< + OpEngineTypes, + >>::ensure_well_formed_attributes( + &validator, EngineApiMessageVersion::V3, &attributes, + ); + assert!(matches!(result, Err(EngineObjectValidationError::InvalidParams(_)))); + } + + /// Before Jovian, min base fee must be None + #[test] + fn test_malformed_attributes_pre_jovian_with_min_base_fee() { + let validator = + OpEngineValidator::new::(get_chainspec(), NoopProvider::default()); + let attributes = get_attributes(Some(b64!("0000000000000000")), Some(1), 1732633200); + + let result = as EngineApiValidator< + OpEngineTypes, + >>::ensure_well_formed_attributes( + &validator, EngineApiMessageVersion::V3, &attributes, + ); + assert!(matches!(result, Err(EngineObjectValidationError::InvalidParams(_)))); + } + + /// After Jovian, min base fee must be Some + #[test] + fn test_malformed_attributes_post_jovian_with_min_base_fee_none() { + let validator = + OpEngineValidator::new::(get_chainspec(), NoopProvider::default()); + let attributes = get_attributes(Some(b64!("0000000000000000")), None, JOVIAN_TIMESTAMP); + + let result = as EngineApiValidator< + OpEngineTypes, + >>::ensure_well_formed_attributes( + &validator, EngineApiMessageVersion::V3, &attributes, + ); + assert!(matches!(result, Err(EngineObjectValidationError::InvalidParams(_)))); + } } diff --git a/crates/optimism/node/src/lib.rs b/crates/optimism/node/src/lib.rs index fc57365b460..9fcc8d4e549 100644 --- a/crates/optimism/node/src/lib.rs +++ b/crates/optimism/node/src/lib.rs @@ -8,7 +8,7 @@ html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256", issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(test), warn(unused_crate_dependencies))] /// CLI argument parsing for the optimism node. @@ -20,7 +20,7 @@ pub mod engine; pub use engine::OpEngineTypes; pub mod node; -pub use node::{OpNetworkPrimitives, OpNode}; +pub use node::*; pub mod rpc; pub use rpc::OpEngineApiBuilder; @@ -35,7 +35,13 @@ pub use reth_optimism_txpool as txpool; pub mod utils; pub use reth_optimism_payload_builder::{ - OpBuiltPayload, OpPayloadAttributes, OpPayloadBuilder, OpPayloadBuilderAttributes, + self as payload, config::OpDAConfig, OpBuiltPayload, OpPayloadAttributes, OpPayloadBuilder, + OpPayloadBuilderAttributes, OpPayloadPrimitives, OpPayloadTypes, }; pub use reth_optimism_evm::*; + +pub use reth_optimism_storage::OpStorage; + +use op_revm as _; +use revm as _; diff --git a/crates/optimism/node/src/node.rs b/crates/optimism/node/src/node.rs index 23506be79c3..cf8485b7b92 100644 --- a/crates/optimism/node/src/node.rs +++ b/crates/optimism/node/src/node.rs @@ -7,72 +7,108 @@ use crate::{ OpEngineApiBuilder, OpEngineTypes, }; use op_alloy_consensus::{interop::SafetyLevel, OpPooledTransaction}; -use reth_chainspec::{EthChainSpec, Hardforks}; -use reth_evm::{ConfigureEvm, EvmFactory, EvmFactoryFor}; -use reth_network::{NetworkConfig, NetworkHandle, NetworkManager, NetworkPrimitives, PeersInfo}; +use op_alloy_rpc_types_engine::OpExecutionData; +use reth_chainspec::{ChainSpecProvider, EthChainSpec, Hardforks}; +use reth_engine_local::LocalPayloadAttributesBuilder; +use reth_evm::ConfigureEvm; +use reth_mantle_forks::MantleHardforks; +use reth_network::{ + types::BasicNetworkPrimitives, NetworkConfig, NetworkHandle, NetworkManager, NetworkPrimitives, + PeersInfo, +}; use reth_node_api::{ - AddOnsContext, FullNodeComponents, KeyHasherTy, NodeAddOns, NodePrimitives, PrimitivesTy, TxTy, + AddOnsContext, BuildNextEnv, EngineTypes, FullNodeComponents, HeaderTy, NodeAddOns, + NodePrimitives, PayloadAttributesBuilder, PayloadTypes, PrimitivesTy, TxTy, }; use reth_node_builder::{ components::{ BasicPayloadServiceBuilder, ComponentsBuilder, ConsensusBuilder, ExecutorBuilder, NetworkBuilder, PayloadBuilderBuilder, PoolBuilder, PoolBuilderConfigOverrides, + TxPoolBuilder, }, node::{FullNodeTypes, NodeTypes}, rpc::{ - EngineValidatorAddOn, EngineValidatorBuilder, EthApiBuilder, RethRpcAddOns, RpcAddOns, - RpcHandle, + BasicEngineValidatorBuilder, EngineApiBuilder, EngineValidatorAddOn, + EngineValidatorBuilder, EthApiBuilder, Identity, PayloadValidatorBuilder, RethRpcAddOns, + RethRpcMiddleware, RethRpcServerHandles, RpcAddOns, RpcContext, RpcHandle, }, BuilderContext, DebugNode, Node, NodeAdapter, NodeComponentsBuilder, }; -use reth_optimism_chainspec::OpChainSpec; +use reth_optimism_chainspec::{OpChainSpec, OpHardfork}; use reth_optimism_consensus::OpBeaconConsensus; -use reth_optimism_evm::{OpEvmConfig, OpNextBlockEnvAttributes}; +use reth_optimism_evm::{OpEvmConfig, OpRethReceiptBuilder}; use reth_optimism_forks::OpHardforks; -use reth_mantle_forks::MantleHardforks; use reth_optimism_payload_builder::{ builder::OpPayloadTransactions, - config::{OpBuilderConfig, OpDAConfig}, + config::{OpBuilderConfig, OpDAConfig, OpGasLimitConfig}, + OpAttributes, OpBuiltPayload, OpPayloadPrimitives, }; -use reth_optimism_primitives::{DepositReceipt, OpPrimitives, OpReceipt, OpTransactionSigned}; +use reth_optimism_primitives::{DepositReceipt, OpPrimitives}; use reth_optimism_rpc::{ eth::{ext::OpEthExtApi, mantle_ext::MantleEthApiExt, OpEthApiBuilder}, + historical::{HistoricalRpc, HistoricalRpcClient}, miner::{MinerApiExtServer, OpMinerExtApi}, witness::{DebugExecutionWitnessApiServer, OpDebugWitnessApi}, - OpEthApi, OpEthApiError, SequencerClient, + SequencerClient, }; +use reth_optimism_storage::OpStorage; use reth_optimism_txpool::{ - conditional::MaybeConditionalTransaction, - interop::MaybeInteropTransaction, supervisor::{SupervisorClient, DEFAULT_SUPERVISOR_URL}, OpPooledTx, }; -use reth_provider::{providers::ProviderFactoryBuilder, CanonStateSubscriptions, EthStorage}; -use reth_rpc_api::DebugApiServer; -use reth_rpc_eth_api::{ext::L2EthApiExtServer, MantleEthApiExtServer}; -use reth_rpc_eth_types::error::FromEvmError; +use reth_provider::{providers::ProviderFactoryBuilder, CanonStateSubscriptions}; +use reth_rpc_api::{eth::RpcTypes, DebugApiServer, L2EthApiExtServer, MantleEthApiExtServer}; use reth_rpc_server_types::RethRpcModule; use reth_tracing::tracing::{debug, info}; use reth_transaction_pool::{ - blobstore::DiskFileBlobStore, CoinbaseTipOrdering, EthPoolTransaction, PoolTransaction, + blobstore::DiskFileBlobStore, EthPoolTransaction, PoolPooledTx, PoolTransaction, TransactionPool, TransactionValidationTaskExecutor, }; -use reth_trie_db::MerklePatriciaTrie; -use revm::context::TxEnv; -use std::sync::Arc; +use reth_trie_common::KeccakKeyHasher; +use serde::de::DeserializeOwned; +use std::{marker::PhantomData, sync::Arc}; +use url::Url; /// Marker trait for Optimism node types with standard engine, chain spec, and primitives. pub trait OpNodeTypes: - NodeTypes + NodeTypes< + Payload = OpEngineTypes, + ChainSpec: OpHardforks + MantleHardforks + Hardforks, + Primitives = OpPrimitives, +> { } /// Blanket impl for all node types that conform to the Optimism spec. impl OpNodeTypes for N where - N: NodeTypes + N: NodeTypes< + Payload = OpEngineTypes, + ChainSpec: OpHardforks + MantleHardforks + Hardforks, + Primitives = OpPrimitives, + > +{ +} + +/// Helper trait for Optimism node types with full configuration including storage and execution +/// data. +pub trait OpFullNodeTypes: + NodeTypes< + ChainSpec: OpHardforks, + Primitives: OpPayloadPrimitives, + Storage = OpStorage, + Payload: EngineTypes, +> +{ +} + +impl OpFullNodeTypes for N where + N: NodeTypes< + ChainSpec: OpHardforks, + Primitives: OpPayloadPrimitives, + Storage = OpStorage, + Payload: EngineTypes, + > { } -/// Storage implementation for Optimism. -pub type OpStorage = EthStorage; /// Type configuration for a regular Optimism node. #[derive(Debug, Default, Clone)] @@ -87,12 +123,30 @@ pub struct OpNode { /// /// By default no throttling is applied. pub da_config: OpDAConfig, + /// Gas limit configuration for the OP builder. + /// Used to control the gas limit of the blocks produced by the OP builder.(configured by the + /// batcher via the `miner_` api) + pub gas_limit_config: OpGasLimitConfig, } +/// A [`ComponentsBuilder`] with its generic arguments set to a stack of Optimism specific builders. +pub type OpNodeComponentBuilder = ComponentsBuilder< + Node, + OpPoolBuilder, + BasicPayloadServiceBuilder, + OpNetworkBuilder, + OpExecutorBuilder, + OpConsensusBuilder, +>; + impl OpNode { /// Creates a new instance of the Optimism node type. pub fn new(args: RollupArgs) -> Self { - Self { args, da_config: OpDAConfig::default() } + Self { + args, + da_config: OpDAConfig::default(), + gas_limit_config: OpGasLimitConfig::default(), + } } /// Configure the data availability configuration for the OP builder. @@ -101,17 +155,14 @@ impl OpNode { self } + /// Configure the gas limit configuration for the OP builder. + pub fn with_gas_limit_config(mut self, gas_limit_config: OpGasLimitConfig) -> Self { + self.gas_limit_config = gas_limit_config; + self + } + /// Returns the components for the given [`RollupArgs`]. - pub fn components( - &self, - ) -> ComponentsBuilder< - Node, - OpPoolBuilder, - BasicPayloadServiceBuilder, - OpNetworkBuilder, - OpExecutorBuilder, - OpConsensusBuilder, - > + pub fn components(&self) -> OpNodeComponentBuilder where Node: FullNodeTypes, { @@ -129,15 +180,27 @@ impl OpNode { ) .executor(OpExecutorBuilder::default()) .payload(BasicPayloadServiceBuilder::new( - OpPayloadBuilder::new(compute_pending_block).with_da_config(self.da_config.clone()), + OpPayloadBuilder::new(compute_pending_block) + .with_da_config(self.da_config.clone()) + .with_gas_limit_config(self.gas_limit_config.clone()), )) - .network(OpNetworkBuilder { - disable_txpool_gossip, - disable_discovery_v4: !discovery_v4, - }) + .network(OpNetworkBuilder::new(disable_txpool_gossip, !discovery_v4)) .consensus(OpConsensusBuilder::default()) } + /// Returns [`OpAddOnsBuilder`] with configured arguments. + pub fn add_ons_builder(&self) -> OpAddOnsBuilder { + OpAddOnsBuilder::default() + .with_sequencer(self.args.sequencer.clone()) + .with_sequencer_headers(self.args.sequencer_headers.clone()) + .with_da_config(self.da_config.clone()) + .with_gas_limit_config(self.gas_limit_config.clone()) + .with_enable_tx_conditional(self.args.enable_tx_conditional) + .with_min_suggested_priority_fee(self.args.min_suggested_priority_fee) + .with_historical_rpc(self.args.historical_rpc.clone()) + .with_flashblocks(self.args.flashblocks_url.clone()) + } + /// Instantiates the [`ProviderFactoryBuilder`] for an opstack node. /// /// # Open a Providerfactory in read-only mode from a datadir @@ -175,14 +238,7 @@ impl OpNode { impl Node for OpNode where - N: FullNodeTypes< - Types: NodeTypes< - Payload = OpEngineTypes, - ChainSpec = OpChainSpec, - Primitives = OpPrimitives, - Storage = OpStorage, - >, - >, + N: FullNodeTypes, { type ComponentsBuilder = ComponentsBuilder< N, @@ -193,19 +249,20 @@ where OpConsensusBuilder, >; - type AddOns = - OpAddOns>::Components>>; + type AddOns = OpAddOns< + NodeAdapter>::Components>, + OpEthApiBuilder, + OpEngineValidatorBuilder, + OpEngineApiBuilder, + BasicEngineValidatorBuilder, + >; fn components_builder(&self) -> Self::ComponentsBuilder { Self::components(self) } fn add_ons(&self) -> Self::AddOns { - Self::AddOns::builder() - .with_sequencer(self.args.sequencer.clone()) - .with_da_config(self.da_config.clone()) - .with_enable_tx_conditional(self.args.enable_tx_conditional) - .build() + self.add_ons_builder().build() } } @@ -216,52 +273,90 @@ where type RpcBlock = alloy_rpc_types_eth::Block; fn rpc_to_primitive_block(rpc_block: Self::RpcBlock) -> reth_node_api::BlockTy { - let alloy_rpc_types_eth::Block { header, transactions, .. } = rpc_block; - reth_optimism_primitives::OpBlock { - header: header.inner, - body: reth_optimism_primitives::OpBlockBody { - transactions: transactions.into_transactions().collect(), - ..Default::default() - }, - } + rpc_block.into_consensus() + } + + fn local_payload_attributes_builder( + chain_spec: &Self::ChainSpec, + ) -> impl PayloadAttributesBuilder<::PayloadAttributes> { + LocalPayloadAttributesBuilder::new(Arc::new(chain_spec.clone())) } } impl NodeTypes for OpNode { type Primitives = OpPrimitives; type ChainSpec = OpChainSpec; - type StateCommitment = MerklePatriciaTrie; type Storage = OpStorage; type Payload = OpEngineTypes; } /// Add-ons w.r.t. optimism. +/// +/// This type provides optimism-specific addons to the node and exposes the RPC server and engine +/// API. #[derive(Debug)] -pub struct OpAddOns -where +pub struct OpAddOns< N: FullNodeComponents, - OpEthApiBuilder: EthApiBuilder, -{ + EthB: EthApiBuilder, + PVB, + EB = OpEngineApiBuilder, + EVB = BasicEngineValidatorBuilder, + RpcMiddleware = Identity, +> { /// Rpc add-ons responsible for launching the RPC servers and instantiating the RPC handlers /// and eth-api. - pub rpc_add_ons: RpcAddOns< - N, - OpEthApiBuilder, - OpEngineValidatorBuilder, - OpEngineApiBuilder, - >, + pub rpc_add_ons: RpcAddOns, /// Data availability configuration for the OP builder. pub da_config: OpDAConfig, + /// Gas limit configuration for the OP builder. + pub gas_limit_config: OpGasLimitConfig, /// Sequencer client, configured to forward submitted transactions to sequencer of given OP /// network. pub sequencer_url: Option, + /// Headers to use for the sequencer client requests. + pub sequencer_headers: Vec, + /// RPC endpoint for historical data. + /// + /// This can be used to forward pre-bedrock rpc requests (op-mainnet). + pub historical_rpc: Option, /// Enable transaction conditionals. enable_tx_conditional: bool, + min_suggested_priority_fee: u64, } -impl Default for OpAddOns +impl OpAddOns where - N: FullNodeComponents>, + N: FullNodeComponents, + EthB: EthApiBuilder, +{ + /// Creates a new instance from components. + #[allow(clippy::too_many_arguments)] + pub const fn new( + rpc_add_ons: RpcAddOns, + da_config: OpDAConfig, + gas_limit_config: OpGasLimitConfig, + sequencer_url: Option, + sequencer_headers: Vec, + historical_rpc: Option, + enable_tx_conditional: bool, + min_suggested_priority_fee: u64, + ) -> Self { + Self { + rpc_add_ons, + da_config, + gas_limit_config, + sequencer_url, + sequencer_headers, + historical_rpc, + enable_tx_conditional, + min_suggested_priority_fee, + } + } +} + +impl Default for OpAddOns +where + N: FullNodeComponents, OpEthApiBuilder: EthApiBuilder, { fn default() -> Self { @@ -269,39 +364,200 @@ where } } -impl OpAddOns +impl + OpAddOns< + N, + OpEthApiBuilder, + OpEngineValidatorBuilder, + OpEngineApiBuilder, + RpcMiddleware, + > where - N: FullNodeComponents>, - OpEthApiBuilder: EthApiBuilder, + N: FullNodeComponents, + OpEthApiBuilder: EthApiBuilder, { /// Build a [`OpAddOns`] using [`OpAddOnsBuilder`]. - pub fn builder() -> OpAddOnsBuilder { + pub fn builder() -> OpAddOnsBuilder { OpAddOnsBuilder::default() } } -impl NodeAddOns for OpAddOns +impl OpAddOns +where + N: FullNodeComponents, + EthB: EthApiBuilder, +{ + /// Maps the [`reth_node_builder::rpc::EngineApiBuilder`] builder type. + pub fn with_engine_api( + self, + engine_api_builder: T, + ) -> OpAddOns { + let Self { + rpc_add_ons, + da_config, + gas_limit_config, + sequencer_url, + sequencer_headers, + historical_rpc, + enable_tx_conditional, + min_suggested_priority_fee, + .. + } = self; + OpAddOns::new( + rpc_add_ons.with_engine_api(engine_api_builder), + da_config, + gas_limit_config, + sequencer_url, + sequencer_headers, + historical_rpc, + enable_tx_conditional, + min_suggested_priority_fee, + ) + } + + /// Maps the [`PayloadValidatorBuilder`] builder type. + pub fn with_payload_validator( + self, + payload_validator_builder: T, + ) -> OpAddOns { + let Self { + rpc_add_ons, + da_config, + gas_limit_config, + sequencer_url, + sequencer_headers, + enable_tx_conditional, + min_suggested_priority_fee, + historical_rpc, + .. + } = self; + OpAddOns::new( + rpc_add_ons.with_payload_validator(payload_validator_builder), + da_config, + gas_limit_config, + sequencer_url, + sequencer_headers, + historical_rpc, + enable_tx_conditional, + min_suggested_priority_fee, + ) + } + + /// Sets the RPC middleware stack for processing RPC requests. + /// + /// This method configures a custom middleware stack that will be applied to all RPC requests + /// across HTTP, `WebSocket`, and IPC transports. The middleware is applied to the RPC service + /// layer, allowing you to intercept, modify, or enhance RPC request processing. + /// + /// See also [`RpcAddOns::with_rpc_middleware`]. + pub fn with_rpc_middleware(self, rpc_middleware: T) -> OpAddOns { + let Self { + rpc_add_ons, + da_config, + gas_limit_config, + sequencer_url, + sequencer_headers, + enable_tx_conditional, + min_suggested_priority_fee, + historical_rpc, + .. + } = self; + OpAddOns::new( + rpc_add_ons.with_rpc_middleware(rpc_middleware), + da_config, + gas_limit_config, + sequencer_url, + sequencer_headers, + historical_rpc, + enable_tx_conditional, + min_suggested_priority_fee, + ) + } + + /// Sets the hook that is run once the rpc server is started. + pub fn on_rpc_started(mut self, hook: F) -> Self + where + F: FnOnce(RpcContext<'_, N, EthB::EthApi>, RethRpcServerHandles) -> eyre::Result<()> + + Send + + 'static, + { + self.rpc_add_ons = self.rpc_add_ons.on_rpc_started(hook); + self + } + + /// Sets the hook that is run to configure the rpc modules. + pub fn extend_rpc_modules(mut self, hook: F) -> Self + where + F: FnOnce(RpcContext<'_, N, EthB::EthApi>) -> eyre::Result<()> + Send + 'static, + { + self.rpc_add_ons = self.rpc_add_ons.extend_rpc_modules(hook); + self + } +} + +impl NodeAddOns + for OpAddOns where N: FullNodeComponents< Types: NodeTypes< - ChainSpec = OpChainSpec, - Primitives = OpPrimitives, - Storage = OpStorage, - Payload = OpEngineTypes, + ChainSpec: OpHardforks, + Primitives: OpPayloadPrimitives, + Payload: PayloadTypes, >, - Evm: ConfigureEvm, + Evm: ConfigureEvm< + NextBlockEnvCtx: BuildNextEnv< + Attrs, + HeaderTy, + ::ChainSpec, + >, + >, + Pool: TransactionPool, >, - OpEthApiError: FromEvmError, - ::Transaction: OpPooledTx, - EvmFactoryFor: EvmFactory>, + EthB: EthApiBuilder, + PVB: Send, + EB: EngineApiBuilder, + EVB: EngineValidatorBuilder, + RpcMiddleware: RethRpcMiddleware, + Attrs: OpAttributes, RpcPayloadAttributes: DeserializeOwned>, { - type Handle = RpcHandle>; + type Handle = RpcHandle; async fn launch_add_ons( self, ctx: reth_node_api::AddOnsContext<'_, N>, ) -> eyre::Result { - let Self { rpc_add_ons, da_config, sequencer_url, enable_tx_conditional } = self; + let Self { + rpc_add_ons, + da_config, + gas_limit_config, + sequencer_url, + sequencer_headers, + enable_tx_conditional, + historical_rpc, + .. + } = self; + + let maybe_pre_bedrock_historical_rpc = historical_rpc + .and_then(|historical_rpc| { + ctx.node + .provider() + .chain_spec() + .op_fork_activation(OpHardfork::Bedrock) + .block_number() + .filter(|activation| *activation > 0) + .map(|bedrock_block| (historical_rpc, bedrock_block)) + }) + .map(|(historical_rpc, bedrock_block)| -> eyre::Result<_> { + info!(target: "reth::cli", %bedrock_block, ?historical_rpc, "Using historical RPC endpoint pre bedrock"); + let provider = ctx.node.provider().clone(); + let client = HistoricalRpcClient::new(&historical_rpc)?; + let layer = HistoricalRpc::new(provider, client, bedrock_block); + Ok(layer) + }) + .transpose()? + ; + + let rpc_add_ons = rpc_add_ons.option_layer_rpc_middleware(maybe_pre_bedrock_historical_rpc); let builder = reth_optimism_payload_builder::OpPayloadBuilder::new( ctx.node.pool().clone(), @@ -309,21 +565,21 @@ where ctx.node.evm_config().clone(), ); // install additional OP specific rpc methods - let debug_ext = OpDebugWitnessApi::new( + let debug_ext = OpDebugWitnessApi::<_, _, _, Attrs>::new( ctx.node.provider().clone(), Box::new(ctx.node.task_executor().clone()), builder, ); - let miner_ext = OpMinerExtApi::new(da_config); + let miner_ext = OpMinerExtApi::new(da_config, gas_limit_config); let sequencer_client = if let Some(url) = sequencer_url { - Some(SequencerClient::new(url).await?) + Some(SequencerClient::new_with_headers(url, sequencer_headers).await?) } else { None }; let tx_conditional_ext: OpEthExtApi = OpEthExtApi::new( - sequencer_client, + sequencer_client.clone(), ctx.node.pool().clone(), ctx.node.provider().clone(), ); @@ -333,12 +589,15 @@ where let provider = ctx.node.provider().clone(); rpc_add_ons - .launch_add_ons_with(ctx, move |modules, auth_modules, registry| { + .launch_add_ons_with(ctx, move |container| { + let reth_node_builder::rpc::RpcModuleContainer { modules, auth_module, registry } = + container; + debug!(target: "reth::cli", "Installing debug payload witness rpc endpoint"); modules.merge_if_module_configured(RethRpcModule::Debug, debug_ext.into_rpc())?; // extend the miner namespace if configured in the regular http server - modules.merge_if_module_configured( + modules.add_or_replace_if_module_configured( RethRpcModule::Miner, miner_ext.clone().into_rpc(), )?; @@ -346,13 +605,13 @@ where // install the miner extension in the authenticated if configured if modules.module_config().contains_any(&RethRpcModule::Miner) { debug!(target: "reth::cli", "Installing miner DA rpc endpoint"); - auth_modules.merge_auth_methods(miner_ext.into_rpc())?; + auth_module.merge_auth_methods(miner_ext.into_rpc())?; } // install the debug namespace in the authenticated if configured if modules.module_config().contains_any(&RethRpcModule::Debug) { debug!(target: "reth::cli", "Installing debug rpc endpoint"); - auth_modules.merge_auth_methods(registry.debug_api().into_rpc())?; + auth_module.merge_auth_methods(registry.debug_api().into_rpc())?; } if enable_tx_conditional { @@ -365,10 +624,11 @@ where // extend the eth namespace with mantle methods info!(target: "reth::cli", "Installing Mantle RPC extension endpoints"); - let eth_api = registry.eth_api(); - let sequencer_client = eth_api.sequencer_client().cloned(); - let mantle_ext: MantleEthApiExt> = - MantleEthApiExt::new(provider.clone(), Arc::new(eth_api.clone()), sequencer_client); + let mantle_ext = MantleEthApiExt::new( + provider.clone(), + Arc::new(registry.eth_api().clone()), + sequencer_client, + ); modules.merge_if_module_configured(RethRpcModule::Eth, mantle_ext.into_rpc())?; Ok(()) @@ -377,114 +637,256 @@ where } } -impl RethRpcAddOns for OpAddOns +impl RethRpcAddOns + for OpAddOns where N: FullNodeComponents< Types: NodeTypes< - ChainSpec = OpChainSpec, - Primitives = OpPrimitives, - Storage = OpStorage, - Payload = OpEngineTypes, + ChainSpec: OpHardforks, + Primitives: OpPayloadPrimitives, + Payload: PayloadTypes, + >, + Evm: ConfigureEvm< + NextBlockEnvCtx: BuildNextEnv< + Attrs, + HeaderTy, + ::ChainSpec, + >, >, - Evm: ConfigureEvm, >, - OpEthApiError: FromEvmError, <::Pool as TransactionPool>::Transaction: OpPooledTx, - EvmFactoryFor: EvmFactory>, + EthB: EthApiBuilder, + PVB: PayloadValidatorBuilder, + EB: EngineApiBuilder, + EVB: EngineValidatorBuilder, + RpcMiddleware: RethRpcMiddleware, + Attrs: OpAttributes, RpcPayloadAttributes: DeserializeOwned>, { - type EthApi = OpEthApi; + type EthApi = EthB::EthApi; fn hooks_mut(&mut self) -> &mut reth_node_builder::rpc::RpcHooks { self.rpc_add_ons.hooks_mut() } } -impl EngineValidatorAddOn for OpAddOns +impl EngineValidatorAddOn + for OpAddOns where - N: FullNodeComponents< - Types: NodeTypes< - ChainSpec = OpChainSpec, - Primitives = OpPrimitives, - Payload = OpEngineTypes, - >, - >, - OpEthApiBuilder: EthApiBuilder, + N: FullNodeComponents, + EthB: EthApiBuilder, + PVB: Send, + EB: EngineApiBuilder, + EVB: EngineValidatorBuilder, + RpcMiddleware: Send, { - type Validator = OpEngineValidator; + type ValidatorBuilder = EVB; - async fn engine_validator(&self, ctx: &AddOnsContext<'_, N>) -> eyre::Result { - OpEngineValidatorBuilder::default().build(ctx).await + fn engine_validator_builder(&self) -> Self::ValidatorBuilder { + EngineValidatorAddOn::engine_validator_builder(&self.rpc_add_ons) } } /// A regular optimism evm and executor builder. -#[derive(Debug, Default, Clone)] +#[derive(Debug, Clone)] #[non_exhaustive] -pub struct OpAddOnsBuilder { +pub struct OpAddOnsBuilder { /// Sequencer client, configured to forward submitted transactions to sequencer of given OP /// network. sequencer_url: Option, + /// Headers to use for the sequencer client requests. + sequencer_headers: Vec, + /// RPC endpoint for historical data. + historical_rpc: Option, /// Data availability configuration for the OP builder. da_config: Option, + /// Gas limit configuration for the OP builder. + gas_limit_config: Option, /// Enable transaction conditionals. enable_tx_conditional: bool, + /// Marker for network types. + _nt: PhantomData, + /// Minimum suggested priority fee (tip) + min_suggested_priority_fee: u64, + /// RPC middleware to use + rpc_middleware: RpcMiddleware, + /// Optional tokio runtime to use for the RPC server. + tokio_runtime: Option, + /// A URL pointing to a secure websocket service that streams out flashblocks. + flashblocks_url: Option, +} + +impl Default for OpAddOnsBuilder { + fn default() -> Self { + Self { + sequencer_url: None, + sequencer_headers: Vec::new(), + historical_rpc: None, + da_config: None, + gas_limit_config: None, + enable_tx_conditional: false, + min_suggested_priority_fee: 1_000_000, + _nt: PhantomData, + rpc_middleware: Identity::new(), + tokio_runtime: None, + flashblocks_url: None, + } + } } -impl OpAddOnsBuilder { +impl OpAddOnsBuilder { /// With a [`SequencerClient`]. pub fn with_sequencer(mut self, sequencer_client: Option) -> Self { self.sequencer_url = sequencer_client; self } + /// With headers to use for the sequencer client requests. + pub fn with_sequencer_headers(mut self, sequencer_headers: Vec) -> Self { + self.sequencer_headers = sequencer_headers; + self + } + /// Configure the data availability configuration for the OP builder. pub fn with_da_config(mut self, da_config: OpDAConfig) -> Self { self.da_config = Some(da_config); self } + /// Configure the gas limit configuration for the OP payload builder. + pub fn with_gas_limit_config(mut self, gas_limit_config: OpGasLimitConfig) -> Self { + self.gas_limit_config = Some(gas_limit_config); + self + } + /// Configure if transaction conditional should be enabled. pub const fn with_enable_tx_conditional(mut self, enable_tx_conditional: bool) -> Self { self.enable_tx_conditional = enable_tx_conditional; self } + + /// Configure the minimum priority fee (tip) + pub const fn with_min_suggested_priority_fee(mut self, min: u64) -> Self { + self.min_suggested_priority_fee = min; + self + } + + /// Configures the endpoint for historical RPC forwarding. + pub fn with_historical_rpc(mut self, historical_rpc: Option) -> Self { + self.historical_rpc = historical_rpc; + self + } + + /// Configures a custom tokio runtime for the RPC server. + /// + /// Caution: This runtime must not be created from within asynchronous context. + pub fn with_tokio_runtime(mut self, tokio_runtime: Option) -> Self { + self.tokio_runtime = tokio_runtime; + self + } + + /// Configure the RPC middleware to use + pub fn with_rpc_middleware(self, rpc_middleware: T) -> OpAddOnsBuilder { + let Self { + sequencer_url, + sequencer_headers, + historical_rpc, + da_config, + gas_limit_config, + enable_tx_conditional, + min_suggested_priority_fee, + tokio_runtime, + _nt, + flashblocks_url, + .. + } = self; + OpAddOnsBuilder { + sequencer_url, + sequencer_headers, + historical_rpc, + da_config, + gas_limit_config, + enable_tx_conditional, + min_suggested_priority_fee, + _nt, + rpc_middleware, + tokio_runtime, + flashblocks_url, + } + } + + /// With a URL pointing to a flashblocks secure websocket subscription. + pub fn with_flashblocks(mut self, flashblocks_url: Option) -> Self { + self.flashblocks_url = flashblocks_url; + self + } } -impl OpAddOnsBuilder { +impl OpAddOnsBuilder { /// Builds an instance of [`OpAddOns`]. - pub fn build(self) -> OpAddOns + pub fn build( + self, + ) -> OpAddOns, PVB, EB, EVB, RpcMiddleware> where - N: FullNodeComponents>, - OpEthApiBuilder: EthApiBuilder, + N: FullNodeComponents, + OpEthApiBuilder: EthApiBuilder, + PVB: PayloadValidatorBuilder + Default, + EB: Default, + EVB: Default, { - let Self { sequencer_url, da_config, enable_tx_conditional } = self; - - OpAddOns { - rpc_add_ons: RpcAddOns::new( - OpEthApiBuilder::default().with_sequencer(sequencer_url.clone()), - Default::default(), - Default::default(), - ), - da_config: da_config.unwrap_or_default(), + let Self { sequencer_url, + sequencer_headers, + da_config, + gas_limit_config, enable_tx_conditional, - } + min_suggested_priority_fee, + historical_rpc, + rpc_middleware, + tokio_runtime, + flashblocks_url, + .. + } = self; + + OpAddOns::new( + RpcAddOns::new( + OpEthApiBuilder::default() + .with_sequencer(sequencer_url.clone()) + .with_sequencer_headers(sequencer_headers.clone()) + .with_min_suggested_priority_fee(min_suggested_priority_fee) + .with_flashblocks(flashblocks_url), + PVB::default(), + EB::default(), + EVB::default(), + rpc_middleware, + ) + .with_tokio_runtime(tokio_runtime), + da_config.unwrap_or_default(), + gas_limit_config.unwrap_or_default(), + sequencer_url, + sequencer_headers, + historical_rpc, + enable_tx_conditional, + min_suggested_priority_fee, + ) } } /// A regular optimism evm and executor builder. -#[derive(Debug, Default, Clone, Copy)] +#[derive(Debug, Copy, Clone, Default)] #[non_exhaustive] pub struct OpExecutorBuilder; impl ExecutorBuilder for OpExecutorBuilder where - Node: FullNodeTypes>, + Node: FullNodeTypes< + Types: NodeTypes, + >, { - type EVM = OpEvmConfig; + type EVM = + OpEvmConfig<::ChainSpec, ::Primitives>; async fn build_evm(self, ctx: &BuilderContext) -> eyre::Result { - let evm_config = OpEvmConfig::optimism(ctx.chain_spec()); + let evm_config = OpEvmConfig::new(ctx.chain_spec(), OpRethReceiptBuilder::default()); Ok(evm_config) } @@ -494,7 +896,7 @@ where /// /// This contains various settings that can be configured and take precedence over the node's /// config. -#[derive(Debug, Clone)] +#[derive(Debug)] pub struct OpPoolBuilder { /// Enforced overrides that are applied to the pool config. pub pool_config_overrides: PoolBuilderConfigOverrides, @@ -520,6 +922,18 @@ impl Default for OpPoolBuilder { } } +impl Clone for OpPoolBuilder { + fn clone(&self) -> Self { + Self { + pool_config_overrides: self.pool_config_overrides.clone(), + enable_tx_conditional: self.enable_tx_conditional, + supervisor_http: self.supervisor_http.clone(), + supervisor_safety_level: self.supervisor_safety_level, + _pd: core::marker::PhantomData, + } + } +} + impl OpPoolBuilder { /// Sets the `enable_tx_conditional` flag on the pool builder. pub const fn with_enable_tx_conditional(mut self, enable_tx_conditional: bool) -> Self { @@ -551,19 +965,16 @@ impl OpPoolBuilder { impl PoolBuilder for OpPoolBuilder where Node: FullNodeTypes>, - T: EthPoolTransaction> - + MaybeConditionalTransaction - + MaybeInteropTransaction, + T: EthPoolTransaction> + OpPooledTx, { type Pool = OpTransactionPool; async fn build_pool(self, ctx: &BuilderContext) -> eyre::Result { let Self { pool_config_overrides, .. } = self; - let data_dir = ctx.config().datadir(); - let blob_store = DiskFileBlobStore::open(data_dir.blobstore(), Default::default())?; + // supervisor used for interop - if ctx.chain_spec().is_interop_active_at_timestamp(ctx.head().timestamp) && - self.supervisor_http == DEFAULT_SUPERVISOR_URL + if ctx.chain_spec().is_interop_active_at_timestamp(ctx.head().timestamp) + && self.supervisor_http == DEFAULT_SUPERVISOR_URL { info!(target: "reth::cli", url=%DEFAULT_SUPERVISOR_URL, @@ -575,11 +986,15 @@ where .build() .await; + let blob_store = reth_node_builder::components::create_blob_store(ctx)?; let validator = TransactionValidationTaskExecutor::eth_builder(ctx.provider().clone()) .no_eip4844() .with_head_timestamp(ctx.head().timestamp) + .with_max_tx_input_bytes(ctx.config().txpool.max_tx_input_bytes) .kzg_settings(ctx.kzg_settings()?) .set_tx_fee_cap(ctx.config().rpc.rpc_tx_fee_cap) + .with_max_tx_gas_limit(ctx.config().txpool.max_tx_gas_limit) + .with_minimum_priority_fee(ctx.config().txpool.minimum_priority_fee) .with_additional_tasks( pool_config_overrides .additional_validation_tasks @@ -594,87 +1009,41 @@ where .with_supervisor(supervisor_client.clone()) }); - let transaction_pool = reth_transaction_pool::Pool::new( - validator, - CoinbaseTipOrdering::default(), - blob_store, - pool_config_overrides.apply(ctx.pool_config()), - ); - info!(target: "reth::cli", "Transaction pool initialized"); + let final_pool_config = pool_config_overrides.apply(ctx.pool_config()); - // spawn txpool maintenance tasks - { - let pool = transaction_pool.clone(); - let chain_events = ctx.provider().canonical_state_stream(); - let client = ctx.provider().clone(); - if !ctx.config().txpool.disable_transactions_backup { - // Use configured backup path or default to data dir - let transactions_path = ctx - .config() - .txpool - .transactions_backup_path - .clone() - .unwrap_or_else(|| data_dir.txpool_transactions()); - - let transactions_backup_config = - reth_transaction_pool::maintain::LocalTransactionBackupConfig::with_local_txs_backup(transactions_path); - - ctx.task_executor().spawn_critical_with_graceful_shutdown_signal( - "local transactions backup task", - |shutdown| { - reth_transaction_pool::maintain::backup_local_transactions_task( - shutdown, - pool.clone(), - transactions_backup_config, - ) - }, - ); - } + let transaction_pool = TxPoolBuilder::new(ctx) + .with_validator(validator) + .build_and_spawn_maintenance_task(blob_store, final_pool_config)?; - // spawn the main maintenance task - ctx.task_executor().spawn_critical( - "txpool maintenance task", - reth_transaction_pool::maintain::maintain_transaction_pool_future( - client, - pool.clone(), - chain_events, - ctx.task_executor().clone(), - reth_transaction_pool::maintain::MaintainPoolConfig { - max_tx_lifetime: pool.config().max_queued_lifetime, - no_local_exemptions: transaction_pool - .config() - .local_transactions_config - .no_exemptions, - ..Default::default() - }, - ), - ); - debug!(target: "reth::cli", "Spawned txpool maintenance task"); + info!(target: "reth::cli", "Transaction pool initialized"); + debug!(target: "reth::cli", "Spawned txpool maintenance task"); + // The Op txpool maintenance task is only spawned when interop is active + if ctx.chain_spec().is_interop_active_at_timestamp(ctx.head().timestamp) { // spawn the Op txpool maintenance task let chain_events = ctx.provider().canonical_state_stream(); ctx.task_executor().spawn_critical( "Op txpool interop maintenance task", reth_optimism_txpool::maintain::maintain_transaction_pool_interop_future( - pool.clone(), + transaction_pool.clone(), chain_events, supervisor_client, ), ); debug!(target: "reth::cli", "Spawned Op interop txpool maintenance task"); + } - if self.enable_tx_conditional { - // spawn the Op txpool maintenance task - let chain_events = ctx.provider().canonical_state_stream(); - ctx.task_executor().spawn_critical( - "Op txpool conditional maintenance task", - reth_optimism_txpool::maintain::maintain_transaction_pool_conditional_future( - pool, - chain_events, - ), - ); - debug!(target: "reth::cli", "Spawned Op conditional txpool maintenance task"); - } + if self.enable_tx_conditional { + // spawn the Op txpool maintenance task + let chain_events = ctx.provider().canonical_state_stream(); + ctx.task_executor().spawn_critical( + "Op txpool conditional maintenance task", + reth_optimism_txpool::maintain::maintain_transaction_pool_conditional_future( + transaction_pool.clone(), + chain_events, + ), + ); + debug!(target: "reth::cli", "Spawned Op conditional txpool maintenance task"); } Ok(transaction_pool) @@ -699,13 +1068,21 @@ pub struct OpPayloadBuilder { /// This data availability configuration specifies constraints for the payload builder /// when assembling payloads pub da_config: OpDAConfig, + /// Gas limit configuration for the OP builder. + /// This is used to configure gas limit related constraints for the payload builder. + pub gas_limit_config: OpGasLimitConfig, } impl OpPayloadBuilder { /// Create a new instance with the given `compute_pending_block` flag and data availability /// config. pub fn new(compute_pending_block: bool) -> Self { - Self { compute_pending_block, best_transactions: (), da_config: OpDAConfig::default() } + Self { + compute_pending_block, + best_transactions: (), + da_config: OpDAConfig::default(), + gas_limit_config: OpGasLimitConfig::default(), + } } /// Configure the data availability configuration for the OP payload builder. @@ -713,38 +1090,49 @@ impl OpPayloadBuilder { self.da_config = da_config; self } + + /// Configure the gas limit configuration for the OP payload builder. + pub fn with_gas_limit_config(mut self, gas_limit_config: OpGasLimitConfig) -> Self { + self.gas_limit_config = gas_limit_config; + self + } } impl OpPayloadBuilder { /// Configures the type responsible for yielding the transactions that should be included in the /// payload. pub fn with_transactions(self, best_transactions: T) -> OpPayloadBuilder { - let Self { compute_pending_block, da_config, .. } = self; - OpPayloadBuilder { compute_pending_block, best_transactions, da_config } + let Self { compute_pending_block, da_config, gas_limit_config, .. } = self; + OpPayloadBuilder { compute_pending_block, best_transactions, da_config, gas_limit_config } } } -impl PayloadBuilderBuilder for OpPayloadBuilder +impl PayloadBuilderBuilder for OpPayloadBuilder where Node: FullNodeTypes< + Provider: ChainSpecProvider, Types: NodeTypes< - Payload = OpEngineTypes, - ChainSpec = OpChainSpec, - Primitives = OpPrimitives, + Primitives: OpPayloadPrimitives, + Payload: PayloadTypes< + BuiltPayload = OpBuiltPayload>, + PayloadBuilderAttributes = Attrs, + >, >, >, Evm: ConfigureEvm< Primitives = PrimitivesTy, - NextBlockEnvCtx = OpNextBlockEnvAttributes, + NextBlockEnvCtx: BuildNextEnv< + Attrs, + HeaderTy, + ::ChainSpec, + >, > + 'static, - Pool: TransactionPool>> - + Unpin - + 'static, + Pool: TransactionPool>> + Unpin + 'static, Txs: OpPayloadTransactions, - ::Transaction: OpPooledTx, + Attrs: OpAttributes>, { type PayloadBuilder = - reth_optimism_payload_builder::OpPayloadBuilder; + reth_optimism_payload_builder::OpPayloadBuilder; async fn build_payload_builder( self, @@ -756,7 +1144,10 @@ where pool, ctx.provider().clone(), evm_config, - OpBuilderConfig { da_config: self.da_config.clone() }, + OpBuilderConfig { + da_config: self.da_config.clone(), + gas_limit_config: self.gas_limit_config.clone(), + }, ) .with_transactions(self.best_transactions.clone()) .set_compute_pending_block(self.compute_pending_block); @@ -765,7 +1156,7 @@ where } /// A basic optimism network builder. -#[derive(Debug, Default, Clone)] +#[derive(Debug, Default)] pub struct OpNetworkBuilder { /// Disable transaction pool gossip pub disable_txpool_gossip: bool, @@ -773,18 +1164,33 @@ pub struct OpNetworkBuilder { pub disable_discovery_v4: bool, } +impl Clone for OpNetworkBuilder { + fn clone(&self) -> Self { + Self::new(self.disable_txpool_gossip, self.disable_discovery_v4) + } +} + +impl OpNetworkBuilder { + /// Creates a new `OpNetworkBuilder`. + pub const fn new(disable_txpool_gossip: bool, disable_discovery_v4: bool) -> Self { + Self { disable_txpool_gossip, disable_discovery_v4 } + } +} + impl OpNetworkBuilder { /// Returns the [`NetworkConfig`] that contains the settings to launch the p2p network. /// /// This applies the configured [`OpNetworkBuilder`] settings. - pub fn network_config( + pub fn network_config( &self, ctx: &BuilderContext, - ) -> eyre::Result::Provider, OpNetworkPrimitives>> + ) -> eyre::Result> where Node: FullNodeTypes>, + NetworkP: NetworkPrimitives, { - let Self { disable_txpool_gossip, disable_discovery_v4 } = self.clone(); + let disable_txpool_gossip = self.disable_txpool_gossip; + let disable_discovery_v4 = self.disable_discovery_v4; let args = &ctx.config().network; let network_builder = ctx .network_config_builder()? @@ -823,16 +1229,13 @@ impl OpNetworkBuilder { impl NetworkBuilder for OpNetworkBuilder where - Node: FullNodeTypes>, - Pool: TransactionPool< - Transaction: PoolTransaction< - Consensus = TxTy, - Pooled = OpPooledTransaction, - >, - > + Unpin + Node: FullNodeTypes>, + Pool: TransactionPool>> + + Unpin + 'static, { - type Network = NetworkHandle; + type Network = + NetworkHandle, PoolPooledTx>>; async fn build_network( self, @@ -874,15 +1277,23 @@ where #[non_exhaustive] pub struct OpEngineValidatorBuilder; -impl EngineValidatorBuilder for OpEngineValidatorBuilder +impl PayloadValidatorBuilder for OpEngineValidatorBuilder where - Types: NodeTypes, - Node: FullNodeComponents, + Node: FullNodeComponents< + Types: NodeTypes< + ChainSpec: OpHardforks, + Payload: PayloadTypes, + >, + >, { - type Validator = OpEngineValidator; + type Validator = OpEngineValidator< + Node::Provider, + <::Primitives as NodePrimitives>::SignedTx, + ::ChainSpec, + >; async fn build(self, ctx: &AddOnsContext<'_, Node>) -> eyre::Result { - Ok(OpEngineValidator::new::>( + Ok(OpEngineValidator::new::( ctx.config.chain.clone(), ctx.node.provider().clone(), )) @@ -890,15 +1301,4 @@ where } /// Network primitive types used by Optimism networks. -#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Hash)] -#[non_exhaustive] -pub struct OpNetworkPrimitives; - -impl NetworkPrimitives for OpNetworkPrimitives { - type BlockHeader = alloy_consensus::Header; - type BlockBody = alloy_consensus::BlockBody; - type Block = alloy_consensus::Block; - type BroadcastedTransaction = OpTransactionSigned; - type PooledTransaction = OpPooledTransaction; - type Receipt = OpReceipt; -} +pub type OpNetworkPrimitives = BasicNetworkPrimitives; diff --git a/crates/optimism/node/src/rpc.rs b/crates/optimism/node/src/rpc.rs index 1126235af0b..db811a7f921 100644 --- a/crates/optimism/node/src/rpc.rs +++ b/crates/optimism/node/src/rpc.rs @@ -1,20 +1,100 @@ //! RPC component builder +//! +//! # Example +//! +//! Builds offline `TraceApi` with only EVM and database. This can be useful +//! for example when downloading a state snapshot (pre-synced node) from some mirror. +//! +//! ```rust +//! use alloy_rpc_types_eth::BlockId; +//! use op_alloy_network::Optimism; +//! use reth_db::test_utils::create_test_rw_db_with_path; +//! use reth_node_builder::{ +//! components::ComponentsBuilder, +//! hooks::OnComponentInitializedHook, +//! rpc::{EthApiBuilder, EthApiCtx}, +//! LaunchContext, NodeConfig, RethFullAdapter, +//! }; +//! use reth_optimism_chainspec::OP_SEPOLIA; +//! use reth_optimism_evm::OpEvmConfig; +//! use reth_optimism_node::{OpExecutorBuilder, OpNetworkPrimitives, OpNode}; +//! use reth_optimism_rpc::OpEthApiBuilder; +//! use reth_optimism_txpool::OpPooledTransaction; +//! use reth_provider::providers::BlockchainProvider; +//! use reth_rpc::TraceApi; +//! use reth_rpc_eth_types::{EthConfig, EthStateCache}; +//! use reth_tasks::{pool::BlockingTaskGuard, TaskManager}; +//! use std::sync::Arc; +//! +//! #[tokio::main] +//! async fn main() { +//! // build core node with all components disabled except EVM and state +//! let sepolia = NodeConfig::new(OP_SEPOLIA.clone()); +//! let db = create_test_rw_db_with_path(sepolia.datadir()); +//! let tasks = TaskManager::current(); +//! let launch_ctx = LaunchContext::new(tasks.executor(), sepolia.datadir()); +//! let node = launch_ctx +//! .with_loaded_toml_config(sepolia) +//! .unwrap() +//! .attach(Arc::new(db)) +//! .with_provider_factory::<_, OpEvmConfig>() +//! .await +//! .unwrap() +//! .with_genesis() +//! .unwrap() +//! .with_metrics_task() // todo: shouldn't be req to set up blockchain db +//! .with_blockchain_db::, _>(move |provider_factory| { +//! Ok(BlockchainProvider::new(provider_factory).unwrap()) +//! }) +//! .unwrap() +//! .with_components( +//! ComponentsBuilder::default() +//! .node_types::>() +//! .noop_pool::() +//! .noop_network::() +//! .noop_consensus() +//! .executor(OpExecutorBuilder::default()) +//! .noop_payload(), +//! Box::new(()) as Box>, +//! ) +//! .await +//! .unwrap(); +//! +//! // build `eth` namespace API +//! let config = EthConfig::default(); +//! let cache = EthStateCache::spawn_with( +//! node.provider_factory().clone(), +//! config.cache, +//! node.task_executor().clone(), +//! ); +//! let ctx = EthApiCtx { components: node.node_adapter(), config, cache }; +//! let eth_api = OpEthApiBuilder::::default().build_eth_api(ctx).await.unwrap(); +//! +//! // build `trace` namespace API +//! let trace_api = TraceApi::new(eth_api, BlockingTaskGuard::new(10), EthConfig::default()); +//! +//! // fetch traces for latest block +//! let traces = trace_api.trace_block(BlockId::latest()).await.unwrap(); +//! } +//! ``` -pub use reth_optimism_rpc::OpEngineApi; +pub use reth_optimism_rpc::{OpEngineApi, OpEthApi, OpEthApiBuilder}; use crate::OP_NAME_CLIENT; use alloy_rpc_types_engine::ClientVersionV1; use op_alloy_rpc_types_engine::OpExecutionData; use reth_chainspec::EthereumHardforks; -use reth_node_api::{AddOnsContext, EngineTypes, FullNodeComponents, NodeTypes}; -use reth_node_builder::rpc::{EngineApiBuilder, EngineValidatorBuilder}; -use reth_node_core::version::{CARGO_PKG_VERSION, CLIENT_CODE, VERGEN_GIT_SHA}; +use reth_node_api::{ + AddOnsContext, EngineApiValidator, EngineTypes, FullNodeComponents, NodeTypes, +}; +use reth_node_builder::rpc::{EngineApiBuilder, PayloadValidatorBuilder}; +use reth_node_core::version::{version_metadata, CLIENT_CODE}; use reth_optimism_rpc::engine::OP_ENGINE_CAPABILITIES; use reth_payload_builder::PayloadStore; use reth_rpc_engine_api::{EngineApi, EngineCapabilities}; /// Builder for basic [`OpEngineApi`] implementation. -#[derive(Debug, Default)] +#[derive(Debug, Default, Clone)] pub struct OpEngineApiBuilder { engine_validator_builder: EV, } @@ -27,7 +107,8 @@ where Payload: EngineTypes, >, >, - EV: EngineValidatorBuilder, + EV: PayloadValidatorBuilder, + EV::Validator: EngineApiValidator<::Payload>, { type EngineApi = OpEngineApi< N::Provider, @@ -44,8 +125,8 @@ where let client = ClientVersionV1 { code: CLIENT_CODE, name: OP_NAME_CLIENT.to_string(), - version: CARGO_PKG_VERSION.to_string(), - commit: VERGEN_GIT_SHA.to_string(), + version: version_metadata().cargo_pkg_version.to_string(), + commit: version_metadata().vergen_git_sha.to_string(), }; let inner = EngineApi::new( ctx.node.provider().clone(), diff --git a/crates/optimism/node/src/utils.rs b/crates/optimism/node/src/utils.rs index a8ab57c7222..42104c9df73 100644 --- a/crates/optimism/node/src/utils.rs +++ b/crates/optimism/node/src/utils.rs @@ -25,6 +25,7 @@ pub async fn setup(num_nodes: usize) -> eyre::Result<(Vec, TaskManager, num_nodes, Arc::new(OpChainSpecBuilder::base_mainnet().genesis(genesis).ecotone_activated().build()), false, + Default::default(), optimism_payload_attributes, ) .await @@ -68,5 +69,6 @@ pub fn optimism_payload_attributes(timestamp: u64) -> OpPayloadBuilderAttribu no_tx_pool: false, gas_limit: Some(30_000_000), eip_1559_params: None, + min_base_fee: None, } } diff --git a/crates/optimism/node/tests/e2e/main.rs b/crates/optimism/node/tests/e2e-testsuite/main.rs similarity index 100% rename from crates/optimism/node/tests/e2e/main.rs rename to crates/optimism/node/tests/e2e-testsuite/main.rs diff --git a/crates/optimism/node/tests/e2e/p2p.rs b/crates/optimism/node/tests/e2e-testsuite/p2p.rs similarity index 100% rename from crates/optimism/node/tests/e2e/p2p.rs rename to crates/optimism/node/tests/e2e-testsuite/p2p.rs diff --git a/crates/optimism/node/tests/e2e/testsuite.rs b/crates/optimism/node/tests/e2e-testsuite/testsuite.rs similarity index 51% rename from crates/optimism/node/tests/e2e/testsuite.rs rename to crates/optimism/node/tests/e2e-testsuite/testsuite.rs index b67c6e97705..b031b3a8266 100644 --- a/crates/optimism/node/tests/e2e/testsuite.rs +++ b/crates/optimism/node/tests/e2e-testsuite/testsuite.rs @@ -1,4 +1,4 @@ -use alloy_primitives::{Address, B256}; +use alloy_primitives::{Address, B256, B64}; use eyre::Result; use op_alloy_rpc_types_engine::OpPayloadAttributes; use reth_e2e_test_utils::testsuite::{ @@ -44,6 +44,52 @@ async fn test_testsuite_op_assert_mine_block() -> Result<()> { transactions: None, no_tx_pool: None, eip_1559_params: None, + min_base_fee: None, + gas_limit: Some(30_000_000), + }, + )); + + test.run::().await?; + + Ok(()) +} + +#[tokio::test] +async fn test_testsuite_op_assert_mine_block_isthmus_activated() -> Result<()> { + reth_tracing::init_test_tracing(); + + let setup = Setup::default() + .with_chain_spec(Arc::new( + OpChainSpecBuilder::default() + .chain(OP_MAINNET.chain) + .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) + .isthmus_activated() + .build() + .into(), + )) + .with_network(NetworkSetup::single_node()); + + let test = + TestBuilder::new().with_setup(setup).with_action(AssertMineBlock::::new( + 0, + vec![], + Some(B256::ZERO), + // TODO: refactor once we have actions to generate payload attributes. + OpPayloadAttributes { + payload_attributes: alloy_rpc_types_engine::PayloadAttributes { + timestamp: std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs(), + prev_randao: B256::random(), + suggested_fee_recipient: Address::random(), + withdrawals: Some(vec![]), + parent_beacon_block_root: Some(B256::ZERO), + }, + transactions: None, + no_tx_pool: None, + eip_1559_params: Some(B64::ZERO), + min_base_fee: None, gas_limit: Some(30_000_000), }, )); diff --git a/crates/optimism/node/tests/it/builder.rs b/crates/optimism/node/tests/it/builder.rs index eba2aed422d..b495fdb47ce 100644 --- a/crates/optimism/node/tests/it/builder.rs +++ b/crates/optimism/node/tests/it/builder.rs @@ -1,11 +1,32 @@ //! Node builder setup tests. +use alloy_primitives::{address, Bytes}; +use core::marker::PhantomData; +use op_revm::{ + precompiles::OpPrecompiles, OpContext, OpHaltReason, OpSpecId, OpTransaction, + OpTransactionError, +}; use reth_db::test_utils::create_test_rw_db; +use reth_evm::{precompiles::PrecompilesMap, Database, Evm, EvmEnv, EvmFactory}; use reth_node_api::{FullNodeComponents, NodeTypesWithDBAdapter}; -use reth_node_builder::{Node, NodeBuilder, NodeConfig}; -use reth_optimism_chainspec::BASE_MAINNET; -use reth_optimism_node::{args::RollupArgs, OpNode}; +use reth_node_builder::{ + components::ExecutorBuilder, BuilderContext, FullNodeTypes, Node, NodeBuilder, NodeConfig, + NodeTypes, +}; +use reth_optimism_chainspec::{OpChainSpec, BASE_MAINNET, OP_SEPOLIA}; +use reth_optimism_evm::{OpBlockExecutorFactory, OpEvm, OpEvmFactory, OpRethReceiptBuilder}; +use reth_optimism_node::{args::RollupArgs, OpEvmConfig, OpExecutorBuilder, OpNode}; +use reth_optimism_primitives::OpPrimitives; use reth_provider::providers::BlockchainProvider; +use revm::{ + context::{BlockEnv, Cfg, ContextTr, TxEnv}, + context_interface::result::EVMError, + inspector::NoOpInspector, + interpreter::interpreter::EthInterpreter, + precompile::{Precompile, PrecompileId, PrecompileOutput, PrecompileResult, Precompiles}, + Inspector, +}; +use std::sync::OnceLock; #[test] fn test_basic_setup() { @@ -36,3 +57,113 @@ fn test_basic_setup() { }) .check_launch(); } + +#[test] +fn test_setup_custom_precompiles() { + /// Unichain custom precompiles. + struct UniPrecompiles; + + impl UniPrecompiles { + /// Returns map of precompiles for Unichain. + fn precompiles(spec_id: OpSpecId) -> PrecompilesMap { + static INSTANCE: OnceLock = OnceLock::new(); + + PrecompilesMap::from_static(INSTANCE.get_or_init(|| { + let mut precompiles = OpPrecompiles::new_with_spec(spec_id).precompiles().clone(); + // Custom precompile. + let precompile = Precompile::new( + PrecompileId::custom("custom"), + address!("0x0000000000000000000000000000000000756e69"), + |_, _| PrecompileResult::Ok(PrecompileOutput::new(0, Bytes::new())), + ); + precompiles.extend([precompile]); + precompiles + })) + } + } + + /// Builds Unichain EVM configuration. + #[derive(Clone, Debug)] + struct UniEvmFactory; + + impl EvmFactory for UniEvmFactory { + type Evm>> = OpEvm; + type Context = OpContext; + type Tx = OpTransaction; + type Error = + EVMError; + type HaltReason = OpHaltReason; + type Spec = OpSpecId; + type BlockEnv = BlockEnv; + type Precompiles = PrecompilesMap; + + fn create_evm( + &self, + db: DB, + input: EvmEnv, + ) -> Self::Evm { + let mut op_evm = OpEvmFactory::default().create_evm(db, input); + *op_evm.components_mut().2 = UniPrecompiles::precompiles(op_evm.ctx().cfg().spec()); + + op_evm + } + + fn create_evm_with_inspector< + DB: Database, + I: Inspector, EthInterpreter>, + >( + &self, + db: DB, + input: EvmEnv, + inspector: I, + ) -> Self::Evm { + let mut op_evm = + OpEvmFactory::default().create_evm_with_inspector(db, input, inspector); + *op_evm.components_mut().2 = UniPrecompiles::precompiles(op_evm.ctx().cfg().spec()); + + op_evm + } + } + + /// Unichain executor builder. + struct UniExecutorBuilder; + + impl ExecutorBuilder for UniExecutorBuilder + where + Node: FullNodeTypes>, + { + type EVM = OpEvmConfig< + OpChainSpec, + ::Primitives, + OpRethReceiptBuilder, + UniEvmFactory, + >; + + async fn build_evm(self, ctx: &BuilderContext) -> eyre::Result { + let OpEvmConfig { executor_factory, block_assembler, _pd: _ } = + OpExecutorBuilder::default().build_evm(ctx).await?; + let uni_executor_factory = OpBlockExecutorFactory::new( + *executor_factory.receipt_builder(), + ctx.chain_spec(), + UniEvmFactory, + ); + let uni_evm_config = OpEvmConfig { + executor_factory: uni_executor_factory, + block_assembler, + _pd: PhantomData, + }; + Ok(uni_evm_config) + } + } + + NodeBuilder::new(NodeConfig::new(OP_SEPOLIA.clone())) + .with_database(create_test_rw_db()) + .with_types::() + .with_components( + OpNode::default() + .components() + // Custom EVM configuration + .executor(UniExecutorBuilder), + ) + .check_launch(); +} diff --git a/crates/optimism/node/tests/it/priority.rs b/crates/optimism/node/tests/it/priority.rs index 38c63777923..f831c65ca93 100644 --- a/crates/optimism/node/tests/it/priority.rs +++ b/crates/optimism/node/tests/it/priority.rs @@ -12,15 +12,15 @@ use reth_e2e_test_utils::{ use reth_node_api::FullNodeTypes; use reth_node_builder::{ components::{BasicPayloadServiceBuilder, ComponentsBuilder}, - EngineNodeLauncher, NodeBuilder, NodeConfig, + EngineNodeLauncher, Node, NodeBuilder, NodeConfig, }; use reth_node_core::args::DatadirArgs; use reth_optimism_chainspec::OpChainSpecBuilder; use reth_optimism_node::{ args::RollupArgs, node::{ - OpAddOns, OpConsensusBuilder, OpExecutorBuilder, OpNetworkBuilder, OpNodeTypes, - OpPayloadBuilder, OpPoolBuilder, + OpConsensusBuilder, OpExecutorBuilder, OpNetworkBuilder, OpNodeComponentBuilder, + OpNodeTypes, OpPayloadBuilder, OpPoolBuilder, }, txpool::OpPooledTransaction, utils::optimism_payload_attributes, @@ -88,14 +88,7 @@ impl OpPayloadTransactions for CustomTxPriority { /// Builds the node with custom transaction priority service within default payload builder. fn build_components( chain_id: ChainId, -) -> ComponentsBuilder< - Node, - OpPoolBuilder, - BasicPayloadServiceBuilder>, - OpNetworkBuilder, - OpExecutorBuilder, - OpConsensusBuilder, -> +) -> OpNodeComponentBuilder> where Node: FullNodeTypes, { @@ -109,7 +102,7 @@ where OpPayloadBuilder::new(compute_pending_block) .with_transactions(CustomTxPriority { chain_id }), )) - .network(OpNetworkBuilder { disable_txpool_gossip, disable_discovery_v4: !discovery_v4 }) + .network(OpNetworkBuilder::new(disable_txpool_gossip, !discovery_v4)) .consensus(OpConsensusBuilder::default()) } @@ -143,7 +136,7 @@ async fn test_custom_block_priority_config() { .with_database(db) .with_types_and_provider::>() .with_components(build_components(config.chain.chain_id())) - .with_add_ons(OpAddOns::default()) + .with_add_ons(OpNode::new(Default::default()).add_ons()) .launch_with_fn(|builder| { let launcher = EngineNodeLauncher::new( tasks.executor(), @@ -179,11 +172,11 @@ async fn test_custom_block_priority_config() { .unwrap(); assert_eq!(block_payloads.len(), 1); let block_payload = block_payloads.first().unwrap(); - let block_payload = block_payload.block().clone(); - assert_eq!(block_payload.body().transactions.len(), 2); // L1 block info tx + end-of-block custom tx + let block = block_payload.block(); + assert_eq!(block.body().transactions.len(), 2); // L1 block info tx + end-of-block custom tx // Check that last transaction in the block looks like a transfer to a random address. - let end_of_block_tx = block_payload.body().transactions.last().unwrap(); + let end_of_block_tx = block.body().transactions.last().unwrap(); let Some(tx) = end_of_block_tx.as_eip1559() else { panic!("expected EIP-1559 transaction"); }; diff --git a/crates/optimism/payload/Cargo.toml b/crates/optimism/payload/Cargo.toml index 8d1875fe753..e75075a12cf 100644 --- a/crates/optimism/payload/Cargo.toml +++ b/crates/optimism/payload/Cargo.toml @@ -44,6 +44,7 @@ op-alloy-consensus.workspace = true alloy-rpc-types-engine.workspace = true alloy-rpc-types-debug.workspace = true alloy-consensus.workspace = true +alloy-evm.workspace = true # misc derive_more.workspace = true diff --git a/crates/optimism/payload/src/builder.rs b/crates/optimism/payload/src/builder.rs index 5bb266ccbc5..3d047c5f617 100644 --- a/crates/optimism/payload/src/builder.rs +++ b/crates/optimism/payload/src/builder.rs @@ -1,38 +1,38 @@ //! Optimism payload builder implementation. - use crate::{ - config::{OpBuilderConfig, OpDAConfig}, - error::OpPayloadBuilderError, - payload::{OpBuiltPayload, OpPayloadBuilderAttributes}, - OpPayloadPrimitives, + config::OpBuilderConfig, error::OpPayloadBuilderError, payload::OpBuiltPayload, OpAttributes, + OpPayloadBuilderAttributes, OpPayloadPrimitives, }; -use alloy_consensus::{Transaction, Typed2718}; -use alloy_primitives::{Bytes, B256, U256}; -use alloy_rlp::Encodable; +use alloy_consensus::{BlockHeader, Transaction, Typed2718}; +use alloy_evm::Evm as AlloyEvm; +use alloy_primitives::{B256, U256}; use alloy_rpc_types_debug::ExecutionWitness; use alloy_rpc_types_engine::PayloadId; -use op_alloy_rpc_types_engine::OpPayloadAttributes; use reth_basic_payload_builder::*; -use reth_chain_state::{ExecutedBlock, ExecutedBlockWithTrieUpdates}; +use reth_chain_state::ExecutedBlock; use reth_chainspec::{ChainSpecProvider, EthChainSpec}; use reth_evm::{ + block::BlockExecutorFor, execute::{ BlockBuilder, BlockBuilderOutcome, BlockExecutionError, BlockExecutor, BlockValidationError, }, - ConfigureEvm, Database, Evm, + op_revm::{constants::L1_BLOCK_CONTRACT, L1BlockInfo}, + ConfigureEvm, Database, }; use reth_execution_types::ExecutionOutcome; -use reth_optimism_evm::OpNextBlockEnvAttributes; use reth_optimism_forks::OpHardforks; use reth_optimism_primitives::{transaction::OpTransaction, ADDRESS_L2_TO_L1_MESSAGE_PASSER}; use reth_optimism_txpool::{ + estimated_da_size::DataAvailabilitySized, interop::{is_valid_interop, MaybeInteropTransaction}, OpPooledTx, }; use reth_payload_builder_primitives::PayloadBuilderError; -use reth_payload_primitives::PayloadBuilderAttributes; +use reth_payload_primitives::{BuildNextEnv, PayloadBuilderAttributes}; use reth_payload_util::{BestPayloadTransactions, NoopPayloadTransactions, PayloadTransactions}; -use reth_primitives_traits::{NodePrimitives, SealedHeader, SignedTransaction, TxTy}; +use reth_primitives_traits::{ + HeaderTy, NodePrimitives, SealedHeader, SealedHeaderFor, SignedTransaction, TxTy, +}; use reth_revm::{ cancelled::CancelOnDrop, database::StateProviderDatabase, db::State, witness::ExecutionWitnessRecord, @@ -40,12 +40,18 @@ use reth_revm::{ use reth_storage_api::{errors::ProviderError, StateProvider, StateProviderFactory}; use reth_transaction_pool::{BestTransactionsAttributes, PoolTransaction, TransactionPool}; use revm::context::{Block, BlockEnv}; -use std::sync::Arc; +use std::{marker::PhantomData, sync::Arc}; use tracing::{debug, trace, warn}; /// Optimism's payload builder -#[derive(Debug, Clone)] -pub struct OpPayloadBuilder { +#[derive(Debug)] +pub struct OpPayloadBuilder< + Pool, + Client, + Evm, + Txs = (), + Attrs = OpPayloadBuilderAttributes::Primitives>>, +> { /// The rollup's compute pending block configuration option. // TODO(clabby): Implement this feature. pub compute_pending_block: bool, @@ -60,9 +66,31 @@ pub struct OpPayloadBuilder { /// The type responsible for yielding the best transactions for the payload if mempool /// transactions are allowed. pub best_transactions: Txs, + /// Marker for the payload attributes type. + _pd: PhantomData, +} + +impl Clone for OpPayloadBuilder +where + Pool: Clone, + Client: Clone, + Evm: ConfigureEvm, + Txs: Clone, +{ + fn clone(&self) -> Self { + Self { + evm_config: self.evm_config.clone(), + pool: self.pool.clone(), + client: self.client.clone(), + config: self.config.clone(), + best_transactions: self.best_transactions.clone(), + compute_pending_block: self.compute_pending_block, + _pd: PhantomData, + } + } } -impl OpPayloadBuilder { +impl OpPayloadBuilder { /// `OpPayloadBuilder` constructor. /// /// Configures the builder with the default settings. @@ -84,11 +112,12 @@ impl OpPayloadBuilder { evm_config, config, best_transactions: (), + _pd: PhantomData, } } } -impl OpPayloadBuilder { +impl OpPayloadBuilder { /// Sets the rollup's compute pending block configuration option. pub const fn set_compute_pending_block(mut self, compute_pending_block: bool) -> Self { self.compute_pending_block = compute_pending_block; @@ -100,7 +129,7 @@ impl OpPayloadBuilder { pub fn with_transactions( self, best_transactions: T, - ) -> OpPayloadBuilder { + ) -> OpPayloadBuilder { let Self { pool, client, compute_pending_block, evm_config, config, .. } = self; OpPayloadBuilder { pool, @@ -109,6 +138,7 @@ impl OpPayloadBuilder { evm_config, best_transactions, config, + _pd: PhantomData, } } @@ -123,12 +153,16 @@ impl OpPayloadBuilder { } } -impl OpPayloadBuilder +impl OpPayloadBuilder where Pool: TransactionPool>, - Client: StateProviderFactory + ChainSpecProvider, + Client: StateProviderFactory + ChainSpecProvider, N: OpPayloadPrimitives, - Evm: ConfigureEvm, + Evm: ConfigureEvm< + Primitives = N, + NextBlockEnvCtx: BuildNextEnv, + >, + Attrs: OpAttributes>, { /// Constructs an Optimism payload from the transactions sent via the /// Payload attributes by the sequencer. If the `no_tx_pool` argument is passed in @@ -140,19 +174,18 @@ where /// a result indicating success with the payload or an error in case of failure. fn build_payload<'a, Txs>( &self, - args: BuildArguments, OpBuiltPayload>, + args: BuildArguments>, best: impl FnOnce(BestTransactionsAttributes) -> Txs + Send + Sync + 'a, ) -> Result>, PayloadBuilderError> where - Txs: PayloadTransactions< - Transaction: PoolTransaction + MaybeInteropTransaction, - >, + Txs: + PayloadTransactions + OpPooledTx>, { let BuildArguments { mut cached_reads, config, cancel, best_payload } = args; let ctx = OpPayloadBuilderCtx { evm_config: self.evm_config.clone(), - da_config: self.config.da_config.clone(), + builder_config: self.config.clone(), chain_spec: self.client.chain_spec(), config, cancel, @@ -164,7 +197,7 @@ where let state_provider = self.client.state_by_block_hash(ctx.parent().hash())?; let state = StateProviderDatabase::new(&state_provider); - if ctx.attributes().no_tx_pool { + if ctx.attributes().no_tx_pool() { builder.build(state, &state_provider, ctx) } else { // sequencer mode we can reuse cachedreads from previous runs @@ -176,16 +209,19 @@ where /// Computes the witness for the payload. pub fn payload_witness( &self, - parent: SealedHeader, - attributes: OpPayloadAttributes, - ) -> Result { - let attributes = OpPayloadBuilderAttributes::try_new(parent.hash(), attributes, 3) - .map_err(PayloadBuilderError::other)?; + parent: SealedHeader, + attributes: Attrs::RpcPayloadAttributes, + ) -> Result + where + Attrs: PayloadBuilderAttributes, + { + let attributes = + Attrs::try_new(parent.hash(), attributes, 3).map_err(PayloadBuilderError::other)?; let config = PayloadConfig { parent_header: Arc::new(parent), attributes }; let ctx = OpPayloadBuilderCtx { evm_config: self.evm_config.clone(), - da_config: self.config.da_config.clone(), + builder_config: self.config.clone(), chain_spec: self.client.chain_spec(), config, cancel: Default::default(), @@ -200,15 +236,20 @@ where } /// Implementation of the [`PayloadBuilder`] trait for [`OpPayloadBuilder`]. -impl PayloadBuilder for OpPayloadBuilder +impl PayloadBuilder + for OpPayloadBuilder where - Client: StateProviderFactory + ChainSpecProvider + Clone, N: OpPayloadPrimitives, + Client: StateProviderFactory + ChainSpecProvider + Clone, Pool: TransactionPool>, - Evm: ConfigureEvm, + Evm: ConfigureEvm< + Primitives = N, + NextBlockEnvCtx: BuildNextEnv, + >, Txs: OpPayloadTransactions, + Attrs: OpAttributes, { - type Attributes = OpPayloadBuilderAttributes; + type Attributes = Attrs; type BuiltPayload = OpBuiltPayload; fn try_build( @@ -232,7 +273,7 @@ where // system txs, hence on_missing_payload we return [MissingPayloadBehaviour::AwaitInProgress]. fn build_empty_payload( &self, - config: PayloadConfig, + config: PayloadConfig, ) -> Result { let args = BuildArguments { config, @@ -277,25 +318,33 @@ impl<'a, Txs> OpBuilder<'a, Txs> { impl OpBuilder<'_, Txs> { /// Builds the payload on top of the state. - pub fn build( + pub fn build( self, db: impl Database, state_provider: impl StateProvider, - ctx: OpPayloadBuilderCtx, + ctx: OpPayloadBuilderCtx, ) -> Result>, PayloadBuilderError> where - EvmConfig: ConfigureEvm, + Evm: ConfigureEvm< + Primitives = N, + NextBlockEnvCtx: BuildNextEnv, + >, ChainSpec: EthChainSpec + OpHardforks, N: OpPayloadPrimitives, - Txs: PayloadTransactions< - Transaction: PoolTransaction + MaybeInteropTransaction, - >, + Txs: + PayloadTransactions + OpPooledTx>, + Attrs: OpAttributes, { let Self { best } = self; - debug!(target: "payload_builder", id=%ctx.payload_id(), parent_header = ?ctx.parent().hash(), parent_number = ctx.parent().number, "building new payload"); + debug!(target: "payload_builder", id=%ctx.payload_id(), parent_header = ?ctx.parent().hash(), parent_number = ctx.parent().number(), "building new payload"); let mut db = State::builder().with_database(db).with_bundle_update().build(); + // Load the L1 block contract into the database cache. If the L1 block contract is not + // pre-loaded the database will panic when trying to fetch the DA footprint gas + // scalar. + db.load_cache_account(L1_BLOCK_CONTRACT).map_err(BlockExecutionError::other)?; + let mut builder = ctx.block_builder(&mut db)?; // 1. apply pre-execution changes @@ -308,7 +357,7 @@ impl OpBuilder<'_, Txs> { let mut info = ctx.execute_sequencer_transactions(&mut builder)?; // 3. if mem pool transactions are requested we execute them - if !ctx.attributes().no_tx_pool { + if !ctx.attributes().no_tx_pool() { let best_txs = best(ctx.best_transaction_attributes(builder.evm_mut().block())); if ctx.execute_best_transactions(&mut info, &mut builder, best_txs)?.is_some() { return Ok(BuildOutcomeKind::Cancelled) @@ -330,21 +379,19 @@ impl OpBuilder<'_, Txs> { let execution_outcome = ExecutionOutcome::new( db.take_bundle(), vec![execution_result.receipts], - block.number, + block.number(), Vec::new(), ); // create the executed block data - let executed: ExecutedBlockWithTrieUpdates = ExecutedBlockWithTrieUpdates { - block: ExecutedBlock { - recovered_block: Arc::new(block), - execution_output: Arc::new(execution_outcome), - hashed_state: Arc::new(hashed_state), - }, - trie: Arc::new(trie_updates), + let executed: ExecutedBlock = ExecutedBlock { + recovered_block: Arc::new(block), + execution_output: Arc::new(execution_outcome), + hashed_state: Arc::new(hashed_state), + trie_updates: Arc::new(trie_updates), }; - let no_tx_pool = ctx.attributes().no_tx_pool; + let no_tx_pool = ctx.attributes().no_tx_pool(); let payload = OpBuiltPayload::new(ctx.payload_id(), sealed_block, info.total_fees, Some(executed)); @@ -360,16 +407,20 @@ impl OpBuilder<'_, Txs> { } /// Builds the payload and returns its [`ExecutionWitness`] based on the state after execution. - pub fn witness( + pub fn witness( self, state_provider: impl StateProvider, - ctx: &OpPayloadBuilderCtx, + ctx: &OpPayloadBuilderCtx, ) -> Result where - Evm: ConfigureEvm, + Evm: ConfigureEvm< + Primitives = N, + NextBlockEnvCtx: BuildNextEnv, + >, ChainSpec: EthChainSpec + OpHardforks, N: OpPayloadPrimitives, Txs: PayloadTransactions>, + Attrs: OpAttributes, { let mut db = State::builder() .with_database(StateProviderDatabase::new(&state_provider)) @@ -458,78 +509,80 @@ impl ExecutionInfo { /// maximum allowed DA limit per block. pub fn is_tx_over_limits( &self, - tx: &(impl Encodable + Transaction), + tx_da_size: u64, block_gas_limit: u64, tx_data_limit: Option, block_data_limit: Option, + tx_gas_limit: u64, + da_footprint_gas_scalar: Option, ) -> bool { - if tx_data_limit.is_some_and(|da_limit| tx.length() as u64 > da_limit) { + if tx_data_limit.is_some_and(|da_limit| tx_da_size > da_limit) { return true; } - if block_data_limit - .is_some_and(|da_limit| self.cumulative_da_bytes_used + (tx.length() as u64) > da_limit) - { + let total_da_bytes_used = self.cumulative_da_bytes_used.saturating_add(tx_da_size); + + if block_data_limit.is_some_and(|da_limit| total_da_bytes_used > da_limit) { return true; } - self.cumulative_gas_used + tx.gas_limit() > block_gas_limit + // Post Jovian: the tx DA footprint must be less than the block gas limit + if let Some(da_footprint_gas_scalar) = da_footprint_gas_scalar { + let tx_da_footprint = + total_da_bytes_used.saturating_mul(da_footprint_gas_scalar as u64); + if tx_da_footprint > block_gas_limit { + return true; + } + } + + self.cumulative_gas_used + tx_gas_limit > block_gas_limit } } /// Container type that holds all necessities to build a new payload. #[derive(derive_more::Debug)] -pub struct OpPayloadBuilderCtx { +pub struct OpPayloadBuilderCtx< + Evm: ConfigureEvm, + ChainSpec, + Attrs = OpPayloadBuilderAttributes::Primitives>>, +> { /// The type that knows how to perform system calls and configure the evm. pub evm_config: Evm, - /// The DA config for the payload builder - pub da_config: OpDAConfig, + /// Additional config for the builder/sequencer, e.g. DA and gas limit + pub builder_config: OpBuilderConfig, /// The chainspec pub chain_spec: Arc, /// How to build the payload. - pub config: PayloadConfig>>, + pub config: PayloadConfig>, /// Marker to check whether the job has been cancelled. pub cancel: CancelOnDrop, /// The currently best payload. pub best_payload: Option>, } -impl OpPayloadBuilderCtx +impl OpPayloadBuilderCtx where - Evm: ConfigureEvm, + Evm: ConfigureEvm< + Primitives: OpPayloadPrimitives, + NextBlockEnvCtx: BuildNextEnv, ChainSpec>, + >, ChainSpec: EthChainSpec + OpHardforks, + Attrs: OpAttributes>, { /// Returns the parent block the payload will be build on. - pub fn parent(&self) -> &SealedHeader { - &self.config.parent_header + pub fn parent(&self) -> &SealedHeaderFor { + self.config.parent_header.as_ref() } /// Returns the builder attributes. - pub const fn attributes(&self) -> &OpPayloadBuilderAttributes> { + pub const fn attributes(&self) -> &Attrs { &self.config.attributes } - /// Returns the extra data for the block. - /// - /// After holocene this extracts the extra data from the payload - pub fn extra_data(&self) -> Result { - if self.is_holocene_active() { - self.attributes() - .get_holocene_extra_data( - self.chain_spec.base_fee_params_at_timestamp( - self.attributes().payload_attributes.timestamp, - ), - ) - .map_err(PayloadBuilderError::other) - } else { - Ok(Default::default()) - } - } - /// Returns the current fee settings for transactions from the mempool - pub fn best_transaction_attributes(&self, block_env: &BlockEnv) -> BestTransactionsAttributes { + pub fn best_transaction_attributes(&self, block_env: impl Block) -> BestTransactionsAttributes { BestTransactionsAttributes::new( - block_env.basefee, + block_env.basefee(), block_env.blob_gasprice().map(|p| p as u64), ) } @@ -539,11 +592,6 @@ where self.attributes().payload_id() } - /// Returns true if holocene is active for the payload. - pub fn is_holocene_active(&self) -> bool { - self.chain_spec.is_holocene_active_at_timestamp(self.attributes().timestamp()) - } - /// Returns true if the fees are higher than the previous payload. pub fn is_better_payload(&self, total_fees: U256) -> bool { is_better_payload(self.best_payload.as_ref(), total_fees) @@ -553,29 +601,27 @@ where pub fn block_builder<'a, DB: Database>( &'a self, db: &'a mut State, - ) -> Result + 'a, PayloadBuilderError> { + ) -> Result< + impl BlockBuilder< + Primitives = Evm::Primitives, + Executor: BlockExecutorFor<'a, Evm::BlockExecutorFactory, DB>, + > + 'a, + PayloadBuilderError, + > { self.evm_config .builder_for_next_block( db, self.parent(), - OpNextBlockEnvAttributes { - timestamp: self.attributes().timestamp(), - suggested_fee_recipient: self.attributes().suggested_fee_recipient(), - prev_randao: self.attributes().prev_randao(), - gas_limit: self.attributes().gas_limit.unwrap_or(self.parent().gas_limit), - parent_beacon_block_root: self.attributes().parent_beacon_block_root(), - extra_data: self.extra_data()?, - }, + Evm::NextBlockEnvCtx::build_next_env( + self.attributes(), + self.parent(), + self.chain_spec.as_ref(), + ) + .map_err(PayloadBuilderError::other)?, ) .map_err(PayloadBuilderError::other) } -} -impl OpPayloadBuilderCtx -where - Evm: ConfigureEvm, - ChainSpec: EthChainSpec + OpHardforks, -{ /// Executes all sequencer transactions that are included in the payload attributes. pub fn execute_sequencer_transactions( &self, @@ -583,7 +629,7 @@ where ) -> Result { let mut info = ExecutionInfo::new(); - for sequencer_tx in &self.attributes().transactions { + for sequencer_tx in self.attributes().sequencer_transactions() { // A sequencer's block should never contain blob transactions. if sequencer_tx.value().is_eip4844() { return Err(PayloadBuilderError::other( @@ -624,24 +670,50 @@ where /// Executes the given best transactions and updates the execution info. /// /// Returns `Ok(Some(())` if the job was cancelled. - pub fn execute_best_transactions( + pub fn execute_best_transactions( &self, info: &mut ExecutionInfo, - builder: &mut impl BlockBuilder, + builder: &mut Builder, mut best_txs: impl PayloadTransactions< - Transaction: PoolTransaction> - + MaybeInteropTransaction, + Transaction: PoolTransaction> + OpPooledTx, >, - ) -> Result, PayloadBuilderError> { - let block_gas_limit = builder.evm_mut().block().gas_limit; - let block_da_limit = self.da_config.max_da_block_size(); - let tx_da_limit = self.da_config.max_da_tx_size(); - let base_fee = builder.evm_mut().block().basefee; + ) -> Result, PayloadBuilderError> + where + Builder: BlockBuilder, + <::Evm as AlloyEvm>::DB: Database, + { + let mut block_gas_limit = builder.evm_mut().block().gas_limit(); + if let Some(gas_limit_config) = self.builder_config.gas_limit_config.gas_limit() { + // If a gas limit is configured, use that limit as target if it's smaller, otherwise use + // the block's actual gas limit. + block_gas_limit = gas_limit_config.min(block_gas_limit); + }; + let block_da_limit = self.builder_config.da_config.max_da_block_size(); + let tx_da_limit = self.builder_config.da_config.max_da_tx_size(); + let base_fee = builder.evm_mut().block().basefee(); while let Some(tx) = best_txs.next(()) { let interop = tx.interop_deadline(); + let tx_da_size = tx.estimated_da_size(); let tx = tx.into_consensus(); - if info.is_tx_over_limits(tx.inner(), block_gas_limit, tx_da_limit, block_da_limit) { + + let da_footprint_gas_scalar = self + .chain_spec + .is_jovian_active_at_timestamp(self.attributes().timestamp()) + .then_some( + L1BlockInfo::fetch_da_footprint_gas_scalar(builder.evm_mut().db_mut()).expect( + "DA footprint should always be available from the database post jovian", + ), + ); + + if info.is_tx_over_limits( + tx_da_size, + block_gas_limit, + tx_da_limit, + block_da_limit, + tx.gas_limit(), + da_footprint_gas_scalar, + ) { // we can't fit this transaction into the block, so we need to mark it as // invalid which also removes all dependent transaction from // the iterator before we can continue @@ -657,11 +729,11 @@ where // We skip invalid cross chain txs, they would be removed on the next block update in // the maintenance job - if let Some(interop) = interop { - if !is_valid_interop(interop, self.config.attributes.timestamp()) { - best_txs.mark_invalid(tx.signer(), tx.nonce()); - continue - } + if let Some(interop) = interop && + !is_valid_interop(interop, self.config.attributes.timestamp()) + { + best_txs.mark_invalid(tx.signer(), tx.nonce()); + continue } // check if the job was cancelled, if so we can exit early if self.cancel.is_cancelled() { @@ -694,9 +766,9 @@ where // add gas used by the transaction to cumulative gas used, before creating the // receipt info.cumulative_gas_used += gas_used; - info.cumulative_da_bytes_used += tx.length() as u64; + info.cumulative_da_bytes_used += tx_da_size; - // update add to total fees + // update and add to total fees let miner_fee = tx .effective_tip_per_gas(base_fee) .expect("fee is always valid; execution succeeded"); diff --git a/crates/optimism/payload/src/config.rs b/crates/optimism/payload/src/config.rs index 469bfc9fe31..c79ee0ece4b 100644 --- a/crates/optimism/payload/src/config.rs +++ b/crates/optimism/payload/src/config.rs @@ -7,12 +7,14 @@ use std::sync::{atomic::AtomicU64, Arc}; pub struct OpBuilderConfig { /// Data availability configuration for the OP builder. pub da_config: OpDAConfig, + /// Gas limit configuration for the OP builder. + pub gas_limit_config: OpGasLimitConfig, } impl OpBuilderConfig { /// Creates a new OP builder configuration with the given data availability configuration. - pub const fn new(da_config: OpDAConfig) -> Self { - Self { da_config } + pub const fn new(da_config: OpDAConfig, gas_limit_config: OpGasLimitConfig) -> Self { + Self { da_config, gas_limit_config } } /// Returns the Data Availability configuration for the OP builder, if it has configured @@ -100,6 +102,40 @@ struct OpDAConfigInner { max_da_block_size: AtomicU64, } +/// Contains the Gas Limit configuration for the OP builder. +/// +/// This type is shareable and can be used to update the Gas Limit configuration for the OP payload +/// builder. +#[derive(Debug, Clone, Default)] +pub struct OpGasLimitConfig { + /// Gas limit for a transaction + /// + /// 0 means use the default gas limit. + gas_limit: Arc, +} + +impl OpGasLimitConfig { + /// Creates a new Gas Limit configuration with the given maximum gas limit. + pub fn new(max_gas_limit: u64) -> Self { + let this = Self::default(); + this.set_gas_limit(max_gas_limit); + this + } + /// Returns the gas limit for a transaction, if any. + pub fn gas_limit(&self) -> Option { + let val = self.gas_limit.load(std::sync::atomic::Ordering::Relaxed); + if val == 0 { + None + } else { + Some(val) + } + } + /// Sets the gas limit for a transaction. 0 means use the default gas limit. + pub fn set_gas_limit(&self, gas_limit: u64) { + self.gas_limit.store(gas_limit, std::sync::atomic::Ordering::Relaxed); + } +} + #[cfg(test)] mod tests { use super::*; @@ -122,4 +158,14 @@ mod tests { let config = OpBuilderConfig::default(); assert!(config.constrained_da_config().is_none()); } + + #[test] + fn test_gas_limit() { + let gas_limit = OpGasLimitConfig::default(); + assert_eq!(gas_limit.gas_limit(), None); + gas_limit.set_gas_limit(50000); + assert_eq!(gas_limit.gas_limit(), Some(50000)); + gas_limit.set_gas_limit(0); + assert_eq!(gas_limit.gas_limit(), None); + } } diff --git a/crates/optimism/payload/src/lib.rs b/crates/optimism/payload/src/lib.rs index 038e7cab833..57f21ef967f 100644 --- a/crates/optimism/payload/src/lib.rs +++ b/crates/optimism/payload/src/lib.rs @@ -6,7 +6,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![allow(clippy::useless_let_if_seq)] extern crate alloc; @@ -16,11 +16,13 @@ pub use builder::OpPayloadBuilder; pub mod error; pub mod payload; use op_alloy_rpc_types_engine::OpExecutionData; -pub use payload::{OpBuiltPayload, OpPayloadAttributes, OpPayloadBuilderAttributes}; +pub use payload::{ + payload_id_optimism, OpBuiltPayload, OpPayloadAttributes, OpPayloadBuilderAttributes, +}; mod traits; -use reth_optimism_primitives::{OpBlock, OpPrimitives}; +use reth_optimism_primitives::OpPrimitives; use reth_payload_primitives::{BuiltPayload, PayloadTypes}; -use reth_primitives_traits::{NodePrimitives, SealedBlock}; +use reth_primitives_traits::{Block, NodePrimitives, SealedBlock}; pub use traits::*; pub mod validator; pub use validator::OpExecutionPayloadValidator; @@ -34,7 +36,7 @@ pub struct OpPayloadTypes(core::marker::Phanto impl PayloadTypes for OpPayloadTypes where - OpBuiltPayload: BuiltPayload>, + OpBuiltPayload: BuiltPayload, { type ExecutionData = OpExecutionData; type BuiltPayload = OpBuiltPayload; @@ -46,6 +48,9 @@ where <::Primitives as NodePrimitives>::Block, >, ) -> Self::ExecutionData { - OpExecutionData::from_block_unchecked(block.hash(), &block.into_block()) + OpExecutionData::from_block_unchecked( + block.hash(), + &block.into_block().into_ethereum_block(), + ) } } diff --git a/crates/optimism/payload/src/payload.rs b/crates/optimism/payload/src/payload.rs index ebdb2b9a762..41b825a2b72 100644 --- a/crates/optimism/payload/src/payload.rs +++ b/crates/optimism/payload/src/payload.rs @@ -2,7 +2,7 @@ use std::{fmt::Debug, sync::Arc}; -use alloy_consensus::Block; +use alloy_consensus::{Block, BlockHeader}; use alloy_eips::{ eip1559::BaseFeeParams, eip2718::Decodable2718, eip4895::Withdrawals, eip7685::Requests, }; @@ -12,18 +12,23 @@ use alloy_rpc_types_engine::{ BlobsBundleV1, ExecutionPayloadEnvelopeV2, ExecutionPayloadFieldV2, ExecutionPayloadV1, ExecutionPayloadV3, PayloadId, }; -use op_alloy_consensus::{encode_holocene_extra_data, EIP1559ParamError}; +use op_alloy_consensus::{encode_holocene_extra_data, encode_jovian_extra_data, EIP1559ParamError}; use op_alloy_rpc_types_engine::{ OpExecutionPayloadEnvelopeV3, OpExecutionPayloadEnvelopeV4, OpExecutionPayloadV4, }; -use reth_chain_state::ExecutedBlockWithTrieUpdates; -use reth_optimism_primitives::OpPrimitives; -use reth_payload_builder::EthPayloadBuilderAttributes; -use reth_payload_primitives::{BuiltPayload, PayloadBuilderAttributes}; -use reth_primitives_traits::{NodePrimitives, SealedBlock, SignedTransaction, WithEncoded}; +use reth_chain_state::ExecutedBlock; +use reth_chainspec::EthChainSpec; +use reth_optimism_evm::OpNextBlockEnvAttributes; +use reth_optimism_forks::OpHardforks; +use reth_payload_builder::{EthPayloadBuilderAttributes, PayloadBuilderError}; +use reth_payload_primitives::{BuildNextEnv, BuiltPayload, PayloadBuilderAttributes}; +use reth_primitives_traits::{ + NodePrimitives, SealedBlock, SealedHeader, SignedTransaction, WithEncoded, +}; /// Re-export for use in downstream arguments. pub use op_alloy_rpc_types_engine::OpPayloadAttributes; +use reth_optimism_primitives::OpPrimitives; /// Optimism Payload Builder Attributes #[derive(Debug, Clone, PartialEq, Eq)] @@ -39,6 +44,8 @@ pub struct OpPayloadBuilderAttributes { pub gas_limit: Option, /// EIP-1559 parameters for the generated payload pub eip_1559_params: Option, + /// Min base fee for the generated payload (only available post-Jovian) + pub min_base_fee: Option, } impl Default for OpPayloadBuilderAttributes { @@ -49,12 +56,14 @@ impl Default for OpPayloadBuilderAttributes { gas_limit: Default::default(), eip_1559_params: Default::default(), transactions: Default::default(), + min_base_fee: Default::default(), } } } impl OpPayloadBuilderAttributes { - /// Extracts the `eip1559` parameters for the payload. + /// Extracts the extra data parameters post-Holocene hardfork. + /// In Holocene, those parameters are the EIP-1559 base fee parameters. pub fn get_holocene_extra_data( &self, default_base_fee_params: BaseFeeParams, @@ -63,9 +72,21 @@ impl OpPayloadBuilderAttributes { .map(|params| encode_holocene_extra_data(params, default_base_fee_params)) .ok_or(EIP1559ParamError::NoEIP1559Params)? } + + /// Extracts the extra data parameters post-Jovian hardfork. + /// Those parameters are the EIP-1559 parameters from Holocene and the minimum base fee. + pub fn get_jovian_extra_data( + &self, + default_base_fee_params: BaseFeeParams, + ) -> Result { + let min_base_fee = self.min_base_fee.ok_or(EIP1559ParamError::MinBaseFeeNotSet)?; + self.eip_1559_params + .map(|params| encode_jovian_extra_data(params, default_base_fee_params, min_base_fee)) + .ok_or(EIP1559ParamError::NoEIP1559Params)? + } } -impl PayloadBuilderAttributes +impl PayloadBuilderAttributes for OpPayloadBuilderAttributes { type RpcPayloadAttributes = OpPayloadAttributes; @@ -86,14 +107,7 @@ impl PayloadBuilderAttributes .unwrap_or_default() .into_iter() .map(|data| { - let mut buf = data.as_ref(); - let tx = Decodable2718::decode_2718(&mut buf).map_err(alloy_rlp::Error::from)?; - - if !buf.is_empty() { - return Err(alloy_rlp::Error::UnexpectedLength); - } - - Ok(WithEncoded::new(data, tx)) + Decodable2718::decode_2718_exact(data.as_ref()).map(|tx| WithEncoded::new(data, tx)) }) .collect::>()?; @@ -113,6 +127,7 @@ impl PayloadBuilderAttributes transactions, gas_limit: attributes.gas_limit, eip_1559_params: attributes.eip_1559_params, + min_base_fee: attributes.min_base_fee, }) } @@ -161,7 +176,7 @@ pub struct OpBuiltPayload { /// Sealed block pub(crate) block: Arc>, /// Block execution data for the payload, if any. - pub(crate) executed_block: Option>, + pub(crate) executed_block: Option>, /// The fees of the block pub(crate) fees: U256, } @@ -174,7 +189,7 @@ impl OpBuiltPayload { id: PayloadId, block: Arc>, fees: U256, - executed_block: Option>, + executed_block: Option>, ) -> Self { Self { id, block, fees, executed_block } } @@ -211,7 +226,7 @@ impl BuiltPayload for OpBuiltPayload { self.fees } - fn executed_block(&self) -> Option> { + fn executed_block(&self) -> Option> { self.executed_block.clone() } @@ -327,7 +342,10 @@ where /// Generates the payload id for the configured payload from the [`OpPayloadAttributes`]. /// /// Returns an 8-byte identifier by hashing the payload components with sha256 hash. -pub(crate) fn payload_id_optimism( +/// +/// Note: This must be updated whenever the [`OpPayloadAttributes`] changes for a hardfork. +/// See also +pub fn payload_id_optimism( parent: &B256, attributes: &OpPayloadAttributes, payload_version: u8, @@ -372,11 +390,54 @@ pub(crate) fn payload_id_optimism( hasher.update(eip_1559_params.as_slice()); } + if let Some(min_base_fee) = attributes.min_base_fee { + hasher.update(min_base_fee.to_be_bytes()); + } + let mut out = hasher.finalize(); out[0] = payload_version; PayloadId::new(out.as_slice()[..8].try_into().expect("sufficient length")) } +impl BuildNextEnv, H, ChainSpec> + for OpNextBlockEnvAttributes +where + H: BlockHeader, + T: SignedTransaction, + ChainSpec: EthChainSpec + OpHardforks, +{ + fn build_next_env( + attributes: &OpPayloadBuilderAttributes, + parent: &SealedHeader, + chain_spec: &ChainSpec, + ) -> Result { + let extra_data = if chain_spec.is_jovian_active_at_timestamp(attributes.timestamp()) { + attributes + .get_jovian_extra_data( + chain_spec.base_fee_params_at_timestamp(attributes.timestamp()), + ) + .map_err(PayloadBuilderError::other)? + } else if chain_spec.is_holocene_active_at_timestamp(attributes.timestamp()) { + attributes + .get_holocene_extra_data( + chain_spec.base_fee_params_at_timestamp(attributes.timestamp()), + ) + .map_err(PayloadBuilderError::other)? + } else { + Default::default() + }; + + Ok(Self { + timestamp: attributes.timestamp(), + suggested_fee_recipient: attributes.suggested_fee_recipient(), + prev_randao: attributes.prev_randao(), + gas_limit: attributes.gas_limit.unwrap_or_else(|| parent.gas_limit()), + parent_beacon_block_root: attributes.parent_beacon_block_root(), + extra_data, + }) + } +} + #[cfg(test)] mod tests { use super::*; @@ -405,6 +466,7 @@ mod tests { no_tx_pool: None, gas_limit: Some(30000000), eip_1559_params: None, + min_base_fee: None, }; // Reth's `PayloadId` should match op-geth's `PayloadId`. This fails @@ -418,6 +480,37 @@ mod tests { ); } + #[test] + fn test_payload_id_parity_op_geth_jovian() { + // + let expected = + PayloadId::new(FixedBytes::<8>::from_str("0x046c65ffc4d659ec").unwrap().into()); + let attrs = OpPayloadAttributes { + payload_attributes: PayloadAttributes { + timestamp: 1728933301, + prev_randao: b256!("0x9158595abbdab2c90635087619aa7042bbebe47642dfab3c9bfb934f6b082765"), + suggested_fee_recipient: address!("0x4200000000000000000000000000000000000011"), + withdrawals: Some([].into()), + parent_beacon_block_root: b256!("0x8fe0193b9bf83cb7e5a08538e494fecc23046aab9a497af3704f4afdae3250ff").into(), + }, + transactions: Some([bytes!("7ef8f8a0dc19cfa777d90980e4875d0a548a881baaa3f83f14d1bc0d3038bc329350e54194deaddeaddeaddeaddeaddeaddeaddeaddead00019442000000000000000000000000000000000000158080830f424080b8a4440a5e20000f424000000000000000000000000300000000670d6d890000000000000125000000000000000000000000000000000000000000000000000000000000000700000000000000000000000000000000000000000000000000000000000000014bf9181db6e381d4384bbf69c48b0ee0eed23c6ca26143c6d2544f9d39997a590000000000000000000000007f83d659683caf2767fd3c720981d51f5bc365bc")].into()), + no_tx_pool: None, + gas_limit: Some(30000000), + eip_1559_params: None, + min_base_fee: Some(100), + }; + + // Reth's `PayloadId` should match op-geth's `PayloadId`. This fails + assert_eq!( + expected, + payload_id_optimism( + &b256!("0x3533bf30edaf9505d0810bf475cbe4e5f4b9889904b9845e83efdeab4e92eb1e"), + &attrs, + EngineApiMessageVersion::V4 as u8 + ) + ); + } + #[test] fn test_get_extra_data_post_holocene() { let attributes: OpPayloadBuilderAttributes = @@ -436,4 +529,50 @@ mod tests { let extra_data = attributes.get_holocene_extra_data(BaseFeeParams::new(80, 60)); assert_eq!(extra_data.unwrap(), Bytes::copy_from_slice(&[0, 0, 0, 0, 80, 0, 0, 0, 60])); } + + #[test] + fn test_get_extra_data_post_jovian() { + let attributes: OpPayloadBuilderAttributes = + OpPayloadBuilderAttributes { + eip_1559_params: Some(B64::from_str("0x0000000800000008").unwrap()), + min_base_fee: Some(10), + ..Default::default() + }; + let extra_data = attributes.get_jovian_extra_data(BaseFeeParams::new(80, 60)); + assert_eq!( + extra_data.unwrap(), + // Version byte is 1 for Jovian, then holocene payload followed by 8 bytes for the + // minimum base fee + Bytes::copy_from_slice(&[1, 0, 0, 0, 8, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 10]) + ); + } + + #[test] + fn test_get_extra_data_post_jovian_default() { + let attributes: OpPayloadBuilderAttributes = + OpPayloadBuilderAttributes { + eip_1559_params: Some(B64::ZERO), + min_base_fee: Some(10), + ..Default::default() + }; + let extra_data = attributes.get_jovian_extra_data(BaseFeeParams::new(80, 60)); + assert_eq!( + extra_data.unwrap(), + // Version byte is 1 for Jovian, then holocene payload followed by 8 bytes for the + // minimum base fee + Bytes::copy_from_slice(&[1, 0, 0, 0, 80, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 10]) + ); + } + + #[test] + fn test_get_extra_data_post_jovian_no_base_fee() { + let attributes: OpPayloadBuilderAttributes = + OpPayloadBuilderAttributes { + eip_1559_params: Some(B64::ZERO), + min_base_fee: None, + ..Default::default() + }; + let extra_data = attributes.get_jovian_extra_data(BaseFeeParams::new(80, 60)); + assert_eq!(extra_data.unwrap_err(), EIP1559ParamError::MinBaseFeeNotSet); + } } diff --git a/crates/optimism/payload/src/traits.rs b/crates/optimism/payload/src/traits.rs index a0d13022cd5..485b8d1df9e 100644 --- a/crates/optimism/payload/src/traits.rs +++ b/crates/optimism/payload/src/traits.rs @@ -1,29 +1,60 @@ -use alloy_consensus::{BlockBody, Header}; +use alloy_consensus::BlockBody; use reth_optimism_primitives::{transaction::OpTransaction, DepositReceipt}; -use reth_primitives_traits::{NodePrimitives, SignedTransaction}; +use reth_payload_primitives::PayloadBuilderAttributes; +use reth_primitives_traits::{FullBlockHeader, NodePrimitives, SignedTransaction, WithEncoded}; + +use crate::OpPayloadBuilderAttributes; /// Helper trait to encapsulate common bounds on [`NodePrimitives`] for OP payload builder. pub trait OpPayloadPrimitives: NodePrimitives< Receipt: DepositReceipt, SignedTx = Self::_TX, - BlockHeader = Header, - BlockBody = BlockBody, + BlockBody = BlockBody, + BlockHeader = Self::_Header, > { /// Helper AT to bound [`NodePrimitives::Block`] type without causing bound cycle. type _TX: SignedTransaction + OpTransaction; + /// Helper AT to bound [`NodePrimitives::Block`] type without causing bound cycle. + type _Header: FullBlockHeader; } -impl OpPayloadPrimitives for T +impl OpPayloadPrimitives for T where Tx: SignedTransaction + OpTransaction, T: NodePrimitives< SignedTx = Tx, Receipt: DepositReceipt, + BlockBody = BlockBody, BlockHeader = Header, - BlockBody = BlockBody, >, + Header: FullBlockHeader, { type _TX = Tx; + type _Header = Header; +} + +/// Attributes for the OP payload builder. +pub trait OpAttributes: PayloadBuilderAttributes { + /// Primitive transaction type. + type Transaction: SignedTransaction; + + /// Whether to use the transaction pool for the payload. + fn no_tx_pool(&self) -> bool; + + /// Sequencer transactions to include in the payload. + fn sequencer_transactions(&self) -> &[WithEncoded]; +} + +impl OpAttributes for OpPayloadBuilderAttributes { + type Transaction = T; + + fn no_tx_pool(&self) -> bool { + self.no_tx_pool + } + + fn sequencer_transactions(&self) -> &[WithEncoded] { + &self.transactions + } } diff --git a/crates/optimism/payload/src/validator.rs b/crates/optimism/payload/src/validator.rs index 274f0edc06f..fa0d610469c 100644 --- a/crates/optimism/payload/src/validator.rs +++ b/crates/optimism/payload/src/validator.rs @@ -27,59 +27,74 @@ where } /// Ensures that the given payload does not violate any consensus rules that concern the block's - /// layout, like: - /// - missing or invalid base fee - /// - invalid extra data - /// - invalid transactions - /// - incorrect hash - /// - block contains blob transactions or blob versioned hashes - /// - block contains l1 withdrawals + /// layout. /// - /// The checks are done in the order that conforms with the engine-API specification. - /// - /// This is intended to be invoked after receiving the payload from the CLI. - /// The additional fields, starting with [`MaybeCancunPayloadFields`](alloy_rpc_types_engine::MaybeCancunPayloadFields), are not part of the payload, but are additional fields starting in the `engine_newPayloadV3` RPC call, See also - /// - /// If the cancun fields are provided this also validates that the versioned hashes in the block - /// are empty as well as those passed in the sidecar. If the payload fields are not provided. - /// - /// Validation according to specs . + /// See also [`ensure_well_formed_payload`]. pub fn ensure_well_formed_payload( &self, payload: OpExecutionData, ) -> Result>, OpPayloadError> { - let OpExecutionData { payload, sidecar } = payload; + ensure_well_formed_payload(self.chain_spec(), payload) + } +} - let expected_hash = payload.block_hash(); +/// Ensures that the given payload does not violate any consensus rules that concern the block's +/// layout, like: +/// - missing or invalid base fee +/// - invalid extra data +/// - invalid transactions +/// - incorrect hash +/// - block contains blob transactions or blob versioned hashes +/// - block contains l1 withdrawals +/// +/// The checks are done in the order that conforms with the engine-API specification. +/// +/// This is intended to be invoked after receiving the payload from the CLI. +/// The additional fields, starting with [`MaybeCancunPayloadFields`](alloy_rpc_types_engine::MaybeCancunPayloadFields), are not part of the payload, but are additional fields starting in the `engine_newPayloadV3` RPC call, See also +/// +/// If the cancun fields are provided this also validates that the versioned hashes in the block +/// are empty as well as those passed in the sidecar. If the payload fields are not provided. +/// +/// Validation according to specs . +pub fn ensure_well_formed_payload( + chain_spec: ChainSpec, + payload: OpExecutionData, +) -> Result>, OpPayloadError> +where + ChainSpec: OpHardforks, + T: SignedTransaction, +{ + let OpExecutionData { payload, sidecar } = payload; - // First parse the block - let sealed_block = payload.try_into_block_with_sidecar(&sidecar)?.seal_slow(); + let expected_hash = payload.block_hash(); - // Ensure the hash included in the payload matches the block hash - if expected_hash != sealed_block.hash() { - return Err(PayloadError::BlockHash { - execution: sealed_block.hash(), - consensus: expected_hash, - })? - } + // First parse the block + let sealed_block = payload.try_into_block_with_sidecar(&sidecar)?.seal_slow(); - shanghai::ensure_well_formed_fields( - sealed_block.body(), - self.is_shanghai_active_at_timestamp(sealed_block.timestamp), - )?; + // Ensure the hash included in the payload matches the block hash + if expected_hash != sealed_block.hash() { + return Err(PayloadError::BlockHash { + execution: sealed_block.hash(), + consensus: expected_hash, + })? + } - cancun::ensure_well_formed_header_and_sidecar_fields( - &sealed_block, - sidecar.canyon(), - self.is_cancun_active_at_timestamp(sealed_block.timestamp), - )?; + shanghai::ensure_well_formed_fields( + sealed_block.body(), + chain_spec.is_shanghai_active_at_timestamp(sealed_block.timestamp), + )?; - prague::ensure_well_formed_fields( - sealed_block.body(), - sidecar.isthmus(), - self.is_prague_active_at_timestamp(sealed_block.timestamp), - )?; + cancun::ensure_well_formed_header_and_sidecar_fields( + &sealed_block, + sidecar.ecotone(), + chain_spec.is_cancun_active_at_timestamp(sealed_block.timestamp), + )?; - Ok(sealed_block) - } + prague::ensure_well_formed_fields( + sealed_block.body(), + sidecar.isthmus(), + chain_spec.is_prague_active_at_timestamp(sealed_block.timestamp), + )?; + + Ok(sealed_block) } diff --git a/crates/optimism/primitives/src/lib.rs b/crates/optimism/primitives/src/lib.rs index 8a447ffc2fa..8100d70c916 100644 --- a/crates/optimism/primitives/src/lib.rs +++ b/crates/optimism/primitives/src/lib.rs @@ -5,7 +5,7 @@ html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256", issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(test), warn(unused_crate_dependencies))] #![cfg_attr(not(feature = "std"), no_std)] #![allow(unused)] diff --git a/crates/optimism/primitives/src/receipt.rs b/crates/optimism/primitives/src/receipt.rs index f3511be7b3c..74f21eab115 100644 --- a/crates/optimism/primitives/src/receipt.rs +++ b/crates/optimism/primitives/src/receipt.rs @@ -1,8 +1,12 @@ +use alloc::vec::Vec; use alloy_consensus::{ Eip2718EncodableReceipt, Eip658Value, Receipt, ReceiptWithBloom, RlpDecodableReceipt, RlpEncodableReceipt, TxReceipt, Typed2718, }; -use alloy_eips::{eip2718::Eip2718Result, Decodable2718, Encodable2718}; +use alloy_eips::{ + eip2718::{Eip2718Result, IsTyped2718}, + Decodable2718, Encodable2718, +}; use alloy_primitives::{Bloom, Log}; use alloy_rlp::{BufMut, Decodable, Encodable, Header}; use op_alloy_consensus::{OpDepositReceipt, OpTxType}; @@ -61,6 +65,17 @@ impl OpReceipt { } } + /// Consumes this and returns the inner [`Receipt`]. + pub fn into_receipt(self) -> Receipt { + match self { + Self::Legacy(receipt) | + Self::Eip2930(receipt) | + Self::Eip1559(receipt) | + Self::Eip7702(receipt) => receipt, + Self::Deposit(receipt) => receipt.inner, + } + } + /// Returns length of RLP-encoded receipt fields with the given [`Bloom`] without an RLP header. pub fn rlp_encoded_fields_length(&self, bloom: &Bloom) -> usize { match self { @@ -343,6 +358,16 @@ impl TxReceipt for OpReceipt { fn logs(&self) -> &[Log] { self.as_receipt().logs() } + + fn into_logs(self) -> Vec { + match self { + Self::Legacy(receipt) | + Self::Eip2930(receipt) | + Self::Eip1559(receipt) | + Self::Eip7702(receipt) => receipt.logs, + Self::Deposit(receipt) => receipt.inner.logs, + } + } } impl Typed2718 for OpReceipt { @@ -351,18 +376,49 @@ impl Typed2718 for OpReceipt { } } +impl IsTyped2718 for OpReceipt { + fn is_type(type_id: u8) -> bool { + ::is_type(type_id) + } +} + impl InMemorySize for OpReceipt { fn size(&self) -> usize { self.as_receipt().size() } } -impl reth_primitives_traits::Receipt for OpReceipt {} +impl From for OpReceipt { + fn from(envelope: op_alloy_consensus::OpReceiptEnvelope) -> Self { + match envelope { + op_alloy_consensus::OpReceiptEnvelope::Legacy(receipt) => Self::Legacy(receipt.receipt), + op_alloy_consensus::OpReceiptEnvelope::Eip2930(receipt) => { + Self::Eip2930(receipt.receipt) + } + op_alloy_consensus::OpReceiptEnvelope::Eip1559(receipt) => { + Self::Eip1559(receipt.receipt) + } + op_alloy_consensus::OpReceiptEnvelope::Eip7702(receipt) => { + Self::Eip7702(receipt.receipt) + } + op_alloy_consensus::OpReceiptEnvelope::Deposit(receipt) => { + Self::Deposit(OpDepositReceipt { + deposit_nonce: receipt.receipt.deposit_nonce, + deposit_receipt_version: receipt.receipt.deposit_receipt_version, + inner: receipt.receipt.inner, + }) + } + } + } +} /// Trait for deposit receipt. pub trait DepositReceipt: reth_primitives_traits::Receipt { - /// Returns deposit receipt if it is a deposit transaction. + /// Converts a `Receipt` into a mutable Optimism deposit receipt. fn as_deposit_receipt_mut(&mut self) -> Option<&mut OpDepositReceipt>; + + /// Extracts an Optimism deposit receipt from `Receipt`. + fn as_deposit_receipt(&self) -> Option<&OpDepositReceipt>; } impl DepositReceipt for OpReceipt { @@ -372,6 +428,13 @@ impl DepositReceipt for OpReceipt { _ => None, } } + + fn as_deposit_receipt(&self) -> Option<&OpDepositReceipt> { + match self { + Self::Deposit(receipt) => Some(receipt), + _ => None, + } + } } #[cfg(feature = "reth-codec")] @@ -572,17 +635,17 @@ pub(super) mod serde_bincode_compat { #[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] struct Data { #[serde_as(as = "serde_bincode_compat::OpReceipt<'_>")] - reseipt: OpReceipt, + receipt: OpReceipt, } let mut bytes = [0u8; 1024]; rand::rng().fill(bytes.as_mut_slice()); let mut data = Data { - reseipt: OpReceipt::arbitrary(&mut arbitrary::Unstructured::new(&bytes)).unwrap(), + receipt: OpReceipt::arbitrary(&mut arbitrary::Unstructured::new(&bytes)).unwrap(), }; - let success = data.reseipt.as_receipt_mut().status.coerce_status(); + let success = data.receipt.as_receipt_mut().status.coerce_status(); // // ensure we don't have an invalid poststate variant - data.reseipt.as_receipt_mut().status = success.into(); + data.receipt.as_receipt_mut().status = success.into(); let encoded = bincode::serialize(&data).unwrap(); let decoded: Data = bincode::deserialize(&encoded).unwrap(); diff --git a/crates/optimism/primitives/src/transaction/mod.rs b/crates/optimism/primitives/src/transaction/mod.rs index 230d8a6a3ea..306f5459046 100644 --- a/crates/optimism/primitives/src/transaction/mod.rs +++ b/crates/optimism/primitives/src/transaction/mod.rs @@ -2,24 +2,11 @@ mod tx_type; -/// Kept for concistency tests +/// Kept for consistency tests #[cfg(test)] mod signed; -pub use op_alloy_consensus::{OpTxType, OpTypedTransaction}; +pub use op_alloy_consensus::{OpTransaction, OpTxType, OpTypedTransaction}; /// Signed transaction. pub type OpTransactionSigned = op_alloy_consensus::OpTxEnvelope; - -/// A trait that represents an optimism transaction, mainly used to indicate whether or not the -/// transaction is a deposit transaction. -pub trait OpTransaction { - /// Whether or not the transaction is a dpeosit transaction. - fn is_deposit(&self) -> bool; -} - -impl OpTransaction for op_alloy_consensus::OpTxEnvelope { - fn is_deposit(&self) -> bool { - Self::is_deposit(self) - } -} diff --git a/crates/optimism/primitives/src/transaction/signed.rs b/crates/optimism/primitives/src/transaction/signed.rs index 88db839b037..820cc112710 100644 --- a/crates/optimism/primitives/src/transaction/signed.rs +++ b/crates/optimism/primitives/src/transaction/signed.rs @@ -4,7 +4,7 @@ use crate::transaction::OpTransaction; use alloc::vec::Vec; use alloy_consensus::{ - transaction::{RlpEcdsaDecodableTx, RlpEcdsaEncodableTx}, + transaction::{RlpEcdsaDecodableTx, RlpEcdsaEncodableTx, SignerRecoverable, TxHashRef}, Sealed, SignableTransaction, Signed, Transaction, TxEip1559, TxEip2930, TxEip7702, TxLegacy, Typed2718, }; @@ -103,11 +103,7 @@ impl OpTransactionSigned { } } -impl SignedTransaction for OpTransactionSigned { - fn tx_hash(&self) -> &TxHash { - self.hash.get_or_init(|| self.recalculate_hash()) - } - +impl SignerRecoverable for OpTransactionSigned { fn recover_signer(&self) -> Result { // Optimism's Deposit transaction does not have a signature. Directly return the // `from` address. @@ -132,10 +128,7 @@ impl SignedTransaction for OpTransactionSigned { recover_signer_unchecked(signature, signature_hash) } - fn recover_signer_unchecked_with_buf( - &self, - buf: &mut Vec, - ) -> Result { + fn recover_unchecked_with_buf(&self, buf: &mut Vec) -> Result { match &self.transaction { // Optimism's Deposit transaction does not have a signature. Directly return the // `from` address. @@ -147,7 +140,15 @@ impl SignedTransaction for OpTransactionSigned { }; recover_signer_unchecked(&self.signature, keccak256(buf)) } +} +impl TxHashRef for OpTransactionSigned { + fn tx_hash(&self) -> &TxHash { + self.hash.get_or_init(|| self.recalculate_hash()) + } +} + +impl SignedTransaction for OpTransactionSigned { fn recalculate_hash(&self) -> B256 { keccak256(self.encoded_2718()) } @@ -503,15 +504,6 @@ impl<'a> arbitrary::Arbitrary<'a> for OpTransactionSigned { ) .unwrap(); - // Both `Some(0)` and `None` values are encoded as empty string byte. This introduces - // ambiguity in roundtrip tests. Patch the mint value of deposit transaction here, so that - // it's `None` if zero. - if let OpTypedTransaction::Deposit(ref mut tx_deposit) = transaction { - if tx_deposit.mint == Some(0) { - tx_deposit.mint = None; - } - } - let signature = if transaction.is_deposit() { TxDeposit::signature() } else { signature }; Ok(Self::new_unhashed(transaction, signature)) diff --git a/crates/optimism/reth/Cargo.toml b/crates/optimism/reth/Cargo.toml index f4f8606114b..d120f04f614 100644 --- a/crates/optimism/reth/Cargo.toml +++ b/crates/optimism/reth/Cargo.toml @@ -19,6 +19,7 @@ reth-network-api = { workspace = true, optional = true } reth-eth-wire = { workspace = true, optional = true } reth-provider = { workspace = true, optional = true } reth-db = { workspace = true, optional = true, features = ["mdbx", "op"] } +reth-codecs = { workspace = true, optional = true } reth-storage-api = { workspace = true, optional = true } reth-node-api = { workspace = true, optional = true } reth-node-core = { workspace = true, optional = true } @@ -30,8 +31,14 @@ reth-rpc = { workspace = true, optional = true } reth-rpc-api = { workspace = true, optional = true } reth-rpc-eth-types = { workspace = true, optional = true } reth-rpc-builder = { workspace = true, optional = true } +reth-exex = { workspace = true, optional = true } reth-transaction-pool = { workspace = true, optional = true } reth-trie = { workspace = true, optional = true } +reth-trie-db = { workspace = true, optional = true } +reth-node-builder = { workspace = true, optional = true } +reth-tasks = { workspace = true, optional = true } +reth-cli-util = { workspace = true, optional = true } +reth-engine-local = { workspace = true, optional = true } # reth-op reth-optimism-primitives.workspace = true @@ -65,6 +72,7 @@ arbitrary = [ "reth-db?/arbitrary", "reth-transaction-pool?/arbitrary", "reth-eth-wire?/arbitrary", + "reth-codecs?/arbitrary", ] test-utils = [ @@ -79,38 +87,55 @@ test-utils = [ "reth-provider?/test-utils", "reth-trie?/test-utils", "reth-transaction-pool?/test-utils", + "reth-node-builder?/test-utils", + "reth-trie-db?/test-utils", + "reth-codecs?/test-utils", ] full = ["consensus", "evm", "node", "provider", "rpc", "trie", "pool", "network"] alloy-compat = ["reth-optimism-primitives/alloy-compat"] -cli = ["dep:reth-optimism-cli"] +cli = ["dep:reth-optimism-cli", "dep:reth-cli-util"] consensus = [ "dep:reth-consensus", "dep:reth-consensus-common", "dep:reth-optimism-consensus", ] evm = ["dep:reth-evm", "dep:reth-optimism-evm", "dep:reth-revm"] +exex = ["provider", "dep:reth-exex"] node-api = ["dep:reth-node-api", "dep:reth-node-core"] node = [ "provider", "consensus", "evm", + "network", "node-api", "dep:reth-optimism-node", + "dep:reth-node-builder", + "dep:reth-engine-local", "rpc", - "trie", + "trie-db", + "pool", ] rpc = [ + "tasks", "dep:reth-rpc", "dep:reth-rpc-builder", "dep:reth-rpc-api", "dep:reth-rpc-eth-types", "dep:reth-optimism-rpc", ] -js-tracer = ["rpc", "reth-rpc/js-tracer"] -network = ["dep:reth-network", "dep:reth-network-api", "dep:reth-eth-wire"] -provider = ["storage-api", "dep:reth-provider", "dep:reth-db"] +tasks = ["dep:reth-tasks"] +js-tracer = [ + "rpc", + "reth-rpc/js-tracer", + "reth-node-builder?/js-tracer", + "reth-optimism-node?/js-tracer", + "reth-rpc-eth-types?/js-tracer", +] +network = ["dep:reth-network", "tasks", "dep:reth-network-api", "dep:reth-eth-wire"] +provider = ["storage-api", "tasks", "dep:reth-provider", "dep:reth-db", "dep:reth-codecs"] pool = ["dep:reth-transaction-pool"] storage-api = ["dep:reth-storage-api"] trie = ["dep:reth-trie"] +trie-db = ["trie", "dep:reth-trie-db"] diff --git a/crates/optimism/reth/src/lib.rs b/crates/optimism/reth/src/lib.rs index f043322404f..eb00eb6576d 100644 --- a/crates/optimism/reth/src/lib.rs +++ b/crates/optimism/reth/src/lib.rs @@ -6,11 +6,11 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(feature = "std"), no_std)] #![allow(unused_crate_dependencies)] -/// Re-exported ethereum types +/// Re-exported optimism types #[doc(inline)] pub use reth_optimism_primitives::*; @@ -22,7 +22,16 @@ pub mod primitives { /// Re-exported cli types #[cfg(feature = "cli")] -pub use reth_optimism_cli as cli; +pub mod cli { + #[doc(inline)] + pub use reth_cli_util::{ + allocator, get_secret_key, hash_or_num_value_parser, load_secret_key, + parse_duration_from_secs, parse_duration_from_secs_or_ms, parse_ether_value, + parse_socket_address, sigsegv_handler, + }; + #[doc(inline)] + pub use reth_optimism_cli::*; +} /// Re-exported pool types #[cfg(feature = "pool")] @@ -43,6 +52,7 @@ pub mod consensus { } /// Re-exported from `reth_chainspec` +#[allow(ambiguous_glob_reexports)] pub mod chainspec { #[doc(inline)] pub use reth_chainspec::*; @@ -63,6 +73,16 @@ pub mod evm { pub use reth_revm as revm; } +/// Re-exported exex types +#[cfg(feature = "exex")] +pub use reth_exex as exex; + +/// Re-exported from `tasks`. +#[cfg(feature = "tasks")] +pub mod tasks { + pub use reth_tasks::*; +} + /// Re-exported reth network types #[cfg(feature = "network")] pub mod network { @@ -84,6 +104,10 @@ pub mod provider { pub use reth_db as db; } +/// Re-exported codec crate +#[cfg(feature = "provider")] +pub use reth_codecs as codec; + /// Re-exported reth storage api types #[cfg(feature = "storage-api")] pub mod storage { @@ -91,22 +115,37 @@ pub mod storage { pub use reth_storage_api::*; } -/// Re-exported ethereum node +/// Re-exported optimism node #[cfg(feature = "node-api")] pub mod node { #[doc(inline)] pub use reth_node_api as api; + #[cfg(feature = "node")] + pub use reth_node_builder as builder; #[doc(inline)] pub use reth_node_core as core; #[cfg(feature = "node")] pub use reth_optimism_node::*; } +/// Re-exported engine types +#[cfg(feature = "node")] +pub mod engine { + #[doc(inline)] + pub use reth_engine_local as local; + #[doc(inline)] + pub use reth_optimism_node::engine::*; +} + /// Re-exported reth trie types #[cfg(feature = "trie")] pub mod trie { #[doc(inline)] pub use reth_trie::*; + + #[cfg(feature = "trie-db")] + #[doc(inline)] + pub use reth_trie_db::*; } /// Re-exported rpc types diff --git a/crates/optimism/rpc/Cargo.toml b/crates/optimism/rpc/Cargo.toml index df2bd0be72b..2d2462a256e 100644 --- a/crates/optimism/rpc/Cargo.toml +++ b/crates/optimism/rpc/Cargo.toml @@ -14,10 +14,9 @@ workspace = true [dependencies] # reth reth-evm.workspace = true -reth-primitives-traits.workspace = true +reth-primitives-traits = { workspace = true, features = ["op"] } reth-storage-api.workspace = true -reth-chain-state.workspace = true -reth-rpc-eth-api.workspace = true +reth-rpc-eth-api = { workspace = true, features = ["op"] } reth-rpc-eth-types.workspace = true reth-rpc-server-types.workspace = true reth-tasks = { workspace = true, features = ["rayon"] } @@ -25,20 +24,20 @@ reth-transaction-pool.workspace = true reth-rpc.workspace = true reth-rpc-api.workspace = true reth-node-api.workspace = true -reth-network-api.workspace = true reth-node-builder.workspace = true reth-chainspec.workspace = true +reth-chain-state.workspace = true reth-rpc-engine-api.workspace = true # op-reth -reth-optimism-chainspec.workspace = true reth-optimism-evm.workspace = true +reth-optimism-flashblocks.workspace = true reth-optimism-payload-builder.workspace = true reth-optimism-txpool.workspace = true -reth-optimism-consensus.workspace = true # TODO remove node-builder import -reth-optimism-primitives = { workspace = true, features = ["reth-codec", "serde-bincode-compat"] } +reth-optimism-primitives = { workspace = true, features = ["reth-codec", "serde-bincode-compat", "serde"] } reth-optimism-forks.workspace = true +reth-mantle-forks.workspace = true # ethereum alloy-eips.workspace = true @@ -51,7 +50,6 @@ alloy-transport.workspace = true alloy-transport-http.workspace = true alloy-consensus.workspace = true alloy-rpc-types-engine.workspace = true -alloy-signer.workspace = true op-alloy-network.workspace = true op-alloy-rpc-types.workspace = true op-alloy-rpc-types-engine.workspace = true @@ -61,15 +59,18 @@ revm.workspace = true op-revm.workspace = true # async -parking_lot.workspace = true tokio.workspace = true +futures.workspace = true +tokio-stream.workspace = true reqwest = { workspace = true, features = ["rustls-tls-native-roots"] } async-trait.workspace = true +tower.workspace = true # rpc jsonrpsee-core.workspace = true jsonrpsee-types.workspace = true jsonrpsee.workspace = true +serde_json.workspace = true # misc eyre.workspace = true @@ -77,8 +78,13 @@ thiserror.workspace = true tracing.workspace = true derive_more = { workspace = true, features = ["constructor"] } +# metrics +reth-metrics.workspace = true +metrics.workspace = true + [dev-dependencies] reth-optimism-chainspec.workspace = true +alloy-op-hardforks.workspace = true [features] client = [ diff --git a/crates/optimism/rpc/src/engine.rs b/crates/optimism/rpc/src/engine.rs index ae986628700..2d706eb0e91 100644 --- a/crates/optimism/rpc/src/engine.rs +++ b/crates/optimism/rpc/src/engine.rs @@ -14,12 +14,12 @@ use op_alloy_rpc_types_engine::{ ProtocolVersion, ProtocolVersionFormatV0, SuperchainSignal, }; use reth_chainspec::EthereumHardforks; -use reth_node_api::{EngineTypes, EngineValidator}; +use reth_node_api::{EngineApiValidator, EngineTypes}; use reth_rpc_api::IntoEngineApiRpcModule; use reth_rpc_engine_api::EngineApi; use reth_storage_api::{BlockReader, HeaderProvider, StateProviderFactory}; use reth_transaction_pool::TransactionPool; -use tracing::{info, trace}; +use tracing::{debug, info, trace}; /// The list of all supported Engine capabilities available over the engine endpoint. /// @@ -225,7 +225,7 @@ pub trait OpEngineApi { /// Returns the execution payload bodies by the range starting at `start`, containing `count` /// blocks. /// - /// WARNING: This method is associated with the BeaconBlocksByRange message in the consensus + /// WARNING: This method is associated with the `BeaconBlocksByRange` message in the consensus /// layer p2p specification, meaning the input should be treated as untrusted or potentially /// adversarial. /// @@ -273,6 +273,16 @@ pub struct OpEngineApi, } +impl Clone + for OpEngineApi +where + PayloadT: EngineTypes, +{ + fn clone(&self) -> Self { + Self { inner: self.inner.clone() } + } +} + #[async_trait::async_trait] impl OpEngineApiServer for OpEngineApi @@ -280,7 +290,7 @@ where Provider: HeaderProvider + BlockReader + StateProviderFactory + 'static, EngineT: EngineTypes, Pool: TransactionPool + 'static, - Validator: EngineValidator, + Validator: EngineApiValidator, ChainSpec: EthereumHardforks + Send + Sync + 'static, { /// Handler for `engine_newPayloadV1` @@ -369,7 +379,7 @@ where &self, payload_id: PayloadId, ) -> RpcResult { - trace!(target: "rpc::engine", "Serving engine_getPayloadV2"); + debug!(target: "rpc::engine", id = %payload_id, "Serving engine_getPayloadV2"); Ok(self.inner.get_payload_v2_metered(payload_id).await?) } diff --git a/crates/optimism/rpc/src/error.rs b/crates/optimism/rpc/src/error.rs index 795c6652adc..3d37cc7b7a8 100644 --- a/crates/optimism/rpc/src/error.rs +++ b/crates/optimism/rpc/src/error.rs @@ -1,15 +1,17 @@ //! RPC errors specific to OP. +use alloy_json_rpc::ErrorPayload; use alloy_rpc_types_eth::{error::EthRpcErrorCode, BlockError}; use alloy_transport::{RpcError, TransportErrorKind}; use jsonrpsee_types::error::{INTERNAL_ERROR_CODE, INVALID_PARAMS_CODE}; use op_revm::{OpHaltReason, OpTransactionError}; +use reth_evm::execute::ProviderError; use reth_optimism_evm::OpBlockExecutionError; -use reth_rpc_eth_api::AsEthApiError; +use reth_rpc_eth_api::{AsEthApiError, EthTxEnvError, TransactionConversionError}; use reth_rpc_eth_types::{error::api::FromEvmHalt, EthApiError}; use reth_rpc_server_types::result::{internal_rpc_err, rpc_err}; use revm::context_interface::result::{EVMError, InvalidTransaction}; -use std::fmt::Display; +use std::{convert::Infallible, fmt::Display}; /// Optimism specific errors, that extend [`EthApiError`]. #[derive(Debug, thiserror::Error)] @@ -65,6 +67,9 @@ pub enum OpInvalidTransactionError { /// A deposit transaction halted post-regolith #[error("deposit transaction halted after regolith")] HaltedDepositPostRegolith, + /// The encoded transaction was missing during evm execution. + #[error("missing enveloped transaction bytes")] + MissingEnvelopedTx, /// Transaction conditional errors. #[error(transparent)] TxConditionalErr(#[from] TxConditionalErr), @@ -74,7 +79,8 @@ impl From for jsonrpsee_types::error::ErrorObject<'st fn from(err: OpInvalidTransactionError) -> Self { match err { OpInvalidTransactionError::DepositSystemTxPostRegolith | - OpInvalidTransactionError::HaltedDepositPostRegolith => { + OpInvalidTransactionError::HaltedDepositPostRegolith | + OpInvalidTransactionError::MissingEnvelopedTx => { rpc_err(EthRpcErrorCode::TransactionRejected.code(), err.to_string(), None) } OpInvalidTransactionError::TxConditionalErr(_) => err.into(), @@ -91,6 +97,7 @@ impl TryFrom for OpInvalidTransactionError { Ok(Self::DepositSystemTxPostRegolith) } OpTransactionError::HaltedDepositPostRegolith => Ok(Self::HaltedDepositPostRegolith), + OpTransactionError::MissingEnvelopedTx => Ok(Self::MissingEnvelopedTx), OpTransactionError::Base(err) => Err(err), OpTransactionError::BvmEth(_) => todo!(), } @@ -141,24 +148,22 @@ pub enum SequencerClientError { /// Wrapper around an [`RpcError`]. #[error(transparent)] HttpError(#[from] RpcError), - /// Thrown when serializing transaction to forward to sequencer - #[error("invalid sequencer transaction")] - InvalidSequencerTransaction, } impl From for jsonrpsee_types::error::ErrorObject<'static> { fn from(err: SequencerClientError) -> Self { - jsonrpsee_types::error::ErrorObject::owned( - INTERNAL_ERROR_CODE, - err.to_string(), - None::, - ) - } -} - -impl From for OpEthApiError { - fn from(error: BlockError) -> Self { - Self::Eth(error.into()) + match err { + SequencerClientError::HttpError(RpcError::ErrorResp(ErrorPayload { + code, + message, + data, + })) => jsonrpsee_types::error::ErrorObject::owned(code as i32, message, data), + err => jsonrpsee_types::error::ErrorObject::owned( + INTERNAL_ERROR_CODE, + err.to_string(), + None::, + ), + } } } @@ -189,3 +194,33 @@ impl FromEvmHalt for OpEthApiError { } } } + +impl From for OpEthApiError { + fn from(value: TransactionConversionError) -> Self { + Self::Eth(EthApiError::from(value)) + } +} + +impl From for OpEthApiError { + fn from(value: EthTxEnvError) -> Self { + Self::Eth(EthApiError::from(value)) + } +} + +impl From for OpEthApiError { + fn from(value: ProviderError) -> Self { + Self::Eth(EthApiError::from(value)) + } +} + +impl From for OpEthApiError { + fn from(value: BlockError) -> Self { + Self::Eth(EthApiError::from(value)) + } +} + +impl From for OpEthApiError { + fn from(value: Infallible) -> Self { + match value {} + } +} diff --git a/crates/optimism/rpc/src/eth/block.rs b/crates/optimism/rpc/src/eth/block.rs index 67211e9d531..0efd9aea988 100644 --- a/crates/optimism/rpc/src/eth/block.rs +++ b/crates/optimism/rpc/src/eth/block.rs @@ -1,94 +1,23 @@ //! Loads and formats OP block RPC response. -use alloy_consensus::{transaction::TransactionMeta, BlockHeader}; -use alloy_rpc_types_eth::BlockId; -use op_alloy_rpc_types::OpTransactionReceipt; -use reth_chainspec::ChainSpecProvider; -use reth_node_api::BlockBody; -use reth_optimism_chainspec::OpChainSpec; -use reth_optimism_primitives::{OpReceipt, OpTransactionSigned}; +use crate::{eth::RpcNodeCore, OpEthApi, OpEthApiError}; use reth_rpc_eth_api::{ - helpers::{EthBlocks, LoadBlock, LoadPendingBlock, LoadReceipt, SpawnBlocking}, - types::RpcTypes, - RpcReceipt, + helpers::{EthBlocks, LoadBlock}, + FromEvmError, RpcConvert, }; -use reth_storage_api::{BlockReader, HeaderProvider, ProviderTx}; -use reth_transaction_pool::{PoolTransaction, TransactionPool}; -use crate::{eth::OpNodeCore, OpEthApi, OpEthApiError, OpReceiptBuilder}; - -impl EthBlocks for OpEthApi +impl EthBlocks for OpEthApi where - Self: LoadBlock< - Error = OpEthApiError, - NetworkTypes: RpcTypes, - Provider: BlockReader, - >, - N: OpNodeCore + HeaderProvider>, + N: RpcNodeCore, + OpEthApiError: FromEvmError, + Rpc: RpcConvert, { - async fn block_receipts( - &self, - block_id: BlockId, - ) -> Result>>, Self::Error> - where - Self: LoadReceipt, - { - if let Some((block, receipts)) = self.load_block_and_receipts(block_id).await? { - let block_number = block.number(); - let base_fee = block.base_fee_per_gas(); - let block_hash = block.hash(); - let excess_blob_gas = block.excess_blob_gas(); - let timestamp = block.timestamp(); - - let mut l1_block_info = reth_optimism_evm::extract_l1_info(block.body())?; - - return block - .body() - .transactions() - .iter() - .zip(receipts.iter()) - .enumerate() - .map(|(idx, (tx, receipt))| -> Result<_, _> { - let meta = TransactionMeta { - tx_hash: tx.tx_hash(), - index: idx as u64, - block_hash, - block_number, - base_fee, - excess_blob_gas, - timestamp, - }; - - // We must clear this cache as different L2 transactions can have different - // L1 costs. A potential improvement here is to only clear the cache if the - // new transaction input has changed, since otherwise the L1 cost wouldn't. - l1_block_info.clear_tx_l1_cost(); - - Ok(OpReceiptBuilder::new( - &self.inner.eth_api.provider().chain_spec(), - tx, - meta, - receipt, - &receipts, - &mut l1_block_info, - )? - .build()) - }) - .collect::, Self::Error>>() - .map(Some) - } - - Ok(None) - } } -impl LoadBlock for OpEthApi +impl LoadBlock for OpEthApi where - Self: LoadPendingBlock< - Pool: TransactionPool< - Transaction: PoolTransaction>, - >, - > + SpawnBlocking, - N: OpNodeCore, + N: RpcNodeCore, + OpEthApiError: FromEvmError, + Rpc: RpcConvert, { } diff --git a/crates/optimism/rpc/src/eth/call.rs b/crates/optimism/rpc/src/eth/call.rs index 01cccf16048..db96bda83f3 100644 --- a/crates/optimism/rpc/src/eth/call.rs +++ b/crates/optimism/rpc/src/eth/call.rs @@ -1,51 +1,30 @@ -use super::OpNodeCore; -use crate::{OpEthApi, OpEthApiError}; -use alloy_consensus::TxType; -use alloy_primitives::{Bytes, TxKind, U256}; -use alloy_rpc_types_eth::transaction::TransactionRequest; -use alloy_signer::Either; -use op_revm::OpTransaction; -use reth_evm::{execute::BlockExecutorFactory, ConfigureEvm, EvmEnv, EvmFactory, SpecFor}; -use reth_node_api::NodePrimitives; +use crate::{eth::RpcNodeCore, OpEthApi, OpEthApiError}; use reth_rpc_eth_api::{ - helpers::{estimate::EstimateCall, Call, EthCall, LoadBlock, LoadState, SpawnBlocking}, - FromEthApiError, FromEvmError, FullEthApiTypes, IntoEthApiError, + helpers::{estimate::EstimateCall, Call, EthCall}, + FromEvmError, RpcConvert, }; -use reth_rpc_eth_types::{revm_utils::CallFees, EthApiError, RpcInvalidTransactionError}; -use reth_storage_api::{ProviderHeader, ProviderTx}; -use revm::{context::TxEnv, context_interface::Block, Database}; -impl EthCall for OpEthApi +impl EthCall for OpEthApi where - Self: EstimateCall + LoadBlock + FullEthApiTypes, - N: OpNodeCore, + N: RpcNodeCore, + OpEthApiError: FromEvmError, + Rpc: RpcConvert, { } -impl EstimateCall for OpEthApi +impl EstimateCall for OpEthApi where - Self: Call, - Self::Error: From, - N: OpNodeCore, + N: RpcNodeCore, + OpEthApiError: FromEvmError, + Rpc: RpcConvert, { } -impl Call for OpEthApi +impl Call for OpEthApi where - Self: LoadState< - Evm: ConfigureEvm< - Primitives: NodePrimitives< - BlockHeader = ProviderHeader, - SignedTx = ProviderTx, - >, - BlockExecutorFactory: BlockExecutorFactory< - EvmFactory: EvmFactory>, - >, - >, - Error: FromEvmError, - > + SpawnBlocking, - Self::Error: From, - N: OpNodeCore, + N: RpcNodeCore, + OpEthApiError: FromEvmError, + Rpc: RpcConvert, { #[inline] fn call_gas_limit(&self) -> u64 { @@ -57,106 +36,8 @@ where self.inner.eth_api.max_simulate_blocks() } - fn create_txn_env( - &self, - evm_env: &EvmEnv>, - request: TransactionRequest, - mut db: impl Database>, - ) -> Result, Self::Error> { - // Ensure that if versioned hashes are set, they're not empty - if request.blob_versioned_hashes.as_ref().is_some_and(|hashes| hashes.is_empty()) { - return Err(RpcInvalidTransactionError::BlobTransactionMissingBlobHashes.into_eth_err()); - } - - let tx_type = if request.authorization_list.is_some() { - TxType::Eip7702 - } else if request.sidecar.is_some() || request.max_fee_per_blob_gas.is_some() { - TxType::Eip4844 - } else if request.max_fee_per_gas.is_some() || request.max_priority_fee_per_gas.is_some() { - TxType::Eip1559 - } else if request.access_list.is_some() { - TxType::Eip2930 - } else { - TxType::Legacy - } as u8; - - let TransactionRequest { - from, - to, - gas_price, - max_fee_per_gas, - max_priority_fee_per_gas, - gas, - value, - input, - nonce, - access_list, - chain_id, - blob_versioned_hashes, - max_fee_per_blob_gas, - authorization_list, - transaction_type: _, - sidecar: _, - } = request; - - let CallFees { max_priority_fee_per_gas, gas_price, max_fee_per_blob_gas } = - CallFees::ensure_fees( - gas_price.map(U256::from), - max_fee_per_gas.map(U256::from), - max_priority_fee_per_gas.map(U256::from), - U256::from(evm_env.block_env.basefee), - blob_versioned_hashes.as_deref(), - max_fee_per_blob_gas.map(U256::from), - evm_env.block_env.blob_gasprice().map(U256::from), - )?; - - let gas_limit = gas.unwrap_or( - // Use maximum allowed gas limit. The reason for this - // is that both Erigon and Geth use pre-configured gas cap even if - // it's possible to derive the gas limit from the block: - // - evm_env.block_env.gas_limit, - ); - - let chain_id = chain_id.unwrap_or(evm_env.cfg_env.chain_id); - - let caller = from.unwrap_or_default(); - - let nonce = if let Some(nonce) = nonce { - nonce - } else { - db.basic(caller).map_err(Into::into)?.map(|acc| acc.nonce).unwrap_or_default() - }; - - let base = TxEnv { - tx_type, - gas_limit, - nonce, - caller, - gas_price: gas_price.saturating_to(), - gas_priority_fee: max_priority_fee_per_gas.map(|v| v.saturating_to()), - kind: to.unwrap_or(TxKind::Create), - value: value.unwrap_or_default(), - data: input - .try_into_unique_input() - .map_err(Self::Error::from_eth_err)? - .unwrap_or_default(), - chain_id: Some(chain_id), - access_list: access_list.unwrap_or_default(), - // EIP-4844 fields - blob_hashes: blob_versioned_hashes.unwrap_or_default(), - max_fee_per_blob_gas: max_fee_per_blob_gas - .map(|v| v.saturating_to()) - .unwrap_or_default(), - // EIP-7702 fields - authorization_list: authorization_list - .unwrap_or_default() - .into_iter() - .map(Either::Left) - .collect(), - }; - - Ok(OpTransaction { base, enveloped_tx: Some(Bytes::new()), deposit: Default::default() }) + #[inline] + fn evm_memory_limit(&self) -> u64 { + self.inner.eth_api.evm_memory_limit() } } diff --git a/crates/optimism/rpc/src/eth/ext.rs b/crates/optimism/rpc/src/eth/ext.rs index 46008d0608b..6c4e1bc7cf1 100644 --- a/crates/optimism/rpc/src/eth/ext.rs +++ b/crates/optimism/rpc/src/eth/ext.rs @@ -10,7 +10,9 @@ use reth_optimism_txpool::conditional::MaybeConditionalTransaction; use reth_rpc_eth_api::L2EthApiExtServer; use reth_rpc_eth_types::utils::recover_raw_transaction; use reth_storage_api::{BlockReaderIdExt, StateProviderFactory}; -use reth_transaction_pool::{PoolTransaction, TransactionOrigin, TransactionPool}; +use reth_transaction_pool::{ + AddedTransactionOutcome, PoolTransaction, TransactionOrigin, TransactionPool, +}; use std::sync::Arc; use tokio::sync::Semaphore; @@ -157,7 +159,7 @@ where } else { // otherwise, add to pool with the appended conditional tx.set_conditional(condition); - let hash = + let AddedTransactionOutcome { hash, .. } = self.pool().add_transaction(TransactionOrigin::Private, tx).await.map_err(|e| { OpEthApiError::Eth(reth_rpc_eth_types::EthApiError::PoolError(e.into())) })?; diff --git a/crates/optimism/rpc/src/eth/fee.rs b/crates/optimism/rpc/src/eth/fee.rs deleted file mode 100644 index f5d16c09528..00000000000 --- a/crates/optimism/rpc/src/eth/fee.rs +++ /dev/null @@ -1,430 +0,0 @@ -use super::{OpEthApi, OpNodeCore}; -use alloy_consensus::{BlockHeader, Transaction, TxReceipt}; -use alloy_eips::{eip7840::BlobParams, BlockNumberOrTag}; -use alloy_primitives::Sealable; -use alloy_primitives::U256; -use alloy_rpc_types_eth::FeeHistory; -use reth_chainspec::{ChainSpecProvider, EthChainSpec, EthereumHardforks}; -use reth_optimism_consensus::next_block_base_fee as optimism_next_block_base_fee; -use reth_primitives_traits::{Block, BlockBody}; -use reth_rpc_eth_api::{ - helpers::{EthFees, LoadBlock, LoadFee}, - FromEthApiError, -}; -use reth_rpc_eth_types::{ - fee_history::calculate_reward_percentiles_for_block, EthApiError, FeeHistoryCache, - GasPriceOracle, -}; -use reth_rpc_server_types::constants::gas_oracle::DEFAULT_MIN_SUGGESTED_PRIORITY_FEE; -use reth_storage_api::{ - BlockIdReader, BlockReader, BlockReaderIdExt, HeaderProvider, ReceiptProvider, - StateProviderFactory, -}; -use tracing::debug; - -impl LoadFee for OpEthApi -where - Self: LoadBlock, - N: OpNodeCore< - Provider: BlockReaderIdExt - + ChainSpecProvider< - ChainSpec: EthChainSpec + EthereumHardforks + reth_optimism_forks::OpHardforks, - > + StateProviderFactory, - >, -{ - #[inline] - fn gas_oracle(&self) -> &GasPriceOracle { - self.inner.eth_api.gas_oracle() - } - - #[inline] - fn fee_history_cache(&self) -> &FeeHistoryCache { - self.inner.eth_api.fee_history_cache() - } - - /// Optimism-specific priority fee suggestion - async fn suggested_priority_fee(&self) -> Result - where - Self: 'static, - { - // Delegate to the Optimism-specific implementation that mirrors op-geth's SuggestOptimismPriorityFee - self.suggest_optimism_priority_fee().await - } -} - -impl EthFees for OpEthApi -where - Self: LoadFee, - N: OpNodeCore< - Provider: BlockReaderIdExt - + ChainSpecProvider< - ChainSpec: EthChainSpec + EthereumHardforks + reth_optimism_forks::OpHardforks, - > + StateProviderFactory, - >, -{ - /// Reports the fee history, for the given amount of blocks, up until the given newest block. - /// - /// If `reward_percentiles` are provided the [`FeeHistory`] will include the _approximated_ - /// rewards for the requested range. - async fn fee_history( - &self, - mut block_count: u64, - mut newest_block: BlockNumberOrTag, - reward_percentiles: Option>, - ) -> Result { - if block_count == 0 { - return Ok(FeeHistory::default()); - } - - // ensure the given reward percentiles aren't excessive - if reward_percentiles.as_ref().map(|perc| perc.len() as u64) - > Some(self.gas_oracle().config().max_reward_percentile_count) - { - return Err(EthApiError::InvalidRewardPercentiles.into()); - } - - // See https://github.com/ethereum/go-ethereum/blob/2754b197c935ee63101cbbca2752338246384fec/eth/gasprice/feehistory.go#L218C8-L225 - let max_fee_history = if reward_percentiles.is_none() { - self.gas_oracle().config().max_header_history - } else { - self.gas_oracle().config().max_block_history - }; - - if block_count > max_fee_history { - debug!( - requested = block_count, - truncated = max_fee_history, - "Sanitizing fee history block count" - ); - block_count = max_fee_history - } - - if newest_block.is_pending() { - // cap the target block since we don't have fee history for the pending block - newest_block = BlockNumberOrTag::Latest; - // account for missing pending block - block_count = block_count.saturating_sub(1); - } - - let end_block = self - .inner - .eth_api - .provider() - .block_number_for_id(newest_block.into()) - .map_err(Self::Error::from_eth_err)? - .ok_or(EthApiError::HeaderNotFound(newest_block.into()))?; - - // need to add 1 to the end block to get the correct (inclusive) range - let end_block_plus = end_block + 1; - // Ensure that we would not be querying outside of genesis - if end_block_plus < block_count { - block_count = end_block_plus; - } - - // If reward percentiles were specified, we - // need to validate that they are monotonically - // increasing and 0 <= p <= 100 - // Note: The types used ensure that the percentiles are never < 0 - if let Some(percentiles) = &reward_percentiles { - if percentiles.windows(2).any(|w| w[0] > w[1] || w[0] > 100.) { - return Err(EthApiError::InvalidRewardPercentiles.into()); - } - } - - // Fetch the headers and ensure we got all of them - // - // Treat a request for 1 block as a request for `newest_block..=newest_block`, - // otherwise `newest_block - 2` - // NOTE: We ensured that block count is capped - let start_block = end_block_plus - block_count; - - // Collect base fees, gas usage ratios and (optionally) reward percentile data - let mut base_fee_per_gas: Vec = Vec::new(); - let mut gas_used_ratio: Vec = Vec::new(); - - let mut base_fee_per_blob_gas: Vec = Vec::new(); - let mut blob_gas_used_ratio: Vec = Vec::new(); - - let mut rewards: Vec> = Vec::new(); - - // Check if the requested range is within the cache bounds - let fee_entries = self.fee_history_cache().get_history(start_block, end_block).await; - - if let Some(fee_entries) = fee_entries { - if fee_entries.len() != block_count as usize { - return Err(EthApiError::InvalidBlockRange.into()); - } - - for entry in &fee_entries { - base_fee_per_gas.push(entry.base_fee_per_gas as u128); - gas_used_ratio.push(entry.gas_used_ratio); - base_fee_per_blob_gas.push(entry.base_fee_per_blob_gas.unwrap_or_default()); - blob_gas_used_ratio.push(entry.blob_gas_used_ratio); - - if let Some(percentiles) = &reward_percentiles { - let mut block_rewards = Vec::with_capacity(percentiles.len()); - for &percentile in percentiles { - block_rewards.push(self.approximate_percentile(entry, percentile)); - } - rewards.push(block_rewards); - } - } - let last_entry = fee_entries.last().expect("is not empty"); - - // Also need to include the `base_fee_per_gas` and `base_fee_per_blob_gas` for the - // next block - // Use Optimism-specific base fee calculation for consistency - // We need to get the full header from cache to use Optimism-specific calculation - let last_header = self - .inner - .eth_api - .cache() - .get_header(last_entry.header_hash) - .await - .map_err(Self::Error::from_eth_err)?; - - let next_base_fee = self - .calculate_optimism_next_block_base_fee(&last_header, last_entry.timestamp) - .unwrap_or_default() as u128; - base_fee_per_gas.push(next_base_fee); - - base_fee_per_blob_gas.push(last_entry.next_block_blob_fee().unwrap_or_default()); - } else { - // read the requested header range - let headers = self - .inner - .eth_api - .provider() - .headers_range(start_block..=end_block) - .map_err(Self::Error::from_eth_err)?; - if headers.len() != block_count as usize { - return Err(EthApiError::InvalidBlockRange.into()); - } - - for header in &headers { - base_fee_per_gas.push(header.base_fee_per_gas().unwrap_or_default() as u128); - gas_used_ratio.push(header.gas_used() as f64 / header.gas_limit() as f64); - - let blob_params = self - .inner - .eth_api - .provider() - .chain_spec() - .blob_params_at_timestamp(header.timestamp()) - .unwrap_or_else(BlobParams::cancun); - - base_fee_per_blob_gas.push(header.blob_fee(blob_params).unwrap_or_default()); - blob_gas_used_ratio.push( - header.blob_gas_used().unwrap_or_default() as f64 - / blob_params.max_blob_gas_per_block() as f64, - ); - - // Percentiles were specified, so we need to collect reward percentile info - if let Some(percentiles) = &reward_percentiles { - let (block, receipts) = self - .inner - .eth_api - .cache() - .get_block_and_receipts(header.hash_slow()) - .await - .map_err(Self::Error::from_eth_err)? - .ok_or(EthApiError::InvalidBlockRange)?; - rewards.push( - calculate_reward_percentiles_for_block( - percentiles, - header.gas_used(), - header.base_fee_per_gas().unwrap_or_default(), - block.body().transactions(), - &receipts, - ) - .unwrap_or_default(), - ); - } - } - - // The spec states that `base_fee_per_gas` "[..] includes the next block after the - // newest of the returned range, because this value can be derived from the - // newest block" - // - // The unwrap is safe since we checked earlier that we got at least 1 header. - let last_header = headers.last().expect("is present"); - let next_base_fee = self - .calculate_optimism_next_block_base_fee(last_header, last_header.timestamp()) - .unwrap_or_default() as u128; - base_fee_per_gas.push(next_base_fee); - - // Same goes for the `base_fee_per_blob_gas`: - // > "[..] includes the next block after the newest of the returned range, because this value can be derived from the newest block. - base_fee_per_blob_gas.push( - last_header - .maybe_next_block_blob_fee( - self.inner - .eth_api - .provider() - .chain_spec() - .blob_params_at_timestamp(last_header.timestamp()), - ) - .unwrap_or_default(), - ); - }; - - Ok(FeeHistory { - base_fee_per_gas, - gas_used_ratio, - base_fee_per_blob_gas, - blob_gas_used_ratio, - oldest_block: start_block, - reward: reward_percentiles.map(|_| rewards), - }) - } -} - -impl OpEthApi -where - N: OpNodeCore< - Provider: BlockReaderIdExt - + ChainSpecProvider< - ChainSpec: EthChainSpec + EthereumHardforks + reth_optimism_forks::OpHardforks, - > + StateProviderFactory, - >, -{ - /// Calculate the next block base fee using Optimism-specific logic - /// - /// This function handles the Optimism-specific base fee calculation that considers: - /// 1. Holocene hardfork activation and `extra_data` decoding - /// 2. Optimism-specific base fee parameters - /// 3. Fallback to standard Ethereum base fee calculation when appropriate - fn calculate_optimism_next_block_base_fee( - &self, - header: &impl BlockHeader, - timestamp: u64, - ) -> Result::Error> { - // Use Optimism-specific next_block_base_fee calculation - // This handles Holocene hardfork logic and extra_data decoding - match optimism_next_block_base_fee( - self.inner.eth_api.provider().chain_spec(), - header, - timestamp, - ) { - Ok(base_fee) => Ok(base_fee), - Err(e) => { - tracing::warn!("Failed to calculate Optimism next block base fee: {:?}", e); - // Fallback to standard calculation if Optimism-specific calculation fails - Ok(header - .next_block_base_fee( - self.inner - .eth_api - .provider() - .chain_spec() - .base_fee_params_at_timestamp(timestamp), - ) - .unwrap_or_default()) - } - } - } - /// Optimism-specific gas price suggestion algorithm - /// - /// This implements the same algorithm as op-geth's `SuggestOptimismPriorityFee`: - /// 1. Start with minimum suggested priority fee from config (default 0.0001 gwei) - /// 2. Check if the last block is at capacity - /// 3. If at capacity, return median + 10% of previous block's effective tips - /// 4. Otherwise, return the minimum suggestion - async fn suggest_optimism_priority_fee( - &self, - ) -> Result::Error> { - let min_suggestion = self - .inner - .eth_api - .gas_oracle() - .config() - .min_suggested_priority_fee - .unwrap_or(DEFAULT_MIN_SUGGESTED_PRIORITY_FEE); - - // Fetch the latest block header - let header = self - .inner - .eth_api - .provider() - .latest_header() - .map_err(::Error::from_eth_err)? - .ok_or(EthApiError::HeaderNotFound(BlockNumberOrTag::Latest.into()))?; - - // Check if block is at capacity - let receipts = self - .inner - .eth_api - .provider() - .receipts_by_block(alloy_eips::HashOrNumber::Hash(header.hash())) - .map_err(::Error::from_eth_err)?; - - // Calculate suggestion based on receipts, min suggestion if None - let suggestion = receipts - .and_then(|receipts| { - // Calculate max gas usage per transaction - let max_tx_gas = receipts - .windows(2) - .map(|w| w[1].cumulative_gas_used() - w[0].cumulative_gas_used()) - .chain(receipts.first().map(|r| r.cumulative_gas_used()).into_iter()) - .max() - .unwrap_or(0); - - // Sanity check the max gas used value - if max_tx_gas > header.gas_limit() { - tracing::error!( - "found tx consuming more gas than the block limit: {}", - max_tx_gas - ); - return None; - } - - // Check if block is at capacity, if not, return None - if header.gas_used() + max_tx_gas <= header.gas_limit() { - return None; - } - - tracing::debug!("Block is at capacity, calculating median + 10%"); - - // Get block for tip calculation - let block = match self - .inner - .eth_api - .provider() - .block_by_hash(header.hash()) - .map_err(::Error::from_eth_err) - { - Ok(Some(block)) => block, - Ok(None) | Err(_) => return None, - }; - - let base_fee = block.header().base_fee_per_gas().unwrap_or_default(); - let mut tips: Vec = block - .body() - .transactions_iter() - .filter_map(|tx| tx.effective_tip_per_gas(base_fee).map(U256::from)) - .collect(); - - if tips.is_empty() { - tracing::error!("block was at capacity but doesn't have transactions"); - None - } else { - tips.sort_unstable(); - let median = tips[tips.len() / 2]; - let new_suggestion = median + median / U256::from(10); - Some(new_suggestion.max(min_suggestion)) - } - }) - .unwrap_or(min_suggestion); - - // The suggestion should be capped by oracle.maxPrice - let final_suggestion = self - .inner - .eth_api - .gas_oracle() - .config() - .max_price - .map(|max_price| suggestion.min(max_price)) - .unwrap_or(suggestion); - - Ok(final_suggestion) - } -} diff --git a/crates/optimism/rpc/src/eth/mantle_ext.rs b/crates/optimism/rpc/src/eth/mantle_ext.rs index 99dead2c798..3f2ef698768 100644 --- a/crates/optimism/rpc/src/eth/mantle_ext.rs +++ b/crates/optimism/rpc/src/eth/mantle_ext.rs @@ -4,7 +4,8 @@ use alloy_eips::BlockNumberOrTag; use alloy_primitives::Bytes; use jsonrpsee::types::ErrorObject; use jsonrpsee_core::RpcResult; -use reth_rpc_eth_api::{EthApiServer, EthApiTypes, MantleEthApiExtServer, PreconfTxEvent, RpcBlock}; +use reth_primitives_traits::TxTy; +use reth_rpc_eth_api::{EthApiServer, EthApiTypes, MantleEthApiExtServer, PreconfTxEvent, RpcBlock, RpcNodeCore}; use reth_rpc_server_types::result::invalid_params_rpc_err; use reth_storage_api::{BlockNumReader, BlockReaderIdExt, StateProviderFactory}; use std::sync::Arc; @@ -56,12 +57,15 @@ impl MantleEthApiExtServer> for MantleEthApiExt where Provider: BlockReaderIdExt + BlockNumReader + StateProviderFactory + Clone + 'static, - EthApi: EthApiTypes + EthApi: RpcNodeCore + + EthApiTypes + EthApiServer< + reth_rpc_eth_api::RpcTxReq, reth_rpc_eth_api::RpcTransaction, RpcBlock, reth_rpc_eth_api::RpcReceipt, reth_rpc_eth_api::RpcHeader, + TxTy, > + Send + Sync + 'static, @@ -147,14 +151,12 @@ where .await .map_err(|err| { // Extract the original error message from the sequencer response - let error_msg = match &err { - SequencerClientError::HttpError(rpc_err) => { - rpc_err.as_error_resp() - .map(|payload| payload.message.to_string()) - .unwrap_or_else(|| err.to_string()) - } - _ => err.to_string(), - }; + // SequencerClientError only has one variant (HttpError), so we can directly destructure + let SequencerClientError::HttpError(rpc_err) = &err; + let error_msg = rpc_err + .as_error_resp() + .map(|payload| payload.message.to_string()) + .unwrap_or_else(|| err.to_string()); ErrorObject::owned( -32000, diff --git a/crates/optimism/rpc/src/eth/mod.rs b/crates/optimism/rpc/src/eth/mod.rs index 4585bce46bd..6c0983157b8 100644 --- a/crates/optimism/rpc/src/eth/mod.rs +++ b/crates/optimism/rpc/src/eth/mod.rs @@ -7,53 +7,55 @@ pub mod transaction; mod block; mod call; -mod fee; mod pending_block; -use alloy_primitives::U256; +use crate::{ + eth::{receipt::OpReceiptConverter, transaction::OpTxInfoMapper}, + OpEthApiError, SequencerClient, +}; +use alloy_consensus::BlockHeader; +use alloy_primitives::{B256, U256}; use eyre::WrapErr; use op_alloy_network::Optimism; pub use receipt::{OpReceiptBuilder, OpReceiptFieldsBuilder}; -use reth_chain_state::CanonStateSubscriptions; -use reth_chainspec::{ChainSpecProvider, EthereumHardforks}; +use reqwest::Url; +use reth_chainspec::{EthereumHardforks, Hardforks}; use reth_evm::ConfigureEvm; -use reth_network_api::NetworkInfo; -use reth_node_api::{FullNodeComponents, NodePrimitives}; +use reth_node_api::{FullNodeComponents, FullNodeTypes, HeaderTy, NodeTypes}; use reth_node_builder::rpc::{EthApiBuilder, EthApiCtx}; -use reth_optimism_primitives::OpPrimitives; -use reth_rpc::eth::{core::EthApiInner, DevSigner}; +use reth_optimism_flashblocks::{ + ExecutionPayloadBaseV1, FlashBlockBuildInfo, FlashBlockCompleteSequenceRx, FlashBlockRx, + FlashBlockService, FlashblocksListeners, PendingBlockRx, PendingFlashBlock, WsFlashBlockStream, +}; +use reth_rpc::eth::core::EthApiInner; use reth_rpc_eth_api::{ helpers::{ - AddDevSigners, EthApiSpec, EthSigner, EthState, LoadState, - SpawnBlocking, Trace, + pending_block::BuildPendingEnv, EthApiSpec, EthFees, EthState, LoadFee, LoadPendingBlock, + LoadState, SpawnBlocking, Trace, }, - EthApiTypes, FromEvmError, FullEthApiServer, RpcNodeCore, RpcNodeCoreExt, -}; -use reth_rpc_eth_types::EthStateCache; -use reth_storage_api::{ - BlockNumReader, BlockReader, BlockReaderIdExt, ProviderBlock, ProviderHeader, ProviderReceipt, - ProviderTx, StageCheckpointReader, StateProviderFactory, + EthApiTypes, FromEvmError, FullEthApiServer, RpcConvert, RpcConverter, RpcNodeCore, + RpcNodeCoreExt, RpcTypes, }; +use reth_rpc_eth_types::{EthStateCache, FeeHistoryCache, GasPriceOracle, PendingBlock}; +use reth_storage_api::{BlockReaderIdExt, ProviderHeader}; use reth_tasks::{ pool::{BlockingTaskGuard, BlockingTaskPool}, TaskSpawner, }; -use reth_transaction_pool::TransactionPool; -use std::{fmt, sync::Arc}; +use std::{ + fmt::{self, Formatter}, + marker::PhantomData, + sync::Arc, + time::Duration, +}; +use tokio::{sync::watch, time}; +use tracing::info; -use crate::{OpEthApiError, SequencerClient}; +/// Maximum duration to wait for a fresh flashblock when one is being built. +const MAX_FLASHBLOCK_WAIT_DURATION: Duration = Duration::from_millis(50); /// Adapter for [`EthApiInner`], which holds all the data required to serve core `eth_` API. -pub type EthApiNodeBackend = EthApiInner< - ::Provider, - ::Pool, - ::Network, - ::Evm, ->; - -/// A helper trait with requirements for [`RpcNodeCore`] to be used in [`OpEthApi`]. -pub trait OpNodeCore: RpcNodeCore {} -impl OpNodeCore for T where T: RpcNodeCore {} +pub type EthApiNodeBackend = EthApiInner; /// OP-Reth `Eth` API implementation. /// @@ -65,62 +67,147 @@ impl OpNodeCore for T where T: RpcNodeCore {} /// /// This type implements the [`FullEthApi`](reth_rpc_eth_api::helpers::FullEthApi) by implemented /// all the `Eth` helper traits and prerequisite traits. -#[derive(Clone)] -pub struct OpEthApi { +pub struct OpEthApi { /// Gateway to node's core components. - inner: Arc>, + inner: Arc>, } -impl OpEthApi -where - N: OpNodeCore< - Provider: BlockReaderIdExt - + ChainSpecProvider - + CanonStateSubscriptions - + Clone - + 'static, - >, -{ +impl Clone for OpEthApi { + fn clone(&self) -> Self { + Self { inner: self.inner.clone() } + } +} + +impl OpEthApi { + /// Creates a new `OpEthApi`. + pub fn new( + eth_api: EthApiNodeBackend, + sequencer_client: Option, + min_suggested_priority_fee: U256, + flashblocks: Option>, + ) -> Self { + let inner = Arc::new(OpEthApiInner { + eth_api, + sequencer_client, + min_suggested_priority_fee, + flashblocks, + }); + Self { inner } + } + + /// Build a [`OpEthApi`] using [`OpEthApiBuilder`]. + pub const fn builder() -> OpEthApiBuilder { + OpEthApiBuilder::new() + } + /// Returns a reference to the [`EthApiNodeBackend`]. - pub fn eth_api(&self) -> &EthApiNodeBackend { + pub fn eth_api(&self) -> &EthApiNodeBackend { self.inner.eth_api() } - /// Returns the configured sequencer client, if any. pub fn sequencer_client(&self) -> Option<&SequencerClient> { self.inner.sequencer_client() } - /// Build a [`OpEthApi`] using [`OpEthApiBuilder`]. - pub const fn builder() -> OpEthApiBuilder { - OpEthApiBuilder::new() + /// Returns a cloned pending block receiver, if any. + pub fn pending_block_rx(&self) -> Option> { + self.inner.flashblocks.as_ref().map(|f| f.pending_block_rx.clone()) + } + + /// Returns a new subscription to received flashblocks. + pub fn subscribe_received_flashblocks(&self) -> Option { + self.inner.flashblocks.as_ref().map(|f| f.received_flashblocks.subscribe()) + } + + /// Returns a new subscription to flashblock sequences. + pub fn subscribe_flashblock_sequence(&self) -> Option { + self.inner.flashblocks.as_ref().map(|f| f.flashblocks_sequence.subscribe()) + } + + /// Returns information about the flashblock currently being built, if any. + fn flashblock_build_info(&self) -> Option { + self.inner.flashblocks.as_ref().and_then(|f| *f.in_progress_rx.borrow()) + } + + /// Extracts pending block if it matches the expected parent hash. + fn extract_matching_block( + &self, + block: Option<&PendingFlashBlock>, + parent_hash: B256, + ) -> Option> { + block.filter(|b| b.block().parent_hash() == parent_hash).map(|b| b.pending.clone()) + } + + /// Awaits a fresh flashblock if one is being built, otherwise returns current. + async fn flashblock( + &self, + parent_hash: B256, + ) -> eyre::Result>> { + let Some(rx) = self.inner.flashblocks.as_ref().map(|f| &f.pending_block_rx) else { + return Ok(None) + }; + + // Check if a flashblock is being built + if let Some(build_info) = self.flashblock_build_info() { + let current_index = rx.borrow().as_ref().map(|b| b.last_flashblock_index); + + // Check if this is the first flashblock or the next consecutive index + let is_next_index = current_index.is_none_or(|idx| build_info.index == idx + 1); + + // Wait only for relevant flashblocks: matching parent and next in sequence + if build_info.parent_hash == parent_hash && is_next_index { + let mut rx_clone = rx.clone(); + // Wait up to MAX_FLASHBLOCK_WAIT_DURATION for a new flashblock to arrive + let _ = time::timeout(MAX_FLASHBLOCK_WAIT_DURATION, rx_clone.changed()).await; + } + } + + // Fall back to current block + Ok(self.extract_matching_block(rx.borrow().as_ref(), parent_hash)) + } + + /// Returns a [`PendingBlock`] that is built out of flashblocks. + /// + /// If flashblocks receiver is not set, then it always returns `None`. + /// + /// It may wait up to 50ms for a fresh flashblock if one is currently being built. + pub async fn pending_flashblock(&self) -> eyre::Result>> + where + OpEthApiError: FromEvmError, + Rpc: RpcConvert, + { + let Some(latest) = self.provider().latest_header()? else { + return Ok(None); + }; + + self.flashblock(latest.hash()).await } } -impl EthApiTypes for OpEthApi +impl EthApiTypes for OpEthApi where - Self: Send + Sync, - N: OpNodeCore, + N: RpcNodeCore, + Rpc: RpcConvert, { type Error = OpEthApiError; - type NetworkTypes = Optimism; - type TransactionCompat = Self; + type NetworkTypes = Rpc::Network; + type RpcConvert = Rpc; - fn tx_resp_builder(&self) -> &Self::TransactionCompat { - self + fn tx_resp_builder(&self) -> &Self::RpcConvert { + self.inner.eth_api.tx_resp_builder() } } -impl RpcNodeCore for OpEthApi +impl RpcNodeCore for OpEthApi where - N: OpNodeCore, + N: RpcNodeCore, + Rpc: RpcConvert, { - type Primitives = OpPrimitives; + type Primitives = N::Primitives; type Provider = N::Provider; type Pool = N::Pool; - type Evm = ::Evm; - type Network = ::Network; - type PayloadBuilder = (); + type Evm = N::Evm; + type Network = N::Network; #[inline] fn pool(&self) -> &Self::Pool { @@ -137,53 +224,38 @@ where self.inner.eth_api.network() } - #[inline] - fn payload_builder(&self) -> &Self::PayloadBuilder { - &() - } - #[inline] fn provider(&self) -> &Self::Provider { self.inner.eth_api.provider() } } -impl RpcNodeCoreExt for OpEthApi +impl RpcNodeCoreExt for OpEthApi where - N: OpNodeCore, + N: RpcNodeCore, + Rpc: RpcConvert, { #[inline] - fn cache(&self) -> &EthStateCache, ProviderReceipt> { + fn cache(&self) -> &EthStateCache { self.inner.eth_api.cache() } } -impl EthApiSpec for OpEthApi +impl EthApiSpec for OpEthApi where - N: OpNodeCore< - Provider: ChainSpecProvider - + BlockNumReader - + StageCheckpointReader, - Network: NetworkInfo, - >, + N: RpcNodeCore, + Rpc: RpcConvert, { - type Transaction = ProviderTx; - #[inline] fn starting_block(&self) -> U256 { self.inner.eth_api.starting_block() } - - #[inline] - fn signers(&self) -> &parking_lot::RwLock>>>> { - self.inner.eth_api.signers() - } } -impl SpawnBlocking for OpEthApi +impl SpawnBlocking for OpEthApi where - Self: Send + Sync + Clone + 'static, - N: OpNodeCore, + N: RpcNodeCore, + Rpc: RpcConvert, { #[inline] fn io_task_spawner(&self) -> impl TaskSpawner { @@ -201,18 +273,45 @@ where } } -impl LoadState for OpEthApi where - N: OpNodeCore< - Provider: StateProviderFactory + ChainSpecProvider, - Pool: TransactionPool, - > +impl LoadFee for OpEthApi +where + N: RpcNodeCore, + OpEthApiError: FromEvmError, + Rpc: RpcConvert, +{ + #[inline] + fn gas_oracle(&self) -> &GasPriceOracle { + self.inner.eth_api.gas_oracle() + } + + #[inline] + fn fee_history_cache(&self) -> &FeeHistoryCache> { + self.inner.eth_api.fee_history_cache() + } + + async fn suggested_priority_fee(&self) -> Result { + self.inner + .eth_api + .gas_oracle() + .op_suggest_tip_cap(self.inner.min_suggested_priority_fee) + .await + .map_err(Into::into) + } +} + +impl LoadState for OpEthApi +where + N: RpcNodeCore, + Rpc: RpcConvert, + Self: LoadPendingBlock, { } -impl EthState for OpEthApi +impl EthState for OpEthApi where - Self: LoadState + SpawnBlocking, - N: OpNodeCore, + N: RpcNodeCore, + Rpc: RpcConvert, + Self: LoadPendingBlock, { #[inline] fn max_proof_window(&self) -> u64 { @@ -220,50 +319,54 @@ where } } - -impl Trace for OpEthApi +impl EthFees for OpEthApi where - Self: RpcNodeCore - + LoadState< - Evm: ConfigureEvm< - Primitives: NodePrimitives< - BlockHeader = ProviderHeader, - SignedTx = ProviderTx, - >, - >, - Error: FromEvmError, - >, - N: OpNodeCore, + N: RpcNodeCore, + OpEthApiError: FromEvmError, + Rpc: RpcConvert, { } -impl AddDevSigners for OpEthApi +impl Trace for OpEthApi where - N: OpNodeCore, + N: RpcNodeCore, + OpEthApiError: FromEvmError, + Rpc: RpcConvert, { - fn with_dev_accounts(&self) { - *self.inner.eth_api.signers().write() = DevSigner::random_signers(20) - } } -impl fmt::Debug for OpEthApi { +impl fmt::Debug for OpEthApi { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("OpEthApi").finish_non_exhaustive() } } /// Container type `OpEthApi` -struct OpEthApiInner { +pub struct OpEthApiInner { /// Gateway to node's core components. - eth_api: EthApiNodeBackend, + eth_api: EthApiNodeBackend, /// Sequencer client, configured to forward submitted transactions to sequencer of given OP /// network. sequencer_client: Option, + /// Minimum priority fee enforced by OP-specific logic. + /// + /// See also + min_suggested_priority_fee: U256, + /// Flashblocks listeners. + /// + /// If set, provides receivers for pending blocks, flashblock sequences, and build status. + flashblocks: Option>, +} + +impl fmt::Debug for OpEthApiInner { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.debug_struct("OpEthApiInner").finish() + } } -impl OpEthApiInner { +impl OpEthApiInner { /// Returns a reference to the [`EthApiNodeBackend`]. - const fn eth_api(&self) -> &EthApiNodeBackend { + const fn eth_api(&self) -> &EthApiNodeBackend { &self.eth_api } @@ -273,18 +376,55 @@ impl OpEthApiInner { } } +/// Converter for OP RPC types. +pub type OpRpcConvert = RpcConverter< + NetworkT, + ::Evm, + OpReceiptConverter<::Provider>, + (), + OpTxInfoMapper<::Provider>, +>; + /// Builds [`OpEthApi`] for Optimism. -#[derive(Debug, Default)] -pub struct OpEthApiBuilder { +#[derive(Debug)] +pub struct OpEthApiBuilder { /// Sequencer client, configured to forward submitted transactions to sequencer of given OP /// network. sequencer_url: Option, + /// Headers to use for the sequencer client requests. + sequencer_headers: Vec, + /// Minimum suggested priority fee (tip) + min_suggested_priority_fee: u64, + /// A URL pointing to a secure websocket connection (wss) that streams out [flashblocks]. + /// + /// [flashblocks]: reth_optimism_flashblocks + flashblocks_url: Option, + /// Marker for network types. + _nt: PhantomData, +} + +impl Default for OpEthApiBuilder { + fn default() -> Self { + Self { + sequencer_url: None, + sequencer_headers: Vec::new(), + min_suggested_priority_fee: 1_000_000, + flashblocks_url: None, + _nt: PhantomData, + } + } } -impl OpEthApiBuilder { +impl OpEthApiBuilder { /// Creates a [`OpEthApiBuilder`] instance from core components. pub const fn new() -> Self { - Self { sequencer_url: None } + Self { + sequencer_url: None, + sequencer_headers: Vec::new(), + min_suggested_priority_fee: 1_000_000, + flashblocks_url: None, + _nt: PhantomData, + } } /// With a [`SequencerClient`]. @@ -292,43 +432,100 @@ impl OpEthApiBuilder { self.sequencer_url = sequencer_url; self } + + /// With headers to use for the sequencer client requests. + pub fn with_sequencer_headers(mut self, sequencer_headers: Vec) -> Self { + self.sequencer_headers = sequencer_headers; + self + } + + /// With minimum suggested priority fee (tip). + pub const fn with_min_suggested_priority_fee(mut self, min: u64) -> Self { + self.min_suggested_priority_fee = min; + self + } + + /// With a subscription to flashblocks secure websocket connection. + pub fn with_flashblocks(mut self, flashblocks_url: Option) -> Self { + self.flashblocks_url = flashblocks_url; + self + } } -impl EthApiBuilder for OpEthApiBuilder +impl EthApiBuilder for OpEthApiBuilder where - N: FullNodeComponents, - OpEthApi: FullEthApiServer, + N: FullNodeComponents< + Evm: ConfigureEvm< + NextBlockEnvCtx: BuildPendingEnv> + + From + + Unpin, + >, + Types: NodeTypes, + >, + NetworkT: RpcTypes, + OpRpcConvert: RpcConvert, + OpEthApi>: + FullEthApiServer, { - type EthApi = OpEthApi; + type EthApi = OpEthApi>; async fn build_eth_api(self, ctx: EthApiCtx<'_, N>) -> eyre::Result { - let Self { sequencer_url } = self; - let eth_api = reth_rpc::EthApiBuilder::new( - ctx.components.provider().clone(), - ctx.components.pool().clone(), - ctx.components.network().clone(), - ctx.components.evm_config().clone(), - ) - .eth_cache(ctx.cache) - .task_spawner(ctx.components.task_executor().clone()) - .gas_cap(ctx.config.rpc_gas_cap.into()) - .max_simulate_blocks(ctx.config.rpc_max_simulate_blocks) - .eth_proof_window(ctx.config.eth_proof_window) - .fee_history_cache_config(ctx.config.fee_history_cache) - .proof_permits(ctx.config.proof_permits) - .gas_oracle_config(ctx.config.gas_oracle) - .build_inner(); + let Self { + sequencer_url, + sequencer_headers, + min_suggested_priority_fee, + flashblocks_url, + .. + } = self; + let rpc_converter = + RpcConverter::new(OpReceiptConverter::new(ctx.components.provider().clone())) + .with_mapper(OpTxInfoMapper::new(ctx.components.provider().clone())); let sequencer_client = if let Some(url) = sequencer_url { Some( - SequencerClient::new(&url) + SequencerClient::new_with_headers(&url, sequencer_headers) .await - .wrap_err_with(|| "Failed to init sequencer client with: {url}")?, + .wrap_err_with(|| format!("Failed to init sequencer client with: {url}"))?, ) } else { None }; - Ok(OpEthApi { inner: Arc::new(OpEthApiInner { eth_api, sequencer_client }) }) + let flashblocks = if let Some(ws_url) = flashblocks_url { + info!(target: "reth:cli", %ws_url, "Launching flashblocks service"); + + let (tx, pending_rx) = watch::channel(None); + let stream = WsFlashBlockStream::new(ws_url); + let service = FlashBlockService::new( + stream, + ctx.components.evm_config().clone(), + ctx.components.provider().clone(), + ctx.components.task_executor().clone(), + ); + + let flashblocks_sequence = service.block_sequence_broadcaster().clone(); + let received_flashblocks = service.flashblocks_broadcaster().clone(); + let in_progress_rx = service.subscribe_in_progress(); + + ctx.components.task_executor().spawn(Box::pin(service.run(tx))); + + Some(FlashblocksListeners::new( + pending_rx, + flashblocks_sequence, + in_progress_rx, + received_flashblocks, + )) + } else { + None + }; + + let eth_api = ctx.eth_api_builder().with_rpc_converter(rpc_converter).build_inner(); + + Ok(OpEthApi::new( + eth_api, + sequencer_client, + U256::from(min_suggested_priority_fee), + flashblocks, + )) } } diff --git a/crates/optimism/rpc/src/eth/pending_block.rs b/crates/optimism/rpc/src/eth/pending_block.rs index b0c13f14b1f..88bf2496592 100644 --- a/crates/optimism/rpc/src/eth/pending_block.rs +++ b/crates/optimism/rpc/src/eth/pending_block.rs @@ -1,109 +1,79 @@ //! Loads OP pending block for a RPC response. -use crate::OpEthApi; +use crate::{OpEthApi, OpEthApiError}; use alloy_consensus::BlockHeader; use alloy_eips::BlockNumberOrTag; -use alloy_primitives::B256; -use reth_chainspec::{ChainSpecProvider, EthChainSpec}; -use reth_evm::ConfigureEvm; -use reth_node_api::NodePrimitives; -use reth_optimism_evm::OpNextBlockEnvAttributes; -use reth_optimism_forks::OpHardforks; -use reth_optimism_primitives::{OpBlock, OpReceipt, OpTransactionSigned}; -use reth_primitives_traits::{RecoveredBlock, SealedHeader}; +use reth_chain_state::BlockState; use reth_rpc_eth_api::{ - helpers::{LoadPendingBlock, SpawnBlocking}, - types::RpcTypes, - EthApiTypes, FromEthApiError, FromEvmError, RpcNodeCore, + helpers::{pending_block::PendingEnvBuilder, LoadPendingBlock, SpawnBlocking}, + FromEvmError, RpcConvert, RpcNodeCore, RpcNodeCoreExt, }; -use reth_rpc_eth_types::{EthApiError, PendingBlock}; -use reth_storage_api::{ - BlockReader, BlockReaderIdExt, ProviderBlock, ProviderHeader, ProviderReceipt, ProviderTx, - ReceiptProvider, StateProviderFactory, +use reth_rpc_eth_types::{ + block::BlockAndReceipts, builder::config::PendingBlockKind, error::FromEthApiError, + EthApiError, PendingBlock, }; -use reth_transaction_pool::{PoolTransaction, TransactionPool}; +use reth_storage_api::{BlockReaderIdExt, StateProviderBox, StateProviderFactory}; -impl LoadPendingBlock for OpEthApi +impl LoadPendingBlock for OpEthApi where - Self: SpawnBlocking - + EthApiTypes< - NetworkTypes: RpcTypes< - Header = alloy_rpc_types_eth::Header>, - >, - Error: FromEvmError, - >, - N: RpcNodeCore< - Provider: BlockReaderIdExt< - Transaction = OpTransactionSigned, - Block = OpBlock, - Receipt = OpReceipt, - Header = alloy_consensus::Header, - > + ChainSpecProvider - + StateProviderFactory, - Pool: TransactionPool>>, - Evm: ConfigureEvm< - Primitives: NodePrimitives< - SignedTx = ProviderTx, - BlockHeader = ProviderHeader, - Receipt = ProviderReceipt, - Block = ProviderBlock, - >, - NextBlockEnvCtx = OpNextBlockEnvAttributes, - >, - >, + N: RpcNodeCore, + OpEthApiError: FromEvmError, + Rpc: RpcConvert, { #[inline] - fn pending_block( - &self, - ) -> &tokio::sync::Mutex< - Option, ProviderReceipt>>, - > { + fn pending_block(&self) -> &tokio::sync::Mutex>> { self.inner.eth_api.pending_block() } - fn next_env_attributes( - &self, - parent: &SealedHeader>, - ) -> Result<::NextBlockEnvCtx, Self::Error> { - Ok(OpNextBlockEnvAttributes { - timestamp: parent.timestamp().saturating_add(12), - suggested_fee_recipient: parent.beneficiary(), - prev_randao: B256::random(), - gas_limit: parent.gas_limit(), - parent_beacon_block_root: parent.parent_beacon_block_root(), - extra_data: parent.extra_data.clone(), - }) + #[inline] + fn pending_env_builder(&self) -> &dyn PendingEnvBuilder { + self.inner.eth_api.pending_env_builder() + } + + #[inline] + fn pending_block_kind(&self) -> PendingBlockKind { + self.inner.eth_api.pending_block_kind() + } + + /// Returns a [`StateProviderBox`] on a mem-pool built pending block overlaying latest. + async fn local_pending_state(&self) -> Result, Self::Error> + where + Self: SpawnBlocking, + { + let Ok(Some(pending_block)) = self.pending_flashblock().await else { + return Ok(None); + }; + + let latest_historical = self + .provider() + .history_by_block_hash(pending_block.block().parent_hash()) + .map_err(Self::Error::from_eth_err)?; + + let state = BlockState::from(pending_block); + + Ok(Some(Box::new(state.state_provider(latest_historical)) as StateProviderBox)) } /// Returns the locally built pending block async fn local_pending_block( &self, - ) -> Result< - Option<( - RecoveredBlock>, - Vec>, - )>, - Self::Error, - > { + ) -> Result>, Self::Error> { + if let Ok(Some(pending)) = self.pending_flashblock().await { + return Ok(Some(pending.into_block_and_receipts())); + } + // See: let latest = self .provider() - .latest_header() - .map_err(Self::Error::from_eth_err)? + .latest_header()? .ok_or(EthApiError::HeaderNotFound(BlockNumberOrTag::Latest.into()))?; - let block_id = latest.hash().into(); - let block = self - .provider() - .recovered_block(block_id, Default::default()) - .map_err(Self::Error::from_eth_err)? - .ok_or(EthApiError::HeaderNotFound(block_id.into()))?; - let receipts = self - .provider() - .receipts_by_block(block_id) + let latest = self + .cache() + .get_block_and_receipts(latest.hash()) + .await .map_err(Self::Error::from_eth_err)? - .ok_or(EthApiError::ReceiptsNotFound(block_id.into()))?; - - Ok(Some((block, receipts))) + .map(|(block, receipts)| BlockAndReceipts { block, receipts }); + Ok(latest) } } diff --git a/crates/optimism/rpc/src/eth/receipt.rs b/crates/optimism/rpc/src/eth/receipt.rs index 7156af5dcef..81ec062ee39 100644 --- a/crates/optimism/rpc/src/eth/receipt.rs +++ b/crates/optimism/rpc/src/eth/receipt.rs @@ -1,64 +1,114 @@ //! Loads and formats OP receipt RPC response. -use crate::{OpEthApi, OpEthApiError}; -use alloy_consensus::transaction::TransactionMeta; +use crate::{eth::RpcNodeCore, OpEthApi, OpEthApiError}; +use alloy_consensus::{BlockHeader, Receipt, TxReceipt}; use alloy_eips::eip2718::Encodable2718; use alloy_rpc_types_eth::{Log, TransactionReceipt}; -use op_alloy_consensus::{OpDepositReceipt, OpDepositReceiptWithBloom, OpReceiptEnvelope}; +use op_alloy_consensus::{OpReceiptEnvelope, OpTransaction}; use op_alloy_rpc_types::{L1BlockInfo, OpTransactionReceipt, OpTransactionReceiptFields}; use op_revm::constants::{GAS_ORACLE_CONTRACT, TOKEN_RATIO_SLOT}; +use op_revm::estimate_tx_compressed_size; use reth_chainspec::ChainSpecProvider; -use reth_node_api::{FullNodeComponents, NodeTypes}; -use reth_optimism_chainspec::OpChainSpec; +use reth_node_api::NodePrimitives; use reth_optimism_evm::RethL1BlockInfo; -use reth_optimism_forks::OpHardforks; -use reth_optimism_primitives::{OpReceipt, OpTransactionSigned}; -use reth_rpc_eth_api::{helpers::LoadReceipt, FromEthApiError, RpcReceipt}; +use reth_mantle_forks::MantleHardforks; +use reth_optimism_primitives::OpReceipt; +use reth_primitives_traits::SealedBlock; +use reth_rpc_eth_api::{ + helpers::LoadReceipt, + transaction::{ConvertReceiptInput, ReceiptConverter}, + RpcConvert, +}; use reth_rpc_eth_types::{receipt::build_receipt, EthApiError}; -use reth_storage_api::{ReceiptProvider, StateProviderFactory, TransactionsProvider}; +use reth_storage_api::{BlockReader, StateProviderFactory}; +use std::fmt::Debug; -impl LoadReceipt for OpEthApi +impl LoadReceipt for OpEthApi where - Self: Send + Sync, - N: FullNodeComponents>, - Self::Provider: TransactionsProvider - + ReceiptProvider, + N: RpcNodeCore, + Rpc: RpcConvert, { - async fn build_transaction_receipt( +} + +/// Converter for OP receipts. +#[derive(Debug, Clone)] +pub struct OpReceiptConverter { + provider: Provider, +} + +impl OpReceiptConverter { + /// Creates a new [`OpReceiptConverter`]. + pub const fn new(provider: Provider) -> Self { + Self { provider } + } +} + +impl ReceiptConverter for OpReceiptConverter +where + N: NodePrimitives, + Provider: BlockReader + + ChainSpecProvider + + StateProviderFactory + + Debug + + 'static, +{ + type RpcReceipt = OpTransactionReceipt; + type Error = OpEthApiError; + + fn convert_receipts( &self, - tx: OpTransactionSigned, - meta: TransactionMeta, - receipt: OpReceipt, - ) -> Result, Self::Error> { - let (block, receipts) = self - .inner - .eth_api - .cache() - .get_block_and_receipts(meta.block_hash) - .await - .map_err(Self::Error::from_eth_err)? - .ok_or(Self::Error::from_eth_err(EthApiError::HeaderNotFound( - meta.block_hash.into(), - )))?; + inputs: Vec>, + ) -> Result, Self::Error> { + let Some(block_number) = inputs.first().map(|r| r.meta.block_number) else { + return Ok(Vec::new()); + }; - let mut l1_block_info = - reth_optimism_evm::extract_l1_info(block.body()).map_err(OpEthApiError::from)?; + let block = self + .provider + .block_by_number(block_number)? + .ok_or(EthApiError::HeaderNotFound(block_number.into()))?; + + self.convert_receipts_with_block(inputs, &SealedBlock::new_unhashed(block)) + } + + fn convert_receipts_with_block( + &self, + inputs: Vec>, + block: &SealedBlock, + ) -> Result, Self::Error> { + let mut l1_block_info = match reth_optimism_evm::extract_l1_info(block.body()) { + Ok(l1_block_info) => l1_block_info, + Err(err) => { + // If it is the genesis block (i.e. block number is 0), there is no L1 info, so + // we return an empty l1_block_info. + if block.header().number() == 0 { + return Ok(vec![]); + } + return Err(err.into()); + } + }; // [TODO] It's a temporary solution to get token ratio from state, we should modify the // receipt - let state = self.inner.eth_api.provider().state_by_block_hash(meta.block_hash).unwrap(); + let state = self.provider.state_by_block_hash(block.hash()).unwrap(); let token_ratio = state.storage(GAS_ORACLE_CONTRACT, TOKEN_RATIO_SLOT.into()).unwrap(); l1_block_info.token_ratio = token_ratio; - Ok(OpReceiptBuilder::new( - &self.inner.eth_api.provider().chain_spec(), - &tx, - meta, - &receipt, - &receipts, - &mut l1_block_info, - )? - .build()) + let mut receipts = Vec::with_capacity(inputs.len()); + + for input in inputs { + // We must clear this cache as different L2 transactions can have different + // L1 costs. A potential improvement here is to only clear the cache if the + // new transaction input has changed, since otherwise the L1 cost wouldn't. + l1_block_info.clear_tx_l1_cost(); + + receipts.push( + OpReceiptBuilder::new(&self.provider.chain_spec(), input, &mut l1_block_info)? + .build(), + ); + } + + Ok(receipts) } } @@ -85,6 +135,24 @@ pub struct OpReceiptFieldsBuilder { /* --------------------------------------- Mantle ---------------------------------------- */ /// The token ratio. pub token_ratio: Option, + /* ---------------------------------------- Canyon ----------------------------------------- */ + /// Deposit receipt version, if this is a deposit transaction. + pub deposit_receipt_version: Option, + /* ---------------------------------------- Ecotone ---------------------------------------- */ + /// The current L1 fee scalar. + pub l1_base_fee_scalar: Option, + /// The current L1 blob base fee. + pub l1_blob_base_fee: Option, + /// The current L1 blob base fee scalar. + pub l1_blob_base_fee_scalar: Option, + /* ---------------------------------------- Isthmus ---------------------------------------- */ + /// The current operator fee scalar. + pub operator_fee_scalar: Option, + /// The current L1 blob base fee scalar. + pub operator_fee_constant: Option, + /* ---------------------------------------- Jovian ----------------------------------------- */ + /// The current DA footprint gas scalar. + pub da_footprint_gas_scalar: Option, } impl OpReceiptFieldsBuilder { @@ -99,14 +167,21 @@ impl OpReceiptFieldsBuilder { l1_base_fee: None, deposit_nonce: None, token_ratio: None, + deposit_receipt_version: None, + l1_base_fee_scalar: None, + l1_blob_base_fee: None, + l1_blob_base_fee_scalar: None, + operator_fee_scalar: None, + operator_fee_constant: None, + da_footprint_gas_scalar: None, } } /// Applies [`L1BlockInfo`](op_revm::L1BlockInfo). - pub fn l1_block_info( + pub fn l1_block_info( mut self, - chain_spec: &OpChainSpec, - tx: &OpTransactionSigned, + chain_spec: &impl MantleHardforks, + tx: &T, l1_block_info: &mut op_revm::L1BlockInfo, ) -> Result { let raw_tx = tx.encoded_2718(); @@ -150,8 +225,11 @@ impl OpReceiptFieldsBuilder { // l1_block_info.operator_fee_constant.map(|constant| constant.saturating_to()); // } + // self.da_footprint_gas_scalar = l1_block_info.da_footprint_gas_scalar; self.token_ratio = l1_block_info.token_ratio.map(|ratio| ratio.saturating_to()); + self.da_footprint_gas_scalar = l1_block_info.da_footprint_gas_scalar; + Ok(self) } @@ -177,13 +255,14 @@ impl OpReceiptFieldsBuilder { l1_fee_scalar, l1_base_fee: l1_gas_price, deposit_nonce, - // deposit_receipt_version, - // l1_base_fee_scalar, - // l1_blob_base_fee, - // l1_blob_base_fee_scalar, - // operator_fee_scalar, - // operator_fee_constant, token_ratio, + deposit_receipt_version, + l1_base_fee_scalar, + l1_blob_base_fee, + l1_blob_base_fee_scalar, + operator_fee_scalar, + operator_fee_constant, + da_footprint_gas_scalar, } = self; OpTransactionReceiptFields { @@ -192,15 +271,16 @@ impl OpReceiptFieldsBuilder { l1_gas_used, l1_fee, l1_fee_scalar, - l1_base_fee_scalar: None, - l1_blob_base_fee: None, - l1_blob_base_fee_scalar: None, - operator_fee_scalar: None, - operator_fee_constant: None, + l1_base_fee_scalar, + l1_blob_base_fee, + l1_blob_base_fee_scalar, + operator_fee_scalar, + operator_fee_constant, token_ratio, + da_footprint_gas_scalar, }, deposit_nonce, - deposit_receipt_version: None, + deposit_receipt_version, } } } @@ -216,44 +296,66 @@ pub struct OpReceiptBuilder { impl OpReceiptBuilder { /// Returns a new builder. - pub fn new( - chain_spec: &OpChainSpec, - transaction: &OpTransactionSigned, - meta: TransactionMeta, - receipt: &OpReceipt, - all_receipts: &[OpReceipt], + pub fn new( + chain_spec: &impl MantleHardforks, + input: ConvertReceiptInput<'_, N>, l1_block_info: &mut op_revm::L1BlockInfo, - ) -> Result { - let timestamp = meta.timestamp; - let block_number = meta.block_number; - let core_receipt = - build_receipt(transaction, meta, receipt, all_receipts, None, |receipt_with_bloom| { - match receipt { - OpReceipt::Legacy(_) => OpReceiptEnvelope::::Legacy(receipt_with_bloom), - OpReceipt::Eip2930(_) => OpReceiptEnvelope::::Eip2930(receipt_with_bloom), - OpReceipt::Eip1559(_) => OpReceiptEnvelope::::Eip1559(receipt_with_bloom), - OpReceipt::Eip7702(_) => OpReceiptEnvelope::::Eip7702(receipt_with_bloom), - OpReceipt::Deposit(receipt) => { - OpReceiptEnvelope::::Deposit(OpDepositReceiptWithBloom:: { - receipt: OpDepositReceipt:: { - inner: receipt_with_bloom.receipt, - deposit_nonce: receipt.deposit_nonce, - deposit_receipt_version: receipt.deposit_receipt_version, - }, - logs_bloom: receipt_with_bloom.logs_bloom, - }) - } + ) -> Result + where + N: NodePrimitives, + { + let timestamp = input.meta.timestamp; + let block_number = input.meta.block_number; + let tx_signed = *input.tx.inner(); + let mut core_receipt = build_receipt(input, None, |receipt, next_log_index, meta| { + let map_logs = move |receipt: alloy_consensus::Receipt| { + let Receipt { status, cumulative_gas_used, logs } = receipt; + let logs = Log::collect_for_receipt(next_log_index, meta, logs); + Receipt { status, cumulative_gas_used, logs } + }; + match receipt { + OpReceipt::Legacy(receipt) => { + OpReceiptEnvelope::Legacy(map_logs(receipt).into_with_bloom()) + } + OpReceipt::Eip2930(receipt) => { + OpReceiptEnvelope::Eip2930(map_logs(receipt).into_with_bloom()) + } + OpReceipt::Eip1559(receipt) => { + OpReceiptEnvelope::Eip1559(map_logs(receipt).into_with_bloom()) } - })?; + OpReceipt::Eip7702(receipt) => { + OpReceiptEnvelope::Eip7702(map_logs(receipt).into_with_bloom()) + } + + OpReceipt::Deposit(receipt) => { + OpReceiptEnvelope::Deposit(receipt.map_inner(map_logs).into_with_bloom()) + } + } + }); + + // In jovian, we're using the blob gas used field to store the current da + // footprint's value. + // We're computing the jovian blob gas used before building the receipt since the inputs get + // consumed by the `build_receipt` function. + chain_spec.is_jovian_active_at_timestamp(timestamp).then(|| { + // Estimate the size of the transaction in bytes and multiply by the DA + // footprint gas scalar. + // Jovian specs: `https://github.com/ethereum-optimism/specs/blob/main/specs/protocol/jovian/exec-engine.md#da-footprint-block-limit` + let da_size = estimate_tx_compressed_size(tx_signed.encoded_2718().as_slice()) + .saturating_div(1_000_000) + .saturating_mul(l1_block_info.da_footprint_gas_scalar.unwrap_or_default().into()); + + core_receipt.blob_gas_used = Some(da_size); + }); let op_receipt_fields = OpReceiptFieldsBuilder::new(timestamp, block_number) - .l1_block_info(chain_spec, transaction, l1_block_info)? + .l1_block_info(chain_spec, tx_signed, l1_block_info)? .build(); Ok(Self { core_receipt, op_receipt_fields }) } - /// Builds [`OpTransactionReceipt`] by combing core (l1) receipt fields and additional OP + /// Builds [`OpTransactionReceipt`] by combining core (l1) receipt fields and additional OP /// receipt fields. pub fn build(self) -> OpTransactionReceipt { let Self { core_receipt: inner, op_receipt_fields } = self; @@ -267,10 +369,18 @@ impl OpReceiptBuilder { #[cfg(test)] mod test { use super::*; - use alloy_consensus::{Block, BlockBody}; - use alloy_primitives::hex; + use alloy_consensus::{transaction::TransactionMeta, Block, BlockBody, Eip658Value, TxEip7702}; + use alloy_op_hardforks::{ + OP_MAINNET_ISTHMUS_TIMESTAMP, OP_MAINNET_JOVIAN_TIMESTAMP, + }; + use alloy_primitives::{hex, Address, Bytes, Signature, U256}; + use op_alloy_consensus::OpTypedTransaction; use op_alloy_network::eip2718::Decodable2718; + use reth_optimism_forks::OpHardforks; + use reth_mantle_forks::MantleChainHardforks; use reth_optimism_chainspec::{BASE_MAINNET, OP_MAINNET}; + use reth_optimism_primitives::{OpPrimitives, OpTransactionSigned}; + use reth_primitives_traits::Recovered; /// OP Mainnet transaction at index 0 in block 124665056. /// @@ -307,6 +417,7 @@ mod test { operator_fee_scalar: None, operator_fee_constant: None, token_ratio: None, + da_footprint_gas_scalar: None, }, deposit_nonce: None, deposit_receipt_version: None, @@ -336,7 +447,7 @@ mod test { assert!(OP_MAINNET.is_fjord_active_at_timestamp(BLOCK_124665056_TIMESTAMP)); let receipt_meta = OpReceiptFieldsBuilder::new(BLOCK_124665056_TIMESTAMP, 124665056) - .l1_block_info(&OP_MAINNET, &tx_1, &mut l1_block_info) + .l1_block_info(&*OP_MAINNET, &tx_1, &mut l1_block_info) .expect("should parse revm l1 info") .build(); @@ -351,6 +462,7 @@ mod test { operator_fee_scalar, operator_fee_constant, token_ratio, + da_footprint_gas_scalar, } = receipt_meta.l1_block_info; assert_eq!( @@ -395,10 +507,54 @@ mod test { "incorrect operator fee constant" ); assert_eq!( - token_ratio, - TX_META_TX_1_OP_MAINNET_BLOCK_124665056.l1_block_info.token_ratio, + token_ratio, TX_META_TX_1_OP_MAINNET_BLOCK_124665056.l1_block_info.token_ratio, "incorrect token ratio" ); + assert_eq!( + da_footprint_gas_scalar, + TX_META_TX_1_OP_MAINNET_BLOCK_124665056.l1_block_info.da_footprint_gas_scalar, + "incorrect da footprint gas scalar" + ); + } + + #[test] + fn op_non_zero_operator_fee_params_included_in_receipt() { + let tx_1 = + OpTransactionSigned::decode_2718(&mut TX_1_OP_MAINNET_BLOCK_124665056.as_slice()) + .unwrap(); + + let mut l1_block_info = op_revm::L1BlockInfo::default(); + + let receipt_meta = OpReceiptFieldsBuilder::new(BLOCK_124665056_TIMESTAMP, 124665056) + .l1_block_info(&*OP_MAINNET, &tx_1, &mut l1_block_info) + .expect("should parse revm l1 info") + .build(); + + let L1BlockInfo { operator_fee_scalar, operator_fee_constant, .. } = + receipt_meta.l1_block_info; + + assert_eq!(operator_fee_scalar, Some(0), "incorrect operator fee scalar"); + assert_eq!(operator_fee_constant, Some(2), "incorrect operator fee constant"); + } + + #[test] + fn op_zero_operator_fee_params_not_included_in_receipt() { + let tx_1 = + OpTransactionSigned::decode_2718(&mut TX_1_OP_MAINNET_BLOCK_124665056.as_slice()) + .unwrap(); + + let mut l1_block_info = op_revm::L1BlockInfo::default(); + + let receipt_meta = OpReceiptFieldsBuilder::new(BLOCK_124665056_TIMESTAMP, 124665056) + .l1_block_info(&*OP_MAINNET, &tx_1, &mut l1_block_info) + .expect("should parse revm l1 info") + .build(); + + let L1BlockInfo { operator_fee_scalar, operator_fee_constant, .. } = + receipt_meta.l1_block_info; + + assert_eq!(operator_fee_scalar, None, "incorrect operator fee scalar"); + assert_eq!(operator_fee_constant, None, "incorrect operator fee constant"); } // @@ -424,7 +580,7 @@ mod test { let tx_1 = OpTransactionSigned::decode_2718(&mut &tx[..]).unwrap(); let receipt_meta = OpReceiptFieldsBuilder::new(1730216981, 21713817) - .l1_block_info(&BASE_MAINNET, &tx_1, &mut l1_block_info) + .l1_block_info(&*BASE_MAINNET, &tx_1, &mut l1_block_info) .expect("should parse revm l1 info") .build(); @@ -439,6 +595,7 @@ mod test { operator_fee_scalar, operator_fee_constant, token_ratio, + da_footprint_gas_scalar, } = receipt_meta.l1_block_info; assert_eq!(l1_gas_price, Some(14121491676), "incorrect l1 base fee (former gas price)"); @@ -451,5 +608,146 @@ mod test { assert_eq!(operator_fee_scalar, None, "incorrect operator fee scalar"); assert_eq!(operator_fee_constant, None, "incorrect operator fee constant"); assert_eq!(token_ratio, None, "incorrect token ratio"); + assert_eq!(da_footprint_gas_scalar, None, "incorrect da footprint gas scalar"); + } + + #[test] + fn da_footprint_gas_scalar_included_in_receipt_post_jovian() { + const DA_FOOTPRINT_GAS_SCALAR: u16 = 10; + + let tx = TxEip7702 { + chain_id: 1u64, + nonce: 0, + max_fee_per_gas: 0x28f000fff, + max_priority_fee_per_gas: 0x28f000fff, + gas_limit: 10, + to: Address::default(), + value: U256::from(3_u64), + input: Bytes::from(vec![1, 2]), + access_list: Default::default(), + authorization_list: Default::default(), + }; + + let signature = Signature::new(U256::default(), U256::default(), true); + + let tx = OpTransactionSigned::new_unhashed(OpTypedTransaction::Eip7702(tx), signature); + + let mut l1_block_info = op_revm::L1BlockInfo { + da_footprint_gas_scalar: Some(DA_FOOTPRINT_GAS_SCALAR), + ..Default::default() + }; + + let mantle_hardforks = MantleChainHardforks::mantle_mainnet(); + + let receipt = OpReceiptFieldsBuilder::new(OP_MAINNET_JOVIAN_TIMESTAMP, u64::MAX) + .l1_block_info(&mantle_hardforks, &tx, &mut l1_block_info) + .expect("should parse revm l1 info") + .build(); + + assert_eq!(receipt.l1_block_info.da_footprint_gas_scalar, Some(DA_FOOTPRINT_GAS_SCALAR)); + } + + #[test] + fn blob_gas_used_included_in_receipt_post_jovian() { + const DA_FOOTPRINT_GAS_SCALAR: u16 = 100; + let tx = TxEip7702 { + chain_id: 1u64, + nonce: 0, + max_fee_per_gas: 0x28f000fff, + max_priority_fee_per_gas: 0x28f000fff, + gas_limit: 10, + to: Address::default(), + value: U256::from(3_u64), + access_list: Default::default(), + authorization_list: Default::default(), + input: Bytes::from(vec![0; 1_000_000]), + }; + + let signature = Signature::new(U256::default(), U256::default(), true); + + let tx = OpTransactionSigned::new_unhashed(OpTypedTransaction::Eip7702(tx), signature); + + let mut l1_block_info = op_revm::L1BlockInfo { + da_footprint_gas_scalar: Some(DA_FOOTPRINT_GAS_SCALAR), + ..Default::default() + }; + + let mantle_hardforks = MantleChainHardforks::mantle_mainnet(); + + let op_receipt = OpReceiptBuilder::new( + &mantle_hardforks, + ConvertReceiptInput:: { + tx: Recovered::new_unchecked(&tx, Address::default()), + receipt: OpReceipt::Eip7702(Receipt { + status: Eip658Value::Eip658(true), + cumulative_gas_used: 100, + logs: vec![], + }), + gas_used: 100, + next_log_index: 0, + meta: TransactionMeta { + timestamp: OP_MAINNET_JOVIAN_TIMESTAMP, + ..Default::default() + }, + }, + &mut l1_block_info, + ) + .unwrap(); + + let expected_blob_gas_used = estimate_tx_compressed_size(tx.encoded_2718().as_slice()) + .saturating_div(1_000_000) + .saturating_mul(DA_FOOTPRINT_GAS_SCALAR.into()); + + assert_eq!(op_receipt.core_receipt.blob_gas_used, Some(expected_blob_gas_used)); + } + + #[test] + fn blob_gas_used_not_included_in_receipt_post_isthmus() { + const DA_FOOTPRINT_GAS_SCALAR: u16 = 100; + let tx = TxEip7702 { + chain_id: 1u64, + nonce: 0, + max_fee_per_gas: 0x28f000fff, + max_priority_fee_per_gas: 0x28f000fff, + gas_limit: 10, + to: Address::default(), + value: U256::from(3_u64), + access_list: Default::default(), + authorization_list: Default::default(), + input: Bytes::from(vec![0; 1_000_000]), + }; + + let signature = Signature::new(U256::default(), U256::default(), true); + + let tx = OpTransactionSigned::new_unhashed(OpTypedTransaction::Eip7702(tx), signature); + + let mut l1_block_info = op_revm::L1BlockInfo { + da_footprint_gas_scalar: Some(DA_FOOTPRINT_GAS_SCALAR), + ..Default::default() + }; + + let mantle_hardforks = MantleChainHardforks::mantle_mainnet(); + + let op_receipt = OpReceiptBuilder::new( + &mantle_hardforks, + ConvertReceiptInput:: { + tx: Recovered::new_unchecked(&tx, Address::default()), + receipt: OpReceipt::Eip7702(Receipt { + status: Eip658Value::Eip658(true), + cumulative_gas_used: 100, + logs: vec![], + }), + gas_used: 100, + next_log_index: 0, + meta: TransactionMeta { + timestamp: OP_MAINNET_ISTHMUS_TIMESTAMP, + ..Default::default() + }, + }, + &mut l1_block_info, + ) + .unwrap(); + + assert_eq!(op_receipt.core_receipt.blob_gas_used, None); } } diff --git a/crates/optimism/rpc/src/eth/transaction.rs b/crates/optimism/rpc/src/eth/transaction.rs index 1235ffcb39e..7ea375dd15d 100644 --- a/crates/optimism/rpc/src/eth/transaction.rs +++ b/crates/optimism/rpc/src/eth/transaction.rs @@ -1,38 +1,53 @@ //! Loads and formats OP transaction RPC response. -use alloy_consensus::{transaction::Recovered, SignableTransaction, Transaction as _}; -use alloy_primitives::{Bytes, Signature, B256}; +use crate::{OpEthApi, OpEthApiError, SequencerClient}; +use alloy_primitives::{Bytes, B256}; use alloy_rpc_types_eth::TransactionInfo; -use op_alloy_consensus::OpTxEnvelope; -use op_alloy_rpc_types::{OpTransactionRequest, Transaction}; -use reth_node_api::FullNodeComponents; -use reth_optimism_primitives::{OpReceipt, OpTransactionSigned}; +use futures::StreamExt; +use op_alloy_consensus::{transaction::OpTransactionInfo, OpTransaction}; +use reth_chain_state::CanonStateSubscriptions; +use reth_optimism_primitives::DepositReceipt; +use reth_primitives_traits::{BlockBody, SignedTransaction}; use reth_rpc_eth_api::{ - helpers::{EthSigner, EthTransactions, LoadTransaction, SpawnBlocking}, - EthApiTypes, FromEthApiError, FullEthApiTypes, RpcNodeCore, RpcNodeCoreExt, TransactionCompat, + helpers::{spec::SignersForRpc, EthTransactions, LoadReceipt, LoadTransaction}, + try_into_op_tx_info, EthApiTypes as _, FromEthApiError, FromEvmError, RpcConvert, RpcNodeCore, + RpcReceipt, TxInfoMapper, }; use reth_rpc_eth_types::{utils::recover_raw_transaction, EthApiError}; -use reth_storage_api::{ - BlockReader, BlockReaderIdExt, ProviderTx, ReceiptProvider, TransactionsProvider, +use reth_storage_api::{errors::ProviderError, ReceiptProvider}; +use reth_transaction_pool::{ + AddedTransactionOutcome, PoolTransaction, TransactionOrigin, TransactionPool, }; -use reth_transaction_pool::{PoolTransaction, TransactionOrigin, TransactionPool}; - -use crate::{eth::OpNodeCore, OpEthApi, OpEthApiError, SequencerClient}; +use std::{ + fmt::{Debug, Formatter}, + future::Future, + time::Duration, +}; +use tokio_stream::wrappers::WatchStream; -impl EthTransactions for OpEthApi +impl EthTransactions for OpEthApi where - Self: LoadTransaction + EthApiTypes, - N: OpNodeCore>>, + N: RpcNodeCore, + OpEthApiError: FromEvmError, + Rpc: RpcConvert, { - fn signers(&self) -> &parking_lot::RwLock>>>> { + fn signers(&self) -> &SignersForRpc { self.inner.eth_api.signers() } + fn send_raw_transaction_sync_timeout(&self) -> Duration { + self.inner.eth_api.send_raw_transaction_sync_timeout() + } + /// Decodes and recovers the transaction and submits it to the pool. /// /// Returns the hash of the transaction. async fn send_raw_transaction(&self, tx: Bytes) -> Result { let recovered = recover_raw_transaction(&tx)?; + + // broadcast raw transaction to subscribers if there is any. + self.eth_api().broadcast_raw_transaction(tx.clone()); + let pool_transaction = ::Transaction::from_pooled(recovered); // On optimism, transactions are forwarded directly to the sequencer to be included in @@ -44,18 +59,15 @@ where })?; // Retain tx in local tx pool after forwarding, for local RPC usage. - let _ = self - .pool() - .add_transaction(TransactionOrigin::Local, pool_transaction) - .await.inspect_err(|err| { - tracing::warn!(target: "rpc::eth", %err, %hash, "successfully sent tx to sequencer, but failed to persist in local tx pool"); + let _ = self.inner.eth_api.add_pool_transaction(pool_transaction).await.inspect_err(|err| { + tracing::warn!(target: "rpc::eth", %err, %hash, "successfully sent tx to sequencer, but failed to persist in local tx pool"); }); - return Ok(hash) + return Ok(hash); } // submit the transaction to the pool with a `Local` origin - let hash = self + let AddedTransactionOutcome { hash, .. } = self .pool() .add_transaction(TransactionOrigin::Local, pool_transaction) .await @@ -63,19 +75,116 @@ where Ok(hash) } + + /// Decodes and recovers the transaction and submits it to the pool. + /// + /// And awaits the receipt, checking both canonical blocks and flashblocks for faster + /// confirmation. + fn send_raw_transaction_sync( + &self, + tx: Bytes, + ) -> impl Future, Self::Error>> + Send { + let this = self.clone(); + let timeout_duration = self.send_raw_transaction_sync_timeout(); + async move { + let mut canonical_stream = this.provider().canonical_state_stream(); + let hash = EthTransactions::send_raw_transaction(&this, tx).await?; + let mut flashblock_stream = this.pending_block_rx().map(WatchStream::new); + + tokio::time::timeout(timeout_duration, async { + loop { + tokio::select! { + biased; + // check if the tx was preconfirmed in a new flashblock + flashblock = async { + if let Some(stream) = &mut flashblock_stream { + stream.next().await + } else { + futures::future::pending().await + } + } => { + if let Some(flashblock) = flashblock.flatten() { + // if flashblocks are supported, attempt to find id from the pending block + if let Some(receipt) = flashblock + .find_and_convert_transaction_receipt(hash, this.tx_resp_builder()) + { + return receipt; + } + } + } + // Listen for regular canonical block updates for inclusion + canonical_notification = canonical_stream.next() => { + if let Some(notification) = canonical_notification { + let chain = notification.committed(); + for block in chain.blocks_iter() { + if block.body().contains_transaction(&hash) + && let Some(receipt) = this.transaction_receipt(hash).await? { + return Ok(receipt); + } + } + } else { + // Canonical stream ended + break; + } + } + } + } + Err(Self::Error::from_eth_err(EthApiError::TransactionConfirmationTimeout { + hash, + duration: timeout_duration, + })) + }) + .await + .unwrap_or_else(|_elapsed| { + Err(Self::Error::from_eth_err(EthApiError::TransactionConfirmationTimeout { + hash, + duration: timeout_duration, + })) + }) + } + } + + /// Returns the transaction receipt for the given hash. + /// + /// With flashblocks, we should also lookup the pending block for the transaction + /// because this is considered confirmed/mined. + fn transaction_receipt( + &self, + hash: B256, + ) -> impl Future>, Self::Error>> + Send + { + let this = self.clone(); + async move { + // first attempt to fetch the mined transaction receipt data + let tx_receipt = this.load_transaction_and_receipt(hash).await?; + + if tx_receipt.is_none() { + // if flashblocks are supported, attempt to find id from the pending block + if let Ok(Some(pending_block)) = this.pending_flashblock().await && + let Some(Ok(receipt)) = pending_block + .find_and_convert_transaction_receipt(hash, this.tx_resp_builder()) + { + return Ok(Some(receipt)); + } + } + let Some((tx, meta, receipt)) = tx_receipt else { return Ok(None) }; + self.build_transaction_receipt(tx, meta, receipt).await.map(Some) + } + } } -impl LoadTransaction for OpEthApi +impl LoadTransaction for OpEthApi where - Self: SpawnBlocking + FullEthApiTypes + RpcNodeCoreExt, - N: OpNodeCore, - Self::Pool: TransactionPool, + N: RpcNodeCore, + OpEthApiError: FromEvmError, + Rpc: RpcConvert, { } -impl OpEthApi +impl OpEthApi where - N: OpNodeCore, + N: RpcNodeCore, + Rpc: RpcConvert, { /// Returns the [`SequencerClient`] if one is set. pub fn raw_tx_forwarder(&self) -> Option { @@ -83,89 +192,42 @@ where } } -impl TransactionCompat for OpEthApi -where - N: FullNodeComponents>, -{ - type Transaction = Transaction; - type Error = OpEthApiError; - - fn fill( - &self, - tx: Recovered, - tx_info: TransactionInfo, - ) -> Result { - let mut tx = tx.convert::(); - let mut deposit_receipt_version = None; - let mut deposit_nonce = None; - - if let OpTxEnvelope::Deposit(tx) = tx.inner_mut() { - // for depost tx we need to fetch the receipt - self.inner - .eth_api - .provider() - .receipt_by_hash(tx.tx_hash()) - .map_err(Self::Error::from_eth_err)? - .inspect(|receipt| { - if let OpReceipt::Deposit(receipt) = receipt { - deposit_receipt_version = receipt.deposit_receipt_version; - deposit_nonce = receipt.deposit_nonce; - } - }); +/// Optimism implementation of [`TxInfoMapper`]. +/// +/// For deposits, receipt is fetched to extract `deposit_nonce` and `deposit_receipt_version`. +/// Otherwise, it works like regular Ethereum implementation, i.e. uses [`TransactionInfo`]. +pub struct OpTxInfoMapper { + provider: Provider, +} - // For consistency with op-geth, we always return `0x0` for mint and eth_value if it is - // missing. This is because op-geth does not distinguish between null and 0, because - // this value is decoded from RLP where null is represented as 0. - tx.inner_mut().mint = Some(tx.mint.unwrap_or_default()); - tx.inner_mut().eth_value = Some(tx.eth_value.unwrap_or_default()); - } +impl Clone for OpTxInfoMapper { + fn clone(&self) -> Self { + Self { provider: self.provider.clone() } + } +} - let TransactionInfo { - block_hash, block_number, index: transaction_index, base_fee, .. - } = tx_info; - - let effective_gas_price = if tx.is_deposit() { - // For deposits, we must always set the `gasPrice` field to 0 in rpc - // deposit tx don't have a gas price field, but serde of `Transaction` will take care of - // it - 0 - } else { - base_fee - .map(|base_fee| { - tx.effective_tip_per_gas(base_fee).unwrap_or_default() + base_fee as u128 - }) - .unwrap_or_else(|| tx.max_fee_per_gas()) - }; - - Ok(Transaction { - inner: alloy_rpc_types_eth::Transaction { - inner: tx, - block_hash, - block_number, - transaction_index, - effective_gas_price: Some(effective_gas_price), - }, - deposit_nonce, - deposit_receipt_version, - }) +impl Debug for OpTxInfoMapper { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("OpTxInfoMapper").finish() } +} - fn build_simulate_v1_transaction( - &self, - request: alloy_rpc_types_eth::TransactionRequest, - ) -> Result { - let request: OpTransactionRequest = request.into(); - let Ok(tx) = request.build_typed_tx() else { - return Err(OpEthApiError::Eth(EthApiError::TransactionConversionError)) - }; - - // Create an empty signature for the transaction. - let signature = Signature::new(Default::default(), Default::default(), false); - Ok(tx.into_signed(signature).into()) +impl OpTxInfoMapper { + /// Creates [`OpTxInfoMapper`] that uses [`ReceiptProvider`] borrowed from given `eth_api`. + pub const fn new(provider: Provider) -> Self { + Self { provider } } +} + +impl TxInfoMapper for OpTxInfoMapper +where + T: OpTransaction + SignedTransaction, + Provider: ReceiptProvider, +{ + type Out = OpTransactionInfo; + type Err = ProviderError; - fn otterscan_api_truncate_input(tx: &mut Self::Transaction) { - let input = tx.inner.inner.inner_mut().input_mut(); - *input = input.slice(..4); + fn try_map(&self, tx: &T, tx_info: TransactionInfo) -> Result { + try_into_op_tx_info(&self.provider, tx, tx_info) } } diff --git a/crates/optimism/rpc/src/historical.rs b/crates/optimism/rpc/src/historical.rs new file mode 100644 index 00000000000..736d962b6db --- /dev/null +++ b/crates/optimism/rpc/src/historical.rs @@ -0,0 +1,417 @@ +//! Client support for optimism historical RPC requests. + +use crate::sequencer::Error; +use alloy_eips::BlockId; +use alloy_json_rpc::{RpcRecv, RpcSend}; +use alloy_primitives::{BlockNumber, B256}; +use alloy_rpc_client::RpcClient; +use jsonrpsee_core::{ + middleware::{Batch, Notification, RpcServiceT}, + server::MethodResponse, +}; +use jsonrpsee_types::{Params, Request}; +use reth_storage_api::{BlockReaderIdExt, TransactionsProvider}; +use std::{future::Future, sync::Arc}; +use tracing::{debug, warn}; + +/// A client that can be used to forward RPC requests for historical data to an endpoint. +/// +/// This is intended to be used for OP-Mainnet pre-bedrock data, allowing users to query historical +/// state. +#[derive(Debug, Clone)] +pub struct HistoricalRpcClient { + inner: Arc, +} + +impl HistoricalRpcClient { + /// Constructs a new historical RPC client with the given endpoint URL. + pub fn new(endpoint: &str) -> Result { + let client = RpcClient::new_http( + endpoint.parse::().map_err(|err| Error::InvalidUrl(err.to_string()))?, + ); + + Ok(Self { + inner: Arc::new(HistoricalRpcClientInner { + historical_endpoint: endpoint.to_string(), + client, + }), + }) + } + + /// Returns a reference to the underlying RPC client + fn client(&self) -> &RpcClient { + &self.inner.client + } + + /// Forwards a JSON-RPC request to the historical endpoint + pub async fn request( + &self, + method: &str, + params: Params, + ) -> Result { + let resp = + self.client().request::(method.to_string(), params).await.inspect_err( + |err| { + warn!( + target: "rpc::historical", + %err, + "HTTP request to historical endpoint failed" + ); + }, + )?; + + Ok(resp) + } + + /// Returns the configured historical endpoint URL + pub fn endpoint(&self) -> &str { + &self.inner.historical_endpoint + } +} + +#[derive(Debug)] +struct HistoricalRpcClientInner { + historical_endpoint: String, + client: RpcClient, +} + +/// A layer that provides historical RPC forwarding functionality for a given service. +#[derive(Debug, Clone)] +pub struct HistoricalRpc

{ + inner: Arc>, +} + +impl

HistoricalRpc

{ + /// Constructs a new historical RPC layer with the given provider, client and bedrock block + /// number. + pub fn new(provider: P, client: HistoricalRpcClient, bedrock_block: BlockNumber) -> Self { + let inner = Arc::new(HistoricalRpcInner { provider, client, bedrock_block }); + + Self { inner } + } +} + +impl tower::Layer for HistoricalRpc

{ + type Service = HistoricalRpcService; + + fn layer(&self, inner: S) -> Self::Service { + HistoricalRpcService::new(inner, self.inner.clone()) + } +} + +/// A service that intercepts RPC calls and forwards pre-bedrock historical requests +/// to a dedicated endpoint. +/// +/// This checks if the request is for a pre-bedrock block and forwards it via the configured +/// historical RPC client. +#[derive(Debug, Clone)] +pub struct HistoricalRpcService { + /// The inner service that handles regular RPC requests + inner: S, + /// The context required to forward historical requests. + historical: Arc>, +} + +impl HistoricalRpcService { + /// Constructs a new historical RPC service with the given inner service, historical client, + /// provider, and bedrock block number. + const fn new(inner: S, historical: Arc>) -> Self { + Self { inner, historical } + } +} + +impl RpcServiceT for HistoricalRpcService +where + S: RpcServiceT + Send + Sync + Clone + 'static, + + P: BlockReaderIdExt + TransactionsProvider + Send + Sync + Clone + 'static, +{ + type MethodResponse = S::MethodResponse; + type NotificationResponse = S::NotificationResponse; + type BatchResponse = S::BatchResponse; + + fn call<'a>(&self, req: Request<'a>) -> impl Future + Send + 'a { + let inner_service = self.inner.clone(); + let historical = self.historical.clone(); + + Box::pin(async move { + // Check if request should be forwarded to historical endpoint + if let Some(response) = historical.maybe_forward_request(&req).await { + return response + } + + // Handle the request with the inner service + inner_service.call(req).await + }) + } + + fn batch<'a>(&self, req: Batch<'a>) -> impl Future + Send + 'a { + self.inner.batch(req) + } + + fn notification<'a>( + &self, + n: Notification<'a>, + ) -> impl Future + Send + 'a { + self.inner.notification(n) + } +} + +#[derive(Debug)] +struct HistoricalRpcInner

{ + /// Provider used to determine if a block is pre-bedrock + provider: P, + /// Client used to forward historical requests + client: HistoricalRpcClient, + /// Bedrock transition block number + bedrock_block: BlockNumber, +} + +impl

HistoricalRpcInner

+where + P: BlockReaderIdExt + TransactionsProvider + Send + Sync + Clone, +{ + /// Checks if a request should be forwarded to the historical endpoint and returns + /// the response if it was forwarded. + async fn maybe_forward_request(&self, req: &Request<'_>) -> Option { + let should_forward = match req.method_name() { + "debug_traceTransaction" | + "eth_getTransactionByHash" | + "eth_getTransactionReceipt" | + "eth_getRawTransactionByHash" => self.should_forward_transaction(req), + method => self.should_forward_block_request(method, req), + }; + + if should_forward { + return self.forward_to_historical(req).await + } + + None + } + + /// Determines if a transaction request should be forwarded + fn should_forward_transaction(&self, req: &Request<'_>) -> bool { + parse_transaction_hash_from_params(&req.params()) + .ok() + .map(|tx_hash| { + // Check if we can find the transaction locally and get its metadata + match self.provider.transaction_by_hash_with_meta(tx_hash) { + Ok(Some((_, meta))) => { + // Transaction found - check if it's pre-bedrock based on block number + let is_pre_bedrock = meta.block_number < self.bedrock_block; + if is_pre_bedrock { + debug!( + target: "rpc::historical", + ?tx_hash, + block_num = meta.block_number, + bedrock = self.bedrock_block, + "transaction found in pre-bedrock block, forwarding to historical endpoint" + ); + } + is_pre_bedrock + } + _ => { + // Transaction not found locally, optimistically forward to historical endpoint + debug!( + target: "rpc::historical", + ?tx_hash, + "transaction not found locally, forwarding to historical endpoint" + ); + true + } + } + }) + .unwrap_or(false) + } + + /// Determines if a block-based request should be forwarded + fn should_forward_block_request(&self, method: &str, req: &Request<'_>) -> bool { + let maybe_block_id = extract_block_id_for_method(method, &req.params()); + + maybe_block_id.map(|block_id| self.is_pre_bedrock(block_id)).unwrap_or(false) + } + + /// Checks if a block ID refers to a pre-bedrock block + fn is_pre_bedrock(&self, block_id: BlockId) -> bool { + match self.provider.block_number_for_id(block_id) { + Ok(Some(num)) => { + debug!( + target: "rpc::historical", + ?block_id, + block_num=num, + bedrock=self.bedrock_block, + "found block number" + ); + num < self.bedrock_block + } + Ok(None) if block_id.is_hash() => { + debug!( + target: "rpc::historical", + ?block_id, + "block hash not found locally, assuming pre-bedrock" + ); + true + } + _ => { + debug!( + target: "rpc::historical", + ?block_id, + "could not determine block number; not forwarding" + ); + false + } + } + } + + /// Forwards a request to the historical endpoint + async fn forward_to_historical(&self, req: &Request<'_>) -> Option { + debug!( + target: "rpc::historical", + method = %req.method_name(), + params=?req.params(), + "forwarding request to historical endpoint" + ); + + let params = req.params(); + let params_str = params.as_str().unwrap_or("[]"); + + let params = serde_json::from_str::(params_str).ok()?; + + let raw = + self.client.request::<_, serde_json::Value>(req.method_name(), params).await.ok()?; + + let payload = jsonrpsee_types::ResponsePayload::success(raw).into(); + Some(MethodResponse::response(req.id.clone(), payload, usize::MAX)) + } +} + +/// Error type for parameter parsing +#[derive(Debug)] +enum ParseError { + InvalidFormat, + MissingParameter, +} + +/// Extracts the block ID from request parameters based on the method name +fn extract_block_id_for_method(method: &str, params: &Params<'_>) -> Option { + match method { + "eth_getBlockByNumber" | + "eth_getBlockByHash" | + "debug_traceBlockByNumber" | + "debug_traceBlockByHash" => parse_block_id_from_params(params, 0), + "eth_getBalance" | + "eth_getCode" | + "eth_getTransactionCount" | + "eth_call" | + "eth_estimateGas" | + "eth_createAccessList" | + "debug_traceCall" => parse_block_id_from_params(params, 1), + "eth_getStorageAt" | "eth_getProof" => parse_block_id_from_params(params, 2), + _ => None, + } +} + +/// Parses a `BlockId` from the given parameters at the specified position. +fn parse_block_id_from_params(params: &Params<'_>, position: usize) -> Option { + let values: Vec = params.parse().ok()?; + let val = values.into_iter().nth(position)?; + serde_json::from_value::(val).ok() +} + +/// Parses a transaction hash from the first parameter. +fn parse_transaction_hash_from_params(params: &Params<'_>) -> Result { + let values: Vec = params.parse().map_err(|_| ParseError::InvalidFormat)?; + let val = values.into_iter().next().ok_or(ParseError::MissingParameter)?; + serde_json::from_value::(val).map_err(|_| ParseError::InvalidFormat) +} + +#[cfg(test)] +mod tests { + use super::*; + use alloy_eips::{BlockId, BlockNumberOrTag}; + use jsonrpsee::types::Params; + use jsonrpsee_core::middleware::layer::Either; + use reth_node_builder::rpc::RethRpcMiddleware; + use reth_storage_api::noop::NoopProvider; + use tower::layer::util::Identity; + + #[test] + fn check_historical_rpc() { + fn assert_historical_rpc() {} + assert_historical_rpc::>(); + assert_historical_rpc::, Identity>>(); + } + + /// Tests that various valid id types can be parsed from the first parameter. + #[test] + fn parses_block_id_from_first_param() { + // Test with a block number + let params_num = Params::new(Some(r#"["0x64"]"#)); // 100 + assert_eq!( + parse_block_id_from_params(¶ms_num, 0).unwrap(), + BlockId::Number(BlockNumberOrTag::Number(100)) + ); + + // Test with the "earliest" tag + let params_tag = Params::new(Some(r#"["earliest"]"#)); + assert_eq!( + parse_block_id_from_params(¶ms_tag, 0).unwrap(), + BlockId::Number(BlockNumberOrTag::Earliest) + ); + } + + /// Tests that the function correctly parses from a position other than 0. + #[test] + fn parses_block_id_from_second_param() { + let params = + Params::new(Some(r#"["0x0000000000000000000000000000000000000000", "latest"]"#)); + let result = parse_block_id_from_params(¶ms, 1).unwrap(); + assert_eq!(result, BlockId::Number(BlockNumberOrTag::Latest)); + } + + /// Tests that the function returns nothing if the parameter is missing or empty. + #[test] + fn defaults_to_latest_when_param_is_missing() { + let params = Params::new(Some(r#"["0x0000000000000000000000000000000000000000"]"#)); + let result = parse_block_id_from_params(¶ms, 1); + assert!(result.is_none()); + } + + /// Tests that the function doesn't parse anything if the parameter is not a valid block id. + #[test] + fn returns_error_for_invalid_input() { + let params = Params::new(Some(r#"[true]"#)); + let result = parse_block_id_from_params(¶ms, 0); + assert!(result.is_none()); + } + + /// Tests that transaction hashes can be parsed from params. + #[test] + fn parses_transaction_hash_from_params() { + let hash = "0xdbdfa0f88b2cf815fdc1621bd20c2bd2b0eed4f0c56c9be2602957b5a60ec702"; + let params_str = format!(r#"["{hash}"]"#); + let params = Params::new(Some(¶ms_str)); + let result = parse_transaction_hash_from_params(¶ms); + assert!(result.is_ok()); + let parsed_hash = result.unwrap(); + assert_eq!(format!("{parsed_hash:?}"), hash); + } + + /// Tests that invalid transaction hash returns error. + #[test] + fn returns_error_for_invalid_tx_hash() { + let params = Params::new(Some(r#"["not_a_hash"]"#)); + let result = parse_transaction_hash_from_params(¶ms); + assert!(result.is_err()); + assert!(matches!(result.unwrap_err(), ParseError::InvalidFormat)); + } + + /// Tests that missing parameter returns appropriate error. + #[test] + fn returns_error_for_missing_parameter() { + let params = Params::new(Some(r#"[]"#)); + let result = parse_transaction_hash_from_params(¶ms); + assert!(result.is_err()); + assert!(matches!(result.unwrap_err(), ParseError::MissingParameter)); + } +} diff --git a/crates/optimism/rpc/src/lib.rs b/crates/optimism/rpc/src/lib.rs index 7fdf62a9e8b..86c9bcfe2d1 100644 --- a/crates/optimism/rpc/src/lib.rs +++ b/crates/optimism/rpc/src/lib.rs @@ -6,11 +6,13 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] pub mod engine; pub mod error; pub mod eth; +pub mod historical; +pub mod metrics; pub mod miner; pub mod sequencer; pub mod witness; @@ -19,5 +21,6 @@ pub mod witness; pub use engine::OpEngineApiClient; pub use engine::{OpEngineApi, OpEngineApiServer, OP_ENGINE_CAPABILITIES}; pub use error::{OpEthApiError, OpInvalidTransactionError, SequencerClientError}; -pub use eth::{mantle_ext::MantleEthApiExt, OpEthApi, OpReceiptBuilder}; +pub use eth::{mantle_ext::MantleEthApiExt, OpEthApi, OpEthApiBuilder, OpReceiptBuilder}; +pub use metrics::SequencerMetrics; pub use sequencer::SequencerClient; diff --git a/crates/optimism/rpc/src/metrics.rs b/crates/optimism/rpc/src/metrics.rs new file mode 100644 index 00000000000..5aa5e3eff3d --- /dev/null +++ b/crates/optimism/rpc/src/metrics.rs @@ -0,0 +1,21 @@ +//! RPC metrics unique for OP-stack. + +use core::time::Duration; +use metrics::Histogram; +use reth_metrics::Metrics; + +/// Optimism sequencer metrics +#[derive(Metrics, Clone)] +#[metrics(scope = "optimism_rpc.sequencer")] +pub struct SequencerMetrics { + /// How long it takes to forward a transaction to the sequencer + pub(crate) sequencer_forward_latency: Histogram, +} + +impl SequencerMetrics { + /// Records the duration it took to forward a transaction + #[inline] + pub fn record_forward_latency(&self, duration: Duration) { + self.sequencer_forward_latency.record(duration.as_secs_f64()); + } +} diff --git a/crates/optimism/rpc/src/miner.rs b/crates/optimism/rpc/src/miner.rs index f18b815f255..f8780f37e82 100644 --- a/crates/optimism/rpc/src/miner.rs +++ b/crates/optimism/rpc/src/miner.rs @@ -3,7 +3,8 @@ use alloy_primitives::U64; use jsonrpsee_core::{async_trait, RpcResult}; pub use op_alloy_rpc_jsonrpsee::traits::MinerApiExtServer; -use reth_optimism_payload_builder::config::OpDAConfig; +use reth_metrics::{metrics::Gauge, Metrics}; +use reth_optimism_payload_builder::config::{OpDAConfig, OpGasLimitConfig}; use tracing::debug; /// Miner API extension for OP, exposes settings for the data availability configuration via the @@ -11,13 +12,15 @@ use tracing::debug; #[derive(Debug, Clone)] pub struct OpMinerExtApi { da_config: OpDAConfig, + gas_limit_config: OpGasLimitConfig, + metrics: OpMinerMetrics, } impl OpMinerExtApi { /// Instantiate the miner API extension with the given, sharable data availability /// configuration. - pub const fn new(da_config: OpDAConfig) -> Self { - Self { da_config } + pub fn new(da_config: OpDAConfig, gas_limit_config: OpGasLimitConfig) -> Self { + Self { da_config, gas_limit_config, metrics: OpMinerMetrics::default() } } } @@ -27,6 +30,49 @@ impl MinerApiExtServer for OpMinerExtApi { async fn set_max_da_size(&self, max_tx_size: U64, max_block_size: U64) -> RpcResult { debug!(target: "rpc", "Setting max DA size: tx={}, block={}", max_tx_size, max_block_size); self.da_config.set_max_da_size(max_tx_size.to(), max_block_size.to()); + + self.metrics.set_max_da_tx_size(max_tx_size.to()); + self.metrics.set_max_da_block_size(max_block_size.to()); + Ok(true) } + + async fn set_gas_limit(&self, gas_limit: U64) -> RpcResult { + debug!(target: "rpc", "Setting gas limit: {}", gas_limit); + self.gas_limit_config.set_gas_limit(gas_limit.to()); + self.metrics.set_gas_limit(gas_limit.to()); + Ok(true) + } +} + +/// Optimism miner metrics +#[derive(Metrics, Clone)] +#[metrics(scope = "optimism_rpc.miner")] +pub struct OpMinerMetrics { + /// Max DA tx size set on the miner + max_da_tx_size: Gauge, + /// Max DA block size set on the miner + max_da_block_size: Gauge, + /// Gas limit set on the miner + gas_limit: Gauge, +} + +impl OpMinerMetrics { + /// Sets the max DA tx size gauge value + #[inline] + pub fn set_max_da_tx_size(&self, size: u64) { + self.max_da_tx_size.set(size as f64); + } + + /// Sets the max DA block size gauge value + #[inline] + pub fn set_max_da_block_size(&self, size: u64) { + self.max_da_block_size.set(size as f64); + } + + /// Sets the gas limit gauge value + #[inline] + pub fn set_gas_limit(&self, gas_limit: u64) { + self.gas_limit.set(gas_limit as f64); + } } diff --git a/crates/optimism/rpc/src/sequencer.rs b/crates/optimism/rpc/src/sequencer.rs index 6e83371cf24..7c72ad285a6 100644 --- a/crates/optimism/rpc/src/sequencer.rs +++ b/crates/optimism/rpc/src/sequencer.rs @@ -1,12 +1,11 @@ //! Helpers for optimism specific RPC implementations. -use crate::SequencerClientError; +use crate::{SequencerClientError, SequencerMetrics}; use alloy_json_rpc::{RpcRecv, RpcSend}; use alloy_primitives::{hex, B256}; use alloy_rpc_client::{BuiltInConnectionString, ClientBuilder, RpcClient as Client}; use alloy_rpc_types_eth::erc4337::TransactionConditional; use alloy_transport_http::Http; -use reth_optimism_txpool::supervisor::metrics::SequencerMetrics; use std::{str::FromStr, sync::Arc, time::Instant}; use thiserror::Error; use tracing::warn; diff --git a/crates/optimism/rpc/src/witness.rs b/crates/optimism/rpc/src/witness.rs index c32f482bbe6..1858b4fd2f1 100644 --- a/crates/optimism/rpc/src/witness.rs +++ b/crates/optimism/rpc/src/witness.rs @@ -3,15 +3,13 @@ use alloy_primitives::B256; use alloy_rpc_types_debug::ExecutionWitness; use jsonrpsee_core::{async_trait, RpcResult}; -use op_alloy_rpc_types_engine::OpPayloadAttributes; use reth_chainspec::ChainSpecProvider; use reth_evm::ConfigureEvm; -use reth_node_api::NodePrimitives; -use reth_optimism_chainspec::OpChainSpec; -use reth_optimism_evm::OpNextBlockEnvAttributes; -use reth_optimism_payload_builder::{OpPayloadBuilder, OpPayloadPrimitives}; +use reth_node_api::{BuildNextEnv, NodePrimitives}; +use reth_optimism_forks::OpHardforks; +use reth_optimism_payload_builder::{OpAttributes, OpPayloadBuilder, OpPayloadPrimitives}; use reth_optimism_txpool::OpPooledTx; -use reth_primitives_traits::SealedHeader; +use reth_primitives_traits::{SealedHeader, TxTy}; pub use reth_rpc_api::DebugExecutionWitnessApiServer; use reth_rpc_server_types::{result::internal_rpc_err, ToRpcResult}; use reth_storage_api::{ @@ -24,16 +22,16 @@ use std::{fmt::Debug, sync::Arc}; use tokio::sync::{oneshot, Semaphore}; /// An extension to the `debug_` namespace of the RPC API. -pub struct OpDebugWitnessApi { - inner: Arc>, +pub struct OpDebugWitnessApi { + inner: Arc>, } -impl OpDebugWitnessApi { +impl OpDebugWitnessApi { /// Creates a new instance of the `OpDebugWitnessApi`. pub fn new( provider: Provider, task_spawner: Box, - builder: OpPayloadBuilder, + builder: OpPayloadBuilder, ) -> Self { let semaphore = Arc::new(Semaphore::new(3)); let inner = OpDebugWitnessApiInner { provider, builder, task_spawner, semaphore }; @@ -41,13 +39,17 @@ impl OpDebugWitnessApi { } } -impl OpDebugWitnessApi +impl OpDebugWitnessApi where EvmConfig: ConfigureEvm, - Provider: NodePrimitivesProvider + BlockReaderIdExt

, + Provider: NodePrimitivesProvider> + + BlockReaderIdExt, { /// Fetches the parent header by hash. - fn parent_header(&self, parent_block_hash: B256) -> ProviderResult { + fn parent_header( + &self, + parent_block_hash: B256, + ) -> ProviderResult> { self.inner .provider .sealed_header_by_hash(parent_block_hash)? @@ -56,25 +58,28 @@ where } #[async_trait] -impl DebugExecutionWitnessApiServer - for OpDebugWitnessApi +impl DebugExecutionWitnessApiServer + for OpDebugWitnessApi where Pool: TransactionPool< Transaction: OpPooledTx::SignedTx>, > + 'static, - Provider: BlockReaderIdExt
+ Provider: BlockReaderIdExt
::BlockHeader> + NodePrimitivesProvider + StateProviderFactory - + ChainSpecProvider + + ChainSpecProvider + Clone + 'static, - EvmConfig: ConfigureEvm - + 'static, + EvmConfig: ConfigureEvm< + Primitives = Provider::Primitives, + NextBlockEnvCtx: BuildNextEnv, + > + 'static, + Attrs: OpAttributes>, { async fn execute_payload( &self, parent_block_hash: B256, - attributes: OpPayloadAttributes, + attributes: Attrs::RpcPayloadAttributes, ) -> RpcResult { let _permit = self.inner.semaphore.acquire().await; @@ -93,20 +98,24 @@ where } } -impl Clone for OpDebugWitnessApi { +impl Clone + for OpDebugWitnessApi +{ fn clone(&self) -> Self { Self { inner: Arc::clone(&self.inner) } } } -impl Debug for OpDebugWitnessApi { +impl Debug + for OpDebugWitnessApi +{ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("OpDebugWitnessApi").finish_non_exhaustive() } } -struct OpDebugWitnessApiInner { +struct OpDebugWitnessApiInner { provider: Provider, - builder: OpPayloadBuilder, + builder: OpPayloadBuilder, task_spawner: Box, semaphore: Arc, } diff --git a/crates/optimism/storage/Cargo.toml b/crates/optimism/storage/Cargo.toml index 0bb7c3a0bd3..aab6ee7d8e0 100644 --- a/crates/optimism/storage/Cargo.toml +++ b/crates/optimism/storage/Cargo.toml @@ -12,19 +12,14 @@ workspace = true [dependencies] # reth -reth-chainspec.workspace = true -reth-primitives-traits.workspace = true -reth-optimism-forks.workspace = true reth-optimism-primitives = { workspace = true, features = ["serde", "reth-codec"] } reth-storage-api = { workspace = true, features = ["db-api"] } # ethereum -alloy-primitives.workspace = true alloy-consensus.workspace = true [dev-dependencies] reth-codecs = { workspace = true, features = ["test-utils"] } -reth-db-api.workspace = true reth-prune-types.workspace = true reth-stages-types.workspace = true @@ -32,12 +27,8 @@ reth-stages-types.workspace = true default = ["std"] std = [ "reth-storage-api/std", - "alloy-primitives/std", "reth-prune-types/std", "reth-stages-types/std", "alloy-consensus/std", - "reth-chainspec/std", - "reth-optimism-forks/std", "reth-optimism-primitives/std", - "reth-primitives-traits/std", ] diff --git a/crates/optimism/storage/src/chain.rs b/crates/optimism/storage/src/chain.rs index 5df84eeae46..e56cd12f36d 100644 --- a/crates/optimism/storage/src/chain.rs +++ b/crates/optimism/storage/src/chain.rs @@ -1,78 +1,6 @@ -use alloc::{vec, vec::Vec}; use alloy_consensus::Header; -use alloy_primitives::BlockNumber; -use core::marker::PhantomData; -use reth_chainspec::{ChainSpecProvider, EthChainSpec, EthereumHardforks}; -use reth_optimism_forks::OpHardforks; use reth_optimism_primitives::OpTransactionSigned; -use reth_primitives_traits::{Block, FullBlockHeader, SignedTransaction}; -use reth_storage_api::{ - errors::ProviderResult, BlockBodyReader, BlockBodyWriter, DBProvider, ReadBodyInput, - StorageLocation, -}; +use reth_storage_api::EmptyBodyStorage; /// Optimism storage implementation. -#[derive(Debug, Clone, Copy)] -pub struct OptStorage(PhantomData<(T, H)>); - -impl BlockBodyWriter> - for OptStorage -where - T: SignedTransaction, - H: FullBlockHeader, -{ - fn write_block_bodies( - &self, - _provider: &Provider, - _bodies: Vec<(u64, Option>)>, - _write_to: StorageLocation, - ) -> ProviderResult<()> { - // noop - Ok(()) - } - - fn remove_block_bodies_above( - &self, - _provider: &Provider, - _block: BlockNumber, - _remove_from: StorageLocation, - ) -> ProviderResult<()> { - // noop - Ok(()) - } -} - -impl BlockBodyReader for OptStorage -where - Provider: ChainSpecProvider + DBProvider, - T: SignedTransaction, - H: FullBlockHeader, -{ - type Block = alloy_consensus::Block; - - fn read_block_bodies( - &self, - provider: &Provider, - inputs: Vec>, - ) -> ProviderResult::Body>> { - let chain_spec = provider.chain_spec(); - - let mut bodies = Vec::with_capacity(inputs.len()); - - for (header, transactions) in inputs { - let mut withdrawals = None; - if chain_spec.is_shanghai_active_at_timestamp(header.timestamp()) { - // after shanghai the body should have an empty withdrawals list - withdrawals.replace(vec![].into()); - } - - bodies.push(alloy_consensus::BlockBody:: { - transactions, - ommers: vec![], - withdrawals, - }); - } - - Ok(bodies) - } -} +pub type OpStorage = EmptyBodyStorage; diff --git a/crates/optimism/storage/src/lib.rs b/crates/optimism/storage/src/lib.rs index 8ef3d88d239..c2507925cfa 100644 --- a/crates/optimism/storage/src/lib.rs +++ b/crates/optimism/storage/src/lib.rs @@ -5,71 +5,30 @@ html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256", issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(feature = "std"), no_std)] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -extern crate alloc; - mod chain; -pub use chain::OptStorage; +pub use chain::OpStorage; #[cfg(test)] mod tests { use reth_codecs::{test_utils::UnusedBits, validate_bitflag_backwards_compat}; - use reth_db_api::models::{ - CompactClientVersion, CompactU256, CompactU64, StoredBlockBodyIndices, - StoredBlockWithdrawals, - }; - use reth_primitives_traits::Account; + use reth_prune_types::{PruneCheckpoint, PruneMode, PruneSegment}; - use reth_stages_types::{ - AccountHashingCheckpoint, CheckpointBlockRange, EntitiesCheckpoint, ExecutionCheckpoint, - HeadersCheckpoint, IndexHistoryCheckpoint, StageCheckpoint, StageUnitCheckpoint, - StorageHashingCheckpoint, - }; #[test] fn test_ensure_backwards_compatibility() { - assert_eq!(Account::bitflag_encoded_bytes(), 2); - assert_eq!(AccountHashingCheckpoint::bitflag_encoded_bytes(), 1); - assert_eq!(CheckpointBlockRange::bitflag_encoded_bytes(), 1); - assert_eq!(CompactClientVersion::bitflag_encoded_bytes(), 0); - assert_eq!(CompactU256::bitflag_encoded_bytes(), 1); - assert_eq!(CompactU64::bitflag_encoded_bytes(), 1); - assert_eq!(EntitiesCheckpoint::bitflag_encoded_bytes(), 1); - assert_eq!(ExecutionCheckpoint::bitflag_encoded_bytes(), 0); - assert_eq!(HeadersCheckpoint::bitflag_encoded_bytes(), 0); - assert_eq!(IndexHistoryCheckpoint::bitflag_encoded_bytes(), 0); - assert_eq!(PruneCheckpoint::bitflag_encoded_bytes(), 1); assert_eq!(PruneMode::bitflag_encoded_bytes(), 1); assert_eq!(PruneSegment::bitflag_encoded_bytes(), 1); - assert_eq!(StageCheckpoint::bitflag_encoded_bytes(), 1); - assert_eq!(StageUnitCheckpoint::bitflag_encoded_bytes(), 1); - assert_eq!(StoredBlockBodyIndices::bitflag_encoded_bytes(), 1); - assert_eq!(StoredBlockWithdrawals::bitflag_encoded_bytes(), 0); - assert_eq!(StorageHashingCheckpoint::bitflag_encoded_bytes(), 1); // In case of failure, refer to the documentation of the // [`validate_bitflag_backwards_compat`] macro for detailed instructions on handling // it. - validate_bitflag_backwards_compat!(Account, UnusedBits::NotZero); - validate_bitflag_backwards_compat!(AccountHashingCheckpoint, UnusedBits::NotZero); - validate_bitflag_backwards_compat!(CheckpointBlockRange, UnusedBits::Zero); - validate_bitflag_backwards_compat!(CompactClientVersion, UnusedBits::Zero); - validate_bitflag_backwards_compat!(CompactU256, UnusedBits::NotZero); - validate_bitflag_backwards_compat!(CompactU64, UnusedBits::NotZero); - validate_bitflag_backwards_compat!(EntitiesCheckpoint, UnusedBits::Zero); - validate_bitflag_backwards_compat!(ExecutionCheckpoint, UnusedBits::Zero); - validate_bitflag_backwards_compat!(HeadersCheckpoint, UnusedBits::Zero); - validate_bitflag_backwards_compat!(IndexHistoryCheckpoint, UnusedBits::Zero); + validate_bitflag_backwards_compat!(PruneCheckpoint, UnusedBits::NotZero); validate_bitflag_backwards_compat!(PruneMode, UnusedBits::Zero); validate_bitflag_backwards_compat!(PruneSegment, UnusedBits::Zero); - validate_bitflag_backwards_compat!(StageCheckpoint, UnusedBits::NotZero); - validate_bitflag_backwards_compat!(StageUnitCheckpoint, UnusedBits::Zero); - validate_bitflag_backwards_compat!(StoredBlockBodyIndices, UnusedBits::Zero); - validate_bitflag_backwards_compat!(StoredBlockWithdrawals, UnusedBits::Zero); - validate_bitflag_backwards_compat!(StorageHashingCheckpoint, UnusedBits::NotZero); } } diff --git a/crates/optimism/txpool/Cargo.toml b/crates/optimism/txpool/Cargo.toml index 6eb087f0980..9e5ea5cbb6e 100644 --- a/crates/optimism/txpool/Cargo.toml +++ b/crates/optimism/txpool/Cargo.toml @@ -37,6 +37,7 @@ op-revm.workspace = true # optimism op-alloy-consensus.workspace = true op-alloy-flz.workspace = true +op-alloy-rpc-types.workspace = true reth-optimism-evm.workspace = true reth-optimism-forks.workspace = true reth-optimism-primitives.workspace = true diff --git a/crates/optimism/txpool/src/estimated_da_size.rs b/crates/optimism/txpool/src/estimated_da_size.rs new file mode 100644 index 00000000000..74c21d75f16 --- /dev/null +++ b/crates/optimism/txpool/src/estimated_da_size.rs @@ -0,0 +1,9 @@ +//! Additional support for estimating the data availability size of transactions. + +/// Helper trait that allows attaching an estimated data availability size. +pub trait DataAvailabilitySized { + /// Get the estimated data availability size of the transaction. + /// + /// Note: it is expected that this value will be cached internally. + fn estimated_da_size(&self) -> u64; +} diff --git a/crates/optimism/txpool/src/lib.rs b/crates/optimism/txpool/src/lib.rs index d5de8774fd9..43421ed3b30 100644 --- a/crates/optimism/txpool/src/lib.rs +++ b/crates/optimism/txpool/src/lib.rs @@ -6,7 +6,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] mod validator; pub use validator::{OpL1BlockInfo, OpTransactionValidator}; @@ -19,6 +19,7 @@ mod error; pub mod interop; pub mod maintain; pub use error::InvalidCrossTx; +pub mod estimated_da_size; use reth_transaction_pool::{CoinbaseTipOrdering, Pool, TransactionValidationTaskExecutor}; diff --git a/crates/optimism/txpool/src/maintain.rs b/crates/optimism/txpool/src/maintain.rs index 571b1ab7b32..c071bf708e4 100644 --- a/crates/optimism/txpool/src/maintain.rs +++ b/crates/optimism/txpool/src/maintain.rs @@ -12,14 +12,15 @@ use crate::{ interop::{is_stale_interop, is_valid_interop, MaybeInteropTransaction}, supervisor::SupervisorClient, }; -use alloy_consensus::{conditional::BlockConditionalAttributes, BlockHeader, Transaction}; +use alloy_consensus::{conditional::BlockConditionalAttributes, BlockHeader}; use futures_util::{future::BoxFuture, FutureExt, Stream, StreamExt}; -use metrics::Gauge; +use metrics::{Gauge, Histogram}; use reth_chain_state::CanonStateNotification; use reth_metrics::{metrics::Counter, Metrics}; use reth_primitives_traits::NodePrimitives; use reth_transaction_pool::{error::PoolTransactionError, PoolTransaction, TransactionPool}; -use std::sync::Arc; +use std::time::Instant; +use tracing::warn; /// Transaction pool maintenance metrics #[derive(Metrics)] @@ -50,7 +51,8 @@ struct MaintainPoolInteropMetrics { /// Counter for interop transactions that became stale and need revalidation stale_interop_transactions: Counter, // TODO: we also should add metric for (hash, counter) to check number of validation per tx - // TODO: we should add some timing metric in here to check supervisor congestion + /// Histogram for measuring supervisor revalidation duration (congestion metric) + supervisor_revalidation_duration_seconds: Histogram, } impl MaintainPoolInteropMetrics { @@ -67,6 +69,12 @@ impl MaintainPoolInteropMetrics { fn inc_stale_tx_interop(&self, count: usize) { self.stale_interop_transactions.increment(count as u64); } + + /// Record supervisor revalidation duration + #[inline] + fn record_supervisor_duration(&self, duration: std::time::Duration) { + self.supervisor_revalidation_duration_seconds.record(duration.as_secs_f64()); + } } /// Returns a spawnable future for maintaining the state of the conditional txs in the transaction /// pool. @@ -153,7 +161,7 @@ pub async fn maintain_transaction_pool_interop( St: Stream> + Send + Unpin + 'static, { let metrics = MaintainPoolInteropMetrics::default(); - let supervisor_client = Arc::new(supervisor_client); + loop { let Some(event) = events.next().await else { break }; if let CanonStateNotification::Commit { new } = event { @@ -161,58 +169,61 @@ pub async fn maintain_transaction_pool_interop( let mut to_remove = Vec::new(); let mut to_revalidate = Vec::new(); let mut interop_count = 0; - for tx in &pool.pooled_transactions() { - // Only interop txs have this field set - if let Some(interop) = tx.transaction.interop_deadline() { + + // scan all pooled interop transactions + for pooled_tx in pool.pooled_transactions() { + if let Some(interop_deadline_val) = pooled_tx.transaction.interop_deadline() { interop_count += 1; - if !is_valid_interop(interop, timestamp) { - // That means tx didn't revalidated during [`OFFSET_TIME`] time - // We could assume that it won't be validated at all and remove it - to_remove.push(*tx.hash()); - } else if is_stale_interop(interop, timestamp, OFFSET_TIME) { - // If tx has less then [`OFFSET_TIME`] of valid time we revalidate it - to_revalidate.push(tx.clone()) + if !is_valid_interop(interop_deadline_val, timestamp) { + to_remove.push(*pooled_tx.transaction.hash()); + } else if is_stale_interop(interop_deadline_val, timestamp, OFFSET_TIME) { + to_revalidate.push(pooled_tx.transaction.clone()); } } } metrics.set_interop_txs_in_pool(interop_count); + if !to_revalidate.is_empty() { metrics.inc_stale_tx_interop(to_revalidate.len()); - let checks_stream = - futures_util::stream::iter(to_revalidate.into_iter().map(|tx| { - let supervisor_client = supervisor_client.clone(); - async move { - let check = supervisor_client - .is_valid_cross_tx( - tx.transaction.access_list(), - tx.transaction.hash(), - timestamp, - Some(TRANSACTION_VALIDITY_WINDOW), - // We could assume that interop is enabled, because - // tx.transaction.interop() would be set only in - // this case - true, - ) - .await; - (tx.clone(), check) + + let revalidation_start = Instant::now(); + let revalidation_stream = supervisor_client.revalidate_interop_txs_stream( + to_revalidate, + timestamp, + TRANSACTION_VALIDITY_WINDOW, + MAX_SUPERVISOR_QUERIES, + ); + + futures_util::pin_mut!(revalidation_stream); + + while let Some((tx_item_from_stream, validation_result)) = + revalidation_stream.next().await + { + match validation_result { + Some(Ok(())) => { + tx_item_from_stream + .set_interop_deadline(timestamp + TRANSACTION_VALIDITY_WINDOW); } - })) - .buffered(MAX_SUPERVISOR_QUERIES); - futures_util::pin_mut!(checks_stream); - while let Some((tx, check)) = checks_stream.next().await { - if let Some(Err(err)) = check { - // We remove only bad transaction. If error caused by supervisor instability - // or other fixable issues transaction would be validated on next state - // change, so we ignore it - if err.is_bad_transaction() { - to_remove.push(*tx.transaction.hash()); + Some(Err(err)) => { + if err.is_bad_transaction() { + to_remove.push(*tx_item_from_stream.hash()); + } + } + None => { + warn!( + target: "txpool", + hash = %tx_item_from_stream.hash(), + "Interop transaction no longer considered cross-chain during revalidation; removing." + ); + to_remove.push(*tx_item_from_stream.hash()); } - } else { - tx.transaction.set_interop_deadline(timestamp + TRANSACTION_VALIDITY_WINDOW) } } + + metrics.record_supervisor_duration(revalidation_start.elapsed()); } + if !to_remove.is_empty() { let removed = pool.remove_transactions(to_remove); metrics.inc_removed_tx_interop(removed.len()); diff --git a/crates/optimism/txpool/src/supervisor/access_list.rs b/crates/optimism/txpool/src/supervisor/access_list.rs index 9b3e4b0f2b4..7565c960c38 100644 --- a/crates/optimism/txpool/src/supervisor/access_list.rs +++ b/crates/optimism/txpool/src/supervisor/access_list.rs @@ -32,7 +32,7 @@ pub fn parse_access_list_items_to_inbox_entries<'a>( /// Max 3 inbox entries can exist per [`AccessListItem`] that points to [`CROSS_L2_INBOX_ADDRESS`]. /// /// Returns `Vec::new()` if [`AccessListItem`] address doesn't point to [`CROSS_L2_INBOX_ADDRESS`]. -// TODO: add url to spec once [pr](https://github.com/ethereum-optimism/specs/pull/612) is merged +// Access-list spec: fn parse_access_list_item_to_inbox_entries( access_list_item: &AccessListItem, ) -> Option> { diff --git a/crates/optimism/txpool/src/supervisor/client.rs b/crates/optimism/txpool/src/supervisor/client.rs index 075a5c92bbb..b362fae2e10 100644 --- a/crates/optimism/txpool/src/supervisor/client.rs +++ b/crates/optimism/txpool/src/supervisor/client.rs @@ -1,17 +1,24 @@ //! This is our custom implementation of validator struct use crate::{ + interop::MaybeInteropTransaction, supervisor::{ metrics::SupervisorMetrics, parse_access_list_items_to_inbox_entries, ExecutingDescriptor, InteropTxValidatorError, }, InvalidCrossTx, }; +use alloy_consensus::Transaction; use alloy_eips::eip2930::AccessList; use alloy_primitives::{TxHash, B256}; use alloy_rpc_client::ReqwestClient; -use futures_util::future::BoxFuture; +use futures_util::{ + future::BoxFuture, + stream::{self, StreamExt}, + Stream, +}; use op_alloy_consensus::interop::SafetyLevel; +use reth_transaction_pool::PoolTransaction; use std::{ borrow::Cow, future::IntoFuture, @@ -21,7 +28,7 @@ use std::{ use tracing::trace; /// Supervisor hosted by op-labs -// TODO: This should be changes to actual supervisor url +// TODO: This should be changed to actual supervisor url pub const DEFAULT_SUPERVISOR_URL: &str = "http://localhost:1337/"; /// The default request timeout to use @@ -106,11 +113,55 @@ impl SupervisorClient { ) .await { + self.inner.metrics.increment_metrics_for_error(&err); trace!(target: "txpool", hash=%hash, err=%err, "Cross chain transaction invalid"); return Some(Err(InvalidCrossTx::ValidationError(err))); } Some(Ok(())) } + + /// Creates a stream that revalidates interop transactions against the supervisor. + /// Returns + /// An implementation of `Stream` that is `Send`-able and tied to the lifetime `'a` of `self`. + /// Each item yielded by the stream is a tuple `(TItem, Option>)`. + /// - The first element is the original `TItem` that was revalidated. + /// - The second element is the `Option>` describes the outcome + /// - `None`: Transaction was not identified as a cross-chain candidate by initial checks. + /// - `Some(Ok(()))`: Supervisor confirmed the transaction is valid. + /// - `Some(Err(InvalidCrossTx))`: Supervisor indicated the transaction is invalid. + pub fn revalidate_interop_txs_stream<'a, TItem, InputIter>( + &'a self, + txs_to_revalidate: InputIter, + current_timestamp: u64, + revalidation_window: u64, + max_concurrent_queries: usize, + ) -> impl Stream>)> + Send + 'a + where + InputIter: IntoIterator + Send + 'a, + InputIter::IntoIter: Send + 'a, + TItem: + MaybeInteropTransaction + PoolTransaction + Transaction + Clone + Send + Sync + 'static, + { + stream::iter(txs_to_revalidate.into_iter().map(move |tx_item| { + let client_for_async_task = self.clone(); + + async move { + let validation_result = client_for_async_task + .is_valid_cross_tx( + tx_item.access_list(), + tx_item.hash(), + current_timestamp, + Some(revalidation_window), + true, + ) + .await; + + // return the original transaction paired with its validation result. + (tx_item, validation_result) + } + })) + .buffered(max_concurrent_queries) + } } /// Holds supervisor data. Inner type of [`SupervisorClient`]. diff --git a/crates/optimism/txpool/src/supervisor/errors.rs b/crates/optimism/txpool/src/supervisor/errors.rs index 7a95f3616d6..9993a5ca5d1 100644 --- a/crates/optimism/txpool/src/supervisor/errors.rs +++ b/crates/optimism/txpool/src/supervisor/errors.rs @@ -1,85 +1,6 @@ use alloy_json_rpc::RpcError; use core::error; -use derive_more; - -/// Supervisor protocol error codes. -/// -/// Specs: -#[derive(thiserror::Error, Debug, Clone, Copy, PartialEq, Eq, derive_more::TryFrom)] -#[repr(i64)] -#[try_from(repr)] -pub enum InvalidInboxEntry { - // -3204XX DEADLINE_EXCEEDED errors - /// Happens when a chain database is not initialized yet. - #[error("chain database is not initialized")] - UninitializedChainDatabase = -320400, - - // -3205XX NOT_FOUND errors - /// Happens when we try to retrieve data that is not available (pruned). - /// It may also happen if we erroneously skip data, that was not considered a conflict, if the - /// DB is corrupted. - #[error("data was skipped or pruned and is not available")] - SkippedData = -320500, - - /// Happens when a chain is unknown, not in the dependency set. - #[error("unsupported chain id")] - UnknownChain = -320501, - - // -3206XX ALREADY_EXISTS errors - /// Happens when we know for sure that there is different canonical data. - #[error("conflicting data exists in the database")] - ConflictingData = -320600, - - /// Happens when data is accepted as compatible, but did not change anything. - /// This happens when a node is deriving an L2 block we already know of being - /// derived from the given source, - /// but without path to skip forward to newer source blocks without doing the known - /// derivation work first. - #[error("data is already known and didn't change anything")] - IneffectiveData = -320601, - - // -3209XX FAILED_PRECONDITION errors - /// Happens when you try to add data to the DB, but it does not actually fit onto - /// the latest data. - /// (by being too old or new). - #[error("data is out of order (too old or new)")] - OutOfOrder = -320900, - - /// Happens when we know for sure that a replacement block is needed before progress - /// can be made. - #[error("waiting for replacement block before progress can be made")] - AwaitingReplacement = -320901, - - // -3211XX OUT_OF_RANGE errors - /// Happens when data is accessed, but access is not allowed, because of a limited - /// scope. - /// E.g. when limiting scope to L2 blocks derived from a specific subset of the L1 - /// chain. - #[error("data access not allowed due to limited scope")] - OutOfScope = -321100, - - // -3212XX UNIMPLEMENTED errors - /// Happens when you try to get the previous block of the first block. - /// E.g. when trying to determine the previous source block for the first L1 block - /// in the database. - #[error("cannot get parent of first block in database")] - NoParentForFirstBlock = -321200, - - // -3214XX UNAVAILABLE errors - /// Happens when data is just not yet available. - #[error("data is not yet available (from the future)")] - FutureData = -321401, - - // -3215XX DATA_LOSS errors - /// Happens when we search the DB, know the data may be there, but is not (e.g. - /// different revision). - #[error("data may exist but was not found (possibly different revision)")] - MissedData = -321500, - - /// Happens when the underlying DB has some I/O issue. - #[error("underlying database has I/O issues or is corrupted")] - DataCorruption = -321501, -} +use op_alloy_rpc_types::SuperchainDAError; /// Failures occurring during validation of inbox entries. #[derive(thiserror::Error, Debug)] @@ -90,7 +11,7 @@ pub enum InteropTxValidatorError { /// Message does not satisfy validation requirements #[error(transparent)] - InvalidEntry(#[from] InvalidInboxEntry), + InvalidEntry(#[from] SuperchainDAError), /// Catch-all variant. #[error("supervisor server error: {0}")] @@ -115,10 +36,10 @@ impl InteropTxValidatorError { { // Try to extract error details from the RPC error if let Some(error_payload) = err.as_error_resp() { - let code = error_payload.code; + let code = error_payload.code as i32; - // Try to convert the error code to an InvalidInboxEntry variant - if let Ok(invalid_entry) = InvalidInboxEntry::try_from(code) { + // Try to convert the error code to an SuperchainDAError variant + if let Ok(invalid_entry) = SuperchainDAError::try_from(code) { return Self::InvalidEntry(invalid_entry); } } diff --git a/crates/optimism/txpool/src/supervisor/metrics.rs b/crates/optimism/txpool/src/supervisor/metrics.rs index 1ccb2178916..cb51a52bfc5 100644 --- a/crates/optimism/txpool/src/supervisor/metrics.rs +++ b/crates/optimism/txpool/src/supervisor/metrics.rs @@ -1,6 +1,11 @@ -//! Optimism supervisor and sequencer metrics +//! Optimism supervisor metrics -use reth_metrics::{metrics::Histogram, Metrics}; +use crate::supervisor::InteropTxValidatorError; +use op_alloy_rpc_types::SuperchainDAError; +use reth_metrics::{ + metrics::{Counter, Histogram}, + Metrics, +}; use std::time::Duration; /// Optimism supervisor metrics @@ -9,6 +14,29 @@ use std::time::Duration; pub struct SupervisorMetrics { /// How long it takes to query the supervisor in the Optimism transaction pool pub(crate) supervisor_query_latency: Histogram, + + /// Counter for the number of times data was skipped + pub(crate) skipped_data_count: Counter, + /// Counter for the number of times an unknown chain was encountered + pub(crate) unknown_chain_count: Counter, + /// Counter for the number of times conflicting data was encountered + pub(crate) conflicting_data_count: Counter, + /// Counter for the number of times ineffective data was encountered + pub(crate) ineffective_data_count: Counter, + /// Counter for the number of times data was out of order + pub(crate) out_of_order_count: Counter, + /// Counter for the number of times data was awaiting replacement + pub(crate) awaiting_replacement_count: Counter, + /// Counter for the number of times data was out of scope + pub(crate) out_of_scope_count: Counter, + /// Counter for the number of times there was no parent for the first block + pub(crate) no_parent_for_first_block_count: Counter, + /// Counter for the number of times future data was encountered + pub(crate) future_data_count: Counter, + /// Counter for the number of times data was missed + pub(crate) missed_data_count: Counter, + /// Counter for the number of times data corruption was encountered + pub(crate) data_corruption_count: Counter, } impl SupervisorMetrics { @@ -17,20 +45,28 @@ impl SupervisorMetrics { pub fn record_supervisor_query(&self, duration: Duration) { self.supervisor_query_latency.record(duration.as_secs_f64()); } -} - -/// Optimism sequencer metrics -#[derive(Metrics, Clone)] -#[metrics(scope = "optimism_transaction_pool.sequencer")] -pub struct SequencerMetrics { - /// How long it takes to forward a transaction to the sequencer - pub(crate) sequencer_forward_latency: Histogram, -} -impl SequencerMetrics { - /// Records the duration it took to forward a transaction - #[inline] - pub fn record_forward_latency(&self, duration: Duration) { - self.sequencer_forward_latency.record(duration.as_secs_f64()); + /// Increments the metrics for the given error + pub fn increment_metrics_for_error(&self, error: &InteropTxValidatorError) { + if let InteropTxValidatorError::InvalidEntry(inner) = error { + match inner { + SuperchainDAError::SkippedData => self.skipped_data_count.increment(1), + SuperchainDAError::UnknownChain => self.unknown_chain_count.increment(1), + SuperchainDAError::ConflictingData => self.conflicting_data_count.increment(1), + SuperchainDAError::IneffectiveData => self.ineffective_data_count.increment(1), + SuperchainDAError::OutOfOrder => self.out_of_order_count.increment(1), + SuperchainDAError::AwaitingReplacement => { + self.awaiting_replacement_count.increment(1) + } + SuperchainDAError::OutOfScope => self.out_of_scope_count.increment(1), + SuperchainDAError::NoParentForFirstBlock => { + self.no_parent_for_first_block_count.increment(1) + } + SuperchainDAError::FutureData => self.future_data_count.increment(1), + SuperchainDAError::MissedData => self.missed_data_count.increment(1), + SuperchainDAError::DataCorruption => self.data_corruption_count.increment(1), + _ => {} + } + } } } diff --git a/crates/optimism/txpool/src/transaction.rs b/crates/optimism/txpool/src/transaction.rs index 23d1952576e..741ec541422 100644 --- a/crates/optimism/txpool/src/transaction.rs +++ b/crates/optimism/txpool/src/transaction.rs @@ -1,8 +1,14 @@ -use crate::{conditional::MaybeConditionalTransaction, interop::MaybeInteropTransaction}; -use alloy_consensus::{ - transaction::Recovered, BlobTransactionSidecar, BlobTransactionValidationError, Typed2718, +use crate::{ + conditional::MaybeConditionalTransaction, estimated_da_size::DataAvailabilitySized, + interop::MaybeInteropTransaction, +}; +use alloy_consensus::{transaction::Recovered, BlobTransactionValidationError, Typed2718}; +use alloy_eips::{ + eip2718::{Encodable2718, WithEncoded}, + eip2930::AccessList, + eip7594::BlobTransactionSidecarVariant, + eip7702::SignedAuthorization, }; -use alloy_eips::{eip2930::AccessList, eip7702::SignedAuthorization}; use alloy_primitives::{Address, Bytes, TxHash, TxKind, B256, U256}; use alloy_rpc_types_eth::erc4337::TransactionConditional; use c_kzg::KzgSettings; @@ -12,9 +18,12 @@ use reth_primitives_traits::{InMemorySize, SignedTransaction}; use reth_transaction_pool::{ EthBlobTransactionSidecar, EthPoolTransaction, EthPooledTransaction, PoolTransaction, }; -use std::sync::{ - atomic::{AtomicU64, Ordering}, - Arc, OnceLock, +use std::{ + borrow::Cow, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, OnceLock, + }, }; /// Marker for no-interop transactions @@ -60,14 +69,14 @@ impl OpPooledTransaction { } } - /// Returns the estimated compressed size of a transaction in bytes scaled by 1e6. + /// Returns the estimated compressed size of a transaction in bytes. /// This value is computed based on the following formula: - /// `max(minTransactionSize, intercept + fastlzCoef*fastlzSize)` + /// `max(minTransactionSize, intercept + fastlzCoef*fastlzSize) / 1e6` /// Uses cached EIP-2718 encoded bytes to avoid recomputing the encoding for each estimation. pub fn estimated_compressed_size(&self) -> u64 { *self .estimated_tx_compressed_size - .get_or_init(|| op_alloy_flz::tx_estimated_size_fjord(self.encoded_2718())) + .get_or_init(|| op_alloy_flz::tx_estimated_size_fjord_bytes(self.encoded_2718())) } /// Returns lazily computed EIP-2718 encoded bytes of the transaction. @@ -106,6 +115,12 @@ impl MaybeInteropTransaction for OpPooledTransaction } } +impl DataAvailabilitySized for OpPooledTransaction { + fn estimated_da_size(&self) -> u64 { + self.estimated_compressed_size() + } +} + impl PoolTransaction for OpPooledTransaction where Cons: SignedTransaction + From, @@ -123,6 +138,11 @@ where self.inner.transaction } + fn into_consensus_with2718(self) -> WithEncoded> { + let encoding = self.encoded_2718().clone(); + self.inner.transaction.into_encoded_with(encoding) + } + fn from_pooled(tx: Recovered) -> Self { let encoded_len = tx.encode_2718_len(); Self::new(tx.convert(), encoded_len) @@ -247,21 +267,21 @@ where fn try_into_pooled_eip4844( self, - _sidecar: Arc, + _sidecar: Arc, ) -> Option> { None } fn try_from_eip4844( _tx: Recovered, - _sidecar: BlobTransactionSidecar, + _sidecar: BlobTransactionSidecarVariant, ) -> Option { None } fn validate_blob( &self, - _sidecar: &BlobTransactionSidecar, + _sidecar: &BlobTransactionSidecarVariant, _settings: &KzgSettings, ) -> Result<(), BlobTransactionValidationError> { Err(BlobTransactionValidationError::NotBlobTransaction(self.ty())) @@ -271,12 +291,21 @@ where /// Helper trait to provide payload builder with access to conditionals and encoded bytes of /// transaction. pub trait OpPooledTx: - MaybeConditionalTransaction + MaybeInteropTransaction + PoolTransaction + MaybeConditionalTransaction + MaybeInteropTransaction + PoolTransaction + DataAvailabilitySized { + /// Returns the EIP-2718 encoded bytes of the transaction. + fn encoded_2718(&self) -> Cow<'_, Bytes>; } -impl OpPooledTx for T where - T: MaybeConditionalTransaction + MaybeInteropTransaction + PoolTransaction + +impl OpPooledTx for OpPooledTransaction +where + Cons: SignedTransaction + From, + Pooled: SignedTransaction + TryFrom, + >::Error: core::error::Error, { + fn encoded_2718(&self) -> Cow<'_, Bytes> { + Cow::Borrowed(self.encoded_2718()) + } } #[cfg(test)] @@ -308,11 +337,11 @@ mod tests { source_hash: Default::default(), from: signer, to: TxKind::Create, - mint: None, + mint: 0, value: U256::ZERO, gas_limit: 0, eth_tx_value: None, - eth_value: None, + eth_value: 0, is_system_transaction: false, input: Default::default(), }; diff --git a/crates/optimism/txpool/src/validator.rs b/crates/optimism/txpool/src/validator.rs index d56a6ead55e..2a064e15587 100644 --- a/crates/optimism/txpool/src/validator.rs +++ b/crates/optimism/txpool/src/validator.rs @@ -1,16 +1,15 @@ -use crate::{interop::MaybeInteropTransaction, supervisor::SupervisorClient, InvalidCrossTx}; +use crate::{supervisor::SupervisorClient, InvalidCrossTx, OpPooledTx}; use alloy_consensus::{BlockHeader, Transaction}; -use alloy_eips::Encodable2718; use op_revm::L1BlockInfo; use parking_lot::RwLock; use reth_chainspec::ChainSpecProvider; +use reth_mantle_forks::MantleHardforks; use reth_optimism_evm::RethL1BlockInfo; use reth_optimism_forks::OpHardforks; -use reth_mantle_forks::MantleHardforks; use reth_primitives_traits::{ transaction::error::InvalidTransactionError, Block, BlockBody, GotExpected, SealedBlock, }; -use reth_storage_api::{BlockReaderIdExt, StateProvider, StateProviderFactory}; +use reth_storage_api::{AccountInfoReader, BlockReaderIdExt, StateProviderFactory}; use reth_transaction_pool::{ error::InvalidPoolTransactionError, EthPoolTransaction, EthTransactionValidator, TransactionOrigin, TransactionValidationOutcome, TransactionValidator, @@ -30,8 +29,6 @@ pub struct OpL1BlockInfo { l1_block_info: RwLock, /// Current block timestamp. timestamp: AtomicU64, - /// Current block number. - number: AtomicU64, } impl OpL1BlockInfo { @@ -45,7 +42,7 @@ impl OpL1BlockInfo { #[derive(Debug, Clone)] pub struct OpTransactionValidator { /// The type that performs the actual validation. - inner: EthTransactionValidator, + inner: Arc>, /// Additional block info required for validation. block_info: Arc, /// If true, ensure that the transaction's sender has enough balance to cover the L1 gas fee @@ -92,8 +89,10 @@ impl OpTransactionValidator { impl OpTransactionValidator where - Client: ChainSpecProvider + StateProviderFactory + BlockReaderIdExt, - Tx: EthPoolTransaction + MaybeInteropTransaction, + Client: ChainSpecProvider + + StateProviderFactory + + BlockReaderIdExt, + Tx: EthPoolTransaction + OpPooledTx, { /// Create a new [`OpTransactionValidator`]. pub fn new(inner: EthTransactionValidator) -> Self { @@ -105,7 +104,6 @@ where // so that we will accept txs into the pool before the first block if block.header().number() == 0 { this.block_info.timestamp.store(block.header().timestamp(), Ordering::Relaxed); - this.block_info.number.store(block.header().number(), Ordering::Relaxed); } else { this.update_l1_block_info(block.header(), block.body().transactions().first()); } @@ -120,7 +118,7 @@ where block_info: OpL1BlockInfo, ) -> Self { Self { - inner, + inner: Arc::new(inner), block_info: Arc::new(block_info), require_l1_data_gas_fee: true, supervisor_client: None, @@ -143,10 +141,9 @@ where T: Transaction, { self.block_info.timestamp.store(header.timestamp(), Ordering::Relaxed); - self.block_info.number.store(header.number(), Ordering::Relaxed); - if let Some(Ok(cost_addition)) = tx.map(reth_optimism_evm::extract_l1_info_from_tx) { - *self.block_info.l1_block_info.write() = cost_addition; + if let Some(Ok(l1_block_info)) = tx.map(reth_optimism_evm::extract_l1_info_from_tx) { + *self.block_info.l1_block_info.write() = l1_block_info; } if self.chain_spec().is_interop_active_at_timestamp(header.timestamp()) { @@ -183,13 +180,13 @@ where &self, origin: TransactionOrigin, transaction: Tx, - state: &mut Option>, + state: &mut Option>, ) -> TransactionValidationOutcome { if transaction.is_eip4844() { return TransactionValidationOutcome::Invalid( transaction, InvalidTransactionError::TxTypeNotSupported.into(), - ) + ); } // Interop cross tx validation @@ -201,7 +198,7 @@ where } err => InvalidPoolTransactionError::Other(Box::new(err)), }; - return TransactionValidationOutcome::Invalid(transaction, err) + return TransactionValidationOutcome::Invalid(transaction, err); } Some(Ok(_)) => { // valid interop tx @@ -217,21 +214,6 @@ where self.apply_op_checks(outcome) } - /// Validates all given transactions. - /// - /// Returns all outcomes for the given transactions in the same order. - /// - /// See also [`Self::validate_one`] - pub async fn validate_all( - &self, - transactions: Vec<(TransactionOrigin, Tx)>, - ) -> Vec> { - futures_util::future::join_all( - transactions.into_iter().map(|(origin, tx)| self.validate_one(origin, tx)), - ) - .await - } - /// Performs the necessary opstack specific checks based on top of the regular eth outcome. fn apply_op_checks( &self, @@ -239,7 +221,7 @@ where ) -> TransactionValidationOutcome { if !self.requires_l1_data_gas_fee() { // no need to check L1 gas fee - return outcome + return outcome; } // ensure that the account has enough balance to cover the L1 gas cost if let TransactionValidationOutcome::Valid { @@ -253,9 +235,7 @@ where { let mut l1_block_info = self.block_info.l1_block_info.read().clone(); - let mut encoded = Vec::with_capacity(valid_tx.transaction().encoded_length()); - let tx = valid_tx.transaction().clone_into_consensus(); - tx.encode_2718(&mut encoded); + let encoded = valid_tx.transaction().encoded_2718(); let cost_addition = match l1_block_info.l1_tx_data_fee( self.chain_spec(), @@ -278,7 +258,7 @@ where GotExpected { got: balance, expected: cost }.into(), ) .into(), - ) + ); } return TransactionValidationOutcome::Valid { @@ -288,7 +268,7 @@ where propagate, bytecode_hash, authorities, - } + }; } outcome } @@ -312,8 +292,10 @@ where impl TransactionValidator for OpTransactionValidator where - Client: ChainSpecProvider + StateProviderFactory + BlockReaderIdExt, - Tx: EthPoolTransaction + MaybeInteropTransaction, + Client: ChainSpecProvider + + StateProviderFactory + + BlockReaderIdExt, + Tx: EthPoolTransaction + OpPooledTx, { type Transaction = Tx; @@ -329,7 +311,21 @@ where &self, transactions: Vec<(TransactionOrigin, Self::Transaction)>, ) -> Vec> { - self.validate_all(transactions).await + futures_util::future::join_all( + transactions.into_iter().map(|(origin, tx)| self.validate_one(origin, tx)), + ) + .await + } + + async fn validate_transactions_with_origin( + &self, + origin: TransactionOrigin, + transactions: impl IntoIterator + Send, + ) -> Vec> { + futures_util::future::join_all( + transactions.into_iter().map(|tx| self.validate_one(origin, tx)), + ) + .await } fn on_new_head_block(&self, new_tip_block: &SealedBlock) diff --git a/crates/payload/basic/src/lib.rs b/crates/payload/basic/src/lib.rs index 048a55b8486..aa2b1f66802 100644 --- a/crates/payload/basic/src/lib.rs +++ b/crates/payload/basic/src/lib.rs @@ -6,7 +6,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] use crate::metrics::PayloadBuilderMetrics; use alloy_eips::merge::SLOT_DURATION; @@ -455,6 +455,10 @@ where Ok(self.config.attributes.clone()) } + fn payload_timestamp(&self) -> Result { + Ok(self.config.attributes.timestamp()) + } + fn resolve_kind( &mut self, kind: PayloadKind, @@ -583,15 +587,15 @@ where let this = self.get_mut(); // check if there is a better payload before returning the best payload - if let Some(fut) = Pin::new(&mut this.maybe_better).as_pin_mut() { - if let Poll::Ready(res) = fut.poll(cx) { - this.maybe_better = None; - if let Ok(Some(payload)) = res.map(|out| out.into_payload()) - .inspect_err(|err| warn!(target: "payload_builder", %err, "failed to resolve pending payload")) - { - debug!(target: "payload_builder", "resolving better payload"); - return Poll::Ready(Ok(payload)) - } + if let Some(fut) = Pin::new(&mut this.maybe_better).as_pin_mut() && + let Poll::Ready(res) = fut.poll(cx) + { + this.maybe_better = None; + if let Ok(Some(payload)) = res.map(|out| out.into_payload()).inspect_err( + |err| warn!(target: "payload_builder", %err, "failed to resolve pending payload"), + ) { + debug!(target: "payload_builder", "resolving better payload"); + return Poll::Ready(Ok(payload)) } } @@ -600,20 +604,20 @@ where return Poll::Ready(Ok(best)) } - if let Some(fut) = Pin::new(&mut this.empty_payload).as_pin_mut() { - if let Poll::Ready(res) = fut.poll(cx) { - this.empty_payload = None; - return match res { - Ok(res) => { - if let Err(err) = &res { - warn!(target: "payload_builder", %err, "failed to resolve empty payload"); - } else { - debug!(target: "payload_builder", "resolving empty payload"); - } - Poll::Ready(res) + if let Some(fut) = Pin::new(&mut this.empty_payload).as_pin_mut() && + let Poll::Ready(res) = fut.poll(cx) + { + this.empty_payload = None; + return match res { + Ok(res) => { + if let Err(err) = &res { + warn!(target: "payload_builder", %err, "failed to resolve empty payload"); + } else { + debug!(target: "payload_builder", "resolving empty payload"); } - Err(err) => Poll::Ready(Err(err.into())), + Poll::Ready(res) } + Err(err) => Poll::Ready(Err(err.into())), } } @@ -726,7 +730,7 @@ impl BuildOutcome { } /// Applies a fn on the current payload. - pub(crate) fn map_payload(self, f: F) -> BuildOutcome

+ pub fn map_payload(self, f: F) -> BuildOutcome

where F: FnOnce(Payload) -> P, { @@ -852,10 +856,12 @@ pub trait PayloadBuilder: Send + Sync + Clone { /// Tells the payload builder how to react to payload request if there's no payload available yet. /// /// This situation can occur if the CL requests a payload before the first payload has been built. +#[derive(Default)] pub enum MissingPayloadBehaviour { /// Await the regular scheduled payload process. AwaitInProgress, /// Race the in progress payload process with an empty payload. + #[default] RaceEmptyPayload, /// Race the in progress payload process with this job. RacePayload(Box Result + Send>), @@ -873,12 +879,6 @@ impl fmt::Debug for MissingPayloadBehaviour { } } -impl Default for MissingPayloadBehaviour { - fn default() -> Self { - Self::RaceEmptyPayload - } -} - /// Checks if the new payload is better than the current best. /// /// This compares the total fees of the blocks, higher is better. diff --git a/crates/payload/builder-primitives/src/lib.rs b/crates/payload/builder-primitives/src/lib.rs index d181531ca32..e6b02c3e550 100644 --- a/crates/payload/builder-primitives/src/lib.rs +++ b/crates/payload/builder-primitives/src/lib.rs @@ -6,7 +6,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] mod events; pub use crate::events::{Events, PayloadEvents}; diff --git a/crates/payload/builder/Cargo.toml b/crates/payload/builder/Cargo.toml index 5ae0425f3e1..166c538f7a1 100644 --- a/crates/payload/builder/Cargo.toml +++ b/crates/payload/builder/Cargo.toml @@ -21,7 +21,7 @@ reth-ethereum-engine-primitives.workspace = true # alloy alloy-consensus.workspace = true -alloy-primitives = { workspace = true, optional = true } +alloy-primitives.workspace = true alloy-rpc-types = { workspace = true, features = ["engine"] } # async @@ -37,14 +37,10 @@ metrics.workspace = true tracing.workspace = true [dev-dependencies] -alloy-primitives.workspace = true -alloy-consensus.workspace = true - tokio = { workspace = true, features = ["sync", "rt"] } [features] test-utils = [ - "alloy-primitives", "reth-chain-state/test-utils", "reth-primitives-traits/test-utils", "tokio/rt", diff --git a/crates/payload/builder/src/lib.rs b/crates/payload/builder/src/lib.rs index 75a1c658219..457ca7fe3c8 100644 --- a/crates/payload/builder/src/lib.rs +++ b/crates/payload/builder/src/lib.rs @@ -75,6 +75,10 @@ //! Ok(self.attributes.clone()) //! } //! +//! fn payload_timestamp(&self) -> Result { +//! Ok(self.attributes.timestamp) +//! } +//! //! fn resolve_kind(&mut self, _kind: PayloadKind) -> (Self::ResolvePayloadFuture, KeepPayloadJobAlive) { //! let payload = self.best_payload(); //! (futures_util::future::ready(payload), KeepPayloadJobAlive::No) @@ -101,7 +105,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] mod metrics; mod service; @@ -122,4 +126,6 @@ pub use traits::{KeepPayloadJobAlive, PayloadJob, PayloadJobGenerator}; // re-export the Ethereum engine primitives for convenience #[doc(inline)] -pub use reth_ethereum_engine_primitives::{EthBuiltPayload, EthPayloadBuilderAttributes}; +pub use reth_ethereum_engine_primitives::{ + BlobSidecars, EthBuiltPayload, EthPayloadBuilderAttributes, +}; diff --git a/crates/payload/builder/src/noop.rs b/crates/payload/builder/src/noop.rs index cbf21f1cebf..3628ef83c0d 100644 --- a/crates/payload/builder/src/noop.rs +++ b/crates/payload/builder/src/noop.rs @@ -50,10 +50,24 @@ where tx.send(Ok(id)).ok() } PayloadServiceCommand::BestPayload(_, tx) => tx.send(None).ok(), - PayloadServiceCommand::PayloadAttributes(_, tx) => tx.send(None).ok(), + PayloadServiceCommand::PayloadTimestamp(_, tx) => tx.send(None).ok(), PayloadServiceCommand::Resolve(_, _, tx) => tx.send(None).ok(), PayloadServiceCommand::Subscribe(_) => None, }; } } } + +impl Default for NoopPayloadBuilderService { + fn default() -> Self { + let (service, _) = Self::new(); + service + } +} + +impl PayloadBuilderHandle { + /// Returns a new noop instance. + pub fn noop() -> Self { + Self::new(mpsc::unbounded_channel().0) + } +} diff --git a/crates/payload/builder/src/service.rs b/crates/payload/builder/src/service.rs index 82807eb721b..f3f1b03ab2e 100644 --- a/crates/payload/builder/src/service.rs +++ b/crates/payload/builder/src/service.rs @@ -8,6 +8,7 @@ use crate::{ PayloadJob, }; use alloy_consensus::BlockHeader; +use alloy_primitives::BlockTimestamp; use alloy_rpc_types::engine::PayloadId; use futures_util::{future::FutureExt, Stream, StreamExt}; use reth_chain_state::CanonStateNotification; @@ -24,11 +25,12 @@ use std::{ use tokio::sync::{ broadcast, mpsc, oneshot::{self, Receiver}, + watch, }; use tokio_stream::wrappers::UnboundedReceiverStream; use tracing::{debug, info, trace, warn}; -type PayloadFuture

= Pin> + Send + Sync>>; +type PayloadFuture

= Pin> + Send>>; /// A communication channel to the [`PayloadBuilderService`] that can retrieve payloads. /// @@ -73,14 +75,14 @@ where self.inner.best_payload(id).await } - /// Returns the payload attributes associated with the given identifier. + /// Returns the payload timestamp associated with the given identifier. /// - /// Note: this returns the attributes of the payload and does not resolve the job. - pub async fn payload_attributes( + /// Note: this returns the timestamp of the payload and does not resolve the job. + pub async fn payload_timestamp( &self, id: PayloadId, - ) -> Option> { - self.inner.payload_attributes(id).await + ) -> Option> { + self.inner.payload_timestamp(id).await } } @@ -166,15 +168,15 @@ impl PayloadBuilderHandle { Ok(PayloadEvents { receiver: rx.await? }) } - /// Returns the payload attributes associated with the given identifier. + /// Returns the payload timestamp associated with the given identifier. /// - /// Note: this returns the attributes of the payload and does not resolve the job. - pub async fn payload_attributes( + /// Note: this returns the timestamp of the payload and does not resolve the job. + pub async fn payload_timestamp( &self, id: PayloadId, - ) -> Option> { + ) -> Option> { let (tx, rx) = oneshot::channel(); - self.to_service.send(PayloadServiceCommand::PayloadAttributes(id, tx)).ok()?; + self.to_service.send(PayloadServiceCommand::PayloadTimestamp(id, tx)).ok()?; rx.await.ok()? } } @@ -218,6 +220,11 @@ where chain_events: St, /// Payload events handler, used to broadcast and subscribe to payload events. payload_events: broadcast::Sender>, + /// We retain latest resolved payload just to make sure that we can handle repeating + /// requests for it gracefully. + cached_payload_rx: watch::Receiver>, + /// Sender half of the cached payload channel. + cached_payload_tx: watch::Sender>, } const PAYLOAD_EVENTS_BUFFER_SIZE: usize = 20; @@ -241,6 +248,8 @@ where let (service_tx, command_rx) = mpsc::unbounded_channel(); let (payload_events, _) = broadcast::channel(PAYLOAD_EVENTS_BUFFER_SIZE); + let (cached_payload_tx, cached_payload_rx) = watch::channel(None); + let service = Self { generator, payload_jobs: Vec::new(), @@ -249,6 +258,8 @@ where metrics: Default::default(), chain_events, payload_events, + cached_payload_rx, + cached_payload_tx, }; let handle = service.handle(); @@ -260,6 +271,12 @@ where PayloadBuilderHandle::new(self.service_tx.clone()) } + /// Create clone on `payload_events` sending handle that could be used by builder to produce + /// additional events during block building + pub fn payload_events_handle(&self) -> broadcast::Sender> { + self.payload_events.clone() + } + /// Returns true if the given payload is currently being built. fn contains_payload(&self, id: PayloadId) -> bool { self.payload_jobs.iter().any(|(_, job_id)| *job_id == id) @@ -286,20 +303,28 @@ where id: PayloadId, kind: PayloadKind, ) -> Option> { - trace!(%id, "resolving payload job"); + debug!(target: "payload_builder", %id, "resolving payload job"); + + if let Some((cached, _, payload)) = &*self.cached_payload_rx.borrow() && + *cached == id + { + return Some(Box::pin(core::future::ready(Ok(payload.clone())))); + } let job = self.payload_jobs.iter().position(|(_, job_id)| *job_id == id)?; let (fut, keep_alive) = self.payload_jobs[job].0.resolve_kind(kind); + let payload_timestamp = self.payload_jobs[job].0.payload_timestamp(); if keep_alive == KeepPayloadJobAlive::No { let (_, id) = self.payload_jobs.swap_remove(job); - trace!(%id, "terminated resolved job"); + debug!(target: "payload_builder", %id, "terminated resolved job"); } // Since the fees will not be known until the payload future is resolved / awaited, we wrap // the future in a new future that will update the metrics. let resolved_metrics = self.metrics.clone(); let payload_events = self.payload_events.clone(); + let cached_payload_tx = self.cached_payload_tx.clone(); let fut = async move { let res = fut.await; @@ -308,6 +333,10 @@ where payload_events.send(Events::BuiltPayload(payload.clone().into())).ok(); } + if let Ok(timestamp) = payload_timestamp { + let _ = cached_payload_tx.send(Some((id, timestamp, payload.clone().into()))); + } + resolved_metrics .set_resolved_revenue(payload.block().number(), f64::from(payload.fees())); } @@ -325,22 +354,25 @@ where Gen::Job: PayloadJob, ::BuiltPayload: Into, { - /// Returns the payload attributes for the given payload. - fn payload_attributes( - &self, - id: PayloadId, - ) -> Option::PayloadAttributes, PayloadBuilderError>> { - let attributes = self + /// Returns the payload timestamp for the given payload. + fn payload_timestamp(&self, id: PayloadId) -> Option> { + if let Some((cached_id, timestamp, _)) = *self.cached_payload_rx.borrow() && + cached_id == id + { + return Some(Ok(timestamp)); + } + + let timestamp = self .payload_jobs .iter() .find(|(_, job_id)| *job_id == id) - .map(|(j, _)| j.payload_attributes()); + .map(|(j, _)| j.payload_timestamp()); - if attributes.is_none() { - trace!(%id, "no matching payload job found to get attributes for"); + if timestamp.is_none() { + trace!(target: "payload_builder", %id, "no matching payload job found to get timestamp for"); } - attributes + timestamp } } @@ -374,10 +406,10 @@ where match job.poll_unpin(cx) { Poll::Ready(Ok(_)) => { this.metrics.set_active_jobs(this.payload_jobs.len()); - trace!(%id, "payload job finished"); + trace!(target: "payload_builder", %id, "payload job finished"); } Poll::Ready(Err(err)) => { - warn!(%err, ?id, "Payload builder job failed; resolving payload"); + warn!(target: "payload_builder",%err, ?id, "Payload builder job failed; resolving payload"); this.metrics.inc_failed_jobs(); this.metrics.set_active_jobs(this.payload_jobs.len()); } @@ -399,13 +431,13 @@ where let mut res = Ok(id); if this.contains_payload(id) { - debug!(%id, parent = %attr.parent(), "Payload job already in progress, ignoring."); + debug!(target: "payload_builder",%id, parent = %attr.parent(), "Payload job already in progress, ignoring."); } else { // no job for this payload yet, create one let parent = attr.parent(); match this.generator.new_payload_job(attr.clone()) { Ok(job) => { - info!(%id, %parent, "New payload job created"); + info!(target: "payload_builder", %id, %parent, "New payload job created"); this.metrics.inc_initiated_jobs(); new_job = true; this.payload_jobs.push((job, id)); @@ -413,7 +445,7 @@ where } Err(err) => { this.metrics.inc_failed_jobs(); - warn!(%err, %id, "Failed to create payload builder job"); + warn!(target: "payload_builder", %err, %id, "Failed to create payload builder job"); res = Err(err); } } @@ -425,9 +457,9 @@ where PayloadServiceCommand::BestPayload(id, tx) => { let _ = tx.send(this.best_payload(id)); } - PayloadServiceCommand::PayloadAttributes(id, tx) => { - let attributes = this.payload_attributes(id); - let _ = tx.send(attributes); + PayloadServiceCommand::PayloadTimestamp(id, tx) => { + let timestamp = this.payload_timestamp(id); + let _ = tx.send(timestamp); } PayloadServiceCommand::Resolve(id, strategy, tx) => { let _ = tx.send(this.resolve(id, strategy)); @@ -455,11 +487,8 @@ pub enum PayloadServiceCommand { ), /// Get the best payload so far BestPayload(PayloadId, oneshot::Sender>>), - /// Get the payload attributes for the given payload - PayloadAttributes( - PayloadId, - oneshot::Sender>>, - ), + /// Get the payload timestamp for the given payload + PayloadTimestamp(PayloadId, oneshot::Sender>>), /// Resolve the payload and return the payload Resolve( PayloadId, @@ -482,8 +511,8 @@ where Self::BestPayload(f0, f1) => { f.debug_tuple("BestPayload").field(&f0).field(&f1).finish() } - Self::PayloadAttributes(f0, f1) => { - f.debug_tuple("PayloadAttributes").field(&f0).field(&f1).finish() + Self::PayloadTimestamp(f0, f1) => { + f.debug_tuple("PayloadTimestamp").field(&f0).field(&f1).finish() } Self::Resolve(f0, f1, _f2) => f.debug_tuple("Resolve").field(&f0).field(&f1).finish(), Self::Subscribe(f0) => f.debug_tuple("Subscribe").field(&f0).finish(), diff --git a/crates/payload/builder/src/test_utils.rs b/crates/payload/builder/src/test_utils.rs index 5058d48f246..bf4e85122ea 100644 --- a/crates/payload/builder/src/test_utils.rs +++ b/crates/payload/builder/src/test_utils.rs @@ -98,6 +98,10 @@ impl PayloadJob for TestPayloadJob { Ok(self.attr.clone()) } + fn payload_timestamp(&self) -> Result { + Ok(self.attr.timestamp) + } + fn resolve_kind( &mut self, _kind: PayloadKind, diff --git a/crates/payload/builder/src/traits.rs b/crates/payload/builder/src/traits.rs index 807bfa186ec..1e4158addde 100644 --- a/crates/payload/builder/src/traits.rs +++ b/crates/payload/builder/src/traits.rs @@ -17,13 +17,12 @@ use std::future::Future; /// empty. /// /// Note: A `PayloadJob` need to be cancel safe because it might be dropped after the CL has requested the payload via `engine_getPayloadV1` (see also [engine API docs](https://github.com/ethereum/execution-apis/blob/6709c2a795b707202e93c4f2867fa0bf2640a84f/src/engine/paris.md#engine_getpayloadv1)) -pub trait PayloadJob: Future> + Send + Sync { +pub trait PayloadJob: Future> { /// Represents the payload attributes type that is used to spawn this payload job. type PayloadAttributes: PayloadBuilderAttributes + std::fmt::Debug; /// Represents the future that resolves the block that's returned to the CL. type ResolvePayloadFuture: Future> + Send - + Sync + 'static; /// Represents the built payload type that is returned to the CL. type BuiltPayload: BuiltPayload + Clone + std::fmt::Debug; @@ -36,6 +35,14 @@ pub trait PayloadJob: Future> + Send + /// Returns the payload attributes for the payload being built. fn payload_attributes(&self) -> Result; + /// Returns the payload timestamp for the payload being built. + /// The default implementation allocates full attributes only to + /// extract the timestamp. Provide your own implementation if you + /// need performance here. + fn payload_timestamp(&self) -> Result { + Ok(self.payload_attributes()?.timestamp()) + } + /// Called when the payload is requested by the CL. /// /// This is invoked on [`engine_getPayloadV2`](https://github.com/ethereum/execution-apis/blob/main/src/engine/shanghai.md#engine_getpayloadv2) and [`engine_getPayloadV1`](https://github.com/ethereum/execution-apis/blob/main/src/engine/paris.md#engine_getpayloadv1). @@ -85,7 +92,7 @@ pub enum KeepPayloadJobAlive { } /// A type that knows how to create new jobs for creating payloads. -pub trait PayloadJobGenerator: Send + Sync { +pub trait PayloadJobGenerator { /// The type that manages the lifecycle of a payload. /// /// This type is a future that yields better payloads. diff --git a/crates/payload/primitives/Cargo.toml b/crates/payload/primitives/Cargo.toml index bb305961fe1..670727e3c6d 100644 --- a/crates/payload/primitives/Cargo.toml +++ b/crates/payload/primitives/Cargo.toml @@ -22,10 +22,11 @@ reth-chain-state.workspace = true alloy-eips.workspace = true alloy-primitives.workspace = true alloy-rpc-types-engine = { workspace = true, features = ["serde"] } -op-alloy-rpc-types-engine = { workspace = true, optional = true } +op-alloy-rpc-types-engine = { workspace = true, optional = true, features = ["serde"] } # misc auto_impl.workspace = true +either.workspace = true serde.workspace = true thiserror.workspace = true tokio = { workspace = true, default-features = false, features = ["sync"] } @@ -44,6 +45,7 @@ std = [ "serde/std", "thiserror/std", "reth-primitives-traits/std", + "either/std", ] op = [ "dep:op-alloy-rpc-types-engine", diff --git a/crates/payload/primitives/src/error.rs b/crates/payload/primitives/src/error.rs index 8a0ab501228..4de4b4ccabe 100644 --- a/crates/payload/primitives/src/error.rs +++ b/crates/payload/primitives/src/error.rs @@ -1,4 +1,4 @@ -//! Error types emitted by types or implementations of this crate. +//! Error types for payload operations. use alloc::{boxed::Box, string::ToString}; use alloy_primitives::B256; @@ -175,7 +175,7 @@ impl EngineObjectValidationError { #[derive(thiserror::Error, Debug)] pub enum InvalidPayloadAttributesError { /// Thrown if the timestamp of the payload attributes is invalid according to the engine specs. - #[error("parent beacon block root not supported before V3")] + #[error("invalid timestamp")] InvalidTimestamp, /// Another type of error that is not covered by the above variants. #[error("Invalid params: {0}")] diff --git a/crates/payload/primitives/src/lib.rs b/crates/payload/primitives/src/lib.rs index 561aeea4152..ca3cccda883 100644 --- a/crates/payload/primitives/src/lib.rs +++ b/crates/payload/primitives/src/lib.rs @@ -1,4 +1,6 @@ -//! This crate defines abstractions to create and update payloads (blocks) +//! Abstractions for working with execution payloads. +//! +//! This crate provides types and traits for execution and building payloads. #![doc( html_logo_url = "https://raw.githubusercontent.com/paradigmxyz/reth/main/assets/reth-docs.png", @@ -6,7 +8,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(feature = "std"), no_std)] extern crate alloc; @@ -22,32 +24,41 @@ pub use error::{ PayloadBuilderError, VersionSpecificValidationError, }; -/// Contains traits to abstract over payload attributes types and default implementations of the -/// [`PayloadAttributes`] trait for ethereum mainnet and optimism types. mod traits; pub use traits::{ - BuiltPayload, PayloadAttributes, PayloadAttributesBuilder, PayloadBuilderAttributes, + BuildNextEnv, BuiltPayload, PayloadAttributes, PayloadAttributesBuilder, + PayloadBuilderAttributes, }; mod payload; pub use payload::{ExecutionPayload, PayloadOrAttributes}; -/// The types that are used by the engine API. +/// Core trait that defines the associated types for working with execution payloads. pub trait PayloadTypes: Send + Sync + Unpin + core::fmt::Debug + Clone + 'static { - /// The execution payload type provided as input + /// The format for execution payload data that can be processed and validated. + /// + /// This type represents the canonical format for block data that includes + /// all necessary information for execution and validation. type ExecutionData: ExecutionPayload; - /// The built payload type. + /// The type representing a successfully built payload/block. type BuiltPayload: BuiltPayload + Clone + Unpin; - /// The RPC payload attributes type the CL node emits via the engine API. + /// Attributes that specify how a payload should be constructed. + /// + /// These attributes typically come from external sources (e.g., consensus layer over RPC such + /// as the Engine API) and contain parameters like timestamp, fee recipient, and randomness. type PayloadAttributes: PayloadAttributes + Unpin; - /// The payload attributes type that contains information about a running payload job. + /// Extended attributes used internally during payload building. + /// + /// This type augments the basic payload attributes with additional information + /// needed during the building process, such as unique identifiers and parent + /// block references. type PayloadBuilderAttributes: PayloadBuilderAttributes + Clone + Unpin; - /// Converts a block into an execution payload. + /// Converts a sealed block into the execution payload format. fn block_to_payload( block: SealedBlock< <::Primitives as NodePrimitives>::Block, @@ -60,6 +71,7 @@ pub trait PayloadTypes: Send + Sync + Unpin + core::fmt::Debug + Clone + 'static /// * If V2, this ensures that the payload timestamp is pre-Cancun. /// * If V3, this ensures that the payload timestamp is within the Cancun timestamp. /// * If V4, this ensures that the payload timestamp is within the Prague timestamp. +/// * If V5, this ensures that the payload timestamp is within the Osaka timestamp. /// /// Otherwise, this will return [`EngineObjectValidationError::UnsupportedFork`]. pub fn validate_payload_timestamp( @@ -133,6 +145,19 @@ pub fn validate_payload_timestamp( // the payload does not fall within the time frame of the Prague fork. return Err(EngineObjectValidationError::UnsupportedFork) } + + let is_osaka = chain_spec.is_osaka_active_at_timestamp(timestamp); + if version.is_v5() && !is_osaka { + // From the Engine API spec: + // + // + // For `engine_getPayloadV5` + // + // 1. Client software MUST return -38005: Unsupported fork error if the timestamp of the + // built payload does not fall within the time frame of the Osaka fork. + return Err(EngineObjectValidationError::UnsupportedFork) + } + Ok(()) } @@ -155,7 +180,10 @@ pub fn validate_withdrawals_presence( .to_error(VersionSpecificValidationError::WithdrawalsNotSupportedInV1)) } } - EngineApiMessageVersion::V2 | EngineApiMessageVersion::V3 | EngineApiMessageVersion::V4 => { + EngineApiMessageVersion::V2 | + EngineApiMessageVersion::V3 | + EngineApiMessageVersion::V4 | + EngineApiMessageVersion::V5 => { if is_shanghai_active && !has_withdrawals { return Err(message_validation_kind .to_error(VersionSpecificValidationError::NoWithdrawalsPostShanghai)) @@ -256,7 +284,7 @@ pub fn validate_parent_beacon_block_root_presence( )) } } - EngineApiMessageVersion::V3 | EngineApiMessageVersion::V4 => { + EngineApiMessageVersion::V3 | EngineApiMessageVersion::V4 | EngineApiMessageVersion::V5 => { if !has_parent_beacon_block_root { return Err(validation_kind .to_error(VersionSpecificValidationError::NoParentBeaconBlockRootPostCancun)) @@ -350,12 +378,16 @@ pub enum EngineApiMessageVersion { /// Version 3 /// /// Added in the Cancun hardfork. - #[default] V3 = 3, /// Version 4 /// /// Added in the Prague hardfork. + #[default] V4 = 4, + /// Version 5 + /// + /// Added in the Osaka hardfork. + V5 = 5, } impl EngineApiMessageVersion { @@ -378,6 +410,22 @@ impl EngineApiMessageVersion { pub const fn is_v4(&self) -> bool { matches!(self, Self::V4) } + + /// Returns true if the version is V5. + pub const fn is_v5(&self) -> bool { + matches!(self, Self::V5) + } + + /// Returns the method name for the given version. + pub const fn method_name(&self) -> &'static str { + match self { + Self::V1 => "engine_newPayloadV1", + Self::V2 => "engine_newPayloadV2", + Self::V3 => "engine_newPayloadV3", + Self::V4 => "engine_newPayloadV4", + Self::V5 => "engine_newPayloadV5", + } + } } /// Determines how we should choose the payload to return. @@ -474,7 +522,7 @@ mod tests { let mut requests_valid_reversed = valid_requests; requests_valid_reversed.reverse(); assert_matches!( - validate_execution_requests(&requests_with_empty), + validate_execution_requests(&requests_valid_reversed), Err(EngineObjectValidationError::InvalidParams(_)) ); diff --git a/crates/payload/primitives/src/payload.rs b/crates/payload/primitives/src/payload.rs index e21aabed75e..709a37768f4 100644 --- a/crates/payload/primitives/src/payload.rs +++ b/crates/payload/primitives/src/payload.rs @@ -1,34 +1,54 @@ +//! Types and traits for execution payload data structures. + use crate::{MessageValidationKind, PayloadAttributes}; use alloc::vec::Vec; -use alloy_eips::{eip4895::Withdrawal, eip7685::Requests}; +use alloy_eips::{eip1898::BlockWithParent, eip4895::Withdrawal, eip7685::Requests, BlockNumHash}; use alloy_primitives::B256; use alloy_rpc_types_engine::ExecutionData; use core::fmt::Debug; use serde::{de::DeserializeOwned, Serialize}; -/// An execution payload. +/// Represents the core data structure of an execution payload. +/// +/// Contains all necessary information to execute and validate a block, including +/// headers, transactions, and consensus fields. Provides a unified interface +/// regardless of protocol version. pub trait ExecutionPayload: Serialize + DeserializeOwned + Debug + Clone + Send + Sync + 'static { - /// Returns the parent hash of the block. + /// Returns the hash of this block's parent. fn parent_hash(&self) -> B256; - /// Returns the hash of the block. + /// Returns this block's hash. fn block_hash(&self) -> B256; - /// Returns the number of the block. + /// Returns this block's number (height). fn block_number(&self) -> u64; - /// Returns the withdrawals for the payload, if it exists. + /// Returns this block's number hash. + fn num_hash(&self) -> BlockNumHash { + BlockNumHash::new(self.block_number(), self.block_hash()) + } + + /// Returns a [`BlockWithParent`] for this block. + fn block_with_parent(&self) -> BlockWithParent { + BlockWithParent::new(self.parent_hash(), self.num_hash()) + } + + /// Returns the withdrawals included in this payload. + /// + /// Returns `None` for pre-Shanghai blocks. fn withdrawals(&self) -> Option<&Vec>; - /// Return the parent beacon block root for the payload, if it exists. + /// Returns the beacon block root associated with this payload. + /// + /// Returns `None` for pre-merge payloads. fn parent_beacon_block_root(&self) -> Option; - /// Returns the timestamp to be used in the payload. + /// Returns this block's timestamp (seconds since Unix epoch). fn timestamp(&self) -> u64; - /// Gas used by the payload + /// Returns the total gas consumed by all transactions in this block. fn gas_used(&self) -> u64; } @@ -62,25 +82,25 @@ impl ExecutionPayload for ExecutionData { } } -/// Either a type that implements the [`ExecutionPayload`] or a type that implements the -/// [`PayloadAttributes`] trait. +/// A unified type for handling both execution payloads and payload attributes. /// -/// This is a helper type to unify pre-validation of version specific fields of the engine API. +/// Enables generic validation and processing logic for both complete payloads +/// and payload attributes, useful for version-specific validation. #[derive(Debug)] pub enum PayloadOrAttributes<'a, Payload, Attributes> { - /// An [`ExecutionPayload`] + /// A complete execution payload containing block data ExecutionPayload(&'a Payload), - /// A payload attributes type. + /// Attributes specifying how to build a new payload PayloadAttributes(&'a Attributes), } impl<'a, Payload, Attributes> PayloadOrAttributes<'a, Payload, Attributes> { - /// Construct a [`PayloadOrAttributes::ExecutionPayload`] variant + /// Creates a `PayloadOrAttributes` from an execution payload reference pub const fn from_execution_payload(payload: &'a Payload) -> Self { Self::ExecutionPayload(payload) } - /// Construct a [`PayloadOrAttributes::PayloadAttributes`] variant + /// Creates a `PayloadOrAttributes` from a payload attributes reference pub const fn from_attributes(attributes: &'a Attributes) -> Self { Self::PayloadAttributes(attributes) } @@ -91,7 +111,7 @@ where Payload: ExecutionPayload, Attributes: PayloadAttributes, { - /// Return the withdrawals for the payload or attributes. + /// Returns withdrawals from either the payload or attributes. pub fn withdrawals(&self) -> Option<&Vec> { match self { Self::ExecutionPayload(payload) => payload.withdrawals(), @@ -99,7 +119,7 @@ where } } - /// Return the timestamp for the payload or attributes. + /// Returns the timestamp from either the payload or attributes. pub fn timestamp(&self) -> u64 { match self { Self::ExecutionPayload(payload) => payload.timestamp(), @@ -107,7 +127,7 @@ where } } - /// Return the parent beacon block root for the payload or attributes. + /// Returns the parent beacon block root from either the payload or attributes. pub fn parent_beacon_block_root(&self) -> Option { match self { Self::ExecutionPayload(payload) => payload.parent_beacon_block_root(), @@ -115,7 +135,7 @@ where } } - /// Return a [`MessageValidationKind`] for the payload or attributes. + /// Determines the validation context based on the contained type. pub const fn message_validation_kind(&self) -> MessageValidationKind { match self { Self::ExecutionPayload { .. } => MessageValidationKind::Payload, @@ -165,19 +185,15 @@ impl ExecutionPayload for op_alloy_rpc_types_engine::OpExecutionData { } } -/// Special implementation for Ethereum types that provides additional helper methods +/// Extended functionality for Ethereum execution payloads impl PayloadOrAttributes<'_, ExecutionData, Attributes> where Attributes: PayloadAttributes, { - /// Return the execution requests from the payload, if available. - /// - /// This will return `Some(requests)` only if: - /// - The payload is an `ExecutionData` (not `PayloadAttributes`) - /// - The payload has Prague payload fields - /// - The Prague fields contain requests (not a hash) + /// Extracts execution layer requests from the payload. /// - /// Returns `None` in all other cases. + /// Returns `Some(requests)` if this is an execution payload with request data, + /// `None` otherwise. pub fn execution_requests(&self) -> Option<&Requests> { if let Self::ExecutionPayload(payload) = self { payload.sidecar.requests() diff --git a/crates/payload/primitives/src/traits.rs b/crates/payload/primitives/src/traits.rs index ee503c90005..160956afa27 100644 --- a/crates/payload/primitives/src/traits.rs +++ b/crates/payload/primitives/src/traits.rs @@ -1,4 +1,7 @@ -use alloc::vec::Vec; +//! Core traits for working with execution payloads. + +use crate::PayloadBuilderError; +use alloc::{boxed::Box, vec::Vec}; use alloy_eips::{ eip4895::{Withdrawal, Withdrawals}, eip7685::Requests, @@ -6,45 +9,52 @@ use alloy_eips::{ use alloy_primitives::{Address, B256, U256}; use alloy_rpc_types_engine::{PayloadAttributes as EthPayloadAttributes, PayloadId}; use core::fmt; -use reth_chain_state::ExecutedBlockWithTrieUpdates; -use reth_primitives_traits::{NodePrimitives, SealedBlock}; +use reth_chain_state::ExecutedBlock; +use reth_primitives_traits::{NodePrimitives, SealedBlock, SealedHeader}; -/// Represents a built payload type that contains a built `SealedBlock` and can be converted into -/// engine API execution payloads. +/// Represents a successfully built execution payload (block). +/// +/// Provides access to the underlying block data, execution results, and associated metadata +/// for payloads ready for execution or propagation. #[auto_impl::auto_impl(&, Arc)] pub trait BuiltPayload: Send + Sync + fmt::Debug { /// The node's primitive types type Primitives: NodePrimitives; - /// Returns the built block (sealed) + /// Returns the built block in its sealed (hash-verified) form. fn block(&self) -> &SealedBlock<::Block>; - /// Returns the fees collected for the built block + /// Returns the total fees collected from all transactions in this block. fn fees(&self) -> U256; - /// Returns the entire execution data for the built block, if available. - fn executed_block(&self) -> Option> { + /// Returns the complete execution result including state updates. + /// + /// Returns `None` if execution data is not available or not tracked. + fn executed_block(&self) -> Option> { None } - /// Returns the EIP-7685 requests for the payload if any. + /// Returns the EIP-7685 execution layer requests included in this block. + /// + /// These are requests generated by the execution layer that need to be + /// processed by the consensus layer (e.g., validator deposits, withdrawals). fn requests(&self) -> Option; } -/// This can be implemented by types that describe a currently running payload job. +/// Attributes used to guide the construction of a new execution payload. /// -/// This is used as a conversion type, transforming a payload attributes type that the engine API -/// receives, into a type that the payload builder can use. -pub trait PayloadBuilderAttributes: Send + Sync + fmt::Debug { - /// The payload attributes that can be used to construct this type. Used as the argument in - /// [`PayloadBuilderAttributes::try_new`]. - type RpcPayloadAttributes; +/// Extends basic payload attributes with additional context needed during the +/// building process, tracking in-progress payload jobs and their parameters. +pub trait PayloadBuilderAttributes: Send + Sync + Unpin + fmt::Debug + 'static { + /// The external payload attributes format this type can be constructed from. + type RpcPayloadAttributes: Send + Sync + 'static; /// The error type used in [`PayloadBuilderAttributes::try_new`]. - type Error: core::error::Error; + type Error: core::error::Error + Send + Sync + 'static; - /// Creates a new payload builder for the given parent block and the attributes. + /// Constructs new builder attributes from external payload attributes. /// - /// Derives the unique [`PayloadId`] for the given parent, attributes and version. + /// Validates attributes and generates a unique [`PayloadId`] based on the + /// parent block, attributes, and version. fn try_new( parent: B256, rpc_payload_attributes: Self::RpcPayloadAttributes, @@ -53,42 +63,48 @@ pub trait PayloadBuilderAttributes: Send + Sync + fmt::Debug { where Self: Sized; - /// Returns the [`PayloadId`] for the running payload job. + /// Returns the unique identifier for this payload build job. fn payload_id(&self) -> PayloadId; - /// Returns the parent block hash for the running payload job. + /// Returns the hash of the parent block this payload builds on. fn parent(&self) -> B256; - /// Returns the timestamp for the running payload job. + /// Returns the timestamp to be used in the payload's header. fn timestamp(&self) -> u64; - /// Returns the parent beacon block root for the running payload job, if it exists. + /// Returns the beacon chain block root from the parent block. + /// + /// Returns `None` for pre-merge blocks or non-beacon contexts. fn parent_beacon_block_root(&self) -> Option; - /// Returns the suggested fee recipient for the running payload job. + /// Returns the address that should receive transaction fees. fn suggested_fee_recipient(&self) -> Address; - /// Returns the prevrandao field for the running payload job. + /// Returns the randomness value for this block. fn prev_randao(&self) -> B256; - /// Returns the withdrawals for the running payload job. + /// Returns the list of withdrawals to be processed in this block. fn withdrawals(&self) -> &Withdrawals; } -/// The execution payload attribute type the CL node emits via the engine API. -/// This trait should be implemented by types that could be used to spawn a payload job. +/// Basic attributes required to initiate payload construction. /// -/// This type is emitted as part of the forkchoiceUpdated call +/// Defines minimal parameters needed to build a new execution payload. +/// Implementations must be serializable for transmission. pub trait PayloadAttributes: serde::de::DeserializeOwned + serde::Serialize + fmt::Debug + Clone + Send + Sync + 'static { - /// Returns the timestamp to be used in the payload job. + /// Returns the timestamp for the new payload. fn timestamp(&self) -> u64; - /// Returns the withdrawals for the given payload attributes. + /// Returns the withdrawals to be included in the payload. + /// + /// `Some` for post-Shanghai blocks, `None` for earlier blocks. fn withdrawals(&self) -> Option<&Vec>; - /// Return the parent beacon block root for the payload attributes. + /// Returns the parent beacon block root. + /// + /// `Some` for post-merge blocks, `None` for pre-merge blocks. fn parent_beacon_block_root(&self) -> Option; } @@ -121,8 +137,55 @@ impl PayloadAttributes for op_alloy_rpc_types_engine::OpPayloadAttributes { } } -/// A builder that can return the current payload attribute. +/// Factory trait for creating payload attributes. +/// +/// Enables different strategies for generating payload attributes based on +/// contextual information. Useful for testing and specialized building. pub trait PayloadAttributesBuilder: Send + Sync + 'static { - /// Return a new payload attribute from the builder. + /// Constructs new payload attributes for the given timestamp. fn build(&self, timestamp: u64) -> Attributes; } + +impl PayloadAttributesBuilder for F +where + F: Fn(u64) -> Attributes + Send + Sync + 'static, +{ + fn build(&self, timestamp: u64) -> Attributes { + self(timestamp) + } +} + +impl PayloadAttributesBuilder for either::Either +where + L: PayloadAttributesBuilder, + R: PayloadAttributesBuilder, +{ + fn build(&self, timestamp: u64) -> Attributes { + match self { + Self::Left(l) => l.build(timestamp), + Self::Right(r) => r.build(timestamp), + } + } +} + +impl PayloadAttributesBuilder + for Box> +where + Attributes: 'static, +{ + fn build(&self, timestamp: u64) -> Attributes { + self.as_ref().build(timestamp) + } +} + +/// Trait to build the EVM environment for the next block from the given payload attributes. +/// +/// Accepts payload attributes from CL, parent header and additional payload builder context. +pub trait BuildNextEnv: Sized { + /// Builds the EVM environment for the next block from the given payload attributes. + fn build_next_env( + attributes: &Attributes, + parent: &SealedHeader

, + ctx: &Ctx, + ) -> Result; +} diff --git a/crates/payload/util/src/lib.rs b/crates/payload/util/src/lib.rs index ffffc936fe1..ccf8d8c1096 100644 --- a/crates/payload/util/src/lib.rs +++ b/crates/payload/util/src/lib.rs @@ -6,7 +6,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] mod traits; mod transaction; diff --git a/crates/payload/validator/src/cancun.rs b/crates/payload/validator/src/cancun.rs index 5a4deb139fd..cea8aca5144 100644 --- a/crates/payload/validator/src/cancun.rs +++ b/crates/payload/validator/src/cancun.rs @@ -11,14 +11,15 @@ use reth_primitives_traits::{AlloyBlockHeader, Block, SealedBlock}; /// - doesn't contain EIP-4844 transactions unless Cancun is active /// - checks blob versioned hashes in block and sidecar match #[inline] -pub fn ensure_well_formed_fields( +pub fn ensure_well_formed_fields( block: &SealedBlock, cancun_sidecar_fields: Option<&CancunPayloadFields>, is_cancun_active: bool, ) -> Result<(), PayloadError> where T: Transaction + Typed2718, - B: Block>, + H: AlloyBlockHeader, + B: Block
>, { ensure_well_formed_header_and_sidecar_fields(block, cancun_sidecar_fields, is_cancun_active)?; ensure_well_formed_transactions_field_with_sidecar( @@ -72,8 +73,8 @@ pub fn ensure_well_formed_header_and_sidecar_fields( /// - doesn't contain EIP-4844 transactions unless Cancun is active /// - checks blob versioned hashes in block and sidecar match #[inline] -pub fn ensure_well_formed_transactions_field_with_sidecar( - block_body: &BlockBody, +pub fn ensure_well_formed_transactions_field_with_sidecar( + block_body: &BlockBody, cancun_sidecar_fields: Option<&CancunPayloadFields>, is_cancun_active: bool, ) -> Result<(), PayloadError> { @@ -89,8 +90,8 @@ pub fn ensure_well_formed_transactions_field_with_sidecar( - block_body: &BlockBody, +pub fn ensure_matching_blob_versioned_hashes( + block_body: &BlockBody, cancun_sidecar_fields: Option<&CancunPayloadFields>, ) -> Result<(), PayloadError> { let num_blob_versioned_hashes = block_body.blob_versioned_hashes_iter().count(); diff --git a/crates/payload/validator/src/lib.rs b/crates/payload/validator/src/lib.rs index de952ebd6af..d7853ffa3b0 100644 --- a/crates/payload/validator/src/lib.rs +++ b/crates/payload/validator/src/lib.rs @@ -6,7 +6,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(feature = "std"), no_std)] pub mod cancun; diff --git a/crates/payload/validator/src/prague.rs b/crates/payload/validator/src/prague.rs index d663469a826..9dff206d74f 100644 --- a/crates/payload/validator/src/prague.rs +++ b/crates/payload/validator/src/prague.rs @@ -10,8 +10,8 @@ use alloy_rpc_types_engine::{PayloadError, PraguePayloadFields}; /// - Prague fields are not present unless Prague is active /// - does not contain EIP-7702 transactions if Prague is not active #[inline] -pub fn ensure_well_formed_fields( - block_body: &BlockBody, +pub fn ensure_well_formed_fields( + block_body: &BlockBody, prague_fields: Option<&PraguePayloadFields>, is_prague_active: bool, ) -> Result<(), PayloadError> { @@ -36,8 +36,8 @@ pub const fn ensure_well_formed_sidecar_fields( /// Checks that transactions field doesn't contain EIP-7702 transactions if Prague is not /// active. #[inline] -pub fn ensure_well_formed_transactions_field( - block_body: &BlockBody, +pub fn ensure_well_formed_transactions_field( + block_body: &BlockBody, is_prague_active: bool, ) -> Result<(), PayloadError> { if !is_prague_active && block_body.has_eip7702_transactions() { diff --git a/crates/primitives-traits/Cargo.toml b/crates/primitives-traits/Cargo.toml index f8ee37fb01d..58d52bddb03 100644 --- a/crates/primitives-traits/Cargo.toml +++ b/crates/primitives-traits/Cargo.toml @@ -27,7 +27,7 @@ revm-bytecode.workspace = true revm-state.workspace = true # op -op-alloy-consensus = { workspace = true, optional = true } +op-alloy-consensus = { workspace = true, optional = true, features = ["k256"] } # crypto secp256k1 = { workspace = true, features = ["recovery"], optional = true } @@ -50,6 +50,7 @@ arbitrary = { workspace = true, features = ["derive"], optional = true } proptest = { workspace = true, optional = true } proptest-arbitrary-interop = { workspace = true, optional = true } rayon = { workspace = true, optional = true } +alloy-rpc-types-eth = { workspace = true, optional = true } [dev-dependencies] reth-codecs.workspace = true @@ -68,8 +69,6 @@ rand.workspace = true rand_08.workspace = true serde.workspace = true serde_json.workspace = true -test-fuzz.workspace = true -modular-bitfield.workspace = true [features] default = ["std"] @@ -93,6 +92,7 @@ std = [ "reth-chainspec/std", "revm-bytecode/std", "revm-state/std", + "alloy-rpc-types-eth?/std", ] secp256k1 = ["alloy-consensus/secp256k1"] test-utils = [ @@ -115,6 +115,7 @@ arbitrary = [ "op-alloy-consensus?/arbitrary", "alloy-trie/arbitrary", "reth-chainspec/arbitrary", + "alloy-rpc-types-eth?/arbitrary", ] serde-bincode-compat = [ "serde", @@ -123,6 +124,8 @@ serde-bincode-compat = [ "alloy-eips/serde-bincode-compat", "op-alloy-consensus?/serde", "op-alloy-consensus?/serde-bincode-compat", + "alloy-genesis/serde-bincode-compat", + "alloy-rpc-types-eth?/serde-bincode-compat", ] serde = [ "dep:serde", @@ -140,6 +143,7 @@ serde = [ "revm-bytecode/serde", "revm-state/serde", "rand_08/serde", + "alloy-rpc-types-eth?/serde", ] reth-codec = [ "dep:reth-codecs", @@ -153,3 +157,4 @@ op = [ rayon = [ "dep:rayon", ] +rpc-compat = ["alloy-rpc-types-eth"] diff --git a/crates/primitives-traits/src/account.rs b/crates/primitives-traits/src/account.rs index b6a7523ea0a..8c4a496dabd 100644 --- a/crates/primitives-traits/src/account.rs +++ b/crates/primitives-traits/src/account.rs @@ -1,3 +1,4 @@ +use crate::InMemorySize; use alloy_consensus::constants::KECCAK_EMPTY; use alloy_genesis::GenesisAccount; use alloy_primitives::{keccak256, Bytes, B256, U256}; @@ -18,9 +19,6 @@ pub mod compact_ids { /// Identifier for [`LegacyAnalyzed`](revm_bytecode::Bytecode::LegacyAnalyzed). pub const LEGACY_ANALYZED_BYTECODE_ID: u8 = 2; - /// Identifier for [`Eof`](revm_bytecode::Bytecode::Eof). - pub const EOF_BYTECODE_ID: u8 = 3; - /// Identifier for [`Eip7702`](revm_bytecode::Bytecode::Eip7702). pub const EIP7702_BYTECODE_ID: u8 = 4; } @@ -91,6 +89,12 @@ impl From for Account { } } +impl InMemorySize for Account { + fn size(&self) -> usize { + size_of::() + } +} + /// Bytecode for an account. /// /// A wrapper around [`revm::primitives::Bytecode`][RevmBytecode] with encoding/decoding support. @@ -125,11 +129,10 @@ impl reth_codecs::Compact for Bytecode { where B: bytes::BufMut + AsMut<[u8]>, { - use compact_ids::{EIP7702_BYTECODE_ID, EOF_BYTECODE_ID, LEGACY_ANALYZED_BYTECODE_ID}; + use compact_ids::{EIP7702_BYTECODE_ID, LEGACY_ANALYZED_BYTECODE_ID}; let bytecode = match &self.0 { RevmBytecode::LegacyAnalyzed(analyzed) => analyzed.bytecode(), - RevmBytecode::Eof(eof) => eof.raw(), RevmBytecode::Eip7702(eip7702) => eip7702.raw(), }; buf.put_u32(bytecode.len() as u32); @@ -143,10 +146,6 @@ impl reth_codecs::Compact for Bytecode { buf.put_slice(map); 1 + 8 + map.len() } - RevmBytecode::Eof(_) => { - buf.put_u8(EOF_BYTECODE_ID); - 1 - } RevmBytecode::Eip7702(_) => { buf.put_u8(EIP7702_BYTECODE_ID); 1 @@ -192,8 +191,8 @@ impl reth_codecs::Compact for Bytecode { revm_bytecode::JumpTable::from_slice(buf, jump_table_len), )) } - EOF_BYTECODE_ID | EIP7702_BYTECODE_ID => { - // EOF and EIP-7702 bytecode objects will be decoded from the raw bytecode + EIP7702_BYTECODE_ID => { + // EIP-7702 bytecode objects will be decoded from the raw bytecode Self(RevmBytecode::new_raw(bytes)) } _ => unreachable!("Junk data in database: unknown Bytecode variant"), @@ -292,16 +291,17 @@ mod tests { } #[test] + #[ignore] fn test_bytecode() { let mut buf = vec![]; let bytecode = Bytecode::new_raw(Bytes::default()); let len = bytecode.to_compact(&mut buf); - assert_eq!(len, 51); + assert_eq!(len, 14); let mut buf = vec![]; let bytecode = Bytecode::new_raw(Bytes::from(&hex!("ffff"))); let len = bytecode.to_compact(&mut buf); - assert_eq!(len, 53); + assert_eq!(len, 17); let mut buf = vec![]; let bytecode = Bytecode(RevmBytecode::LegacyAnalyzed(LegacyAnalyzedBytecode::new( diff --git a/crates/primitives-traits/src/block/body.rs b/crates/primitives-traits/src/block/body.rs index 69d6c0089d3..4dc9a67e887 100644 --- a/crates/primitives-traits/src/block/body.rs +++ b/crates/primitives-traits/src/block/body.rs @@ -5,7 +5,10 @@ use crate::{ MaybeSerdeBincodeCompat, SignedTransaction, }; use alloc::{fmt, vec::Vec}; -use alloy_consensus::{Transaction, Typed2718}; +use alloy_consensus::{ + transaction::{Recovered, TxHashRef}, + Transaction, Typed2718, +}; use alloy_eips::{eip2718::Encodable2718, eip4895::Withdrawals}; use alloy_primitives::{Address, Bytes, B256}; @@ -67,6 +70,13 @@ pub trait BlockBody: self.transactions_iter().find(|tx| tx.tx_hash() == hash) } + /// Returns true if the block body contains a transaction with the given hash. + /// + /// This is a convenience function for `transaction_by_hash().is_some()` + fn contains_transaction(&self, hash: &B256) -> bool { + self.transaction_by_hash(hash).is_some() + } + /// Clones the transactions in the block. /// /// This is a convenience function for `transactions().to_vec()` @@ -102,7 +112,7 @@ pub trait BlockBody: /// Calculate the withdrawals root for the block body. /// - /// Returns `RecoveryError` if there are no withdrawals in the block. + /// Returns `Some(root)` if withdrawals are present, otherwise `None`. fn calculate_withdrawals_root(&self) -> Option { self.withdrawals().map(|withdrawals| { alloy_consensus::proofs::calculate_withdrawals_root(withdrawals.as_slice()) @@ -114,7 +124,7 @@ pub trait BlockBody: /// Calculate the ommers root for the block body. /// - /// Returns `RecoveryError` if there are no ommers in the block. + /// Returns `Some(root)` if ommers are present, otherwise `None`. fn calculate_ommers_root(&self) -> Option { self.ommers().map(alloy_consensus::proofs::calculate_ommers_root) } @@ -150,20 +160,14 @@ pub trait BlockBody: } /// Recover signer addresses for all transactions in the block body. - fn recover_signers(&self) -> Result, RecoveryError> - where - Self::Transaction: SignedTransaction, - { + fn recover_signers(&self) -> Result, RecoveryError> { crate::transaction::recover::recover_signers(self.transactions()) } /// Recover signer addresses for all transactions in the block body. /// /// Returns an error if some transaction's signature is invalid. - fn try_recover_signers(&self) -> Result, RecoveryError> - where - Self::Transaction: SignedTransaction, - { + fn try_recover_signers(&self) -> Result, RecoveryError> { self.recover_signers() } @@ -171,10 +175,7 @@ pub trait BlockBody: /// signature has a low `s` value_. /// /// Returns `RecoveryError`, if some transaction's signature is invalid. - fn recover_signers_unchecked(&self) -> Result, RecoveryError> - where - Self::Transaction: SignedTransaction, - { + fn recover_signers_unchecked(&self) -> Result, RecoveryError> { crate::transaction::recover::recover_signers_unchecked(self.transactions()) } @@ -182,12 +183,21 @@ pub trait BlockBody: /// signature has a low `s` value_. /// /// Returns an error if some transaction's signature is invalid. - fn try_recover_signers_unchecked(&self) -> Result, RecoveryError> - where - Self::Transaction: SignedTransaction, - { + fn try_recover_signers_unchecked(&self) -> Result, RecoveryError> { self.recover_signers_unchecked() } + + /// Recovers signers for all transactions in the block body and returns a vector of + /// [`Recovered`]. + fn recover_transactions(&self) -> Result>, RecoveryError> { + self.recover_signers().map(|signers| { + self.transactions() + .iter() + .zip(signers) + .map(|(tx, signer)| tx.clone().with_signer(signer)) + .collect() + }) + } } impl BlockBody for alloy_consensus::BlockBody diff --git a/crates/primitives-traits/src/block/error.rs b/crates/primitives-traits/src/block/error.rs index 471eeb800bf..ccb727ce88a 100644 --- a/crates/primitives-traits/src/block/error.rs +++ b/crates/primitives-traits/src/block/error.rs @@ -3,6 +3,37 @@ use crate::transaction::signed::RecoveryError; /// Type alias for [`BlockRecoveryError`] with a [`SealedBlock`](crate::SealedBlock) value. +/// +/// This error type is specifically used when recovering a sealed block fails. +/// It contains the original sealed block that could not be recovered, allowing +/// callers to inspect the problematic block or attempt recovery with different +/// parameters. +/// +/// # Example +/// +/// ```rust +/// use alloy_consensus::{Block, BlockBody, Header, Signed, TxEnvelope, TxLegacy}; +/// use alloy_primitives::{Signature, B256}; +/// use reth_primitives_traits::{block::error::SealedBlockRecoveryError, SealedBlock}; +/// +/// // Create a simple block for demonstration +/// let header = Header::default(); +/// let tx = TxLegacy::default(); +/// let signed_tx = Signed::new_unchecked(tx, Signature::test_signature(), B256::ZERO); +/// let envelope = TxEnvelope::Legacy(signed_tx); +/// let body = BlockBody { transactions: vec![envelope], ommers: vec![], withdrawals: None }; +/// let block = Block::new(header, body); +/// let sealed_block = SealedBlock::new_unchecked(block, B256::ZERO); +/// +/// // Simulate a block recovery operation that fails +/// let block_recovery_result: Result<(), SealedBlockRecoveryError<_>> = +/// Err(SealedBlockRecoveryError::new(sealed_block)); +/// +/// // When block recovery fails, you get the error with the original block +/// let error = block_recovery_result.unwrap_err(); +/// let failed_block = error.into_inner(); +/// // Now you can inspect the failed block or try recovery again +/// ``` pub type SealedBlockRecoveryError = BlockRecoveryError>; /// Error when recovering a block from [`SealedBlock`](crate::SealedBlock) to @@ -26,8 +57,11 @@ impl BlockRecoveryError { } } -impl From> for RecoveryError { - fn from(_: BlockRecoveryError) -> Self { - Self +impl From> for RecoveryError +where + T: core::fmt::Debug + Send + Sync + 'static, +{ + fn from(err: BlockRecoveryError) -> Self { + Self::from_source(err) } } diff --git a/crates/primitives-traits/src/block/mod.rs b/crates/primitives-traits/src/block/mod.rs index f3ac7f2bc7c..7705512d633 100644 --- a/crates/primitives-traits/src/block/mod.rs +++ b/crates/primitives-traits/src/block/mod.rs @@ -1,4 +1,26 @@ //! Block abstraction. +//! +//! This module provides the core block types and transformations: +//! +//! ```rust +//! # use reth_primitives_traits::{Block, SealedBlock, RecoveredBlock}; +//! # fn example(block: B) -> Result<(), Box> +//! # where B::Body: reth_primitives_traits::BlockBody { +//! // Basic block flow +//! let block: B = block; +//! +//! // Seal (compute hash) +//! let sealed: SealedBlock = block.seal(); +//! +//! // Recover senders +//! let recovered: RecoveredBlock = sealed.try_recover()?; +//! +//! // Access components +//! let senders = recovered.senders(); +//! let hash = recovered.hash(); +//! # Ok(()) +//! # } +//! ``` pub(crate) mod sealed; pub use sealed::SealedBlock; @@ -28,17 +50,9 @@ pub mod serde_bincode_compat { } /// Helper trait that unifies all behaviour required by block to support full node operations. -pub trait FullBlock: - Block + alloy_rlp::Encodable + alloy_rlp::Decodable -{ -} +pub trait FullBlock: Block {} -impl FullBlock for T where - T: Block - + alloy_rlp::Encodable - + alloy_rlp::Decodable -{ -} +impl FullBlock for T where T: Block {} /// Helper trait to access [`BlockBody::Transaction`] given a [`Block`]. pub type BlockTx = <::Body as BlockBody>::Transaction; @@ -47,7 +61,7 @@ pub type BlockTx = <::Body as BlockBody>::Transaction; /// /// This type defines the structure of a block in the blockchain. /// A [`Block`] is composed of a header and a body. -/// It is expected that a block can always be completely reconstructed from its header and body. +/// It is expected that a block can always be completely reconstructed from its header and body pub trait Block: Send + Sync @@ -168,10 +182,7 @@ pub trait Block: /// transactions. /// /// Returns the block as error if a signature is invalid. - fn try_into_recovered(self) -> Result, BlockRecoveryError> - where - ::Transaction: SignedTransaction, - { + fn try_into_recovered(self) -> Result, BlockRecoveryError> { let Ok(signers) = self.body().recover_signers() else { return Err(BlockRecoveryError::new(self)) }; diff --git a/crates/primitives-traits/src/block/recovered.rs b/crates/primitives-traits/src/block/recovered.rs index c565691dc64..d6bba9d1127 100644 --- a/crates/primitives-traits/src/block/recovered.rs +++ b/crates/primitives-traits/src/block/recovered.rs @@ -6,14 +6,34 @@ use crate::{ Block, BlockBody, InMemorySize, SealedHeader, }; use alloc::vec::Vec; -use alloy_consensus::{transaction::Recovered, BlockHeader}; -use alloy_eips::{eip1898::BlockWithParent, BlockNumHash}; -use alloy_primitives::{Address, BlockHash, BlockNumber, Bloom, Bytes, Sealed, B256, B64, U256}; +use alloy_consensus::{ + transaction::{Recovered, TransactionMeta}, + BlockHeader, +}; +use alloy_eips::{eip1898::BlockWithParent, BlockNumHash, Encodable2718}; +use alloy_primitives::{ + Address, BlockHash, BlockNumber, Bloom, Bytes, Sealed, TxHash, B256, B64, U256, +}; use derive_more::Deref; /// A block with senders recovered from the block's transactions. /// -/// This type is a [`SealedBlock`] with a list of senders that match the transactions in the block. +/// This type represents a [`SealedBlock`] where all transaction senders have been +/// recovered and verified. Recovery is an expensive operation that extracts the +/// sender address from each transaction's signature. +/// +/// # Construction +/// +/// - [`RecoveredBlock::new`] / [`RecoveredBlock::new_unhashed`] - Create with pre-recovered senders +/// (unchecked) +/// - [`RecoveredBlock::try_new`] / [`RecoveredBlock::try_new_unhashed`] - Create with validation +/// - [`RecoveredBlock::try_recover`] - Recover from a block +/// - [`RecoveredBlock::try_recover_sealed`] - Recover from a sealed block +/// +/// # Performance +/// +/// Sender recovery is computationally expensive. Cache recovered blocks when possible +/// to avoid repeated recovery operations. /// /// ## Sealing /// @@ -83,7 +103,7 @@ impl RecoveredBlock { Self { block, senders } } - /// A safer variant of [`Self::new_unhashed`] that checks if the number of senders is equal to + /// A safer variant of [`Self::new`] that checks if the number of senders is equal to /// the number of transactions in the block and recovers the senders from the transactions, if /// not using [`SignedTransaction::recover_signer`](crate::transaction::signed::SignedTransaction) /// to recover the senders. @@ -196,7 +216,7 @@ impl RecoveredBlock { Ok(Self::new(block, senders, hash)) } - /// A safer variant of [`Self::new_unhashed`] that checks if the number of senders is equal to + /// A safer variant of [`Self::new_sealed`] that checks if the number of senders is equal to /// the number of transactions in the block and recovers the senders from the transactions, if /// not using [`SignedTransaction::recover_signer_unchecked`](crate::transaction::signed::SignedTransaction) /// to recover the senders. @@ -210,7 +230,7 @@ impl RecoveredBlock { Self::try_new(block, senders, hash) } - /// A safer variant of [`Self::new`] that checks if the number of senders is equal to + /// A safer variant of [`Self::new_sealed`] that checks if the number of senders is equal to /// the number of transactions in the block and recovers the senders from the transactions, if /// not using [`SignedTransaction::recover_signer_unchecked`](crate::transaction::signed::SignedTransaction) /// to recover the senders. @@ -283,6 +303,24 @@ impl RecoveredBlock { (self.block.into_block(), self.senders) } + /// Returns the `Recovered<&T>` transaction at the given index. + pub fn recovered_transaction( + &self, + idx: usize, + ) -> Option::Transaction>> { + let sender = self.senders.get(idx).copied()?; + self.block.body().transactions().get(idx).map(|tx| Recovered::new_unchecked(tx, sender)) + } + + /// Finds a transaction by hash and returns it with its index and block context. + pub fn find_indexed(&self, tx_hash: TxHash) -> Option> { + self.body() + .transactions_iter() + .enumerate() + .find(|(_, tx)| tx.trie_hash() == tx_hash) + .map(|(index, tx)| IndexedTx { block: self, tx, index }) + } + /// Returns an iterator over all transactions and their sender. #[inline] pub fn transactions_with_sender( @@ -420,9 +458,7 @@ impl Eq for RecoveredBlock {} impl PartialEq for RecoveredBlock { fn eq(&self, other: &Self) -> bool { - self.hash_ref().eq(other.hash_ref()) && - self.block.eq(&other.block) && - self.senders.eq(&other.senders) + self.block.eq(&other.block) && self.senders.eq(&other.senders) } } @@ -446,6 +482,44 @@ impl From> for Sealed { } } +/// Converts a block with recovered transactions into a [`RecoveredBlock`]. +/// +/// This implementation takes an `alloy_consensus::Block` where transactions are of type +/// `Recovered` (transactions with their recovered senders) and converts it into a +/// [`RecoveredBlock`] which stores transactions and senders separately for efficiency. +impl From, H>> + for RecoveredBlock> +where + T: SignedTransaction, + H: crate::block::header::BlockHeader, +{ + fn from(block: alloy_consensus::Block, H>) -> Self { + let header = block.header; + + // Split the recovered transactions into transactions and senders + let (transactions, senders): (Vec, Vec
) = block + .body + .transactions + .into_iter() + .map(|recovered| { + let (tx, sender) = recovered.into_parts(); + (tx, sender) + }) + .unzip(); + + // Reconstruct the block with regular transactions + let body = alloy_consensus::BlockBody { + transactions, + ommers: block.body.ommers, + withdrawals: block.body.withdrawals, + }; + + let block = alloy_consensus::Block::new(header, body); + + Self::new_unhashed(block, senders) + } +} + #[cfg(any(test, feature = "arbitrary"))] impl<'a, B> arbitrary::Arbitrary<'a> for RecoveredBlock where @@ -497,7 +571,7 @@ impl RecoveredBlock { self.block.header_mut() } - /// Returns a mutable reference to the header. + /// Returns a mutable reference to the body. pub const fn block_mut(&mut self) -> &mut B::Body { self.block.body_mut() } @@ -523,6 +597,266 @@ impl RecoveredBlock { } } +/// Transaction with its index and block reference for efficient metadata access. +#[derive(Debug)] +pub struct IndexedTx<'a, B: Block> { + /// Recovered block containing the transaction + block: &'a RecoveredBlock, + /// Transaction matching the hash + tx: &'a ::Transaction, + /// Index of the transaction in the block + index: usize, +} + +impl<'a, B: Block> IndexedTx<'a, B> { + /// Returns the transaction. + pub const fn tx(&self) -> &::Transaction { + self.tx + } + + /// Returns the recovered transaction with the sender. + pub fn recovered_tx(&self) -> Recovered<&::Transaction> { + let sender = self.block.senders[self.index]; + Recovered::new_unchecked(self.tx, sender) + } + + /// Returns the transaction hash. + pub fn tx_hash(&self) -> TxHash { + self.tx.trie_hash() + } + + /// Returns the block hash. + pub fn block_hash(&self) -> B256 { + self.block.hash() + } + + /// Returns the index of the transaction in the block. + pub const fn index(&self) -> usize { + self.index + } + + /// Builds a [`TransactionMeta`] for the indexed transaction. + pub fn meta(&self) -> TransactionMeta { + TransactionMeta { + tx_hash: self.tx.trie_hash(), + index: self.index as u64, + block_hash: self.block.hash(), + block_number: self.block.number(), + base_fee: self.block.base_fee_per_gas(), + timestamp: self.block.timestamp(), + excess_blob_gas: self.block.excess_blob_gas(), + } + } +} + +#[cfg(feature = "rpc-compat")] +mod rpc_compat { + use super::{ + Block as BlockTrait, BlockBody as BlockBodyTrait, RecoveredBlock, SignedTransaction, + }; + use crate::{block::error::BlockRecoveryError, SealedHeader}; + use alloc::vec::Vec; + use alloy_consensus::{ + transaction::{Recovered, TxHashRef}, + Block as CBlock, BlockBody, BlockHeader, Sealable, + }; + use alloy_rpc_types_eth::{Block, BlockTransactions, BlockTransactionsKind, TransactionInfo}; + + impl RecoveredBlock + where + B: BlockTrait, + { + /// Converts the block into an RPC [`Block`] with the given [`BlockTransactionsKind`]. + /// + /// The `tx_resp_builder` closure transforms each transaction into the desired response + /// type. + /// + /// `header_builder` transforms the block header into RPC representation. It takes the + /// consensus header and RLP length of the block which is a common dependency of RPC + /// headers. + pub fn into_rpc_block( + self, + kind: BlockTransactionsKind, + tx_resp_builder: F, + header_builder: impl FnOnce(SealedHeader, usize) -> Result, + ) -> Result, E> + where + F: Fn( + Recovered<<::Body as BlockBodyTrait>::Transaction>, + TransactionInfo, + ) -> Result, + { + match kind { + BlockTransactionsKind::Hashes => self.into_rpc_block_with_tx_hashes(header_builder), + BlockTransactionsKind::Full => { + self.into_rpc_block_full(tx_resp_builder, header_builder) + } + } + } + + /// Converts the block to an RPC [`Block`] without consuming self. + /// + /// For transaction hashes, only necessary parts are cloned for efficiency. + /// For full transactions, the entire block is cloned. + /// + /// The `tx_resp_builder` closure transforms each transaction into the desired response + /// type. + /// + /// `header_builder` transforms the block header into RPC representation. It takes the + /// consensus header and RLP length of the block which is a common dependency of RPC + /// headers. + pub fn clone_into_rpc_block( + &self, + kind: BlockTransactionsKind, + tx_resp_builder: F, + header_builder: impl FnOnce(SealedHeader, usize) -> Result, + ) -> Result, E> + where + F: Fn( + Recovered<<::Body as BlockBodyTrait>::Transaction>, + TransactionInfo, + ) -> Result, + { + match kind { + BlockTransactionsKind::Hashes => self.to_rpc_block_with_tx_hashes(header_builder), + BlockTransactionsKind::Full => { + self.clone().into_rpc_block_full(tx_resp_builder, header_builder) + } + } + } + + /// Creates an RPC [`Block`] with transaction hashes from a reference. + /// + /// Returns [`BlockTransactions::Hashes`] containing only transaction hashes. + /// Efficiently clones only necessary parts, not the entire block. + pub fn to_rpc_block_with_tx_hashes( + &self, + header_builder: impl FnOnce(SealedHeader, usize) -> Result, + ) -> Result, E> { + let transactions = self.body().transaction_hashes_iter().copied().collect(); + let rlp_length = self.rlp_length(); + let header = self.clone_sealed_header(); + let withdrawals = self.body().withdrawals().cloned(); + + let transactions = BlockTransactions::Hashes(transactions); + let uncles = + self.body().ommers().unwrap_or(&[]).iter().map(|h| h.hash_slow()).collect(); + let header = header_builder(header, rlp_length)?; + + Ok(Block { header, uncles, transactions, withdrawals }) + } + + /// Converts the block into an RPC [`Block`] with transaction hashes. + /// + /// Consumes self and returns [`BlockTransactions::Hashes`] containing only transaction + /// hashes. + pub fn into_rpc_block_with_tx_hashes( + self, + f: impl FnOnce(SealedHeader, usize) -> Result, + ) -> Result, E> { + let transactions = self.body().transaction_hashes_iter().copied().collect(); + let rlp_length = self.rlp_length(); + let (header, body) = self.into_sealed_block().split_sealed_header_body(); + let BlockBody { ommers, withdrawals, .. } = body.into_ethereum_body(); + + let transactions = BlockTransactions::Hashes(transactions); + let uncles = ommers.into_iter().map(|h| h.hash_slow()).collect(); + let header = f(header, rlp_length)?; + + Ok(Block { header, uncles, transactions, withdrawals }) + } + + /// Converts the block into an RPC [`Block`] with full transaction objects. + /// + /// Returns [`BlockTransactions::Full`] with complete transaction data. + /// The `tx_resp_builder` closure transforms each transaction with its metadata. + pub fn into_rpc_block_full( + self, + tx_resp_builder: F, + header_builder: impl FnOnce(SealedHeader, usize) -> Result, + ) -> Result, E> + where + F: Fn( + Recovered<<::Body as BlockBodyTrait>::Transaction>, + TransactionInfo, + ) -> Result, + { + let block_number = self.header().number(); + let base_fee = self.header().base_fee_per_gas(); + let block_length = self.rlp_length(); + let block_hash = Some(self.hash()); + + let (block, senders) = self.split_sealed(); + let (header, body) = block.split_sealed_header_body(); + let BlockBody { transactions, ommers, withdrawals } = body.into_ethereum_body(); + + let transactions = transactions + .into_iter() + .zip(senders) + .enumerate() + .map(|(idx, (tx, sender))| { + let tx_info = TransactionInfo { + hash: Some(*tx.tx_hash()), + block_hash, + block_number: Some(block_number), + base_fee, + index: Some(idx as u64), + }; + + tx_resp_builder(Recovered::new_unchecked(tx, sender), tx_info) + }) + .collect::, E>>()?; + + let transactions = BlockTransactions::Full(transactions); + let uncles = ommers.into_iter().map(|h| h.hash_slow()).collect(); + let header = header_builder(header, block_length)?; + + let block = Block { header, uncles, transactions, withdrawals }; + + Ok(block) + } + } + + impl RecoveredBlock> + where + T: SignedTransaction, + { + /// Creates a `RecoveredBlock` from an RPC block. + /// + /// Converts the RPC block to consensus format and recovers transaction senders. + /// Works with any transaction type `U` that can be converted to `T`. + /// + /// # Examples + /// ```ignore + /// let rpc_block: alloy_rpc_types_eth::Block = get_rpc_block(); + /// let recovered = RecoveredBlock::from_rpc_block(rpc_block)?; + /// ``` + pub fn from_rpc_block( + block: alloy_rpc_types_eth::Block, + ) -> Result>> + where + T: From, + { + // Convert to consensus block and then convert transactions + let consensus_block = block.into_consensus().convert_transactions(); + + // Try to recover the block + consensus_block.try_into_recovered() + } + } + + impl TryFrom> for RecoveredBlock> + where + T: SignedTransaction + From, + { + type Error = BlockRecoveryError>; + + fn try_from(block: alloy_rpc_types_eth::Block) -> Result { + Self::from_rpc_block(block) + } + } +} + /// Bincode-compatible [`RecoveredBlock`] serde implementation. #[cfg(feature = "serde-bincode-compat")] pub(super) mod serde_bincode_compat { @@ -622,3 +956,48 @@ pub(super) mod serde_bincode_compat { } } } + +#[cfg(test)] +mod tests { + use super::*; + use alloy_consensus::{Header, TxLegacy}; + use alloy_primitives::{bytes, Signature, TxKind}; + + #[test] + fn test_from_block_with_recovered_transactions() { + let tx = TxLegacy { + chain_id: Some(1), + nonce: 0, + gas_price: 21_000_000_000, + gas_limit: 21_000, + to: TxKind::Call(Address::ZERO), + value: U256::ZERO, + input: bytes!(), + }; + + let signature = Signature::new(U256::from(1), U256::from(2), false); + let sender = Address::from([0x01; 20]); + + let signed_tx = alloy_consensus::TxEnvelope::Legacy( + alloy_consensus::Signed::new_unchecked(tx, signature, B256::ZERO), + ); + + let recovered_tx = Recovered::new_unchecked(signed_tx, sender); + + let header = Header::default(); + let body = alloy_consensus::BlockBody { + transactions: vec![recovered_tx], + ommers: vec![], + withdrawals: None, + }; + let block_with_recovered = alloy_consensus::Block::new(header, body); + + let recovered_block: RecoveredBlock< + alloy_consensus::Block, + > = block_with_recovered.into(); + + assert_eq!(recovered_block.senders().len(), 1); + assert_eq!(recovered_block.senders()[0], sender); + assert_eq!(recovered_block.body().transactions().count(), 1); + } +} diff --git a/crates/primitives-traits/src/block/sealed.rs b/crates/primitives-traits/src/block/sealed.rs index dd0bc0b6652..5c43178146b 100644 --- a/crates/primitives-traits/src/block/sealed.rs +++ b/crates/primitives-traits/src/block/sealed.rs @@ -308,7 +308,8 @@ impl Deref for SealedBlock { impl Encodable for SealedBlock { fn encode(&self, out: &mut dyn BufMut) { - self.body.encode(out); + // TODO: https://github.com/paradigmxyz/reth/issues/18002 + self.clone().into_block().encode(out); } } @@ -349,7 +350,7 @@ impl SealedBlock { self.header.set_hash(hash) } - /// Returns a mutable reference to the header. + /// Returns a mutable reference to the body. pub const fn body_mut(&mut self) -> &mut B::Body { &mut self.body } @@ -469,3 +470,84 @@ pub(super) mod serde_bincode_compat { } } } + +#[cfg(test)] +mod tests { + use super::*; + use alloy_rlp::{Decodable, Encodable}; + + #[test] + fn test_sealed_block_rlp_roundtrip() { + // Create a sample block using alloy_consensus::Block + let header = alloy_consensus::Header { + parent_hash: B256::ZERO, + ommers_hash: B256::ZERO, + beneficiary: Address::ZERO, + state_root: B256::ZERO, + transactions_root: B256::ZERO, + receipts_root: B256::ZERO, + logs_bloom: Default::default(), + difficulty: Default::default(), + number: 42, + gas_limit: 30_000_000, + gas_used: 21_000, + timestamp: 1_000_000, + extra_data: Default::default(), + mix_hash: B256::ZERO, + nonce: Default::default(), + base_fee_per_gas: Some(1_000_000_000), + withdrawals_root: None, + blob_gas_used: None, + excess_blob_gas: None, + parent_beacon_block_root: None, + requests_hash: None, + }; + + // Create a simple transaction + let tx = alloy_consensus::TxLegacy { + chain_id: Some(1), + nonce: 0, + gas_price: 21_000_000_000, + gas_limit: 21_000, + to: alloy_primitives::TxKind::Call(Address::ZERO), + value: alloy_primitives::U256::from(100), + input: alloy_primitives::Bytes::default(), + }; + + let tx_signed = + alloy_consensus::TxEnvelope::Legacy(alloy_consensus::Signed::new_unchecked( + tx, + alloy_primitives::Signature::test_signature(), + B256::ZERO, + )); + + // Create block body with the transaction + let body = alloy_consensus::BlockBody { + transactions: vec![tx_signed], + ommers: vec![], + withdrawals: Some(Default::default()), + }; + + // Create the block + let block = alloy_consensus::Block::new(header, body); + + // Create a sealed block + let sealed_block = SealedBlock::seal_slow(block); + + // Encode the sealed block + let mut encoded = Vec::new(); + sealed_block.encode(&mut encoded); + + // Decode the sealed block + let decoded = SealedBlock::< + alloy_consensus::Block, + >::decode(&mut encoded.as_slice()) + .expect("Failed to decode sealed block"); + + // Verify the roundtrip + assert_eq!(sealed_block.hash(), decoded.hash()); + assert_eq!(sealed_block.header().number, decoded.header().number); + assert_eq!(sealed_block.header().state_root, decoded.header().state_root); + assert_eq!(sealed_block.body().transactions.len(), decoded.body().transactions.len()); + } +} diff --git a/crates/primitives-traits/src/constants/gas_units.rs b/crates/primitives-traits/src/constants/gas_units.rs index 312ae51cbc6..e311e34d0aa 100644 --- a/crates/primitives-traits/src/constants/gas_units.rs +++ b/crates/primitives-traits/src/constants/gas_units.rs @@ -19,11 +19,11 @@ pub const GIGAGAS: u64 = MEGAGAS * 1_000; pub fn format_gas_throughput(gas: u64, execution_duration: Duration) -> String { let gas_per_second = gas as f64 / execution_duration.as_secs_f64(); if gas_per_second < MEGAGAS as f64 { - format!("{:.2} Kgas/second", gas_per_second / KILOGAS as f64) + format!("{:.2}Kgas/second", gas_per_second / KILOGAS as f64) } else if gas_per_second < GIGAGAS as f64 { - format!("{:.2} Mgas/second", gas_per_second / MEGAGAS as f64) + format!("{:.2}Mgas/second", gas_per_second / MEGAGAS as f64) } else { - format!("{:.2} Ggas/second", gas_per_second / GIGAGAS as f64) + format!("{:.2}Ggas/second", gas_per_second / GIGAGAS as f64) } } @@ -36,11 +36,11 @@ pub fn format_gas_throughput(gas: u64, execution_duration: Duration) -> String { pub fn format_gas(gas: u64) -> String { let gas = gas as f64; if gas < MEGAGAS as f64 { - format!("{:.2} Kgas", gas / KILOGAS as f64) + format!("{:.2}Kgas", gas / KILOGAS as f64) } else if gas < GIGAGAS as f64 { - format!("{:.2} Mgas", gas / MEGAGAS as f64) + format!("{:.2}Mgas", gas / MEGAGAS as f64) } else { - format!("{:.2} Ggas", gas / GIGAGAS as f64) + format!("{:.2}Ggas", gas / GIGAGAS as f64) } } @@ -50,17 +50,21 @@ mod tests { #[test] fn test_gas_fmt() { + let gas = 888; + let gas_unit = format_gas(gas); + assert_eq!(gas_unit, "0.89Kgas"); + let gas = 100_000; let gas_unit = format_gas(gas); - assert_eq!(gas_unit, "100.00 Kgas"); + assert_eq!(gas_unit, "100.00Kgas"); let gas = 100_000_000; let gas_unit = format_gas(gas); - assert_eq!(gas_unit, "100.00 Mgas"); + assert_eq!(gas_unit, "100.00Mgas"); let gas = 100_000_000_000; let gas_unit = format_gas(gas); - assert_eq!(gas_unit, "100.00 Ggas"); + assert_eq!(gas_unit, "100.00Ggas"); } #[test] @@ -68,14 +72,14 @@ mod tests { let duration = Duration::from_secs(1); let gas = 100_000; let throughput = format_gas_throughput(gas, duration); - assert_eq!(throughput, "100.00 Kgas/second"); + assert_eq!(throughput, "100.00Kgas/second"); let gas = 100_000_000; let throughput = format_gas_throughput(gas, duration); - assert_eq!(throughput, "100.00 Mgas/second"); + assert_eq!(throughput, "100.00Mgas/second"); let gas = 100_000_000_000; let throughput = format_gas_throughput(gas, duration); - assert_eq!(throughput, "100.00 Ggas/second"); + assert_eq!(throughput, "100.00Ggas/second"); } } diff --git a/crates/primitives-traits/src/constants/mod.rs b/crates/primitives-traits/src/constants/mod.rs index b5bf127fdf3..a9aa18fac31 100644 --- a/crates/primitives-traits/src/constants/mod.rs +++ b/crates/primitives-traits/src/constants/mod.rs @@ -17,6 +17,9 @@ pub const MAXIMUM_GAS_LIMIT_BLOCK: u64 = 2u64.pow(63) - 1; /// The bound divisor of the gas limit, used in update calculations. pub const GAS_LIMIT_BOUND_DIVISOR: u64 = 1024; +/// Maximum transaction gas limit as defined by [EIP-7825](https://eips.ethereum.org/EIPS/eip-7825) activated in `Osaka` hardfork. +pub const MAX_TX_GAS_LIMIT_OSAKA: u64 = 2u64.pow(24); + /// The number of blocks to unwind during a reorg that already became a part of canonical chain. /// /// In reality, the node can end up in this particular situation very rarely. It would happen only diff --git a/crates/primitives-traits/src/extended.rs b/crates/primitives-traits/src/extended.rs index ddf339e7f17..da2bbc533aa 100644 --- a/crates/primitives-traits/src/extended.rs +++ b/crates/primitives-traits/src/extended.rs @@ -3,7 +3,10 @@ use crate::{ transaction::signed::{RecoveryError, SignedTransaction}, }; use alloc::vec::Vec; -use alloy_consensus::Transaction; +use alloy_consensus::{ + transaction::{SignerRecoverable, TxHashRef}, + EthereumTxEnvelope, Transaction, +}; use alloy_eips::{ eip2718::{Eip2718Error, Eip2718Result, IsTyped2718}, eip2930::AccessList, @@ -23,23 +26,24 @@ macro_rules! delegate { }; } -/// A [`SignedTransaction`] implementation that combines two different transaction types. +/// An enum that combines two different transaction types. /// -/// This is intended to be used to extend existing presets, for example the ethereum or optstack -/// transaction types. +/// This is intended to be used to extend existing presets, for example the ethereum or opstack +/// transaction types and receipts /// -/// Note: The other transaction type variants must not overlap with the builtin one, transaction -/// types must be unique. +/// Note: The [`Extended::Other`] variants must not overlap with the builtin one, transaction +/// types must be unique. For example if [`Extended::BuiltIn`] contains an `EIP-1559` type variant, +/// [`Extended::Other`] must not include that type. #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[derive(Debug, Clone, Hash, Eq, PartialEq)] -pub enum ExtendedTxEnvelope { +pub enum Extended { /// The builtin transaction type. BuiltIn(BuiltIn), /// The other transaction type. Other(Other), } -impl Transaction for ExtendedTxEnvelope +impl Transaction for Extended where B: Transaction, T: Transaction, @@ -91,7 +95,7 @@ where fn is_create(&self) -> bool { match self { Self::BuiltIn(tx) => tx.is_create(), - Self::Other(_tx) => false, + Self::Other(tx) => tx.is_create(), } } @@ -116,7 +120,7 @@ where } } -impl IsTyped2718 for ExtendedTxEnvelope +impl IsTyped2718 for Extended where B: IsTyped2718, T: IsTyped2718, @@ -126,7 +130,7 @@ where } } -impl InMemorySize for ExtendedTxEnvelope +impl InMemorySize for Extended where B: InMemorySize, T: InMemorySize, @@ -136,18 +140,11 @@ where } } -impl SignedTransaction for ExtendedTxEnvelope +impl SignerRecoverable for Extended where - B: SignedTransaction + IsTyped2718, - T: SignedTransaction, + B: SignerRecoverable, + T: SignerRecoverable, { - fn tx_hash(&self) -> &TxHash { - match self { - Self::BuiltIn(tx) => tx.tx_hash(), - Self::Other(tx) => tx.tx_hash(), - } - } - fn recover_signer(&self) -> Result { delegate!(self => tx.recover_signer()) } @@ -156,15 +153,29 @@ where delegate!(self => tx.recover_signer_unchecked()) } - fn recover_signer_unchecked_with_buf( - &self, - buf: &mut Vec, - ) -> Result { - delegate!(self => tx.recover_signer_unchecked_with_buf(buf)) + fn recover_unchecked_with_buf(&self, buf: &mut Vec) -> Result { + delegate!(self => tx.recover_unchecked_with_buf(buf)) + } +} + +impl TxHashRef for Extended +where + B: TxHashRef, + T: TxHashRef, +{ + fn tx_hash(&self) -> &TxHash { + delegate!(self => tx.tx_hash()) } } -impl Typed2718 for ExtendedTxEnvelope +impl SignedTransaction for Extended +where + B: SignedTransaction + IsTyped2718 + TxHashRef, + T: SignedTransaction + TxHashRef, +{ +} + +impl Typed2718 for Extended where B: Typed2718, T: Typed2718, @@ -177,7 +188,7 @@ where } } -impl Decodable2718 for ExtendedTxEnvelope +impl Decodable2718 for Extended where B: Decodable2718 + IsTyped2718, T: Decodable2718, @@ -199,7 +210,7 @@ where } } -impl Encodable2718 for ExtendedTxEnvelope +impl Encodable2718 for Extended where B: Encodable2718, T: Encodable2718, @@ -219,7 +230,7 @@ where } } -impl Encodable for ExtendedTxEnvelope +impl Encodable for Extended where B: Encodable, T: Encodable, @@ -239,7 +250,7 @@ where } } -impl Decodable for ExtendedTxEnvelope +impl Decodable for Extended where B: Decodable, T: Decodable, @@ -257,43 +268,75 @@ where } } +impl From> for Extended, Tx> { + fn from(value: EthereumTxEnvelope) -> Self { + Self::BuiltIn(value) + } +} + #[cfg(feature = "op")] mod op { - use crate::ExtendedTxEnvelope; + use crate::Extended; use alloy_consensus::error::ValueError; - use alloy_primitives::{Signature, B256}; - use op_alloy_consensus::{OpPooledTransaction, OpTxEnvelope}; + use alloy_primitives::{Sealed, Signature, B256}; + use op_alloy_consensus::{OpPooledTransaction, OpTransaction, OpTxEnvelope, TxDeposit}; - impl TryFrom> - for ExtendedTxEnvelope + impl OpTransaction for Extended + where + B: OpTransaction, + T: OpTransaction, { - type Error = OpTxEnvelope; + fn is_deposit(&self) -> bool { + match self { + Self::BuiltIn(b) => b.is_deposit(), + Self::Other(t) => t.is_deposit(), + } + } + + fn as_deposit(&self) -> Option<&Sealed> { + match self { + Self::BuiltIn(b) => b.as_deposit(), + Self::Other(t) => t.as_deposit(), + } + } + } + + impl TryFrom> for Extended { + type Error = >::Error; - fn try_from(value: ExtendedTxEnvelope) -> Result { + fn try_from(value: Extended) -> Result { match value { - ExtendedTxEnvelope::BuiltIn(tx) => { - let converted_tx: OpPooledTransaction = - tx.clone().try_into().map_err(|_| tx)?; + Extended::BuiltIn(tx) => { + let converted_tx: OpPooledTransaction = tx.try_into()?; Ok(Self::BuiltIn(converted_tx)) } - ExtendedTxEnvelope::Other(tx) => Ok(Self::Other(tx)), + Extended::Other(tx) => Ok(Self::Other(tx)), } } } - impl From for ExtendedTxEnvelope { + impl From for Extended { fn from(tx: OpPooledTransaction) -> Self { Self::BuiltIn(tx.into()) } } - impl TryFrom> for OpPooledTransaction { + impl From> for Extended { + fn from(tx: Extended) -> Self { + match tx { + Extended::BuiltIn(tx) => Self::BuiltIn(tx.into()), + Extended::Other(tx) => Self::Other(tx), + } + } + } + + impl TryFrom> for OpPooledTransaction { type Error = ValueError; - fn try_from(_tx: ExtendedTxEnvelope) -> Result { + fn try_from(_tx: Extended) -> Result { match _tx { - ExtendedTxEnvelope::BuiltIn(inner) => inner.try_into(), - ExtendedTxEnvelope::Other(_tx) => Err(ValueError::new( + Extended::BuiltIn(inner) => inner.try_into(), + Extended::Other(_tx) => Err(ValueError::new( OpTxEnvelope::Legacy(alloy_consensus::Signed::new_unchecked( alloy_consensus::TxLegacy::default(), Signature::decode_rlp_vrs(&mut &[0u8; 65][..], |_| Ok(false)).unwrap(), @@ -304,6 +347,12 @@ mod op { } } } + + impl From for Extended { + fn from(value: OpTxEnvelope) -> Self { + Self::BuiltIn(value) + } + } } #[cfg(feature = "serde-bincode-compat")] @@ -313,29 +362,29 @@ mod serde_bincode_compat { #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[derive(Debug)] - pub enum ExtendedTxEnvelopeRepr<'a, B: SerdeBincodeCompat, T: SerdeBincodeCompat> { + pub enum ExtendedRepr<'a, B: SerdeBincodeCompat, T: SerdeBincodeCompat> { BuiltIn(B::BincodeRepr<'a>), Other(T::BincodeRepr<'a>), } - impl SerdeBincodeCompat for ExtendedTxEnvelope + impl SerdeBincodeCompat for Extended where B: SerdeBincodeCompat + core::fmt::Debug, T: SerdeBincodeCompat + core::fmt::Debug, { - type BincodeRepr<'a> = ExtendedTxEnvelopeRepr<'a, B, T>; + type BincodeRepr<'a> = ExtendedRepr<'a, B, T>; fn as_repr(&self) -> Self::BincodeRepr<'_> { match self { - Self::BuiltIn(tx) => ExtendedTxEnvelopeRepr::BuiltIn(tx.as_repr()), - Self::Other(tx) => ExtendedTxEnvelopeRepr::Other(tx.as_repr()), + Self::BuiltIn(tx) => ExtendedRepr::BuiltIn(tx.as_repr()), + Self::Other(tx) => ExtendedRepr::Other(tx.as_repr()), } } fn from_repr(repr: Self::BincodeRepr<'_>) -> Self { match repr { - ExtendedTxEnvelopeRepr::BuiltIn(tx_repr) => Self::BuiltIn(B::from_repr(tx_repr)), - ExtendedTxEnvelopeRepr::Other(tx_repr) => Self::Other(T::from_repr(tx_repr)), + ExtendedRepr::BuiltIn(tx_repr) => Self::BuiltIn(B::from_repr(tx_repr)), + ExtendedRepr::Other(tx_repr) => Self::Other(T::from_repr(tx_repr)), } } } @@ -345,7 +394,7 @@ mod serde_bincode_compat { use alloy_primitives::bytes::Buf; #[cfg(feature = "reth-codec")] -impl reth_codecs::Compact for ExtendedTxEnvelope +impl reth_codecs::Compact for Extended where B: Transaction + IsTyped2718 + reth_codecs::Compact, T: Transaction + reth_codecs::Compact, diff --git a/crates/primitives-traits/src/header/error.rs b/crates/primitives-traits/src/header/error.rs deleted file mode 100644 index 3905d831053..00000000000 --- a/crates/primitives-traits/src/header/error.rs +++ /dev/null @@ -1,8 +0,0 @@ -/// Errors that can occur during header sanity checks. -#[derive(Debug, PartialEq, Eq)] -pub enum HeaderError { - /// Represents an error when the block difficulty is too large. - LargeDifficulty, - /// Represents an error when the block extra data is too large. - LargeExtraData, -} diff --git a/crates/primitives-traits/src/header/mod.rs b/crates/primitives-traits/src/header/mod.rs index 7f3a5ab0660..198b9cb3c8f 100644 --- a/crates/primitives-traits/src/header/mod.rs +++ b/crates/primitives-traits/src/header/mod.rs @@ -1,9 +1,6 @@ mod sealed; pub use sealed::{Header, SealedHeader, SealedHeaderFor}; -mod error; -pub use error::HeaderError; - #[cfg(any(test, feature = "test-utils", feature = "arbitrary"))] pub mod test_utils; diff --git a/crates/primitives-traits/src/header/sealed.rs b/crates/primitives-traits/src/header/sealed.rs index b84a7fa622f..bcf69813f97 100644 --- a/crates/primitives-traits/src/header/sealed.rs +++ b/crates/primitives-traits/src/header/sealed.rs @@ -239,6 +239,34 @@ impl SealedHeader { } } +#[cfg(feature = "rpc-compat")] +mod rpc_compat { + use super::*; + + impl SealedHeader { + /// Converts this header into `alloy_rpc_types_eth::Header`. + /// + /// Note: This does not set the total difficulty or size of the block. + pub fn into_rpc_header(self) -> alloy_rpc_types_eth::Header + where + H: Sealable, + { + alloy_rpc_types_eth::Header::from_sealed(self.into()) + } + + /// Converts an `alloy_rpc_types_eth::Header` into a `SealedHeader`. + pub fn from_rpc_header(header: alloy_rpc_types_eth::Header) -> Self { + Self::new(header.inner, header.hash) + } + } + + impl From> for SealedHeader { + fn from(value: alloy_rpc_types_eth::Header) -> Self { + Self::from_rpc_header(value) + } + } +} + /// Bincode-compatible [`SealedHeader`] serde implementation. #[cfg(feature = "serde-bincode-compat")] pub(super) mod serde_bincode_compat { diff --git a/crates/primitives-traits/src/lib.rs b/crates/primitives-traits/src/lib.rs index 4a6d58ab8db..5400f52a204 100644 --- a/crates/primitives-traits/src/lib.rs +++ b/crates/primitives-traits/src/lib.rs @@ -1,10 +1,18 @@ //! Commonly used types and traits in Reth. //! -//! This crate contains various primitive traits used across reth's components. -//! It provides the [`Block`] trait which is used to represent a block and all its components. -//! A [`Block`] is composed of a [`Header`] and a [`BlockBody`]. In ethereum (and optimism), a block -//! body consists of a list of transactions, a list of uncle headers, and a list of withdrawals. For -//! optimism, uncle headers and withdrawals are always empty lists. +//! ## Overview +//! +//! This crate defines various traits and types that form the foundation of the reth stack. +//! The top-level trait is [`Block`] which represents a block in the blockchain. A [`Block`] is +//! composed of a [`Header`] and a [`BlockBody`]. A [`BlockBody`] contains the transactions in the +//! block and additional data that is part of the block. In ethereum, this includes uncle headers +//! and withdrawals. For optimism, uncle headers and withdrawals are always empty lists. +//! +//! The most common types you'll use are: +//! - [`Block`] - A basic block with header and body +//! - [`SealedBlock`] - A block with its hash cached +//! - [`SealedHeader`] - A header with its hash cached +//! - [`RecoveredBlock`] - A sealed block with sender addresses recovered //! //! ## Feature Flags //! @@ -13,20 +21,14 @@ //! types. //! - `reth-codec`: Enables db codec support for reth types including zstd compression for certain //! types. +//! - `rpc-compat`: Adds RPC compatibility functions for the types in this crate, e.g. rpc type +//! conversions. //! - `serde`: Adds serde support for all types. //! - `secp256k1`: Adds secp256k1 support for transaction signing/recovery. (By default the no-std //! friendly `k256` is used) //! - `rayon`: Uses `rayon` for parallel transaction sender recovery in [`BlockBody`] by default. //! - `serde-bincode-compat` provides helpers for dealing with the `bincode` crate. //! -//! ## Overview -//! -//! This crate defines various traits and types that form the foundation of the reth stack. -//! The top-level trait is [`Block`] which represents a block in the blockchain. A [`Block`] is -//! composed of a [`Header`] and a [`BlockBody`]. A [`BlockBody`] contains the transactions in the -//! block any additional data that is part of the block. A [`Header`] contains the metadata of the -//! block. -//! //! ### Sealing (Hashing) //! //! The block hash is derived from the [`Header`] and is used to uniquely identify the block. This @@ -48,19 +50,60 @@ //! #### Naming //! //! The types in this crate support multiple recovery functions, e.g. -//! [`SealedBlock::try_recover_unchecked`] and [`SealedBlock::try_recover_unchecked`]. The `_unchecked` suffix indicates that this function recovers the signer _without ensuring that the signature has a low `s` value_, in other words this rule introduced in [EIP-2](https://github.com/ethereum/EIPs/blob/master/EIPS/eip-2.md) is ignored. +//! [`SealedBlock::try_recover`] and [`SealedBlock::try_recover_unchecked`]. The `_unchecked` suffix indicates that this function recovers the signer _without ensuring that the signature has a low `s` value_, in other words this rule introduced in [EIP-2](https://github.com/ethereum/EIPs/blob/master/EIPS/eip-2.md) is ignored. //! Hence this function is necessary when dealing with pre EIP-2 transactions on the ethereum //! mainnet. Newer transactions must always be recovered with the regular `recover` functions, see //! also [`recover_signer`](crypto::secp256k1::recover_signer). //! +//! ## Error Handling +//! +//! Most operations that can fail return `Result` types: +//! - [`RecoveryError`](transaction::signed::RecoveryError) - Transaction signature recovery failed +//! - [`BlockRecoveryError`](block::error::BlockRecoveryError) - Block-level recovery failed +//! - [`GotExpected`] / [`GotExpectedBoxed`] - Generic error for mismatched values +//! +//! Recovery errors typically indicate invalid signatures or corrupted data. The block recovery +//! error preserves the original block for further inspection. +//! +//! ### Example +//! +//! ```rust +//! # use reth_primitives_traits::{SealedBlock, RecoveredBlock}; +//! # use reth_primitives_traits::block::error::BlockRecoveryError; +//! # fn example(sealed_block: SealedBlock) -> Result<(), BlockRecoveryError>> +//! # where B::Body: reth_primitives_traits::BlockBody { +//! // Attempt to recover senders from a sealed block +//! match sealed_block.try_recover() { +//! Ok(recovered) => { +//! // Successfully recovered all senders +//! println!("Recovered {} senders", recovered.senders().len()); +//! Ok(()) +//! } +//! Err(err) => { +//! // Recovery failed - the block is returned in the error +//! println!("Failed to recover senders for block"); +//! // You can still access the original block +//! let block = err.into_inner(); +//! let hash = block.hash(); +//! Err(BlockRecoveryError::new(block)) +//! } +//! } +//! # } +//! ``` +//! +//! ## Performance Considerations +//! +//! - **Hashing**: Block hashing is expensive. Use [`SealedBlock`] to cache hashes. +//! - **Recovery**: Sender recovery is CPU-intensive. Use [`RecoveredBlock`] to cache results. +//! - **Parallel Recovery**: Enable the `rayon` feature for parallel transaction recovery. +//! //! ## Bincode serde compatibility //! //! The [bincode-crate](https://github.com/bincode-org/bincode) is often used by additional tools when sending data over the network. //! `bincode` crate doesn't work well with optionally serializable serde fields, but some of the consensus types require optional serialization for RPC compatibility. Read more: //! -//! As a workaround this crate introduces the -//! [`SerdeBincodeCompat`](serde_bincode_compat::SerdeBincodeCompat) trait used to a bincode -//! compatible serde representation. +//! As a workaround this crate introduces the `SerdeBincodeCompat` trait (available with the +//! `serde-bincode-compat` feature) used to provide a bincode compatible serde representation. #![doc( html_logo_url = "https://raw.githubusercontent.com/paradigmxyz/reth/main/assets/reth-docs.png", @@ -68,7 +111,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(feature = "std"), no_std)] #[macro_use] @@ -94,16 +137,18 @@ pub use alloy_consensus::{ pub use transaction::{ execute::FillTxEnv, signed::{FullSignedTx, SignedTransaction}, - FullTransaction, Transaction, + FullTransaction, SignerRecoverable, Transaction, }; pub mod block; pub use block::{ body::{BlockBody, FullBlockBody}, header::{AlloyBlockHeader, BlockHeader, FullBlockHeader}, + recovered::IndexedTx, Block, FullBlock, RecoveredBlock, SealedBlock, }; +#[cfg(test)] mod withdrawal; pub use alloy_eips::eip2718::WithEncoded; @@ -112,6 +157,7 @@ pub mod crypto; mod error; pub use error::{GotExpected, GotExpectedBoxed}; +#[cfg(test)] mod log; pub use alloy_primitives::{logs_bloom, Log, LogData}; @@ -123,10 +169,10 @@ pub use storage::StorageEntry; pub mod sync; mod extended; -pub use extended::ExtendedTxEnvelope; +pub use extended::Extended; /// Common header types pub mod header; -pub use header::{Header, HeaderError, SealedHeader, SealedHeaderFor}; +pub use header::{Header, SealedHeader, SealedHeaderFor}; /// Bincode-compatible serde implementations for common abstracted types in Reth. /// @@ -144,7 +190,7 @@ pub use size::InMemorySize; /// Node traits pub mod node; -pub use node::{BlockTy, BodyTy, FullNodePrimitives, HeaderTy, NodePrimitives, ReceiptTy, TxTy}; +pub use node::{BlockTy, BodyTy, HeaderTy, NodePrimitives, ReceiptTy, TxTy}; /// Helper trait that requires de-/serialize implementation since `serde` feature is enabled. #[cfg(feature = "serde")] diff --git a/crates/primitives-traits/src/node.rs b/crates/primitives-traits/src/node.rs index 59181a412cd..f23ff222ab6 100644 --- a/crates/primitives-traits/src/node.rs +++ b/crates/primitives-traits/src/node.rs @@ -1,62 +1,30 @@ use crate::{ - Block, FullBlock, FullBlockBody, FullBlockHeader, FullReceipt, FullSignedTx, - MaybeSerdeBincodeCompat, Receipt, + FullBlock, FullBlockBody, FullBlockHeader, FullReceipt, FullSignedTx, MaybeSerdeBincodeCompat, }; use core::fmt; /// Configures all the primitive types of the node. +/// +/// This trait defines the core types used throughout the node for representing +/// blockchain data. It serves as the foundation for type consistency across +/// different node implementations. pub trait NodePrimitives: Send + Sync + Unpin + Clone + Default + fmt::Debug + PartialEq + Eq + 'static { /// Block primitive. - type Block: Block
+ MaybeSerdeBincodeCompat; + type Block: FullBlock
+ + MaybeSerdeBincodeCompat; /// Block header primitive. type BlockHeader: FullBlockHeader; /// Block body primitive. type BlockBody: FullBlockBody; /// Signed version of the transaction type. + /// + /// This represents the transaction as it exists in the blockchain - the consensus + /// format that includes the signature and can be included in a block. type SignedTx: FullSignedTx; /// A receipt. - type Receipt: Receipt; -} -/// Helper trait that sets trait bounds on [`NodePrimitives`]. -pub trait FullNodePrimitives -where - Self: NodePrimitives< - Block: FullBlock
, - BlockHeader: FullBlockHeader, - BlockBody: FullBlockBody, - SignedTx: FullSignedTx, - Receipt: FullReceipt, - > + Send - + Sync - + Unpin - + Clone - + Default - + fmt::Debug - + PartialEq - + Eq - + 'static, -{ -} - -impl FullNodePrimitives for T where - T: NodePrimitives< - Block: FullBlock
, - BlockHeader: FullBlockHeader, - BlockBody: FullBlockBody, - SignedTx: FullSignedTx, - Receipt: FullReceipt, - > + Send - + Sync - + Unpin - + Clone - + Default - + fmt::Debug - + PartialEq - + Eq - + 'static -{ + type Receipt: FullReceipt; } /// Helper adapter type for accessing [`NodePrimitives`] block header types. diff --git a/crates/primitives-traits/src/receipt.rs b/crates/primitives-traits/src/receipt.rs index a8d632c3569..9be419987f0 100644 --- a/crates/primitives-traits/src/receipt.rs +++ b/crates/primitives-traits/src/receipt.rs @@ -5,6 +5,7 @@ use alloc::vec::Vec; use alloy_consensus::{ Eip2718EncodableReceipt, RlpDecodableReceipt, RlpEncodableReceipt, TxReceipt, Typed2718, }; +use alloy_rlp::{Decodable, Encodable}; use core::fmt; /// Helper trait that unifies all behaviour required by receipt to support full node operations. @@ -13,7 +14,6 @@ pub trait FullReceipt: Receipt + MaybeCompact {} impl FullReceipt for T where T: Receipt + MaybeCompact {} /// Abstraction of a receipt. -#[auto_impl::auto_impl(&, Arc)] pub trait Receipt: Send + Sync @@ -23,6 +23,8 @@ pub trait Receipt: + TxReceipt + RlpEncodableReceipt + RlpDecodableReceipt + + Encodable + + Decodable + Eip2718EncodableReceipt + Typed2718 + MaybeSerde @@ -31,6 +33,26 @@ pub trait Receipt: { } +// Blanket implementation for any type that satisfies all the supertrait bounds +impl Receipt for T where + T: Send + + Sync + + Unpin + + Clone + + fmt::Debug + + TxReceipt + + RlpEncodableReceipt + + RlpDecodableReceipt + + Encodable + + Decodable + + Eip2718EncodableReceipt + + Typed2718 + + MaybeSerde + + InMemorySize + + MaybeSerdeBincodeCompat +{ +} + /// Retrieves gas spent by transactions as a vector of tuples (transaction index, gas used). pub fn gas_spent_by_transactions(receipts: I) -> Vec<(u64, u64)> where diff --git a/crates/primitives-traits/src/serde_bincode_compat.rs b/crates/primitives-traits/src/serde_bincode_compat.rs index 8b3ca7a594b..217ad5ff332 100644 --- a/crates/primitives-traits/src/serde_bincode_compat.rs +++ b/crates/primitives-traits/src/serde_bincode_compat.rs @@ -1,3 +1,38 @@ +//! Bincode compatibility support for reth primitive types. +//! +//! This module provides traits and implementations to work around bincode's limitations +//! with optional serde fields. The bincode crate requires all fields to be present during +//! serialization, which conflicts with types that have `#[serde(skip_serializing_if)]` +//! attributes for RPC compatibility. +//! +//! # Overview +//! +//! The main trait is `SerdeBincodeCompat`, which provides a conversion mechanism between +//! types and their bincode-compatible representations. There are two main ways to implement +//! this trait: +//! +//! 1. **Using RLP encoding** - Implement `RlpBincode` for types that already support RLP +//! 2. **Custom implementation** - Define a custom representation type +//! +//! # Examples +//! +//! ## Using with `serde_with` +//! +//! ```rust +//! # use reth_primitives_traits::serde_bincode_compat::{self, SerdeBincodeCompat}; +//! # use serde::{Deserialize, Serialize}; +//! # use serde_with::serde_as; +//! # use alloy_consensus::Header; +//! #[serde_as] +//! #[derive(Serialize, Deserialize)] +//! struct MyStruct { +//! #[serde_as(as = "serde_bincode_compat::BincodeReprFor<'_, Header>")] +//! data: Header, +//! } +//! ``` + +use alloc::vec::Vec; +use alloy_primitives::Bytes; use core::fmt::Debug; use serde::{de::DeserializeOwned, Serialize}; @@ -9,8 +44,26 @@ pub use block_bincode::{Block, BlockBody}; /// Trait for types that can be serialized and deserialized using bincode. /// +/// This trait provides a workaround for bincode's incompatibility with optional +/// serde fields. It ensures all fields are serialized, making the type bincode-compatible. +/// +/// # Implementation +/// +/// The easiest way to implement this trait is using [`RlpBincode`] for RLP-encodable types: +/// +/// ```rust +/// # use reth_primitives_traits::serde_bincode_compat::RlpBincode; +/// # use alloy_rlp::{RlpEncodable, RlpDecodable}; +/// # #[derive(RlpEncodable, RlpDecodable)] +/// # struct MyType; +/// impl RlpBincode for MyType {} +/// // SerdeBincodeCompat is automatically implemented +/// ``` +/// +/// For custom implementations, see the examples in the `block` module. +/// /// The recommended way to add bincode compatible serialization is via the -/// [`serde_with`] crate and the `serde_as` macro that. See for reference [`header`]. +/// [`serde_with`] crate and the `serde_as` macro. See for reference [`header`]. pub trait SerdeBincodeCompat: Sized + 'static { /// Serde representation of the type for bincode serialization. /// @@ -37,8 +90,58 @@ impl SerdeBincodeCompat for alloy_consensus::Header { } /// Type alias for the [`SerdeBincodeCompat::BincodeRepr`] associated type. +/// +/// This provides a convenient way to refer to the bincode representation type +/// without having to write out the full associated type projection. +/// +/// # Example +/// +/// ```rust +/// # use reth_primitives_traits::serde_bincode_compat::{SerdeBincodeCompat, BincodeReprFor}; +/// fn serialize_to_bincode(value: &T) -> BincodeReprFor<'_, T> { +/// value.as_repr() +/// } +/// ``` pub type BincodeReprFor<'a, T> = ::BincodeRepr<'a>; +/// A helper trait for using RLP-encoding for providing bincode-compatible serialization. +/// +/// By implementing this trait, [`SerdeBincodeCompat`] will be automatically implemented for the +/// type and RLP encoding will be used for serialization and deserialization for bincode +/// compatibility. +/// +/// # Example +/// +/// ```rust +/// # use reth_primitives_traits::serde_bincode_compat::RlpBincode; +/// # use alloy_rlp::{RlpEncodable, RlpDecodable}; +/// #[derive(RlpEncodable, RlpDecodable)] +/// struct MyCustomType { +/// value: u64, +/// data: Vec, +/// } +/// +/// // Simply implement the marker trait +/// impl RlpBincode for MyCustomType {} +/// +/// // Now MyCustomType can be used with bincode through RLP encoding +/// ``` +pub trait RlpBincode: alloy_rlp::Encodable + alloy_rlp::Decodable {} + +impl SerdeBincodeCompat for T { + type BincodeRepr<'a> = Bytes; + + fn as_repr(&self) -> Self::BincodeRepr<'_> { + let mut buf = Vec::new(); + self.encode(&mut buf); + buf.into() + } + + fn from_repr(repr: Self::BincodeRepr<'_>) -> Self { + Self::decode(&mut repr.as_ref()).expect("Failed to decode bincode rlp representation") + } +} + mod block_bincode { use crate::serde_bincode_compat::SerdeBincodeCompat; use alloc::{borrow::Cow, vec::Vec}; diff --git a/crates/primitives-traits/src/size.rs b/crates/primitives-traits/src/size.rs index 0e3ad45aaa5..82c8b5d9c43 100644 --- a/crates/primitives-traits/src/size.rs +++ b/crates/primitives-traits/src/size.rs @@ -1,10 +1,10 @@ use alloc::vec::Vec; use alloy_consensus::{ - EthereumTxEnvelope, Header, TxEip1559, TxEip2930, TxEip4844, TxEip4844Variant, - TxEip4844WithSidecar, TxEip7702, TxLegacy, TxType, + transaction::TxEip4844Sidecar, EthereumTxEnvelope, Header, TxEip1559, TxEip2930, TxEip4844, + TxEip4844Variant, TxEip4844WithSidecar, TxEip7702, TxLegacy, TxType, }; use alloy_eips::eip4895::Withdrawals; -use alloy_primitives::{Signature, TxHash, B256}; +use alloy_primitives::{LogData, Signature, TxHash, B256}; use revm_primitives::Log; /// Trait for calculating a heuristic for the in-memory size of a struct. @@ -50,16 +50,21 @@ macro_rules! impl_in_mem_size { }; } -impl_in_mem_size!( - Header, - TxLegacy, - TxEip2930, - TxEip1559, - TxEip7702, - TxEip4844, - TxEip4844Variant, - TxEip4844WithSidecar -); +impl_in_mem_size!(Header, TxLegacy, TxEip2930, TxEip1559, TxEip7702, TxEip4844); + +impl InMemorySize for TxEip4844Variant { + #[inline] + fn size(&self) -> usize { + Self::size(self) + } +} + +impl InMemorySize for TxEip4844WithSidecar { + #[inline] + fn size(&self) -> usize { + Self::size(self) + } +} #[cfg(feature = "op")] impl_in_mem_size_size_of!(op_alloy_consensus::OpTxType); @@ -69,7 +74,19 @@ impl InMemorySize for alloy_consensus::Receipt { let Self { status, cumulative_gas_used, logs } = self; core::mem::size_of_val(status) + core::mem::size_of_val(cumulative_gas_used) + - logs.capacity() * core::mem::size_of::() + logs.iter().map(|log| log.size()).sum::() + } +} + +impl InMemorySize for LogData { + fn size(&self) -> usize { + self.data.len() + core::mem::size_of_val(self.topics()) + } +} + +impl InMemorySize for Log { + fn size(&self) -> usize { + core::mem::size_of_val(&self.address) + self.data.size() } } @@ -90,9 +107,7 @@ impl InMemorySize for alloy_consensus::BlockBo #[inline] fn size(&self) -> usize { self.transactions.iter().map(T::size).sum::() + - self.transactions.capacity() * core::mem::size_of::() + self.ommers.iter().map(H::size).sum::() + - self.ommers.capacity() * core::mem::size_of::
() + self.withdrawals .as_ref() .map_or(core::mem::size_of::>(), Withdrawals::total_size) diff --git a/crates/primitives-traits/src/transaction/access_list.rs b/crates/primitives-traits/src/transaction/access_list.rs index 8406e5a5b48..e4d5638f562 100644 --- a/crates/primitives-traits/src/transaction/access_list.rs +++ b/crates/primitives-traits/src/transaction/access_list.rs @@ -8,22 +8,11 @@ mod tests { use proptest::proptest; use proptest_arbitrary_interop::arb; use reth_codecs::{add_arbitrary_tests, Compact}; - use serde::{Deserialize, Serialize}; /// This type is kept for compatibility tests after the codec support was added to alloy-eips - /// AccessList type natively + /// `AccessList` type natively #[derive( - Clone, - Debug, - PartialEq, - Eq, - Hash, - Default, - RlpDecodableWrapper, - RlpEncodableWrapper, - Serialize, - Deserialize, - Compact, + Clone, Debug, PartialEq, Eq, Default, RlpDecodableWrapper, RlpEncodableWrapper, Compact, )] #[cfg_attr(any(test, feature = "arbitrary"), derive(arbitrary::Arbitrary))] #[add_arbitrary_tests(compact, rlp)] @@ -36,22 +25,9 @@ mod tests { } // This - #[derive( - Clone, - Debug, - PartialEq, - Eq, - Hash, - Default, - RlpDecodable, - RlpEncodable, - Serialize, - Deserialize, - Compact, - )] + #[derive(Clone, Debug, PartialEq, Eq, Default, RlpDecodable, RlpEncodable, Compact)] #[cfg_attr(any(test, feature = "arbitrary"), derive(arbitrary::Arbitrary))] #[add_arbitrary_tests(compact, rlp)] - #[serde(rename_all = "camelCase")] struct RethAccessListItem { /// Account address that would be loaded at the start of execution address: Address, diff --git a/crates/primitives-traits/src/transaction/error.rs b/crates/primitives-traits/src/transaction/error.rs index d155656c0e6..b87405e4abd 100644 --- a/crates/primitives-traits/src/transaction/error.rs +++ b/crates/primitives-traits/src/transaction/error.rs @@ -61,6 +61,9 @@ pub enum InvalidTransactionError { /// Thrown if the sender of a transaction is a contract. #[error("transaction signer has bytecode set")] SignerAccountHasBytecode, + /// Thrown post Osaka if gas limit is too high. + #[error("gas limit too high")] + GasLimitTooHigh, } /// Represents error variants that can happen when trying to convert a transaction to pooled diff --git a/crates/primitives-traits/src/transaction/mod.rs b/crates/primitives-traits/src/transaction/mod.rs index 43fe7899d99..5620d4916bd 100644 --- a/crates/primitives-traits/src/transaction/mod.rs +++ b/crates/primitives-traits/src/transaction/mod.rs @@ -1,4 +1,15 @@ //! Transaction abstraction +//! +//! This module provides traits for working with blockchain transactions: +//! - [`Transaction`] - Basic transaction interface +//! - [`signed::SignedTransaction`] - Transaction with signature and recovery methods +//! - [`FullTransaction`] - Transaction with database encoding support +//! +//! # Transaction Recovery +//! +//! Transaction senders are not stored directly but recovered from signatures. +//! Use `recover_signer` for post-EIP-2 transactions or `recover_signer_unchecked` +//! for historical transactions. pub mod execute; pub mod signature; @@ -7,7 +18,9 @@ pub mod signed; pub mod error; pub mod recover; -pub use alloy_consensus::transaction::{TransactionInfo, TransactionMeta}; +pub use alloy_consensus::transaction::{ + SignerRecoverable, TransactionInfo, TransactionMeta, TxHashRef, +}; use crate::{InMemorySize, MaybeCompact, MaybeSerde}; use core::{fmt, hash::Hash}; diff --git a/crates/primitives-traits/src/transaction/recover.rs b/crates/primitives-traits/src/transaction/recover.rs index 704f11f58c6..59e6e8a6943 100644 --- a/crates/primitives-traits/src/transaction/recover.rs +++ b/crates/primitives-traits/src/transaction/recover.rs @@ -15,11 +15,11 @@ mod rayon { /// Recovers a list of signers from a transaction list iterator. /// - /// Returns `None`, if some transaction's signature is invalid + /// Returns `Err(RecoveryError)`, if some transaction's signature is invalid pub fn recover_signers<'a, I, T>(txes: I) -> Result, RecoveryError> where T: SignedTransaction, - I: IntoParallelIterator + IntoIterator + Send, + I: IntoParallelIterator, { txes.into_par_iter().map(|tx| tx.recover_signer()).collect() } @@ -27,11 +27,11 @@ mod rayon { /// Recovers a list of signers from a transaction list iterator _without ensuring that the /// signature has a low `s` value_. /// - /// Returns `None`, if some transaction's signature is invalid. + /// Returns `Err(RecoveryError)`, if some transaction's signature is invalid. pub fn recover_signers_unchecked<'a, I, T>(txes: I) -> Result, RecoveryError> where T: SignedTransaction, - I: IntoParallelIterator + IntoIterator + Send, + I: IntoParallelIterator, { txes.into_par_iter().map(|tx| tx.recover_signer_unchecked()).collect() } diff --git a/crates/primitives-traits/src/transaction/signature.rs b/crates/primitives-traits/src/transaction/signature.rs index 2e994f1e5f4..481096b7936 100644 --- a/crates/primitives-traits/src/transaction/signature.rs +++ b/crates/primitives-traits/src/transaction/signature.rs @@ -6,7 +6,7 @@ pub use alloy_primitives::Signature; #[cfg(test)] mod tests { use crate::crypto::secp256k1::recover_signer; - use alloy_primitives::{address, Signature, B256, U256}; + use alloy_primitives::{address, b256, Signature, U256}; use std::str::FromStr; #[test] @@ -22,9 +22,7 @@ mod tests { .unwrap(), false, ); - let hash = - B256::from_str("daf5a779ae972f972197303d7b574746c7ef83eadac0f2791ad23db92e4c8e53") - .unwrap(); + let hash = b256!("0xdaf5a779ae972f972197303d7b574746c7ef83eadac0f2791ad23db92e4c8e53"); let signer = recover_signer(&signature, hash).unwrap(); let expected = address!("0x9d8a62f656a8d1615c1294fd71e9cfb3e4855a4f"); assert_eq!(expected, signer); diff --git a/crates/primitives-traits/src/transaction/signed.rs b/crates/primitives-traits/src/transaction/signed.rs index 79c297edc37..08a6758d8d4 100644 --- a/crates/primitives-traits/src/transaction/signed.rs +++ b/crates/primitives-traits/src/transaction/signed.rs @@ -1,16 +1,13 @@ //! API of a signed transaction. -use crate::{ - crypto::secp256k1::{recover_signer, recover_signer_unchecked}, - InMemorySize, MaybeCompact, MaybeSerde, MaybeSerdeBincodeCompat, -}; -use alloc::{fmt, vec::Vec}; +use crate::{InMemorySize, MaybeCompact, MaybeSerde, MaybeSerdeBincodeCompat}; +use alloc::fmt; use alloy_consensus::{ - transaction::{Recovered, RlpEcdsaEncodableTx}, + transaction::{Recovered, RlpEcdsaEncodableTx, SignerRecoverable, TxHashRef}, EthereumTxEnvelope, SignableTransaction, }; use alloy_eips::eip2718::{Decodable2718, Encodable2718}; -use alloy_primitives::{keccak256, Address, Signature, TxHash, B256}; +use alloy_primitives::{keccak256, Address, Signature, B256}; use alloy_rlp::{Decodable, Encodable}; use core::hash::Hash; @@ -21,6 +18,15 @@ pub trait FullSignedTx: SignedTransaction + MaybeCompact + MaybeSerdeBincodeComp impl FullSignedTx for T where T: SignedTransaction + MaybeCompact + MaybeSerdeBincodeCompat {} /// A signed transaction. +/// +/// # Recovery Methods +/// +/// This trait provides two types of recovery methods: +/// - Standard methods (e.g., `try_recover`) - enforce EIP-2 low-s signature requirement +/// - Unchecked methods (e.g., `try_recover_unchecked`) - skip EIP-2 validation for pre-EIP-2 +/// transactions +/// +/// Use unchecked methods only when dealing with historical pre-EIP-2 transactions. #[auto_impl::auto_impl(&, Arc)] pub trait SignedTransaction: Send @@ -38,10 +44,9 @@ pub trait SignedTransaction: + alloy_consensus::Transaction + MaybeSerde + InMemorySize + + SignerRecoverable + + TxHashRef { - /// Returns reference to transaction hash. - fn tx_hash(&self) -> &TxHash; - /// Returns whether this transaction type can be __broadcasted__ as full transaction over the /// network. /// @@ -52,17 +57,6 @@ pub trait SignedTransaction: !self.is_eip4844() } - /// Recover signer from signature and hash. - /// - /// Returns `RecoveryError` if the transaction's signature is invalid following [EIP-2](https://eips.ethereum.org/EIPS/eip-2), see also `reth_primitive_traits::crypto::secp256k1::recover_signer`. - /// - /// Note: - /// - /// This can fail for some early ethereum mainnet transactions pre EIP-2, use - /// [`Self::recover_signer_unchecked`] if you want to recover the signer without ensuring that - /// the signature has a low `s` value. - fn recover_signer(&self) -> Result; - /// Recover signer from signature and hash. /// /// Returns an error if the transaction's signature is invalid. @@ -70,15 +64,6 @@ pub trait SignedTransaction: self.recover_signer() } - /// Recover signer from signature and hash _without ensuring that the signature has a low `s` - /// value_. - /// - /// Returns `RecoveryError` if the transaction's signature is invalid, see also - /// `reth_primitive_traits::crypto::secp256k1::recover_signer_unchecked`. - fn recover_signer_unchecked(&self) -> Result { - self.recover_signer_unchecked_with_buf(&mut Vec::new()) - } - /// Recover signer from signature and hash _without ensuring that the signature has a low `s` /// value_. /// @@ -87,13 +72,6 @@ pub trait SignedTransaction: self.recover_signer_unchecked() } - /// Same as [`Self::recover_signer_unchecked`] but receives a buffer to operate on. This is used - /// during batch recovery to avoid allocating a new buffer for each transaction. - fn recover_signer_unchecked_with_buf( - &self, - buf: &mut Vec, - ) -> Result; - /// Calculate transaction hash, eip2728 transaction does not contain rlp header and start with /// tx type. fn recalculate_hash(&self) -> B256 { @@ -106,10 +84,16 @@ pub trait SignedTransaction: self.recover_signer().map(|signer| Recovered::new_unchecked(self.clone(), signer)) } + /// Tries to recover signer and return [`Recovered`] by cloning the type. + #[auto_impl(keep_default_for(&, Arc))] + fn try_clone_into_recovered_unchecked(&self) -> Result, RecoveryError> { + self.recover_signer_unchecked().map(|signer| Recovered::new_unchecked(self.clone(), signer)) + } + /// Tries to recover signer and return [`Recovered`]. /// /// Returns `Err(Self)` if the transaction's signature is invalid, see also - /// [`SignedTransaction::recover_signer`]. + /// [`SignerRecoverable::recover_signer`]. #[auto_impl(keep_default_for(&, Arc))] fn try_into_recovered(self) -> Result, Self> { match self.recover_signer() { @@ -122,6 +106,7 @@ pub trait SignedTransaction: /// ensuring that the signature has a low `s` value_ (EIP-2). /// /// Returns `RecoveryError` if the transaction's signature is invalid. + #[deprecated(note = "Use try_into_recovered_unchecked instead")] #[auto_impl(keep_default_for(&, Arc))] fn into_recovered_unchecked(self) -> Result, RecoveryError> { self.recover_signer_unchecked().map(|signer| Recovered::new_unchecked(self, signer)) @@ -134,6 +119,14 @@ pub trait SignedTransaction: fn with_signer(self, signer: Address) -> Recovered { Recovered::new_unchecked(self, signer) } + + /// Returns the [`Recovered`] transaction with the given signer, using a reference to self. + /// + /// Note: assumes the given signer is the signer of this transaction. + #[auto_impl(keep_default_for(&, Arc))] + fn with_signer_ref(&self, signer: Address) -> Recovered<&Self> { + Recovered::new_unchecked(self, signer) + } } impl SignedTransaction for EthereumTxEnvelope @@ -141,35 +134,6 @@ where T: RlpEcdsaEncodableTx + SignableTransaction + Unpin, Self: Clone + PartialEq + Eq + Decodable + Decodable2718 + MaybeSerde + InMemorySize, { - fn tx_hash(&self) -> &TxHash { - match self { - Self::Legacy(tx) => tx.hash(), - Self::Eip2930(tx) => tx.hash(), - Self::Eip1559(tx) => tx.hash(), - Self::Eip7702(tx) => tx.hash(), - Self::Eip4844(tx) => tx.hash(), - } - } - - fn recover_signer(&self) -> Result { - let signature_hash = self.signature_hash(); - recover_signer(self.signature(), signature_hash) - } - - fn recover_signer_unchecked_with_buf( - &self, - buf: &mut Vec, - ) -> Result { - match self { - Self::Legacy(tx) => tx.tx().encode_for_signing(buf), - Self::Eip2930(tx) => tx.tx().encode_for_signing(buf), - Self::Eip1559(tx) => tx.tx().encode_for_signing(buf), - Self::Eip7702(tx) => tx.tx().encode_for_signing(buf), - Self::Eip4844(tx) => tx.tx().encode_for_signing(buf), - } - let signature_hash = keccak256(buf); - recover_signer_unchecked(self.signature(), signature_hash) - } } #[cfg(feature = "op")] @@ -177,106 +141,7 @@ mod op { use super::*; use op_alloy_consensus::{OpPooledTransaction, OpTxEnvelope}; - impl SignedTransaction for OpPooledTransaction { - fn tx_hash(&self) -> &TxHash { - match self { - Self::Legacy(tx) => tx.hash(), - Self::Eip2930(tx) => tx.hash(), - Self::Eip1559(tx) => tx.hash(), - Self::Eip7702(tx) => tx.hash(), - } - } - - fn recover_signer(&self) -> Result { - recover_signer(self.signature(), self.signature_hash()) - } - - fn recover_signer_unchecked_with_buf( - &self, - buf: &mut Vec, - ) -> Result { - match self { - Self::Legacy(tx) => tx.tx().encode_for_signing(buf), - Self::Eip2930(tx) => tx.tx().encode_for_signing(buf), - Self::Eip1559(tx) => tx.tx().encode_for_signing(buf), - Self::Eip7702(tx) => tx.tx().encode_for_signing(buf), - } - let signature_hash = keccak256(buf); - recover_signer_unchecked(self.signature(), signature_hash) - } - } - - impl SignedTransaction for OpTxEnvelope { - fn tx_hash(&self) -> &TxHash { - match self { - Self::Legacy(tx) => tx.hash(), - Self::Eip2930(tx) => tx.hash(), - Self::Eip1559(tx) => tx.hash(), - Self::Eip7702(tx) => tx.hash(), - Self::Deposit(tx) => tx.hash_ref(), - } - } + impl SignedTransaction for OpPooledTransaction {} - fn recover_signer(&self) -> Result { - let signature_hash = match self { - Self::Legacy(tx) => tx.signature_hash(), - Self::Eip2930(tx) => tx.signature_hash(), - Self::Eip1559(tx) => tx.signature_hash(), - Self::Eip7702(tx) => tx.signature_hash(), - // Optimism's Deposit transaction does not have a signature. Directly return the - // `from` address. - Self::Deposit(tx) => return Ok(tx.from), - }; - let signature = match self { - Self::Legacy(tx) => tx.signature(), - Self::Eip2930(tx) => tx.signature(), - Self::Eip1559(tx) => tx.signature(), - Self::Eip7702(tx) => tx.signature(), - Self::Deposit(_) => unreachable!("Deposit transactions should not be handled here"), - }; - recover_signer(signature, signature_hash) - } - - fn recover_signer_unchecked(&self) -> Result { - let signature_hash = match self { - Self::Legacy(tx) => tx.signature_hash(), - Self::Eip2930(tx) => tx.signature_hash(), - Self::Eip1559(tx) => tx.signature_hash(), - Self::Eip7702(tx) => tx.signature_hash(), - // Optimism's Deposit transaction does not have a signature. Directly return the - // `from` address. - Self::Deposit(tx) => return Ok(tx.from), - }; - let signature = match self { - Self::Legacy(tx) => tx.signature(), - Self::Eip2930(tx) => tx.signature(), - Self::Eip1559(tx) => tx.signature(), - Self::Eip7702(tx) => tx.signature(), - Self::Deposit(_) => unreachable!("Deposit transactions should not be handled here"), - }; - recover_signer_unchecked(signature, signature_hash) - } - - fn recover_signer_unchecked_with_buf( - &self, - buf: &mut Vec, - ) -> Result { - match self { - Self::Deposit(tx) => return Ok(tx.from), - Self::Legacy(tx) => tx.tx().encode_for_signing(buf), - Self::Eip2930(tx) => tx.tx().encode_for_signing(buf), - Self::Eip1559(tx) => tx.tx().encode_for_signing(buf), - Self::Eip7702(tx) => tx.tx().encode_for_signing(buf), - } - let signature_hash = keccak256(buf); - let signature = match self { - Self::Legacy(tx) => tx.signature(), - Self::Eip2930(tx) => tx.signature(), - Self::Eip1559(tx) => tx.signature(), - Self::Eip7702(tx) => tx.signature(), - Self::Deposit(_) => unreachable!("Deposit transactions should not be handled here"), - }; - recover_signer_unchecked(signature, signature_hash) - } - } + impl SignedTransaction for OpTxEnvelope {} } diff --git a/crates/primitives/Cargo.toml b/crates/primitives/Cargo.toml index 4f1aa64b30c..1717cc6ec3f 100644 --- a/crates/primitives/Cargo.toml +++ b/crates/primitives/Cargo.toml @@ -14,7 +14,7 @@ workspace = true [dependencies] # reth reth-ethereum-primitives = { workspace = true, features = ["serde"] } -reth-primitives-traits = { workspace = true, features = ["serde"] } +reth-primitives-traits.workspace = true reth-ethereum-forks.workspace = true reth-static-file-types.workspace = true @@ -27,9 +27,6 @@ c-kzg = { workspace = true, features = ["serde"], optional = true } # misc once_cell.workspace = true -# arbitrary utils -arbitrary = { workspace = true, features = ["derive"], optional = true } - [dev-dependencies] # eth reth-primitives-traits = { workspace = true, features = ["arbitrary", "test-utils"] } @@ -70,7 +67,6 @@ asm-keccak = [ "alloy-primitives/asm-keccak", ] arbitrary = [ - "dep:arbitrary", "alloy-eips/arbitrary", "reth-codec", "reth-ethereum-forks/arbitrary", @@ -101,6 +97,7 @@ serde-bincode-compat = [ "alloy-consensus/serde-bincode-compat", "reth-primitives-traits/serde-bincode-compat", "reth-ethereum-primitives/serde-bincode-compat", + "alloy-genesis/serde-bincode-compat", ] [[bench]] diff --git a/crates/primitives/benches/recover_ecdsa_crit.rs b/crates/primitives/benches/recover_ecdsa_crit.rs index da6cd8d00f0..7521b0fa3bb 100644 --- a/crates/primitives/benches/recover_ecdsa_crit.rs +++ b/crates/primitives/benches/recover_ecdsa_crit.rs @@ -1,9 +1,9 @@ #![allow(missing_docs)] +use alloy_consensus::transaction::SignerRecoverable; use alloy_primitives::hex_literal::hex; use alloy_rlp::Decodable; use criterion::{criterion_group, criterion_main, Criterion}; use reth_ethereum_primitives::TransactionSigned; -use reth_primitives_traits::SignedTransaction; /// Benchmarks the recovery of the public key from the ECDSA message using criterion. pub fn criterion_benchmark(c: &mut Criterion) { @@ -12,7 +12,7 @@ pub fn criterion_benchmark(c: &mut Criterion) { let raw =hex!("f88b8212b085028fa6ae00830f424094aad593da0c8116ef7d2d594dd6a63241bccfc26c80a48318b64b000000000000000000000000641c5d790f862a58ec7abcfd644c0442e9c201b32aa0a6ef9e170bca5ffb7ac05433b13b7043de667fbb0b4a5e45d3b54fb2d6efcc63a0037ec2c05c3d60c5f5f78244ce0a3859e3a18a36c61efb061b383507d3ce19d2"); let mut pointer = raw.as_ref(); let tx = TransactionSigned::decode(&mut pointer).unwrap(); - SignedTransaction::recover_signer(&tx).unwrap(); + SignerRecoverable::recover_signer(&tx).unwrap(); } ) }); diff --git a/crates/primitives/src/lib.rs b/crates/primitives/src/lib.rs index 2aa550807d7..9d4d6da235d 100644 --- a/crates/primitives/src/lib.rs +++ b/crates/primitives/src/lib.rs @@ -15,7 +15,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(feature = "std"), no_std)] mod block; @@ -31,8 +31,8 @@ pub use block::{BlockWithSenders, SealedBlockFor, SealedBlockWithSenders}; pub use receipt::{gas_spent_by_transactions, Receipt}; pub use reth_primitives_traits::{ logs_bloom, Account, BlockTy, BodyTy, Bytecode, GotExpected, GotExpectedBoxed, Header, - HeaderError, HeaderTy, Log, LogData, NodePrimitives, ReceiptTy, RecoveredBlock, SealedHeader, - StorageEntry, TxTy, + HeaderTy, Log, LogData, NodePrimitives, ReceiptTy, RecoveredBlock, SealedHeader, StorageEntry, + TxTy, }; pub use static_file::StaticFileSegment; @@ -55,9 +55,6 @@ pub use transaction::{PooledTransactionsElementEcRecovered, TransactionSignedEcR // Re-exports pub use reth_ethereum_forks::*; -#[cfg(any(test, feature = "arbitrary"))] -pub use arbitrary; - #[cfg(feature = "c-kzg")] pub use c_kzg as kzg; diff --git a/crates/prune/db/Cargo.toml b/crates/prune/db/Cargo.toml new file mode 100644 index 00000000000..269a87bf7b6 --- /dev/null +++ b/crates/prune/db/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "reth-prune-db" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true +exclude.workspace = true +description = "Database integration with prune implementation" + +[dependencies] + +[lints] +workspace = true diff --git a/crates/prune/db/src/lib.rs b/crates/prune/db/src/lib.rs new file mode 100644 index 00000000000..ef777085e54 --- /dev/null +++ b/crates/prune/db/src/lib.rs @@ -0,0 +1 @@ +//! An integration of `reth-prune` with `reth-db`. diff --git a/crates/prune/prune/Cargo.toml b/crates/prune/prune/Cargo.toml index 094570b873f..a2d82c26923 100644 --- a/crates/prune/prune/Cargo.toml +++ b/crates/prune/prune/Cargo.toml @@ -13,7 +13,6 @@ workspace = true [dependencies] # reth -reth-chainspec.workspace = true reth-exex-types.workspace = true reth-db-api.workspace = true reth-errors.workspace = true diff --git a/crates/prune/prune/src/builder.rs b/crates/prune/prune/src/builder.rs index f5bb95df3f5..78283710e15 100644 --- a/crates/prune/prune/src/builder.rs +++ b/crates/prune/prune/src/builder.rs @@ -1,13 +1,13 @@ use crate::{segments::SegmentSet, Pruner}; use alloy_eips::eip2718::Encodable2718; -use reth_chainspec::MAINNET_PRUNE_DELETE_LIMIT; use reth_config::PruneConfig; use reth_db_api::{table::Value, transaction::DbTxMut}; use reth_exex_types::FinishedExExHeight; use reth_primitives_traits::NodePrimitives; use reth_provider::{ - providers::StaticFileProvider, BlockReader, DBProvider, DatabaseProviderFactory, - NodePrimitivesProvider, PruneCheckpointWriter, StaticFileProviderFactory, + providers::StaticFileProvider, BlockReader, ChainStateBlockReader, DBProvider, + DatabaseProviderFactory, NodePrimitivesProvider, PruneCheckpointReader, PruneCheckpointWriter, + StaticFileProviderFactory, }; use reth_prune_types::PruneModes; use std::time::Duration; @@ -29,9 +29,6 @@ pub struct PrunerBuilder { } impl PrunerBuilder { - /// Default timeout for a prune run. - pub const DEFAULT_TIMEOUT: Duration = Duration::from_millis(100); - /// Creates a new [`PrunerBuilder`] from the given [`PruneConfig`]. pub fn new(pruner_config: PruneConfig) -> Self { Self::default() @@ -80,9 +77,11 @@ impl PrunerBuilder { where PF: DatabaseProviderFactory< ProviderRW: PruneCheckpointWriter + + PruneCheckpointReader + BlockReader + + ChainStateBlockReader + StaticFileProviderFactory< - Primitives: NodePrimitives, + Primitives: NodePrimitives, >, > + StaticFileProviderFactory< Primitives = ::Primitives, @@ -107,10 +106,13 @@ impl PrunerBuilder { static_file_provider: StaticFileProvider, ) -> Pruner where - Provider: StaticFileProviderFactory> - + DBProvider + Provider: StaticFileProviderFactory< + Primitives: NodePrimitives, + > + DBProvider + BlockReader - + PruneCheckpointWriter, + + ChainStateBlockReader + + PruneCheckpointWriter + + PruneCheckpointReader, { let segments = SegmentSet::::from_components(static_file_provider, self.segments); @@ -128,8 +130,8 @@ impl Default for PrunerBuilder { fn default() -> Self { Self { block_interval: 5, - segments: PruneModes::none(), - delete_limit: MAINNET_PRUNE_DELETE_LIMIT, + segments: PruneModes::default(), + delete_limit: usize::MAX, timeout: None, finished_exex_height: watch::channel(FinishedExExHeight::NoExExs).1, } diff --git a/crates/prune/prune/src/lib.rs b/crates/prune/prune/src/lib.rs index ef3ee0de2db..4da07d495e7 100644 --- a/crates/prune/prune/src/lib.rs +++ b/crates/prune/prune/src/lib.rs @@ -7,7 +7,7 @@ )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] #![allow(missing_docs)] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] mod builder; mod db_ext; diff --git a/crates/prune/prune/src/pruner.rs b/crates/prune/prune/src/pruner.rs index 60fa7a8b5c9..4ef060774b9 100644 --- a/crates/prune/prune/src/pruner.rs +++ b/crates/prune/prune/src/pruner.rs @@ -39,7 +39,7 @@ pub struct Pruner { previous_tip_block_number: Option, /// Maximum total entries to prune (delete from database) per run. delete_limit: usize, - /// Maximum time for a one pruner run. + /// Maximum time for one pruner run. timeout: Option, /// The finished height of all `ExEx`'s. finished_exex_height: watch::Receiver, diff --git a/crates/prune/prune/src/segments/mod.rs b/crates/prune/prune/src/segments/mod.rs index c34e3a322aa..f917c78ea94 100644 --- a/crates/prune/prune/src/segments/mod.rs +++ b/crates/prune/prune/src/segments/mod.rs @@ -1,6 +1,5 @@ mod receipts; mod set; -mod static_file; mod user; use crate::{PruneLimiter, PrunerError}; @@ -8,15 +7,11 @@ use alloy_primitives::{BlockNumber, TxNumber}; use reth_provider::{errors::provider::ProviderResult, BlockReader, PruneCheckpointWriter}; use reth_prune_types::{PruneCheckpoint, PruneMode, PrunePurpose, PruneSegment, SegmentOutput}; pub use set::SegmentSet; -pub use static_file::{ - Headers as StaticFileHeaders, Receipts as StaticFileReceipts, - Transactions as StaticFileTransactions, -}; use std::{fmt::Debug, ops::RangeInclusive}; use tracing::error; pub use user::{ - AccountHistory, Receipts as UserReceipts, ReceiptsByLogs, SenderRecovery, StorageHistory, - TransactionLookup, + AccountHistory, Bodies, MerkleChangeSets, Receipts as UserReceipts, ReceiptsByLogs, + SenderRecovery, StorageHistory, TransactionLookup, }; /// A segment represents a pruning of some portion of the data. @@ -149,6 +144,7 @@ mod tests { use reth_provider::{ providers::BlockchainProvider, test_utils::{create_test_provider_factory, MockEthProvider}, + BlockWriter, }; use reth_testing_utils::generators::{self, random_block_range, BlockRangeParams}; @@ -190,7 +186,7 @@ mod tests { let provider_rw = factory.provider_rw().expect("failed to get provider_rw"); for block in &blocks { provider_rw - .insert_historical_block( + .insert_block( block.clone().try_recover().expect("failed to seal block with senders"), ) .expect("failed to insert block"); @@ -228,7 +224,7 @@ mod tests { let provider_rw = factory.provider_rw().expect("failed to get provider_rw"); for block in &blocks { provider_rw - .insert_historical_block( + .insert_block( block.clone().try_recover().expect("failed to seal block with senders"), ) .expect("failed to insert block"); @@ -274,7 +270,7 @@ mod tests { let provider_rw = factory.provider_rw().expect("failed to get provider_rw"); for block in &blocks { provider_rw - .insert_historical_block( + .insert_block( block.clone().try_recover().expect("failed to seal block with senders"), ) .expect("failed to insert block"); @@ -310,7 +306,7 @@ mod tests { let provider_rw = factory.provider_rw().expect("failed to get provider_rw"); for block in &blocks { provider_rw - .insert_historical_block( + .insert_block( block.clone().try_recover().expect("failed to seal block with senders"), ) .expect("failed to insert block"); diff --git a/crates/prune/prune/src/segments/receipts.rs b/crates/prune/prune/src/segments/receipts.rs index 2c94b5e4a8a..68a12552013 100644 --- a/crates/prune/prune/src/segments/receipts.rs +++ b/crates/prune/prune/src/segments/receipts.rs @@ -1,9 +1,7 @@ -//! Common receipts pruning logic shared between user and static file pruning segments. +//! Common receipts pruning logic. //! //! - [`crate::segments::user::Receipts`] is responsible for pruning receipts according to the //! user-configured settings (for example, on a full node or with a custom prune config) -//! - [`crate::segments::static_file::Receipts`] is responsible for pruning receipts on an archive -//! node after static file producer has finished use crate::{db_ext::DbTxPruneExt, segments::PruneInput, PrunerError}; use reth_db_api::{table::Value, tables, transaction::DbTxMut}; @@ -89,7 +87,7 @@ mod tests { Itertools, }; use reth_db_api::tables; - use reth_provider::{DatabaseProviderFactory, PruneCheckpointReader}; + use reth_provider::{DBProvider, DatabaseProviderFactory, PruneCheckpointReader}; use reth_prune_types::{ PruneCheckpoint, PruneInterruptReason, PruneMode, PruneProgress, PruneSegment, }; @@ -115,8 +113,10 @@ mod tests { for block in &blocks { receipts.reserve_exact(block.transaction_count()); for transaction in &block.body().transactions { - receipts - .push((receipts.len() as u64, random_receipt(&mut rng, transaction, Some(0)))); + receipts.push(( + receipts.len() as u64, + random_receipt(&mut rng, transaction, Some(0), None), + )); } } let receipts_len = receipts.len(); diff --git a/crates/prune/prune/src/segments/set.rs b/crates/prune/prune/src/segments/set.rs index c99defe0841..7ae9e044e20 100644 --- a/crates/prune/prune/src/segments/set.rs +++ b/crates/prune/prune/src/segments/set.rs @@ -1,18 +1,16 @@ use crate::segments::{ - AccountHistory, ReceiptsByLogs, Segment, SenderRecovery, StorageHistory, TransactionLookup, - UserReceipts, + user::ReceiptsByLogs, AccountHistory, Bodies, MerkleChangeSets, Segment, SenderRecovery, + StorageHistory, TransactionLookup, UserReceipts, }; use alloy_eips::eip2718::Encodable2718; use reth_db_api::{table::Value, transaction::DbTxMut}; use reth_primitives_traits::NodePrimitives; use reth_provider::{ - providers::StaticFileProvider, BlockReader, DBProvider, PruneCheckpointWriter, - StaticFileProviderFactory, + providers::StaticFileProvider, BlockReader, ChainStateBlockReader, DBProvider, + PruneCheckpointReader, PruneCheckpointWriter, StaticFileProviderFactory, }; use reth_prune_types::PruneModes; -use super::{StaticFileHeaders, StaticFileReceipts, StaticFileTransactions}; - /// Collection of [`Segment`]. Thread-safe, allocated on the heap. #[derive(Debug)] pub struct SegmentSet { @@ -47,15 +45,18 @@ impl SegmentSet { impl SegmentSet where - Provider: StaticFileProviderFactory> - + DBProvider + Provider: StaticFileProviderFactory< + Primitives: NodePrimitives, + > + DBProvider + PruneCheckpointWriter - + BlockReader, + + PruneCheckpointReader + + BlockReader + + ChainStateBlockReader, { /// Creates a [`SegmentSet`] from an existing components, such as [`StaticFileProvider`] and /// [`PruneModes`]. pub fn from_components( - static_file_provider: StaticFileProvider, + _static_file_provider: StaticFileProvider, prune_modes: PruneModes, ) -> Self { let PruneModes { @@ -64,16 +65,16 @@ where receipts, account_history, storage_history, + bodies_history, + merkle_changesets, receipts_log_filter, } = prune_modes; Self::default() - // Static file headers - .segment(StaticFileHeaders::new(static_file_provider.clone())) - // Static file transactions - .segment(StaticFileTransactions::new(static_file_provider.clone())) - // Static file receipts - .segment(StaticFileReceipts::new(static_file_provider)) + // Bodies - run first since file deletion is fast + .segment_opt(bodies_history.map(Bodies::new)) + // Merkle changesets + .segment(MerkleChangeSets::new(merkle_changesets)) // Account history .segment_opt(account_history.map(AccountHistory::new)) // Storage history diff --git a/crates/prune/prune/src/segments/static_file/headers.rs b/crates/prune/prune/src/segments/static_file/headers.rs deleted file mode 100644 index be4e50fe48b..00000000000 --- a/crates/prune/prune/src/segments/static_file/headers.rs +++ /dev/null @@ -1,339 +0,0 @@ -use crate::{ - db_ext::DbTxPruneExt, - segments::{PruneInput, Segment}, - PruneLimiter, PrunerError, -}; -use alloy_primitives::BlockNumber; -use itertools::Itertools; -use reth_db_api::{ - cursor::{DbCursorRO, RangeWalker}, - tables, - transaction::DbTxMut, -}; -use reth_provider::{providers::StaticFileProvider, DBProvider, StaticFileProviderFactory}; -use reth_prune_types::{ - PruneMode, PrunePurpose, PruneSegment, SegmentOutput, SegmentOutputCheckpoint, -}; -use reth_static_file_types::StaticFileSegment; -use std::num::NonZeroUsize; -use tracing::trace; - -/// Number of header tables to prune in one step -const HEADER_TABLES_TO_PRUNE: usize = 3; - -#[derive(Debug)] -pub struct Headers { - static_file_provider: StaticFileProvider, -} - -impl Headers { - pub const fn new(static_file_provider: StaticFileProvider) -> Self { - Self { static_file_provider } - } -} - -impl> Segment - for Headers -{ - fn segment(&self) -> PruneSegment { - PruneSegment::Headers - } - - fn mode(&self) -> Option { - self.static_file_provider - .get_highest_static_file_block(StaticFileSegment::Headers) - .map(PruneMode::before_inclusive) - } - - fn purpose(&self) -> PrunePurpose { - PrunePurpose::StaticFile - } - - fn prune(&self, provider: &Provider, input: PruneInput) -> Result { - let (block_range_start, block_range_end) = match input.get_next_block_range() { - Some(range) => (*range.start(), *range.end()), - None => { - trace!(target: "pruner", "No headers to prune"); - return Ok(SegmentOutput::done()) - } - }; - - let last_pruned_block = - if block_range_start == 0 { None } else { Some(block_range_start - 1) }; - - let range = last_pruned_block.map_or(0, |block| block + 1)..=block_range_end; - - let mut headers_cursor = provider.tx_ref().cursor_write::()?; - let mut header_tds_cursor = - provider.tx_ref().cursor_write::()?; - let mut canonical_headers_cursor = - provider.tx_ref().cursor_write::()?; - - let mut limiter = input.limiter.floor_deleted_entries_limit_to_multiple_of( - NonZeroUsize::new(HEADER_TABLES_TO_PRUNE).unwrap(), - ); - - let tables_iter = HeaderTablesIter::new( - provider, - &mut limiter, - headers_cursor.walk_range(range.clone())?, - header_tds_cursor.walk_range(range.clone())?, - canonical_headers_cursor.walk_range(range)?, - ); - - let mut last_pruned_block: Option = None; - let mut pruned = 0; - for res in tables_iter { - let HeaderTablesIterItem { pruned_block, entries_pruned } = res?; - last_pruned_block = Some(pruned_block); - pruned += entries_pruned; - } - - let done = last_pruned_block == Some(block_range_end); - let progress = limiter.progress(done); - - Ok(SegmentOutput { - progress, - pruned, - checkpoint: Some(SegmentOutputCheckpoint { - block_number: last_pruned_block, - tx_number: None, - }), - }) - } -} -type Walker<'a, Provider, T> = - RangeWalker<'a, T, <::Tx as DbTxMut>::CursorMut>; - -#[allow(missing_debug_implementations)] -struct HeaderTablesIter<'a, Provider> -where - Provider: DBProvider, -{ - provider: &'a Provider, - limiter: &'a mut PruneLimiter, - headers_walker: Walker<'a, Provider, tables::Headers>, - header_tds_walker: Walker<'a, Provider, tables::HeaderTerminalDifficulties>, - canonical_headers_walker: Walker<'a, Provider, tables::CanonicalHeaders>, -} - -struct HeaderTablesIterItem { - pruned_block: BlockNumber, - entries_pruned: usize, -} - -impl<'a, Provider> HeaderTablesIter<'a, Provider> -where - Provider: DBProvider, -{ - const fn new( - provider: &'a Provider, - limiter: &'a mut PruneLimiter, - headers_walker: Walker<'a, Provider, tables::Headers>, - header_tds_walker: Walker<'a, Provider, tables::HeaderTerminalDifficulties>, - canonical_headers_walker: Walker<'a, Provider, tables::CanonicalHeaders>, - ) -> Self { - Self { provider, limiter, headers_walker, header_tds_walker, canonical_headers_walker } - } -} - -impl Iterator for HeaderTablesIter<'_, Provider> -where - Provider: DBProvider, -{ - type Item = Result; - fn next(&mut self) -> Option { - if self.limiter.is_limit_reached() { - return None - } - - let mut pruned_block_headers = None; - let mut pruned_block_td = None; - let mut pruned_block_canonical = None; - - if let Err(err) = self.provider.tx_ref().prune_table_with_range_step( - &mut self.headers_walker, - self.limiter, - &mut |_| false, - &mut |row| pruned_block_headers = Some(row.0), - ) { - return Some(Err(err.into())) - } - - if let Err(err) = self.provider.tx_ref().prune_table_with_range_step( - &mut self.header_tds_walker, - self.limiter, - &mut |_| false, - &mut |row| pruned_block_td = Some(row.0), - ) { - return Some(Err(err.into())) - } - - if let Err(err) = self.provider.tx_ref().prune_table_with_range_step( - &mut self.canonical_headers_walker, - self.limiter, - &mut |_| false, - &mut |row| pruned_block_canonical = Some(row.0), - ) { - return Some(Err(err.into())) - } - - if ![pruned_block_headers, pruned_block_td, pruned_block_canonical].iter().all_equal() { - return Some(Err(PrunerError::InconsistentData( - "All headers-related tables should be pruned up to the same height", - ))) - } - - pruned_block_headers.map(move |block| { - Ok(HeaderTablesIterItem { pruned_block: block, entries_pruned: HEADER_TABLES_TO_PRUNE }) - }) - } -} - -#[cfg(test)] -mod tests { - use crate::segments::{ - static_file::headers::HEADER_TABLES_TO_PRUNE, PruneInput, PruneLimiter, Segment, - SegmentOutput, - }; - use alloy_primitives::{BlockNumber, B256, U256}; - use assert_matches::assert_matches; - use reth_db_api::{tables, transaction::DbTx}; - use reth_provider::{ - DatabaseProviderFactory, PruneCheckpointReader, PruneCheckpointWriter, - StaticFileProviderFactory, - }; - use reth_prune_types::{ - PruneCheckpoint, PruneInterruptReason, PruneMode, PruneProgress, PruneSegment, - SegmentOutputCheckpoint, - }; - use reth_stages::test_utils::TestStageDB; - use reth_testing_utils::{generators, generators::random_header_range}; - use tracing::trace; - - #[test] - fn prune() { - reth_tracing::init_test_tracing(); - - let db = TestStageDB::default(); - let mut rng = generators::rng(); - - let headers = random_header_range(&mut rng, 0..100, B256::ZERO); - let tx = db.factory.provider_rw().unwrap().into_tx(); - for header in &headers { - TestStageDB::insert_header(None, &tx, header, U256::ZERO).unwrap(); - } - tx.commit().unwrap(); - - assert_eq!(db.table::().unwrap().len(), headers.len()); - assert_eq!(db.table::().unwrap().len(), headers.len()); - assert_eq!(db.table::().unwrap().len(), headers.len()); - - let test_prune = |to_block: BlockNumber, expected_result: (PruneProgress, usize)| { - let segment = super::Headers::new(db.factory.static_file_provider()); - let prune_mode = PruneMode::Before(to_block); - let mut limiter = PruneLimiter::default().set_deleted_entries_limit(10); - let input = PruneInput { - previous_checkpoint: db - .factory - .provider() - .unwrap() - .get_prune_checkpoint(PruneSegment::Headers) - .unwrap(), - to_block, - limiter: limiter.clone(), - }; - - let next_block_number_to_prune = db - .factory - .provider() - .unwrap() - .get_prune_checkpoint(PruneSegment::Headers) - .unwrap() - .and_then(|checkpoint| checkpoint.block_number) - .map(|block_number| block_number + 1) - .unwrap_or_default(); - - let provider = db.factory.database_provider_rw().unwrap(); - let result = segment.prune(&provider, input.clone()).unwrap(); - limiter.increment_deleted_entries_count_by(result.pruned); - trace!(target: "pruner::test", - expected_prune_progress=?expected_result.0, - expected_pruned=?expected_result.1, - result=?result, - "SegmentOutput" - ); - - assert_matches!( - result, - SegmentOutput {progress, pruned, checkpoint: Some(_)} - if (progress, pruned) == expected_result - ); - provider - .save_prune_checkpoint( - PruneSegment::Headers, - result.checkpoint.unwrap().as_prune_checkpoint(prune_mode), - ) - .unwrap(); - provider.commit().expect("commit"); - - let last_pruned_block_number = to_block.min( - next_block_number_to_prune + - (input.limiter.deleted_entries_limit().unwrap() / HEADER_TABLES_TO_PRUNE - 1) - as u64, - ); - - assert_eq!( - db.table::().unwrap().len(), - headers.len() - (last_pruned_block_number + 1) as usize - ); - assert_eq!( - db.table::().unwrap().len(), - headers.len() - (last_pruned_block_number + 1) as usize - ); - assert_eq!( - db.table::().unwrap().len(), - headers.len() - (last_pruned_block_number + 1) as usize - ); - assert_eq!( - db.factory.provider().unwrap().get_prune_checkpoint(PruneSegment::Headers).unwrap(), - Some(PruneCheckpoint { - block_number: Some(last_pruned_block_number), - tx_number: None, - prune_mode - }) - ); - }; - - test_prune( - 3, - (PruneProgress::HasMoreData(PruneInterruptReason::DeletedEntriesLimitReached), 9), - ); - test_prune(3, (PruneProgress::Finished, 3)); - } - - #[test] - fn prune_cannot_be_done() { - let db = TestStageDB::default(); - - let limiter = PruneLimiter::default().set_deleted_entries_limit(0); - - let input = PruneInput { - previous_checkpoint: None, - to_block: 1, - // Less than total number of tables for `Headers` segment - limiter, - }; - - let provider = db.factory.database_provider_rw().unwrap(); - let segment = super::Headers::new(db.factory.static_file_provider()); - let result = segment.prune(&provider, input).unwrap(); - assert_eq!( - result, - SegmentOutput::not_done( - PruneInterruptReason::DeletedEntriesLimitReached, - Some(SegmentOutputCheckpoint::default()) - ) - ); - } -} diff --git a/crates/prune/prune/src/segments/static_file/mod.rs b/crates/prune/prune/src/segments/static_file/mod.rs deleted file mode 100644 index cb9dc79c6cd..00000000000 --- a/crates/prune/prune/src/segments/static_file/mod.rs +++ /dev/null @@ -1,7 +0,0 @@ -mod headers; -mod receipts; -mod transactions; - -pub use headers::Headers; -pub use receipts::Receipts; -pub use transactions::Transactions; diff --git a/crates/prune/prune/src/segments/static_file/receipts.rs b/crates/prune/prune/src/segments/static_file/receipts.rs deleted file mode 100644 index 6a84cce9c41..00000000000 --- a/crates/prune/prune/src/segments/static_file/receipts.rs +++ /dev/null @@ -1,58 +0,0 @@ -use crate::{ - segments::{PruneInput, Segment}, - PrunerError, -}; -use reth_db_api::{table::Value, transaction::DbTxMut}; -use reth_primitives_traits::NodePrimitives; -use reth_provider::{ - errors::provider::ProviderResult, providers::StaticFileProvider, BlockReader, DBProvider, - PruneCheckpointWriter, StaticFileProviderFactory, TransactionsProvider, -}; -use reth_prune_types::{PruneCheckpoint, PruneMode, PrunePurpose, PruneSegment, SegmentOutput}; -use reth_static_file_types::StaticFileSegment; - -#[derive(Debug)] -pub struct Receipts { - static_file_provider: StaticFileProvider, -} - -impl Receipts { - pub const fn new(static_file_provider: StaticFileProvider) -> Self { - Self { static_file_provider } - } -} - -impl Segment for Receipts -where - Provider: StaticFileProviderFactory> - + DBProvider - + PruneCheckpointWriter - + TransactionsProvider - + BlockReader, -{ - fn segment(&self) -> PruneSegment { - PruneSegment::Receipts - } - - fn mode(&self) -> Option { - self.static_file_provider - .get_highest_static_file_block(StaticFileSegment::Receipts) - .map(PruneMode::before_inclusive) - } - - fn purpose(&self) -> PrunePurpose { - PrunePurpose::StaticFile - } - - fn prune(&self, provider: &Provider, input: PruneInput) -> Result { - crate::segments::receipts::prune(provider, input) - } - - fn save_checkpoint( - &self, - provider: &Provider, - checkpoint: PruneCheckpoint, - ) -> ProviderResult<()> { - crate::segments::receipts::save_checkpoint(provider, checkpoint) - } -} diff --git a/crates/prune/prune/src/segments/static_file/transactions.rs b/crates/prune/prune/src/segments/static_file/transactions.rs deleted file mode 100644 index 7005ae15e7d..00000000000 --- a/crates/prune/prune/src/segments/static_file/transactions.rs +++ /dev/null @@ -1,224 +0,0 @@ -use crate::{ - db_ext::DbTxPruneExt, - segments::{PruneInput, Segment}, - PrunerError, -}; -use reth_db_api::{table::Value, tables, transaction::DbTxMut}; -use reth_primitives_traits::NodePrimitives; -use reth_provider::{ - providers::StaticFileProvider, BlockReader, DBProvider, StaticFileProviderFactory, - TransactionsProvider, -}; -use reth_prune_types::{ - PruneMode, PrunePurpose, PruneSegment, SegmentOutput, SegmentOutputCheckpoint, -}; -use reth_static_file_types::StaticFileSegment; -use tracing::trace; - -#[derive(Debug)] -pub struct Transactions { - static_file_provider: StaticFileProvider, -} - -impl Transactions { - pub const fn new(static_file_provider: StaticFileProvider) -> Self { - Self { static_file_provider } - } -} - -impl Segment for Transactions -where - Provider: DBProvider - + TransactionsProvider - + BlockReader - + StaticFileProviderFactory>, -{ - fn segment(&self) -> PruneSegment { - PruneSegment::Transactions - } - - fn mode(&self) -> Option { - self.static_file_provider - .get_highest_static_file_block(StaticFileSegment::Transactions) - .map(PruneMode::before_inclusive) - } - - fn purpose(&self) -> PrunePurpose { - PrunePurpose::StaticFile - } - - fn prune(&self, provider: &Provider, input: PruneInput) -> Result { - let tx_range = match input.get_next_tx_num_range(provider)? { - Some(range) => range, - None => { - trace!(target: "pruner", "No transactions to prune"); - return Ok(SegmentOutput::done()) - } - }; - - let mut limiter = input.limiter; - - let mut last_pruned_transaction = *tx_range.end(); - let (pruned, done) = provider.tx_ref().prune_table_with_range::::SignedTx, - >>( - tx_range, - &mut limiter, - |_| false, - |row| last_pruned_transaction = row.0, - )?; - trace!(target: "pruner", %pruned, %done, "Pruned transactions"); - - let last_pruned_block = provider - .transaction_block(last_pruned_transaction)? - .ok_or(PrunerError::InconsistentData("Block for transaction is not found"))? - // If there's more transactions to prune, set the checkpoint block number to previous, - // so we could finish pruning its transactions on the next run. - .checked_sub(if done { 0 } else { 1 }); - - let progress = limiter.progress(done); - - Ok(SegmentOutput { - progress, - pruned, - checkpoint: Some(SegmentOutputCheckpoint { - block_number: last_pruned_block, - tx_number: Some(last_pruned_transaction), - }), - }) - } -} - -#[cfg(test)] -mod tests { - use crate::segments::{PruneInput, PruneLimiter, Segment}; - use alloy_primitives::{BlockNumber, TxNumber, B256}; - use assert_matches::assert_matches; - use itertools::{ - FoldWhile::{Continue, Done}, - Itertools, - }; - use reth_db_api::tables; - use reth_provider::{ - DatabaseProviderFactory, PruneCheckpointReader, PruneCheckpointWriter, - StaticFileProviderFactory, - }; - use reth_prune_types::{ - PruneCheckpoint, PruneInterruptReason, PruneMode, PruneProgress, PruneSegment, - SegmentOutput, - }; - use reth_stages::test_utils::{StorageKind, TestStageDB}; - use reth_testing_utils::generators::{self, random_block_range, BlockRangeParams}; - use std::ops::Sub; - - #[test] - fn prune() { - let db = TestStageDB::default(); - let mut rng = generators::rng(); - - let blocks = random_block_range( - &mut rng, - 1..=100, - BlockRangeParams { parent: Some(B256::ZERO), tx_count: 2..3, ..Default::default() }, - ); - db.insert_blocks(blocks.iter(), StorageKind::Database(None)).expect("insert blocks"); - - let transactions = - blocks.iter().flat_map(|block| &block.body().transactions).collect::>(); - - assert_eq!(db.table::().unwrap().len(), transactions.len()); - - let test_prune = |to_block: BlockNumber, expected_result: (PruneProgress, usize)| { - let segment = super::Transactions::new(db.factory.static_file_provider()); - let prune_mode = PruneMode::Before(to_block); - let mut limiter = PruneLimiter::default().set_deleted_entries_limit(10); - let input = PruneInput { - previous_checkpoint: db - .factory - .provider() - .unwrap() - .get_prune_checkpoint(PruneSegment::Transactions) - .unwrap(), - to_block, - limiter: limiter.clone(), - }; - - let next_tx_number_to_prune = db - .factory - .provider() - .unwrap() - .get_prune_checkpoint(PruneSegment::Transactions) - .unwrap() - .and_then(|checkpoint| checkpoint.tx_number) - .map(|tx_number| tx_number + 1) - .unwrap_or_default(); - - let provider = db.factory.database_provider_rw().unwrap(); - let result = segment.prune(&provider, input.clone()).unwrap(); - limiter.increment_deleted_entries_count_by(result.pruned); - - assert_matches!( - result, - SegmentOutput {progress, pruned, checkpoint: Some(_)} - if (progress, pruned) == expected_result - ); - - provider - .save_prune_checkpoint( - PruneSegment::Transactions, - result.checkpoint.unwrap().as_prune_checkpoint(prune_mode), - ) - .unwrap(); - provider.commit().expect("commit"); - - let last_pruned_tx_number = blocks - .iter() - .take(to_block as usize) - .map(|block| block.transaction_count()) - .sum::() - .min( - next_tx_number_to_prune as usize + - input.limiter.deleted_entries_limit().unwrap(), - ) - .sub(1); - - let last_pruned_block_number = blocks - .iter() - .fold_while((0, 0), |(_, mut tx_count), block| { - tx_count += block.transaction_count(); - - if tx_count > last_pruned_tx_number { - Done((block.number, tx_count)) - } else { - Continue((block.number, tx_count)) - } - }) - .into_inner() - .0 - .checked_sub(if result.progress.is_finished() { 0 } else { 1 }); - - assert_eq!( - db.table::().unwrap().len(), - transactions.len() - (last_pruned_tx_number + 1) - ); - assert_eq!( - db.factory - .provider() - .unwrap() - .get_prune_checkpoint(PruneSegment::Transactions) - .unwrap(), - Some(PruneCheckpoint { - block_number: last_pruned_block_number, - tx_number: Some(last_pruned_tx_number as TxNumber), - prune_mode - }) - ); - }; - - test_prune( - 6, - (PruneProgress::HasMoreData(PruneInterruptReason::DeletedEntriesLimitReached), 10), - ); - test_prune(6, (PruneProgress::Finished, 2)); - } -} diff --git a/crates/prune/prune/src/segments/user/account_history.rs b/crates/prune/prune/src/segments/user/account_history.rs index 7780a9e07e6..317337f050e 100644 --- a/crates/prune/prune/src/segments/user/account_history.rs +++ b/crates/prune/prune/src/segments/user/account_history.rs @@ -45,7 +45,7 @@ where PrunePurpose::User } - #[instrument(level = "trace", target = "pruner", skip(self, provider), ret)] + #[instrument(target = "pruner", skip(self, provider), ret(level = "trace"))] fn prune(&self, provider: &Provider, input: PruneInput) -> Result { let range = match input.get_next_block_range() { Some(range) => range, @@ -133,7 +133,7 @@ mod tests { use alloy_primitives::{BlockNumber, B256}; use assert_matches::assert_matches; use reth_db_api::{tables, BlockNumberList}; - use reth_provider::{DatabaseProviderFactory, PruneCheckpointReader}; + use reth_provider::{DBProvider, DatabaseProviderFactory, PruneCheckpointReader}; use reth_prune_types::{ PruneCheckpoint, PruneInterruptReason, PruneMode, PruneProgress, PruneSegment, }; diff --git a/crates/prune/prune/src/segments/user/bodies.rs b/crates/prune/prune/src/segments/user/bodies.rs new file mode 100644 index 00000000000..0a6a432754b --- /dev/null +++ b/crates/prune/prune/src/segments/user/bodies.rs @@ -0,0 +1,327 @@ +use crate::{ + segments::{PruneInput, Segment}, + PrunerError, +}; +use reth_provider::{BlockReader, StaticFileProviderFactory}; +use reth_prune_types::{ + PruneMode, PruneProgress, PrunePurpose, PruneSegment, SegmentOutput, SegmentOutputCheckpoint, +}; +use reth_static_file_types::StaticFileSegment; + +/// Segment responsible for pruning transactions in static files. +/// +/// This segment is controlled by the `bodies_history` configuration. +#[derive(Debug)] +pub struct Bodies { + mode: PruneMode, +} + +impl Bodies { + /// Creates a new [`Bodies`] segment with the given prune mode. + pub const fn new(mode: PruneMode) -> Self { + Self { mode } + } +} + +impl Segment for Bodies +where + Provider: StaticFileProviderFactory + BlockReader, +{ + fn segment(&self) -> PruneSegment { + PruneSegment::Bodies + } + + fn mode(&self) -> Option { + Some(self.mode) + } + + fn purpose(&self) -> PrunePurpose { + PrunePurpose::User + } + + fn prune(&self, provider: &Provider, input: PruneInput) -> Result { + let deleted_headers = provider + .static_file_provider() + .delete_segment_below_block(StaticFileSegment::Transactions, input.to_block + 1)?; + + if deleted_headers.is_empty() { + return Ok(SegmentOutput::done()) + } + + let tx_ranges = deleted_headers.iter().filter_map(|header| header.tx_range()); + + let pruned = tx_ranges.clone().map(|range| range.len()).sum::() as usize; + + Ok(SegmentOutput { + progress: PruneProgress::Finished, + pruned, + checkpoint: Some(SegmentOutputCheckpoint { + block_number: Some(input.to_block), + tx_number: tx_ranges.map(|range| range.end()).max(), + }), + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::Pruner; + use alloy_primitives::BlockNumber; + use reth_exex_types::FinishedExExHeight; + use reth_provider::{ + test_utils::{create_test_provider_factory, MockNodeTypesWithDB}, + ProviderFactory, StaticFileWriter, + }; + use reth_prune_types::{PruneMode, PruneProgress, PruneSegment}; + use reth_static_file_types::{ + SegmentHeader, SegmentRangeInclusive, StaticFileSegment, DEFAULT_BLOCKS_PER_STATIC_FILE, + }; + + /// Creates empty static file jars at 500k block intervals up to the tip block. + /// + /// Each jar contains sequential transaction ranges for testing deletion logic. + fn setup_static_file_jars(provider: &P, tip_block: u64) { + let num_jars = (tip_block + 1) / DEFAULT_BLOCKS_PER_STATIC_FILE; + let txs_per_jar = 1000; + let static_file_provider = provider.static_file_provider(); + + let mut writer = + static_file_provider.latest_writer(StaticFileSegment::Transactions).unwrap(); + + for jar_idx in 0..num_jars { + let block_start = jar_idx * DEFAULT_BLOCKS_PER_STATIC_FILE; + let block_end = ((jar_idx + 1) * DEFAULT_BLOCKS_PER_STATIC_FILE - 1).min(tip_block); + + let tx_start = jar_idx * txs_per_jar; + let tx_end = tx_start + txs_per_jar - 1; + + *writer.user_header_mut() = SegmentHeader::new( + SegmentRangeInclusive::new(block_start, block_end), + Some(SegmentRangeInclusive::new(block_start, block_end)), + Some(SegmentRangeInclusive::new(tx_start, tx_end)), + StaticFileSegment::Transactions, + ); + + writer.inner().set_dirty(); + writer.commit().expect("commit empty jar"); + + if jar_idx < num_jars - 1 { + writer.increment_block(block_end + 1).expect("increment block"); + } + } + + static_file_provider.initialize_index().expect("initialize index"); + } + + struct PruneTestCase { + prune_mode: PruneMode, + expected_pruned: usize, + expected_lowest_block: Option, + } + + fn run_prune_test( + factory: &ProviderFactory, + finished_exex_height_rx: &tokio::sync::watch::Receiver, + test_case: PruneTestCase, + tip: BlockNumber, + ) { + let bodies = Bodies::new(test_case.prune_mode); + let segments: Vec>> = vec![Box::new(bodies)]; + + let mut pruner = Pruner::new_with_factory( + factory.clone(), + segments, + 5, + 10000, + None, + finished_exex_height_rx.clone(), + ); + + let result = pruner.run(tip).expect("pruner run"); + + assert_eq!(result.progress, PruneProgress::Finished); + assert_eq!(result.segments.len(), 1); + + let (segment, output) = &result.segments[0]; + assert_eq!(*segment, PruneSegment::Bodies); + assert_eq!(output.pruned, test_case.expected_pruned); + + let static_provider = factory.static_file_provider(); + assert_eq!( + static_provider.get_lowest_static_file_block(StaticFileSegment::Transactions), + test_case.expected_lowest_block + ); + assert_eq!( + static_provider.get_highest_static_file_block(StaticFileSegment::Transactions), + Some(tip) + ); + } + + #[test] + fn bodies_prune_through_pruner() { + let factory = create_test_provider_factory(); + let tip = 2_499_999; + setup_static_file_jars(&factory, tip); + + let (_, finished_exex_height_rx) = tokio::sync::watch::channel(FinishedExExHeight::NoExExs); + + let test_cases = vec![ + // Test 1: PruneMode::Before(750_000) → deletes jar 1 (0-499_999) + PruneTestCase { + prune_mode: PruneMode::Before(750_000), + expected_pruned: 1000, + expected_lowest_block: Some(999_999), + }, + // Test 2: PruneMode::Before(850_000) → no deletion (jar 2: 500_000-999_999 contains + // target) + PruneTestCase { + prune_mode: PruneMode::Before(850_000), + expected_pruned: 0, + expected_lowest_block: Some(999_999), + }, + // Test 3: PruneMode::Before(1_599_999) → deletes jar 2 (500_000-999_999) and jar 3 + // (1_000_000-1_499_999) + PruneTestCase { + prune_mode: PruneMode::Before(1_599_999), + expected_pruned: 2000, + expected_lowest_block: Some(1_999_999), + }, + // Test 4: PruneMode::Distance(500_000) with tip=2_499_999 → deletes jar 4 + // (1_500_000-1_999_999) + PruneTestCase { + prune_mode: PruneMode::Distance(500_000), + expected_pruned: 1000, + expected_lowest_block: Some(2_499_999), + }, + // Test 5: PruneMode::Before(2_300_000) → no deletion (jar 5: 2_000_000-2_499_999 + // contains target) + PruneTestCase { + prune_mode: PruneMode::Before(2_300_000), + expected_pruned: 0, + expected_lowest_block: Some(2_499_999), + }, + ]; + + for test_case in test_cases { + run_prune_test(&factory, &finished_exex_height_rx, test_case, tip); + } + } + + #[test] + fn min_block_updated_on_sync() { + // Regression test: update_index must update min_block to prevent stale values + // that can cause pruner to incorrectly delete static files when PruneMode::Before(0) is + // used. + + struct MinBlockTestCase { + // Block range + initial_range: Option, + updated_range: SegmentRangeInclusive, + // Min block + expected_before_update: Option, + expected_after_update: BlockNumber, + // Test delete_segment_below_block with this value + delete_below_block: BlockNumber, + // Expected number of deleted segments + expected_deleted: usize, + } + + let test_cases = vec![ + // Test 1: Empty initial state (None) -> syncs to block 100 + MinBlockTestCase { + initial_range: None, + updated_range: SegmentRangeInclusive::new(0, 100), + expected_before_update: None, + expected_after_update: 100, + delete_below_block: 1, + expected_deleted: 0, + }, + // Test 2: Genesis state [0..=0] -> syncs to block 100 (eg. op-reth node after op-reth + // init-state) + MinBlockTestCase { + initial_range: Some(SegmentRangeInclusive::new(0, 0)), + updated_range: SegmentRangeInclusive::new(0, 100), + expected_before_update: Some(0), + expected_after_update: 100, + delete_below_block: 1, + expected_deleted: 0, + }, + // Test 3: Existing state [0..=50] -> syncs to block 200 + MinBlockTestCase { + initial_range: Some(SegmentRangeInclusive::new(0, 50)), + updated_range: SegmentRangeInclusive::new(0, 200), + expected_before_update: Some(50), + expected_after_update: 200, + delete_below_block: 150, + expected_deleted: 0, + }, + ]; + + for ( + idx, + MinBlockTestCase { + initial_range, + updated_range, + expected_before_update, + expected_after_update, + delete_below_block, + expected_deleted, + }, + ) in test_cases.into_iter().enumerate() + { + let factory = create_test_provider_factory(); + let static_provider = factory.static_file_provider(); + + let mut writer = + static_provider.latest_writer(StaticFileSegment::Transactions).unwrap(); + + // Set up initial state if provided + if let Some(initial_range) = initial_range { + *writer.user_header_mut() = SegmentHeader::new( + initial_range, + Some(initial_range), + Some(initial_range), + StaticFileSegment::Transactions, + ); + writer.inner().set_dirty(); + writer.commit().unwrap(); + static_provider.initialize_index().unwrap(); + } + + // Verify initial state + assert_eq!( + static_provider.get_lowest_static_file_block(StaticFileSegment::Transactions), + expected_before_update, + "Test case {}: Initial min_block mismatch", + idx + ); + + // Update to new range + *writer.user_header_mut() = SegmentHeader::new( + updated_range, + Some(updated_range), + Some(updated_range), + StaticFileSegment::Transactions, + ); + writer.inner().set_dirty(); + writer.commit().unwrap(); // update_index is called inside + + // Verify min_block was updated (not stuck at stale value) + assert_eq!( + static_provider.get_lowest_static_file_block(StaticFileSegment::Transactions), + Some(expected_after_update), + "Test case {}: min_block should be updated to {} (not stuck at stale value)", + idx, + expected_after_update + ); + + // Verify delete_segment_below_block behaves correctly with updated min_block + let deleted = static_provider + .delete_segment_below_block(StaticFileSegment::Transactions, delete_below_block) + .unwrap(); + + assert_eq!(deleted.len(), expected_deleted); + } + } +} diff --git a/crates/prune/prune/src/segments/user/merkle_change_sets.rs b/crates/prune/prune/src/segments/user/merkle_change_sets.rs new file mode 100644 index 00000000000..89cc4567b7d --- /dev/null +++ b/crates/prune/prune/src/segments/user/merkle_change_sets.rs @@ -0,0 +1,116 @@ +use crate::{ + db_ext::DbTxPruneExt, + segments::{PruneInput, Segment}, + PrunerError, +}; +use alloy_primitives::B256; +use reth_db_api::{models::BlockNumberHashedAddress, table::Value, tables, transaction::DbTxMut}; +use reth_primitives_traits::NodePrimitives; +use reth_provider::{ + errors::provider::ProviderResult, BlockReader, ChainStateBlockReader, DBProvider, + NodePrimitivesProvider, PruneCheckpointWriter, TransactionsProvider, +}; +use reth_prune_types::{ + PruneCheckpoint, PruneMode, PrunePurpose, PruneSegment, SegmentOutput, SegmentOutputCheckpoint, +}; +use tracing::{instrument, trace}; + +#[derive(Debug)] +pub struct MerkleChangeSets { + mode: PruneMode, +} + +impl MerkleChangeSets { + pub const fn new(mode: PruneMode) -> Self { + Self { mode } + } +} + +impl Segment for MerkleChangeSets +where + Provider: DBProvider + + PruneCheckpointWriter + + TransactionsProvider + + BlockReader + + ChainStateBlockReader + + NodePrimitivesProvider>, +{ + fn segment(&self) -> PruneSegment { + PruneSegment::MerkleChangeSets + } + + fn mode(&self) -> Option { + Some(self.mode) + } + + fn purpose(&self) -> PrunePurpose { + PrunePurpose::User + } + + #[instrument(level = "trace", target = "pruner", skip(self, provider), ret)] + fn prune(&self, provider: &Provider, input: PruneInput) -> Result { + let Some(block_range) = input.get_next_block_range() else { + trace!(target: "pruner", "No change sets to prune"); + return Ok(SegmentOutput::done()) + }; + + let block_range_end = *block_range.end(); + let mut limiter = input.limiter; + + // Create range for StoragesTrieChangeSets which uses BlockNumberHashedAddress as key + let storage_range_start: BlockNumberHashedAddress = + (*block_range.start(), B256::ZERO).into(); + let storage_range_end: BlockNumberHashedAddress = + (*block_range.end() + 1, B256::ZERO).into(); + let storage_range = storage_range_start..storage_range_end; + + let mut last_storages_pruned_block = None; + let (storages_pruned, done) = + provider.tx_ref().prune_table_with_range::( + storage_range, + &mut limiter, + |_| false, + |(BlockNumberHashedAddress((block_number, _)), _)| { + last_storages_pruned_block = Some(block_number); + }, + )?; + + trace!(target: "pruner", %storages_pruned, %done, "Pruned storages change sets"); + + let mut last_accounts_pruned_block = block_range_end; + let last_storages_pruned_block = last_storages_pruned_block + // If there's more storage changesets to prune, set the checkpoint block number to + // previous, so we could finish pruning its storage changesets on the next run. + .map(|block_number| if done { block_number } else { block_number.saturating_sub(1) }) + .unwrap_or(block_range_end); + + let (accounts_pruned, done) = + provider.tx_ref().prune_table_with_range::( + block_range, + &mut limiter, + |_| false, + |row| last_accounts_pruned_block = row.0, + )?; + + trace!(target: "pruner", %accounts_pruned, %done, "Pruned accounts change sets"); + + let progress = limiter.progress(done); + + Ok(SegmentOutput { + progress, + pruned: accounts_pruned + storages_pruned, + checkpoint: Some(SegmentOutputCheckpoint { + block_number: Some(last_storages_pruned_block.min(last_accounts_pruned_block)), + tx_number: None, + }), + }) + } + + fn save_checkpoint( + &self, + provider: &Provider, + checkpoint: PruneCheckpoint, + ) -> ProviderResult<()> { + provider.save_prune_checkpoint(PruneSegment::MerkleChangeSets, checkpoint) + } +} diff --git a/crates/prune/prune/src/segments/user/mod.rs b/crates/prune/prune/src/segments/user/mod.rs index 0b787d14dae..b993d3f2616 100644 --- a/crates/prune/prune/src/segments/user/mod.rs +++ b/crates/prune/prune/src/segments/user/mod.rs @@ -1,5 +1,7 @@ mod account_history; +mod bodies; mod history; +mod merkle_change_sets; mod receipts; mod receipts_by_logs; mod sender_recovery; @@ -7,6 +9,8 @@ mod storage_history; mod transaction_lookup; pub use account_history::AccountHistory; +pub use bodies::Bodies; +pub use merkle_change_sets::MerkleChangeSets; pub use receipts::Receipts; pub use receipts_by_logs::ReceiptsByLogs; pub use sender_recovery::SenderRecovery; diff --git a/crates/prune/prune/src/segments/user/receipts.rs b/crates/prune/prune/src/segments/user/receipts.rs index ecb0f3423be..03faddc1d5b 100644 --- a/crates/prune/prune/src/segments/user/receipts.rs +++ b/crates/prune/prune/src/segments/user/receipts.rs @@ -42,7 +42,7 @@ where PrunePurpose::User } - #[instrument(level = "trace", target = "pruner", skip(self, provider), ret)] + #[instrument(target = "pruner", skip(self, provider), ret(level = "trace"))] fn prune(&self, provider: &Provider, input: PruneInput) -> Result { crate::segments::receipts::prune(provider, input) } diff --git a/crates/prune/prune/src/segments/user/receipts_by_logs.rs b/crates/prune/prune/src/segments/user/receipts_by_logs.rs index b413a70394b..9e57bd2411a 100644 --- a/crates/prune/prune/src/segments/user/receipts_by_logs.rs +++ b/crates/prune/prune/src/segments/user/receipts_by_logs.rs @@ -45,7 +45,7 @@ where PrunePurpose::User } - #[instrument(level = "trace", target = "pruner", skip(self, provider), ret)] + #[instrument(target = "pruner", skip(self, provider), ret(level = "trace"))] fn prune(&self, provider: &Provider, input: PruneInput) -> Result { // Contract log filtering removes every receipt possible except the ones in the list. So, // for the other receipts it's as if they had a `PruneMode::Distance()` of @@ -180,7 +180,7 @@ where last_pruned_block = Some( provider - .transaction_block(last_pruned_transaction)? + .block_by_transaction_id(last_pruned_transaction)? .ok_or(PrunerError::InconsistentData("Block for transaction is not found"))? // If there's more receipts to prune, set the checkpoint block number to // previous, so we could finish pruning its receipts on the @@ -227,12 +227,12 @@ where #[cfg(test)] mod tests { - use crate::segments::{PruneInput, PruneLimiter, ReceiptsByLogs, Segment}; + use crate::segments::{user::ReceiptsByLogs, PruneInput, PruneLimiter, Segment}; use alloy_primitives::B256; use assert_matches::assert_matches; use reth_db_api::{cursor::DbCursorRO, tables, transaction::DbTx}; use reth_primitives_traits::InMemorySize; - use reth_provider::{DatabaseProviderFactory, PruneCheckpointReader, TransactionsProvider}; + use reth_provider::{BlockReader, DBProvider, DatabaseProviderFactory, PruneCheckpointReader}; use reth_prune_types::{PruneMode, PruneSegment, ReceiptsLogPruneConfig}; use reth_stages::test_utils::{StorageKind, TestStageDB}; use reth_testing_utils::generators::{ @@ -274,7 +274,7 @@ mod tests { for block in &blocks { receipts.reserve_exact(block.body().size()); for (txi, transaction) in block.body().transactions.iter().enumerate() { - let mut receipt = random_receipt(&mut rng, transaction, Some(1)); + let mut receipt = random_receipt(&mut rng, transaction, Some(1), None); receipt.logs.push(random_log( &mut rng, (txi == (block.transaction_count() - 1)).then_some(deposit_contract_addr), @@ -355,7 +355,7 @@ mod tests { // set by tip - 128 assert!( receipt.logs.iter().any(|l| l.address == deposit_contract_addr) || - provider.transaction_block(tx_num).unwrap().unwrap() > tip - 128, + provider.block_by_transaction_id(tx_num).unwrap().unwrap() > tip - 128, ); } } diff --git a/crates/prune/prune/src/segments/user/sender_recovery.rs b/crates/prune/prune/src/segments/user/sender_recovery.rs index bb0a812aa94..9fbad8c428c 100644 --- a/crates/prune/prune/src/segments/user/sender_recovery.rs +++ b/crates/prune/prune/src/segments/user/sender_recovery.rs @@ -37,7 +37,7 @@ where PrunePurpose::User } - #[instrument(level = "trace", target = "pruner", skip(self, provider), ret)] + #[instrument(target = "pruner", skip(self, provider), ret(level = "trace"))] fn prune(&self, provider: &Provider, input: PruneInput) -> Result { let tx_range = match input.get_next_tx_num_range(provider)? { Some(range) => range, @@ -90,8 +90,8 @@ mod tests { Itertools, }; use reth_db_api::tables; - use reth_primitives_traits::SignedTransaction; - use reth_provider::{DatabaseProviderFactory, PruneCheckpointReader}; + use reth_primitives_traits::SignerRecoverable; + use reth_provider::{DBProvider, DatabaseProviderFactory, PruneCheckpointReader}; use reth_prune_types::{PruneCheckpoint, PruneMode, PruneProgress, PruneSegment}; use reth_stages::test_utils::{StorageKind, TestStageDB}; use reth_testing_utils::generators::{self, random_block_range, BlockRangeParams}; @@ -115,7 +115,7 @@ mod tests { for transaction in &block.body().transactions { transaction_senders.push(( transaction_senders.len() as u64, - SignedTransaction::recover_signer(transaction).expect("recover signer"), + transaction.recover_signer().expect("recover signer"), )); } } diff --git a/crates/prune/prune/src/segments/user/storage_history.rs b/crates/prune/prune/src/segments/user/storage_history.rs index aa9cb846448..a4ad37bf789 100644 --- a/crates/prune/prune/src/segments/user/storage_history.rs +++ b/crates/prune/prune/src/segments/user/storage_history.rs @@ -47,7 +47,7 @@ where PrunePurpose::User } - #[instrument(level = "trace", target = "pruner", skip(self, provider), ret)] + #[instrument(target = "pruner", skip(self, provider), ret(level = "trace"))] fn prune(&self, provider: &Provider, input: PruneInput) -> Result { let range = match input.get_next_block_range() { Some(range) => range, @@ -140,7 +140,7 @@ mod tests { use alloy_primitives::{BlockNumber, B256}; use assert_matches::assert_matches; use reth_db_api::{tables, BlockNumberList}; - use reth_provider::{DatabaseProviderFactory, PruneCheckpointReader}; + use reth_provider::{DBProvider, DatabaseProviderFactory, PruneCheckpointReader}; use reth_prune_types::{PruneCheckpoint, PruneMode, PruneProgress, PruneSegment}; use reth_stages::test_utils::{StorageKind, TestStageDB}; use reth_testing_utils::generators::{ diff --git a/crates/prune/prune/src/segments/user/transaction_lookup.rs b/crates/prune/prune/src/segments/user/transaction_lookup.rs index 92a69dfd127..fed90d84f2d 100644 --- a/crates/prune/prune/src/segments/user/transaction_lookup.rs +++ b/crates/prune/prune/src/segments/user/transaction_lookup.rs @@ -6,9 +6,12 @@ use crate::{ use alloy_eips::eip2718::Encodable2718; use rayon::prelude::*; use reth_db_api::{tables, transaction::DbTxMut}; -use reth_provider::{BlockReader, DBProvider}; -use reth_prune_types::{PruneMode, PrunePurpose, PruneSegment, SegmentOutputCheckpoint}; -use tracing::{instrument, trace}; +use reth_provider::{BlockReader, DBProvider, PruneCheckpointReader, StaticFileProviderFactory}; +use reth_prune_types::{ + PruneCheckpoint, PruneMode, PrunePurpose, PruneSegment, SegmentOutputCheckpoint, +}; +use reth_static_file_types::StaticFileSegment; +use tracing::{debug, instrument, trace}; #[derive(Debug)] pub struct TransactionLookup { @@ -23,7 +26,10 @@ impl TransactionLookup { impl Segment for TransactionLookup where - Provider: DBProvider + BlockReader, + Provider: DBProvider + + BlockReader + + PruneCheckpointReader + + StaticFileProviderFactory, { fn segment(&self) -> PruneSegment { PruneSegment::TransactionLookup @@ -37,8 +43,37 @@ where PrunePurpose::User } - #[instrument(level = "trace", target = "pruner", skip(self, provider), ret)] - fn prune(&self, provider: &Provider, input: PruneInput) -> Result { + #[instrument(target = "pruner", skip(self, provider), ret(level = "trace"))] + fn prune( + &self, + provider: &Provider, + mut input: PruneInput, + ) -> Result { + // It is not possible to prune TransactionLookup data for which we don't have transaction + // data. If the TransactionLookup checkpoint is lagging behind (which can happen e.g. when + // pre-merge history is dropped and then later tx lookup pruning is enabled) then we can + // only prune from the lowest static file. + if let Some(lowest_range) = + provider.static_file_provider().get_lowest_range(StaticFileSegment::Transactions) && + input + .previous_checkpoint + .is_none_or(|checkpoint| checkpoint.block_number < Some(lowest_range.start())) + { + let new_checkpoint = lowest_range.start().saturating_sub(1); + if let Some(body_indices) = provider.block_body_indices(new_checkpoint)? { + input.previous_checkpoint = Some(PruneCheckpoint { + block_number: Some(new_checkpoint), + tx_number: Some(body_indices.last_tx_num()), + prune_mode: self.mode, + }); + debug!( + target: "pruner", + static_file_checkpoint = ?input.previous_checkpoint, + "Using static file transaction checkpoint as TransactionLookup starting point" + ); + } + } + let (start, end) = match input.get_next_tx_num_range(provider)? { Some(range) => range, None => { @@ -117,7 +152,7 @@ mod tests { Itertools, }; use reth_db_api::tables; - use reth_provider::{DatabaseProviderFactory, PruneCheckpointReader}; + use reth_provider::{DBProvider, DatabaseProviderFactory, PruneCheckpointReader}; use reth_prune_types::{ PruneCheckpoint, PruneInterruptReason, PruneMode, PruneProgress, PruneSegment, }; @@ -135,7 +170,7 @@ mod tests { 1..=10, BlockRangeParams { parent: Some(B256::ZERO), tx_count: 2..3, ..Default::default() }, ); - db.insert_blocks(blocks.iter(), StorageKind::Database(None)).expect("insert blocks"); + db.insert_blocks(blocks.iter(), StorageKind::Static).expect("insert blocks"); let mut tx_hash_numbers = Vec::new(); for block in &blocks { @@ -148,11 +183,11 @@ mod tests { db.insert_tx_hash_numbers(tx_hash_numbers).expect("insert tx hash numbers"); assert_eq!( - db.table::().unwrap().len(), + db.count_entries::().unwrap(), blocks.iter().map(|block| block.transaction_count()).sum::() ); assert_eq!( - db.table::().unwrap().len(), + db.count_entries::().unwrap(), db.table::().unwrap().len() ); diff --git a/crates/prune/types/Cargo.toml b/crates/prune/types/Cargo.toml index 42a6d7f2082..30adbb14d91 100644 --- a/crates/prune/types/Cargo.toml +++ b/crates/prune/types/Cargo.toml @@ -16,6 +16,7 @@ reth-codecs = { workspace = true, optional = true } alloy-primitives.workspace = true derive_more.workspace = true +strum = { workspace = true, features = ["derive"] } thiserror.workspace = true modular-bitfield = { workspace = true, optional = true } @@ -27,13 +28,11 @@ reth-codecs.workspace = true alloy-primitives = { workspace = true, features = ["serde"] } serde.workspace = true -modular-bitfield.workspace = true arbitrary = { workspace = true, features = ["derive"] } assert_matches.workspace = true proptest.workspace = true proptest-arbitrary-interop.workspace = true serde_json.workspace = true -test-fuzz.workspace = true toml.workspace = true [features] @@ -44,8 +43,10 @@ std = [ "serde?/std", "serde_json/std", "thiserror/std", + "strum/std", ] test-utils = [ + "std", "dep:arbitrary", "reth-codecs?/test-utils", ] diff --git a/crates/prune/types/src/lib.rs b/crates/prune/types/src/lib.rs index ef8ef882b8c..315063278b2 100644 --- a/crates/prune/types/src/lib.rs +++ b/crates/prune/types/src/lib.rs @@ -6,7 +6,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(feature = "std"), no_std)] extern crate alloc; @@ -30,7 +30,7 @@ pub use pruner::{ SegmentOutputCheckpoint, }; pub use segment::{PrunePurpose, PruneSegment, PruneSegmentError}; -pub use target::{PruneModes, MINIMUM_PRUNING_DISTANCE}; +pub use target::{PruneModes, UnwindTargetPrunedError, MINIMUM_PRUNING_DISTANCE}; /// Configuration for pruning receipts not associated with logs emitted by the specified contracts. #[derive(Debug, Clone, PartialEq, Eq, Default)] @@ -96,12 +96,11 @@ impl ReceiptsLogPruneConfig { let mut lowest = None; for mode in self.values() { - if mode.is_distance() { - if let Some((block, _)) = + if mode.is_distance() && + let Some((block, _)) = mode.prune_target_block(tip, PruneSegment::ContractLogs, PrunePurpose::User)? - { - lowest = Some(lowest.unwrap_or(u64::MAX).min(block)); - } + { + lowest = Some(lowest.unwrap_or(u64::MAX).min(block)); } } diff --git a/crates/prune/types/src/mode.rs b/crates/prune/types/src/mode.rs index 42d34b30cc7..0565087673d 100644 --- a/crates/prune/types/src/mode.rs +++ b/crates/prune/types/src/mode.rs @@ -18,6 +18,7 @@ pub enum PruneMode { } #[cfg(any(test, feature = "test-utils"))] +#[allow(clippy::derivable_impls)] impl Default for PruneMode { fn default() -> Self { Self::Full @@ -128,7 +129,11 @@ mod tests { // Test for a scenario where there are no minimum blocks and Full can be used assert_eq!( - PruneMode::Full.prune_target_block(tip, PruneSegment::Transactions, PrunePurpose::User), + PruneMode::Full.prune_target_block( + tip, + PruneSegment::TransactionLookup, + PrunePurpose::User + ), Ok(Some((tip, PruneMode::Full))), ); } diff --git a/crates/prune/types/src/segment.rs b/crates/prune/types/src/segment.rs index 443acf1ed79..36e39fcb585 100644 --- a/crates/prune/types/src/segment.rs +++ b/crates/prune/types/src/segment.rs @@ -1,9 +1,16 @@ +#![allow(deprecated)] // necessary to all defining deprecated `PruneSegment` variants + use crate::MINIMUM_PRUNING_DISTANCE; use derive_more::Display; +use strum::{EnumIter, IntoEnumIterator}; use thiserror::Error; /// Segment of the data that can be pruned. -#[derive(Debug, Display, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash)] +/// +/// VERY IMPORTANT NOTE: new variants must be added to the end of this enum, and old variants which +/// are no longer used must not be removed from this enum. The variant index is encoded directly +/// when writing to the `PruneCheckpoint` table, so changing the order here will corrupt the table. +#[derive(Debug, Display, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash, EnumIter)] #[cfg_attr(test, derive(arbitrary::Arbitrary))] #[cfg_attr(any(test, feature = "reth-codec"), derive(reth_codecs::Compact))] #[cfg_attr(any(test, feature = "reth-codec"), reth_codecs::add_arbitrary_tests(compact))] @@ -21,27 +28,64 @@ pub enum PruneSegment { AccountHistory, /// Prune segment responsible for the `StorageChangeSets` and `StoragesHistory` tables. StorageHistory, - /// Prune segment responsible for the `CanonicalHeaders`, `Headers` and - /// `HeaderTerminalDifficulties` tables. + #[deprecated = "Variant indexes cannot be changed"] + #[strum(disabled)] + /// Prune segment responsible for the `CanonicalHeaders`, `Headers` tables. Headers, + #[deprecated = "Variant indexes cannot be changed"] + #[strum(disabled)] /// Prune segment responsible for the `Transactions` table. Transactions, + /// Prune segment responsible for all rows in `AccountsTrieChangeSets` and + /// `StoragesTrieChangeSets` table. + MerkleChangeSets, + /// Prune segment responsible for bodies (transactions in static files). + Bodies, +} + +#[cfg(test)] +#[allow(clippy::derivable_impls)] +impl Default for PruneSegment { + fn default() -> Self { + Self::SenderRecovery + } } impl PruneSegment { + /// Returns an iterator over all variants of [`PruneSegment`]. + /// + /// Excludes deprecated variants that are no longer used, but can still be found in the + /// database. + pub fn variants() -> impl Iterator { + Self::iter() + } + /// Returns minimum number of blocks to keep in the database for this segment. pub const fn min_blocks(&self, purpose: PrunePurpose) -> u64 { match self { - Self::SenderRecovery | Self::TransactionLookup | Self::Headers | Self::Transactions => { - 0 - } + Self::SenderRecovery | Self::TransactionLookup => 0, Self::Receipts if purpose.is_static_file() => 0, - Self::ContractLogs | Self::AccountHistory | Self::StorageHistory => { - MINIMUM_PRUNING_DISTANCE - } + Self::ContractLogs | + Self::AccountHistory | + Self::StorageHistory | + Self::MerkleChangeSets | + Self::Bodies | Self::Receipts => MINIMUM_PRUNING_DISTANCE, + #[expect(deprecated)] + #[expect(clippy::match_same_arms)] + Self::Headers | Self::Transactions => 0, } } + + /// Returns true if this is [`Self::AccountHistory`]. + pub const fn is_account_history(&self) -> bool { + matches!(self, Self::AccountHistory) + } + + /// Returns true if this is [`Self::StorageHistory`]. + pub const fn is_storage_history(&self) -> bool { + matches!(self, Self::StorageHistory) + } } /// Prune purpose. @@ -74,8 +118,18 @@ pub enum PruneSegmentError { } #[cfg(test)] -impl Default for PruneSegment { - fn default() -> Self { - Self::SenderRecovery +mod tests { + use super::*; + + #[test] + fn test_prune_segment_iter_excludes_deprecated() { + let segments: Vec = PruneSegment::variants().collect(); + + // Verify deprecated variants are not included derived iter + #[expect(deprecated)] + { + assert!(!segments.contains(&PruneSegment::Headers)); + assert!(!segments.contains(&PruneSegment::Transactions)); + } } } diff --git a/crates/prune/types/src/target.rs b/crates/prune/types/src/target.rs index 9edeb71ec97..3ff18554a9b 100644 --- a/crates/prune/types/src/target.rs +++ b/crates/prune/types/src/target.rs @@ -1,4 +1,8 @@ -use crate::{PruneMode, ReceiptsLogPruneConfig}; +use alloy_primitives::BlockNumber; +use derive_more::Display; +use thiserror::Error; + +use crate::{PruneCheckpoint, PruneMode, PruneSegment, ReceiptsLogPruneConfig}; /// Minimum distance from the tip necessary for the node to work correctly: /// 1. Minimum 2 epochs (32 blocks per epoch) required to handle any reorg according to the @@ -7,8 +11,38 @@ use crate::{PruneMode, ReceiptsLogPruneConfig}; /// unwind is required. pub const MINIMUM_PRUNING_DISTANCE: u64 = 32 * 2 + 10_000; +/// Type of history that can be pruned +#[derive(Debug, Error, PartialEq, Eq, Clone)] +pub enum UnwindTargetPrunedError { + /// The target block is beyond the history limit + #[error("Cannot unwind to block {target_block} as it is beyond the {history_type} limit. Latest block: {latest_block}, History limit: {limit}")] + TargetBeyondHistoryLimit { + /// The latest block number + latest_block: BlockNumber, + /// The target block number + target_block: BlockNumber, + /// The type of history that is beyond the limit + history_type: HistoryType, + /// The limit of the history + limit: u64, + }, +} + +#[derive(Debug, Display, Clone, PartialEq, Eq)] +pub enum HistoryType { + /// Account history + AccountHistory, + /// Storage history + StorageHistory, +} + +/// Default pruning mode for merkle changesets +const fn default_merkle_changesets_mode() -> PruneMode { + PruneMode::Distance(MINIMUM_PRUNING_DISTANCE) +} + /// Pruning configuration for every segment of the data that can be pruned. -#[derive(Debug, Clone, Default, Eq, PartialEq)] +#[derive(Debug, Clone, Eq, PartialEq)] #[cfg_attr(any(test, feature = "serde"), derive(serde::Serialize, serde::Deserialize))] #[cfg_attr(any(test, feature = "serde"), serde(default))] pub struct PruneModes { @@ -46,20 +80,53 @@ pub struct PruneModes { ) )] pub storage_history: Option, + /// Bodies History pruning configuration. + #[cfg_attr( + any(test, feature = "serde"), + serde( + skip_serializing_if = "Option::is_none", + deserialize_with = "deserialize_opt_prune_mode_with_min_blocks::" + ) + )] + pub bodies_history: Option, + /// Merkle Changesets pruning configuration for `AccountsTrieChangeSets` and + /// `StoragesTrieChangeSets`. + #[cfg_attr( + any(test, feature = "serde"), + serde( + default = "default_merkle_changesets_mode", + deserialize_with = "deserialize_prune_mode_with_min_blocks::" + ) + )] + pub merkle_changesets: PruneMode, /// Receipts pruning configuration by retaining only those receipts that contain logs emitted /// by the specified addresses, discarding others. This setting is overridden by `receipts`. /// /// The [`BlockNumber`](`crate::BlockNumber`) represents the starting block from which point /// onwards the receipts are preserved. + #[cfg_attr( + any(test, feature = "serde"), + serde(skip_serializing_if = "ReceiptsLogPruneConfig::is_empty") + )] pub receipts_log_filter: ReceiptsLogPruneConfig, } -impl PruneModes { - /// Sets pruning to no target. - pub fn none() -> Self { - Self::default() +impl Default for PruneModes { + fn default() -> Self { + Self { + sender_recovery: None, + transaction_lookup: None, + receipts: None, + account_history: None, + storage_history: None, + bodies_history: None, + merkle_changesets: default_merkle_changesets_mode(), + receipts_log_filter: ReceiptsLogPruneConfig::default(), + } } +} +impl PruneModes { /// Sets pruning to all targets. pub fn all() -> Self { Self { @@ -68,6 +135,8 @@ impl PruneModes { receipts: Some(PruneMode::Full), account_history: Some(PruneMode::Full), storage_history: Some(PruneMode::Full), + bodies_history: Some(PruneMode::Full), + merkle_changesets: PruneMode::Full, receipts_log_filter: Default::default(), } } @@ -77,20 +146,85 @@ impl PruneModes { self.receipts.is_some() || !self.receipts_log_filter.is_empty() } - /// Returns true if all prune modes are set to [`None`]. - pub fn is_empty(&self) -> bool { - self == &Self::none() + /// Returns an error if we can't unwind to the targeted block because the target block is + /// outside the range. + /// + /// This is only relevant for certain tables that are required by other stages + /// + /// See also + pub fn ensure_unwind_target_unpruned( + &self, + latest_block: u64, + target_block: u64, + checkpoints: &[(PruneSegment, PruneCheckpoint)], + ) -> Result<(), UnwindTargetPrunedError> { + let distance = latest_block.saturating_sub(target_block); + for (prune_mode, history_type, checkpoint) in &[ + ( + self.account_history, + HistoryType::AccountHistory, + checkpoints.iter().find(|(segment, _)| segment.is_account_history()), + ), + ( + self.storage_history, + HistoryType::StorageHistory, + checkpoints.iter().find(|(segment, _)| segment.is_storage_history()), + ), + ] { + if let Some(PruneMode::Distance(limit)) = prune_mode { + // check if distance exceeds the configured limit + if distance > *limit { + // but only if have haven't pruned the target yet, if we dont have a checkpoint + // yet, it's fully unpruned yet + let pruned_height = checkpoint + .and_then(|checkpoint| checkpoint.1.block_number) + .unwrap_or(latest_block); + if pruned_height >= target_block { + // we've pruned the target block already and can't unwind past it + return Err(UnwindTargetPrunedError::TargetBeyondHistoryLimit { + latest_block, + target_block, + history_type: history_type.clone(), + limit: *limit, + }) + } + } + } + } + Ok(()) } } +/// Deserializes [`PruneMode`] and validates that the value is not less than the const +/// generic parameter `MIN_BLOCKS`. This parameter represents the number of blocks that needs to be +/// left in database after the pruning. +/// +/// 1. For [`PruneMode::Full`], it fails if `MIN_BLOCKS > 0`. +/// 2. For [`PruneMode::Distance`], it fails if `distance < MIN_BLOCKS + 1`. `+ 1` is needed because +/// `PruneMode::Distance(0)` means that we leave zero blocks from the latest, meaning we have one +/// block in the database. +#[cfg(any(test, feature = "serde"))] +fn deserialize_prune_mode_with_min_blocks< + 'de, + const MIN_BLOCKS: u64, + D: serde::Deserializer<'de>, +>( + deserializer: D, +) -> Result { + use serde::Deserialize; + let prune_mode = PruneMode::deserialize(deserializer)?; + serde_deserialize_validate::(&prune_mode)?; + Ok(prune_mode) +} + /// Deserializes [`Option`] and validates that the value is not less than the const /// generic parameter `MIN_BLOCKS`. This parameter represents the number of blocks that needs to be /// left in database after the pruning. /// /// 1. For [`PruneMode::Full`], it fails if `MIN_BLOCKS > 0`. -/// 2. For [`PruneMode::Distance(distance`)], it fails if `distance < MIN_BLOCKS + 1`. `+ 1` is -/// needed because `PruneMode::Distance(0)` means that we leave zero blocks from the latest, -/// meaning we have one block in the database. +/// 2. For [`PruneMode::Distance`], it fails if `distance < MIN_BLOCKS + 1`. `+ 1` is needed because +/// `PruneMode::Distance(0)` means that we leave zero blocks from the latest, meaning we have one +/// block in the database. #[cfg(any(test, feature = "serde"))] fn deserialize_opt_prune_mode_with_min_blocks< 'de, @@ -99,12 +233,21 @@ fn deserialize_opt_prune_mode_with_min_blocks< >( deserializer: D, ) -> Result, D::Error> { - use alloc::format; use serde::Deserialize; let prune_mode = Option::::deserialize(deserializer)?; + if let Some(prune_mode) = prune_mode.as_ref() { + serde_deserialize_validate::(prune_mode)?; + } + Ok(prune_mode) +} +#[cfg(any(test, feature = "serde"))] +fn serde_deserialize_validate<'a, 'de, const MIN_BLOCKS: u64, D: serde::Deserializer<'de>>( + prune_mode: &'a PruneMode, +) -> Result<(), D::Error> { + use alloc::format; match prune_mode { - Some(PruneMode::Full) if MIN_BLOCKS > 0 => { + PruneMode::Full if MIN_BLOCKS > 0 => { Err(serde::de::Error::invalid_value( serde::de::Unexpected::Str("full"), // This message should have "expected" wording @@ -112,15 +255,15 @@ fn deserialize_opt_prune_mode_with_min_blocks< .as_str(), )) } - Some(PruneMode::Distance(distance)) if distance < MIN_BLOCKS => { + PruneMode::Distance(distance) if *distance < MIN_BLOCKS => { Err(serde::de::Error::invalid_value( - serde::de::Unexpected::Unsigned(distance), + serde::de::Unexpected::Unsigned(*distance), // This message should have "expected" wording &format!("prune mode that leaves at least {MIN_BLOCKS} blocks in the database") .as_str(), )) } - _ => Ok(prune_mode), + _ => Ok(()), } } @@ -149,4 +292,165 @@ mod tests { Err(err) if err.to_string() == "invalid value: string \"full\", expected prune mode that leaves at least 10 blocks in the database" ); } + + #[test] + fn test_unwind_target_unpruned() { + // Test case 1: No pruning configured - should always succeed + let prune_modes = PruneModes::default(); + assert!(prune_modes.ensure_unwind_target_unpruned(1000, 500, &[]).is_ok()); + assert!(prune_modes.ensure_unwind_target_unpruned(1000, 0, &[]).is_ok()); + + // Test case 2: Distance pruning within limit - should succeed + let prune_modes = PruneModes { + account_history: Some(PruneMode::Distance(100)), + storage_history: Some(PruneMode::Distance(100)), + ..Default::default() + }; + // Distance is 50, limit is 100 - OK + assert!(prune_modes.ensure_unwind_target_unpruned(1000, 950, &[]).is_ok()); + + // Test case 3: Distance exceeds limit with no checkpoint + // NOTE: Current implementation assumes pruned_height = latest_block when no checkpoint + // exists This means it will fail because it assumes we've pruned up to block 1000 > + // target 800 + let prune_modes = + PruneModes { account_history: Some(PruneMode::Distance(100)), ..Default::default() }; + // Distance is 200 > 100, no checkpoint - current impl treats as pruned up to latest_block + let result = prune_modes.ensure_unwind_target_unpruned(1000, 800, &[]); + assert_matches!( + result, + Err(UnwindTargetPrunedError::TargetBeyondHistoryLimit { + latest_block: 1000, + target_block: 800, + history_type: HistoryType::AccountHistory, + limit: 100 + }) + ); + + // Test case 4: Distance exceeds limit and target is pruned - should fail + let prune_modes = + PruneModes { account_history: Some(PruneMode::Distance(100)), ..Default::default() }; + let checkpoints = vec![( + PruneSegment::AccountHistory, + PruneCheckpoint { + block_number: Some(850), + tx_number: None, + prune_mode: PruneMode::Distance(100), + }, + )]; + // Distance is 200 > 100, and checkpoint shows we've pruned up to block 850 > target 800 + let result = prune_modes.ensure_unwind_target_unpruned(1000, 800, &checkpoints); + assert_matches!( + result, + Err(UnwindTargetPrunedError::TargetBeyondHistoryLimit { + latest_block: 1000, + target_block: 800, + history_type: HistoryType::AccountHistory, + limit: 100 + }) + ); + + // Test case 5: Storage history exceeds limit and is pruned - should fail + let prune_modes = + PruneModes { storage_history: Some(PruneMode::Distance(50)), ..Default::default() }; + let checkpoints = vec![( + PruneSegment::StorageHistory, + PruneCheckpoint { + block_number: Some(960), + tx_number: None, + prune_mode: PruneMode::Distance(50), + }, + )]; + // Distance is 100 > 50, and checkpoint shows we've pruned up to block 960 > target 900 + let result = prune_modes.ensure_unwind_target_unpruned(1000, 900, &checkpoints); + assert_matches!( + result, + Err(UnwindTargetPrunedError::TargetBeyondHistoryLimit { + latest_block: 1000, + target_block: 900, + history_type: HistoryType::StorageHistory, + limit: 50 + }) + ); + + // Test case 6: Distance exceeds limit but target block not pruned yet - should succeed + let prune_modes = + PruneModes { account_history: Some(PruneMode::Distance(100)), ..Default::default() }; + let checkpoints = vec![( + PruneSegment::AccountHistory, + PruneCheckpoint { + block_number: Some(700), + tx_number: None, + prune_mode: PruneMode::Distance(100), + }, + )]; + // Distance is 200 > 100, but checkpoint shows we've only pruned up to block 700 < target + // 800 + assert!(prune_modes.ensure_unwind_target_unpruned(1000, 800, &checkpoints).is_ok()); + + // Test case 7: Both account and storage history configured, only one fails + let prune_modes = PruneModes { + account_history: Some(PruneMode::Distance(200)), + storage_history: Some(PruneMode::Distance(50)), + ..Default::default() + }; + let checkpoints = vec![ + ( + PruneSegment::AccountHistory, + PruneCheckpoint { + block_number: Some(700), + tx_number: None, + prune_mode: PruneMode::Distance(200), + }, + ), + ( + PruneSegment::StorageHistory, + PruneCheckpoint { + block_number: Some(960), + tx_number: None, + prune_mode: PruneMode::Distance(50), + }, + ), + ]; + // For target 900: account history OK (distance 100 < 200), storage history fails (distance + // 100 > 50, pruned at 960) + let result = prune_modes.ensure_unwind_target_unpruned(1000, 900, &checkpoints); + assert_matches!( + result, + Err(UnwindTargetPrunedError::TargetBeyondHistoryLimit { + latest_block: 1000, + target_block: 900, + history_type: HistoryType::StorageHistory, + limit: 50 + }) + ); + + // Test case 8: Edge case - exact boundary + let prune_modes = + PruneModes { account_history: Some(PruneMode::Distance(100)), ..Default::default() }; + let checkpoints = vec![( + PruneSegment::AccountHistory, + PruneCheckpoint { + block_number: Some(900), + tx_number: None, + prune_mode: PruneMode::Distance(100), + }, + )]; + // Distance is exactly 100, checkpoint at exactly the target block + assert!(prune_modes.ensure_unwind_target_unpruned(1000, 900, &checkpoints).is_ok()); + + // Test case 9: Full pruning mode - should succeed (no distance check) + let prune_modes = PruneModes { + account_history: Some(PruneMode::Full), + storage_history: Some(PruneMode::Full), + ..Default::default() + }; + assert!(prune_modes.ensure_unwind_target_unpruned(1000, 0, &[]).is_ok()); + + // Test case 10: Edge case - saturating subtraction (target > latest) + let prune_modes = + PruneModes { account_history: Some(PruneMode::Distance(100)), ..Default::default() }; + // Target block (1500) > latest block (1000) - distance should be 0 + assert!(prune_modes.ensure_unwind_target_unpruned(1000, 1500, &[]).is_ok()); + } } diff --git a/crates/ress/protocol/src/lib.rs b/crates/ress/protocol/src/lib.rs index 8eb0040620c..82820cc5a31 100644 --- a/crates/ress/protocol/src/lib.rs +++ b/crates/ress/protocol/src/lib.rs @@ -1,5 +1,30 @@ -//! `ress` protocol is an `RLPx` subprotocol for stateless nodes. -//! following [RLPx specs](https://github.com/ethereum/devp2p/blob/master/rlpx.md) +//! RESS protocol for stateless Ethereum nodes. +//! +//! Enables stateless nodes to fetch execution witnesses, bytecode, and block data from +//! stateful peers for minimal on-disk state with full execution capability. +//! +//! ## Node Types +//! +//! - **Stateless**: Minimal state, requests data on-demand +//! - **Stateful**: Full Ethereum nodes providing state data +//! +//! Valid connections: Stateless ↔ Stateless ✅, Stateless ↔ Stateful ✅, Stateful ↔ Stateful ❌ +//! +//! ## Messages +//! +//! - `NodeType (0x00)`: Handshake +//! - `GetHeaders/Headers (0x01/0x02)`: Block headers +//! - `GetBlockBodies/BlockBodies (0x03/0x04)`: Block bodies +//! - `GetBytecode/Bytecode (0x05/0x06)`: Contract bytecode +//! - `GetWitness/Witness (0x07/0x08)`: Execution witnesses +//! +//! ## Flow +//! +//! 1. Exchange `NodeType` for compatibility +//! 2. Download ancestor blocks via headers/bodies +//! 3. For new payloads: request witness → get missing bytecode → execute +//! +//! Protocol version: `ress/1` #![doc( html_logo_url = "https://raw.githubusercontent.com/paradigmxyz/reth/main/assets/reth-docs.png", @@ -7,7 +32,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] mod types; pub use types::*; diff --git a/crates/ress/protocol/src/provider.rs b/crates/ress/protocol/src/provider.rs index bac8b1f5f0a..ef4ee493314 100644 --- a/crates/ress/protocol/src/provider.rs +++ b/crates/ress/protocol/src/provider.rs @@ -14,6 +14,9 @@ pub trait RessProtocolProvider: Send + Sync { /// Return block headers. fn headers(&self, request: GetHeaders) -> ProviderResult> { + if request.limit == 0 { + return Ok(Vec::new()); + } let mut total_bytes = 0; let mut block_hash = request.start_hash; let mut headers = Vec::new(); diff --git a/crates/ress/provider/Cargo.toml b/crates/ress/provider/Cargo.toml index 39216c03d6a..7bd9beee2b4 100644 --- a/crates/ress/provider/Cargo.toml +++ b/crates/ress/provider/Cargo.toml @@ -13,7 +13,8 @@ workspace = true [dependencies] reth-ress-protocol.workspace = true reth-primitives-traits.workspace = true -reth-provider.workspace = true +reth-storage-api.workspace = true +reth-errors.workspace = true reth-evm.workspace = true reth-revm = { workspace = true, features = ["witness"] } reth-chain-state.workspace = true diff --git a/crates/ress/provider/src/lib.rs b/crates/ress/provider/src/lib.rs index 5cafb244423..d986eb9e953 100644 --- a/crates/ress/provider/src/lib.rs +++ b/crates/ress/provider/src/lib.rs @@ -1,17 +1,21 @@ //! Reth implementation of [`reth_ress_protocol::RessProtocolProvider`]. +#![doc( + html_logo_url = "https://raw.githubusercontent.com/paradigmxyz/reth/main/assets/reth-docs.png", + html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256", + issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" +)] #![cfg_attr(not(test), warn(unused_crate_dependencies))] +#![cfg_attr(docsrs, feature(doc_cfg))] use alloy_consensus::BlockHeader as _; use alloy_primitives::{Bytes, B256}; use parking_lot::Mutex; -use reth_chain_state::{ExecutedBlock, ExecutedBlockWithTrieUpdates, MemoryOverlayStateProvider}; +use reth_chain_state::{ExecutedBlock, MemoryOverlayStateProvider}; +use reth_errors::{ProviderError, ProviderResult}; use reth_ethereum_primitives::{Block, BlockBody, EthPrimitives}; use reth_evm::{execute::Executor, ConfigureEvm}; use reth_primitives_traits::{Block as _, Header, RecoveredBlock}; -use reth_provider::{ - BlockReader, BlockSource, ProviderError, ProviderResult, StateProvider, StateProviderFactory, -}; use reth_ress_protocol::RessProtocolProvider; use reth_revm::{database::StateProviderDatabase, db::State, witness::ExecutionWitnessRecord}; use reth_tasks::TaskSpawner; @@ -26,6 +30,7 @@ use recorder::StateWitnessRecorderDatabase; mod pending_state; pub use pending_state::*; +use reth_storage_api::{BlockReader, BlockSource, StateProviderFactory}; /// Reth provider implementing [`RessProtocolProvider`]. #[expect(missing_debug_implementations)] @@ -113,19 +118,13 @@ where let mut executed = self.pending_state.executed_block(&ancestor_hash); // If it's not present, attempt to lookup invalid block. - if executed.is_none() { - if let Some(invalid) = + if executed.is_none() && + let Some(invalid) = self.pending_state.invalid_recovered_block(&ancestor_hash) - { - trace!(target: "reth::ress_provider", %block_hash, %ancestor_hash, "Using invalid ancestor block for witness construction"); - executed = Some(ExecutedBlockWithTrieUpdates { - block: ExecutedBlock { - recovered_block: invalid, - ..Default::default() - }, - ..Default::default() - }); - } + { + trace!(target: "reth::ress_provider", %block_hash, %ancestor_hash, "Using invalid ancestor block for witness construction"); + executed = + Some(ExecutedBlock { recovered_block: invalid, ..Default::default() }); } let Some(executed) = executed else { @@ -159,7 +158,8 @@ where let witness_state_provider = self.provider.state_by_block_hash(ancestor_hash)?; let mut trie_input = TrieInput::default(); for block in executed_ancestors.into_iter().rev() { - trie_input.append_cached_ref(&block.trie, &block.hashed_state); + let trie_updates = block.trie_updates.as_ref(); + trie_input.append_cached_ref(trie_updates, &block.hashed_state); } let mut hashed_state = db.into_state(); hashed_state.extend(record.hashed_state); @@ -183,7 +183,8 @@ where }; // Insert witness into the cache. - self.witness_cache.lock().insert(block_hash, Arc::new(witness.clone())); + let cached_witness = Arc::new(witness.clone()); + self.witness_cache.lock().insert(block_hash, cached_witness); Ok(witness) } diff --git a/crates/ress/provider/src/pending_state.rs b/crates/ress/provider/src/pending_state.rs index bc0ea4702cb..f536acdb60a 100644 --- a/crates/ress/provider/src/pending_state.rs +++ b/crates/ress/provider/src/pending_state.rs @@ -5,11 +5,11 @@ use alloy_primitives::{ }; use futures::StreamExt; use parking_lot::RwLock; -use reth_chain_state::ExecutedBlockWithTrieUpdates; +use reth_chain_state::ExecutedBlock; use reth_ethereum_primitives::EthPrimitives; -use reth_node_api::{BeaconConsensusEngineEvent, NodePrimitives}; +use reth_node_api::{ConsensusEngineEvent, NodePrimitives}; use reth_primitives_traits::{Bytecode, RecoveredBlock}; -use reth_provider::BlockNumReader; +use reth_storage_api::BlockNumReader; use reth_tokio_util::EventStream; use std::{collections::BTreeMap, sync::Arc}; use tracing::*; @@ -20,14 +20,14 @@ pub struct PendingState(Arc>>); #[derive(Default, Debug)] struct PendingStateInner { - blocks_by_hash: B256Map>, + blocks_by_hash: B256Map>, invalid_blocks_by_hash: B256Map>>, block_hashes_by_number: BTreeMap, } impl PendingState { /// Insert executed block with trie updates. - pub fn insert_block(&self, block: ExecutedBlockWithTrieUpdates) { + pub fn insert_block(&self, block: ExecutedBlock) { let mut this = self.0.write(); let block_hash = block.recovered_block.hash(); this.block_hashes_by_number @@ -46,13 +46,13 @@ impl PendingState { } /// Returns only valid executed blocks by hash. - pub fn executed_block(&self, hash: &B256) -> Option> { + pub fn executed_block(&self, hash: &B256) -> Option> { self.0.read().blocks_by_hash.get(hash).cloned() } /// Returns valid recovered block. pub fn recovered_block(&self, hash: &B256) -> Option>> { - self.executed_block(hash).map(|b| b.recovered_block.clone()) + self.executed_block(hash).map(|b| b.recovered_block) } /// Returns invalid recovered block. @@ -93,7 +93,7 @@ impl PendingState { /// A task to maintain pending state based on consensus engine events. pub async fn maintain_pending_state

( - mut events: EventStream>, + mut events: EventStream>, provider: P, pending_state: PendingState, ) where @@ -101,18 +101,18 @@ pub async fn maintain_pending_state

( { while let Some(event) = events.next().await { match event { - BeaconConsensusEngineEvent::CanonicalBlockAdded(block, _) | - BeaconConsensusEngineEvent::ForkBlockAdded(block, _) => { + ConsensusEngineEvent::CanonicalBlockAdded(block, _) | + ConsensusEngineEvent::ForkBlockAdded(block, _) => { trace!(target: "reth::ress_provider", block = ? block.recovered_block().num_hash(), "Insert block into pending state"); pending_state.insert_block(block); } - BeaconConsensusEngineEvent::InvalidBlock(block) => { + ConsensusEngineEvent::InvalidBlock(block) => { if let Ok(block) = block.try_recover() { trace!(target: "reth::ress_provider", block = ?block.num_hash(), "Insert invalid block into pending state"); pending_state.insert_invalid_block(Arc::new(block)); } } - BeaconConsensusEngineEvent::ForkchoiceUpdated(state, status) => { + ConsensusEngineEvent::ForkchoiceUpdated(state, status) => { if status.is_valid() { let target = state.finalized_block_hash; if let Ok(Some(block_number)) = provider.block_number(target) { @@ -122,8 +122,9 @@ pub async fn maintain_pending_state

( } } // ignore - BeaconConsensusEngineEvent::CanonicalChainCommitted(_, _) | - BeaconConsensusEngineEvent::LiveSyncProgress(_) => (), + ConsensusEngineEvent::CanonicalChainCommitted(_, _) | + ConsensusEngineEvent::BlockReceived(_) | + ConsensusEngineEvent::LiveSyncProgress(_) => (), } } } diff --git a/crates/ress/provider/src/recorder.rs b/crates/ress/provider/src/recorder.rs index b692dd9a4d1..ec5afacbf0c 100644 --- a/crates/ress/provider/src/recorder.rs +++ b/crates/ress/provider/src/recorder.rs @@ -8,6 +8,7 @@ use reth_trie::{HashedPostState, HashedStorage}; /// The state witness recorder that records all state accesses during execution. /// It does so by implementing the [`reth_revm::Database`] and recording accesses of accounts and /// slots. +#[derive(Debug)] pub(crate) struct StateWitnessRecorderDatabase { database: D, state: HashedPostState, diff --git a/crates/revm/Cargo.toml b/crates/revm/Cargo.toml index 1e5d36ec22a..92036e39085 100644 --- a/crates/revm/Cargo.toml +++ b/crates/revm/Cargo.toml @@ -27,7 +27,6 @@ revm.workspace = true [dev-dependencies] reth-trie.workspace = true reth-ethereum-forks.workspace = true -alloy-primitives.workspace = true alloy-consensus.workspace = true [features] @@ -54,5 +53,19 @@ serde = [ "reth-trie?/serde", "reth-ethereum-forks/serde", "reth-primitives-traits/serde", + "reth-storage-api/serde", ] portable = ["revm/portable"] +optional-balance-check = ["revm/optional_balance_check"] +optional-block-gas-limit = ["revm/optional_block_gas_limit"] +optional-eip3541 = ["revm/optional_eip3541"] +optional-eip3607 = ["revm/optional_eip3607"] +optional-no-base-fee = ["revm/optional_no_base_fee"] +optional-checks = [ + "optional-balance-check", + "optional-block-gas-limit", + "optional-eip3541", + "optional-eip3607", + "optional-no-base-fee", +] +memory_limit = ["revm/memory_limit"] diff --git a/crates/revm/src/cached.rs b/crates/revm/src/cached.rs index bf4bd6d5d1b..d40e814c12a 100644 --- a/crates/revm/src/cached.rs +++ b/crates/revm/src/cached.rs @@ -146,11 +146,11 @@ impl Database for CachedReadsDbMut<'_, DB> { } fn block_hash(&mut self, number: u64) -> Result { - let code = match self.cached.block_hashes.entry(number) { + let hash = match self.cached.block_hashes.entry(number) { Entry::Occupied(entry) => *entry.get(), Entry::Vacant(entry) => *entry.insert(self.db.block_hash_ref(number)?), }; - Ok(code) + Ok(hash) } } diff --git a/crates/revm/src/cancelled.rs b/crates/revm/src/cancelled.rs index b692d2db7bb..e535882355c 100644 --- a/crates/revm/src/cancelled.rs +++ b/crates/revm/src/cancelled.rs @@ -108,4 +108,40 @@ mod tests { c.cancel(); assert!(cloned_cancel.is_cancelled()); } + + #[test] + fn test_cancelondrop_clone_behavior() { + let cancel = CancelOnDrop::default(); + assert!(!cancel.is_cancelled()); + + // Clone the CancelOnDrop + let cloned_cancel = cancel.clone(); + assert!(!cloned_cancel.is_cancelled()); + + // Drop the original - this should set the cancelled flag + drop(cancel); + + // The cloned instance should now see the cancelled flag as true + assert!(cloned_cancel.is_cancelled()); + } + + #[test] + fn test_cancelondrop_multiple_clones() { + let cancel = CancelOnDrop::default(); + let clone1 = cancel.clone(); + let clone2 = cancel.clone(); + let clone3 = cancel.clone(); + + assert!(!cancel.is_cancelled()); + assert!(!clone1.is_cancelled()); + assert!(!clone2.is_cancelled()); + assert!(!clone3.is_cancelled()); + + // Drop one clone - this should cancel all instances + drop(clone1); + + assert!(cancel.is_cancelled()); + assert!(clone2.is_cancelled()); + assert!(clone3.is_cancelled()); + } } diff --git a/crates/revm/src/database.rs b/crates/revm/src/database.rs index fafe990c3b1..6b829c3d734 100644 --- a/crates/revm/src/database.rs +++ b/crates/revm/src/database.rs @@ -2,7 +2,7 @@ use crate::primitives::alloy_primitives::{BlockNumber, StorageKey, StorageValue} use alloy_primitives::{Address, B256, U256}; use core::ops::{Deref, DerefMut}; use reth_primitives_traits::Account; -use reth_storage_api::{AccountReader, BlockHashReader, StateProvider}; +use reth_storage_api::{AccountReader, BlockHashReader, BytecodeReader, StateProvider}; use reth_storage_errors::provider::{ProviderError, ProviderResult}; use revm::{bytecode::Bytecode, state::AccountInfo, Database, DatabaseRef}; @@ -47,7 +47,7 @@ impl EvmStateProvider for T { &self, code_hash: &B256, ) -> ProviderResult> { - ::bytecode_by_hash(self, code_hash) + ::bytecode_by_hash(self, code_hash) } fn storage( @@ -61,7 +61,7 @@ impl EvmStateProvider for T { /// A [Database] and [`DatabaseRef`] implementation that uses [`EvmStateProvider`] as the underlying /// data source. -#[derive(Debug, Clone)] +#[derive(Clone)] pub struct StateProviderDatabase(pub DB); impl StateProviderDatabase { @@ -76,6 +76,12 @@ impl StateProviderDatabase { } } +impl core::fmt::Debug for StateProviderDatabase { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("StateProviderDatabase").finish_non_exhaustive() + } +} + impl AsRef for StateProviderDatabase { fn as_ref(&self) -> &DB { self diff --git a/crates/revm/src/either.rs b/crates/revm/src/either.rs deleted file mode 100644 index e26d2ccb721..00000000000 --- a/crates/revm/src/either.rs +++ /dev/null @@ -1,49 +0,0 @@ -use alloy_primitives::{Address, B256, U256}; -use revm::{bytecode::Bytecode, state::AccountInfo, Database}; - -/// An enum type that can hold either of two different [`Database`] implementations. -/// -/// This allows flexible usage of different [`Database`] types in the same context. -#[derive(Debug, Clone)] -pub enum Either { - /// A value of type `L`. - Left(L), - /// A value of type `R`. - Right(R), -} - -impl Database for Either -where - L: Database, - R: Database, -{ - type Error = L::Error; - - fn basic(&mut self, address: Address) -> Result, Self::Error> { - match self { - Self::Left(db) => db.basic(address), - Self::Right(db) => db.basic(address), - } - } - - fn code_by_hash(&mut self, code_hash: B256) -> Result { - match self { - Self::Left(db) => db.code_by_hash(code_hash), - Self::Right(db) => db.code_by_hash(code_hash), - } - } - - fn storage(&mut self, address: Address, index: U256) -> Result { - match self { - Self::Left(db) => db.storage(address, index), - Self::Right(db) => db.storage(address, index), - } - } - - fn block_hash(&mut self, number: u64) -> Result { - match self { - Self::Left(db) => db.block_hash(number), - Self::Right(db) => db.block_hash(number), - } - } -} diff --git a/crates/revm/src/lib.rs b/crates/revm/src/lib.rs index caaae237c8a..acf7548304b 100644 --- a/crates/revm/src/lib.rs +++ b/crates/revm/src/lib.rs @@ -6,7 +6,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(feature = "std"), no_std)] extern crate alloc; @@ -30,9 +30,6 @@ pub mod test_utils; // Convenience re-exports. pub use revm::{self, database::State, *}; -/// Either type for flexible usage of different database types in the same context. -pub mod either; - /// Helper types for execution witness generation. #[cfg(feature = "witness")] pub mod witness; diff --git a/crates/revm/src/test_utils.rs b/crates/revm/src/test_utils.rs index d32f7a9e7a7..e0d40070878 100644 --- a/crates/revm/src/test_utils.rs +++ b/crates/revm/src/test_utils.rs @@ -4,8 +4,8 @@ use alloy_primitives::{ }; use reth_primitives_traits::{Account, Bytecode}; use reth_storage_api::{ - AccountReader, BlockHashReader, HashedPostStateProvider, StateProofProvider, StateProvider, - StateRootProvider, StorageRootProvider, + AccountReader, BlockHashReader, BytecodeReader, HashedPostStateProvider, StateProofProvider, + StateProvider, StateRootProvider, StorageRootProvider, }; use reth_storage_errors::provider::ProviderResult; use reth_trie::{ @@ -158,7 +158,9 @@ impl StateProvider for StateProviderTest { ) -> ProviderResult> { Ok(self.accounts.get(&account).and_then(|(storage, _)| storage.get(&storage_key).copied())) } +} +impl BytecodeReader for StateProviderTest { fn bytecode_by_hash(&self, code_hash: &B256) -> ProviderResult> { Ok(self.contracts.get(code_hash).cloned()) } diff --git a/crates/rpc/ipc/Cargo.toml b/crates/rpc/ipc/Cargo.toml index d1284af0a5b..58639f2c44e 100644 --- a/crates/rpc/ipc/Cargo.toml +++ b/crates/rpc/ipc/Cargo.toml @@ -17,7 +17,6 @@ futures.workspace = true tokio = { workspace = true, features = ["net", "time", "rt-multi-thread"] } tokio-util = { workspace = true, features = ["codec"] } tokio-stream.workspace = true -async-trait.workspace = true pin-project.workspace = true tower.workspace = true diff --git a/crates/rpc/ipc/src/client/mod.rs b/crates/rpc/ipc/src/client/mod.rs index d0b27adaaf0..f6138dd2e17 100644 --- a/crates/rpc/ipc/src/client/mod.rs +++ b/crates/rpc/ipc/src/client/mod.rs @@ -20,7 +20,6 @@ pub(crate) struct Sender { inner: SendHalf, } -#[async_trait::async_trait] impl TransportSenderT for Sender { type Error = IpcError; @@ -47,7 +46,6 @@ pub(crate) struct Receiver { pub(crate) inner: FramedRead, } -#[async_trait::async_trait] impl TransportReceiverT for Receiver { type Error = IpcError; diff --git a/crates/rpc/ipc/src/lib.rs b/crates/rpc/ipc/src/lib.rs index ae7a8b221f2..bde2196a318 100644 --- a/crates/rpc/ipc/src/lib.rs +++ b/crates/rpc/ipc/src/lib.rs @@ -10,7 +10,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] pub mod client; pub mod server; diff --git a/crates/rpc/ipc/src/server/connection.rs b/crates/rpc/ipc/src/server/connection.rs index aaf6731d045..e8b827078ba 100644 --- a/crates/rpc/ipc/src/server/connection.rs +++ b/crates/rpc/ipc/src/server/connection.rs @@ -1,4 +1,4 @@ -//! A IPC connection. +//! An IPC connection. use crate::stream_codec::StreamCodec; use futures::{stream::FuturesUnordered, FutureExt, Sink, Stream}; @@ -54,7 +54,7 @@ where } } -/// Drives an [IpcConn] forward. +/// Drives an [`IpcConn`] forward. /// /// This forwards received requests from the connection to the service and sends responses to the /// connection. diff --git a/crates/rpc/ipc/src/server/ipc.rs b/crates/rpc/ipc/src/server/ipc.rs index 33ed8d2d553..fda19c7cb31 100644 --- a/crates/rpc/ipc/src/server/ipc.rs +++ b/crates/rpc/ipc/src/server/ipc.rs @@ -3,17 +3,13 @@ use futures::{stream::FuturesOrdered, StreamExt}; use jsonrpsee::{ batch_response_error, - core::{ - server::helpers::prepare_error, - tracing::server::{rx_log_from_json, tx_log_from_str}, - JsonRawValue, - }, + core::{server::helpers::prepare_error, JsonRawValue}, server::middleware::rpc::RpcServiceT, types::{ error::{reject_too_big_request, ErrorCode}, ErrorObject, Id, InvalidRequest, Notification, Request, }, - BatchResponseBuilder, MethodResponse, ResponsePayload, + BatchResponseBuilder, MethodResponse, }; use std::sync::Arc; use tokio::sync::OwnedSemaphorePermit; @@ -31,13 +27,13 @@ pub(crate) struct Batch { // Batch responses must be sent back as a single message so we read the results from each // request in the batch and read the results off of a new channel, `rx_batch`, and then send the // complete batch response back to the client over `tx`. -#[instrument(name = "batch", skip(b), level = "TRACE")] +#[instrument(name = "batch", skip(b))] pub(crate) async fn process_batch_request( b: Batch, max_response_body_size: usize, ) -> Option where - for<'a> S: RpcServiceT<'a> + Send, + S: RpcServiceT + Send, { let Batch { data, rpc_service } = b; @@ -69,8 +65,8 @@ where .collect(); while let Some(response) = pending_calls.next().await { - if let Err(too_large) = batch_response.append(&response) { - return Some(too_large.to_result()) + if let Err(too_large) = batch_response.append(response) { + return Some(too_large.to_json().to_string()) } } @@ -78,10 +74,10 @@ where None } else { let batch_resp = batch_response.finish(); - Some(MethodResponse::from_batch(batch_resp).to_result()) + Some(MethodResponse::from_batch(batch_resp).to_json().to_string()) } } else { - Some(batch_response_error(Id::Null, ErrorObject::from(ErrorCode::ParseError))) + Some(batch_response_error(Id::Null, ErrorObject::from(ErrorCode::ParseError)).to_string()) } } @@ -90,7 +86,7 @@ pub(crate) async fn process_single_request( rpc_service: &S, ) -> Option where - for<'a> S: RpcServiceT<'a> + Send, + S: RpcServiceT + Send, { if let Ok(req) = serde_json::from_slice::>(&data) { Some(execute_call_with_tracing(req, rpc_service).await) @@ -102,26 +98,17 @@ where } } -#[instrument(name = "method_call", fields(method = req.method.as_ref()), skip(req, rpc_service), level = "TRACE")] +#[instrument(name = "method_call", fields(method = req.method.as_ref()), skip(req, rpc_service))] pub(crate) async fn execute_call_with_tracing<'a, S>( req: Request<'a>, rpc_service: &S, ) -> MethodResponse where - for<'b> S: RpcServiceT<'b> + Send, + S: RpcServiceT + Send, { rpc_service.call(req).await } -#[instrument(name = "notification", fields(method = notif.method.as_ref()), skip(notif, max_log_length), level = "TRACE")] -fn execute_notification(notif: &Notif<'_>, max_log_length: u32) -> MethodResponse { - rx_log_from_json(notif, max_log_length); - let response = - MethodResponse::response(Id::Null, ResponsePayload::success(String::new()), usize::MAX); - tx_log_from_str(response.as_result(), max_log_length); - response -} - pub(crate) async fn call_with_service( request: String, rpc_service: S, @@ -130,7 +117,7 @@ pub(crate) async fn call_with_service( conn: Arc, ) -> Option where - for<'a> S: RpcServiceT<'a> + Send, + S: RpcServiceT + Send, { enum Kind { Single, @@ -148,17 +135,17 @@ where let data = request.into_bytes(); if data.len() > max_request_body_size { - return Some(batch_response_error( - Id::Null, - reject_too_big_request(max_request_body_size as u32), - )) + return Some( + batch_response_error(Id::Null, reject_too_big_request(max_request_body_size as u32)) + .to_string(), + ) } // Single request or notification let res = if matches!(request_kind, Kind::Single) { let response = process_single_request(data, &rpc_service).await; match response { - Some(response) if response.is_method_call() => Some(response.to_result()), + Some(response) if response.is_method_call() => Some(response.to_json().to_string()), _ => { // subscription responses are sent directly over the sink, return a response here // would lead to duplicate responses for the subscription response diff --git a/crates/rpc/ipc/src/server/mod.rs b/crates/rpc/ipc/src/server/mod.rs index 3fee58d3a15..75431b915a5 100644 --- a/crates/rpc/ipc/src/server/mod.rs +++ b/crates/rpc/ipc/src/server/mod.rs @@ -9,13 +9,12 @@ use interprocess::local_socket::{ GenericFilePath, ListenerOptions, ToFsName, }; use jsonrpsee::{ - core::TEN_MB_SIZE_BYTES, + core::{middleware::layer::RpcLoggerLayer, JsonRawValue, TEN_MB_SIZE_BYTES}, server::{ - middleware::rpc::{RpcLoggerLayer, RpcServiceT}, - stop_channel, ConnectionGuard, ConnectionPermit, IdProvider, RandomIntegerIdProvider, - ServerHandle, StopHandle, + middleware::rpc::RpcServiceT, stop_channel, ConnectionGuard, ConnectionPermit, IdProvider, + RandomIntegerIdProvider, ServerHandle, StopHandle, }, - BoundedSubscriptions, MethodSink, Methods, + BoundedSubscriptions, MethodResponse, MethodSink, Methods, }; use std::{ future::Future, @@ -66,7 +65,7 @@ impl IpcServer { impl IpcServer where - RpcMiddleware: for<'a> Layer> + Clone + Send + 'static, + RpcMiddleware: for<'a> Layer + Clone + Send + 'static, HttpMiddleware: Layer< TowerServiceNoHttp, Service: Service< @@ -140,7 +139,20 @@ where .to_fs_name::() .and_then(|name| ListenerOptions::new().name(name).create_tokio()) { - Ok(listener) => listener, + Ok(listener) => { + #[cfg(unix)] + { + // set permissions only on unix + use std::os::unix::fs::PermissionsExt; + if let Some(perms_str) = &self.cfg.ipc_socket_permissions && + let Ok(mode) = u32::from_str_radix(&perms_str.replace("0o", ""), 8) + { + let perms = std::fs::Permissions::from_mode(mode); + let _ = std::fs::set_permissions(&self.endpoint, perms); + } + } + listener + } Err(err) => { on_ready .send(Err(IpcServerStartError { endpoint: self.endpoint.clone(), source: err })) @@ -257,7 +269,7 @@ pub struct IpcServerStartError { /// Data required by the server to handle requests received via an IPC connection #[derive(Debug, Clone)] -#[expect(dead_code)] +#[allow(dead_code)] pub(crate) struct ServiceData { /// Registered server methods. pub(crate) methods: Methods, @@ -292,7 +304,7 @@ impl Default for RpcServiceBuilder { impl RpcServiceBuilder { /// Create a new [`RpcServiceBuilder`]. - pub fn new() -> Self { + pub const fn new() -> Self { Self(tower::ServiceBuilder::new()) } } @@ -357,7 +369,8 @@ pub struct TowerServiceNoHttp { impl Service for TowerServiceNoHttp where RpcMiddleware: for<'a> Layer, - for<'a> >::Service: Send + Sync + 'static + RpcServiceT<'a>, + for<'a> >::Service: + Send + Sync + 'static + RpcServiceT, { /// The response of a handled RPC call /// @@ -378,7 +391,7 @@ where fn call(&mut self, request: String) -> Self::Future { trace!("{:?}", request); - let cfg = RpcServiceCfg::CallsAndSubscriptions { + let cfg = RpcServiceCfg { bounded_subscriptions: BoundedSubscriptions::new( self.inner.server_cfg.max_subscriptions_per_connection, ), @@ -430,12 +443,12 @@ struct ProcessConnection<'a, HttpMiddleware, RpcMiddleware> { } /// Spawns the IPC connection onto a new task -#[instrument(name = "connection", skip_all, fields(conn_id = %params.conn_id), level = "INFO")] -fn process_connection<'b, RpcMiddleware, HttpMiddleware>( +#[instrument(name = "connection", skip_all, fields(conn_id = %params.conn_id))] +fn process_connection( params: ProcessConnection<'_, HttpMiddleware, RpcMiddleware>, ) where RpcMiddleware: Layer + Clone + Send + 'static, - for<'a> >::Service: RpcServiceT<'a>, + for<'a> >::Service: RpcServiceT, HttpMiddleware: Layer> + Send + 'static, >>::Service: Send + Service< @@ -464,7 +477,7 @@ fn process_connection<'b, RpcMiddleware, HttpMiddleware>( local_socket_stream, )); - let (tx, rx) = mpsc::channel::(server_cfg.message_buffer_capacity as usize); + let (tx, rx) = mpsc::channel::>(server_cfg.message_buffer_capacity as usize); let method_sink = MethodSink::new_with_limit(tx, server_cfg.max_response_body_size); let tower_service = TowerServiceNoHttp { inner: ServiceData { @@ -493,7 +506,7 @@ async fn to_ipc_service( ipc: IpcConn>, service: S, stop_handle: StopHandle, - rx: mpsc::Receiver, + rx: mpsc::Receiver>, ) where S: Service> + Send + 'static, S::Error: Into>, @@ -520,7 +533,7 @@ async fn to_ipc_service( } item = rx_item.next() => { if let Some(item) = item { - conn.push_back(item); + conn.push_back(item.to_string()); } } _ = &mut stopped => { @@ -550,6 +563,8 @@ pub struct Settings { message_buffer_capacity: u32, /// Custom tokio runtime to run the server on. tokio_runtime: Option, + /// The permissions to create the IPC socket with. + ipc_socket_permissions: Option, } impl Default for Settings { @@ -562,6 +577,7 @@ impl Default for Settings { max_subscriptions_per_connection: 1024, message_buffer_capacity: 1024, tokio_runtime: None, + ipc_socket_permissions: None, } } } @@ -612,7 +628,7 @@ impl Builder { self } - /// Set the maximum number of connections allowed. Default is 1024. + /// Set the maximum number of subscriptions per connection. Default is 1024. pub const fn max_subscriptions_per_connection(mut self, max: u32) -> Self { self.settings.max_subscriptions_per_connection = max; self @@ -620,7 +636,7 @@ impl Builder { /// The server enforces backpressure which means that /// `n` messages can be buffered and if the client - /// can't keep with up the server. + /// can't keep up with the server. /// /// This `capacity` is applied per connection and /// applies globally on the connection which implies @@ -648,6 +664,12 @@ impl Builder { self } + /// Sets the permissions for the IPC socket file. + pub fn set_ipc_socket_permissions(mut self, permissions: Option) -> Self { + self.settings.ipc_socket_permissions = permissions; + self + } + /// Configure custom `subscription ID` provider for the server to use /// to when getting new subscription calls. /// @@ -712,59 +734,6 @@ impl Builder { /// /// The builder itself exposes a similar API as the [`tower::ServiceBuilder`] /// where it is possible to compose layers to the middleware. - /// - /// ``` - /// use std::{ - /// net::SocketAddr, - /// sync::{ - /// atomic::{AtomicUsize, Ordering}, - /// Arc, - /// }, - /// time::Instant, - /// }; - /// - /// use futures_util::future::BoxFuture; - /// use jsonrpsee::{ - /// server::{middleware::rpc::RpcServiceT, ServerBuilder}, - /// types::Request, - /// MethodResponse, - /// }; - /// use reth_ipc::server::{Builder, RpcServiceBuilder}; - /// - /// #[derive(Clone)] - /// struct MyMiddleware { - /// service: S, - /// count: Arc, - /// } - /// - /// impl<'a, S> RpcServiceT<'a> for MyMiddleware - /// where - /// S: RpcServiceT<'a> + Send + Sync + Clone + 'static, - /// { - /// type Future = BoxFuture<'a, MethodResponse>; - /// - /// fn call(&self, req: Request<'a>) -> Self::Future { - /// tracing::info!("MyMiddleware processed call {}", req.method); - /// let count = self.count.clone(); - /// let service = self.service.clone(); - /// - /// Box::pin(async move { - /// let rp = service.call(req).await; - /// // Modify the state. - /// count.fetch_add(1, Ordering::Relaxed); - /// rp - /// }) - /// } - /// } - /// - /// // Create a state per connection - /// // NOTE: The service type can be omitted once `start` is called on the server. - /// let m = RpcServiceBuilder::new().layer_fn(move |service: ()| MyMiddleware { - /// service, - /// count: Arc::new(AtomicUsize::new(0)), - /// }); - /// let builder = Builder::default().set_rpc_middleware(m); - /// ``` pub fn set_rpc_middleware( self, rpc_middleware: RpcServiceBuilder, @@ -808,8 +777,8 @@ mod tests { use futures::future::select; use jsonrpsee::{ core::{ - client, - client::{ClientT, Error, Subscription, SubscriptionClientT}, + client::{self, ClientT, Error, Subscription, SubscriptionClientT}, + middleware::{Batch, BatchEntry, Notification}, params::BatchRequestBuilder, }, rpc_params, @@ -821,6 +790,24 @@ mod tests { use tokio::sync::broadcast; use tokio_stream::wrappers::BroadcastStream; + #[tokio::test] + #[cfg(unix)] + async fn test_ipc_socket_permissions() { + use std::os::unix::fs::PermissionsExt; + let endpoint = &dummy_name(); + let perms = "0777"; + let server = Builder::default() + .set_ipc_socket_permissions(Some(perms.to_string())) + .build(endpoint.clone()); + let module = RpcModule::new(()); + let handle = server.start(module).await.unwrap(); + tokio::spawn(handle.stopped()); + + let meta = std::fs::metadata(endpoint).unwrap(); + let perms = meta.permissions(); + assert_eq!(perms.mode() & 0o777, 0o777); + } + async fn pipe_from_stream_with_bounded_buffer( pending: PendingSubscriptionSink, stream: BroadcastStream, @@ -838,7 +825,8 @@ mod tests { // received new item from the stream. Either::Right((Some(Ok(item)), c)) => { - let notif = SubscriptionMessage::from_json(&item)?; + let raw_value = serde_json::value::to_raw_value(&item)?; + let notif = SubscriptionMessage::from(raw_value); // NOTE: this will block until there a spot in the queue // and you might want to do something smarter if it's @@ -1035,13 +1023,18 @@ mod tests { #[derive(Clone)] struct ModifyRequestIf(S); - impl<'a, S> RpcServiceT<'a> for ModifyRequestIf + impl RpcServiceT for ModifyRequestIf where - S: Send + Sync + RpcServiceT<'a>, + S: Send + Sync + RpcServiceT, { - type Future = S::Future; - - fn call(&self, mut req: Request<'a>) -> Self::Future { + type MethodResponse = S::MethodResponse; + type NotificationResponse = S::NotificationResponse; + type BatchResponse = S::BatchResponse; + + fn call<'a>( + &self, + mut req: Request<'a>, + ) -> impl Future + Send + 'a { // Re-direct all calls that isn't `say_hello` to `say_goodbye` if req.method == "say_hello" { req.method = "say_goodbye".into(); @@ -1051,6 +1044,46 @@ mod tests { self.0.call(req) } + + fn batch<'a>( + &self, + mut batch: Batch<'a>, + ) -> impl Future + Send + 'a { + for call in batch.iter_mut() { + match call { + Ok(BatchEntry::Call(req)) => { + if req.method == "say_hello" { + req.method = "say_goodbye".into(); + } else if req.method == "say_goodbye" { + req.method = "say_hello".into(); + } + } + Ok(BatchEntry::Notification(n)) => { + if n.method == "say_hello" { + n.method = "say_goodbye".into(); + } else if n.method == "say_goodbye" { + n.method = "say_hello".into(); + } + } + // Invalid request, we don't care about it. + Err(_err) => {} + } + } + + self.0.batch(batch) + } + + fn notification<'a>( + &self, + mut n: Notification<'a>, + ) -> impl Future + Send + 'a { + if n.method == "say_hello" { + n.method = "say_goodbye".into(); + } else if n.method == "say_goodbye" { + n.method = "say_hello".into(); + } + self.0.notification(n) + } } reth_tracing::init_test_tracing(); diff --git a/crates/rpc/ipc/src/server/rpc_service.rs b/crates/rpc/ipc/src/server/rpc_service.rs index c8ea7d916bc..f7fcdace4c4 100644 --- a/crates/rpc/ipc/src/server/rpc_service.rs +++ b/crates/rpc/ipc/src/server/rpc_service.rs @@ -1,15 +1,19 @@ //! JSON-RPC service middleware. -use futures_util::future::BoxFuture; +use futures::{ + future::Either, + stream::{FuturesOrdered, StreamExt}, +}; use jsonrpsee::{ + core::middleware::{Batch, BatchEntry}, server::{ middleware::rpc::{ResponseFuture, RpcServiceT}, IdProvider, }, - types::{error::reject_too_many_subscriptions, ErrorCode, ErrorObject, Request}, - BoundedSubscriptions, ConnectionId, MethodCallback, MethodResponse, MethodSink, Methods, - SubscriptionState, + types::{error::reject_too_many_subscriptions, ErrorCode, ErrorObject, Id, Request}, + BatchResponse, BatchResponseBuilder, BoundedSubscriptions, ConnectionId, MethodCallback, + MethodResponse, MethodSink, Methods, SubscriptionState, }; -use std::sync::Arc; +use std::{future::Future, sync::Arc}; /// JSON-RPC service middleware. #[derive(Clone, Debug)] @@ -21,17 +25,11 @@ pub struct RpcService { } /// Configuration of the `RpcService`. -#[expect(dead_code)] #[derive(Clone, Debug)] -pub(crate) enum RpcServiceCfg { - /// The server supports only calls. - OnlyCalls, - /// The server supports both method calls and subscriptions. - CallsAndSubscriptions { - bounded_subscriptions: BoundedSubscriptions, - sink: MethodSink, - id_provider: Arc, - }, +pub(crate) struct RpcServiceCfg { + pub(crate) bounded_subscriptions: BoundedSubscriptions, + pub(crate) sink: MethodSink, + pub(crate) id_provider: Arc, } impl RpcService { @@ -46,12 +44,12 @@ impl RpcService { } } -impl<'a> RpcServiceT<'a> for RpcService { - // The rpc module is already boxing the futures and - // it's used to under the hood by the RpcService. - type Future = ResponseFuture>; +impl RpcServiceT for RpcService { + type MethodResponse = MethodResponse; + type NotificationResponse = Option; + type BatchResponse = BatchResponse; - fn call(&self, req: Request<'a>) -> Self::Future { + fn call<'a>(&self, req: Request<'a>) -> impl Future + Send + 'a { let conn_id = self.conn_id; let max_response_body_size = self.max_response_body_size; @@ -78,30 +76,20 @@ impl<'a> RpcServiceT<'a> for RpcService { ResponseFuture::future(fut) } MethodCallback::Subscription(callback) => { - let RpcServiceCfg::CallsAndSubscriptions { - bounded_subscriptions, - sink, - id_provider, - } = &self.cfg - else { - tracing::warn!("Subscriptions not supported"); - let rp = - MethodResponse::error(id, ErrorObject::from(ErrorCode::InternalError)); - return ResponseFuture::ready(rp); - }; - - if let Some(p) = bounded_subscriptions.acquire() { + let cfg = &self.cfg; + + if let Some(p) = cfg.bounded_subscriptions.acquire() { let conn_state = SubscriptionState { conn_id, - id_provider: &**id_provider, + id_provider: &*cfg.id_provider, subscription_permit: p, }; let fut = - callback(id.clone(), params, sink.clone(), conn_state, extensions); + callback(id.clone(), params, cfg.sink.clone(), conn_state, extensions); ResponseFuture::future(fut) } else { - let max = bounded_subscriptions.max(); + let max = cfg.bounded_subscriptions.max(); let rp = MethodResponse::error(id, reject_too_many_subscriptions(max)); ResponseFuture::ready(rp) } @@ -110,17 +98,50 @@ impl<'a> RpcServiceT<'a> for RpcService { // Don't adhere to any resource or subscription limits; always let unsubscribing // happen! - let RpcServiceCfg::CallsAndSubscriptions { .. } = self.cfg else { - tracing::warn!("Subscriptions not supported"); - let rp = - MethodResponse::error(id, ErrorObject::from(ErrorCode::InternalError)); - return ResponseFuture::ready(rp); - }; - let rp = callback(id, params, conn_id, max_response_body_size, extensions); ResponseFuture::ready(rp) } }, } } + + fn batch<'a>(&self, req: Batch<'a>) -> impl Future + Send + 'a { + let entries: Vec<_> = req.into_iter().collect(); + + let mut got_notif = false; + let mut batch_response = BatchResponseBuilder::new_with_limit(self.max_response_body_size); + + let mut pending_calls: FuturesOrdered<_> = entries + .into_iter() + .filter_map(|v| match v { + Ok(BatchEntry::Call(call)) => Some(Either::Right(self.call(call))), + Ok(BatchEntry::Notification(_n)) => { + got_notif = true; + None + } + Err(_err) => Some(Either::Left(async { + MethodResponse::error(Id::Null, ErrorObject::from(ErrorCode::InvalidRequest)) + })), + }) + .collect(); + async move { + while let Some(response) = pending_calls.next().await { + if let Err(too_large) = batch_response.append(response) { + let mut error_batch = BatchResponseBuilder::new_with_limit(1); + let _ = error_batch.append(too_large); + return error_batch.finish(); + } + } + + batch_response.finish() + } + } + + #[allow(clippy::manual_async_fn)] + fn notification<'a>( + &self, + _n: jsonrpsee::core::middleware::Notification<'a>, + ) -> impl Future + Send + 'a { + async move { None } + } } diff --git a/crates/rpc/ipc/src/stream_codec.rs b/crates/rpc/ipc/src/stream_codec.rs index 4205081e3de..aa5cda16b7f 100644 --- a/crates/rpc/ipc/src/stream_codec.rs +++ b/crates/rpc/ipc/src/stream_codec.rs @@ -209,7 +209,7 @@ mod tests { let request2 = codec .decode(&mut buf) .expect("There should be no error in first 2nd test") - .expect("There should be aa request in 2nd whitespace test"); + .expect("There should be a request in 2nd whitespace test"); // TODO: maybe actually trim it out assert_eq!(request2, "\n\n\n\n{ test: 2 }"); diff --git a/crates/rpc/rpc-api/Cargo.toml b/crates/rpc/rpc-api/Cargo.toml index 4eb9a6e653c..7d170d342f2 100644 --- a/crates/rpc/rpc-api/Cargo.toml +++ b/crates/rpc/rpc-api/Cargo.toml @@ -16,6 +16,8 @@ workspace = true reth-rpc-eth-api.workspace = true reth-engine-primitives.workspace = true reth-network-peers.workspace = true +reth-trie-common.workspace = true +reth-chain-state.workspace = true # ethereum alloy-eips.workspace = true diff --git a/crates/rpc/rpc-api/src/admin.rs b/crates/rpc/rpc-api/src/admin.rs index e6484937783..2c6de0bcd1b 100644 --- a/crates/rpc/rpc-api/src/admin.rs +++ b/crates/rpc/rpc-api/src/admin.rs @@ -45,4 +45,9 @@ pub trait AdminApi { /// Returns the ENR of the node. #[method(name = "nodeInfo")] async fn node_info(&self) -> RpcResult; + + /// Clears all transactions from the transaction pool. + /// Returns the number of transactions that were removed from the pool. + #[method(name = "clearTxpool")] + async fn clear_txpool(&self) -> RpcResult; } diff --git a/crates/rpc/rpc-api/src/debug.rs b/crates/rpc/rpc-api/src/debug.rs index c3246c333b4..0fca5f18457 100644 --- a/crates/rpc/rpc-api/src/debug.rs +++ b/crates/rpc/rpc-api/src/debug.rs @@ -1,17 +1,19 @@ use alloy_eips::{BlockId, BlockNumberOrTag}; use alloy_genesis::ChainConfig; +use alloy_json_rpc::RpcObject; use alloy_primitives::{Address, Bytes, B256}; use alloy_rpc_types_debug::ExecutionWitness; -use alloy_rpc_types_eth::{transaction::TransactionRequest, Block, Bundle, StateContext}; +use alloy_rpc_types_eth::{Block, Bundle, StateContext}; use alloy_rpc_types_trace::geth::{ BlockTraceResult, GethDebugTracingCallOptions, GethDebugTracingOptions, GethTrace, TraceResult, }; use jsonrpsee::{core::RpcResult, proc_macros::rpc}; +use reth_trie_common::{updates::TrieUpdates, HashedPostState}; /// Debug rpc interface. #[cfg_attr(not(feature = "client"), rpc(server, namespace = "debug"))] #[cfg_attr(feature = "client", rpc(server, client, namespace = "debug"))] -pub trait DebugApi { +pub trait DebugApi { /// Returns an RLP-encoded header. #[method(name = "getRawHeader")] async fn raw_header(&self, block_id: BlockId) -> RpcResult; @@ -53,7 +55,7 @@ pub trait DebugApi { /// This expects an rlp encoded block /// /// Note, the parent of this block must be present, or it will fail. For the second parameter - /// see [GethDebugTracingOptions] reference. + /// see [`GethDebugTracingOptions`] reference. #[method(name = "traceBlock")] async fn debug_trace_block( &self, @@ -63,7 +65,7 @@ pub trait DebugApi { /// Similar to `debug_traceBlock`, `debug_traceBlockByHash` accepts a block hash and will replay /// the block that is already present in the database. For the second parameter see - /// [GethDebugTracingOptions]. + /// [`GethDebugTracingOptions`]. #[method(name = "traceBlockByHash")] async fn debug_trace_block_by_hash( &self, @@ -72,8 +74,8 @@ pub trait DebugApi { ) -> RpcResult>; /// Similar to `debug_traceBlockByHash`, `debug_traceBlockByNumber` accepts a block number - /// [BlockNumberOrTag] and will replay the block that is already present in the database. - /// For the second parameter see [GethDebugTracingOptions]. + /// [`BlockNumberOrTag`] and will replay the block that is already present in the database. + /// For the second parameter see [`GethDebugTracingOptions`]. #[method(name = "traceBlockByNumber")] async fn debug_trace_block_by_number( &self, @@ -99,12 +101,12 @@ pub trait DebugApi { /// The block can optionally be specified either by hash or by number as /// the second argument. /// The trace can be configured similar to `debug_traceTransaction`, - /// see [GethDebugTracingOptions]. The method returns the same output as + /// see [`GethDebugTracingOptions`]. The method returns the same output as /// `debug_traceTransaction`. #[method(name = "traceCall")] async fn debug_trace_call( &self, - request: TransactionRequest, + request: TxReq, block_id: Option, opts: Option, ) -> RpcResult; @@ -115,7 +117,7 @@ pub trait DebugApi { /// /// The first argument is a list of bundles. Each bundle can overwrite the block headers. This /// will affect all transaction in that bundle. - /// BlockNumber and transaction_index are optional. Transaction_index + /// `BlockNumber` and `transaction_index` are optional. `Transaction_index` /// specifies the number of tx in the block to replay and -1 means all transactions should be /// replayed. /// The trace can be configured similar to `debug_traceTransaction`. @@ -127,7 +129,7 @@ pub trait DebugApi { #[method(name = "traceCallMany")] async fn debug_trace_call_many( &self, - bundles: Vec, + bundles: Vec>, state_context: Option, opts: Option, ) -> RpcResult>>; @@ -220,7 +222,7 @@ pub trait DebugApi { /// Returns the raw value of a key stored in the database. #[method(name = "dbGet")] - async fn debug_db_get(&self, key: String) -> RpcResult<()>; + async fn debug_db_get(&self, key: String) -> RpcResult>; /// Retrieves the state that corresponds to the block number and returns a list of accounts /// (including storage and code). @@ -359,6 +361,15 @@ pub trait DebugApi { #[method(name = "startGoTrace")] async fn debug_start_go_trace(&self, file: String) -> RpcResult<()>; + /// Returns the state root of the `HashedPostState` on top of the state for the given block with + /// trie updates. + #[method(name = "stateRootWithUpdates")] + async fn debug_state_root_with_updates( + &self, + hashed_state: HashedPostState, + block_id: Option, + ) -> RpcResult<(B256, TrieUpdates)>; + /// Stops an ongoing CPU profile. #[method(name = "stopCPUProfile")] async fn debug_stop_cpu_profile(&self) -> RpcResult<()>; @@ -382,7 +393,7 @@ pub trait DebugApi { /// Returns the structured logs created during the execution of EVM against a block pulled /// from the pool of bad ones and returns them as a JSON object. For the second parameter see - /// TraceConfig reference. + /// `TraceConfig` reference. #[method(name = "traceBadBlock")] async fn debug_trace_bad_block( &self, diff --git a/crates/rpc/rpc-api/src/engine.rs b/crates/rpc/rpc-api/src/engine.rs index 1687aec7ce2..bf097eec2f7 100644 --- a/crates/rpc/rpc-api/src/engine.rs +++ b/crates/rpc/rpc-api/src/engine.rs @@ -15,8 +15,7 @@ use alloy_rpc_types_engine::{ ExecutionPayloadV3, ForkchoiceState, ForkchoiceUpdated, PayloadId, PayloadStatus, }; use alloy_rpc_types_eth::{ - state::StateOverride, transaction::TransactionRequest, BlockOverrides, - EIP1186AccountProofResponse, Filter, Log, SyncStatus, + state::StateOverride, BlockOverrides, EIP1186AccountProofResponse, Filter, Log, SyncStatus, }; use alloy_serde::JsonStorageKey; use jsonrpsee::{core::RpcResult, proc_macros::rpc, RpcModule}; @@ -166,6 +165,19 @@ pub trait EngineApi { payload_id: PayloadId, ) -> RpcResult; + /// Post Osaka payload handler. + /// + /// See also . + /// + /// Returns the most recent version of the payload that is available in the corresponding + /// payload build process at the time of receiving this call. Note: + /// > Provider software MAY stop the corresponding build process after serving this call. + #[method(name = "getPayloadV5")] + async fn get_payload_v5( + &self, + payload_id: PayloadId, + ) -> RpcResult; + /// See also #[method(name = "getPayloadBodiesByHashV1")] async fn get_payload_bodies_by_hash_v1( @@ -178,7 +190,7 @@ pub trait EngineApi { /// Returns the execution payload bodies by the range starting at `start`, containing `count` /// blocks. /// - /// WARNING: This method is associated with the BeaconBlocksByRange message in the consensus + /// WARNING: This method is associated with the `BeaconBlocksByRange` message in the consensus /// layer p2p specification, meaning the input should be treated as untrusted or potentially /// adversarial. /// @@ -192,9 +204,9 @@ pub trait EngineApi { count: U64, ) -> RpcResult; - /// This function will return the ClientVersionV1 object. + /// This function will return the [`ClientVersionV1`] object. /// See also: - /// make fmt + /// /// /// /// - When connected to a single execution client, the consensus client **MUST** receive an @@ -212,7 +224,7 @@ pub trait EngineApi { #[method(name = "exchangeCapabilities")] async fn exchange_capabilities(&self, capabilities: Vec) -> RpcResult>; - /// Fetch blobs for the consensus layer from the in-memory blob cache. + /// Fetch blobs for the consensus layer from the blob store. #[method(name = "getBlobsV1")] async fn get_blobs_v1( &self, @@ -220,21 +232,24 @@ pub trait EngineApi { ) -> RpcResult>>; /// Fetch blobs for the consensus layer from the blob store. + /// + /// Returns a response only if blobs and proofs are present for _all_ of the versioned hashes: + /// 2. Client software MUST return null in case of any missing or older version blobs. #[method(name = "getBlobsV2")] async fn get_blobs_v2( &self, versioned_hashes: Vec, - ) -> RpcResult>>; + ) -> RpcResult>>; } -/// A subset of the ETH rpc interface: +/// A subset of the ETH rpc interface: /// /// This also includes additional eth functions required by optimism. /// /// Specifically for the engine auth server: #[cfg_attr(not(feature = "client"), rpc(server, namespace = "eth"))] #[cfg_attr(feature = "client", rpc(server, client, namespace = "eth"))] -pub trait EngineEthApi { +pub trait EngineEthApi { /// Returns an object with data about the sync status or false. #[method(name = "syncing")] fn syncing(&self) -> RpcResult; @@ -251,7 +266,7 @@ pub trait EngineEthApi { #[method(name = "call")] async fn call( &self, - request: TransactionRequest, + request: TxReq, block_id: Option, state_overrides: Option, block_overrides: Option>, diff --git a/crates/rpc/rpc-api/src/ganache.rs b/crates/rpc/rpc-api/src/ganache.rs deleted file mode 100644 index 0f46b481efe..00000000000 --- a/crates/rpc/rpc-api/src/ganache.rs +++ /dev/null @@ -1,75 +0,0 @@ -use alloy_primitives::U256; -use alloy_rpc_types_anvil::MineOptions; -use jsonrpsee::{core::RpcResult, proc_macros::rpc}; - -/// Ganache rpc interface. -/// https://github.com/trufflesuite/ganache/tree/develop/docs -#[cfg_attr(not(feature = "client"), rpc(server, namespace = "evm"))] -#[cfg_attr(feature = "client", rpc(server, client, namespace = "evm"))] -pub trait GanacheApi { - // TODO Ganache is deprecated and this method is not implemented by Anvil and Hardhat. - // #[method(name = "addAccount")] - // async fn evm_add_account(&self, address: Address, passphrase: B256) -> RpcResult; - - /// Jump forward in time by the given amount of time, in seconds. - /// - /// Returns the total time adjustment, in seconds. - #[method(name = "increaseTime")] - async fn evm_increase_time(&self, seconds: U256) -> RpcResult; - - /// Force a single block to be mined. - /// - /// Mines a block independent of whether or not mining is started or stopped. Will mine an empty - /// block if there are no available transactions to mine. - /// - /// Returns "0x0". May return additional meta-data in the future. - #[method(name = "mine")] - async fn evm_mine(&self, opts: Option) -> RpcResult; - - // TODO Ganache is deprecated and this method is not implemented by Anvil and Hardhat. - // #[method(name = "removeAccount")] - // async fn evm_remove_account(address: Address, passphrase: B256) -> RpcResult; - - /// Revert the state of the blockchain to a previous snapshot. Takes a single parameter, which - /// is the snapshot id to revert to. This deletes the given snapshot, as well as any snapshots - /// taken after (e.g.: reverting to id 0x1 will delete snapshots with ids 0x1, 0x2, etc.). - /// - /// Returns `true` if a snapshot was reverted, otherwise `false`. - #[method(name = "revert")] - async fn evm_revert(&self, snapshot_id: U256) -> RpcResult; - - // TODO Ganache is deprecated and this method is not implemented by Anvil and Hardhat. - // #[method(name = "setAccountBalance")] - // async fn evm_set_account_balance(address: Address, balance: U256) -> RpcResult; - - // TODO Ganache is deprecated and this method is not implemented by Anvil and Hardhat. - // #[method(name = "setAccountCode")] - // async fn evm_set_account_code(address: Address, code: Bytes) -> RpcResult; - - // TODO Ganache is deprecated and this method is not implemented by Anvil and Hardhat. - // #[method(name = "setAccountNonce")] - // async fn evm_set_account_nonce(address: Address, nonce: U256) -> RpcResult; - - // TODO Ganache is deprecated and this method is not implemented by Anvil and Hardhat. - // #[method(name = "setAccountStorageAt")] - // async fn evm_set_account_storage_at(address: Address, slot: U256, value: B256) -> - // RpcResult; - - /// Sets the internal clock time to the given timestamp. - /// - /// **Warning** This will allow you to move backwards in time, which may cause new blocks to - /// appear to be mined before old blocks. This will result in an invalid state. - /// - /// Returns the amount of seconds between the given timestamp and now. - #[method(name = "setTime")] - async fn evm_set_time(&self, timestamp: u64) -> RpcResult; - - /// Snapshot the state of the blockchain at the current block. Takes no parameters. Returns the - /// id of the snapshot that was created. A snapshot can only be reverted once. After a - /// successful evm_revert, the same snapshot id cannot be used again. Consider creating a new - /// snapshot after each evm_revert if you need to revert to the same point multiple times. - /// - /// Returns the hex-encoded identifier for this snapshot. - #[method(name = "snapshot")] - async fn evm_snapshot(&self) -> RpcResult; -} diff --git a/crates/rpc/rpc-api/src/lib.rs b/crates/rpc/rpc-api/src/lib.rs index 5f4a75ef469..e68e47cd219 100644 --- a/crates/rpc/rpc-api/src/lib.rs +++ b/crates/rpc/rpc-api/src/lib.rs @@ -12,13 +12,12 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] mod admin; mod anvil; mod debug; mod engine; -mod ganache; mod hardhat; mod mev; mod miner; @@ -69,7 +68,6 @@ pub mod clients { anvil::AnvilApiClient, debug::{DebugApiClient, DebugExecutionWitnessApiClient}, engine::{EngineApiClient, EngineEthApiClient}, - ganache::GanacheApiClient, hardhat::HardhatApiClient, mev::{MevFullApiClient, MevSimApiClient}, miner::MinerApiClient, diff --git a/crates/rpc/rpc-api/src/mev.rs b/crates/rpc/rpc-api/src/mev.rs index 4980b5cc671..274fcbf9316 100644 --- a/crates/rpc/rpc-api/src/mev.rs +++ b/crates/rpc/rpc-api/src/mev.rs @@ -1,6 +1,4 @@ -use alloy_rpc_types_mev::{ - SendBundleRequest, SendBundleResponse, SimBundleOverrides, SimBundleResponse, -}; +use alloy_rpc_types_mev::{EthBundleHash, MevSendBundle, SimBundleOverrides, SimBundleResponse}; use jsonrpsee::proc_macros::rpc; /// Mev rpc interface. @@ -12,7 +10,7 @@ pub trait MevSimApi { #[method(name = "simBundle")] async fn sim_bundle( &self, - bundle: SendBundleRequest, + bundle: MevSendBundle, sim_overrides: SimBundleOverrides, ) -> jsonrpsee::core::RpcResult; } @@ -26,15 +24,15 @@ pub trait MevFullApi { #[method(name = "sendBundle")] async fn send_bundle( &self, - request: SendBundleRequest, - ) -> jsonrpsee::core::RpcResult; + request: MevSendBundle, + ) -> jsonrpsee::core::RpcResult; /// Similar to `mev_sendBundle` but instead of submitting a bundle to the relay, it returns /// a simulation result. Only fully matched bundles can be simulated. #[method(name = "simBundle")] async fn sim_bundle( &self, - bundle: SendBundleRequest, + bundle: MevSendBundle, sim_overrides: SimBundleOverrides, ) -> jsonrpsee::core::RpcResult; } diff --git a/crates/rpc/rpc-api/src/otterscan.rs b/crates/rpc/rpc-api/src/otterscan.rs index eb2cb21a2ba..c8651e608f5 100644 --- a/crates/rpc/rpc-api/src/otterscan.rs +++ b/crates/rpc/rpc-api/src/otterscan.rs @@ -1,4 +1,4 @@ -use alloy_eips::BlockId; +use alloy_eips::{eip1898::LenientBlockNumberOrTag, BlockId}; use alloy_json_rpc::RpcObject; use alloy_primitives::{Address, Bytes, TxHash, B256}; use alloy_rpc_types_trace::otterscan::{ @@ -19,7 +19,10 @@ pub trait Otterscan { /// /// Ref: #[method(name = "getHeaderByNumber", aliases = ["erigon_getHeaderByNumber"])] - async fn get_header_by_number(&self, block_number: u64) -> RpcResult>; + async fn get_header_by_number( + &self, + block_number: LenientBlockNumberOrTag, + ) -> RpcResult>; /// Check if a certain address contains a deployed code. #[method(name = "hasCode")] @@ -44,12 +47,16 @@ pub trait Otterscan { #[method(name = "traceTransaction")] async fn trace_transaction(&self, tx_hash: TxHash) -> RpcResult>>; - /// Tailor-made and expanded version of eth_getBlockByNumber for block details page in + /// Tailor-made and expanded version of `eth_getBlockByNumber` for block details page in /// Otterscan. #[method(name = "getBlockDetails")] - async fn get_block_details(&self, block_number: u64) -> RpcResult>; + async fn get_block_details( + &self, + block_number: LenientBlockNumberOrTag, + ) -> RpcResult>; - /// Tailor-made and expanded version of eth_getBlockByHash for block details page in Otterscan. + /// Tailor-made and expanded version of `eth_getBlockByHash` for block details page in + /// Otterscan. #[method(name = "getBlockDetailsByHash")] async fn get_block_details_by_hash(&self, block_hash: B256) -> RpcResult>; @@ -57,7 +64,7 @@ pub trait Otterscan { #[method(name = "getBlockTransactions")] async fn get_block_transactions( &self, - block_number: u64, + block_number: LenientBlockNumberOrTag, page_number: usize, page_size: usize, ) -> RpcResult>; @@ -67,7 +74,7 @@ pub trait Otterscan { async fn search_transactions_before( &self, address: Address, - block_number: u64, + block_number: LenientBlockNumberOrTag, page_size: usize, ) -> RpcResult; @@ -76,7 +83,7 @@ pub trait Otterscan { async fn search_transactions_after( &self, address: Address, - block_number: u64, + block_number: LenientBlockNumberOrTag, page_size: usize, ) -> RpcResult; diff --git a/crates/rpc/rpc-api/src/reth.rs b/crates/rpc/rpc-api/src/reth.rs index 0589ffc00ce..de0402624a9 100644 --- a/crates/rpc/rpc-api/src/reth.rs +++ b/crates/rpc/rpc-api/src/reth.rs @@ -3,6 +3,9 @@ use alloy_primitives::{Address, U256}; use jsonrpsee::{core::RpcResult, proc_macros::rpc}; use std::collections::HashMap; +// Required for the subscription attribute below +use reth_chain_state as _; + /// Reth API namespace for reth-specific methods #[cfg_attr(not(feature = "client"), rpc(server, namespace = "reth"))] #[cfg_attr(feature = "client", rpc(server, client, namespace = "reth"))] @@ -13,4 +16,12 @@ pub trait RethApi { &self, block_id: BlockId, ) -> RpcResult>; + + /// Subscribe to json `ChainNotifications` + #[subscription( + name = "subscribeChainNotifications", + unsubscribe = "unsubscribeChainNotifications", + item = reth_chain_state::CanonStateNotification + )] + async fn reth_subscribe_chain_notifications(&self) -> jsonrpsee::core::SubscriptionResult; } diff --git a/crates/rpc/rpc-api/src/trace.rs b/crates/rpc/rpc-api/src/trace.rs index 41e2b4c1c3e..1c4b148a098 100644 --- a/crates/rpc/rpc-api/src/trace.rs +++ b/crates/rpc/rpc-api/src/trace.rs @@ -1,8 +1,6 @@ use alloy_eips::BlockId; use alloy_primitives::{map::HashSet, Bytes, B256}; -use alloy_rpc_types_eth::{ - state::StateOverride, transaction::TransactionRequest, BlockOverrides, Index, -}; +use alloy_rpc_types_eth::{state::StateOverride, BlockOverrides, Index}; use alloy_rpc_types_trace::{ filter::TraceFilter, opcode::{BlockOpcodeGas, TransactionOpcodeGas}, @@ -13,12 +11,12 @@ use jsonrpsee::{core::RpcResult, proc_macros::rpc}; /// Ethereum trace API #[cfg_attr(not(feature = "client"), rpc(server, namespace = "trace"))] #[cfg_attr(feature = "client", rpc(server, client, namespace = "trace"))] -pub trait TraceApi { +pub trait TraceApi { /// Executes the given call and returns a number of possible traces for it. #[method(name = "call")] async fn trace_call( &self, - call: TransactionRequest, + call: TxReq, trace_types: HashSet, block_id: Option, state_overrides: Option, @@ -31,7 +29,7 @@ pub trait TraceApi { #[method(name = "callMany")] async fn trace_call_many( &self, - calls: Vec<(TransactionRequest, HashSet)>, + calls: Vec<(TxReq, HashSet)>, block_id: Option, ) -> RpcResult>; @@ -80,7 +78,7 @@ pub trait TraceApi { /// `indices` represent the index positions of the traces. /// /// Note: This expects a list of indices but only one is supported since this function returns a - /// single [LocalizedTransactionTrace]. + /// single [`LocalizedTransactionTrace`]. #[method(name = "get")] async fn trace_get( &self, diff --git a/crates/rpc/rpc-api/src/validation.rs b/crates/rpc/rpc-api/src/validation.rs index 5e4f2e26143..9ff47b5eaf2 100644 --- a/crates/rpc/rpc-api/src/validation.rs +++ b/crates/rpc/rpc-api/src/validation.rs @@ -3,6 +3,7 @@ use alloy_rpc_types_beacon::relay::{ BuilderBlockValidationRequest, BuilderBlockValidationRequestV2, BuilderBlockValidationRequestV3, BuilderBlockValidationRequestV4, + BuilderBlockValidationRequestV5, }; use jsonrpsee::proc_macros::rpc; @@ -37,4 +38,11 @@ pub trait BlockSubmissionValidationApi { &self, request: BuilderBlockValidationRequestV4, ) -> jsonrpsee::core::RpcResult<()>; + + /// A Request to validate a block submission. + #[method(name = "validateBuilderSubmissionV5")] + async fn validate_builder_submission_v5( + &self, + request: BuilderBlockValidationRequestV5, + ) -> jsonrpsee::core::RpcResult<()>; } diff --git a/crates/rpc/rpc-builder/Cargo.toml b/crates/rpc/rpc-builder/Cargo.toml index 92ef9cfbc12..e7178405b3b 100644 --- a/crates/rpc/rpc-builder/Cargo.toml +++ b/crates/rpc/rpc-builder/Cargo.toml @@ -43,6 +43,7 @@ reth-metrics = { workspace = true, features = ["common"] } metrics.workspace = true # misc +dyn-clone.workspace = true serde = { workspace = true, features = ["derive"] } thiserror.workspace = true tracing.workspace = true @@ -52,10 +53,7 @@ alloy-provider = { workspace = true, features = ["ws", "ipc"] } alloy-network.workspace = true [dev-dependencies] -reth-primitives-traits.workspace = true reth-ethereum-primitives.workspace = true -reth-chainspec.workspace = true -reth-network-api.workspace = true reth-network-peers.workspace = true reth-evm-ethereum.workspace = true reth-ethereum-engine-primitives.workspace = true @@ -65,7 +63,6 @@ reth-rpc-api = { workspace = true, features = ["client"] } reth-rpc-engine-api.workspace = true reth-tracing.workspace = true reth-transaction-pool = { workspace = true, features = ["test-utils"] } -reth-rpc-types-compat.workspace = true reth-engine-primitives.workspace = true reth-node-ethereum.workspace = true @@ -75,6 +72,5 @@ alloy-rpc-types-trace.workspace = true alloy-eips.workspace = true alloy-rpc-types-engine.workspace = true -tokio = { workspace = true, features = ["rt", "rt-multi-thread"] } serde_json.workspace = true clap = { workspace = true, features = ["derive"] } diff --git a/crates/rpc/rpc-builder/src/auth.rs b/crates/rpc/rpc-builder/src/auth.rs index 4885ef99dff..0d0a6165ff7 100644 --- a/crates/rpc/rpc-builder/src/auth.rs +++ b/crates/rpc/rpc-builder/src/auth.rs @@ -1,37 +1,43 @@ -use crate::error::{RpcError, ServerKind}; +use crate::{ + error::{RpcError, ServerKind}, + middleware::RethRpcMiddleware, +}; use http::header::AUTHORIZATION; use jsonrpsee::{ - core::RegisterMethodError, - http_client::{transport::HttpBackend, HeaderMap}, + core::{client::SubscriptionClientT, RegisterMethodError}, + http_client::HeaderMap, server::{AlreadyStoppedError, RpcModule}, + ws_client::RpcServiceBuilder, Methods, }; use reth_rpc_api::servers::*; use reth_rpc_eth_types::EthSubscriptionIdProvider; use reth_rpc_layer::{ - secret_to_bearer_header, AuthClientLayer, AuthClientService, AuthLayer, JwtAuthValidator, - JwtSecret, + secret_to_bearer_header, AuthClientLayer, AuthLayer, JwtAuthValidator, JwtSecret, }; use reth_rpc_server_types::constants; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use tower::layer::util::Identity; pub use jsonrpsee::server::ServerBuilder; +use jsonrpsee::server::{ServerConfig, ServerConfigBuilder}; pub use reth_ipc::server::Builder as IpcServerBuilder; /// Server configuration for the auth server. #[derive(Debug)] -pub struct AuthServerConfig { +pub struct AuthServerConfig { /// Where the server should listen. pub(crate) socket_addr: SocketAddr, /// The secret for the auth layer of the server. pub(crate) secret: JwtSecret, /// Configs for JSON-RPC Http. - pub(crate) server_config: ServerBuilder, + pub(crate) server_config: ServerConfigBuilder, /// Configs for IPC server pub(crate) ipc_server_config: Option>, /// IPC endpoint pub(crate) ipc_endpoint: Option, + /// Configurable RPC middleware + pub(crate) rpc_middleware: RpcMiddleware, } // === impl AuthServerConfig === @@ -41,23 +47,51 @@ impl AuthServerConfig { pub const fn builder(secret: JwtSecret) -> AuthServerConfigBuilder { AuthServerConfigBuilder::new(secret) } - +} +impl AuthServerConfig { /// Returns the address the server will listen on. pub const fn address(&self) -> SocketAddr { self.socket_addr } + /// Configures the rpc middleware. + pub fn with_rpc_middleware(self, rpc_middleware: T) -> AuthServerConfig { + let Self { socket_addr, secret, server_config, ipc_server_config, ipc_endpoint, .. } = self; + AuthServerConfig { + socket_addr, + secret, + server_config, + ipc_server_config, + ipc_endpoint, + rpc_middleware, + } + } + /// Convenience function to start a server in one step. - pub async fn start(self, module: AuthRpcModule) -> Result { - let Self { socket_addr, secret, server_config, ipc_server_config, ipc_endpoint } = self; + pub async fn start(self, module: AuthRpcModule) -> Result + where + RpcMiddleware: RethRpcMiddleware, + { + let Self { + socket_addr, + secret, + server_config, + ipc_server_config, + ipc_endpoint, + rpc_middleware, + } = self; // Create auth middleware. let middleware = tower::ServiceBuilder::new().layer(AuthLayer::new(JwtAuthValidator::new(secret))); + let rpc_middleware = RpcServiceBuilder::default().layer(rpc_middleware); + // By default, both http and ws are enabled. - let server = server_config + let server = ServerBuilder::new() + .set_config(server_config.build()) .set_http_middleware(middleware) + .set_rpc_middleware(rpc_middleware) .build(socket_addr) .await .map_err(|err| RpcError::server_error(err, ServerKind::Auth(socket_addr)))?; @@ -67,16 +101,17 @@ impl AuthServerConfig { .map_err(|err| RpcError::server_error(err, ServerKind::Auth(socket_addr)))?; let handle = server.start(module.inner.clone()); - let mut ipc_handle: Option = None; - if let Some(ipc_server_config) = ipc_server_config { + let ipc_handle = if let Some(ipc_server_config) = ipc_server_config { let ipc_endpoint_str = ipc_endpoint .clone() .unwrap_or_else(|| constants::DEFAULT_ENGINE_API_IPC_ENDPOINT.to_string()); let ipc_server = ipc_server_config.build(ipc_endpoint_str); let res = ipc_server.start(module.inner).await?; - ipc_handle = Some(res); - } + Some(res) + } else { + None + }; Ok(AuthServerHandle { handle: Some(handle), local_addr, secret, ipc_endpoint, ipc_handle }) } @@ -84,12 +119,13 @@ impl AuthServerConfig { /// Builder type for configuring an `AuthServerConfig`. #[derive(Debug)] -pub struct AuthServerConfigBuilder { +pub struct AuthServerConfigBuilder { socket_addr: Option, secret: JwtSecret, - server_config: Option>, + server_config: Option, ipc_server_config: Option>, ipc_endpoint: Option, + rpc_middleware: RpcMiddleware, } // === impl AuthServerConfigBuilder === @@ -103,6 +139,22 @@ impl AuthServerConfigBuilder { server_config: None, ipc_server_config: None, ipc_endpoint: None, + rpc_middleware: Identity::new(), + } + } +} + +impl AuthServerConfigBuilder { + /// Configures the rpc middleware. + pub fn with_rpc_middleware(self, rpc_middleware: T) -> AuthServerConfigBuilder { + let Self { socket_addr, secret, server_config, ipc_server_config, ipc_endpoint, .. } = self; + AuthServerConfigBuilder { + socket_addr, + secret, + server_config, + ipc_server_config, + ipc_endpoint, + rpc_middleware, } } @@ -128,7 +180,7 @@ impl AuthServerConfigBuilder { /// /// Note: this always configures an [`EthSubscriptionIdProvider`] /// [`IdProvider`](jsonrpsee::server::IdProvider) for convenience. - pub fn with_server_config(mut self, config: ServerBuilder) -> Self { + pub fn with_server_config(mut self, config: ServerConfigBuilder) -> Self { self.server_config = Some(config.set_id_provider(EthSubscriptionIdProvider::default())); self } @@ -148,25 +200,27 @@ impl AuthServerConfigBuilder { } /// Build the `AuthServerConfig`. - pub fn build(self) -> AuthServerConfig { + pub fn build(self) -> AuthServerConfig { AuthServerConfig { socket_addr: self.socket_addr.unwrap_or_else(|| { SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), constants::DEFAULT_AUTH_PORT) }), secret: self.secret, server_config: self.server_config.unwrap_or_else(|| { - ServerBuilder::new() - // This needs to large enough to handle large eth_getLogs responses and maximum - // payload bodies limit for `engine_getPayloadBodiesByRangeV` - // ~750MB per response should be enough + ServerConfig::builder() + // This needs to large enough to handle large eth_getLogs responses and + // maximum payload bodies limit for + // `engine_getPayloadBodiesByRangeV` ~750MB per + // response should be enough .max_response_body_size(750 * 1024 * 1024) - // Connections to this server are always authenticated, hence this only affects - // connections from the CL or any other client that uses JWT, this should be - // more than enough so that the CL (or multiple CL nodes) will never get rate - // limited + // Connections to this server are always authenticated, hence this only + // affects connections from the CL or any other + // client that uses JWT, this should be + // more than enough so that the CL (or multiple CL nodes) will never get + // rate limited .max_connections(500) - // bump the default request size slightly, there aren't any methods exposed with - // dynamic request params that can exceed this + // bump the default request size slightly, there aren't any methods exposed + // with dynamic request params that can exceed this .max_request_body_size(128 * 1024 * 1024) .set_id_provider(EthSubscriptionIdProvider::default()) }), @@ -178,6 +232,7 @@ impl AuthServerConfigBuilder { .set_id_provider(EthSubscriptionIdProvider::default()) }), ipc_endpoint: self.ipc_endpoint, + rpc_middleware: self.rpc_middleware, } } } @@ -297,9 +352,11 @@ impl AuthServerHandle { } /// Returns a http client connected to the server. + /// + /// This client uses the JWT token to authenticate requests. pub fn http_client( &self, - ) -> jsonrpsee::http_client::HttpClient> { + ) -> impl SubscriptionClientT + use<> + Clone + Send + Sync + Unpin + 'static { // Create a middleware that adds a new JWT token to every request. let secret_layer = AuthClientLayer::new(self.secret); let middleware = tower::ServiceBuilder::default().layer(secret_layer); diff --git a/crates/rpc/rpc-builder/src/config.rs b/crates/rpc/rpc-builder/src/config.rs index ed879d04740..4d57bdec7d8 100644 --- a/crates/rpc/rpc-builder/src/config.rs +++ b/crates/rpc/rpc-builder/src/config.rs @@ -1,11 +1,10 @@ -use std::{net::SocketAddr, path::PathBuf}; - -use jsonrpsee::server::ServerBuilder; +use jsonrpsee::server::ServerConfigBuilder; use reth_node_core::{args::RpcServerArgs, utils::get_or_create_jwt_secret_from_path}; use reth_rpc::ValidationApiConfig; use reth_rpc_eth_types::{EthConfig, EthStateCacheConfig, GasPriceOracleConfig}; use reth_rpc_layer::{JwtError, JwtSecret}; use reth_rpc_server_types::RpcModuleSelection; +use std::{net::SocketAddr, path::PathBuf}; use tower::layer::util::Identity; use tracing::{debug, warn}; @@ -49,8 +48,8 @@ pub trait RethRpcServerConfig { /// settings in the [`TransportRpcModuleConfig`]. fn transport_rpc_module_config(&self) -> TransportRpcModuleConfig; - /// Returns the default server builder for http/ws - fn http_ws_server_builder(&self) -> ServerBuilder; + /// Returns the default server config for http/ws + fn http_ws_server_builder(&self) -> ServerConfigBuilder; /// Returns the default ipc server builder fn ipc_server_builder(&self) -> IpcServerBuilder; @@ -104,6 +103,9 @@ impl RethRpcServerConfig for RpcServerArgs { .state_cache(self.state_cache_config()) .gpo_config(self.gas_price_oracle_config()) .proof_permits(self.rpc_proof_permits) + .pending_block_kind(self.rpc_pending_block) + .raw_tx_forwarder(self.rpc_forwarder.clone()) + .rpc_evm_memory_limit(self.rpc_evm_memory_limit) } fn flashbots_config(&self) -> ValidationApiConfig { @@ -161,8 +163,8 @@ impl RethRpcServerConfig for RpcServerArgs { config } - fn http_ws_server_builder(&self) -> ServerBuilder { - ServerBuilder::new() + fn http_ws_server_builder(&self) -> ServerConfigBuilder { + ServerConfigBuilder::new() .max_connections(self.rpc_max_connections.get()) .max_request_body_size(self.rpc_max_request_size_bytes()) .max_response_body_size(self.rpc_max_response_size_bytes()) @@ -175,6 +177,7 @@ impl RethRpcServerConfig for RpcServerArgs { .max_request_body_size(self.rpc_max_request_size_bytes()) .max_response_body_size(self.rpc_max_response_size_bytes()) .max_connections(self.rpc_max_connections.get()) + .set_ipc_socket_permissions(self.ipc_socket_permissions.clone()) } fn rpc_server_config(&self) -> RpcServerConfig { @@ -193,12 +196,16 @@ impl RethRpcServerConfig for RpcServerArgs { .with_http_address(socket_address) .with_http(self.http_ws_server_builder()) .with_http_cors(self.http_corsdomain.clone()) - .with_ws_cors(self.ws_allowed_origins.clone()); + .with_http_disable_compression(self.http_disable_compression); } if self.ws { let socket_address = SocketAddr::new(self.ws_addr, self.ws_port); - config = config.with_ws_address(socket_address).with_ws(self.http_ws_server_builder()); + // Ensure WS CORS is applied regardless of HTTP being enabled + config = config + .with_ws_address(socket_address) + .with_ws(self.http_ws_server_builder()) + .with_ws_cors(self.ws_allowed_origins.clone()); } if self.is_ipc_enabled() { diff --git a/crates/rpc/rpc-builder/src/eth.rs b/crates/rpc/rpc-builder/src/eth.rs index a4909c5029a..1ef0fd33be3 100644 --- a/crates/rpc/rpc-builder/src/eth.rs +++ b/crates/rpc/rpc-builder/src/eth.rs @@ -21,14 +21,14 @@ where /// Returns a new instance with the additional handlers for the `eth` namespace. /// /// This will spawn all necessary tasks for the additional handlers. - pub fn bootstrap(config: EthConfig, executor: Tasks, eth_api: EthApi) -> Self - where - Tasks: TaskSpawner + Clone + 'static, - { - let filter = - EthFilter::new(eth_api.clone(), config.filter_config(), Box::new(executor.clone())); + pub fn bootstrap( + config: EthConfig, + executor: Box, + eth_api: EthApi, + ) -> Self { + let filter = EthFilter::new(eth_api.clone(), config.filter_config(), executor.clone()); - let pubsub = EthPubSub::with_spawner(eth_api.clone(), Box::new(executor)); + let pubsub = EthPubSub::with_spawner(eth_api.clone(), executor); Self { api: eth_api, filter, pubsub } } diff --git a/crates/rpc/rpc-builder/src/lib.rs b/crates/rpc/rpc-builder/src/lib.rs index 18716fd30ee..6bd4223f60f 100644 --- a/crates/rpc/rpc-builder/src/lib.rs +++ b/crates/rpc/rpc-builder/src/lib.rs @@ -17,58 +17,60 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] use crate::{auth::AuthRpcModule, error::WsHttpSamePortError, metrics::RpcRequestMetrics}; +use alloy_network::{Ethereum, IntoWallet}; use alloy_provider::{fillers::RecommendedFillers, Provider, ProviderBuilder}; use core::marker::PhantomData; use error::{ConflictingModules, RpcError, ServerKind}; use http::{header::AUTHORIZATION, HeaderMap}; use jsonrpsee::{ core::RegisterMethodError, - server::{ - middleware::rpc::{RpcService, RpcServiceT}, - AlreadyStoppedError, IdProvider, RpcServiceBuilder, ServerHandle, - }, + server::{middleware::rpc::RpcServiceBuilder, AlreadyStoppedError, IdProvider, ServerHandle}, Methods, RpcModule, }; use reth_chainspec::{ChainSpecProvider, EthereumHardforks}; use reth_consensus::{ConsensusError, FullConsensus}; use reth_evm::ConfigureEvm; use reth_network_api::{noop::NoopNetwork, NetworkInfo, Peers}; -use reth_primitives_traits::NodePrimitives; +use reth_primitives_traits::{NodePrimitives, TxTy}; use reth_rpc::{ AdminApi, DebugApi, EngineEthApi, EthApi, EthApiBuilder, EthBundle, MinerApi, NetApi, OtterscanApi, RPCApi, RethApi, TraceApi, TxPoolApi, ValidationApiConfig, Web3Api, }; use reth_rpc_api::servers::*; use reth_rpc_eth_api::{ - helpers::{Call, EthApiSpec, EthTransactions, LoadPendingBlock, TraceExt}, - EthApiServer, EthApiTypes, FullEthApiServer, RpcBlock, RpcHeader, RpcReceipt, RpcTransaction, + helpers::{ + pending_block::PendingEnvBuilder, Call, EthApiSpec, EthTransactions, LoadPendingBlock, + TraceExt, + }, + node::RpcNodeCoreAdapter, + EthApiServer, EthApiTypes, FullEthApiServer, RpcBlock, RpcConvert, RpcConverter, RpcHeader, + RpcNodeCore, RpcReceipt, RpcTransaction, RpcTxReq, }; -use reth_rpc_eth_types::{EthConfig, EthSubscriptionIdProvider}; +use reth_rpc_eth_types::{receipt::EthReceiptConverter, EthConfig, EthSubscriptionIdProvider}; use reth_rpc_layer::{AuthLayer, Claims, CompressionLayer, JwtAuthValidator, JwtSecret}; use reth_storage_api::{ - AccountReader, BlockReader, BlockReaderIdExt, ChangeSetReader, FullRpcProvider, ProviderBlock, + AccountReader, BlockReader, ChangeSetReader, FullRpcProvider, ProviderBlock, StateProviderFactory, }; use reth_tasks::{pool::BlockingTaskGuard, TaskSpawner, TokioTaskExecutor}; -use reth_transaction_pool::{noop::NoopTransactionPool, PoolTransaction, TransactionPool}; +use reth_transaction_pool::{noop::NoopTransactionPool, TransactionPool}; use serde::{Deserialize, Serialize}; use std::{ collections::HashMap, fmt::Debug, net::{Ipv4Addr, SocketAddr, SocketAddrV4}, - sync::Arc, time::{Duration, SystemTime, UNIX_EPOCH}, }; -use tower::Layer; use tower_http::cors::CorsLayer; pub use cors::CorsDomainError; // re-export for convenience pub use jsonrpsee::server::ServerBuilder; +use jsonrpsee::server::ServerConfigBuilder; pub use reth_ipc::server::{ Builder as IpcServerBuilder, RpcServiceBuilder as IpcRpcServiceBuilder, }; @@ -81,6 +83,9 @@ pub mod auth; /// RPC server utilities. pub mod config; +/// Utils for installing Rpc middleware +pub mod middleware; + /// Cors utilities. mod cors; @@ -93,6 +98,7 @@ pub use eth::EthHandlers; // Rpc server metrics mod metrics; +use crate::middleware::RethRpcMiddleware; pub use metrics::{MeteredRequestFuture, RpcRequestMetricsService}; use reth_chain_state::CanonStateSubscriptions; use reth_rpc::eth::sim_bundle::EthSimBundle; @@ -100,46 +106,11 @@ use reth_rpc::eth::sim_bundle::EthSimBundle; // Rpc rate limiter pub mod rate_limiter; -/// Convenience function for starting a server in one step. -#[expect(clippy::too_many_arguments)] -pub async fn launch( - provider: Provider, - pool: Pool, - network: Network, - module_config: impl Into, - server_config: impl Into, - executor: Tasks, - evm_config: EvmConfig, - eth: EthApi, - consensus: Arc>, -) -> Result -where - N: NodePrimitives, - Provider: FullRpcProvider - + CanonStateSubscriptions - + AccountReader - + ChangeSetReader, - Pool: TransactionPool + 'static, - Network: NetworkInfo + Peers + Clone + 'static, - Tasks: TaskSpawner + Clone + 'static, - EvmConfig: ConfigureEvm + 'static, - EthApi: FullEthApiServer, -{ - let module_config = module_config.into(); - server_config - .into() - .start( - &RpcModuleBuilder::new(provider, pool, network, executor, evm_config, consensus) - .build(module_config, eth), - ) - .await -} - /// A builder type to configure the RPC module: See [`RpcModule`] /// /// This is the main entrypoint and the easiest way to configure an RPC server. #[derive(Debug, Clone)] -pub struct RpcModuleBuilder { +pub struct RpcModuleBuilder { /// The Provider type to when creating all rpc handlers provider: Provider, /// The Pool type to when creating all rpc handlers @@ -147,7 +118,7 @@ pub struct RpcModuleBuilder, /// Defines how the EVM should be configured before execution. evm_config: EvmConfig, /// The consensus implementation. @@ -158,18 +129,15 @@ pub struct RpcModuleBuilder - RpcModuleBuilder -where - N: NodePrimitives, - EvmConfig: Clone, +impl + RpcModuleBuilder { /// Create a new instance of the builder pub const fn new( provider: Provider, pool: Pool, network: Network, - executor: Tasks, + executor: Box, evm_config: EvmConfig, consensus: Consensus, ) -> Self { @@ -180,12 +148,7 @@ where pub fn with_provider

( self, provider: P, - ) -> RpcModuleBuilder - where - P: BlockReader - + StateProviderFactory - + 'static, - { + ) -> RpcModuleBuilder { let Self { pool, network, executor, evm_config, consensus, _primitives, .. } = self; RpcModuleBuilder { provider, network, pool, executor, evm_config, consensus, _primitives } } @@ -194,10 +157,7 @@ where pub fn with_pool

( self, pool: P, - ) -> RpcModuleBuilder - where - P: TransactionPool> + 'static, - { + ) -> RpcModuleBuilder { let Self { provider, network, executor, evm_config, consensus, _primitives, .. } = self; RpcModuleBuilder { provider, network, pool, executor, evm_config, consensus, _primitives } } @@ -209,8 +169,7 @@ where /// [`EthApi`] which requires a [`TransactionPool`] implementation. pub fn with_noop_pool( self, - ) -> RpcModuleBuilder - { + ) -> RpcModuleBuilder { let Self { provider, executor, network, evm_config, consensus, _primitives, .. } = self; RpcModuleBuilder { provider, @@ -227,10 +186,7 @@ where pub fn with_network( self, network: Net, - ) -> RpcModuleBuilder - where - Net: NetworkInfo + Peers + 'static, - { + ) -> RpcModuleBuilder { let Self { provider, pool, executor, evm_config, consensus, _primitives, .. } = self; RpcModuleBuilder { provider, network, pool, executor, evm_config, consensus, _primitives } } @@ -242,7 +198,7 @@ where /// [`EthApi`] which requires a [`NetworkInfo`] implementation. pub fn with_noop_network( self, - ) -> RpcModuleBuilder { + ) -> RpcModuleBuilder { let Self { provider, pool, executor, evm_config, consensus, _primitives, .. } = self; RpcModuleBuilder { provider, @@ -256,30 +212,22 @@ where } /// Configure the task executor to use for additional tasks. - pub fn with_executor( - self, - executor: T, - ) -> RpcModuleBuilder - where - T: TaskSpawner + 'static, - { + pub fn with_executor(self, executor: Box) -> Self { let Self { pool, network, provider, evm_config, consensus, _primitives, .. } = self; - RpcModuleBuilder { provider, network, pool, executor, evm_config, consensus, _primitives } + Self { provider, network, pool, executor, evm_config, consensus, _primitives } } /// Configure [`TokioTaskExecutor`] as the task executor to use for additional tasks. /// /// This will spawn additional tasks directly via `tokio::task::spawn`, See /// [`TokioTaskExecutor`]. - pub fn with_tokio_executor( - self, - ) -> RpcModuleBuilder { + pub fn with_tokio_executor(self) -> Self { let Self { pool, network, provider, evm_config, consensus, _primitives, .. } = self; - RpcModuleBuilder { + Self { provider, network, pool, - executor: TokioTaskExecutor::default(), + executor: Box::new(TokioTaskExecutor::default()), evm_config, consensus, _primitives, @@ -290,11 +238,7 @@ where pub fn with_evm_config( self, evm_config: E, - ) -> RpcModuleBuilder - where - EvmConfig: 'static, - E: ConfigureEvm + Clone, - { + ) -> RpcModuleBuilder { let Self { provider, pool, executor, network, consensus, _primitives, .. } = self; RpcModuleBuilder { provider, network, pool, executor, evm_config, consensus, _primitives } } @@ -303,18 +247,26 @@ where pub fn with_consensus( self, consensus: C, - ) -> RpcModuleBuilder { + ) -> RpcModuleBuilder { let Self { provider, network, pool, executor, evm_config, _primitives, .. } = self; RpcModuleBuilder { provider, network, pool, executor, evm_config, consensus, _primitives } } /// Instantiates a new [`EthApiBuilder`] from the configured components. - pub fn eth_api_builder(&self) -> EthApiBuilder + #[expect(clippy::type_complexity)] + pub fn eth_api_builder( + &self, + ) -> EthApiBuilder< + RpcNodeCoreAdapter, + RpcConverter>, + > where - Provider: BlockReaderIdExt + Clone, + Provider: Clone, Pool: Clone, Network: Clone, EvmConfig: Clone, + RpcNodeCoreAdapter: + RpcNodeCore, Evm = EvmConfig>, { EthApiBuilder::new( self.provider.clone(), @@ -329,34 +281,37 @@ where /// Note: This spawns all necessary tasks. /// /// See also [`EthApiBuilder`]. - pub fn bootstrap_eth_api(&self) -> EthApi + #[expect(clippy::type_complexity)] + pub fn bootstrap_eth_api( + &self, + ) -> EthApi< + RpcNodeCoreAdapter, + RpcConverter>, + > where - Provider: BlockReaderIdExt - + StateProviderFactory - + CanonStateSubscriptions - + ChainSpecProvider - + Clone - + Unpin - + 'static, + Provider: Clone, Pool: Clone, - EvmConfig: Clone, Network: Clone, + EvmConfig: ConfigureEvm + Clone, + RpcNodeCoreAdapter: + RpcNodeCore, Evm = EvmConfig>, + RpcConverter>: RpcConvert, + (): PendingEnvBuilder, { self.eth_api_builder().build() } } -impl - RpcModuleBuilder +impl + RpcModuleBuilder where N: NodePrimitives, Provider: FullRpcProvider + CanonStateSubscriptions + AccountReader + ChangeSetReader, - Pool: TransactionPool + 'static, + Pool: TransactionPool + Clone + 'static, Network: NetworkInfo + Peers + Clone + 'static, - Tasks: TaskSpawner + Clone + 'static, EvmConfig: ConfigureEvm + 'static, Consensus: FullConsensus + Clone + 'static, { @@ -366,7 +321,6 @@ where /// This behaves exactly as [`RpcModuleBuilder::build`] for the [`TransportRpcModules`], but /// also configures the auth (engine api) server, which exposes a subset of the `eth_` /// namespace. - #[expect(clippy::type_complexity)] pub fn build_with_auth_server( self, module_config: TransportRpcModuleConfig, @@ -375,7 +329,7 @@ where ) -> ( TransportRpcModules, AuthRpcModule, - RpcRegistryInner, + RpcRegistryInner, ) where EthApi: FullEthApiServer, @@ -403,7 +357,7 @@ where self, config: RpcModuleConfig, eth: EthApi, - ) -> RpcRegistryInner + ) -> RpcRegistryInner where EthApi: EthApiTypes + 'static, { @@ -449,9 +403,9 @@ where } } -impl Default for RpcModuleBuilder { +impl Default for RpcModuleBuilder { fn default() -> Self { - Self::new((), (), (), (), (), ()) + Self::new((), (), (), Box::new(TokioTaskExecutor::default()), (), ()) } } @@ -499,7 +453,7 @@ pub struct RpcModuleConfigBuilder { impl RpcModuleConfigBuilder { /// Configures a custom eth namespace config - pub const fn eth(mut self, eth: EthConfig) -> Self { + pub fn eth(mut self, eth: EthConfig) -> Self { self.eth = Some(eth); self } @@ -539,7 +493,6 @@ pub struct RpcRegistryInner< Provider: BlockReader, Pool, Network, - Tasks, EthApi: EthApiTypes, EvmConfig, Consensus, @@ -547,10 +500,10 @@ pub struct RpcRegistryInner< provider: Provider, pool: Pool, network: Network, - executor: Tasks, + executor: Box, evm_config: EvmConfig, consensus: Consensus, - /// Holds a all `eth_` namespace handlers + /// Holds all `eth_` namespace handlers eth: EthHandlers, /// to put trace calls behind semaphore blocking_pool_guard: BlockingTaskGuard, @@ -562,8 +515,8 @@ pub struct RpcRegistryInner< // === impl RpcRegistryInner === -impl - RpcRegistryInner +impl + RpcRegistryInner where N: NodePrimitives, Provider: StateProviderFactory @@ -574,7 +527,6 @@ where + 'static, Pool: Send + Sync + Clone + 'static, Network: Clone + 'static, - Tasks: TaskSpawner + Clone + 'static, EthApi: EthApiTypes + 'static, EvmConfig: ConfigureEvm, { @@ -584,7 +536,7 @@ where provider: Provider, pool: Pool, network: Network, - executor: Tasks, + executor: Box, consensus: Consensus, config: RpcModuleConfig, evm_config: EvmConfig, @@ -595,7 +547,7 @@ where { let blocking_pool_guard = BlockingTaskGuard::new(config.eth.max_tracing_requests); - let eth = EthHandlers::bootstrap(config.eth, executor.clone(), eth_api); + let eth = EthHandlers::bootstrap(config.eth.clone(), executor.clone(), eth_api); Self { provider, @@ -612,8 +564,8 @@ where } } -impl - RpcRegistryInner +impl + RpcRegistryInner where Provider: BlockReader, EthApi: EthApiTypes, @@ -634,8 +586,8 @@ where } /// Returns a reference to the tasks type - pub const fn tasks(&self) -> &Tasks { - &self.executor + pub const fn tasks(&self) -> &(dyn TaskSpawner + 'static) { + &*self.executor } /// Returns a reference to the provider @@ -658,8 +610,8 @@ where } } -impl - RpcRegistryInner +impl + RpcRegistryInner where Network: NetworkInfo + Clone + 'static, EthApi: EthApiTypes, @@ -667,11 +619,12 @@ where EvmConfig: ConfigureEvm, { /// Instantiates `AdminApi` - pub fn admin_api(&self) -> AdminApi + pub fn admin_api(&self) -> AdminApi where Network: Peers, + Pool: TransactionPool + Clone + 'static, { - AdminApi::new(self.network.clone(), self.provider.chain_spec()) + AdminApi::new(self.network.clone(), self.provider.chain_spec(), self.pool.clone()) } /// Instantiates `Web3Api` @@ -683,6 +636,7 @@ where pub fn register_admin(&mut self) -> &mut Self where Network: Peers, + Pool: TransactionPool + Clone + 'static, { let adminapi = self.admin_api(); self.modules.insert(RethRpcModule::Admin, adminapi.into_rpc().into()); @@ -697,8 +651,8 @@ where } } -impl - RpcRegistryInner +impl + RpcRegistryInner where N: NodePrimitives, Provider: FullRpcProvider< @@ -707,14 +661,16 @@ where Receipt = N::Receipt, Transaction = N::SignedTx, > + AccountReader - + ChangeSetReader, + + ChangeSetReader + + CanonStateSubscriptions, Network: NetworkInfo + Peers + Clone + 'static, - Tasks: TaskSpawner + Clone + 'static, EthApi: EthApiServer< + RpcTxReq, RpcTransaction, RpcBlock, RpcReceipt, RpcHeader, + TxTy, > + EthApiTypes, EvmConfig: ConfigureEvm + 'static, { @@ -736,7 +692,7 @@ where /// If called outside of the tokio runtime. See also [`Self::eth_api`] pub fn register_ots(&mut self) -> &mut Self where - EthApi: TraceExt + EthTransactions, + EthApi: TraceExt + EthTransactions, { let otterscan_api = self.otterscan_api(); self.modules.insert(RethRpcModule::Ots, otterscan_api.into_rpc().into()); @@ -812,8 +768,8 @@ where } } -impl - RpcRegistryInner +impl + RpcRegistryInner where N: NodePrimitives, Provider: FullRpcProvider< @@ -824,7 +780,6 @@ where > + AccountReader + ChangeSetReader, Network: NetworkInfo + Peers + Clone + 'static, - Tasks: TaskSpawner + Clone + 'static, EthApi: EthApiTypes, EvmConfig: ConfigureEvm, { @@ -833,11 +788,12 @@ where /// # Panics /// /// If called outside of the tokio runtime. See also [`Self::eth_api`] - pub fn trace_api(&self) -> TraceApi - where - EthApi: TraceExt, - { - TraceApi::new(self.eth_api().clone(), self.blocking_pool_guard.clone(), self.eth_config) + pub fn trace_api(&self) -> TraceApi { + TraceApi::new( + self.eth_api().clone(), + self.blocking_pool_guard.clone(), + self.eth_config.clone(), + ) } /// Instantiates [`EthBundle`] Api @@ -858,16 +814,8 @@ where /// # Panics /// /// If called outside of the tokio runtime. See also [`Self::eth_api`] - pub fn debug_api(&self) -> DebugApi - where - EthApi: EthApiSpec + EthTransactions + TraceExt, - EvmConfig::Primitives: NodePrimitives>, - { - DebugApi::new( - self.eth_api().clone(), - self.blocking_pool_guard.clone(), - self.evm_config.clone(), - ) + pub fn debug_api(&self) -> DebugApi { + DebugApi::new(self.eth_api().clone(), self.blocking_pool_guard.clone()) } /// Instantiates `NetApi` @@ -885,22 +833,21 @@ where /// Instantiates `RethApi` pub fn reth_api(&self) -> RethApi { - RethApi::new(self.provider.clone(), Box::new(self.executor.clone())) + RethApi::new(self.provider.clone(), self.executor.clone()) } } -impl - RpcRegistryInner +impl + RpcRegistryInner where N: NodePrimitives, Provider: FullRpcProvider + CanonStateSubscriptions + AccountReader + ChangeSetReader, - Pool: TransactionPool + 'static, + Pool: TransactionPool + Clone + 'static, Network: NetworkInfo + Peers + Clone + 'static, - Tasks: TaskSpawner + Clone + 'static, - EthApi: FullEthApiServer, + EthApi: FullEthApiServer, EvmConfig: ConfigureEvm + 'static, Consensus: FullConsensus + Clone + 'static, { @@ -975,23 +922,22 @@ where let namespaces: Vec<_> = namespaces.collect(); namespaces .iter() - .copied() .map(|namespace| { self.modules - .entry(namespace) - .or_insert_with(|| match namespace { - RethRpcModule::Admin => { - AdminApi::new(self.network.clone(), self.provider.chain_spec()) - .into_rpc() - .into() - } - RethRpcModule::Debug => DebugApi::new( - eth_api.clone(), - self.blocking_pool_guard.clone(), - self.evm_config.clone(), + .entry(namespace.clone()) + .or_insert_with(|| match namespace.clone() { + RethRpcModule::Admin => AdminApi::new( + self.network.clone(), + self.provider.chain_spec(), + self.pool.clone(), ) .into_rpc() .into(), + RethRpcModule::Debug => { + DebugApi::new(eth_api.clone(), self.blocking_pool_guard.clone()) + .into_rpc() + .into() + } RethRpcModule::Eth => { // merge all eth handlers let mut module = eth_api.clone().into_rpc(); @@ -1015,14 +961,14 @@ where RethRpcModule::Trace => TraceApi::new( eth_api.clone(), self.blocking_pool_guard.clone(), - self.eth_config, + self.eth_config.clone(), ) .into_rpc() .into(), RethRpcModule::Web3 => Web3Api::new(self.network.clone()).into_rpc().into(), RethRpcModule::Txpool => TxPoolApi::new( self.eth.api.pool().clone(), - self.eth.api.tx_resp_builder().clone(), + dyn_clone::clone(self.eth.api.tx_resp_builder()), ) .into_rpc() .into(), @@ -1036,14 +982,16 @@ where .into(), RethRpcModule::Ots => OtterscanApi::new(eth_api.clone()).into_rpc().into(), RethRpcModule::Reth => { - RethApi::new(self.provider.clone(), Box::new(self.executor.clone())) + RethApi::new(self.provider.clone(), self.executor.clone()) .into_rpc() .into() } // only relevant for Ethereum and configured in `EthereumAddOns` // implementation // TODO: can we get rid of this here? - RethRpcModule::Flashbots => Default::default(), + // Custom modules are not handled here - they should be registered via + // extend_rpc_modules + RethRpcModule::Flashbots | RethRpcModule::Other(_) => Default::default(), RethRpcModule::Miner => MinerApi::default().into_rpc().into(), RethRpcModule::Mev => { EthSimBundle::new(eth_api.clone(), self.blocking_pool_guard.clone()) @@ -1071,13 +1019,15 @@ where #[derive(Debug)] pub struct RpcServerConfig { /// Configs for JSON-RPC Http. - http_server_config: Option>, + http_server_config: Option, /// Allowed CORS Domains for http http_cors_domains: Option, /// Address where to bind the http server to http_addr: Option, + /// Control whether http responses should be compressed + http_disable_compression: bool, /// Configs for WS server - ws_server_config: Option>, + ws_server_config: Option, /// Allowed CORS Domains for ws. ws_cors_domains: Option, /// Address where to bind the ws server to @@ -1089,7 +1039,7 @@ pub struct RpcServerConfig { /// JWT secret for authentication jwt_secret: Option, /// Configurable RPC middleware - rpc_middleware: RpcServiceBuilder, + rpc_middleware: RpcMiddleware, } // === impl RpcServerConfig === @@ -1101,25 +1051,26 @@ impl Default for RpcServerConfig { http_server_config: None, http_cors_domains: None, http_addr: None, + http_disable_compression: false, ws_server_config: None, ws_cors_domains: None, ws_addr: None, ipc_server_config: None, ipc_endpoint: None, jwt_secret: None, - rpc_middleware: RpcServiceBuilder::new(), + rpc_middleware: Default::default(), } } } impl RpcServerConfig { /// Creates a new config with only http set - pub fn http(config: ServerBuilder) -> Self { + pub fn http(config: ServerConfigBuilder) -> Self { Self::default().with_http(config) } /// Creates a new config with only ws set - pub fn ws(config: ServerBuilder) -> Self { + pub fn ws(config: ServerConfigBuilder) -> Self { Self::default().with_ws(config) } @@ -1132,7 +1083,7 @@ impl RpcServerConfig { /// /// Note: this always configures an [`EthSubscriptionIdProvider`] [`IdProvider`] for /// convenience. To set a custom [`IdProvider`], please use [`Self::with_id_provider`]. - pub fn with_http(mut self, config: ServerBuilder) -> Self { + pub fn with_http(mut self, config: ServerConfigBuilder) -> Self { self.http_server_config = Some(config.set_id_provider(EthSubscriptionIdProvider::default())); self @@ -1142,7 +1093,7 @@ impl RpcServerConfig { /// /// Note: this always configures an [`EthSubscriptionIdProvider`] [`IdProvider`] for /// convenience. To set a custom [`IdProvider`], please use [`Self::with_id_provider`]. - pub fn with_ws(mut self, config: ServerBuilder) -> Self { + pub fn with_ws(mut self, config: ServerConfigBuilder) -> Self { self.ws_server_config = Some(config.set_id_provider(EthSubscriptionIdProvider::default())); self } @@ -1159,11 +1110,12 @@ impl RpcServerConfig { impl RpcServerConfig { /// Configure rpc middleware - pub fn set_rpc_middleware(self, rpc_middleware: RpcServiceBuilder) -> RpcServerConfig { + pub fn set_rpc_middleware(self, rpc_middleware: T) -> RpcServerConfig { RpcServerConfig { http_server_config: self.http_server_config, http_cors_domains: self.http_cors_domains, http_addr: self.http_addr, + http_disable_compression: self.http_disable_compression, ws_server_config: self.ws_server_config, ws_cors_domains: self.ws_cors_domains, ws_addr: self.ws_addr, @@ -1185,6 +1137,12 @@ impl RpcServerConfig { self } + /// Configure whether HTTP responses should be compressed + pub const fn with_http_disable_compression(mut self, http_disable_compression: bool) -> Self { + self.http_disable_compression = http_disable_compression; + self + } + /// Configure the cors domains for HTTP pub fn with_http_cors(mut self, cors_domain: Option) -> Self { self.http_cors_domains = cors_domain; @@ -1216,11 +1174,11 @@ impl RpcServerConfig { where I: IdProvider + Clone + 'static, { - if let Some(http) = self.http_server_config { - self.http_server_config = Some(http.set_id_provider(id_provider.clone())); + if let Some(config) = self.http_server_config { + self.http_server_config = Some(config.set_id_provider(id_provider.clone())); } - if let Some(ws) = self.ws_server_config { - self.ws_server_config = Some(ws.set_id_provider(id_provider.clone())); + if let Some(config) = self.ws_server_config { + self.ws_server_config = Some(config.set_id_provider(id_provider.clone())); } if let Some(ipc) = self.ipc_server_config { self.ipc_server_config = Some(ipc.set_id_provider(id_provider)); @@ -1243,6 +1201,23 @@ impl RpcServerConfig { self } + /// Configures a custom tokio runtime for the rpc server. + pub fn with_tokio_runtime(mut self, tokio_runtime: Option) -> Self { + let Some(tokio_runtime) = tokio_runtime else { return self }; + if let Some(http_server_config) = self.http_server_config { + self.http_server_config = + Some(http_server_config.custom_tokio_runtime(tokio_runtime.clone())); + } + if let Some(ws_server_config) = self.ws_server_config { + self.ws_server_config = + Some(ws_server_config.custom_tokio_runtime(tokio_runtime.clone())); + } + if let Some(ipc_server_config) = self.ipc_server_config { + self.ipc_server_config = Some(ipc_server_config.custom_tokio_runtime(tokio_runtime)); + } + self + } + /// Returns true if any server is configured. /// /// If no server is configured, no server will be launched on [`RpcServerConfig::start`]. @@ -1279,8 +1254,12 @@ impl RpcServerConfig { /// Returns a [`CompressionLayer`] that adds compression support (gzip, deflate, brotli, zstd) /// based on the client's `Accept-Encoding` header - fn maybe_compression_layer() -> Option { - Some(CompressionLayer::new()) + fn maybe_compression_layer(disable_compression: bool) -> Option { + if disable_compression { + None + } else { + Some(CompressionLayer::new()) + } } /// Builds and starts the configured server(s): http, ws, ipc. @@ -1290,9 +1269,7 @@ impl RpcServerConfig { /// Returns the [`RpcServerHandle`] with the handle to the started servers. pub async fn start(self, modules: &TransportRpcModules) -> Result where - RpcMiddleware: Layer> + Clone + Send + 'static, - for<'a> >>::Service: - Send + Sync + 'static + RpcServiceT<'a>, + RpcMiddleware: RethRpcMiddleware, { let mut http_handle = None; let mut ws_handle = None; @@ -1342,24 +1319,29 @@ impl RpcServerConfig { // we merge this into one server using the http setup modules.config.ensure_ws_http_identical()?; - if let Some(builder) = self.http_server_config { - let server = builder + if let Some(config) = self.http_server_config { + let server = ServerBuilder::new() .set_http_middleware( tower::ServiceBuilder::new() .option_layer(Self::maybe_cors_layer(cors)?) .option_layer(Self::maybe_jwt_layer(self.jwt_secret)) - .option_layer(Self::maybe_compression_layer()), + .option_layer(Self::maybe_compression_layer( + self.http_disable_compression, + )), ) .set_rpc_middleware( - self.rpc_middleware.clone().layer( - modules - .http - .as_ref() - .or(modules.ws.as_ref()) - .map(RpcRequestMetrics::same_port) - .unwrap_or_default(), - ), + RpcServiceBuilder::default() + .layer( + modules + .http + .as_ref() + .or(modules.ws.as_ref()) + .map(RpcRequestMetrics::same_port) + .unwrap_or_default(), + ) + .layer(self.rpc_middleware.clone()), ) + .set_config(config.build()) .build(http_socket_addr) .await .map_err(|err| { @@ -1390,18 +1372,18 @@ impl RpcServerConfig { let mut http_local_addr = None; let mut http_server = None; - if let Some(builder) = self.ws_server_config { - let server = builder - .ws_only() + if let Some(config) = self.ws_server_config { + let server = ServerBuilder::new() + .set_config(config.ws_only().build()) .set_http_middleware( tower::ServiceBuilder::new() .option_layer(Self::maybe_cors_layer(self.ws_cors_domains.clone())?) .option_layer(Self::maybe_jwt_layer(self.jwt_secret)), ) .set_rpc_middleware( - self.rpc_middleware - .clone() - .layer(modules.ws.as_ref().map(RpcRequestMetrics::ws).unwrap_or_default()), + RpcServiceBuilder::default() + .layer(modules.ws.as_ref().map(RpcRequestMetrics::ws).unwrap_or_default()) + .layer(self.rpc_middleware.clone()), ) .build(ws_socket_addr) .await @@ -1415,19 +1397,21 @@ impl RpcServerConfig { ws_server = Some(server); } - if let Some(builder) = self.http_server_config { - let server = builder - .http_only() + if let Some(config) = self.http_server_config { + let server = ServerBuilder::new() + .set_config(config.http_only().build()) .set_http_middleware( tower::ServiceBuilder::new() .option_layer(Self::maybe_cors_layer(self.http_cors_domains.clone())?) .option_layer(Self::maybe_jwt_layer(self.jwt_secret)) - .option_layer(Self::maybe_compression_layer()), + .option_layer(Self::maybe_compression_layer(self.http_disable_compression)), ) .set_rpc_middleware( - self.rpc_middleware.clone().layer( - modules.http.as_ref().map(RpcRequestMetrics::http).unwrap_or_default(), - ), + RpcServiceBuilder::default() + .layer( + modules.http.as_ref().map(RpcRequestMetrics::http).unwrap_or_default(), + ) + .layer(self.rpc_middleware.clone()), ) .build(http_socket_addr) .await @@ -1596,9 +1580,9 @@ impl TransportRpcModuleConfig { let ws_modules = self.ws.as_ref().map(RpcModuleSelection::to_selection).unwrap_or_default(); - let http_not_ws = http_modules.difference(&ws_modules).copied().collect(); - let ws_not_http = ws_modules.difference(&http_modules).copied().collect(); - let overlap = http_modules.intersection(&ws_modules).copied().collect(); + let http_not_ws = http_modules.difference(&ws_modules).cloned().collect(); + let ws_not_http = ws_modules.difference(&http_modules).cloned().collect(); + let overlap = http_modules.intersection(&ws_modules).cloned().collect(); Err(WsHttpSamePortError::ConflictingModules(Box::new(ConflictingModules { overlap, @@ -1734,7 +1718,7 @@ impl TransportRpcModules { /// Returns all unique endpoints installed for the given module. /// /// Note: In case of duplicate method names this only record the first occurrence. - pub fn methods_by_module(&self, module: RethRpcModule) -> Methods { + pub fn methods_by_module(&self, module: RethRpcModule) -> Methods { self.methods_by(|name| name.starts_with(module.as_str())) } @@ -1934,6 +1918,73 @@ impl TransportRpcModules { self.replace_ipc(other)?; Ok(true) } + + /// Adds or replaces given [`Methods`] in http module. + /// + /// Returns `true` if the methods were replaced or added, `false` otherwise. + pub fn add_or_replace_http( + &mut self, + other: impl Into, + ) -> Result { + let other = other.into(); + self.remove_http_methods(other.method_names()); + self.merge_http(other) + } + + /// Adds or replaces given [`Methods`] in ws module. + /// + /// Returns `true` if the methods were replaced or added, `false` otherwise. + pub fn add_or_replace_ws( + &mut self, + other: impl Into, + ) -> Result { + let other = other.into(); + self.remove_ws_methods(other.method_names()); + self.merge_ws(other) + } + + /// Adds or replaces given [`Methods`] in ipc module. + /// + /// Returns `true` if the methods were replaced or added, `false` otherwise. + pub fn add_or_replace_ipc( + &mut self, + other: impl Into, + ) -> Result { + let other = other.into(); + self.remove_ipc_methods(other.method_names()); + self.merge_ipc(other) + } + + /// Adds or replaces given [`Methods`] in all configured network modules. + pub fn add_or_replace_configured( + &mut self, + other: impl Into, + ) -> Result<(), RegisterMethodError> { + let other = other.into(); + self.add_or_replace_http(other.clone())?; + self.add_or_replace_ws(other.clone())?; + self.add_or_replace_ipc(other)?; + Ok(()) + } + /// Adds or replaces the given [`Methods`] in the transport modules where the specified + /// [`RethRpcModule`] is configured. + pub fn add_or_replace_if_module_configured( + &mut self, + module: RethRpcModule, + other: impl Into, + ) -> Result<(), RegisterMethodError> { + let other = other.into(); + if self.module_config().contains_http(&module) { + self.add_or_replace_http(other.clone())?; + } + if self.module_config().contains_ws(&module) { + self.add_or_replace_ws(other.clone())?; + } + if self.module_config().contains_ipc(&module) { + self.add_or_replace_ipc(other)?; + } + Ok(()) + } } /// Returns the methods installed in the given module that match the given filter. @@ -2067,6 +2118,21 @@ impl RpcServerHandle { self.new_http_provider_for() } + /// Returns a new [`alloy_network::Ethereum`] http provider with its recommended fillers and + /// installed wallet. + pub fn eth_http_provider_with_wallet( + &self, + wallet: W, + ) -> Option + Clone + Unpin + 'static> + where + W: IntoWallet, + { + let rpc_url = self.http_url()?; + let provider = + ProviderBuilder::new().wallet(wallet).connect_http(rpc_url.parse().expect("valid url")); + Some(provider) + } + /// Returns an http provider from the rpc server handle for the /// specified [`alloy_network::Network`]. /// @@ -2089,6 +2155,24 @@ impl RpcServerHandle { self.new_ws_provider_for().await } + /// Returns a new [`alloy_network::Ethereum`] ws provider with its recommended fillers and + /// installed wallet. + pub async fn eth_ws_provider_with_wallet( + &self, + wallet: W, + ) -> Option + Clone + Unpin + 'static> + where + W: IntoWallet, + { + let rpc_url = self.ws_url()?; + let provider = ProviderBuilder::new() + .wallet(wallet) + .connect(&rpc_url) + .await + .expect("failed to create ws client"); + Some(provider) + } + /// Returns an ws provider from the rpc server handle for the /// specified [`alloy_network::Network`]. /// @@ -2457,4 +2541,56 @@ mod tests { assert!(modules.ipc.as_ref().unwrap().method("anything").is_some()); assert!(modules.ws.as_ref().unwrap().method("anything").is_some()); } + + #[test] + fn test_add_or_replace_if_module_configured() { + // Create a config that enables RethRpcModule::Eth for HTTP and WS, but NOT IPC + let config = TransportRpcModuleConfig::default() + .with_http([RethRpcModule::Eth]) + .with_ws([RethRpcModule::Eth]); + + // Create HTTP module with an existing method (to test "replace") + let mut http_module = RpcModule::new(()); + http_module.register_method("eth_existing", |_, _, _| "original").unwrap(); + + // Create WS module with the same existing method + let mut ws_module = RpcModule::new(()); + ws_module.register_method("eth_existing", |_, _, _| "original").unwrap(); + + // Create IPC module (empty, to ensure no changes) + let ipc_module = RpcModule::new(()); + + // Set up TransportRpcModules with the config and modules + let mut modules = TransportRpcModules { + config, + http: Some(http_module), + ws: Some(ws_module), + ipc: Some(ipc_module), + }; + + // Create new methods: one to replace an existing method, one to add a new one + let mut new_module = RpcModule::new(()); + new_module.register_method("eth_existing", |_, _, _| "replaced").unwrap(); // Replace + new_module.register_method("eth_new", |_, _, _| "added").unwrap(); // Add + let new_methods: Methods = new_module.into(); + + // Call the function for RethRpcModule::Eth + let result = modules.add_or_replace_if_module_configured(RethRpcModule::Eth, new_methods); + assert!(result.is_ok(), "Function should succeed"); + + // Verify HTTP: existing method still exists (replaced), new method added + let http = modules.http.as_ref().unwrap(); + assert!(http.method("eth_existing").is_some()); + assert!(http.method("eth_new").is_some()); + + // Verify WS: existing method still exists (replaced), new method added + let ws = modules.ws.as_ref().unwrap(); + assert!(ws.method("eth_existing").is_some()); + assert!(ws.method("eth_new").is_some()); + + // Verify IPC: no changes (Eth not configured for IPC) + let ipc = modules.ipc.as_ref().unwrap(); + assert!(ipc.method("eth_existing").is_none()); + assert!(ipc.method("eth_new").is_none()); + } } diff --git a/crates/rpc/rpc-builder/src/metrics.rs b/crates/rpc/rpc-builder/src/metrics.rs index f38dae0ce63..f32d90ed095 100644 --- a/crates/rpc/rpc-builder/src/metrics.rs +++ b/crates/rpc/rpc-builder/src/metrics.rs @@ -1,4 +1,9 @@ -use jsonrpsee::{server::middleware::rpc::RpcServiceT, types::Request, MethodResponse, RpcModule}; +use jsonrpsee::{ + core::middleware::{Batch, Notification}, + server::middleware::rpc::RpcServiceT, + types::Request, + MethodResponse, RpcModule, +}; use reth_metrics::{ metrics::{Counter, Histogram}, Metrics, @@ -99,13 +104,15 @@ impl RpcRequestMetricsService { } } -impl<'a, S> RpcServiceT<'a> for RpcRequestMetricsService +impl RpcServiceT for RpcRequestMetricsService where - S: RpcServiceT<'a> + Send + Sync + Clone + 'static, + S: RpcServiceT + Send + Sync + Clone + 'static, { - type Future = MeteredRequestFuture; + type MethodResponse = S::MethodResponse; + type NotificationResponse = S::NotificationResponse; + type BatchResponse = S::BatchResponse; - fn call(&self, req: Request<'a>) -> Self::Future { + fn call<'a>(&self, req: Request<'a>) -> impl Future + Send + 'a { self.metrics.inner.connection_metrics.requests_started_total.increment(1); let call_metrics = self.metrics.inner.call_metrics.get_key_value(req.method.as_ref()); if let Some((_, call_metrics)) = &call_metrics { @@ -118,6 +125,17 @@ where method: call_metrics.map(|(method, _)| *method), } } + + fn batch<'a>(&self, req: Batch<'a>) -> impl Future + Send + 'a { + self.inner.batch(req) + } + + fn notification<'a>( + &self, + n: Notification<'a>, + ) -> impl Future + Send + 'a { + self.inner.notification(n) + } } impl Drop for RpcRequestMetricsService { diff --git a/crates/rpc/rpc-builder/src/middleware.rs b/crates/rpc/rpc-builder/src/middleware.rs new file mode 100644 index 00000000000..c03f63501fc --- /dev/null +++ b/crates/rpc/rpc-builder/src/middleware.rs @@ -0,0 +1,37 @@ +use jsonrpsee::server::middleware::rpc::RpcService; +use tower::Layer; + +/// A Helper alias trait for the RPC middleware supported by the server. +pub trait RethRpcMiddleware: + Layer< + RpcService, + Service: jsonrpsee::server::middleware::rpc::RpcServiceT< + MethodResponse = jsonrpsee::MethodResponse, + BatchResponse = jsonrpsee::MethodResponse, + NotificationResponse = jsonrpsee::MethodResponse, + > + Send + + Sync + + Clone + + 'static, + > + Clone + + Send + + 'static +{ +} + +impl RethRpcMiddleware for T where + T: Layer< + RpcService, + Service: jsonrpsee::server::middleware::rpc::RpcServiceT< + MethodResponse = jsonrpsee::MethodResponse, + BatchResponse = jsonrpsee::MethodResponse, + NotificationResponse = jsonrpsee::MethodResponse, + > + Send + + Sync + + Clone + + 'static, + > + Clone + + Send + + 'static +{ +} diff --git a/crates/rpc/rpc-builder/src/rate_limiter.rs b/crates/rpc/rpc-builder/src/rate_limiter.rs index 85df0eee61c..902f480377d 100644 --- a/crates/rpc/rpc-builder/src/rate_limiter.rs +++ b/crates/rpc/rpc-builder/src/rate_limiter.rs @@ -1,6 +1,6 @@ //! [`jsonrpsee`] helper layer for rate limiting certain methods. -use jsonrpsee::{server::middleware::rpc::RpcServiceT, types::Request, MethodResponse}; +use jsonrpsee::{server::middleware::rpc::RpcServiceT, types::Request}; use std::{ future::Future, pin::Pin, @@ -61,13 +61,15 @@ impl RpcRequestRateLimitingService { } } -impl<'a, S> RpcServiceT<'a> for RpcRequestRateLimitingService +impl RpcServiceT for RpcRequestRateLimitingService where - S: RpcServiceT<'a> + Send + Sync + Clone + 'static, + S: RpcServiceT + Send + Sync + Clone + 'static, { - type Future = RateLimitingRequestFuture; + type MethodResponse = S::MethodResponse; + type NotificationResponse = S::NotificationResponse; + type BatchResponse = S::BatchResponse; - fn call(&self, req: Request<'a>) -> Self::Future { + fn call<'a>(&self, req: Request<'a>) -> impl Future + Send + 'a { let method_name = req.method_name(); if method_name.starts_with("trace_") || method_name.starts_with("debug_") { RateLimitingRequestFuture { @@ -81,6 +83,20 @@ where RateLimitingRequestFuture { fut: self.inner.call(req), guard: None, permit: None } } } + + fn batch<'a>( + &self, + requests: jsonrpsee::core::middleware::Batch<'a>, + ) -> impl Future + Send + 'a { + self.inner.batch(requests) + } + + fn notification<'a>( + &self, + n: jsonrpsee::core::middleware::Notification<'a>, + ) -> impl Future + Send + 'a { + self.inner.notification(n) + } } /// Response future. @@ -98,7 +114,7 @@ impl std::fmt::Debug for RateLimitingRequestFuture { } } -impl> Future for RateLimitingRequestFuture { +impl Future for RateLimitingRequestFuture { type Output = F::Output; fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { diff --git a/crates/rpc/rpc-builder/tests/it/http.rs b/crates/rpc/rpc-builder/tests/it/http.rs index a038285ea15..6be4d5d965d 100644 --- a/crates/rpc/rpc-builder/tests/it/http.rs +++ b/crates/rpc/rpc-builder/tests/it/http.rs @@ -2,7 +2,7 @@ //! Standalone http tests use crate::utils::{launch_http, launch_http_ws, launch_ws}; -use alloy_eips::{BlockId, BlockNumberOrTag}; +use alloy_eips::{eip1898::LenientBlockNumberOrTag, BlockId, BlockNumberOrTag}; use alloy_primitives::{hex_literal::hex, Address, Bytes, TxHash, B256, B64, U256, U64}; use alloy_rpc_types_eth::{ transaction::TransactionRequest, Block, FeeHistory, Filter, Header, Index, Log, @@ -18,7 +18,7 @@ use jsonrpsee::{ rpc_params, types::error::ErrorCode, }; -use reth_ethereum_primitives::Receipt; +use reth_ethereum_primitives::{Receipt, TransactionSigned}; use reth_network_peers::NodeRecord; use reth_rpc_api::{ clients::{AdminApiClient, EthApiClient}, @@ -176,24 +176,38 @@ where .unwrap(); // Implemented - EthApiClient::::protocol_version(client).await.unwrap(); - EthApiClient::::chain_id(client).await.unwrap(); - EthApiClient::::accounts(client).await.unwrap(); - EthApiClient::::get_account( + EthApiClient::::protocol_version( client, - address, - block_number.into(), ) .await .unwrap(); - EthApiClient::::block_number(client).await.unwrap(); - EthApiClient::::get_code(client, address, None) + EthApiClient::::chain_id(client) + .await + .unwrap(); + EthApiClient::::accounts(client) .await .unwrap(); - EthApiClient::::send_raw_transaction(client, tx) + EthApiClient::::get_account( + client, + address, + block_number.into(), + ) + .await + .unwrap(); + EthApiClient::::block_number(client) .await .unwrap(); - EthApiClient::::fee_history( + EthApiClient::::get_code( + client, address, None, + ) + .await + .unwrap(); + EthApiClient::::send_raw_transaction( + client, tx, + ) + .await + .unwrap(); + EthApiClient::::fee_history( client, U64::from(0), block_number, @@ -201,13 +215,17 @@ where ) .await .unwrap(); - EthApiClient::::balance(client, address, None) - .await - .unwrap(); - EthApiClient::::transaction_count(client, address, None) - .await - .unwrap(); - EthApiClient::::storage_at( + EthApiClient::::balance( + client, address, None, + ) + .await + .unwrap(); + EthApiClient::::transaction_count( + client, address, None, + ) + .await + .unwrap(); + EthApiClient::::storage_at( client, address, U256::default().into(), @@ -215,72 +233,80 @@ where ) .await .unwrap(); - EthApiClient::::block_by_hash(client, hash, false) - .await - .unwrap(); - EthApiClient::::block_by_number( + EthApiClient::::block_by_hash( + client, hash, false, + ) + .await + .unwrap(); + EthApiClient::::block_by_number( client, block_number, false, ) .await .unwrap(); - EthApiClient::::block_transaction_count_by_number( + EthApiClient::::block_transaction_count_by_number( client, block_number, ) .await .unwrap(); - EthApiClient::::block_transaction_count_by_hash( + EthApiClient::::block_transaction_count_by_hash( client, hash, ) .await .unwrap(); - EthApiClient::::block_uncles_count_by_hash(client, hash) + EthApiClient::::block_uncles_count_by_hash(client, hash) .await .unwrap(); - EthApiClient::::block_uncles_count_by_number( + EthApiClient::::block_uncles_count_by_number( client, block_number, ) .await .unwrap(); - EthApiClient::::uncle_by_block_hash_and_index( + EthApiClient::::uncle_by_block_hash_and_index( client, hash, index, ) .await .unwrap(); - EthApiClient::::uncle_by_block_number_and_index( + EthApiClient::::uncle_by_block_number_and_index( client, block_number, index, ) .await .unwrap(); - EthApiClient::::sign(client, address, bytes.clone()) - .await - .unwrap_err(); - EthApiClient::::sign_typed_data( + EthApiClient::::sign( + client, + address, + bytes.clone(), + ) + .await + .unwrap_err(); + EthApiClient::::sign_typed_data( client, address, typed_data, ) .await .unwrap_err(); - EthApiClient::::transaction_by_hash(client, tx_hash) - .await - .unwrap(); - EthApiClient::::transaction_by_block_hash_and_index( + EthApiClient::::transaction_by_hash( + client, tx_hash, + ) + .await + .unwrap(); + EthApiClient::::transaction_by_block_hash_and_index( client, hash, index, ) .await .unwrap(); - EthApiClient::::transaction_by_block_number_and_index( + EthApiClient::::transaction_by_block_number_and_index( client, block_number, index, ) .await .unwrap(); - EthApiClient::::create_access_list( + EthApiClient::::create_access_list( client, call_request.clone(), Some(block_number.into()), @@ -288,7 +314,7 @@ where ) .await .unwrap_err(); - EthApiClient::::estimate_gas( + EthApiClient::::estimate_gas( client, call_request.clone(), Some(block_number.into()), @@ -296,7 +322,7 @@ where ) .await .unwrap_err(); - EthApiClient::::call( + EthApiClient::::call( client, call_request.clone(), Some(block_number.into()), @@ -305,56 +331,107 @@ where ) .await .unwrap_err(); - EthApiClient::::syncing(client).await.unwrap(); - EthApiClient::::send_transaction( + EthApiClient::::syncing(client) + .await + .unwrap(); + EthApiClient::::send_transaction( client, transaction_request.clone(), ) .await .unwrap_err(); - EthApiClient::::sign_transaction( + EthApiClient::::sign_transaction( client, transaction_request, ) .await .unwrap_err(); - EthApiClient::::hashrate(client).await.unwrap(); - EthApiClient::::submit_hashrate( + EthApiClient::::hashrate(client) + .await + .unwrap(); + EthApiClient::::submit_hashrate( client, U256::default(), B256::default(), ) .await .unwrap(); - EthApiClient::::gas_price(client).await.unwrap_err(); - EthApiClient::::max_priority_fee_per_gas(client) + EthApiClient::::gas_price(client) .await .unwrap_err(); - EthApiClient::::get_proof(client, address, vec![], None) + EthApiClient::::max_priority_fee_per_gas(client) .await - .unwrap(); + .unwrap_err(); + EthApiClient::::get_proof( + client, + address, + vec![], + None, + ) + .await + .unwrap(); // Unimplemented - assert!(is_unimplemented( - EthApiClient::::author(client).await.err().unwrap() - )); - assert!(is_unimplemented( - EthApiClient::::is_mining(client).await.err().unwrap() - )); - assert!(is_unimplemented( - EthApiClient::::get_work(client).await.err().unwrap() - )); - assert!(is_unimplemented( - EthApiClient::::submit_work( - client, - B64::default(), - B256::default(), - B256::default() + assert!( + is_unimplemented( + EthApiClient::< + TransactionRequest, + Transaction, + Block, + Receipt, + Header, + TransactionSigned, + >::author(client) + .await + .err() + .unwrap() ) - .await - .err() - .unwrap() - )); + ); + assert!( + is_unimplemented( + EthApiClient::< + TransactionRequest, + Transaction, + Block, + Receipt, + Header, + TransactionSigned, + >::is_mining(client) + .await + .err() + .unwrap() + ) + ); + assert!( + is_unimplemented( + EthApiClient::< + TransactionRequest, + Transaction, + Block, + Receipt, + Header, + TransactionSigned, + >::get_work(client) + .await + .err() + .unwrap() + ) + ); + assert!( + is_unimplemented( + EthApiClient::< + TransactionRequest, + Transaction, + Block, + Receipt, + Header, + TransactionSigned, + >::submit_work(client, B64::default(), B256::default(), B256::default()) + .await + .err() + .unwrap() + ) + ); EthCallBundleApiClient::call_bundle(client, Default::default()).await.unwrap_err(); } @@ -364,11 +441,11 @@ where { let block_id = BlockId::Number(BlockNumberOrTag::default()); - DebugApiClient::raw_header(client, block_id).await.unwrap(); - DebugApiClient::raw_block(client, block_id).await.unwrap_err(); - DebugApiClient::raw_transaction(client, B256::default()).await.unwrap(); - DebugApiClient::raw_receipts(client, block_id).await.unwrap(); - assert!(is_unimplemented(DebugApiClient::bad_blocks(client).await.err().unwrap())); + DebugApiClient::::raw_header(client, block_id).await.unwrap(); + DebugApiClient::::raw_block(client, block_id).await.unwrap_err(); + DebugApiClient::::raw_transaction(client, B256::default()).await.unwrap(); + DebugApiClient::::raw_receipts(client, block_id).await.unwrap(); + DebugApiClient::::bad_blocks(client).await.unwrap(); } async fn test_basic_net_calls(client: &C) @@ -395,22 +472,39 @@ where count: None, }; - TraceApiClient::trace_raw_transaction(client, Bytes::default(), HashSet::default(), None) - .await - .unwrap_err(); - TraceApiClient::trace_call_many(client, vec![], Some(BlockNumberOrTag::Latest.into())) - .await - .unwrap_err(); - TraceApiClient::replay_transaction(client, B256::default(), HashSet::default()) - .await - .err() - .unwrap(); - TraceApiClient::trace_block(client, block_id).await.unwrap_err(); - TraceApiClient::replay_block_transactions(client, block_id, HashSet::default()) - .await - .unwrap_err(); + TraceApiClient::::trace_raw_transaction( + client, + Bytes::default(), + HashSet::default(), + None, + ) + .await + .unwrap_err(); + TraceApiClient::::trace_call_many( + client, + vec![], + Some(BlockNumberOrTag::Latest.into()), + ) + .await + .unwrap_err(); + TraceApiClient::::replay_transaction( + client, + B256::default(), + HashSet::default(), + ) + .await + .err() + .unwrap(); + TraceApiClient::::trace_block(client, block_id).await.unwrap_err(); + TraceApiClient::::replay_block_transactions( + client, + block_id, + HashSet::default(), + ) + .await + .unwrap_err(); - TraceApiClient::trace_filter(client, trace_filter).await.unwrap(); + TraceApiClient::::trace_filter(client, trace_filter).await.unwrap(); } async fn test_basic_web3_calls(client: &C) @@ -434,9 +528,12 @@ where let nonce = 1; let block_hash = B256::default(); - OtterscanClient::::get_header_by_number(client, block_number) - .await - .unwrap(); + OtterscanClient::::get_header_by_number( + client, + LenientBlockNumberOrTag::new(BlockNumberOrTag::Number(block_number)), + ) + .await + .unwrap(); OtterscanClient::::has_code(client, address, None).await.unwrap(); OtterscanClient::::has_code(client, address, Some(block_number.into())) @@ -451,7 +548,13 @@ where OtterscanClient::::trace_transaction(client, tx_hash).await.unwrap(); - OtterscanClient::::get_block_details(client, block_number) + OtterscanClient::::get_block_details( + client, + LenientBlockNumberOrTag::new(BlockNumberOrTag::Number(block_number)), + ) + .await + .unwrap_err(); + OtterscanClient::::get_block_details(client, Default::default()) .await .unwrap_err(); @@ -461,7 +564,7 @@ where OtterscanClient::::get_block_transactions( client, - block_number, + LenientBlockNumberOrTag::new(BlockNumberOrTag::Number(block_number)), page_number, page_size, ) @@ -473,7 +576,7 @@ where OtterscanClient::::search_transactions_before( client, address, - block_number, + LenientBlockNumberOrTag::new(BlockNumberOrTag::Number(block_number)), page_size, ) .await @@ -484,7 +587,7 @@ where OtterscanClient::::search_transactions_after( client, address, - block_number, + LenientBlockNumberOrTag::new(BlockNumberOrTag::Number(block_number)), page_size, ) .await @@ -1591,3 +1694,47 @@ async fn test_eth_fee_history_raw() { ) .await; } + +#[tokio::test(flavor = "multi_thread")] +async fn test_debug_db_get() { + reth_tracing::init_test_tracing(); + + let handle = launch_http(vec![RethRpcModule::Debug]).await; + let client = handle.http_client().unwrap(); + + let valid_test_cases = [ + "0x630000000000000000000000000000000000000000000000000000000000000000", + "c00000000000000000000000000000000", + ]; + + for key in valid_test_cases { + DebugApiClient::<()>::debug_db_get(&client, key.into()).await.unwrap(); + } + + // Invalid test cases + let test_cases = [ + ("0x0000", "Key must be 33 bytes, got 2"), + ("00", "Key must be 33 bytes, got 2"), + ( + "0x000000000000000000000000000000000000000000000000000000000000000000", + "Key prefix must be 0x63", + ), + ("000000000000000000000000000000000", "Key prefix must be 0x63"), + ("0xc0000000000000000000000000000000000000000000000000000000000000000", "Invalid hex key"), + ]; + + let match_error_msg = |err: jsonrpsee::core::client::Error, expected: String| -> bool { + match err { + jsonrpsee::core::client::Error::Call(error_obj) => { + error_obj.code() == ErrorCode::InvalidParams.code() && + error_obj.message() == expected + } + _ => false, + } + }; + + for (key, expected) in test_cases { + let err = DebugApiClient::<()>::debug_db_get(&client, key.into()).await.unwrap_err(); + assert!(match_error_msg(err, expected.into())); + } +} diff --git a/crates/rpc/rpc-builder/tests/it/middleware.rs b/crates/rpc/rpc-builder/tests/it/middleware.rs index dcf5d43a2c5..9a70356bcac 100644 --- a/crates/rpc/rpc-builder/tests/it/middleware.rs +++ b/crates/rpc/rpc-builder/tests/it/middleware.rs @@ -1,16 +1,16 @@ use crate::utils::{test_address, test_rpc_builder}; -use alloy_rpc_types_eth::{Block, Header, Receipt, Transaction}; +use alloy_rpc_types_eth::{Block, Header, Receipt, Transaction, TransactionRequest}; use jsonrpsee::{ - server::{middleware::rpc::RpcServiceT, RpcServiceBuilder}, + core::middleware::{Batch, Notification}, + server::middleware::rpc::RpcServiceT, types::Request, - MethodResponse, }; +use reth_ethereum_primitives::TransactionSigned; use reth_rpc_builder::{RpcServerConfig, TransportRpcModuleConfig}; use reth_rpc_eth_api::EthApiClient; use reth_rpc_server_types::RpcModuleSelection; use std::{ future::Future, - pin::Pin, sync::{ atomic::{AtomicUsize, Ordering}, Arc, @@ -37,13 +37,15 @@ struct MyMiddlewareService { count: Arc, } -impl<'a, S> RpcServiceT<'a> for MyMiddlewareService +impl RpcServiceT for MyMiddlewareService where - S: RpcServiceT<'a> + Send + Sync + Clone + 'static, + S: RpcServiceT + Send + Sync + Clone + 'static, { - type Future = Pin + Send + 'a>>; + type MethodResponse = S::MethodResponse; + type NotificationResponse = S::NotificationResponse; + type BatchResponse = S::BatchResponse; - fn call(&self, req: Request<'a>) -> Self::Future { + fn call<'a>(&self, req: Request<'a>) -> impl Future + Send + 'a { tracing::info!("MyMiddleware processed call {}", req.method); let count = self.count.clone(); let service = self.service.clone(); @@ -54,6 +56,17 @@ where rp }) } + + fn batch<'a>(&self, req: Batch<'a>) -> impl Future + Send + 'a { + self.service.batch(req) + } + + fn notification<'a>( + &self, + n: Notification<'a>, + ) -> impl Future + Send + 'a { + self.service.notification(n) + } } #[tokio::test(flavor = "multi_thread")] @@ -67,13 +80,17 @@ async fn test_rpc_middleware() { let handle = RpcServerConfig::http(Default::default()) .with_http_address(test_address()) - .set_rpc_middleware(RpcServiceBuilder::new().layer(mylayer.clone())) + .set_rpc_middleware(mylayer.clone()) .start(&modules) .await .unwrap(); let client = handle.http_client().unwrap(); - EthApiClient::::protocol_version(&client).await.unwrap(); + EthApiClient::::protocol_version( + &client, + ) + .await + .unwrap(); let count = mylayer.count.load(Ordering::Relaxed); assert_eq!(count, 1); } diff --git a/crates/rpc/rpc-builder/tests/it/utils.rs b/crates/rpc/rpc-builder/tests/it/utils.rs index 3b21e848ce8..673a1f79fc2 100644 --- a/crates/rpc/rpc-builder/tests/it/utils.rs +++ b/crates/rpc/rpc-builder/tests/it/utils.rs @@ -1,11 +1,10 @@ -use std::net::{Ipv4Addr, SocketAddr, SocketAddrV4}; - use alloy_rpc_types_engine::{ClientCode, ClientVersionV1}; use reth_chainspec::MAINNET; use reth_consensus::noop::NoopConsensus; -use reth_engine_primitives::BeaconConsensusEngineHandle; +use reth_engine_primitives::ConsensusEngineHandle; use reth_ethereum_engine_primitives::EthEngineTypes; use reth_ethereum_primitives::EthPrimitives; +use std::net::{Ipv4Addr, SocketAddr, SocketAddrV4}; use reth_evm_ethereum::EthEvmConfig; use reth_network_api::noop::NoopNetwork; @@ -35,7 +34,7 @@ pub const fn test_address() -> SocketAddr { pub async fn launch_auth(secret: JwtSecret) -> AuthServerHandle { let config = AuthServerConfig::builder(secret).socket_addr(test_address()).build(); let (tx, _rx) = unbounded_channel(); - let beacon_engine_handle = BeaconConsensusEngineHandle::::new(tx); + let beacon_engine_handle = ConsensusEngineHandle::::new(tx); let client = ClientVersionV1 { code: ClientCode::RH, name: "Reth".to_string(), @@ -118,20 +117,14 @@ pub async fn launch_http_ws_same_port(modules: impl Into) -> } /// Returns an [`RpcModuleBuilder`] with testing components. -pub fn test_rpc_builder() -> RpcModuleBuilder< - EthPrimitives, - NoopProvider, - TestPool, - NoopNetwork, - TokioTaskExecutor, - EthEvmConfig, - NoopConsensus, -> { +pub fn test_rpc_builder( +) -> RpcModuleBuilder +{ RpcModuleBuilder::default() .with_provider(NoopProvider::default()) .with_pool(TestPoolBuilder::default().into()) .with_network(NoopNetwork::default()) - .with_executor(TokioTaskExecutor::default()) + .with_executor(Box::new(TokioTaskExecutor::default())) .with_evm_config(EthEvmConfig::mainnet()) .with_consensus(NoopConsensus::default()) } diff --git a/crates/rpc/rpc-convert/Cargo.toml b/crates/rpc/rpc-convert/Cargo.toml new file mode 100644 index 00000000000..af43e9c54a2 --- /dev/null +++ b/crates/rpc/rpc-convert/Cargo.toml @@ -0,0 +1,62 @@ +[package] +name = "reth-rpc-convert" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true +description = "Compatibility layer for reth-primitives and ethereum RPC types" + +[lints] +workspace = true + +[dependencies] +# reth +reth-primitives-traits.workspace = true +reth-storage-api = { workspace = true, optional = true } +reth-evm.workspace = true +reth-ethereum-primitives.workspace = true + +# ethereum +alloy-primitives.workspace = true +alloy-rpc-types-eth = { workspace = true, features = ["serde"] } +alloy-signer.workspace = true +alloy-consensus.workspace = true +alloy-network.workspace = true +alloy-json-rpc.workspace = true + +# optimism +op-alloy-consensus = { workspace = true, optional = true } +op-alloy-rpc-types = { workspace = true, optional = true } +op-alloy-network = { workspace = true, optional = true } +reth-optimism-primitives = { workspace = true, optional = true } +op-revm = { workspace = true, optional = true } + +# revm +revm-context.workspace = true + +# io +jsonrpsee-types.workspace = true + +# error +thiserror.workspace = true + +auto_impl.workspace = true +dyn-clone.workspace = true + +[dev-dependencies] +serde_json.workspace = true + +[features] +default = [] +op = [ + "dep:op-alloy-consensus", + "dep:op-alloy-rpc-types", + "dep:op-alloy-network", + "dep:reth-optimism-primitives", + "dep:reth-storage-api", + "dep:op-revm", + "reth-evm/op", + "reth-primitives-traits/op", +] diff --git a/crates/rpc/rpc-convert/src/block.rs b/crates/rpc/rpc-convert/src/block.rs new file mode 100644 index 00000000000..144bcdcac97 --- /dev/null +++ b/crates/rpc/rpc-convert/src/block.rs @@ -0,0 +1,47 @@ +//! Conversion traits for block responses to primitive block types. + +use alloy_network::Network; +use std::convert::Infallible; + +/// Trait for converting network block responses to primitive block types. +pub trait TryFromBlockResponse { + /// The error type returned if the conversion fails. + type Error: core::error::Error + Send + Sync + Unpin; + + /// Converts a network block response to a primitive block type. + /// + /// # Returns + /// + /// Returns `Ok(Self)` on successful conversion, or `Err(Self::Error)` if the conversion fails. + fn from_block_response(block_response: N::BlockResponse) -> Result + where + Self: Sized; +} + +impl TryFromBlockResponse for alloy_consensus::Block +where + N::BlockResponse: Into, +{ + type Error = Infallible; + + fn from_block_response(block_response: N::BlockResponse) -> Result { + Ok(block_response.into()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use alloy_consensus::{Block, TxEnvelope}; + use alloy_network::Ethereum; + use alloy_rpc_types_eth::BlockTransactions; + + #[test] + fn test_try_from_block_response() { + let rpc_block: alloy_rpc_types_eth::Block = + alloy_rpc_types_eth::Block::new(Default::default(), BlockTransactions::Full(vec![])); + let result = + as TryFromBlockResponse>::from_block_response(rpc_block); + assert!(result.is_ok()); + } +} diff --git a/crates/rpc/rpc-eth-types/src/revm_utils.rs b/crates/rpc/rpc-convert/src/fees.rs similarity index 50% rename from crates/rpc/rpc-eth-types/src/revm_utils.rs rename to crates/rpc/rpc-convert/src/fees.rs index 53a75ebbb07..46f8fc8c207 100644 --- a/crates/rpc/rpc-eth-types/src/revm_utils.rs +++ b/crates/rpc/rpc-convert/src/fees.rs @@ -1,57 +1,6 @@ -//! utilities for working with revm - -use alloy_primitives::{keccak256, Address, B256, U256}; -use alloy_rpc_types_eth::{ - state::{AccountOverride, StateOverride}, - BlockOverrides, -}; -use reth_evm::TransactionEnv; -use revm::{ - context::BlockEnv, - database::{CacheDB, State}, - state::{Account, AccountStatus, Bytecode, EvmStorageSlot}, - Database, DatabaseCommit, -}; -use std::{ - cmp::min, - collections::{BTreeMap, HashMap}, -}; - -use super::{EthApiError, EthResult, RpcInvalidTransactionError}; - -/// Calculates the caller gas allowance. -/// -/// `allowance = (account.balance - tx.value) / tx.gas_price` -/// -/// Returns an error if the caller has insufficient funds. -/// Caution: This assumes non-zero `env.gas_price`. Otherwise, zero allowance will be returned. -/// -/// Note: this takes the mut [Database] trait because the loaded sender can be reused for the -/// following operation like `eth_call`. -pub fn caller_gas_allowance(db: &mut DB, env: &impl TransactionEnv) -> EthResult -where - DB: Database, - EthApiError: From<::Error>, -{ - // Get the caller account. - let caller = db.basic(env.caller())?; - // Get the caller balance. - let balance = caller.map(|acc| acc.balance).unwrap_or_default(); - // Get transaction value. - let value = env.value(); - // Subtract transferred value from the caller balance. Return error if the caller has - // insufficient funds. - let balance = balance - .checked_sub(env.value()) - .ok_or_else(|| RpcInvalidTransactionError::InsufficientFunds { cost: value, balance })?; - - Ok(balance - // Calculate the amount of gas the caller can afford with the specified gas price. - .checked_div(U256::from(env.gas_price())) - // This will be 0 if gas price is 0. It is fine, because we check it before. - .unwrap_or_default() - .saturating_to()) -} +use alloy_primitives::{B256, U256}; +use std::cmp::min; +use thiserror::Error; /// Helper type for representing the fees of a `TransactionRequest` #[derive(Debug)] @@ -69,8 +18,6 @@ pub struct CallFees { pub max_fee_per_blob_gas: Option, } -// === impl CallFees === - impl CallFees { /// Ensures the fields of a `TransactionRequest` are not conflicting. /// @@ -90,6 +37,8 @@ impl CallFees { /// missing values, bypassing fee checks wrt. `baseFeePerGas`. /// /// This mirrors geth's behaviour when transaction requests are executed: + /// + /// [`BlockEnv`]: revm_context::BlockEnv pub fn ensure_fees( call_gas_price: Option, call_max_fee: Option, @@ -98,14 +47,14 @@ impl CallFees { blob_versioned_hashes: Option<&[B256]>, max_fee_per_blob_gas: Option, block_blob_fee: Option, - ) -> EthResult { + ) -> Result { /// Get the effective gas price of a transaction as specfified in EIP-1559 with relevant /// checks. fn get_effective_gas_price( max_fee_per_gas: Option, max_priority_fee_per_gas: Option, block_base_fee: U256, - ) -> EthResult { + ) -> Result { match max_fee_per_gas { Some(max_fee) => { let max_priority_fee_per_gas = max_priority_fee_per_gas.unwrap_or(U256::ZERO); @@ -115,25 +64,25 @@ impl CallFees { max_fee < block_base_fee { // `base_fee_per_gas` is greater than the `max_fee_per_gas` - return Err(RpcInvalidTransactionError::FeeCapTooLow.into()) + return Err(CallFeesError::FeeCapTooLow) } if max_fee < max_priority_fee_per_gas { return Err( // `max_priority_fee_per_gas` is greater than the `max_fee_per_gas` - RpcInvalidTransactionError::TipAboveFeeCap.into(), + CallFeesError::TipAboveFeeCap, ) } // ref Ok(min( max_fee, - block_base_fee.checked_add(max_priority_fee_per_gas).ok_or_else(|| { - EthApiError::from(RpcInvalidTransactionError::TipVeryHigh) - })?, + block_base_fee + .checked_add(max_priority_fee_per_gas) + .ok_or(CallFeesError::TipVeryHigh)?, )) } None => Ok(block_base_fee .checked_add(max_priority_fee_per_gas.unwrap_or(U256::ZERO)) - .ok_or(EthApiError::from(RpcInvalidTransactionError::TipVeryHigh))?), + .ok_or(CallFeesError::TipVeryHigh)?), } } @@ -176,7 +125,7 @@ impl CallFees { // Ensure blob_hashes are present if !has_blob_hashes { // Blob transaction but no blob hashes - return Err(RpcInvalidTransactionError::BlobTransactionMissingBlobHashes.into()) + return Err(CallFeesError::BlobTransactionMissingBlobHashes) } Ok(Self { @@ -187,169 +136,38 @@ impl CallFees { } _ => { // this fallback covers incompatible combinations of fields - Err(EthApiError::ConflictingFeeFieldsInRequest) + Err(CallFeesError::ConflictingFeeFieldsInRequest) } } } } -/// Helper trait implemented for databases that support overriding block hashes. -/// -/// Used for applying [`BlockOverrides::block_hash`] -pub trait OverrideBlockHashes { - /// Overrides the given block hashes. - fn override_block_hashes(&mut self, block_hashes: BTreeMap); -} - -impl OverrideBlockHashes for CacheDB { - fn override_block_hashes(&mut self, block_hashes: BTreeMap) { - self.cache - .block_hashes - .extend(block_hashes.into_iter().map(|(num, hash)| (U256::from(num), hash))) - } -} - -impl OverrideBlockHashes for State { - fn override_block_hashes(&mut self, block_hashes: BTreeMap) { - self.block_hashes.extend(block_hashes); - } -} - -/// Applies the given block overrides to the env and updates overridden block hashes in the db. -pub fn apply_block_overrides( - overrides: BlockOverrides, - db: &mut impl OverrideBlockHashes, - env: &mut BlockEnv, -) { - let BlockOverrides { - number, - difficulty, - time, - gas_limit, - coinbase, - random, - base_fee, - block_hash, - } = overrides; - - if let Some(block_hashes) = block_hash { - // override block hashes - db.override_block_hashes(block_hashes); - } - - if let Some(number) = number { - env.number = number.saturating_to(); - } - if let Some(difficulty) = difficulty { - env.difficulty = difficulty; - } - if let Some(time) = time { - env.timestamp = time; - } - if let Some(gas_limit) = gas_limit { - env.gas_limit = gas_limit; - } - if let Some(coinbase) = coinbase { - env.beneficiary = coinbase; - } - if let Some(random) = random { - env.prevrandao = Some(random); - } - if let Some(base_fee) = base_fee { - env.basefee = base_fee.saturating_to(); - } -} - -/// Applies the given state overrides (a set of [`AccountOverride`]) to the [`CacheDB`]. -pub fn apply_state_overrides(overrides: StateOverride, db: &mut DB) -> EthResult<()> -where - DB: Database + DatabaseCommit, - EthApiError: From, -{ - for (account, account_overrides) in overrides { - apply_account_override(account, account_overrides, db)?; - } - Ok(()) -} - -/// Applies a single [`AccountOverride`] to the [`CacheDB`]. -fn apply_account_override( - account: Address, - account_override: AccountOverride, - db: &mut DB, -) -> EthResult<()> -where - DB: Database + DatabaseCommit, - EthApiError: From, -{ - let mut info = db.basic(account)?.unwrap_or_default(); - - if let Some(nonce) = account_override.nonce { - info.nonce = nonce; - } - if let Some(code) = account_override.code { - // we need to set both the bytecode and the codehash - info.code_hash = keccak256(&code); - info.code = Some( - Bytecode::new_raw_checked(code) - .map_err(|err| EthApiError::InvalidBytecode(err.to_string()))?, - ); - } - if let Some(balance) = account_override.balance { - info.balance = balance; - } - - // Create a new account marked as touched - let mut acc = - revm::state::Account { info, status: AccountStatus::Touched, storage: HashMap::default() }; - - let storage_diff = match (account_override.state, account_override.state_diff) { - (Some(_), Some(_)) => return Err(EthApiError::BothStateAndStateDiffInOverride(account)), - (None, None) => None, - // If we need to override the entire state, we firstly mark account as destroyed to clear - // its storage, and then we mark it is "NewlyCreated" to make sure that old storage won't be - // used. - (Some(state), None) => { - // Destroy the account to ensure that its storage is cleared - db.commit(HashMap::from_iter([( - account, - Account { - status: AccountStatus::SelfDestructed | AccountStatus::Touched, - ..Default::default() - }, - )])); - // Mark the account as created to ensure that old storage is not read - acc.mark_created(); - Some(state) - } - (None, Some(state)) => Some(state), - }; - - if let Some(state) = storage_diff { - for (slot, value) in state { - acc.storage.insert( - slot.into(), - EvmStorageSlot { - // we use inverted value here to ensure that storage is treated as changed - original_value: (!value).into(), - present_value: value.into(), - is_cold: false, - }, - ); - } - } - - db.commit(HashMap::from_iter([(account, acc)])); - - Ok(()) +/// Error coming from decoding and validating transaction request fees. +#[derive(Debug, Error)] +pub enum CallFeesError { + /// Thrown when a call or transaction request (`eth_call`, `eth_estimateGas`, + /// `eth_sendTransaction`) contains conflicting fields (legacy, EIP-1559) + #[error("both gasPrice and (maxFeePerGas or maxPriorityFeePerGas) specified")] + ConflictingFeeFieldsInRequest, + /// Thrown post London if the transaction's fee is less than the base fee of the block + #[error("max fee per gas less than block base fee")] + FeeCapTooLow, + /// Thrown to ensure no one is able to specify a transaction with a tip higher than the total + /// fee cap. + #[error("max priority fee per gas higher than max fee per gas")] + TipAboveFeeCap, + /// A sanity error to avoid huge numbers specified in the tip field. + #[error("max priority fee per gas higher than 2^256-1")] + TipVeryHigh, + /// Blob transaction has no versioned hashes + #[error("blob transaction missing blob hashes")] + BlobTransactionMissingBlobHashes, } #[cfg(test)] mod tests { use super::*; use alloy_consensus::constants::GWEI_TO_WEI; - use alloy_primitives::{address, bytes}; - use reth_revm::db::EmptyDB; #[test] fn test_ensure_0_fallback() { @@ -460,38 +278,4 @@ mod tests { ); assert!(call_fees.is_err()); } - - #[test] - fn state_override_state() { - let code = bytes!( - "0x63d0e30db05f525f5f6004601c3473c02aaa39b223fe8d0a0e5c4f27ead9083c756cc25af15f5260205ff3" - ); - let to = address!("0x2260FAC5E5542a773Aa44fBCfeDf7C193bc2C599"); - - let mut db = State::builder().with_database(CacheDB::new(EmptyDB::new())).build(); - - let acc_override = AccountOverride::default().with_code(code.clone()); - apply_account_override(to, acc_override, &mut db).unwrap(); - - let account = db.basic(to).unwrap().unwrap(); - assert!(account.code.is_some()); - assert_eq!(account.code_hash, keccak256(&code)); - } - - #[test] - fn state_override_cache_db() { - let code = bytes!( - "0x63d0e30db05f525f5f6004601c3473c02aaa39b223fe8d0a0e5c4f27ead9083c756cc25af15f5260205ff3" - ); - let to = address!("0x2260FAC5E5542a773Aa44fBCfeDf7C193bc2C599"); - - let mut db = CacheDB::new(EmptyDB::new()); - - let acc_override = AccountOverride::default().with_code(code.clone()); - apply_account_override(to, acc_override, &mut db).unwrap(); - - let account = db.basic(to).unwrap().unwrap(); - assert!(account.code.is_some()); - assert_eq!(account.code_hash, keccak256(&code)); - } } diff --git a/crates/rpc/rpc-convert/src/lib.rs b/crates/rpc/rpc-convert/src/lib.rs new file mode 100644 index 00000000000..9844b17b604 --- /dev/null +++ b/crates/rpc/rpc-convert/src/lib.rs @@ -0,0 +1,29 @@ +//! Reth compatibility and utils for RPC types +//! +//! This crate various helper functions to convert between reth primitive types and rpc types. + +#![doc( + html_logo_url = "https://raw.githubusercontent.com/paradigmxyz/reth/main/assets/reth-docs.png", + html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256", + issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" +)] +#![cfg_attr(not(test), warn(unused_crate_dependencies))] +#![cfg_attr(docsrs, feature(doc_cfg))] + +pub mod block; +mod fees; +pub mod receipt; +mod rpc; +pub mod transaction; + +pub use block::TryFromBlockResponse; +pub use fees::{CallFees, CallFeesError}; +pub use receipt::TryFromReceiptResponse; +pub use rpc::*; +pub use transaction::{ + EthTxEnvError, IntoRpcTx, RpcConvert, RpcConverter, TransactionConversionError, + TryFromTransactionResponse, TryIntoSimTx, TxInfoMapper, +}; + +#[cfg(feature = "op")] +pub use transaction::op::*; diff --git a/crates/rpc/rpc-convert/src/receipt.rs b/crates/rpc/rpc-convert/src/receipt.rs new file mode 100644 index 00000000000..5f37c1cad5e --- /dev/null +++ b/crates/rpc/rpc-convert/src/receipt.rs @@ -0,0 +1,99 @@ +//! Conversion traits for receipt responses to primitive receipt types. + +use alloy_network::Network; +use std::convert::Infallible; + +/// Trait for converting network receipt responses to primitive receipt types. +pub trait TryFromReceiptResponse { + /// The error type returned if the conversion fails. + type Error: core::error::Error + Send + Sync + Unpin; + + /// Converts a network receipt response to a primitive receipt type. + /// + /// # Returns + /// + /// Returns `Ok(Self)` on successful conversion, or `Err(Self::Error)` if the conversion fails. + fn from_receipt_response(receipt_response: N::ReceiptResponse) -> Result + where + Self: Sized; +} + +impl TryFromReceiptResponse for reth_ethereum_primitives::Receipt { + type Error = Infallible; + + fn from_receipt_response( + receipt_response: alloy_rpc_types_eth::TransactionReceipt, + ) -> Result { + Ok(receipt_response.into_inner().into()) + } +} + +#[cfg(feature = "op")] +impl TryFromReceiptResponse for reth_optimism_primitives::OpReceipt { + type Error = Infallible; + + fn from_receipt_response( + receipt_response: op_alloy_rpc_types::OpTransactionReceipt, + ) -> Result { + Ok(receipt_response.inner.inner.map_logs(Into::into).into()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use alloy_consensus::ReceiptEnvelope; + use alloy_network::Ethereum; + use reth_ethereum_primitives::Receipt; + + #[test] + fn test_try_from_receipt_response() { + let rpc_receipt = alloy_rpc_types_eth::TransactionReceipt { + inner: ReceiptEnvelope::Eip1559(Default::default()), + transaction_hash: Default::default(), + transaction_index: None, + block_hash: None, + block_number: None, + gas_used: 0, + effective_gas_price: 0, + blob_gas_used: None, + blob_gas_price: None, + from: Default::default(), + to: None, + contract_address: None, + }; + let result = + >::from_receipt_response(rpc_receipt); + assert!(result.is_ok()); + } + + #[cfg(feature = "op")] + #[test] + fn test_try_from_receipt_response_optimism() { + use op_alloy_consensus::OpReceiptEnvelope; + use op_alloy_network::Optimism; + use op_alloy_rpc_types::OpTransactionReceipt; + use reth_optimism_primitives::OpReceipt; + + let op_receipt = OpTransactionReceipt { + inner: alloy_rpc_types_eth::TransactionReceipt { + inner: OpReceiptEnvelope::Eip1559(Default::default()), + transaction_hash: Default::default(), + transaction_index: None, + block_hash: None, + block_number: None, + gas_used: 0, + effective_gas_price: 0, + blob_gas_used: None, + blob_gas_price: None, + from: Default::default(), + to: None, + contract_address: None, + }, + l1_block_info: Default::default(), + }; + let result = + >::from_receipt_response(op_receipt); + assert!(result.is_ok()); + } +} diff --git a/crates/rpc/rpc-convert/src/rpc.rs b/crates/rpc/rpc-convert/src/rpc.rs new file mode 100644 index 00000000000..cf67bc11add --- /dev/null +++ b/crates/rpc/rpc-convert/src/rpc.rs @@ -0,0 +1,102 @@ +use std::{fmt::Debug, future::Future}; + +use alloy_consensus::{EthereumTxEnvelope, SignableTransaction, TxEip4844}; +use alloy_json_rpc::RpcObject; +use alloy_network::{ + primitives::HeaderResponse, Network, ReceiptResponse, TransactionResponse, TxSigner, +}; +use alloy_primitives::Signature; +use alloy_rpc_types_eth::TransactionRequest; + +/// RPC types used by the `eth_` RPC API. +/// +/// This is a subset of [`Network`] trait with only RPC response types kept. +pub trait RpcTypes: Send + Sync + Clone + Unpin + Debug + 'static { + /// Header response type. + type Header: RpcObject + HeaderResponse; + /// Receipt response type. + type Receipt: RpcObject + ReceiptResponse; + /// Transaction response type. + type TransactionResponse: RpcObject + TransactionResponse; + /// Transaction response type. + type TransactionRequest: RpcObject + AsRef + AsMut; +} + +impl RpcTypes for T +where + T: Network + AsMut> + Unpin, +{ + type Header = T::HeaderResponse; + type Receipt = T::ReceiptResponse; + type TransactionResponse = T::TransactionResponse; + type TransactionRequest = T::TransactionRequest; +} + +/// Adapter for network specific transaction response. +pub type RpcTransaction = ::TransactionResponse; + +/// Adapter for network specific receipt response. +pub type RpcReceipt = ::Receipt; + +/// Adapter for network specific header response. +pub type RpcHeader = ::Header; + +/// Adapter for network specific block type. +pub type RpcBlock = alloy_rpc_types_eth::Block, RpcHeader>; + +/// Adapter for network specific transaction request. +pub type RpcTxReq = ::TransactionRequest; + +/// Error for [`SignableTxRequest`] trait. +#[derive(Debug, thiserror::Error)] +pub enum SignTxRequestError { + /// The transaction request is invalid. + #[error("invalid transaction request")] + InvalidTransactionRequest, + + /// The signer is not supported. + #[error(transparent)] + SignerNotSupported(#[from] alloy_signer::Error), +} + +/// An abstraction over transaction requests that can be signed. +pub trait SignableTxRequest: Send + Sync + 'static { + /// Attempts to build a transaction request and sign it with the given signer. + fn try_build_and_sign( + self, + signer: impl TxSigner + Send, + ) -> impl Future> + Send; +} + +impl SignableTxRequest> for TransactionRequest { + async fn try_build_and_sign( + self, + signer: impl TxSigner + Send, + ) -> Result, SignTxRequestError> { + let mut tx = + self.build_typed_tx().map_err(|_| SignTxRequestError::InvalidTransactionRequest)?; + let signature = signer.sign_transaction(&mut tx).await?; + Ok(tx.into_signed(signature).into()) + } +} + +#[cfg(feature = "op")] +impl SignableTxRequest + for op_alloy_rpc_types::OpTransactionRequest +{ + async fn try_build_and_sign( + self, + signer: impl TxSigner + Send, + ) -> Result { + let mut tx = + self.build_typed_tx().map_err(|_| SignTxRequestError::InvalidTransactionRequest)?; + let signature = signer.sign_transaction(&mut tx).await?; + + // sanity check + if tx.is_deposit() { + return Err(SignTxRequestError::InvalidTransactionRequest); + } + + Ok(tx.into_signed(signature).into()) + } +} diff --git a/crates/rpc/rpc-convert/src/transaction.rs b/crates/rpc/rpc-convert/src/transaction.rs new file mode 100644 index 00000000000..6766ec43fb0 --- /dev/null +++ b/crates/rpc/rpc-convert/src/transaction.rs @@ -0,0 +1,1202 @@ +//! Compatibility functions for rpc `Transaction` type. +use crate::{ + fees::{CallFees, CallFeesError}, + RpcHeader, RpcReceipt, RpcTransaction, RpcTxReq, RpcTypes, SignableTxRequest, +}; +use alloy_consensus::{ + error::ValueError, transaction::Recovered, EthereumTxEnvelope, Sealable, TxEip4844, +}; +use alloy_network::Network; +use alloy_primitives::{Address, TxKind, U256}; +use alloy_rpc_types_eth::{ + request::{TransactionInputError, TransactionRequest}, + Transaction, TransactionInfo, +}; +use core::error; +use dyn_clone::DynClone; +use reth_evm::{ + revm::context_interface::{either::Either, Block}, + BlockEnvFor, ConfigureEvm, EvmEnvFor, TxEnvFor, +}; +use reth_primitives_traits::{ + BlockTy, HeaderTy, NodePrimitives, SealedBlock, SealedHeader, SealedHeaderFor, TransactionMeta, + TxTy, +}; +use revm_context::{BlockEnv, CfgEnv, TxEnv}; +use std::{convert::Infallible, error::Error, fmt::Debug, marker::PhantomData}; +use thiserror::Error; + +/// Input for [`RpcConvert::convert_receipts`]. +#[derive(Debug, Clone)] +pub struct ConvertReceiptInput<'a, N: NodePrimitives> { + /// Primitive receipt. + pub receipt: N::Receipt, + /// Transaction the receipt corresponds to. + pub tx: Recovered<&'a N::SignedTx>, + /// Gas used by the transaction. + pub gas_used: u64, + /// Number of logs emitted before this transaction. + pub next_log_index: usize, + /// Metadata for the transaction. + pub meta: TransactionMeta, +} + +/// A type that knows how to convert primitive receipts to RPC representations. +pub trait ReceiptConverter: Debug + 'static { + /// RPC representation. + type RpcReceipt; + + /// Error that may occur during conversion. + type Error; + + /// Converts a set of primitive receipts to RPC representations. It is guaranteed that all + /// receipts are from the same block. + fn convert_receipts( + &self, + receipts: Vec>, + ) -> Result, Self::Error>; + + /// Converts a set of primitive receipts to RPC representations. It is guaranteed that all + /// receipts are from `block`. + fn convert_receipts_with_block( + &self, + receipts: Vec>, + _block: &SealedBlock, + ) -> Result, Self::Error> { + self.convert_receipts(receipts) + } +} + +/// A type that knows how to convert a consensus header into an RPC header. +pub trait HeaderConverter: Debug + Send + Sync + Unpin + Clone + 'static { + /// An associated RPC conversion error. + type Err: error::Error; + + /// Converts a consensus header into an RPC header. + fn convert_header( + &self, + header: SealedHeader, + block_size: usize, + ) -> Result; +} + +/// Default implementation of [`HeaderConverter`] that uses [`FromConsensusHeader`] to convert +/// headers. +impl HeaderConverter for () +where + Rpc: FromConsensusHeader, +{ + type Err = Infallible; + + fn convert_header( + &self, + header: SealedHeader, + block_size: usize, + ) -> Result { + Ok(Rpc::from_consensus_header(header, block_size)) + } +} + +/// Conversion trait for obtaining RPC header from a consensus header. +pub trait FromConsensusHeader { + /// Takes a consensus header and converts it into `self`. + fn from_consensus_header(header: SealedHeader, block_size: usize) -> Self; +} + +impl FromConsensusHeader for alloy_rpc_types_eth::Header { + fn from_consensus_header(header: SealedHeader, block_size: usize) -> Self { + Self::from_consensus(header.into(), None, Some(U256::from(block_size))) + } +} + +/// Responsible for the conversions from and into RPC requests and responses. +/// +/// The JSON-RPC schema and the Node primitives are configurable using the [`RpcConvert::Network`] +/// and [`RpcConvert::Primitives`] associated types respectively. +/// +/// A generic implementation [`RpcConverter`] should be preferred over a manual implementation. As +/// long as its trait bound requirements are met, the implementation is created automatically and +/// can be used in RPC method handlers for all the conversions. +#[auto_impl::auto_impl(&, Box, Arc)] +pub trait RpcConvert: Send + Sync + Unpin + Debug + DynClone + 'static { + /// Associated lower layer consensus types to convert from and into types of [`Self::Network`]. + type Primitives: NodePrimitives; + + /// The EVM configuration. + type Evm: ConfigureEvm; + + /// Associated upper layer JSON-RPC API network requests and responses to convert from and into + /// types of [`Self::Primitives`]. + type Network: RpcTypes>>; + + /// An associated RPC conversion error. + type Error: error::Error + Into>; + + /// Wrapper for `fill()` with default `TransactionInfo` + /// Create a new rpc transaction result for a _pending_ signed transaction, setting block + /// environment related fields to `None`. + fn fill_pending( + &self, + tx: Recovered>, + ) -> Result, Self::Error> { + self.fill(tx, TransactionInfo::default()) + } + + /// Create a new rpc transaction result for a mined transaction, using the given block hash, + /// number, and tx index fields to populate the corresponding fields in the rpc result. + /// + /// The block hash, number, and tx index fields should be from the original block where the + /// transaction was mined. + fn fill( + &self, + tx: Recovered>, + tx_info: TransactionInfo, + ) -> Result, Self::Error>; + + /// Builds a fake transaction from a transaction request for inclusion into block built in + /// `eth_simulateV1`. + fn build_simulate_v1_transaction( + &self, + request: RpcTxReq, + ) -> Result, Self::Error>; + + /// Creates a transaction environment for execution based on `request` with corresponding + /// `cfg_env` and `block_env`. + fn tx_env( + &self, + request: RpcTxReq, + evm_env: &EvmEnvFor, + ) -> Result, Self::Error>; + + /// Converts a set of primitive receipts to RPC representations. It is guaranteed that all + /// receipts are from the same block. + fn convert_receipts( + &self, + receipts: Vec>, + ) -> Result>, Self::Error>; + + /// Converts a set of primitive receipts to RPC representations. It is guaranteed that all + /// receipts are from the same block. + /// + /// Also accepts the corresponding block in case the receipt requires additional metadata. + fn convert_receipts_with_block( + &self, + receipts: Vec>, + block: &SealedBlock>, + ) -> Result>, Self::Error>; + + /// Converts a primitive header to an RPC header. + fn convert_header( + &self, + header: SealedHeaderFor, + block_size: usize, + ) -> Result, Self::Error>; +} + +dyn_clone::clone_trait_object!( + + RpcConvert +); + +/// Converts `self` into `T`. The opposite of [`FromConsensusTx`]. +/// +/// Should create an RPC transaction response object based on a consensus transaction, its signer +/// [`Address`] and an additional context [`IntoRpcTx::TxInfo`]. +/// +/// Avoid implementing [`IntoRpcTx`] and use [`FromConsensusTx`] instead. Implementing it +/// automatically provides an implementation of [`IntoRpcTx`] thanks to the blanket implementation +/// in this crate. +/// +/// Prefer using [`IntoRpcTx`] over [`FromConsensusTx`] when specifying trait bounds on a generic +/// function to ensure that types that only implement [`IntoRpcTx`] can be used as well. +pub trait IntoRpcTx { + /// An additional context, usually [`TransactionInfo`] in a wrapper that carries some + /// implementation specific extra information. + type TxInfo; + /// An associated RPC conversion error. + type Err: error::Error; + + /// Performs the conversion consuming `self` with `signer` and `tx_info`. See [`IntoRpcTx`] + /// for details. + fn into_rpc_tx(self, signer: Address, tx_info: Self::TxInfo) -> Result; +} + +/// Converts `T` into `self`. It is reciprocal of [`IntoRpcTx`]. +/// +/// Should create an RPC transaction response object based on a consensus transaction, its signer +/// [`Address`] and an additional context [`FromConsensusTx::TxInfo`]. +/// +/// Prefer implementing [`FromConsensusTx`] over [`IntoRpcTx`] because it automatically provides an +/// implementation of [`IntoRpcTx`] thanks to the blanket implementation in this crate. +/// +/// Prefer using [`IntoRpcTx`] over using [`FromConsensusTx`] when specifying trait bounds on a +/// generic function. This way, types that directly implement [`IntoRpcTx`] can be used as arguments +/// as well. +pub trait FromConsensusTx: Sized { + /// An additional context, usually [`TransactionInfo`] in a wrapper that carries some + /// implementation specific extra information. + type TxInfo; + /// An associated RPC conversion error. + type Err: error::Error; + + /// Performs the conversion consuming `tx` with `signer` and `tx_info`. See [`FromConsensusTx`] + /// for details. + fn from_consensus_tx(tx: T, signer: Address, tx_info: Self::TxInfo) -> Result; +} + +impl> + FromConsensusTx for Transaction +{ + type TxInfo = TransactionInfo; + type Err = Infallible; + + fn from_consensus_tx( + tx: TxIn, + signer: Address, + tx_info: Self::TxInfo, + ) -> Result { + Ok(Self::from_transaction(Recovered::new_unchecked(tx.into(), signer), tx_info)) + } +} + +impl IntoRpcTx for ConsensusTx +where + ConsensusTx: alloy_consensus::Transaction, + RpcTx: FromConsensusTx, + >::Err: Debug, +{ + type TxInfo = RpcTx::TxInfo; + type Err = >::Err; + + fn into_rpc_tx(self, signer: Address, tx_info: Self::TxInfo) -> Result { + RpcTx::from_consensus_tx(self, signer, tx_info) + } +} + +/// Converts `Tx` into `RpcTx` +/// +/// Where: +/// * `Tx` is a transaction from the consensus layer. +/// * `RpcTx` is a transaction response object of the RPC API +/// +/// The conversion function is accompanied by `signer`'s address and `tx_info` providing extra +/// context about a transaction in a block. +/// +/// The `RpcTxConverter` has two blanket implementations: +/// * `()` assuming `Tx` implements [`IntoRpcTx`] and is used as default for [`RpcConverter`]. +/// * `Fn(Tx, Address, TxInfo) -> RpcTx` and can be applied using +/// [`RpcConverter::with_rpc_tx_converter`]. +/// +/// One should prefer to implement [`IntoRpcTx`] for `Tx` to get the `RpcTxConverter` implementation +/// for free, thanks to the blanket implementation, unless the conversion requires more context. For +/// example, some configuration parameters or access handles to database, network, etc. +pub trait RpcTxConverter: Clone + Debug + Unpin + Send + Sync + 'static { + /// An associated error that can happen during the conversion. + type Err; + + /// Performs the conversion of `tx` from `Tx` into `RpcTx`. + /// + /// See [`RpcTxConverter`] for more information. + fn convert_rpc_tx(&self, tx: Tx, signer: Address, tx_info: TxInfo) -> Result; +} + +impl RpcTxConverter for () +where + Tx: IntoRpcTx, +{ + type Err = Tx::Err; + + fn convert_rpc_tx( + &self, + tx: Tx, + signer: Address, + tx_info: Tx::TxInfo, + ) -> Result { + tx.into_rpc_tx(signer, tx_info) + } +} + +impl RpcTxConverter for F +where + F: Fn(Tx, Address, TxInfo) -> Result + Clone + Debug + Unpin + Send + Sync + 'static, +{ + type Err = E; + + fn convert_rpc_tx(&self, tx: Tx, signer: Address, tx_info: TxInfo) -> Result { + self(tx, signer, tx_info) + } +} + +/// Converts `TxReq` into `SimTx`. +/// +/// Where: +/// * `TxReq` is a transaction request received from an RPC API +/// * `SimTx` is the corresponding consensus layer transaction for execution simulation +/// +/// The `SimTxConverter` has two blanket implementations: +/// * `()` assuming `TxReq` implements [`TryIntoSimTx`] and is used as default for [`RpcConverter`]. +/// * `Fn(TxReq) -> Result>` and can be applied using +/// [`RpcConverter::with_sim_tx_converter`]. +/// +/// One should prefer to implement [`TryIntoSimTx`] for `TxReq` to get the `SimTxConverter` +/// implementation for free, thanks to the blanket implementation, unless the conversion requires +/// more context. For example, some configuration parameters or access handles to database, network, +/// etc. +pub trait SimTxConverter: Clone + Debug + Unpin + Send + Sync + 'static { + /// An associated error that can occur during the conversion. + type Err: Error; + + /// Performs the conversion from `tx_req` into `SimTx`. + /// + /// See [`SimTxConverter`] for more information. + fn convert_sim_tx(&self, tx_req: TxReq) -> Result; +} + +impl SimTxConverter for () +where + TxReq: TryIntoSimTx + Debug, +{ + type Err = ValueError; + + fn convert_sim_tx(&self, tx_req: TxReq) -> Result { + tx_req.try_into_sim_tx() + } +} + +impl SimTxConverter for F +where + TxReq: Debug, + E: Error, + F: Fn(TxReq) -> Result + Clone + Debug + Unpin + Send + Sync + 'static, +{ + type Err = E; + + fn convert_sim_tx(&self, tx_req: TxReq) -> Result { + self(tx_req) + } +} + +/// Converts `self` into `T`. +/// +/// Should create a fake transaction for simulation using [`TransactionRequest`]. +pub trait TryIntoSimTx +where + Self: Sized, +{ + /// Performs the conversion. + /// + /// Should return a signed typed transaction envelope for the [`eth_simulateV1`] endpoint with a + /// dummy signature or an error if [required fields] are missing. + /// + /// [`eth_simulateV1`]: + /// [required fields]: TransactionRequest::buildable_type + fn try_into_sim_tx(self) -> Result>; +} + +/// Adds extra context to [`TransactionInfo`]. +pub trait TxInfoMapper { + /// An associated output type that carries [`TransactionInfo`] with some extra context. + type Out; + /// An associated error that can occur during the mapping. + type Err; + + /// Performs the conversion. + fn try_map(&self, tx: &T, tx_info: TransactionInfo) -> Result; +} + +impl TxInfoMapper for () { + type Out = TransactionInfo; + type Err = Infallible; + + fn try_map(&self, _tx: &T, tx_info: TransactionInfo) -> Result { + Ok(tx_info) + } +} + +impl TryIntoSimTx> for TransactionRequest { + fn try_into_sim_tx(self) -> Result, ValueError> { + Self::build_typed_simulate_transaction(self) + } +} + +/// Converts `TxReq` into `TxEnv`. +/// +/// Where: +/// * `TxReq` is a transaction request received from an RPC API +/// * `TxEnv` is the corresponding transaction environment for execution +/// +/// The `TxEnvConverter` has two blanket implementations: +/// * `()` assuming `TxReq` implements [`TryIntoTxEnv`] and is used as default for [`RpcConverter`]. +/// * `Fn(TxReq, &CfgEnv, &BlockEnv) -> Result` and can be applied using +/// [`RpcConverter::with_tx_env_converter`]. +/// +/// One should prefer to implement [`TryIntoTxEnv`] for `TxReq` to get the `TxEnvConverter` +/// implementation for free, thanks to the blanket implementation, unless the conversion requires +/// more context. For example, some configuration parameters or access handles to database, network, +/// etc. +pub trait TxEnvConverter: + Debug + Send + Sync + Unpin + Clone + 'static +{ + /// An associated error that can occur during conversion. + type Error; + + /// Converts a rpc transaction request into a transaction environment. + /// + /// See [`TxEnvConverter`] for more information. + fn convert_tx_env( + &self, + tx_req: TxReq, + evm_env: &EvmEnvFor, + ) -> Result, Self::Error>; +} + +impl TxEnvConverter for () +where + TxReq: TryIntoTxEnv, BlockEnvFor>, + Evm: ConfigureEvm, +{ + type Error = TxReq::Err; + + fn convert_tx_env( + &self, + tx_req: TxReq, + evm_env: &EvmEnvFor, + ) -> Result, Self::Error> { + tx_req.try_into_tx_env(&evm_env.cfg_env, &evm_env.block_env) + } +} + +/// Converts rpc transaction requests into transaction environment using a closure. +impl TxEnvConverter for F +where + F: Fn(TxReq, &EvmEnvFor) -> Result, E> + + Debug + + Send + + Sync + + Unpin + + Clone + + 'static, + TxReq: Clone, + Evm: ConfigureEvm, + E: error::Error + Send + Sync + 'static, +{ + type Error = E; + + fn convert_tx_env( + &self, + tx_req: TxReq, + evm_env: &EvmEnvFor, + ) -> Result, Self::Error> { + self(tx_req, evm_env) + } +} + +/// Converts `self` into `T`. +/// +/// Should create an executable transaction environment using [`TransactionRequest`]. +pub trait TryIntoTxEnv { + /// An associated error that can occur during the conversion. + type Err; + + /// Performs the conversion. + fn try_into_tx_env( + self, + cfg_env: &CfgEnv, + block_env: &BlockEnv, + ) -> Result; +} + +/// An Ethereum specific transaction environment error than can occur during conversion from +/// [`TransactionRequest`]. +#[derive(Debug, Error)] +pub enum EthTxEnvError { + /// Error while decoding or validating transaction request fees. + #[error(transparent)] + CallFees(#[from] CallFeesError), + /// Both data and input fields are set and not equal. + #[error(transparent)] + Input(#[from] TransactionInputError), +} + +impl TryIntoTxEnv for TransactionRequest { + type Err = EthTxEnvError; + + fn try_into_tx_env( + self, + cfg_env: &CfgEnv, + block_env: &BlockEnv, + ) -> Result { + // Ensure that if versioned hashes are set, they're not empty + if self.blob_versioned_hashes.as_ref().is_some_and(|hashes| hashes.is_empty()) { + return Err(CallFeesError::BlobTransactionMissingBlobHashes.into()) + } + + let tx_type = self.minimal_tx_type() as u8; + + let Self { + from, + to, + gas_price, + max_fee_per_gas, + max_priority_fee_per_gas, + gas, + value, + input, + nonce, + access_list, + chain_id, + blob_versioned_hashes, + max_fee_per_blob_gas, + authorization_list, + transaction_type: _, + sidecar: _, + } = self; + + let CallFees { max_priority_fee_per_gas, gas_price, max_fee_per_blob_gas } = + CallFees::ensure_fees( + gas_price.map(U256::from), + max_fee_per_gas.map(U256::from), + max_priority_fee_per_gas.map(U256::from), + U256::from(block_env.basefee), + blob_versioned_hashes.as_deref(), + max_fee_per_blob_gas.map(U256::from), + block_env.blob_gasprice().map(U256::from), + )?; + + let gas_limit = gas.unwrap_or( + // Use maximum allowed gas limit. The reason for this + // is that both Erigon and Geth use pre-configured gas cap even if + // it's possible to derive the gas limit from the block: + // + block_env.gas_limit, + ); + + let chain_id = chain_id.unwrap_or(cfg_env.chain_id); + + let caller = from.unwrap_or_default(); + + let nonce = nonce.unwrap_or_default(); + + let env = TxEnv { + tx_type, + gas_limit, + nonce, + caller, + gas_price: gas_price.saturating_to(), + gas_priority_fee: max_priority_fee_per_gas.map(|v| v.saturating_to()), + kind: to.unwrap_or(TxKind::Create), + value: value.unwrap_or_default(), + data: input.try_into_unique_input().map_err(EthTxEnvError::from)?.unwrap_or_default(), + chain_id: Some(chain_id), + access_list: access_list.unwrap_or_default(), + // EIP-4844 fields + blob_hashes: blob_versioned_hashes.unwrap_or_default(), + max_fee_per_blob_gas: max_fee_per_blob_gas + .map(|v| v.saturating_to()) + .unwrap_or_default(), + // EIP-7702 fields + authorization_list: authorization_list + .unwrap_or_default() + .into_iter() + .map(Either::Left) + .collect(), + }; + + Ok(env) + } +} + +/// Conversion into transaction RPC response failed. +#[derive(Debug, Clone, Error)] +#[error("Failed to convert transaction into RPC response: {0}")] +pub struct TransactionConversionError(String); + +/// Generic RPC response object converter for `Evm` and network `Network`. +/// +/// The main purpose of this struct is to provide an implementation of [`RpcConvert`] for generic +/// associated types. This struct can then be used for conversions in RPC method handlers. +/// +/// An [`RpcConvert`] implementation is generated if the following traits are implemented for the +/// network and EVM associated primitives: +/// * [`FromConsensusTx`]: from signed transaction into RPC response object. +/// * [`TryIntoSimTx`]: from RPC transaction request into a simulated transaction. +/// * [`TryIntoTxEnv`] or [`TxEnvConverter`]: from RPC transaction request into an executable +/// transaction. +/// * [`TxInfoMapper`]: from [`TransactionInfo`] into [`FromConsensusTx::TxInfo`]. Should be +/// implemented for a dedicated struct that is assigned to `Map`. If [`FromConsensusTx::TxInfo`] +/// is [`TransactionInfo`] then `()` can be used as `Map` which trivially passes over the input +/// object. +#[derive(Debug)] +pub struct RpcConverter< + Network, + Evm, + Receipt, + Header = (), + Map = (), + SimTx = (), + RpcTx = (), + TxEnv = (), +> { + network: PhantomData, + evm: PhantomData, + receipt_converter: Receipt, + header_converter: Header, + mapper: Map, + tx_env_converter: TxEnv, + sim_tx_converter: SimTx, + rpc_tx_converter: RpcTx, +} + +impl RpcConverter { + /// Creates a new [`RpcConverter`] with `receipt_converter` and `mapper`. + pub const fn new(receipt_converter: Receipt) -> Self { + Self { + network: PhantomData, + evm: PhantomData, + receipt_converter, + header_converter: (), + mapper: (), + tx_env_converter: (), + sim_tx_converter: (), + rpc_tx_converter: (), + } + } +} + +impl + RpcConverter +{ + /// Converts the network type + pub fn with_network( + self, + ) -> RpcConverter { + let Self { + receipt_converter, + header_converter, + mapper, + evm, + sim_tx_converter, + rpc_tx_converter, + tx_env_converter, + .. + } = self; + RpcConverter { + receipt_converter, + header_converter, + mapper, + network: Default::default(), + evm, + sim_tx_converter, + rpc_tx_converter, + tx_env_converter, + } + } + + /// Converts the transaction environment type. + pub fn with_tx_env_converter( + self, + tx_env_converter: TxEnvNew, + ) -> RpcConverter { + let Self { + receipt_converter, + header_converter, + mapper, + network, + evm, + sim_tx_converter, + rpc_tx_converter, + tx_env_converter: _, + .. + } = self; + RpcConverter { + receipt_converter, + header_converter, + mapper, + network, + evm, + sim_tx_converter, + rpc_tx_converter, + tx_env_converter, + } + } + + /// Configures the header converter. + pub fn with_header_converter( + self, + header_converter: HeaderNew, + ) -> RpcConverter { + let Self { + receipt_converter, + header_converter: _, + mapper, + network, + evm, + sim_tx_converter, + rpc_tx_converter, + tx_env_converter, + } = self; + RpcConverter { + receipt_converter, + header_converter, + mapper, + network, + evm, + sim_tx_converter, + rpc_tx_converter, + tx_env_converter, + } + } + + /// Configures the mapper. + pub fn with_mapper( + self, + mapper: MapNew, + ) -> RpcConverter { + let Self { + receipt_converter, + header_converter, + mapper: _, + network, + evm, + sim_tx_converter, + rpc_tx_converter, + tx_env_converter, + } = self; + RpcConverter { + receipt_converter, + header_converter, + mapper, + network, + evm, + sim_tx_converter, + rpc_tx_converter, + tx_env_converter, + } + } + + /// Swaps the simulate transaction converter with `sim_tx_converter`. + pub fn with_sim_tx_converter( + self, + sim_tx_converter: SimTxNew, + ) -> RpcConverter { + let Self { + receipt_converter, + header_converter, + mapper, + network, + evm, + rpc_tx_converter, + tx_env_converter, + .. + } = self; + RpcConverter { + receipt_converter, + header_converter, + mapper, + network, + evm, + sim_tx_converter, + rpc_tx_converter, + tx_env_converter, + } + } + + /// Swaps the RPC transaction converter with `rpc_tx_converter`. + pub fn with_rpc_tx_converter( + self, + rpc_tx_converter: RpcTxNew, + ) -> RpcConverter { + let Self { + receipt_converter, + header_converter, + mapper, + network, + evm, + sim_tx_converter, + tx_env_converter, + .. + } = self; + RpcConverter { + receipt_converter, + header_converter, + mapper, + network, + evm, + sim_tx_converter, + rpc_tx_converter, + tx_env_converter, + } + } + + /// Converts `self` into a boxed converter. + pub fn erased( + self, + ) -> Box< + dyn RpcConvert< + Primitives = ::Primitives, + Network = ::Network, + Error = ::Error, + Evm = ::Evm, + >, + > + where + Self: RpcConvert, + { + Box::new(self) + } +} + +impl Default + for RpcConverter +where + Receipt: Default, + Header: Default, + Map: Default, + SimTx: Default, + RpcTx: Default, + TxEnv: Default, +{ + fn default() -> Self { + Self { + network: Default::default(), + evm: Default::default(), + receipt_converter: Default::default(), + header_converter: Default::default(), + mapper: Default::default(), + sim_tx_converter: Default::default(), + rpc_tx_converter: Default::default(), + tx_env_converter: Default::default(), + } + } +} + +impl< + Network, + Evm, + Receipt: Clone, + Header: Clone, + Map: Clone, + SimTx: Clone, + RpcTx: Clone, + TxEnv: Clone, + > Clone for RpcConverter +{ + fn clone(&self) -> Self { + Self { + network: Default::default(), + evm: Default::default(), + receipt_converter: self.receipt_converter.clone(), + header_converter: self.header_converter.clone(), + mapper: self.mapper.clone(), + sim_tx_converter: self.sim_tx_converter.clone(), + rpc_tx_converter: self.rpc_tx_converter.clone(), + tx_env_converter: self.tx_env_converter.clone(), + } + } +} + +impl RpcConvert + for RpcConverter +where + N: NodePrimitives, + Network: RpcTypes>, + Evm: ConfigureEvm + 'static, + Receipt: ReceiptConverter< + N, + RpcReceipt = RpcReceipt, + Error: From + + From + + From<>>::Err> + + From + + From + + Error + + Unpin + + Sync + + Send + + Into>, + > + Send + + Sync + + Unpin + + Clone + + Debug, + Header: HeaderConverter, RpcHeader>, + Map: TxInfoMapper> + Clone + Debug + Unpin + Send + Sync + 'static, + SimTx: SimTxConverter, TxTy>, + RpcTx: + RpcTxConverter, Network::TransactionResponse, >>::Out>, + TxEnv: TxEnvConverter, Evm>, +{ + type Primitives = N; + type Evm = Evm; + type Network = Network; + type Error = Receipt::Error; + + fn fill( + &self, + tx: Recovered>, + tx_info: TransactionInfo, + ) -> Result { + let (tx, signer) = tx.into_parts(); + let tx_info = self.mapper.try_map(&tx, tx_info)?; + + self.rpc_tx_converter.convert_rpc_tx(tx, signer, tx_info).map_err(Into::into) + } + + fn build_simulate_v1_transaction( + &self, + request: RpcTxReq, + ) -> Result, Self::Error> { + Ok(self + .sim_tx_converter + .convert_sim_tx(request) + .map_err(|e| TransactionConversionError(e.to_string()))?) + } + + fn tx_env( + &self, + request: RpcTxReq, + evm_env: &EvmEnvFor, + ) -> Result, Self::Error> { + self.tx_env_converter.convert_tx_env(request, evm_env).map_err(Into::into) + } + + fn convert_receipts( + &self, + receipts: Vec>, + ) -> Result>, Self::Error> { + self.receipt_converter.convert_receipts(receipts) + } + + fn convert_receipts_with_block( + &self, + receipts: Vec>, + block: &SealedBlock>, + ) -> Result>, Self::Error> { + self.receipt_converter.convert_receipts_with_block(receipts, block) + } + + fn convert_header( + &self, + header: SealedHeaderFor, + block_size: usize, + ) -> Result, Self::Error> { + Ok(self.header_converter.convert_header(header, block_size)?) + } +} + +/// Optimism specific RPC transaction compatibility implementations. +#[cfg(feature = "op")] +pub mod op { + use super::*; + use alloy_consensus::SignableTransaction; + use alloy_primitives::{Address, Bytes, Signature}; + use op_alloy_consensus::{ + transaction::{OpDepositInfo, OpTransactionInfo}, + OpTxEnvelope, + }; + use op_alloy_rpc_types::OpTransactionRequest; + use op_revm::OpTransaction; + use reth_optimism_primitives::DepositReceipt; + use reth_primitives_traits::SignedTransaction; + use reth_storage_api::{errors::ProviderError, ReceiptProvider}; + + /// Creates [`OpTransactionInfo`] by adding [`OpDepositInfo`] to [`TransactionInfo`] if `tx` is + /// a deposit. + pub fn try_into_op_tx_info( + provider: &T, + tx: &Tx, + tx_info: TransactionInfo, + ) -> Result + where + Tx: op_alloy_consensus::OpTransaction + SignedTransaction, + T: ReceiptProvider, + { + let deposit_meta = if tx.is_deposit() { + provider.receipt_by_hash(*tx.tx_hash())?.and_then(|receipt| { + receipt.as_deposit_receipt().map(|receipt| OpDepositInfo { + deposit_receipt_version: receipt.deposit_receipt_version, + deposit_nonce: receipt.deposit_nonce, + }) + }) + } else { + None + } + .unwrap_or_default(); + + Ok(OpTransactionInfo::new(tx_info, deposit_meta)) + } + + impl FromConsensusTx + for op_alloy_rpc_types::Transaction + { + type TxInfo = OpTransactionInfo; + type Err = Infallible; + + fn from_consensus_tx( + tx: T, + signer: Address, + tx_info: Self::TxInfo, + ) -> Result { + Ok(Self::from_transaction(Recovered::new_unchecked(tx, signer), tx_info)) + } + } + + impl TryIntoSimTx for OpTransactionRequest { + fn try_into_sim_tx(self) -> Result> { + let tx = self + .build_typed_tx() + .map_err(|request| ValueError::new(request, "Required fields missing"))?; + + // Create an empty signature for the transaction. + let signature = Signature::new(Default::default(), Default::default(), false); + + Ok(tx.into_signed(signature).into()) + } + } + + impl TryIntoTxEnv> for OpTransactionRequest { + type Err = EthTxEnvError; + + fn try_into_tx_env( + self, + cfg_env: &CfgEnv, + block_env: &BlockEnv, + ) -> Result, Self::Err> { + Ok(OpTransaction { + base: self.as_ref().clone().try_into_tx_env(cfg_env, block_env)?, + enveloped_tx: Some(Bytes::new()), + deposit: Default::default(), + }) + } + } +} + +/// Trait for converting network transaction responses to primitive transaction types. +pub trait TryFromTransactionResponse { + /// The error type returned if the conversion fails. + type Error: core::error::Error + Send + Sync + Unpin; + + /// Converts a network transaction response to a primitive transaction type. + /// + /// # Returns + /// + /// Returns `Ok(Self)` on successful conversion, or `Err(Self::Error)` if the conversion fails. + fn from_transaction_response( + transaction_response: N::TransactionResponse, + ) -> Result + where + Self: Sized; +} + +impl TryFromTransactionResponse + for reth_ethereum_primitives::TransactionSigned +{ + type Error = Infallible; + + fn from_transaction_response(transaction_response: Transaction) -> Result { + Ok(transaction_response.into_inner().into()) + } +} + +#[cfg(feature = "op")] +impl TryFromTransactionResponse + for reth_optimism_primitives::OpTransactionSigned +{ + type Error = Infallible; + + fn from_transaction_response( + transaction_response: op_alloy_rpc_types::Transaction, + ) -> Result { + Ok(transaction_response.inner.into_inner()) + } +} + +#[cfg(test)] +mod transaction_response_tests { + use super::*; + use alloy_consensus::{transaction::Recovered, EthereumTxEnvelope, Signed, TxLegacy}; + use alloy_network::Ethereum; + use alloy_primitives::{Address, Signature, B256, U256}; + use alloy_rpc_types_eth::Transaction; + + #[test] + fn test_ethereum_transaction_conversion() { + let signed_tx = Signed::new_unchecked( + TxLegacy::default(), + Signature::new(U256::ONE, U256::ONE, false), + B256::ZERO, + ); + let envelope = EthereumTxEnvelope::Legacy(signed_tx); + + let tx_response = Transaction { + inner: Recovered::new_unchecked(envelope, Address::ZERO), + block_hash: None, + block_number: None, + transaction_index: None, + effective_gas_price: None, + }; + + let result = >::from_transaction_response(tx_response); + assert!(result.is_ok()); + } + + #[cfg(feature = "op")] + mod op { + use super::*; + use crate::transaction::TryIntoTxEnv; + use revm_context::{BlockEnv, CfgEnv}; + + #[test] + fn test_optimism_transaction_conversion() { + use op_alloy_consensus::OpTxEnvelope; + use op_alloy_network::Optimism; + use reth_optimism_primitives::OpTransactionSigned; + + let signed_tx = Signed::new_unchecked( + TxLegacy::default(), + Signature::new(U256::ONE, U256::ONE, false), + B256::ZERO, + ); + let envelope = OpTxEnvelope::Legacy(signed_tx); + + let inner_tx = Transaction { + inner: Recovered::new_unchecked(envelope, Address::ZERO), + block_hash: None, + block_number: None, + transaction_index: None, + effective_gas_price: None, + }; + + let tx_response = op_alloy_rpc_types::Transaction { + inner: inner_tx, + deposit_nonce: None, + deposit_receipt_version: None, + }; + + let result = >::from_transaction_response(tx_response); + + assert!(result.is_ok()); + } + + #[test] + fn test_op_into_tx_env() { + use op_alloy_rpc_types::OpTransactionRequest; + use op_revm::{transaction::OpTxTr, OpSpecId}; + use revm_context::Transaction; + + let s = r#"{"from":"0x0000000000000000000000000000000000000000","to":"0x6d362b9c3ab68c0b7c79e8a714f1d7f3af63655f","input":"0x1626ba7ec8ee0d506e864589b799a645ddb88b08f5d39e8049f9f702b3b61fa15e55fc73000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000550000002d6db27c52e3c11c1cf24072004ac75cba49b25bf45f513902e469755e1f3bf2ca8324ad16930b0a965c012a24bb1101f876ebebac047bd3b6bf610205a27171eaaeffe4b5e5589936f4e542d637b627311b0000000000000000000000","data":"0x1626ba7ec8ee0d506e864589b799a645ddb88b08f5d39e8049f9f702b3b61fa15e55fc73000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000550000002d6db27c52e3c11c1cf24072004ac75cba49b25bf45f513902e469755e1f3bf2ca8324ad16930b0a965c012a24bb1101f876ebebac047bd3b6bf610205a27171eaaeffe4b5e5589936f4e542d637b627311b0000000000000000000000","chainId":"0x7a69"}"#; + + let req: OpTransactionRequest = serde_json::from_str(s).unwrap(); + + let cfg = CfgEnv::::default(); + let block_env = BlockEnv::default(); + let tx_env = req.try_into_tx_env(&cfg, &block_env).unwrap(); + assert_eq!(tx_env.gas_limit(), block_env.gas_limit); + assert_eq!(tx_env.gas_price(), 0); + assert!(tx_env.enveloped_tx().unwrap().is_empty()); + } + } +} diff --git a/crates/rpc/rpc-e2e-tests/Cargo.toml b/crates/rpc/rpc-e2e-tests/Cargo.toml new file mode 100644 index 00000000000..78c04740497 --- /dev/null +++ b/crates/rpc/rpc-e2e-tests/Cargo.toml @@ -0,0 +1,43 @@ +[package] +name = "reth-rpc-e2e-tests" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true +description = "RPC end-to-end tests including execution-apis compatibility testing" + +[lints] +workspace = true + +[dependencies] +# reth +reth-e2e-test-utils.workspace = true +reth-rpc-api = { workspace = true, features = ["client"] } + +# ethereum +alloy-rpc-types-engine.workspace = true + +# async +tokio.workspace = true +futures-util.workspace = true + +# misc +eyre.workspace = true +serde_json.workspace = true +tracing.workspace = true +jsonrpsee.workspace = true + +# required for the Action trait +reth-node-api.workspace = true + +[dev-dependencies] +reth-tracing.workspace = true +reth-chainspec.workspace = true +reth-node-ethereum.workspace = true +alloy-genesis.workspace = true + +[[test]] +name = "e2e_testsuite" +path = "tests/e2e-testsuite/main.rs" diff --git a/crates/rpc/rpc-e2e-tests/README.md b/crates/rpc/rpc-e2e-tests/README.md new file mode 100644 index 00000000000..03d6081cb2d --- /dev/null +++ b/crates/rpc/rpc-e2e-tests/README.md @@ -0,0 +1,174 @@ +# Reth RPC E2E Tests + +This crate contains end-to-end tests for Reth's RPC implementation, including compatibility testing against the official execution-apis test suite. + +## Overview + +The RPC compatibility testing framework enables: +1. Importing pre-built blockchain data from RLP files +2. Initializing nodes with specific forkchoice states +3. Running standardized RPC test cases from the execution-apis repository +4. Comparing responses against expected results + +## Architecture + +### Key Components + +1. **`RunRpcCompatTests` Action**: Executes RPC test cases from .io files +2. **`InitializeFromExecutionApis` Action**: Applies forkchoice state from JSON files with automatic retry for syncing nodes +3. **Test Data Format**: Uses execution-apis .io file format for test cases + +### Test Data Structure + +Expected directory structure: +``` +test_data_path/ +├── chain.rlp # Pre-built blockchain data +├── headfcu.json # Initial forkchoice state +├── genesis.json # Genesis configuration (optional) +└── eth_getLogs/ # Test cases for eth_getLogs + ├── contract-addr.io + ├── no-topics.io + ├── topic-exact-match.io + └── topic-wildcard.io +``` + +### .io File Format + +Test files use a simple request-response format: +``` +// Optional comment describing the test +// speconly: marks test as specification-only +>> {"jsonrpc":"2.0","id":1,"method":"eth_getLogs","params":[...]} +<< {"jsonrpc":"2.0","id":1,"result":[...]} +``` + +## Usage + +### Basic Example + +```rust +use alloy_genesis::Genesis; +use reth_chainspec::ChainSpec; +use reth_e2e_test_utils::testsuite::{ + actions::{MakeCanonical, UpdateBlockInfo}, + setup::{NetworkSetup, Setup}, + TestBuilder, +}; +use reth_rpc_e2e_tests::rpc_compat::{InitializeFromExecutionApis, RunRpcCompatTests}; + +#[tokio::test] +async fn test_eth_get_logs_compat() -> Result<()> { + let test_data_path = "../execution-apis/tests"; + let chain_rlp_path = PathBuf::from(&test_data_path).join("chain.rlp"); + let fcu_json_path = PathBuf::from(&test_data_path).join("headfcu.json"); + let genesis_path = PathBuf::from(&test_data_path).join("genesis.json"); + + // Parse genesis.json to get chain spec with all hardfork configuration + let genesis_json = std::fs::read_to_string(&genesis_path)?; + let genesis: Genesis = serde_json::from_str(&genesis_json)?; + let chain_spec: ChainSpec = genesis.into(); + let chain_spec = Arc::new(chain_spec); + + let setup = Setup::::default() + .with_chain_spec(chain_spec) + .with_network(NetworkSetup::single_node()); + + let test = TestBuilder::new() + .with_setup_and_import(setup, chain_rlp_path) + .with_action(UpdateBlockInfo::default()) + .with_action( + InitializeFromExecutionApis::new() + .with_fcu_json(fcu_json_path.to_string_lossy()), + ) + .with_action(MakeCanonical::new()) + .with_action(RunRpcCompatTests::new( + vec!["eth_getLogs".to_string()], + test_data_path.to_string_lossy(), + )); + + test.run::().await?; + Ok(()) +} +``` + +### Running Tests + +To run the official execution-apis test suite: + +1. Clone the execution-apis repository: + ```bash + git clone https://github.com/ethereum/execution-apis.git + ``` + +2. Set the test data path environment variable: + ```bash + export EXECUTION_APIS_TEST_PATH=/path/to/execution-apis/tests + ``` + +3. Run the execution-apis compatibility test: + ```bash + cargo nextest run --test e2e_testsuite test_execution_apis_compat + ``` + +This will auto-discover all RPC method directories and test each file individually, providing detailed per-file results. + +### Custom Test Data + +You can create custom test cases following the same format: + +1. Create a directory structure matching the execution-apis format +2. Write .io files with request-response pairs +3. Use the same testing framework with your custom path + +### Test Multiple RPC Methods + +```rust +let methods_to_test = vec![ + "eth_blockNumber".to_string(), + "eth_call".to_string(), + "eth_getLogs".to_string(), + "eth_getTransactionReceipt".to_string(), +]; + +RunRpcCompatTests::new(methods_to_test, test_data_path) + .with_fail_fast(true) // Stop on first failure +``` + +## Implementation Details + +### JSON-RPC Request Handling + +The framework handles various parameter formats: +- Empty parameters: `[]` +- Array parameters: `[param1, param2, ...]` +- Object parameters: Wrapped in array `[{...}]` + +### Response Comparison + +- **Numbers**: Compared with floating-point tolerance +- **Arrays**: Element-by-element comparison +- **Objects**: Key-by-key comparison (extra fields in actual response are allowed) +- **Errors**: Only presence is checked, not exact message + +### Error Handling + +- Parse errors are reported with context +- RPC errors are captured and compared +- Test failures include detailed diffs + +## Benefits + +1. **Standardization**: Uses official execution-apis test format +2. **Flexibility**: Works with custom test data +3. **Integration**: Seamlessly integrates with e2e test framework +4. **Extensibility**: Easy to add new RPC methods +5. **Debugging**: Detailed error reporting with fail-fast option + +## Future Enhancements + +- Support for batch requests +- WebSocket testing +- Performance benchmarking +- Automatic test discovery +- Parallel test execution \ No newline at end of file diff --git a/crates/rpc/rpc-types-compat/src/lib.rs b/crates/rpc/rpc-e2e-tests/src/lib.rs similarity index 54% rename from crates/rpc/rpc-types-compat/src/lib.rs rename to crates/rpc/rpc-e2e-tests/src/lib.rs index 40e2a20c4a9..376f03964f5 100644 --- a/crates/rpc/rpc-types-compat/src/lib.rs +++ b/crates/rpc/rpc-e2e-tests/src/lib.rs @@ -1,6 +1,4 @@ -//! Reth compatibility and utils for RPC types -//! -//! This crate various helper functions to convert between reth primitive types and rpc types. +//! RPC end-to-end tests including execution-apis compatibility testing. #![doc( html_logo_url = "https://raw.githubusercontent.com/paradigmxyz/reth/main/assets/reth-docs.png", @@ -8,8 +6,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] -pub mod block; -pub mod transaction; -pub use transaction::TransactionCompat; +/// RPC compatibility test actions for the e2e test framework +pub mod rpc_compat; diff --git a/crates/rpc/rpc-e2e-tests/src/rpc_compat.rs b/crates/rpc/rpc-e2e-tests/src/rpc_compat.rs new file mode 100644 index 00000000000..d6b48e3f4fb --- /dev/null +++ b/crates/rpc/rpc-e2e-tests/src/rpc_compat.rs @@ -0,0 +1,514 @@ +//! RPC compatibility test actions for testing RPC methods against execution-apis test data. + +use eyre::{eyre, Result}; +use futures_util::future::BoxFuture; +use jsonrpsee::core::client::ClientT; +use reth_e2e_test_utils::testsuite::{actions::Action, BlockInfo, Environment}; +use reth_node_api::EngineTypes; +use serde_json::Value; +use std::path::Path; +use tracing::{debug, info}; + +/// Test case from execution-apis .io file format +#[derive(Debug, Clone)] +pub struct RpcTestCase { + /// The test name (filename without .io extension) + pub name: String, + /// Request to send (as JSON value) + pub request: Value, + /// Expected response (as JSON value) + pub expected_response: Value, + /// Whether this test is spec-only + pub spec_only: bool, +} + +/// Action that runs RPC compatibility tests from execution-apis test data +#[derive(Debug)] +pub struct RunRpcCompatTests { + /// RPC methods to test (e.g. `eth_getLogs`) + pub methods: Vec, + /// Path to the execution-apis tests directory + pub test_data_path: String, + /// Whether to stop on first failure + pub fail_fast: bool, +} + +impl RunRpcCompatTests { + /// Create a new RPC compatibility test runner + pub fn new(methods: Vec, test_data_path: impl Into) -> Self { + Self { methods, test_data_path: test_data_path.into(), fail_fast: false } + } + + /// Set whether to stop on first failure + pub const fn with_fail_fast(mut self, fail_fast: bool) -> Self { + self.fail_fast = fail_fast; + self + } + + /// Parse a .io test file + fn parse_io_file(content: &str) -> Result { + let mut lines = content.lines(); + let mut spec_only = false; + let mut request_line = None; + let mut response_line = None; + + // Skip comments and look for spec_only marker + for line in lines.by_ref() { + let line = line.trim(); + if line.starts_with("//") { + if line.contains("speconly:") { + spec_only = true; + } + } else if let Some(stripped) = line.strip_prefix(">>") { + request_line = Some(stripped.trim()); + break; + } + } + + // Look for response + for line in lines { + let line = line.trim(); + if let Some(stripped) = line.strip_prefix("<<") { + response_line = Some(stripped.trim()); + break; + } + } + + let request_str = + request_line.ok_or_else(|| eyre!("No request found in test file (>> marker)"))?; + let response_str = + response_line.ok_or_else(|| eyre!("No response found in test file (<< marker)"))?; + + // Parse request + let request: Value = serde_json::from_str(request_str) + .map_err(|e| eyre!("Failed to parse request: {}", e))?; + + // Parse response + let expected_response: Value = serde_json::from_str(response_str) + .map_err(|e| eyre!("Failed to parse response: {}", e))?; + + Ok(RpcTestCase { name: String::new(), request, expected_response, spec_only }) + } + + /// Compare JSON values with special handling for numbers and errors + /// Uses iterative approach to avoid stack overflow with deeply nested structures + fn compare_json_values(actual: &Value, expected: &Value, path: &str) -> Result<()> { + // Stack to hold work items: (actual, expected, path) + let mut work_stack = vec![(actual, expected, path.to_string())]; + + while let Some((actual, expected, current_path)) = work_stack.pop() { + match (actual, expected) { + // Number comparison: handle different representations + (Value::Number(a), Value::Number(b)) => { + let a_f64 = a.as_f64().ok_or_else(|| eyre!("Invalid number"))?; + let b_f64 = b.as_f64().ok_or_else(|| eyre!("Invalid number"))?; + // Use a reasonable epsilon for floating point comparison + const EPSILON: f64 = 1e-10; + if (a_f64 - b_f64).abs() > EPSILON { + return Err(eyre!("Number mismatch at {}: {} != {}", current_path, a, b)); + } + } + // Array comparison + (Value::Array(a), Value::Array(b)) => { + if a.len() != b.len() { + return Err(eyre!( + "Array length mismatch at {}: {} != {}", + current_path, + a.len(), + b.len() + )); + } + // Add array elements to work stack in reverse order + // so they are processed in correct order + for (i, (av, bv)) in a.iter().zip(b.iter()).enumerate().rev() { + work_stack.push((av, bv, format!("{current_path}[{i}]"))); + } + } + // Object comparison + (Value::Object(a), Value::Object(b)) => { + // Check all keys in expected are present in actual + for (key, expected_val) in b { + if let Some(actual_val) = a.get(key) { + work_stack.push(( + actual_val, + expected_val, + format!("{current_path}.{key}"), + )); + } else { + return Err(eyre!("Missing key at {}.{}", current_path, key)); + } + } + } + // Direct value comparison + (a, b) => { + if a != b { + return Err(eyre!("Value mismatch at {}: {:?} != {:?}", current_path, a, b)); + } + } + } + } + Ok(()) + } + + /// Execute a single test case + async fn execute_test_case( + &self, + test_case: &RpcTestCase, + env: &Environment, + ) -> Result<()> { + let node_client = &env.node_clients[env.active_node_idx]; + + // Extract method and params from request + let method = test_case + .request + .get("method") + .and_then(|v| v.as_str()) + .ok_or_else(|| eyre!("Request missing method field"))?; + + let params = test_case.request.get("params").cloned().unwrap_or(Value::Array(vec![])); + + // Make the RPC request using jsonrpsee + // We need to handle the case where the RPC might return an error + use jsonrpsee::core::params::ArrayParams; + + let response_result: Result = match params { + Value::Array(ref arr) => { + // Use ArrayParams for array parameters + let mut array_params = ArrayParams::new(); + for param in arr { + array_params + .insert(param.clone()) + .map_err(|e| eyre!("Failed to insert param: {}", e))?; + } + node_client.rpc.request(method, array_params).await + } + _ => { + // For non-array params, wrap in an array + let mut array_params = ArrayParams::new(); + array_params.insert(params).map_err(|e| eyre!("Failed to insert param: {}", e))?; + node_client.rpc.request(method, array_params).await + } + }; + + // Build actual response object to match execution-apis format + let actual_response = match response_result { + Ok(response) => { + serde_json::json!({ + "jsonrpc": "2.0", + "id": test_case.request.get("id").cloned().unwrap_or(Value::Null), + "result": response + }) + } + Err(err) => { + // RPC error - build error response + serde_json::json!({ + "jsonrpc": "2.0", + "id": test_case.request.get("id").cloned().unwrap_or(Value::Null), + "error": { + "code": -32000, // Generic error code + "message": err.to_string() + } + }) + } + }; + + // Compare responses + let expected_result = test_case.expected_response.get("result"); + let expected_error = test_case.expected_response.get("error"); + let actual_result = actual_response.get("result"); + let actual_error = actual_response.get("error"); + + match (expected_result, expected_error) { + (Some(expected), None) => { + // Expected success response + if let Some(actual) = actual_result { + Self::compare_json_values(actual, expected, "result")?; + } else if let Some(error) = actual_error { + return Err(eyre!("Expected success response but got error: {}", error)); + } else { + return Err(eyre!("Expected success response but got neither result nor error")); + } + } + (None, Some(_)) => { + // Expected error response - just check that we got an error + if actual_error.is_none() { + return Err(eyre!("Expected error response but got success")); + } + debug!("Both responses are errors (expected behavior)"); + } + _ => { + return Err(eyre!("Invalid expected response format")); + } + } + + Ok(()) + } +} + +impl Action for RunRpcCompatTests +where + Engine: EngineTypes, +{ + fn execute<'a>(&'a mut self, env: &'a mut Environment) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + let mut total_tests = 0; + let mut passed_tests = 0; + + for method in &self.methods { + info!("Running RPC compatibility tests for {}", method); + + let method_dir = Path::new(&self.test_data_path).join(method); + if !method_dir.exists() { + return Err(eyre!("Test directory does not exist: {}", method_dir.display())); + } + + // Read all .io files in the method directory + let entries = std::fs::read_dir(&method_dir) + .map_err(|e| eyre!("Failed to read directory: {}", e))?; + + for entry in entries { + let entry = entry?; + let path = entry.path(); + + if path.extension().and_then(|s| s.to_str()) == Some("io") { + let test_name = path + .file_stem() + .and_then(|s| s.to_str()) + .unwrap_or("unknown") + .to_string(); + + let content = std::fs::read_to_string(&path) + .map_err(|e| eyre!("Failed to read test file: {}", e))?; + + match Self::parse_io_file(&content) { + Ok(mut test_case) => { + test_case.name = test_name.clone(); + total_tests += 1; + + match self.execute_test_case(&test_case, env).await { + Ok(_) => { + info!("✓ {}/{}: PASS", method, test_name); + passed_tests += 1; + } + Err(e) => { + info!("✗ {}/{}: FAIL - {}", method, test_name, e); + + if self.fail_fast { + return Err(eyre!("Test failed (fail-fast enabled)")); + } + } + } + } + Err(e) => { + info!("✗ {}/{}: PARSE ERROR - {}", method, test_name, e); + if self.fail_fast { + return Err(e); + } + } + } + } + } + } + + info!("RPC compatibility test results: {}/{} passed", passed_tests, total_tests); + + if passed_tests < total_tests { + return Err(eyre!("Some tests failed: {}/{} passed", passed_tests, total_tests)); + } + + Ok(()) + }) + } +} + +/// Action to initialize the chain from execution-apis test data +#[derive(Debug)] +pub struct InitializeFromExecutionApis { + /// Path to the base.rlp file (if different from default) + pub chain_rlp_path: Option, + /// Path to the headfcu.json file (if different from default) + pub fcu_json_path: Option, +} + +impl Default for InitializeFromExecutionApis { + fn default() -> Self { + Self::new() + } +} + +impl InitializeFromExecutionApis { + /// Create with default paths (assumes execution-apis/tests structure) + pub const fn new() -> Self { + Self { chain_rlp_path: None, fcu_json_path: None } + } + + /// Set custom chain RLP path + pub fn with_chain_rlp(mut self, path: impl Into) -> Self { + self.chain_rlp_path = Some(path.into()); + self + } + + /// Set custom FCU JSON path + pub fn with_fcu_json(mut self, path: impl Into) -> Self { + self.fcu_json_path = Some(path.into()); + self + } +} + +impl Action for InitializeFromExecutionApis +where + Engine: EngineTypes, +{ + fn execute<'a>(&'a mut self, env: &'a mut Environment) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + // Load forkchoice state + let fcu_path = self + .fcu_json_path + .as_ref() + .map(Path::new) + .ok_or_else(|| eyre!("FCU JSON path is required"))?; + + let fcu_state = reth_e2e_test_utils::setup_import::load_forkchoice_state(fcu_path)?; + + info!( + "Applying forkchoice state - head: {}, safe: {}, finalized: {}", + fcu_state.head_block_hash, + fcu_state.safe_block_hash, + fcu_state.finalized_block_hash + ); + + // Apply forkchoice update to each node + for (idx, client) in env.node_clients.iter().enumerate() { + debug!("Applying forkchoice update to node {}", idx); + + // Wait for the node to finish syncing imported blocks + let mut retries = 0; + const MAX_RETRIES: u32 = 10; + const RETRY_DELAY_MS: u64 = 500; + + loop { + let response = + reth_rpc_api::clients::EngineApiClient::::fork_choice_updated_v3( + &client.engine.http_client(), + fcu_state, + None, + ) + .await + .map_err(|e| eyre!("Failed to update forkchoice on node {}: {}", idx, e))?; + + match response.payload_status.status { + alloy_rpc_types_engine::PayloadStatusEnum::Valid => { + debug!("Forkchoice update successful on node {}", idx); + break; + } + alloy_rpc_types_engine::PayloadStatusEnum::Syncing => { + if retries >= MAX_RETRIES { + return Err(eyre!( + "Node {} still syncing after {} retries", + idx, + MAX_RETRIES + )); + } + debug!("Node {} is syncing, retrying in {}ms...", idx, RETRY_DELAY_MS); + tokio::time::sleep(std::time::Duration::from_millis(RETRY_DELAY_MS)) + .await; + retries += 1; + } + _ => { + return Err(eyre!( + "Invalid forkchoice state on node {}: {:?}", + idx, + response.payload_status + )); + } + } + } + } + + // Update environment state + env.active_node_state_mut()?.current_block_info = Some(BlockInfo { + hash: fcu_state.head_block_hash, + number: 0, // Will be updated when we fetch the actual block + timestamp: 0, + }); + + info!("Successfully initialized chain from execution-apis test data"); + Ok(()) + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn test_compare_json_values_deeply_nested() { + // Test that the iterative comparison handles deeply nested structures + // without stack overflow + let mut nested = json!({"value": 0}); + let mut expected = json!({"value": 0}); + + // Create a deeply nested structure + for i in 1..1000 { + nested = json!({"level": i, "nested": nested}); + expected = json!({"level": i, "nested": expected}); + } + + // Should not panic with stack overflow + RunRpcCompatTests::compare_json_values(&nested, &expected, "root").unwrap(); + } + + #[test] + fn test_compare_json_values_arrays() { + // Test array comparison + let actual = json!([1, 2, 3, 4, 5]); + let expected = json!([1, 2, 3, 4, 5]); + + RunRpcCompatTests::compare_json_values(&actual, &expected, "root").unwrap(); + + // Test array length mismatch + let actual = json!([1, 2, 3]); + let expected = json!([1, 2, 3, 4, 5]); + + let result = RunRpcCompatTests::compare_json_values(&actual, &expected, "root"); + assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains("Array length mismatch")); + } + + #[test] + fn test_compare_json_values_objects() { + // Test object comparison + let actual = json!({"a": 1, "b": 2, "c": 3}); + let expected = json!({"a": 1, "b": 2, "c": 3}); + + RunRpcCompatTests::compare_json_values(&actual, &expected, "root").unwrap(); + + // Test missing key + let actual = json!({"a": 1, "b": 2}); + let expected = json!({"a": 1, "b": 2, "c": 3}); + + let result = RunRpcCompatTests::compare_json_values(&actual, &expected, "root"); + assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains("Missing key")); + } + + #[test] + fn test_compare_json_values_numbers() { + // Test number comparison with floating point + let actual = json!({"value": 1.00000000001}); + let expected = json!({"value": 1.0}); + + // Should be equal within epsilon (1e-10) + RunRpcCompatTests::compare_json_values(&actual, &expected, "root").unwrap(); + + // Test significant difference + let actual = json!({"value": 1.1}); + let expected = json!({"value": 1.0}); + + let result = RunRpcCompatTests::compare_json_values(&actual, &expected, "root"); + assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains("Number mismatch")); + } +} diff --git a/crates/rpc/rpc-e2e-tests/testdata/rpc-compat/chain.rlp b/crates/rpc/rpc-e2e-tests/testdata/rpc-compat/chain.rlp new file mode 100644 index 00000000000..ae681adf9f0 Binary files /dev/null and b/crates/rpc/rpc-e2e-tests/testdata/rpc-compat/chain.rlp differ diff --git a/crates/rpc/rpc-e2e-tests/testdata/rpc-compat/eth_getLogs/contract-addr.io b/crates/rpc/rpc-e2e-tests/testdata/rpc-compat/eth_getLogs/contract-addr.io new file mode 100644 index 00000000000..674a7eb4f81 --- /dev/null +++ b/crates/rpc/rpc-e2e-tests/testdata/rpc-compat/eth_getLogs/contract-addr.io @@ -0,0 +1,3 @@ +// queries for logs from a specific contract across a range of blocks +>> {"jsonrpc":"2.0","id":1,"method":"eth_getLogs","params":[{"address":["0x7dcd17433742f4c0ca53122ab541d0ba67fc27df"],"fromBlock":"0x1","toBlock":"0x4","topics":null}]} +<< {"jsonrpc":"2.0","id":1,"result":[{"address":"0x7dcd17433742f4c0ca53122ab541d0ba67fc27df","topics":["0x00000000000000000000000000000000000000000000000000000000656d6974","0xf4da19d6c17928e683661a52829cf391d3dc26d581152b81ce595a1207944f09"],"data":"0x0000000000000000000000000000000000000000000000000000000000000000","blockNumber":"0x2","transactionHash":"0x5bc704d4eb4ce7fe319705d2f888516961426a177f2799c9f934b5df7466dd33","transactionIndex":"0x2","blockHash":"0x28a64e8d846382eb270941251be3a3e1547809e7eb70939c3530faa8f4599570","logIndex":"0xa","removed":false},{"address":"0x7dcd17433742f4c0ca53122ab541d0ba67fc27df","topics":["0x00000000000000000000000000000000000000000000000000000000656d6974","0x4238ace0bf7e66fd40fea01bdf43f4f30423f48432efd0da3af5fcb17a977fd4"],"data":"0x0000000000000000000000000000000000000000000000000000000000000001","blockNumber":"0x4","transactionHash":"0xf047c5133c96c405a79d01038b4ccf8208c03e296dd9f6bea083727c9513f805","transactionIndex":"0x0","blockHash":"0x94540b21748e45497c41518ed68b2a0c16d728e917b665ae50d51f6895242e53","logIndex":"0x0","removed":false}]} diff --git a/crates/rpc/rpc-e2e-tests/testdata/rpc-compat/eth_getLogs/no-topics.io b/crates/rpc/rpc-e2e-tests/testdata/rpc-compat/eth_getLogs/no-topics.io new file mode 100644 index 00000000000..89ec5bcd058 --- /dev/null +++ b/crates/rpc/rpc-e2e-tests/testdata/rpc-compat/eth_getLogs/no-topics.io @@ -0,0 +1,3 @@ +// queries for all logs across a range of blocks +>> {"jsonrpc":"2.0","id":1,"method":"eth_getLogs","params":[{"address":null,"fromBlock":"0x1","toBlock":"0x3","topics":null}]} +<< {"jsonrpc":"2.0","id":1,"result":[{"address":"0x882e7e5d12617c267a72948e716f231fa79e6d51","topics":["0xabbb5caa7dda850e60932de0934eb1f9d0f59695050f761dc64e443e5030a569"],"data":"0x0000000000000000000000000000000000000000000000000000000000000001","blockNumber":"0x2","transactionHash":"0x25d8b4a27c4578e5de6441f98881cf050ab2d9f28ceb28559ece0b65f555e9d8","transactionIndex":"0x0","blockHash":"0x28a64e8d846382eb270941251be3a3e1547809e7eb70939c3530faa8f4599570","logIndex":"0x0","removed":false},{"address":"0x882e7e5d12617c267a72948e716f231fa79e6d51","topics":["0xd9d16d34ffb15ba3a3d852f0d403e2ce1d691fb54de27ac87cd2f993f3ec330f"],"data":"0x0000000000000000000000000000000000000000000000000000000000000002","blockNumber":"0x2","transactionHash":"0x25d8b4a27c4578e5de6441f98881cf050ab2d9f28ceb28559ece0b65f555e9d8","transactionIndex":"0x0","blockHash":"0x28a64e8d846382eb270941251be3a3e1547809e7eb70939c3530faa8f4599570","logIndex":"0x1","removed":false},{"address":"0x882e7e5d12617c267a72948e716f231fa79e6d51","topics":["0x679795a0195a1b76cdebb7c51d74e058aee92919b8c3389af86ef24535e8a28c"],"data":"0x0000000000000000000000000000000000000000000000000000000000000003","blockNumber":"0x2","transactionHash":"0x25d8b4a27c4578e5de6441f98881cf050ab2d9f28ceb28559ece0b65f555e9d8","transactionIndex":"0x0","blockHash":"0x28a64e8d846382eb270941251be3a3e1547809e7eb70939c3530faa8f4599570","logIndex":"0x2","removed":false},{"address":"0x882e7e5d12617c267a72948e716f231fa79e6d51","topics":["0xc3a24b0501bd2c13a7e57f2db4369ec4c223447539fc0724a9d55ac4a06ebd4d"],"data":"0x0000000000000000000000000000000000000000000000000000000000000004","blockNumber":"0x2","transactionHash":"0x25d8b4a27c4578e5de6441f98881cf050ab2d9f28ceb28559ece0b65f555e9d8","transactionIndex":"0x0","blockHash":"0x28a64e8d846382eb270941251be3a3e1547809e7eb70939c3530faa8f4599570","logIndex":"0x3","removed":false},{"address":"0x882e7e5d12617c267a72948e716f231fa79e6d51","topics":["0x91da3fd0782e51c6b3986e9e672fd566868e71f3dbc2d6c2cd6fbb3e361af2a7"],"data":"0x0000000000000000000000000000000000000000000000000000000000000005","blockNumber":"0x2","transactionHash":"0x25d8b4a27c4578e5de6441f98881cf050ab2d9f28ceb28559ece0b65f555e9d8","transactionIndex":"0x0","blockHash":"0x28a64e8d846382eb270941251be3a3e1547809e7eb70939c3530faa8f4599570","logIndex":"0x4","removed":false},{"address":"0x882e7e5d12617c267a72948e716f231fa79e6d51","topics":["0x89832631fb3c3307a103ba2c84ab569c64d6182a18893dcd163f0f1c2090733a"],"data":"0x0000000000000000000000000000000000000000000000000000000000000006","blockNumber":"0x2","transactionHash":"0x25d8b4a27c4578e5de6441f98881cf050ab2d9f28ceb28559ece0b65f555e9d8","transactionIndex":"0x0","blockHash":"0x28a64e8d846382eb270941251be3a3e1547809e7eb70939c3530faa8f4599570","logIndex":"0x5","removed":false},{"address":"0x882e7e5d12617c267a72948e716f231fa79e6d51","topics":["0x8819ef417987f8ae7a81f42cdfb18815282fe989326fbff903d13cf0e03ace29"],"data":"0x0000000000000000000000000000000000000000000000000000000000000007","blockNumber":"0x2","transactionHash":"0x25d8b4a27c4578e5de6441f98881cf050ab2d9f28ceb28559ece0b65f555e9d8","transactionIndex":"0x0","blockHash":"0x28a64e8d846382eb270941251be3a3e1547809e7eb70939c3530faa8f4599570","logIndex":"0x6","removed":false},{"address":"0x882e7e5d12617c267a72948e716f231fa79e6d51","topics":["0xb7c774451310d1be4108bc180d1b52823cb0ee0274a6c0081bcaf94f115fb96d"],"data":"0x0000000000000000000000000000000000000000000000000000000000000008","blockNumber":"0x2","transactionHash":"0x25d8b4a27c4578e5de6441f98881cf050ab2d9f28ceb28559ece0b65f555e9d8","transactionIndex":"0x0","blockHash":"0x28a64e8d846382eb270941251be3a3e1547809e7eb70939c3530faa8f4599570","logIndex":"0x7","removed":false},{"address":"0x882e7e5d12617c267a72948e716f231fa79e6d51","topics":["0x6add646517a5b0f6793cd5891b7937d28a5b2981a5d88ebc7cd776088fea9041"],"data":"0x0000000000000000000000000000000000000000000000000000000000000009","blockNumber":"0x2","transactionHash":"0x25d8b4a27c4578e5de6441f98881cf050ab2d9f28ceb28559ece0b65f555e9d8","transactionIndex":"0x0","blockHash":"0x28a64e8d846382eb270941251be3a3e1547809e7eb70939c3530faa8f4599570","logIndex":"0x8","removed":false},{"address":"0x882e7e5d12617c267a72948e716f231fa79e6d51","topics":["0x6cde3cea4b3a3fb2488b2808bae7556f4a405e50f65e1794383bc026131b13c3"],"data":"0x000000000000000000000000000000000000000000000000000000000000000a","blockNumber":"0x2","transactionHash":"0x25d8b4a27c4578e5de6441f98881cf050ab2d9f28ceb28559ece0b65f555e9d8","transactionIndex":"0x0","blockHash":"0x28a64e8d846382eb270941251be3a3e1547809e7eb70939c3530faa8f4599570","logIndex":"0x9","removed":false},{"address":"0x7dcd17433742f4c0ca53122ab541d0ba67fc27df","topics":["0x00000000000000000000000000000000000000000000000000000000656d6974","0xf4da19d6c17928e683661a52829cf391d3dc26d581152b81ce595a1207944f09"],"data":"0x0000000000000000000000000000000000000000000000000000000000000000","blockNumber":"0x2","transactionHash":"0x5bc704d4eb4ce7fe319705d2f888516961426a177f2799c9f934b5df7466dd33","transactionIndex":"0x2","blockHash":"0x28a64e8d846382eb270941251be3a3e1547809e7eb70939c3530faa8f4599570","logIndex":"0xa","removed":false},{"address":"0xa788ca96a910bac854f95b794776c1ad847dcdd5","topics":["0x101e368776582e57ab3d116ffe2517c0a585cd5b23174b01e275c2d8329c3d83"],"data":"0x0000000000000000000000000000000000000000000000000000000000000001","blockNumber":"0x3","transactionHash":"0xfecc4d4439d77962b8c27e7e652294717fcd75379cab400bbefb2975a960344c","transactionIndex":"0x1","blockHash":"0x30adf30837c967524cbcf881c024c194eee010b3750feef2e45a674979b2cd36","logIndex":"0x0","removed":false},{"address":"0xa788ca96a910bac854f95b794776c1ad847dcdd5","topics":["0x7dfe757ecd65cbd7922a9c0161e935dd7fdbcc0e999689c7d31633896b1fc60b"],"data":"0x0000000000000000000000000000000000000000000000000000000000000002","blockNumber":"0x3","transactionHash":"0xfecc4d4439d77962b8c27e7e652294717fcd75379cab400bbefb2975a960344c","transactionIndex":"0x1","blockHash":"0x30adf30837c967524cbcf881c024c194eee010b3750feef2e45a674979b2cd36","logIndex":"0x1","removed":false},{"address":"0xa788ca96a910bac854f95b794776c1ad847dcdd5","topics":["0x88601476d11616a71c5be67555bd1dff4b1cbf21533d2669b768b61518cfe1c3"],"data":"0x0000000000000000000000000000000000000000000000000000000000000003","blockNumber":"0x3","transactionHash":"0xfecc4d4439d77962b8c27e7e652294717fcd75379cab400bbefb2975a960344c","transactionIndex":"0x1","blockHash":"0x30adf30837c967524cbcf881c024c194eee010b3750feef2e45a674979b2cd36","logIndex":"0x2","removed":false},{"address":"0xa788ca96a910bac854f95b794776c1ad847dcdd5","topics":["0xcbc4e5fb02c3d1de23a9f1e014b4d2ee5aeaea9505df5e855c9210bf472495af"],"data":"0x0000000000000000000000000000000000000000000000000000000000000004","blockNumber":"0x3","transactionHash":"0xfecc4d4439d77962b8c27e7e652294717fcd75379cab400bbefb2975a960344c","transactionIndex":"0x1","blockHash":"0x30adf30837c967524cbcf881c024c194eee010b3750feef2e45a674979b2cd36","logIndex":"0x3","removed":false},{"address":"0xa788ca96a910bac854f95b794776c1ad847dcdd5","topics":["0x2e174c10e159ea99b867ce3205125c24a42d128804e4070ed6fcc8cc98166aa0"],"data":"0x0000000000000000000000000000000000000000000000000000000000000005","blockNumber":"0x3","transactionHash":"0xfecc4d4439d77962b8c27e7e652294717fcd75379cab400bbefb2975a960344c","transactionIndex":"0x1","blockHash":"0x30adf30837c967524cbcf881c024c194eee010b3750feef2e45a674979b2cd36","logIndex":"0x4","removed":false},{"address":"0xa788ca96a910bac854f95b794776c1ad847dcdd5","topics":["0xa9bc9a3a348c357ba16b37005d7e6b3236198c0e939f4af8c5f19b8deeb8ebc0"],"data":"0x0000000000000000000000000000000000000000000000000000000000000006","blockNumber":"0x3","transactionHash":"0xfecc4d4439d77962b8c27e7e652294717fcd75379cab400bbefb2975a960344c","transactionIndex":"0x1","blockHash":"0x30adf30837c967524cbcf881c024c194eee010b3750feef2e45a674979b2cd36","logIndex":"0x5","removed":false},{"address":"0xa788ca96a910bac854f95b794776c1ad847dcdd5","topics":["0x75f96ab15d697e93042dc45b5c896c4b27e89bb6eaf39475c5c371cb2513f7d2"],"data":"0x0000000000000000000000000000000000000000000000000000000000000007","blockNumber":"0x3","transactionHash":"0xfecc4d4439d77962b8c27e7e652294717fcd75379cab400bbefb2975a960344c","transactionIndex":"0x1","blockHash":"0x30adf30837c967524cbcf881c024c194eee010b3750feef2e45a674979b2cd36","logIndex":"0x6","removed":false},{"address":"0xa788ca96a910bac854f95b794776c1ad847dcdd5","topics":["0x3be6fd20d5acfde5b873b48692cd31f4d3c7e8ee8a813af4696af8859e5ca6c6"],"data":"0x0000000000000000000000000000000000000000000000000000000000000008","blockNumber":"0x3","transactionHash":"0xfecc4d4439d77962b8c27e7e652294717fcd75379cab400bbefb2975a960344c","transactionIndex":"0x1","blockHash":"0x30adf30837c967524cbcf881c024c194eee010b3750feef2e45a674979b2cd36","logIndex":"0x7","removed":false},{"address":"0xa788ca96a910bac854f95b794776c1ad847dcdd5","topics":["0x625b35f5e76f098dd7c3a05b10e2e5e78a4a01228d60c3b143426cdf36d26455"],"data":"0x0000000000000000000000000000000000000000000000000000000000000009","blockNumber":"0x3","transactionHash":"0xfecc4d4439d77962b8c27e7e652294717fcd75379cab400bbefb2975a960344c","transactionIndex":"0x1","blockHash":"0x30adf30837c967524cbcf881c024c194eee010b3750feef2e45a674979b2cd36","logIndex":"0x8","removed":false},{"address":"0xa788ca96a910bac854f95b794776c1ad847dcdd5","topics":["0xc575c31fea594a6eb97c8e9d3f9caee4c16218c6ef37e923234c0fe9014a61e7"],"data":"0x000000000000000000000000000000000000000000000000000000000000000a","blockNumber":"0x3","transactionHash":"0xfecc4d4439d77962b8c27e7e652294717fcd75379cab400bbefb2975a960344c","transactionIndex":"0x1","blockHash":"0x30adf30837c967524cbcf881c024c194eee010b3750feef2e45a674979b2cd36","logIndex":"0x9","removed":false}]} diff --git a/crates/rpc/rpc-e2e-tests/testdata/rpc-compat/eth_getLogs/topic-exact-match.io b/crates/rpc/rpc-e2e-tests/testdata/rpc-compat/eth_getLogs/topic-exact-match.io new file mode 100644 index 00000000000..30366e8005e --- /dev/null +++ b/crates/rpc/rpc-e2e-tests/testdata/rpc-compat/eth_getLogs/topic-exact-match.io @@ -0,0 +1,3 @@ +// queries for logs with two topics, with both topics set explicitly +>> {"jsonrpc":"2.0","id":1,"method":"eth_getLogs","params":[{"address":null,"fromBlock":"0x3","toBlock":"0x6","topics":[["0x00000000000000000000000000000000000000000000000000000000656d6974"],["0x4238ace0bf7e66fd40fea01bdf43f4f30423f48432efd0da3af5fcb17a977fd4"]]}]} +<< {"jsonrpc":"2.0","id":1,"result":[{"address":"0x7dcd17433742f4c0ca53122ab541d0ba67fc27df","topics":["0x00000000000000000000000000000000000000000000000000000000656d6974","0x4238ace0bf7e66fd40fea01bdf43f4f30423f48432efd0da3af5fcb17a977fd4"],"data":"0x0000000000000000000000000000000000000000000000000000000000000001","blockNumber":"0x4","transactionHash":"0xf047c5133c96c405a79d01038b4ccf8208c03e296dd9f6bea083727c9513f805","transactionIndex":"0x0","blockHash":"0x94540b21748e45497c41518ed68b2a0c16d728e917b665ae50d51f6895242e53","logIndex":"0x0","removed":false}]} diff --git a/crates/rpc/rpc-e2e-tests/testdata/rpc-compat/eth_getLogs/topic-wildcard.io b/crates/rpc/rpc-e2e-tests/testdata/rpc-compat/eth_getLogs/topic-wildcard.io new file mode 100644 index 00000000000..9a798698c25 --- /dev/null +++ b/crates/rpc/rpc-e2e-tests/testdata/rpc-compat/eth_getLogs/topic-wildcard.io @@ -0,0 +1,3 @@ +// queries for logs with two topics, performing a wildcard match in topic position zero +>> {"jsonrpc":"2.0","id":1,"method":"eth_getLogs","params":[{"address":null,"fromBlock":"0x3","toBlock":"0x6","topics":[[],["0x4238ace0bf7e66fd40fea01bdf43f4f30423f48432efd0da3af5fcb17a977fd4"]]}]} +<< {"jsonrpc":"2.0","id":1,"result":[{"address":"0x7dcd17433742f4c0ca53122ab541d0ba67fc27df","topics":["0x00000000000000000000000000000000000000000000000000000000656d6974","0x4238ace0bf7e66fd40fea01bdf43f4f30423f48432efd0da3af5fcb17a977fd4"],"data":"0x0000000000000000000000000000000000000000000000000000000000000001","blockNumber":"0x4","transactionHash":"0xf047c5133c96c405a79d01038b4ccf8208c03e296dd9f6bea083727c9513f805","transactionIndex":"0x0","blockHash":"0x94540b21748e45497c41518ed68b2a0c16d728e917b665ae50d51f6895242e53","logIndex":"0x0","removed":false}]} diff --git a/crates/rpc/rpc-e2e-tests/testdata/rpc-compat/eth_syncing/eth_syncing.io b/crates/rpc/rpc-e2e-tests/testdata/rpc-compat/eth_syncing/eth_syncing.io new file mode 100644 index 00000000000..3aba3c1eb79 --- /dev/null +++ b/crates/rpc/rpc-e2e-tests/testdata/rpc-compat/eth_syncing/eth_syncing.io @@ -0,0 +1,3 @@ +// checks client syncing status +>> {"jsonrpc":"2.0","id":1,"method":"eth_syncing"} +<< {"jsonrpc":"2.0","id":1,"result":false} diff --git a/crates/rpc/rpc-e2e-tests/testdata/rpc-compat/forkenv.json b/crates/rpc/rpc-e2e-tests/testdata/rpc-compat/forkenv.json new file mode 100644 index 00000000000..3da23534337 --- /dev/null +++ b/crates/rpc/rpc-e2e-tests/testdata/rpc-compat/forkenv.json @@ -0,0 +1,27 @@ +{ + "HIVE_CANCUN_BLOB_BASE_FEE_UPDATE_FRACTION": "3338477", + "HIVE_CANCUN_BLOB_MAX": "6", + "HIVE_CANCUN_BLOB_TARGET": "3", + "HIVE_CANCUN_TIMESTAMP": "420", + "HIVE_CHAIN_ID": "3503995874084926", + "HIVE_FORK_ARROW_GLACIER": "30", + "HIVE_FORK_BERLIN": "24", + "HIVE_FORK_BYZANTIUM": "9", + "HIVE_FORK_CONSTANTINOPLE": "12", + "HIVE_FORK_GRAY_GLACIER": "33", + "HIVE_FORK_HOMESTEAD": "0", + "HIVE_FORK_ISTANBUL": "18", + "HIVE_FORK_LONDON": "27", + "HIVE_FORK_MUIR_GLACIER": "21", + "HIVE_FORK_PETERSBURG": "15", + "HIVE_FORK_SPURIOUS": "6", + "HIVE_FORK_TANGERINE": "3", + "HIVE_MERGE_BLOCK_ID": "36", + "HIVE_NETWORK_ID": "3503995874084926", + "HIVE_PRAGUE_BLOB_BASE_FEE_UPDATE_FRACTION": "5007716", + "HIVE_PRAGUE_BLOB_MAX": "9", + "HIVE_PRAGUE_BLOB_TARGET": "6", + "HIVE_PRAGUE_TIMESTAMP": "450", + "HIVE_SHANGHAI_TIMESTAMP": "390", + "HIVE_TERMINAL_TOTAL_DIFFICULTY": "4732736" +} \ No newline at end of file diff --git a/crates/rpc/rpc-e2e-tests/testdata/rpc-compat/genesis.json b/crates/rpc/rpc-e2e-tests/testdata/rpc-compat/genesis.json new file mode 100644 index 00000000000..0c29edcb252 --- /dev/null +++ b/crates/rpc/rpc-e2e-tests/testdata/rpc-compat/genesis.json @@ -0,0 +1,141 @@ +{ + "config": { + "chainId": 3503995874084926, + "homesteadBlock": 0, + "eip150Block": 3, + "eip155Block": 6, + "eip158Block": 6, + "byzantiumBlock": 9, + "constantinopleBlock": 12, + "petersburgBlock": 15, + "istanbulBlock": 18, + "muirGlacierBlock": 21, + "berlinBlock": 24, + "londonBlock": 27, + "arrowGlacierBlock": 30, + "grayGlacierBlock": 33, + "mergeNetsplitBlock": 36, + "shanghaiTime": 390, + "cancunTime": 420, + "pragueTime": 450, + "terminalTotalDifficulty": 4732736, + "depositContractAddress": "0x0000000000000000000000000000000000000000", + "ethash": {}, + "blobSchedule": { + "cancun": { + "target": 3, + "max": 6, + "baseFeeUpdateFraction": 3338477 + }, + "prague": { + "target": 6, + "max": 9, + "baseFeeUpdateFraction": 5007716 + } + } + }, + "nonce": "0x0", + "timestamp": "0x0", + "extraData": "0x68697665636861696e", + "gasLimit": "0x23f3e20", + "difficulty": "0x20000", + "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "coinbase": "0x0000000000000000000000000000000000000000", + "alloc": { + "00000961ef480eb55e80d19ad83579a64c007002": { + "code": "0x3373fffffffffffffffffffffffffffffffffffffffe1460cb5760115f54807fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff146101f457600182026001905f5b5f82111560685781019083028483029004916001019190604d565b909390049250505036603814608857366101f457346101f4575f5260205ff35b34106101f457600154600101600155600354806003026004013381556001015f35815560010160203590553360601b5f5260385f601437604c5fa0600101600355005b6003546002548082038060101160df575060105b5f5b8181146101835782810160030260040181604c02815460601b8152601401816001015481526020019060020154807fffffffffffffffffffffffffffffffff00000000000000000000000000000000168252906010019060401c908160381c81600701538160301c81600601538160281c81600501538160201c81600401538160181c81600301538160101c81600201538160081c81600101535360010160e1565b910180921461019557906002556101a0565b90505f6002555f6003555b5f54807fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff14156101cd57505f5b6001546002828201116101e25750505f6101e8565b01600290035b5f555f600155604c025ff35b5f5ffd", + "balance": "0x1" + }, + "0000bbddc7ce488642fb579f8b00f3a590007251": { + "code": "0x3373fffffffffffffffffffffffffffffffffffffffe1460d35760115f54807fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1461019a57600182026001905f5b5f82111560685781019083028483029004916001019190604d565b9093900492505050366060146088573661019a573461019a575f5260205ff35b341061019a57600154600101600155600354806004026004013381556001015f358155600101602035815560010160403590553360601b5f5260605f60143760745fa0600101600355005b6003546002548082038060021160e7575060025b5f5b8181146101295782810160040260040181607402815460601b815260140181600101548152602001816002015481526020019060030154905260010160e9565b910180921461013b5790600255610146565b90505f6002555f6003555b5f54807fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff141561017357505f5b6001546001828201116101885750505f61018e565b01600190035b5f555f6001556074025ff35b5f5ffd", + "balance": "0x1" + }, + "0000f90827f1c53a10cb7a02335b175320002935": { + "code": "0x3373fffffffffffffffffffffffffffffffffffffffe14604657602036036042575f35600143038111604257611fff81430311604257611fff9006545f5260205ff35b5f5ffd5b5f35611fff60014303065500", + "balance": "0x1" + }, + "000f3df6d732807ef1319fb7b8bb8522d0beac02": { + "code": "0x3373fffffffffffffffffffffffffffffffffffffffe14604d57602036146024575f5ffd5b5f35801560495762001fff810690815414603c575f5ffd5b62001fff01545f5260205ff35b5f5ffd5b62001fff42064281555f359062001fff015500", + "balance": "0x2a" + }, + "0c2c51a0990aee1d73c1228de158688341557508": { + "balance": "0xc097ce7bc90715b34b9f1000000000" + }, + "14e46043e63d0e3cdcf2530519f4cfaf35058cb2": { + "balance": "0xc097ce7bc90715b34b9f1000000000" + }, + "16c57edf7fa9d9525378b0b81bf8a3ced0620c1c": { + "balance": "0xc097ce7bc90715b34b9f1000000000" + }, + "1f4924b14f34e24159387c0a4cdbaa32f3ddb0cf": { + "balance": "0xc097ce7bc90715b34b9f1000000000" + }, + "1f5bde34b4afc686f136c7a3cb6ec376f7357759": { + "balance": "0xc097ce7bc90715b34b9f1000000000" + }, + "2d389075be5be9f2246ad654ce152cf05990b209": { + "balance": "0xc097ce7bc90715b34b9f1000000000" + }, + "3ae75c08b4c907eb63a8960c45b86e1e9ab6123c": { + "balance": "0xc097ce7bc90715b34b9f1000000000" + }, + "4340ee1b812acb40a1eb561c019c327b243b92df": { + "balance": "0xc097ce7bc90715b34b9f1000000000" + }, + "4a0f1452281bcec5bd90c3dce6162a5995bfe9df": { + "balance": "0xc097ce7bc90715b34b9f1000000000" + }, + "4dde844b71bcdf95512fb4dc94e84fb67b512ed8": { + "balance": "0xc097ce7bc90715b34b9f1000000000" + }, + "5f552da00dfb4d3749d9e62dcee3c918855a86a0": { + "balance": "0xc097ce7bc90715b34b9f1000000000" + }, + "654aa64f5fbefb84c270ec74211b81ca8c44a72e": { + "balance": "0xc097ce7bc90715b34b9f1000000000" + }, + "717f8aa2b982bee0e29f573d31df288663e1ce16": { + "balance": "0xc097ce7bc90715b34b9f1000000000" + }, + "7435ed30a8b4aeb0877cef0c6e8cffe834eb865f": { + "balance": "0xc097ce7bc90715b34b9f1000000000" + }, + "7dcd17433742f4c0ca53122ab541d0ba67fc27df": { + "code": "0x3680600080376000206000548082558060010160005560005263656d697460206000a2", + "balance": "0x0" + }, + "83c7e323d189f18725ac510004fdc2941f8c4a78": { + "balance": "0xc097ce7bc90715b34b9f1000000000" + }, + "84e75c28348fb86acea1a93a39426d7d60f4cc46": { + "balance": "0xc097ce7bc90715b34b9f1000000000" + }, + "8bebc8ba651aee624937e7d897853ac30c95a067": { + "storage": { + "0x0000000000000000000000000000000000000000000000000000000000000001": "0x0000000000000000000000000000000000000000000000000000000000000001", + "0x0000000000000000000000000000000000000000000000000000000000000002": "0x0000000000000000000000000000000000000000000000000000000000000002", + "0x0000000000000000000000000000000000000000000000000000000000000003": "0x0000000000000000000000000000000000000000000000000000000000000003" + }, + "balance": "0x1", + "nonce": "0x1" + }, + "c7b99a164efd027a93f147376cc7da7c67c6bbe0": { + "balance": "0xc097ce7bc90715b34b9f1000000000" + }, + "d803681e487e6ac18053afc5a6cd813c86ec3e4d": { + "balance": "0xc097ce7bc90715b34b9f1000000000" + }, + "e7d13f7aa2a838d24c59b40186a0aca1e21cffcc": { + "balance": "0xc097ce7bc90715b34b9f1000000000" + }, + "eda8645ba6948855e3b3cd596bbb07596d59c603": { + "balance": "0xc097ce7bc90715b34b9f1000000000" + } + }, + "number": "0x0", + "gasUsed": "0x0", + "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "baseFeePerGas": null, + "excessBlobGas": null, + "blobGasUsed": null +} \ No newline at end of file diff --git a/crates/rpc/rpc-e2e-tests/testdata/rpc-compat/headfcu.json b/crates/rpc/rpc-e2e-tests/testdata/rpc-compat/headfcu.json new file mode 100644 index 00000000000..cc39610b4f1 --- /dev/null +++ b/crates/rpc/rpc-e2e-tests/testdata/rpc-compat/headfcu.json @@ -0,0 +1,13 @@ +{ + "jsonrpc": "2.0", + "id": "fcu45", + "method": "engine_forkchoiceUpdatedV3", + "params": [ + { + "headBlockHash": "0xaf51811799f22260e5b4e1f95504dae760505f102dcb2e9ca7d897d8a40124a1", + "safeBlockHash": "0xaf51811799f22260e5b4e1f95504dae760505f102dcb2e9ca7d897d8a40124a1", + "finalizedBlockHash": "0xaf51811799f22260e5b4e1f95504dae760505f102dcb2e9ca7d897d8a40124a1" + }, + null + ] +} \ No newline at end of file diff --git a/crates/rpc/rpc-e2e-tests/tests/e2e-testsuite/main.rs b/crates/rpc/rpc-e2e-tests/tests/e2e-testsuite/main.rs new file mode 100644 index 00000000000..e1a4a249799 --- /dev/null +++ b/crates/rpc/rpc-e2e-tests/tests/e2e-testsuite/main.rs @@ -0,0 +1,173 @@ +//! RPC compatibility tests using execution-apis test data + +use alloy_genesis::Genesis; +use eyre::Result; +use reth_chainspec::ChainSpec; +use reth_e2e_test_utils::testsuite::{ + actions::{MakeCanonical, UpdateBlockInfo}, + setup::{NetworkSetup, Setup}, + TestBuilder, +}; +use reth_node_ethereum::{EthEngineTypes, EthereumNode}; +use reth_rpc_e2e_tests::rpc_compat::{InitializeFromExecutionApis, RunRpcCompatTests}; +use std::{env, path::PathBuf, sync::Arc}; +use tracing::{debug, info}; + +/// Test repo-local RPC method compatibility with execution-apis test data +/// +/// This test: +/// 1. Initializes a node with chain data from testdata (chain.rlp) +/// 2. Applies the forkchoice state from headfcu.json +/// 3. Runs tests cases in the local repository, some of which are execution-api tests +#[tokio::test(flavor = "multi_thread")] +async fn test_local_rpc_tests_compat() -> Result<()> { + reth_tracing::init_test_tracing(); + + // Use local test data + let test_data_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("testdata/rpc-compat"); + + assert!(test_data_path.exists(), "Test data path does not exist: {}", test_data_path.display()); + + info!("Using test data from: {}", test_data_path.display()); + + // Paths to test files + let chain_rlp_path = test_data_path.join("chain.rlp"); + let fcu_json_path = test_data_path.join("headfcu.json"); + let genesis_path = test_data_path.join("genesis.json"); + + // Verify required files exist + if !chain_rlp_path.exists() { + return Err(eyre::eyre!("chain.rlp not found at {}", chain_rlp_path.display())); + } + if !fcu_json_path.exists() { + return Err(eyre::eyre!("headfcu.json not found at {}", fcu_json_path.display())); + } + if !genesis_path.exists() { + return Err(eyre::eyre!("genesis.json not found at {}", genesis_path.display())); + } + + // Load genesis from test data + let genesis_json = std::fs::read_to_string(&genesis_path)?; + + // Parse the Genesis struct from JSON and convert it to ChainSpec + // This properly handles all the hardfork configuration from the config section + let genesis: Genesis = serde_json::from_str(&genesis_json)?; + let chain_spec: ChainSpec = genesis.into(); + let chain_spec = Arc::new(chain_spec); + + // Create test setup with imported chain + let setup = Setup::::default() + .with_chain_spec(chain_spec) + .with_network(NetworkSetup::single_node()); + + // Build and run the test + let test = TestBuilder::new() + .with_setup_and_import(setup, chain_rlp_path) + .with_action(UpdateBlockInfo::default()) + .with_action( + InitializeFromExecutionApis::new().with_fcu_json(fcu_json_path.to_string_lossy()), + ) + .with_action(MakeCanonical::new()) + .with_action(RunRpcCompatTests::new( + vec!["eth_getLogs".to_string(), "eth_syncing".to_string()], + test_data_path.to_string_lossy(), + )); + + test.run::().await?; + + Ok(()) +} + +/// Test RPC method compatibility with execution-apis test data from environment variable +/// +/// This test: +/// 1. Reads test data path from `EXECUTION_APIS_TEST_PATH` environment variable +/// 2. Auto-discovers all RPC method directories (starting with `eth_`) +/// 3. Initializes a node with chain data from that directory (chain.rlp) +/// 4. Applies the forkchoice state from headfcu.json +/// 5. Runs all discovered RPC test cases individually (each test file reported separately) +#[tokio::test(flavor = "multi_thread")] +async fn test_execution_apis_compat() -> Result<()> { + reth_tracing::init_test_tracing(); + + // Get test data path from environment variable + let test_data_path = match env::var("EXECUTION_APIS_TEST_PATH") { + Ok(path) => path, + Err(_) => { + info!("SKIPPING: EXECUTION_APIS_TEST_PATH environment variable not set. Please set it to the path of execution-apis/tests directory to run this test."); + return Ok(()); + } + }; + + let test_data_path = PathBuf::from(test_data_path); + + if !test_data_path.exists() { + return Err(eyre::eyre!("Test data path does not exist: {}", test_data_path.display())); + } + + info!("Using execution-apis test data from: {}", test_data_path.display()); + + // Auto-discover RPC method directories + let mut rpc_methods = Vec::new(); + if let Ok(entries) = std::fs::read_dir(&test_data_path) { + for entry in entries.flatten() { + if let Some(name) = entry.file_name().to_str() { + // Search for an underscore to get all namespaced directories + if entry.path().is_dir() && name.contains('_') { + rpc_methods.push(name.to_string()); + } + } + } + } + + if rpc_methods.is_empty() { + return Err(eyre::eyre!( + "No RPC method directories (containing a '_' indicating namespacing) found in {}", + test_data_path.display() + )); + } + + rpc_methods.sort(); + debug!("Found RPC method test directories: {:?}", rpc_methods); + + // Paths to chain config files + let chain_rlp_path = test_data_path.join("chain.rlp"); + let genesis_path = test_data_path.join("genesis.json"); + let fcu_json_path = test_data_path.join("headfcu.json"); + + // Verify required files exist + if !chain_rlp_path.exists() { + return Err(eyre::eyre!("chain.rlp not found at {}", chain_rlp_path.display())); + } + if !fcu_json_path.exists() { + return Err(eyre::eyre!("headfcu.json not found at {}", fcu_json_path.display())); + } + if !genesis_path.exists() { + return Err(eyre::eyre!("genesis.json not found at {}", genesis_path.display())); + } + + // Load genesis from test data + let genesis_json = std::fs::read_to_string(&genesis_path)?; + let genesis: Genesis = serde_json::from_str(&genesis_json)?; + let chain_spec: ChainSpec = genesis.into(); + let chain_spec = Arc::new(chain_spec); + + // Create test setup with imported chain + let setup = Setup::::default() + .with_chain_spec(chain_spec) + .with_network(NetworkSetup::single_node()); + + // Build and run the test with all discovered methods + let test = TestBuilder::new() + .with_setup_and_import(setup, chain_rlp_path) + .with_action(UpdateBlockInfo::default()) + .with_action( + InitializeFromExecutionApis::new().with_fcu_json(fcu_json_path.to_string_lossy()), + ) + .with_action(MakeCanonical::new()) + .with_action(RunRpcCompatTests::new(rpc_methods, test_data_path.to_string_lossy())); + + test.run::().await?; + + Ok(()) +} diff --git a/crates/rpc/rpc-engine-api/Cargo.toml b/crates/rpc/rpc-engine-api/Cargo.toml index e4499605b5b..825eb485fc2 100644 --- a/crates/rpc/rpc-engine-api/Cargo.toml +++ b/crates/rpc/rpc-engine-api/Cargo.toml @@ -23,12 +23,11 @@ reth-tasks.workspace = true reth-engine-primitives.workspace = true reth-transaction-pool.workspace = true reth-primitives-traits.workspace = true -reth-rpc-server-types.workspace = true # ethereum alloy-eips.workspace = true alloy-primitives.workspace = true -alloy-rpc-types-engine = { workspace = true, features = ["jsonrpsee-types"] } +alloy-rpc-types-engine.workspace = true # async tokio = { workspace = true, features = ["sync"] } @@ -50,7 +49,6 @@ parking_lot.workspace = true reth-ethereum-engine-primitives.workspace = true reth-provider = { workspace = true, features = ["test-utils"] } reth-ethereum-primitives.workspace = true -reth-primitives-traits.workspace = true reth-payload-builder = { workspace = true, features = ["test-utils"] } reth-testing-utils.workspace = true alloy-rlp.workspace = true diff --git a/crates/rpc/rpc-engine-api/src/capabilities.rs b/crates/rpc/rpc-engine-api/src/capabilities.rs index 97bd30c41e2..67a5a1b72d7 100644 --- a/crates/rpc/rpc-engine-api/src/capabilities.rs +++ b/crates/rpc/rpc-engine-api/src/capabilities.rs @@ -10,6 +10,7 @@ pub const CAPABILITIES: &[&str] = &[ "engine_getPayloadV2", "engine_getPayloadV3", "engine_getPayloadV4", + "engine_getPayloadV5", "engine_newPayloadV1", "engine_newPayloadV2", "engine_newPayloadV3", @@ -17,6 +18,7 @@ pub const CAPABILITIES: &[&str] = &[ "engine_getPayloadBodiesByHashV1", "engine_getPayloadBodiesByRangeV1", "engine_getBlobsV1", + "engine_getBlobsV2", ]; // The list of all supported Engine capabilities available over the engine endpoint. diff --git a/crates/rpc/rpc-engine-api/src/engine_api.rs b/crates/rpc/rpc-engine-api/src/engine_api.rs index a4afc14732e..6aeadeecba5 100644 --- a/crates/rpc/rpc-engine-api/src/engine_api.rs +++ b/crates/rpc/rpc-engine-api/src/engine_api.rs @@ -18,21 +18,23 @@ use async_trait::async_trait; use jsonrpsee_core::{server::RpcModule, RpcResult}; use parking_lot::Mutex; use reth_chainspec::EthereumHardforks; -use reth_engine_primitives::{BeaconConsensusEngineHandle, EngineTypes, EngineValidator}; +use reth_engine_primitives::{ConsensusEngineHandle, EngineApiValidator, EngineTypes}; use reth_payload_builder::PayloadStore; use reth_payload_primitives::{ - validate_payload_timestamp, EngineApiMessageVersion, ExecutionPayload, - PayloadBuilderAttributes, PayloadOrAttributes, PayloadTypes, + validate_payload_timestamp, EngineApiMessageVersion, ExecutionPayload, PayloadOrAttributes, + PayloadTypes, }; use reth_primitives_traits::{Block, BlockBody}; use reth_rpc_api::{EngineApiServer, IntoEngineApiRpcModule}; -use reth_rpc_server_types::result::internal_rpc_err; use reth_storage_api::{BlockReader, HeaderProvider, StateProviderFactory}; use reth_tasks::TaskSpawner; use reth_transaction_pool::TransactionPool; -use std::{sync::Arc, time::Instant}; +use std::{ + sync::Arc, + time::{Instant, SystemTime}, +}; use tokio::sync::oneshot; -use tracing::{trace, warn}; +use tracing::{debug, trace, warn}; /// The Engine API response sender. pub type EngineApiSender = oneshot::Sender>; @@ -40,7 +42,7 @@ pub type EngineApiSender = oneshot::Sender>; /// The upper limit for payload bodies request. const MAX_PAYLOAD_BODIES_LIMIT: u64 = 1024; -/// The upper limit blobs `eth_getBlobs`. +/// The upper limit for blobs in `engine_getBlobsVx`. const MAX_BLOB_LIMIT: usize = 128; /// The Engine API implementation that grants the Consensus layer access to data and @@ -62,30 +64,13 @@ pub struct EngineApi>, } -struct EngineApiInner { - /// The provider to interact with the chain. - provider: Provider, - /// Consensus configuration - chain_spec: Arc, - /// The channel to send messages to the beacon consensus engine. - beacon_consensus: BeaconConsensusEngineHandle, - /// The type that can communicate with the payload service to retrieve payloads. - payload_store: PayloadStore, - /// For spawning and executing async tasks - task_spawner: Box, - /// The latency and response type metrics for engine api calls - metrics: EngineApiMetrics, - /// Identification of the execution client used by the consensus client - client: ClientVersionV1, - /// The list of all supported Engine capabilities available over the engine endpoint. - capabilities: EngineCapabilities, - /// Transaction pool. - tx_pool: Pool, - /// Engine validator. - validator: Validator, - /// Start time of the latest payload request - latest_new_payload_response: Mutex>, - accept_execution_requests_hash: bool, +impl + EngineApi +{ + /// Returns the configured chainspec. + pub fn chain_spec(&self) -> &Arc { + &self.inner.chain_spec + } } impl @@ -94,7 +79,7 @@ where Provider: HeaderProvider + BlockReader + StateProviderFactory + 'static, PayloadT: PayloadTypes, Pool: TransactionPool + 'static, - Validator: EngineValidator, + Validator: EngineApiValidator, ChainSpec: EthereumHardforks + Send + Sync + 'static, { /// Create new instance of [`EngineApi`]. @@ -102,7 +87,7 @@ where pub fn new( provider: Provider, chain_spec: Arc, - beacon_consensus: BeaconConsensusEngineHandle, + beacon_consensus: ConsensusEngineHandle, payload_store: PayloadStore, tx_pool: Pool, task_spawner: Box, @@ -136,15 +121,12 @@ where Ok(vec![self.inner.client.clone()]) } - /// Fetches the attributes for the payload with the given id. - async fn get_payload_attributes( - &self, - payload_id: PayloadId, - ) -> EngineApiResult { + /// Fetches the timestamp of the payload with the given id. + async fn get_payload_timestamp(&self, payload_id: PayloadId) -> EngineApiResult { Ok(self .inner .payload_store - .payload_attributes(payload_id) + .payload_timestamp(payload_id) .await .ok_or(EngineApiError::UnknownPayload)??) } @@ -298,6 +280,11 @@ where self.inner.metrics.new_payload_response.update_response_metrics(&res, gas_used, elapsed); Ok(res?) } + + /// Returns whether the engine accepts execution requests hash. + pub fn accept_execution_requests_hash(&self) -> bool { + self.inner.accept_execution_requests_hash + } } impl @@ -306,7 +293,7 @@ where Provider: HeaderProvider + BlockReader + StateProviderFactory + 'static, EngineT: EngineTypes, Pool: TransactionPool + 'static, - Validator: EngineValidator, + Validator: EngineApiValidator, ChainSpec: EthereumHardforks + Send + Sync + 'static, { /// Sends a message to the beacon consensus engine to update the fork choice _without_ @@ -389,6 +376,40 @@ where res } + /// Helper function for retrieving the build payload by id. + async fn get_built_payload( + &self, + payload_id: PayloadId, + ) -> EngineApiResult { + self.inner + .payload_store + .resolve(payload_id) + .await + .ok_or(EngineApiError::UnknownPayload)? + .map_err(|_| EngineApiError::UnknownPayload) + } + + /// Helper function for validating the payload timestamp and retrieving & converting the payload + /// into desired envelope. + async fn get_payload_inner( + &self, + payload_id: PayloadId, + version: EngineApiMessageVersion, + ) -> EngineApiResult + where + EngineT::BuiltPayload: TryInto, + { + // validate timestamp according to engine rules + let timestamp = self.get_payload_timestamp(payload_id).await?; + validate_payload_timestamp(&self.inner.chain_spec, version, timestamp)?; + + // Now resolve the payload + self.get_built_payload(payload_id).await?.try_into().map_err(|_| { + warn!(?version, "could not transform built payload"); + EngineApiError::UnknownPayload + }) + } + /// Returns the most recent version of the payload that is available in the corresponding /// payload build process at the time of receiving this call. /// @@ -402,17 +423,10 @@ where &self, payload_id: PayloadId, ) -> EngineApiResult { - self.inner - .payload_store - .resolve(payload_id) - .await - .ok_or(EngineApiError::UnknownPayload)? - .map_err(|_| EngineApiError::UnknownPayload)? - .try_into() - .map_err(|_| { - warn!("could not transform built payload into ExecutionPayloadV1"); - EngineApiError::UnknownPayload - }) + self.get_built_payload(payload_id).await?.try_into().map_err(|_| { + warn!(version = ?EngineApiMessageVersion::V1, "could not transform built payload"); + EngineApiError::UnknownPayload + }) } /// Metrics version of `get_payload_v1` @@ -437,28 +451,7 @@ where &self, payload_id: PayloadId, ) -> EngineApiResult { - // First we fetch the payload attributes to check the timestamp - let attributes = self.get_payload_attributes(payload_id).await?; - - // validate timestamp according to engine rules - validate_payload_timestamp( - &self.inner.chain_spec, - EngineApiMessageVersion::V2, - attributes.timestamp(), - )?; - - // Now resolve the payload - self.inner - .payload_store - .resolve(payload_id) - .await - .ok_or(EngineApiError::UnknownPayload)? - .map_err(|_| EngineApiError::UnknownPayload)? - .try_into() - .map_err(|_| { - warn!("could not transform built payload into ExecutionPayloadV2"); - EngineApiError::UnknownPayload - }) + self.get_payload_inner(payload_id, EngineApiMessageVersion::V2).await } /// Metrics version of `get_payload_v2` @@ -483,28 +476,7 @@ where &self, payload_id: PayloadId, ) -> EngineApiResult { - // First we fetch the payload attributes to check the timestamp - let attributes = self.get_payload_attributes(payload_id).await?; - - // validate timestamp according to engine rules - validate_payload_timestamp( - &self.inner.chain_spec, - EngineApiMessageVersion::V3, - attributes.timestamp(), - )?; - - // Now resolve the payload - self.inner - .payload_store - .resolve(payload_id) - .await - .ok_or(EngineApiError::UnknownPayload)? - .map_err(|_| EngineApiError::UnknownPayload)? - .try_into() - .map_err(|_| { - warn!("could not transform built payload into ExecutionPayloadV3"); - EngineApiError::UnknownPayload - }) + self.get_payload_inner(payload_id, EngineApiMessageVersion::V3).await } /// Metrics version of `get_payload_v3` @@ -529,28 +501,7 @@ where &self, payload_id: PayloadId, ) -> EngineApiResult { - // First we fetch the payload attributes to check the timestamp - let attributes = self.get_payload_attributes(payload_id).await?; - - // validate timestamp according to engine rules - validate_payload_timestamp( - &self.inner.chain_spec, - EngineApiMessageVersion::V4, - attributes.timestamp(), - )?; - - // Now resolve the payload - self.inner - .payload_store - .resolve(payload_id) - .await - .ok_or(EngineApiError::UnknownPayload)? - .map_err(|_| EngineApiError::UnknownPayload)? - .try_into() - .map_err(|_| { - warn!("could not transform built payload into ExecutionPayloadV3"); - EngineApiError::UnknownPayload - }) + self.get_payload_inner(payload_id, EngineApiMessageVersion::V4).await } /// Metrics version of `get_payload_v4` @@ -564,6 +515,33 @@ where res } + /// Handler for `engine_getPayloadV5` + /// + /// Returns the most recent version of the payload that is available in the corresponding + /// payload build process at the time of receiving this call. + /// + /// See also + /// + /// Note: + /// > Provider software MAY stop the corresponding build process after serving this call. + pub async fn get_payload_v5( + &self, + payload_id: PayloadId, + ) -> EngineApiResult { + self.get_payload_inner(payload_id, EngineApiMessageVersion::V5).await + } + + /// Metrics version of `get_payload_v5` + pub async fn get_payload_v5_metered( + &self, + payload_id: PayloadId, + ) -> EngineApiResult { + let start = Instant::now(); + let res = Self::get_payload_v5(self, payload_id).await; + self.inner.metrics.latency.get_payload_v5.record(start.elapsed()); + res + } + /// Fetches all the blocks for the provided range starting at `start`, containing `count` /// blocks and returns the mapped payload bodies. pub async fn get_payload_bodies_by_range_with( @@ -597,13 +575,18 @@ where // > Client software MUST NOT return trailing null values if the request extends past the current latest known block. // truncate the end if it's greater than the last block - if let Ok(best_block) = inner.provider.best_block_number() { - if end > best_block { + if let Ok(best_block) = inner.provider.best_block_number() + && end > best_block { end = best_block; } - } + // Check if the requested range starts before the earliest available block due to pruning/expiry + let earliest_block = inner.provider.earliest_block_number().unwrap_or(0); for num in start..=end { + if num < earliest_block { + result.push(None); + continue; + } let block_result = inner.provider.block(BlockHashOrNumber::Number(num)); match block_result { Ok(block) => { @@ -778,17 +761,27 @@ where &self, versioned_hashes: Vec, ) -> EngineApiResult>> { + // Only allow this method before Osaka fork + let current_timestamp = + SystemTime::now().duration_since(SystemTime::UNIX_EPOCH).unwrap_or_default().as_secs(); + if self.inner.chain_spec.is_osaka_active_at_timestamp(current_timestamp) { + return Err(EngineApiError::EngineObjectValidationError( + reth_payload_primitives::EngineObjectValidationError::UnsupportedFork, + )); + } + if versioned_hashes.len() > MAX_BLOB_LIMIT { return Err(EngineApiError::BlobRequestTooLarge { len: versioned_hashes.len() }) } self.inner .tx_pool - .get_blobs_for_versioned_hashes(&versioned_hashes) + .get_blobs_for_versioned_hashes_v1(&versioned_hashes) .map_err(|err| EngineApiError::Internal(Box::new(err))) } - fn get_blobs_v1_metered( + /// Metered version of `get_blobs_v1`. + pub fn get_blobs_v1_metered( &self, versioned_hashes: Vec, ) -> EngineApiResult>> { @@ -807,25 +800,64 @@ where res } -} -impl - EngineApiInner -where - PayloadT: PayloadTypes, -{ - /// Tracks the elapsed time between the new payload response and the received forkchoice update - /// request. - fn record_elapsed_time_on_fcu(&self) { - if let Some(start_time) = self.latest_new_payload_response.lock().take() { - let elapsed_time = start_time.elapsed(); - self.metrics.latency.new_payload_forkchoice_updated_time_diff.record(elapsed_time); + fn get_blobs_v2( + &self, + versioned_hashes: Vec, + ) -> EngineApiResult>> { + // Check if Osaka fork is active + let current_timestamp = + SystemTime::now().duration_since(SystemTime::UNIX_EPOCH).unwrap_or_default().as_secs(); + if !self.inner.chain_spec.is_osaka_active_at_timestamp(current_timestamp) { + return Err(EngineApiError::EngineObjectValidationError( + reth_payload_primitives::EngineObjectValidationError::UnsupportedFork, + )); } + + if versioned_hashes.len() > MAX_BLOB_LIMIT { + return Err(EngineApiError::BlobRequestTooLarge { len: versioned_hashes.len() }) + } + + self.inner + .tx_pool + .get_blobs_for_versioned_hashes_v2(&versioned_hashes) + .map_err(|err| EngineApiError::Internal(Box::new(err))) } - /// Updates the timestamp for the latest new payload response. - fn on_new_payload_response(&self) { - self.latest_new_payload_response.lock().replace(Instant::now()); + /// Metered version of `get_blobs_v2`. + pub fn get_blobs_v2_metered( + &self, + versioned_hashes: Vec, + ) -> EngineApiResult>> { + let hashes_len = versioned_hashes.len(); + let start = Instant::now(); + let res = Self::get_blobs_v2(self, versioned_hashes); + self.inner.metrics.latency.get_blobs_v2.record(start.elapsed()); + + if let Ok(blobs) = &res { + let blobs_found = blobs.iter().flatten().count(); + + self.inner + .metrics + .blob_metrics + .get_blobs_requests_blobs_total + .increment(hashes_len as u64); + self.inner + .metrics + .blob_metrics + .get_blobs_requests_blobs_in_blobpool_total + .increment(blobs_found as u64); + + if blobs_found == hashes_len { + self.inner.metrics.blob_metrics.get_blobs_requests_success_total.increment(1); + } else { + self.inner.metrics.blob_metrics.get_blobs_requests_failure_total.increment(1); + } + } else { + self.inner.metrics.blob_metrics.get_blobs_requests_failure_total.increment(1); + } + + res } } @@ -837,7 +869,7 @@ where Provider: HeaderProvider + BlockReader + StateProviderFactory + 'static, EngineT: EngineTypes, Pool: TransactionPool + 'static, - Validator: EngineValidator, + Validator: EngineApiValidator, ChainSpec: EthereumHardforks + Send + Sync + 'static, { /// Handler for `engine_newPayloadV1` @@ -977,7 +1009,7 @@ where &self, payload_id: PayloadId, ) -> RpcResult { - trace!(target: "rpc::engine", "Serving engine_getPayloadV2"); + debug!(target: "rpc::engine", id = %payload_id, "Serving engine_getPayloadV2"); Ok(self.get_payload_v2_metered(payload_id).await?) } @@ -1015,6 +1047,23 @@ where Ok(self.get_payload_v4_metered(payload_id).await?) } + /// Handler for `engine_getPayloadV5` + /// + /// Returns the most recent version of the payload that is available in the corresponding + /// payload build process at the time of receiving this call. + /// + /// See also + /// + /// Note: + /// > Provider software MAY stop the corresponding build process after serving this call. + async fn get_payload_v5( + &self, + payload_id: PayloadId, + ) -> RpcResult { + trace!(target: "rpc::engine", "Serving engine_getPayloadV5"); + Ok(self.get_payload_v5_metered(payload_id).await?) + } + /// Handler for `engine_getPayloadBodiesByHashV1` /// See also async fn get_payload_bodies_by_hash_v1( @@ -1032,7 +1081,7 @@ where /// Returns the execution payload bodies by the range starting at `start`, containing `count` /// blocks. /// - /// WARNING: This method is associated with the BeaconBlocksByRange message in the consensus + /// WARNING: This method is associated with the `BeaconBlocksByRange` message in the consensus /// layer p2p specification, meaning the input should be treated as untrusted or potentially /// adversarial. /// @@ -1077,10 +1126,10 @@ where async fn get_blobs_v2( &self, - _versioned_hashes: Vec, - ) -> RpcResult>> { + versioned_hashes: Vec, + ) -> RpcResult>> { trace!(target: "rpc::engine", "Serving engine_getBlobsV2"); - Err(internal_rpc_err("unimplemented")) + Ok(self.get_blobs_v2_metered(versioned_hashes)?) } } @@ -1105,6 +1154,63 @@ where } } +impl Clone + for EngineApi +where + PayloadT: PayloadTypes, +{ + fn clone(&self) -> Self { + Self { inner: Arc::clone(&self.inner) } + } +} + +/// The container type for the engine API internals. +struct EngineApiInner { + /// The provider to interact with the chain. + provider: Provider, + /// Consensus configuration + chain_spec: Arc, + /// The channel to send messages to the beacon consensus engine. + beacon_consensus: ConsensusEngineHandle, + /// The type that can communicate with the payload service to retrieve payloads. + payload_store: PayloadStore, + /// For spawning and executing async tasks + task_spawner: Box, + /// The latency and response type metrics for engine api calls + metrics: EngineApiMetrics, + /// Identification of the execution client used by the consensus client + client: ClientVersionV1, + /// The list of all supported Engine capabilities available over the engine endpoint. + capabilities: EngineCapabilities, + /// Transaction pool. + tx_pool: Pool, + /// Engine validator. + validator: Validator, + /// Start time of the latest payload request + latest_new_payload_response: Mutex>, + accept_execution_requests_hash: bool, +} + +impl + EngineApiInner +where + PayloadT: PayloadTypes, +{ + /// Tracks the elapsed time between the new payload response and the received forkchoice update + /// request. + fn record_elapsed_time_on_fcu(&self) { + if let Some(start_time) = self.latest_new_payload_response.lock().take() { + let elapsed_time = start_time.elapsed(); + self.metrics.latency.new_payload_forkchoice_updated_time_diff.record(elapsed_time); + } + } + + /// Updates the timestamp for the latest new payload response. + fn on_new_payload_response(&self) { + self.latest_new_payload_response.lock().replace(Instant::now()); + } +} + #[cfg(test)] mod tests { use super::*; @@ -1146,7 +1252,7 @@ mod tests { let api = EngineApi::new( provider.clone(), chain_spec.clone(), - BeaconConsensusEngineHandle::new(to_engine), + ConsensusEngineHandle::new(to_engine), payload_store.into(), NoopTransactionPool::default(), task_executor, diff --git a/crates/rpc/rpc-engine-api/src/error.rs b/crates/rpc/rpc-engine-api/src/error.rs index 2825caee93b..6155c004c36 100644 --- a/crates/rpc/rpc-engine-api/src/error.rs +++ b/crates/rpc/rpc-engine-api/src/error.rs @@ -1,4 +1,8 @@ -use alloy_primitives::{B256, U256}; +use alloy_primitives::B256; +use alloy_rpc_types_engine::{ + ForkchoiceUpdateError, INVALID_FORK_CHOICE_STATE_ERROR, INVALID_FORK_CHOICE_STATE_ERROR_MSG, + INVALID_PAYLOAD_ATTRIBUTES_ERROR, INVALID_PAYLOAD_ATTRIBUTES_ERROR_MSG, +}; use jsonrpsee_types::error::{ INTERNAL_ERROR_CODE, INVALID_PARAMS_CODE, INVALID_PARAMS_MSG, SERVER_ERROR_MSG, }; @@ -55,17 +59,6 @@ pub enum EngineApiError { /// Requested number of items count: u64, }, - /// Terminal total difficulty mismatch during transition configuration exchange. - #[error( - "invalid transition terminal total difficulty: \ - execution: {execution}, consensus: {consensus}" - )] - TerminalTD { - /// Execution terminal total difficulty value. - execution: U256, - /// Consensus terminal total difficulty value. - consensus: U256, - }, /// Terminal block hash mismatch during transition configuration exchange. #[error( "invalid transition terminal block hash: \ @@ -171,7 +164,23 @@ impl From for jsonrpsee_types::error::ErrorObject<'static> { ), // Error responses from the consensus engine EngineApiError::ForkChoiceUpdate(ref err) => match err { - BeaconForkChoiceUpdateError::ForkchoiceUpdateError(err) => (*err).into(), + BeaconForkChoiceUpdateError::ForkchoiceUpdateError(err) => match err { + ForkchoiceUpdateError::UpdatedInvalidPayloadAttributes => { + jsonrpsee_types::error::ErrorObject::owned( + INVALID_PAYLOAD_ATTRIBUTES_ERROR, + INVALID_PAYLOAD_ATTRIBUTES_ERROR_MSG, + None::<()>, + ) + } + ForkchoiceUpdateError::InvalidState | + ForkchoiceUpdateError::UnknownFinalBlock => { + jsonrpsee_types::error::ErrorObject::owned( + INVALID_FORK_CHOICE_STATE_ERROR, + INVALID_FORK_CHOICE_STATE_ERROR_MSG, + None::<()>, + ) + } + }, BeaconForkChoiceUpdateError::EngineUnavailable | BeaconForkChoiceUpdateError::Internal(_) => { jsonrpsee_types::error::ErrorObject::owned( @@ -182,7 +191,6 @@ impl From for jsonrpsee_types::error::ErrorObject<'static> { } }, // Any other server error - EngineApiError::TerminalTD { .. } | EngineApiError::TerminalBlockHash { .. } | EngineApiError::NewPayload(_) | EngineApiError::Internal(_) | diff --git a/crates/rpc/rpc-engine-api/src/lib.rs b/crates/rpc/rpc-engine-api/src/lib.rs index 65088eac5af..9ce8d21763b 100644 --- a/crates/rpc/rpc-engine-api/src/lib.rs +++ b/crates/rpc/rpc-engine-api/src/lib.rs @@ -7,7 +7,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] /// The Engine API implementation. mod engine_api; diff --git a/crates/rpc/rpc-engine-api/src/metrics.rs b/crates/rpc/rpc-engine-api/src/metrics.rs index ef6ccd2aacf..95156e490b7 100644 --- a/crates/rpc/rpc-engine-api/src/metrics.rs +++ b/crates/rpc/rpc-engine-api/src/metrics.rs @@ -46,12 +46,16 @@ pub(crate) struct EngineApiLatencyMetrics { pub(crate) get_payload_v3: Histogram, /// Latency for `engine_getPayloadV4` pub(crate) get_payload_v4: Histogram, + /// Latency for `engine_getPayloadV5` + pub(crate) get_payload_v5: Histogram, /// Latency for `engine_getPayloadBodiesByRangeV1` pub(crate) get_payload_bodies_by_range_v1: Histogram, /// Latency for `engine_getPayloadBodiesByHashV1` pub(crate) get_payload_bodies_by_hash_v1: Histogram, /// Latency for `engine_getBlobsV1` pub(crate) get_blobs_v1: Histogram, + /// Latency for `engine_getBlobsV2` + pub(crate) get_blobs_v2: Histogram, } /// Metrics for engine API forkchoiceUpdated responses. @@ -113,6 +117,14 @@ pub(crate) struct BlobMetrics { pub(crate) blob_count: Counter, /// Count of blob misses pub(crate) blob_misses: Counter, + /// Number of blobs requested via getBlobsV2 + pub(crate) get_blobs_requests_blobs_total: Counter, + /// Number of blobs requested via getBlobsV2 that are present in the blobpool + pub(crate) get_blobs_requests_blobs_in_blobpool_total: Counter, + /// Number of times getBlobsV2 responded with “hit” + pub(crate) get_blobs_requests_success_total: Counter, + /// Number of times getBlobsV2 responded with “miss” + pub(crate) get_blobs_requests_failure_total: Counter, } impl NewPayloadStatusResponseMetrics { diff --git a/crates/rpc/rpc-engine-api/tests/it/payload.rs b/crates/rpc/rpc-engine-api/tests/it/payload.rs index 477fda2b1f5..81359969c76 100644 --- a/crates/rpc/rpc-engine-api/tests/it/payload.rs +++ b/crates/rpc/rpc-engine-api/tests/it/payload.rs @@ -2,7 +2,7 @@ use alloy_eips::eip4895::Withdrawals; use alloy_primitives::Bytes; -use alloy_rlp::{Decodable, Error as RlpError}; +use alloy_rlp::Decodable; use alloy_rpc_types_engine::{ ExecutionPayload, ExecutionPayloadBodyV1, ExecutionPayloadSidecar, ExecutionPayloadV1, PayloadError, @@ -87,16 +87,6 @@ fn payload_validation_conversion() { Err(PayloadError::ExtraData(data)) if data == block_with_invalid_extra_data ); - // Zero base fee - let block_with_zero_base_fee = transform_block(block.clone(), |mut b| { - b.header.base_fee_per_gas = Some(0); - b - }); - assert_matches!( - block_with_zero_base_fee.try_into_block_with_sidecar::(&ExecutionPayloadSidecar::none()), - Err(PayloadError::BaseFee(val)) if val.is_zero() - ); - // Invalid encoded transactions let mut payload_with_invalid_txs = ExecutionPayloadV1::from_block_unchecked(block.hash(), &block.into_block()); @@ -105,5 +95,5 @@ fn payload_validation_conversion() { *tx = Bytes::new(); }); let payload_with_invalid_txs = payload_with_invalid_txs.try_into_block::(); - assert_matches!(payload_with_invalid_txs, Err(PayloadError::Decode(RlpError::InputTooShort))); + assert_matches!(payload_with_invalid_txs, Err(PayloadError::Decode(_))); } diff --git a/crates/rpc/rpc-eth-api/Cargo.toml b/crates/rpc/rpc-eth-api/Cargo.toml index 66e11f4686f..0775489eb08 100644 --- a/crates/rpc/rpc-eth-api/Cargo.toml +++ b/crates/rpc/rpc-eth-api/Cargo.toml @@ -13,14 +13,15 @@ workspace = true [dependencies] # reth -revm = { workspace = true, features = ["optional_block_gas_limit", "optional_eip3607", "optional_no_base_fee"] } +revm = { workspace = true, features = ["optional_block_gas_limit", "optional_eip3607", "optional_no_base_fee", "optional_fee_charge", "memory_limit"] } +reth-chain-state.workspace = true revm-inspectors.workspace = true -reth-primitives-traits.workspace = true +reth-primitives-traits = { workspace = true, features = ["rpc-compat"] } reth-errors.workspace = true reth-evm.workspace = true -reth-provider.workspace = true +reth-storage-api.workspace = true reth-revm.workspace = true -reth-rpc-types-compat.workspace = true +reth-rpc-convert.workspace = true reth-tasks = { workspace = true, features = ["rayon"] } reth-transaction-pool.workspace = true reth-chainspec.workspace = true @@ -29,9 +30,9 @@ reth-rpc-server-types.workspace = true reth-network-api.workspace = true reth-node-api.workspace = true reth-trie-common = { workspace = true, features = ["eip1186"] } -reth-payload-builder.workspace = true # ethereum +alloy-evm = { workspace = true, features = ["overrides", "call-util"] } alloy-rlp.workspace = true alloy-serde.workspace = true alloy-eips.workspace = true @@ -62,3 +63,9 @@ tracing.workspace = true [features] js-tracer = ["revm-inspectors/js-tracer", "reth-rpc-eth-types/js-tracer"] client = ["jsonrpsee/client", "jsonrpsee/async-client"] +op = [ + "reth-evm/op", + "reth-primitives-traits/op", + "reth-rpc-convert/op", + "alloy-evm/op", +] diff --git a/crates/rpc/rpc-eth-api/src/bundle.rs b/crates/rpc/rpc-eth-api/src/bundle.rs index 75ec169ac7d..b47ef1b3bb3 100644 --- a/crates/rpc/rpc-eth-api/src/bundle.rs +++ b/crates/rpc/rpc-eth-api/src/bundle.rs @@ -4,8 +4,8 @@ use alloy_primitives::{Bytes, B256}; use alloy_rpc_types_mev::{ - CancelBundleRequest, CancelPrivateTransactionRequest, EthBundleHash, EthCallBundle, - EthCallBundleResponse, EthSendBundle, PrivateTransactionRequest, + EthBundleHash, EthCallBundle, EthCallBundleResponse, EthCancelBundle, + EthCancelPrivateTransaction, EthSendBundle, EthSendPrivateTransaction, }; use jsonrpsee::proc_macros::rpc; @@ -43,19 +43,19 @@ pub trait EthBundleApi { /// `eth_cancelBundle` is used to prevent a submitted bundle from being included on-chain. See [bundle cancellations](https://docs.flashbots.net/flashbots-auction/advanced/bundle-cancellations) for more information. #[method(name = "cancelBundle")] - async fn cancel_bundle(&self, request: CancelBundleRequest) -> jsonrpsee::core::RpcResult<()>; + async fn cancel_bundle(&self, request: EthCancelBundle) -> jsonrpsee::core::RpcResult<()>; /// `eth_sendPrivateTransaction` is used to send a single transaction to Flashbots. Flashbots will attempt to build a block including the transaction for the next 25 blocks. See [Private Transactions](https://docs.flashbots.net/flashbots-protect/additional-documentation/eth-sendPrivateTransaction) for more info. #[method(name = "sendPrivateTransaction")] async fn send_private_transaction( &self, - request: PrivateTransactionRequest, + request: EthSendPrivateTransaction, ) -> jsonrpsee::core::RpcResult; /// The `eth_sendPrivateRawTransaction` method can be used to send private transactions to /// the RPC endpoint. Private transactions are protected from frontrunning and kept /// private until included in a block. A request to this endpoint needs to follow - /// the standard eth_sendRawTransaction + /// the standard `eth_sendRawTransaction` #[method(name = "sendPrivateRawTransaction")] async fn send_private_raw_transaction(&self, bytes: Bytes) -> jsonrpsee::core::RpcResult; @@ -63,10 +63,10 @@ pub trait EthBundleApi { /// submitted for future blocks. /// /// A transaction can only be cancelled if the request is signed by the same key as the - /// eth_sendPrivateTransaction call submitting the transaction in first place. + /// `eth_sendPrivateTransaction` call submitting the transaction in first place. #[method(name = "cancelPrivateTransaction")] async fn cancel_private_transaction( &self, - request: CancelPrivateTransactionRequest, + request: EthCancelPrivateTransaction, ) -> jsonrpsee::core::RpcResult; } diff --git a/crates/rpc/rpc-eth-api/src/core.rs b/crates/rpc/rpc-eth-api/src/core.rs index 9d612ec33b6..40f19c86227 100644 --- a/crates/rpc/rpc-eth-api/src/core.rs +++ b/crates/rpc/rpc-eth-api/src/core.rs @@ -1,5 +1,9 @@ //! Implementation of the [`jsonrpsee`] generated [`EthApiServer`] trait. Handles RPC requests for //! the `eth_` namespace. +use crate::{ + helpers::{EthApiSpec, EthBlocks, EthCall, EthFees, EthState, EthTransactions, FullEthApi}, + RpcBlock, RpcHeader, RpcReceipt, RpcTransaction, +}; use alloy_dyn_abi::TypedData; use alloy_eips::{eip2930::AccessListResult, BlockId, BlockNumberOrTag}; use alloy_json_rpc::RpcObject; @@ -7,28 +11,27 @@ use alloy_primitives::{Address, Bytes, B256, B64, U256, U64}; use alloy_rpc_types_eth::{ simulate::{SimulatePayload, SimulatedBlock}, state::{EvmOverrides, StateOverride}, - transaction::TransactionRequest, BlockOverrides, Bundle, EIP1186AccountProofResponse, EthCallResponse, FeeHistory, Index, StateContext, SyncStatus, Work, }; use alloy_serde::JsonStorageKey; use jsonrpsee::{core::RpcResult, proc_macros::rpc}; +use reth_primitives_traits::TxTy; +use reth_rpc_convert::RpcTxReq; +use reth_rpc_eth_types::FillTransactionResult; use reth_rpc_server_types::{result::internal_rpc_err, ToRpcResult}; use tracing::trace; -use crate::{ - helpers::{EthApiSpec, EthBlocks, EthCall, EthFees, EthState, EthTransactions, FullEthApi}, - RpcBlock, RpcHeader, RpcReceipt, RpcTransaction, -}; - /// Helper trait, unifies functionality that must be supported to implement all RPC methods for /// server. pub trait FullEthApiServer: EthApiServer< + RpcTxReq, RpcTransaction, RpcBlock, RpcReceipt, RpcHeader, + TxTy, > + FullEthApi + Clone { @@ -36,19 +39,29 @@ pub trait FullEthApiServer: impl FullEthApiServer for T where T: EthApiServer< + RpcTxReq, RpcTransaction, RpcBlock, RpcReceipt, RpcHeader, + TxTy, > + FullEthApi + Clone { } -/// Eth rpc interface: +/// Eth rpc interface: #[cfg_attr(not(feature = "client"), rpc(server, namespace = "eth"))] #[cfg_attr(feature = "client", rpc(server, client, namespace = "eth"))] -pub trait EthApi { +pub trait EthApi< + TxReq: RpcObject, + T: RpcObject, + B: RpcObject, + R: RpcObject, + H: RpcObject, + RawTx: RpcObject, +> +{ /// Returns the protocol version encoded as a string. #[method(name = "protocolVersion")] async fn protocol_version(&self) -> RpcResult; @@ -213,7 +226,7 @@ pub trait EthApi { #[method(name = "simulateV1")] async fn simulate_v1( &self, - opts: SimulatePayload, + opts: SimulatePayload, block_number: Option, ) -> RpcResult>>; @@ -221,18 +234,22 @@ pub trait EthApi { #[method(name = "call")] async fn call( &self, - request: TransactionRequest, + request: TxReq, block_number: Option, state_overrides: Option, block_overrides: Option>, ) -> RpcResult; + /// Fills the defaults on a given unsigned transaction. + #[method(name = "fillTransaction")] + async fn fill_transaction(&self, request: TxReq) -> RpcResult>; + /// Simulate arbitrary number of transactions at an arbitrary blockchain index, with the /// optionality of state overrides #[method(name = "callMany")] async fn call_many( &self, - bundles: Vec, + bundles: Vec>, state_context: Option, state_override: Option, ) -> RpcResult>>; @@ -247,14 +264,14 @@ pub trait EthApi { /// It returns list of addresses and storage keys used by the transaction, plus the gas /// consumed when the access list is added. That is, it gives you the list of addresses and /// storage keys that will be used by that transaction, plus the gas consumed if the access - /// list is included. Like eth_estimateGas, this is an estimation; the list could change + /// list is included. Like `eth_estimateGas`, this is an estimation; the list could change /// when the transaction is actually mined. Adding an accessList to your transaction does /// not necessary result in lower gas usage compared to a transaction without an access /// list. #[method(name = "createAccessList")] async fn create_access_list( &self, - request: TransactionRequest, + request: TxReq, block_number: Option, state_override: Option, ) -> RpcResult; @@ -264,7 +281,7 @@ pub trait EthApi { #[method(name = "estimateGas")] async fn estimate_gas( &self, - request: TransactionRequest, + request: TxReq, block_number: Option, state_override: Option, ) -> RpcResult; @@ -332,12 +349,18 @@ pub trait EthApi { /// Sends transaction; will block waiting for signer to return the /// transaction hash. #[method(name = "sendTransaction")] - async fn send_transaction(&self, request: TransactionRequest) -> RpcResult; + async fn send_transaction(&self, request: TxReq) -> RpcResult; /// Sends signed transaction, returning its hash. #[method(name = "sendRawTransaction")] async fn send_raw_transaction(&self, bytes: Bytes) -> RpcResult; + /// Sends a signed transaction and awaits the transaction receipt. + /// + /// This will return a timeout error if the transaction isn't included within some time period. + #[method(name = "sendRawTransactionSync")] + async fn send_raw_transaction_sync(&self, bytes: Bytes) -> RpcResult; + /// Returns an Ethereum specific signature with: sign(keccak256("\x19Ethereum Signed Message:\n" /// + len(message) + message))). #[method(name = "sign")] @@ -346,7 +369,7 @@ pub trait EthApi { /// Signs a transaction that can be submitted to the network at a later time using with /// `sendRawTransaction.` #[method(name = "signTransaction")] - async fn sign_transaction(&self, transaction: TransactionRequest) -> RpcResult; + async fn sign_transaction(&self, transaction: TxReq) -> RpcResult; /// Signs data via [EIP-712](https://github.com/ethereum/EIPs/blob/master/EIPS/eip-712.md). #[method(name = "signTypedData")] @@ -376,10 +399,12 @@ pub trait EthApi { #[async_trait::async_trait] impl EthApiServer< + RpcTxReq, RpcTransaction, RpcBlock, RpcReceipt, RpcHeader, + TxTy, > for T where T: FullEthApi, @@ -405,7 +430,7 @@ where /// Handler for: `eth_accounts` fn accounts(&self) -> RpcResult> { trace!(target: "rpc::eth", "Serving eth_accounts"); - Ok(EthApiSpec::accounts(self)) + Ok(EthTransactions::accounts(self)) } /// Handler for: `eth_blockNumber` @@ -460,7 +485,12 @@ where /// Handler for: `eth_getUncleCountByBlockHash` async fn block_uncles_count_by_hash(&self, hash: B256) -> RpcResult> { trace!(target: "rpc::eth", ?hash, "Serving eth_getUncleCountByBlockHash"); - Ok(EthBlocks::ommers(self, hash.into())?.map(|ommers| U256::from(ommers.len()))) + + if let Some(block) = self.block_by_hash(hash, false).await? { + Ok(Some(U256::from(block.uncles.len()))) + } else { + Ok(None) + } } /// Handler for: `eth_getUncleCountByBlockNumber` @@ -469,7 +499,12 @@ where number: BlockNumberOrTag, ) -> RpcResult> { trace!(target: "rpc::eth", ?number, "Serving eth_getUncleCountByBlockNumber"); - Ok(EthBlocks::ommers(self, number.into())?.map(|ommers| U256::from(ommers.len()))) + + if let Some(block) = self.block_by_number(number, false).await? { + Ok(Some(U256::from(block.uncles.len()))) + } else { + Ok(None) + } } /// Handler for: `eth_getBlockReceipts` @@ -638,7 +673,7 @@ where /// Handler for: `eth_simulateV1` async fn simulate_v1( &self, - payload: SimulatePayload, + payload: SimulatePayload>, block_number: Option, ) -> RpcResult>>> { trace!(target: "rpc::eth", ?block_number, "Serving eth_simulateV1"); @@ -649,7 +684,7 @@ where /// Handler for: `eth_call` async fn call( &self, - request: TransactionRequest, + request: RpcTxReq, block_number: Option, state_overrides: Option, block_overrides: Option>, @@ -664,10 +699,19 @@ where .await?) } + /// Handler for: `eth_fillTransaction` + async fn fill_transaction( + &self, + request: RpcTxReq, + ) -> RpcResult>> { + trace!(target: "rpc::eth", ?request, "Serving eth_fillTransaction"); + Ok(EthTransactions::fill_transaction(self, request).await?) + } + /// Handler for: `eth_callMany` async fn call_many( &self, - bundles: Vec, + bundles: Vec>>, state_context: Option, state_override: Option, ) -> RpcResult>> { @@ -678,7 +722,7 @@ where /// Handler for: `eth_createAccessList` async fn create_access_list( &self, - request: TransactionRequest, + request: RpcTxReq, block_number: Option, state_override: Option, ) -> RpcResult { @@ -689,7 +733,7 @@ where /// Handler for: `eth_estimateGas` async fn estimate_gas( &self, - request: TransactionRequest, + request: RpcTxReq, block_number: Option, state_override: Option, ) -> RpcResult { @@ -781,7 +825,7 @@ where } /// Handler for: `eth_sendTransaction` - async fn send_transaction(&self, request: TransactionRequest) -> RpcResult { + async fn send_transaction(&self, request: RpcTxReq) -> RpcResult { trace!(target: "rpc::eth", ?request, "Serving eth_sendTransaction"); Ok(EthTransactions::send_transaction(self, request).await?) } @@ -792,6 +836,12 @@ where Ok(EthTransactions::send_raw_transaction(self, tx).await?) } + /// Handler for: `eth_sendRawTransactionSync` + async fn send_raw_transaction_sync(&self, tx: Bytes) -> RpcResult> { + trace!(target: "rpc::eth", ?tx, "Serving eth_sendRawTransactionSync"); + Ok(EthTransactions::send_raw_transaction_sync(self, tx).await?) + } + /// Handler for: `eth_sign` async fn sign(&self, address: Address, message: Bytes) -> RpcResult { trace!(target: "rpc::eth", ?address, ?message, "Serving eth_sign"); @@ -799,7 +849,7 @@ where } /// Handler for: `eth_signTransaction` - async fn sign_transaction(&self, request: TransactionRequest) -> RpcResult { + async fn sign_transaction(&self, request: RpcTxReq) -> RpcResult { trace!(target: "rpc::eth", ?request, "Serving eth_signTransaction"); Ok(EthTransactions::sign_transaction(self, request).await?) } diff --git a/crates/rpc/rpc-eth-api/src/helpers/block.rs b/crates/rpc/rpc-eth-api/src/helpers/block.rs index b88bd9b65c8..17e4b000b35 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/block.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/block.rs @@ -5,17 +5,15 @@ use crate::{ node::RpcNodeCoreExt, EthApiTypes, FromEthApiError, FullEthApiTypes, RpcBlock, RpcNodeCore, RpcReceipt, }; +use alloy_consensus::{transaction::TxHashRef, TxReceipt}; use alloy_eips::BlockId; -use alloy_primitives::{Sealable, U256}; use alloy_rlp::Encodable; -use alloy_rpc_types_eth::{Block, BlockTransactions, Header, Index}; +use alloy_rpc_types_eth::{Block, BlockTransactions, Index}; use futures::Future; use reth_node_api::BlockBody; -use reth_primitives_traits::{RecoveredBlock, SealedBlock}; -use reth_provider::{ - BlockIdReader, BlockReader, BlockReaderIdExt, ProviderHeader, ProviderReceipt, ProviderTx, -}; -use reth_rpc_types_compat::block::from_block; +use reth_primitives_traits::{AlloyBlockHeader, RecoveredBlock, SealedHeader, TransactionMeta}; +use reth_rpc_convert::{transaction::ConvertReceiptInput, RpcConvert, RpcHeader}; +use reth_storage_api::{BlockIdReader, BlockReader, ProviderHeader, ProviderReceipt, ProviderTx}; use reth_transaction_pool::{PoolTransaction, TransactionPool}; use std::sync::Arc; @@ -24,7 +22,7 @@ pub type BlockReceiptsResult = Result>>, E>; /// Result type of the fetched block and its receipts. pub type BlockAndReceiptsResult = Result< Option<( - SealedBlock<<::Provider as BlockReader>::Block>, + Arc::Provider as BlockReader>::Block>>, Arc::Provider>>>, )>, ::Error, @@ -32,13 +30,14 @@ pub type BlockAndReceiptsResult = Result< /// Block related functions for the [`EthApiServer`](crate::EthApiServer) trait in the /// `eth_` namespace. -pub trait EthBlocks: LoadBlock { +pub trait EthBlocks: + LoadBlock> +{ /// Returns the block header for the given block id. - #[expect(clippy::type_complexity)] fn rpc_block_header( &self, block_id: BlockId, - ) -> impl Future>>, Self::Error>> + Send + ) -> impl Future>, Self::Error>> + Send where Self: FullEthApiTypes, { @@ -60,7 +59,11 @@ pub trait EthBlocks: LoadBlock { async move { let Some(block) = self.recovered_block(block_id).await? else { return Ok(None) }; - let block = from_block((*block).clone(), full.into(), self.tx_resp_builder())?; + let block = block.clone_into_rpc_block( + full.into(), + |tx, tx_info| self.tx_resp_builder().fill(tx, tx_info), + |header, size| self.tx_resp_builder().convert_header(header, size), + )?; Ok(Some(block)) } } @@ -79,7 +82,7 @@ pub trait EthBlocks: LoadBlock { .provider() .pending_block() .map_err(Self::Error::from_eth_err)? - .map(|block| block.body().transactions().len())) + .map(|block| block.body().transaction_count())); } let block_hash = match self @@ -108,7 +111,60 @@ pub trait EthBlocks: LoadBlock { block_id: BlockId, ) -> impl Future> + Send where - Self: LoadReceipt; + Self: LoadReceipt, + { + async move { + if let Some((block, receipts)) = self.load_block_and_receipts(block_id).await? { + let block_number = block.number(); + let base_fee = block.base_fee_per_gas(); + let block_hash = block.hash(); + let excess_blob_gas = block.excess_blob_gas(); + let timestamp = block.timestamp(); + let mut gas_used = 0; + let mut next_log_index = 0; + + let inputs = block + .transactions_recovered() + .zip(Arc::unwrap_or_clone(receipts)) + .enumerate() + .map(|(idx, (tx, receipt))| { + let meta = TransactionMeta { + tx_hash: *tx.tx_hash(), + index: idx as u64, + block_hash, + block_number, + base_fee, + excess_blob_gas, + timestamp, + }; + + let cumulative_gas_used = receipt.cumulative_gas_used(); + let logs_len = receipt.logs().len(); + + let input = ConvertReceiptInput { + tx, + gas_used: cumulative_gas_used - gas_used, + next_log_index, + meta, + receipt, + }; + + gas_used = cumulative_gas_used; + next_log_index += logs_len; + + input + }) + .collect::>(); + + return self + .tx_resp_builder() + .convert_receipts_with_block(inputs, block.sealed_block()) + .map(Some) + } + + Ok(None) + } + } /// Helper method that loads a block and all its receipts. fn load_block_and_receipts( @@ -129,24 +185,24 @@ pub trait EthBlocks: LoadBlock { .pending_block_and_receipts() .map_err(Self::Error::from_eth_err)? { - return Ok(Some((block, Arc::new(receipts)))); + return Ok(Some((Arc::new(block), Arc::new(receipts)))); } // If no pending block from provider, build the pending block locally. - if let Some((block, receipts)) = self.local_pending_block().await? { - return Ok(Some((block.into_sealed_block(), Arc::new(receipts)))); + if let Some(pending) = self.local_pending_block().await? { + return Ok(Some((pending.block, pending.receipts))); } } if let Some(block_hash) = - self.provider().block_hash_for_id(block_id).map_err(Self::Error::from_eth_err)? - { - return self + self.provider().block_hash_for_id(block_id).map_err(Self::Error::from_eth_err)? && + let Some((block, receipts)) = self .cache() .get_block_and_receipts(block_hash) .await - .map_err(Self::Error::from_eth_err) - .map(|b| b.map(|(b, r)| (b.clone_sealed_block(), r))) + .map_err(Self::Error::from_eth_err)? + { + return Ok(Some((block, receipts))); } Ok(None) @@ -160,8 +216,15 @@ pub trait EthBlocks: LoadBlock { fn ommers( &self, block_id: BlockId, - ) -> Result>>, Self::Error> { - self.provider().ommers_by_id(block_id).map_err(Self::Error::from_eth_err) + ) -> impl Future>>, Self::Error>> + Send + { + async move { + if let Some(block) = self.recovered_block(block_id).await? { + Ok(block.body().ommers().map(|o| o.to_vec())) + } else { + Ok(None) + } + } } /// Returns uncle block at given index in given block. @@ -181,20 +244,30 @@ pub trait EthBlocks: LoadBlock { .map_err(Self::Error::from_eth_err)? .and_then(|block| block.body().ommers().map(|o| o.to_vec())) } else { - self.provider().ommers_by_id(block_id).map_err(Self::Error::from_eth_err)? + self.recovered_block(block_id) + .await? + .map(|block| block.body().ommers().map(|o| o.to_vec()).unwrap_or_default()) } .unwrap_or_default(); - Ok(uncles.into_iter().nth(index.into()).map(|header| { - let block = alloy_consensus::Block::::uncle(header); - let size = U256::from(block.length()); - Block { - uncles: vec![], - header: Header::from_consensus(block.header.seal_slow(), None, Some(size)), - transactions: BlockTransactions::Uncle, - withdrawals: None, - } - })) + uncles + .into_iter() + .nth(index.into()) + .map(|header| { + let block = + alloy_consensus::Block::::uncle(header); + let size = block.length(); + let header = self + .tx_resp_builder() + .convert_header(SealedHeader::new_unhashed(block.header), size)?; + Ok(Block { + uncles: vec![], + header, + transactions: BlockTransactions::Uncle, + withdrawals: None, + }) + }) + .transpose() } } } @@ -202,13 +275,7 @@ pub trait EthBlocks: LoadBlock { /// Loads a block from database. /// /// Behaviour shared by several `eth_` RPC methods, not exclusive to `eth_` blocks RPC methods. -pub trait LoadBlock: - LoadPendingBlock - + SpawnBlocking - + RpcNodeCoreExt< - Pool: TransactionPool>>, - > -{ +pub trait LoadBlock: LoadPendingBlock + SpawnBlocking + RpcNodeCoreExt { /// Returns the block object for the given block id. #[expect(clippy::type_complexity)] fn recovered_block( @@ -223,17 +290,15 @@ pub trait LoadBlock: async move { if block_id.is_pending() { // Pending block can be fetched directly without need for caching - if let Some(pending_block) = self - .provider() - .pending_block_with_senders() - .map_err(Self::Error::from_eth_err)? + if let Some(pending_block) = + self.provider().pending_block().map_err(Self::Error::from_eth_err)? { return Ok(Some(Arc::new(pending_block))); } // If no pending block from provider, try to get local pending block return match self.local_pending_block().await? { - Some((block, _)) => Ok(Some(Arc::new(block))), + Some(pending) => Ok(Some(pending.block)), None => Ok(None), }; } diff --git a/crates/rpc/rpc-eth-api/src/helpers/blocking_task.rs b/crates/rpc/rpc-eth-api/src/helpers/blocking_task.rs index 1032c6085f5..886ff639141 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/blocking_task.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/blocking_task.rs @@ -60,6 +60,29 @@ pub trait SpawnBlocking: EthApiTypes + Clone + Send + Sync + 'static { async move { rx.await.map_err(|_| EthApiError::InternalEthError)? } } + /// Executes the future on a new blocking task. + /// + /// Note: This is expected for futures that are dominated by blocking IO operations, for tracing + /// or CPU bound operations in general use [`spawn_tracing`](Self::spawn_tracing). + fn spawn_blocking_io_fut( + &self, + f: F, + ) -> impl Future> + Send + where + Fut: Future> + Send + 'static, + F: FnOnce(Self) -> Fut + Send + 'static, + R: Send + 'static, + { + let (tx, rx) = oneshot::channel(); + let this = self.clone(); + self.io_task_spawner().spawn_blocking(Box::pin(async move { + let res = f(this).await; + let _ = tx.send(res); + })); + + async move { rx.await.map_err(|_| EthApiError::InternalEthError)? } + } + /// Executes a blocking task on the tracing pool. /// /// Note: This is expected for futures that are predominantly CPU bound, as it uses `rayon` diff --git a/crates/rpc/rpc-eth-api/src/helpers/call.rs b/crates/rpc/rpc-eth-api/src/helpers/call.rs index c47f549329b..05f0de87464 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/call.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/call.rs @@ -1,41 +1,41 @@ //! Loads a pending block from database. Helper trait for `eth_` transaction, call and trace RPC //! methods. +use core::fmt; + use super::{LoadBlock, LoadPendingBlock, LoadState, LoadTransaction, SpawnBlocking, Trace}; use crate::{ helpers::estimate::EstimateCall, FromEvmError, FullEthApiTypes, RpcBlock, RpcNodeCore, }; -use alloy_consensus::BlockHeader; +use alloy_consensus::{transaction::TxHashRef, BlockHeader}; use alloy_eips::eip2930::AccessListResult; +use alloy_evm::overrides::{apply_block_overrides, apply_state_overrides, OverrideBlockHashes}; +use alloy_network::TransactionBuilder; use alloy_primitives::{Bytes, B256, U256}; use alloy_rpc_types_eth::{ simulate::{SimBlock, SimulatePayload, SimulatedBlock}, state::{EvmOverrides, StateOverride}, - transaction::TransactionRequest, BlockId, Bundle, EthCallResponse, StateContext, TransactionInfo, }; use futures::Future; use reth_errors::{ProviderError, RethError}; use reth_evm::{ - ConfigureEvm, Evm, EvmEnv, EvmEnvFor, HaltReasonFor, InspectorFor, SpecFor, TransactionEnv, - TxEnvFor, -}; -use reth_node_api::{BlockBody, NodePrimitives}; -use reth_primitives_traits::{Recovered, SealedHeader, SignedTransaction}; -use reth_provider::{BlockIdReader, ProviderHeader, ProviderTx}; -use reth_revm::{ - database::StateProviderDatabase, - db::{CacheDB, State}, - DatabaseRef, + env::BlockEnvironment, ConfigureEvm, Evm, EvmEnvFor, HaltReasonFor, InspectorFor, + TransactionEnv, TxEnvFor, }; +use reth_node_api::BlockBody; +use reth_primitives_traits::Recovered; +use reth_revm::{database::StateProviderDatabase, db::State}; +use reth_rpc_convert::{RpcConvert, RpcTxReq}; use reth_rpc_eth_types::{ cache::db::{StateCacheDbRefMutWrapper, StateProviderTraitObjWrapper}, error::{api::FromEvmHalt, ensure_success, FromEthApiError}, - revm_utils::{apply_block_overrides, apply_state_overrides, caller_gas_allowance}, simulate::{self, EthSimulateError}, - EthApiError, RevertError, RpcInvalidTransactionError, StateCacheDb, + EthApiError, RevertError, StateCacheDb, }; +use reth_storage_api::{BlockIdReader, ProviderTx}; use revm::{ + context::Block, context_interface::{ result::{ExecutionResult, ResultAndState}, Transaction, @@ -43,7 +43,7 @@ use revm::{ Database, DatabaseCommit, }; use revm_inspectors::{access_list::AccessListInspector, transfer::TransferInspector}; -use tracing::trace; +use tracing::{trace, warn}; /// Result type for `eth_simulateV1` RPC method. pub type SimulatedBlocksResult = Result>>, E>; @@ -54,7 +54,7 @@ pub trait EthCall: EstimateCall + Call + LoadPendingBlock + LoadBlock + FullEthA /// Estimate gas needed for execution of the `request` at the [`BlockId`]. fn estimate_gas_at( &self, - request: TransactionRequest, + request: RpcTxReq<::Network>, at: BlockId, state_override: Option, ) -> impl Future> + Send { @@ -67,7 +67,7 @@ pub trait EthCall: EstimateCall + Call + LoadPendingBlock + LoadBlock + FullEthA /// See also: fn simulate_v1( &self, - payload: SimulatePayload, + payload: SimulatePayload::Network>>, block: Option, ) -> impl Future> + Send { async move { @@ -112,37 +112,41 @@ pub trait EthCall: EstimateCall + Call + LoadPendingBlock + LoadBlock + FullEthA // If not explicitly required, we disable nonce check evm_env.cfg_env.disable_nonce_check = true; evm_env.cfg_env.disable_base_fee = true; - evm_env.block_env.basefee = 0; + evm_env.cfg_env.tx_gas_limit_cap = Some(u64::MAX); + evm_env.block_env.inner_mut().basefee = 0; } let SimBlock { block_overrides, state_overrides, calls } = block; if let Some(block_overrides) = block_overrides { // ensure we don't allow uncapped gas limit per block - if let Some(gas_limit_override) = block_overrides.gas_limit { - if gas_limit_override > evm_env.block_env.gas_limit && - gas_limit_override > this.call_gas_limit() - { - return Err( - EthApiError::other(EthSimulateError::GasLimitReached).into() - ) - } + if let Some(gas_limit_override) = block_overrides.gas_limit && + gas_limit_override > evm_env.block_env.gas_limit() && + gas_limit_override > this.call_gas_limit() + { + return Err(EthApiError::other(EthSimulateError::GasLimitReached).into()) } - apply_block_overrides(block_overrides, &mut db, &mut evm_env.block_env); + apply_block_overrides( + block_overrides, + &mut db, + evm_env.block_env.inner_mut(), + ); } if let Some(state_overrides) = state_overrides { - apply_state_overrides(state_overrides, &mut db)?; + apply_state_overrides(state_overrides, &mut db) + .map_err(Self::Error::from_eth_err)?; } - let block_env = evm_env.block_env.clone(); + let block_gas_limit = evm_env.block_env.gas_limit(); let chain_id = evm_env.cfg_env.chain_id; let default_gas_limit = { - let total_specified_gas = calls.iter().filter_map(|tx| tx.gas).sum::(); + let total_specified_gas = + calls.iter().filter_map(|tx| tx.as_ref().gas_limit()).sum::(); let txs_without_gas_limit = - calls.iter().filter(|tx| tx.gas.is_none()).count(); + calls.iter().filter(|tx| tx.as_ref().gas_limit().is_none()).count(); - if total_specified_gas > block_env.gas_limit { + if total_specified_gas > block_gas_limit { return Err(EthApiError::Other(Box::new( EthSimulateError::BlockGasLimitExceeded, )) @@ -150,8 +154,7 @@ pub trait EthCall: EstimateCall + Call + LoadPendingBlock + LoadBlock + FullEthA } if txs_without_gas_limit > 0 { - (block_env.gas_limit - total_specified_gas) / - txs_without_gas_limit as u64 + (block_gas_limit - total_specified_gas) / txs_without_gas_limit as u64 } else { 0 } @@ -159,7 +162,9 @@ pub trait EthCall: EstimateCall + Call + LoadPendingBlock + LoadBlock + FullEthA let ctx = this .evm_config() - .context_for_next_block(&parent, this.next_env_attributes(&parent)?); + .context_for_next_block(&parent, this.next_env_attributes(&parent)?) + .map_err(RethError::other) + .map_err(Self::Error::from_eth_err)?; let (result, results) = if trace_transfers { // prepare inspector to capture transfer inside the evm so they are recorded // and included in logs @@ -187,18 +192,15 @@ pub trait EthCall: EstimateCall + Call + LoadPendingBlock + LoadBlock + FullEthA )? }; + parent = result.block.clone_sealed_header(); + let block = simulate::build_simulated_block( result.block, results, - return_full_transactions, + return_full_transactions.into(), this.tx_resp_builder(), )?; - parent = SealedHeader::new( - block.inner.header.inner.clone(), - block.inner.header.hash, - ); - blocks.push(block); } @@ -211,12 +213,12 @@ pub trait EthCall: EstimateCall + Call + LoadPendingBlock + LoadBlock + FullEthA /// Executes the call request (`eth_call`) and returns the output fn call( &self, - request: TransactionRequest, + request: RpcTxReq<::Network>, block_number: Option, overrides: EvmOverrides, ) -> impl Future> + Send { async move { - let (res, _env) = + let res = self.transact_call_at(request, block_number.unwrap_or_default(), overrides).await?; ensure_success(res.result) @@ -227,7 +229,7 @@ pub trait EthCall: EstimateCall + Call + LoadPendingBlock + LoadBlock + FullEthA /// optionality of state overrides fn call_many( &self, - bundles: Vec, + bundles: Vec::Network>>>, state_context: Option, mut state_override: Option, ) -> impl Future>, Self::Error>> + Send { @@ -281,7 +283,8 @@ pub trait EthCall: EstimateCall + Call + LoadPendingBlock + LoadBlock + FullEthA let this = self.clone(); self.spawn_with_state_at_block(at.into(), move |state| { let mut all_results = Vec::with_capacity(bundles.len()); - let mut db = CacheDB::new(StateProviderDatabase::new(state)); + let mut db = + State::builder().with_database(StateProviderDatabase::new(state)).build(); if replay_block_txs { // only need to replay the transactions in the block if not all transactions are @@ -289,13 +292,13 @@ pub trait EthCall: EstimateCall + Call + LoadPendingBlock + LoadBlock + FullEthA let block_transactions = block.transactions_recovered().take(num_txs); for tx in block_transactions { let tx_env = RpcNodeCore::evm_config(&this).tx_env(tx); - let (res, _) = this.transact(&mut db, evm_env.clone(), tx_env)?; + let res = this.transact(&mut db, evm_env.clone(), tx_env)?; db.commit(res.state); } } // transact all bundles - for bundle in bundles { + for (bundle_index, bundle) in bundles.into_iter().enumerate() { let Bundle { transactions, block_override } = bundle; if transactions.is_empty() { // Skip empty bundles @@ -306,15 +309,30 @@ pub trait EthCall: EstimateCall + Call + LoadPendingBlock + LoadBlock + FullEthA let block_overrides = block_override.map(Box::new); // transact all transactions in the bundle - for tx in transactions { + for (tx_index, tx) in transactions.into_iter().enumerate() { // Apply overrides, state overrides are only applied for the first tx in the // request let overrides = EvmOverrides::new(state_override.take(), block_overrides.clone()); - let (current_evm_env, prepared_tx) = - this.prepare_call_env(evm_env.clone(), tx, &mut db, overrides)?; - let (res, _) = this.transact(&mut db, current_evm_env, prepared_tx)?; + let (current_evm_env, prepared_tx) = this + .prepare_call_env(evm_env.clone(), tx, &mut db, overrides) + .map_err(|err| { + Self::Error::from_eth_err(EthApiError::call_many_error( + bundle_index, + tx_index, + err.into(), + )) + })?; + let res = this.transact(&mut db, current_evm_env, prepared_tx).map_err( + |err| { + Self::Error::from_eth_err(EthApiError::call_many_error( + bundle_index, + tx_index, + err.into(), + )) + }, + )?; match ensure_success::<_, Self::Error>(res.result) { Ok(output) => { @@ -343,11 +361,11 @@ pub trait EthCall: EstimateCall + Call + LoadPendingBlock + LoadBlock + FullEthA } } - /// Creates [`AccessListResult`] for the [`TransactionRequest`] at the given + /// Creates [`AccessListResult`] for the [`RpcTxReq`] at the given /// [`BlockId`], or latest block. fn create_access_list_at( &self, - request: TransactionRequest, + request: RpcTxReq<::Network>, block_number: Option, state_override: Option, ) -> impl Future> + Send @@ -358,105 +376,113 @@ pub trait EthCall: EstimateCall + Call + LoadPendingBlock + LoadBlock + FullEthA let block_id = block_number.unwrap_or_default(); let (evm_env, at) = self.evm_env_at(block_id).await?; - self.spawn_blocking_io(move |this| { - this.create_access_list_with(evm_env, at, request, state_override) + self.spawn_blocking_io_fut(move |this| async move { + this.create_access_list_with(evm_env, at, request, state_override).await }) .await } } - /// Creates [`AccessListResult`] for the [`TransactionRequest`] at the given + /// Creates [`AccessListResult`] for the [`RpcTxReq`] at the given /// [`BlockId`]. fn create_access_list_with( &self, mut evm_env: EvmEnvFor, at: BlockId, - mut request: TransactionRequest, + request: RpcTxReq<::Network>, state_override: Option, - ) -> Result + ) -> impl Future> + Send where Self: Trace, { - let state = self.state_at_block_id(at)?; - let mut db = CacheDB::new(StateProviderDatabase::new(state)); + self.spawn_blocking_io_fut(move |this| async move { + let state = this.state_at_block_id(at).await?; + let mut db = State::builder().with_database(StateProviderDatabase::new(state)).build(); - if let Some(state_overrides) = state_override { - apply_state_overrides(state_overrides, &mut db)?; - } + if let Some(state_overrides) = state_override { + apply_state_overrides(state_overrides, &mut db) + .map_err(Self::Error::from_eth_err)?; + } - let mut tx_env = self.create_txn_env(&evm_env, request.clone(), &mut db)?; + let mut tx_env = this.create_txn_env(&evm_env, request.clone(), &mut db)?; - // we want to disable this in eth_createAccessList, since this is common practice used by - // other node impls and providers - evm_env.cfg_env.disable_block_gas_limit = true; + // we want to disable this in eth_createAccessList, since this is common practice used + // by other node impls and providers + evm_env.cfg_env.disable_block_gas_limit = true; - // The basefee should be ignored for eth_createAccessList - // See: - // - evm_env.cfg_env.disable_base_fee = true; + // The basefee should be ignored for eth_createAccessList + // See: + // + evm_env.cfg_env.disable_base_fee = true; - // Disabled because eth_createAccessList is sometimes used with non-eoa senders - evm_env.cfg_env.disable_eip3607 = true; + // Disabled because eth_createAccessList is sometimes used with non-eoa senders + evm_env.cfg_env.disable_eip3607 = true; - if request.gas.is_none() && tx_env.gas_price() > 0 { - let cap = caller_gas_allowance(&mut db, &tx_env)?; - // no gas limit was provided in the request, so we need to cap the request's gas limit - tx_env.set_gas_limit(cap.min(evm_env.block_env.gas_limit)); - } - - // can consume the list since we're not using the request anymore - let initial = request.access_list.take().unwrap_or_default(); + if request.as_ref().gas_limit().is_none() && tx_env.gas_price() > 0 { + let cap = this.caller_gas_allowance(&mut db, &evm_env, &tx_env)?; + // no gas limit was provided in the request, so we need to cap the request's gas + // limit + tx_env.set_gas_limit(cap.min(evm_env.block_env.gas_limit())); + } - let mut inspector = AccessListInspector::new(initial); + // can consume the list since we're not using the request anymore + let initial = request.as_ref().access_list().cloned().unwrap_or_default(); + + let mut inspector = AccessListInspector::new(initial); + + let result = this.inspect(&mut db, evm_env.clone(), tx_env.clone(), &mut inspector)?; + let access_list = inspector.into_access_list(); + tx_env.set_access_list(access_list.clone()); + match result.result { + ExecutionResult::Halt { reason, gas_used } => { + let error = + Some(Self::Error::from_evm_halt(reason, tx_env.gas_limit()).to_string()); + return Ok(AccessListResult { + access_list, + gas_used: U256::from(gas_used), + error, + }) + } + ExecutionResult::Revert { output, gas_used } => { + let error = Some(RevertError::new(output).to_string()); + return Ok(AccessListResult { + access_list, + gas_used: U256::from(gas_used), + error, + }) + } + ExecutionResult::Success { .. } => {} + }; - let (result, (evm_env, mut tx_env)) = - self.inspect(&mut db, evm_env, tx_env, &mut inspector)?; - let access_list = inspector.into_access_list(); - tx_env.set_access_list(access_list.clone()); - match result.result { - ExecutionResult::Halt { reason, gas_used } => { - let error = - Some(Self::Error::from_evm_halt(reason, tx_env.gas_limit()).to_string()); - return Ok(AccessListResult { access_list, gas_used: U256::from(gas_used), error }) - } - ExecutionResult::Revert { output, gas_used } => { - let error = Some(RevertError::new(output).to_string()); - return Ok(AccessListResult { access_list, gas_used: U256::from(gas_used), error }) - } - ExecutionResult::Success { .. } => {} - }; - - // transact again to get the exact gas used - let (result, (_, tx_env)) = self.transact(&mut db, evm_env, tx_env)?; - let res = match result.result { - ExecutionResult::Halt { reason, gas_used } => { - let error = - Some(Self::Error::from_evm_halt(reason, tx_env.gas_limit()).to_string()); - AccessListResult { access_list, gas_used: U256::from(gas_used), error } - } - ExecutionResult::Revert { output, gas_used } => { - let error = Some(RevertError::new(output).to_string()); - AccessListResult { access_list, gas_used: U256::from(gas_used), error } - } - ExecutionResult::Success { gas_used, .. } => { - AccessListResult { access_list, gas_used: U256::from(gas_used), error: None } - } - }; + // transact again to get the exact gas used + let gas_limit = tx_env.gas_limit(); + let result = this.transact(&mut db, evm_env, tx_env)?; + let res = match result.result { + ExecutionResult::Halt { reason, gas_used } => { + let error = Some(Self::Error::from_evm_halt(reason, gas_limit).to_string()); + AccessListResult { access_list, gas_used: U256::from(gas_used), error } + } + ExecutionResult::Revert { output, gas_used } => { + let error = Some(RevertError::new(output).to_string()); + AccessListResult { access_list, gas_used: U256::from(gas_used), error } + } + ExecutionResult::Success { gas_used, .. } => { + AccessListResult { access_list, gas_used: U256::from(gas_used), error: None } + } + }; - Ok(res) + Ok(res) + }) } } /// Executes code on state. pub trait Call: LoadState< - Evm: ConfigureEvm< - Primitives: NodePrimitives< - BlockHeader = ProviderHeader, - SignedTx = ProviderTx, - >, - >, - Error: FromEvmError, + RpcConvert: RpcConvert, + Error: FromEvmError + + From<::Error> + + From, > + SpawnBlocking { /// Returns default gas limit to use for `eth_call` and tracing RPC methods. @@ -467,72 +493,80 @@ pub trait Call: /// Returns the maximum number of blocks accepted for `eth_simulateV1`. fn max_simulate_blocks(&self) -> u64; + /// Returns the maximum memory the EVM can allocate per RPC request. + fn evm_memory_limit(&self) -> u64; + + /// Returns the max gas limit that the caller can afford given a transaction environment. + fn caller_gas_allowance( + &self, + mut db: impl Database>, + _evm_env: &EvmEnvFor, + tx_env: &TxEnvFor, + ) -> Result { + alloy_evm::call::caller_gas_allowance(&mut db, tx_env).map_err(Self::Error::from_eth_err) + } + /// Executes the closure with the state that corresponds to the given [`BlockId`]. - fn with_state_at_block(&self, at: BlockId, f: F) -> Result + fn with_state_at_block( + &self, + at: BlockId, + f: F, + ) -> impl Future> + Send where - F: FnOnce(StateProviderTraitObjWrapper<'_>) -> Result, + R: Send + 'static, + F: FnOnce(Self, StateProviderTraitObjWrapper<'_>) -> Result + + Send + + 'static, { - let state = self.state_at_block_id(at)?; - f(StateProviderTraitObjWrapper(&state)) + self.spawn_blocking_io_fut(move |this| async move { + let state = this.state_at_block_id(at).await?; + f(this, StateProviderTraitObjWrapper(&state)) + }) } /// Executes the `TxEnv` against the given [Database] without committing state /// changes. - #[expect(clippy::type_complexity)] fn transact( &self, db: DB, evm_env: EvmEnvFor, tx_env: TxEnvFor, - ) -> Result< - (ResultAndState>, (EvmEnvFor, TxEnvFor)), - Self::Error, - > + ) -> Result>, Self::Error> where - DB: Database, + DB: Database + fmt::Debug, { - let mut evm = self.evm_config().evm_with_env(db, evm_env.clone()); - let res = evm.transact(tx_env.clone()).map_err(Self::Error::from_evm_err)?; + let mut evm = self.evm_config().evm_with_env(db, evm_env); + let res = evm.transact(tx_env).map_err(Self::Error::from_evm_err)?; - Ok((res, (evm_env, tx_env))) + Ok(res) } - /// Executes the [`EvmEnv`] against the given [Database] without committing state + /// Executes the [`reth_evm::EvmEnv`] against the given [Database] without committing state /// changes. - #[expect(clippy::type_complexity)] fn transact_with_inspector( &self, db: DB, evm_env: EvmEnvFor, tx_env: TxEnvFor, inspector: I, - ) -> Result< - (ResultAndState>, (EvmEnvFor, TxEnvFor)), - Self::Error, - > + ) -> Result>, Self::Error> where - DB: Database, + DB: Database + fmt::Debug, I: InspectorFor, { - let mut evm = self.evm_config().evm_with_env_and_inspector(db, evm_env.clone(), inspector); - let res = evm.transact(tx_env.clone()).map_err(Self::Error::from_evm_err)?; + let mut evm = self.evm_config().evm_with_env_and_inspector(db, evm_env, inspector); + let res = evm.transact(tx_env).map_err(Self::Error::from_evm_err)?; - Ok((res, (evm_env, tx_env))) + Ok(res) } /// Executes the call request at the given [`BlockId`]. - #[expect(clippy::type_complexity)] fn transact_call_at( &self, - request: TransactionRequest, + request: RpcTxReq<::Network>, at: BlockId, overrides: EvmOverrides, - ) -> impl Future< - Output = Result< - (ResultAndState>, (EvmEnvFor, TxEnvFor)), - Self::Error, - >, - > + Send + ) -> impl Future>, Self::Error>> + Send where Self: LoadPendingBlock, { @@ -552,16 +586,16 @@ pub trait Call: F: FnOnce(StateProviderTraitObjWrapper<'_>) -> Result + Send + 'static, R: Send + 'static, { - self.spawn_tracing(move |this| { - let state = this.state_at_block_id(at)?; + self.spawn_blocking_io_fut(move |this| async move { + let state = this.state_at_block_id(at).await?; f(StateProviderTraitObjWrapper(&state)) }) } - /// Prepares the state and env for the given [`TransactionRequest`] at the given [`BlockId`] and + /// Prepares the state and env for the given [`RpcTxReq`] at the given [`BlockId`] and /// executes the closure on a new task returning the result of the closure. /// - /// This returns the configured [`EvmEnv`] for the given [`TransactionRequest`] at + /// This returns the configured [`reth_evm::EvmEnv`] for the given [`RpcTxReq`] at /// the given [`BlockId`] and with configured call settings: `prepare_call_env`. /// /// This is primarily used by `eth_call`. @@ -575,7 +609,7 @@ pub trait Call: /// instead, where blocking IO is less problematic. fn spawn_with_call_at( &self, - request: TransactionRequest, + request: RpcTxReq<::Network>, at: BlockId, overrides: EvmOverrides, f: F, @@ -594,10 +628,11 @@ pub trait Call: async move { let (evm_env, at) = self.evm_env_at(at).await?; let this = self.clone(); - self.spawn_blocking_io(move |_| { - let state = this.state_at_block_id(at)?; - let mut db = - CacheDB::new(StateProviderDatabase::new(StateProviderTraitObjWrapper(&state))); + self.spawn_blocking_io_fut(move |_| async move { + let state = this.state_at_block_id(at).await?; + let mut db = State::builder() + .with_database(StateProviderDatabase::new(StateProviderTraitObjWrapper(&state))) + .build(); let (evm_env, tx_env) = this.prepare_call_env(evm_env, request, &mut db, overrides)?; @@ -648,7 +683,8 @@ pub trait Call: let this = self.clone(); self.spawn_with_state_at_block(parent_block.into(), move |state| { - let mut db = CacheDB::new(StateProviderDatabase::new(state)); + let mut db = + State::builder().with_database(StateProviderDatabase::new(state)).build(); let block_txs = block.transactions_recovered(); // replay all transactions prior to the targeted transaction @@ -656,7 +692,7 @@ pub trait Call: let tx_env = RpcNodeCore::evm_config(&this).tx_env(tx); - let (res, _) = this.transact(&mut db, evm_env, tx_env)?; + let res = this.transact(&mut db, evm_env, tx_env)?; f(tx_info, res, db) }) .await @@ -667,7 +703,7 @@ pub trait Call: /// Replays all the transactions until the target transaction is found. /// /// All transactions before the target transaction are executed and their changes are written to - /// the _runtime_ db ([`CacheDB`]). + /// the _runtime_ db ([`State`]). /// /// Note: This assumes the target transaction is in the given iterator. /// Returns the index of the target transaction in the given iterator. @@ -679,7 +715,7 @@ pub trait Call: target_tx_hash: B256, ) -> Result where - DB: Database + DatabaseCommit, + DB: Database + DatabaseCommit + core::fmt::Debug, I: IntoIterator>>, { let mut evm = self.evm_config().evm_with_env(db, evm_env); @@ -697,18 +733,28 @@ pub trait Call: Ok(index) } - /// Configures a new `TxEnv` for the [`TransactionRequest`] /// - /// All `TxEnv` fields are derived from the given [`TransactionRequest`], if fields are - /// `None`, they fall back to the [`EvmEnv`]'s settings. + /// All `TxEnv` fields are derived from the given [`RpcTxReq`], if fields are + /// `None`, they fall back to the [`reth_evm::EvmEnv`]'s settings. fn create_txn_env( &self, - evm_env: &EvmEnv>, - request: TransactionRequest, - db: impl Database>, - ) -> Result, Self::Error>; + evm_env: &EvmEnvFor, + mut request: RpcTxReq<::Network>, + mut db: impl Database>, + ) -> Result, Self::Error> { + if request.as_ref().nonce().is_none() { + let nonce = db + .basic(request.as_ref().from().unwrap_or_default()) + .map_err(Into::into)? + .map(|acc| acc.nonce) + .unwrap_or_default(); + request.as_mut().set_nonce(nonce); + } + + Ok(self.tx_resp_builder().tx_env(request, evm_env)?) + } - /// Prepares the [`EvmEnv`] for execution of calls. + /// Prepares the [`reth_evm::EvmEnv`] for execution of calls. /// /// Does not commit any changes to the underlying database. /// @@ -725,23 +771,31 @@ pub trait Call: fn prepare_call_env( &self, mut evm_env: EvmEnvFor, - mut request: TransactionRequest, - db: &mut CacheDB, + mut request: RpcTxReq<::Network>, + db: &mut DB, overrides: EvmOverrides, ) -> Result<(EvmEnvFor, TxEnvFor), Self::Error> where - DB: DatabaseRef, - EthApiError: From<::Error>, + DB: Database + DatabaseCommit + OverrideBlockHashes, + EthApiError: From<::Error>, { - if request.gas > Some(self.call_gas_limit()) { - // configured gas exceeds limit - return Err( - EthApiError::InvalidTransaction(RpcInvalidTransactionError::GasTooHigh).into() - ) + // track whether the request has a gas limit set + let request_has_gas_limit = request.as_ref().gas_limit().is_some(); + + if let Some(requested_gas) = request.as_ref().gas_limit() { + let global_gas_cap = self.call_gas_limit(); + if global_gas_cap != 0 && global_gas_cap < requested_gas { + warn!(target: "rpc::eth::call", ?request, ?global_gas_cap, "Capping gas limit to global gas cap"); + request.as_mut().set_gas_limit(global_gas_cap); + } + } else { + // cap request's gas limit to call gas limit + request.as_mut().set_gas_limit(self.call_gas_limit()); } - // apply configured gas cap - evm_env.block_env.gas_limit = self.call_gas_limit(); + // Disable block gas limit check to allow executing transactions with higher gas limit (call + // gas limit): https://github.com/paradigmxyz/reth/issues/18577 + evm_env.cfg_env.disable_block_gas_limit = true; // Disabled because eth_call is sometimes used with eoa senders // See @@ -752,27 +806,42 @@ pub trait Call: // evm_env.cfg_env.disable_base_fee = true; + // Disable EIP-7825 transaction gas limit to support larger transactions + evm_env.cfg_env.tx_gas_limit_cap = Some(u64::MAX); + + // Disable additional fee charges, e.g. opstack operator fee charge + // See: + // + evm_env.cfg_env.disable_fee_charge = true; + + evm_env.cfg_env.memory_limit = self.evm_memory_limit(); + // set nonce to None so that the correct nonce is chosen by the EVM - request.nonce = None; + request.as_mut().take_nonce(); if let Some(block_overrides) = overrides.block { - apply_block_overrides(*block_overrides, db, &mut evm_env.block_env); + apply_block_overrides(*block_overrides, db, evm_env.block_env.inner_mut()); } if let Some(state_overrides) = overrides.state { - apply_state_overrides(state_overrides, db)?; + apply_state_overrides(state_overrides, db) + .map_err(EthApiError::from_state_overrides_err)?; } - let request_gas = request.gas; let mut tx_env = self.create_txn_env(&evm_env, request, &mut *db)?; - if request_gas.is_none() { + // lower the basefee to 0 to avoid breaking EVM invariants (basefee < gasprice): + if tx_env.gas_price() == 0 { + evm_env.block_env.inner_mut().basefee = 0; + } + + if !request_has_gas_limit { // No gas limit was provided in the request, so we need to cap the transaction gas limit if tx_env.gas_price() > 0 { // If gas price is specified, cap transaction gas limit with caller allowance trace!(target: "rpc::eth::call", ?tx_env, "Applying gas limit cap with caller allowance"); - let cap = caller_gas_allowance(db, &tx_env)?; + let cap = self.caller_gas_allowance(db, &evm_env, &tx_env)?; // ensure we cap gas_limit to the block's - tx_env.set_gas_limit(cap.min(evm_env.block_env.gas_limit)); + tx_env.set_gas_limit(cap.min(evm_env.block_env.gas_limit())); } } diff --git a/crates/rpc/rpc-eth-api/src/helpers/config.rs b/crates/rpc/rpc-eth-api/src/helpers/config.rs new file mode 100644 index 00000000000..c4014e6f204 --- /dev/null +++ b/crates/rpc/rpc-eth-api/src/helpers/config.rs @@ -0,0 +1,173 @@ +//! Loads chain configuration. + +use alloy_consensus::Header; +use alloy_eips::eip7910::{EthConfig, EthForkConfig, SystemContract}; +use alloy_evm::precompiles::Precompile; +use alloy_primitives::Address; +use jsonrpsee::{core::RpcResult, proc_macros::rpc}; +use reth_chainspec::{ChainSpecProvider, EthChainSpec, EthereumHardforks, Hardforks, Head}; +use reth_errors::{ProviderError, RethError}; +use reth_evm::{precompiles::PrecompilesMap, ConfigureEvm, Evm}; +use reth_node_api::NodePrimitives; +use reth_revm::db::EmptyDB; +use reth_rpc_eth_types::EthApiError; +use reth_storage_api::BlockReaderIdExt; +use std::collections::BTreeMap; + +/// RPC endpoint support for [EIP-7910](https://eips.ethereum.org/EIPS/eip-7910) +#[cfg_attr(not(feature = "client"), rpc(server, namespace = "eth"))] +#[cfg_attr(feature = "client", rpc(server, client, namespace = "eth"))] +pub trait EthConfigApi { + /// Returns an object with data about recent and upcoming fork configurations. + #[method(name = "config")] + fn config(&self) -> RpcResult; +} + +/// Handler for the `eth_config` RPC endpoint. +/// +/// Ref: +#[derive(Debug, Clone)] +pub struct EthConfigHandler { + provider: Provider, + evm_config: Evm, +} + +impl EthConfigHandler +where + Provider: ChainSpecProvider + + BlockReaderIdExt

+ + 'static, + Evm: ConfigureEvm> + 'static, +{ + /// Creates a new [`EthConfigHandler`]. + pub const fn new(provider: Provider, evm_config: Evm) -> Self { + Self { provider, evm_config } + } + + /// Returns fork config for specific timestamp. + /// Returns [`None`] if no blob params were found for this fork. + fn build_fork_config_at( + &self, + timestamp: u64, + precompiles: BTreeMap, + ) -> Option { + let chain_spec = self.provider.chain_spec(); + + let mut system_contracts = BTreeMap::::default(); + + if chain_spec.is_cancun_active_at_timestamp(timestamp) { + system_contracts.extend(SystemContract::cancun()); + } + + if chain_spec.is_prague_active_at_timestamp(timestamp) { + system_contracts + .extend(SystemContract::prague(chain_spec.deposit_contract().map(|c| c.address))); + } + + // Fork config only exists for timestamp-based hardforks. + let fork_id = chain_spec + .fork_id(&Head { timestamp, number: u64::MAX, ..Default::default() }) + .hash + .0 + .into(); + + Some(EthForkConfig { + activation_time: timestamp, + blob_schedule: chain_spec.blob_params_at_timestamp(timestamp)?, + chain_id: chain_spec.chain().id(), + fork_id, + precompiles, + system_contracts, + }) + } + + fn config(&self) -> Result { + let chain_spec = self.provider.chain_spec(); + let latest = self + .provider + .latest_header()? + .ok_or_else(|| ProviderError::BestBlockNotFound)? + .into_header(); + + let current_precompiles = evm_to_precompiles_map( + self.evm_config.evm_for_block(EmptyDB::default(), &latest).map_err(RethError::other)?, + ); + + let mut fork_timestamps = + chain_spec.forks_iter().filter_map(|(_, cond)| cond.as_timestamp()).collect::>(); + fork_timestamps.sort_unstable(); + fork_timestamps.dedup(); + + let (current_fork_idx, current_fork_timestamp) = fork_timestamps + .iter() + .position(|ts| &latest.timestamp < ts) + .and_then(|idx| idx.checked_sub(1)) + .or_else(|| fork_timestamps.len().checked_sub(1)) + .and_then(|idx| fork_timestamps.get(idx).map(|ts| (idx, *ts))) + .ok_or_else(|| RethError::msg("no active timestamp fork found"))?; + + let current = self + .build_fork_config_at(current_fork_timestamp, current_precompiles) + .ok_or_else(|| RethError::msg("no fork config for current fork"))?; + + let mut config = EthConfig { current, next: None, last: None }; + + if let Some(next_fork_timestamp) = fork_timestamps.get(current_fork_idx + 1).copied() { + let fake_header = { + let mut header = latest.clone(); + header.timestamp = next_fork_timestamp; + header + }; + let next_precompiles = evm_to_precompiles_map( + self.evm_config + .evm_for_block(EmptyDB::default(), &fake_header) + .map_err(RethError::other)?, + ); + + config.next = self.build_fork_config_at(next_fork_timestamp, next_precompiles); + } else { + // If there is no fork scheduled, there is no "last" or "final" fork scheduled. + return Ok(config); + } + + let last_fork_timestamp = fork_timestamps.last().copied().unwrap(); + let fake_header = { + let mut header = latest; + header.timestamp = last_fork_timestamp; + header + }; + let last_precompiles = evm_to_precompiles_map( + self.evm_config + .evm_for_block(EmptyDB::default(), &fake_header) + .map_err(RethError::other)?, + ); + + config.last = self.build_fork_config_at(last_fork_timestamp, last_precompiles); + + Ok(config) + } +} + +impl EthConfigApiServer for EthConfigHandler +where + Provider: ChainSpecProvider + + BlockReaderIdExt
+ + 'static, + Evm: ConfigureEvm> + 'static, +{ + fn config(&self) -> RpcResult { + Ok(self.config().map_err(EthApiError::from)?) + } +} + +fn evm_to_precompiles_map( + evm: impl Evm, +) -> BTreeMap { + let precompiles = evm.precompiles(); + precompiles + .addresses() + .filter_map(|address| { + Some((precompiles.get(address)?.precompile_id().name().to_string(), *address)) + }) + .collect() +} diff --git a/crates/rpc/rpc-eth-api/src/helpers/estimate.rs b/crates/rpc/rpc-eth-api/src/helpers/estimate.rs index fa5da0d96e0..13a74d1cf00 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/estimate.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/estimate.rs @@ -2,27 +2,32 @@ use super::{Call, LoadPendingBlock}; use crate::{AsEthApiError, FromEthApiError, IntoEthApiError}; -use alloy_primitives::U256; -use alloy_rpc_types_eth::{state::StateOverride, transaction::TransactionRequest, BlockId}; +use alloy_evm::overrides::apply_state_overrides; +use alloy_network::TransactionBuilder; +use alloy_primitives::{TxKind, U256}; +use alloy_rpc_types_eth::{state::StateOverride, BlockId}; use futures::Future; use reth_errors::ProviderError; -use reth_evm::{Database, EvmEnvFor, TransactionEnv, TxEnvFor}; -use reth_provider::StateProvider; -use reth_revm::{database::StateProviderDatabase, db::CacheDB}; +use reth_evm::{ConfigureEvm, Database, Evm, EvmEnvFor, EvmFor, TransactionEnv, TxEnvFor}; +use reth_revm::{database::StateProviderDatabase, db::State}; +use reth_rpc_convert::{RpcConvert, RpcTxReq}; use reth_rpc_eth_types::{ - error::api::FromEvmHalt, - revm_utils::{apply_state_overrides, caller_gas_allowance}, + error::{api::FromEvmHalt, FromEvmError}, EthApiError, RevertError, RpcInvalidTransactionError, }; use reth_rpc_server_types::constants::gas_oracle::{CALL_STIPEND_GAS, ESTIMATE_GAS_ERROR_RATIO}; -use revm::context_interface::{result::ExecutionResult, Transaction}; +use reth_storage_api::StateProvider; +use revm::{ + context::Block, + context_interface::{result::ExecutionResult, Transaction}, +}; use tracing::trace; /// Gas execution estimates pub trait EstimateCall: Call { /// Estimates the gas usage of the `request` with the state. /// - /// This will execute the [`TransactionRequest`] and find the best gas limit via binary search. + /// This will execute the [`RpcTxReq`] and find the best gas limit via binary search. /// /// ## EVM settings /// @@ -34,7 +39,7 @@ pub trait EstimateCall: Call { fn estimate_gas_with( &self, mut evm_env: EvmEnvFor, - mut request: TransactionRequest, + mut request: RpcTxReq<::Network>, state: S, state_override: Option, ) -> Result @@ -51,103 +56,110 @@ pub trait EstimateCall: Call { evm_env.cfg_env.disable_base_fee = true; // set nonce to None so that the correct nonce is chosen by the EVM - request.nonce = None; + request.as_mut().take_nonce(); // Keep a copy of gas related request values - let tx_request_gas_limit = request.gas; - let tx_request_gas_price = request.gas_price; + let tx_request_gas_limit = request.as_ref().gas_limit(); + let tx_request_gas_price = request.as_ref().gas_price(); // the gas limit of the corresponding block - let block_env_gas_limit = evm_env.block_env.gas_limit; + let max_gas_limit = evm_env.cfg_env.tx_gas_limit_cap.map_or_else( + || evm_env.block_env.gas_limit(), + |cap| cap.min(evm_env.block_env.gas_limit()), + ); // Determine the highest possible gas limit, considering both the request's specified limit // and the block's limit. let mut highest_gas_limit = tx_request_gas_limit .map(|mut tx_gas_limit| { - if block_env_gas_limit < tx_gas_limit { + if max_gas_limit < tx_gas_limit { // requested gas limit is higher than the allowed gas limit, capping - tx_gas_limit = block_env_gas_limit; + tx_gas_limit = max_gas_limit; } tx_gas_limit }) - .unwrap_or(block_env_gas_limit); + .unwrap_or(max_gas_limit); // Configure the evm env - let mut db = CacheDB::new(StateProviderDatabase::new(state)); - let mut tx_env = self.create_txn_env(&evm_env, request, &mut db)?; + let mut db = State::builder().with_database(StateProviderDatabase::new(state)).build(); // Apply any state overrides if specified. if let Some(state_override) = state_override { apply_state_overrides(state_override, &mut db).map_err(Self::Error::from_eth_err)?; } - // [TODO]: Due to the existence of tokenRatio, simple transfer transactions in Mantle cannot be optimized. - // if tx_env.input().is_empty() { - // if let TxKind::Call(to) = tx_env.kind() { - // if let Ok(code) = db.db.account_code(&to) { - // let no_code_callee = code.map(|code| code.is_empty()).unwrap_or(true); - // if no_code_callee { - // // If the tx is a simple transfer (call to an account with no code) we - // can // shortcircuit. But simply returning - // // `MIN_TRANSACTION_GAS` is dangerous because there might be additional - // // field combos that bump the price up, so we try executing the function - // // with the minimum gas limit to make sure. - // let mut tx_env = tx_env.clone(); - // tx_env.set_gas_limit(MIN_TRANSACTION_GAS); - // if let Ok((res, _)) = self.transact(&mut db, evm_env.clone(), tx_env) { - // if res.result.is_success() { - // return Ok(U256::from(MIN_TRANSACTION_GAS)) - // } - // } - // } - // } - // } - // } + let mut tx_env = self.create_txn_env(&evm_env, request, &mut db)?; + + // Check if this is a basic transfer (no input data to account with no code) + let is_basic_transfer = if tx_env.input().is_empty() && + let TxKind::Call(to) = tx_env.kind() && + let Ok(code) = db.database.account_code(&to) + { + code.map(|code| code.is_empty()).unwrap_or(true) + } else { + false + }; // Check funds of the sender (only useful to check if transaction gas price is more than 0). // // The caller allowance is check by doing `(account.balance - tx.value) / tx.gas_price` if tx_env.gas_price() > 0 { // cap the highest gas limit by max gas caller can afford with given gas price - highest_gas_limit = highest_gas_limit - .min(caller_gas_allowance(&mut db, &tx_env).map_err(Self::Error::from_eth_err)?); + highest_gas_limit = + highest_gas_limit.min(self.caller_gas_allowance(&mut db, &evm_env, &tx_env)?); } // If the provided gas limit is less than computed cap, use that tx_env.set_gas_limit(tx_env.gas_limit().min(highest_gas_limit)); - trace!(target: "rpc::eth::estimate", ?evm_env, ?tx_env, "Starting gas estimation"); + // Create EVM instance once and reuse it throughout the entire estimation process + let mut evm = self.evm_config().evm_with_env(&mut db, evm_env); + + // Mantle not working for basic transfers MIN_TRANSACTION_GAS, so we disable it for now + // For basic transfers, try using minimum gas before running full binary search + // if is_basic_transfer { + // // If the tx is a simple transfer (call to an account with no code) we can + // // shortcircuit. But simply returning + // // `MIN_TRANSACTION_GAS` is dangerous because there might be additional + // // field combos that bump the price up, so we try executing the function + // // with the minimum gas limit to make sure. + // let mut min_tx_env = tx_env.clone(); + // min_tx_env.set_gas_limit(MIN_TRANSACTION_GAS); + + // // Reuse the same EVM instance + // if let Ok(res) = evm.transact(min_tx_env).map_err(Self::Error::from_evm_err) + // && res.result.is_success() + // { + // return Ok(U256::from(MIN_TRANSACTION_GAS)); + // } + // } + + trace!(target: "rpc::eth::estimate", ?tx_env, gas_limit = tx_env.gas_limit(), is_basic_transfer, "Starting gas estimation"); // Execute the transaction with the highest possible gas limit. - let (mut res, (mut evm_env, mut tx_env)) = - match self.transact(&mut db, evm_env.clone(), tx_env.clone()) { - // Handle the exceptional case where the transaction initialization uses too much - // gas. If the gas price or gas limit was specified in the request, - // retry the transaction with the block's gas limit to determine if - // the failure was due to insufficient gas. - Err(err) - if err.is_gas_too_high() && - (tx_request_gas_limit.is_some() || tx_request_gas_price.is_some()) => - { - return Err(self.map_out_of_gas_err( - block_env_gas_limit, - evm_env, - tx_env, - &mut db, - )) - } - Err(err) if err.is_gas_too_low() => { - // This failed because the configured gas cost of the tx was lower than what - // actually consumed by the tx This can happen if the - // request provided fee values manually and the resulting gas cost exceeds the - // sender's allowance, so we return the appropriate error here - return Err(RpcInvalidTransactionError::GasRequiredExceedsAllowance { - gas_limit: tx_env.gas_limit(), - } - .into_eth_err()) + let mut res = match evm.transact(tx_env.clone()).map_err(Self::Error::from_evm_err) { + // Handle the exceptional case where the transaction initialization uses too much + // gas. If the gas price or gas limit was specified in the request, + // retry the transaction with the block's gas limit to determine if + // the failure was due to insufficient gas. + Err(err) + if err.is_gas_too_high() + && (tx_request_gas_limit.is_some() || tx_request_gas_price.is_some()) => + { + return Self::map_out_of_gas_err(&mut evm, tx_env, max_gas_limit); + } + Err(err) if err.is_gas_too_low() => { + // This failed because the configured gas cost of the tx was lower than what + // actually consumed by the tx This can happen if the + // request provided fee values manually and the resulting gas cost exceeds the + // sender's allowance, so we return the appropriate error here + return Err(RpcInvalidTransactionError::GasRequiredExceedsAllowance { + gas_limit: tx_env.gas_limit(), } - // Propagate other results (successful or other errors). - ethres => ethres?, - }; + .into_eth_err()); + } + // Propagate other results (successful or other errors). + ethres => ethres?, + }; // [TODO]: Maybe not working for Mantle let gas_refund = match res.result { @@ -155,17 +167,17 @@ pub trait EstimateCall: Call { ExecutionResult::Halt { reason, .. } => { // here we don't check for invalid opcode because already executed with highest gas // limit - return Err(Self::Error::from_evm_halt(reason, tx_env.gas_limit())) + return Err(Self::Error::from_evm_halt(reason, tx_env.gas_limit())); } ExecutionResult::Revert { output, .. } => { // if price or limit was included in the request then we can execute the request // again with the block's gas limit to check if revert is gas related or not return if tx_request_gas_limit.is_some() || tx_request_gas_price.is_some() { - Err(self.map_out_of_gas_err(block_env_gas_limit, evm_env, tx_env, &mut db)) + Self::map_out_of_gas_err(&mut evm, tx_env, max_gas_limit) } else { // the transaction did revert Err(RpcInvalidTransactionError::Revert(RevertError::new(output)).into_eth_err()) - } + }; } }; @@ -191,10 +203,13 @@ pub trait EstimateCall: Call { let optimistic_gas_limit = (gas_used + gas_refund + CALL_STIPEND_GAS) * 64 / 63; if optimistic_gas_limit < highest_gas_limit { // Set the transaction's gas limit to the calculated optimistic gas limit. - tx_env.set_gas_limit(optimistic_gas_limit); + let mut optimistic_tx_env = tx_env.clone(); + optimistic_tx_env.set_gas_limit(optimistic_gas_limit); + // Re-execute the transaction with the new gas limit and update the result and // environment. - (res, (evm_env, tx_env)) = self.transact(&mut db, evm_env, tx_env)?; + res = evm.transact(optimistic_tx_env).map_err(Self::Error::from_evm_err)?; + // Update the gas used based on the new result. gas_used = res.result.gas_used(); // Update the gas limit estimates (highest and lowest) based on the execution result. @@ -212,24 +227,24 @@ pub trait EstimateCall: Call { ((highest_gas_limit as u128 + lowest_gas_limit as u128) / 2) as u64, ); - trace!(target: "rpc::eth::estimate", ?evm_env, ?tx_env, ?highest_gas_limit, ?lowest_gas_limit, ?mid_gas_limit, "Starting binary search for gas"); + trace!(target: "rpc::eth::estimate", ?highest_gas_limit, ?lowest_gas_limit, ?mid_gas_limit, "Starting binary search for gas"); // Binary search narrows the range to find the minimum gas limit needed for the transaction // to succeed. - while (highest_gas_limit - lowest_gas_limit) > 1 { + while lowest_gas_limit + 1 < highest_gas_limit { // An estimation error is allowed once the current gas limit range used in the binary // search is small enough (less than 1.5% of the highest gas limit) // { // Decrease the highest gas limit if gas is too high highest_gas_limit = mid_gas_limit; @@ -241,7 +256,7 @@ pub trait EstimateCall: Call { // Handle other cases, including successful transactions. ethres => { // Unpack the result and environment if the transaction was successful. - (res, (evm_env, tx_env)) = ethres?; + res = ethres?; // Update the estimated gas range based on the transaction result. update_estimated_gas_range( res.result, @@ -262,7 +277,7 @@ pub trait EstimateCall: Call { /// Estimate gas needed for execution of the `request` at the [`BlockId`]. fn estimate_gas_at( &self, - request: TransactionRequest, + request: RpcTxReq<::Network>, at: BlockId, state_override: Option, ) -> impl Future> + Send @@ -272,8 +287,8 @@ pub trait EstimateCall: Call { async move { let (evm_env, at) = self.evm_env_at(at).await?; - self.spawn_blocking_io(move |this| { - let state = this.state_at_block_id(at)?; + self.spawn_blocking_io_fut(move |this| async move { + let state = this.state_at_block_id(at).await?; EstimateCall::estimate_gas_with(&this, evm_env, request, state, state_override) }) .await @@ -284,34 +299,31 @@ pub trait EstimateCall: Call { /// or not #[inline] fn map_out_of_gas_err( - &self, - env_gas_limit: u64, - evm_env: EvmEnvFor, + evm: &mut EvmFor, mut tx_env: TxEnvFor, - db: &mut DB, - ) -> Self::Error + max_gas_limit: u64, + ) -> Result where DB: Database, EthApiError: From, { let req_gas_limit = tx_env.gas_limit(); - tx_env.set_gas_limit(env_gas_limit); - let (res, _) = match self.transact(db, evm_env, tx_env) { - Ok(res) => res, - Err(err) => return err, - }; - match res.result { + tx_env.set_gas_limit(max_gas_limit); + + let retry_res = evm.transact(tx_env).map_err(Self::Error::from_evm_err)?; + + match retry_res.result { ExecutionResult::Success { .. } => { - // transaction succeeded by manually increasing the gas limit to - // highest, which means the caller lacks funds to pay for the tx - RpcInvalidTransactionError::BasicOutOfGas(req_gas_limit).into_eth_err() + // Transaction succeeded by manually increasing the gas limit, + // which means the caller lacks funds to pay for the tx + Err(RpcInvalidTransactionError::BasicOutOfGas(req_gas_limit).into_eth_err()) } ExecutionResult::Revert { output, .. } => { // reverted again after bumping the limit - RpcInvalidTransactionError::Revert(RevertError::new(output)).into_eth_err() + Err(RpcInvalidTransactionError::Revert(RevertError::new(output)).into_eth_err()) } ExecutionResult::Halt { reason, .. } => { - Self::Error::from_evm_halt(reason, req_gas_limit) + Err(Self::Error::from_evm_halt(reason, req_gas_limit)) } } } diff --git a/crates/rpc/rpc-eth-api/src/helpers/fee.rs b/crates/rpc/rpc-eth-api/src/helpers/fee.rs index 78effdb3f57..b0d736981c2 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/fee.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/fee.rs @@ -7,18 +7,22 @@ use alloy_eips::eip7840::BlobParams; use alloy_primitives::U256; use alloy_rpc_types_eth::{BlockNumberOrTag, FeeHistory}; use futures::Future; -use reth_chainspec::EthChainSpec; +use reth_chainspec::{ChainSpecProvider, EthChainSpec}; use reth_primitives_traits::BlockBody; -use reth_provider::{BlockIdReader, ChainSpecProvider, HeaderProvider}; use reth_rpc_eth_types::{ - fee_history::calculate_reward_percentiles_for_block, EthApiError, FeeHistoryCache, - FeeHistoryEntry, GasPriceOracle, RpcInvalidTransactionError, + fee_history::calculate_reward_percentiles_for_block, utils::checked_blob_gas_used_ratio, + EthApiError, FeeHistoryCache, FeeHistoryEntry, GasPriceOracle, RpcInvalidTransactionError, }; +use reth_storage_api::{BlockIdReader, BlockReaderIdExt, HeaderProvider, ProviderHeader}; use tracing::debug; /// Fee related functions for the [`EthApiServer`](crate::EthApiServer) trait in the /// `eth_` namespace. -pub trait EthFees: LoadFee { +pub trait EthFees: + LoadFee< + Provider: ChainSpecProvider>>, +> +{ /// Returns a suggestion for a gas price for legacy transactions. /// /// See also: @@ -86,8 +90,6 @@ pub trait EthFees: LoadFee { if newest_block.is_pending() { // cap the target block since we don't have fee history for the pending block newest_block = BlockNumberOrTag::Latest; - // account for missing pending block - block_count = block_count.saturating_sub(1); } let end_block = self @@ -107,10 +109,10 @@ pub trait EthFees: LoadFee { // need to validate that they are monotonically // increasing and 0 <= p <= 100 // Note: The types used ensure that the percentiles are never < 0 - if let Some(percentiles) = &reward_percentiles { - if percentiles.windows(2).any(|w| w[0] > w[1] || w[0] > 100.) { - return Err(EthApiError::InvalidRewardPercentiles.into()) - } + if let Some(percentiles) = &reward_percentiles && + percentiles.windows(2).any(|w| w[0] > w[1] || w[0] > 100.) + { + return Err(EthApiError::InvalidRewardPercentiles.into()) } // Fetch the headers and ensure we got all of them @@ -138,7 +140,8 @@ pub trait EthFees: LoadFee { } for entry in &fee_entries { - base_fee_per_gas.push(entry.base_fee_per_gas as u128); + base_fee_per_gas + .push(entry.header.base_fee_per_gas().unwrap_or_default() as u128); gas_used_ratio.push(entry.gas_used_ratio); base_fee_per_blob_gas.push(entry.base_fee_per_blob_gas.unwrap_or_default()); blob_gas_used_ratio.push(entry.blob_gas_used_ratio); @@ -155,8 +158,12 @@ pub trait EthFees: LoadFee { // Also need to include the `base_fee_per_gas` and `base_fee_per_blob_gas` for the // next block - base_fee_per_gas - .push(last_entry.next_block_base_fee(self.provider().chain_spec()) as u128); + base_fee_per_gas.push( + self.provider() + .chain_spec() + .next_block_base_fee(&last_entry.header, last_entry.header.timestamp()) + .unwrap_or_default() as u128, + ); base_fee_per_blob_gas.push(last_entry.next_block_blob_fee().unwrap_or_default()); } else { @@ -168,20 +175,21 @@ pub trait EthFees: LoadFee { return Err(EthApiError::InvalidBlockRange.into()) } - + let chain_spec = self.provider().chain_spec(); for header in &headers { base_fee_per_gas.push(header.base_fee_per_gas().unwrap_or_default() as u128); gas_used_ratio.push(header.gas_used() as f64 / header.gas_limit() as f64); - let blob_params = self.provider() - .chain_spec() + let blob_params = chain_spec .blob_params_at_timestamp(header.timestamp()) .unwrap_or_else(BlobParams::cancun); base_fee_per_blob_gas.push(header.blob_fee(blob_params).unwrap_or_default()); blob_gas_used_ratio.push( - header.blob_gas_used().unwrap_or_default() as f64 - / blob_params.max_blob_gas_per_block() as f64, + checked_blob_gas_used_ratio( + header.blob_gas_used().unwrap_or_default(), + blob_params.max_blob_gas_per_block(), + ) ); // Percentiles were specified, so we need to collect reward percentile info @@ -211,18 +219,16 @@ pub trait EthFees: LoadFee { // The unwrap is safe since we checked earlier that we got at least 1 header. let last_header = headers.last().expect("is present"); base_fee_per_gas.push( - last_header.next_block_base_fee( - self.provider() - .chain_spec() - .base_fee_params_at_timestamp(last_header.timestamp())).unwrap_or_default() as u128 + chain_spec + .next_block_base_fee(last_header.header(), last_header.timestamp()) + .unwrap_or_default() as u128, ); - // Same goes for the `base_fee_per_blob_gas`: // > "[..] includes the next block after the newest of the returned range, because this value can be derived from the newest block. base_fee_per_blob_gas.push( last_header .maybe_next_block_blob_fee( - self.provider().chain_spec().blob_params_at_timestamp(last_header.timestamp()) + chain_spec.blob_params_at_timestamp(last_header.timestamp()) ).unwrap_or_default() ); }; @@ -240,7 +246,11 @@ pub trait EthFees: LoadFee { /// Approximates reward at a given percentile for a specific block /// Based on the configured resolution - fn approximate_percentile(&self, entry: &FeeHistoryEntry, requested_percentile: f64) -> u128 { + fn approximate_percentile( + &self, + entry: &FeeHistoryEntry>, + requested_percentile: f64, + ) -> u128 { let resolution = self.fee_history_cache().resolution(); let rounded_percentile = (requested_percentile * resolution as f64).round() / resolution as f64; @@ -256,7 +266,10 @@ pub trait EthFees: LoadFee { /// Loads fee from database. /// /// Behaviour shared by several `eth_` RPC methods, not exclusive to `eth_` fees RPC methods. -pub trait LoadFee: LoadBlock { +pub trait LoadFee: LoadBlock +where + Self::Provider: BlockReaderIdExt, +{ /// Returns a handle for reading gas price. /// /// Data access in default (L1) trait method implementations. @@ -265,7 +278,7 @@ pub trait LoadFee: LoadBlock { /// Returns a handle for reading fee history data from memory. /// /// Data access in default (L1) trait method implementations. - fn fee_history_cache(&self) -> &FeeHistoryCache; + fn fee_history_cache(&self) -> &FeeHistoryCache>; /// Returns the gas price if it is set, otherwise fetches a suggested gas price for legacy /// transactions. @@ -335,10 +348,9 @@ pub trait LoadFee: LoadBlock { /// /// See also: fn gas_price(&self) -> impl Future> + Send { - let header = self.recovered_block(BlockNumberOrTag::Latest.into()); - let suggested_tip = self.suggested_priority_fee(); async move { - let (header, suggested_tip) = futures::try_join!(header, suggested_tip)?; + let header = self.provider().latest_header().map_err(Self::Error::from_eth_err)?; + let suggested_tip = self.suggested_priority_fee().await?; let base_fee = header.and_then(|h| h.base_fee_per_gas()).unwrap_or_default(); Ok(suggested_tip + U256::from(base_fee)) } @@ -347,8 +359,9 @@ pub trait LoadFee: LoadBlock { /// Returns a suggestion for a base fee for blob transactions. fn blob_base_fee(&self) -> impl Future> + Send { async move { - self.recovered_block(BlockNumberOrTag::Latest.into()) - .await? + self.provider() + .latest_header() + .map_err(Self::Error::from_eth_err)? .and_then(|h| { h.maybe_next_block_blob_fee( self.provider().chain_spec().blob_params_at_timestamp(h.timestamp()), diff --git a/crates/rpc/rpc-eth-api/src/helpers/mod.rs b/crates/rpc/rpc-eth-api/src/helpers/mod.rs index 27d23da74b2..19a72ccafb7 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/mod.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/mod.rs @@ -17,6 +17,7 @@ pub mod block; pub mod blocking_task; pub mod call; +pub mod config; pub mod estimate; pub mod fee; pub mod pending_block; @@ -33,7 +34,7 @@ pub use call::{Call, EthCall}; pub use fee::{EthFees, LoadFee}; pub use pending_block::LoadPendingBlock; pub use receipt::LoadReceipt; -pub use signer::{AddDevSigners, EthSigner}; +pub use signer::EthSigner; pub use spec::EthApiSpec; pub use state::{EthState, LoadState}; pub use trace::Trace; diff --git a/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs b/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs index b8c682651f7..1dda44d090e 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs @@ -2,33 +2,39 @@ //! RPC methods. use super::SpawnBlocking; -use crate::{types::RpcTypes, EthApiTypes, FromEthApiError, FromEvmError, RpcNodeCore}; +use crate::{EthApiTypes, FromEthApiError, FromEvmError, RpcNodeCore}; use alloy_consensus::{BlockHeader, Transaction}; use alloy_eips::eip7840::BlobParams; +use alloy_primitives::{B256, U256}; use alloy_rpc_types_eth::BlockNumberOrTag; use futures::Future; -use reth_chainspec::{EthChainSpec, EthereumHardforks}; -use reth_errors::{BlockExecutionError, BlockValidationError, RethError}; +use reth_chain_state::{BlockState, ExecutedBlock}; +use reth_chainspec::{ChainSpecProvider, EthChainSpec}; +use reth_errors::{BlockExecutionError, BlockValidationError, ProviderError, RethError}; use reth_evm::{ - execute::{BlockBuilder, BlockBuilderOutcome}, - ConfigureEvm, Evm, SpecFor, + execute::{BlockBuilder, BlockBuilderOutcome, ExecutionOutcome}, + ConfigureEvm, Evm, NextBlockEnvAttributes, }; -use reth_node_api::NodePrimitives; -use reth_primitives_traits::{ - transaction::error::InvalidTransactionError, Receipt, RecoveredBlock, SealedHeader, +use reth_primitives_traits::{transaction::error::InvalidTransactionError, HeaderTy, SealedHeader}; +use reth_revm::{database::StateProviderDatabase, db::State}; +use reth_rpc_convert::RpcConvert; +use reth_rpc_eth_types::{ + block::BlockAndReceipts, builder::config::PendingBlockKind, EthApiError, PendingBlock, + PendingBlockEnv, PendingBlockEnvOrigin, }; -use reth_provider::{ - BlockReader, BlockReaderIdExt, ChainSpecProvider, ProviderBlock, ProviderError, ProviderHeader, - ProviderReceipt, ProviderTx, ReceiptProvider, StateProviderFactory, +use reth_storage_api::{ + noop::NoopProvider, BlockReader, BlockReaderIdExt, ProviderHeader, ProviderTx, ReceiptProvider, + StateProviderBox, StateProviderFactory, }; -use reth_revm::{database::StateProviderDatabase, db::State}; -use reth_rpc_eth_types::{EthApiError, PendingBlock, PendingBlockEnv, PendingBlockEnvOrigin}; use reth_transaction_pool::{ - error::InvalidPoolTransactionError, BestTransactionsAttributes, PoolTransaction, - TransactionPool, + error::InvalidPoolTransactionError, BestTransactions, BestTransactionsAttributes, + PoolTransaction, TransactionPool, }; use revm::context_interface::Block; -use std::time::{Duration, Instant}; +use std::{ + sync::Arc, + time::{Duration, Instant}, +}; use tokio::sync::Mutex; use tracing::debug; @@ -37,64 +43,44 @@ use tracing::debug; /// Behaviour shared by several `eth_` RPC methods, not exclusive to `eth_` blocks RPC methods. pub trait LoadPendingBlock: EthApiTypes< - NetworkTypes: RpcTypes< - Header = alloy_rpc_types_eth::Header>, - >, Error: FromEvmError, - > + RpcNodeCore< - Provider: BlockReaderIdExt - + ChainSpecProvider - + StateProviderFactory, - Evm: ConfigureEvm< - Primitives: NodePrimitives< - BlockHeader = ProviderHeader, - SignedTx = ProviderTx, - Receipt = ProviderReceipt, - Block = ProviderBlock, - >, - >, - > + RpcConvert: RpcConvert, + > + RpcNodeCore { /// Returns a handle to the pending block. /// /// Data access in default (L1) trait method implementations. - #[expect(clippy::type_complexity)] - fn pending_block( - &self, - ) -> &Mutex, ProviderReceipt>>>; + fn pending_block(&self) -> &Mutex>>; + + /// Returns a [`PendingEnvBuilder`] for the pending block. + fn pending_env_builder(&self) -> &dyn PendingEnvBuilder; + + /// Returns the pending block kind + fn pending_block_kind(&self) -> PendingBlockKind; /// Configures the [`PendingBlockEnv`] for the pending block /// /// If no pending block is available, this will derive it from the `latest` block - #[expect(clippy::type_complexity)] - fn pending_block_env_and_cfg( - &self, - ) -> Result< - PendingBlockEnv< - ProviderBlock, - ProviderReceipt, - SpecFor, - >, - Self::Error, - > { - if let Some(block) = - self.provider().pending_block_with_senders().map_err(Self::Error::from_eth_err)? - { - if let Some(receipts) = self + fn pending_block_env_and_cfg(&self) -> Result, Self::Error> { + if let Some(block) = self.provider().pending_block().map_err(Self::Error::from_eth_err)? && + let Some(receipts) = self .provider() .receipts_by_block(block.hash().into()) .map_err(Self::Error::from_eth_err)? - { - // Note: for the PENDING block we assume it is past the known merge block and - // thus this will not fail when looking up the total - // difficulty value for the blockenv. - let evm_env = self.evm_config().evm_env(block.header()); - - return Ok(PendingBlockEnv::new( - evm_env, - PendingBlockEnvOrigin::ActualPending(block, receipts), - )); - } + { + // Note: for the PENDING block we assume it is past the known merge block and + // thus this will not fail when looking up the total + // difficulty value for the blockenv. + let evm_env = self + .evm_config() + .evm_env(block.header()) + .map_err(RethError::other) + .map_err(Self::Error::from_eth_err)?; + + return Ok(PendingBlockEnv::new( + evm_env, + PendingBlockEnvOrigin::ActualPending(Arc::new(block), Arc::new(receipts)), + )); } // no pending block from the CL yet, so we use the latest block and modify the env @@ -118,32 +104,47 @@ pub trait LoadPendingBlock: fn next_env_attributes( &self, parent: &SealedHeader>, - ) -> Result<::NextBlockEnvCtx, Self::Error>; + ) -> Result<::NextBlockEnvCtx, Self::Error> { + Ok(self.pending_env_builder().pending_env_attributes(parent)?) + } - /// Returns the locally built pending block - #[expect(clippy::type_complexity)] - fn local_pending_block( + /// Returns a [`StateProviderBox`] on a mem-pool built pending block overlaying latest. + fn local_pending_state( &self, - ) -> impl Future< - Output = Result< - Option<( - RecoveredBlock<::Block>, - Vec>, - )>, - Self::Error, - >, - > + Send + ) -> impl Future, Self::Error>> + Send + where + Self: SpawnBlocking, + { + async move { + let Some(pending_block) = self.pool_pending_block().await? else { + return Ok(None); + }; + + let latest_historical = self + .provider() + .history_by_block_hash(pending_block.block().parent_hash()) + .map_err(Self::Error::from_eth_err)?; + + let state = BlockState::from(pending_block); + + Ok(Some(Box::new(state.state_provider(latest_historical)) as StateProviderBox)) + } + } + + /// Returns a mem-pool built pending block. + fn pool_pending_block( + &self, + ) -> impl Future>, Self::Error>> + Send where Self: SpawnBlocking, - Self::Pool: - TransactionPool>>, { async move { + if self.pending_block_kind().is_none() { + return Ok(None); + } let pending = self.pending_block_env_and_cfg()?; let parent = match pending.origin { - PendingBlockEnvOrigin::ActualPending(block, receipts) => { - return Ok(Some((block, receipts))); - } + PendingBlockEnvOrigin::ActualPending(..) => return Ok(None), PendingBlockEnvOrigin::DerivedFromLatest(parent) => parent, }; @@ -152,19 +153,18 @@ pub trait LoadPendingBlock: let now = Instant::now(); - // check if the block is still good + // Is the pending block cached? if let Some(pending_block) = lock.as_ref() { - // this is guaranteed to be the `latest` header - if pending.evm_env.block_env.number == pending_block.block.number() && - parent.hash() == pending_block.block.parent_hash() && + // Is the cached block not expired and latest is its parent? + if pending.evm_env.block_env.number() == U256::from(pending_block.block().number()) && + parent.hash() == pending_block.block().parent_hash() && now <= pending_block.expires_at { - return Ok(Some((pending_block.block.clone(), pending_block.receipts.clone()))); + return Ok(Some(pending_block.clone())); } } - // no pending block from the CL yet, so we need to build it ourselves via txpool - let (sealed_block, receipts) = match self + let executed_block = match self .spawn_blocking_io(move |this| { // we rebuild the block this.build_block(&parent) @@ -178,14 +178,41 @@ pub trait LoadPendingBlock: } }; - let now = Instant::now(); - *lock = Some(PendingBlock::new( - now + Duration::from_secs(1), - sealed_block.clone(), - receipts.clone(), - )); + let pending = PendingBlock::with_executed_block( + Instant::now() + Duration::from_secs(1), + executed_block, + ); + + *lock = Some(pending.clone()); + + Ok(Some(pending)) + } + } + + /// Returns the locally built pending block + fn local_pending_block( + &self, + ) -> impl Future>, Self::Error>> + Send + where + Self: SpawnBlocking, + Self::Pool: + TransactionPool>>, + { + async move { + if self.pending_block_kind().is_none() { + return Ok(None); + } - Ok(Some((sealed_block, receipts))) + let pending = self.pending_block_env_and_cfg()?; + + Ok(match pending.origin { + PendingBlockEnvOrigin::ActualPending(block, receipts) => { + Some(BlockAndReceipts { block, receipts }) + } + PendingBlockEnvOrigin::DerivedFromLatest(..) => { + self.pool_pending_block().await?.map(PendingBlock::into_block_and_receipts) + } + }) } } @@ -195,14 +222,10 @@ pub trait LoadPendingBlock: /// /// After Cancun, if the origin is the actual pending block, the block includes the EIP-4788 pre /// block contract call using the parent beacon block root received from the CL. - #[expect(clippy::type_complexity)] fn build_block( &self, parent: &SealedHeader>, - ) -> Result< - (RecoveredBlock>, Vec>), - Self::Error, - > + ) -> Result, Self::Error> where Self::Pool: TransactionPool>>, @@ -232,50 +255,56 @@ pub trait LoadPendingBlock: .unwrap_or_else(BlobParams::cancun); let mut cumulative_gas_used = 0; let mut sum_blob_gas_used = 0; - let block_gas_limit: u64 = block_env.gas_limit; + let block_gas_limit: u64 = block_env.gas_limit(); - let mut best_txs = - self.pool().best_transactions_with_attributes(BestTransactionsAttributes::new( - block_env.basefee, - block_env.blob_gasprice().map(|gasprice| gasprice as u64), - )); + // Only include transactions if not configured as Empty + if !self.pending_block_kind().is_empty() { + let mut best_txs = self + .pool() + .best_transactions_with_attributes(BestTransactionsAttributes::new( + block_env.basefee(), + block_env.blob_gasprice().map(|gasprice| gasprice as u64), + )) + // freeze to get a block as fast as possible + .without_updates(); - while let Some(pool_tx) = best_txs.next() { - // ensure we still have capacity for this transaction - if cumulative_gas_used + pool_tx.gas_limit() > block_gas_limit { - // we can't fit this transaction into the block, so we need to mark it as invalid - // which also removes all dependent transaction from the iterator before we can - // continue - best_txs.mark_invalid( - &pool_tx, - InvalidPoolTransactionError::ExceedsGasLimit( - pool_tx.gas_limit(), - block_gas_limit, - ), - ); - continue - } + while let Some(pool_tx) = best_txs.next() { + // ensure we still have capacity for this transaction + if cumulative_gas_used + pool_tx.gas_limit() > block_gas_limit { + // we can't fit this transaction into the block, so we need to mark it as + // invalid which also removes all dependent transaction from + // the iterator before we can continue + best_txs.mark_invalid( + &pool_tx, + InvalidPoolTransactionError::ExceedsGasLimit( + pool_tx.gas_limit(), + block_gas_limit, + ), + ); + continue + } - if pool_tx.origin.is_private() { - // we don't want to leak any state changes made by private transactions, so we mark - // them as invalid here which removes all dependent transactions from the iterator - // before we can continue - best_txs.mark_invalid( - &pool_tx, - InvalidPoolTransactionError::Consensus( - InvalidTransactionError::TxTypeNotSupported, - ), - ); - continue - } + if pool_tx.origin.is_private() { + // we don't want to leak any state changes made by private transactions, so we + // mark them as invalid here which removes all dependent + // transactions from the iteratorbefore we can continue + best_txs.mark_invalid( + &pool_tx, + InvalidPoolTransactionError::Consensus( + InvalidTransactionError::TxTypeNotSupported, + ), + ); + continue + } - // convert tx to a signed transaction - let tx = pool_tx.to_consensus(); + // convert tx to a signed transaction + let tx = pool_tx.to_consensus(); - // There's only limited amount of blob space available per block, so we need to check if - // the EIP-4844 can still fit in the block - if let Some(tx_blob_gas) = tx.blob_gas_used() { - if sum_blob_gas_used + tx_blob_gas > blob_params.max_blob_gas_per_block() { + // There's only limited amount of blob space available per block, so we need to + // check if the EIP-4844 can still fit in the block + if let Some(tx_blob_gas) = tx.blob_gas_used() && + sum_blob_gas_used + tx_blob_gas > blob_params.max_blob_gas_per_block() + { // we can't fit this _blob_ transaction into the block, so we mark it as // invalid, which removes its dependent transactions from // the iterator. This is similar to the gas limit condition @@ -289,49 +318,106 @@ pub trait LoadPendingBlock: ); continue } - } - let gas_used = match builder.execute_transaction(tx.clone()) { - Ok(gas_used) => gas_used, - Err(BlockExecutionError::Validation(BlockValidationError::InvalidTx { - error, - .. - })) => { - if error.is_nonce_too_low() { - // if the nonce is too low, we can skip this transaction - } else { - // if the transaction is invalid, we can skip it and all of its - // descendants - best_txs.mark_invalid( - &pool_tx, - InvalidPoolTransactionError::Consensus( - InvalidTransactionError::TxTypeNotSupported, - ), - ); + let gas_used = match builder.execute_transaction(tx.clone()) { + Ok(gas_used) => gas_used, + Err(BlockExecutionError::Validation(BlockValidationError::InvalidTx { + error, + .. + })) => { + if error.is_nonce_too_low() { + // if the nonce is too low, we can skip this transaction + } else { + // if the transaction is invalid, we can skip it and all of its + // descendants + best_txs.mark_invalid( + &pool_tx, + InvalidPoolTransactionError::Consensus( + InvalidTransactionError::TxTypeNotSupported, + ), + ); + } + continue } - continue - } - // this is an error that we should treat as fatal for this attempt - Err(err) => return Err(Self::Error::from_eth_err(err)), - }; + // this is an error that we should treat as fatal for this attempt + Err(err) => return Err(Self::Error::from_eth_err(err)), + }; - // add to the total blob gas used if the transaction successfully executed - if let Some(tx_blob_gas) = tx.blob_gas_used() { - sum_blob_gas_used += tx_blob_gas; + // add to the total blob gas used if the transaction successfully executed + if let Some(tx_blob_gas) = tx.blob_gas_used() { + sum_blob_gas_used += tx_blob_gas; - // if we've reached the max data gas per block, we can skip blob txs entirely - if sum_blob_gas_used == blob_params.max_blob_gas_per_block() { - best_txs.skip_blobs(); + // if we've reached the max data gas per block, we can skip blob txs entirely + if sum_blob_gas_used == blob_params.max_blob_gas_per_block() { + best_txs.skip_blobs(); + } } - } - // add gas used by the transaction to cumulative gas used, before creating the receipt - cumulative_gas_used += gas_used; + // add gas used by the transaction to cumulative gas used, before creating the + // receipt + cumulative_gas_used += gas_used; + } } - let BlockBuilderOutcome { execution_result, block, .. } = - builder.finish(&state_provider).map_err(Self::Error::from_eth_err)?; + let BlockBuilderOutcome { execution_result, block, hashed_state, trie_updates } = + builder.finish(NoopProvider::default()).map_err(Self::Error::from_eth_err)?; + + let execution_outcome = ExecutionOutcome::new( + db.take_bundle(), + vec![execution_result.receipts], + block.number(), + vec![execution_result.requests], + ); + + Ok(ExecutedBlock { + recovered_block: block.into(), + execution_output: Arc::new(execution_outcome), + hashed_state: Arc::new(hashed_state), + trie_updates: Arc::new(trie_updates), + }) + } +} + +/// A type that knows how to build a [`ConfigureEvm::NextBlockEnvCtx`] for a pending block. +pub trait PendingEnvBuilder: Send + Sync + Unpin + 'static { + /// Builds a [`ConfigureEvm::NextBlockEnvCtx`] for pending block. + fn pending_env_attributes( + &self, + parent: &SealedHeader>, + ) -> Result; +} + +/// Trait that should be implemented on [`ConfigureEvm::NextBlockEnvCtx`] to provide a way for it to +/// build an environment for pending block. +/// +/// This assumes that next environment building doesn't require any additional context, for more +/// complex implementations one should implement [`PendingEnvBuilder`] on their custom type. +pub trait BuildPendingEnv
{ + /// Builds a [`ConfigureEvm::NextBlockEnvCtx`] for pending block. + fn build_pending_env(parent: &SealedHeader
) -> Self; +} + +impl PendingEnvBuilder for () +where + Evm: ConfigureEvm>>, +{ + fn pending_env_attributes( + &self, + parent: &SealedHeader>, + ) -> Result { + Ok(Evm::NextBlockEnvCtx::build_pending_env(parent)) + } +} - Ok((block, execution_result.receipts)) +impl BuildPendingEnv for NextBlockEnvAttributes { + fn build_pending_env(parent: &SealedHeader) -> Self { + Self { + timestamp: parent.timestamp().saturating_add(12), + suggested_fee_recipient: parent.beneficiary(), + prev_randao: B256::random(), + gas_limit: parent.gas_limit(), + parent_beacon_block_root: parent.parent_beacon_block_root().map(|_| B256::ZERO), + withdrawals: parent.withdrawals_root().map(|_| Default::default()), + } } } diff --git a/crates/rpc/rpc-eth-api/src/helpers/receipt.rs b/crates/rpc/rpc-eth-api/src/helpers/receipt.rs index b211676f41b..12215fbff1e 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/receipt.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/receipt.rs @@ -1,17 +1,30 @@ //! Loads a receipt from database. Helper trait for `eth_` block and transaction RPC methods, that //! loads receipt data w.r.t. network. -use alloy_consensus::transaction::TransactionMeta; -use futures::Future; -use reth_provider::{ProviderReceipt, ProviderTx, ReceiptProvider, TransactionsProvider}; - use crate::{EthApiTypes, RpcNodeCoreExt, RpcReceipt}; +use alloy_consensus::{transaction::TransactionMeta, TxReceipt}; +use futures::Future; +use reth_primitives_traits::SignerRecoverable; +use reth_rpc_convert::{transaction::ConvertReceiptInput, RpcConvert}; +use reth_rpc_eth_types::{ + error::FromEthApiError, utils::calculate_gas_used_and_next_log_index, EthApiError, +}; +use reth_storage_api::{ProviderReceipt, ProviderTx}; /// Assembles transaction receipt data w.r.t to network. /// /// Behaviour shared by several `eth_` RPC methods, not exclusive to `eth_` receipts RPC methods. pub trait LoadReceipt: - EthApiTypes + RpcNodeCoreExt + Send + Sync + EthApiTypes< + RpcConvert: RpcConvert< + Primitives = Self::Primitives, + Error = Self::Error, + Network = Self::NetworkTypes, + >, + Error: FromEthApiError, + > + RpcNodeCoreExt + + Send + + Sync { /// Helper method for `eth_getBlockReceipts` and `eth_getTransactionReceipt`. fn build_transaction_receipt( @@ -19,5 +32,34 @@ pub trait LoadReceipt: tx: ProviderTx, meta: TransactionMeta, receipt: ProviderReceipt, - ) -> impl Future, Self::Error>> + Send; + ) -> impl Future, Self::Error>> + Send { + async move { + let hash = meta.block_hash; + // get all receipts for the block + let all_receipts = self + .cache() + .get_receipts(hash) + .await + .map_err(Self::Error::from_eth_err)? + .ok_or(EthApiError::HeaderNotFound(hash.into()))?; + + let (gas_used, next_log_index) = + calculate_gas_used_and_next_log_index(meta.index, &all_receipts); + + Ok(self + .tx_resp_builder() + .convert_receipts(vec![ConvertReceiptInput { + tx: tx + .try_into_recovered_unchecked() + .map_err(Self::Error::from_eth_err)? + .as_recovered_ref(), + gas_used: receipt.cumulative_gas_used() - gas_used, + receipt, + next_log_index, + meta, + }])? + .pop() + .unwrap()) + } + } } diff --git a/crates/rpc/rpc-eth-api/src/helpers/signer.rs b/crates/rpc/rpc-eth-api/src/helpers/signer.rs index 62f8b75b869..c54c8943c0a 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/signer.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/signer.rs @@ -12,7 +12,7 @@ pub type Result = result::Result; /// An Ethereum Signer used via RPC. #[async_trait::async_trait] -pub trait EthSigner: Send + Sync + DynClone { +pub trait EthSigner: Send + Sync + DynClone { /// Returns the available accounts for this signer. fn accounts(&self) -> Vec
; @@ -25,18 +25,10 @@ pub trait EthSigner: Send + Sync + DynClone { async fn sign(&self, address: Address, message: &[u8]) -> Result; /// signs a transaction request using the given account in request - async fn sign_transaction(&self, request: TransactionRequest, address: &Address) -> Result; + async fn sign_transaction(&self, request: TxReq, address: &Address) -> Result; /// Encodes and signs the typed data according EIP-712. Payload must implement Eip712 trait. fn sign_typed_data(&self, address: Address, payload: &TypedData) -> Result; } dyn_clone::clone_trait_object!( EthSigner); - -/// Adds 20 random dev signers for access via the API. Used in dev mode. -#[auto_impl::auto_impl(&)] -pub trait AddDevSigners { - /// Generates 20 random developer accounts. - /// Used in DEV mode. - fn with_dev_accounts(&self); -} diff --git a/crates/rpc/rpc-eth-api/src/helpers/spec.rs b/crates/rpc/rpc-eth-api/src/helpers/spec.rs index 13ad9b778b2..39c9f67cc69 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/spec.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/spec.rs @@ -1,36 +1,24 @@ //! Loads chain metadata. -use alloy_primitives::{Address, U256, U64}; +use alloy_primitives::{U256, U64}; use alloy_rpc_types_eth::{Stage, SyncInfo, SyncStatus}; use futures::Future; -use reth_chainspec::{ChainInfo, EthereumHardforks}; +use reth_chainspec::ChainInfo; use reth_errors::{RethError, RethResult}; use reth_network_api::NetworkInfo; -use reth_provider::{BlockNumReader, ChainSpecProvider, StageCheckpointReader}; +use reth_rpc_convert::RpcTxReq; +use reth_storage_api::{BlockNumReader, StageCheckpointReader, TransactionsProvider}; -use crate::{helpers::EthSigner, RpcNodeCore}; +use crate::{helpers::EthSigner, EthApiTypes, RpcNodeCore}; /// `Eth` API trait. /// /// Defines core functionality of the `eth` API implementation. #[auto_impl::auto_impl(&, Arc)] -pub trait EthApiSpec: - RpcNodeCore< - Provider: ChainSpecProvider - + BlockNumReader - + StageCheckpointReader, - Network: NetworkInfo, -> -{ - /// The transaction type signers are using. - type Transaction; - +pub trait EthApiSpec: RpcNodeCore + EthApiTypes { /// Returns the block node is started on. fn starting_block(&self) -> U256; - /// Returns a handle to the signers owned by provider. - fn signers(&self) -> &parking_lot::RwLock>>>; - /// Returns the current ethereum protocol version. fn protocol_version(&self) -> impl Future> + Send { async move { @@ -49,11 +37,6 @@ pub trait EthApiSpec: Ok(self.provider().chain_info()?) } - /// Returns a list of addresses owned by provider. - fn accounts(&self) -> Vec
{ - self.signers().read().iter().flat_map(|s| s.accounts()).collect() - } - /// Returns `true` if the network is undergoing sync. fn is_syncing(&self) -> bool { self.network().is_syncing() @@ -88,3 +71,9 @@ pub trait EthApiSpec: Ok(status) } } + +/// A handle to [`EthSigner`]s with its generics set from [`TransactionsProvider`] and +/// [`reth_rpc_convert::RpcTypes`]. +pub type SignersForRpc = parking_lot::RwLock< + Vec::Transaction, RpcTxReq>>>, +>; diff --git a/crates/rpc/rpc-eth-api/src/helpers/state.rs b/crates/rpc/rpc-eth-api/src/helpers/state.rs index c0f8bce86c2..1b3dbfcdee6 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/state.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/state.rs @@ -1,5 +1,6 @@ //! Loads a pending block from database. Helper trait for `eth_` block, transaction, call and trace //! RPC methods. + use super::{EthApiSpec, LoadPendingBlock, SpawnBlocking}; use crate::{EthApiTypes, FromEthApiError, RpcNodeCore, RpcNodeCoreExt}; use alloy_consensus::constants::KECCAK_EMPTY; @@ -8,14 +9,15 @@ use alloy_primitives::{Address, Bytes, B256, U256}; use alloy_rpc_types_eth::{Account, AccountInfo, EIP1186AccountProofResponse}; use alloy_serde::JsonStorageKey; use futures::Future; -use reth_chainspec::{EthChainSpec, EthereumHardforks}; use reth_errors::RethError; use reth_evm::{ConfigureEvm, EvmEnvFor}; -use reth_provider::{ - BlockIdReader, BlockNumReader, ChainSpecProvider, StateProvider, StateProviderBox, - StateProviderFactory, +use reth_rpc_convert::RpcConvert; +use reth_rpc_eth_types::{ + error::FromEvmError, EthApiError, PendingBlockEnv, RpcInvalidTransactionError, +}; +use reth_storage_api::{ + BlockIdReader, BlockNumReader, StateProvider, StateProviderBox, StateProviderFactory, }; -use reth_rpc_eth_types::{EthApiError, PendingBlockEnv, RpcInvalidTransactionError}; use reth_transaction_pool::TransactionPool; /// Helper methods for `eth_` methods relating to state (accounts). @@ -50,9 +52,10 @@ pub trait EthState: LoadState + SpawnBlocking { address: Address, block_id: Option, ) -> impl Future> + Send { - self.spawn_blocking_io(move |this| { + self.spawn_blocking_io_fut(move |this| async move { Ok(this - .state_at_block_id_or_latest(block_id)? + .state_at_block_id_or_latest(block_id) + .await? .account_balance(&address) .map_err(Self::Error::from_eth_err)? .unwrap_or_default()) @@ -66,9 +69,10 @@ pub trait EthState: LoadState + SpawnBlocking { index: JsonStorageKey, block_id: Option, ) -> impl Future> + Send { - self.spawn_blocking_io(move |this| { + self.spawn_blocking_io_fut(move |this| async move { Ok(B256::new( - this.state_at_block_id_or_latest(block_id)? + this.state_at_block_id_or_latest(block_id) + .await? .storage(address, index.as_b256()) .map_err(Self::Error::from_eth_err)? .unwrap_or_default() @@ -111,8 +115,8 @@ pub trait EthState: LoadState + SpawnBlocking { return Err(EthApiError::ExceedsMaxProofWindow.into()) } - self.spawn_blocking_io(move |this| { - let state = this.state_at_block_id(block_id)?; + self.spawn_blocking_io_fut(move |this| async move { + let state = this.state_at_block_id(block_id).await?; let storage_keys = keys.iter().map(|key| key.as_b256()).collect::>(); let proof = state .proof(Default::default(), address, &storage_keys) @@ -129,8 +133,8 @@ pub trait EthState: LoadState + SpawnBlocking { address: Address, block_id: BlockId, ) -> impl Future, Self::Error>> + Send { - self.spawn_blocking_io(move |this| { - let state = this.state_at_block_id(block_id)?; + self.spawn_blocking_io_fut(move |this| async move { + let state = this.state_at_block_id(block_id).await?; let account = state.basic_account(&address).map_err(Self::Error::from_eth_err)?; let Some(account) = account else { return Ok(None) }; @@ -166,8 +170,8 @@ pub trait EthState: LoadState + SpawnBlocking { address: Address, block_id: BlockId, ) -> impl Future> + Send { - self.spawn_blocking_io(move |this| { - let state = this.state_at_block_id(block_id)?; + self.spawn_blocking_io_fut(move |this| async move { + let state = this.state_at_block_id(block_id).await?; let account = state .basic_account(&address) .map_err(Self::Error::from_eth_err)? @@ -194,12 +198,11 @@ pub trait EthState: LoadState + SpawnBlocking { /// /// Behaviour shared by several `eth_` RPC methods, not exclusive to `eth_` state RPC methods. pub trait LoadState: - EthApiTypes - + RpcNodeCoreExt< - Provider: StateProviderFactory - + ChainSpecProvider, - Pool: TransactionPool, - > + LoadPendingBlock + + EthApiTypes< + Error: FromEvmError + FromEthApiError, + RpcConvert: RpcConvert, + > + RpcNodeCoreExt { /// Returns the state at the given block number fn state_at_hash(&self, block_hash: B256) -> Result { @@ -210,8 +213,22 @@ pub trait LoadState: /// /// Note: if not [`BlockNumberOrTag::Pending`](alloy_eips::BlockNumberOrTag) then this /// will only return canonical state. See also - fn state_at_block_id(&self, at: BlockId) -> Result { - self.provider().state_by_block_id(at).map_err(Self::Error::from_eth_err) + fn state_at_block_id( + &self, + at: BlockId, + ) -> impl Future> + Send + where + Self: SpawnBlocking, + { + async move { + if at.is_pending() && + let Ok(Some(state)) = self.local_pending_state().await + { + return Ok(state) + } + + self.provider().state_by_block_id(at).map_err(Self::Error::from_eth_err) + } } /// Returns the _latest_ state @@ -225,11 +242,16 @@ pub trait LoadState: fn state_at_block_id_or_latest( &self, block_id: Option, - ) -> Result { - if let Some(block_id) = block_id { - self.state_at_block_id(block_id) - } else { - Ok(self.latest_state()?) + ) -> impl Future> + Send + where + Self: SpawnBlocking, + { + async move { + if let Some(block_id) = block_id { + self.state_at_block_id(block_id).await + } else { + Ok(self.latest_state()?) + } } } @@ -244,7 +266,7 @@ pub trait LoadState: at: BlockId, ) -> impl Future, BlockId), Self::Error>> + Send where - Self: LoadPendingBlock + SpawnBlocking, + Self: SpawnBlocking, { async move { if at.is_pending() { @@ -259,7 +281,11 @@ pub trait LoadState: let header = self.cache().get_header(block_hash).await.map_err(Self::Error::from_eth_err)?; - let evm_env = self.evm_config().evm_env(&header); + let evm_env = self + .evm_config() + .evm_env(&header) + .map_err(RethError::other) + .map_err(Self::Error::from_eth_err)?; Ok((evm_env, block_hash.into())) } @@ -278,17 +304,15 @@ pub trait LoadState: { self.spawn_blocking_io(move |this| { // first fetch the on chain nonce of the account - let on_chain_account_nonce = this + let mut next_nonce = this .latest_state()? .account_nonce(&address) .map_err(Self::Error::from_eth_err)? .unwrap_or_default(); - let mut next_nonce = on_chain_account_nonce; // Retrieve the highest consecutive transaction for the sender from the transaction pool - if let Some(highest_tx) = this - .pool() - .get_highest_consecutive_transaction_by_sender(address, on_chain_account_nonce) + if let Some(highest_tx) = + this.pool().get_highest_consecutive_transaction_by_sender(address, next_nonce) { // Return the nonce of the highest consecutive transaction + 1 next_nonce = highest_tx.nonce().checked_add(1).ok_or_else(|| { @@ -314,18 +338,20 @@ pub trait LoadState: where Self: SpawnBlocking, { - self.spawn_blocking_io(move |this| { + self.spawn_blocking_io_fut(move |this| async move { // first fetch the on chain nonce of the account let on_chain_account_nonce = this - .state_at_block_id_or_latest(block_id)? + .state_at_block_id_or_latest(block_id) + .await? .account_nonce(&address) .map_err(Self::Error::from_eth_err)? .unwrap_or_default(); if block_id == Some(BlockId::pending()) { - // for pending tag we need to find the highest nonce in the pool - if let Some(highest_pool_tx) = - this.pool().get_highest_transaction_by_sender(address) + // for pending tag we need to find the highest nonce of txn in the pending state. + if let Some(highest_pool_tx) = this + .pool() + .get_highest_consecutive_transaction_by_sender(address, on_chain_account_nonce) { { // and the corresponding txcount is nonce + 1 of the highest tx in the pool @@ -358,9 +384,10 @@ pub trait LoadState: where Self: SpawnBlocking, { - self.spawn_blocking_io(move |this| { + self.spawn_blocking_io_fut(move |this| async move { Ok(this - .state_at_block_id_or_latest(block_id)? + .state_at_block_id_or_latest(block_id) + .await? .account_code(&address) .map_err(Self::Error::from_eth_err)? .unwrap_or_default() diff --git a/crates/rpc/rpc-eth-api/src/helpers/trace.rs b/crates/rpc/rpc-eth-api/src/helpers/trace.rs index a7e3840a18b..30ba12165ea 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/trace.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/trace.rs @@ -2,65 +2,44 @@ use super::{Call, LoadBlock, LoadPendingBlock, LoadState, LoadTransaction}; use crate::FromEvmError; -use alloy_consensus::BlockHeader; +use alloy_consensus::{transaction::TxHashRef, BlockHeader}; use alloy_primitives::B256; use alloy_rpc_types_eth::{BlockId, TransactionInfo}; use futures::Future; use reth_chainspec::ChainSpecProvider; use reth_errors::ProviderError; use reth_evm::{ - system_calls::SystemCaller, ConfigureEvm, Database, Evm, EvmEnvFor, HaltReasonFor, - InspectorFor, TxEnvFor, + evm::EvmFactoryExt, system_calls::SystemCaller, tracing::TracingCtx, ConfigureEvm, Database, + Evm, EvmEnvFor, EvmFor, HaltReasonFor, InspectorFor, TxEnvFor, }; -use reth_node_api::NodePrimitives; -use reth_primitives_traits::{BlockBody, RecoveredBlock, SignedTransaction}; -use reth_provider::{BlockReader, ProviderBlock, ProviderHeader, ProviderTx}; -use reth_revm::{database::StateProviderDatabase, db::CacheDB}; +use reth_primitives_traits::{BlockBody, Recovered, RecoveredBlock}; +use reth_revm::{database::StateProviderDatabase, db::State}; use reth_rpc_eth_types::{ cache::db::{StateCacheDb, StateCacheDbRefMutWrapper, StateProviderTraitObjWrapper}, EthApiError, }; -use revm::{ - context_interface::result::{ExecutionResult, ResultAndState}, - state::EvmState, - DatabaseCommit, -}; +use reth_storage_api::{ProviderBlock, ProviderTx}; +use revm::{context::Block, context_interface::result::ResultAndState, DatabaseCommit}; use revm_inspectors::tracing::{TracingInspector, TracingInspectorConfig}; use std::sync::Arc; /// Executes CPU heavy tasks. -pub trait Trace: - LoadState< - Provider: BlockReader, - Evm: ConfigureEvm< - Primitives: NodePrimitives< - BlockHeader = ProviderHeader, - SignedTx = ProviderTx, - >, - >, - Error: FromEvmError, -> -{ - /// Executes the [`reth_evm::EvmEnv`] against the given [Database] without committing state - /// changes. - #[expect(clippy::type_complexity)] +pub trait Trace: LoadState> { + /// Executes the [`TxEnvFor`] with [`reth_evm::EvmEnv`] against the given [Database] without + /// committing state changes. fn inspect( &self, db: DB, evm_env: EvmEnvFor, tx_env: TxEnvFor, inspector: I, - ) -> Result< - (ResultAndState>, (EvmEnvFor, TxEnvFor)), - Self::Error, - > + ) -> Result>, Self::Error> where DB: Database, I: InspectorFor, { - let mut evm = self.evm_config().evm_with_env_and_inspector(db, evm_env.clone(), inspector); - let res = evm.transact(tx_env.clone()).map_err(Self::Error::from_evm_err)?; - Ok((res, (evm_env, tx_env))) + let mut evm = self.evm_config().evm_with_env_and_inspector(db, evm_env, inspector); + evm.transact(tx_env).map_err(Self::Error::from_evm_err) } /// Executes the transaction on top of the given [`BlockId`] with a tracer configured by the @@ -77,18 +56,21 @@ pub trait Trace: config: TracingInspectorConfig, at: BlockId, f: F, - ) -> Result + ) -> impl Future> + Send where Self: Call, + R: Send + 'static, F: FnOnce( - TracingInspector, - ResultAndState>, - ) -> Result, + TracingInspector, + ResultAndState>, + ) -> Result + + Send + + 'static, { - self.with_state_at_block(at, |state| { - let mut db = CacheDB::new(StateProviderDatabase::new(state)); + self.with_state_at_block(at, move |this, state| { + let mut db = State::builder().with_database(StateProviderDatabase::new(state)).build(); let mut inspector = TracingInspector::new(config); - let (res, _) = self.inspect(&mut db, evm_env, tx_env, &mut inspector)?; + let res = this.inspect(&mut db, evm_env, tx_env, &mut inspector)?; f(inspector, res) }) } @@ -121,9 +103,9 @@ pub trait Trace: { let this = self.clone(); self.spawn_with_state_at_block(at, move |state| { - let mut db = CacheDB::new(StateProviderDatabase::new(state)); + let mut db = State::builder().with_database(StateProviderDatabase::new(state)).build(); let mut inspector = TracingInspector::new(config); - let (res, _) = this.inspect(&mut db, evm_env, tx_env, &mut inspector)?; + let res = this.inspect(&mut db, evm_env, tx_env, &mut inspector)?; f(inspector, res, db) }) } @@ -202,7 +184,8 @@ pub trait Trace: let this = self.clone(); self.spawn_with_state_at_block(parent_block.into(), move |state| { - let mut db = CacheDB::new(StateProviderDatabase::new(state)); + let mut db = + State::builder().with_database(StateProviderDatabase::new(state)).build(); let block_txs = block.transactions_recovered(); this.apply_pre_execution_changes(&block, &mut db, &evm_env)?; @@ -211,7 +194,7 @@ pub trait Trace: this.replay_transactions_until(&mut db, evm_env.clone(), block_txs, *tx.tx_hash())?; let tx_env = this.evm_config().tx_env(tx); - let (res, _) = this.inspect( + let res = this.inspect( StateCacheDbRefMutWrapper(&mut db), evm_env, tx_env, @@ -242,10 +225,11 @@ pub trait Trace: Self: LoadBlock, F: Fn( TransactionInfo, - TracingInspector, - ExecutionResult>, - &EvmState, - &StateCacheDb<'_>, + TracingCtx< + '_, + Recovered<&ProviderTx>, + EvmFor, TracingInspector>, + >, ) -> Result + Send + 'static, @@ -282,15 +266,16 @@ pub trait Trace: Self: LoadBlock, F: Fn( TransactionInfo, - Insp, - ExecutionResult>, - &EvmState, - &StateCacheDb<'_>, + TracingCtx< + '_, + Recovered<&ProviderTx>, + EvmFor, Insp>, + >, ) -> Result + Send + 'static, Setup: FnMut() -> Insp + Send + 'static, - Insp: for<'a, 'b> InspectorFor>, + Insp: Clone + for<'a, 'b> InspectorFor>, R: Send + 'static, { async move { @@ -311,66 +296,52 @@ pub trait Trace: } // replay all transactions of the block - self.spawn_tracing(move |this| { + self.spawn_blocking_io_fut(move |this| async move { // we need to get the state of the parent block because we're replaying this block // on top of its parent block's state let state_at = block.parent_hash(); let block_hash = block.hash(); - let block_number = evm_env.block_env.number; - let base_fee = evm_env.block_env.basefee; + let block_number = evm_env.block_env.number().saturating_to(); + let base_fee = evm_env.block_env.basefee(); // now get the state - let state = this.state_at_block_id(state_at.into())?; - let mut db = - CacheDB::new(StateProviderDatabase::new(StateProviderTraitObjWrapper(&state))); + let state = this.state_at_block_id(state_at.into()).await?; + let mut db = State::builder() + .with_database(StateProviderDatabase::new(StateProviderTraitObjWrapper(&state))) + .build(); this.apply_pre_execution_changes(&block, &mut db, &evm_env)?; // prepare transactions, we do everything upfront to reduce time spent with open // state - let max_transactions = - highest_index.map_or(block.body().transaction_count(), |highest| { + let max_transactions = highest_index.map_or_else( + || block.body().transaction_count(), + |highest| { // we need + 1 because the index is 0-based highest as usize + 1 - }); - let mut results = Vec::with_capacity(max_transactions); + }, + ); + + let mut idx = 0; - let mut transactions = block - .transactions_recovered() - .take(max_transactions) - .enumerate() - .map(|(idx, tx)| { + let results = this + .evm_config() + .evm_factory() + .create_tracer(StateCacheDbRefMutWrapper(&mut db), evm_env, inspector_setup()) + .try_trace_many(block.transactions_recovered().take(max_transactions), |ctx| { let tx_info = TransactionInfo { - hash: Some(*tx.tx_hash()), - index: Some(idx as u64), + hash: Some(*ctx.tx.tx_hash()), + index: Some(idx), block_hash: Some(block_hash), block_number: Some(block_number), base_fee: Some(base_fee), }; - let tx_env = this.evm_config().tx_env(tx); - (tx_info, tx_env) - }) - .peekable(); - - while let Some((tx_info, tx)) = transactions.next() { - let mut inspector = inspector_setup(); - let (res, _) = this.inspect( - StateCacheDbRefMutWrapper(&mut db), - evm_env.clone(), - tx, - &mut inspector, - )?; - let ResultAndState { result, state } = res; - results.push(f(tx_info, inspector, result, &state, &db)?); + idx += 1; - // need to apply the state changes of this transaction before executing the - // next transaction, but only if there's a next transaction - if transactions.peek().is_some() { - // commit the state changes to the DB - db.commit(state) - } - } + f(tx_info, ctx) + }) + .collect::>()?; Ok(Some(results)) }) @@ -383,7 +354,7 @@ pub trait Trace: /// /// This /// 1. fetches all transactions of the block - /// 2. configures the EVM evn + /// 2. configures the EVM env /// 3. loops over all transactions and executes them /// 4. calls the callback with the transaction info, the execution result, the changed state /// _after_ the transaction [`StateProviderDatabase`] and the database that points to the @@ -401,10 +372,11 @@ pub trait Trace: // state and db F: Fn( TransactionInfo, - TracingInspector, - ExecutionResult>, - &EvmState, - &StateCacheDb<'_>, + TracingCtx< + '_, + Recovered<&ProviderTx>, + EvmFor, TracingInspector>, + >, ) -> Result + Send + 'static, @@ -418,10 +390,10 @@ pub trait Trace: /// /// This /// 1. fetches all transactions of the block - /// 2. configures the EVM evn + /// 2. configures the EVM env /// 3. loops over all transactions and executes them /// 4. calls the callback with the transaction info, the execution result, the changed state - /// _after_ the transaction [`EvmState`] and the database that points to the state right + /// _after_ the transaction `EvmState` and the database that points to the state right /// _before_ the transaction, in other words the state the transaction was executed on: /// `changed_state = tx(cached_state)` /// @@ -440,15 +412,16 @@ pub trait Trace: // state and db F: Fn( TransactionInfo, - Insp, - ExecutionResult>, - &EvmState, - &StateCacheDb<'_>, + TracingCtx< + '_, + Recovered<&ProviderTx>, + EvmFor, Insp>, + >, ) -> Result + Send + 'static, Setup: FnMut() -> Insp + Send + 'static, - Insp: for<'a, 'b> InspectorFor>, + Insp: Clone + for<'a, 'b> InspectorFor>, R: Send + 'static, { self.trace_block_until_with_inspector(block_id, block, None, insp_setup, f) diff --git a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs index 3f4ff966733..8a49208cd8c 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs @@ -1,28 +1,38 @@ //! Database access for `eth_` transaction RPC methods. Loads transaction and receipt data w.r.t. //! network. -use super::{EthApiSpec, EthSigner, LoadBlock, LoadReceipt, LoadState, SpawnBlocking}; +use super::{EthApiSpec, EthSigner, LoadBlock, LoadFee, LoadReceipt, LoadState, SpawnBlocking}; use crate::{ - helpers::estimate::EstimateCall, FromEthApiError, FullEthApiTypes, IntoEthApiError, - RpcNodeCore, RpcNodeCoreExt, RpcReceipt, RpcTransaction, + helpers::{estimate::EstimateCall, spec::SignersForRpc}, + FromEthApiError, FullEthApiTypes, IntoEthApiError, RpcNodeCore, RpcNodeCoreExt, RpcReceipt, + RpcTransaction, +}; +use alloy_consensus::{ + transaction::{SignerRecoverable, TransactionMeta, TxHashRef}, + BlockHeader, Transaction, }; -use alloy_consensus::{transaction::TransactionMeta, BlockHeader, Transaction}; use alloy_dyn_abi::TypedData; use alloy_eips::{eip2718::Encodable2718, BlockId}; -use alloy_network::TransactionBuilder; -use alloy_primitives::{Address, Bytes, TxHash, B256}; -use alloy_rpc_types_eth::{transaction::TransactionRequest, BlockNumberOrTag, TransactionInfo}; -use futures::Future; +use alloy_network::{TransactionBuilder, TransactionBuilder4844}; +use alloy_primitives::{Address, Bytes, TxHash, B256, U256}; +use alloy_rpc_types_eth::{BlockNumberOrTag, TransactionInfo}; +use futures::{Future, StreamExt}; +use reth_chain_state::CanonStateSubscriptions; use reth_node_api::BlockBody; -use reth_primitives_traits::{RecoveredBlock, SignedTransaction}; -use reth_provider::{ +use reth_primitives_traits::{RecoveredBlock, SignedTransaction, TxTy}; +use reth_rpc_convert::{transaction::RpcConvert, RpcTxReq}; +use reth_rpc_eth_types::{ + utils::binary_search, EthApiError, EthApiError::TransactionConfirmationTimeout, + FillTransactionResult, SignError, TransactionSource, +}; +use reth_storage_api::{ BlockNumReader, BlockReaderIdExt, ProviderBlock, ProviderReceipt, ProviderTx, ReceiptProvider, TransactionsProvider, }; -use reth_rpc_eth_types::{utils::binary_search, EthApiError, SignError, TransactionSource}; -use reth_rpc_types_compat::transaction::TransactionCompat; -use reth_transaction_pool::{PoolTransaction, TransactionOrigin, TransactionPool}; -use std::sync::Arc; +use reth_transaction_pool::{ + AddedTransactionOutcome, PoolTransaction, TransactionOrigin, TransactionPool, +}; +use std::{sync::Arc, time::Duration}; /// Transaction related functions for the [`EthApiServer`](crate::EthApiServer) trait in /// the `eth_` namespace. @@ -34,7 +44,7 @@ use std::sync::Arc; /// /// ## Calls /// -/// There are subtle differences between when transacting [`TransactionRequest`]: +/// There are subtle differences between when transacting [`RpcTxReq`]: /// /// The endpoints `eth_call` and `eth_estimateGas` and `eth_createAccessList` should always /// __disable__ the base fee check in the [`CfgEnv`](revm::context::CfgEnv). @@ -50,8 +60,15 @@ pub trait EthTransactions: LoadTransaction { /// Returns a handle for signing data. /// /// Signer access in default (L1) trait method implementations. - #[expect(clippy::type_complexity)] - fn signers(&self) -> &parking_lot::RwLock>>>>; + fn signers(&self) -> &SignersForRpc; + + /// Returns a list of addresses owned by provider. + fn accounts(&self) -> Vec
{ + self.signers().read().iter().flat_map(|s| s.accounts()).collect() + } + + /// Returns the timeout duration for `send_raw_transaction_sync` RPC method. + fn send_raw_transaction_sync_timeout(&self) -> Duration; /// Decodes and recovers the transaction and submits it to the pool. /// @@ -61,6 +78,47 @@ pub trait EthTransactions: LoadTransaction { tx: Bytes, ) -> impl Future> + Send; + /// Decodes and recovers the transaction and submits it to the pool. + /// + /// And awaits the receipt. + fn send_raw_transaction_sync( + &self, + tx: Bytes, + ) -> impl Future, Self::Error>> + Send + where + Self: LoadReceipt + 'static, + { + let this = self.clone(); + let timeout_duration = self.send_raw_transaction_sync_timeout(); + async move { + let mut stream = this.provider().canonical_state_stream(); + let hash = EthTransactions::send_raw_transaction(&this, tx).await?; + tokio::time::timeout(timeout_duration, async { + while let Some(notification) = stream.next().await { + let chain = notification.committed(); + for block in chain.blocks_iter() { + if block.body().contains_transaction(&hash) && + let Some(receipt) = this.transaction_receipt(hash).await? + { + return Ok(receipt); + } + } + } + Err(Self::Error::from_eth_err(TransactionConfirmationTimeout { + hash, + duration: timeout_duration, + })) + }) + .await + .unwrap_or_else(|_elapsed| { + Err(Self::Error::from_eth_err(TransactionConfirmationTimeout { + hash, + duration: timeout_duration, + })) + }) + } + } + /// Returns the transaction by hash. /// /// Checks the pool and state. @@ -175,8 +233,8 @@ pub trait EthTransactions: LoadTransaction { where Self: 'static, { - let provider = self.provider().clone(); - self.spawn_blocking_io(move |_| { + self.spawn_blocking_io(move |this| { + let provider = this.provider(); let (tx, meta) = match provider .transaction_by_hash_with_meta(hash) .map_err(Self::Error::from_eth_err)? @@ -241,19 +299,16 @@ pub trait EthTransactions: LoadTransaction { { async move { // Check the pool first - if include_pending { - if let Some(tx) = + if include_pending && + let Some(tx) = RpcNodeCore::pool(self).get_transaction_by_sender_and_nonce(sender, nonce) - { - let transaction = tx.transaction.clone_into_consensus(); - return Ok(Some(self.tx_resp_builder().fill_pending(transaction)?)); - } + { + let transaction = tx.transaction.clone_into_consensus(); + return Ok(Some(self.tx_resp_builder().fill_pending(transaction)?)); } - // Check if the sender is a contract - if !self.get_code(sender, None).await?.is_empty() { - return Ok(None); - } + // Note: we can't optimize for contracts (account with code) and cannot shortcircuit if + // the address has code, because with 7702 EOAs can also have code let highest = self.transaction_count(sender, None).await?.saturating_to::(); @@ -317,10 +372,10 @@ pub trait EthTransactions: LoadTransaction { Self: LoadBlock, { async move { - if let Some(block) = self.recovered_block(block_id).await? { - if let Some(tx) = block.body().transactions().get(index) { - return Ok(Some(tx.encoded_2718().into())) - } + if let Some(block) = self.recovered_block(block_id).await? && + let Some(tx) = block.body().transactions().get(index) + { + return Ok(Some(tx.encoded_2718().into())) } Ok(None) @@ -331,13 +386,13 @@ pub trait EthTransactions: LoadTransaction { /// Returns the hash of the signed transaction. fn send_transaction( &self, - mut request: TransactionRequest, + mut request: RpcTxReq, ) -> impl Future> + Send where Self: EthApiSpec + LoadBlock + EstimateCall, { async move { - let from = match request.from { + let from = match request.as_ref().from() { Some(from) => from, None => return Err(SignError::NoAccount.into_eth_err()), }; @@ -347,18 +402,18 @@ pub trait EthTransactions: LoadTransaction { } // set nonce if not already set before - if request.nonce.is_none() { + if request.as_ref().nonce().is_none() { let nonce = self.next_available_nonce(from).await?; - request.nonce = Some(nonce); + request.as_mut().set_nonce(nonce); } let chain_id = self.chain_id(); - request.chain_id = Some(chain_id.to()); + request.as_mut().set_chain_id(chain_id.to()); let estimated_gas = self.estimate_gas_at(request.clone(), BlockId::pending(), None).await?; let gas_limit = estimated_gas; - request.set_gas_limit(gas_limit.to()); + request.as_mut().set_gas_limit(gas_limit.to()); let transaction = self.sign_request(&from, request).await?.with_signer(from); @@ -369,7 +424,7 @@ pub trait EthTransactions: LoadTransaction { .map_err(|_| EthApiError::TransactionConversionError)?; // submit the transaction to the pool with a `Local` origin - let hash = self + let AddedTransactionOutcome { hash, .. } = self .pool() .add_transaction(TransactionOrigin::Local, pool_transaction) .await @@ -379,11 +434,80 @@ pub trait EthTransactions: LoadTransaction { } } + /// Fills the defaults on a given unsigned transaction. + fn fill_transaction( + &self, + mut request: RpcTxReq, + ) -> impl Future>, Self::Error>> + Send + where + Self: EthApiSpec + LoadBlock + EstimateCall + LoadFee, + { + async move { + let from = match request.as_ref().from() { + Some(from) => from, + None => return Err(SignError::NoAccount.into_eth_err()), + }; + + if request.as_ref().value().is_none() { + request.as_mut().set_value(U256::ZERO); + } + + if request.as_ref().nonce().is_none() { + let nonce = self.next_available_nonce(from).await?; + request.as_mut().set_nonce(nonce); + } + + let chain_id = self.chain_id(); + request.as_mut().set_chain_id(chain_id.to()); + + if request.as_ref().has_eip4844_fields() && + request.as_ref().max_fee_per_blob_gas().is_none() + { + let blob_fee = self.blob_base_fee().await?; + request.as_mut().set_max_fee_per_blob_gas(blob_fee.to()); + } + + if request.as_ref().blob_sidecar().is_some() && + request.as_ref().blob_versioned_hashes.is_none() + { + request.as_mut().populate_blob_hashes(); + } + + if request.as_ref().gas_limit().is_none() { + let estimated_gas = + self.estimate_gas_at(request.clone(), BlockId::pending(), None).await?; + request.as_mut().set_gas_limit(estimated_gas.to()); + } + + if request.as_ref().gas_price().is_none() { + let tip = if let Some(tip) = request.as_ref().max_priority_fee_per_gas() { + tip + } else { + let tip = self.suggested_priority_fee().await?.to::(); + request.as_mut().set_max_priority_fee_per_gas(tip); + tip + }; + if request.as_ref().max_fee_per_gas().is_none() { + let header = + self.provider().latest_header().map_err(Self::Error::from_eth_err)?; + let base_fee = header.and_then(|h| h.base_fee_per_gas()).unwrap_or_default(); + request.as_mut().set_max_fee_per_gas(base_fee as u128 + tip); + } + } + + let tx = self.tx_resp_builder().build_simulate_v1_transaction(request)?; + + let raw = tx.encoded_2718().into(); + + Ok(FillTransactionResult { raw, tx }) + } + } + /// Signs a transaction, with configured signers. fn sign_request( &self, from: &Address, - txn: TransactionRequest, + txn: RpcTxReq, ) -> impl Future, Self::Error>> + Send { async move { self.find_signer(from)? @@ -414,10 +538,10 @@ pub trait EthTransactions: LoadTransaction { /// Returns the EIP-2718 encoded signed transaction. fn sign_transaction( &self, - request: TransactionRequest, + request: RpcTxReq, ) -> impl Future> + Send { async move { - let from = match request.from { + let from = match request.as_ref().from() { Some(from) => from, None => return Err(SignError::NoAccount.into_eth_err()), }; @@ -441,7 +565,10 @@ pub trait EthTransactions: LoadTransaction { fn find_signer( &self, account: &Address, - ) -> Result> + 'static)>, Self::Error> { + ) -> Result< + Box, RpcTxReq> + 'static>, + Self::Error, + > { self.signers() .read() .iter() @@ -483,7 +610,7 @@ pub trait LoadTransaction: SpawnBlocking + FullEthApiTypes + RpcNodeCoreExt { // part of pending block) and already. We don't need to // check for pre EIP-2 because this transaction could be pre-EIP-2. let transaction = tx - .into_recovered_unchecked() + .try_into_recovered_unchecked() .map_err(|_| EthApiError::InvalidTransactionSignature)?; let tx = TransactionSource::Block { diff --git a/crates/rpc/rpc-eth-api/src/lib.rs b/crates/rpc/rpc-eth-api/src/lib.rs index c6b40c3caec..57b87dd5eb6 100644 --- a/crates/rpc/rpc-eth-api/src/lib.rs +++ b/crates/rpc/rpc-eth-api/src/lib.rs @@ -10,7 +10,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] pub mod bundle; pub mod core; @@ -27,10 +27,10 @@ pub use ext::{L2EthApiExtServer, MantleEthApiExtServer, PreconfTxEvent, PreconfS pub use filter::{EngineEthFilter, EthFilterApiServer, QueryLimits}; pub use node::{RpcNodeCore, RpcNodeCoreExt}; pub use pubsub::EthPubSubApiServer; +pub use reth_rpc_convert::*; pub use reth_rpc_eth_types::error::{ AsEthApiError, FromEthApiError, FromEvmError, IntoEthApiError, }; -pub use reth_rpc_types_compat::TransactionCompat; pub use types::{EthApiTypes, FullEthApiTypes, RpcBlock, RpcHeader, RpcReceipt, RpcTransaction}; #[cfg(feature = "client")] diff --git a/crates/rpc/rpc-eth-api/src/node.rs b/crates/rpc/rpc-eth-api/src/node.rs index 2b3bd7026f0..bde95b9c572 100644 --- a/crates/rpc/rpc-eth-api/src/node.rs +++ b/crates/rpc/rpc-eth-api/src/node.rs @@ -1,30 +1,53 @@ //! Helper trait for interfacing with [`FullNodeComponents`]. -use reth_node_api::{FullNodeComponents, NodeTypes, PrimitivesTy}; -use reth_payload_builder::PayloadBuilderHandle; -use reth_provider::{BlockReader, ProviderBlock, ProviderReceipt}; +use reth_chain_state::CanonStateSubscriptions; +use reth_chainspec::{ChainSpecProvider, EthChainSpec, EthereumHardforks, Hardforks}; +use reth_evm::ConfigureEvm; +use reth_network_api::NetworkInfo; +use reth_node_api::{FullNodeComponents, NodePrimitives, PrimitivesTy}; +use reth_primitives_traits::{BlockTy, HeaderTy, ReceiptTy, TxTy}; use reth_rpc_eth_types::EthStateCache; +use reth_storage_api::{ + BlockReader, BlockReaderIdExt, StageCheckpointReader, StateProviderFactory, +}; +use reth_transaction_pool::{PoolTransaction, TransactionPool}; -/// Helper trait to relax trait bounds on [`FullNodeComponents`]. +/// Helper trait that provides the same interface as [`FullNodeComponents`] but without requiring +/// implementation of trait bounds. /// -/// Helpful when defining types that would otherwise have a generic `N: FullNodeComponents`. Using -/// `N: RpcNodeCore` instead, allows access to all the associated types on [`FullNodeComponents`] -/// that are used in RPC, but with more flexibility since they have no trait bounds (asides auto -/// traits). -pub trait RpcNodeCore: Clone + Send + Sync { +/// This trait is structurally equivalent to [`FullNodeComponents`], exposing the same associated +/// types and methods. However, it doesn't enforce the trait bounds required by +/// [`FullNodeComponents`]. This makes it useful for RPC types that need access to node components +/// where the full trait bounds of the components are not necessary. +/// +/// Every type that is a [`FullNodeComponents`] also implements this trait. +pub trait RpcNodeCore: Clone + Send + Sync + Unpin + 'static { /// Blockchain data primitives. - type Primitives: Send + Sync + Clone + Unpin; + type Primitives: NodePrimitives; /// The provider type used to interact with the node. - type Provider: Send + Sync + Clone + Unpin; + type Provider: BlockReaderIdExt< + Block = BlockTy, + Receipt = ReceiptTy, + Header = HeaderTy, + Transaction = TxTy, + > + ChainSpecProvider< + ChainSpec: EthChainSpec
> + + Hardforks + + EthereumHardforks, + > + StateProviderFactory + + CanonStateSubscriptions + + StageCheckpointReader + + Send + + Sync + + Clone + + Unpin + + 'static; /// The transaction pool of the node. - type Pool: Send + Sync + Clone + Unpin; + type Pool: TransactionPool>>; /// The node's EVM configuration, defining settings for the Ethereum Virtual Machine. - type Evm: Send + Sync + Clone + Unpin; + type Evm: ConfigureEvm + Send + Sync + 'static; /// Network API. - type Network: Send + Sync + Clone; - - /// Builds new blocks. - type PayloadBuilder: Send + Sync + Clone; + type Network: NetworkInfo + Clone; /// Returns the transaction pool of the node. fn pool(&self) -> &Self::Pool; @@ -35,23 +58,19 @@ pub trait RpcNodeCore: Clone + Send + Sync { /// Returns the handle to the network fn network(&self) -> &Self::Network; - /// Returns the handle to the payload builder service. - fn payload_builder(&self) -> &Self::PayloadBuilder; - /// Returns the provider of the node. fn provider(&self) -> &Self::Provider; } impl RpcNodeCore for T where - T: FullNodeComponents, + T: FullNodeComponents>, { type Primitives = PrimitivesTy; type Provider = T::Provider; type Pool = T::Pool; type Evm = T::Evm; type Network = T::Network; - type PayloadBuilder = PayloadBuilderHandle<::Payload>; #[inline] fn pool(&self) -> &Self::Pool { @@ -68,11 +87,6 @@ where FullNodeComponents::network(self) } - #[inline] - fn payload_builder(&self) -> &Self::PayloadBuilder { - FullNodeComponents::payload_builder_handle(self) - } - #[inline] fn provider(&self) -> &Self::Provider { FullNodeComponents::provider(self) @@ -83,7 +97,69 @@ where /// server. pub trait RpcNodeCoreExt: RpcNodeCore { /// Returns handle to RPC cache service. - fn cache( - &self, - ) -> &EthStateCache, ProviderReceipt>; + fn cache(&self) -> &EthStateCache; +} + +/// An adapter that allows to construct [`RpcNodeCore`] from components. +#[derive(Debug, Clone)] +pub struct RpcNodeCoreAdapter { + provider: Provider, + pool: Pool, + network: Network, + evm_config: Evm, +} + +impl RpcNodeCoreAdapter { + /// Creates a new `RpcNodeCoreAdapter` instance. + pub const fn new(provider: Provider, pool: Pool, network: Network, evm_config: Evm) -> Self { + Self { provider, pool, network, evm_config } + } +} + +impl RpcNodeCore for RpcNodeCoreAdapter +where + Provider: BlockReaderIdExt< + Block = BlockTy, + Receipt = ReceiptTy, + Header = HeaderTy, + Transaction = TxTy, + > + ChainSpecProvider< + ChainSpec: EthChainSpec
> + + Hardforks + + EthereumHardforks, + > + StateProviderFactory + + CanonStateSubscriptions + + StageCheckpointReader + + Send + + Sync + + Unpin + + Clone + + 'static, + Evm: ConfigureEvm + Clone + 'static, + Pool: TransactionPool>> + + Unpin + + 'static, + Network: NetworkInfo + Clone + Unpin + 'static, +{ + type Primitives = Evm::Primitives; + type Provider = Provider; + type Pool = Pool; + type Evm = Evm; + type Network = Network; + + fn pool(&self) -> &Self::Pool { + &self.pool + } + + fn evm_config(&self) -> &Self::Evm { + &self.evm_config + } + + fn network(&self) -> &Self::Network { + &self.network + } + + fn provider(&self) -> &Self::Provider { + &self.provider + } } diff --git a/crates/rpc/rpc-eth-api/src/types.rs b/crates/rpc/rpc-eth-api/src/types.rs index cdd6d50d2ca..ed4fcfa5c80 100644 --- a/crates/rpc/rpc-eth-api/src/types.rs +++ b/crates/rpc/rpc-eth-api/src/types.rs @@ -1,43 +1,20 @@ //! Trait for specifying `eth` network dependent API types. use crate::{AsEthApiError, FromEthApiError, RpcNodeCore}; -use alloy_json_rpc::RpcObject; -use alloy_network::{Network, ReceiptResponse, TransactionResponse}; use alloy_rpc_types_eth::Block; -use reth_provider::{ProviderTx, ReceiptProvider, TransactionsProvider}; -use reth_rpc_types_compat::TransactionCompat; -use reth_transaction_pool::{PoolTransaction, TransactionPool}; +use reth_rpc_convert::{RpcConvert, SignableTxRequest}; +pub use reth_rpc_convert::{RpcTransaction, RpcTxReq, RpcTypes}; +use reth_storage_api::ProviderTx; use std::{ error::Error, fmt::{self}, }; -/// RPC types used by the `eth_` RPC API. -/// -/// This is a subset of [`alloy_network::Network`] trait with only RPC response types kept. -pub trait RpcTypes { - /// Header response type. - type Header: RpcObject; - /// Receipt response type. - type Receipt: RpcObject + ReceiptResponse; - /// Transaction response type. - type Transaction: RpcObject + TransactionResponse; -} - -impl RpcTypes for T -where - T: Network, -{ - type Header = T::HeaderResponse; - type Receipt = T::ReceiptResponse; - type Transaction = T::TransactionResponse; -} - /// Network specific `eth` API types. /// /// This trait defines the network specific rpc types and helpers required for the `eth_` and -/// adjacent endpoints. `NetworkTypes` is [`Network`] as defined by the alloy crate, see also -/// [`alloy_network::Ethereum`]. +/// adjacent endpoints. `NetworkTypes` is [`alloy_network::Network`] as defined by the alloy crate, +/// see also [`alloy_network::Ethereum`]. /// /// This type is stateful so that it can provide additional context if necessary, e.g. populating /// receipts with additional data. @@ -52,15 +29,12 @@ pub trait EthApiTypes: Send + Sync + Clone { /// Blockchain primitive types, specific to network, e.g. block and transaction. type NetworkTypes: RpcTypes; /// Conversion methods for transaction RPC type. - type TransactionCompat: Send + Sync + Clone + fmt::Debug; + type RpcConvert: Send + Sync + fmt::Debug; /// Returns reference to transaction response builder. - fn tx_resp_builder(&self) -> &Self::TransactionCompat; + fn tx_resp_builder(&self) -> &Self::RpcConvert; } -/// Adapter for network specific transaction type. -pub type RpcTransaction = ::Transaction; - /// Adapter for network specific block type. pub type RpcBlock = Block, RpcHeader>; @@ -76,15 +50,14 @@ pub type RpcError = ::Error; /// Helper trait holds necessary trait bounds on [`EthApiTypes`] to implement `eth` API. pub trait FullEthApiTypes where - Self: RpcNodeCore< - Provider: TransactionsProvider + ReceiptProvider, - Pool: TransactionPool< - Transaction: PoolTransaction>, + Self: RpcNodeCore + + EthApiTypes< + NetworkTypes: RpcTypes< + TransactionRequest: SignableTxRequest>, >, - > + EthApiTypes< - TransactionCompat: TransactionCompat< - ::Transaction, - Transaction = RpcTransaction, + RpcConvert: RpcConvert< + Primitives = Self::Primitives, + Network = Self::NetworkTypes, Error = RpcError, >, >, @@ -92,15 +65,14 @@ where } impl FullEthApiTypes for T where - T: RpcNodeCore< - Provider: TransactionsProvider + ReceiptProvider, - Pool: TransactionPool< - Transaction: PoolTransaction>, + T: RpcNodeCore + + EthApiTypes< + NetworkTypes: RpcTypes< + TransactionRequest: SignableTxRequest>, >, - > + EthApiTypes< - TransactionCompat: TransactionCompat< - ::Transaction, - Transaction = RpcTransaction, + RpcConvert: RpcConvert< + Primitives = ::Primitives, + Network = Self::NetworkTypes, Error = RpcError, >, > diff --git a/crates/rpc/rpc-eth-types/Cargo.toml b/crates/rpc/rpc-eth-types/Cargo.toml index 20254eea731..7eed1aa3db1 100644 --- a/crates/rpc/rpc-eth-types/Cargo.toml +++ b/crates/rpc/rpc-eth-types/Cargo.toml @@ -18,22 +18,26 @@ reth-errors.workspace = true reth-evm.workspace = true reth-execution-types.workspace = true reth-metrics.workspace = true -reth-ethereum-primitives.workspace = true -reth-primitives-traits.workspace = true +reth-ethereum-primitives = { workspace = true, features = ["rpc"] } +reth-primitives-traits = { workspace = true, features = ["rpc-compat"] } reth-storage-api.workspace = true reth-revm.workspace = true reth-rpc-server-types.workspace = true -reth-rpc-types-compat.workspace = true +reth-rpc-convert.workspace = true reth-tasks.workspace = true reth-transaction-pool.workspace = true reth-trie.workspace = true # ethereum alloy-eips.workspace = true +alloy-evm = { workspace = true, features = ["overrides", "call-util"] } alloy-primitives.workspace = true alloy-consensus.workspace = true alloy-sol-types.workspace = true +alloy-transport.workspace = true +alloy-rpc-client = { workspace = true, features = ["reqwest"] } alloy-rpc-types-eth.workspace = true +alloy-network.workspace = true revm.workspace = true revm-inspectors.workspace = true @@ -45,6 +49,7 @@ jsonrpsee-types.workspace = true futures.workspace = true tokio.workspace = true tokio-stream.workspace = true +reqwest = { workspace = true, features = ["rustls-tls-native-roots"] } # metrics metrics.workspace = true diff --git a/crates/rpc/rpc-eth-types/src/block.rs b/crates/rpc/rpc-eth-types/src/block.rs new file mode 100644 index 00000000000..624ce53c26f --- /dev/null +++ b/crates/rpc/rpc-eth-types/src/block.rs @@ -0,0 +1,47 @@ +//! Block related types for RPC API. + +use std::sync::Arc; + +use alloy_primitives::TxHash; +use reth_primitives_traits::{ + BlockTy, IndexedTx, NodePrimitives, ReceiptTy, RecoveredBlock, SealedBlock, +}; + +/// A pair of an [`Arc`] wrapped [`RecoveredBlock`] and its corresponding receipts. +/// +/// This type is used throughout the RPC layer to efficiently pass around +/// blocks with their execution receipts, avoiding unnecessary cloning. +#[derive(Debug, Clone)] +pub struct BlockAndReceipts { + /// The recovered block. + pub block: Arc>>, + /// The receipts for the block. + pub receipts: Arc>>, +} + +impl BlockAndReceipts { + /// Creates a new [`BlockAndReceipts`] instance. + pub const fn new( + block: Arc>>, + receipts: Arc>>, + ) -> Self { + Self { block, receipts } + } + + /// Finds a transaction by hash and returns it along with its corresponding receipt. + /// + /// Returns `None` if the transaction is not found in this block. + pub fn find_transaction_and_receipt_by_hash( + &self, + tx_hash: TxHash, + ) -> Option<(IndexedTx<'_, N::Block>, &N::Receipt)> { + let indexed_tx = self.block.find_indexed(tx_hash)?; + let receipt = self.receipts.get(indexed_tx.index())?; + Some((indexed_tx, receipt)) + } + + /// Returns the underlying sealed block. + pub fn sealed_block(&self) -> &SealedBlock> { + self.block.sealed_block() + } +} diff --git a/crates/rpc/rpc-eth-types/src/builder/config.rs b/crates/rpc/rpc-eth-types/src/builder/config.rs index 6503e83f2e5..ded50ab4a83 100644 --- a/crates/rpc/rpc-eth-types/src/builder/config.rs +++ b/crates/rpc/rpc-eth-types/src/builder/config.rs @@ -3,20 +3,62 @@ use std::time::Duration; use crate::{ - EthStateCacheConfig, FeeHistoryCacheConfig, GasPriceOracleConfig, RPC_DEFAULT_GAS_CAP, + EthStateCacheConfig, FeeHistoryCacheConfig, ForwardConfig, GasPriceOracleConfig, + RPC_DEFAULT_GAS_CAP, }; +use reqwest::Url; use reth_rpc_server_types::constants::{ default_max_tracing_requests, DEFAULT_ETH_PROOF_WINDOW, DEFAULT_MAX_BLOCKS_PER_FILTER, DEFAULT_MAX_LOGS_PER_RESPONSE, DEFAULT_MAX_SIMULATE_BLOCKS, DEFAULT_MAX_TRACE_FILTER_BLOCKS, - DEFAULT_PROOF_PERMITS, + DEFAULT_PROOF_PERMITS, RPC_DEFAULT_SEND_RAW_TX_SYNC_TIMEOUT_SECS, }; use serde::{Deserialize, Serialize}; /// Default value for stale filter ttl pub const DEFAULT_STALE_FILTER_TTL: Duration = Duration::from_secs(5 * 60); +/// Config for the locally built pending block +#[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize, Deserialize, Default)] +#[serde(rename_all = "lowercase")] +pub enum PendingBlockKind { + /// Return a pending block with header only, no transactions included + Empty, + /// Return null/no pending block + None, + /// Return a pending block with all transactions from the mempool (default behavior) + #[default] + Full, +} + +impl std::str::FromStr for PendingBlockKind { + type Err = String; + + fn from_str(s: &str) -> Result { + match s.to_lowercase().as_str() { + "empty" => Ok(Self::Empty), + "none" => Ok(Self::None), + "full" => Ok(Self::Full), + _ => Err(format!( + "Invalid pending block kind: {s}. Valid options are: empty, none, full" + )), + } + } +} + +impl PendingBlockKind { + /// Returns true if the pending block kind is `None` + pub const fn is_none(&self) -> bool { + matches!(self, Self::None) + } + + /// Returns true if the pending block kind is `Empty` + pub const fn is_empty(&self) -> bool { + matches!(self, Self::Empty) + } +} + /// Additional config values for the eth namespace. -#[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] pub struct EthConfig { /// Settings for the caching layer pub cache: EthStateCacheConfig, @@ -45,6 +87,16 @@ pub struct EthConfig { pub fee_history_cache: FeeHistoryCacheConfig, /// The maximum number of getproof calls that can be executed concurrently. pub proof_permits: usize, + /// Maximum batch size for transaction pool insertions. + pub max_batch_size: usize, + /// Controls how pending blocks are built when requested via RPC methods + pub pending_block_kind: PendingBlockKind, + /// The raw transaction forwarder. + pub raw_tx_forwarder: ForwardConfig, + /// Timeout duration for `send_raw_transaction_sync` RPC method. + pub send_raw_transaction_sync_timeout: Duration, + /// Maximum memory the EVM can allocate per RPC request. + pub rpc_evm_memory_limit: u64, } impl EthConfig { @@ -72,6 +124,11 @@ impl Default for EthConfig { stale_filter_ttl: DEFAULT_STALE_FILTER_TTL, fee_history_cache: FeeHistoryCacheConfig::default(), proof_permits: DEFAULT_PROOF_PERMITS, + max_batch_size: 1, + pending_block_kind: PendingBlockKind::Full, + raw_tx_forwarder: ForwardConfig::default(), + send_raw_transaction_sync_timeout: RPC_DEFAULT_SEND_RAW_TX_SYNC_TIMEOUT_SECS, + rpc_evm_memory_limit: (1 << 32) - 1, } } } @@ -136,6 +193,38 @@ impl EthConfig { self.proof_permits = permits; self } + + /// Configures the maximum batch size for transaction pool insertions + pub const fn max_batch_size(mut self, max_batch_size: usize) -> Self { + self.max_batch_size = max_batch_size; + self + } + + /// Configures the pending block config + pub const fn pending_block_kind(mut self, pending_block_kind: PendingBlockKind) -> Self { + self.pending_block_kind = pending_block_kind; + self + } + + /// Configures the raw transaction forwarder. + pub fn raw_tx_forwarder(mut self, tx_forwarder: Option) -> Self { + if let Some(tx_forwarder) = tx_forwarder { + self.raw_tx_forwarder.tx_forwarder = Some(tx_forwarder); + } + self + } + + /// Configures the timeout duration for `send_raw_transaction_sync` RPC method. + pub const fn send_raw_transaction_sync_timeout(mut self, timeout: Duration) -> Self { + self.send_raw_transaction_sync_timeout = timeout; + self + } + + /// Configures the maximum memory the EVM can allocate per RPC request. + pub const fn rpc_evm_memory_limit(mut self, memory_limit: u64) -> Self { + self.rpc_evm_memory_limit = memory_limit; + self + } } /// Config for the filter diff --git a/crates/rpc/rpc-eth-types/src/cache/db.rs b/crates/rpc/rpc-eth-types/src/cache/db.rs index 633a4482e74..8209af0fa53 100644 --- a/crates/rpc/rpc-eth-types/src/cache/db.rs +++ b/crates/rpc/rpc-eth-types/src/cache/db.rs @@ -5,16 +5,17 @@ use alloy_primitives::{Address, B256, U256}; use reth_errors::ProviderResult; use reth_revm::{database::StateProviderDatabase, DatabaseRef}; -use reth_storage_api::{HashedPostStateProvider, StateProvider}; +use reth_storage_api::{BytecodeReader, HashedPostStateProvider, StateProvider}; use reth_trie::{HashedStorage, MultiProofTargets}; use revm::{ - database::{BundleState, CacheDB}, + database::{BundleState, State}, + primitives::HashMap, state::{AccountInfo, Bytecode}, - Database, + Database, DatabaseCommit, }; -/// Helper alias type for the state's [`CacheDB`] -pub type StateCacheDb<'a> = CacheDB>>; +/// Helper alias type for the state's [`State`] +pub type StateCacheDb<'a> = State>>; /// Hack to get around 'higher-ranked lifetime error', see /// @@ -154,13 +155,6 @@ impl StateProvider for StateProviderTraitObjWrapper<'_> { self.0.storage(account, storage_key) } - fn bytecode_by_hash( - &self, - code_hash: &B256, - ) -> reth_errors::ProviderResult> { - self.0.bytecode_by_hash(code_hash) - } - fn account_code( &self, addr: &Address, @@ -177,11 +171,25 @@ impl StateProvider for StateProviderTraitObjWrapper<'_> { } } +impl BytecodeReader for StateProviderTraitObjWrapper<'_> { + fn bytecode_by_hash( + &self, + code_hash: &B256, + ) -> reth_errors::ProviderResult> { + self.0.bytecode_by_hash(code_hash) + } +} + /// Hack to get around 'higher-ranked lifetime error', see /// -#[expect(missing_debug_implementations)] pub struct StateCacheDbRefMutWrapper<'a, 'b>(pub &'b mut StateCacheDb<'a>); +impl<'a, 'b> core::fmt::Debug for StateCacheDbRefMutWrapper<'a, 'b> { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("StateCacheDbRefMutWrapper").finish_non_exhaustive() + } +} + impl<'a> Database for StateCacheDbRefMutWrapper<'a, '_> { type Error = as Database>::Error; fn basic(&mut self, address: Address) -> Result, Self::Error> { @@ -220,3 +228,9 @@ impl<'a> DatabaseRef for StateCacheDbRefMutWrapper<'a, '_> { self.0.block_hash_ref(number) } } + +impl DatabaseCommit for StateCacheDbRefMutWrapper<'_, '_> { + fn commit(&mut self, changes: HashMap) { + self.0.commit(changes) + } +} diff --git a/crates/rpc/rpc-eth-types/src/cache/mod.rs b/crates/rpc/rpc-eth-types/src/cache/mod.rs index fa5594b18d9..e0bea2cf463 100644 --- a/crates/rpc/rpc-eth-types/src/cache/mod.rs +++ b/crates/rpc/rpc-eth-types/src/cache/mod.rs @@ -4,7 +4,7 @@ use super::{EthStateCacheConfig, MultiConsumerLruCache}; use alloy_consensus::BlockHeader; use alloy_eips::BlockHashOrNumber; use alloy_primitives::B256; -use futures::{future::Either, Stream, StreamExt}; +use futures::{future::Either, stream::FuturesOrdered, Stream, StreamExt}; use reth_chain_state::CanonStateNotification; use reth_errors::{ProviderError, ProviderResult}; use reth_execution_types::Chain; @@ -41,6 +41,9 @@ type ReceiptsResponseSender = oneshot::Sender = oneshot::Sender>>>; +type CachedBlockAndReceiptsResponseSender = + oneshot::Sender<(Option>>, Option>>)>; + /// The type that can send the response to a requested header type HeaderResponseSender = oneshot::Sender>; @@ -67,17 +70,17 @@ type HeaderLruCache = MultiConsumerLruCache { - to_service: UnboundedSender>, +pub struct EthStateCache { + to_service: UnboundedSender>, } -impl Clone for EthStateCache { +impl Clone for EthStateCache { fn clone(&self) -> Self { Self { to_service: self.to_service.clone() } } } -impl EthStateCache { +impl EthStateCache { /// Creates and returns both [`EthStateCache`] frontend and the memory bound service. fn create( provider: Provider, @@ -88,7 +91,7 @@ impl EthStateCache { max_concurrent_db_operations: usize, ) -> (Self, EthStateCacheService) where - Provider: BlockReader, + Provider: BlockReader, { let (to_service, rx) = unbounded_channel(); let service = EthStateCacheService { @@ -111,7 +114,7 @@ impl EthStateCache { /// See also [`Self::spawn_with`] pub fn spawn(provider: Provider, config: EthStateCacheConfig) -> Self where - Provider: BlockReader + Clone + Unpin + 'static, + Provider: BlockReader + Clone + Unpin + 'static, { Self::spawn_with(provider, config, TokioTaskExecutor::default()) } @@ -126,7 +129,7 @@ impl EthStateCache { executor: Tasks, ) -> Self where - Provider: BlockReader + Clone + Unpin + 'static, + Provider: BlockReader + Clone + Unpin + 'static, Tasks: TaskSpawner + Clone + 'static, { let EthStateCacheConfig { @@ -153,7 +156,7 @@ impl EthStateCache { pub async fn get_recovered_block( &self, block_hash: B256, - ) -> ProviderResult>>> { + ) -> ProviderResult>>> { let (response_tx, rx) = oneshot::channel(); let _ = self.to_service.send(CacheAction::GetBlockWithSenders { block_hash, response_tx }); rx.await.map_err(|_| CacheServiceUnavailable)? @@ -162,7 +165,10 @@ impl EthStateCache { /// Requests the receipts for the block hash /// /// Returns `None` if the block was not found. - pub async fn get_receipts(&self, block_hash: B256) -> ProviderResult>>> { + pub async fn get_receipts( + &self, + block_hash: B256, + ) -> ProviderResult>>> { let (response_tx, rx) = oneshot::channel(); let _ = self.to_service.send(CacheAction::GetReceipts { block_hash, response_tx }); rx.await.map_err(|_| CacheServiceUnavailable)? @@ -172,7 +178,7 @@ impl EthStateCache { pub async fn get_block_and_receipts( &self, block_hash: B256, - ) -> ProviderResult>, Arc>)>> { + ) -> ProviderResult>, Arc>)>> { let block = self.get_recovered_block(block_hash); let receipts = self.get_receipts(block_hash); @@ -185,7 +191,7 @@ impl EthStateCache { pub async fn get_receipts_and_maybe_block( &self, block_hash: B256, - ) -> ProviderResult>, Option>>)>> { + ) -> ProviderResult>, Option>>)>> { let (response_tx, rx) = oneshot::channel(); let _ = self.to_service.send(CacheAction::GetCachedBlock { block_hash, response_tx }); @@ -197,10 +203,37 @@ impl EthStateCache { Ok(receipts?.map(|r| (r, block))) } + /// Retrieves both block and receipts from cache if available. + pub async fn maybe_cached_block_and_receipts( + &self, + block_hash: B256, + ) -> ProviderResult<(Option>>, Option>>)> { + let (response_tx, rx) = oneshot::channel(); + let _ = self + .to_service + .send(CacheAction::GetCachedBlockAndReceipts { block_hash, response_tx }); + rx.await.map_err(|_| CacheServiceUnavailable.into()) + } + + /// Streams cached receipts and blocks for a list of block hashes, preserving input order. + #[allow(clippy::type_complexity)] + pub fn get_receipts_and_maybe_block_stream<'a>( + &'a self, + hashes: Vec, + ) -> impl Stream< + Item = ProviderResult< + Option<(Arc>, Option>>)>, + >, + > + 'a { + let futures = hashes.into_iter().map(move |hash| self.get_receipts_and_maybe_block(hash)); + + futures.collect::>() + } + /// Requests the header for the given hash. /// /// Returns an error if the header is not found. - pub async fn get_header(&self, block_hash: B256) -> ProviderResult { + pub async fn get_header(&self, block_hash: B256) -> ProviderResult { let (response_tx, rx) = oneshot::channel(); let _ = self.to_service.send(CacheAction::GetHeader { block_hash, response_tx }); rx.await.map_err(|_| CacheServiceUnavailable)? @@ -217,7 +250,7 @@ impl EthStateCache { &self, block_hash: B256, max_blocks: usize, - ) -> Option>>> { + ) -> Option>>> { let (response_tx, rx) = oneshot::channel(); let _ = self.to_service.send(CacheAction::GetCachedParentBlocks { block_hash, @@ -424,6 +457,11 @@ where let _ = response_tx.send(this.full_block_cache.get(&block_hash).cloned()); } + CacheAction::GetCachedBlockAndReceipts { block_hash, response_tx } => { + let block = this.full_block_cache.get(&block_hash).cloned(); + let receipts = this.receipts_cache.get(&block_hash).cloned(); + let _ = response_tx.send((block, receipts)); + } CacheAction::GetBlockWithSenders { block_hash, response_tx } => { if let Some(block) = this.full_block_cache.get(&block_hash).cloned() { let _ = response_tx.send(Ok(Some(block))); @@ -501,7 +539,7 @@ where this.action_task_spawner.spawn_blocking(Box::pin(async move { // Acquire permit let _permit = rate_limiter.acquire().await; - let header = provider.header(&block_hash).and_then(|header| { + let header = provider.header(block_hash).and_then(|header| { header.ok_or_else(|| { ProviderError::HeaderNotFound(block_hash.into()) }) @@ -612,6 +650,10 @@ enum CacheAction { block_hash: B256, response_tx: CachedBlockResponseSender, }, + GetCachedBlockAndReceipts { + block_hash: B256, + response_tx: CachedBlockAndReceiptsResponseSender, + }, BlockWithSendersResult { block_hash: B256, res: ProviderResult>>>, @@ -741,7 +783,7 @@ impl Drop for ActionSender { /// /// Reorged blocks are removed from the cache. pub async fn cache_new_blocks_task( - eth_state_cache: EthStateCache, + eth_state_cache: EthStateCache, mut events: St, ) where St: Stream> + Unpin + 'static, diff --git a/crates/rpc/rpc-eth-types/src/cache/multi_consumer.rs b/crates/rpc/rpc-eth-types/src/cache/multi_consumer.rs index bae39c78f0f..dec5dcb09a0 100644 --- a/crates/rpc/rpc-eth-types/src/cache/multi_consumer.rs +++ b/crates/rpc/rpc-eth-types/src/cache/multi_consumer.rs @@ -100,11 +100,11 @@ where { let size = value.size(); - if self.cache.limiter().is_over_the_limit(self.cache.len() + 1) { - if let Some((_, evicted)) = self.cache.pop_oldest() { - // update tracked memory with the evicted value - self.memory_usage = self.memory_usage.saturating_sub(evicted.size()); - } + if self.cache.limiter().is_over_the_limit(self.cache.len() + 1) && + let Some((_, evicted)) = self.cache.pop_oldest() + { + // update tracked memory with the evicted value + self.memory_usage = self.memory_usage.saturating_sub(evicted.size()); } if self.cache.insert(key, value) { diff --git a/crates/rpc/rpc-eth-types/src/error/mod.rs b/crates/rpc/rpc-eth-types/src/error/mod.rs index 6f586bc1ce8..b8814785478 100644 --- a/crates/rpc/rpc-eth-types/src/error/mod.rs +++ b/crates/rpc/rpc-eth-types/src/error/mod.rs @@ -3,13 +3,16 @@ pub mod api; use crate::error::api::FromEvmHalt; use alloy_eips::BlockId; -use alloy_primitives::{Address, Bytes, U256}; +use alloy_evm::{call::CallError, overrides::StateOverrideError}; +use alloy_primitives::{Address, Bytes, B256, U256}; use alloy_rpc_types_eth::{error::EthRpcErrorCode, request::TransactionInputError, BlockError}; use alloy_sol_types::{ContractError, RevertReason}; +use alloy_transport::{RpcError, TransportErrorKind}; pub use api::{AsEthApiError, FromEthApiError, FromEvmError, IntoEthApiError}; use core::time::Duration; -use reth_errors::{BlockExecutionError, RethError}; +use reth_errors::{BlockExecutionError, BlockValidationError, RethError}; use reth_primitives_traits::transaction::{error::InvalidTransactionError, signed::RecoveryError}; +use reth_rpc_convert::{CallFeesError, EthTxEnvError, TransactionConversionError}; use reth_rpc_server_types::result::{ block_id_to_str, internal_rpc_err, invalid_params_rpc_err, rpc_err, rpc_error_with_code, }; @@ -22,7 +25,7 @@ use revm::context_interface::result::{ }; use revm_inspectors::tracing::MuxError; use std::convert::Infallible; -use tracing::error; +use tokio::sync::oneshot::error::RecvError; /// A trait to convert an error to an RPC error. pub trait ToRpcError: core::error::Error + Send + Sync + 'static { @@ -36,6 +39,19 @@ impl ToRpcError for jsonrpsee_types::ErrorObject<'static> { } } +impl ToRpcError for RpcError { + fn to_rpc_error(&self) -> jsonrpsee_types::ErrorObject<'static> { + match self { + Self::ErrorResp(payload) => jsonrpsee_types::error::ErrorObject::owned( + payload.code as i32, + payload.message.clone(), + payload.data.clone(), + ), + err => internal_rpc_err(err.to_string()), + } + } +} + /// Result alias pub type EthResult = Result; @@ -53,13 +69,21 @@ pub enum EthApiError { InvalidTransactionSignature, /// Errors related to the transaction pool #[error(transparent)] - PoolError(RpcPoolError), + PoolError(#[from] RpcPoolError), /// Header not found for block hash/number/tag #[error("header not found")] HeaderNotFound(BlockId), /// Header range not found for start block hash/number/tag to end block hash/number/tag #[error("header range not found, start block {0:?}, end block {1:?}")] HeaderRangeNotFound(BlockId, BlockId), + /// Thrown when historical data is not available because it has been pruned + /// + /// This error is intended for use as a standard response when historical data is + /// requested that has been pruned according to the node's data retention policy. + /// + /// See also + #[error("pruned history unavailable")] + PrunedHistoryUnavailable, /// Receipts not found for block hash/number/tag #[error("receipts not found")] ReceiptsNotFound(BlockId), @@ -146,6 +170,32 @@ pub enum EthApiError { /// Error thrown when tracing with a muxTracer fails #[error(transparent)] MuxTracerError(#[from] MuxError), + /// Error thrown when waiting for transaction confirmation times out + #[error( + "Transaction {hash} was added to the mempool but wasn't confirmed within {duration:?}." + )] + TransactionConfirmationTimeout { + /// Hash of the transaction that timed out + hash: B256, + /// Duration that was waited before timing out + duration: Duration, + }, + /// Error thrown when batch tx response channel fails + #[error(transparent)] + BatchTxRecvError(#[from] RecvError), + /// Error thrown when batch tx send channel fails + #[error("Batch transaction sender channel closed")] + BatchTxSendError, + /// Error that occurred during `call_many` execution with bundle and transaction context + #[error("call_many error in bundle {bundle_index} and transaction {tx_index}: {}", .error.message())] + CallManyError { + /// Bundle index where the error occurred + bundle_index: usize, + /// Transaction index within the bundle where the error occurred + tx_index: usize, + /// The underlying error object + error: jsonrpsee_types::ErrorObject<'static>, + }, /// Any other error #[error("{0}")] Other(Box), @@ -157,15 +207,59 @@ impl EthApiError { Self::Other(Box::new(err)) } + /// Creates a new [`EthApiError::CallManyError`] variant. + pub const fn call_many_error( + bundle_index: usize, + tx_index: usize, + error: jsonrpsee_types::ErrorObject<'static>, + ) -> Self { + Self::CallManyError { bundle_index, tx_index, error } + } + /// Returns `true` if error is [`RpcInvalidTransactionError::GasTooHigh`] pub const fn is_gas_too_high(&self) -> bool { - matches!(self, Self::InvalidTransaction(RpcInvalidTransactionError::GasTooHigh)) + matches!( + self, + Self::InvalidTransaction( + RpcInvalidTransactionError::GasTooHigh | + RpcInvalidTransactionError::GasLimitTooHigh + ) + ) } /// Returns `true` if error is [`RpcInvalidTransactionError::GasTooLow`] pub const fn is_gas_too_low(&self) -> bool { matches!(self, Self::InvalidTransaction(RpcInvalidTransactionError::GasTooLow)) } + + /// Returns the [`RpcInvalidTransactionError`] if this is a [`EthApiError::InvalidTransaction`] + pub const fn as_invalid_transaction(&self) -> Option<&RpcInvalidTransactionError> { + match self { + Self::InvalidTransaction(e) => Some(e), + _ => None, + } + } + + /// Converts the given [`StateOverrideError`] into a new [`EthApiError`] instance. + pub fn from_state_overrides_err(err: StateOverrideError) -> Self + where + E: Into, + { + err.into() + } + + /// Converts the given [`CallError`] into a new [`EthApiError`] instance. + pub fn from_call_err(err: CallError) -> Self + where + E: Into, + { + err.into() + } + + /// Converts this error into the rpc error object. + pub fn into_rpc_err(self) -> jsonrpsee_types::error::ErrorObject<'static> { + self.into() + } } impl From for jsonrpsee_types::error::ErrorObject<'static> { @@ -193,18 +287,12 @@ impl From for jsonrpsee_types::error::ErrorObject<'static> { EthApiError::UnknownBlockOrTxIndex | EthApiError::TransactionNotFound => { rpc_error_with_code(EthRpcErrorCode::ResourceNotFound.code(), error.to_string()) } - // TODO(onbjerg): We rewrite the error message here because op-node does string matching - // on the error message. - // - // Until https://github.com/ethereum-optimism/optimism/pull/11759 is released, this must be kept around. - EthApiError::HeaderNotFound(id) => rpc_error_with_code( - EthRpcErrorCode::ResourceNotFound.code(), - format!("block not found: {}", block_id_to_str(id)), - ), - EthApiError::ReceiptsNotFound(id) => rpc_error_with_code( - EthRpcErrorCode::ResourceNotFound.code(), - format!("{error}: {}", block_id_to_str(id)), - ), + EthApiError::HeaderNotFound(id) | EthApiError::ReceiptsNotFound(id) => { + rpc_error_with_code( + EthRpcErrorCode::ResourceNotFound.code(), + format!("block not found: {}", block_id_to_str(id)), + ) + } EthApiError::HeaderRangeNotFound(start_id, end_id) => rpc_error_with_code( EthRpcErrorCode::ResourceNotFound.code(), format!( @@ -213,6 +301,10 @@ impl From for jsonrpsee_types::error::ErrorObject<'static> { block_id_to_str(end_id), ), ), + err @ EthApiError::TransactionConfirmationTimeout { .. } => rpc_error_with_code( + EthRpcErrorCode::TransactionConfirmationTimeout.code(), + err.to_string(), + ), EthApiError::Unsupported(msg) => internal_rpc_err(msg), EthApiError::InternalJsTracerError(msg) => internal_rpc_err(msg), EthApiError::InvalidParams(msg) => invalid_params_rpc_err(msg), @@ -224,8 +316,88 @@ impl From for jsonrpsee_types::error::ErrorObject<'static> { internal_rpc_err(err.to_string()) } err @ EthApiError::TransactionInputError(_) => invalid_params_rpc_err(err.to_string()), + EthApiError::PrunedHistoryUnavailable => rpc_error_with_code(4444, error.to_string()), EthApiError::Other(err) => err.to_rpc_error(), EthApiError::MuxTracerError(msg) => internal_rpc_err(msg.to_string()), + EthApiError::BatchTxRecvError(err) => internal_rpc_err(err.to_string()), + EthApiError::BatchTxSendError => { + internal_rpc_err("Batch transaction sender channel closed".to_string()) + } + EthApiError::CallManyError { bundle_index, tx_index, error } => { + jsonrpsee_types::error::ErrorObject::owned( + error.code(), + format!( + "call_many error in bundle {bundle_index} and transaction {tx_index}: {}", + error.message() + ), + error.data(), + ) + } + } + } +} + +impl From for EthApiError { + fn from(_: TransactionConversionError) -> Self { + Self::TransactionConversionError + } +} + +impl From> for EthApiError +where + E: Into, +{ + fn from(value: CallError) -> Self { + match value { + CallError::Database(err) => err.into(), + CallError::InsufficientFunds(insufficient_funds_error) => { + Self::InvalidTransaction(RpcInvalidTransactionError::InsufficientFunds { + cost: insufficient_funds_error.cost, + balance: insufficient_funds_error.balance, + }) + } + } + } +} + +impl From> for EthApiError +where + E: Into, +{ + fn from(value: StateOverrideError) -> Self { + match value { + StateOverrideError::InvalidBytecode(bytecode_decode_error) => { + Self::InvalidBytecode(bytecode_decode_error.to_string()) + } + StateOverrideError::BothStateAndStateDiff(address) => { + Self::BothStateAndStateDiffInOverride(address) + } + StateOverrideError::Database(err) => err.into(), + } + } +} + +impl From for EthApiError { + fn from(value: EthTxEnvError) -> Self { + match value { + EthTxEnvError::CallFees(CallFeesError::BlobTransactionMissingBlobHashes) => { + Self::InvalidTransaction( + RpcInvalidTransactionError::BlobTransactionMissingBlobHashes, + ) + } + EthTxEnvError::CallFees(CallFeesError::FeeCapTooLow) => { + Self::InvalidTransaction(RpcInvalidTransactionError::FeeCapTooLow) + } + EthTxEnvError::CallFees(CallFeesError::ConflictingFeeFieldsInRequest) => { + Self::ConflictingFeeFieldsInRequest + } + EthTxEnvError::CallFees(CallFeesError::TipAboveFeeCap) => { + Self::InvalidTransaction(RpcInvalidTransactionError::TipAboveFeeCap) + } + EthTxEnvError::CallFees(CallFeesError::TipVeryHigh) => { + Self::InvalidTransaction(RpcInvalidTransactionError::TipVeryHigh) + } + EthTxEnvError::Input(err) => Self::TransactionInputError(err), } } } @@ -253,7 +425,30 @@ impl From for EthApiError { impl From for EthApiError { fn from(error: BlockExecutionError) -> Self { - Self::Internal(error.into()) + match error { + BlockExecutionError::Validation(validation_error) => match validation_error { + BlockValidationError::InvalidTx { error, .. } => { + if let Some(invalid_tx) = error.as_invalid_tx_err() { + Self::InvalidTransaction(RpcInvalidTransactionError::from( + invalid_tx.clone(), + )) + } else { + Self::InvalidTransaction(RpcInvalidTransactionError::other( + rpc_error_with_code( + EthRpcErrorCode::TransactionRejected.code(), + error.to_string(), + ), + )) + } + } + _ => Self::Internal(RethError::Execution(BlockExecutionError::Validation( + validation_error, + ))), + }, + BlockExecutionError::Internal(internal_error) => { + Self::Internal(RethError::Execution(BlockExecutionError::Internal(internal_error))) + } + } } } @@ -267,7 +462,6 @@ impl From for EthApiError { } ProviderError::BestBlockNotFound => Self::HeaderNotFound(BlockId::latest()), ProviderError::BlockNumberForTransactionIndexNotFound => Self::UnknownBlockOrTxIndex, - ProviderError::TotalDifficultyNotFound(num) => Self::HeaderNotFound(num.into()), ProviderError::FinalizedBlockNotFound => Self::HeaderNotFound(BlockId::finalized()), ProviderError::SafeBlockNotFound => Self::HeaderNotFound(BlockId::safe()), err => Self::Internal(err.into()), @@ -284,18 +478,32 @@ impl From for EthApiError { } } -impl From> for EthApiError +impl From> for EthApiError where T: Into, + TxError: reth_evm::InvalidTxError, { - fn from(err: EVMError) -> Self { + fn from(err: EVMError) -> Self { match err { - EVMError::Transaction(invalid_tx) => match invalid_tx { - InvalidTransaction::NonceTooLow { tx, state } => { - Self::InvalidTransaction(RpcInvalidTransactionError::NonceTooLow { tx, state }) + EVMError::Transaction(invalid_tx) => { + // Try to get the underlying InvalidTransaction if available + if let Some(eth_tx_err) = invalid_tx.as_invalid_tx_err() { + // Handle the special NonceTooLow case + match eth_tx_err { + InvalidTransaction::NonceTooLow { tx, state } => { + Self::InvalidTransaction(RpcInvalidTransactionError::NonceTooLow { + tx: *tx, + state: *state, + }) + } + _ => RpcInvalidTransactionError::from(eth_tx_err.clone()).into(), + } + } else { + // For custom transaction errors that don't wrap InvalidTransaction, + // convert to a custom error message + Self::EvmCustom(invalid_tx.to_string()) } - _ => RpcInvalidTransactionError::from(invalid_tx).into(), - }, + } EVMError::Header(err) => err.into(), EVMError::Database(err) => err.into(), EVMError::Custom(err) => Self::EvmCustom(err), @@ -381,6 +589,9 @@ pub enum RpcInvalidTransactionError { /// Thrown if the transaction gas exceeds the limit #[error("intrinsic gas too high")] GasTooHigh, + /// Thrown if the transaction gas limit exceeds the maximum + #[error("gas limit too high")] + GasLimitTooHigh, /// Thrown if a transaction is not supported in the current network configuration. #[error("transaction type not supported")] TxTypeNotSupported, @@ -408,6 +619,9 @@ pub enum RpcInvalidTransactionError { /// Contains the gas limit. #[error("out of gas: gas exhausted during memory expansion: {0}")] MemoryOutOfGas(u64), + /// Memory limit was exceeded during memory expansion. + #[error("out of memory: memory limit exceeded during memory expansion")] + MemoryLimitOutOfGas, /// Gas limit was exceeded during precompile execution. /// Contains the gas limit. #[error("out of gas: gas exhausted during precompiled contract execution: {0}")] @@ -456,15 +670,18 @@ pub enum RpcInvalidTransactionError { /// Blob transaction is a create transaction #[error("blob transaction is a create transaction")] BlobTransactionIsCreate, - /// EOF crate should have `to` address - #[error("EOF crate should have `to` address")] - EofCrateShouldHaveToAddress, /// EIP-7702 is not enabled. #[error("EIP-7702 authorization list not supported")] AuthorizationListNotSupported, /// EIP-7702 transaction has invalid fields set. #[error("EIP-7702 authorization list has invalid fields")] AuthorizationListInvalidFields, + /// Transaction priority fee is below the minimum required priority fee. + #[error("transaction priority fee below minimum required priority fee {minimum_priority_fee}")] + PriorityFeeBelowMinimum { + /// Minimum required priority fee. + minimum_priority_fee: u128, + }, /// Any other error #[error("{0}")] Other(Box), @@ -475,16 +692,18 @@ impl RpcInvalidTransactionError { pub fn other(err: E) -> Self { Self::Other(Box::new(err)) } -} -impl RpcInvalidTransactionError { /// Returns the rpc error code for this error. pub const fn error_code(&self) -> i32 { match self { Self::InvalidChainId | Self::GasTooLow | Self::GasTooHigh | - Self::GasRequiredExceedsAllowance { .. } => EthRpcErrorCode::InvalidInput.code(), + Self::GasRequiredExceedsAllowance { .. } | + Self::NonceTooLow { .. } | + Self::NonceTooHigh { .. } | + Self::FeeCapTooLow | + Self::FeeCapVeryHigh => EthRpcErrorCode::InvalidInput.code(), Self::Revert(_) => EthRpcErrorCode::ExecutionError.code(), _ => EthRpcErrorCode::TransactionRejected.code(), } @@ -493,7 +712,7 @@ impl RpcInvalidTransactionError { /// Converts the halt error /// /// Takes the configured gas limit of the transaction which is attached to the error - pub const fn halt(reason: HaltReason, gas_limit: u64) -> Self { + pub fn halt(reason: HaltReason, gas_limit: u64) -> Self { match reason { HaltReason::OutOfGas(err) => Self::out_of_gas(err, gas_limit), HaltReason::NonceOverflow => Self::NonceMaxValue, @@ -507,11 +726,17 @@ impl RpcInvalidTransactionError { OutOfGasError::Basic | OutOfGasError::ReentrancySentry => { Self::BasicOutOfGas(gas_limit) } - OutOfGasError::Memory | OutOfGasError::MemoryLimit => Self::MemoryOutOfGas(gas_limit), + OutOfGasError::Memory => Self::MemoryOutOfGas(gas_limit), + OutOfGasError::MemoryLimit => Self::MemoryLimitOutOfGas, OutOfGasError::Precompile => Self::PrecompileOutOfGas(gas_limit), OutOfGasError::InvalidOperand => Self::InvalidOperandOutOfGas(gas_limit), } } + + /// Converts this error into the rpc error object. + pub fn into_rpc_err(self) -> jsonrpsee_types::error::ErrorObject<'static> { + self.into() + } } impl From for jsonrpsee_types::error::ErrorObject<'static> { @@ -534,10 +759,13 @@ impl From for jsonrpsee_types::error::ErrorObject<'s impl From for RpcInvalidTransactionError { fn from(err: InvalidTransaction) -> Self { match err { - InvalidTransaction::InvalidChainId => Self::InvalidChainId, + InvalidTransaction::InvalidChainId | InvalidTransaction::MissingChainId => { + Self::InvalidChainId + } InvalidTransaction::PriorityFeeGreaterThanMaxFee => Self::TipAboveFeeCap, InvalidTransaction::GasPriceLessThanBasefee => Self::FeeCapTooLow, - InvalidTransaction::CallerGasLimitMoreThanBlock => { + InvalidTransaction::CallerGasLimitMoreThanBlock | + InvalidTransaction::TxGasLimitGreaterThanCap { .. } => { // tx.gas > block.gas_limit Self::GasTooHigh } @@ -566,12 +794,11 @@ impl From for RpcInvalidTransactionError { InvalidTransaction::BlobVersionedHashesNotSupported => { Self::BlobVersionedHashesNotSupported } - InvalidTransaction::BlobGasPriceGreaterThanMax => Self::BlobFeeCapTooLow, + InvalidTransaction::BlobGasPriceGreaterThanMax { .. } => Self::BlobFeeCapTooLow, InvalidTransaction::EmptyBlobs => Self::BlobTransactionMissingBlobHashes, InvalidTransaction::BlobVersionNotSupported => Self::BlobHashVersionMismatch, InvalidTransaction::TooManyBlobs { have, .. } => Self::TooManyBlobs { have }, InvalidTransaction::BlobCreateTransaction => Self::BlobTransactionIsCreate, - InvalidTransaction::EofCreateShouldHaveToAddress => Self::EofCrateShouldHaveToAddress, InvalidTransaction::AuthorizationListNotSupported => { Self::AuthorizationListNotSupported } @@ -585,6 +812,7 @@ impl From for RpcInvalidTransactionError { InvalidTransaction::Eip7873MissingTarget => { Self::other(internal_rpc_err(err.to_string())) } + InvalidTransaction::Str(_) => Self::other(internal_rpc_err(err.to_string())), } } } @@ -617,6 +845,7 @@ impl From for RpcInvalidTransactionError { InvalidTransactionError::TipAboveFeeCap => Self::TipAboveFeeCap, InvalidTransactionError::FeeCapTooLow => Self::FeeCapTooLow, InvalidTransactionError::SignerAccountHasBytecode => Self::SenderNoEOA, + InvalidTransactionError::GasLimitTooHigh => Self::GasLimitTooHigh, } } } @@ -689,11 +918,14 @@ pub enum RpcPoolError { /// When the transaction exceeds the block gas limit #[error("exceeds block gas limit")] ExceedsGasLimit, + /// When the transaction gas limit exceeds the maximum transaction gas limit + #[error("exceeds max transaction gas limit")] + MaxTxGasLimitExceeded, /// Thrown when a new transaction is added to the pool, but then immediately discarded to /// respect the tx fee exceeds the configured cap #[error("tx fee ({max_tx_fee_wei} wei) exceeds the configured cap ({tx_fee_cap_wei} wei)")] ExceedsFeeCap { - /// max fee in wei of new tx submitted to the pull (e.g. 0.11534 ETH) + /// max fee in wei of new tx submitted to the pool (e.g. 0.11534 ETH) max_tx_fee_wei: u128, /// configured tx fee cap in wei (e.g. 1.0 ETH) tx_fee_cap_wei: u128, @@ -702,8 +934,13 @@ pub enum RpcPoolError { #[error("negative value")] NegativeValue, /// When oversized data is encountered - #[error("oversized data")] - OversizedData, + #[error("oversized data: transaction size {size}, limit {limit}")] + OversizedData { + /// Size of the transaction/input data that exceeded the limit. + size: usize, + /// Configured limit that was exceeded. + limit: usize, + }, /// When the max initcode size is exceeded #[error("max initcode size exceeded")] ExceedsMaxInitCodeSize, @@ -737,7 +974,23 @@ impl From for jsonrpsee_types::error::ErrorObject<'static> { RpcPoolError::TxPoolOverflow => { rpc_error_with_code(EthRpcErrorCode::TransactionRejected.code(), error.to_string()) } - error => internal_rpc_err(error.to_string()), + RpcPoolError::AlreadyKnown | + RpcPoolError::InvalidSender | + RpcPoolError::Underpriced | + RpcPoolError::ReplaceUnderpriced | + RpcPoolError::ExceedsGasLimit | + RpcPoolError::MaxTxGasLimitExceeded | + RpcPoolError::ExceedsFeeCap { .. } | + RpcPoolError::NegativeValue | + RpcPoolError::OversizedData { .. } | + RpcPoolError::ExceedsMaxInitCodeSize | + RpcPoolError::PoolTransactionError(_) | + RpcPoolError::Eip4844(_) | + RpcPoolError::Eip7702(_) | + RpcPoolError::AddressAlreadyReserved => { + rpc_error_with_code(EthRpcErrorCode::InvalidInput.code(), error.to_string()) + } + RpcPoolError::Other(other) => internal_rpc_err(other.to_string()), } } } @@ -763,6 +1016,7 @@ impl From for RpcPoolError { match err { InvalidPoolTransactionError::Consensus(err) => Self::Invalid(err.into()), InvalidPoolTransactionError::ExceedsGasLimit(_, _) => Self::ExceedsGasLimit, + InvalidPoolTransactionError::MaxTxGasLimitExceeded(_, _) => Self::MaxTxGasLimitExceeded, InvalidPoolTransactionError::ExceedsFeeCap { max_tx_fee_wei, tx_fee_cap_wei } => { Self::ExceedsFeeCap { max_tx_fee_wei, tx_fee_cap_wei } } @@ -772,14 +1026,24 @@ impl From for RpcPoolError { InvalidPoolTransactionError::IntrinsicGasTooLow => { Self::Invalid(RpcInvalidTransactionError::GasTooLow) } - InvalidPoolTransactionError::OversizedData(_, _) => Self::OversizedData, + InvalidPoolTransactionError::OversizedData { size, limit } => { + Self::OversizedData { size, limit } + } InvalidPoolTransactionError::Underpriced => Self::Underpriced, + InvalidPoolTransactionError::Eip2681 => { + Self::Invalid(RpcInvalidTransactionError::NonceMaxValue) + } InvalidPoolTransactionError::Other(err) => Self::PoolTransactionError(err), InvalidPoolTransactionError::Eip4844(err) => Self::Eip4844(err), InvalidPoolTransactionError::Eip7702(err) => Self::Eip7702(err), InvalidPoolTransactionError::Overdraft { cost, balance } => { Self::Invalid(RpcInvalidTransactionError::InsufficientFunds { cost, balance }) } + InvalidPoolTransactionError::PriorityFeeBelowMinimum { minimum_priority_fee } => { + Self::Invalid(RpcInvalidTransactionError::PriorityFeeBelowMinimum { + minimum_priority_fee, + }) + } } } } @@ -870,6 +1134,47 @@ mod tests { assert_eq!(err.message(), "block not found: finalized"); } + #[test] + fn receipts_not_found_message() { + let err: jsonrpsee_types::error::ErrorObject<'static> = + EthApiError::ReceiptsNotFound(BlockId::hash(b256!( + "0x1a15e3c30cf094a99826869517b16d185d45831d3a494f01030b0001a9d3ebb9" + ))) + .into(); + assert_eq!( + err.message(), + "block not found: hash 0x1a15e3c30cf094a99826869517b16d185d45831d3a494f01030b0001a9d3ebb9" + ); + let err: jsonrpsee_types::error::ErrorObject<'static> = + EthApiError::ReceiptsNotFound(BlockId::hash_canonical(b256!( + "0x1a15e3c30cf094a99826869517b16d185d45831d3a494f01030b0001a9d3ebb9" + ))) + .into(); + assert_eq!( + err.message(), + "block not found: canonical hash 0x1a15e3c30cf094a99826869517b16d185d45831d3a494f01030b0001a9d3ebb9" + ); + let err: jsonrpsee_types::error::ErrorObject<'static> = + EthApiError::ReceiptsNotFound(BlockId::number(100000)).into(); + assert_eq!(err.code(), EthRpcErrorCode::ResourceNotFound.code()); + assert_eq!(err.message(), "block not found: 0x186a0"); + let err: jsonrpsee_types::error::ErrorObject<'static> = + EthApiError::ReceiptsNotFound(BlockId::latest()).into(); + assert_eq!(err.message(), "block not found: latest"); + let err: jsonrpsee_types::error::ErrorObject<'static> = + EthApiError::ReceiptsNotFound(BlockId::safe()).into(); + assert_eq!(err.message(), "block not found: safe"); + let err: jsonrpsee_types::error::ErrorObject<'static> = + EthApiError::ReceiptsNotFound(BlockId::finalized()).into(); + assert_eq!(err.message(), "block not found: finalized"); + let err: jsonrpsee_types::error::ErrorObject<'static> = + EthApiError::ReceiptsNotFound(BlockId::pending()).into(); + assert_eq!(err.message(), "block not found: pending"); + let err: jsonrpsee_types::error::ErrorObject<'static> = + EthApiError::ReceiptsNotFound(BlockId::earliest()).into(); + assert_eq!(err.message(), "block not found: earliest"); + } + #[test] fn revert_err_display() { let revert = Revert::from("test_revert_reason"); diff --git a/crates/rpc/rpc-eth-types/src/fee_history.rs b/crates/rpc/rpc-eth-types/src/fee_history.rs index da67e92dcd3..3eaf69d2c4c 100644 --- a/crates/rpc/rpc-eth-types/src/fee_history.rs +++ b/crates/rpc/rpc-eth-types/src/fee_history.rs @@ -6,9 +6,8 @@ use std::{ sync::{atomic::Ordering::SeqCst, Arc}, }; -use alloy_consensus::{BlockHeader, Transaction, TxReceipt}; -use alloy_eips::{eip1559::calc_next_block_base_fee, eip7840::BlobParams}; -use alloy_primitives::B256; +use alloy_consensus::{BlockHeader, Header, Transaction, TxReceipt}; +use alloy_eips::eip7840::BlobParams; use alloy_rpc_types_eth::TxGasAndReward; use futures::{ future::{Fuse, FusedFuture}, @@ -23,17 +22,22 @@ use reth_storage_api::BlockReaderIdExt; use serde::{Deserialize, Serialize}; use tracing::trace; +use crate::utils::checked_blob_gas_used_ratio; + use super::{EthApiError, EthStateCache}; /// Contains cached fee history entries for blocks. /// /// Purpose for this is to provide cached data for `eth_feeHistory`. #[derive(Debug, Clone)] -pub struct FeeHistoryCache { - inner: Arc, +pub struct FeeHistoryCache { + inner: Arc>, } -impl FeeHistoryCache { +impl FeeHistoryCache +where + H: BlockHeader + Clone, +{ /// Creates new `FeeHistoryCache` instance, initialize it with the more recent data, set bounds pub fn new(config: FeeHistoryCacheConfig) -> Self { let inner = FeeHistoryCacheInner { @@ -73,9 +77,9 @@ impl FeeHistoryCache { /// Insert block data into the cache. async fn insert_blocks<'a, I, B, R, C>(&self, blocks: I, chain_spec: &C) where - B: Block + 'a, - R: TxReceipt, - I: IntoIterator, Arc>)>, + B: Block
+ 'a, + R: TxReceipt + 'a, + I: IntoIterator, &'a [R])>, C: EthChainSpec, { let mut entries = self.inner.entries.write().await; @@ -83,16 +87,16 @@ impl FeeHistoryCache { let percentiles = self.predefined_percentiles(); // Insert all new blocks and calculate approximated rewards for (block, receipts) in blocks { - let mut fee_history_entry = FeeHistoryEntry::new( + let mut fee_history_entry = FeeHistoryEntry::::new( block, chain_spec.blob_params_at_timestamp(block.header().timestamp()), ); fee_history_entry.rewards = calculate_reward_percentiles_for_block( &percentiles, - fee_history_entry.gas_used, - fee_history_entry.base_fee_per_gas, + fee_history_entry.header.gas_used(), + fee_history_entry.header.base_fee_per_gas().unwrap_or_default(), block.body().transactions(), - &receipts, + receipts, ) .unwrap_or_default(); entries.insert(block.number(), fee_history_entry); @@ -132,7 +136,7 @@ impl FeeHistoryCache { self.inner.lower_bound.load(SeqCst) } - /// Collect fee history for given range. + /// Collect fee history for the given range (inclusive `start_block..=end_block`). /// /// This function retrieves fee history entries from the cache for the specified range. /// If the requested range (`start_block` to `end_block`) is within the cache bounds, @@ -142,7 +146,11 @@ impl FeeHistoryCache { &self, start_block: u64, end_block: u64, - ) -> Option> { + ) -> Option>> { + if end_block < start_block { + // invalid range, return None + return None + } let lower_bound = self.lower_bound(); let upper_bound = self.upper_bound(); if start_block >= lower_bound && end_block <= upper_bound { @@ -194,7 +202,7 @@ impl Default for FeeHistoryCacheConfig { /// Container type for shared state in [`FeeHistoryCache`] #[derive(Debug)] -struct FeeHistoryCacheInner { +struct FeeHistoryCacheInner { /// Stores the lower bound of the cache lower_bound: AtomicU64, /// Stores the upper bound of the cache @@ -203,21 +211,22 @@ struct FeeHistoryCacheInner { /// and max number of blocks config: FeeHistoryCacheConfig, /// Stores the entries of the cache - entries: tokio::sync::RwLock>, + entries: tokio::sync::RwLock>>, } /// Awaits for new chain events and directly inserts them into the cache so they're available /// immediately before they need to be fetched from disk. pub async fn fee_history_cache_new_blocks_task( - fee_history_cache: FeeHistoryCache, + fee_history_cache: FeeHistoryCache, mut events: St, provider: Provider, - cache: EthStateCache, + cache: EthStateCache, ) where St: Stream> + Unpin + 'static, Provider: BlockReaderIdExt + ChainSpecProvider + 'static, N: NodePrimitives, + N::BlockHeader: BlockHeader + Clone, { // We're listening for new blocks emitted when the node is in live sync. // If the node transitions to stage sync, we need to fetch the missing blocks @@ -225,13 +234,13 @@ pub async fn fee_history_cache_new_blocks_task( let mut fetch_missing_block = Fuse::terminated(); loop { - if fetch_missing_block.is_terminated() { - if let Some(block_number) = missing_blocks.pop_front() { - trace!(target: "rpc::fee", ?block_number, "Fetching missing block for fee history cache"); - if let Ok(Some(hash)) = provider.block_hash(block_number) { - // fetch missing block - fetch_missing_block = cache.get_block_and_receipts(hash).boxed().fuse(); - } + if fetch_missing_block.is_terminated() && + let Some(block_number) = missing_blocks.pop_front() + { + trace!(target: "rpc::fee", ?block_number, "Fetching missing block for fee history cache"); + if let Ok(Some(hash)) = provider.block_hash(block_number) { + // fetch missing block + fetch_missing_block = cache.get_block_and_receipts(hash).boxed().fuse(); } } @@ -241,7 +250,7 @@ pub async fn fee_history_cache_new_blocks_task( res = &mut fetch_missing_block => { if let Ok(res) = res { let res = res.as_ref() - .map(|(b, r)| (b.sealed_block(), r.clone())); + .map(|(b, r)| (b.sealed_block(), r.as_slice())); fee_history_cache.insert_blocks(res, &chain_spec).await; } } @@ -252,13 +261,12 @@ pub async fn fee_history_cache_new_blocks_task( }; let committed = event.committed(); - let (blocks, receipts): (Vec<_>, Vec<_>) = committed + let blocks_and_receipts = committed .blocks_and_receipts() .map(|(block, receipts)| { - (block.clone_sealed_block(), Arc::new(receipts.clone())) - }) - .unzip(); - fee_history_cache.insert_blocks(blocks.iter().zip(receipts), &chain_spec).await; + (block.sealed_block(), receipts.as_slice()) + }); + fee_history_cache.insert_blocks(blocks_and_receipts, &chain_spec).await; // keep track of missing blocks missing_blocks = fee_history_cache.missing_consecutive_blocks().await; @@ -333,9 +341,9 @@ where /// A cached entry for a block's fee history. #[derive(Debug, Clone)] -pub struct FeeHistoryEntry { - /// The base fee per gas for this block. - pub base_fee_per_gas: u64, +pub struct FeeHistoryEntry { + /// The full block header. + pub header: H, /// Gas used ratio this block. pub gas_used_ratio: f64, /// The base per blob gas for EIP-4844. @@ -346,64 +354,42 @@ pub struct FeeHistoryEntry { /// Calculated as the ratio of blob gas used and the available blob data gas per block. /// Will be zero if no blob gas was used or pre EIP-4844. pub blob_gas_used_ratio: f64, - /// The excess blob gas of the block. - pub excess_blob_gas: Option, - /// The total amount of blob gas consumed by the transactions within the block, - /// added in EIP-4844 - pub blob_gas_used: Option, - /// Gas used by this block. - pub gas_used: u64, - /// Gas limit by this block. - pub gas_limit: u64, - /// Hash of the block. - pub header_hash: B256, /// Approximated rewards for the configured percentiles. pub rewards: Vec, - /// The timestamp of the block. - pub timestamp: u64, /// Blob parameters for this block. pub blob_params: Option, } -impl FeeHistoryEntry { +impl FeeHistoryEntry +where + H: BlockHeader + Clone, +{ /// Creates a new entry from a sealed block. /// /// Note: This does not calculate the rewards for the block. - pub fn new(block: &SealedBlock, blob_params: Option) -> Self { + pub fn new(block: &SealedBlock, blob_params: Option) -> Self + where + B: Block
, + { + let header = block.header(); Self { - base_fee_per_gas: block.header().base_fee_per_gas().unwrap_or_default(), - gas_used_ratio: block.header().gas_used() as f64 / block.header().gas_limit() as f64, - base_fee_per_blob_gas: block - .header() + header: block.header().clone(), + gas_used_ratio: header.gas_used() as f64 / header.gas_limit() as f64, + base_fee_per_blob_gas: header .excess_blob_gas() .and_then(|excess_blob_gas| Some(blob_params?.calc_blob_fee(excess_blob_gas))), - blob_gas_used_ratio: block.body().blob_gas_used() as f64 / + blob_gas_used_ratio: checked_blob_gas_used_ratio( + block.body().blob_gas_used(), blob_params .as_ref() .map(|params| params.max_blob_gas_per_block()) - .unwrap_or(alloy_eips::eip4844::MAX_DATA_GAS_PER_BLOCK_DENCUN) - as f64, - excess_blob_gas: block.header().excess_blob_gas(), - blob_gas_used: block.header().blob_gas_used(), - gas_used: block.header().gas_used(), - header_hash: block.hash(), - gas_limit: block.header().gas_limit(), + .unwrap_or(alloy_eips::eip4844::MAX_DATA_GAS_PER_BLOCK_DENCUN), + ), rewards: Vec::new(), - timestamp: block.header().timestamp(), blob_params, } } - /// Returns the base fee for the next block according to the EIP-1559 spec. - pub fn next_block_base_fee(&self, chain_spec: impl EthChainSpec) -> u64 { - calc_next_block_base_fee( - self.gas_used, - self.gas_limit, - self.base_fee_per_gas, - chain_spec.base_fee_params_at_timestamp(self.timestamp), - ) - } - /// Returns the blob fee for the next block according to the EIP-4844 spec. /// /// Returns `None` if `excess_blob_gas` is None. @@ -418,8 +404,12 @@ impl FeeHistoryEntry { /// /// Returns a `None` if no excess blob gas is set, no EIP-4844 support pub fn next_block_excess_blob_gas(&self) -> Option { - self.excess_blob_gas.and_then(|excess_blob_gas| { - Some(self.blob_params?.next_block_excess_blob_gas(excess_blob_gas, self.blob_gas_used?)) + self.header.excess_blob_gas().and_then(|excess_blob_gas| { + Some(self.blob_params?.next_block_excess_blob_gas_osaka( + excess_blob_gas, + self.header.blob_gas_used()?, + self.header.base_fee_per_gas()?, + )) }) } } diff --git a/crates/rpc/rpc-eth-types/src/gas_oracle.rs b/crates/rpc/rpc-eth-types/src/gas_oracle.rs index ea2d05b6351..7bbf6433c6d 100644 --- a/crates/rpc/rpc-eth-types/src/gas_oracle.rs +++ b/crates/rpc/rpc-eth-types/src/gas_oracle.rs @@ -2,22 +2,20 @@ //! previous blocks. use super::{EthApiError, EthResult, EthStateCache, RpcInvalidTransactionError}; -use alloy_consensus::{constants::GWEI_TO_WEI, BlockHeader, Transaction}; +use alloy_consensus::{constants::GWEI_TO_WEI, BlockHeader, Transaction, TxReceipt}; use alloy_eips::BlockNumberOrTag; use alloy_primitives::{B256, U256}; use alloy_rpc_types_eth::BlockId; use derive_more::{Deref, DerefMut, From, Into}; use itertools::Itertools; -use reth_primitives_traits::{BlockBody, SignedTransaction}; use reth_rpc_server_types::{ constants, constants::gas_oracle::{ DEFAULT_GAS_PRICE_BLOCKS, DEFAULT_GAS_PRICE_PERCENTILE, DEFAULT_IGNORE_GAS_PRICE, - DEFAULT_MAX_GAS_PRICE, DEFAULT_MIN_SUGGESTED_PRIORITY_FEE, MAX_HEADER_HISTORY, - MAX_REWARD_PERCENTILE_COUNT, SAMPLE_NUMBER, + DEFAULT_MAX_GAS_PRICE, MAX_HEADER_HISTORY, MAX_REWARD_PERCENTILE_COUNT, SAMPLE_NUMBER, }, }; -use reth_storage_api::{BlockReader, BlockReaderIdExt}; +use reth_storage_api::{BlockReaderIdExt, NodePrimitivesProvider}; use schnellru::{ByLength, LruMap}; use serde::{Deserialize, Serialize}; use std::fmt::{self, Debug, Formatter}; @@ -51,19 +49,13 @@ pub struct GasPriceOracleConfig { pub max_reward_percentile_count: u64, /// The default gas price to use if there are no blocks to use - pub default: Option, + pub default_suggested_fee: Option, /// The maximum gas price to use for the estimate pub max_price: Option, /// The minimum gas price, under which the sample will be ignored pub ignore_price: Option, - - /// The minimum suggested priority fee for Optimism chains - /// - /// This is used by Optimism-specific gas price algorithms. - /// If not set, defaults to [`DEFAULT_MIN_SUGGESTED_PRIORITY_FEE`] (0.0001 gwei = 100,000 wei). - pub min_suggested_priority_fee: Option, } impl Default for GasPriceOracleConfig { @@ -74,10 +66,9 @@ impl Default for GasPriceOracleConfig { max_header_history: MAX_HEADER_HISTORY, max_block_history: MAX_HEADER_HISTORY, max_reward_percentile_count: MAX_REWARD_PERCENTILE_COUNT, - default: None, + default_suggested_fee: None, max_price: Some(DEFAULT_MAX_GAS_PRICE), ignore_price: Some(DEFAULT_IGNORE_GAS_PRICE), - min_suggested_priority_fee: Some(DEFAULT_MIN_SUGGESTED_PRIORITY_FEE), } } } @@ -86,12 +77,12 @@ impl Default for GasPriceOracleConfig { #[derive(Debug)] pub struct GasPriceOracle where - Provider: BlockReader, + Provider: NodePrimitivesProvider, { /// The type used to subscribe to block events and get block info provider: Provider, /// The cache for blocks - cache: EthStateCache, + cache: EthStateCache, /// The config for the oracle oracle_config: GasPriceOracleConfig, /// The price under which the sample will be ignored. @@ -103,13 +94,13 @@ where impl GasPriceOracle where - Provider: BlockReaderIdExt, + Provider: BlockReaderIdExt + NodePrimitivesProvider, { /// Creates and returns the [`GasPriceOracle`]. pub fn new( provider: Provider, mut oracle_config: GasPriceOracleConfig, - cache: EthStateCache, + cache: EthStateCache, ) -> Self { // sanitize the percentile to be less than 100 if oracle_config.percentile > 100 { @@ -121,20 +112,17 @@ where // this is the number of blocks that we will cache the values for let cached_values = (oracle_config.blocks * 5).max(oracle_config.max_block_history as u32); let inner = Mutex::new(GasPriceOracleInner { - last_price: Default::default(), + last_price: GasPriceOracleResult { + block_hash: B256::ZERO, + price: oracle_config + .default_suggested_fee + .unwrap_or_else(|| GasPriceOracleResult::default().price), + }, lowest_effective_tip_cache: EffectiveTipLruCache(LruMap::new(ByLength::new( cached_values, ))), }); - // if the price is less than the min suggested priority fee, then we use the min suggested priority fee - if oracle_config.min_suggested_priority_fee.is_none() - || oracle_config.min_suggested_priority_fee.unwrap_or_default() <= U256::ZERO - { - warn!("Sanitizing invalid optimism gasprice oracle min priority fee suggestion, using default"); - oracle_config.min_suggested_priority_fee = Some(DEFAULT_MIN_SUGGESTED_PRIORITY_FEE); - } - Self { provider, oracle_config, cache, ignore_price, inner } } @@ -216,10 +204,10 @@ where }; // constrain to the max price - if let Some(max_price) = self.oracle_config.max_price { - if price > max_price { - price = max_price; - } + if let Some(max_price) = self.oracle_config.max_price && + price > max_price + { + price = max_price; } inner.last_price = GasPriceOracleResult { block_hash: header.hash(), price }; @@ -248,7 +236,7 @@ where let parent_hash = block.parent_hash(); // sort the functions by ascending effective tip first - let sorted_transactions = block.body().transactions_iter().sorted_by_cached_key(|tx| { + let sorted_transactions = block.transactions_recovered().sorted_by_cached_key(|tx| { if let Some(base_fee) = base_fee_per_gas { (*tx).effective_tip_per_gas(base_fee) } else { @@ -266,17 +254,15 @@ where }; // ignore transactions with a tip under the configured threshold - if let Some(ignore_under) = self.ignore_price { - if effective_tip < Some(ignore_under) { - continue - } + if let Some(ignore_under) = self.ignore_price && + effective_tip < Some(ignore_under) + { + continue } // check if the sender was the coinbase, if so, ignore - if let Ok(sender) = tx.recover_signer() { - if sender == block.beneficiary() { - continue - } + if tx.signer() == block.beneficiary() { + continue } // a `None` effective_gas_tip represents a transaction where the max_fee_per_gas is @@ -291,8 +277,115 @@ where Ok(Some((parent_hash, prices))) } -} + /// Suggests a max priority fee value using a simplified and more predictable algorithm + /// appropriate for chains like Optimism with a single known block builder. + /// + /// It returns either: + /// - The minimum suggested priority fee when blocks have capacity + /// - 10% above the median effective priority fee from the last block when at capacity + /// + /// A block is considered at capacity if its total gas used plus the maximum single transaction + /// gas would exceed the block's gas limit. + pub async fn op_suggest_tip_cap(&self, min_suggested_priority_fee: U256) -> EthResult { + let header = self + .provider + .sealed_header_by_number_or_tag(BlockNumberOrTag::Latest)? + .ok_or(EthApiError::HeaderNotFound(BlockId::latest()))?; + + let mut inner = self.inner.lock().await; + + // if we have stored a last price, then we check whether or not it was for the same head + if inner.last_price.block_hash == header.hash() { + return Ok(inner.last_price.price); + } + + let mut suggestion = min_suggested_priority_fee; + + // find the maximum gas used by any of the transactions in the block to use as the + // capacity margin for the block, if no receipts are found return the + // suggested_min_priority_fee + let receipts = self + .cache + .get_receipts(header.hash()) + .await? + .ok_or(EthApiError::ReceiptsNotFound(BlockId::latest()))?; + + let mut max_tx_gas_used = 0u64; + let mut last_cumulative_gas = 0; + for receipt in receipts.as_ref() { + let cumulative_gas = receipt.cumulative_gas_used(); + // get the gas used by each transaction in the block, by subtracting the + // cumulative gas used of the previous transaction from the cumulative gas used of + // the current transaction. This is because there is no gas_used() + // method on the Receipt trait. + let gas_used = cumulative_gas - last_cumulative_gas; + max_tx_gas_used = max_tx_gas_used.max(gas_used); + last_cumulative_gas = cumulative_gas; + } + + // if the block is at capacity, the suggestion must be increased + if header.gas_used() + max_tx_gas_used > header.gas_limit() { + let Some(median_tip) = self.get_block_median_tip(header.hash()).await? else { + return Ok(suggestion); + }; + + let new_suggestion = median_tip + median_tip / U256::from(10); + + if new_suggestion > suggestion { + suggestion = new_suggestion; + } + } + + // constrain to the max price + if let Some(max_price) = self.oracle_config.max_price && + suggestion > max_price + { + suggestion = max_price; + } + + inner.last_price = GasPriceOracleResult { block_hash: header.hash(), price: suggestion }; + + Ok(suggestion) + } + + /// Get the median tip value for the given block. This is useful for determining + /// tips when a block is at capacity. + /// + /// If the block cannot be found or has no transactions, this will return `None`. + pub async fn get_block_median_tip(&self, block_hash: B256) -> EthResult> { + // check the cache (this will hit the disk if the block is not cached) + let Some(block) = self.cache.get_recovered_block(block_hash).await? else { + return Ok(None) + }; + + let base_fee_per_gas = block.base_fee_per_gas(); + + // Filter, sort and collect the prices + let prices = block + .transactions_recovered() + .filter_map(|tx| { + if let Some(base_fee) = base_fee_per_gas { + (*tx).effective_tip_per_gas(base_fee) + } else { + Some((*tx).priority_fee_or_price()) + } + }) + .sorted() + .collect::>(); + + let median = if prices.is_empty() { + // if there are no prices, return `None` + None + } else if prices.len() % 2 == 1 { + Some(U256::from(prices[prices.len() / 2])) + } else { + Some(U256::from((prices[prices.len() / 2 - 1] + prices[prices.len() / 2]) / 2)) + }; + + Ok(median) + } +} /// Container type for mutable inner state of the [`GasPriceOracle`] #[derive(Debug)] struct GasPriceOracleInner { diff --git a/crates/rpc/rpc-eth-types/src/lib.rs b/crates/rpc/rpc-eth-types/src/lib.rs index 8d92fda9c33..7378ad99629 100644 --- a/crates/rpc/rpc-eth-types/src/lib.rs +++ b/crates/rpc/rpc-eth-types/src/lib.rs @@ -5,9 +5,10 @@ html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256", issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(test), warn(unused_crate_dependencies))] +pub mod block; pub mod builder; pub mod cache; pub mod error; @@ -17,9 +18,9 @@ pub mod id_provider; pub mod logs_utils; pub mod pending_block; pub mod receipt; -pub mod revm_utils; pub mod simulate; pub mod transaction; +pub mod tx_forward; pub mod utils; pub use builder::config::{EthConfig, EthFilterConfig}; @@ -34,5 +35,5 @@ pub use gas_oracle::{ }; pub use id_provider::EthSubscriptionIdProvider; pub use pending_block::{PendingBlock, PendingBlockEnv, PendingBlockEnvOrigin}; -pub use receipt::EthReceiptBuilder; -pub use transaction::TransactionSource; +pub use transaction::{FillTransactionResult, TransactionSource}; +pub use tx_forward::ForwardConfig; diff --git a/crates/rpc/rpc-eth-types/src/logs_utils.rs b/crates/rpc/rpc-eth-types/src/logs_utils.rs index 0ea14bb160e..1d93de4bb1f 100644 --- a/crates/rpc/rpc-eth-types/src/logs_utils.rs +++ b/crates/rpc/rpc-eth-types/src/logs_utils.rs @@ -5,7 +5,7 @@ use alloy_consensus::TxReceipt; use alloy_eips::{eip2718::Encodable2718, BlockNumHash}; use alloy_primitives::TxHash; -use alloy_rpc_types_eth::{FilteredParams, Log}; +use alloy_rpc_types_eth::{Filter, Log}; use reth_chainspec::ChainInfo; use reth_errors::ProviderError; use reth_primitives_traits::{BlockBody, RecoveredBlock, SignedTransaction}; @@ -14,8 +14,9 @@ use std::sync::Arc; /// Returns all matching of a block's receipts when the transaction hashes are known. pub fn matching_block_logs_with_tx_hashes<'a, I, R>( - filter: &FilteredParams, + filter: &Filter, block_num_hash: BlockNumHash, + block_timestamp: u64, tx_hashes_and_receipts: I, removed: bool, ) -> Vec @@ -23,13 +24,18 @@ where I: IntoIterator, R: TxReceipt + 'a, { + if !filter.matches_block(&block_num_hash) { + return vec![]; + } + let mut all_logs = Vec::new(); // Tracks the index of a log in the entire block. let mut log_index: u64 = 0; + // Iterate over transaction hashes and receipts and append matching logs. for (receipt_idx, (tx_hash, receipt)) in tx_hashes_and_receipts.into_iter().enumerate() { for log in receipt.logs() { - if log_matches_filter(block_num_hash, log, filter) { + if filter.matches(log) { let log = Log { inner: log.clone(), block_hash: Some(block_num_hash.hash), @@ -39,7 +45,7 @@ where transaction_index: Some(receipt_idx as u64), log_index: Some(log_index), removed, - block_timestamp: None, + block_timestamp: Some(block_timestamp), }; all_logs.push(log); } @@ -63,7 +69,7 @@ pub enum ProviderOrBlock<'a, P: BlockReader> { pub fn append_matching_block_logs

( all_logs: &mut Vec, provider_or_block: ProviderOrBlock<'_, P>, - filter: &FilteredParams, + filter: &Filter, block_num_hash: BlockNumHash, receipts: &[P::Receipt], removed: bool, @@ -86,7 +92,7 @@ where let mut transaction_hash = None; for log in receipt.logs() { - if log_matches_filter(block_num_hash, log, filter) { + if filter.matches(log) { // if this is the first match in the receipt's logs, look up the transaction hash if transaction_hash.is_none() { transaction_hash = match &provider_or_block { @@ -139,24 +145,9 @@ where Ok(()) } -/// Returns true if the log matches the filter and should be included -pub fn log_matches_filter( - block: BlockNumHash, - log: &alloy_primitives::Log, - params: &FilteredParams, -) -> bool { - if params.filter.is_some() && - (!params.filter_block_range(block.number) || - !params.filter_block_hash(block.hash) || - !params.filter_address(&log.address) || - !params.filter_topics(log.topics())) - { - return false - } - true -} - -/// Computes the block range based on the filter range and current block numbers +/// Computes the block range based on the filter range and current block numbers. +/// +/// This returns `(min(best,from), min(best,to))`. pub fn get_filter_block_range( from_block: Option, to_block: Option, diff --git a/crates/rpc/rpc-eth-types/src/pending_block.rs b/crates/rpc/rpc-eth-types/src/pending_block.rs index 7990e2334b1..3150fffdc56 100644 --- a/crates/rpc/rpc-eth-types/src/pending_block.rs +++ b/crates/rpc/rpc-eth-types/src/pending_block.rs @@ -2,30 +2,35 @@ //! //! Types used in block building. -use std::time::Instant; +use std::{sync::Arc, time::Instant}; -use alloy_consensus::BlockHeader; +use crate::{block::BlockAndReceipts, utils::calculate_gas_used_and_next_log_index}; +use alloy_consensus::{BlockHeader, TxReceipt}; use alloy_eips::{BlockId, BlockNumberOrTag}; -use alloy_primitives::B256; +use alloy_primitives::{BlockHash, TxHash, B256}; use derive_more::Constructor; +use reth_chain_state::{BlockState, ExecutedBlock}; use reth_ethereum_primitives::Receipt; -use reth_evm::EvmEnv; -use reth_primitives_traits::{Block, RecoveredBlock, SealedHeader}; +use reth_evm::{ConfigureEvm, EvmEnvFor}; +use reth_primitives_traits::{ + Block, BlockTy, IndexedTx, NodePrimitives, ReceiptTy, RecoveredBlock, SealedHeader, +}; +use reth_rpc_convert::{transaction::ConvertReceiptInput, RpcConvert, RpcTypes}; -/// Configured [`EvmEnv`] for a pending block. +/// Configured [`reth_evm::EvmEnv`] for a pending block. #[derive(Debug, Clone, Constructor)] -pub struct PendingBlockEnv { - /// Configured [`EvmEnv`] for the pending block. - pub evm_env: EvmEnv, +pub struct PendingBlockEnv { + /// Configured [`reth_evm::EvmEnv`] for the pending block. + pub evm_env: EvmEnvFor, /// Origin block for the config - pub origin: PendingBlockEnvOrigin, + pub origin: PendingBlockEnvOrigin, ReceiptTy>, } /// The origin for a configured [`PendingBlockEnv`] #[derive(Clone, Debug)] pub enum PendingBlockEnvOrigin { /// The pending block as received from the CL. - ActualPending(RecoveredBlock, Vec), + ActualPending(Arc>, Arc>), /// The _modified_ header of the latest block. /// /// This derives the pending state based on the latest header by modifying: @@ -42,7 +47,7 @@ impl PendingBlockEnvOrigin { } /// Consumes the type and returns the actual pending block. - pub fn into_actual_pending(self) -> Option> { + pub fn into_actual_pending(self) -> Option>> { match self { Self::ActualPending(block, _) => Some(block), _ => None, @@ -73,13 +78,108 @@ impl PendingBlockEnvOrigin { } } +/// A type alias for a pair of an [`Arc`] wrapped [`RecoveredBlock`] and a vector of +/// [`NodePrimitives::Receipt`]. +pub type PendingBlockAndReceipts = BlockAndReceipts; + /// Locally built pending block for `pending` tag. -#[derive(Debug, Constructor)] -pub struct PendingBlock { +#[derive(Debug, Clone, Constructor)] +pub struct PendingBlock { /// Timestamp when the pending block is considered outdated. pub expires_at: Instant, - /// The locally built pending block. - pub block: RecoveredBlock, /// The receipts for the pending block - pub receipts: Vec, + pub receipts: Arc>>, + /// The locally built pending block with execution output. + pub executed_block: ExecutedBlock, +} + +impl PendingBlock { + /// Creates a new instance of [`PendingBlock`] with `executed_block` as its output that should + /// not be used past `expires_at`. + pub fn with_executed_block(expires_at: Instant, executed_block: ExecutedBlock) -> Self { + Self { + expires_at, + receipts: Arc::new( + executed_block.execution_output.receipts.iter().flatten().cloned().collect(), + ), + executed_block, + } + } + + /// Returns the locally built pending [`RecoveredBlock`]. + pub const fn block(&self) -> &Arc>> { + &self.executed_block.recovered_block + } + + /// Converts this [`PendingBlock`] into a pair of [`RecoveredBlock`] and a vector of + /// [`NodePrimitives::Receipt`]s, taking self. + pub fn into_block_and_receipts(self) -> PendingBlockAndReceipts { + BlockAndReceipts { block: self.executed_block.recovered_block, receipts: self.receipts } + } + + /// Returns a pair of [`RecoveredBlock`] and a vector of [`NodePrimitives::Receipt`]s by + /// cloning from borrowed self. + pub fn to_block_and_receipts(&self) -> PendingBlockAndReceipts { + BlockAndReceipts { + block: self.executed_block.recovered_block.clone(), + receipts: self.receipts.clone(), + } + } + + /// Returns a hash of the parent block for this `executed_block`. + pub fn parent_hash(&self) -> BlockHash { + self.executed_block.recovered_block().parent_hash() + } + + /// Finds a transaction by hash and returns it along with its corresponding receipt. + /// + /// Returns `None` if the transaction is not found in this block. + pub fn find_transaction_and_receipt_by_hash( + &self, + tx_hash: TxHash, + ) -> Option<(IndexedTx<'_, N::Block>, &N::Receipt)> { + let indexed_tx = self.executed_block.recovered_block().find_indexed(tx_hash)?; + let receipt = self.receipts.get(indexed_tx.index())?; + Some((indexed_tx, receipt)) + } + + /// Returns the rpc transaction receipt for the given transaction hash if it exists. + /// + /// This uses the given converter to turn [`Self::find_transaction_and_receipt_by_hash`] into + /// the rpc format. + pub fn find_and_convert_transaction_receipt( + &self, + tx_hash: TxHash, + converter: &C, + ) -> Option::Receipt, C::Error>> + where + C: RpcConvert, + { + let (tx, receipt) = self.find_transaction_and_receipt_by_hash(tx_hash)?; + let meta = tx.meta(); + let all_receipts = &self.receipts; + + let (gas_used, next_log_index) = + calculate_gas_used_and_next_log_index(meta.index, all_receipts); + + converter + .convert_receipts_with_block( + vec![ConvertReceiptInput { + tx: tx.recovered_tx(), + gas_used: receipt.cumulative_gas_used() - gas_used, + receipt: receipt.clone(), + next_log_index, + meta, + }], + self.executed_block.sealed_block(), + ) + .map(|mut receipts| receipts.pop()) + .transpose() + } +} + +impl From> for BlockState { + fn from(pending_block: PendingBlock) -> Self { + Self::new(pending_block.executed_block) + } } diff --git a/crates/rpc/rpc-eth-types/src/receipt.rs b/crates/rpc/rpc-eth-types/src/receipt.rs index 6bf2c5318ef..48dbf1e5add 100644 --- a/crates/rpc/rpc-eth-types/src/receipt.rs +++ b/crates/rpc/rpc-eth-types/src/receipt.rs @@ -1,83 +1,40 @@ //! RPC receipt response builder, extends a layer one receipt with layer two data. -use super::EthResult; -use alloy_consensus::{transaction::TransactionMeta, ReceiptEnvelope, TxReceipt}; +use crate::EthApiError; +use alloy_consensus::{ReceiptEnvelope, Transaction}; use alloy_eips::eip7840::BlobParams; use alloy_primitives::{Address, TxKind}; -use alloy_rpc_types_eth::{Log, ReceiptWithBloom, TransactionReceipt}; -use reth_ethereum_primitives::{Receipt, TransactionSigned, TxType}; -use reth_primitives_traits::SignedTransaction; +use alloy_rpc_types_eth::{Log, TransactionReceipt}; +use reth_chainspec::EthChainSpec; +use reth_ethereum_primitives::Receipt; +use reth_primitives_traits::{NodePrimitives, TransactionMeta}; +use reth_rpc_convert::transaction::{ConvertReceiptInput, ReceiptConverter}; +use std::sync::Arc; /// Builds an [`TransactionReceipt`] obtaining the inner receipt envelope from the given closure. -pub fn build_receipt( - transaction: &T, - meta: TransactionMeta, - receipt: &R, - all_receipts: &[R], +pub fn build_receipt( + input: ConvertReceiptInput<'_, N>, blob_params: Option, - build_envelope: impl FnOnce(ReceiptWithBloom>) -> E, -) -> EthResult> + build_rpc_receipt: impl FnOnce(N::Receipt, usize, TransactionMeta) -> E, +) -> TransactionReceipt where - R: TxReceipt, - T: SignedTransaction, + N: NodePrimitives, { - // Note: we assume this transaction is valid, because it's mined (or part of pending block) - // and we don't need to check for pre EIP-2 - let from = transaction.recover_signer_unchecked()?; + let ConvertReceiptInput { tx, meta, receipt, gas_used, next_log_index } = input; + let from = tx.signer(); - // get the previous transaction cumulative gas used - let gas_used = if meta.index == 0 { - receipt.cumulative_gas_used() - } else { - let prev_tx_idx = (meta.index - 1) as usize; - all_receipts - .get(prev_tx_idx) - .map(|prev_receipt| receipt.cumulative_gas_used() - prev_receipt.cumulative_gas_used()) - .unwrap_or_default() - }; - - let blob_gas_used = transaction.blob_gas_used(); + let blob_gas_used = tx.blob_gas_used(); // Blob gas price should only be present if the transaction is a blob transaction let blob_gas_price = blob_gas_used.and_then(|_| Some(blob_params?.calc_blob_fee(meta.excess_blob_gas?))); - let logs_bloom = receipt.bloom(); - - // get number of logs in the block - let mut num_logs = 0; - for prev_receipt in all_receipts.iter().take(meta.index as usize) { - num_logs += prev_receipt.logs().len(); - } - - let logs: Vec = receipt - .logs() - .iter() - .enumerate() - .map(|(tx_log_idx, log)| Log { - inner: log.clone(), - block_hash: Some(meta.block_hash), - block_number: Some(meta.block_number), - block_timestamp: Some(meta.timestamp), - transaction_hash: Some(meta.tx_hash), - transaction_index: Some(meta.index), - log_index: Some((num_logs + tx_log_idx) as u64), - removed: false, - }) - .collect(); - - let rpc_receipt = alloy_rpc_types_eth::Receipt { - status: receipt.status_or_post_state(), - cumulative_gas_used: receipt.cumulative_gas_used(), - logs, - }; - - let (contract_address, to) = match transaction.kind() { - TxKind::Create => (Some(from.create(transaction.nonce())), None), + let (contract_address, to) = match tx.kind() { + TxKind::Create => (Some(from.create(tx.nonce())), None), TxKind::Call(addr) => (None, Some(Address(*addr))), }; - Ok(TransactionReceipt { - inner: build_envelope(ReceiptWithBloom { receipt: rpc_receipt, logs_bloom }), + TransactionReceipt { + inner: build_rpc_receipt(receipt, next_log_index, meta), transaction_hash: meta.tx_hash, transaction_index: Some(meta.index), block_hash: Some(meta.block_hash), @@ -86,54 +43,76 @@ where to, gas_used, contract_address, - effective_gas_price: transaction.effective_gas_price(meta.base_fee), + effective_gas_price: tx.effective_gas_price(meta.base_fee), // EIP-4844 fields blob_gas_price, blob_gas_used, - }) + } } -/// Receipt response builder. -#[derive(Debug)] -pub struct EthReceiptBuilder { - /// The base response body, contains L1 fields. - pub base: TransactionReceipt, +/// Converter for Ethereum receipts. +#[derive(derive_more::Debug)] +pub struct EthReceiptConverter< + ChainSpec, + Builder = fn(Receipt, usize, TransactionMeta) -> ReceiptEnvelope, +> { + chain_spec: Arc, + #[debug(skip)] + build_rpc_receipt: Builder, } -impl EthReceiptBuilder { - /// Returns a new builder with the base response body (L1 fields) set. - /// - /// Note: This requires _all_ block receipts because we need to calculate the gas used by the - /// transaction. - pub fn new( - transaction: &TransactionSigned, - meta: TransactionMeta, - receipt: &Receipt, - all_receipts: &[Receipt], - blob_params: Option, - ) -> EthResult { - let base = build_receipt( - transaction, - meta, - receipt, - all_receipts, - blob_params, - |receipt_with_bloom| match receipt.tx_type { - TxType::Legacy => ReceiptEnvelope::Legacy(receipt_with_bloom), - TxType::Eip2930 => ReceiptEnvelope::Eip2930(receipt_with_bloom), - TxType::Eip1559 => ReceiptEnvelope::Eip1559(receipt_with_bloom), - TxType::Eip4844 => ReceiptEnvelope::Eip4844(receipt_with_bloom), - TxType::Eip7702 => ReceiptEnvelope::Eip7702(receipt_with_bloom), - #[expect(unreachable_patterns)] - _ => unreachable!(), +impl Clone for EthReceiptConverter +where + Builder: Clone, +{ + fn clone(&self) -> Self { + Self { + chain_spec: self.chain_spec.clone(), + build_rpc_receipt: self.build_rpc_receipt.clone(), + } + } +} + +impl EthReceiptConverter { + /// Creates a new converter with the given chain spec. + pub const fn new(chain_spec: Arc) -> Self { + Self { + chain_spec, + build_rpc_receipt: |receipt, next_log_index, meta| { + receipt.into_rpc(next_log_index, meta).into() }, - )?; + } + } - Ok(Self { base }) + /// Sets new builder for the converter. + pub fn with_builder( + self, + build_rpc_receipt: Builder, + ) -> EthReceiptConverter { + EthReceiptConverter { chain_spec: self.chain_spec, build_rpc_receipt } } +} + +impl ReceiptConverter for EthReceiptConverter +where + N: NodePrimitives, + ChainSpec: EthChainSpec + 'static, + Builder: Fn(N::Receipt, usize, TransactionMeta) -> Rpc + 'static, +{ + type RpcReceipt = TransactionReceipt; + type Error = EthApiError; + + fn convert_receipts( + &self, + inputs: Vec>, + ) -> Result, Self::Error> { + let mut receipts = Vec::with_capacity(inputs.len()); + + for input in inputs { + let blob_params = self.chain_spec.blob_params_at_timestamp(input.meta.timestamp); + receipts.push(build_receipt(input, blob_params, &self.build_rpc_receipt)); + } - /// Builds a receipt response from the base response body, and any set additional fields. - pub fn build(self) -> TransactionReceipt { - self.base + Ok(receipts) } } diff --git a/crates/rpc/rpc-eth-types/src/simulate.rs b/crates/rpc/rpc-eth-types/src/simulate.rs index a82bb934f21..ec63443da3d 100644 --- a/crates/rpc/rpc-eth-types/src/simulate.rs +++ b/crates/rpc/rpc-eth-types/src/simulate.rs @@ -7,25 +7,24 @@ use crate::{ }, EthApiError, RevertError, }; -use alloy_consensus::{BlockHeader, Transaction as _, TxType}; +use alloy_consensus::{transaction::TxHashRef, BlockHeader, Transaction as _}; use alloy_eips::eip2718::WithEncoded; +use alloy_network::TransactionBuilder; use alloy_rpc_types_eth::{ simulate::{SimCallResult, SimulateError, SimulatedBlock}, - transaction::TransactionRequest, - Block, BlockTransactionsKind, Header, + BlockTransactionsKind, }; use jsonrpsee_types::ErrorObject; use reth_evm::{ execute::{BlockBuilder, BlockBuilderOutcome, BlockExecutor}, Evm, }; -use reth_primitives_traits::{ - block::BlockTx, BlockBody as _, Recovered, RecoveredBlock, SignedTransaction, TxTy, -}; +use reth_primitives_traits::{BlockBody as _, BlockTy, NodePrimitives, Recovered, RecoveredBlock}; +use reth_rpc_convert::{RpcBlock, RpcConvert, RpcTxReq}; use reth_rpc_server_types::result::rpc_err; -use reth_rpc_types_compat::{block::from_block, TransactionCompat}; use reth_storage_api::noop::NoopProvider; use revm::{ + context::Block, context_interface::result::ExecutionResult, primitives::{Address, Bytes, TxKind}, Database, @@ -61,10 +60,12 @@ impl ToRpcError for EthSimulateError { /// given [`BlockExecutor`]. /// /// Returns all executed transactions and the result of the execution. +/// +/// [`TransactionRequest`]: alloy_rpc_types_eth::TransactionRequest #[expect(clippy::type_complexity)] pub fn execute_transactions( mut builder: S, - calls: Vec, + calls: Vec>, default_gas_limit: u64, chain_id: u64, tx_resp_builder: &T, @@ -77,7 +78,7 @@ pub fn execute_transactions( > where S: BlockBuilder>>>>, - T: TransactionCompat>, + T: RpcConvert, { builder.apply_pre_execution_changes()?; @@ -88,7 +89,7 @@ where let tx = resolve_transaction( call, default_gas_limit, - builder.evm().block().basefee, + builder.evm().block().basefee(), chain_id, builder.evm_mut().db_mut(), tx_resp_builder, @@ -111,8 +112,10 @@ where /// them into primitive transactions. /// /// This will set the defaults as defined in -pub fn resolve_transaction>( - mut tx: TransactionRequest, +/// +/// [`TransactionRequest`]: alloy_rpc_types_eth::TransactionRequest +pub fn resolve_transaction( + mut tx: RpcTxReq, default_gas_limit: u64, block_base_fee_per_gas: u64, chain_id: u64, @@ -121,57 +124,56 @@ pub fn resolve_transaction>( ) -> Result, EthApiError> where DB::Error: Into, + T: RpcConvert>, { // If we're missing any fields we try to fill nonce, gas and // gas price. - let tx_type = tx.preferred_type(); + let tx_type = tx.as_ref().output_tx_type(); - let from = if let Some(from) = tx.from { + let from = if let Some(from) = tx.as_ref().from() { from } else { - tx.from = Some(Address::ZERO); + tx.as_mut().set_from(Address::ZERO); Address::ZERO }; - if tx.nonce.is_none() { - tx.nonce = - Some(db.basic(from).map_err(Into::into)?.map(|acc| acc.nonce).unwrap_or_default()); + if tx.as_ref().nonce().is_none() { + tx.as_mut().set_nonce( + db.basic(from).map_err(Into::into)?.map(|acc| acc.nonce).unwrap_or_default(), + ); } - if tx.gas.is_none() { - tx.gas = Some(default_gas_limit); + if tx.as_ref().gas_limit().is_none() { + tx.as_mut().set_gas_limit(default_gas_limit); } - if tx.chain_id.is_none() { - tx.chain_id = Some(chain_id); + if tx.as_ref().chain_id().is_none() { + tx.as_mut().set_chain_id(chain_id); } - if tx.to.is_none() { - tx.to = Some(TxKind::Create); + if tx.as_ref().kind().is_none() { + tx.as_mut().set_kind(TxKind::Create); } // if we can't build the _entire_ transaction yet, we need to check the fee values - if tx.buildable_type().is_none() { - match tx_type { - TxType::Legacy | TxType::Eip2930 => { - if tx.gas_price.is_none() { - tx.gas_price = Some(block_base_fee_per_gas as u128); - } + if tx.as_ref().output_tx_type_checked().is_none() { + if tx_type.is_legacy() || tx_type.is_eip2930() { + if tx.as_ref().gas_price().is_none() { + tx.as_mut().set_gas_price(block_base_fee_per_gas as u128); } - _ => { - // set dynamic 1559 fees - if tx.max_fee_per_gas.is_none() { - let mut max_fee_per_gas = block_base_fee_per_gas as u128; - if let Some(prio_fee) = tx.max_priority_fee_per_gas { - // if a prio fee is provided we need to select the max fee accordingly - // because the base fee must be higher than the prio fee. - max_fee_per_gas = prio_fee.max(max_fee_per_gas); - } - tx.max_fee_per_gas = Some(max_fee_per_gas); - } - if tx.max_priority_fee_per_gas.is_none() { - tx.max_priority_fee_per_gas = Some(0); + } else { + // set dynamic 1559 fees + if tx.as_ref().max_fee_per_gas().is_none() { + let mut max_fee_per_gas = block_base_fee_per_gas as u128; + if let Some(prio_fee) = tx.as_ref().max_priority_fee_per_gas() { + // if a prio fee is provided we need to select the max fee accordingly + // because the base fee must be higher than the prio fee. + max_fee_per_gas = prio_fee.max(max_fee_per_gas); } + tx.as_mut().set_max_fee_per_gas(max_fee_per_gas); + } + if tx.as_ref().max_priority_fee_per_gas().is_none() { + tx.as_mut().set_max_priority_fee_per_gas(0); } } } @@ -184,31 +186,29 @@ where } /// Handles outputs of the calls execution and builds a [`SimulatedBlock`]. -#[expect(clippy::type_complexity)] -pub fn build_simulated_block( - block: RecoveredBlock, +pub fn build_simulated_block( + block: RecoveredBlock>, results: Vec>, - full_transactions: bool, + txs_kind: BlockTransactionsKind, tx_resp_builder: &T, -) -> Result>>, T::Error> +) -> Result>, T::Error> where - T: TransactionCompat, Error: FromEthApiError + FromEvmHalt>, - B: reth_primitives_traits::Block, + T: RpcConvert>, { let mut calls: Vec = Vec::with_capacity(results.len()); let mut log_index = 0; - for (index, (result, tx)) in results.iter().zip(block.body().transactions()).enumerate() { + for (index, (result, tx)) in results.into_iter().zip(block.body().transactions()).enumerate() { let call = match result { ExecutionResult::Halt { reason, gas_used } => { - let error = T::Error::from_evm_halt(reason.clone(), tx.gas_limit()); + let error = T::Error::from_evm_halt(reason, tx.gas_limit()); SimCallResult { return_data: Bytes::new(), error: Some(SimulateError { message: error.to_string(), code: error.into().code(), }), - gas_used: *gas_used, + gas_used, logs: Vec::new(), status: false, } @@ -216,26 +216,26 @@ where ExecutionResult::Revert { output, gas_used } => { let error = RevertError::new(output.clone()); SimCallResult { - return_data: output.clone(), + return_data: output, error: Some(SimulateError { code: error.error_code(), message: error.to_string(), }), - gas_used: *gas_used, + gas_used, status: false, logs: Vec::new(), } } ExecutionResult::Success { output, gas_used, logs, .. } => SimCallResult { - return_data: output.clone().into_data(), + return_data: output.into_data(), error: None, - gas_used: *gas_used, + gas_used, logs: logs - .iter() + .into_iter() .map(|log| { log_index += 1; alloy_rpc_types_eth::Log { - inner: log.clone(), + inner: log, log_index: Some(log_index - 1), transaction_index: Some(index as u64), transaction_hash: Some(*tx.tx_hash()), @@ -252,9 +252,10 @@ where calls.push(call); } - let txs_kind = - if full_transactions { BlockTransactionsKind::Full } else { BlockTransactionsKind::Hashes }; - - let block = from_block(block, txs_kind, tx_resp_builder)?; + let block = block.into_rpc_block( + txs_kind, + |tx, tx_info| tx_resp_builder.fill(tx, tx_info), + |header, size| tx_resp_builder.convert_header(header, size), + )?; Ok(SimulatedBlock { inner: block, calls }) } diff --git a/crates/rpc/rpc-eth-types/src/transaction.rs b/crates/rpc/rpc-eth-types/src/transaction.rs index 34d80d91145..3d099f01188 100644 --- a/crates/rpc/rpc-eth-types/src/transaction.rs +++ b/crates/rpc/rpc-eth-types/src/transaction.rs @@ -2,11 +2,21 @@ //! //! Transaction wrapper that labels transaction with its origin. -use alloy_primitives::B256; +use alloy_primitives::{Bytes, B256}; use alloy_rpc_types_eth::TransactionInfo; use reth_ethereum_primitives::TransactionSigned; -use reth_primitives_traits::{Recovered, SignedTransaction}; -use reth_rpc_types_compat::TransactionCompat; +use reth_primitives_traits::{NodePrimitives, Recovered, SignedTransaction}; +use reth_rpc_convert::{RpcConvert, RpcTransaction}; +use serde::{Deserialize, Serialize}; + +/// Response type for `eth_fillTransaction` RPC method. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FillTransactionResult { + /// RLP-encoded transaction bytes + pub raw: Bytes, + /// Filled transaction object + pub tx: T, +} /// Represents from where a transaction was fetched. #[derive(Debug, Clone, Eq, PartialEq)] @@ -39,10 +49,13 @@ impl TransactionSource { } /// Conversion into network specific transaction type. - pub fn into_transaction>( + pub fn into_transaction( self, resp_builder: &Builder, - ) -> Result { + ) -> Result, Builder::Error> + where + Builder: RpcConvert>, + { match self { Self::Pool(tx) => resp_builder.fill_pending(tx), Self::Block { transaction, index, block_hash, block_number, base_fee } => { diff --git a/crates/rpc/rpc-eth-types/src/tx_forward.rs b/crates/rpc/rpc-eth-types/src/tx_forward.rs new file mode 100644 index 00000000000..07499a5a9f5 --- /dev/null +++ b/crates/rpc/rpc-eth-types/src/tx_forward.rs @@ -0,0 +1,22 @@ +//! Consist of types adjacent to the fee history cache and its configs + +use alloy_rpc_client::RpcClient; +use reqwest::Url; +use serde::{Deserialize, Serialize}; +use std::fmt::Debug; + +/// Configuration for the transaction forwarder. +#[derive(Debug, PartialEq, Eq, Clone, Default, Serialize, Deserialize)] +pub struct ForwardConfig { + /// The raw transaction forwarder. + /// + /// Default is `None` + pub tx_forwarder: Option, +} + +impl ForwardConfig { + /// Builds an [`RpcClient`] from the forwarder URL, if configured. + pub fn forwarder_client(&self) -> Option { + self.tx_forwarder.clone().map(RpcClient::new_http) + } +} diff --git a/crates/rpc/rpc-eth-types/src/utils.rs b/crates/rpc/rpc-eth-types/src/utils.rs index e7873920a9c..4a613c1915b 100644 --- a/crates/rpc/rpc-eth-types/src/utils.rs +++ b/crates/rpc/rpc-eth-types/src/utils.rs @@ -1,24 +1,47 @@ //! Commonly used code snippets use super::{EthApiError, EthResult}; +use alloy_consensus::TxReceipt; use reth_primitives_traits::{Recovered, SignedTransaction}; use std::future::Future; +/// Calculates the gas used and next log index for a transaction at the given index +pub fn calculate_gas_used_and_next_log_index( + tx_index: u64, + all_receipts: &[impl TxReceipt], +) -> (u64, usize) { + let mut gas_used = 0; + let mut next_log_index = 0; + + if tx_index > 0 { + for receipt in all_receipts.iter().take(tx_index as usize) { + gas_used = receipt.cumulative_gas_used(); + next_log_index += receipt.logs().len(); + } + } + + (gas_used, next_log_index) +} + /// Recovers a [`SignedTransaction`] from an enveloped encoded byte stream. /// /// This is a helper function that returns the appropriate RPC-specific error if the input data is /// malformed. /// -/// See [`alloy_eips::eip2718::Decodable2718::decode_2718`] -pub fn recover_raw_transaction(mut data: &[u8]) -> EthResult> { +/// This function uses [`alloy_eips::eip2718::Decodable2718::decode_2718_exact`] to ensure +/// that the entire input buffer is consumed and no trailing bytes are allowed. +/// +/// See [`alloy_eips::eip2718::Decodable2718::decode_2718_exact`] +pub fn recover_raw_transaction(data: &[u8]) -> EthResult> { if data.is_empty() { return Err(EthApiError::EmptyRawTransactionData) } let transaction = - T::decode_2718(&mut data).map_err(|_| EthApiError::FailedToDecodeSignedTransaction)?; + T::decode_2718_exact(data).map_err(|_| EthApiError::FailedToDecodeSignedTransaction)?; - transaction.try_into_recovered().or(Err(EthApiError::InvalidTransactionSignature)) + SignedTransaction::try_into_recovered(transaction) + .or(Err(EthApiError::InvalidTransactionSignature)) } /// Performs a binary search within a given block range to find the desired block number. @@ -57,6 +80,19 @@ where Ok(num) } +/// Calculates the blob gas used ratio for a block, accounting for the case where +/// `max_blob_gas_per_block` is zero. +/// +/// Returns `0.0` if `blob_gas_used` is `0`, otherwise returns the ratio +/// `blob_gas_used/max_blob_gas_per_block`. +pub fn checked_blob_gas_used_ratio(blob_gas_used: u64, max_blob_gas_per_block: u64) -> f64 { + if blob_gas_used == 0 { + 0.0 + } else { + blob_gas_used as f64 / max_blob_gas_per_block as f64 + } +} + #[cfg(test)] mod tests { use super::*; @@ -83,4 +119,16 @@ mod tests { binary_search(1, 10, |mid| Box::pin(async move { Ok(mid >= 11) })).await; assert_eq!(num, Ok(10)); } + + #[test] + fn test_checked_blob_gas_used_ratio() { + // No blob gas used, max blob gas per block is 0 + assert_eq!(checked_blob_gas_used_ratio(0, 0), 0.0); + // Blob gas used is zero, max blob gas per block is non-zero + assert_eq!(checked_blob_gas_used_ratio(0, 100), 0.0); + // Blob gas used is non-zero, max blob gas per block is non-zero + assert_eq!(checked_blob_gas_used_ratio(50, 100), 0.5); + // Blob gas used is non-zero and equal to max blob gas per block + assert_eq!(checked_blob_gas_used_ratio(100, 100), 1.0); + } } diff --git a/crates/rpc/rpc-layer/src/auth_layer.rs b/crates/rpc/rpc-layer/src/auth_layer.rs index af8a2045ede..345d5c98d48 100644 --- a/crates/rpc/rpc-layer/src/auth_layer.rs +++ b/crates/rpc/rpc-layer/src/auth_layer.rs @@ -155,7 +155,7 @@ mod tests { use crate::JwtAuthValidator; use alloy_rpc_types_engine::{Claims, JwtError, JwtSecret}; use jsonrpsee::{ - server::{RandomStringIdProvider, ServerBuilder, ServerHandle}, + server::{RandomStringIdProvider, ServerBuilder, ServerConfig, ServerHandle}, RpcModule, }; use reqwest::{header, StatusCode}; @@ -260,7 +260,9 @@ mod tests { // Create a layered server let server = ServerBuilder::default() - .set_id_provider(RandomStringIdProvider::new(16)) + .set_config( + ServerConfig::builder().set_id_provider(RandomStringIdProvider::new(16)).build(), + ) .set_http_middleware(middleware) .build(addr.parse::().unwrap()) .await diff --git a/crates/rpc/rpc-layer/src/jwt_validator.rs b/crates/rpc/rpc-layer/src/jwt_validator.rs index 917773adc9f..8804ab398ac 100644 --- a/crates/rpc/rpc-layer/src/jwt_validator.rs +++ b/crates/rpc/rpc-layer/src/jwt_validator.rs @@ -47,8 +47,12 @@ fn get_bearer(headers: &HeaderMap) -> Option { let header = headers.get(header::AUTHORIZATION)?; let auth: &str = header.to_str().ok()?; let prefix = "Bearer "; - let index = auth.find(prefix)?; - let token: &str = &auth[index + prefix.len()..]; + + if !auth.starts_with(prefix) { + return None; + } + + let token: &str = &auth[prefix.len()..]; Some(token.into()) } @@ -93,4 +97,28 @@ mod tests { let token = get_bearer(&headers); assert!(token.is_none()); } + + #[test] + fn auth_header_bearer_in_middle() { + // Test that "Bearer " must be at the start of the header, not in the middle + let jwt = "valid_token"; + let bearer = format!("NotBearer Bearer {jwt}"); + let mut headers = HeaderMap::new(); + headers.insert(header::AUTHORIZATION, bearer.parse().unwrap()); + let token = get_bearer(&headers); + // Function should return None since "Bearer " is not at the start + assert!(token.is_none()); + } + + #[test] + fn auth_header_bearer_without_space() { + // Test that "BearerBearer" is not treated as "Bearer " + let jwt = "valid_token"; + let bearer = format!("BearerBearer {jwt}"); + let mut headers = HeaderMap::new(); + headers.insert(header::AUTHORIZATION, bearer.parse().unwrap()); + let token = get_bearer(&headers); + // Function should return None since header doesn't start with "Bearer " + assert!(token.is_none()); + } } diff --git a/crates/rpc/rpc-layer/src/lib.rs b/crates/rpc/rpc-layer/src/lib.rs index 79a92114524..b7f6f29cbb9 100644 --- a/crates/rpc/rpc-layer/src/lib.rs +++ b/crates/rpc/rpc-layer/src/lib.rs @@ -6,7 +6,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] use http::HeaderMap; use jsonrpsee_http_client::HttpResponse; diff --git a/crates/rpc/rpc-server-types/src/constants.rs b/crates/rpc/rpc-server-types/src/constants.rs index 61ca892deeb..8861af7b54d 100644 --- a/crates/rpc/rpc-server-types/src/constants.rs +++ b/crates/rpc/rpc-server-types/src/constants.rs @@ -1,4 +1,4 @@ -use std::cmp::max; +use std::{cmp::max, time::Duration}; /// The default port for the http server pub const DEFAULT_HTTP_RPC_PORT: u16 = 8545; @@ -61,6 +61,9 @@ pub const DEFAULT_TX_FEE_CAP_WEI: u128 = 1_000_000_000_000_000_000u128; /// second block time, and a month on a 2 second block time. pub const MAX_ETH_PROOF_WINDOW: u64 = 28 * 24 * 60 * 60 / 2; +/// Default timeout for send raw transaction sync in seconds. +pub const RPC_DEFAULT_SEND_RAW_TX_SYNC_TIMEOUT_SECS: Duration = Duration::from_secs(30); + /// GPO specific constants pub mod gas_oracle { use alloy_primitives::U256; @@ -87,9 +90,6 @@ pub mod gas_oracle { /// The default minimum gas price, under which the sample will be ignored pub const DEFAULT_IGNORE_GAS_PRICE: U256 = U256::from_limbs([2u64, 0, 0, 0]); - /// 0.0001 gwei, for Optimism fee suggestion - pub const DEFAULT_MIN_SUGGESTED_PRIORITY_FEE: U256 = U256::from_limbs([100_000u64, 0, 0, 0]); - /// The default gas limit for `eth_call` and adjacent calls. /// /// This is different from the default to regular 30M block gas limit @@ -107,18 +107,6 @@ pub mod gas_oracle { /// Cache specific constants pub mod cache { - // TODO: memory based limiter is currently disabled pending - /// Default cache size for the block cache: 500MB - /// - /// With an average block size of ~100kb this should be able to cache ~5000 blocks. - pub const DEFAULT_BLOCK_CACHE_SIZE_BYTES_MB: usize = 500; - - /// Default cache size for the receipts cache: 500MB - pub const DEFAULT_RECEIPT_CACHE_SIZE_BYTES_MB: usize = 500; - - /// Default cache size for the env cache: 1MB - pub const DEFAULT_ENV_CACHE_SIZE_BYTES_MB: usize = 1; - /// Default cache size for the block cache: 5000 blocks. pub const DEFAULT_BLOCK_CACHE_MAX_LEN: u32 = 5000; diff --git a/crates/rpc/rpc-server-types/src/lib.rs b/crates/rpc/rpc-server-types/src/lib.rs index c20b578816b..2db91a5edf4 100644 --- a/crates/rpc/rpc-server-types/src/lib.rs +++ b/crates/rpc/rpc-server-types/src/lib.rs @@ -5,7 +5,7 @@ html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256", issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(test), warn(unused_crate_dependencies))] /// Common RPC constants. @@ -13,6 +13,9 @@ pub mod constants; pub mod result; mod module; -pub use module::{RethRpcModule, RpcModuleSelection}; +pub use module::{ + DefaultRpcModuleValidator, LenientRpcModuleValidator, RethRpcModule, RpcModuleSelection, + RpcModuleValidator, +}; pub use result::ToRpcResult; diff --git a/crates/rpc/rpc-server-types/src/module.rs b/crates/rpc/rpc-server-types/src/module.rs index fdca41cc196..db9268d5d6e 100644 --- a/crates/rpc/rpc-server-types/src/module.rs +++ b/crates/rpc/rpc-server-types/src/module.rs @@ -1,7 +1,7 @@ use std::{collections::HashSet, fmt, str::FromStr}; use serde::{Deserialize, Serialize, Serializer}; -use strum::{AsRefStr, EnumIter, IntoStaticStr, ParseError, VariantArray, VariantNames}; +use strum::{ParseError, VariantNames}; /// Describes the modules that should be installed. /// @@ -98,12 +98,17 @@ impl RpcModuleSelection { } } + /// Returns true if all modules are selected + pub const fn is_all(&self) -> bool { + matches!(self, Self::All) + } + /// Returns an iterator over all configured [`RethRpcModule`] pub fn iter_selection(&self) -> Box + '_> { match self { Self::All => Box::new(RethRpcModule::modules().into_iter()), - Self::Standard => Box::new(Self::STANDARD_MODULES.iter().copied()), - Self::Selection(s) => Box::new(s.iter().copied()), + Self::Standard => Box::new(Self::STANDARD_MODULES.iter().cloned()), + Self::Selection(s) => Box::new(s.iter().cloned()), } } @@ -149,6 +154,64 @@ impl RpcModuleSelection { Self::Selection(s) => s.contains(module), } } + + /// Adds a module to the selection. + /// + /// If the selection is `All`, this is a no-op. + /// Otherwise, converts to a `Selection` and adds the module. + pub fn push(&mut self, module: RethRpcModule) { + if !self.is_all() { + let mut modules = self.to_selection(); + modules.insert(module); + *self = Self::Selection(modules); + } + } + + /// Returns a new selection with the given module added. + /// + /// If the selection is `All`, returns `All`. + /// Otherwise, converts to a `Selection` and adds the module. + pub fn append(self, module: RethRpcModule) -> Self { + if self.is_all() { + Self::All + } else { + let mut modules = self.into_selection(); + modules.insert(module); + Self::Selection(modules) + } + } + + /// Extends the selection with modules from an iterator. + /// + /// If the selection is `All`, this is a no-op. + /// Otherwise, converts to a `Selection` and adds the modules. + pub fn extend(&mut self, iter: I) + where + I: IntoIterator, + { + if !self.is_all() { + let mut modules = self.to_selection(); + modules.extend(iter); + *self = Self::Selection(modules); + } + } + + /// Returns a new selection with modules from an iterator added. + /// + /// If the selection is `All`, returns `All`. + /// Otherwise, converts to a `Selection` and adds the modules. + pub fn extended(self, iter: I) -> Self + where + I: IntoIterator, + { + if self.is_all() { + Self::All + } else { + let mut modules = self.into_selection(); + modules.extend(iter); + Self::Selection(modules) + } + } } impl From<&HashSet> for RpcModuleSelection { @@ -165,7 +228,7 @@ impl From> for RpcModuleSelection { impl From<&[RethRpcModule]> for RpcModuleSelection { fn from(s: &[RethRpcModule]) -> Self { - Self::Selection(s.iter().copied().collect()) + Self::Selection(s.iter().cloned().collect()) } } @@ -177,7 +240,7 @@ impl From> for RpcModuleSelection { impl From<[RethRpcModule; N]> for RpcModuleSelection { fn from(s: [RethRpcModule; N]) -> Self { - Self::Selection(s.iter().copied().collect()) + Self::Selection(s.iter().cloned().collect()) } } @@ -186,7 +249,7 @@ impl<'a> FromIterator<&'a RethRpcModule> for RpcModuleSelection { where I: IntoIterator, { - iter.into_iter().copied().collect() + iter.into_iter().cloned().collect() } } @@ -230,20 +293,7 @@ impl fmt::Display for RpcModuleSelection { } /// Represents RPC modules that are supported by reth -#[derive( - Debug, - Clone, - Copy, - Eq, - PartialEq, - Hash, - AsRefStr, - IntoStaticStr, - VariantNames, - VariantArray, - EnumIter, - Deserialize, -)] +#[derive(Debug, Clone, Eq, PartialEq, Hash, VariantNames, Deserialize)] #[serde(rename_all = "snake_case")] #[strum(serialize_all = "kebab-case")] pub enum RethRpcModule { @@ -273,36 +323,90 @@ pub enum RethRpcModule { Miner, /// `mev_` module Mev, + /// Custom RPC module not part of the standard set + #[strum(default)] + #[serde(untagged)] + Other(String), } // === impl RethRpcModule === impl RethRpcModule { - /// Returns the number of variants in the enum + /// All standard variants (excludes Other) + const STANDARD_VARIANTS: &'static [Self] = &[ + Self::Admin, + Self::Debug, + Self::Eth, + Self::Net, + Self::Trace, + Self::Txpool, + Self::Web3, + Self::Rpc, + Self::Reth, + Self::Ots, + Self::Flashbots, + Self::Miner, + Self::Mev, + ]; + + /// Returns the number of standard variants (excludes Other) pub const fn variant_count() -> usize { - ::VARIANTS.len() + Self::STANDARD_VARIANTS.len() } - /// Returns all variant names of the enum + /// Returns all variant names including Other (for parsing) pub const fn all_variant_names() -> &'static [&'static str] { ::VARIANTS } - /// Returns all variants of the enum + /// Returns standard variant names (excludes "other") for CLI display + pub fn standard_variant_names() -> impl Iterator { + ::VARIANTS.iter().copied().filter(|&name| name != "other") + } + + /// Returns all standard variants (excludes Other) pub const fn all_variants() -> &'static [Self] { - ::VARIANTS + Self::STANDARD_VARIANTS } - /// Returns all variants of the enum - pub fn modules() -> impl IntoIterator { - use strum::IntoEnumIterator; - Self::iter() + /// Returns iterator over standard modules only + pub fn modules() -> impl IntoIterator + Clone { + Self::STANDARD_VARIANTS.iter().cloned() } /// Returns the string representation of the module. - #[inline] - pub fn as_str(&self) -> &'static str { - self.into() + pub fn as_str(&self) -> &str { + match self { + Self::Other(s) => s.as_str(), + _ => self.as_ref(), // Uses AsRefStr trait + } + } + + /// Returns true if this is an `Other` variant. + pub const fn is_other(&self) -> bool { + matches!(self, Self::Other(_)) + } +} + +impl AsRef for RethRpcModule { + fn as_ref(&self) -> &str { + match self { + Self::Other(s) => s.as_str(), + // For standard variants, use the derive-generated static strings + Self::Admin => "admin", + Self::Debug => "debug", + Self::Eth => "eth", + Self::Net => "net", + Self::Trace => "trace", + Self::Txpool => "txpool", + Self::Web3 => "web3", + Self::Rpc => "rpc", + Self::Reth => "reth", + Self::Ots => "ots", + Self::Flashbots => "flashbots", + Self::Miner => "miner", + Self::Mev => "mev", + } } } @@ -324,7 +428,8 @@ impl FromStr for RethRpcModule { "flashbots" => Self::Flashbots, "miner" => Self::Miner, "mev" => Self::Mev, - _ => return Err(ParseError::VariantNotFound), + // Any unknown module becomes Other + other => Self::Other(other.to_string()), }) } } @@ -347,7 +452,81 @@ impl Serialize for RethRpcModule { where S: Serializer, { - s.serialize_str(self.as_ref()) + s.serialize_str(self.as_str()) + } +} + +/// Trait for validating RPC module selections. +/// +/// This allows customizing how RPC module names are validated when parsing +/// CLI arguments or configuration. +pub trait RpcModuleValidator: Clone + Send + Sync + 'static { + /// Parse and validate an RPC module selection string. + fn parse_selection(s: &str) -> Result; + + /// Validates RPC module selection that was already parsed. + /// + /// This is used to validate modules that were parsed as `Other` variants + /// to ensure they meet the validation rules of the specific implementation. + fn validate_selection(modules: &RpcModuleSelection, arg_name: &str) -> Result<(), String> { + // Re-validate the modules using the parser's validator + // This is necessary because the clap value parser accepts any input + // and we need to validate according to the specific parser's rules + let RpcModuleSelection::Selection(module_set) = modules else { + // All or Standard variants are always valid + return Ok(()); + }; + + for module in module_set { + let RethRpcModule::Other(name) = module else { + // Standard modules are always valid + continue; + }; + + // Try to parse and validate using the configured validator + // This will check for typos and other validation rules + Self::parse_selection(name) + .map_err(|e| format!("Invalid RPC module '{name}' in {arg_name}: {e}"))?; + } + + Ok(()) + } +} + +/// Default validator that rejects unknown module names. +/// +/// This validator only accepts known RPC module names. +#[derive(Debug, Clone, Copy)] +pub struct DefaultRpcModuleValidator; + +impl RpcModuleValidator for DefaultRpcModuleValidator { + fn parse_selection(s: &str) -> Result { + // First try standard parsing + let selection = RpcModuleSelection::from_str(s) + .map_err(|e| format!("Failed to parse RPC modules: {}", e))?; + + // Validate each module in the selection + if let RpcModuleSelection::Selection(modules) = &selection { + for module in modules { + if let RethRpcModule::Other(name) = module { + return Err(format!("Unknown RPC module: '{}'", name)); + } + } + } + + Ok(selection) + } +} + +/// Lenient validator that accepts any module name without validation. +/// +/// This validator accepts any module name, including unknown ones. +#[derive(Debug, Clone, Copy)] +pub struct LenientRpcModuleValidator; + +impl RpcModuleValidator for LenientRpcModuleValidator { + fn parse_selection(s: &str) -> Result { + RpcModuleSelection::from_str(s).map_err(|e| format!("Failed to parse RPC modules: {}", e)) } } @@ -514,6 +693,52 @@ mod test { assert!(!RpcModuleSelection::are_identical(Some(&standard), Some(&non_matching_standard))); } + #[test] + fn test_rpc_module_selection_append() { + // Test append on Standard selection + let selection = RpcModuleSelection::Standard; + let new_selection = selection.append(RethRpcModule::Admin); + assert!(new_selection.contains(&RethRpcModule::Eth)); + assert!(new_selection.contains(&RethRpcModule::Net)); + assert!(new_selection.contains(&RethRpcModule::Web3)); + assert!(new_selection.contains(&RethRpcModule::Admin)); + + // Test append on empty Selection + let selection = RpcModuleSelection::Selection(HashSet::new()); + let new_selection = selection.append(RethRpcModule::Eth); + assert!(new_selection.contains(&RethRpcModule::Eth)); + assert_eq!(new_selection.len(), 1); + + // Test append on All (should return All) + let selection = RpcModuleSelection::All; + let new_selection = selection.append(RethRpcModule::Eth); + assert_eq!(new_selection, RpcModuleSelection::All); + } + + #[test] + fn test_rpc_module_selection_extend() { + // Test extend on Standard selection + let mut selection = RpcModuleSelection::Standard; + selection.extend(vec![RethRpcModule::Admin, RethRpcModule::Debug]); + assert!(selection.contains(&RethRpcModule::Eth)); + assert!(selection.contains(&RethRpcModule::Net)); + assert!(selection.contains(&RethRpcModule::Web3)); + assert!(selection.contains(&RethRpcModule::Admin)); + assert!(selection.contains(&RethRpcModule::Debug)); + + // Test extend on empty Selection + let mut selection = RpcModuleSelection::Selection(HashSet::new()); + selection.extend(vec![RethRpcModule::Eth, RethRpcModule::Admin]); + assert!(selection.contains(&RethRpcModule::Eth)); + assert!(selection.contains(&RethRpcModule::Admin)); + assert_eq!(selection.len(), 2); + + // Test extend on All (should be no-op) + let mut selection = RpcModuleSelection::All; + selection.extend(vec![RethRpcModule::Eth, RethRpcModule::Admin]); + assert_eq!(selection, RpcModuleSelection::All); + } + #[test] fn test_rpc_module_selection_from_str() { // Test empty string returns default selection @@ -559,10 +784,12 @@ mod test { assert!(result.is_ok()); assert_eq!(result.unwrap(), expected_selection); - // Test invalid selection should return error + // Test custom module selections now work (no longer return errors) let result = RpcModuleSelection::from_str("invalid,unknown"); - assert!(result.is_err()); - assert_eq!(result.unwrap_err(), ParseError::VariantNotFound); + assert!(result.is_ok()); + let selection = result.unwrap(); + assert!(selection.contains(&RethRpcModule::Other("invalid".to_string()))); + assert!(selection.contains(&RethRpcModule::Other("unknown".to_string()))); // Test single valid selection: "eth" let result = RpcModuleSelection::from_str("eth"); @@ -570,9 +797,160 @@ mod test { let expected_selection = RpcModuleSelection::from([RethRpcModule::Eth]); assert_eq!(result.unwrap(), expected_selection); - // Test single invalid selection: "unknown" + // Test single custom module selection: "unknown" now becomes Other let result = RpcModuleSelection::from_str("unknown"); + assert!(result.is_ok()); + let expected_selection = + RpcModuleSelection::from([RethRpcModule::Other("unknown".to_string())]); + assert_eq!(result.unwrap(), expected_selection); + } + + #[test] + fn test_rpc_module_other_variant() { + // Test parsing custom module + let custom_module = RethRpcModule::from_str("myCustomModule").unwrap(); + assert_eq!(custom_module, RethRpcModule::Other("myCustomModule".to_string())); + + // Test as_str for Other variant + assert_eq!(custom_module.as_str(), "myCustomModule"); + + // Test as_ref for Other variant + assert_eq!(custom_module.as_ref(), "myCustomModule"); + + // Test Display impl + assert_eq!(custom_module.to_string(), "myCustomModule"); + } + + #[test] + fn test_rpc_module_selection_with_mixed_modules() { + // Test selection with both standard and custom modules + let result = RpcModuleSelection::from_str("eth,admin,myCustomModule,anotherCustom"); + assert!(result.is_ok()); + + let selection = result.unwrap(); + assert!(selection.contains(&RethRpcModule::Eth)); + assert!(selection.contains(&RethRpcModule::Admin)); + assert!(selection.contains(&RethRpcModule::Other("myCustomModule".to_string()))); + assert!(selection.contains(&RethRpcModule::Other("anotherCustom".to_string()))); + } + + #[test] + fn test_rpc_module_all_excludes_custom() { + // Test that All selection doesn't include custom modules + let all_selection = RpcModuleSelection::All; + + // All should contain standard modules + assert!(all_selection.contains(&RethRpcModule::Eth)); + assert!(all_selection.contains(&RethRpcModule::Admin)); + + // But All doesn't explicitly contain custom modules + // (though contains() returns true for all modules when selection is All) + assert_eq!(all_selection.len(), RethRpcModule::variant_count()); + } + + #[test] + fn test_rpc_module_equality_with_other() { + let other1 = RethRpcModule::Other("custom".to_string()); + let other2 = RethRpcModule::Other("custom".to_string()); + let other3 = RethRpcModule::Other("different".to_string()); + + assert_eq!(other1, other2); + assert_ne!(other1, other3); + assert_ne!(other1, RethRpcModule::Eth); + } + + #[test] + fn test_rpc_module_is_other() { + // Standard modules should return false + assert!(!RethRpcModule::Eth.is_other()); + assert!(!RethRpcModule::Admin.is_other()); + assert!(!RethRpcModule::Debug.is_other()); + + // Other variants should return true + assert!(RethRpcModule::Other("custom".to_string()).is_other()); + assert!(RethRpcModule::Other("mycustomrpc".to_string()).is_other()); + } + + #[test] + fn test_standard_variant_names_excludes_other() { + let standard_names: Vec<_> = RethRpcModule::standard_variant_names().collect(); + + // Verify "other" is not in the list + assert!(!standard_names.contains(&"other")); + + // Should have exactly as many names as STANDARD_VARIANTS + assert_eq!(standard_names.len(), RethRpcModule::STANDARD_VARIANTS.len()); + + // Verify all standard variants have their names in the list + for variant in RethRpcModule::STANDARD_VARIANTS { + assert!(standard_names.contains(&variant.as_ref())); + } + } + + #[test] + fn test_default_validator_accepts_standard_modules() { + // Should accept standard modules + let result = DefaultRpcModuleValidator::parse_selection("eth,admin,debug"); + assert!(result.is_ok()); + + let selection = result.unwrap(); + assert!(matches!(selection, RpcModuleSelection::Selection(_))); + } + + #[test] + fn test_default_validator_rejects_unknown_modules() { + // Should reject unknown module names + let result = DefaultRpcModuleValidator::parse_selection("eth,mycustom"); + assert!(result.is_err()); + assert!(result.unwrap_err().contains("Unknown RPC module: 'mycustom'")); + + let result = DefaultRpcModuleValidator::parse_selection("unknownmodule"); + assert!(result.is_err()); + assert!(result.unwrap_err().contains("Unknown RPC module: 'unknownmodule'")); + + let result = DefaultRpcModuleValidator::parse_selection("eth,admin,xyz123"); + assert!(result.is_err()); + assert!(result.unwrap_err().contains("Unknown RPC module: 'xyz123'")); + } + + #[test] + fn test_default_validator_all_selection() { + // Should accept "all" selection + let result = DefaultRpcModuleValidator::parse_selection("all"); + assert!(result.is_ok()); + assert_eq!(result.unwrap(), RpcModuleSelection::All); + } + + #[test] + fn test_default_validator_none_selection() { + // Should accept "none" selection + let result = DefaultRpcModuleValidator::parse_selection("none"); + assert!(result.is_ok()); + assert_eq!(result.unwrap(), RpcModuleSelection::Selection(Default::default())); + } + + #[test] + fn test_lenient_validator_accepts_unknown_modules() { + // Lenient validator should accept any module name without validation + let result = LenientRpcModuleValidator::parse_selection("eht,adimn,xyz123,customrpc"); + assert!(result.is_ok()); + + let selection = result.unwrap(); + if let RpcModuleSelection::Selection(modules) = selection { + assert!(modules.contains(&RethRpcModule::Other("eht".to_string()))); + assert!(modules.contains(&RethRpcModule::Other("adimn".to_string()))); + assert!(modules.contains(&RethRpcModule::Other("xyz123".to_string()))); + assert!(modules.contains(&RethRpcModule::Other("customrpc".to_string()))); + } else { + panic!("Expected Selection variant"); + } + } + + #[test] + fn test_default_validator_mixed_standard_and_custom() { + // Should reject mix of standard and custom modules + let result = DefaultRpcModuleValidator::parse_selection("eth,admin,mycustom,debug"); assert!(result.is_err()); - assert_eq!(result.unwrap_err(), ParseError::VariantNotFound); + assert!(result.unwrap_err().contains("Unknown RPC module: 'mycustom'")); } } diff --git a/crates/rpc/rpc-testing-util/Cargo.toml b/crates/rpc/rpc-testing-util/Cargo.toml index 65b10feef98..2d074ef2368 100644 --- a/crates/rpc/rpc-testing-util/Cargo.toml +++ b/crates/rpc/rpc-testing-util/Cargo.toml @@ -36,4 +36,3 @@ similar-asserts.workspace = true tokio = { workspace = true, features = ["rt-multi-thread", "macros", "rt"] } reth-rpc-eth-api.workspace = true jsonrpsee-http-client.workspace = true -alloy-rpc-types-trace.workspace = true diff --git a/crates/rpc/rpc-testing-util/src/debug.rs b/crates/rpc/rpc-testing-util/src/debug.rs index fa3facead5c..65fc3e86e02 100644 --- a/crates/rpc/rpc-testing-util/src/debug.rs +++ b/crates/rpc/rpc-testing-util/src/debug.rs @@ -15,7 +15,7 @@ use alloy_rpc_types_trace::{ }; use futures::{Stream, StreamExt}; use jsonrpsee::core::client::Error as RpcError; -use reth_ethereum_primitives::Receipt; +use reth_ethereum_primitives::{Receipt, TransactionSigned}; use reth_rpc_api::{clients::DebugApiClient, EthApiClient}; const NOOP_TRACER: &str = include_str!("../assets/noop-tracer.js"); @@ -77,7 +77,9 @@ pub trait DebugApiExt { impl DebugApiExt for T where - T: EthApiClient + DebugApiClient + Sync, + T: EthApiClient + + DebugApiClient + + Sync, { type Provider = T; @@ -132,11 +134,13 @@ where let stream = futures::stream::iter(blocks.into_iter().map(move |(block, opts)| async move { let trace_future = match block { - BlockId::Hash(hash) => self.debug_trace_block_by_hash(hash.block_hash, opts), - BlockId::Number(tag) => self.debug_trace_block_by_number(tag, opts), + BlockId::Hash(hash) => { + self.debug_trace_block_by_hash(hash.block_hash, opts).await + } + BlockId::Number(tag) => self.debug_trace_block_by_number(tag, opts).await, }; - match trace_future.await { + match trace_future { Ok(result) => Ok((result, block)), Err(err) => Err((err, block)), } diff --git a/crates/rpc/rpc-testing-util/src/lib.rs b/crates/rpc/rpc-testing-util/src/lib.rs index ebf5090b715..6be9f74403f 100644 --- a/crates/rpc/rpc-testing-util/src/lib.rs +++ b/crates/rpc/rpc-testing-util/src/lib.rs @@ -6,7 +6,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] pub mod debug; pub mod trace; diff --git a/crates/rpc/rpc-testing-util/src/trace.rs b/crates/rpc/rpc-testing-util/src/trace.rs index b556a895045..8f71d1c4554 100644 --- a/crates/rpc/rpc-testing-util/src/trace.rs +++ b/crates/rpc/rpc-testing-util/src/trace.rs @@ -250,7 +250,7 @@ impl std::fmt::Debug for ReplayTransactionStream<'_> { } } -impl TraceApiExt for T { +impl + Sync> TraceApiExt for T { type Provider = T; fn trace_block_buffered(&self, params: I, n: usize) -> TraceBlockStream<'_> diff --git a/crates/rpc/rpc-testing-util/tests/it/trace.rs b/crates/rpc/rpc-testing-util/tests/it/trace.rs index c733e6bde67..19e0b202dc6 100644 --- a/crates/rpc/rpc-testing-util/tests/it/trace.rs +++ b/crates/rpc/rpc-testing-util/tests/it/trace.rs @@ -1,14 +1,14 @@ //! Integration tests for the trace API. use alloy_primitives::map::HashSet; -use alloy_rpc_types_eth::{Block, Header, Transaction}; +use alloy_rpc_types_eth::{Block, Header, Transaction, TransactionRequest}; use alloy_rpc_types_trace::{ filter::TraceFilter, parity::TraceType, tracerequest::TraceCallRequest, }; use futures::StreamExt; use jsonrpsee::http_client::HttpClientBuilder; use jsonrpsee_http_client::HttpClient; -use reth_ethereum_primitives::Receipt; +use reth_ethereum_primitives::{Receipt, TransactionSigned}; use reth_rpc_api_testing_util::{debug::DebugApiExt, trace::TraceApiExt, utils::parse_env_url}; use reth_rpc_eth_api::EthApiClient; use std::time::Instant; @@ -112,12 +112,18 @@ async fn debug_trace_block_entire_chain() { let url = url.unwrap(); let client = HttpClientBuilder::default().build(url).unwrap(); - let current_block: u64 = - >::block_number(&client) - .await - .unwrap() - .try_into() - .unwrap(); + let current_block: u64 = >::block_number(&client) + .await + .unwrap() + .try_into() + .unwrap(); let range = 0..=current_block; let mut stream = client.debug_trace_block_buffered_unordered(range, None, 20); let now = Instant::now(); @@ -141,12 +147,18 @@ async fn debug_trace_block_opcodes_entire_chain() { let url = url.unwrap(); let client = HttpClientBuilder::default().build(url).unwrap(); - let current_block: u64 = - >::block_number(&client) - .await - .unwrap() - .try_into() - .unwrap(); + let current_block: u64 = >::block_number(&client) + .await + .unwrap() + .try_into() + .unwrap(); let range = 0..=current_block; println!("Tracing blocks {range:?} for opcodes"); let mut stream = client.trace_block_opcode_gas_unordered(range, 2).enumerate(); diff --git a/crates/rpc/rpc-types-compat/Cargo.toml b/crates/rpc/rpc-types-compat/Cargo.toml deleted file mode 100644 index 76cb92505e0..00000000000 --- a/crates/rpc/rpc-types-compat/Cargo.toml +++ /dev/null @@ -1,25 +0,0 @@ -[package] -name = "reth-rpc-types-compat" -version.workspace = true -edition.workspace = true -rust-version.workspace = true -license.workspace = true -homepage.workspace = true -repository.workspace = true -description = "Compatibility layer for reth-primitives and ethereum RPC types" - -[lints] -workspace = true - -[dependencies] -# reth -reth-primitives-traits.workspace = true - -# ethereum -alloy-primitives.workspace = true -alloy-rpc-types-eth = { workspace = true, default-features = false, features = ["serde"] } -alloy-consensus.workspace = true - -# io -serde.workspace = true -jsonrpsee-types.workspace = true diff --git a/crates/rpc/rpc-types-compat/src/block.rs b/crates/rpc/rpc-types-compat/src/block.rs deleted file mode 100644 index 4f73a2c3f2f..00000000000 --- a/crates/rpc/rpc-types-compat/src/block.rs +++ /dev/null @@ -1,101 +0,0 @@ -//! Compatibility functions for rpc `Block` type. - -use crate::transaction::TransactionCompat; -use alloy_consensus::{transaction::Recovered, BlockBody, BlockHeader, Sealable}; -use alloy_primitives::U256; -use alloy_rpc_types_eth::{ - Block, BlockTransactions, BlockTransactionsKind, Header, TransactionInfo, -}; -use reth_primitives_traits::{ - Block as BlockTrait, BlockBody as BlockBodyTrait, RecoveredBlock, SignedTransaction, -}; - -/// Converts the given primitive block into a [`Block`] response with the given -/// [`BlockTransactionsKind`] -/// -/// If a `block_hash` is provided, then this is used, otherwise the block hash is computed. -#[expect(clippy::type_complexity)] -pub fn from_block( - block: RecoveredBlock, - kind: BlockTransactionsKind, - tx_resp_builder: &T, -) -> Result>, T::Error> -where - T: TransactionCompat<<::Body as BlockBodyTrait>::Transaction>, - B: BlockTrait, -{ - match kind { - BlockTransactionsKind::Hashes => Ok(from_block_with_tx_hashes::(block)), - BlockTransactionsKind::Full => from_block_full::(block, tx_resp_builder), - } -} - -/// Create a new [`Block`] response from a [`RecoveredBlock`], using the -/// total difficulty to populate its field in the rpc response. -/// -/// This will populate the `transactions` field with only the hashes of the transactions in the -/// block: [`BlockTransactions::Hashes`] -pub fn from_block_with_tx_hashes(block: RecoveredBlock) -> Block> -where - B: BlockTrait, -{ - let transactions = block.body().transaction_hashes_iter().copied().collect(); - let rlp_length = block.rlp_length(); - let (header, body) = block.into_sealed_block().split_sealed_header_body(); - let BlockBody { ommers, withdrawals, .. } = body.into_ethereum_body(); - - let transactions = BlockTransactions::Hashes(transactions); - let uncles = ommers.into_iter().map(|h| h.hash_slow()).collect(); - let header = Header::from_consensus(header.into(), None, Some(U256::from(rlp_length))); - - Block { header, uncles, transactions, withdrawals } -} - -/// Create a new [`Block`] response from a [`RecoveredBlock`], using the -/// total difficulty to populate its field in the rpc response. -/// -/// This will populate the `transactions` field with the _full_ -/// [`TransactionCompat::Transaction`] objects: [`BlockTransactions::Full`] -#[expect(clippy::type_complexity)] -pub fn from_block_full( - block: RecoveredBlock, - tx_resp_builder: &T, -) -> Result>, T::Error> -where - T: TransactionCompat<<::Body as BlockBodyTrait>::Transaction>, - B: BlockTrait, -{ - let block_number = block.header().number(); - let base_fee = block.header().base_fee_per_gas(); - let block_length = block.rlp_length(); - let block_hash = Some(block.hash()); - - let (block, senders) = block.split_sealed(); - let (header, body) = block.split_sealed_header_body(); - let BlockBody { transactions, ommers, withdrawals } = body.into_ethereum_body(); - - let transactions = transactions - .into_iter() - .zip(senders) - .enumerate() - .map(|(idx, (tx, sender))| { - let tx_info = TransactionInfo { - hash: Some(*tx.tx_hash()), - block_hash, - block_number: Some(block_number), - base_fee, - index: Some(idx as u64), - }; - - tx_resp_builder.fill(Recovered::new_unchecked(tx, sender), tx_info) - }) - .collect::, T::Error>>()?; - - let transactions = BlockTransactions::Full(transactions); - let uncles = ommers.into_iter().map(|h| h.hash_slow()).collect(); - let header = Header::from_consensus(header.into(), None, Some(U256::from(block_length))); - - let block = Block { header, uncles, transactions, withdrawals }; - - Ok(block) -} diff --git a/crates/rpc/rpc-types-compat/src/transaction.rs b/crates/rpc/rpc-types-compat/src/transaction.rs deleted file mode 100644 index b722c9aa48e..00000000000 --- a/crates/rpc/rpc-types-compat/src/transaction.rs +++ /dev/null @@ -1,49 +0,0 @@ -//! Compatibility functions for rpc `Transaction` type. - -use alloy_consensus::transaction::Recovered; -use alloy_rpc_types_eth::{request::TransactionRequest, TransactionInfo}; -use core::error; -use serde::{Deserialize, Serialize}; -use std::fmt; - -/// Builds RPC transaction w.r.t. network. -pub trait TransactionCompat: Send + Sync + Unpin + Clone + fmt::Debug { - /// RPC transaction response type. - type Transaction: Serialize - + for<'de> Deserialize<'de> - + Send - + Sync - + Unpin - + Clone - + fmt::Debug; - - /// RPC transaction error type. - type Error: error::Error + Into>; - - /// Wrapper for `fill()` with default `TransactionInfo` - /// Create a new rpc transaction result for a _pending_ signed transaction, setting block - /// environment related fields to `None`. - fn fill_pending(&self, tx: Recovered) -> Result { - self.fill(tx, TransactionInfo::default()) - } - - /// Create a new rpc transaction result for a mined transaction, using the given block hash, - /// number, and tx index fields to populate the corresponding fields in the rpc result. - /// - /// The block hash, number, and tx index fields should be from the original block where the - /// transaction was mined. - fn fill( - &self, - tx: Recovered, - tx_inf: TransactionInfo, - ) -> Result; - - /// Builds a fake transaction from a transaction request for inclusion into block built in - /// `eth_simulateV1`. - fn build_simulate_v1_transaction(&self, request: TransactionRequest) -> Result; - - /// Truncates the input of a transaction to only the first 4 bytes. - // todo: remove in favour of using constructor on `TransactionResponse` or similar - // . - fn otterscan_api_truncate_input(tx: &mut Self::Transaction); -} diff --git a/crates/rpc/rpc/Cargo.toml b/crates/rpc/rpc/Cargo.toml index 5fda4ea0ccb..a47fa5ebcdf 100644 --- a/crates/rpc/rpc/Cargo.toml +++ b/crates/rpc/rpc/Cargo.toml @@ -18,7 +18,6 @@ reth-primitives-traits.workspace = true reth-rpc-api.workspace = true reth-rpc-eth-api.workspace = true reth-engine-primitives.workspace = true -reth-ethereum-primitives.workspace = true reth-errors.workspace = true reth-metrics.workspace = true reth-storage-api.workspace = true @@ -29,38 +28,42 @@ reth-network-api.workspace = true reth-rpc-engine-api.workspace = true reth-revm = { workspace = true, features = ["witness"] } reth-tasks = { workspace = true, features = ["rayon"] } -reth-rpc-types-compat.workspace = true +reth-rpc-convert.workspace = true revm-inspectors.workspace = true reth-network-peers = { workspace = true, features = ["secp256k1"] } reth-evm.workspace = true +reth-evm-ethereum.workspace = true reth-rpc-eth-types.workspace = true reth-rpc-server-types.workspace = true reth-network-types.workspace = true reth-consensus.workspace = true +reth-consensus-common.workspace = true reth-node-api.workspace = true +reth-trie-common.workspace = true # ethereum -alloy-evm.workspace = true +alloy-evm = { workspace = true, features = ["overrides"] } alloy-consensus.workspace = true alloy-signer.workspace = true -alloy-signer-local.workspace = true +alloy-signer-local = { workspace = true, features = ["mnemonic"] } alloy-eips = { workspace = true, features = ["kzg"] } alloy-dyn-abi.workspace = true alloy-genesis.workspace = true alloy-network.workspace = true alloy-primitives.workspace = true alloy-rlp.workspace = true +alloy-rpc-client.workspace = true alloy-rpc-types-beacon = { workspace = true, features = ["ssz"] } alloy-rpc-types.workspace = true -alloy-rpc-types-eth = { workspace = true, features = ["jsonrpsee-types", "serde"] } +alloy-rpc-types-eth = { workspace = true, features = ["serde"] } alloy-rpc-types-debug.workspace = true alloy-rpc-types-trace.workspace = true alloy-rpc-types-mev.workspace = true alloy-rpc-types-txpool.workspace = true alloy-rpc-types-admin.workspace = true -alloy-rpc-types-engine.workspace = true +alloy-rpc-types-engine = { workspace = true, features = ["kzg"] } alloy-serde.workspace = true -revm = { workspace = true, features = ["optional_block_gas_limit", "optional_eip3607", "optional_no_base_fee"] } +revm = { workspace = true, features = ["optional_block_gas_limit", "optional_eip3607", "optional_no_base_fee", "memory_limit"] } revm-primitives = { workspace = true, features = ["serde"] } # rpc @@ -81,24 +84,30 @@ pin-project.workspace = true parking_lot.workspace = true # misc +dyn-clone.workspace = true tracing.workspace = true tracing-futures.workspace = true futures.workspace = true serde.workspace = true +sha2.workspace = true thiserror.workspace = true derive_more.workspace = true +itertools.workspace = true [dev-dependencies] -reth-evm-ethereum.workspace = true +reth-ethereum-primitives.workspace = true reth-testing-utils.workspace = true reth-transaction-pool = { workspace = true, features = ["test-utils"] } reth-provider = { workspace = true, features = ["test-utils"] } +reth-db-api.workspace = true -alloy-consensus.workspace = true rand.workspace = true -jsonrpsee-types.workspace = true jsonrpsee = { workspace = true, features = ["client"] } [features] -js-tracer = ["revm-inspectors/js-tracer", "reth-rpc-eth-types/js-tracer"] +js-tracer = [ + "revm-inspectors/js-tracer", + "reth-rpc-eth-types/js-tracer", + "reth-rpc-eth-api/js-tracer", +] diff --git a/crates/rpc/rpc/src/admin.rs b/crates/rpc/rpc/src/admin.rs index 731021fb435..af5e1ae2ef9 100644 --- a/crates/rpc/rpc/src/admin.rs +++ b/crates/rpc/rpc/src/admin.rs @@ -13,29 +13,34 @@ use reth_network_peers::{id2pk, AnyNode, NodeRecord}; use reth_network_types::PeerKind; use reth_rpc_api::AdminApiServer; use reth_rpc_server_types::ToRpcResult; +use reth_transaction_pool::TransactionPool; +use revm_primitives::keccak256; /// `admin` API implementation. /// /// This type provides the functionality for handling `admin` related requests. -pub struct AdminApi { +pub struct AdminApi { /// An interface to interact with the network network: N, /// The specification of the blockchain's configuration. chain_spec: Arc, + /// The transaction pool + pool: Pool, } -impl AdminApi { +impl AdminApi { /// Creates a new instance of `AdminApi`. - pub const fn new(network: N, chain_spec: Arc) -> Self { - Self { network, chain_spec } + pub const fn new(network: N, chain_spec: Arc, pool: Pool) -> Self { + Self { network, chain_spec, pool } } } #[async_trait] -impl AdminApiServer for AdminApi +impl AdminApiServer for AdminApi where N: NetworkInfo + Peers + 'static, ChainSpec: EthChainSpec + EthereumHardforks + Send + Sync + 'static, + Pool: TransactionPool + 'static, { /// Handler for `admin_addPeer` fn add_peer(&self, record: NodeRecord) -> RpcResult { @@ -70,34 +75,25 @@ where let mut infos = Vec::with_capacity(peers.len()); for peer in peers { - if let Ok(pk) = id2pk(peer.remote_id) { - infos.push(PeerInfo { - id: pk.to_string(), - name: peer.client_version.to_string(), - enode: peer.enode, - enr: peer.enr, - caps: peer - .capabilities - .capabilities() - .iter() - .map(|cap| cap.to_string()) - .collect(), - network: PeerNetworkInfo { - remote_address: peer.remote_addr, - local_address: peer.local_addr.unwrap_or_else(|| self.network.local_addr()), - inbound: peer.direction.is_incoming(), - trusted: peer.kind.is_trusted(), - static_node: peer.kind.is_static(), - }, - protocols: PeerProtocolInfo { - eth: Some(EthPeerInfo::Info(EthInfo { - version: peer.status.version as u64, - })), - snap: None, - other: Default::default(), - }, - }) - } + infos.push(PeerInfo { + id: keccak256(peer.remote_id.as_slice()).to_string(), + name: peer.client_version.to_string(), + enode: peer.enode, + enr: peer.enr, + caps: peer.capabilities.capabilities().iter().map(|cap| cap.to_string()).collect(), + network: PeerNetworkInfo { + remote_address: peer.remote_addr, + local_address: peer.local_addr.unwrap_or_else(|| self.network.local_addr()), + inbound: peer.direction.is_incoming(), + trusted: peer.kind.is_trusted(), + static_node: peer.kind.is_static(), + }, + protocols: PeerProtocolInfo { + eth: Some(EthPeerInfo::Info(EthInfo { version: peer.status.version as u64 })), + snap: None, + other: Default::default(), + }, + }) } Ok(infos) @@ -189,9 +185,17 @@ where ) -> jsonrpsee::core::SubscriptionResult { Err("admin_peerEvents is not implemented yet".into()) } + + /// Handler for `admin_clearTxpool` + async fn clear_txpool(&self) -> RpcResult { + let all_hashes = self.pool.all_transaction_hashes(); + let count = all_hashes.len() as u64; + let _ = self.pool.remove_transactions(all_hashes); + Ok(count) + } } -impl std::fmt::Debug for AdminApi { +impl std::fmt::Debug for AdminApi { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("AdminApi").finish_non_exhaustive() } diff --git a/crates/rpc/rpc/src/aliases.rs b/crates/rpc/rpc/src/aliases.rs new file mode 100644 index 00000000000..8854f1b607d --- /dev/null +++ b/crates/rpc/rpc/src/aliases.rs @@ -0,0 +1,13 @@ +use reth_evm::ConfigureEvm; +use reth_rpc_convert::RpcConvert; +use reth_rpc_eth_types::EthApiError; + +/// Boxed RPC converter. +pub type DynRpcConverter = Box< + dyn RpcConvert< + Primitives = ::Primitives, + Network = Network, + Error = Error, + Evm = Evm, + >, +>; diff --git a/crates/rpc/rpc/src/debug.rs b/crates/rpc/rpc/src/debug.rs index 85c6e65a446..75d3b4ad7cc 100644 --- a/crates/rpc/rpc/src/debug.rs +++ b/crates/rpc/rpc/src/debug.rs @@ -1,12 +1,15 @@ -use alloy_consensus::BlockHeader; +use alloy_consensus::{ + transaction::{SignerRecoverable, TxHashRef}, + BlockHeader, +}; use alloy_eips::{eip2718::Encodable2718, BlockId, BlockNumberOrTag}; +use alloy_evm::env::BlockEnvironment; use alloy_genesis::ChainConfig; -use alloy_primitives::{Address, Bytes, B256}; +use alloy_primitives::{hex::decode, uint, Address, Bytes, B256}; use alloy_rlp::{Decodable, Encodable}; use alloy_rpc_types_debug::ExecutionWitness; use alloy_rpc_types_eth::{ - state::EvmOverrides, transaction::TransactionRequest, Block as RpcBlock, BlockError, Bundle, - StateContext, TransactionInfo, + state::EvmOverrides, Block as RpcBlock, BlockError, Bundle, StateContext, TransactionInfo, }; use alloy_rpc_types_trace::geth::{ call::FlatCallFrame, BlockTraceResult, FourByteFrame, GethDebugBuiltInTracerType, @@ -16,16 +19,12 @@ use alloy_rpc_types_trace::geth::{ use async_trait::async_trait; use jsonrpsee::core::RpcResult; use reth_chainspec::{ChainSpecProvider, EthChainSpec, EthereumHardforks}; +use reth_errors::RethError; use reth_evm::{execute::Executor, ConfigureEvm, EvmEnvFor, TxEnvFor}; -use reth_primitives_traits::{ - Block as _, BlockBody, NodePrimitives, ReceiptWithBloom, RecoveredBlock, SignedTransaction, -}; -use reth_revm::{ - database::StateProviderDatabase, - db::{CacheDB, State}, - witness::ExecutionWitnessRecord, -}; +use reth_primitives_traits::{Block as _, BlockBody, ReceiptWithBloom, RecoveredBlock}; +use reth_revm::{database::StateProviderDatabase, db::State, witness::ExecutionWitnessRecord}; use reth_rpc_api::DebugApiServer; +use reth_rpc_convert::RpcTxReq; use reth_rpc_eth_api::{ helpers::{EthTransactions, TraceExt}, EthApiTypes, FromEthApiError, RpcNodeCore, @@ -34,10 +33,11 @@ use reth_rpc_eth_types::{EthApiError, StateCacheDb}; use reth_rpc_server_types::{result::internal_rpc_err, ToRpcResult}; use reth_storage_api::{ BlockIdReader, BlockReaderIdExt, HeaderProvider, ProviderBlock, ReceiptProviderIdExt, - StateProofProvider, StateProvider, StateProviderFactory, TransactionVariant, + StateProofProvider, StateProviderFactory, StateRootProvider, TransactionVariant, }; use reth_tasks::pool::BlockingTaskGuard; -use revm::{context_interface::Transaction, state::EvmState, DatabaseCommit}; +use reth_trie_common::{updates::TrieUpdates, HashedPostState}; +use revm::{context::Block, context_interface::Transaction, state::EvmState, DatabaseCommit}; use revm_inspectors::tracing::{ FourByteInspector, MuxInspector, TracingInspector, TracingInspectorConfig, TransactionContext, }; @@ -47,16 +47,16 @@ use tokio::sync::{AcquireError, OwnedSemaphorePermit}; /// `debug` API implementation. /// /// This type provides the functionality for handling `debug` related requests. -pub struct DebugApi { - inner: Arc>, +pub struct DebugApi { + inner: Arc>, } // === impl DebugApi === -impl DebugApi { +impl DebugApi { /// Create a new instance of the [`DebugApi`] - pub fn new(eth: Eth, blocking_task_guard: BlockingTaskGuard, evm_config: Evm) -> Self { - let inner = Arc::new(DebugApiInner { eth_api: eth, blocking_task_guard, evm_config }); + pub fn new(eth_api: Eth, blocking_task_guard: BlockingTaskGuard) -> Self { + let inner = Arc::new(DebugApiInner { eth_api, blocking_task_guard }); Self { inner } } @@ -66,7 +66,7 @@ impl DebugApi { } } -impl DebugApi { +impl DebugApi { /// Access the underlying provider. pub fn provider(&self) -> &Eth::Provider { self.inner.eth_api.provider() @@ -75,10 +75,9 @@ impl DebugApi { // === impl DebugApi === -impl DebugApi +impl DebugApi where Eth: EthApiTypes + TraceExt + 'static, - Evm: ConfigureEvm>> + 'static, { /// Acquires a permit to execute a tracing call. async fn acquire_trace_permit(&self) -> Result { @@ -97,7 +96,8 @@ where self.eth_api() .spawn_with_state_at_block(block.parent_hash().into(), move |state| { let mut results = Vec::with_capacity(block.body().transactions().len()); - let mut db = CacheDB::new(StateProviderDatabase::new(state)); + let mut db = + State::builder().with_database(StateProviderDatabase::new(state)).build(); this.eth_api().apply_pre_execution_changes(&block, &mut db, &evm_env)?; @@ -150,7 +150,12 @@ where .map_err(BlockError::RlpDecodeRawBlock) .map_err(Eth::Error::from_eth_err)?; - let evm_env = self.eth_api().evm_config().evm_env(block.header()); + let evm_env = self + .eth_api() + .evm_config() + .evm_env(block.header()) + .map_err(RethError::other) + .map_err(Eth::Error::from_eth_err)?; // Depending on EIP-2 we need to recover the transactions differently let senders = @@ -161,8 +166,6 @@ where .iter() .map(|tx| tx.recover_signer().map_err(Eth::Error::from_eth_err)) .collect::, _>>()? - .into_iter() - .collect() } else { block .body() @@ -170,8 +173,6 @@ where .iter() .map(|tx| tx.recover_signer_unchecked().map_err(Eth::Error::from_eth_err)) .collect::, _>>()? - .into_iter() - .collect() }; self.trace_block(Arc::new(block.into_recovered_with_signers(senders)), evm_env, opts).await @@ -226,7 +227,8 @@ where // configure env for the target transaction let tx = transaction.into_recovered(); - let mut db = CacheDB::new(StateProviderDatabase::new(state)); + let mut db = + State::builder().with_database(StateProviderDatabase::new(state)).build(); this.eth_api().apply_pre_execution_changes(&block, &mut db, &evm_env)?; @@ -264,18 +266,20 @@ where /// - `debug_traceCall` executes with __enabled__ basefee check, `eth_call` does not: pub async fn debug_trace_call( &self, - call: TransactionRequest, + call: RpcTxReq, block_id: Option, opts: GethDebugTracingCallOptions, ) -> Result { let at = block_id.unwrap_or_default(); - let GethDebugTracingCallOptions { tracing_options, state_overrides, block_overrides } = - opts; + let GethDebugTracingCallOptions { + tracing_options, state_overrides, block_overrides, .. + } = opts; let overrides = EvmOverrides::new(state_overrides, block_overrides.map(Box::new)); let GethDebugTracingOptions { config, tracer, tracer_config, .. } = tracing_options; let this = self.clone(); if let Some(tracer) = tracer { + #[allow(unreachable_patterns)] return match tracer { GethDebugTracerType::BuiltInTracer(tracer) => match tracer { GethDebugBuiltInTracerType::FourByteTracer => { @@ -287,7 +291,7 @@ where Ok(inspector) }) .await?; - return Ok(FourByteFrame::from(&inspector).into()) + Ok(FourByteFrame::from(&inspector).into()) } GethDebugBuiltInTracerType::CallTracer => { let call_config = tracer_config @@ -301,16 +305,17 @@ where let frame = self .eth_api() .spawn_with_call_at(call, at, overrides, move |db, evm_env, tx_env| { - let (res, (_, tx_env)) = + let gas_limit = tx_env.gas_limit(); + let res = this.eth_api().inspect(db, evm_env, tx_env, &mut inspector)?; let frame = inspector - .with_transaction_gas_limit(tx_env.gas_limit()) + .with_transaction_gas_limit(gas_limit) .into_geth_builder() .geth_call_traces(call_config, res.result.gas_used()); Ok(frame.into()) }) .await?; - return Ok(frame) + Ok(frame) } GethDebugBuiltInTracerType::PreStateTracer => { let prestate_config = tracer_config @@ -327,21 +332,22 @@ where // see let db = db.0; - let (res, (_, tx_env)) = this.eth_api().inspect( + let gas_limit = tx_env.gas_limit(); + let res = this.eth_api().inspect( &mut *db, evm_env, tx_env, &mut inspector, )?; let frame = inspector - .with_transaction_gas_limit(tx_env.gas_limit()) + .with_transaction_gas_limit(gas_limit) .into_geth_builder() .geth_prestate_traces(&res, &prestate_config, db) .map_err(Eth::Error::from_eth_err)?; Ok(frame) }) .await?; - return Ok(frame.into()) + Ok(frame.into()) } GethDebugBuiltInTracerType::NoopTracer => Ok(NoopFrame::default().into()), GethDebugBuiltInTracerType::MuxTracer => { @@ -361,14 +367,14 @@ where let db = db.0; let tx_info = TransactionInfo { - block_number: Some(evm_env.block_env.number), - base_fee: Some(evm_env.block_env.basefee), + block_number: Some(evm_env.block_env.number().saturating_to()), + base_fee: Some(evm_env.block_env.basefee()), hash: None, block_hash: None, index: None, }; - let (res, _) = this.eth_api().inspect( + let res = this.eth_api().inspect( &mut *db, evm_env, tx_env, @@ -380,7 +386,7 @@ where Ok(frame.into()) }) .await?; - return Ok(frame) + Ok(frame) } GethDebugBuiltInTracerType::FlatCallTracer => { let flat_call_config = tracer_config @@ -395,18 +401,23 @@ where .inner .eth_api .spawn_with_call_at(call, at, overrides, move |db, evm_env, tx_env| { - let (_res, (_, tx_env)) = - this.eth_api().inspect(db, evm_env, tx_env, &mut inspector)?; + let gas_limit = tx_env.gas_limit(); + this.eth_api().inspect(db, evm_env, tx_env, &mut inspector)?; let tx_info = TransactionInfo::default(); let frame: FlatCallFrame = inspector - .with_transaction_gas_limit(tx_env.gas_limit()) + .with_transaction_gas_limit(gas_limit) .into_parity_builder() .into_localized_transaction_traces(tx_info); Ok(frame) }) .await?; - return Ok(frame.into()); + Ok(frame.into()) + } + _ => { + // Note: this match is non-exhaustive in case we need to add support for + // additional tracers + Err(EthApiError::Unsupported("unsupported tracer").into()) } }, #[cfg(not(feature = "js-tracer"))] @@ -429,7 +440,7 @@ where let mut inspector = revm_inspectors::tracing::js::JsInspector::new(code, config) .map_err(Eth::Error::from_eth_err)?; - let (res, _) = this.eth_api().inspect( + let res = this.eth_api().inspect( &mut *db, evm_env.clone(), tx_env.clone(), @@ -443,6 +454,11 @@ where Ok(GethTrace::JS(res)) } + _ => { + // Note: this match is non-exhaustive in case we need to add support for + // additional tracers + Err(EthApiError::Unsupported("unsupported tracer").into()) + } } } @@ -454,9 +470,9 @@ where let (res, tx_gas_limit, inspector) = self .eth_api() .spawn_with_call_at(call, at, overrides, move |db, evm_env, tx_env| { - let (res, (_, tx_env)) = - this.eth_api().inspect(db, evm_env, tx_env, &mut inspector)?; - Ok((res, tx_env.gas_limit(), inspector)) + let gas_limit = tx_env.gas_limit(); + let res = this.eth_api().inspect(db, evm_env, tx_env, &mut inspector)?; + Ok((res, gas_limit, inspector)) }) .await?; let gas_used = res.result.gas_used(); @@ -474,7 +490,7 @@ where /// Each following bundle increments block number by 1 and block timestamp by 12 seconds pub async fn debug_trace_call_many( &self, - bundles: Vec, + bundles: Vec>>, state_context: Option, opts: Option, ) -> Result>, Eth::Error> { @@ -517,7 +533,8 @@ where .spawn_with_state_at_block(at.into(), move |state| { // the outer vec for the bundles let mut all_bundles = Vec::with_capacity(bundles.len()); - let mut db = CacheDB::new(StateProviderDatabase::new(state)); + let mut db = + State::builder().with_database(StateProviderDatabase::new(state)).build(); if replay_block_txs { // only need to replay the transactions in the block if not all transactions are @@ -527,7 +544,7 @@ where // Execute all transactions until index for tx in transactions { let tx_env = this.eth_api().evm_config().tx_env(tx); - let (res, _) = this.eth_api().transact(&mut db, evm_env.clone(), tx_env)?; + let res = this.eth_api().transact(&mut db, evm_env.clone(), tx_env)?; db.commit(res.state); } } @@ -573,8 +590,8 @@ where results.push(trace); } // Increment block_env number and timestamp for the next bundle - evm_env.block_env.number += 1; - evm_env.block_env.timestamp += 12; + evm_env.block_env.inner_mut().number += uint!(1_U256); + evm_env.block_env.inner_mut().timestamp += uint!(12_U256); all_bundles.push(results); } @@ -629,12 +646,12 @@ where .eth_api() .spawn_with_state_at_block(block.parent_hash().into(), move |state_provider| { let db = StateProviderDatabase::new(&state_provider); - let block_executor = this.inner.evm_config.batch_executor(db); + let block_executor = this.eth_api().evm_config().executor(db); let mut witness_record = ExecutionWitnessRecord::default(); let _ = block_executor - .execute_with_state_closure(&(*block).clone(), |statedb: &State<_>| { + .execute_with_state_closure(&block, |statedb: &State<_>| { witness_record.record_executed_state(statedb); }) .map_err(|err| EthApiError::Internal(err.into()))?; @@ -662,7 +679,6 @@ where }; let range = smallest..block_number; - // TODO: Check if headers_range errors when one of the headers in the range is missing exec_witness.headers = self .provider() .headers_range(range) @@ -726,17 +742,17 @@ where .map(|c| c.tx_index.map(|i| i as u64)) .unwrap_or_default(), block_hash: transaction_context.as_ref().map(|c| c.block_hash).unwrap_or_default(), - block_number: Some(evm_env.block_env.number), - base_fee: Some(evm_env.block_env.basefee), + block_number: Some(evm_env.block_env.number().saturating_to()), + base_fee: Some(evm_env.block_env.basefee()), }; if let Some(tracer) = tracer { + #[allow(unreachable_patterns)] return match tracer { GethDebugTracerType::BuiltInTracer(tracer) => match tracer { GethDebugBuiltInTracerType::FourByteTracer => { let mut inspector = FourByteInspector::default(); - let (res, _) = - self.eth_api().inspect(db, evm_env, tx_env, &mut inspector)?; + let res = self.eth_api().inspect(db, evm_env, tx_env, &mut inspector)?; return Ok((FourByteFrame::from(&inspector).into(), res.state)) } GethDebugBuiltInTracerType::CallTracer => { @@ -751,10 +767,10 @@ where )) }); - let (res, (_, tx_env)) = - self.eth_api().inspect(db, evm_env, tx_env, &mut inspector)?; + let gas_limit = tx_env.gas_limit(); + let res = self.eth_api().inspect(db, evm_env, tx_env, &mut inspector)?; - inspector.set_transaction_gas_limit(tx_env.gas_limit()); + inspector.set_transaction_gas_limit(gas_limit); let frame = inspector .geth_builder() @@ -773,10 +789,11 @@ where TracingInspectorConfig::from_geth_prestate_config(&prestate_config), ) }); - let (res, (_, tx_env)) = + let gas_limit = tx_env.gas_limit(); + let res = self.eth_api().inspect(&mut *db, evm_env, tx_env, &mut inspector)?; - inspector.set_transaction_gas_limit(tx_env.gas_limit()); + inspector.set_transaction_gas_limit(gas_limit); let frame = inspector .geth_builder() .geth_prestate_traces(&res, &prestate_config, db) @@ -796,7 +813,7 @@ where let mut inspector = MuxInspector::try_from_config(mux_config) .map_err(Eth::Error::from_eth_err)?; - let (res, _) = + let res = self.eth_api().inspect(&mut *db, evm_env, tx_env, &mut inspector)?; let frame = inspector .try_into_mux_frame(&res, db, tx_info) @@ -813,15 +830,20 @@ where TracingInspectorConfig::from_flat_call_config(&flat_call_config), ); - let (res, (_, tx_env)) = - self.eth_api().inspect(db, evm_env, tx_env, &mut inspector)?; + let gas_limit = tx_env.gas_limit(); + let res = self.eth_api().inspect(db, evm_env, tx_env, &mut inspector)?; let frame: FlatCallFrame = inspector - .with_transaction_gas_limit(tx_env.gas_limit()) + .with_transaction_gas_limit(gas_limit) .into_parity_builder() .into_localized_transaction_traces(tx_info); return Ok((frame.into(), res.state)); } + _ => { + // Note: this match is non-exhaustive in case we need to add support for + // additional tracers + Err(EthApiError::Unsupported("unsupported tracer").into()) + } }, #[cfg(not(feature = "js-tracer"))] GethDebugTracerType::JsTracer(_) => { @@ -837,8 +859,12 @@ where transaction_context.unwrap_or_default(), ) .map_err(Eth::Error::from_eth_err)?; - let (res, (evm_env, tx_env)) = - self.eth_api().inspect(&mut *db, evm_env, tx_env, &mut inspector)?; + let res = self.eth_api().inspect( + &mut *db, + evm_env.clone(), + tx_env.clone(), + &mut inspector, + )?; let state = res.state.clone(); let result = inspector @@ -846,6 +872,11 @@ where .map_err(Eth::Error::from_eth_err)?; Ok((GethTrace::JS(result), state)) } + _ => { + // Note: this match is non-exhaustive in case we need to add support for + // additional tracers + Err(EthApiError::Unsupported("unsupported tracer").into()) + } } } @@ -854,26 +885,45 @@ where let inspector_config = TracingInspectorConfig::from_geth_config(config); TracingInspector::new(inspector_config) }); - let (res, (_, tx_env)) = self.eth_api().inspect(db, evm_env, tx_env, &mut inspector)?; + let gas_limit = tx_env.gas_limit(); + let res = self.eth_api().inspect(db, evm_env, tx_env, &mut inspector)?; let gas_used = res.result.gas_used(); let return_value = res.result.into_output().unwrap_or_default(); - inspector.set_transaction_gas_limit(tx_env.gas_limit()); + inspector.set_transaction_gas_limit(gas_limit); let frame = inspector.geth_builder().geth_traces(gas_used, return_value, *config); Ok((frame.into(), res.state)) } + + /// Returns the state root of the `HashedPostState` on top of the state for the given block with + /// trie updates. + async fn debug_state_root_with_updates( + &self, + hashed_state: HashedPostState, + block_id: Option, + ) -> Result<(B256, TrieUpdates), Eth::Error> { + self.inner + .eth_api + .spawn_blocking_io(move |this| { + let state = this + .provider() + .state_by_block_id(block_id.unwrap_or_default()) + .map_err(Eth::Error::from_eth_err)?; + state.state_root_with_updates(hashed_state).map_err(Eth::Error::from_eth_err) + }) + .await + } } #[async_trait] -impl DebugApiServer for DebugApi +impl DebugApiServer> for DebugApi where Eth: EthApiTypes + EthTransactions + TraceExt + 'static, - Evm: ConfigureEvm>> + 'static, { /// Handler for `debug_getRawHeader` async fn raw_header(&self, block_id: BlockId) -> RpcResult { let header = match block_id { - BlockId::Hash(hash) => self.provider().header(&hash.into()).to_rpc_result()?, + BlockId::Hash(hash) => self.provider().header(hash.into()).to_rpc_result()?, BlockId::Number(number_or_tag) => { let number = self .provider() @@ -940,7 +990,7 @@ where /// Handler for `debug_getBadBlocks` async fn bad_blocks(&self) -> RpcResult> { - Err(internal_rpc_err("unimplemented")) + Ok(vec![]) } /// Handler for `debug_traceChain` @@ -1003,7 +1053,7 @@ where /// Handler for `debug_traceCall` async fn debug_trace_call( &self, - request: TransactionRequest, + request: RpcTxReq, block_id: Option, opts: Option, ) -> RpcResult { @@ -1015,7 +1065,7 @@ where async fn debug_trace_call_many( &self, - bundles: Vec, + bundles: Vec>>, state_context: Option, opts: Option, ) -> RpcResult>> { @@ -1093,8 +1143,38 @@ where Ok(()) } - async fn debug_db_get(&self, _key: String) -> RpcResult<()> { - Ok(()) + /// `debug_db_get` - database key lookup + /// + /// Currently supported: + /// * Contract bytecode associated with a code hash. The key format is: `<0x63>` + /// * Prefix byte: 0x63 (required) + /// * Code hash: 32 bytes + /// Must be provided as either: + /// * Hex string: "0x63..." (66 hex characters after 0x) + /// * Raw byte string: raw byte string (33 bytes) + /// See Geth impl: + async fn debug_db_get(&self, key: String) -> RpcResult> { + let key_bytes = if key.starts_with("0x") { + decode(&key).map_err(|_| EthApiError::InvalidParams("Invalid hex key".to_string()))? + } else { + key.into_bytes() + }; + + if key_bytes.len() != 33 { + return Err(EthApiError::InvalidParams(format!( + "Key must be 33 bytes, got {}", + key_bytes.len() + )) + .into()); + } + if key_bytes[0] != 0x63 { + return Err(EthApiError::InvalidParams("Key prefix must be 0x63".to_string()).into()); + } + + let code_hash = B256::from_slice(&key_bytes[1..33]); + + // No block ID is provided, so it defaults to the latest block + self.debug_code_by_hash(code_hash, None).await.map_err(Into::into) } async fn debug_dump_block(&self, _number: BlockId) -> RpcResult<()> { @@ -1217,6 +1297,14 @@ where Ok(()) } + async fn debug_state_root_with_updates( + &self, + hashed_state: HashedPostState, + block_id: Option, + ) -> RpcResult<(B256, TrieUpdates)> { + Self::debug_state_root_with_updates(self, hashed_state, block_id).await.map_err(Into::into) + } + async fn debug_stop_cpu_profile(&self) -> RpcResult<()> { Ok(()) } @@ -1265,23 +1353,21 @@ where } } -impl std::fmt::Debug for DebugApi { +impl std::fmt::Debug for DebugApi { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("DebugApi").finish_non_exhaustive() } } -impl Clone for DebugApi { +impl Clone for DebugApi { fn clone(&self) -> Self { Self { inner: Arc::clone(&self.inner) } } } -struct DebugApiInner { +struct DebugApiInner { /// The implementation of `eth` API eth_api: Eth, // restrict the number of concurrent calls to blocking calls blocking_task_guard: BlockingTaskGuard, - /// block executor for debug & trace apis - evm_config: Evm, } diff --git a/crates/rpc/rpc/src/engine.rs b/crates/rpc/rpc/src/engine.rs index 824e5fb40d7..b7e62fadb75 100644 --- a/crates/rpc/rpc/src/engine.rs +++ b/crates/rpc/rpc/src/engine.rs @@ -1,12 +1,13 @@ use alloy_eips::{BlockId, BlockNumberOrTag}; use alloy_primitives::{Address, Bytes, B256, U256, U64}; use alloy_rpc_types_eth::{ - state::StateOverride, transaction::TransactionRequest, BlockOverrides, - EIP1186AccountProofResponse, Filter, Log, SyncStatus, + state::StateOverride, BlockOverrides, EIP1186AccountProofResponse, Filter, Log, SyncStatus, }; use alloy_serde::JsonStorageKey; use jsonrpsee::core::RpcResult as Result; +use reth_primitives_traits::TxTy; use reth_rpc_api::{EngineEthApiServer, EthApiServer}; +use reth_rpc_convert::RpcTxReq; /// Re-export for convenience pub use reth_rpc_engine_api::EngineApi; use reth_rpc_eth_api::{ @@ -16,7 +17,7 @@ use tracing_futures::Instrument; macro_rules! engine_span { () => { - tracing::trace_span!(target: "rpc", "engine") + tracing::info_span!(target: "rpc", "engine") }; } @@ -36,14 +37,20 @@ impl EngineEthApi { } #[async_trait::async_trait] -impl EngineEthApiServer, RpcReceipt> - for EngineEthApi +impl + EngineEthApiServer< + RpcTxReq, + RpcBlock, + RpcReceipt, + > for EngineEthApi where Eth: EthApiServer< + RpcTxReq, RpcTransaction, RpcBlock, RpcReceipt, RpcHeader, + TxTy, > + FullEthApiTypes, EthFilter: EngineEthFilter, { @@ -71,7 +78,7 @@ where /// Handler for: `eth_call` async fn call( &self, - request: TransactionRequest, + request: RpcTxReq, block_id: Option, state_overrides: Option, block_overrides: Option>, diff --git a/crates/rpc/rpc/src/eth/builder.rs b/crates/rpc/rpc/src/eth/builder.rs index 83b9a074a15..ff01903736b 100644 --- a/crates/rpc/rpc/src/eth/builder.rs +++ b/crates/rpc/rpc/src/eth/builder.rs @@ -1,63 +1,137 @@ //! `EthApiBuilder` implementation -use crate::{ - eth::{core::EthApiInner, EthTxBuilder}, - EthApi, -}; +use crate::{eth::core::EthApiInner, EthApi}; +use alloy_network::Ethereum; use reth_chain_state::CanonStateSubscriptions; use reth_chainspec::ChainSpecProvider; -use reth_node_api::NodePrimitives; +use reth_primitives_traits::HeaderTy; +use reth_rpc_convert::{RpcConvert, RpcConverter}; +use reth_rpc_eth_api::{ + helpers::pending_block::PendingEnvBuilder, node::RpcNodeCoreAdapter, RpcNodeCore, +}; use reth_rpc_eth_types::{ - fee_history::fee_history_cache_new_blocks_task, EthStateCache, EthStateCacheConfig, - FeeHistoryCache, FeeHistoryCacheConfig, GasCap, GasPriceOracle, GasPriceOracleConfig, + builder::config::PendingBlockKind, fee_history::fee_history_cache_new_blocks_task, + receipt::EthReceiptConverter, EthStateCache, EthStateCacheConfig, FeeHistoryCache, + FeeHistoryCacheConfig, ForwardConfig, GasCap, GasPriceOracle, GasPriceOracleConfig, }; use reth_rpc_server_types::constants::{ DEFAULT_ETH_PROOF_WINDOW, DEFAULT_MAX_SIMULATE_BLOCKS, DEFAULT_PROOF_PERMITS, }; -use reth_storage_api::{BlockReaderIdExt, StateProviderFactory}; use reth_tasks::{pool::BlockingTaskPool, TaskSpawner, TokioTaskExecutor}; -use std::sync::Arc; +use std::{sync::Arc, time::Duration}; /// A helper to build the `EthApi` handler instance. /// /// This builder type contains all settings to create an [`EthApiInner`] or an [`EthApi`] instance /// directly. #[derive(Debug)] -pub struct EthApiBuilder -where - Provider: BlockReaderIdExt, -{ - provider: Provider, - pool: Pool, - network: Network, - evm_config: EvmConfig, +pub struct EthApiBuilder { + components: N, + rpc_converter: Rpc, gas_cap: GasCap, max_simulate_blocks: u64, eth_proof_window: u64, fee_history_cache_config: FeeHistoryCacheConfig, proof_permits: usize, eth_state_cache_config: EthStateCacheConfig, - eth_cache: Option>, + eth_cache: Option>, gas_oracle_config: GasPriceOracleConfig, - gas_oracle: Option>, + gas_oracle: Option>, blocking_task_pool: Option, task_spawner: Box, + next_env: NextEnv, + max_batch_size: usize, + pending_block_kind: PendingBlockKind, + raw_tx_forwarder: ForwardConfig, + send_raw_transaction_sync_timeout: Duration, + evm_memory_limit: u64, } -impl EthApiBuilder +impl + EthApiBuilder< + RpcNodeCoreAdapter, + RpcConverter>, + > where - Provider: BlockReaderIdExt, + RpcNodeCoreAdapter: + RpcNodeCore, Evm = EvmConfig>, { /// Creates a new `EthApiBuilder` instance. - pub fn new(provider: Provider, pool: Pool, network: Network, evm_config: EvmConfig) -> Self + pub fn new(provider: Provider, pool: Pool, network: Network, evm_config: EvmConfig) -> Self { + Self::new_with_components(RpcNodeCoreAdapter::new(provider, pool, network, evm_config)) + } +} + +impl EthApiBuilder { + /// Apply a function to the builder + pub fn apply(self, f: F) -> Self + where + F: FnOnce(Self) -> Self, + { + f(self) + } + + /// Converts the RPC converter type of this builder + pub fn map_converter(self, f: F) -> EthApiBuilder where - Provider: BlockReaderIdExt, + F: FnOnce(Rpc) -> R, { + let Self { + components, + rpc_converter, + gas_cap, + max_simulate_blocks, + eth_proof_window, + fee_history_cache_config, + proof_permits, + eth_state_cache_config, + eth_cache, + gas_oracle_config, + gas_oracle, + blocking_task_pool, + task_spawner, + next_env, + max_batch_size, + pending_block_kind, + raw_tx_forwarder, + send_raw_transaction_sync_timeout, + evm_memory_limit, + } = self; + EthApiBuilder { + components, + rpc_converter: f(rpc_converter), + gas_cap, + max_simulate_blocks, + eth_proof_window, + fee_history_cache_config, + proof_permits, + eth_state_cache_config, + eth_cache, + gas_oracle_config, + gas_oracle, + blocking_task_pool, + task_spawner, + next_env, + max_batch_size, + pending_block_kind, + raw_tx_forwarder, + send_raw_transaction_sync_timeout, + evm_memory_limit, + } + } +} + +impl EthApiBuilder>> +where + N: RpcNodeCore>, +{ + /// Creates a new `EthApiBuilder` instance with the provided components. + pub fn new_with_components(components: N) -> Self { + let rpc_converter = + RpcConverter::new(EthReceiptConverter::new(components.provider().chain_spec())); Self { - provider, - pool, - network, - evm_config, + components, + rpc_converter, eth_cache: None, gas_oracle: None, gas_cap: GasCap::default(), @@ -69,15 +143,124 @@ where task_spawner: TokioTaskExecutor::default().boxed(), gas_oracle_config: Default::default(), eth_state_cache_config: Default::default(), + next_env: Default::default(), + max_batch_size: 1, + pending_block_kind: PendingBlockKind::Full, + raw_tx_forwarder: ForwardConfig::default(), + send_raw_transaction_sync_timeout: Duration::from_secs(30), + evm_memory_limit: (1 << 32) - 1, } } +} +impl EthApiBuilder +where + N: RpcNodeCore, +{ /// Configures the task spawner used to spawn additional tasks. pub fn task_spawner(mut self, spawner: impl TaskSpawner + 'static) -> Self { self.task_spawner = Box::new(spawner); self } + /// Changes the configured converter. + pub fn with_rpc_converter( + self, + rpc_converter: RpcNew, + ) -> EthApiBuilder { + let Self { + components, + rpc_converter: _, + gas_cap, + max_simulate_blocks, + eth_proof_window, + fee_history_cache_config, + proof_permits, + eth_state_cache_config, + eth_cache, + gas_oracle, + blocking_task_pool, + task_spawner, + gas_oracle_config, + next_env, + max_batch_size, + pending_block_kind, + raw_tx_forwarder, + send_raw_transaction_sync_timeout, + evm_memory_limit, + } = self; + EthApiBuilder { + components, + rpc_converter, + gas_cap, + max_simulate_blocks, + eth_proof_window, + fee_history_cache_config, + proof_permits, + eth_state_cache_config, + eth_cache, + gas_oracle, + blocking_task_pool, + task_spawner, + gas_oracle_config, + next_env, + max_batch_size, + pending_block_kind, + raw_tx_forwarder, + send_raw_transaction_sync_timeout, + evm_memory_limit, + } + } + + /// Changes the configured pending environment builder. + pub fn with_pending_env_builder( + self, + next_env: NextEnvNew, + ) -> EthApiBuilder { + let Self { + components, + rpc_converter, + gas_cap, + max_simulate_blocks, + eth_proof_window, + fee_history_cache_config, + proof_permits, + eth_state_cache_config, + eth_cache, + gas_oracle, + blocking_task_pool, + task_spawner, + gas_oracle_config, + next_env: _, + max_batch_size, + pending_block_kind, + raw_tx_forwarder, + send_raw_transaction_sync_timeout, + evm_memory_limit, + } = self; + EthApiBuilder { + components, + rpc_converter, + gas_cap, + max_simulate_blocks, + eth_proof_window, + fee_history_cache_config, + proof_permits, + eth_state_cache_config, + eth_cache, + gas_oracle, + blocking_task_pool, + task_spawner, + gas_oracle_config, + next_env, + max_batch_size, + pending_block_kind, + raw_tx_forwarder, + send_raw_transaction_sync_timeout, + evm_memory_limit, + } + } + /// Sets `eth_cache` config for the cache that will be used if no [`EthStateCache`] is /// configured. pub const fn eth_state_cache_config( @@ -89,10 +272,7 @@ where } /// Sets `eth_cache` instance - pub fn eth_cache( - mut self, - eth_cache: EthStateCache, - ) -> Self { + pub fn eth_cache(mut self, eth_cache: EthStateCache) -> Self { self.eth_cache = Some(eth_cache); self } @@ -105,7 +285,7 @@ where } /// Sets `gas_oracle` instance - pub fn gas_oracle(mut self, gas_oracle: GasPriceOracle) -> Self { + pub fn gas_oracle(mut self, gas_oracle: GasPriceOracle) -> Self { self.gas_oracle = Some(gas_oracle); self } @@ -149,6 +329,130 @@ where self } + /// Sets the max batch size for batching transaction insertions. + pub const fn max_batch_size(mut self, max_batch_size: usize) -> Self { + self.max_batch_size = max_batch_size; + self + } + + /// Sets the pending block kind + pub const fn pending_block_kind(mut self, pending_block_kind: PendingBlockKind) -> Self { + self.pending_block_kind = pending_block_kind; + self + } + + /// Sets the raw transaction forwarder. + pub fn raw_tx_forwarder(mut self, tx_forwarder: ForwardConfig) -> Self { + self.raw_tx_forwarder = tx_forwarder; + self + } + + /// Returns the gas cap. + pub const fn get_gas_cap(&self) -> &GasCap { + &self.gas_cap + } + + /// Returns the maximum simulate blocks. + pub const fn get_max_simulate_blocks(&self) -> u64 { + self.max_simulate_blocks + } + + /// Returns the ETH proof window. + pub const fn get_eth_proof_window(&self) -> u64 { + self.eth_proof_window + } + + /// Returns a reference to the fee history cache config. + pub const fn get_fee_history_cache_config(&self) -> &FeeHistoryCacheConfig { + &self.fee_history_cache_config + } + + /// Returns the proof permits. + pub const fn get_proof_permits(&self) -> usize { + self.proof_permits + } + + /// Returns a reference to the ETH state cache config. + pub const fn get_eth_state_cache_config(&self) -> &EthStateCacheConfig { + &self.eth_state_cache_config + } + + /// Returns a reference to the gas oracle config. + pub const fn get_gas_oracle_config(&self) -> &GasPriceOracleConfig { + &self.gas_oracle_config + } + + /// Returns the max batch size. + pub const fn get_max_batch_size(&self) -> usize { + self.max_batch_size + } + + /// Returns the pending block kind. + pub const fn get_pending_block_kind(&self) -> PendingBlockKind { + self.pending_block_kind + } + + /// Returns a reference to the raw tx forwarder config. + pub const fn get_raw_tx_forwarder(&self) -> &ForwardConfig { + &self.raw_tx_forwarder + } + + /// Returns a mutable reference to the fee history cache config. + pub const fn fee_history_cache_config_mut(&mut self) -> &mut FeeHistoryCacheConfig { + &mut self.fee_history_cache_config + } + + /// Returns a mutable reference to the ETH state cache config. + pub const fn eth_state_cache_config_mut(&mut self) -> &mut EthStateCacheConfig { + &mut self.eth_state_cache_config + } + + /// Returns a mutable reference to the gas oracle config. + pub const fn gas_oracle_config_mut(&mut self) -> &mut GasPriceOracleConfig { + &mut self.gas_oracle_config + } + + /// Returns a mutable reference to the raw tx forwarder config. + pub const fn raw_tx_forwarder_mut(&mut self) -> &mut ForwardConfig { + &mut self.raw_tx_forwarder + } + + /// Modifies the fee history cache configuration using a closure. + pub fn modify_fee_history_cache_config(mut self, f: F) -> Self + where + F: FnOnce(&mut FeeHistoryCacheConfig), + { + f(&mut self.fee_history_cache_config); + self + } + + /// Modifies the ETH state cache configuration using a closure. + pub fn modify_eth_state_cache_config(mut self, f: F) -> Self + where + F: FnOnce(&mut EthStateCacheConfig), + { + f(&mut self.eth_state_cache_config); + self + } + + /// Modifies the gas oracle configuration using a closure. + pub fn modify_gas_oracle_config(mut self, f: F) -> Self + where + F: FnOnce(&mut GasPriceOracleConfig), + { + f(&mut self.gas_oracle_config); + self + } + + /// Modifies the raw tx forwarder configuration using a closure. + pub fn modify_raw_tx_forwarder(mut self, f: F) -> Self + where + F: FnOnce(&mut ForwardConfig), + { + f(&mut self.raw_tx_forwarder); + self + } + /// Builds the [`EthApiInner`] instance. /// /// If not configured, this will spawn the cache backend: [`EthStateCache::spawn`]. @@ -157,22 +461,14 @@ where /// /// This function panics if the blocking task pool cannot be built. /// This will panic if called outside the context of a Tokio runtime. - pub fn build_inner(self) -> EthApiInner + pub fn build_inner(self) -> EthApiInner where - Provider: BlockReaderIdExt - + StateProviderFactory - + ChainSpecProvider - + CanonStateSubscriptions< - Primitives: NodePrimitives, - > + Clone - + Unpin - + 'static, + Rpc: RpcConvert, + NextEnv: PendingEnvBuilder, { let Self { - provider, - pool, - network, - evm_config, + components, + rpc_converter, eth_state_cache_config, gas_oracle_config, eth_cache, @@ -184,29 +480,35 @@ where fee_history_cache_config, proof_permits, task_spawner, + next_env, + max_batch_size, + pending_block_kind, + raw_tx_forwarder, + send_raw_transaction_sync_timeout, + evm_memory_limit, } = self; + let provider = components.provider().clone(); + let eth_cache = eth_cache .unwrap_or_else(|| EthStateCache::spawn(provider.clone(), eth_state_cache_config)); let gas_oracle = gas_oracle.unwrap_or_else(|| { GasPriceOracle::new(provider.clone(), gas_oracle_config, eth_cache.clone()) }); - let fee_history_cache = FeeHistoryCache::new(fee_history_cache_config); + let fee_history_cache = + FeeHistoryCache::>::new(fee_history_cache_config); let new_canonical_blocks = provider.canonical_state_stream(); let fhc = fee_history_cache.clone(); let cache = eth_cache.clone(); - let prov = provider.clone(); task_spawner.spawn_critical( "cache canonical blocks for fee history task", Box::pin(async move { - fee_history_cache_new_blocks_task(fhc, new_canonical_blocks, prov, cache).await; + fee_history_cache_new_blocks_task(fhc, new_canonical_blocks, provider, cache).await; }), ); EthApiInner::new( - provider, - pool, - network, + components, eth_cache, gas_oracle, gas_cap, @@ -216,9 +518,15 @@ where BlockingTaskPool::build().expect("failed to build blocking task pool") }), fee_history_cache, - evm_config, task_spawner, proof_permits, + rpc_converter, + next_env, + max_batch_size, + pending_block_kind, + raw_tx_forwarder.forwarder_client(), + send_raw_transaction_sync_timeout, + evm_memory_limit, ) } @@ -230,17 +538,23 @@ where /// /// This function panics if the blocking task pool cannot be built. /// This will panic if called outside the context of a Tokio runtime. - pub fn build(self) -> EthApi + pub fn build(self) -> EthApi where - Provider: BlockReaderIdExt - + StateProviderFactory - + CanonStateSubscriptions< - Primitives: NodePrimitives, - > + ChainSpecProvider - + Clone - + Unpin - + 'static, + Rpc: RpcConvert, + NextEnv: PendingEnvBuilder, { - EthApi { inner: Arc::new(self.build_inner()), tx_resp_builder: EthTxBuilder } + EthApi { inner: Arc::new(self.build_inner()) } + } + + /// Sets the timeout for `send_raw_transaction_sync` RPC method. + pub const fn send_raw_transaction_sync_timeout(mut self, timeout: Duration) -> Self { + self.send_raw_transaction_sync_timeout = timeout; + self + } + + /// Sets the maximum memory the EVM can allocate per RPC request. + pub const fn evm_memory_limit(mut self, memory_limit: u64) -> Self { + self.evm_memory_limit = memory_limit; + self } } diff --git a/crates/rpc/rpc/src/eth/bundle.rs b/crates/rpc/rpc/src/eth/bundle.rs index 07ed06fdc90..d49b5486d3d 100644 --- a/crates/rpc/rpc/src/eth/bundle.rs +++ b/crates/rpc/rpc/src/eth/bundle.rs @@ -1,14 +1,14 @@ //! `Eth` bundle implementation and helpers. -use alloy_consensus::{EnvKzgSettings, Transaction as _}; +use alloy_consensus::{transaction::TxHashRef, EnvKzgSettings, Transaction as _}; use alloy_eips::eip7840::BlobParams; -use alloy_primitives::{Keccak256, U256}; +use alloy_evm::env::BlockEnvironment; +use alloy_primitives::{uint, Keccak256, U256}; use alloy_rpc_types_mev::{EthCallBundle, EthCallBundleResponse, EthCallBundleTransactionResult}; use jsonrpsee::core::RpcResult; use reth_chainspec::{ChainSpecProvider, EthChainSpec}; use reth_evm::{ConfigureEvm, Evm}; -use reth_primitives_traits::SignedTransaction; -use reth_revm::{database::StateProviderDatabase, db::CacheDB}; +use reth_revm::{database::StateProviderDatabase, State}; use reth_rpc_eth_api::{ helpers::{Call, EthTransactions, LoadPendingBlock}, EthCallBundleApiServer, FromEthApiError, FromEvmError, @@ -18,7 +18,9 @@ use reth_tasks::pool::BlockingTaskGuard; use reth_transaction_pool::{ EthBlobTransactionSidecar, EthPoolTransaction, PoolPooledTx, PoolTransaction, TransactionPool, }; -use revm::{context_interface::result::ResultAndState, DatabaseCommit, DatabaseRef}; +use revm::{ + context::Block, context_interface::result::ResultAndState, DatabaseCommit, DatabaseRef, +}; use std::sync::Arc; /// `Eth` bundle implementation. @@ -88,18 +90,18 @@ where let (mut evm_env, at) = self.eth_api().evm_env_at(block_id).await?; if let Some(coinbase) = coinbase { - evm_env.block_env.beneficiary = coinbase; + evm_env.block_env.inner_mut().beneficiary = coinbase; } // need to adjust the timestamp for the next block if let Some(timestamp) = timestamp { - evm_env.block_env.timestamp = timestamp; + evm_env.block_env.inner_mut().timestamp = U256::from(timestamp); } else { - evm_env.block_env.timestamp += 12; + evm_env.block_env.inner_mut().timestamp += uint!(12_U256); } if let Some(difficulty) = difficulty { - evm_env.block_env.difficulty = U256::from(difficulty); + evm_env.block_env.inner_mut().difficulty = U256::from(difficulty); } // Validate that the bundle does not contain more than MAX_BLOB_NUMBER_PER_BLOCK blob @@ -110,7 +112,7 @@ where .eth_api() .provider() .chain_spec() - .blob_params_at_timestamp(evm_env.block_env.timestamp) + .blob_params_at_timestamp(evm_env.block_env.timestamp().saturating_to()) .unwrap_or_else(BlobParams::cancun); if transactions.iter().filter_map(|tx| tx.blob_gas_used()).sum::() > blob_params.max_blob_gas_per_block() @@ -124,31 +126,31 @@ where } // default to call gas limit unless user requests a smaller limit - evm_env.block_env.gas_limit = self.inner.eth_api.call_gas_limit(); + evm_env.block_env.inner_mut().gas_limit = self.inner.eth_api.call_gas_limit(); if let Some(gas_limit) = gas_limit { - if gas_limit > evm_env.block_env.gas_limit { + if gas_limit > evm_env.block_env.gas_limit() { return Err( EthApiError::InvalidTransaction(RpcInvalidTransactionError::GasTooHigh).into() ) } - evm_env.block_env.gas_limit = gas_limit; + evm_env.block_env.inner_mut().gas_limit = gas_limit; } if let Some(base_fee) = base_fee { - evm_env.block_env.basefee = base_fee.try_into().unwrap_or(u64::MAX); + evm_env.block_env.inner_mut().basefee = base_fee.try_into().unwrap_or(u64::MAX); } - let state_block_number = evm_env.block_env.number; + let state_block_number = evm_env.block_env.number(); // use the block number of the request - evm_env.block_env.number = block_number; + evm_env.block_env.inner_mut().number = U256::from(block_number); let eth_api = self.eth_api().clone(); self.eth_api() .spawn_with_state_at_block(at, move |state| { - let coinbase = evm_env.block_env.beneficiary; - let basefee = evm_env.block_env.basefee; - let db = CacheDB::new(StateProviderDatabase::new(state)); + let coinbase = evm_env.block_env.beneficiary(); + let basefee = evm_env.block_env.basefee(); + let db = State::builder().with_database(StateProviderDatabase::new(state)).build(); let initial_coinbase = db .basic_ref(coinbase) @@ -253,7 +255,7 @@ where eth_sent_to_coinbase, gas_fees: total_gas_fees, results, - state_block_number, + state_block_number: state_block_number.to(), total_gas_used, }; @@ -273,7 +275,7 @@ where } } -/// Container type for `EthBundle` internals +/// Container type for `EthBundle` internals #[derive(Debug)] struct EthBundleInner { /// Access to commonly used code of the `eth` namespace diff --git a/crates/rpc/rpc/src/eth/core.rs b/crates/rpc/rpc/src/eth/core.rs index fe1f8bdcd4c..4084168c4f6 100644 --- a/crates/rpc/rpc/src/eth/core.rs +++ b/crates/rpc/rpc/src/eth/core.rs @@ -1,49 +1,55 @@ //! Implementation of the [`jsonrpsee`] generated [`EthApiServer`](crate::EthApi) trait //! Handles RPC requests for the `eth_` namespace. -use std::sync::Arc; +use std::{sync::Arc, time::Duration}; -use crate::{eth::EthTxBuilder, EthApiBuilder}; +use crate::{eth::helpers::types::EthRpcConverter, EthApiBuilder}; use alloy_consensus::BlockHeader; use alloy_eips::BlockNumberOrTag; use alloy_network::Ethereum; use alloy_primitives::{Bytes, U256}; +use alloy_rpc_client::RpcClient; use derive_more::Deref; +use reth_chainspec::{ChainSpec, ChainSpecProvider}; +use reth_evm_ethereum::EthEvmConfig; +use reth_network_api::noop::NoopNetwork; use reth_node_api::{FullNodeComponents, FullNodeTypes}; +use reth_rpc_convert::{RpcConvert, RpcConverter}; use reth_rpc_eth_api::{ - helpers::{EthSigner, SpawnBlocking}, - node::RpcNodeCoreExt, + helpers::{pending_block::PendingEnvBuilder, spec::SignersForRpc, SpawnBlocking}, + node::{RpcNodeCoreAdapter, RpcNodeCoreExt}, EthApiTypes, RpcNodeCore, }; use reth_rpc_eth_types::{ + builder::config::PendingBlockKind, receipt::EthReceiptConverter, tx_forward::ForwardConfig, EthApiError, EthStateCache, FeeHistoryCache, GasCap, GasPriceOracle, PendingBlock, }; -use reth_storage_api::{ - BlockReader, BlockReaderIdExt, NodePrimitivesProvider, ProviderBlock, ProviderReceipt, -}; +use reth_storage_api::{noop::NoopProvider, BlockReaderIdExt, ProviderHeader}; use reth_tasks::{ pool::{BlockingTaskGuard, BlockingTaskPool}, TaskSpawner, TokioTaskExecutor, }; -use tokio::sync::{broadcast, Mutex}; +use reth_transaction_pool::{ + blobstore::BlobSidecarConverter, noop::NoopTransactionPool, AddedTransactionOutcome, + BatchTxProcessor, BatchTxRequest, TransactionPool, +}; +use tokio::sync::{broadcast, mpsc, Mutex}; const DEFAULT_BROADCAST_CAPACITY: usize = 2000; -/// Helper type alias for [`EthApi`] with components from the given [`FullNodeComponents`]. -pub type EthApiFor = EthApi< - ::Provider, - ::Pool, - ::Network, +/// Helper type alias for [`RpcConverter`] with components from the given [`FullNodeComponents`]. +pub type EthRpcConverterFor = RpcConverter< + NetworkT, ::Evm, + EthReceiptConverter<<::Provider as ChainSpecProvider>::ChainSpec>, >; /// Helper type alias for [`EthApi`] with components from the given [`FullNodeComponents`]. -pub type EthApiBuilderFor = EthApiBuilder< - ::Provider, - ::Pool, - ::Network, - ::Evm, ->; +pub type EthApiFor = EthApi>; + +/// Helper type alias for [`EthApi`] with components from the given [`FullNodeComponents`]. +pub type EthApiBuilderFor = + EthApiBuilder>; /// `Eth` API implementation. /// @@ -60,26 +66,27 @@ pub type EthApiBuilderFor = EthApiBuilder< /// While this type requires various unrestricted generic components, trait bounds are enforced when /// additional traits are implemented for this type. #[derive(Deref)] -pub struct EthApi { +pub struct EthApi { /// All nested fields bundled together. #[deref] - pub(super) inner: Arc>, - /// Transaction RPC response builder. - pub tx_resp_builder: EthTxBuilder, + pub(super) inner: Arc>, } -impl Clone for EthApi +impl Clone for EthApi where - Provider: BlockReader, + N: RpcNodeCore, + Rpc: RpcConvert, { fn clone(&self) -> Self { - Self { inner: self.inner.clone(), tx_resp_builder: EthTxBuilder } + Self { inner: self.inner.clone() } } } -impl EthApi -where - Provider: BlockReaderIdExt, +impl + EthApi< + RpcNodeCoreAdapter, + EthRpcConverter, + > { /// Convenience fn to obtain a new [`EthApiBuilder`] instance with mandatory components. /// @@ -93,6 +100,7 @@ where /// # Create an instance with noop ethereum implementations /// /// ```no_run + /// use alloy_network::Ethereum; /// use reth_evm_ethereum::EthEvmConfig; /// use reth_network_api::noop::NoopNetwork; /// use reth_provider::noop::NoopProvider; @@ -106,35 +114,51 @@ where /// ) /// .build(); /// ``` - pub fn builder( + #[expect(clippy::type_complexity)] + pub fn builder( provider: Provider, pool: Pool, network: Network, evm_config: EvmConfig, - ) -> EthApiBuilder { + ) -> EthApiBuilder< + RpcNodeCoreAdapter, + RpcConverter>, + > + where + RpcNodeCoreAdapter: + RpcNodeCore, Evm = EvmConfig>, + { EthApiBuilder::new(provider, pool, network, evm_config) } +} +impl EthApi +where + N: RpcNodeCore, + Rpc: RpcConvert, + (): PendingEnvBuilder, +{ /// Creates a new, shareable instance using the default tokio task spawner. #[expect(clippy::too_many_arguments)] pub fn new( - provider: Provider, - pool: Pool, - network: Network, - eth_cache: EthStateCache, - gas_oracle: GasPriceOracle, + components: N, + eth_cache: EthStateCache, + gas_oracle: GasPriceOracle, gas_cap: impl Into, max_simulate_blocks: u64, eth_proof_window: u64, blocking_task_pool: BlockingTaskPool, - fee_history_cache: FeeHistoryCache, - evm_config: EvmConfig, + fee_history_cache: FeeHistoryCache>, proof_permits: usize, + rpc_converter: Rpc, + max_batch_size: usize, + pending_block_kind: PendingBlockKind, + raw_tx_forwarder: ForwardConfig, + send_raw_transaction_sync_timeout: Duration, + evm_memory_limit: u64, ) -> Self { let inner = EthApiInner::new( - provider, - pool, - network, + components, eth_cache, gas_oracle, gas_cap, @@ -142,42 +166,45 @@ where eth_proof_window, blocking_task_pool, fee_history_cache, - evm_config, TokioTaskExecutor::default().boxed(), proof_permits, + rpc_converter, + (), + max_batch_size, + pending_block_kind, + raw_tx_forwarder.forwarder_client(), + send_raw_transaction_sync_timeout, + evm_memory_limit, ); - Self { inner: Arc::new(inner), tx_resp_builder: EthTxBuilder } + Self { inner: Arc::new(inner) } } } -impl EthApiTypes for EthApi +impl EthApiTypes for EthApi where - Self: Send + Sync, - Provider: BlockReader, + N: RpcNodeCore, + Rpc: RpcConvert, { type Error = EthApiError; - type NetworkTypes = Ethereum; - type TransactionCompat = EthTxBuilder; + type NetworkTypes = Rpc::Network; + type RpcConvert = Rpc; - fn tx_resp_builder(&self) -> &Self::TransactionCompat { + fn tx_resp_builder(&self) -> &Self::RpcConvert { &self.tx_resp_builder } } -impl RpcNodeCore for EthApi +impl RpcNodeCore for EthApi where - Provider: BlockReader + NodePrimitivesProvider + Clone + Unpin, - Pool: Send + Sync + Clone + Unpin, - Network: Send + Sync + Clone, - EvmConfig: Send + Sync + Clone + Unpin, + N: RpcNodeCore, + Rpc: RpcConvert, { - type Primitives = Provider::Primitives; - type Provider = Provider; - type Pool = Pool; - type Evm = EvmConfig; - type Network = Network; - type PayloadBuilder = (); + type Primitives = N::Primitives; + type Provider = N::Provider; + type Pool = N::Pool; + type Evm = N::Evm; + type Network = N::Network; fn pool(&self) -> &Self::Pool { self.inner.pool() @@ -191,44 +218,36 @@ where self.inner.network() } - fn payload_builder(&self) -> &Self::PayloadBuilder { - &() - } - fn provider(&self) -> &Self::Provider { self.inner.provider() } } -impl RpcNodeCoreExt - for EthApi +impl RpcNodeCoreExt for EthApi where - Provider: BlockReader + NodePrimitivesProvider + Clone + Unpin, - Pool: Send + Sync + Clone + Unpin, - Network: Send + Sync + Clone, - EvmConfig: Send + Sync + Clone + Unpin, + N: RpcNodeCore, + Rpc: RpcConvert, { #[inline] - fn cache(&self) -> &EthStateCache, ProviderReceipt> { + fn cache(&self) -> &EthStateCache { self.inner.cache() } } -impl std::fmt::Debug - for EthApi +impl std::fmt::Debug for EthApi where - Provider: BlockReader, + N: RpcNodeCore, + Rpc: RpcConvert, { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("EthApi").finish_non_exhaustive() } } -impl SpawnBlocking - for EthApi +impl SpawnBlocking for EthApi where - Self: Clone + Send + Sync + 'static, - Provider: BlockReader, + N: RpcNodeCore, + Rpc: RpcConvert, { #[inline] fn io_task_spawner(&self) -> impl TaskSpawner { @@ -248,19 +267,15 @@ where /// Container type `EthApi` #[expect(missing_debug_implementations)] -pub struct EthApiInner { - /// The transaction pool. - pool: Pool, - /// The provider that can interact with the chain. - provider: Provider, - /// An interface to interact with the network - network: Network, +pub struct EthApiInner { + /// The components of the node. + components: N, /// All configured Signers - signers: parking_lot::RwLock>>>, + signers: SignersForRpc, /// The async cache frontend for eth related data - eth_cache: EthStateCache, + eth_cache: EthStateCache, /// The async gas oracle frontend for gas price suggestions - gas_oracle: GasPriceOracle, + gas_oracle: GasPriceOracle, /// Maximum gas limit for `eth_call` and call tracing RPC methods. gas_cap: u64, /// Maximum number of blocks for `eth_simulateV1`. @@ -272,46 +287,75 @@ pub struct EthApiInner { /// The type that can spawn tasks which would otherwise block. task_spawner: Box, /// Cached pending block if any - pending_block: Mutex>>, + pending_block: Mutex>>, /// A pool dedicated to CPU heavy blocking tasks. blocking_task_pool: BlockingTaskPool, /// Cache for block fees history - fee_history_cache: FeeHistoryCache, - /// The type that defines how to configure the EVM - evm_config: EvmConfig, + fee_history_cache: FeeHistoryCache>, /// Guard for getproof calls blocking_task_guard: BlockingTaskGuard, /// Transaction broadcast channel raw_tx_sender: broadcast::Sender, + + /// Raw transaction forwarder + raw_tx_forwarder: Option, + + /// Converter for RPC types. + tx_resp_builder: Rpc, + + /// Builder for pending block environment. + next_env_builder: Box>, + + /// Transaction batch sender for batching tx insertions + tx_batch_sender: + mpsc::UnboundedSender::Transaction>>, + + /// Configuration for pending block construction. + pending_block_kind: PendingBlockKind, + + /// Timeout duration for `send_raw_transaction_sync` RPC method. + send_raw_transaction_sync_timeout: Duration, + + /// Blob sidecar converter + blob_sidecar_converter: BlobSidecarConverter, + + /// Maximum memory the EVM can allocate per RPC request. + evm_memory_limit: u64, } -impl EthApiInner +impl EthApiInner where - Provider: BlockReaderIdExt, + N: RpcNodeCore, + Rpc: RpcConvert, { /// Creates a new, shareable instance using the default tokio task spawner. #[expect(clippy::too_many_arguments)] pub fn new( - provider: Provider, - pool: Pool, - network: Network, - eth_cache: EthStateCache, - gas_oracle: GasPriceOracle, + components: N, + eth_cache: EthStateCache, + gas_oracle: GasPriceOracle, gas_cap: impl Into, max_simulate_blocks: u64, eth_proof_window: u64, blocking_task_pool: BlockingTaskPool, - fee_history_cache: FeeHistoryCache, - evm_config: EvmConfig, + fee_history_cache: FeeHistoryCache>, task_spawner: Box, proof_permits: usize, + tx_resp_builder: Rpc, + next_env: impl PendingEnvBuilder, + max_batch_size: usize, + pending_block_kind: PendingBlockKind, + raw_tx_forwarder: Option, + send_raw_transaction_sync_timeout: Duration, + evm_memory_limit: u64, ) -> Self { let signers = parking_lot::RwLock::new(Default::default()); // get the block number of the latest block let starting_block = U256::from( - provider + components + .provider() .header_by_number_or_tag(BlockNumberOrTag::Latest) .ok() .flatten() @@ -321,10 +365,13 @@ where let (raw_tx_sender, _) = broadcast::channel(DEFAULT_BROADCAST_CAPACITY); + // Create tx pool insertion batcher + let (processor, tx_batch_sender) = + BatchTxProcessor::new(components.pool().clone(), max_batch_size); + task_spawner.spawn_critical("tx-batcher", Box::pin(processor)); + Self { - provider, - pool, - network, + components, signers, eth_cache, gas_oracle, @@ -336,37 +383,56 @@ where pending_block: Default::default(), blocking_task_pool, fee_history_cache, - evm_config, blocking_task_guard: BlockingTaskGuard::new(proof_permits), raw_tx_sender, + raw_tx_forwarder, + tx_resp_builder, + next_env_builder: Box::new(next_env), + tx_batch_sender, + pending_block_kind, + send_raw_transaction_sync_timeout, + blob_sidecar_converter: BlobSidecarConverter::new(), + evm_memory_limit, } } } -impl EthApiInner +impl EthApiInner where - Provider: BlockReader, + N: RpcNodeCore, + Rpc: RpcConvert, { /// Returns a handle to data on disk. #[inline] - pub const fn provider(&self) -> &Provider { - &self.provider + pub fn provider(&self) -> &N::Provider { + self.components.provider() + } + + /// Returns a handle to the transaction response builder. + #[inline] + pub const fn tx_resp_builder(&self) -> &Rpc { + &self.tx_resp_builder } /// Returns a handle to data in memory. #[inline] - pub const fn cache(&self) -> &EthStateCache { + pub const fn cache(&self) -> &EthStateCache { &self.eth_cache } /// Returns a handle to the pending block. #[inline] - pub const fn pending_block( - &self, - ) -> &Mutex>> { + pub const fn pending_block(&self) -> &Mutex>> { &self.pending_block } + /// Returns a type that knows how to build a [`reth_evm::ConfigureEvm::NextBlockEnvCtx`] for a + /// pending block. + #[inline] + pub const fn pending_env_builder(&self) -> &dyn PendingEnvBuilder { + &*self.next_env_builder + } + /// Returns a handle to the task spawner. #[inline] pub const fn task_spawner(&self) -> &dyn TaskSpawner { @@ -381,14 +447,14 @@ where /// Returns a handle to the EVM config. #[inline] - pub const fn evm_config(&self) -> &EvmConfig { - &self.evm_config + pub fn evm_config(&self) -> &N::Evm { + self.components.evm_config() } /// Returns a handle to the transaction pool. #[inline] - pub const fn pool(&self) -> &Pool { - &self.pool + pub fn pool(&self) -> &N::Pool { + self.components.pool() } /// Returns the gas cap. @@ -405,21 +471,19 @@ where /// Returns a handle to the gas oracle. #[inline] - pub const fn gas_oracle(&self) -> &GasPriceOracle { + pub const fn gas_oracle(&self) -> &GasPriceOracle { &self.gas_oracle } /// Returns a handle to the fee history cache. #[inline] - pub const fn fee_history_cache(&self) -> &FeeHistoryCache { + pub const fn fee_history_cache(&self) -> &FeeHistoryCache> { &self.fee_history_cache } /// Returns a handle to the signers. #[inline] - pub const fn signers( - &self, - ) -> &parking_lot::RwLock>>> { + pub const fn signers(&self) -> &SignersForRpc { &self.signers } @@ -431,8 +495,8 @@ where /// Returns the inner `Network` #[inline] - pub const fn network(&self) -> &Network { - &self.network + pub fn network(&self) -> &N::Network { + self.components.network() } /// The maximum number of blocks into the past for generating state proofs. @@ -458,11 +522,65 @@ where pub fn broadcast_raw_transaction(&self, raw_tx: Bytes) { let _ = self.raw_tx_sender.send(raw_tx); } + + /// Returns the transaction batch sender + #[inline] + const fn tx_batch_sender( + &self, + ) -> &mpsc::UnboundedSender::Transaction>> { + &self.tx_batch_sender + } + + /// Adds an _unvalidated_ transaction into the pool via the transaction batch sender. + #[inline] + pub async fn add_pool_transaction( + &self, + transaction: ::Transaction, + ) -> Result { + let (response_tx, response_rx) = tokio::sync::oneshot::channel(); + let request = reth_transaction_pool::BatchTxRequest::new(transaction, response_tx); + + self.tx_batch_sender() + .send(request) + .map_err(|_| reth_rpc_eth_types::EthApiError::BatchTxSendError)?; + + Ok(response_rx.await??) + } + + /// Returns the pending block kind + #[inline] + pub const fn pending_block_kind(&self) -> PendingBlockKind { + self.pending_block_kind + } + + /// Returns a handle to the raw transaction forwarder. + #[inline] + pub const fn raw_tx_forwarder(&self) -> Option<&RpcClient> { + self.raw_tx_forwarder.as_ref() + } + + /// Returns the timeout duration for `send_raw_transaction_sync` RPC method. + #[inline] + pub const fn send_raw_transaction_sync_timeout(&self) -> Duration { + self.send_raw_transaction_sync_timeout + } + + /// Returns a handle to the blob sidecar converter. + #[inline] + pub const fn blob_sidecar_converter(&self) -> &BlobSidecarConverter { + &self.blob_sidecar_converter + } + + /// Returns the EVM memory limit. + #[inline] + pub const fn evm_memory_limit(&self) -> u64 { + self.evm_memory_limit + } } #[cfg(test)] mod tests { - use crate::{EthApi, EthApiBuilder}; + use crate::{eth::helpers::types::EthRpcConverter, EthApi, EthApiBuilder}; use alloy_consensus::{Block, BlockBody, Header}; use alloy_eips::BlockNumberOrTag; use alloy_primitives::{Signature, B256, U64}; @@ -470,31 +588,41 @@ mod tests { use jsonrpsee_types::error::INVALID_PARAMS_CODE; use rand::Rng; use reth_chain_state::CanonStateSubscriptions; - use reth_chainspec::{BaseFeeParams, ChainSpec, ChainSpecProvider}; + use reth_chainspec::{ChainSpec, ChainSpecProvider, EthChainSpec}; use reth_ethereum_primitives::TransactionSigned; use reth_evm_ethereum::EthEvmConfig; use reth_network_api::noop::NoopNetwork; - use reth_provider::test_utils::{MockEthProvider, NoopProvider}; - use reth_rpc_eth_api::EthApiServer; + use reth_provider::{ + test_utils::{MockEthProvider, NoopProvider}, + StageCheckpointReader, + }; + use reth_rpc_eth_api::{node::RpcNodeCoreAdapter, EthApiServer}; use reth_storage_api::{BlockReader, BlockReaderIdExt, StateProviderFactory}; use reth_testing_utils::generators; use reth_transaction_pool::test_utils::{testing_pool, TestPool}; + type FakeEthApi

= EthApi< + RpcNodeCoreAdapter, + EthRpcConverter, + >; + fn build_test_eth_api< P: BlockReaderIdExt< Block = reth_ethereum_primitives::Block, Receipt = reth_ethereum_primitives::Receipt, Header = alloy_consensus::Header, + Transaction = reth_ethereum_primitives::TransactionSigned, > + BlockReader + ChainSpecProvider + StateProviderFactory + CanonStateSubscriptions + + StageCheckpointReader + Unpin + Clone + 'static, >( provider: P, - ) -> EthApi { + ) -> FakeEthApi

{ EthApiBuilder::new( provider.clone(), testing_pool(), @@ -510,7 +638,7 @@ mod tests { mut oldest_block: Option, block_count: u64, mock_provider: MockEthProvider, - ) -> (EthApi, Vec, Vec) { + ) -> (FakeEthApi, Vec, Vec) { let mut rng = generators::rng(); // Build mock data @@ -582,11 +710,11 @@ mod tests { // Add final base fee (for the next block outside of the request) let last_header = last_header.unwrap(); - base_fees_per_gas.push(BaseFeeParams::ethereum().next_block_base_fee( - last_header.gas_used, - last_header.gas_limit, - last_header.base_fee_per_gas.unwrap_or_default(), - ) as u128); + let spec = mock_provider.chain_spec(); + base_fees_per_gas.push( + spec.next_block_base_fee(&last_header, last_header.timestamp).unwrap_or_default() + as u128, + ); let eth_api = build_test_eth_api(mock_provider); @@ -596,7 +724,7 @@ mod tests { /// Invalid block range #[tokio::test] async fn test_fee_history_empty() { - let response = as EthApiServer<_, _, _, _>>::fee_history( + let response = as EthApiServer<_, _, _, _, _, _>>::fee_history( &build_test_eth_api(NoopProvider::default()), U64::from(1), BlockNumberOrTag::Latest, @@ -618,7 +746,7 @@ mod tests { let (eth_api, _, _) = prepare_eth_api(newest_block, oldest_block, block_count, MockEthProvider::default()); - let response = as EthApiServer<_, _, _, _>>::fee_history( + let response = as EthApiServer<_, _, _, _, _, _>>::fee_history( ð_api, U64::from(newest_block + 1), newest_block.into(), @@ -641,7 +769,7 @@ mod tests { let (eth_api, _, _) = prepare_eth_api(newest_block, oldest_block, block_count, MockEthProvider::default()); - let response = as EthApiServer<_, _, _, _>>::fee_history( + let response = as EthApiServer<_, _, _, _, _, _>>::fee_history( ð_api, U64::from(1), (newest_block + 1000).into(), @@ -664,7 +792,7 @@ mod tests { let (eth_api, _, _) = prepare_eth_api(newest_block, oldest_block, block_count, MockEthProvider::default()); - let response = as EthApiServer<_, _, _, _>>::fee_history( + let response = as EthApiServer<_, _, _, _, _, _>>::fee_history( ð_api, U64::from(0), newest_block.into(), diff --git a/crates/rpc/rpc/src/eth/filter.rs b/crates/rpc/rpc/src/eth/filter.rs index 95e0e2daaf1..22b14d7a174 100644 --- a/crates/rpc/rpc/src/eth/filter.rs +++ b/crates/rpc/rpc/src/eth/filter.rs @@ -1,18 +1,26 @@ //! `eth_` `Filter` RPC handler implementation use alloy_consensus::BlockHeader; -use alloy_primitives::TxHash; +use alloy_eips::BlockNumberOrTag; +use alloy_primitives::{Sealable, TxHash}; use alloy_rpc_types_eth::{ - BlockNumHash, Filter, FilterBlockOption, FilterChanges, FilterId, FilteredParams, Log, + BlockNumHash, Filter, FilterBlockOption, FilterChanges, FilterId, Log, PendingTransactionFilterKind, }; use async_trait::async_trait; -use futures::future::TryFutureExt; +use futures::{ + future::TryFutureExt, + stream::{FuturesOrdered, StreamExt}, + Future, +}; +use itertools::Itertools; use jsonrpsee::{core::RpcResult, server::IdProvider}; use reth_errors::ProviderError; +use reth_primitives_traits::{NodePrimitives, SealedHeader}; use reth_rpc_eth_api::{ - EngineEthFilter, EthApiTypes, EthFilterApiServer, FullEthApiTypes, QueryLimits, RpcNodeCoreExt, - RpcTransaction, TransactionCompat, + helpers::{EthBlocks, LoadReceipt}, + EngineEthFilter, EthApiTypes, EthFilterApiServer, FullEthApiTypes, QueryLimits, RpcConvert, + RpcNodeCoreExt, RpcTransaction, }; use reth_rpc_eth_types::{ logs_utils::{self, append_matching_block_logs, ProviderOrBlock}, @@ -21,28 +29,32 @@ use reth_rpc_eth_types::{ use reth_rpc_server_types::{result::rpc_error_with_code, ToRpcResult}; use reth_storage_api::{ BlockHashReader, BlockIdReader, BlockNumReader, BlockReader, HeaderProvider, ProviderBlock, - ProviderReceipt, + ProviderReceipt, ReceiptProvider, }; use reth_tasks::TaskSpawner; use reth_transaction_pool::{NewSubpoolTransactionStream, PoolTransaction, TransactionPool}; use std::{ - collections::HashMap, + collections::{HashMap, VecDeque}, fmt, - future::Future, - iter::StepBy, + iter::{Peekable, StepBy}, ops::RangeInclusive, + pin::Pin, sync::Arc, time::{Duration, Instant}, }; use tokio::{ - sync::{mpsc::Receiver, Mutex}, + sync::{mpsc::Receiver, oneshot, Mutex}, time::MissedTickBehavior, }; -use tracing::{error, trace}; +use tracing::{debug, error, trace}; impl EngineEthFilter for EthFilter where - Eth: FullEthApiTypes + RpcNodeCoreExt + 'static, + Eth: FullEthApiTypes + + RpcNodeCoreExt + + LoadReceipt + + EthBlocks + + 'static, { /// Returns logs matching given filter object, no query limits fn logs( @@ -51,13 +63,31 @@ where limits: QueryLimits, ) -> impl Future>> + Send { trace!(target: "rpc::eth", "Serving eth_getLogs"); - self.inner.logs_for_filter(filter, limits).map_err(|e| e.into()) + self.logs_for_filter(filter, limits).map_err(|e| e.into()) } } +/// Threshold for deciding between cached and range mode processing +const CACHED_MODE_BLOCK_THRESHOLD: u64 = 250; + +/// Threshold for bloom filter matches that triggers reduced caching +const HIGH_BLOOM_MATCH_THRESHOLD: usize = 20; + +/// Threshold for bloom filter matches that triggers moderately reduced caching +const MODERATE_BLOOM_MATCH_THRESHOLD: usize = 10; + +/// Minimum block count to apply bloom filter match adjustments +const BLOOM_ADJUSTMENT_MIN_BLOCKS: u64 = 100; + /// The maximum number of headers we read at once when handling a range filter. const MAX_HEADERS_RANGE: u64 = 1_000; // with ~530bytes per header this is ~500kb +/// Threshold for enabling parallel processing in range mode +const PARALLEL_PROCESSING_THRESHOLD: usize = 1000; + +/// Default concurrency for parallel processing +const DEFAULT_PARALLEL_CONCURRENCY: usize = 4; + /// `Eth` filter RPC implementation. /// /// This type handles `eth_` rpc requests related to filters (`eth_getLogs`). @@ -169,7 +199,11 @@ where impl EthFilter where - Eth: FullEthApiTypes + RpcNodeCoreExt, + Eth: FullEthApiTypes + + RpcNodeCoreExt + + LoadReceipt + + EthBlocks + + 'static, { /// Access the underlying provider. fn provider(&self) -> &Eth::Provider { @@ -244,8 +278,9 @@ where }; let logs = self .inner + .clone() .get_logs_in_block_range( - &filter, + *filter, from_block_number, to_block_number, self.inner.query_limits, @@ -274,14 +309,23 @@ where } }; - self.inner.logs_for_filter(filter, self.inner.query_limits).await + self.logs_for_filter(filter, self.inner.query_limits).await + } + + /// Returns logs matching given filter object. + async fn logs_for_filter( + &self, + filter: Filter, + limits: QueryLimits, + ) -> Result, EthFilterError> { + self.inner.clone().logs_for_filter(filter, limits).await } } #[async_trait] impl EthFilterApiServer> for EthFilter where - Eth: FullEthApiTypes + RpcNodeCoreExt + 'static, + Eth: FullEthApiTypes + RpcNodeCoreExt + LoadReceipt + EthBlocks + 'static, { /// Handler for `eth_newFilter` async fn new_filter(&self, filter: Filter) -> RpcResult { @@ -314,7 +358,7 @@ where let stream = self.pool().new_pending_pool_transactions_listener(); let full_txs_receiver = FullTransactionsReceiver::new( stream, - self.inner.eth_api.tx_resp_builder().clone(), + dyn_clone::clone(self.inner.eth_api.tx_resp_builder()), ); FilterKind::PendingTransaction(PendingTransactionKind::FullTransaction(Arc::new( full_txs_receiver, @@ -322,8 +366,6 @@ where } }; - //let filter = FilterKind::PendingTransaction(transaction_kind); - // Install the filter and propagate any errors self.inner.install_filter(transaction_kind).await } @@ -364,7 +406,7 @@ where /// Handler for `eth_getLogs` async fn logs(&self, filter: Filter) -> RpcResult> { trace!(target: "rpc::eth", "Serving eth_getLogs"); - Ok(self.inner.logs_for_filter(filter, self.inner.query_limits).await?) + Ok(self.logs_for_filter(filter, self.inner.query_limits).await?) } } @@ -398,7 +440,11 @@ struct EthFilterInner { impl EthFilterInner where - Eth: RpcNodeCoreExt + EthApiTypes, + Eth: RpcNodeCoreExt + + EthApiTypes + + LoadReceipt + + EthBlocks + + 'static, { /// Access the underlying provider. fn provider(&self) -> &Eth::Provider { @@ -406,15 +452,13 @@ where } /// Access the underlying [`EthStateCache`]. - fn eth_cache( - &self, - ) -> &EthStateCache, ProviderReceipt> { + fn eth_cache(&self) -> &EthStateCache { self.eth_api.cache() } /// Returns logs matching given filter object. async fn logs_for_filter( - &self, + self: Arc, filter: Filter, limits: QueryLimits, ) -> Result, EthFilterError> { @@ -443,7 +487,7 @@ where maybe_block .map(ProviderOrBlock::Block) .unwrap_or_else(|| ProviderOrBlock::Provider(self.provider())), - &FilteredParams::new(Some(filter)), + &filter, block_num_hash, &receipts, false, @@ -453,10 +497,43 @@ where Ok(all_logs) } FilterBlockOption::Range { from_block, to_block } => { - // compute the range - let info = self.provider().chain_info()?; + // Handle special case where from block is pending + if from_block.is_some_and(|b| b.is_pending()) { + let to_block = to_block.unwrap_or(BlockNumberOrTag::Pending); + if !(to_block.is_pending() || to_block.is_number()) { + // always empty range + return Ok(Vec::new()); + } + // Try to get pending block and receipts + if let Ok(Some(pending_block)) = self.eth_api.local_pending_block().await { + if let BlockNumberOrTag::Number(to_block) = to_block && + to_block < pending_block.block.number() + { + // this block range is empty based on the user input + return Ok(Vec::new()); + } - // we start at the most recent block if unset in filter + let info = self.provider().chain_info()?; + if pending_block.block.number() > info.best_number { + // only consider the pending block if it is ahead of the chain + let mut all_logs = Vec::new(); + let timestamp = pending_block.block.timestamp(); + let block_num_hash = pending_block.block.num_hash(); + append_matching_block_logs( + &mut all_logs, + ProviderOrBlock::::Block(pending_block.block), + &filter, + block_num_hash, + &pending_block.receipts, + false, // removed = false for pending blocks + timestamp, + )?; + return Ok(all_logs); + } + } + } + + let info = self.provider().chain_info()?; let start_block = info.best_number; let from = from_block .map(|num| self.provider().convert_block_number(num)) @@ -466,9 +543,18 @@ where .map(|num| self.provider().convert_block_number(num)) .transpose()? .flatten(); + + if let Some(f) = from && + f > info.best_number + { + // start block higher than local head, can return empty + return Ok(Vec::new()); + } + let (from_block_number, to_block_number) = logs_utils::get_filter_block_range(from, to, start_block, info); - self.get_logs_in_block_range(&filter, from_block_number, to_block_number, limits) + + self.get_logs_in_block_range(filter, from_block_number, to_block_number, limits) .await } } @@ -480,7 +566,12 @@ where kind: FilterKind>, ) -> RpcResult { let last_poll_block_number = self.provider().best_block_number().to_rpc_result()?; - let id = FilterId::from(self.id_provider.next_id()); + let subscription_id = self.id_provider.next_id(); + + let id = match subscription_id { + jsonrpsee_types::SubscriptionId::Num(n) => FilterId::Num(n), + jsonrpsee_types::SubscriptionId::Str(s) => FilterId::Str(s.into_owned()), + }; let mut filters = self.active_filters.inner.lock().await; filters.insert( id.clone(), @@ -499,14 +590,15 @@ where /// - underlying database error /// - amount of matches exceeds configured limit async fn get_logs_in_block_range( - &self, - filter: &Filter, + self: Arc, + filter: Filter, from_block: u64, to_block: u64, limits: QueryLimits, ) -> Result, EthFilterError> { trace!(target: "rpc::eth::filter", from=from_block, to=to_block, ?filter, "finding logs in range"); + // perform boundary checks first if to_block < from_block { return Err(EthFilterError::InvalidBlockRangeParams) } @@ -517,65 +609,115 @@ where return Err(EthFilterError::QueryExceedsMaxBlocks(max_blocks_per_filter)) } + let (tx, rx) = oneshot::channel(); + let this = self.clone(); + self.task_spawner.spawn_blocking(Box::pin(async move { + let res = + this.get_logs_in_block_range_inner(&filter, from_block, to_block, limits).await; + let _ = tx.send(res); + })); + + rx.await.map_err(|_| EthFilterError::InternalError)? + } + + /// Returns all logs in the given _inclusive_ range that match the filter + /// + /// Note: This function uses a mix of blocking db operations for fetching indices and header + /// ranges and utilizes the rpc cache for optimistically fetching receipts and blocks. + /// This function is considered blocking and should thus be spawned on a blocking task. + /// + /// Returns an error if: + /// - underlying database error + async fn get_logs_in_block_range_inner( + self: Arc, + filter: &Filter, + from_block: u64, + to_block: u64, + limits: QueryLimits, + ) -> Result, EthFilterError> { let mut all_logs = Vec::new(); - let filter_params = FilteredParams::new(Some(filter.clone())); + let mut matching_headers = Vec::new(); - // derive bloom filters from filter input, so we can check headers for matching logs - let address_filter = FilteredParams::address_filter(&filter.address); - let topics_filter = FilteredParams::topics_filter(&filter.topics); + // get current chain tip to determine processing mode + let chain_tip = self.provider().best_block_number()?; - // loop over the range of new blocks and check logs if the filter matches the log's bloom - // filter + // first collect all headers that match the bloom filter for cached mode decision for (from, to) in BlockRangeInclusiveIter::new(from_block..=to_block, self.max_headers_range) { let headers = self.provider().headers_range(from..=to)?; - for (idx, header) in headers.iter().enumerate() { - // only if filter matches - if FilteredParams::matches_address(header.logs_bloom(), &address_filter) && - FilteredParams::matches_topics(header.logs_bloom(), &topics_filter) - { - // these are consecutive headers, so we can use the parent hash of the next - // block to get the current header's hash - let block_hash = match headers.get(idx + 1) { - Some(child) => child.parent_hash(), - None => self - .provider() - .block_hash(header.number())? - .ok_or_else(|| ProviderError::HeaderNotFound(header.number().into()))?, - }; - - let num_hash = BlockNumHash::new(header.number(), block_hash); - if let Some((receipts, maybe_block)) = - self.eth_cache().get_receipts_and_maybe_block(num_hash.hash).await? - { - append_matching_block_logs( - &mut all_logs, - maybe_block - .map(ProviderOrBlock::Block) - .unwrap_or_else(|| ProviderOrBlock::Provider(self.provider())), - &filter_params, - num_hash, - &receipts, - false, - header.timestamp(), - )?; - - // size check but only if range is multiple blocks, so we always return all - // logs of a single block - let is_multi_block_range = from_block != to_block; - if let Some(max_logs_per_response) = limits.max_logs_per_response { - if is_multi_block_range && all_logs.len() > max_logs_per_response { - return Err(EthFilterError::QueryExceedsMaxResults { - max_logs: max_logs_per_response, - from_block, - to_block: num_hash.number.saturating_sub(1), - }); - } - } - } + let mut headers_iter = headers.into_iter().peekable(); + + while let Some(header) = headers_iter.next() { + if !filter.matches_bloom(header.logs_bloom()) { + continue } + + let current_number = header.number(); + + let block_hash = match headers_iter.peek() { + Some(next_header) if next_header.number() == current_number + 1 => { + // Headers are consecutive, use the more efficient parent_hash + next_header.parent_hash() + } + _ => { + // Headers not consecutive or last header, calculate hash + header.hash_slow() + } + }; + + matching_headers.push(SealedHeader::new(header, block_hash)); + } + } + + // initialize the appropriate range mode based on collected headers + let mut range_mode = RangeMode::new( + self.clone(), + matching_headers, + from_block, + to_block, + self.max_headers_range, + chain_tip, + ); + + // iterate through the range mode to get receipts and blocks + while let Some(ReceiptBlockResult { receipts, recovered_block, header }) = + range_mode.next().await? + { + let num_hash = header.num_hash(); + append_matching_block_logs( + &mut all_logs, + recovered_block + .map(ProviderOrBlock::Block) + .unwrap_or_else(|| ProviderOrBlock::Provider(self.provider())), + filter, + num_hash, + &receipts, + false, + header.timestamp(), + )?; + + // size check but only if range is multiple blocks, so we always return all + // logs of a single block + let is_multi_block_range = from_block != to_block; + if let Some(max_logs_per_response) = limits.max_logs_per_response && + is_multi_block_range && + all_logs.len() > max_logs_per_response + { + debug!( + target: "rpc::eth::filter", + logs_found = all_logs.len(), + max_logs_per_response, + from_block, + to_block = num_hash.number.saturating_sub(1), + "Query exceeded max logs per response limit" + ); + return Err(EthFilterError::QueryExceedsMaxResults { + max_logs: max_logs_per_response, + from_block, + to_block: num_hash.number.saturating_sub(1), + }); } } @@ -642,7 +784,7 @@ struct FullTransactionsReceiver { impl FullTransactionsReceiver where T: PoolTransaction + 'static, - TxCompat: TransactionCompat, + TxCompat: RpcConvert>, { /// Creates a new `FullTransactionsReceiver` encapsulating the provided transaction stream. fn new(stream: NewSubpoolTransactionStream, tx_resp_builder: TxCompat) -> Self { @@ -650,7 +792,7 @@ where } /// Returns all new pending transactions received since the last poll. - async fn drain(&self) -> FilterChanges { + async fn drain(&self) -> FilterChanges> { let mut pending_txs = Vec::new(); let mut prepared_stream = self.txs_stream.lock().await; @@ -669,20 +811,20 @@ where } } -/// Helper trait for [FullTransactionsReceiver] to erase the `Transaction` type. +/// Helper trait for [`FullTransactionsReceiver`] to erase the `Transaction` type. #[async_trait] trait FullTransactionsFilter: fmt::Debug + Send + Sync + Unpin + 'static { async fn drain(&self) -> FilterChanges; } #[async_trait] -impl FullTransactionsFilter +impl FullTransactionsFilter> for FullTransactionsReceiver where T: PoolTransaction + 'static, - TxCompat: TransactionCompat + 'static, + TxCompat: RpcConvert> + 'static, { - async fn drain(&self) -> FilterChanges { + async fn drain(&self) -> FilterChanges> { Self::drain(self).await } } @@ -797,11 +939,355 @@ impl From for EthFilterError { } } +/// Helper type for the common pattern of returning receipts, block and the original header that is +/// a match for the filter. +struct ReceiptBlockResult

+where + P: ReceiptProvider + BlockReader, +{ + /// We always need the entire receipts for the matching block. + receipts: Arc>>, + /// Block can be optional and we can fetch it lazily when needed. + recovered_block: Option>>>, + /// The header of the block. + header: SealedHeader<

::Header>, +} + +/// Represents different modes for processing block ranges when filtering logs +enum RangeMode< + Eth: RpcNodeCoreExt + + EthApiTypes + + LoadReceipt + + EthBlocks + + 'static, +> { + /// Use cache-based processing for recent blocks + Cached(CachedMode), + /// Use range-based processing for older blocks + Range(RangeBlockMode), +} + +impl< + Eth: RpcNodeCoreExt + + EthApiTypes + + LoadReceipt + + EthBlocks + + 'static, + > RangeMode +{ + /// Creates a new `RangeMode`. + fn new( + filter_inner: Arc>, + sealed_headers: Vec::Header>>, + from_block: u64, + to_block: u64, + max_headers_range: u64, + chain_tip: u64, + ) -> Self { + let block_count = to_block - from_block + 1; + let distance_from_tip = chain_tip.saturating_sub(to_block); + + // Determine if we should use cached mode based on range characteristics + let use_cached_mode = + Self::should_use_cached_mode(&sealed_headers, block_count, distance_from_tip); + + if use_cached_mode && !sealed_headers.is_empty() { + Self::Cached(CachedMode { filter_inner, headers_iter: sealed_headers.into_iter() }) + } else { + Self::Range(RangeBlockMode { + filter_inner, + iter: sealed_headers.into_iter().peekable(), + next: VecDeque::new(), + max_range: max_headers_range as usize, + pending_tasks: FuturesOrdered::new(), + }) + } + } + + /// Determines whether to use cached mode based on bloom filter matches and range size + const fn should_use_cached_mode( + headers: &[SealedHeader<::Header>], + block_count: u64, + distance_from_tip: u64, + ) -> bool { + // Headers are already filtered by bloom, so count equals length + let bloom_matches = headers.len(); + + // Calculate adjusted threshold based on bloom matches + let adjusted_threshold = Self::calculate_adjusted_threshold(block_count, bloom_matches); + + block_count <= adjusted_threshold && distance_from_tip <= adjusted_threshold + } + + /// Calculates the adjusted cache threshold based on bloom filter matches + const fn calculate_adjusted_threshold(block_count: u64, bloom_matches: usize) -> u64 { + // Only apply adjustments for larger ranges + if block_count <= BLOOM_ADJUSTMENT_MIN_BLOCKS { + return CACHED_MODE_BLOCK_THRESHOLD; + } + + match bloom_matches { + n if n > HIGH_BLOOM_MATCH_THRESHOLD => CACHED_MODE_BLOCK_THRESHOLD / 2, + n if n > MODERATE_BLOOM_MATCH_THRESHOLD => (CACHED_MODE_BLOCK_THRESHOLD * 3) / 4, + _ => CACHED_MODE_BLOCK_THRESHOLD, + } + } + + /// Gets the next (receipts, `maybe_block`, header, `block_hash`) tuple. + async fn next(&mut self) -> Result>, EthFilterError> { + match self { + Self::Cached(cached) => cached.next().await, + Self::Range(range) => range.next().await, + } + } +} + +/// Mode for processing blocks using cache optimization for recent blocks +struct CachedMode< + Eth: RpcNodeCoreExt + + EthApiTypes + + LoadReceipt + + EthBlocks + + 'static, +> { + filter_inner: Arc>, + headers_iter: std::vec::IntoIter::Header>>, +} + +impl< + Eth: RpcNodeCoreExt + + EthApiTypes + + LoadReceipt + + EthBlocks + + 'static, + > CachedMode +{ + async fn next(&mut self) -> Result>, EthFilterError> { + for header in self.headers_iter.by_ref() { + // Use get_receipts_and_maybe_block which has automatic fallback to provider + if let Some((receipts, maybe_block)) = + self.filter_inner.eth_cache().get_receipts_and_maybe_block(header.hash()).await? + { + return Ok(Some(ReceiptBlockResult { + receipts, + recovered_block: maybe_block, + header, + })); + } + } + + Ok(None) // No more headers + } +} + +/// Type alias for parallel receipt fetching task futures used in `RangeBlockMode` +type ReceiptFetchFuture

= + Pin>, EthFilterError>> + Send>>; + +/// Mode for processing blocks using range queries for older blocks +struct RangeBlockMode< + Eth: RpcNodeCoreExt + + EthApiTypes + + LoadReceipt + + EthBlocks + + 'static, +> { + filter_inner: Arc>, + iter: Peekable::Header>>>, + next: VecDeque>, + max_range: usize, + // Stream of ongoing receipt fetching tasks + pending_tasks: FuturesOrdered>, +} + +impl< + Eth: RpcNodeCoreExt + + EthApiTypes + + LoadReceipt + + EthBlocks + + 'static, + > RangeBlockMode +{ + async fn next(&mut self) -> Result>, EthFilterError> { + loop { + // First, try to return any already processed result from buffer + if let Some(result) = self.next.pop_front() { + return Ok(Some(result)); + } + + // Try to get a completed task result if there are pending tasks + if let Some(task_result) = self.pending_tasks.next().await { + self.next.extend(task_result?); + continue; + } + + // No pending tasks - try to generate more work + let Some(next_header) = self.iter.next() else { + // No more headers to process + return Ok(None); + }; + + let mut range_headers = Vec::with_capacity(self.max_range); + range_headers.push(next_header); + + // Collect consecutive blocks up to max_range size + while range_headers.len() < self.max_range { + let Some(peeked) = self.iter.peek() else { break }; + let Some(last_header) = range_headers.last() else { break }; + + let expected_next = last_header.number() + 1; + if peeked.number() != expected_next { + debug!( + target: "rpc::eth::filter", + last_block = last_header.number(), + next_block = peeked.number(), + expected = expected_next, + range_size = range_headers.len(), + "Non-consecutive block detected, stopping range collection" + ); + break; // Non-consecutive block, stop here + } + + let Some(next_header) = self.iter.next() else { break }; + range_headers.push(next_header); + } + + // Check if we should use parallel processing for large ranges + let remaining_headers = self.iter.len() + range_headers.len(); + if remaining_headers >= PARALLEL_PROCESSING_THRESHOLD { + self.spawn_parallel_tasks(range_headers); + // Continue loop to await the spawned tasks + } else { + // Process small range sequentially and add results to buffer + if let Some(result) = self.process_small_range(range_headers).await? { + return Ok(Some(result)); + } + // Continue loop to check for more work + } + } + } + + /// Process a small range of headers sequentially + /// + /// This is used when the remaining headers count is below [`PARALLEL_PROCESSING_THRESHOLD`]. + async fn process_small_range( + &mut self, + range_headers: Vec::Header>>, + ) -> Result>, EthFilterError> { + // Process each header individually to avoid queuing for all receipts + for header in range_headers { + // First check if already cached to avoid unnecessary provider calls + let (maybe_block, maybe_receipts) = self + .filter_inner + .eth_cache() + .maybe_cached_block_and_receipts(header.hash()) + .await?; + + let receipts = match maybe_receipts { + Some(receipts) => receipts, + None => { + // Not cached - fetch directly from provider + match self.filter_inner.provider().receipts_by_block(header.hash().into())? { + Some(receipts) => Arc::new(receipts), + None => continue, // No receipts found + } + } + }; + + if !receipts.is_empty() { + self.next.push_back(ReceiptBlockResult { + receipts, + recovered_block: maybe_block, + header, + }); + } + } + + Ok(self.next.pop_front()) + } + + /// Spawn parallel tasks for processing a large range of headers + /// + /// This is used when the remaining headers count is at or above + /// [`PARALLEL_PROCESSING_THRESHOLD`]. + fn spawn_parallel_tasks( + &mut self, + range_headers: Vec::Header>>, + ) { + // Split headers into chunks + let chunk_size = std::cmp::max(range_headers.len() / DEFAULT_PARALLEL_CONCURRENCY, 1); + let header_chunks = range_headers + .into_iter() + .chunks(chunk_size) + .into_iter() + .map(|chunk| chunk.collect::>()) + .collect::>(); + + // Spawn each chunk as a separate task directly into the FuturesOrdered stream + for chunk_headers in header_chunks { + let filter_inner = self.filter_inner.clone(); + let chunk_task = Box::pin(async move { + let chunk_task = tokio::task::spawn_blocking(move || { + let mut chunk_results = Vec::new(); + + for header in chunk_headers { + // Fetch directly from provider - RangeMode is used for older blocks + // unlikely to be cached + let receipts = match filter_inner + .provider() + .receipts_by_block(header.hash().into())? + { + Some(receipts) => Arc::new(receipts), + None => continue, // No receipts found + }; + + if !receipts.is_empty() { + chunk_results.push(ReceiptBlockResult { + receipts, + recovered_block: None, + header, + }); + } + } + + Ok(chunk_results) + }); + + // Await the blocking task and handle the result + match chunk_task.await { + Ok(Ok(chunk_results)) => Ok(chunk_results), + Ok(Err(e)) => Err(e), + Err(join_err) => { + trace!(target: "rpc::eth::filter", error = ?join_err, "Task join error"); + Err(EthFilterError::InternalError) + } + } + }); + + self.pending_tasks.push_back(chunk_task); + } + } +} + #[cfg(test)] mod tests { use super::*; + use crate::{eth::EthApi, EthApiBuilder}; + use alloy_network::Ethereum; + use alloy_primitives::FixedBytes; use rand::Rng; + use reth_chainspec::{ChainSpec, ChainSpecProvider}; + use reth_ethereum_primitives::TxType; + use reth_evm_ethereum::EthEvmConfig; + use reth_network_api::noop::NoopNetwork; + use reth_provider::test_utils::MockEthProvider; + use reth_rpc_convert::RpcConverter; + use reth_rpc_eth_api::node::RpcNodeCoreAdapter; + use reth_rpc_eth_types::receipt::EthReceiptConverter; + use reth_tasks::TokioTaskExecutor; use reth_testing_utils::generators; + use reth_transaction_pool::test_utils::{testing_pool, TestPool}; + use std::{collections::VecDeque, sync::Arc}; #[test] fn test_block_range_iter() { @@ -824,4 +1310,567 @@ mod tests { assert_eq!(end, *range.end()); } + + // Helper function to create a test EthApi instance + #[expect(clippy::type_complexity)] + fn build_test_eth_api( + provider: MockEthProvider, + ) -> EthApi< + RpcNodeCoreAdapter, + RpcConverter>, + > { + EthApiBuilder::new( + provider.clone(), + testing_pool(), + NoopNetwork::default(), + EthEvmConfig::new(provider.chain_spec()), + ) + .build() + } + + #[tokio::test] + async fn test_range_block_mode_empty_range() { + let provider = MockEthProvider::default(); + let eth_api = build_test_eth_api(provider); + + let eth_filter = super::EthFilter::new( + eth_api, + EthFilterConfig::default(), + Box::new(TokioTaskExecutor::default()), + ); + let filter_inner = eth_filter.inner; + + let headers = vec![]; + let max_range = 100; + + let mut range_mode = RangeBlockMode { + filter_inner, + iter: headers.into_iter().peekable(), + next: VecDeque::new(), + max_range, + pending_tasks: FuturesOrdered::new(), + }; + + let result = range_mode.next().await; + assert!(result.is_ok()); + assert!(result.unwrap().is_none()); + } + + #[tokio::test] + async fn test_range_block_mode_queued_results_priority() { + let provider = MockEthProvider::default(); + let eth_api = build_test_eth_api(provider); + + let eth_filter = super::EthFilter::new( + eth_api, + EthFilterConfig::default(), + Box::new(TokioTaskExecutor::default()), + ); + let filter_inner = eth_filter.inner; + + let headers = vec![ + SealedHeader::new( + alloy_consensus::Header { number: 100, ..Default::default() }, + FixedBytes::random(), + ), + SealedHeader::new( + alloy_consensus::Header { number: 101, ..Default::default() }, + FixedBytes::random(), + ), + ]; + + // create specific mock results to test ordering + let expected_block_hash_1 = FixedBytes::from([1u8; 32]); + let expected_block_hash_2 = FixedBytes::from([2u8; 32]); + + // create mock receipts to test receipt handling + let mock_receipt_1 = reth_ethereum_primitives::Receipt { + tx_type: TxType::Legacy, + cumulative_gas_used: 100_000, + logs: vec![], + success: true, + }; + let mock_receipt_2 = reth_ethereum_primitives::Receipt { + tx_type: TxType::Eip1559, + cumulative_gas_used: 200_000, + logs: vec![], + success: true, + }; + let mock_receipt_3 = reth_ethereum_primitives::Receipt { + tx_type: TxType::Eip2930, + cumulative_gas_used: 150_000, + logs: vec![], + success: false, // Different success status + }; + + let mock_result_1 = ReceiptBlockResult { + receipts: Arc::new(vec![mock_receipt_1.clone(), mock_receipt_2.clone()]), + recovered_block: None, + header: SealedHeader::new( + alloy_consensus::Header { number: 42, ..Default::default() }, + expected_block_hash_1, + ), + }; + + let mock_result_2 = ReceiptBlockResult { + receipts: Arc::new(vec![mock_receipt_3.clone()]), + recovered_block: None, + header: SealedHeader::new( + alloy_consensus::Header { number: 43, ..Default::default() }, + expected_block_hash_2, + ), + }; + + let mut range_mode = RangeBlockMode { + filter_inner, + iter: headers.into_iter().peekable(), + next: VecDeque::from([mock_result_1, mock_result_2]), // Queue two results + max_range: 100, + pending_tasks: FuturesOrdered::new(), + }; + + // first call should return the first queued result (FIFO order) + let result1 = range_mode.next().await; + assert!(result1.is_ok()); + let receipt_result1 = result1.unwrap().unwrap(); + assert_eq!(receipt_result1.header.hash(), expected_block_hash_1); + assert_eq!(receipt_result1.header.number, 42); + + // verify receipts + assert_eq!(receipt_result1.receipts.len(), 2); + assert_eq!(receipt_result1.receipts[0].tx_type, mock_receipt_1.tx_type); + assert_eq!( + receipt_result1.receipts[0].cumulative_gas_used, + mock_receipt_1.cumulative_gas_used + ); + assert_eq!(receipt_result1.receipts[0].success, mock_receipt_1.success); + assert_eq!(receipt_result1.receipts[1].tx_type, mock_receipt_2.tx_type); + assert_eq!( + receipt_result1.receipts[1].cumulative_gas_used, + mock_receipt_2.cumulative_gas_used + ); + assert_eq!(receipt_result1.receipts[1].success, mock_receipt_2.success); + + // second call should return the second queued result + let result2 = range_mode.next().await; + assert!(result2.is_ok()); + let receipt_result2 = result2.unwrap().unwrap(); + assert_eq!(receipt_result2.header.hash(), expected_block_hash_2); + assert_eq!(receipt_result2.header.number, 43); + + // verify receipts + assert_eq!(receipt_result2.receipts.len(), 1); + assert_eq!(receipt_result2.receipts[0].tx_type, mock_receipt_3.tx_type); + assert_eq!( + receipt_result2.receipts[0].cumulative_gas_used, + mock_receipt_3.cumulative_gas_used + ); + assert_eq!(receipt_result2.receipts[0].success, mock_receipt_3.success); + + // queue should now be empty + assert!(range_mode.next.is_empty()); + + let result3 = range_mode.next().await; + assert!(result3.is_ok()); + } + + #[tokio::test] + async fn test_range_block_mode_single_block_no_receipts() { + let provider = MockEthProvider::default(); + let eth_api = build_test_eth_api(provider); + + let eth_filter = super::EthFilter::new( + eth_api, + EthFilterConfig::default(), + Box::new(TokioTaskExecutor::default()), + ); + let filter_inner = eth_filter.inner; + + let headers = vec![SealedHeader::new( + alloy_consensus::Header { number: 100, ..Default::default() }, + FixedBytes::random(), + )]; + + let mut range_mode = RangeBlockMode { + filter_inner, + iter: headers.into_iter().peekable(), + next: VecDeque::new(), + max_range: 100, + pending_tasks: FuturesOrdered::new(), + }; + + let result = range_mode.next().await; + assert!(result.is_ok()); + } + + #[tokio::test] + async fn test_range_block_mode_provider_receipts() { + let provider = MockEthProvider::default(); + + let header_1 = alloy_consensus::Header { number: 100, ..Default::default() }; + let header_2 = alloy_consensus::Header { number: 101, ..Default::default() }; + let header_3 = alloy_consensus::Header { number: 102, ..Default::default() }; + + let block_hash_1 = FixedBytes::random(); + let block_hash_2 = FixedBytes::random(); + let block_hash_3 = FixedBytes::random(); + + provider.add_header(block_hash_1, header_1.clone()); + provider.add_header(block_hash_2, header_2.clone()); + provider.add_header(block_hash_3, header_3.clone()); + + // create mock receipts to test provider fetching with mock logs + let mock_log = alloy_primitives::Log { + address: alloy_primitives::Address::ZERO, + data: alloy_primitives::LogData::new_unchecked(vec![], alloy_primitives::Bytes::new()), + }; + + let receipt_100_1 = reth_ethereum_primitives::Receipt { + tx_type: TxType::Legacy, + cumulative_gas_used: 21_000, + logs: vec![mock_log.clone()], + success: true, + }; + let receipt_100_2 = reth_ethereum_primitives::Receipt { + tx_type: TxType::Eip1559, + cumulative_gas_used: 42_000, + logs: vec![mock_log.clone()], + success: true, + }; + let receipt_101_1 = reth_ethereum_primitives::Receipt { + tx_type: TxType::Eip2930, + cumulative_gas_used: 30_000, + logs: vec![mock_log.clone()], + success: false, + }; + + provider.add_receipts(100, vec![receipt_100_1.clone(), receipt_100_2.clone()]); + provider.add_receipts(101, vec![receipt_101_1.clone()]); + + let eth_api = build_test_eth_api(provider); + + let eth_filter = super::EthFilter::new( + eth_api, + EthFilterConfig::default(), + Box::new(TokioTaskExecutor::default()), + ); + let filter_inner = eth_filter.inner; + + let headers = vec![ + SealedHeader::new(header_1, block_hash_1), + SealedHeader::new(header_2, block_hash_2), + SealedHeader::new(header_3, block_hash_3), + ]; + + let mut range_mode = RangeBlockMode { + filter_inner, + iter: headers.into_iter().peekable(), + next: VecDeque::new(), + max_range: 3, // include the 3 blocks in the first queried results + pending_tasks: FuturesOrdered::new(), + }; + + // first call should fetch receipts from provider and return first block with receipts + let result = range_mode.next().await; + assert!(result.is_ok()); + let receipt_result = result.unwrap().unwrap(); + + assert_eq!(receipt_result.header.hash(), block_hash_1); + assert_eq!(receipt_result.header.number, 100); + assert_eq!(receipt_result.receipts.len(), 2); + + // verify receipts + assert_eq!(receipt_result.receipts[0].tx_type, receipt_100_1.tx_type); + assert_eq!( + receipt_result.receipts[0].cumulative_gas_used, + receipt_100_1.cumulative_gas_used + ); + assert_eq!(receipt_result.receipts[0].success, receipt_100_1.success); + + assert_eq!(receipt_result.receipts[1].tx_type, receipt_100_2.tx_type); + assert_eq!( + receipt_result.receipts[1].cumulative_gas_used, + receipt_100_2.cumulative_gas_used + ); + assert_eq!(receipt_result.receipts[1].success, receipt_100_2.success); + + // second call should return the second block with receipts + let result2 = range_mode.next().await; + assert!(result2.is_ok()); + let receipt_result2 = result2.unwrap().unwrap(); + + assert_eq!(receipt_result2.header.hash(), block_hash_2); + assert_eq!(receipt_result2.header.number, 101); + assert_eq!(receipt_result2.receipts.len(), 1); + + // verify receipts + assert_eq!(receipt_result2.receipts[0].tx_type, receipt_101_1.tx_type); + assert_eq!( + receipt_result2.receipts[0].cumulative_gas_used, + receipt_101_1.cumulative_gas_used + ); + assert_eq!(receipt_result2.receipts[0].success, receipt_101_1.success); + + // third call should return None since no more blocks with receipts + let result3 = range_mode.next().await; + assert!(result3.is_ok()); + assert!(result3.unwrap().is_none()); + } + + #[tokio::test] + async fn test_range_block_mode_iterator_exhaustion() { + let provider = MockEthProvider::default(); + + let header_100 = alloy_consensus::Header { number: 100, ..Default::default() }; + let header_101 = alloy_consensus::Header { number: 101, ..Default::default() }; + + let block_hash_100 = FixedBytes::random(); + let block_hash_101 = FixedBytes::random(); + + // Associate headers with hashes first + provider.add_header(block_hash_100, header_100.clone()); + provider.add_header(block_hash_101, header_101.clone()); + + // Add mock receipts so headers are actually processed + let mock_receipt = reth_ethereum_primitives::Receipt { + tx_type: TxType::Legacy, + cumulative_gas_used: 21_000, + logs: vec![], + success: true, + }; + provider.add_receipts(100, vec![mock_receipt.clone()]); + provider.add_receipts(101, vec![mock_receipt.clone()]); + + let eth_api = build_test_eth_api(provider); + + let eth_filter = super::EthFilter::new( + eth_api, + EthFilterConfig::default(), + Box::new(TokioTaskExecutor::default()), + ); + let filter_inner = eth_filter.inner; + + let headers = vec![ + SealedHeader::new(header_100, block_hash_100), + SealedHeader::new(header_101, block_hash_101), + ]; + + let mut range_mode = RangeBlockMode { + filter_inner, + iter: headers.into_iter().peekable(), + next: VecDeque::new(), + max_range: 1, + pending_tasks: FuturesOrdered::new(), + }; + + let result1 = range_mode.next().await; + assert!(result1.is_ok()); + assert!(result1.unwrap().is_some()); // Should have processed block 100 + + assert!(range_mode.iter.peek().is_some()); // Should still have block 101 + + let result2 = range_mode.next().await; + assert!(result2.is_ok()); + assert!(result2.unwrap().is_some()); // Should have processed block 101 + + // now iterator should be exhausted + assert!(range_mode.iter.peek().is_none()); + + // further calls should return None + let result3 = range_mode.next().await; + assert!(result3.is_ok()); + assert!(result3.unwrap().is_none()); + } + + #[tokio::test] + async fn test_cached_mode_with_mock_receipts() { + // create test data + let test_hash = FixedBytes::from([42u8; 32]); + let test_block_number = 100u64; + let test_header = SealedHeader::new( + alloy_consensus::Header { + number: test_block_number, + gas_used: 50_000, + ..Default::default() + }, + test_hash, + ); + + // add a mock receipt to the provider with a mock log + let mock_log = alloy_primitives::Log { + address: alloy_primitives::Address::ZERO, + data: alloy_primitives::LogData::new_unchecked(vec![], alloy_primitives::Bytes::new()), + }; + + let mock_receipt = reth_ethereum_primitives::Receipt { + tx_type: TxType::Legacy, + cumulative_gas_used: 21_000, + logs: vec![mock_log], + success: true, + }; + + let provider = MockEthProvider::default(); + provider.add_header(test_hash, test_header.header().clone()); + provider.add_receipts(test_block_number, vec![mock_receipt.clone()]); + + let eth_api = build_test_eth_api(provider); + let eth_filter = super::EthFilter::new( + eth_api, + EthFilterConfig::default(), + Box::new(TokioTaskExecutor::default()), + ); + let filter_inner = eth_filter.inner; + + let headers = vec![test_header.clone()]; + + let mut cached_mode = CachedMode { filter_inner, headers_iter: headers.into_iter() }; + + // should find the receipt from provider fallback (cache will be empty) + let result = cached_mode.next().await.expect("next should succeed"); + let receipt_block_result = result.expect("should have receipt result"); + assert_eq!(receipt_block_result.header.hash(), test_hash); + assert_eq!(receipt_block_result.header.number, test_block_number); + assert_eq!(receipt_block_result.receipts.len(), 1); + assert_eq!(receipt_block_result.receipts[0].tx_type, mock_receipt.tx_type); + assert_eq!( + receipt_block_result.receipts[0].cumulative_gas_used, + mock_receipt.cumulative_gas_used + ); + assert_eq!(receipt_block_result.receipts[0].success, mock_receipt.success); + + // iterator should be exhausted + let result2 = cached_mode.next().await; + assert!(result2.is_ok()); + assert!(result2.unwrap().is_none()); + } + + #[tokio::test] + async fn test_cached_mode_empty_headers() { + let provider = MockEthProvider::default(); + let eth_api = build_test_eth_api(provider); + + let eth_filter = super::EthFilter::new( + eth_api, + EthFilterConfig::default(), + Box::new(TokioTaskExecutor::default()), + ); + let filter_inner = eth_filter.inner; + + let headers: Vec> = vec![]; + + let mut cached_mode = CachedMode { filter_inner, headers_iter: headers.into_iter() }; + + // should immediately return None for empty headers + let result = cached_mode.next().await.expect("next should succeed"); + assert!(result.is_none()); + } + + #[tokio::test] + async fn test_non_consecutive_headers_after_bloom_filter() { + let provider = MockEthProvider::default(); + + // Create 4 headers where only blocks 100 and 102 will match bloom filter + let mut expected_hashes = vec![]; + let mut prev_hash = alloy_primitives::B256::default(); + + // Create a transaction for blocks that will have receipts + use alloy_consensus::TxLegacy; + use reth_ethereum_primitives::{TransactionSigned, TxType}; + + let tx_inner = TxLegacy { + chain_id: Some(1), + nonce: 0, + gas_price: 21_000, + gas_limit: 21_000, + to: alloy_primitives::TxKind::Call(alloy_primitives::Address::ZERO), + value: alloy_primitives::U256::ZERO, + input: alloy_primitives::Bytes::new(), + }; + let signature = alloy_primitives::Signature::test_signature(); + let tx = TransactionSigned::new_unhashed(tx_inner.into(), signature); + + for i in 100u64..=103 { + let header = alloy_consensus::Header { + number: i, + parent_hash: prev_hash, + // Set bloom to match filter only for blocks 100 and 102 + logs_bloom: if i == 100 || i == 102 { + alloy_primitives::Bloom::from([1u8; 256]) + } else { + alloy_primitives::Bloom::default() + }, + ..Default::default() + }; + + let hash = header.hash_slow(); + expected_hashes.push(hash); + prev_hash = hash; + + // Add transaction to blocks that will have receipts (100 and 102) + let transactions = if i == 100 || i == 102 { vec![tx.clone()] } else { vec![] }; + + let block = reth_ethereum_primitives::Block { + header, + body: reth_ethereum_primitives::BlockBody { transactions, ..Default::default() }, + }; + provider.add_block(hash, block); + } + + // Add receipts with logs only to blocks that match bloom + let mock_log = alloy_primitives::Log { + address: alloy_primitives::Address::ZERO, + data: alloy_primitives::LogData::new_unchecked(vec![], alloy_primitives::Bytes::new()), + }; + + let receipt = reth_ethereum_primitives::Receipt { + tx_type: TxType::Legacy, + cumulative_gas_used: 21_000, + logs: vec![mock_log], + success: true, + }; + + provider.add_receipts(100, vec![receipt.clone()]); + provider.add_receipts(101, vec![]); + provider.add_receipts(102, vec![receipt.clone()]); + provider.add_receipts(103, vec![]); + + // Add block body indices for each block so receipts can be fetched + use reth_db_api::models::StoredBlockBodyIndices; + provider + .add_block_body_indices(100, StoredBlockBodyIndices { first_tx_num: 0, tx_count: 1 }); + provider + .add_block_body_indices(101, StoredBlockBodyIndices { first_tx_num: 1, tx_count: 0 }); + provider + .add_block_body_indices(102, StoredBlockBodyIndices { first_tx_num: 1, tx_count: 1 }); + provider + .add_block_body_indices(103, StoredBlockBodyIndices { first_tx_num: 2, tx_count: 0 }); + + let eth_api = build_test_eth_api(provider); + let eth_filter = EthFilter::new( + eth_api, + EthFilterConfig::default(), + Box::new(TokioTaskExecutor::default()), + ); + + // Use default filter which will match any non-empty bloom + let filter = Filter::default(); + + // Get logs in the range - this will trigger the bloom filtering + let logs = eth_filter + .inner + .clone() + .get_logs_in_block_range(filter, 100, 103, QueryLimits::default()) + .await + .expect("should succeed"); + + // We should get logs from blocks 100 and 102 only (bloom filtered) + assert_eq!(logs.len(), 2); + + assert_eq!(logs[0].block_number, Some(100)); + assert_eq!(logs[1].block_number, Some(102)); + + // Each block hash should be the hash of its own header, not derived from any other header + assert_eq!(logs[0].block_hash, Some(expected_hashes[0])); // block 100 + assert_eq!(logs[1].block_hash, Some(expected_hashes[2])); // block 102 + } } diff --git a/crates/rpc/rpc/src/eth/helpers/block.rs b/crates/rpc/rpc/src/eth/helpers/block.rs index 6304b73dcc1..8077802804b 100644 --- a/crates/rpc/rpc/src/eth/helpers/block.rs +++ b/crates/rpc/rpc/src/eth/helpers/block.rs @@ -1,83 +1,26 @@ //! Contains RPC handler implementations specific to blocks. -use alloy_consensus::{transaction::TransactionMeta, BlockHeader}; -use alloy_rpc_types_eth::{BlockId, TransactionReceipt}; -use reth_chainspec::{ChainSpecProvider, EthChainSpec}; -use reth_primitives_traits::BlockBody; +use reth_rpc_convert::RpcConvert; use reth_rpc_eth_api::{ - helpers::{EthBlocks, LoadBlock, LoadPendingBlock, LoadReceipt, SpawnBlocking}, - types::RpcTypes, - RpcNodeCoreExt, RpcReceipt, + helpers::{EthBlocks, LoadBlock, LoadPendingBlock}, + FromEvmError, RpcNodeCore, }; -use reth_rpc_eth_types::{EthApiError, EthReceiptBuilder}; -use reth_storage_api::{BlockReader, ProviderTx}; -use reth_transaction_pool::{PoolTransaction, TransactionPool}; +use reth_rpc_eth_types::EthApiError; use crate::EthApi; -impl EthBlocks for EthApi +impl EthBlocks for EthApi where - Self: LoadBlock< - Error = EthApiError, - NetworkTypes: RpcTypes, - Provider: BlockReader< - Transaction = reth_ethereum_primitives::TransactionSigned, - Receipt = reth_ethereum_primitives::Receipt, - >, - >, - Provider: BlockReader + ChainSpecProvider, + N: RpcNodeCore, + EthApiError: FromEvmError, + Rpc: RpcConvert, { - async fn block_receipts( - &self, - block_id: BlockId, - ) -> Result>>, Self::Error> - where - Self: LoadReceipt, - { - if let Some((block, receipts)) = self.load_block_and_receipts(block_id).await? { - let block_number = block.number(); - let base_fee = block.base_fee_per_gas(); - let block_hash = block.hash(); - let excess_blob_gas = block.excess_blob_gas(); - let timestamp = block.timestamp(); - let blob_params = self.provider().chain_spec().blob_params_at_timestamp(timestamp); - - return block - .body() - .transactions() - .iter() - .zip(receipts.iter()) - .enumerate() - .map(|(idx, (tx, receipt))| { - let meta = TransactionMeta { - tx_hash: *tx.tx_hash(), - index: idx as u64, - block_hash, - block_number, - base_fee, - excess_blob_gas, - timestamp, - }; - EthReceiptBuilder::new(tx, meta, receipt, &receipts, blob_params) - .map(|builder| builder.build()) - }) - .collect::, Self::Error>>() - .map(Some) - } - - Ok(None) - } } -impl LoadBlock for EthApi +impl LoadBlock for EthApi where - Self: LoadPendingBlock - + SpawnBlocking - + RpcNodeCoreExt< - Pool: TransactionPool< - Transaction: PoolTransaction>, - >, - >, - Provider: BlockReader, + Self: LoadPendingBlock, + N: RpcNodeCore, + Rpc: RpcConvert, { } diff --git a/crates/rpc/rpc/src/eth/helpers/call.rs b/crates/rpc/rpc/src/eth/helpers/call.rs index 33d303d8f09..ad9f020bd0c 100644 --- a/crates/rpc/rpc/src/eth/helpers/call.rs +++ b/crates/rpc/rpc/src/eth/helpers/call.rs @@ -1,41 +1,26 @@ //! Contains RPC handler implementations specific to endpoints that call/execute within evm. use crate::EthApi; -use alloy_consensus::TxType; -use alloy_evm::block::BlockExecutorFactory; -use alloy_primitives::{TxKind, U256}; -use alloy_rpc_types::TransactionRequest; -use alloy_signer::Either; -use reth_evm::{ConfigureEvm, EvmEnv, EvmFactory, SpecFor}; -use reth_node_api::NodePrimitives; +use reth_rpc_convert::RpcConvert; use reth_rpc_eth_api::{ - helpers::{estimate::EstimateCall, Call, EthCall, LoadPendingBlock, LoadState, SpawnBlocking}, - FromEthApiError, FromEvmError, FullEthApiTypes, IntoEthApiError, + helpers::{estimate::EstimateCall, Call, EthCall}, + FromEvmError, RpcNodeCore, }; -use reth_rpc_eth_types::{revm_utils::CallFees, EthApiError, RpcInvalidTransactionError}; -use reth_storage_api::{BlockReader, ProviderHeader, ProviderTx}; -use revm::{context::TxEnv, context_interface::Block, Database}; +use reth_rpc_eth_types::EthApiError; -impl EthCall for EthApi +impl EthCall for EthApi where - Self: EstimateCall + LoadPendingBlock + FullEthApiTypes, - Provider: BlockReader, + N: RpcNodeCore, + EthApiError: FromEvmError, + Rpc: RpcConvert, { } -impl Call for EthApi +impl Call for EthApi where - Self: LoadState< - Evm: ConfigureEvm< - BlockExecutorFactory: BlockExecutorFactory>, - Primitives: NodePrimitives< - BlockHeader = ProviderHeader, - SignedTx = ProviderTx, - >, - >, - Error: FromEvmError, - > + SpawnBlocking, - Provider: BlockReader, + N: RpcNodeCore, + EthApiError: FromEvmError, + Rpc: RpcConvert, { #[inline] fn call_gas_limit(&self) -> u64 { @@ -47,113 +32,16 @@ where self.inner.max_simulate_blocks() } - fn create_txn_env( - &self, - evm_env: &EvmEnv>, - request: TransactionRequest, - mut db: impl Database>, - ) -> Result { - // Ensure that if versioned hashes are set, they're not empty - if request.blob_versioned_hashes.as_ref().is_some_and(|hashes| hashes.is_empty()) { - return Err(RpcInvalidTransactionError::BlobTransactionMissingBlobHashes.into_eth_err()); - } - - let tx_type = if request.authorization_list.is_some() { - TxType::Eip7702 - } else if request.sidecar.is_some() || request.max_fee_per_blob_gas.is_some() { - TxType::Eip4844 - } else if request.max_fee_per_gas.is_some() || request.max_priority_fee_per_gas.is_some() { - TxType::Eip1559 - } else if request.access_list.is_some() { - TxType::Eip2930 - } else { - TxType::Legacy - } as u8; - - let TransactionRequest { - from, - to, - gas_price, - max_fee_per_gas, - max_priority_fee_per_gas, - gas, - value, - input, - nonce, - access_list, - chain_id, - blob_versioned_hashes, - max_fee_per_blob_gas, - authorization_list, - transaction_type: _, - sidecar: _, - } = request; - - let CallFees { max_priority_fee_per_gas, gas_price, max_fee_per_blob_gas } = - CallFees::ensure_fees( - gas_price.map(U256::from), - max_fee_per_gas.map(U256::from), - max_priority_fee_per_gas.map(U256::from), - U256::from(evm_env.block_env.basefee), - blob_versioned_hashes.as_deref(), - max_fee_per_blob_gas.map(U256::from), - evm_env.block_env.blob_gasprice().map(U256::from), - )?; - - let gas_limit = gas.unwrap_or( - // Use maximum allowed gas limit. The reason for this - // is that both Erigon and Geth use pre-configured gas cap even if - // it's possible to derive the gas limit from the block: - // - evm_env.block_env.gas_limit, - ); - - let chain_id = chain_id.unwrap_or(evm_env.cfg_env.chain_id); - - let caller = from.unwrap_or_default(); - - let nonce = if let Some(nonce) = nonce { - nonce - } else { - db.basic(caller).map_err(Into::into)?.map(|acc| acc.nonce).unwrap_or_default() - }; - - let env = TxEnv { - tx_type, - gas_limit, - nonce, - caller, - gas_price: gas_price.saturating_to(), - gas_priority_fee: max_priority_fee_per_gas.map(|v| v.saturating_to()), - kind: to.unwrap_or(TxKind::Create), - value: value.unwrap_or_default(), - data: input - .try_into_unique_input() - .map_err(Self::Error::from_eth_err)? - .unwrap_or_default(), - chain_id: Some(chain_id), - access_list: access_list.unwrap_or_default(), - // EIP-4844 fields - blob_hashes: blob_versioned_hashes.unwrap_or_default(), - max_fee_per_blob_gas: max_fee_per_blob_gas - .map(|v| v.saturating_to()) - .unwrap_or_default(), - // EIP-7702 fields - authorization_list: authorization_list - .unwrap_or_default() - .into_iter() - .map(Either::Left) - .collect(), - }; - - Ok(env) + #[inline] + fn evm_memory_limit(&self) -> u64 { + self.inner.evm_memory_limit() } } -impl EstimateCall for EthApi +impl EstimateCall for EthApi where - Self: Call, - Provider: BlockReader, + N: RpcNodeCore, + EthApiError: FromEvmError, + Rpc: RpcConvert, { } diff --git a/crates/rpc/rpc/src/eth/helpers/fees.rs b/crates/rpc/rpc/src/eth/helpers/fees.rs index 9ee8b9702be..1d26644b47b 100644 --- a/crates/rpc/rpc/src/eth/helpers/fees.rs +++ b/crates/rpc/rpc/src/eth/helpers/fees.rs @@ -1,25 +1,28 @@ //! Contains RPC handler implementations for fee history. -use reth_chainspec::{ChainSpecProvider, EthChainSpec, EthereumHardforks}; -use reth_rpc_eth_api::helpers::{EthFees, LoadBlock, LoadFee}; -use reth_rpc_eth_types::{FeeHistoryCache, GasPriceOracle}; -use reth_storage_api::{BlockReader, BlockReaderIdExt, StateProviderFactory}; +use reth_rpc_convert::RpcConvert; +use reth_rpc_eth_api::{ + helpers::{EthFees, LoadFee}, + FromEvmError, RpcNodeCore, +}; +use reth_rpc_eth_types::{EthApiError, FeeHistoryCache, GasPriceOracle}; +use reth_storage_api::ProviderHeader; use crate::EthApi; -impl EthFees for EthApi +impl EthFees for EthApi where - Self: LoadFee, - Provider: BlockReader, + N: RpcNodeCore, + EthApiError: FromEvmError, + Rpc: RpcConvert, { } -impl LoadFee for EthApi +impl LoadFee for EthApi where - Self: LoadBlock, - Provider: BlockReaderIdExt - + ChainSpecProvider - + StateProviderFactory, + N: RpcNodeCore, + EthApiError: FromEvmError, + Rpc: RpcConvert, { #[inline] fn gas_oracle(&self) -> &GasPriceOracle { @@ -27,7 +30,7 @@ where } #[inline] - fn fee_history_cache(&self) -> &FeeHistoryCache { + fn fee_history_cache(&self) -> &FeeHistoryCache> { self.inner.fee_history_cache() } } diff --git a/crates/rpc/rpc/src/eth/helpers/mod.rs b/crates/rpc/rpc/src/eth/helpers/mod.rs index 03e0443a15b..15fcf612d9a 100644 --- a/crates/rpc/rpc/src/eth/helpers/mod.rs +++ b/crates/rpc/rpc/src/eth/helpers/mod.rs @@ -2,6 +2,7 @@ //! files. pub mod signer; +pub mod sync_listener; pub mod types; mod block; @@ -13,3 +14,5 @@ mod spec; mod state; mod trace; mod transaction; + +pub use sync_listener::SyncListener; diff --git a/crates/rpc/rpc/src/eth/helpers/pending_block.rs b/crates/rpc/rpc/src/eth/helpers/pending_block.rs index ad30c5f3da8..0c08c12e0e9 100644 --- a/crates/rpc/rpc/src/eth/helpers/pending_block.rs +++ b/crates/rpc/rpc/src/eth/helpers/pending_block.rs @@ -1,77 +1,31 @@ //! Support for building a pending block with transactions from local view of mempool. -use alloy_consensus::BlockHeader; -use reth_chainspec::{ChainSpecProvider, EthChainSpec, EthereumHardforks}; -use reth_evm::{ConfigureEvm, NextBlockEnvAttributes}; -use reth_node_api::NodePrimitives; -use reth_primitives_traits::SealedHeader; +use crate::EthApi; +use reth_rpc_convert::RpcConvert; use reth_rpc_eth_api::{ - helpers::{LoadPendingBlock, SpawnBlocking}, - types::RpcTypes, + helpers::{pending_block::PendingEnvBuilder, LoadPendingBlock}, FromEvmError, RpcNodeCore, }; -use reth_rpc_eth_types::PendingBlock; -use reth_storage_api::{ - BlockReader, BlockReaderIdExt, ProviderBlock, ProviderHeader, ProviderReceipt, ProviderTx, - StateProviderFactory, -}; -use reth_transaction_pool::{PoolTransaction, TransactionPool}; -use revm_primitives::B256; - -use crate::EthApi; +use reth_rpc_eth_types::{builder::config::PendingBlockKind, EthApiError, PendingBlock}; -impl LoadPendingBlock - for EthApi +impl LoadPendingBlock for EthApi where - Self: SpawnBlocking< - NetworkTypes: RpcTypes

, - Error: FromEvmError, - > + RpcNodeCore< - Provider: BlockReaderIdExt< - Transaction = reth_ethereum_primitives::TransactionSigned, - Block = reth_ethereum_primitives::Block, - Receipt = reth_ethereum_primitives::Receipt, - Header = alloy_consensus::Header, - > + ChainSpecProvider - + StateProviderFactory, - Pool: TransactionPool< - Transaction: PoolTransaction>, - >, - Evm: ConfigureEvm< - Primitives: NodePrimitives< - BlockHeader = ProviderHeader, - SignedTx = ProviderTx, - Receipt = ProviderReceipt, - Block = ProviderBlock, - >, - NextBlockEnvCtx = NextBlockEnvAttributes, - >, - >, - Provider: BlockReader< - Block = reth_ethereum_primitives::Block, - Receipt = reth_ethereum_primitives::Receipt, - >, + N: RpcNodeCore, + EthApiError: FromEvmError, + Rpc: RpcConvert, { #[inline] - fn pending_block( - &self, - ) -> &tokio::sync::Mutex< - Option, ProviderReceipt>>, - > { + fn pending_block(&self) -> &tokio::sync::Mutex>> { self.inner.pending_block() } - fn next_env_attributes( - &self, - parent: &SealedHeader>, - ) -> Result<::NextBlockEnvCtx, Self::Error> { - Ok(NextBlockEnvAttributes { - timestamp: parent.timestamp().saturating_add(12), - suggested_fee_recipient: parent.beneficiary(), - prev_randao: B256::random(), - gas_limit: parent.gas_limit(), - parent_beacon_block_root: parent.parent_beacon_block_root(), - withdrawals: None, - }) + #[inline] + fn pending_env_builder(&self) -> &dyn PendingEnvBuilder { + self.inner.pending_env_builder() + } + + #[inline] + fn pending_block_kind(&self) -> PendingBlockKind { + self.inner.pending_block_kind() } } diff --git a/crates/rpc/rpc/src/eth/helpers/receipt.rs b/crates/rpc/rpc/src/eth/helpers/receipt.rs index 9d0a744ee80..358ef57f768 100644 --- a/crates/rpc/rpc/src/eth/helpers/receipt.rs +++ b/crates/rpc/rpc/src/eth/helpers/receipt.rs @@ -1,38 +1,14 @@ //! Builds an RPC receipt response w.r.t. data layout of network. -use alloy_consensus::transaction::TransactionMeta; -use reth_chainspec::{ChainSpecProvider, EthChainSpec}; -use reth_ethereum_primitives::{Receipt, TransactionSigned}; -use reth_rpc_eth_api::{helpers::LoadReceipt, FromEthApiError, RpcNodeCoreExt, RpcReceipt}; -use reth_rpc_eth_types::{EthApiError, EthReceiptBuilder}; -use reth_storage_api::{BlockReader, ReceiptProvider, TransactionsProvider}; - use crate::EthApi; +use reth_rpc_convert::RpcConvert; +use reth_rpc_eth_api::{helpers::LoadReceipt, FromEvmError, RpcNodeCore}; +use reth_rpc_eth_types::EthApiError; -impl LoadReceipt for EthApi +impl LoadReceipt for EthApi where - Self: RpcNodeCoreExt< - Provider: TransactionsProvider - + ReceiptProvider, - >, - Provider: BlockReader + ChainSpecProvider, + N: RpcNodeCore, + EthApiError: FromEvmError, + Rpc: RpcConvert, { - async fn build_transaction_receipt( - &self, - tx: TransactionSigned, - meta: TransactionMeta, - receipt: Receipt, - ) -> Result, Self::Error> { - let hash = meta.block_hash; - // get all receipts for the block - let all_receipts = self - .cache() - .get_receipts(hash) - .await - .map_err(Self::Error::from_eth_err)? - .ok_or(EthApiError::HeaderNotFound(hash.into()))?; - let blob_params = self.provider().chain_spec().blob_params_at_timestamp(meta.timestamp); - - Ok(EthReceiptBuilder::new(&tx, meta, &receipt, &all_receipts, blob_params)?.build()) - } } diff --git a/crates/rpc/rpc/src/eth/helpers/signer.rs b/crates/rpc/rpc/src/eth/helpers/signer.rs index 01a07c4436d..2c18245d542 100644 --- a/crates/rpc/rpc/src/eth/helpers/signer.rs +++ b/crates/rpc/rpc/src/eth/helpers/signer.rs @@ -1,28 +1,14 @@ //! An abstraction over ethereum signers. -use std::collections::HashMap; - -use crate::EthApi; use alloy_dyn_abi::TypedData; use alloy_eips::eip2718::Decodable2718; -use alloy_network::{eip2718::Encodable2718, EthereumWallet, TransactionBuilder}; use alloy_primitives::{eip191_hash_message, Address, Signature, B256}; -use alloy_rpc_types_eth::TransactionRequest; use alloy_signer::SignerSync; -use alloy_signer_local::PrivateKeySigner; -use reth_rpc_eth_api::helpers::{signer::Result, AddDevSigners, EthSigner}; +use alloy_signer_local::{coins_bip39::English, MnemonicBuilder, PrivateKeySigner}; +use reth_rpc_convert::SignableTxRequest; +use reth_rpc_eth_api::helpers::{signer::Result, EthSigner}; use reth_rpc_eth_types::SignError; -use reth_storage_api::BlockReader; - -impl AddDevSigners - for EthApi -where - Provider: BlockReader, -{ - fn with_dev_accounts(&self) { - *self.inner.signers().write() = DevSigner::random_signers(20) - } -} +use std::collections::HashMap; /// Holds developer keys #[derive(Debug, Clone)] @@ -32,15 +18,11 @@ pub struct DevSigner { } impl DevSigner { - /// Generates a random dev signer which satisfies [`EthSigner`] trait - pub fn random() -> Box> { - let mut signers = Self::random_signers(1); - signers.pop().expect("expect to generate at least one signer") - } - /// Generates provided number of random dev signers /// which satisfy [`EthSigner`] trait - pub fn random_signers(num: u32) -> Vec + 'static>> { + pub fn random_signers>( + num: u32, + ) -> Vec + 'static>> { let mut signers = Vec::with_capacity(num as usize); for _ in 0..num { let sk = PrivateKeySigner::random(); @@ -49,11 +31,37 @@ impl DevSigner { let addresses = vec![address]; let accounts = HashMap::from([(address, sk)]); - signers.push(Box::new(Self { addresses, accounts }) as Box>); + signers.push(Box::new(Self { addresses, accounts }) as Box>); } signers } + /// Generates dev signers deterministically from a fixed mnemonic. + /// Uses the Ethereum derivation path: `m/44'/60'/0'/0/{index}` + pub fn from_mnemonic>( + mnemonic: &str, + num: u32, + ) -> Vec + 'static>> { + let mut signers = Vec::with_capacity(num as usize); + + for i in 0..num { + let sk = MnemonicBuilder::::default() + .phrase(mnemonic) + .index(i) + .expect("invalid derivation path") + .build() + .expect("failed to build signer from mnemonic"); + + let address = sk.address(); + let addresses = vec![address]; + let accounts = HashMap::from([(address, sk)]); + + signers.push(Box::new(Self { addresses, accounts }) as Box>); + } + + signers + } + fn get_key(&self, account: Address) -> Result<&PrivateKeySigner> { self.accounts.get(&account).ok_or(SignError::NoAccount) } @@ -65,7 +73,7 @@ impl DevSigner { } #[async_trait::async_trait] -impl EthSigner for DevSigner { +impl> EthSigner for DevSigner { fn accounts(&self) -> Vec
{ self.addresses.clone() } @@ -81,21 +89,17 @@ impl EthSigner for DevSigner { self.sign_hash(hash, address) } - async fn sign_transaction(&self, request: TransactionRequest, address: &Address) -> Result { + async fn sign_transaction(&self, request: TxReq, address: &Address) -> Result { // create local signer wallet from signing key let signer = self.accounts.get(address).ok_or(SignError::NoAccount)?.clone(); - let wallet = EthereumWallet::from(signer); // build and sign transaction with signer - let txn_envelope = - request.build(&wallet).await.map_err(|_| SignError::InvalidTransactionRequest)?; - - // decode transaction into signed transaction type - let encoded = txn_envelope.encoded_2718(); - let txn_signed = T::decode_2718(&mut encoded.as_ref()) + let tx = request + .try_build_and_sign(&signer) + .await .map_err(|_| SignError::InvalidTransactionRequest)?; - Ok(txn_signed) + Ok(tx) } fn sign_typed_data(&self, address: Address, payload: &TypedData) -> Result { @@ -109,7 +113,7 @@ mod tests { use super::*; use alloy_consensus::Transaction; use alloy_primitives::{Bytes, U256}; - use alloy_rpc_types_eth::TransactionInput; + use alloy_rpc_types_eth::{TransactionInput, TransactionRequest}; use reth_ethereum_primitives::TransactionSigned; use revm_primitives::TxKind; diff --git a/crates/rpc/rpc/src/eth/helpers/spec.rs b/crates/rpc/rpc/src/eth/helpers/spec.rs index a4a8ad7531a..fdae08f8f1e 100644 --- a/crates/rpc/rpc/src/eth/helpers/spec.rs +++ b/crates/rpc/rpc/src/eth/helpers/spec.rs @@ -1,31 +1,15 @@ use alloy_primitives::U256; -use reth_chainspec::{ChainSpecProvider, EthereumHardforks}; -use reth_network_api::NetworkInfo; +use reth_rpc_convert::RpcConvert; use reth_rpc_eth_api::{helpers::EthApiSpec, RpcNodeCore}; -use reth_storage_api::{BlockNumReader, BlockReader, ProviderTx, StageCheckpointReader}; use crate::EthApi; -impl EthApiSpec for EthApi +impl EthApiSpec for EthApi where - Self: RpcNodeCore< - Provider: ChainSpecProvider - + BlockNumReader - + StageCheckpointReader, - Network: NetworkInfo, - >, - Provider: BlockReader, + N: RpcNodeCore, + Rpc: RpcConvert, { - type Transaction = ProviderTx; - fn starting_block(&self) -> U256 { self.inner.starting_block() } - - fn signers( - &self, - ) -> &parking_lot::RwLock>>> - { - self.inner.signers() - } } diff --git a/crates/rpc/rpc/src/eth/helpers/state.rs b/crates/rpc/rpc/src/eth/helpers/state.rs index 19b857fa986..3d9cc763097 100644 --- a/crates/rpc/rpc/src/eth/helpers/state.rs +++ b/crates/rpc/rpc/src/eth/helpers/state.rs @@ -1,102 +1,72 @@ //! Contains RPC handler implementations specific to state. -use reth_chainspec::{ChainSpecProvider, EthereumHardforks}; -use reth_storage_api::{BlockReader, StateProviderFactory}; -use reth_transaction_pool::TransactionPool; - +use crate::EthApi; +use reth_rpc_convert::RpcConvert; use reth_rpc_eth_api::{ - helpers::{EthState, LoadState, SpawnBlocking}, - RpcNodeCoreExt, + helpers::{EthState, LoadPendingBlock, LoadState}, + RpcNodeCore, }; -use crate::EthApi; - -impl EthState for EthApi +impl EthState for EthApi where - Self: LoadState + SpawnBlocking, - Provider: BlockReader, + N: RpcNodeCore, + Rpc: RpcConvert, + Self: LoadPendingBlock, { fn max_proof_window(&self) -> u64 { self.inner.eth_proof_window() } } -impl LoadState for EthApi +impl LoadState for EthApi where - Self: RpcNodeCoreExt< - Provider: BlockReader - + StateProviderFactory - + ChainSpecProvider, - Pool: TransactionPool, - >, - Provider: BlockReader, + N: RpcNodeCore, + Rpc: RpcConvert, + Self: LoadPendingBlock, { } #[cfg(test)] mod tests { + use crate::eth::helpers::types::EthRpcConverter; + use super::*; - use alloy_eips::eip1559::ETHEREUM_BLOCK_GAS_LIMIT_30M; use alloy_primitives::{Address, StorageKey, StorageValue, U256}; + use reth_chainspec::ChainSpec; use reth_evm_ethereum::EthEvmConfig; use reth_network_api::noop::NoopNetwork; - use reth_provider::test_utils::{ExtendedAccount, MockEthProvider, NoopProvider}; - use reth_rpc_eth_api::helpers::EthState; - use reth_rpc_eth_types::{ - EthStateCache, FeeHistoryCache, FeeHistoryCacheConfig, GasPriceOracle, - }; - use reth_rpc_server_types::constants::{ - DEFAULT_ETH_PROOF_WINDOW, DEFAULT_MAX_SIMULATE_BLOCKS, DEFAULT_PROOF_PERMITS, + use reth_provider::{ + test_utils::{ExtendedAccount, MockEthProvider, NoopProvider}, + ChainSpecProvider, }; - use reth_tasks::pool::BlockingTaskPool; + use reth_rpc_eth_api::{helpers::EthState, node::RpcNodeCoreAdapter}; use reth_transaction_pool::test_utils::{testing_pool, TestPool}; use std::collections::HashMap; - fn noop_eth_api() -> EthApi { + fn noop_eth_api() -> EthApi< + RpcNodeCoreAdapter, + EthRpcConverter, + > { + let provider = NoopProvider::default(); let pool = testing_pool(); let evm_config = EthEvmConfig::mainnet(); - let cache = EthStateCache::spawn(NoopProvider::default(), Default::default()); - EthApi::new( - NoopProvider::default(), - pool, - NoopNetwork::default(), - cache.clone(), - GasPriceOracle::new(NoopProvider::default(), Default::default(), cache), - ETHEREUM_BLOCK_GAS_LIMIT_30M, - DEFAULT_MAX_SIMULATE_BLOCKS, - DEFAULT_ETH_PROOF_WINDOW, - BlockingTaskPool::build().expect("failed to build tracing pool"), - FeeHistoryCache::new(FeeHistoryCacheConfig::default()), - evm_config, - DEFAULT_PROOF_PERMITS, - ) + EthApi::builder(provider, pool, NoopNetwork::default(), evm_config).build() } fn mock_eth_api( accounts: HashMap, - ) -> EthApi { + ) -> EthApi< + RpcNodeCoreAdapter, + EthRpcConverter, + > { let pool = testing_pool(); let mock_provider = MockEthProvider::default(); let evm_config = EthEvmConfig::new(mock_provider.chain_spec()); mock_provider.extend_accounts(accounts); - let cache = EthStateCache::spawn(mock_provider.clone(), Default::default()); - EthApi::new( - mock_provider.clone(), - pool, - (), - cache.clone(), - GasPriceOracle::new(mock_provider, Default::default(), cache), - ETHEREUM_BLOCK_GAS_LIMIT_30M, - DEFAULT_MAX_SIMULATE_BLOCKS, - DEFAULT_ETH_PROOF_WINDOW + 1, - BlockingTaskPool::build().expect("failed to build tracing pool"), - FeeHistoryCache::new(FeeHistoryCacheConfig::default()), - evm_config, - DEFAULT_PROOF_PERMITS, - ) + EthApi::builder(mock_provider, pool, NoopNetwork::default(), evm_config).build() } #[tokio::test] diff --git a/crates/rpc/rpc/src/eth/helpers/sync_listener.rs b/crates/rpc/rpc/src/eth/helpers/sync_listener.rs new file mode 100644 index 00000000000..e444f76d3af --- /dev/null +++ b/crates/rpc/rpc/src/eth/helpers/sync_listener.rs @@ -0,0 +1,134 @@ +//! A utility Future to asynchronously wait until a node has finished syncing. + +use futures::Stream; +use pin_project::pin_project; +use reth_network_api::NetworkInfo; +use std::{ + future::Future, + pin::Pin, + task::{ready, Context, Poll}, +}; + +/// This future resolves once the node is no longer syncing: [`NetworkInfo::is_syncing`]. +#[must_use = "futures do nothing unless polled"] +#[pin_project] +#[derive(Debug)] +pub struct SyncListener { + #[pin] + tick: St, + network_info: N, +} + +impl SyncListener { + /// Create a new [`SyncListener`] using the given tick stream. + pub const fn new(network_info: N, tick: St) -> Self { + Self { tick, network_info } + } +} + +impl Future for SyncListener +where + N: NetworkInfo, + St: Stream + Unpin, +{ + type Output = (); + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let mut this = self.project(); + + if !this.network_info.is_syncing() { + return Poll::Ready(()); + } + + loop { + let tick_event = ready!(this.tick.as_mut().poll_next(cx)); + + match tick_event { + Some(_) => { + if !this.network_info.is_syncing() { + return Poll::Ready(()); + } + } + None => return Poll::Ready(()), + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use alloy_rpc_types_admin::EthProtocolInfo; + use futures::stream; + use reth_network_api::{NetworkError, NetworkStatus}; + use std::{ + net::{IpAddr, SocketAddr}, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, + }; + + #[derive(Clone)] + struct TestNetwork { + syncing: Arc, + } + + impl NetworkInfo for TestNetwork { + fn local_addr(&self) -> SocketAddr { + (IpAddr::from([0, 0, 0, 0]), 0).into() + } + + async fn network_status(&self) -> Result { + #[allow(deprecated)] + Ok(NetworkStatus { + client_version: "test".to_string(), + protocol_version: 5, + eth_protocol_info: EthProtocolInfo { + network: 1, + difficulty: None, + genesis: Default::default(), + config: Default::default(), + head: Default::default(), + }, + capabilities: vec![], + }) + } + + fn chain_id(&self) -> u64 { + 1 + } + + fn is_syncing(&self) -> bool { + self.syncing.load(Ordering::SeqCst) + } + + fn is_initially_syncing(&self) -> bool { + self.is_syncing() + } + } + + #[tokio::test] + async fn completes_immediately_if_not_syncing() { + let network = TestNetwork { syncing: Arc::new(AtomicBool::new(false)) }; + let fut = SyncListener::new(network, stream::pending::<()>()); + fut.await; + } + + #[tokio::test] + async fn resolves_when_syncing_stops() { + use tokio::sync::mpsc::unbounded_channel; + use tokio_stream::wrappers::UnboundedReceiverStream; + + let syncing = Arc::new(AtomicBool::new(true)); + let network = TestNetwork { syncing: syncing.clone() }; + let (tx, rx) = unbounded_channel(); + let listener = SyncListener::new(network, UnboundedReceiverStream::new(rx)); + let handle = tokio::spawn(listener); + + syncing.store(false, Ordering::Relaxed); + let _ = tx.send(()); + + handle.await.unwrap(); + } +} diff --git a/crates/rpc/rpc/src/eth/helpers/trace.rs b/crates/rpc/rpc/src/eth/helpers/trace.rs index 98f3e255818..3e00f2df0c4 100644 --- a/crates/rpc/rpc/src/eth/helpers/trace.rs +++ b/crates/rpc/rpc/src/eth/helpers/trace.rs @@ -1,27 +1,15 @@ //! Contains RPC handler implementations specific to tracing. -use reth_evm::ConfigureEvm; -use reth_node_api::NodePrimitives; -use reth_rpc_eth_api::{ - helpers::{LoadState, Trace}, - FromEvmError, -}; -use reth_storage_api::{BlockReader, ProviderHeader, ProviderTx}; +use reth_rpc_convert::RpcConvert; +use reth_rpc_eth_api::{helpers::Trace, FromEvmError, RpcNodeCore}; +use reth_rpc_eth_types::EthApiError; use crate::EthApi; -impl Trace for EthApi +impl Trace for EthApi where - Self: LoadState< - Provider: BlockReader, - Evm: ConfigureEvm< - Primitives: NodePrimitives< - BlockHeader = ProviderHeader, - SignedTx = ProviderTx, - >, - >, - Error: FromEvmError, - >, - Provider: BlockReader, + N: RpcNodeCore, + EthApiError: FromEvmError, + Rpc: RpcConvert, { } diff --git a/crates/rpc/rpc/src/eth/helpers/transaction.rs b/crates/rpc/rpc/src/eth/helpers/transaction.rs index e7efc43ac45..7889dd1f54c 100644 --- a/crates/rpc/rpc/src/eth/helpers/transaction.rs +++ b/crates/rpc/rpc/src/eth/helpers/transaction.rs @@ -1,101 +1,187 @@ //! Contains RPC handler implementations specific to transactions +use std::time::Duration; + use crate::EthApi; -use alloy_primitives::{Bytes, B256}; +use alloy_consensus::BlobTransactionValidationError; +use alloy_eips::{eip7594::BlobTransactionSidecarVariant, BlockId, Typed2718}; +use alloy_primitives::{hex, Bytes, B256}; +use reth_chainspec::{ChainSpecProvider, EthereumHardforks}; +use reth_primitives_traits::AlloyBlockHeader; +use reth_rpc_convert::RpcConvert; use reth_rpc_eth_api::{ - helpers::{EthSigner, EthTransactions, LoadTransaction, SpawnBlocking}, - FromEthApiError, FullEthApiTypes, RpcNodeCore, RpcNodeCoreExt, + helpers::{spec::SignersForRpc, EthTransactions, LoadTransaction}, + FromEvmError, RpcNodeCore, +}; +use reth_rpc_eth_types::{error::RpcPoolError, utils::recover_raw_transaction, EthApiError}; +use reth_storage_api::BlockReaderIdExt; +use reth_transaction_pool::{ + error::Eip4844PoolTransactionError, AddedTransactionOutcome, EthBlobTransactionSidecar, + EthPoolTransaction, PoolTransaction, TransactionPool, }; -use reth_rpc_eth_types::utils::recover_raw_transaction; -use reth_storage_api::{BlockReader, BlockReaderIdExt, ProviderTx, TransactionsProvider}; -use reth_transaction_pool::{PoolTransaction, TransactionOrigin, TransactionPool}; -impl EthTransactions - for EthApi +impl EthTransactions for EthApi where - Self: LoadTransaction, - Provider: BlockReader>, + N: RpcNodeCore, + EthApiError: FromEvmError, + Rpc: RpcConvert, { #[inline] - fn signers(&self) -> &parking_lot::RwLock>>>> { + fn signers(&self) -> &SignersForRpc { self.inner.signers() } + #[inline] + fn send_raw_transaction_sync_timeout(&self) -> Duration { + self.inner.send_raw_transaction_sync_timeout() + } + /// Decodes and recovers the transaction and submits it to the pool. /// /// Returns the hash of the transaction. async fn send_raw_transaction(&self, tx: Bytes) -> Result { let recovered = recover_raw_transaction(&tx)?; + let mut pool_transaction = + ::Transaction::from_pooled(recovered); + + // TODO: remove this after Osaka transition + // Convert legacy blob sidecars to EIP-7594 format + if pool_transaction.is_eip4844() { + let EthBlobTransactionSidecar::Present(sidecar) = pool_transaction.take_blob() else { + return Err(EthApiError::PoolError(RpcPoolError::Eip4844( + Eip4844PoolTransactionError::MissingEip4844BlobSidecar, + ))); + }; + + let sidecar = match sidecar { + BlobTransactionSidecarVariant::Eip4844(sidecar) => { + let latest = self + .provider() + .latest_header()? + .ok_or(EthApiError::HeaderNotFound(BlockId::latest()))?; + // Convert to EIP-7594 if next block is Osaka + if self + .provider() + .chain_spec() + .is_osaka_active_at_timestamp(latest.timestamp().saturating_add(12)) + { + BlobTransactionSidecarVariant::Eip7594( + self.blob_sidecar_converter().convert(sidecar).await.ok_or_else( + || { + RpcPoolError::Eip4844( + Eip4844PoolTransactionError::InvalidEip4844Blob( + BlobTransactionValidationError::InvalidProof, + ), + ) + }, + )?, + ) + } else { + BlobTransactionSidecarVariant::Eip4844(sidecar) + } + } + sidecar => sidecar, + }; + + pool_transaction = + EthPoolTransaction::try_from_eip4844(pool_transaction.into_consensus(), sidecar) + .ok_or_else(|| { + RpcPoolError::Eip4844( + Eip4844PoolTransactionError::MissingEip4844BlobSidecar, + ) + })?; + } + + // forward the transaction to the specific endpoint if configured. + if let Some(client) = self.raw_tx_forwarder() { + tracing::debug!(target: "rpc::eth", hash = %pool_transaction.hash(), "forwarding raw transaction to forwarder"); + let rlp_hex = hex::encode_prefixed(&tx); + + // broadcast raw transaction to subscribers if there is any. + self.broadcast_raw_transaction(tx); + + let hash = + client.request("eth_sendRawTransaction", (rlp_hex,)).await.inspect_err(|err| { + tracing::debug!(target: "rpc::eth", %err, hash=% *pool_transaction.hash(), "failed to forward raw transaction"); + }).map_err(EthApiError::other)?; + + // Retain tx in local tx pool after forwarding, for local RPC usage. + let _ = self.inner.add_pool_transaction(pool_transaction).await; + + return Ok(hash); + } + // broadcast raw transaction to subscribers if there is any. self.broadcast_raw_transaction(tx); - let pool_transaction = ::Transaction::from_pooled(recovered); - // submit the transaction to the pool with a `Local` origin - let hash = self - .pool() - .add_transaction(TransactionOrigin::Local, pool_transaction) - .await - .map_err(Self::Error::from_eth_err)?; + let AddedTransactionOutcome { hash, .. } = + self.inner.add_pool_transaction(pool_transaction).await?; Ok(hash) } } -impl LoadTransaction - for EthApi +impl LoadTransaction for EthApi where - Self: SpawnBlocking - + FullEthApiTypes - + RpcNodeCoreExt, - Provider: BlockReader, + N: RpcNodeCore, + EthApiError: FromEvmError, + Rpc: RpcConvert, { } #[cfg(test)] mod tests { use super::*; - use alloy_eips::eip1559::ETHEREUM_BLOCK_GAS_LIMIT_30M; - use alloy_primitives::{hex_literal::hex, Bytes}; - use reth_chainspec::ChainSpecProvider; + use crate::eth::helpers::types::EthRpcConverter; + use alloy_consensus::{Block, Header, SidecarBuilder, SimpleCoder, Transaction}; + use alloy_primitives::{Address, U256}; + use alloy_rpc_types_eth::request::TransactionRequest; + use reth_chainspec::{ChainSpec, ChainSpecBuilder}; use reth_evm_ethereum::EthEvmConfig; use reth_network_api::noop::NoopNetwork; - use reth_provider::test_utils::NoopProvider; - use reth_rpc_eth_api::helpers::EthTransactions; - use reth_rpc_eth_types::{ - EthStateCache, FeeHistoryCache, FeeHistoryCacheConfig, GasPriceOracle, - }; - use reth_rpc_server_types::constants::{ - DEFAULT_ETH_PROOF_WINDOW, DEFAULT_MAX_SIMULATE_BLOCKS, DEFAULT_PROOF_PERMITS, + use reth_provider::{ + test_utils::{ExtendedAccount, MockEthProvider}, + ChainSpecProvider, }; - use reth_tasks::pool::BlockingTaskPool; - use reth_transaction_pool::{test_utils::testing_pool, TransactionPool}; + use reth_rpc_eth_api::node::RpcNodeCoreAdapter; + use reth_transaction_pool::test_utils::{testing_pool, TestPool}; + use std::collections::HashMap; - #[tokio::test] - async fn send_raw_transaction() { - let noop_provider = NoopProvider::default(); - let noop_network_provider = NoopNetwork::default(); + fn mock_eth_api( + accounts: HashMap, + ) -> EthApi< + RpcNodeCoreAdapter, + EthRpcConverter, + > { + let mock_provider = MockEthProvider::default() + .with_chain_spec(ChainSpecBuilder::mainnet().cancun_activated().build()); + mock_provider.extend_accounts(accounts); + let evm_config = EthEvmConfig::new(mock_provider.chain_spec()); let pool = testing_pool(); - let evm_config = EthEvmConfig::new(noop_provider.chain_spec()); - let cache = EthStateCache::spawn(noop_provider.clone(), Default::default()); - let fee_history_cache = FeeHistoryCache::new(FeeHistoryCacheConfig::default()); - let eth_api = EthApi::new( - noop_provider.clone(), - pool.clone(), - noop_network_provider, - cache.clone(), - GasPriceOracle::new(noop_provider, Default::default(), cache.clone()), - ETHEREUM_BLOCK_GAS_LIMIT_30M, - DEFAULT_MAX_SIMULATE_BLOCKS, - DEFAULT_ETH_PROOF_WINDOW, - BlockingTaskPool::build().expect("failed to build tracing pool"), - fee_history_cache, - evm_config, - DEFAULT_PROOF_PERMITS, - ); + let genesis_header = Header { + number: 0, + gas_limit: 30_000_000, + timestamp: 1, + excess_blob_gas: Some(0), + base_fee_per_gas: Some(1000000000), + blob_gas_used: Some(0), + ..Default::default() + }; + + let genesis_hash = B256::ZERO; + mock_provider.add_block(genesis_hash, Block::new(genesis_header, Default::default())); + + EthApi::builder(mock_provider, pool, NoopNetwork::default(), evm_config).build() + } + + #[tokio::test] + async fn send_raw_transaction() { + let eth_api = mock_eth_api(Default::default()); + let pool = eth_api.pool(); // https://etherscan.io/tx/0xa694b71e6c128a2ed8e2e0f6770bddbe52e3bb8f10e8472f9a79ab81497a8b5d let tx_1 = Bytes::from(hex!( @@ -126,4 +212,205 @@ mod tests { assert!(pool.get(&tx_1_result).is_some(), "tx1 not found in the pool"); assert!(pool.get(&tx_2_result).is_some(), "tx2 not found in the pool"); } + + #[tokio::test] + async fn test_fill_transaction_fills_chain_id() { + let address = Address::random(); + let accounts = HashMap::from([( + address, + ExtendedAccount::new(0, U256::from(10_000_000_000_000_000_000u64)), // 10 ETH + )]); + + let eth_api = mock_eth_api(accounts); + + let tx_req = TransactionRequest { + from: Some(address), + to: Some(Address::random().into()), + gas: Some(21_000), + ..Default::default() + }; + + let filled = + eth_api.fill_transaction(tx_req).await.expect("fill_transaction should succeed"); + + // Should fill with the chain id from provider + assert!(filled.tx.chain_id().is_some()); + } + + #[tokio::test] + async fn test_fill_transaction_fills_nonce() { + let address = Address::random(); + let nonce = 42u64; + + let accounts = HashMap::from([( + address, + ExtendedAccount::new(nonce, U256::from(1_000_000_000_000_000_000u64)), // 1 ETH + )]); + + let eth_api = mock_eth_api(accounts); + + let tx_req = TransactionRequest { + from: Some(address), + to: Some(Address::random().into()), + value: Some(U256::from(1000)), + gas: Some(21_000), + ..Default::default() + }; + + let filled = + eth_api.fill_transaction(tx_req).await.expect("fill_transaction should succeed"); + + assert_eq!(filled.tx.nonce(), nonce); + } + + #[tokio::test] + async fn test_fill_transaction_preserves_provided_fields() { + let address = Address::random(); + let provided_nonce = 100u64; + let provided_gas_limit = 50_000u64; + + let accounts = HashMap::from([( + address, + ExtendedAccount::new(42, U256::from(10_000_000_000_000_000_000u64)), + )]); + + let eth_api = mock_eth_api(accounts); + + let tx_req = TransactionRequest { + from: Some(address), + to: Some(Address::random().into()), + value: Some(U256::from(1000)), + nonce: Some(provided_nonce), + gas: Some(provided_gas_limit), + ..Default::default() + }; + + let filled = + eth_api.fill_transaction(tx_req).await.expect("fill_transaction should succeed"); + + // Should preserve the provided nonce and gas limit + assert_eq!(filled.tx.nonce(), provided_nonce); + assert_eq!(filled.tx.gas_limit(), provided_gas_limit); + } + + #[tokio::test] + async fn test_fill_transaction_fills_all_missing_fields() { + let address = Address::random(); + + let balance = U256::from(100u128) * U256::from(1_000_000_000_000_000_000u128); + let accounts = HashMap::from([(address, ExtendedAccount::new(5, balance))]); + + let eth_api = mock_eth_api(accounts); + + // Create a simple transfer transaction + let tx_req = TransactionRequest { + from: Some(address), + to: Some(Address::random().into()), + ..Default::default() + }; + + let filled = + eth_api.fill_transaction(tx_req).await.expect("fill_transaction should succeed"); + + assert!(filled.tx.is_eip1559()); + } + + #[tokio::test] + async fn test_fill_transaction_eip4844_blob_fee() { + let address = Address::random(); + let accounts = HashMap::from([( + address, + ExtendedAccount::new(0, U256::from(10_000_000_000_000_000_000u64)), + )]); + + let eth_api = mock_eth_api(accounts); + + let mut builder = SidecarBuilder::::new(); + builder.ingest(b"dummy blob"); + + // EIP-4844 blob transaction with versioned hashes but no blob fee + let tx_req = TransactionRequest { + from: Some(address), + to: Some(Address::random().into()), + sidecar: Some(builder.build().unwrap()), + ..Default::default() + }; + + let filled = + eth_api.fill_transaction(tx_req).await.expect("fill_transaction should succeed"); + + // Blob transaction should have max_fee_per_blob_gas filled + assert!( + filled.tx.max_fee_per_blob_gas().is_some(), + "max_fee_per_blob_gas should be filled for blob tx" + ); + assert!( + filled.tx.blob_versioned_hashes().is_some(), + "blob_versioned_hashes should be preserved" + ); + } + + #[tokio::test] + async fn test_fill_transaction_eip4844_preserves_blob_fee() { + let address = Address::random(); + let accounts = HashMap::from([( + address, + ExtendedAccount::new(0, U256::from(10_000_000_000_000_000_000u64)), + )]); + + let eth_api = mock_eth_api(accounts); + + let provided_blob_fee = 5000000u128; + + let mut builder = SidecarBuilder::::new(); + builder.ingest(b"dummy blob"); + + // EIP-4844 blob transaction with blob fee already set + let tx_req = TransactionRequest { + from: Some(address), + to: Some(Address::random().into()), + transaction_type: Some(3), // EIP-4844 + sidecar: Some(builder.build().unwrap()), + max_fee_per_blob_gas: Some(provided_blob_fee), // Already set + ..Default::default() + }; + + let filled = + eth_api.fill_transaction(tx_req).await.expect("fill_transaction should succeed"); + + // Should preserve the provided blob fee + assert_eq!( + filled.tx.max_fee_per_blob_gas(), + Some(provided_blob_fee), + "should preserve provided max_fee_per_blob_gas" + ); + } + + #[tokio::test] + async fn test_fill_transaction_non_blob_tx_no_blob_fee() { + let address = Address::random(); + let accounts = HashMap::from([( + address, + ExtendedAccount::new(0, U256::from(10_000_000_000_000_000_000u64)), + )]); + + let eth_api = mock_eth_api(accounts); + + // EIP-1559 transaction without blob fields + let tx_req = TransactionRequest { + from: Some(address), + to: Some(Address::random().into()), + transaction_type: Some(2), // EIP-1559 + ..Default::default() + }; + + let filled = + eth_api.fill_transaction(tx_req).await.expect("fill_transaction should succeed"); + + // Non-blob transaction should NOT have blob fee filled + assert!( + filled.tx.max_fee_per_blob_gas().is_none(), + "max_fee_per_blob_gas should not be set for non-blob tx" + ); + } } diff --git a/crates/rpc/rpc/src/eth/helpers/types.rs b/crates/rpc/rpc/src/eth/helpers/types.rs index 1465e6e9eeb..0c1d59a6ca3 100644 --- a/crates/rpc/rpc/src/eth/helpers/types.rs +++ b/crates/rpc/rpc/src/eth/helpers/types.rs @@ -1,97 +1,27 @@ //! L1 `eth` API types. -use alloy_consensus::{SignableTransaction, Transaction as _, TxEnvelope}; -use alloy_network::{Ethereum, Network}; -use alloy_primitives::Signature; -use alloy_rpc_types::TransactionRequest; -use alloy_rpc_types_eth::{Transaction, TransactionInfo}; -use reth_ethereum_primitives::TransactionSigned; -use reth_primitives_traits::Recovered; -use reth_rpc_eth_api::EthApiTypes; -use reth_rpc_eth_types::EthApiError; -use reth_rpc_types_compat::TransactionCompat; +use alloy_network::Ethereum; +use reth_evm_ethereum::EthEvmConfig; +use reth_rpc_convert::RpcConverter; +use reth_rpc_eth_types::receipt::EthReceiptConverter; -/// A standalone [`EthApiTypes`] implementation for Ethereum. -#[derive(Debug, Clone, Copy, Default)] -pub struct EthereumEthApiTypes(EthTxBuilder); - -impl EthApiTypes for EthereumEthApiTypes { - type Error = EthApiError; - type NetworkTypes = Ethereum; - type TransactionCompat = EthTxBuilder; - - fn tx_resp_builder(&self) -> &Self::TransactionCompat { - &self.0 - } -} - -/// Builds RPC transaction response for l1. -#[derive(Debug, Clone, Copy, Default)] -#[non_exhaustive] -pub struct EthTxBuilder; - -impl TransactionCompat for EthTxBuilder -where - Self: Send + Sync, -{ - type Transaction = ::TransactionResponse; - - type Error = EthApiError; - - fn fill( - &self, - tx: Recovered, - tx_info: TransactionInfo, - ) -> Result { - let tx = tx.convert::(); - - let TransactionInfo { - block_hash, block_number, index: transaction_index, base_fee, .. - } = tx_info; - - let effective_gas_price = base_fee - .map(|base_fee| { - tx.effective_tip_per_gas(base_fee).unwrap_or_default() + base_fee as u128 - }) - .unwrap_or_else(|| tx.max_fee_per_gas()); - - Ok(Transaction { - inner: tx, - block_hash, - block_number, - transaction_index, - effective_gas_price: Some(effective_gas_price), - }) - } - - fn build_simulate_v1_transaction( - &self, - request: TransactionRequest, - ) -> Result { - let Ok(tx) = request.build_typed_tx() else { - return Err(EthApiError::TransactionConversionError) - }; - let signature = Signature::new(Default::default(), Default::default(), false); - Ok(tx.into_signed(signature).into()) - } - - fn otterscan_api_truncate_input(tx: &mut Self::Transaction) { - let input = tx.inner.inner_mut().input_mut(); - *input = input.slice(..4); - } -} +/// An [`RpcConverter`] with its generics set to Ethereum specific. +pub type EthRpcConverter = + RpcConverter>; //tests for simulate #[cfg(test)] mod tests { use super::*; - use alloy_consensus::TxType; + use alloy_consensus::{Transaction, TxType}; + use alloy_rpc_types_eth::TransactionRequest; + use reth_chainspec::MAINNET; use reth_rpc_eth_types::simulate::resolve_transaction; use revm::database::CacheDB; #[test] fn test_resolve_transaction_empty_request() { - let builder = EthTxBuilder::default(); + let builder = EthRpcConverter::new(EthReceiptConverter::new(MAINNET.clone())); let mut db = CacheDB::>::default(); let tx = TransactionRequest::default(); let result = resolve_transaction(tx, 21000, 0, 1, &mut db, &builder).unwrap(); @@ -106,7 +36,7 @@ mod tests { #[test] fn test_resolve_transaction_legacy() { let mut db = CacheDB::>::default(); - let builder = EthTxBuilder::default(); + let builder = EthRpcConverter::new(EthReceiptConverter::new(MAINNET.clone())); let tx = TransactionRequest { gas_price: Some(100), ..Default::default() }; @@ -122,7 +52,7 @@ mod tests { #[test] fn test_resolve_transaction_partial_eip1559() { let mut db = CacheDB::>::default(); - let builder = EthTxBuilder::default(); + let rpc_converter = EthRpcConverter::new(EthReceiptConverter::new(MAINNET.clone())); let tx = TransactionRequest { max_fee_per_gas: Some(200), @@ -130,7 +60,7 @@ mod tests { ..Default::default() }; - let result = resolve_transaction(tx, 21000, 0, 1, &mut db, &builder).unwrap(); + let result = resolve_transaction(tx, 21000, 0, 1, &mut db, &rpc_converter).unwrap(); assert_eq!(result.tx_type(), TxType::Eip1559); let tx = result.into_inner(); diff --git a/crates/rpc/rpc/src/eth/mod.rs b/crates/rpc/rpc/src/eth/mod.rs index e6adb5617d0..af8619de867 100644 --- a/crates/rpc/rpc/src/eth/mod.rs +++ b/crates/rpc/rpc/src/eth/mod.rs @@ -11,13 +11,10 @@ pub mod sim_bundle; /// Implementation of `eth` namespace API. pub use builder::EthApiBuilder; pub use bundle::EthBundle; -pub use core::EthApi; +pub use core::{EthApi, EthApiFor}; pub use filter::EthFilter; pub use pubsub::EthPubSub; -pub use helpers::{ - signer::DevSigner, - types::{EthTxBuilder, EthereumEthApiTypes}, -}; +pub use helpers::{signer::DevSigner, sync_listener::SyncListener}; pub use reth_rpc_eth_api::{EthApiServer, EthApiTypes, FullEthApiServer, RpcNodeCore}; diff --git a/crates/rpc/rpc/src/eth/pubsub.rs b/crates/rpc/rpc/src/eth/pubsub.rs index 300f582ff84..985cdf3129e 100644 --- a/crates/rpc/rpc/src/eth/pubsub.rs +++ b/crates/rpc/rpc/src/eth/pubsub.rs @@ -2,10 +2,10 @@ use std::sync::Arc; -use alloy_primitives::TxHash; +use alloy_primitives::{TxHash, U256}; use alloy_rpc_types_eth::{ pubsub::{Params, PubSubSyncStatus, SubscriptionKind, SyncStatusMetadata}, - FilteredParams, Header, Log, + Filter, Header, Log, }; use futures::StreamExt; use jsonrpsee::{ @@ -15,7 +15,7 @@ use reth_chain_state::CanonStateSubscriptions; use reth_network_api::NetworkInfo; use reth_primitives_traits::NodePrimitives; use reth_rpc_eth_api::{ - pubsub::EthPubSubApiServer, EthApiTypes, RpcNodeCore, RpcTransaction, TransactionCompat, + pubsub::EthPubSubApiServer, EthApiTypes, RpcConvert, RpcNodeCore, RpcTransaction, }; use reth_rpc_eth_types::logs_utils; use reth_rpc_server_types::result::{internal_rpc_err, invalid_params_rpc_err}; @@ -36,8 +36,6 @@ use tracing::error; pub struct EthPubSub { /// All nested fields bundled together. inner: Arc>, - /// The type that's used to spawn subscription tasks. - subscription_task_spawner: Box, } // === impl EthPubSub === @@ -52,141 +50,194 @@ impl EthPubSub { /// Creates a new, shareable instance. pub fn with_spawner(eth_api: Eth, subscription_task_spawner: Box) -> Self { - let inner = EthPubSubInner { eth_api }; - Self { inner: Arc::new(inner), subscription_task_spawner } + let inner = EthPubSubInner { eth_api, subscription_task_spawner }; + Self { inner: Arc::new(inner) } } } -#[async_trait::async_trait] -impl EthPubSubApiServer> for EthPubSub +impl EthPubSub where Eth: RpcNodeCore< - Provider: BlockNumReader + CanonStateSubscriptions, + Provider: BlockNumReader + CanonStateSubscriptions, Pool: TransactionPool, Network: NetworkInfo, - > + EthApiTypes>> - + 'static, + > + EthApiTypes< + RpcConvert: RpcConvert< + Primitives: NodePrimitives>, + >, + >, { - /// Handler for `eth_subscribe` - async fn subscribe( + /// Returns the current sync status for the `syncing` subscription + pub fn sync_status(&self, is_syncing: bool) -> PubSubSyncStatus { + self.inner.sync_status(is_syncing) + } + + /// Returns a stream that yields all transaction hashes emitted by the txpool. + pub fn pending_transaction_hashes_stream(&self) -> impl Stream { + self.inner.pending_transaction_hashes_stream() + } + + /// Returns a stream that yields all transactions emitted by the txpool. + pub fn full_pending_transaction_stream( &self, - pending: PendingSubscriptionSink, - kind: SubscriptionKind, - params: Option, - ) -> jsonrpsee::core::SubscriptionResult { - let sink = pending.accept().await?; - let pubsub = self.inner.clone(); - self.subscription_task_spawner.spawn(Box::pin(async move { - let _ = handle_accepted(pubsub, sink, kind, params).await; - })); + ) -> impl Stream::Transaction>> { + self.inner.full_pending_transaction_stream() + } - Ok(()) + /// Returns a stream that yields all new RPC blocks. + pub fn new_headers_stream(&self) -> impl Stream> { + self.inner.new_headers_stream() } -} -/// The actual handler for an accepted [`EthPubSub::subscribe`] call. -async fn handle_accepted( - pubsub: Arc>, - accepted_sink: SubscriptionSink, - kind: SubscriptionKind, - params: Option, -) -> Result<(), ErrorObject<'static>> -where - Eth: RpcNodeCore< - Provider: BlockNumReader + CanonStateSubscriptions, - Pool: TransactionPool, - Network: NetworkInfo, - > + EthApiTypes>>, -{ - match kind { - SubscriptionKind::NewHeads => { - pipe_from_stream(accepted_sink, pubsub.new_headers_stream()).await - } - SubscriptionKind::Logs => { - // if no params are provided, used default filter params - let filter = match params { - Some(Params::Logs(filter)) => FilteredParams::new(Some(*filter)), - Some(Params::Bool(_)) => { - return Err(invalid_params_rpc_err("Invalid params for logs")) - } - _ => FilteredParams::default(), - }; - pipe_from_stream(accepted_sink, pubsub.log_stream(filter)).await - } - SubscriptionKind::NewPendingTransactions => { - if let Some(params) = params { - match params { - Params::Bool(true) => { - // full transaction objects requested - let stream = pubsub.full_pending_transaction_stream().filter_map(|tx| { - let tx_value = match pubsub - .eth_api - .tx_resp_builder() - .fill_pending(tx.transaction.to_consensus()) - { - Ok(tx) => Some(tx), - Err(err) => { - error!(target = "rpc", - %err, - "Failed to fill transaction with block context" - ); - None - } - }; - std::future::ready(tx_value) - }); - return pipe_from_stream(accepted_sink, stream).await - } - Params::Bool(false) | Params::None => { - // only hashes requested + /// Returns a stream that yields all logs that match the given filter. + pub fn log_stream(&self, filter: Filter) -> impl Stream { + self.inner.log_stream(filter) + } + + /// The actual handler for an accepted [`EthPubSub::subscribe`] call. + pub async fn handle_accepted( + &self, + accepted_sink: SubscriptionSink, + kind: SubscriptionKind, + params: Option, + ) -> Result<(), ErrorObject<'static>> { + #[allow(unreachable_patterns)] + match kind { + SubscriptionKind::NewHeads => { + pipe_from_stream(accepted_sink, self.new_headers_stream()).await + } + SubscriptionKind::Logs => { + // if no params are provided, used default filter params + let filter = match params { + Some(Params::Logs(filter)) => *filter, + Some(Params::Bool(_)) => { + return Err(invalid_params_rpc_err("Invalid params for logs")) } - Params::Logs(_) => { - return Err(invalid_params_rpc_err( - "Invalid params for newPendingTransactions", - )) + _ => Default::default(), + }; + pipe_from_stream(accepted_sink, self.log_stream(filter)).await + } + SubscriptionKind::NewPendingTransactions => { + if let Some(params) = params { + match params { + Params::Bool(true) => { + // full transaction objects requested + let stream = self.full_pending_transaction_stream().filter_map(|tx| { + let tx_value = match self + .inner + .eth_api + .tx_resp_builder() + .fill_pending(tx.transaction.to_consensus()) + { + Ok(tx) => Some(tx), + Err(err) => { + error!(target = "rpc", + %err, + "Failed to fill transaction with block context" + ); + None + } + }; + std::future::ready(tx_value) + }); + return pipe_from_stream(accepted_sink, stream).await + } + Params::Bool(false) | Params::None => { + // only hashes requested + } + Params::Logs(_) => { + return Err(invalid_params_rpc_err( + "Invalid params for newPendingTransactions", + )) + } } } + + pipe_from_stream(accepted_sink, self.pending_transaction_hashes_stream()).await } + SubscriptionKind::Syncing => { + // get new block subscription + let mut canon_state = BroadcastStream::new( + self.inner.eth_api.provider().subscribe_to_canonical_state(), + ); + // get current sync status + let mut initial_sync_status = self.inner.eth_api.network().is_syncing(); + let current_sub_res = self.sync_status(initial_sync_status); - pipe_from_stream(accepted_sink, pubsub.pending_transaction_hashes_stream()).await - } - SubscriptionKind::Syncing => { - // get new block subscription - let mut canon_state = - BroadcastStream::new(pubsub.eth_api.provider().subscribe_to_canonical_state()); - // get current sync status - let mut initial_sync_status = pubsub.eth_api.network().is_syncing(); - let current_sub_res = pubsub.sync_status(initial_sync_status); - - // send the current status immediately - let msg = SubscriptionMessage::from_json(¤t_sub_res) + // send the current status immediately + let msg = SubscriptionMessage::new( + accepted_sink.method_name(), + accepted_sink.subscription_id(), + ¤t_sub_res, + ) .map_err(SubscriptionSerializeError::new)?; - if accepted_sink.send(msg).await.is_err() { - return Ok(()) - } - while canon_state.next().await.is_some() { - let current_syncing = pubsub.eth_api.network().is_syncing(); - // Only send a new response if the sync status has changed - if current_syncing != initial_sync_status { - // Update the sync status on each new block - initial_sync_status = current_syncing; + if accepted_sink.send(msg).await.is_err() { + return Ok(()) + } + + while canon_state.next().await.is_some() { + let current_syncing = self.inner.eth_api.network().is_syncing(); + // Only send a new response if the sync status has changed + if current_syncing != initial_sync_status { + // Update the sync status on each new block + initial_sync_status = current_syncing; - // send a new message now that the status changed - let sync_status = pubsub.sync_status(current_syncing); - let msg = SubscriptionMessage::from_json(&sync_status) + // send a new message now that the status changed + let sync_status = self.sync_status(current_syncing); + let msg = SubscriptionMessage::new( + accepted_sink.method_name(), + accepted_sink.subscription_id(), + &sync_status, + ) .map_err(SubscriptionSerializeError::new)?; - if accepted_sink.send(msg).await.is_err() { - break + + if accepted_sink.send(msg).await.is_err() { + break + } } } - } - Ok(()) + Ok(()) + } + _ => { + // TODO: implement once https://github.com/alloy-rs/alloy/pull/2974 is released + Err(invalid_params_rpc_err("Unsupported subscription kind")) + } } } } +#[async_trait::async_trait] +impl EthPubSubApiServer> for EthPubSub +where + Eth: RpcNodeCore< + Provider: BlockNumReader + CanonStateSubscriptions, + Pool: TransactionPool, + Network: NetworkInfo, + > + EthApiTypes< + RpcConvert: RpcConvert< + Primitives: NodePrimitives>, + >, + > + 'static, +{ + /// Handler for `eth_subscribe` + async fn subscribe( + &self, + pending: PendingSubscriptionSink, + kind: SubscriptionKind, + params: Option, + ) -> jsonrpsee::core::SubscriptionResult { + let sink = pending.accept().await?; + let pubsub = self.clone(); + self.inner.subscription_task_spawner.spawn(Box::pin(async move { + let _ = pubsub.handle_accepted(sink, kind, params).await; + })); + + Ok(()) + } +} + /// Helper to convert a serde error into an [`ErrorObject`] #[derive(Debug, thiserror::Error)] #[error("Failed to serialize subscription item: {0}")] @@ -227,7 +278,12 @@ where break Ok(()) }, }; - let msg = SubscriptionMessage::from_json(&item).map_err(SubscriptionSerializeError::new)?; + let msg = SubscriptionMessage::new( + sink.method_name(), + sink.subscription_id(), + &item + ).map_err(SubscriptionSerializeError::new)?; + if sink.send(msg).await.is_err() { break Ok(()); } @@ -247,6 +303,8 @@ impl std::fmt::Debug for EthPubSub { struct EthPubSubInner { /// The `eth` API. eth_api: EthApi, + /// The type that's used to spawn subscription tasks. + subscription_task_spawner: Box, } // == impl EthPubSubInner === @@ -300,15 +358,23 @@ where /// Returns a stream that yields all new RPC blocks. fn new_headers_stream(&self) -> impl Stream> { self.eth_api.provider().canonical_state_stream().flat_map(|new_chain| { - let headers = new_chain.committed().headers().collect::>(); - futures::stream::iter( - headers.into_iter().map(|h| Header::from_consensus(h.into(), None, None)), - ) + let headers = new_chain + .committed() + .blocks_iter() + .map(|block| { + Header::from_consensus( + block.clone_sealed_header().into(), + None, + Some(U256::from(block.rlp_length())), + ) + }) + .collect::>(); + futures::stream::iter(headers) }) } /// Returns a stream that yields all logs that match the given filter. - fn log_stream(&self, filter: FilteredParams) -> impl Stream { + fn log_stream(&self, filter: Filter) -> impl Stream { BroadcastStream::new(self.eth_api.provider().subscribe_to_canonical_state()) .map(move |canon_state| { canon_state.expect("new block subscription never ends").block_receipts() @@ -318,6 +384,7 @@ where let all_logs = logs_utils::matching_block_logs_with_tx_hashes( &filter, block_receipts.block, + block_receipts.timestamp, block_receipts.tx_receipts.iter().map(|(tx, receipt)| (*tx, receipt)), removed, ); diff --git a/crates/rpc/rpc/src/eth/sim_bundle.rs b/crates/rpc/rpc/src/eth/sim_bundle.rs index 0bd3fb67076..fa3fd46e45c 100644 --- a/crates/rpc/rpc/src/eth/sim_bundle.rs +++ b/crates/rpc/rpc/src/eth/sim_bundle.rs @@ -1,29 +1,30 @@ //! `Eth` Sim bundle implementation and helpers. -use alloy_consensus::BlockHeader; +use alloy_consensus::{transaction::TxHashRef, BlockHeader}; use alloy_eips::BlockNumberOrTag; +use alloy_evm::{env::BlockEnvironment, overrides::apply_block_overrides}; use alloy_primitives::U256; use alloy_rpc_types_eth::BlockId; use alloy_rpc_types_mev::{ - BundleItem, Inclusion, Privacy, RefundConfig, SendBundleRequest, SimBundleLogs, - SimBundleOverrides, SimBundleResponse, Validity, + BundleItem, Inclusion, MevSendBundle, Privacy, RefundConfig, SimBundleLogs, SimBundleOverrides, + SimBundleResponse, Validity, }; use jsonrpsee::core::RpcResult; use reth_evm::{ConfigureEvm, Evm}; -use reth_primitives_traits::{Recovered, SignedTransaction}; -use reth_revm::{database::StateProviderDatabase, db::CacheDB}; +use reth_primitives_traits::Recovered; +use reth_revm::{database::StateProviderDatabase, State}; use reth_rpc_api::MevSimApiServer; use reth_rpc_eth_api::{ helpers::{block::LoadBlock, Call, EthTransactions}, FromEthApiError, FromEvmError, }; -use reth_rpc_eth_types::{ - revm_utils::apply_block_overrides, utils::recover_raw_transaction, EthApiError, -}; +use reth_rpc_eth_types::{utils::recover_raw_transaction, EthApiError}; use reth_storage_api::ProviderTx; use reth_tasks::pool::BlockingTaskGuard; use reth_transaction_pool::{PoolPooledTx, PoolTransaction, TransactionPool}; -use revm::{context_interface::result::ResultAndState, DatabaseCommit, DatabaseRef}; +use revm::{ + context::Block, context_interface::result::ResultAndState, DatabaseCommit, DatabaseRef, +}; use std::{sync::Arc, time::Duration}; use tracing::trace; @@ -89,7 +90,7 @@ where /// inclusion, validity and privacy settings from parent bundles. fn parse_and_flatten_bundle( &self, - request: &SendBundleRequest, + request: &MevSendBundle, ) -> Result>>, EthApiError> { let mut items = Vec::new(); @@ -220,7 +221,7 @@ where async fn sim_bundle_inner( &self, - request: SendBundleRequest, + request: MevSendBundle, overrides: SimBundleOverrides, logs: bool, ) -> Result { @@ -243,12 +244,13 @@ where .spawn_with_state_at_block(current_block_id, move |state| { // Setup environment let current_block_number = current_block.number(); - let coinbase = evm_env.block_env.beneficiary; - let basefee = evm_env.block_env.basefee; - let mut db = CacheDB::new(StateProviderDatabase::new(state)); + let coinbase = evm_env.block_env.beneficiary(); + let basefee = evm_env.block_env.basefee(); + let mut db = + State::builder().with_database(StateProviderDatabase::new(state)).build(); // apply overrides - apply_block_overrides(block_overrides, &mut db, &mut evm_env.block_env); + apply_block_overrides(block_overrides, &mut db, evm_env.block_env.inner_mut()); let initial_coinbase_balance = DatabaseRef::basic_ref(&db, coinbase) .map_err(EthApiError::from_eth_err)? @@ -416,7 +418,7 @@ where { async fn sim_bundle( &self, - request: SendBundleRequest, + request: MevSendBundle, overrides: SimBundleOverrides, ) -> RpcResult { trace!("mev_simBundle called, request: {:?}, overrides: {:?}", request, overrides); @@ -425,7 +427,7 @@ where let timeout = override_timeout .map(Duration::from_secs) - .filter(|&custom_duration| custom_duration <= MAX_SIM_TIMEOUT) + .map(|d| d.min(MAX_SIM_TIMEOUT)) .unwrap_or(DEFAULT_SIM_TIMEOUT); let bundle_res = diff --git a/crates/rpc/rpc/src/lib.rs b/crates/rpc/rpc/src/lib.rs index bac57b63035..b5a20c19cf6 100644 --- a/crates/rpc/rpc/src/lib.rs +++ b/crates/rpc/rpc/src/lib.rs @@ -22,7 +22,7 @@ html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256", issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(test), warn(unused_crate_dependencies))] use http as _; @@ -33,6 +33,7 @@ use pin_project as _; use tower as _; mod admin; +mod aliases; mod debug; mod engine; pub mod eth; @@ -47,13 +48,15 @@ mod validation; mod web3; pub use admin::AdminApi; +pub use aliases::*; pub use debug::DebugApi; pub use engine::{EngineApi, EngineEthApi}; -pub use eth::{EthApi, EthApiBuilder, EthBundle, EthFilter, EthPubSub}; +pub use eth::{helpers::SyncListener, EthApi, EthApiBuilder, EthBundle, EthFilter, EthPubSub}; pub use miner::MinerApi; pub use net::NetApi; pub use otterscan::OtterscanApi; pub use reth::RethApi; +pub use reth_rpc_convert::RpcTypes; pub use rpc::RPCApi; pub use trace::TraceApi; pub use txpool::TxPoolApi; diff --git a/crates/rpc/rpc/src/otterscan.rs b/crates/rpc/rpc/src/otterscan.rs index 6eb2ec992ac..334e8d7dea4 100644 --- a/crates/rpc/rpc/src/otterscan.rs +++ b/crates/rpc/rpc/src/otterscan.rs @@ -1,5 +1,5 @@ -use alloy_consensus::{BlockHeader, Transaction, Typed2718}; -use alloy_eips::{BlockId, BlockNumberOrTag}; +use alloy_consensus::{BlockHeader, Typed2718}; +use alloy_eips::{eip1898::LenientBlockNumberOrTag, BlockId}; use alloy_network::{ReceiptResponse, TransactionResponse}; use alloy_primitives::{Address, Bytes, TxHash, B256, U256}; use alloy_rpc_types_eth::{BlockTransactions, TransactionReceipt}; @@ -12,10 +12,12 @@ use alloy_rpc_types_trace::{ }; use async_trait::async_trait; use jsonrpsee::{core::RpcResult, types::ErrorObjectOwned}; +use reth_primitives_traits::TxTy; use reth_rpc_api::{EthApiServer, OtterscanServer}; +use reth_rpc_convert::RpcTxReq; use reth_rpc_eth_api::{ helpers::{EthTransactions, TraceExt}, - FullEthApiTypes, RpcBlock, RpcHeader, RpcReceipt, RpcTransaction, TransactionCompat, + FullEthApiTypes, RpcBlock, RpcHeader, RpcReceipt, RpcTransaction, }; use reth_rpc_eth_types::{utils::binary_search, EthApiError}; use reth_rpc_server_types::result::internal_rpc_err; @@ -67,10 +69,12 @@ impl OtterscanServer, RpcHeader where Eth: EthApiServer< + RpcTxReq, RpcTransaction, RpcBlock, RpcReceipt, RpcHeader, + TxTy, > + EthTransactions + TraceExt + 'static, @@ -78,9 +82,9 @@ where /// Handler for `ots_getHeaderByNumber` and `erigon_getHeaderByNumber` async fn get_header_by_number( &self, - block_number: u64, + block_number: LenientBlockNumberOrTag, ) -> RpcResult>> { - self.eth.header_by_number(BlockNumberOrTag::Number(block_number)).await + self.eth.header_by_number(block_number.into()).await } /// Handler for `ots_hasCode` @@ -116,7 +120,6 @@ where TransferKind::Create => OperationType::OpCreate, TransferKind::Create2 => OperationType::OpCreate2, TransferKind::SelfDestruct => OperationType::OpSelfDestruct, - TransferKind::EofCreate => OperationType::OpEofCreate, }, }) .collect::>() @@ -174,11 +177,11 @@ where /// Handler for `ots_getBlockDetails` async fn get_block_details( &self, - block_number: u64, + block_number: LenientBlockNumberOrTag, ) -> RpcResult>> { + let block_number = block_number.into_inner(); + let block = self.eth.block_by_number(block_number, true); let block_id = block_number.into(); - let block = self.eth.block_by_number(block_id, true); - let block_id = block_id.into(); let receipts = self.eth.block_receipts(block_id); let (block, receipts) = futures::try_join!(block, receipts)?; self.block_details( @@ -205,16 +208,16 @@ where /// Handler for `ots_getBlockTransactions` async fn get_block_transactions( &self, - block_number: u64, + block_number: LenientBlockNumberOrTag, page_number: usize, page_size: usize, ) -> RpcResult< OtsBlockTransactions, RpcHeader>, > { - let block_id = block_number.into(); + let block_number = block_number.into_inner(); // retrieve full block and its receipts - let block = self.eth.block_by_number(block_id, true); - let block_id = block_id.into(); + let block = self.eth.block_by_number(block_number, true); + let block_id = block_number.into(); let receipts = self.eth.block_receipts(block_id); let (block, receipts) = futures::try_join!(block, receipts)?; @@ -241,15 +244,6 @@ where // Crop transactions *transactions = transactions.drain(page_start..page_end).collect::>(); - // The input field returns only the 4 bytes method selector instead of the entire - // calldata byte blob - // See also: - for tx in transactions.iter_mut() { - if tx.input().len() > 4 { - Eth::TransactionCompat::otterscan_api_truncate_input(tx); - } - } - // Crop receipts and transform them into OtsTransactionReceipt let timestamp = Some(block.header.timestamp()); let receipts = receipts @@ -293,7 +287,7 @@ where async fn search_transactions_before( &self, _address: Address, - _block_number: u64, + _block_number: LenientBlockNumberOrTag, _page_size: usize, ) -> RpcResult { Err(internal_rpc_err("unimplemented")) @@ -303,7 +297,7 @@ where async fn search_transactions_after( &self, _address: Address, - _block_number: u64, + _block_number: LenientBlockNumberOrTag, _page_size: usize, ) -> RpcResult { Err(internal_rpc_err("unimplemented")) @@ -348,8 +342,11 @@ where num.into(), None, TracingInspectorConfig::default_parity(), - |tx_info, inspector, _, _, _| { - Ok(inspector.into_parity_builder().into_localized_transaction_traces(tx_info)) + |tx_info, mut ctx| { + Ok(ctx + .take_inspector() + .into_parity_builder() + .into_localized_transaction_traces(tx_info)) }, ) .await diff --git a/crates/rpc/rpc/src/reth.rs b/crates/rpc/rpc/src/reth.rs index a032db1084d..8f8decd7f4a 100644 --- a/crates/rpc/rpc/src/reth.rs +++ b/crates/rpc/rpc/src/reth.rs @@ -3,10 +3,15 @@ use std::{collections::HashMap, future::Future, sync::Arc}; use alloy_eips::BlockId; use alloy_primitives::{Address, U256}; use async_trait::async_trait; -use jsonrpsee::core::RpcResult; +use futures::StreamExt; +use jsonrpsee::{core::RpcResult, PendingSubscriptionSink, SubscriptionMessage, SubscriptionSink}; +use jsonrpsee_types::ErrorObject; +use reth_chain_state::{CanonStateNotificationStream, CanonStateSubscriptions}; use reth_errors::RethResult; +use reth_primitives_traits::NodePrimitives; use reth_rpc_api::RethApiServer; use reth_rpc_eth_types::{EthApiError, EthResult}; +use reth_rpc_server_types::result::internal_rpc_err; use reth_storage_api::{BlockReaderIdExt, ChangeSetReader, StateProviderFactory}; use reth_tasks::TaskSpawner; use tokio::sync::oneshot; @@ -88,7 +93,11 @@ where #[async_trait] impl RethApiServer for RethApi where - Provider: BlockReaderIdExt + ChangeSetReader + StateProviderFactory + 'static, + Provider: BlockReaderIdExt + + ChangeSetReader + + StateProviderFactory + + CanonStateSubscriptions + + 'static, { /// Handler for `reth_getBalanceChangesInBlock` async fn reth_get_balance_changes_in_block( @@ -97,6 +106,50 @@ where ) -> RpcResult> { Ok(Self::balance_changes_in_block(self, block_id).await?) } + + /// Handler for `reth_subscribeChainNotifications` + async fn reth_subscribe_chain_notifications( + &self, + pending: PendingSubscriptionSink, + ) -> jsonrpsee::core::SubscriptionResult { + let sink = pending.accept().await?; + let stream = self.provider().canonical_state_stream(); + self.inner.task_spawner.spawn(Box::pin(async move { + let _ = pipe_from_stream(sink, stream).await; + })); + + Ok(()) + } +} + +/// Pipes all stream items to the subscription sink. +async fn pipe_from_stream( + sink: SubscriptionSink, + mut stream: CanonStateNotificationStream, +) -> Result<(), ErrorObject<'static>> { + loop { + tokio::select! { + _ = sink.closed() => { + // connection dropped + break Ok(()) + } + maybe_item = stream.next() => { + let item = match maybe_item { + Some(item) => item, + None => { + // stream ended + break Ok(()) + }, + }; + let msg = SubscriptionMessage::new(sink.method_name(), sink.subscription_id(), &item) + .map_err(|e| internal_rpc_err(e.to_string()))?; + + if sink.send(msg).await.is_err() { + break Ok(()); + } + } + } + } } impl std::fmt::Debug for RethApi { diff --git a/crates/rpc/rpc/src/trace.rs b/crates/rpc/rpc/src/trace.rs index a2c621045d0..e1e6bc26544 100644 --- a/crates/rpc/rpc/src/trace.rs +++ b/crates/rpc/rpc/src/trace.rs @@ -1,10 +1,12 @@ use alloy_consensus::BlockHeader as _; use alloy_eips::BlockId; use alloy_evm::block::calc::{base_block_reward_pre_merge, block_reward, ommer_reward}; -use alloy_primitives::{map::HashSet, Bytes, B256, U256}; +use alloy_primitives::{ + map::{HashMap, HashSet}, + Address, BlockHash, Bytes, B256, U256, +}; use alloy_rpc_types_eth::{ state::{EvmOverrides, StateOverride}, - transaction::TransactionRequest, BlockOverrides, Index, }; use alloy_rpc_types_trace::{ @@ -18,8 +20,9 @@ use jsonrpsee::core::RpcResult; use reth_chainspec::{ChainSpecProvider, EthChainSpec, EthereumHardfork, MAINNET, SEPOLIA}; use reth_evm::ConfigureEvm; use reth_primitives_traits::{BlockBody, BlockHeader}; -use reth_revm::{database::StateProviderDatabase, db::CacheDB}; +use reth_revm::{database::StateProviderDatabase, State}; use reth_rpc_api::TraceApiServer; +use reth_rpc_convert::RpcTxReq; use reth_rpc_eth_api::{ helpers::{Call, LoadPendingBlock, LoadTransaction, Trace, TraceExt}, FromEthApiError, RpcNodeCore, @@ -31,8 +34,10 @@ use reth_transaction_pool::{PoolPooledTx, PoolTransaction, TransactionPool}; use revm::DatabaseCommit; use revm_inspectors::{ opcode::OpcodeGasInspector, + storage::StorageInspector, tracing::{parity::populate_state_diff, TracingInspector, TracingInspectorConfig}, }; +use serde::{Deserialize, Serialize}; use std::sync::Arc; use tokio::sync::{AcquireError, OwnedSemaphorePermit}; @@ -87,7 +92,7 @@ where /// Executes the given call and returns a number of possible traces for it. pub async fn trace_call( &self, - trace_request: TraceCallRequest, + trace_request: TraceCallRequest>, ) -> Result { let at = trace_request.block_id.unwrap_or_default(); let config = TracingInspectorConfig::from_parity_config(&trace_request.trace_types); @@ -101,7 +106,7 @@ where // let db = db.0; - let (res, _) = this.eth_api().inspect(&mut *db, evm_env, tx_env, &mut inspector)?; + let res = this.eth_api().inspect(&mut *db, evm_env, tx_env, &mut inspector)?; let trace_res = inspector .into_parity_builder() .into_trace_results_with_state(&res, &trace_request.trace_types, &db) @@ -142,7 +147,7 @@ where /// Note: Allows tracing dependent transactions, hence all transactions are traced in sequence pub async fn trace_call_many( &self, - calls: Vec<(TransactionRequest, HashSet)>, + calls: Vec<(RpcTxReq, HashSet)>, block_id: Option, ) -> Result, Eth::Error> { let at = block_id.unwrap_or(BlockId::pending()); @@ -153,7 +158,8 @@ where self.eth_api() .spawn_with_state_at_block(at, move |state| { let mut results = Vec::with_capacity(calls.len()); - let mut db = CacheDB::new(StateProviderDatabase::new(state)); + let mut db = + State::builder().with_database(StateProviderDatabase::new(state)).build(); let mut calls = calls.into_iter().peekable(); @@ -166,8 +172,7 @@ where )?; let config = TracingInspectorConfig::from_parity_config(&trace_types); let mut inspector = TracingInspector::new(config); - let (res, _) = - this.eth_api().inspect(&mut db, evm_env, tx_env, &mut inspector)?; + let res = this.eth_api().inspect(&mut db, evm_env, tx_env, &mut inspector)?; let trace_res = inspector .into_parity_builder() @@ -358,7 +363,7 @@ where ) -> Result, Eth::Error> { // We'll reuse the matcher across multiple blocks that are traced in parallel let matcher = Arc::new(filter.matcher()); - let TraceFilter { from_block, to_block, after, count, .. } = filter; + let TraceFilter { from_block, to_block, mut after, count, .. } = filter; let start = from_block.unwrap_or(0); let latest_block = self.provider().best_block_number().map_err(Eth::Error::from_eth_err)?; @@ -384,78 +389,97 @@ where .into()) } - // fetch all blocks in that range - let blocks = self - .provider() - .recovered_block_range(start..=end) - .map_err(Eth::Error::from_eth_err)? - .into_iter() - .map(Arc::new) - .collect::>(); - - // trace all blocks - let mut block_traces = Vec::with_capacity(blocks.len()); - for block in &blocks { - let matcher = matcher.clone(); - let traces = self.eth_api().trace_block_until( - block.hash().into(), - Some(block.clone()), - None, - TracingInspectorConfig::default_parity(), - move |tx_info, inspector, _, _, _| { - let mut traces = - inspector.into_parity_builder().into_localized_transaction_traces(tx_info); - traces.retain(|trace| matcher.matches(&trace.trace)); - Ok(Some(traces)) - }, - ); - block_traces.push(traces); - } - - let block_traces = futures::future::try_join_all(block_traces).await?; - let mut all_traces = block_traces - .into_iter() - .flatten() - .flat_map(|traces| traces.into_iter().flatten().flat_map(|traces| traces.into_iter())) - .collect::>(); - - // add reward traces for all blocks - for block in &blocks { - if let Some(base_block_reward) = self.calculate_base_block_reward(block.header())? { - all_traces.extend( - self.extract_reward_traces( - block.header(), - block.body().ommers(), - base_block_reward, - ) - .into_iter() - .filter(|trace| matcher.matches(&trace.trace)), + let mut all_traces = Vec::new(); + let mut block_traces = Vec::with_capacity(self.inner.eth_config.max_tracing_requests); + for chunk_start in (start..end).step_by(self.inner.eth_config.max_tracing_requests) { + let chunk_end = + std::cmp::min(chunk_start + self.inner.eth_config.max_tracing_requests as u64, end); + + // fetch all blocks in that chunk + let blocks = self + .eth_api() + .spawn_blocking_io(move |this| { + Ok(this + .provider() + .recovered_block_range(chunk_start..=chunk_end) + .map_err(Eth::Error::from_eth_err)? + .into_iter() + .map(Arc::new) + .collect::>()) + }) + .await?; + + // trace all blocks + for block in &blocks { + let matcher = matcher.clone(); + let traces = self.eth_api().trace_block_until( + block.hash().into(), + Some(block.clone()), + None, + TracingInspectorConfig::default_parity(), + move |tx_info, mut ctx| { + let mut traces = ctx + .take_inspector() + .into_parity_builder() + .into_localized_transaction_traces(tx_info); + traces.retain(|trace| matcher.matches(&trace.trace)); + Ok(Some(traces)) + }, ); - } else { - // no block reward, means we're past the Paris hardfork and don't expect any rewards - // because the blocks in ascending order - break + block_traces.push(traces); } - } - // Skips the first `after` number of matching traces. - // If `after` is greater than or equal to the number of matched traces, it returns an empty - // array. - if let Some(after) = after.map(|a| a as usize) { - if after < all_traces.len() { - all_traces.drain(..after); - } else { - return Ok(vec![]) + #[allow(clippy::iter_with_drain)] + let block_traces = futures::future::try_join_all(block_traces.drain(..)).await?; + all_traces.extend(block_traces.into_iter().flatten().flat_map(|traces| { + traces.into_iter().flatten().flat_map(|traces| traces.into_iter()) + })); + + // add reward traces for all blocks + for block in &blocks { + if let Some(base_block_reward) = self.calculate_base_block_reward(block.header())? { + all_traces.extend( + self.extract_reward_traces( + block.header(), + block.body().ommers(), + base_block_reward, + ) + .into_iter() + .filter(|trace| matcher.matches(&trace.trace)), + ); + } else { + // no block reward, means we're past the Paris hardfork and don't expect any + // rewards because the blocks in ascending order + break + } } - } - // Return at most `count` of traces - if let Some(count) = count { - let count = count as usize; - if count < all_traces.len() { - all_traces.truncate(count); + // Skips the first `after` number of matching traces. + if let Some(cutoff) = after.map(|a| a as usize) && + cutoff < all_traces.len() + { + all_traces.drain(..cutoff); + // we removed the first `after` traces + after = None; } - }; + + // Return at most `count` of traces + if let Some(count) = count { + let count = count as usize; + if count < all_traces.len() { + all_traces.truncate(count); + return Ok(all_traces) + } + }; + } + + // If `after` is greater than or equal to the number of matched traces, it returns an + // empty array. + if let Some(cutoff) = after.map(|a| a as usize) && + cutoff >= all_traces.len() + { + return Ok(vec![]) + } Ok(all_traces) } @@ -469,9 +493,11 @@ where block_id, None, TracingInspectorConfig::default_parity(), - |tx_info, inspector, _, _, _| { - let traces = - inspector.into_parity_builder().into_localized_transaction_traces(tx_info); + |tx_info, mut ctx| { + let traces = ctx + .take_inspector() + .into_parity_builder() + .into_localized_transaction_traces(tx_info); Ok(traces) }, ); @@ -482,14 +508,14 @@ where let mut maybe_traces = maybe_traces.map(|traces| traces.into_iter().flatten().collect::>()); - if let (Some(block), Some(traces)) = (maybe_block, maybe_traces.as_mut()) { - if let Some(base_block_reward) = self.calculate_base_block_reward(block.header())? { - traces.extend(self.extract_reward_traces( - block.header(), - block.body().ommers(), - base_block_reward, - )); - } + if let (Some(block), Some(traces)) = (maybe_block, maybe_traces.as_mut()) && + let Some(base_block_reward) = self.calculate_base_block_reward(block.header())? + { + traces.extend(self.extract_reward_traces( + block.header(), + block.body().ommers(), + base_block_reward, + )); } Ok(maybe_traces) @@ -506,14 +532,16 @@ where block_id, None, TracingInspectorConfig::from_parity_config(&trace_types), - move |tx_info, inspector, res, state, db| { - let mut full_trace = - inspector.into_parity_builder().into_trace_results(&res, &trace_types); + move |tx_info, mut ctx| { + let mut full_trace = ctx + .take_inspector() + .into_parity_builder() + .into_trace_results(&ctx.result, &trace_types); // If statediffs were requested, populate them with the account balance and // nonce from pre-state if let Some(ref mut state_diff) = full_trace.state_diff { - populate_state_diff(state_diff, db, state.iter()) + populate_state_diff(state_diff, &ctx.db, ctx.state.iter()) .map_err(Eth::Error::from_eth_err)?; } @@ -541,10 +569,10 @@ where block_id, None, OpcodeGasInspector::default, - move |tx_info, inspector, _res, _, _| { + move |tx_info, ctx| { let trace = TransactionOpcodeGas { transaction_hash: tx_info.hash.expect("tx hash is set"), - opcode_gas: inspector.opcode_gas_iter().collect(), + opcode_gas: ctx.inspector.opcode_gas_iter().collect(), }; Ok(trace) }, @@ -561,10 +589,45 @@ where transactions, })) } + + /// Returns all storage slots accessed during transaction execution along with their access + /// counts. + pub async fn trace_block_storage_access( + &self, + block_id: BlockId, + ) -> Result, Eth::Error> { + let res = self + .eth_api() + .trace_block_inspector( + block_id, + None, + StorageInspector::default, + move |tx_info, ctx| { + let trace = TransactionStorageAccess { + transaction_hash: tx_info.hash.expect("tx hash is set"), + storage_access: ctx.inspector.accessed_slots().clone(), + unique_loads: ctx.inspector.unique_loads(), + warm_loads: ctx.inspector.warm_loads(), + }; + Ok(trace) + }, + ) + .await?; + + let Some(transactions) = res else { return Ok(None) }; + + let Some(block) = self.eth_api().recovered_block(block_id).await? else { return Ok(None) }; + + Ok(Some(BlockStorageAccess { + block_hash: block.hash(), + block_number: block.number(), + transactions, + })) + } } #[async_trait] -impl TraceApiServer for TraceApi +impl TraceApiServer> for TraceApi where Eth: TraceExt + 'static, { @@ -573,7 +636,7 @@ where /// Handler for `trace_call` async fn trace_call( &self, - call: TransactionRequest, + call: RpcTxReq, trace_types: HashSet, block_id: Option, state_overrides: Option, @@ -588,7 +651,7 @@ where /// Handler for `trace_callMany` async fn trace_call_many( &self, - calls: Vec<(TransactionRequest, HashSet)>, + calls: Vec<(RpcTxReq, HashSet)>, block_id: Option, ) -> RpcResult> { let _permit = self.acquire_trace_permit().await; @@ -646,6 +709,7 @@ where /// # Limitations /// This currently requires block filter fields, since reth does not have address indices yet. async fn trace_filter(&self, filter: TraceFilter) -> RpcResult> { + let _permit = self.inner.blocking_task_guard.clone().acquire_many_owned(2).await; Ok(Self::trace_filter(self, filter).await.map_err(Into::into)?) } @@ -707,6 +771,33 @@ struct TraceApiInner { eth_config: EthConfig, } +/// Response type for storage tracing that contains all accessed storage slots +/// for a transaction. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TransactionStorageAccess { + /// Hash of the transaction + pub transaction_hash: B256, + /// Tracks storage slots and access counter. + pub storage_access: HashMap>, + /// Number of unique storage loads + pub unique_loads: u64, + /// Number of warm storage loads + pub warm_loads: u64, +} + +/// Response type for storage tracing that contains all accessed storage slots +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct BlockStorageAccess { + /// The block hash + pub block_hash: BlockHash, + /// The block's number + pub block_number: u64, + /// All executed transactions in the block in the order they were executed + pub transactions: Vec, +} + /// Helper to construct a [`LocalizedTransactionTrace`] that describes a reward to the block /// beneficiary. fn reward_trace(header: &H, reward: RewardAction) -> LocalizedTransactionTrace { diff --git a/crates/rpc/rpc/src/txpool.rs b/crates/rpc/rpc/src/txpool.rs index 7facf6dd91e..5c7bcd45a84 100644 --- a/crates/rpc/rpc/src/txpool.rs +++ b/crates/rpc/rpc/src/txpool.rs @@ -8,8 +8,10 @@ use alloy_rpc_types_txpool::{ }; use async_trait::async_trait; use jsonrpsee::core::RpcResult; +use reth_primitives_traits::NodePrimitives; use reth_rpc_api::TxPoolApiServer; -use reth_rpc_types_compat::TransactionCompat; +use reth_rpc_convert::{RpcConvert, RpcTypes}; +use reth_rpc_eth_api::RpcTransaction; use reth_transaction_pool::{ AllPoolTransactions, PoolConsensusTx, PoolTransaction, TransactionPool, }; @@ -35,18 +37,21 @@ impl TxPoolApi { impl TxPoolApi where Pool: TransactionPool> + 'static, - Eth: TransactionCompat>, + Eth: RpcConvert>>, { - fn content(&self) -> Result, Eth::Error> { + fn content(&self) -> Result>, Eth::Error> { #[inline] fn insert( tx: &Tx, - content: &mut BTreeMap>, + content: &mut BTreeMap< + Address, + BTreeMap::TransactionResponse>, + >, resp_builder: &RpcTxB, ) -> Result<(), RpcTxB::Error> where Tx: PoolTransaction, - RpcTxB: TransactionCompat, + RpcTxB: RpcConvert>, { content.entry(tx.sender()).or_default().insert( tx.nonce().to_string(), @@ -71,10 +76,10 @@ where } #[async_trait] -impl TxPoolApiServer for TxPoolApi +impl TxPoolApiServer> for TxPoolApi where Pool: TransactionPool> + 'static, - Eth: TransactionCompat> + 'static, + Eth: RpcConvert>> + 'static, { /// Returns the number of transactions currently pending for inclusion in the next block(s), as /// well as the ones that are being scheduled for future execution only. @@ -83,8 +88,8 @@ where /// Handler for `txpool_status` async fn txpool_status(&self) -> RpcResult { trace!(target: "rpc::eth", "Serving txpool_status"); - let all = self.pool.all_transactions(); - Ok(TxpoolStatus { pending: all.pending.len() as u64, queued: all.queued.len() as u64 }) + let (pending, queued) = self.pool.pending_and_queued_txn_count(); + Ok(TxpoolStatus { pending: pending as u64, queued: queued as u64 }) } /// Returns a summary of all the transactions currently pending for inclusion in the next @@ -128,7 +133,7 @@ where async fn txpool_content_from( &self, from: Address, - ) -> RpcResult> { + ) -> RpcResult>> { trace!(target: "rpc::eth", ?from, "Serving txpool_contentFrom"); Ok(self.content().map_err(Into::into)?.remove_from(&from)) } @@ -138,7 +143,7 @@ where /// /// See [here](https://geth.ethereum.org/docs/rpc/ns-txpool#txpool_content) for more details /// Handler for `txpool_content` - async fn txpool_content(&self) -> RpcResult> { + async fn txpool_content(&self) -> RpcResult>> { trace!(target: "rpc::eth", "Serving txpool_content"); Ok(self.content().map_err(Into::into)?) } diff --git a/crates/rpc/rpc/src/validation.rs b/crates/rpc/rpc/src/validation.rs index d21e8f13e74..d03846a4279 100644 --- a/crates/rpc/rpc/src/validation.rs +++ b/crates/rpc/rpc/src/validation.rs @@ -5,10 +5,11 @@ use alloy_eips::{eip4844::kzg_to_versioned_hash, eip7685::RequestsOrHash}; use alloy_rpc_types_beacon::relay::{ BidTrace, BuilderBlockValidationRequest, BuilderBlockValidationRequestV2, BuilderBlockValidationRequestV3, BuilderBlockValidationRequestV4, + BuilderBlockValidationRequestV5, }; use alloy_rpc_types_engine::{ - BlobsBundleV1, CancunPayloadFields, ExecutionData, ExecutionPayload, ExecutionPayloadSidecar, - PraguePayloadFields, + BlobsBundleV1, BlobsBundleV2, CancunPayloadFields, ExecutionData, ExecutionPayload, + ExecutionPayloadSidecar, PraguePayloadFields, }; use async_trait::async_trait; use core::fmt; @@ -16,12 +17,17 @@ use jsonrpsee::core::RpcResult; use jsonrpsee_types::error::ErrorObject; use reth_chainspec::{ChainSpecProvider, EthereumHardforks}; use reth_consensus::{Consensus, FullConsensus}; +use reth_consensus_common::validation::MAX_RLP_BLOCK_SIZE; use reth_engine_primitives::PayloadValidator; use reth_errors::{BlockExecutionError, ConsensusError, ProviderError}; use reth_evm::{execute::Executor, ConfigureEvm}; use reth_execution_types::BlockExecutionOutput; -use reth_metrics::{metrics, metrics::Gauge, Metrics}; -use reth_node_api::NewPayloadError; +use reth_metrics::{ + metrics, + metrics::{gauge, Gauge}, + Metrics, +}; +use reth_node_api::{NewPayloadError, PayloadTypes}; use reth_primitives_traits::{ constants::GAS_LIMIT_BOUND_DIVISOR, BlockBody, GotExpected, NodePrimitives, RecoveredBlock, SealedBlock, SealedHeaderFor, @@ -33,20 +39,22 @@ use reth_storage_api::{BlockReaderIdExt, StateProviderFactory}; use reth_tasks::TaskSpawner; use revm_primitives::{Address, B256, U256}; use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; use std::{collections::HashSet, sync::Arc}; use tokio::sync::{oneshot, RwLock}; use tracing::warn; /// The type that implements the `validation` rpc namespace trait #[derive(Clone, Debug, derive_more::Deref)] -pub struct ValidationApi { +pub struct ValidationApi { #[deref] - inner: Arc>, + inner: Arc>, } -impl ValidationApi +impl ValidationApi where E: ConfigureEvm, + T: PayloadTypes, { /// Create a new instance of the [`ValidationApi`] pub fn new( @@ -56,10 +64,7 @@ where config: ValidationApiConfig, task_spawner: Box, payload_validator: Arc< - dyn PayloadValidator< - Block = ::Block, - ExecutionData = ExecutionData, - >, + dyn PayloadValidator::Block>, >, ) -> Self { let ValidationApiConfig { disallow, validation_window } = config; @@ -77,6 +82,11 @@ where }); inner.metrics.disallow_size.set(inner.disallow.len() as f64); + + let disallow_hash = hash_disallow_list(&inner.disallow); + let hash_gauge = gauge!("builder_validation_disallow_hash", "hash" => disallow_hash); + hash_gauge.set(1.0); + Self { inner } } @@ -101,13 +111,14 @@ where } } -impl ValidationApi +impl ValidationApi where Provider: BlockReaderIdExt
::BlockHeader> + ChainSpecProvider + StateProviderFactory + 'static, E: ConfigureEvm + 'static, + T: PayloadTypes, { /// Validates the given block and a [`BidTrace`] against it. pub async fn validate_message_against_block( @@ -132,10 +143,10 @@ where if self.disallow.contains(sender) { return Err(ValidationApiError::Blacklist(*sender)) } - if let Some(to) = tx.to() { - if self.disallow.contains(&to) { - return Err(ValidationApiError::Blacklist(to)) - } + if let Some(to) = tx.to() && + self.disallow.contains(&to) + { + return Err(ValidationApiError::Blacklist(to)) } } } @@ -174,7 +185,7 @@ where let output = executor.execute_with_state_closure(&block, |state| { if !self.disallow.is_empty() { // Check whether the submission interacted with any blacklisted account by scanning - // the `State`'s cache that records everything read form database during execution. + // the `State`'s cache that records everything read from database during execution. for account in state.cache.accounts.keys() { if self.disallow.contains(account) { accessed_blacklisted = Some(*account); @@ -229,7 +240,7 @@ where expected: header.gas_limit(), })) } else if header.gas_used() != message.gas_used { - return Err(ValidationApiError::GasUsedMismatch(GotExpected { + Err(ValidationApiError::GasUsedMismatch(GotExpected { got: message.gas_used, expected: header.gas_used(), })) @@ -297,7 +308,7 @@ where } } - if balance_after >= balance_before + message.value { + if balance_after >= balance_before.saturating_add(message.value) { return Ok(()) } @@ -323,10 +334,10 @@ where return Err(ValidationApiError::ProposerPayment) } - if let Some(block_base_fee) = block.header().base_fee_per_gas() { - if tx.effective_tip_per_gas(block_base_fee).unwrap_or_default() != 0 { - return Err(ValidationApiError::ProposerPayment) - } + if let Some(block_base_fee) = block.header().base_fee_per_gas() && + tx.effective_tip_per_gas(block_base_fee).unwrap_or_default() != 0 + { + return Err(ValidationApiError::ProposerPayment) } Ok(()) @@ -355,6 +366,24 @@ where Ok(versioned_hashes) } + /// Validates the given [`BlobsBundleV1`] and returns versioned hashes for blobs. + pub fn validate_blobs_bundle_v2( + &self, + blobs_bundle: BlobsBundleV2, + ) -> Result, ValidationApiError> { + let versioned_hashes = blobs_bundle + .commitments + .iter() + .map(|c| kzg_to_versioned_hash(c.as_slice())) + .collect::>(); + + blobs_bundle + .try_into_sidecar() + .map_err(|_| ValidationApiError::InvalidBlobsBundle)? + .validate(&versioned_hashes, EnvKzgSettings::default().get())?; + + Ok(versioned_hashes) + } /// Core logic for validating the builder submission v3 async fn validate_builder_submission_v3( @@ -404,10 +433,50 @@ where ) .await } + + /// Core logic for validating the builder submission v5 + async fn validate_builder_submission_v5( + &self, + request: BuilderBlockValidationRequestV5, + ) -> Result<(), ValidationApiError> { + let block = self.payload_validator.ensure_well_formed_payload(ExecutionData { + payload: ExecutionPayload::V3(request.request.execution_payload), + sidecar: ExecutionPayloadSidecar::v4( + CancunPayloadFields { + parent_beacon_block_root: request.parent_beacon_block_root, + versioned_hashes: self + .validate_blobs_bundle_v2(request.request.blobs_bundle)?, + }, + PraguePayloadFields { + requests: RequestsOrHash::Requests( + request.request.execution_requests.to_requests(), + ), + }, + ), + })?; + + // Check block size as per EIP-7934 (only applies when Osaka hardfork is active) + let chain_spec = self.provider.chain_spec(); + if chain_spec.is_osaka_active_at_timestamp(block.timestamp()) && + block.rlp_length() > MAX_RLP_BLOCK_SIZE + { + return Err(ValidationApiError::Consensus(ConsensusError::BlockTooLarge { + rlp_length: block.rlp_length(), + max_rlp_length: MAX_RLP_BLOCK_SIZE, + })); + } + + self.validate_message_against_block( + block, + request.request.message, + request.registered_gas_limit, + ) + .await + } } #[async_trait] -impl BlockSubmissionValidationApiServer for ValidationApi +impl BlockSubmissionValidationApiServer for ValidationApi where Provider: BlockReaderIdExt
::BlockHeader> + ChainSpecProvider @@ -415,6 +484,7 @@ where + Clone + 'static, E: ConfigureEvm + 'static, + T: PayloadTypes, { async fn validate_builder_submission_v1( &self, @@ -467,20 +537,34 @@ where rx.await.map_err(|_| internal_rpc_err("Internal blocking task error"))? } + + /// Validates a block submitted to the relay + async fn validate_builder_submission_v5( + &self, + request: BuilderBlockValidationRequestV5, + ) -> RpcResult<()> { + let this = self.clone(); + let (tx, rx) = oneshot::channel(); + + self.task_spawner.spawn_blocking(Box::pin(async move { + let result = Self::validate_builder_submission_v5(&this, request) + .await + .map_err(ErrorObject::from); + let _ = tx.send(result); + })); + + rx.await.map_err(|_| internal_rpc_err("Internal blocking task error"))? + } } -pub struct ValidationApiInner { +pub struct ValidationApiInner { /// The provider that can interact with the chain. provider: Provider, /// Consensus implementation. consensus: Arc>, /// Execution payload validator. - payload_validator: Arc< - dyn PayloadValidator< - Block = ::Block, - ExecutionData = ExecutionData, - >, - >, + payload_validator: + Arc::Block>>, /// Block executor factory. evm_config: E, /// Set of disallowed addresses @@ -498,7 +582,23 @@ pub struct ValidationApiInner { metrics: ValidationMetrics, } -impl fmt::Debug for ValidationApiInner { +/// Calculates a deterministic hash of the blocklist for change detection. +/// +/// This function sorts addresses to ensure deterministic output regardless of +/// insertion order, then computes a SHA256 hash of the concatenated addresses. +fn hash_disallow_list(disallow: &HashSet
) -> String { + let mut sorted: Vec<_> = disallow.iter().collect(); + sorted.sort(); // sort for deterministic hashing + + let mut hasher = Sha256::new(); + for addr in sorted { + hasher.update(addr.as_slice()); + } + + format!("{:x}", hasher.finalize()) +} + +impl fmt::Debug for ValidationApiInner { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("ValidationApiInner").finish_non_exhaustive() } @@ -597,3 +697,63 @@ pub(crate) struct ValidationMetrics { /// The number of entries configured in the builder validation disallow list. pub(crate) disallow_size: Gauge, } + +#[cfg(test)] +mod tests { + use super::hash_disallow_list; + use revm_primitives::Address; + use std::collections::HashSet; + + #[test] + fn test_hash_disallow_list_deterministic() { + let mut addresses = HashSet::new(); + addresses.insert(Address::from([1u8; 20])); + addresses.insert(Address::from([2u8; 20])); + + let hash1 = hash_disallow_list(&addresses); + let hash2 = hash_disallow_list(&addresses); + + assert_eq!(hash1, hash2); + } + + #[test] + fn test_hash_disallow_list_different_content() { + let mut addresses1 = HashSet::new(); + addresses1.insert(Address::from([1u8; 20])); + + let mut addresses2 = HashSet::new(); + addresses2.insert(Address::from([2u8; 20])); + + let hash1 = hash_disallow_list(&addresses1); + let hash2 = hash_disallow_list(&addresses2); + + assert_ne!(hash1, hash2); + } + + #[test] + fn test_hash_disallow_list_order_independent() { + let mut addresses1 = HashSet::new(); + addresses1.insert(Address::from([1u8; 20])); + addresses1.insert(Address::from([2u8; 20])); + + let mut addresses2 = HashSet::new(); + addresses2.insert(Address::from([2u8; 20])); // Different insertion order + addresses2.insert(Address::from([1u8; 20])); + + let hash1 = hash_disallow_list(&addresses1); + let hash2 = hash_disallow_list(&addresses2); + + assert_eq!(hash1, hash2); + } + + #[test] + //ensures parity with rbuilder hashing https://github.com/flashbots/rbuilder/blob/962c8444cdd490a216beda22c7eec164db9fc3ac/crates/rbuilder/src/live_builder/block_list_provider.rs#L248 + fn test_disallow_list_hash_rbuilder_parity() { + let json = r#"["0x05E0b5B40B7b66098C2161A5EE11C5740A3A7C45","0x01e2919679362dFBC9ee1644Ba9C6da6D6245BB1","0x03893a7c7463AE47D46bc7f091665f1893656003","0x04DBA1194ee10112fE6C3207C0687DEf0e78baCf"]"#; + let blocklist: Vec
= serde_json::from_str(json).unwrap(); + let blocklist: HashSet
= blocklist.into_iter().collect(); + let expected_hash = "ee14e9d115e182f61871a5a385ab2f32ecf434f3b17bdbacc71044810d89e608"; + let hash = hash_disallow_list(&blocklist); + assert_eq!(expected_hash, hash); + } +} diff --git a/crates/stages/api/Cargo.toml b/crates/stages/api/Cargo.toml index 515c2712466..c8eb81289d0 100644 --- a/crates/stages/api/Cargo.toml +++ b/crates/stages/api/Cargo.toml @@ -44,6 +44,7 @@ auto_impl.workspace = true [dev-dependencies] assert_matches.workspace = true reth-provider = { workspace = true, features = ["test-utils"] } +tokio = { workspace = true, features = ["sync", "rt-multi-thread"] } tokio-stream.workspace = true reth-testing-utils.workspace = true @@ -51,7 +52,7 @@ reth-testing-utils.workspace = true test-utils = [ "reth-consensus/test-utils", "reth-network-p2p/test-utils", - "reth-primitives-traits/test-utils", "reth-provider/test-utils", "reth-stages-types/test-utils", + "reth-primitives-traits/test-utils", ] diff --git a/crates/stages/api/src/error.rs b/crates/stages/api/src/error.rs index 92b1d974542..b4bbf390e22 100644 --- a/crates/stages/api/src/error.rs +++ b/crates/stages/api/src/error.rs @@ -4,7 +4,7 @@ use reth_consensus::ConsensusError; use reth_errors::{BlockExecutionError, DatabaseError, RethError}; use reth_network_p2p::error::DownloadError; use reth_provider::ProviderError; -use reth_prune::{PruneSegment, PruneSegmentError, PrunerError}; +use reth_prune::{PruneSegment, PruneSegmentError, PrunerError, UnwindTargetPrunedError}; use reth_static_file_types::StaticFileSegment; use thiserror::Error; use tokio::sync::broadcast::error::SendError; @@ -163,4 +163,7 @@ pub enum PipelineError { /// The pipeline encountered an unwind when `fail_on_unwind` was set to `true`. #[error("unexpected unwind")] UnexpectedUnwind, + /// Unwind target pruned error. + #[error(transparent)] + UnwindTargetPruned(#[from] UnwindTargetPrunedError), } diff --git a/crates/stages/api/src/lib.rs b/crates/stages/api/src/lib.rs index ec01876c995..1fb6e2be743 100644 --- a/crates/stages/api/src/lib.rs +++ b/crates/stages/api/src/lib.rs @@ -9,7 +9,7 @@ html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256", issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(test), warn(unused_crate_dependencies))] mod error; diff --git a/crates/stages/api/src/metrics/listener.rs b/crates/stages/api/src/metrics/listener.rs index aba001a92f1..2ae367eb364 100644 --- a/crates/stages/api/src/metrics/listener.rs +++ b/crates/stages/api/src/metrics/listener.rs @@ -4,6 +4,7 @@ use std::{ future::Future, pin::Pin, task::{ready, Context, Poll}, + time::Duration, }; use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender}; use tracing::trace; @@ -28,6 +29,8 @@ pub enum MetricEvent { /// Maximum known block number reachable by this stage. /// If specified, `entities_total` metric is updated. max_block_number: Option, + /// The duration of stage iteration including database commit. + elapsed: Duration, }, } @@ -49,20 +52,12 @@ impl MetricsListener { trace!(target: "sync::metrics", ?event, "Metric event received"); match event { MetricEvent::SyncHeight { height } => { - for stage_id in StageId::ALL { - self.handle_event(MetricEvent::StageCheckpoint { - stage_id, - checkpoint: StageCheckpoint { - block_number: height, - stage_checkpoint: None, - }, - max_block_number: Some(height), - }); - } + self.update_all_stages_height(height); } - MetricEvent::StageCheckpoint { stage_id, checkpoint, max_block_number } => { + MetricEvent::StageCheckpoint { stage_id, checkpoint, max_block_number, elapsed } => { let stage_metrics = self.sync_metrics.get_stage_metrics(stage_id); + stage_metrics.total_elapsed.increment(elapsed.as_secs_f64()); stage_metrics.checkpoint.set(checkpoint.block_number as f64); let (processed, total) = match checkpoint.entities() { @@ -78,6 +73,17 @@ impl MetricsListener { } } } + + /// Updates all stage checkpoints to the given height efficiently. + fn update_all_stages_height(&mut self, height: BlockNumber) { + for stage_id in StageId::ALL { + let stage_metrics = self.sync_metrics.get_stage_metrics(stage_id); + let height_f64 = height as f64; + stage_metrics.checkpoint.set(height_f64); + stage_metrics.entities_processed.set(height_f64); + stage_metrics.entities_total.set(height_f64); + } + } } impl Future for MetricsListener { diff --git a/crates/stages/api/src/metrics/sync_metrics.rs b/crates/stages/api/src/metrics/sync_metrics.rs index b89d7b8822e..754a2b22fcc 100644 --- a/crates/stages/api/src/metrics/sync_metrics.rs +++ b/crates/stages/api/src/metrics/sync_metrics.rs @@ -4,6 +4,7 @@ use std::collections::HashMap; #[derive(Debug, Default)] pub(crate) struct SyncMetrics { + /// Stage metrics by stage. pub(crate) stages: HashMap, } @@ -26,4 +27,6 @@ pub(crate) struct StageMetrics { pub(crate) entities_processed: Gauge, /// The number of total entities of the last commit for a stage, if applicable. pub(crate) entities_total: Gauge, + /// The number of seconds spent executing the stage and committing the data. + pub(crate) total_elapsed: Gauge, } diff --git a/crates/stages/api/src/pipeline/builder.rs b/crates/stages/api/src/pipeline/builder.rs index 45bdc2d8942..56b895cac7d 100644 --- a/crates/stages/api/src/pipeline/builder.rs +++ b/crates/stages/api/src/pipeline/builder.rs @@ -90,6 +90,8 @@ impl PipelineBuilder { progress: Default::default(), metrics_tx, fail_on_unwind, + last_detached_head_unwind_target: None, + detached_head_attempts: 0, } } } diff --git a/crates/stages/api/src/pipeline/mod.rs b/crates/stages/api/src/pipeline/mod.rs index 37152967a62..e8542c36da6 100644 --- a/crates/stages/api/src/pipeline/mod.rs +++ b/crates/stages/api/src/pipeline/mod.rs @@ -7,14 +7,17 @@ pub use event::*; use futures_util::Future; use reth_primitives_traits::constants::BEACON_CONSENSUS_REORG_UNWIND_DEPTH; use reth_provider::{ - providers::ProviderNodeTypes, writer::UnifiedStorageWriter, ChainStateBlockReader, - ChainStateBlockWriter, DatabaseProviderFactory, ProviderFactory, StageCheckpointReader, - StageCheckpointWriter, + providers::ProviderNodeTypes, BlockHashReader, BlockNumReader, ChainStateBlockReader, + ChainStateBlockWriter, DBProvider, DatabaseProviderFactory, ProviderFactory, + PruneCheckpointReader, StageCheckpointReader, StageCheckpointWriter, }; use reth_prune::PrunerBuilder; use reth_static_file::StaticFileProducer; use reth_tokio_util::{EventSender, EventStream}; -use std::pin::Pin; +use std::{ + pin::Pin, + time::{Duration, Instant}, +}; use tokio::sync::watch; use tracing::*; @@ -83,6 +86,12 @@ pub struct Pipeline { /// Whether an unwind should fail the syncing process. Should only be set when downloading /// blocks from trusted sources and expecting them to be valid. fail_on_unwind: bool, + /// Block that was chosen as a target of the last unwind triggered by + /// [`StageError::DetachedHead`] error. + last_detached_head_unwind_target: Option, + /// Number of consecutive unwind attempts due to [`StageError::DetachedHead`] for the current + /// fork. + detached_head_attempts: u64, } impl Pipeline { @@ -110,6 +119,14 @@ impl Pipeline { pub fn events(&self) -> EventStream { self.event_sender.new_listener() } + + /// Get a mutable reference to a stage by index. + pub fn stage( + &mut self, + idx: usize, + ) -> &mut dyn Stage< as DatabaseProviderFactory>::ProviderRW> { + &mut self.stages[idx] + } } impl Pipeline { @@ -124,6 +141,7 @@ impl Pipeline { stage_id, checkpoint: provider.get_stage_checkpoint(stage_id)?.unwrap_or_default(), max_block_number: None, + elapsed: Duration::default(), }); } Ok(()) @@ -280,6 +298,16 @@ impl Pipeline { to: BlockNumber, bad_block: Option, ) -> Result<(), PipelineError> { + // Add validation before starting unwind + let provider = self.provider_factory.provider()?; + let latest_block = provider.last_block_number()?; + + // Get the actual pruning configuration + let prune_modes = provider.prune_modes_ref(); + + let checkpoints = provider.get_prune_checkpoints()?; + prune_modes.ensure_unwind_target_unpruned(latest_block, to, &checkpoints)?; + // Unwind stages in reverse order of execution let unwind_pipeline = self.stages.iter_mut().rev(); @@ -315,6 +343,7 @@ impl Pipeline { "Starting unwind" ); while checkpoint.block_number > to { + let unwind_started_at = Instant::now(); let input = UnwindInput { checkpoint, unwind_to: to, bad_block }; self.event_sender.notify(PipelineEvent::Unwind { stage_id, input }); @@ -330,6 +359,13 @@ impl Pipeline { done = checkpoint.block_number == to, "Stage unwound" ); + + provider_rw.save_stage_checkpoint(stage_id, checkpoint)?; + + // Notify event listeners and update metrics. + self.event_sender + .notify(PipelineEvent::Unwound { stage_id, result: unwind_output }); + if let Some(metrics_tx) = &mut self.metrics_tx { let _ = metrics_tx.send(MetricEvent::StageCheckpoint { stage_id, @@ -337,12 +373,9 @@ impl Pipeline { // We assume it was set in the previous execute iteration, so it // doesn't change when we unwind. max_block_number: None, + elapsed: unwind_started_at.elapsed(), }); } - provider_rw.save_stage_checkpoint(stage_id, checkpoint)?; - - self.event_sender - .notify(PipelineEvent::Unwound { stage_id, result: unwind_output }); // update finalized block if needed let last_saved_finalized_block_number = @@ -358,7 +391,7 @@ impl Pipeline { ))?; } - UnifiedStorageWriter::commit_unwind(provider_rw)?; + provider_rw.commit()?; stage.post_unwind_commit()?; @@ -383,8 +416,7 @@ impl Pipeline { ) -> Result { let total_stages = self.stages.len(); - let stage = &mut self.stages[stage_index]; - let stage_id = stage.id(); + let stage_id = self.stage(stage_index).id(); let mut made_progress = false; let target = self.max_block.or(previous_stage); @@ -422,15 +454,15 @@ impl Pipeline { target, }); - if let Err(err) = stage.execute_ready(exec_input).await { + if let Err(err) = self.stage(stage_index).execute_ready(exec_input).await { self.event_sender.notify(PipelineEvent::Error { stage_id }); - - match on_stage_error(&self.provider_factory, stage_id, prev_checkpoint, err)? { + match self.on_stage_error(stage_id, prev_checkpoint, err)? { Some(ctrl) => return Ok(ctrl), None => continue, }; } + let stage_started_at = Instant::now(); let provider_rw = self.provider_factory.database_provider_rw()?; self.event_sender.notify(PipelineEvent::Run { @@ -443,20 +475,18 @@ impl Pipeline { target, }); - match stage.execute(&provider_rw, exec_input) { + match self.stage(stage_index).execute(&provider_rw, exec_input) { Ok(out @ ExecOutput { checkpoint, done }) => { - made_progress |= - checkpoint.block_number != prev_checkpoint.unwrap_or_default().block_number; - - if let Some(metrics_tx) = &mut self.metrics_tx { - let _ = metrics_tx.send(MetricEvent::StageCheckpoint { - stage_id, - checkpoint, - max_block_number: target, - }); - } + // Update stage checkpoint. provider_rw.save_stage_checkpoint(stage_id, checkpoint)?; + // Commit processed data to the database. + provider_rw.commit()?; + + // Invoke stage post commit hook. + self.stage(stage_index).post_execute_commit()?; + + // Notify event listeners and update metrics. self.event_sender.notify(PipelineEvent::Ran { pipeline_stages_progress: PipelineStagesProgress { current: stage_index + 1, @@ -465,13 +495,19 @@ impl Pipeline { stage_id, result: out.clone(), }); + if let Some(metrics_tx) = &mut self.metrics_tx { + let _ = metrics_tx.send(MetricEvent::StageCheckpoint { + stage_id, + checkpoint, + max_block_number: target, + elapsed: stage_started_at.elapsed(), + }); + } - UnifiedStorageWriter::commit(provider_rw)?; - - stage.post_execute_commit()?; - + let block_number = checkpoint.block_number; + let prev_block_number = prev_checkpoint.unwrap_or_default().block_number; + made_progress |= block_number != prev_block_number; if done { - let block_number = checkpoint.block_number; return Ok(if made_progress { ControlFlow::Continue { block_number } } else { @@ -483,101 +519,125 @@ impl Pipeline { drop(provider_rw); self.event_sender.notify(PipelineEvent::Error { stage_id }); - if let Some(ctrl) = - on_stage_error(&self.provider_factory, stage_id, prev_checkpoint, err)? - { + if let Some(ctrl) = self.on_stage_error(stage_id, prev_checkpoint, err)? { return Ok(ctrl) } } } } } -} -fn on_stage_error( - factory: &ProviderFactory, - stage_id: StageId, - prev_checkpoint: Option, - err: StageError, -) -> Result, PipelineError> { - if let StageError::DetachedHead { local_head, header, error } = err { - warn!(target: "sync::pipeline", stage = %stage_id, ?local_head, ?header, %error, "Stage encountered detached head"); - - // We unwind because of a detached head. - let unwind_to = - local_head.block.number.saturating_sub(BEACON_CONSENSUS_REORG_UNWIND_DEPTH).max(1); - Ok(Some(ControlFlow::Unwind { target: unwind_to, bad_block: local_head })) - } else if let StageError::Block { block, error } = err { - match error { - BlockErrorKind::Validation(validation_error) => { - error!( - target: "sync::pipeline", - stage = %stage_id, - bad_block = %block.block.number, - "Stage encountered a validation error: {validation_error}" - ); - - // FIXME: When handling errors, we do not commit the database transaction. This - // leads to the Merkle stage not clearing its checkpoint, and restarting from an - // invalid place. - let provider_rw = factory.database_provider_rw()?; - provider_rw.save_stage_checkpoint_progress(StageId::MerkleExecute, vec![])?; - provider_rw.save_stage_checkpoint( - StageId::MerkleExecute, - prev_checkpoint.unwrap_or_default(), - )?; - - UnifiedStorageWriter::commit(provider_rw)?; - - // We unwind because of a validation error. If the unwind itself - // fails, we bail entirely, - // otherwise we restart the execution loop from the - // beginning. - Ok(Some(ControlFlow::Unwind { - target: prev_checkpoint.unwrap_or_default().block_number, - bad_block: block, - })) + fn on_stage_error( + &mut self, + stage_id: StageId, + prev_checkpoint: Option, + err: StageError, + ) -> Result, PipelineError> { + if let StageError::DetachedHead { local_head, header, error } = err { + warn!(target: "sync::pipeline", stage = %stage_id, ?local_head, ?header, %error, "Stage encountered detached head"); + + if let Some(last_detached_head_unwind_target) = self.last_detached_head_unwind_target { + if local_head.block.hash == last_detached_head_unwind_target && + header.block.number == local_head.block.number + 1 + { + self.detached_head_attempts += 1; + } else { + self.detached_head_attempts = 1; + } + } else { + self.detached_head_attempts = 1; } - BlockErrorKind::Execution(execution_error) => { - error!( - target: "sync::pipeline", - stage = %stage_id, - bad_block = %block.block.number, - "Stage encountered an execution error: {execution_error}" - ); - // We unwind because of an execution error. If the unwind itself - // fails, we bail entirely, - // otherwise we restart - // the execution loop from the beginning. - Ok(Some(ControlFlow::Unwind { - target: prev_checkpoint.unwrap_or_default().block_number, - bad_block: block, - })) + // We unwind because of a detached head. + let unwind_to = local_head + .block + .number + .saturating_sub( + BEACON_CONSENSUS_REORG_UNWIND_DEPTH.saturating_mul(self.detached_head_attempts), + ) + .max(1); + + self.last_detached_head_unwind_target = self.provider_factory.block_hash(unwind_to)?; + Ok(Some(ControlFlow::Unwind { target: unwind_to, bad_block: local_head })) + } else if let StageError::Block { block, error } = err { + match error { + BlockErrorKind::Validation(validation_error) => { + error!( + target: "sync::pipeline", + stage = %stage_id, + bad_block = %block.block.number, + "Stage encountered a validation error: {validation_error}" + ); + + // FIXME: When handling errors, we do not commit the database transaction. This + // leads to the Merkle stage not clearing its checkpoint, and restarting from an + // invalid place. + // Only reset MerkleExecute checkpoint if MerkleExecute itself failed + if stage_id == StageId::MerkleExecute { + let provider_rw = self.provider_factory.database_provider_rw()?; + provider_rw + .save_stage_checkpoint_progress(StageId::MerkleExecute, vec![])?; + provider_rw.save_stage_checkpoint( + StageId::MerkleExecute, + prev_checkpoint.unwrap_or_default(), + )?; + + provider_rw.commit()?; + } + + // We unwind because of a validation error. If the unwind itself + // fails, we bail entirely, + // otherwise we restart the execution loop from the + // beginning. + Ok(Some(ControlFlow::Unwind { + target: prev_checkpoint.unwrap_or_default().block_number, + bad_block: block, + })) + } + BlockErrorKind::Execution(execution_error) => { + error!( + target: "sync::pipeline", + stage = %stage_id, + bad_block = %block.block.number, + "Stage encountered an execution error: {execution_error}" + ); + + // We unwind because of an execution error. If the unwind itself + // fails, we bail entirely, + // otherwise we restart + // the execution loop from the beginning. + Ok(Some(ControlFlow::Unwind { + target: prev_checkpoint.unwrap_or_default().block_number, + bad_block: block, + })) + } } - } - } else if let StageError::MissingStaticFileData { block, segment } = err { - error!( - target: "sync::pipeline", - stage = %stage_id, - bad_block = %block.block.number, - segment = %segment, - "Stage is missing static file data." - ); + } else if let StageError::MissingStaticFileData { block, segment } = err { + error!( + target: "sync::pipeline", + stage = %stage_id, + bad_block = %block.block.number, + segment = %segment, + "Stage is missing static file data." + ); - Ok(Some(ControlFlow::Unwind { target: block.block.number - 1, bad_block: block })) - } else if err.is_fatal() { - error!(target: "sync::pipeline", stage = %stage_id, "Stage encountered a fatal error: {err}"); - Err(err.into()) - } else { - // On other errors we assume they are recoverable if we discard the - // transaction and run the stage again. - warn!( - target: "sync::pipeline", - stage = %stage_id, - "Stage encountered a non-fatal error: {err}. Retrying..." - ); - Ok(None) + Ok(Some(ControlFlow::Unwind { + target: block.block.number.saturating_sub(1), + bad_block: block, + })) + } else if err.is_fatal() { + error!(target: "sync::pipeline", stage = %stage_id, "Stage encountered a fatal error: {err}"); + Err(err.into()) + } else { + // On other errors we assume they are recoverable if we discard the + // transaction and run the stage again. + warn!( + target: "sync::pipeline", + stage = %stage_id, + "Stage encountered a non-fatal error: {err}. Retrying..." + ); + Ok(None) + } } } diff --git a/crates/stages/api/src/pipeline/set.rs b/crates/stages/api/src/pipeline/set.rs index 8aea87ba035..c39dafae99f 100644 --- a/crates/stages/api/src/pipeline/set.rs +++ b/crates/stages/api/src/pipeline/set.rs @@ -73,16 +73,15 @@ impl StageSetBuilder { fn upsert_stage_state(&mut self, stage: Box>, added_at_index: usize) { let stage_id = stage.id(); - if self.stages.insert(stage.id(), StageEntry { stage, enabled: true }).is_some() { - if let Some(to_remove) = self + if self.stages.insert(stage.id(), StageEntry { stage, enabled: true }).is_some() && + let Some(to_remove) = self .order .iter() .enumerate() .find(|(i, id)| *i != added_at_index && **id == stage_id) .map(|(i, _)| i) - { - self.order.remove(to_remove); - } + { + self.order.remove(to_remove); } } @@ -264,10 +263,10 @@ impl StageSetBuilder { pub fn build(mut self) -> Vec>> { let mut stages = Vec::new(); for id in &self.order { - if let Some(entry) = self.stages.remove(id) { - if entry.enabled { - stages.push(entry.stage); - } + if let Some(entry) = self.stages.remove(id) && + entry.enabled + { + stages.push(entry.stage); } } stages diff --git a/crates/stages/api/src/stage.rs b/crates/stages/api/src/stage.rs index 368269782a2..9fc3038c69c 100644 --- a/crates/stages/api/src/stage.rs +++ b/crates/stages/api/src/stage.rs @@ -111,7 +111,7 @@ impl ExecInput { // body. let end_block_body = provider .block_body_indices(end_block_number)? - .ok_or(ProviderError::BlockBodyIndicesNotFound(target_block))?; + .ok_or(ProviderError::BlockBodyIndicesNotFound(end_block_number))?; (end_block_number, false, end_block_body.next_tx_num()) }; @@ -165,6 +165,11 @@ pub struct ExecOutput { } impl ExecOutput { + /// Mark the stage as not done, checkpointing at the given place. + pub const fn in_progress(checkpoint: StageCheckpoint) -> Self { + Self { checkpoint, done: false } + } + /// Mark the stage as done, checkpointing at the given place. pub const fn done(checkpoint: StageCheckpoint) -> Self { Self { checkpoint, done: true } @@ -184,7 +189,7 @@ pub struct UnwindOutput { /// transactions, and persist their results to a database. /// /// Stages must have a unique [ID][StageId] and implement a way to "roll forwards" -/// ([Stage::execute]) and a way to "roll back" ([Stage::unwind]). +/// ([`Stage::execute`]) and a way to "roll back" ([`Stage::unwind`]). /// /// Stages are executed as part of a pipeline where they are executed serially. /// @@ -271,4 +276,4 @@ pub trait StageExt: Stage { } } -impl> StageExt for S {} +impl + ?Sized> StageExt for S {} diff --git a/crates/stages/stages/Cargo.toml b/crates/stages/stages/Cargo.toml index 600cd9f9905..32114c58e1b 100644 --- a/crates/stages/stages/Cargo.toml +++ b/crates/stages/stages/Cargo.toml @@ -21,6 +21,9 @@ reth-db.workspace = true reth-db-api.workspace = true reth-etl.workspace = true reth-evm = { workspace = true, features = ["metrics"] } +reth-era-downloader.workspace = true +reth-era-utils.workspace = true +reth-era.workspace = true reth-exex.workspace = true reth-fs-util.workspace = true reth-network-p2p.workspace = true @@ -57,9 +60,8 @@ rayon.workspace = true num-traits.workspace = true tempfile = { workspace = true, optional = true } bincode.workspace = true -blake3.workspace = true reqwest = { workspace = true, default-features = false, features = ["rustls-tls-native-roots", "blocking"] } -serde = { workspace = true, features = ["derive"] } +eyre.workspace = true [dev-dependencies] # reth @@ -68,22 +70,19 @@ reth-db = { workspace = true, features = ["test-utils", "mdbx"] } reth-ethereum-primitives = { workspace = true, features = ["test-utils"] } reth-ethereum-consensus.workspace = true reth-evm-ethereum.workspace = true -reth-execution-errors.workspace = true reth-consensus = { workspace = true, features = ["test-utils"] } reth-network-p2p = { workspace = true, features = ["test-utils"] } reth-downloaders.workspace = true -reth-revm.workspace = true reth-static-file.workspace = true reth-stages-api = { workspace = true, features = ["test-utils"] } reth-testing-utils.workspace = true reth-trie = { workspace = true, features = ["test-utils"] } reth-provider = { workspace = true, features = ["test-utils"] } reth-network-peers.workspace = true -reth-tracing.workspace = true alloy-primitives = { workspace = true, features = ["getrandom", "rand"] } alloy-rlp.workspace = true -itertools.workspace = true + tokio = { workspace = true, features = ["rt", "sync", "macros"] } assert_matches.workspace = true rand.workspace = true diff --git a/crates/stages/stages/benches/README.md b/crates/stages/stages/benches/README.md index 7c482c59c60..c3d3268e318 100644 --- a/crates/stages/stages/benches/README.md +++ b/crates/stages/stages/benches/README.md @@ -13,10 +13,10 @@ It will generate a flamegraph report without running any criterion analysis. ``` cargo bench --package reth-stages --bench criterion --features test-utils -- --profile-time=2 ``` -Flamegraph reports can be find at `target/criterion/Stages/$STAGE_LABEL/profile/flamegraph.svg` +Flamegraph reports can be found at `target/criterion/Stages/$STAGE_LABEL/profile/flamegraph.svg` ## External DB support To choose an external DB, just pass an environment variable to the `cargo bench` command. -* Account Hashing Stage: `ACCOUNT_HASHING_DB=` \ No newline at end of file +* Account Hashing Stage: `ACCOUNT_HASHING_DB=` diff --git a/crates/stages/stages/benches/criterion.rs b/crates/stages/stages/benches/criterion.rs index 08f700789ab..655b990f254 100644 --- a/crates/stages/stages/benches/criterion.rs +++ b/crates/stages/stages/benches/criterion.rs @@ -5,7 +5,9 @@ use alloy_primitives::BlockNumber; use criterion::{criterion_main, measurement::WallTime, BenchmarkGroup, Criterion}; use reth_config::config::{EtlConfig, TransactionLookupConfig}; use reth_db::{test_utils::TempDatabase, Database, DatabaseEnv}; -use reth_provider::{test_utils::MockNodeTypesWithDB, DatabaseProvider, DatabaseProviderFactory}; +use reth_provider::{ + test_utils::MockNodeTypesWithDB, DBProvider, DatabaseProvider, DatabaseProviderFactory, +}; use reth_stages::{ stages::{MerkleStage, SenderRecoveryStage, TransactionLookupStage}, test_utils::TestStageDB, @@ -113,7 +115,7 @@ fn merkle(c: &mut Criterion, runtime: &Runtime) { let db = setup::txs_testdata(DEFAULT_NUM_BLOCKS); - let stage = MerkleStage::Both { clean_threshold: u64::MAX }; + let stage = MerkleStage::Both { rebuild_threshold: u64::MAX, incremental_threshold: u64::MAX }; measure_stage( runtime, &mut group, @@ -124,7 +126,7 @@ fn merkle(c: &mut Criterion, runtime: &Runtime) { "Merkle-incremental".to_string(), ); - let stage = MerkleStage::Both { clean_threshold: 0 }; + let stage = MerkleStage::Both { rebuild_threshold: 0, incremental_threshold: 0 }; measure_stage( runtime, &mut group, diff --git a/crates/stages/stages/benches/setup/mod.rs b/crates/stages/stages/benches/setup/mod.rs index 074e239da75..b6010dd6f39 100644 --- a/crates/stages/stages/benches/setup/mod.rs +++ b/crates/stages/stages/benches/setup/mod.rs @@ -1,15 +1,11 @@ #![expect(unreachable_pub)] -use alloy_primitives::{Address, B256, U256}; +use alloy_primitives::{Address, B256}; use itertools::concat; use reth_db::{test_utils::TempDatabase, Database, DatabaseEnv}; -use reth_db_api::{ - cursor::DbCursorRO, - tables, - transaction::{DbTx, DbTxMut}, -}; use reth_primitives_traits::{Account, SealedBlock, SealedHeader}; use reth_provider::{ - test_utils::MockNodeTypesWithDB, DatabaseProvider, DatabaseProviderFactory, TrieWriter, + test_utils::MockNodeTypesWithDB, DBProvider, DatabaseProvider, DatabaseProviderFactory, + TrieWriter, }; use reth_stages::{ stages::{AccountHashingStage, StorageHashingStage}, @@ -161,9 +157,10 @@ pub(crate) fn txs_testdata(num_blocks: u64) -> TestStageDB { let offset = transitions.len() as u64; - let provider_rw = db.factory.provider_rw().unwrap(); db.insert_changesets(transitions, None).unwrap(); - provider_rw.write_trie_updates(&updates).unwrap(); + + let provider_rw = db.factory.provider_rw().unwrap(); + provider_rw.write_trie_updates(updates).unwrap(); provider_rw.commit().unwrap(); let (transitions, final_state) = random_changeset_range( @@ -196,13 +193,6 @@ pub(crate) fn txs_testdata(num_blocks: u64) -> TestStageDB { ); db.insert_blocks(blocks.iter(), StorageKind::Static).unwrap(); - - // initialize TD - db.commit(|tx| { - let (head, _) = tx.cursor_read::()?.first()?.unwrap_or_default(); - Ok(tx.put::(head, U256::from(0).into())?) - }) - .unwrap(); } db diff --git a/crates/stages/stages/src/lib.rs b/crates/stages/stages/src/lib.rs index d0e6a2f22c0..bdd68f03a2e 100644 --- a/crates/stages/stages/src/lib.rs +++ b/crates/stages/stages/src/lib.rs @@ -16,7 +16,6 @@ //! # use reth_downloaders::bodies::bodies::BodiesDownloaderBuilder; //! # use reth_downloaders::headers::reverse_headers::ReverseHeadersDownloaderBuilder; //! # use reth_network_p2p::test_utils::{TestBodiesClient, TestHeadersClient}; -//! # use reth_evm_ethereum::execute::EthExecutorProvider; //! # use alloy_primitives::B256; //! # use reth_chainspec::MAINNET; //! # use reth_prune_types::PruneModes; @@ -47,11 +46,12 @@ //! # provider_factory.clone() //! # ); //! # let (tip_tx, tip_rx) = watch::channel(B256::default()); -//! # let executor_provider = EthExecutorProvider::mainnet(); +//! # let executor_provider = EthEvmConfig::mainnet(); //! # let static_file_producer = StaticFileProducer::new( //! # provider_factory.clone(), //! # PruneModes::default() //! # ); +//! # let era_import_source = None; //! // Create a pipeline that can fully sync //! # let pipeline = //! Pipeline::::builder() @@ -65,6 +65,7 @@ //! executor_provider, //! StageConfig::default(), //! PruneModes::default(), +//! era_import_source, //! )) //! .build(provider_factory, static_file_producer); //! ``` @@ -78,7 +79,7 @@ html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256", issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(test), warn(unused_crate_dependencies))] #[expect(missing_docs)] diff --git a/crates/stages/stages/src/sets.rs b/crates/stages/stages/src/sets.rs index 3162836444c..48a2a995809 100644 --- a/crates/stages/stages/src/sets.rs +++ b/crates/stages/stages/src/sets.rs @@ -38,9 +38,10 @@ //! ``` use crate::{ stages::{ - AccountHashingStage, BodyStage, ExecutionStage, FinishStage, HeaderStage, - IndexAccountHistoryStage, IndexStorageHistoryStage, MerkleStage, PruneSenderRecoveryStage, - PruneStage, SenderRecoveryStage, StorageHashingStage, TransactionLookupStage, + AccountHashingStage, BodyStage, EraImportSource, EraStage, ExecutionStage, FinishStage, + HeaderStage, IndexAccountHistoryStage, IndexStorageHistoryStage, MerkleChangeSets, + MerkleStage, PruneSenderRecoveryStage, PruneStage, SenderRecoveryStage, + StorageHashingStage, TransactionLookupStage, }, StageSet, StageSetBuilder, }; @@ -53,7 +54,7 @@ use reth_primitives_traits::{Block, NodePrimitives}; use reth_provider::HeaderSyncGapProvider; use reth_prune_types::PruneModes; use reth_stages_api::Stage; -use std::{ops::Not, sync::Arc}; +use std::sync::Arc; use tokio::sync::watch; /// A set containing all stages to run a fully syncing instance of reth. @@ -74,6 +75,7 @@ use tokio::sync::watch; /// - [`AccountHashingStage`] /// - [`StorageHashingStage`] /// - [`MerkleStage`] (execute) +/// - [`MerkleChangeSets`] /// - [`TransactionLookupStage`] /// - [`IndexStorageHistoryStage`] /// - [`IndexAccountHistoryStage`] @@ -115,6 +117,7 @@ where evm_config: E, stages_config: StageConfig, prune_modes: PruneModes, + era_import_source: Option, ) -> Self { Self { online: OnlineStages::new( @@ -123,6 +126,7 @@ where header_downloader, body_downloader, stages_config.clone(), + era_import_source, ), evm_config, consensus, @@ -197,6 +201,8 @@ where body_downloader: B, /// Configuration for each stage in the pipeline stages_config: StageConfig, + /// Optional source of ERA1 files. The `EraStage` does nothing unless this is specified. + era_import_source: Option, } impl OnlineStages @@ -211,8 +217,9 @@ where header_downloader: H, body_downloader: B, stages_config: StageConfig, + era_import_source: Option, ) -> Self { - Self { provider, tip, header_downloader, body_downloader, stages_config } + Self { provider, tip, header_downloader, body_downloader, stages_config, era_import_source } } } @@ -259,9 +266,18 @@ where B: BodyDownloader + 'static, HeaderStage: Stage, BodyStage: Stage, + EraStage<::Header, ::Body, EraImportSource>: + Stage, { fn builder(self) -> StageSetBuilder { - StageSetBuilder::default() + let mut builder = StageSetBuilder::default(); + + if self.era_import_source.is_some() { + builder = builder + .add_stage(EraStage::new(self.era_import_source, self.stages_config.etl.clone())); + } + + builder .add_stage(HeaderStage::new( self.provider, self.header_downloader, @@ -327,12 +343,12 @@ where stages_config: self.stages_config.clone(), prune_modes: self.prune_modes.clone(), }) - // If any prune modes are set, add the prune stage. - .add_stage_opt(self.prune_modes.is_empty().not().then(|| { - // Prune stage should be added after all hashing stages, because otherwise it will - // delete - PruneStage::new(self.prune_modes.clone(), self.stages_config.prune.commit_threshold) - })) + // Prune stage should be added after all hashing stages, because otherwise it will + // delete + .add_stage(PruneStage::new( + self.prune_modes.clone(), + self.stages_config.prune.commit_threshold, + )) } } @@ -378,6 +394,13 @@ where } /// A set containing all stages that hash account state. +/// +/// This includes: +/// - [`MerkleStage`] (unwind) +/// - [`AccountHashingStage`] +/// - [`StorageHashingStage`] +/// - [`MerkleStage`] (execute) +/// - [`MerkleChangeSets`] #[derive(Debug, Default)] #[non_exhaustive] pub struct HashingStages { @@ -390,6 +413,7 @@ where MerkleStage: Stage, AccountHashingStage: Stage, StorageHashingStage: Stage, + MerkleChangeSets: Stage, { fn builder(self) -> StageSetBuilder { StageSetBuilder::default() @@ -402,7 +426,11 @@ where self.stages_config.storage_hashing, self.stages_config.etl.clone(), )) - .add_stage(MerkleStage::new_execution(self.stages_config.merkle.clean_threshold)) + .add_stage(MerkleStage::new_execution( + self.stages_config.merkle.rebuild_threshold, + self.stages_config.merkle.incremental_threshold, + )) + .add_stage(MerkleChangeSets::new()) } } @@ -432,12 +460,12 @@ where .add_stage(IndexStorageHistoryStage::new( self.stages_config.index_storage_history, self.stages_config.etl.clone(), - self.prune_modes.account_history, + self.prune_modes.storage_history, )) .add_stage(IndexAccountHistoryStage::new( self.stages_config.index_account_history, self.stages_config.etl.clone(), - self.prune_modes.storage_history, + self.prune_modes.account_history, )) } } diff --git a/crates/stages/stages/src/stages/bodies.rs b/crates/stages/stages/src/stages/bodies.rs index 26ea3c44275..7b6090ca86b 100644 --- a/crates/stages/stages/src/stages/bodies.rs +++ b/crates/stages/stages/src/stages/bodies.rs @@ -8,7 +8,7 @@ use reth_db_api::{ use reth_network_p2p::bodies::{downloader::BodyDownloader, response::BlockResponse}; use reth_provider::{ providers::StaticFileWriter, BlockReader, BlockWriter, DBProvider, ProviderError, - StaticFileProviderFactory, StatsReader, StorageLocation, + StaticFileProviderFactory, StatsReader, }; use reth_stages_api::{ EntitiesCheckpoint, ExecInput, ExecOutput, Stage, StageCheckpoint, StageError, StageId, @@ -65,70 +65,61 @@ impl BodyStage { pub const fn new(downloader: D) -> Self { Self { downloader, buffer: None } } +} - /// Ensures that static files and database are in sync. - fn ensure_consistency( - &self, - provider: &Provider, - unwind_block: Option, - ) -> Result<(), StageError> - where - Provider: DBProvider + BlockReader + StaticFileProviderFactory, - { - // Get id for the next tx_num of zero if there are no transactions. - let next_tx_num = provider - .tx_ref() - .cursor_read::()? - .last()? - .map(|(id, _)| id + 1) - .unwrap_or_default(); - - let static_file_provider = provider.static_file_provider(); - - // Make sure Transactions static file is at the same height. If it's further, this - // input execution was interrupted previously and we need to unwind the static file. - let next_static_file_tx_num = static_file_provider - .get_highest_static_file_tx(StaticFileSegment::Transactions) - .map(|id| id + 1) - .unwrap_or_default(); - - match next_static_file_tx_num.cmp(&next_tx_num) { - // If static files are ahead, we are currently unwinding the stage or we didn't reach - // the database commit in a previous stage run. So, our only solution is to unwind the - // static files and proceed from the database expected height. - Ordering::Greater => { - let highest_db_block = - provider.tx_ref().entries::()? as u64; - let mut static_file_producer = - static_file_provider.latest_writer(StaticFileSegment::Transactions)?; - static_file_producer - .prune_transactions(next_static_file_tx_num - next_tx_num, highest_db_block)?; - // Since this is a database <-> static file inconsistency, we commit the change - // straight away. - static_file_producer.commit()?; - } - // If static files are behind, then there was some corruption or loss of files. This - // error will trigger an unwind, that will bring the database to the same height as the - // static files. - Ordering::Less => { - // If we are already in the process of unwind, this might be fine because we will - // fix the inconsistency right away. - if let Some(unwind_to) = unwind_block { - let next_tx_num_after_unwind = provider - .block_body_indices(unwind_to)? - .map(|b| b.next_tx_num()) - .ok_or(ProviderError::BlockBodyIndicesNotFound(unwind_to))?; - - // This means we need a deeper unwind. - if next_tx_num_after_unwind > next_static_file_tx_num { - return Err(missing_static_data_error( - next_static_file_tx_num.saturating_sub(1), - &static_file_provider, - provider, - StaticFileSegment::Transactions, - )?) - } - } else { +/// Ensures that static files and database are in sync. +pub(crate) fn ensure_consistency( + provider: &Provider, + unwind_block: Option, +) -> Result<(), StageError> +where + Provider: DBProvider + BlockReader + StaticFileProviderFactory, +{ + // Get id for the next tx_num of zero if there are no transactions. + let next_tx_num = provider + .tx_ref() + .cursor_read::()? + .last()? + .map(|(id, _)| id + 1) + .unwrap_or_default(); + + let static_file_provider = provider.static_file_provider(); + + // Make sure Transactions static file is at the same height. If it's further, this + // input execution was interrupted previously and we need to unwind the static file. + let next_static_file_tx_num = static_file_provider + .get_highest_static_file_tx(StaticFileSegment::Transactions) + .map(|id| id + 1) + .unwrap_or_default(); + + match next_static_file_tx_num.cmp(&next_tx_num) { + // If static files are ahead, we are currently unwinding the stage or we didn't reach + // the database commit in a previous stage run. So, our only solution is to unwind the + // static files and proceed from the database expected height. + Ordering::Greater => { + let highest_db_block = provider.tx_ref().entries::()? as u64; + let mut static_file_producer = + static_file_provider.latest_writer(StaticFileSegment::Transactions)?; + static_file_producer + .prune_transactions(next_static_file_tx_num - next_tx_num, highest_db_block)?; + // Since this is a database <-> static file inconsistency, we commit the change + // straight away. + static_file_producer.commit()?; + } + // If static files are behind, then there was some corruption or loss of files. This + // error will trigger an unwind, that will bring the database to the same height as the + // static files. + Ordering::Less => { + // If we are already in the process of unwind, this might be fine because we will + // fix the inconsistency right away. + if let Some(unwind_to) = unwind_block { + let next_tx_num_after_unwind = provider + .block_body_indices(unwind_to)? + .map(|b| b.next_tx_num()) + .ok_or(ProviderError::BlockBodyIndicesNotFound(unwind_to))?; + + // This means we need a deeper unwind. + if next_tx_num_after_unwind > next_static_file_tx_num { return Err(missing_static_data_error( next_static_file_tx_num.saturating_sub(1), &static_file_provider, @@ -136,12 +127,19 @@ impl BodyStage { StaticFileSegment::Transactions, )?) } + } else { + return Err(missing_static_data_error( + next_static_file_tx_num.saturating_sub(1), + &static_file_provider, + provider, + StaticFileSegment::Transactions, + )?) } - Ordering::Equal => {} } - - Ok(()) + Ordering::Equal => {} } + + Ok(()) } impl Stage for BodyStage @@ -194,7 +192,7 @@ where } let (from_block, to_block) = input.next_block_range().into_inner(); - self.ensure_consistency(provider, None)?; + ensure_consistency(provider, None)?; debug!(target: "sync::stages::bodies", stage_progress = from_block, target = to_block, "Commencing sync"); @@ -208,8 +206,6 @@ where .into_iter() .map(|response| (response.block_number(), response.into_body())) .collect(), - // We are writing transactions directly to static files. - StorageLocation::StaticFiles, )?; // The stage is "done" if: @@ -231,8 +227,8 @@ where ) -> Result { self.buffer.take(); - self.ensure_consistency(provider, Some(input.unwind_to))?; - provider.remove_bodies_above(input.unwind_to, StorageLocation::Both)?; + ensure_consistency(provider, Some(input.unwind_to))?; + provider.remove_bodies_above(input.unwind_to)?; Ok(UnwindOutput { checkpoint: StageCheckpoint::new(input.unwind_to) @@ -309,7 +305,7 @@ mod tests { assert!(runner.validate_execution(input, output.ok()).is_ok(), "execution validation"); } - /// Same as [partial_body_download] except the `batch_size` is not hit. + /// Same as [`partial_body_download`] except the `batch_size` is not hit. #[tokio::test] async fn full_body_download() { let (stage_progress, previous_stage) = (1, 20); @@ -348,7 +344,7 @@ mod tests { assert!(runner.validate_execution(input, output.ok()).is_ok(), "execution validation"); } - /// Same as [full_body_download] except we have made progress before + /// Same as [`full_body_download`] except we have made progress before #[tokio::test] async fn sync_from_previous_progress() { let (stage_progress, previous_stage) = (1, 21); @@ -584,7 +580,7 @@ mod tests { ..Default::default() }, ); - self.db.insert_headers_with_td(blocks.iter().map(|block| block.sealed_header()))?; + self.db.insert_headers(blocks.iter().map(|block| block.sealed_header()))?; if let Some(progress) = blocks.get(start as usize) { // Insert last progress data { @@ -704,11 +700,10 @@ mod tests { // Validate sequentiality only after prev progress, // since the data before is mocked and can contain gaps - if number > prev_progress { - if let Some(prev_key) = prev_number { + if number > prev_progress + && let Some(prev_key) = prev_number { assert_eq!(prev_key + 1, number, "Body entries must be sequential"); } - } // Validate that the current entry is below or equals to the highest allowed block assert!( diff --git a/crates/stages/stages/src/stages/era.rs b/crates/stages/stages/src/stages/era.rs new file mode 100644 index 00000000000..6fa10a297c7 --- /dev/null +++ b/crates/stages/stages/src/stages/era.rs @@ -0,0 +1,614 @@ +use crate::{StageCheckpoint, StageId}; +use alloy_primitives::{BlockHash, BlockNumber}; +use futures_util::{Stream, StreamExt}; +use reqwest::{Client, Url}; +use reth_config::config::EtlConfig; +use reth_db_api::{table::Value, transaction::DbTxMut}; +use reth_era::{era1_file::Era1Reader, era_file_ops::StreamReader}; +use reth_era_downloader::{read_dir, EraClient, EraMeta, EraStream, EraStreamConfig}; +use reth_era_utils as era; +use reth_etl::Collector; +use reth_primitives_traits::{FullBlockBody, FullBlockHeader, NodePrimitives}; +use reth_provider::{ + BlockReader, BlockWriter, DBProvider, StageCheckpointWriter, StaticFileProviderFactory, + StaticFileWriter, +}; +use reth_stages_api::{ExecInput, ExecOutput, Stage, StageError, UnwindInput, UnwindOutput}; +use reth_static_file_types::StaticFileSegment; +use std::{ + fmt::{Debug, Formatter}, + iter, + path::Path, + task::{ready, Context, Poll}, +}; + +type Item = + Box> + Send + Sync + Unpin>; +type ThreadSafeEraStream = + Box>> + Send + Sync + Unpin>; + +/// The [ERA1](https://github.com/eth-clients/e2store-format-specs/blob/main/formats/era1.md) +/// pre-merge history stage. +/// +/// Imports block headers and bodies from genesis up to the last pre-merge block. Receipts are +/// generated by execution. Execution is not done in this stage. +pub struct EraStage { + /// The `source` creates `stream`. + source: Option, + /// A map of block hash to block height collected when processing headers and inserted into + /// database afterward. + hash_collector: Collector, + /// Last extracted iterator of block `Header` and `Body` pairs. + item: Option>, + /// A stream of [`Item`]s, i.e. iterators over block `Header` and `Body` pairs. + stream: Option>, +} + +trait EraStreamFactory { + fn create(self, input: ExecInput) -> Result, StageError>; +} + +impl EraStreamFactory for EraImportSource +where + Header: FullBlockHeader + Value, + Body: FullBlockBody, +{ + fn create(self, input: ExecInput) -> Result, StageError> { + match self { + Self::Path(path) => Self::convert( + read_dir(path, input.next_block()).map_err(|e| StageError::Fatal(e.into()))?, + ), + Self::Url(url, folder) => { + let _ = reth_fs_util::create_dir_all(&folder); + let client = EraClient::new(Client::new(), url, folder); + + Self::convert(EraStream::new( + client, + EraStreamConfig::default().start_from(input.next_block()), + )) + } + } + } +} + +impl EraImportSource { + fn convert( + stream: impl Stream> + + Send + + Sync + + 'static + + Unpin, + ) -> Result, StageError> + where + Header: FullBlockHeader + Value, + Body: FullBlockBody, + { + Ok(Box::new(Box::pin(stream.map(|meta| { + meta.and_then(|meta| { + let file = reth_fs_util::open(meta.path())?; + let reader = Era1Reader::new(file); + let iter = reader.iter(); + let iter = iter.map(era::decode); + let iter = iter.chain( + iter::once_with(move || match meta.mark_as_processed() { + Ok(..) => None, + Err(e) => Some(Err(e)), + }) + .flatten(), + ); + + Ok(Box::new(iter) as Item) + }) + })))) + } +} + +impl Debug for EraStage { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("EraStage") + .field("source", &self.source) + .field("hash_collector", &self.hash_collector) + .field("item", &self.item.is_some()) + .field("stream", &"dyn Stream") + .finish() + } +} + +impl EraStage { + /// Creates a new [`EraStage`]. + pub fn new(source: Option, etl_config: EtlConfig) -> Self { + Self { + source, + item: None, + stream: None, + hash_collector: Collector::new(etl_config.file_size, etl_config.dir), + } + } +} + +impl Stage for EraStage +where + Provider: DBProvider + + StaticFileProviderFactory + + BlockWriter + + BlockReader + + StageCheckpointWriter, + F: EraStreamFactory + Send + Sync + Clone, + N: NodePrimitives, +{ + fn id(&self) -> StageId { + StageId::Era + } + + fn poll_execute_ready( + &mut self, + cx: &mut Context<'_>, + input: ExecInput, + ) -> Poll> { + if input.target_reached() || self.item.is_some() { + return Poll::Ready(Ok(())); + } + + if self.stream.is_none() && + let Some(source) = self.source.clone() + { + self.stream.replace(source.create(input)?); + } + if let Some(stream) = &mut self.stream && + let Some(next) = ready!(stream.poll_next_unpin(cx)) + .transpose() + .map_err(|e| StageError::Fatal(e.into()))? + { + self.item.replace(next); + } + + Poll::Ready(Ok(())) + } + + fn execute(&mut self, provider: &Provider, input: ExecInput) -> Result { + let height = if let Some(era) = self.item.take() { + let static_file_provider = provider.static_file_provider(); + + // Consistency check of expected headers in static files vs DB is done on + // provider::sync_gap when poll_execute_ready is polled. + let last_header_number = static_file_provider + .get_highest_static_file_block(StaticFileSegment::Headers) + .unwrap_or_default(); + + // Although headers were downloaded in reverse order, the collector iterates it in + // ascending order + let mut writer = static_file_provider.latest_writer(StaticFileSegment::Headers)?; + + let height = era::process_iter( + era, + &mut writer, + provider, + &mut self.hash_collector, + last_header_number..=input.target(), + ) + .map_err(|e| StageError::Fatal(e.into()))?; + + if !self.hash_collector.is_empty() { + era::build_index(provider, &mut self.hash_collector) + .map_err(|e| StageError::Recoverable(e.into()))?; + self.hash_collector.clear(); + } + + era::save_stage_checkpoints( + &provider, + input.checkpoint().block_number, + height, + height, + input.target(), + )?; + + height + } else { + // No era files to process. Return the highest block we're aware of to avoid + // limiting subsequent stages with an outdated checkpoint. + // + // This can happen when: + // 1. Era import is complete (all pre-merge blocks imported) + // 2. No era import source was configured + // + // We return max(checkpoint, highest_header, target) to ensure we don't return + // a stale checkpoint that could limit subsequent stages like Headers. + let highest_header = provider + .static_file_provider() + .get_highest_static_file_block(StaticFileSegment::Headers) + .unwrap_or_default(); + + let checkpoint = input.checkpoint().block_number; + let from_target = input.target.unwrap_or(checkpoint); + + checkpoint.max(highest_header).max(from_target) + }; + + Ok(ExecOutput { checkpoint: StageCheckpoint::new(height), done: height >= input.target() }) + } + + fn unwind( + &mut self, + _provider: &Provider, + input: UnwindInput, + ) -> Result { + Ok(UnwindOutput { checkpoint: input.checkpoint.with_block_number(input.unwind_to) }) + } +} + +/// Describes where to get the era files from. +#[derive(Debug, Clone)] +pub enum EraImportSource { + /// Remote HTTP accessible host. + Url(Url, Box), + /// Local directory. + Path(Box), +} + +impl EraImportSource { + /// Maybe constructs a new `EraImportSource` depending on the arguments. + /// + /// Only one of `url` or `path` should be provided, but upholding this invariant is delegated + /// above so that both parameters can be accepted. + /// + /// # Arguments + /// * The `path` uses a directory as the import source. It and its contents must be readable. + /// * The `url` uses an HTTP client to list and download files. + /// * The `default` gives the default [`Url`] if none of the previous parameters are provided. + /// * For any [`Url`] the `folder` is used as the download directory for storing files + /// temporarily. It and its contents must be readable and writable. + pub fn maybe_new( + path: Option>, + url: Option, + default: impl FnOnce() -> Option, + folder: impl FnOnce() -> Box, + ) -> Option { + path.map(Self::Path).or_else(|| url.or_else(default).map(|url| Self::Url(url, folder()))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::{ + stage_test_suite, ExecuteStageTestRunner, StageTestRunner, UnwindStageTestRunner, + }; + use alloy_primitives::B256; + use assert_matches::assert_matches; + use reth_db_api::tables; + use reth_provider::BlockHashReader; + use reth_testing_utils::generators::{self, random_header}; + use test_runner::EraTestRunner; + + #[tokio::test] + async fn test_era_range_ends_below_target() { + let era_cap = 2; + let target = 20000; + + let mut runner = EraTestRunner::default(); + + let input = ExecInput { target: Some(era_cap), checkpoint: None }; + runner.seed_execution(input).unwrap(); + + let input = ExecInput { target: Some(target), checkpoint: None }; + let output = runner.execute(input).await.unwrap(); + + runner.commit(); + + assert_matches!( + output, + Ok(ExecOutput { + checkpoint: StageCheckpoint { block_number, stage_checkpoint: None }, + done: false + }) if block_number == era_cap + ); + + let output = output.unwrap(); + let validation_output = runner.validate_execution(input, Some(output.clone())); + + assert_matches!(validation_output, Ok(())); + + runner.take_responses(); + + let input = ExecInput { target: Some(target), checkpoint: Some(output.checkpoint) }; + let output = runner.execute(input).await.unwrap(); + + runner.commit(); + + assert_matches!( + output, + Ok(ExecOutput { + checkpoint: StageCheckpoint { block_number, stage_checkpoint: None }, + done: true + }) if block_number == target + ); + + let validation_output = runner.validate_execution(input, output.ok()); + + assert_matches!(validation_output, Ok(())); + } + + mod test_runner { + use super::*; + use crate::test_utils::{TestRunnerError, TestStageDB}; + use alloy_consensus::{BlockBody, Header}; + use futures_util::stream; + use reth_db_api::{ + cursor::DbCursorRO, + models::{StoredBlockBodyIndices, StoredBlockOmmers}, + transaction::DbTx, + }; + use reth_ethereum_primitives::TransactionSigned; + use reth_primitives_traits::{SealedBlock, SealedHeader}; + use reth_provider::{BlockNumReader, HeaderProvider, TransactionsProvider}; + use reth_testing_utils::generators::{ + random_block_range, random_signed_tx, BlockRangeParams, + }; + use tokio::sync::watch; + + pub(crate) struct EraTestRunner { + channel: (watch::Sender, watch::Receiver), + db: TestStageDB, + responses: Option)>>, + } + + impl Default for EraTestRunner { + fn default() -> Self { + Self { + channel: watch::channel(B256::ZERO), + db: TestStageDB::default(), + responses: Default::default(), + } + } + } + + impl StageTestRunner for EraTestRunner { + type S = EraStage, StubResponses>; + + fn db(&self) -> &TestStageDB { + &self.db + } + + fn stage(&self) -> Self::S { + EraStage::new(self.responses.clone().map(StubResponses), EtlConfig::default()) + } + } + + impl ExecuteStageTestRunner for EraTestRunner { + type Seed = Vec>; + + fn seed_execution(&mut self, input: ExecInput) -> Result { + let start = input.checkpoint().block_number; + let end = input.target(); + + let static_file_provider = self.db.factory.static_file_provider(); + + let mut rng = generators::rng(); + + // Static files do not support gaps in headers, so we need to generate 0 to end + let blocks = random_block_range( + &mut rng, + 0..=end, + BlockRangeParams { + parent: Some(B256::ZERO), + tx_count: 0..2, + ..Default::default() + }, + ); + self.db.insert_headers(blocks.iter().map(|block| block.sealed_header()))?; + if let Some(progress) = blocks.get(start as usize) { + // Insert last progress data + { + let tx = self.db.factory.provider_rw()?.into_tx(); + let mut static_file_producer = static_file_provider + .get_writer(start, StaticFileSegment::Transactions)?; + + let body = StoredBlockBodyIndices { + first_tx_num: 0, + tx_count: progress.transaction_count() as u64, + }; + + static_file_producer.set_block_range(0..=progress.number); + + body.tx_num_range().try_for_each(|tx_num| { + let transaction = random_signed_tx(&mut rng); + static_file_producer.append_transaction(tx_num, &transaction).map(drop) + })?; + + if body.tx_count != 0 { + tx.put::( + body.last_tx_num(), + progress.number, + )?; + } + + tx.put::(progress.number, body)?; + + if !progress.ommers_hash_is_empty() { + tx.put::( + progress.number, + StoredBlockOmmers { ommers: progress.body().ommers.clone() }, + )?; + } + + static_file_producer.commit()?; + tx.commit()?; + } + } + self.responses.replace( + blocks.iter().map(|v| (v.header().clone(), v.body().clone())).collect(), + ); + Ok(blocks) + } + + /// Validate stored headers and bodies + fn validate_execution( + &self, + input: ExecInput, + output: Option, + ) -> Result<(), TestRunnerError> { + let initial_checkpoint = input.checkpoint().block_number; + match output { + Some(output) if output.checkpoint.block_number > initial_checkpoint => { + let provider = self.db.factory.provider()?; + + for block_num in initial_checkpoint.. + output + .checkpoint + .block_number + .min(self.responses.as_ref().map(|v| v.len()).unwrap_or_default() + as BlockNumber) + { + // look up the header hash + let hash = provider.block_hash(block_num)?.expect("no header hash"); + + // validate the header number + assert_eq!(provider.block_number(hash)?, Some(block_num)); + + // validate the header + let header = provider.header_by_number(block_num)?; + assert!(header.is_some()); + let header = SealedHeader::seal_slow(header.unwrap()); + assert_eq!(header.hash(), hash); + } + + self.validate_db_blocks( + output.checkpoint.block_number, + output.checkpoint.block_number, + )?; + } + _ => self.check_no_header_entry_above(initial_checkpoint)?, + }; + Ok(()) + } + + async fn after_execution(&self, headers: Self::Seed) -> Result<(), TestRunnerError> { + let tip = if headers.is_empty() { + let tip = random_header(&mut generators::rng(), 0, None); + self.db.insert_headers(iter::once(&tip))?; + tip.hash() + } else { + headers.last().unwrap().hash() + }; + self.send_tip(tip); + Ok(()) + } + } + + impl UnwindStageTestRunner for EraTestRunner { + fn validate_unwind(&self, _input: UnwindInput) -> Result<(), TestRunnerError> { + Ok(()) + } + } + + impl EraTestRunner { + pub(crate) fn check_no_header_entry_above( + &self, + block: BlockNumber, + ) -> Result<(), TestRunnerError> { + self.db + .ensure_no_entry_above_by_value::(block, |val| val)?; + self.db.ensure_no_entry_above::(block, |key| key)?; + self.db.ensure_no_entry_above::(block, |key| key)?; + Ok(()) + } + + pub(crate) fn send_tip(&self, tip: B256) { + self.channel.0.send(tip).expect("failed to send tip"); + } + + /// Validate that the inserted block data is valid + pub(crate) fn validate_db_blocks( + &self, + prev_progress: BlockNumber, + highest_block: BlockNumber, + ) -> Result<(), TestRunnerError> { + let static_file_provider = self.db.factory.static_file_provider(); + + self.db.query(|tx| { + // Acquire cursors on body related tables + let mut bodies_cursor = tx.cursor_read::()?; + let mut ommers_cursor = tx.cursor_read::()?; + let mut tx_block_cursor = tx.cursor_read::()?; + + let first_body_key = match bodies_cursor.first()? { + Some((key, _)) => key, + None => return Ok(()), + }; + + let mut prev_number: Option = None; + + + for entry in bodies_cursor.walk(Some(first_body_key))? { + let (number, body) = entry?; + + // Validate sequentiality only after prev progress, + // since the data before is mocked and can contain gaps + if number > prev_progress + && let Some(prev_key) = prev_number { + assert_eq!(prev_key + 1, number, "Body entries must be sequential"); + } + + // Validate that the current entry is below or equals to the highest allowed block + assert!( + number <= highest_block, + "We wrote a block body outside of our synced range. Found block with number {number}, highest block according to stage is {highest_block}", + ); + + let header = static_file_provider.header_by_number(number)?.expect("to be present"); + // Validate that ommers exist if any + let stored_ommers = ommers_cursor.seek_exact(number)?; + if header.ommers_hash_is_empty() { + assert!(stored_ommers.is_none(), "Unexpected ommers entry"); + } else { + assert!(stored_ommers.is_some(), "Missing ommers entry"); + } + + let tx_block_id = tx_block_cursor.seek_exact(body.last_tx_num())?.map(|(_,b)| b); + if body.tx_count == 0 { + assert_ne!(tx_block_id,Some(number)); + } else { + assert_eq!(tx_block_id, Some(number)); + } + + for tx_id in body.tx_num_range() { + assert!(static_file_provider.transaction_by_id(tx_id)?.is_some(), "Transaction is missing."); + } + + prev_number = Some(number); + } + Ok(()) + })?; + Ok(()) + } + + pub(crate) fn take_responses(&mut self) { + self.responses.take(); + } + + pub(crate) fn commit(&self) { + self.db.factory.static_file_provider().commit().unwrap(); + } + } + + #[derive(Clone)] + pub(crate) struct StubResponses(Vec<(Header, BlockBody)>); + + impl EraStreamFactory> for StubResponses { + fn create( + self, + _input: ExecInput, + ) -> Result>, StageError> + { + let stream = stream::iter(vec![self.0]); + + Ok(Box::new(Box::pin(stream.map(|meta| { + Ok(Box::new(meta.into_iter().map(Ok)) + as Item>) + })))) + } + } + } + + stage_test_suite!(EraTestRunner, era); +} diff --git a/crates/stages/stages/src/stages/execution.rs b/crates/stages/stages/src/stages/execution.rs index 6833eddc1f5..1666e79baf3 100644 --- a/crates/stages/stages/src/stages/execution.rs +++ b/crates/stages/stages/src/stages/execution.rs @@ -1,5 +1,5 @@ -use crate::stages::MERKLE_STAGE_DEFAULT_CLEAN_THRESHOLD; -use alloy_consensus::{BlockHeader, Header}; +use crate::stages::MERKLE_STAGE_DEFAULT_INCREMENTAL_THRESHOLD; +use alloy_consensus::BlockHeader; use alloy_primitives::BlockNumber; use num_traits::Zero; use reth_config::config::ExecutionConfig; @@ -8,12 +8,12 @@ use reth_db::{static_file::HeaderMask, tables}; use reth_evm::{execute::Executor, metrics::ExecutorMetrics, ConfigureEvm}; use reth_execution_types::Chain; use reth_exex::{ExExManagerHandle, ExExNotification, ExExNotificationSource}; -use reth_primitives_traits::{format_gas_throughput, Block, BlockBody, NodePrimitives}; +use reth_primitives_traits::{format_gas_throughput, BlockBody, NodePrimitives}; use reth_provider::{ providers::{StaticFileProvider, StaticFileWriter}, BlockHashReader, BlockReader, DBProvider, ExecutionOutcome, HeaderProvider, - LatestStateProviderRef, OriginalValuesKnown, ProviderError, StateCommitmentProvider, - StateWriter, StaticFileProviderFactory, StatsReader, StorageLocation, TransactionVariant, + LatestStateProviderRef, OriginalValuesKnown, ProviderError, StateWriter, + StaticFileProviderFactory, StatsReader, TransactionVariant, }; use reth_revm::database::StateProviderDatabase; use reth_stages_api::{ @@ -39,7 +39,6 @@ use super::missing_static_data_error; /// Input tables: /// - [`tables::CanonicalHeaders`] get next block to execute. /// - [`tables::Headers`] get for revm environment variables. -/// - [`tables::HeaderTerminalDifficulties`] /// - [`tables::BlockBodyIndices`] to get tx number /// - [`tables::Transactions`] to execute /// @@ -71,7 +70,6 @@ where evm_config: E, /// The consensus instance for validating blocks. consensus: Arc>, - /// The consensu /// The commit thresholds of the execution stage. thresholds: ExecutionStageThresholds, /// The highest threshold (in number of blocks) for switching between incremental @@ -119,7 +117,7 @@ where /// Create an execution stage with the provided executor. /// - /// The commit threshold will be set to [`MERKLE_STAGE_DEFAULT_CLEAN_THRESHOLD`]. + /// The commit threshold will be set to [`MERKLE_STAGE_DEFAULT_INCREMENTAL_THRESHOLD`]. pub fn new_with_executor( evm_config: E, consensus: Arc>, @@ -128,7 +126,7 @@ where evm_config, consensus, ExecutionStageThresholds::default(), - MERKLE_STAGE_DEFAULT_CLEAN_THRESHOLD, + MERKLE_STAGE_DEFAULT_INCREMENTAL_THRESHOLD, ExExManagerHandle::empty(), ) } @@ -257,11 +255,11 @@ where + BlockReader< Block = ::Block, Header = ::BlockHeader, - > + StaticFileProviderFactory - + StatsReader + > + StaticFileProviderFactory< + Primitives: NodePrimitives, + > + StatsReader + BlockHashReader - + StateWriter::Receipt> - + StateCommitmentProvider, + + StateWriter::Receipt>, { /// Return the id of the stage fn id(&self) -> StageId { @@ -453,7 +451,7 @@ where } // write output - provider.write_state(&state, OriginalValuesKnown::Yes, StorageLocation::StaticFiles)?; + provider.write_state(&state, OriginalValuesKnown::Yes)?; let db_write_duration = time.elapsed(); debug!( @@ -505,8 +503,7 @@ where // Unwind account and storage changesets, as well as receipts. // // This also updates `PlainStorageState` and `PlainAccountState`. - let bundle_state_with_receipts = - provider.take_state_above(unwind_to, StorageLocation::Both)?; + let bundle_state_with_receipts = provider.take_state_above(unwind_to)?; // Prepare the input for post unwind commit hook, where an `ExExNotification` will be sent. if self.exex_manager_handle.has_exexs() { @@ -532,9 +529,8 @@ where if let Some(stage_checkpoint) = stage_checkpoint.as_mut() { for block_number in range { stage_checkpoint.progress.processed -= provider - .block_by_number(block_number)? + .header_by_number(block_number)? .ok_or_else(|| ProviderError::HeaderNotFound(block_number.into()))? - .header() .gas_used(); } } @@ -561,12 +557,15 @@ where } } -fn execution_checkpoint( +fn execution_checkpoint( provider: &StaticFileProvider, start_block: BlockNumber, max_block: BlockNumber, checkpoint: StageCheckpoint, -) -> Result { +) -> Result +where + N: NodePrimitives, +{ Ok(match checkpoint.execution_stage_checkpoint() { // If checkpoint block range fully matches our range, // we take the previously used stage checkpoint as-is. @@ -628,10 +627,14 @@ fn execution_checkpoint( }) } -fn calculate_gas_used_from_headers( +/// Calculates the total amount of gas used from the headers in the given range. +pub fn calculate_gas_used_from_headers( provider: &StaticFileProvider, range: RangeInclusive, -) -> Result { +) -> Result +where + N: NodePrimitives, +{ debug!(target: "sync::stages::execution", ?range, "Calculating gas used from headers"); let mut gas_total = 0; @@ -641,10 +644,10 @@ fn calculate_gas_used_from_headers( for entry in provider.fetch_range_iter( StaticFileSegment::Headers, *range.start()..*range.end() + 1, - |cursor, number| cursor.get_one::>(number.into()), + |cursor, number| cursor.get_one::>(number.into()), )? { - let Header { gas_used, .. } = entry?; - gas_total += gas_used; + let entry = entry?; + gas_total += entry.gas_used(); } let duration = start.elapsed(); @@ -656,7 +659,7 @@ fn calculate_gas_used_from_headers( #[cfg(test)] mod tests { use super::*; - use crate::test_utils::TestStageDB; + use crate::{stages::MERKLE_STAGE_DEFAULT_REBUILD_THRESHOLD, test_utils::TestStageDB}; use alloy_primitives::{address, hex_literal::hex, keccak256, Address, B256, U256}; use alloy_rlp::Decodable; use assert_matches::assert_matches; @@ -670,8 +673,8 @@ mod tests { use reth_evm_ethereum::EthEvmConfig; use reth_primitives_traits::{Account, Bytecode, SealedBlock, StorageEntry}; use reth_provider::{ - test_utils::create_test_provider_factory, AccountReader, DatabaseProviderFactory, - ReceiptProvider, StaticFileProviderFactory, + test_utils::create_test_provider_factory, AccountReader, BlockWriter, + DatabaseProviderFactory, ReceiptProvider, StaticFileProviderFactory, }; use reth_prune::PruneModes; use reth_prune_types::{PruneMode, ReceiptsLogPruneConfig}; @@ -693,7 +696,7 @@ mod tests { max_cumulative_gas: None, max_duration: None, }, - MERKLE_STAGE_DEFAULT_CLEAN_THRESHOLD, + MERKLE_STAGE_DEFAULT_REBUILD_THRESHOLD, ExExManagerHandle::empty(), ) } @@ -732,8 +735,8 @@ mod tests { let genesis = SealedBlock::::decode(&mut genesis_rlp).unwrap(); let mut block_rlp = hex!("f90262f901f9a075c371ba45999d87f4542326910a11af515897aebce5265d3f6acd1f1161f82fa01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347942adc25665018aa1fe0e6bc666dac8fc2697ff9baa098f2dcd87c8ae4083e7017a05456c14eea4b1db2032126e27b3b1563d57d7cc0a08151d548273f6683169524b66ca9fe338b9ce42bc3540046c828fd939ae23bcba03f4e5c2ec5b2170b711d97ee755c160457bb58d8daa338e835ec02ae6860bbabb901000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000083020000018502540be40082a8798203e800a00000000000000000000000000000000000000000000000000000000000000000880000000000000000f863f861800a8405f5e10094100000000000000000000000000000000000000080801ba07e09e26678ed4fac08a249ebe8ed680bf9051a5e14ad223e4b2b9d26e0208f37a05f6e3f188e3e6eab7d7d3b6568f5eac7d687b08d307d3154ccd8c87b4630509bc0").as_slice(); let block = SealedBlock::::decode(&mut block_rlp).unwrap(); - provider.insert_historical_block(genesis.try_recover().unwrap()).unwrap(); - provider.insert_historical_block(block.clone().try_recover().unwrap()).unwrap(); + provider.insert_block(genesis.try_recover().unwrap()).unwrap(); + provider.insert_block(block.clone().try_recover().unwrap()).unwrap(); provider .static_file_provider() .latest_writer(StaticFileSegment::Headers) @@ -773,8 +776,8 @@ mod tests { let genesis = SealedBlock::::decode(&mut genesis_rlp).unwrap(); let mut block_rlp = hex!("f90262f901f9a075c371ba45999d87f4542326910a11af515897aebce5265d3f6acd1f1161f82fa01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347942adc25665018aa1fe0e6bc666dac8fc2697ff9baa098f2dcd87c8ae4083e7017a05456c14eea4b1db2032126e27b3b1563d57d7cc0a08151d548273f6683169524b66ca9fe338b9ce42bc3540046c828fd939ae23bcba03f4e5c2ec5b2170b711d97ee755c160457bb58d8daa338e835ec02ae6860bbabb901000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000083020000018502540be40082a8798203e800a00000000000000000000000000000000000000000000000000000000000000000880000000000000000f863f861800a8405f5e10094100000000000000000000000000000000000000080801ba07e09e26678ed4fac08a249ebe8ed680bf9051a5e14ad223e4b2b9d26e0208f37a05f6e3f188e3e6eab7d7d3b6568f5eac7d687b08d307d3154ccd8c87b4630509bc0").as_slice(); let block = SealedBlock::::decode(&mut block_rlp).unwrap(); - provider.insert_historical_block(genesis.try_recover().unwrap()).unwrap(); - provider.insert_historical_block(block.clone().try_recover().unwrap()).unwrap(); + provider.insert_block(genesis.try_recover().unwrap()).unwrap(); + provider.insert_block(block.clone().try_recover().unwrap()).unwrap(); provider .static_file_provider() .latest_writer(StaticFileSegment::Headers) @@ -814,8 +817,8 @@ mod tests { let genesis = SealedBlock::::decode(&mut genesis_rlp).unwrap(); let mut block_rlp = hex!("f90262f901f9a075c371ba45999d87f4542326910a11af515897aebce5265d3f6acd1f1161f82fa01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347942adc25665018aa1fe0e6bc666dac8fc2697ff9baa098f2dcd87c8ae4083e7017a05456c14eea4b1db2032126e27b3b1563d57d7cc0a08151d548273f6683169524b66ca9fe338b9ce42bc3540046c828fd939ae23bcba03f4e5c2ec5b2170b711d97ee755c160457bb58d8daa338e835ec02ae6860bbabb901000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000083020000018502540be40082a8798203e800a00000000000000000000000000000000000000000000000000000000000000000880000000000000000f863f861800a8405f5e10094100000000000000000000000000000000000000080801ba07e09e26678ed4fac08a249ebe8ed680bf9051a5e14ad223e4b2b9d26e0208f37a05f6e3f188e3e6eab7d7d3b6568f5eac7d687b08d307d3154ccd8c87b4630509bc0").as_slice(); let block = SealedBlock::::decode(&mut block_rlp).unwrap(); - provider.insert_historical_block(genesis.try_recover().unwrap()).unwrap(); - provider.insert_historical_block(block.clone().try_recover().unwrap()).unwrap(); + provider.insert_block(genesis.try_recover().unwrap()).unwrap(); + provider.insert_block(block.clone().try_recover().unwrap()).unwrap(); provider .static_file_provider() .latest_writer(StaticFileSegment::Headers) @@ -847,8 +850,8 @@ mod tests { let genesis = SealedBlock::::decode(&mut genesis_rlp).unwrap(); let mut block_rlp = hex!("f90262f901f9a075c371ba45999d87f4542326910a11af515897aebce5265d3f6acd1f1161f82fa01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347942adc25665018aa1fe0e6bc666dac8fc2697ff9baa098f2dcd87c8ae4083e7017a05456c14eea4b1db2032126e27b3b1563d57d7cc0a08151d548273f6683169524b66ca9fe338b9ce42bc3540046c828fd939ae23bcba03f4e5c2ec5b2170b711d97ee755c160457bb58d8daa338e835ec02ae6860bbabb901000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000083020000018502540be40082a8798203e800a00000000000000000000000000000000000000000000000000000000000000000880000000000000000f863f861800a8405f5e10094100000000000000000000000000000000000000080801ba07e09e26678ed4fac08a249ebe8ed680bf9051a5e14ad223e4b2b9d26e0208f37a05f6e3f188e3e6eab7d7d3b6568f5eac7d687b08d307d3154ccd8c87b4630509bc0").as_slice(); let block = SealedBlock::::decode(&mut block_rlp).unwrap(); - provider.insert_historical_block(genesis.try_recover().unwrap()).unwrap(); - provider.insert_historical_block(block.clone().try_recover().unwrap()).unwrap(); + provider.insert_block(genesis.try_recover().unwrap()).unwrap(); + provider.insert_block(block.clone().try_recover().unwrap()).unwrap(); provider .static_file_provider() .latest_writer(StaticFileSegment::Headers) @@ -892,7 +895,7 @@ mod tests { // If there is a pruning configuration, then it's forced to use the database. // This way we test both cases. - let modes = [None, Some(PruneModes::none())]; + let modes = [None, Some(PruneModes::default())]; let random_filter = ReceiptsLogPruneConfig(BTreeMap::from([( Address::random(), PruneMode::Distance(100000), @@ -989,8 +992,8 @@ mod tests { let genesis = SealedBlock::::decode(&mut genesis_rlp).unwrap(); let mut block_rlp = hex!("f90262f901f9a075c371ba45999d87f4542326910a11af515897aebce5265d3f6acd1f1161f82fa01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347942adc25665018aa1fe0e6bc666dac8fc2697ff9baa098f2dcd87c8ae4083e7017a05456c14eea4b1db2032126e27b3b1563d57d7cc0a08151d548273f6683169524b66ca9fe338b9ce42bc3540046c828fd939ae23bcba03f4e5c2ec5b2170b711d97ee755c160457bb58d8daa338e835ec02ae6860bbabb901000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000083020000018502540be40082a8798203e800a00000000000000000000000000000000000000000000000000000000000000000880000000000000000f863f861800a8405f5e10094100000000000000000000000000000000000000080801ba07e09e26678ed4fac08a249ebe8ed680bf9051a5e14ad223e4b2b9d26e0208f37a05f6e3f188e3e6eab7d7d3b6568f5eac7d687b08d307d3154ccd8c87b4630509bc0").as_slice(); let block = SealedBlock::::decode(&mut block_rlp).unwrap(); - provider.insert_historical_block(genesis.try_recover().unwrap()).unwrap(); - provider.insert_historical_block(block.clone().try_recover().unwrap()).unwrap(); + provider.insert_block(genesis.try_recover().unwrap()).unwrap(); + provider.insert_block(block.clone().try_recover().unwrap()).unwrap(); provider .static_file_provider() .latest_writer(StaticFileSegment::Headers) @@ -1029,7 +1032,7 @@ mod tests { // If there is a pruning configuration, then it's forced to use the database. // This way we test both cases. - let modes = [None, Some(PruneModes::none())]; + let modes = [None, Some(PruneModes::default())]; let random_filter = ReceiptsLogPruneConfig(BTreeMap::from([( Address::random(), PruneMode::Before(100000), @@ -1061,6 +1064,8 @@ mod tests { ) .unwrap(); + provider.static_file_provider().commit().unwrap(); + assert_matches!(result, UnwindOutput { checkpoint: StageCheckpoint { block_number: 0, @@ -1097,8 +1102,8 @@ mod tests { let genesis = SealedBlock::::decode(&mut genesis_rlp).unwrap(); let mut block_rlp = hex!("f9025ff901f7a0c86e8cc0310ae7c531c758678ddbfd16fc51c8cef8cec650b032de9869e8b94fa01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347942adc25665018aa1fe0e6bc666dac8fc2697ff9baa050554882fbbda2c2fd93fdc466db9946ea262a67f7a76cc169e714f105ab583da00967f09ef1dfed20c0eacfaa94d5cd4002eda3242ac47eae68972d07b106d192a0e3c8b47fbfc94667ef4cceb17e5cc21e3b1eebd442cebb27f07562b33836290db90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008302000001830f42408238108203e800a00000000000000000000000000000000000000000000000000000000000000000880000000000000000f862f860800a83061a8094095e7baea6a6c7c4c2dfeb977efac326af552d8780801ba072ed817487b84ba367d15d2f039b5fc5f087d0a8882fbdf73e8cb49357e1ce30a0403d800545b8fc544f92ce8124e2255f8c3c6af93f28243a120585d4c4c6a2a3c0").as_slice(); let block = SealedBlock::::decode(&mut block_rlp).unwrap(); - provider.insert_historical_block(genesis.try_recover().unwrap()).unwrap(); - provider.insert_historical_block(block.clone().try_recover().unwrap()).unwrap(); + provider.insert_block(genesis.try_recover().unwrap()).unwrap(); + provider.insert_block(block.clone().try_recover().unwrap()).unwrap(); provider .static_file_provider() .latest_writer(StaticFileSegment::Headers) diff --git a/crates/stages/stages/src/stages/finish.rs b/crates/stages/stages/src/stages/finish.rs index 1b9e624b41b..8d676c35b99 100644 --- a/crates/stages/stages/src/stages/finish.rs +++ b/crates/stages/stages/src/stages/finish.rs @@ -72,7 +72,7 @@ mod tests { let start = input.checkpoint().block_number; let mut rng = generators::rng(); let head = random_header(&mut rng, start, None); - self.db.insert_headers_with_td(std::iter::once(&head))?; + self.db.insert_headers(std::iter::once(&head))?; // use previous progress as seed size let end = input.target.unwrap_or_default() + 1; @@ -82,7 +82,7 @@ mod tests { } let mut headers = random_header_range(&mut rng, start + 1..end, head.hash()); - self.db.insert_headers_with_td(headers.iter())?; + self.db.insert_headers(headers.iter())?; headers.insert(0, head); Ok(headers) } diff --git a/crates/stages/stages/src/stages/hashing_account.rs b/crates/stages/stages/src/stages/hashing_account.rs index 726f9e82183..1e48f2d38e0 100644 --- a/crates/stages/stages/src/stages/hashing_account.rs +++ b/crates/stages/stages/src/stages/hashing_account.rs @@ -64,14 +64,14 @@ impl AccountHashingStage { opts: SeedOpts, ) -> Result, StageError> where - N::Primitives: reth_primitives_traits::FullNodePrimitives< + N::Primitives: reth_primitives_traits::NodePrimitives< Block = reth_ethereum_primitives::Block, BlockHeader = reth_primitives_traits::Header, >, { use alloy_primitives::U256; use reth_db_api::models::AccountBeforeTx; - use reth_provider::{StaticFileProviderFactory, StaticFileWriter}; + use reth_provider::{BlockWriter, StaticFileProviderFactory, StaticFileWriter}; use reth_testing_utils::{ generators, generators::{random_block_range, random_eoa_accounts, BlockRangeParams}, @@ -86,7 +86,7 @@ impl AccountHashingStage { ); for block in blocks { - provider.insert_historical_block(block.try_recover().unwrap()).unwrap(); + provider.insert_block(block.try_recover().unwrap()).unwrap(); } provider .static_file_provider() @@ -180,7 +180,7 @@ where }); // Flush to ETL when channels length reaches MAXIMUM_CHANNELS - if !channels.is_empty() && channels.len() % MAXIMUM_CHANNELS == 0 { + if !channels.is_empty() && channels.len().is_multiple_of(MAXIMUM_CHANNELS) { collect(&mut channels, &mut collector)?; } } @@ -193,7 +193,7 @@ where let total_hashes = collector.len(); let interval = (total_hashes / 10).max(1); for (index, item) in collector.iter()?.enumerate() { - if index > 0 && index % interval == 0 { + if index > 0 && index.is_multiple_of(interval) { info!( target: "sync::stages::hashing_account", progress = %format!("{:.2}%", (index as f64 / total_hashes as f64) * 100.0), @@ -344,7 +344,7 @@ mod tests { done: true, }) if block_number == previous_stage && processed == total && - total == runner.db.table::().unwrap().len() as u64 + total == runner.db.count_entries::().unwrap() as u64 ); // Validate the stage execution @@ -453,7 +453,7 @@ mod tests { let provider = self.db.factory.database_provider_rw()?; let res = Ok(AccountHashingStage::seed( &provider, - SeedOpts { blocks: 1..=input.target(), accounts: 10, txs: 0..3 }, + SeedOpts { blocks: 0..=input.target(), accounts: 10, txs: 0..3 }, ) .unwrap()); provider.commit().expect("failed to commit"); diff --git a/crates/stages/stages/src/stages/hashing_storage.rs b/crates/stages/stages/src/stages/hashing_storage.rs index af811828fb2..c52f800a018 100644 --- a/crates/stages/stages/src/stages/hashing_storage.rs +++ b/crates/stages/stages/src/stages/hashing_storage.rs @@ -110,7 +110,7 @@ where }); // Flush to ETL when channels length reaches MAXIMUM_CHANNELS - if !channels.is_empty() && channels.len() % MAXIMUM_CHANNELS == 0 { + if !channels.is_empty() && channels.len().is_multiple_of(MAXIMUM_CHANNELS) { collect(&mut channels, &mut collector)?; } } @@ -121,7 +121,7 @@ where let interval = (total_hashes / 10).max(1); let mut cursor = tx.cursor_dup_write::()?; for (index, item) in collector.iter()?.enumerate() { - if index > 0 && index % interval == 0 { + if index > 0 && index.is_multiple_of(interval) { info!( target: "sync::stages::hashing_storage", progress = %format!("{:.2}%", (index as f64 / total_hashes as f64) * 100.0), @@ -266,7 +266,7 @@ mod tests { }, .. }) if processed == previous_checkpoint.progress.processed + 1 && - total == runner.db.table::().unwrap().len() as u64); + total == runner.db.count_entries::().unwrap() as u64); // Continue from checkpoint input.checkpoint = Some(checkpoint); @@ -280,7 +280,7 @@ mod tests { }, .. }) if processed == total && - total == runner.db.table::().unwrap().len() as u64); + total == runner.db.count_entries::().unwrap() as u64); // Validate the stage execution assert!( diff --git a/crates/stages/stages/src/stages/headers.rs b/crates/stages/stages/src/stages/headers.rs index 15992c232cb..8ad39be5eb8 100644 --- a/crates/stages/stages/src/stages/headers.rs +++ b/crates/stages/stages/src/stages/headers.rs @@ -16,15 +16,14 @@ use reth_network_p2p::headers::{ }; use reth_primitives_traits::{serde_bincode_compat, FullBlockHeader, NodePrimitives, SealedHeader}; use reth_provider::{ - providers::StaticFileWriter, BlockHashReader, DBProvider, HeaderProvider, - HeaderSyncGapProvider, StaticFileProviderFactory, + providers::StaticFileWriter, BlockHashReader, DBProvider, HeaderSyncGapProvider, + StaticFileProviderFactory, }; use reth_stages_api::{ CheckpointBlockRange, EntitiesCheckpoint, ExecInput, ExecOutput, HeadersCheckpoint, Stage, StageCheckpoint, StageError, StageId, UnwindInput, UnwindOutput, }; use reth_static_file_types::StaticFileSegment; -use reth_storage_errors::provider::ProviderError; use std::task::{ready, Context, Poll}; use tokio::sync::watch; @@ -107,11 +106,6 @@ where .get_highest_static_file_block(StaticFileSegment::Headers) .unwrap_or_default(); - // Find the latest total difficulty - let mut td = static_file_provider - .header_td_by_number(last_header_number)? - .ok_or(ProviderError::TotalDifficultyNotFound(last_header_number))?; - // Although headers were downloaded in reverse order, the collector iterates it in ascending // order let mut writer = static_file_provider.latest_writer(StaticFileSegment::Headers)?; @@ -119,7 +113,7 @@ where for (index, header) in self.header_collector.iter()?.enumerate() { let (_, header_buf) = header?; - if index > 0 && index % interval == 0 && total_headers > 100 { + if index > 0 && index.is_multiple_of(interval) && total_headers > 100 { info!(target: "sync::stages::headers", progress = %format!("{:.2}%", (index as f64 / total_headers as f64) * 100.0), "Writing headers"); } @@ -134,37 +128,33 @@ where } last_header_number = header.number(); - // Increase total difficulty - td += header.difficulty(); - // Append to Headers segment - writer.append_header(header, td, header_hash)?; + writer.append_header(header, header_hash)?; } info!(target: "sync::stages::headers", total = total_headers, "Writing headers hash index"); let mut cursor_header_numbers = provider.tx_ref().cursor_write::>()?; - let mut first_sync = false; - // If we only have the genesis block hash, then we are at first sync, and we can remove it, // add it to the collector and use tx.append on all hashes. - if provider.tx_ref().entries::>()? == 1 { - if let Some((hash, block_number)) = cursor_header_numbers.last()? { - if block_number.value()? == 0 { - self.hash_collector.insert(hash.key()?, 0)?; - cursor_header_numbers.delete_current()?; - first_sync = true; - } - } - } + let first_sync = if provider.tx_ref().entries::>()? == 1 && + let Some((hash, block_number)) = cursor_header_numbers.last()? && + block_number.value()? == 0 + { + self.hash_collector.insert(hash.key()?, 0)?; + cursor_header_numbers.delete_current()?; + true + } else { + false + }; // Since ETL sorts all entries by hashes, we are either appending (first sync) or inserting // in order (further syncs). for (index, hash_to_number) in self.hash_collector.iter()?.enumerate() { let (hash, number) = hash_to_number?; - if index > 0 && index % interval == 0 && total_headers > 100 { + if index > 0 && index.is_multiple_of(interval) && total_headers > 100 { info!(target: "sync::stages::headers", progress = %format!("{:.2}%", (index as f64 / total_headers as f64) * 100.0), "Writing headers hash index"); } @@ -214,7 +204,6 @@ where let target = SyncTarget::Tip(*self.tip.borrow()); let gap = HeaderSyncGap { local_head, target }; let tip = gap.target.tip(); - self.sync_gap = Some(gap.clone()); // Nothing to sync if gap.is_closed() { @@ -225,6 +214,7 @@ where "Target block already reached" ); self.is_etl_ready = true; + self.sync_gap = Some(gap); return Poll::Ready(Ok(())) } @@ -232,7 +222,10 @@ where let local_head_number = gap.local_head.number(); // let the downloader know what to sync - self.downloader.update_sync_gap(gap.local_head, gap.target); + if self.sync_gap != Some(gap.clone()) { + self.sync_gap = Some(gap.clone()); + self.downloader.update_sync_gap(gap.local_head, gap.target); + } // We only want to stop once we have all the headers on ETL filespace (disk). loop { @@ -263,13 +256,17 @@ where } Some(Err(HeadersDownloaderError::DetachedHead { local_head, header, error })) => { error!(target: "sync::stages::headers", %error, "Cannot attach header to head"); + self.sync_gap = None; return Poll::Ready(Err(StageError::DetachedHead { local_head: Box::new(local_head.block_with_parent()), header: Box::new(header.block_with_parent()), error, })) } - None => return Poll::Ready(Err(StageError::ChannelClosed)), + None => { + self.sync_gap = None; + return Poll::Ready(Err(StageError::ChannelClosed)) + } } } } @@ -279,7 +276,7 @@ where fn execute(&mut self, provider: &Provider, input: ExecInput) -> Result { let current_checkpoint = input.checkpoint(); - if self.sync_gap.as_ref().ok_or(StageError::MissingSyncGap)?.is_closed() { + if self.sync_gap.take().ok_or(StageError::MissingSyncGap)?.is_closed() { self.is_etl_ready = false; return Ok(ExecOutput::done(current_checkpoint)) } @@ -336,9 +333,6 @@ where (input.unwind_to + 1).., )?; provider.tx_ref().unwind_table_by_num::(input.unwind_to)?; - provider - .tx_ref() - .unwind_table_by_num::(input.unwind_to)?; let unfinalized_headers_unwound = provider.tx_ref().unwind_table_by_num::(input.unwind_to)?; @@ -395,13 +389,9 @@ mod tests { }; use alloy_primitives::B256; use assert_matches::assert_matches; - use reth_ethereum_primitives::BlockBody; - use reth_execution_types::ExecutionOutcome; - use reth_primitives_traits::{RecoveredBlock, SealedBlock}; - use reth_provider::{BlockWriter, ProviderFactory, StaticFileProviderFactory}; + use reth_provider::{DatabaseProviderFactory, ProviderFactory, StaticFileProviderFactory}; use reth_stages_api::StageUnitCheckpoint; use reth_testing_utils::generators::{self, random_header, random_header_range}; - use reth_trie::{updates::TrieUpdates, HashedPostStateSorted}; use std::sync::Arc; use test_runner::HeadersTestRunner; @@ -413,7 +403,7 @@ mod tests { ReverseHeadersDownloader, ReverseHeadersDownloaderBuilder, }; use reth_network_p2p::test_utils::{TestHeaderDownloader, TestHeadersClient}; - use reth_provider::{test_utils::MockNodeTypesWithDB, BlockNumReader}; + use reth_provider::{test_utils::MockNodeTypesWithDB, BlockNumReader, HeaderProvider}; use tokio::sync::watch; pub(crate) struct HeadersTestRunner { @@ -467,7 +457,7 @@ mod tests { let start = input.checkpoint().block_number; let headers = random_header_range(&mut rng, 0..start + 1, B256::ZERO); let head = headers.last().cloned().unwrap(); - self.db.insert_headers_with_td(headers.iter())?; + self.db.insert_headers(headers.iter())?; // use previous checkpoint as seed size let end = input.target.unwrap_or_default() + 1; @@ -491,9 +481,6 @@ mod tests { match output { Some(output) if output.checkpoint.block_number > initial_checkpoint => { let provider = self.db.factory.provider()?; - let mut td = provider - .header_td_by_number(initial_checkpoint.saturating_sub(1))? - .unwrap_or_default(); for block_num in initial_checkpoint..output.checkpoint.block_number { // look up the header hash @@ -507,10 +494,6 @@ mod tests { assert!(header.is_some()); let header = SealedHeader::seal_slow(header.unwrap()); assert_eq!(header.hash(), hash); - - // validate the header total difficulty - td += header.difficulty; - assert_eq!(provider.header_td_by_number(block_num)?, Some(td)); } } _ => self.check_no_header_entry_above(initial_checkpoint)?, @@ -565,10 +548,6 @@ mod tests { .ensure_no_entry_above_by_value::(block, |val| val)?; self.db.ensure_no_entry_above::(block, |key| key)?; self.db.ensure_no_entry_above::(block, |key| key)?; - self.db.ensure_no_entry_above::( - block, - |num| num, - )?; Ok(()) } @@ -623,30 +602,20 @@ mod tests { assert!(runner.stage().header_collector.is_empty()); // let's insert some blocks using append_blocks_with_state - let sealed_headers = - random_header_range(&mut generators::rng(), tip.number..tip.number + 10, tip.hash()); - - // make them sealed blocks with senders by converting them to empty blocks - let sealed_blocks = sealed_headers - .iter() - .map(|header| { - RecoveredBlock::new_sealed( - SealedBlock::from_sealed_parts(header.clone(), BlockBody::default()), - vec![], - ) - }) - .collect(); + let sealed_headers = random_header_range( + &mut generators::rng(), + tip.number + 1..tip.number + 10, + tip.hash(), + ); + + let provider = runner.db().factory.database_provider_rw().unwrap(); + let static_file_provider = provider.static_file_provider(); + let mut writer = static_file_provider.latest_writer(StaticFileSegment::Headers).unwrap(); + for header in sealed_headers { + writer.append_header(header.header(), &header.hash()).unwrap(); + } + drop(writer); - // append the blocks - let provider = runner.db().factory.provider_rw().unwrap(); - provider - .append_blocks_with_state( - sealed_blocks, - &ExecutionOutcome::default(), - HashedPostStateSorted::default(), - TrieUpdates::default(), - ) - .unwrap(); provider.commit().unwrap(); // now we can unwind 10 blocks diff --git a/crates/stages/stages/src/stages/index_account_history.rs b/crates/stages/stages/src/stages/index_account_history.rs index 37db4f5f9fd..c8d6464cf3f 100644 --- a/crates/stages/stages/src/stages/index_account_history.rs +++ b/crates/stages/stages/src/stages/index_account_history.rs @@ -67,23 +67,22 @@ where ) }) .transpose()? - .flatten() + .flatten() && + target_prunable_block > input.checkpoint().block_number { - if target_prunable_block > input.checkpoint().block_number { - input.checkpoint = Some(StageCheckpoint::new(target_prunable_block)); - - // Save prune checkpoint only if we don't have one already. - // Otherwise, pruner may skip the unpruned range of blocks. - if provider.get_prune_checkpoint(PruneSegment::AccountHistory)?.is_none() { - provider.save_prune_checkpoint( - PruneSegment::AccountHistory, - PruneCheckpoint { - block_number: Some(target_prunable_block), - tx_number: None, - prune_mode, - }, - )?; - } + input.checkpoint = Some(StageCheckpoint::new(target_prunable_block)); + + // Save prune checkpoint only if we don't have one already. + // Otherwise, pruner may skip the unpruned range of blocks. + if provider.get_prune_checkpoint(PruneSegment::AccountHistory)?.is_none() { + provider.save_prune_checkpoint( + PruneSegment::AccountHistory, + PruneCheckpoint { + block_number: Some(target_prunable_block), + tx_number: None, + prune_mode, + }, + )?; } } diff --git a/crates/stages/stages/src/stages/index_storage_history.rs b/crates/stages/stages/src/stages/index_storage_history.rs index 09c9030cb39..2ec4094c1ec 100644 --- a/crates/stages/stages/src/stages/index_storage_history.rs +++ b/crates/stages/stages/src/stages/index_storage_history.rs @@ -70,23 +70,22 @@ where ) }) .transpose()? - .flatten() + .flatten() && + target_prunable_block > input.checkpoint().block_number { - if target_prunable_block > input.checkpoint().block_number { - input.checkpoint = Some(StageCheckpoint::new(target_prunable_block)); - - // Save prune checkpoint only if we don't have one already. - // Otherwise, pruner may skip the unpruned range of blocks. - if provider.get_prune_checkpoint(PruneSegment::StorageHistory)?.is_none() { - provider.save_prune_checkpoint( - PruneSegment::StorageHistory, - PruneCheckpoint { - block_number: Some(target_prunable_block), - tx_number: None, - prune_mode, - }, - )?; - } + input.checkpoint = Some(StageCheckpoint::new(target_prunable_block)); + + // Save prune checkpoint only if we don't have one already. + // Otherwise, pruner may skip the unpruned range of blocks. + if provider.get_prune_checkpoint(PruneSegment::StorageHistory)?.is_none() { + provider.save_prune_checkpoint( + PruneSegment::StorageHistory, + PruneCheckpoint { + block_number: Some(target_prunable_block), + tx_number: None, + prune_mode, + }, + )?; } } diff --git a/crates/stages/stages/src/stages/merkle.rs b/crates/stages/stages/src/stages/merkle.rs index 55173876e96..a3a3ac88483 100644 --- a/crates/stages/stages/src/stages/merkle.rs +++ b/crates/stages/stages/src/stages/merkle.rs @@ -1,4 +1,4 @@ -use alloy_consensus::BlockHeader; +use alloy_consensus::{constants::KECCAK_EMPTY, BlockHeader}; use alloy_primitives::{BlockNumber, Sealable, B256}; use reth_codecs::Compact; use reth_consensus::ConsensusError; @@ -13,7 +13,7 @@ use reth_provider::{ }; use reth_stages_api::{ BlockErrorKind, EntitiesCheckpoint, ExecInput, ExecOutput, MerkleCheckpoint, Stage, - StageCheckpoint, StageError, StageId, UnwindInput, UnwindOutput, + StageCheckpoint, StageError, StageId, StorageRootMerkleCheckpoint, UnwindInput, UnwindOutput, }; use reth_trie::{IntermediateStateRootState, StateRoot, StateRootProgress, StoredSubNode}; use reth_trie_db::DatabaseStateRoot; @@ -40,11 +40,17 @@ Once you have this information, please submit a github issue at https://github.c /// The default threshold (in number of blocks) for switching from incremental trie building /// of changes to whole rebuild. -pub const MERKLE_STAGE_DEFAULT_CLEAN_THRESHOLD: u64 = 5_000; +pub const MERKLE_STAGE_DEFAULT_REBUILD_THRESHOLD: u64 = 100_000; + +/// The default threshold (in number of blocks) to run the stage in incremental mode. The +/// incremental mode will calculate the state root for a large range of blocks by calculating the +/// new state root for this many blocks, in batches, repeating until we reach the desired block +/// number. +pub const MERKLE_STAGE_DEFAULT_INCREMENTAL_THRESHOLD: u64 = 7_000; /// The merkle hashing stage uses input from /// [`AccountHashingStage`][crate::stages::AccountHashingStage] and -/// [`StorageHashingStage`][crate::stages::AccountHashingStage] to calculate intermediate hashes +/// [`StorageHashingStage`][crate::stages::StorageHashingStage] to calculate intermediate hashes /// and state roots. /// /// This stage should be run with the above two stages, otherwise it is a no-op. @@ -67,9 +73,15 @@ pub const MERKLE_STAGE_DEFAULT_CLEAN_THRESHOLD: u64 = 5_000; pub enum MerkleStage { /// The execution portion of the merkle stage. Execution { + // TODO: make struct for holding incremental settings, for code reuse between `Execution` + // variant and `Both` /// The threshold (in number of blocks) for switching from incremental trie building /// of changes to whole rebuild. - clean_threshold: u64, + rebuild_threshold: u64, + /// The threshold (in number of blocks) to run the stage in incremental mode. The + /// incremental mode will calculate the state root by calculating the new state root for + /// some number of blocks, repeating until we reach the desired block number. + incremental_threshold: u64, }, /// The unwind portion of the merkle stage. Unwind, @@ -78,14 +90,21 @@ pub enum MerkleStage { Both { /// The threshold (in number of blocks) for switching from incremental trie building /// of changes to whole rebuild. - clean_threshold: u64, + rebuild_threshold: u64, + /// The threshold (in number of blocks) to run the stage in incremental mode. The + /// incremental mode will calculate the state root by calculating the new state root for + /// some number of blocks, repeating until we reach the desired block number. + incremental_threshold: u64, }, } impl MerkleStage { /// Stage default for the [`MerkleStage::Execution`]. pub const fn default_execution() -> Self { - Self::Execution { clean_threshold: MERKLE_STAGE_DEFAULT_CLEAN_THRESHOLD } + Self::Execution { + rebuild_threshold: MERKLE_STAGE_DEFAULT_REBUILD_THRESHOLD, + incremental_threshold: MERKLE_STAGE_DEFAULT_INCREMENTAL_THRESHOLD, + } } /// Stage default for the [`MerkleStage::Unwind`]. @@ -94,8 +113,8 @@ impl MerkleStage { } /// Create new instance of [`MerkleStage::Execution`]. - pub const fn new_execution(clean_threshold: u64) -> Self { - Self::Execution { clean_threshold } + pub const fn new_execution(rebuild_threshold: u64, incremental_threshold: u64) -> Self { + Self::Execution { rebuild_threshold, incremental_threshold } } /// Gets the hashing progress @@ -154,14 +173,18 @@ where /// Execute the stage. fn execute(&mut self, provider: &Provider, input: ExecInput) -> Result { - let threshold = match self { + let (threshold, incremental_threshold) = match self { Self::Unwind => { info!(target: "sync::stages::merkle::unwind", "Stage is always skipped"); return Ok(ExecOutput::done(StageCheckpoint::new(input.target()))) } - Self::Execution { clean_threshold } => *clean_threshold, + Self::Execution { rebuild_threshold, incremental_threshold } => { + (*rebuild_threshold, *incremental_threshold) + } #[cfg(any(test, feature = "test-utils"))] - Self::Both { clean_threshold } => *clean_threshold, + Self::Both { rebuild_threshold, incremental_threshold } => { + (*rebuild_threshold, *incremental_threshold) + } }; let range = input.next_block_range(); @@ -173,10 +196,11 @@ where .ok_or_else(|| ProviderError::HeaderNotFound(to_block.into()))?; let target_block_root = target_block.state_root(); - let mut checkpoint = self.get_execution_checkpoint(provider)?; let (trie_root, entities_checkpoint) = if range.is_empty() { (target_block_root, input.checkpoint().entities_stage_checkpoint().unwrap_or_default()) } else if to_block - from_block > threshold || from_block == 1 { + let mut checkpoint = self.get_execution_checkpoint(provider)?; + // if there are more blocks than threshold it is faster to rebuild the trie let mut entities_checkpoint = if let Some(checkpoint) = checkpoint.as_ref().filter(|c| c.target_block == to_block) @@ -223,14 +247,37 @@ where })?; match progress { StateRootProgress::Progress(state, hashed_entries_walked, updates) => { - provider.write_trie_updates(&updates)?; + provider.write_trie_updates(updates)?; - let checkpoint = MerkleCheckpoint::new( + let mut checkpoint = MerkleCheckpoint::new( to_block, - state.last_account_key, - state.walker_stack.into_iter().map(StoredSubNode::from).collect(), - state.hash_builder.into(), + state.account_root_state.last_hashed_key, + state + .account_root_state + .walker_stack + .into_iter() + .map(StoredSubNode::from) + .collect(), + state.account_root_state.hash_builder.into(), ); + + // Save storage root state if present + if let Some(storage_state) = state.storage_root_state { + checkpoint.storage_root_checkpoint = + Some(StorageRootMerkleCheckpoint::new( + storage_state.state.last_hashed_key, + storage_state + .state + .walker_stack + .into_iter() + .map(StoredSubNode::from) + .collect(), + storage_state.state.hash_builder.into(), + storage_state.account.nonce, + storage_state.account.balance, + storage_state.account.bytecode_hash.unwrap_or(KECCAK_EMPTY), + )); + } self.save_execution_checkpoint(provider, Some(checkpoint))?; entities_checkpoint.processed += hashed_entries_walked as u64; @@ -243,7 +290,7 @@ where }) } StateRootProgress::Complete(root, hashed_entries_walked, updates) => { - provider.write_trie_updates(&updates)?; + provider.write_trie_updates(updates)?; entities_checkpoint.processed += hashed_entries_walked as u64; @@ -251,15 +298,33 @@ where } } } else { - debug!(target: "sync::stages::merkle::exec", current = ?current_block_number, target = ?to_block, "Updating trie"); - let (root, updates) = - StateRoot::incremental_root_with_updates(provider.tx_ref(), range) + debug!(target: "sync::stages::merkle::exec", current = ?current_block_number, target = ?to_block, "Updating trie in chunks"); + let mut final_root = None; + for start_block in range.step_by(incremental_threshold as usize) { + let chunk_to = std::cmp::min(start_block + incremental_threshold, to_block); + let chunk_range = start_block..=chunk_to; + debug!( + target: "sync::stages::merkle::exec", + current = ?current_block_number, + target = ?to_block, + incremental_threshold, + chunk_range = ?chunk_range, + "Processing chunk" + ); + let (root, updates) = + StateRoot::incremental_root_with_updates(provider.tx_ref(), chunk_range) .map_err(|e| { error!(target: "sync::stages::merkle", %e, ?current_block_number, ?to_block, "Incremental state root failed! {INVALID_STATE_ROOT_ERROR_MESSAGE}"); StageError::Fatal(Box::new(e)) })?; + provider.write_trie_updates(updates)?; + final_root = Some(root); + } - provider.write_trie_updates(&updates)?; + // if we had no final root, we must have not looped above, which should not be possible + let final_root = final_root.ok_or(StageError::Fatal( + "Incremental merkle hashing did not produce a final root".into(), + ))?; let total_hashed_entries = (provider.count_entries::()? + provider.count_entries::()?) @@ -272,8 +337,8 @@ where processed: total_hashed_entries, total: total_hashed_entries, }; - - (root, entities_checkpoint) + // Save the checkpoint + (final_root, entities_checkpoint) }; // Reset the checkpoint @@ -335,12 +400,21 @@ where validate_state_root(block_root, SealedHeader::seal_slow(target), input.unwind_to)?; // Validation passed, apply unwind changes to the database. - provider.write_trie_updates(&updates)?; - - // TODO(alexey): update entities checkpoint + provider.write_trie_updates(updates)?; + + // Update entities checkpoint to reflect the unwind operation + // Since we're unwinding, we need to recalculate the total entities at the target block + let accounts = tx.entries::()?; + let storages = tx.entries::()?; + let total = (accounts + storages) as u64; + entities_checkpoint.total = total; + entities_checkpoint.processed = total; } - Ok(UnwindOutput { checkpoint: StageCheckpoint::new(input.unwind_to) }) + Ok(UnwindOutput { + checkpoint: StageCheckpoint::new(input.unwind_to) + .with_entities_stage_checkpoint(entities_checkpoint), + }) } } @@ -419,8 +493,8 @@ mod tests { done: true }) if block_number == previous_stage && processed == total && total == ( - runner.db.table::().unwrap().len() + - runner.db.table::().unwrap().len() + runner.db.count_entries::().unwrap() + + runner.db.count_entries::().unwrap() ) as u64 ); @@ -459,8 +533,8 @@ mod tests { done: true }) if block_number == previous_stage && processed == total && total == ( - runner.db.table::().unwrap().len() + - runner.db.table::().unwrap().len() + runner.db.count_entries::().unwrap() + + runner.db.count_entries::().unwrap() ) as u64 ); @@ -468,14 +542,79 @@ mod tests { assert!(runner.validate_execution(input, result.ok()).is_ok(), "execution validation"); } + #[tokio::test] + async fn execute_chunked_merkle() { + let (previous_stage, stage_progress) = (200, 100); + let clean_threshold = 100; + let incremental_threshold = 10; + + // Set up the runner + let mut runner = + MerkleTestRunner { db: TestStageDB::default(), clean_threshold, incremental_threshold }; + + let input = ExecInput { + target: Some(previous_stage), + checkpoint: Some(StageCheckpoint::new(stage_progress)), + }; + + runner.seed_execution(input).expect("failed to seed execution"); + let rx = runner.execute(input); + + // Assert the successful result + let result = rx.await.unwrap(); + assert_matches!( + result, + Ok(ExecOutput { + checkpoint: StageCheckpoint { + block_number, + stage_checkpoint: Some(StageUnitCheckpoint::Entities(EntitiesCheckpoint { + processed, + total + })) + }, + done: true + }) if block_number == previous_stage && processed == total && + total == ( + runner.db.count_entries::().unwrap() + + runner.db.count_entries::().unwrap() + ) as u64 + ); + + // Validate the stage execution + let provider = runner.db.factory.provider().unwrap(); + let header = provider.header_by_number(previous_stage).unwrap().unwrap(); + let expected_root = header.state_root; + + let actual_root = runner + .db + .query(|tx| { + Ok(StateRoot::incremental_root_with_updates( + tx, + stage_progress + 1..=previous_stage, + )) + }) + .unwrap(); + + assert_eq!( + actual_root.unwrap().0, + expected_root, + "State root mismatch after chunked processing" + ); + } + struct MerkleTestRunner { db: TestStageDB, clean_threshold: u64, + incremental_threshold: u64, } impl Default for MerkleTestRunner { fn default() -> Self { - Self { db: TestStageDB::default(), clean_threshold: 10000 } + Self { + db: TestStageDB::default(), + clean_threshold: 10000, + incremental_threshold: 10000, + } } } @@ -487,7 +626,10 @@ mod tests { } fn stage(&self) -> Self::S { - Self::S::Both { clean_threshold: self.clean_threshold } + Self::S::Both { + rebuild_threshold: self.clean_threshold, + incremental_threshold: self.incremental_threshold, + } } } @@ -596,7 +738,7 @@ mod tests { let hash = last_header.hash_slow(); writer.prune_headers(1).unwrap(); writer.commit().unwrap(); - writer.append_header(&last_header, U256::ZERO, &hash).unwrap(); + writer.append_header(&last_header, &hash).unwrap(); writer.commit().unwrap(); Ok(blocks) diff --git a/crates/stages/stages/src/stages/merkle_changesets.rs b/crates/stages/stages/src/stages/merkle_changesets.rs new file mode 100644 index 00000000000..dd4d8cf2017 --- /dev/null +++ b/crates/stages/stages/src/stages/merkle_changesets.rs @@ -0,0 +1,401 @@ +use crate::stages::merkle::INVALID_STATE_ROOT_ERROR_MESSAGE; +use alloy_consensus::BlockHeader; +use alloy_primitives::BlockNumber; +use reth_consensus::ConsensusError; +use reth_primitives_traits::{GotExpected, SealedHeader}; +use reth_provider::{ + ChainStateBlockReader, DBProvider, HeaderProvider, ProviderError, PruneCheckpointReader, + PruneCheckpointWriter, StageCheckpointReader, TrieWriter, +}; +use reth_prune_types::{PruneCheckpoint, PruneMode, PruneSegment}; +use reth_stages_api::{ + BlockErrorKind, ExecInput, ExecOutput, Stage, StageCheckpoint, StageError, StageId, + UnwindInput, UnwindOutput, +}; +use reth_trie::{updates::TrieUpdates, HashedPostState, KeccakKeyHasher, StateRoot, TrieInput}; +use reth_trie_db::{DatabaseHashedPostState, DatabaseStateRoot}; +use std::ops::Range; +use tracing::{debug, error}; + +/// The `MerkleChangeSets` stage. +/// +/// This stage processes and maintains trie changesets from the finalized block to the latest block. +#[derive(Debug, Clone)] +pub struct MerkleChangeSets { + /// The number of blocks to retain changesets for, used as a fallback when the finalized block + /// is not found. Defaults to 64 (2 epochs in beacon chain). + retention_blocks: u64, +} + +impl MerkleChangeSets { + /// Creates a new `MerkleChangeSets` stage with default retention blocks of 64. + pub const fn new() -> Self { + Self { retention_blocks: 64 } + } + + /// Creates a new `MerkleChangeSets` stage with a custom finalized block height. + pub const fn with_retention_blocks(retention_blocks: u64) -> Self { + Self { retention_blocks } + } + + /// Returns the range of blocks which are already computed. Will return an empty range if none + /// have been computed. + fn computed_range( + provider: &Provider, + checkpoint: Option, + ) -> Result, StageError> + where + Provider: PruneCheckpointReader, + { + let to = checkpoint.map(|chk| chk.block_number).unwrap_or_default(); + + // Get the prune checkpoint for MerkleChangeSets to use as the lower bound. If there's no + // prune checkpoint or if the pruned block number is None, return empty range + let Some(from) = provider + .get_prune_checkpoint(PruneSegment::MerkleChangeSets)? + .and_then(|chk| chk.block_number) + // prune checkpoint indicates the last block pruned, so the block after is the start of + // the computed data + .map(|block_number| block_number + 1) + else { + return Ok(0..0) + }; + + Ok(from..to + 1) + } + + /// Determines the target range for changeset computation based on the checkpoint and provider + /// state. + /// + /// Returns the target range (exclusive end) to compute changesets for. + fn determine_target_range( + &self, + provider: &Provider, + ) -> Result, StageError> + where + Provider: StageCheckpointReader + ChainStateBlockReader, + { + // Get merkle checkpoint which represents our target end block + let merkle_checkpoint = provider + .get_stage_checkpoint(StageId::MerkleExecute)? + .map(|checkpoint| checkpoint.block_number) + .unwrap_or(0); + + let target_end = merkle_checkpoint + 1; // exclusive + + // Calculate the target range based on the finalized block and the target block. + // We maintain changesets from the finalized block to the latest block. + let finalized_block = provider.last_finalized_block_number()?; + + // Calculate the fallback start position based on retention blocks + let retention_based_start = merkle_checkpoint.saturating_sub(self.retention_blocks); + + // If the finalized block was way in the past then we don't want to generate changesets for + // all of those past blocks; we only care about the recent history. + // + // Use maximum of finalized_block and retention_based_start if finalized_block exists, + // otherwise just use retention_based_start. + let mut target_start = finalized_block + .map(|finalized| finalized.saturating_add(1).max(retention_based_start)) + .unwrap_or(retention_based_start); + + // We cannot revert the genesis block; target_start must be >0 + target_start = target_start.max(1); + + Ok(target_start..target_end) + } + + /// Calculates the trie updates given a [`TrieInput`], asserting that the resulting state root + /// matches the expected one for the block. + fn calculate_block_trie_updates( + provider: &Provider, + block_number: BlockNumber, + input: TrieInput, + ) -> Result { + let (root, trie_updates) = + StateRoot::overlay_root_from_nodes_with_updates(provider.tx_ref(), input).map_err( + |e| { + error!( + target: "sync::stages::merkle_changesets", + %e, + ?block_number, + "Incremental state root failed! {INVALID_STATE_ROOT_ERROR_MESSAGE}"); + StageError::Fatal(Box::new(e)) + }, + )?; + + let block = provider + .header_by_number(block_number)? + .ok_or_else(|| ProviderError::HeaderNotFound(block_number.into()))?; + + let (got, expected) = (root, block.state_root()); + if got != expected { + // Only seal the header when we need it for the error + let header = SealedHeader::seal_slow(block); + error!( + target: "sync::stages::merkle_changesets", + ?block_number, + ?got, + ?expected, + "Failed to verify block state root! {INVALID_STATE_ROOT_ERROR_MESSAGE}", + ); + return Err(StageError::Block { + error: BlockErrorKind::Validation(ConsensusError::BodyStateRootDiff( + GotExpected { got, expected }.into(), + )), + block: Box::new(header.block_with_parent()), + }) + } + + Ok(trie_updates) + } + + fn populate_range( + provider: &Provider, + target_range: Range, + ) -> Result<(), StageError> + where + Provider: StageCheckpointReader + + TrieWriter + + DBProvider + + HeaderProvider + + ChainStateBlockReader, + { + let target_start = target_range.start; + let target_end = target_range.end; + debug!( + target: "sync::stages::merkle_changesets", + ?target_range, + "Starting trie changeset computation", + ); + + // We need to distinguish a cumulative revert and a per-block revert. A cumulative revert + // reverts changes starting at db tip all the way to a block. A per-block revert only + // reverts a block's changes. + // + // We need to calculate the cumulative HashedPostState reverts for every block in the + // target range. The cumulative HashedPostState revert for block N can be calculated as: + // + // + // ``` + // // where `extend` overwrites any shared keys + // cumulative_state_revert(N) = cumulative_state_revert(N + 1).extend(get_block_state_revert(N)) + // ``` + // + // We need per-block reverts to calculate the prefix set for each individual block. By + // using the per-block reverts to calculate cumulative reverts on-the-fly we can save a + // bunch of memory. + debug!( + target: "sync::stages::merkle_changesets", + ?target_range, + "Computing per-block state reverts", + ); + let mut per_block_state_reverts = Vec::new(); + for block_number in target_range.clone() { + per_block_state_reverts.push(HashedPostState::from_reverts::( + provider.tx_ref(), + block_number..=block_number, + )?); + } + + // Helper to retrieve state revert data for a specific block from the pre-computed array + let get_block_state_revert = |block_number: BlockNumber| -> &HashedPostState { + let index = (block_number - target_start) as usize; + &per_block_state_reverts[index] + }; + + // Helper to accumulate state reverts from a given block to the target end + let compute_cumulative_state_revert = |block_number: BlockNumber| -> HashedPostState { + let mut cumulative_revert = HashedPostState::default(); + for n in (block_number..target_end).rev() { + cumulative_revert.extend_ref(get_block_state_revert(n)) + } + cumulative_revert + }; + + // To calculate the changeset for a block, we first need the TrieUpdates which are + // generated as a result of processing the block. To get these we need: + // 1) The TrieUpdates which revert the db's trie to _prior_ to the block + // 2) The HashedPostState to revert the db's state to _after_ the block + // + // To get (1) for `target_start` we need to do a big state root calculation which takes + // into account all changes between that block and db tip. For each block after the + // `target_start` we can update (1) using the TrieUpdates which were output by the previous + // block, only targeting the state changes of that block. + debug!( + target: "sync::stages::merkle_changesets", + ?target_start, + "Computing trie state at starting block", + ); + let mut input = TrieInput::default(); + input.state = compute_cumulative_state_revert(target_start); + input.prefix_sets = input.state.construct_prefix_sets(); + // target_start will be >= 1, see `determine_target_range`. + input.nodes = + Self::calculate_block_trie_updates(provider, target_start - 1, input.clone())?; + + for block_number in target_range { + debug!( + target: "sync::stages::merkle_changesets", + ?block_number, + "Computing trie updates for block", + ); + // Revert the state so that this block has been just processed, meaning we take the + // cumulative revert of the subsequent block. + input.state = compute_cumulative_state_revert(block_number + 1); + + // Construct prefix sets from only this block's `HashedPostState`, because we only care + // about trie updates which occurred as a result of this block being processed. + input.prefix_sets = get_block_state_revert(block_number).construct_prefix_sets(); + + // Calculate the trie updates for this block, then apply those updates to the reverts. + // We calculate the overlay which will be passed into the next step using the trie + // reverts prior to them being updated. + let this_trie_updates = + Self::calculate_block_trie_updates(provider, block_number, input.clone())?; + + let trie_overlay = input.nodes.clone().into_sorted(); + input.nodes.extend_ref(&this_trie_updates); + let this_trie_updates = this_trie_updates.into_sorted(); + + // Write the changesets to the DB using the trie updates produced by the block, and the + // trie reverts as the overlay. + debug!( + target: "sync::stages::merkle_changesets", + ?block_number, + "Writing trie changesets for block", + ); + provider.write_trie_changesets( + block_number, + &this_trie_updates, + Some(&trie_overlay), + )?; + } + + Ok(()) + } +} + +impl Default for MerkleChangeSets { + fn default() -> Self { + Self::new() + } +} + +impl Stage for MerkleChangeSets +where + Provider: StageCheckpointReader + + TrieWriter + + DBProvider + + HeaderProvider + + ChainStateBlockReader + + PruneCheckpointReader + + PruneCheckpointWriter, +{ + fn id(&self) -> StageId { + StageId::MerkleChangeSets + } + + fn execute(&mut self, provider: &Provider, input: ExecInput) -> Result { + // Get merkle checkpoint and assert that the target is the same. + let merkle_checkpoint = provider + .get_stage_checkpoint(StageId::MerkleExecute)? + .map(|checkpoint| checkpoint.block_number) + .unwrap_or(0); + + if input.target.is_none_or(|target| merkle_checkpoint != target) { + return Err(StageError::Fatal(eyre::eyre!("Cannot sync stage to block {:?} when MerkleExecute is at block {merkle_checkpoint:?}", input.target).into())) + } + + let mut target_range = self.determine_target_range(provider)?; + + // Get the previously computed range. This will be updated to reflect the populating of the + // target range. + let mut computed_range = Self::computed_range(provider, input.checkpoint)?; + debug!( + target: "sync::stages::merkle_changesets", + ?computed_range, + ?target_range, + "Got computed and target ranges", + ); + + // We want the target range to not include any data already computed previously, if + // possible, so we start the target range from the end of the computed range if that is + // greater. + // + // ------------------------------> Block # + // |------computed-----| + // |-----target-----| + // |--actual--| + // + // However, if the target start is less than the previously computed start, we don't want to + // do this, as it would leave a gap of data at `target_range.start..=computed_range.start`. + // + // ------------------------------> Block # + // |---computed---| + // |-------target-------| + // |-------actual-------| + // + if target_range.start >= computed_range.start { + target_range.start = target_range.start.max(computed_range.end); + } + + // If target range is empty (target_start >= target_end), stage is already successfully + // executed. + if target_range.start >= target_range.end { + return Ok(ExecOutput::done(StageCheckpoint::new(target_range.end.saturating_sub(1)))); + } + + // If our target range is a continuation of the already computed range then we can keep the + // already computed data. + if target_range.start == computed_range.end { + // Clear from target_start onwards to ensure no stale data exists + provider.clear_trie_changesets_from(target_range.start)?; + computed_range.end = target_range.end; + } else { + // If our target range is not a continuation of the already computed range then we + // simply clear the computed data, to make sure there's no gaps or conflicts. + provider.clear_trie_changesets()?; + computed_range = target_range.clone(); + } + + // Populate the target range with changesets + Self::populate_range(provider, target_range)?; + + // Update the prune checkpoint to reflect that all data before `computed_range.start` + // is not available. + provider.save_prune_checkpoint( + PruneSegment::MerkleChangeSets, + PruneCheckpoint { + block_number: Some(computed_range.start.saturating_sub(1)), + tx_number: None, + prune_mode: PruneMode::Before(computed_range.start), + }, + )?; + + // `computed_range.end` is exclusive. + let checkpoint = StageCheckpoint::new(computed_range.end.saturating_sub(1)); + + Ok(ExecOutput::done(checkpoint)) + } + + fn unwind( + &mut self, + provider: &Provider, + input: UnwindInput, + ) -> Result { + // Unwinding is trivial; just clear everything after the target block. + provider.clear_trie_changesets_from(input.unwind_to + 1)?; + + let mut computed_range = Self::computed_range(provider, Some(input.checkpoint))?; + computed_range.end = input.unwind_to + 1; + if computed_range.start > computed_range.end { + computed_range.start = computed_range.end; + } + + // `computed_range.end` is exclusive + let checkpoint = StageCheckpoint::new(computed_range.end.saturating_sub(1)); + + Ok(UnwindOutput { checkpoint }) + } +} diff --git a/crates/stages/stages/src/stages/mod.rs b/crates/stages/stages/src/stages/mod.rs index 8977fa8a10b..58fa7cfb324 100644 --- a/crates/stages/stages/src/stages/mod.rs +++ b/crates/stages/stages/src/stages/mod.rs @@ -16,15 +16,16 @@ mod index_account_history; mod index_storage_history; /// Stage for computing state root. mod merkle; +/// Stage for computing merkle changesets. +mod merkle_changesets; mod prune; -/// The s3 download stage -mod s3; /// The sender recovery stage. mod sender_recovery; /// The transaction lookup stage mod tx_lookup; pub use bodies::*; +pub use era::*; pub use execution::*; pub use finish::*; pub use hashing_account::*; @@ -33,12 +34,14 @@ pub use headers::*; pub use index_account_history::*; pub use index_storage_history::*; pub use merkle::*; +pub use merkle_changesets::*; pub use prune::*; -pub use s3::*; pub use sender_recovery::*; pub use tx_lookup::*; +mod era; mod utils; + use utils::*; #[cfg(test)] @@ -61,15 +64,15 @@ mod tests { }; use reth_ethereum_consensus::EthBeaconConsensus; use reth_ethereum_primitives::Block; - use reth_evm_ethereum::execute::EthExecutorProvider; + use reth_evm_ethereum::EthEvmConfig; use reth_exex::ExExManagerHandle; use reth_primitives_traits::{Account, Bytecode, SealedBlock}; use reth_provider::{ providers::{StaticFileProvider, StaticFileWriter}, test_utils::MockNodeTypesWithDB, - AccountExtReader, BlockBodyIndicesProvider, DatabaseProviderFactory, ProviderFactory, - ProviderResult, ReceiptProvider, StageCheckpointWriter, StaticFileProviderFactory, - StorageReader, + AccountExtReader, BlockBodyIndicesProvider, BlockWriter, DatabaseProviderFactory, + ProviderFactory, ProviderResult, ReceiptProvider, StageCheckpointWriter, + StaticFileProviderFactory, StorageReader, }; use reth_prune_types::{PruneMode, PruneModes}; use reth_stages_api::{ @@ -93,8 +96,8 @@ mod tests { let genesis = SealedBlock::::decode(&mut genesis_rlp).unwrap(); let mut block_rlp = hex!("f90262f901f9a075c371ba45999d87f4542326910a11af515897aebce5265d3f6acd1f1161f82fa01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347942adc25665018aa1fe0e6bc666dac8fc2697ff9baa098f2dcd87c8ae4083e7017a05456c14eea4b1db2032126e27b3b1563d57d7cc0a08151d548273f6683169524b66ca9fe338b9ce42bc3540046c828fd939ae23bcba03f4e5c2ec5b2170b711d97ee755c160457bb58d8daa338e835ec02ae6860bbabb901000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000083020000018502540be40082a8798203e800a00000000000000000000000000000000000000000000000000000000000000000880000000000000000f863f861800a8405f5e10094100000000000000000000000000000000000000080801ba07e09e26678ed4fac08a249ebe8ed680bf9051a5e14ad223e4b2b9d26e0208f37a05f6e3f188e3e6eab7d7d3b6568f5eac7d687b08d307d3154ccd8c87b4630509bc0").as_slice(); let block = SealedBlock::::decode(&mut block_rlp).unwrap(); - provider_rw.insert_historical_block(genesis.try_recover().unwrap()).unwrap(); - provider_rw.insert_historical_block(block.clone().try_recover().unwrap()).unwrap(); + provider_rw.insert_block(genesis.try_recover().unwrap()).unwrap(); + provider_rw.insert_block(block.clone().try_recover().unwrap()).unwrap(); // Fill with bogus blocks to respect PruneMode distance. let mut head = block.hash(); @@ -106,7 +109,7 @@ mod tests { generators::BlockParams { parent: Some(head), ..Default::default() }, ); head = nblock.hash(); - provider_rw.insert_historical_block(nblock.try_recover().unwrap()).unwrap(); + provider_rw.insert_block(nblock.try_recover().unwrap()).unwrap(); } provider_rw .static_file_provider() @@ -154,7 +157,7 @@ mod tests { // Check execution and create receipts and changesets according to the pruning // configuration let mut execution_stage = ExecutionStage::new( - EthExecutorProvider::ethereum(Arc::new( + EthEvmConfig::ethereum(Arc::new( ChainSpecBuilder::mainnet().berlin_activated().build(), )), Arc::new(EthBeaconConsensus::new(Arc::new( @@ -166,7 +169,7 @@ mod tests { max_cumulative_gas: None, max_duration: None, }, - MERKLE_STAGE_DEFAULT_CLEAN_THRESHOLD, + MERKLE_STAGE_DEFAULT_REBUILD_THRESHOLD, ExExManagerHandle::empty(), ); @@ -210,7 +213,7 @@ mod tests { if prune_modes.storage_history == Some(PruneMode::Full) { // Full is not supported - assert!(acc_indexing_stage.execute(&provider, input).is_err()); + assert!(storage_indexing_stage.execute(&provider, input).is_err()); } else { storage_indexing_stage.execute(&provider, input).unwrap(); @@ -225,7 +228,7 @@ mod tests { // In an unpruned configuration there is 1 receipt, 3 changed accounts and 1 changed // storage. - let mut prune = PruneModes::none(); + let mut prune = PruneModes::default(); check_pruning(test_db.factory.clone(), prune.clone(), 1, 3, 1).await; prune.receipts = Some(PruneMode::Full); @@ -277,7 +280,7 @@ mod tests { for block in &blocks { let mut block_receipts = Vec::with_capacity(block.transaction_count()); for transaction in &block.body().transactions { - block_receipts.push((tx_num, random_receipt(&mut rng, transaction, Some(0)))); + block_receipts.push((tx_num, random_receipt(&mut rng, transaction, Some(0), None))); tx_num += 1; } receipts.push((block.number, block_receipts)); diff --git a/crates/stages/stages/src/stages/prune.rs b/crates/stages/stages/src/stages/prune.rs index 1eab645580c..f6fb7f90ae1 100644 --- a/crates/stages/stages/src/stages/prune.rs +++ b/crates/stages/stages/src/stages/prune.rs @@ -1,7 +1,7 @@ use reth_db_api::{table::Value, transaction::DbTxMut}; use reth_primitives_traits::NodePrimitives; use reth_provider::{ - BlockReader, DBProvider, PruneCheckpointReader, PruneCheckpointWriter, + BlockReader, ChainStateBlockReader, DBProvider, PruneCheckpointReader, PruneCheckpointWriter, StaticFileProviderFactory, }; use reth_prune::{ @@ -42,7 +42,10 @@ where + PruneCheckpointReader + PruneCheckpointWriter + BlockReader - + StaticFileProviderFactory>, + + ChainStateBlockReader + + StaticFileProviderFactory< + Primitives: NodePrimitives, + >, { fn id(&self) -> StageId { StageId::Prune @@ -100,9 +103,18 @@ where // We cannot recover the data that was pruned in `execute`, so we just update the // checkpoints. let prune_checkpoints = provider.get_prune_checkpoints()?; + let unwind_to_last_tx = + provider.block_body_indices(input.unwind_to)?.map(|i| i.last_tx_num()); + for (segment, mut checkpoint) in prune_checkpoints { - checkpoint.block_number = Some(input.unwind_to); - provider.save_prune_checkpoint(segment, checkpoint)?; + // Only update the checkpoint if unwind_to is lower than the existing checkpoint. + if let Some(block) = checkpoint.block_number && + input.unwind_to < block + { + checkpoint.block_number = Some(input.unwind_to); + checkpoint.tx_number = unwind_to_last_tx; + provider.save_prune_checkpoint(segment, checkpoint)?; + } } Ok(UnwindOutput { checkpoint: StageCheckpoint::new(input.unwind_to) }) } @@ -119,7 +131,7 @@ impl PruneSenderRecoveryStage { /// Create new prune sender recovery stage with the given prune mode and commit threshold. pub fn new(prune_mode: PruneMode, commit_threshold: usize) -> Self { Self(PruneStage::new( - PruneModes { sender_recovery: Some(prune_mode), ..PruneModes::none() }, + PruneModes { sender_recovery: Some(prune_mode), ..PruneModes::default() }, commit_threshold, )) } @@ -131,7 +143,10 @@ where + PruneCheckpointReader + PruneCheckpointWriter + BlockReader - + StaticFileProviderFactory>, + + ChainStateBlockReader + + StaticFileProviderFactory< + Primitives: NodePrimitives, + >, { fn id(&self) -> StageId { StageId::PruneSenderRecovery @@ -172,7 +187,7 @@ mod tests { }; use alloy_primitives::B256; use reth_ethereum_primitives::Block; - use reth_primitives_traits::SealedBlock; + use reth_primitives_traits::{SealedBlock, SignerRecoverable}; use reth_provider::{ providers::StaticFileWriter, TransactionsProvider, TransactionsProviderExt, }; diff --git a/crates/stages/stages/src/stages/s3/downloader/error.rs b/crates/stages/stages/src/stages/s3/downloader/error.rs deleted file mode 100644 index 49f4b418aad..00000000000 --- a/crates/stages/stages/src/stages/s3/downloader/error.rs +++ /dev/null @@ -1,31 +0,0 @@ -use alloy_primitives::B256; -use reth_fs_util::FsPathError; - -/// Possible downloader error variants. -#[derive(Debug, thiserror::Error)] -pub enum DownloaderError { - /// Requires a valid `total_size` {0} - #[error("requires a valid total_size")] - InvalidMetadataTotalSize(Option), - #[error("tried to access chunk on index {0}, but there's only {1} chunks")] - /// Invalid chunk access - InvalidChunk(usize, usize), - // File hash mismatch. - #[error("file hash does not match the expected one {0} != {1} ")] - InvalidFileHash(B256, B256), - // Empty content length returned from the server. - #[error("metadata got an empty content length from server")] - EmptyContentLength, - /// Reqwest error - #[error(transparent)] - FsPath(#[from] FsPathError), - /// Reqwest error - #[error(transparent)] - Reqwest(#[from] reqwest::Error), - /// Std Io error - #[error(transparent)] - StdIo(#[from] std::io::Error), - /// Bincode error - #[error(transparent)] - Bincode(#[from] bincode::Error), -} diff --git a/crates/stages/stages/src/stages/s3/downloader/fetch.rs b/crates/stages/stages/src/stages/s3/downloader/fetch.rs deleted file mode 100644 index 1d8bba739fd..00000000000 --- a/crates/stages/stages/src/stages/s3/downloader/fetch.rs +++ /dev/null @@ -1,184 +0,0 @@ -use crate::stages::s3::downloader::{worker::spawn_workers, RemainingChunkRange}; - -use super::{ - error::DownloaderError, - meta::Metadata, - worker::{WorkerRequest, WorkerResponse}, -}; -use alloy_primitives::B256; -use reqwest::{header::CONTENT_LENGTH, Client}; -use std::{ - collections::HashMap, - fs::{File, OpenOptions}, - io::BufReader, - path::Path, -}; -use tracing::{debug, error, info}; - -/// Downloads file from url to data file path. -/// -/// If a `file_hash` is passed, it will verify it at the end. -/// -/// ## Details -/// -/// 1) A [`Metadata`] file is created or opened in `{target_dir}/download/{filename}.metadata`. It -/// tracks the download progress including total file size, downloaded bytes, chunk sizes, and -/// ranges that still need downloading. Allows for resumability. -/// 2) The target file is preallocated with the total size of the file in -/// `{target_dir}/download/{filename}`. -/// 3) Multiple `workers` are spawned for downloading of specific chunks of the file. -/// 4) `Orchestrator` manages workers, distributes chunk ranges, and ensures the download progresses -/// efficiently by dynamically assigning tasks to workers as they become available. -/// 5) Once the file is downloaded: -/// * If `file_hash` is `Some`, verifies its blake3 hash. -/// * Deletes the metadata file -/// * Moves downloaded file to target directory. -pub async fn fetch( - filename: &str, - target_dir: &Path, - url: &str, - mut concurrent: u64, - file_hash: Option, -) -> Result<(), DownloaderError> { - // Create a temporary directory to download files to, before moving them to target_dir. - let download_dir = target_dir.join("download"); - reth_fs_util::create_dir_all(&download_dir)?; - - let data_file = download_dir.join(filename); - let mut metadata = metadata(&data_file, url).await?; - if metadata.is_done() { - return Ok(()) - } - - // Ensure the file is preallocated so we can download it concurrently - { - let file = OpenOptions::new() - .create(true) - .truncate(true) - .read(true) - .write(true) - .open(&data_file)?; - - if file.metadata()?.len() as usize != metadata.total_size { - info!(target: "sync::stages::s3::downloader", ?filename, length = metadata.total_size, "Preallocating space."); - file.set_len(metadata.total_size as u64)?; - } - } - - while !metadata.is_done() { - info!(target: "sync::stages::s3::downloader", ?filename, "Downloading."); - - // Find the missing file chunks and the minimum number of workers required - let missing_chunks = metadata.needed_ranges(); - concurrent = concurrent - .min(std::thread::available_parallelism()?.get() as u64) - .min(missing_chunks.len() as u64); - - let mut orchestrator_rx = spawn_workers(url, concurrent, &data_file); - - let mut workers = HashMap::new(); - let mut missing_chunks = missing_chunks.into_iter(); - - // Distribute chunk ranges to workers when they free up - while let Some(worker_msg) = orchestrator_rx.recv().await { - debug!(target: "sync::stages::s3::downloader", ?worker_msg, "received message from worker"); - - let available_worker = match worker_msg { - WorkerResponse::Ready { worker_id, tx } => { - debug!(target: "sync::stages::s3::downloader", ?worker_id, "Worker ready."); - workers.insert(worker_id, tx); - worker_id - } - WorkerResponse::DownloadedChunk { worker_id, chunk_index, written_bytes } => { - metadata.update_chunk(chunk_index, written_bytes)?; - worker_id - } - WorkerResponse::Err { worker_id, error } => { - error!(target: "sync::stages::s3::downloader", ?worker_id, "Worker found an error: {:?}", error); - return Err(error) - } - }; - - let msg = if let Some(RemainingChunkRange { index, start, end }) = missing_chunks.next() - { - debug!(target: "sync::stages::s3::downloader", ?available_worker, start, end, "Worker download request."); - WorkerRequest::Download { chunk_index: index, start, end } - } else { - debug!(target: "sync::stages::s3::downloader", ?available_worker, "Sent Finish command to worker."); - WorkerRequest::Finish - }; - - let _ = workers.get(&available_worker).expect("should exist").send(msg); - } - } - - if let Some(file_hash) = file_hash { - info!(target: "sync::stages::s3::downloader", ?filename, "Checking file integrity."); - check_file_hash(&data_file, &file_hash)?; - } - - // No longer need the metadata file. - metadata.delete()?; - - // Move downloaded file to desired directory. - let file_directory = target_dir.join(filename); - reth_fs_util::rename(data_file, &file_directory)?; - info!(target: "sync::stages::s3::downloader", ?file_directory, "Moved file from temporary to target directory."); - - Ok(()) -} - -/// Creates a metadata file used to keep track of the downloaded chunks. Useful on resuming after a -/// shutdown. -async fn metadata(data_file: &Path, url: &str) -> Result { - if Metadata::file_path(data_file).exists() { - debug!(target: "sync::stages::s3::downloader", ?data_file, "Loading metadata "); - return Metadata::load(data_file) - } - - let client = Client::new(); - let resp = client.head(url).send().await?; - let total_length: usize = resp - .headers() - .get(CONTENT_LENGTH) - .and_then(|v| v.to_str().ok()) - .and_then(|s| s.parse().ok()) - .ok_or(DownloaderError::EmptyContentLength)?; - - debug!(target: "sync::stages::s3::downloader", ?data_file, "Creating metadata "); - - Metadata::builder(data_file).with_total_size(total_length).build() -} - -/// Ensures the file on path has the expected blake3 hash. -fn check_file_hash(path: &Path, expected: &B256) -> Result<(), DownloaderError> { - let mut reader = BufReader::new(File::open(path)?); - let mut hasher = blake3::Hasher::new(); - std::io::copy(&mut reader, &mut hasher)?; - - let file_hash = hasher.finalize(); - if file_hash.as_bytes() != expected { - return Err(DownloaderError::InvalidFileHash(file_hash.as_bytes().into(), *expected)) - } - - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - use alloy_primitives::b256; - - #[tokio::test] - async fn test_download() { - reth_tracing::init_test_tracing(); - - let b3sum = b256!("0xe9908f4992ae39c4d1fe9984dd743ae3f8e9a84a4a5af768128833605ff72723"); - let url = "https://link.testfile.org/15MB"; - - let file = tempfile::NamedTempFile::new().unwrap(); - let filename = file.path().file_name().unwrap().to_str().unwrap(); - let target_dir = file.path().parent().unwrap(); - fetch(filename, target_dir, url, 4, Some(b3sum)).await.unwrap(); - } -} diff --git a/crates/stages/stages/src/stages/s3/downloader/meta.rs b/crates/stages/stages/src/stages/s3/downloader/meta.rs deleted file mode 100644 index 7ff4213fffc..00000000000 --- a/crates/stages/stages/src/stages/s3/downloader/meta.rs +++ /dev/null @@ -1,195 +0,0 @@ -use super::{error::DownloaderError, RemainingChunkRange}; -use serde::{Deserialize, Serialize}; -use std::{ - fs::File, - ops::RangeInclusive, - path::{Path, PathBuf}, -}; -use tracing::info; - -/// Tracks download progress and manages chunked downloads for resumable file transfers. -#[derive(Debug)] -pub struct Metadata { - /// Total file size - pub total_size: usize, - /// Total file size - pub downloaded: usize, - /// Download chunk size. Default 150MB. - pub chunk_size: usize, - /// Remaining download ranges for each chunk. - /// - `Some(RangeInclusive)`: range to be downloaded. - /// - `None`: Chunk fully downloaded. - chunks: Vec>>, - /// Path with the stored metadata. - path: PathBuf, -} - -impl Metadata { - /// Build a [`Metadata`] using a builder. - pub fn builder(data_file: &Path) -> MetadataBuilder { - MetadataBuilder::new(Self::file_path(data_file)) - } - - /// Returns the metadata file path of a data file: `{data_file}.metadata` - pub fn file_path(data_file: &Path) -> PathBuf { - data_file.with_file_name(format!( - "{}.metadata", - data_file.file_name().unwrap_or_default().to_string_lossy() - )) - } - - /// Returns a list of all chunks with their remaining ranges to be downloaded: - /// `RemainingChunkRange`. - pub fn needed_ranges(&self) -> Vec { - self.chunks - .iter() - .enumerate() - .filter(|(_, remaining)| remaining.is_some()) - .map(|(index, remaining)| { - let range = remaining.as_ref().expect("qed"); - RemainingChunkRange { index, start: *range.start(), end: *range.end() } - }) - .collect() - } - - /// Updates a downloaded chunk. - pub fn update_chunk( - &mut self, - index: usize, - downloaded_bytes: usize, - ) -> Result<(), DownloaderError> { - self.downloaded += downloaded_bytes; - - let num_chunks = self.chunks.len(); - if index >= self.chunks.len() { - return Err(DownloaderError::InvalidChunk(index, num_chunks)) - } - - // Update chunk with downloaded range - if let Some(range) = &self.chunks[index] { - let start = range.start() + downloaded_bytes; - if start > *range.end() { - self.chunks[index] = None; - } else { - self.chunks[index] = Some(start..=*range.end()); - } - } - - let file = self.path.file_stem().unwrap_or_default().to_string_lossy().into_owned(); - info!( - target: "sync::stages::s3::downloader", - file, - "{}/{}", self.downloaded / 1024 / 1024, self.total_size / 1024 / 1024); - - self.commit() - } - - /// Commits the [`Metadata`] to file. - pub fn commit(&self) -> Result<(), DownloaderError> { - Ok(reth_fs_util::atomic_write_file(&self.path, |file| { - bincode::serialize_into(file, &MetadataFile::from(self)) - })?) - } - - /// Loads a [`Metadata`] file from disk using the target data file. - pub fn load(data_file: &Path) -> Result { - let metadata_file_path = Self::file_path(data_file); - let MetadataFile { total_size, downloaded, chunk_size, chunks } = - bincode::deserialize_from(File::open(&metadata_file_path)?)?; - - Ok(Self { total_size, downloaded, chunk_size, chunks, path: metadata_file_path }) - } - - /// Returns true if we have downloaded all chunks. - pub fn is_done(&self) -> bool { - !self.chunks.iter().any(|c| c.is_some()) - } - - /// Deletes [`Metadata`] file from disk. - pub fn delete(self) -> Result<(), DownloaderError> { - Ok(reth_fs_util::remove_file(&self.path)?) - } -} - -/// A builder that can configure [Metadata] -#[derive(Debug)] -pub struct MetadataBuilder { - /// Path with the stored metadata. - metadata_path: PathBuf, - /// Total file size - total_size: Option, - /// Download chunk size. Default 150MB. - chunk_size: usize, -} - -impl MetadataBuilder { - const fn new(metadata_path: PathBuf) -> Self { - Self { - metadata_path, - total_size: None, - chunk_size: 150 * (1024 * 1024), // 150MB - } - } - - pub const fn with_total_size(mut self, total_size: usize) -> Self { - self.total_size = Some(total_size); - self - } - - pub const fn with_chunk_size(mut self, chunk_size: usize) -> Self { - self.chunk_size = chunk_size; - self - } - - /// Returns a [Metadata] if - pub fn build(&self) -> Result { - match &self.total_size { - Some(total_size) if *total_size > 0 => { - let chunks = (0..*total_size) - .step_by(self.chunk_size) - .map(|start| { - Some(start..=(start + self.chunk_size).min(*total_size).saturating_sub(1)) - }) - .collect(); - - let metadata = Metadata { - path: self.metadata_path.clone(), - total_size: *total_size, - downloaded: 0, - chunk_size: self.chunk_size, - chunks, - }; - metadata.commit()?; - - Ok(metadata) - } - _ => Err(DownloaderError::InvalidMetadataTotalSize(self.total_size)), - } - } -} - -/// Helper type that can serialize and deserialize [`Metadata`] to disk. -#[derive(Debug, Serialize, Deserialize)] -struct MetadataFile { - /// Total file size - total_size: usize, - /// Total file size - downloaded: usize, - /// Download chunk size. Default 150MB. - chunk_size: usize, - /// Remaining download ranges for each chunk. - /// - `Some(RangeInclusive)`: range to be downloaded. - /// - `None`: Chunk fully downloaded. - chunks: Vec>>, -} - -impl From<&Metadata> for MetadataFile { - fn from(metadata: &Metadata) -> Self { - Self { - total_size: metadata.total_size, - downloaded: metadata.downloaded, - chunk_size: metadata.chunk_size, - chunks: metadata.chunks.clone(), - } - } -} diff --git a/crates/stages/stages/src/stages/s3/downloader/mod.rs b/crates/stages/stages/src/stages/s3/downloader/mod.rs deleted file mode 100644 index d42c8251a07..00000000000 --- a/crates/stages/stages/src/stages/s3/downloader/mod.rs +++ /dev/null @@ -1,38 +0,0 @@ -//! Provides functionality for downloading files in chunks from a remote source. It supports -//! concurrent downloads, resuming interrupted downloads, and file integrity verification. - -mod error; -mod fetch; -mod meta; -mod worker; - -pub(crate) use error::DownloaderError; -pub use fetch::fetch; -pub use meta::Metadata; - -/// Response sent by the fetch task to `S3Stage` once it has downloaded all files of a block -/// range. -pub(crate) enum S3DownloaderResponse { - /// A new block range was downloaded. - AddedNewRange, - /// The last requested block range was downloaded. - Done, -} - -impl S3DownloaderResponse { - /// Whether the downloaded block range is the last requested one. - pub(crate) const fn is_done(&self) -> bool { - matches!(self, Self::Done) - } -} - -/// Chunk nth remaining range to be downloaded. -#[derive(Debug)] -pub struct RemainingChunkRange { - /// The nth chunk - pub index: usize, - /// Start of range - pub start: usize, - /// End of range - pub end: usize, -} diff --git a/crates/stages/stages/src/stages/s3/downloader/worker.rs b/crates/stages/stages/src/stages/s3/downloader/worker.rs deleted file mode 100644 index 779160892fe..00000000000 --- a/crates/stages/stages/src/stages/s3/downloader/worker.rs +++ /dev/null @@ -1,107 +0,0 @@ -use super::error::DownloaderError; -use reqwest::{header::RANGE, Client}; -use std::path::{Path, PathBuf}; -use tokio::{ - fs::OpenOptions, - io::{AsyncSeekExt, AsyncWriteExt, BufWriter}, - sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender}, -}; -use tracing::debug; - -/// Responses sent by a worker. -#[derive(Debug)] -pub(crate) enum WorkerResponse { - /// Worker has been spawned and awaiting work. - Ready { worker_id: u64, tx: UnboundedSender }, - /// Worker has downloaded - DownloadedChunk { worker_id: u64, chunk_index: usize, written_bytes: usize }, - /// Worker has encountered an error. - Err { worker_id: u64, error: DownloaderError }, -} - -/// Requests sent to a worker. -#[derive(Debug)] -pub(crate) enum WorkerRequest { - /// Requests a range to be downloaded. - Download { chunk_index: usize, start: usize, end: usize }, - /// Signals a worker exit. - Finish, -} - -/// Spawns the requested number of workers and returns a `UnboundedReceiver` that all of them will -/// respond to. -pub(crate) fn spawn_workers( - url: &str, - worker_count: u64, - data_file: &Path, -) -> UnboundedReceiver { - // Create channels for communication between workers and orchestrator - let (orchestrator_tx, orchestrator_rx) = unbounded_channel(); - - // Initiate workers - for worker_id in 0..worker_count { - let orchestrator_tx = orchestrator_tx.clone(); - let data_file = data_file.to_path_buf(); - let url = url.to_string(); - debug!(target: "sync::stages::s3::downloader", ?worker_id, "Spawning."); - - tokio::spawn(async move { - if let Err(error) = worker_fetch(worker_id, &orchestrator_tx, data_file, url).await { - let _ = orchestrator_tx.send(WorkerResponse::Err { worker_id, error }); - } - }); - } - - orchestrator_rx -} - -/// Downloads requested chunk ranges to the data file. -async fn worker_fetch( - worker_id: u64, - orchestrator_tx: &UnboundedSender, - data_file: PathBuf, - url: String, -) -> Result<(), DownloaderError> { - let client = Client::new(); - let mut data_file = BufWriter::new(OpenOptions::new().write(true).open(data_file).await?); - - // Signals readiness to download - let (tx, mut rx) = unbounded_channel::(); - orchestrator_tx.send(WorkerResponse::Ready { worker_id, tx }).unwrap_or_else(|_| { - debug!("Failed to notify orchestrator of readiness"); - }); - - while let Some(req) = rx.recv().await { - debug!( - target: "sync::stages::s3::downloader", - worker_id, - ?req, - "received from orchestrator" - ); - - match req { - WorkerRequest::Download { chunk_index, start, end } => { - data_file.seek(tokio::io::SeekFrom::Start(start as u64)).await?; - - let mut response = - client.get(&url).header(RANGE, format!("bytes={start}-{end}")).send().await?; - - let mut written_bytes = 0; - while let Some(chunk) = response.chunk().await? { - written_bytes += chunk.len(); - data_file.write_all(&chunk).await?; - } - data_file.flush().await?; - - let _ = orchestrator_tx.send(WorkerResponse::DownloadedChunk { - worker_id, - chunk_index, - written_bytes, - }); - } - WorkerRequest::Finish => break, - } - } - - Ok(()) -} diff --git a/crates/stages/stages/src/stages/s3/filelist.rs b/crates/stages/stages/src/stages/s3/filelist.rs deleted file mode 100644 index 683c4a20886..00000000000 --- a/crates/stages/stages/src/stages/s3/filelist.rs +++ /dev/null @@ -1,21 +0,0 @@ -use alloy_primitives::B256; - -/// File list to be downloaded with their hashes. -pub(crate) static DOWNLOAD_FILE_LIST: [[(&str, B256); 3]; 2] = [ - [ - ("static_file_transactions_0_499999", B256::ZERO), - ("static_file_transactions_0_499999.off", B256::ZERO), - ("static_file_transactions_0_499999.conf", B256::ZERO), - // ("static_file_blockmeta_0_499999", B256::ZERO), - // ("static_file_blockmeta_0_499999.off", B256::ZERO), - // ("static_file_blockmeta_0_499999.conf", B256::ZERO), - ], - [ - ("static_file_transactions_500000_999999", B256::ZERO), - ("static_file_transactions_500000_999999.off", B256::ZERO), - ("static_file_transactions_500000_999999.conf", B256::ZERO), - // ("static_file_blockmeta_500000_999999", B256::ZERO), - // ("static_file_blockmeta_500000_999999.off", B256::ZERO), - // ("static_file_blockmeta_500000_999999.conf", B256::ZERO), - ], -]; diff --git a/crates/stages/stages/src/stages/s3/mod.rs b/crates/stages/stages/src/stages/s3/mod.rs deleted file mode 100644 index 1c656cc1ecf..00000000000 --- a/crates/stages/stages/src/stages/s3/mod.rs +++ /dev/null @@ -1,294 +0,0 @@ -mod downloader; -pub use downloader::{fetch, Metadata}; -use downloader::{DownloaderError, S3DownloaderResponse}; - -mod filelist; -use filelist::DOWNLOAD_FILE_LIST; - -use reth_db_api::transaction::DbTxMut; -use reth_provider::{ - DBProvider, StageCheckpointReader, StageCheckpointWriter, StaticFileProviderFactory, -}; -use reth_stages_api::{ - ExecInput, ExecOutput, Stage, StageCheckpoint, StageError, StageId, UnwindInput, UnwindOutput, -}; -use reth_static_file_types::StaticFileSegment; -use std::{ - path::PathBuf, - task::{ready, Context, Poll}, -}; -use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver}; - -/// S3 `StageId` -const S3_STAGE_ID: StageId = StageId::Other("S3"); - -/// The S3 stage. -#[derive(Default, Debug)] -#[non_exhaustive] -pub struct S3Stage { - /// Static file directory. - static_file_directory: PathBuf, - /// Remote server URL. - url: String, - /// Maximum number of connections per download. - max_concurrent_requests: u64, - /// Channel to receive the downloaded ranges from the fetch task. - fetch_rx: Option>>, -} - -impl Stage for S3Stage -where - Provider: DBProvider - + StaticFileProviderFactory - + StageCheckpointReader - + StageCheckpointWriter, -{ - fn id(&self) -> StageId { - S3_STAGE_ID - } - - fn poll_execute_ready( - &mut self, - cx: &mut Context<'_>, - input: ExecInput, - ) -> Poll> { - loop { - // We are currently fetching and may have downloaded ranges that we can process. - if let Some(rx) = &mut self.fetch_rx { - // Whether we have downloaded all the required files. - let mut is_done = false; - - let response = match ready!(rx.poll_recv(cx)) { - Some(Ok(response)) => { - is_done = response.is_done(); - Ok(()) - } - Some(Err(_)) => todo!(), // TODO: DownloaderError -> StageError - None => Err(StageError::ChannelClosed), - }; - - if is_done { - self.fetch_rx = None; - } - - return Poll::Ready(response) - } - - // Spawns the downloader task if there are any missing files - if let Some(fetch_rx) = self.maybe_spawn_fetch(input) { - self.fetch_rx = Some(fetch_rx); - - // Polls fetch_rx & registers waker - continue - } - - // No files to be downloaded - return Poll::Ready(Ok(())) - } - } - - fn execute(&mut self, provider: &Provider, input: ExecInput) -> Result - where - Provider: DBProvider - + StaticFileProviderFactory - + StageCheckpointReader - + StageCheckpointWriter, - { - // Re-initializes the provider to detect the new additions - provider.static_file_provider().initialize_index()?; - - // TODO logic for appending tx_block - - // let (_, _to_block) = input.next_block_range().into_inner(); - // let static_file_provider = provider.static_file_provider(); - // let mut _tx_block_cursor = - // provider.tx_ref().cursor_write::()?; - - // tx_block_cursor.append(indice.last_tx_num(), &block_number)?; - - // let checkpoint = StageCheckpoint { block_number: highest_block, stage_checkpoint: None }; - // provider.save_stage_checkpoint(StageId::Bodies, checkpoint)?; - // provider.save_stage_checkpoint(S3_STAGE_ID, checkpoint)?; - - // // TODO: verify input.target according to s3 stage specifications - // let done = highest_block == to_block; - - Ok(ExecOutput { checkpoint: StageCheckpoint::new(input.target()), done: true }) - } - - fn unwind( - &mut self, - _provider: &Provider, - input: UnwindInput, - ) -> Result { - // TODO - Ok(UnwindOutput { checkpoint: StageCheckpoint::new(input.unwind_to) }) - } -} - -impl S3Stage { - /// It will only spawn a task to fetch files from the remote server, it there are any missing - /// static files. - /// - /// Every time a block range is ready with all the necessary files, it sends a - /// [`S3DownloaderResponse`] to `self.fetch_rx`. If it's the last requested block range, the - /// response will have `is_done` set to true. - fn maybe_spawn_fetch( - &self, - input: ExecInput, - ) -> Option>> { - let checkpoint = input.checkpoint(); - // TODO: input target can only be certain numbers. eg. 499_999 , 999_999 etc. - - // Create a list of all the missing files per block range that need to be downloaded. - let mut requests = vec![]; - for block_range_files in &DOWNLOAD_FILE_LIST { - let (_, block_range) = - StaticFileSegment::parse_filename(block_range_files[0].0).expect("qed"); - - if block_range.end() <= checkpoint.block_number { - continue - } - - let mut block_range_requests = vec![]; - for (filename, file_hash) in block_range_files { - // If the file already exists, then we are resuming a previously interrupted stage - // run. - if self.static_file_directory.join(filename).exists() { - // TODO: check hash if the file already exists - continue - } - - block_range_requests.push((filename, file_hash)); - } - - requests.push((block_range, block_range_requests)); - } - - // Return None, if we have downloaded all the files that are required. - if requests.is_empty() { - return None - } - - let static_file_directory = self.static_file_directory.clone(); - let url = self.url.clone(); - let max_concurrent_requests = self.max_concurrent_requests; - - let (fetch_tx, fetch_rx) = unbounded_channel(); - tokio::spawn(async move { - let mut requests_iter = requests.into_iter().peekable(); - - while let Some((_, file_requests)) = requests_iter.next() { - for (filename, file_hash) in file_requests { - if let Err(err) = fetch( - filename, - &static_file_directory, - &format!("{url}/{filename}"), - max_concurrent_requests, - Some(*file_hash), - ) - .await - { - let _ = fetch_tx.send(Err(err)); - return - } - } - - let response = if requests_iter.peek().is_none() { - S3DownloaderResponse::Done - } else { - S3DownloaderResponse::AddedNewRange - }; - - let _ = fetch_tx.send(Ok(response)); - } - }); - - Some(fetch_rx) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::test_utils::{ - ExecuteStageTestRunner, StageTestRunner, TestRunnerError, TestStageDB, - UnwindStageTestRunner, - }; - use reth_primitives_traits::SealedHeader; - use reth_testing_utils::{ - generators, - generators::{random_header, random_header_range}, - }; - - // stage_test_suite_ext!(S3TestRunner, s3); - - #[derive(Default)] - struct S3TestRunner { - db: TestStageDB, - } - - impl StageTestRunner for S3TestRunner { - type S = S3Stage; - - fn db(&self) -> &TestStageDB { - &self.db - } - - fn stage(&self) -> Self::S { - S3Stage::default() - } - } - - impl ExecuteStageTestRunner for S3TestRunner { - type Seed = Vec; - - fn seed_execution(&mut self, input: ExecInput) -> Result { - let start = input.checkpoint().block_number; - let mut rng = generators::rng(); - let head = random_header(&mut rng, start, None); - self.db.insert_headers_with_td(std::iter::once(&head))?; - - // use previous progress as seed size - let end = input.target.unwrap_or_default() + 1; - - if start + 1 >= end { - return Ok(Vec::default()) - } - - let mut headers = random_header_range(&mut rng, start + 1..end, head.hash()); - self.db.insert_headers_with_td(headers.iter())?; - headers.insert(0, head); - Ok(headers) - } - - fn validate_execution( - &self, - input: ExecInput, - output: Option, - ) -> Result<(), TestRunnerError> { - if let Some(output) = output { - assert!(output.done, "stage should always be done"); - assert_eq!( - output.checkpoint.block_number, - input.target(), - "stage progress should always match progress of previous stage" - ); - } - Ok(()) - } - } - - impl UnwindStageTestRunner for S3TestRunner { - fn validate_unwind(&self, _input: UnwindInput) -> Result<(), TestRunnerError> { - Ok(()) - } - } - - #[test] - fn parse_files() { - for block_range_files in &DOWNLOAD_FILE_LIST { - let (_, _) = StaticFileSegment::parse_filename(block_range_files[0].0).expect("qed"); - } - } -} diff --git a/crates/stages/stages/src/stages/sender_recovery.rs b/crates/stages/stages/src/stages/sender_recovery.rs index 2954c79e184..947f0620954 100644 --- a/crates/stages/stages/src/stages/sender_recovery.rs +++ b/crates/stages/stages/src/stages/sender_recovery.rs @@ -315,7 +315,7 @@ fn recover_sender( // value is greater than `secp256k1n / 2` if past EIP-2. There are transactions // pre-homestead which have large `s` values, so using [Signature::recover_signer] here // would not be backwards-compatible. - let sender = tx.recover_signer_unchecked_with_buf(rlp_buf).map_err(|_| { + let sender = tx.recover_unchecked_with_buf(rlp_buf).map_err(|_| { SenderRecoveryStageError::FailedRecovery(FailedSenderRecoveryError { tx: tx_id }) })?; @@ -376,7 +376,7 @@ mod tests { use assert_matches::assert_matches; use reth_db_api::cursor::DbCursorRO; use reth_ethereum_primitives::{Block, TransactionSigned}; - use reth_primitives_traits::SealedBlock; + use reth_primitives_traits::{SealedBlock, SignerRecoverable}; use reth_provider::{ providers::StaticFileWriter, BlockBodyIndicesProvider, DatabaseProviderFactory, PruneCheckpointWriter, StaticFileProviderFactory, TransactionsProvider, @@ -490,7 +490,7 @@ mod tests { ExecOutput { checkpoint: StageCheckpoint::new(expected_progress).with_entities_stage_checkpoint( EntitiesCheckpoint { - processed: runner.db.table::().unwrap().len() + processed: runner.db.count_entries::().unwrap() as u64, total: total_transactions } @@ -599,7 +599,7 @@ mod tests { /// /// 1. If there are any entries in the [`tables::TransactionSenders`] table above a given /// block number. - /// 2. If the is no requested block entry in the bodies table, but + /// 2. If there is no requested block entry in the bodies table, but /// [`tables::TransactionSenders`] is not empty. fn ensure_no_senders_by_block(&self, block: BlockNumber) -> Result<(), TestRunnerError> { let body_result = self diff --git a/crates/stages/stages/src/stages/tx_lookup.rs b/crates/stages/stages/src/stages/tx_lookup.rs index 71a790ccb14..8b1c531736b 100644 --- a/crates/stages/stages/src/stages/tx_lookup.rs +++ b/crates/stages/stages/src/stages/tx_lookup.rs @@ -88,28 +88,27 @@ where ) }) .transpose()? - .flatten() + .flatten() && + target_prunable_block > input.checkpoint().block_number { - if target_prunable_block > input.checkpoint().block_number { - input.checkpoint = Some(StageCheckpoint::new(target_prunable_block)); - - // Save prune checkpoint only if we don't have one already. - // Otherwise, pruner may skip the unpruned range of blocks. - if provider.get_prune_checkpoint(PruneSegment::TransactionLookup)?.is_none() { - let target_prunable_tx_number = provider - .block_body_indices(target_prunable_block)? - .ok_or(ProviderError::BlockBodyIndicesNotFound(target_prunable_block))? - .last_tx_num(); - - provider.save_prune_checkpoint( - PruneSegment::TransactionLookup, - PruneCheckpoint { - block_number: Some(target_prunable_block), - tx_number: Some(target_prunable_tx_number), - prune_mode, - }, - )?; - } + input.checkpoint = Some(StageCheckpoint::new(target_prunable_block)); + + // Save prune checkpoint only if we don't have one already. + // Otherwise, pruner may skip the unpruned range of blocks. + if provider.get_prune_checkpoint(PruneSegment::TransactionLookup)?.is_none() { + let target_prunable_tx_number = provider + .block_body_indices(target_prunable_block)? + .ok_or(ProviderError::BlockBodyIndicesNotFound(target_prunable_block))? + .last_tx_num(); + + provider.save_prune_checkpoint( + PruneSegment::TransactionLookup, + PruneCheckpoint { + block_number: Some(target_prunable_block), + tx_number: Some(target_prunable_tx_number), + prune_mode, + }, + )?; } } if input.target_reached() { @@ -154,7 +153,7 @@ where let interval = (total_hashes / 10).max(1); for (index, hash_to_number) in hash_collector.iter()?.enumerate() { let (hash, number) = hash_to_number?; - if index > 0 && index % interval == 0 { + if index > 0 && index.is_multiple_of(interval) { info!( target: "sync::stages::transaction_lookup", ?append_only, @@ -213,10 +212,10 @@ where // Delete all transactions that belong to this block for tx_id in body.tx_num_range() { // First delete the transaction and hash to id mapping - if let Some(transaction) = static_file_provider.transaction_by_id(tx_id)? { - if tx_hash_number_cursor.seek_exact(transaction.trie_hash())?.is_some() { - tx_hash_number_cursor.delete_current()?; - } + if let Some(transaction) = static_file_provider.transaction_by_id(tx_id)? && + tx_hash_number_cursor.seek_exact(transaction.trie_hash())?.is_some() + { + tx_hash_number_cursor.delete_current()?; } } } @@ -265,7 +264,6 @@ mod tests { use reth_primitives_traits::SealedBlock; use reth_provider::{ providers::StaticFileWriter, BlockBodyIndicesProvider, DatabaseProviderFactory, - StaticFileProviderFactory, }; use reth_stages_api::StageUnitCheckpoint; use reth_testing_utils::generators::{ @@ -321,7 +319,7 @@ mod tests { total })) }, done: true }) if block_number == previous_stage && processed == total && - total == runner.db.factory.static_file_provider().count_entries::().unwrap() as u64 + total == runner.db.count_entries::().unwrap() as u64 ); // Validate the stage execution @@ -367,7 +365,7 @@ mod tests { total })) }, done: true }) if block_number == previous_stage && processed == total && - total == runner.db.factory.static_file_provider().count_entries::().unwrap() as u64 + total == runner.db.count_entries::().unwrap() as u64 ); // Validate the stage execution @@ -460,7 +458,7 @@ mod tests { /// /// 1. If there are any entries in the [`tables::TransactionHashNumbers`] table above a /// given block number. - /// 2. If the is no requested block entry in the bodies table, but + /// 2. If there is no requested block entry in the bodies table, but /// [`tables::TransactionHashNumbers`] is not empty. fn ensure_no_hash_by_block(&self, number: BlockNumber) -> Result<(), TestRunnerError> { let body_result = self @@ -538,11 +536,10 @@ mod tests { }) .transpose() .expect("prune target block for transaction lookup") - .flatten() + .flatten() && + target_prunable_block > input.checkpoint().block_number { - if target_prunable_block > input.checkpoint().block_number { - input.checkpoint = Some(StageCheckpoint::new(target_prunable_block)); - } + input.checkpoint = Some(StageCheckpoint::new(target_prunable_block)); } let start_block = input.next_block(); let end_block = output.checkpoint.block_number; diff --git a/crates/stages/stages/src/stages/utils.rs b/crates/stages/stages/src/stages/utils.rs index 2020ad04106..f4bb960e7aa 100644 --- a/crates/stages/stages/src/stages/utils.rs +++ b/crates/stages/stages/src/stages/utils.rs @@ -77,7 +77,7 @@ where let (block_number, key) = partial_key_factory(entry?); cache.entry(key).or_default().push(block_number); - if idx > 0 && idx % interval == 0 && total_changesets > 1000 { + if idx > 0 && idx.is_multiple_of(interval) && total_changesets > 1000 { info!(target: "sync::stages::index_history", progress = %format!("{:.4}%", (idx as f64 / total_changesets as f64) * 100.0), "Collecting indices"); } @@ -124,14 +124,14 @@ where // observability let total_entries = collector.len(); - let interval = (total_entries / 100).max(1); + let interval = (total_entries / 10).max(1); for (index, element) in collector.iter()?.enumerate() { let (k, v) = element?; let sharded_key = decode_key(k)?; let new_list = BlockNumberList::decompress_owned(v)?; - if index > 0 && index % interval == 0 && total_entries > 100 { + if index > 0 && index.is_multiple_of(interval) && total_entries > 10 { info!(target: "sync::stages::index_history", progress = %format!("{:.2}%", (index as f64 / total_entries as f64) * 100.0), "Writing indices"); } @@ -156,12 +156,11 @@ where // If it's not the first sync, there might an existing shard already, so we need to // merge it with the one coming from the collector - if !append_only { - if let Some((_, last_database_shard)) = + if !append_only && + let Some((_, last_database_shard)) = write_cursor.seek_exact(sharded_key_factory(current_partial, u64::MAX))? - { - current_list.extend(last_database_shard.iter()); - } + { + current_list.extend(last_database_shard.iter()); } } @@ -265,10 +264,10 @@ where // To be extra safe, we make sure that the last tx num matches the last block from its indices. // If not, get it. loop { - if let Some(indices) = provider.block_body_indices(last_block)? { - if indices.last_tx_num() <= last_tx_num { - break - } + if let Some(indices) = provider.block_body_indices(last_block)? && + indices.last_tx_num() <= last_tx_num + { + break } if last_block == 0 { break diff --git a/crates/stages/stages/src/test_utils/test_db.rs b/crates/stages/stages/src/test_utils/test_db.rs index f3e29c1fa66..3fe1c7f1f97 100644 --- a/crates/stages/stages/src/test_utils/test_db.rs +++ b/crates/stages/stages/src/test_utils/test_db.rs @@ -1,4 +1,4 @@ -use alloy_primitives::{keccak256, Address, BlockNumber, TxHash, TxNumber, B256, U256}; +use alloy_primitives::{keccak256, Address, BlockNumber, TxHash, TxNumber, B256}; use reth_chainspec::MAINNET; use reth_db::{ test_utils::{create_test_rw_db, create_test_rw_db_with_path, create_test_static_files_dir}, @@ -19,7 +19,7 @@ use reth_primitives_traits::{Account, SealedBlock, SealedHeader, StorageEntry}; use reth_provider::{ providers::{StaticFileProvider, StaticFileProviderRWRefMut, StaticFileWriter}, test_utils::MockNodeTypesWithDB, - HistoryWriter, ProviderError, ProviderFactory, StaticFileProviderFactory, + HistoryWriter, ProviderError, ProviderFactory, StaticFileProviderFactory, StatsReader, }; use reth_static_file_types::StaticFileSegment; use reth_storage_errors::provider::ProviderResult; @@ -103,6 +103,11 @@ impl TestStageDB { }) } + /// Return the number of entries in the table or static file segment + pub fn count_entries(&self) -> ProviderResult { + self.factory.provider()?.count_entries::() + } + /// Check that there is no table entry above a given /// number by [`Table::Key`] pub fn ensure_no_entry_above(&self, num: u64, mut selector: F) -> ProviderResult<()> @@ -145,7 +150,6 @@ impl TestStageDB { writer: Option<&mut StaticFileProviderRWRefMut<'_, EthPrimitives>>, tx: &TX, header: &SealedHeader, - td: U256, ) -> ProviderResult<()> { if let Some(writer) = writer { // Backfill: some tests start at a forward block number, but static files require no @@ -155,14 +159,13 @@ impl TestStageDB { for block_number in 0..header.number { let mut prev = header.clone_header(); prev.number = block_number; - writer.append_header(&prev, U256::ZERO, &B256::ZERO)?; + writer.append_header(&prev, &B256::ZERO)?; } } - writer.append_header(header.header(), td, &header.hash())?; + writer.append_header(header.header(), &header.hash())?; } else { tx.put::(header.number, header.hash())?; - tx.put::(header.number, td.into())?; tx.put::(header.number, header.header().clone())?; } @@ -170,20 +173,16 @@ impl TestStageDB { Ok(()) } - fn insert_headers_inner<'a, I, const TD: bool>(&self, headers: I) -> ProviderResult<()> + fn insert_headers_inner<'a, I>(&self, headers: I) -> ProviderResult<()> where I: IntoIterator, { let provider = self.factory.static_file_provider(); let mut writer = provider.latest_writer(StaticFileSegment::Headers)?; let tx = self.factory.provider_rw()?.into_tx(); - let mut td = U256::ZERO; for header in headers { - if TD { - td += header.difficulty; - } - Self::insert_header(Some(&mut writer), &tx, header, td)?; + Self::insert_header(Some(&mut writer), &tx, header)?; } writer.commit()?; @@ -198,17 +197,7 @@ impl TestStageDB { where I: IntoIterator, { - self.insert_headers_inner::(headers) - } - - /// Inserts total difficulty of headers into the corresponding static file and tables. - /// - /// Superset functionality of [`TestStageDB::insert_headers`]. - pub fn insert_headers_with_td<'a, I>(&self, headers: I) -> ProviderResult<()> - where - I: IntoIterator, - { - self.insert_headers_inner::(headers) + self.insert_headers_inner::(headers) } /// Insert ordered collection of [`SealedBlock`] into corresponding tables. @@ -235,7 +224,7 @@ impl TestStageDB { .then(|| provider.latest_writer(StaticFileSegment::Headers).unwrap()); blocks.iter().try_for_each(|block| { - Self::insert_header(headers_writer.as_mut(), &tx, block.sealed_header(), U256::ZERO) + Self::insert_header(headers_writer.as_mut(), &tx, block.sealed_header()) })?; if let Some(mut writer) = headers_writer { diff --git a/crates/stages/types/Cargo.toml b/crates/stages/types/Cargo.toml index c88f53dcdaa..19e15304896 100644 --- a/crates/stages/types/Cargo.toml +++ b/crates/stages/types/Cargo.toml @@ -26,10 +26,8 @@ modular-bitfield = { workspace = true, optional = true } reth-codecs.workspace = true alloy-primitives = { workspace = true, features = ["arbitrary", "rand"] } arbitrary = { workspace = true, features = ["derive"] } -modular-bitfield.workspace = true proptest.workspace = true proptest-arbitrary-interop.workspace = true -test-fuzz.workspace = true rand.workspace = true bytes.workspace = true diff --git a/crates/stages/types/src/checkpoints.rs b/crates/stages/types/src/checkpoints.rs index 8c9ca3128a2..16bee1387f6 100644 --- a/crates/stages/types/src/checkpoints.rs +++ b/crates/stages/types/src/checkpoints.rs @@ -1,6 +1,6 @@ use super::StageId; use alloc::{format, string::String, vec::Vec}; -use alloy_primitives::{Address, BlockNumber, B256}; +use alloy_primitives::{Address, BlockNumber, B256, U256}; use core::ops::RangeInclusive; use reth_trie_common::{hash_builder::HashBuilderState, StoredSubNode}; @@ -15,6 +15,8 @@ pub struct MerkleCheckpoint { pub walker_stack: Vec, /// The hash builder state. pub state: HashBuilderState, + /// Optional storage root checkpoint for the last processed account. + pub storage_root_checkpoint: Option, } impl MerkleCheckpoint { @@ -25,7 +27,7 @@ impl MerkleCheckpoint { walker_stack: Vec, state: HashBuilderState, ) -> Self { - Self { target_block, last_account_key, walker_stack, state } + Self { target_block, last_account_key, walker_stack, state, storage_root_checkpoint: None } } } @@ -50,6 +52,22 @@ impl reth_codecs::Compact for MerkleCheckpoint { } len += self.state.to_compact(buf); + + // Encode the optional storage root checkpoint + match &self.storage_root_checkpoint { + Some(checkpoint) => { + // one means Some + buf.put_u8(1); + len += 1; + len += checkpoint.to_compact(buf); + } + None => { + // zero means None + buf.put_u8(0); + len += 1; + } + } + len } @@ -68,12 +86,137 @@ impl reth_codecs::Compact for MerkleCheckpoint { buf = rest; } - let (state, buf) = HashBuilderState::from_compact(buf, 0); - (Self { target_block, last_account_key, walker_stack, state }, buf) + let (state, mut buf) = HashBuilderState::from_compact(buf, 0); + + // Decode the storage root checkpoint if it exists + let (storage_root_checkpoint, buf) = if buf.is_empty() { + (None, buf) + } else { + match buf.get_u8() { + 1 => { + let (checkpoint, rest) = StorageRootMerkleCheckpoint::from_compact(buf, 0); + (Some(checkpoint), rest) + } + _ => (None, buf), + } + }; + + (Self { target_block, last_account_key, walker_stack, state, storage_root_checkpoint }, buf) } } -/// Saves the progress of AccountHashing stage. +/// Saves the progress of a storage root computation. +/// +/// This contains the walker stack, hash builder state, and the last storage key processed. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StorageRootMerkleCheckpoint { + /// The last storage key processed. + pub last_storage_key: B256, + /// Previously recorded walker stack. + pub walker_stack: Vec, + /// The hash builder state. + pub state: HashBuilderState, + /// The account nonce. + pub account_nonce: u64, + /// The account balance. + pub account_balance: U256, + /// The account bytecode hash. + pub account_bytecode_hash: B256, +} + +impl StorageRootMerkleCheckpoint { + /// Creates a new storage root merkle checkpoint. + pub const fn new( + last_storage_key: B256, + walker_stack: Vec, + state: HashBuilderState, + account_nonce: u64, + account_balance: U256, + account_bytecode_hash: B256, + ) -> Self { + Self { + last_storage_key, + walker_stack, + state, + account_nonce, + account_balance, + account_bytecode_hash, + } + } +} + +#[cfg(any(test, feature = "reth-codec"))] +impl reth_codecs::Compact for StorageRootMerkleCheckpoint { + fn to_compact(&self, buf: &mut B) -> usize + where + B: bytes::BufMut + AsMut<[u8]>, + { + let mut len = 0; + + buf.put_slice(self.last_storage_key.as_slice()); + len += self.last_storage_key.len(); + + buf.put_u16(self.walker_stack.len() as u16); + len += 2; + for item in &self.walker_stack { + len += item.to_compact(buf); + } + + len += self.state.to_compact(buf); + + // Encode account fields + buf.put_u64(self.account_nonce); + len += 8; + + let balance_len = self.account_balance.byte_len() as u8; + buf.put_u8(balance_len); + len += 1; + len += self.account_balance.to_compact(buf); + + buf.put_slice(self.account_bytecode_hash.as_slice()); + len += 32; + + len + } + + fn from_compact(mut buf: &[u8], _len: usize) -> (Self, &[u8]) { + use bytes::Buf; + + let last_storage_key = B256::from_slice(&buf[..32]); + buf.advance(32); + + let walker_stack_len = buf.get_u16() as usize; + let mut walker_stack = Vec::with_capacity(walker_stack_len); + for _ in 0..walker_stack_len { + let (item, rest) = StoredSubNode::from_compact(buf, 0); + walker_stack.push(item); + buf = rest; + } + + let (state, mut buf) = HashBuilderState::from_compact(buf, 0); + + // Decode account fields + let account_nonce = buf.get_u64(); + let balance_len = buf.get_u8() as usize; + let (account_balance, mut buf) = U256::from_compact(buf, balance_len); + let account_bytecode_hash = B256::from_slice(&buf[..32]); + buf.advance(32); + + ( + Self { + last_storage_key, + walker_stack, + state, + account_nonce, + account_balance, + account_bytecode_hash, + }, + buf, + ) + } +} + +/// Saves the progress of `AccountHashing` stage. #[derive(Default, Debug, Copy, Clone, PartialEq, Eq)] #[cfg_attr(any(test, feature = "test-utils"), derive(arbitrary::Arbitrary))] #[cfg_attr(any(test, feature = "reth-codec"), derive(reth_codecs::Compact))] @@ -88,7 +231,7 @@ pub struct AccountHashingCheckpoint { pub progress: EntitiesCheckpoint, } -/// Saves the progress of StorageHashing stage. +/// Saves the progress of `StorageHashing` stage. #[derive(Default, Debug, Copy, Clone, PartialEq, Eq)] #[cfg_attr(any(test, feature = "test-utils"), derive(arbitrary::Arbitrary))] #[cfg_attr(any(test, feature = "reth-codec"), derive(reth_codecs::Compact))] @@ -144,6 +287,17 @@ pub struct IndexHistoryCheckpoint { pub progress: EntitiesCheckpoint, } +/// Saves the progress of `MerkleChangeSets` stage. +#[derive(Default, Debug, Copy, Clone, PartialEq, Eq)] +#[cfg_attr(any(test, feature = "test-utils"), derive(arbitrary::Arbitrary))] +#[cfg_attr(any(test, feature = "reth-codec"), derive(reth_codecs::Compact))] +#[cfg_attr(any(test, feature = "reth-codec"), reth_codecs::add_arbitrary_tests(compact))] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct MerkleChangeSetsCheckpoint { + /// Block range which this checkpoint is valid for. + pub block_range: CheckpointBlockRange, +} + /// Saves the progress of abstract stage iterating over or downloading entities. #[derive(Debug, Default, PartialEq, Eq, Clone, Copy)] #[cfg_attr(any(test, feature = "test-utils"), derive(arbitrary::Arbitrary))] @@ -243,6 +397,9 @@ impl StageCheckpoint { StageId::IndexStorageHistory | StageId::IndexAccountHistory => { StageUnitCheckpoint::IndexHistory(IndexHistoryCheckpoint::default()) } + StageId::MerkleChangeSets => { + StageUnitCheckpoint::MerkleChangeSets(MerkleChangeSetsCheckpoint::default()) + } _ => return self, }); _ = self.stage_checkpoint.map(|mut checkpoint| checkpoint.set_block_range(from, to)); @@ -268,6 +425,7 @@ impl StageCheckpoint { progress: entities, .. }) => Some(entities), + StageUnitCheckpoint::MerkleChangeSets(_) => None, } } } @@ -281,9 +439,9 @@ impl StageCheckpoint { #[cfg_attr(any(test, feature = "reth-codec"), reth_codecs::add_arbitrary_tests(compact))] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] pub enum StageUnitCheckpoint { - /// Saves the progress of AccountHashing stage. + /// Saves the progress of `AccountHashing` stage. Account(AccountHashingCheckpoint), - /// Saves the progress of StorageHashing stage. + /// Saves the progress of `StorageHashing` stage. Storage(StorageHashingCheckpoint), /// Saves the progress of abstract stage iterating over or downloading entities. Entities(EntitiesCheckpoint), @@ -293,6 +451,8 @@ pub enum StageUnitCheckpoint { Headers(HeadersCheckpoint), /// Saves the progress of Index History stage. IndexHistory(IndexHistoryCheckpoint), + /// Saves the progress of `MerkleChangeSets` stage. + MerkleChangeSets(MerkleChangeSetsCheckpoint), } impl StageUnitCheckpoint { @@ -303,7 +463,8 @@ impl StageUnitCheckpoint { Self::Account(AccountHashingCheckpoint { block_range, .. }) | Self::Storage(StorageHashingCheckpoint { block_range, .. }) | Self::Execution(ExecutionCheckpoint { block_range, .. }) | - Self::IndexHistory(IndexHistoryCheckpoint { block_range, .. }) => { + Self::IndexHistory(IndexHistoryCheckpoint { block_range, .. }) | + Self::MerkleChangeSets(MerkleChangeSetsCheckpoint { block_range, .. }) => { let old_range = *block_range; *block_range = CheckpointBlockRange { from, to }; @@ -401,12 +562,22 @@ stage_unit_checkpoints!( index_history_stage_checkpoint, /// Sets the stage checkpoint to index history. with_index_history_stage_checkpoint + ), + ( + 6, + MerkleChangeSets, + MerkleChangeSetsCheckpoint, + /// Returns the merkle changesets stage checkpoint, if any. + merkle_changesets_stage_checkpoint, + /// Sets the stage checkpoint to merkle changesets. + with_merkle_changesets_stage_checkpoint ) ); #[cfg(test)] mod tests { use super::*; + use alloy_primitives::b256; use rand::Rng; use reth_codecs::Compact; @@ -422,6 +593,68 @@ mod tests { node: None, }], state: HashBuilderState::default(), + storage_root_checkpoint: None, + }; + + let mut buf = Vec::new(); + let encoded = checkpoint.to_compact(&mut buf); + let (decoded, _) = MerkleCheckpoint::from_compact(&buf, encoded); + assert_eq!(decoded, checkpoint); + } + + #[test] + fn storage_root_merkle_checkpoint_roundtrip() { + let mut rng = rand::rng(); + let checkpoint = StorageRootMerkleCheckpoint { + last_storage_key: rng.random(), + walker_stack: vec![StoredSubNode { + key: B256::random_with(&mut rng).to_vec(), + nibble: Some(rng.random()), + node: None, + }], + state: HashBuilderState::default(), + account_nonce: 0, + account_balance: U256::ZERO, + account_bytecode_hash: B256::ZERO, + }; + + let mut buf = Vec::new(); + let encoded = checkpoint.to_compact(&mut buf); + let (decoded, _) = StorageRootMerkleCheckpoint::from_compact(&buf, encoded); + assert_eq!(decoded, checkpoint); + } + + #[test] + fn merkle_checkpoint_with_storage_root_roundtrip() { + let mut rng = rand::rng(); + + // Create a storage root checkpoint + let storage_checkpoint = StorageRootMerkleCheckpoint { + last_storage_key: rng.random(), + walker_stack: vec![StoredSubNode { + key: B256::random_with(&mut rng).to_vec(), + nibble: Some(rng.random()), + node: None, + }], + state: HashBuilderState::default(), + account_nonce: 1, + account_balance: U256::from(1), + account_bytecode_hash: b256!( + "0x0fffffffffffffffffffffffffffffff0fffffffffffffffffffffffffffffff" + ), + }; + + // Create a merkle checkpoint with the storage root checkpoint + let checkpoint = MerkleCheckpoint { + target_block: rng.random(), + last_account_key: rng.random(), + walker_stack: vec![StoredSubNode { + key: B256::random_with(&mut rng).to_vec(), + nibble: Some(rng.random()), + node: None, + }], + state: HashBuilderState::default(), + storage_root_checkpoint: Some(storage_checkpoint), }; let mut buf = Vec::new(); diff --git a/crates/stages/types/src/id.rs b/crates/stages/types/src/id.rs index e1d466eff32..8c0a91c8731 100644 --- a/crates/stages/types/src/id.rs +++ b/crates/stages/types/src/id.rs @@ -1,3 +1,7 @@ +use alloc::vec::Vec; +#[cfg(feature = "std")] +use std::{collections::HashMap, sync::OnceLock}; + /// Stage IDs for all known stages. /// /// For custom stages, use [`StageId::Other`] @@ -8,6 +12,7 @@ pub enum StageId { note = "Static Files are generated outside of the pipeline and do not require a separate stage" )] StaticFile, + Era, Headers, Bodies, SenderRecovery, @@ -20,15 +25,23 @@ pub enum StageId { TransactionLookup, IndexStorageHistory, IndexAccountHistory, + MerkleChangeSets, Prune, Finish, /// Other custom stage with a provided string identifier. Other(&'static str), } +/// One-time-allocated stage ids encoded as raw Vecs, useful for database +/// clients to reference them for queries instead of encoding anew per query +/// (sad heap allocation required). +#[cfg(feature = "std")] +static ENCODED_STAGE_IDS: OnceLock>> = OnceLock::new(); + impl StageId { /// All supported Stages - pub const ALL: [Self; 14] = [ + pub const ALL: [Self; 16] = [ + Self::Era, Self::Headers, Self::Bodies, Self::SenderRecovery, @@ -41,6 +54,7 @@ impl StageId { Self::TransactionLookup, Self::IndexStorageHistory, Self::IndexAccountHistory, + Self::MerkleChangeSets, Self::Prune, Self::Finish, ]; @@ -63,6 +77,7 @@ impl StageId { match self { #[expect(deprecated)] Self::StaticFile => "StaticFile", + Self::Era => "Era", Self::Headers => "Headers", Self::Bodies => "Bodies", Self::SenderRecovery => "SenderRecovery", @@ -75,6 +90,7 @@ impl StageId { Self::TransactionLookup => "TransactionLookup", Self::IndexAccountHistory => "IndexAccountHistory", Self::IndexStorageHistory => "IndexStorageHistory", + Self::MerkleChangeSets => "MerkleChangeSets", Self::Prune => "Prune", Self::Finish => "Finish", Self::Other(s) => s, @@ -83,7 +99,7 @@ impl StageId { /// Returns true if it's a downloading stage [`StageId::Headers`] or [`StageId::Bodies`] pub const fn is_downloading_stage(&self) -> bool { - matches!(self, Self::Headers | Self::Bodies) + matches!(self, Self::Era | Self::Headers | Self::Bodies) } /// Returns `true` if it's [`TransactionLookup`](StageId::TransactionLookup) stage. @@ -95,6 +111,25 @@ impl StageId { pub const fn is_finish(&self) -> bool { matches!(self, Self::Finish) } + + /// Get a pre-encoded raw Vec, for example, to be used as the DB key for + /// `tables::StageCheckpoints` and `tables::StageCheckpointProgresses` + pub fn get_pre_encoded(&self) -> Option<&Vec> { + #[cfg(not(feature = "std"))] + { + None + } + #[cfg(feature = "std")] + ENCODED_STAGE_IDS + .get_or_init(|| { + let mut map = HashMap::with_capacity(Self::ALL.len()); + for stage_id in Self::ALL { + map.insert(stage_id, stage_id.to_string().into_bytes()); + } + map + }) + .get(self) + } } impl core::fmt::Display for StageId { @@ -109,6 +144,7 @@ mod tests { #[test] fn stage_id_as_string() { + assert_eq!(StageId::Era.to_string(), "Era"); assert_eq!(StageId::Headers.to_string(), "Headers"); assert_eq!(StageId::Bodies.to_string(), "Bodies"); assert_eq!(StageId::SenderRecovery.to_string(), "SenderRecovery"); @@ -129,14 +165,8 @@ mod tests { fn is_downloading_stage() { assert!(StageId::Headers.is_downloading_stage()); assert!(StageId::Bodies.is_downloading_stage()); + assert!(StageId::Era.is_downloading_stage()); assert!(!StageId::Execution.is_downloading_stage()); } - - // Multiple places around the codebase assume headers is the first stage. - // Feel free to remove this test if the assumption changes. - #[test] - fn stage_all_headers_first() { - assert_eq!(*StageId::ALL.first().unwrap(), StageId::Headers); - } } diff --git a/crates/stages/types/src/lib.rs b/crates/stages/types/src/lib.rs index 13d59de3433..83585fee7ce 100644 --- a/crates/stages/types/src/lib.rs +++ b/crates/stages/types/src/lib.rs @@ -6,7 +6,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(feature = "std"), no_std)] extern crate alloc; @@ -18,8 +18,8 @@ pub use id::StageId; mod checkpoints; pub use checkpoints::{ AccountHashingCheckpoint, CheckpointBlockRange, EntitiesCheckpoint, ExecutionCheckpoint, - HeadersCheckpoint, IndexHistoryCheckpoint, MerkleCheckpoint, StageCheckpoint, - StageUnitCheckpoint, StorageHashingCheckpoint, + HeadersCheckpoint, IndexHistoryCheckpoint, MerkleChangeSetsCheckpoint, MerkleCheckpoint, + StageCheckpoint, StageUnitCheckpoint, StorageHashingCheckpoint, StorageRootMerkleCheckpoint, }; mod execution; diff --git a/crates/stateless/Cargo.toml b/crates/stateless/Cargo.toml index d452521aa15..8adbae28ae3 100644 --- a/crates/stateless/Cargo.toml +++ b/crates/stateless/Cargo.toml @@ -22,10 +22,9 @@ alloy-rpc-types-debug.workspace = true # reth reth-ethereum-consensus.workspace = true reth-primitives-traits.workspace = true -reth-ethereum-primitives.workspace = true +reth-ethereum-primitives = { workspace = true, features = ["serde", "serde-bincode-compat"] } reth-errors.workspace = true reth-evm.workspace = true -reth-evm-ethereum.workspace = true reth-revm.workspace = true reth-trie-common.workspace = true reth-trie-sparse.workspace = true @@ -35,3 +34,13 @@ reth-consensus.workspace = true # misc thiserror.workspace = true itertools.workspace = true +serde.workspace = true +serde_with.workspace = true + +k256 = { workspace = true, optional = true } +secp256k1 = { workspace = true, optional = true } + +[features] +default = ["k256"] +k256 = ["dep:k256"] +secp256k1 = ["dep:secp256k1"] diff --git a/crates/stateless/src/lib.rs b/crates/stateless/src/lib.rs index cc99afe4169..6813638485e 100644 --- a/crates/stateless/src/lib.rs +++ b/crates/stateless/src/lib.rs @@ -29,16 +29,42 @@ html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256", issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(test), warn(unused_crate_dependencies))] #![no_std] extern crate alloc; -pub(crate) mod root; +mod recover_block; +/// Sparse trie implementation for stateless validation +pub mod trie; + +#[doc(inline)] +pub use recover_block::UncompressedPublicKey; +#[doc(inline)] +pub use trie::StatelessTrie; +#[doc(inline)] +pub use validation::stateless_validation_with_trie; + /// Implementation of stateless validation pub mod validation; pub(crate) mod witness_db; #[doc(inline)] pub use alloy_rpc_types_debug::ExecutionWitness; + +use reth_ethereum_primitives::Block; + +/// `StatelessInput` is a convenience structure for serializing the input needed +/// for the stateless validation function. +#[serde_with::serde_as] +#[derive(Clone, Debug, Default, serde::Serialize, serde::Deserialize)] +pub struct StatelessInput { + /// The block being executed in the stateless validation function + #[serde_as( + as = "reth_primitives_traits::serde_bincode_compat::Block" + )] + pub block: Block, + /// `ExecutionWitness` for the stateless validation function + pub witness: ExecutionWitness, +} diff --git a/crates/stateless/src/recover_block.rs b/crates/stateless/src/recover_block.rs new file mode 100644 index 00000000000..15db1fe55e1 --- /dev/null +++ b/crates/stateless/src/recover_block.rs @@ -0,0 +1,143 @@ +use crate::validation::StatelessValidationError; +use alloc::vec::Vec; +use alloy_consensus::BlockHeader; +use alloy_primitives::{Address, Signature, B256}; +use core::ops::Deref; +use reth_chainspec::EthereumHardforks; +use reth_ethereum_primitives::{Block, TransactionSigned}; +use reth_primitives_traits::{Block as _, RecoveredBlock}; +use serde::{Deserialize, Serialize}; +use serde_with::{serde_as, Bytes}; + +#[cfg(all(feature = "k256", feature = "secp256k1"))] +use k256 as _; + +/// Serialized uncompressed public key +#[serde_as] +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct UncompressedPublicKey(#[serde_as(as = "Bytes")] pub [u8; 65]); + +impl Deref for UncompressedPublicKey { + type Target = [u8]; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +/// Verifies all transactions in a block against a list of public keys and signatures. +/// +/// Returns a `RecoveredBlock` +pub(crate) fn recover_block_with_public_keys( + block: Block, + public_keys: Vec, + chain_spec: &ChainSpec, +) -> Result, StatelessValidationError> +where + ChainSpec: EthereumHardforks, +{ + if block.body().transactions.len() != public_keys.len() { + return Err(StatelessValidationError::Custom( + "Number of public keys must match number of transactions", + )); + } + + // Determine if we're in the Homestead fork for signature validation + let is_homestead = chain_spec.is_homestead_active_at_block(block.header().number()); + + // Verify each transaction signature against its corresponding public key + let senders = public_keys + .iter() + .zip(block.body().transactions()) + .map(|(vk, tx)| verify_and_compute_sender(vk, tx, is_homestead)) + .collect::, _>>()?; + + // Create RecoveredBlock with verified senders + let block_hash = block.hash_slow(); + Ok(RecoveredBlock::new(block, senders, block_hash)) +} + +/// Verifies a transaction using its signature and the given public key. +/// +/// Note: If the signature or the public key is incorrect, then this method +/// will return an error. +/// +/// Returns the address derived from the public key. +fn verify_and_compute_sender( + vk: &UncompressedPublicKey, + tx: &TransactionSigned, + is_homestead: bool, +) -> Result { + let sig = tx.signature(); + + // non-normalized signatures are only valid pre-homestead + let sig_is_normalized = sig.normalize_s().is_none(); + if is_homestead && !sig_is_normalized { + return Err(StatelessValidationError::HomesteadSignatureNotNormalized); + } + let sig_hash = tx.signature_hash(); + #[cfg(all(feature = "k256", feature = "secp256k1"))] + { + let _ = verify_and_compute_sender_unchecked_k256; + } + #[cfg(feature = "secp256k1")] + { + verify_and_compute_sender_unchecked_secp256k1(vk, sig, sig_hash) + } + #[cfg(all(feature = "k256", not(feature = "secp256k1")))] + { + verify_and_compute_sender_unchecked_k256(vk, sig, sig_hash) + } + #[cfg(not(any(feature = "secp256k1", feature = "k256")))] + { + let _ = vk; + let _ = tx; + let _: B256 = sig_hash; + let _: &Signature = sig; + + unimplemented!("Must choose either k256 or secp256k1 feature") + } +} +#[cfg(feature = "k256")] +fn verify_and_compute_sender_unchecked_k256( + vk: &UncompressedPublicKey, + sig: &Signature, + sig_hash: B256, +) -> Result { + use k256::ecdsa::{signature::hazmat::PrehashVerifier, VerifyingKey}; + + let vk = + VerifyingKey::from_sec1_bytes(vk).map_err(|_| StatelessValidationError::SignerRecovery)?; + + sig.to_k256() + .and_then(|sig| vk.verify_prehash(sig_hash.as_slice(), &sig)) + .map_err(|_| StatelessValidationError::SignerRecovery)?; + + Ok(Address::from_public_key(&vk)) +} + +#[cfg(feature = "secp256k1")] +fn verify_and_compute_sender_unchecked_secp256k1( + vk: &UncompressedPublicKey, + sig: &Signature, + sig_hash: B256, +) -> Result { + use secp256k1::{ecdsa::Signature as SecpSignature, Message, PublicKey, SECP256K1}; + + let public_key = + PublicKey::from_slice(vk).map_err(|_| StatelessValidationError::SignerRecovery)?; + + let mut sig_bytes = [0u8; 64]; + sig_bytes[0..32].copy_from_slice(&sig.r().to_be_bytes::<32>()); + sig_bytes[32..64].copy_from_slice(&sig.s().to_be_bytes::<32>()); + + let signature = SecpSignature::from_compact(&sig_bytes) + .map_err(|_| StatelessValidationError::SignerRecovery)?; + + let message = Message::from_digest(sig_hash.0); + SECP256K1 + .verify_ecdsa(&message, &signature, &public_key) + .map_err(|_| StatelessValidationError::SignerRecovery)?; + + Ok(Address::from_raw_public_key(&vk[1..])) +} diff --git a/crates/stateless/src/root.rs b/crates/stateless/src/root.rs deleted file mode 100644 index d1788cfbb9c..00000000000 --- a/crates/stateless/src/root.rs +++ /dev/null @@ -1,96 +0,0 @@ -// Copied and modified from ress: https://github.com/paradigmxyz/ress/blob/06bf2c4788e45b8fcbd640e38b6243e6f87c4d0e/crates/engine/src/tree/root.rs - -use alloc::vec::Vec; -use alloy_primitives::B256; -use alloy_rlp::{Decodable, Encodable}; -use itertools::Itertools; -use reth_trie_common::{ - HashedPostState, Nibbles, TrieAccount, EMPTY_ROOT_HASH, TRIE_ACCOUNT_RLP_MAX_SIZE, -}; -use reth_trie_sparse::{errors::SparseStateTrieResult, SparseStateTrie, SparseTrie}; - -/// Calculates the post-execution state root by applying state changes to a sparse trie. -/// -/// This function takes a [`SparseStateTrie`] with the pre-state and a [`HashedPostState`] -/// containing account and storage changes resulting from block execution (state diff). -/// -/// It modifies the input `trie` in place to reflect these changes and then calculates the -/// final post-execution state root. -pub(crate) fn calculate_state_root( - trie: &mut SparseStateTrie, - state: HashedPostState, -) -> SparseStateTrieResult { - // 1. Apply storage‑slot updates and compute each contract’s storage root - // - // - // We walk over every (address, storage) pair in deterministic order - // and update the corresponding per‑account storage trie in‑place. - // When we’re done we collect (address, updated_storage_trie) in a `Vec` - // so that we can insert them back into the outer state trie afterwards ― this avoids - // borrowing issues. - let mut storage_results = Vec::with_capacity(state.storages.len()); - - for (address, storage) in state.storages.into_iter().sorted_unstable_by_key(|(addr, _)| *addr) { - // Take the existing storage trie (or create an empty, “revealed” one) - let mut storage_trie = - trie.take_storage_trie(&address).unwrap_or_else(SparseTrie::revealed_empty); - - if storage.wiped { - storage_trie.wipe()?; - } - - // Apply slot‑level changes - for (hashed_slot, value) in - storage.storage.into_iter().sorted_unstable_by_key(|(slot, _)| *slot) - { - let nibbles = Nibbles::unpack(hashed_slot); - if value.is_zero() { - storage_trie.remove_leaf(&nibbles)?; - } else { - storage_trie.update_leaf(nibbles, alloy_rlp::encode_fixed_size(&value).to_vec())?; - } - } - - // Finalise the storage‑trie root before pushing the result - storage_trie.root(); - storage_results.push((address, storage_trie)); - } - - // Insert every updated storage trie back into the outer state trie - for (address, storage_trie) in storage_results { - trie.insert_storage_trie(address, storage_trie); - } - - // 2. Apply account‑level updates and (re)encode the account nodes - // Update accounts with new values - // TODO: upstream changes into reth so that `SparseStateTrie::update_account` handles this - let mut account_rlp_buf = Vec::with_capacity(TRIE_ACCOUNT_RLP_MAX_SIZE); - - for (hashed_address, account) in - state.accounts.into_iter().sorted_unstable_by_key(|(addr, _)| *addr) - { - let nibbles = Nibbles::unpack(hashed_address); - let account = account.unwrap_or_default(); - - // Determine which storage root should be used for this account - let storage_root = if let Some(storage_trie) = trie.storage_trie_mut(&hashed_address) { - storage_trie.root() - } else if let Some(value) = trie.get_account_value(&hashed_address) { - TrieAccount::decode(&mut &value[..])?.storage_root - } else { - EMPTY_ROOT_HASH - }; - - // Decide whether to remove or update the account leaf - if account.is_empty() && storage_root == EMPTY_ROOT_HASH { - trie.remove_account_leaf(&nibbles)?; - } else { - account_rlp_buf.clear(); - account.into_trie_account(storage_root).encode(&mut account_rlp_buf); - trie.update_account_leaf(nibbles, account_rlp_buf.clone())?; - } - } - - // Return new state root - trie.root() -} diff --git a/crates/stateless/src/trie.rs b/crates/stateless/src/trie.rs new file mode 100644 index 00000000000..49d1f6cf0fd --- /dev/null +++ b/crates/stateless/src/trie.rs @@ -0,0 +1,311 @@ +use crate::validation::StatelessValidationError; +use alloc::{format, vec::Vec}; +use alloy_primitives::{keccak256, map::B256Map, Address, B256, U256}; +use alloy_rlp::{Decodable, Encodable}; +use alloy_rpc_types_debug::ExecutionWitness; +use alloy_trie::{TrieAccount, EMPTY_ROOT_HASH}; +use itertools::Itertools; +use reth_errors::ProviderError; +use reth_revm::state::Bytecode; +use reth_trie_common::{HashedPostState, Nibbles, TRIE_ACCOUNT_RLP_MAX_SIZE}; +use reth_trie_sparse::{ + errors::SparseStateTrieResult, + provider::{DefaultTrieNodeProvider, DefaultTrieNodeProviderFactory}, + SparseStateTrie, SparseTrie, SparseTrieInterface, +}; + +/// Trait for stateless trie implementations that can be used for stateless validation. +pub trait StatelessTrie: core::fmt::Debug { + /// Initialize the stateless trie using the `ExecutionWitness` + fn new( + witness: &ExecutionWitness, + pre_state_root: B256, + ) -> Result<(Self, B256Map), StatelessValidationError> + where + Self: Sized; + + /// Returns the `TrieAccount` that corresponds to the `Address` + /// + /// This method will error if the `ExecutionWitness` is not able to guarantee + /// that the account is missing from the Trie _and_ the witness was complete. + fn account(&self, address: Address) -> Result, ProviderError>; + + /// Returns the storage slot value that corresponds to the given (address, slot) tuple. + /// + /// This method will error if the `ExecutionWitness` is not able to guarantee + /// that the storage was missing from the Trie _and_ the witness was complete. + fn storage(&self, address: Address, slot: U256) -> Result; + + /// Computes the new state root from the `HashedPostState`. + fn calculate_state_root( + &mut self, + state: HashedPostState, + ) -> Result; +} + +/// `StatelessSparseTrie` structure for usage during stateless validation +#[derive(Debug)] +pub struct StatelessSparseTrie { + inner: SparseStateTrie, +} + +impl StatelessSparseTrie { + /// Initialize the stateless trie using the `ExecutionWitness` + /// + /// Note: Currently this method does not check that the `ExecutionWitness` + /// is complete for all of the preimage keys. + pub fn new( + witness: &ExecutionWitness, + pre_state_root: B256, + ) -> Result<(Self, B256Map), StatelessValidationError> { + verify_execution_witness(witness, pre_state_root) + .map(|(inner, bytecode)| (Self { inner }, bytecode)) + } + + /// Returns the `TrieAccount` that corresponds to the `Address` + /// + /// This method will error if the `ExecutionWitness` is not able to guarantee + /// that the account is missing from the Trie _and_ the witness was complete. + pub fn account(&self, address: Address) -> Result, ProviderError> { + let hashed_address = keccak256(address); + + if let Some(bytes) = self.inner.get_account_value(&hashed_address) { + let account = TrieAccount::decode(&mut bytes.as_slice())?; + return Ok(Some(account)) + } + + if !self.inner.check_valid_account_witness(hashed_address) { + return Err(ProviderError::TrieWitnessError(format!( + "incomplete account witness for {hashed_address:?}" + ))); + } + + Ok(None) + } + + /// Returns the storage slot value that corresponds to the given (address, slot) tuple. + /// + /// This method will error if the `ExecutionWitness` is not able to guarantee + /// that the storage was missing from the Trie _and_ the witness was complete. + pub fn storage(&self, address: Address, slot: U256) -> Result { + let hashed_address = keccak256(address); + let hashed_slot = keccak256(B256::from(slot)); + + if let Some(raw) = self.inner.get_storage_slot_value(&hashed_address, &hashed_slot) { + return Ok(U256::decode(&mut raw.as_slice())?) + } + + // Storage slot value is not present in the trie, validate that the witness is complete. + // If the account exists in the trie... + if let Some(bytes) = self.inner.get_account_value(&hashed_address) { + // ...check that its storage is either empty or the storage trie was sufficiently + // revealed... + let account = TrieAccount::decode(&mut bytes.as_slice())?; + if account.storage_root != EMPTY_ROOT_HASH && + !self.inner.check_valid_storage_witness(hashed_address, hashed_slot) + { + return Err(ProviderError::TrieWitnessError(format!( + "incomplete storage witness: prover must supply exclusion proof for slot {hashed_slot:?} in account {hashed_address:?}" + ))); + } + } else if !self.inner.check_valid_account_witness(hashed_address) { + // ...else if account is missing, validate that the account trie was sufficiently + // revealed. + return Err(ProviderError::TrieWitnessError(format!( + "incomplete account witness for {hashed_address:?}" + ))); + } + + Ok(U256::ZERO) + } + + /// Computes the new state root from the `HashedPostState`. + pub fn calculate_state_root( + &mut self, + state: HashedPostState, + ) -> Result { + calculate_state_root(&mut self.inner, state) + .map_err(|_e| StatelessValidationError::StatelessStateRootCalculationFailed) + } +} + +impl StatelessTrie for StatelessSparseTrie { + fn new( + witness: &ExecutionWitness, + pre_state_root: B256, + ) -> Result<(Self, B256Map), StatelessValidationError> { + Self::new(witness, pre_state_root) + } + + fn account(&self, address: Address) -> Result, ProviderError> { + self.account(address) + } + + fn storage(&self, address: Address, slot: U256) -> Result { + self.storage(address, slot) + } + + fn calculate_state_root( + &mut self, + state: HashedPostState, + ) -> Result { + self.calculate_state_root(state) + } +} + +/// Verifies execution witness [`ExecutionWitness`] against an expected pre-state root. +/// +/// This function takes the RLP-encoded values provided in [`ExecutionWitness`] +/// (which includes state trie nodes, storage trie nodes, and contract bytecode) +/// and uses it to populate a new [`SparseStateTrie`]. +/// +/// If the computed root hash matches the `pre_state_root`, it signifies that the +/// provided execution witness is consistent with that pre-state root. In this case, the function +/// returns the populated [`SparseStateTrie`] and a [`B256Map`] containing the +/// contract bytecode (mapping code hash to [`Bytecode`]). +/// +/// The bytecode has a separate mapping because the [`SparseStateTrie`] does not store the +/// contract bytecode, only the hash of it (code hash). +/// +/// If the roots do not match, it returns an error indicating the witness is invalid +/// for the given `pre_state_root` (see `StatelessValidationError::PreStateRootMismatch`). +// Note: This approach might be inefficient for ZKVMs requiring minimal memory operations, which +// would explain why they have for the most part re-implemented this function. +fn verify_execution_witness( + witness: &ExecutionWitness, + pre_state_root: B256, +) -> Result<(SparseStateTrie, B256Map), StatelessValidationError> { + let provider_factory = DefaultTrieNodeProviderFactory; + let mut trie = SparseStateTrie::new(); + let mut state_witness = B256Map::default(); + let mut bytecode = B256Map::default(); + + for rlp_encoded in &witness.state { + let hash = keccak256(rlp_encoded); + state_witness.insert(hash, rlp_encoded.clone()); + } + for rlp_encoded in &witness.codes { + let hash = keccak256(rlp_encoded); + bytecode.insert(hash, Bytecode::new_raw(rlp_encoded.clone())); + } + + // Reveal the witness with our state root + // This method builds a trie using the sparse trie using the state_witness with + // the root being the pre_state_root. + // Here are some things to note: + // - You can pass in more witnesses than is needed for the block execution. + // - If you try to get an account and it has not been seen. This means that the account + // was not inserted into the Trie. It does not mean that the account does not exist. + // In order to determine an account not existing, we must do an exclusion proof. + trie.reveal_witness(pre_state_root, &state_witness) + .map_err(|_e| StatelessValidationError::WitnessRevealFailed { pre_state_root })?; + + // Calculate the root + let computed_root = trie + .root(&provider_factory) + .map_err(|_e| StatelessValidationError::StatelessPreStateRootCalculationFailed)?; + + if computed_root == pre_state_root { + Ok((trie, bytecode)) + } else { + Err(StatelessValidationError::PreStateRootMismatch { + got: computed_root, + expected: pre_state_root, + }) + } +} + +// Copied and modified from ress: https://github.com/paradigmxyz/ress/blob/06bf2c4788e45b8fcbd640e38b6243e6f87c4d0e/crates/engine/src/tree/root.rs +/// Calculates the post-execution state root by applying state changes to a sparse trie. +/// +/// This function takes a [`SparseStateTrie`] with the pre-state and a [`HashedPostState`] +/// containing account and storage changes resulting from block execution (state diff). +/// +/// It modifies the input `trie` in place to reflect these changes and then calculates the +/// final post-execution state root. +fn calculate_state_root( + trie: &mut SparseStateTrie, + state: HashedPostState, +) -> SparseStateTrieResult { + // 1. Apply storage‑slot updates and compute each contract’s storage root + // + // + // We walk over every (address, storage) pair in deterministic order + // and update the corresponding per‑account storage trie in‑place. + // When we’re done we collect (address, updated_storage_trie) in a `Vec` + // so that we can insert them back into the outer state trie afterwards ― this avoids + // borrowing issues. + let mut storage_results = Vec::with_capacity(state.storages.len()); + + // In `verify_execution_witness` a `DefaultTrieNodeProviderFactory` is used, so we use the same + // again in here. + let provider_factory = DefaultTrieNodeProviderFactory; + let storage_provider = DefaultTrieNodeProvider; + + for (address, storage) in state.storages.into_iter().sorted_unstable_by_key(|(addr, _)| *addr) { + // Take the existing storage trie (or create an empty, “revealed” one) + let mut storage_trie = + trie.take_storage_trie(&address).unwrap_or_else(SparseTrie::revealed_empty); + + if storage.wiped { + storage_trie.wipe()?; + } + + // Apply slot‑level changes + for (hashed_slot, value) in + storage.storage.into_iter().sorted_unstable_by_key(|(slot, _)| *slot) + { + let nibbles = Nibbles::unpack(hashed_slot); + if value.is_zero() { + storage_trie.remove_leaf(&nibbles, &storage_provider)?; + } else { + storage_trie.update_leaf( + nibbles, + alloy_rlp::encode_fixed_size(&value).to_vec(), + &storage_provider, + )?; + } + } + + // Finalise the storage‑trie root before pushing the result + storage_trie.root(); + storage_results.push((address, storage_trie)); + } + + // Insert every updated storage trie back into the outer state trie + for (address, storage_trie) in storage_results { + trie.insert_storage_trie(address, storage_trie); + } + + // 2. Apply account‑level updates and (re)encode the account nodes + // Update accounts with new values + // TODO: upstream changes into reth so that `SparseStateTrie::update_account` handles this + let mut account_rlp_buf = Vec::with_capacity(TRIE_ACCOUNT_RLP_MAX_SIZE); + + for (hashed_address, account) in + state.accounts.into_iter().sorted_unstable_by_key(|(addr, _)| *addr) + { + let nibbles = Nibbles::unpack(hashed_address); + + // Determine which storage root should be used for this account + let storage_root = if let Some(storage_trie) = trie.storage_trie_mut(&hashed_address) { + storage_trie.root() + } else if let Some(value) = trie.get_account_value(&hashed_address) { + TrieAccount::decode(&mut &value[..])?.storage_root + } else { + EMPTY_ROOT_HASH + }; + + // Decide whether to remove or update the account leaf + if let Some(account) = account { + account_rlp_buf.clear(); + account.into_trie_account(storage_root).encode(&mut account_rlp_buf); + trie.update_account_leaf(nibbles, account_rlp_buf.clone(), &provider_factory)?; + } else { + trie.remove_account_leaf(&nibbles, &provider_factory)?; + } + } + + // Return new state root + trie.root(&provider_factory) +} diff --git a/crates/stateless/src/validation.rs b/crates/stateless/src/validation.rs index 4a07acba60a..db5f317ab22 100644 --- a/crates/stateless/src/validation.rs +++ b/crates/stateless/src/validation.rs @@ -1,24 +1,32 @@ -use crate::{witness_db::WitnessDatabase, ExecutionWitness}; +use crate::{ + recover_block::{recover_block_with_public_keys, UncompressedPublicKey}, + trie::{StatelessSparseTrie, StatelessTrie}, + witness_db::WitnessDatabase, + ExecutionWitness, +}; use alloc::{ collections::BTreeMap, + fmt::Debug, string::{String, ToString}, sync::Arc, vec::Vec, }; -use alloy_consensus::{Block, BlockHeader, Header}; -use alloy_primitives::{keccak256, map::B256Map, B256}; -use alloy_rlp::Decodable; -use reth_chainspec::ChainSpec; +use alloy_consensus::{BlockHeader, Header}; +use alloy_primitives::{keccak256, B256}; +use reth_chainspec::{EthChainSpec, EthereumHardforks}; use reth_consensus::{Consensus, HeaderValidator}; use reth_errors::ConsensusError; use reth_ethereum_consensus::{validate_block_post_execution, EthBeaconConsensus}; -use reth_ethereum_primitives::TransactionSigned; -use reth_evm::{execute::Executor, ConfigureEvm}; -use reth_evm_ethereum::execute::EthExecutorProvider; -use reth_primitives_traits::RecoveredBlock; -use reth_revm::state::Bytecode; +use reth_ethereum_primitives::{Block, EthPrimitives, EthereumReceipt}; +use reth_evm::{ + execute::{BlockExecutionOutput, Executor}, + ConfigureEvm, +}; +use reth_primitives_traits::{RecoveredBlock, SealedHeader}; use reth_trie_common::{HashedPostState, KeccakKeyHasher}; -use reth_trie_sparse::{blinded::DefaultBlindedProviderFactory, SparseStateTrie}; + +/// BLOCKHASH ancestor lookup window limit per EVM (number of most recent blocks accessible). +const BLOCKHASH_ANCESTOR_LIMIT: usize = 256; /// Errors that can occur during stateless validation. #[derive(Debug, thiserror::Error)] @@ -68,7 +76,7 @@ pub enum StatelessValidationError { HeaderDeserializationFailed, /// Error when the computed state root does not match the one in the block header. - #[error("mismatched post- state root: {got}\n {expected}")] + #[error("mismatched post-state root: {got}\n {expected}")] PostStateRootMismatch { /// The computed post-state root got: B256, @@ -84,6 +92,18 @@ pub enum StatelessValidationError { /// The expected pre-state root from the previous block expected: B256, }, + + /// Error during signer recovery. + #[error("signer recovery failed")] + SignerRecovery, + + /// Error when signature has non-normalized s value in homestead block. + #[error("signature s value not normalized for homestead block")] + HomesteadSignatureNotNormalized, + + /// Custom error. + #[error("{0}")] + Custom(&'static str), } /// Performs stateless validation of a block using the provided witness data. @@ -102,10 +122,10 @@ pub enum StatelessValidationError { /// the pre state reads. /// /// 2. **Pre-State Verification:** Retrieves the expected `pre_state_root` from the parent header -/// from `ancestor_headers`. Verifies the provided [`ExecutionWitness`] against this root using -/// [`verify_execution_witness`]. +/// from `ancestor_headers`. Verifies the provided [`ExecutionWitness`] against the +/// `pre_state_root`. /// -/// 3. **Chain Verification:** The code currently does not verify the [`ChainSpec`] and expects a +/// 3. **Chain Verification:** The code currently does not verify the [`EthChainSpec`] and expects a /// higher level function to assert that this is correct by, for example, asserting that it is /// equal to the Ethereum Mainnet `ChainSpec` or asserting against the genesis hash that this /// `ChainSpec` defines. @@ -121,17 +141,53 @@ pub enum StatelessValidationError { /// /// If all steps succeed the function returns `Some` containing the hash of the validated /// `current_block`. -pub fn stateless_validation( - current_block: RecoveredBlock>, +pub fn stateless_validation( + current_block: Block, + public_keys: Vec, witness: ExecutionWitness, chain_spec: Arc, -) -> Result { - let mut ancestor_headers: Vec
= witness + evm_config: E, +) -> Result<(B256, BlockExecutionOutput), StatelessValidationError> +where + ChainSpec: Send + Sync + EthChainSpec
+ EthereumHardforks + Debug, + E: ConfigureEvm + Clone + 'static, +{ + stateless_validation_with_trie::( + current_block, + public_keys, + witness, + chain_spec, + evm_config, + ) +} + +/// Performs stateless validation of a block using a custom `StatelessTrie` implementation. +/// +/// This is a generic version of `stateless_validation` that allows users to provide their own +/// implementation of the `StatelessTrie` for custom trie backends or optimizations. +/// +/// See `stateless_validation` for detailed documentation of the validation process. +pub fn stateless_validation_with_trie( + current_block: Block, + public_keys: Vec, + witness: ExecutionWitness, + chain_spec: Arc, + evm_config: E, +) -> Result<(B256, BlockExecutionOutput), StatelessValidationError> +where + T: StatelessTrie, + ChainSpec: Send + Sync + EthChainSpec
+ EthereumHardforks + Debug, + E: ConfigureEvm + Clone + 'static, +{ + let current_block = recover_block_with_public_keys(current_block, public_keys, &*chain_spec)?; + + let mut ancestor_headers: Vec<_> = witness .headers .iter() - .map(|serialized_header| { - let bytes = serialized_header.as_ref(); - Header::decode(&mut &bytes[..]) + .map(|bytes| { + let hash = keccak256(bytes); + alloy_rlp::decode_exact::
(bytes) + .map(|h| SealedHeader::new(h, hash)) .map_err(|_| StatelessValidationError::HeaderDeserializationFailed) }) .collect::>()?; @@ -139,32 +195,37 @@ pub fn stateless_validation( // ascending order. ancestor_headers.sort_by_key(|header| header.number()); - // Validate block against pre-execution consensus rules - validate_block_consensus(chain_spec.clone(), ¤t_block)?; + // Enforce BLOCKHASH ancestor headers limit (256 most recent blocks) + let count = ancestor_headers.len(); + if count > BLOCKHASH_ANCESTOR_LIMIT { + return Err(StatelessValidationError::AncestorHeaderLimitExceeded { + count, + limit: BLOCKHASH_ANCESTOR_LIMIT, + }); + } // Check that the ancestor headers form a contiguous chain and are not just random headers. let ancestor_hashes = compute_ancestor_hashes(¤t_block, &ancestor_headers)?; - // Get the last ancestor header and retrieve its state root. - // - // There should be at least one ancestor header, this is because we need the parent header to - // retrieve the previous state root. + // There should be at least one ancestor header. // The edge case here would be the genesis block, but we do not create proofs for the genesis // block. - let pre_state_root = match ancestor_headers.last() { - Some(prev_header) => prev_header.state_root, + let parent = match ancestor_headers.last() { + Some(prev_header) => prev_header, None => return Err(StatelessValidationError::MissingAncestorHeader), }; + // Validate block against pre-execution consensus rules + validate_block_consensus(chain_spec.clone(), ¤t_block, parent)?; + // First verify that the pre-state reads are correct - let (mut sparse_trie, bytecode) = verify_execution_witness(&witness, pre_state_root)?; + let (mut trie, bytecode) = T::new(&witness, parent.state_root)?; // Create an in-memory database that will use the reads to validate the block - let db = WitnessDatabase::new(&sparse_trie, bytecode, ancestor_hashes); + let db = WitnessDatabase::new(&trie, bytecode, ancestor_hashes); // Execute the block - let basic_block_executor = EthExecutorProvider::ethereum(chain_spec.clone()); - let executor = basic_block_executor.batch_executor(db); + let executor = evm_config.executor(db); let output = executor .execute(¤t_block) .map_err(|e| StatelessValidationError::StatelessExecutionFailed(e.to_string()))?; @@ -175,8 +236,7 @@ pub fn stateless_validation( // Compute and check the post state root let hashed_state = HashedPostState::from_bundle_state::(&output.state.state); - let state_root = crate::root::calculate_state_root(&mut sparse_trie, hashed_state) - .map_err(|_e| StatelessValidationError::StatelessStateRootCalculationFailed)?; + let state_root = trie.calculate_state_root(hashed_state)?; if state_root != current_block.state_root { return Err(StatelessValidationError::PostStateRootMismatch { got: state_root, @@ -185,102 +245,43 @@ pub fn stateless_validation( } // Return block hash - Ok(current_block.hash_slow()) + Ok((current_block.hash_slow(), output)) } /// Performs consensus validation checks on a block without execution or state validation. /// /// This function validates a block against Ethereum consensus rules by: /// -/// 1. **Difficulty Validation:** Validates the header with total difficulty to verify proof-of-work -/// (pre-merge) or to enforce post-merge requirements. -/// -/// 2. **Header Validation:** Validates the sealed header against protocol specifications, +/// 1. **Header Validation:** Validates the sealed header against protocol specifications, /// including: /// - Gas limit checks /// - Base fee validation for EIP-1559 /// - Withdrawals root validation for Shanghai fork /// - Blob-related fields validation for Cancun fork /// -/// 3. **Pre-Execution Validation:** Validates block structure, transaction format, signature +/// 2. **Pre-Execution Validation:** Validates block structure, transaction format, signature /// validity, and other pre-execution requirements. /// /// This function acts as a preliminary validation before executing and validating the state /// transition function. -fn validate_block_consensus( +fn validate_block_consensus( chain_spec: Arc, - block: &RecoveredBlock>, -) -> Result<(), StatelessValidationError> { + block: &RecoveredBlock, + parent: &SealedHeader
, +) -> Result<(), StatelessValidationError> +where + ChainSpec: Send + Sync + EthChainSpec
+ EthereumHardforks + Debug, +{ let consensus = EthBeaconConsensus::new(chain_spec); consensus.validate_header(block.sealed_header())?; + consensus.validate_header_against_parent(block.sealed_header(), parent)?; consensus.validate_block_pre_execution(block)?; Ok(()) } -/// Verifies execution witness [`ExecutionWitness`] against an expected pre-state root. -/// -/// This function takes the RLP-encoded values provided in [`ExecutionWitness`] -/// (which includes state trie nodes, storage trie nodes, and contract bytecode) -/// and uses it to populate a new [`SparseStateTrie`]. -/// -/// If the computed root hash matches the `pre_state_root`, it signifies that the -/// provided execution witness is consistent with that pre-state root. In this case, the function -/// returns the populated [`SparseStateTrie`] and a [`B256Map`] containing the -/// contract bytecode (mapping code hash to [`Bytecode`]). -/// -/// The bytecode has a separate mapping because the [`SparseStateTrie`] does not store the -/// contract bytecode, only the hash of it (code hash). -/// -/// If the roots do not match, it returns `None`, indicating the witness is invalid -/// for the given `pre_state_root`. -// Note: This approach might be inefficient for ZKVMs requiring minimal memory operations, which -// would explain why they have for the most part re-implemented this function. -pub fn verify_execution_witness( - witness: &ExecutionWitness, - pre_state_root: B256, -) -> Result<(SparseStateTrie, B256Map), StatelessValidationError> { - let mut trie = SparseStateTrie::new(DefaultBlindedProviderFactory); - let mut state_witness = B256Map::default(); - let mut bytecode = B256Map::default(); - - for rlp_encoded in &witness.state { - let hash = keccak256(rlp_encoded); - state_witness.insert(hash, rlp_encoded.clone()); - } - for rlp_encoded in &witness.codes { - let hash = keccak256(rlp_encoded); - bytecode.insert(hash, Bytecode::new_raw(rlp_encoded.clone())); - } - - // Reveal the witness with our state root - // This method builds a trie using the sparse trie using the state_witness with - // the root being the pre_state_root. - // Here are some things to note: - // - You can pass in more witnesses than is needed for the block execution. - // - If you try to get an account and it has not been seen. This means that the account - // was not inserted into the Trie. It does not mean that the account does not exist. - // In order to determine an account not existing, we must do an exclusion proof. - trie.reveal_witness(pre_state_root, &state_witness) - .map_err(|_e| StatelessValidationError::WitnessRevealFailed { pre_state_root })?; - - // Calculate the root - let computed_root = trie - .root() - .map_err(|_e| StatelessValidationError::StatelessPreStateRootCalculationFailed)?; - - if computed_root == pre_state_root { - Ok((trie, bytecode)) - } else { - Err(StatelessValidationError::PreStateRootMismatch { - got: computed_root, - expected: pre_state_root, - }) - } -} - /// Verifies the contiguity, number of ancestor headers and extracts their hashes. /// /// This function is used to prepare the data required for the `BLOCKHASH` @@ -295,19 +296,19 @@ pub fn verify_execution_witness( /// If both checks pass, it returns a [`BTreeMap`] mapping the block number of each /// ancestor header to its corresponding block hash. fn compute_ancestor_hashes( - current_block: &RecoveredBlock>, - ancestor_headers: &[Header], + current_block: &RecoveredBlock, + ancestor_headers: &[SealedHeader], ) -> Result, StatelessValidationError> { let mut ancestor_hashes = BTreeMap::new(); - let mut child_header = current_block.header(); + let mut child_header = current_block.sealed_header(); // Next verify that headers supplied are contiguous for parent_header in ancestor_headers.iter().rev() { let parent_hash = child_header.parent_hash(); ancestor_hashes.insert(parent_header.number, parent_hash); - if parent_hash != parent_header.hash_slow() { + if parent_hash != parent_header.hash() { return Err(StatelessValidationError::InvalidAncestorChain); // Blocks must be contiguous } diff --git a/crates/stateless/src/witness_db.rs b/crates/stateless/src/witness_db.rs index de4cfdf59b1..4a99c286ad3 100644 --- a/crates/stateless/src/witness_db.rs +++ b/crates/stateless/src/witness_db.rs @@ -1,25 +1,26 @@ //! Provides the [`WitnessDatabase`] type, an implementation of [`reth_revm::Database`] //! specifically designed for stateless execution environments. +use crate::trie::StatelessTrie; use alloc::{collections::btree_map::BTreeMap, format}; -use alloy_primitives::{keccak256, map::B256Map, Address, B256, U256}; -use alloy_rlp::Decodable; -use alloy_trie::{TrieAccount, EMPTY_ROOT_HASH}; +use alloy_primitives::{map::B256Map, Address, B256, U256}; use reth_errors::ProviderError; use reth_revm::{bytecode::Bytecode, state::AccountInfo, Database}; -use reth_trie_sparse::SparseStateTrie; /// An EVM database implementation backed by witness data. /// /// This struct implements the [`reth_revm::Database`] trait, allowing the EVM to execute /// transactions using: -/// - Account and storage slot data provided by a [`reth_trie_sparse::SparseStateTrie`]. +/// - Account and storage slot data provided by a [`StatelessTrie`] implementation. /// - Bytecode and ancestor block hashes provided by in-memory maps. /// /// This is designed for stateless execution scenarios where direct access to a full node's /// database is not available or desired. #[derive(Debug)] -pub(crate) struct WitnessDatabase<'a> { +pub(crate) struct WitnessDatabase<'a, T> +where + T: StatelessTrie, +{ /// Map of block numbers to block hashes. /// This is used to service the `BLOCKHASH` opcode. // TODO: use Vec instead -- ancestors should be contiguous @@ -34,10 +35,13 @@ pub(crate) struct WitnessDatabase<'a> { /// TODO: Ideally we do not have this trie and instead a simple map. /// TODO: Then as a corollary we can avoid unnecessary hashing in `Database::storage` /// TODO: and `Database::basic` without needing to cache the hashed Addresses and Keys - trie: &'a SparseStateTrie, + trie: &'a T, } -impl<'a> WitnessDatabase<'a> { +impl<'a, T> WitnessDatabase<'a, T> +where + T: StatelessTrie, +{ /// Creates a new [`WitnessDatabase`] instance. /// /// # Assumptions @@ -52,7 +56,7 @@ impl<'a> WitnessDatabase<'a> { /// contiguous chain of blocks. The caller is responsible for verifying the contiguity and /// the block limit. pub(crate) const fn new( - trie: &'a SparseStateTrie, + trie: &'a T, bytecode: B256Map, ancestor_hashes: BTreeMap, ) -> Self { @@ -60,80 +64,42 @@ impl<'a> WitnessDatabase<'a> { } } -impl Database for WitnessDatabase<'_> { +impl Database for WitnessDatabase<'_, T> +where + T: StatelessTrie, +{ /// The database error type. type Error = ProviderError; /// Get basic account information by hashing the address and looking up the account RLP - /// in the underlying [`SparseStateTrie`]. + /// in the underlying [`StatelessTrie`] implementation. /// /// Returns `Ok(None)` if the account is not found in the trie. fn basic(&mut self, address: Address) -> Result, Self::Error> { - let hashed_address = keccak256(address); - - if let Some(bytes) = self.trie.get_account_value(&hashed_address) { - let account = TrieAccount::decode(&mut bytes.as_slice())?; - return Ok(Some(AccountInfo { + self.trie.account(address).map(|opt| { + opt.map(|account| AccountInfo { balance: account.balance, nonce: account.nonce, code_hash: account.code_hash, code: None, - })); - } - - if !self.trie.check_valid_account_witness(hashed_address) { - return Err(ProviderError::TrieWitnessError(format!( - "incomplete account witness for {hashed_address:?}" - ))); - } - - Ok(None) + }) + }) } /// Get storage value of an account at a specific slot. /// /// Returns `U256::ZERO` if the slot is not found in the trie. fn storage(&mut self, address: Address, slot: U256) -> Result { - let hashed_address = keccak256(address); - let hashed_slot = keccak256(B256::from(slot)); - - if let Some(raw) = self.trie.get_storage_slot_value(&hashed_address, &hashed_slot) { - return Ok(U256::decode(&mut raw.as_slice())?) - } - - // Storage slot value is not present in the trie, validate that the witness is complete. - // If the account exists in the trie... - if let Some(bytes) = self.trie.get_account_value(&hashed_address) { - // ...check that its storage is either empty or the storage trie was sufficiently - // revealed... - let account = TrieAccount::decode(&mut bytes.as_slice())?; - if account.storage_root != EMPTY_ROOT_HASH && - !self.trie.check_valid_storage_witness(hashed_address, hashed_slot) - { - return Err(ProviderError::TrieWitnessError(format!( - "incomplete storage witness: prover must supply exclusion proof for slot {hashed_slot:?} in account {hashed_address:?}" - ))); - } - } else if !self.trie.check_valid_account_witness(hashed_address) { - // ...else if account is missing, validate that the account trie was sufficiently - // revealed. - return Err(ProviderError::TrieWitnessError(format!( - "incomplete account witness for {hashed_address:?}" - ))); - } - - Ok(U256::ZERO) + self.trie.storage(address, slot) } /// Get account code by its hash from the provided bytecode map. /// /// Returns an error if the bytecode for the given hash is not found in the map. fn code_by_hash(&mut self, code_hash: B256) -> Result { - let bytecode = self.bytecode.get(&code_hash).ok_or_else(|| { + self.bytecode.get(&code_hash).cloned().ok_or_else(|| { ProviderError::TrieWitnessError(format!("bytecode for {code_hash} not found")) - })?; - - Ok(bytecode.clone()) + }) } /// Get block hash by block number from the provided ancestor hashes map. diff --git a/crates/static-file/static-file/Cargo.toml b/crates/static-file/static-file/Cargo.toml index 38cfac36207..7ea23e0132f 100644 --- a/crates/static-file/static-file/Cargo.toml +++ b/crates/static-file/static-file/Cargo.toml @@ -31,7 +31,6 @@ rayon.workspace = true parking_lot = { workspace = true, features = ["send_guard", "arc_lock"] } [dev-dependencies] -reth-db = { workspace = true, features = ["test-utils"] } reth-stages = { workspace = true, features = ["test-utils"] } reth-testing-utils.workspace = true diff --git a/crates/static-file/static-file/README.md b/crates/static-file/static-file/README.md index a414ded0c67..fe40b73d13e 100644 --- a/crates/static-file/static-file/README.md +++ b/crates/static-file/static-file/README.md @@ -2,11 +2,11 @@ ## Overview -Data that has reached a finalized state and won't undergo further changes (essentially frozen) should be read without concerns of modification. This makes it unsuitable for traditional databases. +Data that has reached a finalized state and won't undergo further changes (essentially frozen) should be read without concerns about modification. This makes it unsuitable for traditional databases. This crate aims to copy this data from the current database to multiple static files, aggregated by block ranges. At every 500_000th block, a new static file is created. -Below are four diagrams illustrating on how data is served from static files to the provider. A glossary is also provided to explain the different (linked) components involved in these processes. +Below are four diagrams illustrating how data is served from static files to the provider. A glossary is also provided to explain the different (linked) components involved in these processes. ### Query Diagrams ([`Provider`](../../storage/provider/src/providers/database/mod.rs#L41)) @@ -106,9 +106,9 @@ In descending order of abstraction hierarchy: [`StaticFileProducer`](../../static-file/static-file/src/static_file_producer.rs#L25): A `reth` hook service that when triggered, **copies** finalized data from the database to the latest static file. Upon completion, it updates the internal index at `StaticFileProvider` with the new highest block and transaction on each specific segment. -[`StaticFileProvider`](../../storage/provider/src/providers/static_file/manager.rs#L44) A provider similar to `DatabaseProvider`, **managing all existing static_file files** and selecting the optimal one (by range and segment type) to fulfill a request. **A single instance is shared across all components and should be instantiated only once within `ProviderFactory`**. An immutable reference is given every time `ProviderFactory` creates a new `DatabaseProvider`. +[`StaticFileProvider`](../../storage/provider/src/providers/static_file/manager.rs#L44) A provider similar to `DatabaseProvider`, **managing all existing static files** and selecting the optimal one (by range and segment type) to fulfill a request. **A single instance is shared across all components and should be instantiated only once within `ProviderFactory`**. An immutable reference is given every time `ProviderFactory` creates a new `DatabaseProvider`. -[`StaticFileJarProvider`](../../storage/provider/src/providers/static_file/jar.rs#L42) A provider similar to `DatabaseProvider` that provides access to a **single static file segment data** one a specific block range. +[`StaticFileJarProvider`](../../storage/provider/src/providers/static_file/jar.rs#L42) A provider similar to `DatabaseProvider` that provides access to a **single static file segment data** on a specific block range. [`StaticFileCursor`](../../storage/db/src/static_file/cursor.rs#L11) An elevated abstraction of `NippyJarCursor` for simplified access. It associates the bitmasks with type decoding. For instance, `cursor.get_two::>(tx_number)` would yield `Tx` and `Signature`, eliminating the need to manage masks or invoke a decoder/decompressor. @@ -116,4 +116,4 @@ In descending order of abstraction hierarchy: [`NippyJarCursor`](../../storage/nippy-jar/src/cursor.rs#L12) Accessor of data in a `NippyJar` file. It enables queries either by row number (e.g., block number 1) or by a predefined key not part of the file (e.g., transaction hashes). **Currently, only queries by row number are being used.** If a file has multiple columns (e.g., `Header | HeaderTD | HeaderHash`), and one wishes to access only one of the column values, this can be accomplished by bitmasks. (e.g., for `HeaderTD`, the mask would be `0b010`). -[`NippyJar`](../../storage/nippy-jar/src/lib.rs#92) An append-or-truncate-only file format. It supports multiple columns, compression (e.g., Zstd (with and without dictionaries), lz4, uncompressed) and inclusion filters (e.g., cuckoo filter: `is hash X part of this dataset`). StaticFiles are organized by block ranges. (e.g., `TransactionStaticFile_0_-_499_999.jar` contains a transaction per row for all transactions between block `0` and block `499_999`). For more check the struct documentation. +[`NippyJar`](../../storage/nippy-jar/src/lib.rs#92) An append-or-truncate-only file format. It supports multiple columns, compression (e.g., Zstd (with and without dictionaries), lz4, uncompressed) and inclusion filters (e.g., cuckoo filter: `is hash X part of this dataset`). StaticFiles are organized by block ranges. (e.g., `TransactionStaticFile_0_-_499_999.jar` contains a transaction per row for all transactions between block `0` and block `499_999`). For more, check the struct documentation. diff --git a/crates/static-file/static-file/src/lib.rs b/crates/static-file/static-file/src/lib.rs index 1e9ffa15c66..7288129ca33 100644 --- a/crates/static-file/static-file/src/lib.rs +++ b/crates/static-file/static-file/src/lib.rs @@ -5,7 +5,7 @@ html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256", issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(test), warn(unused_crate_dependencies))] pub mod segments; diff --git a/crates/static-file/static-file/src/segments/headers.rs b/crates/static-file/static-file/src/segments/headers.rs deleted file mode 100644 index 5232061caaf..00000000000 --- a/crates/static-file/static-file/src/segments/headers.rs +++ /dev/null @@ -1,62 +0,0 @@ -use crate::segments::Segment; -use alloy_primitives::BlockNumber; -use reth_codecs::Compact; -use reth_db_api::{cursor::DbCursorRO, table::Value, tables, transaction::DbTx}; -use reth_primitives_traits::NodePrimitives; -use reth_provider::{providers::StaticFileWriter, DBProvider, StaticFileProviderFactory}; -use reth_static_file_types::StaticFileSegment; -use reth_storage_errors::provider::ProviderResult; -use std::ops::RangeInclusive; - -/// Static File segment responsible for [`StaticFileSegment::Headers`] part of data. -#[derive(Debug, Default)] -pub struct Headers; - -impl Segment for Headers -where - Provider: StaticFileProviderFactory> - + DBProvider, -{ - fn segment(&self) -> StaticFileSegment { - StaticFileSegment::Headers - } - - fn copy_to_static_files( - &self, - provider: Provider, - block_range: RangeInclusive, - ) -> ProviderResult<()> { - let static_file_provider = provider.static_file_provider(); - let mut static_file_writer = - static_file_provider.get_writer(*block_range.start(), StaticFileSegment::Headers)?; - - let mut headers_cursor = provider - .tx_ref() - .cursor_read::::BlockHeader>>( - )?; - let headers_walker = headers_cursor.walk_range(block_range.clone())?; - - let mut header_td_cursor = - provider.tx_ref().cursor_read::()?; - let header_td_walker = header_td_cursor.walk_range(block_range.clone())?; - - let mut canonical_headers_cursor = - provider.tx_ref().cursor_read::()?; - let canonical_headers_walker = canonical_headers_cursor.walk_range(block_range)?; - - for ((header_entry, header_td_entry), canonical_header_entry) in - headers_walker.zip(header_td_walker).zip(canonical_headers_walker) - { - let (header_block, header) = header_entry?; - let (header_td_block, header_td) = header_td_entry?; - let (canonical_header_block, canonical_header) = canonical_header_entry?; - - debug_assert_eq!(header_block, header_td_block); - debug_assert_eq!(header_td_block, canonical_header_block); - - static_file_writer.append_header(&header, header_td.0, &canonical_header)?; - } - - Ok(()) - } -} diff --git a/crates/static-file/static-file/src/segments/mod.rs b/crates/static-file/static-file/src/segments/mod.rs index fc79effdd5a..a1499a2eaa8 100644 --- a/crates/static-file/static-file/src/segments/mod.rs +++ b/crates/static-file/static-file/src/segments/mod.rs @@ -1,11 +1,5 @@ //! `StaticFile` segment implementations and utilities. -mod transactions; -pub use transactions::Transactions; - -mod headers; -pub use headers::Headers; - mod receipts; pub use receipts::Receipts; diff --git a/crates/static-file/static-file/src/segments/transactions.rs b/crates/static-file/static-file/src/segments/transactions.rs deleted file mode 100644 index 74cb58ed708..00000000000 --- a/crates/static-file/static-file/src/segments/transactions.rs +++ /dev/null @@ -1,60 +0,0 @@ -use crate::segments::Segment; -use alloy_primitives::BlockNumber; -use reth_codecs::Compact; -use reth_db_api::{cursor::DbCursorRO, table::Value, tables, transaction::DbTx}; -use reth_primitives_traits::NodePrimitives; -use reth_provider::{ - providers::StaticFileWriter, BlockReader, DBProvider, StaticFileProviderFactory, -}; -use reth_static_file_types::StaticFileSegment; -use reth_storage_errors::provider::{ProviderError, ProviderResult}; -use std::ops::RangeInclusive; - -/// Static File segment responsible for [`StaticFileSegment::Transactions`] part of data. -#[derive(Debug, Default)] -pub struct Transactions; - -impl Segment for Transactions -where - Provider: StaticFileProviderFactory> - + DBProvider - + BlockReader, -{ - fn segment(&self) -> StaticFileSegment { - StaticFileSegment::Transactions - } - - /// Write transactions from database table [`tables::Transactions`] to static files with segment - /// [`StaticFileSegment::Transactions`] for the provided block range. - fn copy_to_static_files( - &self, - provider: Provider, - block_range: RangeInclusive, - ) -> ProviderResult<()> { - let static_file_provider = provider.static_file_provider(); - let mut static_file_writer = static_file_provider - .get_writer(*block_range.start(), StaticFileSegment::Transactions)?; - - for block in block_range { - static_file_writer.increment_block(block)?; - - let block_body_indices = provider - .block_body_indices(block)? - .ok_or(ProviderError::BlockBodyIndicesNotFound(block))?; - - let mut transactions_cursor = provider.tx_ref().cursor_read::::SignedTx, - >>()?; - let transactions_walker = - transactions_cursor.walk_range(block_body_indices.tx_num_range())?; - - for entry in transactions_walker { - let (tx_number, transaction) = entry?; - - static_file_writer.append_transaction(tx_number, &transaction)?; - } - } - - Ok(()) - } -} diff --git a/crates/static-file/static-file/src/static_file_producer.rs b/crates/static-file/static-file/src/static_file_producer.rs index 491419ef4b6..03337f1fd7d 100644 --- a/crates/static-file/static-file/src/static_file_producer.rs +++ b/crates/static-file/static-file/src/static_file_producer.rs @@ -30,7 +30,7 @@ pub type StaticFileProducerResult = ProviderResult; pub type StaticFileProducerWithResult = (StaticFileProducer, StaticFileProducerResult); -/// Static File producer. It's a wrapper around [`StaticFileProducer`] that allows to share it +/// Static File producer. It's a wrapper around [`StaticFileProducerInner`] that allows to share it /// between threads. #[derive(Debug)] pub struct StaticFileProducer(Arc>>); @@ -131,12 +131,6 @@ where let mut segments = Vec::<(Box>, RangeInclusive)>::new(); - if let Some(block_range) = targets.transactions.clone() { - segments.push((Box::new(segments::Transactions), block_range)); - } - if let Some(block_range) = targets.headers.clone() { - segments.push((Box::new(segments::Headers), block_range)); - } if let Some(block_range) = targets.receipts.clone() { segments.push((Box::new(segments::Receipts), block_range)); } @@ -178,17 +172,11 @@ where /// Returns highest block numbers for all static file segments. pub fn copy_to_static_files(&self) -> ProviderResult { let provider = self.provider.database_provider_ro()?; - let stages_checkpoints = [StageId::Headers, StageId::Execution, StageId::Bodies] - .into_iter() + let stages_checkpoints = std::iter::once(StageId::Execution) .map(|stage| provider.get_stage_checkpoint(stage).map(|c| c.map(|c| c.block_number))) .collect::, _>>()?; - let highest_static_files = HighestStaticFiles { - headers: stages_checkpoints[0], - receipts: stages_checkpoints[1], - transactions: stages_checkpoints[2], - block_meta: stages_checkpoints[2], - }; + let highest_static_files = HighestStaticFiles { receipts: stages_checkpoints[0] }; let targets = self.get_static_file_targets(highest_static_files)?; self.run(targets)?; @@ -205,9 +193,6 @@ where let highest_static_files = self.provider.static_file_provider().get_highest_static_files(); let targets = StaticFileTargets { - headers: finalized_block_numbers.headers.and_then(|finalized_block_number| { - self.get_static_file_target(highest_static_files.headers, finalized_block_number) - }), // StaticFile receipts only if they're not pruned according to the user configuration receipts: if self.prune_modes.receipts.is_none() && self.prune_modes.receipts_log_filter.is_empty() @@ -221,15 +206,6 @@ where } else { None }, - transactions: finalized_block_numbers.transactions.and_then(|finalized_block_number| { - self.get_static_file_target( - highest_static_files.transactions, - finalized_block_number, - ) - }), - block_meta: finalized_block_numbers.block_meta.and_then(|finalized_block_number| { - self.get_static_file_target(highest_static_files.block_meta, finalized_block_number) - }), }; trace!( @@ -259,9 +235,8 @@ mod tests { use crate::static_file_producer::{ StaticFileProducer, StaticFileProducerInner, StaticFileTargets, }; - use alloy_primitives::{B256, U256}; + use alloy_primitives::B256; use assert_matches::assert_matches; - use reth_db_api::{database::Database, transaction::DbTx}; use reth_provider::{ providers::StaticFileWriter, test_utils::MockNodeTypesWithDB, ProviderError, ProviderFactory, StaticFileProviderFactory, @@ -293,19 +268,17 @@ mod tests { .expect("get static file writer for headers"); static_file_writer.prune_headers(blocks.len() as u64).unwrap(); static_file_writer.commit().expect("prune headers"); + drop(static_file_writer); - let tx = db.factory.db_ref().tx_mut().expect("init tx"); - for block in &blocks { - TestStageDB::insert_header(None, &tx, block.sealed_header(), U256::ZERO) - .expect("insert block header"); - } - tx.commit().expect("commit tx"); + db.insert_blocks(blocks.iter(), StorageKind::Database(None)).expect("insert blocks"); let mut receipts = Vec::new(); for block in &blocks { for transaction in &block.body().transactions { - receipts - .push((receipts.len() as u64, random_receipt(&mut rng, transaction, Some(0)))); + receipts.push(( + receipts.len() as u64, + random_receipt(&mut rng, transaction, Some(0), None), + )); } } db.insert_receipts(receipts).expect("insert receipts"); @@ -322,90 +295,36 @@ mod tests { StaticFileProducerInner::new(provider_factory.clone(), PruneModes::default()); let targets = static_file_producer - .get_static_file_targets(HighestStaticFiles { - headers: Some(1), - receipts: Some(1), - transactions: Some(1), - block_meta: None, - }) + .get_static_file_targets(HighestStaticFiles { receipts: Some(1) }) .expect("get static file targets"); - assert_eq!( - targets, - StaticFileTargets { - headers: Some(0..=1), - receipts: Some(0..=1), - transactions: Some(0..=1), - block_meta: None - } - ); + assert_eq!(targets, StaticFileTargets { receipts: Some(0..=1) }); assert_matches!(static_file_producer.run(targets), Ok(_)); assert_eq!( provider_factory.static_file_provider().get_highest_static_files(), - HighestStaticFiles { - headers: Some(1), - receipts: Some(1), - transactions: Some(1), - block_meta: None - } + HighestStaticFiles { receipts: Some(1) } ); let targets = static_file_producer - .get_static_file_targets(HighestStaticFiles { - headers: Some(3), - receipts: Some(3), - transactions: Some(3), - block_meta: None, - }) + .get_static_file_targets(HighestStaticFiles { receipts: Some(3) }) .expect("get static file targets"); - assert_eq!( - targets, - StaticFileTargets { - headers: Some(2..=3), - receipts: Some(2..=3), - transactions: Some(2..=3), - block_meta: None - } - ); + assert_eq!(targets, StaticFileTargets { receipts: Some(2..=3) }); assert_matches!(static_file_producer.run(targets), Ok(_)); assert_eq!( provider_factory.static_file_provider().get_highest_static_files(), - HighestStaticFiles { - headers: Some(3), - receipts: Some(3), - transactions: Some(3), - block_meta: None - } + HighestStaticFiles { receipts: Some(3) } ); let targets = static_file_producer - .get_static_file_targets(HighestStaticFiles { - headers: Some(4), - receipts: Some(4), - transactions: Some(4), - block_meta: None, - }) + .get_static_file_targets(HighestStaticFiles { receipts: Some(4) }) .expect("get static file targets"); - assert_eq!( - targets, - StaticFileTargets { - headers: Some(4..=4), - receipts: Some(4..=4), - transactions: Some(4..=4), - block_meta: None - } - ); + assert_eq!(targets, StaticFileTargets { receipts: Some(4..=4) }); assert_matches!( static_file_producer.run(targets), Err(ProviderError::BlockBodyIndicesNotFound(4)) ); assert_eq!( provider_factory.static_file_provider().get_highest_static_files(), - HighestStaticFiles { - headers: Some(3), - receipts: Some(3), - transactions: Some(3), - block_meta: None - } + HighestStaticFiles { receipts: Some(3) } ); } @@ -429,12 +348,7 @@ mod tests { std::thread::sleep(Duration::from_millis(100)); } let targets = locked_producer - .get_static_file_targets(HighestStaticFiles { - headers: Some(1), - receipts: Some(1), - transactions: Some(1), - block_meta: None, - }) + .get_static_file_targets(HighestStaticFiles { receipts: Some(1) }) .expect("get static file targets"); assert_matches!(locked_producer.run(targets.clone()), Ok(_)); tx.send(targets).unwrap(); diff --git a/crates/static-file/types/src/lib.rs b/crates/static-file/types/src/lib.rs index 5d638493643..9606b0ec98b 100644 --- a/crates/static-file/types/src/lib.rs +++ b/crates/static-file/types/src/lib.rs @@ -6,7 +6,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(feature = "std"), no_std)] extern crate alloc; @@ -27,44 +27,15 @@ pub const DEFAULT_BLOCKS_PER_STATIC_FILE: u64 = 500_000; /// Highest static file block numbers, per data segment. #[derive(Debug, Clone, Copy, Default, Eq, PartialEq)] pub struct HighestStaticFiles { - /// Highest static file block of headers, inclusive. - /// If [`None`], no static file is available. - pub headers: Option, /// Highest static file block of receipts, inclusive. /// If [`None`], no static file is available. pub receipts: Option, - /// Highest static file block of transactions, inclusive. - /// If [`None`], no static file is available. - pub transactions: Option, - /// Highest static file block of transactions, inclusive. - /// If [`None`], no static file is available. - pub block_meta: Option, } impl HighestStaticFiles { - /// Returns the highest static file if it exists for a segment - pub const fn highest(&self, segment: StaticFileSegment) -> Option { - match segment { - StaticFileSegment::Headers => self.headers, - StaticFileSegment::Transactions => self.transactions, - StaticFileSegment::Receipts => self.receipts, - StaticFileSegment::BlockMeta => self.block_meta, - } - } - - /// Returns a mutable reference to a static file segment - pub const fn as_mut(&mut self, segment: StaticFileSegment) -> &mut Option { - match segment { - StaticFileSegment::Headers => &mut self.headers, - StaticFileSegment::Transactions => &mut self.transactions, - StaticFileSegment::Receipts => &mut self.receipts, - StaticFileSegment::BlockMeta => &mut self.block_meta, - } - } - /// Returns an iterator over all static file segments fn iter(&self) -> impl Iterator> { - [self.headers, self.transactions, self.receipts, self.block_meta].into_iter() + [self.receipts].into_iter() } /// Returns the minimum block of all segments. @@ -81,43 +52,28 @@ impl HighestStaticFiles { /// Static File targets, per data segment, measured in [`BlockNumber`]. #[derive(Debug, Clone, Eq, PartialEq)] pub struct StaticFileTargets { - /// Targeted range of headers. - pub headers: Option>, /// Targeted range of receipts. pub receipts: Option>, - /// Targeted range of transactions. - pub transactions: Option>, - /// Targeted range of block meta. - pub block_meta: Option>, } impl StaticFileTargets { /// Returns `true` if any of the targets are [Some]. pub const fn any(&self) -> bool { - self.headers.is_some() || - self.receipts.is_some() || - self.transactions.is_some() || - self.block_meta.is_some() + self.receipts.is_some() } /// Returns `true` if all targets are either [`None`] or has beginning of the range equal to the /// highest static file. pub fn is_contiguous_to_highest_static_files(&self, static_files: HighestStaticFiles) -> bool { - [ - (self.headers.as_ref(), static_files.headers), - (self.receipts.as_ref(), static_files.receipts), - (self.transactions.as_ref(), static_files.transactions), - (self.block_meta.as_ref(), static_files.block_meta), - ] - .iter() - .all(|(target_block_range, highest_static_fileted_block)| { - target_block_range.is_none_or(|target_block_range| { - *target_block_range.start() == - highest_static_fileted_block.map_or(0, |highest_static_fileted_block| { - highest_static_fileted_block + 1 - }) - }) - }) + core::iter::once(&(self.receipts.as_ref(), static_files.receipts)).all( + |(target_block_range, highest_static_file_block)| { + target_block_range.is_none_or(|target_block_range| { + *target_block_range.start() == + highest_static_file_block + .map_or(0, |highest_static_file_block| highest_static_file_block + 1) + }) + }, + ) } } @@ -135,54 +91,9 @@ pub const fn find_fixed_range( mod tests { use super::*; - #[test] - fn test_highest_static_files_highest() { - let files = HighestStaticFiles { - headers: Some(100), - receipts: Some(200), - transactions: None, - block_meta: None, - }; - - // Test for headers segment - assert_eq!(files.highest(StaticFileSegment::Headers), Some(100)); - - // Test for receipts segment - assert_eq!(files.highest(StaticFileSegment::Receipts), Some(200)); - - // Test for transactions segment - assert_eq!(files.highest(StaticFileSegment::Transactions), None); - } - - #[test] - fn test_highest_static_files_as_mut() { - let mut files = HighestStaticFiles::default(); - - // Modify headers value - *files.as_mut(StaticFileSegment::Headers) = Some(150); - assert_eq!(files.headers, Some(150)); - - // Modify receipts value - *files.as_mut(StaticFileSegment::Receipts) = Some(250); - assert_eq!(files.receipts, Some(250)); - - // Modify transactions value - *files.as_mut(StaticFileSegment::Transactions) = Some(350); - assert_eq!(files.transactions, Some(350)); - - // Modify block meta value - *files.as_mut(StaticFileSegment::BlockMeta) = Some(350); - assert_eq!(files.block_meta, Some(350)); - } - #[test] fn test_highest_static_files_min() { - let files = HighestStaticFiles { - headers: Some(300), - receipts: Some(100), - transactions: None, - block_meta: None, - }; + let files = HighestStaticFiles { receipts: Some(100) }; // Minimum value among the available segments assert_eq!(files.min_block_num(), Some(100)); @@ -194,15 +105,10 @@ mod tests { #[test] fn test_highest_static_files_max() { - let files = HighestStaticFiles { - headers: Some(300), - receipts: Some(100), - transactions: Some(500), - block_meta: Some(500), - }; + let files = HighestStaticFiles { receipts: Some(100) }; // Maximum value among the available segments - assert_eq!(files.max_block_num(), Some(500)); + assert_eq!(files.max_block_num(), Some(100)); let empty_files = HighestStaticFiles::default(); // No values, should return None diff --git a/crates/static-file/types/src/segment.rs b/crates/static-file/types/src/segment.rs index 185eff18eae..ca7d9ef24d5 100644 --- a/crates/static-file/types/src/segment.rs +++ b/crates/static-file/types/src/segment.rs @@ -37,10 +37,6 @@ pub enum StaticFileSegment { #[strum(serialize = "receipts")] /// Static File segment responsible for the `Receipts` table. Receipts, - #[strum(serialize = "blockmeta")] - /// Static File segment responsible for the `BlockBodyIndices`, `BlockOmmers`, - /// `BlockWithdrawals` tables. - BlockMeta, } impl StaticFileSegment { @@ -50,15 +46,13 @@ impl StaticFileSegment { Self::Headers => "headers", Self::Transactions => "transactions", Self::Receipts => "receipts", - Self::BlockMeta => "blockmeta", } } /// Returns an iterator over all segments. pub fn iter() -> impl Iterator { - // The order of segments is significant and must be maintained to ensure correctness. For - // example, Transactions require BlockBodyIndices from Blockmeta to be sound. - [Self::Headers, Self::BlockMeta, Self::Transactions, Self::Receipts].into_iter() + // The order of segments is significant and must be maintained to ensure correctness. + [Self::Headers, Self::Transactions, Self::Receipts].into_iter() } /// Returns the default configuration of the segment. @@ -69,7 +63,7 @@ impl StaticFileSegment { /// Returns the number of columns for the segment pub const fn columns(&self) -> usize { match self { - Self::Headers | Self::BlockMeta => 3, + Self::Headers => 3, Self::Transactions | Self::Receipts => 1, } } @@ -133,11 +127,6 @@ impl StaticFileSegment { matches!(self, Self::Headers) } - /// Returns `true` if the segment is `StaticFileSegment::BlockMeta`. - pub const fn is_block_meta(&self) -> bool { - matches!(self, Self::BlockMeta) - } - /// Returns `true` if the segment is `StaticFileSegment::Receipts`. pub const fn is_receipts(&self) -> bool { matches!(self, Self::Receipts) @@ -150,7 +139,7 @@ impl StaticFileSegment { /// Returns `true` if a segment row is linked to a block. pub const fn is_block_based(&self) -> bool { - matches!(self, Self::Headers | Self::BlockMeta) + matches!(self, Self::Headers) } } @@ -228,12 +217,12 @@ impl SegmentHeader { /// Number of transactions. pub fn tx_len(&self) -> Option { - self.tx_range.as_ref().map(|r| (r.end() + 1) - r.start()) + self.tx_range.as_ref().map(|r| r.len()) } /// Number of blocks. pub fn block_len(&self) -> Option { - self.block_range.as_ref().map(|r| (r.end() + 1) - r.start()) + self.block_range.as_ref().map(|r| r.len()) } /// Increments block end range depending on segment @@ -340,6 +329,12 @@ impl SegmentRangeInclusive { pub const fn end(&self) -> u64 { self.end } + + /// Returns the length of the inclusive range. + #[allow(clippy::len_without_is_empty)] + pub const fn len(&self) -> u64 { + self.end.saturating_sub(self.start).saturating_add(1) + } } impl core::fmt::Display for SegmentRangeInclusive { diff --git a/crates/storage/codecs/derive/Cargo.toml b/crates/storage/codecs/derive/Cargo.toml index 6d1bffb4abe..3728e0c1750 100644 --- a/crates/storage/codecs/derive/Cargo.toml +++ b/crates/storage/codecs/derive/Cargo.toml @@ -15,7 +15,6 @@ workspace = true proc-macro = true [dependencies] -convert_case.workspace = true proc-macro2.workspace = true quote.workspace = true syn.workspace = true diff --git a/crates/storage/codecs/derive/src/arbitrary.rs b/crates/storage/codecs/derive/src/arbitrary.rs index 5713bb9b0ff..552e7d592d2 100644 --- a/crates/storage/codecs/derive/src/arbitrary.rs +++ b/crates/storage/codecs/derive/src/arbitrary.rs @@ -23,11 +23,11 @@ pub fn maybe_generate_tests( let mut iter = args.into_iter().peekable(); // we check if there's a crate argument which is used from inside the codecs crate directly - if let Some(arg) = iter.peek() { - if arg.to_string() == "crate" { - is_crate = true; - iter.next(); - } + if let Some(arg) = iter.peek() && + arg.to_string() == "crate" + { + is_crate = true; + iter.next(); } for arg in iter { diff --git a/crates/storage/codecs/derive/src/compact/flags.rs b/crates/storage/codecs/derive/src/compact/flags.rs index b6bad462917..c3e0b988cf6 100644 --- a/crates/storage/codecs/derive/src/compact/flags.rs +++ b/crates/storage/codecs/derive/src/compact/flags.rs @@ -51,7 +51,7 @@ pub(crate) fn generate_flag_struct( quote! { buf.get_u8(), }; - total_bytes.into() + total_bytes ]; let docs = format!( @@ -64,11 +64,11 @@ pub(crate) fn generate_flag_struct( impl<'a> #ident<'a> { #[doc = #bitflag_encoded_bytes] pub const fn bitflag_encoded_bytes() -> usize { - #total_bytes as usize + #total_bytes } #[doc = #bitflag_unused_bits] pub const fn bitflag_unused_bits() -> usize { - #unused_bits as usize + #unused_bits } } } @@ -77,11 +77,11 @@ pub(crate) fn generate_flag_struct( impl #ident { #[doc = #bitflag_encoded_bytes] pub const fn bitflag_encoded_bytes() -> usize { - #total_bytes as usize + #total_bytes } #[doc = #bitflag_unused_bits] pub const fn bitflag_unused_bits() -> usize { - #unused_bits as usize + #unused_bits } } } @@ -123,8 +123,8 @@ fn build_struct_field_flags( fields: Vec<&StructFieldDescriptor>, field_flags: &mut Vec, is_zstd: bool, -) -> u8 { - let mut total_bits = 0; +) -> usize { + let mut total_bits: usize = 0; // Find out the adequate bit size for the length of each field, if applicable. for field in fields { @@ -138,7 +138,7 @@ fn build_struct_field_flags( let name = format_ident!("{name}_len"); let bitsize = get_bit_size(ftype); let bsize = format_ident!("B{bitsize}"); - total_bits += bitsize; + total_bits += bitsize as usize; field_flags.push(quote! { pub #name: #bsize , @@ -170,7 +170,7 @@ fn build_struct_field_flags( /// skipped field. /// /// Returns the total number of bytes used by the flags struct and how many unused bits. -fn pad_flag_struct(total_bits: u8, field_flags: &mut Vec) -> (u8, u8) { +fn pad_flag_struct(total_bits: usize, field_flags: &mut Vec) -> (usize, usize) { let remaining = 8 - total_bits % 8; if remaining == 8 { (total_bits / 8, 0) diff --git a/crates/storage/codecs/derive/src/compact/generator.rs b/crates/storage/codecs/derive/src/compact/generator.rs index e6c06f44ad8..d72fc4644e9 100644 --- a/crates/storage/codecs/derive/src/compact/generator.rs +++ b/crates/storage/codecs/derive/src/compact/generator.rs @@ -2,7 +2,6 @@ use super::*; use crate::ZstdConfig; -use convert_case::{Case, Casing}; use syn::{Attribute, LitStr}; /// Generates code to implement the `Compact` trait for a data type. @@ -20,11 +19,6 @@ pub fn generate_from_to( let to_compact = generate_to_compact(fields, ident, zstd.clone(), &reth_codecs); let from_compact = generate_from_compact(fields, ident, zstd); - let snake_case_ident = ident.to_string().to_case(Case::Snake); - - let fuzz = format_ident!("fuzz_test_{snake_case_ident}"); - let test = format_ident!("fuzz_{snake_case_ident}"); - let lifetime = if has_lifetime { quote! { 'a } } else { @@ -58,33 +52,8 @@ pub fn generate_from_to( } }; - let fuzz_tests = if has_lifetime { - quote! {} - } else { - quote! { - #[cfg(test)] - #[expect(dead_code)] - #[test_fuzz::test_fuzz] - fn #fuzz(obj: #ident) { - use #reth_codecs::Compact; - let mut buf = vec![]; - let len = obj.clone().to_compact(&mut buf); - let (same_obj, buf) = #ident::from_compact(buf.as_ref(), len); - assert_eq!(obj, same_obj); - } - - #[test] - #[expect(missing_docs)] - pub fn #test() { - #fuzz(#ident::default()) - } - } - }; - // Build function quote! { - #fuzz_tests - #impl_compact { fn to_compact(&self, buf: &mut B) -> usize where B: #reth_codecs::__private::bytes::BufMut + AsMut<[u8]> { let mut flags = #flags::default(); diff --git a/crates/storage/codecs/derive/src/compact/mod.rs b/crates/storage/codecs/derive/src/compact/mod.rs index ae349cd06e5..ed43286923b 100644 --- a/crates/storage/codecs/derive/src/compact/mod.rs +++ b/crates/storage/codecs/derive/src/compact/mod.rs @@ -82,7 +82,7 @@ pub fn get_fields(data: &Data) -> FieldList { ); load_field(&data_fields.unnamed[0], &mut fields, false); } - syn::Fields::Unit => todo!(), + syn::Fields::Unit => unimplemented!("Compact does not support unit structs"), }, Data::Enum(data) => { for variant in &data.variants { @@ -106,7 +106,7 @@ pub fn get_fields(data: &Data) -> FieldList { } } } - Data::Union(_) => todo!(), + Data::Union(_) => unimplemented!("Compact does not support union types"), } fields @@ -171,28 +171,15 @@ fn load_field_from_segments( /// /// If so, we use another impl to code/decode its data. fn should_use_alt_impl(ftype: &str, segment: &syn::PathSegment) -> bool { - if ftype == "Vec" || ftype == "Option" { - if let syn::PathArguments::AngleBracketed(ref args) = segment.arguments { - if let Some(syn::GenericArgument::Type(syn::Type::Path(arg_path))) = args.args.last() { - if let (Some(path), 1) = - (arg_path.path.segments.first(), arg_path.path.segments.len()) - { - if [ - "B256", - "Address", - "Address", - "Bloom", - "TxHash", - "BlockHash", - "CompactPlaceholder", - ] - .contains(&path.ident.to_string().as_str()) - { - return true - } - } - } - } + if (ftype == "Vec" || ftype == "Option") && + let syn::PathArguments::AngleBracketed(ref args) = segment.arguments && + let Some(syn::GenericArgument::Type(syn::Type::Path(arg_path))) = args.args.last() && + let (Some(path), 1) = (arg_path.path.segments.first(), arg_path.path.segments.len()) && + ["B256", "Address", "Address", "Bloom", "TxHash", "BlockHash", "CompactPlaceholder"] + .iter() + .any(|&s| path.ident == s) + { + return true } false } @@ -251,11 +238,11 @@ mod tests { impl TestStruct { #[doc = "Used bytes by [`TestStructFlags`]"] pub const fn bitflag_encoded_bytes() -> usize { - 2u8 as usize + 2usize } #[doc = "Unused bits for new fields by [`TestStructFlags`]"] pub const fn bitflag_unused_bits() -> usize { - 1u8 as usize + 1usize } } @@ -290,21 +277,6 @@ mod tests { } } } - #[cfg(test)] - #[expect(dead_code)] - #[test_fuzz::test_fuzz] - fn fuzz_test_test_struct(obj: TestStruct) { - use reth_codecs::Compact; - let mut buf = vec![]; - let len = obj.clone().to_compact(&mut buf); - let (same_obj, buf) = TestStruct::from_compact(buf.as_ref(), len); - assert_eq!(obj, same_obj); - } - #[test] - #[expect(missing_docs)] - pub fn fuzz_test_struct() { - fuzz_test_test_struct(TestStruct::default()) - } impl reth_codecs::Compact for TestStruct { fn to_compact(&self, buf: &mut B) -> usize where B: reth_codecs::__private::bytes::BufMut + AsMut<[u8]> { let mut flags = TestStructFlags::default(); diff --git a/crates/storage/codecs/derive/src/compact/structs.rs b/crates/storage/codecs/derive/src/compact/structs.rs index f8ebda33499..4bafe730624 100644 --- a/crates/storage/codecs/derive/src/compact/structs.rs +++ b/crates/storage/codecs/derive/src/compact/structs.rs @@ -155,7 +155,7 @@ impl<'a> StructHandler<'a> { let (#name, new_buf) = #ident_type::#from_compact_ident(buf, flags.#len() as usize); }); } else { - todo!() + unreachable!("flag-type fields are always compact in Compact derive") } self.lines.push(quote! { buf = new_buf; diff --git a/crates/storage/codecs/derive/src/lib.rs b/crates/storage/codecs/derive/src/lib.rs index a835e8fab3c..8214366ac2e 100644 --- a/crates/storage/codecs/derive/src/lib.rs +++ b/crates/storage/codecs/derive/src/lib.rs @@ -7,7 +7,7 @@ )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] #![allow(unreachable_pub, missing_docs)] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] use proc_macro::TokenStream; use quote::{format_ident, quote}; @@ -69,8 +69,8 @@ pub fn derive_zstd(input: TokenStream) -> TokenStream { let mut decompressor = None; for attr in &input.attrs { - if attr.path().is_ident("reth_zstd") { - if let Err(err) = attr.parse_nested_meta(|meta| { + if attr.path().is_ident("reth_zstd") && + let Err(err) = attr.parse_nested_meta(|meta| { if meta.path.is_ident("compressor") { let value = meta.value()?; let path: syn::Path = value.parse()?; @@ -83,9 +83,9 @@ pub fn derive_zstd(input: TokenStream) -> TokenStream { return Err(meta.error("unsupported attribute")) } Ok(()) - }) { - return err.to_compile_error().into() - } + }) + { + return err.to_compile_error().into() } } diff --git a/crates/storage/codecs/src/alloy/authorization_list.rs b/crates/storage/codecs/src/alloy/authorization_list.rs index 1816d3dc202..23fa28a9e26 100644 --- a/crates/storage/codecs/src/alloy/authorization_list.rs +++ b/crates/storage/codecs/src/alloy/authorization_list.rs @@ -7,7 +7,7 @@ use bytes::Buf; use core::ops::Deref; use reth_codecs_derive::add_arbitrary_tests; -/// Authorization acts as bridge which simplifies Compact implementation for AlloyAuthorization. +/// Authorization acts as bridge which simplifies Compact implementation for `AlloyAuthorization`. /// /// Notice: Make sure this struct is 1:1 with `alloy_eips::eip7702::Authorization` #[derive(Debug, Clone, PartialEq, Eq, Default, Compact)] diff --git a/crates/storage/codecs/src/alloy/header.rs b/crates/storage/codecs/src/alloy/header.rs index eefb25a5193..b82760d166a 100644 --- a/crates/storage/codecs/src/alloy/header.rs +++ b/crates/storage/codecs/src/alloy/header.rs @@ -3,6 +3,7 @@ use crate::Compact; use alloy_consensus::Header as AlloyHeader; use alloy_primitives::{Address, BlockNumber, Bloom, Bytes, B256, U256}; +use reth_codecs_derive::{add_arbitrary_tests, generate_tests}; /// Block header /// @@ -19,6 +20,7 @@ use alloy_primitives::{Address, BlockNumber, Bloom, Bytes, B256, U256}; #[cfg_attr(feature = "test-utils", allow(unreachable_pub), visibility::make(pub))] #[derive(Debug, Clone, PartialEq, Eq, Hash, Default, Compact)] #[reth_codecs(crate = "crate")] +#[add_arbitrary_tests(crate, compact)] pub(crate) struct Header { parent_hash: B256, ommers_hash: B256, @@ -56,6 +58,7 @@ pub(crate) struct Header { #[cfg_attr(feature = "test-utils", allow(unreachable_pub), visibility::make(pub))] #[derive(Debug, Clone, PartialEq, Eq, Hash, Default, Compact)] #[reth_codecs(crate = "crate")] +#[add_arbitrary_tests(crate, compact)] pub(crate) struct HeaderExt { requests_hash: Option, } @@ -135,6 +138,8 @@ impl Compact for AlloyHeader { } } +generate_tests!(#[crate, compact] AlloyHeader, AlloyHeaderTests); + #[cfg(test)] mod tests { use super::*; diff --git a/crates/storage/codecs/src/alloy/transaction/eip1559.rs b/crates/storage/codecs/src/alloy/transaction/eip1559.rs index 6d910a6900c..f13422a2dea 100644 --- a/crates/storage/codecs/src/alloy/transaction/eip1559.rs +++ b/crates/storage/codecs/src/alloy/transaction/eip1559.rs @@ -53,7 +53,8 @@ impl Compact for AlloyTxEip1559 { } fn from_compact(buf: &[u8], len: usize) -> (Self, &[u8]) { - let (tx, _) = TxEip1559::from_compact(buf, len); + // Return the remaining slice from the inner from_compact to advance the cursor correctly. + let (tx, remaining) = TxEip1559::from_compact(buf, len); let alloy_tx = Self { chain_id: tx.chain_id, @@ -67,6 +68,6 @@ impl Compact for AlloyTxEip1559 { input: tx.input, }; - (alloy_tx, buf) + (alloy_tx, remaining) } } diff --git a/crates/storage/codecs/src/alloy/transaction/eip2930.rs b/crates/storage/codecs/src/alloy/transaction/eip2930.rs index aeb08f361be..a5c25a84d4f 100644 --- a/crates/storage/codecs/src/alloy/transaction/eip2930.rs +++ b/crates/storage/codecs/src/alloy/transaction/eip2930.rs @@ -52,7 +52,8 @@ impl Compact for AlloyTxEip2930 { } fn from_compact(buf: &[u8], len: usize) -> (Self, &[u8]) { - let (tx, _) = TxEip2930::from_compact(buf, len); + // Return the remaining slice from the inner from_compact to advance the cursor correctly. + let (tx, remaining) = TxEip2930::from_compact(buf, len); let alloy_tx = Self { chain_id: tx.chain_id, nonce: tx.nonce, @@ -63,6 +64,6 @@ impl Compact for AlloyTxEip2930 { access_list: tx.access_list, input: tx.input, }; - (alloy_tx, buf) + (alloy_tx, remaining) } } diff --git a/crates/storage/codecs/src/alloy/transaction/eip4844.rs b/crates/storage/codecs/src/alloy/transaction/eip4844.rs index 6367f3e08e7..6ea1927f7d5 100644 --- a/crates/storage/codecs/src/alloy/transaction/eip4844.rs +++ b/crates/storage/codecs/src/alloy/transaction/eip4844.rs @@ -68,7 +68,8 @@ impl Compact for AlloyTxEip4844 { } fn from_compact(buf: &[u8], len: usize) -> (Self, &[u8]) { - let (tx, _) = TxEip4844::from_compact(buf, len); + // Return the remaining slice from the inner from_compact to advance the cursor correctly. + let (tx, remaining) = TxEip4844::from_compact(buf, len); let alloy_tx = Self { chain_id: tx.chain_id, nonce: tx.nonce, @@ -82,7 +83,7 @@ impl Compact for AlloyTxEip4844 { max_fee_per_blob_gas: tx.max_fee_per_blob_gas, input: tx.input, }; - (alloy_tx, buf) + (alloy_tx, remaining) } } diff --git a/crates/storage/codecs/src/alloy/transaction/eip7702.rs b/crates/storage/codecs/src/alloy/transaction/eip7702.rs index eab10af0b66..95de81c3804 100644 --- a/crates/storage/codecs/src/alloy/transaction/eip7702.rs +++ b/crates/storage/codecs/src/alloy/transaction/eip7702.rs @@ -57,7 +57,8 @@ impl Compact for AlloyTxEip7702 { } fn from_compact(buf: &[u8], len: usize) -> (Self, &[u8]) { - let (tx, _) = TxEip7702::from_compact(buf, len); + // Return the remaining slice from the inner from_compact to advance the cursor correctly. + let (tx, remaining) = TxEip7702::from_compact(buf, len); let alloy_tx = Self { chain_id: tx.chain_id, nonce: tx.nonce, @@ -70,6 +71,6 @@ impl Compact for AlloyTxEip7702 { access_list: tx.access_list, authorization_list: tx.authorization_list, }; - (alloy_tx, buf) + (alloy_tx, remaining) } } diff --git a/crates/storage/codecs/src/alloy/transaction/ethereum.rs b/crates/storage/codecs/src/alloy/transaction/ethereum.rs index 14d51b866fb..7824f60301a 100644 --- a/crates/storage/codecs/src/alloy/transaction/ethereum.rs +++ b/crates/storage/codecs/src/alloy/transaction/ethereum.rs @@ -112,7 +112,8 @@ impl Envelope } } -pub(super) trait CompactEnvelope: Sized { +/// Compact serialization for transaction envelopes with compression and bitfield packing. +pub trait CompactEnvelope: Sized { /// Takes a buffer which can be written to. *Ideally*, it returns the length written to. fn to_compact(&self, buf: &mut B) -> usize where diff --git a/crates/storage/codecs/src/alloy/transaction/legacy.rs b/crates/storage/codecs/src/alloy/transaction/legacy.rs index 60250ba64af..c4caf97ac38 100644 --- a/crates/storage/codecs/src/alloy/transaction/legacy.rs +++ b/crates/storage/codecs/src/alloy/transaction/legacy.rs @@ -42,7 +42,7 @@ pub(crate) struct TxLegacy { value: U256, /// Input has two uses depending if transaction is Create or Call (if `to` field is None or /// Some). pub init: An unlimited size byte array specifying the - /// EVM-code for the account initialisation procedure CREATE, + /// EVM-code for the account initialization procedure CREATE, /// data: An unlimited size byte array specifying the /// input data of the message call, formally Td. input: Bytes, @@ -67,7 +67,8 @@ impl Compact for AlloyTxLegacy { } fn from_compact(buf: &[u8], len: usize) -> (Self, &[u8]) { - let (tx, _) = TxLegacy::from_compact(buf, len); + // Return the remaining slice from the inner from_compact to advance the cursor correctly. + let (tx, remaining) = TxLegacy::from_compact(buf, len); let alloy_tx = Self { chain_id: tx.chain_id, @@ -79,6 +80,6 @@ impl Compact for AlloyTxLegacy { input: tx.input, }; - (alloy_tx, buf) + (alloy_tx, remaining) } } diff --git a/crates/storage/codecs/src/alloy/transaction/mod.rs b/crates/storage/codecs/src/alloy/transaction/mod.rs index 47881b6f87a..f841ff24f17 100644 --- a/crates/storage/codecs/src/alloy/transaction/mod.rs +++ b/crates/storage/codecs/src/alloy/transaction/mod.rs @@ -56,7 +56,7 @@ where cond_mod!(eip1559, eip2930, eip4844, eip7702, legacy, txtype); mod ethereum; -pub use ethereum::{Envelope, FromTxCompact, ToTxCompact}; +pub use ethereum::{CompactEnvelope, Envelope, FromTxCompact, ToTxCompact}; #[cfg(all(feature = "test-utils", feature = "op"))] pub mod optimism; diff --git a/crates/storage/codecs/src/alloy/transaction/optimism.rs b/crates/storage/codecs/src/alloy/transaction/optimism.rs index 013b6f8bcf7..f6667efc81f 100644 --- a/crates/storage/codecs/src/alloy/transaction/optimism.rs +++ b/crates/storage/codecs/src/alloy/transaction/optimism.rs @@ -41,7 +41,7 @@ pub(crate) struct TxDeposit { value: U256, gas_limit: u64, is_system_transaction: bool, - eth_value: Option, + eth_value: u128, eth_tx_value: Option, input: Bytes, } @@ -55,7 +55,10 @@ impl Compact for AlloyTxDeposit { source_hash: self.source_hash, from: self.from, to: self.to, - mint: self.mint, + mint: match self.mint { + 0 => None, + v => Some(v), + }, value: self.value, gas_limit: self.gas_limit, is_system_transaction: self.is_system_transaction, @@ -67,12 +70,13 @@ impl Compact for AlloyTxDeposit { } fn from_compact(buf: &[u8], len: usize) -> (Self, &[u8]) { - let (tx, _) = TxDeposit::from_compact(buf, len); + // Return the remaining slice from the inner from_compact to advance the cursor correctly. + let (tx, remaining) = TxDeposit::from_compact(buf, len); let alloy_tx = Self { source_hash: tx.source_hash, from: tx.from, to: tx.to, - mint: tx.mint, + mint: tx.mint.unwrap_or_default(), value: tx.value, gas_limit: tx.gas_limit, is_system_transaction: tx.is_system_transaction, @@ -80,7 +84,7 @@ impl Compact for AlloyTxDeposit { eth_value: tx.eth_value, eth_tx_value: tx.eth_tx_value, }; - (alloy_tx, buf) + (alloy_tx, remaining) } } diff --git a/crates/storage/codecs/src/alloy/withdrawal.rs b/crates/storage/codecs/src/alloy/withdrawal.rs index 09e80d1faa7..6234eea3412 100644 --- a/crates/storage/codecs/src/alloy/withdrawal.rs +++ b/crates/storage/codecs/src/alloy/withdrawal.rs @@ -6,7 +6,7 @@ use alloy_eips::eip4895::{Withdrawal as AlloyWithdrawal, Withdrawals}; use alloy_primitives::Address; use reth_codecs_derive::add_arbitrary_tests; -/// Withdrawal acts as bridge which simplifies Compact implementation for AlloyWithdrawal. +/// Withdrawal acts as bridge which simplifies Compact implementation for `AlloyWithdrawal`. /// /// Notice: Make sure this struct is 1:1 with `alloy_eips::eip4895::Withdrawal` #[derive(Debug, Clone, PartialEq, Eq, Default, Compact)] diff --git a/crates/storage/codecs/src/lib.rs b/crates/storage/codecs/src/lib.rs index a9cb7f2fcd1..1ac37966c2e 100644 --- a/crates/storage/codecs/src/lib.rs +++ b/crates/storage/codecs/src/lib.rs @@ -14,7 +14,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(feature = "std"), no_std)] extern crate alloc; @@ -312,10 +312,9 @@ where return (None, buf) } - let (len, mut buf) = decode_varuint(buf); + let (len, buf) = decode_varuint(buf); - let (element, _) = T::from_compact(&buf[..len], len); - buf.advance(len); + let (element, buf) = T::from_compact(buf, len); (Some(element), buf) } diff --git a/crates/storage/db-api/Cargo.toml b/crates/storage/db-api/Cargo.toml index 3f7e5c7b1a7..bd77b9d63d7 100644 --- a/crates/storage/db-api/Cargo.toml +++ b/crates/storage/db-api/Cargo.toml @@ -28,7 +28,7 @@ alloy-genesis.workspace = true alloy-consensus.workspace = true # optimism -reth-optimism-primitives = { workspace = true, optional = true } +reth-optimism-primitives = { workspace = true, optional = true, features = ["serde", "reth-codec"] } # codecs modular-bitfield.workspace = true diff --git a/crates/storage/db-api/src/cursor.rs b/crates/storage/db-api/src/cursor.rs index 3aeee949ea1..068b64a3c97 100644 --- a/crates/storage/db-api/src/cursor.rs +++ b/crates/storage/db-api/src/cursor.rs @@ -87,7 +87,7 @@ pub trait DbDupCursorRO { /// | `key` | `subkey` | **Equivalent starting position** | /// |--------|----------|-----------------------------------------| /// | `None` | `None` | [`DbCursorRO::first()`] | - /// | `Some` | `None` | [`DbCursorRO::seek()`] | + /// | `Some` | `None` | [`DbCursorRO::seek_exact()`] | /// | `None` | `Some` | [`DbDupCursorRO::seek_by_key_subkey()`] | /// | `Some` | `Some` | [`DbDupCursorRO::seek_by_key_subkey()`] | fn walk_dup( diff --git a/crates/storage/db-api/src/lib.rs b/crates/storage/db-api/src/lib.rs index 96a3253a32f..f39b2c49708 100644 --- a/crates/storage/db-api/src/lib.rs +++ b/crates/storage/db-api/src/lib.rs @@ -57,7 +57,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] /// Common types used throughout the abstraction. pub mod common; diff --git a/crates/storage/db-api/src/mock.rs b/crates/storage/db-api/src/mock.rs index ece47f81ee5..60f69ae8f0d 100644 --- a/crates/storage/db-api/src/mock.rs +++ b/crates/storage/db-api/src/mock.rs @@ -1,4 +1,7 @@ -//! Mock database +//! Mock database implementation for testing and development. +//! +//! Provides lightweight mock implementations of database traits. All operations +//! are no-ops that return default values without persisting data. use crate::{ common::{IterPairResult, PairResult, ValueOnlyResult}, @@ -7,6 +10,7 @@ use crate::{ ReverseWalker, Walker, }, database::Database, + database_metrics::DatabaseMetrics, table::{DupSort, Encode, Table, TableImporter}, transaction::{DbTx, DbTxMut}, DatabaseError, @@ -14,30 +18,50 @@ use crate::{ use core::ops::Bound; use std::{collections::BTreeMap, ops::RangeBounds}; -/// Mock database used for testing with inner `BTreeMap` structure -// TODO +/// Mock database implementation for testing and development. +/// +/// Provides a lightweight implementation of the [`Database`] trait suitable +/// for testing scenarios where actual database operations are not required. #[derive(Clone, Debug, Default)] pub struct DatabaseMock { - /// Main data. TODO (Make it table aware) + /// Internal data storage using a `BTreeMap`. + /// + /// TODO: Make the mock database table-aware by properly utilizing + /// this data structure to simulate realistic database behavior during testing. pub data: BTreeMap, Vec>, } impl Database for DatabaseMock { type TX = TxMock; type TXMut = TxMock; + + /// Creates a new read-only transaction. + /// + /// This always succeeds and returns a default [`TxMock`] instance. + /// The mock transaction doesn't actually perform any database operations. fn tx(&self) -> Result { Ok(TxMock::default()) } + /// Creates a new read-write transaction. + /// + /// This always succeeds and returns a default [`TxMock`] instance. + /// The mock transaction doesn't actually perform any database operations. fn tx_mut(&self) -> Result { Ok(TxMock::default()) } } -/// Mock read only tx +impl DatabaseMetrics for DatabaseMock {} + +/// Mock transaction implementation for testing and development. +/// +/// Implements both [`DbTx`] and [`DbTxMut`] traits. All operations are no-ops +/// that return success or default values, suitable for testing database operations +/// without side effects. #[derive(Debug, Clone, Default)] pub struct TxMock { - /// Table representation + /// Internal table representation (currently unused). _table: BTreeMap, Vec>, } @@ -45,10 +69,20 @@ impl DbTx for TxMock { type Cursor = CursorMock; type DupCursor = CursorMock; + /// Retrieves a value by key from the specified table. + /// + /// **Mock behavior**: Always returns `None` regardless of the key. + /// This simulates a table with no data, which is typical for testing + /// scenarios where you want to verify that read operations are called + /// correctly without actually storing data. fn get(&self, _key: T::Key) -> Result, DatabaseError> { Ok(None) } + /// Retrieves a value by encoded key from the specified table. + /// + /// **Mock behavior**: Always returns `None` regardless of the encoded key. + /// This is equivalent to [`Self::get`] but works with pre-encoded keys. fn get_by_encoded_key( &self, _key: &::Encoded, @@ -56,24 +90,48 @@ impl DbTx for TxMock { Ok(None) } + /// Commits the transaction. + /// + /// **Mock behavior**: Always returns `Ok(true)`, indicating successful commit. + /// No actual data is persisted since this is a mock implementation. fn commit(self) -> Result { Ok(true) } + /// Aborts the transaction. + /// + /// **Mock behavior**: No-op. Since no data is actually stored in the mock, + /// there's nothing to rollback. fn abort(self) {} + /// Creates a read-only cursor for the specified table. + /// + /// **Mock behavior**: Returns a default [`CursorMock`] that will not + /// iterate over any data (all cursor operations return `None`). fn cursor_read(&self) -> Result, DatabaseError> { Ok(CursorMock { _cursor: 0 }) } + /// Creates a read-only duplicate cursor for the specified duplicate sort table. + /// + /// **Mock behavior**: Returns a default [`CursorMock`] that will not + /// iterate over any data (all cursor operations return `None`). fn cursor_dup_read(&self) -> Result, DatabaseError> { Ok(CursorMock { _cursor: 0 }) } + /// Returns the number of entries in the specified table. + /// + /// **Mock behavior**: Returns the length of the internal `_table` `BTreeMap`, + /// which is typically 0 since no data is actually stored. fn entries(&self) -> Result { Ok(self._table.len()) } + /// Disables long read transaction safety checks. + /// + /// **Mock behavior**: No-op. This is a performance optimization that + /// doesn't apply to the mock implementation. fn disable_long_read_transaction_safety(&mut self) {} } @@ -81,10 +139,19 @@ impl DbTxMut for TxMock { type CursorMut = CursorMock; type DupCursorMut = CursorMock; + /// Inserts or updates a key-value pair in the specified table. + /// + /// **Mock behavior**: Always returns `Ok(())` without actually storing + /// the data. This allows tests to verify that write operations are called + /// correctly without side effects. fn put(&self, _key: T::Key, _value: T::Value) -> Result<(), DatabaseError> { Ok(()) } + /// Deletes a key-value pair from the specified table. + /// + /// **Mock behavior**: Always returns `Ok(true)`, indicating successful + /// deletion, without actually removing any data. fn delete( &self, _key: T::Key, @@ -93,14 +160,26 @@ impl DbTxMut for TxMock { Ok(true) } + /// Clears all entries from the specified table. + /// + /// **Mock behavior**: Always returns `Ok(())` without actually clearing + /// any data. This simulates successful table clearing for testing purposes. fn clear(&self) -> Result<(), DatabaseError> { Ok(()) } + /// Creates a write cursor for the specified table. + /// + /// **Mock behavior**: Returns a default [`CursorMock`] that will not + /// iterate over any data and all write operations will be no-ops. fn cursor_write(&self) -> Result, DatabaseError> { Ok(CursorMock { _cursor: 0 }) } + /// Creates a write duplicate cursor for the specified duplicate sort table. + /// + /// **Mock behavior**: Returns a default [`CursorMock`] that will not + /// iterate over any data and all write operations will be no-ops. fn cursor_dup_write(&self) -> Result, DatabaseError> { Ok(CursorMock { _cursor: 0 }) } @@ -108,41 +187,61 @@ impl DbTxMut for TxMock { impl TableImporter for TxMock {} -/// Cursor that iterates over table +/// Mock cursor implementation for testing and development. +/// +/// Implements all cursor traits. All operations are no-ops that return empty +/// results, suitable for testing cursor operations without side effects. #[derive(Debug)] pub struct CursorMock { + /// Internal cursor position (currently unused). _cursor: u32, } impl DbCursorRO for CursorMock { + /// Moves to the first entry in the table. + /// **Mock behavior**: Always returns `None`. fn first(&mut self) -> PairResult { Ok(None) } + /// Seeks to an exact key match. + /// **Mock behavior**: Always returns `None`. fn seek_exact(&mut self, _key: T::Key) -> PairResult { Ok(None) } + /// Seeks to the first key greater than or equal to the given key. + /// **Mock behavior**: Always returns `None`. fn seek(&mut self, _key: T::Key) -> PairResult { Ok(None) } + /// Moves to the next entry. + /// **Mock behavior**: Always returns `None`. fn next(&mut self) -> PairResult { Ok(None) } + /// Moves to the previous entry. + /// **Mock behavior**: Always returns `None`. fn prev(&mut self) -> PairResult { Ok(None) } + /// Moves to the last entry in the table. + /// **Mock behavior**: Always returns `None`. fn last(&mut self) -> PairResult { Ok(None) } + /// Returns the current entry without moving the cursor. + /// **Mock behavior**: Always returns `None`. fn current(&mut self) -> PairResult { Ok(None) } + /// Creates a forward walker starting from the given key. + /// **Mock behavior**: Returns an empty walker that won't iterate over any data. fn walk(&mut self, start_key: Option) -> Result, DatabaseError> { let start: IterPairResult = match start_key { Some(key) => >::seek(self, key).transpose(), @@ -152,6 +251,8 @@ impl DbCursorRO for CursorMock { Ok(Walker::new(self, start)) } + /// Creates a range walker for the specified key range. + /// **Mock behavior**: Returns an empty walker that won't iterate over any data. fn walk_range( &mut self, range: impl RangeBounds, @@ -174,6 +275,8 @@ impl DbCursorRO for CursorMock { Ok(RangeWalker::new(self, start, end_key)) } + /// Creates a backward walker starting from the given key. + /// **Mock behavior**: Returns an empty walker that won't iterate over any data. fn walk_back( &mut self, start_key: Option, @@ -187,18 +290,26 @@ impl DbCursorRO for CursorMock { } impl DbDupCursorRO for CursorMock { + /// Moves to the next duplicate entry. + /// **Mock behavior**: Always returns `None`. fn next_dup(&mut self) -> PairResult { Ok(None) } + /// Moves to the next entry with a different key. + /// **Mock behavior**: Always returns `None`. fn next_no_dup(&mut self) -> PairResult { Ok(None) } + /// Moves to the next duplicate value. + /// **Mock behavior**: Always returns `None`. fn next_dup_val(&mut self) -> ValueOnlyResult { Ok(None) } + /// Seeks to a specific key-subkey combination. + /// **Mock behavior**: Always returns `None`. fn seek_by_key_subkey( &mut self, _key: ::Key, @@ -207,6 +318,8 @@ impl DbDupCursorRO for CursorMock { Ok(None) } + /// Creates a duplicate walker for the specified key and subkey. + /// **Mock behavior**: Returns an empty walker that won't iterate over any data. fn walk_dup( &mut self, _key: Option<::Key>, @@ -217,6 +330,8 @@ impl DbDupCursorRO for CursorMock { } impl DbCursorRW for CursorMock { + /// Inserts or updates a key-value pair at the current cursor position. + /// **Mock behavior**: Always succeeds without modifying any data. fn upsert( &mut self, _key: ::Key, @@ -225,6 +340,8 @@ impl DbCursorRW for CursorMock { Ok(()) } + /// Inserts a key-value pair at the current cursor position. + /// **Mock behavior**: Always succeeds without modifying any data. fn insert( &mut self, _key: ::Key, @@ -233,6 +350,8 @@ impl DbCursorRW for CursorMock { Ok(()) } + /// Appends a key-value pair at the end of the table. + /// **Mock behavior**: Always succeeds without modifying any data. fn append( &mut self, _key: ::Key, @@ -241,16 +360,22 @@ impl DbCursorRW for CursorMock { Ok(()) } + /// Deletes the entry at the current cursor position. + /// **Mock behavior**: Always succeeds without modifying any data. fn delete_current(&mut self) -> Result<(), DatabaseError> { Ok(()) } } impl DbDupCursorRW for CursorMock { + /// Deletes all duplicate entries at the current cursor position. + /// **Mock behavior**: Always succeeds without modifying any data. fn delete_current_duplicates(&mut self) -> Result<(), DatabaseError> { Ok(()) } + /// Appends a duplicate key-value pair. + /// **Mock behavior**: Always succeeds without modifying any data. fn append_dup(&mut self, _key: ::Key, _value: ::Value) -> Result<(), DatabaseError> { Ok(()) } diff --git a/crates/storage/db-api/src/models/accounts.rs b/crates/storage/db-api/src/models/accounts.rs index ad6e37e0ecb..41a11e1c7e5 100644 --- a/crates/storage/db-api/src/models/accounts.rs +++ b/crates/storage/db-api/src/models/accounts.rs @@ -1,14 +1,13 @@ //! Account related models and types. -use std::ops::{Range, RangeInclusive}; - use crate::{ impl_fixed_arbitrary, table::{Decode, Encode}, DatabaseError, }; -use alloy_primitives::{Address, BlockNumber, StorageKey}; +use alloy_primitives::{Address, BlockNumber, StorageKey, B256}; use serde::{Deserialize, Serialize}; +use std::ops::{Bound, Range, RangeBounds, RangeInclusive}; /// [`BlockNumber`] concatenated with [`Address`]. /// @@ -71,6 +70,81 @@ impl Decode for BlockNumberAddress { } } +/// A [`RangeBounds`] over a range of [`BlockNumberAddress`]s. Used to conveniently convert from a +/// range of [`BlockNumber`]s. +#[derive(Debug)] +pub struct BlockNumberAddressRange { + /// Starting bound of the range. + pub start: Bound, + /// Ending bound of the range. + pub end: Bound, +} + +impl RangeBounds for BlockNumberAddressRange { + fn start_bound(&self) -> Bound<&BlockNumberAddress> { + self.start.as_ref() + } + + fn end_bound(&self) -> Bound<&BlockNumberAddress> { + self.end.as_ref() + } +} + +impl> From for BlockNumberAddressRange { + fn from(r: R) -> Self { + let start = match r.start_bound() { + Bound::Included(n) => Bound::Included(BlockNumberAddress((*n, Address::ZERO))), + Bound::Excluded(n) => Bound::Included(BlockNumberAddress((n + 1, Address::ZERO))), + Bound::Unbounded => Bound::Unbounded, + }; + + let end = match r.end_bound() { + Bound::Included(n) => Bound::Excluded(BlockNumberAddress((n + 1, Address::ZERO))), + Bound::Excluded(n) => Bound::Excluded(BlockNumberAddress((*n, Address::ZERO))), + Bound::Unbounded => Bound::Unbounded, + }; + + Self { start, end } + } +} + +/// [`BlockNumber`] concatenated with [`B256`] (hashed address). +/// +/// Since it's used as a key, it isn't compressed when encoding it. +#[derive( + Debug, Default, Copy, Clone, PartialEq, Eq, Serialize, Deserialize, Ord, PartialOrd, Hash, +)] +pub struct BlockNumberHashedAddress(pub (BlockNumber, B256)); + +impl From<(BlockNumber, B256)> for BlockNumberHashedAddress { + fn from(tpl: (BlockNumber, B256)) -> Self { + Self(tpl) + } +} + +impl Encode for BlockNumberHashedAddress { + type Encoded = [u8; 40]; + + fn encode(self) -> Self::Encoded { + let block_number = self.0 .0; + let hashed_address = self.0 .1; + + let mut buf = [0u8; 40]; + + buf[..8].copy_from_slice(&block_number.to_be_bytes()); + buf[8..].copy_from_slice(hashed_address.as_slice()); + buf + } +} + +impl Decode for BlockNumberHashedAddress { + fn decode(value: &[u8]) -> Result { + let num = u64::from_be_bytes(value[..8].try_into().map_err(|_| DatabaseError::Decode)?); + let hash = B256::from_slice(&value[8..]); + Ok(Self((num, hash))) + } +} + /// [`Address`] concatenated with [`StorageKey`]. Used by `reth_etl` and history stages. /// /// Since it's used as a key, it isn't compressed when encoding it. @@ -102,18 +176,22 @@ impl Decode for AddressStorageKey { } } -impl_fixed_arbitrary!((BlockNumberAddress, 28), (AddressStorageKey, 52)); +impl_fixed_arbitrary!( + (BlockNumberAddress, 28), + (BlockNumberHashedAddress, 40), + (AddressStorageKey, 52) +); #[cfg(test)] mod tests { use super::*; + use alloy_primitives::address; use rand::{rng, Rng}; - use std::str::FromStr; #[test] fn test_block_number_address() { let num = 1u64; - let hash = Address::from_str("ba5e000000000000000000000000000000000000").unwrap(); + let hash = address!("0xba5e000000000000000000000000000000000000"); let key = BlockNumberAddress((num, hash)); let mut bytes = [0u8; 28]; @@ -135,10 +213,35 @@ mod tests { assert_eq!(bytes, Encode::encode(key)); } + #[test] + fn test_block_number_hashed_address() { + let num = 1u64; + let hash = B256::from_slice(&[0xba; 32]); + let key = BlockNumberHashedAddress((num, hash)); + + let mut bytes = [0u8; 40]; + bytes[..8].copy_from_slice(&num.to_be_bytes()); + bytes[8..].copy_from_slice(hash.as_slice()); + + let encoded = Encode::encode(key); + assert_eq!(encoded, bytes); + + let decoded: BlockNumberHashedAddress = Decode::decode(&encoded).unwrap(); + assert_eq!(decoded, key); + } + + #[test] + fn test_block_number_hashed_address_rand() { + let mut bytes = [0u8; 40]; + rng().fill(bytes.as_mut_slice()); + let key = BlockNumberHashedAddress::arbitrary(&mut Unstructured::new(&bytes)).unwrap(); + assert_eq!(bytes, Encode::encode(key)); + } + #[test] fn test_address_storage_key() { let storage_key = StorageKey::random(); - let address = Address::from_str("ba5e000000000000000000000000000000000000").unwrap(); + let address = address!("0xba5e000000000000000000000000000000000000"); let key = AddressStorageKey((address, storage_key)); let mut bytes = [0u8; 52]; diff --git a/crates/storage/db-api/src/models/integer_list.rs b/crates/storage/db-api/src/models/integer_list.rs index c252d5ee0c8..aae9e204528 100644 --- a/crates/storage/db-api/src/models/integer_list.rs +++ b/crates/storage/db-api/src/models/integer_list.rs @@ -70,14 +70,14 @@ impl IntegerList { self.0.clear(); } - /// Serializes a [`IntegerList`] into a sequence of bytes. + /// Serializes an [`IntegerList`] into a sequence of bytes. pub fn to_bytes(&self) -> Vec { let mut vec = Vec::with_capacity(self.0.serialized_size()); self.0.serialize_into(&mut vec).expect("not able to encode IntegerList"); vec } - /// Serializes a [`IntegerList`] into a sequence of bytes. + /// Serializes an [`IntegerList`] into a sequence of bytes. pub fn to_mut_bytes(&self, buf: &mut B) { self.0.serialize_into(buf.writer()).unwrap(); } diff --git a/crates/storage/db-api/src/models/mod.rs b/crates/storage/db-api/src/models/mod.rs index a255703266a..31d9b301f8c 100644 --- a/crates/storage/db-api/src/models/mod.rs +++ b/crates/storage/db-api/src/models/mod.rs @@ -12,7 +12,9 @@ use reth_ethereum_primitives::{Receipt, TransactionSigned, TxType}; use reth_primitives_traits::{Account, Bytecode, StorageEntry}; use reth_prune_types::{PruneCheckpoint, PruneSegment}; use reth_stages_types::StageCheckpoint; -use reth_trie_common::{StoredNibbles, StoredNibblesSubKey, *}; +use reth_trie_common::{ + StorageTrieEntry, StoredNibbles, StoredNibblesSubKey, TrieChangeSetsEntry, *, +}; use serde::{Deserialize, Serialize}; pub mod accounts; @@ -128,7 +130,7 @@ impl Encode for StoredNibbles { fn encode(self) -> Self::Encoded { // NOTE: This used to be `to_compact`, but all it does is append the bytes to the buffer, // so we can just use the implementation of `Into>` to reuse the buffer. - self.0.into() + self.0.to_vec() } } @@ -215,10 +217,11 @@ impl_compression_for_compact!( Header, Account, Log, - Receipt, + Receipt, TxType, StorageEntry, BranchNodeCompact, + TrieChangeSetsEntry, StoredNibbles, StoredNibblesSubKey, StorageTrieEntry, @@ -256,7 +259,7 @@ macro_rules! impl_compression_fixed_compact { } fn compress_to_buf>(&self, buf: &mut B) { - let _ = Compact::to_compact(self, buf); + let _ = Compact::to_compact(self, buf); } } @@ -278,7 +281,7 @@ impl_compression_fixed_compact!(B256, Address); macro_rules! add_wrapper_struct { ($(($name:tt, $wrapper:tt)),+) => { $( - /// Wrapper struct so it can use StructFlags from Compact, when used as pure table values. + /// Wrapper struct so it can use `StructFlags` from Compact, when used as pure table values. #[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize, Compact)] #[cfg_attr(any(test, feature = "arbitrary"), derive(arbitrary::Arbitrary))] #[add_arbitrary_tests(compact)] diff --git a/crates/storage/db-api/src/models/sharded_key.rs b/crates/storage/db-api/src/models/sharded_key.rs index d1de1bd400c..fdd583f0f55 100644 --- a/crates/storage/db-api/src/models/sharded_key.rs +++ b/crates/storage/db-api/src/models/sharded_key.rs @@ -55,7 +55,7 @@ impl Encode for ShardedKey { impl Decode for ShardedKey { fn decode(value: &[u8]) -> Result { - let (key, highest_tx_number) = value.split_last_chunk().unwrap(); + let (key, highest_tx_number) = value.split_last_chunk().ok_or(DatabaseError::Decode)?; let key = T::decode(key)?; let highest_tx_number = u64::from_be_bytes(*highest_tx_number); Ok(Self::new(key, highest_tx_number)) diff --git a/crates/storage/db-api/src/models/storage_sharded_key.rs b/crates/storage/db-api/src/models/storage_sharded_key.rs index a7a1ffb71be..6c7e40e2730 100644 --- a/crates/storage/db-api/src/models/storage_sharded_key.rs +++ b/crates/storage/db-api/src/models/storage_sharded_key.rs @@ -19,16 +19,16 @@ const STORAGE_SHARD_KEY_BYTES_SIZE: usize = 20 + 32 + 8; /// Sometimes data can be too big to be saved for a single key. This helps out by dividing the data /// into different shards. Example: /// -/// `Address | StorageKey | 200` -> data is from transition 0 to 200. +/// `Address | StorageKey | 200` -> data is from block 0 to 200. /// -/// `Address | StorageKey | 300` -> data is from transition 201 to 300. +/// `Address | StorageKey | 300` -> data is from block 201 to 300. #[derive( Debug, Default, Clone, Eq, Ord, PartialOrd, PartialEq, AsRef, Serialize, Deserialize, Hash, )] pub struct StorageShardedKey { /// Storage account address. pub address: Address, - /// Storage slot with highest transition id. + /// Storage slot with highest block number. #[as_ref] pub sharded_key: ShardedKey, } @@ -70,14 +70,14 @@ impl Decode for StorageShardedKey { if value.len() != STORAGE_SHARD_KEY_BYTES_SIZE { return Err(DatabaseError::Decode) } - let tx_num_index = value.len() - 8; + let block_num_index = value.len() - 8; - let highest_tx_number = u64::from_be_bytes( - value[tx_num_index..].try_into().map_err(|_| DatabaseError::Decode)?, + let highest_block_number = u64::from_be_bytes( + value[block_num_index..].try_into().map_err(|_| DatabaseError::Decode)?, ); let address = Address::decode(&value[..20])?; let storage_key = B256::decode(&value[20..52])?; - Ok(Self { address, sharded_key: ShardedKey::new(storage_key, highest_tx_number) }) + Ok(Self { address, sharded_key: ShardedKey::new(storage_key, highest_block_number) }) } } diff --git a/crates/storage/db-api/src/table.rs b/crates/storage/db-api/src/table.rs index 5715852a5dd..54517908de7 100644 --- a/crates/storage/db-api/src/table.rs +++ b/crates/storage/db-api/src/table.rs @@ -139,6 +139,9 @@ pub trait TableImporter: DbTxMut { } /// Imports table data from another transaction within a range. + /// + /// This method works correctly with both regular and `DupSort` tables. For `DupSort` tables, + /// all duplicate entries within the range are preserved during import. fn import_table_with_range( &self, source_tx: &R, diff --git a/crates/storage/db-api/src/tables/mod.rs b/crates/storage/db-api/src/tables/mod.rs index 5c8c02e9f25..cf2a20fff04 100644 --- a/crates/storage/db-api/src/tables/mod.rs +++ b/crates/storage/db-api/src/tables/mod.rs @@ -21,8 +21,8 @@ use crate::{ accounts::BlockNumberAddress, blocks::{HeaderHash, StoredBlockOmmers}, storage_sharded_key::StorageShardedKey, - AccountBeforeTx, ClientVersion, CompactU256, IntegerList, ShardedKey, - StoredBlockBodyIndices, StoredBlockWithdrawals, + AccountBeforeTx, BlockNumberHashedAddress, ClientVersion, CompactU256, IntegerList, + ShardedKey, StoredBlockBodyIndices, StoredBlockWithdrawals, }, table::{Decode, DupSort, Encode, Table, TableInfo}, }; @@ -32,7 +32,9 @@ use reth_ethereum_primitives::{Receipt, TransactionSigned}; use reth_primitives_traits::{Account, Bytecode, StorageEntry}; use reth_prune_types::{PruneCheckpoint, PruneSegment}; use reth_stages_types::StageCheckpoint; -use reth_trie_common::{BranchNodeCompact, StorageTrieEntry, StoredNibbles, StoredNibblesSubKey}; +use reth_trie_common::{ + BranchNodeCompact, StorageTrieEntry, StoredNibbles, StoredNibblesSubKey, TrieChangeSetsEntry, +}; use serde::{Deserialize, Serialize}; use std::fmt; @@ -306,7 +308,8 @@ tables! { type Value = HeaderHash; } - /// Stores the total difficulty from a block header. + /// Stores the total difficulty from block headers. + /// Note: Deprecated. table HeaderTerminalDifficulties { type Key = BlockNumber; type Value = CompactU256; @@ -405,8 +408,7 @@ tables! { /// the shard that equal or more than asked. For example: /// * For N=50 we would get first shard. /// * for N=150 we would get second shard. - /// * If max block number is 200 and we ask for N=250 we would fetch last shard and - /// know that needed entry is in `AccountPlainState`. + /// * If max block number is 200 and we ask for N=250 we would fetch last shard and know that needed entry is in `AccountPlainState`. /// * If there were no shard we would get `None` entry or entry of different storage key. /// /// Code example can be found in `reth_provider::HistoricalStateProviderRef` @@ -428,8 +430,7 @@ tables! { /// the shard that equal or more than asked. For example: /// * For N=50 we would get first shard. /// * for N=150 we would get second shard. - /// * If max block number is 200 and we ask for N=250 we would fetch last shard and - /// know that needed entry is in `StoragePlainState`. + /// * If max block number is 200 and we ask for N=250 we would fetch last shard and know that needed entry is in `StoragePlainState`. /// * If there were no shard we would get `None` entry or entry of different storage key. /// /// Code example can be found in `reth_provider::HistoricalStateProviderRef` @@ -481,13 +482,27 @@ tables! { type Value = BranchNodeCompact; } - /// From HashedAddress => NibblesSubKey => Intermediate value + /// From `HashedAddress` => `NibblesSubKey` => Intermediate value table StoragesTrie { type Key = B256; type Value = StorageTrieEntry; type SubKey = StoredNibblesSubKey; } + /// Stores the state of a node in the accounts trie prior to a particular block being executed. + table AccountsTrieChangeSets { + type Key = BlockNumber; + type Value = TrieChangeSetsEntry; + type SubKey = StoredNibblesSubKey; + } + + /// Stores the state of a node in a storage trie prior to a particular block being executed. + table StoragesTrieChangeSets { + type Key = BlockNumberHashedAddress; + type Value = TrieChangeSetsEntry; + type SubKey = StoredNibblesSubKey; + } + /// Stores the transaction sender for each canonical transaction. /// It is needed to speed up execution stage and allows fetching signer without doing /// transaction signed recovery @@ -532,8 +547,8 @@ tables! { pub enum ChainStateKey { /// Last finalized block key LastFinalizedBlock, - /// Last finalized block key - LastSafeBlockBlock, + /// Last safe block key + LastSafeBlock, } impl Encode for ChainStateKey { @@ -542,7 +557,7 @@ impl Encode for ChainStateKey { fn encode(self) -> Self::Encoded { match self { Self::LastFinalizedBlock => [0], - Self::LastSafeBlockBlock => [1], + Self::LastSafeBlock => [1], } } } @@ -551,7 +566,7 @@ impl Decode for ChainStateKey { fn decode(value: &[u8]) -> Result { match value { [0] => Ok(Self::LastFinalizedBlock), - [1] => Ok(Self::LastSafeBlockBlock), + [1] => Ok(Self::LastSafeBlock), _ => Err(crate::DatabaseError::Decode), } } diff --git a/crates/storage/db-api/src/transaction.rs b/crates/storage/db-api/src/transaction.rs index 96f609419f5..d6028b7c5e3 100644 --- a/crates/storage/db-api/src/transaction.rs +++ b/crates/storage/db-api/src/transaction.rs @@ -50,6 +50,12 @@ pub trait DbTxMut: Send + Sync { /// Put value to database fn put(&self, key: T::Key, value: T::Value) -> Result<(), DatabaseError>; + /// Append value with the largest key to database. This should have the same + /// outcome as `put`, but databases like MDBX provide dedicated modes to make + /// it much faster, typically from O(logN) down to O(1) thanks to no lookup. + fn append(&self, key: T::Key, value: T::Value) -> Result<(), DatabaseError> { + self.put::(key, value) + } /// Delete value from database fn delete(&self, key: T::Key, value: Option) -> Result; diff --git a/crates/storage/db-common/Cargo.toml b/crates/storage/db-common/Cargo.toml index 7d05bc9815f..bc8fc7a666d 100644 --- a/crates/storage/db-common/Cargo.toml +++ b/crates/storage/db-common/Cargo.toml @@ -22,12 +22,17 @@ reth-stages-types.workspace = true reth-fs-util.workspace = true reth-node-types.workspace = true reth-static-file-types.workspace = true +reth-execution-errors.workspace = true +reth-mantle-forks = { workspace = true, features = ["std"] } # eth alloy-consensus.workspace = true alloy-genesis.workspace = true alloy-primitives.workspace = true +# revm +revm.workspace = true + # misc eyre.workspace = true thiserror.workspace = true @@ -43,7 +48,6 @@ tracing.workspace = true [dev-dependencies] reth-db = { workspace = true, features = ["mdbx"] } reth-provider = { workspace = true, features = ["test-utils"] } -alloy-consensus.workspace = true [lints] workspace = true diff --git a/crates/storage/db-common/src/db_tool/mod.rs b/crates/storage/db-common/src/db_tool/mod.rs index 5866ad8ae2a..e9d7f81b0f6 100644 --- a/crates/storage/db-common/src/db_tool/mod.rs +++ b/crates/storage/db-common/src/db_tool/mod.rs @@ -185,7 +185,7 @@ pub struct ListFilter { impl ListFilter { /// If `search` has a list of bytes, then filter for rows that have this sequence. - pub fn has_search(&self) -> bool { + pub const fn has_search(&self) -> bool { !self.search.is_empty() } diff --git a/crates/storage/db-common/src/init.rs b/crates/storage/db-common/src/init.rs index a29d02a42c4..d5a16376e9c 100644 --- a/crates/storage/db-common/src/init.rs +++ b/crates/storage/db-common/src/init.rs @@ -2,23 +2,29 @@ use alloy_consensus::BlockHeader; use alloy_genesis::GenesisAccount; -use alloy_primitives::{map::HashMap, Address, B256, U256}; +use alloy_primitives::{keccak256, map::HashMap, Address, B256, U256}; use reth_chainspec::EthChainSpec; use reth_codecs::Compact; use reth_config::config::EtlConfig; use reth_db_api::{tables, transaction::DbTxMut, DatabaseError}; use reth_etl::Collector; -use reth_primitives_traits::{Account, Bytecode, GotExpected, NodePrimitives, StorageEntry}; +use reth_execution_errors::StateRootError; +use reth_primitives_traits::{ + Account, Bytecode, GotExpected, NodePrimitives, SealedHeader, StorageEntry, +}; use reth_provider::{ - errors::provider::ProviderResult, providers::StaticFileWriter, writer::UnifiedStorageWriter, - BlockHashReader, BlockNumReader, BundleStateInit, ChainSpecProvider, DBProvider, - DatabaseProviderFactory, ExecutionOutcome, HashingWriter, HeaderProvider, HistoryWriter, - OriginalValuesKnown, ProviderError, RevertsInit, StageCheckpointReader, StageCheckpointWriter, - StateWriter, StaticFileProviderFactory, StorageLocation, TrieWriter, + errors::provider::ProviderResult, providers::StaticFileWriter, BlockHashReader, BlockNumReader, + BundleStateInit, ChainSpecProvider, DBProvider, DatabaseProviderFactory, ExecutionOutcome, + HashingWriter, HeaderProvider, HistoryWriter, OriginalValuesKnown, ProviderError, RevertsInit, + StageCheckpointReader, StageCheckpointWriter, StateWriter, StaticFileProviderFactory, + TrieWriter, }; use reth_stages_types::{StageCheckpoint, StageId}; use reth_static_file_types::StaticFileSegment; -use reth_trie::{IntermediateStateRootState, StateRoot as StateRootComputer, StateRootProgress}; +use reth_trie::{ + prefix_set::{TriePrefixSets, TriePrefixSetsMut}, + IntermediateStateRootState, Nibbles, StateRoot as StateRootComputer, StateRootProgress, +}; use reth_trie_db::DatabaseStateRoot; use serde::{Deserialize, Serialize}; use std::io::BufRead; @@ -63,6 +69,9 @@ pub enum InitStorageError { /// Provider error. #[error(transparent)] Provider(#[from] ProviderError), + /// State root error while computing the state root + #[error(transparent)] + StateRootError(#[from] StateRootError), /// State root doesn't match the expected one. #[error("state root mismatch: {_0}")] StateRootMismatch(GotExpected), @@ -88,6 +97,7 @@ where + HeaderProvider + HashingWriter + StateWriter + + TrieWriter + AsRef, PF::ChainSpec: EthChainSpec
::BlockHeader>, { @@ -138,22 +148,22 @@ where insert_genesis_state(&provider_rw, alloc.iter())?; + // compute state root to populate trie tables + compute_state_root(&provider_rw, None)?; + // insert sync stage for stage in StageId::ALL { provider_rw.save_stage_checkpoint(stage, Default::default())?; } - let static_file_provider = provider_rw.static_file_provider(); // Static file segments start empty, so we need to initialize the genesis block. - let segment = StaticFileSegment::Receipts; - static_file_provider.latest_writer(segment)?.increment_block(0)?; - - let segment = StaticFileSegment::Transactions; - static_file_provider.latest_writer(segment)?.increment_block(0)?; + let static_file_provider = provider_rw.static_file_provider(); + static_file_provider.latest_writer(StaticFileSegment::Receipts)?.increment_block(0)?; + static_file_provider.latest_writer(StaticFileSegment::Transactions)?.increment_block(0)?; // `commit_unwind`` will first commit the DB and then the static file provider, which is // necessary on `init_genesis`. - UnifiedStorageWriter::commit_unwind(provider_rw)?; + provider_rw.commit()?; Ok(hash) } @@ -253,11 +263,7 @@ where Vec::new(), ); - provider.write_state( - &execution_outcome, - OriginalValuesKnown::Yes, - StorageLocation::Database, - )?; + provider.write_state(&execution_outcome, OriginalValuesKnown::Yes)?; trace!(target: "reth::cli", "Inserted state"); @@ -341,9 +347,8 @@ where match static_file_provider.block_hash(0) { Ok(None) | Err(ProviderError::MissingStaticFileBlock(StaticFileSegment::Headers, 0)) => { - let (difficulty, hash) = (header.difficulty(), block_hash); let mut writer = static_file_provider.latest_writer(StaticFileSegment::Headers)?; - writer.append_header(header, difficulty, &hash)?; + writer.append_header(header, &block_hash)?; } Ok(Some(_)) => {} Err(e) => return Err(e), @@ -385,11 +390,16 @@ where } let block = provider_rw.last_block_number()?; - let hash = provider_rw.block_hash(block)?.unwrap(); - let expected_state_root = provider_rw + + let hash = provider_rw + .block_hash(block)? + .ok_or_else(|| eyre::eyre!("Block hash not found for block {}", block))?; + let header = provider_rw .header_by_number(block)? - .ok_or_else(|| ProviderError::HeaderNotFound(block.into()))? - .state_root(); + .map(SealedHeader::seal_slow) + .ok_or_else(|| ProviderError::HeaderNotFound(block.into()))?; + + let expected_state_root = header.state_root(); // first line can be state root let dump_state_root = parse_state_root(&mut reader)?; @@ -397,6 +407,7 @@ where error!(target: "reth::cli", ?dump_state_root, ?expected_state_root, + header=?header.num_hash(), "State root from state dump does not match state root in current header." ); return Err(InitStorageError::StateRootMismatch(GotExpected { @@ -415,11 +426,14 @@ where // remaining lines are accounts let collector = parse_accounts(&mut reader, etl_config)?; - // write state to db - dump_state(collector, provider_rw, block)?; + // write state to db and collect prefix sets + let mut prefix_sets = TriePrefixSetsMut::default(); + dump_state(collector, provider_rw, block, &mut prefix_sets)?; + + info!(target: "reth::cli", "All accounts written to database, starting state root computation (may take some time)"); // compute and compare state root. this advances the stage checkpoints. - let computed_state_root = compute_state_root(provider_rw)?; + let computed_state_root = compute_state_root(provider_rw, Some(prefix_sets.freeze()))?; if computed_state_root == expected_state_root { info!(target: "reth::cli", ?computed_state_root, @@ -432,6 +446,12 @@ where "Computed state root does not match state root in state dump" ); + // Export computed state for debugging purposes + info!(target: "reth::cli", "Exporting computed state for debugging..."); + if let Err(e) = export_state_on_mismatch(provider_rw, "computed_state_mismatch.json", Some(computed_state_root)) { + tracing::warn!(target: "reth::cli", error = ?e, "Failed to export computed state for debugging"); + } + return Err(InitStorageError::StateRootMismatch(GotExpected { got: computed_state_root, expected: expected_state_root, @@ -476,7 +496,8 @@ fn parse_accounts( let GenesisAccountWithAddress { genesis_account, address } = serde_json::from_str(&line)?; collector.insert(address, genesis_account)?; - if !collector.is_empty() && collector.len() % AVERAGE_COUNT_ACCOUNTS_PER_GB_STATE_DUMP == 0 + if !collector.is_empty() && + collector.len().is_multiple_of(AVERAGE_COUNT_ACCOUNTS_PER_GB_STATE_DUMP) { info!(target: "reth::cli", parsed_new_accounts=collector.len(), @@ -494,6 +515,7 @@ fn dump_state( mut collector: Collector, provider_rw: &Provider, block: u64, + prefix_sets: &mut TriePrefixSetsMut, ) -> Result<(), eyre::Error> where Provider: StaticFileProviderFactory @@ -513,9 +535,25 @@ where let (address, _) = Address::from_compact(address.as_slice(), address.len()); let (account, _) = GenesisAccount::from_compact(account.as_slice(), account.len()); + // Add to prefix sets + let hashed_address = keccak256(address); + prefix_sets.account_prefix_set.insert(Nibbles::unpack(hashed_address)); + + // Add storage keys to prefix sets if storage exists + if let Some(ref storage) = account.storage { + for key in storage.keys() { + let hashed_key = keccak256(key); + prefix_sets + .storage_prefix_sets + .entry(hashed_address) + .or_default() + .insert(Nibbles::unpack(hashed_key)); + } + } + accounts.push((address, account)); - if (index > 0 && index % AVERAGE_COUNT_ACCOUNTS_PER_GB_STATE_DUMP == 0) || + if (index > 0 && index.is_multiple_of(AVERAGE_COUNT_ACCOUNTS_PER_GB_STATE_DUMP)) || index == accounts_len - 1 { total_inserted_accounts += accounts.len(); @@ -552,7 +590,10 @@ where /// Computes the state root (from scratch) based on the accounts and storages present in the /// database. -fn compute_state_root(provider: &Provider) -> eyre::Result +fn compute_state_root( + provider: &Provider, + prefix_sets: Option, +) -> Result where Provider: DBProvider + TrieWriter, { @@ -563,16 +604,20 @@ where let mut total_flushed_updates = 0; loop { - match StateRootComputer::from_tx(tx) - .with_intermediate_state(intermediate_state) - .root_with_progress()? - { + let mut state_root = + StateRootComputer::from_tx(tx).with_intermediate_state(intermediate_state); + + if let Some(sets) = prefix_sets.clone() { + state_root = state_root.with_prefix_sets(sets); + } + + match state_root.root_with_progress()? { StateRootProgress::Progress(state, _, updates) => { - let updated_len = provider.write_trie_updates(&updates)?; + let updated_len = provider.write_trie_updates(updates)?; total_flushed_updates += updated_len; trace!(target: "reth::cli", - last_account_key = %state.last_account_key, + last_account_key = %state.account_root_state.last_hashed_key, updated_len, total_flushed_updates, "Flushing trie updates" @@ -580,7 +625,7 @@ where intermediate_state = Some(*state); - if total_flushed_updates % SOFT_LIMIT_COUNT_FLUSHED_UPDATES == 0 { + if total_flushed_updates.is_multiple_of(SOFT_LIMIT_COUNT_FLUSHED_UPDATES) { info!(target: "reth::cli", total_flushed_updates, "Flushing trie updates" @@ -588,7 +633,7 @@ where } } StateRootProgress::Complete(root, _, updates) => { - let updated_len = provider.write_trie_updates(&updates)?; + let updated_len = provider.write_trie_updates(updates)?; total_flushed_updates += updated_len; trace!(target: "reth::cli", @@ -621,6 +666,37 @@ struct GenesisAccountWithAddress { address: Address, } +/// Export computed state for debugging when state root mismatch occurs. +/// +/// This function exports all accounts and their storage from the database to a JSON file. +/// It's useful for debugging state root mismatches by comparing the computed state with +/// the expected state. +/// +/// In the init stage, all state has been written to the database, so `bundle_state` is empty. +/// We pass `false` to export all account storage details from the database. +fn export_state_on_mismatch( + provider: &Provider, + filename: &str, + state_root: Option, +) -> eyre::Result<()> +where + Provider: DBProvider, +{ + use reth_mantle_forks::debug::state_export::export_full_state_with_bundle; + use revm::database::BundleState; + + info!(target: "reth::cli", "Starting full state export to file: {}", filename); + + // In init stage, all state is in the database, bundle_state is empty + let empty_bundle = BundleState::default(); + + // Export with false to include all account storage (init scenario) + export_full_state_with_bundle(provider, &empty_bundle, filename, state_root, false)?; + + info!(target: "reth::cli", "State export completed: {}", filename); + Ok(()) +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/storage/db-common/src/lib.rs b/crates/storage/db-common/src/lib.rs index 173e5314340..22e49abfb05 100644 --- a/crates/storage/db-common/src/lib.rs +++ b/crates/storage/db-common/src/lib.rs @@ -6,7 +6,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] pub mod init; diff --git a/crates/storage/db-models/Cargo.toml b/crates/storage/db-models/Cargo.toml index eb74e227e6d..8f6a101eb78 100644 --- a/crates/storage/db-models/Cargo.toml +++ b/crates/storage/db-models/Cargo.toml @@ -36,12 +36,10 @@ reth-primitives-traits = { workspace = true, features = ["arbitrary", "reth-code reth-codecs.workspace = true bytes.workspace = true -modular-bitfield.workspace = true arbitrary = { workspace = true, features = ["derive"] } proptest.workspace = true proptest-arbitrary-interop.workspace = true -test-fuzz.workspace = true [features] default = ["std"] diff --git a/crates/storage/db-models/src/accounts.rs b/crates/storage/db-models/src/accounts.rs index 477c18f1c00..cbae5d84aa6 100644 --- a/crates/storage/db-models/src/accounts.rs +++ b/crates/storage/db-models/src/accounts.rs @@ -27,10 +27,7 @@ impl reth_codecs::Compact for AccountBeforeTx { // for now put full bytes and later compress it. buf.put_slice(self.address.as_slice()); - let mut acc_len = 0; - if let Some(account) = self.info { - acc_len = account.to_compact(buf); - } + let acc_len = if let Some(account) = self.info { account.to_compact(buf) } else { 0 }; acc_len + 20 } diff --git a/crates/storage/db-models/src/client_version.rs b/crates/storage/db-models/src/client_version.rs index e87a82e729d..ce6ced8a653 100644 --- a/crates/storage/db-models/src/client_version.rs +++ b/crates/storage/db-models/src/client_version.rs @@ -18,7 +18,7 @@ pub struct ClientVersion { impl ClientVersion { /// Returns `true` if no version fields are set. - pub fn is_empty(&self) -> bool { + pub const fn is_empty(&self) -> bool { self.version.is_empty() && self.git_sha.is_empty() && self.build_timestamp.is_empty() } } @@ -29,9 +29,10 @@ impl reth_codecs::Compact for ClientVersion { where B: bytes::BufMut + AsMut<[u8]>, { - self.version.to_compact(buf); - self.git_sha.to_compact(buf); - self.build_timestamp.to_compact(buf) + let version_size = self.version.to_compact(buf); + let git_sha_size = self.git_sha.to_compact(buf); + let build_timestamp_size = self.build_timestamp.to_compact(buf); + version_size + git_sha_size + build_timestamp_size } fn from_compact(buf: &[u8], len: usize) -> (Self, &[u8]) { diff --git a/crates/storage/db-models/src/lib.rs b/crates/storage/db-models/src/lib.rs index 87a1b3f62c6..db1c99b5e16 100644 --- a/crates/storage/db-models/src/lib.rs +++ b/crates/storage/db-models/src/lib.rs @@ -6,7 +6,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(feature = "std"), no_std)] extern crate alloc; diff --git a/crates/storage/db/Cargo.toml b/crates/storage/db/Cargo.toml index 719c7b785c1..2dd2517acfd 100644 --- a/crates/storage/db/Cargo.toml +++ b/crates/storage/db/Cargo.toml @@ -46,6 +46,7 @@ strum = { workspace = true, features = ["derive"], optional = true } [dev-dependencies] # reth libs with arbitrary reth-primitives-traits = { workspace = true, features = ["reth-codec"] } +reth-prune-types.workspace = true alloy-primitives = { workspace = true, features = ["getrandom"] } alloy-consensus.workspace = true @@ -81,6 +82,7 @@ test-utils = [ "reth-db-api/test-utils", "reth-nippy-jar/test-utils", "reth-primitives-traits/test-utils", + "reth-prune-types/test-utils", ] bench = ["reth-db-api/bench"] arbitrary = [ @@ -88,6 +90,7 @@ arbitrary = [ "alloy-primitives/arbitrary", "alloy-consensus/arbitrary", "reth-primitives-traits/arbitrary", + "reth-prune-types/arbitrary", ] op = [ "reth-db-api/op", @@ -109,3 +112,8 @@ harness = false name = "get" required-features = ["test-utils"] harness = false + +[[bench]] +name = "put" +required-features = ["test-utils"] +harness = false diff --git a/crates/storage/db/benches/criterion.rs b/crates/storage/db/benches/criterion.rs index 64d6fbdbfdf..7d62384c164 100644 --- a/crates/storage/db/benches/criterion.rs +++ b/crates/storage/db/benches/criterion.rs @@ -31,7 +31,6 @@ pub fn db(c: &mut Criterion) { group.warm_up_time(std::time::Duration::from_millis(200)); measure_table_db::(&mut group); - measure_table_db::(&mut group); measure_table_db::(&mut group); measure_table_db::(&mut group); measure_table_db::(&mut group); @@ -48,7 +47,6 @@ pub fn serialization(c: &mut Criterion) { group.warm_up_time(std::time::Duration::from_millis(200)); measure_table_serialization::(&mut group); - measure_table_serialization::(&mut group); measure_table_serialization::(&mut group); measure_table_serialization::(&mut group); measure_table_serialization::(&mut group); diff --git a/crates/storage/db/benches/put.rs b/crates/storage/db/benches/put.rs new file mode 100644 index 00000000000..b91634734ad --- /dev/null +++ b/crates/storage/db/benches/put.rs @@ -0,0 +1,44 @@ +#![allow(missing_docs)] + +use alloy_primitives::B256; +use criterion::{criterion_group, criterion_main, Criterion}; +use reth_db::{test_utils::create_test_rw_db_with_path, CanonicalHeaders, Database}; +use reth_db_api::transaction::DbTxMut; + +mod utils; +use utils::BENCH_DB_PATH; + +const NUM_BLOCKS: u64 = 1_000_000; + +criterion_group! { + name = benches; + config = Criterion::default(); + targets = put +} +criterion_main!(benches); + +// Small benchmark showing that `append` is much faster than `put` when keys are put in order +fn put(c: &mut Criterion) { + let mut group = c.benchmark_group("Put"); + + let setup = || { + let _ = std::fs::remove_dir_all(BENCH_DB_PATH); + create_test_rw_db_with_path(BENCH_DB_PATH).tx_mut().expect("tx") + }; + + group.bench_function("put", |b| { + b.iter_with_setup(setup, |tx| { + for i in 0..NUM_BLOCKS { + tx.put::(i, B256::ZERO).unwrap(); + } + }) + }); + + group.bench_function("append", |b| { + b.iter_with_setup(setup, |tx| { + for i in 0..NUM_BLOCKS { + tx.append::(i, B256::ZERO).unwrap(); + } + }) + }); +} diff --git a/crates/storage/db/src/implementation/mdbx/cursor.rs b/crates/storage/db/src/implementation/mdbx/cursor.rs index 0bbb75ce4b5..5ca6eacb6c7 100644 --- a/crates/storage/db/src/implementation/mdbx/cursor.rs +++ b/crates/storage/db/src/implementation/mdbx/cursor.rs @@ -345,3 +345,110 @@ impl DbDupCursorRW for Cursor { ) } } + +#[cfg(test)] +mod tests { + use crate::{ + mdbx::{DatabaseArguments, DatabaseEnv, DatabaseEnvKind}, + tables::StorageChangeSets, + Database, + }; + use alloy_primitives::{address, Address, B256, U256}; + use reth_db_api::{ + cursor::{DbCursorRO, DbDupCursorRW}, + models::{BlockNumberAddress, ClientVersion}, + table::TableImporter, + transaction::{DbTx, DbTxMut}, + }; + use reth_primitives_traits::StorageEntry; + use std::sync::Arc; + use tempfile::TempDir; + + fn create_test_db() -> Arc { + let path = TempDir::new().unwrap(); + let mut db = DatabaseEnv::open( + path.path(), + DatabaseEnvKind::RW, + DatabaseArguments::new(ClientVersion::default()), + ) + .unwrap(); + db.create_tables().unwrap(); + Arc::new(db) + } + + #[test] + fn test_import_table_with_range_works_on_dupsort() { + let addr1 = address!("0000000000000000000000000000000000000001"); + let addr2 = address!("0000000000000000000000000000000000000002"); + let addr3 = address!("0000000000000000000000000000000000000003"); + let source_db = create_test_db(); + let target_db = create_test_db(); + let test_data = vec![ + ( + BlockNumberAddress((100, addr1)), + StorageEntry { key: B256::with_last_byte(1), value: U256::from(100) }, + ), + ( + BlockNumberAddress((100, addr1)), + StorageEntry { key: B256::with_last_byte(2), value: U256::from(200) }, + ), + ( + BlockNumberAddress((100, addr1)), + StorageEntry { key: B256::with_last_byte(3), value: U256::from(300) }, + ), + ( + BlockNumberAddress((101, addr1)), + StorageEntry { key: B256::with_last_byte(1), value: U256::from(400) }, + ), + ( + BlockNumberAddress((101, addr2)), + StorageEntry { key: B256::with_last_byte(1), value: U256::from(500) }, + ), + ( + BlockNumberAddress((101, addr2)), + StorageEntry { key: B256::with_last_byte(2), value: U256::from(600) }, + ), + ( + BlockNumberAddress((102, addr3)), + StorageEntry { key: B256::with_last_byte(1), value: U256::from(700) }, + ), + ]; + + // setup data + let tx = source_db.tx_mut().unwrap(); + { + let mut cursor = tx.cursor_dup_write::().unwrap(); + for (key, value) in &test_data { + cursor.append_dup(*key, *value).unwrap(); + } + } + tx.commit().unwrap(); + + // import data from source db to target + let source_tx = source_db.tx().unwrap(); + let target_tx = target_db.tx_mut().unwrap(); + + target_tx + .import_table_with_range::( + &source_tx, + Some(BlockNumberAddress((100, Address::ZERO))), + BlockNumberAddress((102, Address::repeat_byte(0xff))), + ) + .unwrap(); + target_tx.commit().unwrap(); + + // fetch all data from target db + let verify_tx = target_db.tx().unwrap(); + let mut cursor = verify_tx.cursor_dup_read::().unwrap(); + let copied: Vec<_> = cursor.walk(None).unwrap().collect::, _>>().unwrap(); + + // verify each entry matches the test data + assert_eq!(copied.len(), test_data.len(), "Should copy all entries including duplicates"); + for ((copied_key, copied_value), (expected_key, expected_value)) in + copied.iter().zip(test_data.iter()) + { + assert_eq!(copied_key, expected_key); + assert_eq!(copied_value, expected_value); + } + } +} diff --git a/crates/storage/db/src/implementation/mdbx/mod.rs b/crates/storage/db/src/implementation/mdbx/mod.rs index 5cb8652d8f3..b00bfd3c9a5 100644 --- a/crates/storage/db/src/implementation/mdbx/mod.rs +++ b/crates/storage/db/src/implementation/mdbx/mod.rs @@ -23,6 +23,7 @@ use reth_libmdbx::{ use reth_storage_errors::db::LogLevel; use reth_tracing::tracing::error; use std::{ + collections::HashMap, ops::{Deref, Range}, path::Path, sync::Arc, @@ -99,6 +100,25 @@ pub struct DatabaseArguments { /// /// This flag affects only at environment opening but can't be changed after. exclusive: Option, + /// MDBX allows up to 32767 readers (`MDBX_READERS_LIMIT`). This arg is to configure the max + /// readers. + max_readers: Option, + /// Defines the synchronization strategy used by the MDBX database when writing data to disk. + /// + /// This determines how aggressively MDBX ensures data durability versus prioritizing + /// performance. The available modes are: + /// + /// - [`SyncMode::Durable`]: Ensures all transactions are fully flushed to disk before they are + /// considered committed. This provides the highest level of durability and crash safety + /// but may have a performance cost. + /// - [`SyncMode::SafeNoSync`]: Skips certain fsync operations to improve write performance. + /// This mode still maintains database integrity but may lose the most recent transactions if + /// the system crashes unexpectedly. + /// + /// Choose `Durable` if consistency and crash safety are critical (e.g., production + /// environments). Choose `SafeNoSync` if performance is more important and occasional data + /// loss is acceptable (e.g., testing or ephemeral data). + sync_mode: SyncMode, } impl Default for DatabaseArguments { @@ -113,7 +133,7 @@ impl DatabaseArguments { Self { client_version, geometry: Geometry { - size: Some(0..(4 * TERABYTE)), + size: Some(0..(8 * TERABYTE)), growth_step: Some(4 * GIGABYTE as isize), shrink_threshold: Some(0), page_size: Some(PageSize::Set(default_page_size())), @@ -121,6 +141,8 @@ impl DatabaseArguments { log_level: None, max_read_transaction_duration: None, exclusive: None, + max_readers: None, + sync_mode: SyncMode::Durable, } } @@ -132,6 +154,15 @@ impl DatabaseArguments { self } + /// Sets the database sync mode. + pub const fn with_sync_mode(mut self, sync_mode: Option) -> Self { + if let Some(sync_mode) = sync_mode { + self.sync_mode = sync_mode; + } + + self + } + /// Configures the database growth step in bytes. pub const fn with_growth_step(mut self, growth_step: Option) -> Self { if let Some(growth_step) = growth_step { @@ -146,12 +177,20 @@ impl DatabaseArguments { self } + /// Set the maximum duration of a read transaction. + pub const fn max_read_transaction_duration( + &mut self, + max_read_transaction_duration: Option, + ) { + self.max_read_transaction_duration = max_read_transaction_duration; + } + /// Set the maximum duration of a read transaction. pub const fn with_max_read_transaction_duration( mut self, max_read_transaction_duration: Option, ) -> Self { - self.max_read_transaction_duration = max_read_transaction_duration; + self.max_read_transaction_duration(max_read_transaction_duration); self } @@ -161,6 +200,12 @@ impl DatabaseArguments { self } + /// Set `max_readers` flag. + pub const fn with_max_readers(mut self, max_readers: Option) -> Self { + self.max_readers = max_readers; + self + } + /// Returns the client version if any. pub const fn client_version(&self) -> &ClientVersion { &self.client_version @@ -172,6 +217,12 @@ impl DatabaseArguments { pub struct DatabaseEnv { /// Libmdbx-sys environment. inner: Environment, + /// Opened DBIs for reuse. + /// Important: Do not manually close these DBIs, like via `mdbx_dbi_close`. + /// More generally, do not dynamically create, re-open, or drop tables at + /// runtime. It's better to perform table creation and migration only once + /// at startup. + dbis: Arc>, /// Cache for metric handles. If `None`, metrics are not recorded. metrics: Option>, /// Write lock for when dealing with a read-write environment. @@ -183,16 +234,18 @@ impl Database for DatabaseEnv { type TXMut = tx::Tx; fn tx(&self) -> Result { - Tx::new_with_metrics( + Tx::new( self.inner.begin_ro_txn().map_err(|e| DatabaseError::InitTx(e.into()))?, + self.dbis.clone(), self.metrics.clone(), ) .map_err(|e| DatabaseError::InitTx(e.into())) } fn tx_mut(&self) -> Result { - Tx::new_with_metrics( + Tx::new( self.inner.begin_rw_txn().map_err(|e| DatabaseError::InitTx(e.into()))?, + self.dbis.clone(), self.metrics.clone(), ) .map_err(|e| DatabaseError::InitTx(e.into())) @@ -302,7 +355,7 @@ impl DatabaseEnv { DatabaseEnvKind::RW => { // enable writemap mode in RW mode inner_env.write_map(); - Mode::ReadWrite { sync_mode: SyncMode::Durable } + Mode::ReadWrite { sync_mode: args.sync_mode } } }; @@ -367,7 +420,7 @@ impl DatabaseEnv { ..Default::default() }); // Configure more readers - inner_env.set_max_readers(DEFAULT_MAX_READERS); + inner_env.set_max_readers(args.max_readers.unwrap_or(DEFAULT_MAX_READERS)); // This parameter sets the maximum size of the "reclaimed list", and the unit of measurement // is "pages". Reclaimed list is the list of freed pages that's populated during the // lifetime of DB transaction, and through which MDBX searches when it needs to insert new @@ -427,6 +480,7 @@ impl DatabaseEnv { let env = Self { inner: inner_env.open(path).map_err(|e| DatabaseError::Open(e.into()))?, + dbis: Arc::default(), metrics: None, _lock_file, }; @@ -441,25 +495,60 @@ impl DatabaseEnv { } /// Creates all the tables defined in [`Tables`], if necessary. - pub fn create_tables(&self) -> Result<(), DatabaseError> { - self.create_tables_for::() + /// + /// This keeps tracks of the created table handles and stores them for better efficiency. + pub fn create_tables(&mut self) -> Result<(), DatabaseError> { + self.create_and_track_tables_for::() } /// Creates all the tables defined in the given [`TableSet`], if necessary. - pub fn create_tables_for(&self) -> Result<(), DatabaseError> { + /// + /// This keeps tracks of the created table handles and stores them for better efficiency. + pub fn create_and_track_tables_for(&mut self) -> Result<(), DatabaseError> { + let handles = self._create_tables::()?; + // Note: This is okay because self has mutable access here and `DatabaseEnv` must be Arc'ed + // before it can be shared. + let dbis = Arc::make_mut(&mut self.dbis); + dbis.extend(handles); + + Ok(()) + } + + /// Creates all the tables defined in [`Tables`], if necessary. + /// + /// If this type is unique the created handle for the tables will be updated. + /// + /// This is recommended to be called during initialization to create and track additional tables + /// after the default [`Self::create_tables`] are created. + pub fn create_tables_for(self: &mut Arc) -> Result<(), DatabaseError> { + let handles = self._create_tables::()?; + if let Some(db) = Arc::get_mut(self) { + // Note: The db is unique and the dbis as well, and they can also be cloned. + let dbis = Arc::make_mut(&mut db.dbis); + dbis.extend(handles); + } + Ok(()) + } + + /// Creates the tables and returns the identifiers of the tables. + fn _create_tables( + &self, + ) -> Result, DatabaseError> { + let mut handles = Vec::new(); let tx = self.inner.begin_rw_txn().map_err(|e| DatabaseError::InitTx(e.into()))?; for table in TS::tables() { let flags = if table.is_dupsort() { DatabaseFlags::DUP_SORT } else { DatabaseFlags::default() }; - tx.create_db(Some(table.name()), flags) + let db = tx + .create_db(Some(table.name()), flags) .map_err(|e| DatabaseError::CreateTable(e.into()))?; + handles.push((table.name(), db.dbi())); } tx.commit().map_err(|e| DatabaseError::Commit(e.into()))?; - - Ok(()) + Ok(handles) } /// Records version that accesses the database with write privileges. @@ -519,14 +608,15 @@ mod tests { fn create_test_db(kind: DatabaseEnvKind) -> Arc { Arc::new(create_test_db_with_path( kind, - &tempfile::TempDir::new().expect(ERROR_TEMPDIR).into_path(), + &tempfile::TempDir::new().expect(ERROR_TEMPDIR).keep(), )) } /// Create database for testing with specified path fn create_test_db_with_path(kind: DatabaseEnvKind, path: &Path) -> DatabaseEnv { - let env = DatabaseEnv::open(path, kind, DatabaseArguments::new(ClientVersion::default())) - .expect(ERROR_DB_CREATION); + let mut env = + DatabaseEnv::open(path, kind, DatabaseArguments::new(ClientVersion::default())) + .expect(ERROR_DB_CREATION); env.create_tables().expect(ERROR_TABLE_CREATION); env } @@ -884,9 +974,7 @@ mod tests { // Seek exact let exact = cursor.seek_exact(missing_key).unwrap(); assert_eq!(exact, None); - assert_eq!(cursor.current(), Ok(Some((missing_key + 1, B256::ZERO)))); - assert_eq!(cursor.prev(), Ok(Some((missing_key - 1, B256::ZERO)))); - assert_eq!(cursor.prev(), Ok(Some((missing_key - 2, B256::ZERO)))); + assert_eq!(cursor.current(), Ok(None)); } #[test] @@ -971,11 +1059,14 @@ mod tests { // Seek & delete key2 again assert_eq!(cursor.seek_exact(key2), Ok(None)); - assert_eq!(cursor.delete_current(), Ok(())); + assert_eq!( + cursor.delete_current(), + Err(DatabaseError::Delete(reth_libmdbx::Error::NoData.into())) + ); // Assert that key1 is still there assert_eq!(cursor.seek_exact(key1), Ok(Some((key1, Account::default())))); - // Assert that key3 was deleted - assert_eq!(cursor.seek_exact(key3), Ok(None)); + // Assert that key3 is still there + assert_eq!(cursor.seek_exact(key3), Ok(Some((key3, Account::default())))); } #[test] @@ -1170,7 +1261,7 @@ mod tests { #[test] fn db_closure_put_get() { - let path = TempDir::new().expect(ERROR_TEMPDIR).into_path(); + let path = TempDir::new().expect(ERROR_TEMPDIR).keep(); let value = Account { nonce: 18446744073709551615, @@ -1249,6 +1340,34 @@ mod tests { } } + #[test] + fn db_walk_dup_with_not_existing_key() { + let env = create_test_db(DatabaseEnvKind::RW); + let key = Address::from_str("0xa2c122be93b0074270ebee7f6b7292c7deb45047") + .expect(ERROR_ETH_ADDRESS); + + // PUT (0,0) + let value00 = StorageEntry::default(); + env.update(|tx| tx.put::(key, value00).expect(ERROR_PUT)).unwrap(); + + // PUT (2,2) + let value22 = StorageEntry { key: B256::with_last_byte(2), value: U256::from(2) }; + env.update(|tx| tx.put::(key, value22).expect(ERROR_PUT)).unwrap(); + + // PUT (1,1) + let value11 = StorageEntry { key: B256::with_last_byte(1), value: U256::from(1) }; + env.update(|tx| tx.put::(key, value11).expect(ERROR_PUT)).unwrap(); + + // Try to walk_dup with not existing key should immediately return None + { + let tx = env.tx().expect(ERROR_INIT_TX); + let mut cursor = tx.cursor_dup_read::().unwrap(); + let not_existing_key = Address::ZERO; + let mut walker = cursor.walk_dup(Some(not_existing_key), None).unwrap(); + assert_eq!(walker.next(), None); + } + } + #[test] fn db_iterate_over_all_dup_values() { let env = create_test_db(DatabaseEnvKind::RW); @@ -1343,8 +1462,9 @@ mod tests { let db: Arc = create_test_db(DatabaseEnvKind::RW); let real_key = address!("0xa2c122be93b0074270ebee7f6b7292c7deb45047"); - for i in 1..5 { - let key = ShardedKey::new(real_key, i * 100); + let shards = 5; + for i in 1..=shards { + let key = ShardedKey::new(real_key, if i == shards { u64::MAX } else { i * 100 }); let list = IntegerList::new_pre_sorted([i * 100u64]); db.update(|tx| tx.put::(key.clone(), list.clone()).expect("")) diff --git a/crates/storage/db/src/implementation/mdbx/tx.rs b/crates/storage/db/src/implementation/mdbx/tx.rs index d2b20f5ae38..0ca4d44a6cd 100644 --- a/crates/storage/db/src/implementation/mdbx/tx.rs +++ b/crates/storage/db/src/implementation/mdbx/tx.rs @@ -14,6 +14,7 @@ use reth_storage_errors::db::{DatabaseWriteError, DatabaseWriteOperation}; use reth_tracing::tracing::{debug, trace, warn}; use std::{ backtrace::Backtrace, + collections::HashMap, marker::PhantomData, sync::{ atomic::{AtomicBool, Ordering}, @@ -31,6 +32,9 @@ pub struct Tx { /// Libmdbx-sys transaction. pub inner: Transaction, + /// Cached MDBX DBIs for reuse. + dbis: Arc>, + /// Handler for metrics with its own [Drop] implementation for cases when the transaction isn't /// closed by [`Tx::commit`] or [`Tx::abort`], but we still need to report it in the metrics. /// @@ -39,17 +43,12 @@ pub struct Tx { } impl Tx { - /// Creates new `Tx` object with a `RO` or `RW` transaction. - #[inline] - pub const fn new(inner: Transaction) -> Self { - Self::new_inner(inner, None) - } - /// Creates new `Tx` object with a `RO` or `RW` transaction and optionally enables metrics. #[inline] #[track_caller] - pub(crate) fn new_with_metrics( + pub(crate) fn new( inner: Transaction, + dbis: Arc>, env_metrics: Option>, ) -> reth_libmdbx::Result { let metrics_handler = env_metrics @@ -60,12 +59,7 @@ impl Tx { Ok(handler) }) .transpose()?; - Ok(Self::new_inner(inner, metrics_handler)) - } - - #[inline] - const fn new_inner(inner: Transaction, metrics_handler: Option>) -> Self { - Self { inner, metrics_handler } + Ok(Self { inner, dbis, metrics_handler }) } /// Gets this transaction ID. @@ -75,10 +69,14 @@ impl Tx { /// Gets a table database handle if it exists, otherwise creates it. pub fn get_dbi(&self) -> Result { - self.inner - .open_db(Some(T::NAME)) - .map(|db| db.dbi()) - .map_err(|e| DatabaseError::Open(e.into())) + if let Some(dbi) = self.dbis.get(T::NAME) { + Ok(*dbi) + } else { + self.inner + .open_db(Some(T::NAME)) + .map(|db| db.dbi()) + .map_err(|e| DatabaseError::Open(e.into())) + } } /// Create db Cursor @@ -342,28 +340,64 @@ impl DbTx for Tx { } } +#[derive(Clone, Copy)] +enum PutKind { + /// Default kind that inserts a new key-value or overwrites an existed key. + Upsert, + /// Append the key-value to the end of the table -- fast path when the new + /// key is the highest so far, like the latest block number. + Append, +} + +impl PutKind { + const fn into_operation_and_flags(self) -> (Operation, DatabaseWriteOperation, WriteFlags) { + match self { + Self::Upsert => { + (Operation::PutUpsert, DatabaseWriteOperation::PutUpsert, WriteFlags::UPSERT) + } + Self::Append => { + (Operation::PutAppend, DatabaseWriteOperation::PutAppend, WriteFlags::APPEND) + } + } + } +} + +impl Tx { + /// The inner implementation mapping to `mdbx_put` that supports different + /// put kinds like upserting and appending. + fn put( + &self, + kind: PutKind, + key: T::Key, + value: T::Value, + ) -> Result<(), DatabaseError> { + let key = key.encode(); + let value = value.compress(); + let (operation, write_operation, flags) = kind.into_operation_and_flags(); + self.execute_with_operation_metric::(operation, Some(value.as_ref().len()), |tx| { + tx.put(self.get_dbi::()?, key.as_ref(), value, flags).map_err(|e| { + DatabaseWriteError { + info: e.into(), + operation: write_operation, + table_name: T::NAME, + key: key.into(), + } + .into() + }) + }) + } +} + impl DbTxMut for Tx { type CursorMut = Cursor; type DupCursorMut = Cursor; fn put(&self, key: T::Key, value: T::Value) -> Result<(), DatabaseError> { - let key = key.encode(); - let value = value.compress(); - self.execute_with_operation_metric::( - Operation::Put, - Some(value.as_ref().len()), - |tx| { - tx.put(self.get_dbi::()?, key.as_ref(), value, WriteFlags::UPSERT).map_err(|e| { - DatabaseWriteError { - info: e.into(), - operation: DatabaseWriteOperation::Put, - table_name: T::NAME, - key: key.into(), - } - .into() - }) - }, - ) + self.put::(PutKind::Upsert, key, value) + } + + fn append(&self, key: T::Key, value: T::Value) -> Result<(), DatabaseError> { + self.put::(PutKind::Append, key, value) } fn delete( diff --git a/crates/storage/db/src/lib.rs b/crates/storage/db/src/lib.rs index bf9f2354dd5..a6306723847 100644 --- a/crates/storage/db/src/lib.rs +++ b/crates/storage/db/src/lib.rs @@ -1,6 +1,6 @@ //! MDBX implementation for reth's database abstraction layer. //! -//! This crate is an implementation of [`reth-db-api`] for MDBX, as well as a few other common +//! This crate is an implementation of `reth-db-api` for MDBX, as well as a few other common //! database types. //! //! # Overview @@ -13,7 +13,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] mod implementation; pub mod lockfile; @@ -162,7 +162,7 @@ pub mod test_utils { /// Get a temporary directory path to use for the database pub fn tempdir_path() -> PathBuf { let builder = tempfile::Builder::new().prefix("reth-test-").rand_bytes(8).tempdir(); - builder.expect(ERROR_TEMPDIR).into_path() + builder.expect(ERROR_TEMPDIR).keep() } /// Create read/write database for testing diff --git a/crates/storage/db/src/lockfile.rs b/crates/storage/db/src/lockfile.rs index a1a9946b570..5e25d14ae3a 100644 --- a/crates/storage/db/src/lockfile.rs +++ b/crates/storage/db/src/lockfile.rs @@ -44,33 +44,50 @@ impl StorageLock { #[cfg(any(test, not(feature = "disable-lock")))] fn try_acquire_file_lock(path: &Path) -> Result { let file_path = path.join(LOCKFILE_NAME); - if let Some(process_lock) = ProcessUID::parse(&file_path)? { - if process_lock.pid != (process::id() as usize) && process_lock.is_active() { - reth_tracing::tracing::error!( - target: "reth::db::lockfile", - path = ?file_path, - pid = process_lock.pid, - start_time = process_lock.start_time, - "Storage lock already taken." - ); - return Err(StorageLockError::Taken(process_lock.pid)) - } + if let Some(process_lock) = ProcessUID::parse(&file_path)? && + process_lock.pid != (process::id() as usize) && + process_lock.is_active() + { + reth_tracing::tracing::error!( + target: "reth::db::lockfile", + path = ?file_path, + pid = process_lock.pid, + start_time = process_lock.start_time, + "Storage lock already taken." + ); + return Err(StorageLockError::Taken(process_lock.pid)) } Ok(Self(Arc::new(StorageLockInner::new(file_path)?))) } } -impl Drop for StorageLock { +impl Drop for StorageLockInner { fn drop(&mut self) { // The lockfile is not created in disable-lock mode, so we don't need to delete it. #[cfg(any(test, not(feature = "disable-lock")))] - if Arc::strong_count(&self.0) == 1 && self.0.file_path.exists() { - // TODO: should only happen during tests that the file does not exist: tempdir is - // getting dropped first. However, tempdir shouldn't be dropped - // before any of the storage providers. - if let Err(err) = reth_fs_util::remove_file(&self.0.file_path) { - reth_tracing::tracing::error!(%err, "Failed to delete lock file"); + { + let file_path = &self.file_path; + if file_path.exists() { + if let Ok(Some(process_uid)) = ProcessUID::parse(file_path) { + // Only remove if the lock file belongs to our process + if process_uid.pid == process::id() as usize { + if let Err(err) = reth_fs_util::remove_file(file_path) { + reth_tracing::tracing::error!(%err, "Failed to delete lock file"); + } + } else { + reth_tracing::tracing::warn!( + "Lock file belongs to different process (PID: {}), not removing", + process_uid.pid + ); + } + } else { + // If we can't parse the lock file, still try to remove it + // as it might be corrupted or from a previous run + if let Err(err) = reth_fs_util::remove_file(file_path) { + reth_tracing::tracing::error!(%err, "Failed to delete lock file"); + } + } } } } @@ -125,15 +142,15 @@ impl ProcessUID { /// Parses [`Self`] from a file. fn parse(path: &Path) -> Result, StorageLockError> { - if path.exists() { - if let Ok(contents) = reth_fs_util::read_to_string(path) { - let mut lines = contents.lines(); - if let (Some(Ok(pid)), Some(Ok(start_time))) = ( - lines.next().map(str::trim).map(str::parse), - lines.next().map(str::trim).map(str::parse), - ) { - return Ok(Some(Self { pid, start_time })); - } + if path.exists() && + let Ok(contents) = reth_fs_util::read_to_string(path) + { + let mut lines = contents.lines(); + if let (Some(Ok(pid)), Some(Ok(start_time))) = ( + lines.next().map(str::trim).map(str::parse), + lines.next().map(str::trim).map(str::parse), + ) { + return Ok(Some(Self { pid, start_time })); } } Ok(None) diff --git a/crates/storage/db/src/mdbx.rs b/crates/storage/db/src/mdbx.rs index 9042299afdc..fb0fd8501e3 100644 --- a/crates/storage/db/src/mdbx.rs +++ b/crates/storage/db/src/mdbx.rs @@ -41,8 +41,8 @@ pub fn init_db_for, TS: TableSet>( args: DatabaseArguments, ) -> eyre::Result { let client_version = args.client_version().clone(); - let db = create_db(path, args)?; - db.create_tables_for::()?; + let mut db = create_db(path, args)?; + db.create_and_track_tables_for::()?; db.record_client_version(client_version)?; Ok(db) } diff --git a/crates/storage/db/src/metrics.rs b/crates/storage/db/src/metrics.rs index 40790950969..444c9ce5707 100644 --- a/crates/storage/db/src/metrics.rs +++ b/crates/storage/db/src/metrics.rs @@ -197,8 +197,10 @@ impl TransactionOutcome { pub(crate) enum Operation { /// Database get operation. Get, - /// Database put operation. - Put, + /// Database put upsert operation. + PutUpsert, + /// Database put append operation. + PutAppend, /// Database delete operation. Delete, /// Database cursor upsert operation. @@ -220,7 +222,8 @@ impl Operation { pub(crate) const fn as_str(&self) -> &'static str { match self { Self::Get => "get", - Self::Put => "put", + Self::PutUpsert => "put-upsert", + Self::PutAppend => "put-append", Self::Delete => "delete", Self::CursorUpsert => "cursor-upsert", Self::CursorInsert => "cursor-insert", diff --git a/crates/storage/db/src/static_file/masks.rs b/crates/storage/db/src/static_file/masks.rs index 6dbc384fab8..17833e7ee29 100644 --- a/crates/storage/db/src/static_file/masks.rs +++ b/crates/storage/db/src/static_file/masks.rs @@ -1,13 +1,10 @@ use crate::{ add_static_file_mask, static_file::mask::{ColumnSelectorOne, ColumnSelectorTwo}, - BlockBodyIndices, HeaderTerminalDifficulties, + HeaderTerminalDifficulties, }; use alloy_primitives::BlockHash; -use reth_db_api::{ - models::{StaticFileBlockWithdrawals, StoredBlockOmmers}, - table::Table, -}; +use reth_db_api::table::Table; // HEADER MASKS add_static_file_mask! { @@ -45,17 +42,3 @@ add_static_file_mask! { #[doc = "Mask for selecting a single transaction from Transactions static file segment"] TransactionMask, T, 0b1 } - -// BLOCK_META MASKS -add_static_file_mask! { - #[doc = "Mask for a `StoredBlockBodyIndices` from BlockMeta static file segment"] - BodyIndicesMask, ::Value, 0b001 -} -add_static_file_mask! { - #[doc = "Mask for a `StoredBlockOmmers` from BlockMeta static file segment"] - OmmersMask, StoredBlockOmmers, 0b010 -} -add_static_file_mask! { - #[doc = "Mask for a `StaticFileBlockWithdrawals` from BlockMeta static file segment"] - WithdrawalsMask, StaticFileBlockWithdrawals, 0b100 -} diff --git a/crates/storage/db/src/static_file/mod.rs b/crates/storage/db/src/static_file/mod.rs index cbcf87d8939..f2c9ce45fbc 100644 --- a/crates/storage/db/src/static_file/mod.rs +++ b/crates/storage/db/src/static_file/mod.rs @@ -33,25 +33,22 @@ pub fn iter_static_files(path: &Path) -> Result::load(&entry.path())?; + { + let jar = NippyJar::::load(&entry.path())?; - let (block_range, tx_range) = ( - jar.user_header().block_range().copied(), - jar.user_header().tx_range().copied(), - ); + let (block_range, tx_range) = + (jar.user_header().block_range().copied(), jar.user_header().tx_range().copied()); - if let Some(block_range) = block_range { - match static_files.entry(segment) { - Entry::Occupied(mut entry) => { - entry.get_mut().push((block_range, tx_range)); - } - Entry::Vacant(entry) => { - entry.insert(vec![(block_range, tx_range)]); - } + if let Some(block_range) = block_range { + match static_files.entry(segment) { + Entry::Occupied(mut entry) => { + entry.get_mut().push((block_range, tx_range)); + } + Entry::Vacant(entry) => { + entry.insert(vec![(block_range, tx_range)]); } } } diff --git a/crates/storage/db/src/utils.rs b/crates/storage/db/src/utils.rs index cf6a0341ef7..713fb37d4f1 100644 --- a/crates/storage/db/src/utils.rs +++ b/crates/storage/db/src/utils.rs @@ -25,8 +25,9 @@ pub fn is_database_empty>(path: P) -> bool { true } else if path.is_file() { false - } else if let Ok(dir) = path.read_dir() { - dir.count() == 0 + } else if let Ok(mut dir) = path.read_dir() { + // Check if directory has any entries without counting all of them + dir.next().is_none() } else { true } diff --git a/crates/storage/errors/src/db.rs b/crates/storage/errors/src/db.rs index 63f59cd6a69..b12ad28898f 100644 --- a/crates/storage/errors/src/db.rs +++ b/crates/storage/errors/src/db.rs @@ -106,8 +106,10 @@ pub enum DatabaseWriteOperation { CursorInsert, /// Append duplicate cursor. CursorAppendDup, - /// Put. - Put, + /// Put upsert. + PutUpsert, + /// Put append. + PutAppend, } /// Database log level. diff --git a/crates/storage/errors/src/lib.rs b/crates/storage/errors/src/lib.rs index aa8254ee174..eca6cd47a45 100644 --- a/crates/storage/errors/src/lib.rs +++ b/crates/storage/errors/src/lib.rs @@ -6,7 +6,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(feature = "std"), no_std)] extern crate alloc; @@ -21,8 +21,5 @@ pub mod lockfile; pub mod provider; pub use provider::{ProviderError, ProviderResult}; -/// Writer error -pub mod writer; - /// Any error pub mod any; diff --git a/crates/storage/errors/src/provider.rs b/crates/storage/errors/src/provider.rs index d72032e322b..ed5230c18fb 100644 --- a/crates/storage/errors/src/provider.rs +++ b/crates/storage/errors/src/provider.rs @@ -1,4 +1,4 @@ -use crate::{any::AnyError, db::DatabaseError, writer::UnifiedStorageWriterError}; +use crate::{any::AnyError, db::DatabaseError}; use alloc::{boxed::Box, string::String}; use alloy_eips::{BlockHashOrNumber, HashOrNumber}; use alloy_primitives::{Address, BlockHash, BlockNumber, TxNumber, B256}; @@ -58,9 +58,6 @@ pub enum ProviderError { /// The account address. address: Address, }, - /// The total difficulty for a block is missing. - #[error("total difficulty not found for block #{_0}")] - TotalDifficultyNotFound(BlockNumber), /// When required header related data was not found but was required. #[error("no header found for {_0:?}")] HeaderNotFound(BlockHashOrNumber), @@ -128,12 +125,20 @@ pub enum ProviderError { /// Consistent view error. #[error("failed to initialize consistent view: {_0}")] ConsistentView(Box), - /// Storage writer error. - #[error(transparent)] - UnifiedStorageWriterError(#[from] UnifiedStorageWriterError), /// Received invalid output from configured storage implementation. #[error("received invalid output from storage")] InvalidStorageOutput, + /// Missing trie updates. + #[error("missing trie updates for block {0}")] + MissingTrieUpdates(B256), + /// Insufficient changesets to revert to the requested block. + #[error("insufficient changesets to revert to block #{requested}. Available changeset range: {available:?}")] + InsufficientChangesets { + /// The block number requested for reversion + requested: BlockNumber, + /// The available range of blocks with changesets + available: core::ops::RangeInclusive, + }, /// Any other error type wrapped into a cloneable [`AnyError`]. #[error(transparent)] Other(#[from] AnyError), diff --git a/crates/storage/errors/src/writer.rs b/crates/storage/errors/src/writer.rs deleted file mode 100644 index 3e060d7005d..00000000000 --- a/crates/storage/errors/src/writer.rs +++ /dev/null @@ -1,25 +0,0 @@ -use crate::db::DatabaseError; -use reth_static_file_types::StaticFileSegment; - -/// `UnifiedStorageWriter` related errors -/// `StorageWriter` related errors -#[derive(Clone, Debug, derive_more::Display, PartialEq, Eq, derive_more::Error)] -pub enum UnifiedStorageWriterError { - /// Database writer is missing - #[display("Database writer is missing")] - MissingDatabaseWriter, - /// Static file writer is missing - #[display("Static file writer is missing")] - MissingStaticFileWriter, - /// Static file writer is of wrong segment - #[display("Static file writer is of wrong segment: got {_0}, expected {_1}")] - IncorrectStaticFileWriter(StaticFileSegment, StaticFileSegment), - /// Database-related errors. - Database(DatabaseError), -} - -impl From for UnifiedStorageWriterError { - fn from(error: DatabaseError) -> Self { - Self::Database(error) - } -} diff --git a/crates/storage/libmdbx-rs/Cargo.toml b/crates/storage/libmdbx-rs/Cargo.toml index 6b7956f4675..8fa931a3495 100644 --- a/crates/storage/libmdbx-rs/Cargo.toml +++ b/crates/storage/libmdbx-rs/Cargo.toml @@ -17,7 +17,6 @@ reth-mdbx-sys.workspace = true bitflags.workspace = true byteorder.workspace = true derive_more.workspace = true -indexmap.workspace = true parking_lot.workspace = true smallvec.workspace = true thiserror.workspace = true diff --git a/crates/storage/libmdbx-rs/README.md b/crates/storage/libmdbx-rs/README.md index 0ead0242b8f..df115ee69a0 100644 --- a/crates/storage/libmdbx-rs/README.md +++ b/crates/storage/libmdbx-rs/README.md @@ -12,8 +12,8 @@ To update the libmdbx version you must clone it and copy the `dist/` folder in ` Make sure to follow the [building steps](https://libmdbx.dqdkfa.ru/usage.html#getting). ```bash -# clone libmmdbx to a repository outside at specific tag -git clone https://gitflic.ru/project/erthink/libmdbx.git ../libmdbx --branch v0.7.0 +# clone libmdbx to a repository outside at specific tag +git clone https://github.com/erthink/libmdbx.git ../libmdbx --branch v0.7.0 make -C ../libmdbx dist # copy the `libmdbx/dist/` folder just created into `mdbx-sys/libmdbx` diff --git a/crates/storage/libmdbx-rs/benches/transaction.rs b/crates/storage/libmdbx-rs/benches/transaction.rs index 35c403606df..311e5b5c184 100644 --- a/crates/storage/libmdbx-rs/benches/transaction.rs +++ b/crates/storage/libmdbx-rs/benches/transaction.rs @@ -66,8 +66,6 @@ fn bench_put_rand(c: &mut Criterion) { let txn = env.begin_ro_txn().unwrap(); let db = txn.open_db(None).unwrap(); - txn.prime_for_permaopen(db); - let db = txn.commit_and_rebind_open_dbs().unwrap().2.remove(0); let mut items: Vec<(String, String)> = (0..n).map(|n| (get_key(n), get_data(n))).collect(); items.shuffle(&mut StdRng::from_seed(Default::default())); diff --git a/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/CMakeLists.txt b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/CMakeLists.txt index 0f10b8775b3..7d0e3f434cf 100644 --- a/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/CMakeLists.txt +++ b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/CMakeLists.txt @@ -1,38 +1,21 @@ -## -## Copyright 2020-2024 Leonid Yuriev -## and other libmdbx authors: please see AUTHORS file. -## All rights reserved. -## -## Redistribution and use in source and binary forms, with or without -## modification, are permitted only as authorized by the OpenLDAP -## Public License. -## -## A copy of this license is available in the file LICENSE in the -## top-level directory of the distribution or, alternatively, at -## . -## - -## -## libmdbx = { Revised and extended descendant of Symas LMDB. } -## Please see README.md at https://gitflic.ru/project/erthink/libmdbx -## -## Libmdbx is superior to LMDB in terms of features and reliability, -## not inferior in performance. libmdbx works on Linux, FreeBSD, MacOS X -## and other systems compliant with POSIX.1-2008, but also support Windows -## as a complementary platform. -## -## The next version is under active non-public development and will be -## released as MithrilDB and libmithrildb for libraries & packages. -## Admittedly mythical Mithril is resembling silver but being stronger and -## lighter than steel. Therefore MithrilDB is rightly relevant name. -## -## MithrilDB will be radically different from libmdbx by the new database -## format and API based on C++17, as well as the Apache 2.0 License. -## The goal of this revolution is to provide a clearer and robust API, -## add more features and new valuable properties of database. -## -## The Future will (be) Positive. Всё будет хорошо. -## +# Copyright (c) 2020-2025 Леонид Юрьев aka Leonid Yuriev ############################################### +# SPDX-License-Identifier: Apache-2.0 +# +# Donations are welcome to ETH `0xD104d8f8B2dC312aaD74899F83EBf3EEBDC1EA3A`. Всё будет хорошо! + +# libmdbx = { Revised and extended descendant of Symas LMDB. } Please see README.md at +# https://gitflic.ru/project/erthink/libmdbx +# +# Libmdbx is superior to LMDB in terms of features and reliability, not inferior in performance. libmdbx works on Linux, +# FreeBSD, MacOS X and other systems compliant with POSIX.1-2008, but also support Windows as a complementary platform. +# +# The next version is under active non-public development and will be released as MithrilDB and libmithrildb for +# libraries & packages. Admittedly mythical Mithril is resembling silver but being stronger and lighter than steel. +# Therefore MithrilDB is rightly relevant name. +# +# MithrilDB will be radically different from libmdbx by the new database format and API based on C++17, as well as the +# Apache 2.0 License. The goal of this revolution is to provide a clearer and robust API, add more features and new +# valuable properties of database. if(CMAKE_VERSION VERSION_LESS 3.8.2) cmake_minimum_required(VERSION 3.0.2) @@ -68,41 +51,156 @@ else() set(CMAKE_INTERPROCEDURAL_OPTIMIZATION_AVAILABLE FALSE) endif() -if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.git" AND - EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/CMakeLists.txt" AND - EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/core.c" AND - EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/alloy.c" AND - EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/config.h.in" AND - EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/version.c.in" AND - EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/man1" AND - EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/mdbx_chk.c" AND - EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/mdbx.c++") +cmake_policy(SET CMP0054 NEW) + +if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.git" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/COPYRIGHT" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/LICENSE" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/NOTICE" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/README.md" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/mdbx.h" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/mdbx.h++" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/alloy.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/api-cold.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/api-copy.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/api-cursor.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/api-dbi.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/api-env.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/api-extra.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/api-key-transform.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/api-misc.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/api-opts.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/api-range-estimate.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/api-txn-data.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/api-txn.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/atomics-ops.h" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/atomics-types.h" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/audit.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/chk.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/cogs.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/cogs.h" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/coherency.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/config.h.in" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/cursor.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/cursor.h" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/dbi.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/dbi.h" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/debug_begin.h" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/debug_end.h" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/dpl.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/dpl.h" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/dxb.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/env.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/essentials.h" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/gc-get.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/gc-put.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/gc.h" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/global.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/internals.h" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/layout-dxb.h" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/layout-lck.h" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/lck-posix.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/lck-windows.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/lck.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/lck.h" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/logging_and_debug.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/logging_and_debug.h" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/man1/mdbx_chk.1" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/man1/mdbx_copy.1" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/man1/mdbx_drop.1" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/man1/mdbx_dump.1" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/man1/mdbx_load.1" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/man1/mdbx_stat.1" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/mdbx.c++" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/meta.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/meta.h" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/mvcc-readers.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/node.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/node.h" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/ntdll.def" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/options.h" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/osal.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/osal.h" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/page-get.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/page-iov.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/page-iov.h" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/page-ops.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/page-ops.h" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/tree-search.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/pnl.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/pnl.h" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/preface.h" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/proto.h" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/refund.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/sort.h" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/spill.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/spill.h" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/table.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/tls.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/tls.h" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/tools/chk.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/tools/copy.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/tools/drop.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/tools/dump.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/tools/load.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/tools/stat.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/tools/wingetopt.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/tools/wingetopt.h" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/tree-ops.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/txl.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/txl.h" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/txn.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/unaligned.h" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/utils.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/utils.h" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/version.c.in" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/walk.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/walk.h" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/windows-import.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/windows-import.h" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/CMakeLists.txt") set(MDBX_AMALGAMATED_SOURCE FALSE) find_program(GIT git) if(NOT GIT) message(SEND_ERROR "Git command-line tool not found") endif() set(MDBX_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/src") -elseif(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/VERSION.txt" AND - EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/mdbx.c" AND - EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/mdbx.c++" AND - EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/config.h.in" AND - EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/man1" AND - EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/mdbx_chk.c") +elseif( + EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/VERSION.json" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/LICENSE" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/NOTICE" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/mdbx.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/mdbx.c++" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/mdbx.h" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/mdbx.h++" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/mdbx_chk.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/mdbx_copy.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/mdbx_dump.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/mdbx_load.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/mdbx_stat.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/mdbx_drop.c" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/ntdll.def" + AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/config.h.in") set(MDBX_AMALGAMATED_SOURCE TRUE) set(MDBX_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}") else() - message(FATAL_ERROR "\n" - "Please don't use tarballs nor zips which are automatically provided by Github! " - "These archives do not contain version information and thus are unfit to build libmdbx. " - "You can vote for ability of disabling auto-creation such unsuitable archives at https://github.community/t/disable-tarball\n" - "Instead of above, just clone the git repository, either download a tarball or zip with the properly amalgamated source core. " - "For embedding libmdbx use a git-submodule or the amalgamated source code.\n" - "Please, avoid using any other techniques.") + message( + FATAL_ERROR + "\nThe set of libmdbx source code files is incomplete! " + "Instead just follow the https://libmdbx.dqdkfa.ru/usage.html " "PLEASE, AVOID USING ANY OTHER TECHNIQUES.") endif() +# Provide version +include(cmake/utils.cmake) +set(MDBX_BUILD_METADATA + "${MDBX_BUILD_METADATA}" + CACHE STRING "An extra/custom information provided during libmdbx build") +semver_provide(MDBX "${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_CURRENT_BINARY_DIR}" "${MDBX_BUILD_METADATA}" FALSE) +message(STATUS "libmdbx version is ${MDBX_VERSION}") + if(DEFINED PROJECT_NAME) - option(MDBX_FORCE_BUILD_AS_MAIN_PROJECT "Force libmdbx to full control build options even it added as a subdirectory to your project." OFF) + option(MDBX_FORCE_BUILD_AS_MAIN_PROJECT + "Force libmdbx to full control build options even it added as a subdirectory to your project." OFF) endif() if(DEFINED PROJECT_NAME AND NOT MDBX_FORCE_BUILD_AS_MAIN_PROJECT) @@ -115,6 +213,20 @@ if(DEFINED PROJECT_NAME AND NOT MDBX_FORCE_BUILD_AS_MAIN_PROJECT) else() set(SUBPROJECT OFF) set(NOT_SUBPROJECT ON) + + # Setup Apple stuff which should be set prior to the first project() or enable_language() + if(APPLE) + # Enable universal binaries for macOS (target arm64 and x86_64) + if(NOT DEFINED CMAKE_OSX_ARCHITECTURES) + set(CMAKE_OSX_ARCHITECTURES "arm64;x86_64") + endif() + + # Set the minimum macOS deployment target if not already defined + if(NOT DEFINED CMAKE_OSX_DEPLOYMENT_TARGET) + set(CMAKE_OSX_DEPLOYMENT_TARGET "13.0") + endif() + endif() + project(libmdbx C) if(NOT MDBX_AMALGAMATED_SOURCE AND NOT DEFINED BUILD_TESTING) set(BUILD_TESTING ON) @@ -130,10 +242,11 @@ elseif(DEFINED MDBX_ENABLE_TESTS AND MDBX_ENABLE_TESTS) endif() # Try to find a C++ compiler unless sure that this is unnecessary. -if (NOT CMAKE_CXX_COMPILER_LOADED) +if(NOT CMAKE_CXX_COMPILER_LOADED) include(CheckLanguage) - if(NOT DEFINED MDBX_BUILD_CXX OR MDBX_BUILD_CXX - OR (NOT MDBX_AMALGAMATED_SOURCE AND (NOT DEFINED MDBX_ENABLE_TESTS OR MDBX_ENABLE_TESTS))) + if(NOT DEFINED MDBX_BUILD_CXX + OR MDBX_BUILD_CXX + OR (NOT MDBX_AMALGAMATED_SOURCE AND (NOT DEFINED MDBX_ENABLE_TESTS OR MDBX_ENABLE_TESTS))) check_language(CXX) if(CMAKE_CXX_COMPILER) enable_language(CXX) @@ -145,9 +258,9 @@ endif() # Set default build type to Release. This is to ease a User's life. if(NOT CMAKE_BUILD_TYPE) - set(CMAKE_BUILD_TYPE Release CACHE STRING - "Choose the type of build, options are: Debug Release RelWithDebInfo MinSizeRel." - FORCE) + set(CMAKE_BUILD_TYPE + Release + CACHE STRING "Choose the type of build, options are: Debug Release RelWithDebInfo MinSizeRel." FORCE) endif() string(TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UPPERCASE) @@ -189,8 +302,9 @@ include(FindPackageMessage) include(GNUInstallDirs) if(CMAKE_C_COMPILER_ID STREQUAL "MSVC" AND MSVC_VERSION LESS 1900) - message(SEND_ERROR "MSVC compiler ${MSVC_VERSION} is too old for building MDBX." - " At least 'Microsoft Visual Studio 2015' is required.") + message( + SEND_ERROR "MSVC compiler ${MSVC_VERSION} is too old for building MDBX." + " At least \"Microsoft C/C++ Compiler\" version 19.0.24234.1 (Visual Studio 2015 Update 3) is required.") endif() if(NOT DEFINED THREADS_PREFER_PTHREAD_FLAG) @@ -198,14 +312,15 @@ if(NOT DEFINED THREADS_PREFER_PTHREAD_FLAG) endif() find_package(Threads REQUIRED) -include(cmake/utils.cmake) include(cmake/compiler.cmake) include(cmake/profile.cmake) # Workaround for `-pthread` toolchain/cmake bug -if(NOT APPLE AND NOT MSVC - AND CMAKE_USE_PTHREADS_INIT AND NOT CMAKE_THREAD_LIBS_INIT - AND (CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_CLANG)) +if(NOT APPLE + AND NOT MSVC + AND CMAKE_USE_PTHREADS_INIT + AND NOT CMAKE_THREAD_LIBS_INIT + AND (CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_CLANG)) check_compiler_flag("-pthread" CC_HAS_PTHREAD) if(CC_HAS_PTHREAD AND NOT CMAKE_EXE_LINKER_FLAGS MATCHES "-pthread") message(STATUS "Force add -pthread for linker flags to avoid troubles") @@ -215,12 +330,12 @@ if(NOT APPLE AND NOT MSVC endif() endif() -CHECK_FUNCTION_EXISTS(pow NOT_NEED_LIBM) +check_function_exists(pow NOT_NEED_LIBM) if(NOT_NEED_LIBM) set(LIB_MATH "") else() set(CMAKE_REQUIRED_LIBRARIES m) - CHECK_FUNCTION_EXISTS(pow HAVE_LIBM) + check_function_exists(pow HAVE_LIBM) if(HAVE_LIBM) set(LIB_MATH m) else() @@ -239,46 +354,79 @@ if(SUBPROJECT) else() option(BUILD_SHARED_LIBS "Build shared libraries (DLLs)" ON) option(CMAKE_POSITION_INDEPENDENT_CODE "Generate position independent (PIC)" ON) - if (CC_HAS_ARCH_NATIVE) + if(CC_HAS_ARCH_NATIVE) option(BUILD_FOR_NATIVE_CPU "Generate code for the compiling machine CPU" OFF) endif() if(CMAKE_INTERPROCEDURAL_OPTIMIZATION_AVAILABLE - OR GCC_LTO_AVAILABLE OR MSVC_LTO_AVAILABLE OR CLANG_LTO_AVAILABLE) - if((CMAKE_CONFIGURATION_TYPES OR NOT CMAKE_BUILD_TYPE_UPPERCASE STREQUAL "DEBUG") AND - ((MSVC_LTO_AVAILABLE AND NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 19) OR - (GCC_LTO_AVAILABLE AND NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 7) OR - (CLANG_LTO_AVAILABLE AND NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 5))) + OR GCC_LTO_AVAILABLE + OR MSVC_LTO_AVAILABLE + OR CLANG_LTO_AVAILABLE) + if((CMAKE_CONFIGURATION_TYPES OR NOT CMAKE_BUILD_TYPE_UPPERCASE STREQUAL "DEBUG") + AND ((MSVC_LTO_AVAILABLE AND NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 19) + OR (GCC_LTO_AVAILABLE AND NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 7) + OR (CLANG_LTO_AVAILABLE AND NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 5))) set(INTERPROCEDURAL_OPTIMIZATION_DEFAULT ON) else() set(INTERPROCEDURAL_OPTIMIZATION_DEFAULT OFF) endif() - option(INTERPROCEDURAL_OPTIMIZATION "Enable interprocedural/LTO optimization." ${INTERPROCEDURAL_OPTIMIZATION_DEFAULT}) + option(INTERPROCEDURAL_OPTIMIZATION "Enable interprocedural/LTO optimization." + ${INTERPROCEDURAL_OPTIMIZATION_DEFAULT}) endif() if(INTERPROCEDURAL_OPTIMIZATION) if(GCC_LTO_AVAILABLE) set(LTO_ENABLED TRUE) - set(CMAKE_AR ${CMAKE_GCC_AR} CACHE PATH "Path to ar program with LTO-plugin" FORCE) - set(CMAKE_C_COMPILER_AR ${CMAKE_AR} CACHE PATH "Path to ar program with LTO-plugin" FORCE) - set(CMAKE_CXX_COMPILER_AR ${CMAKE_AR} CACHE PATH "Path to ar program with LTO-plugin" FORCE) - set(CMAKE_NM ${CMAKE_GCC_NM} CACHE PATH "Path to nm program with LTO-plugin" FORCE) - set(CMAKE_RANLIB ${CMAKE_GCC_RANLIB} CACHE PATH "Path to ranlib program with LTO-plugin" FORCE) - set(CMAKE_C_COMPILER_RANLIB ${CMAKE_RANLIB} CACHE PATH "Path to ranlib program with LTO-plugin" FORCE) - set(CMAKE_CXX_COMPILER_RANLIB ${CMAKE_RANLIB} CACHE PATH "Path to ranlib program with LTO-plugin" FORCE) + set(CMAKE_AR + ${CMAKE_GCC_AR} + CACHE PATH "Path to ar program with LTO-plugin" FORCE) + set(CMAKE_C_COMPILER_AR + ${CMAKE_AR} + CACHE PATH "Path to ar program with LTO-plugin" FORCE) + set(CMAKE_CXX_COMPILER_AR + ${CMAKE_AR} + CACHE PATH "Path to ar program with LTO-plugin" FORCE) + set(CMAKE_NM + ${CMAKE_GCC_NM} + CACHE PATH "Path to nm program with LTO-plugin" FORCE) + set(CMAKE_RANLIB + ${CMAKE_GCC_RANLIB} + CACHE PATH "Path to ranlib program with LTO-plugin" FORCE) + set(CMAKE_C_COMPILER_RANLIB + ${CMAKE_RANLIB} + CACHE PATH "Path to ranlib program with LTO-plugin" FORCE) + set(CMAKE_CXX_COMPILER_RANLIB + ${CMAKE_RANLIB} + CACHE PATH "Path to ranlib program with LTO-plugin" FORCE) message(STATUS "MDBX indulge Link-Time Optimization by GCC") elseif(CLANG_LTO_AVAILABLE) set(LTO_ENABLED TRUE) if(CMAKE_CLANG_LD) - set(CMAKE_LINKER ${CMAKE_CLANG_LD} CACHE PATH "Path to lld or ld program with LTO-plugin" FORCE) + set(CMAKE_LINKER + ${CMAKE_CLANG_LD} + CACHE PATH "Path to lld or ld program with LTO-plugin" FORCE) endif() - set(CMAKE_AR ${CMAKE_CLANG_AR} CACHE PATH "Path to ar program with LTO-plugin" FORCE) - set(CMAKE_C_COMPILER_AR ${CMAKE_AR} CACHE PATH "Path to ar program with LTO-plugin" FORCE) - set(CMAKE_CXX_COMPILER_AR ${CMAKE_AR} CACHE PATH "Path to ar program with LTO-plugin" FORCE) - set(CMAKE_NM ${CMAKE_CLANG_NM} CACHE PATH "Path to nm program with LTO-plugin" FORCE) - set(CMAKE_RANLIB ${CMAKE_CLANG_RANLIB} CACHE PATH "Path to ranlib program with LTO-plugin" FORCE) - set(CMAKE_C_COMPILER_RANLIB ${CMAKE_RANLIB} CACHE PATH "Path to ranlib program with LTO-plugin" FORCE) - set(CMAKE_CXX_COMPILER_RANLIB ${CMAKE_RANLIB} CACHE PATH "Path to ranlib program with LTO-plugin" FORCE) + set(CMAKE_AR + ${CMAKE_CLANG_AR} + CACHE PATH "Path to ar program with LTO-plugin" FORCE) + set(CMAKE_C_COMPILER_AR + ${CMAKE_AR} + CACHE PATH "Path to ar program with LTO-plugin" FORCE) + set(CMAKE_CXX_COMPILER_AR + ${CMAKE_AR} + CACHE PATH "Path to ar program with LTO-plugin" FORCE) + set(CMAKE_NM + ${CMAKE_CLANG_NM} + CACHE PATH "Path to nm program with LTO-plugin" FORCE) + set(CMAKE_RANLIB + ${CMAKE_CLANG_RANLIB} + CACHE PATH "Path to ranlib program with LTO-plugin" FORCE) + set(CMAKE_C_COMPILER_RANLIB + ${CMAKE_RANLIB} + CACHE PATH "Path to ranlib program with LTO-plugin" FORCE) + set(CMAKE_CXX_COMPILER_RANLIB + ${CMAKE_RANLIB} + CACHE PATH "Path to ranlib program with LTO-plugin" FORCE) message(STATUS "MDBX indulge Link-Time Optimization by CLANG") elseif(MSVC_LTO_AVAILABLE) set(LTO_ENABLED TRUE) @@ -298,40 +446,41 @@ else() if(NOT MDBX_AMALGAMATED_SOURCE) find_program(VALGRIND valgrind) if(VALGRIND) - # LY: cmake is ugly and nasty. - # - therefore memcheck-options should be defined before including ctest; - # - otherwise ctest may ignore it. + # (LY) cmake is ugly and nasty. Therefore memcheck-options should be defined before including ctest. Otherwise + # ctest may ignore it. set(MEMORYCHECK_SUPPRESSIONS_FILE - "${CMAKE_CURRENT_SOURCE_DIR}/test/valgrind_suppress.txt" - CACHE FILEPATH "Suppressions file for Valgrind" FORCE) + "${CMAKE_CURRENT_SOURCE_DIR}/test/valgrind_suppress.txt" + CACHE FILEPATH "Suppressions file for Valgrind" FORCE) set(MEMORYCHECK_COMMAND_OPTIONS - "--trace-children=yes --leak-check=full --track-origins=yes --error-exitcode=42 --error-markers=@ --errors-for-leak-kinds=definite --fair-sched=yes --suppressions=${MEMORYCHECK_SUPPRESSIONS_FILE}" - CACHE STRING "Valgrind options" FORCE) - set(VALGRIND_COMMAND_OPTIONS "${MEMORYCHECK_COMMAND_OPTIONS}" CACHE STRING "Valgrind options" FORCE) + "--trace-children=yes --leak-check=full --track-origins=yes --track-origins=yes --error-exitcode=42 --error-markers=@ --errors-for-leak-kinds=definite --fair-sched=yes --suppressions=${MEMORYCHECK_SUPPRESSIONS_FILE}" + CACHE STRING "Valgrind options" FORCE) + set(VALGRIND_COMMAND_OPTIONS + "${MEMORYCHECK_COMMAND_OPTIONS}" + CACHE STRING "Valgrind options" FORCE) endif() # Enable 'make tags' target. find_program(CTAGS ctags) if(CTAGS) - add_custom_target(tags COMMAND ${CTAGS} -R -f tags + add_custom_target( + tags + COMMAND ${CTAGS} -R -f tags WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) add_custom_target(ctags DEPENDS tags) endif(CTAGS) if(UNIX) - find_program(CLANG_FORMAT - NAMES clang-format-13 clang-format) + find_program(CLANG_FORMAT NAMES clang-format-13 clang-format) if(CLANG_FORMAT) execute_process(COMMAND ${CLANG_FORMAT} "--version" OUTPUT_VARIABLE clang_format_version_info) string(REGEX MATCH "version ([0-9]+)\\.([0-9]+)\\.([0-9]+)(.*)?" clang_format_version_info CLANG_FORMAT_VERSION) if(clang_format_version_info AND NOT CLANG_FORMAT_VERSION VERSION_LESS 13.0) # Enable 'make reformat' target. - add_custom_target(reformat + add_custom_target( + reformat VERBATIM - COMMAND - git ls-files | - grep -E \\.\(c|cxx|cc|cpp|h|hxx|hpp\)\(\\.in\)?\$ | - xargs ${CLANG_FORMAT} -i --style=file + COMMAND git ls-files | grep -E \\.\(c|cxx|cc|cpp|h|hxx|hpp\)\(\\.in\)?\$ | xargs ${CLANG_FORMAT} -i + --style=file WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) endif() endif() @@ -339,12 +488,16 @@ else() if(NOT "${PROJECT_BINARY_DIR}" STREQUAL "${PROJECT_SOURCE_DIR}") add_custom_target(distclean) - add_custom_command(TARGET distclean + add_custom_command( + TARGET distclean + POST_BUILD COMMAND ${CMAKE_COMMAND} -E remove_directory "${PROJECT_BINARY_DIR}" COMMENT "Removing the build directory and its content") elseif(IS_DIRECTORY .git AND GIT) add_custom_target(distclean) - add_custom_command(TARGET distclean + add_custom_command( + TARGET distclean + POST_BUILD WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} COMMAND ${GIT} submodule foreach --recursive git clean -f -X -d COMMAND ${GIT} clean -f -X -d @@ -355,12 +508,12 @@ else() set(MDBX_MANAGE_BUILD_FLAGS_DEFAULT ON) endif(SUBPROJECT) -option(MDBX_MANAGE_BUILD_FLAGS "Allow libmdbx to configure/manage/override its own build flags" ${MDBX_MANAGE_BUILD_FLAGS_DEFAULT}) +option(MDBX_MANAGE_BUILD_FLAGS "Allow libmdbx to configure/manage/override its own build flags" + ${MDBX_MANAGE_BUILD_FLAGS_DEFAULT}) if(MDBX_MANAGE_BUILD_FLAGS) setup_compile_flags() endif() -list(FIND CMAKE_C_COMPILE_FEATURES c_std_11 HAS_C11) list(FIND CMAKE_CXX_COMPILE_FEATURES cxx_std_11 HAS_CXX11) list(FIND CMAKE_CXX_COMPILE_FEATURES cxx_std_14 HAS_CXX14) list(FIND CMAKE_CXX_COMPILE_FEATURES cxx_std_17 HAS_CXX17) @@ -372,14 +525,11 @@ if(NOT DEFINED MDBX_CXX_STANDARD) endif() if(DEFINED CMAKE_CXX_STANDARD) set(MDBX_CXX_STANDARD ${CMAKE_CXX_STANDARD}) - elseif(NOT HAS_CXX23 LESS 0 - AND NOT (CMAKE_COMPILER_IS_CLANG AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 12)) + elseif(NOT HAS_CXX23 LESS 0 AND NOT (CMAKE_COMPILER_IS_CLANG AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 12)) set(MDBX_CXX_STANDARD 23) - elseif(NOT HAS_CXX20 LESS 0 - AND NOT (CMAKE_COMPILER_IS_CLANG AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 10)) + elseif(NOT HAS_CXX20 LESS 0 AND NOT (CMAKE_COMPILER_IS_CLANG AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 10)) set(MDBX_CXX_STANDARD 20) - elseif(NOT HAS_CXX17 LESS 0 - AND NOT (CMAKE_COMPILER_IS_CLANG AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5)) + elseif(NOT HAS_CXX17 LESS 0 AND NOT (CMAKE_COMPILER_IS_CLANG AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5)) set(MDBX_CXX_STANDARD 17) elseif(NOT HAS_CXX14 LESS 0) set(MDBX_CXX_STANDARD 14) @@ -391,21 +541,36 @@ if(NOT DEFINED MDBX_CXX_STANDARD) set(MDBX_CXX_STANDARD 98) endif() endif() + +list(FIND CMAKE_C_COMPILE_FEATURES c_std_11 HAS_C11) +list(FIND CMAKE_C_COMPILE_FEATURES c_std_23 HAS_C23) if(NOT DEFINED MDBX_C_STANDARD) - # MSVC >= 19.28 (Microsoft Visual Studio 16.8) is mad! - # It unable process Windows SDK headers in the C11 mode! - if(MSVC AND MSVC_VERSION GREATER 1927 AND NOT MSVC_VERSION GREATER 1929) + if(DEFINED ENV{CMAKE_C_STANDARD}) + set(CMAKE_C_STANDARD $ENV{CMAKE_C_STANDARD}) + endif() + if(DEFINED CMAKE_C_STANDARD) + set(MDBX_C_STANDARD ${CMAKE_C_STANDARD}) + elseif( + MSVC + # MSVC >= 19.28 (Microsoft Visual Studio 16.8) is mad! It unable process Windows SDK headers in the C11 mode! + AND MSVC_VERSION GREATER 1927 + AND NOT MSVC_VERSION GREATER 1929) set(MDBX_C_STANDARD 99) set(C_FALLBACK_11 OFF) set(C_FALLBACK_GNU11 OFF) - elseif(HAS_C11 LESS 0 AND NOT C_FALLBACK_GNU11 AND NOT C_FALLBACK_11) + elseif(NOT HAS_C23 LESS 0) + set(MDBX_C_STANDARD 23) + elseif( + HAS_C11 LESS 0 + AND NOT C_FALLBACK_GNU11 + AND NOT C_FALLBACK_11) set(MDBX_C_STANDARD 99) else() set(MDBX_C_STANDARD 11) endif() endif() -if(${CMAKE_SYSTEM_NAME} STREQUAL "Windows" AND EXISTS "${MDBX_SOURCE_DIR}/ntdll.def") +if(WIN32 AND EXISTS "${MDBX_SOURCE_DIR}/ntdll.def") if(MSVC) if(NOT MSVC_LIB_EXE) # Find lib.exe @@ -416,10 +581,12 @@ if(${CMAKE_SYSTEM_NAME} STREQUAL "Windows" AND EXISTS "${MDBX_SOURCE_DIR}/ntdll. if(MSVC_LIB_EXE) message(STATUS "Found MSVC's lib tool: ${MSVC_LIB_EXE}") set(MDBX_NTDLL_EXTRA_IMPLIB "${CMAKE_CURRENT_BINARY_DIR}/mdbx_ntdll_extra.lib") - add_custom_command(OUTPUT "${MDBX_NTDLL_EXTRA_IMPLIB}" + add_custom_command( + OUTPUT "${MDBX_NTDLL_EXTRA_IMPLIB}" COMMENT "Create extra-import-library for ntdll.dll" MAIN_DEPENDENCY "${MDBX_SOURCE_DIR}/ntdll.def" - COMMAND ${MSVC_LIB_EXE} /def:"${MDBX_SOURCE_DIR}/ntdll.def" /out:"${MDBX_NTDLL_EXTRA_IMPLIB}" ${INITIAL_CMAKE_STATIC_LINKER_FLAGS}) + COMMAND ${MSVC_LIB_EXE} /def:"${MDBX_SOURCE_DIR}/ntdll.def" /out:"${MDBX_NTDLL_EXTRA_IMPLIB}" + ${INITIAL_CMAKE_STATIC_LINKER_FLAGS}) else() message(WARNING "MSVC's lib tool not found") endif() @@ -433,7 +600,8 @@ if(${CMAKE_SYSTEM_NAME} STREQUAL "Windows" AND EXISTS "${MDBX_SOURCE_DIR}/ntdll. if(DLLTOOL) message(STATUS "Found dlltool: ${DLLTOOL}") set(MDBX_NTDLL_EXTRA_IMPLIB "${CMAKE_CURRENT_BINARY_DIR}/mdbx_ntdll_extra.a") - add_custom_command(OUTPUT "${MDBX_NTDLL_EXTRA_IMPLIB}" + add_custom_command( + OUTPUT "${MDBX_NTDLL_EXTRA_IMPLIB}" COMMENT "Create extra-import-library for ntdll.dll" MAIN_DEPENDENCY "${MDBX_SOURCE_DIR}/ntdll.def" COMMAND ${DLLTOOL} -d "${MDBX_SOURCE_DIR}/ntdll.def" -l "${MDBX_NTDLL_EXTRA_IMPLIB}") @@ -443,21 +611,20 @@ if(${CMAKE_SYSTEM_NAME} STREQUAL "Windows" AND EXISTS "${MDBX_SOURCE_DIR}/ntdll. endif() if(MDBX_NTDLL_EXTRA_IMPLIB) - # LY: Sometimes CMake requires a nightmarish magic for simple things. - # 1) create a target out of the library compilation result + # Sometimes CMake requires a nightmarish magic for simple things. + # + # (1) create a target out of the library compilation result add_custom_target(ntdll_extra_target DEPENDS "${MDBX_NTDLL_EXTRA_IMPLIB}") - # 2) create an library target out of the library compilation result + # (2) create an library target out of the library compilation result add_library(ntdll_extra STATIC IMPORTED GLOBAL) add_dependencies(ntdll_extra ntdll_extra_target) - # 3) specify where the library is (and where to find the headers) - set_target_properties(ntdll_extra - PROPERTIES - IMPORTED_LOCATION "${MDBX_NTDLL_EXTRA_IMPLIB}") + # (3) specify where the library is (and where to find the headers) + set_target_properties(ntdll_extra PROPERTIES IMPORTED_LOCATION "${MDBX_NTDLL_EXTRA_IMPLIB}") endif() endif() -################################################################################ -################################################################################ +# ###################################################################################################################### +# ~~~ # # #### ##### ##### # #### # # #### # # # # # # # # # ## # # @@ -466,71 +633,75 @@ endif() # # # # # # # # # ## # # # #### # # # #### # # #### # +# ~~~ +# ###################################################################################################################### -set(MDBX_BUILD_OPTIONS ENABLE_UBSAN ENABLE_ASAN MDBX_USE_VALGRIND ENABLE_GPROF ENABLE_GCOV) -macro(add_mdbx_option NAME DESCRIPTION DEFAULT) - list(APPEND MDBX_BUILD_OPTIONS ${NAME}) - if(NOT ${DEFAULT} STREQUAL "AUTO") - option(${NAME} "${DESCRIPTION}" ${DEFAULT}) - elseif(NOT DEFINED ${NAME}) - set(${NAME}_AUTO ON) - endif() -endmacro() +set(MDBX_BUILD_OPTIONS ENABLE_UBSAN ENABLE_ASAN ENABLE_MEMCHECK ENABLE_GPROF ENABLE_GCOV) if(IOS) set(MDBX_BUILD_TOOLS_DEFAULT OFF) if(NOT_SUBPROJECT) cmake_policy(SET CMP0006 OLD) - set(CMAKE_XCODE_ATTRIBUTE_CODE_SIGNING_ALLOWED "NO") + set(CMAKE_XCODE_ATTRIBUTE_CODE_SIGNING_ALLOWED NO) endif() else() set(MDBX_BUILD_TOOLS_DEFAULT ON) endif() -add_mdbx_option(MDBX_INSTALL_STATIC "Build and install libmdbx for static linking" OFF) -add_mdbx_option(MDBX_BUILD_SHARED_LIBRARY "Build libmdbx as shared library (DLL)" ${BUILD_SHARED_LIBS}) -add_mdbx_option(MDBX_BUILD_TOOLS "Build MDBX tools (mdbx_chk/stat/dump/load/copy)" ${MDBX_BUILD_TOOLS_DEFAULT}) -CMAKE_DEPENDENT_OPTION(MDBX_INSTALL_MANPAGES "Install man-pages for MDBX tools (mdbx_chk/stat/dump/load/copy)" ON MDBX_BUILD_TOOLS OFF) -add_mdbx_option(MDBX_TXN_CHECKOWNER "Checking transaction matches the calling thread inside libmdbx's API" ON) -add_mdbx_option(MDBX_ENV_CHECKPID "Paranoid checking PID inside libmdbx's API" AUTO) +add_option(MDBX INSTALL_STATIC "Build and install libmdbx for static linking" OFF) +add_option(MDBX BUILD_SHARED_LIBRARY "Build libmdbx as shared library (DLL)" ${BUILD_SHARED_LIBS}) +add_option(MDBX BUILD_TOOLS "Build MDBX tools (mdbx_chk/stat/dump/load/copy/drop)" ${MDBX_BUILD_TOOLS_DEFAULT}) +cmake_dependent_option(MDBX_INSTALL_MANPAGES "Install man-pages for MDBX tools (mdbx_chk/stat/dump/load/copy)" ON + MDBX_BUILD_TOOLS OFF) +add_option(MDBX TXN_CHECKOWNER "Checking transaction matches the calling thread inside libmdbx's API" ON) +add_option(MDBX ENV_CHECKPID "Checking PID inside libmdbx's API against reuse DB environment after the fork()" AUTO) mark_as_advanced(MDBX_ENV_CHECKPID) if(${CMAKE_SYSTEM_NAME} STREQUAL "Linux") - add_mdbx_option(MDBX_DISABLE_GNU_SOURCE "Don't use GNU/Linux libc extensions" OFF) + add_option(MDBX DISABLE_GNU_SOURCE "Don't use GNU/Linux libc extensions" OFF) mark_as_advanced(MDBX_DISABLE_GNU_SOURCE) endif() if(${CMAKE_SYSTEM_NAME} STREQUAL "Darwin" OR IOS) - add_mdbx_option(MDBX_OSX_SPEED_INSTEADOF_DURABILITY "Disable use fcntl(F_FULLFSYNC) in favor of speed" OFF) - mark_as_advanced(MDBX_OSX_SPEED_INSTEADOF_DURABILITY) + add_option(MDBX APPLE_SPEED_INSTEADOF_DURABILITY "Disable use fcntl(F_FULLFSYNC) in favor of speed" OFF) + mark_as_advanced(MDBX_APPLE_SPEED_INSTEADOF_DURABILITY) endif() -if(${CMAKE_SYSTEM_NAME} STREQUAL "Windows") +if(WIN32) if(MDBX_NTDLL_EXTRA_IMPLIB) - add_mdbx_option(MDBX_WITHOUT_MSVC_CRT "Avoid dependence from MSVC CRT and use ntdll.dll instead" OFF) + add_option(MDBX WITHOUT_MSVC_CRT "Avoid dependence from MSVC CRT and use ntdll.dll instead" OFF) endif() set(MDBX_AVOID_MSYNC_DEFAULT ON) else() - add_mdbx_option(MDBX_USE_OFDLOCKS "Use Open file description locks (aka OFD locks, non-POSIX)" AUTO) + add_option(MDBX USE_OFDLOCKS "Use Open file description locks (aka OFD locks, non-POSIX)" AUTO) mark_as_advanced(MDBX_USE_OFDLOCKS) + add_option(MDBX USE_MINCORE "Use Unix' mincore() to determine whether DB-pages are resident in memory" ON) + mark_as_advanced(MDBX_USE_MINCORE) set(MDBX_AVOID_MSYNC_DEFAULT OFF) endif() -add_mdbx_option(MDBX_AVOID_MSYNC "Controls dirty pages tracking, spilling and persisting in MDBX_WRITEMAP mode" ${MDBX_AVOID_MSYNC_DEFAULT}) -add_mdbx_option(MDBX_LOCKING "Locking method (Windows=-1, SysV=5, POSIX=1988, POSIX=2001, POSIX=2008, Futexes=1995)" AUTO) +add_option( + MDBX AVOID_MSYNC + "Disable in-memory database updating with consequent flush-to-disk/msync syscall in `MDBX_WRITEMAP` mode" + ${MDBX_AVOID_MSYNC_DEFAULT}) +add_option(MDBX MMAP_NEEDS_JOLT "Assume system needs explicit syscall to sync/flush/write modified mapped memory" AUTO) +mark_as_advanced(MDBX_MMAP_NEEDS_JOLT) +add_option(MDBX LOCKING "Locking method (Windows=-1, SystemV=5, POSIX=1988, POSIX=2001, POSIX=2008)" AUTO) mark_as_advanced(MDBX_LOCKING) -add_mdbx_option(MDBX_TRUST_RTC "Does a system have battery-backed Real-Time Clock or just a fake" AUTO) +add_option(MDBX TRUST_RTC "Does a system have battery-backed Real-Time Clock or just a fake" AUTO) mark_as_advanced(MDBX_TRUST_RTC) -add_mdbx_option(MDBX_FORCE_ASSERTIONS "Force enable assertion checking" OFF) -add_mdbx_option(MDBX_DISABLE_VALIDATION "Disable some checks to reduce an overhead and detection probability of database corruption to a values closer to the LMDB" OFF) +add_option(MDBX FORCE_ASSERTIONS "Force enable assertion checking" OFF) +add_option( + MDBX + DISABLE_VALIDATION + "Disable some checks to reduce an overhead and detection probability of database corruption to a values closer to the LMDB" + OFF) mark_as_advanced(MDBX_DISABLE_VALIDATION) -add_mdbx_option(MDBX_ENABLE_REFUND "Zerocost auto-compactification during write-transactions" ON) -add_mdbx_option(MDBX_ENABLE_MADVISE "Using POSIX' madvise() and/or similar hints" ON) -if (CMAKE_TARGET_BITNESS GREATER 32) - set(MDBX_BIGFOOT_DEFAULT ON) -else() - set(MDBX_BIGFOOT_DEFAULT OFF) -endif() -add_mdbx_option(MDBX_ENABLE_BIGFOOT "Chunking long list of retired pages during huge transactions commit to avoid use sequences of pages" ${MDBX_BIGFOOT_DEFAULT}) -add_mdbx_option(MDBX_ENABLE_PGOP_STAT "Gathering statistics for page operations" ON) -add_mdbx_option(MDBX_ENABLE_PROFGC "Profiling of GC search and updates" OFF) +add_option(MDBX ENABLE_REFUND "Zerocost auto-compactification during write-transactions" ON) +add_option(MDBX ENABLE_BIGFOOT + "Chunking long list of retired pages during huge transactions commit to avoid use sequences of pages" ON) +add_option(MDBX ENABLE_PGOP_STAT "Gathering statistics for page operations" ON) +add_option(MDBX ENABLE_PROFGC "Profiling of GC search and updates" OFF) mark_as_advanced(MDBX_ENABLE_PROFGC) +add_option(MDBX ENABLE_DBI_SPARSE + "Support for sparse sets of DBI handles to reduce overhead when starting and processing transactions" ON) +add_option(MDBX ENABLE_DBI_LOCKFREE "Support for deferred releasing and a lockfree path to quickly open DBI handles" ON) if(NOT MDBX_AMALGAMATED_SOURCE) if(CMAKE_CONFIGURATION_TYPES OR CMAKE_BUILD_TYPE_UPPERCASE STREQUAL "DEBUG") @@ -538,23 +709,25 @@ if(NOT MDBX_AMALGAMATED_SOURCE) else() set(MDBX_ALLOY_BUILD_DEFAULT ON) endif() - add_mdbx_option(MDBX_ALLOY_BUILD "Build MDBX library through single/alloyed object file" ${MDBX_ALLOY_BUILD_DEFAULT}) + add_option(MDBX ALLOY_BUILD "Build MDBX library through single/alloyed object file" ${MDBX_ALLOY_BUILD_DEFAULT}) endif() if((MDBX_BUILD_TOOLS OR MDBX_ENABLE_TESTS) AND MDBX_BUILD_SHARED_LIBRARY) - add_mdbx_option(MDBX_LINK_TOOLS_NONSTATIC "Link MDBX tools with non-static libmdbx" OFF) + add_option(MDBX LINK_TOOLS_NONSTATIC "Link MDBX tools with non-static libmdbx" OFF) else() unset(MDBX_LINK_TOOLS_NONSTATIC CACHE) endif() -if(CMAKE_CXX_COMPILER_LOADED AND MDBX_CXX_STANDARD LESS 83 AND NOT MDBX_CXX_STANDARD LESS 11) +if(CMAKE_CXX_COMPILER_LOADED + AND MDBX_CXX_STANDARD LESS 83 + AND NOT MDBX_CXX_STANDARD LESS 11) if(NOT MDBX_AMALGAMATED_SOURCE) option(MDBX_ENABLE_TESTS "Build MDBX tests" ${BUILD_TESTING}) endif() if(NOT MDBX_WITHOUT_MSVC_CRT - AND NOT (CMAKE_COMPILER_IS_GNUCXX AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.8) - AND NOT (CMAKE_COMPILER_IS_CLANG AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 3.9) - AND NOT (MSVC AND MSVC_VERSION LESS 1900)) + AND NOT (CMAKE_COMPILER_IS_GNUCXX AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.8) + AND NOT (CMAKE_COMPILER_IS_CLANG AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 3.9) + AND NOT (MSVC AND MSVC_VERSION LESS 1900)) option(MDBX_BUILD_CXX "Build C++ portion" ON) else() set(MDBX_BUILD_CXX FALSE) @@ -564,8 +737,11 @@ else() set(MDBX_ENABLE_TESTS FALSE) endif() -################################################################################ -################################################################################ +if(CI) + add_definitions(-DMDBX_CI="${CI}") +endif() + +# ###################################################################################################################### if(MDBX_BUILD_CXX AND NOT CMAKE_CXX_COMPILER_LOADED) message(FATAL_ERROR "MDBX_BUILD_CXX=${MDBX_BUILD_CXX}: The C++ compiler is required to build the C++API.") @@ -576,10 +752,6 @@ if(MDBX_BUILD_CXX) probe_libcxx_filesystem() endif() -# Get version -fetch_version(MDBX "${CMAKE_CURRENT_SOURCE_DIR}" FALSE) -message(STATUS "libmdbx version is ${MDBX_VERSION}") - # sources list set(LIBMDBX_PUBLIC_HEADERS mdbx.h) set(LIBMDBX_SOURCES mdbx.h "${CMAKE_CURRENT_BINARY_DIR}/config.h") @@ -587,8 +759,7 @@ if(MDBX_AMALGAMATED_SOURCE) list(APPEND LIBMDBX_SOURCES mdbx.c) else() # generate version file - configure_file("${MDBX_SOURCE_DIR}/version.c.in" - "${CMAKE_CURRENT_BINARY_DIR}/version.c" ESCAPE_QUOTES) + configure_file("${MDBX_SOURCE_DIR}/version.c.in" "${CMAKE_CURRENT_BINARY_DIR}/version.c" ESCAPE_QUOTES) file(SHA256 "${CMAKE_CURRENT_BINARY_DIR}/version.c" MDBX_SOURCERY_DIGEST) string(MAKE_C_IDENTIFIER "${MDBX_GIT_DESCRIBE}" MDBX_SOURCERY_SUFFIX) set(MDBX_BUILD_SOURCERY "${MDBX_SOURCERY_DIGEST}_${MDBX_SOURCERY_SUFFIX}") @@ -597,14 +768,89 @@ else() list(APPEND LIBMDBX_SOURCES "${MDBX_SOURCE_DIR}/alloy.c") include_directories("${MDBX_SOURCE_DIR}" "${CMAKE_CURRENT_BINARY_DIR}") else() - list(APPEND LIBMDBX_SOURCES - "${CMAKE_CURRENT_BINARY_DIR}/version.c" - "${MDBX_SOURCE_DIR}/options.h" "${MDBX_SOURCE_DIR}/base.h" - "${MDBX_SOURCE_DIR}/internals.h" "${MDBX_SOURCE_DIR}/osal.h" - "${MDBX_SOURCE_DIR}/core.c" "${MDBX_SOURCE_DIR}/osal.c" - "${MDBX_SOURCE_DIR}/lck-posix.c") + list( + APPEND + LIBMDBX_SOURCES + "${MDBX_SOURCE_DIR}/api-cold.c" + "${MDBX_SOURCE_DIR}/api-copy.c" + "${MDBX_SOURCE_DIR}/api-cursor.c" + "${MDBX_SOURCE_DIR}/api-dbi.c" + "${MDBX_SOURCE_DIR}/api-env.c" + "${MDBX_SOURCE_DIR}/api-extra.c" + "${MDBX_SOURCE_DIR}/api-key-transform.c" + "${MDBX_SOURCE_DIR}/api-misc.c" + "${MDBX_SOURCE_DIR}/api-opts.c" + "${MDBX_SOURCE_DIR}/api-range-estimate.c" + "${MDBX_SOURCE_DIR}/api-txn-data.c" + "${MDBX_SOURCE_DIR}/api-txn.c" + "${MDBX_SOURCE_DIR}/atomics-ops.h" + "${MDBX_SOURCE_DIR}/atomics-types.h" + "${MDBX_SOURCE_DIR}/audit.c" + "${MDBX_SOURCE_DIR}/chk.c" + "${MDBX_SOURCE_DIR}/cogs.c" + "${MDBX_SOURCE_DIR}/cogs.h" + "${MDBX_SOURCE_DIR}/coherency.c" + "${MDBX_SOURCE_DIR}/cursor.c" + "${MDBX_SOURCE_DIR}/cursor.h" + "${MDBX_SOURCE_DIR}/dbi.c" + "${MDBX_SOURCE_DIR}/dbi.h" + "${MDBX_SOURCE_DIR}/dpl.c" + "${MDBX_SOURCE_DIR}/dpl.h" + "${MDBX_SOURCE_DIR}/dxb.c" + "${MDBX_SOURCE_DIR}/env.c" + "${MDBX_SOURCE_DIR}/essentials.h" + "${MDBX_SOURCE_DIR}/gc-get.c" + "${MDBX_SOURCE_DIR}/gc-put.c" + "${MDBX_SOURCE_DIR}/gc.h" + "${MDBX_SOURCE_DIR}/global.c" + "${MDBX_SOURCE_DIR}/internals.h" + "${MDBX_SOURCE_DIR}/layout-dxb.h" + "${MDBX_SOURCE_DIR}/layout-lck.h" + "${MDBX_SOURCE_DIR}/lck.c" + "${MDBX_SOURCE_DIR}/lck.h" + "${MDBX_SOURCE_DIR}/logging_and_debug.c" + "${MDBX_SOURCE_DIR}/logging_and_debug.h" + "${MDBX_SOURCE_DIR}/meta.c" + "${MDBX_SOURCE_DIR}/meta.h" + "${MDBX_SOURCE_DIR}/mvcc-readers.c" + "${MDBX_SOURCE_DIR}/node.c" + "${MDBX_SOURCE_DIR}/node.h" + "${MDBX_SOURCE_DIR}/options.h" + "${MDBX_SOURCE_DIR}/osal.c" + "${MDBX_SOURCE_DIR}/osal.h" + "${MDBX_SOURCE_DIR}/page-get.c" + "${MDBX_SOURCE_DIR}/page-iov.c" + "${MDBX_SOURCE_DIR}/page-iov.h" + "${MDBX_SOURCE_DIR}/page-ops.c" + "${MDBX_SOURCE_DIR}/page-ops.h" + "${MDBX_SOURCE_DIR}/tree-search.c" + "${MDBX_SOURCE_DIR}/pnl.c" + "${MDBX_SOURCE_DIR}/pnl.h" + "${MDBX_SOURCE_DIR}/preface.h" + "${MDBX_SOURCE_DIR}/proto.h" + "${MDBX_SOURCE_DIR}/refund.c" + "${MDBX_SOURCE_DIR}/sort.h" + "${MDBX_SOURCE_DIR}/spill.c" + "${MDBX_SOURCE_DIR}/spill.h" + "${MDBX_SOURCE_DIR}/table.c" + "${MDBX_SOURCE_DIR}/tls.c" + "${MDBX_SOURCE_DIR}/tls.h" + "${MDBX_SOURCE_DIR}/tree-ops.c" + "${MDBX_SOURCE_DIR}/txl.c" + "${MDBX_SOURCE_DIR}/txl.h" + "${MDBX_SOURCE_DIR}/txn.c" + "${MDBX_SOURCE_DIR}/unaligned.h" + "${MDBX_SOURCE_DIR}/utils.c" + "${MDBX_SOURCE_DIR}/utils.h" + "${MDBX_SOURCE_DIR}/walk.c" + "${MDBX_SOURCE_DIR}/walk.h" + "${CMAKE_CURRENT_BINARY_DIR}/version.c") + if(NOT MSVC) + list(APPEND LIBMDBX_SOURCES "${MDBX_SOURCE_DIR}/lck-posix.c") + endif() if(NOT APPLE) - list(APPEND LIBMDBX_SOURCES "${MDBX_SOURCE_DIR}/lck-windows.c") + list(APPEND LIBMDBX_SOURCES "${MDBX_SOURCE_DIR}/windows-import.h" "${MDBX_SOURCE_DIR}/windows-import.c" + "${MDBX_SOURCE_DIR}/lck-windows.c") endif() include_directories("${MDBX_SOURCE_DIR}") endif() @@ -617,31 +863,26 @@ else() message(STATUS "Use C${MDBX_C_STANDARD} for libmdbx but C++ portion is disabled") endif() -if(SUBPROJECT AND MSVC) - if(MSVC_VERSION LESS 1900) - message(FATAL_ERROR "At least \"Microsoft C/C++ Compiler\" version 19.0.24234.1 (Visual Studio 2015 Update 3) is required.") - endif() +if(MSVC) add_compile_options("/utf-8") endif() macro(target_setup_options TARGET) if(DEFINED INTERPROCEDURAL_OPTIMIZATION) - set_target_properties(${TARGET} PROPERTIES - INTERPROCEDURAL_OPTIMIZATION $) + set_target_properties(${TARGET} PROPERTIES INTERPROCEDURAL_OPTIMIZATION $) endif() if(NOT C_FALLBACK_GNU11 AND NOT C_FALLBACK_11) - set_target_properties(${TARGET} PROPERTIES - C_STANDARD ${MDBX_C_STANDARD} C_STANDARD_REQUIRED ON) + set_target_properties(${TARGET} PROPERTIES C_STANDARD ${MDBX_C_STANDARD} C_STANDARD_REQUIRED ON) endif() if(MDBX_BUILD_CXX) - set_target_properties(${TARGET} PROPERTIES - CXX_STANDARD ${MDBX_CXX_STANDARD} CXX_STANDARD_REQUIRED ON) + if(NOT CXX_FALLBACK_GNU11 AND NOT CXX_FALLBACK_11) + set_target_properties(${TARGET} PROPERTIES CXX_STANDARD ${MDBX_CXX_STANDARD} CXX_STANDARD_REQUIRED ON) + endif() if(MSVC AND NOT MSVC_VERSION LESS 1910) target_compile_options(${TARGET} INTERFACE "/Zc:__cplusplus") endif() endif() - if(CC_HAS_FASTMATH - AND NOT (CMAKE_COMPILER_IS_CLANG AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 10)) + if(CC_HAS_FASTMATH AND NOT (CMAKE_COMPILER_IS_CLANG AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 10)) target_compile_options(${TARGET} PRIVATE "-ffast-math") endif() if(CC_HAS_VISIBILITY) @@ -658,8 +899,15 @@ macro(libmdbx_setup_libs TARGET MODE) else() target_link_libraries(${TARGET} ${MODE} Threads::Threads) endif() - if(${CMAKE_SYSTEM_NAME} STREQUAL "Windows") - target_link_libraries(${TARGET} ${MODE} ntdll user32 kernel32 advapi32) + if(WIN32) + target_link_libraries( + ${TARGET} + ${MODE} + ntdll + user32 + kernel32 + advapi32 + ole32) if(MDBX_NTDLL_EXTRA_IMPLIB AND MDBX_WITHOUT_MSVC_CRT) target_link_libraries(${TARGET} ${MODE} ntdll_extra) endif() @@ -669,8 +917,9 @@ macro(libmdbx_setup_libs TARGET MODE) target_link_libraries(${TARGET} ${MODE} log) endif() if(LIBCXX_FILESYSTEM AND MDBX_BUILD_CXX) - if(CMAKE_COMPILER_IS_ELBRUSCXX AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 1.25.23 - AND NOT CMAKE_VERSION VERSION_LESS 3.13) + if(CMAKE_COMPILER_IS_ELBRUSCXX + AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 1.25.23 + AND NOT CMAKE_VERSION VERSION_LESS 3.13) target_link_options(${TARGET} PUBLIC "-Wl,--allow-multiple-definition") endif() target_link_libraries(${TARGET} PUBLIC ${LIBCXX_FILESYSTEM}) @@ -694,13 +943,16 @@ else() endif() target_include_directories(mdbx-static INTERFACE "${CMAKE_CURRENT_SOURCE_DIR}") -################################################################################ +# ###################################################################################################################### # build shared library if(MDBX_BUILD_SHARED_LIBRARY) add_library(mdbx SHARED ${LIBMDBX_SOURCES}) set_target_properties(mdbx PROPERTIES PUBLIC_HEADER "${LIBMDBX_PUBLIC_HEADERS}") - target_compile_definitions(mdbx PRIVATE LIBMDBX_EXPORTS MDBX_BUILD_SHARED_LIBRARY=1 INTERFACE LIBMDBX_IMPORTS) + target_compile_definitions( + mdbx + PRIVATE LIBMDBX_EXPORTS MDBX_BUILD_SHARED_LIBRARY=1 + INTERFACE LIBMDBX_IMPORTS) target_setup_options(mdbx) libmdbx_setup_libs(mdbx PRIVATE) if(MSVC) @@ -726,8 +978,8 @@ if(MDBX_BUILD_SHARED_LIBRARY AND MDBX_LINK_TOOLS_NONSTATIC) # when building, don't use the install RPATH already (but later on when installing) set(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE) - # add the automatically determined parts of the RPATH - # which point to directories outside the build tree to the install RPATH + # add the automatically determined parts of the RPATH which point to directories outside the build tree to the install + # RPATH set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) # the RPATH to be used when installing, but only if it's not a system directory @@ -739,26 +991,40 @@ if(MDBX_BUILD_SHARED_LIBRARY AND MDBX_LINK_TOOLS_NONSTATIC) set(CMAKE_INSTALL_RPATH "\$ORIGIN/../lib") endif() endif() + + if(WIN32) + # Windows don't have RPATH feature, therefore we should prepare PATH or copy DLL(s) + set(TOOL_MDBX_DLLCRUTCH "Crutch for ${CMAKE_SYSTEM_NAME}") + if(NOT CMAKE_CONFIGURATION_TYPES AND NOT CMAKE_VERSION VERSION_LESS 3.0) + # will use LOCATION property to compose DLLPATH + cmake_policy(SET CMP0026 OLD) + endif() + else() + set(TOOL_MDBX_DLLCRUTCH FALSE) + endif() else() set(TOOL_MDBX_LIB mdbx-static) + set(TOOL_MDBX_DLLCRUTCH FALSE) endif() # build mdbx-tools if(MDBX_BUILD_TOOLS) - if(NOT MDBX_AMALGAMATED_SOURCE AND ${CMAKE_SYSTEM_NAME} STREQUAL "Windows") - set(WINGETOPT_SRC ${MDBX_SOURCE_DIR}/wingetopt.c ${MDBX_SOURCE_DIR}/wingetopt.h) - else() - set(WINGETOPT_SRC "") + set(WINGETOPT_SRC "") + if(WIN32) + set(WINGETOPT_SRC ${MDBX_SOURCE_DIR}/tools/wingetopt.c ${MDBX_SOURCE_DIR}/tools/wingetopt.h) endif() - foreach(TOOL mdbx_chk mdbx_copy mdbx_stat mdbx_dump mdbx_load mdbx_drop) - add_executable(${TOOL} mdbx.h ${MDBX_SOURCE_DIR}/${TOOL}.c ${WINGETOPT_SRC}) + foreach(TOOL chk copy stat dump load drop) + if(MDBX_AMALGAMATED_SOURCE) + add_executable(mdbx_${TOOL} mdbx.h ${MDBX_SOURCE_DIR}/mdbx_${TOOL}.c) + else() + add_executable(mdbx_${TOOL} mdbx.h ${MDBX_SOURCE_DIR}/tools/${TOOL}.c ${WINGETOPT_SRC}) + endif() if(NOT C_FALLBACK_GNU11 AND NOT C_FALLBACK_11) - set_target_properties(${TOOL} PROPERTIES - C_STANDARD ${MDBX_C_STANDARD} C_STANDARD_REQUIRED ON) + set_target_properties(mdbx_${TOOL} PROPERTIES C_STANDARD ${MDBX_C_STANDARD} C_STANDARD_REQUIRED ON) endif() - target_setup_options(${TOOL}) - target_link_libraries(${TOOL} ${TOOL_MDBX_LIB}) + target_setup_options(mdbx_${TOOL}) + target_link_libraries(mdbx_${TOOL} ${TOOL_MDBX_LIB}) endforeach() if(LIB_MATH) target_link_libraries(mdbx_chk ${LIB_MATH}) @@ -766,7 +1032,7 @@ if(MDBX_BUILD_TOOLS) endif() endif() -################################################################################ +# ###################################################################################################################### # mdbx-shared-lib installation if(NOT DEFINED MDBX_DLL_INSTALL_DESTINATION) @@ -778,19 +1044,28 @@ if(NOT DEFINED MDBX_DLL_INSTALL_DESTINATION) endif() if(MDBX_BUILD_SHARED_LIBRARY) if(CMAKE_VERSION VERSION_LESS 3.12) - install(TARGETS mdbx EXPORT libmdbx + install( + TARGETS mdbx + EXPORT libmdbx LIBRARY DESTINATION ${MDBX_DLL_INSTALL_DESTINATION} COMPONENT runtime ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT devel PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} COMPONENT devel - INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} COMPONENT devel) + INCLUDES + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + COMPONENT devel) else() - install(TARGETS mdbx EXPORT libmdbx - LIBRARY DESTINATION ${MDBX_DLL_INSTALL_DESTINATION} COMPONENT runtime - NAMELINK_COMPONENT devel + install( + TARGETS mdbx + EXPORT libmdbx + LIBRARY DESTINATION ${MDBX_DLL_INSTALL_DESTINATION} + COMPONENT runtime + NAMELINK_COMPONENT devel OBJECTS DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT devel ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT devel PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} COMPONENT devel - INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} COMPONENT devel) + INCLUDES + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + COMPONENT devel) endif() endif(MDBX_BUILD_SHARED_LIBRARY) @@ -799,29 +1074,16 @@ if(MDBX_BUILD_TOOLS) if(NOT DEFINED MDBX_TOOLS_INSTALL_DESTINATION) set(MDBX_TOOLS_INSTALL_DESTINATION ${CMAKE_INSTALL_BINDIR}) endif() - install( - TARGETS - mdbx_chk - mdbx_stat - mdbx_copy - mdbx_dump - mdbx_load - mdbx_drop - RUNTIME - DESTINATION ${MDBX_TOOLS_INSTALL_DESTINATION} - COMPONENT runtime) + install(TARGETS mdbx_chk mdbx_stat mdbx_copy mdbx_dump mdbx_load mdbx_drop + RUNTIME DESTINATION ${MDBX_TOOLS_INSTALL_DESTINATION} COMPONENT runtime) if(MDBX_INSTALL_MANPAGES) if(NOT DEFINED MDBX_MAN_INSTALL_DESTINATION) set(MDBX_MAN_INSTALL_DESTINATION ${CMAKE_INSTALL_MANDIR}/man1) endif() install( - FILES - "${MDBX_SOURCE_DIR}/man1/mdbx_chk.1" - "${MDBX_SOURCE_DIR}/man1/mdbx_stat.1" - "${MDBX_SOURCE_DIR}/man1/mdbx_copy.1" - "${MDBX_SOURCE_DIR}/man1/mdbx_dump.1" - "${MDBX_SOURCE_DIR}/man1/mdbx_load.1" - "${MDBX_SOURCE_DIR}/man1/mdbx_drop.1" + FILES "${MDBX_SOURCE_DIR}/man1/mdbx_chk.1" "${MDBX_SOURCE_DIR}/man1/mdbx_stat.1" + "${MDBX_SOURCE_DIR}/man1/mdbx_copy.1" "${MDBX_SOURCE_DIR}/man1/mdbx_dump.1" + "${MDBX_SOURCE_DIR}/man1/mdbx_load.1" "${MDBX_SOURCE_DIR}/man1/mdbx_drop.1" DESTINATION ${MDBX_MAN_INSTALL_DESTINATION} COMPONENT doc) endif() @@ -830,28 +1092,41 @@ endif(MDBX_BUILD_TOOLS) # mdbx-static-lib installation if(MDBX_INSTALL_STATIC) if(CMAKE_VERSION VERSION_LESS 3.12) - install(TARGETS mdbx-static EXPORT libmdbx + install( + TARGETS mdbx-static + EXPORT libmdbx LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT devel OBJECTS DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT devel ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT devel PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} COMPONENT devel - INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} COMPONENT devel) + INCLUDES + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + COMPONENT devel) else() - install(TARGETS mdbx-static EXPORT libmdbx - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT devel - NAMELINK_COMPONENT devel + install( + TARGETS mdbx-static + EXPORT libmdbx + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + COMPONENT devel + NAMELINK_COMPONENT devel OBJECTS DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT devel ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT devel PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} COMPONENT devel - INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} COMPONENT devel) + INCLUDES + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + COMPONENT devel) endif() endif(MDBX_INSTALL_STATIC) -################################################################################ +# ###################################################################################################################### # collect options & build info if(NOT DEFINED MDBX_BUILD_TIMESTAMP) - string(TIMESTAMP MDBX_BUILD_TIMESTAMP UTC) + if(NOT "$ENV{SOURCE_DATE_EPOCH}" STREQUAL "") + set(MDBX_BUILD_TIMESTAMP "$ENV{SOURCE_DATE_EPOCH}") + else() + string(TIMESTAMP MDBX_BUILD_TIMESTAMP UTC) + endif() endif() set(MDBX_BUILD_FLAGS ${CMAKE_C_FLAGS}) if(MDBX_BUILD_CXX) @@ -890,16 +1165,18 @@ string(REPLACE ";" " " MDBX_BUILD_FLAGS "${MDBX_BUILD_FLAGS}") if(CMAKE_CONFIGURATION_TYPES) # add dynamic part via per-configuration define message(STATUS "MDBX Compile Flags: ${MDBX_BUILD_FLAGS} ") - add_definitions(-DMDBX_BUILD_FLAGS_CONFIG="$<$:${CMAKE_C_FLAGS_DEBUG} ${CMAKE_C_DEFINES_DEBUG}>$<$:${CMAKE_C_FLAGS_RELEASE} ${CMAKE_C_DEFINES_RELEASE}>$<$:${CMAKE_C_FLAGS_RELWITHDEBINFO} ${CMAKE_C_DEFINES_RELWITHDEBINFO}>$<$:${CMAKE_C_FLAGS_MINSIZEREL} ${CMAKE_C_DEFINES_MINSIZEREL}>") + add_definitions( + -DMDBX_BUILD_FLAGS_CONFIG="$<$:${CMAKE_C_FLAGS_DEBUG} ${CMAKE_C_DEFINES_DEBUG}>$<$:${CMAKE_C_FLAGS_RELEASE} ${CMAKE_C_DEFINES_RELEASE}>$<$:${CMAKE_C_FLAGS_RELWITHDEBINFO} ${CMAKE_C_DEFINES_RELWITHDEBINFO}>$<$:${CMAKE_C_FLAGS_MINSIZEREL} ${CMAKE_C_DEFINES_MINSIZEREL}>" + ) else() message(STATUS "MDBX Compile Flags: ${MDBX_BUILD_FLAGS}") endif() # get compiler info -execute_process(COMMAND sh -c "${CMAKE_C_COMPILER} --version | head -1" +execute_process( + COMMAND sh -c "${CMAKE_C_COMPILER} --version | head -1" OUTPUT_VARIABLE MDBX_BUILD_COMPILER - OUTPUT_STRIP_TRAILING_WHITESPACE - ERROR_QUIET + OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET RESULT_VARIABLE rc) if(rc OR NOT MDBX_BUILD_COMPILER) string(STRIP "${CMAKE_C_COMPILER_ID}-${CMAKE_C_COMPILER_VERSION}" MDBX_BUILD_COMPILER) @@ -922,14 +1199,15 @@ else() else() set(MDBX_BUILD_TARGET "unknown") endif() - if(CMAKE_C_COMPILER_ABI - AND NOT (CMAKE_C_COMPILER_ABI MATCHES ".*${MDBX_BUILD_TARGET}.*" OR MDBX_BUILD_TARGET MATCHES ".*${CMAKE_C_COMPILER_ABI}.*")) + if(CMAKE_C_COMPILER_ABI AND NOT (CMAKE_C_COMPILER_ABI MATCHES ".*${MDBX_BUILD_TARGET}.*" + OR MDBX_BUILD_TARGET MATCHES ".*${CMAKE_C_COMPILER_ABI}.*")) string(CONCAT MDBX_BUILD_TARGET "${MDBX_BUILD_TARGET}-${CMAKE_C_COMPILER_ABI}") endif() if(CMAKE_C_PLATFORM_ID - AND NOT (CMAKE_SYSTEM_NAME - AND (CMAKE_C_PLATFORM_ID MATCHES ".*${CMAKE_SYSTEM_NAME}.*" OR CMAKE_SYSTEM_NAME MATCHES ".*${CMAKE_C_PLATFORM_ID}.*")) - AND NOT (CMAKE_C_PLATFORM_ID MATCHES ".*${CMAKE_C_PLATFORM_ID}.*" OR MDBX_BUILD_TARGET MATCHES ".*${CMAKE_C_PLATFORM_ID}.*")) + AND NOT (CMAKE_SYSTEM_NAME AND (CMAKE_C_PLATFORM_ID MATCHES ".*${CMAKE_SYSTEM_NAME}.*" + OR CMAKE_SYSTEM_NAME MATCHES ".*${CMAKE_C_PLATFORM_ID}.*")) + AND NOT (CMAKE_C_PLATFORM_ID MATCHES ".*${CMAKE_C_PLATFORM_ID}.*" OR MDBX_BUILD_TARGET MATCHES + ".*${CMAKE_C_PLATFORM_ID}.*")) string(CONCAT MDBX_BUILD_TARGET "${MDBX_BUILD_TARGET}-${CMAKE_C_COMPILER_ABI}") endif() if(CMAKE_SYSTEM_NAME) @@ -964,11 +1242,10 @@ foreach(item IN LISTS options) endforeach(item) # provide config.h for library build info -configure_file("${MDBX_SOURCE_DIR}/config.h.in" - "${CMAKE_CURRENT_BINARY_DIR}/config.h" ESCAPE_QUOTES) +configure_file("${MDBX_SOURCE_DIR}/config.h.in" "${CMAKE_CURRENT_BINARY_DIR}/config.h" ESCAPE_QUOTES) add_definitions(-DMDBX_CONFIG_H="${CMAKE_CURRENT_BINARY_DIR}/config.h") -################################################################################ +# ###################################################################################################################### if(NOT MDBX_AMALGAMATED_SOURCE AND MDBX_ENABLE_TESTS) if(NOT CMAKE_CXX_COMPILER_LOADED) @@ -977,17 +1254,16 @@ if(NOT MDBX_AMALGAMATED_SOURCE AND MDBX_ENABLE_TESTS) add_subdirectory(test) endif() -################################################################################ +# ###################################################################################################################### -if (NOT SUBPROJECT) +if(NOT SUBPROJECT) set(PACKAGE "libmdbx") set(CPACK_PACKAGE_VERSION_MAJOR ${MDBX_VERSION_MAJOR}) set(CPACK_PACKAGE_VERSION_MINOR ${MDBX_VERSION_MINOR}) - set(CPACK_PACKAGE_VERSION_PATCH ${MDBX_VERSION_RELEASE}) - set(CPACK_PACKAGE_VERSION_COMMIT ${MDBX_VERSION_REVISION}) - set(PACKAGE_VERSION "${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}.${CPACK_PACKAGE_VERSION_COMMIT}") + set(CPACK_PACKAGE_VERSION_PATCH ${MDBX_VERSION_PATCH}) + set(CPACK_PACKAGE_VERSION_TWEAK ${MDBX_VERSION_TWEAK}) + set(PACKAGE_VERSION ${MDBX_VERSION}) message(STATUS "libmdbx package version is ${PACKAGE_VERSION}") - file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/VERSION.txt" "${MDBX_VERSION_MAJOR}.${MDBX_VERSION_MINOR}.${MDBX_VERSION_RELEASE}.${MDBX_VERSION_REVISION}") endif() cmake_policy(POP) diff --git a/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/GNUmakefile b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/GNUmakefile index 35e7849e8b6..32cfc9a05e6 100644 --- a/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/GNUmakefile +++ b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/GNUmakefile @@ -1,4 +1,4 @@ -# This makefile is for GNU Make 3.80 or above, and nowadays provided +# This makefile is for GNU Make 3.81 or above, and nowadays provided # just for compatibility and preservation of traditions. # # Please use CMake in case of any difficulties or @@ -16,6 +16,7 @@ ifneq ($(make_lt_3_81),0) $(error Please use GNU Make 3.81 or above) endif make_ge_4_1 := $(shell expr "$(MAKE_VERx3)" ">=" " 4 1") +make_ge_4_4 := $(shell expr "$(MAKE_VERx3)" ">=" " 4 4") SRC_PROBE_C := $(shell [ -f mdbx.c ] && echo mdbx.c || echo src/osal.c) SRC_PROBE_CXX := $(shell [ -f mdbx.c++ ] && echo mdbx.c++ || echo src/mdbx.c++) UNAME := $(shell uname -s 2>/dev/null || echo Unknown) @@ -51,17 +52,24 @@ CC ?= gcc CXX ?= g++ CFLAGS_EXTRA ?= LD ?= ld +CMAKE ?= cmake +CMAKE_OPT ?= +CTEST ?= ctest +CTEST_OPT ?= +# target directory for `make dist` +DIST_DIR ?= dist # build options MDBX_BUILD_OPTIONS ?=-DNDEBUG=1 -MDBX_BUILD_TIMESTAMP ?=$(shell date +%Y-%m-%dT%H:%M:%S%z) -MDBX_BUILD_CXX ?= YES +MDBX_BUILD_TIMESTAMP ?=$(if $(SOURCE_DATE_EPOCH),$(SOURCE_DATE_EPOCH),$(shell date +%Y-%m-%dT%H:%M:%S%z)) +MDBX_BUILD_CXX ?=YES +MDBX_BUILD_METADATA ?= # probe and compose common compiler flags with variable expansion trick (seems this work two times per session for GNU Make 3.81) CFLAGS ?= $(strip $(eval CFLAGS := -std=gnu11 -O2 -g -Wall -Werror -Wextra -Wpedantic -ffunction-sections -fPIC -fvisibility=hidden -pthread -Wno-error=attributes $$(shell for opt in -fno-semantic-interposition -Wno-unused-command-line-argument -Wno-tautological-compare; do [ -z "$$$$($(CC) '-DMDBX_BUILD_FLAGS="probe"' $$$${opt} -c $(SRC_PROBE_C) -o /dev/null >/dev/null 2>&1 || echo failed)" ] && echo "$$$${opt} "; done)$(CFLAGS_EXTRA))$(CFLAGS)) # choosing C++ standard with variable expansion trick (seems this work two times per session for GNU Make 3.81) -CXXSTD ?= $(eval CXXSTD := $$(shell for std in gnu++23 c++23 gnu++2b c++2b gnu++20 c++20 gnu++2a c++2a gnu++17 c++17 gnu++1z c++1z gnu++14 c++14 gnu++1y c++1y gnu+11 c++11 gnu++0x c++0x; do $(CXX) -std=$$$${std} -c $(SRC_PROBE_CXX) -o /dev/null 2>probe4std-$$$${std}.err >/dev/null && echo "-std=$$$${std}" && exit; done))$(CXXSTD) +CXXSTD ?= $(eval CXXSTD := $$(shell for std in gnu++23 c++23 gnu++2b c++2b gnu++20 c++20 gnu++2a c++2a gnu++17 c++17 gnu++1z c++1z gnu++14 c++14 gnu++1y c++1y gnu+11 c++11 gnu++0x c++0x; do $(CXX) -std=$$$${std} -DMDBX_BUILD_CXX=1 -c $(SRC_PROBE_CXX) -o /dev/null 2>probe4std-$$$${std}.err >/dev/null && echo "-std=$$$${std}" && exit; done))$(CXXSTD) CXXFLAGS ?= $(strip $(CXXSTD) $(filter-out -std=gnu11,$(CFLAGS))) # libraries and options for linking @@ -78,6 +86,13 @@ LDFLAGS ?= $(eval LDFLAGS := $$(shell $$(uname2ldflags)))$(LDFLAGS) LIB_STDCXXFS ?= $(eval LIB_STDCXXFS := $$(shell echo '$$(cxx_filesystem_probe)' | cat mdbx.h++ - | sed $$$$'1s/\xef\xbb\xbf//' | $(CXX) -x c++ $(CXXFLAGS) -Wno-error - -Wl,--allow-multiple-definition -lstdc++fs $(LIBS) $(LDFLAGS) $(EXE_LDFLAGS) -o /dev/null 2>probe4lstdfs.err >/dev/null && echo '-Wl,--allow-multiple-definition -lstdc++fs'))$(LIB_STDCXXFS) endif +ifneq ($(make_ge_4_4),1) +.NOTPARALLEL: +WAIT = +else +WAIT = .WAIT +endif + ################################################################################ define uname2sosuffix @@ -121,12 +136,13 @@ endef SO_SUFFIX := $(shell $(uname2sosuffix)) HEADERS := mdbx.h mdbx.h++ LIBRARIES := libmdbx.a libmdbx.$(SO_SUFFIX) -TOOLS := mdbx_stat mdbx_copy mdbx_dump mdbx_load mdbx_chk mdbx_drop +TOOLS := chk copy drop dump load stat +MDBX_TOOLS := $(addprefix mdbx_,$(TOOLS)) MANPAGES := mdbx_stat.1 mdbx_copy.1 mdbx_dump.1 mdbx_load.1 mdbx_chk.1 mdbx_drop.1 TIP := // TIP: .PHONY: all help options lib libs tools clean install uninstall check_buildflags_tag tools-static -.PHONY: install-strip install-no-strip strip libmdbx mdbx show-options lib-static lib-shared +.PHONY: install-strip install-no-strip strip libmdbx mdbx show-options lib-static lib-shared cmake-build ninja boolean = $(if $(findstring $(strip $($1)),YES Yes yes y ON On on 1 true True TRUE),1,$(if $(findstring $(strip $($1)),NO No no n OFF Off off 0 false False FALSE),,$(error Wrong value `$($1)` of $1 for YES/NO option))) select_by = $(if $(call boolean,$(1)),$(2),$(3)) @@ -148,7 +164,11 @@ else $(info $(TIP) Use `make V=1` for verbose.) endif -all: show-options $(LIBRARIES) $(TOOLS) +ifeq ($(UNAME),Darwin) + $(info $(TIP) Use `brew install gnu-sed gnu-tar` and add ones to the beginning of the PATH.) +endif + +all: show-options $(LIBRARIES) $(MDBX_TOOLS) help: @echo " make all - build libraries and tools" @@ -160,6 +180,7 @@ help: @echo " make clean " @echo " make install " @echo " make uninstall " + @echo " make cmake-build | ninja - build by CMake & Ninja" @echo "" @echo " make strip - strip debug symbols from binaries" @echo " make install-no-strip - install explicitly without strip" @@ -175,6 +196,7 @@ show-options: @echo " MDBX_BUILD_OPTIONS = $(MDBX_BUILD_OPTIONS)" @echo " MDBX_BUILD_CXX = $(MDBX_BUILD_CXX)" @echo " MDBX_BUILD_TIMESTAMP = $(MDBX_BUILD_TIMESTAMP)" + @echo " MDBX_BUILD_METADATA = $(MDBX_BUILD_METADATA)" @echo '$(TIP) Use `make options` to listing available build options.' @echo $(call select_by,MDBX_BUILD_CXX," CXX =`which $(CXX)` | `$(CXX) --version | head -1`"," CC =`which $(CC)` | `$(CC) --version | head -1`") @echo $(call select_by,MDBX_BUILD_CXX," CXXFLAGS =$(CXXFLAGS)"," CFLAGS =$(CFLAGS)") @@ -202,38 +224,39 @@ options: @echo "" @echo " MDBX_BUILD_OPTIONS = $(MDBX_BUILD_OPTIONS)" @echo " MDBX_BUILD_TIMESTAMP = $(MDBX_BUILD_TIMESTAMP)" + @echo " MDBX_BUILD_METADATA = $(MDBX_BUILD_METADATA)" @echo "" @echo "## Assortment items for MDBX_BUILD_OPTIONS:" @echo "## Note that the defaults should already be correct for most platforms;" @echo "## you should not need to change any of these. Read their descriptions" @echo "## in README and source code (see mdbx.c) if you do." - @grep -h '#ifndef MDBX_' mdbx.c | grep -v BUILD | uniq | sed 's/#ifndef / /' + @grep -h '#ifndef MDBX_' mdbx.c | grep -v BUILD | sort -u | sed 's/#ifndef / /' lib libs libmdbx mdbx: libmdbx.a libmdbx.$(SO_SUFFIX) -tools: $(TOOLS) -tools-static: $(addsuffix .static,$(TOOLS)) $(addsuffix .static-lto,$(TOOLS)) +tools: $(MDBX_TOOLS) +tools-static: $(addsuffix .static,$(MDBX_TOOLS)) $(addsuffix .static-lto,$(MDBX_TOOLS)) strip: all - @echo ' STRIP libmdbx.$(SO_SUFFIX) $(TOOLS)' - $(TRACE )strip libmdbx.$(SO_SUFFIX) $(TOOLS) + @echo ' STRIP libmdbx.$(SO_SUFFIX) $(MDBX_TOOLS)' + $(TRACE )strip libmdbx.$(SO_SUFFIX) $(MDBX_TOOLS) clean: @echo ' REMOVE ...' - $(QUIET)rm -rf $(TOOLS) mdbx_test @* *.[ao] *.[ls]o *.$(SO_SUFFIX) *.dSYM *~ tmp.db/* \ - *.gcov *.log *.err src/*.o test/*.o mdbx_example dist \ - config.h src/config.h src/version.c *.tar* buildflags.tag \ - mdbx_*.static mdbx_*.static-lto + $(QUIET)rm -rf $(MDBX_TOOLS) mdbx_test @* *.[ao] *.[ls]o *.$(SO_SUFFIX) *.dSYM *~ tmp.db/* \ + *.gcov *.log *.err src/*.o test/*.o mdbx_example dist @dist-check \ + config.h src/config.h src/version.c *.tar* @buildflags.tag @dist-checked.tag \ + mdbx_*.static mdbx_*.static-lto CMakeFiles MDBX_BUILD_FLAGS =$(strip MDBX_BUILD_CXX=$(MDBX_BUILD_CXX) $(MDBX_BUILD_OPTIONS) $(call select_by,MDBX_BUILD_CXX,$(CXXFLAGS) $(LDFLAGS) $(LIB_STDCXXFS) $(LIBS),$(CFLAGS) $(LDFLAGS) $(LIBS))) check_buildflags_tag: - $(QUIET)if [ "$(MDBX_BUILD_FLAGS)" != "$$(cat buildflags.tag 2>&1)" ]; then \ + $(QUIET)if [ "$(MDBX_BUILD_FLAGS)" != "$$(cat @buildflags.tag 2>&1)" ]; then \ echo -n " CLEAN for build with specified flags..." && \ $(MAKE) IOARENA=false CXXSTD= -s clean >/dev/null && echo " Ok" && \ - echo '$(MDBX_BUILD_FLAGS)' > buildflags.tag; \ + echo '$(MDBX_BUILD_FLAGS)' > @buildflags.tag; \ fi -buildflags.tag: check_buildflags_tag +@buildflags.tag: check_buildflags_tag lib-static libmdbx.a: mdbx-static.o $(call select_by,MDBX_BUILD_CXX,mdbx++-static.o) @echo ' AR $@' @@ -243,32 +266,46 @@ lib-shared libmdbx.$(SO_SUFFIX): mdbx-dylib.o $(call select_by,MDBX_BUILD_CXX,md @echo ' LD $@' $(QUIET)$(call select_by,MDBX_BUILD_CXX,$(CXX) $(CXXFLAGS),$(CC) $(CFLAGS)) $^ -pthread -shared $(LDFLAGS) $(call select_by,MDBX_BUILD_CXX,$(LIB_STDCXXFS)) $(LIBS) -o $@ +ninja-assertions: CMAKE_OPT += -DMDBX_FORCE_ASSERTIONS=ON +ninja-assertions: cmake-build +ninja-debug: CMAKE_OPT += -DCMAKE_BUILD_TYPE=Debug +ninja-debug: cmake-build +ninja: cmake-build +cmake-build: + @echo " RUN: cmake -G Ninja && cmake --build" + $(QUIET)mkdir -p @cmake-ninja-build && $(CMAKE) $(CMAKE_OPT) -G Ninja -S . -B @cmake-ninja-build && $(CMAKE) --build @cmake-ninja-build + +ctest: cmake-build + @echo " RUN: ctest .." + $(QUIET)$(CTEST) --test-dir @cmake-ninja-build --parallel `(nproc | sysctl -n hw.ncpu | echo 2) 2>/dev/null` --schedule-random $(CTEST_OPT) ################################################################################ # Amalgamated source code, i.e. distributed after `make dist` MAN_SRCDIR := man1/ -config.h: buildflags.tag mdbx.c $(lastword $(MAKEFILE_LIST)) +config.h: @buildflags.tag $(WAIT) mdbx.c $(lastword $(MAKEFILE_LIST)) LICENSE NOTICE @echo ' MAKE $@' $(QUIET)(echo '#define MDBX_BUILD_TIMESTAMP "$(MDBX_BUILD_TIMESTAMP)"' \ - && echo "#define MDBX_BUILD_FLAGS \"$$(cat buildflags.tag)\"" \ + && echo "#define MDBX_BUILD_FLAGS \"$$(cat @buildflags.tag)\"" \ && echo '#define MDBX_BUILD_COMPILER "$(shell (LC_ALL=C $(CC) --version || echo 'Please use GCC or CLANG compatible compiler') | head -1)"' \ && echo '#define MDBX_BUILD_TARGET "$(shell set -o pipefail; (LC_ALL=C $(CC) -v 2>&1 | grep -i '^Target:' | cut -d ' ' -f 2- || (LC_ALL=C $(CC) --version | grep -qi e2k && echo E2K) || echo 'Please use GCC or CLANG compatible compiler') | head -1)"' \ + && echo '#define MDBX_BUILD_CXX $(call select_by,MDBX_BUILD_CXX,1,0)' \ + && echo '#define MDBX_BUILD_METADATA "$(MDBX_BUILD_METADATA)"' \ ) >$@ -mdbx-dylib.o: config.h mdbx.c mdbx.h $(lastword $(MAKEFILE_LIST)) +mdbx-dylib.o: config.h mdbx.c mdbx.h $(lastword $(MAKEFILE_LIST)) LICENSE NOTICE @echo ' CC $@' $(QUIET)$(CC) $(CFLAGS) $(MDBX_BUILD_OPTIONS) '-DMDBX_CONFIG_H="config.h"' -DLIBMDBX_EXPORTS=1 -c mdbx.c -o $@ -mdbx-static.o: config.h mdbx.c mdbx.h $(lastword $(MAKEFILE_LIST)) +mdbx-static.o: config.h mdbx.c mdbx.h $(lastword $(MAKEFILE_LIST)) LICENSE NOTICE @echo ' CC $@' $(QUIET)$(CC) $(CFLAGS) $(MDBX_BUILD_OPTIONS) '-DMDBX_CONFIG_H="config.h"' -ULIBMDBX_EXPORTS -c mdbx.c -o $@ -mdbx++-dylib.o: config.h mdbx.c++ mdbx.h mdbx.h++ $(lastword $(MAKEFILE_LIST)) +mdbx++-dylib.o: config.h mdbx.c++ mdbx.h mdbx.h++ $(lastword $(MAKEFILE_LIST)) LICENSE NOTICE @echo ' CC $@' $(QUIET)$(CXX) $(CXXFLAGS) $(MDBX_BUILD_OPTIONS) '-DMDBX_CONFIG_H="config.h"' -DLIBMDBX_EXPORTS=1 -c mdbx.c++ -o $@ -mdbx++-static.o: config.h mdbx.c++ mdbx.h mdbx.h++ $(lastword $(MAKEFILE_LIST)) +mdbx++-static.o: config.h mdbx.c++ mdbx.h mdbx.h++ $(lastword $(MAKEFILE_LIST)) LICENSE NOTICE @echo ' CC $@' $(QUIET)$(CXX) $(CXXFLAGS) $(MDBX_BUILD_OPTIONS) '-DMDBX_CONFIG_H="config.h"' -ULIBMDBX_EXPORTS -c mdbx.c++ -o $@ @@ -285,11 +322,10 @@ mdbx_%.static-lto: mdbx_%.c config.h mdbx.c mdbx.h $(QUIET)$(CC) $(CFLAGS) -Os -flto $(MDBX_BUILD_OPTIONS) '-DLIBMDBX_API=' '-DMDBX_CONFIG_H="config.h"' \ $< mdbx.c $(EXE_LDFLAGS) $(LIBS) -static -Wl,--strip-all -o $@ - -install: $(LIBRARIES) $(TOOLS) $(HEADERS) +install: $(LIBRARIES) $(MDBX_TOOLS) $(HEADERS) @echo ' INSTALLING...' $(QUIET)mkdir -p $(DESTDIR)$(prefix)/bin$(suffix) && \ - $(INSTALL) -p $(EXE_INSTALL_FLAGS) $(TOOLS) $(DESTDIR)$(prefix)/bin$(suffix)/ && \ + $(INSTALL) -p $(EXE_INSTALL_FLAGS) $(MDBX_TOOLS) $(DESTDIR)$(prefix)/bin$(suffix)/ && \ mkdir -p $(DESTDIR)$(prefix)/lib$(suffix)/ && \ $(INSTALL) -p $(EXE_INSTALL_FLAGS) $(filter-out libmdbx.a,$(LIBRARIES)) $(DESTDIR)$(prefix)/lib$(suffix)/ && \ mkdir -p $(DESTDIR)$(prefix)/lib$(suffix)/ && \ @@ -307,7 +343,7 @@ install-no-strip: install uninstall: @echo ' UNINSTALLING/REMOVE...' - $(QUIET)rm -f $(addprefix $(DESTDIR)$(prefix)/bin$(suffix)/,$(TOOLS)) \ + $(QUIET)rm -f $(addprefix $(DESTDIR)$(prefix)/bin$(suffix)/,$(MDBX_TOOLS)) \ $(addprefix $(DESTDIR)$(prefix)/lib$(suffix)/,$(LIBRARIES)) \ $(addprefix $(DESTDIR)$(prefix)/include/,$(HEADERS)) \ $(addprefix $(DESTDIR)$(mandir)/man1/,$(MANPAGES)) @@ -347,18 +383,17 @@ bench-$(1)_$(2).txt: $(3) $(IOARENA) $(lastword $(MAKEFILE_LIST)) $(QUIET)(export LD_LIBRARY_PATH="./:$$$${LD_LIBRARY_PATH}"; \ ldd $(IOARENA) | grep -i $(1) && \ $(IOARENA) -D $(1) -B batch -m $(BENCH_CRUD_MODE) -n $(2) \ - | tee $$@ | grep throughput | sed 's/throughput/batch×N/' && \ + | tee $$@ | grep throughput | $(SED) 's/throughput/batch×N/' && \ $(IOARENA) -D $(1) -B crud -m $(BENCH_CRUD_MODE) -n $(2) \ - | tee -a $$@ | grep throughput | sed 's/throughput/ crud/' && \ + | tee -a $$@ | grep throughput | $(SED) 's/throughput/ crud/' && \ $(IOARENA) -D $(1) -B iterate,get,iterate,get,iterate -m $(BENCH_CRUD_MODE) -r 4 -n $(2) \ - | tee -a $$@ | grep throughput | sed '0,/throughput/{s/throughput/iterate/};s/throughput/ get/' && \ + | tee -a $$@ | grep throughput | $(SED) '0,/throughput/{s/throughput/iterate/};s/throughput/ get/' && \ $(IOARENA) -D $(1) -B delete -m $(BENCH_CRUD_MODE) -n $(2) \ - | tee -a $$@ | grep throughput | sed 's/throughput/ delete/' && \ + | tee -a $$@ | grep throughput | $(SED) 's/throughput/ delete/' && \ true) || mv -f $$@ $$@.error endef - $(eval $(call bench-rule,mdbx,$(NN),libmdbx.$(SO_SUFFIX))) $(eval $(call bench-rule,sophia,$(NN))) diff --git a/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/LICENSE b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/LICENSE index 05ad7571e44..f433b1a53f5 100644 --- a/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/LICENSE +++ b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/LICENSE @@ -1,47 +1,177 @@ -The OpenLDAP Public License - Version 2.8, 17 August 2003 - -Redistribution and use of this software and associated documentation -("Software"), with or without modification, are permitted provided -that the following conditions are met: - -1. Redistributions in source form must retain copyright statements - and notices, - -2. Redistributions in binary form must reproduce applicable copyright - statements and notices, this list of conditions, and the following - disclaimer in the documentation and/or other materials provided - with the distribution, and - -3. Redistributions must contain a verbatim copy of this document. - -The OpenLDAP Foundation may revise this license from time to time. -Each revision is distinguished by a version number. You may use -this Software under terms of this license revision or under the -terms of any subsequent revision of the license. - -THIS SOFTWARE IS PROVIDED BY THE OPENLDAP FOUNDATION AND ITS -CONTRIBUTORS ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, -INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY -AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT -SHALL THE OPENLDAP FOUNDATION, ITS CONTRIBUTORS, OR THE AUTHOR(S) -OR OWNER(S) OF THE SOFTWARE BE LIABLE FOR ANY DIRECT, INDIRECT, -INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. - -The names of the authors and copyright holders must not be used in -advertising or otherwise to promote the sale, use or other dealing -in this Software without specific, written prior permission. Title -to copyright in this Software shall at all times remain with copyright -holders. - -OpenLDAP is a registered trademark of the OpenLDAP Foundation. - -Copyright 1999-2003 The OpenLDAP Foundation, Redwood City, -California, USA. All Rights Reserved. Permission to copy and -distribute verbatim copies of this document is granted. + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/Makefile b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/Makefile index 599e4787418..8a176ceb101 100644 --- a/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/Makefile +++ b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/Makefile @@ -1,14 +1,15 @@ # This is thunk-Makefile for calling GNU Make 3.80 or above -all help options \ +all help options cmake-build ninja \ clean install install-no-strip install-strip strip tools uninstall \ bench bench-clean bench-couple bench-quartet bench-triplet re-bench \ lib libs lib-static lib-shared tools-static \ libmdbx mdbx mdbx_chk mdbx_copy mdbx_drop mdbx_dump mdbx_load mdbx_stat \ check dist memcheck cross-gcc cross-qemu doxygen gcc-analyzer reformat \ -release-assets tags test build-test mdbx_test smoke smoke-fault smoke-singleprocess \ -smoke-assertion test-assertion long-test-assertion \ -test-asan test-leak test-singleprocess test-ubsan test-valgrind: +release-assets tags build-test mdbx_test \ +smoke smoke-fault smoke-singleprocess smoke-assertion smoke-memcheck \ +test test-assertion test-long test-long-assertion test-ci test-ci-extra \ +test-asan test-leak test-singleprocess test-ubsan test-memcheck: @CC=$(CC) \ CXX=`if test -n "$(CXX)" && which "$(CXX)" > /dev/null; then echo "$(CXX)"; elif test -n "$(CCC)" && which "$(CCC)" > /dev/null; then echo "$(CCC)"; else echo "c++"; fi` \ `which gmake || which gnumake || echo 'echo "GNU Make 3.80 or above is required"; exit 2;'` \ diff --git a/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/NOTICE b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/NOTICE new file mode 100644 index 00000000000..dd58a0b540d --- /dev/null +++ b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/NOTICE @@ -0,0 +1,39 @@ +libmdbx (aka MDBX) is an extremely fast, compact, powerful, embeddedable, +transactional key-value storage engine with open-source code. MDBX has a +specific set of properties and capabilities, focused on creating unique +lightweight solutions. + +Please visit https://libmdbx.dqdkfa.ru for more information, changelog, +documentation, C++ API description and links to the original git repo +with the source code. Questions, feedback and suggestions are welcome +to the Telegram' group https://t.me/libmdbx. + +Donations are welcome to the Ethereum/ERC-20 `0xD104d8f8B2dC312aaD74899F83EBf3EEBDC1EA3A`. +Всё будет хорошо! + +Copyright 2015-2025 Леонид Юрьев aka Leonid Yuriev +SPDX-License-Identifier: Apache-2.0 +For notes about the license change, credits and acknowledgments, +please refer to the COPYRIGHT file within libmdbx source. + +--- + +On 2022-04-15, without any warnings or following explanations, the +Github administration deleted _libmdbx_, my account and all other +projects (status 404). A few months later, without any involvement or +notification from/to me, the projects were restored/opened in the "public +read-only archive" status from some kind of incomplete backup. I regard +these actions of Github as malicious sabotage, and I consider the Github +service itself to have lost trust forever. + +As a result of what has happened, I will never, under any circumstances, +post the primary sources (aka origins) of my projects on Github, or rely +in any way on the Github infrastructure. + +Nevertheless, realizing that it is more convenient for users of +_libmdbx_ and other my projects to access ones on Github, I do not want +to restrict their freedom or create inconvenience, and therefore I place +mirrors (aka mirrors) of such repositories on Github since 2025. At the +same time, I would like to emphasize once again that these are only +mirrors that can be frozen, blocked or deleted at any time, as was the +case in 2022. diff --git a/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/VERSION.json b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/VERSION.json new file mode 100644 index 00000000000..534d22e15c6 --- /dev/null +++ b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/VERSION.json @@ -0,0 +1 @@ +{ "git_describe": "v0.13.7-0-g566b0f93", "git_timestamp": "2025-07-30T11:44:04+03:00", "git_tree": "7777cbdf5aa4c1ce85ff902a4c3e6170edd42495", "git_commit": "566b0f93c7c9a3bdffb8fb3dc0ce8ca42641bd72", "semver": "0.13.7" } diff --git a/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/VERSION.txt b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/VERSION.txt deleted file mode 100644 index 6dda1b89047..00000000000 --- a/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/VERSION.txt +++ /dev/null @@ -1 +0,0 @@ -0.12.13.0 diff --git a/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/cmake/compiler.cmake b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/cmake/compiler.cmake index 73cd35028fe..a3d789ce9af 100644 --- a/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/cmake/compiler.cmake +++ b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/cmake/compiler.cmake @@ -1,17 +1,5 @@ -## Copyright (c) 2012-2024 Leonid Yuriev . -## -## Licensed under the Apache License, Version 2.0 (the "License"); -## you may not use this file except in compliance with the License. -## You may obtain a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -## See the License for the specific language governing permissions and -## limitations under the License. -## +# Copyright (c) 2010-2025 Леонид Юрьев aka Leonid Yuriev ############################################### +# SPDX-License-Identifier: Apache-2.0 if(CMAKE_VERSION VERSION_LESS 3.8.2) cmake_minimum_required(VERSION 3.0.2) @@ -43,9 +31,11 @@ if(NOT CMAKE_VERSION VERSION_LESS 3.9) cmake_policy(SET CMP0069 NEW) endif() +cmake_policy(SET CMP0054 NEW) + if(CMAKE_VERSION MATCHES ".*MSVC.*" AND CMAKE_VERSION VERSION_LESS 3.16) message(FATAL_ERROR "CMake from MSVC kit is unfit! " - "Please use MSVC2019 with modern CMake the original CMake from https://cmake.org/download/") + "Please use MSVC-2019 with modern CMake the original CMake from https://cmake.org/download/") endif() if(NOT (CMAKE_C_COMPILER_LOADED OR CMAKE_CXX_COMPILER_LOADED)) @@ -67,11 +57,11 @@ include(CheckLibraryExists) include(CheckIncludeFiles) # Check if the same compile family is used for both C and CXX -if(CMAKE_C_COMPILER_LOADED AND CMAKE_CXX_COMPILER_LOADED AND - NOT (CMAKE_C_COMPILER_ID STREQUAL CMAKE_CXX_COMPILER_ID)) +if(CMAKE_C_COMPILER_LOADED + AND CMAKE_CXX_COMPILER_LOADED + AND NOT (CMAKE_C_COMPILER_ID STREQUAL CMAKE_CXX_COMPILER_ID)) message(WARNING "CMAKE_C_COMPILER_ID (${CMAKE_C_COMPILER_ID}) is different " - "from CMAKE_CXX_COMPILER_ID (${CMAKE_CXX_COMPILER_ID}). " - "The final binary may be unusable.") + "from CMAKE_CXX_COMPILER_ID (${CMAKE_CXX_COMPILER_ID}). " "The final binary may be unusable.") endif() if(CMAKE_CXX_COMPILER_LOADED) @@ -88,27 +78,29 @@ macro(check_compiler_flag flag variable) endif() endmacro(check_compiler_flag) -# We support building with Clang and gcc. First check -# what we're using for build. +# We support building with Clang and gcc. First check what we're using for build. if(CMAKE_C_COMPILER_LOADED AND CMAKE_C_COMPILER_ID MATCHES ".*[Cc][Ll][Aa][Nn][Gg].*") - set(CMAKE_COMPILER_IS_CLANG ON) - set(CMAKE_COMPILER_IS_GNUCC OFF) + set(CMAKE_COMPILER_IS_CLANG ON) + set(CMAKE_COMPILER_IS_GNUCC OFF) endif() if(CMAKE_CXX_COMPILER_LOADED AND CMAKE_CXX_COMPILER_ID MATCHES ".*[Cc][Ll][Aa][Nn][Gg].*") - set(CMAKE_COMPILER_IS_CLANG ON) + set(CMAKE_COMPILER_IS_CLANG ON) set(CMAKE_COMPILER_IS_GNUCXX OFF) endif() if(CMAKE_C_COMPILER_LOADED) # Check for Elbrus lcc - execute_process(COMMAND ${CMAKE_C_COMPILER} --version + execute_process( + COMMAND ${CMAKE_C_COMPILER} --version OUTPUT_VARIABLE tmp_lcc_probe_version - RESULT_VARIABLE tmp_lcc_probe_result ERROR_QUIET) + RESULT_VARIABLE tmp_lcc_probe_result + ERROR_QUIET) if(tmp_lcc_probe_result EQUAL 0) string(FIND "${tmp_lcc_probe_version}" "lcc:" tmp_lcc_marker) string(FIND "${tmp_lcc_probe_version}" ":e2k-" tmp_e2k_marker) if(tmp_lcc_marker GREATER -1 AND tmp_e2k_marker GREATER tmp_lcc_marker) - execute_process(COMMAND ${CMAKE_C_COMPILER} -print-version + execute_process( + COMMAND ${CMAKE_C_COMPILER} -print-version OUTPUT_VARIABLE CMAKE_C_COMPILER_VERSION RESULT_VARIABLE tmp_lcc_probe_result OUTPUT_STRIP_TRAILING_WHITESPACE) @@ -127,14 +119,17 @@ endif() if(CMAKE_CXX_COMPILER_LOADED) # Check for Elbrus l++ - execute_process(COMMAND ${CMAKE_CXX_COMPILER} --version + execute_process( + COMMAND ${CMAKE_CXX_COMPILER} --version OUTPUT_VARIABLE tmp_lxx_probe_version - RESULT_VARIABLE tmp_lxx_probe_result ERROR_QUIET) + RESULT_VARIABLE tmp_lxx_probe_result + ERROR_QUIET) if(tmp_lxx_probe_result EQUAL 0) string(FIND "${tmp_lxx_probe_version}" "lcc:" tmp_lcc_marker) string(FIND "${tmp_lxx_probe_version}" ":e2k-" tmp_e2k_marker) if(tmp_lcc_marker GREATER -1 AND tmp_e2k_marker GREATER tmp_lcc_marker) - execute_process(COMMAND ${CMAKE_CXX_COMPILER} -print-version + execute_process( + COMMAND ${CMAKE_CXX_COMPILER} -print-version OUTPUT_VARIABLE CMAKE_CXX_COMPILER_VERSION RESULT_VARIABLE tmp_lxx_probe_result OUTPUT_STRIP_TRAILING_WHITESPACE) @@ -151,20 +146,17 @@ if(CMAKE_CXX_COMPILER_LOADED) unset(tmp_lxx_probe_result) endif() -# Hard coding the compiler version is ugly from cmake POV, but -# at least gives user a friendly error message. The most critical -# demand for C++ compiler is support of C++11 lambdas, added -# only in version 4.5 https://gcc.gnu.org/projects/cxx0x.html +# Hard coding the compiler version is ugly from cmake POV, but at least gives user a friendly error message. The most +# critical demand for C++ compiler is support of C++11 lambdas, added only in version 4.5 +# https://gcc.gnu.org/projects/cxx0x.html if(CMAKE_COMPILER_IS_GNUCC) - if(CMAKE_C_COMPILER_VERSION VERSION_LESS 4.5 - AND NOT CMAKE_COMPILER_IS_ELBRUSC) + if(CMAKE_C_COMPILER_VERSION VERSION_LESS 4.5 AND NOT CMAKE_COMPILER_IS_ELBRUSC) message(FATAL_ERROR " Your GCC version is ${CMAKE_C_COMPILER_VERSION}, please update") endif() endif() if(CMAKE_COMPILER_IS_GNUCXX) - if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.5 - AND NOT CMAKE_COMPILER_IS_ELBRUSCXX) + if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.5 AND NOT CMAKE_COMPILER_IS_ELBRUSCXX) message(FATAL_ERROR " Your G++ version is ${CMAKE_CXX_COMPILER_VERSION}, please update") endif() @@ -174,7 +166,8 @@ if(CMAKE_CL_64) set(MSVC64 1) endif() if(WIN32 AND CMAKE_COMPILER_IS_GNU${CMAKE_PRIMARY_LANG}) - execute_process(COMMAND ${CMAKE_${CMAKE_PRIMARY_LANG}_COMPILER} -dumpmachine + execute_process( + COMMAND ${CMAKE_${CMAKE_PRIMARY_LANG}_COMPILER} -dumpmachine OUTPUT_VARIABLE __GCC_TARGET_MACHINE OUTPUT_STRIP_TRAILING_WHITESPACE) if(__GCC_TARGET_MACHINE MATCHES "amd64|x86_64|AMD64") @@ -184,9 +177,12 @@ if(WIN32 AND CMAKE_COMPILER_IS_GNU${CMAKE_PRIMARY_LANG}) endif() if(NOT DEFINED IOS) - if(APPLE AND (CMAKE_SYSTEM_NAME STREQUAL "iOS" - OR DEFINED CMAKE_IOS_DEVELOPER_ROOT - OR DEFINED IOS_PLATFORM OR DEFINED IOS_ARCH)) + if(APPLE + AND (CMAKE_SYSTEM_NAME STREQUAL "iOS" + OR DEFINED CMAKE_IOS_DEVELOPER_ROOT + OR DEFINED IOS_PLATFORM + OR DEFINED IOS_ARCH + )) set(IOS TRUE) else() set(IOS FALSE) @@ -194,9 +190,9 @@ if(NOT DEFINED IOS) endif() if(NOT DEFINED CMAKE_TARGET_BITNESS) - if (CMAKE_SIZEOF_VOID_P LESS 4) + if(CMAKE_SIZEOF_VOID_P LESS 4) set(CMAKE_TARGET_BITNESS 16) - elseif (CMAKE_SIZEOF_VOID_P LESS 8) + elseif(CMAKE_SIZEOF_VOID_P LESS 8) set(CMAKE_TARGET_BITNESS 32) else() set(CMAKE_TARGET_BITNESS 64) @@ -237,12 +233,18 @@ if(NOT CMAKE_SYSTEM_ARCH) set(MIPS32 TRUE) endif() endif() - elseif(CMAKE_COMPILER_IS_ELBRUSC OR CMAKE_COMPILER_IS_ELBRUSCXX - OR CMAKE_${CMAKE_PRIMARY_LANG}_COMPILER_ID STREQUAL "LCC" - OR CMAKE_SYSTEM_PROCESSOR MATCHES "e2k.*|E2K.*|elbrus.*|ELBRUS.*") + elseif( + CMAKE_COMPILER_IS_ELBRUSC + OR CMAKE_COMPILER_IS_ELBRUSCXX + OR CMAKE_${CMAKE_PRIMARY_LANG}_COMPILER_ID STREQUAL "LCC" + OR CMAKE_SYSTEM_PROCESSOR MATCHES "e2k.*|E2K.*|elbrus.*|ELBRUS.*") set(E2K TRUE) set(CMAKE_SYSTEM_ARCH "Elbrus") - elseif(MSVC64 OR MINGW64 OR MINGW OR (MSVC AND NOT CMAKE_CROSSCOMPILING)) + elseif( + MSVC64 + OR MINGW64 + OR MINGW + OR (MSVC AND NOT CMAKE_CROSSCOMPILING)) if(CMAKE_TARGET_BITNESS EQUAL 64) set(X86_64 TRUE) set(CMAKE_SYSTEM_ARCH "x86_64") @@ -322,15 +324,19 @@ if(NOT DEFINED CMAKE_HOST_CAN_RUN_EXECUTABLES_BUILT_FOR_TARGET) set(CMAKE_HOST_CAN_RUN_EXECUTABLES_BUILT_FOR_TARGET TRUE) elseif(CMAKE_CROSSCOMPILING AND NOT CMAKE_CROSSCOMPILING_EMULATOR) set(CMAKE_HOST_CAN_RUN_EXECUTABLES_BUILT_FOR_TARGET FALSE) - elseif(CMAKE_SYSTEM_NAME STREQUAL CMAKE_HOST_SYSTEM_NAME - AND ((CMAKE_SYSTEM_PROCESSOR STREQUAL CMAKE_HOST_PROCESSOR) - OR (CMAKE_SYSTEM_ARCH STREQUAL CMAKE_HOST_ARCH) - OR (WIN32 AND CMAKE_HOST_WIN32 AND X86_32 AND CMAKE_HOST_ARCH STREQUAL "x86_64"))) + elseif( + CMAKE_SYSTEM_NAME STREQUAL CMAKE_HOST_SYSTEM_NAME + AND ((CMAKE_SYSTEM_PROCESSOR STREQUAL CMAKE_HOST_PROCESSOR) + OR (CMAKE_SYSTEM_ARCH STREQUAL CMAKE_HOST_ARCH) + OR (WIN32 + AND CMAKE_HOST_WIN32 + AND X86_32 + AND CMAKE_HOST_ARCH STREQUAL "x86_64" + ) + )) set(CMAKE_HOST_CAN_RUN_EXECUTABLES_BUILT_FOR_TARGET TRUE) - message(STATUS - "Assume СAN RUN A BUILT EXECUTABLES," - " since host (${CMAKE_HOST_SYSTEM_NAME}-${CMAKE_HOST_ARCH})" - " match target (${CMAKE_SYSTEM_NAME}-${CMAKE_SYSTEM_ARCH})") + message(STATUS "Assume СAN RUN A BUILT EXECUTABLES," " since host (${CMAKE_HOST_SYSTEM_NAME}-${CMAKE_HOST_ARCH})" + " match target (${CMAKE_SYSTEM_NAME}-${CMAKE_SYSTEM_ARCH})") else() if(CMAKE_C_COMPILER_LOADED) include(CheckCSourceRuns) @@ -352,9 +358,8 @@ if(MSVC) check_compiler_flag("/fsanitize=undefined" CC_HAS_UBSAN) else() # - # GCC started to warn for unused result starting from 4.2, and - # this is when it introduced -Wno-unused-result - # GCC can also be built on top of llvm runtime (on mac). + # GCC started to warn for unused result starting from 4.2, and this is when it introduced -Wno-unused-result GCC can + # also be built on top of llvm runtime (on mac). check_compiler_flag("-Wno-unknown-pragmas" CC_HAS_WNO_UNKNOWN_PRAGMAS) check_compiler_flag("-Wextra" CC_HAS_WEXTRA) check_compiler_flag("-Werror" CC_HAS_WERROR) @@ -379,15 +384,21 @@ else() # Check for an omp support set(CMAKE_REQUIRED_FLAGS "-fopenmp -Werror") if(CMAKE_CXX_COMPILER_LOADED) - check_cxx_source_compiles("int main(void) { - #pragma omp parallel - return 0; - }" HAVE_OPENMP) + check_cxx_source_compiles( + "int main(void) { + #pragma omp for + for(int i = 0, j = 0; i != 42; i = 1 + i * 12345) j += i % 43; + return j; + }" + HAVE_OPENMP) else() - check_c_source_compiles("int main(void) { - #pragma omp parallel - return 0; - }" HAVE_OPENMP) + check_c_source_compiles( + "int main(void) { + #pragma omp for + for(int i = 0, j = 0; i != 42; i = 1 + i * 12345) j += i % 43; + return j; + }" + HAVE_OPENMP) endif() set(CMAKE_REQUIRED_FLAGS "") endif() @@ -396,9 +407,13 @@ endif() if(CMAKE_CXX_COMPILER_LOADED) list(FIND CMAKE_CXX_COMPILE_FEATURES cxx_std_11 HAS_CXX11) if(HAS_CXX11 LESS 0) - check_cxx_compiler_flag("-std=gnu++11" CXX_FALLBACK_GNU11) - if(NOT CXX_FALLBACK_GNU11) - check_cxx_compiler_flag("-std=c++11" CXX_FALLBACK_11) + if(MSVC) + check_cxx_compiler_flag("/std:c++11" CXX_FALLBACK_11) + else() + check_cxx_compiler_flag("-std=gnu++11" CXX_FALLBACK_GNU11) + if(NOT CXX_FALLBACK_GNU11) + check_cxx_compiler_flag("-std=c++11" CXX_FALLBACK_11) + endif() endif() endif() endif() @@ -407,7 +422,7 @@ endif() if(CMAKE_C_COMPILER_LOADED) list(FIND CMAKE_C_COMPILE_FEATURES c_std_11 HAS_C11) if(HAS_C11 LESS 0) - if (MSVC) + if(MSVC) check_c_compiler_flag("/std:c11" C_FALLBACK_11) else() check_c_compiler_flag("-std=gnu11" C_FALLBACK_GNU11) @@ -419,13 +434,17 @@ if(CMAKE_C_COMPILER_LOADED) endif() # Check for LTO support by GCC -if(CMAKE_COMPILER_IS_GNU${CMAKE_PRIMARY_LANG} AND NOT CMAKE_COMPILER_IS_ELBRUSC AND NOT CMAKE_COMPILER_IS_ELBRUSCXX) +if(CMAKE_COMPILER_IS_GNU${CMAKE_PRIMARY_LANG} + AND NOT CMAKE_COMPILER_IS_ELBRUSC + AND NOT CMAKE_COMPILER_IS_ELBRUSCXX) unset(gcc_collect) unset(gcc_lto_wrapper) if(NOT CMAKE_${CMAKE_PRIMARY_LANG}_COMPILER_VERSION VERSION_LESS 4.7) - execute_process(COMMAND ${CMAKE_${CMAKE_PRIMARY_LANG}_COMPILER} -v - OUTPUT_VARIABLE gcc_info_v ERROR_VARIABLE gcc_info_v) + execute_process( + COMMAND ${CMAKE_${CMAKE_PRIMARY_LANG}_COMPILER} -v + OUTPUT_VARIABLE gcc_info_v + ERROR_VARIABLE gcc_info_v) string(REGEX MATCH "^(.+\nCOLLECT_GCC=)([^ \n]+)(\n.+)$" gcc_collect_valid ${gcc_info_v}) if(gcc_collect_valid) @@ -434,7 +453,8 @@ if(CMAKE_COMPILER_IS_GNU${CMAKE_PRIMARY_LANG} AND NOT CMAKE_COMPILER_IS_ELBRUSC string(REGEX MATCH "^(.+\nCOLLECT_LTO_WRAPPER=)([^ \n]+/lto-wrapper)(\n.+)$" gcc_lto_wrapper_valid ${gcc_info_v}) if(gcc_lto_wrapper_valid) - string(REGEX REPLACE "^(.+\nCOLLECT_LTO_WRAPPER=)([^ \n]+/lto-wrapper)(\n.+)$" "\\2" gcc_lto_wrapper ${gcc_info_v}) + string(REGEX REPLACE "^(.+\nCOLLECT_LTO_WRAPPER=)([^ \n]+/lto-wrapper)(\n.+)$" "\\2" gcc_lto_wrapper + ${gcc_info_v}) endif() set(gcc_suffix "") @@ -447,13 +467,25 @@ if(CMAKE_COMPILER_IS_GNU${CMAKE_PRIMARY_LANG} AND NOT CMAKE_COMPILER_IS_ELBRUSC get_filename_component(gcc_dir ${CMAKE_${CMAKE_PRIMARY_LANG}_COMPILER} DIRECTORY) if(NOT CMAKE_GCC_AR) - find_program(CMAKE_GCC_AR NAMES "gcc${gcc_suffix}-ar" "gcc-ar${gcc_suffix}" PATHS "${gcc_dir}" NO_DEFAULT_PATH) + find_program( + CMAKE_GCC_AR + NAMES "gcc${gcc_suffix}-ar" "gcc-ar${gcc_suffix}" + PATHS "${gcc_dir}" + NO_DEFAULT_PATH) endif() if(NOT CMAKE_GCC_NM) - find_program(CMAKE_GCC_NM NAMES "gcc${gcc_suffix}-nm" "gcc-nm${gcc_suffix}" PATHS "${gcc_dir}" NO_DEFAULT_PATH) + find_program( + CMAKE_GCC_NM + NAMES "gcc${gcc_suffix}-nm" "gcc-nm${gcc_suffix}" + PATHS "${gcc_dir}" + NO_DEFAULT_PATH) endif() if(NOT CMAKE_GCC_RANLIB) - find_program(CMAKE_GCC_RANLIB NAMES "gcc${gcc_suffix}-ranlib" "gcc-ranlib${gcc_suffix}" PATHS "${gcc_dir}" NO_DEFAULT_PATH) + find_program( + CMAKE_GCC_RANLIB + NAMES "gcc${gcc_suffix}-ranlib" "gcc-ranlib${gcc_suffix}" + PATHS "${gcc_dir}" + NO_DEFAULT_PATH) endif() unset(gcc_dir) @@ -465,9 +497,16 @@ if(CMAKE_COMPILER_IS_GNU${CMAKE_PRIMARY_LANG} AND NOT CMAKE_COMPILER_IS_ELBRUSC unset(gcc_info_v) endif() - if(CMAKE_GCC_AR AND CMAKE_GCC_NM AND CMAKE_GCC_RANLIB AND gcc_lto_wrapper) + if(CMAKE_GCC_AR + AND CMAKE_GCC_NM + AND CMAKE_GCC_RANLIB + AND gcc_lto_wrapper) message(STATUS "Found GCC's LTO toolset: ${gcc_lto_wrapper}, ${CMAKE_GCC_AR}, ${CMAKE_GCC_RANLIB}") - set(GCC_LTO_CFLAGS "-flto -fno-fat-lto-objects -fuse-linker-plugin") + if(CMAKE_${CMAKE_PRIMARY_LANG}_COMPILER_VERSION VERSION_LESS 11.4) + set(GCC_LTO_CFLAGS "-flto -fno-fat-lto-objects -fuse-linker-plugin") + else() + set(GCC_LTO_CFLAGS "-flto=auto -fno-fat-lto-objects -fuse-linker-plugin") + endif() set(GCC_LTO_AVAILABLE TRUE) message(STATUS "Link-Time Optimization by GCC is available") else() @@ -491,8 +530,11 @@ endif() # Check for LTO support by CLANG if(CMAKE_COMPILER_IS_CLANG) if(NOT CMAKE_${CMAKE_PRIMARY_LANG}_COMPILER_VERSION VERSION_LESS 3.5) - execute_process(COMMAND ${CMAKE_${CMAKE_PRIMARY_LANG}_COMPILER} -print-search-dirs - OUTPUT_VARIABLE clang_search_dirs RESULT_VARIABLE clang_probe_result ERROR_QUIET) + execute_process( + COMMAND ${CMAKE_${CMAKE_PRIMARY_LANG}_COMPILER} -print-search-dirs + OUTPUT_VARIABLE clang_search_dirs + RESULT_VARIABLE clang_probe_result + ERROR_QUIET) unset(clang_bindirs) unset(clang_bindirs_x) @@ -503,13 +545,21 @@ if(CMAKE_COMPILER_IS_CLANG) if(regexp_valid) string(REGEX REPLACE "(^|\n.*)(.*programs: =)([^\n]+)((\n.*)|$)" "\\3" list ${clang_search_dirs}) string(REPLACE ":" ";" list "${list}") + set(libs_extra_subdirs "lib;../lib;lib64;../lib64;lib32;../lib32") foreach(dir IN LISTS list) get_filename_component(dir "${dir}" REALPATH) if(dir MATCHES ".*llvm.*" OR dir MATCHES ".*clang.*") - list(APPEND clang_bindirs "${dir}") + set(list_suffix "") else() - list(APPEND clang_bindirs_x "${dir}") + set(list_suffix "_x") endif() + list(APPEND clang_bindirs${list_suffix} "${dir}") + foreach(subdir IN LISTS libs_extra_subdirs) + get_filename_component(subdir "${dir}/${subdir}" REALPATH) + if(EXISTS "${subdir}") + list(APPEND clang_libdirs${list_suffix} "${subdir}") + endif() + endforeach() endforeach() list(APPEND clang_bindirs "${clang_bindirs_x}") list(REMOVE_DUPLICATES clang_bindirs) @@ -521,10 +571,11 @@ if(CMAKE_COMPILER_IS_CLANG) foreach(dir IN LISTS list) get_filename_component(dir "${dir}" REALPATH) if(dir MATCHES ".*llvm.*" OR dir MATCHES ".*clang.*") - list(APPEND clang_libdirs "${dir}") + set(list_suffix "") else() - list(APPEND clang_libdirs_x "${dir}") + set(list_suffix "_x") endif() + list(APPEND clang_libdirs${list_suffix} "${dir}") endforeach() list(APPEND clang_libdirs "${clang_libdirs_x}") list(REMOVE_DUPLICATES clang_libdirs) @@ -545,24 +596,46 @@ if(CMAKE_COMPILER_IS_CLANG) endif() if(NOT CMAKE_CLANG_LD AND clang_bindirs) - find_program(CMAKE_CLANG_LD NAMES lld-link ld.lld "ld${CMAKE_TARGET_BITNESS}.lld" lld llvm-link llvm-ld PATHS ${clang_bindirs} NO_DEFAULT_PATH) + find_program( + CMAKE_CLANG_LD + NAMES lld-link ld.lld "ld${CMAKE_TARGET_BITNESS}.lld" lld llvm-link llvm-ld + PATHS ${clang_bindirs} + NO_DEFAULT_PATH) endif() if(NOT CMAKE_CLANG_AR AND clang_bindirs) - find_program(CMAKE_CLANG_AR NAMES llvm-ar ar PATHS ${clang_bindirs} NO_DEFAULT_PATH) + find_program( + CMAKE_CLANG_AR + NAMES llvm-ar ar + PATHS ${clang_bindirs} + NO_DEFAULT_PATH) endif() if(NOT CMAKE_CLANG_NM AND clang_bindirs) - find_program(CMAKE_CLANG_NM NAMES llvm-nm nm PATHS ${clang_bindirs} NO_DEFAULT_PATH) + find_program( + CMAKE_CLANG_NM + NAMES llvm-nm nm + PATHS ${clang_bindirs} + NO_DEFAULT_PATH) endif() if(NOT CMAKE_CLANG_RANLIB AND clang_bindirs) - find_program(CMAKE_CLANG_RANLIB NAMES llvm-ranlib ranlib PATHS ${clang_bindirs} NO_DEFAULT_PATH) + find_program( + CMAKE_CLANG_RANLIB + NAMES llvm-ranlib ranlib + PATHS ${clang_bindirs} + NO_DEFAULT_PATH) endif() set(clang_lto_plugin_name "LLVMgold${CMAKE_SHARED_LIBRARY_SUFFIX}") if(NOT CMAKE_LD_GOLD AND clang_bindirs) - find_program(CMAKE_LD_GOLD NAMES ld.gold PATHS ${clang_bindirs}) + find_program( + CMAKE_LD_GOLD + NAMES ld.gold + PATHS ${clang_bindirs}) endif() if(NOT CLANG_LTO_PLUGIN AND clang_libdirs) - find_file(CLANG_LTO_PLUGIN ${clang_lto_plugin_name} PATHS ${clang_libdirs} NO_DEFAULT_PATH) + find_file( + CLANG_LTO_PLUGIN ${clang_lto_plugin_name} + PATHS ${clang_libdirs} + NO_DEFAULT_PATH) endif() if(CLANG_LTO_PLUGIN) @@ -577,7 +650,9 @@ if(CMAKE_COMPILER_IS_CLANG) message(STATUS "Could NOT find CLANG/LLVM's linker (lld, llvm-ld, llvm-link) for LTO.") endif() - if(CMAKE_CLANG_AR AND CMAKE_CLANG_RANLIB AND CMAKE_CLANG_NM) + if(CMAKE_CLANG_AR + AND CMAKE_CLANG_RANLIB + AND CMAKE_CLANG_NM) message(STATUS "Found CLANG/LLVM's binutils for LTO: ${CMAKE_CLANG_AR}, ${CMAKE_CLANG_RANLIB}, ${CMAKE_CLANG_NM}") else() message(STATUS "Could NOT find CLANG/LLVM's binutils (ar, ranlib, nm) for LTO.") @@ -590,15 +665,16 @@ if(CMAKE_COMPILER_IS_CLANG) unset(clang_search_dirs) endif() - if(CMAKE_CLANG_AR AND CMAKE_CLANG_NM AND CMAKE_CLANG_RANLIB - AND ((CLANG_LTO_PLUGIN AND CMAKE_LD_GOLD) - OR (CMAKE_CLANG_LD - AND NOT (CMAKE_HOST_SYSTEM_NAME STREQUAL "Linux" - AND CMAKE_SYSTEM_NAME STREQUAL "Linux")) - OR APPLE)) + if(CMAKE_CLANG_AR + AND CMAKE_CLANG_NM + AND CMAKE_CLANG_RANLIB + AND ((CLANG_LTO_PLUGIN AND CMAKE_LD_GOLD) + OR CMAKE_CLANG_LD + OR APPLE)) if(ANDROID AND CMAKE_${CMAKE_PRIMARY_LANG}_COMPILER_VERSION VERSION_LESS 12) set(CLANG_LTO_AVAILABLE FALSE) - message(STATUS "Link-Time Optimization by CLANG/LLVM is available but unusable due https://reviews.llvm.org/D79919") + message( + STATUS "Link-Time Optimization by CLANG/LLVM is available but unusable due https://reviews.llvm.org/D79919") else() set(CLANG_LTO_AVAILABLE TRUE) message(STATUS "Link-Time Optimization by CLANG/LLVM is available") @@ -625,17 +701,22 @@ if(CMAKE_COMPILER_IS_CLANG) endif() # Perform build type specific configuration. -option(ENABLE_BACKTRACE "Enable output of fiber backtrace information in 'show +option( + ENABLE_BACKTRACE + "Enable output of fiber backtrace information in 'show fiber' administrative command. Only works on x86 architectures, if compiled with gcc. If GNU binutils and binutils-dev libraries are installed, backtrace is output with resolved function (symbol) names. Otherwise only frame - addresses are printed." OFF) + addresses are printed." + OFF) set(HAVE_BFD FALSE) if(ENABLE_BACKTRACE) if(NOT (X86_32 OR X86_64) OR NOT CMAKE_COMPILER_IS_GNU${CMAKE_PRIMARY_LANG}) # We only know this option to work with gcc - message(FATAL_ERROR "ENABLE_BACKTRACE option is set but the system + message( + FATAL_ERROR + "ENABLE_BACKTRACE option is set but the system is not x86 based (${CMAKE_SYSTEM_PROCESSOR}) or the compiler is not GNU GCC (${CMAKE_${CMAKE_PRIMARY_LANG}_COMPILER}).") endif() @@ -652,11 +733,13 @@ if(ENABLE_BACKTRACE) check_include_files(bfd.h HAVE_BFD_H) set(CMAKE_REQUIRED_DEFINITIONS) find_package(ZLIB) - if(HAVE_BFD_LIB AND HAVE_BFD_H AND HAVE_IBERTY_LIB AND ZLIB_FOUND) + if(HAVE_BFD_LIB + AND HAVE_BFD_H + AND HAVE_IBERTY_LIB + AND ZLIB_FOUND) set(HAVE_BFD ON) set(BFD_LIBRARIES ${BFD_LIBRARY} ${IBERTY_LIBRARY} ${ZLIB_LIBRARIES}) - find_package_message(BFD_LIBRARIES "Found libbfd and dependencies" - ${BFD_LIBRARIES}) + find_package_message(BFD_LIBRARIES "Found libbfd and dependencies" ${BFD_LIBRARIES}) if(TARGET_OS_FREEBSD AND NOT TARGET_OS_DEBIAN_FREEBSD) set(BFD_LIBRARIES ${BFD_LIBRARIES} iconv) endif() @@ -667,16 +750,30 @@ macro(setup_compile_flags) # save initial C/CXX flags if(NOT INITIAL_CMAKE_FLAGS_SAVED) if(CMAKE_CXX_COMPILER_LOADED) - set(INITIAL_CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS} CACHE STRING "Initial CMake's flags" FORCE) + set(INITIAL_CMAKE_CXX_FLAGS + ${CMAKE_CXX_FLAGS} + CACHE STRING "Initial CMake's flags" FORCE) endif() if(CMAKE_C_COMPILER_LOADED) - set(INITIAL_CMAKE_C_FLAGS ${CMAKE_C_FLAGS} CACHE STRING "Initial CMake's flags" FORCE) + set(INITIAL_CMAKE_C_FLAGS + ${CMAKE_C_FLAGS} + CACHE STRING "Initial CMake's flags" FORCE) endif() - set(INITIAL_CMAKE_EXE_LINKER_FLAGS ${CMAKE_EXE_LINKER_FLAGS} CACHE STRING "Initial CMake's flags" FORCE) - set(INITIAL_CMAKE_SHARED_LINKER_FLAGS ${CMAKE_SHARED_LINKER_FLAGS} CACHE STRING "Initial CMake's flags" FORCE) - set(INITIAL_CMAKE_STATIC_LINKER_FLAGS ${CMAKE_STATIC_LINKER_FLAGS} CACHE STRING "Initial CMake's flags" FORCE) - set(INITIAL_CMAKE_MODULE_LINKER_FLAGS ${CMAKE_MODULE_LINKER_FLAGS} CACHE STRING "Initial CMake's flags" FORCE) - set(INITIAL_CMAKE_FLAGS_SAVED TRUE CACHE INTERNAL "State of initial CMake's flags" FORCE) + set(INITIAL_CMAKE_EXE_LINKER_FLAGS + ${CMAKE_EXE_LINKER_FLAGS} + CACHE STRING "Initial CMake's flags" FORCE) + set(INITIAL_CMAKE_SHARED_LINKER_FLAGS + ${CMAKE_SHARED_LINKER_FLAGS} + CACHE STRING "Initial CMake's flags" FORCE) + set(INITIAL_CMAKE_STATIC_LINKER_FLAGS + ${CMAKE_STATIC_LINKER_FLAGS} + CACHE STRING "Initial CMake's flags" FORCE) + set(INITIAL_CMAKE_MODULE_LINKER_FLAGS + ${CMAKE_MODULE_LINKER_FLAGS} + CACHE STRING "Initial CMake's flags" FORCE) + set(INITIAL_CMAKE_FLAGS_SAVED + TRUE + CACHE INTERNAL "State of initial CMake's flags" FORCE) endif() # reset C/CXX flags @@ -717,14 +814,13 @@ macro(setup_compile_flags) add_compile_flags("C;CXX" "-fno-semantic-interposition") endif() if(MSVC) - # checks for /EHa or /clr options exists, - # i.e. is enabled structured async WinNT exceptions + # checks for /EHa or /clr options exists, i.e. is enabled structured async WinNT exceptions string(REGEX MATCH "^(.* )*[-/]EHc*a( .*)*$" msvc_async_eh_enabled "${CXX_FLAGS}" "${C_FLAGS}") string(REGEX MATCH "^(.* )*[-/]clr( .*)*$" msvc_clr_enabled "${CXX_FLAGS}" "${C_FLAGS}") # remote any /EH? options string(REGEX REPLACE "( *[-/]-*EH[csa]+ *)+" "" CXX_FLAGS "${CXX_FLAGS}") string(REGEX REPLACE "( *[-/]-*EH[csa]+ *)+" "" C_FLAGS "${C_FLAGS}") - if (msvc_clr_enabled STREQUAL "") + if(msvc_clr_enabled STREQUAL "") if(NOT msvc_async_eh_enabled STREQUAL "") add_compile_flags("C;CXX" "/EHa") else() @@ -733,8 +829,9 @@ macro(setup_compile_flags) endif() endif(MSVC) - if(CC_HAS_WNO_ATTRIBUTES AND CMAKE_COMPILER_IS_GNU${CMAKE_PRIMARY_LANG} - AND CMAKE_${CMAKE_PRIMARY_LANG}_COMPILER_VERSION VERSION_LESS 9) + if(CC_HAS_WNO_ATTRIBUTES + AND CMAKE_COMPILER_IS_GNU${CMAKE_PRIMARY_LANG} + AND CMAKE_${CMAKE_PRIMARY_LANG}_COMPILER_VERSION VERSION_LESS 9) # GCC < 9.x generates false-positive warnings for optimization attributes add_compile_flags("C;CXX" "-Wno-attributes") if(LTO_ENABLED) @@ -742,22 +839,17 @@ macro(setup_compile_flags) endif() endif() - # In C a global variable without a storage specifier (static/extern) and - # without an initialiser is called a ’tentative definition’. The - # language permits multiple tentative definitions in the single - # translation unit; i.e. int foo; int foo; is perfectly ok. GNU - # toolchain goes even further, allowing multiple tentative definitions - # in *different* translation units. Internally, variables introduced via - # tentative definitions are implemented as ‘common’ symbols. Linker - # permits multiple definitions if they are common symbols, and it picks - # one arbitrarily for inclusion in the binary being linked. + # In C a global variable without a storage specifier (static/extern) and without an initialiser is called a ’tentative + # definition’. The language permits multiple tentative definitions in the single translation unit; i.e. int foo; int + # foo; is perfectly ok. GNU toolchain goes even further, allowing multiple tentative definitions in *different* + # translation units. Internally, variables introduced via tentative definitions are implemented as ‘common’ symbols. + # Linker permits multiple definitions if they are common symbols, and it picks one arbitrarily for inclusion in the + # binary being linked. # - # -fno-common forces GNU toolchain to behave in a more - # standard-conformant way in respect to tentative definitions and it - # prevents common symbols generation. Since we are a cross-platform - # project it really makes sense. There are toolchains that don’t - # implement GNU style handling of the tentative definitions and there - # are platforms lacking proper support for common symbols (osx). + # -fno-common forces GNU toolchain to behave in a more standard-conformant way in respect to tentative definitions and + # it prevents common symbols generation. Since we are a cross-platform project it really makes sense. There are + # toolchains that don’t implement GNU style handling of the tentative definitions and there are platforms lacking + # proper support for common symbols (osx). if(CC_HAS_FNO_COMMON) add_compile_flags("C;CXX" "-fno-common") endif() @@ -776,10 +868,8 @@ macro(setup_compile_flags) add_compile_flags("C;CXX" "/Gy") endif() - # We must set -fno-omit-frame-pointer here, since we rely - # on frame pointer when getting a backtrace, and it must - # be used consistently across all object files. - # The same reasoning applies to -fno-stack-protector switch. + # We must set -fno-omit-frame-pointer here, since we rely on frame pointer when getting a backtrace, and it must be + # used consistently across all object files. The same reasoning applies to -fno-stack-protector switch. if(ENABLE_BACKTRACE) if(CC_HAS_FNO_OMIT_FRAME_POINTER) add_compile_flags("C;CXX" "-fno-omit-frame-pointer") @@ -788,7 +878,9 @@ macro(setup_compile_flags) if(MSVC) if(MSVC_VERSION LESS 1900) - message(FATAL_ERROR "At least \"Microsoft C/C++ Compiler\" version 19.0.24234.1 (Visual Studio 2015 Update 3) is required.") + message( + FATAL_ERROR + "At least \"Microsoft C/C++ Compiler\" version 19.0.24234.1 (Visual Studio 2015 Update 3) is required.") endif() if(NOT MSVC_VERSION LESS 1910) add_compile_flags("CXX" "/Zc:__cplusplus") @@ -809,9 +901,11 @@ macro(setup_compile_flags) add_definitions("-D__STDC_CONSTANT_MACROS=1") add_definitions("-D_HAS_EXCEPTIONS=1") - # Only add -Werror if it's a debug build, done by developers. - # Release builds should not cause extra trouble. - if(CC_HAS_WERROR AND (CI OR CMAKE_CONFIGURATION_TYPES OR CMAKE_BUILD_TYPE STREQUAL "Debug")) + # Only add -Werror if it's a debug build, done by developers. Release builds should not cause extra trouble. + if(CC_HAS_WERROR + AND (CI + OR CMAKE_CONFIGURATION_TYPES + OR CMAKE_BUILD_TYPE STREQUAL "Debug")) if(MSVC) add_compile_flags("C;CXX" "/WX") elseif(CMAKE_COMPILER_IS_CLANG) @@ -827,17 +921,15 @@ macro(setup_compile_flags) endif() endif() - - if(CMAKE_COMPILER_IS_GNU${CMAKE_PRIMARY_LANG} - AND CMAKE_${CMAKE_PRIMARY_LANG}_COMPILER_VERSION VERSION_LESS 5) + if(CMAKE_COMPILER_IS_GNU${CMAKE_PRIMARY_LANG} AND CMAKE_${CMAKE_PRIMARY_LANG}_COMPILER_VERSION VERSION_LESS 5) # G++ bug. http://gcc.gnu.org/bugzilla/show_bug.cgi?id=31488 add_compile_flags("CXX" "-Wno-invalid-offsetof") endif() if(MINGW) - # Disable junk MINGW's warnings that issued due to incompatibilities - # and shortcomings of MINGW, - # since the code is checked by builds with GCC, CLANG and MSVC. - add_compile_flags("C;CXX" "-Wno-format-extra-args" "-Wno-format" "-Wno-cast-function-type" "-Wno-implicit-fallthrough") + # Disable junk MINGW's warnings that issued due to incompatibilities and shortcomings of MINGW, since the code is + # checked by builds with GCC, CLANG and MSVC. + add_compile_flags("C;CXX" "-Wno-format-extra-args" "-Wno-format" "-Wno-cast-function-type" + "-Wno-implicit-fallthrough") endif() if(ENABLE_ASAN) @@ -891,32 +983,46 @@ macro(setup_compile_flags) endif() endif() - if(MSVC AND NOT CMAKE_COMPILER_IS_CLANG AND LTO_ENABLED) + if(MSVC + AND NOT CMAKE_COMPILER_IS_CLANG + AND LTO_ENABLED) add_compile_flags("C;CXX" "/GL") foreach(linkmode IN ITEMS EXE SHARED STATIC MODULE) set(${linkmode}_LINKER_FLAGS "${${linkmode}_LINKER_FLAGS} /LTCG") - string(REGEX REPLACE "^(.*)(/INCREMENTAL)(:YES)?(:NO)?( ?.*)$" "\\1\\2:NO\\5" ${linkmode}_LINKER_FLAGS "${${linkmode}_LINKER_FLAGS}") + string(REGEX REPLACE "^(.*)(/INCREMENTAL)(:YES)?(:NO)?( ?.*)$" "\\1\\2:NO\\5" ${linkmode}_LINKER_FLAGS + "${${linkmode}_LINKER_FLAGS}") string(STRIP "${${linkmode}_LINKER_FLAGS}" ${linkmode}_LINKER_FLAGS) - foreach(config IN LISTS CMAKE_CONFIGURATION_TYPES ITEMS Release MinSizeRel RelWithDebInfo Debug) + foreach( + config IN + LISTS CMAKE_CONFIGURATION_TYPES + ITEMS Release MinSizeRel RelWithDebInfo Debug) string(TOUPPER "${config}" config_uppercase) if(DEFINED "CMAKE_${linkmode}_LINKER_FLAGS_${config_uppercase}") - string(REGEX REPLACE "^(.*)(/INCREMENTAL)(:YES)?(:NO)?( ?.*)$" "\\1\\2:NO\\5" altered_flags "${CMAKE_${linkmode}_LINKER_FLAGS_${config_uppercase}}") + string(REGEX REPLACE "^(.*)(/INCREMENTAL)(:YES)?(:NO)?( ?.*)$" "\\1\\2:NO\\5" altered_flags + "${CMAKE_${linkmode}_LINKER_FLAGS_${config_uppercase}}") string(STRIP "${altered_flags}" altered_flags) if(NOT "${altered_flags}" STREQUAL "${CMAKE_${linkmode}_LINKER_FLAGS_${config_uppercase}}") - set(CMAKE_${linkmode}_LINKER_FLAGS_${config_uppercase} "${altered_flags}" CACHE STRING "Altered: '/INCREMENTAL' removed for LTO" FORCE) + set(CMAKE_${linkmode}_LINKER_FLAGS_${config_uppercase} + "${altered_flags}" + CACHE STRING "Altered: '/INCREMENTAL' removed for LTO" FORCE) endif() endif() endforeach(config) endforeach(linkmode) unset(linkmode) - foreach(config IN LISTS CMAKE_CONFIGURATION_TYPES ITEMS Release MinSizeRel RelWithDebInfo) + foreach( + config IN + LISTS CMAKE_CONFIGURATION_TYPES + ITEMS Release MinSizeRel RelWithDebInfo) foreach(lang IN ITEMS C CXX) string(TOUPPER "${config}" config_uppercase) if(DEFINED "CMAKE_${lang}_FLAGS_${config_uppercase}") string(REPLACE "/O2" "/Ox" altered_flags "${CMAKE_${lang}_FLAGS_${config_uppercase}}") if(NOT "${altered_flags}" STREQUAL "${CMAKE_${lang}_FLAGS_${config_uppercase}}") - set(CMAKE_${lang}_FLAGS_${config_uppercase} "${altered_flags}" CACHE STRING "Altered: '/O2' replaced by '/Ox' for LTO" FORCE) + set(CMAKE_${lang}_FLAGS_${config_uppercase} + "${altered_flags}" + CACHE STRING "Altered: '/O2' replaced by '/Ox' for LTO" FORCE) endif() endif() unset(config_uppercase) @@ -949,17 +1055,29 @@ macro(setup_compile_flags) # push C/CXX flags into the cache if(CMAKE_CXX_COMPILER_LOADED) - set(CMAKE_CXX_FLAGS ${CXX_FLAGS} CACHE STRING "Flags used by the C++ compiler during all build types" FORCE) + set(CMAKE_CXX_FLAGS + ${CXX_FLAGS} + CACHE STRING "Flags used by the C++ compiler during all build types" FORCE) unset(CXX_FLAGS) endif() if(CMAKE_C_COMPILER_LOADED) - set(CMAKE_C_FLAGS ${C_FLAGS} CACHE STRING "Flags used by the C compiler during all build types" FORCE) + set(CMAKE_C_FLAGS + ${C_FLAGS} + CACHE STRING "Flags used by the C compiler during all build types" FORCE) unset(C_FLAGS) endif() - set(CMAKE_EXE_LINKER_FLAGS ${EXE_LINKER_FLAGS} CACHE STRING "Flags used by the linker" FORCE) - set(CMAKE_SHARED_LINKER_FLAGS ${SHARED_LINKER_FLAGS} CACHE STRING "Flags used by the linker during the creation of dll's" FORCE) - set(CMAKE_STATIC_LINKER_FLAGS ${STATIC_LINKER_FLAGS} CACHE STRING "Flags used by the linker during the creation of static libraries" FORCE) - set(CMAKE_MODULE_LINKER_FLAGS ${MODULE_LINKER_FLAGS} CACHE STRING "Flags used by the linker during the creation of modules" FORCE) + set(CMAKE_EXE_LINKER_FLAGS + ${EXE_LINKER_FLAGS} + CACHE STRING "Flags used by the linker" FORCE) + set(CMAKE_SHARED_LINKER_FLAGS + ${SHARED_LINKER_FLAGS} + CACHE STRING "Flags used by the linker during the creation of dll's" FORCE) + set(CMAKE_STATIC_LINKER_FLAGS + ${STATIC_LINKER_FLAGS} + CACHE STRING "Flags used by the linker during the creation of static libraries" FORCE) + set(CMAKE_MODULE_LINKER_FLAGS + ${MODULE_LINKER_FLAGS} + CACHE STRING "Flags used by the linker during the creation of modules" FORCE) unset(EXE_LINKER_FLAGS) unset(SHARED_LINKER_FLAGS) unset(STATIC_LINKER_FLAGS) @@ -969,7 +1087,9 @@ endmacro(setup_compile_flags) macro(probe_libcxx_filesystem) if(CMAKE_CXX_COMPILER_LOADED AND NOT DEFINED LIBCXX_FILESYSTEM) list(FIND CMAKE_CXX_COMPILE_FEATURES cxx_std_11 HAS_CXX11) - if(NOT HAS_CXX11 LESS 0 OR CXX_FALLBACK_GNU11 OR CXX_FALLBACK_11) + if(NOT HAS_CXX11 LESS 0 + OR CXX_FALLBACK_GNU11 + OR CXX_FALLBACK_11) include(CMakePushCheckState) include(CheckCXXSourceCompiles) cmake_push_check_state() @@ -981,8 +1101,7 @@ macro(probe_libcxx_filesystem) if(NOT DEFINED CMAKE_CXX_STANDARD) list(FIND CMAKE_CXX_COMPILE_FEATURES cxx_std_14 HAS_CXX14) list(FIND CMAKE_CXX_COMPILE_FEATURES cxx_std_17 HAS_CXX17) - if(NOT HAS_CXX17 LESS 0 - AND NOT (CMAKE_COMPILER_IS_CLANG AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5)) + if(NOT HAS_CXX17 LESS 0 AND NOT (CMAKE_COMPILER_IS_CLANG AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5)) set(CMAKE_CXX_STANDARD 17) elseif(NOT HAS_CXX14 LESS 0) set(CMAKE_CXX_STANDARD 14) @@ -1004,7 +1123,8 @@ macro(probe_libcxx_filesystem) endif() set(CMAKE_REQUIRED_FLAGS ${stdfs_probe_flags}) - set(stdfs_probe_code [[ + set(stdfs_probe_code + [[ #if defined(__SIZEOF_INT128__) && !defined(__GLIBCXX_TYPE_INT_N_0) && defined(__clang__) && __clang_major__ < 4 #define __GLIBCXX_BITSIZE_INT_N_0 128 #define __GLIBCXX_TYPE_INT_N_0 __int128 diff --git a/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/cmake/profile.cmake b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/cmake/profile.cmake index cf6bf87b47e..6a8a466ef34 100644 --- a/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/cmake/profile.cmake +++ b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/cmake/profile.cmake @@ -1,17 +1,5 @@ -## Copyright (c) 2012-2024 Leonid Yuriev . -## -## Licensed under the Apache License, Version 2.0 (the "License"); -## you may not use this file except in compliance with the License. -## You may obtain a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -## See the License for the specific language governing permissions and -## limitations under the License. -## +# Copyright (c) 2012-2025 Леонид Юрьев aka Leonid Yuriev ############################################### +# SPDX-License-Identifier: Apache-2.0 if(CMAKE_VERSION VERSION_LESS 3.8.2) cmake_minimum_required(VERSION 3.0.2) @@ -24,32 +12,47 @@ endif() cmake_policy(PUSH) cmake_policy(VERSION ${CMAKE_MINIMUM_REQUIRED_VERSION}) +unset(MEMCHECK_OPTION_NAME) +if(NOT DEFINED ENABLE_MEMCHECK) + if(DEFINED MDBX_USE_VALGRIND) + set(MEMCHECK_OPTION_NAME "MDBX_USE_VALGRIND") + elseif(DEFINED ENABLE_VALGRIND) + set(MEMCHECK_OPTION_NAME "ENABLE_VALGRIND") + else() + set(MEMCHECK_OPTION_NAME "ENABLE_MEMCHECK") + endif() + if(MEMCHECK_OPTION_NAME STREQUAL "ENABLE_MEMCHECK") + option(ENABLE_MEMCHECK "Enable integration with valgrind, a memory analyzing tool" OFF) + elseif(${MEMCHECK_OPTION_NAME}) + set(ENABLE_MEMCHECK ON) + else() + set(ENABLE_MEMCHECK OFF) + endif() +endif() + include(CheckLibraryExists) check_library_exists(gcov __gcov_flush "" HAVE_GCOV) -option(ENABLE_GCOV - "Enable integration with gcov, a code coverage program" OFF) - -option(ENABLE_GPROF - "Enable integration with gprof, a performance analyzing tool" OFF) - -if(CMAKE_CXX_COMPILER_LOADED) - include(CheckIncludeFileCXX) - check_include_file_cxx(valgrind/memcheck.h HAVE_VALGRIND_MEMCHECK_H) -else() - include(CheckIncludeFile) - check_include_file(valgrind/memcheck.h HAVE_VALGRIND_MEMCHECK_H) -endif() +option(ENABLE_GCOV "Enable integration with gcov, a code coverage program" OFF) -option(MDBX_USE_VALGRIND "Enable integration with valgrind, a memory analyzing tool" OFF) -if(MDBX_USE_VALGRIND AND NOT HAVE_VALGRIND_MEMCHECK_H) - message(FATAL_ERROR "MDBX_USE_VALGRIND option is set but valgrind/memcheck.h is not found") -endif() +option(ENABLE_GPROF "Enable integration with gprof, a performance analyzing tool" OFF) -option(ENABLE_ASAN - "Enable AddressSanitizer, a fast memory error detector based on compiler instrumentation" OFF) +option(ENABLE_ASAN "Enable AddressSanitizer, a fast memory error detector based on compiler instrumentation" OFF) option(ENABLE_UBSAN - "Enable UndefinedBehaviorSanitizer, a fast undefined behavior detector based on compiler instrumentation" OFF) + "Enable UndefinedBehaviorSanitizer, a fast undefined behavior detector based on compiler instrumentation" OFF) + +if(ENABLE_MEMCHECK) + if(CMAKE_CXX_COMPILER_LOADED) + include(CheckIncludeFileCXX) + check_include_file_cxx(valgrind/memcheck.h HAVE_VALGRIND_MEMCHECK_H) + else() + include(CheckIncludeFile) + check_include_file(valgrind/memcheck.h HAVE_VALGRIND_MEMCHECK_H) + endif() + if(NOT HAVE_VALGRIND_MEMCHECK_H) + message(FATAL_ERROR "${MEMCHECK_OPTION_NAME} option is set but valgrind/memcheck.h is not found") + endif() +endif() cmake_policy(POP) diff --git a/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/cmake/utils.cmake b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/cmake/utils.cmake index 0fa578458d0..abb4cd30980 100644 --- a/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/cmake/utils.cmake +++ b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/cmake/utils.cmake @@ -1,17 +1,5 @@ -## Copyright (c) 2012-2024 Leonid Yuriev . -## -## Licensed under the Apache License, Version 2.0 (the "License"); -## you may not use this file except in compliance with the License. -## You may obtain a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -## See the License for the specific language governing permissions and -## limitations under the License. -## +# Copyright (c) 2012-2025 Леонид Юрьев aka Leonid Yuriev ############################################### +# SPDX-License-Identifier: Apache-2.0 if(CMAKE_VERSION VERSION_LESS 3.8.2) cmake_minimum_required(VERSION 3.0.2) @@ -24,6 +12,21 @@ endif() cmake_policy(PUSH) cmake_policy(VERSION ${CMAKE_MINIMUM_REQUIRED_VERSION}) +macro(add_option HIVE NAME DESCRIPTION DEFAULT) + list(APPEND ${HIVE}_BUILD_OPTIONS ${HIVE}_${NAME}) + if(NOT ${DEFAULT} STREQUAL "AUTO") + option(${HIVE}_${NAME} "${DESCRIPTION}" ${DEFAULT}) + elseif(NOT DEFINED ${HIVE}_${NAME}) + set(${HIVE}_${NAME}_AUTO ON) + endif() +endmacro() + +macro(set_if_undefined VARNAME) + if(NOT DEFINED "${VARNAME}") + set("${VARNAME}" ${ARGN}) + endif() +endmacro() + macro(add_compile_flags languages) foreach(_lang ${languages}) string(REPLACE ";" " " _flags "${ARGN}") @@ -61,9 +64,8 @@ macro(set_source_files_compile_flags) set(_lang "") if("${_file_ext}" STREQUAL ".m") set(_lang OBJC) - # CMake believes that Objective C is a flavor of C++, not C, - # and uses g++ compiler for .m files. - # LANGUAGE property forces CMake to use CC for ${file} + # CMake believes that Objective C is a flavor of C++, not C, and uses g++ compiler for .m files. LANGUAGE property + # forces CMake to use CC for ${file} set_source_files_properties(${file} PROPERTIES LANGUAGE C) elseif("${_file_ext}" STREQUAL ".mm") set(_lang OBJCXX) @@ -77,210 +79,446 @@ macro(set_source_files_compile_flags) set(_flags "${_flags} ${CMAKE_${_lang}_FLAGS}") endif() # message(STATUS "Set (${file} ${_flags}") - set_source_files_properties(${file} PROPERTIES COMPILE_FLAGS - "${_flags}") + set_source_files_properties(${file} PROPERTIES COMPILE_FLAGS "${_flags}") endif() endforeach() unset(_file_ext) unset(_lang) endmacro(set_source_files_compile_flags) -macro(fetch_version name source_root_directory parent_scope) - set(${name}_VERSION "") - set(${name}_GIT_DESCRIBE "") - set(${name}_GIT_TIMESTAMP "") - set(${name}_GIT_TREE "") - set(${name}_GIT_COMMIT "") - set(${name}_GIT_REVISION 0) - set(${name}_GIT_VERSION "") - if(GIT AND EXISTS "${source_root_directory}/.git") - execute_process(COMMAND ${GIT} show --no-patch --format=%cI HEAD - OUTPUT_VARIABLE ${name}_GIT_TIMESTAMP - OUTPUT_STRIP_TRAILING_WHITESPACE - WORKING_DIRECTORY ${source_root_directory} - RESULT_VARIABLE rc) - if(rc OR "${name}_GIT_TIMESTAMP" STREQUAL "%cI") - execute_process(COMMAND ${GIT} show --no-patch --format=%ci HEAD - OUTPUT_VARIABLE ${name}_GIT_TIMESTAMP - OUTPUT_STRIP_TRAILING_WHITESPACE - WORKING_DIRECTORY ${source_root_directory} - RESULT_VARIABLE rc) - if(rc OR "${name}_GIT_TIMESTAMP" STREQUAL "%ci") - message(FATAL_ERROR "Please install latest version of git (`show --no-patch --format=%cI HEAD` failed)") +macro(semver_parse str) + set(_semver_ok FALSE) + set(_semver_err "") + set(_semver_major 0) + set(_semver_minor 0) + set(_semver_patch 0) + set(_semver_tweak_withdot "") + set(_semver_tweak "") + set(_semver_extra "") + set(_semver_prerelease_withdash "") + set(_semver_prerelease "") + set(_semver_buildmetadata_withplus "") + set(_semver_buildmetadata "") + if("${str}" MATCHES + "^v?(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)(\\.(0|[1-9][0-9]*))?([-+]-*[0-9a-zA-Z]+.*)?$") + set(_semver_major ${CMAKE_MATCH_1}) + set(_semver_minor ${CMAKE_MATCH_2}) + set(_semver_patch ${CMAKE_MATCH_3}) + set(_semver_tweak_withdot ${CMAKE_MATCH_4}) + set(_semver_tweak ${CMAKE_MATCH_5}) + set(_semver_extra "${CMAKE_MATCH_6}") + if("${_semver_extra}" STREQUAL "") + set(_semver_ok TRUE) + elseif("${_semver_extra}" MATCHES "^([.-][a-zA-Z0-9-]+)*(\\+[^+]+)?$") + set(_semver_prerelease_withdash "${CMAKE_MATCH_1}") + if(NOT "${_semver_prerelease_withdash}" STREQUAL "") + string(SUBSTRING "${_semver_prerelease_withdash}" 1 -1 _semver_prerelease) endif() + set(_semver_buildmetadata_withplus "${CMAKE_MATCH_2}") + if(NOT "${_semver_buildmetadata_withplus}" STREQUAL "") + string(SUBSTRING "${_semver_buildmetadata_withplus}" 1 -1 _semver_buildmetadata) + endif() + set(_semver_ok TRUE) + else() + set(_semver_err + "Поля prerelease и/или buildmetadata (строка `-foo+bar` в составе `0.0.0[.0][-foo][+bar]`) не соответствуют SemVer-спецификации" + ) endif() + else() + set(_semver_err "Версионная отметка в целом не соответствует шаблону `0.0.0[.0][-foo][+bar]` SemVer-спецификации") + endif() +endmacro(semver_parse) + +function(_semver_parse_probe str expect) + semver_parse(${str}) + if(expect AND NOT _semver_ok) + message(FATAL_ERROR "semver_parse(${str}) expect SUCCESS, got ${_semver_ok}: ${_semver_err}") + elseif(NOT expect AND _semver_ok) + message(FATAL_ERROR "semver_parse(${str}) expect FAIL, got ${_semver_ok}") + endif() +endfunction() + +function(semver_parse_selfcheck) + _semver_parse_probe("0.0.4" TRUE) + _semver_parse_probe("v1.2.3" TRUE) + _semver_parse_probe("10.20.30" TRUE) + _semver_parse_probe("10.20.30.42" TRUE) + _semver_parse_probe("1.1.2-prerelease+meta" TRUE) + _semver_parse_probe("1.1.2+meta" TRUE) + _semver_parse_probe("1.1.2+meta-valid" TRUE) + _semver_parse_probe("1.0.0-alpha" TRUE) + _semver_parse_probe("1.0.0-beta" TRUE) + _semver_parse_probe("1.0.0-alpha.beta" TRUE) + _semver_parse_probe("1.0.0-alpha.beta.1" TRUE) + _semver_parse_probe("1.0.0-alpha.1" TRUE) + _semver_parse_probe("1.0.0-alpha0.valid" TRUE) + _semver_parse_probe("1.0.0-alpha.0valid" TRUE) + _semver_parse_probe("1.0.0-alpha-a.b-c-somethinglong+build.1-aef.1-its-okay" TRUE) + _semver_parse_probe("1.0.0-rc.1+build.1" TRUE) + _semver_parse_probe("2.0.0-rc.1+build.123" TRUE) + _semver_parse_probe("1.2.3-beta" TRUE) + _semver_parse_probe("10.2.3-DEV-SNAPSHOT" TRUE) + _semver_parse_probe("1.2.3-SNAPSHOT-123" TRUE) + _semver_parse_probe("1.0.0" TRUE) + _semver_parse_probe("2.0.0" TRUE) + _semver_parse_probe("1.1.7" TRUE) + _semver_parse_probe("2.0.0+build.1848" TRUE) + _semver_parse_probe("2.0.1-alpha.1227" TRUE) + _semver_parse_probe("1.0.0-alpha+beta" TRUE) + _semver_parse_probe("1.2.3----RC-SNAPSHOT.12.9.1--.12+788" TRUE) + _semver_parse_probe("1.2.3----R-S.12.9.1--.12+meta" TRUE) + _semver_parse_probe("1.2.3----RC-SNAPSHOT.12.9.1--.12" TRUE) + _semver_parse_probe("1.0.0+0.build.1-rc.10000aaa-kk-0.1" TRUE) + _semver_parse_probe("99999999999999999999999.999999999999999999.99999999999999999" TRUE) + _semver_parse_probe("v1.0.0-0A.is.legal" TRUE) - execute_process(COMMAND ${GIT} show --no-patch --format=%T HEAD - OUTPUT_VARIABLE ${name}_GIT_TREE + _semver_parse_probe("1" FALSE) + _semver_parse_probe("1.2" FALSE) + # _semver_parse_probe("1.2.3-0123" FALSE) _semver_parse_probe("1.2.3-0123.0123" FALSE) + _semver_parse_probe("1.1.2+.123" FALSE) + _semver_parse_probe("+invalid" FALSE) + _semver_parse_probe("-invalid" FALSE) + _semver_parse_probe("-invalid+invalid" FALSE) + _semver_parse_probe("-invalid.01" FALSE) + _semver_parse_probe("alpha" FALSE) + _semver_parse_probe("alpha.beta" FALSE) + _semver_parse_probe("alpha.beta.1" FALSE) + _semver_parse_probe("alpha.1" FALSE) + _semver_parse_probe("alpha+beta" FALSE) + _semver_parse_probe("alpha_beta" FALSE) + _semver_parse_probe("alpha." FALSE) + _semver_parse_probe("alpha.." FALSE) + _semver_parse_probe("beta" FALSE) + _semver_parse_probe("1.0.0-alpha_beta" FALSE) + _semver_parse_probe("-alpha." FALSE) + _semver_parse_probe("1.0.0-alpha.." FALSE) + _semver_parse_probe("1.0.0-alpha..1" FALSE) + _semver_parse_probe("1.0.0-alpha...1" FALSE) + _semver_parse_probe("1.0.0-alpha....1" FALSE) + _semver_parse_probe("1.0.0-alpha.....1" FALSE) + _semver_parse_probe("1.0.0-alpha......1" FALSE) + _semver_parse_probe("1.0.0-alpha.......1" FALSE) + _semver_parse_probe("01.1.1" FALSE) + _semver_parse_probe("1.01.1" FALSE) + _semver_parse_probe("1.1.01" FALSE) + _semver_parse_probe("1.2" FALSE) + _semver_parse_probe("1.2.3.DEV" FALSE) + _semver_parse_probe("1.2-SNAPSHOT" FALSE) + _semver_parse_probe("1.2.31.2.3----RC-SNAPSHOT.12.09.1--..12+788" FALSE) + _semver_parse_probe("1.2-RC-SNAPSHOT" FALSE) + _semver_parse_probe("-1.0.3-gamma+b7718" FALSE) + _semver_parse_probe("+justmeta" FALSE) + _semver_parse_probe("9.8.7+meta+meta" FALSE) + _semver_parse_probe("9.8.7-whatever+meta+meta" FALSE) + _semver_parse_probe( + "99999999999999999999999.999999999999999999.99999999999999999----RC-SNAPSHOT.12.09.1--------------------------------..12" + FALSE) +endfunction() + +macro(git_get_versioninfo source_root_directory) + set(_git_describe "") + set(_git_timestamp "") + set(_git_tree "") + set(_git_commit "") + set(_git_last_vtag "") + set(_git_trailing_commits 0) + set(_git_is_dirty FALSE) + + execute_process( + COMMAND ${GIT} show --no-patch --format=%cI HEAD + OUTPUT_VARIABLE _git_timestamp + OUTPUT_STRIP_TRAILING_WHITESPACE + WORKING_DIRECTORY ${source_root_directory} + RESULT_VARIABLE _rc) + if(_rc OR "${_git_timestamp}" STREQUAL "%cI") + execute_process( + COMMAND ${GIT} show --no-patch --format=%ci HEAD + OUTPUT_VARIABLE _git_timestamp OUTPUT_STRIP_TRAILING_WHITESPACE WORKING_DIRECTORY ${source_root_directory} - RESULT_VARIABLE rc) - if(rc OR "${name}_GIT_TREE" STREQUAL "") - message(FATAL_ERROR "Please install latest version of git (`show --no-patch --format=%T HEAD` failed)") + RESULT_VARIABLE _rc) + if(_rc OR "${_git_timestamp}" STREQUAL "%ci") + message(FATAL_ERROR "Please install latest version of git (`show --no-patch --format=%cI HEAD` failed)") endif() + endif() - execute_process(COMMAND ${GIT} show --no-patch --format=%H HEAD - OUTPUT_VARIABLE ${name}_GIT_COMMIT + execute_process( + COMMAND ${GIT} show --no-patch --format=%T HEAD + OUTPUT_VARIABLE _git_tree + OUTPUT_STRIP_TRAILING_WHITESPACE + WORKING_DIRECTORY ${source_root_directory} + RESULT_VARIABLE _rc) + if(_rc OR "${_git_tree}" STREQUAL "") + message(FATAL_ERROR "Please install latest version of git (`show --no-patch --format=%T HEAD` failed)") + endif() + + execute_process( + COMMAND ${GIT} show --no-patch --format=%H HEAD + OUTPUT_VARIABLE _git_commit + OUTPUT_STRIP_TRAILING_WHITESPACE + WORKING_DIRECTORY ${source_root_directory} + RESULT_VARIABLE _rc) + if(_rc OR "${_git_commit}" STREQUAL "") + message(FATAL_ERROR "Please install latest version of git (`show --no-patch --format=%H HEAD` failed)") + endif() + + execute_process( + COMMAND ${GIT} status --untracked-files=no --porcelain + OUTPUT_VARIABLE _git_status + OUTPUT_STRIP_TRAILING_WHITESPACE + WORKING_DIRECTORY ${source_root_directory} + RESULT_VARIABLE _rc) + if(_rc) + message(FATAL_ERROR "Please install latest version of git (`status --untracked-files=no --porcelain` failed)") + endif() + if(NOT "${_git_status}" STREQUAL "") + set(_git_commit "DIRTY-${_git_commit}") + set(_git_is_dirty TRUE) + endif() + unset(_git_status) + + execute_process( + COMMAND ${GIT} describe --tags --abbrev=0 "--match=v[0-9]*" + OUTPUT_VARIABLE _git_last_vtag + OUTPUT_STRIP_TRAILING_WHITESPACE + WORKING_DIRECTORY ${source_root_directory} + RESULT_VARIABLE _rc) + if(_rc OR "${_git_last_vtag}" STREQUAL "") + execute_process( + COMMAND ${GIT} tag + OUTPUT_VARIABLE _git_tags_dump OUTPUT_STRIP_TRAILING_WHITESPACE WORKING_DIRECTORY ${source_root_directory} - RESULT_VARIABLE rc) - if(rc OR "${name}_GIT_COMMIT" STREQUAL "") - message(FATAL_ERROR "Please install latest version of git (`show --no-patch --format=%H HEAD` failed)") - endif() - - execute_process(COMMAND ${GIT} rev-list --tags --count - OUTPUT_VARIABLE tag_count + RESULT_VARIABLE _rc) + execute_process( + COMMAND ${GIT} rev-list --count --no-merges --remove-empty HEAD + OUTPUT_VARIABLE _git_whole_count OUTPUT_STRIP_TRAILING_WHITESPACE WORKING_DIRECTORY ${source_root_directory} - RESULT_VARIABLE rc) - if(rc) - message(FATAL_ERROR "Please install latest version of git (`git rev-list --tags --count` failed)") + RESULT_VARIABLE _rc) + if(_rc) + message( + FATAL_ERROR + "Please install latest version of git (`git rev-list --count --no-merges --remove-empty HEAD` failed)") endif() - - if(tag_count EQUAL 0) - execute_process(COMMAND ${GIT} rev-list --all --count - OUTPUT_VARIABLE whole_count - OUTPUT_STRIP_TRAILING_WHITESPACE - WORKING_DIRECTORY ${source_root_directory} - RESULT_VARIABLE rc) - if(rc) - message(FATAL_ERROR "Please install latest version of git (`git rev-list --all --count` failed)") - endif() - if(whole_count GREATER 42) - message(FATAL_ERROR "Please fetch tags (no any tags for ${whole_count} commits)") - endif() - set(${name}_GIT_VERSION "0;0;0") - execute_process(COMMAND ${GIT} rev-list --count --all --no-merges - OUTPUT_VARIABLE ${name}_GIT_REVISION - OUTPUT_STRIP_TRAILING_WHITESPACE - WORKING_DIRECTORY ${source_root_directory} - RESULT_VARIABLE rc) - if(rc OR "${name}_GIT_REVISION" STREQUAL "") - message(FATAL_ERROR "Please install latest version of git (`rev-list --count --all --no-merges` failed)") - endif() - else(tag_count EQUAL 0) - execute_process(COMMAND ${GIT} describe --tags --long --dirty=-dirty "--match=v[0-9]*" - OUTPUT_VARIABLE ${name}_GIT_DESCRIBE + if(_git_whole_count GREATER 42 AND "${_git_tags_dump}" STREQUAL "") + message(FATAL_ERROR "Please fetch tags (`describe --tags --abbrev=0 --match=v[0-9]*` failed)") + else() + message(NOTICE "Falling back to version `0.0.0` (have you made an initial release?") + endif() + set(_git_last_vtag "0.0.0") + set(_git_trailing_commits ${_git_whole_count}) + execute_process( + COMMAND ${GIT} describe --tags --dirty --long --always + OUTPUT_VARIABLE _git_describe + OUTPUT_STRIP_TRAILING_WHITESPACE + WORKING_DIRECTORY ${source_root_directory} + RESULT_VARIABLE _rc) + if(_rc OR "${_git_describe}" STREQUAL "") + execute_process( + COMMAND ${GIT} describe --tags --all --dirty --long --always + OUTPUT_VARIABLE _git_describe OUTPUT_STRIP_TRAILING_WHITESPACE WORKING_DIRECTORY ${source_root_directory} - RESULT_VARIABLE rc) - if(rc OR "${name}_GIT_DESCRIBE" STREQUAL "") - if(_whole_count GREATER 42) - message(FATAL_ERROR "Please fetch tags (`describe --tags --long --dirty --match=v[0-9]*` failed)") - else() - execute_process(COMMAND ${GIT} describe --all --long --dirty=-dirty - OUTPUT_VARIABLE ${name}_GIT_DESCRIBE - OUTPUT_STRIP_TRAILING_WHITESPACE - WORKING_DIRECTORY ${source_root_directory} - RESULT_VARIABLE rc) - if(rc OR "${name}_GIT_DESCRIBE" STREQUAL "") - message(FATAL_ERROR "Please install latest version of git (`git rev-list --tags --count` and/or `git rev-list --all --count` failed)") - endif() - endif() + RESULT_VARIABLE _rc) + if(_rc OR "${_git_describe}" STREQUAL "") + message(FATAL_ERROR "Please install latest version of git (`describe --tags --all --long` failed)") endif() + endif() + else() + execute_process( + COMMAND ${GIT} describe --tags --dirty --long "--match=v[0-9]*" + OUTPUT_VARIABLE _git_describe + OUTPUT_STRIP_TRAILING_WHITESPACE + WORKING_DIRECTORY ${source_root_directory} + RESULT_VARIABLE _rc) + if(_rc OR "${_git_describe}" STREQUAL "") + message(FATAL_ERROR "Please install latest version of git (`describe --tags --long --match=v[0-9]*`)") + endif() + execute_process( + COMMAND ${GIT} rev-list --count "${_git_last_vtag}..HEAD" + OUTPUT_VARIABLE _git_trailing_commits + OUTPUT_STRIP_TRAILING_WHITESPACE + WORKING_DIRECTORY ${source_root_directory} + RESULT_VARIABLE _rc) + if(_rc OR "${_git_trailing_commits}" STREQUAL "") + message(FATAL_ERROR "Please install latest version of git (`rev-list --count ${_git_last_vtag}..HEAD` failed)") + endif() + endif() +endmacro(git_get_versioninfo) - execute_process(COMMAND ${GIT} describe --tags --abbrev=0 "--match=v[0-9]*" - OUTPUT_VARIABLE last_release_tag - OUTPUT_STRIP_TRAILING_WHITESPACE - WORKING_DIRECTORY ${source_root_directory} - RESULT_VARIABLE rc) - if(rc) - message(FATAL_ERROR "Please install latest version of git (`describe --tags --abbrev=0 --match=v[0-9]*` failed)") - endif() - if (last_release_tag) - set(git_revlist_arg "${last_release_tag}..HEAD") +macro(semver_provide name source_root_directory build_directory_for_json_output build_metadata parent_scope) + set(_semver "") + set(_git_describe "") + set(_git_timestamp "") + set(_git_tree "") + set(_git_commit "") + set(_version_from "") + set(_git_root FALSE) + + find_program(GIT git) + if(GIT) + execute_process( + COMMAND ${GIT} rev-parse --show-toplevel + OUTPUT_VARIABLE _git_root + ERROR_VARIABLE _git_root_error + OUTPUT_STRIP_TRAILING_WHITESPACE + WORKING_DIRECTORY ${source_root_directory} + RESULT_VARIABLE _rc) + if(_rc OR "${_git_root}" STREQUAL "") + if(EXISTS "${source_root_directory}/.git") + message(ERROR "`git rev-parse --show-toplevel` failed '${_git_root_error}'") else() - execute_process(COMMAND ${GIT} tag --sort=-version:refname - OUTPUT_VARIABLE tag_list - OUTPUT_STRIP_TRAILING_WHITESPACE - WORKING_DIRECTORY ${source_root_directory} - RESULT_VARIABLE rc) - if(rc) - message(FATAL_ERROR "Please install latest version of git (`tag --sort=-version:refname` failed)") - endif() - string(REGEX REPLACE "\n" ";" tag_list "${tag_list}") - set(git_revlist_arg "HEAD") - foreach(tag IN LISTS tag_list) - if(NOT last_release_tag) - string(REGEX MATCH "^v[0-9]+(\.[0-9]+)+" last_release_tag "${tag}") - set(git_revlist_arg "${tag}..HEAD") - endif() - endforeach(tag) + message(VERBOSE "`git rev-parse --show-toplevel` failed '${_git_root_error}'") endif() - execute_process(COMMAND ${GIT} rev-list --count "${git_revlist_arg}" - OUTPUT_VARIABLE ${name}_GIT_REVISION - OUTPUT_STRIP_TRAILING_WHITESPACE - WORKING_DIRECTORY ${source_root_directory} - RESULT_VARIABLE rc) - if(rc OR "${name}_GIT_REVISION" STREQUAL "") - message(FATAL_ERROR "Please install latest version of git (`rev-list --count ${git_revlist_arg}` failed)") + else() + set(_source_root "${source_root_directory}") + if(NOT CMAKE_VERSION VERSION_LESS 3.19) + file(REAL_PATH "${_git_root}" _git_root) + file(REAL_PATH "${_source_root}" _source_root) endif() - - string(REGEX MATCH "^(v)?([0-9]+)\\.([0-9]+)\\.([0-9]+)(.*)?" git_version_valid "${${name}_GIT_DESCRIBE}") - if(git_version_valid) - string(REGEX REPLACE "^(v)?([0-9]+)\\.([0-9]+)\\.([0-9]+)(.*)?" "\\2;\\3;\\4" ${name}_GIT_VERSION ${${name}_GIT_DESCRIBE}) - else() - string(REGEX MATCH "^(v)?([0-9]+)\\.([0-9]+)(.*)?" git_version_valid "${${name}_GIT_DESCRIBE}") - if(git_version_valid) - string(REGEX REPLACE "^(v)?([0-9]+)\\.([0-9]+)(.*)?" "\\2;\\3;0" ${name}_GIT_VERSION ${${name}_GIT_DESCRIBE}) - else() - message(AUTHOR_WARNING "Bad ${name} version \"${${name}_GIT_DESCRIBE}\"; falling back to 0.0.0 (have you made an initial release?)") - set(${name}_GIT_VERSION "0;0;0") - endif() + if(_source_root STREQUAL _git_root AND EXISTS "${_git_root}/VERSION.json") + message( + FATAL_ERROR + "Несколько источников информации о версии, допустим только один из: репозиторий git, либо файл VERSION.json" + ) endif() - endif(tag_count EQUAL 0) + endif() endif() - if(NOT ${name}_GIT_VERSION OR NOT ${name}_GIT_TIMESTAMP OR ${name}_GIT_REVISION STREQUAL "") - if(GIT AND EXISTS "${source_root_directory}/.git") - message(WARNING "Unable to retrieve ${name} version from git.") - endif() - set(${name}_GIT_VERSION "0;0;0;0") - set(${name}_GIT_TIMESTAMP "") - set(${name}_GIT_REVISION 0) + if(EXISTS "${source_root_directory}/VERSION.json") + set(_version_from "${source_root_directory}/VERSION.json") - # Try to get version from VERSION file - set(version_file "${source_root_directory}/VERSION.txt") - if(NOT EXISTS "${version_file}") - set(version_file "${source_root_directory}/VERSION") + if(CMAKE_VERSION VERSION_LESS 3.19) + message(FATAL_ERROR "Требуется CMake версии >= 3.19 для чтения VERSION.json") endif() - if(EXISTS "${version_file}") - file(STRINGS "${version_file}" ${name}_VERSION LIMIT_COUNT 1 LIMIT_INPUT 42) + file( + STRINGS "${_version_from}" _versioninfo_json NEWLINE_CONSUME + LIMIT_COUNT 9 + LIMIT_INPUT 999 + ENCODING UTF-8) + string(JSON _git_describe GET ${_versioninfo_json} git_describe) + string(JSON _git_timestamp GET "${_versioninfo_json}" "git_timestamp") + string(JSON _git_tree GET "${_versioninfo_json}" "git_tree") + string(JSON _git_commit GET "${_versioninfo_json}" "git_commit") + string(JSON _semver GET "${_versioninfo_json}" "semver") + unset(_json_object) + if(NOT _semver) + message(FATAL_ERROR "Unable to retrieve ${name} version from \"${_version_from}\" file.") endif() - - if(NOT ${name}_VERSION) - message(WARNING "Unable to retrieve ${name} version from \"${version_file}\" file.") - set(${name}_VERSION_LIST ${${name}_GIT_VERSION}) - string(REPLACE ";" "." ${name}_VERSION "${${name}_GIT_VERSION}") - else() - string(REPLACE "." ";" ${name}_VERSION_LIST ${${name}_VERSION}) + semver_parse("${_semver}") + if(NOT _semver_ok) + message(FATAL_ERROR "SemVer `${_semver}` from ${_version_from}: ${_semver_err}") + endif() + elseif(_git_root AND _source_root STREQUAL _git_root) + set(_version_from git) + git_get_versioninfo(${source_root_directory}) + semver_parse(${_git_last_vtag}) + if(NOT _semver_ok) + message(FATAL_ERROR "Git tag `${_git_last_vtag}`: ${_semver_err}") + endif() + if(_git_trailing_commits GREATER 0 AND "${_semver_tweak}" STREQUAL "") + set(_semver_tweak ${_git_trailing_commits}) endif() + elseif(GIT) + message( + FATAL_ERROR + "Нет источника информации о версии (${source_root_directory}), требуется один из: репозиторий git, либо VERSION.json" + ) else() - list(APPEND ${name}_GIT_VERSION ${${name}_GIT_REVISION}) - set(${name}_VERSION_LIST ${${name}_GIT_VERSION}) - string(REPLACE ";" "." ${name}_VERSION "${${name}_GIT_VERSION}") + message(FATAL_ERROR "Требуется git для получения информации о версии") endif() - list(GET ${name}_VERSION_LIST 0 "${name}_VERSION_MAJOR") - list(GET ${name}_VERSION_LIST 1 "${name}_VERSION_MINOR") - list(GET ${name}_VERSION_LIST 2 "${name}_VERSION_RELEASE") - list(GET ${name}_VERSION_LIST 3 "${name}_VERSION_REVISION") + if(NOT _git_describe + OR NOT _git_timestamp + OR NOT _git_tree + OR NOT _git_commit + OR "${_semver_major}" STREQUAL "" + OR "${_semver_minor}" STREQUAL "" + OR "${_semver_patch}" STREQUAL "") + message(ERROR "Unable to retrieve ${name} version from ${_version_from}.") + endif() + + set(_semver "${_semver_major}.${_semver_minor}.${_semver_patch}") + if("${_semver_tweak}" STREQUAL "") + set(_semver_tweak 0) + elseif(_semver_tweak GREATER 0) + string(APPEND _semver ".${_semver_tweak}") + endif() + if(NOT "${_semver_prerelease}" STREQUAL "") + string(APPEND _semver "-${_semver_prerelease}") + endif() + if(_git_is_dirty) + string(APPEND _semver "-DIRTY") + endif() + + set(_semver_complete "${_semver}") + if(NOT "${build_metadata}" STREQUAL "") + string(APPEND _semver_complete "+${build_metadata}") + endif() + + set(${name}_VERSION "${_semver_complete}") + set(${name}_VERSION_PURE "${_semver}") + set(${name}_VERSION_MAJOR ${_semver_major}) + set(${name}_VERSION_MINOR ${_semver_minor}) + set(${name}_VERSION_PATCH ${_semver_patch}) + set(${name}_VERSION_TWEAK ${_semver_tweak}) + set(${name}_VERSION_PRERELEASE "${_semver_prerelease}") + set(${name}_GIT_DESCRIBE "${_git_describe}") + set(${name}_GIT_TIMESTAMP "${_git_timestamp}") + set(${name}_GIT_TREE "${_git_tree}") + set(${name}_GIT_COMMIT "${_git_commit}") if(${parent_scope}) - set(${name}_VERSION_MAJOR "${${name}_VERSION_MAJOR}" PARENT_SCOPE) - set(${name}_VERSION_MINOR "${${name}_VERSION_MINOR}" PARENT_SCOPE) - set(${name}_VERSION_RELEASE "${${name}_VERSION_RELEASE}" PARENT_SCOPE) - set(${name}_VERSION_REVISION "${${name}_VERSION_REVISION}" PARENT_SCOPE) - set(${name}_VERSION "${${name}_VERSION}" PARENT_SCOPE) + set(${name}_VERSION + "${_semver_complete}" + PARENT_SCOPE) + set(${name}_VERSION_PURE + "${_semver}" + PARENT_SCOPE) + set(${name}_VERSION_MAJOR + ${_semver_major} + PARENT_SCOPE) + set(${name}_VERSION_MINOR + ${_semver_minor} + PARENT_SCOPE) + set(${name}_VERSION_PATCH + ${_semver_patch} + PARENT_SCOPE) + set(${name}_VERSION_TWEAK + "${_semver_tweak}" + PARENT_SCOPE) + set(${name}_VERSION_PRERELEASE + "${_semver_prerelease}" + PARENT_SCOPE) + set(${name}_GIT_DESCRIBE + "${_git_describe}" + PARENT_SCOPE) + set(${name}_GIT_TIMESTAMP + "${_git_timestamp}" + PARENT_SCOPE) + set(${name}_GIT_TREE + "${_git_tree}" + PARENT_SCOPE) + set(${name}_GIT_COMMIT + "${_git_commit}" + PARENT_SCOPE) + endif() - set(${name}_GIT_DESCRIBE "${${name}_GIT_DESCRIBE}" PARENT_SCOPE) - set(${name}_GIT_TIMESTAMP "${${name}_GIT_TIMESTAMP}" PARENT_SCOPE) - set(${name}_GIT_TREE "${${name}_GIT_TREE}" PARENT_SCOPE) - set(${name}_GIT_COMMIT "${${name}_GIT_COMMIT}" PARENT_SCOPE) - set(${name}_GIT_REVISION "${${name}_GIT_REVISION}" PARENT_SCOPE) - set(${name}_GIT_VERSION "${${name}_GIT_VERSION}" PARENT_SCOPE) + if(_version_from STREQUAL "git") + string( + CONFIGURE + "{ + \"git_describe\" : \"@_git_describe@\", + \"git_timestamp\" : \"@_git_timestamp@\", + \"git_tree\" : \"@_git_tree@\", + \"git_commit\" : \"@_git_commit@\", + \"semver\" : \"@_semver@\"\n}" + _versioninfo_json + @ONLY ESCAPE_QUOTES) + file(WRITE "${build_directory_for_json_output}/VERSION.json" "${_versioninfo_json}") endif() -endmacro(fetch_version) +endmacro(semver_provide) cmake_policy(POP) diff --git a/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/config.h.in b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/config.h.in index 05c561b1e1d..5d53860cfc2 100644 --- a/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/config.h.in +++ b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/config.h.in @@ -5,12 +5,15 @@ /* clang-format off */ #cmakedefine LTO_ENABLED -#cmakedefine MDBX_USE_VALGRIND +#cmakedefine ENABLE_MEMCHECK #cmakedefine ENABLE_GPROF #cmakedefine ENABLE_GCOV #cmakedefine ENABLE_ASAN #cmakedefine ENABLE_UBSAN #cmakedefine01 MDBX_FORCE_ASSERTIONS +#if !defined(MDBX_BUILD_TEST) && !defined(MDBX_BUILD_CXX) +#cmakedefine01 MDBX_BUILD_CXX +#endif /* Common */ #cmakedefine01 MDBX_TXN_CHECKOWNER @@ -29,23 +32,36 @@ #cmakedefine01 MDBX_DISABLE_VALIDATION #cmakedefine01 MDBX_AVOID_MSYNC #cmakedefine01 MDBX_ENABLE_REFUND -#cmakedefine01 MDBX_ENABLE_MADVISE #cmakedefine01 MDBX_ENABLE_BIGFOOT #cmakedefine01 MDBX_ENABLE_PGOP_STAT #cmakedefine01 MDBX_ENABLE_PROFGC +#cmakedefine01 MDBX_ENABLE_DBI_SPARSE +#cmakedefine01 MDBX_ENABLE_DBI_LOCKFREE /* Windows */ +#if defined(MDBX_BUILD_TEST) || !defined(MDBX_BUILD_CXX) || MDBX_BUILD_CXX +#define MDBX_WITHOUT_MSVC_CRT 0 +#else #cmakedefine01 MDBX_WITHOUT_MSVC_CRT +#endif /* MDBX_WITHOUT_MSVC_CRT */ /* MacOS & iOS */ -#cmakedefine01 MDBX_OSX_SPEED_INSTEADOF_DURABILITY +#cmakedefine01 MDBX_APPLE_SPEED_INSTEADOF_DURABILITY /* POSIX */ #cmakedefine01 MDBX_DISABLE_GNU_SOURCE + #cmakedefine MDBX_USE_OFDLOCKS_AUTO #ifndef MDBX_USE_OFDLOCKS_AUTO #cmakedefine01 MDBX_USE_OFDLOCKS -#endif +#endif /* MDBX_USE_OFDLOCKS */ + +#cmakedefine MDBX_MMAP_NEEDS_JOLT_AUTO +#ifndef MDBX_MMAP_NEEDS_JOLT_AUTO +#cmakedefine01 MDBX_MMAP_NEEDS_JOLT +#endif /* MDBX_MMAP_NEEDS_JOLT */ + +#cmakedefine01 MDBX_USE_MINCORE /* Build Info */ #ifndef MDBX_BUILD_TIMESTAMP @@ -63,6 +79,9 @@ #ifndef MDBX_BUILD_FLAGS #cmakedefine MDBX_BUILD_FLAGS "@MDBX_BUILD_FLAGS@" #endif +#ifndef MDBX_BUILD_METADATA +#cmakedefine MDBX_BUILD_METADATA "@MDBX_BUILD_METADATA@" +#endif #cmakedefine MDBX_BUILD_SOURCERY @MDBX_BUILD_SOURCERY@ /* *INDENT-ON* */ diff --git a/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/man1/mdbx_chk.1 b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/man1/mdbx_chk.1 index 7b182325b31..bc6de4b7758 100644 --- a/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/man1/mdbx_chk.1 +++ b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/man1/mdbx_chk.1 @@ -1,6 +1,6 @@ -.\" Copyright 2015-2024 Leonid Yuriev . +.\" Copyright 2015-2025 Leonid Yuriev . .\" Copying restrictions apply. See COPYRIGHT/LICENSE. -.TH MDBX_CHK 1 "2024-03-13" "MDBX 0.12.10" +.TH MDBX_CHK 1 "2024-08-29" "MDBX 0.13" .SH NAME mdbx_chk \- MDBX checking tool .SH SYNOPSIS @@ -22,12 +22,12 @@ mdbx_chk \- MDBX checking tool [\c .BR \-i ] [\c -.BI \-s \ subdb\fR] +.BI \-s \ table\fR] .BR \ dbpath .SH DESCRIPTION The .B mdbx_chk -utility intended to check an MDBX database file. +utility is intended to check an MDBX database file. .SH OPTIONS .TP .BR \-V @@ -55,7 +55,7 @@ check, including full check of all meta-pages and actual size of database file. .BR \-w Open environment in read-write mode and lock for writing while checking. This could be impossible if environment already used by another process(s) -in an incompatible read-write mode. This allow rollback to last steady commit +in an incompatible read-write mode. This allows rollback to last steady commit (in case environment was not closed properly) and then check transaction IDs of meta-pages. Otherwise, without \fB\-w\fP option environment will be opened in read-only mode. @@ -69,8 +69,8 @@ pages. Ignore wrong order errors, which will likely false-positive if custom comparator(s) was used. .TP -.BR \-s \ subdb -Verify and show info only for a specific subdatabase. +.BR \-s \ table +Verify and show info only for a specific table. .TP .BR \-0 | \-1 | \-2 Using specific meta-page 0, or 2 for checking. @@ -90,7 +90,7 @@ then forcibly loads ones by sequential access and tries to lock database pages i .TP .BR \-n Open MDBX environment(s) which do not use subdirectories. -This is legacy option. For now MDBX handles this automatically. +This is a legacy option. For now MDBX handles this automatically. .SH DIAGNOSTICS Exit status is zero if no errors occur. Errors result in a non-zero exit status diff --git a/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/man1/mdbx_copy.1 b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/man1/mdbx_copy.1 index 33a017fd7c4..83ec8553bd4 100644 --- a/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/man1/mdbx_copy.1 +++ b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/man1/mdbx_copy.1 @@ -1,8 +1,8 @@ -.\" Copyright 2015-2024 Leonid Yuriev . +.\" Copyright 2015-2025 Leonid Yuriev . .\" Copyright 2015,2016 Peter-Service R&D LLC . .\" Copyright 2012-2015 Howard Chu, Symas Corp. All Rights Reserved. .\" Copying restrictions apply. See COPYRIGHT/LICENSE. -.TH MDBX_COPY 1 "2024-03-13" "MDBX 0.12.10" +.TH MDBX_COPY 1 "2024-08-29" "MDBX 0.13" .SH NAME mdbx_copy \- MDBX environment copy tool .SH SYNOPSIS @@ -14,6 +14,10 @@ mdbx_copy \- MDBX environment copy tool [\c .BR \-c ] [\c +.BR \-d ] +[\c +.BR \-p ] +[\c .BR \-n ] .B src_path [\c @@ -45,6 +49,22 @@ or unused pages will be omitted from the copy. This option will slow down the backup process as it is more CPU-intensive. Currently it fails if the environment has suffered a page leak. .TP +.BR \-d +Alters geometry to enforce the copy to be a dynamic size DB, +which could be growth and shrink by reasonable steps on the fly. +.TP +.BR \-p +Use read transaction parking/ousting during copying MVCC-snapshot. +This allows the writing transaction to oust the read +transaction used to copy the database if copying takes so long +that it will interfere with the recycling old MVCC snapshots +and may lead to an overflow of the database. +However, if the reading transaction is ousted the copy will +be aborted until successful completion. Thus, this option +allows copy the database without interfering with write +transactions and a threat of database overflow, but at the cost +that copying will be aborted to prevent such conditions. +.TP .BR \-u Warms up the DB before copying via notifying OS kernel of subsequent access to the database pages. .TP diff --git a/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/man1/mdbx_drop.1 b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/man1/mdbx_drop.1 index 70d8355c9a0..a6abbe4198d 100644 --- a/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/man1/mdbx_drop.1 +++ b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/man1/mdbx_drop.1 @@ -1,7 +1,7 @@ -.\" Copyright 2021-2024 Leonid Yuriev . +.\" Copyright 2021-2025 Leonid Yuriev . .\" Copyright 2014-2021 Howard Chu, Symas Corp. All Rights Reserved. .\" Copying restrictions apply. See COPYRIGHT/LICENSE. -.TH MDBX_DROP 1 "2024-03-13" "MDBX 0.12.10" +.TH MDBX_DROP 1 "2024-08-29" "MDBX 0.13" .SH NAME mdbx_drop \- MDBX database delete tool .SH SYNOPSIS @@ -11,7 +11,7 @@ mdbx_drop \- MDBX database delete tool [\c .BR \-d ] [\c -.BI \-s \ subdb\fR] +.BI \-s \ table\fR] [\c .BR \-n ] .BR \ dbpath @@ -28,8 +28,8 @@ Write the library version number to the standard output, and exit. .BR \-d Delete the specified database, don't just empty it. .TP -.BR \-s \ subdb -Operate on a specific subdatabase. If no database is specified, only the main database is dropped. +.BR \-s \ table +Operate on a specific table. If no table is specified, only the main table is dropped. .TP .BR \-n Dump an MDBX database which does not use subdirectories. diff --git a/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/man1/mdbx_dump.1 b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/man1/mdbx_dump.1 index c809d7d3fb9..030617c3478 100644 --- a/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/man1/mdbx_dump.1 +++ b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/man1/mdbx_dump.1 @@ -1,8 +1,8 @@ -.\" Copyright 2015-2024 Leonid Yuriev . +.\" Copyright 2015-2025 Leonid Yuriev . .\" Copyright 2015,2016 Peter-Service R&D LLC . .\" Copyright 2014-2015 Howard Chu, Symas Corp. All Rights Reserved. .\" Copying restrictions apply. See COPYRIGHT/LICENSE. -.TH MDBX_DUMP 1 "2024-03-13" "MDBX 0.12.10" +.TH MDBX_DUMP 1 "2024-08-29" "MDBX 0.13" .SH NAME mdbx_dump \- MDBX environment export tool .SH SYNOPSIS @@ -19,7 +19,7 @@ mdbx_dump \- MDBX environment export tool .BR \-p ] [\c .BR \-a \ | -.BI \-s \ subdb\fR] +.BI \-s \ table\fR] [\c .BR \-r ] [\c @@ -58,10 +58,10 @@ are considered printing characters, and databases dumped in this manner may be less portable to external systems. .TP .BR \-a -Dump all of the subdatabases in the environment. +Dump all of the tables in the environment. .TP -.BR \-s \ subdb -Dump a specific subdatabase. If no database is specified, only the main database is dumped. +.BR \-s \ table +Dump a specific table. If no database is specified, only the main table is dumped. .TP .BR \-r Rescure mode. Ignore some errors to dump corrupted DB. diff --git a/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/man1/mdbx_load.1 b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/man1/mdbx_load.1 index 675865cd6cf..27b533e398a 100644 --- a/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/man1/mdbx_load.1 +++ b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/man1/mdbx_load.1 @@ -1,8 +1,8 @@ -.\" Copyright 2015-2024 Leonid Yuriev . +.\" Copyright 2015-2025 Leonid Yuriev . .\" Copyright 2015,2016 Peter-Service R&D LLC . .\" Copyright 2014-2015 Howard Chu, Symas Corp. All Rights Reserved. .\" Copying restrictions apply. See COPYRIGHT/LICENSE. -.TH MDBX_LOAD 1 "2024-03-13" "MDBX 0.12.10" +.TH MDBX_LOAD 1 "2024-08-29" "MDBX 0.13" .SH NAME mdbx_load \- MDBX environment import tool .SH SYNOPSIS @@ -16,7 +16,7 @@ mdbx_load \- MDBX environment import tool [\c .BI \-f \ file\fR] [\c -.BI \-s \ subdb\fR] +.BI \-s \ table\fR] [\c .BR \-N ] [\c @@ -71,11 +71,11 @@ on a database that uses custom compare functions. .BR \-f \ file Read from the specified file instead of from the standard input. .TP -.BR \-s \ subdb -Load a specific subdatabase. If no database is specified, data is loaded into the main database. +.BR \-s \ table +Load a specific table. If no table is specified, data is loaded into the main table. .TP .BR \-N -Don't overwrite existing records when loading into an already existing database; just skip them. +Don't overwrite existing records when loading into an already existing table; just skip them. .TP .BR \-T Load data from simple text files. The input must be paired lines of text, where the first diff --git a/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/man1/mdbx_stat.1 b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/man1/mdbx_stat.1 index 38cd735cef8..68a698c1cb2 100644 --- a/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/man1/mdbx_stat.1 +++ b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/man1/mdbx_stat.1 @@ -1,8 +1,8 @@ -.\" Copyright 2015-2024 Leonid Yuriev . +.\" Copyright 2015-2025 Leonid Yuriev . .\" Copyright 2015,2016 Peter-Service R&D LLC . .\" Copyright 2012-2015 Howard Chu, Symas Corp. All Rights Reserved. .\" Copying restrictions apply. See COPYRIGHT/LICENSE. -.TH MDBX_STAT 1 "2024-03-13" "MDBX 0.12.10" +.TH MDBX_STAT 1 "2024-08-29" "MDBX 0.13" .SH NAME mdbx_stat \- MDBX environment status tool .SH SYNOPSIS @@ -21,7 +21,7 @@ mdbx_stat \- MDBX environment status tool .BR \-r [ r ]] [\c .BR \-a \ | -.BI \-s \ subdb\fR] +.BI \-s \ table\fR] .BR \ dbpath [\c .BR \-n ] @@ -61,10 +61,10 @@ table and clear them. The reader table will be printed again after the check is performed. .TP .BR \-a -Display the status of all of the subdatabases in the environment. +Display the status of all of the tables in the environment. .TP -.BR \-s \ subdb -Display the status of a specific subdatabase. +.BR \-s \ table +Display the status of a specific table. .TP .BR \-n Display the status of an MDBX database which does not use subdirectories. diff --git a/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx.c b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx.c index 2601c5ada5a..ae5de1be4c9 100644 --- a/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx.c +++ b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx.c @@ -1,36 +1,26 @@ -/* - * Copyright 2015-2024 Leonid Yuriev - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . */ +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 +/* clang-format off */ -#define xMDBX_ALLOY 1 -#define MDBX_BUILD_SOURCERY e156c1a97c017ce89d6541cd9464ae5a9761d76b3fd2f1696521f5f3792904fc_v0_12_13_0_g1fff1f67 -#ifdef MDBX_CONFIG_H -#include MDBX_CONFIG_H -#endif +#define xMDBX_ALLOY 1 /* alloyed build */ + +#define MDBX_BUILD_SOURCERY 6b5df6869d2bf5419e3a8189d9cc849cc9911b9c8a951b9750ed0a261ce43724_v0_13_7_0_g566b0f93 #define LIBMDBX_INTERNALS -#ifdef xMDBX_TOOLS #define MDBX_DEPRECATED -#endif /* xMDBX_TOOLS */ -#ifdef xMDBX_ALLOY -/* Amalgamated build */ -#define MDBX_INTERNAL_FUNC static -#define MDBX_INTERNAL_VAR static -#else -/* Non-amalgamated build */ -#define MDBX_INTERNAL_FUNC -#define MDBX_INTERNAL_VAR extern -#endif /* xMDBX_ALLOY */ +#ifdef MDBX_CONFIG_H +#include MDBX_CONFIG_H +#endif + +/* Undefine the NDEBUG if debugging is enforced by MDBX_DEBUG */ +#if (defined(MDBX_DEBUG) && MDBX_DEBUG > 0) || (defined(MDBX_FORCE_ASSERTIONS) && MDBX_FORCE_ASSERTIONS) +#undef NDEBUG +#ifndef MDBX_DEBUG +/* Чтобы избежать включения отладки только из-за включения assert-проверок */ +#define MDBX_DEBUG 0 +#endif +#endif /*----------------------------------------------------------------------------*/ @@ -48,14 +38,59 @@ #endif /* MDBX_DISABLE_GNU_SOURCE */ /* Should be defined before any includes */ -#if !defined(_FILE_OFFSET_BITS) && !defined(__ANDROID_API__) && \ - !defined(ANDROID) +#if !defined(_FILE_OFFSET_BITS) && !defined(__ANDROID_API__) && !defined(ANDROID) #define _FILE_OFFSET_BITS 64 -#endif +#endif /* _FILE_OFFSET_BITS */ -#ifdef __APPLE__ +#if defined(__APPLE__) && !defined(_DARWIN_C_SOURCE) #define _DARWIN_C_SOURCE -#endif +#endif /* _DARWIN_C_SOURCE */ + +#if (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) && !defined(__USE_MINGW_ANSI_STDIO) +#define __USE_MINGW_ANSI_STDIO 1 +#endif /* MinGW */ + +#if defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS) + +#ifndef _WIN32_WINNT +#define _WIN32_WINNT 0x0601 /* Windows 7 */ +#endif /* _WIN32_WINNT */ + +#if !defined(_CRT_SECURE_NO_WARNINGS) +#define _CRT_SECURE_NO_WARNINGS +#endif /* _CRT_SECURE_NO_WARNINGS */ +#if !defined(UNICODE) +#define UNICODE +#endif /* UNICODE */ + +#if !defined(_NO_CRT_STDIO_INLINE) && MDBX_BUILD_SHARED_LIBRARY && !defined(xMDBX_TOOLS) && MDBX_WITHOUT_MSVC_CRT +#define _NO_CRT_STDIO_INLINE +#endif /* _NO_CRT_STDIO_INLINE */ + +#elif !defined(_POSIX_C_SOURCE) +#define _POSIX_C_SOURCE 200809L +#endif /* Windows */ + +#ifdef __cplusplus + +#ifndef NOMINMAX +#define NOMINMAX +#endif /* NOMINMAX */ + +/* Workaround for modern libstdc++ with CLANG < 4.x */ +#if defined(__SIZEOF_INT128__) && !defined(__GLIBCXX_TYPE_INT_N_0) && defined(__clang__) && __clang_major__ < 4 +#define __GLIBCXX_BITSIZE_INT_N_0 128 +#define __GLIBCXX_TYPE_INT_N_0 __int128 +#endif /* Workaround for modern libstdc++ with CLANG < 4.x */ + +#ifdef _MSC_VER +/* Workaround for MSVC' header `extern "C"` vs `std::` redefinition bug */ +#if defined(__SANITIZE_ADDRESS__) && !defined(_DISABLE_VECTOR_ANNOTATION) +#define _DISABLE_VECTOR_ANNOTATION +#endif /* _DISABLE_VECTOR_ANNOTATION */ +#endif /* _MSC_VER */ + +#endif /* __cplusplus */ #ifdef _MSC_VER #if _MSC_FULL_VER < 190024234 @@ -77,12 +112,8 @@ * and how to and where you can obtain the latest "Visual Studio 2015" build * with all fixes. */ -#error \ - "At least \"Microsoft C/C++ Compiler\" version 19.00.24234 (Visual Studio 2015 Update 3) is required." +#error "At least \"Microsoft C/C++ Compiler\" version 19.00.24234 (Visual Studio 2015 Update 3) is required." #endif -#ifndef _CRT_SECURE_NO_WARNINGS -#define _CRT_SECURE_NO_WARNINGS -#endif /* _CRT_SECURE_NO_WARNINGS */ #if _MSC_VER > 1800 #pragma warning(disable : 4464) /* relative include path contains '..' */ #endif @@ -90,124 +121,78 @@ #pragma warning(disable : 5045) /* will insert Spectre mitigation... */ #endif #if _MSC_VER > 1914 -#pragma warning( \ - disable : 5105) /* winbase.h(9531): warning C5105: macro expansion \ - producing 'defined' has undefined behavior */ +#pragma warning(disable : 5105) /* winbase.h(9531): warning C5105: macro expansion \ + producing 'defined' has undefined behavior */ +#endif +#if _MSC_VER < 1920 +/* avoid "error C2219: syntax error: type qualifier must be after '*'" */ +#define __restrict #endif #if _MSC_VER > 1930 #pragma warning(disable : 6235) /* is always a constant */ -#pragma warning(disable : 6237) /* is never evaluated and might \ +#pragma warning(disable : 6237) /* is never evaluated and might \ have side effects */ +#pragma warning(disable : 5286) /* implicit conversion from enum type 'type 1' to enum type 'type 2' */ +#pragma warning(disable : 5287) /* operands are different enum types 'type 1' and 'type 2' */ #endif #pragma warning(disable : 4710) /* 'xyz': function not inlined */ -#pragma warning(disable : 4711) /* function 'xyz' selected for automatic \ +#pragma warning(disable : 4711) /* function 'xyz' selected for automatic \ inline expansion */ -#pragma warning(disable : 4201) /* nonstandard extension used: nameless \ +#pragma warning(disable : 4201) /* nonstandard extension used: nameless \ struct/union */ #pragma warning(disable : 4702) /* unreachable code */ #pragma warning(disable : 4706) /* assignment within conditional expression */ #pragma warning(disable : 4127) /* conditional expression is constant */ -#pragma warning(disable : 4324) /* 'xyz': structure was padded due to \ +#pragma warning(disable : 4324) /* 'xyz': structure was padded due to \ alignment specifier */ #pragma warning(disable : 4310) /* cast truncates constant value */ -#pragma warning(disable : 4820) /* bytes padding added after data member for \ +#pragma warning(disable : 4820) /* bytes padding added after data member for \ alignment */ -#pragma warning(disable : 4548) /* expression before comma has no effect; \ +#pragma warning(disable : 4548) /* expression before comma has no effect; \ expected expression with side - effect */ -#pragma warning(disable : 4366) /* the result of the unary '&' operator may be \ +#pragma warning(disable : 4366) /* the result of the unary '&' operator may be \ unaligned */ -#pragma warning(disable : 4200) /* nonstandard extension used: zero-sized \ +#pragma warning(disable : 4200) /* nonstandard extension used: zero-sized \ array in struct/union */ -#pragma warning(disable : 4204) /* nonstandard extension used: non-constant \ +#pragma warning(disable : 4204) /* nonstandard extension used: non-constant \ aggregate initializer */ -#pragma warning( \ - disable : 4505) /* unreferenced local function has been removed */ -#endif /* _MSC_VER (warnings) */ +#pragma warning(disable : 4505) /* unreferenced local function has been removed */ +#endif /* _MSC_VER (warnings) */ #if defined(__GNUC__) && __GNUC__ < 9 #pragma GCC diagnostic ignored "-Wattributes" #endif /* GCC < 9 */ -#if (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) && \ - !defined(__USE_MINGW_ANSI_STDIO) -#define __USE_MINGW_ANSI_STDIO 1 -#endif /* MinGW */ - -#if (defined(_WIN32) || defined(_WIN64)) && !defined(UNICODE) -#define UNICODE -#endif /* UNICODE */ - -#include "mdbx.h" -/* - * Copyright 2015-2024 Leonid Yuriev - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - */ - - /*----------------------------------------------------------------------------*/ /* Microsoft compiler generates a lot of warning for self includes... */ #ifdef _MSC_VER #pragma warning(push, 1) -#pragma warning(disable : 4548) /* expression before comma has no effect; \ +#pragma warning(disable : 4548) /* expression before comma has no effect; \ expected expression with side - effect */ -#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ +#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ * semantics are not enabled. Specify /EHsc */ -#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ - * mode specified; termination on exception is \ +#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ + * mode specified; termination on exception is \ * not guaranteed. Specify /EHsc */ #endif /* _MSC_VER (warnings) */ -#if defined(_WIN32) || defined(_WIN64) -#if !defined(_CRT_SECURE_NO_WARNINGS) -#define _CRT_SECURE_NO_WARNINGS -#endif /* _CRT_SECURE_NO_WARNINGS */ -#if !defined(_NO_CRT_STDIO_INLINE) && MDBX_BUILD_SHARED_LIBRARY && \ - !defined(xMDBX_TOOLS) && MDBX_WITHOUT_MSVC_CRT -#define _NO_CRT_STDIO_INLINE -#endif -#elif !defined(_POSIX_C_SOURCE) -#define _POSIX_C_SOURCE 200809L -#endif /* Windows */ - /*----------------------------------------------------------------------------*/ /* basic C99 includes */ + #include #include #include #include #include +#include #include #include #include #include #include -#if (-6 & 5) || CHAR_BIT != 8 || UINT_MAX < 0xffffffff || ULONG_MAX % 0xFFFF -#error \ - "Sanity checking failed: Two's complement, reasonably sized integer types" -#endif - -#ifndef SSIZE_MAX -#define SSIZE_MAX INTPTR_MAX -#endif - -#if UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul || defined(_WIN64) -#define MDBX_WORDBITS 64 -#else -#define MDBX_WORDBITS 32 -#endif /* MDBX_WORDBITS */ - /*----------------------------------------------------------------------------*/ /* feature testing */ @@ -219,6 +204,14 @@ #define __has_include(x) (0) #endif +#ifndef __has_attribute +#define __has_attribute(x) (0) +#endif + +#ifndef __has_cpp_attribute +#define __has_cpp_attribute(x) 0 +#endif + #ifndef __has_feature #define __has_feature(x) (0) #endif @@ -241,8 +234,7 @@ #ifndef __GNUC_PREREQ #if defined(__GNUC__) && defined(__GNUC_MINOR__) -#define __GNUC_PREREQ(maj, min) \ - ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min)) +#define __GNUC_PREREQ(maj, min) ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min)) #else #define __GNUC_PREREQ(maj, min) (0) #endif @@ -250,8 +242,7 @@ #ifndef __CLANG_PREREQ #ifdef __clang__ -#define __CLANG_PREREQ(maj, min) \ - ((__clang_major__ << 16) + __clang_minor__ >= ((maj) << 16) + (min)) +#define __CLANG_PREREQ(maj, min) ((__clang_major__ << 16) + __clang_minor__ >= ((maj) << 16) + (min)) #else #define __CLANG_PREREQ(maj, min) (0) #endif @@ -259,13 +250,51 @@ #ifndef __GLIBC_PREREQ #if defined(__GLIBC__) && defined(__GLIBC_MINOR__) -#define __GLIBC_PREREQ(maj, min) \ - ((__GLIBC__ << 16) + __GLIBC_MINOR__ >= ((maj) << 16) + (min)) +#define __GLIBC_PREREQ(maj, min) ((__GLIBC__ << 16) + __GLIBC_MINOR__ >= ((maj) << 16) + (min)) #else #define __GLIBC_PREREQ(maj, min) (0) #endif #endif /* __GLIBC_PREREQ */ +/*----------------------------------------------------------------------------*/ +/* pre-requirements */ + +#if (-6 & 5) || CHAR_BIT != 8 || UINT_MAX < 0xffffffff || ULONG_MAX % 0xFFFF +#error "Sanity checking failed: Two's complement, reasonably sized integer types" +#endif + +#ifndef SSIZE_MAX +#define SSIZE_MAX INTPTR_MAX +#endif + +#if defined(__GNUC__) && !__GNUC_PREREQ(4, 2) +/* Actually libmdbx was not tested with compilers older than GCC 4.2. + * But you could ignore this warning at your own risk. + * In such case please don't rise up an issues related ONLY to old compilers. + */ +#warning "libmdbx required GCC >= 4.2" +#endif + +#if defined(__clang__) && !__CLANG_PREREQ(3, 8) +/* Actually libmdbx was not tested with CLANG older than 3.8. + * But you could ignore this warning at your own risk. + * In such case please don't rise up an issues related ONLY to old compilers. + */ +#warning "libmdbx required CLANG >= 3.8" +#endif + +#if defined(__GLIBC__) && !__GLIBC_PREREQ(2, 12) +/* Actually libmdbx was not tested with something older than glibc 2.12. + * But you could ignore this warning at your own risk. + * In such case please don't rise up an issues related ONLY to old systems. + */ +#warning "libmdbx was only tested with GLIBC >= 2.12." +#endif + +#ifdef __SANITIZE_THREAD__ +#warning "libmdbx don't compatible with ThreadSanitizer, you will get a lot of false-positive issues." +#endif /* __SANITIZE_THREAD__ */ + /*----------------------------------------------------------------------------*/ /* C11' alignas() */ @@ -295,8 +324,7 @@ #endif #endif /* __extern_C */ -#if !defined(nullptr) && !defined(__cplusplus) || \ - (__cplusplus < 201103L && !defined(_MSC_VER)) +#if !defined(nullptr) && !defined(__cplusplus) || (__cplusplus < 201103L && !defined(_MSC_VER)) #define nullptr NULL #endif @@ -308,9 +336,8 @@ #endif #endif /* Apple OSX & iOS */ -#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \ - defined(__BSD__) || defined(__bsdi__) || defined(__DragonFly__) || \ - defined(__APPLE__) || defined(__MACH__) +#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__BSD__) || defined(__bsdi__) || \ + defined(__DragonFly__) || defined(__APPLE__) || defined(__MACH__) #include #include #include @@ -327,8 +354,7 @@ #endif #else #include -#if !(defined(__sun) || defined(__SVR4) || defined(__svr4__) || \ - defined(_WIN32) || defined(_WIN64)) +#if !(defined(__sun) || defined(__SVR4) || defined(__svr4__) || defined(_WIN32) || defined(_WIN64)) #include #endif /* !Solaris */ #endif /* !xBSD */ @@ -382,12 +408,14 @@ __extern_C key_t ftok(const char *, int); #ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN #endif /* WIN32_LEAN_AND_MEAN */ -#include -#include #include #include #include +/* После подгрузки windows.h, чтобы избежать проблем со сборкой MINGW и т.п. */ +#include +#include + #else /*----------------------------------------------------------------------*/ #include @@ -415,11 +443,6 @@ __extern_C key_t ftok(const char *, int); #if __ANDROID_API__ >= 21 #include #endif -#if defined(_FILE_OFFSET_BITS) && _FILE_OFFSET_BITS != MDBX_WORDBITS -#error "_FILE_OFFSET_BITS != MDBX_WORDBITS" (_FILE_OFFSET_BITS != MDBX_WORDBITS) -#elif defined(__FILE_OFFSET_BITS) && __FILE_OFFSET_BITS != MDBX_WORDBITS -#error "__FILE_OFFSET_BITS != MDBX_WORDBITS" (__FILE_OFFSET_BITS != MDBX_WORDBITS) -#endif #endif /* Android */ #if defined(HAVE_SYS_STAT_H) || __has_include() @@ -435,43 +458,38 @@ __extern_C key_t ftok(const char *, int); /*----------------------------------------------------------------------------*/ /* Byteorder */ -#if defined(i386) || defined(__386) || defined(__i386) || defined(__i386__) || \ - defined(i486) || defined(__i486) || defined(__i486__) || defined(i586) || \ - defined(__i586) || defined(__i586__) || defined(i686) || \ - defined(__i686) || defined(__i686__) || defined(_M_IX86) || \ - defined(_X86_) || defined(__THW_INTEL__) || defined(__I86__) || \ - defined(__INTEL__) || defined(__x86_64) || defined(__x86_64__) || \ - defined(__amd64__) || defined(__amd64) || defined(_M_X64) || \ - defined(_M_AMD64) || defined(__IA32__) || defined(__INTEL__) +#if defined(i386) || defined(__386) || defined(__i386) || defined(__i386__) || defined(i486) || defined(__i486) || \ + defined(__i486__) || defined(i586) || defined(__i586) || defined(__i586__) || defined(i686) || defined(__i686) || \ + defined(__i686__) || defined(_M_IX86) || defined(_X86_) || defined(__THW_INTEL__) || defined(__I86__) || \ + defined(__INTEL__) || defined(__x86_64) || defined(__x86_64__) || defined(__amd64__) || defined(__amd64) || \ + defined(_M_X64) || defined(_M_AMD64) || defined(__IA32__) || defined(__INTEL__) #ifndef __ia32__ /* LY: define neutral __ia32__ for x86 and x86-64 */ #define __ia32__ 1 #endif /* __ia32__ */ -#if !defined(__amd64__) && \ - (defined(__x86_64) || defined(__x86_64__) || defined(__amd64) || \ - defined(_M_X64) || defined(_M_AMD64)) +#if !defined(__amd64__) && \ + (defined(__x86_64) || defined(__x86_64__) || defined(__amd64) || defined(_M_X64) || defined(_M_AMD64)) /* LY: define trusty __amd64__ for all AMD64/x86-64 arch */ #define __amd64__ 1 #endif /* __amd64__ */ #endif /* all x86 */ -#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || \ - !defined(__ORDER_BIG_ENDIAN__) +#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || !defined(__ORDER_BIG_ENDIAN__) -#if defined(__GLIBC__) || defined(__GNU_LIBRARY__) || \ - defined(__ANDROID_API__) || defined(HAVE_ENDIAN_H) || __has_include() +#if defined(__GLIBC__) || defined(__GNU_LIBRARY__) || defined(__ANDROID_API__) || defined(HAVE_ENDIAN_H) || \ + __has_include() #include -#elif defined(__APPLE__) || defined(__MACH__) || defined(__OpenBSD__) || \ - defined(HAVE_MACHINE_ENDIAN_H) || __has_include() +#elif defined(__APPLE__) || defined(__MACH__) || defined(__OpenBSD__) || defined(HAVE_MACHINE_ENDIAN_H) || \ + __has_include() #include #elif defined(HAVE_SYS_ISA_DEFS_H) || __has_include() #include -#elif (defined(HAVE_SYS_TYPES_H) && defined(HAVE_SYS_ENDIAN_H)) || \ +#elif (defined(HAVE_SYS_TYPES_H) && defined(HAVE_SYS_ENDIAN_H)) || \ (__has_include() && __has_include()) #include #include -#elif defined(__bsdi__) || defined(__DragonFly__) || defined(__FreeBSD__) || \ - defined(__NetBSD__) || defined(HAVE_SYS_PARAM_H) || __has_include() +#elif defined(__bsdi__) || defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || \ + defined(HAVE_SYS_PARAM_H) || __has_include() #include #endif /* OS */ @@ -487,27 +505,19 @@ __extern_C key_t ftok(const char *, int); #define __ORDER_LITTLE_ENDIAN__ 1234 #define __ORDER_BIG_ENDIAN__ 4321 -#if defined(__LITTLE_ENDIAN__) || \ - (defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)) || \ - defined(__ARMEL__) || defined(__THUMBEL__) || defined(__AARCH64EL__) || \ - defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \ - defined(_M_ARM) || defined(_M_ARM64) || defined(__e2k__) || \ - defined(__elbrus_4c__) || defined(__elbrus_8c__) || defined(__bfin__) || \ - defined(__BFIN__) || defined(__ia64__) || defined(_IA64) || \ - defined(__IA64__) || defined(__ia64) || defined(_M_IA64) || \ - defined(__itanium__) || defined(__ia32__) || defined(__CYGWIN__) || \ - defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) || \ - defined(__WINDOWS__) +#if defined(__LITTLE_ENDIAN__) || (defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)) || defined(__ARMEL__) || \ + defined(__THUMBEL__) || defined(__AARCH64EL__) || defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \ + defined(_M_ARM) || defined(_M_ARM64) || defined(__e2k__) || defined(__elbrus_4c__) || defined(__elbrus_8c__) || \ + defined(__bfin__) || defined(__BFIN__) || defined(__ia64__) || defined(_IA64) || defined(__IA64__) || \ + defined(__ia64) || defined(_M_IA64) || defined(__itanium__) || defined(__ia32__) || defined(__CYGWIN__) || \ + defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) || defined(__WINDOWS__) #define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__ -#elif defined(__BIG_ENDIAN__) || \ - (defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) || \ - defined(__ARMEB__) || defined(__THUMBEB__) || defined(__AARCH64EB__) || \ - defined(__MIPSEB__) || defined(_MIPSEB) || defined(__MIPSEB) || \ - defined(__m68k__) || defined(M68000) || defined(__hppa__) || \ - defined(__hppa) || defined(__HPPA__) || defined(__sparc__) || \ - defined(__sparc) || defined(__370__) || defined(__THW_370__) || \ - defined(__s390__) || defined(__s390x__) || defined(__SYSC_ZARCH__) +#elif defined(__BIG_ENDIAN__) || (defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) || defined(__ARMEB__) || \ + defined(__THUMBEB__) || defined(__AARCH64EB__) || defined(__MIPSEB__) || defined(_MIPSEB) || defined(__MIPSEB) || \ + defined(__m68k__) || defined(M68000) || defined(__hppa__) || defined(__hppa) || defined(__HPPA__) || \ + defined(__sparc__) || defined(__sparc) || defined(__370__) || defined(__THW_370__) || defined(__s390__) || \ + defined(__s390x__) || defined(__SYSC_ZARCH__) #define __BYTE_ORDER__ __ORDER_BIG_ENDIAN__ #else @@ -517,6 +527,12 @@ __extern_C key_t ftok(const char *, int); #endif #endif /* __BYTE_ORDER__ || __ORDER_LITTLE_ENDIAN__ || __ORDER_BIG_ENDIAN__ */ +#if UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul || defined(_WIN64) +#define MDBX_WORDBITS 64 +#else +#define MDBX_WORDBITS 32 +#endif /* MDBX_WORDBITS */ + /*----------------------------------------------------------------------------*/ /* Availability of CMOV or equivalent */ @@ -527,17 +543,14 @@ __extern_C key_t ftok(const char *, int); #define MDBX_HAVE_CMOV 1 #elif defined(__thumb__) || defined(__thumb) || defined(__TARGET_ARCH_THUMB) #define MDBX_HAVE_CMOV 0 -#elif defined(_M_ARM) || defined(_M_ARM64) || defined(__aarch64__) || \ - defined(__aarch64) || defined(__arm__) || defined(__arm) || \ - defined(__CC_ARM) +#elif defined(_M_ARM) || defined(_M_ARM64) || defined(__aarch64__) || defined(__aarch64) || defined(__arm__) || \ + defined(__arm) || defined(__CC_ARM) #define MDBX_HAVE_CMOV 1 -#elif (defined(__riscv__) || defined(__riscv64)) && \ - (defined(__riscv_b) || defined(__riscv_bitmanip)) +#elif (defined(__riscv__) || defined(__riscv64)) && (defined(__riscv_b) || defined(__riscv_bitmanip)) #define MDBX_HAVE_CMOV 1 -#elif defined(i686) || defined(__i686) || defined(__i686__) || \ - (defined(_M_IX86) && _M_IX86 > 600) || defined(__x86_64) || \ - defined(__x86_64__) || defined(__amd64__) || defined(__amd64) || \ - defined(_M_X64) || defined(_M_AMD64) +#elif defined(i686) || defined(__i686) || defined(__i686__) || (defined(_M_IX86) && _M_IX86 > 600) || \ + defined(__x86_64) || defined(__x86_64__) || defined(__amd64__) || defined(__amd64) || defined(_M_X64) || \ + defined(_M_AMD64) #define MDBX_HAVE_CMOV 1 #else #define MDBX_HAVE_CMOV 0 @@ -563,8 +576,7 @@ __extern_C key_t ftok(const char *, int); #endif #elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) #include -#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ - (defined(HP_IA64) || defined(__ia64)) +#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && (defined(HP_IA64) || defined(__ia64)) #include #elif defined(__IBMC__) && defined(__powerpc) #include @@ -586,29 +598,26 @@ __extern_C key_t ftok(const char *, int); #endif /* Compiler */ #if !defined(__noop) && !defined(_MSC_VER) -#define __noop \ - do { \ +#define __noop \ + do { \ } while (0) #endif /* __noop */ -#if defined(__fallthrough) && \ - (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) +#if defined(__fallthrough) && (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) #undef __fallthrough #endif /* __fallthrough workaround for MinGW */ #ifndef __fallthrough -#if defined(__cplusplus) && (__has_cpp_attribute(fallthrough) && \ - (!defined(__clang__) || __clang__ > 4)) || \ +#if defined(__cplusplus) && (__has_cpp_attribute(fallthrough) && (!defined(__clang__) || __clang__ > 4)) || \ __cplusplus >= 201703L #define __fallthrough [[fallthrough]] #elif __GNUC_PREREQ(8, 0) && defined(__cplusplus) && __cplusplus >= 201103L #define __fallthrough [[fallthrough]] -#elif __GNUC_PREREQ(7, 0) && \ - (!defined(__LCC__) || (__LCC__ == 124 && __LCC_MINOR__ >= 12) || \ - (__LCC__ == 125 && __LCC_MINOR__ >= 5) || (__LCC__ >= 126)) +#elif __GNUC_PREREQ(7, 0) && (!defined(__LCC__) || (__LCC__ == 124 && __LCC_MINOR__ >= 12) || \ + (__LCC__ == 125 && __LCC_MINOR__ >= 5) || (__LCC__ >= 126)) #define __fallthrough __attribute__((__fallthrough__)) -#elif defined(__clang__) && defined(__cplusplus) && __cplusplus >= 201103L && \ - __has_feature(cxx_attributes) && __has_warning("-Wimplicit-fallthrough") +#elif defined(__clang__) && defined(__cplusplus) && __cplusplus >= 201103L && __has_feature(cxx_attributes) && \ + __has_warning("-Wimplicit-fallthrough") #define __fallthrough [[clang::fallthrough]] #else #define __fallthrough @@ -621,8 +630,8 @@ __extern_C key_t ftok(const char *, int); #elif defined(_MSC_VER) #define __unreachable() __assume(0) #else -#define __unreachable() \ - do { \ +#define __unreachable() \ + do { \ } while (1) #endif #endif /* __unreachable */ @@ -631,9 +640,9 @@ __extern_C key_t ftok(const char *, int); #if defined(__GNUC__) || defined(__clang__) || __has_builtin(__builtin_prefetch) #define __prefetch(ptr) __builtin_prefetch(ptr) #else -#define __prefetch(ptr) \ - do { \ - (void)(ptr); \ +#define __prefetch(ptr) \ + do { \ + (void)(ptr); \ } while (0) #endif #endif /* __prefetch */ @@ -643,11 +652,11 @@ __extern_C key_t ftok(const char *, int); #endif /* offsetof */ #ifndef container_of -#define container_of(ptr, type, member) \ - ((type *)((char *)(ptr) - offsetof(type, member))) +#define container_of(ptr, type, member) ((type *)((char *)(ptr) - offsetof(type, member))) #endif /* container_of */ /*----------------------------------------------------------------------------*/ +/* useful attributes */ #ifndef __always_inline #if defined(__GNUC__) || __has_attribute(__always_inline__) @@ -715,8 +724,7 @@ __extern_C key_t ftok(const char *, int); #ifndef __hot #if defined(__OPTIMIZE__) -#if defined(__clang__) && !__has_attribute(__hot__) && \ - __has_attribute(__section__) && \ +#if defined(__clang__) && !__has_attribute(__hot__) && __has_attribute(__section__) && \ (defined(__linux__) || defined(__gnu_linux__)) /* just put frequently used functions in separate section */ #define __hot __attribute__((__section__("text.hot"))) __optimize("O3") @@ -732,8 +740,7 @@ __extern_C key_t ftok(const char *, int); #ifndef __cold #if defined(__OPTIMIZE__) -#if defined(__clang__) && !__has_attribute(__cold__) && \ - __has_attribute(__section__) && \ +#if defined(__clang__) && !__has_attribute(__cold__) && __has_attribute(__section__) && \ (defined(__linux__) || defined(__gnu_linux__)) /* just put infrequently used functions in separate section */ #define __cold __attribute__((__section__("text.unlikely"))) __optimize("Os") @@ -756,8 +763,7 @@ __extern_C key_t ftok(const char *, int); #endif /* __flatten */ #ifndef likely -#if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && \ - !defined(__COVERITY__) +#if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__) #define likely(cond) __builtin_expect(!!(cond), 1) #else #define likely(x) (!!(x)) @@ -765,8 +771,7 @@ __extern_C key_t ftok(const char *, int); #endif /* likely */ #ifndef unlikely -#if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && \ - !defined(__COVERITY__) +#if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__) #define unlikely(cond) __builtin_expect(!!(cond), 0) #else #define unlikely(x) (!!(x)) @@ -781,29 +786,41 @@ __extern_C key_t ftok(const char *, int); #endif #endif /* __anonymous_struct_extension__ */ -#ifndef expect_with_probability -#if defined(__builtin_expect_with_probability) || \ - __has_builtin(__builtin_expect_with_probability) || __GNUC_PREREQ(9, 0) -#define expect_with_probability(expr, value, prob) \ - __builtin_expect_with_probability(expr, value, prob) -#else -#define expect_with_probability(expr, value, prob) (expr) -#endif -#endif /* expect_with_probability */ - #ifndef MDBX_WEAK_IMPORT_ATTRIBUTE #ifdef WEAK_IMPORT_ATTRIBUTE #define MDBX_WEAK_IMPORT_ATTRIBUTE WEAK_IMPORT_ATTRIBUTE #elif __has_attribute(__weak__) && __has_attribute(__weak_import__) #define MDBX_WEAK_IMPORT_ATTRIBUTE __attribute__((__weak__, __weak_import__)) -#elif __has_attribute(__weak__) || \ - (defined(__GNUC__) && __GNUC__ >= 4 && defined(__ELF__)) +#elif __has_attribute(__weak__) || (defined(__GNUC__) && __GNUC__ >= 4 && defined(__ELF__)) #define MDBX_WEAK_IMPORT_ATTRIBUTE __attribute__((__weak__)) #else #define MDBX_WEAK_IMPORT_ATTRIBUTE #endif #endif /* MDBX_WEAK_IMPORT_ATTRIBUTE */ +#if !defined(__thread) && (defined(_MSC_VER) || defined(__DMC__)) +#define __thread __declspec(thread) +#endif /* __thread */ + +#ifndef MDBX_EXCLUDE_FOR_GPROF +#ifdef ENABLE_GPROF +#define MDBX_EXCLUDE_FOR_GPROF __attribute__((__no_instrument_function__, __no_profile_instrument_function__)) +#else +#define MDBX_EXCLUDE_FOR_GPROF +#endif /* ENABLE_GPROF */ +#endif /* MDBX_EXCLUDE_FOR_GPROF */ + +/*----------------------------------------------------------------------------*/ + +#ifndef expect_with_probability +#if defined(__builtin_expect_with_probability) || __has_builtin(__builtin_expect_with_probability) || \ + __GNUC_PREREQ(9, 0) +#define expect_with_probability(expr, value, prob) __builtin_expect_with_probability(expr, value, prob) +#else +#define expect_with_probability(expr, value, prob) (expr) +#endif +#endif /* expect_with_probability */ + #ifndef MDBX_GOOFY_MSVC_STATIC_ANALYZER #ifdef _PREFAST_ #define MDBX_GOOFY_MSVC_STATIC_ANALYZER 1 @@ -815,20 +832,27 @@ __extern_C key_t ftok(const char *, int); #if MDBX_GOOFY_MSVC_STATIC_ANALYZER || (defined(_MSC_VER) && _MSC_VER > 1919) #define MDBX_ANALYSIS_ASSUME(expr) __analysis_assume(expr) #ifdef _PREFAST_ -#define MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(warn_id) \ - __pragma(prefast(suppress : warn_id)) +#define MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(warn_id) __pragma(prefast(suppress : warn_id)) #else -#define MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(warn_id) \ - __pragma(warning(suppress : warn_id)) +#define MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(warn_id) __pragma(warning(suppress : warn_id)) #endif #else #define MDBX_ANALYSIS_ASSUME(expr) assert(expr) #define MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(warn_id) #endif /* MDBX_GOOFY_MSVC_STATIC_ANALYZER */ +#ifndef FLEXIBLE_ARRAY_MEMBERS +#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || (!defined(__cplusplus) && defined(_MSC_VER)) +#define FLEXIBLE_ARRAY_MEMBERS 1 +#else +#define FLEXIBLE_ARRAY_MEMBERS 0 +#endif +#endif /* FLEXIBLE_ARRAY_MEMBERS */ + /*----------------------------------------------------------------------------*/ +/* Valgrind and Address Sanitizer */ -#if defined(MDBX_USE_VALGRIND) +#if defined(ENABLE_MEMCHECK) #include #ifndef VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE /* LY: available since Valgrind 3.10 */ @@ -850,7 +874,7 @@ __extern_C key_t ftok(const char *, int); #define VALGRIND_CHECK_MEM_IS_ADDRESSABLE(a, s) (0) #define VALGRIND_CHECK_MEM_IS_DEFINED(a, s) (0) #define RUNNING_ON_VALGRIND (0) -#endif /* MDBX_USE_VALGRIND */ +#endif /* ENABLE_MEMCHECK */ #ifdef __SANITIZE_ADDRESS__ #include @@ -877,8 +901,7 @@ template char (&__ArraySizeHelper(T (&array)[N]))[N]; #define CONCAT(a, b) a##b #define XCONCAT(a, b) CONCAT(a, b) -#define MDBX_TETRAD(a, b, c, d) \ - ((uint32_t)(a) << 24 | (uint32_t)(b) << 16 | (uint32_t)(c) << 8 | (d)) +#define MDBX_TETRAD(a, b, c, d) ((uint32_t)(a) << 24 | (uint32_t)(b) << 16 | (uint32_t)(c) << 8 | (d)) #define MDBX_STRING_TETRAD(str) MDBX_TETRAD(str[0], str[1], str[2], str[3]) @@ -892,14 +915,13 @@ template char (&__ArraySizeHelper(T (&array)[N]))[N]; #elif defined(_MSC_VER) #include #define STATIC_ASSERT_MSG(expr, msg) _STATIC_ASSERT(expr) -#elif (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) || \ - __has_feature(c_static_assert) +#elif (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) || __has_feature(c_static_assert) #define STATIC_ASSERT_MSG(expr, msg) _Static_assert(expr, msg) #else -#define STATIC_ASSERT_MSG(expr, msg) \ - switch (0) { \ - case 0: \ - case (expr):; \ +#define STATIC_ASSERT_MSG(expr, msg) \ + switch (0) { \ + case 0: \ + case (expr):; \ } #endif #endif /* STATIC_ASSERT */ @@ -908,42 +930,37 @@ template char (&__ArraySizeHelper(T (&array)[N]))[N]; #define STATIC_ASSERT(expr) STATIC_ASSERT_MSG(expr, #expr) #endif -#ifndef __Wpedantic_format_voidptr -MDBX_MAYBE_UNUSED MDBX_PURE_FUNCTION static __inline const void * -__Wpedantic_format_voidptr(const void *ptr) { - return ptr; -} -#define __Wpedantic_format_voidptr(ARG) __Wpedantic_format_voidptr(ARG) -#endif /* __Wpedantic_format_voidptr */ +/*----------------------------------------------------------------------------*/ -#if defined(__GNUC__) && !__GNUC_PREREQ(4, 2) -/* Actually libmdbx was not tested with compilers older than GCC 4.2. - * But you could ignore this warning at your own risk. - * In such case please don't rise up an issues related ONLY to old compilers. - */ -#warning "libmdbx required GCC >= 4.2" -#endif +#if defined(_MSC_VER) && _MSC_VER >= 1900 +/* LY: MSVC 2015/2017/2019 has buggy/inconsistent PRIuPTR/PRIxPTR macros + * for internal format-args checker. */ +#undef PRIuPTR +#undef PRIiPTR +#undef PRIdPTR +#undef PRIxPTR +#define PRIuPTR "Iu" +#define PRIiPTR "Ii" +#define PRIdPTR "Id" +#define PRIxPTR "Ix" +#define PRIuSIZE "zu" +#define PRIiSIZE "zi" +#define PRIdSIZE "zd" +#define PRIxSIZE "zx" +#endif /* fix PRI*PTR for _MSC_VER */ -#if defined(__clang__) && !__CLANG_PREREQ(3, 8) -/* Actually libmdbx was not tested with CLANG older than 3.8. - * But you could ignore this warning at your own risk. - * In such case please don't rise up an issues related ONLY to old compilers. - */ -#warning "libmdbx required CLANG >= 3.8" -#endif +#ifndef PRIuSIZE +#define PRIuSIZE PRIuPTR +#define PRIiSIZE PRIiPTR +#define PRIdSIZE PRIdPTR +#define PRIxSIZE PRIxPTR +#endif /* PRI*SIZE macros for MSVC */ -#if defined(__GLIBC__) && !__GLIBC_PREREQ(2, 12) -/* Actually libmdbx was not tested with something older than glibc 2.12. - * But you could ignore this warning at your own risk. - * In such case please don't rise up an issues related ONLY to old systems. - */ -#warning "libmdbx was only tested with GLIBC >= 2.12." +#ifdef _MSC_VER +#pragma warning(pop) #endif -#ifdef __SANITIZE_THREAD__ -#warning \ - "libmdbx don't compatible with ThreadSanitizer, you will get a lot of false-positive issues." -#endif /* __SANITIZE_THREAD__ */ +/*----------------------------------------------------------------------------*/ #if __has_warning("-Wnested-anon-types") #if defined(__clang__) @@ -980,80 +997,34 @@ __Wpedantic_format_voidptr(const void *ptr) { #endif #endif /* -Walignment-reduction-ignored */ -#ifndef MDBX_EXCLUDE_FOR_GPROF -#ifdef ENABLE_GPROF -#define MDBX_EXCLUDE_FOR_GPROF \ - __attribute__((__no_instrument_function__, \ - __no_profile_instrument_function__)) +#ifdef xMDBX_ALLOY +/* Amalgamated build */ +#define MDBX_INTERNAL static #else -#define MDBX_EXCLUDE_FOR_GPROF -#endif /* ENABLE_GPROF */ -#endif /* MDBX_EXCLUDE_FOR_GPROF */ - -#ifdef __cplusplus -extern "C" { -#endif +/* Non-amalgamated build */ +#define MDBX_INTERNAL +#endif /* xMDBX_ALLOY */ -/* https://en.wikipedia.org/wiki/Operating_system_abstraction_layer */ +#include "mdbx.h" -/* - * Copyright 2015-2024 Leonid Yuriev - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - */ +/*----------------------------------------------------------------------------*/ +/* Basic constants and types */ +typedef struct iov_ctx iov_ctx_t; +/// /*----------------------------------------------------------------------------*/ -/* C11 Atomics */ - -#if defined(__cplusplus) && !defined(__STDC_NO_ATOMICS__) && __has_include() -#include -#define MDBX_HAVE_C11ATOMICS -#elif !defined(__cplusplus) && \ - (__STDC_VERSION__ >= 201112L || __has_extension(c_atomic)) && \ - !defined(__STDC_NO_ATOMICS__) && \ - (__GNUC_PREREQ(4, 9) || __CLANG_PREREQ(3, 8) || \ - !(defined(__GNUC__) || defined(__clang__))) -#include -#define MDBX_HAVE_C11ATOMICS -#elif defined(__GNUC__) || defined(__clang__) -#elif defined(_MSC_VER) -#pragma warning(disable : 4163) /* 'xyz': not available as an intrinsic */ -#pragma warning(disable : 4133) /* 'function': incompatible types - from \ - 'size_t' to 'LONGLONG' */ -#pragma warning(disable : 4244) /* 'return': conversion from 'LONGLONG' to \ - 'std::size_t', possible loss of data */ -#pragma warning(disable : 4267) /* 'function': conversion from 'size_t' to \ - 'long', possible loss of data */ -#pragma intrinsic(_InterlockedExchangeAdd, _InterlockedCompareExchange) -#pragma intrinsic(_InterlockedExchangeAdd64, _InterlockedCompareExchange64) -#elif defined(__APPLE__) -#include -#else -#error FIXME atomic-ops -#endif - -/*----------------------------------------------------------------------------*/ -/* Memory/Compiler barriers, cache coherence */ +/* Memory/Compiler barriers, cache coherence */ #if __has_include() #include -#elif defined(__mips) || defined(__mips__) || defined(__mips64) || \ - defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \ - defined(__MWERKS__) || defined(__sgi) +#elif defined(__mips) || defined(__mips__) || defined(__mips64) || defined(__mips64__) || defined(_M_MRX000) || \ + defined(_MIPS_) || defined(__MWERKS__) || defined(__sgi) /* MIPS should have explicit cache control */ #include #endif -MDBX_MAYBE_UNUSED static __inline void osal_compiler_barrier(void) { +MDBX_MAYBE_UNUSED static inline void osal_compiler_barrier(void) { #if defined(__clang__) || defined(__GNUC__) __asm__ __volatile__("" ::: "memory"); #elif defined(_MSC_VER) @@ -1062,18 +1033,16 @@ MDBX_MAYBE_UNUSED static __inline void osal_compiler_barrier(void) { __memory_barrier(); #elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) __compiler_barrier(); -#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ - (defined(HP_IA64) || defined(__ia64)) +#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && (defined(HP_IA64) || defined(__ia64)) _Asm_sched_fence(/* LY: no-arg meaning 'all expect ALU', e.g. 0x3D3D */); -#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \ - defined(__ppc64__) || defined(__powerpc64__) +#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || defined(__ppc64__) || defined(__powerpc64__) __fence(); #else #error "Could not guess the kind of compiler, please report to us." #endif } -MDBX_MAYBE_UNUSED static __inline void osal_memory_barrier(void) { +MDBX_MAYBE_UNUSED static inline void osal_memory_barrier(void) { #ifdef MDBX_HAVE_C11ATOMICS atomic_thread_fence(memory_order_seq_cst); #elif defined(__ATOMIC_SEQ_CST) @@ -1094,11 +1063,9 @@ MDBX_MAYBE_UNUSED static __inline void osal_memory_barrier(void) { #endif #elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) __machine_rw_barrier(); -#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ - (defined(HP_IA64) || defined(__ia64)) +#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && (defined(HP_IA64) || defined(__ia64)) _Asm_mf(); -#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \ - defined(__ppc64__) || defined(__powerpc64__) +#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || defined(__ppc64__) || defined(__powerpc64__) __lwsync(); #else #error "Could not guess the kind of compiler, please report to us." @@ -1113,7 +1080,7 @@ MDBX_MAYBE_UNUSED static __inline void osal_memory_barrier(void) { #define HAVE_SYS_TYPES_H typedef HANDLE osal_thread_t; typedef unsigned osal_thread_key_t; -#define MAP_FAILED NULL +#define MAP_FAILED nullptr #define HIGH_DWORD(v) ((DWORD)((sizeof(v) > 4) ? ((uint64_t)(v) >> 32) : 0)) #define THREAD_CALL WINAPI #define THREAD_RESULT DWORD @@ -1125,15 +1092,13 @@ typedef CRITICAL_SECTION osal_fastmutex_t; #if !defined(_MSC_VER) && !defined(__try) #define __try -#define __except(COND) if (false) +#define __except(COND) if (/* (void)(COND), */ false) #endif /* stub for MSVC's __try/__except */ #if MDBX_WITHOUT_MSVC_CRT #ifndef osal_malloc -static inline void *osal_malloc(size_t bytes) { - return HeapAlloc(GetProcessHeap(), 0, bytes); -} +static inline void *osal_malloc(size_t bytes) { return HeapAlloc(GetProcessHeap(), 0, bytes); } #endif /* osal_malloc */ #ifndef osal_calloc @@ -1144,8 +1109,7 @@ static inline void *osal_calloc(size_t nelem, size_t size) { #ifndef osal_realloc static inline void *osal_realloc(void *ptr, size_t bytes) { - return ptr ? HeapReAlloc(GetProcessHeap(), 0, ptr, bytes) - : HeapAlloc(GetProcessHeap(), 0, bytes); + return ptr ? HeapReAlloc(GetProcessHeap(), 0, ptr, bytes) : HeapAlloc(GetProcessHeap(), 0, bytes); } #endif /* osal_realloc */ @@ -1191,29 +1155,16 @@ typedef pthread_mutex_t osal_fastmutex_t; #endif /* Platform */ #if __GLIBC_PREREQ(2, 12) || defined(__FreeBSD__) || defined(malloc_usable_size) -/* malloc_usable_size() already provided */ +#define osal_malloc_usable_size(ptr) malloc_usable_size(ptr) #elif defined(__APPLE__) -#define malloc_usable_size(ptr) malloc_size(ptr) +#define osal_malloc_usable_size(ptr) malloc_size(ptr) #elif defined(_MSC_VER) && !MDBX_WITHOUT_MSVC_CRT -#define malloc_usable_size(ptr) _msize(ptr) -#endif /* malloc_usable_size */ +#define osal_malloc_usable_size(ptr) _msize(ptr) +#endif /* osal_malloc_usable_size */ /*----------------------------------------------------------------------------*/ /* OS abstraction layer stuff */ -MDBX_INTERNAL_VAR unsigned sys_pagesize; -MDBX_MAYBE_UNUSED MDBX_INTERNAL_VAR unsigned sys_pagesize_ln2, - sys_allocation_granularity; - -/* Get the size of a memory page for the system. - * This is the basic size that the platform's memory manager uses, and is - * fundamental to the use of memory-mapped files. */ -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline size_t -osal_syspagesize(void) { - assert(sys_pagesize > 0 && (sys_pagesize & (sys_pagesize - 1)) == 0); - return sys_pagesize; -} - #if defined(_WIN32) || defined(_WIN64) typedef wchar_t pathchar_t; #define MDBX_PRIsPATH "ls" @@ -1225,7 +1176,7 @@ typedef char pathchar_t; typedef struct osal_mmap { union { void *base; - struct MDBX_lockinfo *lck; + struct shared_lck *lck; }; mdbx_filehandle_t fd; size_t limit; /* mapping length, but NOT a size of file nor DB */ @@ -1236,25 +1187,6 @@ typedef struct osal_mmap { #endif } osal_mmap_t; -typedef union bin128 { - __anonymous_struct_extension__ struct { - uint64_t x, y; - }; - __anonymous_struct_extension__ struct { - uint32_t a, b, c, d; - }; -} bin128_t; - -#if defined(_WIN32) || defined(_WIN64) -typedef union osal_srwlock { - __anonymous_struct_extension__ struct { - long volatile readerCount; - long volatile writerCount; - }; - RTL_SRWLOCK native; -} osal_srwlock_t; -#endif /* Windows */ - #ifndef MDBX_HAVE_PWRITEV #if defined(_WIN32) || defined(_WIN64) @@ -1263,14 +1195,21 @@ typedef union osal_srwlock { #elif defined(__ANDROID_API__) #if __ANDROID_API__ < 24 +/* https://android-developers.googleblog.com/2017/09/introducing-android-native-development.html + * https://android.googlesource.com/platform/bionic/+/master/docs/32-bit-abi.md */ #define MDBX_HAVE_PWRITEV 0 +#if defined(_FILE_OFFSET_BITS) && _FILE_OFFSET_BITS != MDBX_WORDBITS +#error "_FILE_OFFSET_BITS != MDBX_WORDBITS and __ANDROID_API__ < 24" (_FILE_OFFSET_BITS != MDBX_WORDBITS) +#elif defined(__FILE_OFFSET_BITS) && __FILE_OFFSET_BITS != MDBX_WORDBITS +#error "__FILE_OFFSET_BITS != MDBX_WORDBITS and __ANDROID_API__ < 24" (__FILE_OFFSET_BITS != MDBX_WORDBITS) +#endif #else #define MDBX_HAVE_PWRITEV 1 #endif #elif defined(__APPLE__) || defined(__MACH__) || defined(_DARWIN_C_SOURCE) -#if defined(MAC_OS_X_VERSION_MIN_REQUIRED) && defined(MAC_OS_VERSION_11_0) && \ +#if defined(MAC_OS_X_VERSION_MIN_REQUIRED) && defined(MAC_OS_VERSION_11_0) && \ MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_VERSION_11_0 /* FIXME: add checks for IOS versions, etc */ #define MDBX_HAVE_PWRITEV 1 @@ -1288,20 +1227,20 @@ typedef union osal_srwlock { typedef struct ior_item { #if defined(_WIN32) || defined(_WIN64) OVERLAPPED ov; -#define ior_svg_gap4terminator 1 +#define ior_sgv_gap4terminator 1 #define ior_sgv_element FILE_SEGMENT_ELEMENT #else size_t offset; #if MDBX_HAVE_PWRITEV size_t sgvcnt; -#define ior_svg_gap4terminator 0 +#define ior_sgv_gap4terminator 0 #define ior_sgv_element struct iovec #endif /* MDBX_HAVE_PWRITEV */ #endif /* !Windows */ union { MDBX_val single; #if defined(ior_sgv_element) - ior_sgv_element sgv[1 + ior_svg_gap4terminator]; + ior_sgv_element sgv[1 + ior_sgv_gap4terminator]; #endif /* ior_sgv_element */ }; } ior_item_t; @@ -1337,45 +1276,33 @@ typedef struct osal_ioring { char *boundary; } osal_ioring_t; -#ifndef __cplusplus - /* Actually this is not ioring for now, but on the way. */ -MDBX_INTERNAL_FUNC int osal_ioring_create(osal_ioring_t * +MDBX_INTERNAL int osal_ioring_create(osal_ioring_t * #if defined(_WIN32) || defined(_WIN64) - , - bool enable_direct, - mdbx_filehandle_t overlapped_fd + , + bool enable_direct, mdbx_filehandle_t overlapped_fd #endif /* Windows */ ); -MDBX_INTERNAL_FUNC int osal_ioring_resize(osal_ioring_t *, size_t items); -MDBX_INTERNAL_FUNC void osal_ioring_destroy(osal_ioring_t *); -MDBX_INTERNAL_FUNC void osal_ioring_reset(osal_ioring_t *); -MDBX_INTERNAL_FUNC int osal_ioring_add(osal_ioring_t *ctx, const size_t offset, - void *data, const size_t bytes); +MDBX_INTERNAL int osal_ioring_resize(osal_ioring_t *, size_t items); +MDBX_INTERNAL void osal_ioring_destroy(osal_ioring_t *); +MDBX_INTERNAL void osal_ioring_reset(osal_ioring_t *); +MDBX_INTERNAL int osal_ioring_add(osal_ioring_t *ctx, const size_t offset, void *data, const size_t bytes); typedef struct osal_ioring_write_result { int err; unsigned wops; } osal_ioring_write_result_t; -MDBX_INTERNAL_FUNC osal_ioring_write_result_t -osal_ioring_write(osal_ioring_t *ior, mdbx_filehandle_t fd); +MDBX_INTERNAL osal_ioring_write_result_t osal_ioring_write(osal_ioring_t *ior, mdbx_filehandle_t fd); -typedef struct iov_ctx iov_ctx_t; -MDBX_INTERNAL_FUNC void osal_ioring_walk( - osal_ioring_t *ior, iov_ctx_t *ctx, - void (*callback)(iov_ctx_t *ctx, size_t offset, void *data, size_t bytes)); +MDBX_INTERNAL void osal_ioring_walk(osal_ioring_t *ior, iov_ctx_t *ctx, + void (*callback)(iov_ctx_t *ctx, size_t offset, void *data, size_t bytes)); -MDBX_MAYBE_UNUSED static inline unsigned -osal_ioring_left(const osal_ioring_t *ior) { - return ior->slots_left; -} +MDBX_MAYBE_UNUSED static inline unsigned osal_ioring_left(const osal_ioring_t *ior) { return ior->slots_left; } -MDBX_MAYBE_UNUSED static inline unsigned -osal_ioring_used(const osal_ioring_t *ior) { +MDBX_MAYBE_UNUSED static inline unsigned osal_ioring_used(const osal_ioring_t *ior) { return ior->allocated - ior->slots_left; } -MDBX_MAYBE_UNUSED static inline int -osal_ioring_prepare(osal_ioring_t *ior, size_t items, size_t bytes) { +MDBX_MAYBE_UNUSED static inline int osal_ioring_prepare(osal_ioring_t *ior, size_t items, size_t bytes) { items = (items > 32) ? items : 32; #if defined(_WIN32) || defined(_WIN64) if (ior->direct) { @@ -1394,14 +1321,12 @@ osal_ioring_prepare(osal_ioring_t *ior, size_t items, size_t bytes) { /*----------------------------------------------------------------------------*/ /* libc compatibility stuff */ -#if (!defined(__GLIBC__) && __GLIBC_PREREQ(2, 1)) && \ - (defined(_GNU_SOURCE) || defined(_BSD_SOURCE)) +#if (!defined(__GLIBC__) && __GLIBC_PREREQ(2, 1)) && (defined(_GNU_SOURCE) || defined(_BSD_SOURCE)) #define osal_asprintf asprintf #define osal_vasprintf vasprintf #else -MDBX_MAYBE_UNUSED MDBX_INTERNAL_FUNC - MDBX_PRINTF_ARGS(2, 3) int osal_asprintf(char **strp, const char *fmt, ...); -MDBX_INTERNAL_FUNC int osal_vasprintf(char **strp, const char *fmt, va_list ap); +MDBX_MAYBE_UNUSED MDBX_INTERNAL MDBX_PRINTF_ARGS(2, 3) int osal_asprintf(char **strp, const char *fmt, ...); +MDBX_INTERNAL int osal_vasprintf(char **strp, const char *fmt, va_list ap); #endif #if !defined(MADV_DODUMP) && defined(MADV_CORE) @@ -1412,8 +1337,7 @@ MDBX_INTERNAL_FUNC int osal_vasprintf(char **strp, const char *fmt, va_list ap); #define MADV_DONTDUMP MADV_NOCORE #endif /* MADV_NOCORE -> MADV_DONTDUMP */ -MDBX_MAYBE_UNUSED MDBX_INTERNAL_FUNC void osal_jitter(bool tiny); -MDBX_MAYBE_UNUSED static __inline void jitter4testing(bool tiny); +MDBX_MAYBE_UNUSED MDBX_INTERNAL void osal_jitter(bool tiny); /* max bytes to write in one call */ #if defined(_WIN64) @@ -1423,14 +1347,12 @@ MDBX_MAYBE_UNUSED static __inline void jitter4testing(bool tiny); #else #define MAX_WRITE UINT32_C(0x3f000000) -#if defined(F_GETLK64) && defined(F_SETLK64) && defined(F_SETLKW64) && \ - !defined(__ANDROID_API__) +#if defined(F_GETLK64) && defined(F_SETLK64) && defined(F_SETLKW64) && !defined(__ANDROID_API__) #define MDBX_F_SETLK F_SETLK64 #define MDBX_F_SETLKW F_SETLKW64 #define MDBX_F_GETLK F_GETLK64 -#if (__GLIBC_PREREQ(2, 28) && \ - (defined(__USE_LARGEFILE64) || defined(__LARGEFILE64_SOURCE) || \ - defined(_USE_LARGEFILE64) || defined(_LARGEFILE64_SOURCE))) || \ +#if (__GLIBC_PREREQ(2, 28) && (defined(__USE_LARGEFILE64) || defined(__LARGEFILE64_SOURCE) || \ + defined(_USE_LARGEFILE64) || defined(_LARGEFILE64_SOURCE))) || \ defined(fcntl64) #define MDBX_FCNTL fcntl64 #else @@ -1448,8 +1370,7 @@ MDBX_MAYBE_UNUSED static __inline void jitter4testing(bool tiny); #define MDBX_STRUCT_FLOCK struct flock #endif /* MDBX_F_SETLK, MDBX_F_SETLKW, MDBX_F_GETLK */ -#if defined(F_OFD_SETLK64) && defined(F_OFD_SETLKW64) && \ - defined(F_OFD_GETLK64) && !defined(__ANDROID_API__) +#if defined(F_OFD_SETLK64) && defined(F_OFD_SETLKW64) && defined(F_OFD_GETLK64) && !defined(__ANDROID_API__) #define MDBX_F_OFD_SETLK F_OFD_SETLK64 #define MDBX_F_OFD_SETLKW F_OFD_SETLKW64 #define MDBX_F_OFD_GETLK F_OFD_GETLK64 @@ -1458,23 +1379,17 @@ MDBX_MAYBE_UNUSED static __inline void jitter4testing(bool tiny); #define MDBX_F_OFD_SETLKW F_OFD_SETLKW #define MDBX_F_OFD_GETLK F_OFD_GETLK #ifndef OFF_T_MAX -#define OFF_T_MAX \ - (((sizeof(off_t) > 4) ? INT64_MAX : INT32_MAX) & ~(size_t)0xFffff) +#define OFF_T_MAX (((sizeof(off_t) > 4) ? INT64_MAX : INT32_MAX) & ~(size_t)0xFffff) #endif /* OFF_T_MAX */ #endif /* MDBX_F_OFD_SETLK64, MDBX_F_OFD_SETLKW64, MDBX_F_OFD_GETLK64 */ -#endif - -#if defined(__linux__) || defined(__gnu_linux__) -MDBX_INTERNAL_VAR uint32_t linux_kernel_version; -MDBX_INTERNAL_VAR bool mdbx_RunningOnWSL1 /* Windows Subsystem 1 for Linux */; -#endif /* Linux */ +#endif /* !Windows */ #ifndef osal_strdup LIBMDBX_API char *osal_strdup(const char *str); #endif -MDBX_MAYBE_UNUSED static __inline int osal_get_errno(void) { +MDBX_MAYBE_UNUSED static inline int osal_get_errno(void) { #if defined(_WIN32) || defined(_WIN64) DWORD rc = GetLastError(); #else @@ -1484,40 +1399,32 @@ MDBX_MAYBE_UNUSED static __inline int osal_get_errno(void) { } #ifndef osal_memalign_alloc -MDBX_INTERNAL_FUNC int osal_memalign_alloc(size_t alignment, size_t bytes, - void **result); +MDBX_INTERNAL int osal_memalign_alloc(size_t alignment, size_t bytes, void **result); #endif #ifndef osal_memalign_free -MDBX_INTERNAL_FUNC void osal_memalign_free(void *ptr); -#endif - -MDBX_INTERNAL_FUNC int osal_condpair_init(osal_condpair_t *condpair); -MDBX_INTERNAL_FUNC int osal_condpair_lock(osal_condpair_t *condpair); -MDBX_INTERNAL_FUNC int osal_condpair_unlock(osal_condpair_t *condpair); -MDBX_INTERNAL_FUNC int osal_condpair_signal(osal_condpair_t *condpair, - bool part); -MDBX_INTERNAL_FUNC int osal_condpair_wait(osal_condpair_t *condpair, bool part); -MDBX_INTERNAL_FUNC int osal_condpair_destroy(osal_condpair_t *condpair); - -MDBX_INTERNAL_FUNC int osal_fastmutex_init(osal_fastmutex_t *fastmutex); -MDBX_INTERNAL_FUNC int osal_fastmutex_acquire(osal_fastmutex_t *fastmutex); -MDBX_INTERNAL_FUNC int osal_fastmutex_release(osal_fastmutex_t *fastmutex); -MDBX_INTERNAL_FUNC int osal_fastmutex_destroy(osal_fastmutex_t *fastmutex); - -MDBX_INTERNAL_FUNC int osal_pwritev(mdbx_filehandle_t fd, struct iovec *iov, - size_t sgvcnt, uint64_t offset); -MDBX_INTERNAL_FUNC int osal_pread(mdbx_filehandle_t fd, void *buf, size_t count, - uint64_t offset); -MDBX_INTERNAL_FUNC int osal_pwrite(mdbx_filehandle_t fd, const void *buf, - size_t count, uint64_t offset); -MDBX_INTERNAL_FUNC int osal_write(mdbx_filehandle_t fd, const void *buf, - size_t count); - -MDBX_INTERNAL_FUNC int -osal_thread_create(osal_thread_t *thread, - THREAD_RESULT(THREAD_CALL *start_routine)(void *), - void *arg); -MDBX_INTERNAL_FUNC int osal_thread_join(osal_thread_t thread); +MDBX_INTERNAL void osal_memalign_free(void *ptr); +#endif + +MDBX_INTERNAL int osal_condpair_init(osal_condpair_t *condpair); +MDBX_INTERNAL int osal_condpair_lock(osal_condpair_t *condpair); +MDBX_INTERNAL int osal_condpair_unlock(osal_condpair_t *condpair); +MDBX_INTERNAL int osal_condpair_signal(osal_condpair_t *condpair, bool part); +MDBX_INTERNAL int osal_condpair_wait(osal_condpair_t *condpair, bool part); +MDBX_INTERNAL int osal_condpair_destroy(osal_condpair_t *condpair); + +MDBX_INTERNAL int osal_fastmutex_init(osal_fastmutex_t *fastmutex); +MDBX_INTERNAL int osal_fastmutex_acquire(osal_fastmutex_t *fastmutex); +MDBX_INTERNAL int osal_fastmutex_release(osal_fastmutex_t *fastmutex); +MDBX_INTERNAL int osal_fastmutex_destroy(osal_fastmutex_t *fastmutex); + +MDBX_INTERNAL int osal_pwritev(mdbx_filehandle_t fd, struct iovec *iov, size_t sgvcnt, uint64_t offset); +MDBX_INTERNAL int osal_pread(mdbx_filehandle_t fd, void *buf, size_t count, uint64_t offset); +MDBX_INTERNAL int osal_pwrite(mdbx_filehandle_t fd, const void *buf, size_t count, uint64_t offset); +MDBX_INTERNAL int osal_write(mdbx_filehandle_t fd, const void *buf, size_t count); + +MDBX_INTERNAL int osal_thread_create(osal_thread_t *thread, THREAD_RESULT(THREAD_CALL *start_routine)(void *), + void *arg); +MDBX_INTERNAL int osal_thread_join(osal_thread_t thread); enum osal_syncmode_bits { MDBX_SYNC_NONE = 0, @@ -1527,11 +1434,10 @@ enum osal_syncmode_bits { MDBX_SYNC_IODQ = 8 }; -MDBX_INTERNAL_FUNC int osal_fsync(mdbx_filehandle_t fd, - const enum osal_syncmode_bits mode_bits); -MDBX_INTERNAL_FUNC int osal_ftruncate(mdbx_filehandle_t fd, uint64_t length); -MDBX_INTERNAL_FUNC int osal_fseek(mdbx_filehandle_t fd, uint64_t pos); -MDBX_INTERNAL_FUNC int osal_filesize(mdbx_filehandle_t fd, uint64_t *length); +MDBX_INTERNAL int osal_fsync(mdbx_filehandle_t fd, const enum osal_syncmode_bits mode_bits); +MDBX_INTERNAL int osal_ftruncate(mdbx_filehandle_t fd, uint64_t length); +MDBX_INTERNAL int osal_fseek(mdbx_filehandle_t fd, uint64_t pos); +MDBX_INTERNAL int osal_filesize(mdbx_filehandle_t fd, uint64_t *length); enum osal_openfile_purpose { MDBX_OPEN_DXB_READ, @@ -1546,7 +1452,7 @@ enum osal_openfile_purpose { MDBX_OPEN_DELETE }; -MDBX_MAYBE_UNUSED static __inline bool osal_isdirsep(pathchar_t c) { +MDBX_MAYBE_UNUSED static inline bool osal_isdirsep(pathchar_t c) { return #if defined(_WIN32) || defined(_WIN64) c == '\\' || @@ -1554,50 +1460,39 @@ MDBX_MAYBE_UNUSED static __inline bool osal_isdirsep(pathchar_t c) { c == '/'; } -MDBX_INTERNAL_FUNC bool osal_pathequal(const pathchar_t *l, const pathchar_t *r, - size_t len); -MDBX_INTERNAL_FUNC pathchar_t *osal_fileext(const pathchar_t *pathname, - size_t len); -MDBX_INTERNAL_FUNC int osal_fileexists(const pathchar_t *pathname); -MDBX_INTERNAL_FUNC int osal_openfile(const enum osal_openfile_purpose purpose, - const MDBX_env *env, - const pathchar_t *pathname, - mdbx_filehandle_t *fd, - mdbx_mode_t unix_mode_bits); -MDBX_INTERNAL_FUNC int osal_closefile(mdbx_filehandle_t fd); -MDBX_INTERNAL_FUNC int osal_removefile(const pathchar_t *pathname); -MDBX_INTERNAL_FUNC int osal_removedirectory(const pathchar_t *pathname); -MDBX_INTERNAL_FUNC int osal_is_pipe(mdbx_filehandle_t fd); -MDBX_INTERNAL_FUNC int osal_lockfile(mdbx_filehandle_t fd, bool wait); +MDBX_INTERNAL bool osal_pathequal(const pathchar_t *l, const pathchar_t *r, size_t len); +MDBX_INTERNAL pathchar_t *osal_fileext(const pathchar_t *pathname, size_t len); +MDBX_INTERNAL int osal_fileexists(const pathchar_t *pathname); +MDBX_INTERNAL int osal_openfile(const enum osal_openfile_purpose purpose, const MDBX_env *env, + const pathchar_t *pathname, mdbx_filehandle_t *fd, mdbx_mode_t unix_mode_bits); +MDBX_INTERNAL int osal_closefile(mdbx_filehandle_t fd); +MDBX_INTERNAL int osal_removefile(const pathchar_t *pathname); +MDBX_INTERNAL int osal_removedirectory(const pathchar_t *pathname); +MDBX_INTERNAL int osal_is_pipe(mdbx_filehandle_t fd); +MDBX_INTERNAL int osal_lockfile(mdbx_filehandle_t fd, bool wait); #define MMAP_OPTION_TRUNCATE 1 #define MMAP_OPTION_SEMAPHORE 2 -MDBX_INTERNAL_FUNC int osal_mmap(const int flags, osal_mmap_t *map, size_t size, - const size_t limit, const unsigned options); -MDBX_INTERNAL_FUNC int osal_munmap(osal_mmap_t *map); +MDBX_INTERNAL int osal_mmap(const int flags, osal_mmap_t *map, size_t size, const size_t limit, const unsigned options, + const pathchar_t *pathname4logging); +MDBX_INTERNAL int osal_munmap(osal_mmap_t *map); #define MDBX_MRESIZE_MAY_MOVE 0x00000100 #define MDBX_MRESIZE_MAY_UNMAP 0x00000200 -MDBX_INTERNAL_FUNC int osal_mresize(const int flags, osal_mmap_t *map, - size_t size, size_t limit); +MDBX_INTERNAL int osal_mresize(const int flags, osal_mmap_t *map, size_t size, size_t limit); #if defined(_WIN32) || defined(_WIN64) typedef struct { unsigned limit, count; HANDLE handles[31]; } mdbx_handle_array_t; -MDBX_INTERNAL_FUNC int -osal_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array); -MDBX_INTERNAL_FUNC int -osal_resume_threads_after_remap(mdbx_handle_array_t *array); +MDBX_INTERNAL int osal_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array); +MDBX_INTERNAL int osal_resume_threads_after_remap(mdbx_handle_array_t *array); #endif /* Windows */ -MDBX_INTERNAL_FUNC int osal_msync(const osal_mmap_t *map, size_t offset, - size_t length, - enum osal_syncmode_bits mode_bits); -MDBX_INTERNAL_FUNC int osal_check_fs_rdonly(mdbx_filehandle_t handle, - const pathchar_t *pathname, - int err); -MDBX_INTERNAL_FUNC int osal_check_fs_incore(mdbx_filehandle_t handle); - -MDBX_MAYBE_UNUSED static __inline uint32_t osal_getpid(void) { +MDBX_INTERNAL int osal_msync(const osal_mmap_t *map, size_t offset, size_t length, enum osal_syncmode_bits mode_bits); +MDBX_INTERNAL int osal_check_fs_rdonly(mdbx_filehandle_t handle, const pathchar_t *pathname, int err); +MDBX_INTERNAL int osal_check_fs_incore(mdbx_filehandle_t handle); +MDBX_INTERNAL int osal_check_fs_local(mdbx_filehandle_t handle, int flags); + +MDBX_MAYBE_UNUSED static inline uint32_t osal_getpid(void) { STATIC_ASSERT(sizeof(mdbx_pid_t) <= sizeof(uint32_t)); #if defined(_WIN32) || defined(_WIN64) return GetCurrentProcessId(); @@ -1607,7 +1502,7 @@ MDBX_MAYBE_UNUSED static __inline uint32_t osal_getpid(void) { #endif } -MDBX_MAYBE_UNUSED static __inline uintptr_t osal_thread_self(void) { +MDBX_MAYBE_UNUSED static inline uintptr_t osal_thread_self(void) { mdbx_tid_t thunk; STATIC_ASSERT(sizeof(uintptr_t) >= sizeof(thunk)); #if defined(_WIN32) || defined(_WIN64) @@ -1620,274 +1515,51 @@ MDBX_MAYBE_UNUSED static __inline uintptr_t osal_thread_self(void) { #if !defined(_WIN32) && !defined(_WIN64) #if defined(__ANDROID_API__) || defined(ANDROID) || defined(BIONIC) -MDBX_INTERNAL_FUNC int osal_check_tid4bionic(void); +MDBX_INTERNAL int osal_check_tid4bionic(void); #else -static __inline int osal_check_tid4bionic(void) { return 0; } +static inline int osal_check_tid4bionic(void) { return 0; } #endif /* __ANDROID_API__ || ANDROID) || BIONIC */ -MDBX_MAYBE_UNUSED static __inline int -osal_pthread_mutex_lock(pthread_mutex_t *mutex) { +MDBX_MAYBE_UNUSED static inline int osal_pthread_mutex_lock(pthread_mutex_t *mutex) { int err = osal_check_tid4bionic(); return unlikely(err) ? err : pthread_mutex_lock(mutex); } #endif /* !Windows */ -MDBX_INTERNAL_FUNC uint64_t osal_monotime(void); -MDBX_INTERNAL_FUNC uint64_t osal_cputime(size_t *optional_page_faults); -MDBX_INTERNAL_FUNC uint64_t osal_16dot16_to_monotime(uint32_t seconds_16dot16); -MDBX_INTERNAL_FUNC uint32_t osal_monotime_to_16dot16(uint64_t monotime); +MDBX_INTERNAL uint64_t osal_monotime(void); +MDBX_INTERNAL uint64_t osal_cputime(size_t *optional_page_faults); +MDBX_INTERNAL uint64_t osal_16dot16_to_monotime(uint32_t seconds_16dot16); +MDBX_INTERNAL uint32_t osal_monotime_to_16dot16(uint64_t monotime); -MDBX_MAYBE_UNUSED static inline uint32_t -osal_monotime_to_16dot16_noUnderflow(uint64_t monotime) { +MDBX_MAYBE_UNUSED static inline uint32_t osal_monotime_to_16dot16_noUnderflow(uint64_t monotime) { uint32_t seconds_16dot16 = osal_monotime_to_16dot16(monotime); return seconds_16dot16 ? seconds_16dot16 : /* fix underflow */ (monotime > 0); } -MDBX_INTERNAL_FUNC bin128_t osal_bootid(void); /*----------------------------------------------------------------------------*/ -/* lck stuff */ - -/// \brief Initialization of synchronization primitives linked with MDBX_env -/// instance both in LCK-file and within the current process. -/// \param -/// global_uniqueness_flag = true - denotes that there are no other processes -/// working with DB and LCK-file. Thus the function MUST initialize -/// shared synchronization objects in memory-mapped LCK-file. -/// global_uniqueness_flag = false - denotes that at least one process is -/// already working with DB and LCK-file, including the case when DB -/// has already been opened in the current process. Thus the function -/// MUST NOT initialize shared synchronization objects in memory-mapped -/// LCK-file that are already in use. -/// \return Error code or zero on success. -MDBX_INTERNAL_FUNC int osal_lck_init(MDBX_env *env, - MDBX_env *inprocess_neighbor, - int global_uniqueness_flag); - -/// \brief Disconnects from shared interprocess objects and destructs -/// synchronization objects linked with MDBX_env instance -/// within the current process. -/// \param -/// inprocess_neighbor = NULL - if the current process does not have other -/// instances of MDBX_env linked with the DB being closed. -/// Thus the function MUST check for other processes working with DB or -/// LCK-file, and keep or destroy shared synchronization objects in -/// memory-mapped LCK-file depending on the result. -/// inprocess_neighbor = not-NULL - pointer to another instance of MDBX_env -/// (anyone of there is several) working with DB or LCK-file within the -/// current process. Thus the function MUST NOT try to acquire exclusive -/// lock and/or try to destruct shared synchronization objects linked with -/// DB or LCK-file. Moreover, the implementation MUST ensure correct work -/// of other instances of MDBX_env within the current process, e.g. -/// restore POSIX-fcntl locks after the closing of file descriptors. -/// \return Error code (MDBX_PANIC) or zero on success. -MDBX_INTERNAL_FUNC int osal_lck_destroy(MDBX_env *env, - MDBX_env *inprocess_neighbor); - -/// \brief Connects to shared interprocess locking objects and tries to acquire -/// the maximum lock level (shared if exclusive is not available) -/// Depending on implementation or/and platform (Windows) this function may -/// acquire the non-OS super-level lock (e.g. for shared synchronization -/// objects initialization), which will be downgraded to OS-exclusive or -/// shared via explicit calling of osal_lck_downgrade(). -/// \return -/// MDBX_RESULT_TRUE (-1) - if an exclusive lock was acquired and thus -/// the current process is the first and only after the last use of DB. -/// MDBX_RESULT_FALSE (0) - if a shared lock was acquired and thus -/// DB has already been opened and now is used by other processes. -/// Otherwise (not 0 and not -1) - error code. -MDBX_INTERNAL_FUNC int osal_lck_seize(MDBX_env *env); - -/// \brief Downgrades the level of initially acquired lock to -/// operational level specified by argument. The reason for such downgrade: -/// - unblocking of other processes that are waiting for access, i.e. -/// if (env->me_flags & MDBX_EXCLUSIVE) != 0, then other processes -/// should be made aware that access is unavailable rather than -/// wait for it. -/// - freeing locks that interfere file operation (especially for Windows) -/// (env->me_flags & MDBX_EXCLUSIVE) == 0 - downgrade to shared lock. -/// (env->me_flags & MDBX_EXCLUSIVE) != 0 - downgrade to exclusive -/// operational lock. -/// \return Error code or zero on success -MDBX_INTERNAL_FUNC int osal_lck_downgrade(MDBX_env *env); - -/// \brief Locks LCK-file or/and table of readers for (de)registering. -/// \return Error code or zero on success -MDBX_INTERNAL_FUNC int osal_rdt_lock(MDBX_env *env); - -/// \brief Unlocks LCK-file or/and table of readers after (de)registering. -MDBX_INTERNAL_FUNC void osal_rdt_unlock(MDBX_env *env); - -/// \brief Acquires lock for DB change (on writing transaction start) -/// Reading transactions will not be blocked. -/// Declared as LIBMDBX_API because it is used in mdbx_chk. -/// \return Error code or zero on success -LIBMDBX_API int mdbx_txn_lock(MDBX_env *env, bool dont_wait); - -/// \brief Releases lock once DB changes is made (after writing transaction -/// has finished). -/// Declared as LIBMDBX_API because it is used in mdbx_chk. -LIBMDBX_API void mdbx_txn_unlock(MDBX_env *env); - -/// \brief Sets alive-flag of reader presence (indicative lock) for PID of -/// the current process. The function does no more than needed for -/// the correct working of osal_rpid_check() in other processes. -/// \return Error code or zero on success -MDBX_INTERNAL_FUNC int osal_rpid_set(MDBX_env *env); - -/// \brief Resets alive-flag of reader presence (indicative lock) -/// for PID of the current process. The function does no more than needed -/// for the correct working of osal_rpid_check() in other processes. -/// \return Error code or zero on success -MDBX_INTERNAL_FUNC int osal_rpid_clear(MDBX_env *env); - -/// \brief Checks for reading process status with the given pid with help of -/// alive-flag of presence (indicative lock) or using another way. -/// \return -/// MDBX_RESULT_TRUE (-1) - if the reader process with the given PID is alive -/// and working with DB (indicative lock is present). -/// MDBX_RESULT_FALSE (0) - if the reader process with the given PID is absent -/// or not working with DB (indicative lock is not present). -/// Otherwise (not 0 and not -1) - error code. -MDBX_INTERNAL_FUNC int osal_rpid_check(MDBX_env *env, uint32_t pid); - -#if defined(_WIN32) || defined(_WIN64) - -MDBX_INTERNAL_FUNC int osal_mb2w(const char *const src, wchar_t **const pdst); - -typedef void(WINAPI *osal_srwlock_t_function)(osal_srwlock_t *); -MDBX_INTERNAL_VAR osal_srwlock_t_function osal_srwlock_Init, - osal_srwlock_AcquireShared, osal_srwlock_ReleaseShared, - osal_srwlock_AcquireExclusive, osal_srwlock_ReleaseExclusive; - -#if _WIN32_WINNT < 0x0600 /* prior to Windows Vista */ -typedef enum _FILE_INFO_BY_HANDLE_CLASS { - FileBasicInfo, - FileStandardInfo, - FileNameInfo, - FileRenameInfo, - FileDispositionInfo, - FileAllocationInfo, - FileEndOfFileInfo, - FileStreamInfo, - FileCompressionInfo, - FileAttributeTagInfo, - FileIdBothDirectoryInfo, - FileIdBothDirectoryRestartInfo, - FileIoPriorityHintInfo, - FileRemoteProtocolInfo, - MaximumFileInfoByHandleClass -} FILE_INFO_BY_HANDLE_CLASS, - *PFILE_INFO_BY_HANDLE_CLASS; - -typedef struct _FILE_END_OF_FILE_INFO { - LARGE_INTEGER EndOfFile; -} FILE_END_OF_FILE_INFO, *PFILE_END_OF_FILE_INFO; - -#define REMOTE_PROTOCOL_INFO_FLAG_LOOPBACK 0x00000001 -#define REMOTE_PROTOCOL_INFO_FLAG_OFFLINE 0x00000002 - -typedef struct _FILE_REMOTE_PROTOCOL_INFO { - USHORT StructureVersion; - USHORT StructureSize; - DWORD Protocol; - USHORT ProtocolMajorVersion; - USHORT ProtocolMinorVersion; - USHORT ProtocolRevision; - USHORT Reserved; - DWORD Flags; - struct { - DWORD Reserved[8]; - } GenericReserved; - struct { - DWORD Reserved[16]; - } ProtocolSpecificReserved; -} FILE_REMOTE_PROTOCOL_INFO, *PFILE_REMOTE_PROTOCOL_INFO; - -#endif /* _WIN32_WINNT < 0x0600 (prior to Windows Vista) */ - -typedef BOOL(WINAPI *MDBX_GetFileInformationByHandleEx)( - _In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass, - _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize); -MDBX_INTERNAL_VAR MDBX_GetFileInformationByHandleEx - mdbx_GetFileInformationByHandleEx; - -typedef BOOL(WINAPI *MDBX_GetVolumeInformationByHandleW)( - _In_ HANDLE hFile, _Out_opt_ LPWSTR lpVolumeNameBuffer, - _In_ DWORD nVolumeNameSize, _Out_opt_ LPDWORD lpVolumeSerialNumber, - _Out_opt_ LPDWORD lpMaximumComponentLength, - _Out_opt_ LPDWORD lpFileSystemFlags, - _Out_opt_ LPWSTR lpFileSystemNameBuffer, _In_ DWORD nFileSystemNameSize); -MDBX_INTERNAL_VAR MDBX_GetVolumeInformationByHandleW - mdbx_GetVolumeInformationByHandleW; - -typedef DWORD(WINAPI *MDBX_GetFinalPathNameByHandleW)(_In_ HANDLE hFile, - _Out_ LPWSTR lpszFilePath, - _In_ DWORD cchFilePath, - _In_ DWORD dwFlags); -MDBX_INTERNAL_VAR MDBX_GetFinalPathNameByHandleW mdbx_GetFinalPathNameByHandleW; - -typedef BOOL(WINAPI *MDBX_SetFileInformationByHandle)( - _In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass, - _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize); -MDBX_INTERNAL_VAR MDBX_SetFileInformationByHandle - mdbx_SetFileInformationByHandle; - -typedef NTSTATUS(NTAPI *MDBX_NtFsControlFile)( - IN HANDLE FileHandle, IN OUT HANDLE Event, - IN OUT PVOID /* PIO_APC_ROUTINE */ ApcRoutine, IN OUT PVOID ApcContext, - OUT PIO_STATUS_BLOCK IoStatusBlock, IN ULONG FsControlCode, - IN OUT PVOID InputBuffer, IN ULONG InputBufferLength, - OUT OPTIONAL PVOID OutputBuffer, IN ULONG OutputBufferLength); -MDBX_INTERNAL_VAR MDBX_NtFsControlFile mdbx_NtFsControlFile; - -typedef uint64_t(WINAPI *MDBX_GetTickCount64)(void); -MDBX_INTERNAL_VAR MDBX_GetTickCount64 mdbx_GetTickCount64; - -#if !defined(_WIN32_WINNT_WIN8) || _WIN32_WINNT < _WIN32_WINNT_WIN8 -typedef struct _WIN32_MEMORY_RANGE_ENTRY { - PVOID VirtualAddress; - SIZE_T NumberOfBytes; -} WIN32_MEMORY_RANGE_ENTRY, *PWIN32_MEMORY_RANGE_ENTRY; -#endif /* Windows 8.x */ - -typedef BOOL(WINAPI *MDBX_PrefetchVirtualMemory)( - HANDLE hProcess, ULONG_PTR NumberOfEntries, - PWIN32_MEMORY_RANGE_ENTRY VirtualAddresses, ULONG Flags); -MDBX_INTERNAL_VAR MDBX_PrefetchVirtualMemory mdbx_PrefetchVirtualMemory; - -typedef enum _SECTION_INHERIT { ViewShare = 1, ViewUnmap = 2 } SECTION_INHERIT; -typedef NTSTATUS(NTAPI *MDBX_NtExtendSection)(IN HANDLE SectionHandle, - IN PLARGE_INTEGER NewSectionSize); -MDBX_INTERNAL_VAR MDBX_NtExtendSection mdbx_NtExtendSection; - -static __inline bool mdbx_RunningUnderWine(void) { - return !mdbx_NtExtendSection; -} - -typedef LSTATUS(WINAPI *MDBX_RegGetValueA)(HKEY hkey, LPCSTR lpSubKey, - LPCSTR lpValue, DWORD dwFlags, - LPDWORD pdwType, PVOID pvData, - LPDWORD pcbData); -MDBX_INTERNAL_VAR MDBX_RegGetValueA mdbx_RegGetValueA; - -NTSYSAPI ULONG RtlRandomEx(PULONG Seed); - -typedef BOOL(WINAPI *MDBX_SetFileIoOverlappedRange)(HANDLE FileHandle, - PUCHAR OverlappedRangeStart, - ULONG Length); -MDBX_INTERNAL_VAR MDBX_SetFileIoOverlappedRange mdbx_SetFileIoOverlappedRange; +MDBX_INTERNAL void osal_ctor(void); +MDBX_INTERNAL void osal_dtor(void); +#if defined(_WIN32) || defined(_WIN64) +MDBX_INTERNAL int osal_mb2w(const char *const src, wchar_t **const pdst); #endif /* Windows */ -#endif /* !__cplusplus */ +typedef union bin128 { + __anonymous_struct_extension__ struct { + uint64_t x, y; + }; + __anonymous_struct_extension__ struct { + uint32_t a, b, c, d; + }; +} bin128_t; + +MDBX_INTERNAL bin128_t osal_guid(const MDBX_env *); /*----------------------------------------------------------------------------*/ -MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static __always_inline uint64_t -osal_bswap64(uint64_t v) { -#if __GNUC_PREREQ(4, 4) || __CLANG_PREREQ(4, 0) || \ - __has_builtin(__builtin_bswap64) +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint64_t osal_bswap64(uint64_t v) { +#if __GNUC_PREREQ(4, 4) || __CLANG_PREREQ(4, 0) || __has_builtin(__builtin_bswap64) return __builtin_bswap64(v); #elif defined(_MSC_VER) && !defined(__clang__) return _byteswap_uint64(v); @@ -1896,19 +1568,14 @@ osal_bswap64(uint64_t v) { #elif defined(bswap_64) return bswap_64(v); #else - return v << 56 | v >> 56 | ((v << 40) & UINT64_C(0x00ff000000000000)) | - ((v << 24) & UINT64_C(0x0000ff0000000000)) | - ((v << 8) & UINT64_C(0x000000ff00000000)) | - ((v >> 8) & UINT64_C(0x00000000ff000000)) | - ((v >> 24) & UINT64_C(0x0000000000ff0000)) | - ((v >> 40) & UINT64_C(0x000000000000ff00)); + return v << 56 | v >> 56 | ((v << 40) & UINT64_C(0x00ff000000000000)) | ((v << 24) & UINT64_C(0x0000ff0000000000)) | + ((v << 8) & UINT64_C(0x000000ff00000000)) | ((v >> 8) & UINT64_C(0x00000000ff000000)) | + ((v >> 24) & UINT64_C(0x0000000000ff0000)) | ((v >> 40) & UINT64_C(0x000000000000ff00)); #endif } -MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static __always_inline uint32_t -osal_bswap32(uint32_t v) { -#if __GNUC_PREREQ(4, 4) || __CLANG_PREREQ(4, 0) || \ - __has_builtin(__builtin_bswap32) +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint32_t osal_bswap32(uint32_t v) { +#if __GNUC_PREREQ(4, 4) || __CLANG_PREREQ(4, 0) || __has_builtin(__builtin_bswap32) return __builtin_bswap32(v); #elif defined(_MSC_VER) && !defined(__clang__) return _byteswap_ulong(v); @@ -1917,50 +1584,14 @@ osal_bswap32(uint32_t v) { #elif defined(bswap_32) return bswap_32(v); #else - return v << 24 | v >> 24 | ((v << 8) & UINT32_C(0x00ff0000)) | - ((v >> 8) & UINT32_C(0x0000ff00)); + return v << 24 | v >> 24 | ((v << 8) & UINT32_C(0x00ff0000)) | ((v >> 8) & UINT32_C(0x0000ff00)); #endif } -/*----------------------------------------------------------------------------*/ - -#if defined(_MSC_VER) && _MSC_VER >= 1900 -/* LY: MSVC 2015/2017/2019 has buggy/inconsistent PRIuPTR/PRIxPTR macros - * for internal format-args checker. */ -#undef PRIuPTR -#undef PRIiPTR -#undef PRIdPTR -#undef PRIxPTR -#define PRIuPTR "Iu" -#define PRIiPTR "Ii" -#define PRIdPTR "Id" -#define PRIxPTR "Ix" -#define PRIuSIZE "zu" -#define PRIiSIZE "zi" -#define PRIdSIZE "zd" -#define PRIxSIZE "zx" -#endif /* fix PRI*PTR for _MSC_VER */ - -#ifndef PRIuSIZE -#define PRIuSIZE PRIuPTR -#define PRIiSIZE PRIiPTR -#define PRIdSIZE PRIdPTR -#define PRIxSIZE PRIxPTR -#endif /* PRI*SIZE macros for MSVC */ - -#ifdef _MSC_VER -#pragma warning(pop) -#endif - -#define mdbx_sourcery_anchor XCONCAT(mdbx_sourcery_, MDBX_BUILD_SOURCERY) -#if defined(xMDBX_TOOLS) -extern LIBMDBX_API const char *const mdbx_sourcery_anchor; -#endif - /******************************************************************************* - ******************************************************************************* ******************************************************************************* * + * BUILD TIME * * #### ##### ##### # #### # # #### * # # # # # # # # ## # # @@ -1981,23 +1612,15 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Using fsync() with chance of data lost on power failure */ #define MDBX_OSX_WANNA_SPEED 1 -#ifndef MDBX_OSX_SPEED_INSTEADOF_DURABILITY +#ifndef MDBX_APPLE_SPEED_INSTEADOF_DURABILITY /** Choices \ref MDBX_OSX_WANNA_DURABILITY or \ref MDBX_OSX_WANNA_SPEED * for OSX & iOS */ -#define MDBX_OSX_SPEED_INSTEADOF_DURABILITY MDBX_OSX_WANNA_DURABILITY -#endif /* MDBX_OSX_SPEED_INSTEADOF_DURABILITY */ - -/** Controls using of POSIX' madvise() and/or similar hints. */ -#ifndef MDBX_ENABLE_MADVISE -#define MDBX_ENABLE_MADVISE 1 -#elif !(MDBX_ENABLE_MADVISE == 0 || MDBX_ENABLE_MADVISE == 1) -#error MDBX_ENABLE_MADVISE must be defined as 0 or 1 -#endif /* MDBX_ENABLE_MADVISE */ +#define MDBX_APPLE_SPEED_INSTEADOF_DURABILITY MDBX_OSX_WANNA_DURABILITY +#endif /* MDBX_APPLE_SPEED_INSTEADOF_DURABILITY */ /** Controls checking PID against reuse DB environment after the fork() */ #ifndef MDBX_ENV_CHECKPID -#if (defined(MADV_DONTFORK) && MDBX_ENABLE_MADVISE) || defined(_WIN32) || \ - defined(_WIN64) +#if defined(MADV_DONTFORK) || defined(_WIN32) || defined(_WIN64) /* PID check could be omitted: * - on Linux when madvise(MADV_DONTFORK) is available, i.e. after the fork() * mapped pages will not be available for child process. @@ -2026,8 +1649,7 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Does a system have battery-backed Real-Time Clock or just a fake. */ #ifndef MDBX_TRUST_RTC -#if defined(__linux__) || defined(__gnu_linux__) || defined(__NetBSD__) || \ - defined(__OpenBSD__) +#if defined(__linux__) || defined(__gnu_linux__) || defined(__NetBSD__) || defined(__OpenBSD__) #define MDBX_TRUST_RTC 0 /* a lot of embedded systems have a fake RTC */ #else #define MDBX_TRUST_RTC 1 @@ -2062,24 +1684,21 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Controls using Unix' mincore() to determine whether DB-pages * are resident in memory. */ -#ifndef MDBX_ENABLE_MINCORE +#ifndef MDBX_USE_MINCORE #if defined(MINCORE_INCORE) || !(defined(_WIN32) || defined(_WIN64)) -#define MDBX_ENABLE_MINCORE 1 +#define MDBX_USE_MINCORE 1 #else -#define MDBX_ENABLE_MINCORE 0 +#define MDBX_USE_MINCORE 0 #endif -#elif !(MDBX_ENABLE_MINCORE == 0 || MDBX_ENABLE_MINCORE == 1) -#error MDBX_ENABLE_MINCORE must be defined as 0 or 1 -#endif /* MDBX_ENABLE_MINCORE */ +#define MDBX_USE_MINCORE_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_USE_MINCORE) +#elif !(MDBX_USE_MINCORE == 0 || MDBX_USE_MINCORE == 1) +#error MDBX_USE_MINCORE must be defined as 0 or 1 +#endif /* MDBX_USE_MINCORE */ /** Enables chunking long list of retired pages during huge transactions commit * to avoid use sequences of pages. */ #ifndef MDBX_ENABLE_BIGFOOT -#if MDBX_WORDBITS >= 64 || defined(DOXYGEN) #define MDBX_ENABLE_BIGFOOT 1 -#else -#define MDBX_ENABLE_BIGFOOT 0 -#endif #elif !(MDBX_ENABLE_BIGFOOT == 0 || MDBX_ENABLE_BIGFOOT == 1) #error MDBX_ENABLE_BIGFOOT must be defined as 0 or 1 #endif /* MDBX_ENABLE_BIGFOOT */ @@ -2094,25 +1713,27 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #ifndef MDBX_PNL_PREALLOC_FOR_RADIXSORT #define MDBX_PNL_PREALLOC_FOR_RADIXSORT 1 -#elif !(MDBX_PNL_PREALLOC_FOR_RADIXSORT == 0 || \ - MDBX_PNL_PREALLOC_FOR_RADIXSORT == 1) +#elif !(MDBX_PNL_PREALLOC_FOR_RADIXSORT == 0 || MDBX_PNL_PREALLOC_FOR_RADIXSORT == 1) #error MDBX_PNL_PREALLOC_FOR_RADIXSORT must be defined as 0 or 1 #endif /* MDBX_PNL_PREALLOC_FOR_RADIXSORT */ #ifndef MDBX_DPL_PREALLOC_FOR_RADIXSORT #define MDBX_DPL_PREALLOC_FOR_RADIXSORT 1 -#elif !(MDBX_DPL_PREALLOC_FOR_RADIXSORT == 0 || \ - MDBX_DPL_PREALLOC_FOR_RADIXSORT == 1) +#elif !(MDBX_DPL_PREALLOC_FOR_RADIXSORT == 0 || MDBX_DPL_PREALLOC_FOR_RADIXSORT == 1) #error MDBX_DPL_PREALLOC_FOR_RADIXSORT must be defined as 0 or 1 #endif /* MDBX_DPL_PREALLOC_FOR_RADIXSORT */ -/** Controls dirty pages tracking, spilling and persisting in MDBX_WRITEMAP - * mode. 0/OFF = Don't track dirty pages at all, don't spill ones, and use - * msync() to persist data. This is by-default on Linux and other systems where - * kernel provides properly LRU tracking and effective flushing on-demand. 1/ON - * = Tracking of dirty pages but with LRU labels for spilling and explicit - * persist ones by write(). This may be reasonable for systems which low - * performance of msync() and/or LRU tracking. */ +/** Controls dirty pages tracking, spilling and persisting in `MDBX_WRITEMAP` + * mode, i.e. disables in-memory database updating with consequent + * flush-to-disk/msync syscall. + * + * 0/OFF = Don't track dirty pages at all, don't spill ones, and use msync() to + * persist data. This is by-default on Linux and other systems where kernel + * provides properly LRU tracking and effective flushing on-demand. + * + * 1/ON = Tracking of dirty pages but with LRU labels for spilling and explicit + * persist ones by write(). This may be reasonable for goofy systems (Windows) + * which low performance of msync() and/or zany LRU tracking. */ #ifndef MDBX_AVOID_MSYNC #if defined(_WIN32) || defined(_WIN64) #define MDBX_AVOID_MSYNC 1 @@ -2123,6 +1744,22 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #error MDBX_AVOID_MSYNC must be defined as 0 or 1 #endif /* MDBX_AVOID_MSYNC */ +/** Управляет механизмом поддержки разреженных наборов DBI-хендлов для снижения + * накладных расходов при запуске и обработке транзакций. */ +#ifndef MDBX_ENABLE_DBI_SPARSE +#define MDBX_ENABLE_DBI_SPARSE 1 +#elif !(MDBX_ENABLE_DBI_SPARSE == 0 || MDBX_ENABLE_DBI_SPARSE == 1) +#error MDBX_ENABLE_DBI_SPARSE must be defined as 0 or 1 +#endif /* MDBX_ENABLE_DBI_SPARSE */ + +/** Управляет механизмом отложенного освобождения и поддержки пути быстрого + * открытия DBI-хендлов без захвата блокировок. */ +#ifndef MDBX_ENABLE_DBI_LOCKFREE +#define MDBX_ENABLE_DBI_LOCKFREE 1 +#elif !(MDBX_ENABLE_DBI_LOCKFREE == 0 || MDBX_ENABLE_DBI_LOCKFREE == 1) +#error MDBX_ENABLE_DBI_LOCKFREE must be defined as 0 or 1 +#endif /* MDBX_ENABLE_DBI_LOCKFREE */ + /** Controls sort order of internal page number lists. * This mostly experimental/advanced option with not for regular MDBX users. * \warning The database format depend on this option and libmdbx built with @@ -2135,7 +1772,11 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Avoid dependence from MSVC CRT and use ntdll.dll instead. */ #ifndef MDBX_WITHOUT_MSVC_CRT +#if defined(MDBX_BUILD_CXX) && !MDBX_BUILD_CXX #define MDBX_WITHOUT_MSVC_CRT 1 +#else +#define MDBX_WITHOUT_MSVC_CRT 0 +#endif #elif !(MDBX_WITHOUT_MSVC_CRT == 0 || MDBX_WITHOUT_MSVC_CRT == 1) #error MDBX_WITHOUT_MSVC_CRT must be defined as 0 or 1 #endif /* MDBX_WITHOUT_MSVC_CRT */ @@ -2143,12 +1784,11 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Size of buffer used during copying a environment/database file. */ #ifndef MDBX_ENVCOPY_WRITEBUF #define MDBX_ENVCOPY_WRITEBUF 1048576u -#elif MDBX_ENVCOPY_WRITEBUF < 65536u || MDBX_ENVCOPY_WRITEBUF > 1073741824u || \ - MDBX_ENVCOPY_WRITEBUF % 65536u +#elif MDBX_ENVCOPY_WRITEBUF < 65536u || MDBX_ENVCOPY_WRITEBUF > 1073741824u || MDBX_ENVCOPY_WRITEBUF % 65536u #error MDBX_ENVCOPY_WRITEBUF must be defined in range 65536..1073741824 and be multiple of 65536 #endif /* MDBX_ENVCOPY_WRITEBUF */ -/** Forces assertion checking */ +/** Forces assertion checking. */ #ifndef MDBX_FORCE_ASSERTIONS #define MDBX_FORCE_ASSERTIONS 0 #elif !(MDBX_FORCE_ASSERTIONS == 0 || MDBX_FORCE_ASSERTIONS == 1) @@ -2163,15 +1803,14 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #else #define MDBX_ASSUME_MALLOC_OVERHEAD (sizeof(void *) * 2u) #endif -#elif MDBX_ASSUME_MALLOC_OVERHEAD < 0 || MDBX_ASSUME_MALLOC_OVERHEAD > 64 || \ - MDBX_ASSUME_MALLOC_OVERHEAD % 4 +#elif MDBX_ASSUME_MALLOC_OVERHEAD < 0 || MDBX_ASSUME_MALLOC_OVERHEAD > 64 || MDBX_ASSUME_MALLOC_OVERHEAD % 4 #error MDBX_ASSUME_MALLOC_OVERHEAD must be defined in range 0..64 and be multiple of 4 #endif /* MDBX_ASSUME_MALLOC_OVERHEAD */ /** If defined then enables integration with Valgrind, * a memory analyzing tool. */ -#ifndef MDBX_USE_VALGRIND -#endif /* MDBX_USE_VALGRIND */ +#ifndef ENABLE_MEMCHECK +#endif /* ENABLE_MEMCHECK */ /** If defined then enables use C11 atomics, * otherwise detects ones availability automatically. */ @@ -2191,18 +1830,24 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #define MDBX_HAVE_BUILTIN_CPU_SUPPORTS 0 #elif defined(__e2k__) #define MDBX_HAVE_BUILTIN_CPU_SUPPORTS 0 -#elif __has_builtin(__builtin_cpu_supports) || \ - defined(__BUILTIN_CPU_SUPPORTS__) || \ +#elif __has_builtin(__builtin_cpu_supports) || defined(__BUILTIN_CPU_SUPPORTS__) || \ (defined(__ia32__) && __GNUC_PREREQ(4, 8) && __GLIBC_PREREQ(2, 23)) #define MDBX_HAVE_BUILTIN_CPU_SUPPORTS 1 #else #define MDBX_HAVE_BUILTIN_CPU_SUPPORTS 0 #endif -#elif !(MDBX_HAVE_BUILTIN_CPU_SUPPORTS == 0 || \ - MDBX_HAVE_BUILTIN_CPU_SUPPORTS == 1) +#elif !(MDBX_HAVE_BUILTIN_CPU_SUPPORTS == 0 || MDBX_HAVE_BUILTIN_CPU_SUPPORTS == 1) #error MDBX_HAVE_BUILTIN_CPU_SUPPORTS must be defined as 0 or 1 #endif /* MDBX_HAVE_BUILTIN_CPU_SUPPORTS */ +/** if enabled then instead of the returned error `MDBX_REMOTE`, only a warning is issued, when + * the database being opened in non-read-only mode is located in a file system exported via NFS. */ +#ifndef MDBX_ENABLE_NON_READONLY_EXPORT +#define MDBX_ENABLE_NON_READONLY_EXPORT 0 +#elif !(MDBX_ENABLE_NON_READONLY_EXPORT == 0 || MDBX_ENABLE_NON_READONLY_EXPORT == 1) +#error MDBX_ENABLE_NON_READONLY_EXPORT must be defined as 0 or 1 +#endif /* MDBX_ENABLE_NON_READONLY_EXPORT */ + //------------------------------------------------------------------------------ /** Win32 File Locking API for \ref MDBX_LOCKING */ @@ -2220,27 +1865,20 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** POSIX-2008 Robust Mutexes for \ref MDBX_LOCKING */ #define MDBX_LOCKING_POSIX2008 2008 -/** BeOS Benaphores, aka Futexes for \ref MDBX_LOCKING */ -#define MDBX_LOCKING_BENAPHORE 1995 - /** Advanced: Choices the locking implementation (autodetection by default). */ #if defined(_WIN32) || defined(_WIN64) #define MDBX_LOCKING MDBX_LOCKING_WIN32FILES #else #ifndef MDBX_LOCKING -#if defined(_POSIX_THREAD_PROCESS_SHARED) && \ - _POSIX_THREAD_PROCESS_SHARED >= 200112L && !defined(__FreeBSD__) +#if defined(_POSIX_THREAD_PROCESS_SHARED) && _POSIX_THREAD_PROCESS_SHARED >= 200112L && !defined(__FreeBSD__) /* Some platforms define the EOWNERDEAD error code even though they * don't support Robust Mutexes. If doubt compile with -MDBX_LOCKING=2001. */ -#if defined(EOWNERDEAD) && _POSIX_THREAD_PROCESS_SHARED >= 200809L && \ - ((defined(_POSIX_THREAD_ROBUST_PRIO_INHERIT) && \ - _POSIX_THREAD_ROBUST_PRIO_INHERIT > 0) || \ - (defined(_POSIX_THREAD_ROBUST_PRIO_PROTECT) && \ - _POSIX_THREAD_ROBUST_PRIO_PROTECT > 0) || \ - defined(PTHREAD_MUTEX_ROBUST) || defined(PTHREAD_MUTEX_ROBUST_NP)) && \ - (!defined(__GLIBC__) || \ - __GLIBC_PREREQ(2, 10) /* troubles with Robust mutexes before 2.10 */) +#if defined(EOWNERDEAD) && _POSIX_THREAD_PROCESS_SHARED >= 200809L && \ + ((defined(_POSIX_THREAD_ROBUST_PRIO_INHERIT) && _POSIX_THREAD_ROBUST_PRIO_INHERIT > 0) || \ + (defined(_POSIX_THREAD_ROBUST_PRIO_PROTECT) && _POSIX_THREAD_ROBUST_PRIO_PROTECT > 0) || \ + defined(PTHREAD_MUTEX_ROBUST) || defined(PTHREAD_MUTEX_ROBUST_NP)) && \ + (!defined(__GLIBC__) || __GLIBC_PREREQ(2, 10) /* troubles with Robust mutexes before 2.10 */) #define MDBX_LOCKING MDBX_LOCKING_POSIX2008 #else #define MDBX_LOCKING MDBX_LOCKING_POSIX2001 @@ -2258,12 +1896,9 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Advanced: Using POSIX OFD-locks (autodetection by default). */ #ifndef MDBX_USE_OFDLOCKS -#if ((defined(F_OFD_SETLK) && defined(F_OFD_SETLKW) && \ - defined(F_OFD_GETLK)) || \ - (defined(F_OFD_SETLK64) && defined(F_OFD_SETLKW64) && \ - defined(F_OFD_GETLK64))) && \ - !defined(MDBX_SAFE4QEMU) && \ - !defined(__sun) /* OFD-lock are broken on Solaris */ +#if ((defined(F_OFD_SETLK) && defined(F_OFD_SETLKW) && defined(F_OFD_GETLK)) || \ + (defined(F_OFD_SETLK64) && defined(F_OFD_SETLKW64) && defined(F_OFD_GETLK64))) && \ + !defined(MDBX_SAFE4QEMU) && !defined(__sun) /* OFD-lock are broken on Solaris */ #define MDBX_USE_OFDLOCKS 1 #else #define MDBX_USE_OFDLOCKS 0 @@ -2277,8 +1912,7 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Advanced: Using sendfile() syscall (autodetection by default). */ #ifndef MDBX_USE_SENDFILE -#if ((defined(__linux__) || defined(__gnu_linux__)) && \ - !defined(__ANDROID_API__)) || \ +#if ((defined(__linux__) || defined(__gnu_linux__)) && !defined(__ANDROID_API__)) || \ (defined(__ANDROID_API__) && __ANDROID_API__ >= 21) #define MDBX_USE_SENDFILE 1 #else @@ -2299,30 +1933,15 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #error MDBX_USE_COPYFILERANGE must be defined as 0 or 1 #endif /* MDBX_USE_COPYFILERANGE */ -/** Advanced: Using sync_file_range() syscall (autodetection by default). */ -#ifndef MDBX_USE_SYNCFILERANGE -#if ((defined(__linux__) || defined(__gnu_linux__)) && \ - defined(SYNC_FILE_RANGE_WRITE) && !defined(__ANDROID_API__)) || \ - (defined(__ANDROID_API__) && __ANDROID_API__ >= 26) -#define MDBX_USE_SYNCFILERANGE 1 -#else -#define MDBX_USE_SYNCFILERANGE 0 -#endif -#elif !(MDBX_USE_SYNCFILERANGE == 0 || MDBX_USE_SYNCFILERANGE == 1) -#error MDBX_USE_SYNCFILERANGE must be defined as 0 or 1 -#endif /* MDBX_USE_SYNCFILERANGE */ - //------------------------------------------------------------------------------ #ifndef MDBX_CPU_WRITEBACK_INCOHERENT -#if defined(__ia32__) || defined(__e2k__) || defined(__hppa) || \ - defined(__hppa__) || defined(DOXYGEN) +#if defined(__ia32__) || defined(__e2k__) || defined(__hppa) || defined(__hppa__) || defined(DOXYGEN) #define MDBX_CPU_WRITEBACK_INCOHERENT 0 #else #define MDBX_CPU_WRITEBACK_INCOHERENT 1 #endif -#elif !(MDBX_CPU_WRITEBACK_INCOHERENT == 0 || \ - MDBX_CPU_WRITEBACK_INCOHERENT == 1) +#elif !(MDBX_CPU_WRITEBACK_INCOHERENT == 0 || MDBX_CPU_WRITEBACK_INCOHERENT == 1) #error MDBX_CPU_WRITEBACK_INCOHERENT must be defined as 0 or 1 #endif /* MDBX_CPU_WRITEBACK_INCOHERENT */ @@ -2332,35 +1951,35 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #else #define MDBX_MMAP_INCOHERENT_FILE_WRITE 0 #endif -#elif !(MDBX_MMAP_INCOHERENT_FILE_WRITE == 0 || \ - MDBX_MMAP_INCOHERENT_FILE_WRITE == 1) +#elif !(MDBX_MMAP_INCOHERENT_FILE_WRITE == 0 || MDBX_MMAP_INCOHERENT_FILE_WRITE == 1) #error MDBX_MMAP_INCOHERENT_FILE_WRITE must be defined as 0 or 1 #endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */ #ifndef MDBX_MMAP_INCOHERENT_CPU_CACHE -#if defined(__mips) || defined(__mips__) || defined(__mips64) || \ - defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \ - defined(__MWERKS__) || defined(__sgi) +#if defined(__mips) || defined(__mips__) || defined(__mips64) || defined(__mips64__) || defined(_M_MRX000) || \ + defined(_MIPS_) || defined(__MWERKS__) || defined(__sgi) /* MIPS has cache coherency issues. */ #define MDBX_MMAP_INCOHERENT_CPU_CACHE 1 #else /* LY: assume no relevant mmap/dcache issues. */ #define MDBX_MMAP_INCOHERENT_CPU_CACHE 0 #endif -#elif !(MDBX_MMAP_INCOHERENT_CPU_CACHE == 0 || \ - MDBX_MMAP_INCOHERENT_CPU_CACHE == 1) +#elif !(MDBX_MMAP_INCOHERENT_CPU_CACHE == 0 || MDBX_MMAP_INCOHERENT_CPU_CACHE == 1) #error MDBX_MMAP_INCOHERENT_CPU_CACHE must be defined as 0 or 1 #endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */ -#ifndef MDBX_MMAP_USE_MS_ASYNC -#if MDBX_MMAP_INCOHERENT_FILE_WRITE || MDBX_MMAP_INCOHERENT_CPU_CACHE -#define MDBX_MMAP_USE_MS_ASYNC 1 +/** Assume system needs explicit syscall to sync/flush/write modified mapped + * memory. */ +#ifndef MDBX_MMAP_NEEDS_JOLT +#if MDBX_MMAP_INCOHERENT_FILE_WRITE || MDBX_MMAP_INCOHERENT_CPU_CACHE || !(defined(__linux__) || defined(__gnu_linux__)) +#define MDBX_MMAP_NEEDS_JOLT 1 #else -#define MDBX_MMAP_USE_MS_ASYNC 0 +#define MDBX_MMAP_NEEDS_JOLT 0 #endif -#elif !(MDBX_MMAP_USE_MS_ASYNC == 0 || MDBX_MMAP_USE_MS_ASYNC == 1) -#error MDBX_MMAP_USE_MS_ASYNC must be defined as 0 or 1 -#endif /* MDBX_MMAP_USE_MS_ASYNC */ +#define MDBX_MMAP_NEEDS_JOLT_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_MMAP_NEEDS_JOLT) +#elif !(MDBX_MMAP_NEEDS_JOLT == 0 || MDBX_MMAP_NEEDS_JOLT == 1) +#error MDBX_MMAP_NEEDS_JOLT must be defined as 0 or 1 +#endif /* MDBX_MMAP_NEEDS_JOLT */ #ifndef MDBX_64BIT_ATOMIC #if MDBX_WORDBITS >= 64 || defined(DOXYGEN) @@ -2407,8 +2026,7 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #endif /* MDBX_64BIT_CAS */ #ifndef MDBX_UNALIGNED_OK -#if defined(__ALIGNED__) || defined(__SANITIZE_UNDEFINED__) || \ - defined(ENABLE_UBSAN) +#if defined(__ALIGNED__) || defined(__SANITIZE_UNDEFINED__) || defined(ENABLE_UBSAN) #define MDBX_UNALIGNED_OK 0 /* no unaligned access allowed */ #elif defined(__ARM_FEATURE_UNALIGNED) #define MDBX_UNALIGNED_OK 4 /* ok unaligned for 32-bit words */ @@ -2442,6 +2060,19 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #endif #endif /* MDBX_CACHELINE_SIZE */ +/* Max length of iov-vector passed to writev() call, used for auxilary writes */ +#ifndef MDBX_AUXILARY_IOV_MAX +#define MDBX_AUXILARY_IOV_MAX 64 +#endif +#if defined(IOV_MAX) && IOV_MAX < MDBX_AUXILARY_IOV_MAX +#undef MDBX_AUXILARY_IOV_MAX +#define MDBX_AUXILARY_IOV_MAX IOV_MAX +#endif /* MDBX_AUXILARY_IOV_MAX */ + +/* An extra/custom information provided during library build */ +#ifndef MDBX_BUILD_METADATA +#define MDBX_BUILD_METADATA "" +#endif /* MDBX_BUILD_METADATA */ /** @} end of build options */ /******************************************************************************* ******************************************************************************* @@ -2456,6 +2087,9 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #else #define MDBX_DEBUG 1 #endif +#endif +#if MDBX_DEBUG < 0 || MDBX_DEBUG > 2 +#error "The MDBX_DEBUG must be defined to 0, 1 or 2" #endif /* MDBX_DEBUG */ #else @@ -2475,187 +2109,76 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; * Also enables \ref MDBX_DBG_AUDIT if `MDBX_DEBUG >= 2`. * * \ingroup build_option */ -#define MDBX_DEBUG 0...7 +#define MDBX_DEBUG 0...2 /** Disables using of GNU libc extensions. */ #define MDBX_DISABLE_GNU_SOURCE 0 or 1 #endif /* DOXYGEN */ -/* Undefine the NDEBUG if debugging is enforced by MDBX_DEBUG */ -#if MDBX_DEBUG -#undef NDEBUG -#endif - -#ifndef __cplusplus -/*----------------------------------------------------------------------------*/ -/* Debug and Logging stuff */ - -#define MDBX_RUNTIME_FLAGS_INIT \ - ((MDBX_DEBUG) > 0) * MDBX_DBG_ASSERT + ((MDBX_DEBUG) > 1) * MDBX_DBG_AUDIT +#ifndef MDBX_64BIT_ATOMIC +#error "The MDBX_64BIT_ATOMIC must be defined before" +#endif /* MDBX_64BIT_ATOMIC */ -extern uint8_t runtime_flags; -extern uint8_t loglevel; -extern MDBX_debug_func *debug_logger; +#ifndef MDBX_64BIT_CAS +#error "The MDBX_64BIT_CAS must be defined before" +#endif /* MDBX_64BIT_CAS */ -MDBX_MAYBE_UNUSED static __inline void jitter4testing(bool tiny) { -#if MDBX_DEBUG - if (MDBX_DBG_JITTER & runtime_flags) - osal_jitter(tiny); +#if defined(__cplusplus) && !defined(__STDC_NO_ATOMICS__) && __has_include() +#include +#define MDBX_HAVE_C11ATOMICS +#elif !defined(__cplusplus) && (__STDC_VERSION__ >= 201112L || __has_extension(c_atomic)) && \ + !defined(__STDC_NO_ATOMICS__) && \ + (__GNUC_PREREQ(4, 9) || __CLANG_PREREQ(3, 8) || !(defined(__GNUC__) || defined(__clang__))) +#include +#define MDBX_HAVE_C11ATOMICS +#elif defined(__GNUC__) || defined(__clang__) +#elif defined(_MSC_VER) +#pragma warning(disable : 4163) /* 'xyz': not available as an intrinsic */ +#pragma warning(disable : 4133) /* 'function': incompatible types - from \ + 'size_t' to 'LONGLONG' */ +#pragma warning(disable : 4244) /* 'return': conversion from 'LONGLONG' to \ + 'std::size_t', possible loss of data */ +#pragma warning(disable : 4267) /* 'function': conversion from 'size_t' to \ + 'long', possible loss of data */ +#pragma intrinsic(_InterlockedExchangeAdd, _InterlockedCompareExchange) +#pragma intrinsic(_InterlockedExchangeAdd64, _InterlockedCompareExchange64) +#elif defined(__APPLE__) +#include #else - (void)tiny; +#error FIXME atomic-ops #endif -} -MDBX_INTERNAL_FUNC void MDBX_PRINTF_ARGS(4, 5) - debug_log(int level, const char *function, int line, const char *fmt, ...) - MDBX_PRINTF_ARGS(4, 5); -MDBX_INTERNAL_FUNC void debug_log_va(int level, const char *function, int line, - const char *fmt, va_list args); +typedef enum mdbx_memory_order { + mo_Relaxed, + mo_AcquireRelease + /* , mo_SequentialConsistency */ +} mdbx_memory_order_t; -#if MDBX_DEBUG -#define LOG_ENABLED(msg) unlikely(msg <= loglevel) -#define AUDIT_ENABLED() unlikely((runtime_flags & MDBX_DBG_AUDIT)) -#else /* MDBX_DEBUG */ -#define LOG_ENABLED(msg) (msg < MDBX_LOG_VERBOSE && msg <= loglevel) -#define AUDIT_ENABLED() (0) -#endif /* MDBX_DEBUG */ +typedef union { + volatile uint32_t weak; +#ifdef MDBX_HAVE_C11ATOMICS + volatile _Atomic uint32_t c11a; +#endif /* MDBX_HAVE_C11ATOMICS */ +} mdbx_atomic_uint32_t; -#if MDBX_FORCE_ASSERTIONS -#define ASSERT_ENABLED() (1) -#elif MDBX_DEBUG -#define ASSERT_ENABLED() likely((runtime_flags & MDBX_DBG_ASSERT)) -#else -#define ASSERT_ENABLED() (0) -#endif /* assertions */ - -#define DEBUG_EXTRA(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_EXTRA)) \ - debug_log(MDBX_LOG_EXTRA, __func__, __LINE__, fmt, __VA_ARGS__); \ - } while (0) - -#define DEBUG_EXTRA_PRINT(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_EXTRA)) \ - debug_log(MDBX_LOG_EXTRA, NULL, 0, fmt, __VA_ARGS__); \ - } while (0) - -#define TRACE(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_TRACE)) \ - debug_log(MDBX_LOG_TRACE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ - } while (0) - -#define DEBUG(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_DEBUG)) \ - debug_log(MDBX_LOG_DEBUG, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ - } while (0) - -#define VERBOSE(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_VERBOSE)) \ - debug_log(MDBX_LOG_VERBOSE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ - } while (0) - -#define NOTICE(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_NOTICE)) \ - debug_log(MDBX_LOG_NOTICE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ - } while (0) - -#define WARNING(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_WARN)) \ - debug_log(MDBX_LOG_WARN, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ - } while (0) - -#undef ERROR /* wingdi.h \ - Yeah, morons from M$ put such definition to the public header. */ - -#define ERROR(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_ERROR)) \ - debug_log(MDBX_LOG_ERROR, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ - } while (0) - -#define FATAL(fmt, ...) \ - debug_log(MDBX_LOG_FATAL, __func__, __LINE__, fmt "\n", __VA_ARGS__); - -#if MDBX_DEBUG -#define ASSERT_FAIL(env, msg, func, line) mdbx_assert_fail(env, msg, func, line) -#else /* MDBX_DEBUG */ -MDBX_NORETURN __cold void assert_fail(const char *msg, const char *func, - unsigned line); -#define ASSERT_FAIL(env, msg, func, line) \ - do { \ - (void)(env); \ - assert_fail(msg, func, line); \ - } while (0) -#endif /* MDBX_DEBUG */ - -#define ENSURE_MSG(env, expr, msg) \ - do { \ - if (unlikely(!(expr))) \ - ASSERT_FAIL(env, msg, __func__, __LINE__); \ - } while (0) - -#define ENSURE(env, expr) ENSURE_MSG(env, expr, #expr) - -/* assert(3) variant in environment context */ -#define eASSERT(env, expr) \ - do { \ - if (ASSERT_ENABLED()) \ - ENSURE(env, expr); \ - } while (0) - -/* assert(3) variant in cursor context */ -#define cASSERT(mc, expr) eASSERT((mc)->mc_txn->mt_env, expr) - -/* assert(3) variant in transaction context */ -#define tASSERT(txn, expr) eASSERT((txn)->mt_env, expr) - -#ifndef xMDBX_TOOLS /* Avoid using internal eASSERT() */ -#undef assert -#define assert(expr) eASSERT(NULL, expr) -#endif - -#endif /* __cplusplus */ - -/*----------------------------------------------------------------------------*/ -/* Atomics */ - -enum MDBX_memory_order { - mo_Relaxed, - mo_AcquireRelease - /* , mo_SequentialConsistency */ -}; - -typedef union { - volatile uint32_t weak; -#ifdef MDBX_HAVE_C11ATOMICS - volatile _Atomic uint32_t c11a; -#endif /* MDBX_HAVE_C11ATOMICS */ -} MDBX_atomic_uint32_t; - -typedef union { - volatile uint64_t weak; -#if defined(MDBX_HAVE_C11ATOMICS) && (MDBX_64BIT_CAS || MDBX_64BIT_ATOMIC) - volatile _Atomic uint64_t c11a; -#endif -#if !defined(MDBX_HAVE_C11ATOMICS) || !MDBX_64BIT_CAS || !MDBX_64BIT_ATOMIC - __anonymous_struct_extension__ struct { -#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ - MDBX_atomic_uint32_t low, high; -#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - MDBX_atomic_uint32_t high, low; +typedef union { + volatile uint64_t weak; +#if defined(MDBX_HAVE_C11ATOMICS) && (MDBX_64BIT_CAS || MDBX_64BIT_ATOMIC) + volatile _Atomic uint64_t c11a; +#endif +#if !defined(MDBX_HAVE_C11ATOMICS) || !MDBX_64BIT_CAS || !MDBX_64BIT_ATOMIC + __anonymous_struct_extension__ struct { +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + mdbx_atomic_uint32_t low, high; +#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + mdbx_atomic_uint32_t high, low; #else #error "FIXME: Unsupported byte order" #endif /* __BYTE_ORDER__ */ }; #endif -} MDBX_atomic_uint64_t; +} mdbx_atomic_uint64_t; #ifdef MDBX_HAVE_C11ATOMICS @@ -2671,92 +2194,20 @@ typedef union { #define MDBX_c11a_rw(type, ptr) (&(ptr)->c11a) #endif /* Crutches for C11 atomic compiler's bugs */ -#define mo_c11_store(fence) \ - (((fence) == mo_Relaxed) ? memory_order_relaxed \ - : ((fence) == mo_AcquireRelease) ? memory_order_release \ +#define mo_c11_store(fence) \ + (((fence) == mo_Relaxed) ? memory_order_relaxed \ + : ((fence) == mo_AcquireRelease) ? memory_order_release \ : memory_order_seq_cst) -#define mo_c11_load(fence) \ - (((fence) == mo_Relaxed) ? memory_order_relaxed \ - : ((fence) == mo_AcquireRelease) ? memory_order_acquire \ +#define mo_c11_load(fence) \ + (((fence) == mo_Relaxed) ? memory_order_relaxed \ + : ((fence) == mo_AcquireRelease) ? memory_order_acquire \ : memory_order_seq_cst) #endif /* MDBX_HAVE_C11ATOMICS */ -#ifndef __cplusplus - -#ifdef MDBX_HAVE_C11ATOMICS -#define osal_memory_fence(order, write) \ - atomic_thread_fence((write) ? mo_c11_store(order) : mo_c11_load(order)) -#else /* MDBX_HAVE_C11ATOMICS */ -#define osal_memory_fence(order, write) \ - do { \ - osal_compiler_barrier(); \ - if (write && order > (MDBX_CPU_WRITEBACK_INCOHERENT ? mo_Relaxed \ - : mo_AcquireRelease)) \ - osal_memory_barrier(); \ - } while (0) -#endif /* MDBX_HAVE_C11ATOMICS */ - -#if defined(MDBX_HAVE_C11ATOMICS) && defined(__LCC__) -#define atomic_store32(p, value, order) \ - ({ \ - const uint32_t value_to_store = (value); \ - atomic_store_explicit(MDBX_c11a_rw(uint32_t, p), value_to_store, \ - mo_c11_store(order)); \ - value_to_store; \ - }) -#define atomic_load32(p, order) \ - atomic_load_explicit(MDBX_c11a_ro(uint32_t, p), mo_c11_load(order)) -#define atomic_store64(p, value, order) \ - ({ \ - const uint64_t value_to_store = (value); \ - atomic_store_explicit(MDBX_c11a_rw(uint64_t, p), value_to_store, \ - mo_c11_store(order)); \ - value_to_store; \ - }) -#define atomic_load64(p, order) \ - atomic_load_explicit(MDBX_c11a_ro(uint64_t, p), mo_c11_load(order)) -#endif /* LCC && MDBX_HAVE_C11ATOMICS */ - -#ifndef atomic_store32 -MDBX_MAYBE_UNUSED static __always_inline uint32_t -atomic_store32(MDBX_atomic_uint32_t *p, const uint32_t value, - enum MDBX_memory_order order) { - STATIC_ASSERT(sizeof(MDBX_atomic_uint32_t) == 4); -#ifdef MDBX_HAVE_C11ATOMICS - assert(atomic_is_lock_free(MDBX_c11a_rw(uint32_t, p))); - atomic_store_explicit(MDBX_c11a_rw(uint32_t, p), value, mo_c11_store(order)); -#else /* MDBX_HAVE_C11ATOMICS */ - if (order != mo_Relaxed) - osal_compiler_barrier(); - p->weak = value; - osal_memory_fence(order, true); -#endif /* MDBX_HAVE_C11ATOMICS */ - return value; -} -#endif /* atomic_store32 */ - -#ifndef atomic_load32 -MDBX_MAYBE_UNUSED static __always_inline uint32_t atomic_load32( - const volatile MDBX_atomic_uint32_t *p, enum MDBX_memory_order order) { - STATIC_ASSERT(sizeof(MDBX_atomic_uint32_t) == 4); -#ifdef MDBX_HAVE_C11ATOMICS - assert(atomic_is_lock_free(MDBX_c11a_ro(uint32_t, p))); - return atomic_load_explicit(MDBX_c11a_ro(uint32_t, p), mo_c11_load(order)); -#else /* MDBX_HAVE_C11ATOMICS */ - osal_memory_fence(order, false); - const uint32_t value = p->weak; - if (order != mo_Relaxed) - osal_compiler_barrier(); - return value; -#endif /* MDBX_HAVE_C11ATOMICS */ -} -#endif /* atomic_load32 */ - -#endif /* !__cplusplus */ +#define SAFE64_INVALID_THRESHOLD UINT64_C(0xffffFFFF00000000) -/*----------------------------------------------------------------------------*/ -/* Basic constants and types */ +#pragma pack(push, 4) /* A stamp that identifies a file as an MDBX file. * There's nothing special about this value other than that it is easily @@ -2765,8 +2216,10 @@ MDBX_MAYBE_UNUSED static __always_inline uint32_t atomic_load32( /* FROZEN: The version number for a database's datafile format. */ #define MDBX_DATA_VERSION 3 -/* The version number for a database's lockfile format. */ -#define MDBX_LOCK_VERSION 5 + +#define MDBX_DATA_MAGIC ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + MDBX_DATA_VERSION) +#define MDBX_DATA_MAGIC_LEGACY_COMPAT ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + 2) +#define MDBX_DATA_MAGIC_LEGACY_DEVEL ((MDBX_MAGIC << 8) + 255) /* handle for the DB used to track free pages. */ #define FREE_DBI 0 @@ -2783,203 +2236,285 @@ MDBX_MAYBE_UNUSED static __always_inline uint32_t atomic_load32( * MDBX uses 32 bit for page numbers. This limits database * size up to 2^44 bytes, in case of 4K pages. */ typedef uint32_t pgno_t; -typedef MDBX_atomic_uint32_t atomic_pgno_t; +typedef mdbx_atomic_uint32_t atomic_pgno_t; #define PRIaPGNO PRIu32 #define MAX_PAGENO UINT32_C(0x7FFFffff) #define MIN_PAGENO NUM_METAS -#define SAFE64_INVALID_THRESHOLD UINT64_C(0xffffFFFF00000000) +/* An invalid page number. + * Mainly used to denote an empty tree. */ +#define P_INVALID (~(pgno_t)0) /* A transaction ID. */ typedef uint64_t txnid_t; -typedef MDBX_atomic_uint64_t atomic_txnid_t; +typedef mdbx_atomic_uint64_t atomic_txnid_t; #define PRIaTXN PRIi64 #define MIN_TXNID UINT64_C(1) #define MAX_TXNID (SAFE64_INVALID_THRESHOLD - 1) #define INITIAL_TXNID (MIN_TXNID + NUM_METAS - 1) #define INVALID_TXNID UINT64_MAX -/* LY: for testing non-atomic 64-bit txnid on 32-bit arches. - * #define xMDBX_TXNID_STEP (UINT32_MAX / 3) */ -#ifndef xMDBX_TXNID_STEP -#if MDBX_64BIT_CAS -#define xMDBX_TXNID_STEP 1u -#else -#define xMDBX_TXNID_STEP 2u -#endif -#endif /* xMDBX_TXNID_STEP */ -/* Used for offsets within a single page. - * Since memory pages are typically 4 or 8KB in size, 12-13 bits, - * this is plenty. */ +/* Used for offsets within a single page. */ typedef uint16_t indx_t; -#define MEGABYTE ((size_t)1 << 20) - -/*----------------------------------------------------------------------------*/ -/* Core structures for database and shared memory (i.e. format definition) */ -#pragma pack(push, 4) - -/* Information about a single database in the environment. */ -typedef struct MDBX_db { - uint16_t md_flags; /* see mdbx_dbi_open */ - uint16_t md_depth; /* depth of this tree */ - uint32_t md_xsize; /* key-size for MDBX_DUPFIXED (LEAF2 pages) */ - pgno_t md_root; /* the root page of this tree */ - pgno_t md_branch_pages; /* number of internal pages */ - pgno_t md_leaf_pages; /* number of leaf pages */ - pgno_t md_overflow_pages; /* number of overflow pages */ - uint64_t md_seq; /* table sequence counter */ - uint64_t md_entries; /* number of data items */ - uint64_t md_mod_txnid; /* txnid of last committed modification */ -} MDBX_db; +typedef struct tree { + uint16_t flags; /* see mdbx_dbi_open */ + uint16_t height; /* height of this tree */ + uint32_t dupfix_size; /* key-size for MDBX_DUPFIXED (DUPFIX pages) */ + pgno_t root; /* the root page of this tree */ + pgno_t branch_pages; /* number of branch pages */ + pgno_t leaf_pages; /* number of leaf pages */ + pgno_t large_pages; /* number of large pages */ + uint64_t sequence; /* table sequence counter */ + uint64_t items; /* number of data items */ + uint64_t mod_txnid; /* txnid of last committed modification */ +} tree_t; /* database size-related parameters */ -typedef struct MDBX_geo { +typedef struct geo { uint16_t grow_pv; /* datafile growth step as a 16-bit packed (exponential quantized) value */ uint16_t shrink_pv; /* datafile shrink threshold as a 16-bit packed (exponential quantized) value */ pgno_t lower; /* minimal size of datafile in pages */ pgno_t upper; /* maximal size of datafile in pages */ - pgno_t now; /* current size of datafile in pages */ - pgno_t next; /* first unused page in the datafile, + union { + pgno_t now; /* current size of datafile in pages */ + pgno_t end_pgno; + }; + union { + pgno_t first_unallocated; /* first unused page in the datafile, but actually the file may be shorter. */ -} MDBX_geo; + pgno_t next_pgno; + }; +} geo_t; /* Meta page content. * A meta page is the start point for accessing a database snapshot. - * Pages 0-1 are meta pages. Transaction N writes meta page (N % 2). */ -typedef struct MDBX_meta { + * Pages 0-2 are meta pages. */ +typedef struct meta { /* Stamp identifying this as an MDBX file. * It must be set to MDBX_MAGIC with MDBX_DATA_VERSION. */ - uint32_t mm_magic_and_version[2]; + uint32_t magic_and_version[2]; - /* txnid that committed this page, the first of a two-phase-update pair */ + /* txnid that committed this meta, the first of a two-phase-update pair */ union { - MDBX_atomic_uint32_t mm_txnid_a[2]; + mdbx_atomic_uint32_t txnid_a[2]; uint64_t unsafe_txnid; }; - uint16_t mm_extra_flags; /* extra DB flags, zero (nothing) for now */ - uint8_t mm_validator_id; /* ID of checksum and page validation method, - * zero (nothing) for now */ - uint8_t mm_extra_pagehdr; /* extra bytes in the page header, - * zero (nothing) for now */ + uint16_t reserve16; /* extra flags, zero (nothing) for now */ + uint8_t validator_id; /* ID of checksum and page validation method, + * zero (nothing) for now */ + int8_t extra_pagehdr; /* extra bytes in the page header, + * zero (nothing) for now */ + + geo_t geometry; /* database size-related parameters */ - MDBX_geo mm_geo; /* database size-related parameters */ + union { + struct { + tree_t gc, main; + } trees; + __anonymous_struct_extension__ struct { + uint16_t gc_flags; + uint16_t gc_height; + uint32_t pagesize; + }; + }; - MDBX_db mm_dbs[CORE_DBS]; /* first is free space, 2nd is main db */ - /* The size of pages used in this DB */ -#define mm_psize mm_dbs[FREE_DBI].md_xsize - MDBX_canary mm_canary; + MDBX_canary canary; -#define MDBX_DATASIGN_NONE 0u -#define MDBX_DATASIGN_WEAK 1u -#define SIGN_IS_STEADY(sign) ((sign) > MDBX_DATASIGN_WEAK) -#define META_IS_STEADY(meta) \ - SIGN_IS_STEADY(unaligned_peek_u64_volatile(4, (meta)->mm_sign)) +#define DATASIGN_NONE 0u +#define DATASIGN_WEAK 1u +#define SIGN_IS_STEADY(sign) ((sign) > DATASIGN_WEAK) union { - uint32_t mm_sign[2]; + uint32_t sign[2]; uint64_t unsafe_sign; }; - /* txnid that committed this page, the second of a two-phase-update pair */ - MDBX_atomic_uint32_t mm_txnid_b[2]; + /* txnid that committed this meta, the second of a two-phase-update pair */ + mdbx_atomic_uint32_t txnid_b[2]; /* Number of non-meta pages which were put in GC after COW. May be 0 in case * DB was previously handled by libmdbx without corresponding feature. - * This value in couple with mr_snapshot_pages_retired allows fast estimation - * of "how much reader is restraining GC recycling". */ - uint32_t mm_pages_retired[2]; + * This value in couple with reader.snapshot_pages_retired allows fast + * estimation of "how much reader is restraining GC recycling". */ + uint32_t pages_retired[2]; /* The analogue /proc/sys/kernel/random/boot_id or similar to determine * whether the system was rebooted after the last use of the database files. * If there was no reboot, but there is no need to rollback to the last * steady sync point. Zeros mean that no relevant information is available * from the system. */ - bin128_t mm_bootid; + bin128_t bootid; -} MDBX_meta; + /* GUID базы данных, начиная с v0.13.1 */ + bin128_t dxbid; +} meta_t; #pragma pack(1) -/* Common header for all page types. The page type depends on mp_flags. +typedef enum page_type { + P_BRANCH = 0x01u /* branch page */, + P_LEAF = 0x02u /* leaf page */, + P_LARGE = 0x04u /* large/overflow page */, + P_META = 0x08u /* meta page */, + P_LEGACY_DIRTY = 0x10u /* legacy P_DIRTY flag prior to v0.10 958fd5b9 */, + P_BAD = P_LEGACY_DIRTY /* explicit flag for invalid/bad page */, + P_DUPFIX = 0x20u /* for MDBX_DUPFIXED records */, + P_SUBP = 0x40u /* for MDBX_DUPSORT sub-pages */, + P_SPILLED = 0x2000u /* spilled in parent txn */, + P_LOOSE = 0x4000u /* page was dirtied then freed, can be reused */, + P_FROZEN = 0x8000u /* used for retire page with known status */, + P_ILL_BITS = (uint16_t)~(P_BRANCH | P_LEAF | P_DUPFIX | P_LARGE | P_SPILLED), + + page_broken = 0, + page_large = P_LARGE, + page_branch = P_BRANCH, + page_leaf = P_LEAF, + page_dupfix_leaf = P_DUPFIX, + page_sub_leaf = P_SUBP | P_LEAF, + page_sub_dupfix_leaf = P_SUBP | P_DUPFIX, + page_sub_broken = P_SUBP, +} page_type_t; + +/* Common header for all page types. The page type depends on flags. * - * P_BRANCH and P_LEAF pages have unsorted 'MDBX_node's at the end, with - * sorted mp_ptrs[] entries referring to them. Exception: P_LEAF2 pages - * omit mp_ptrs and pack sorted MDBX_DUPFIXED values after the page header. + * P_BRANCH and P_LEAF pages have unsorted 'node_t's at the end, with + * sorted entries[] entries referring to them. Exception: P_DUPFIX pages + * omit entries and pack sorted MDBX_DUPFIXED values after the page header. * - * P_OVERFLOW records occupy one or more contiguous pages where only the - * first has a page header. They hold the real data of F_BIGDATA nodes. + * P_LARGE records occupy one or more contiguous pages where only the + * first has a page header. They hold the real data of N_BIG nodes. * * P_SUBP sub-pages are small leaf "pages" with duplicate data. - * A node with flag F_DUPDATA but not F_SUBDATA contains a sub-page. - * (Duplicate data can also go in sub-databases, which use normal pages.) + * A node with flag N_DUP but not N_TREE contains a sub-page. + * (Duplicate data can also go in tables, which use normal pages.) * - * P_META pages contain MDBX_meta, the start point of an MDBX snapshot. + * P_META pages contain meta_t, the start point of an MDBX snapshot. * - * Each non-metapage up to MDBX_meta.mm_last_pg is reachable exactly once + * Each non-metapage up to meta_t.mm_last_pg is reachable exactly once * in the snapshot: Either used by a database or listed in a GC record. */ -typedef struct MDBX_page { -#define IS_FROZEN(txn, p) ((p)->mp_txnid < (txn)->mt_txnid) -#define IS_SPILLED(txn, p) ((p)->mp_txnid == (txn)->mt_txnid) -#define IS_SHADOWED(txn, p) ((p)->mp_txnid > (txn)->mt_txnid) -#define IS_VALID(txn, p) ((p)->mp_txnid <= (txn)->mt_front) -#define IS_MODIFIABLE(txn, p) ((p)->mp_txnid == (txn)->mt_front) - uint64_t mp_txnid; /* txnid which created page, maybe zero in legacy DB */ - uint16_t mp_leaf2_ksize; /* key size if this is a LEAF2 page */ -#define P_BRANCH 0x01u /* branch page */ -#define P_LEAF 0x02u /* leaf page */ -#define P_OVERFLOW 0x04u /* overflow page */ -#define P_META 0x08u /* meta page */ -#define P_LEGACY_DIRTY 0x10u /* legacy P_DIRTY flag prior to v0.10 958fd5b9 */ -#define P_BAD P_LEGACY_DIRTY /* explicit flag for invalid/bad page */ -#define P_LEAF2 0x20u /* for MDBX_DUPFIXED records */ -#define P_SUBP 0x40u /* for MDBX_DUPSORT sub-pages */ -#define P_SPILLED 0x2000u /* spilled in parent txn */ -#define P_LOOSE 0x4000u /* page was dirtied then freed, can be reused */ -#define P_FROZEN 0x8000u /* used for retire page with known status */ -#define P_ILL_BITS \ - ((uint16_t)~(P_BRANCH | P_LEAF | P_LEAF2 | P_OVERFLOW | P_SPILLED)) - uint16_t mp_flags; +typedef struct page { + uint64_t txnid; /* txnid which created page, maybe zero in legacy DB */ + uint16_t dupfix_ksize; /* key size if this is a DUPFIX page */ + uint16_t flags; union { - uint32_t mp_pages; /* number of overflow pages */ + uint32_t pages; /* number of overflow pages */ __anonymous_struct_extension__ struct { - indx_t mp_lower; /* lower bound of free space */ - indx_t mp_upper; /* upper bound of free space */ + indx_t lower; /* lower bound of free space */ + indx_t upper; /* upper bound of free space */ }; }; - pgno_t mp_pgno; /* page number */ - -#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ - (!defined(__cplusplus) && defined(_MSC_VER)) - indx_t mp_ptrs[] /* dynamic size */; -#endif /* C99 */ -} MDBX_page; + pgno_t pgno; /* page number */ -#define PAGETYPE_WHOLE(p) ((uint8_t)(p)->mp_flags) - -/* Drop legacy P_DIRTY flag for sub-pages for compatilibity */ -#define PAGETYPE_COMPAT(p) \ - (unlikely(PAGETYPE_WHOLE(p) & P_SUBP) \ - ? PAGETYPE_WHOLE(p) & ~(P_SUBP | P_LEGACY_DIRTY) \ - : PAGETYPE_WHOLE(p)) +#if FLEXIBLE_ARRAY_MEMBERS + indx_t entries[] /* dynamic size */; +#endif /* FLEXIBLE_ARRAY_MEMBERS */ +} page_t; /* Size of the page header, excluding dynamic data at the end */ -#define PAGEHDRSZ offsetof(MDBX_page, mp_ptrs) +#define PAGEHDRSZ 20u -/* Pointer displacement without casting to char* to avoid pointer-aliasing */ -#define ptr_disp(ptr, disp) ((void *)(((intptr_t)(ptr)) + ((intptr_t)(disp)))) +/* Header for a single key/data pair within a page. + * Used in pages of type P_BRANCH and P_LEAF without P_DUPFIX. + * We guarantee 2-byte alignment for 'node_t's. + * + * Leaf node flags describe node contents. N_BIG says the node's + * data part is the page number of an overflow page with actual data. + * N_DUP and N_TREE can be combined giving duplicate data in + * a sub-page/table, and named databases (just N_TREE). */ +typedef struct node { +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + union { + uint32_t dsize; + uint32_t child_pgno; + }; + uint8_t flags; /* see node_flags */ + uint8_t extra; + uint16_t ksize; /* key size */ +#else + uint16_t ksize; /* key size */ + uint8_t extra; + uint8_t flags; /* see node_flags */ + union { + uint32_t child_pgno; + uint32_t dsize; + }; +#endif /* __BYTE_ORDER__ */ -/* Pointer distance as signed number of bytes */ -#define ptr_dist(more, less) (((intptr_t)(more)) - ((intptr_t)(less))) +#if FLEXIBLE_ARRAY_MEMBERS + uint8_t payload[] /* key and data are appended here */; +#endif /* FLEXIBLE_ARRAY_MEMBERS */ +} node_t; + +/* Size of the node header, excluding dynamic data at the end */ +#define NODESIZE 8u -#define mp_next(mp) \ - (*(MDBX_page **)ptr_disp((mp)->mp_ptrs, sizeof(void *) - sizeof(uint32_t))) +typedef enum node_flags { + N_BIG = 0x01 /* data put on large page */, + N_TREE = 0x02 /* data is a b-tree */, + N_DUP = 0x04 /* data has duplicates */ +} node_flags_t; #pragma pack(pop) -typedef struct profgc_stat { +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint8_t page_type(const page_t *mp) { return mp->flags; } + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint8_t page_type_compat(const page_t *mp) { + /* Drop legacy P_DIRTY flag for sub-pages for compatilibity, + * for assertions only. */ + return unlikely(mp->flags & P_SUBP) ? mp->flags & ~(P_SUBP | P_LEGACY_DIRTY) : mp->flags; +} + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_leaf(const page_t *mp) { + return (mp->flags & P_LEAF) != 0; +} + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_dupfix_leaf(const page_t *mp) { + return (mp->flags & P_DUPFIX) != 0; +} + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_branch(const page_t *mp) { + return (mp->flags & P_BRANCH) != 0; +} + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_largepage(const page_t *mp) { + return (mp->flags & P_LARGE) != 0; +} + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_subpage(const page_t *mp) { + return (mp->flags & P_SUBP) != 0; +} + +/* The version number for a database's lockfile format. */ +#define MDBX_LOCK_VERSION 6 + +#if MDBX_LOCKING == MDBX_LOCKING_WIN32FILES + +#define MDBX_LCK_SIGN UINT32_C(0xF10C) +typedef void osal_ipclock_t; +#elif MDBX_LOCKING == MDBX_LOCKING_SYSV + +#define MDBX_LCK_SIGN UINT32_C(0xF18D) +typedef mdbx_pid_t osal_ipclock_t; + +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || MDBX_LOCKING == MDBX_LOCKING_POSIX2008 + +#define MDBX_LCK_SIGN UINT32_C(0x8017) +typedef pthread_mutex_t osal_ipclock_t; + +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988 + +#define MDBX_LCK_SIGN UINT32_C(0xFC29) +typedef sem_t osal_ipclock_t; + +#else +#error "FIXME" +#endif /* MDBX_LOCKING */ + +/* Статистика профилирования работы GC */ +typedef struct gc_prof_stat { /* Монотонное время по "настенным часам" * затраченное на чтение и поиск внутри GC */ uint64_t rtime_monotonic; @@ -2995,42 +2530,44 @@ typedef struct profgc_stat { uint32_t spe_counter; /* page faults (hard page faults) */ uint32_t majflt; -} profgc_stat_t; - -/* Statistics of page operations overall of all (running, completed and aborted) - * transactions */ -typedef struct pgop_stat { - MDBX_atomic_uint64_t newly; /* Quantity of a new pages added */ - MDBX_atomic_uint64_t cow; /* Quantity of pages copied for update */ - MDBX_atomic_uint64_t clone; /* Quantity of parent's dirty pages clones + /* Для разборок с pnl_merge() */ + struct { + uint64_t time; + uint64_t volume; + uint32_t calls; + } pnl_merge; +} gc_prof_stat_t; + +/* Statistics of pages operations for all transactions, + * including incomplete and aborted. */ +typedef struct pgops { + mdbx_atomic_uint64_t newly; /* Quantity of a new pages added */ + mdbx_atomic_uint64_t cow; /* Quantity of pages copied for update */ + mdbx_atomic_uint64_t clone; /* Quantity of parent's dirty pages clones for nested transactions */ - MDBX_atomic_uint64_t split; /* Page splits */ - MDBX_atomic_uint64_t merge; /* Page merges */ - MDBX_atomic_uint64_t spill; /* Quantity of spilled dirty pages */ - MDBX_atomic_uint64_t unspill; /* Quantity of unspilled/reloaded pages */ - MDBX_atomic_uint64_t - wops; /* Number of explicit write operations (not a pages) to a disk */ - MDBX_atomic_uint64_t - msync; /* Number of explicit msync/flush-to-disk operations */ - MDBX_atomic_uint64_t - fsync; /* Number of explicit fsync/flush-to-disk operations */ - - MDBX_atomic_uint64_t prefault; /* Number of prefault write operations */ - MDBX_atomic_uint64_t mincore; /* Number of mincore() calls */ - - MDBX_atomic_uint32_t - incoherence; /* number of https://libmdbx.dqdkfa.ru/dead-github/issues/269 - caught */ - MDBX_atomic_uint32_t reserved; + mdbx_atomic_uint64_t split; /* Page splits */ + mdbx_atomic_uint64_t merge; /* Page merges */ + mdbx_atomic_uint64_t spill; /* Quantity of spilled dirty pages */ + mdbx_atomic_uint64_t unspill; /* Quantity of unspilled/reloaded pages */ + mdbx_atomic_uint64_t wops; /* Number of explicit write operations (not a pages) to a disk */ + mdbx_atomic_uint64_t msync; /* Number of explicit msync/flush-to-disk operations */ + mdbx_atomic_uint64_t fsync; /* Number of explicit fsync/flush-to-disk operations */ + + mdbx_atomic_uint64_t prefault; /* Number of prefault write operations */ + mdbx_atomic_uint64_t mincore; /* Number of mincore() calls */ + + mdbx_atomic_uint32_t incoherence; /* number of https://libmdbx.dqdkfa.ru/dead-github/issues/269 + caught */ + mdbx_atomic_uint32_t reserved; /* Статистика для профилирования GC. - * Логически эти данные может быть стоит вынести в другую структуру, + * Логически эти данные, возможно, стоит вынести в другую структуру, * но разница будет сугубо косметическая. */ struct { /* Затраты на поддержку данных пользователя */ - profgc_stat_t work; + gc_prof_stat_t work; /* Затраты на поддержку и обновления самой GC */ - profgc_stat_t self; + gc_prof_stat_t self; /* Итераций обновления GC, * больше 1 если были повторы/перезапуски */ uint32_t wloops; @@ -3045,33 +2582,6 @@ typedef struct pgop_stat { } gc_prof; } pgop_stat_t; -#if MDBX_LOCKING == MDBX_LOCKING_WIN32FILES -#define MDBX_CLOCK_SIGN UINT32_C(0xF10C) -typedef void osal_ipclock_t; -#elif MDBX_LOCKING == MDBX_LOCKING_SYSV - -#define MDBX_CLOCK_SIGN UINT32_C(0xF18D) -typedef mdbx_pid_t osal_ipclock_t; -#ifndef EOWNERDEAD -#define EOWNERDEAD MDBX_RESULT_TRUE -#endif - -#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ - MDBX_LOCKING == MDBX_LOCKING_POSIX2008 -#define MDBX_CLOCK_SIGN UINT32_C(0x8017) -typedef pthread_mutex_t osal_ipclock_t; -#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988 -#define MDBX_CLOCK_SIGN UINT32_C(0xFC29) -typedef sem_t osal_ipclock_t; -#else -#error "FIXME" -#endif /* MDBX_LOCKING */ - -#if MDBX_LOCKING > MDBX_LOCKING_SYSV && !defined(__cplusplus) -MDBX_INTERNAL_FUNC int osal_ipclock_stub(osal_ipclock_t *ipc); -MDBX_INTERNAL_FUNC int osal_ipclock_destroy(osal_ipclock_t *ipc); -#endif /* MDBX_LOCKING */ - /* Reader Lock Table * * Readers don't acquire any locks for their data access. Instead, they @@ -3081,8 +2591,9 @@ MDBX_INTERNAL_FUNC int osal_ipclock_destroy(osal_ipclock_t *ipc); * read transactions started by the same thread need no further locking to * proceed. * - * If MDBX_NOTLS is set, the slot address is not saved in thread-specific data. - * No reader table is used if the database is on a read-only filesystem. + * If MDBX_NOSTICKYTHREADS is set, the slot address is not saved in + * thread-specific data. No reader table is used if the database is on a + * read-only filesystem. * * Since the database uses multi-version concurrency control, readers don't * actually need any locking. This table is used to keep track of which @@ -3111,14 +2622,14 @@ MDBX_INTERNAL_FUNC int osal_ipclock_destroy(osal_ipclock_t *ipc); * many old transactions together. */ /* The actual reader record, with cacheline padding. */ -typedef struct MDBX_reader { - /* Current Transaction ID when this transaction began, or (txnid_t)-1. +typedef struct reader_slot { + /* Current Transaction ID when this transaction began, or INVALID_TXNID. * Multiple readers that start at the same time will probably have the * same ID here. Again, it's not important to exclude them from * anything; all we need to know is which version of the DB they * started from so we can avoid overwriting any data used in that * particular version. */ - MDBX_atomic_uint64_t /* txnid_t */ mr_txnid; + atomic_txnid_t txnid; /* The information we store in a single slot of the reader table. * In addition to a transaction ID, we also record the process and @@ -3129,708 +2640,320 @@ typedef struct MDBX_reader { * We simply re-init the table when we know that we're the only process * opening the lock file. */ + /* Псевдо thread_id для пометки вытесненных читающих транзакций. */ +#define MDBX_TID_TXN_OUSTED (UINT64_MAX - 1) + + /* Псевдо thread_id для пометки припаркованных читающих транзакций. */ +#define MDBX_TID_TXN_PARKED UINT64_MAX + /* The thread ID of the thread owning this txn. */ - MDBX_atomic_uint64_t mr_tid; + mdbx_atomic_uint64_t tid; /* The process ID of the process owning this reader txn. */ - MDBX_atomic_uint32_t mr_pid; + mdbx_atomic_uint32_t pid; /* The number of pages used in the reader's MVCC snapshot, - * i.e. the value of meta->mm_geo.next and txn->mt_next_pgno */ - atomic_pgno_t mr_snapshot_pages_used; + * i.e. the value of meta->geometry.first_unallocated and + * txn->geo.first_unallocated */ + atomic_pgno_t snapshot_pages_used; /* Number of retired pages at the time this reader starts transaction. So, - * at any time the difference mm_pages_retired - mr_snapshot_pages_retired - * will give the number of pages which this reader restraining from reuse. */ - MDBX_atomic_uint64_t mr_snapshot_pages_retired; -} MDBX_reader; + * at any time the difference meta.pages_retired - + * reader.snapshot_pages_retired will give the number of pages which this + * reader restraining from reuse. */ + mdbx_atomic_uint64_t snapshot_pages_retired; +} reader_slot_t; /* The header for the reader table (a memory-mapped lock file). */ -typedef struct MDBX_lockinfo { +typedef struct shared_lck { /* Stamp identifying this as an MDBX file. * It must be set to MDBX_MAGIC with with MDBX_LOCK_VERSION. */ - uint64_t mti_magic_and_version; + uint64_t magic_and_version; /* Format of this lock file. Must be set to MDBX_LOCK_FORMAT. */ - uint32_t mti_os_and_format; + uint32_t os_and_format; /* Flags which environment was opened. */ - MDBX_atomic_uint32_t mti_envmode; + mdbx_atomic_uint32_t envmode; /* Threshold of un-synced-with-disk pages for auto-sync feature, * zero means no-threshold, i.e. auto-sync is disabled. */ - atomic_pgno_t mti_autosync_threshold; + atomic_pgno_t autosync_threshold; /* Low 32-bit of txnid with which meta-pages was synced, * i.e. for sync-polling in the MDBX_NOMETASYNC mode. */ #define MDBX_NOMETASYNC_LAZY_UNK (UINT32_MAX / 3) #define MDBX_NOMETASYNC_LAZY_FD (MDBX_NOMETASYNC_LAZY_UNK + UINT32_MAX / 8) -#define MDBX_NOMETASYNC_LAZY_WRITEMAP \ - (MDBX_NOMETASYNC_LAZY_UNK - UINT32_MAX / 8) - MDBX_atomic_uint32_t mti_meta_sync_txnid; +#define MDBX_NOMETASYNC_LAZY_WRITEMAP (MDBX_NOMETASYNC_LAZY_UNK - UINT32_MAX / 8) + mdbx_atomic_uint32_t meta_sync_txnid; /* Period for timed auto-sync feature, i.e. at the every steady checkpoint - * the mti_unsynced_timeout sets to the current_time + mti_autosync_period. + * the mti_unsynced_timeout sets to the current_time + autosync_period. * The time value is represented in a suitable system-dependent form, for * example clock_gettime(CLOCK_BOOTTIME) or clock_gettime(CLOCK_MONOTONIC). * Zero means timed auto-sync is disabled. */ - MDBX_atomic_uint64_t mti_autosync_period; + mdbx_atomic_uint64_t autosync_period; /* Marker to distinguish uniqueness of DB/CLK. */ - MDBX_atomic_uint64_t mti_bait_uniqueness; + mdbx_atomic_uint64_t bait_uniqueness; /* Paired counter of processes that have mlock()ed part of mmapped DB. - * The (mti_mlcnt[0] - mti_mlcnt[1]) > 0 means at least one process + * The (mlcnt[0] - mlcnt[1]) > 0 means at least one process * lock at least one page, so therefore madvise() could return EINVAL. */ - MDBX_atomic_uint32_t mti_mlcnt[2]; + mdbx_atomic_uint32_t mlcnt[2]; MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/ /* Statistics of costly ops of all (running, completed and aborted) * transactions */ - pgop_stat_t mti_pgop_stat; + pgop_stat_t pgops; MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/ - /* Write transaction lock. */ #if MDBX_LOCKING > 0 - osal_ipclock_t mti_wlock; + /* Write transaction lock. */ + osal_ipclock_t wrt_lock; #endif /* MDBX_LOCKING > 0 */ - atomic_txnid_t mti_oldest_reader; + atomic_txnid_t cached_oldest; /* Timestamp of entering an out-of-sync state. Value is represented in a * suitable system-dependent form, for example clock_gettime(CLOCK_BOOTTIME) * or clock_gettime(CLOCK_MONOTONIC). */ - MDBX_atomic_uint64_t mti_eoos_timestamp; + mdbx_atomic_uint64_t eoos_timestamp; /* Number un-synced-with-disk pages for auto-sync feature. */ - MDBX_atomic_uint64_t mti_unsynced_pages; + mdbx_atomic_uint64_t unsynced_pages; /* Timestamp of the last readers check. */ - MDBX_atomic_uint64_t mti_reader_check_timestamp; + mdbx_atomic_uint64_t readers_check_timestamp; /* Number of page which was discarded last time by madvise(DONTNEED). */ - atomic_pgno_t mti_discarded_tail; + atomic_pgno_t discarded_tail; /* Shared anchor for tracking readahead edge and enabled/disabled status. */ - pgno_t mti_readahead_anchor; + pgno_t readahead_anchor; /* Shared cache for mincore() results */ struct { pgno_t begin[4]; uint64_t mask[4]; - } mti_mincore_cache; + } mincore_cache; MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/ - /* Readeaders registration lock. */ #if MDBX_LOCKING > 0 - osal_ipclock_t mti_rlock; + /* Readeaders table lock. */ + osal_ipclock_t rdt_lock; #endif /* MDBX_LOCKING > 0 */ /* The number of slots that have been used in the reader table. * This always records the maximum count, it is not decremented * when readers release their slots. */ - MDBX_atomic_uint32_t mti_numreaders; - MDBX_atomic_uint32_t mti_readers_refresh_flag; + mdbx_atomic_uint32_t rdt_length; + mdbx_atomic_uint32_t rdt_refresh_flag; -#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ - (!defined(__cplusplus) && defined(_MSC_VER)) +#if FLEXIBLE_ARRAY_MEMBERS MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/ - MDBX_reader mti_readers[] /* dynamic size */; -#endif /* C99 */ -} MDBX_lockinfo; + reader_slot_t rdt[] /* dynamic size */; /* Lockfile format signature: version, features and field layout */ -#define MDBX_LOCK_FORMAT \ - (MDBX_CLOCK_SIGN * 27733 + (unsigned)sizeof(MDBX_reader) * 13 + \ - (unsigned)offsetof(MDBX_reader, mr_snapshot_pages_used) * 251 + \ - (unsigned)offsetof(MDBX_lockinfo, mti_oldest_reader) * 83 + \ - (unsigned)offsetof(MDBX_lockinfo, mti_numreaders) * 37 + \ - (unsigned)offsetof(MDBX_lockinfo, mti_readers) * 29) - -#define MDBX_DATA_MAGIC \ - ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + MDBX_DATA_VERSION) - -#define MDBX_DATA_MAGIC_LEGACY_COMPAT \ - ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + 2) - -#define MDBX_DATA_MAGIC_LEGACY_DEVEL ((MDBX_MAGIC << 8) + 255) +#define MDBX_LOCK_FORMAT \ + (MDBX_LCK_SIGN * 27733 + (unsigned)sizeof(reader_slot_t) * 13 + \ + (unsigned)offsetof(reader_slot_t, snapshot_pages_used) * 251 + (unsigned)offsetof(lck_t, cached_oldest) * 83 + \ + (unsigned)offsetof(lck_t, rdt_length) * 37 + (unsigned)offsetof(lck_t, rdt) * 29) +#endif /* FLEXIBLE_ARRAY_MEMBERS */ +} lck_t; #define MDBX_LOCK_MAGIC ((MDBX_MAGIC << 8) + MDBX_LOCK_VERSION) -/* The maximum size of a database page. - * - * It is 64K, but value-PAGEHDRSZ must fit in MDBX_page.mp_upper. - * - * MDBX will use database pages < OS pages if needed. - * That causes more I/O in write transactions: The OS must - * know (read) the whole page before writing a partial page. - * - * Note that we don't currently support Huge pages. On Linux, - * regular data files cannot use Huge pages, and in general - * Huge pages aren't actually pageable. We rely on the OS - * demand-pager to read our data and page it out when memory - * pressure from other processes is high. So until OSs have - * actual paging support for Huge pages, they're not viable. */ -#define MAX_PAGESIZE MDBX_MAX_PAGESIZE -#define MIN_PAGESIZE MDBX_MIN_PAGESIZE - -#define MIN_MAPSIZE (MIN_PAGESIZE * MIN_PAGENO) +#define MDBX_READERS_LIMIT 32767 + +#define MIN_MAPSIZE (MDBX_MIN_PAGESIZE * MIN_PAGENO) #if defined(_WIN32) || defined(_WIN64) #define MAX_MAPSIZE32 UINT32_C(0x38000000) #else #define MAX_MAPSIZE32 UINT32_C(0x7f000000) #endif -#define MAX_MAPSIZE64 ((MAX_PAGENO + 1) * (uint64_t)MAX_PAGESIZE) +#define MAX_MAPSIZE64 ((MAX_PAGENO + 1) * (uint64_t)MDBX_MAX_PAGESIZE) #if MDBX_WORDBITS >= 64 #define MAX_MAPSIZE MAX_MAPSIZE64 -#define MDBX_PGL_LIMIT ((size_t)MAX_PAGENO) +#define PAGELIST_LIMIT ((size_t)MAX_PAGENO) #else #define MAX_MAPSIZE MAX_MAPSIZE32 -#define MDBX_PGL_LIMIT (MAX_MAPSIZE32 / MIN_PAGESIZE) +#define PAGELIST_LIMIT (MAX_MAPSIZE32 / MDBX_MIN_PAGESIZE) #endif /* MDBX_WORDBITS */ -#define MDBX_READERS_LIMIT 32767 -#define MDBX_RADIXSORT_THRESHOLD 142 #define MDBX_GOLD_RATIO_DBL 1.6180339887498948482 +#define MEGABYTE ((size_t)1 << 20) /*----------------------------------------------------------------------------*/ -/* An PNL is an Page Number List, a sorted array of IDs. - * The first element of the array is a counter for how many actual page-numbers - * are in the list. By default PNLs are sorted in descending order, this allow - * cut off a page with lowest pgno (at the tail) just truncating the list. The - * sort order of PNLs is controlled by the MDBX_PNL_ASCENDING build option. */ -typedef pgno_t *MDBX_PNL; +union logger_union { + void *ptr; + MDBX_debug_func *fmt; + MDBX_debug_func_nofmt *nofmt; +}; -#if MDBX_PNL_ASCENDING -#define MDBX_PNL_ORDERED(first, last) ((first) < (last)) -#define MDBX_PNL_DISORDERED(first, last) ((first) >= (last)) -#else -#define MDBX_PNL_ORDERED(first, last) ((first) > (last)) -#define MDBX_PNL_DISORDERED(first, last) ((first) <= (last)) -#endif +struct libmdbx_globals { + bin128_t bootid; + unsigned sys_pagesize, sys_allocation_granularity; + uint8_t sys_pagesize_ln2; + uint8_t runtime_flags; + uint8_t loglevel; +#if defined(_WIN32) || defined(_WIN64) + bool running_under_Wine; +#elif defined(__linux__) || defined(__gnu_linux__) + bool running_on_WSL1 /* Windows Subsystem 1 for Linux */; + uint32_t linux_kernel_version; +#endif /* Linux */ + union logger_union logger; + osal_fastmutex_t debug_lock; + size_t logger_buffer_size; + char *logger_buffer; +}; -/* List of txnid, only for MDBX_txn.tw.lifo_reclaimed */ -typedef txnid_t *MDBX_TXL; +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ -/* An Dirty-Page list item is an pgno/pointer pair. */ -typedef struct MDBX_dp { - MDBX_page *ptr; - pgno_t pgno, npages; -} MDBX_dp; +extern struct libmdbx_globals globals; +#if defined(_WIN32) || defined(_WIN64) +extern struct libmdbx_imports imports; +#endif /* Windows */ -/* An DPL (dirty-page list) is a sorted array of MDBX_DPs. */ -typedef struct MDBX_dpl { - size_t sorted; - size_t length; - size_t pages_including_loose; /* number of pages, but not an entries. */ - size_t detent; /* allocated size excluding the MDBX_DPL_RESERVE_GAP */ -#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ - (!defined(__cplusplus) && defined(_MSC_VER)) - MDBX_dp items[] /* dynamic size with holes at zero and after the last */; -#endif -} MDBX_dpl; +#ifndef __Wpedantic_format_voidptr +MDBX_MAYBE_UNUSED static inline const void *__Wpedantic_format_voidptr(const void *ptr) { return ptr; } +#define __Wpedantic_format_voidptr(ARG) __Wpedantic_format_voidptr(ARG) +#endif /* __Wpedantic_format_voidptr */ -/* PNL sizes */ -#define MDBX_PNL_GRANULATE_LOG2 10 -#define MDBX_PNL_GRANULATE (1 << MDBX_PNL_GRANULATE_LOG2) -#define MDBX_PNL_INITIAL \ - (MDBX_PNL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t)) +MDBX_INTERNAL void MDBX_PRINTF_ARGS(4, 5) debug_log(int level, const char *function, int line, const char *fmt, ...) + MDBX_PRINTF_ARGS(4, 5); +MDBX_INTERNAL void debug_log_va(int level, const char *function, int line, const char *fmt, va_list args); -#define MDBX_TXL_GRANULATE 32 -#define MDBX_TXL_INITIAL \ - (MDBX_TXL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t)) -#define MDBX_TXL_MAX \ - ((1u << 26) - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t)) - -#define MDBX_PNL_ALLOCLEN(pl) ((pl)[-1]) -#define MDBX_PNL_GETSIZE(pl) ((size_t)((pl)[0])) -#define MDBX_PNL_SETSIZE(pl, size) \ - do { \ - const size_t __size = size; \ - assert(__size < INT_MAX); \ - (pl)[0] = (pgno_t)__size; \ - } while (0) -#define MDBX_PNL_FIRST(pl) ((pl)[1]) -#define MDBX_PNL_LAST(pl) ((pl)[MDBX_PNL_GETSIZE(pl)]) -#define MDBX_PNL_BEGIN(pl) (&(pl)[1]) -#define MDBX_PNL_END(pl) (&(pl)[MDBX_PNL_GETSIZE(pl) + 1]) +#if MDBX_DEBUG +#define LOG_ENABLED(LVL) unlikely(LVL <= globals.loglevel) +#define AUDIT_ENABLED() unlikely((globals.runtime_flags & (unsigned)MDBX_DBG_AUDIT)) +#else /* MDBX_DEBUG */ +#define LOG_ENABLED(LVL) (LVL < MDBX_LOG_VERBOSE && LVL <= globals.loglevel) +#define AUDIT_ENABLED() (0) +#endif /* LOG_ENABLED() & AUDIT_ENABLED() */ -#if MDBX_PNL_ASCENDING -#define MDBX_PNL_EDGE(pl) ((pl) + 1) -#define MDBX_PNL_LEAST(pl) MDBX_PNL_FIRST(pl) -#define MDBX_PNL_MOST(pl) MDBX_PNL_LAST(pl) +#if MDBX_FORCE_ASSERTIONS +#define ASSERT_ENABLED() (1) +#elif MDBX_DEBUG +#define ASSERT_ENABLED() likely((globals.runtime_flags & (unsigned)MDBX_DBG_ASSERT)) #else -#define MDBX_PNL_EDGE(pl) ((pl) + MDBX_PNL_GETSIZE(pl)) -#define MDBX_PNL_LEAST(pl) MDBX_PNL_LAST(pl) -#define MDBX_PNL_MOST(pl) MDBX_PNL_FIRST(pl) -#endif - -#define MDBX_PNL_SIZEOF(pl) ((MDBX_PNL_GETSIZE(pl) + 1) * sizeof(pgno_t)) -#define MDBX_PNL_IS_EMPTY(pl) (MDBX_PNL_GETSIZE(pl) == 0) - -/*----------------------------------------------------------------------------*/ -/* Internal structures */ - -/* Auxiliary DB info. - * The information here is mostly static/read-only. There is - * only a single copy of this record in the environment. */ -typedef struct MDBX_dbx { - MDBX_val md_name; /* name of the database */ - MDBX_cmp_func *md_cmp; /* function for comparing keys */ - MDBX_cmp_func *md_dcmp; /* function for comparing data items */ - size_t md_klen_min, md_klen_max; /* min/max key length for the database */ - size_t md_vlen_min, - md_vlen_max; /* min/max value/data length for the database */ -} MDBX_dbx; +#define ASSERT_ENABLED() (0) +#endif /* ASSERT_ENABLED() */ -typedef struct troika { - uint8_t fsm, recent, prefer_steady, tail_and_flags; -#if MDBX_WORDBITS > 32 /* Workaround for false-positives from Valgrind */ - uint32_t unused_pad; -#endif -#define TROIKA_HAVE_STEADY(troika) ((troika)->fsm & 7) -#define TROIKA_STRICT_VALID(troika) ((troika)->tail_and_flags & 64) -#define TROIKA_VALID(troika) ((troika)->tail_and_flags & 128) -#define TROIKA_TAIL(troika) ((troika)->tail_and_flags & 3) - txnid_t txnid[NUM_METAS]; -} meta_troika_t; +#define DEBUG_EXTRA(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_EXTRA)) \ + debug_log(MDBX_LOG_EXTRA, __func__, __LINE__, fmt, __VA_ARGS__); \ + } while (0) -/* A database transaction. - * Every operation requires a transaction handle. */ -struct MDBX_txn { -#define MDBX_MT_SIGNATURE UINT32_C(0x93D53A31) - uint32_t mt_signature; - - /* Transaction Flags */ - /* mdbx_txn_begin() flags */ -#define MDBX_TXN_RO_BEGIN_FLAGS (MDBX_TXN_RDONLY | MDBX_TXN_RDONLY_PREPARE) -#define MDBX_TXN_RW_BEGIN_FLAGS \ - (MDBX_TXN_NOMETASYNC | MDBX_TXN_NOSYNC | MDBX_TXN_TRY) - /* Additional flag for sync_locked() */ -#define MDBX_SHRINK_ALLOWED UINT32_C(0x40000000) - -#define MDBX_TXN_DRAINED_GC 0x20 /* GC was depleted up to oldest reader */ - -#define TXN_FLAGS \ - (MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_DIRTY | MDBX_TXN_SPILLS | \ - MDBX_TXN_HAS_CHILD | MDBX_TXN_INVALID | MDBX_TXN_DRAINED_GC) - -#if (TXN_FLAGS & (MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_RO_BEGIN_FLAGS)) || \ - ((MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_RO_BEGIN_FLAGS | TXN_FLAGS) & \ - MDBX_SHRINK_ALLOWED) -#error "Oops, some txn flags overlapped or wrong" -#endif - uint32_t mt_flags; - - MDBX_txn *mt_parent; /* parent of a nested txn */ - /* Nested txn under this txn, set together with flag MDBX_TXN_HAS_CHILD */ - MDBX_txn *mt_child; - MDBX_geo mt_geo; - /* next unallocated page */ -#define mt_next_pgno mt_geo.next - /* corresponding to the current size of datafile */ -#define mt_end_pgno mt_geo.now +#define DEBUG_EXTRA_PRINT(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_EXTRA)) \ + debug_log(MDBX_LOG_EXTRA, nullptr, 0, fmt, __VA_ARGS__); \ + } while (0) - /* The ID of this transaction. IDs are integers incrementing from - * INITIAL_TXNID. Only committed write transactions increment the ID. If a - * transaction aborts, the ID may be re-used by the next writer. */ - txnid_t mt_txnid; - txnid_t mt_front; - - MDBX_env *mt_env; /* the DB environment */ - /* Array of records for each DB known in the environment. */ - MDBX_dbx *mt_dbxs; - /* Array of MDBX_db records for each known DB */ - MDBX_db *mt_dbs; - /* Array of sequence numbers for each DB handle */ - MDBX_atomic_uint32_t *mt_dbiseqs; - - /* Transaction DBI Flags */ -#define DBI_DIRTY MDBX_DBI_DIRTY /* DB was written in this txn */ -#define DBI_STALE MDBX_DBI_STALE /* Named-DB record is older than txnID */ -#define DBI_FRESH MDBX_DBI_FRESH /* Named-DB handle opened in this txn */ -#define DBI_CREAT MDBX_DBI_CREAT /* Named-DB handle created in this txn */ -#define DBI_VALID 0x10 /* DB handle is valid, see also DB_VALID */ -#define DBI_USRVALID 0x20 /* As DB_VALID, but not set for FREE_DBI */ -#define DBI_AUDITED 0x40 /* Internal flag for accounting during audit */ - /* Array of flags for each DB */ - uint8_t *mt_dbistate; - /* Number of DB records in use, or 0 when the txn is finished. - * This number only ever increments until the txn finishes; we - * don't decrement it when individual DB handles are closed. */ - MDBX_dbi mt_numdbs; - size_t mt_owner; /* thread ID that owns this transaction */ - MDBX_canary mt_canary; - void *mt_userctx; /* User-settable context */ - MDBX_cursor **mt_cursors; +#define TRACE(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_TRACE)) \ + debug_log(MDBX_LOG_TRACE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ + } while (0) - union { - struct { - /* For read txns: This thread/txn's reader table slot, or NULL. */ - MDBX_reader *reader; - } to; - struct { - meta_troika_t troika; - /* In write txns, array of cursors for each DB */ - MDBX_PNL relist; /* Reclaimed GC pages */ - txnid_t last_reclaimed; /* ID of last used record */ -#if MDBX_ENABLE_REFUND - pgno_t loose_refund_wl /* FIXME: describe */; -#endif /* MDBX_ENABLE_REFUND */ - /* a sequence to spilling dirty page with LRU policy */ - unsigned dirtylru; - /* dirtylist room: Dirty array size - dirty pages visible to this txn. - * Includes ancestor txns' dirty pages not hidden by other txns' - * dirty/spilled pages. Thus commit(nested txn) has room to merge - * dirtylist into mt_parent after freeing hidden mt_parent pages. */ - size_t dirtyroom; - /* For write txns: Modified pages. Sorted when not MDBX_WRITEMAP. */ - MDBX_dpl *dirtylist; - /* The list of reclaimed txns from GC */ - MDBX_TXL lifo_reclaimed; - /* The list of pages that became unused during this transaction. */ - MDBX_PNL retired_pages; - /* The list of loose pages that became unused and may be reused - * in this transaction, linked through `mp_next`. */ - MDBX_page *loose_pages; - /* Number of loose pages (tw.loose_pages) */ - size_t loose_count; - union { - struct { - size_t least_removed; - /* The sorted list of dirty pages we temporarily wrote to disk - * because the dirty list was full. page numbers in here are - * shifted left by 1, deleted slots have the LSB set. */ - MDBX_PNL list; - } spilled; - size_t writemap_dirty_npages; - size_t writemap_spilled_npages; - }; - } tw; - }; -}; +#define DEBUG(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_DEBUG)) \ + debug_log(MDBX_LOG_DEBUG, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ + } while (0) -#if MDBX_WORDBITS >= 64 -#define CURSOR_STACK 32 -#else -#define CURSOR_STACK 24 -#endif +#define VERBOSE(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_VERBOSE)) \ + debug_log(MDBX_LOG_VERBOSE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ + } while (0) -struct MDBX_xcursor; +#define NOTICE(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_NOTICE)) \ + debug_log(MDBX_LOG_NOTICE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ + } while (0) -/* Cursors are used for all DB operations. - * A cursor holds a path of (page pointer, key index) from the DB - * root to a position in the DB, plus other state. MDBX_DUPSORT - * cursors include an xcursor to the current data item. Write txns - * track their cursors and keep them up to date when data moves. - * Exception: An xcursor's pointer to a P_SUBP page can be stale. - * (A node with F_DUPDATA but no F_SUBDATA contains a subpage). */ -struct MDBX_cursor { -#define MDBX_MC_LIVE UINT32_C(0xFE05D5B1) -#define MDBX_MC_READY4CLOSE UINT32_C(0x2817A047) -#define MDBX_MC_WAIT4EOT UINT32_C(0x90E297A7) - uint32_t mc_signature; - /* The database handle this cursor operates on */ - MDBX_dbi mc_dbi; - /* Next cursor on this DB in this txn */ - MDBX_cursor *mc_next; - /* Backup of the original cursor if this cursor is a shadow */ - MDBX_cursor *mc_backup; - /* Context used for databases with MDBX_DUPSORT, otherwise NULL */ - struct MDBX_xcursor *mc_xcursor; - /* The transaction that owns this cursor */ - MDBX_txn *mc_txn; - /* The database record for this cursor */ - MDBX_db *mc_db; - /* The database auxiliary record for this cursor */ - MDBX_dbx *mc_dbx; - /* The mt_dbistate for this database */ - uint8_t *mc_dbistate; - uint8_t mc_snum; /* number of pushed pages */ - uint8_t mc_top; /* index of top page, normally mc_snum-1 */ - - /* Cursor state flags. */ -#define C_INITIALIZED 0x01 /* cursor has been initialized and is valid */ -#define C_EOF 0x02 /* No more data */ -#define C_SUB 0x04 /* Cursor is a sub-cursor */ -#define C_DEL 0x08 /* last op was a cursor_del */ -#define C_UNTRACK 0x10 /* Un-track cursor when closing */ -#define C_GCU \ - 0x20 /* Происходит подготовка к обновлению GC, поэтому \ - * можно брать страницы из GC даже для FREE_DBI */ - uint8_t mc_flags; - - /* Cursor checking flags. */ -#define CC_BRANCH 0x01 /* same as P_BRANCH for CHECK_LEAF_TYPE() */ -#define CC_LEAF 0x02 /* same as P_LEAF for CHECK_LEAF_TYPE() */ -#define CC_OVERFLOW 0x04 /* same as P_OVERFLOW for CHECK_LEAF_TYPE() */ -#define CC_UPDATING 0x08 /* update/rebalance pending */ -#define CC_SKIPORD 0x10 /* don't check keys ordering */ -#define CC_LEAF2 0x20 /* same as P_LEAF2 for CHECK_LEAF_TYPE() */ -#define CC_RETIRING 0x40 /* refs to child pages may be invalid */ -#define CC_PAGECHECK 0x80 /* perform page checking, see MDBX_VALIDATION */ - uint8_t mc_checking; - - MDBX_page *mc_pg[CURSOR_STACK]; /* stack of pushed pages */ - indx_t mc_ki[CURSOR_STACK]; /* stack of page indices */ -}; +#define WARNING(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_WARN)) \ + debug_log(MDBX_LOG_WARN, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ + } while (0) -#define CHECK_LEAF_TYPE(mc, mp) \ - (((PAGETYPE_WHOLE(mp) ^ (mc)->mc_checking) & \ - (CC_BRANCH | CC_LEAF | CC_OVERFLOW | CC_LEAF2)) == 0) - -/* Context for sorted-dup records. - * We could have gone to a fully recursive design, with arbitrarily - * deep nesting of sub-databases. But for now we only handle these - * levels - main DB, optional sub-DB, sorted-duplicate DB. */ -typedef struct MDBX_xcursor { - /* A sub-cursor for traversing the Dup DB */ - MDBX_cursor mx_cursor; - /* The database record for this Dup DB */ - MDBX_db mx_db; - /* The auxiliary DB record for this Dup DB */ - MDBX_dbx mx_dbx; -} MDBX_xcursor; - -typedef struct MDBX_cursor_couple { - MDBX_cursor outer; - void *mc_userctx; /* User-settable context */ - MDBX_xcursor inner; -} MDBX_cursor_couple; +#undef ERROR /* wingdi.h \ + Yeah, morons from M$ put such definition to the public header. */ -/* The database environment. */ -struct MDBX_env { - /* ----------------------------------------------------- mostly static part */ -#define MDBX_ME_SIGNATURE UINT32_C(0x9A899641) - MDBX_atomic_uint32_t me_signature; - /* Failed to update the meta page. Probably an I/O error. */ -#define MDBX_FATAL_ERROR UINT32_C(0x80000000) - /* Some fields are initialized. */ -#define MDBX_ENV_ACTIVE UINT32_C(0x20000000) - /* me_txkey is set */ -#define MDBX_ENV_TXKEY UINT32_C(0x10000000) - /* Legacy MDBX_MAPASYNC (prior v0.9) */ -#define MDBX_DEPRECATED_MAPASYNC UINT32_C(0x100000) - /* Legacy MDBX_COALESCE (prior v0.12) */ -#define MDBX_DEPRECATED_COALESCE UINT32_C(0x2000000) -#define ENV_INTERNAL_FLAGS (MDBX_FATAL_ERROR | MDBX_ENV_ACTIVE | MDBX_ENV_TXKEY) - uint32_t me_flags; - osal_mmap_t me_dxb_mmap; /* The main data file */ -#define me_map me_dxb_mmap.base -#define me_lazy_fd me_dxb_mmap.fd - mdbx_filehandle_t me_dsync_fd, me_fd4meta; -#if defined(_WIN32) || defined(_WIN64) -#define me_overlapped_fd me_ioring.overlapped_fd - HANDLE me_data_lock_event; -#endif /* Windows */ - osal_mmap_t me_lck_mmap; /* The lock file */ -#define me_lfd me_lck_mmap.fd - struct MDBX_lockinfo *me_lck; - - unsigned me_psize; /* DB page size, initialized from me_os_psize */ - uint16_t me_leaf_nodemax; /* max size of a leaf-node */ - uint16_t me_branch_nodemax; /* max size of a branch-node */ - uint16_t me_subpage_limit; - uint16_t me_subpage_room_threshold; - uint16_t me_subpage_reserve_prereq; - uint16_t me_subpage_reserve_limit; - atomic_pgno_t me_mlocked_pgno; - uint8_t me_psize2log; /* log2 of DB page size */ - int8_t me_stuck_meta; /* recovery-only: target meta page or less that zero */ - uint16_t me_merge_threshold, - me_merge_threshold_gc; /* pages emptier than this are candidates for - merging */ - unsigned me_os_psize; /* OS page size, from osal_syspagesize() */ - unsigned me_maxreaders; /* size of the reader table */ - MDBX_dbi me_maxdbs; /* size of the DB table */ - uint32_t me_pid; /* process ID of this env */ - osal_thread_key_t me_txkey; /* thread-key for readers */ - pathchar_t *me_pathname; /* path to the DB files */ - void *me_pbuf; /* scratch area for DUPSORT put() */ - MDBX_txn *me_txn0; /* preallocated write transaction */ - - MDBX_dbx *me_dbxs; /* array of static DB info */ - uint16_t *me_dbflags; /* array of flags from MDBX_db.md_flags */ - MDBX_atomic_uint32_t *me_dbiseqs; /* array of dbi sequence numbers */ - unsigned - me_maxgc_ov1page; /* Number of pgno_t fit in a single overflow page */ - unsigned me_maxgc_per_branch; - uint32_t me_live_reader; /* have liveness lock in reader table */ - void *me_userctx; /* User-settable context */ - MDBX_hsr_func *me_hsr_callback; /* Callback for kicking laggard readers */ - size_t me_madv_threshold; +#define ERROR(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_ERROR)) \ + debug_log(MDBX_LOG_ERROR, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ + } while (0) - struct { - unsigned dp_reserve_limit; - unsigned rp_augment_limit; - unsigned dp_limit; - unsigned dp_initial; - uint8_t dp_loose_limit; - uint8_t spill_max_denominator; - uint8_t spill_min_denominator; - uint8_t spill_parent4child_denominator; - unsigned merge_threshold_16dot16_percent; -#if !(defined(_WIN32) || defined(_WIN64)) - unsigned writethrough_threshold; -#endif /* Windows */ - bool prefault_write; - union { - unsigned all; - /* tracks options with non-auto values but tuned by user */ - struct { - unsigned dp_limit : 1; - unsigned rp_augment_limit : 1; - unsigned prefault_write : 1; - } non_auto; - } flags; - } me_options; +#define FATAL(fmt, ...) debug_log(MDBX_LOG_FATAL, __func__, __LINE__, fmt "\n", __VA_ARGS__); - /* struct me_dbgeo used for accepting db-geo params from user for the new - * database creation, i.e. when mdbx_env_set_geometry() was called before - * mdbx_env_open(). */ - struct { - size_t lower; /* minimal size of datafile */ - size_t upper; /* maximal size of datafile */ - size_t now; /* current size of datafile */ - size_t grow; /* step to grow datafile */ - size_t shrink; /* threshold to shrink datafile */ - } me_dbgeo; +#if MDBX_DEBUG +#define ASSERT_FAIL(env, msg, func, line) mdbx_assert_fail(env, msg, func, line) +#else /* MDBX_DEBUG */ +MDBX_NORETURN __cold void assert_fail(const char *msg, const char *func, unsigned line); +#define ASSERT_FAIL(env, msg, func, line) \ + do { \ + (void)(env); \ + assert_fail(msg, func, line); \ + } while (0) +#endif /* MDBX_DEBUG */ -#if MDBX_LOCKING == MDBX_LOCKING_SYSV - union { - key_t key; - int semid; - } me_sysv_ipc; -#endif /* MDBX_LOCKING == MDBX_LOCKING_SYSV */ - bool me_incore; +#define ENSURE_MSG(env, expr, msg) \ + do { \ + if (unlikely(!(expr))) \ + ASSERT_FAIL(env, msg, __func__, __LINE__); \ + } while (0) - MDBX_env *me_lcklist_next; +#define ENSURE(env, expr) ENSURE_MSG(env, expr, #expr) - /* --------------------------------------------------- mostly volatile part */ +/* assert(3) variant in environment context */ +#define eASSERT(env, expr) \ + do { \ + if (ASSERT_ENABLED()) \ + ENSURE(env, expr); \ + } while (0) - MDBX_txn *me_txn; /* current write transaction */ - osal_fastmutex_t me_dbi_lock; - MDBX_dbi me_numdbs; /* number of DBs opened */ - bool me_prefault_write; +/* assert(3) variant in cursor context */ +#define cASSERT(mc, expr) eASSERT((mc)->txn->env, expr) - MDBX_page *me_dp_reserve; /* list of malloc'ed blocks for re-use */ - unsigned me_dp_reserve_len; - /* PNL of pages that became unused in a write txn */ - MDBX_PNL me_retired_pages; - osal_ioring_t me_ioring; +/* assert(3) variant in transaction context */ +#define tASSERT(txn, expr) eASSERT((txn)->env, expr) -#if defined(_WIN32) || defined(_WIN64) - osal_srwlock_t me_remap_guard; - /* Workaround for LockFileEx and WriteFile multithread bug */ - CRITICAL_SECTION me_windowsbug_lock; - char *me_pathname_char; /* cache of multi-byte representation of pathname - to the DB files */ -#else - osal_fastmutex_t me_remap_guard; +#ifndef xMDBX_TOOLS /* Avoid using internal eASSERT() */ +#undef assert +#define assert(expr) eASSERT(nullptr, expr) #endif - /* -------------------------------------------------------------- debugging */ - +MDBX_MAYBE_UNUSED static inline void jitter4testing(bool tiny) { #if MDBX_DEBUG - MDBX_assert_func *me_assert_func; /* Callback for assertion failures */ -#endif -#ifdef MDBX_USE_VALGRIND - int me_valgrind_handle; -#endif -#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__) - MDBX_atomic_uint32_t me_ignore_EDEADLK; - pgno_t me_poison_edge; -#endif /* MDBX_USE_VALGRIND || __SANITIZE_ADDRESS__ */ - -#ifndef xMDBX_DEBUG_SPILLING -#define xMDBX_DEBUG_SPILLING 0 -#endif -#if xMDBX_DEBUG_SPILLING == 2 - size_t debug_dirtied_est, debug_dirtied_act; -#endif /* xMDBX_DEBUG_SPILLING */ - - /* ------------------------------------------------- stub for lck-less mode */ - MDBX_atomic_uint64_t - x_lckless_stub[(sizeof(MDBX_lockinfo) + MDBX_CACHELINE_SIZE - 1) / - sizeof(MDBX_atomic_uint64_t)]; -}; - -#ifndef __cplusplus -/*----------------------------------------------------------------------------*/ -/* Cache coherence and mmap invalidation */ - -#if MDBX_CPU_WRITEBACK_INCOHERENT -#define osal_flush_incoherent_cpu_writeback() osal_memory_barrier() -#else -#define osal_flush_incoherent_cpu_writeback() osal_compiler_barrier() -#endif /* MDBX_CPU_WRITEBACK_INCOHERENT */ - -MDBX_MAYBE_UNUSED static __inline void -osal_flush_incoherent_mmap(const void *addr, size_t nbytes, - const intptr_t pagesize) { -#if MDBX_MMAP_INCOHERENT_FILE_WRITE - char *const begin = (char *)(-pagesize & (intptr_t)addr); - char *const end = - (char *)(-pagesize & (intptr_t)((char *)addr + nbytes + pagesize - 1)); - int err = msync(begin, end - begin, MS_SYNC | MS_INVALIDATE) ? errno : 0; - eASSERT(nullptr, err == 0); - (void)err; -#else - (void)pagesize; -#endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */ - -#if MDBX_MMAP_INCOHERENT_CPU_CACHE -#ifdef DCACHE - /* MIPS has cache coherency issues. - * Note: for any nbytes >= on-chip cache size, entire is flushed. */ - cacheflush((void *)addr, nbytes, DCACHE); + if (globals.runtime_flags & (unsigned)MDBX_DBG_JITTER) + osal_jitter(tiny); #else -#error "Oops, cacheflush() not available" -#endif /* DCACHE */ -#endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */ - -#if !MDBX_MMAP_INCOHERENT_FILE_WRITE && !MDBX_MMAP_INCOHERENT_CPU_CACHE - (void)addr; - (void)nbytes; + (void)tiny; #endif } -/*----------------------------------------------------------------------------*/ -/* Internal prototypes */ - -MDBX_INTERNAL_FUNC int cleanup_dead_readers(MDBX_env *env, int rlocked, - int *dead); -MDBX_INTERNAL_FUNC int rthc_alloc(osal_thread_key_t *key, MDBX_reader *begin, - MDBX_reader *end); -MDBX_INTERNAL_FUNC void rthc_remove(const osal_thread_key_t key); - -MDBX_INTERNAL_FUNC void global_ctor(void); -MDBX_INTERNAL_FUNC void osal_ctor(void); -MDBX_INTERNAL_FUNC void global_dtor(void); -MDBX_INTERNAL_FUNC void osal_dtor(void); -MDBX_INTERNAL_FUNC void thread_dtor(void *ptr); - -#endif /* !__cplusplus */ - -#define MDBX_IS_ERROR(rc) \ - ((rc) != MDBX_RESULT_TRUE && (rc) != MDBX_RESULT_FALSE) - -/* Internal error codes, not exposed outside libmdbx */ -#define MDBX_NO_ROOT (MDBX_LAST_ADDED_ERRCODE + 10) - -/* Debugging output value of a cursor DBI: Negative in a sub-cursor. */ -#define DDBI(mc) \ - (((mc)->mc_flags & C_SUB) ? -(int)(mc)->mc_dbi : (int)(mc)->mc_dbi) +MDBX_MAYBE_UNUSED MDBX_INTERNAL void page_list(page_t *mp); +MDBX_INTERNAL const char *pagetype_caption(const uint8_t type, char buf4unknown[16]); /* Key size which fits in a DKBUF (debug key buffer). */ -#define DKBUF_MAX 511 -#define DKBUF char _kbuf[DKBUF_MAX * 4 + 2] -#define DKEY(x) mdbx_dump_val(x, _kbuf, DKBUF_MAX * 2 + 1) -#define DVAL(x) mdbx_dump_val(x, _kbuf + DKBUF_MAX * 2 + 1, DKBUF_MAX * 2 + 1) +#define DKBUF_MAX 127 +#define DKBUF char dbg_kbuf[DKBUF_MAX * 4 + 2] +#define DKEY(x) mdbx_dump_val(x, dbg_kbuf, DKBUF_MAX * 2 + 1) +#define DVAL(x) mdbx_dump_val(x, dbg_kbuf + DKBUF_MAX * 2 + 1, DKBUF_MAX * 2 + 1) #if MDBX_DEBUG #define DKBUF_DEBUG DKBUF @@ -3842,102 +2965,24 @@ MDBX_INTERNAL_FUNC void thread_dtor(void *ptr); #define DVAL_DEBUG(x) ("-") #endif -/* An invalid page number. - * Mainly used to denote an empty tree. */ -#define P_INVALID (~(pgno_t)0) +MDBX_INTERNAL void log_error(const int err, const char *func, unsigned line); + +MDBX_MAYBE_UNUSED static inline int log_if_error(const int err, const char *func, unsigned line) { + if (unlikely(err != MDBX_SUCCESS)) + log_error(err, func, line); + return err; +} + +#define LOG_IFERR(err) log_if_error((err), __func__, __LINE__) /* Test if the flags f are set in a flag word w. */ #define F_ISSET(w, f) (((w) & (f)) == (f)) /* Round n up to an even number. */ -#define EVEN(n) (((n) + 1UL) & -2L) /* sign-extending -2 to match n+1U */ - -/* Default size of memory map. - * This is certainly too small for any actual applications. Apps should - * always set the size explicitly using mdbx_env_set_geometry(). */ -#define DEFAULT_MAPSIZE MEGABYTE - -/* Number of slots in the reader table. - * This value was chosen somewhat arbitrarily. The 61 is a prime number, - * and such readers plus a couple mutexes fit into single 4KB page. - * Applications should set the table size using mdbx_env_set_maxreaders(). */ -#define DEFAULT_READERS 61 +#define EVEN_CEIL(n) (((n) + 1UL) & -2L) /* sign-extending -2 to match n+1U */ -/* Test if a page is a leaf page */ -#define IS_LEAF(p) (((p)->mp_flags & P_LEAF) != 0) -/* Test if a page is a LEAF2 page */ -#define IS_LEAF2(p) unlikely(((p)->mp_flags & P_LEAF2) != 0) -/* Test if a page is a branch page */ -#define IS_BRANCH(p) (((p)->mp_flags & P_BRANCH) != 0) -/* Test if a page is an overflow page */ -#define IS_OVERFLOW(p) unlikely(((p)->mp_flags & P_OVERFLOW) != 0) -/* Test if a page is a sub page */ -#define IS_SUBP(p) (((p)->mp_flags & P_SUBP) != 0) - -/* Header for a single key/data pair within a page. - * Used in pages of type P_BRANCH and P_LEAF without P_LEAF2. - * We guarantee 2-byte alignment for 'MDBX_node's. - * - * Leaf node flags describe node contents. F_BIGDATA says the node's - * data part is the page number of an overflow page with actual data. - * F_DUPDATA and F_SUBDATA can be combined giving duplicate data in - * a sub-page/sub-database, and named databases (just F_SUBDATA). */ -typedef struct MDBX_node { -#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ - union { - uint32_t mn_dsize; - uint32_t mn_pgno32; - }; - uint8_t mn_flags; /* see mdbx_node flags */ - uint8_t mn_extra; - uint16_t mn_ksize; /* key size */ -#else - uint16_t mn_ksize; /* key size */ - uint8_t mn_extra; - uint8_t mn_flags; /* see mdbx_node flags */ - union { - uint32_t mn_pgno32; - uint32_t mn_dsize; - }; -#endif /* __BYTE_ORDER__ */ - - /* mdbx_node Flags */ -#define F_BIGDATA 0x01 /* data put on overflow page */ -#define F_SUBDATA 0x02 /* data is a sub-database */ -#define F_DUPDATA 0x04 /* data has duplicates */ - - /* valid flags for mdbx_node_add() */ -#define NODE_ADD_FLAGS (F_DUPDATA | F_SUBDATA | MDBX_RESERVE | MDBX_APPEND) - -#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ - (!defined(__cplusplus) && defined(_MSC_VER)) - uint8_t mn_data[] /* key and data are appended here */; -#endif /* C99 */ -} MDBX_node; - -#define DB_PERSISTENT_FLAGS \ - (MDBX_REVERSEKEY | MDBX_DUPSORT | MDBX_INTEGERKEY | MDBX_DUPFIXED | \ - MDBX_INTEGERDUP | MDBX_REVERSEDUP) - -/* mdbx_dbi_open() flags */ -#define DB_USABLE_FLAGS (DB_PERSISTENT_FLAGS | MDBX_CREATE | MDBX_DB_ACCEDE) - -#define DB_VALID 0x8000 /* DB handle is valid, for me_dbflags */ -#define DB_INTERNAL_FLAGS DB_VALID - -#if DB_INTERNAL_FLAGS & DB_USABLE_FLAGS -#error "Oops, some flags overlapped or wrong" -#endif -#if DB_PERSISTENT_FLAGS & ~DB_USABLE_FLAGS -#error "Oops, some flags overlapped or wrong" -#endif - -/* Max length of iov-vector passed to writev() call, used for auxilary writes */ -#define MDBX_AUXILARY_IOV_MAX 64 -#if defined(IOV_MAX) && IOV_MAX < MDBX_AUXILARY_IOV_MAX -#undef MDBX_AUXILARY_IOV_MAX -#define MDBX_AUXILARY_IOV_MAX IOV_MAX -#endif /* MDBX_AUXILARY_IOV_MAX */ +/* Round n down to an even number. */ +#define EVEN_FLOOR(n) ((n) & ~(size_t)1) /* * / @@ -3948,17397 +2993,13206 @@ typedef struct MDBX_node { */ #define CMP2INT(a, b) (((a) != (b)) ? (((a) < (b)) ? -1 : 1) : 0) -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline pgno_t -int64pgno(int64_t i64) { - if (likely(i64 >= (int64_t)MIN_PAGENO && i64 <= (int64_t)MAX_PAGENO + 1)) - return (pgno_t)i64; - return (i64 < (int64_t)MIN_PAGENO) ? MIN_PAGENO : MAX_PAGENO; -} +/* Pointer displacement without casting to char* to avoid pointer-aliasing */ +#define ptr_disp(ptr, disp) ((void *)(((intptr_t)(ptr)) + ((intptr_t)(disp)))) -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline pgno_t -pgno_add(size_t base, size_t augend) { - assert(base <= MAX_PAGENO + 1 && augend < MAX_PAGENO); - return int64pgno((int64_t)base + (int64_t)augend); -} +/* Pointer distance as signed number of bytes */ +#define ptr_dist(more, less) (((intptr_t)(more)) - ((intptr_t)(less))) -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline pgno_t -pgno_sub(size_t base, size_t subtrahend) { - assert(base >= MIN_PAGENO && base <= MAX_PAGENO + 1 && - subtrahend < MAX_PAGENO); - return int64pgno((int64_t)base - (int64_t)subtrahend); -} +#define MDBX_ASAN_POISON_MEMORY_REGION(addr, size) \ + do { \ + TRACE("POISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), (size_t)(size), __LINE__); \ + ASAN_POISON_MEMORY_REGION(addr, size); \ + } while (0) + +#define MDBX_ASAN_UNPOISON_MEMORY_REGION(addr, size) \ + do { \ + TRACE("UNPOISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), (size_t)(size), __LINE__); \ + ASAN_UNPOISON_MEMORY_REGION(addr, size); \ + } while (0) -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __always_inline bool -is_powerof2(size_t x) { - return (x & (x - 1)) == 0; +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline size_t branchless_abs(intptr_t value) { + assert(value > INT_MIN); + const size_t expanded_sign = (size_t)(value >> (sizeof(value) * CHAR_BIT - 1)); + return ((size_t)value + expanded_sign) ^ expanded_sign; } -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __always_inline size_t -floor_powerof2(size_t value, size_t granularity) { +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline bool is_powerof2(size_t x) { return (x & (x - 1)) == 0; } + +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline size_t floor_powerof2(size_t value, size_t granularity) { assert(is_powerof2(granularity)); return value & ~(granularity - 1); } -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __always_inline size_t -ceil_powerof2(size_t value, size_t granularity) { +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline size_t ceil_powerof2(size_t value, size_t granularity) { return floor_powerof2(value + granularity - 1, granularity); } -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static unsigned -log2n_powerof2(size_t value_uintptr) { - assert(value_uintptr > 0 && value_uintptr < INT32_MAX && - is_powerof2(value_uintptr)); - assert((value_uintptr & -(intptr_t)value_uintptr) == value_uintptr); - const uint32_t value_uint32 = (uint32_t)value_uintptr; -#if __GNUC_PREREQ(4, 1) || __has_builtin(__builtin_ctz) - STATIC_ASSERT(sizeof(value_uint32) <= sizeof(unsigned)); - return __builtin_ctz(value_uint32); -#elif defined(_MSC_VER) - unsigned long index; - STATIC_ASSERT(sizeof(value_uint32) <= sizeof(long)); - _BitScanForward(&index, value_uint32); - return index; -#else - static const uint8_t debruijn_ctz32[32] = { - 0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8, - 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9}; - return debruijn_ctz32[(uint32_t)(value_uint32 * 0x077CB531ul) >> 27]; -#endif -} +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED MDBX_INTERNAL unsigned log2n_powerof2(size_t value_uintptr); -/* Only a subset of the mdbx_env flags can be changed - * at runtime. Changing other flags requires closing the - * environment and re-opening it with the new flags. */ -#define ENV_CHANGEABLE_FLAGS \ - (MDBX_SAFE_NOSYNC | MDBX_NOMETASYNC | MDBX_DEPRECATED_MAPASYNC | \ - MDBX_NOMEMINIT | MDBX_COALESCE | MDBX_PAGEPERTURB | MDBX_ACCEDE | \ - MDBX_VALIDATION) -#define ENV_CHANGELESS_FLAGS \ - (MDBX_NOSUBDIR | MDBX_RDONLY | MDBX_WRITEMAP | MDBX_NOTLS | MDBX_NORDAHEAD | \ - MDBX_LIFORECLAIM | MDBX_EXCLUSIVE) -#define ENV_USABLE_FLAGS (ENV_CHANGEABLE_FLAGS | ENV_CHANGELESS_FLAGS) +MDBX_NOTHROW_CONST_FUNCTION MDBX_INTERNAL uint64_t rrxmrrxmsx_0(uint64_t v); -#if !defined(__cplusplus) || CONSTEXPR_ENUM_FLAGS_OPERATIONS -MDBX_MAYBE_UNUSED static void static_checks(void) { - STATIC_ASSERT_MSG(INT16_MAX - CORE_DBS == MDBX_MAX_DBI, - "Oops, MDBX_MAX_DBI or CORE_DBS?"); - STATIC_ASSERT_MSG((unsigned)(MDBX_DB_ACCEDE | MDBX_CREATE) == - ((DB_USABLE_FLAGS | DB_INTERNAL_FLAGS) & - (ENV_USABLE_FLAGS | ENV_INTERNAL_FLAGS)), - "Oops, some flags overlapped or wrong"); - STATIC_ASSERT_MSG((ENV_INTERNAL_FLAGS & ENV_USABLE_FLAGS) == 0, - "Oops, some flags overlapped or wrong"); -} -#endif /* Disabled for MSVC 19.0 (VisualStudio 2015) */ +struct monotime_cache { + uint64_t value; + int expire_countdown; +}; -#ifdef __cplusplus +MDBX_MAYBE_UNUSED static inline uint64_t monotime_since_cached(uint64_t begin_timestamp, struct monotime_cache *cache) { + if (cache->expire_countdown) + cache->expire_countdown -= 1; + else { + cache->value = osal_monotime(); + cache->expire_countdown = 42 / 3; + } + return cache->value - begin_timestamp; } + +/* An PNL is an Page Number List, a sorted array of IDs. + * + * The first element of the array is a counter for how many actual page-numbers + * are in the list. By default PNLs are sorted in descending order, this allow + * cut off a page with lowest pgno (at the tail) just truncating the list. The + * sort order of PNLs is controlled by the MDBX_PNL_ASCENDING build option. */ +typedef pgno_t *pnl_t; +typedef const pgno_t *const_pnl_t; + +#if MDBX_PNL_ASCENDING +#define MDBX_PNL_ORDERED(first, last) ((first) < (last)) +#define MDBX_PNL_DISORDERED(first, last) ((first) >= (last)) +#else +#define MDBX_PNL_ORDERED(first, last) ((first) > (last)) +#define MDBX_PNL_DISORDERED(first, last) ((first) <= (last)) #endif -#define MDBX_ASAN_POISON_MEMORY_REGION(addr, size) \ - do { \ - TRACE("POISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), \ - (size_t)(size), __LINE__); \ - ASAN_POISON_MEMORY_REGION(addr, size); \ - } while (0) +#define MDBX_PNL_GRANULATE_LOG2 10 +#define MDBX_PNL_GRANULATE (1 << MDBX_PNL_GRANULATE_LOG2) +#define MDBX_PNL_INITIAL (MDBX_PNL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t)) -#define MDBX_ASAN_UNPOISON_MEMORY_REGION(addr, size) \ - do { \ - TRACE("UNPOISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), \ - (size_t)(size), __LINE__); \ - ASAN_UNPOISON_MEMORY_REGION(addr, size); \ +#define MDBX_PNL_ALLOCLEN(pl) ((pl)[-1]) +#define MDBX_PNL_GETSIZE(pl) ((size_t)((pl)[0])) +#define MDBX_PNL_SETSIZE(pl, size) \ + do { \ + const size_t __size = size; \ + assert(__size < INT_MAX); \ + (pl)[0] = (pgno_t)__size; \ } while (0) -/* - * Copyright 2015-2024 Leonid Yuriev . - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * This code is derived from "LMDB engine" written by - * Howard Chu (Symas Corporation), which itself derived from btree.c - * written by Martin Hedenfalk. - * - * --- - * - * Portions Copyright 2011-2015 Howard Chu, Symas Corp. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - * - * --- - * - * Portions Copyright (c) 2009, 2010 Martin Hedenfalk - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ +#define MDBX_PNL_FIRST(pl) ((pl)[1]) +#define MDBX_PNL_LAST(pl) ((pl)[MDBX_PNL_GETSIZE(pl)]) +#define MDBX_PNL_BEGIN(pl) (&(pl)[1]) +#define MDBX_PNL_END(pl) (&(pl)[MDBX_PNL_GETSIZE(pl) + 1]) +#if MDBX_PNL_ASCENDING +#define MDBX_PNL_EDGE(pl) ((pl) + 1) +#define MDBX_PNL_LEAST(pl) MDBX_PNL_FIRST(pl) +#define MDBX_PNL_MOST(pl) MDBX_PNL_LAST(pl) +#else +#define MDBX_PNL_EDGE(pl) ((pl) + MDBX_PNL_GETSIZE(pl)) +#define MDBX_PNL_LEAST(pl) MDBX_PNL_LAST(pl) +#define MDBX_PNL_MOST(pl) MDBX_PNL_FIRST(pl) +#endif -/*------------------------------------------------------------------------------ - * Internal inline functions */ +#define MDBX_PNL_SIZEOF(pl) ((MDBX_PNL_GETSIZE(pl) + 1) * sizeof(pgno_t)) +#define MDBX_PNL_IS_EMPTY(pl) (MDBX_PNL_GETSIZE(pl) == 0) -MDBX_NOTHROW_CONST_FUNCTION static size_t branchless_abs(intptr_t value) { - assert(value > INT_MIN); - const size_t expanded_sign = - (size_t)(value >> (sizeof(value) * CHAR_BIT - 1)); - return ((size_t)value + expanded_sign) ^ expanded_sign; -} +MDBX_MAYBE_UNUSED static inline size_t pnl_size2bytes(size_t size) { + assert(size > 0 && size <= PAGELIST_LIMIT); +#if MDBX_PNL_PREALLOC_FOR_RADIXSORT -/* Pack/Unpack 16-bit values for Grow step & Shrink threshold */ -MDBX_NOTHROW_CONST_FUNCTION static __inline pgno_t me2v(size_t m, size_t e) { - assert(m < 2048 && e < 8); - return (pgno_t)(32768 + ((m + 1) << (e + 8))); + size += size; +#endif /* MDBX_PNL_PREALLOC_FOR_RADIXSORT */ + STATIC_ASSERT(MDBX_ASSUME_MALLOC_OVERHEAD + + (PAGELIST_LIMIT * (MDBX_PNL_PREALLOC_FOR_RADIXSORT + 1) + MDBX_PNL_GRANULATE + 3) * sizeof(pgno_t) < + SIZE_MAX / 4 * 3); + size_t bytes = + ceil_powerof2(MDBX_ASSUME_MALLOC_OVERHEAD + sizeof(pgno_t) * (size + 3), MDBX_PNL_GRANULATE * sizeof(pgno_t)) - + MDBX_ASSUME_MALLOC_OVERHEAD; + return bytes; } -MDBX_NOTHROW_CONST_FUNCTION static __inline uint16_t v2me(size_t v, size_t e) { - assert(v > (e ? me2v(2047, e - 1) : 32768)); - assert(v <= me2v(2047, e)); - size_t m = (v - 32768 + ((size_t)1 << (e + 8)) - 1) >> (e + 8); - m -= m > 0; - assert(m < 2048 && e < 8); - // f e d c b a 9 8 7 6 5 4 3 2 1 0 - // 1 e e e m m m m m m m m m m m 1 - const uint16_t pv = (uint16_t)(0x8001 + (e << 12) + (m << 1)); - assert(pv != 65535); - return pv; +MDBX_MAYBE_UNUSED static inline pgno_t pnl_bytes2size(const size_t bytes) { + size_t size = bytes / sizeof(pgno_t); + assert(size > 3 && size <= PAGELIST_LIMIT + /* alignment gap */ 65536); + size -= 3; +#if MDBX_PNL_PREALLOC_FOR_RADIXSORT + size >>= 1; +#endif /* MDBX_PNL_PREALLOC_FOR_RADIXSORT */ + return (pgno_t)size; } -/* Convert 16-bit packed (exponential quantized) value to number of pages */ -MDBX_NOTHROW_CONST_FUNCTION static pgno_t pv2pages(uint16_t pv) { - if ((pv & 0x8001) != 0x8001) - return pv; - if (pv == 65535) - return 65536; - // f e d c b a 9 8 7 6 5 4 3 2 1 0 - // 1 e e e m m m m m m m m m m m 1 - return me2v((pv >> 1) & 2047, (pv >> 12) & 7); -} +MDBX_INTERNAL pnl_t pnl_alloc(size_t size); -/* Convert number of pages to 16-bit packed (exponential quantized) value */ -MDBX_NOTHROW_CONST_FUNCTION static uint16_t pages2pv(size_t pages) { - if (pages < 32769 || (pages < 65536 && (pages & 1) == 0)) - return (uint16_t)pages; - if (pages <= me2v(2047, 0)) - return v2me(pages, 0); - if (pages <= me2v(2047, 1)) - return v2me(pages, 1); - if (pages <= me2v(2047, 2)) - return v2me(pages, 2); - if (pages <= me2v(2047, 3)) - return v2me(pages, 3); - if (pages <= me2v(2047, 4)) - return v2me(pages, 4); - if (pages <= me2v(2047, 5)) - return v2me(pages, 5); - if (pages <= me2v(2047, 6)) - return v2me(pages, 6); - return (pages < me2v(2046, 7)) ? v2me(pages, 7) : 65533; -} +MDBX_INTERNAL void pnl_free(pnl_t pnl); -/*------------------------------------------------------------------------------ - * Unaligned access */ +MDBX_INTERNAL int pnl_reserve(pnl_t __restrict *__restrict ppnl, const size_t wanna); -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __always_inline size_t -field_alignment(size_t alignment_baseline, size_t field_offset) { - size_t merge = alignment_baseline | (size_t)field_offset; - return merge & -(int)merge; +MDBX_MAYBE_UNUSED static inline int __must_check_result pnl_need(pnl_t __restrict *__restrict ppnl, size_t num) { + assert(MDBX_PNL_GETSIZE(*ppnl) <= PAGELIST_LIMIT && MDBX_PNL_ALLOCLEN(*ppnl) >= MDBX_PNL_GETSIZE(*ppnl)); + assert(num <= PAGELIST_LIMIT); + const size_t wanna = MDBX_PNL_GETSIZE(*ppnl) + num; + return likely(MDBX_PNL_ALLOCLEN(*ppnl) >= wanna) ? MDBX_SUCCESS : pnl_reserve(ppnl, wanna); } -/* read-thunk for UB-sanitizer */ -MDBX_NOTHROW_PURE_FUNCTION static __always_inline uint8_t -peek_u8(const uint8_t *const __restrict ptr) { - return *ptr; +MDBX_MAYBE_UNUSED static inline void pnl_append_prereserved(__restrict pnl_t pnl, pgno_t pgno) { + assert(MDBX_PNL_GETSIZE(pnl) < MDBX_PNL_ALLOCLEN(pnl)); + if (AUDIT_ENABLED()) { + for (size_t i = MDBX_PNL_GETSIZE(pnl); i > 0; --i) + assert(pgno != pnl[i]); + } + *pnl += 1; + MDBX_PNL_LAST(pnl) = pgno; } -/* write-thunk for UB-sanitizer */ -static __always_inline void poke_u8(uint8_t *const __restrict ptr, - const uint8_t v) { - *ptr = v; -} +MDBX_INTERNAL void pnl_shrink(pnl_t __restrict *__restrict ppnl); -MDBX_NOTHROW_PURE_FUNCTION static __always_inline uint16_t -unaligned_peek_u16(const size_t expected_alignment, const void *const ptr) { - assert((uintptr_t)ptr % expected_alignment == 0); - if (MDBX_UNALIGNED_OK >= 2 || (expected_alignment % sizeof(uint16_t)) == 0) - return *(const uint16_t *)ptr; - else { -#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || \ - defined(_M_X64) || defined(_M_IA64) - return *(const __unaligned uint16_t *)ptr; -#else - uint16_t v; - memcpy(&v, ptr, sizeof(v)); - return v; -#endif /* _MSC_VER || __unaligned */ - } -} +MDBX_INTERNAL int __must_check_result spill_append_span(__restrict pnl_t *ppnl, pgno_t pgno, size_t n); -static __always_inline void unaligned_poke_u16(const size_t expected_alignment, - void *const __restrict ptr, - const uint16_t v) { - assert((uintptr_t)ptr % expected_alignment == 0); - if (MDBX_UNALIGNED_OK >= 2 || (expected_alignment % sizeof(v)) == 0) - *(uint16_t *)ptr = v; - else { -#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || \ - defined(_M_X64) || defined(_M_IA64) - *((uint16_t __unaligned *)ptr) = v; -#else - memcpy(ptr, &v, sizeof(v)); -#endif /* _MSC_VER || __unaligned */ - } -} +MDBX_INTERNAL int __must_check_result pnl_append_span(__restrict pnl_t *ppnl, pgno_t pgno, size_t n); -MDBX_NOTHROW_PURE_FUNCTION static __always_inline uint32_t unaligned_peek_u32( - const size_t expected_alignment, const void *const __restrict ptr) { - assert((uintptr_t)ptr % expected_alignment == 0); - if (MDBX_UNALIGNED_OK >= 4 || (expected_alignment % sizeof(uint32_t)) == 0) - return *(const uint32_t *)ptr; - else if ((expected_alignment % sizeof(uint16_t)) == 0) { - const uint16_t lo = - ((const uint16_t *)ptr)[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__]; - const uint16_t hi = - ((const uint16_t *)ptr)[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__]; - return lo | (uint32_t)hi << 16; - } else { -#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || \ - defined(_M_X64) || defined(_M_IA64) - return *(const __unaligned uint32_t *)ptr; -#else - uint32_t v; - memcpy(&v, ptr, sizeof(v)); - return v; -#endif /* _MSC_VER || __unaligned */ - } -} +MDBX_INTERNAL int __must_check_result pnl_insert_span(__restrict pnl_t *ppnl, pgno_t pgno, size_t n); -static __always_inline void unaligned_poke_u32(const size_t expected_alignment, - void *const __restrict ptr, - const uint32_t v) { - assert((uintptr_t)ptr % expected_alignment == 0); - if (MDBX_UNALIGNED_OK >= 4 || (expected_alignment % sizeof(v)) == 0) - *(uint32_t *)ptr = v; - else if ((expected_alignment % sizeof(uint16_t)) == 0) { - ((uint16_t *)ptr)[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__] = (uint16_t)v; - ((uint16_t *)ptr)[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__] = - (uint16_t)(v >> 16); - } else { -#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || \ - defined(_M_X64) || defined(_M_IA64) - *((uint32_t __unaligned *)ptr) = v; -#else - memcpy(ptr, &v, sizeof(v)); -#endif /* _MSC_VER || __unaligned */ - } -} +MDBX_INTERNAL size_t pnl_search_nochk(const pnl_t pnl, pgno_t pgno); -MDBX_NOTHROW_PURE_FUNCTION static __always_inline uint64_t unaligned_peek_u64( - const size_t expected_alignment, const void *const __restrict ptr) { - assert((uintptr_t)ptr % expected_alignment == 0); - if (MDBX_UNALIGNED_OK >= 8 || (expected_alignment % sizeof(uint64_t)) == 0) - return *(const uint64_t *)ptr; - else if ((expected_alignment % sizeof(uint32_t)) == 0) { - const uint32_t lo = - ((const uint32_t *)ptr)[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__]; - const uint32_t hi = - ((const uint32_t *)ptr)[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__]; - return lo | (uint64_t)hi << 32; - } else { -#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || \ - defined(_M_X64) || defined(_M_IA64) - return *(const __unaligned uint64_t *)ptr; -#else - uint64_t v; - memcpy(&v, ptr, sizeof(v)); - return v; -#endif /* _MSC_VER || __unaligned */ - } +MDBX_INTERNAL void pnl_sort_nochk(pnl_t pnl); + +MDBX_INTERNAL bool pnl_check(const const_pnl_t pnl, const size_t limit); + +MDBX_MAYBE_UNUSED static inline bool pnl_check_allocated(const const_pnl_t pnl, const size_t limit) { + return pnl == nullptr || (MDBX_PNL_ALLOCLEN(pnl) >= MDBX_PNL_GETSIZE(pnl) && pnl_check(pnl, limit)); } -static __always_inline uint64_t -unaligned_peek_u64_volatile(const size_t expected_alignment, - const volatile void *const __restrict ptr) { - assert((uintptr_t)ptr % expected_alignment == 0); - assert(expected_alignment % sizeof(uint32_t) == 0); - if (MDBX_UNALIGNED_OK >= 8 || (expected_alignment % sizeof(uint64_t)) == 0) - return *(const volatile uint64_t *)ptr; - else { -#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || \ - defined(_M_X64) || defined(_M_IA64) - return *(const volatile __unaligned uint64_t *)ptr; -#else - const uint32_t lo = ((const volatile uint32_t *) - ptr)[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__]; - const uint32_t hi = ((const volatile uint32_t *) - ptr)[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__]; - return lo | (uint64_t)hi << 32; -#endif /* _MSC_VER || __unaligned */ - } +MDBX_MAYBE_UNUSED static inline void pnl_sort(pnl_t pnl, size_t limit4check) { + pnl_sort_nochk(pnl); + assert(pnl_check(pnl, limit4check)); + (void)limit4check; } -static __always_inline void unaligned_poke_u64(const size_t expected_alignment, - void *const __restrict ptr, - const uint64_t v) { - assert((uintptr_t)ptr % expected_alignment == 0); - if (MDBX_UNALIGNED_OK >= 8 || (expected_alignment % sizeof(v)) == 0) - *(uint64_t *)ptr = v; - else if ((expected_alignment % sizeof(uint32_t)) == 0) { - ((uint32_t *)ptr)[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__] = (uint32_t)v; - ((uint32_t *)ptr)[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__] = - (uint32_t)(v >> 32); - } else { -#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || \ - defined(_M_X64) || defined(_M_IA64) - *((uint64_t __unaligned *)ptr) = v; -#else - memcpy(ptr, &v, sizeof(v)); -#endif /* _MSC_VER || __unaligned */ +MDBX_MAYBE_UNUSED static inline size_t pnl_search(const pnl_t pnl, pgno_t pgno, size_t limit) { + assert(pnl_check_allocated(pnl, limit)); + if (MDBX_HAVE_CMOV) { + /* cmov-ускоренный бинарный поиск может читать (но не использовать) один + * элемент за концом данных, этот элемент в пределах выделенного участка + * памяти, но не инициализирован. */ + VALGRIND_MAKE_MEM_DEFINED(MDBX_PNL_END(pnl), sizeof(pgno_t)); + } + assert(pgno < limit); + (void)limit; + size_t n = pnl_search_nochk(pnl, pgno); + if (MDBX_HAVE_CMOV) { + VALGRIND_MAKE_MEM_UNDEFINED(MDBX_PNL_END(pnl), sizeof(pgno_t)); } + return n; } -#define UNALIGNED_PEEK_8(ptr, struct, field) \ - peek_u8(ptr_disp(ptr, offsetof(struct, field))) -#define UNALIGNED_POKE_8(ptr, struct, field, value) \ - poke_u8(ptr_disp(ptr, offsetof(struct, field)), value) +MDBX_INTERNAL size_t pnl_merge(pnl_t dst, const pnl_t src); -#define UNALIGNED_PEEK_16(ptr, struct, field) \ - unaligned_peek_u16(1, ptr_disp(ptr, offsetof(struct, field))) -#define UNALIGNED_POKE_16(ptr, struct, field, value) \ - unaligned_poke_u16(1, ptr_disp(ptr, offsetof(struct, field)), value) +#ifdef __cplusplus +} +#endif /* __cplusplus */ -#define UNALIGNED_PEEK_32(ptr, struct, field) \ - unaligned_peek_u32(1, ptr_disp(ptr, offsetof(struct, field))) -#define UNALIGNED_POKE_32(ptr, struct, field, value) \ - unaligned_poke_u32(1, ptr_disp(ptr, offsetof(struct, field)), value) +#define mdbx_sourcery_anchor XCONCAT(mdbx_sourcery_, MDBX_BUILD_SOURCERY) +#if defined(xMDBX_TOOLS) +extern LIBMDBX_API const char *const mdbx_sourcery_anchor; +#endif -#define UNALIGNED_PEEK_64(ptr, struct, field) \ - unaligned_peek_u64(1, ptr_disp(ptr, offsetof(struct, field))) -#define UNALIGNED_POKE_64(ptr, struct, field, value) \ - unaligned_poke_u64(1, ptr_disp(ptr, offsetof(struct, field)), value) +#define MDBX_IS_ERROR(rc) ((rc) != MDBX_RESULT_TRUE && (rc) != MDBX_RESULT_FALSE) -/* Get the page number pointed to by a branch node */ -MDBX_NOTHROW_PURE_FUNCTION static __always_inline pgno_t -node_pgno(const MDBX_node *const __restrict node) { - pgno_t pgno = UNALIGNED_PEEK_32(node, MDBX_node, mn_pgno32); - if (sizeof(pgno) > 4) - pgno |= ((uint64_t)UNALIGNED_PEEK_8(node, MDBX_node, mn_extra)) << 32; - return pgno; -} - -/* Set the page number in a branch node */ -static __always_inline void node_set_pgno(MDBX_node *const __restrict node, - pgno_t pgno) { - assert(pgno >= MIN_PAGENO && pgno <= MAX_PAGENO); +/*----------------------------------------------------------------------------*/ - UNALIGNED_POKE_32(node, MDBX_node, mn_pgno32, (uint32_t)pgno); - if (sizeof(pgno) > 4) - UNALIGNED_POKE_8(node, MDBX_node, mn_extra, - (uint8_t)((uint64_t)pgno >> 32)); +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline pgno_t int64pgno(int64_t i64) { + if (likely(i64 >= (int64_t)MIN_PAGENO && i64 <= (int64_t)MAX_PAGENO + 1)) + return (pgno_t)i64; + return (i64 < (int64_t)MIN_PAGENO) ? MIN_PAGENO : MAX_PAGENO; } -/* Get the size of the data in a leaf node */ -MDBX_NOTHROW_PURE_FUNCTION static __always_inline size_t -node_ds(const MDBX_node *const __restrict node) { - return UNALIGNED_PEEK_32(node, MDBX_node, mn_dsize); +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline pgno_t pgno_add(size_t base, size_t augend) { + assert(base <= MAX_PAGENO + 1 && augend < MAX_PAGENO); + return int64pgno((int64_t)base + (int64_t)augend); } -/* Set the size of the data for a leaf node */ -static __always_inline void node_set_ds(MDBX_node *const __restrict node, - size_t size) { - assert(size < INT_MAX); - UNALIGNED_POKE_32(node, MDBX_node, mn_dsize, (uint32_t)size); +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline pgno_t pgno_sub(size_t base, size_t subtrahend) { + assert(base >= MIN_PAGENO && base <= MAX_PAGENO + 1 && subtrahend < MAX_PAGENO); + return int64pgno((int64_t)base - (int64_t)subtrahend); } -/* The size of a key in a node */ -MDBX_NOTHROW_PURE_FUNCTION static __always_inline size_t -node_ks(const MDBX_node *const __restrict node) { - return UNALIGNED_PEEK_16(node, MDBX_node, mn_ksize); -} +/*----------------------------------------------------------------------------*/ -/* Set the size of the key for a leaf node */ -static __always_inline void node_set_ks(MDBX_node *const __restrict node, - size_t size) { - assert(size < INT16_MAX); - UNALIGNED_POKE_16(node, MDBX_node, mn_ksize, (uint16_t)size); -} +typedef struct dp dp_t; +typedef struct dpl dpl_t; +typedef struct kvx kvx_t; +typedef struct meta_ptr meta_ptr_t; +typedef struct inner_cursor subcur_t; +typedef struct cursor_couple cursor_couple_t; +typedef struct defer_free_item defer_free_item_t; -MDBX_NOTHROW_PURE_FUNCTION static __always_inline uint8_t -node_flags(const MDBX_node *const __restrict node) { - return UNALIGNED_PEEK_8(node, MDBX_node, mn_flags); -} +typedef struct troika { + uint8_t fsm, recent, prefer_steady, tail_and_flags; +#if MDBX_WORDBITS > 32 /* Workaround for false-positives from Valgrind */ + uint32_t unused_pad; +#endif +#define TROIKA_HAVE_STEADY(troika) ((troika)->fsm & 7u) +#define TROIKA_STRICT_VALID(troika) ((troika)->tail_and_flags & 64u) +#define TROIKA_VALID(troika) ((troika)->tail_and_flags & 128u) +#define TROIKA_TAIL(troika) ((troika)->tail_and_flags & 3u) + txnid_t txnid[NUM_METAS]; +} troika_t; -static __always_inline void node_set_flags(MDBX_node *const __restrict node, - uint8_t flags) { - UNALIGNED_POKE_8(node, MDBX_node, mn_flags, flags); -} +typedef struct page_get_result { + page_t *page; + int err; +} pgr_t; -/* Size of the node header, excluding dynamic data at the end */ -#define NODESIZE offsetof(MDBX_node, mn_data) +typedef struct node_search_result { + node_t *node; + bool exact; +} nsr_t; -/* Address of the key for the node */ -MDBX_NOTHROW_PURE_FUNCTION static __always_inline void * -node_key(const MDBX_node *const __restrict node) { - return ptr_disp(node, NODESIZE); -} +typedef struct bind_reader_slot_result { + int err; + reader_slot_t *rslot; +} bsr_t; -/* Address of the data for a node */ -MDBX_NOTHROW_PURE_FUNCTION static __always_inline void * -node_data(const MDBX_node *const __restrict node) { - return ptr_disp(node_key(node), node_ks(node)); -} +#ifndef __cplusplus -/* Size of a node in a leaf page with a given key and data. - * This is node header plus key plus data size. */ -MDBX_NOTHROW_CONST_FUNCTION static __always_inline size_t -node_size_len(const size_t key_len, const size_t value_len) { - return NODESIZE + EVEN(key_len + value_len); -} -MDBX_NOTHROW_PURE_FUNCTION static __always_inline size_t -node_size(const MDBX_val *key, const MDBX_val *value) { - return node_size_len(key ? key->iov_len : 0, value ? value->iov_len : 0); -} +#ifdef MDBX_HAVE_C11ATOMICS +#define osal_memory_fence(order, write) atomic_thread_fence((write) ? mo_c11_store(order) : mo_c11_load(order)) +#else /* MDBX_HAVE_C11ATOMICS */ +#define osal_memory_fence(order, write) \ + do { \ + osal_compiler_barrier(); \ + if (write && order > (MDBX_CPU_WRITEBACK_INCOHERENT ? mo_Relaxed : mo_AcquireRelease)) \ + osal_memory_barrier(); \ + } while (0) +#endif /* MDBX_HAVE_C11ATOMICS */ -MDBX_NOTHROW_PURE_FUNCTION static __always_inline pgno_t -peek_pgno(const void *const __restrict ptr) { - if (sizeof(pgno_t) == sizeof(uint32_t)) - return (pgno_t)unaligned_peek_u32(1, ptr); - else if (sizeof(pgno_t) == sizeof(uint64_t)) - return (pgno_t)unaligned_peek_u64(1, ptr); - else { - pgno_t pgno; - memcpy(&pgno, ptr, sizeof(pgno)); - return pgno; - } -} +#if defined(MDBX_HAVE_C11ATOMICS) && defined(__LCC__) +#define atomic_store32(p, value, order) \ + ({ \ + const uint32_t value_to_store = (value); \ + atomic_store_explicit(MDBX_c11a_rw(uint32_t, p), value_to_store, mo_c11_store(order)); \ + value_to_store; \ + }) +#define atomic_load32(p, order) atomic_load_explicit(MDBX_c11a_ro(uint32_t, p), mo_c11_load(order)) +#define atomic_store64(p, value, order) \ + ({ \ + const uint64_t value_to_store = (value); \ + atomic_store_explicit(MDBX_c11a_rw(uint64_t, p), value_to_store, mo_c11_store(order)); \ + value_to_store; \ + }) +#define atomic_load64(p, order) atomic_load_explicit(MDBX_c11a_ro(uint64_t, p), mo_c11_load(order)) +#endif /* LCC && MDBX_HAVE_C11ATOMICS */ -static __always_inline void poke_pgno(void *const __restrict ptr, - const pgno_t pgno) { - if (sizeof(pgno) == sizeof(uint32_t)) - unaligned_poke_u32(1, ptr, pgno); - else if (sizeof(pgno) == sizeof(uint64_t)) - unaligned_poke_u64(1, ptr, pgno); - else - memcpy(ptr, &pgno, sizeof(pgno)); +#ifndef atomic_store32 +MDBX_MAYBE_UNUSED static __always_inline uint32_t atomic_store32(mdbx_atomic_uint32_t *p, const uint32_t value, + enum mdbx_memory_order order) { + STATIC_ASSERT(sizeof(mdbx_atomic_uint32_t) == 4); +#ifdef MDBX_HAVE_C11ATOMICS + assert(atomic_is_lock_free(MDBX_c11a_rw(uint32_t, p))); + atomic_store_explicit(MDBX_c11a_rw(uint32_t, p), value, mo_c11_store(order)); +#else /* MDBX_HAVE_C11ATOMICS */ + if (order != mo_Relaxed) + osal_compiler_barrier(); + p->weak = value; + osal_memory_fence(order, true); +#endif /* MDBX_HAVE_C11ATOMICS */ + return value; } +#endif /* atomic_store32 */ -MDBX_NOTHROW_PURE_FUNCTION static __always_inline pgno_t -node_largedata_pgno(const MDBX_node *const __restrict node) { - assert(node_flags(node) & F_BIGDATA); - return peek_pgno(node_data(node)); +#ifndef atomic_load32 +MDBX_MAYBE_UNUSED static __always_inline uint32_t atomic_load32(const volatile mdbx_atomic_uint32_t *p, + enum mdbx_memory_order order) { + STATIC_ASSERT(sizeof(mdbx_atomic_uint32_t) == 4); +#ifdef MDBX_HAVE_C11ATOMICS + assert(atomic_is_lock_free(MDBX_c11a_ro(uint32_t, p))); + return atomic_load_explicit(MDBX_c11a_ro(uint32_t, p), mo_c11_load(order)); +#else /* MDBX_HAVE_C11ATOMICS */ + osal_memory_fence(order, false); + const uint32_t value = p->weak; + if (order != mo_Relaxed) + osal_compiler_barrier(); + return value; +#endif /* MDBX_HAVE_C11ATOMICS */ } +#endif /* atomic_load32 */ /*------------------------------------------------------------------------------ - * Nodes, Keys & Values length limitation factors: - * - * BRANCH_NODE_MAX - * Branch-page must contain at least two nodes, within each a key and a child - * page number. But page can't be split if it contains less that 4 keys, - * i.e. a page should not overflow before adding the fourth key. Therefore, - * at least 3 branch-node should fit in the single branch-page. Further, the - * first node of a branch-page doesn't contain a key, i.e. the first node - * is always require space just for itself. Thus: - * PAGEROOM = pagesize - page_hdr_len; - * BRANCH_NODE_MAX = even_floor( - * (PAGEROOM - sizeof(indx_t) - NODESIZE) / (3 - 1) - sizeof(indx_t)); - * KEYLEN_MAX = BRANCH_NODE_MAX - node_hdr_len; - * - * LEAF_NODE_MAX - * Leaf-node must fit into single leaf-page, where a value could be placed on - * a large/overflow page. However, may require to insert a nearly page-sized - * node between two large nodes are already fill-up a page. In this case the - * page must be split to two if some pair of nodes fits on one page, or - * otherwise the page should be split to the THREE with a single node - * per each of ones. Such 1-into-3 page splitting is costly and complex since - * requires TWO insertion into the parent page, that could lead to split it - * and so on up to the root. Therefore double-splitting is avoided here and - * the maximum node size is half of a leaf page space: - * LEAF_NODE_MAX = even_floor(PAGEROOM / 2 - sizeof(indx_t)); - * DATALEN_NO_OVERFLOW = LEAF_NODE_MAX - NODESIZE - KEYLEN_MAX; - * - * - SubDatabase-node must fit into one leaf-page: - * SUBDB_NAME_MAX = LEAF_NODE_MAX - node_hdr_len - sizeof(MDBX_db); - * - * - Dupsort values itself are a keys in a dupsort-subdb and couldn't be longer - * than the KEYLEN_MAX. But dupsort node must not great than LEAF_NODE_MAX, - * since dupsort value couldn't be placed on a large/overflow page: - * DUPSORT_DATALEN_MAX = min(KEYLEN_MAX, - * max(DATALEN_NO_OVERFLOW, sizeof(MDBX_db)); - */ - -#define PAGEROOM(pagesize) ((pagesize) - PAGEHDRSZ) -#define EVEN_FLOOR(n) ((n) & ~(size_t)1) -#define BRANCH_NODE_MAX(pagesize) \ - (EVEN_FLOOR((PAGEROOM(pagesize) - sizeof(indx_t) - NODESIZE) / (3 - 1) - \ - sizeof(indx_t))) -#define LEAF_NODE_MAX(pagesize) \ - (EVEN_FLOOR(PAGEROOM(pagesize) / 2) - sizeof(indx_t)) -#define MAX_GC1OVPAGE(pagesize) (PAGEROOM(pagesize) / sizeof(pgno_t) - 1) - -static __inline size_t keysize_max(size_t pagesize, MDBX_db_flags_t flags) { - assert(pagesize >= MIN_PAGESIZE && pagesize <= MAX_PAGESIZE && - is_powerof2(pagesize)); - STATIC_ASSERT(BRANCH_NODE_MAX(MIN_PAGESIZE) - NODESIZE >= 8); - if (flags & MDBX_INTEGERKEY) - return 8 /* sizeof(uint64_t) */; - - const intptr_t max_branch_key = BRANCH_NODE_MAX(pagesize) - NODESIZE; - STATIC_ASSERT(LEAF_NODE_MAX(MIN_PAGESIZE) - NODESIZE - - /* sizeof(uint64) as a key */ 8 > - sizeof(MDBX_db)); - if (flags & - (MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_REVERSEDUP | MDBX_INTEGERDUP)) { - const intptr_t max_dupsort_leaf_key = - LEAF_NODE_MAX(pagesize) - NODESIZE - sizeof(MDBX_db); - return (max_branch_key < max_dupsort_leaf_key) ? max_branch_key - : max_dupsort_leaf_key; - } - return max_branch_key; -} - -static __inline size_t valsize_max(size_t pagesize, MDBX_db_flags_t flags) { - assert(pagesize >= MIN_PAGESIZE && pagesize <= MAX_PAGESIZE && - is_powerof2(pagesize)); - - if (flags & MDBX_INTEGERDUP) - return 8 /* sizeof(uint64_t) */; - - if (flags & (MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_REVERSEDUP)) - return keysize_max(pagesize, 0); + * safe read/write volatile 64-bit fields on 32-bit architectures. */ - const unsigned page_ln2 = log2n_powerof2(pagesize); - const size_t hard = 0x7FF00000ul; - const size_t hard_pages = hard >> page_ln2; - STATIC_ASSERT(MDBX_PGL_LIMIT <= MAX_PAGENO); - const size_t pages_limit = MDBX_PGL_LIMIT / 4; - const size_t limit = - (hard_pages < pages_limit) ? hard : (pages_limit << page_ln2); - return (limit < MAX_MAPSIZE / 2) ? limit : MAX_MAPSIZE / 2; -} +/* LY: for testing non-atomic 64-bit txnid on 32-bit arches. + * #define xMDBX_TXNID_STEP (UINT32_MAX / 3) */ +#ifndef xMDBX_TXNID_STEP +#if MDBX_64BIT_CAS +#define xMDBX_TXNID_STEP 1u +#else +#define xMDBX_TXNID_STEP 2u +#endif +#endif /* xMDBX_TXNID_STEP */ -__cold int mdbx_env_get_maxkeysize(const MDBX_env *env) { - return mdbx_env_get_maxkeysize_ex(env, MDBX_DUPSORT); +#ifndef atomic_store64 +MDBX_MAYBE_UNUSED static __always_inline uint64_t atomic_store64(mdbx_atomic_uint64_t *p, const uint64_t value, + enum mdbx_memory_order order) { + STATIC_ASSERT(sizeof(mdbx_atomic_uint64_t) == 8); +#if MDBX_64BIT_ATOMIC +#if __GNUC_PREREQ(11, 0) + STATIC_ASSERT(__alignof__(mdbx_atomic_uint64_t) >= sizeof(uint64_t)); +#endif /* GNU C >= 11 */ +#ifdef MDBX_HAVE_C11ATOMICS + assert(atomic_is_lock_free(MDBX_c11a_rw(uint64_t, p))); + atomic_store_explicit(MDBX_c11a_rw(uint64_t, p), value, mo_c11_store(order)); +#else /* MDBX_HAVE_C11ATOMICS */ + if (order != mo_Relaxed) + osal_compiler_barrier(); + p->weak = value; + osal_memory_fence(order, true); +#endif /* MDBX_HAVE_C11ATOMICS */ +#else /* !MDBX_64BIT_ATOMIC */ + osal_compiler_barrier(); + atomic_store32(&p->low, (uint32_t)value, mo_Relaxed); + jitter4testing(true); + atomic_store32(&p->high, (uint32_t)(value >> 32), order); + jitter4testing(true); +#endif /* !MDBX_64BIT_ATOMIC */ + return value; } +#endif /* atomic_store64 */ -__cold int mdbx_env_get_maxkeysize_ex(const MDBX_env *env, - MDBX_db_flags_t flags) { - if (unlikely(!env || env->me_signature.weak != MDBX_ME_SIGNATURE)) - return -1; - - return (int)mdbx_limits_keysize_max((intptr_t)env->me_psize, flags); +#ifndef atomic_load64 +MDBX_MAYBE_UNUSED static +#if MDBX_64BIT_ATOMIC + __always_inline +#endif /* MDBX_64BIT_ATOMIC */ + uint64_t + atomic_load64(const volatile mdbx_atomic_uint64_t *p, enum mdbx_memory_order order) { + STATIC_ASSERT(sizeof(mdbx_atomic_uint64_t) == 8); +#if MDBX_64BIT_ATOMIC +#ifdef MDBX_HAVE_C11ATOMICS + assert(atomic_is_lock_free(MDBX_c11a_ro(uint64_t, p))); + return atomic_load_explicit(MDBX_c11a_ro(uint64_t, p), mo_c11_load(order)); +#else /* MDBX_HAVE_C11ATOMICS */ + osal_memory_fence(order, false); + const uint64_t value = p->weak; + if (order != mo_Relaxed) + osal_compiler_barrier(); + return value; +#endif /* MDBX_HAVE_C11ATOMICS */ +#else /* !MDBX_64BIT_ATOMIC */ + osal_compiler_barrier(); + uint64_t value = (uint64_t)atomic_load32(&p->high, order) << 32; + jitter4testing(true); + value |= atomic_load32(&p->low, (order == mo_Relaxed) ? mo_Relaxed : mo_AcquireRelease); + jitter4testing(true); + for (;;) { + osal_compiler_barrier(); + uint64_t again = (uint64_t)atomic_load32(&p->high, order) << 32; + jitter4testing(true); + again |= atomic_load32(&p->low, (order == mo_Relaxed) ? mo_Relaxed : mo_AcquireRelease); + jitter4testing(true); + if (likely(value == again)) + return value; + value = again; + } +#endif /* !MDBX_64BIT_ATOMIC */ } +#endif /* atomic_load64 */ -size_t mdbx_default_pagesize(void) { - size_t pagesize = osal_syspagesize(); - ENSURE(nullptr, is_powerof2(pagesize)); - pagesize = (pagesize >= MIN_PAGESIZE) ? pagesize : MIN_PAGESIZE; - pagesize = (pagesize <= MAX_PAGESIZE) ? pagesize : MAX_PAGESIZE; - return pagesize; +MDBX_MAYBE_UNUSED static __always_inline void atomic_yield(void) { +#if defined(_WIN32) || defined(_WIN64) + YieldProcessor(); +#elif defined(__ia32__) || defined(__e2k__) + __builtin_ia32_pause(); +#elif defined(__ia64__) +#if defined(__HP_cc__) || defined(__HP_aCC__) + _Asm_hint(_HINT_PAUSE); +#else + __asm__ __volatile__("hint @pause"); +#endif +#elif defined(__aarch64__) || (defined(__ARM_ARCH) && __ARM_ARCH > 6) || defined(__ARM_ARCH_6K__) +#ifdef __CC_ARM + __yield(); +#else + __asm__ __volatile__("yield"); +#endif +#elif (defined(__mips64) || defined(__mips64__)) && defined(__mips_isa_rev) && __mips_isa_rev >= 2 + __asm__ __volatile__("pause"); +#elif defined(__mips) || defined(__mips__) || defined(__mips64) || defined(__mips64__) || defined(_M_MRX000) || \ + defined(_MIPS_) || defined(__MWERKS__) || defined(__sgi) + __asm__ __volatile__(".word 0x00000140"); +#elif defined(__linux__) || defined(__gnu_linux__) || defined(_UNIX03_SOURCE) + sched_yield(); +#elif (defined(_GNU_SOURCE) && __GLIBC_PREREQ(2, 1)) || defined(_OPEN_THREADS) + pthread_yield(); +#endif } -__cold intptr_t mdbx_limits_keysize_max(intptr_t pagesize, - MDBX_db_flags_t flags) { - if (pagesize < 1) - pagesize = (intptr_t)mdbx_default_pagesize(); - if (unlikely(pagesize < (intptr_t)MIN_PAGESIZE || - pagesize > (intptr_t)MAX_PAGESIZE || - !is_powerof2((size_t)pagesize))) - return -1; - - return keysize_max(pagesize, flags); +#if MDBX_64BIT_CAS +MDBX_MAYBE_UNUSED static __always_inline bool atomic_cas64(mdbx_atomic_uint64_t *p, uint64_t c, uint64_t v) { +#ifdef MDBX_HAVE_C11ATOMICS + STATIC_ASSERT(sizeof(long long) >= sizeof(uint64_t)); + assert(atomic_is_lock_free(MDBX_c11a_rw(uint64_t, p))); + return atomic_compare_exchange_strong(MDBX_c11a_rw(uint64_t, p), &c, v); +#elif defined(__GNUC__) || defined(__clang__) + return __sync_bool_compare_and_swap(&p->weak, c, v); +#elif defined(_MSC_VER) + return c == (uint64_t)_InterlockedCompareExchange64((volatile __int64 *)&p->weak, v, c); +#elif defined(__APPLE__) + return OSAtomicCompareAndSwap64Barrier(c, v, &p->weak); +#else +#error FIXME: Unsupported compiler +#endif } +#endif /* MDBX_64BIT_CAS */ -__cold int mdbx_env_get_maxvalsize_ex(const MDBX_env *env, - MDBX_db_flags_t flags) { - if (unlikely(!env || env->me_signature.weak != MDBX_ME_SIGNATURE)) - return -1; - - return (int)mdbx_limits_valsize_max((intptr_t)env->me_psize, flags); +MDBX_MAYBE_UNUSED static __always_inline bool atomic_cas32(mdbx_atomic_uint32_t *p, uint32_t c, uint32_t v) { +#ifdef MDBX_HAVE_C11ATOMICS + STATIC_ASSERT(sizeof(int) >= sizeof(uint32_t)); + assert(atomic_is_lock_free(MDBX_c11a_rw(uint32_t, p))); + return atomic_compare_exchange_strong(MDBX_c11a_rw(uint32_t, p), &c, v); +#elif defined(__GNUC__) || defined(__clang__) + return __sync_bool_compare_and_swap(&p->weak, c, v); +#elif defined(_MSC_VER) + STATIC_ASSERT(sizeof(volatile long) == sizeof(volatile uint32_t)); + return c == (uint32_t)_InterlockedCompareExchange((volatile long *)&p->weak, v, c); +#elif defined(__APPLE__) + return OSAtomicCompareAndSwap32Barrier(c, v, &p->weak); +#else +#error FIXME: Unsupported compiler +#endif } -__cold intptr_t mdbx_limits_valsize_max(intptr_t pagesize, - MDBX_db_flags_t flags) { - if (pagesize < 1) - pagesize = (intptr_t)mdbx_default_pagesize(); - if (unlikely(pagesize < (intptr_t)MIN_PAGESIZE || - pagesize > (intptr_t)MAX_PAGESIZE || - !is_powerof2((size_t)pagesize))) - return -1; - - return valsize_max(pagesize, flags); +MDBX_MAYBE_UNUSED static __always_inline uint32_t atomic_add32(mdbx_atomic_uint32_t *p, uint32_t v) { +#ifdef MDBX_HAVE_C11ATOMICS + STATIC_ASSERT(sizeof(int) >= sizeof(uint32_t)); + assert(atomic_is_lock_free(MDBX_c11a_rw(uint32_t, p))); + return atomic_fetch_add(MDBX_c11a_rw(uint32_t, p), v); +#elif defined(__GNUC__) || defined(__clang__) + return __sync_fetch_and_add(&p->weak, v); +#elif defined(_MSC_VER) + STATIC_ASSERT(sizeof(volatile long) == sizeof(volatile uint32_t)); + return (uint32_t)_InterlockedExchangeAdd((volatile long *)&p->weak, v); +#elif defined(__APPLE__) + return OSAtomicAdd32Barrier(v, &p->weak); +#else +#error FIXME: Unsupported compiler +#endif } -__cold intptr_t mdbx_limits_pairsize4page_max(intptr_t pagesize, - MDBX_db_flags_t flags) { - if (pagesize < 1) - pagesize = (intptr_t)mdbx_default_pagesize(); - if (unlikely(pagesize < (intptr_t)MIN_PAGESIZE || - pagesize > (intptr_t)MAX_PAGESIZE || - !is_powerof2((size_t)pagesize))) - return -1; - - if (flags & - (MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERDUP | MDBX_REVERSEDUP)) - return BRANCH_NODE_MAX(pagesize) - NODESIZE; +#define atomic_sub32(p, v) atomic_add32(p, 0 - (v)) - return LEAF_NODE_MAX(pagesize) - NODESIZE; +MDBX_MAYBE_UNUSED static __always_inline uint64_t safe64_txnid_next(uint64_t txnid) { + txnid += xMDBX_TXNID_STEP; +#if !MDBX_64BIT_CAS + /* avoid overflow of low-part in safe64_reset() */ + txnid += (UINT32_MAX == (uint32_t)txnid); +#endif + return txnid; } -__cold int mdbx_env_get_pairsize4page_max(const MDBX_env *env, - MDBX_db_flags_t flags) { - if (unlikely(!env || env->me_signature.weak != MDBX_ME_SIGNATURE)) - return -1; - - return (int)mdbx_limits_pairsize4page_max((intptr_t)env->me_psize, flags); +/* Atomically make target value >= SAFE64_INVALID_THRESHOLD */ +MDBX_MAYBE_UNUSED static __always_inline void safe64_reset(mdbx_atomic_uint64_t *p, bool single_writer) { + if (single_writer) { +#if MDBX_64BIT_ATOMIC && MDBX_WORDBITS >= 64 + atomic_store64(p, UINT64_MAX, mo_AcquireRelease); +#else + atomic_store32(&p->high, UINT32_MAX, mo_AcquireRelease); +#endif /* MDBX_64BIT_ATOMIC && MDBX_WORDBITS >= 64 */ + } else { +#if MDBX_64BIT_CAS && MDBX_64BIT_ATOMIC + /* atomically make value >= SAFE64_INVALID_THRESHOLD by 64-bit operation */ + atomic_store64(p, UINT64_MAX, mo_AcquireRelease); +#elif MDBX_64BIT_CAS + /* atomically make value >= SAFE64_INVALID_THRESHOLD by 32-bit operation */ + atomic_store32(&p->high, UINT32_MAX, mo_AcquireRelease); +#else + /* it is safe to increment low-part to avoid ABA, since xMDBX_TXNID_STEP > 1 + * and overflow was preserved in safe64_txnid_next() */ + STATIC_ASSERT(xMDBX_TXNID_STEP > 1); + atomic_add32(&p->low, 1) /* avoid ABA in safe64_reset_compare() */; + atomic_store32(&p->high, UINT32_MAX, mo_AcquireRelease); + atomic_add32(&p->low, 1) /* avoid ABA in safe64_reset_compare() */; +#endif /* MDBX_64BIT_CAS && MDBX_64BIT_ATOMIC */ + } + assert(p->weak >= SAFE64_INVALID_THRESHOLD); + jitter4testing(true); } -__cold intptr_t mdbx_limits_valsize4page_max(intptr_t pagesize, - MDBX_db_flags_t flags) { - if (pagesize < 1) - pagesize = (intptr_t)mdbx_default_pagesize(); - if (unlikely(pagesize < (intptr_t)MIN_PAGESIZE || - pagesize > (intptr_t)MAX_PAGESIZE || - !is_powerof2((size_t)pagesize))) - return -1; - - if (flags & - (MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERDUP | MDBX_REVERSEDUP)) - return valsize_max(pagesize, flags); - - return PAGEROOM(pagesize); +MDBX_MAYBE_UNUSED static __always_inline bool safe64_reset_compare(mdbx_atomic_uint64_t *p, uint64_t compare) { + /* LY: This function is used to reset `txnid` from hsr-handler in case + * the asynchronously cancellation of read transaction. Therefore, + * there may be a collision between the cleanup performed here and + * asynchronous termination and restarting of the read transaction + * in another process/thread. In general we MUST NOT reset the `txnid` + * if a new transaction was started (i.e. if `txnid` was changed). */ +#if MDBX_64BIT_CAS + bool rc = atomic_cas64(p, compare, UINT64_MAX); +#else + /* LY: There is no gold ratio here since shared mutex is too costly, + * in such way we must acquire/release it for every update of txnid, + * i.e. twice for each read transaction). */ + bool rc = false; + if (likely(atomic_load32(&p->low, mo_AcquireRelease) == (uint32_t)compare && + atomic_cas32(&p->high, (uint32_t)(compare >> 32), UINT32_MAX))) { + if (unlikely(atomic_load32(&p->low, mo_AcquireRelease) != (uint32_t)compare)) + atomic_cas32(&p->high, UINT32_MAX, (uint32_t)(compare >> 32)); + else + rc = true; + } +#endif /* MDBX_64BIT_CAS */ + jitter4testing(true); + return rc; } -__cold int mdbx_env_get_valsize4page_max(const MDBX_env *env, - MDBX_db_flags_t flags) { - if (unlikely(!env || env->me_signature.weak != MDBX_ME_SIGNATURE)) - return -1; - - return (int)mdbx_limits_valsize4page_max((intptr_t)env->me_psize, flags); +MDBX_MAYBE_UNUSED static __always_inline void safe64_write(mdbx_atomic_uint64_t *p, const uint64_t v) { + assert(p->weak >= SAFE64_INVALID_THRESHOLD); +#if MDBX_64BIT_ATOMIC && MDBX_64BIT_CAS + atomic_store64(p, v, mo_AcquireRelease); +#else /* MDBX_64BIT_ATOMIC */ + osal_compiler_barrier(); + /* update low-part but still value >= SAFE64_INVALID_THRESHOLD */ + atomic_store32(&p->low, (uint32_t)v, mo_Relaxed); + assert(p->weak >= SAFE64_INVALID_THRESHOLD); + jitter4testing(true); + /* update high-part from SAFE64_INVALID_THRESHOLD to actual value */ + atomic_store32(&p->high, (uint32_t)(v >> 32), mo_AcquireRelease); +#endif /* MDBX_64BIT_ATOMIC */ + assert(p->weak == v); + jitter4testing(true); } -/* Calculate the size of a leaf node. - * - * The size depends on the environment's page size; if a data item - * is too large it will be put onto an large/overflow page and the node - * size will only include the key and not the data. Sizes are always - * rounded up to an even number of bytes, to guarantee 2-byte alignment - * of the MDBX_node headers. */ -MDBX_NOTHROW_PURE_FUNCTION static __always_inline size_t -leaf_size(const MDBX_env *env, const MDBX_val *key, const MDBX_val *data) { - size_t node_bytes = node_size(key, data); - if (node_bytes > env->me_leaf_nodemax) { - /* put on large/overflow page */ - node_bytes = node_size_len(key->iov_len, 0) + sizeof(pgno_t); - } - - return node_bytes + sizeof(indx_t); +MDBX_MAYBE_UNUSED static __always_inline uint64_t safe64_read(const mdbx_atomic_uint64_t *p) { + jitter4testing(true); + uint64_t v; + do + v = atomic_load64(p, mo_AcquireRelease); + while (!MDBX_64BIT_ATOMIC && unlikely(v != p->weak)); + return v; } -/* Calculate the size of a branch node. - * - * The size should depend on the environment's page size but since - * we currently don't support spilling large keys onto large/overflow - * pages, it's simply the size of the MDBX_node header plus the - * size of the key. Sizes are always rounded up to an even number - * of bytes, to guarantee 2-byte alignment of the MDBX_node headers. - * - * [in] env The environment handle. - * [in] key The key for the node. - * - * Returns The number of bytes needed to store the node. */ -MDBX_NOTHROW_PURE_FUNCTION static __always_inline size_t -branch_size(const MDBX_env *env, const MDBX_val *key) { - /* Size of a node in a branch page with a given key. - * This is just the node header plus the key, there is no data. */ - size_t node_bytes = node_size(key, nullptr); - if (unlikely(node_bytes > env->me_branch_nodemax)) { - /* put on large/overflow page */ - /* not implemented */ - mdbx_panic("node_size(key) %zu > %u branch_nodemax", node_bytes, - env->me_branch_nodemax); - node_bytes = node_size(key, nullptr) + sizeof(pgno_t); - } - - return node_bytes + sizeof(indx_t); +#if 0 /* unused for now */ +MDBX_MAYBE_UNUSED static __always_inline bool safe64_is_valid(uint64_t v) { +#if MDBX_WORDBITS >= 64 + return v < SAFE64_INVALID_THRESHOLD; +#else + return (v >> 32) != UINT32_MAX; +#endif /* MDBX_WORDBITS */ } -MDBX_NOTHROW_CONST_FUNCTION static __always_inline uint16_t -flags_db2sub(uint16_t db_flags) { - uint16_t sub_flags = db_flags & MDBX_DUPFIXED; - - /* MDBX_INTEGERDUP => MDBX_INTEGERKEY */ -#define SHIFT_INTEGERDUP_TO_INTEGERKEY 2 - STATIC_ASSERT((MDBX_INTEGERDUP >> SHIFT_INTEGERDUP_TO_INTEGERKEY) == - MDBX_INTEGERKEY); - sub_flags |= (db_flags & MDBX_INTEGERDUP) >> SHIFT_INTEGERDUP_TO_INTEGERKEY; - - /* MDBX_REVERSEDUP => MDBX_REVERSEKEY */ -#define SHIFT_REVERSEDUP_TO_REVERSEKEY 5 - STATIC_ASSERT((MDBX_REVERSEDUP >> SHIFT_REVERSEDUP_TO_REVERSEKEY) == - MDBX_REVERSEKEY); - sub_flags |= (db_flags & MDBX_REVERSEDUP) >> SHIFT_REVERSEDUP_TO_REVERSEKEY; - - return sub_flags; +MDBX_MAYBE_UNUSED static __always_inline bool + safe64_is_valid_ptr(const mdbx_atomic_uint64_t *p) { +#if MDBX_64BIT_ATOMIC + return atomic_load64(p, mo_AcquireRelease) < SAFE64_INVALID_THRESHOLD; +#else + return atomic_load32(&p->high, mo_AcquireRelease) != UINT32_MAX; +#endif /* MDBX_64BIT_ATOMIC */ } +#endif /* unused for now */ -/*----------------------------------------------------------------------------*/ - -MDBX_NOTHROW_PURE_FUNCTION static __always_inline size_t -pgno2bytes(const MDBX_env *env, size_t pgno) { - eASSERT(env, (1u << env->me_psize2log) == env->me_psize); - return ((size_t)pgno) << env->me_psize2log; +/* non-atomic write with safety for reading a half-updated value */ +MDBX_MAYBE_UNUSED static __always_inline void safe64_update(mdbx_atomic_uint64_t *p, const uint64_t v) { +#if MDBX_64BIT_ATOMIC + atomic_store64(p, v, mo_Relaxed); +#else + safe64_reset(p, true); + safe64_write(p, v); +#endif /* MDBX_64BIT_ATOMIC */ } -MDBX_NOTHROW_PURE_FUNCTION static __always_inline MDBX_page * -pgno2page(const MDBX_env *env, size_t pgno) { - return ptr_disp(env->me_map, pgno2bytes(env, pgno)); +/* non-atomic increment with safety for reading a half-updated value */ +MDBX_MAYBE_UNUSED static +#if MDBX_64BIT_ATOMIC + __always_inline +#endif /* MDBX_64BIT_ATOMIC */ + void + safe64_inc(mdbx_atomic_uint64_t *p, const uint64_t v) { + assert(v > 0); + safe64_update(p, safe64_read(p) + v); } -MDBX_NOTHROW_PURE_FUNCTION static __always_inline pgno_t -bytes2pgno(const MDBX_env *env, size_t bytes) { - eASSERT(env, (env->me_psize >> env->me_psize2log) == 1); - return (pgno_t)(bytes >> env->me_psize2log); -} +#endif /* !__cplusplus */ -MDBX_NOTHROW_PURE_FUNCTION static size_t -pgno_align2os_bytes(const MDBX_env *env, size_t pgno) { - return ceil_powerof2(pgno2bytes(env, pgno), env->me_os_psize); -} +/* Internal prototypes */ -MDBX_NOTHROW_PURE_FUNCTION static pgno_t pgno_align2os_pgno(const MDBX_env *env, - size_t pgno) { - return bytes2pgno(env, pgno_align2os_bytes(env, pgno)); +/* audit.c */ +MDBX_INTERNAL int audit_ex(MDBX_txn *txn, size_t retired_stored, bool dont_filter_gc); + +/* mvcc-readers.c */ +MDBX_INTERNAL bsr_t mvcc_bind_slot(MDBX_env *env); +MDBX_MAYBE_UNUSED MDBX_INTERNAL pgno_t mvcc_largest_this(MDBX_env *env, pgno_t largest); +MDBX_INTERNAL txnid_t mvcc_shapshot_oldest(MDBX_env *const env, const txnid_t steady); +MDBX_INTERNAL pgno_t mvcc_snapshot_largest(const MDBX_env *env, pgno_t last_used_page); +MDBX_INTERNAL txnid_t mvcc_kick_laggards(MDBX_env *env, const txnid_t straggler); +MDBX_INTERNAL int mvcc_cleanup_dead(MDBX_env *env, int rlocked, int *dead); +MDBX_INTERNAL txnid_t mvcc_kick_laggards(MDBX_env *env, const txnid_t laggard); + +/* dxb.c */ +MDBX_INTERNAL int dxb_setup(MDBX_env *env, const int lck_rc, const mdbx_mode_t mode_bits); +MDBX_INTERNAL int __must_check_result dxb_read_header(MDBX_env *env, meta_t *meta, const int lck_exclusive, + const mdbx_mode_t mode_bits); +enum resize_mode { implicit_grow, impilict_shrink, explicit_resize }; +MDBX_INTERNAL int __must_check_result dxb_resize(MDBX_env *const env, const pgno_t used_pgno, const pgno_t size_pgno, + pgno_t limit_pgno, const enum resize_mode mode); +MDBX_INTERNAL int dxb_set_readahead(const MDBX_env *env, const pgno_t edge, const bool enable, const bool force_whole); +MDBX_INTERNAL int __must_check_result dxb_sync_locked(MDBX_env *env, unsigned flags, meta_t *const pending, + troika_t *const troika); +#if defined(ENABLE_MEMCHECK) || defined(__SANITIZE_ADDRESS__) +MDBX_INTERNAL void dxb_sanitize_tail(MDBX_env *env, MDBX_txn *txn); +#else +static inline void dxb_sanitize_tail(MDBX_env *env, MDBX_txn *txn) { + (void)env; + (void)txn; } +#endif /* ENABLE_MEMCHECK || __SANITIZE_ADDRESS__ */ -MDBX_NOTHROW_PURE_FUNCTION static size_t -bytes_align2os_bytes(const MDBX_env *env, size_t bytes) { - return ceil_powerof2(ceil_powerof2(bytes, env->me_psize), env->me_os_psize); -} +/* txn.c */ +MDBX_INTERNAL bool txn_refund(MDBX_txn *txn); +MDBX_INTERNAL txnid_t txn_snapshot_oldest(const MDBX_txn *const txn); +MDBX_INTERNAL int txn_abort(MDBX_txn *txn); +MDBX_INTERNAL int txn_renew(MDBX_txn *txn, unsigned flags); +MDBX_INTERNAL int txn_park(MDBX_txn *txn, bool autounpark); +MDBX_INTERNAL int txn_unpark(MDBX_txn *txn); +MDBX_INTERNAL int txn_check_badbits_parked(const MDBX_txn *txn, int bad_bits); +MDBX_INTERNAL void txn_done_cursors(MDBX_txn *txn, const bool merge); -/* Address of first usable data byte in a page, after the header */ -MDBX_NOTHROW_PURE_FUNCTION static __always_inline void * -page_data(const MDBX_page *mp) { - return ptr_disp(mp, PAGEHDRSZ); -} +#define TXN_END_NAMES \ + {"committed", "empty-commit", "abort", "reset", "fail-begin", "fail-beginchild", "ousted", nullptr} +enum { + /* txn_end operation number, for logging */ + TXN_END_COMMITTED, + TXN_END_PURE_COMMIT, + TXN_END_ABORT, + TXN_END_RESET, + TXN_END_FAIL_BEGIN, + TXN_END_FAIL_BEGINCHILD, + TXN_END_OUSTED, + + TXN_END_OPMASK = 0x07 /* mask for txn_end() operation number */, + TXN_END_UPDATE = 0x10 /* update env state (DBIs) */, + TXN_END_FREE = 0x20 /* free txn unless it is env.basal_txn */, + TXN_END_EOTDONE = 0x40 /* txn's cursors already closed */, + TXN_END_SLOT = 0x80 /* release any reader slot if NOSTICKYTHREADS */ +}; +MDBX_INTERNAL int txn_end(MDBX_txn *txn, unsigned mode); +MDBX_INTERNAL int txn_write(MDBX_txn *txn, iov_ctx_t *ctx); +MDBX_INTERNAL void txn_take_gcprof(const MDBX_txn *txn, MDBX_commit_latency *latency); +MDBX_INTERNAL void txn_merge(MDBX_txn *const parent, MDBX_txn *const txn, const size_t parent_retired_len); + +/* env.c */ +MDBX_INTERNAL int env_open(MDBX_env *env, mdbx_mode_t mode); +MDBX_INTERNAL int env_info(const MDBX_env *env, const MDBX_txn *txn, MDBX_envinfo *out, size_t bytes, troika_t *troika); +MDBX_INTERNAL int env_sync(MDBX_env *env, bool force, bool nonblock); +MDBX_INTERNAL int env_close(MDBX_env *env, bool resurrect_after_fork); +MDBX_INTERNAL MDBX_txn *env_owned_wrtxn(const MDBX_env *env); +MDBX_INTERNAL int __must_check_result env_page_auxbuffer(MDBX_env *env); +MDBX_INTERNAL unsigned env_setup_pagesize(MDBX_env *env, const size_t pagesize); + +/* api-opt.c */ +MDBX_INTERNAL void env_options_init(MDBX_env *env); +MDBX_INTERNAL void env_options_adjust_defaults(MDBX_env *env); +MDBX_INTERNAL void env_options_adjust_dp_limit(MDBX_env *env); +MDBX_INTERNAL pgno_t default_dp_limit(const MDBX_env *env); + +/* tree.c */ +MDBX_INTERNAL int tree_drop(MDBX_cursor *mc, const bool may_have_tables); +MDBX_INTERNAL int __must_check_result tree_rebalance(MDBX_cursor *mc); +MDBX_INTERNAL int __must_check_result tree_propagate_key(MDBX_cursor *mc, const MDBX_val *key); +MDBX_INTERNAL void recalculate_merge_thresholds(MDBX_env *env); +MDBX_INTERNAL void recalculate_subpage_thresholds(MDBX_env *env); + +/* table.c */ +MDBX_INTERNAL int __must_check_result tbl_fetch(MDBX_txn *txn, size_t dbi); +MDBX_INTERNAL int __must_check_result tbl_setup(const MDBX_env *env, volatile kvx_t *const kvx, const tree_t *const db); + +/* coherency.c */ +MDBX_INTERNAL bool coherency_check_meta(const MDBX_env *env, const volatile meta_t *meta, bool report); +MDBX_INTERNAL int coherency_fetch_head(MDBX_txn *txn, const meta_ptr_t head, uint64_t *timestamp); +MDBX_INTERNAL int coherency_check_written(const MDBX_env *env, const txnid_t txnid, const volatile meta_t *meta, + const intptr_t pgno, uint64_t *timestamp); +MDBX_INTERNAL int coherency_timeout(uint64_t *timestamp, intptr_t pgno, const MDBX_env *env); + +/* List of txnid */ +typedef txnid_t *txl_t; +typedef const txnid_t *const_txl_t; + +enum txl_rules { + txl_granulate = 32, + txl_initial = txl_granulate - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t), + txl_max = (1u << 26) - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t) +}; -MDBX_NOTHROW_PURE_FUNCTION static __always_inline const MDBX_page * -data_page(const void *data) { - return container_of(data, MDBX_page, mp_ptrs); -} +MDBX_INTERNAL txl_t txl_alloc(void); -MDBX_NOTHROW_PURE_FUNCTION static __always_inline MDBX_meta * -page_meta(MDBX_page *mp) { - return (MDBX_meta *)page_data(mp); -} +MDBX_INTERNAL void txl_free(txl_t txl); -/* Number of nodes on a page */ -MDBX_NOTHROW_PURE_FUNCTION static __always_inline size_t -page_numkeys(const MDBX_page *mp) { - return mp->mp_lower >> 1; -} +MDBX_INTERNAL int __must_check_result txl_append(txl_t __restrict *ptxl, txnid_t id); -/* The amount of space remaining in the page */ -MDBX_NOTHROW_PURE_FUNCTION static __always_inline size_t -page_room(const MDBX_page *mp) { - return mp->mp_upper - mp->mp_lower; -} +MDBX_INTERNAL void txl_sort(txl_t txl); -/* Maximum free space in an empty page */ -MDBX_NOTHROW_PURE_FUNCTION static __always_inline size_t -page_space(const MDBX_env *env) { - STATIC_ASSERT(PAGEHDRSZ % 2 == 0); - return env->me_psize - PAGEHDRSZ; -} +MDBX_INTERNAL bool txl_contain(const txl_t txl, txnid_t id); -MDBX_NOTHROW_PURE_FUNCTION static __always_inline size_t -page_used(const MDBX_env *env, const MDBX_page *mp) { - return page_space(env) - page_room(mp); -} +/*------------------------------------------------------------------------------ + * Unaligned access */ -/* The percentage of space used in the page, in a percents. */ -MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static __inline double -page_fill(const MDBX_env *env, const MDBX_page *mp) { - return page_used(env, mp) * 100.0 / page_space(env); +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline size_t field_alignment(size_t alignment_baseline, + size_t field_offset) { + size_t merge = alignment_baseline | (size_t)field_offset; + return merge & -(int)merge; } -/* The number of large/overflow pages needed to store the given size. */ -MDBX_NOTHROW_PURE_FUNCTION static __always_inline pgno_t -number_of_ovpages(const MDBX_env *env, size_t bytes) { - return bytes2pgno(env, PAGEHDRSZ - 1 + bytes) + 1; -} +/* read-thunk for UB-sanitizer */ +MDBX_NOTHROW_PURE_FUNCTION static inline uint8_t peek_u8(const uint8_t *__restrict ptr) { return *ptr; } -__cold static const char *pagetype_caption(const uint8_t type, - char buf4unknown[16]) { - switch (type) { - case P_BRANCH: - return "branch"; - case P_LEAF: - return "leaf"; - case P_LEAF | P_SUBP: - return "subleaf"; - case P_LEAF | P_LEAF2: - return "dupfixed-leaf"; - case P_LEAF | P_LEAF2 | P_SUBP: - return "dupfixed-subleaf"; - case P_LEAF | P_LEAF2 | P_SUBP | P_LEGACY_DIRTY: - return "dupfixed-subleaf.legacy-dirty"; - case P_OVERFLOW: - return "large"; - default: - snprintf(buf4unknown, 16, "unknown_0x%x", type); - return buf4unknown; +/* write-thunk for UB-sanitizer */ +static inline void poke_u8(uint8_t *__restrict ptr, const uint8_t v) { *ptr = v; } + +static inline void *bcopy_2(void *__restrict dst, const void *__restrict src) { + uint8_t *__restrict d = (uint8_t *)dst; + const uint8_t *__restrict s = (uint8_t *)src; + d[0] = s[0]; + d[1] = s[1]; + return d; +} + +static inline void *bcopy_4(void *const __restrict dst, const void *const __restrict src) { + uint8_t *__restrict d = (uint8_t *)dst; + const uint8_t *__restrict s = (uint8_t *)src; + d[0] = s[0]; + d[1] = s[1]; + d[2] = s[2]; + d[3] = s[3]; + return d; +} + +static inline void *bcopy_8(void *const __restrict dst, const void *const __restrict src) { + uint8_t *__restrict d = (uint8_t *)dst; + const uint8_t *__restrict s = (uint8_t *)src; + d[0] = s[0]; + d[1] = s[1]; + d[2] = s[2]; + d[3] = s[3]; + d[4] = s[4]; + d[5] = s[5]; + d[6] = s[6]; + d[7] = s[7]; + return d; +} + +MDBX_NOTHROW_PURE_FUNCTION static inline uint16_t unaligned_peek_u16(const size_t expected_alignment, + const void *const ptr) { + assert((uintptr_t)ptr % expected_alignment == 0); + if (MDBX_UNALIGNED_OK >= 2 || (expected_alignment % sizeof(uint16_t)) == 0) + return *(const uint16_t *)ptr; + else { +#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64) + return *(const __unaligned uint16_t *)ptr; +#else + uint16_t v; + bcopy_2((uint8_t *)&v, (const uint8_t *)ptr); + return v; +#endif /* _MSC_VER || __unaligned */ } } -__cold static int MDBX_PRINTF_ARGS(2, 3) - bad_page(const MDBX_page *mp, const char *fmt, ...) { - if (LOG_ENABLED(MDBX_LOG_ERROR)) { - static const MDBX_page *prev; - if (prev != mp) { - char buf4unknown[16]; - prev = mp; - debug_log(MDBX_LOG_ERROR, "badpage", 0, - "corrupted %s-page #%u, mod-txnid %" PRIaTXN "\n", - pagetype_caption(PAGETYPE_WHOLE(mp), buf4unknown), mp->mp_pgno, - mp->mp_txnid); - } - - va_list args; - va_start(args, fmt); - debug_log_va(MDBX_LOG_ERROR, "badpage", 0, fmt, args); - va_end(args); +static inline void unaligned_poke_u16(const size_t expected_alignment, void *const __restrict ptr, const uint16_t v) { + assert((uintptr_t)ptr % expected_alignment == 0); + if (MDBX_UNALIGNED_OK >= 2 || (expected_alignment % sizeof(v)) == 0) + *(uint16_t *)ptr = v; + else { +#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64) + *((uint16_t __unaligned *)ptr) = v; +#else + bcopy_2((uint8_t *)ptr, (const uint8_t *)&v); +#endif /* _MSC_VER || __unaligned */ } - return MDBX_CORRUPTED; } -__cold static void MDBX_PRINTF_ARGS(2, 3) - poor_page(const MDBX_page *mp, const char *fmt, ...) { - if (LOG_ENABLED(MDBX_LOG_NOTICE)) { - static const MDBX_page *prev; - if (prev != mp) { - char buf4unknown[16]; - prev = mp; - debug_log(MDBX_LOG_NOTICE, "poorpage", 0, - "suboptimal %s-page #%u, mod-txnid %" PRIaTXN "\n", - pagetype_caption(PAGETYPE_WHOLE(mp), buf4unknown), mp->mp_pgno, - mp->mp_txnid); - } - - va_list args; - va_start(args, fmt); - debug_log_va(MDBX_LOG_NOTICE, "poorpage", 0, fmt, args); - va_end(args); +MDBX_NOTHROW_PURE_FUNCTION static inline uint32_t unaligned_peek_u32(const size_t expected_alignment, + const void *const __restrict ptr) { + assert((uintptr_t)ptr % expected_alignment == 0); + if (MDBX_UNALIGNED_OK >= 4 || (expected_alignment % sizeof(uint32_t)) == 0) + return *(const uint32_t *)ptr; + else if ((expected_alignment % sizeof(uint16_t)) == 0) { + const uint16_t lo = ((const uint16_t *)ptr)[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__]; + const uint16_t hi = ((const uint16_t *)ptr)[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__]; + return lo | (uint32_t)hi << 16; + } else { +#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64) + return *(const __unaligned uint32_t *)ptr; +#else + uint32_t v; + bcopy_4((uint8_t *)&v, (const uint8_t *)ptr); + return v; +#endif /* _MSC_VER || __unaligned */ } } -/* Address of node i in page p */ -MDBX_NOTHROW_PURE_FUNCTION static __always_inline MDBX_node * -page_node(const MDBX_page *mp, size_t i) { - assert(PAGETYPE_COMPAT(mp) == P_LEAF || PAGETYPE_WHOLE(mp) == P_BRANCH); - assert(page_numkeys(mp) > i); - assert(mp->mp_ptrs[i] % 2 == 0); - return ptr_disp(mp, mp->mp_ptrs[i] + PAGEHDRSZ); -} - -/* The address of a key in a LEAF2 page. - * LEAF2 pages are used for MDBX_DUPFIXED sorted-duplicate sub-DBs. - * There are no node headers, keys are stored contiguously. */ -MDBX_NOTHROW_PURE_FUNCTION static __always_inline void * -page_leaf2key(const MDBX_page *mp, size_t i, size_t keysize) { - assert(PAGETYPE_COMPAT(mp) == (P_LEAF | P_LEAF2)); - assert(mp->mp_leaf2_ksize == keysize); - (void)keysize; - return ptr_disp(mp, PAGEHDRSZ + i * mp->mp_leaf2_ksize); -} - -/* Set the node's key into keyptr. */ -static __always_inline void get_key(const MDBX_node *node, MDBX_val *keyptr) { - keyptr->iov_len = node_ks(node); - keyptr->iov_base = node_key(node); -} - -/* Set the node's key into keyptr, if requested. */ -static __always_inline void -get_key_optional(const MDBX_node *node, MDBX_val *keyptr /* __may_null */) { - if (keyptr) - get_key(node, keyptr); -} - -/*------------------------------------------------------------------------------ - * safe read/write volatile 64-bit fields on 32-bit architectures. */ - -#ifndef atomic_store64 -MDBX_MAYBE_UNUSED static __always_inline uint64_t -atomic_store64(MDBX_atomic_uint64_t *p, const uint64_t value, - enum MDBX_memory_order order) { - STATIC_ASSERT(sizeof(MDBX_atomic_uint64_t) == 8); -#if MDBX_64BIT_ATOMIC -#if __GNUC_PREREQ(11, 0) - STATIC_ASSERT(__alignof__(MDBX_atomic_uint64_t) >= sizeof(uint64_t)); -#endif /* GNU C >= 11 */ -#ifdef MDBX_HAVE_C11ATOMICS - assert(atomic_is_lock_free(MDBX_c11a_rw(uint64_t, p))); - atomic_store_explicit(MDBX_c11a_rw(uint64_t, p), value, mo_c11_store(order)); -#else /* MDBX_HAVE_C11ATOMICS */ - if (order != mo_Relaxed) - osal_compiler_barrier(); - p->weak = value; - osal_memory_fence(order, true); -#endif /* MDBX_HAVE_C11ATOMICS */ -#else /* !MDBX_64BIT_ATOMIC */ - osal_compiler_barrier(); - atomic_store32(&p->low, (uint32_t)value, mo_Relaxed); - jitter4testing(true); - atomic_store32(&p->high, (uint32_t)(value >> 32), order); - jitter4testing(true); -#endif /* !MDBX_64BIT_ATOMIC */ - return value; -} -#endif /* atomic_store64 */ - -#ifndef atomic_load64 -MDBX_MAYBE_UNUSED static -#if MDBX_64BIT_ATOMIC - __always_inline -#endif /* MDBX_64BIT_ATOMIC */ - uint64_t - atomic_load64(const volatile MDBX_atomic_uint64_t *p, - enum MDBX_memory_order order) { - STATIC_ASSERT(sizeof(MDBX_atomic_uint64_t) == 8); -#if MDBX_64BIT_ATOMIC -#ifdef MDBX_HAVE_C11ATOMICS - assert(atomic_is_lock_free(MDBX_c11a_ro(uint64_t, p))); - return atomic_load_explicit(MDBX_c11a_ro(uint64_t, p), mo_c11_load(order)); -#else /* MDBX_HAVE_C11ATOMICS */ - osal_memory_fence(order, false); - const uint64_t value = p->weak; - if (order != mo_Relaxed) - osal_compiler_barrier(); - return value; -#endif /* MDBX_HAVE_C11ATOMICS */ -#else /* !MDBX_64BIT_ATOMIC */ - osal_compiler_barrier(); - uint64_t value = (uint64_t)atomic_load32(&p->high, order) << 32; - jitter4testing(true); - value |= atomic_load32(&p->low, (order == mo_Relaxed) ? mo_Relaxed - : mo_AcquireRelease); - jitter4testing(true); - for (;;) { - osal_compiler_barrier(); - uint64_t again = (uint64_t)atomic_load32(&p->high, order) << 32; - jitter4testing(true); - again |= atomic_load32(&p->low, (order == mo_Relaxed) ? mo_Relaxed - : mo_AcquireRelease); - jitter4testing(true); - if (likely(value == again)) - return value; - value = again; +static inline void unaligned_poke_u32(const size_t expected_alignment, void *const __restrict ptr, const uint32_t v) { + assert((uintptr_t)ptr % expected_alignment == 0); + if (MDBX_UNALIGNED_OK >= 4 || (expected_alignment % sizeof(v)) == 0) + *(uint32_t *)ptr = v; + else if ((expected_alignment % sizeof(uint16_t)) == 0) { + ((uint16_t *)ptr)[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__] = (uint16_t)v; + ((uint16_t *)ptr)[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__] = (uint16_t)(v >> 16); + } else { +#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64) + *((uint32_t __unaligned *)ptr) = v; +#else + bcopy_4((uint8_t *)ptr, (const uint8_t *)&v); +#endif /* _MSC_VER || __unaligned */ } -#endif /* !MDBX_64BIT_ATOMIC */ } -#endif /* atomic_load64 */ -static __always_inline void atomic_yield(void) { -#if defined(_WIN32) || defined(_WIN64) - YieldProcessor(); -#elif defined(__ia32__) || defined(__e2k__) - __builtin_ia32_pause(); -#elif defined(__ia64__) -#if defined(__HP_cc__) || defined(__HP_aCC__) - _Asm_hint(_HINT_PAUSE); -#else - __asm__ __volatile__("hint @pause"); -#endif -#elif defined(__aarch64__) || (defined(__ARM_ARCH) && __ARM_ARCH > 6) || \ - defined(__ARM_ARCH_6K__) -#ifdef __CC_ARM - __yield(); +MDBX_NOTHROW_PURE_FUNCTION static inline uint64_t unaligned_peek_u64(const size_t expected_alignment, + const void *const __restrict ptr) { + assert((uintptr_t)ptr % expected_alignment == 0); + if (MDBX_UNALIGNED_OK >= 8 || (expected_alignment % sizeof(uint64_t)) == 0) + return *(const uint64_t *)ptr; + else if ((expected_alignment % sizeof(uint32_t)) == 0) { + const uint32_t lo = ((const uint32_t *)ptr)[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__]; + const uint32_t hi = ((const uint32_t *)ptr)[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__]; + return lo | (uint64_t)hi << 32; + } else { +#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64) + return *(const __unaligned uint64_t *)ptr; #else - __asm__ __volatile__("yield"); -#endif -#elif (defined(__mips64) || defined(__mips64__)) && defined(__mips_isa_rev) && \ - __mips_isa_rev >= 2 - __asm__ __volatile__("pause"); -#elif defined(__mips) || defined(__mips__) || defined(__mips64) || \ - defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \ - defined(__MWERKS__) || defined(__sgi) - __asm__ __volatile__(".word 0x00000140"); -#elif defined(__linux__) || defined(__gnu_linux__) || defined(_UNIX03_SOURCE) - sched_yield(); -#elif (defined(_GNU_SOURCE) && __GLIBC_PREREQ(2, 1)) || defined(_OPEN_THREADS) - pthread_yield(); -#endif + uint64_t v; + bcopy_8((uint8_t *)&v, (const uint8_t *)ptr); + return v; +#endif /* _MSC_VER || __unaligned */ + } } -#if MDBX_64BIT_CAS -static __always_inline bool atomic_cas64(MDBX_atomic_uint64_t *p, uint64_t c, - uint64_t v) { -#ifdef MDBX_HAVE_C11ATOMICS - STATIC_ASSERT(sizeof(long long) >= sizeof(uint64_t)); - assert(atomic_is_lock_free(MDBX_c11a_rw(uint64_t, p))); - return atomic_compare_exchange_strong(MDBX_c11a_rw(uint64_t, p), &c, v); -#elif defined(__GNUC__) || defined(__clang__) - return __sync_bool_compare_and_swap(&p->weak, c, v); -#elif defined(_MSC_VER) - return c == (uint64_t)_InterlockedCompareExchange64( - (volatile __int64 *)&p->weak, v, c); -#elif defined(__APPLE__) - return OSAtomicCompareAndSwap64Barrier(c, v, &p->weak); +static inline uint64_t unaligned_peek_u64_volatile(const size_t expected_alignment, + const volatile void *const __restrict ptr) { + assert((uintptr_t)ptr % expected_alignment == 0); + assert(expected_alignment % sizeof(uint32_t) == 0); + if (MDBX_UNALIGNED_OK >= 8 || (expected_alignment % sizeof(uint64_t)) == 0) + return *(const volatile uint64_t *)ptr; + else { +#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64) + return *(const volatile __unaligned uint64_t *)ptr; #else -#error FIXME: Unsupported compiler -#endif + const uint32_t lo = ((const volatile uint32_t *)ptr)[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__]; + const uint32_t hi = ((const volatile uint32_t *)ptr)[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__]; + return lo | (uint64_t)hi << 32; +#endif /* _MSC_VER || __unaligned */ + } } -#endif /* MDBX_64BIT_CAS */ -static __always_inline bool atomic_cas32(MDBX_atomic_uint32_t *p, uint32_t c, - uint32_t v) { -#ifdef MDBX_HAVE_C11ATOMICS - STATIC_ASSERT(sizeof(int) >= sizeof(uint32_t)); - assert(atomic_is_lock_free(MDBX_c11a_rw(uint32_t, p))); - return atomic_compare_exchange_strong(MDBX_c11a_rw(uint32_t, p), &c, v); -#elif defined(__GNUC__) || defined(__clang__) - return __sync_bool_compare_and_swap(&p->weak, c, v); -#elif defined(_MSC_VER) - STATIC_ASSERT(sizeof(volatile long) == sizeof(volatile uint32_t)); - return c == - (uint32_t)_InterlockedCompareExchange((volatile long *)&p->weak, v, c); -#elif defined(__APPLE__) - return OSAtomicCompareAndSwap32Barrier(c, v, &p->weak); +static inline void unaligned_poke_u64(const size_t expected_alignment, void *const __restrict ptr, const uint64_t v) { + assert((uintptr_t)ptr % expected_alignment == 0); + if (MDBX_UNALIGNED_OK >= 8 || (expected_alignment % sizeof(v)) == 0) + *(uint64_t *)ptr = v; + else if ((expected_alignment % sizeof(uint32_t)) == 0) { + ((uint32_t *)ptr)[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__] = (uint32_t)v; + ((uint32_t *)ptr)[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__] = (uint32_t)(v >> 32); + } else { +#if defined(__unaligned) || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_X64) || defined(_M_IA64) + *((uint64_t __unaligned *)ptr) = v; #else -#error FIXME: Unsupported compiler -#endif + bcopy_8((uint8_t *)ptr, (const uint8_t *)&v); +#endif /* _MSC_VER || __unaligned */ + } } -static __always_inline uint32_t atomic_add32(MDBX_atomic_uint32_t *p, - uint32_t v) { -#ifdef MDBX_HAVE_C11ATOMICS - STATIC_ASSERT(sizeof(int) >= sizeof(uint32_t)); - assert(atomic_is_lock_free(MDBX_c11a_rw(uint32_t, p))); - return atomic_fetch_add(MDBX_c11a_rw(uint32_t, p), v); -#elif defined(__GNUC__) || defined(__clang__) - return __sync_fetch_and_add(&p->weak, v); -#elif defined(_MSC_VER) - STATIC_ASSERT(sizeof(volatile long) == sizeof(volatile uint32_t)); - return (uint32_t)_InterlockedExchangeAdd((volatile long *)&p->weak, v); -#elif defined(__APPLE__) - return OSAtomicAdd32Barrier(v, &p->weak); -#else -#error FIXME: Unsupported compiler -#endif -} +#define UNALIGNED_PEEK_8(ptr, struct, field) peek_u8(ptr_disp(ptr, offsetof(struct, field))) +#define UNALIGNED_POKE_8(ptr, struct, field, value) poke_u8(ptr_disp(ptr, offsetof(struct, field)), value) -#define atomic_sub32(p, v) atomic_add32(p, 0 - (v)) +#define UNALIGNED_PEEK_16(ptr, struct, field) unaligned_peek_u16(1, ptr_disp(ptr, offsetof(struct, field))) +#define UNALIGNED_POKE_16(ptr, struct, field, value) \ + unaligned_poke_u16(1, ptr_disp(ptr, offsetof(struct, field)), value) -static __always_inline uint64_t safe64_txnid_next(uint64_t txnid) { - txnid += xMDBX_TXNID_STEP; -#if !MDBX_64BIT_CAS - /* avoid overflow of low-part in safe64_reset() */ - txnid += (UINT32_MAX == (uint32_t)txnid); -#endif - return txnid; -} +#define UNALIGNED_PEEK_32(ptr, struct, field) unaligned_peek_u32(1, ptr_disp(ptr, offsetof(struct, field))) +#define UNALIGNED_POKE_32(ptr, struct, field, value) \ + unaligned_poke_u32(1, ptr_disp(ptr, offsetof(struct, field)), value) -/* Atomically make target value >= SAFE64_INVALID_THRESHOLD */ -static __always_inline void safe64_reset(MDBX_atomic_uint64_t *p, - bool single_writer) { - if (single_writer) { -#if MDBX_64BIT_ATOMIC && MDBX_WORDBITS >= 64 - atomic_store64(p, UINT64_MAX, mo_AcquireRelease); -#else - atomic_store32(&p->high, UINT32_MAX, mo_AcquireRelease); -#endif /* MDBX_64BIT_ATOMIC && MDBX_WORDBITS >= 64 */ - } else { -#if MDBX_64BIT_CAS && MDBX_64BIT_ATOMIC - /* atomically make value >= SAFE64_INVALID_THRESHOLD by 64-bit operation */ - atomic_store64(p, UINT64_MAX, mo_AcquireRelease); -#elif MDBX_64BIT_CAS - /* atomically make value >= SAFE64_INVALID_THRESHOLD by 32-bit operation */ - atomic_store32(&p->high, UINT32_MAX, mo_AcquireRelease); -#else - /* it is safe to increment low-part to avoid ABA, since xMDBX_TXNID_STEP > 1 - * and overflow was preserved in safe64_txnid_next() */ - STATIC_ASSERT(xMDBX_TXNID_STEP > 1); - atomic_add32(&p->low, 1) /* avoid ABA in safe64_reset_compare() */; - atomic_store32(&p->high, UINT32_MAX, mo_AcquireRelease); - atomic_add32(&p->low, 1) /* avoid ABA in safe64_reset_compare() */; -#endif /* MDBX_64BIT_CAS && MDBX_64BIT_ATOMIC */ +#define UNALIGNED_PEEK_64(ptr, struct, field) unaligned_peek_u64(1, ptr_disp(ptr, offsetof(struct, field))) +#define UNALIGNED_POKE_64(ptr, struct, field, value) \ + unaligned_poke_u64(1, ptr_disp(ptr, offsetof(struct, field)), value) + +MDBX_NOTHROW_PURE_FUNCTION static inline pgno_t peek_pgno(const void *const __restrict ptr) { + if (sizeof(pgno_t) == sizeof(uint32_t)) + return (pgno_t)unaligned_peek_u32(1, ptr); + else if (sizeof(pgno_t) == sizeof(uint64_t)) + return (pgno_t)unaligned_peek_u64(1, ptr); + else { + pgno_t pgno; + memcpy(&pgno, ptr, sizeof(pgno)); + return pgno; } - assert(p->weak >= SAFE64_INVALID_THRESHOLD); - jitter4testing(true); } -static __always_inline bool safe64_reset_compare(MDBX_atomic_uint64_t *p, - txnid_t compare) { - /* LY: This function is used to reset `mr_txnid` from hsr-handler in case - * the asynchronously cancellation of read transaction. Therefore, - * there may be a collision between the cleanup performed here and - * asynchronous termination and restarting of the read transaction - * in another process/thread. In general we MUST NOT reset the `mr_txnid` - * if a new transaction was started (i.e. if `mr_txnid` was changed). */ -#if MDBX_64BIT_CAS - bool rc = atomic_cas64(p, compare, UINT64_MAX); -#else - /* LY: There is no gold ratio here since shared mutex is too costly, - * in such way we must acquire/release it for every update of mr_txnid, - * i.e. twice for each read transaction). */ - bool rc = false; - if (likely(atomic_load32(&p->low, mo_AcquireRelease) == (uint32_t)compare && - atomic_cas32(&p->high, (uint32_t)(compare >> 32), UINT32_MAX))) { - if (unlikely(atomic_load32(&p->low, mo_AcquireRelease) != - (uint32_t)compare)) - atomic_cas32(&p->high, UINT32_MAX, (uint32_t)(compare >> 32)); - else - rc = true; - } -#endif /* MDBX_64BIT_CAS */ - jitter4testing(true); - return rc; +static inline void poke_pgno(void *const __restrict ptr, const pgno_t pgno) { + if (sizeof(pgno) == sizeof(uint32_t)) + unaligned_poke_u32(1, ptr, pgno); + else if (sizeof(pgno) == sizeof(uint64_t)) + unaligned_poke_u64(1, ptr, pgno); + else + memcpy(ptr, &pgno, sizeof(pgno)); } +#if defined(_WIN32) || defined(_WIN64) -static __always_inline void safe64_write(MDBX_atomic_uint64_t *p, - const uint64_t v) { - assert(p->weak >= SAFE64_INVALID_THRESHOLD); -#if MDBX_64BIT_ATOMIC && MDBX_64BIT_CAS - atomic_store64(p, v, mo_AcquireRelease); -#else /* MDBX_64BIT_ATOMIC */ - osal_compiler_barrier(); - /* update low-part but still value >= SAFE64_INVALID_THRESHOLD */ - atomic_store32(&p->low, (uint32_t)v, mo_Relaxed); - assert(p->weak >= SAFE64_INVALID_THRESHOLD); - jitter4testing(true); - /* update high-part from SAFE64_INVALID_THRESHOLD to actual value */ - atomic_store32(&p->high, (uint32_t)(v >> 32), mo_AcquireRelease); -#endif /* MDBX_64BIT_ATOMIC */ - assert(p->weak == v); - jitter4testing(true); -} +typedef union osal_srwlock { + __anonymous_struct_extension__ struct { + long volatile readerCount; + long volatile writerCount; + }; + RTL_SRWLOCK native; +} osal_srwlock_t; -static __always_inline uint64_t safe64_read(const MDBX_atomic_uint64_t *p) { - jitter4testing(true); - uint64_t v; - do - v = atomic_load64(p, mo_AcquireRelease); - while (!MDBX_64BIT_ATOMIC && unlikely(v != p->weak)); - return v; -} +typedef void(WINAPI *osal_srwlock_t_function)(osal_srwlock_t *); -#if 0 /* unused for now */ -MDBX_MAYBE_UNUSED static __always_inline bool safe64_is_valid(uint64_t v) { -#if MDBX_WORDBITS >= 64 - return v < SAFE64_INVALID_THRESHOLD; -#else - return (v >> 32) != UINT32_MAX; -#endif /* MDBX_WORDBITS */ -} +#if _WIN32_WINNT < 0x0600 /* prior to Windows Vista */ +typedef enum _FILE_INFO_BY_HANDLE_CLASS { + FileBasicInfo, + FileStandardInfo, + FileNameInfo, + FileRenameInfo, + FileDispositionInfo, + FileAllocationInfo, + FileEndOfFileInfo, + FileStreamInfo, + FileCompressionInfo, + FileAttributeTagInfo, + FileIdBothDirectoryInfo, + FileIdBothDirectoryRestartInfo, + FileIoPriorityHintInfo, + FileRemoteProtocolInfo, + MaximumFileInfoByHandleClass +} FILE_INFO_BY_HANDLE_CLASS, + *PFILE_INFO_BY_HANDLE_CLASS; -MDBX_MAYBE_UNUSED static __always_inline bool - safe64_is_valid_ptr(const MDBX_atomic_uint64_t *p) { -#if MDBX_64BIT_ATOMIC - return atomic_load64(p, mo_AcquireRelease) < SAFE64_INVALID_THRESHOLD; -#else - return atomic_load32(&p->high, mo_AcquireRelease) != UINT32_MAX; -#endif /* MDBX_64BIT_ATOMIC */ -} -#endif /* unused for now */ +typedef struct _FILE_END_OF_FILE_INFO { + LARGE_INTEGER EndOfFile; +} FILE_END_OF_FILE_INFO, *PFILE_END_OF_FILE_INFO; -/* non-atomic write with safety for reading a half-updated value */ -static __always_inline void safe64_update(MDBX_atomic_uint64_t *p, - const uint64_t v) { -#if MDBX_64BIT_ATOMIC - atomic_store64(p, v, mo_Relaxed); -#else - safe64_reset(p, true); - safe64_write(p, v); -#endif /* MDBX_64BIT_ATOMIC */ -} +#define REMOTE_PROTOCOL_INFO_FLAG_LOOPBACK 0x00000001 +#define REMOTE_PROTOCOL_INFO_FLAG_OFFLINE 0x00000002 -/* non-atomic increment with safety for reading a half-updated value */ -MDBX_MAYBE_UNUSED static -#if MDBX_64BIT_ATOMIC - __always_inline -#endif /* MDBX_64BIT_ATOMIC */ - void - safe64_inc(MDBX_atomic_uint64_t *p, const uint64_t v) { - assert(v > 0); - safe64_update(p, safe64_read(p) + v); -} +typedef struct _FILE_REMOTE_PROTOCOL_INFO { + USHORT StructureVersion; + USHORT StructureSize; + DWORD Protocol; + USHORT ProtocolMajorVersion; + USHORT ProtocolMinorVersion; + USHORT ProtocolRevision; + USHORT Reserved; + DWORD Flags; + struct { + DWORD Reserved[8]; + } GenericReserved; + struct { + DWORD Reserved[16]; + } ProtocolSpecificReserved; +} FILE_REMOTE_PROTOCOL_INFO, *PFILE_REMOTE_PROTOCOL_INFO; -/*----------------------------------------------------------------------------*/ -/* rthc (tls keys and destructors) */ +#endif /* _WIN32_WINNT < 0x0600 (prior to Windows Vista) */ -typedef struct rthc_entry_t { - MDBX_reader *begin; - MDBX_reader *end; - osal_thread_key_t thr_tls_key; -} rthc_entry_t; +typedef BOOL(WINAPI *MDBX_GetFileInformationByHandleEx)(_In_ HANDLE hFile, + _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass, + _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize); -#if MDBX_DEBUG -#define RTHC_INITIAL_LIMIT 1 -#else -#define RTHC_INITIAL_LIMIT 16 -#endif +typedef BOOL(WINAPI *MDBX_GetVolumeInformationByHandleW)( + _In_ HANDLE hFile, _Out_opt_ LPWSTR lpVolumeNameBuffer, _In_ DWORD nVolumeNameSize, + _Out_opt_ LPDWORD lpVolumeSerialNumber, _Out_opt_ LPDWORD lpMaximumComponentLength, + _Out_opt_ LPDWORD lpFileSystemFlags, _Out_opt_ LPWSTR lpFileSystemNameBuffer, _In_ DWORD nFileSystemNameSize); -static bin128_t bootid; +typedef DWORD(WINAPI *MDBX_GetFinalPathNameByHandleW)(_In_ HANDLE hFile, _Out_ LPWSTR lpszFilePath, + _In_ DWORD cchFilePath, _In_ DWORD dwFlags); -#if defined(_WIN32) || defined(_WIN64) -static CRITICAL_SECTION rthc_critical_section; -static CRITICAL_SECTION lcklist_critical_section; -#else +typedef BOOL(WINAPI *MDBX_SetFileInformationByHandle)(_In_ HANDLE hFile, + _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass, + _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize); -static pthread_mutex_t lcklist_mutex = PTHREAD_MUTEX_INITIALIZER; -static pthread_mutex_t rthc_mutex = PTHREAD_MUTEX_INITIALIZER; -static pthread_cond_t rthc_cond = PTHREAD_COND_INITIALIZER; -static osal_thread_key_t rthc_key; -static MDBX_atomic_uint32_t rthc_pending; +typedef NTSTATUS(NTAPI *MDBX_NtFsControlFile)(IN HANDLE FileHandle, IN OUT HANDLE Event, + IN OUT PVOID /* PIO_APC_ROUTINE */ ApcRoutine, IN OUT PVOID ApcContext, + OUT PIO_STATUS_BLOCK IoStatusBlock, IN ULONG FsControlCode, + IN OUT PVOID InputBuffer, IN ULONG InputBufferLength, + OUT OPTIONAL PVOID OutputBuffer, IN ULONG OutputBufferLength); -static __inline uint64_t rthc_signature(const void *addr, uint8_t kind) { - uint64_t salt = osal_thread_self() * UINT64_C(0xA2F0EEC059629A17) ^ - UINT64_C(0x01E07C6FDB596497) * (uintptr_t)(addr); -#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ - return salt << 8 | kind; -#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - return (uint64_t)kind << 56 | salt >> 8; -#else -#error "FIXME: Unsupported byte order" -#endif /* __BYTE_ORDER__ */ -} +typedef uint64_t(WINAPI *MDBX_GetTickCount64)(void); -#define MDBX_THREAD_RTHC_REGISTERED(addr) rthc_signature(addr, 0x0D) -#define MDBX_THREAD_RTHC_COUNTED(addr) rthc_signature(addr, 0xC0) -static __thread uint64_t rthc_thread_state -#if __has_attribute(tls_model) && \ - (defined(__PIC__) || defined(__pic__) || MDBX_BUILD_SHARED_LIBRARY) - __attribute__((tls_model("local-dynamic"))) -#endif - ; +#if !defined(_WIN32_WINNT_WIN8) || _WIN32_WINNT < _WIN32_WINNT_WIN8 +typedef struct _WIN32_MEMORY_RANGE_ENTRY { + PVOID VirtualAddress; + SIZE_T NumberOfBytes; +} WIN32_MEMORY_RANGE_ENTRY, *PWIN32_MEMORY_RANGE_ENTRY; +#endif /* Windows 8.x */ -#if defined(__APPLE__) && defined(__SANITIZE_ADDRESS__) && \ - !defined(MDBX_ATTRIBUTE_NO_SANITIZE_ADDRESS) -/* Avoid ASAN-trap due the target TLS-variable feed by Darwin's tlv_free() */ -#define MDBX_ATTRIBUTE_NO_SANITIZE_ADDRESS \ - __attribute__((__no_sanitize_address__, __noinline__)) -#else -#define MDBX_ATTRIBUTE_NO_SANITIZE_ADDRESS __inline -#endif +typedef BOOL(WINAPI *MDBX_PrefetchVirtualMemory)(HANDLE hProcess, ULONG_PTR NumberOfEntries, + PWIN32_MEMORY_RANGE_ENTRY VirtualAddresses, ULONG Flags); -MDBX_ATTRIBUTE_NO_SANITIZE_ADDRESS static uint64_t rthc_read(const void *rthc) { - return *(volatile uint64_t *)rthc; -} +typedef enum _SECTION_INHERIT { ViewShare = 1, ViewUnmap = 2 } SECTION_INHERIT; -MDBX_ATTRIBUTE_NO_SANITIZE_ADDRESS static uint64_t -rthc_compare_and_clean(const void *rthc, const uint64_t signature) { -#if MDBX_64BIT_CAS - return atomic_cas64((MDBX_atomic_uint64_t *)rthc, signature, 0); -#elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ - return atomic_cas32((MDBX_atomic_uint32_t *)rthc, (uint32_t)signature, 0); -#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - return atomic_cas32((MDBX_atomic_uint32_t *)rthc, (uint32_t)(signature >> 32), - 0); -#else -#error "FIXME: Unsupported byte order" -#endif -} +typedef NTSTATUS(NTAPI *MDBX_NtExtendSection)(IN HANDLE SectionHandle, IN PLARGE_INTEGER NewSectionSize); -static __inline int rthc_atexit(void (*dtor)(void *), void *obj, - void *dso_symbol) { -#ifndef MDBX_HAVE_CXA_THREAD_ATEXIT_IMPL -#if defined(LIBCXXABI_HAS_CXA_THREAD_ATEXIT_IMPL) || \ - defined(HAVE___CXA_THREAD_ATEXIT_IMPL) || __GLIBC_PREREQ(2, 18) || \ - defined(BIONIC) -#define MDBX_HAVE_CXA_THREAD_ATEXIT_IMPL 1 -#else -#define MDBX_HAVE_CXA_THREAD_ATEXIT_IMPL 0 -#endif -#endif /* MDBX_HAVE_CXA_THREAD_ATEXIT_IMPL */ +typedef LSTATUS(WINAPI *MDBX_RegGetValueA)(HKEY hkey, LPCSTR lpSubKey, LPCSTR lpValue, DWORD dwFlags, LPDWORD pdwType, + PVOID pvData, LPDWORD pcbData); -#ifndef MDBX_HAVE_CXA_THREAD_ATEXIT -#if defined(LIBCXXABI_HAS_CXA_THREAD_ATEXIT) || \ - defined(HAVE___CXA_THREAD_ATEXIT) -#define MDBX_HAVE_CXA_THREAD_ATEXIT 1 -#elif !MDBX_HAVE_CXA_THREAD_ATEXIT_IMPL && \ - (defined(__linux__) || defined(__gnu_linux__)) -#define MDBX_HAVE_CXA_THREAD_ATEXIT 1 -#else -#define MDBX_HAVE_CXA_THREAD_ATEXIT 0 -#endif -#endif /* MDBX_HAVE_CXA_THREAD_ATEXIT */ +typedef long(WINAPI *MDBX_CoCreateGuid)(bin128_t *guid); - int rc = MDBX_ENOSYS; -#if MDBX_HAVE_CXA_THREAD_ATEXIT_IMPL && !MDBX_HAVE_CXA_THREAD_ATEXIT -#define __cxa_thread_atexit __cxa_thread_atexit_impl -#endif -#if MDBX_HAVE_CXA_THREAD_ATEXIT || defined(__cxa_thread_atexit) - extern int __cxa_thread_atexit(void (*dtor)(void *), void *obj, - void *dso_symbol) MDBX_WEAK_IMPORT_ATTRIBUTE; - if (&__cxa_thread_atexit) - rc = __cxa_thread_atexit(dtor, obj, dso_symbol); -#elif defined(__APPLE__) || defined(_DARWIN_C_SOURCE) - extern void _tlv_atexit(void (*termfunc)(void *objAddr), void *objAddr) - MDBX_WEAK_IMPORT_ATTRIBUTE; - if (&_tlv_atexit) { - (void)dso_symbol; - _tlv_atexit(dtor, obj); - rc = 0; - } -#else - (void)dtor; - (void)obj; - (void)dso_symbol; -#endif - return rc; -} +NTSYSAPI ULONG RtlRandomEx(PULONG Seed); -__cold static void workaround_glibc_bug21031(void) { - /* Workaround for https://sourceware.org/bugzilla/show_bug.cgi?id=21031 - * - * Due race between pthread_key_delete() and __nptl_deallocate_tsd() - * The destructor(s) of thread-local-storage object(s) may be running - * in another thread(s) and be blocked or not finished yet. - * In such case we get a SEGFAULT after unload this library DSO. - * - * So just by yielding a few timeslices we give a chance - * to such destructor(s) for completion and avoids segfault. */ - sched_yield(); - sched_yield(); - sched_yield(); -} -#endif +typedef BOOL(WINAPI *MDBX_SetFileIoOverlappedRange)(HANDLE FileHandle, PUCHAR OverlappedRangeStart, ULONG Length); + +struct libmdbx_imports { + osal_srwlock_t_function srwl_Init; + osal_srwlock_t_function srwl_AcquireShared; + osal_srwlock_t_function srwl_ReleaseShared; + osal_srwlock_t_function srwl_AcquireExclusive; + osal_srwlock_t_function srwl_ReleaseExclusive; + MDBX_NtExtendSection NtExtendSection; + MDBX_GetFileInformationByHandleEx GetFileInformationByHandleEx; + MDBX_GetVolumeInformationByHandleW GetVolumeInformationByHandleW; + MDBX_GetFinalPathNameByHandleW GetFinalPathNameByHandleW; + MDBX_SetFileInformationByHandle SetFileInformationByHandle; + MDBX_NtFsControlFile NtFsControlFile; + MDBX_PrefetchVirtualMemory PrefetchVirtualMemory; + MDBX_GetTickCount64 GetTickCount64; + MDBX_RegGetValueA RegGetValueA; + MDBX_SetFileIoOverlappedRange SetFileIoOverlappedRange; + MDBX_CoCreateGuid CoCreateGuid; +}; -static unsigned rthc_count, rthc_limit; -static rthc_entry_t *rthc_table; -static rthc_entry_t rthc_table_static[RTHC_INITIAL_LIMIT]; +MDBX_INTERNAL void windows_import(void); +#endif /* Windows */ -static __inline void rthc_lock(void) { -#if defined(_WIN32) || defined(_WIN64) - EnterCriticalSection(&rthc_critical_section); -#else - ENSURE(nullptr, osal_pthread_mutex_lock(&rthc_mutex) == 0); -#endif -} +enum signatures { + env_signature = INT32_C(0x1A899641), + txn_signature = INT32_C(0x13D53A31), + cur_signature_live = INT32_C(0x7E05D5B1), + cur_signature_ready4dispose = INT32_C(0x2817A047), + cur_signature_wait4eot = INT32_C(0x10E297A7) +}; -static __inline void rthc_unlock(void) { -#if defined(_WIN32) || defined(_WIN64) - LeaveCriticalSection(&rthc_critical_section); -#else - ENSURE(nullptr, pthread_mutex_unlock(&rthc_mutex) == 0); -#endif -} +/*----------------------------------------------------------------------------*/ -static __inline int thread_key_create(osal_thread_key_t *key) { - int rc; -#if defined(_WIN32) || defined(_WIN64) - *key = TlsAlloc(); - rc = (*key != TLS_OUT_OF_INDEXES) ? MDBX_SUCCESS : GetLastError(); -#else - rc = pthread_key_create(key, nullptr); -#endif - TRACE("&key = %p, value %" PRIuPTR ", rc %d", __Wpedantic_format_voidptr(key), - (uintptr_t)*key, rc); - return rc; -} +/* An dirty-page list item is an pgno/pointer pair. */ +struct dp { + page_t *ptr; + pgno_t pgno, npages; +}; -static __inline void thread_key_delete(osal_thread_key_t key) { - TRACE("key = %" PRIuPTR, (uintptr_t)key); -#if defined(_WIN32) || defined(_WIN64) - ENSURE(nullptr, TlsFree(key)); -#else - ENSURE(nullptr, pthread_key_delete(key) == 0); - workaround_glibc_bug21031(); -#endif -} +enum dpl_rules { + dpl_gap_edging = 2, + dpl_gap_mergesort = 16, + dpl_reserve_gap = dpl_gap_mergesort + dpl_gap_edging, + dpl_insertion_threshold = 42 +}; -static __inline void *thread_rthc_get(osal_thread_key_t key) { -#if defined(_WIN32) || defined(_WIN64) - return TlsGetValue(key); -#else - return pthread_getspecific(key); -#endif -} +/* An DPL (dirty-page list) is a lazy-sorted array of MDBX_DPs. */ +struct dpl { + size_t sorted; + size_t length; + /* number of pages, but not an entries. */ + size_t pages_including_loose; + /* allocated size excluding the dpl_reserve_gap */ + size_t detent; + /* dynamic size with holes at zero and after the last */ + dp_t items[dpl_reserve_gap]; +}; -static void thread_rthc_set(osal_thread_key_t key, const void *value) { -#if defined(_WIN32) || defined(_WIN64) - ENSURE(nullptr, TlsSetValue(key, (void *)value)); -#else - const uint64_t sign_registered = - MDBX_THREAD_RTHC_REGISTERED(&rthc_thread_state); - const uint64_t sign_counted = MDBX_THREAD_RTHC_COUNTED(&rthc_thread_state); - if (value && unlikely(rthc_thread_state != sign_registered && - rthc_thread_state != sign_counted)) { - rthc_thread_state = sign_registered; - TRACE("thread registered 0x%" PRIxPTR, osal_thread_self()); - if (rthc_atexit(thread_dtor, &rthc_thread_state, - (void *)&mdbx_version /* dso_anchor */)) { - ENSURE(nullptr, pthread_setspecific(rthc_key, &rthc_thread_state) == 0); - rthc_thread_state = sign_counted; - const unsigned count_before = atomic_add32(&rthc_pending, 1); - ENSURE(nullptr, count_before < INT_MAX); - NOTICE("fallback to pthreads' tsd, key %" PRIuPTR ", count %u", - (uintptr_t)rthc_key, count_before); - (void)count_before; - } - } - ENSURE(nullptr, pthread_setspecific(key, value) == 0); -#endif -} +/*----------------------------------------------------------------------------*/ +/* Internal structures */ -/* dtor called for thread, i.e. for all mdbx's environment objects */ -__cold void thread_dtor(void *rthc) { - rthc_lock(); - TRACE(">> pid %d, thread 0x%" PRIxPTR ", rthc %p", osal_getpid(), - osal_thread_self(), rthc); +/* Comparing/ordering and length constraints */ +typedef struct clc { + MDBX_cmp_func *cmp; /* comparator */ + size_t lmin, lmax; /* min/max length constraints */ +} clc_t; - const uint32_t self_pid = osal_getpid(); - for (size_t i = 0; i < rthc_count; ++i) { - const osal_thread_key_t key = rthc_table[i].thr_tls_key; - MDBX_reader *const reader = thread_rthc_get(key); - if (reader < rthc_table[i].begin || reader >= rthc_table[i].end) - continue; -#if !defined(_WIN32) && !defined(_WIN64) - if (pthread_setspecific(key, nullptr) != 0) { - TRACE("== thread 0x%" PRIxPTR - ", rthc %p: ignore race with tsd-key deletion", - osal_thread_self(), __Wpedantic_format_voidptr(reader)); - continue /* ignore race with tsd-key deletion by mdbx_env_close() */; - } -#endif +/* Вспомогательная информация о table. + * + * Совокупность потребностей: + * 1. Для транзакций и основного курсора нужны все поля. + * 2. Для вложенного dupsort-курсора нужен компаратор значений, который изнутри + * курсора будет выглядеть как компаратор ключей. Плюс заглушка компаратора + * значений, которая не должна использоваться в штатных ситуациях, но + * требуется хотя-бы для отслеживания таких обращений. + * 3. Использование компараторов для курсора и вложенного dupsort-курсора + * должно выглядеть одинаково. + * 4. Желательно минимизировать объём данных размещаемых внутри вложенного + * dupsort-курсора. + * 5. Желательно чтобы объем всей структуры был степенью двойки. + * + * Решение: + * - не храним в dupsort-курсоре ничего лишнего, а только tree; + * - в курсоры помещаем только указатель на clc_t, который будет указывать + * на соответствующее clc-поле в общей kvx-таблице привязанной к env; + * - компаратор размещаем в начале clc_t, в kvx_t сначала размещаем clc + * для ключей, потом для значений, а имя БД в конце kvx_t. + * - тогда в курсоре clc[0] будет содержать информацию для ключей, + * а clc[1] для значений, причем компаратор значений для dupsort-курсора + * будет попадать на MDBX_val с именем, что приведет к SIGSEGV при попытке + * использования такого компаратора. + * - размер kvx_t становится равным 8 словам. + * + * Трюки и прочая экономия на спичках: + * - не храним dbi внутри курсора, вместо этого вычисляем его как разницу между + * dbi_state курсора и началом таблицы dbi_state в транзакции. Смысл тут в + * экономии кол-ва полей при инициализации курсора. Затрат это не создает, + * так как dbi требуется для последующего доступа к массивам в транзакции, + * т.е. при вычислении dbi разыменовывается тот-же указатель на txn + * и читается та же кэш-линия с указателями. */ +typedef struct clc2 { + clc_t k; /* для ключей */ + clc_t v; /* для значений */ +} clc2_t; + +struct kvx { + clc2_t clc; + MDBX_val name; /* имя table */ +}; - TRACE("== thread 0x%" PRIxPTR - ", rthc %p, [%zi], %p ... %p (%+i), rtch-pid %i, " - "current-pid %i", - osal_thread_self(), __Wpedantic_format_voidptr(reader), i, - __Wpedantic_format_voidptr(rthc_table[i].begin), - __Wpedantic_format_voidptr(rthc_table[i].end), - (int)(reader - rthc_table[i].begin), reader->mr_pid.weak, self_pid); - if (atomic_load32(&reader->mr_pid, mo_Relaxed) == self_pid) { - TRACE("==== thread 0x%" PRIxPTR ", rthc %p, cleanup", osal_thread_self(), - __Wpedantic_format_voidptr(reader)); - (void)atomic_cas32(&reader->mr_pid, self_pid, 0); - } - } +/* Non-shared DBI state flags inside transaction */ +enum dbi_state { + DBI_DIRTY = 0x01 /* DB was written in this txn */, + DBI_STALE = 0x02 /* Named-DB record is older than txnID */, + DBI_FRESH = 0x04 /* Named-DB handle opened in this txn */, + DBI_CREAT = 0x08 /* Named-DB handle created in this txn */, + DBI_VALID = 0x10 /* Handle is valid, see also DB_VALID */, + DBI_OLDEN = 0x40 /* Handle was closed/reopened outside txn */, + DBI_LINDO = 0x80 /* Lazy initialization done for DBI-slot */, +}; -#if defined(_WIN32) || defined(_WIN64) - TRACE("<< thread 0x%" PRIxPTR ", rthc %p", osal_thread_self(), rthc); - rthc_unlock(); -#else - const uint64_t sign_registered = MDBX_THREAD_RTHC_REGISTERED(rthc); - const uint64_t sign_counted = MDBX_THREAD_RTHC_COUNTED(rthc); - const uint64_t state = rthc_read(rthc); - if (state == sign_registered && - rthc_compare_and_clean(rthc, sign_registered)) { - TRACE("== thread 0x%" PRIxPTR - ", rthc %p, pid %d, self-status %s (0x%08" PRIx64 ")", - osal_thread_self(), rthc, osal_getpid(), "registered", state); - } else if (state == sign_counted && - rthc_compare_and_clean(rthc, sign_counted)) { - TRACE("== thread 0x%" PRIxPTR - ", rthc %p, pid %d, self-status %s (0x%08" PRIx64 ")", - osal_thread_self(), rthc, osal_getpid(), "counted", state); - ENSURE(nullptr, atomic_sub32(&rthc_pending, 1) > 0); - } else { - WARNING("thread 0x%" PRIxPTR - ", rthc %p, pid %d, self-status %s (0x%08" PRIx64 ")", - osal_thread_self(), rthc, osal_getpid(), "wrong", state); - } +enum txn_flags { + txn_ro_begin_flags = MDBX_TXN_RDONLY | MDBX_TXN_RDONLY_PREPARE, + txn_rw_begin_flags = MDBX_TXN_NOMETASYNC | MDBX_TXN_NOSYNC | MDBX_TXN_TRY, + txn_shrink_allowed = UINT32_C(0x40000000), + txn_parked = MDBX_TXN_PARKED, + txn_gc_drained = 0x40 /* GC was depleted up to oldest reader */, + txn_state_flags = MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_DIRTY | MDBX_TXN_SPILLS | MDBX_TXN_HAS_CHILD | + MDBX_TXN_INVALID | txn_gc_drained +}; - if (atomic_load32(&rthc_pending, mo_AcquireRelease) == 0) { - TRACE("== thread 0x%" PRIxPTR ", rthc %p, pid %d, wake", osal_thread_self(), - rthc, osal_getpid()); - ENSURE(nullptr, pthread_cond_broadcast(&rthc_cond) == 0); - } +/* A database transaction. + * Every operation requires a transaction handle. */ +struct MDBX_txn { + int32_t signature; + uint32_t flags; /* Transaction Flags */ + size_t n_dbi; + size_t owner; /* thread ID that owns this transaction */ - TRACE("<< thread 0x%" PRIxPTR ", rthc %p", osal_thread_self(), rthc); - /* Allow tail call optimization, i.e. gcc should generate the jmp instruction - * instead of a call for pthread_mutex_unlock() and therefore CPU could not - * return to current DSO's code section, which may be unloaded immediately - * after the mutex got released. */ - pthread_mutex_unlock(&rthc_mutex); -#endif -} + MDBX_txn *parent; /* parent of a nested txn */ + MDBX_txn *nested; /* nested txn under this txn, + set together with MDBX_TXN_HAS_CHILD */ + geo_t geo; -MDBX_EXCLUDE_FOR_GPROF -__cold void global_dtor(void) { - TRACE(">> pid %d", osal_getpid()); + /* The ID of this transaction. IDs are integers incrementing from + * INITIAL_TXNID. Only committed write transactions increment the ID. If a + * transaction aborts, the ID may be re-used by the next writer. */ + txnid_t txnid, front_txnid; - rthc_lock(); -#if !defined(_WIN32) && !defined(_WIN64) - uint64_t *rthc = pthread_getspecific(rthc_key); - TRACE("== thread 0x%" PRIxPTR ", rthc %p, pid %d, self-status 0x%08" PRIx64 - ", left %d", - osal_thread_self(), __Wpedantic_format_voidptr(rthc), osal_getpid(), - rthc ? rthc_read(rthc) : ~UINT64_C(0), - atomic_load32(&rthc_pending, mo_Relaxed)); - if (rthc) { - const uint64_t sign_registered = MDBX_THREAD_RTHC_REGISTERED(rthc); - const uint64_t sign_counted = MDBX_THREAD_RTHC_COUNTED(rthc); - const uint64_t state = rthc_read(rthc); - if (state == sign_registered && - rthc_compare_and_clean(rthc, sign_registered)) { - TRACE("== thread 0x%" PRIxPTR - ", rthc %p, pid %d, self-status %s (0x%08" PRIx64 ")", - osal_thread_self(), __Wpedantic_format_voidptr(rthc), osal_getpid(), - "registered", state); - } else if (state == sign_counted && - rthc_compare_and_clean(rthc, sign_counted)) { - TRACE("== thread 0x%" PRIxPTR - ", rthc %p, pid %d, self-status %s (0x%08" PRIx64 ")", - osal_thread_self(), __Wpedantic_format_voidptr(rthc), osal_getpid(), - "counted", state); - ENSURE(nullptr, atomic_sub32(&rthc_pending, 1) > 0); - } else { - WARNING("thread 0x%" PRIxPTR - ", rthc %p, pid %d, self-status %s (0x%08" PRIx64 ")", - osal_thread_self(), __Wpedantic_format_voidptr(rthc), - osal_getpid(), "wrong", state); - } - } + MDBX_env *env; /* the DB environment */ + tree_t *dbs; /* Array of tree_t records for each known DB */ - struct timespec abstime; - ENSURE(nullptr, clock_gettime(CLOCK_REALTIME, &abstime) == 0); - abstime.tv_nsec += 1000000000l / 10; - if (abstime.tv_nsec >= 1000000000l) { - abstime.tv_nsec -= 1000000000l; - abstime.tv_sec += 1; - } -#if MDBX_DEBUG > 0 - abstime.tv_sec += 600; -#endif +#if MDBX_ENABLE_DBI_SPARSE + unsigned *__restrict dbi_sparse; +#endif /* MDBX_ENABLE_DBI_SPARSE */ - for (unsigned left; - (left = atomic_load32(&rthc_pending, mo_AcquireRelease)) > 0;) { - NOTICE("tls-cleanup: pid %d, pending %u, wait for...", osal_getpid(), left); - const int rc = pthread_cond_timedwait(&rthc_cond, &rthc_mutex, &abstime); - if (rc && rc != EINTR) - break; - } - thread_key_delete(rthc_key); -#endif + /* Array of non-shared txn's flags of DBI. + * Модификатор __restrict тут полезен и безопасен в текущем понимании, + * так как пересечение возможно только с dbi_state курсоров, + * и происходит по-чтению до последующего изменения/записи. */ + uint8_t *__restrict dbi_state; - const uint32_t self_pid = osal_getpid(); - for (size_t i = 0; i < rthc_count; ++i) { - const osal_thread_key_t key = rthc_table[i].thr_tls_key; - thread_key_delete(key); - for (MDBX_reader *rthc = rthc_table[i].begin; rthc < rthc_table[i].end; - ++rthc) { - TRACE("== [%zi] = key %" PRIuPTR ", %p ... %p, rthc %p (%+i), " - "rthc-pid %i, current-pid %i", - i, (uintptr_t)key, __Wpedantic_format_voidptr(rthc_table[i].begin), - __Wpedantic_format_voidptr(rthc_table[i].end), - __Wpedantic_format_voidptr(rthc), (int)(rthc - rthc_table[i].begin), - rthc->mr_pid.weak, self_pid); - if (atomic_load32(&rthc->mr_pid, mo_Relaxed) == self_pid) { - atomic_store32(&rthc->mr_pid, 0, mo_AcquireRelease); - TRACE("== cleanup %p", __Wpedantic_format_voidptr(rthc)); - } - } - } + /* Array of sequence numbers for each DB handle. */ + uint32_t *__restrict dbi_seqs; - rthc_limit = rthc_count = 0; - if (rthc_table != rthc_table_static) - osal_free(rthc_table); - rthc_table = nullptr; - rthc_unlock(); + /* Массив с головами односвязных списков отслеживания курсоров. */ + MDBX_cursor **cursors; -#if defined(_WIN32) || defined(_WIN64) - DeleteCriticalSection(&lcklist_critical_section); - DeleteCriticalSection(&rthc_critical_section); + /* "Канареечные" маркеры/счетчики */ + MDBX_canary canary; + + /* User-settable context */ + void *userctx; + + union { + struct { + /* For read txns: This thread/txn's reader table slot, or nullptr. */ + reader_slot_t *reader; + } to; + struct { + troika_t troika; + pnl_t __restrict repnl; /* Reclaimed GC pages */ + struct { + /* The list of reclaimed txn-ids from GC */ + txl_t __restrict retxl; + txnid_t last_reclaimed; /* ID of last used record */ + uint64_t time_acc; + } gc; + bool prefault_write_activated; +#if MDBX_ENABLE_REFUND + pgno_t loose_refund_wl /* FIXME: describe */; +#endif /* MDBX_ENABLE_REFUND */ + /* a sequence to spilling dirty page with LRU policy */ + unsigned dirtylru; + /* dirtylist room: Dirty array size - dirty pages visible to this txn. + * Includes ancestor txns' dirty pages not hidden by other txns' + * dirty/spilled pages. Thus commit(nested txn) has room to merge + * dirtylist into parent after freeing hidden parent pages. */ + size_t dirtyroom; + /* For write txns: Modified pages. Sorted when not MDBX_WRITEMAP. */ + dpl_t *__restrict dirtylist; + /* The list of pages that became unused during this transaction. */ + pnl_t __restrict retired_pages; + /* The list of loose pages that became unused and may be reused + * in this transaction, linked through `page_next()`. */ + page_t *__restrict loose_pages; + /* Number of loose pages (tw.loose_pages) */ + size_t loose_count; + union { + struct { + size_t least_removed; + /* The sorted list of dirty pages we temporarily wrote to disk + * because the dirty list was full. page numbers in here are + * shifted left by 1, deleted slots have the LSB set. */ + pnl_t __restrict list; + } spilled; + size_t writemap_dirty_npages; + size_t writemap_spilled_npages; + }; + /* In write txns, next is located the array of cursors for each DB */ + } tw; + }; +}; + +#define CURSOR_STACK_SIZE (16 + MDBX_WORDBITS / 4) + +struct MDBX_cursor { + int32_t signature; + union { + /* Тут некоторые трюки/заморочки с тем чтобы во всех основных сценариях + * проверять состояние курсора одной простой операцией сравнения, + * и при этом ни на каплю не усложнять код итерации стека курсора. + * + * Поэтому решение такое: + * - поля flags и top сделаны знаковыми, а их отрицательные значения + * используются для обозначения не-установленного/не-инициализированного + * состояния курсора; + * - для инвалидации/сброса курсора достаточно записать отрицательное + * значение в объединенное поле top_and_flags; + * - все проверки состояния сводятся к сравнению одного из полей + * flags/snum/snum_and_flags, которые в зависимости от сценария, + * трактуются либо как знаковые, либо как безнаковые. */ + __anonymous_struct_extension__ struct { +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + int8_t flags; + /* индекс вершины стека, меньше нуля для не-инициализированного курсора */ + int8_t top; #else - /* LY: yielding a few timeslices to give a more chance - * to racing destructor(s) for completion. */ - workaround_glibc_bug21031(); + int8_t top; + int8_t flags; #endif + }; + int16_t top_and_flags; + }; + /* флаги проверки, в том числе биты для проверки типа листовых страниц. */ + uint8_t checking; + + /* Указывает на txn->dbi_state[] для DBI этого курсора. + * Модификатор __restrict тут полезен и безопасен в текущем понимании, + * так как пересечение возможно только с dbi_state транзакции, + * и происходит по-чтению до последующего изменения/записи. */ + uint8_t *__restrict dbi_state; + /* Связь списка отслеживания курсоров в транзакции */ + MDBX_txn *txn; + /* Указывает на tree->dbs[] для DBI этого курсора. */ + tree_t *tree; + /* Указывает на env->kvs[] для DBI этого курсора. */ + clc2_t *clc; + subcur_t *__restrict subcur; + page_t *pg[CURSOR_STACK_SIZE]; /* stack of pushed pages */ + indx_t ki[CURSOR_STACK_SIZE]; /* stack of page indices */ + MDBX_cursor *next; + /* Состояние на момент старта вложенной транзакции */ + MDBX_cursor *backup; +}; - osal_dtor(); - TRACE("<< pid %d\n", osal_getpid()); -} - -__cold int rthc_alloc(osal_thread_key_t *pkey, MDBX_reader *begin, - MDBX_reader *end) { - assert(pkey != NULL); -#ifndef NDEBUG - *pkey = (osal_thread_key_t)0xBADBADBAD; -#endif /* NDEBUG */ +struct inner_cursor { + MDBX_cursor cursor; + tree_t nested_tree; +}; - rthc_lock(); - TRACE(">> rthc_count %u, rthc_limit %u", rthc_count, rthc_limit); - int rc; - if (rthc_count == rthc_limit) { - rthc_entry_t *new_table = - osal_realloc((rthc_table == rthc_table_static) ? nullptr : rthc_table, - sizeof(rthc_entry_t) * rthc_limit * 2); - if (new_table == nullptr) { - rc = MDBX_ENOMEM; - goto bailout; - } - if (rthc_table == rthc_table_static) - memcpy(new_table, rthc_table, sizeof(rthc_entry_t) * rthc_limit); - rthc_table = new_table; - rthc_limit *= 2; - } +struct cursor_couple { + MDBX_cursor outer; + void *userctx; /* User-settable context */ + subcur_t inner; +}; - rc = thread_key_create(&rthc_table[rthc_count].thr_tls_key); - if (rc != MDBX_SUCCESS) - goto bailout; +enum env_flags { + /* Failed to update the meta page. Probably an I/O error. */ + ENV_FATAL_ERROR = INT32_MIN /* 0x80000000 */, + /* Some fields are initialized. */ + ENV_ACTIVE = UINT32_C(0x20000000), + /* me_txkey is set */ + ENV_TXKEY = UINT32_C(0x10000000), + /* Legacy MDBX_MAPASYNC (prior v0.9) */ + DEPRECATED_MAPASYNC = UINT32_C(0x100000), + /* Legacy MDBX_COALESCE (prior v0.12) */ + DEPRECATED_COALESCE = UINT32_C(0x2000000), + ENV_INTERNAL_FLAGS = ENV_FATAL_ERROR | ENV_ACTIVE | ENV_TXKEY, + /* Only a subset of the mdbx_env flags can be changed + * at runtime. Changing other flags requires closing the + * environment and re-opening it with the new flags. */ + ENV_CHANGEABLE_FLAGS = MDBX_SAFE_NOSYNC | MDBX_NOMETASYNC | DEPRECATED_MAPASYNC | MDBX_NOMEMINIT | + DEPRECATED_COALESCE | MDBX_PAGEPERTURB | MDBX_ACCEDE | MDBX_VALIDATION, + ENV_CHANGELESS_FLAGS = MDBX_NOSUBDIR | MDBX_RDONLY | MDBX_WRITEMAP | MDBX_NOSTICKYTHREADS | MDBX_NORDAHEAD | + MDBX_LIFORECLAIM | MDBX_EXCLUSIVE, + ENV_USABLE_FLAGS = ENV_CHANGEABLE_FLAGS | ENV_CHANGELESS_FLAGS +}; - *pkey = rthc_table[rthc_count].thr_tls_key; - TRACE("== [%i] = key %" PRIuPTR ", %p ... %p", rthc_count, (uintptr_t)*pkey, - __Wpedantic_format_voidptr(begin), __Wpedantic_format_voidptr(end)); +/* The database environment. */ +struct MDBX_env { + /* ----------------------------------------------------- mostly static part */ + mdbx_atomic_uint32_t signature; + uint32_t flags; + unsigned ps; /* DB page size, initialized from me_os_psize */ + osal_mmap_t dxb_mmap; /* The main data file */ +#define lazy_fd dxb_mmap.fd + mdbx_filehandle_t dsync_fd, fd4meta; +#if defined(_WIN32) || defined(_WIN64) + HANDLE dxb_lock_event; +#endif /* Windows */ + osal_mmap_t lck_mmap; /* The lock file */ + lck_t *lck; + + uint16_t leaf_nodemax; /* max size of a leaf-node */ + uint16_t branch_nodemax; /* max size of a branch-node */ + uint16_t subpage_limit; + uint16_t subpage_room_threshold; + uint16_t subpage_reserve_prereq; + uint16_t subpage_reserve_limit; + atomic_pgno_t mlocked_pgno; + uint8_t ps2ln; /* log2 of DB page size */ + int8_t stuck_meta; /* recovery-only: target meta page or less that zero */ + uint16_t merge_threshold, merge_threshold_gc; /* pages emptier than this are + candidates for merging */ + unsigned max_readers; /* size of the reader table */ + MDBX_dbi max_dbi; /* size of the DB table */ + uint32_t pid; /* process ID of this env */ + osal_thread_key_t me_txkey; /* thread-key for readers */ + struct { /* path to the DB files */ + pathchar_t *lck, *dxb, *specified; + void *buffer; + } pathname; + void *page_auxbuf; /* scratch area for DUPSORT put() */ + MDBX_txn *basal_txn; /* preallocated write transaction */ + kvx_t *kvs; /* array of auxiliary key-value properties */ + uint8_t *__restrict dbs_flags; /* array of flags from tree_t.flags */ + mdbx_atomic_uint32_t *dbi_seqs; /* array of dbi sequence numbers */ + unsigned maxgc_large1page; /* Number of pgno_t fit in a single large page */ + unsigned maxgc_per_branch; + uint32_t registered_reader_pid; /* have liveness lock in reader table */ + void *userctx; /* User-settable context */ + MDBX_hsr_func *hsr_callback; /* Callback for kicking laggard readers */ + size_t madv_threshold; - rthc_table[rthc_count].begin = begin; - rthc_table[rthc_count].end = end; - ++rthc_count; - TRACE("<< key %" PRIuPTR ", rthc_count %u, rthc_limit %u", (uintptr_t)*pkey, - rthc_count, rthc_limit); - rthc_unlock(); - return MDBX_SUCCESS; + struct { + unsigned dp_reserve_limit; + unsigned rp_augment_limit; + unsigned dp_limit; + unsigned dp_initial; + uint64_t gc_time_limit; + uint8_t dp_loose_limit; + uint8_t spill_max_denominator; + uint8_t spill_min_denominator; + uint8_t spill_parent4child_denominator; + unsigned merge_threshold_16dot16_percent; +#if !(defined(_WIN32) || defined(_WIN64)) + unsigned writethrough_threshold; +#endif /* Windows */ + bool prefault_write; + bool prefer_waf_insteadof_balance; /* Strive to minimize WAF instead of + balancing pages fullment */ + bool need_dp_limit_adjust; + struct { + uint16_t limit; + uint16_t room_threshold; + uint16_t reserve_prereq; + uint16_t reserve_limit; + } subpage; -bailout: - rthc_unlock(); - return rc; -} + union { + unsigned all; + /* tracks options with non-auto values but tuned by user */ + struct { + unsigned dp_limit : 1; + unsigned rp_augment_limit : 1; + unsigned prefault_write : 1; + } non_auto; + } flags; + } options; -__cold void rthc_remove(const osal_thread_key_t key) { - thread_key_delete(key); - rthc_lock(); - TRACE(">> key %zu, rthc_count %u, rthc_limit %u", (uintptr_t)key, rthc_count, - rthc_limit); + /* struct geo_in_bytes used for accepting db-geo params from user for the new + * database creation, i.e. when mdbx_env_set_geometry() was called before + * mdbx_env_open(). */ + struct { + size_t lower; /* minimal size of datafile */ + size_t upper; /* maximal size of datafile */ + size_t now; /* current size of datafile */ + size_t grow; /* step to grow datafile */ + size_t shrink; /* threshold to shrink datafile */ + } geo_in_bytes; - for (size_t i = 0; i < rthc_count; ++i) { - if (key == rthc_table[i].thr_tls_key) { - const uint32_t self_pid = osal_getpid(); - TRACE("== [%zi], %p ...%p, current-pid %d", i, - __Wpedantic_format_voidptr(rthc_table[i].begin), - __Wpedantic_format_voidptr(rthc_table[i].end), self_pid); - - for (MDBX_reader *rthc = rthc_table[i].begin; rthc < rthc_table[i].end; - ++rthc) { - if (atomic_load32(&rthc->mr_pid, mo_Relaxed) == self_pid) { - atomic_store32(&rthc->mr_pid, 0, mo_AcquireRelease); - TRACE("== cleanup %p", __Wpedantic_format_voidptr(rthc)); - } - } - if (--rthc_count > 0) - rthc_table[i] = rthc_table[rthc_count]; - else if (rthc_table != rthc_table_static) { - osal_free(rthc_table); - rthc_table = rthc_table_static; - rthc_limit = RTHC_INITIAL_LIMIT; - } - break; - } - } +#if MDBX_LOCKING == MDBX_LOCKING_SYSV + union { + key_t key; + int semid; + } me_sysv_ipc; +#endif /* MDBX_LOCKING == MDBX_LOCKING_SYSV */ + bool incore; - TRACE("<< key %zu, rthc_count %u, rthc_limit %u", (size_t)key, rthc_count, - rthc_limit); - rthc_unlock(); -} +#if MDBX_ENABLE_DBI_LOCKFREE + defer_free_item_t *defer_free; +#endif /* MDBX_ENABLE_DBI_LOCKFREE */ -//------------------------------------------------------------------------------ + /* -------------------------------------------------------------- debugging */ -#define RTHC_ENVLIST_END ((MDBX_env *)((uintptr_t)50459)) -static MDBX_env *inprocess_lcklist_head = RTHC_ENVLIST_END; +#if MDBX_DEBUG + MDBX_assert_func *assert_func; /* Callback for assertion failures */ +#endif +#ifdef ENABLE_MEMCHECK + int valgrind_handle; +#endif +#if defined(ENABLE_MEMCHECK) || defined(__SANITIZE_ADDRESS__) + pgno_t poison_edge; +#endif /* ENABLE_MEMCHECK || __SANITIZE_ADDRESS__ */ -static __inline void lcklist_lock(void) { -#if defined(_WIN32) || defined(_WIN64) - EnterCriticalSection(&lcklist_critical_section); -#else - ENSURE(nullptr, osal_pthread_mutex_lock(&lcklist_mutex) == 0); +#ifndef xMDBX_DEBUG_SPILLING +#define xMDBX_DEBUG_SPILLING 0 #endif -} +#if xMDBX_DEBUG_SPILLING == 2 + size_t debug_dirtied_est, debug_dirtied_act; +#endif /* xMDBX_DEBUG_SPILLING */ + + /* --------------------------------------------------- mostly volatile part */ + + MDBX_txn *txn; /* current write transaction */ + osal_fastmutex_t dbi_lock; + unsigned n_dbi; /* number of DBs opened */ + + unsigned shadow_reserve_len; + page_t *__restrict shadow_reserve; /* list of malloc'ed blocks for re-use */ + + osal_ioring_t ioring; -static __inline void lcklist_unlock(void) { #if defined(_WIN32) || defined(_WIN64) - LeaveCriticalSection(&lcklist_critical_section); + osal_srwlock_t remap_guard; + /* Workaround for LockFileEx and WriteFile multithread bug */ + CRITICAL_SECTION windowsbug_lock; + char *pathname_char; /* cache of multi-byte representation of pathname + to the DB files */ #else - ENSURE(nullptr, pthread_mutex_unlock(&lcklist_mutex) == 0); + osal_fastmutex_t remap_guard; #endif -} -MDBX_NOTHROW_CONST_FUNCTION static uint64_t rrxmrrxmsx_0(uint64_t v) { - /* Pelle Evensen's mixer, https://bit.ly/2HOfynt */ - v ^= (v << 39 | v >> 25) ^ (v << 14 | v >> 50); - v *= UINT64_C(0xA24BAED4963EE407); - v ^= (v << 40 | v >> 24) ^ (v << 15 | v >> 49); - v *= UINT64_C(0x9FB21C651E98DF25); - return v ^ v >> 28; + /* ------------------------------------------------- stub for lck-less mode */ + mdbx_atomic_uint64_t lckless_placeholder[(sizeof(lck_t) + MDBX_CACHELINE_SIZE - 1) / sizeof(mdbx_atomic_uint64_t)]; +}; + +/*----------------------------------------------------------------------------*/ + +/* pseudo-error code, not exposed outside libmdbx */ +#define MDBX_NO_ROOT (MDBX_LAST_ADDED_ERRCODE + 33) + +/* Number of slots in the reader table. + * This value was chosen somewhat arbitrarily. The 61 is a prime number, + * and such readers plus a couple mutexes fit into single 4KB page. + * Applications should set the table size using mdbx_env_set_maxreaders(). */ +#define DEFAULT_READERS 61 + +enum db_flags { + DB_PERSISTENT_FLAGS = + MDBX_REVERSEKEY | MDBX_DUPSORT | MDBX_INTEGERKEY | MDBX_DUPFIXED | MDBX_INTEGERDUP | MDBX_REVERSEDUP, + + /* mdbx_dbi_open() flags */ + DB_USABLE_FLAGS = DB_PERSISTENT_FLAGS | MDBX_CREATE | MDBX_DB_ACCEDE, + + DB_VALID = 0x80u /* DB handle is valid, for dbs_flags */, + DB_POISON = 0x7fu /* update pending */, + DB_INTERNAL_FLAGS = DB_VALID +}; + +#if !defined(__cplusplus) || CONSTEXPR_ENUM_FLAGS_OPERATIONS +MDBX_MAYBE_UNUSED static void static_checks(void) { + STATIC_ASSERT(MDBX_WORDBITS == sizeof(void *) * CHAR_BIT); + STATIC_ASSERT(UINT64_C(0x80000000) == (uint32_t)ENV_FATAL_ERROR); + STATIC_ASSERT_MSG(INT16_MAX - CORE_DBS == MDBX_MAX_DBI, "Oops, MDBX_MAX_DBI or CORE_DBS?"); + STATIC_ASSERT_MSG((unsigned)(MDBX_DB_ACCEDE | MDBX_CREATE) == + ((DB_USABLE_FLAGS | DB_INTERNAL_FLAGS) & (ENV_USABLE_FLAGS | ENV_INTERNAL_FLAGS)), + "Oops, some flags overlapped or wrong"); + STATIC_ASSERT_MSG((DB_INTERNAL_FLAGS & DB_USABLE_FLAGS) == 0, "Oops, some flags overlapped or wrong"); + STATIC_ASSERT_MSG((DB_PERSISTENT_FLAGS & ~DB_USABLE_FLAGS) == 0, "Oops, some flags overlapped or wrong"); + STATIC_ASSERT(DB_PERSISTENT_FLAGS <= UINT8_MAX); + STATIC_ASSERT_MSG((ENV_INTERNAL_FLAGS & ENV_USABLE_FLAGS) == 0, "Oops, some flags overlapped or wrong"); + + STATIC_ASSERT_MSG((txn_state_flags & (txn_rw_begin_flags | txn_ro_begin_flags)) == 0, + "Oops, some txn flags overlapped or wrong"); + STATIC_ASSERT_MSG(((txn_rw_begin_flags | txn_ro_begin_flags | txn_state_flags) & txn_shrink_allowed) == 0, + "Oops, some txn flags overlapped or wrong"); + + STATIC_ASSERT(sizeof(reader_slot_t) == 32); +#if MDBX_LOCKING > 0 + STATIC_ASSERT(offsetof(lck_t, wrt_lock) % MDBX_CACHELINE_SIZE == 0); + STATIC_ASSERT(offsetof(lck_t, rdt_lock) % MDBX_CACHELINE_SIZE == 0); +#else + STATIC_ASSERT(offsetof(lck_t, cached_oldest) % MDBX_CACHELINE_SIZE == 0); + STATIC_ASSERT(offsetof(lck_t, rdt_length) % MDBX_CACHELINE_SIZE == 0); +#endif /* MDBX_LOCKING */ + STATIC_ASSERT(offsetof(lck_t, rdt) % MDBX_CACHELINE_SIZE == 0); + +#if FLEXIBLE_ARRAY_MEMBERS + STATIC_ASSERT(NODESIZE == offsetof(node_t, payload)); + STATIC_ASSERT(PAGEHDRSZ == offsetof(page_t, entries)); +#endif /* FLEXIBLE_ARRAY_MEMBERS */ + STATIC_ASSERT(sizeof(clc_t) == 3 * sizeof(void *)); + STATIC_ASSERT(sizeof(kvx_t) == 8 * sizeof(void *)); + +#if MDBX_WORDBITS == 64 +#define KVX_SIZE_LN2 6 +#else +#define KVX_SIZE_LN2 5 +#endif + STATIC_ASSERT(sizeof(kvx_t) == (1u << KVX_SIZE_LN2)); } +#endif /* Disabled for MSVC 19.0 (VisualStudio 2015) */ -static int uniq_peek(const osal_mmap_t *pending, osal_mmap_t *scan) { - int rc; - uint64_t bait; - MDBX_lockinfo *const pending_lck = pending->lck; - MDBX_lockinfo *const scan_lck = scan->lck; - if (pending_lck) { - bait = atomic_load64(&pending_lck->mti_bait_uniqueness, mo_AcquireRelease); - rc = MDBX_SUCCESS; - } else { - bait = 0 /* hush MSVC warning */; - rc = osal_msync(scan, 0, sizeof(MDBX_lockinfo), MDBX_SYNC_DATA); - if (rc == MDBX_SUCCESS) - rc = osal_pread(pending->fd, &bait, sizeof(scan_lck->mti_bait_uniqueness), - offsetof(MDBX_lockinfo, mti_bait_uniqueness)); - } - if (likely(rc == MDBX_SUCCESS) && - bait == atomic_load64(&scan_lck->mti_bait_uniqueness, mo_AcquireRelease)) - rc = MDBX_RESULT_TRUE; +/******************************************************************************/ - TRACE("uniq-peek: %s, bait 0x%016" PRIx64 ",%s rc %d", - pending_lck ? "mem" : "file", bait, - (rc == MDBX_RESULT_TRUE) ? " found," : (rc ? " FAILED," : ""), rc); - return rc; +/* valid flags for mdbx_node_add() */ +#define NODE_ADD_FLAGS (N_DUP | N_TREE | MDBX_RESERVE | MDBX_APPEND) + +/* Get the page number pointed to by a branch node */ +MDBX_NOTHROW_PURE_FUNCTION static inline pgno_t node_pgno(const node_t *const __restrict node) { + pgno_t pgno = UNALIGNED_PEEK_32(node, node_t, child_pgno); + return pgno; } -static int uniq_poke(const osal_mmap_t *pending, osal_mmap_t *scan, - uint64_t *abra) { - if (*abra == 0) { - const uintptr_t tid = osal_thread_self(); - uintptr_t uit = 0; - memcpy(&uit, &tid, (sizeof(tid) < sizeof(uit)) ? sizeof(tid) : sizeof(uit)); - *abra = rrxmrrxmsx_0(osal_monotime() + UINT64_C(5873865991930747) * uit); - } - const uint64_t cadabra = - rrxmrrxmsx_0(*abra + UINT64_C(7680760450171793) * (unsigned)osal_getpid()) - << 24 | - *abra >> 40; - MDBX_lockinfo *const scan_lck = scan->lck; - atomic_store64(&scan_lck->mti_bait_uniqueness, cadabra, mo_AcquireRelease); - *abra = *abra * UINT64_C(6364136223846793005) + 1; - return uniq_peek(pending, scan); +/* Set the page number in a branch node */ +static inline void node_set_pgno(node_t *const __restrict node, pgno_t pgno) { + assert(pgno >= MIN_PAGENO && pgno <= MAX_PAGENO); + + UNALIGNED_POKE_32(node, node_t, child_pgno, (uint32_t)pgno); } -__cold static int uniq_check(const osal_mmap_t *pending, MDBX_env **found) { - *found = nullptr; - uint64_t salt = 0; - for (MDBX_env *scan = inprocess_lcklist_head; scan != RTHC_ENVLIST_END; - scan = scan->me_lcklist_next) { - MDBX_lockinfo *const scan_lck = scan->me_lck_mmap.lck; - int err = atomic_load64(&scan_lck->mti_bait_uniqueness, mo_AcquireRelease) - ? uniq_peek(pending, &scan->me_lck_mmap) - : uniq_poke(pending, &scan->me_lck_mmap, &salt); - if (err == MDBX_ENODATA) { - uint64_t length = 0; - if (likely(osal_filesize(pending->fd, &length) == MDBX_SUCCESS && - length == 0)) { - /* LY: skip checking since LCK-file is empty, i.e. just created. */ - DEBUG("uniq-probe: %s", "unique (new/empty lck)"); - return MDBX_RESULT_TRUE; - } - } - if (err == MDBX_RESULT_TRUE) - err = uniq_poke(pending, &scan->me_lck_mmap, &salt); - if (err == MDBX_RESULT_TRUE) { - (void)osal_msync(&scan->me_lck_mmap, 0, sizeof(MDBX_lockinfo), - MDBX_SYNC_KICK); - err = uniq_poke(pending, &scan->me_lck_mmap, &salt); - } - if (err == MDBX_RESULT_TRUE) { - err = uniq_poke(pending, &scan->me_lck_mmap, &salt); - *found = scan; - DEBUG("uniq-probe: found %p", __Wpedantic_format_voidptr(*found)); - return MDBX_RESULT_FALSE; - } - if (unlikely(err != MDBX_SUCCESS)) { - DEBUG("uniq-probe: failed rc %d", err); - return err; - } - } +/* Get the size of the data in a leaf node */ +MDBX_NOTHROW_PURE_FUNCTION static inline size_t node_ds(const node_t *const __restrict node) { + return UNALIGNED_PEEK_32(node, node_t, dsize); +} - DEBUG("uniq-probe: %s", "unique"); - return MDBX_RESULT_TRUE; +/* Set the size of the data for a leaf node */ +static inline void node_set_ds(node_t *const __restrict node, size_t size) { + assert(size < INT_MAX); + UNALIGNED_POKE_32(node, node_t, dsize, (uint32_t)size); } -static int lcklist_detach_locked(MDBX_env *env) { - MDBX_env *inprocess_neighbor = nullptr; - int rc = MDBX_SUCCESS; - if (env->me_lcklist_next != nullptr) { - ENSURE(env, env->me_lcklist_next != nullptr); - ENSURE(env, inprocess_lcklist_head != RTHC_ENVLIST_END); - for (MDBX_env **ptr = &inprocess_lcklist_head; *ptr != RTHC_ENVLIST_END; - ptr = &(*ptr)->me_lcklist_next) { - if (*ptr == env) { - *ptr = env->me_lcklist_next; - env->me_lcklist_next = nullptr; - break; - } - } - ENSURE(env, env->me_lcklist_next == nullptr); - } +/* The size of a key in a node */ +MDBX_NOTHROW_PURE_FUNCTION static inline size_t node_ks(const node_t *const __restrict node) { + return UNALIGNED_PEEK_16(node, node_t, ksize); +} - rc = likely(osal_getpid() == env->me_pid) - ? uniq_check(&env->me_lck_mmap, &inprocess_neighbor) - : MDBX_PANIC; - if (!inprocess_neighbor && env->me_live_reader) - (void)osal_rpid_clear(env); - if (!MDBX_IS_ERROR(rc)) - rc = osal_lck_destroy(env, inprocess_neighbor); - return rc; +/* Set the size of the key for a leaf node */ +static inline void node_set_ks(node_t *const __restrict node, size_t size) { + assert(size < INT16_MAX); + UNALIGNED_POKE_16(node, node_t, ksize, (uint16_t)size); } -/*------------------------------------------------------------------------------ - * LY: State of the art quicksort-based sorting, with internal stack - * and network-sort for small chunks. - * Thanks to John M. Gamble for the http://pages.ripco.net/~jgamble/nw.html */ +MDBX_NOTHROW_PURE_FUNCTION static inline uint8_t node_flags(const node_t *const __restrict node) { + return UNALIGNED_PEEK_8(node, node_t, flags); +} -#if MDBX_HAVE_CMOV -#define SORT_CMP_SWAP(TYPE, CMP, a, b) \ - do { \ - const TYPE swap_tmp = (a); \ - const bool swap_cmp = expect_with_probability(CMP(swap_tmp, b), 0, .5); \ - (a) = swap_cmp ? swap_tmp : b; \ - (b) = swap_cmp ? b : swap_tmp; \ - } while (0) -#else -#define SORT_CMP_SWAP(TYPE, CMP, a, b) \ - do \ - if (expect_with_probability(!CMP(a, b), 0, .5)) { \ - const TYPE swap_tmp = (a); \ - (a) = (b); \ - (b) = swap_tmp; \ - } \ - while (0) -#endif +static inline void node_set_flags(node_t *const __restrict node, uint8_t flags) { + UNALIGNED_POKE_8(node, node_t, flags, flags); +} -// 3 comparators, 3 parallel operations -// o-----^--^--o -// | | -// o--^--|--v--o -// | | -// o--v--v-----o -// -// [[1,2]] -// [[0,2]] -// [[0,1]] -#define SORT_NETWORK_3(TYPE, CMP, begin) \ - do { \ - SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[2]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[2]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[1]); \ - } while (0) +/* Address of the key for the node */ +MDBX_NOTHROW_PURE_FUNCTION static inline void *node_key(const node_t *const __restrict node) { + return ptr_disp(node, NODESIZE); +} -// 5 comparators, 3 parallel operations -// o--^--^--------o -// | | -// o--v--|--^--^--o -// | | | -// o--^--v--|--v--o -// | | -// o--v-----v-----o -// -// [[0,1],[2,3]] -// [[0,2],[1,3]] -// [[1,2]] -#define SORT_NETWORK_4(TYPE, CMP, begin) \ - do { \ - SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[1]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[3]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[2]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[3]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[2]); \ - } while (0) +/* Address of the data for a node */ +MDBX_NOTHROW_PURE_FUNCTION static inline void *node_data(const node_t *const __restrict node) { + return ptr_disp(node_key(node), node_ks(node)); +} -// 9 comparators, 5 parallel operations -// o--^--^-----^-----------o -// | | | -// o--|--|--^--v-----^--^--o -// | | | | | -// o--|--v--|--^--^--|--v--o -// | | | | | -// o--|-----v--|--v--|--^--o -// | | | | -// o--v--------v-----v--v--o -// -// [[0,4],[1,3]] -// [[0,2]] -// [[2,4],[0,1]] -// [[2,3],[1,4]] -// [[1,2],[3,4]] -#define SORT_NETWORK_5(TYPE, CMP, begin) \ - do { \ - SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[4]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[3]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[2]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[4]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[1]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[3]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[4]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[2]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[4]); \ - } while (0) +/* Size of a node in a leaf page with a given key and data. + * This is node header plus key plus data size. */ +MDBX_NOTHROW_CONST_FUNCTION static inline size_t node_size_len(const size_t key_len, const size_t value_len) { + return NODESIZE + EVEN_CEIL(key_len + value_len); +} +MDBX_NOTHROW_PURE_FUNCTION static inline size_t node_size(const MDBX_val *key, const MDBX_val *value) { + return node_size_len(key ? key->iov_len : 0, value ? value->iov_len : 0); +} -// 12 comparators, 6 parallel operations -// o-----^--^--^-----------------o -// | | | -// o--^--|--v--|--^--------^-----o -// | | | | | -// o--v--v-----|--|--^--^--|--^--o -// | | | | | | -// o-----^--^--v--|--|--|--v--v--o -// | | | | | -// o--^--|--v-----v--|--v--------o -// | | | -// o--v--v-----------v-----------o -// -// [[1,2],[4,5]] -// [[0,2],[3,5]] -// [[0,1],[3,4],[2,5]] -// [[0,3],[1,4]] -// [[2,4],[1,3]] -// [[2,3]] -#define SORT_NETWORK_6(TYPE, CMP, begin) \ - do { \ - SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[2]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[4], begin[5]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[2]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[5]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[1]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[4]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[5]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[3]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[4]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[4]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[3]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[3]); \ - } while (0) +MDBX_NOTHROW_PURE_FUNCTION static inline pgno_t node_largedata_pgno(const node_t *const __restrict node) { + assert(node_flags(node) & N_BIG); + return peek_pgno(node_data(node)); +} -// 16 comparators, 6 parallel operations -// o--^--------^-----^-----------------o -// | | | -// o--|--^-----|--^--v--------^--^-----o -// | | | | | | -// o--|--|--^--v--|--^-----^--|--v-----o -// | | | | | | | -// o--|--|--|-----v--|--^--v--|--^--^--o -// | | | | | | | | -// o--v--|--|--^-----v--|--^--v--|--v--o -// | | | | | | -// o-----v--|--|--------v--v-----|--^--o -// | | | | -// o--------v--v-----------------v--v--o -// -// [[0,4],[1,5],[2,6]] -// [[0,2],[1,3],[4,6]] -// [[2,4],[3,5],[0,1]] -// [[2,3],[4,5]] -// [[1,4],[3,6]] -// [[1,2],[3,4],[5,6]] -#define SORT_NETWORK_7(TYPE, CMP, begin) \ - do { \ - SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[4]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[5]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[6]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[2]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[3]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[4], begin[6]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[4]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[5]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[1]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[3]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[4], begin[5]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[4]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[6]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[2]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[4]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[5], begin[6]); \ - } while (0) +MDBX_INTERNAL int __must_check_result node_read_bigdata(MDBX_cursor *mc, const node_t *node, MDBX_val *data, + const page_t *mp); -// 19 comparators, 6 parallel operations -// o--^--------^-----^-----------------o -// | | | -// o--|--^-----|--^--v--------^--^-----o -// | | | | | | -// o--|--|--^--v--|--^-----^--|--v-----o -// | | | | | | | -// o--|--|--|--^--v--|--^--v--|--^--^--o -// | | | | | | | | | -// o--v--|--|--|--^--v--|--^--v--|--v--o -// | | | | | | | -// o-----v--|--|--|--^--v--v-----|--^--o -// | | | | | | -// o--------v--|--v--|--^--------v--v--o -// | | | -// o-----------v-----v--v--------------o -// -// [[0,4],[1,5],[2,6],[3,7]] -// [[0,2],[1,3],[4,6],[5,7]] -// [[2,4],[3,5],[0,1],[6,7]] -// [[2,3],[4,5]] -// [[1,4],[3,6]] -// [[1,2],[3,4],[5,6]] -#define SORT_NETWORK_8(TYPE, CMP, begin) \ - do { \ - SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[4]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[5]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[6]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[7]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[2]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[3]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[4], begin[6]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[5], begin[7]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[4]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[5]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[1]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[6], begin[7]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[3]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[4], begin[5]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[4]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[6]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[2]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[4]); \ - SORT_CMP_SWAP(TYPE, CMP, begin[5], begin[6]); \ - } while (0) +static inline int __must_check_result node_read(MDBX_cursor *mc, const node_t *node, MDBX_val *data, const page_t *mp) { + data->iov_len = node_ds(node); + data->iov_base = node_data(node); + if (likely(node_flags(node) != N_BIG)) + return MDBX_SUCCESS; + return node_read_bigdata(mc, node, data, mp); +} -#define SORT_INNER(TYPE, CMP, begin, end, len) \ - switch (len) { \ - default: \ - assert(false); \ - __unreachable(); \ - case 0: \ - case 1: \ - break; \ - case 2: \ - SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[1]); \ - break; \ - case 3: \ - SORT_NETWORK_3(TYPE, CMP, begin); \ - break; \ - case 4: \ - SORT_NETWORK_4(TYPE, CMP, begin); \ - break; \ - case 5: \ - SORT_NETWORK_5(TYPE, CMP, begin); \ - break; \ - case 6: \ - SORT_NETWORK_6(TYPE, CMP, begin); \ - break; \ - case 7: \ - SORT_NETWORK_7(TYPE, CMP, begin); \ - break; \ - case 8: \ - SORT_NETWORK_8(TYPE, CMP, begin); \ - break; \ - } - -#define SORT_SWAP(TYPE, a, b) \ - do { \ - const TYPE swap_tmp = (a); \ - (a) = (b); \ - (b) = swap_tmp; \ - } while (0) +/*----------------------------------------------------------------------------*/ -#define SORT_PUSH(low, high) \ - do { \ - top->lo = (low); \ - top->hi = (high); \ - ++top; \ - } while (0) +MDBX_INTERNAL nsr_t node_search(MDBX_cursor *mc, const MDBX_val *key); -#define SORT_POP(low, high) \ - do { \ - --top; \ - low = top->lo; \ - high = top->hi; \ - } while (0) +MDBX_INTERNAL int __must_check_result node_add_branch(MDBX_cursor *mc, size_t indx, const MDBX_val *key, pgno_t pgno); -#define SORT_IMPL(NAME, EXPECT_LOW_CARDINALITY_OR_PRESORTED, TYPE, CMP) \ - \ - static __inline bool NAME##_is_sorted(const TYPE *first, const TYPE *last) { \ - while (++first <= last) \ - if (expect_with_probability(CMP(first[0], first[-1]), 1, .1)) \ - return false; \ - return true; \ - } \ - \ - typedef struct { \ - TYPE *lo, *hi; \ - } NAME##_stack; \ - \ - __hot static void NAME(TYPE *const __restrict begin, \ - TYPE *const __restrict end) { \ - NAME##_stack stack[sizeof(size_t) * CHAR_BIT], *__restrict top = stack; \ - \ - TYPE *__restrict hi = end - 1; \ - TYPE *__restrict lo = begin; \ - while (true) { \ - const ptrdiff_t len = hi - lo; \ - if (len < 8) { \ - SORT_INNER(TYPE, CMP, lo, hi + 1, len + 1); \ - if (unlikely(top == stack)) \ - break; \ - SORT_POP(lo, hi); \ - continue; \ - } \ - \ - TYPE *__restrict mid = lo + (len >> 1); \ - SORT_CMP_SWAP(TYPE, CMP, *lo, *mid); \ - SORT_CMP_SWAP(TYPE, CMP, *mid, *hi); \ - SORT_CMP_SWAP(TYPE, CMP, *lo, *mid); \ - \ - TYPE *right = hi - 1; \ - TYPE *left = lo + 1; \ - while (1) { \ - while (expect_with_probability(CMP(*left, *mid), 0, .5)) \ - ++left; \ - while (expect_with_probability(CMP(*mid, *right), 0, .5)) \ - --right; \ - if (unlikely(left > right)) { \ - if (EXPECT_LOW_CARDINALITY_OR_PRESORTED) { \ - if (NAME##_is_sorted(lo, right)) \ - lo = right + 1; \ - if (NAME##_is_sorted(left, hi)) \ - hi = left; \ - } \ - break; \ - } \ - SORT_SWAP(TYPE, *left, *right); \ - mid = (mid == left) ? right : (mid == right) ? left : mid; \ - ++left; \ - --right; \ - } \ - \ - if (right - lo > hi - left) { \ - SORT_PUSH(lo, right); \ - lo = left; \ - } else { \ - SORT_PUSH(left, hi); \ - hi = right; \ - } \ - } \ - \ - if (AUDIT_ENABLED()) { \ - for (TYPE *scan = begin + 1; scan < end; ++scan) \ - assert(CMP(scan[-1], scan[0])); \ - } \ - } +MDBX_INTERNAL int __must_check_result node_add_leaf(MDBX_cursor *mc, size_t indx, const MDBX_val *key, MDBX_val *data, + unsigned flags); -/*------------------------------------------------------------------------------ - * LY: radix sort for large chunks */ +MDBX_INTERNAL int __must_check_result node_add_dupfix(MDBX_cursor *mc, size_t indx, const MDBX_val *key); -#define RADIXSORT_IMPL(NAME, TYPE, EXTRACT_KEY, BUFFER_PREALLOCATED, END_GAP) \ - \ - __hot static bool NAME##_radixsort(TYPE *const begin, const size_t length) { \ - TYPE *tmp; \ - if (BUFFER_PREALLOCATED) { \ - tmp = begin + length + END_GAP; \ - /* memset(tmp, 0xDeadBeef, sizeof(TYPE) * length); */ \ - } else { \ - tmp = osal_malloc(sizeof(TYPE) * length); \ - if (unlikely(!tmp)) \ - return false; \ - } \ - \ - size_t key_shift = 0, key_diff_mask; \ - do { \ - struct { \ - pgno_t a[256], b[256]; \ - } counters; \ - memset(&counters, 0, sizeof(counters)); \ - \ - key_diff_mask = 0; \ - size_t prev_key = EXTRACT_KEY(begin) >> key_shift; \ - TYPE *r = begin, *end = begin + length; \ - do { \ - const size_t key = EXTRACT_KEY(r) >> key_shift; \ - counters.a[key & 255]++; \ - counters.b[(key >> 8) & 255]++; \ - key_diff_mask |= prev_key ^ key; \ - prev_key = key; \ - } while (++r != end); \ - \ - pgno_t ta = 0, tb = 0; \ - for (size_t i = 0; i < 256; ++i) { \ - const pgno_t ia = counters.a[i]; \ - counters.a[i] = ta; \ - ta += ia; \ - const pgno_t ib = counters.b[i]; \ - counters.b[i] = tb; \ - tb += ib; \ - } \ - \ - r = begin; \ - do { \ - const size_t key = EXTRACT_KEY(r) >> key_shift; \ - tmp[counters.a[key & 255]++] = *r; \ - } while (++r != end); \ - \ - if (unlikely(key_diff_mask < 256)) { \ - memcpy(begin, tmp, ptr_dist(end, begin)); \ - break; \ - } \ - end = (r = tmp) + length; \ - do { \ - const size_t key = EXTRACT_KEY(r) >> key_shift; \ - begin[counters.b[(key >> 8) & 255]++] = *r; \ - } while (++r != end); \ - \ - key_shift += 16; \ - } while (key_diff_mask >> 16); \ - \ - if (!(BUFFER_PREALLOCATED)) \ - osal_free(tmp); \ - return true; \ +MDBX_INTERNAL void node_del(MDBX_cursor *mc, size_t ksize); + +MDBX_INTERNAL node_t *node_shrink(page_t *mp, size_t indx, node_t *node); + +#if MDBX_ENABLE_DBI_SPARSE + +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED MDBX_INTERNAL size_t dbi_bitmap_ctz_fallback(const MDBX_txn *txn, + intptr_t bmi); + +static inline size_t dbi_bitmap_ctz(const MDBX_txn *txn, intptr_t bmi) { + tASSERT(txn, bmi > 0); + STATIC_ASSERT(sizeof(bmi) >= sizeof(txn->dbi_sparse[0])); +#if __GNUC_PREREQ(4, 1) || __has_builtin(__builtin_ctzl) + if (sizeof(txn->dbi_sparse[0]) <= sizeof(int)) + return __builtin_ctz((int)bmi); + if (sizeof(txn->dbi_sparse[0]) == sizeof(long)) + return __builtin_ctzl((long)bmi); +#if (defined(__SIZEOF_LONG_LONG__) && __SIZEOF_LONG_LONG__ == 8) || __has_builtin(__builtin_ctzll) + return __builtin_ctzll(bmi); +#endif /* have(long long) && long long == uint64_t */ +#endif /* GNU C */ + +#if defined(_MSC_VER) + unsigned long index; + if (sizeof(txn->dbi_sparse[0]) > 4) { +#if defined(_M_AMD64) || defined(_M_ARM64) || defined(_M_X64) + _BitScanForward64(&index, bmi); + return index; +#else + if (bmi > UINT32_MAX) { + _BitScanForward(&index, (uint32_t)((uint64_t)bmi >> 32)); + return index; + } +#endif } + _BitScanForward(&index, (uint32_t)bmi); + return index; +#endif /* MSVC */ -/*------------------------------------------------------------------------------ - * LY: Binary search */ + return dbi_bitmap_ctz_fallback(txn, bmi); +} + +/* LY: Макрос целенаправленно сделан с одним циклом, чтобы сохранить возможность + * использования оператора break */ +#define TXN_FOREACH_DBI_FROM(TXN, I, FROM) \ + for (size_t bitmap_chunk = CHAR_BIT * sizeof(TXN->dbi_sparse[0]), bitmap_item = TXN->dbi_sparse[0] >> FROM, \ + I = FROM; \ + I < TXN->n_dbi; ++I) \ + if (bitmap_item == 0) { \ + I = (I - 1) | (bitmap_chunk - 1); \ + bitmap_item = TXN->dbi_sparse[(1 + I) / bitmap_chunk]; \ + if (!bitmap_item) \ + /* coverity[const_overflow] */ \ + I += bitmap_chunk; \ + continue; \ + } else if ((bitmap_item & 1) == 0) { \ + size_t bitmap_skip = dbi_bitmap_ctz(txn, bitmap_item); \ + bitmap_item >>= bitmap_skip; \ + I += bitmap_skip - 1; \ + continue; \ + } else if (bitmap_item >>= 1, TXN->dbi_state[I]) -#if defined(__clang__) && __clang_major__ > 4 && defined(__ia32__) -#define WORKAROUND_FOR_CLANG_OPTIMIZER_BUG(size, flag) \ - do \ - __asm __volatile("" \ - : "+r"(size) \ - : "r" /* the `b` constraint is more suitable here, but \ - cause CLANG to allocate and push/pop an one more \ - register, so using the `r` which avoids this. */ \ - (flag)); \ - while (0) #else -#define WORKAROUND_FOR_CLANG_OPTIMIZER_BUG(size, flag) \ - do { \ - /* nope for non-clang or non-x86 */; \ - } while (0) -#endif /* Workaround for CLANG */ -#define BINARY_SEARCH_STEP(TYPE_LIST, CMP, it, size, key) \ - do { \ - } while (0) +#define TXN_FOREACH_DBI_FROM(TXN, I, SKIP) \ + for (size_t I = SKIP; I < TXN->n_dbi; ++I) \ + if (TXN->dbi_state[I]) -#define SEARCH_IMPL(NAME, TYPE_LIST, TYPE_ARG, CMP) \ - static __always_inline const TYPE_LIST *NAME( \ - const TYPE_LIST *it, size_t length, const TYPE_ARG item) { \ - const TYPE_LIST *const begin = it, *const end = begin + length; \ - \ - if (MDBX_HAVE_CMOV) \ - do { \ - /* Адаптивно-упрощенный шаг двоичного поиска: \ - * - без переходов при наличии cmov или аналога; \ - * - допускает лишние итерации; \ - * - но ищет пока size > 2, что требует дозавершения поиска \ - * среди остающихся 0-1-2 элементов. */ \ - const TYPE_LIST *const middle = it + (length >> 1); \ - length = (length + 1) >> 1; \ - const bool flag = expect_with_probability(CMP(*middle, item), 0, .5); \ - WORKAROUND_FOR_CLANG_OPTIMIZER_BUG(length, flag); \ - it = flag ? middle : it; \ - } while (length > 2); \ - else \ - while (length > 2) { \ - /* Вариант с использованием условного перехода. Основное отличие в \ - * том, что при "не равно" (true от компаратора) переход делается на 1 \ - * ближе к концу массива. Алгоритмически это верно и обеспечивает \ - * чуть-чуть более быструю сходимость, но зато требует больше \ - * вычислений при true от компаратора. Также ВАЖНО(!) не допускается \ - * спекулятивное выполнение при size == 0. */ \ - const TYPE_LIST *const middle = it + (length >> 1); \ - length = (length + 1) >> 1; \ - const bool flag = expect_with_probability(CMP(*middle, item), 0, .5); \ - if (flag) { \ - it = middle + 1; \ - length -= 1; \ - } \ - } \ - it += length > 1 && expect_with_probability(CMP(*it, item), 0, .5); \ - it += length > 0 && expect_with_probability(CMP(*it, item), 0, .5); \ - \ - if (AUDIT_ENABLED()) { \ - for (const TYPE_LIST *scan = begin; scan < it; ++scan) \ - assert(CMP(*scan, item)); \ - for (const TYPE_LIST *scan = it; scan < end; ++scan) \ - assert(!CMP(*scan, item)); \ - (void)begin, (void)end; \ - } \ - \ - return it; \ - } +#endif /* MDBX_ENABLE_DBI_SPARSE */ -/*----------------------------------------------------------------------------*/ +#define TXN_FOREACH_DBI_ALL(TXN, I) TXN_FOREACH_DBI_FROM(TXN, I, 0) +#define TXN_FOREACH_DBI_USER(TXN, I) TXN_FOREACH_DBI_FROM(TXN, I, CORE_DBS) -static __always_inline size_t pnl_size2bytes(size_t size) { - assert(size > 0 && size <= MDBX_PGL_LIMIT); -#if MDBX_PNL_PREALLOC_FOR_RADIXSORT - size += size; -#endif /* MDBX_PNL_PREALLOC_FOR_RADIXSORT */ - STATIC_ASSERT(MDBX_ASSUME_MALLOC_OVERHEAD + - (MDBX_PGL_LIMIT * (MDBX_PNL_PREALLOC_FOR_RADIXSORT + 1) + - MDBX_PNL_GRANULATE + 3) * - sizeof(pgno_t) < - SIZE_MAX / 4 * 3); - size_t bytes = - ceil_powerof2(MDBX_ASSUME_MALLOC_OVERHEAD + sizeof(pgno_t) * (size + 3), - MDBX_PNL_GRANULATE * sizeof(pgno_t)) - - MDBX_ASSUME_MALLOC_OVERHEAD; - return bytes; -} +MDBX_INTERNAL int dbi_import(MDBX_txn *txn, const size_t dbi); -static __always_inline pgno_t pnl_bytes2size(const size_t bytes) { - size_t size = bytes / sizeof(pgno_t); - assert(size > 3 && size <= MDBX_PGL_LIMIT + /* alignment gap */ 65536); - size -= 3; -#if MDBX_PNL_PREALLOC_FOR_RADIXSORT - size >>= 1; -#endif /* MDBX_PNL_PREALLOC_FOR_RADIXSORT */ - return (pgno_t)size; +struct dbi_snap_result { + uint32_t sequence; + unsigned flags; +}; +MDBX_INTERNAL struct dbi_snap_result dbi_snap(const MDBX_env *env, const size_t dbi); + +MDBX_INTERNAL int dbi_update(MDBX_txn *txn, int keep); + +static inline uint8_t dbi_state(const MDBX_txn *txn, const size_t dbi) { + STATIC_ASSERT((int)DBI_DIRTY == MDBX_DBI_DIRTY && (int)DBI_STALE == MDBX_DBI_STALE && + (int)DBI_FRESH == MDBX_DBI_FRESH && (int)DBI_CREAT == MDBX_DBI_CREAT); + +#if MDBX_ENABLE_DBI_SPARSE + const size_t bitmap_chunk = CHAR_BIT * sizeof(txn->dbi_sparse[0]); + const size_t bitmap_indx = dbi / bitmap_chunk; + const size_t bitmap_mask = (size_t)1 << dbi % bitmap_chunk; + return likely(dbi < txn->n_dbi && (txn->dbi_sparse[bitmap_indx] & bitmap_mask) != 0) ? txn->dbi_state[dbi] : 0; +#else + return likely(dbi < txn->n_dbi) ? txn->dbi_state[dbi] : 0; +#endif /* MDBX_ENABLE_DBI_SPARSE */ } -static MDBX_PNL pnl_alloc(size_t size) { - size_t bytes = pnl_size2bytes(size); - MDBX_PNL pl = osal_malloc(bytes); - if (likely(pl)) { -#if __GLIBC_PREREQ(2, 12) || defined(__FreeBSD__) || defined(malloc_usable_size) - bytes = malloc_usable_size(pl); -#endif /* malloc_usable_size */ - pl[0] = pnl_bytes2size(bytes); - assert(pl[0] >= size); - pl += 1; - *pl = 0; - } - return pl; +static inline bool dbi_changed(const MDBX_txn *txn, const size_t dbi) { + const MDBX_env *const env = txn->env; + eASSERT(env, dbi_state(txn, dbi) & DBI_LINDO); + const uint32_t snap_seq = atomic_load32(&env->dbi_seqs[dbi], mo_AcquireRelease); + return unlikely(snap_seq != txn->dbi_seqs[dbi]); } -static void pnl_free(MDBX_PNL pl) { - if (likely(pl)) - osal_free(pl - 1); +static inline int dbi_check(const MDBX_txn *txn, const size_t dbi) { + const uint8_t state = dbi_state(txn, dbi); + if (likely((state & DBI_LINDO) != 0 && !dbi_changed(txn, dbi))) + return (state & DBI_VALID) ? MDBX_SUCCESS : MDBX_BAD_DBI; + + /* Медленный путь: ленивая до-инициализацяи и импорт */ + return dbi_import((MDBX_txn *)txn, dbi); } -/* Shrink the PNL to the default size if it has grown larger */ -static void pnl_shrink(MDBX_PNL *ppl) { - assert(pnl_bytes2size(pnl_size2bytes(MDBX_PNL_INITIAL)) >= MDBX_PNL_INITIAL && - pnl_bytes2size(pnl_size2bytes(MDBX_PNL_INITIAL)) < - MDBX_PNL_INITIAL * 3 / 2); - assert(MDBX_PNL_GETSIZE(*ppl) <= MDBX_PGL_LIMIT && - MDBX_PNL_ALLOCLEN(*ppl) >= MDBX_PNL_GETSIZE(*ppl)); - MDBX_PNL_SETSIZE(*ppl, 0); - if (unlikely(MDBX_PNL_ALLOCLEN(*ppl) > - MDBX_PNL_INITIAL * (MDBX_PNL_PREALLOC_FOR_RADIXSORT ? 8 : 4) - - MDBX_CACHELINE_SIZE / sizeof(pgno_t))) { - size_t bytes = pnl_size2bytes(MDBX_PNL_INITIAL * 2); - MDBX_PNL pl = osal_realloc(*ppl - 1, bytes); - if (likely(pl)) { -#if __GLIBC_PREREQ(2, 12) || defined(__FreeBSD__) || defined(malloc_usable_size) - bytes = malloc_usable_size(pl); -#endif /* malloc_usable_size */ - *pl = pnl_bytes2size(bytes); - *ppl = pl + 1; - } - } +static inline uint32_t dbi_seq_next(const MDBX_env *const env, size_t dbi) { + uint32_t v = atomic_load32(&env->dbi_seqs[dbi], mo_AcquireRelease) + 1; + return v ? v : 1; } -/* Grow the PNL to the size growed to at least given size */ -static int pnl_reserve(MDBX_PNL *ppl, const size_t wanna) { - const size_t allocated = MDBX_PNL_ALLOCLEN(*ppl); - assert(MDBX_PNL_GETSIZE(*ppl) <= MDBX_PGL_LIMIT && - MDBX_PNL_ALLOCLEN(*ppl) >= MDBX_PNL_GETSIZE(*ppl)); - if (likely(allocated >= wanna)) - return MDBX_SUCCESS; +MDBX_INTERNAL int dbi_open(MDBX_txn *txn, const MDBX_val *const name, unsigned user_flags, MDBX_dbi *dbi, + MDBX_cmp_func *keycmp, MDBX_cmp_func *datacmp); - if (unlikely(wanna > /* paranoia */ MDBX_PGL_LIMIT)) { - ERROR("PNL too long (%zu > %zu)", wanna, (size_t)MDBX_PGL_LIMIT); - return MDBX_TXN_FULL; +MDBX_INTERNAL int dbi_bind(MDBX_txn *txn, const size_t dbi, unsigned user_flags, MDBX_cmp_func *keycmp, + MDBX_cmp_func *datacmp); + +typedef struct defer_free_item { + struct defer_free_item *next; + uint64_t timestamp; +} defer_free_item_t; + +MDBX_INTERNAL int dbi_defer_release(MDBX_env *const env, defer_free_item_t *const chain); +MDBX_INTERNAL int dbi_close_release(MDBX_env *env, MDBX_dbi dbi); +MDBX_INTERNAL const tree_t *dbi_dig(const MDBX_txn *txn, const size_t dbi, tree_t *fallback); + +struct dbi_rename_result { + defer_free_item_t *defer; + int err; +}; + +MDBX_INTERNAL struct dbi_rename_result dbi_rename_locked(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val new_name); + +MDBX_NOTHROW_CONST_FUNCTION MDBX_INTERNAL pgno_t pv2pages(uint16_t pv); + +MDBX_NOTHROW_CONST_FUNCTION MDBX_INTERNAL uint16_t pages2pv(size_t pages); + +MDBX_MAYBE_UNUSED MDBX_INTERNAL bool pv2pages_verify(void); + +/*------------------------------------------------------------------------------ + * Nodes, Keys & Values length limitation factors: + * + * BRANCH_NODE_MAX + * Branch-page must contain at least two nodes, within each a key and a child + * page number. But page can't be split if it contains less that 4 keys, + * i.e. a page should not overflow before adding the fourth key. Therefore, + * at least 3 branch-node should fit in the single branch-page. Further, the + * first node of a branch-page doesn't contain a key, i.e. the first node + * is always require space just for itself. Thus: + * PAGESPACE = pagesize - page_hdr_len; + * BRANCH_NODE_MAX = even_floor( + * (PAGESPACE - sizeof(indx_t) - NODESIZE) / (3 - 1) - sizeof(indx_t)); + * KEYLEN_MAX = BRANCH_NODE_MAX - node_hdr_len; + * + * LEAF_NODE_MAX + * Leaf-node must fit into single leaf-page, where a value could be placed on + * a large/overflow page. However, may require to insert a nearly page-sized + * node between two large nodes are already fill-up a page. In this case the + * page must be split to two if some pair of nodes fits on one page, or + * otherwise the page should be split to the THREE with a single node + * per each of ones. Such 1-into-3 page splitting is costly and complex since + * requires TWO insertion into the parent page, that could lead to split it + * and so on up to the root. Therefore double-splitting is avoided here and + * the maximum node size is half of a leaf page space: + * LEAF_NODE_MAX = even_floor(PAGESPACE / 2 - sizeof(indx_t)); + * DATALEN_NO_OVERFLOW = LEAF_NODE_MAX - NODESIZE - KEYLEN_MAX; + * + * - Table-node must fit into one leaf-page: + * TABLE_NAME_MAX = LEAF_NODE_MAX - node_hdr_len - sizeof(tree_t); + * + * - Dupsort values itself are a keys in a dupsort-table and couldn't be longer + * than the KEYLEN_MAX. But dupsort node must not great than LEAF_NODE_MAX, + * since dupsort value couldn't be placed on a large/overflow page: + * DUPSORT_DATALEN_MAX = min(KEYLEN_MAX, + * max(DATALEN_NO_OVERFLOW, sizeof(tree_t)); + */ + +#define PAGESPACE(pagesize) ((pagesize) - PAGEHDRSZ) + +#define BRANCH_NODE_MAX(pagesize) \ + (EVEN_FLOOR((PAGESPACE(pagesize) - sizeof(indx_t) - NODESIZE) / (3 - 1) - sizeof(indx_t))) + +#define LEAF_NODE_MAX(pagesize) (EVEN_FLOOR(PAGESPACE(pagesize) / 2) - sizeof(indx_t)) + +#define MAX_GC1OVPAGE(pagesize) (PAGESPACE(pagesize) / sizeof(pgno_t) - 1) + +MDBX_NOTHROW_CONST_FUNCTION static inline size_t keysize_max(size_t pagesize, MDBX_db_flags_t flags) { + assert(pagesize >= MDBX_MIN_PAGESIZE && pagesize <= MDBX_MAX_PAGESIZE && is_powerof2(pagesize)); + STATIC_ASSERT(BRANCH_NODE_MAX(MDBX_MIN_PAGESIZE) - NODESIZE >= 8); + if (flags & MDBX_INTEGERKEY) + return 8 /* sizeof(uint64_t) */; + + const intptr_t max_branch_key = BRANCH_NODE_MAX(pagesize) - NODESIZE; + STATIC_ASSERT(LEAF_NODE_MAX(MDBX_MIN_PAGESIZE) - NODESIZE - + /* sizeof(uint64) as a key */ 8 > + sizeof(tree_t)); + if (flags & (MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_REVERSEDUP | MDBX_INTEGERDUP)) { + const intptr_t max_dupsort_leaf_key = LEAF_NODE_MAX(pagesize) - NODESIZE - sizeof(tree_t); + return (max_branch_key < max_dupsort_leaf_key) ? max_branch_key : max_dupsort_leaf_key; } + return max_branch_key; +} - const size_t size = (wanna + wanna - allocated < MDBX_PGL_LIMIT) - ? wanna + wanna - allocated - : MDBX_PGL_LIMIT; - size_t bytes = pnl_size2bytes(size); - MDBX_PNL pl = osal_realloc(*ppl - 1, bytes); - if (likely(pl)) { -#if __GLIBC_PREREQ(2, 12) || defined(__FreeBSD__) || defined(malloc_usable_size) - bytes = malloc_usable_size(pl); -#endif /* malloc_usable_size */ - *pl = pnl_bytes2size(bytes); - assert(*pl >= wanna); - *ppl = pl + 1; - return MDBX_SUCCESS; +MDBX_NOTHROW_CONST_FUNCTION static inline size_t env_keysize_max(const MDBX_env *env, MDBX_db_flags_t flags) { + size_t size_max; + if (flags & MDBX_INTEGERKEY) + size_max = 8 /* sizeof(uint64_t) */; + else { + const intptr_t max_branch_key = env->branch_nodemax - NODESIZE; + STATIC_ASSERT(LEAF_NODE_MAX(MDBX_MIN_PAGESIZE) - NODESIZE - + /* sizeof(uint64) as a key */ 8 > + sizeof(tree_t)); + if (flags & (MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_REVERSEDUP | MDBX_INTEGERDUP)) { + const intptr_t max_dupsort_leaf_key = env->leaf_nodemax - NODESIZE - sizeof(tree_t); + size_max = (max_branch_key < max_dupsort_leaf_key) ? max_branch_key : max_dupsort_leaf_key; + } else + size_max = max_branch_key; } - return MDBX_ENOMEM; + eASSERT(env, size_max == keysize_max(env->ps, flags)); + return size_max; } -/* Make room for num additional elements in an PNL */ -static __always_inline int __must_check_result pnl_need(MDBX_PNL *ppl, - size_t num) { - assert(MDBX_PNL_GETSIZE(*ppl) <= MDBX_PGL_LIMIT && - MDBX_PNL_ALLOCLEN(*ppl) >= MDBX_PNL_GETSIZE(*ppl)); - assert(num <= MDBX_PGL_LIMIT); - const size_t wanna = MDBX_PNL_GETSIZE(*ppl) + num; - return likely(MDBX_PNL_ALLOCLEN(*ppl) >= wanna) ? MDBX_SUCCESS - : pnl_reserve(ppl, wanna); +MDBX_NOTHROW_CONST_FUNCTION static inline size_t keysize_min(MDBX_db_flags_t flags) { + return (flags & MDBX_INTEGERKEY) ? 4 /* sizeof(uint32_t) */ : 0; } -static __always_inline void pnl_xappend(MDBX_PNL pl, pgno_t pgno) { - assert(MDBX_PNL_GETSIZE(pl) < MDBX_PNL_ALLOCLEN(pl)); - if (AUDIT_ENABLED()) { - for (size_t i = MDBX_PNL_GETSIZE(pl); i > 0; --i) - assert(pgno != pl[i]); - } - *pl += 1; - MDBX_PNL_LAST(pl) = pgno; +MDBX_NOTHROW_CONST_FUNCTION static inline size_t valsize_min(MDBX_db_flags_t flags) { + if (flags & MDBX_INTEGERDUP) + return 4 /* sizeof(uint32_t) */; + else if (flags & MDBX_DUPFIXED) + return sizeof(indx_t); + else + return 0; } -/* Append an pgno range onto an unsorted PNL */ -__always_inline static int __must_check_result pnl_append_range(bool spilled, - MDBX_PNL *ppl, - pgno_t pgno, - size_t n) { - assert(n > 0); - int rc = pnl_need(ppl, n); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; +MDBX_NOTHROW_CONST_FUNCTION static inline size_t valsize_max(size_t pagesize, MDBX_db_flags_t flags) { + assert(pagesize >= MDBX_MIN_PAGESIZE && pagesize <= MDBX_MAX_PAGESIZE && is_powerof2(pagesize)); - const MDBX_PNL pnl = *ppl; -#if MDBX_PNL_ASCENDING - size_t w = MDBX_PNL_GETSIZE(pnl); - do { - pnl[++w] = pgno; - pgno += spilled ? 2 : 1; - } while (--n); - MDBX_PNL_SETSIZE(pnl, w); -#else - size_t w = MDBX_PNL_GETSIZE(pnl) + n; - MDBX_PNL_SETSIZE(pnl, w); - do { - pnl[w--] = pgno; - pgno += spilled ? 2 : 1; - } while (--n); -#endif + if (flags & MDBX_INTEGERDUP) + return 8 /* sizeof(uint64_t) */; - return MDBX_SUCCESS; + if (flags & (MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_REVERSEDUP)) + return keysize_max(pagesize, 0); + + const unsigned page_ln2 = log2n_powerof2(pagesize); + const size_t hard = 0x7FF00000ul; + const size_t hard_pages = hard >> page_ln2; + STATIC_ASSERT(PAGELIST_LIMIT <= MAX_PAGENO); + const size_t pages_limit = PAGELIST_LIMIT / 4; + const size_t limit = (hard_pages < pages_limit) ? hard : (pages_limit << page_ln2); + return (limit < MAX_MAPSIZE / 2) ? limit : MAX_MAPSIZE / 2; } -/* Append an pgno range into the sorted PNL */ -__hot static int __must_check_result pnl_insert_range(MDBX_PNL *ppl, - pgno_t pgno, size_t n) { - assert(n > 0); - int rc = pnl_need(ppl, n); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; +MDBX_NOTHROW_CONST_FUNCTION static inline size_t env_valsize_max(const MDBX_env *env, MDBX_db_flags_t flags) { + size_t size_max; + if (flags & MDBX_INTEGERDUP) + size_max = 8 /* sizeof(uint64_t) */; + else if (flags & (MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_REVERSEDUP)) + size_max = env_keysize_max(env, 0); + else { + const size_t hard = 0x7FF00000ul; + const size_t hard_pages = hard >> env->ps2ln; + STATIC_ASSERT(PAGELIST_LIMIT <= MAX_PAGENO); + const size_t pages_limit = PAGELIST_LIMIT / 4; + const size_t limit = (hard_pages < pages_limit) ? hard : (pages_limit << env->ps2ln); + size_max = (limit < MAX_MAPSIZE / 2) ? limit : MAX_MAPSIZE / 2; + } + eASSERT(env, size_max == valsize_max(env->ps, flags)); + return size_max; +} - const MDBX_PNL pnl = *ppl; - size_t r = MDBX_PNL_GETSIZE(pnl), w = r + n; - MDBX_PNL_SETSIZE(pnl, w); - while (r && MDBX_PNL_DISORDERED(pnl[r], pgno)) - pnl[w--] = pnl[r--]; +/*----------------------------------------------------------------------------*/ - for (pgno_t fill = MDBX_PNL_ASCENDING ? pgno + n : pgno; w > r; --w) - pnl[w] = MDBX_PNL_ASCENDING ? --fill : fill++; +MDBX_NOTHROW_PURE_FUNCTION static inline size_t leaf_size(const MDBX_env *env, const MDBX_val *key, + const MDBX_val *data) { + size_t node_bytes = node_size(key, data); + if (node_bytes > env->leaf_nodemax) + /* put on large/overflow page */ + node_bytes = node_size_len(key->iov_len, 0) + sizeof(pgno_t); - return MDBX_SUCCESS; + return node_bytes + sizeof(indx_t); } -__hot static bool pnl_check(const pgno_t *pl, const size_t limit) { - assert(limit >= MIN_PAGENO - MDBX_ENABLE_REFUND); - if (likely(MDBX_PNL_GETSIZE(pl))) { - if (unlikely(MDBX_PNL_GETSIZE(pl) > MDBX_PGL_LIMIT)) - return false; - if (unlikely(MDBX_PNL_LEAST(pl) < MIN_PAGENO)) - return false; - if (unlikely(MDBX_PNL_MOST(pl) >= limit)) - return false; - - if ((!MDBX_DISABLE_VALIDATION || AUDIT_ENABLED()) && - likely(MDBX_PNL_GETSIZE(pl) > 1)) { - const pgno_t *scan = MDBX_PNL_BEGIN(pl); - const pgno_t *const end = MDBX_PNL_END(pl); - pgno_t prev = *scan++; - do { - if (unlikely(!MDBX_PNL_ORDERED(prev, *scan))) - return false; - prev = *scan; - } while (likely(++scan != end)); - } +MDBX_NOTHROW_PURE_FUNCTION static inline size_t branch_size(const MDBX_env *env, const MDBX_val *key) { + /* Size of a node in a branch page with a given key. + * This is just the node header plus the key, there is no data. */ + size_t node_bytes = node_size(key, nullptr); + if (unlikely(node_bytes > env->branch_nodemax)) { + /* put on large/overflow page, not implemented */ + mdbx_panic("node_size(key) %zu > %u branch_nodemax", node_bytes, env->branch_nodemax); + node_bytes = node_size(key, nullptr) + sizeof(pgno_t); } - return true; -} -static __always_inline bool pnl_check_allocated(const pgno_t *pl, - const size_t limit) { - return pl == nullptr || (MDBX_PNL_ALLOCLEN(pl) >= MDBX_PNL_GETSIZE(pl) && - pnl_check(pl, limit)); + return node_bytes + sizeof(indx_t); } -static __always_inline void -pnl_merge_inner(pgno_t *__restrict dst, const pgno_t *__restrict src_a, - const pgno_t *__restrict src_b, - const pgno_t *__restrict const src_b_detent) { - do { -#if MDBX_HAVE_CMOV - const bool flag = MDBX_PNL_ORDERED(*src_b, *src_a); -#if defined(__LCC__) || __CLANG_PREREQ(13, 0) - // lcc 1.26: 13ШК (подготовка и первая итерация) + 7ШК (цикл), БЕЗ loop-mode - // gcc>=7: cmp+jmp с возвратом в тело цикла (WTF?) - // gcc<=6: cmov×3 - // clang<=12: cmov×3 - // clang>=13: cmov, set+add/sub - *dst = flag ? *src_a-- : *src_b--; -#else - // gcc: cmov, cmp+set+add/sub - // clang<=5: cmov×2, set+add/sub - // clang>=6: cmov, set+add/sub - *dst = flag ? *src_a : *src_b; - src_b += (ptrdiff_t)flag - 1; - src_a -= flag; -#endif - --dst; -#else /* MDBX_HAVE_CMOV */ - while (MDBX_PNL_ORDERED(*src_b, *src_a)) - *dst-- = *src_a--; - *dst-- = *src_b--; -#endif /* !MDBX_HAVE_CMOV */ - } while (likely(src_b > src_b_detent)); -} +MDBX_NOTHROW_CONST_FUNCTION static inline uint16_t flags_db2sub(uint16_t db_flags) { + uint16_t sub_flags = db_flags & MDBX_DUPFIXED; -/* Merge a PNL onto a PNL. The destination PNL must be big enough */ -__hot static size_t pnl_merge(MDBX_PNL dst, const MDBX_PNL src) { - assert(pnl_check_allocated(dst, MAX_PAGENO + 1)); - assert(pnl_check(src, MAX_PAGENO + 1)); - const size_t src_len = MDBX_PNL_GETSIZE(src); - const size_t dst_len = MDBX_PNL_GETSIZE(dst); - size_t total = dst_len; - assert(MDBX_PNL_ALLOCLEN(dst) >= total); - if (likely(src_len > 0)) { - total += src_len; - if (!MDBX_DEBUG && total < (MDBX_HAVE_CMOV ? 21 : 12)) - goto avoid_call_libc_for_short_cases; - if (dst_len == 0 || - MDBX_PNL_ORDERED(MDBX_PNL_LAST(dst), MDBX_PNL_FIRST(src))) - memcpy(MDBX_PNL_END(dst), MDBX_PNL_BEGIN(src), src_len * sizeof(pgno_t)); - else if (MDBX_PNL_ORDERED(MDBX_PNL_LAST(src), MDBX_PNL_FIRST(dst))) { - memmove(MDBX_PNL_BEGIN(dst) + src_len, MDBX_PNL_BEGIN(dst), - dst_len * sizeof(pgno_t)); - memcpy(MDBX_PNL_BEGIN(dst), MDBX_PNL_BEGIN(src), - src_len * sizeof(pgno_t)); - } else { - avoid_call_libc_for_short_cases: - dst[0] = /* the detent */ (MDBX_PNL_ASCENDING ? 0 : P_INVALID); - pnl_merge_inner(dst + total, dst + dst_len, src + src_len, src); - } - MDBX_PNL_SETSIZE(dst, total); - } - assert(pnl_check_allocated(dst, MAX_PAGENO + 1)); - return total; -} + /* MDBX_INTEGERDUP => MDBX_INTEGERKEY */ +#define SHIFT_INTEGERDUP_TO_INTEGERKEY 2 + STATIC_ASSERT((MDBX_INTEGERDUP >> SHIFT_INTEGERDUP_TO_INTEGERKEY) == MDBX_INTEGERKEY); + sub_flags |= (db_flags & MDBX_INTEGERDUP) >> SHIFT_INTEGERDUP_TO_INTEGERKEY; -static void spill_remove(MDBX_txn *txn, size_t idx, size_t npages) { - tASSERT(txn, idx > 0 && idx <= MDBX_PNL_GETSIZE(txn->tw.spilled.list) && - txn->tw.spilled.least_removed > 0); - txn->tw.spilled.least_removed = (idx < txn->tw.spilled.least_removed) - ? idx - : txn->tw.spilled.least_removed; - txn->tw.spilled.list[idx] |= 1; - MDBX_PNL_SETSIZE(txn->tw.spilled.list, - MDBX_PNL_GETSIZE(txn->tw.spilled.list) - - (idx == MDBX_PNL_GETSIZE(txn->tw.spilled.list))); + /* MDBX_REVERSEDUP => MDBX_REVERSEKEY */ +#define SHIFT_REVERSEDUP_TO_REVERSEKEY 5 + STATIC_ASSERT((MDBX_REVERSEDUP >> SHIFT_REVERSEDUP_TO_REVERSEKEY) == MDBX_REVERSEKEY); + sub_flags |= (db_flags & MDBX_REVERSEDUP) >> SHIFT_REVERSEDUP_TO_REVERSEKEY; - while (unlikely(npages > 1)) { - const pgno_t pgno = (txn->tw.spilled.list[idx] >> 1) + 1; - if (MDBX_PNL_ASCENDING) { - if (++idx > MDBX_PNL_GETSIZE(txn->tw.spilled.list) || - (txn->tw.spilled.list[idx] >> 1) != pgno) - return; - } else { - if (--idx < 1 || (txn->tw.spilled.list[idx] >> 1) != pgno) - return; - txn->tw.spilled.least_removed = (idx < txn->tw.spilled.least_removed) - ? idx - : txn->tw.spilled.least_removed; - } - txn->tw.spilled.list[idx] |= 1; - MDBX_PNL_SETSIZE(txn->tw.spilled.list, - MDBX_PNL_GETSIZE(txn->tw.spilled.list) - - (idx == MDBX_PNL_GETSIZE(txn->tw.spilled.list))); - --npages; - } + return sub_flags; } -static MDBX_PNL spill_purge(MDBX_txn *txn) { - tASSERT(txn, txn->tw.spilled.least_removed > 0); - const MDBX_PNL sl = txn->tw.spilled.list; - if (txn->tw.spilled.least_removed != INT_MAX) { - size_t len = MDBX_PNL_GETSIZE(sl), r, w; - for (w = r = txn->tw.spilled.least_removed; r <= len; ++r) { - sl[w] = sl[r]; - w += 1 - (sl[r] & 1); - } - for (size_t i = 1; i < w; ++i) - tASSERT(txn, (sl[i] & 1) == 0); - MDBX_PNL_SETSIZE(sl, w - 1); - txn->tw.spilled.least_removed = INT_MAX; - } else { - for (size_t i = 1; i <= MDBX_PNL_GETSIZE(sl); ++i) - tASSERT(txn, (sl[i] & 1) == 0); +static inline bool check_table_flags(unsigned flags) { + switch (flags & ~(MDBX_REVERSEKEY | MDBX_INTEGERKEY)) { + default: + NOTICE("invalid db-flags 0x%x", flags); + return false; + case MDBX_DUPSORT: + case MDBX_DUPSORT | MDBX_REVERSEDUP: + case MDBX_DUPSORT | MDBX_DUPFIXED: + case MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_REVERSEDUP: + case MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERDUP: + case MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERDUP | MDBX_REVERSEDUP: + case MDBX_DB_DEFAULTS: + return (flags & (MDBX_REVERSEKEY | MDBX_INTEGERKEY)) != (MDBX_REVERSEKEY | MDBX_INTEGERKEY); } - return sl; -} - -#if MDBX_PNL_ASCENDING -#define MDBX_PNL_EXTRACT_KEY(ptr) (*(ptr)) -#else -#define MDBX_PNL_EXTRACT_KEY(ptr) (P_INVALID - *(ptr)) -#endif -RADIXSORT_IMPL(pgno, pgno_t, MDBX_PNL_EXTRACT_KEY, - MDBX_PNL_PREALLOC_FOR_RADIXSORT, 0) - -SORT_IMPL(pgno_sort, false, pgno_t, MDBX_PNL_ORDERED) - -__hot __noinline static void pnl_sort_nochk(MDBX_PNL pnl) { - if (likely(MDBX_PNL_GETSIZE(pnl) < MDBX_RADIXSORT_THRESHOLD) || - unlikely(!pgno_radixsort(&MDBX_PNL_FIRST(pnl), MDBX_PNL_GETSIZE(pnl)))) - pgno_sort(MDBX_PNL_BEGIN(pnl), MDBX_PNL_END(pnl)); } -static __inline void pnl_sort(MDBX_PNL pnl, size_t limit4check) { - pnl_sort_nochk(pnl); - assert(pnl_check(pnl, limit4check)); - (void)limit4check; +static inline int tbl_setup_ifneed(const MDBX_env *env, volatile kvx_t *const kvx, const tree_t *const db) { + return likely(kvx->clc.v.lmax) ? MDBX_SUCCESS : tbl_setup(env, kvx, db); } -/* Search for an pgno in an PNL. - * Returns The index of the first item greater than or equal to pgno. */ -SEARCH_IMPL(pgno_bsearch, pgno_t, pgno_t, MDBX_PNL_ORDERED) +/*----------------------------------------------------------------------------*/ -__hot __noinline static size_t pnl_search_nochk(const MDBX_PNL pnl, - pgno_t pgno) { - const pgno_t *begin = MDBX_PNL_BEGIN(pnl); - const pgno_t *it = pgno_bsearch(begin, MDBX_PNL_GETSIZE(pnl), pgno); - const pgno_t *end = begin + MDBX_PNL_GETSIZE(pnl); - assert(it >= begin && it <= end); - if (it != begin) - assert(MDBX_PNL_ORDERED(it[-1], pgno)); - if (it != end) - assert(!MDBX_PNL_ORDERED(it[0], pgno)); - return it - begin + 1; +MDBX_NOTHROW_PURE_FUNCTION static inline size_t pgno2bytes(const MDBX_env *env, size_t pgno) { + eASSERT(env, (1u << env->ps2ln) == env->ps); + return ((size_t)pgno) << env->ps2ln; } -static __inline size_t pnl_search(const MDBX_PNL pnl, pgno_t pgno, - size_t limit) { - assert(pnl_check_allocated(pnl, limit)); - if (MDBX_HAVE_CMOV) { - /* cmov-ускоренный бинарный поиск может читать (но не использовать) один - * элемент за концом данных, этот элемент в пределах выделенного участка - * памяти, но не инициализирован. */ - VALGRIND_MAKE_MEM_DEFINED(MDBX_PNL_END(pnl), sizeof(pgno_t)); - } - assert(pgno < limit); - (void)limit; - size_t n = pnl_search_nochk(pnl, pgno); - if (MDBX_HAVE_CMOV) { - VALGRIND_MAKE_MEM_UNDEFINED(MDBX_PNL_END(pnl), sizeof(pgno_t)); - } - return n; +MDBX_NOTHROW_PURE_FUNCTION static inline page_t *pgno2page(const MDBX_env *env, size_t pgno) { + return ptr_disp(env->dxb_mmap.base, pgno2bytes(env, pgno)); } -static __inline size_t search_spilled(const MDBX_txn *txn, pgno_t pgno) { - tASSERT(txn, (txn->mt_flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); - const MDBX_PNL pnl = txn->tw.spilled.list; - if (likely(!pnl)) - return 0; - pgno <<= 1; - size_t n = pnl_search(pnl, pgno, (size_t)MAX_PAGENO + MAX_PAGENO + 1); - return (n <= MDBX_PNL_GETSIZE(pnl) && pnl[n] == pgno) ? n : 0; +MDBX_NOTHROW_PURE_FUNCTION static inline pgno_t bytes2pgno(const MDBX_env *env, size_t bytes) { + eASSERT(env, (env->ps >> env->ps2ln) == 1); + return (pgno_t)(bytes >> env->ps2ln); } -static __inline bool intersect_spilled(const MDBX_txn *txn, pgno_t pgno, - size_t npages) { - const MDBX_PNL pnl = txn->tw.spilled.list; - if (likely(!pnl)) - return false; - const size_t len = MDBX_PNL_GETSIZE(pnl); - if (LOG_ENABLED(MDBX_LOG_EXTRA)) { - DEBUG_EXTRA("PNL len %zu [", len); - for (size_t i = 1; i <= len; ++i) - DEBUG_EXTRA_PRINT(" %li", (pnl[i] & 1) ? -(long)(pnl[i] >> 1) - : (long)(pnl[i] >> 1)); - DEBUG_EXTRA_PRINT("%s\n", "]"); - } - const pgno_t spilled_range_begin = pgno << 1; - const pgno_t spilled_range_last = ((pgno + (pgno_t)npages) << 1) - 1; -#if MDBX_PNL_ASCENDING - const size_t n = - pnl_search(pnl, spilled_range_begin, (size_t)(MAX_PAGENO + 1) << 1); - assert(n && - (n == MDBX_PNL_GETSIZE(pnl) + 1 || spilled_range_begin <= pnl[n])); - const bool rc = n <= MDBX_PNL_GETSIZE(pnl) && pnl[n] <= spilled_range_last; -#else - const size_t n = - pnl_search(pnl, spilled_range_last, (size_t)MAX_PAGENO + MAX_PAGENO + 1); - assert(n && (n == MDBX_PNL_GETSIZE(pnl) + 1 || spilled_range_last >= pnl[n])); - const bool rc = n <= MDBX_PNL_GETSIZE(pnl) && pnl[n] >= spilled_range_begin; -#endif - if (ASSERT_ENABLED()) { - bool check = false; - for (size_t i = 0; i < npages; ++i) - check |= search_spilled(txn, (pgno_t)(pgno + i)) != 0; - assert(check == rc); - } - return rc; -} +MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL size_t bytes_align2os_bytes(const MDBX_env *env, size_t bytes); -/*----------------------------------------------------------------------------*/ +MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL size_t pgno_align2os_bytes(const MDBX_env *env, size_t pgno); -static __always_inline size_t txl_size2bytes(const size_t size) { - assert(size > 0 && size <= MDBX_TXL_MAX * 2); - size_t bytes = - ceil_powerof2(MDBX_ASSUME_MALLOC_OVERHEAD + sizeof(txnid_t) * (size + 2), - MDBX_TXL_GRANULATE * sizeof(txnid_t)) - - MDBX_ASSUME_MALLOC_OVERHEAD; - return bytes; +MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL pgno_t pgno_align2os_pgno(const MDBX_env *env, size_t pgno); + +MDBX_NOTHROW_PURE_FUNCTION static inline pgno_t largechunk_npages(const MDBX_env *env, size_t bytes) { + return bytes2pgno(env, PAGEHDRSZ - 1 + bytes) + 1; } -static __always_inline size_t txl_bytes2size(const size_t bytes) { - size_t size = bytes / sizeof(txnid_t); - assert(size > 2 && size <= MDBX_TXL_MAX * 2); - return size - 2; +MDBX_NOTHROW_PURE_FUNCTION static inline MDBX_val get_key(const node_t *node) { + MDBX_val key; + key.iov_len = node_ks(node); + key.iov_base = node_key(node); + return key; } -static MDBX_TXL txl_alloc(void) { - size_t bytes = txl_size2bytes(MDBX_TXL_INITIAL); - MDBX_TXL tl = osal_malloc(bytes); - if (likely(tl)) { -#if __GLIBC_PREREQ(2, 12) || defined(__FreeBSD__) || defined(malloc_usable_size) - bytes = malloc_usable_size(tl); -#endif /* malloc_usable_size */ - tl[0] = txl_bytes2size(bytes); - assert(tl[0] >= MDBX_TXL_INITIAL); - tl += 1; - *tl = 0; - } - return tl; +static inline void get_key_optional(const node_t *node, MDBX_val *keyptr /* __may_null */) { + if (keyptr) + *keyptr = get_key(node); } -static void txl_free(MDBX_TXL tl) { - if (likely(tl)) - osal_free(tl - 1); +MDBX_NOTHROW_PURE_FUNCTION static inline void *page_data(const page_t *mp) { return ptr_disp(mp, PAGEHDRSZ); } + +MDBX_NOTHROW_PURE_FUNCTION static inline const page_t *data_page(const void *data) { + return container_of(data, page_t, entries); } -static int txl_reserve(MDBX_TXL *ptl, const size_t wanna) { - const size_t allocated = (size_t)MDBX_PNL_ALLOCLEN(*ptl); - assert(MDBX_PNL_GETSIZE(*ptl) <= MDBX_TXL_MAX && - MDBX_PNL_ALLOCLEN(*ptl) >= MDBX_PNL_GETSIZE(*ptl)); - if (likely(allocated >= wanna)) - return MDBX_SUCCESS; +MDBX_NOTHROW_PURE_FUNCTION static inline meta_t *page_meta(page_t *mp) { return (meta_t *)page_data(mp); } - if (unlikely(wanna > /* paranoia */ MDBX_TXL_MAX)) { - ERROR("TXL too long (%zu > %zu)", wanna, (size_t)MDBX_TXL_MAX); - return MDBX_TXN_FULL; - } +MDBX_NOTHROW_PURE_FUNCTION static inline size_t page_numkeys(const page_t *mp) { return mp->lower >> 1; } - const size_t size = (wanna + wanna - allocated < MDBX_TXL_MAX) - ? wanna + wanna - allocated - : MDBX_TXL_MAX; - size_t bytes = txl_size2bytes(size); - MDBX_TXL tl = osal_realloc(*ptl - 1, bytes); - if (likely(tl)) { -#if __GLIBC_PREREQ(2, 12) || defined(__FreeBSD__) || defined(malloc_usable_size) - bytes = malloc_usable_size(tl); -#endif /* malloc_usable_size */ - *tl = txl_bytes2size(bytes); - assert(*tl >= wanna); - *ptl = tl + 1; - return MDBX_SUCCESS; - } - return MDBX_ENOMEM; +MDBX_NOTHROW_PURE_FUNCTION static inline size_t page_room(const page_t *mp) { return mp->upper - mp->lower; } + +MDBX_NOTHROW_PURE_FUNCTION static inline size_t page_space(const MDBX_env *env) { + STATIC_ASSERT(PAGEHDRSZ % 2 == 0); + return env->ps - PAGEHDRSZ; } -static __always_inline int __must_check_result txl_need(MDBX_TXL *ptl, - size_t num) { - assert(MDBX_PNL_GETSIZE(*ptl) <= MDBX_TXL_MAX && - MDBX_PNL_ALLOCLEN(*ptl) >= MDBX_PNL_GETSIZE(*ptl)); - assert(num <= MDBX_PGL_LIMIT); - const size_t wanna = (size_t)MDBX_PNL_GETSIZE(*ptl) + num; - return likely(MDBX_PNL_ALLOCLEN(*ptl) >= wanna) ? MDBX_SUCCESS - : txl_reserve(ptl, wanna); +MDBX_NOTHROW_PURE_FUNCTION static inline size_t page_used(const MDBX_env *env, const page_t *mp) { + return page_space(env) - page_room(mp); } -static __always_inline void txl_xappend(MDBX_TXL tl, txnid_t id) { - assert(MDBX_PNL_GETSIZE(tl) < MDBX_PNL_ALLOCLEN(tl)); - tl[0] += 1; - MDBX_PNL_LAST(tl) = id; +/* The percentage of space used in the page, in a percents. */ +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline unsigned page_fill_percentum_x10(const MDBX_env *env, + const page_t *mp) { + const size_t space = page_space(env); + return (unsigned)((page_used(env, mp) * 1000 + space / 2) / space); } -#define TXNID_SORT_CMP(first, last) ((first) > (last)) -SORT_IMPL(txnid_sort, false, txnid_t, TXNID_SORT_CMP) -static void txl_sort(MDBX_TXL tl) { - txnid_sort(MDBX_PNL_BEGIN(tl), MDBX_PNL_END(tl)); +MDBX_NOTHROW_PURE_FUNCTION static inline node_t *page_node(const page_t *mp, size_t i) { + assert(page_type_compat(mp) == P_LEAF || page_type(mp) == P_BRANCH); + assert(page_numkeys(mp) > i); + assert(mp->entries[i] % 2 == 0); + return ptr_disp(mp, mp->entries[i] + PAGEHDRSZ); } -static int __must_check_result txl_append(MDBX_TXL *ptl, txnid_t id) { - if (unlikely(MDBX_PNL_GETSIZE(*ptl) == MDBX_PNL_ALLOCLEN(*ptl))) { - int rc = txl_need(ptl, MDBX_TXL_GRANULATE); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - } - txl_xappend(*ptl, id); - return MDBX_SUCCESS; +MDBX_NOTHROW_PURE_FUNCTION static inline void *page_dupfix_ptr(const page_t *mp, size_t i, size_t keysize) { + assert(page_type_compat(mp) == (P_LEAF | P_DUPFIX) && i == (indx_t)i && mp->dupfix_ksize == keysize); + (void)keysize; + return ptr_disp(mp, PAGEHDRSZ + mp->dupfix_ksize * (indx_t)i); +} + +MDBX_NOTHROW_PURE_FUNCTION static inline MDBX_val page_dupfix_key(const page_t *mp, size_t i, size_t keysize) { + MDBX_val r; + r.iov_base = page_dupfix_ptr(mp, i, keysize); + r.iov_len = mp->dupfix_ksize; + return r; } /*----------------------------------------------------------------------------*/ -#define MDBX_DPL_GAP_MERGESORT 16 -#define MDBX_DPL_GAP_EDGING 2 -#define MDBX_DPL_RESERVE_GAP (MDBX_DPL_GAP_MERGESORT + MDBX_DPL_GAP_EDGING) +MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL int cmp_int_unaligned(const MDBX_val *a, const MDBX_val *b); -static __always_inline size_t dpl_size2bytes(ptrdiff_t size) { - assert(size > CURSOR_STACK && (size_t)size <= MDBX_PGL_LIMIT); -#if MDBX_DPL_PREALLOC_FOR_RADIXSORT - size += size; -#endif /* MDBX_DPL_PREALLOC_FOR_RADIXSORT */ - STATIC_ASSERT(MDBX_ASSUME_MALLOC_OVERHEAD + sizeof(MDBX_dpl) + - (MDBX_PGL_LIMIT * (MDBX_DPL_PREALLOC_FOR_RADIXSORT + 1) + - MDBX_DPL_RESERVE_GAP) * - sizeof(MDBX_dp) + - MDBX_PNL_GRANULATE * sizeof(void *) * 2 < - SIZE_MAX / 4 * 3); - size_t bytes = - ceil_powerof2(MDBX_ASSUME_MALLOC_OVERHEAD + sizeof(MDBX_dpl) + - ((size_t)size + MDBX_DPL_RESERVE_GAP) * sizeof(MDBX_dp), - MDBX_PNL_GRANULATE * sizeof(void *) * 2) - - MDBX_ASSUME_MALLOC_OVERHEAD; - return bytes; -} +#if MDBX_UNALIGNED_OK < 2 || (MDBX_DEBUG || MDBX_FORCE_ASSERTIONS || !defined(NDEBUG)) +MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL int +/* Compare two items pointing at 2-byte aligned unsigned int's. */ +cmp_int_align2(const MDBX_val *a, const MDBX_val *b); +#else +#define cmp_int_align2 cmp_int_unaligned +#endif /* !MDBX_UNALIGNED_OK || debug */ -static __always_inline size_t dpl_bytes2size(const ptrdiff_t bytes) { - size_t size = (bytes - sizeof(MDBX_dpl)) / sizeof(MDBX_dp); - size -= MDBX_DPL_RESERVE_GAP; -#if MDBX_DPL_PREALLOC_FOR_RADIXSORT - size >>= 1; -#endif /* MDBX_DPL_PREALLOC_FOR_RADIXSORT */ - assert(size > CURSOR_STACK && size <= MDBX_PGL_LIMIT + MDBX_PNL_GRANULATE); - return size; -} +#if MDBX_UNALIGNED_OK < 4 || (MDBX_DEBUG || MDBX_FORCE_ASSERTIONS || !defined(NDEBUG)) +MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL int +/* Compare two items pointing at 4-byte aligned unsigned int's. */ +cmp_int_align4(const MDBX_val *a, const MDBX_val *b); +#else +#define cmp_int_align4 cmp_int_unaligned +#endif /* !MDBX_UNALIGNED_OK || debug */ -static __always_inline size_t dpl_setlen(MDBX_dpl *dl, size_t len) { - static const MDBX_page dpl_stub_pageE = {INVALID_TXNID, - 0, - P_BAD, - {0}, - /* pgno */ ~(pgno_t)0}; - assert(dpl_stub_pageE.mp_flags == P_BAD && - dpl_stub_pageE.mp_pgno == P_INVALID); - dl->length = len; - dl->items[len + 1].ptr = (MDBX_page *)&dpl_stub_pageE; - dl->items[len + 1].pgno = P_INVALID; - dl->items[len + 1].npages = 1; - return len; -} +/* Compare two items lexically */ +MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL int cmp_lexical(const MDBX_val *a, const MDBX_val *b); -static __always_inline void dpl_clear(MDBX_dpl *dl) { - static const MDBX_page dpl_stub_pageB = {INVALID_TXNID, - 0, - P_BAD, - {0}, - /* pgno */ 0}; - assert(dpl_stub_pageB.mp_flags == P_BAD && dpl_stub_pageB.mp_pgno == 0); - dl->sorted = dpl_setlen(dl, 0); - dl->pages_including_loose = 0; - dl->items[0].ptr = (MDBX_page *)&dpl_stub_pageB; - dl->items[0].pgno = 0; - dl->items[0].npages = 1; - assert(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID); -} +/* Compare two items in reverse byte order */ +MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL int cmp_reverse(const MDBX_val *a, const MDBX_val *b); -static void dpl_free(MDBX_txn *txn) { - if (likely(txn->tw.dirtylist)) { - osal_free(txn->tw.dirtylist); - txn->tw.dirtylist = NULL; - } -} +/* Fast non-lexically comparator */ +MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL int cmp_lenfast(const MDBX_val *a, const MDBX_val *b); -static MDBX_dpl *dpl_reserve(MDBX_txn *txn, size_t size) { - tASSERT(txn, (txn->mt_flags & MDBX_TXN_RDONLY) == 0); - tASSERT(txn, (txn->mt_flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); +MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL bool eq_fast_slowpath(const uint8_t *a, const uint8_t *b, size_t l); - size_t bytes = - dpl_size2bytes((size < MDBX_PGL_LIMIT) ? size : MDBX_PGL_LIMIT); - MDBX_dpl *const dl = osal_realloc(txn->tw.dirtylist, bytes); - if (likely(dl)) { -#if __GLIBC_PREREQ(2, 12) || defined(__FreeBSD__) || defined(malloc_usable_size) - bytes = malloc_usable_size(dl); -#endif /* malloc_usable_size */ - dl->detent = dpl_bytes2size(bytes); - tASSERT(txn, txn->tw.dirtylist == NULL || dl->length <= dl->detent); - txn->tw.dirtylist = dl; - } - return dl; +MDBX_NOTHROW_PURE_FUNCTION static inline bool eq_fast(const MDBX_val *a, const MDBX_val *b) { + return unlikely(a->iov_len == b->iov_len) && eq_fast_slowpath(a->iov_base, b->iov_base, a->iov_len); } -static int dpl_alloc(MDBX_txn *txn) { - tASSERT(txn, (txn->mt_flags & MDBX_TXN_RDONLY) == 0); - tASSERT(txn, (txn->mt_flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); +MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL int cmp_equal_or_greater(const MDBX_val *a, const MDBX_val *b); - const size_t wanna = (txn->mt_env->me_options.dp_initial < txn->mt_geo.upper) - ? txn->mt_env->me_options.dp_initial - : txn->mt_geo.upper; -#if MDBX_FORCE_ASSERTIONS || MDBX_DEBUG - if (txn->tw.dirtylist) - /* обнуляем чтобы не сработал ассерт внутри dpl_reserve() */ - txn->tw.dirtylist->sorted = txn->tw.dirtylist->length = 0; -#endif /* asertions enabled */ - if (unlikely(!txn->tw.dirtylist || txn->tw.dirtylist->detent < wanna || - txn->tw.dirtylist->detent > wanna + wanna) && - unlikely(!dpl_reserve(txn, wanna))) - return MDBX_ENOMEM; +MDBX_NOTHROW_PURE_FUNCTION MDBX_INTERNAL int cmp_equal_or_wrong(const MDBX_val *a, const MDBX_val *b); - dpl_clear(txn->tw.dirtylist); - return MDBX_SUCCESS; +static inline MDBX_cmp_func *builtin_keycmp(MDBX_db_flags_t flags) { + return (flags & MDBX_REVERSEKEY) ? cmp_reverse : (flags & MDBX_INTEGERKEY) ? cmp_int_align2 : cmp_lexical; } -#define MDBX_DPL_EXTRACT_KEY(ptr) ((ptr)->pgno) -RADIXSORT_IMPL(dpl, MDBX_dp, MDBX_DPL_EXTRACT_KEY, - MDBX_DPL_PREALLOC_FOR_RADIXSORT, 1) +static inline MDBX_cmp_func *builtin_datacmp(MDBX_db_flags_t flags) { + return !(flags & MDBX_DUPSORT) + ? cmp_lenfast + : ((flags & MDBX_INTEGERDUP) ? cmp_int_unaligned + : ((flags & MDBX_REVERSEDUP) ? cmp_reverse : cmp_lexical)); +} -#define DP_SORT_CMP(first, last) ((first).pgno < (last).pgno) -SORT_IMPL(dp_sort, false, MDBX_dp, DP_SORT_CMP) +/*----------------------------------------------------------------------------*/ -__hot __noinline static MDBX_dpl *dpl_sort_slowpath(const MDBX_txn *txn) { - tASSERT(txn, (txn->mt_flags & MDBX_TXN_RDONLY) == 0); - tASSERT(txn, (txn->mt_flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); +MDBX_INTERNAL uint32_t combine_durability_flags(const uint32_t a, const uint32_t b); - MDBX_dpl *dl = txn->tw.dirtylist; - assert(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID); - const size_t unsorted = dl->length - dl->sorted; - if (likely(unsorted < MDBX_RADIXSORT_THRESHOLD) || - unlikely(!dpl_radixsort(dl->items + 1, dl->length))) { - if (dl->sorted > unsorted / 4 + 4 && - (MDBX_DPL_PREALLOC_FOR_RADIXSORT || - dl->length + unsorted < dl->detent + MDBX_DPL_GAP_MERGESORT)) { - MDBX_dp *const sorted_begin = dl->items + 1; - MDBX_dp *const sorted_end = sorted_begin + dl->sorted; - MDBX_dp *const end = - dl->items + (MDBX_DPL_PREALLOC_FOR_RADIXSORT - ? dl->length + dl->length + 1 - : dl->detent + MDBX_DPL_RESERVE_GAP); - MDBX_dp *const tmp = end - unsorted; - assert(dl->items + dl->length + 1 < tmp); - /* copy unsorted to the end of allocated space and sort it */ - memcpy(tmp, sorted_end, unsorted * sizeof(MDBX_dp)); - dp_sort(tmp, tmp + unsorted); - /* merge two parts from end to begin */ - MDBX_dp *__restrict w = dl->items + dl->length; - MDBX_dp *__restrict l = dl->items + dl->sorted; - MDBX_dp *__restrict r = end - 1; - do { - const bool cmp = expect_with_probability(l->pgno > r->pgno, 0, .5); -#if defined(__LCC__) || __CLANG_PREREQ(13, 0) || !MDBX_HAVE_CMOV - *w = cmp ? *l-- : *r--; -#else - *w = cmp ? *l : *r; - l -= cmp; - r += (ptrdiff_t)cmp - 1; -#endif - } while (likely(--w > l)); - assert(r == tmp - 1); - assert(dl->items[0].pgno == 0 && - dl->items[dl->length + 1].pgno == P_INVALID); - if (ASSERT_ENABLED()) - for (size_t i = 0; i <= dl->length; ++i) - assert(dl->items[i].pgno < dl->items[i + 1].pgno); - } else { - dp_sort(dl->items + 1, dl->items + dl->length + 1); - assert(dl->items[0].pgno == 0 && - dl->items[dl->length + 1].pgno == P_INVALID); - } - } else { - assert(dl->items[0].pgno == 0 && - dl->items[dl->length + 1].pgno == P_INVALID); - } - dl->sorted = dl->length; - return dl; +MDBX_CONST_FUNCTION static inline lck_t *lckless_stub(const MDBX_env *env) { + uintptr_t stub = (uintptr_t)&env->lckless_placeholder; + /* align to avoid false-positive alarm from UndefinedBehaviorSanitizer */ + stub = (stub + MDBX_CACHELINE_SIZE - 1) & ~(MDBX_CACHELINE_SIZE - 1); + return (lck_t *)stub; +} + +#if !(defined(_WIN32) || defined(_WIN64)) +MDBX_CONST_FUNCTION static inline int ignore_enosys(int err) { +#ifdef ENOSYS + if (err == ENOSYS) + return MDBX_RESULT_TRUE; +#endif /* ENOSYS */ +#ifdef ENOIMPL + if (err == ENOIMPL) + return MDBX_RESULT_TRUE; +#endif /* ENOIMPL */ +#ifdef ENOTSUP + if (err == ENOTSUP) + return MDBX_RESULT_TRUE; +#endif /* ENOTSUP */ +#ifdef ENOSUPP + if (err == ENOSUPP) + return MDBX_RESULT_TRUE; +#endif /* ENOSUPP */ +#ifdef EOPNOTSUPP + if (err == EOPNOTSUPP) + return MDBX_RESULT_TRUE; +#endif /* EOPNOTSUPP */ + return err; } -static __always_inline MDBX_dpl *dpl_sort(const MDBX_txn *txn) { - tASSERT(txn, (txn->mt_flags & MDBX_TXN_RDONLY) == 0); - tASSERT(txn, (txn->mt_flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); +MDBX_MAYBE_UNUSED MDBX_CONST_FUNCTION static inline int ignore_enosys_and_eagain(int err) { + return (err == EAGAIN) ? MDBX_RESULT_TRUE : ignore_enosys(err); +} - MDBX_dpl *dl = txn->tw.dirtylist; - assert(dl->length <= MDBX_PGL_LIMIT); - assert(dl->sorted <= dl->length); - assert(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID); - return likely(dl->sorted == dl->length) ? dl : dpl_sort_slowpath(txn); +MDBX_MAYBE_UNUSED MDBX_CONST_FUNCTION static inline int ignore_enosys_and_einval(int err) { + return (err == EINVAL) ? MDBX_RESULT_TRUE : ignore_enosys(err); } -/* Returns the index of the first dirty-page whose pgno - * member is greater than or equal to id. */ -#define DP_SEARCH_CMP(dp, id) ((dp).pgno < (id)) -SEARCH_IMPL(dp_bsearch, MDBX_dp, pgno_t, DP_SEARCH_CMP) +MDBX_MAYBE_UNUSED MDBX_CONST_FUNCTION static inline int ignore_enosys_and_eremote(int err) { + return (err == MDBX_EREMOTE) ? MDBX_RESULT_TRUE : ignore_enosys(err); +} -__hot __noinline static size_t dpl_search(const MDBX_txn *txn, pgno_t pgno) { - tASSERT(txn, (txn->mt_flags & MDBX_TXN_RDONLY) == 0); - tASSERT(txn, (txn->mt_flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); +#endif /* defined(_WIN32) || defined(_WIN64) */ - MDBX_dpl *dl = txn->tw.dirtylist; - assert(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID); - if (AUDIT_ENABLED()) { - for (const MDBX_dp *ptr = dl->items + dl->sorted; --ptr > dl->items;) { - assert(ptr[0].pgno < ptr[1].pgno); - assert(ptr[0].pgno >= NUM_METAS); +static inline int check_env(const MDBX_env *env, const bool wanna_active) { + if (unlikely(!env)) + return MDBX_EINVAL; + + if (unlikely(env->signature.weak != env_signature)) + return MDBX_EBADSIGN; + + if (unlikely(env->flags & ENV_FATAL_ERROR)) + return MDBX_PANIC; + + if (wanna_active) { +#if MDBX_ENV_CHECKPID + if (unlikely(env->pid != osal_getpid()) && env->pid) { + ((MDBX_env *)env)->flags |= ENV_FATAL_ERROR; + return MDBX_PANIC; } +#endif /* MDBX_ENV_CHECKPID */ + if (unlikely((env->flags & ENV_ACTIVE) == 0)) + return MDBX_EPERM; + eASSERT(env, env->dxb_mmap.base != nullptr); } - switch (dl->length - dl->sorted) { - default: - /* sort a whole */ - dpl_sort_slowpath(txn); - break; - case 0: - /* whole sorted cases */ - break; + return MDBX_SUCCESS; +} -#define LINEAR_SEARCH_CASE(N) \ - case N: \ - if (dl->items[dl->length - N + 1].pgno == pgno) \ - return dl->length - N + 1; \ - __fallthrough +static __always_inline int check_txn(const MDBX_txn *txn, int bad_bits) { + if (unlikely(!txn)) + return MDBX_EINVAL; - /* use linear scan until the threshold */ - LINEAR_SEARCH_CASE(7); /* fall through */ - LINEAR_SEARCH_CASE(6); /* fall through */ - LINEAR_SEARCH_CASE(5); /* fall through */ - LINEAR_SEARCH_CASE(4); /* fall through */ - LINEAR_SEARCH_CASE(3); /* fall through */ - LINEAR_SEARCH_CASE(2); /* fall through */ - case 1: - if (dl->items[dl->length].pgno == pgno) - return dl->length; - /* continue bsearch on the sorted part */ - break; + if (unlikely(txn->signature != txn_signature)) + return MDBX_EBADSIGN; + + if (bad_bits) { + if (unlikely(!txn->env->dxb_mmap.base)) + return MDBX_EPERM; + + if (unlikely(txn->flags & bad_bits)) { + if ((bad_bits & MDBX_TXN_RDONLY) && unlikely(txn->flags & MDBX_TXN_RDONLY)) + return MDBX_EACCESS; + if ((bad_bits & MDBX_TXN_PARKED) == 0) + return MDBX_BAD_TXN; + return txn_check_badbits_parked(txn, bad_bits); + } } - return dp_bsearch(dl->items + 1, dl->sorted, pgno) - dl->items; + + tASSERT(txn, (txn->flags & MDBX_TXN_FINISHED) || + (txn->flags & MDBX_NOSTICKYTHREADS) == (txn->env->flags & MDBX_NOSTICKYTHREADS)); +#if MDBX_TXN_CHECKOWNER + if ((txn->flags & (MDBX_NOSTICKYTHREADS | MDBX_TXN_FINISHED)) != MDBX_NOSTICKYTHREADS && + !(bad_bits /* abort/reset/txn-break */ == 0 && + ((txn->flags & (MDBX_TXN_RDONLY | MDBX_TXN_FINISHED)) == (MDBX_TXN_RDONLY | MDBX_TXN_FINISHED))) && + unlikely(txn->owner != osal_thread_self())) + return txn->owner ? MDBX_THREAD_MISMATCH : MDBX_BAD_TXN; +#endif /* MDBX_TXN_CHECKOWNER */ + + return MDBX_SUCCESS; } -MDBX_NOTHROW_PURE_FUNCTION static __inline unsigned -dpl_npages(const MDBX_dpl *dl, size_t i) { - assert(0 <= (intptr_t)i && i <= dl->length); - unsigned n = dl->items[i].npages; - assert(n == (IS_OVERFLOW(dl->items[i].ptr) ? dl->items[i].ptr->mp_pages : 1)); - return n; +static inline int check_txn_rw(const MDBX_txn *txn, int bad_bits) { + return check_txn(txn, (bad_bits | MDBX_TXN_RDONLY) & ~MDBX_TXN_PARKED); } -MDBX_NOTHROW_PURE_FUNCTION static __inline pgno_t -dpl_endpgno(const MDBX_dpl *dl, size_t i) { - return dpl_npages(dl, i) + dl->items[i].pgno; +/*----------------------------------------------------------------------------*/ + +MDBX_INTERNAL void mincore_clean_cache(const MDBX_env *const env); + +MDBX_INTERNAL void update_mlcnt(const MDBX_env *env, const pgno_t new_aligned_mlocked_pgno, + const bool lock_not_release); + +MDBX_INTERNAL void munlock_after(const MDBX_env *env, const pgno_t aligned_pgno, const size_t end_bytes); + +MDBX_INTERNAL void munlock_all(const MDBX_env *env); + +/*----------------------------------------------------------------------------*/ +/* Cache coherence and mmap invalidation */ +#ifndef MDBX_CPU_WRITEBACK_INCOHERENT +#error "The MDBX_CPU_WRITEBACK_INCOHERENT must be defined before" +#elif MDBX_CPU_WRITEBACK_INCOHERENT +#define osal_flush_incoherent_cpu_writeback() osal_memory_barrier() +#else +#define osal_flush_incoherent_cpu_writeback() osal_compiler_barrier() +#endif /* MDBX_CPU_WRITEBACK_INCOHERENT */ + +MDBX_MAYBE_UNUSED static inline void osal_flush_incoherent_mmap(const void *addr, size_t nbytes, + const intptr_t pagesize) { +#ifndef MDBX_MMAP_INCOHERENT_FILE_WRITE +#error "The MDBX_MMAP_INCOHERENT_FILE_WRITE must be defined before" +#elif MDBX_MMAP_INCOHERENT_FILE_WRITE + char *const begin = (char *)(-pagesize & (intptr_t)addr); + char *const end = (char *)(-pagesize & (intptr_t)((char *)addr + nbytes + pagesize - 1)); + int err = msync(begin, end - begin, MS_SYNC | MS_INVALIDATE) ? errno : 0; + eASSERT(nullptr, err == 0); + (void)err; +#else + (void)pagesize; +#endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */ + +#ifndef MDBX_MMAP_INCOHERENT_CPU_CACHE +#error "The MDBX_MMAP_INCOHERENT_CPU_CACHE must be defined before" +#elif MDBX_MMAP_INCOHERENT_CPU_CACHE +#ifdef DCACHE + /* MIPS has cache coherency issues. + * Note: for any nbytes >= on-chip cache size, entire is flushed. */ + cacheflush((void *)addr, nbytes, DCACHE); +#else +#error "Oops, cacheflush() not available" +#endif /* DCACHE */ +#endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */ + +#if !MDBX_MMAP_INCOHERENT_FILE_WRITE && !MDBX_MMAP_INCOHERENT_CPU_CACHE + (void)addr; + (void)nbytes; +#endif } -static __inline bool dpl_intersect(const MDBX_txn *txn, pgno_t pgno, - size_t npages) { - tASSERT(txn, (txn->mt_flags & MDBX_TXN_RDONLY) == 0); - tASSERT(txn, (txn->mt_flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); +/* Состояние курсора. + * + * плохой/poor: + * - неустановленный курсор с незаполненым стеком; + * - следует пропускать во всех циклах отслеживания/корректировки + * позиций курсоров; + * - допускаются только операции предполагающие установку абсолютной позиции; + * - в остальных случаях возвращается ENODATA. + * + * У таких курсоров top = -1 и flags < 0, что позволяет дешево проверять и + * пропускать такие курсоры в циклах отслеживания/корректировки по условию + * probe_cursor->top < this_cursor->top. + * + * пустой/hollow: + * - частично инициализированный курсор, но без доступной пользователю позиции, + * поэтому нельзя выполнить какую-либо операцию без абсолютного (не + * относительного) позиционирования; + * - ki[top] может быть некорректным, в том числе >= page_numkeys(pg[top]). + * + * У таких курсоров top >= 0, но flags < 0 (есть флажок z_hollow). + * + * установленный/pointed: + * - полностью инициализированный курсор с конкретной позицией с данными; + * - можно прочитать текущую строку, удалить её, либо выполнить + * относительное перемещение; + * - может иметь флажки z_after_delete, z_eof_hard и z_eof_soft; + * - наличие z_eof_soft означает что курсор перемещен за пределы данных, + * поэтому нелья прочитать текущие данные, либо удалить их. + * + * У таких курсоров top >= 0 и flags >= 0 (нет флажка z_hollow). + * + * наполненный данными/filled: + * - это установленный/pointed курсор без флагов z_eof_soft; + * - за курсором есть даные, возможны CRUD операции в текущей позиции. + * + * У таких курсоров top >= 0 и (unsigned)flags < z_eof_soft. + * + * Изменения состояния. + * + * - Сбрасывается состояние курсора посредством top_and_flags |= z_poor_mark, + * что равносильно top = -1 вместе с flags |= z_poor_mark; + * - При позиционировании курсора сначала устанавливается top, а flags + * только в самом конце при отсутстви ошибок. + * - Повторное позиционирование first/last может начинаться + * с установки/обнуления только top без сброса flags, что позволяет работать + * быстрому пути внутри tree_search_finalize(). + * + * - Заморочки с концом данных: + * - mdbx_cursor_get(NEXT) выполняет две операции (перемещение и чтение), + * поэтому перемещение на последнюю строку строку всегда успешно, + * а ошибка возвращается только при последующем next(). + * Однако, из-за этой двойственности семантика ситуации возврата ошибки + * из mdbx_cursor_get(NEXT) допускает разночтение/неопределенность, ибо + * не понятно к чему относится ошибка: + * - Если к чтению данных, то курсор перемещен и стоит после последней + * строки. Соответственно, чтение в текущей позиции запрещено, + * а при выполнении prev() курсор вернется на последнюю строку; + * - Если же ошибка относится к перемещению, то курсор не перемещен и + * остается на последней строке. Соответственно, чтение в текущей + * позиции допустимо, а при выполнении prev() курсор встанет + * на пред-последнюю строку. + * - Пикантность в том, что пользователи (так или иначе) полагаются + * на оба варианта поведения, при этом конечно ожидают что после + * ошибки MDBX_NEXT функция mdbx_cursor_eof() будет возвращать true. + * - далее добавляется схожая ситуация с MDBX_GET_RANGE, MDBX_LOWERBOUND, + * MDBX_GET_BOTH_RANGE и MDBX_UPPERBOUND. Тут при неуспехе поиска курсор + * может/должен стоять после последней строки. + * - далее добавляется MDBX_LAST. Тут курсор должен стоять на последней + * строке и допускать чтение в текузщей позиции, + * но mdbx_cursor_eof() должен возвращать true. + * + * Решение = делаем два флажка z_eof_soft и z_eof_hard: + * - Когда установлен только z_eof_soft, + * функция mdbx_cursor_eof() возвращает true, но допускается + * чтение данных в текущей позиции, а prev() передвигает курсор + * на пред-последнюю строку. + * - Когда установлен z_eof_hard, чтение данных в текущей позиции + * не допускается, и mdbx_cursor_eof() также возвращает true, + * а prev() устанавливает курсора на последюю строку. */ +enum cursor_state { + /* Это вложенный курсор для вложенного дерева/страницы и является + inner-элементом struct cursor_couple. */ + z_inner = 0x01, + + /* Происходит подготовка к обновлению GC, + поэтому можно брать страницы из GC даже для FREE_DBI. */ + z_gcu_preparation = 0x02, + + /* Курсор только-что создан, поэтому допускается авто-установка + в начало/конец, вместо возврата ошибки. */ + z_fresh = 0x04, + + /* Предыдущей операцией было удаление, поэтому курсор уже физически указывает + на следующий элемент и соответствующая операция перемещения должна + игнорироваться. */ + z_after_delete = 0x08, + + /* */ + z_disable_tree_search_fastpath = 0x10, + + /* Курсор логически в конце данных, но физически на последней строке, + * ki[top] == page_numkeys(pg[top]) - 1 и читать данные в текущей позиции. */ + z_eof_soft = 0x20, + + /* Курсор логически за концом данных, поэтому следующий переход "назад" + должен игнорироваться и/или приводить к установке на последнюю строку. + В текущем же состоянии нельзя делать CRUD операции. */ + z_eof_hard = 0x40, + + /* За курсором нет данных, логически его позиция не определена, + нельзя делать CRUD операции в текущей позиции. + Относительное перемещение запрещено. */ + z_hollow = -128 /* 0x80 */, + + /* Маски для сброса/установки состояния. */ + z_clear_mask = z_inner | z_gcu_preparation, + z_poor_mark = z_eof_hard | z_hollow | z_disable_tree_search_fastpath, + z_fresh_mark = z_poor_mark | z_fresh +}; - MDBX_dpl *dl = txn->tw.dirtylist; - assert(dl->sorted == dl->length); - assert(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID); - size_t const n = dpl_search(txn, pgno); - assert(n >= 1 && n <= dl->length + 1); - assert(pgno <= dl->items[n].pgno); - assert(pgno > dl->items[n - 1].pgno); - const bool rc = - /* intersection with founded */ pgno + npages > dl->items[n].pgno || - /* intersection with prev */ dpl_endpgno(dl, n - 1) > pgno; - if (ASSERT_ENABLED()) { - bool check = false; - for (size_t i = 1; i <= dl->length; ++i) { - const MDBX_page *const dp = dl->items[i].ptr; - if (!(dp->mp_pgno /* begin */ >= /* end */ pgno + npages || - dpl_endpgno(dl, i) /* end */ <= /* begin */ pgno)) - check |= true; - } - assert(check == rc); - } - return rc; +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_inner(const MDBX_cursor *mc) { + return (mc->flags & z_inner) != 0; } -MDBX_NOTHROW_PURE_FUNCTION static __always_inline size_t -dpl_exist(const MDBX_txn *txn, pgno_t pgno) { - tASSERT(txn, (txn->mt_flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); - MDBX_dpl *dl = txn->tw.dirtylist; - size_t i = dpl_search(txn, pgno); - assert((int)i > 0); - return (dl->items[i].pgno == pgno) ? i : 0; +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_poor(const MDBX_cursor *mc) { + const bool r = mc->top < 0; + cASSERT(mc, r == (mc->top_and_flags < 0)); + if (r && mc->subcur) + cASSERT(mc, mc->subcur->cursor.flags < 0 && mc->subcur->cursor.top < 0); + return r; } -MDBX_MAYBE_UNUSED static const MDBX_page *debug_dpl_find(const MDBX_txn *txn, - const pgno_t pgno) { - tASSERT(txn, (txn->mt_flags & MDBX_TXN_RDONLY) == 0); - const MDBX_dpl *dl = txn->tw.dirtylist; - if (dl) { - tASSERT(txn, (txn->mt_flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); - assert(dl->items[0].pgno == 0 && - dl->items[dl->length + 1].pgno == P_INVALID); - for (size_t i = dl->length; i > dl->sorted; --i) - if (dl->items[i].pgno == pgno) - return dl->items[i].ptr; +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_pointed(const MDBX_cursor *mc) { + const bool r = mc->top >= 0; + cASSERT(mc, r == (mc->top_and_flags >= 0)); + if (!r && mc->subcur) + cASSERT(mc, is_poor(&mc->subcur->cursor)); + return r; +} - if (dl->sorted) { - const size_t i = dp_bsearch(dl->items + 1, dl->sorted, pgno) - dl->items; - if (dl->items[i].pgno == pgno) - return dl->items[i].ptr; - } +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_hollow(const MDBX_cursor *mc) { + const bool r = mc->flags < 0; + if (!r) { + cASSERT(mc, mc->top >= 0); + cASSERT(mc, (mc->flags & z_eof_hard) || mc->ki[mc->top] < page_numkeys(mc->pg[mc->top])); + } else if (mc->subcur) + cASSERT(mc, is_poor(&mc->subcur->cursor)); + return r; +} + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_eof(const MDBX_cursor *mc) { + const bool r = z_eof_soft <= (uint8_t)mc->flags; + return r; +} + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_filled(const MDBX_cursor *mc) { + const bool r = z_eof_hard > (uint8_t)mc->flags; + return r; +} + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool inner_filled(const MDBX_cursor *mc) { + return mc->subcur && is_filled(&mc->subcur->cursor); +} + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool inner_pointed(const MDBX_cursor *mc) { + return mc->subcur && is_pointed(&mc->subcur->cursor); +} + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool inner_hollow(const MDBX_cursor *mc) { + const bool r = !mc->subcur || is_hollow(&mc->subcur->cursor); +#if MDBX_DEBUG || MDBX_FORCE_ASSERTIONS + if (!r) { + cASSERT(mc, is_filled(mc)); + const page_t *mp = mc->pg[mc->top]; + const node_t *node = page_node(mp, mc->ki[mc->top]); + cASSERT(mc, node_flags(node) & N_DUP); + } +#endif /* MDBX_DEBUG || MDBX_FORCE_ASSERTIONS */ + return r; +} + +MDBX_MAYBE_UNUSED static inline void inner_gone(MDBX_cursor *mc) { + if (mc->subcur) { + TRACE("reset inner cursor %p", __Wpedantic_format_voidptr(&mc->subcur->cursor)); + mc->subcur->nested_tree.root = 0; + mc->subcur->cursor.top_and_flags = z_inner | z_poor_mark; + } +} + +MDBX_MAYBE_UNUSED static inline void be_poor(MDBX_cursor *mc) { + const bool inner = is_inner(mc); + if (inner) { + mc->tree->root = 0; + mc->top_and_flags = z_inner | z_poor_mark; } else { - tASSERT(txn, (txn->mt_flags & MDBX_WRITEMAP) != 0 && !MDBX_AVOID_MSYNC); + mc->top_and_flags |= z_poor_mark; + inner_gone(mc); } - return nullptr; + cASSERT(mc, is_poor(mc) && !is_pointed(mc) && !is_filled(mc)); + cASSERT(mc, inner == is_inner(mc)); } -static void dpl_remove_ex(const MDBX_txn *txn, size_t i, size_t npages) { - tASSERT(txn, (txn->mt_flags & MDBX_TXN_RDONLY) == 0); - tASSERT(txn, (txn->mt_flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); +MDBX_MAYBE_UNUSED static inline void be_filled(MDBX_cursor *mc) { + cASSERT(mc, mc->top >= 0); + cASSERT(mc, mc->ki[mc->top] < page_numkeys(mc->pg[mc->top])); + const bool inner = is_inner(mc); + mc->flags &= z_clear_mask; + cASSERT(mc, is_filled(mc)); + cASSERT(mc, inner == is_inner(mc)); +} - MDBX_dpl *dl = txn->tw.dirtylist; - assert((intptr_t)i > 0 && i <= dl->length); - assert(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID); - dl->pages_including_loose -= npages; - dl->sorted -= dl->sorted >= i; - dl->length -= 1; - memmove(dl->items + i, dl->items + i + 1, - (dl->length - i + 2) * sizeof(dl->items[0])); - assert(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID); +MDBX_MAYBE_UNUSED static inline bool is_related(const MDBX_cursor *base, const MDBX_cursor *scan) { + cASSERT(base, base->top >= 0); + return base->top <= scan->top && base != scan; } -static void dpl_remove(const MDBX_txn *txn, size_t i) { - dpl_remove_ex(txn, i, dpl_npages(txn->tw.dirtylist, i)); +/* Флаги контроля/проверки курсора. */ +enum cursor_checking { + z_branch = 0x01 /* same as P_BRANCH for check_leaf_type() */, + z_leaf = 0x02 /* same as P_LEAF for check_leaf_type() */, + z_largepage = 0x04 /* same as P_LARGE for check_leaf_type() */, + z_updating = 0x08 /* update/rebalance pending */, + z_ignord = 0x10 /* don't check keys ordering */, + z_dupfix = 0x20 /* same as P_DUPFIX for check_leaf_type() */, + z_retiring = 0x40 /* refs to child pages may be invalid */, + z_pagecheck = 0x80 /* perform page checking, see MDBX_VALIDATION */ +}; + +MDBX_INTERNAL int __must_check_result cursor_validate(const MDBX_cursor *mc); + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline size_t cursor_dbi(const MDBX_cursor *mc) { + cASSERT(mc, mc->txn && mc->txn->signature == txn_signature); + size_t dbi = mc->dbi_state - mc->txn->dbi_state; + cASSERT(mc, dbi < mc->txn->env->n_dbi); + return dbi; } -static __noinline void txn_lru_reduce(MDBX_txn *txn) { - NOTICE("lru-reduce %u -> %u", txn->tw.dirtylru, txn->tw.dirtylru >> 1); - tASSERT(txn, (txn->mt_flags & (MDBX_TXN_RDONLY | MDBX_WRITEMAP)) == 0); - do { - txn->tw.dirtylru >>= 1; - MDBX_dpl *dl = txn->tw.dirtylist; - for (size_t i = 1; i <= dl->length; ++i) { - size_t *const ptr = - ptr_disp(dl->items[i].ptr, -(ptrdiff_t)sizeof(size_t)); - *ptr >>= 1; - } - txn = txn->mt_parent; - } while (txn); +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool cursor_dbi_changed(const MDBX_cursor *mc) { + return dbi_changed(mc->txn, cursor_dbi(mc)); } -MDBX_NOTHROW_PURE_FUNCTION static __inline uint32_t dpl_age(const MDBX_txn *txn, - size_t i) { - tASSERT(txn, (txn->mt_flags & (MDBX_TXN_RDONLY | MDBX_WRITEMAP)) == 0); - const MDBX_dpl *dl = txn->tw.dirtylist; - assert((intptr_t)i > 0 && i <= dl->length); - size_t *const ptr = ptr_disp(dl->items[i].ptr, -(ptrdiff_t)sizeof(size_t)); - return txn->tw.dirtylru - (uint32_t)*ptr; +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint8_t *cursor_dbi_state(const MDBX_cursor *mc) { + return mc->dbi_state; } -static __inline uint32_t txn_lru_turn(MDBX_txn *txn) { - txn->tw.dirtylru += 1; - if (unlikely(txn->tw.dirtylru > UINT32_MAX / 3) && - (txn->mt_flags & MDBX_WRITEMAP) == 0) - txn_lru_reduce(txn); - return txn->tw.dirtylru; +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool cursor_is_gc(const MDBX_cursor *mc) { + return mc->dbi_state == mc->txn->dbi_state + FREE_DBI; } -static __always_inline int __must_check_result dpl_append(MDBX_txn *txn, - pgno_t pgno, - MDBX_page *page, - size_t npages) { - tASSERT(txn, (txn->mt_flags & MDBX_TXN_RDONLY) == 0); - tASSERT(txn, (txn->mt_flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); - const MDBX_dp dp = {page, pgno, (pgno_t)npages}; - if ((txn->mt_flags & MDBX_WRITEMAP) == 0) { - size_t *const ptr = ptr_disp(page, -(ptrdiff_t)sizeof(size_t)); - *ptr = txn->tw.dirtylru; - } +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool cursor_is_main(const MDBX_cursor *mc) { + return mc->dbi_state == mc->txn->dbi_state + MAIN_DBI; +} - MDBX_dpl *dl = txn->tw.dirtylist; - tASSERT(txn, dl->length <= MDBX_PGL_LIMIT + MDBX_PNL_GRANULATE); - tASSERT(txn, dl->items[0].pgno == 0 && - dl->items[dl->length + 1].pgno == P_INVALID); - if (AUDIT_ENABLED()) { - for (size_t i = dl->length; i > 0; --i) { - assert(dl->items[i].pgno != dp.pgno); - if (unlikely(dl->items[i].pgno == dp.pgno)) { - ERROR("Page %u already exist in the DPL at %zu", dp.pgno, i); - return MDBX_PROBLEM; - } - } +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool cursor_is_core(const MDBX_cursor *mc) { + return mc->dbi_state < mc->txn->dbi_state + CORE_DBS; +} + +MDBX_MAYBE_UNUSED static inline int cursor_dbi_dbg(const MDBX_cursor *mc) { + /* Debugging output value of a cursor's DBI: Negative for a sub-cursor. */ + const int dbi = cursor_dbi(mc); + return (mc->flags & z_inner) ? -dbi : dbi; +} + +MDBX_MAYBE_UNUSED static inline int __must_check_result cursor_push(MDBX_cursor *mc, page_t *mp, indx_t ki) { + TRACE("pushing page %" PRIaPGNO " on db %d cursor %p", mp->pgno, cursor_dbi_dbg(mc), __Wpedantic_format_voidptr(mc)); + if (unlikely(mc->top >= CURSOR_STACK_SIZE - 1)) { + be_poor(mc); + mc->txn->flags |= MDBX_TXN_ERROR; + return MDBX_CURSOR_FULL; } + mc->top += 1; + mc->pg[mc->top] = mp; + mc->ki[mc->top] = ki; + return MDBX_SUCCESS; +} - if (unlikely(dl->length == dl->detent)) { - if (unlikely(dl->detent >= MDBX_PGL_LIMIT)) { - ERROR("DPL is full (MDBX_PGL_LIMIT %zu)", MDBX_PGL_LIMIT); - return MDBX_TXN_FULL; - } - const size_t size = (dl->detent < MDBX_PNL_INITIAL * 42) - ? dl->detent + dl->detent - : dl->detent + dl->detent / 2; - dl = dpl_reserve(txn, size); - if (unlikely(!dl)) - return MDBX_ENOMEM; - tASSERT(txn, dl->length < dl->detent); - } +MDBX_MAYBE_UNUSED static inline void cursor_pop(MDBX_cursor *mc) { + TRACE("popped page %" PRIaPGNO " off db %d cursor %p", mc->pg[mc->top]->pgno, cursor_dbi_dbg(mc), + __Wpedantic_format_voidptr(mc)); + cASSERT(mc, mc->top >= 0); + mc->top -= 1; +} - /* Сортировка нужна для быстрого поиска, используем несколько тактик: - * 1) Сохраняем упорядоченность при естественной вставке в нужном порядке. - * 2) Добавляем в не-сортированный хвост, который сортируем и сливаем - * с отсортированной головой по необходимости, а пока хвост короткий - * ищем в нём сканированием, избегая большой пересортировки. - * 3) Если не-сортированный хвост короткий, а добавляемый элемент близок - * к концу отсортированной головы, то выгоднее сразу вставить элемент - * в нужное место. - * - * Алгоритмически: - * - добавлять в не-сортированный хвост следует только если вставка сильно - * дорогая, т.е. если целевая позиция элемента сильно далека от конца; - * - для быстрой проверки достаточно сравнить добавляемый элемент с отстоящим - * от конца на максимально-приемлемое расстояние; - * - если список короче, либо элемент в этой позиции меньше вставляемого, - * то следует перемещать элементы и вставлять в отсортированную голову; - * - если не-сортированный хвост длиннее, либо элемент в этой позиции больше, - * то следует добавлять в не-сортированный хвост. */ +MDBX_NOTHROW_PURE_FUNCTION static inline bool check_leaf_type(const MDBX_cursor *mc, const page_t *mp) { + return (((page_type(mp) ^ mc->checking) & (z_branch | z_leaf | z_largepage | z_dupfix)) == 0); +} - dl->pages_including_loose += npages; - MDBX_dp *i = dl->items + dl->length; +MDBX_INTERNAL int cursor_check(const MDBX_cursor *mc, int txn_bad_bits); -#define MDBX_DPL_INSERTION_THRESHOLD 42 - const ptrdiff_t pivot = (ptrdiff_t)dl->length - MDBX_DPL_INSERTION_THRESHOLD; -#if MDBX_HAVE_CMOV - const pgno_t pivot_pgno = - dl->items[(dl->length < MDBX_DPL_INSERTION_THRESHOLD) - ? 0 - : dl->length - MDBX_DPL_INSERTION_THRESHOLD] - .pgno; -#endif /* MDBX_HAVE_CMOV */ +/* без необходимости доступа к данным, без активации припаркованных транзакций. */ +static inline int cursor_check_pure(const MDBX_cursor *mc) { + return cursor_check(mc, MDBX_TXN_BLOCKED - MDBX_TXN_PARKED); +} - /* copy the stub beyond the end */ - i[2] = i[1]; - dl->length += 1; +/* для чтения данных, с активацией припаркованных транзакций. */ +static inline int cursor_check_ro(const MDBX_cursor *mc) { return cursor_check(mc, MDBX_TXN_BLOCKED); } - if (likely(pivot <= (ptrdiff_t)dl->sorted) && -#if MDBX_HAVE_CMOV - pivot_pgno < dp.pgno) { -#else - (pivot <= 0 || dl->items[pivot].pgno < dp.pgno)) { -#endif /* MDBX_HAVE_CMOV */ - dl->sorted += 1; +/* для записи данных. */ +static inline int cursor_check_rw(const MDBX_cursor *mc) { + return cursor_check(mc, (MDBX_TXN_BLOCKED - MDBX_TXN_PARKED) | MDBX_TXN_RDONLY); +} - /* сдвигаем несортированный хвост */ - while (i >= dl->items + dl->sorted) { -#if !defined(__GNUC__) /* пытаемся избежать вызова memmove() */ - i[1] = *i; -#elif MDBX_WORDBITS == 64 && \ - (defined(__SIZEOF_INT128__) || \ - (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128)) - STATIC_ASSERT(sizeof(MDBX_dp) == sizeof(__uint128_t)); - ((__uint128_t *)i)[1] = *(volatile __uint128_t *)i; -#else - i[1].ptr = i->ptr; - i[1].pgno = i->pgno; - i[1].npages = i->npages; -#endif - --i; - } - /* ищем нужную позицию сдвигая отсортированные элементы */ - while (i->pgno > pgno) { - tASSERT(txn, i > dl->items); - i[1] = *i; - --i; - } - tASSERT(txn, i->pgno < dp.pgno); - } +MDBX_INTERNAL MDBX_cursor *cursor_eot(MDBX_cursor *mc, MDBX_txn *txn, const bool merge); +MDBX_INTERNAL int cursor_shadow(MDBX_cursor *mc, MDBX_txn *nested, const size_t dbi); - i[1] = dp; - assert(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID); - assert(dl->sorted <= dl->length); - return MDBX_SUCCESS; -} +MDBX_INTERNAL MDBX_cursor *cursor_cpstk(const MDBX_cursor *csrc, MDBX_cursor *cdst); -/*----------------------------------------------------------------------------*/ +MDBX_INTERNAL int __must_check_result cursor_ops(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, + const MDBX_cursor_op op); + +MDBX_INTERNAL int __must_check_result cursor_check_multiple(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, + unsigned flags); -uint8_t runtime_flags = MDBX_RUNTIME_FLAGS_INIT; -uint8_t loglevel = MDBX_LOG_FATAL; -MDBX_debug_func *debug_logger; +MDBX_INTERNAL int __must_check_result cursor_put_checklen(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, + unsigned flags); -static __must_check_result __inline int page_retire(MDBX_cursor *mc, - MDBX_page *mp); +MDBX_INTERNAL int __must_check_result cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, unsigned flags); -static int __must_check_result page_dirty(MDBX_txn *txn, MDBX_page *mp, - size_t npages); -typedef struct page_result { - MDBX_page *page; +MDBX_INTERNAL int __must_check_result cursor_validate_updating(MDBX_cursor *mc); + +MDBX_INTERNAL int __must_check_result cursor_del(MDBX_cursor *mc, unsigned flags); + +MDBX_INTERNAL int __must_check_result cursor_sibling_left(MDBX_cursor *mc); +MDBX_INTERNAL int __must_check_result cursor_sibling_right(MDBX_cursor *mc); + +typedef struct cursor_set_result { int err; -} pgr_t; + bool exact; +} csr_t; -static txnid_t kick_longlived_readers(MDBX_env *env, const txnid_t laggard); +MDBX_INTERNAL csr_t cursor_seek(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, MDBX_cursor_op op); -static pgr_t page_new(MDBX_cursor *mc, const unsigned flags); -static pgr_t page_new_large(MDBX_cursor *mc, const size_t npages); -static int page_touch(MDBX_cursor *mc); -static int cursor_touch(MDBX_cursor *const mc, const MDBX_val *key, - const MDBX_val *data); +MDBX_INTERNAL int __must_check_result inner_first(MDBX_cursor *__restrict mc, MDBX_val *__restrict data); +MDBX_INTERNAL int __must_check_result inner_last(MDBX_cursor *__restrict mc, MDBX_val *__restrict data); +MDBX_INTERNAL int __must_check_result outer_first(MDBX_cursor *__restrict mc, MDBX_val *__restrict key, + MDBX_val *__restrict data); +MDBX_INTERNAL int __must_check_result outer_last(MDBX_cursor *__restrict mc, MDBX_val *__restrict key, + MDBX_val *__restrict data); -#define MDBX_END_NAMES \ - { \ - "committed", "empty-commit", "abort", "reset", "reset-tmp", "fail-begin", \ - "fail-beginchild" \ - } -enum { - /* txn_end operation number, for logging */ - MDBX_END_COMMITTED, - MDBX_END_PURE_COMMIT, - MDBX_END_ABORT, - MDBX_END_RESET, - MDBX_END_RESET_TMP, - MDBX_END_FAIL_BEGIN, - MDBX_END_FAIL_BEGINCHILD -}; -#define MDBX_END_OPMASK 0x0F /* mask for txn_end() operation number */ -#define MDBX_END_UPDATE 0x10 /* update env state (DBIs) */ -#define MDBX_END_FREE 0x20 /* free txn unless it is MDBX_env.me_txn0 */ -#define MDBX_END_EOTDONE 0x40 /* txn's cursors already closed */ -#define MDBX_END_SLOT 0x80 /* release any reader slot if MDBX_NOTLS */ -static int txn_end(MDBX_txn *txn, const unsigned mode); - -static __always_inline pgr_t page_get_inline(const uint16_t ILL, - const MDBX_cursor *const mc, - const pgno_t pgno, - const txnid_t front); - -static pgr_t page_get_any(const MDBX_cursor *const mc, const pgno_t pgno, - const txnid_t front) { - return page_get_inline(P_ILL_BITS, mc, pgno, front); -} +MDBX_INTERNAL int __must_check_result inner_next(MDBX_cursor *__restrict mc, MDBX_val *__restrict data); +MDBX_INTERNAL int __must_check_result inner_prev(MDBX_cursor *__restrict mc, MDBX_val *__restrict data); +MDBX_INTERNAL int __must_check_result outer_next(MDBX_cursor *__restrict mc, MDBX_val *__restrict key, + MDBX_val *__restrict data, MDBX_cursor_op op); +MDBX_INTERNAL int __must_check_result outer_prev(MDBX_cursor *__restrict mc, MDBX_val *__restrict key, + MDBX_val *__restrict data, MDBX_cursor_op op); -__hot static pgr_t page_get_three(const MDBX_cursor *const mc, - const pgno_t pgno, const txnid_t front) { - return page_get_inline(P_ILL_BITS | P_OVERFLOW, mc, pgno, front); -} +MDBX_INTERNAL int cursor_init4walk(cursor_couple_t *couple, const MDBX_txn *const txn, tree_t *const tree, + kvx_t *const kvx); -static pgr_t page_get_large(const MDBX_cursor *const mc, const pgno_t pgno, - const txnid_t front) { - return page_get_inline(P_ILL_BITS | P_BRANCH | P_LEAF | P_LEAF2, mc, pgno, - front); +MDBX_INTERNAL int __must_check_result cursor_init(MDBX_cursor *mc, const MDBX_txn *txn, size_t dbi); + +MDBX_INTERNAL int __must_check_result cursor_dupsort_setup(MDBX_cursor *mc, const node_t *node, const page_t *mp); + +MDBX_INTERNAL int __must_check_result cursor_touch(MDBX_cursor *const mc, const MDBX_val *key, const MDBX_val *data); + +/*----------------------------------------------------------------------------*/ + +/* Update sub-page pointer, if any, in mc->subcur. + * Needed when the node which contains the sub-page may have moved. + * Called with mp = mc->pg[mc->top], ki = mc->ki[mc->top]. */ +MDBX_MAYBE_UNUSED static inline void cursor_inner_refresh(const MDBX_cursor *mc, const page_t *mp, unsigned ki) { + cASSERT(mc, is_leaf(mp)); + const node_t *node = page_node(mp, ki); + if ((node_flags(node) & (N_DUP | N_TREE)) == N_DUP) + mc->subcur->cursor.pg[0] = node_data(node); +} + +MDBX_MAYBE_UNUSED MDBX_INTERNAL bool cursor_is_tracked(const MDBX_cursor *mc); + +static inline void cursor_reset(cursor_couple_t *couple) { + couple->outer.top_and_flags = z_fresh_mark; + couple->inner.cursor.top_and_flags = z_fresh_mark | z_inner; +} + +static inline void cursor_drown(cursor_couple_t *couple) { + couple->outer.top_and_flags = z_poor_mark; + couple->inner.cursor.top_and_flags = z_poor_mark | z_inner; + couple->outer.txn = nullptr; + couple->inner.cursor.txn = nullptr; + couple->outer.tree = nullptr; + /* сохраняем clc-указатель, так он используется для вычисления dbi в mdbx_cursor_renew(). */ + couple->outer.dbi_state = nullptr; + couple->inner.cursor.dbi_state = nullptr; +} + +static inline size_t dpl_setlen(dpl_t *dl, size_t len) { + static const page_t dpl_stub_pageE = {INVALID_TXNID, + 0, + P_BAD, + {0}, + /* pgno */ ~(pgno_t)0}; + assert(dpl_stub_pageE.flags == P_BAD && dpl_stub_pageE.pgno == P_INVALID); + dl->length = len; + dl->items[len + 1].ptr = (page_t *)&dpl_stub_pageE; + dl->items[len + 1].pgno = P_INVALID; + dl->items[len + 1].npages = 1; + return len; } -static __always_inline int __must_check_result page_get(const MDBX_cursor *mc, - const pgno_t pgno, - MDBX_page **mp, - const txnid_t front) { - pgr_t ret = page_get_three(mc, pgno, front); - *mp = ret.page; - return ret.err; +static inline void dpl_clear(dpl_t *dl) { + static const page_t dpl_stub_pageB = {INVALID_TXNID, + 0, + P_BAD, + {0}, + /* pgno */ 0}; + assert(dpl_stub_pageB.flags == P_BAD && dpl_stub_pageB.pgno == 0); + dl->sorted = dpl_setlen(dl, 0); + dl->pages_including_loose = 0; + dl->items[0].ptr = (page_t *)&dpl_stub_pageB; + dl->items[0].pgno = 0; + dl->items[0].npages = 1; + assert(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID); } -static int __must_check_result page_search_root(MDBX_cursor *mc, - const MDBX_val *key, int flags); +MDBX_INTERNAL int __must_check_result dpl_alloc(MDBX_txn *txn); -#define MDBX_PS_MODIFY 1 -#define MDBX_PS_ROOTONLY 2 -#define MDBX_PS_FIRST 4 -#define MDBX_PS_LAST 8 -static int __must_check_result page_search(MDBX_cursor *mc, const MDBX_val *key, - int flags); -static int __must_check_result page_merge(MDBX_cursor *csrc, MDBX_cursor *cdst); +MDBX_INTERNAL void dpl_free(MDBX_txn *txn); -#define MDBX_SPLIT_REPLACE MDBX_APPENDDUP /* newkey is not new */ -static int __must_check_result page_split(MDBX_cursor *mc, - const MDBX_val *const newkey, - MDBX_val *const newdata, - pgno_t newpgno, const unsigned naf); - -static int coherency_timeout(uint64_t *timestamp, intptr_t pgno, - const MDBX_env *env); -static int __must_check_result validate_meta_copy(MDBX_env *env, - const MDBX_meta *meta, - MDBX_meta *dest); -static int __must_check_result override_meta(MDBX_env *env, size_t target, - txnid_t txnid, - const MDBX_meta *shape); -static int __must_check_result read_header(MDBX_env *env, MDBX_meta *meta, - const int lck_exclusive, - const mdbx_mode_t mode_bits); -static int __must_check_result sync_locked(MDBX_env *env, unsigned flags, - MDBX_meta *const pending, - meta_troika_t *const troika); -static int env_close(MDBX_env *env); - -struct node_result { - MDBX_node *node; - bool exact; -}; +MDBX_INTERNAL dpl_t *dpl_reserve(MDBX_txn *txn, size_t size); -static struct node_result node_search(MDBX_cursor *mc, const MDBX_val *key); - -static int __must_check_result node_add_branch(MDBX_cursor *mc, size_t indx, - const MDBX_val *key, - pgno_t pgno); -static int __must_check_result node_add_leaf(MDBX_cursor *mc, size_t indx, - const MDBX_val *key, - MDBX_val *data, unsigned flags); -static int __must_check_result node_add_leaf2(MDBX_cursor *mc, size_t indx, - const MDBX_val *key); - -static void node_del(MDBX_cursor *mc, size_t ksize); -static MDBX_node *node_shrink(MDBX_page *mp, size_t indx, MDBX_node *node); -static int __must_check_result node_move(MDBX_cursor *csrc, MDBX_cursor *cdst, - bool fromleft); -static int __must_check_result node_read(MDBX_cursor *mc, const MDBX_node *leaf, - MDBX_val *data, const MDBX_page *mp); -static int __must_check_result rebalance(MDBX_cursor *mc); -static int __must_check_result update_key(MDBX_cursor *mc, const MDBX_val *key); - -static void cursor_pop(MDBX_cursor *mc); -static int __must_check_result cursor_push(MDBX_cursor *mc, MDBX_page *mp); - -static int __must_check_result audit_ex(MDBX_txn *txn, size_t retired_stored, - bool dont_filter_gc); - -static int __must_check_result page_check(const MDBX_cursor *const mc, - const MDBX_page *const mp); -static int __must_check_result cursor_check(const MDBX_cursor *mc); -static int __must_check_result cursor_get(MDBX_cursor *mc, MDBX_val *key, - MDBX_val *data, MDBX_cursor_op op); -static int __must_check_result cursor_put_checklen(MDBX_cursor *mc, - const MDBX_val *key, - MDBX_val *data, - unsigned flags); -static int __must_check_result cursor_put_nochecklen(MDBX_cursor *mc, - const MDBX_val *key, - MDBX_val *data, - unsigned flags); -static int __must_check_result cursor_check_updating(MDBX_cursor *mc); -static int __must_check_result cursor_del(MDBX_cursor *mc, - MDBX_put_flags_t flags); -static int __must_check_result delete(MDBX_txn *txn, MDBX_dbi dbi, - const MDBX_val *key, const MDBX_val *data, - unsigned flags); -#define SIBLING_LEFT 0 -#define SIBLING_RIGHT 2 -static int __must_check_result cursor_sibling(MDBX_cursor *mc, int dir); -static int __must_check_result cursor_next(MDBX_cursor *mc, MDBX_val *key, - MDBX_val *data, MDBX_cursor_op op); -static int __must_check_result cursor_prev(MDBX_cursor *mc, MDBX_val *key, - MDBX_val *data, MDBX_cursor_op op); -struct cursor_set_result { - int err; - bool exact; -}; +MDBX_INTERNAL __noinline dpl_t *dpl_sort_slowpath(const MDBX_txn *txn); -static struct cursor_set_result cursor_set(MDBX_cursor *mc, MDBX_val *key, - MDBX_val *data, MDBX_cursor_op op); -static int __must_check_result cursor_first(MDBX_cursor *mc, MDBX_val *key, - MDBX_val *data); -static int __must_check_result cursor_last(MDBX_cursor *mc, MDBX_val *key, - MDBX_val *data); - -static int __must_check_result cursor_init(MDBX_cursor *mc, const MDBX_txn *txn, - size_t dbi); -static int __must_check_result cursor_xinit0(MDBX_cursor *mc); -static int __must_check_result cursor_xinit1(MDBX_cursor *mc, MDBX_node *node, - const MDBX_page *mp); -static int __must_check_result cursor_xinit2(MDBX_cursor *mc, - MDBX_xcursor *src_mx, - bool new_dupdata); -static void cursor_copy(const MDBX_cursor *csrc, MDBX_cursor *cdst); - -static int __must_check_result drop_tree(MDBX_cursor *mc, - const bool may_have_subDBs); -static int __must_check_result fetch_sdb(MDBX_txn *txn, size_t dbi); -static int __must_check_result setup_dbx(MDBX_dbx *const dbx, - const MDBX_db *const db, - const unsigned pagesize); - -static __inline MDBX_cmp_func *get_default_keycmp(MDBX_db_flags_t flags); -static __inline MDBX_cmp_func *get_default_datacmp(MDBX_db_flags_t flags); +static inline dpl_t *dpl_sort(const MDBX_txn *txn) { + tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0); + tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); -__cold const char *mdbx_liberr2str(int errnum) { - /* Table of descriptions for MDBX errors */ - static const char *const tbl[] = { - "MDBX_KEYEXIST: Key/data pair already exists", - "MDBX_NOTFOUND: No matching key/data pair found", - "MDBX_PAGE_NOTFOUND: Requested page not found", - "MDBX_CORRUPTED: Database is corrupted", - "MDBX_PANIC: Environment had fatal error", - "MDBX_VERSION_MISMATCH: DB version mismatch libmdbx", - "MDBX_INVALID: File is not an MDBX file", - "MDBX_MAP_FULL: Environment mapsize limit reached", - "MDBX_DBS_FULL: Too many DBI-handles (maxdbs reached)", - "MDBX_READERS_FULL: Too many readers (maxreaders reached)", - NULL /* MDBX_TLS_FULL (-30789): unused in MDBX */, - "MDBX_TXN_FULL: Transaction has too many dirty pages," - " i.e transaction is too big", - "MDBX_CURSOR_FULL: Cursor stack limit reachedn - this usually indicates" - " corruption, i.e branch-pages loop", - "MDBX_PAGE_FULL: Internal error - Page has no more space", - "MDBX_UNABLE_EXTEND_MAPSIZE: Database engine was unable to extend" - " mapping, e.g. since address space is unavailable or busy," - " or Operation system not supported such operations", - "MDBX_INCOMPATIBLE: Environment or database is not compatible" - " with the requested operation or the specified flags", - "MDBX_BAD_RSLOT: Invalid reuse of reader locktable slot," - " e.g. read-transaction already run for current thread", - "MDBX_BAD_TXN: Transaction is not valid for requested operation," - " e.g. had errored and be must aborted, has a child, or is invalid", - "MDBX_BAD_VALSIZE: Invalid size or alignment of key or data" - " for target database, either invalid subDB name", - "MDBX_BAD_DBI: The specified DBI-handle is invalid" - " or changed by another thread/transaction", - "MDBX_PROBLEM: Unexpected internal error, transaction should be aborted", - "MDBX_BUSY: Another write transaction is running," - " or environment is already used while opening with MDBX_EXCLUSIVE flag", - }; + dpl_t *dl = txn->tw.dirtylist; + tASSERT(txn, dl->length <= PAGELIST_LIMIT); + tASSERT(txn, dl->sorted <= dl->length); + tASSERT(txn, dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID); + return likely(dl->sorted == dl->length) ? dl : dpl_sort_slowpath(txn); +} - if (errnum >= MDBX_KEYEXIST && errnum <= MDBX_BUSY) { - int i = errnum - MDBX_KEYEXIST; - return tbl[i]; - } +MDBX_INTERNAL __noinline size_t dpl_search(const MDBX_txn *txn, pgno_t pgno); - switch (errnum) { - case MDBX_SUCCESS: - return "MDBX_SUCCESS: Successful"; - case MDBX_EMULTIVAL: - return "MDBX_EMULTIVAL: The specified key has" - " more than one associated value"; - case MDBX_EBADSIGN: - return "MDBX_EBADSIGN: Wrong signature of a runtime object(s)," - " e.g. memory corruption or double-free"; - case MDBX_WANNA_RECOVERY: - return "MDBX_WANNA_RECOVERY: Database should be recovered," - " but this could NOT be done automatically for now" - " since it opened in read-only mode"; - case MDBX_EKEYMISMATCH: - return "MDBX_EKEYMISMATCH: The given key value is mismatched to the" - " current cursor position"; - case MDBX_TOO_LARGE: - return "MDBX_TOO_LARGE: Database is too large for current system," - " e.g. could NOT be mapped into RAM"; - case MDBX_THREAD_MISMATCH: - return "MDBX_THREAD_MISMATCH: A thread has attempted to use a not" - " owned object, e.g. a transaction that started by another thread"; - case MDBX_TXN_OVERLAPPING: - return "MDBX_TXN_OVERLAPPING: Overlapping read and write transactions for" - " the current thread"; - case MDBX_DUPLICATED_CLK: - return "MDBX_DUPLICATED_CLK: Alternative/Duplicate LCK-file is exists, " - "please keep one and remove unused other"; - default: - return NULL; - } +MDBX_MAYBE_UNUSED MDBX_INTERNAL const page_t *debug_dpl_find(const MDBX_txn *txn, const pgno_t pgno); + +MDBX_NOTHROW_PURE_FUNCTION static inline unsigned dpl_npages(const dpl_t *dl, size_t i) { + assert(0 <= (intptr_t)i && i <= dl->length); + unsigned n = dl->items[i].npages; + assert(n == (is_largepage(dl->items[i].ptr) ? dl->items[i].ptr->pages : 1)); + return n; } -__cold const char *mdbx_strerror_r(int errnum, char *buf, size_t buflen) { - const char *msg = mdbx_liberr2str(errnum); - if (!msg && buflen > 0 && buflen < INT_MAX) { -#if defined(_WIN32) || defined(_WIN64) - DWORD size = FormatMessageA( - FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, NULL, - errnum, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), buf, (DWORD)buflen, - NULL); - while (size && buf[size - 1] <= ' ') - --size; - buf[size] = 0; - return size ? buf : "FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM) failed"; -#elif defined(_GNU_SOURCE) && defined(__GLIBC__) - /* GNU-specific */ - if (errnum > 0) - msg = strerror_r(errnum, buf, buflen); -#elif (_POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600) - /* XSI-compliant */ - if (errnum > 0 && strerror_r(errnum, buf, buflen) == 0) - msg = buf; -#else - if (errnum > 0) { - msg = strerror(errnum); - if (msg) { - strncpy(buf, msg, buflen); - msg = buf; - } - } -#endif - if (!msg) { - (void)snprintf(buf, buflen, "error %d", errnum); - msg = buf; - } - buf[buflen - 1] = '\0'; - } - return msg; +MDBX_NOTHROW_PURE_FUNCTION static inline pgno_t dpl_endpgno(const dpl_t *dl, size_t i) { + return dpl_npages(dl, i) + dl->items[i].pgno; } -__cold const char *mdbx_strerror(int errnum) { -#if defined(_WIN32) || defined(_WIN64) - static char buf[1024]; - return mdbx_strerror_r(errnum, buf, sizeof(buf)); -#else - const char *msg = mdbx_liberr2str(errnum); - if (!msg) { - if (errnum > 0) - msg = strerror(errnum); - if (!msg) { - static char buf[32]; - (void)snprintf(buf, sizeof(buf) - 1, "error %d", errnum); - msg = buf; +static inline bool dpl_intersect(const MDBX_txn *txn, pgno_t pgno, size_t npages) { + tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0); + tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); + + dpl_t *dl = txn->tw.dirtylist; + tASSERT(txn, dl->sorted == dl->length); + tASSERT(txn, dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID); + size_t const n = dpl_search(txn, pgno); + tASSERT(txn, n >= 1 && n <= dl->length + 1); + tASSERT(txn, pgno <= dl->items[n].pgno); + tASSERT(txn, pgno > dl->items[n - 1].pgno); + const bool rc = + /* intersection with founded */ pgno + npages > dl->items[n].pgno || + /* intersection with prev */ dpl_endpgno(dl, n - 1) > pgno; + if (ASSERT_ENABLED()) { + bool check = false; + for (size_t i = 1; i <= dl->length; ++i) { + const page_t *const dp = dl->items[i].ptr; + if (!(dp->pgno /* begin */ >= /* end */ pgno + npages || dpl_endpgno(dl, i) /* end */ <= /* begin */ pgno)) + check |= true; } + tASSERT(txn, check == rc); } - return msg; -#endif + return rc; } -#if defined(_WIN32) || defined(_WIN64) /* Bit of madness for Windows */ -const char *mdbx_strerror_r_ANSI2OEM(int errnum, char *buf, size_t buflen) { - const char *msg = mdbx_liberr2str(errnum); - if (!msg && buflen > 0 && buflen < INT_MAX) { - DWORD size = FormatMessageA( - FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, NULL, - errnum, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), buf, (DWORD)buflen, - NULL); - while (size && buf[size - 1] <= ' ') - --size; - buf[size] = 0; - if (!size) - msg = "FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM) failed"; - else if (!CharToOemBuffA(buf, buf, size)) - msg = "CharToOemBuffA() failed"; - else - msg = buf; - } - return msg; +MDBX_NOTHROW_PURE_FUNCTION static inline size_t dpl_exist(const MDBX_txn *txn, pgno_t pgno) { + tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); + dpl_t *dl = txn->tw.dirtylist; + size_t i = dpl_search(txn, pgno); + tASSERT(txn, (int)i > 0); + return (dl->items[i].pgno == pgno) ? i : 0; } -const char *mdbx_strerror_ANSI2OEM(int errnum) { - static char buf[1024]; - return mdbx_strerror_r_ANSI2OEM(errnum, buf, sizeof(buf)); -} -#endif /* Bit of madness for Windows */ +MDBX_INTERNAL void dpl_remove_ex(const MDBX_txn *txn, size_t i, size_t npages); -__cold void debug_log_va(int level, const char *function, int line, - const char *fmt, va_list args) { - if (debug_logger) - debug_logger(level, function, line, fmt, args); - else { -#if defined(_WIN32) || defined(_WIN64) - if (IsDebuggerPresent()) { - int prefix_len = 0; - char *prefix = nullptr; - if (function && line > 0) - prefix_len = osal_asprintf(&prefix, "%s:%d ", function, line); - else if (function) - prefix_len = osal_asprintf(&prefix, "%s: ", function); - else if (line > 0) - prefix_len = osal_asprintf(&prefix, "%d: ", line); - if (prefix_len > 0 && prefix) { - OutputDebugStringA(prefix); - osal_free(prefix); - } - char *msg = nullptr; - int msg_len = osal_vasprintf(&msg, fmt, args); - if (msg_len > 0 && msg) { - OutputDebugStringA(msg); - osal_free(msg); - } - } -#else - if (function && line > 0) - fprintf(stderr, "%s:%d ", function, line); - else if (function) - fprintf(stderr, "%s: ", function); - else if (line > 0) - fprintf(stderr, "%d: ", line); - vfprintf(stderr, fmt, args); - fflush(stderr); -#endif - } -} - -__cold void debug_log(int level, const char *function, int line, - const char *fmt, ...) { - va_list args; - va_start(args, fmt); - debug_log_va(level, function, line, fmt, args); - va_end(args); +static inline void dpl_remove(const MDBX_txn *txn, size_t i) { + dpl_remove_ex(txn, i, dpl_npages(txn->tw.dirtylist, i)); } -/* Dump a val in ascii or hexadecimal. */ -const char *mdbx_dump_val(const MDBX_val *val, char *const buf, - const size_t bufsize) { - if (!val) - return ""; - if (!val->iov_len) - return ""; - if (!buf || bufsize < 4) - return nullptr; - - if (!val->iov_base) { - int len = snprintf(buf, bufsize, "", val->iov_len); - assert(len > 0 && (size_t)len < bufsize); - (void)len; - return buf; - } +MDBX_INTERNAL int __must_check_result dpl_append(MDBX_txn *txn, pgno_t pgno, page_t *page, size_t npages); - bool is_ascii = true; - const uint8_t *const data = val->iov_base; - for (size_t i = 0; i < val->iov_len; i++) - if (data[i] < ' ' || data[i] > '~') { - is_ascii = false; - break; - } +MDBX_MAYBE_UNUSED MDBX_INTERNAL bool dpl_check(MDBX_txn *txn); - if (is_ascii) { - int len = - snprintf(buf, bufsize, "%.*s", - (val->iov_len > INT_MAX) ? INT_MAX : (int)val->iov_len, data); - assert(len > 0 && (size_t)len < bufsize); - (void)len; - } else { - char *const detent = buf + bufsize - 2; - char *ptr = buf; - *ptr++ = '<'; - for (size_t i = 0; i < val->iov_len && ptr < detent; i++) { - const char hex[16] = {'0', '1', '2', '3', '4', '5', '6', '7', - '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'}; - *ptr++ = hex[data[i] >> 4]; - *ptr++ = hex[data[i] & 15]; - } - if (ptr < detent) - *ptr++ = '>'; - *ptr = '\0'; - } - return buf; +MDBX_NOTHROW_PURE_FUNCTION static inline uint32_t dpl_age(const MDBX_txn *txn, size_t i) { + tASSERT(txn, (txn->flags & (MDBX_TXN_RDONLY | MDBX_WRITEMAP)) == 0); + const dpl_t *dl = txn->tw.dirtylist; + assert((intptr_t)i > 0 && i <= dl->length); + size_t *const ptr = ptr_disp(dl->items[i].ptr, -(ptrdiff_t)sizeof(size_t)); + return txn->tw.dirtylru - (uint32_t)*ptr; } -/*------------------------------------------------------------------------------ - LY: debug stuff */ +MDBX_INTERNAL void dpl_lru_reduce(MDBX_txn *txn); -static const char *leafnode_type(MDBX_node *n) { - static const char *const tp[2][2] = {{"", ": DB"}, - {": sub-page", ": sub-DB"}}; - return (node_flags(n) & F_BIGDATA) - ? ": large page" - : tp[!!(node_flags(n) & F_DUPDATA)][!!(node_flags(n) & F_SUBDATA)]; +static inline uint32_t dpl_lru_turn(MDBX_txn *txn) { + txn->tw.dirtylru += 1; + if (unlikely(txn->tw.dirtylru > UINT32_MAX / 3) && (txn->flags & MDBX_WRITEMAP) == 0) + dpl_lru_reduce(txn); + return txn->tw.dirtylru; } -/* Display all the keys in the page. */ -MDBX_MAYBE_UNUSED static void page_list(MDBX_page *mp) { - pgno_t pgno = mp->mp_pgno; - const char *type; - MDBX_node *node; - size_t i, nkeys, nsize, total = 0; - MDBX_val key; - DKBUF; +MDBX_INTERNAL void dpl_sift(MDBX_txn *const txn, pnl_t pl, const bool spilled); - switch (PAGETYPE_WHOLE(mp)) { - case P_BRANCH: - type = "Branch page"; - break; - case P_LEAF: - type = "Leaf page"; - break; - case P_LEAF | P_SUBP: - type = "Leaf sub-page"; - break; - case P_LEAF | P_LEAF2: - type = "Leaf2 page"; - break; - case P_LEAF | P_LEAF2 | P_SUBP: - type = "Leaf2 sub-page"; - break; - case P_OVERFLOW: - VERBOSE("Overflow page %" PRIaPGNO " pages %u\n", pgno, mp->mp_pages); - return; - case P_META: - VERBOSE("Meta-page %" PRIaPGNO " txnid %" PRIu64 "\n", pgno, - unaligned_peek_u64(4, page_meta(mp)->mm_txnid_a)); - return; - default: - VERBOSE("Bad page %" PRIaPGNO " flags 0x%X\n", pgno, mp->mp_flags); - return; - } +MDBX_INTERNAL void dpl_release_shadows(MDBX_txn *txn); - nkeys = page_numkeys(mp); - VERBOSE("%s %" PRIaPGNO " numkeys %zu\n", type, pgno, nkeys); +typedef struct gc_update_context { + unsigned loop; + pgno_t prev_first_unallocated; + bool dense; + size_t reserve_adj; + size_t retired_stored; + size_t amount, reserved, cleaned_slot, reused_slot, fill_idx; + txnid_t cleaned_id, rid; +#if MDBX_ENABLE_BIGFOOT + txnid_t bigfoot; +#endif /* MDBX_ENABLE_BIGFOOT */ + union { + MDBX_cursor cursor; + cursor_couple_t couple; + }; +} gcu_t; - for (i = 0; i < nkeys; i++) { - if (IS_LEAF2(mp)) { /* LEAF2 pages have no mp_ptrs[] or node headers */ - key.iov_len = nsize = mp->mp_leaf2_ksize; - key.iov_base = page_leaf2key(mp, i, nsize); - total += nsize; - VERBOSE("key %zu: nsize %zu, %s\n", i, nsize, DKEY(&key)); - continue; - } - node = page_node(mp, i); - key.iov_len = node_ks(node); - key.iov_base = node->mn_data; - nsize = NODESIZE + key.iov_len; - if (IS_BRANCH(mp)) { - VERBOSE("key %zu: page %" PRIaPGNO ", %s\n", i, node_pgno(node), - DKEY(&key)); - total += nsize; - } else { - if (node_flags(node) & F_BIGDATA) - nsize += sizeof(pgno_t); - else - nsize += node_ds(node); - total += nsize; - nsize += sizeof(indx_t); - VERBOSE("key %zu: nsize %zu, %s%s\n", i, nsize, DKEY(&key), - leafnode_type(node)); - } - total = EVEN(total); - } - VERBOSE("Total: header %zu + contents %zu + unused %zu\n", - IS_LEAF2(mp) ? PAGEHDRSZ : PAGEHDRSZ + mp->mp_lower, total, - page_room(mp)); +static inline int gc_update_init(MDBX_txn *txn, gcu_t *ctx) { + memset(ctx, 0, offsetof(gcu_t, cursor)); + ctx->dense = txn->txnid <= MIN_TXNID; +#if MDBX_ENABLE_BIGFOOT + ctx->bigfoot = txn->txnid; +#endif /* MDBX_ENABLE_BIGFOOT */ + return cursor_init(&ctx->cursor, txn, FREE_DBI); } -/*----------------------------------------------------------------------------*/ - -/* Check if there is an initialized xcursor, so XCURSOR_REFRESH() is proper */ -#define XCURSOR_INITED(mc) \ - ((mc)->mc_xcursor && ((mc)->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)) - -/* Update sub-page pointer, if any, in mc->mc_xcursor. - * Needed when the node which contains the sub-page may have moved. - * Called with mp = mc->mc_pg[mc->mc_top], ki = mc->mc_ki[mc->mc_top]. */ -#define XCURSOR_REFRESH(mc, mp, ki) \ - do { \ - MDBX_page *xr_pg = (mp); \ - MDBX_node *xr_node = page_node(xr_pg, ki); \ - if ((node_flags(xr_node) & (F_DUPDATA | F_SUBDATA)) == F_DUPDATA) \ - (mc)->mc_xcursor->mx_cursor.mc_pg[0] = node_data(xr_node); \ - } while (0) - -MDBX_MAYBE_UNUSED static bool cursor_is_tracked(const MDBX_cursor *mc) { - for (MDBX_cursor *scan = mc->mc_txn->mt_cursors[mc->mc_dbi]; scan; - scan = scan->mc_next) - if (mc == ((mc->mc_flags & C_SUB) ? &scan->mc_xcursor->mx_cursor : scan)) - return true; - return false; -} +#define ALLOC_DEFAULT 0 +#define ALLOC_RESERVE 1 +#define ALLOC_UNIMPORTANT 2 +MDBX_INTERNAL pgr_t gc_alloc_ex(const MDBX_cursor *const mc, const size_t num, uint8_t flags); -/* Perform act while tracking temporary cursor mn */ -#define WITH_CURSOR_TRACKING(mn, act) \ - do { \ - cASSERT(&(mn), \ - mn.mc_txn->mt_cursors != NULL /* must be not rdonly txt */); \ - cASSERT(&(mn), !cursor_is_tracked(&(mn))); \ - MDBX_cursor mc_dummy; \ - MDBX_cursor **tracking_head = &(mn).mc_txn->mt_cursors[mn.mc_dbi]; \ - MDBX_cursor *tracked = &(mn); \ - if ((mn).mc_flags & C_SUB) { \ - mc_dummy.mc_flags = C_INITIALIZED; \ - mc_dummy.mc_top = 0; \ - mc_dummy.mc_snum = 0; \ - mc_dummy.mc_xcursor = (MDBX_xcursor *)&(mn); \ - tracked = &mc_dummy; \ - } \ - tracked->mc_next = *tracking_head; \ - *tracking_head = tracked; \ - { \ - act; \ - } \ - *tracking_head = tracked->mc_next; \ - } while (0) +MDBX_INTERNAL pgr_t gc_alloc_single(const MDBX_cursor *const mc); +MDBX_INTERNAL int gc_update(MDBX_txn *txn, gcu_t *ctx); -int mdbx_cmp(const MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *a, - const MDBX_val *b) { - eASSERT(NULL, txn->mt_signature == MDBX_MT_SIGNATURE); - return txn->mt_dbxs[dbi].md_cmp(a, b); -} +MDBX_INTERNAL int lck_setup(MDBX_env *env, mdbx_mode_t mode); +#if MDBX_LOCKING > MDBX_LOCKING_SYSV +MDBX_INTERNAL int lck_ipclock_stubinit(osal_ipclock_t *ipc); +MDBX_INTERNAL int lck_ipclock_destroy(osal_ipclock_t *ipc); +#endif /* MDBX_LOCKING > MDBX_LOCKING_SYSV */ -int mdbx_dcmp(const MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *a, - const MDBX_val *b) { - eASSERT(NULL, txn->mt_signature == MDBX_MT_SIGNATURE); - return txn->mt_dbxs[dbi].md_dcmp(a, b); -} +MDBX_INTERNAL int lck_init(MDBX_env *env, MDBX_env *inprocess_neighbor, int global_uniqueness_flag); -/* Allocate memory for a page. - * Re-use old malloc'ed pages first for singletons, otherwise just malloc. - * Set MDBX_TXN_ERROR on failure. */ -static MDBX_page *page_malloc(MDBX_txn *txn, size_t num) { - MDBX_env *env = txn->mt_env; - MDBX_page *np = env->me_dp_reserve; - size_t size = env->me_psize; - if (likely(num == 1 && np)) { - eASSERT(env, env->me_dp_reserve_len > 0); - MDBX_ASAN_UNPOISON_MEMORY_REGION(np, size); - VALGRIND_MEMPOOL_ALLOC(env, ptr_disp(np, -(ptrdiff_t)sizeof(size_t)), - size + sizeof(size_t)); - VALGRIND_MAKE_MEM_DEFINED(&mp_next(np), sizeof(MDBX_page *)); - env->me_dp_reserve = mp_next(np); - env->me_dp_reserve_len -= 1; - } else { - size = pgno2bytes(env, num); - void *const ptr = osal_malloc(size + sizeof(size_t)); - if (unlikely(!ptr)) { - txn->mt_flags |= MDBX_TXN_ERROR; - return nullptr; - } - VALGRIND_MEMPOOL_ALLOC(env, ptr, size + sizeof(size_t)); - np = ptr_disp(ptr, sizeof(size_t)); - } +MDBX_INTERNAL int lck_destroy(MDBX_env *env, MDBX_env *inprocess_neighbor, const uint32_t current_pid); - if ((env->me_flags & MDBX_NOMEMINIT) == 0) { - /* For a single page alloc, we init everything after the page header. - * For multi-page, we init the final page; if the caller needed that - * many pages they will be filling in at least up to the last page. */ - size_t skip = PAGEHDRSZ; - if (num > 1) - skip += pgno2bytes(env, num - 1); - memset(ptr_disp(np, skip), 0, size - skip); - } -#if MDBX_DEBUG - np->mp_pgno = 0; -#endif - VALGRIND_MAKE_MEM_UNDEFINED(np, size); - np->mp_flags = 0; - np->mp_pages = (pgno_t)num; - return np; -} +MDBX_INTERNAL int lck_seize(MDBX_env *env); -/* Free a shadow dirty page */ -static void dpage_free(MDBX_env *env, MDBX_page *dp, size_t npages) { - VALGRIND_MAKE_MEM_UNDEFINED(dp, pgno2bytes(env, npages)); - MDBX_ASAN_UNPOISON_MEMORY_REGION(dp, pgno2bytes(env, npages)); - if (unlikely(env->me_flags & MDBX_PAGEPERTURB)) - memset(dp, -1, pgno2bytes(env, npages)); - if (npages == 1 && - env->me_dp_reserve_len < env->me_options.dp_reserve_limit) { - MDBX_ASAN_POISON_MEMORY_REGION(dp, env->me_psize); - MDBX_ASAN_UNPOISON_MEMORY_REGION(&mp_next(dp), sizeof(MDBX_page *)); - mp_next(dp) = env->me_dp_reserve; - VALGRIND_MEMPOOL_FREE(env, ptr_disp(dp, -(ptrdiff_t)sizeof(size_t))); - env->me_dp_reserve = dp; - env->me_dp_reserve_len += 1; - } else { - /* large pages just get freed directly */ - void *const ptr = ptr_disp(dp, -(ptrdiff_t)sizeof(size_t)); - VALGRIND_MEMPOOL_FREE(env, ptr); - osal_free(ptr); - } -} +MDBX_INTERNAL int lck_downgrade(MDBX_env *env); -/* Return all dirty pages to dpage list */ -static void dlist_free(MDBX_txn *txn) { - tASSERT(txn, (txn->mt_flags & (MDBX_TXN_RDONLY | MDBX_WRITEMAP)) == 0); - MDBX_env *env = txn->mt_env; - MDBX_dpl *const dl = txn->tw.dirtylist; +MDBX_MAYBE_UNUSED MDBX_INTERNAL int lck_upgrade(MDBX_env *env, bool dont_wait); - for (size_t i = 1; i <= dl->length; i++) - dpage_free(env, dl->items[i].ptr, dpl_npages(dl, i)); +MDBX_INTERNAL int lck_rdt_lock(MDBX_env *env); - dpl_clear(dl); -} +MDBX_INTERNAL void lck_rdt_unlock(MDBX_env *env); -static __always_inline MDBX_db *outer_db(MDBX_cursor *mc) { - cASSERT(mc, (mc->mc_flags & C_SUB) != 0); - MDBX_xcursor *mx = container_of(mc->mc_db, MDBX_xcursor, mx_db); - MDBX_cursor_couple *couple = container_of(mx, MDBX_cursor_couple, inner); - cASSERT(mc, mc->mc_db == &couple->outer.mc_xcursor->mx_db); - cASSERT(mc, mc->mc_dbx == &couple->outer.mc_xcursor->mx_dbx); - return couple->outer.mc_db; -} +MDBX_INTERNAL int lck_txn_lock(MDBX_env *env, bool dont_wait); -MDBX_MAYBE_UNUSED __cold static bool dirtylist_check(MDBX_txn *txn) { - tASSERT(txn, (txn->mt_flags & MDBX_TXN_RDONLY) == 0); - const MDBX_dpl *const dl = txn->tw.dirtylist; - if (!dl) { - tASSERT(txn, (txn->mt_flags & MDBX_WRITEMAP) != 0 && !MDBX_AVOID_MSYNC); - return true; - } - tASSERT(txn, (txn->mt_flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); +MDBX_INTERNAL void lck_txn_unlock(MDBX_env *env); - assert(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID); - tASSERT(txn, txn->tw.dirtyroom + dl->length == - (txn->mt_parent ? txn->mt_parent->tw.dirtyroom - : txn->mt_env->me_options.dp_limit)); +MDBX_INTERNAL int lck_rpid_set(MDBX_env *env); - if (!AUDIT_ENABLED()) - return true; +MDBX_INTERNAL int lck_rpid_clear(MDBX_env *env); - size_t loose = 0, pages = 0; - for (size_t i = dl->length; i > 0; --i) { - const MDBX_page *const dp = dl->items[i].ptr; - if (!dp) - continue; +MDBX_INTERNAL int lck_rpid_check(MDBX_env *env, uint32_t pid); - tASSERT(txn, dp->mp_pgno == dl->items[i].pgno); - if (unlikely(dp->mp_pgno != dl->items[i].pgno)) - return false; +static inline uint64_t meta_sign_calculate(const meta_t *meta) { + uint64_t sign = DATASIGN_NONE; +#if 0 /* TODO */ + sign = hippeus_hash64(...); +#else + (void)meta; +#endif + /* LY: newer returns DATASIGN_NONE or DATASIGN_WEAK */ + return (sign > DATASIGN_WEAK) ? sign : ~sign; +} - if ((txn->mt_flags & MDBX_WRITEMAP) == 0) { - const uint32_t age = dpl_age(txn, i); - tASSERT(txn, age < UINT32_MAX / 3); - if (unlikely(age > UINT32_MAX / 3)) - return false; - } +static inline uint64_t meta_sign_get(const volatile meta_t *meta) { return unaligned_peek_u64_volatile(4, meta->sign); } - tASSERT(txn, dp->mp_flags == P_LOOSE || IS_MODIFIABLE(txn, dp)); - if (dp->mp_flags == P_LOOSE) { - loose += 1; - } else if (unlikely(!IS_MODIFIABLE(txn, dp))) - return false; +static inline void meta_sign_as_steady(meta_t *meta) { unaligned_poke_u64(4, meta->sign, meta_sign_calculate(meta)); } - const unsigned num = dpl_npages(dl, i); - pages += num; - tASSERT(txn, txn->mt_next_pgno >= dp->mp_pgno + num); - if (unlikely(txn->mt_next_pgno < dp->mp_pgno + num)) - return false; +static inline bool meta_is_steady(const volatile meta_t *meta) { return SIGN_IS_STEADY(meta_sign_get(meta)); } - if (i < dl->sorted) { - tASSERT(txn, dl->items[i + 1].pgno >= dp->mp_pgno + num); - if (unlikely(dl->items[i + 1].pgno < dp->mp_pgno + num)) - return false; - } +MDBX_INTERNAL troika_t meta_tap(const MDBX_env *env); +MDBX_INTERNAL unsigned meta_eq_mask(const troika_t *troika); +MDBX_INTERNAL bool meta_should_retry(const MDBX_env *env, troika_t *troika); +MDBX_MAYBE_UNUSED MDBX_INTERNAL bool troika_verify_fsm(void); - const size_t rpa = - pnl_search(txn->tw.relist, dp->mp_pgno, txn->mt_next_pgno); - tASSERT(txn, rpa > MDBX_PNL_GETSIZE(txn->tw.relist) || - txn->tw.relist[rpa] != dp->mp_pgno); - if (rpa <= MDBX_PNL_GETSIZE(txn->tw.relist) && - unlikely(txn->tw.relist[rpa] == dp->mp_pgno)) - return false; - if (num > 1) { - const size_t rpb = - pnl_search(txn->tw.relist, dp->mp_pgno + num - 1, txn->mt_next_pgno); - tASSERT(txn, rpa == rpb); - if (unlikely(rpa != rpb)) - return false; - } - } +struct meta_ptr { + txnid_t txnid; + union { + const volatile meta_t *ptr_v; + const meta_t *ptr_c; + }; + size_t is_steady; +}; - tASSERT(txn, loose == txn->tw.loose_count); - if (unlikely(loose != txn->tw.loose_count)) - return false; +MDBX_INTERNAL meta_ptr_t meta_ptr(const MDBX_env *env, unsigned n); +MDBX_INTERNAL txnid_t meta_txnid(const volatile meta_t *meta); +MDBX_INTERNAL txnid_t recent_committed_txnid(const MDBX_env *env); +MDBX_INTERNAL int meta_sync(const MDBX_env *env, const meta_ptr_t head); - tASSERT(txn, pages == dl->pages_including_loose); - if (unlikely(pages != dl->pages_including_loose)) - return false; +MDBX_INTERNAL const char *durable_caption(const meta_t *const meta); +MDBX_INTERNAL void meta_troika_dump(const MDBX_env *env, const troika_t *troika); - for (size_t i = 1; i <= MDBX_PNL_GETSIZE(txn->tw.retired_pages); ++i) { - const MDBX_page *const dp = debug_dpl_find(txn, txn->tw.retired_pages[i]); - tASSERT(txn, !dp); - if (unlikely(dp)) - return false; - } +#define METAPAGE(env, n) page_meta(pgno2page(env, n)) +#define METAPAGE_END(env) METAPAGE(env, NUM_METAS) - return true; +static inline meta_ptr_t meta_recent(const MDBX_env *env, const troika_t *troika) { + meta_ptr_t r; + r.txnid = troika->txnid[troika->recent]; + r.ptr_v = METAPAGE(env, troika->recent); + r.is_steady = (troika->fsm >> troika->recent) & 1; + return r; } -#if MDBX_ENABLE_REFUND -static void refund_reclaimed(MDBX_txn *txn) { - /* Scanning in descend order */ - pgno_t next_pgno = txn->mt_next_pgno; - const MDBX_PNL pnl = txn->tw.relist; - tASSERT(txn, MDBX_PNL_GETSIZE(pnl) && MDBX_PNL_MOST(pnl) == next_pgno - 1); -#if MDBX_PNL_ASCENDING - size_t i = MDBX_PNL_GETSIZE(pnl); - tASSERT(txn, pnl[i] == next_pgno - 1); - while (--next_pgno, --i > 0 && pnl[i] == next_pgno - 1) - ; - MDBX_PNL_SETSIZE(pnl, i); -#else - size_t i = 1; - tASSERT(txn, pnl[i] == next_pgno - 1); - size_t len = MDBX_PNL_GETSIZE(pnl); - while (--next_pgno, ++i <= len && pnl[i] == next_pgno - 1) - ; - MDBX_PNL_SETSIZE(pnl, len -= i - 1); - for (size_t move = 0; move < len; ++move) - pnl[1 + move] = pnl[i + move]; -#endif - VERBOSE("refunded %" PRIaPGNO " pages: %" PRIaPGNO " -> %" PRIaPGNO, - txn->mt_next_pgno - next_pgno, txn->mt_next_pgno, next_pgno); - txn->mt_next_pgno = next_pgno; - tASSERT(txn, pnl_check_allocated(txn->tw.relist, txn->mt_next_pgno - 1)); +static inline meta_ptr_t meta_prefer_steady(const MDBX_env *env, const troika_t *troika) { + meta_ptr_t r; + r.txnid = troika->txnid[troika->prefer_steady]; + r.ptr_v = METAPAGE(env, troika->prefer_steady); + r.is_steady = (troika->fsm >> troika->prefer_steady) & 1; + return r; } -static void refund_loose(MDBX_txn *txn) { - tASSERT(txn, txn->tw.loose_pages != nullptr); - tASSERT(txn, txn->tw.loose_count > 0); +static inline meta_ptr_t meta_tail(const MDBX_env *env, const troika_t *troika) { + const uint8_t tail = troika->tail_and_flags & 3; + MDBX_ANALYSIS_ASSUME(tail < NUM_METAS); + meta_ptr_t r; + r.txnid = troika->txnid[tail]; + r.ptr_v = METAPAGE(env, tail); + r.is_steady = (troika->fsm >> tail) & 1; + return r; +} - MDBX_dpl *const dl = txn->tw.dirtylist; - if (dl) { - tASSERT(txn, dl->length >= txn->tw.loose_count); - tASSERT(txn, (txn->mt_flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); - } else { - tASSERT(txn, (txn->mt_flags & MDBX_WRITEMAP) != 0 && !MDBX_AVOID_MSYNC); - } +static inline bool meta_is_used(const troika_t *troika, unsigned n) { + return n == troika->recent || n == troika->prefer_steady; +} - pgno_t onstack[MDBX_CACHELINE_SIZE * 8 / sizeof(pgno_t)]; - MDBX_PNL suitable = onstack; +static inline bool meta_bootid_match(const meta_t *meta) { - if (!dl || dl->length - dl->sorted > txn->tw.loose_count) { - /* Dirty list is useless since unsorted. */ - if (pnl_bytes2size(sizeof(onstack)) < txn->tw.loose_count) { - suitable = pnl_alloc(txn->tw.loose_count); - if (unlikely(!suitable)) - return /* this is not a reason for transaction fail */; - } + return memcmp(&meta->bootid, &globals.bootid, 16) == 0 && (globals.bootid.x | globals.bootid.y) != 0; +} - /* Collect loose-pages which may be refunded. */ - tASSERT(txn, txn->mt_next_pgno >= MIN_PAGENO + txn->tw.loose_count); - pgno_t most = MIN_PAGENO; - size_t w = 0; - for (const MDBX_page *lp = txn->tw.loose_pages; lp; lp = mp_next(lp)) { - tASSERT(txn, lp->mp_flags == P_LOOSE); - tASSERT(txn, txn->mt_next_pgno > lp->mp_pgno); - if (likely(txn->mt_next_pgno - txn->tw.loose_count <= lp->mp_pgno)) { - tASSERT(txn, - w < ((suitable == onstack) ? pnl_bytes2size(sizeof(onstack)) - : MDBX_PNL_ALLOCLEN(suitable))); - suitable[++w] = lp->mp_pgno; - most = (lp->mp_pgno > most) ? lp->mp_pgno : most; - } - MDBX_ASAN_UNPOISON_MEMORY_REGION(&mp_next(lp), sizeof(MDBX_page *)); - VALGRIND_MAKE_MEM_DEFINED(&mp_next(lp), sizeof(MDBX_page *)); - } +static inline bool meta_weak_acceptable(const MDBX_env *env, const meta_t *meta, const int lck_exclusive) { + return lck_exclusive + ? /* exclusive lock */ meta_bootid_match(meta) + : /* db already opened */ env->lck_mmap.lck && (env->lck_mmap.lck->envmode.weak & MDBX_RDONLY) == 0; +} - if (most + 1 == txn->mt_next_pgno) { - /* Sort suitable list and refund pages at the tail. */ - MDBX_PNL_SETSIZE(suitable, w); - pnl_sort(suitable, MAX_PAGENO + 1); +MDBX_NOTHROW_PURE_FUNCTION static inline txnid_t constmeta_txnid(const meta_t *meta) { + const txnid_t a = unaligned_peek_u64(4, &meta->txnid_a); + const txnid_t b = unaligned_peek_u64(4, &meta->txnid_b); + return likely(a == b) ? a : 0; +} - /* Scanning in descend order */ - const intptr_t step = MDBX_PNL_ASCENDING ? -1 : 1; - const intptr_t begin = - MDBX_PNL_ASCENDING ? MDBX_PNL_GETSIZE(suitable) : 1; - const intptr_t end = - MDBX_PNL_ASCENDING ? 0 : MDBX_PNL_GETSIZE(suitable) + 1; - tASSERT(txn, suitable[begin] >= suitable[end - step]); - tASSERT(txn, most == suitable[begin]); +static inline void meta_update_begin(const MDBX_env *env, meta_t *meta, txnid_t txnid) { + eASSERT(env, meta >= METAPAGE(env, 0) && meta < METAPAGE_END(env)); + eASSERT(env, unaligned_peek_u64(4, meta->txnid_a) < txnid && unaligned_peek_u64(4, meta->txnid_b) < txnid); + (void)env; +#if (defined(__amd64__) || defined(__e2k__)) && !defined(ENABLE_UBSAN) && MDBX_UNALIGNED_OK >= 8 + atomic_store64((mdbx_atomic_uint64_t *)&meta->txnid_b, 0, mo_AcquireRelease); + atomic_store64((mdbx_atomic_uint64_t *)&meta->txnid_a, txnid, mo_AcquireRelease); +#else + atomic_store32(&meta->txnid_b[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__], 0, mo_AcquireRelease); + atomic_store32(&meta->txnid_b[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__], 0, mo_AcquireRelease); + atomic_store32(&meta->txnid_a[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__], (uint32_t)txnid, mo_AcquireRelease); + atomic_store32(&meta->txnid_a[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__], (uint32_t)(txnid >> 32), mo_AcquireRelease); +#endif +} - for (intptr_t i = begin + step; i != end; i += step) { - if (suitable[i] != most - 1) - break; - most -= 1; - } - const size_t refunded = txn->mt_next_pgno - most; - DEBUG("refund-suitable %zu pages %" PRIaPGNO " -> %" PRIaPGNO, refunded, - most, txn->mt_next_pgno); - txn->mt_next_pgno = most; - txn->tw.loose_count -= refunded; - if (dl) { - txn->tw.dirtyroom += refunded; - dl->pages_including_loose -= refunded; - assert(txn->tw.dirtyroom <= txn->mt_env->me_options.dp_limit); +static inline void meta_update_end(const MDBX_env *env, meta_t *meta, txnid_t txnid) { + eASSERT(env, meta >= METAPAGE(env, 0) && meta < METAPAGE_END(env)); + eASSERT(env, unaligned_peek_u64(4, meta->txnid_a) == txnid); + eASSERT(env, unaligned_peek_u64(4, meta->txnid_b) < txnid); + (void)env; + jitter4testing(true); + memcpy(&meta->bootid, &globals.bootid, 16); +#if (defined(__amd64__) || defined(__e2k__)) && !defined(ENABLE_UBSAN) && MDBX_UNALIGNED_OK >= 8 + atomic_store64((mdbx_atomic_uint64_t *)&meta->txnid_b, txnid, mo_AcquireRelease); +#else + atomic_store32(&meta->txnid_b[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__], (uint32_t)txnid, mo_AcquireRelease); + atomic_store32(&meta->txnid_b[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__], (uint32_t)(txnid >> 32), mo_AcquireRelease); +#endif +} - /* Filter-out dirty list */ - size_t r = 0; - w = 0; - if (dl->sorted) { - do { - if (dl->items[++r].pgno < most) { - if (++w != r) - dl->items[w] = dl->items[r]; - } - } while (r < dl->sorted); - dl->sorted = w; - } - while (r < dl->length) { - if (dl->items[++r].pgno < most) { - if (++w != r) - dl->items[w] = dl->items[r]; - } - } - dpl_setlen(dl, w); - tASSERT(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length == - (txn->mt_parent ? txn->mt_parent->tw.dirtyroom - : txn->mt_env->me_options.dp_limit)); - } - goto unlink_loose; - } - } else { - /* Dirtylist is mostly sorted, just refund loose pages at the end. */ - dpl_sort(txn); - tASSERT(txn, - dl->length < 2 || dl->items[1].pgno < dl->items[dl->length].pgno); - tASSERT(txn, dl->sorted == dl->length); +static inline void meta_set_txnid(const MDBX_env *env, meta_t *meta, const txnid_t txnid) { + eASSERT(env, !env->dxb_mmap.base || meta < METAPAGE(env, 0) || meta >= METAPAGE_END(env)); + (void)env; + /* update inconsistently since this function used ONLY for filling meta-image + * for writing, but not the actual meta-page */ + memcpy(&meta->bootid, &globals.bootid, 16); + unaligned_poke_u64(4, meta->txnid_a, txnid); + unaligned_poke_u64(4, meta->txnid_b, txnid); +} - /* Scan dirtylist tail-forward and cutoff suitable pages. */ - size_t n; - for (n = dl->length; dl->items[n].pgno == txn->mt_next_pgno - 1 && - dl->items[n].ptr->mp_flags == P_LOOSE; - --n) { - tASSERT(txn, n > 0); - MDBX_page *dp = dl->items[n].ptr; - DEBUG("refund-sorted page %" PRIaPGNO, dp->mp_pgno); - tASSERT(txn, dp->mp_pgno == dl->items[n].pgno); - txn->mt_next_pgno -= 1; - } - dpl_setlen(dl, n); +static inline uint8_t meta_cmp2int(txnid_t a, txnid_t b, uint8_t s) { + return unlikely(a == b) ? 1 * s : (a > b) ? 2 * s : 0 * s; +} - if (dl->sorted != dl->length) { - const size_t refunded = dl->sorted - dl->length; - dl->sorted = dl->length; - txn->tw.loose_count -= refunded; - txn->tw.dirtyroom += refunded; - dl->pages_including_loose -= refunded; - tASSERT(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length == - (txn->mt_parent ? txn->mt_parent->tw.dirtyroom - : txn->mt_env->me_options.dp_limit)); +static inline uint8_t meta_cmp2recent(uint8_t ab_cmp2int, bool a_steady, bool b_steady) { + assert(ab_cmp2int < 3 /* && a_steady< 2 && b_steady < 2 */); + return ab_cmp2int > 1 || (ab_cmp2int == 1 && a_steady > b_steady); +} - /* Filter-out loose chain & dispose refunded pages. */ - unlink_loose: - for (MDBX_page **link = &txn->tw.loose_pages; *link;) { - MDBX_page *dp = *link; - tASSERT(txn, dp->mp_flags == P_LOOSE); - MDBX_ASAN_UNPOISON_MEMORY_REGION(&mp_next(dp), sizeof(MDBX_page *)); - VALGRIND_MAKE_MEM_DEFINED(&mp_next(dp), sizeof(MDBX_page *)); - if (txn->mt_next_pgno > dp->mp_pgno) { - link = &mp_next(dp); - } else { - *link = mp_next(dp); - if ((txn->mt_flags & MDBX_WRITEMAP) == 0) - dpage_free(txn->mt_env, dp, 1); - } - } - } - } +static inline uint8_t meta_cmp2steady(uint8_t ab_cmp2int, bool a_steady, bool b_steady) { + assert(ab_cmp2int < 3 /* && a_steady< 2 && b_steady < 2 */); + return a_steady > b_steady || (a_steady == b_steady && ab_cmp2int > 1); +} - tASSERT(txn, dirtylist_check(txn)); - if (suitable != onstack) - pnl_free(suitable); - txn->tw.loose_refund_wl = txn->mt_next_pgno; +static inline bool meta_choice_recent(txnid_t a_txnid, bool a_steady, txnid_t b_txnid, bool b_steady) { + return meta_cmp2recent(meta_cmp2int(a_txnid, b_txnid, 1), a_steady, b_steady); } -static bool txn_refund(MDBX_txn *txn) { - const pgno_t before = txn->mt_next_pgno; +static inline bool meta_choice_steady(txnid_t a_txnid, bool a_steady, txnid_t b_txnid, bool b_steady) { + return meta_cmp2steady(meta_cmp2int(a_txnid, b_txnid, 1), a_steady, b_steady); +} - if (txn->tw.loose_pages && txn->tw.loose_refund_wl > txn->mt_next_pgno) - refund_loose(txn); +MDBX_INTERNAL meta_t *meta_init_triplet(const MDBX_env *env, void *buffer); - while (true) { - if (MDBX_PNL_GETSIZE(txn->tw.relist) == 0 || - MDBX_PNL_MOST(txn->tw.relist) != txn->mt_next_pgno - 1) - break; +MDBX_INTERNAL int meta_validate(MDBX_env *env, meta_t *const meta, const page_t *const page, const unsigned meta_number, + unsigned *guess_pagesize); - refund_reclaimed(txn); - if (!txn->tw.loose_pages || txn->tw.loose_refund_wl <= txn->mt_next_pgno) - break; +MDBX_INTERNAL int __must_check_result meta_validate_copy(MDBX_env *env, const meta_t *meta, meta_t *dest); - const pgno_t memo = txn->mt_next_pgno; - refund_loose(txn); - if (memo == txn->mt_next_pgno) - break; - } +MDBX_INTERNAL int __must_check_result meta_override(MDBX_env *env, size_t target, txnid_t txnid, const meta_t *shape); - if (before == txn->mt_next_pgno) - return false; +MDBX_INTERNAL int meta_wipe_steady(MDBX_env *env, txnid_t inclusive_upto); - if (txn->tw.spilled.list) - /* Squash deleted pagenums if we refunded any */ - spill_purge(txn); +#if !(defined(_WIN32) || defined(_WIN64)) +#define MDBX_WRITETHROUGH_THRESHOLD_DEFAULT 2 +#endif - return true; -} -#else /* MDBX_ENABLE_REFUND */ -static __inline bool txn_refund(MDBX_txn *txn) { - (void)txn; - /* No online auto-compactification. */ - return false; +struct iov_ctx { + MDBX_env *env; + osal_ioring_t *ior; + mdbx_filehandle_t fd; + int err; +#ifndef MDBX_NEED_WRITTEN_RANGE +#define MDBX_NEED_WRITTEN_RANGE 1 +#endif /* MDBX_NEED_WRITTEN_RANGE */ +#if MDBX_NEED_WRITTEN_RANGE + pgno_t flush_begin; + pgno_t flush_end; +#endif /* MDBX_NEED_WRITTEN_RANGE */ + uint64_t coherency_timestamp; +}; + +MDBX_INTERNAL __must_check_result int iov_init(MDBX_txn *const txn, iov_ctx_t *ctx, size_t items, size_t npages, + mdbx_filehandle_t fd, bool check_coherence); + +static inline bool iov_empty(const iov_ctx_t *ctx) { return osal_ioring_used(ctx->ior) == 0; } + +MDBX_INTERNAL __must_check_result int iov_page(MDBX_txn *txn, iov_ctx_t *ctx, page_t *dp, size_t npages); + +MDBX_INTERNAL __must_check_result int iov_write(iov_ctx_t *ctx); + +MDBX_INTERNAL void spill_remove(MDBX_txn *txn, size_t idx, size_t npages); +MDBX_INTERNAL pnl_t spill_purge(MDBX_txn *txn); +MDBX_INTERNAL int spill_slowpath(MDBX_txn *const txn, MDBX_cursor *const m0, const intptr_t wanna_spill_entries, + const intptr_t wanna_spill_npages, const size_t need); +/*----------------------------------------------------------------------------*/ + +static inline size_t spill_search(const MDBX_txn *txn, pgno_t pgno) { + tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); + const pnl_t pnl = txn->tw.spilled.list; + if (likely(!pnl)) + return 0; + pgno <<= 1; + size_t n = pnl_search(pnl, pgno, (size_t)MAX_PAGENO + MAX_PAGENO + 1); + return (n <= MDBX_PNL_GETSIZE(pnl) && pnl[n] == pgno) ? n : 0; } -#endif /* MDBX_ENABLE_REFUND */ -__cold static void kill_page(MDBX_txn *txn, MDBX_page *mp, pgno_t pgno, - size_t npages) { - MDBX_env *const env = txn->mt_env; - DEBUG("kill %zu page(s) %" PRIaPGNO, npages, pgno); - eASSERT(env, pgno >= NUM_METAS && npages); - if (!IS_FROZEN(txn, mp)) { - const size_t bytes = pgno2bytes(env, npages); - memset(mp, -1, bytes); - mp->mp_pgno = pgno; - if ((txn->mt_flags & MDBX_WRITEMAP) == 0) - osal_pwrite(env->me_lazy_fd, mp, bytes, pgno2bytes(env, pgno)); - } else { - struct iovec iov[MDBX_AUXILARY_IOV_MAX]; - iov[0].iov_len = env->me_psize; - iov[0].iov_base = ptr_disp(env->me_pbuf, env->me_psize); - size_t iov_off = pgno2bytes(env, pgno), n = 1; - while (--npages) { - iov[n] = iov[0]; - if (++n == MDBX_AUXILARY_IOV_MAX) { - osal_pwritev(env->me_lazy_fd, iov, MDBX_AUXILARY_IOV_MAX, iov_off); - iov_off += pgno2bytes(env, MDBX_AUXILARY_IOV_MAX); - n = 0; - } - } - osal_pwritev(env->me_lazy_fd, iov, n, iov_off); +static inline bool spill_intersect(const MDBX_txn *txn, pgno_t pgno, size_t npages) { + const pnl_t pnl = txn->tw.spilled.list; + if (likely(!pnl)) + return false; + const size_t len = MDBX_PNL_GETSIZE(pnl); + if (LOG_ENABLED(MDBX_LOG_EXTRA)) { + DEBUG_EXTRA("PNL len %zu [", len); + for (size_t i = 1; i <= len; ++i) + DEBUG_EXTRA_PRINT(" %li", (pnl[i] & 1) ? -(long)(pnl[i] >> 1) : (long)(pnl[i] >> 1)); + DEBUG_EXTRA_PRINT("%s\n", "]"); + } + const pgno_t spilled_range_begin = pgno << 1; + const pgno_t spilled_range_last = ((pgno + (pgno_t)npages) << 1) - 1; +#if MDBX_PNL_ASCENDING + const size_t n = pnl_search(pnl, spilled_range_begin, (size_t)(MAX_PAGENO + 1) << 1); + tASSERT(txn, n && (n == MDBX_PNL_GETSIZE(pnl) + 1 || spilled_range_begin <= pnl[n])); + const bool rc = n <= MDBX_PNL_GETSIZE(pnl) && pnl[n] <= spilled_range_last; +#else + const size_t n = pnl_search(pnl, spilled_range_last, (size_t)MAX_PAGENO + MAX_PAGENO + 1); + tASSERT(txn, n && (n == MDBX_PNL_GETSIZE(pnl) + 1 || spilled_range_last >= pnl[n])); + const bool rc = n <= MDBX_PNL_GETSIZE(pnl) && pnl[n] >= spilled_range_begin; +#endif + if (ASSERT_ENABLED()) { + bool check = false; + for (size_t i = 0; i < npages; ++i) + check |= spill_search(txn, (pgno_t)(pgno + i)) != 0; + tASSERT(txn, check == rc); } + return rc; } -/* Remove page from dirty list, etc */ -static __inline void page_wash(MDBX_txn *txn, size_t di, MDBX_page *const mp, - const size_t npages) { - tASSERT(txn, (txn->mt_flags & MDBX_TXN_RDONLY) == 0); - mp->mp_txnid = INVALID_TXNID; - mp->mp_flags = P_BAD; +static inline int txn_spill(MDBX_txn *const txn, MDBX_cursor *const m0, const size_t need) { + tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0); + tASSERT(txn, !m0 || cursor_is_tracked(m0)); + + const intptr_t wanna_spill_entries = txn->tw.dirtylist ? (need - txn->tw.dirtyroom - txn->tw.loose_count) : 0; + const intptr_t wanna_spill_npages = + need + (txn->tw.dirtylist ? txn->tw.dirtylist->pages_including_loose : txn->tw.writemap_dirty_npages) - + txn->tw.loose_count - txn->env->options.dp_limit; - if (txn->tw.dirtylist) { - tASSERT(txn, (txn->mt_flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); - tASSERT(txn, - MDBX_AVOID_MSYNC || (di && txn->tw.dirtylist->items[di].ptr == mp)); - if (!MDBX_AVOID_MSYNC || di) { - dpl_remove_ex(txn, di, npages); - txn->tw.dirtyroom++; - tASSERT(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length == - (txn->mt_parent ? txn->mt_parent->tw.dirtyroom - : txn->mt_env->me_options.dp_limit)); - if (!MDBX_AVOID_MSYNC || !(txn->mt_flags & MDBX_WRITEMAP)) { - dpage_free(txn->mt_env, mp, npages); - return; - } - } - } else { - tASSERT(txn, (txn->mt_flags & MDBX_WRITEMAP) && !MDBX_AVOID_MSYNC && !di); - txn->tw.writemap_dirty_npages -= (txn->tw.writemap_dirty_npages > npages) - ? npages - : txn->tw.writemap_dirty_npages; - } - VALGRIND_MAKE_MEM_UNDEFINED(mp, PAGEHDRSZ); - VALGRIND_MAKE_MEM_NOACCESS(page_data(mp), - pgno2bytes(txn->mt_env, npages) - PAGEHDRSZ); - MDBX_ASAN_POISON_MEMORY_REGION(page_data(mp), - pgno2bytes(txn->mt_env, npages) - PAGEHDRSZ); -} + /* production mode */ + if (likely(wanna_spill_npages < 1 && wanna_spill_entries < 1) +#if xMDBX_DEBUG_SPILLING == 1 + /* debug mode: always try to spill if xMDBX_DEBUG_SPILLING == 1 */ + && txn->txnid % 23 > 11 +#endif + ) + return MDBX_SUCCESS; -static __inline bool suitable4loose(const MDBX_txn *txn, pgno_t pgno) { - /* TODO: - * 1) при включенной "экономии последовательностей" проверить, что - * страница не примыкает к какой-либо из уже находящийся в reclaimed. - * 2) стоит подумать над тем, чтобы при большом loose-списке отбрасывать - половину в reclaimed. */ - return txn->tw.loose_count < txn->mt_env->me_options.dp_loose_limit && - (!MDBX_ENABLE_REFUND || - /* skip pages near to the end in favor of compactification */ - txn->mt_next_pgno > pgno + txn->mt_env->me_options.dp_loose_limit || - txn->mt_next_pgno <= txn->mt_env->me_options.dp_loose_limit); + return spill_slowpath(txn, m0, wanna_spill_entries, wanna_spill_npages, need); } -/* Retire, loosen or free a single page. - * - * For dirty pages, saves single pages to a list for future reuse in this same - * txn. It has been pulled from the GC and already resides on the dirty list, - * but has been deleted. Use these pages first before pulling again from the GC. - * - * If the page wasn't dirtied in this txn, just add it - * to this txn's free list. */ -static int page_retire_ex(MDBX_cursor *mc, const pgno_t pgno, - MDBX_page *mp /* maybe null */, - unsigned pageflags /* maybe unknown/zero */) { - int rc; - MDBX_txn *const txn = mc->mc_txn; - tASSERT(txn, !mp || (mp->mp_pgno == pgno && mp->mp_flags == pageflags)); +MDBX_INTERNAL int __must_check_result tree_search_finalize(MDBX_cursor *mc, const MDBX_val *key, int flags); +MDBX_INTERNAL int tree_search_lowest(MDBX_cursor *mc); - /* During deleting entire subtrees, it is reasonable and possible to avoid - * reading leaf pages, i.e. significantly reduce hard page-faults & IOPs: - * - mp is null, i.e. the page has not yet been read; - * - pagetype is known and the P_LEAF bit is set; - * - we can determine the page status via scanning the lists - * of dirty and spilled pages. - * - * On the other hand, this could be suboptimal for WRITEMAP mode, since - * requires support the list of dirty pages and avoid explicit spilling. - * So for flexibility and avoid extra internal dependencies we just - * fallback to reading if dirty list was not allocated yet. */ - size_t di = 0, si = 0, npages = 1; - enum page_status { - unknown, - frozen, - spilled, - shadowed, - modifable - } status = unknown; +enum page_search_flags { + Z_MODIFY = 1, + Z_ROOTONLY = 2, + Z_FIRST = 4, + Z_LAST = 8, +}; +MDBX_INTERNAL int __must_check_result tree_search(MDBX_cursor *mc, const MDBX_val *key, int flags); - if (unlikely(!mp)) { - if (ASSERT_ENABLED() && pageflags) { - pgr_t check; - check = page_get_any(mc, pgno, txn->mt_front); - if (unlikely(check.err != MDBX_SUCCESS)) - return check.err; - tASSERT(txn, - (check.page->mp_flags & ~P_SPILLED) == (pageflags & ~P_FROZEN)); - tASSERT(txn, !(pageflags & P_FROZEN) || IS_FROZEN(txn, check.page)); - } - if (pageflags & P_FROZEN) { - status = frozen; - if (ASSERT_ENABLED()) { - for (MDBX_txn *scan = txn; scan; scan = scan->mt_parent) { - tASSERT(txn, !txn->tw.spilled.list || !search_spilled(scan, pgno)); - tASSERT(txn, !scan->tw.dirtylist || !debug_dpl_find(scan, pgno)); - } - } - goto status_done; - } else if (pageflags && txn->tw.dirtylist) { - if ((di = dpl_exist(txn, pgno)) != 0) { - mp = txn->tw.dirtylist->items[di].ptr; - tASSERT(txn, IS_MODIFIABLE(txn, mp)); - status = modifable; - goto status_done; - } - if ((si = search_spilled(txn, pgno)) != 0) { - status = spilled; - goto status_done; - } - for (MDBX_txn *parent = txn->mt_parent; parent; - parent = parent->mt_parent) { - if (dpl_exist(parent, pgno)) { - status = shadowed; - goto status_done; - } - if (search_spilled(parent, pgno)) { - status = spilled; - goto status_done; - } - } - status = frozen; - goto status_done; - } +#define MDBX_SPLIT_REPLACE MDBX_APPENDDUP /* newkey is not new */ +MDBX_INTERNAL int __must_check_result page_split(MDBX_cursor *mc, const MDBX_val *const newkey, MDBX_val *const newdata, + pgno_t newpgno, const unsigned naf); - pgr_t pg = page_get_any(mc, pgno, txn->mt_front); - if (unlikely(pg.err != MDBX_SUCCESS)) - return pg.err; - mp = pg.page; - tASSERT(txn, !pageflags || mp->mp_flags == pageflags); - pageflags = mp->mp_flags; - } +/*----------------------------------------------------------------------------*/ - if (IS_FROZEN(txn, mp)) { - status = frozen; - tASSERT(txn, !IS_MODIFIABLE(txn, mp)); - tASSERT(txn, !IS_SPILLED(txn, mp)); - tASSERT(txn, !IS_SHADOWED(txn, mp)); - tASSERT(txn, !debug_dpl_find(txn, pgno)); - tASSERT(txn, !txn->tw.spilled.list || !search_spilled(txn, pgno)); - } else if (IS_MODIFIABLE(txn, mp)) { - status = modifable; - if (txn->tw.dirtylist) - di = dpl_exist(txn, pgno); - tASSERT(txn, (txn->mt_flags & MDBX_WRITEMAP) || !IS_SPILLED(txn, mp)); - tASSERT(txn, !txn->tw.spilled.list || !search_spilled(txn, pgno)); - } else if (IS_SHADOWED(txn, mp)) { - status = shadowed; - tASSERT(txn, !txn->tw.spilled.list || !search_spilled(txn, pgno)); - tASSERT(txn, !debug_dpl_find(txn, pgno)); - } else { - tASSERT(txn, IS_SPILLED(txn, mp)); - status = spilled; - si = search_spilled(txn, pgno); - tASSERT(txn, !debug_dpl_find(txn, pgno)); - } +MDBX_INTERNAL int MDBX_PRINTF_ARGS(2, 3) bad_page(const page_t *mp, const char *fmt, ...); -status_done: - if (likely((pageflags & P_OVERFLOW) == 0)) { - STATIC_ASSERT(P_BRANCH == 1); - const bool is_branch = pageflags & P_BRANCH; - if (unlikely(mc->mc_flags & C_SUB)) { - MDBX_db *outer = outer_db(mc); - cASSERT(mc, !is_branch || outer->md_branch_pages > 0); - outer->md_branch_pages -= is_branch; - cASSERT(mc, is_branch || outer->md_leaf_pages > 0); - outer->md_leaf_pages -= 1 - is_branch; - } - cASSERT(mc, !is_branch || mc->mc_db->md_branch_pages > 0); - mc->mc_db->md_branch_pages -= is_branch; - cASSERT(mc, (pageflags & P_LEAF) == 0 || mc->mc_db->md_leaf_pages > 0); - mc->mc_db->md_leaf_pages -= (pageflags & P_LEAF) != 0; - } else { - npages = mp->mp_pages; - cASSERT(mc, mc->mc_db->md_overflow_pages >= npages); - mc->mc_db->md_overflow_pages -= (pgno_t)npages; - } +MDBX_INTERNAL void MDBX_PRINTF_ARGS(2, 3) poor_page(const page_t *mp, const char *fmt, ...); - if (status == frozen) { - retire: - DEBUG("retire %zu page %" PRIaPGNO, npages, pgno); - rc = pnl_append_range(false, &txn->tw.retired_pages, pgno, npages); - tASSERT(txn, dirtylist_check(txn)); - return rc; - } +MDBX_NOTHROW_PURE_FUNCTION static inline bool is_frozen(const MDBX_txn *txn, const page_t *mp) { + return mp->txnid < txn->txnid; +} - /* Возврат страниц в нераспределенный "хвост" БД. - * Содержимое страниц не уничтожается, а для вложенных транзакций граница - * нераспределенного "хвоста" БД сдвигается только при их коммите. */ - if (MDBX_ENABLE_REFUND && unlikely(pgno + npages == txn->mt_next_pgno)) { - const char *kind = nullptr; - if (status == modifable) { - /* Страница испачкана в этой транзакции, но до этого могла быть - * аллоцирована, испачкана и пролита в одной из родительских транзакций. - * Её МОЖНО вытолкнуть в нераспределенный хвост. */ - kind = "dirty"; - /* Remove from dirty list */ - page_wash(txn, di, mp, npages); - } else if (si) { - /* Страница пролита в этой транзакции, т.е. она аллоцирована - * и запачкана в этой или одной из родительских транзакций. - * Её МОЖНО вытолкнуть в нераспределенный хвост. */ - kind = "spilled"; - tASSERT(txn, status == spilled); - spill_remove(txn, si, npages); - } else { - /* Страница аллоцирована, запачкана и возможно пролита в одной - * из родительских транзакций. - * Её МОЖНО вытолкнуть в нераспределенный хвост. */ - kind = "parent's"; - if (ASSERT_ENABLED() && mp) { - kind = nullptr; - for (MDBX_txn *parent = txn->mt_parent; parent; - parent = parent->mt_parent) { - if (search_spilled(parent, pgno)) { - kind = "parent-spilled"; - tASSERT(txn, status == spilled); - break; - } - if (mp == debug_dpl_find(parent, pgno)) { - kind = "parent-dirty"; - tASSERT(txn, status == shadowed); - break; - } - } - tASSERT(txn, kind != nullptr); - } - tASSERT(txn, status == spilled || status == shadowed); +MDBX_NOTHROW_PURE_FUNCTION static inline bool is_spilled(const MDBX_txn *txn, const page_t *mp) { + return mp->txnid == txn->txnid; +} + +MDBX_NOTHROW_PURE_FUNCTION static inline bool is_shadowed(const MDBX_txn *txn, const page_t *mp) { + return mp->txnid > txn->txnid; +} + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_correct(const MDBX_txn *txn, const page_t *mp) { + return mp->txnid <= txn->front_txnid; +} + +MDBX_NOTHROW_PURE_FUNCTION static inline bool is_modifable(const MDBX_txn *txn, const page_t *mp) { + return mp->txnid == txn->front_txnid; +} + +MDBX_INTERNAL int __must_check_result page_check(const MDBX_cursor *const mc, const page_t *const mp); + +MDBX_INTERNAL pgr_t page_get_any(const MDBX_cursor *const mc, const pgno_t pgno, const txnid_t front); + +MDBX_INTERNAL pgr_t page_get_three(const MDBX_cursor *const mc, const pgno_t pgno, const txnid_t front); + +MDBX_INTERNAL pgr_t page_get_large(const MDBX_cursor *const mc, const pgno_t pgno, const txnid_t front); + +static inline int __must_check_result page_get(const MDBX_cursor *mc, const pgno_t pgno, page_t **mp, + const txnid_t front) { + pgr_t ret = page_get_three(mc, pgno, front); + *mp = ret.page; + return ret.err; +} + +/*----------------------------------------------------------------------------*/ + +MDBX_INTERNAL int __must_check_result page_dirty(MDBX_txn *txn, page_t *mp, size_t npages); +MDBX_INTERNAL pgr_t page_new(MDBX_cursor *mc, const unsigned flags); +MDBX_INTERNAL pgr_t page_new_large(MDBX_cursor *mc, const size_t npages); +MDBX_INTERNAL int page_touch_modifable(MDBX_txn *txn, const page_t *const mp); +MDBX_INTERNAL int page_touch_unmodifable(MDBX_txn *txn, MDBX_cursor *mc, const page_t *const mp); + +static inline int page_touch(MDBX_cursor *mc) { + page_t *const mp = mc->pg[mc->top]; + MDBX_txn *txn = mc->txn; + + tASSERT(txn, mc->txn->flags & MDBX_TXN_DIRTY); + tASSERT(txn, F_ISSET(*cursor_dbi_state(mc), DBI_LINDO | DBI_VALID | DBI_DIRTY)); + tASSERT(txn, !is_largepage(mp)); + if (ASSERT_ENABLED()) { + if (mc->flags & z_inner) { + subcur_t *mx = container_of(mc->tree, subcur_t, nested_tree); + cursor_couple_t *couple = container_of(mx, cursor_couple_t, inner); + tASSERT(txn, mc->tree == &couple->outer.subcur->nested_tree); + tASSERT(txn, &mc->clc->k == &couple->outer.clc->v); + tASSERT(txn, *couple->outer.dbi_state & DBI_DIRTY); } - DEBUG("refunded %zu %s page %" PRIaPGNO, npages, kind, pgno); - txn->mt_next_pgno = pgno; - txn_refund(txn); - return MDBX_SUCCESS; + tASSERT(txn, dpl_check(txn)); } - if (status == modifable) { - /* Dirty page from this transaction */ - /* If suitable we can reuse it through loose list */ - if (likely(npages == 1 && suitable4loose(txn, pgno)) && - (di || !txn->tw.dirtylist)) { - DEBUG("loosen dirty page %" PRIaPGNO, pgno); - if (MDBX_DEBUG != 0 || unlikely(txn->mt_env->me_flags & MDBX_PAGEPERTURB)) - memset(page_data(mp), -1, txn->mt_env->me_psize - PAGEHDRSZ); - mp->mp_txnid = INVALID_TXNID; - mp->mp_flags = P_LOOSE; - mp_next(mp) = txn->tw.loose_pages; - txn->tw.loose_pages = mp; - txn->tw.loose_count++; -#if MDBX_ENABLE_REFUND - txn->tw.loose_refund_wl = (pgno + 2 > txn->tw.loose_refund_wl) - ? pgno + 2 - : txn->tw.loose_refund_wl; -#endif /* MDBX_ENABLE_REFUND */ - VALGRIND_MAKE_MEM_NOACCESS(page_data(mp), - txn->mt_env->me_psize - PAGEHDRSZ); - MDBX_ASAN_POISON_MEMORY_REGION(page_data(mp), - txn->mt_env->me_psize - PAGEHDRSZ); + if (is_modifable(txn, mp)) { + if (!txn->tw.dirtylist) { + tASSERT(txn, (txn->flags & MDBX_WRITEMAP) && !MDBX_AVOID_MSYNC); return MDBX_SUCCESS; } + return is_subpage(mp) ? MDBX_SUCCESS : page_touch_modifable(txn, mp); + } + return page_touch_unmodifable(txn, mc, mp); +} -#if !MDBX_DEBUG && !defined(MDBX_USE_VALGRIND) && !defined(__SANITIZE_ADDRESS__) - if (unlikely(txn->mt_env->me_flags & MDBX_PAGEPERTURB)) -#endif - { - /* Страница могла быть изменена в одной из родительских транзакций, - * в том числе, позже выгружена и затем снова загружена и изменена. - * В обоих случаях её нельзя затирать на диске и помечать недоступной - * в asan и/или valgrind */ - for (MDBX_txn *parent = txn->mt_parent; - parent && (parent->mt_flags & MDBX_TXN_SPILLS); - parent = parent->mt_parent) { - if (intersect_spilled(parent, pgno, npages)) - goto skip_invalidate; - if (dpl_intersect(parent, pgno, npages)) - goto skip_invalidate; - } +MDBX_INTERNAL void page_copy(page_t *const dst, const page_t *const src, const size_t size); +MDBX_INTERNAL pgr_t __must_check_result page_unspill(MDBX_txn *const txn, const page_t *const mp); -#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__) - if (MDBX_DEBUG != 0 || unlikely(txn->mt_env->me_flags & MDBX_PAGEPERTURB)) -#endif - kill_page(txn, mp, pgno, npages); - if ((txn->mt_flags & MDBX_WRITEMAP) == 0) { - VALGRIND_MAKE_MEM_NOACCESS(page_data(pgno2page(txn->mt_env, pgno)), - pgno2bytes(txn->mt_env, npages) - PAGEHDRSZ); - MDBX_ASAN_POISON_MEMORY_REGION(page_data(pgno2page(txn->mt_env, pgno)), - pgno2bytes(txn->mt_env, npages) - - PAGEHDRSZ); - } - } - skip_invalidate: +MDBX_INTERNAL page_t *page_shadow_alloc(MDBX_txn *txn, size_t num); - /* wash dirty page */ - page_wash(txn, di, mp, npages); +MDBX_INTERNAL void page_shadow_release(MDBX_env *env, page_t *dp, size_t npages); - reclaim: - DEBUG("reclaim %zu %s page %" PRIaPGNO, npages, "dirty", pgno); - rc = pnl_insert_range(&txn->tw.relist, pgno, npages); - tASSERT(txn, pnl_check_allocated(txn->tw.relist, - txn->mt_next_pgno - MDBX_ENABLE_REFUND)); - tASSERT(txn, dirtylist_check(txn)); - return rc; - } +MDBX_INTERNAL int page_retire_ex(MDBX_cursor *mc, const pgno_t pgno, page_t *mp /* maybe null */, + unsigned pageflags /* maybe unknown/zero */); - if (si) { - /* Page ws spilled in this txn */ - spill_remove(txn, si, npages); - /* Страница могла быть выделена и затем пролита в этой транзакции, - * тогда её необходимо поместить в reclaimed-список. - * Либо она могла быть выделена в одной из родительских транзакций и затем - * пролита в этой транзакции, тогда её необходимо поместить в - * retired-список для последующей фильтрации при коммите. */ - for (MDBX_txn *parent = txn->mt_parent; parent; - parent = parent->mt_parent) { - if (dpl_exist(parent, pgno)) - goto retire; - } - /* Страница точно была выделена в этой транзакции - * и теперь может быть использована повторно. */ - goto reclaim; - } +static inline int page_retire(MDBX_cursor *mc, page_t *mp) { return page_retire_ex(mc, mp->pgno, mp, mp->flags); } - if (status == shadowed) { - /* Dirty page MUST BE a clone from (one of) parent transaction(s). */ - if (ASSERT_ENABLED()) { - const MDBX_page *parent_dp = nullptr; - /* Check parent(s)'s dirty lists. */ - for (MDBX_txn *parent = txn->mt_parent; parent && !parent_dp; - parent = parent->mt_parent) { - tASSERT(txn, !search_spilled(parent, pgno)); - parent_dp = debug_dpl_find(parent, pgno); +static inline void page_wash(MDBX_txn *txn, size_t di, page_t *const mp, const size_t npages) { + tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0); + mp->txnid = INVALID_TXNID; + mp->flags = P_BAD; + + if (txn->tw.dirtylist) { + tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); + tASSERT(txn, MDBX_AVOID_MSYNC || (di && txn->tw.dirtylist->items[di].ptr == mp)); + if (!MDBX_AVOID_MSYNC || di) { + dpl_remove_ex(txn, di, npages); + txn->tw.dirtyroom++; + tASSERT(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length == + (txn->parent ? txn->parent->tw.dirtyroom : txn->env->options.dp_limit)); + if (!MDBX_AVOID_MSYNC || !(txn->flags & MDBX_WRITEMAP)) { + page_shadow_release(txn->env, mp, npages); + return; } - tASSERT(txn, parent_dp && (!mp || parent_dp == mp)); } - /* Страница была выделена в родительской транзакции и теперь может быть - * использована повторно, но только внутри этой транзакции, либо дочерних. - */ - goto reclaim; + } else { + tASSERT(txn, (txn->flags & MDBX_WRITEMAP) && !MDBX_AVOID_MSYNC && !di); + txn->tw.writemap_dirty_npages -= (txn->tw.writemap_dirty_npages > npages) ? npages : txn->tw.writemap_dirty_npages; } - - /* Страница может входить в доступный читателям MVCC-снимок, либо же она - * могла быть выделена, а затем пролита в одной из родительских - * транзакций. Поэтому пока помещаем её в retired-список, который будет - * фильтроваться относительно dirty- и spilled-списков родительских - * транзакций при коммите дочерних транзакций, либо же будет записан - * в GC в неизменном виде. */ - goto retire; + VALGRIND_MAKE_MEM_UNDEFINED(mp, PAGEHDRSZ); + VALGRIND_MAKE_MEM_NOACCESS(page_data(mp), pgno2bytes(txn->env, npages) - PAGEHDRSZ); + MDBX_ASAN_POISON_MEMORY_REGION(page_data(mp), pgno2bytes(txn->env, npages) - PAGEHDRSZ); } -static __inline int page_retire(MDBX_cursor *mc, MDBX_page *mp) { - return page_retire_ex(mc, mp->mp_pgno, mp, mp->mp_flags); -} +MDBX_INTERNAL size_t page_subleaf2_reserve(const MDBX_env *env, size_t host_page_room, size_t subpage_len, + size_t item_len); -typedef struct iov_ctx { - MDBX_env *env; - osal_ioring_t *ior; - mdbx_filehandle_t fd; - int err; -#ifndef MDBX_NEED_WRITTEN_RANGE -#define MDBX_NEED_WRITTEN_RANGE 1 -#endif /* MDBX_NEED_WRITTEN_RANGE */ -#if MDBX_NEED_WRITTEN_RANGE - pgno_t flush_begin; - pgno_t flush_end; -#endif /* MDBX_NEED_WRITTEN_RANGE */ - uint64_t coherency_timestamp; -} iov_ctx_t; - -__must_check_result static int iov_init(MDBX_txn *const txn, iov_ctx_t *ctx, - size_t items, size_t npages, - mdbx_filehandle_t fd, - bool check_coherence) { - ctx->env = txn->mt_env; - ctx->ior = &txn->mt_env->me_ioring; - ctx->fd = fd; - ctx->coherency_timestamp = - (check_coherence || txn->mt_env->me_lck->mti_pgop_stat.incoherence.weak) - ? 0 - : UINT64_MAX /* не выполнять сверку */; - ctx->err = osal_ioring_prepare(ctx->ior, items, - pgno_align2os_bytes(txn->mt_env, npages)); - if (likely(ctx->err == MDBX_SUCCESS)) { -#if MDBX_NEED_WRITTEN_RANGE - ctx->flush_begin = MAX_PAGENO; - ctx->flush_end = MIN_PAGENO; -#endif /* MDBX_NEED_WRITTEN_RANGE */ - osal_ioring_reset(ctx->ior); - } - return ctx->err; -} +#define page_next(mp) (*(page_t **)ptr_disp((mp)->entries, sizeof(void *) - sizeof(uint32_t))) -static inline bool iov_empty(const iov_ctx_t *ctx) { - return osal_ioring_used(ctx->ior) == 0; +MDBX_INTERNAL void rthc_ctor(void); +MDBX_INTERNAL void rthc_dtor(const uint32_t current_pid); +MDBX_INTERNAL void rthc_lock(void); +MDBX_INTERNAL void rthc_unlock(void); + +MDBX_INTERNAL int rthc_register(MDBX_env *const env); +MDBX_INTERNAL int rthc_remove(MDBX_env *const env); +MDBX_INTERNAL int rthc_uniq_check(const osal_mmap_t *pending, MDBX_env **found); + +/* dtor called for thread, i.e. for all mdbx's environment objects */ +MDBX_INTERNAL void rthc_thread_dtor(void *rthc); + +static inline void *thread_rthc_get(osal_thread_key_t key) { +#if defined(_WIN32) || defined(_WIN64) + return TlsGetValue(key); +#else + return pthread_getspecific(key); +#endif } -static void iov_callback4dirtypages(iov_ctx_t *ctx, size_t offset, void *data, - size_t bytes) { - MDBX_env *const env = ctx->env; - eASSERT(env, (env->me_flags & MDBX_WRITEMAP) == 0); +MDBX_INTERNAL void thread_rthc_set(osal_thread_key_t key, const void *value); - MDBX_page *wp = (MDBX_page *)data; - eASSERT(env, wp->mp_pgno == bytes2pgno(env, offset)); - eASSERT(env, bytes2pgno(env, bytes) >= (IS_OVERFLOW(wp) ? wp->mp_pages : 1u)); - eASSERT(env, (wp->mp_flags & P_ILL_BITS) == 0); +#if !defined(_WIN32) && !defined(_WIN64) +MDBX_INTERNAL void rthc_afterfork(void); +MDBX_INTERNAL void workaround_glibc_bug21031(void); +#endif /* !Windows */ - if (likely(ctx->err == MDBX_SUCCESS)) { - const MDBX_page *const rp = ptr_disp(env->me_map, offset); - VALGRIND_MAKE_MEM_DEFINED(rp, bytes); - MDBX_ASAN_UNPOISON_MEMORY_REGION(rp, bytes); - osal_flush_incoherent_mmap(rp, bytes, env->me_os_psize); - /* check with timeout as the workaround - * for https://libmdbx.dqdkfa.ru/dead-github/issues/269 - * - * Проблема проявляется только при неупорядоченности: если записанная - * последней мета-страница "обгоняет" ранее записанные, т.е. когда - * записанное в файл позже становится видимым в отображении раньше, - * чем записанное ранее. - * - * Исходно здесь всегда выполнялась полная сверка. Это давало полную - * гарантию защиты от проявления проблемы, но порождало накладные расходы. - * В некоторых сценариях наблюдалось снижение производительности до 10-15%, - * а в синтетических тестах до 30%. Конечно никто не вникал в причины, - * а просто останавливался на мнении "libmdbx не быстрее LMDB", - * например: https://clck.ru/3386er - * - * Поэтому после серии экспериментов и тестов реализовано следующее: - * 0. Посредством опции сборки MDBX_FORCE_CHECK_MMAP_COHERENCY=1 - * можно включить полную сверку после записи. - * Остальные пункты являются взвешенным компромиссом между полной - * гарантией обнаружения проблемы и бесполезными затратами на системах - * без этого недостатка. - * 1. При старте транзакций проверяется соответствие выбранной мета-страницы - * корневым страницам b-tree проверяется. Эта проверка показала себя - * достаточной без сверки после записи. При обнаружении "некогерентности" - * эти случаи подсчитываются, а при их ненулевом счетчике выполняется - * полная сверка. Таким образом, произойдет переключение в режим полной - * сверки, если показавшая себя достаточной проверка заметит проявление - * проблемы хоты-бы раз. - * 2. Сверка не выполняется при фиксации транзакции, так как: - * - при наличии проблемы "не-когерентности" (при отложенном копировании - * или обновлении PTE, после возврата из write-syscall), проверка - * в этом процессе не гарантирует актуальность данных в другом - * процессе, который может запустить транзакцию сразу после коммита; - * - сверка только последнего блока позволяет почти восстановить - * производительность в больших транзакциях, но одновременно размывает - * уверенность в отсутствии сбоев, чем обесценивает всю затею; - * - после записи данных будет записана мета-страница, соответствие - * которой корневым страницам b-tree проверяется при старте - * транзакций, и только эта проверка показала себя достаточной; - * 3. При спиллинге производится полная сверка записанных страниц. Тут был - * соблазн сверять не полностью, а например начало и конец каждого блока. - * Но при спиллинге возможна ситуация повторного вытеснения страниц, в - * том числе large/overflow. При этом возникает риск прочитать в текущей - * транзакции старую версию страницы, до повторной записи. В этом случае - * могут возникать крайне редкие невоспроизводимые ошибки. С учетом того - * что спиллинг выполняет крайне редко, решено отказаться от экономии - * в пользу надежности. */ -#ifndef MDBX_FORCE_CHECK_MMAP_COHERENCY -#define MDBX_FORCE_CHECK_MMAP_COHERENCY 0 -#endif /* MDBX_FORCE_CHECK_MMAP_COHERENCY */ - if ((MDBX_FORCE_CHECK_MMAP_COHERENCY || - ctx->coherency_timestamp != UINT64_MAX) && - unlikely(memcmp(wp, rp, bytes))) { - ctx->coherency_timestamp = 0; - env->me_lck->mti_pgop_stat.incoherence.weak = - (env->me_lck->mti_pgop_stat.incoherence.weak >= INT32_MAX) - ? INT32_MAX - : env->me_lck->mti_pgop_stat.incoherence.weak + 1; - WARNING("catch delayed/non-arrived page %" PRIaPGNO " %s", wp->mp_pgno, - "(workaround for incoherent flaw of unified page/buffer cache)"); - do - if (coherency_timeout(&ctx->coherency_timestamp, wp->mp_pgno, env) != - MDBX_RESULT_TRUE) { - ctx->err = MDBX_PROBLEM; - break; - } - while (unlikely(memcmp(wp, rp, bytes))); - } - } - - if (likely(bytes == env->me_psize)) - dpage_free(env, wp, 1); - else { - do { - eASSERT(env, wp->mp_pgno == bytes2pgno(env, offset)); - eASSERT(env, (wp->mp_flags & P_ILL_BITS) == 0); - size_t npages = IS_OVERFLOW(wp) ? wp->mp_pages : 1u; - size_t chunk = pgno2bytes(env, npages); - eASSERT(env, bytes >= chunk); - MDBX_page *next = ptr_disp(wp, chunk); - dpage_free(env, wp, npages); - wp = next; - offset += chunk; - bytes -= chunk; - } while (bytes); - } -} - -static void iov_complete(iov_ctx_t *ctx) { - if ((ctx->env->me_flags & MDBX_WRITEMAP) == 0) - osal_ioring_walk(ctx->ior, ctx, iov_callback4dirtypages); - osal_ioring_reset(ctx->ior); -} - -__must_check_result static int iov_write(iov_ctx_t *ctx) { - eASSERT(ctx->env, !iov_empty(ctx)); - osal_ioring_write_result_t r = osal_ioring_write(ctx->ior, ctx->fd); -#if MDBX_ENABLE_PGOP_STAT - ctx->env->me_lck->mti_pgop_stat.wops.weak += r.wops; -#endif /* MDBX_ENABLE_PGOP_STAT */ - ctx->err = r.err; - if (unlikely(ctx->err != MDBX_SUCCESS)) - ERROR("Write error: %s", mdbx_strerror(ctx->err)); - iov_complete(ctx); - return ctx->err; -} - -__must_check_result static int iov_page(MDBX_txn *txn, iov_ctx_t *ctx, - MDBX_page *dp, size_t npages) { - MDBX_env *const env = txn->mt_env; - tASSERT(txn, ctx->err == MDBX_SUCCESS); - tASSERT(txn, dp->mp_pgno >= MIN_PAGENO && dp->mp_pgno < txn->mt_next_pgno); - tASSERT(txn, IS_MODIFIABLE(txn, dp)); - tASSERT(txn, !(dp->mp_flags & ~(P_BRANCH | P_LEAF | P_LEAF2 | P_OVERFLOW))); - - if (IS_SHADOWED(txn, dp)) { - tASSERT(txn, !(txn->mt_flags & MDBX_WRITEMAP)); - dp->mp_txnid = txn->mt_txnid; - tASSERT(txn, IS_SPILLED(txn, dp)); -#if MDBX_AVOID_MSYNC - doit:; -#endif /* MDBX_AVOID_MSYNC */ - int err = osal_ioring_add(ctx->ior, pgno2bytes(env, dp->mp_pgno), dp, - pgno2bytes(env, npages)); - if (unlikely(err != MDBX_SUCCESS)) { - ctx->err = err; - if (unlikely(err != MDBX_RESULT_TRUE)) { - iov_complete(ctx); - return err; - } - err = iov_write(ctx); - tASSERT(txn, iov_empty(ctx)); - if (likely(err == MDBX_SUCCESS)) { - err = osal_ioring_add(ctx->ior, pgno2bytes(env, dp->mp_pgno), dp, - pgno2bytes(env, npages)); - if (unlikely(err != MDBX_SUCCESS)) { - iov_complete(ctx); - return ctx->err = err; - } - } - tASSERT(txn, ctx->err == MDBX_SUCCESS); - } - } else { - tASSERT(txn, txn->mt_flags & MDBX_WRITEMAP); -#if MDBX_AVOID_MSYNC - goto doit; -#endif /* MDBX_AVOID_MSYNC */ - } - -#if MDBX_NEED_WRITTEN_RANGE - ctx->flush_begin = - (ctx->flush_begin < dp->mp_pgno) ? ctx->flush_begin : dp->mp_pgno; - ctx->flush_end = (ctx->flush_end > dp->mp_pgno + (pgno_t)npages) - ? ctx->flush_end - : dp->mp_pgno + (pgno_t)npages; -#endif /* MDBX_NEED_WRITTEN_RANGE */ - return MDBX_SUCCESS; -} - -static int spill_page(MDBX_txn *txn, iov_ctx_t *ctx, MDBX_page *dp, - const size_t npages) { - tASSERT(txn, !(txn->mt_flags & MDBX_WRITEMAP)); -#if MDBX_ENABLE_PGOP_STAT - txn->mt_env->me_lck->mti_pgop_stat.spill.weak += npages; -#endif /* MDBX_ENABLE_PGOP_STAT */ - const pgno_t pgno = dp->mp_pgno; - int err = iov_page(txn, ctx, dp, npages); - if (likely(err == MDBX_SUCCESS)) - err = pnl_append_range(true, &txn->tw.spilled.list, pgno << 1, npages); - return err; -} - -/* Set unspillable LRU-label for dirty pages watched by txn. - * Returns the number of pages marked as unspillable. */ -static size_t cursor_keep(const MDBX_txn *const txn, const MDBX_cursor *mc) { - tASSERT(txn, (txn->mt_flags & (MDBX_TXN_RDONLY | MDBX_WRITEMAP)) == 0); - size_t keep = 0; - while ((mc->mc_flags & C_INITIALIZED) && mc->mc_snum) { - tASSERT(txn, mc->mc_top == mc->mc_snum - 1); - const MDBX_page *mp; - size_t i = 0; - do { - mp = mc->mc_pg[i]; - tASSERT(txn, !IS_SUBP(mp)); - if (IS_MODIFIABLE(txn, mp)) { - size_t const n = dpl_search(txn, mp->mp_pgno); - if (txn->tw.dirtylist->items[n].pgno == mp->mp_pgno && - /* не считаем дважды */ dpl_age(txn, n)) { - size_t *const ptr = ptr_disp(txn->tw.dirtylist->items[n].ptr, - -(ptrdiff_t)sizeof(size_t)); - *ptr = txn->tw.dirtylru; - tASSERT(txn, dpl_age(txn, n) == 0); - ++keep; - } - } - } while (++i < mc->mc_snum); - - tASSERT(txn, IS_LEAF(mp)); - if (!mc->mc_xcursor || mc->mc_ki[mc->mc_top] >= page_numkeys(mp)) - break; - if (!(node_flags(page_node(mp, mc->mc_ki[mc->mc_top])) & F_SUBDATA)) - break; - mc = &mc->mc_xcursor->mx_cursor; - } - return keep; -} - -static size_t txn_keep(MDBX_txn *txn, MDBX_cursor *m0) { - tASSERT(txn, (txn->mt_flags & (MDBX_TXN_RDONLY | MDBX_WRITEMAP)) == 0); - txn_lru_turn(txn); - size_t keep = m0 ? cursor_keep(txn, m0) : 0; - for (size_t i = FREE_DBI; i < txn->mt_numdbs; ++i) - if (F_ISSET(txn->mt_dbistate[i], DBI_DIRTY | DBI_VALID) && - txn->mt_dbs[i].md_root != P_INVALID) - for (MDBX_cursor *mc = txn->mt_cursors[i]; mc; mc = mc->mc_next) - if (mc != m0) - keep += cursor_keep(txn, mc); - return keep; +static inline void thread_key_delete(osal_thread_key_t key) { + TRACE("key = %" PRIuPTR, (uintptr_t)key); +#if defined(_WIN32) || defined(_WIN64) + ENSURE(nullptr, TlsFree(key)); +#else + ENSURE(nullptr, pthread_key_delete(key) == 0); + workaround_glibc_bug21031(); +#endif } -/* Returns the spilling priority (0..255) for a dirty page: - * 0 = should be spilled; - * ... - * > 255 = must not be spilled. */ -MDBX_NOTHROW_PURE_FUNCTION static unsigned -spill_prio(const MDBX_txn *txn, const size_t i, const uint32_t reciprocal) { - MDBX_dpl *const dl = txn->tw.dirtylist; - const uint32_t age = dpl_age(txn, i); - const size_t npages = dpl_npages(dl, i); - const pgno_t pgno = dl->items[i].pgno; - if (age == 0) { - DEBUG("skip %s %zu page %" PRIaPGNO, "keep", npages, pgno); - return 256; - } +typedef struct walk_tbl { + MDBX_val name; + tree_t *internal, *nested; +} walk_tbl_t; - MDBX_page *const dp = dl->items[i].ptr; - if (dp->mp_flags & (P_LOOSE | P_SPILLED)) { - DEBUG("skip %s %zu page %" PRIaPGNO, - (dp->mp_flags & P_LOOSE) ? "loose" : "parent-spilled", npages, pgno); - return 256; - } +typedef int walk_func(const size_t pgno, const unsigned number, void *const ctx, const int deep, + const walk_tbl_t *table, const size_t page_size, const page_type_t page_type, + const MDBX_error_t err, const size_t nentries, const size_t payload_bytes, + const size_t header_bytes, const size_t unused_bytes); - /* Can't spill twice, - * make sure it's not already in a parent's spill list(s). */ - MDBX_txn *parent = txn->mt_parent; - if (parent && (parent->mt_flags & MDBX_TXN_SPILLS)) { - do - if (intersect_spilled(parent, pgno, npages)) { - DEBUG("skip-2 parent-spilled %zu page %" PRIaPGNO, npages, pgno); - dp->mp_flags |= P_SPILLED; - return 256; - } - while ((parent = parent->mt_parent) != nullptr); - } +typedef enum walk_options { dont_check_keys_ordering = 1 } walk_options_t; - tASSERT(txn, age * (uint64_t)reciprocal < UINT32_MAX); - unsigned prio = age * reciprocal >> 24; - tASSERT(txn, prio < 256); - if (likely(npages == 1)) - return prio = 256 - prio; +MDBX_INTERNAL int walk_pages(MDBX_txn *txn, walk_func *visitor, void *user, walk_options_t options); - /* make a large/overflow pages be likely to spill */ - size_t factor = npages | npages >> 1; - factor |= factor >> 2; - factor |= factor >> 4; - factor |= factor >> 8; - factor |= factor >> 16; - factor = (size_t)prio * log2n_powerof2(factor + 1) + /* golden ratio */ 157; - factor = (factor < 256) ? 255 - factor : 0; - tASSERT(txn, factor < 256 && factor < (256 - prio)); - return prio = (unsigned)factor; -} +/// -/* Spill pages from the dirty list back to disk. - * This is intended to prevent running into MDBX_TXN_FULL situations, - * but note that they may still occur in a few cases: - * - * 1) our estimate of the txn size could be too small. Currently this - * seems unlikely, except with a large number of MDBX_MULTIPLE items. - * - * 2) child txns may run out of space if their parents dirtied a - * lot of pages and never spilled them. TODO: we probably should do - * a preemptive spill during mdbx_txn_begin() of a child txn, if - * the parent's dirtyroom is below a given threshold. - * - * Otherwise, if not using nested txns, it is expected that apps will - * not run into MDBX_TXN_FULL any more. The pages are flushed to disk - * the same way as for a txn commit, e.g. their dirty status is cleared. - * If the txn never references them again, they can be left alone. - * If the txn only reads them, they can be used without any fuss. - * If the txn writes them again, they can be dirtied immediately without - * going thru all of the work of page_touch(). Such references are - * handled by page_unspill(). - * - * Also note, we never spill DB root pages, nor pages of active cursors, - * because we'll need these back again soon anyway. And in nested txns, - * we can't spill a page in a child txn if it was already spilled in a - * parent txn. That would alter the parent txns' data even though - * the child hasn't committed yet, and we'd have no way to undo it if - * the child aborted. */ -__cold static int txn_spill_slowpath(MDBX_txn *const txn, MDBX_cursor *const m0, - const intptr_t wanna_spill_entries, - const intptr_t wanna_spill_npages, - const size_t need); - -static __inline int txn_spill(MDBX_txn *const txn, MDBX_cursor *const m0, - const size_t need) { - tASSERT(txn, (txn->mt_flags & MDBX_TXN_RDONLY) == 0); - tASSERT(txn, !m0 || cursor_is_tracked(m0)); +#define MDBX_RADIXSORT_THRESHOLD 142 - const intptr_t wanna_spill_entries = - txn->tw.dirtylist ? (need - txn->tw.dirtyroom - txn->tw.loose_count) : 0; - const intptr_t wanna_spill_npages = - need + - (txn->tw.dirtylist ? txn->tw.dirtylist->pages_including_loose - : txn->tw.writemap_dirty_npages) - - txn->tw.loose_count - txn->mt_env->me_options.dp_limit; +/* --------------------------------------------------------------------------- + * LY: State of the art quicksort-based sorting, with internal stack + * and network-sort for small chunks. + * Thanks to John M. Gamble for the http://pages.ripco.net/~jgamble/nw.html */ - /* production mode */ - if (likely(wanna_spill_npages < 1 && wanna_spill_entries < 1) -#if xMDBX_DEBUG_SPILLING == 1 - /* debug mode: always try to spill if xMDBX_DEBUG_SPILLING == 1 */ - && txn->mt_txnid % 23 > 11 +#if MDBX_HAVE_CMOV +#define SORT_CMP_SWAP(TYPE, CMP, a, b) \ + do { \ + const TYPE swap_tmp = (a); \ + const bool swap_cmp = expect_with_probability(CMP(swap_tmp, b), 0, .5); \ + (a) = swap_cmp ? swap_tmp : b; \ + (b) = swap_cmp ? b : swap_tmp; \ + } while (0) +#else +#define SORT_CMP_SWAP(TYPE, CMP, a, b) \ + do \ + if (expect_with_probability(!CMP(a, b), 0, .5)) { \ + const TYPE swap_tmp = (a); \ + (a) = (b); \ + (b) = swap_tmp; \ + } \ + while (0) #endif - ) - return MDBX_SUCCESS; - return txn_spill_slowpath(txn, m0, wanna_spill_entries, wanna_spill_npages, - need); -} +// 3 comparators, 3 parallel operations +// o-----^--^--o +// | | +// o--^--|--v--o +// | | +// o--v--v-----o +// +// [[1,2]] +// [[0,2]] +// [[0,1]] +#define SORT_NETWORK_3(TYPE, CMP, begin) \ + do { \ + SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[2]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[2]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[1]); \ + } while (0) -static size_t spill_gate(const MDBX_env *env, intptr_t part, - const size_t total) { - const intptr_t spill_min = - env->me_options.spill_min_denominator - ? (total + env->me_options.spill_min_denominator - 1) / - env->me_options.spill_min_denominator - : 1; - const intptr_t spill_max = - total - (env->me_options.spill_max_denominator - ? total / env->me_options.spill_max_denominator - : 0); - part = (part < spill_max) ? part : spill_max; - part = (part > spill_min) ? part : spill_min; - eASSERT(env, part >= 0 && (size_t)part <= total); - return (size_t)part; -} +// 5 comparators, 3 parallel operations +// o--^--^--------o +// | | +// o--v--|--^--^--o +// | | | +// o--^--v--|--v--o +// | | +// o--v-----v-----o +// +// [[0,1],[2,3]] +// [[0,2],[1,3]] +// [[1,2]] +#define SORT_NETWORK_4(TYPE, CMP, begin) \ + do { \ + SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[1]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[3]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[2]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[3]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[2]); \ + } while (0) -__cold static int txn_spill_slowpath(MDBX_txn *const txn, MDBX_cursor *const m0, - const intptr_t wanna_spill_entries, - const intptr_t wanna_spill_npages, - const size_t need) { - tASSERT(txn, (txn->mt_flags & MDBX_TXN_RDONLY) == 0); +// 9 comparators, 5 parallel operations +// o--^--^-----^-----------o +// | | | +// o--|--|--^--v-----^--^--o +// | | | | | +// o--|--v--|--^--^--|--v--o +// | | | | | +// o--|-----v--|--v--|--^--o +// | | | | +// o--v--------v-----v--v--o +// +// [[0,4],[1,3]] +// [[0,2]] +// [[2,4],[0,1]] +// [[2,3],[1,4]] +// [[1,2],[3,4]] +#define SORT_NETWORK_5(TYPE, CMP, begin) \ + do { \ + SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[4]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[3]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[2]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[4]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[1]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[3]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[4]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[2]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[4]); \ + } while (0) - int rc = MDBX_SUCCESS; - if (unlikely(txn->tw.loose_count >= - (txn->tw.dirtylist ? txn->tw.dirtylist->pages_including_loose - : txn->tw.writemap_dirty_npages))) - goto done; +// 12 comparators, 6 parallel operations +// o-----^--^--^-----------------o +// | | | +// o--^--|--v--|--^--------^-----o +// | | | | | +// o--v--v-----|--|--^--^--|--^--o +// | | | | | | +// o-----^--^--v--|--|--|--v--v--o +// | | | | | +// o--^--|--v-----v--|--v--------o +// | | | +// o--v--v-----------v-----------o +// +// [[1,2],[4,5]] +// [[0,2],[3,5]] +// [[0,1],[3,4],[2,5]] +// [[0,3],[1,4]] +// [[2,4],[1,3]] +// [[2,3]] +#define SORT_NETWORK_6(TYPE, CMP, begin) \ + do { \ + SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[2]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[4], begin[5]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[2]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[5]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[1]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[4]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[5]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[3]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[4]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[4]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[3]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[3]); \ + } while (0) - const size_t dirty_entries = - txn->tw.dirtylist ? (txn->tw.dirtylist->length - txn->tw.loose_count) : 1; - const size_t dirty_npages = - (txn->tw.dirtylist ? txn->tw.dirtylist->pages_including_loose - : txn->tw.writemap_dirty_npages) - - txn->tw.loose_count; - const size_t need_spill_entries = - spill_gate(txn->mt_env, wanna_spill_entries, dirty_entries); - const size_t need_spill_npages = - spill_gate(txn->mt_env, wanna_spill_npages, dirty_npages); - - const size_t need_spill = (need_spill_entries > need_spill_npages) - ? need_spill_entries - : need_spill_npages; - if (!need_spill) - goto done; +// 16 comparators, 6 parallel operations +// o--^--------^-----^-----------------o +// | | | +// o--|--^-----|--^--v--------^--^-----o +// | | | | | | +// o--|--|--^--v--|--^-----^--|--v-----o +// | | | | | | | +// o--|--|--|-----v--|--^--v--|--^--^--o +// | | | | | | | | +// o--v--|--|--^-----v--|--^--v--|--v--o +// | | | | | | +// o-----v--|--|--------v--v-----|--^--o +// | | | | +// o--------v--v-----------------v--v--o +// +// [[0,4],[1,5],[2,6]] +// [[0,2],[1,3],[4,6]] +// [[2,4],[3,5],[0,1]] +// [[2,3],[4,5]] +// [[1,4],[3,6]] +// [[1,2],[3,4],[5,6]] +#define SORT_NETWORK_7(TYPE, CMP, begin) \ + do { \ + SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[4]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[5]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[6]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[2]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[3]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[4], begin[6]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[4]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[5]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[1]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[3]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[4], begin[5]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[4]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[6]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[2]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[4]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[5], begin[6]); \ + } while (0) - if (txn->mt_flags & MDBX_WRITEMAP) { - NOTICE("%s-spilling %zu dirty-entries, %zu dirty-npages", "msync", - dirty_entries, dirty_npages); - const MDBX_env *env = txn->mt_env; - tASSERT(txn, txn->tw.spilled.list == nullptr); - rc = - osal_msync(&txn->mt_env->me_dxb_mmap, 0, - pgno_align2os_bytes(env, txn->mt_next_pgno), MDBX_SYNC_KICK); - if (unlikely(rc != MDBX_SUCCESS)) - goto bailout; -#if MDBX_AVOID_MSYNC - MDBX_ANALYSIS_ASSUME(txn->tw.dirtylist != nullptr); - tASSERT(txn, dirtylist_check(txn)); - env->me_lck->mti_unsynced_pages.weak += - txn->tw.dirtylist->pages_including_loose - txn->tw.loose_count; - dpl_clear(txn->tw.dirtylist); - txn->tw.dirtyroom = env->me_options.dp_limit - txn->tw.loose_count; - for (MDBX_page *lp = txn->tw.loose_pages; lp != nullptr; lp = mp_next(lp)) { - tASSERT(txn, lp->mp_flags == P_LOOSE); - rc = dpl_append(txn, lp->mp_pgno, lp, 1); - if (unlikely(rc != MDBX_SUCCESS)) - goto bailout; - MDBX_ASAN_UNPOISON_MEMORY_REGION(&mp_next(lp), sizeof(MDBX_page *)); - VALGRIND_MAKE_MEM_DEFINED(&mp_next(lp), sizeof(MDBX_page *)); - } - tASSERT(txn, dirtylist_check(txn)); -#else - tASSERT(txn, txn->tw.dirtylist == nullptr); - env->me_lck->mti_unsynced_pages.weak += txn->tw.writemap_dirty_npages; - txn->tw.writemap_spilled_npages += txn->tw.writemap_dirty_npages; - txn->tw.writemap_dirty_npages = 0; -#endif /* MDBX_AVOID_MSYNC */ - goto done; - } +// 19 comparators, 6 parallel operations +// o--^--------^-----^-----------------o +// | | | +// o--|--^-----|--^--v--------^--^-----o +// | | | | | | +// o--|--|--^--v--|--^-----^--|--v-----o +// | | | | | | | +// o--|--|--|--^--v--|--^--v--|--^--^--o +// | | | | | | | | | +// o--v--|--|--|--^--v--|--^--v--|--v--o +// | | | | | | | +// o-----v--|--|--|--^--v--v-----|--^--o +// | | | | | | +// o--------v--|--v--|--^--------v--v--o +// | | | +// o-----------v-----v--v--------------o +// +// [[0,4],[1,5],[2,6],[3,7]] +// [[0,2],[1,3],[4,6],[5,7]] +// [[2,4],[3,5],[0,1],[6,7]] +// [[2,3],[4,5]] +// [[1,4],[3,6]] +// [[1,2],[3,4],[5,6]] +#define SORT_NETWORK_8(TYPE, CMP, begin) \ + do { \ + SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[4]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[5]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[6]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[7]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[2]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[3]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[4], begin[6]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[5], begin[7]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[4]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[5]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[1]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[6], begin[7]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[2], begin[3]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[4], begin[5]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[4]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[6]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[1], begin[2]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[3], begin[4]); \ + SORT_CMP_SWAP(TYPE, CMP, begin[5], begin[6]); \ + } while (0) - NOTICE("%s-spilling %zu dirty-entries, %zu dirty-npages", "write", - need_spill_entries, need_spill_npages); - MDBX_ANALYSIS_ASSUME(txn->tw.dirtylist != nullptr); - tASSERT(txn, txn->tw.dirtylist->length - txn->tw.loose_count >= 1); - tASSERT(txn, txn->tw.dirtylist->pages_including_loose - txn->tw.loose_count >= - need_spill_npages); - if (!txn->tw.spilled.list) { - txn->tw.spilled.least_removed = INT_MAX; - txn->tw.spilled.list = pnl_alloc(need_spill); - if (unlikely(!txn->tw.spilled.list)) { - rc = MDBX_ENOMEM; - bailout: - txn->mt_flags |= MDBX_TXN_ERROR; - return rc; - } - } else { - /* purge deleted slots */ - spill_purge(txn); - rc = pnl_reserve(&txn->tw.spilled.list, need_spill); - (void)rc /* ignore since the resulting list may be shorter - and pnl_append() will increase pnl on demand */ - ; - } +#define SORT_INNER(TYPE, CMP, begin, end, len) \ + switch (len) { \ + default: \ + assert(false); \ + __unreachable(); \ + case 0: \ + case 1: \ + break; \ + case 2: \ + SORT_CMP_SWAP(TYPE, CMP, begin[0], begin[1]); \ + break; \ + case 3: \ + SORT_NETWORK_3(TYPE, CMP, begin); \ + break; \ + case 4: \ + SORT_NETWORK_4(TYPE, CMP, begin); \ + break; \ + case 5: \ + SORT_NETWORK_5(TYPE, CMP, begin); \ + break; \ + case 6: \ + SORT_NETWORK_6(TYPE, CMP, begin); \ + break; \ + case 7: \ + SORT_NETWORK_7(TYPE, CMP, begin); \ + break; \ + case 8: \ + SORT_NETWORK_8(TYPE, CMP, begin); \ + break; \ + } + +#define SORT_SWAP(TYPE, a, b) \ + do { \ + const TYPE swap_tmp = (a); \ + (a) = (b); \ + (b) = swap_tmp; \ + } while (0) - /* Сортируем чтобы запись на диск была полее последовательна */ - MDBX_dpl *const dl = dpl_sort(txn); +#define SORT_PUSH(low, high) \ + do { \ + top->lo = (low); \ + top->hi = (high); \ + ++top; \ + } while (0) - /* Preserve pages which may soon be dirtied again */ - const size_t unspillable = txn_keep(txn, m0); - if (unspillable + txn->tw.loose_count >= dl->length) { -#if xMDBX_DEBUG_SPILLING == 1 /* avoid false failure in debug mode */ - if (likely(txn->tw.dirtyroom + txn->tw.loose_count >= need)) - return MDBX_SUCCESS; -#endif /* xMDBX_DEBUG_SPILLING */ - ERROR("all %zu dirty pages are unspillable since referenced " - "by a cursor(s), use fewer cursors or increase " - "MDBX_opt_txn_dp_limit", - unspillable); - goto done; +#define SORT_POP(low, high) \ + do { \ + --top; \ + low = top->lo; \ + high = top->hi; \ + } while (0) + +#define SORT_IMPL(NAME, EXPECT_LOW_CARDINALITY_OR_PRESORTED, TYPE, CMP) \ + \ + static inline bool NAME##_is_sorted(const TYPE *first, const TYPE *last) { \ + while (++first <= last) \ + if (expect_with_probability(CMP(first[0], first[-1]), 1, .1)) \ + return false; \ + return true; \ + } \ + \ + typedef struct { \ + TYPE *lo, *hi; \ + } NAME##_stack; \ + \ + __hot static void NAME(TYPE *const __restrict begin, TYPE *const __restrict end) { \ + NAME##_stack stack[sizeof(size_t) * CHAR_BIT], *__restrict top = stack; \ + \ + TYPE *__restrict hi = end - 1; \ + TYPE *__restrict lo = begin; \ + while (true) { \ + const ptrdiff_t len = hi - lo; \ + if (len < 8) { \ + SORT_INNER(TYPE, CMP, lo, hi + 1, len + 1); \ + if (unlikely(top == stack)) \ + break; \ + SORT_POP(lo, hi); \ + continue; \ + } \ + \ + TYPE *__restrict mid = lo + (len >> 1); \ + SORT_CMP_SWAP(TYPE, CMP, *lo, *mid); \ + SORT_CMP_SWAP(TYPE, CMP, *mid, *hi); \ + SORT_CMP_SWAP(TYPE, CMP, *lo, *mid); \ + \ + TYPE *right = hi - 1; \ + TYPE *left = lo + 1; \ + while (1) { \ + while (expect_with_probability(CMP(*left, *mid), 0, .5)) \ + ++left; \ + while (expect_with_probability(CMP(*mid, *right), 0, .5)) \ + --right; \ + if (unlikely(left > right)) { \ + if (EXPECT_LOW_CARDINALITY_OR_PRESORTED) { \ + if (NAME##_is_sorted(lo, right)) \ + lo = right + 1; \ + if (NAME##_is_sorted(left, hi)) \ + hi = left; \ + } \ + break; \ + } \ + SORT_SWAP(TYPE, *left, *right); \ + mid = (mid == left) ? right : (mid == right) ? left : mid; \ + ++left; \ + --right; \ + } \ + \ + if (right - lo > hi - left) { \ + SORT_PUSH(lo, right); \ + lo = left; \ + } else { \ + SORT_PUSH(left, hi); \ + hi = right; \ + } \ + } \ + \ + if (AUDIT_ENABLED()) { \ + for (TYPE *scan = begin + 1; scan < end; ++scan) \ + assert(CMP(scan[-1], scan[0])); \ + } \ } - /* Подзадача: Вытолкнуть часть страниц на диск в соответствии с LRU, - * но при этом учесть важные поправки: - * - лучше выталкивать старые large/overflow страницы, так будет освобождено - * больше памяти, а также так как они (в текущем понимании) гораздо реже - * повторно изменяются; - * - при прочих равных лучше выталкивать смежные страницы, так будет - * меньше I/O операций; - * - желательно потратить на это меньше времени чем std::partial_sort_copy; - * - * Решение: - * - Квантуем весь диапазон lru-меток до 256 значений и задействуем один - * проход 8-битного radix-sort. В результате получаем 256 уровней - * "свежести", в том числе значение lru-метки, старее которой страницы - * должны быть выгружены; - * - Двигаемся последовательно в сторону увеличения номеров страниц - * и выталкиваем страницы с lru-меткой старее отсекающего значения, - * пока не вытолкнем достаточно; - * - Встречая страницы смежные с выталкиваемыми для уменьшения кол-ва - * I/O операций выталкиваем и их, если они попадают в первую половину - * между выталкиваемыми и самыми свежими lru-метками; - * - дополнительно при сортировке умышленно старим large/overflow страницы, - * тем самым повышая их шансы на выталкивание. */ +/*------------------------------------------------------------------------------ + * LY: radix sort for large chunks */ - /* get min/max of LRU-labels */ - uint32_t age_max = 0; - for (size_t i = 1; i <= dl->length; ++i) { - const uint32_t age = dpl_age(txn, i); - age_max = (age_max >= age) ? age_max : age; +#define RADIXSORT_IMPL(NAME, TYPE, EXTRACT_KEY, BUFFER_PREALLOCATED, END_GAP) \ + \ + __hot static bool NAME##_radixsort(TYPE *const begin, const size_t length) { \ + TYPE *tmp; \ + if (BUFFER_PREALLOCATED) { \ + tmp = begin + length + END_GAP; \ + /* memset(tmp, 0xDeadBeef, sizeof(TYPE) * length); */ \ + } else { \ + tmp = osal_malloc(sizeof(TYPE) * length); \ + if (unlikely(!tmp)) \ + return false; \ + } \ + \ + size_t key_shift = 0, key_diff_mask; \ + do { \ + struct { \ + pgno_t a[256], b[256]; \ + } counters; \ + memset(&counters, 0, sizeof(counters)); \ + \ + key_diff_mask = 0; \ + size_t prev_key = EXTRACT_KEY(begin) >> key_shift; \ + TYPE *r = begin, *end = begin + length; \ + do { \ + const size_t key = EXTRACT_KEY(r) >> key_shift; \ + counters.a[key & 255]++; \ + counters.b[(key >> 8) & 255]++; \ + key_diff_mask |= prev_key ^ key; \ + prev_key = key; \ + } while (++r != end); \ + \ + pgno_t ta = 0, tb = 0; \ + for (size_t i = 0; i < 256; ++i) { \ + const pgno_t ia = counters.a[i]; \ + counters.a[i] = ta; \ + ta += ia; \ + const pgno_t ib = counters.b[i]; \ + counters.b[i] = tb; \ + tb += ib; \ + } \ + \ + r = begin; \ + do { \ + const size_t key = EXTRACT_KEY(r) >> key_shift; \ + tmp[counters.a[key & 255]++] = *r; \ + } while (++r != end); \ + \ + if (unlikely(key_diff_mask < 256)) { \ + memcpy(begin, tmp, ptr_dist(end, begin)); \ + break; \ + } \ + end = (r = tmp) + length; \ + do { \ + const size_t key = EXTRACT_KEY(r) >> key_shift; \ + begin[counters.b[(key >> 8) & 255]++] = *r; \ + } while (++r != end); \ + \ + key_shift += 16; \ + } while (key_diff_mask >> 16); \ + \ + if (!(BUFFER_PREALLOCATED)) \ + osal_free(tmp); \ + return true; \ } - VERBOSE("lru-head %u, age-max %u", txn->tw.dirtylru, age_max); +/*------------------------------------------------------------------------------ + * LY: Binary search */ - /* half of 8-bit radix-sort */ - pgno_t radix_entries[256], radix_npages[256]; - memset(&radix_entries, 0, sizeof(radix_entries)); - memset(&radix_npages, 0, sizeof(radix_npages)); - size_t spillable_entries = 0, spillable_npages = 0; - const uint32_t reciprocal = (UINT32_C(255) << 24) / (age_max + 1); - for (size_t i = 1; i <= dl->length; ++i) { - const unsigned prio = spill_prio(txn, i, reciprocal); - size_t *const ptr = ptr_disp(dl->items[i].ptr, -(ptrdiff_t)sizeof(size_t)); - TRACE("page %" PRIaPGNO - ", lru %zu, is_multi %c, npages %u, age %u of %u, prio %u", - dl->items[i].pgno, *ptr, (dl->items[i].npages > 1) ? 'Y' : 'N', - dpl_npages(dl, i), dpl_age(txn, i), age_max, prio); - if (prio < 256) { - radix_entries[prio] += 1; - spillable_entries += 1; - const pgno_t npages = dpl_npages(dl, i); - radix_npages[prio] += npages; - spillable_npages += npages; - } +#if defined(__clang__) && __clang_major__ > 4 && defined(__ia32__) +#define WORKAROUND_FOR_CLANG_OPTIMIZER_BUG(size, flag) \ + do \ + __asm __volatile("" \ + : "+r"(size) \ + : "r" /* the `b` constraint is more suitable here, but \ + cause CLANG to allocate and push/pop an one more \ + register, so using the `r` which avoids this. */ \ + (flag)); \ + while (0) +#else +#define WORKAROUND_FOR_CLANG_OPTIMIZER_BUG(size, flag) \ + do { \ + /* nope for non-clang or non-x86 */; \ + } while (0) +#endif /* Workaround for CLANG */ + +#define SEARCH_IMPL(NAME, TYPE_LIST, TYPE_ARG, CMP) \ + static __always_inline const TYPE_LIST *NAME( \ + const TYPE_LIST *it, size_t length, const TYPE_ARG item) { \ + const TYPE_LIST *const begin = it, *const end = begin + length; \ + \ + if (MDBX_HAVE_CMOV) \ + do { \ + /* Адаптивно-упрощенный шаг двоичного поиска: \ + * - без переходов при наличии cmov или аналога; \ + * - допускает лишние итерации; \ + * - но ищет пока size > 2, что требует дозавершения поиска \ + * среди остающихся 0-1-2 элементов. */ \ + const TYPE_LIST *const middle = it + (length >> 1); \ + length = (length + 1) >> 1; \ + const bool flag = expect_with_probability(CMP(*middle, item), 0, .5); \ + WORKAROUND_FOR_CLANG_OPTIMIZER_BUG(length, flag); \ + it = flag ? middle : it; \ + } while (length > 2); \ + else \ + while (length > 2) { \ + /* Вариант с использованием условного перехода. Основное отличие в \ + * том, что при "не равно" (true от компаратора) переход делается на 1 \ + * ближе к концу массива. Алгоритмически это верно и обеспечивает \ + * чуть-чуть более быструю сходимость, но зато требует больше \ + * вычислений при true от компаратора. Также ВАЖНО(!) не допускается \ + * спекулятивное выполнение при size == 0. */ \ + const TYPE_LIST *const middle = it + (length >> 1); \ + length = (length + 1) >> 1; \ + const bool flag = expect_with_probability(CMP(*middle, item), 0, .5); \ + if (flag) { \ + it = middle + 1; \ + length -= 1; \ + } \ + } \ + it += length > 1 && expect_with_probability(CMP(*it, item), 0, .5); \ + it += length > 0 && expect_with_probability(CMP(*it, item), 0, .5); \ + \ + if (AUDIT_ENABLED()) { \ + for (const TYPE_LIST *scan = begin; scan < it; ++scan) \ + assert(CMP(*scan, item)); \ + for (const TYPE_LIST *scan = it; scan < end; ++scan) \ + assert(!CMP(*scan, item)); \ + (void)begin, (void)end; \ + } \ + \ + return it; \ } +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 - tASSERT(txn, spillable_npages >= spillable_entries); - pgno_t spilled_entries = 0, spilled_npages = 0; - if (likely(spillable_entries > 0)) { - size_t prio2spill = 0, prio2adjacent = 128, - amount_entries = radix_entries[0], amount_npages = radix_npages[0]; - for (size_t i = 1; i < 256; i++) { - if (amount_entries < need_spill_entries || - amount_npages < need_spill_npages) { - prio2spill = i; - prio2adjacent = i + (257 - i) / 2; - amount_entries += radix_entries[i]; - amount_npages += radix_npages[i]; - } else if (amount_entries + amount_entries < - spillable_entries + need_spill_entries - /* РАВНОЗНАЧНО: amount - need_spill < spillable - amount */ - || amount_npages + amount_npages < - spillable_npages + need_spill_npages) { - prio2adjacent = i; - amount_entries += radix_entries[i]; - amount_npages += radix_npages[i]; - } else - break; - } +__cold size_t mdbx_default_pagesize(void) { + size_t pagesize = globals.sys_pagesize; + ENSURE(nullptr, is_powerof2(pagesize)); + pagesize = (pagesize >= MDBX_MIN_PAGESIZE) ? pagesize : MDBX_MIN_PAGESIZE; + pagesize = (pagesize <= MDBX_MAX_PAGESIZE) ? pagesize : MDBX_MAX_PAGESIZE; + return pagesize; +} - VERBOSE("prio2spill %zu, prio2adjacent %zu, spillable %zu/%zu," - " wanna-spill %zu/%zu, amount %zu/%zu", - prio2spill, prio2adjacent, spillable_entries, spillable_npages, - need_spill_entries, need_spill_npages, amount_entries, - amount_npages); - tASSERT(txn, prio2spill < prio2adjacent && prio2adjacent <= 256); +__cold intptr_t mdbx_limits_dbsize_min(intptr_t pagesize) { + if (pagesize < 1) + pagesize = (intptr_t)mdbx_default_pagesize(); + else if (unlikely(pagesize < (intptr_t)MDBX_MIN_PAGESIZE || pagesize > (intptr_t)MDBX_MAX_PAGESIZE || + !is_powerof2((size_t)pagesize))) + return -1; - iov_ctx_t ctx; - rc = - iov_init(txn, &ctx, amount_entries, amount_npages, -#if defined(_WIN32) || defined(_WIN64) - txn->mt_env->me_overlapped_fd ? txn->mt_env->me_overlapped_fd : -#endif - txn->mt_env->me_lazy_fd, - true); - if (unlikely(rc != MDBX_SUCCESS)) - goto bailout; + return MIN_PAGENO * pagesize; +} - size_t r = 0, w = 0; - pgno_t last = 0; - while (r < dl->length && (spilled_entries < need_spill_entries || - spilled_npages < need_spill_npages)) { - dl->items[++w] = dl->items[++r]; - unsigned prio = spill_prio(txn, w, reciprocal); - if (prio > prio2spill && - (prio >= prio2adjacent || last != dl->items[w].pgno)) - continue; - - const size_t e = w; - last = dpl_endpgno(dl, w); - while (--w && dpl_endpgno(dl, w) == dl->items[w + 1].pgno && - spill_prio(txn, w, reciprocal) < prio2adjacent) - ; - - for (size_t i = w; ++i <= e;) { - const unsigned npages = dpl_npages(dl, i); - prio = spill_prio(txn, i, reciprocal); - DEBUG("%sspill[%zu] %u page %" PRIaPGNO " (age %d, prio %u)", - (prio > prio2spill) ? "co-" : "", i, npages, dl->items[i].pgno, - dpl_age(txn, i), prio); - tASSERT(txn, prio < 256); - ++spilled_entries; - spilled_npages += npages; - rc = spill_page(txn, &ctx, dl->items[i].ptr, npages); - if (unlikely(rc != MDBX_SUCCESS)) - goto failed; - } - } - - VERBOSE("spilled entries %u, spilled npages %u", spilled_entries, - spilled_npages); - tASSERT(txn, spillable_entries == 0 || spilled_entries > 0); - tASSERT(txn, spilled_npages >= spilled_entries); - - failed: - while (r < dl->length) - dl->items[++w] = dl->items[++r]; - tASSERT(txn, r - w == spilled_entries || rc != MDBX_SUCCESS); - - dl->sorted = dpl_setlen(dl, w); - txn->tw.dirtyroom += spilled_entries; - txn->tw.dirtylist->pages_including_loose -= spilled_npages; - tASSERT(txn, dirtylist_check(txn)); - - if (!iov_empty(&ctx)) { - tASSERT(txn, rc == MDBX_SUCCESS); - rc = iov_write(&ctx); - } - if (unlikely(rc != MDBX_SUCCESS)) - goto bailout; - - txn->mt_env->me_lck->mti_unsynced_pages.weak += spilled_npages; - pnl_sort(txn->tw.spilled.list, (size_t)txn->mt_next_pgno << 1); - txn->mt_flags |= MDBX_TXN_SPILLS; - NOTICE("spilled %u dirty-entries, %u dirty-npages, now have %zu dirty-room", - spilled_entries, spilled_npages, txn->tw.dirtyroom); - } else { - tASSERT(txn, rc == MDBX_SUCCESS); - for (size_t i = 1; i <= dl->length; ++i) { - MDBX_page *dp = dl->items[i].ptr; - VERBOSE( - "unspillable[%zu]: pgno %u, npages %u, flags 0x%04X, age %u, prio %u", - i, dp->mp_pgno, dpl_npages(dl, i), dp->mp_flags, dpl_age(txn, i), - spill_prio(txn, i, reciprocal)); - } - } - -#if xMDBX_DEBUG_SPILLING == 2 - if (txn->tw.loose_count + txn->tw.dirtyroom <= need / 2 + 1) - ERROR("dirty-list length: before %zu, after %zu, parent %zi, loose %zu; " - "needed %zu, spillable %zu; " - "spilled %u dirty-entries, now have %zu dirty-room", - dl->length + spilled_entries, dl->length, - (txn->mt_parent && txn->mt_parent->tw.dirtylist) - ? (intptr_t)txn->mt_parent->tw.dirtylist->length - : -1, - txn->tw.loose_count, need, spillable_entries, spilled_entries, - txn->tw.dirtyroom); - ENSURE(txn->mt_env, txn->tw.loose_count + txn->tw.dirtyroom > need / 2); -#endif /* xMDBX_DEBUG_SPILLING */ - -done: - return likely(txn->tw.dirtyroom + txn->tw.loose_count > - ((need > CURSOR_STACK) ? CURSOR_STACK : need)) - ? MDBX_SUCCESS - : MDBX_TXN_FULL; -} - -/*----------------------------------------------------------------------------*/ - -static bool meta_bootid_match(const MDBX_meta *meta) { - return memcmp(&meta->mm_bootid, &bootid, 16) == 0 && - (bootid.x | bootid.y) != 0; -} - -static bool meta_weak_acceptable(const MDBX_env *env, const MDBX_meta *meta, - const int lck_exclusive) { - return lck_exclusive - ? /* exclusive lock */ meta_bootid_match(meta) - : /* db already opened */ env->me_lck_mmap.lck && - (env->me_lck_mmap.lck->mti_envmode.weak & MDBX_RDONLY) == 0; -} - -#define METAPAGE(env, n) page_meta(pgno2page(env, n)) -#define METAPAGE_END(env) METAPAGE(env, NUM_METAS) - -MDBX_NOTHROW_PURE_FUNCTION static txnid_t -constmeta_txnid(const MDBX_meta *meta) { - const txnid_t a = unaligned_peek_u64(4, &meta->mm_txnid_a); - const txnid_t b = unaligned_peek_u64(4, &meta->mm_txnid_b); - return likely(a == b) ? a : 0; -} - -typedef struct { - uint64_t txnid; - size_t is_steady; -} meta_snap_t; - -static __always_inline txnid_t -atomic_load_txnid(const volatile MDBX_atomic_uint32_t *ptr) { -#if (defined(__amd64__) || defined(__e2k__)) && !defined(ENABLE_UBSAN) && \ - MDBX_UNALIGNED_OK >= 8 - return atomic_load64((const volatile MDBX_atomic_uint64_t *)ptr, - mo_AcquireRelease); -#else - const uint32_t l = atomic_load32( - &ptr[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__], mo_AcquireRelease); - const uint32_t h = atomic_load32( - &ptr[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__], mo_AcquireRelease); - return (uint64_t)h << 32 | l; -#endif -} +__cold intptr_t mdbx_limits_dbsize_max(intptr_t pagesize) { + if (pagesize < 1) + pagesize = (intptr_t)mdbx_default_pagesize(); + else if (unlikely(pagesize < (intptr_t)MDBX_MIN_PAGESIZE || pagesize > (intptr_t)MDBX_MAX_PAGESIZE || + !is_powerof2((size_t)pagesize))) + return -1; -static __inline meta_snap_t meta_snap(const volatile MDBX_meta *meta) { - txnid_t txnid = atomic_load_txnid(meta->mm_txnid_a); - jitter4testing(true); - size_t is_steady = META_IS_STEADY(meta) && txnid >= MIN_TXNID; - jitter4testing(true); - if (unlikely(txnid != atomic_load_txnid(meta->mm_txnid_b))) - txnid = is_steady = 0; - meta_snap_t r = {txnid, is_steady}; - return r; + STATIC_ASSERT(MAX_MAPSIZE < INTPTR_MAX); + const uint64_t limit = (1 + (uint64_t)MAX_PAGENO) * pagesize; + return (limit < MAX_MAPSIZE) ? (intptr_t)limit : (intptr_t)MAX_MAPSIZE; } -static __inline txnid_t meta_txnid(const volatile MDBX_meta *meta) { - return meta_snap(meta).txnid; -} +__cold intptr_t mdbx_limits_txnsize_max(intptr_t pagesize) { + if (pagesize < 1) + pagesize = (intptr_t)mdbx_default_pagesize(); + else if (unlikely(pagesize < (intptr_t)MDBX_MIN_PAGESIZE || pagesize > (intptr_t)MDBX_MAX_PAGESIZE || + !is_powerof2((size_t)pagesize))) + return -1; -static __inline void meta_update_begin(const MDBX_env *env, MDBX_meta *meta, - txnid_t txnid) { - eASSERT(env, meta >= METAPAGE(env, 0) && meta < METAPAGE_END(env)); - eASSERT(env, unaligned_peek_u64(4, meta->mm_txnid_a) < txnid && - unaligned_peek_u64(4, meta->mm_txnid_b) < txnid); - (void)env; -#if (defined(__amd64__) || defined(__e2k__)) && !defined(ENABLE_UBSAN) && \ - MDBX_UNALIGNED_OK >= 8 - atomic_store64((MDBX_atomic_uint64_t *)&meta->mm_txnid_b, 0, - mo_AcquireRelease); - atomic_store64((MDBX_atomic_uint64_t *)&meta->mm_txnid_a, txnid, - mo_AcquireRelease); -#else - atomic_store32(&meta->mm_txnid_b[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__], - 0, mo_AcquireRelease); - atomic_store32(&meta->mm_txnid_b[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__], - 0, mo_AcquireRelease); - atomic_store32(&meta->mm_txnid_a[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__], - (uint32_t)txnid, mo_AcquireRelease); - atomic_store32(&meta->mm_txnid_a[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__], - (uint32_t)(txnid >> 32), mo_AcquireRelease); -#endif -} - -static __inline void meta_update_end(const MDBX_env *env, MDBX_meta *meta, - txnid_t txnid) { - eASSERT(env, meta >= METAPAGE(env, 0) && meta < METAPAGE_END(env)); - eASSERT(env, unaligned_peek_u64(4, meta->mm_txnid_a) == txnid); - eASSERT(env, unaligned_peek_u64(4, meta->mm_txnid_b) < txnid); - (void)env; - jitter4testing(true); - memcpy(&meta->mm_bootid, &bootid, 16); -#if (defined(__amd64__) || defined(__e2k__)) && !defined(ENABLE_UBSAN) && \ - MDBX_UNALIGNED_OK >= 8 - atomic_store64((MDBX_atomic_uint64_t *)&meta->mm_txnid_b, txnid, - mo_AcquireRelease); -#else - atomic_store32(&meta->mm_txnid_b[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__], - (uint32_t)txnid, mo_AcquireRelease); - atomic_store32(&meta->mm_txnid_b[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__], - (uint32_t)(txnid >> 32), mo_AcquireRelease); -#endif + STATIC_ASSERT(MAX_MAPSIZE < INTPTR_MAX); + const uint64_t pgl_limit = pagesize * (uint64_t)(PAGELIST_LIMIT / MDBX_GOLD_RATIO_DBL); + const uint64_t map_limit = (uint64_t)(MAX_MAPSIZE / MDBX_GOLD_RATIO_DBL); + return (pgl_limit < map_limit) ? (intptr_t)pgl_limit : (intptr_t)map_limit; } -static __inline void meta_set_txnid(const MDBX_env *env, MDBX_meta *meta, - const txnid_t txnid) { - eASSERT(env, - !env->me_map || meta < METAPAGE(env, 0) || meta >= METAPAGE_END(env)); - (void)env; - /* update inconsistently since this function used ONLY for filling meta-image - * for writing, but not the actual meta-page */ - memcpy(&meta->mm_bootid, &bootid, 16); - unaligned_poke_u64(4, meta->mm_txnid_a, txnid); - unaligned_poke_u64(4, meta->mm_txnid_b, txnid); -} +__cold intptr_t mdbx_limits_keysize_max(intptr_t pagesize, MDBX_db_flags_t flags) { + if (pagesize < 1) + pagesize = (intptr_t)mdbx_default_pagesize(); + if (unlikely(pagesize < (intptr_t)MDBX_MIN_PAGESIZE || pagesize > (intptr_t)MDBX_MAX_PAGESIZE || + !is_powerof2((size_t)pagesize))) + return -1; -static __inline uint64_t meta_sign(const MDBX_meta *meta) { - uint64_t sign = MDBX_DATASIGN_NONE; -#if 0 /* TODO */ - sign = hippeus_hash64(...); -#else - (void)meta; -#endif - /* LY: newer returns MDBX_DATASIGN_NONE or MDBX_DATASIGN_WEAK */ - return (sign > MDBX_DATASIGN_WEAK) ? sign : ~sign; + return keysize_max(pagesize, flags); } -typedef struct { - txnid_t txnid; - union { - const volatile MDBX_meta *ptr_v; - const MDBX_meta *ptr_c; - }; - size_t is_steady; -} meta_ptr_t; - -static meta_ptr_t meta_ptr(const MDBX_env *env, unsigned n) { - eASSERT(env, n < NUM_METAS); - meta_ptr_t r; - meta_snap_t snap = meta_snap(r.ptr_v = METAPAGE(env, n)); - r.txnid = snap.txnid; - r.is_steady = snap.is_steady; - return r; -} +__cold int mdbx_env_get_maxkeysize_ex(const MDBX_env *env, MDBX_db_flags_t flags) { + if (unlikely(!env || env->signature.weak != env_signature)) + return -1; -static __always_inline uint8_t meta_cmp2int(txnid_t a, txnid_t b, uint8_t s) { - return unlikely(a == b) ? 1 * s : (a > b) ? 2 * s : 0 * s; + return (int)mdbx_limits_keysize_max((intptr_t)env->ps, flags); } -static __always_inline uint8_t meta_cmp2recent(uint8_t ab_cmp2int, - bool a_steady, bool b_steady) { - assert(ab_cmp2int < 3 /* && a_steady< 2 && b_steady < 2 */); - return ab_cmp2int > 1 || (ab_cmp2int == 1 && a_steady > b_steady); -} +__cold int mdbx_env_get_maxkeysize(const MDBX_env *env) { return mdbx_env_get_maxkeysize_ex(env, MDBX_DUPSORT); } -static __always_inline uint8_t meta_cmp2steady(uint8_t ab_cmp2int, - bool a_steady, bool b_steady) { - assert(ab_cmp2int < 3 /* && a_steady< 2 && b_steady < 2 */); - return a_steady > b_steady || (a_steady == b_steady && ab_cmp2int > 1); -} +__cold intptr_t mdbx_limits_keysize_min(MDBX_db_flags_t flags) { return keysize_min(flags); } -static __inline bool meta_choice_recent(txnid_t a_txnid, bool a_steady, - txnid_t b_txnid, bool b_steady) { - return meta_cmp2recent(meta_cmp2int(a_txnid, b_txnid, 1), a_steady, b_steady); -} +__cold intptr_t mdbx_limits_valsize_max(intptr_t pagesize, MDBX_db_flags_t flags) { + if (pagesize < 1) + pagesize = (intptr_t)mdbx_default_pagesize(); + if (unlikely(pagesize < (intptr_t)MDBX_MIN_PAGESIZE || pagesize > (intptr_t)MDBX_MAX_PAGESIZE || + !is_powerof2((size_t)pagesize))) + return -1; -static __inline bool meta_choice_steady(txnid_t a_txnid, bool a_steady, - txnid_t b_txnid, bool b_steady) { - return meta_cmp2steady(meta_cmp2int(a_txnid, b_txnid, 1), a_steady, b_steady); + return valsize_max(pagesize, flags); } -MDBX_MAYBE_UNUSED static uint8_t meta_cmp2pack(uint8_t c01, uint8_t c02, - uint8_t c12, bool s0, bool s1, - bool s2) { - assert(c01 < 3 && c02 < 3 && c12 < 3); - /* assert(s0 < 2 && s1 < 2 && s2 < 2); */ - const uint8_t recent = meta_cmp2recent(c01, s0, s1) - ? (meta_cmp2recent(c02, s0, s2) ? 0 : 2) - : (meta_cmp2recent(c12, s1, s2) ? 1 : 2); - const uint8_t prefer_steady = meta_cmp2steady(c01, s0, s1) - ? (meta_cmp2steady(c02, s0, s2) ? 0 : 2) - : (meta_cmp2steady(c12, s1, s2) ? 1 : 2); - - uint8_t tail; - if (recent == 0) - tail = meta_cmp2steady(c12, s1, s2) ? 2 : 1; - else if (recent == 1) - tail = meta_cmp2steady(c02, s0, s2) ? 2 : 0; - else - tail = meta_cmp2steady(c01, s0, s1) ? 1 : 0; - - const bool valid = - c01 != 1 || s0 != s1 || c02 != 1 || s0 != s2 || c12 != 1 || s1 != s2; - const bool strict = (c01 != 1 || s0 != s1) && (c02 != 1 || s0 != s2) && - (c12 != 1 || s1 != s2); - return tail | recent << 2 | prefer_steady << 4 | strict << 6 | valid << 7; -} +__cold int mdbx_env_get_maxvalsize_ex(const MDBX_env *env, MDBX_db_flags_t flags) { + if (unlikely(!env || env->signature.weak != env_signature)) + return -1; -static __inline void meta_troika_unpack(meta_troika_t *troika, - const uint8_t packed) { - troika->recent = (packed >> 2) & 3; - troika->prefer_steady = (packed >> 4) & 3; - troika->tail_and_flags = packed & 0xC3; -#if MDBX_WORDBITS > 32 /* Workaround for false-positives from Valgrind */ - troika->unused_pad = 0; -#endif + return (int)mdbx_limits_valsize_max((intptr_t)env->ps, flags); } -static const uint8_t troika_fsm_map[2 * 2 * 2 * 3 * 3 * 3] = { - 232, 201, 216, 216, 232, 233, 232, 232, 168, 201, 216, 152, 168, 233, 232, - 168, 233, 201, 216, 201, 233, 233, 232, 233, 168, 201, 152, 216, 232, 169, - 232, 168, 168, 193, 152, 152, 168, 169, 232, 168, 169, 193, 152, 194, 233, - 169, 232, 169, 232, 201, 216, 216, 232, 201, 232, 232, 168, 193, 216, 152, - 168, 193, 232, 168, 193, 193, 210, 194, 225, 193, 225, 193, 168, 137, 212, - 214, 232, 233, 168, 168, 168, 137, 212, 150, 168, 233, 168, 168, 169, 137, - 216, 201, 233, 233, 168, 169, 168, 137, 148, 214, 232, 169, 168, 168, 40, - 129, 148, 150, 168, 169, 168, 40, 169, 129, 152, 194, 233, 169, 168, 169, - 168, 137, 214, 214, 232, 201, 168, 168, 168, 129, 214, 150, 168, 193, 168, - 168, 129, 129, 210, 194, 225, 193, 161, 129, 212, 198, 212, 214, 228, 228, - 212, 212, 148, 201, 212, 150, 164, 233, 212, 148, 233, 201, 216, 201, 233, - 233, 216, 233, 148, 198, 148, 214, 228, 164, 212, 148, 148, 194, 148, 150, - 164, 169, 212, 148, 169, 194, 152, 194, 233, 169, 216, 169, 214, 198, 214, - 214, 228, 198, 212, 214, 150, 194, 214, 150, 164, 193, 212, 150, 194, 194, - 210, 194, 225, 193, 210, 194}; - -__hot static meta_troika_t meta_tap(const MDBX_env *env) { - meta_snap_t snap; - meta_troika_t troika; - snap = meta_snap(METAPAGE(env, 0)); - troika.txnid[0] = snap.txnid; - troika.fsm = (uint8_t)snap.is_steady << 0; - snap = meta_snap(METAPAGE(env, 1)); - troika.txnid[1] = snap.txnid; - troika.fsm += (uint8_t)snap.is_steady << 1; - troika.fsm += meta_cmp2int(troika.txnid[0], troika.txnid[1], 8); - snap = meta_snap(METAPAGE(env, 2)); - troika.txnid[2] = snap.txnid; - troika.fsm += (uint8_t)snap.is_steady << 2; - troika.fsm += meta_cmp2int(troika.txnid[0], troika.txnid[2], 8 * 3); - troika.fsm += meta_cmp2int(troika.txnid[1], troika.txnid[2], 8 * 3 * 3); +__cold intptr_t mdbx_limits_valsize_min(MDBX_db_flags_t flags) { return valsize_min(flags); } - meta_troika_unpack(&troika, troika_fsm_map[troika.fsm]); - return troika; -} +__cold intptr_t mdbx_limits_pairsize4page_max(intptr_t pagesize, MDBX_db_flags_t flags) { + if (pagesize < 1) + pagesize = (intptr_t)mdbx_default_pagesize(); + if (unlikely(pagesize < (intptr_t)MDBX_MIN_PAGESIZE || pagesize > (intptr_t)MDBX_MAX_PAGESIZE || + !is_powerof2((size_t)pagesize))) + return -1; -static txnid_t recent_committed_txnid(const MDBX_env *env) { - const txnid_t m0 = meta_txnid(METAPAGE(env, 0)); - const txnid_t m1 = meta_txnid(METAPAGE(env, 1)); - const txnid_t m2 = meta_txnid(METAPAGE(env, 2)); - return (m0 > m1) ? ((m0 > m2) ? m0 : m2) : ((m1 > m2) ? m1 : m2); -} + if (flags & (MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERDUP | MDBX_REVERSEDUP)) + return BRANCH_NODE_MAX(pagesize) - NODESIZE; -static __inline bool meta_eq(const meta_troika_t *troika, size_t a, size_t b) { - assert(a < NUM_METAS && b < NUM_METAS); - return troika->txnid[a] == troika->txnid[b] && - (((troika->fsm >> a) ^ (troika->fsm >> b)) & 1) == 0 && - troika->txnid[a]; + return LEAF_NODE_MAX(pagesize) - NODESIZE; } -static unsigned meta_eq_mask(const meta_troika_t *troika) { - return meta_eq(troika, 0, 1) | meta_eq(troika, 1, 2) << 1 | - meta_eq(troika, 2, 0) << 2; -} +__cold int mdbx_env_get_pairsize4page_max(const MDBX_env *env, MDBX_db_flags_t flags) { + if (unlikely(!env || env->signature.weak != env_signature)) + return -1; -__hot static bool meta_should_retry(const MDBX_env *env, - meta_troika_t *troika) { - const meta_troika_t prev = *troika; - *troika = meta_tap(env); - return prev.fsm != troika->fsm || prev.txnid[0] != troika->txnid[0] || - prev.txnid[1] != troika->txnid[1] || prev.txnid[2] != troika->txnid[2]; + return (int)mdbx_limits_pairsize4page_max((intptr_t)env->ps, flags); } -static __always_inline meta_ptr_t meta_recent(const MDBX_env *env, - const meta_troika_t *troika) { - meta_ptr_t r; - r.txnid = troika->txnid[troika->recent]; - r.ptr_v = METAPAGE(env, troika->recent); - r.is_steady = (troika->fsm >> troika->recent) & 1; - return r; -} +__cold intptr_t mdbx_limits_valsize4page_max(intptr_t pagesize, MDBX_db_flags_t flags) { + if (pagesize < 1) + pagesize = (intptr_t)mdbx_default_pagesize(); + if (unlikely(pagesize < (intptr_t)MDBX_MIN_PAGESIZE || pagesize > (intptr_t)MDBX_MAX_PAGESIZE || + !is_powerof2((size_t)pagesize))) + return -1; -static __always_inline meta_ptr_t -meta_prefer_steady(const MDBX_env *env, const meta_troika_t *troika) { - meta_ptr_t r; - r.txnid = troika->txnid[troika->prefer_steady]; - r.ptr_v = METAPAGE(env, troika->prefer_steady); - r.is_steady = (troika->fsm >> troika->prefer_steady) & 1; - return r; -} + if (flags & (MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERDUP | MDBX_REVERSEDUP)) + return valsize_max(pagesize, flags); -static __always_inline meta_ptr_t meta_tail(const MDBX_env *env, - const meta_troika_t *troika) { - const uint8_t tail = troika->tail_and_flags & 3; - MDBX_ANALYSIS_ASSUME(tail < NUM_METAS); - meta_ptr_t r; - r.txnid = troika->txnid[tail]; - r.ptr_v = METAPAGE(env, tail); - r.is_steady = (troika->fsm >> tail) & 1; - return r; + return PAGESPACE(pagesize); } -static const char *durable_caption(const volatile MDBX_meta *const meta) { - if (META_IS_STEADY(meta)) - return (unaligned_peek_u64_volatile(4, meta->mm_sign) == - meta_sign((const MDBX_meta *)meta)) - ? "Steady" - : "Tainted"; - return "Weak"; -} +__cold int mdbx_env_get_valsize4page_max(const MDBX_env *env, MDBX_db_flags_t flags) { + if (unlikely(!env || env->signature.weak != env_signature)) + return -1; -__cold static void meta_troika_dump(const MDBX_env *env, - const meta_troika_t *troika) { - const meta_ptr_t recent = meta_recent(env, troika); - const meta_ptr_t prefer_steady = meta_prefer_steady(env, troika); - const meta_ptr_t tail = meta_tail(env, troika); - NOTICE("%" PRIaTXN ".%c:%" PRIaTXN ".%c:%" PRIaTXN ".%c, fsm=0x%02x, " - "head=%d-%" PRIaTXN ".%c, " - "base=%d-%" PRIaTXN ".%c, " - "tail=%d-%" PRIaTXN ".%c, " - "valid %c, strict %c", - troika->txnid[0], (troika->fsm & 1) ? 's' : 'w', troika->txnid[1], - (troika->fsm & 2) ? 's' : 'w', troika->txnid[2], - (troika->fsm & 4) ? 's' : 'w', troika->fsm, troika->recent, - recent.txnid, recent.is_steady ? 's' : 'w', troika->prefer_steady, - prefer_steady.txnid, prefer_steady.is_steady ? 's' : 'w', - troika->tail_and_flags % NUM_METAS, tail.txnid, - tail.is_steady ? 's' : 'w', TROIKA_VALID(troika) ? 'Y' : 'N', - TROIKA_STRICT_VALID(troika) ? 'Y' : 'N'); + return (int)mdbx_limits_valsize4page_max((intptr_t)env->ps, flags); } /*----------------------------------------------------------------------------*/ -static __inline MDBX_CONST_FUNCTION MDBX_lockinfo * -lckless_stub(const MDBX_env *env) { - uintptr_t stub = (uintptr_t)&env->x_lckless_stub; - /* align to avoid false-positive alarm from UndefinedBehaviorSanitizer */ - stub = (stub + MDBX_CACHELINE_SIZE - 1) & ~(MDBX_CACHELINE_SIZE - 1); - return (MDBX_lockinfo *)stub; +static size_t estimate_rss(size_t database_bytes) { + return database_bytes + database_bytes / 64 + (512 + MDBX_WORDBITS * 16) * MEGABYTE; } -/* Find oldest txnid still referenced. */ -static txnid_t find_oldest_reader(MDBX_env *const env, const txnid_t steady) { - const uint32_t nothing_changed = MDBX_STRING_TETRAD("None"); - eASSERT(env, steady <= env->me_txn0->mt_txnid); +__cold int mdbx_env_warmup(const MDBX_env *env, const MDBX_txn *txn, MDBX_warmup_flags_t flags, + unsigned timeout_seconds_16dot16) { + if (unlikely(env == nullptr && txn == nullptr)) + return LOG_IFERR(MDBX_EINVAL); + if (unlikely(flags > (MDBX_warmup_force | MDBX_warmup_oomsafe | MDBX_warmup_lock | MDBX_warmup_touchlimit | + MDBX_warmup_release))) + return LOG_IFERR(MDBX_EINVAL); - MDBX_lockinfo *const lck = env->me_lck_mmap.lck; - if (unlikely(lck == NULL /* exclusive without-lck mode */)) { - eASSERT(env, env->me_lck == lckless_stub(env)); - env->me_lck->mti_readers_refresh_flag.weak = nothing_changed; - return env->me_lck->mti_oldest_reader.weak = steady; + if (txn) { + int err = check_txn(txn, MDBX_TXN_FINISHED | MDBX_TXN_ERROR); + if (unlikely(err != MDBX_SUCCESS)) + return LOG_IFERR(err); + } + if (env) { + int err = check_env(env, false); + if (unlikely(err != MDBX_SUCCESS)) + return LOG_IFERR(err); + if (txn && unlikely(txn->env != env)) + return LOG_IFERR(MDBX_EINVAL); + } else { + env = txn->env; } - const txnid_t prev_oldest = - atomic_load64(&lck->mti_oldest_reader, mo_AcquireRelease); - eASSERT(env, steady >= prev_oldest); + const uint64_t timeout_monotime = (timeout_seconds_16dot16 && (flags & MDBX_warmup_force)) + ? osal_monotime() + osal_16dot16_to_monotime(timeout_seconds_16dot16) + : 0; - txnid_t new_oldest = prev_oldest; - while (nothing_changed != - atomic_load32(&lck->mti_readers_refresh_flag, mo_AcquireRelease)) { - lck->mti_readers_refresh_flag.weak = nothing_changed; - jitter4testing(false); - const size_t snap_nreaders = - atomic_load32(&lck->mti_numreaders, mo_AcquireRelease); - new_oldest = steady; + if (flags & MDBX_warmup_release) + munlock_all(env); - for (size_t i = 0; i < snap_nreaders; ++i) { - const uint32_t pid = - atomic_load32(&lck->mti_readers[i].mr_pid, mo_AcquireRelease); - if (!pid) - continue; - jitter4testing(true); + pgno_t used_pgno; + if (txn) { + used_pgno = txn->geo.first_unallocated; + } else { + const troika_t troika = meta_tap(env); + used_pgno = meta_recent(env, &troika).ptr_v->geometry.first_unallocated; + } + const size_t used_range = pgno_align2os_bytes(env, used_pgno); + const pgno_t mlock_pgno = bytes2pgno(env, used_range); - const txnid_t rtxn = safe64_read(&lck->mti_readers[i].mr_txnid); - if (unlikely(rtxn < prev_oldest)) { - if (unlikely(nothing_changed == - atomic_load32(&lck->mti_readers_refresh_flag, - mo_AcquireRelease)) && - safe64_reset_compare(&lck->mti_readers[i].mr_txnid, rtxn)) { - NOTICE("kick stuck reader[%zu of %zu].pid_%u %" PRIaTXN - " < prev-oldest %" PRIaTXN ", steady-txn %" PRIaTXN, - i, snap_nreaders, pid, rtxn, prev_oldest, steady); - } - continue; + int rc = MDBX_SUCCESS; + if (flags & MDBX_warmup_touchlimit) { + const size_t estimated_rss = estimate_rss(used_range); +#if defined(_WIN32) || defined(_WIN64) + SIZE_T current_ws_lower, current_ws_upper; + if (GetProcessWorkingSetSize(GetCurrentProcess(), ¤t_ws_lower, ¤t_ws_upper) && + current_ws_lower < estimated_rss) { + const SIZE_T ws_lower = estimated_rss; + const SIZE_T ws_upper = + (MDBX_WORDBITS == 32 && ws_lower > MEGABYTE * 2048) ? ws_lower : ws_lower + MDBX_WORDBITS * MEGABYTE * 32; + if (!SetProcessWorkingSetSize(GetCurrentProcess(), ws_lower, ws_upper)) { + rc = (int)GetLastError(); + WARNING("SetProcessWorkingSetSize(%zu, %zu) error %d", ws_lower, ws_upper, rc); } - - if (rtxn < new_oldest) { - new_oldest = rtxn; - if (!MDBX_DEBUG && !MDBX_FORCE_ASSERTIONS && new_oldest == prev_oldest) - break; + } +#endif /* Windows */ +#ifdef RLIMIT_RSS + struct rlimit rss; + if (getrlimit(RLIMIT_RSS, &rss) == 0 && rss.rlim_cur < estimated_rss) { + rss.rlim_cur = estimated_rss; + if (rss.rlim_max < estimated_rss) + rss.rlim_max = estimated_rss; + if (setrlimit(RLIMIT_RSS, &rss)) { + rc = errno; + WARNING("setrlimit(%s, {%zu, %zu}) error %d", "RLIMIT_RSS", (size_t)rss.rlim_cur, (size_t)rss.rlim_max, rc); + } + } +#endif /* RLIMIT_RSS */ +#ifdef RLIMIT_MEMLOCK + if (flags & MDBX_warmup_lock) { + struct rlimit memlock; + if (getrlimit(RLIMIT_MEMLOCK, &memlock) == 0 && memlock.rlim_cur < estimated_rss) { + memlock.rlim_cur = estimated_rss; + if (memlock.rlim_max < estimated_rss) + memlock.rlim_max = estimated_rss; + if (setrlimit(RLIMIT_MEMLOCK, &memlock)) { + rc = errno; + WARNING("setrlimit(%s, {%zu, %zu}) error %d", "RLIMIT_MEMLOCK", (size_t)memlock.rlim_cur, + (size_t)memlock.rlim_max, rc); + } } } +#endif /* RLIMIT_MEMLOCK */ + (void)estimated_rss; } - if (new_oldest != prev_oldest) { - VERBOSE("update oldest %" PRIaTXN " -> %" PRIaTXN, prev_oldest, new_oldest); - eASSERT(env, new_oldest >= lck->mti_oldest_reader.weak); - atomic_store64(&lck->mti_oldest_reader, new_oldest, mo_Relaxed); +#if defined(MLOCK_ONFAULT) && \ + ((defined(_GNU_SOURCE) && __GLIBC_PREREQ(2, 27)) || (defined(__ANDROID_API__) && __ANDROID_API__ >= 30)) && \ + (defined(__linux__) || defined(__gnu_linux__)) + if ((flags & MDBX_warmup_lock) != 0 && globals.linux_kernel_version >= 0x04040000 && + atomic_load32(&env->mlocked_pgno, mo_AcquireRelease) < mlock_pgno) { + if (mlock2(env->dxb_mmap.base, used_range, MLOCK_ONFAULT)) { + rc = errno; + WARNING("mlock2(%zu, %s) error %d", used_range, "MLOCK_ONFAULT", rc); + } else { + update_mlcnt(env, mlock_pgno, true); + rc = MDBX_SUCCESS; + } + if (rc != EINVAL) + flags -= MDBX_warmup_lock; } - return new_oldest; -} +#endif /* MLOCK_ONFAULT */ -static txnid_t txn_oldest_reader(const MDBX_txn *const txn) { - return find_oldest_reader(txn->mt_env, - txn->tw.troika.txnid[txn->tw.troika.prefer_steady]); -} + int err = MDBX_ENOSYS; + err = dxb_set_readahead(env, used_pgno, true, true); + if (err != MDBX_SUCCESS && rc == MDBX_SUCCESS) + rc = err; -/* Find largest mvcc-snapshot still referenced. */ -static pgno_t find_largest_snapshot(const MDBX_env *env, - pgno_t last_used_page) { - MDBX_lockinfo *const lck = env->me_lck_mmap.lck; - if (likely(lck != NULL /* check for exclusive without-lck mode */)) { - retry:; - const size_t snap_nreaders = - atomic_load32(&lck->mti_numreaders, mo_AcquireRelease); - for (size_t i = 0; i < snap_nreaders; ++i) { - if (atomic_load32(&lck->mti_readers[i].mr_pid, mo_AcquireRelease)) { - /* jitter4testing(true); */ - const pgno_t snap_pages = atomic_load32( - &lck->mti_readers[i].mr_snapshot_pages_used, mo_Relaxed); - const txnid_t snap_txnid = safe64_read(&lck->mti_readers[i].mr_txnid); - if (unlikely( - snap_pages != - atomic_load32(&lck->mti_readers[i].mr_snapshot_pages_used, - mo_AcquireRelease) || - snap_txnid != safe64_read(&lck->mti_readers[i].mr_txnid))) - goto retry; - if (last_used_page < snap_pages && snap_txnid <= env->me_txn0->mt_txnid) - last_used_page = snap_pages; + if ((flags & MDBX_warmup_force) != 0 && (rc == MDBX_SUCCESS || rc == MDBX_ENOSYS)) { + const volatile uint8_t *ptr = env->dxb_mmap.base; + size_t offset = 0, unused = 42; +#if !(defined(_WIN32) || defined(_WIN64)) + if (flags & MDBX_warmup_oomsafe) { + const int null_fd = open("/dev/null", O_WRONLY); + if (unlikely(null_fd < 0)) + rc = errno; + else { + struct iovec iov[MDBX_AUXILARY_IOV_MAX]; + for (;;) { + unsigned i; + for (i = 0; i < MDBX_AUXILARY_IOV_MAX && offset < used_range; ++i) { + iov[i].iov_base = (void *)(ptr + offset); + iov[i].iov_len = 1; + offset += globals.sys_pagesize; + } + if (unlikely(writev(null_fd, iov, i) < 0)) { + rc = errno; + if (rc == EFAULT) + rc = ENOMEM; + break; + } + if (offset >= used_range) { + rc = MDBX_SUCCESS; + break; + } + if (timeout_seconds_16dot16 && osal_monotime() > timeout_monotime) { + rc = MDBX_RESULT_TRUE; + break; + } + } + close(null_fd); + } + } else +#endif /* Windows */ + for (;;) { + unused += ptr[offset]; + offset += globals.sys_pagesize; + if (offset >= used_range) { + rc = MDBX_SUCCESS; + break; + } + if (timeout_seconds_16dot16 && osal_monotime() > timeout_monotime) { + rc = MDBX_RESULT_TRUE; + break; + } } + (void)unused; + } + + if ((flags & MDBX_warmup_lock) != 0 && (rc == MDBX_SUCCESS || rc == MDBX_ENOSYS) && + atomic_load32(&env->mlocked_pgno, mo_AcquireRelease) < mlock_pgno) { +#if defined(_WIN32) || defined(_WIN64) + if (VirtualLock(env->dxb_mmap.base, used_range)) { + update_mlcnt(env, mlock_pgno, true); + rc = MDBX_SUCCESS; + } else { + rc = (int)GetLastError(); + WARNING("%s(%zu) error %d", "VirtualLock", used_range, rc); + } +#elif defined(_POSIX_MEMLOCK_RANGE) + if (mlock(env->dxb_mmap.base, used_range) == 0) { + update_mlcnt(env, mlock_pgno, true); + rc = MDBX_SUCCESS; + } else { + rc = errno; + WARNING("%s(%zu) error %d", "mlock", used_range, rc); } +#else + rc = MDBX_ENOSYS; +#endif } - return last_used_page; + return LOG_IFERR(rc); } -/* Add a page to the txn's dirty list */ -__hot static int __must_check_result page_dirty(MDBX_txn *txn, MDBX_page *mp, - size_t npages) { - tASSERT(txn, (txn->mt_flags & MDBX_TXN_RDONLY) == 0); - mp->mp_txnid = txn->mt_front; - if (!txn->tw.dirtylist) { - tASSERT(txn, (txn->mt_flags & MDBX_WRITEMAP) != 0 && !MDBX_AVOID_MSYNC); - txn->tw.writemap_dirty_npages += npages; - tASSERT(txn, txn->tw.spilled.list == nullptr); - return MDBX_SUCCESS; - } - tASSERT(txn, (txn->mt_flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); +/*----------------------------------------------------------------------------*/ -#if xMDBX_DEBUG_SPILLING == 2 - txn->mt_env->debug_dirtied_act += 1; - ENSURE(txn->mt_env, - txn->mt_env->debug_dirtied_act < txn->mt_env->debug_dirtied_est); - ENSURE(txn->mt_env, txn->tw.dirtyroom + txn->tw.loose_count > 0); -#endif /* xMDBX_DEBUG_SPILLING == 2 */ +__cold int mdbx_env_get_fd(const MDBX_env *env, mdbx_filehandle_t *arg) { + int rc = check_env(env, true); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); - int rc; - if (unlikely(txn->tw.dirtyroom == 0)) { - if (txn->tw.loose_count) { - MDBX_page *lp = txn->tw.loose_pages; - DEBUG("purge-and-reclaim loose page %" PRIaPGNO, lp->mp_pgno); - rc = pnl_insert_range(&txn->tw.relist, lp->mp_pgno, 1); - if (unlikely(rc != MDBX_SUCCESS)) - goto bailout; - size_t di = dpl_search(txn, lp->mp_pgno); - tASSERT(txn, txn->tw.dirtylist->items[di].ptr == lp); - dpl_remove(txn, di); - MDBX_ASAN_UNPOISON_MEMORY_REGION(&mp_next(lp), sizeof(MDBX_page *)); - VALGRIND_MAKE_MEM_DEFINED(&mp_next(lp), sizeof(MDBX_page *)); - txn->tw.loose_pages = mp_next(lp); - txn->tw.loose_count--; - txn->tw.dirtyroom++; - if (!MDBX_AVOID_MSYNC || !(txn->mt_flags & MDBX_WRITEMAP)) - dpage_free(txn->mt_env, lp, 1); - } else { - ERROR("Dirtyroom is depleted, DPL length %zu", txn->tw.dirtylist->length); - if (!MDBX_AVOID_MSYNC || !(txn->mt_flags & MDBX_WRITEMAP)) - dpage_free(txn->mt_env, mp, npages); - return MDBX_TXN_FULL; - } - } + if (unlikely(!arg)) + return LOG_IFERR(MDBX_EINVAL); - rc = dpl_append(txn, mp->mp_pgno, mp, npages); - if (unlikely(rc != MDBX_SUCCESS)) { - bailout: - txn->mt_flags |= MDBX_TXN_ERROR; - return rc; + *arg = env->lazy_fd; + return MDBX_SUCCESS; +} + +__cold int mdbx_env_set_flags(MDBX_env *env, MDBX_env_flags_t flags, bool onoff) { + int rc = check_env(env, false); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); + + if (unlikely(flags & ((env->flags & ENV_ACTIVE) ? ~ENV_CHANGEABLE_FLAGS : ~ENV_USABLE_FLAGS))) + return LOG_IFERR(MDBX_EPERM); + + if (unlikely(env->flags & MDBX_RDONLY)) + return LOG_IFERR(MDBX_EACCESS); + + const bool lock_needed = (env->flags & ENV_ACTIVE) && !env_owned_wrtxn(env); + bool should_unlock = false; + if (lock_needed) { + rc = lck_txn_lock(env, false); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); + should_unlock = true; } - txn->tw.dirtyroom--; - tASSERT(txn, dirtylist_check(txn)); + + if (onoff) + env->flags = combine_durability_flags(env->flags, flags); + else + env->flags &= ~flags; + + if (should_unlock) + lck_txn_unlock(env); return MDBX_SUCCESS; } -static void mincore_clean_cache(const MDBX_env *const env) { - memset(env->me_lck->mti_mincore_cache.begin, -1, - sizeof(env->me_lck->mti_mincore_cache.begin)); +__cold int mdbx_env_get_flags(const MDBX_env *env, unsigned *flags) { + int rc = check_env(env, false); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); + + if (unlikely(!flags)) + return LOG_IFERR(MDBX_EINVAL); + + *flags = env->flags & ENV_USABLE_FLAGS; + return MDBX_SUCCESS; } -#if !(defined(_WIN32) || defined(_WIN64)) -MDBX_MAYBE_UNUSED static __always_inline int ignore_enosys(int err) { -#ifdef ENOSYS - if (err == ENOSYS) - return MDBX_RESULT_TRUE; -#endif /* ENOSYS */ -#ifdef ENOIMPL - if (err == ENOIMPL) - return MDBX_RESULT_TRUE; -#endif /* ENOIMPL */ -#ifdef ENOTSUP - if (err == ENOTSUP) - return MDBX_RESULT_TRUE; -#endif /* ENOTSUP */ -#ifdef ENOSUPP - if (err == ENOSUPP) - return MDBX_RESULT_TRUE; -#endif /* ENOSUPP */ -#ifdef EOPNOTSUPP - if (err == EOPNOTSUPP) - return MDBX_RESULT_TRUE; -#endif /* EOPNOTSUPP */ - if (err == EAGAIN) - return MDBX_RESULT_TRUE; - return err; +__cold int mdbx_env_set_userctx(MDBX_env *env, void *ctx) { + int rc = check_env(env, false); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); + + env->userctx = ctx; + return MDBX_SUCCESS; } -#endif /* defined(_WIN32) || defined(_WIN64) */ -#if MDBX_ENABLE_MADVISE -/* Turn on/off readahead. It's harmful when the DB is larger than RAM. */ -__cold static int set_readahead(const MDBX_env *env, const pgno_t edge, - const bool enable, const bool force_whole) { - eASSERT(env, edge >= NUM_METAS && edge <= MAX_PAGENO + 1); - eASSERT(env, (enable & 1) == (enable != 0)); - const bool toggle = force_whole || - ((enable ^ env->me_lck->mti_readahead_anchor) & 1) || - !env->me_lck->mti_readahead_anchor; - const pgno_t prev_edge = env->me_lck->mti_readahead_anchor >> 1; - const size_t limit = env->me_dxb_mmap.limit; - size_t offset = - toggle ? 0 - : pgno_align2os_bytes(env, (prev_edge < edge) ? prev_edge : edge); - offset = (offset < limit) ? offset : limit; +__cold void *mdbx_env_get_userctx(const MDBX_env *env) { return env ? env->userctx : nullptr; } - size_t length = - pgno_align2os_bytes(env, (prev_edge < edge) ? edge : prev_edge); - length = (length < limit) ? length : limit; - length -= offset; +__cold int mdbx_env_set_assert(MDBX_env *env, MDBX_assert_func *func) { + int rc = check_env(env, false); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); - eASSERT(env, 0 <= (intptr_t)length); - if (length == 0) - return MDBX_SUCCESS; +#if MDBX_DEBUG + env->assert_func = func; + return MDBX_SUCCESS; +#else + (void)func; + return LOG_IFERR(MDBX_ENOSYS); +#endif +} - NOTICE("readahead %s %u..%u", enable ? "ON" : "OFF", bytes2pgno(env, offset), - bytes2pgno(env, offset + length)); +__cold int mdbx_env_set_hsr(MDBX_env *env, MDBX_hsr_func *hsr) { + int rc = check_env(env, false); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); -#if defined(F_RDAHEAD) - if (toggle && unlikely(fcntl(env->me_lazy_fd, F_RDAHEAD, enable) == -1)) - return errno; -#endif /* F_RDAHEAD */ + env->hsr_callback = hsr; + return MDBX_SUCCESS; +} - int err; - void *const ptr = ptr_disp(env->me_map, offset); - if (enable) { -#if defined(MADV_NORMAL) - err = - madvise(ptr, length, MADV_NORMAL) ? ignore_enosys(errno) : MDBX_SUCCESS; - if (unlikely(MDBX_IS_ERROR(err))) - return err; -#elif defined(POSIX_MADV_NORMAL) - err = ignore_enosys(posix_madvise(ptr, length, POSIX_MADV_NORMAL)); - if (unlikely(MDBX_IS_ERROR(err))) - return err; -#elif defined(POSIX_FADV_NORMAL) && defined(POSIX_FADV_WILLNEED) - err = ignore_enosys( - posix_fadvise(env->me_lazy_fd, offset, length, POSIX_FADV_NORMAL)); - if (unlikely(MDBX_IS_ERROR(err))) - return err; -#elif defined(_WIN32) || defined(_WIN64) - /* no madvise on Windows */ -#else -#warning "FIXME" -#endif - if (toggle) { - /* NOTE: Seems there is a bug in the Mach/Darwin/OSX kernel, - * because MADV_WILLNEED with offset != 0 may cause SIGBUS - * on following access to the hinted region. - * 19.6.0 Darwin Kernel Version 19.6.0: Tue Jan 12 22:13:05 PST 2021; - * root:xnu-6153.141.16~1/RELEASE_X86_64 x86_64 */ -#if defined(F_RDADVISE) - struct radvisory hint; - hint.ra_offset = offset; - hint.ra_count = - unlikely(length > INT_MAX && sizeof(length) > sizeof(hint.ra_count)) - ? INT_MAX - : (int)length; - (void)/* Ignore ENOTTY for DB on the ram-disk and so on */ fcntl( - env->me_lazy_fd, F_RDADVISE, &hint); -#elif defined(MADV_WILLNEED) - err = madvise(ptr, length, MADV_WILLNEED) ? ignore_enosys(errno) - : MDBX_SUCCESS; - if (unlikely(MDBX_IS_ERROR(err))) - return err; -#elif defined(POSIX_MADV_WILLNEED) - err = ignore_enosys(posix_madvise(ptr, length, POSIX_MADV_WILLNEED)); - if (unlikely(MDBX_IS_ERROR(err))) - return err; -#elif defined(_WIN32) || defined(_WIN64) - if (mdbx_PrefetchVirtualMemory) { - WIN32_MEMORY_RANGE_ENTRY hint; - hint.VirtualAddress = ptr; - hint.NumberOfBytes = length; - (void)mdbx_PrefetchVirtualMemory(GetCurrentProcess(), 1, &hint, 0); - } -#elif defined(POSIX_FADV_WILLNEED) - err = ignore_enosys( - posix_fadvise(env->me_lazy_fd, offset, length, POSIX_FADV_WILLNEED)); - if (unlikely(MDBX_IS_ERROR(err))) - return err; -#else -#warning "FIXME" -#endif - } - } else { - mincore_clean_cache(env); -#if defined(MADV_RANDOM) - err = - madvise(ptr, length, MADV_RANDOM) ? ignore_enosys(errno) : MDBX_SUCCESS; - if (unlikely(MDBX_IS_ERROR(err))) - return err; -#elif defined(POSIX_MADV_RANDOM) - err = ignore_enosys(posix_madvise(ptr, length, POSIX_MADV_RANDOM)); - if (unlikely(MDBX_IS_ERROR(err))) - return err; -#elif defined(POSIX_FADV_RANDOM) - err = ignore_enosys( - posix_fadvise(env->me_lazy_fd, offset, length, POSIX_FADV_RANDOM)); - if (unlikely(MDBX_IS_ERROR(err))) - return err; -#elif defined(_WIN32) || defined(_WIN64) - /* no madvise on Windows */ -#else -#warning "FIXME" -#endif /* MADV_RANDOM */ - } - - env->me_lck->mti_readahead_anchor = (enable & 1) + (edge << 1); - err = MDBX_SUCCESS; - return err; +__cold MDBX_hsr_func *mdbx_env_get_hsr(const MDBX_env *env) { + return likely(env && env->signature.weak == env_signature) ? env->hsr_callback : nullptr; } -#endif /* MDBX_ENABLE_MADVISE */ -__cold static void update_mlcnt(const MDBX_env *env, - const pgno_t new_aligned_mlocked_pgno, - const bool lock_not_release) { - for (;;) { - const pgno_t mlock_pgno_before = - atomic_load32(&env->me_mlocked_pgno, mo_AcquireRelease); - eASSERT(env, - pgno_align2os_pgno(env, mlock_pgno_before) == mlock_pgno_before); - eASSERT(env, pgno_align2os_pgno(env, new_aligned_mlocked_pgno) == - new_aligned_mlocked_pgno); - if (lock_not_release ? (mlock_pgno_before >= new_aligned_mlocked_pgno) - : (mlock_pgno_before <= new_aligned_mlocked_pgno)) - break; - if (likely(atomic_cas32(&((MDBX_env *)env)->me_mlocked_pgno, - mlock_pgno_before, new_aligned_mlocked_pgno))) - for (;;) { - MDBX_atomic_uint32_t *const mlcnt = env->me_lck->mti_mlcnt; - const int32_t snap_locked = atomic_load32(mlcnt + 0, mo_Relaxed); - const int32_t snap_unlocked = atomic_load32(mlcnt + 1, mo_Relaxed); - if (mlock_pgno_before == 0 && (snap_locked - snap_unlocked) < INT_MAX) { - eASSERT(env, lock_not_release); - if (unlikely(!atomic_cas32(mlcnt + 0, snap_locked, snap_locked + 1))) - continue; - } - if (new_aligned_mlocked_pgno == 0 && - (snap_locked - snap_unlocked) > 0) { - eASSERT(env, !lock_not_release); - if (unlikely( - !atomic_cas32(mlcnt + 1, snap_unlocked, snap_unlocked + 1))) - continue; - } - NOTICE("%s-pages %u..%u, mlocked-process(es) %u -> %u", - lock_not_release ? "lock" : "unlock", - lock_not_release ? mlock_pgno_before : new_aligned_mlocked_pgno, - lock_not_release ? new_aligned_mlocked_pgno : mlock_pgno_before, - snap_locked - snap_unlocked, - atomic_load32(mlcnt + 0, mo_Relaxed) - - atomic_load32(mlcnt + 1, mo_Relaxed)); - return; - } - } +#if defined(_WIN32) || defined(_WIN64) +__cold int mdbx_env_get_pathW(const MDBX_env *env, const wchar_t **arg) { + int rc = check_env(env, true); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); + + if (unlikely(!arg)) + return LOG_IFERR(MDBX_EINVAL); + + *arg = env->pathname.specified; + return MDBX_SUCCESS; } +#endif /* Windows */ + +__cold int mdbx_env_get_path(const MDBX_env *env, const char **arg) { + int rc = check_env(env, true); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); + + if (unlikely(!arg)) + return LOG_IFERR(MDBX_EINVAL); -__cold static void munlock_after(const MDBX_env *env, const pgno_t aligned_pgno, - const size_t end_bytes) { - if (atomic_load32(&env->me_mlocked_pgno, mo_AcquireRelease) > aligned_pgno) { - int err = MDBX_ENOSYS; - const size_t munlock_begin = pgno2bytes(env, aligned_pgno); - const size_t munlock_size = end_bytes - munlock_begin; - eASSERT(env, end_bytes % env->me_os_psize == 0 && - munlock_begin % env->me_os_psize == 0 && - munlock_size % env->me_os_psize == 0); -#if defined(_WIN32) || defined(_WIN64) - err = VirtualUnlock(ptr_disp(env->me_map, munlock_begin), munlock_size) - ? MDBX_SUCCESS - : (int)GetLastError(); - if (err == ERROR_NOT_LOCKED) - err = MDBX_SUCCESS; -#elif defined(_POSIX_MEMLOCK_RANGE) - err = munlock(ptr_disp(env->me_map, munlock_begin), munlock_size) - ? errno - : MDBX_SUCCESS; -#endif - if (likely(err == MDBX_SUCCESS)) - update_mlcnt(env, aligned_pgno, false); - else { #if defined(_WIN32) || defined(_WIN64) - WARNING("VirtualUnlock(%zu, %zu) error %d", munlock_begin, munlock_size, - err); -#else - WARNING("munlock(%zu, %zu) error %d", munlock_begin, munlock_size, err); -#endif + if (!env->pathname_char) { + *arg = nullptr; + DWORD flags = /* WC_ERR_INVALID_CHARS */ 0x80; + size_t mb_len = + WideCharToMultiByte(CP_THREAD_ACP, flags, env->pathname.specified, -1, nullptr, 0, nullptr, nullptr); + rc = mb_len ? MDBX_SUCCESS : (int)GetLastError(); + if (rc == ERROR_INVALID_FLAGS) { + mb_len = WideCharToMultiByte(CP_THREAD_ACP, flags = 0, env->pathname.specified, -1, nullptr, 0, nullptr, nullptr); + rc = mb_len ? MDBX_SUCCESS : (int)GetLastError(); + } + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); + + char *const mb_pathname = osal_malloc(mb_len); + if (!mb_pathname) + return LOG_IFERR(MDBX_ENOMEM); + if (mb_len != (size_t)WideCharToMultiByte(CP_THREAD_ACP, flags, env->pathname.specified, -1, mb_pathname, + (int)mb_len, nullptr, nullptr)) { + rc = (int)GetLastError(); + osal_free(mb_pathname); + return LOG_IFERR(rc); } + if (env->pathname_char || + InterlockedCompareExchangePointer((PVOID volatile *)&env->pathname_char, mb_pathname, nullptr)) + osal_free(mb_pathname); } + *arg = env->pathname_char; +#else + *arg = env->pathname.specified; +#endif /* Windows */ + return MDBX_SUCCESS; +} + +/*------------------------------------------------------------------------------ + * Legacy API */ + +#ifndef LIBMDBX_NO_EXPORTS_LEGACY_API + +LIBMDBX_API int mdbx_txn_begin(MDBX_env *env, MDBX_txn *parent, MDBX_txn_flags_t flags, MDBX_txn **ret) { + return __inline_mdbx_txn_begin(env, parent, flags, ret); } -__cold static void munlock_all(const MDBX_env *env) { - munlock_after(env, 0, bytes_align2os_bytes(env, env->me_dxb_mmap.current)); +LIBMDBX_API int mdbx_txn_commit(MDBX_txn *txn) { return __inline_mdbx_txn_commit(txn); } + +LIBMDBX_API __cold int mdbx_env_stat(const MDBX_env *env, MDBX_stat *stat, size_t bytes) { + return __inline_mdbx_env_stat(env, stat, bytes); } -__cold static unsigned default_rp_augment_limit(const MDBX_env *env) { - /* default rp_augment_limit = npages / 3 */ - const size_t augment = env->me_dbgeo.now / 3 >> env->me_psize2log; - eASSERT(env, augment < MDBX_PGL_LIMIT); - return pnl_bytes2size(pnl_size2bytes( - (augment > MDBX_PNL_INITIAL) ? augment : MDBX_PNL_INITIAL)); +LIBMDBX_API __cold int mdbx_env_info(const MDBX_env *env, MDBX_envinfo *info, size_t bytes) { + return __inline_mdbx_env_info(env, info, bytes); } -static bool default_prefault_write(const MDBX_env *env) { - return !MDBX_MMAP_INCOHERENT_FILE_WRITE && !env->me_incore && - (env->me_flags & (MDBX_WRITEMAP | MDBX_RDONLY)) == MDBX_WRITEMAP; +LIBMDBX_API int mdbx_dbi_flags(const MDBX_txn *txn, MDBX_dbi dbi, unsigned *flags) { + return __inline_mdbx_dbi_flags(txn, dbi, flags); } -static void adjust_defaults(MDBX_env *env) { - if (!env->me_options.flags.non_auto.rp_augment_limit) - env->me_options.rp_augment_limit = default_rp_augment_limit(env); - if (!env->me_options.flags.non_auto.prefault_write) - env->me_options.prefault_write = default_prefault_write(env); +LIBMDBX_API __cold int mdbx_env_sync(MDBX_env *env) { return __inline_mdbx_env_sync(env); } - const size_t basis = env->me_dbgeo.now; - /* TODO: use options? */ - const unsigned factor = 9; - size_t threshold = (basis < ((size_t)65536 << factor)) - ? 65536 /* minimal threshold */ - : (basis > (MEGABYTE * 4 << factor)) - ? MEGABYTE * 4 /* maximal threshold */ - : basis >> factor; - threshold = (threshold < env->me_dbgeo.shrink || !env->me_dbgeo.shrink) - ? threshold - : env->me_dbgeo.shrink; +LIBMDBX_API __cold int mdbx_env_sync_poll(MDBX_env *env) { return __inline_mdbx_env_sync_poll(env); } - env->me_madv_threshold = - bytes2pgno(env, bytes_align2os_bytes(env, threshold)); +LIBMDBX_API __cold int mdbx_env_close(MDBX_env *env) { return __inline_mdbx_env_close(env); } + +LIBMDBX_API __cold int mdbx_env_set_mapsize(MDBX_env *env, size_t size) { + return __inline_mdbx_env_set_mapsize(env, size); } -enum resize_mode { implicit_grow, impilict_shrink, explicit_resize }; +LIBMDBX_API __cold int mdbx_env_set_maxdbs(MDBX_env *env, MDBX_dbi dbs) { + return __inline_mdbx_env_set_maxdbs(env, dbs); +} -__cold static int dxb_resize(MDBX_env *const env, const pgno_t used_pgno, - const pgno_t size_pgno, pgno_t limit_pgno, - const enum resize_mode mode) { - /* Acquire guard to avoid collision between read and write txns - * around me_dbgeo and me_dxb_mmap */ -#if defined(_WIN32) || defined(_WIN64) - osal_srwlock_AcquireExclusive(&env->me_remap_guard); - int rc = MDBX_SUCCESS; - mdbx_handle_array_t *suspended = NULL; - mdbx_handle_array_t array_onstack; -#else - int rc = osal_fastmutex_acquire(&env->me_remap_guard); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; -#endif +LIBMDBX_API __cold int mdbx_env_get_maxdbs(const MDBX_env *env, MDBX_dbi *dbs) { + return __inline_mdbx_env_get_maxdbs(env, dbs); +} - const size_t prev_size = env->me_dxb_mmap.current; - const size_t prev_limit = env->me_dxb_mmap.limit; - const pgno_t prev_limit_pgno = bytes2pgno(env, prev_limit); - eASSERT(env, limit_pgno >= size_pgno); - eASSERT(env, size_pgno >= used_pgno); - if (mode < explicit_resize && size_pgno <= prev_limit_pgno) { - /* The actual mapsize may be less since the geo.upper may be changed - * by other process. Avoids remapping until it necessary. */ - limit_pgno = prev_limit_pgno; - } - const size_t limit_bytes = pgno_align2os_bytes(env, limit_pgno); - const size_t size_bytes = pgno_align2os_bytes(env, size_pgno); -#if MDBX_ENABLE_MADVISE || defined(MDBX_USE_VALGRIND) - const void *const prev_map = env->me_dxb_mmap.base; -#endif /* MDBX_ENABLE_MADVISE || MDBX_USE_VALGRIND */ +LIBMDBX_API __cold int mdbx_env_set_maxreaders(MDBX_env *env, unsigned readers) { + return __inline_mdbx_env_set_maxreaders(env, readers); +} - VERBOSE("resize/%d datafile/mapping: " - "present %" PRIuPTR " -> %" PRIuPTR ", " - "limit %" PRIuPTR " -> %" PRIuPTR, - mode, prev_size, size_bytes, prev_limit, limit_bytes); +LIBMDBX_API __cold int mdbx_env_get_maxreaders(const MDBX_env *env, unsigned *readers) { + return __inline_mdbx_env_get_maxreaders(env, readers); +} - eASSERT(env, limit_bytes >= size_bytes); - eASSERT(env, bytes2pgno(env, size_bytes) >= size_pgno); - eASSERT(env, bytes2pgno(env, limit_bytes) >= limit_pgno); +LIBMDBX_API __cold int mdbx_env_set_syncbytes(MDBX_env *env, size_t threshold) { + return __inline_mdbx_env_set_syncbytes(env, threshold); +} - unsigned mresize_flags = - env->me_flags & (MDBX_RDONLY | MDBX_WRITEMAP | MDBX_UTTERLY_NOSYNC); - if (mode >= impilict_shrink) - mresize_flags |= MDBX_SHRINK_ALLOWED; +LIBMDBX_API __cold int mdbx_env_get_syncbytes(const MDBX_env *env, size_t *threshold) { + return __inline_mdbx_env_get_syncbytes(env, threshold); +} - if (limit_bytes == env->me_dxb_mmap.limit && - size_bytes == env->me_dxb_mmap.current && - size_bytes == env->me_dxb_mmap.filesize) - goto bailout; +LIBMDBX_API __cold int mdbx_env_set_syncperiod(MDBX_env *env, unsigned seconds_16dot16) { + return __inline_mdbx_env_set_syncperiod(env, seconds_16dot16); +} -#if defined(_WIN32) || defined(_WIN64) - if ((env->me_flags & MDBX_NOTLS) == 0 && - ((size_bytes < env->me_dxb_mmap.current && mode > implicit_grow) || - limit_bytes != env->me_dxb_mmap.limit)) { - /* 1) Windows allows only extending a read-write section, but not a - * corresponding mapped view. Therefore in other cases we must suspend - * the local threads for safe remap. - * 2) At least on Windows 10 1803 the entire mapped section is unavailable - * for short time during NtExtendSection() or VirtualAlloc() execution. - * 3) Under Wine runtime environment on Linux a section extending is not - * supported. - * - * THEREFORE LOCAL THREADS SUSPENDING IS ALWAYS REQUIRED! */ - array_onstack.limit = ARRAY_LENGTH(array_onstack.handles); - array_onstack.count = 0; - suspended = &array_onstack; - rc = osal_suspend_threads_before_remap(env, &suspended); - if (rc != MDBX_SUCCESS) { - ERROR("failed suspend-for-remap: errcode %d", rc); - goto bailout; - } - mresize_flags |= (mode < explicit_resize) - ? MDBX_MRESIZE_MAY_UNMAP - : MDBX_MRESIZE_MAY_UNMAP | MDBX_MRESIZE_MAY_MOVE; - } -#else /* Windows */ - MDBX_lockinfo *const lck = env->me_lck_mmap.lck; - if (mode == explicit_resize && limit_bytes != env->me_dxb_mmap.limit && - !(env->me_flags & MDBX_NOTLS)) { - mresize_flags |= MDBX_MRESIZE_MAY_UNMAP | MDBX_MRESIZE_MAY_MOVE; - if (lck) { - int err = osal_rdt_lock(env) /* lock readers table until remap done */; - if (unlikely(MDBX_IS_ERROR(err))) { - rc = err; +LIBMDBX_API __cold int mdbx_env_get_syncperiod(const MDBX_env *env, unsigned *seconds_16dot16) { + return __inline_mdbx_env_get_syncperiod(env, seconds_16dot16); +} + +LIBMDBX_API __cold uint64_t mdbx_key_from_int64(const int64_t i64) { return __inline_mdbx_key_from_int64(i64); } + +LIBMDBX_API __cold uint32_t mdbx_key_from_int32(const int32_t i32) { return __inline_mdbx_key_from_int32(i32); } + +LIBMDBX_API __cold intptr_t mdbx_limits_pgsize_min(void) { return __inline_mdbx_limits_pgsize_min(); } + +LIBMDBX_API __cold intptr_t mdbx_limits_pgsize_max(void) { return __inline_mdbx_limits_pgsize_max(); } + +#endif /* LIBMDBX_NO_EXPORTS_LEGACY_API */ +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \note Please refer to the COPYRIGHT file for explanations license change, +/// credits and acknowledgments. +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 + +typedef struct compacting_context { + MDBX_env *env; + MDBX_txn *txn; + MDBX_copy_flags_t flags; + pgno_t first_unallocated; + osal_condpair_t condpair; + volatile unsigned head; + volatile unsigned tail; + uint8_t *write_buf[2]; + size_t write_len[2]; + /* Error code. Never cleared if set. Both threads can set nonzero + * to fail the copy. Not mutex-protected, expects atomic int. */ + volatile int error; + mdbx_filehandle_t fd; +} ctx_t; + +__cold static int compacting_walk_tree(ctx_t *ctx, tree_t *tree); + +/* Dedicated writer thread for compacting copy. */ +__cold static THREAD_RESULT THREAD_CALL compacting_write_thread(void *arg) { + ctx_t *const ctx = arg; + +#if defined(EPIPE) && !(defined(_WIN32) || defined(_WIN64)) + sigset_t sigset; + sigemptyset(&sigset); + sigaddset(&sigset, SIGPIPE); + ctx->error = pthread_sigmask(SIG_BLOCK, &sigset, nullptr); +#endif /* EPIPE */ + + osal_condpair_lock(&ctx->condpair); + while (!ctx->error) { + while (ctx->tail == ctx->head && !ctx->error) { + int err = osal_condpair_wait(&ctx->condpair, true); + if (err != MDBX_SUCCESS) { + ctx->error = err; goto bailout; } - - /* looking for readers from this process */ - const size_t snap_nreaders = - atomic_load32(&lck->mti_numreaders, mo_AcquireRelease); - eASSERT(env, mode == explicit_resize); - for (size_t i = 0; i < snap_nreaders; ++i) { - if (lck->mti_readers[i].mr_pid.weak == env->me_pid && - lck->mti_readers[i].mr_tid.weak != osal_thread_self()) { - /* the base address of the mapping can't be changed since - * the other reader thread from this process exists. */ - osal_rdt_unlock(env); - mresize_flags &= ~(MDBX_MRESIZE_MAY_UNMAP | MDBX_MRESIZE_MAY_MOVE); - break; + } + const unsigned toggle = ctx->tail & 1; + size_t wsize = ctx->write_len[toggle]; + if (wsize == 0) { + ctx->tail += 1; + break /* EOF */; + } + ctx->write_len[toggle] = 0; + uint8_t *ptr = ctx->write_buf[toggle]; + if (!ctx->error) { + int err = osal_write(ctx->fd, ptr, wsize); + if (err != MDBX_SUCCESS) { +#if defined(EPIPE) && !(defined(_WIN32) || defined(_WIN64)) + if (err == EPIPE) { + /* Collect the pending SIGPIPE, + * otherwise at least OS X gives it to the process on thread-exit. */ + int unused; + sigwait(&sigset, &unused); } +#endif /* EPIPE */ + ctx->error = err; + goto bailout; } } + ctx->tail += 1; + osal_condpair_signal(&ctx->condpair, false); } -#endif /* ! Windows */ +bailout: + osal_condpair_unlock(&ctx->condpair); + return (THREAD_RESULT)0; +} - const pgno_t aligned_munlock_pgno = - (mresize_flags & (MDBX_MRESIZE_MAY_UNMAP | MDBX_MRESIZE_MAY_MOVE)) - ? 0 - : bytes2pgno(env, size_bytes); - if (mresize_flags & (MDBX_MRESIZE_MAY_UNMAP | MDBX_MRESIZE_MAY_MOVE)) { - mincore_clean_cache(env); - if ((env->me_flags & MDBX_WRITEMAP) && - env->me_lck->mti_unsynced_pages.weak) { -#if MDBX_ENABLE_PGOP_STAT - env->me_lck->mti_pgop_stat.msync.weak += 1; -#endif /* MDBX_ENABLE_PGOP_STAT */ - rc = osal_msync(&env->me_dxb_mmap, 0, pgno_align2os_bytes(env, used_pgno), - MDBX_SYNC_NONE); - if (unlikely(rc != MDBX_SUCCESS)) - goto bailout; - } +/* Give buffer and/or MDBX_EOF to writer thread, await unused buffer. */ +__cold static int compacting_toggle_write_buffers(ctx_t *ctx) { + osal_condpair_lock(&ctx->condpair); + eASSERT(ctx->env, ctx->head - ctx->tail < 2 || ctx->error); + ctx->head += 1; + osal_condpair_signal(&ctx->condpair, true); + while (!ctx->error && ctx->head - ctx->tail == 2 /* both buffers in use */) { + if (ctx->flags & MDBX_CP_THROTTLE_MVCC) + mdbx_txn_park(ctx->txn, false); + int err = osal_condpair_wait(&ctx->condpair, false); + if (err == MDBX_SUCCESS && (ctx->flags & MDBX_CP_THROTTLE_MVCC) != 0) + err = mdbx_txn_unpark(ctx->txn, false); + if (err != MDBX_SUCCESS) + ctx->error = err; } - munlock_after(env, aligned_munlock_pgno, size_bytes); + osal_condpair_unlock(&ctx->condpair); + return ctx->error; +} -#if MDBX_ENABLE_MADVISE - if (size_bytes < prev_size && mode > implicit_grow) { - NOTICE("resize-MADV_%s %u..%u", - (env->me_flags & MDBX_WRITEMAP) ? "REMOVE" : "DONTNEED", size_pgno, - bytes2pgno(env, prev_size)); - const uint32_t munlocks_before = - atomic_load32(&env->me_lck->mti_mlcnt[1], mo_Relaxed); - rc = MDBX_RESULT_TRUE; -#if defined(MADV_REMOVE) - if (env->me_flags & MDBX_WRITEMAP) - rc = madvise(ptr_disp(env->me_map, size_bytes), prev_size - size_bytes, - MADV_REMOVE) - ? ignore_enosys(errno) - : MDBX_SUCCESS; -#endif /* MADV_REMOVE */ -#if defined(MADV_DONTNEED) - if (rc == MDBX_RESULT_TRUE) - rc = madvise(ptr_disp(env->me_map, size_bytes), prev_size - size_bytes, - MADV_DONTNEED) - ? ignore_enosys(errno) - : MDBX_SUCCESS; -#elif defined(POSIX_MADV_DONTNEED) - if (rc == MDBX_RESULT_TRUE) - rc = ignore_enosys(posix_madvise(ptr_disp(env->me_map, size_bytes), - prev_size - size_bytes, - POSIX_MADV_DONTNEED)); -#elif defined(POSIX_FADV_DONTNEED) - if (rc == MDBX_RESULT_TRUE) - rc = ignore_enosys(posix_fadvise(env->me_lazy_fd, size_bytes, - prev_size - size_bytes, - POSIX_FADV_DONTNEED)); -#endif /* MADV_DONTNEED */ - if (unlikely(MDBX_IS_ERROR(rc))) { - const uint32_t mlocks_after = - atomic_load32(&env->me_lck->mti_mlcnt[0], mo_Relaxed); - if (rc == MDBX_EINVAL) { - const int severity = - (mlocks_after - munlocks_before) ? MDBX_LOG_NOTICE : MDBX_LOG_WARN; - if (LOG_ENABLED(severity)) - debug_log(severity, __func__, __LINE__, - "%s-madvise: ignore EINVAL (%d) since some pages maybe " - "locked (%u/%u mlcnt-processes)", - "resize", rc, mlocks_after, munlocks_before); - } else { - ERROR("%s-madvise(%s, %zu, +%zu), %u/%u mlcnt-processes, err %d", - "mresize", "DONTNEED", size_bytes, prev_size - size_bytes, - mlocks_after, munlocks_before, rc); - goto bailout; +static int compacting_put_bytes(ctx_t *ctx, const void *src, size_t bytes, pgno_t pgno, pgno_t npages) { + assert(pgno == 0 || bytes > PAGEHDRSZ); + while (bytes > 0) { + const size_t side = ctx->head & 1; + const size_t left = MDBX_ENVCOPY_WRITEBUF - ctx->write_len[side]; + if (left < (pgno ? PAGEHDRSZ : 1)) { + int err = compacting_toggle_write_buffers(ctx); + if (unlikely(err != MDBX_SUCCESS)) + return err; + continue; + } + const size_t chunk = (bytes < left) ? bytes : left; + void *const dst = ctx->write_buf[side] + ctx->write_len[side]; + if (src) { + memcpy(dst, src, chunk); + if (pgno) { + assert(chunk > PAGEHDRSZ); + page_t *mp = dst; + mp->pgno = pgno; + if (mp->txnid == 0) + mp->txnid = ctx->txn->txnid; + if (mp->flags == P_LARGE) { + assert(bytes <= pgno2bytes(ctx->env, npages)); + mp->pages = npages; + } + pgno = 0; } + src = ptr_disp(src, chunk); } else - env->me_lck->mti_discarded_tail.weak = size_pgno; + memset(dst, 0, chunk); + bytes -= chunk; + ctx->write_len[side] += chunk; } -#endif /* MDBX_ENABLE_MADVISE */ + return MDBX_SUCCESS; +} - rc = osal_mresize(mresize_flags, &env->me_dxb_mmap, size_bytes, limit_bytes); - eASSERT(env, env->me_dxb_mmap.limit >= env->me_dxb_mmap.current); +static int compacting_put_page(ctx_t *ctx, const page_t *mp, const size_t head_bytes, const size_t tail_bytes, + const pgno_t npages) { + if (tail_bytes) { + assert(head_bytes + tail_bytes <= ctx->env->ps); + assert(npages == 1 && (page_type(mp) == P_BRANCH || page_type(mp) == P_LEAF)); + } else { + assert(head_bytes <= pgno2bytes(ctx->env, npages)); + assert((npages == 1 && page_type(mp) == (P_LEAF | P_DUPFIX)) || page_type(mp) == P_LARGE); + } -#if MDBX_ENABLE_MADVISE - if (rc == MDBX_SUCCESS) { - eASSERT(env, limit_bytes == env->me_dxb_mmap.limit); - eASSERT(env, size_bytes <= env->me_dxb_mmap.filesize); - if (mode == explicit_resize) - eASSERT(env, size_bytes == env->me_dxb_mmap.current); - else - eASSERT(env, size_bytes <= env->me_dxb_mmap.current); - env->me_lck->mti_discarded_tail.weak = size_pgno; - const bool readahead = - !(env->me_flags & MDBX_NORDAHEAD) && - mdbx_is_readahead_reasonable(size_bytes, -(intptr_t)prev_size); - const bool force = limit_bytes != prev_limit || - env->me_dxb_mmap.base != prev_map -#if defined(_WIN32) || defined(_WIN64) - || prev_size > size_bytes -#endif /* Windows */ - ; - rc = set_readahead(env, size_pgno, readahead, force); + const pgno_t pgno = ctx->first_unallocated; + ctx->first_unallocated += npages; + int err = compacting_put_bytes(ctx, mp, head_bytes, pgno, npages); + if (unlikely(err != MDBX_SUCCESS)) + return err; + err = compacting_put_bytes(ctx, nullptr, pgno2bytes(ctx->env, npages) - (head_bytes + tail_bytes), 0, 0); + if (unlikely(err != MDBX_SUCCESS)) + return err; + return compacting_put_bytes(ctx, ptr_disp(mp, ctx->env->ps - tail_bytes), tail_bytes, 0, 0); +} + +__cold static int compacting_walk(ctx_t *ctx, MDBX_cursor *mc, pgno_t *const parent_pgno, txnid_t parent_txnid) { + mc->top = 0; + mc->ki[0] = 0; + int rc = page_get(mc, *parent_pgno, &mc->pg[0], parent_txnid); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + rc = tree_search_finalize(mc, nullptr, Z_FIRST); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + /* Make cursor pages writable */ + const intptr_t deep_limit = mc->top + 1; + void *const buf = osal_malloc(pgno2bytes(ctx->env, deep_limit + 1)); + if (buf == nullptr) + return MDBX_ENOMEM; + + void *ptr = buf; + for (intptr_t i = 0; i <= mc->top; i++) { + page_copy(ptr, mc->pg[i], ctx->env->ps); + mc->pg[i] = ptr; + ptr = ptr_disp(ptr, ctx->env->ps); } -#endif /* MDBX_ENABLE_MADVISE */ + /* This is writable space for a leaf page. Usually not needed. */ + page_t *const leaf = ptr; -bailout: - if (rc == MDBX_SUCCESS) { - eASSERT(env, env->me_dxb_mmap.limit >= env->me_dxb_mmap.current); - eASSERT(env, limit_bytes == env->me_dxb_mmap.limit); - eASSERT(env, size_bytes <= env->me_dxb_mmap.filesize); - if (mode == explicit_resize) - eASSERT(env, size_bytes == env->me_dxb_mmap.current); - else - eASSERT(env, size_bytes <= env->me_dxb_mmap.current); - /* update env-geo to avoid influences */ - env->me_dbgeo.now = env->me_dxb_mmap.current; - env->me_dbgeo.upper = env->me_dxb_mmap.limit; - adjust_defaults(env); -#ifdef MDBX_USE_VALGRIND - if (prev_limit != env->me_dxb_mmap.limit || prev_map != env->me_map) { - VALGRIND_DISCARD(env->me_valgrind_handle); - env->me_valgrind_handle = 0; - if (env->me_dxb_mmap.limit) - env->me_valgrind_handle = - VALGRIND_CREATE_BLOCK(env->me_map, env->me_dxb_mmap.limit, "mdbx"); - } -#endif /* MDBX_USE_VALGRIND */ - } else { - if (rc != MDBX_UNABLE_EXTEND_MAPSIZE && rc != MDBX_EPERM) { - ERROR("failed resize datafile/mapping: " - "present %" PRIuPTR " -> %" PRIuPTR ", " - "limit %" PRIuPTR " -> %" PRIuPTR ", errcode %d", - prev_size, size_bytes, prev_limit, limit_bytes, rc); + while (mc->top >= 0) { + page_t *mp = mc->pg[mc->top]; + const size_t nkeys = page_numkeys(mp); + if (is_leaf(mp)) { + if (!(mc->flags & z_inner) /* may have nested N_TREE or N_BIG nodes */) { + for (size_t i = 0; i < nkeys; i++) { + node_t *node = page_node(mp, i); + if (node_flags(node) == N_BIG) { + /* Need writable leaf */ + if (mp != leaf) { + mc->pg[mc->top] = leaf; + page_copy(leaf, mp, ctx->env->ps); + mp = leaf; + node = page_node(mp, i); + } + + const pgr_t lp = page_get_large(mc, node_largedata_pgno(node), mp->txnid); + if (unlikely((rc = lp.err) != MDBX_SUCCESS)) + goto bailout; + const size_t datasize = node_ds(node); + const pgno_t npages = largechunk_npages(ctx->env, datasize); + poke_pgno(node_data(node), ctx->first_unallocated); + rc = compacting_put_page(ctx, lp.page, PAGEHDRSZ + datasize, 0, npages); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + } else if (node_flags(node) & N_TREE) { + if (!MDBX_DISABLE_VALIDATION && unlikely(node_ds(node) != sizeof(tree_t))) { + ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid dupsort sub-tree node size", + (unsigned)node_ds(node)); + rc = MDBX_CORRUPTED; + goto bailout; + } + + /* Need writable leaf */ + if (mp != leaf) { + mc->pg[mc->top] = leaf; + page_copy(leaf, mp, ctx->env->ps); + mp = leaf; + node = page_node(mp, i); + } + + tree_t *nested = nullptr; + if (node_flags(node) & N_DUP) { + rc = cursor_dupsort_setup(mc, node, mp); + if (likely(rc == MDBX_SUCCESS)) { + nested = &mc->subcur->nested_tree; + rc = compacting_walk(ctx, &mc->subcur->cursor, &nested->root, mp->txnid); + } + } else { + cASSERT(mc, (mc->flags & z_inner) == 0 && mc->subcur == 0); + cursor_couple_t *couple = container_of(mc, cursor_couple_t, outer); + nested = &couple->inner.nested_tree; + memcpy(nested, node_data(node), sizeof(tree_t)); + rc = compacting_walk_tree(ctx, nested); + } + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + memcpy(node_data(node), nested, sizeof(tree_t)); + } + } + } } else { - WARNING("unable resize datafile/mapping: " - "present %" PRIuPTR " -> %" PRIuPTR ", " - "limit %" PRIuPTR " -> %" PRIuPTR ", errcode %d", - prev_size, size_bytes, prev_limit, limit_bytes, rc); - eASSERT(env, env->me_dxb_mmap.limit >= env->me_dxb_mmap.current); + mc->ki[mc->top]++; + if (mc->ki[mc->top] < nkeys) { + for (;;) { + const node_t *node = page_node(mp, mc->ki[mc->top]); + rc = page_get(mc, node_pgno(node), &mp, mp->txnid); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + mc->top += 1; + if (unlikely(mc->top >= deep_limit)) { + rc = MDBX_CURSOR_FULL; + goto bailout; + } + mc->ki[mc->top] = 0; + if (!is_branch(mp)) { + mc->pg[mc->top] = mp; + break; + } + /* Whenever we advance to a sibling branch page, + * we must proceed all the way down to its first leaf. */ + page_copy(mc->pg[mc->top], mp, ctx->env->ps); + } + continue; + } } - if (!env->me_dxb_mmap.base) { - env->me_flags |= MDBX_FATAL_ERROR; - if (env->me_txn) - env->me_txn->mt_flags |= MDBX_TXN_ERROR; - rc = MDBX_PANIC; + + const pgno_t pgno = ctx->first_unallocated; + if (likely(!is_dupfix_leaf(mp))) { + rc = compacting_put_page(ctx, mp, PAGEHDRSZ + mp->lower, ctx->env->ps - (PAGEHDRSZ + mp->upper), 1); + } else { + rc = compacting_put_page(ctx, mp, PAGEHDRSZ + page_numkeys(mp) * mp->dupfix_ksize, 0, 1); } - } + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; -#if defined(_WIN32) || defined(_WIN64) - int err = MDBX_SUCCESS; - osal_srwlock_ReleaseExclusive(&env->me_remap_guard); - if (suspended) { - err = osal_resume_threads_after_remap(suspended); - if (suspended != &array_onstack) - osal_free(suspended); - } -#else - if (env->me_lck_mmap.lck && - (mresize_flags & (MDBX_MRESIZE_MAY_UNMAP | MDBX_MRESIZE_MAY_MOVE)) != 0) - osal_rdt_unlock(env); - int err = osal_fastmutex_release(&env->me_remap_guard); -#endif /* Windows */ - if (err != MDBX_SUCCESS) { - FATAL("failed resume-after-remap: errcode %d", err); - return MDBX_PANIC; + if (mc->top) { + /* Update parent if there is one */ + node_set_pgno(page_node(mc->pg[mc->top - 1], mc->ki[mc->top - 1]), pgno); + cursor_pop(mc); + } else { + /* Otherwise we're done */ + *parent_pgno = pgno; + break; + } } + +bailout: + osal_free(buf); return rc; } -static int meta_unsteady(int err, MDBX_env *env, const txnid_t early_than, - const pgno_t pgno) { - MDBX_meta *const meta = METAPAGE(env, pgno); - const txnid_t txnid = constmeta_txnid(meta); - if (unlikely(err != MDBX_SUCCESS) || !META_IS_STEADY(meta) || - !(txnid < early_than)) - return err; +__cold static int compacting_walk_tree(ctx_t *ctx, tree_t *tree) { + if (unlikely(tree->root == P_INVALID)) + return MDBX_SUCCESS; /* empty db */ - WARNING("wipe txn #%" PRIaTXN ", meta %" PRIaPGNO, txnid, pgno); - const uint64_t wipe = MDBX_DATASIGN_NONE; - const void *ptr = &wipe; - size_t bytes = sizeof(meta->mm_sign), - offset = ptr_dist(&meta->mm_sign, env->me_map); - if (env->me_flags & MDBX_WRITEMAP) { - unaligned_poke_u64(4, meta->mm_sign, wipe); - osal_flush_incoherent_cpu_writeback(); - if (!MDBX_AVOID_MSYNC) { - err = - osal_msync(&env->me_dxb_mmap, 0, pgno_align2os_bytes(env, NUM_METAS), - MDBX_SYNC_DATA | MDBX_SYNC_IODQ); -#if MDBX_ENABLE_PGOP_STAT - env->me_lck->mti_pgop_stat.msync.weak += 1; -#endif /* MDBX_ENABLE_PGOP_STAT */ - return err; + cursor_couple_t couple; + memset(&couple, 0, sizeof(couple)); + couple.inner.cursor.signature = ~cur_signature_live; + kvx_t kvx = {.clc = {.k = {.lmin = INT_MAX}, .v = {.lmin = INT_MAX}}}; + int rc = cursor_init4walk(&couple, ctx->txn, tree, &kvx); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + couple.outer.checking |= z_ignord | z_pagecheck; + couple.inner.cursor.checking |= z_ignord | z_pagecheck; + if (!tree->mod_txnid) + tree->mod_txnid = ctx->txn->txnid; + return compacting_walk(ctx, &couple.outer, &tree->root, tree->mod_txnid); +} + +__cold static void compacting_fixup_meta(MDBX_env *env, meta_t *meta) { + eASSERT(env, meta->trees.gc.mod_txnid || meta->trees.gc.root == P_INVALID); + eASSERT(env, meta->trees.main.mod_txnid || meta->trees.main.root == P_INVALID); + + /* Calculate filesize taking in account shrink/growing thresholds */ + if (meta->geometry.first_unallocated != meta->geometry.now) { + meta->geometry.now = meta->geometry.first_unallocated; + const size_t aligner = pv2pages(meta->geometry.grow_pv ? meta->geometry.grow_pv : meta->geometry.shrink_pv); + if (aligner) { + const pgno_t aligned = pgno_align2os_pgno(env, meta->geometry.first_unallocated + aligner - + meta->geometry.first_unallocated % aligner); + meta->geometry.now = aligned; } - ptr = data_page(meta); - offset = ptr_dist(ptr, env->me_map); - bytes = env->me_psize; } -#if MDBX_ENABLE_PGOP_STAT - env->me_lck->mti_pgop_stat.wops.weak += 1; -#endif /* MDBX_ENABLE_PGOP_STAT */ - err = osal_pwrite(env->me_fd4meta, ptr, bytes, offset); - if (likely(err == MDBX_SUCCESS) && env->me_fd4meta == env->me_lazy_fd) { - err = osal_fsync(env->me_lazy_fd, MDBX_SYNC_DATA | MDBX_SYNC_IODQ); -#if MDBX_ENABLE_PGOP_STAT - env->me_lck->mti_pgop_stat.fsync.weak += 1; -#endif /* MDBX_ENABLE_PGOP_STAT */ + if (meta->geometry.now < meta->geometry.lower) + meta->geometry.now = meta->geometry.lower; + if (meta->geometry.now > meta->geometry.upper) + meta->geometry.now = meta->geometry.upper; + + /* Update signature */ + assert(meta->geometry.now >= meta->geometry.first_unallocated); + meta_sign_as_steady(meta); +} + +/* Make resizable */ +__cold static void meta_make_sizeable(meta_t *meta) { + meta->geometry.lower = MIN_PAGENO; + if (meta->geometry.grow_pv == 0) { + const pgno_t step = 1 + (meta->geometry.upper - meta->geometry.lower) / 42; + meta->geometry.grow_pv = pages2pv(step); + } + if (meta->geometry.shrink_pv == 0) { + const pgno_t step = pv2pages(meta->geometry.grow_pv) << 1; + meta->geometry.shrink_pv = pages2pv(step); } - return err; } -__cold static int wipe_steady(MDBX_txn *txn, txnid_t last_steady) { - MDBX_env *const env = txn->mt_env; - int err = MDBX_SUCCESS; +__cold static int copy_with_compacting(MDBX_env *env, MDBX_txn *txn, mdbx_filehandle_t fd, uint8_t *buffer, + const bool dest_is_pipe, const MDBX_copy_flags_t flags) { + const size_t meta_bytes = pgno2bytes(env, NUM_METAS); + uint8_t *const data_buffer = buffer + ceil_powerof2(meta_bytes, globals.sys_pagesize); + meta_t *const meta = meta_init_triplet(env, buffer); + meta_set_txnid(env, meta, txn->txnid); - /* early than last_steady */ - err = meta_unsteady(err, env, last_steady, 0); - err = meta_unsteady(err, env, last_steady, 1); - err = meta_unsteady(err, env, last_steady, 2); + if (flags & MDBX_CP_FORCE_DYNAMIC_SIZE) + meta_make_sizeable(meta); - /* the last_steady */ - err = meta_unsteady(err, env, last_steady + 1, 0); - err = meta_unsteady(err, env, last_steady + 1, 1); - err = meta_unsteady(err, env, last_steady + 1, 2); + /* copy canary sequences if present */ + if (txn->canary.v) { + meta->canary = txn->canary; + meta->canary.v = constmeta_txnid(meta); + } - osal_flush_incoherent_mmap(env->me_map, pgno2bytes(env, NUM_METAS), - env->me_os_psize); + if (txn->dbs[MAIN_DBI].root == P_INVALID) { + /* When the DB is empty, handle it specially to + * fix any breakage like page leaks from ITS#8174. */ + meta->trees.main.flags = txn->dbs[MAIN_DBI].flags; + compacting_fixup_meta(env, meta); + if (dest_is_pipe) { + if (flags & MDBX_CP_THROTTLE_MVCC) + mdbx_txn_park(txn, false); + int rc = osal_write(fd, buffer, meta_bytes); + if (likely(rc == MDBX_SUCCESS) && (flags & MDBX_CP_THROTTLE_MVCC) != 0) + rc = mdbx_txn_unpark(txn, false); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + } + } else { + /* Count free pages + GC pages. */ + cursor_couple_t couple; + int rc = cursor_init(&couple.outer, txn, FREE_DBI); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + pgno_t gc_npages = txn->dbs[FREE_DBI].branch_pages + txn->dbs[FREE_DBI].leaf_pages + txn->dbs[FREE_DBI].large_pages; + MDBX_val key, data; + rc = outer_first(&couple.outer, &key, &data); + while (rc == MDBX_SUCCESS) { + const pnl_t pnl = data.iov_base; + if (unlikely(data.iov_len % sizeof(pgno_t) || data.iov_len < MDBX_PNL_SIZEOF(pnl))) { + ERROR("%s/%d: %s %zu", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid GC-record length", data.iov_len); + return MDBX_CORRUPTED; + } + if (unlikely(!pnl_check(pnl, txn->geo.first_unallocated))) { + ERROR("%s/%d: %s", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid GC-record content"); + return MDBX_CORRUPTED; + } + gc_npages += MDBX_PNL_GETSIZE(pnl); + rc = outer_next(&couple.outer, &key, &data, MDBX_NEXT); + } + if (unlikely(rc != MDBX_NOTFOUND)) + return rc; - /* force oldest refresh */ - atomic_store32(&env->me_lck->mti_readers_refresh_flag, true, mo_Relaxed); + meta->geometry.first_unallocated = txn->geo.first_unallocated - gc_npages; + meta->trees.main = txn->dbs[MAIN_DBI]; - tASSERT(txn, (txn->mt_flags & MDBX_TXN_RDONLY) == 0); - txn->tw.troika = meta_tap(env); - for (MDBX_txn *scan = txn->mt_env->me_txn0; scan; scan = scan->mt_child) - if (scan != txn) - scan->tw.troika = txn->tw.troika; - return err; -} + ctx_t ctx; + memset(&ctx, 0, sizeof(ctx)); + rc = osal_condpair_init(&ctx.condpair); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; -//------------------------------------------------------------------------------ + memset(data_buffer, 0, 2 * (size_t)MDBX_ENVCOPY_WRITEBUF); + ctx.write_buf[0] = data_buffer; + ctx.write_buf[1] = data_buffer + (size_t)MDBX_ENVCOPY_WRITEBUF; + ctx.first_unallocated = NUM_METAS; + ctx.env = env; + ctx.fd = fd; + ctx.txn = txn; + ctx.flags = flags; -MDBX_MAYBE_UNUSED __hot static pgno_t * -scan4seq_fallback(pgno_t *range, const size_t len, const size_t seq) { - assert(seq > 0 && len > seq); -#if MDBX_PNL_ASCENDING - assert(range[-1] == len); - const pgno_t *const detent = range + len - seq; - const ptrdiff_t offset = (ptrdiff_t)seq; - const pgno_t target = (pgno_t)offset; - if (likely(len > seq + 3)) { - do { - const pgno_t diff0 = range[offset + 0] - range[0]; - const pgno_t diff1 = range[offset + 1] - range[1]; - const pgno_t diff2 = range[offset + 2] - range[2]; - const pgno_t diff3 = range[offset + 3] - range[3]; - if (diff0 == target) - return range + 0; - if (diff1 == target) - return range + 1; - if (diff2 == target) - return range + 2; - if (diff3 == target) - return range + 3; - range += 4; - } while (range + 3 < detent); - if (range == detent) - return nullptr; - } - do - if (range[offset] - *range == target) - return range; - while (++range < detent); -#else - assert(range[-(ptrdiff_t)len] == len); - const pgno_t *const detent = range - len + seq; - const ptrdiff_t offset = -(ptrdiff_t)seq; - const pgno_t target = (pgno_t)offset; - if (likely(len > seq + 3)) { - do { - const pgno_t diff0 = range[-0] - range[offset - 0]; - const pgno_t diff1 = range[-1] - range[offset - 1]; - const pgno_t diff2 = range[-2] - range[offset - 2]; - const pgno_t diff3 = range[-3] - range[offset - 3]; - /* Смысл вычислений до ветвлений в том, чтобы позволить компилятору - * загружать и вычислять все значения параллельно. */ - if (diff0 == target) - return range - 0; - if (diff1 == target) - return range - 1; - if (diff2 == target) - return range - 2; - if (diff3 == target) - return range - 3; - range -= 4; - } while (range > detent + 3); - if (range == detent) - return nullptr; - } - do - if (*range - range[offset] == target) - return range; - while (--range > detent); -#endif /* MDBX_PNL sort-order */ - return nullptr; -} + osal_thread_t thread; + int thread_err = osal_thread_create(&thread, compacting_write_thread, &ctx); + if (likely(thread_err == MDBX_SUCCESS)) { + if (dest_is_pipe) { + if (!meta->trees.main.mod_txnid) + meta->trees.main.mod_txnid = txn->txnid; + compacting_fixup_meta(env, meta); + if (flags & MDBX_CP_THROTTLE_MVCC) + mdbx_txn_park(txn, false); + rc = osal_write(fd, buffer, meta_bytes); + if (likely(rc == MDBX_SUCCESS) && (flags & MDBX_CP_THROTTLE_MVCC) != 0) + rc = mdbx_txn_unpark(txn, false); + } + if (likely(rc == MDBX_SUCCESS)) + rc = compacting_walk_tree(&ctx, &meta->trees.main); + if (ctx.write_len[ctx.head & 1]) + /* toggle to flush non-empty buffers */ + compacting_toggle_write_buffers(&ctx); -MDBX_MAYBE_UNUSED static const pgno_t *scan4range_checker(const MDBX_PNL pnl, - const size_t seq) { - size_t begin = MDBX_PNL_ASCENDING ? 1 : MDBX_PNL_GETSIZE(pnl); -#if MDBX_PNL_ASCENDING - while (seq <= MDBX_PNL_GETSIZE(pnl) - begin) { - if (pnl[begin + seq] - pnl[begin] == seq) - return pnl + begin; - ++begin; + if (likely(rc == MDBX_SUCCESS) && unlikely(meta->geometry.first_unallocated != ctx.first_unallocated)) { + if (ctx.first_unallocated > meta->geometry.first_unallocated) { + ERROR("the source DB %s: post-compactification used pages %" PRIaPGNO " %c expected %" PRIaPGNO, + "has double-used pages or other corruption", ctx.first_unallocated, '>', + meta->geometry.first_unallocated); + rc = MDBX_CORRUPTED; /* corrupted DB */ + } + if (ctx.first_unallocated < meta->geometry.first_unallocated) { + WARNING("the source DB %s: post-compactification used pages %" PRIaPGNO " %c expected %" PRIaPGNO, + "has page leak(s)", ctx.first_unallocated, '<', meta->geometry.first_unallocated); + if (dest_is_pipe) + /* the root within already written meta-pages is wrong */ + rc = MDBX_CORRUPTED; + } + /* fixup meta */ + meta->geometry.first_unallocated = ctx.first_unallocated; + } + + /* toggle with empty buffers to exit thread's loop */ + eASSERT(env, (ctx.write_len[ctx.head & 1]) == 0); + compacting_toggle_write_buffers(&ctx); + thread_err = osal_thread_join(thread); + eASSERT(env, (ctx.tail == ctx.head && ctx.write_len[ctx.head & 1] == 0) || ctx.error); + osal_condpair_destroy(&ctx.condpair); + } + if (unlikely(thread_err != MDBX_SUCCESS)) + return thread_err; + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + if (unlikely(ctx.error != MDBX_SUCCESS)) + return ctx.error; + if (!dest_is_pipe) + compacting_fixup_meta(env, meta); } -#else - while (begin > seq) { - if (pnl[begin - seq] - pnl[begin] == seq) - return pnl + begin; - --begin; + + if (flags & MDBX_CP_THROTTLE_MVCC) + mdbx_txn_park(txn, false); + + /* Extend file if required */ + if (meta->geometry.now != meta->geometry.first_unallocated) { + const size_t whole_size = pgno2bytes(env, meta->geometry.now); + if (!dest_is_pipe) + return osal_ftruncate(fd, whole_size); + + const size_t used_size = pgno2bytes(env, meta->geometry.first_unallocated); + memset(data_buffer, 0, (size_t)MDBX_ENVCOPY_WRITEBUF); + for (size_t offset = used_size; offset < whole_size;) { + const size_t chunk = + ((size_t)MDBX_ENVCOPY_WRITEBUF < whole_size - offset) ? (size_t)MDBX_ENVCOPY_WRITEBUF : whole_size - offset; + int rc = osal_write(fd, data_buffer, chunk); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + offset += chunk; + } } -#endif /* MDBX_PNL sort-order */ - return nullptr; + return MDBX_SUCCESS; } -#if defined(_MSC_VER) && !defined(__builtin_clz) && \ - !__has_builtin(__builtin_clz) -MDBX_MAYBE_UNUSED static __always_inline size_t __builtin_clz(uint32_t value) { - unsigned long index; - _BitScanReverse(&index, value); - return 31 - index; -} -#endif /* _MSC_VER */ +//---------------------------------------------------------------------------- -#if defined(_MSC_VER) && !defined(__builtin_clzl) && \ - !__has_builtin(__builtin_clzl) -MDBX_MAYBE_UNUSED static __always_inline size_t __builtin_clzl(size_t value) { - unsigned long index; -#ifdef _WIN64 - assert(sizeof(value) == 8); - _BitScanReverse64(&index, value); - return 63 - index; -#else - assert(sizeof(value) == 4); - _BitScanReverse(&index, value); - return 31 - index; -#endif -} -#endif /* _MSC_VER */ +__cold static int copy_asis(MDBX_env *env, MDBX_txn *txn, mdbx_filehandle_t fd, uint8_t *buffer, + const bool dest_is_pipe, const MDBX_copy_flags_t flags) { + bool should_unlock = false; + if ((txn->flags & MDBX_TXN_RDONLY) != 0 && (flags & MDBX_CP_RENEW_TXN) != 0) { + /* Try temporarily block writers until we snapshot the meta pages */ + int err = lck_txn_lock(env, true); + if (likely(err == MDBX_SUCCESS)) + should_unlock = true; + else if (unlikely(err != MDBX_BUSY)) + return err; + } -#if !MDBX_PNL_ASCENDING + jitter4testing(false); + int rc = MDBX_SUCCESS; + const size_t meta_bytes = pgno2bytes(env, NUM_METAS); + troika_t troika = meta_tap(env); + /* Make a snapshot of meta-pages, + * but writing ones after the data was flushed */ +retry_snap_meta: + memcpy(buffer, env->dxb_mmap.base, meta_bytes); + const meta_ptr_t recent = meta_recent(env, &troika); + meta_t *headcopy = /* LY: get pointer to the snapshot copy */ + ptr_disp(buffer, ptr_dist(recent.ptr_c, env->dxb_mmap.base)); + jitter4testing(false); + if (txn->flags & MDBX_TXN_RDONLY) { + if (recent.txnid != txn->txnid) { + if (flags & MDBX_CP_RENEW_TXN) + rc = mdbx_txn_renew(txn); + else { + rc = MDBX_MVCC_RETARDED; + for (size_t n = 0; n < NUM_METAS; ++n) { + meta_t *const meta = page_meta(ptr_disp(buffer, pgno2bytes(env, n))); + if (troika.txnid[n] == txn->txnid && ((/* is_steady */ (troika.fsm >> n) & 1) || rc != MDBX_SUCCESS)) { + rc = MDBX_SUCCESS; + headcopy = meta; + } else if (troika.txnid[n] > txn->txnid) + meta_set_txnid(env, meta, 0); + } + } + } + if (should_unlock) + lck_txn_unlock(env); + else { + troika_t snap = meta_tap(env); + if (memcmp(&troika, &snap, sizeof(troika_t)) && rc == MDBX_SUCCESS) { + troika = snap; + goto retry_snap_meta; + } + } + } + if (unlikely(rc != MDBX_SUCCESS)) + return rc; -#if !defined(MDBX_ATTRIBUTE_TARGET) && \ - (__has_attribute(__target__) || __GNUC_PREREQ(5, 0)) -#define MDBX_ATTRIBUTE_TARGET(target) __attribute__((__target__(target))) -#endif /* MDBX_ATTRIBUTE_TARGET */ + if (txn->flags & MDBX_TXN_RDONLY) + eASSERT(env, meta_txnid(headcopy) == txn->txnid); + if (flags & MDBX_CP_FORCE_DYNAMIC_SIZE) + meta_make_sizeable(headcopy); + /* Update signature to steady */ + meta_sign_as_steady(headcopy); -#ifndef MDBX_GCC_FASTMATH_i686_SIMD_WORKAROUND -/* Workaround for GCC's bug with `-m32 -march=i686 -Ofast` - * gcc/i686-buildroot-linux-gnu/12.2.0/include/xmmintrin.h:814:1: - * error: inlining failed in call to 'always_inline' '_mm_movemask_ps': - * target specific option mismatch */ -#if !defined(__FAST_MATH__) || !__FAST_MATH__ || !defined(__GNUC__) || \ - defined(__e2k__) || defined(__clang__) || defined(__amd64__) || \ - defined(__SSE2__) -#define MDBX_GCC_FASTMATH_i686_SIMD_WORKAROUND 0 -#else -#define MDBX_GCC_FASTMATH_i686_SIMD_WORKAROUND 1 -#endif -#endif /* MDBX_GCC_FASTMATH_i686_SIMD_WORKAROUND */ + /* Copy the data */ + const size_t whole_size = pgno_align2os_bytes(env, txn->geo.end_pgno); + const size_t used_size = pgno2bytes(env, txn->geo.first_unallocated); + jitter4testing(false); -#if defined(__SSE2__) && defined(__SSE__) -#define MDBX_ATTRIBUTE_TARGET_SSE2 /* nope */ -#elif (defined(_M_IX86_FP) && _M_IX86_FP >= 2) || defined(__amd64__) -#define __SSE2__ -#define MDBX_ATTRIBUTE_TARGET_SSE2 /* nope */ -#elif defined(MDBX_ATTRIBUTE_TARGET) && defined(__ia32__) && \ - !MDBX_GCC_FASTMATH_i686_SIMD_WORKAROUND -#define MDBX_ATTRIBUTE_TARGET_SSE2 MDBX_ATTRIBUTE_TARGET("sse,sse2") -#endif /* __SSE2__ */ + if (flags & MDBX_CP_THROTTLE_MVCC) + mdbx_txn_park(txn, false); -#if defined(__AVX2__) -#define MDBX_ATTRIBUTE_TARGET_AVX2 /* nope */ -#elif defined(MDBX_ATTRIBUTE_TARGET) && defined(__ia32__) && \ - !MDBX_GCC_FASTMATH_i686_SIMD_WORKAROUND -#define MDBX_ATTRIBUTE_TARGET_AVX2 MDBX_ATTRIBUTE_TARGET("sse,sse2,avx,avx2") -#endif /* __AVX2__ */ + if (dest_is_pipe) + rc = osal_write(fd, buffer, meta_bytes); -#if defined(MDBX_ATTRIBUTE_TARGET_AVX2) -#if defined(__AVX512BW__) -#define MDBX_ATTRIBUTE_TARGET_AVX512BW /* nope */ -#elif defined(MDBX_ATTRIBUTE_TARGET) && defined(__ia32__) && \ - !MDBX_GCC_FASTMATH_i686_SIMD_WORKAROUND && \ - (__GNUC_PREREQ(6, 0) || __CLANG_PREREQ(5, 0)) -#define MDBX_ATTRIBUTE_TARGET_AVX512BW \ - MDBX_ATTRIBUTE_TARGET("sse,sse2,avx,avx2,avx512bw") -#endif /* __AVX512BW__ */ -#endif /* MDBX_ATTRIBUTE_TARGET_AVX2 for MDBX_ATTRIBUTE_TARGET_AVX512BW */ + uint8_t *const data_buffer = buffer + ceil_powerof2(meta_bytes, globals.sys_pagesize); +#if MDBX_USE_COPYFILERANGE + static bool copyfilerange_unavailable; +#if (defined(__linux__) || defined(__gnu_linux__)) + if (globals.linux_kernel_version >= 0x05030000 && globals.linux_kernel_version < 0x05130000) + copyfilerange_unavailable = true; +#endif /* linux */ + bool not_the_same_filesystem = false; + if (!copyfilerange_unavailable) { + struct statfs statfs_info; + if (fstatfs(fd, &statfs_info) || statfs_info.f_type == /* ECRYPTFS_SUPER_MAGIC */ 0xf15f) + /* avoid use copyfilerange_unavailable() to ecryptfs due bugs */ + not_the_same_filesystem = true; + } +#endif /* MDBX_USE_COPYFILERANGE */ -#ifdef MDBX_ATTRIBUTE_TARGET_SSE2 -MDBX_ATTRIBUTE_TARGET_SSE2 static __always_inline unsigned -diffcmp2mask_sse2(const pgno_t *const ptr, const ptrdiff_t offset, - const __m128i pattern) { - const __m128i f = _mm_loadu_si128((const __m128i *)ptr); - const __m128i l = _mm_loadu_si128((const __m128i *)(ptr + offset)); - const __m128i cmp = _mm_cmpeq_epi32(_mm_sub_epi32(f, l), pattern); - return _mm_movemask_ps(*(const __m128 *)&cmp); -} + for (size_t offset = meta_bytes; rc == MDBX_SUCCESS && offset < used_size;) { + if (flags & MDBX_CP_THROTTLE_MVCC) { + rc = mdbx_txn_unpark(txn, false); + if (unlikely(rc != MDBX_SUCCESS)) + break; + } -MDBX_MAYBE_UNUSED __hot MDBX_ATTRIBUTE_TARGET_SSE2 static pgno_t * -scan4seq_sse2(pgno_t *range, const size_t len, const size_t seq) { - assert(seq > 0 && len > seq); -#if MDBX_PNL_ASCENDING -#error "FIXME: Not implemented" -#endif /* MDBX_PNL_ASCENDING */ - assert(range[-(ptrdiff_t)len] == len); - pgno_t *const detent = range - len + seq; - const ptrdiff_t offset = -(ptrdiff_t)seq; - const pgno_t target = (pgno_t)offset; - const __m128i pattern = _mm_set1_epi32(target); - uint8_t mask; - if (likely(len > seq + 3)) { - do { - mask = (uint8_t)diffcmp2mask_sse2(range - 3, offset, pattern); - if (mask) { -#ifndef __SANITIZE_ADDRESS__ - found: -#endif /* __SANITIZE_ADDRESS__ */ - return range + 28 - __builtin_clz(mask); +#if MDBX_USE_SENDFILE + static bool sendfile_unavailable; + if (dest_is_pipe && likely(!sendfile_unavailable)) { + off_t in_offset = offset; + const ssize_t written = sendfile(fd, env->lazy_fd, &in_offset, used_size - offset); + if (likely(written > 0)) { + offset = in_offset; + if (flags & MDBX_CP_THROTTLE_MVCC) + rc = mdbx_txn_park(txn, false); + continue; } - range -= 4; - } while (range > detent + 3); - if (range == detent) - return nullptr; - } + rc = MDBX_ENODATA; + if (written == 0 || ignore_enosys_and_eagain(rc = errno) != MDBX_RESULT_TRUE) + break; + sendfile_unavailable = true; + } +#endif /* MDBX_USE_SENDFILE */ - /* Далее происходит чтение от 4 до 12 лишних байт, которые могут быть не - * только за пределами региона выделенного под PNL, но и пересекать границу - * страницы памяти. Что может приводить как к ошибкам ASAN, так и к падению. - * Поэтому проверяем смещение на странице, а с ASAN всегда страхуемся. */ -#ifndef __SANITIZE_ADDRESS__ - const unsigned on_page_safe_mask = 0xff0 /* enough for '-15' bytes offset */; - if (likely(on_page_safe_mask & (uintptr_t)(range + offset)) && - !RUNNING_ON_VALGRIND) { - const unsigned extra = (unsigned)(detent + 4 - range); - assert(extra > 0 && extra < 4); - mask = 0xF << extra; - mask &= diffcmp2mask_sse2(range - 3, offset, pattern); - if (mask) - goto found; - return nullptr; +#if MDBX_USE_COPYFILERANGE + if (!dest_is_pipe && !not_the_same_filesystem && likely(!copyfilerange_unavailable)) { + off_t in_offset = offset, out_offset = offset; + ssize_t bytes_copied = copy_file_range(env->lazy_fd, &in_offset, fd, &out_offset, used_size - offset, 0); + if (likely(bytes_copied > 0)) { + offset = in_offset; + if (flags & MDBX_CP_THROTTLE_MVCC) + rc = mdbx_txn_park(txn, false); + continue; + } + rc = MDBX_ENODATA; + if (bytes_copied == 0) + break; + rc = errno; + if (rc == EXDEV || rc == /* workaround for ecryptfs bug(s), + maybe useful for others FS */ + EINVAL) + not_the_same_filesystem = true; + else if (ignore_enosys_and_eagain(rc) == MDBX_RESULT_TRUE) + copyfilerange_unavailable = true; + else + break; + } +#endif /* MDBX_USE_COPYFILERANGE */ + + /* fallback to portable */ + const size_t chunk = + ((size_t)MDBX_ENVCOPY_WRITEBUF < used_size - offset) ? (size_t)MDBX_ENVCOPY_WRITEBUF : used_size - offset; + /* copy to avoid EFAULT in case swapped-out */ + memcpy(data_buffer, ptr_disp(env->dxb_mmap.base, offset), chunk); + if (flags & MDBX_CP_THROTTLE_MVCC) + mdbx_txn_park(txn, false); + rc = osal_write(fd, data_buffer, chunk); + offset += chunk; } -#endif /* __SANITIZE_ADDRESS__ */ - do - if (*range - range[offset] == target) - return range; - while (--range != detent); - return nullptr; -} -#endif /* MDBX_ATTRIBUTE_TARGET_SSE2 */ -#ifdef MDBX_ATTRIBUTE_TARGET_AVX2 -MDBX_ATTRIBUTE_TARGET_AVX2 static __always_inline unsigned -diffcmp2mask_avx2(const pgno_t *const ptr, const ptrdiff_t offset, - const __m256i pattern) { - const __m256i f = _mm256_loadu_si256((const __m256i *)ptr); - const __m256i l = _mm256_loadu_si256((const __m256i *)(ptr + offset)); - const __m256i cmp = _mm256_cmpeq_epi32(_mm256_sub_epi32(f, l), pattern); - return _mm256_movemask_ps(*(const __m256 *)&cmp); -} - -MDBX_ATTRIBUTE_TARGET_AVX2 static __always_inline unsigned -diffcmp2mask_sse2avx(const pgno_t *const ptr, const ptrdiff_t offset, - const __m128i pattern) { - const __m128i f = _mm_loadu_si128((const __m128i *)ptr); - const __m128i l = _mm_loadu_si128((const __m128i *)(ptr + offset)); - const __m128i cmp = _mm_cmpeq_epi32(_mm_sub_epi32(f, l), pattern); - return _mm_movemask_ps(*(const __m128 *)&cmp); -} - -MDBX_MAYBE_UNUSED __hot MDBX_ATTRIBUTE_TARGET_AVX2 static pgno_t * -scan4seq_avx2(pgno_t *range, const size_t len, const size_t seq) { - assert(seq > 0 && len > seq); -#if MDBX_PNL_ASCENDING -#error "FIXME: Not implemented" -#endif /* MDBX_PNL_ASCENDING */ - assert(range[-(ptrdiff_t)len] == len); - pgno_t *const detent = range - len + seq; - const ptrdiff_t offset = -(ptrdiff_t)seq; - const pgno_t target = (pgno_t)offset; - const __m256i pattern = _mm256_set1_epi32(target); - uint8_t mask; - if (likely(len > seq + 7)) { - do { - mask = (uint8_t)diffcmp2mask_avx2(range - 7, offset, pattern); - if (mask) { -#ifndef __SANITIZE_ADDRESS__ - found: -#endif /* __SANITIZE_ADDRESS__ */ - return range + 24 - __builtin_clz(mask); + /* Extend file if required */ + if (likely(rc == MDBX_SUCCESS) && whole_size != used_size) { + if (!dest_is_pipe) + rc = osal_ftruncate(fd, whole_size); + else { + memset(data_buffer, 0, (size_t)MDBX_ENVCOPY_WRITEBUF); + for (size_t offset = used_size; rc == MDBX_SUCCESS && offset < whole_size;) { + const size_t chunk = + ((size_t)MDBX_ENVCOPY_WRITEBUF < whole_size - offset) ? (size_t)MDBX_ENVCOPY_WRITEBUF : whole_size - offset; + rc = osal_write(fd, data_buffer, chunk); + offset += chunk; } - range -= 8; - } while (range > detent + 7); - if (range == detent) - return nullptr; + } } - /* Далее происходит чтение от 4 до 28 лишних байт, которые могут быть не - * только за пределами региона выделенного под PNL, но и пересекать границу - * страницы памяти. Что может приводить как к ошибкам ASAN, так и к падению. - * Поэтому проверяем смещение на странице, а с ASAN всегда страхуемся. */ -#ifndef __SANITIZE_ADDRESS__ - const unsigned on_page_safe_mask = 0xfe0 /* enough for '-31' bytes offset */; - if (likely(on_page_safe_mask & (uintptr_t)(range + offset)) && - !RUNNING_ON_VALGRIND) { - const unsigned extra = (unsigned)(detent + 8 - range); - assert(extra > 0 && extra < 8); - mask = 0xFF << extra; - mask &= diffcmp2mask_avx2(range - 7, offset, pattern); - if (mask) - goto found; - return nullptr; - } -#endif /* __SANITIZE_ADDRESS__ */ - if (range - 3 > detent) { - mask = diffcmp2mask_sse2avx(range - 3, offset, *(const __m128i *)&pattern); - if (mask) - return range + 28 - __builtin_clz(mask); - range -= 4; - } - while (range > detent) { - if (*range - range[offset] == target) - return range; - --range; - } - return nullptr; + return rc; } -#endif /* MDBX_ATTRIBUTE_TARGET_AVX2 */ -#ifdef MDBX_ATTRIBUTE_TARGET_AVX512BW -MDBX_ATTRIBUTE_TARGET_AVX512BW static __always_inline unsigned -diffcmp2mask_avx512bw(const pgno_t *const ptr, const ptrdiff_t offset, - const __m512i pattern) { - const __m512i f = _mm512_loadu_si512((const __m512i *)ptr); - const __m512i l = _mm512_loadu_si512((const __m512i *)(ptr + offset)); - return _mm512_cmpeq_epi32_mask(_mm512_sub_epi32(f, l), pattern); -} +//---------------------------------------------------------------------------- -MDBX_MAYBE_UNUSED __hot MDBX_ATTRIBUTE_TARGET_AVX512BW static pgno_t * -scan4seq_avx512bw(pgno_t *range, const size_t len, const size_t seq) { - assert(seq > 0 && len > seq); -#if MDBX_PNL_ASCENDING -#error "FIXME: Not implemented" -#endif /* MDBX_PNL_ASCENDING */ - assert(range[-(ptrdiff_t)len] == len); - pgno_t *const detent = range - len + seq; - const ptrdiff_t offset = -(ptrdiff_t)seq; - const pgno_t target = (pgno_t)offset; - const __m512i pattern = _mm512_set1_epi32(target); - unsigned mask; - if (likely(len > seq + 15)) { - do { - mask = diffcmp2mask_avx512bw(range - 15, offset, pattern); - if (mask) { -#ifndef __SANITIZE_ADDRESS__ - found: -#endif /* __SANITIZE_ADDRESS__ */ - return range + 16 - __builtin_clz(mask); - } - range -= 16; - } while (range > detent + 15); - if (range == detent) - return nullptr; - } +__cold static int copy2fd(MDBX_txn *txn, mdbx_filehandle_t fd, MDBX_copy_flags_t flags) { + if (unlikely(txn->flags & MDBX_TXN_DIRTY)) + return MDBX_BAD_TXN; - /* Далее происходит чтение от 4 до 60 лишних байт, которые могут быть не - * только за пределами региона выделенного под PNL, но и пересекать границу - * страницы памяти. Что может приводить как к ошибкам ASAN, так и к падению. - * Поэтому проверяем смещение на странице, а с ASAN всегда страхуемся. */ -#ifndef __SANITIZE_ADDRESS__ - const unsigned on_page_safe_mask = 0xfc0 /* enough for '-63' bytes offset */; - if (likely(on_page_safe_mask & (uintptr_t)(range + offset)) && - !RUNNING_ON_VALGRIND) { - const unsigned extra = (unsigned)(detent + 16 - range); - assert(extra > 0 && extra < 16); - mask = 0xFFFF << extra; - mask &= diffcmp2mask_avx512bw(range - 15, offset, pattern); - if (mask) - goto found; - return nullptr; + int rc = MDBX_SUCCESS; + if (txn->flags & MDBX_TXN_RDONLY) { + if (flags & MDBX_CP_THROTTLE_MVCC) { + rc = mdbx_txn_park(txn, true); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + } + } else if (unlikely(flags & (MDBX_CP_THROTTLE_MVCC | MDBX_CP_RENEW_TXN))) + return MDBX_EINVAL; + + const int dest_is_pipe = osal_is_pipe(fd); + if (MDBX_IS_ERROR(dest_is_pipe)) + return dest_is_pipe; + + if (!dest_is_pipe) { + rc = osal_fseek(fd, 0); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; } -#endif /* __SANITIZE_ADDRESS__ */ - if (range - 7 > detent) { - mask = diffcmp2mask_avx2(range - 7, offset, *(const __m256i *)&pattern); - if (mask) - return range + 24 - __builtin_clz(mask); - range -= 8; + + MDBX_env *const env = txn->env; + const size_t buffer_size = + pgno_align2os_bytes(env, NUM_METAS) + + ceil_powerof2(((flags & MDBX_CP_COMPACT) ? 2 * (size_t)MDBX_ENVCOPY_WRITEBUF : (size_t)MDBX_ENVCOPY_WRITEBUF), + globals.sys_pagesize); + + uint8_t *buffer = nullptr; + rc = osal_memalign_alloc(globals.sys_pagesize, buffer_size, (void **)&buffer); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + if (!dest_is_pipe) { + /* Firstly write a stub to meta-pages. + * Now we sure to incomplete copy will not be used. */ + memset(buffer, -1, pgno2bytes(env, NUM_METAS)); + rc = osal_write(fd, buffer, pgno2bytes(env, NUM_METAS)); } - if (range - 3 > detent) { - mask = diffcmp2mask_sse2avx(range - 3, offset, *(const __m128i *)&pattern); - if (mask) - return range + 28 - __builtin_clz(mask); - range -= 4; + + if (likely(rc == MDBX_SUCCESS)) + rc = mdbx_txn_unpark(txn, false); + if (likely(rc == MDBX_SUCCESS)) { + memset(buffer, 0, pgno2bytes(env, NUM_METAS)); + rc = ((flags & MDBX_CP_COMPACT) ? copy_with_compacting : copy_asis)(env, txn, fd, buffer, dest_is_pipe, flags); + + if (likely(rc == MDBX_SUCCESS)) + rc = mdbx_txn_unpark(txn, false); } - while (range > detent) { - if (*range - range[offset] == target) - return range; - --range; + + if (txn->flags & MDBX_TXN_RDONLY) { + if (flags & MDBX_CP_THROTTLE_MVCC) + mdbx_txn_park(txn, true); + else if (flags & MDBX_CP_DISPOSE_TXN) + mdbx_txn_reset(txn); } - return nullptr; -} -#endif /* MDBX_ATTRIBUTE_TARGET_AVX512BW */ -#if (defined(__ARM_NEON) || defined(__ARM_NEON__)) && \ - (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) -static __always_inline size_t diffcmp2mask_neon(const pgno_t *const ptr, - const ptrdiff_t offset, - const uint32x4_t pattern) { - const uint32x4_t f = vld1q_u32(ptr); - const uint32x4_t l = vld1q_u32(ptr + offset); - const uint16x4_t cmp = vmovn_u32(vceqq_u32(vsubq_u32(f, l), pattern)); - if (sizeof(size_t) > 7) - return vget_lane_u64(vreinterpret_u64_u16(cmp), 0); - else - return vget_lane_u32(vreinterpret_u32_u8(vmovn_u16(vcombine_u16(cmp, cmp))), - 0); -} + if (!dest_is_pipe) { + if (likely(rc == MDBX_SUCCESS) && (flags & MDBX_CP_DONT_FLUSH) == 0) + rc = osal_fsync(fd, MDBX_SYNC_DATA | MDBX_SYNC_SIZE); -__hot static pgno_t *scan4seq_neon(pgno_t *range, const size_t len, - const size_t seq) { - assert(seq > 0 && len > seq); -#if MDBX_PNL_ASCENDING -#error "FIXME: Not implemented" -#endif /* MDBX_PNL_ASCENDING */ - assert(range[-(ptrdiff_t)len] == len); - pgno_t *const detent = range - len + seq; - const ptrdiff_t offset = -(ptrdiff_t)seq; - const pgno_t target = (pgno_t)offset; - const uint32x4_t pattern = vmovq_n_u32(target); - size_t mask; - if (likely(len > seq + 3)) { - do { - mask = diffcmp2mask_neon(range - 3, offset, pattern); - if (mask) { -#ifndef __SANITIZE_ADDRESS__ - found: -#endif /* __SANITIZE_ADDRESS__ */ - return ptr_disp(range, -(__builtin_clzl(mask) >> sizeof(size_t) / 4)); - } - range -= 4; - } while (range > detent + 3); - if (range == detent) - return nullptr; - } + /* Write actual meta */ + if (likely(rc == MDBX_SUCCESS)) + rc = osal_pwrite(fd, buffer, pgno2bytes(env, NUM_METAS), 0); - /* Далее происходит чтение от 4 до 12 лишних байт, которые могут быть не - * только за пределами региона выделенного под PNL, но и пересекать границу - * страницы памяти. Что может приводить как к ошибкам ASAN, так и к падению. - * Поэтому проверяем смещение на странице, а с ASAN всегда страхуемся. */ -#ifndef __SANITIZE_ADDRESS__ - const unsigned on_page_safe_mask = 0xff0 /* enough for '-15' bytes offset */; - if (likely(on_page_safe_mask & (uintptr_t)(range + offset)) && - !RUNNING_ON_VALGRIND) { - const unsigned extra = (unsigned)(detent + 4 - range); - assert(extra > 0 && extra < 4); - mask = (~(size_t)0) << (extra * sizeof(size_t) * 2); - mask &= diffcmp2mask_neon(range - 3, offset, pattern); - if (mask) - goto found; - return nullptr; + if (likely(rc == MDBX_SUCCESS) && (flags & MDBX_CP_DONT_FLUSH) == 0) + rc = osal_fsync(fd, MDBX_SYNC_DATA | MDBX_SYNC_IODQ); } -#endif /* __SANITIZE_ADDRESS__ */ - do - if (*range - range[offset] == target) - return range; - while (--range != detent); - return nullptr; + + osal_memalign_free(buffer); + return rc; } -#endif /* __ARM_NEON || __ARM_NEON__ */ -#if defined(__AVX512BW__) && defined(MDBX_ATTRIBUTE_TARGET_AVX512BW) -#define scan4seq_default scan4seq_avx512bw -#define scan4seq_impl scan4seq_default -#elif defined(__AVX2__) && defined(MDBX_ATTRIBUTE_TARGET_AVX2) -#define scan4seq_default scan4seq_avx2 -#elif defined(__SSE2__) && defined(MDBX_ATTRIBUTE_TARGET_SSE2) -#define scan4seq_default scan4seq_sse2 -#elif (defined(__ARM_NEON) || defined(__ARM_NEON__)) && \ - (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) -#define scan4seq_default scan4seq_neon -/* Choosing of another variants should be added here. */ -#endif /* scan4seq_default */ +__cold static int copy2pathname(MDBX_txn *txn, const pathchar_t *dest_path, MDBX_copy_flags_t flags) { + if (unlikely(!dest_path || *dest_path == '\0')) + return MDBX_EINVAL; -#endif /* MDBX_PNL_ASCENDING */ + /* The destination path must exist, but the destination file must not. + * We don't want the OS to cache the writes, since the source data is + * already in the OS cache. */ + mdbx_filehandle_t newfd = INVALID_HANDLE_VALUE; + int rc = osal_openfile(MDBX_OPEN_COPY, txn->env, dest_path, &newfd, +#if defined(_WIN32) || defined(_WIN64) + (mdbx_mode_t)-1 +#else + S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP +#endif + ); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; -#ifndef scan4seq_default -#define scan4seq_default scan4seq_fallback -#endif /* scan4seq_default */ +#if defined(_WIN32) || defined(_WIN64) + /* no locking required since the file opened with ShareMode == 0 */ +#else + MDBX_STRUCT_FLOCK lock_op; + memset(&lock_op, 0, sizeof(lock_op)); + lock_op.l_type = F_WRLCK; + lock_op.l_whence = SEEK_SET; + lock_op.l_start = 0; + lock_op.l_len = OFF_T_MAX; + const int err_fcntl = MDBX_FCNTL(newfd, MDBX_F_SETLK, &lock_op) ? errno : MDBX_SUCCESS; -#ifdef scan4seq_impl -/* The scan4seq_impl() is the best or no alternatives */ -#elif !MDBX_HAVE_BUILTIN_CPU_SUPPORTS -/* The scan4seq_default() will be used since no cpu-features detection support - * from compiler. Please don't ask to implement cpuid-based detection and don't - * make such PRs. */ -#define scan4seq_impl scan4seq_default + const int err_flock = +#ifdef LOCK_EX + flock(newfd, LOCK_EX | LOCK_NB) ? errno : MDBX_SUCCESS; #else -/* Selecting the most appropriate implementation at runtime, - * depending on the available CPU features. */ -static pgno_t *scan4seq_resolver(pgno_t *range, const size_t len, - const size_t seq); -static pgno_t *(*scan4seq_impl)(pgno_t *range, const size_t len, - const size_t seq) = scan4seq_resolver; - -static pgno_t *scan4seq_resolver(pgno_t *range, const size_t len, - const size_t seq) { - pgno_t *(*choice)(pgno_t *range, const size_t len, const size_t seq) = - nullptr; -#if __has_builtin(__builtin_cpu_init) || defined(__BUILTIN_CPU_INIT__) || \ - __GNUC_PREREQ(4, 8) - __builtin_cpu_init(); -#endif /* __builtin_cpu_init() */ -#ifdef MDBX_ATTRIBUTE_TARGET_SSE2 - if (__builtin_cpu_supports("sse2")) - choice = scan4seq_sse2; -#endif /* MDBX_ATTRIBUTE_TARGET_SSE2 */ -#ifdef MDBX_ATTRIBUTE_TARGET_AVX2 - if (__builtin_cpu_supports("avx2")) - choice = scan4seq_avx2; -#endif /* MDBX_ATTRIBUTE_TARGET_AVX2 */ -#ifdef MDBX_ATTRIBUTE_TARGET_AVX512BW - if (__builtin_cpu_supports("avx512bw")) - choice = scan4seq_avx512bw; -#endif /* MDBX_ATTRIBUTE_TARGET_AVX512BW */ - /* Choosing of another variants should be added here. */ - scan4seq_impl = choice ? choice : scan4seq_default; - return scan4seq_impl(range, len, seq); -} -#endif /* scan4seq_impl */ + MDBX_ENOSYS; +#endif /* LOCK_EX */ -//------------------------------------------------------------------------------ + const int err_check_fs_local = + /* avoid call osal_check_fs_local() on success */ + (!err_fcntl && !err_flock && !MDBX_DEBUG) ? MDBX_SUCCESS : +#if !defined(__ANDROID_API__) || __ANDROID_API__ >= 24 + osal_check_fs_local(newfd, 0); +#else + MDBX_ENOSYS; +#endif -/* Allocate page numbers and memory for writing. Maintain mt_last_reclaimed, - * mt_relist and mt_next_pgno. Set MDBX_TXN_ERROR on failure. - * - * If there are free pages available from older transactions, they - * are re-used first. Otherwise allocate a new page at mt_next_pgno. - * Do not modify the GC, just merge GC records into mt_relist - * and move mt_last_reclaimed to say which records were consumed. Only this - * function can create mt_relist and move - * mt_last_reclaimed/mt_next_pgno. - * - * [in] mc cursor A cursor handle identifying the transaction and - * database for which we are allocating. - * [in] num the number of pages to allocate. - * - * Returns 0 on success, non-zero on failure.*/ + const bool flock_may_fail = +#if defined(__linux__) || defined(__gnu_linux__) + err_check_fs_local != 0; +#else + true; +#endif /* Linux */ -#define MDBX_ALLOC_DEFAULT 0 -#define MDBX_ALLOC_RESERVE 1 -#define MDBX_ALLOC_UNIMPORTANT 2 -#define MDBX_ALLOC_COALESCE 4 /* внутреннее состояние */ -#define MDBX_ALLOC_SHOULD_SCAN 8 /* внутреннее состояние */ -#define MDBX_ALLOC_LIFO 16 /* внутреннее состояние */ + if (!err_fcntl && + (err_flock == EWOULDBLOCK || err_flock == EAGAIN || ignore_enosys_and_eremote(err_flock) == MDBX_RESULT_TRUE)) { + rc = err_flock; + if (flock_may_fail) { + WARNING("ignore %s(%" MDBX_PRIsPATH ") error %d: since %s done, local/remote-fs check %d", "flock", dest_path, + err_flock, "fcntl-lock", err_check_fs_local); + rc = MDBX_SUCCESS; + } + } else if (!err_flock && err_check_fs_local == MDBX_RESULT_TRUE && + ignore_enosys_and_eremote(err_fcntl) == MDBX_RESULT_TRUE) { + WARNING("ignore %s(%" MDBX_PRIsPATH ") error %d: since %s done, local/remote-fs check %d", "fcntl-lock", dest_path, + err_fcntl, "flock", err_check_fs_local); + } else if (err_fcntl || err_flock) { + ERROR("file-lock(%" MDBX_PRIsPATH ") failed: fcntl-lock %d, flock %d, local/remote-fs check %d", dest_path, + err_fcntl, err_flock, err_check_fs_local); + if (err_fcntl == ENOLCK || err_flock == ENOLCK) + rc = ENOLCK; + else if (err_fcntl == EWOULDBLOCK || err_flock == EWOULDBLOCK) + rc = EWOULDBLOCK; + else if (EWOULDBLOCK != EAGAIN && (err_fcntl == EAGAIN || err_flock == EAGAIN)) + rc = EAGAIN; + else + rc = (err_fcntl && ignore_enosys_and_eremote(err_fcntl) != MDBX_RESULT_TRUE) ? err_fcntl : err_flock; + } +#endif /* Windows / POSIX */ -static __inline bool is_gc_usable(MDBX_txn *txn, const MDBX_cursor *mc, - const uint8_t flags) { - /* If txn is updating the GC, then the retired-list cannot play catch-up with - * itself by growing while trying to save it. */ - if (mc->mc_dbi == FREE_DBI && !(flags & MDBX_ALLOC_RESERVE) && - !(mc->mc_flags & C_GCU)) - return false; + if (rc == MDBX_SUCCESS) + rc = copy2fd(txn, newfd, flags); - /* avoid search inside empty tree and while tree is updating, - https://libmdbx.dqdkfa.ru/dead-github/issues/31 */ - if (unlikely(txn->mt_dbs[FREE_DBI].md_entries == 0)) { - txn->mt_flags |= MDBX_TXN_DRAINED_GC; - return false; + if (newfd != INVALID_HANDLE_VALUE) { + int err = osal_closefile(newfd); + if (rc == MDBX_SUCCESS && err != rc) + rc = err; + if (rc != MDBX_SUCCESS) + (void)osal_removefile(dest_path); } - - return true; + return rc; } -__hot static bool is_already_reclaimed(const MDBX_txn *txn, txnid_t id) { - const size_t len = MDBX_PNL_GETSIZE(txn->tw.lifo_reclaimed); - for (size_t i = 1; i <= len; ++i) - if (txn->tw.lifo_reclaimed[i] == id) - return true; - return false; +//---------------------------------------------------------------------------- + +__cold int mdbx_txn_copy2fd(MDBX_txn *txn, mdbx_filehandle_t fd, MDBX_copy_flags_t flags) { + int rc = check_txn(txn, MDBX_TXN_BLOCKED); + if (likely(rc == MDBX_SUCCESS)) + rc = copy2fd(txn, fd, flags); + if (flags & MDBX_CP_DISPOSE_TXN) + mdbx_txn_abort(txn); + return LOG_IFERR(rc); } -__hot static pgno_t relist_get_single(MDBX_txn *txn) { - const size_t len = MDBX_PNL_GETSIZE(txn->tw.relist); - assert(len > 0); - pgno_t *target = MDBX_PNL_EDGE(txn->tw.relist); - const ptrdiff_t dir = MDBX_PNL_ASCENDING ? 1 : -1; +__cold int mdbx_env_copy2fd(MDBX_env *env, mdbx_filehandle_t fd, MDBX_copy_flags_t flags) { + if (unlikely(flags & (MDBX_CP_DISPOSE_TXN | MDBX_CP_RENEW_TXN))) + return LOG_IFERR(MDBX_EINVAL); - /* Есть ТРИ потенциально выигрышные, но противо-направленные тактики: - * - * 1. Стараться использовать страницы с наименьшими номерами. Так обмен с - * диском будет более кучным, а у страниц ближе к концу БД будет больше шансов - * попасть под авто-компактификацию. Частично эта тактика уже реализована, но - * для её эффективности требуется явно приоритезировать выделение страниц: - * - поддерживать два relist, для ближних и для дальних страниц; - * - использовать страницы из дальнего списка, если первый пуст, - * а второй слишком большой, либо при пустой GC. - * - * 2. Стараться выделять страницы последовательно. Так записываемые на диск - * регионы будут линейными, что принципиально ускоряет запись на HDD. - * Одновременно, в среднем это не повлияет на чтение, точнее говоря, если - * порядок чтения не совпадает с порядком изменения (иначе говоря, если - * чтение не коррелирует с обновлениями и/или вставками) то не повлияет, иначе - * может ускорить. Однако, последовательности в среднем достаточно редки. - * Поэтому для эффективности требуется аккумулировать и поддерживать в ОЗУ - * огромные списки страниц, а затем сохранять их обратно в БД. Текущий формат - * БД (без сжатых битовых карт) для этого крайне не удачен. Поэтому эта тактика не - * имеет шансов быть успешной без смены формата БД (Mithril). - * - * 3. Стараться экономить последовательности страниц. Это позволяет избегать - * лишнего чтения/поиска в GC при более-менее постоянном размещении и/или - * обновлении данных требующих более одной страницы. Проблема в том, что без - * информации от приложения библиотека не может знать насколько - * востребованными будут последовательности в ближайшей перспективе, а - * экономия последовательностей "на всякий случай" не только затратна - * сама-по-себе, но и работает во вред (добавляет хаоса). - * - * Поэтому: - * - в TODO добавляется разделение relist на «ближние» и «дальние» страницы, - * с последующей реализацией первой тактики; - * - преимущественное использование последовательностей отправляется - * в MithrilDB как составляющая "HDD frendly" feature; - * - реализованная в 3757eb72f7c6b46862f8f17881ac88e8cecc1979 экономия - * последовательностей отключается через MDBX_ENABLE_SAVING_SEQUENCES=0. - * - * В качестве альтернативы для безусловной «экономии» последовательностей, - * в следующих версиях libmdbx, вероятно, будет предложено - * API для взаимодействия с GC: - * - получение размера GC, включая гистограммы размеров последовательностей - * и близости к концу БД; - * - включение формирования "линейного запаса" для последующего использования - * в рамках текущей транзакции; - * - намеренная загрузка GC в память для коагуляции и "выпрямления"; - * - намеренное копирование данных из страниц в конце БД для последующего - * из освобождения, т.е. контролируемая компактификация по запросу. */ + int rc = check_env(env, true); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); -#ifndef MDBX_ENABLE_SAVING_SEQUENCES -#define MDBX_ENABLE_SAVING_SEQUENCES 0 -#endif - if (MDBX_ENABLE_SAVING_SEQUENCES && unlikely(target[dir] == *target + 1) && - len > 2) { - /* Пытаемся пропускать последовательности при наличии одиночных элементов. - * TODO: необходимо кэшировать пропускаемые последовательности - * чтобы не сканировать список сначала при каждом выделении. */ - pgno_t *scan = target + dir + dir; - size_t left = len; - do { - if (likely(scan[-dir] != *scan - 1 && *scan + 1 != scan[dir])) { -#if MDBX_PNL_ASCENDING - target = scan; - break; -#else - /* вырезаем элемент с перемещением хвоста */ - const pgno_t pgno = *scan; - MDBX_PNL_SETSIZE(txn->tw.relist, len - 1); - while (++scan <= target) - scan[-1] = *scan; - return pgno; -#endif - } - scan += dir; - } while (--left > 2); - } + MDBX_txn *txn = nullptr; + rc = mdbx_txn_begin(env, nullptr, MDBX_TXN_RDONLY, &txn); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); - const pgno_t pgno = *target; -#if MDBX_PNL_ASCENDING - /* вырезаем элемент с перемещением хвоста */ - MDBX_PNL_SETSIZE(txn->tw.relist, len - 1); - for (const pgno_t *const end = txn->tw.relist + len - 1; target <= end; - ++target) - *target = target[1]; -#else - /* перемещать хвост не нужно, просто усекам список */ - MDBX_PNL_SETSIZE(txn->tw.relist, len - 1); -#endif - return pgno; + rc = copy2fd(txn, fd, flags | MDBX_CP_DISPOSE_TXN | MDBX_CP_RENEW_TXN); + mdbx_txn_abort(txn); + return LOG_IFERR(rc); } -__hot static pgno_t relist_get_sequence(MDBX_txn *txn, const size_t num, - uint8_t flags) { - const size_t len = MDBX_PNL_GETSIZE(txn->tw.relist); - pgno_t *edge = MDBX_PNL_EDGE(txn->tw.relist); - assert(len >= num && num > 1); - const size_t seq = num - 1; -#if !MDBX_PNL_ASCENDING - if (edge[-(ptrdiff_t)seq] - *edge == seq) { - if (unlikely(flags & MDBX_ALLOC_RESERVE)) - return P_INVALID; - assert(edge == scan4range_checker(txn->tw.relist, seq)); - /* перемещать хвост не нужно, просто усекам список */ - MDBX_PNL_SETSIZE(txn->tw.relist, len - num); - return *edge; - } -#endif - pgno_t *target = scan4seq_impl(edge, len, seq); - assert(target == scan4range_checker(txn->tw.relist, seq)); - if (target) { - if (unlikely(flags & MDBX_ALLOC_RESERVE)) - return P_INVALID; - const pgno_t pgno = *target; - /* вырезаем найденную последовательность с перемещением хвоста */ - MDBX_PNL_SETSIZE(txn->tw.relist, len - num); -#if MDBX_PNL_ASCENDING - for (const pgno_t *const end = txn->tw.relist + len - num; target <= end; - ++target) - *target = target[num]; -#else - for (const pgno_t *const end = txn->tw.relist + len; ++target <= end;) - target[-(ptrdiff_t)num] = *target; -#endif - return pgno; +__cold int mdbx_txn_copy2pathname(MDBX_txn *txn, const char *dest_path, MDBX_copy_flags_t flags) { +#if defined(_WIN32) || defined(_WIN64) + wchar_t *dest_pathW = nullptr; + int rc = osal_mb2w(dest_path, &dest_pathW); + if (likely(rc == MDBX_SUCCESS)) { + rc = mdbx_txn_copy2pathnameW(txn, dest_pathW, flags); + osal_free(dest_pathW); } - return 0; + return LOG_IFERR(rc); } -#if MDBX_ENABLE_MINCORE -static __inline bool bit_tas(uint64_t *field, char bit) { - const uint64_t m = UINT64_C(1) << bit; - const bool r = (*field & m) != 0; - *field |= m; - return r; +__cold int mdbx_txn_copy2pathnameW(MDBX_txn *txn, const wchar_t *dest_path, MDBX_copy_flags_t flags) { +#endif /* Windows */ + int rc = check_txn(txn, MDBX_TXN_BLOCKED); + if (likely(rc == MDBX_SUCCESS)) + rc = copy2pathname(txn, dest_path, flags); + if (flags & MDBX_CP_DISPOSE_TXN) + mdbx_txn_abort(txn); + return LOG_IFERR(rc); } -static bool mincore_fetch(MDBX_env *const env, const size_t unit_begin) { - MDBX_lockinfo *const lck = env->me_lck; - for (size_t i = 1; i < ARRAY_LENGTH(lck->mti_mincore_cache.begin); ++i) { - const ptrdiff_t dist = unit_begin - lck->mti_mincore_cache.begin[i]; - if (likely(dist >= 0 && dist < 64)) { - const pgno_t tmp_begin = lck->mti_mincore_cache.begin[i]; - const uint64_t tmp_mask = lck->mti_mincore_cache.mask[i]; - do { - lck->mti_mincore_cache.begin[i] = lck->mti_mincore_cache.begin[i - 1]; - lck->mti_mincore_cache.mask[i] = lck->mti_mincore_cache.mask[i - 1]; - } while (--i); - lck->mti_mincore_cache.begin[0] = tmp_begin; - lck->mti_mincore_cache.mask[0] = tmp_mask; - return bit_tas(lck->mti_mincore_cache.mask, (char)dist); - } +__cold int mdbx_env_copy(MDBX_env *env, const char *dest_path, MDBX_copy_flags_t flags) { +#if defined(_WIN32) || defined(_WIN64) + wchar_t *dest_pathW = nullptr; + int rc = osal_mb2w(dest_path, &dest_pathW); + if (likely(rc == MDBX_SUCCESS)) { + rc = mdbx_env_copyW(env, dest_pathW, flags); + osal_free(dest_pathW); } + return LOG_IFERR(rc); +} - size_t pages = 64; - unsigned unit_log = sys_pagesize_ln2; - unsigned shift = 0; - if (env->me_psize > env->me_os_psize) { - unit_log = env->me_psize2log; - shift = env->me_psize2log - sys_pagesize_ln2; - pages <<= shift; - } +__cold int mdbx_env_copyW(MDBX_env *env, const wchar_t *dest_path, MDBX_copy_flags_t flags) { +#endif /* Windows */ + if (unlikely(flags & (MDBX_CP_DISPOSE_TXN | MDBX_CP_RENEW_TXN))) + return LOG_IFERR(MDBX_EINVAL); - const size_t offset = unit_begin << unit_log; - size_t length = pages << sys_pagesize_ln2; - if (offset + length > env->me_dxb_mmap.current) { - length = env->me_dxb_mmap.current - offset; - pages = length >> sys_pagesize_ln2; - } + int rc = check_env(env, true); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); -#if MDBX_ENABLE_PGOP_STAT - env->me_lck->mti_pgop_stat.mincore.weak += 1; -#endif /* MDBX_ENABLE_PGOP_STAT */ - uint8_t *const vector = alloca(pages); - if (unlikely(mincore(ptr_disp(env->me_dxb_mmap.base, offset), length, - (void *)vector))) { - NOTICE("mincore(+%zu, %zu), err %d", offset, length, errno); - return false; - } + MDBX_txn *txn = nullptr; + rc = mdbx_txn_begin(env, nullptr, MDBX_TXN_RDONLY, &txn); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); - for (size_t i = 1; i < ARRAY_LENGTH(lck->mti_mincore_cache.begin); ++i) { - lck->mti_mincore_cache.begin[i] = lck->mti_mincore_cache.begin[i - 1]; - lck->mti_mincore_cache.mask[i] = lck->mti_mincore_cache.mask[i - 1]; - } - lck->mti_mincore_cache.begin[0] = unit_begin; + rc = copy2pathname(txn, dest_path, flags | MDBX_CP_DISPOSE_TXN | MDBX_CP_RENEW_TXN); + mdbx_txn_abort(txn); + return LOG_IFERR(rc); +} +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 - uint64_t mask = 0; -#ifdef MINCORE_INCORE - STATIC_ASSERT(MINCORE_INCORE == 1); -#endif - for (size_t i = 0; i < pages; ++i) { - uint64_t bit = (vector[i] & 1) == 0; - bit <<= i >> shift; - mask |= bit; - } +MDBX_cursor *mdbx_cursor_create(void *context) { + cursor_couple_t *couple = osal_calloc(1, sizeof(cursor_couple_t)); + if (unlikely(!couple)) + return nullptr; - lck->mti_mincore_cache.mask[0] = ~mask; - return bit_tas(lck->mti_mincore_cache.mask, 0); + VALGRIND_MAKE_MEM_UNDEFINED(couple, sizeof(cursor_couple_t)); + couple->outer.signature = cur_signature_ready4dispose; + couple->outer.next = &couple->outer; + couple->userctx = context; + cursor_reset(couple); + VALGRIND_MAKE_MEM_DEFINED(&couple->outer.backup, sizeof(couple->outer.backup)); + VALGRIND_MAKE_MEM_DEFINED(&couple->outer.tree, sizeof(couple->outer.tree)); + VALGRIND_MAKE_MEM_DEFINED(&couple->outer.clc, sizeof(couple->outer.clc)); + VALGRIND_MAKE_MEM_DEFINED(&couple->outer.dbi_state, sizeof(couple->outer.dbi_state)); + VALGRIND_MAKE_MEM_DEFINED(&couple->outer.subcur, sizeof(couple->outer.subcur)); + VALGRIND_MAKE_MEM_DEFINED(&couple->outer.txn, sizeof(couple->outer.txn)); + return &couple->outer; } -#endif /* MDBX_ENABLE_MINCORE */ -MDBX_MAYBE_UNUSED static __inline bool mincore_probe(MDBX_env *const env, - const pgno_t pgno) { -#if MDBX_ENABLE_MINCORE - const size_t offset_aligned = - floor_powerof2(pgno2bytes(env, pgno), env->me_os_psize); - const unsigned unit_log2 = (env->me_psize2log > sys_pagesize_ln2) - ? env->me_psize2log - : sys_pagesize_ln2; - const size_t unit_begin = offset_aligned >> unit_log2; - eASSERT(env, (unit_begin << unit_log2) == offset_aligned); - const ptrdiff_t dist = unit_begin - env->me_lck->mti_mincore_cache.begin[0]; - if (likely(dist >= 0 && dist < 64)) - return bit_tas(env->me_lck->mti_mincore_cache.mask, (char)dist); - return mincore_fetch(env, unit_begin); -#else - (void)env; - (void)pgno; - return false; -#endif /* MDBX_ENABLE_MINCORE */ +int mdbx_cursor_renew(MDBX_txn *txn, MDBX_cursor *mc) { + return likely(mc) ? mdbx_cursor_bind(txn, mc, (kvx_t *)mc->clc - txn->env->kvs) : LOG_IFERR(MDBX_EINVAL); } -static __inline pgr_t page_alloc_finalize(MDBX_env *const env, - MDBX_txn *const txn, - const MDBX_cursor *const mc, - const pgno_t pgno, const size_t num) { -#if MDBX_ENABLE_PROFGC - size_t majflt_before; - const uint64_t cputime_before = osal_cputime(&majflt_before); - profgc_stat_t *const prof = (mc->mc_dbi == FREE_DBI) - ? &env->me_lck->mti_pgop_stat.gc_prof.self - : &env->me_lck->mti_pgop_stat.gc_prof.work; -#else - (void)mc; -#endif /* MDBX_ENABLE_PROFGC */ - ENSURE(env, pgno >= NUM_METAS); +int mdbx_cursor_reset(MDBX_cursor *mc) { + int rc = cursor_check(mc, MDBX_TXN_FINISHED); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); - pgr_t ret; - bool need_clean = (env->me_flags & MDBX_PAGEPERTURB) != 0; - if (env->me_flags & MDBX_WRITEMAP) { - ret.page = pgno2page(env, pgno); - MDBX_ASAN_UNPOISON_MEMORY_REGION(ret.page, pgno2bytes(env, num)); - VALGRIND_MAKE_MEM_UNDEFINED(ret.page, pgno2bytes(env, num)); + cursor_reset((cursor_couple_t *)mc); + return MDBX_SUCCESS; +} - /* Содержимое выделенной страницы не нужно, но если страница отсутствует - * в ОЗУ (что весьма вероятно), то любое обращение к ней приведет - * к page-fault: - * - прерыванию по отсутствию страницы; - * - переключение контекста в режим ядра с засыпанием процесса; - * - чтение страницы с диска; - * - обновление PTE и пробуждением процесса; - * - переключение контекста по доступности ЦПУ. - * - * Пытаемся минимизировать накладные расходы записывая страницу, что при - * наличии unified page cache приведет к появлению страницы в ОЗУ без чтения - * с диска. При этом запись на диск должна быть отложена адекватным ядром, - * так как страница отображена в память в режиме чтения-записи и следом в - * неё пишет ЦПУ. */ +int mdbx_cursor_bind(MDBX_txn *txn, MDBX_cursor *mc, MDBX_dbi dbi) { + if (unlikely(!mc)) + return LOG_IFERR(MDBX_EINVAL); - /* В случае если страница в памяти процесса, то излишняя запись может быть - * достаточно дорогой. Кроме системного вызова и копирования данных, в особо - * одаренных ОС при этом могут включаться файловая система, выделяться - * временная страница, пополняться очереди асинхронного выполнения, - * обновляться PTE с последующей генерацией page-fault и чтением данных из - * грязной I/O очереди. Из-за этого штраф за лишнюю запись может быть - * сравним с избегаемым ненужным чтением. */ - if (env->me_prefault_write) { - void *const pattern = ptr_disp( - env->me_pbuf, need_clean ? env->me_psize : env->me_psize * 2); - size_t file_offset = pgno2bytes(env, pgno); - if (likely(num == 1)) { - if (!mincore_probe(env, pgno)) { - osal_pwrite(env->me_lazy_fd, pattern, env->me_psize, file_offset); -#if MDBX_ENABLE_PGOP_STAT - env->me_lck->mti_pgop_stat.prefault.weak += 1; -#endif /* MDBX_ENABLE_PGOP_STAT */ - need_clean = false; - } - } else { - struct iovec iov[MDBX_AUXILARY_IOV_MAX]; - size_t n = 0, cleared = 0; - for (size_t i = 0; i < num; ++i) { - if (!mincore_probe(env, pgno + (pgno_t)i)) { - ++cleared; - iov[n].iov_len = env->me_psize; - iov[n].iov_base = pattern; - if (unlikely(++n == MDBX_AUXILARY_IOV_MAX)) { - osal_pwritev(env->me_lazy_fd, iov, MDBX_AUXILARY_IOV_MAX, - file_offset); -#if MDBX_ENABLE_PGOP_STAT - env->me_lck->mti_pgop_stat.prefault.weak += 1; -#endif /* MDBX_ENABLE_PGOP_STAT */ - file_offset += pgno2bytes(env, MDBX_AUXILARY_IOV_MAX); - n = 0; - } - } - } - if (likely(n > 0)) { - osal_pwritev(env->me_lazy_fd, iov, n, file_offset); -#if MDBX_ENABLE_PGOP_STAT - env->me_lck->mti_pgop_stat.prefault.weak += 1; -#endif /* MDBX_ENABLE_PGOP_STAT */ - } - if (cleared == num) - need_clean = false; - } - } - } else { - ret.page = page_malloc(txn, num); - if (unlikely(!ret.page)) { - ret.err = MDBX_ENOMEM; - goto bailout; - } + if (unlikely(mc->signature != cur_signature_ready4dispose && mc->signature != cur_signature_live)) { + int rc = (mc->signature == cur_signature_wait4eot) ? MDBX_EINVAL : MDBX_EBADSIGN; + return LOG_IFERR(rc); } - if (unlikely(need_clean)) - memset(ret.page, -1, pgno2bytes(env, num)); + int rc = check_txn(txn, MDBX_TXN_FINISHED | MDBX_TXN_HAS_CHILD); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); - VALGRIND_MAKE_MEM_UNDEFINED(ret.page, pgno2bytes(env, num)); - ret.page->mp_pgno = pgno; - ret.page->mp_leaf2_ksize = 0; - ret.page->mp_flags = 0; - if ((ASSERT_ENABLED() || AUDIT_ENABLED()) && num > 1) { - ret.page->mp_pages = (pgno_t)num; - ret.page->mp_flags = P_OVERFLOW; + if (unlikely(dbi == FREE_DBI && !(txn->flags & MDBX_TXN_RDONLY))) + return LOG_IFERR(MDBX_EACCESS); + + rc = dbi_check(txn, dbi); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); + + if (unlikely(mc->backup)) /* Cursor from parent transaction */ + LOG_IFERR(MDBX_EINVAL); + + if (mc->signature == cur_signature_live) { + if (mc->txn == txn && cursor_dbi(mc) == dbi) + return MDBX_SUCCESS; + rc = mdbx_cursor_unbind(mc); + if (unlikely(rc != MDBX_SUCCESS)) + return (rc == MDBX_BAD_TXN) ? MDBX_EINVAL : rc; } + cASSERT(mc, mc->next == mc); - ret.err = page_dirty(txn, ret.page, (pgno_t)num); -bailout: - tASSERT(txn, pnl_check_allocated(txn->tw.relist, - txn->mt_next_pgno - MDBX_ENABLE_REFUND)); -#if MDBX_ENABLE_PROFGC - size_t majflt_after; - prof->xtime_cpu += osal_cputime(&majflt_after) - cputime_before; - prof->majflt += (uint32_t)(majflt_after - majflt_before); -#endif /* MDBX_ENABLE_PROFGC */ - return ret; + rc = cursor_init(mc, txn, dbi); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); + + mc->next = txn->cursors[dbi]; + txn->cursors[dbi] = mc; + return MDBX_SUCCESS; } -static pgr_t page_alloc_slowpath(const MDBX_cursor *const mc, const size_t num, - uint8_t flags) { -#if MDBX_ENABLE_PROFGC - const uint64_t monotime_before = osal_monotime(); -#endif /* MDBX_ENABLE_PROFGC */ +int mdbx_cursor_unbind(MDBX_cursor *mc) { + if (unlikely(!mc)) + return LOG_IFERR(MDBX_EINVAL); - pgr_t ret; - MDBX_txn *const txn = mc->mc_txn; - MDBX_env *const env = txn->mt_env; -#if MDBX_ENABLE_PROFGC - profgc_stat_t *const prof = (mc->mc_dbi == FREE_DBI) - ? &env->me_lck->mti_pgop_stat.gc_prof.self - : &env->me_lck->mti_pgop_stat.gc_prof.work; - prof->spe_counter += 1; -#endif /* MDBX_ENABLE_PROFGC */ + if (unlikely(mc->signature != cur_signature_live)) + return (mc->signature == cur_signature_ready4dispose) ? MDBX_SUCCESS : LOG_IFERR(MDBX_EBADSIGN); - eASSERT(env, num > 0 || (flags & MDBX_ALLOC_RESERVE)); - eASSERT(env, pnl_check_allocated(txn->tw.relist, - txn->mt_next_pgno - MDBX_ENABLE_REFUND)); + if (unlikely(mc->backup)) /* Cursor from parent transaction */ + /* TODO: реализовать при переходе на двусвязный список курсоров */ + return LOG_IFERR(MDBX_EINVAL); - pgno_t pgno = 0; - size_t newnext; - if (num > 1) { -#if MDBX_ENABLE_PROFGC - prof->xpages += 1; -#endif /* MDBX_ENABLE_PROFGC */ - if (MDBX_PNL_GETSIZE(txn->tw.relist) >= num) { - eASSERT(env, MDBX_PNL_LAST(txn->tw.relist) < txn->mt_next_pgno && - MDBX_PNL_FIRST(txn->tw.relist) < txn->mt_next_pgno); - pgno = relist_get_sequence(txn, num, flags); - if (likely(pgno)) - goto done; + int rc = check_txn(mc->txn, MDBX_TXN_FINISHED | MDBX_TXN_HAS_CHILD); + if (unlikely(rc != MDBX_SUCCESS)) { + for (const MDBX_txn *txn = mc->txn; rc == MDBX_BAD_TXN && check_txn(txn, MDBX_TXN_FINISHED) == MDBX_SUCCESS; + txn = txn->nested) + if (dbi_state(txn, cursor_dbi(mc)) == 0) + /* специальный случай: курсор прикреплён к родительской транзакции, но соответствующий dbi-дескриптор ещё + * не использовался во вложенной транзакции, т.е. курсор ещё не импортирован в дочернюю транзакцию и не имеет + * связанного сохранённого состояния (поэтому mc→backup равен nullptr). */ + rc = MDBX_EINVAL; + return LOG_IFERR(rc); + } + + if (unlikely(!mc->txn || mc->txn->signature != txn_signature)) { + ERROR("Wrong cursor's transaction %p 0x%x", __Wpedantic_format_voidptr(mc->txn), mc->txn ? mc->txn->signature : 0); + return LOG_IFERR(MDBX_PROBLEM); + } + + if (mc->next != mc) { + const size_t dbi = cursor_dbi(mc); + cASSERT(mc, dbi < mc->txn->n_dbi); + cASSERT(mc, &mc->txn->env->kvs[dbi].clc == mc->clc); + if (dbi < mc->txn->n_dbi) { + MDBX_cursor **prev = &mc->txn->cursors[dbi]; + while (/* *prev && */ *prev != mc) { + ENSURE(mc->txn->env, (*prev)->signature == cur_signature_live || (*prev)->signature == cur_signature_wait4eot); + prev = &(*prev)->next; + } + cASSERT(mc, *prev == mc); + *prev = mc->next; } - } else { - eASSERT(env, num == 0 || MDBX_PNL_GETSIZE(txn->tw.relist) == 0); - eASSERT(env, !(flags & MDBX_ALLOC_RESERVE) || num == 0); + mc->next = mc; } + cursor_drown((cursor_couple_t *)mc); + mc->signature = cur_signature_ready4dispose; + return MDBX_SUCCESS; +} - //--------------------------------------------------------------------------- +int mdbx_cursor_open(MDBX_txn *txn, MDBX_dbi dbi, MDBX_cursor **ret) { + if (unlikely(!ret)) + return LOG_IFERR(MDBX_EINVAL); + *ret = nullptr; - if (unlikely(!is_gc_usable(txn, mc, flags))) { - eASSERT(env, (txn->mt_flags & MDBX_TXN_DRAINED_GC) || num > 1); - goto no_gc; + MDBX_cursor *const mc = mdbx_cursor_create(nullptr); + if (unlikely(!mc)) + return LOG_IFERR(MDBX_ENOMEM); + + int rc = mdbx_cursor_bind(txn, mc, dbi); + if (unlikely(rc != MDBX_SUCCESS)) { + mdbx_cursor_close(mc); + return LOG_IFERR(rc); } - eASSERT(env, (flags & (MDBX_ALLOC_COALESCE | MDBX_ALLOC_LIFO | - MDBX_ALLOC_SHOULD_SCAN)) == 0); - flags += (env->me_flags & MDBX_LIFORECLAIM) ? MDBX_ALLOC_LIFO : 0; + *ret = mc; + return MDBX_SUCCESS; +} - if (/* Не коагулируем записи при подготовке резерва для обновления GC. - * Иначе попытка увеличить резерв может приводить к необходимости ещё - * большего резерва из-за увеличения списка переработанных страниц. */ - (flags & MDBX_ALLOC_RESERVE) == 0) { - if (txn->mt_dbs[FREE_DBI].md_branch_pages && - MDBX_PNL_GETSIZE(txn->tw.relist) < env->me_maxgc_ov1page / 2) - flags += MDBX_ALLOC_COALESCE; +void mdbx_cursor_close(MDBX_cursor *cursor) { + if (likely(cursor)) { + int err = mdbx_cursor_close2(cursor); + if (unlikely(err != MDBX_SUCCESS)) + mdbx_panic("%s:%d error %d (%s) while closing cursor", __func__, __LINE__, err, mdbx_liberr2str(err)); } +} - MDBX_cursor *const gc = ptr_disp(env->me_txn0, sizeof(MDBX_txn)); - eASSERT(env, mc != gc && gc->mc_next == nullptr); - gc->mc_txn = txn; - gc->mc_flags = 0; - - env->me_prefault_write = env->me_options.prefault_write; - if (env->me_prefault_write) { - /* Проверка посредством minicore() существенно снижает затраты, но в - * простейших случаях (тривиальный бенчмарк) интегральная производительность - * становится вдвое меньше. А на платформах без mincore() и с проблемной - * подсистемой виртуальной памяти ситуация может быть многократно хуже. - * Поэтому избегаем затрат в ситуациях когда prefault-write скорее всего не - * нужна. */ - const bool readahead_enabled = env->me_lck->mti_readahead_anchor & 1; - const pgno_t readahead_edge = env->me_lck->mti_readahead_anchor >> 1; - if (/* Не суетимся если GC почти пустая и БД маленькая */ - (txn->mt_dbs[FREE_DBI].md_branch_pages == 0 && - txn->mt_geo.now < 1234) || - /* Не суетимся если страница в зоне включенного упреждающего чтения */ - (readahead_enabled && pgno + num < readahead_edge)) - env->me_prefault_write = false; +int mdbx_cursor_close2(MDBX_cursor *mc) { + if (unlikely(!mc)) + return LOG_IFERR(MDBX_EINVAL); + + if (mc->signature == cur_signature_ready4dispose) { + if (unlikely(mc->txn || mc->backup)) + return LOG_IFERR(MDBX_PANIC); + cursor_drown((cursor_couple_t *)mc); + mc->signature = 0; + osal_free(mc); + return MDBX_SUCCESS; } -retry_gc_refresh_oldest:; - txnid_t oldest = txn_oldest_reader(txn); -retry_gc_have_oldest: - if (unlikely(oldest >= txn->mt_txnid)) { - ERROR("unexpected/invalid oldest-readed txnid %" PRIaTXN - " for current-txnid %" PRIaTXN, - oldest, txn->mt_txnid); - ret.err = MDBX_PROBLEM; - goto fail; + if (unlikely(mc->signature != cur_signature_live)) + return LOG_IFERR(MDBX_EBADSIGN); + + MDBX_txn *const txn = mc->txn; + int rc = check_txn(txn, MDBX_TXN_FINISHED); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); + + if (mc->backup) { + /* Cursor closed before nested txn ends */ + cursor_reset((cursor_couple_t *)mc); + mc->signature = cur_signature_wait4eot; + return MDBX_SUCCESS; } - const txnid_t detent = oldest + 1; - txnid_t id = 0; - MDBX_cursor_op op = MDBX_FIRST; - if (flags & MDBX_ALLOC_LIFO) { - if (!txn->tw.lifo_reclaimed) { - txn->tw.lifo_reclaimed = txl_alloc(); - if (unlikely(!txn->tw.lifo_reclaimed)) { - ret.err = MDBX_ENOMEM; - goto fail; + if (mc->next != mc) { + const size_t dbi = cursor_dbi(mc); + cASSERT(mc, dbi < mc->txn->n_dbi); + cASSERT(mc, &mc->txn->env->kvs[dbi].clc == mc->clc); + if (likely(dbi < txn->n_dbi)) { + MDBX_cursor **prev = &txn->cursors[dbi]; + while (/* *prev && */ *prev != mc) { + ENSURE(txn->env, (*prev)->signature == cur_signature_live || (*prev)->signature == cur_signature_wait4eot); + prev = &(*prev)->next; } + tASSERT(txn, *prev == mc); + *prev = mc->next; } - /* Begin lookup backward from oldest reader */ - id = detent - 1; - op = MDBX_SET_RANGE; - } else if (txn->tw.last_reclaimed) { - /* Continue lookup forward from last-reclaimed */ - id = txn->tw.last_reclaimed + 1; - if (id >= detent) - goto depleted_gc; - op = MDBX_SET_RANGE; + mc->next = mc; } + cursor_drown((cursor_couple_t *)mc); + mc->signature = 0; + osal_free(mc); + return MDBX_SUCCESS; +} -next_gc:; - MDBX_val key; - key.iov_base = &id; - key.iov_len = sizeof(id); +int mdbx_cursor_copy(const MDBX_cursor *src, MDBX_cursor *dest) { + int rc = cursor_check(src, MDBX_TXN_FINISHED | MDBX_TXN_HAS_CHILD); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); -#if MDBX_ENABLE_PROFGC - prof->rsteps += 1; -#endif /* MDBX_ENABLE_PROFGC */ + rc = mdbx_cursor_bind(src->txn, dest, cursor_dbi(src)); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; - /* Seek first/next GC record */ - ret.err = cursor_get(gc, &key, NULL, op); - if (unlikely(ret.err != MDBX_SUCCESS)) { - if (unlikely(ret.err != MDBX_NOTFOUND)) - goto fail; - if ((flags & MDBX_ALLOC_LIFO) && op == MDBX_SET_RANGE) { - op = MDBX_PREV; - goto next_gc; - } - goto depleted_gc; - } - if (unlikely(key.iov_len != sizeof(txnid_t))) { - ret.err = MDBX_CORRUPTED; - goto fail; - } - id = unaligned_peek_u64(4, key.iov_base); - if (flags & MDBX_ALLOC_LIFO) { - op = MDBX_PREV; - if (id >= detent || is_already_reclaimed(txn, id)) - goto next_gc; - } else { - op = MDBX_NEXT; - if (unlikely(id >= detent)) - goto depleted_gc; + assert(dest->tree == src->tree); + assert(cursor_dbi(dest) == cursor_dbi(src)); +again: + assert(dest->clc == src->clc); + assert(dest->txn == src->txn); + dest->top_and_flags = src->top_and_flags; + for (intptr_t i = 0; i <= src->top; ++i) { + dest->ki[i] = src->ki[i]; + dest->pg[i] = src->pg[i]; + } + + if (src->subcur) { + dest->subcur->nested_tree = src->subcur->nested_tree; + src = &src->subcur->cursor; + dest = &dest->subcur->cursor; + goto again; } - txn->mt_flags &= ~MDBX_TXN_DRAINED_GC; - - /* Reading next GC record */ - MDBX_val data; - MDBX_page *const mp = gc->mc_pg[gc->mc_top]; - if (unlikely((ret.err = node_read(gc, page_node(mp, gc->mc_ki[gc->mc_top]), - &data, mp)) != MDBX_SUCCESS)) - goto fail; - pgno_t *gc_pnl = (pgno_t *)data.iov_base; - if (unlikely(data.iov_len % sizeof(pgno_t) || - data.iov_len < MDBX_PNL_SIZEOF(gc_pnl) || - !pnl_check(gc_pnl, txn->mt_next_pgno))) { - ret.err = MDBX_CORRUPTED; - goto fail; - } + return MDBX_SUCCESS; +} - const size_t gc_len = MDBX_PNL_GETSIZE(gc_pnl); - TRACE("gc-read: id #%" PRIaTXN " len %zu, re-list will %zu ", id, gc_len, - gc_len + MDBX_PNL_GETSIZE(txn->tw.relist)); +int mdbx_txn_release_all_cursors_ex(const MDBX_txn *txn, bool unbind, size_t *count) { + int rc = check_txn(txn, MDBX_TXN_FINISHED | MDBX_TXN_HAS_CHILD); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); - if (unlikely(gc_len + MDBX_PNL_GETSIZE(txn->tw.relist) >= - env->me_maxgc_ov1page)) { - /* Don't try to coalesce too much. */ - if (flags & MDBX_ALLOC_SHOULD_SCAN) { - eASSERT(env, flags & MDBX_ALLOC_COALESCE); - eASSERT(env, !(flags & MDBX_ALLOC_RESERVE)); - eASSERT(env, num > 0); -#if MDBX_ENABLE_PROFGC - env->me_lck->mti_pgop_stat.gc_prof.coalescences += 1; -#endif /* MDBX_ENABLE_PROFGC */ - TRACE("clear %s %s", "MDBX_ALLOC_COALESCE", "since got threshold"); - if (MDBX_PNL_GETSIZE(txn->tw.relist) >= num) { - eASSERT(env, MDBX_PNL_LAST(txn->tw.relist) < txn->mt_next_pgno && - MDBX_PNL_FIRST(txn->tw.relist) < txn->mt_next_pgno); - if (likely(num == 1)) { - pgno = relist_get_single(txn); - goto done; - } - pgno = relist_get_sequence(txn, num, flags); - if (likely(pgno)) - goto done; + size_t n = 0; + do { + TXN_FOREACH_DBI_FROM(txn, i, MAIN_DBI) { + MDBX_cursor *mc = txn->cursors[i], *next = nullptr; + if (mc) { + txn->cursors[i] = nullptr; + do { + next = mc->next; + if (mc->signature == cur_signature_live) { + mc->signature = cur_signature_wait4eot; + cursor_drown((cursor_couple_t *)mc); + } else + ENSURE(nullptr, mc->signature == cur_signature_wait4eot); + if (mc->backup) { + MDBX_cursor *bk = mc->backup; + mc->next = bk->next; + mc->backup = bk->backup; + bk->backup = nullptr; + bk->signature = 0; + osal_free(bk); + } else { + mc->signature = cur_signature_ready4dispose; + mc->next = mc; + ++n; + if (!unbind) { + mc->signature = 0; + osal_free(mc); + } + } + } while ((mc = next) != nullptr); } - flags -= MDBX_ALLOC_COALESCE | MDBX_ALLOC_SHOULD_SCAN; - } - if (unlikely(/* list is too long already */ MDBX_PNL_GETSIZE( - txn->tw.relist) >= env->me_options.rp_augment_limit) && - ((/* not a slot-request from gc-update */ num && - /* have enough unallocated space */ txn->mt_geo.upper >= - txn->mt_next_pgno + num) || - gc_len + MDBX_PNL_GETSIZE(txn->tw.relist) >= MDBX_PGL_LIMIT)) { - /* Stop reclaiming to avoid large/overflow the page list. This is a rare - * case while search for a continuously multi-page region in a - * large database, see https://libmdbx.dqdkfa.ru/dead-github/issues/123 */ - NOTICE("stop reclaiming %s: %zu (current) + %zu " - "(chunk) -> %zu, rp_augment_limit %u", - likely(gc_len + MDBX_PNL_GETSIZE(txn->tw.relist) < MDBX_PGL_LIMIT) - ? "since rp_augment_limit was reached" - : "to avoid PNL overflow", - MDBX_PNL_GETSIZE(txn->tw.relist), gc_len, - gc_len + MDBX_PNL_GETSIZE(txn->tw.relist), - env->me_options.rp_augment_limit); - goto depleted_gc; } - } + txn = txn->parent; + } while (txn); - /* Remember ID of readed GC record */ - txn->tw.last_reclaimed = id; - if (flags & MDBX_ALLOC_LIFO) { - ret.err = txl_append(&txn->tw.lifo_reclaimed, id); - if (unlikely(ret.err != MDBX_SUCCESS)) - goto fail; - } + if (count) + *count = n; + return MDBX_SUCCESS; +} - /* Append PNL from GC record to tw.relist */ - ret.err = pnl_need(&txn->tw.relist, gc_len); - if (unlikely(ret.err != MDBX_SUCCESS)) - goto fail; +int mdbx_cursor_compare(const MDBX_cursor *l, const MDBX_cursor *r, bool ignore_multival) { + const int incomparable = INT16_MAX + 1; - if (LOG_ENABLED(MDBX_LOG_EXTRA)) { - DEBUG_EXTRA("readed GC-pnl txn %" PRIaTXN " root %" PRIaPGNO - " len %zu, PNL", - id, txn->mt_dbs[FREE_DBI].md_root, gc_len); - for (size_t i = gc_len; i; i--) - DEBUG_EXTRA_PRINT(" %" PRIaPGNO, gc_pnl[i]); - DEBUG_EXTRA_PRINT(", next_pgno %u\n", txn->mt_next_pgno); - } + if (unlikely(!l)) + return r ? -incomparable * 9 : 0; + else if (unlikely(!r)) + return incomparable * 9; - /* Merge in descending sorted order */ - pnl_merge(txn->tw.relist, gc_pnl); - flags |= MDBX_ALLOC_SHOULD_SCAN; - if (AUDIT_ENABLED()) { - if (unlikely(!pnl_check(txn->tw.relist, txn->mt_next_pgno))) { - ret.err = MDBX_CORRUPTED; - goto fail; - } - } else { - eASSERT(env, pnl_check_allocated(txn->tw.relist, txn->mt_next_pgno)); + if (unlikely(cursor_check_pure(l) != MDBX_SUCCESS)) + return (cursor_check_pure(r) == MDBX_SUCCESS) ? -incomparable * 8 : 0; + if (unlikely(cursor_check_pure(r) != MDBX_SUCCESS)) + return (cursor_check_pure(l) == MDBX_SUCCESS) ? incomparable * 8 : 0; + + if (unlikely(l->clc != r->clc)) { + if (l->txn->env != r->txn->env) + return (l->txn->env > r->txn->env) ? incomparable * 7 : -incomparable * 7; + if (l->txn->txnid != r->txn->txnid) + return (l->txn->txnid > r->txn->txnid) ? incomparable * 6 : -incomparable * 6; + return (l->clc > r->clc) ? incomparable * 5 : -incomparable * 5; } - eASSERT(env, dirtylist_check(txn)); + assert(cursor_dbi(l) == cursor_dbi(r)); - eASSERT(env, MDBX_PNL_GETSIZE(txn->tw.relist) == 0 || - MDBX_PNL_MOST(txn->tw.relist) < txn->mt_next_pgno); - if (MDBX_ENABLE_REFUND && MDBX_PNL_GETSIZE(txn->tw.relist) && - unlikely(MDBX_PNL_MOST(txn->tw.relist) == txn->mt_next_pgno - 1)) { - /* Refund suitable pages into "unallocated" space */ - txn_refund(txn); + int diff = is_pointed(l) - is_pointed(r); + if (unlikely(diff)) + return (diff > 0) ? incomparable * 4 : -incomparable * 4; + if (unlikely(!is_pointed(l))) + return 0; + + intptr_t detent = (l->top <= r->top) ? l->top : r->top; + for (intptr_t i = 0; i <= detent; ++i) { + diff = l->ki[i] - r->ki[i]; + if (diff) + return diff; } - eASSERT(env, pnl_check_allocated(txn->tw.relist, - txn->mt_next_pgno - MDBX_ENABLE_REFUND)); + if (unlikely(l->top != r->top)) + return (l->top > r->top) ? incomparable * 3 : -incomparable * 3; - /* Done for a kick-reclaim mode, actually no page needed */ - if (unlikely(num == 0)) { - eASSERT(env, ret.err == MDBX_SUCCESS); - TRACE("%s: last id #%" PRIaTXN ", re-len %zu", "early-exit for slot", id, - MDBX_PNL_GETSIZE(txn->tw.relist)); - goto early_exit; + assert((l->subcur != nullptr) == (r->subcur != nullptr)); + if (unlikely((l->subcur != nullptr) != (r->subcur != nullptr))) + return l->subcur ? incomparable * 2 : -incomparable * 2; + if (ignore_multival || !l->subcur) + return 0; + +#if MDBX_DEBUG + if (is_pointed(&l->subcur->cursor)) { + const page_t *mp = l->pg[l->top]; + const node_t *node = page_node(mp, l->ki[l->top]); + assert(node_flags(node) & N_DUP); } + if (is_pointed(&r->subcur->cursor)) { + const page_t *mp = r->pg[r->top]; + const node_t *node = page_node(mp, r->ki[r->top]); + assert(node_flags(node) & N_DUP); + } +#endif /* MDBX_DEBUG */ - /* TODO: delete reclaimed records */ + l = &l->subcur->cursor; + r = &r->subcur->cursor; + diff = is_pointed(l) - is_pointed(r); + if (unlikely(diff)) + return (diff > 0) ? incomparable * 2 : -incomparable * 2; + if (unlikely(!is_pointed(l))) + return 0; - eASSERT(env, op == MDBX_PREV || op == MDBX_NEXT); - if (flags & MDBX_ALLOC_COALESCE) { - TRACE("%s: last id #%" PRIaTXN ", re-len %zu", "coalesce-continue", id, - MDBX_PNL_GETSIZE(txn->tw.relist)); - goto next_gc; + detent = (l->top <= r->top) ? l->top : r->top; + for (intptr_t i = 0; i <= detent; ++i) { + diff = l->ki[i] - r->ki[i]; + if (diff) + return diff; } + if (unlikely(l->top != r->top)) + return (l->top > r->top) ? incomparable : -incomparable; -scan: - eASSERT(env, flags & MDBX_ALLOC_SHOULD_SCAN); - eASSERT(env, num > 0); - if (MDBX_PNL_GETSIZE(txn->tw.relist) >= num) { - eASSERT(env, MDBX_PNL_LAST(txn->tw.relist) < txn->mt_next_pgno && - MDBX_PNL_FIRST(txn->tw.relist) < txn->mt_next_pgno); - if (likely(num == 1)) { - eASSERT(env, !(flags & MDBX_ALLOC_RESERVE)); - pgno = relist_get_single(txn); - goto done; + return (l->flags & z_eof_hard) - (r->flags & z_eof_hard); +} + +int mdbx_cursor_count_ex(const MDBX_cursor *mc, size_t *count, MDBX_stat *ns, size_t bytes) { + int rc = cursor_check_ro(mc); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); + + if (ns) { + const size_t size_before_modtxnid = offsetof(MDBX_stat, ms_mod_txnid); + if (unlikely(bytes != sizeof(MDBX_stat)) && bytes != size_before_modtxnid) + return LOG_IFERR(MDBX_EINVAL); + memset(ns, 0, sizeof(*ns)); + } + + size_t nvals = 0; + if (is_filled(mc)) { + nvals = 1; + if (!inner_hollow(mc)) { + const page_t *mp = mc->pg[mc->top]; + const node_t *node = page_node(mp, mc->ki[mc->top]); + cASSERT(mc, node_flags(node) & N_DUP); + const tree_t *nt = &mc->subcur->nested_tree; + nvals = unlikely(nt->items > PTRDIFF_MAX) ? PTRDIFF_MAX : (size_t)nt->items; + if (ns) { + ns->ms_psize = (unsigned)node_ds(node); + if (node_flags(node) & N_TREE) { + ns->ms_psize = mc->txn->env->ps; + ns->ms_depth = nt->height; + ns->ms_branch_pages = nt->branch_pages; + } + cASSERT(mc, nt->large_pages == 0); + ns->ms_leaf_pages = nt->leaf_pages; + ns->ms_entries = nt->items; + if (likely(bytes >= offsetof(MDBX_stat, ms_mod_txnid) + sizeof(ns->ms_mod_txnid))) + ns->ms_mod_txnid = nt->mod_txnid; + } } - pgno = relist_get_sequence(txn, num, flags); - if (likely(pgno)) - goto done; - } - flags -= MDBX_ALLOC_SHOULD_SCAN; - if (ret.err == MDBX_SUCCESS) { - TRACE("%s: last id #%" PRIaTXN ", re-len %zu", "continue-search", id, - MDBX_PNL_GETSIZE(txn->tw.relist)); - goto next_gc; } -depleted_gc: - TRACE("%s: last id #%" PRIaTXN ", re-len %zu", "gc-depleted", id, - MDBX_PNL_GETSIZE(txn->tw.relist)); - ret.err = MDBX_NOTFOUND; - if (flags & MDBX_ALLOC_SHOULD_SCAN) - goto scan; - txn->mt_flags |= MDBX_TXN_DRAINED_GC; + if (likely(count)) + *count = nvals; - //------------------------------------------------------------------------- + return MDBX_SUCCESS; +} - /* There is no suitable pages in the GC and to be able to allocate - * we should CHOICE one of: - * - make a new steady checkpoint if reclaiming was stopped by - * the last steady-sync, or wipe it in the MDBX_UTTERLY_NOSYNC mode; - * - kick lagging reader(s) if reclaiming was stopped by ones of it. - * - extend the database file. */ +int mdbx_cursor_count(const MDBX_cursor *mc, size_t *count) { + if (unlikely(count == nullptr)) + return LOG_IFERR(MDBX_EINVAL); - /* Will use new pages from the map if nothing is suitable in the GC. */ - newnext = txn->mt_next_pgno + num; + return mdbx_cursor_count_ex(mc, count, nullptr, 0); +} - /* Does reclaiming stopped at the last steady point? */ - const meta_ptr_t recent = meta_recent(env, &txn->tw.troika); - const meta_ptr_t prefer_steady = meta_prefer_steady(env, &txn->tw.troika); - if (recent.ptr_c != prefer_steady.ptr_c && prefer_steady.is_steady && - detent == prefer_steady.txnid + 1) { - DEBUG("gc-kick-steady: recent %" PRIaTXN "-%s, steady %" PRIaTXN - "-%s, detent %" PRIaTXN, - recent.txnid, durable_caption(recent.ptr_c), prefer_steady.txnid, - durable_caption(prefer_steady.ptr_c), detent); - const pgno_t autosync_threshold = - atomic_load32(&env->me_lck->mti_autosync_threshold, mo_Relaxed); - const uint64_t autosync_period = - atomic_load64(&env->me_lck->mti_autosync_period, mo_Relaxed); - uint64_t eoos_timestamp; - /* wipe the last steady-point if one of: - * - UTTERLY_NOSYNC mode AND auto-sync threshold is NOT specified - * - UTTERLY_NOSYNC mode AND free space at steady-point is exhausted - * otherwise, make a new steady-point if one of: - * - auto-sync threshold is specified and reached; - * - upper limit of database size is reached; - * - database is full (with the current file size) - * AND auto-sync threshold it NOT specified */ - if (F_ISSET(env->me_flags, MDBX_UTTERLY_NOSYNC) && - ((autosync_threshold | autosync_period) == 0 || - newnext >= prefer_steady.ptr_c->mm_geo.now)) { - /* wipe steady checkpoint in MDBX_UTTERLY_NOSYNC mode - * without any auto-sync threshold(s). */ -#if MDBX_ENABLE_PROFGC - env->me_lck->mti_pgop_stat.gc_prof.wipes += 1; -#endif /* MDBX_ENABLE_PROFGC */ - ret.err = wipe_steady(txn, detent); - DEBUG("gc-wipe-steady, rc %d", ret.err); - if (unlikely(ret.err != MDBX_SUCCESS)) - goto fail; - eASSERT(env, prefer_steady.ptr_c != - meta_prefer_steady(env, &txn->tw.troika).ptr_c); - goto retry_gc_refresh_oldest; - } - if ((autosync_threshold && - atomic_load64(&env->me_lck->mti_unsynced_pages, mo_Relaxed) >= - autosync_threshold) || - (autosync_period && - (eoos_timestamp = - atomic_load64(&env->me_lck->mti_eoos_timestamp, mo_Relaxed)) && - osal_monotime() - eoos_timestamp >= autosync_period) || - newnext >= txn->mt_geo.upper || - ((num == 0 || newnext >= txn->mt_end_pgno) && - (autosync_threshold | autosync_period) == 0)) { - /* make steady checkpoint. */ -#if MDBX_ENABLE_PROFGC - env->me_lck->mti_pgop_stat.gc_prof.flushes += 1; -#endif /* MDBX_ENABLE_PROFGC */ - MDBX_meta meta = *recent.ptr_c; - ret.err = sync_locked(env, env->me_flags & MDBX_WRITEMAP, &meta, - &txn->tw.troika); - DEBUG("gc-make-steady, rc %d", ret.err); - eASSERT(env, ret.err != MDBX_RESULT_TRUE); - if (unlikely(ret.err != MDBX_SUCCESS)) - goto fail; - eASSERT(env, prefer_steady.ptr_c != - meta_prefer_steady(env, &txn->tw.troika).ptr_c); - goto retry_gc_refresh_oldest; - } - } +int mdbx_cursor_on_first(const MDBX_cursor *mc) { + int rc = cursor_check_pure(mc); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); - if (unlikely(true == atomic_load32(&env->me_lck->mti_readers_refresh_flag, - mo_AcquireRelease))) { - oldest = txn_oldest_reader(txn); - if (oldest >= detent) - goto retry_gc_have_oldest; + for (intptr_t i = 0; i <= mc->top; ++i) { + if (mc->ki[i]) + return MDBX_RESULT_FALSE; } - /* Avoid kick lagging reader(s) if is enough unallocated space - * at the end of database file. */ - if (!(flags & MDBX_ALLOC_RESERVE) && newnext <= txn->mt_end_pgno) { - eASSERT(env, pgno == 0); - goto done; - } + return MDBX_RESULT_TRUE; +} - if (oldest < txn->mt_txnid - xMDBX_TXNID_STEP) { - oldest = kick_longlived_readers(env, oldest); - if (oldest >= detent) - goto retry_gc_have_oldest; +int mdbx_cursor_on_first_dup(const MDBX_cursor *mc) { + int rc = cursor_check_pure(mc); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); + + if (is_filled(mc) && mc->subcur) { + mc = &mc->subcur->cursor; + for (intptr_t i = 0; i <= mc->top; ++i) { + if (mc->ki[i]) + return MDBX_RESULT_FALSE; + } } - //--------------------------------------------------------------------------- + return MDBX_RESULT_TRUE; +} -no_gc: - eASSERT(env, pgno == 0); -#ifndef MDBX_ENABLE_BACKLOG_DEPLETED -#define MDBX_ENABLE_BACKLOG_DEPLETED 0 -#endif /* MDBX_ENABLE_BACKLOG_DEPLETED*/ - if (MDBX_ENABLE_BACKLOG_DEPLETED && - unlikely(!(txn->mt_flags & MDBX_TXN_DRAINED_GC))) { - ret.err = MDBX_BACKLOG_DEPLETED; - goto fail; - } - if (flags & MDBX_ALLOC_RESERVE) { - ret.err = MDBX_NOTFOUND; - goto fail; +int mdbx_cursor_on_last(const MDBX_cursor *mc) { + int rc = cursor_check_pure(mc); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); + + for (intptr_t i = 0; i <= mc->top; ++i) { + size_t nkeys = page_numkeys(mc->pg[i]); + if (mc->ki[i] < nkeys - 1) + return MDBX_RESULT_FALSE; } - /* Will use new pages from the map if nothing is suitable in the GC. */ - newnext = txn->mt_next_pgno + num; - if (newnext <= txn->mt_end_pgno) - goto done; + return MDBX_RESULT_TRUE; +} - if (newnext > txn->mt_geo.upper || !txn->mt_geo.grow_pv) { - NOTICE("gc-alloc: next %zu > upper %" PRIaPGNO, newnext, txn->mt_geo.upper); - ret.err = MDBX_MAP_FULL; - goto fail; +int mdbx_cursor_on_last_dup(const MDBX_cursor *mc) { + int rc = cursor_check_pure(mc); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); + + if (is_filled(mc) && mc->subcur) { + mc = &mc->subcur->cursor; + for (intptr_t i = 0; i <= mc->top; ++i) { + size_t nkeys = page_numkeys(mc->pg[i]); + if (mc->ki[i] < nkeys - 1) + return MDBX_RESULT_FALSE; + } } - eASSERT(env, newnext > txn->mt_end_pgno); - const size_t grow_step = pv2pages(txn->mt_geo.grow_pv); - size_t aligned = pgno_align2os_pgno( - env, (pgno_t)(newnext + grow_step - newnext % grow_step)); + return MDBX_RESULT_TRUE; +} - if (aligned > txn->mt_geo.upper) - aligned = txn->mt_geo.upper; - eASSERT(env, aligned >= newnext); +int mdbx_cursor_eof(const MDBX_cursor *mc) { + int rc = cursor_check_pure(mc); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); - VERBOSE("try growth datafile to %zu pages (+%zu)", aligned, - aligned - txn->mt_end_pgno); - ret.err = dxb_resize(env, txn->mt_next_pgno, (pgno_t)aligned, - txn->mt_geo.upper, implicit_grow); - if (ret.err != MDBX_SUCCESS) { - ERROR("unable growth datafile to %zu pages (+%zu), errcode %d", aligned, - aligned - txn->mt_end_pgno, ret.err); - goto fail; - } - env->me_txn->mt_end_pgno = (pgno_t)aligned; - eASSERT(env, pgno == 0); + return is_eof(mc) ? MDBX_RESULT_TRUE : MDBX_RESULT_FALSE; +} - //--------------------------------------------------------------------------- +int mdbx_cursor_get(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, MDBX_cursor_op op) { + int rc = cursor_check_ro(mc); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); -done: - ret.err = MDBX_SUCCESS; - if (likely((flags & MDBX_ALLOC_RESERVE) == 0)) { - if (pgno) { - eASSERT(env, pgno + num <= txn->mt_next_pgno && pgno >= NUM_METAS); - eASSERT(env, pnl_check_allocated(txn->tw.relist, - txn->mt_next_pgno - MDBX_ENABLE_REFUND)); - } else { - pgno = txn->mt_next_pgno; - txn->mt_next_pgno += (pgno_t)num; - eASSERT(env, txn->mt_next_pgno <= txn->mt_end_pgno); - eASSERT(env, pgno >= NUM_METAS && pgno + num <= txn->mt_next_pgno); + return LOG_IFERR(cursor_ops(mc, key, data, op)); +} + +__hot static int scan_confinue(MDBX_cursor *mc, MDBX_predicate_func *predicate, void *context, void *arg, MDBX_val *key, + MDBX_val *value, MDBX_cursor_op turn_op) { + int rc; + switch (turn_op) { + case MDBX_NEXT: + case MDBX_NEXT_NODUP: + for (;;) { + rc = predicate(context, key, value, arg); + if (rc != MDBX_RESULT_FALSE) + return rc; + rc = outer_next(mc, key, value, turn_op); + if (unlikely(rc != MDBX_SUCCESS)) + return (rc == MDBX_NOTFOUND) ? MDBX_RESULT_FALSE : rc; } - ret = page_alloc_finalize(env, txn, mc, pgno, num); - if (unlikely(ret.err != MDBX_SUCCESS)) { - fail: - eASSERT(env, ret.err != MDBX_SUCCESS); - eASSERT(env, pnl_check_allocated(txn->tw.relist, - txn->mt_next_pgno - MDBX_ENABLE_REFUND)); - int level; - const char *what; - if (flags & MDBX_ALLOC_RESERVE) { - level = - (flags & MDBX_ALLOC_UNIMPORTANT) ? MDBX_LOG_DEBUG : MDBX_LOG_NOTICE; - what = num ? "reserve-pages" : "fetch-slot"; - } else { - txn->mt_flags |= MDBX_TXN_ERROR; - level = MDBX_LOG_ERROR; - what = "pages"; + case MDBX_PREV: + case MDBX_PREV_NODUP: + for (;;) { + rc = predicate(context, key, value, arg); + if (rc != MDBX_RESULT_FALSE) + return rc; + rc = outer_prev(mc, key, value, turn_op); + if (unlikely(rc != MDBX_SUCCESS)) + return (rc == MDBX_NOTFOUND) ? MDBX_RESULT_FALSE : rc; + } + + case MDBX_NEXT_DUP: + if (mc->subcur) + for (;;) { + rc = predicate(context, key, value, arg); + if (rc != MDBX_RESULT_FALSE) + return rc; + rc = inner_next(&mc->subcur->cursor, value); + if (unlikely(rc != MDBX_SUCCESS)) + return (rc == MDBX_NOTFOUND) ? MDBX_RESULT_FALSE : rc; } - if (LOG_ENABLED(level)) - debug_log(level, __func__, __LINE__, - "unable alloc %zu %s, alloc-flags 0x%x, err %d, txn-flags " - "0x%x, re-list-len %zu, loose-count %zu, gc: height %u, " - "branch %zu, leaf %zu, large %zu, entries %zu\n", - num, what, flags, ret.err, txn->mt_flags, - MDBX_PNL_GETSIZE(txn->tw.relist), txn->tw.loose_count, - txn->mt_dbs[FREE_DBI].md_depth, - (size_t)txn->mt_dbs[FREE_DBI].md_branch_pages, - (size_t)txn->mt_dbs[FREE_DBI].md_leaf_pages, - (size_t)txn->mt_dbs[FREE_DBI].md_overflow_pages, - (size_t)txn->mt_dbs[FREE_DBI].md_entries); - ret.page = NULL; + return MDBX_NOTFOUND; + + case MDBX_PREV_DUP: + if (mc->subcur) + for (;;) { + rc = predicate(context, key, value, arg); + if (rc != MDBX_RESULT_FALSE) + return rc; + rc = inner_prev(&mc->subcur->cursor, value); + if (unlikely(rc != MDBX_SUCCESS)) + return (rc == MDBX_NOTFOUND) ? MDBX_RESULT_FALSE : rc; + } + return MDBX_NOTFOUND; + + default: + for (;;) { + rc = predicate(context, key, value, arg); + if (rc != MDBX_RESULT_FALSE) + return rc; + rc = cursor_ops(mc, key, value, turn_op); + if (unlikely(rc != MDBX_SUCCESS)) + return (rc == MDBX_NOTFOUND) ? MDBX_RESULT_FALSE : rc; } - } else { - early_exit: - DEBUG("return NULL for %zu pages for ALLOC_%s, rc %d", num, - num ? "RESERVE" : "SLOT", ret.err); - ret.page = NULL; } +} -#if MDBX_ENABLE_PROFGC - prof->rtime_monotonic += osal_monotime() - monotime_before; -#endif /* MDBX_ENABLE_PROFGC */ - return ret; +int mdbx_cursor_scan(MDBX_cursor *mc, MDBX_predicate_func *predicate, void *context, MDBX_cursor_op start_op, + MDBX_cursor_op turn_op, void *arg) { + if (unlikely(!predicate)) + return LOG_IFERR(MDBX_EINVAL); + + const unsigned valid_start_mask = 1 << MDBX_FIRST | 1 << MDBX_FIRST_DUP | 1 << MDBX_LAST | 1 << MDBX_LAST_DUP | + 1 << MDBX_GET_CURRENT | 1 << MDBX_GET_MULTIPLE; + if (unlikely(start_op > 30 || ((1 << start_op) & valid_start_mask) == 0)) + return LOG_IFERR(MDBX_EINVAL); + + const unsigned valid_turn_mask = 1 << MDBX_NEXT | 1 << MDBX_NEXT_DUP | 1 << MDBX_NEXT_NODUP | 1 << MDBX_PREV | + 1 << MDBX_PREV_DUP | 1 << MDBX_PREV_NODUP | 1 << MDBX_NEXT_MULTIPLE | + 1 << MDBX_PREV_MULTIPLE; + if (unlikely(turn_op > 30 || ((1 << turn_op) & valid_turn_mask) == 0)) + return LOG_IFERR(MDBX_EINVAL); + + MDBX_val key = {nullptr, 0}, value = {nullptr, 0}; + int rc = mdbx_cursor_get(mc, &key, &value, start_op); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); + return LOG_IFERR(scan_confinue(mc, predicate, context, arg, &key, &value, turn_op)); } -__hot static pgr_t page_alloc(const MDBX_cursor *const mc) { - MDBX_txn *const txn = mc->mc_txn; - tASSERT(txn, mc->mc_txn->mt_flags & MDBX_TXN_DIRTY); - tASSERT(txn, F_ISSET(txn->mt_dbistate[mc->mc_dbi], DBI_DIRTY | DBI_VALID)); +int mdbx_cursor_scan_from(MDBX_cursor *mc, MDBX_predicate_func *predicate, void *context, MDBX_cursor_op from_op, + MDBX_val *key, MDBX_val *value, MDBX_cursor_op turn_op, void *arg) { + if (unlikely(!predicate || !key)) + return LOG_IFERR(MDBX_EINVAL); - /* If there are any loose pages, just use them */ - while (likely(txn->tw.loose_pages)) { -#if MDBX_ENABLE_REFUND - if (unlikely(txn->tw.loose_refund_wl > txn->mt_next_pgno)) { - txn_refund(txn); - if (!txn->tw.loose_pages) - break; - } -#endif /* MDBX_ENABLE_REFUND */ + const unsigned valid_start_mask = 1 << MDBX_GET_BOTH | 1 << MDBX_GET_BOTH_RANGE | 1 << MDBX_SET_KEY | + 1 << MDBX_GET_MULTIPLE | 1 << MDBX_SET_LOWERBOUND | 1 << MDBX_SET_UPPERBOUND; + if (unlikely(from_op < MDBX_TO_KEY_LESSER_THAN && ((1 << from_op) & valid_start_mask) == 0)) + return LOG_IFERR(MDBX_EINVAL); - MDBX_page *lp = txn->tw.loose_pages; - MDBX_ASAN_UNPOISON_MEMORY_REGION(lp, txn->mt_env->me_psize); - VALGRIND_MAKE_MEM_DEFINED(&mp_next(lp), sizeof(MDBX_page *)); - txn->tw.loose_pages = mp_next(lp); - txn->tw.loose_count--; - DEBUG_EXTRA("db %d use loose page %" PRIaPGNO, DDBI(mc), lp->mp_pgno); - tASSERT(txn, lp->mp_pgno < txn->mt_next_pgno); - tASSERT(txn, lp->mp_pgno >= NUM_METAS); - VALGRIND_MAKE_MEM_UNDEFINED(page_data(lp), page_space(txn->mt_env)); - lp->mp_txnid = txn->mt_front; - pgr_t ret = {lp, MDBX_SUCCESS}; - return ret; - } + const unsigned valid_turn_mask = 1 << MDBX_NEXT | 1 << MDBX_NEXT_DUP | 1 << MDBX_NEXT_NODUP | 1 << MDBX_PREV | + 1 << MDBX_PREV_DUP | 1 << MDBX_PREV_NODUP | 1 << MDBX_NEXT_MULTIPLE | + 1 << MDBX_PREV_MULTIPLE; + if (unlikely(turn_op > 30 || ((1 << turn_op) & valid_turn_mask) == 0)) + return LOG_IFERR(MDBX_EINVAL); - if (likely(MDBX_PNL_GETSIZE(txn->tw.relist) > 0)) - return page_alloc_finalize(txn->mt_env, txn, mc, relist_get_single(txn), 1); + int rc = mdbx_cursor_get(mc, key, value, from_op); + if (unlikely(MDBX_IS_ERROR(rc))) + return LOG_IFERR(rc); - return page_alloc_slowpath(mc, 1, MDBX_ALLOC_DEFAULT); + cASSERT(mc, key != nullptr); + MDBX_val stub; + if (!value) { + value = &stub; + rc = cursor_ops(mc, key, value, MDBX_GET_CURRENT); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); + } + return LOG_IFERR(scan_confinue(mc, predicate, context, arg, key, value, turn_op)); } -/* Copy the used portions of a page. */ -__hot static void page_copy(MDBX_page *const dst, const MDBX_page *const src, - const size_t size) { - STATIC_ASSERT(UINT16_MAX > MAX_PAGESIZE - PAGEHDRSZ); - STATIC_ASSERT(MIN_PAGESIZE > PAGEHDRSZ + NODESIZE * 4); - void *copy_dst = dst; - const void *copy_src = src; - size_t copy_len = size; - if (src->mp_flags & P_LEAF2) { - copy_len = PAGEHDRSZ + src->mp_leaf2_ksize * page_numkeys(src); - if (unlikely(copy_len > size)) - goto bailout; +int mdbx_cursor_get_batch(MDBX_cursor *mc, size_t *count, MDBX_val *pairs, size_t limit, MDBX_cursor_op op) { + if (unlikely(!count)) + return LOG_IFERR(MDBX_EINVAL); + + *count = 0; + if (unlikely(limit < 4 || limit > INTPTR_MAX - 2)) + return LOG_IFERR(MDBX_EINVAL); + + int rc = cursor_check_ro(mc); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); + + if (unlikely(mc->subcur)) + return LOG_IFERR(MDBX_INCOMPATIBLE) /* must be a non-dupsort table */; + + switch (op) { + case MDBX_NEXT: + if (unlikely(is_eof(mc))) + return LOG_IFERR(is_pointed(mc) ? MDBX_NOTFOUND : MDBX_ENODATA); + break; + + case MDBX_FIRST: + if (!is_filled(mc)) { + rc = outer_first(mc, nullptr, nullptr); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); + } + break; + + default: + DEBUG("unhandled/unimplemented cursor operation %u", op); + return LOG_IFERR(MDBX_EINVAL); } - if ((src->mp_flags & (P_LEAF2 | P_OVERFLOW)) == 0) { - size_t upper = src->mp_upper, lower = src->mp_lower; - intptr_t unused = upper - lower; - /* If page isn't full, just copy the used portion. Adjust - * alignment so memcpy may copy words instead of bytes. */ - if (unused > MDBX_CACHELINE_SIZE * 3) { - lower = ceil_powerof2(lower + PAGEHDRSZ, sizeof(void *)); - upper = floor_powerof2(upper + PAGEHDRSZ, sizeof(void *)); - if (unlikely(upper > copy_len)) + + const page_t *mp = mc->pg[mc->top]; + size_t nkeys = page_numkeys(mp); + size_t ki = mc->ki[mc->top]; + size_t n = 0; + while (n + 2 <= limit) { + cASSERT(mc, ki < nkeys); + if (unlikely(ki >= nkeys)) + goto sibling; + + const node_t *leaf = page_node(mp, ki); + pairs[n] = get_key(leaf); + rc = node_read(mc, leaf, &pairs[n + 1], mp); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + + n += 2; + if (++ki == nkeys) { + sibling: + rc = cursor_sibling_right(mc); + if (rc != MDBX_SUCCESS) { + if (rc == MDBX_NOTFOUND) + rc = MDBX_RESULT_TRUE; goto bailout; - memcpy(copy_dst, copy_src, lower); - copy_dst = ptr_disp(copy_dst, upper); - copy_src = ptr_disp(copy_src, upper); - copy_len -= upper; + } + + mp = mc->pg[mc->top]; + DEBUG("next page is %" PRIaPGNO ", key index %u", mp->pgno, mc->ki[mc->top]); + if (!MDBX_DISABLE_VALIDATION && unlikely(!check_leaf_type(mc, mp))) { + ERROR("unexpected leaf-page #%" PRIaPGNO " type 0x%x seen by cursor", mp->pgno, mp->flags); + rc = MDBX_CORRUPTED; + goto bailout; + } + nkeys = page_numkeys(mp); + ki = 0; } } - memcpy(copy_dst, copy_src, copy_len); - return; + mc->ki[mc->top] = (indx_t)ki; bailout: - if (src->mp_flags & P_LEAF2) - bad_page(src, "%s addr %p, n-keys %zu, ksize %u", - "invalid/corrupted source page", __Wpedantic_format_voidptr(src), - page_numkeys(src), src->mp_leaf2_ksize); - else - bad_page(src, "%s addr %p, upper %u", "invalid/corrupted source page", - __Wpedantic_format_voidptr(src), src->mp_upper); - memset(dst, -1, size); + *count = n; + return LOG_IFERR(rc); } -/* Pull a page off the txn's spill list, if present. - * - * If a page being referenced was spilled to disk in this txn, bring - * it back and make it dirty/writable again. */ -static pgr_t __must_check_result page_unspill(MDBX_txn *const txn, - const MDBX_page *const mp) { - VERBOSE("unspill page %" PRIaPGNO, mp->mp_pgno); - tASSERT(txn, (txn->mt_flags & MDBX_WRITEMAP) == 0); - tASSERT(txn, IS_SPILLED(txn, mp)); - const MDBX_txn *scan = txn; - pgr_t ret; - do { - tASSERT(txn, (scan->mt_flags & MDBX_TXN_SPILLS) != 0); - const size_t si = search_spilled(scan, mp->mp_pgno); - if (!si) - continue; - const unsigned npages = IS_OVERFLOW(mp) ? mp->mp_pages : 1; - ret.page = page_malloc(txn, npages); - if (unlikely(!ret.page)) { - ret.err = MDBX_ENOMEM; - return ret; - } - page_copy(ret.page, mp, pgno2bytes(txn->mt_env, npages)); - if (scan == txn) { - /* If in current txn, this page is no longer spilled. - * If it happens to be the last page, truncate the spill list. - * Otherwise mark it as deleted by setting the LSB. */ - spill_remove(txn, si, npages); - } /* otherwise, if belonging to a parent txn, the - * page remains spilled until child commits */ +/*----------------------------------------------------------------------------*/ - ret.err = page_dirty(txn, ret.page, npages); - if (unlikely(ret.err != MDBX_SUCCESS)) - return ret; -#if MDBX_ENABLE_PGOP_STAT - txn->mt_env->me_lck->mti_pgop_stat.unspill.weak += npages; -#endif /* MDBX_ENABLE_PGOP_STAT */ - ret.page->mp_flags |= (scan == txn) ? 0 : P_SPILLED; - ret.err = MDBX_SUCCESS; - return ret; - } while (likely((scan = scan->mt_parent) != nullptr && - (scan->mt_flags & MDBX_TXN_SPILLS) != 0)); - ERROR("Page %" PRIaPGNO " mod-txnid %" PRIaTXN - " not found in the spill-list(s), current txn %" PRIaTXN - " front %" PRIaTXN ", root txn %" PRIaTXN " front %" PRIaTXN, - mp->mp_pgno, mp->mp_txnid, txn->mt_txnid, txn->mt_front, - txn->mt_env->me_txn0->mt_txnid, txn->mt_env->me_txn0->mt_front); - ret.err = MDBX_PROBLEM; - ret.page = NULL; - return ret; +int mdbx_cursor_set_userctx(MDBX_cursor *mc, void *ctx) { + int rc = cursor_check(mc, 0); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); + + cursor_couple_t *couple = container_of(mc, cursor_couple_t, outer); + couple->userctx = ctx; + return MDBX_SUCCESS; } -/* Touch a page: make it dirty and re-insert into tree with updated pgno. - * Set MDBX_TXN_ERROR on failure. - * - * [in] mc cursor pointing to the page to be touched - * - * Returns 0 on success, non-zero on failure. */ -__hot static int page_touch(MDBX_cursor *mc) { - const MDBX_page *const mp = mc->mc_pg[mc->mc_top]; - MDBX_page *np; - MDBX_txn *txn = mc->mc_txn; - int rc; +void *mdbx_cursor_get_userctx(const MDBX_cursor *mc) { + if (unlikely(!mc)) + return nullptr; - tASSERT(txn, mc->mc_txn->mt_flags & MDBX_TXN_DIRTY); - tASSERT(txn, F_ISSET(*mc->mc_dbistate, DBI_DIRTY | DBI_VALID)); - tASSERT(txn, !IS_OVERFLOW(mp)); - if (ASSERT_ENABLED()) { - if (mc->mc_flags & C_SUB) { - MDBX_xcursor *mx = container_of(mc->mc_db, MDBX_xcursor, mx_db); - MDBX_cursor_couple *couple = container_of(mx, MDBX_cursor_couple, inner); - tASSERT(txn, mc->mc_db == &couple->outer.mc_xcursor->mx_db); - tASSERT(txn, mc->mc_dbx == &couple->outer.mc_xcursor->mx_dbx); - tASSERT(txn, *couple->outer.mc_dbistate & DBI_DIRTY); - } - tASSERT(txn, dirtylist_check(txn)); - } + if (unlikely(mc->signature != cur_signature_ready4dispose && mc->signature != cur_signature_live)) + return nullptr; - if (IS_MODIFIABLE(txn, mp)) { - if (!txn->tw.dirtylist) { - tASSERT(txn, (txn->mt_flags & MDBX_WRITEMAP) && !MDBX_AVOID_MSYNC); - return MDBX_SUCCESS; - } - if (IS_SUBP(mp)) - return MDBX_SUCCESS; - tASSERT(txn, (txn->mt_flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); - const size_t n = dpl_search(txn, mp->mp_pgno); - if (MDBX_AVOID_MSYNC && - unlikely(txn->tw.dirtylist->items[n].pgno != mp->mp_pgno)) { - tASSERT(txn, (txn->mt_flags & MDBX_WRITEMAP)); - tASSERT(txn, n > 0 && n <= txn->tw.dirtylist->length + 1); - VERBOSE("unspill page %" PRIaPGNO, mp->mp_pgno); - np = (MDBX_page *)mp; -#if MDBX_ENABLE_PGOP_STAT - txn->mt_env->me_lck->mti_pgop_stat.unspill.weak += 1; -#endif /* MDBX_ENABLE_PGOP_STAT */ - return page_dirty(txn, np, 1); - } - tASSERT(txn, n > 0 && n <= txn->tw.dirtylist->length); - tASSERT(txn, txn->tw.dirtylist->items[n].pgno == mp->mp_pgno && - txn->tw.dirtylist->items[n].ptr == mp); - if (!MDBX_AVOID_MSYNC || (txn->mt_flags & MDBX_WRITEMAP) == 0) { - size_t *const ptr = - ptr_disp(txn->tw.dirtylist->items[n].ptr, -(ptrdiff_t)sizeof(size_t)); - *ptr = txn->tw.dirtylru; - } - return MDBX_SUCCESS; - } - if (IS_SUBP(mp)) { - np = (MDBX_page *)mp; - np->mp_txnid = txn->mt_front; - return MDBX_SUCCESS; - } - tASSERT(txn, !IS_OVERFLOW(mp) && !IS_SUBP(mp)); + cursor_couple_t *couple = container_of(mc, cursor_couple_t, outer); + return couple->userctx; +} - if (IS_FROZEN(txn, mp)) { - /* CoW the page */ - rc = pnl_need(&txn->tw.retired_pages, 1); - if (unlikely(rc != MDBX_SUCCESS)) - goto fail; - const pgr_t par = page_alloc(mc); - rc = par.err; - np = par.page; - if (unlikely(rc != MDBX_SUCCESS)) - goto fail; +MDBX_txn *mdbx_cursor_txn(const MDBX_cursor *mc) { + if (unlikely(!mc || mc->signature != cur_signature_live)) + return nullptr; + MDBX_txn *txn = mc->txn; + if (unlikely(!txn || txn->signature != txn_signature || (txn->flags & MDBX_TXN_FINISHED))) + return nullptr; + return (txn->flags & MDBX_TXN_HAS_CHILD) ? txn->env->txn : txn; +} - const pgno_t pgno = np->mp_pgno; - DEBUG("touched db %d page %" PRIaPGNO " -> %" PRIaPGNO, DDBI(mc), - mp->mp_pgno, pgno); - tASSERT(txn, mp->mp_pgno != pgno); - pnl_xappend(txn->tw.retired_pages, mp->mp_pgno); - /* Update the parent page, if any, to point to the new page */ - if (mc->mc_top) { - MDBX_page *parent = mc->mc_pg[mc->mc_top - 1]; - MDBX_node *node = page_node(parent, mc->mc_ki[mc->mc_top - 1]); - node_set_pgno(node, pgno); - } else { - mc->mc_db->md_root = pgno; - } +MDBX_dbi mdbx_cursor_dbi(const MDBX_cursor *mc) { + if (unlikely(!mc || mc->signature != cur_signature_live)) + return UINT_MAX; + return cursor_dbi(mc); +} -#if MDBX_ENABLE_PGOP_STAT - txn->mt_env->me_lck->mti_pgop_stat.cow.weak += 1; -#endif /* MDBX_ENABLE_PGOP_STAT */ - page_copy(np, mp, txn->mt_env->me_psize); - np->mp_pgno = pgno; - np->mp_txnid = txn->mt_front; - } else if (IS_SPILLED(txn, mp)) { - pgr_t pur = page_unspill(txn, mp); - np = pur.page; - rc = pur.err; - if (likely(rc == MDBX_SUCCESS)) { - tASSERT(txn, np != nullptr); - goto done; - } - goto fail; - } else { - if (unlikely(!txn->mt_parent)) { - ERROR("Unexpected not frozen/modifiable/spilled but shadowed %s " - "page %" PRIaPGNO " mod-txnid %" PRIaTXN "," - " without parent transaction, current txn %" PRIaTXN - " front %" PRIaTXN, - IS_BRANCH(mp) ? "branch" : "leaf", mp->mp_pgno, mp->mp_txnid, - mc->mc_txn->mt_txnid, mc->mc_txn->mt_front); - rc = MDBX_PROBLEM; - goto fail; - } +/*----------------------------------------------------------------------------*/ - DEBUG("clone db %d page %" PRIaPGNO, DDBI(mc), mp->mp_pgno); - tASSERT(txn, - txn->tw.dirtylist->length <= MDBX_PGL_LIMIT + MDBX_PNL_GRANULATE); - /* No - copy it */ - np = page_malloc(txn, 1); - if (unlikely(!np)) { - rc = MDBX_ENOMEM; - goto fail; - } - page_copy(np, mp, txn->mt_env->me_psize); +int mdbx_cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, MDBX_put_flags_t flags) { + if (unlikely(key == nullptr || data == nullptr)) + return LOG_IFERR(MDBX_EINVAL); - /* insert a clone of parent's dirty page, so don't touch dirtyroom */ - rc = page_dirty(txn, np, 1); - if (unlikely(rc != MDBX_SUCCESS)) - goto fail; + int rc = cursor_check_rw(mc); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); -#if MDBX_ENABLE_PGOP_STAT - txn->mt_env->me_lck->mti_pgop_stat.clone.weak += 1; -#endif /* MDBX_ENABLE_PGOP_STAT */ + if (unlikely(flags & MDBX_MULTIPLE)) { + rc = cursor_check_multiple(mc, key, data, flags); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); } -done: - /* Adjust cursors pointing to mp */ - mc->mc_pg[mc->mc_top] = np; - MDBX_cursor *m2 = txn->mt_cursors[mc->mc_dbi]; - if (mc->mc_flags & C_SUB) { - for (; m2; m2 = m2->mc_next) { - MDBX_cursor *m3 = &m2->mc_xcursor->mx_cursor; - if (m3->mc_snum < mc->mc_snum) - continue; - if (m3->mc_pg[mc->mc_top] == mp) - m3->mc_pg[mc->mc_top] = np; - } - } else { - for (; m2; m2 = m2->mc_next) { - if (m2->mc_snum < mc->mc_snum) - continue; - if (m2 == mc) - continue; - if (m2->mc_pg[mc->mc_top] == mp) { - m2->mc_pg[mc->mc_top] = np; - if (XCURSOR_INITED(m2) && IS_LEAF(np)) - XCURSOR_REFRESH(m2, np, m2->mc_ki[mc->mc_top]); - } - } + if (flags & MDBX_RESERVE) { + if (unlikely(mc->tree->flags & (MDBX_DUPSORT | MDBX_REVERSEDUP | MDBX_INTEGERDUP | MDBX_DUPFIXED))) + return LOG_IFERR(MDBX_INCOMPATIBLE); + data->iov_base = nullptr; } - return MDBX_SUCCESS; -fail: - txn->mt_flags |= MDBX_TXN_ERROR; - return rc; + return LOG_IFERR(cursor_put_checklen(mc, key, data, flags)); } -static int meta_sync(const MDBX_env *env, const meta_ptr_t head) { - eASSERT(env, atomic_load32(&env->me_lck->mti_meta_sync_txnid, mo_Relaxed) != - (uint32_t)head.txnid); - /* Функция может вызываться (в том числе) при (env->me_flags & - * MDBX_NOMETASYNC) == 0 и env->me_fd4meta == env->me_dsync_fd, например если - * предыдущая транзакция была выполненна с флагом MDBX_NOMETASYNC. */ +int mdbx_cursor_del(MDBX_cursor *mc, MDBX_put_flags_t flags) { + int rc = cursor_check_rw(mc); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); - int rc = MDBX_RESULT_TRUE; - if (env->me_flags & MDBX_WRITEMAP) { - if (!MDBX_AVOID_MSYNC) { - rc = osal_msync(&env->me_dxb_mmap, 0, pgno_align2os_bytes(env, NUM_METAS), - MDBX_SYNC_DATA | MDBX_SYNC_IODQ); -#if MDBX_ENABLE_PGOP_STAT - env->me_lck->mti_pgop_stat.msync.weak += 1; -#endif /* MDBX_ENABLE_PGOP_STAT */ - } else { -#if MDBX_ENABLE_PGOP_ST - env->me_lck->mti_pgop_stat.wops.weak += 1; -#endif /* MDBX_ENABLE_PGOP_STAT */ - const MDBX_page *page = data_page(head.ptr_c); - rc = osal_pwrite(env->me_fd4meta, page, env->me_psize, - ptr_dist(page, env->me_map)); + return LOG_IFERR(cursor_del(mc, flags)); +} - if (likely(rc == MDBX_SUCCESS) && env->me_fd4meta == env->me_lazy_fd) { - rc = osal_fsync(env->me_lazy_fd, MDBX_SYNC_DATA | MDBX_SYNC_IODQ); -#if MDBX_ENABLE_PGOP_STAT - env->me_lck->mti_pgop_stat.fsync.weak += 1; -#endif /* MDBX_ENABLE_PGOP_STAT */ - } - } - } else { - rc = osal_fsync(env->me_lazy_fd, MDBX_SYNC_DATA | MDBX_SYNC_IODQ); -#if MDBX_ENABLE_PGOP_STAT - env->me_lck->mti_pgop_stat.fsync.weak += 1; -#endif /* MDBX_ENABLE_PGOP_STAT */ - } +__cold int mdbx_cursor_ignord(MDBX_cursor *mc) { + int rc = cursor_check(mc, 0); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); - if (likely(rc == MDBX_SUCCESS)) - env->me_lck->mti_meta_sync_txnid.weak = (uint32_t)head.txnid; - return rc; + mc->checking |= z_ignord; + if (mc->subcur) + mc->subcur->cursor.checking |= z_ignord; + + return MDBX_SUCCESS; } +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 -__cold static int env_sync(MDBX_env *env, bool force, bool nonblock) { - bool locked = false; - int rc = MDBX_RESULT_TRUE /* means "nothing to sync" */; +int mdbx_dbi_open2(MDBX_txn *txn, const MDBX_val *name, MDBX_db_flags_t flags, MDBX_dbi *dbi) { + return LOG_IFERR(dbi_open(txn, name, flags, dbi, nullptr, nullptr)); +} -retry:; - unsigned flags = env->me_flags & ~(MDBX_NOMETASYNC | MDBX_SHRINK_ALLOWED); - if (unlikely((flags & (MDBX_RDONLY | MDBX_FATAL_ERROR | MDBX_ENV_ACTIVE)) != - MDBX_ENV_ACTIVE)) { - rc = MDBX_EACCESS; - if (!(flags & MDBX_ENV_ACTIVE)) - rc = MDBX_EPERM; - if (flags & MDBX_FATAL_ERROR) - rc = MDBX_PANIC; - goto bailout; - } +int mdbx_dbi_open_ex2(MDBX_txn *txn, const MDBX_val *name, MDBX_db_flags_t flags, MDBX_dbi *dbi, MDBX_cmp_func *keycmp, + MDBX_cmp_func *datacmp) { + return LOG_IFERR(dbi_open(txn, name, flags, dbi, keycmp, datacmp)); +} - const bool inside_txn = (env->me_txn0->mt_owner == osal_thread_self()); - const meta_troika_t troika = - (inside_txn | locked) ? env->me_txn0->tw.troika : meta_tap(env); - const meta_ptr_t head = meta_recent(env, &troika); - const uint64_t unsynced_pages = - atomic_load64(&env->me_lck->mti_unsynced_pages, mo_Relaxed); - if (unsynced_pages == 0) { - const uint32_t synched_meta_txnid_u32 = - atomic_load32(&env->me_lck->mti_meta_sync_txnid, mo_Relaxed); - if (synched_meta_txnid_u32 == (uint32_t)head.txnid && head.is_steady) - goto bailout; +static int dbi_open_cstr(MDBX_txn *txn, const char *name_cstr, MDBX_db_flags_t flags, MDBX_dbi *dbi, + MDBX_cmp_func *keycmp, MDBX_cmp_func *datacmp) { + MDBX_val thunk, *name; + if (name_cstr == MDBX_CHK_MAIN || name_cstr == MDBX_CHK_GC || name_cstr == MDBX_CHK_META) + name = (void *)name_cstr; + else { + thunk.iov_len = strlen(name_cstr); + thunk.iov_base = (void *)name_cstr; + name = &thunk; } + return dbi_open(txn, name, flags, dbi, keycmp, datacmp); +} - if (!inside_txn && locked && (env->me_flags & MDBX_WRITEMAP) && - unlikely(head.ptr_c->mm_geo.next > - bytes2pgno(env, env->me_dxb_mmap.current))) { - - if (unlikely(env->me_stuck_meta >= 0) && - troika.recent != (uint8_t)env->me_stuck_meta) { - NOTICE("skip %s since wagering meta-page (%u) is mispatch the recent " - "meta-page (%u)", - "sync datafile", env->me_stuck_meta, troika.recent); - rc = MDBX_RESULT_TRUE; - } else { - rc = dxb_resize(env, head.ptr_c->mm_geo.next, head.ptr_c->mm_geo.now, - head.ptr_c->mm_geo.upper, implicit_grow); - if (unlikely(rc != MDBX_SUCCESS)) - goto bailout; - } - } +int mdbx_dbi_open(MDBX_txn *txn, const char *name, MDBX_db_flags_t flags, MDBX_dbi *dbi) { + return LOG_IFERR(dbi_open_cstr(txn, name, flags, dbi, nullptr, nullptr)); +} - const size_t autosync_threshold = - atomic_load32(&env->me_lck->mti_autosync_threshold, mo_Relaxed); - const uint64_t autosync_period = - atomic_load64(&env->me_lck->mti_autosync_period, mo_Relaxed); - uint64_t eoos_timestamp; - if (force || (autosync_threshold && unsynced_pages >= autosync_threshold) || - (autosync_period && - (eoos_timestamp = - atomic_load64(&env->me_lck->mti_eoos_timestamp, mo_Relaxed)) && - osal_monotime() - eoos_timestamp >= autosync_period)) - flags &= MDBX_WRITEMAP /* clear flags for full steady sync */; +int mdbx_dbi_open_ex(MDBX_txn *txn, const char *name, MDBX_db_flags_t flags, MDBX_dbi *dbi, MDBX_cmp_func *keycmp, + MDBX_cmp_func *datacmp) { + return LOG_IFERR(dbi_open_cstr(txn, name, flags, dbi, keycmp, datacmp)); +} - if (!inside_txn) { - if (!locked) { -#if MDBX_ENABLE_PGOP_STAT - unsigned wops = 0; -#endif /* MDBX_ENABLE_PGOP_STAT */ +__cold int mdbx_drop(MDBX_txn *txn, MDBX_dbi dbi, bool del) { + int rc = check_txn_rw(txn, MDBX_TXN_BLOCKED); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); - int err; - /* pre-sync to avoid latency for writer */ - if (unsynced_pages > /* FIXME: define threshold */ 42 && - (flags & MDBX_SAFE_NOSYNC) == 0) { - eASSERT(env, ((flags ^ env->me_flags) & MDBX_WRITEMAP) == 0); - if (flags & MDBX_WRITEMAP) { - /* Acquire guard to avoid collision with remap */ -#if defined(_WIN32) || defined(_WIN64) - osal_srwlock_AcquireShared(&env->me_remap_guard); -#else - err = osal_fastmutex_acquire(&env->me_remap_guard); - if (unlikely(err != MDBX_SUCCESS)) - return err; -#endif - const size_t usedbytes = - pgno_align2os_bytes(env, head.ptr_c->mm_geo.next); - err = osal_msync(&env->me_dxb_mmap, 0, usedbytes, MDBX_SYNC_DATA); -#if defined(_WIN32) || defined(_WIN64) - osal_srwlock_ReleaseShared(&env->me_remap_guard); -#else - int unlock_err = osal_fastmutex_release(&env->me_remap_guard); - if (unlikely(unlock_err != MDBX_SUCCESS) && err == MDBX_SUCCESS) - err = unlock_err; -#endif - } else - err = osal_fsync(env->me_lazy_fd, MDBX_SYNC_DATA); + cursor_couple_t cx; + rc = cursor_init(&cx.outer, txn, dbi); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); - if (unlikely(err != MDBX_SUCCESS)) - return err; + if (txn->dbs[dbi].height) { + cx.outer.next = txn->cursors[dbi]; + txn->cursors[dbi] = &cx.outer; + rc = tree_drop(&cx.outer, dbi == MAIN_DBI || (cx.outer.tree->flags & MDBX_DUPSORT)); + txn->cursors[dbi] = cx.outer.next; + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); + } -#if MDBX_ENABLE_PGOP_STAT - wops = 1; -#endif /* MDBX_ENABLE_PGOP_STAT */ - /* pre-sync done */ - rc = MDBX_SUCCESS /* means "some data was synced" */; - } + /* Invalidate the dropped DB's cursors */ + for (MDBX_cursor *mc = txn->cursors[dbi]; mc; mc = mc->next) + be_poor(mc); - err = mdbx_txn_lock(env, nonblock); - if (unlikely(err != MDBX_SUCCESS)) - return err; + if (!del || dbi < CORE_DBS) { + /* reset the DB record, mark it dirty */ + txn->dbi_state[dbi] |= DBI_DIRTY; + txn->dbs[dbi].height = 0; + txn->dbs[dbi].branch_pages = 0; + txn->dbs[dbi].leaf_pages = 0; + txn->dbs[dbi].large_pages = 0; + txn->dbs[dbi].items = 0; + txn->dbs[dbi].root = P_INVALID; + txn->dbs[dbi].sequence = 0; + /* txn->dbs[dbi].mod_txnid = txn->txnid; */ + txn->flags |= MDBX_TXN_DIRTY; + return MDBX_SUCCESS; + } - locked = true; -#if MDBX_ENABLE_PGOP_STAT - env->me_lck->mti_pgop_stat.wops.weak += wops; -#endif /* MDBX_ENABLE_PGOP_STAT */ - env->me_txn0->tw.troika = meta_tap(env); - eASSERT(env, !env->me_txn && !env->me_txn0->mt_child); - goto retry; + MDBX_env *const env = txn->env; + MDBX_val name = env->kvs[dbi].name; + rc = cursor_init(&cx.outer, txn, MAIN_DBI); + if (likely(rc == MDBX_SUCCESS)) { + rc = cursor_seek(&cx.outer, &name, nullptr, MDBX_SET).err; + if (likely(rc == MDBX_SUCCESS)) { + cx.outer.next = txn->cursors[MAIN_DBI]; + txn->cursors[MAIN_DBI] = &cx.outer; + rc = cursor_del(&cx.outer, N_TREE); + txn->cursors[MAIN_DBI] = cx.outer.next; + if (likely(rc == MDBX_SUCCESS)) { + tASSERT(txn, txn->dbi_state[MAIN_DBI] & DBI_DIRTY); + tASSERT(txn, txn->flags & MDBX_TXN_DIRTY); + txn->dbi_state[dbi] = DBI_LINDO | DBI_OLDEN; + rc = osal_fastmutex_acquire(&env->dbi_lock); + if (likely(rc == MDBX_SUCCESS)) + return LOG_IFERR(dbi_close_release(env, dbi)); + } } - eASSERT(env, head.txnid == recent_committed_txnid(env)); - env->me_txn0->mt_txnid = head.txnid; - txn_oldest_reader(env->me_txn0); - flags |= MDBX_SHRINK_ALLOWED; } - eASSERT(env, inside_txn || locked); - eASSERT(env, !inside_txn || (flags & MDBX_SHRINK_ALLOWED) == 0); + txn->flags |= MDBX_TXN_ERROR; + return LOG_IFERR(rc); +} - if (!head.is_steady && unlikely(env->me_stuck_meta >= 0) && - troika.recent != (uint8_t)env->me_stuck_meta) { - NOTICE("skip %s since wagering meta-page (%u) is mispatch the recent " - "meta-page (%u)", - "sync datafile", env->me_stuck_meta, troika.recent); - rc = MDBX_RESULT_TRUE; - goto bailout; - } - if (!head.is_steady || ((flags & MDBX_SAFE_NOSYNC) == 0 && unsynced_pages)) { - DEBUG("meta-head %" PRIaPGNO ", %s, sync_pending %" PRIu64, - data_page(head.ptr_c)->mp_pgno, durable_caption(head.ptr_c), - unsynced_pages); - MDBX_meta meta = *head.ptr_c; - rc = sync_locked(env, flags, &meta, &env->me_txn0->tw.troika); - if (unlikely(rc != MDBX_SUCCESS)) - goto bailout; +__cold int mdbx_dbi_rename(MDBX_txn *txn, MDBX_dbi dbi, const char *name_cstr) { + MDBX_val thunk, *name; + if (name_cstr == MDBX_CHK_MAIN || name_cstr == MDBX_CHK_GC || name_cstr == MDBX_CHK_META) + name = (void *)name_cstr; + else { + thunk.iov_len = strlen(name_cstr); + thunk.iov_base = (void *)name_cstr; + name = &thunk; } - - /* LY: sync meta-pages if MDBX_NOMETASYNC enabled - * and someone was not synced above. */ - if (atomic_load32(&env->me_lck->mti_meta_sync_txnid, mo_Relaxed) != - (uint32_t)head.txnid) - rc = meta_sync(env, head); - -bailout: - if (locked) - mdbx_txn_unlock(env); - return rc; + return mdbx_dbi_rename2(txn, dbi, name); } -static __inline int check_env(const MDBX_env *env, const bool wanna_active) { - if (unlikely(!env)) - return MDBX_EINVAL; +__cold int mdbx_dbi_rename2(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *new_name) { + int rc = check_txn_rw(txn, MDBX_TXN_BLOCKED); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); - if (unlikely(env->me_signature.weak != MDBX_ME_SIGNATURE)) - return MDBX_EBADSIGN; + if (unlikely(new_name == MDBX_CHK_MAIN || new_name->iov_base == MDBX_CHK_MAIN || new_name == MDBX_CHK_GC || + new_name->iov_base == MDBX_CHK_GC || new_name == MDBX_CHK_META || new_name->iov_base == MDBX_CHK_META)) + return LOG_IFERR(MDBX_EINVAL); - if (unlikely(env->me_flags & MDBX_FATAL_ERROR)) - return MDBX_PANIC; + if (unlikely(dbi < CORE_DBS)) + return LOG_IFERR(MDBX_EINVAL); + rc = dbi_check(txn, dbi); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); - if (wanna_active) { -#if MDBX_ENV_CHECKPID - if (unlikely(env->me_pid != osal_getpid()) && env->me_pid) { - ((MDBX_env *)env)->me_flags |= MDBX_FATAL_ERROR; - return MDBX_PANIC; - } -#endif /* MDBX_ENV_CHECKPID */ - if (unlikely((env->me_flags & MDBX_ENV_ACTIVE) == 0)) - return MDBX_EPERM; - eASSERT(env, env->me_map != nullptr); + rc = osal_fastmutex_acquire(&txn->env->dbi_lock); + if (likely(rc == MDBX_SUCCESS)) { + struct dbi_rename_result pair = dbi_rename_locked(txn, dbi, *new_name); + if (pair.defer) + pair.defer->next = nullptr; + dbi_defer_release(txn->env, pair.defer); + rc = pair.err; } - - return MDBX_SUCCESS; + return LOG_IFERR(rc); } -__cold int mdbx_env_sync_ex(MDBX_env *env, bool force, bool nonblock) { +int mdbx_dbi_close(MDBX_env *env, MDBX_dbi dbi) { int rc = check_env(env, true); if (unlikely(rc != MDBX_SUCCESS)) - return rc; + return LOG_IFERR(rc); - return env_sync(env, force, nonblock); -} - -/* Back up parent txn's cursors, then grab the originals for tracking */ -static int cursor_shadow(MDBX_txn *parent, MDBX_txn *nested) { - tASSERT(parent, parent->mt_cursors[FREE_DBI] == nullptr); - nested->mt_cursors[FREE_DBI] = nullptr; - for (int i = parent->mt_numdbs; --i > FREE_DBI;) { - nested->mt_cursors[i] = NULL; - MDBX_cursor *mc = parent->mt_cursors[i]; - if (mc != NULL) { - size_t size = mc->mc_xcursor ? sizeof(MDBX_cursor) + sizeof(MDBX_xcursor) - : sizeof(MDBX_cursor); - for (MDBX_cursor *bk; mc; mc = bk->mc_next) { - bk = mc; - if (mc->mc_signature != MDBX_MC_LIVE) - continue; - bk = osal_malloc(size); - if (unlikely(!bk)) - return MDBX_ENOMEM; -#if MDBX_DEBUG - memset(bk, 0xCD, size); - VALGRIND_MAKE_MEM_UNDEFINED(bk, size); -#endif /* MDBX_DEBUG */ - *bk = *mc; - mc->mc_backup = bk; - /* Kill pointers into src to reduce abuse: The - * user may not use mc until dst ends. But we need a valid - * txn pointer here for cursor fixups to keep working. */ - mc->mc_txn = nested; - mc->mc_db = &nested->mt_dbs[i]; - mc->mc_dbistate = &nested->mt_dbistate[i]; - MDBX_xcursor *mx = mc->mc_xcursor; - if (mx != NULL) { - *(MDBX_xcursor *)(bk + 1) = *mx; - mx->mx_cursor.mc_txn = mc->mc_txn; - mx->mx_cursor.mc_dbistate = mc->mc_dbistate; - } - mc->mc_next = nested->mt_cursors[i]; - nested->mt_cursors[i] = mc; - } - } - } - return MDBX_SUCCESS; -} - -/* Close this txn's cursors, give parent txn's cursors back to parent. - * - * [in] txn the transaction handle. - * [in] merge true to keep changes to parent cursors, false to revert. - * - * Returns 0 on success, non-zero on failure. */ -static void cursors_eot(MDBX_txn *txn, const bool merge) { - tASSERT(txn, txn->mt_cursors[FREE_DBI] == nullptr); - for (intptr_t i = txn->mt_numdbs; --i > FREE_DBI;) { - MDBX_cursor *mc = txn->mt_cursors[i]; - if (!mc) - continue; - txn->mt_cursors[i] = nullptr; - do { - const unsigned stage = mc->mc_signature; - MDBX_cursor *const next = mc->mc_next; - MDBX_cursor *const bk = mc->mc_backup; - ENSURE(txn->mt_env, - stage == MDBX_MC_LIVE || (stage == MDBX_MC_WAIT4EOT && bk)); - cASSERT(mc, mc->mc_dbi == (MDBX_dbi)i); - if (bk) { - MDBX_xcursor *mx = mc->mc_xcursor; - tASSERT(txn, txn->mt_parent != NULL); - /* Zap: Using uninitialized memory '*mc->mc_backup'. */ - MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(6001); - ENSURE(txn->mt_env, bk->mc_signature == MDBX_MC_LIVE); - tASSERT(txn, mx == bk->mc_xcursor); - if (merge) { - /* Restore pointers to parent txn */ - mc->mc_next = bk->mc_next; - mc->mc_backup = bk->mc_backup; - mc->mc_txn = bk->mc_txn; - mc->mc_db = bk->mc_db; - mc->mc_dbistate = bk->mc_dbistate; - if (mx) { - mx->mx_cursor.mc_txn = mc->mc_txn; - mx->mx_cursor.mc_dbistate = mc->mc_dbistate; - } - } else { - /* Restore from backup, i.e. rollback/abort nested txn */ - *mc = *bk; - if (mx) - *mx = *(MDBX_xcursor *)(bk + 1); - } - bk->mc_signature = 0; - osal_free(bk); - if (stage == MDBX_MC_WAIT4EOT /* Cursor was closed by user */) - mc->mc_signature = stage /* Promote closed state to parent txn */; - } else { - ENSURE(txn->mt_env, stage == MDBX_MC_LIVE); - mc->mc_signature = MDBX_MC_READY4CLOSE /* Cursor may be reused */; - mc->mc_flags = 0 /* reset C_UNTRACK */; - } - mc = next; - } while (mc); - } -} - -#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__) -/* Find largest mvcc-snapshot still referenced by this process. */ -static pgno_t find_largest_this(MDBX_env *env, pgno_t largest) { - MDBX_lockinfo *const lck = env->me_lck_mmap.lck; - if (likely(lck != NULL /* exclusive mode */)) { - const size_t snap_nreaders = - atomic_load32(&lck->mti_numreaders, mo_AcquireRelease); - for (size_t i = 0; i < snap_nreaders; ++i) { - retry: - if (atomic_load32(&lck->mti_readers[i].mr_pid, mo_AcquireRelease) == - env->me_pid) { - /* jitter4testing(true); */ - const pgno_t snap_pages = atomic_load32( - &lck->mti_readers[i].mr_snapshot_pages_used, mo_Relaxed); - const txnid_t snap_txnid = safe64_read(&lck->mti_readers[i].mr_txnid); - if (unlikely( - snap_pages != - atomic_load32(&lck->mti_readers[i].mr_snapshot_pages_used, - mo_AcquireRelease) || - snap_txnid != safe64_read(&lck->mti_readers[i].mr_txnid))) - goto retry; - if (largest < snap_pages && - atomic_load64(&lck->mti_oldest_reader, mo_AcquireRelease) <= - /* ignore pending updates */ snap_txnid && - snap_txnid <= MAX_TXNID) - largest = snap_pages; - } - } - } - return largest; -} - -static void txn_valgrind(MDBX_env *env, MDBX_txn *txn) { -#if !defined(__SANITIZE_ADDRESS__) - if (!RUNNING_ON_VALGRIND) - return; -#endif - - if (txn) { /* transaction start */ - if (env->me_poison_edge < txn->mt_next_pgno) - env->me_poison_edge = txn->mt_next_pgno; - VALGRIND_MAKE_MEM_DEFINED(env->me_map, pgno2bytes(env, txn->mt_next_pgno)); - MDBX_ASAN_UNPOISON_MEMORY_REGION(env->me_map, - pgno2bytes(env, txn->mt_next_pgno)); - /* don't touch more, it should be already poisoned */ - } else { /* transaction end */ - bool should_unlock = false; - pgno_t last = MAX_PAGENO + 1; - if (env->me_txn0 && env->me_txn0->mt_owner == osal_thread_self()) { - /* inside write-txn */ - last = meta_recent(env, &env->me_txn0->tw.troika).ptr_v->mm_geo.next; - } else if (env->me_flags & MDBX_RDONLY) { - /* read-only mode, no write-txn, no wlock mutex */ - last = NUM_METAS; - } else if (mdbx_txn_lock(env, true) == MDBX_SUCCESS) { - /* no write-txn */ - last = NUM_METAS; - should_unlock = true; - } else { - /* write txn is running, therefore shouldn't poison any memory range */ - return; - } - - last = find_largest_this(env, last); - const pgno_t edge = env->me_poison_edge; - if (edge > last) { - eASSERT(env, last >= NUM_METAS); - env->me_poison_edge = last; - VALGRIND_MAKE_MEM_NOACCESS(ptr_disp(env->me_map, pgno2bytes(env, last)), - pgno2bytes(env, edge - last)); - MDBX_ASAN_POISON_MEMORY_REGION( - ptr_disp(env->me_map, pgno2bytes(env, last)), - pgno2bytes(env, edge - last)); - } - if (should_unlock) - mdbx_txn_unlock(env); - } -} -#endif /* MDBX_USE_VALGRIND || __SANITIZE_ADDRESS__ */ + if (unlikely(dbi < CORE_DBS)) + return (dbi == MAIN_DBI) ? MDBX_SUCCESS : LOG_IFERR(MDBX_BAD_DBI); -typedef struct { - int err; - MDBX_reader *rslot; -} bind_rslot_result; + if (unlikely(dbi >= env->max_dbi)) + return LOG_IFERR(MDBX_BAD_DBI); -static bind_rslot_result bind_rslot(MDBX_env *env, const uintptr_t tid) { - eASSERT(env, env->me_lck_mmap.lck); - eASSERT(env, env->me_lck->mti_magic_and_version == MDBX_LOCK_MAGIC); - eASSERT(env, env->me_lck->mti_os_and_format == MDBX_LOCK_FORMAT); + rc = osal_fastmutex_acquire(&env->dbi_lock); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); - bind_rslot_result result = {osal_rdt_lock(env), nullptr}; - if (unlikely(MDBX_IS_ERROR(result.err))) - return result; - if (unlikely(env->me_flags & MDBX_FATAL_ERROR)) { - osal_rdt_unlock(env); - result.err = MDBX_PANIC; - return result; - } - if (unlikely(!env->me_map)) { - osal_rdt_unlock(env); - result.err = MDBX_EPERM; - return result; + if (unlikely(dbi >= env->n_dbi)) { + rc = MDBX_BAD_DBI; + bailout: + osal_fastmutex_release(&env->dbi_lock); + return LOG_IFERR(rc); } - if (unlikely(env->me_live_reader != env->me_pid)) { - result.err = osal_rpid_set(env); - if (unlikely(result.err != MDBX_SUCCESS)) { - osal_rdt_unlock(env); - return result; + while (env->basal_txn && (env->dbs_flags[dbi] & DB_VALID) && (env->basal_txn->flags & MDBX_TXN_FINISHED) == 0) { + /* LY: Опасный код, так как env->txn может быть изменено в другом потоке. + * К сожалению тут нет надежного решения и может быть падение при неверном + * использовании API (вызове mdbx_dbi_close конкурентно с завершением + * пишущей транзакции). + * + * Для минимизации вероятности падения сначала проверяем dbi-флаги + * в basal_txn, а уже после в env->txn. Таким образом, падение может быть + * только при коллизии с завершением вложенной транзакции. + * + * Альтернативно можно попробовать выполнять обновление/put записи в + * mainDb соответствующей таблице закрываемого хендла. Семантически это + * верный путь, но проблема в текущем API, в котором исторически dbi-хендл + * живет и закрывается вне транзакции. Причем проблема не только в том, + * что нет указателя на текущую пишущую транзакцию, а в том что + * пользователь точно не ожидает что закрытие хендла приведет к + * скрытой/непрозрачной активности внутри транзакции потенциально + * выполняемой в другом потоке. Другими словами, проблема может быть + * только при неверном использовании API и если пользователь это + * допускает, то точно не будет ожидать скрытых действий внутри + * транзакции, и поэтому этот путь потенциально более опасен. */ + const MDBX_txn *const hazard = env->txn; + osal_compiler_barrier(); + if ((dbi_state(env->basal_txn, dbi) & (DBI_LINDO | DBI_DIRTY | DBI_CREAT)) > DBI_LINDO) { + rc = MDBX_DANGLING_DBI; + goto bailout; } - env->me_live_reader = env->me_pid; - } - - result.err = MDBX_SUCCESS; - size_t slot, nreaders; - while (1) { - nreaders = env->me_lck->mti_numreaders.weak; - for (slot = 0; slot < nreaders; slot++) - if (!atomic_load32(&env->me_lck->mti_readers[slot].mr_pid, - mo_AcquireRelease)) - break; - - if (likely(slot < env->me_maxreaders)) - break; - - result.err = cleanup_dead_readers(env, true, NULL); - if (result.err != MDBX_RESULT_TRUE) { - osal_rdt_unlock(env); - result.err = - (result.err == MDBX_SUCCESS) ? MDBX_READERS_FULL : result.err; - return result; + osal_memory_barrier(); + if (unlikely(hazard != env->txn)) + continue; + if (hazard != env->basal_txn && hazard && (hazard->flags & MDBX_TXN_FINISHED) == 0 && + hazard->signature == txn_signature && + (dbi_state(hazard, dbi) & (DBI_LINDO | DBI_DIRTY | DBI_CREAT)) > DBI_LINDO) { + rc = MDBX_DANGLING_DBI; + goto bailout; } + osal_compiler_barrier(); + if (likely(hazard == env->txn)) + break; } - - result.rslot = &env->me_lck->mti_readers[slot]; - /* Claim the reader slot, carefully since other code - * uses the reader table un-mutexed: First reset the - * slot, next publish it in lck->mti_numreaders. After - * that, it is safe for mdbx_env_close() to touch it. - * When it will be closed, we can finally claim it. */ - atomic_store32(&result.rslot->mr_pid, 0, mo_AcquireRelease); - safe64_reset(&result.rslot->mr_txnid, true); - if (slot == nreaders) - env->me_lck->mti_numreaders.weak = (uint32_t)++nreaders; - result.rslot->mr_tid.weak = (env->me_flags & MDBX_NOTLS) ? 0 : tid; - atomic_store32(&result.rslot->mr_pid, env->me_pid, mo_AcquireRelease); - osal_rdt_unlock(env); - - if (likely(env->me_flags & MDBX_ENV_TXKEY)) { - eASSERT(env, env->me_live_reader == env->me_pid); - thread_rthc_set(env->me_txkey, result.rslot); - } - return result; + rc = dbi_close_release(env, dbi); + return LOG_IFERR(rc); } -__cold int mdbx_thread_register(const MDBX_env *env) { - int rc = check_env(env, true); +int mdbx_dbi_flags_ex(const MDBX_txn *txn, MDBX_dbi dbi, unsigned *flags, unsigned *state) { + int rc = check_txn(txn, MDBX_TXN_BLOCKED - MDBX_TXN_ERROR - MDBX_TXN_PARKED); if (unlikely(rc != MDBX_SUCCESS)) - return rc; + return LOG_IFERR(rc); - if (unlikely(!env->me_lck_mmap.lck)) - return (env->me_flags & MDBX_EXCLUSIVE) ? MDBX_EINVAL : MDBX_EPERM; + rc = dbi_check(txn, dbi); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); - if (unlikely((env->me_flags & MDBX_ENV_TXKEY) == 0)) { - eASSERT(env, !env->me_lck_mmap.lck || (env->me_flags & MDBX_NOTLS)); - return MDBX_EINVAL /* MDBX_NOTLS mode */; - } + if (unlikely(!flags || !state)) + return LOG_IFERR(MDBX_EINVAL); - eASSERT(env, (env->me_flags & (MDBX_NOTLS | MDBX_ENV_TXKEY | - MDBX_EXCLUSIVE)) == MDBX_ENV_TXKEY); - MDBX_reader *r = thread_rthc_get(env->me_txkey); - if (unlikely(r != NULL)) { - eASSERT(env, r->mr_pid.weak == env->me_pid); - eASSERT(env, r->mr_tid.weak == osal_thread_self()); - if (unlikely(r->mr_pid.weak != env->me_pid)) - return MDBX_BAD_RSLOT; - return MDBX_RESULT_TRUE /* already registered */; - } + *flags = txn->dbs[dbi].flags & DB_PERSISTENT_FLAGS; + *state = txn->dbi_state[dbi] & (DBI_FRESH | DBI_CREAT | DBI_DIRTY | DBI_STALE); + return MDBX_SUCCESS; +} - const uintptr_t tid = osal_thread_self(); - if (env->me_txn0 && unlikely(env->me_txn0->mt_owner == tid)) - return MDBX_TXN_OVERLAPPING; - return bind_rslot((MDBX_env *)env, tid).err; +static void stat_get(const tree_t *db, MDBX_stat *st, size_t bytes) { + st->ms_depth = db->height; + st->ms_branch_pages = db->branch_pages; + st->ms_leaf_pages = db->leaf_pages; + st->ms_overflow_pages = db->large_pages; + st->ms_entries = db->items; + if (likely(bytes >= offsetof(MDBX_stat, ms_mod_txnid) + sizeof(st->ms_mod_txnid))) + st->ms_mod_txnid = db->mod_txnid; } -__cold int mdbx_thread_unregister(const MDBX_env *env) { - int rc = check_env(env, true); +__cold int mdbx_dbi_stat(const MDBX_txn *txn, MDBX_dbi dbi, MDBX_stat *dest, size_t bytes) { + int rc = check_txn(txn, MDBX_TXN_BLOCKED); if (unlikely(rc != MDBX_SUCCESS)) - return rc; + return LOG_IFERR(rc); - if (unlikely(!env->me_lck_mmap.lck)) - return MDBX_RESULT_TRUE; + rc = dbi_check(txn, dbi); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); - if (unlikely((env->me_flags & MDBX_ENV_TXKEY) == 0)) { - eASSERT(env, !env->me_lck_mmap.lck || (env->me_flags & MDBX_NOTLS)); - return MDBX_RESULT_TRUE /* MDBX_NOTLS mode */; - } + if (unlikely(txn->flags & MDBX_TXN_BLOCKED)) + return LOG_IFERR(MDBX_BAD_TXN); - eASSERT(env, (env->me_flags & (MDBX_NOTLS | MDBX_ENV_TXKEY | - MDBX_EXCLUSIVE)) == MDBX_ENV_TXKEY); - MDBX_reader *r = thread_rthc_get(env->me_txkey); - if (unlikely(r == NULL)) - return MDBX_RESULT_TRUE /* not registered */; + if (unlikely(txn->dbi_state[dbi] & DBI_STALE)) { + rc = tbl_fetch((MDBX_txn *)txn, dbi); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); + } - eASSERT(env, r->mr_pid.weak == env->me_pid); - eASSERT(env, r->mr_tid.weak == osal_thread_self()); - if (unlikely(r->mr_pid.weak != env->me_pid || - r->mr_tid.weak != osal_thread_self())) - return MDBX_BAD_RSLOT; + if (unlikely(!dest)) + return LOG_IFERR(MDBX_EINVAL); - eASSERT(env, r->mr_txnid.weak >= SAFE64_INVALID_THRESHOLD); - if (unlikely(r->mr_txnid.weak < SAFE64_INVALID_THRESHOLD)) - return MDBX_BUSY /* transaction is still active */; + const size_t size_before_modtxnid = offsetof(MDBX_stat, ms_mod_txnid); + if (unlikely(bytes != sizeof(MDBX_stat)) && bytes != size_before_modtxnid) + return LOG_IFERR(MDBX_EINVAL); - atomic_store32(&r->mr_pid, 0, mo_Relaxed); - atomic_store32(&env->me_lck->mti_readers_refresh_flag, true, - mo_AcquireRelease); - thread_rthc_set(env->me_txkey, nullptr); + dest->ms_psize = txn->env->ps; + stat_get(&txn->dbs[dbi], dest, bytes); return MDBX_SUCCESS; } -/* check against https://libmdbx.dqdkfa.ru/dead-github/issues/269 */ -static bool coherency_check(const MDBX_env *env, const txnid_t txnid, - const volatile MDBX_db *dbs, - const volatile MDBX_meta *meta, bool report) { - const txnid_t freedb_mod_txnid = dbs[FREE_DBI].md_mod_txnid; - const txnid_t maindb_mod_txnid = dbs[MAIN_DBI].md_mod_txnid; - const pgno_t last_pgno = meta->mm_geo.now; - - const pgno_t freedb_root_pgno = dbs[FREE_DBI].md_root; - const MDBX_page *freedb_root = (env->me_map && freedb_root_pgno < last_pgno) - ? pgno2page(env, freedb_root_pgno) - : nullptr; - - const pgno_t maindb_root_pgno = dbs[MAIN_DBI].md_root; - const MDBX_page *maindb_root = (env->me_map && maindb_root_pgno < last_pgno) - ? pgno2page(env, maindb_root_pgno) - : nullptr; - const uint64_t magic_and_version = - unaligned_peek_u64_volatile(4, &meta->mm_magic_and_version); +__cold int mdbx_enumerate_tables(const MDBX_txn *txn, MDBX_table_enum_func *func, void *ctx) { + if (unlikely(!func)) + return LOG_IFERR(MDBX_EINVAL); - bool ok = true; - if (freedb_root_pgno != P_INVALID && - unlikely(freedb_root_pgno >= last_pgno)) { - if (report) - WARNING( - "catch invalid %sdb root %" PRIaPGNO " for meta_txnid %" PRIaTXN - " %s", - "free", freedb_root_pgno, txnid, - (env->me_stuck_meta < 0) - ? "(workaround for incoherent flaw of unified page/buffer cache)" - : "(wagering meta)"); - ok = false; - } - if (maindb_root_pgno != P_INVALID && - unlikely(maindb_root_pgno >= last_pgno)) { - if (report) - WARNING( - "catch invalid %sdb root %" PRIaPGNO " for meta_txnid %" PRIaTXN - " %s", - "main", maindb_root_pgno, txnid, - (env->me_stuck_meta < 0) - ? "(workaround for incoherent flaw of unified page/buffer cache)" - : "(wagering meta)"); - ok = false; - } - if (unlikely(txnid < freedb_mod_txnid || - (!freedb_mod_txnid && freedb_root && - likely(magic_and_version == MDBX_DATA_MAGIC)))) { - if (report) - WARNING( - "catch invalid %sdb.mod_txnid %" PRIaTXN " for meta_txnid %" PRIaTXN - " %s", - "free", freedb_mod_txnid, txnid, - (env->me_stuck_meta < 0) - ? "(workaround for incoherent flaw of unified page/buffer cache)" - : "(wagering meta)"); - ok = false; - } - if (unlikely(txnid < maindb_mod_txnid || - (!maindb_mod_txnid && maindb_root && - likely(magic_and_version == MDBX_DATA_MAGIC)))) { - if (report) - WARNING( - "catch invalid %sdb.mod_txnid %" PRIaTXN " for meta_txnid %" PRIaTXN - " %s", - "main", maindb_mod_txnid, txnid, - (env->me_stuck_meta < 0) - ? "(workaround for incoherent flaw of unified page/buffer cache)" - : "(wagering meta)"); - ok = false; - } - if (likely(freedb_root && freedb_mod_txnid && - (size_t)ptr_dist(env->me_dxb_mmap.base, freedb_root) < - env->me_dxb_mmap.limit)) { - VALGRIND_MAKE_MEM_DEFINED(freedb_root, sizeof(freedb_root->mp_txnid)); - MDBX_ASAN_UNPOISON_MEMORY_REGION(freedb_root, - sizeof(freedb_root->mp_txnid)); - const txnid_t root_txnid = freedb_root->mp_txnid; - if (unlikely(root_txnid != freedb_mod_txnid)) { - if (report) - WARNING("catch invalid root_page %" PRIaPGNO " mod_txnid %" PRIaTXN - " for %sdb.mod_txnid %" PRIaTXN " %s", - freedb_root_pgno, root_txnid, "free", freedb_mod_txnid, - (env->me_stuck_meta < 0) ? "(workaround for incoherent flaw of " - "unified page/buffer cache)" - : "(wagering meta)"); - ok = false; + int rc = check_txn(txn, MDBX_TXN_BLOCKED); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); + + cursor_couple_t cx; + rc = cursor_init(&cx.outer, txn, MAIN_DBI); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); + + cx.outer.next = txn->cursors[MAIN_DBI]; + txn->cursors[MAIN_DBI] = &cx.outer; + for (rc = outer_first(&cx.outer, nullptr, nullptr); rc == MDBX_SUCCESS; + rc = outer_next(&cx.outer, nullptr, nullptr, MDBX_NEXT_NODUP)) { + node_t *node = page_node(cx.outer.pg[cx.outer.top], cx.outer.ki[cx.outer.top]); + if (node_flags(node) != N_TREE) + continue; + if (unlikely(node_ds(node) != sizeof(tree_t))) { + ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid dupsort sub-tree node size", + (unsigned)node_ds(node)); + rc = MDBX_CORRUPTED; + break; } - } - if (likely(maindb_root && maindb_mod_txnid && - (size_t)ptr_dist(env->me_dxb_mmap.base, maindb_root) < - env->me_dxb_mmap.limit)) { - VALGRIND_MAKE_MEM_DEFINED(maindb_root, sizeof(maindb_root->mp_txnid)); - MDBX_ASAN_UNPOISON_MEMORY_REGION(maindb_root, - sizeof(maindb_root->mp_txnid)); - const txnid_t root_txnid = maindb_root->mp_txnid; - if (unlikely(root_txnid != maindb_mod_txnid)) { - if (report) - WARNING("catch invalid root_page %" PRIaPGNO " mod_txnid %" PRIaTXN - " for %sdb.mod_txnid %" PRIaTXN " %s", - maindb_root_pgno, root_txnid, "main", maindb_mod_txnid, - (env->me_stuck_meta < 0) ? "(workaround for incoherent flaw of " - "unified page/buffer cache)" - : "(wagering meta)"); - ok = false; + + tree_t reside; + const tree_t *tree = memcpy(&reside, node_data(node), sizeof(reside)); + const MDBX_val name = {node_key(node), node_ks(node)}; + const MDBX_env *const env = txn->env; + MDBX_dbi dbi = 0; + for (size_t i = CORE_DBS; i < env->n_dbi; ++i) { + if (i >= txn->n_dbi || !(env->dbs_flags[i] & DB_VALID)) + continue; + if (env->kvs[MAIN_DBI].clc.k.cmp(&name, &env->kvs[i].name)) + continue; + + tree = dbi_dig(txn, i, &reside); + dbi = (MDBX_dbi)i; + break; } - } - if (unlikely(!ok) && report) - env->me_lck->mti_pgop_stat.incoherence.weak = - (env->me_lck->mti_pgop_stat.incoherence.weak >= INT32_MAX) - ? INT32_MAX - : env->me_lck->mti_pgop_stat.incoherence.weak + 1; - return ok; -} -__cold static int coherency_timeout(uint64_t *timestamp, intptr_t pgno, - const MDBX_env *env) { - if (likely(timestamp && *timestamp == 0)) - *timestamp = osal_monotime(); - else if (unlikely(!timestamp || osal_monotime() - *timestamp > - osal_16dot16_to_monotime(65536 / 10))) { - if (pgno >= 0 && pgno != env->me_stuck_meta) - ERROR("bailout waiting for %" PRIuSIZE " page arrival %s", pgno, - "(workaround for incoherent flaw of unified page/buffer cache)"); - else if (env->me_stuck_meta < 0) - ERROR("bailout waiting for valid snapshot (%s)", - "workaround for incoherent flaw of unified page/buffer cache"); - return MDBX_PROBLEM; + MDBX_stat stat; + stat_get(tree, &stat, sizeof(stat)); + rc = func(ctx, txn, &name, tree->flags, &stat, dbi); + if (rc != MDBX_SUCCESS) + goto bailout; } + rc = (rc == MDBX_NOTFOUND) ? MDBX_SUCCESS : rc; - osal_memory_fence(mo_AcquireRelease, true); -#if defined(_WIN32) || defined(_WIN64) - SwitchToThread(); -#elif defined(__linux__) || defined(__gnu_linux__) || defined(_UNIX03_SOURCE) - sched_yield(); -#elif (defined(_GNU_SOURCE) && __GLIBC_PREREQ(2, 1)) || defined(_OPEN_THREADS) - pthread_yield(); -#else - usleep(42); -#endif - return MDBX_RESULT_TRUE; +bailout: + txn->cursors[MAIN_DBI] = cx.outer.next; + return LOG_IFERR(rc); } +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 -/* check with timeout as the workaround - * for https://libmdbx.dqdkfa.ru/dead-github/issues/269 */ -__hot static int coherency_check_head(MDBX_txn *txn, const meta_ptr_t head, - uint64_t *timestamp) { - /* Copy the DB info and flags */ - txn->mt_geo = head.ptr_v->mm_geo; - memcpy(txn->mt_dbs, head.ptr_c->mm_dbs, CORE_DBS * sizeof(MDBX_db)); - txn->mt_canary = head.ptr_v->mm_canary; +__cold static intptr_t reasonable_db_maxsize(void) { + static intptr_t cached_result; + if (cached_result == 0) { + intptr_t pagesize, total_ram_pages; + if (unlikely(mdbx_get_sysraminfo(&pagesize, &total_ram_pages, nullptr) != MDBX_SUCCESS)) + /* the 32-bit limit is good enough for fallback */ + return cached_result = MAX_MAPSIZE32; - if (unlikely(!coherency_check(txn->mt_env, head.txnid, txn->mt_dbs, - head.ptr_v, *timestamp == 0))) - return coherency_timeout(timestamp, -1, txn->mt_env); - return MDBX_SUCCESS; -} +#if defined(__SANITIZE_ADDRESS__) + total_ram_pages >>= 4; +#endif /* __SANITIZE_ADDRESS__ */ + if (RUNNING_ON_VALGRIND) + total_ram_pages >>= 4; -static int coherency_check_written(const MDBX_env *env, const txnid_t txnid, - const volatile MDBX_meta *meta, - const intptr_t pgno, uint64_t *timestamp) { - const bool report = !(timestamp && *timestamp); - const txnid_t head_txnid = meta_txnid(meta); - if (unlikely(head_txnid < MIN_TXNID || head_txnid < txnid)) { - if (report) { - env->me_lck->mti_pgop_stat.incoherence.weak = - (env->me_lck->mti_pgop_stat.incoherence.weak >= INT32_MAX) - ? INT32_MAX - : env->me_lck->mti_pgop_stat.incoherence.weak + 1; - WARNING("catch %s txnid %" PRIaTXN " for meta_%" PRIaPGNO " %s", - (head_txnid < MIN_TXNID) ? "invalid" : "unexpected", head_txnid, - bytes2pgno(env, ptr_dist(meta, env->me_map)), - "(workaround for incoherent flaw of unified page/buffer cache)"); + if (unlikely((size_t)total_ram_pages * 2 > MAX_MAPSIZE / (size_t)pagesize)) + return cached_result = MAX_MAPSIZE; + assert(MAX_MAPSIZE >= (size_t)(total_ram_pages * pagesize * 2)); + + /* Suggesting should not be more than golden ratio of the size of RAM. */ + cached_result = (intptr_t)((size_t)total_ram_pages * 207 >> 7) * pagesize; + + /* Round to the nearest human-readable granulation. */ + for (size_t unit = MEGABYTE; unit; unit <<= 5) { + const size_t floor = floor_powerof2(cached_result, unit); + const size_t ceil = ceil_powerof2(cached_result, unit); + const size_t threshold = (size_t)cached_result >> 4; + const bool down = cached_result - floor < ceil - cached_result || ceil > MAX_MAPSIZE; + if (threshold < (down ? cached_result - floor : ceil - cached_result)) + break; + cached_result = down ? floor : ceil; } - return coherency_timeout(timestamp, pgno, env); } - if (unlikely(!coherency_check(env, head_txnid, meta->mm_dbs, meta, report))) - return coherency_timeout(timestamp, pgno, env); - return MDBX_SUCCESS; + return cached_result; } -static bool check_meta_coherency(const MDBX_env *env, - const volatile MDBX_meta *meta, bool report) { - uint64_t timestamp = 0; - return coherency_check_written(env, 0, meta, -1, - report ? ×tamp : nullptr) == MDBX_SUCCESS; +__cold static int check_alternative_lck_absent(const pathchar_t *lck_pathname) { + int err = osal_fileexists(lck_pathname); + if (unlikely(err != MDBX_RESULT_FALSE)) { + if (err == MDBX_RESULT_TRUE) + err = MDBX_DUPLICATED_CLK; + ERROR("Alternative/Duplicate LCK-file '%" MDBX_PRIsPATH "' error %d", lck_pathname, err); + } + return err; } -/* Common code for mdbx_txn_begin() and mdbx_txn_renew(). */ -static int txn_renew(MDBX_txn *txn, const unsigned flags) { - MDBX_env *env = txn->mt_env; +__cold static int env_handle_pathname(MDBX_env *env, const pathchar_t *pathname, const mdbx_mode_t mode) { + memset(&env->pathname, 0, sizeof(env->pathname)); + if (unlikely(!pathname || !*pathname)) + return MDBX_EINVAL; + int rc; +#if defined(_WIN32) || defined(_WIN64) + const DWORD dwAttrib = GetFileAttributesW(pathname); + if (dwAttrib == INVALID_FILE_ATTRIBUTES) { + rc = GetLastError(); + if (rc != MDBX_ENOFILE) + return rc; + if (mode == 0 || (env->flags & MDBX_RDONLY) != 0) + /* can't open existing */ + return rc; -#if MDBX_ENV_CHECKPID - if (unlikely(env->me_pid != osal_getpid())) { - env->me_flags |= MDBX_FATAL_ERROR; - return MDBX_PANIC; + /* auto-create directory if requested */ + if ((env->flags & MDBX_NOSUBDIR) == 0 && !CreateDirectoryW(pathname, nullptr)) { + rc = GetLastError(); + if (rc != ERROR_ALREADY_EXISTS) + return rc; + } + } else { + /* ignore passed MDBX_NOSUBDIR flag and set it automatically */ + env->flags |= MDBX_NOSUBDIR; + if (dwAttrib & FILE_ATTRIBUTE_DIRECTORY) + env->flags -= MDBX_NOSUBDIR; } -#endif /* MDBX_ENV_CHECKPID */ - - STATIC_ASSERT(sizeof(MDBX_reader) == 32); -#if MDBX_LOCKING > 0 - STATIC_ASSERT(offsetof(MDBX_lockinfo, mti_wlock) % MDBX_CACHELINE_SIZE == 0); - STATIC_ASSERT(offsetof(MDBX_lockinfo, mti_rlock) % MDBX_CACHELINE_SIZE == 0); #else - STATIC_ASSERT( - offsetof(MDBX_lockinfo, mti_oldest_reader) % MDBX_CACHELINE_SIZE == 0); - STATIC_ASSERT(offsetof(MDBX_lockinfo, mti_numreaders) % MDBX_CACHELINE_SIZE == - 0); -#endif /* MDBX_LOCKING */ - STATIC_ASSERT(offsetof(MDBX_lockinfo, mti_readers) % MDBX_CACHELINE_SIZE == - 0); - - const uintptr_t tid = osal_thread_self(); - if (flags & MDBX_TXN_RDONLY) { - eASSERT(env, (flags & ~(MDBX_TXN_RO_BEGIN_FLAGS | MDBX_WRITEMAP)) == 0); - txn->mt_flags = - MDBX_TXN_RDONLY | (env->me_flags & (MDBX_NOTLS | MDBX_WRITEMAP)); - MDBX_reader *r = txn->to.reader; - STATIC_ASSERT(sizeof(uintptr_t) <= sizeof(r->mr_tid)); - if (likely(env->me_flags & MDBX_ENV_TXKEY)) { - eASSERT(env, !(env->me_flags & MDBX_NOTLS)); - r = thread_rthc_get(env->me_txkey); - if (likely(r)) { - if (unlikely(!r->mr_pid.weak) && - (runtime_flags & MDBX_DBG_LEGACY_MULTIOPEN)) { - thread_rthc_set(env->me_txkey, nullptr); - r = nullptr; - } else { - eASSERT(env, r->mr_pid.weak == env->me_pid); - eASSERT(env, r->mr_tid.weak == osal_thread_self()); - } - } - } else { - eASSERT(env, !env->me_lck_mmap.lck || (env->me_flags & MDBX_NOTLS)); - } - - if (likely(r)) { - if (unlikely(r->mr_pid.weak != env->me_pid || - r->mr_txnid.weak < SAFE64_INVALID_THRESHOLD)) - return MDBX_BAD_RSLOT; - } else if (env->me_lck_mmap.lck) { - bind_rslot_result brs = bind_rslot(env, tid); - if (unlikely(brs.err != MDBX_SUCCESS)) - return brs.err; - r = brs.rslot; - } - txn->to.reader = r; - if (flags & (MDBX_TXN_RDONLY_PREPARE - MDBX_TXN_RDONLY)) { - eASSERT(env, txn->mt_txnid == 0); - eASSERT(env, txn->mt_owner == 0); - eASSERT(env, txn->mt_numdbs == 0); - if (likely(r)) { - eASSERT(env, r->mr_snapshot_pages_used.weak == 0); - eASSERT(env, r->mr_txnid.weak >= SAFE64_INVALID_THRESHOLD); - atomic_store32(&r->mr_snapshot_pages_used, 0, mo_Relaxed); - } - txn->mt_flags = MDBX_TXN_RDONLY | MDBX_TXN_FINISHED; - return MDBX_SUCCESS; - } - - /* Seek & fetch the last meta */ - uint64_t timestamp = 0; - size_t loop = 0; - meta_troika_t troika = meta_tap(env); - while (1) { - const meta_ptr_t head = - likely(env->me_stuck_meta < 0) - ? /* regular */ meta_recent(env, &troika) - : /* recovery mode */ meta_ptr(env, env->me_stuck_meta); - if (likely(r)) { - safe64_reset(&r->mr_txnid, false); - atomic_store32(&r->mr_snapshot_pages_used, head.ptr_v->mm_geo.next, - mo_Relaxed); - atomic_store64( - &r->mr_snapshot_pages_retired, - unaligned_peek_u64_volatile(4, head.ptr_v->mm_pages_retired), - mo_Relaxed); - safe64_write(&r->mr_txnid, head.txnid); - eASSERT(env, r->mr_pid.weak == osal_getpid()); - eASSERT(env, - r->mr_tid.weak == - ((env->me_flags & MDBX_NOTLS) ? 0 : osal_thread_self())); - eASSERT(env, r->mr_txnid.weak == head.txnid || - (r->mr_txnid.weak >= SAFE64_INVALID_THRESHOLD && - head.txnid < env->me_lck->mti_oldest_reader.weak)); - atomic_store32(&env->me_lck->mti_readers_refresh_flag, true, - mo_AcquireRelease); - } else { - /* exclusive mode without lck */ - eASSERT(env, !env->me_lck_mmap.lck && env->me_lck == lckless_stub(env)); - } - jitter4testing(true); - - /* Snap the state from current meta-head */ - txn->mt_txnid = head.txnid; - if (likely(env->me_stuck_meta < 0) && - unlikely(meta_should_retry(env, &troika) || - head.txnid < atomic_load64(&env->me_lck->mti_oldest_reader, - mo_AcquireRelease))) { - if (unlikely(++loop > 42)) { - ERROR("bailout waiting for valid snapshot (%s)", - "metapages are too volatile"); - rc = MDBX_PROBLEM; - txn->mt_txnid = INVALID_TXNID; - if (likely(r)) - safe64_reset(&r->mr_txnid, false); - goto bailout; - } - timestamp = 0; - continue; - } - - rc = coherency_check_head(txn, head, ×tamp); - jitter4testing(false); - if (likely(rc == MDBX_SUCCESS)) - break; - - if (unlikely(rc != MDBX_RESULT_TRUE)) { - txn->mt_txnid = INVALID_TXNID; - if (likely(r)) - safe64_reset(&r->mr_txnid, false); - goto bailout; - } - } - - if (unlikely(txn->mt_txnid < MIN_TXNID || txn->mt_txnid > MAX_TXNID)) { - ERROR("%s", "environment corrupted by died writer, must shutdown!"); - if (likely(r)) - safe64_reset(&r->mr_txnid, false); - txn->mt_txnid = INVALID_TXNID; - rc = MDBX_CORRUPTED; - goto bailout; - } - eASSERT(env, txn->mt_txnid >= env->me_lck->mti_oldest_reader.weak); - txn->mt_dbxs = env->me_dbxs; /* mostly static anyway */ - ENSURE(env, txn->mt_txnid >= - /* paranoia is appropriate here */ env->me_lck - ->mti_oldest_reader.weak); - txn->mt_numdbs = env->me_numdbs; - } else { - eASSERT(env, (flags & ~(MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_SPILLS | - MDBX_WRITEMAP)) == 0); - if (unlikely(txn->mt_owner == tid || - /* not recovery mode */ env->me_stuck_meta >= 0)) - return MDBX_BUSY; - MDBX_lockinfo *const lck = env->me_lck_mmap.lck; - if (lck && (env->me_flags & MDBX_NOTLS) == 0 && - (runtime_flags & MDBX_DBG_LEGACY_OVERLAP) == 0) { - const size_t snap_nreaders = - atomic_load32(&lck->mti_numreaders, mo_AcquireRelease); - for (size_t i = 0; i < snap_nreaders; ++i) { - if (atomic_load32(&lck->mti_readers[i].mr_pid, mo_Relaxed) == - env->me_pid && - unlikely(atomic_load64(&lck->mti_readers[i].mr_tid, mo_Relaxed) == - tid)) { - const txnid_t txnid = safe64_read(&lck->mti_readers[i].mr_txnid); - if (txnid >= MIN_TXNID && txnid <= MAX_TXNID) - return MDBX_TXN_OVERLAPPING; - } - } - } - - /* Not yet touching txn == env->me_txn0, it may be active */ - jitter4testing(false); - rc = mdbx_txn_lock(env, !!(flags & MDBX_TXN_TRY)); - if (unlikely(rc)) + struct stat st; + if (stat(pathname, &st) != 0) { + rc = errno; + if (rc != MDBX_ENOFILE) return rc; - if (unlikely(env->me_flags & MDBX_FATAL_ERROR)) { - mdbx_txn_unlock(env); - return MDBX_PANIC; - } -#if defined(_WIN32) || defined(_WIN64) - if (unlikely(!env->me_map)) { - mdbx_txn_unlock(env); - return MDBX_EPERM; - } -#endif /* Windows */ - - txn->tw.troika = meta_tap(env); - const meta_ptr_t head = meta_recent(env, &txn->tw.troika); - uint64_t timestamp = 0; - while ("workaround for https://libmdbx.dqdkfa.ru/dead-github/issues/269") { - rc = coherency_check_head(txn, head, ×tamp); - if (likely(rc == MDBX_SUCCESS)) - break; - if (unlikely(rc != MDBX_RESULT_TRUE)) - goto bailout; - } - eASSERT(env, meta_txnid(head.ptr_v) == head.txnid); - txn->mt_txnid = safe64_txnid_next(head.txnid); - if (unlikely(txn->mt_txnid > MAX_TXNID)) { - rc = MDBX_TXN_FULL; - ERROR("txnid overflow, raise %d", rc); - goto bailout; - } + if (mode == 0 || (env->flags & MDBX_RDONLY) != 0) + /* can't open non-existing */ + return rc /* MDBX_ENOFILE */; - txn->mt_flags = flags; - txn->mt_child = NULL; - txn->tw.loose_pages = NULL; - txn->tw.loose_count = 0; -#if MDBX_ENABLE_REFUND - txn->tw.loose_refund_wl = 0; -#endif /* MDBX_ENABLE_REFUND */ - MDBX_PNL_SETSIZE(txn->tw.retired_pages, 0); - txn->tw.spilled.list = NULL; - txn->tw.spilled.least_removed = 0; - txn->tw.last_reclaimed = 0; - if (txn->tw.lifo_reclaimed) - MDBX_PNL_SETSIZE(txn->tw.lifo_reclaimed, 0); - env->me_txn = txn; - txn->mt_numdbs = env->me_numdbs; - memcpy(txn->mt_dbiseqs, env->me_dbiseqs, txn->mt_numdbs * sizeof(unsigned)); - - if ((txn->mt_flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC) { - rc = dpl_alloc(txn); - if (unlikely(rc != MDBX_SUCCESS)) - goto bailout; - txn->tw.dirtyroom = txn->mt_env->me_options.dp_limit; - txn->tw.dirtylru = MDBX_DEBUG ? UINT32_MAX / 3 - 42 : 0; - } else { - tASSERT(txn, txn->tw.dirtylist == nullptr); - txn->tw.dirtylist = nullptr; - txn->tw.dirtyroom = MAX_PAGENO; - txn->tw.dirtylru = 0; + /* auto-create directory if requested */ + const mdbx_mode_t dir_mode = + (/* inherit read/write permissions for group and others */ mode & (S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)) | + /* always add read/write/search for owner */ S_IRWXU | + ((mode & S_IRGRP) ? /* +search if readable by group */ S_IXGRP : 0) | + ((mode & S_IROTH) ? /* +search if readable by others */ S_IXOTH : 0); + if ((env->flags & MDBX_NOSUBDIR) == 0 && mkdir(pathname, dir_mode)) { + rc = errno; + if (rc != EEXIST) + return rc; } - eASSERT(env, txn->tw.writemap_dirty_npages == 0); - eASSERT(env, txn->tw.writemap_spilled_npages == 0); + } else { + /* ignore passed MDBX_NOSUBDIR flag and set it automatically */ + env->flags |= MDBX_NOSUBDIR; + if (S_ISDIR(st.st_mode)) + env->flags -= MDBX_NOSUBDIR; } +#endif - /* Setup db info */ - osal_compiler_barrier(); - memset(txn->mt_cursors, 0, sizeof(MDBX_cursor *) * txn->mt_numdbs); - for (size_t i = CORE_DBS; i < txn->mt_numdbs; i++) { - const unsigned db_flags = env->me_dbflags[i]; - txn->mt_dbs[i].md_flags = db_flags & DB_PERSISTENT_FLAGS; - txn->mt_dbistate[i] = - (db_flags & DB_VALID) ? DBI_VALID | DBI_USRVALID | DBI_STALE : 0; - } - txn->mt_dbistate[MAIN_DBI] = DBI_VALID | DBI_USRVALID; - rc = - setup_dbx(&txn->mt_dbxs[MAIN_DBI], &txn->mt_dbs[MAIN_DBI], env->me_psize); - if (unlikely(rc != MDBX_SUCCESS)) - goto bailout; - txn->mt_dbistate[FREE_DBI] = DBI_VALID; - txn->mt_front = - txn->mt_txnid + ((flags & (MDBX_WRITEMAP | MDBX_RDONLY)) == 0); + static const pathchar_t dxb_name[] = MDBX_DATANAME; + static const pathchar_t lck_name[] = MDBX_LOCKNAME; + static const pathchar_t lock_suffix[] = MDBX_LOCK_SUFFIX; - if (unlikely(env->me_flags & MDBX_FATAL_ERROR)) { - WARNING("%s", "environment had fatal error, must shutdown!"); - rc = MDBX_PANIC; - } else { - const size_t size_bytes = pgno2bytes(env, txn->mt_end_pgno); - const size_t used_bytes = pgno2bytes(env, txn->mt_next_pgno); - const size_t required_bytes = - (txn->mt_flags & MDBX_TXN_RDONLY) ? used_bytes : size_bytes; - eASSERT(env, env->me_dxb_mmap.limit >= env->me_dxb_mmap.current); - if (unlikely(required_bytes > env->me_dxb_mmap.current)) { - /* Размер БД (для пишущих транзакций) или используемых данных (для - * читающих транзакций) больше предыдущего/текущего размера внутри - * процесса, увеличиваем. Сюда также попадает случай увеличения верхней - * границы размера БД и отображения. В читающих транзакциях нельзя - * изменять размер файла, который может быть больше необходимого этой - * транзакции. */ - if (txn->mt_geo.upper > MAX_PAGENO + 1 || - bytes2pgno(env, pgno2bytes(env, txn->mt_geo.upper)) != - txn->mt_geo.upper) { - rc = MDBX_UNABLE_EXTEND_MAPSIZE; - goto bailout; - } - rc = dxb_resize(env, txn->mt_next_pgno, txn->mt_end_pgno, - txn->mt_geo.upper, implicit_grow); - if (unlikely(rc != MDBX_SUCCESS)) - goto bailout; - eASSERT(env, env->me_dxb_mmap.limit >= env->me_dxb_mmap.current); - } else if (unlikely(size_bytes < env->me_dxb_mmap.current)) { - /* Размер БД меньше предыдущего/текущего размера внутри процесса, можно - * уменьшить, но всё сложнее: - * - размер файла согласован со всеми читаемыми снимками на момент - * коммита последней транзакции; - * - в читающей транзакции размер файла может быть больше и него нельзя - * изменять, в том числе менять madvise (меньша размера файла нельзя, - * а за размером нет смысла). - * - в пишущей транзакции уменьшать размер файла можно только после - * проверки размера читаемых снимков, но в этом нет смысла, так как - * это будет сделано при фиксации транзакции. - * - * В сухом остатке, можно только установить dxb_mmap.current равным - * размеру файла, а это проще сделать без вызова dxb_resize() и усложения - * внутренней логики. - * - * В этой тактике есть недостаток: если пишущите транзакции не регулярны, - * и при завершении такой транзакции файл БД остаётся не-уменьшеным из-за - * читающих транзакций использующих предыдущие снимки. */ -#if defined(_WIN32) || defined(_WIN64) - osal_srwlock_AcquireShared(&env->me_remap_guard); -#else - rc = osal_fastmutex_acquire(&env->me_remap_guard); -#endif - if (likely(rc == MDBX_SUCCESS)) { - eASSERT(env, env->me_dxb_mmap.limit >= env->me_dxb_mmap.current); - rc = osal_filesize(env->me_dxb_mmap.fd, &env->me_dxb_mmap.filesize); - if (likely(rc == MDBX_SUCCESS)) { - eASSERT(env, env->me_dxb_mmap.filesize >= required_bytes); - if (env->me_dxb_mmap.current > env->me_dxb_mmap.filesize) - env->me_dxb_mmap.current = - (env->me_dxb_mmap.limit < env->me_dxb_mmap.filesize) - ? env->me_dxb_mmap.limit - : (size_t)env->me_dxb_mmap.filesize; - } #if defined(_WIN32) || defined(_WIN64) - osal_srwlock_ReleaseShared(&env->me_remap_guard); + assert(dxb_name[0] == '\\' && lck_name[0] == '\\'); + const size_t pathname_len = wcslen(pathname); #else - int err = osal_fastmutex_release(&env->me_remap_guard); - if (unlikely(err) && likely(rc == MDBX_SUCCESS)) - rc = err; + assert(dxb_name[0] == '/' && lck_name[0] == '/'); + const size_t pathname_len = strlen(pathname); #endif - } - if (unlikely(rc != MDBX_SUCCESS)) - goto bailout; + assert(!osal_isdirsep(lock_suffix[0])); + size_t base_len = pathname_len; + static const size_t dxb_name_len = ARRAY_LENGTH(dxb_name) - 1; + if (env->flags & MDBX_NOSUBDIR) { + if (base_len > dxb_name_len && osal_pathequal(pathname + base_len - dxb_name_len, dxb_name, dxb_name_len)) { + env->flags -= MDBX_NOSUBDIR; + base_len -= dxb_name_len; + } else if (base_len == dxb_name_len - 1 && osal_isdirsep(dxb_name[0]) && osal_isdirsep(lck_name[0]) && + osal_pathequal(pathname + base_len - dxb_name_len + 1, dxb_name + 1, dxb_name_len - 1)) { + env->flags -= MDBX_NOSUBDIR; + base_len -= dxb_name_len - 1; } - eASSERT(env, - pgno2bytes(env, txn->mt_next_pgno) <= env->me_dxb_mmap.current); - eASSERT(env, env->me_dxb_mmap.limit >= env->me_dxb_mmap.current); - if (txn->mt_flags & MDBX_TXN_RDONLY) { -#if defined(_WIN32) || defined(_WIN64) - if (((used_bytes > env->me_dbgeo.lower && env->me_dbgeo.shrink) || - (mdbx_RunningUnderWine() && - /* under Wine acquisition of remap_guard is always required, - * since Wine don't support section extending, - * i.e. in both cases unmap+map are required. */ - used_bytes < env->me_dbgeo.upper && env->me_dbgeo.grow)) && - /* avoid recursive use SRW */ (txn->mt_flags & MDBX_NOTLS) == 0) { - txn->mt_flags |= MDBX_SHRINK_ALLOWED; - osal_srwlock_AcquireShared(&env->me_remap_guard); + } + + const size_t suflen_with_NOSUBDIR = sizeof(lock_suffix) + sizeof(pathchar_t); + const size_t suflen_without_NOSUBDIR = sizeof(lck_name) + sizeof(dxb_name); + const size_t enough4any = + (suflen_with_NOSUBDIR > suflen_without_NOSUBDIR) ? suflen_with_NOSUBDIR : suflen_without_NOSUBDIR; + const size_t bytes_needed = sizeof(pathchar_t) * (base_len * 2 + pathname_len + 1) + enough4any; + env->pathname.buffer = osal_malloc(bytes_needed); + if (!env->pathname.buffer) + return MDBX_ENOMEM; + + env->pathname.specified = env->pathname.buffer; + env->pathname.dxb = env->pathname.specified + pathname_len + 1; + env->pathname.lck = env->pathname.dxb + base_len + dxb_name_len + 1; + rc = MDBX_SUCCESS; + pathchar_t *const buf = env->pathname.buffer; + if (base_len) { + memcpy(buf, pathname, sizeof(pathchar_t) * pathname_len); + if (env->flags & MDBX_NOSUBDIR) { + const pathchar_t *const lck_ext = osal_fileext(lck_name, ARRAY_LENGTH(lck_name)); + if (lck_ext) { + pathchar_t *pathname_ext = osal_fileext(buf, pathname_len); + memcpy(pathname_ext ? pathname_ext : buf + pathname_len, lck_ext, + sizeof(pathchar_t) * (ARRAY_END(lck_name) - lck_ext)); + rc = check_alternative_lck_absent(buf); } -#endif /* Windows */ } else { - if (unlikely((txn->mt_dbs[FREE_DBI].md_flags & DB_PERSISTENT_FLAGS) != - MDBX_INTEGERKEY)) { - ERROR("unexpected/invalid db-flags 0x%x for %s", - txn->mt_dbs[FREE_DBI].md_flags, "GC/FreeDB"); - rc = MDBX_INCOMPATIBLE; - goto bailout; - } + memcpy(buf + base_len, dxb_name, sizeof(dxb_name)); + memcpy(buf + base_len + dxb_name_len, lock_suffix, sizeof(lock_suffix)); + rc = check_alternative_lck_absent(buf); + } - tASSERT(txn, txn == env->me_txn0); - MDBX_cursor *const gc = ptr_disp(txn, sizeof(MDBX_txn)); - rc = cursor_init(gc, txn, FREE_DBI); - if (rc != MDBX_SUCCESS) - goto bailout; + memcpy(env->pathname.dxb, pathname, sizeof(pathchar_t) * (base_len + 1)); + memcpy(env->pathname.lck, pathname, sizeof(pathchar_t) * base_len); + if (env->flags & MDBX_NOSUBDIR) { + memcpy(env->pathname.lck + base_len, lock_suffix, sizeof(lock_suffix)); + } else { + memcpy(env->pathname.dxb + base_len, dxb_name, sizeof(dxb_name)); + memcpy(env->pathname.lck + base_len, lck_name, sizeof(lck_name)); } -#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__) - txn_valgrind(env, txn); -#endif /* MDBX_USE_VALGRIND || __SANITIZE_ADDRESS__ */ - txn->mt_owner = tid; - return MDBX_SUCCESS; + } else { + assert(!(env->flags & MDBX_NOSUBDIR)); + memcpy(buf, dxb_name + 1, sizeof(dxb_name) - sizeof(pathchar_t)); + memcpy(buf + dxb_name_len - 1, lock_suffix, sizeof(lock_suffix)); + rc = check_alternative_lck_absent(buf); + + memcpy(env->pathname.dxb, dxb_name + 1, sizeof(dxb_name) - sizeof(pathchar_t)); + memcpy(env->pathname.lck, lck_name + 1, sizeof(lck_name) - sizeof(pathchar_t)); } -bailout: - tASSERT(txn, rc != MDBX_SUCCESS); - txn_end(txn, MDBX_END_SLOT | MDBX_END_EOTDONE | MDBX_END_FAIL_BEGIN); + + memcpy(env->pathname.specified, pathname, sizeof(pathchar_t) * (pathname_len + 1)); return rc; } -static __always_inline int check_txn(const MDBX_txn *txn, int bad_bits) { - if (unlikely(!txn)) - return MDBX_EINVAL; - - if (unlikely(txn->mt_signature != MDBX_MT_SIGNATURE)) - return MDBX_EBADSIGN; - - if (unlikely(txn->mt_flags & bad_bits)) - return MDBX_BAD_TXN; - - tASSERT(txn, (txn->mt_flags & MDBX_TXN_FINISHED) || - (txn->mt_flags & MDBX_NOTLS) == - ((txn->mt_flags & MDBX_TXN_RDONLY) - ? txn->mt_env->me_flags & MDBX_NOTLS - : 0)); -#if MDBX_TXN_CHECKOWNER - STATIC_ASSERT(MDBX_NOTLS > MDBX_TXN_FINISHED + MDBX_TXN_RDONLY); - if (unlikely(txn->mt_owner != osal_thread_self()) && - (txn->mt_flags & (MDBX_NOTLS | MDBX_TXN_FINISHED | MDBX_TXN_RDONLY)) < - (MDBX_TXN_FINISHED | MDBX_TXN_RDONLY)) - return txn->mt_owner ? MDBX_THREAD_MISMATCH : MDBX_BAD_TXN; -#endif /* MDBX_TXN_CHECKOWNER */ +/*----------------------------------------------------------------------------*/ - if (bad_bits && unlikely(!txn->mt_env->me_map)) - return MDBX_EPERM; +__cold int mdbx_env_create(MDBX_env **penv) { + if (unlikely(!penv)) + return LOG_IFERR(MDBX_EINVAL); + *penv = nullptr; - return MDBX_SUCCESS; -} +#ifdef MDBX_HAVE_C11ATOMICS + if (unlikely(!atomic_is_lock_free((const volatile uint32_t *)penv))) { + ERROR("lock-free atomic ops for %u-bit types is required", 32); + return LOG_IFERR(MDBX_INCOMPATIBLE); + } +#if MDBX_64BIT_ATOMIC + if (unlikely(!atomic_is_lock_free((const volatile uint64_t *)penv))) { + ERROR("lock-free atomic ops for %u-bit types is required", 64); + return LOG_IFERR(MDBX_INCOMPATIBLE); + } +#endif /* MDBX_64BIT_ATOMIC */ +#endif /* MDBX_HAVE_C11ATOMICS */ -static __always_inline int check_txn_rw(const MDBX_txn *txn, int bad_bits) { - int err = check_txn(txn, bad_bits); - if (unlikely(err)) - return err; + if (unlikely(!is_powerof2(globals.sys_pagesize) || globals.sys_pagesize < MDBX_MIN_PAGESIZE)) { + ERROR("unsuitable system pagesize %u", globals.sys_pagesize); + return LOG_IFERR(MDBX_INCOMPATIBLE); + } - if (unlikely(txn->mt_flags & MDBX_TXN_RDONLY)) - return MDBX_EACCESS; +#if defined(__linux__) || defined(__gnu_linux__) + if (unlikely(globals.linux_kernel_version < 0x04000000)) { + /* 2022-09-01: Прошло уже более двух лет после окончания какой-либо + * поддержки самого "долгоиграющего" ядра 3.16.85 ветки 3.x */ + ERROR("too old linux kernel %u.%u.%u.%u, the >= 4.0.0 is required", globals.linux_kernel_version >> 24, + (globals.linux_kernel_version >> 16) & 255, (globals.linux_kernel_version >> 8) & 255, + globals.linux_kernel_version & 255); + return LOG_IFERR(MDBX_INCOMPATIBLE); + } +#endif /* Linux */ - return MDBX_SUCCESS; -} + MDBX_env *env = osal_calloc(1, sizeof(MDBX_env)); + if (unlikely(!env)) + return LOG_IFERR(MDBX_ENOMEM); -int mdbx_txn_renew(MDBX_txn *txn) { - if (unlikely(!txn)) - return MDBX_EINVAL; + env->max_readers = DEFAULT_READERS; + env->max_dbi = env->n_dbi = CORE_DBS; + env->lazy_fd = env->dsync_fd = env->fd4meta = env->lck_mmap.fd = INVALID_HANDLE_VALUE; + env->stuck_meta = -1; - if (unlikely(txn->mt_signature != MDBX_MT_SIGNATURE)) - return MDBX_EBADSIGN; + env_options_init(env); + env_setup_pagesize(env, (globals.sys_pagesize < MDBX_MAX_PAGESIZE) ? globals.sys_pagesize : MDBX_MAX_PAGESIZE); - if (unlikely((txn->mt_flags & MDBX_TXN_RDONLY) == 0)) - return MDBX_EINVAL; + int rc = osal_fastmutex_init(&env->dbi_lock); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; - int rc; - if (unlikely(txn->mt_owner != 0 || !(txn->mt_flags & MDBX_TXN_FINISHED))) { - rc = mdbx_txn_reset(txn); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; +#if defined(_WIN32) || defined(_WIN64) + imports.srwl_Init(&env->remap_guard); + InitializeCriticalSection(&env->windowsbug_lock); +#else + rc = osal_fastmutex_init(&env->remap_guard); + if (unlikely(rc != MDBX_SUCCESS)) { + osal_fastmutex_destroy(&env->dbi_lock); + goto bailout; } - rc = txn_renew(txn, MDBX_TXN_RDONLY); - if (rc == MDBX_SUCCESS) { - tASSERT(txn, txn->mt_owner == osal_thread_self()); - DEBUG("renew txn %" PRIaTXN "%c %p on env %p, root page %" PRIaPGNO - "/%" PRIaPGNO, - txn->mt_txnid, (txn->mt_flags & MDBX_TXN_RDONLY) ? 'r' : 'w', - (void *)txn, (void *)txn->mt_env, txn->mt_dbs[MAIN_DBI].md_root, - txn->mt_dbs[FREE_DBI].md_root); +#if MDBX_LOCKING > MDBX_LOCKING_SYSV + lck_t *const stub = lckless_stub(env); + rc = lck_ipclock_stubinit(&stub->wrt_lock); +#endif /* MDBX_LOCKING */ + if (unlikely(rc != MDBX_SUCCESS)) { + osal_fastmutex_destroy(&env->remap_guard); + osal_fastmutex_destroy(&env->dbi_lock); + goto bailout; } - return rc; +#endif /* Windows */ + + VALGRIND_CREATE_MEMPOOL(env, 0, 0); + env->signature.weak = env_signature; + *penv = env; + return MDBX_SUCCESS; + +bailout: + osal_free(env); + return LOG_IFERR(rc); } -int mdbx_txn_set_userctx(MDBX_txn *txn, void *ctx) { - int rc = check_txn(txn, MDBX_TXN_FINISHED); +__cold int mdbx_env_turn_for_recovery(MDBX_env *env, unsigned target) { + if (unlikely(target >= NUM_METAS)) + return LOG_IFERR(MDBX_EINVAL); + int rc = check_env(env, true); if (unlikely(rc != MDBX_SUCCESS)) - return rc; + return LOG_IFERR(rc); - txn->mt_userctx = ctx; - return MDBX_SUCCESS; -} + if (unlikely((env->flags & (MDBX_EXCLUSIVE | MDBX_RDONLY)) != MDBX_EXCLUSIVE)) + return LOG_IFERR(MDBX_EPERM); + + const meta_t *const target_meta = METAPAGE(env, target); + txnid_t new_txnid = constmeta_txnid(target_meta); + if (new_txnid < MIN_TXNID) + new_txnid = MIN_TXNID; + for (unsigned n = 0; n < NUM_METAS; ++n) { + if (n == target) + continue; + page_t *const page = pgno2page(env, n); + meta_t meta = *page_meta(page); + if (meta_validate(env, &meta, page, n, nullptr) != MDBX_SUCCESS) { + int err = meta_override(env, n, 0, nullptr); + if (unlikely(err != MDBX_SUCCESS)) + return LOG_IFERR(err); + } else { + txnid_t txnid = constmeta_txnid(&meta); + if (new_txnid <= txnid) + new_txnid = safe64_txnid_next(txnid); + } + } -void *mdbx_txn_get_userctx(const MDBX_txn *txn) { - return check_txn(txn, MDBX_TXN_FINISHED) ? nullptr : txn->mt_userctx; + if (unlikely(new_txnid > MAX_TXNID)) { + ERROR("txnid overflow, raise %d", MDBX_TXN_FULL); + return LOG_IFERR(MDBX_TXN_FULL); + } + return LOG_IFERR(meta_override(env, target, new_txnid, target_meta)); } -int mdbx_txn_begin_ex(MDBX_env *env, MDBX_txn *parent, MDBX_txn_flags_t flags, - MDBX_txn **ret, void *context) { - if (unlikely(!ret)) - return MDBX_EINVAL; - *ret = NULL; +__cold int mdbx_env_open_for_recovery(MDBX_env *env, const char *pathname, unsigned target_meta, bool writeable) { +#if defined(_WIN32) || defined(_WIN64) + wchar_t *pathnameW = nullptr; + int rc = osal_mb2w(pathname, &pathnameW); + if (likely(rc == MDBX_SUCCESS)) { + rc = mdbx_env_open_for_recoveryW(env, pathnameW, target_meta, writeable); + osal_free(pathnameW); + } + return LOG_IFERR(rc); +} - if (unlikely((flags & ~MDBX_TXN_RW_BEGIN_FLAGS) && - (parent || (flags & ~MDBX_TXN_RO_BEGIN_FLAGS)))) - return MDBX_EINVAL; +__cold int mdbx_env_open_for_recoveryW(MDBX_env *env, const wchar_t *pathname, unsigned target_meta, bool writeable) { +#endif /* Windows */ - int rc = check_env(env, true); + if (unlikely(target_meta >= NUM_METAS)) + return LOG_IFERR(MDBX_EINVAL); + int rc = check_env(env, false); if (unlikely(rc != MDBX_SUCCESS)) - return rc; - - if (unlikely(env->me_flags & MDBX_RDONLY & - ~flags)) /* write txn in RDONLY env */ - return MDBX_EACCESS; + return LOG_IFERR(rc); + if (unlikely(env->dxb_mmap.base)) + return LOG_IFERR(MDBX_EPERM); - flags |= env->me_flags & MDBX_WRITEMAP; + env->stuck_meta = (int8_t)target_meta; + return +#if defined(_WIN32) || defined(_WIN64) + mdbx_env_openW +#else + mdbx_env_open +#endif /* Windows */ + (env, pathname, writeable ? MDBX_EXCLUSIVE : MDBX_EXCLUSIVE | MDBX_RDONLY, 0); +} - MDBX_txn *txn = nullptr; - if (parent) { - /* Nested transactions: Max 1 child, write txns only, no writemap */ - rc = check_txn_rw(parent, - MDBX_TXN_RDONLY | MDBX_WRITEMAP | MDBX_TXN_BLOCKED); - if (unlikely(rc != MDBX_SUCCESS)) { - if (rc == MDBX_BAD_TXN && (parent->mt_flags & (MDBX_TXN_RDONLY | MDBX_TXN_BLOCKED)) == 0) { - ERROR("%s mode is incompatible with nested transactions", "MDBX_WRITEMAP"); - rc = MDBX_INCOMPATIBLE; - } - return rc; - } +__cold int mdbx_env_delete(const char *pathname, MDBX_env_delete_mode_t mode) { +#if defined(_WIN32) || defined(_WIN64) + wchar_t *pathnameW = nullptr; + int rc = osal_mb2w(pathname, &pathnameW); + if (likely(rc == MDBX_SUCCESS)) { + rc = mdbx_env_deleteW(pathnameW, mode); + osal_free(pathnameW); + } + return LOG_IFERR(rc); +} - if (env->me_options.spill_parent4child_denominator) { - /* Spill dirty-pages of parent to provide dirtyroom for child txn */ - rc = txn_spill(parent, nullptr, - parent->tw.dirtylist->length / - env->me_options.spill_parent4child_denominator); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - } - tASSERT(parent, audit_ex(parent, 0, false) == 0); +__cold int mdbx_env_deleteW(const wchar_t *pathname, MDBX_env_delete_mode_t mode) { +#endif /* Windows */ - flags |= parent->mt_flags & (MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_SPILLS); - } else if (flags & MDBX_TXN_RDONLY) { - if (env->me_txn0 && - unlikely(env->me_txn0->mt_owner == osal_thread_self()) && - (runtime_flags & MDBX_DBG_LEGACY_OVERLAP) == 0) - return MDBX_TXN_OVERLAPPING; - } else { - /* Reuse preallocated write txn. However, do not touch it until - * txn_renew() succeeds, since it currently may be active. */ - txn = env->me_txn0; - goto renew; + switch (mode) { + default: + return LOG_IFERR(MDBX_EINVAL); + case MDBX_ENV_JUST_DELETE: + case MDBX_ENV_ENSURE_UNUSED: + case MDBX_ENV_WAIT_FOR_UNUSED: + break; } - const size_t base = (flags & MDBX_TXN_RDONLY) - ? sizeof(MDBX_txn) - sizeof(txn->tw) + sizeof(txn->to) - : sizeof(MDBX_txn); - const size_t size = - base + env->me_maxdbs * (sizeof(MDBX_db) + sizeof(MDBX_cursor *) + 1); - txn = osal_malloc(size); - if (unlikely(txn == nullptr)) { - DEBUG("calloc: %s", "failed"); - return MDBX_ENOMEM; - } -#if MDBX_DEBUG - memset(txn, 0xCD, size); - VALGRIND_MAKE_MEM_UNDEFINED(txn, size); -#endif /* MDBX_DEBUG */ - MDBX_ANALYSIS_ASSUME(size > base); - memset(txn, 0, - (MDBX_GOOFY_MSVC_STATIC_ANALYZER && base > size) ? size : base); - txn->mt_dbs = ptr_disp(txn, base); - txn->mt_cursors = ptr_disp(txn->mt_dbs, sizeof(MDBX_db) * env->me_maxdbs); -#if MDBX_DEBUG - txn->mt_cursors[FREE_DBI] = nullptr; /* avoid SIGSEGV in an assertion later */ -#endif /* MDBX_DEBUG */ - txn->mt_dbistate = ptr_disp(txn, size - env->me_maxdbs); - txn->mt_dbxs = env->me_dbxs; /* static */ - txn->mt_flags = flags; - txn->mt_env = env; +#ifdef __e2k__ /* https://bugs.mcst.ru/bugzilla/show_bug.cgi?id=6011 */ + MDBX_env *const dummy_env = alloca(sizeof(MDBX_env)); +#else + MDBX_env dummy_env_silo, *const dummy_env = &dummy_env_silo; +#endif + memset(dummy_env, 0, sizeof(*dummy_env)); + dummy_env->flags = (mode == MDBX_ENV_ENSURE_UNUSED) ? MDBX_EXCLUSIVE : MDBX_ENV_DEFAULTS; + dummy_env->ps = (unsigned)mdbx_default_pagesize(); - if (parent) { - tASSERT(parent, dirtylist_check(parent)); - txn->mt_dbiseqs = parent->mt_dbiseqs; - txn->mt_geo = parent->mt_geo; - rc = dpl_alloc(txn); - if (likely(rc == MDBX_SUCCESS)) { - const size_t len = - MDBX_PNL_GETSIZE(parent->tw.relist) + parent->tw.loose_count; - txn->tw.relist = - pnl_alloc((len > MDBX_PNL_INITIAL) ? len : MDBX_PNL_INITIAL); - if (unlikely(!txn->tw.relist)) - rc = MDBX_ENOMEM; - } - if (unlikely(rc != MDBX_SUCCESS)) { - nested_failed: - pnl_free(txn->tw.relist); - dpl_free(txn); - osal_free(txn); - return rc; + STATIC_ASSERT(sizeof(dummy_env->flags) == sizeof(MDBX_env_flags_t)); + int rc = MDBX_RESULT_TRUE, err = env_handle_pathname(dummy_env, pathname, 0); + if (likely(err == MDBX_SUCCESS)) { + mdbx_filehandle_t clk_handle = INVALID_HANDLE_VALUE, dxb_handle = INVALID_HANDLE_VALUE; + if (mode > MDBX_ENV_JUST_DELETE) { + err = osal_openfile(MDBX_OPEN_DELETE, dummy_env, dummy_env->pathname.dxb, &dxb_handle, 0); + err = (err == MDBX_ENOFILE) ? MDBX_SUCCESS : err; + if (err == MDBX_SUCCESS) { + err = osal_openfile(MDBX_OPEN_DELETE, dummy_env, dummy_env->pathname.lck, &clk_handle, 0); + err = (err == MDBX_ENOFILE) ? MDBX_SUCCESS : err; + } + if (err == MDBX_SUCCESS && clk_handle != INVALID_HANDLE_VALUE) + err = osal_lockfile(clk_handle, mode == MDBX_ENV_WAIT_FOR_UNUSED); + if (err == MDBX_SUCCESS && dxb_handle != INVALID_HANDLE_VALUE) + err = osal_lockfile(dxb_handle, mode == MDBX_ENV_WAIT_FOR_UNUSED); } - /* Move loose pages to reclaimed list */ - if (parent->tw.loose_count) { - do { - MDBX_page *lp = parent->tw.loose_pages; - tASSERT(parent, lp->mp_flags == P_LOOSE); - rc = pnl_insert_range(&parent->tw.relist, lp->mp_pgno, 1); - if (unlikely(rc != MDBX_SUCCESS)) - goto nested_failed; - MDBX_ASAN_UNPOISON_MEMORY_REGION(&mp_next(lp), sizeof(MDBX_page *)); - VALGRIND_MAKE_MEM_DEFINED(&mp_next(lp), sizeof(MDBX_page *)); - parent->tw.loose_pages = mp_next(lp); - /* Remove from dirty list */ - page_wash(parent, dpl_exist(parent, lp->mp_pgno), lp, 1); - } while (parent->tw.loose_pages); - parent->tw.loose_count = 0; -#if MDBX_ENABLE_REFUND - parent->tw.loose_refund_wl = 0; -#endif /* MDBX_ENABLE_REFUND */ - tASSERT(parent, dirtylist_check(parent)); + if (err == MDBX_SUCCESS) { + err = osal_removefile(dummy_env->pathname.dxb); + if (err == MDBX_SUCCESS) + rc = MDBX_SUCCESS; + else if (err == MDBX_ENOFILE) + err = MDBX_SUCCESS; } - txn->tw.dirtyroom = parent->tw.dirtyroom; - txn->tw.dirtylru = parent->tw.dirtylru; - dpl_sort(parent); - if (parent->tw.spilled.list) - spill_purge(parent); - - tASSERT(txn, MDBX_PNL_ALLOCLEN(txn->tw.relist) >= - MDBX_PNL_GETSIZE(parent->tw.relist)); - memcpy(txn->tw.relist, parent->tw.relist, - MDBX_PNL_SIZEOF(parent->tw.relist)); - eASSERT(env, pnl_check_allocated( - txn->tw.relist, - (txn->mt_next_pgno /* LY: intentional assignment here, - only for assertion */ - = parent->mt_next_pgno) - - MDBX_ENABLE_REFUND)); + if (err == MDBX_SUCCESS) { + err = osal_removefile(dummy_env->pathname.lck); + if (err == MDBX_SUCCESS) + rc = MDBX_SUCCESS; + else if (err == MDBX_ENOFILE) + err = MDBX_SUCCESS; + } - txn->tw.last_reclaimed = parent->tw.last_reclaimed; - if (parent->tw.lifo_reclaimed) { - txn->tw.lifo_reclaimed = parent->tw.lifo_reclaimed; - parent->tw.lifo_reclaimed = - (void *)(intptr_t)MDBX_PNL_GETSIZE(parent->tw.lifo_reclaimed); + if (err == MDBX_SUCCESS && !(dummy_env->flags & MDBX_NOSUBDIR) && + (/* pathname != "." */ pathname[0] != '.' || pathname[1] != 0) && + (/* pathname != ".." */ pathname[0] != '.' || pathname[1] != '.' || pathname[2] != 0)) { + err = osal_removedirectory(pathname); + if (err == MDBX_SUCCESS) + rc = MDBX_SUCCESS; + else if (err == MDBX_ENOFILE) + err = MDBX_SUCCESS; } - txn->tw.retired_pages = parent->tw.retired_pages; - parent->tw.retired_pages = - (void *)(intptr_t)MDBX_PNL_GETSIZE(parent->tw.retired_pages); + if (dxb_handle != INVALID_HANDLE_VALUE) + osal_closefile(dxb_handle); + if (clk_handle != INVALID_HANDLE_VALUE) + osal_closefile(clk_handle); + } else if (err == MDBX_ENOFILE) + err = MDBX_SUCCESS; - txn->mt_txnid = parent->mt_txnid; - txn->mt_front = parent->mt_front + 1; -#if MDBX_ENABLE_REFUND - txn->tw.loose_refund_wl = 0; -#endif /* MDBX_ENABLE_REFUND */ - txn->mt_canary = parent->mt_canary; - parent->mt_flags |= MDBX_TXN_HAS_CHILD; - parent->mt_child = txn; - txn->mt_parent = parent; - txn->mt_numdbs = parent->mt_numdbs; - txn->mt_owner = parent->mt_owner; - memcpy(txn->mt_dbs, parent->mt_dbs, txn->mt_numdbs * sizeof(MDBX_db)); - txn->tw.troika = parent->tw.troika; - /* Copy parent's mt_dbistate, but clear DB_NEW */ - for (size_t i = 0; i < txn->mt_numdbs; i++) - txn->mt_dbistate[i] = - parent->mt_dbistate[i] & ~(DBI_FRESH | DBI_CREAT | DBI_DIRTY); - tASSERT(parent, - parent->tw.dirtyroom + parent->tw.dirtylist->length == - (parent->mt_parent ? parent->mt_parent->tw.dirtyroom - : parent->mt_env->me_options.dp_limit)); - tASSERT(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length == - (txn->mt_parent ? txn->mt_parent->tw.dirtyroom - : txn->mt_env->me_options.dp_limit)); - env->me_txn = txn; - rc = cursor_shadow(parent, txn); - if (AUDIT_ENABLED() && ASSERT_ENABLED()) { - txn->mt_signature = MDBX_MT_SIGNATURE; - tASSERT(txn, audit_ex(txn, 0, false) == 0); - } - if (unlikely(rc != MDBX_SUCCESS)) - txn_end(txn, MDBX_END_FAIL_BEGINCHILD); - } else { /* MDBX_TXN_RDONLY */ - txn->mt_dbiseqs = env->me_dbiseqs; - renew: - rc = txn_renew(txn, flags); + osal_free(dummy_env->pathname.buffer); + return LOG_IFERR((err == MDBX_SUCCESS) ? rc : err); +} + +__cold int mdbx_env_open(MDBX_env *env, const char *pathname, MDBX_env_flags_t flags, mdbx_mode_t mode) { +#if defined(_WIN32) || defined(_WIN64) + wchar_t *pathnameW = nullptr; + int rc = osal_mb2w(pathname, &pathnameW); + if (likely(rc == MDBX_SUCCESS)) { + rc = mdbx_env_openW(env, pathnameW, flags, mode); + osal_free(pathnameW); + if (rc == MDBX_SUCCESS) + /* force to make cache of the multi-byte pathname representation */ + mdbx_env_get_path(env, &pathname); } + return LOG_IFERR(rc); +} - if (unlikely(rc != MDBX_SUCCESS)) { - if (txn != env->me_txn0) - osal_free(txn); +__cold int mdbx_env_openW(MDBX_env *env, const wchar_t *pathname, MDBX_env_flags_t flags, mdbx_mode_t mode) { +#endif /* Windows */ + + int rc = check_env(env, false); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); + + if (unlikely(flags & ~ENV_USABLE_FLAGS)) + return LOG_IFERR(MDBX_EINVAL); + + if (unlikely(env->lazy_fd != INVALID_HANDLE_VALUE || (env->flags & ENV_ACTIVE) != 0 || env->dxb_mmap.base)) + return LOG_IFERR(MDBX_EPERM); + + /* Pickup previously mdbx_env_set_flags(), + * but avoid MDBX_UTTERLY_NOSYNC by disjunction */ + const uint32_t saved_me_flags = env->flags; + flags = combine_durability_flags(flags | DEPRECATED_COALESCE, env->flags); + + if (flags & MDBX_RDONLY) { + /* Silently ignore irrelevant flags when we're only getting read access */ + flags &= ~(MDBX_WRITEMAP | DEPRECATED_MAPASYNC | MDBX_SAFE_NOSYNC | MDBX_NOMETASYNC | DEPRECATED_COALESCE | + MDBX_LIFORECLAIM | MDBX_NOMEMINIT | MDBX_ACCEDE); + mode = 0; } else { - if (flags & (MDBX_TXN_RDONLY_PREPARE - MDBX_TXN_RDONLY)) - eASSERT(env, txn->mt_flags == (MDBX_TXN_RDONLY | MDBX_TXN_FINISHED)); - else if (flags & MDBX_TXN_RDONLY) - eASSERT(env, (txn->mt_flags & - ~(MDBX_NOTLS | MDBX_TXN_RDONLY | MDBX_WRITEMAP | - /* Win32: SRWL flag */ MDBX_SHRINK_ALLOWED)) == 0); - else { - eASSERT(env, (txn->mt_flags & - ~(MDBX_WRITEMAP | MDBX_SHRINK_ALLOWED | MDBX_NOMETASYNC | - MDBX_SAFE_NOSYNC | MDBX_TXN_SPILLS)) == 0); - assert(!txn->tw.spilled.list && !txn->tw.spilled.least_removed); +#if MDBX_MMAP_INCOHERENT_FILE_WRITE + /* Temporary `workaround` for OpenBSD kernel's flaw. + * See https://libmdbx.dqdkfa.ru/dead-github/issues/67 */ + if ((flags & MDBX_WRITEMAP) == 0) { + if (flags & MDBX_ACCEDE) + flags |= MDBX_WRITEMAP; + else { + debug_log(MDBX_LOG_ERROR, __func__, __LINE__, + "System (i.e. OpenBSD) requires MDBX_WRITEMAP because " + "of an internal flaw(s) in a file/buffer/page cache.\n"); + return LOG_IFERR(42 /* ENOPROTOOPT */); + } } - txn->mt_signature = MDBX_MT_SIGNATURE; - txn->mt_userctx = context; - *ret = txn; - DEBUG("begin txn %" PRIaTXN "%c %p on env %p, root page %" PRIaPGNO - "/%" PRIaPGNO, - txn->mt_txnid, (flags & MDBX_TXN_RDONLY) ? 'r' : 'w', (void *)txn, - (void *)env, txn->mt_dbs[MAIN_DBI].md_root, - txn->mt_dbs[FREE_DBI].md_root); +#endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */ } - return rc; -} - -int mdbx_txn_info(const MDBX_txn *txn, MDBX_txn_info *info, bool scan_rlt) { - int rc = check_txn(txn, MDBX_TXN_FINISHED); + env->flags = (flags & ~ENV_FATAL_ERROR); + rc = env_handle_pathname(env, pathname, mode); if (unlikely(rc != MDBX_SUCCESS)) - return rc; + goto bailout; - if (unlikely(!info)) - return MDBX_EINVAL; + env->kvs = osal_calloc(env->max_dbi, sizeof(env->kvs[0])); + env->dbs_flags = osal_calloc(env->max_dbi, sizeof(env->dbs_flags[0])); + env->dbi_seqs = osal_calloc(env->max_dbi, sizeof(env->dbi_seqs[0])); + if (unlikely(!(env->kvs && env->dbs_flags && env->dbi_seqs))) { + rc = MDBX_ENOMEM; + goto bailout; + } - MDBX_env *const env = txn->mt_env; -#if MDBX_ENV_CHECKPID - if (unlikely(env->me_pid != osal_getpid())) { - env->me_flags |= MDBX_FATAL_ERROR; - return MDBX_PANIC; + if ((flags & MDBX_RDONLY) == 0) { + MDBX_txn *txn = nullptr; + const intptr_t bitmap_bytes = +#if MDBX_ENABLE_DBI_SPARSE + ceil_powerof2(env->max_dbi, CHAR_BIT * sizeof(txn->dbi_sparse[0])) / CHAR_BIT; +#else + 0; +#endif /* MDBX_ENABLE_DBI_SPARSE */ + const size_t base = sizeof(MDBX_txn) + sizeof(cursor_couple_t); + const size_t size = base + bitmap_bytes + + env->max_dbi * (sizeof(txn->dbs[0]) + sizeof(txn->cursors[0]) + sizeof(txn->dbi_seqs[0]) + + sizeof(txn->dbi_state[0])); + + txn = osal_calloc(1, size); + if (unlikely(!txn)) { + rc = MDBX_ENOMEM; + goto bailout; + } + txn->dbs = ptr_disp(txn, base); + txn->cursors = ptr_disp(txn->dbs, env->max_dbi * sizeof(txn->dbs[0])); + txn->dbi_seqs = ptr_disp(txn->cursors, env->max_dbi * sizeof(txn->cursors[0])); + txn->dbi_state = ptr_disp(txn, size - env->max_dbi * sizeof(txn->dbi_state[0])); +#if MDBX_ENABLE_DBI_SPARSE + txn->dbi_sparse = ptr_disp(txn->dbi_state, -bitmap_bytes); +#endif /* MDBX_ENABLE_DBI_SPARSE */ + txn->env = env; + txn->flags = MDBX_TXN_FINISHED; + env->basal_txn = txn; + txn->tw.retired_pages = pnl_alloc(MDBX_PNL_INITIAL); + txn->tw.repnl = pnl_alloc(MDBX_PNL_INITIAL); + if (unlikely(!txn->tw.retired_pages || !txn->tw.repnl)) { + rc = MDBX_ENOMEM; + goto bailout; + } + env_options_adjust_defaults(env); } -#endif /* MDBX_ENV_CHECKPID */ - info->txn_id = txn->mt_txnid; - info->txn_space_used = pgno2bytes(env, txn->mt_geo.next); + rc = env_open(env, mode); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; - if (txn->mt_flags & MDBX_TXN_RDONLY) { - meta_ptr_t head; - uint64_t head_retired; - meta_troika_t troika = meta_tap(env); - do { - /* fetch info from volatile head */ - head = meta_recent(env, &troika); - head_retired = - unaligned_peek_u64_volatile(4, head.ptr_v->mm_pages_retired); - info->txn_space_limit_soft = pgno2bytes(env, head.ptr_v->mm_geo.now); - info->txn_space_limit_hard = pgno2bytes(env, head.ptr_v->mm_geo.upper); - info->txn_space_leftover = - pgno2bytes(env, head.ptr_v->mm_geo.now - head.ptr_v->mm_geo.next); - } while (unlikely(meta_should_retry(env, &troika))); - - info->txn_reader_lag = head.txnid - info->txn_id; - info->txn_space_dirty = info->txn_space_retired = 0; - uint64_t reader_snapshot_pages_retired; - if (txn->to.reader && - head_retired > - (reader_snapshot_pages_retired = atomic_load64( - &txn->to.reader->mr_snapshot_pages_retired, mo_Relaxed))) { - info->txn_space_dirty = info->txn_space_retired = pgno2bytes( - env, (pgno_t)(head_retired - reader_snapshot_pages_retired)); +#if MDBX_DEBUG + const troika_t troika = meta_tap(env); + const meta_ptr_t head = meta_recent(env, &troika); + const tree_t *db = &head.ptr_c->trees.main; + + DEBUG("opened database version %u, pagesize %u", (uint8_t)unaligned_peek_u64(4, head.ptr_c->magic_and_version), + env->ps); + DEBUG("using meta page %" PRIaPGNO ", txn %" PRIaTXN, data_page(head.ptr_c)->pgno, head.txnid); + DEBUG("depth: %u", db->height); + DEBUG("entries: %" PRIu64, db->items); + DEBUG("branch pages: %" PRIaPGNO, db->branch_pages); + DEBUG("leaf pages: %" PRIaPGNO, db->leaf_pages); + DEBUG("large/overflow pages: %" PRIaPGNO, db->large_pages); + DEBUG("root: %" PRIaPGNO, db->root); + DEBUG("schema_altered: %" PRIaTXN, db->mod_txnid); +#endif /* MDBX_DEBUG */ - size_t retired_next_reader = 0; - MDBX_lockinfo *const lck = env->me_lck_mmap.lck; - if (scan_rlt && info->txn_reader_lag > 1 && lck) { - /* find next more recent reader */ - txnid_t next_reader = head.txnid; - const size_t snap_nreaders = - atomic_load32(&lck->mti_numreaders, mo_AcquireRelease); - for (size_t i = 0; i < snap_nreaders; ++i) { - retry: - if (atomic_load32(&lck->mti_readers[i].mr_pid, mo_AcquireRelease)) { - jitter4testing(true); - const txnid_t snap_txnid = - safe64_read(&lck->mti_readers[i].mr_txnid); - const uint64_t snap_retired = - atomic_load64(&lck->mti_readers[i].mr_snapshot_pages_retired, - mo_AcquireRelease); - if (unlikely(snap_retired != - atomic_load64( - &lck->mti_readers[i].mr_snapshot_pages_retired, - mo_Relaxed)) || - snap_txnid != safe64_read(&lck->mti_readers[i].mr_txnid)) - goto retry; - if (snap_txnid <= txn->mt_txnid) { - retired_next_reader = 0; - break; - } - if (snap_txnid < next_reader) { - next_reader = snap_txnid; - retired_next_reader = pgno2bytes( - env, (pgno_t)(snap_retired - - atomic_load64( - &txn->to.reader->mr_snapshot_pages_retired, - mo_Relaxed))); - } - } - } - } - info->txn_space_dirty = retired_next_reader; - } + if (likely(rc == MDBX_SUCCESS)) { + dxb_sanitize_tail(env, nullptr); } else { - info->txn_space_limit_soft = pgno2bytes(env, txn->mt_geo.now); - info->txn_space_limit_hard = pgno2bytes(env, txn->mt_geo.upper); - info->txn_space_retired = pgno2bytes( - env, txn->mt_child ? (size_t)txn->tw.retired_pages - : MDBX_PNL_GETSIZE(txn->tw.retired_pages)); - info->txn_space_leftover = pgno2bytes(env, txn->tw.dirtyroom); - info->txn_space_dirty = pgno2bytes( - env, txn->tw.dirtylist ? txn->tw.dirtylist->pages_including_loose - : (txn->tw.writemap_dirty_npages + - txn->tw.writemap_spilled_npages)); - info->txn_reader_lag = INT64_MAX; - MDBX_lockinfo *const lck = env->me_lck_mmap.lck; - if (scan_rlt && lck) { - txnid_t oldest_snapshot = txn->mt_txnid; - const size_t snap_nreaders = - atomic_load32(&lck->mti_numreaders, mo_AcquireRelease); - if (snap_nreaders) { - oldest_snapshot = txn_oldest_reader(txn); - if (oldest_snapshot == txn->mt_txnid - 1) { - /* check if there is at least one reader */ - bool exists = false; - for (size_t i = 0; i < snap_nreaders; ++i) { - if (atomic_load32(&lck->mti_readers[i].mr_pid, mo_Relaxed) && - txn->mt_txnid > safe64_read(&lck->mti_readers[i].mr_txnid)) { - exists = true; - break; - } - } - oldest_snapshot += !exists; - } - } - info->txn_reader_lag = txn->mt_txnid - oldest_snapshot; + bailout: + if (likely(env_close(env, false) == MDBX_SUCCESS)) { + env->flags = saved_me_flags; + } else { + rc = MDBX_PANIC; + env->flags = saved_me_flags | ENV_FATAL_ERROR; } } - - return MDBX_SUCCESS; + return LOG_IFERR(rc); } -MDBX_env *mdbx_txn_env(const MDBX_txn *txn) { - if (unlikely(!txn || txn->mt_signature != MDBX_MT_SIGNATURE || - txn->mt_env->me_signature.weak != MDBX_ME_SIGNATURE)) - return NULL; - return txn->mt_env; -} +/*----------------------------------------------------------------------------*/ -uint64_t mdbx_txn_id(const MDBX_txn *txn) { - if (unlikely(!txn || txn->mt_signature != MDBX_MT_SIGNATURE)) - return 0; - return txn->mt_txnid; -} +#if !(defined(_WIN32) || defined(_WIN64)) +__cold int mdbx_env_resurrect_after_fork(MDBX_env *env) { + if (unlikely(!env)) + return LOG_IFERR(MDBX_EINVAL); -int mdbx_txn_flags(const MDBX_txn *txn) { - if (unlikely(!txn || txn->mt_signature != MDBX_MT_SIGNATURE)) { - assert((-1 & (int)MDBX_TXN_INVALID) != 0); - return -1; - } - assert(0 == (int)(txn->mt_flags & MDBX_TXN_INVALID)); - return txn->mt_flags; -} + if (unlikely(env->signature.weak != env_signature)) + return LOG_IFERR(MDBX_EBADSIGN); -/* Check for misused dbi handles */ -static __inline bool dbi_changed(const MDBX_txn *txn, size_t dbi) { - if (txn->mt_dbiseqs == txn->mt_env->me_dbiseqs) - return false; - if (likely( - txn->mt_dbiseqs[dbi].weak == - atomic_load32((MDBX_atomic_uint32_t *)&txn->mt_env->me_dbiseqs[dbi], - mo_AcquireRelease))) - return false; - return true; -} + if (unlikely(env->flags & ENV_FATAL_ERROR)) + return LOG_IFERR(MDBX_PANIC); -static __inline unsigned dbi_seq(const MDBX_env *const env, size_t slot) { - unsigned v = env->me_dbiseqs[slot].weak + 1; - return v + (v == 0); -} - -static void dbi_import_locked(MDBX_txn *txn) { - const MDBX_env *const env = txn->mt_env; - size_t n = env->me_numdbs; - for (size_t i = CORE_DBS; i < n; ++i) { - if (i >= txn->mt_numdbs) { - txn->mt_cursors[i] = NULL; - if (txn->mt_dbiseqs != env->me_dbiseqs) - txn->mt_dbiseqs[i].weak = 0; - txn->mt_dbistate[i] = 0; - } - if ((dbi_changed(txn, i) && - (txn->mt_dbistate[i] & (DBI_CREAT | DBI_DIRTY | DBI_FRESH)) == 0) || - ((env->me_dbflags[i] & DB_VALID) && - !(txn->mt_dbistate[i] & DBI_VALID))) { - tASSERT(txn, - (txn->mt_dbistate[i] & (DBI_CREAT | DBI_DIRTY | DBI_FRESH)) == 0); - txn->mt_dbiseqs[i] = env->me_dbiseqs[i]; - txn->mt_dbs[i].md_flags = env->me_dbflags[i] & DB_PERSISTENT_FLAGS; - txn->mt_dbistate[i] = 0; - if (env->me_dbflags[i] & DB_VALID) { - txn->mt_dbistate[i] = DBI_VALID | DBI_USRVALID | DBI_STALE; - tASSERT(txn, txn->mt_dbxs[i].md_cmp != NULL); - tASSERT(txn, txn->mt_dbxs[i].md_name.iov_base != NULL); - } - } - } - while (unlikely(n < txn->mt_numdbs)) - if (txn->mt_cursors[txn->mt_numdbs - 1] == NULL && - (txn->mt_dbistate[txn->mt_numdbs - 1] & DBI_USRVALID) == 0) - txn->mt_numdbs -= 1; - else { - if ((txn->mt_dbistate[n] & DBI_USRVALID) == 0) { - if (txn->mt_dbiseqs != env->me_dbiseqs) - txn->mt_dbiseqs[n].weak = 0; - txn->mt_dbistate[n] = 0; - } - ++n; + if (unlikely((env->flags & ENV_ACTIVE) == 0)) + return MDBX_SUCCESS; + + const uint32_t new_pid = osal_getpid(); + if (unlikely(env->pid == new_pid)) + return MDBX_SUCCESS; + + if (!atomic_cas32(&env->signature, env_signature, ~env_signature)) + return LOG_IFERR(MDBX_EBADSIGN); + + if (env->txn) + txn_abort(env->basal_txn); + env->registered_reader_pid = 0; + int rc = env_close(env, true); + env->signature.weak = env_signature; + if (likely(rc == MDBX_SUCCESS)) { + rc = (env->flags & MDBX_EXCLUSIVE) ? MDBX_BUSY : env_open(env, 0); + if (unlikely(rc != MDBX_SUCCESS && env_close(env, false) != MDBX_SUCCESS)) { + rc = MDBX_PANIC; + env->flags |= ENV_FATAL_ERROR; } - txn->mt_numdbs = (MDBX_dbi)n; + } + return LOG_IFERR(rc); } +#endif /* Windows */ -/* Import DBI which opened after txn started into context */ -__cold static bool dbi_import(MDBX_txn *txn, MDBX_dbi dbi) { - if (dbi < CORE_DBS || - (dbi >= txn->mt_numdbs && dbi >= txn->mt_env->me_numdbs)) - return false; +__cold int mdbx_env_close_ex(MDBX_env *env, bool dont_sync) { + page_t *dp; + int rc = MDBX_SUCCESS; - ENSURE(txn->mt_env, - osal_fastmutex_acquire(&txn->mt_env->me_dbi_lock) == MDBX_SUCCESS); - dbi_import_locked(txn); - ENSURE(txn->mt_env, - osal_fastmutex_release(&txn->mt_env->me_dbi_lock) == MDBX_SUCCESS); - return txn->mt_dbistate[dbi] & DBI_USRVALID; -} + if (unlikely(!env)) + return LOG_IFERR(MDBX_EINVAL); -/* Export or close DBI handles opened in this txn. */ -static void dbi_update(MDBX_txn *txn, int keep) { - tASSERT(txn, !txn->mt_parent && txn == txn->mt_env->me_txn0); - MDBX_dbi n = txn->mt_numdbs; - if (n) { - bool locked = false; - MDBX_env *const env = txn->mt_env; - - for (size_t i = n; --i >= CORE_DBS;) { - if (likely((txn->mt_dbistate[i] & DBI_CREAT) == 0)) - continue; - if (!locked) { - ENSURE(env, osal_fastmutex_acquire(&env->me_dbi_lock) == MDBX_SUCCESS); - locked = true; - } - if (env->me_numdbs <= i || - txn->mt_dbiseqs[i].weak != env->me_dbiseqs[i].weak) - continue /* dbi explicitly closed and/or then re-opened by other txn */; - if (keep) { - env->me_dbflags[i] = txn->mt_dbs[i].md_flags | DB_VALID; - } else { - const MDBX_val name = env->me_dbxs[i].md_name; - if (name.iov_base) { - env->me_dbxs[i].md_name.iov_base = nullptr; - eASSERT(env, env->me_dbflags[i] == 0); - atomic_store32(&env->me_dbiseqs[i], dbi_seq(env, i), - mo_AcquireRelease); - env->me_dbxs[i].md_name.iov_len = 0; - if (name.iov_len) - osal_free(name.iov_base); - } else { - eASSERT(env, name.iov_len == 0); - eASSERT(env, env->me_dbflags[i] == 0); - } - } - } + if (unlikely(env->signature.weak != env_signature)) + return LOG_IFERR(MDBX_EBADSIGN); - n = env->me_numdbs; - if (n > CORE_DBS && unlikely(!(env->me_dbflags[n - 1] & DB_VALID))) { - if (!locked) { - ENSURE(env, osal_fastmutex_acquire(&env->me_dbi_lock) == MDBX_SUCCESS); - locked = true; - } +#if MDBX_ENV_CHECKPID || !(defined(_WIN32) || defined(_WIN64)) + /* Check the PID even if MDBX_ENV_CHECKPID=0 on non-Windows + * platforms (i.e. where fork() is available). + * This is required to legitimize a call after fork() + * from a child process, that should be allowed to free resources. */ + if (unlikely(env->pid != osal_getpid())) + env->flags |= ENV_FATAL_ERROR; +#endif /* MDBX_ENV_CHECKPID */ - n = env->me_numdbs; - while (n > CORE_DBS && !(env->me_dbflags[n - 1] & DB_VALID)) - --n; - env->me_numdbs = n; - } + if (env->dxb_mmap.base && (env->flags & (MDBX_RDONLY | ENV_FATAL_ERROR)) == 0 && env->basal_txn) { + if (env->basal_txn->owner && env->basal_txn->owner != osal_thread_self()) + return LOG_IFERR(MDBX_BUSY); + } else + dont_sync = true; - if (unlikely(locked)) - ENSURE(env, osal_fastmutex_release(&env->me_dbi_lock) == MDBX_SUCCESS); + if (!atomic_cas32(&env->signature, env_signature, 0)) + return LOG_IFERR(MDBX_EBADSIGN); + + if (!dont_sync) { +#if defined(_WIN32) || defined(_WIN64) + /* On windows, without blocking is impossible to determine whether another + * process is running a writing transaction or not. + * Because in the "owner died" condition kernel don't release + * file lock immediately. */ + rc = env_sync(env, true, false); + rc = (rc == MDBX_RESULT_TRUE) ? MDBX_SUCCESS : rc; +#else + struct stat st; + if (unlikely(fstat(env->lazy_fd, &st))) + rc = errno; + else if (st.st_nlink > 0 /* don't sync deleted files */) { + rc = env_sync(env, true, true); + rc = (rc == MDBX_BUSY || rc == EAGAIN || rc == EACCES || rc == EBUSY || rc == EWOULDBLOCK || + rc == MDBX_RESULT_TRUE) + ? MDBX_SUCCESS + : rc; + } +#endif /* Windows */ } -} -/* Filter-out pgno list from transaction's dirty-page list */ -static void dpl_sift(MDBX_txn *const txn, MDBX_PNL pl, const bool spilled) { - tASSERT(txn, (txn->mt_flags & MDBX_TXN_RDONLY) == 0); - tASSERT(txn, (txn->mt_flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); - if (MDBX_PNL_GETSIZE(pl) && txn->tw.dirtylist->length) { - tASSERT(txn, pnl_check_allocated(pl, (size_t)txn->mt_next_pgno << spilled)); - MDBX_dpl *dl = dpl_sort(txn); + if (env->basal_txn && (MDBX_TXN_CHECKOWNER ? env->basal_txn->owner == osal_thread_self() : !!env->basal_txn->owner)) + lck_txn_unlock(env); - /* Scanning in ascend order */ - const intptr_t step = MDBX_PNL_ASCENDING ? 1 : -1; - const intptr_t begin = MDBX_PNL_ASCENDING ? 1 : MDBX_PNL_GETSIZE(pl); - const intptr_t end = MDBX_PNL_ASCENDING ? MDBX_PNL_GETSIZE(pl) + 1 : 0; - tASSERT(txn, pl[begin] <= pl[end - step]); + eASSERT(env, env->signature.weak == 0); + rc = env_close(env, false) ? MDBX_PANIC : rc; + ENSURE(env, osal_fastmutex_destroy(&env->dbi_lock) == MDBX_SUCCESS); +#if defined(_WIN32) || defined(_WIN64) + /* remap_guard don't have destructor (Slim Reader/Writer Lock) */ + DeleteCriticalSection(&env->windowsbug_lock); +#else + ENSURE(env, osal_fastmutex_destroy(&env->remap_guard) == MDBX_SUCCESS); +#endif /* Windows */ - size_t w, r = dpl_search(txn, pl[begin] >> spilled); - tASSERT(txn, dl->sorted == dl->length); - for (intptr_t i = begin; r <= dl->length;) { /* scan loop */ - assert(i != end); - tASSERT(txn, !spilled || (pl[i] & 1) == 0); - pgno_t pl_pgno = pl[i] >> spilled; - pgno_t dp_pgno = dl->items[r].pgno; - if (likely(dp_pgno != pl_pgno)) { - const bool cmp = dp_pgno < pl_pgno; - r += cmp; - i += cmp ? 0 : step; - if (likely(i != end)) - continue; - return; - } +#if MDBX_LOCKING > MDBX_LOCKING_SYSV + lck_t *const stub = lckless_stub(env); + /* может вернуть ошибку в дочернем процессе после fork() */ + lck_ipclock_destroy(&stub->wrt_lock); +#endif /* MDBX_LOCKING */ - /* update loop */ - unsigned npages; - w = r; - remove_dl: - npages = dpl_npages(dl, r); - dl->pages_including_loose -= npages; - if (!MDBX_AVOID_MSYNC || !(txn->mt_flags & MDBX_WRITEMAP)) - dpage_free(txn->mt_env, dl->items[r].ptr, npages); - ++r; - next_i: - i += step; - if (unlikely(i == end)) { - while (r <= dl->length) - dl->items[w++] = dl->items[r++]; - } else { - while (r <= dl->length) { - assert(i != end); - tASSERT(txn, !spilled || (pl[i] & 1) == 0); - pl_pgno = pl[i] >> spilled; - dp_pgno = dl->items[r].pgno; - if (dp_pgno < pl_pgno) - dl->items[w++] = dl->items[r++]; - else if (dp_pgno > pl_pgno) - goto next_i; - else - goto remove_dl; - } - } - dl->sorted = dpl_setlen(dl, w - 1); - txn->tw.dirtyroom += r - w; - tASSERT(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length == - (txn->mt_parent ? txn->mt_parent->tw.dirtyroom - : txn->mt_env->me_options.dp_limit)); - return; - } + while ((dp = env->shadow_reserve) != nullptr) { + MDBX_ASAN_UNPOISON_MEMORY_REGION(dp, env->ps); + VALGRIND_MAKE_MEM_DEFINED(&page_next(dp), sizeof(page_t *)); + env->shadow_reserve = page_next(dp); + void *const ptr = ptr_disp(dp, -(ptrdiff_t)sizeof(size_t)); + osal_free(ptr); } + VALGRIND_DESTROY_MEMPOOL(env); + osal_free(env); + + return LOG_IFERR(rc); } -/* End a transaction, except successful commit of a nested transaction. - * May be called twice for readonly txns: First reset it, then abort. - * [in] txn the transaction handle to end - * [in] mode why and how to end the transaction */ -static int txn_end(MDBX_txn *txn, const unsigned mode) { - MDBX_env *env = txn->mt_env; - static const char *const names[] = MDBX_END_NAMES; +/*----------------------------------------------------------------------------*/ -#if MDBX_ENV_CHECKPID - if (unlikely(txn->mt_env->me_pid != osal_getpid())) { - env->me_flags |= MDBX_FATAL_ERROR; +static int env_info_snap(const MDBX_env *env, const MDBX_txn *txn, MDBX_envinfo *out, const size_t bytes, + troika_t *const troika) { + const size_t size_before_bootid = offsetof(MDBX_envinfo, mi_bootid); + const size_t size_before_pgop_stat = offsetof(MDBX_envinfo, mi_pgop_stat); + const size_t size_before_dxbid = offsetof(MDBX_envinfo, mi_dxbid); + if (unlikely(env->flags & ENV_FATAL_ERROR)) return MDBX_PANIC; - } -#endif /* MDBX_ENV_CHECKPID */ - DEBUG("%s txn %" PRIaTXN "%c %p on mdbenv %p, root page %" PRIaPGNO - "/%" PRIaPGNO, - names[mode & MDBX_END_OPMASK], txn->mt_txnid, - (txn->mt_flags & MDBX_TXN_RDONLY) ? 'r' : 'w', (void *)txn, (void *)env, - txn->mt_dbs[MAIN_DBI].md_root, txn->mt_dbs[FREE_DBI].md_root); + /* is the environment open? + * (https://libmdbx.dqdkfa.ru/dead-github/issues/171) */ + if (unlikely(!env->dxb_mmap.base)) { + /* environment not yet opened */ +#if 1 + /* default behavior: returns the available info but zeroed the rest */ + memset(out, 0, bytes); + out->mi_geo.lower = env->geo_in_bytes.lower; + out->mi_geo.upper = env->geo_in_bytes.upper; + out->mi_geo.shrink = env->geo_in_bytes.shrink; + out->mi_geo.grow = env->geo_in_bytes.grow; + out->mi_geo.current = env->geo_in_bytes.now; + out->mi_maxreaders = env->max_readers; + out->mi_dxb_pagesize = env->ps; + out->mi_sys_pagesize = globals.sys_pagesize; + if (likely(bytes > size_before_bootid)) { + out->mi_bootid.current.x = globals.bootid.x; + out->mi_bootid.current.y = globals.bootid.y; + } + return MDBX_SUCCESS; +#else + /* some users may prefer this behavior: return appropriate error */ + return MDBX_EPERM; +#endif + } - if (!(mode & MDBX_END_EOTDONE)) /* !(already closed cursors) */ - cursors_eot(txn, false); + *troika = (txn && !(txn->flags & MDBX_TXN_RDONLY)) ? txn->tw.troika : meta_tap(env); + const meta_ptr_t head = meta_recent(env, troika); + const meta_t *const meta0 = METAPAGE(env, 0); + const meta_t *const meta1 = METAPAGE(env, 1); + const meta_t *const meta2 = METAPAGE(env, 2); + out->mi_recent_txnid = head.txnid; + out->mi_meta_txnid[0] = troika->txnid[0]; + out->mi_meta_sign[0] = unaligned_peek_u64(4, meta0->sign); + out->mi_meta_txnid[1] = troika->txnid[1]; + out->mi_meta_sign[1] = unaligned_peek_u64(4, meta1->sign); + out->mi_meta_txnid[2] = troika->txnid[2]; + out->mi_meta_sign[2] = unaligned_peek_u64(4, meta2->sign); + if (likely(bytes > size_before_bootid)) { + memcpy(&out->mi_bootid.meta[0], &meta0->bootid, 16); + memcpy(&out->mi_bootid.meta[1], &meta1->bootid, 16); + memcpy(&out->mi_bootid.meta[2], &meta2->bootid, 16); + if (likely(bytes > size_before_dxbid)) + memcpy(&out->mi_dxbid, &meta0->dxbid, 16); + } - int rc = MDBX_SUCCESS; - if (txn->mt_flags & MDBX_TXN_RDONLY) { - if (txn->to.reader) { - MDBX_reader *slot = txn->to.reader; - eASSERT(env, slot->mr_pid.weak == env->me_pid); - if (likely(!(txn->mt_flags & MDBX_TXN_FINISHED))) { - ENSURE(env, txn->mt_txnid >= - /* paranoia is appropriate here */ env->me_lck - ->mti_oldest_reader.weak); - eASSERT(env, - txn->mt_txnid == slot->mr_txnid.weak && - slot->mr_txnid.weak >= env->me_lck->mti_oldest_reader.weak); -#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__) - atomic_add32(&env->me_ignore_EDEADLK, 1); - txn_valgrind(env, nullptr); - atomic_sub32(&env->me_ignore_EDEADLK, 1); -#endif /* MDBX_USE_VALGRIND || __SANITIZE_ADDRESS__ */ - atomic_store32(&slot->mr_snapshot_pages_used, 0, mo_Relaxed); - safe64_reset(&slot->mr_txnid, false); - atomic_store32(&env->me_lck->mti_readers_refresh_flag, true, - mo_Relaxed); - } else { - eASSERT(env, slot->mr_pid.weak == env->me_pid); - eASSERT(env, slot->mr_txnid.weak >= SAFE64_INVALID_THRESHOLD); - } - if (mode & MDBX_END_SLOT) { - if ((env->me_flags & MDBX_ENV_TXKEY) == 0) - atomic_store32(&slot->mr_pid, 0, mo_Relaxed); - txn->to.reader = NULL; - } - } -#if defined(_WIN32) || defined(_WIN64) - if (txn->mt_flags & MDBX_SHRINK_ALLOWED) - osal_srwlock_ReleaseShared(&env->me_remap_guard); -#endif - txn->mt_numdbs = 0; /* prevent further DBI activity */ - txn->mt_flags = MDBX_TXN_RDONLY | MDBX_TXN_FINISHED; - txn->mt_owner = 0; - } else if (!(txn->mt_flags & MDBX_TXN_FINISHED)) { - ENSURE(env, txn->mt_txnid >= - /* paranoia is appropriate here */ env->me_lck - ->mti_oldest_reader.weak); -#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__) - if (txn == env->me_txn0) - txn_valgrind(env, nullptr); -#endif /* MDBX_USE_VALGRIND || __SANITIZE_ADDRESS__ */ - - txn->mt_flags = MDBX_TXN_FINISHED; - txn->mt_owner = 0; - env->me_txn = txn->mt_parent; - pnl_free(txn->tw.spilled.list); - txn->tw.spilled.list = nullptr; - if (txn == env->me_txn0) { - eASSERT(env, txn->mt_parent == NULL); - /* Export or close DBI handles created in this txn */ - dbi_update(txn, mode & MDBX_END_UPDATE); - pnl_shrink(&txn->tw.retired_pages); - pnl_shrink(&txn->tw.relist); - if (!(env->me_flags & MDBX_WRITEMAP)) - dlist_free(txn); - /* The writer mutex was locked in mdbx_txn_begin. */ - mdbx_txn_unlock(env); - } else { - eASSERT(env, txn->mt_parent != NULL); - MDBX_txn *const parent = txn->mt_parent; - eASSERT(env, parent->mt_signature == MDBX_MT_SIGNATURE); - eASSERT(env, parent->mt_child == txn && - (parent->mt_flags & MDBX_TXN_HAS_CHILD) != 0); - eASSERT(env, pnl_check_allocated(txn->tw.relist, - txn->mt_next_pgno - MDBX_ENABLE_REFUND)); - eASSERT(env, memcmp(&txn->tw.troika, &parent->tw.troika, - sizeof(meta_troika_t)) == 0); - - if (txn->tw.lifo_reclaimed) { - eASSERT(env, MDBX_PNL_GETSIZE(txn->tw.lifo_reclaimed) >= - (uintptr_t)parent->tw.lifo_reclaimed); - MDBX_PNL_SETSIZE(txn->tw.lifo_reclaimed, - (uintptr_t)parent->tw.lifo_reclaimed); - parent->tw.lifo_reclaimed = txn->tw.lifo_reclaimed; - } + const volatile meta_t *txn_meta = head.ptr_v; + out->mi_last_pgno = txn_meta->geometry.first_unallocated - 1; + out->mi_geo.current = pgno2bytes(env, txn_meta->geometry.now); + if (txn) { + out->mi_last_pgno = txn->geo.first_unallocated - 1; + out->mi_geo.current = pgno2bytes(env, txn->geo.end_pgno); + + const txnid_t wanna_meta_txnid = (txn->flags & MDBX_TXN_RDONLY) ? txn->txnid : txn->txnid - xMDBX_TXNID_STEP; + txn_meta = (out->mi_meta_txnid[0] == wanna_meta_txnid) ? meta0 : txn_meta; + txn_meta = (out->mi_meta_txnid[1] == wanna_meta_txnid) ? meta1 : txn_meta; + txn_meta = (out->mi_meta_txnid[2] == wanna_meta_txnid) ? meta2 : txn_meta; + } + out->mi_geo.lower = pgno2bytes(env, txn_meta->geometry.lower); + out->mi_geo.upper = pgno2bytes(env, txn_meta->geometry.upper); + out->mi_geo.shrink = pgno2bytes(env, pv2pages(txn_meta->geometry.shrink_pv)); + out->mi_geo.grow = pgno2bytes(env, pv2pages(txn_meta->geometry.grow_pv)); + out->mi_mapsize = env->dxb_mmap.limit; + + const lck_t *const lck = env->lck; + out->mi_maxreaders = env->max_readers; + out->mi_numreaders = env->lck_mmap.lck ? atomic_load32(&lck->rdt_length, mo_Relaxed) : INT32_MAX; + out->mi_dxb_pagesize = env->ps; + out->mi_sys_pagesize = globals.sys_pagesize; - if (txn->tw.retired_pages) { - eASSERT(env, MDBX_PNL_GETSIZE(txn->tw.retired_pages) >= - (uintptr_t)parent->tw.retired_pages); - MDBX_PNL_SETSIZE(txn->tw.retired_pages, - (uintptr_t)parent->tw.retired_pages); - parent->tw.retired_pages = txn->tw.retired_pages; - } + if (likely(bytes > size_before_bootid)) { + const uint64_t unsynced_pages = + atomic_load64(&lck->unsynced_pages, mo_Relaxed) + + ((uint32_t)out->mi_recent_txnid != atomic_load32(&lck->meta_sync_txnid, mo_Relaxed)); + out->mi_unsync_volume = pgno2bytes(env, (size_t)unsynced_pages); + const uint64_t monotime_now = osal_monotime(); + uint64_t ts = atomic_load64(&lck->eoos_timestamp, mo_Relaxed); + out->mi_since_sync_seconds16dot16 = ts ? osal_monotime_to_16dot16_noUnderflow(monotime_now - ts) : 0; + ts = atomic_load64(&lck->readers_check_timestamp, mo_Relaxed); + out->mi_since_reader_check_seconds16dot16 = ts ? osal_monotime_to_16dot16_noUnderflow(monotime_now - ts) : 0; + out->mi_autosync_threshold = pgno2bytes(env, atomic_load32(&lck->autosync_threshold, mo_Relaxed)); + out->mi_autosync_period_seconds16dot16 = + osal_monotime_to_16dot16_noUnderflow(atomic_load64(&lck->autosync_period, mo_Relaxed)); + out->mi_bootid.current.x = globals.bootid.x; + out->mi_bootid.current.y = globals.bootid.y; + out->mi_mode = env->lck_mmap.lck ? lck->envmode.weak : env->flags; + } - parent->mt_child = nullptr; - parent->mt_flags &= ~MDBX_TXN_HAS_CHILD; - parent->tw.dirtylru = txn->tw.dirtylru; - tASSERT(parent, dirtylist_check(parent)); - tASSERT(parent, audit_ex(parent, 0, false) == 0); - dlist_free(txn); - dpl_free(txn); - pnl_free(txn->tw.relist); + if (likely(bytes > size_before_pgop_stat)) { +#if MDBX_ENABLE_PGOP_STAT + out->mi_pgop_stat.newly = atomic_load64(&lck->pgops.newly, mo_Relaxed); + out->mi_pgop_stat.cow = atomic_load64(&lck->pgops.cow, mo_Relaxed); + out->mi_pgop_stat.clone = atomic_load64(&lck->pgops.clone, mo_Relaxed); + out->mi_pgop_stat.split = atomic_load64(&lck->pgops.split, mo_Relaxed); + out->mi_pgop_stat.merge = atomic_load64(&lck->pgops.merge, mo_Relaxed); + out->mi_pgop_stat.spill = atomic_load64(&lck->pgops.spill, mo_Relaxed); + out->mi_pgop_stat.unspill = atomic_load64(&lck->pgops.unspill, mo_Relaxed); + out->mi_pgop_stat.wops = atomic_load64(&lck->pgops.wops, mo_Relaxed); + out->mi_pgop_stat.prefault = atomic_load64(&lck->pgops.prefault, mo_Relaxed); + out->mi_pgop_stat.mincore = atomic_load64(&lck->pgops.mincore, mo_Relaxed); + out->mi_pgop_stat.msync = atomic_load64(&lck->pgops.msync, mo_Relaxed); + out->mi_pgop_stat.fsync = atomic_load64(&lck->pgops.fsync, mo_Relaxed); +#else + memset(&out->mi_pgop_stat, 0, sizeof(out->mi_pgop_stat)); +#endif /* MDBX_ENABLE_PGOP_STAT*/ + } - if (parent->mt_geo.upper != txn->mt_geo.upper || - parent->mt_geo.now != txn->mt_geo.now) { - /* undo resize performed by child txn */ - rc = dxb_resize(env, parent->mt_next_pgno, parent->mt_geo.now, - parent->mt_geo.upper, impilict_shrink); - if (rc == MDBX_EPERM) { - /* unable undo resize (it is regular for Windows), - * therefore promote size changes from child to the parent txn */ - WARNING("unable undo resize performed by child txn, promote to " - "the parent (%u->%u, %u->%u)", - txn->mt_geo.now, parent->mt_geo.now, txn->mt_geo.upper, - parent->mt_geo.upper); - parent->mt_geo.now = txn->mt_geo.now; - parent->mt_geo.upper = txn->mt_geo.upper; - parent->mt_flags |= MDBX_TXN_DIRTY; - rc = MDBX_SUCCESS; - } else if (unlikely(rc != MDBX_SUCCESS)) { - ERROR("error %d while undo resize performed by child txn, fail " - "the parent", - rc); - parent->mt_flags |= MDBX_TXN_ERROR; - if (!env->me_dxb_mmap.base) - env->me_flags |= MDBX_FATAL_ERROR; - } + txnid_t overall_latter_reader_txnid = out->mi_recent_txnid; + txnid_t self_latter_reader_txnid = overall_latter_reader_txnid; + if (env->lck_mmap.lck) { + for (size_t i = 0; i < out->mi_numreaders; ++i) { + const uint32_t pid = atomic_load32(&lck->rdt[i].pid, mo_AcquireRelease); + if (pid) { + const txnid_t txnid = safe64_read(&lck->rdt[i].txnid); + if (overall_latter_reader_txnid > txnid) + overall_latter_reader_txnid = txnid; + if (pid == env->pid && self_latter_reader_txnid > txnid) + self_latter_reader_txnid = txnid; } } } + out->mi_self_latter_reader_txnid = self_latter_reader_txnid; + out->mi_latter_reader_txnid = overall_latter_reader_txnid; - eASSERT(env, txn == env->me_txn0 || txn->mt_owner == 0); - if ((mode & MDBX_END_FREE) != 0 && txn != env->me_txn0) { - txn->mt_signature = 0; - osal_free(txn); - } - - return rc; + osal_compiler_barrier(); + return MDBX_SUCCESS; } -int mdbx_txn_reset(MDBX_txn *txn) { - int rc = check_txn(txn, 0); +__cold int env_info(const MDBX_env *env, const MDBX_txn *txn, MDBX_envinfo *out, size_t bytes, troika_t *troika) { + MDBX_envinfo snap; + int rc = env_info_snap(env, txn, &snap, sizeof(snap), troika); if (unlikely(rc != MDBX_SUCCESS)) return rc; - /* This call is only valid for read-only txns */ - if (unlikely((txn->mt_flags & MDBX_TXN_RDONLY) == 0)) - return MDBX_EINVAL; - - /* LY: don't close DBI-handles */ - rc = txn_end(txn, MDBX_END_RESET | MDBX_END_UPDATE); - if (rc == MDBX_SUCCESS) { - tASSERT(txn, txn->mt_signature == MDBX_MT_SIGNATURE); - tASSERT(txn, txn->mt_owner == 0); - } - return rc; -} - -int mdbx_txn_break(MDBX_txn *txn) { - do { - int rc = check_txn(txn, 0); + eASSERT(env, sizeof(snap) >= bytes); + while (1) { + rc = env_info_snap(env, txn, out, bytes, troika); if (unlikely(rc != MDBX_SUCCESS)) return rc; - txn->mt_flags |= MDBX_TXN_ERROR; - if (txn->mt_flags & MDBX_TXN_RDONLY) - break; - txn = txn->mt_child; - } while (txn); - return MDBX_SUCCESS; + snap.mi_since_sync_seconds16dot16 = out->mi_since_sync_seconds16dot16; + snap.mi_since_reader_check_seconds16dot16 = out->mi_since_reader_check_seconds16dot16; + if (likely(memcmp(&snap, out, bytes) == 0)) + return MDBX_SUCCESS; + memcpy(&snap, out, bytes); + } } -int mdbx_txn_abort(MDBX_txn *txn) { - int rc = check_txn(txn, 0); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; +__cold int mdbx_env_info_ex(const MDBX_env *env, const MDBX_txn *txn, MDBX_envinfo *arg, size_t bytes) { + if (unlikely((env == nullptr && txn == nullptr) || arg == nullptr)) + return LOG_IFERR(MDBX_EINVAL); - if (txn->mt_flags & MDBX_TXN_RDONLY) - /* LY: don't close DBI-handles */ - return txn_end(txn, MDBX_END_ABORT | MDBX_END_UPDATE | MDBX_END_SLOT | - MDBX_END_FREE); + const size_t size_before_bootid = offsetof(MDBX_envinfo, mi_bootid); + const size_t size_before_pgop_stat = offsetof(MDBX_envinfo, mi_pgop_stat); + const size_t size_before_dxbid = offsetof(MDBX_envinfo, mi_dxbid); + if (unlikely(bytes != sizeof(MDBX_envinfo)) && bytes != size_before_bootid && bytes != size_before_pgop_stat && + bytes != size_before_dxbid) + return LOG_IFERR(MDBX_EINVAL); - if (unlikely(txn->mt_flags & MDBX_TXN_FINISHED)) - return MDBX_BAD_TXN; + if (txn) { + int err = check_txn(txn, MDBX_TXN_BLOCKED - MDBX_TXN_ERROR); + if (unlikely(err != MDBX_SUCCESS)) + return LOG_IFERR(err); + } + if (env) { + int err = check_env(env, false); + if (unlikely(err != MDBX_SUCCESS)) + return LOG_IFERR(err); + if (txn && unlikely(txn->env != env)) + return LOG_IFERR(MDBX_EINVAL); + } else { + env = txn->env; + } - if (txn->mt_child) - mdbx_txn_abort(txn->mt_child); + troika_t troika; + return LOG_IFERR(env_info(env, txn, arg, bytes, &troika)); +} - tASSERT(txn, (txn->mt_flags & MDBX_TXN_ERROR) || dirtylist_check(txn)); - return txn_end(txn, MDBX_END_ABORT | MDBX_END_SLOT | MDBX_END_FREE); +__cold int mdbx_preopen_snapinfo(const char *pathname, MDBX_envinfo *out, size_t bytes) { +#if defined(_WIN32) || defined(_WIN64) + wchar_t *pathnameW = nullptr; + int rc = osal_mb2w(pathname, &pathnameW); + if (likely(rc == MDBX_SUCCESS)) { + rc = mdbx_preopen_snapinfoW(pathnameW, out, bytes); + osal_free(pathnameW); + } + return LOG_IFERR(rc); } -/* Count all the pages in each DB and in the GC and make sure - * it matches the actual number of pages being used. */ -__cold static int audit_ex(MDBX_txn *txn, size_t retired_stored, - bool dont_filter_gc) { - size_t pending = 0; - if ((txn->mt_flags & MDBX_TXN_RDONLY) == 0) - pending = txn->tw.loose_count + MDBX_PNL_GETSIZE(txn->tw.relist) + - (MDBX_PNL_GETSIZE(txn->tw.retired_pages) - retired_stored); +__cold int mdbx_preopen_snapinfoW(const wchar_t *pathname, MDBX_envinfo *out, size_t bytes) { +#endif /* Windows */ + if (unlikely(!out)) + return LOG_IFERR(MDBX_EINVAL); - MDBX_cursor_couple cx; - int rc = cursor_init(&cx.outer, txn, FREE_DBI); + const size_t size_before_bootid = offsetof(MDBX_envinfo, mi_bootid); + const size_t size_before_pgop_stat = offsetof(MDBX_envinfo, mi_pgop_stat); + const size_t size_before_dxbid = offsetof(MDBX_envinfo, mi_dxbid); + if (unlikely(bytes != sizeof(MDBX_envinfo)) && bytes != size_before_bootid && bytes != size_before_pgop_stat && + bytes != size_before_dxbid) + return LOG_IFERR(MDBX_EINVAL); + + memset(out, 0, bytes); + if (likely(bytes > size_before_bootid)) { + out->mi_bootid.current.x = globals.bootid.x; + out->mi_bootid.current.y = globals.bootid.y; + } + + MDBX_env env; + memset(&env, 0, sizeof(env)); + env.pid = osal_getpid(); + if (unlikely(!is_powerof2(globals.sys_pagesize) || globals.sys_pagesize < MDBX_MIN_PAGESIZE)) { + ERROR("unsuitable system pagesize %u", globals.sys_pagesize); + return LOG_IFERR(MDBX_INCOMPATIBLE); + } + out->mi_sys_pagesize = globals.sys_pagesize; + env.flags = MDBX_RDONLY | MDBX_NORDAHEAD | MDBX_ACCEDE | MDBX_VALIDATION; + env.stuck_meta = -1; + env.lck_mmap.fd = INVALID_HANDLE_VALUE; + env.lazy_fd = INVALID_HANDLE_VALUE; + env.dsync_fd = INVALID_HANDLE_VALUE; + env.fd4meta = INVALID_HANDLE_VALUE; +#if defined(_WIN32) || defined(_WIN64) + env.dxb_lock_event = INVALID_HANDLE_VALUE; + env.ioring.overlapped_fd = INVALID_HANDLE_VALUE; +#endif /* Windows */ + env_options_init(&env); + + int rc = env_handle_pathname(&env, pathname, 0); if (unlikely(rc != MDBX_SUCCESS)) - return rc; + goto bailout; + rc = osal_openfile(MDBX_OPEN_DXB_READ, &env, env.pathname.dxb, &env.lazy_fd, 0); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; - size_t gc = 0; - MDBX_val key, data; - while ((rc = cursor_get(&cx.outer, &key, &data, MDBX_NEXT)) == 0) { - if (!dont_filter_gc) { - if (unlikely(key.iov_len != sizeof(txnid_t))) - return MDBX_CORRUPTED; - txnid_t id = unaligned_peek_u64(4, key.iov_base); - if (txn->tw.lifo_reclaimed) { - for (size_t i = 1; i <= MDBX_PNL_GETSIZE(txn->tw.lifo_reclaimed); ++i) - if (id == txn->tw.lifo_reclaimed[i]) - goto skip; - } else if (id <= txn->tw.last_reclaimed) - goto skip; - } + meta_t header; + rc = dxb_read_header(&env, &header, 0, 0); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; - gc += *(pgno_t *)data.iov_base; - skip:; + out->mi_dxb_pagesize = env_setup_pagesize(&env, header.pagesize); + out->mi_geo.lower = pgno2bytes(&env, header.geometry.lower); + out->mi_geo.upper = pgno2bytes(&env, header.geometry.upper); + out->mi_geo.shrink = pgno2bytes(&env, pv2pages(header.geometry.shrink_pv)); + out->mi_geo.grow = pgno2bytes(&env, pv2pages(header.geometry.grow_pv)); + out->mi_geo.current = pgno2bytes(&env, header.geometry.now); + out->mi_last_pgno = header.geometry.first_unallocated - 1; + + const unsigned n = 0; + out->mi_recent_txnid = constmeta_txnid(&header); + out->mi_meta_sign[n] = unaligned_peek_u64(4, &header.sign); + if (likely(bytes > size_before_bootid)) { + memcpy(&out->mi_bootid.meta[n], &header.bootid, 16); + if (likely(bytes > size_before_dxbid)) + memcpy(&out->mi_dxbid, &header.dxbid, 16); } - tASSERT(txn, rc == MDBX_NOTFOUND); - for (size_t i = FREE_DBI; i < txn->mt_numdbs; i++) - txn->mt_dbistate[i] &= ~DBI_AUDITED; +bailout: + env_close(&env, false); + return LOG_IFERR(rc); +} - size_t used = NUM_METAS; - for (size_t i = FREE_DBI; i <= MAIN_DBI; i++) { - if (!(txn->mt_dbistate[i] & DBI_VALID)) - continue; - rc = cursor_init(&cx.outer, txn, i); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - txn->mt_dbistate[i] |= DBI_AUDITED; - if (txn->mt_dbs[i].md_root == P_INVALID) - continue; - used += (size_t)txn->mt_dbs[i].md_branch_pages + - (size_t)txn->mt_dbs[i].md_leaf_pages + - (size_t)txn->mt_dbs[i].md_overflow_pages; +/*----------------------------------------------------------------------------*/ - if (i != MAIN_DBI) - continue; - rc = page_search(&cx.outer, NULL, MDBX_PS_FIRST); - while (rc == MDBX_SUCCESS) { - MDBX_page *mp = cx.outer.mc_pg[cx.outer.mc_top]; - for (size_t j = 0; j < page_numkeys(mp); j++) { - MDBX_node *node = page_node(mp, j); - if (node_flags(node) == F_SUBDATA) { - if (unlikely(node_ds(node) != sizeof(MDBX_db))) - return MDBX_CORRUPTED; - MDBX_db db_copy, *db; - memcpy(db = &db_copy, node_data(node), sizeof(db_copy)); - if ((txn->mt_flags & MDBX_TXN_RDONLY) == 0) { - for (MDBX_dbi k = txn->mt_numdbs; --k > MAIN_DBI;) { - if ((txn->mt_dbistate[k] & DBI_VALID) && - /* txn->mt_dbxs[k].md_name.iov_base && */ - node_ks(node) == txn->mt_dbxs[k].md_name.iov_len && - memcmp(node_key(node), txn->mt_dbxs[k].md_name.iov_base, - node_ks(node)) == 0) { - txn->mt_dbistate[k] |= DBI_AUDITED; - if (!(txn->mt_dbistate[k] & MDBX_DBI_STALE)) - db = txn->mt_dbs + k; - break; - } - } - } - used += (size_t)db->md_branch_pages + (size_t)db->md_leaf_pages + - (size_t)db->md_overflow_pages; - } - } - rc = cursor_sibling(&cx.outer, SIBLING_RIGHT); - } - tASSERT(txn, rc == MDBX_NOTFOUND); - } +__cold int mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower, intptr_t size_now, intptr_t size_upper, + intptr_t growth_step, intptr_t shrink_threshold, intptr_t pagesize) { + int rc = check_env(env, false); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); - for (size_t i = FREE_DBI; i < txn->mt_numdbs; i++) { - if ((txn->mt_dbistate[i] & (DBI_VALID | DBI_AUDITED | DBI_STALE)) != - DBI_VALID) - continue; - for (MDBX_txn *t = txn; t; t = t->mt_parent) - if (F_ISSET(t->mt_dbistate[i], DBI_DIRTY | DBI_CREAT)) { - used += (size_t)t->mt_dbs[i].md_branch_pages + - (size_t)t->mt_dbs[i].md_leaf_pages + - (size_t)t->mt_dbs[i].md_overflow_pages; - txn->mt_dbistate[i] |= DBI_AUDITED; - break; - } - MDBX_ANALYSIS_ASSUME(txn != nullptr); - if (!(txn->mt_dbistate[i] & DBI_AUDITED)) { - WARNING("audit %s@%" PRIaTXN - ": unable account dbi %zd / \"%*s\", state 0x%02x", - txn->mt_parent ? "nested-" : "", txn->mt_txnid, i, - (int)txn->mt_dbxs[i].md_name.iov_len, - (const char *)txn->mt_dbxs[i].md_name.iov_base, - txn->mt_dbistate[i]); - } + MDBX_txn *const txn_owned = env_owned_wrtxn(env); + bool should_unlock = false; + +#if MDBX_DEBUG && 0 /* минимальные шаги для проверки/отладки уже не нужны */ + if (growth_step < 0) { + growth_step = 1; + if (shrink_threshold < 0) + shrink_threshold = 1; } +#endif /* MDBX_DEBUG */ - if (pending + gc + used == txn->mt_next_pgno) - return MDBX_SUCCESS; + if (env->dxb_mmap.base) { + /* env already mapped */ + if (unlikely(env->flags & MDBX_RDONLY)) + return LOG_IFERR(MDBX_EACCESS); - if ((txn->mt_flags & MDBX_TXN_RDONLY) == 0) - ERROR("audit @%" PRIaTXN ": %zu(pending) = %zu(loose) + " - "%zu(reclaimed) + %zu(retired-pending) - %zu(retired-stored)", - txn->mt_txnid, pending, txn->tw.loose_count, - MDBX_PNL_GETSIZE(txn->tw.relist), - txn->tw.retired_pages ? MDBX_PNL_GETSIZE(txn->tw.retired_pages) : 0, - retired_stored); - ERROR("audit @%" PRIaTXN ": %zu(pending) + %zu" - "(gc) + %zu(count) = %zu(total) <> %zu" - "(allocated)", - txn->mt_txnid, pending, gc, used, pending + gc + used, - (size_t)txn->mt_next_pgno); - return MDBX_PROBLEM; -} + if (!txn_owned) { + int err = lck_txn_lock(env, false); + if (unlikely(err != MDBX_SUCCESS)) + return LOG_IFERR(err); + should_unlock = true; + env->basal_txn->tw.troika = meta_tap(env); + eASSERT(env, !env->txn && !env->basal_txn->nested); + env->basal_txn->txnid = env->basal_txn->tw.troika.txnid[env->basal_txn->tw.troika.recent]; + txn_snapshot_oldest(env->basal_txn); + } -typedef struct gc_update_context { - size_t retired_stored, loop; - size_t settled, cleaned_slot, reused_slot, filled_slot; - txnid_t cleaned_id, rid; - bool lifo, dense; -#if MDBX_ENABLE_BIGFOOT - txnid_t bigfoot; -#endif /* MDBX_ENABLE_BIGFOOT */ - MDBX_cursor cursor; -} gcu_context_t; + /* get untouched params from current TXN or DB */ + if (pagesize <= 0 || pagesize >= INT_MAX) + pagesize = env->ps; + const geo_t *const geo = env->txn ? &env->txn->geo : &meta_recent(env, &env->basal_txn->tw.troika).ptr_c->geometry; + if (size_lower < 0) + size_lower = pgno2bytes(env, geo->lower); + if (size_now < 0) + size_now = pgno2bytes(env, geo->now); + if (size_upper < 0) + size_upper = pgno2bytes(env, geo->upper); + if (growth_step < 0) + growth_step = pgno2bytes(env, pv2pages(geo->grow_pv)); + if (shrink_threshold < 0) + shrink_threshold = pgno2bytes(env, pv2pages(geo->shrink_pv)); -static __inline int gcu_context_init(MDBX_txn *txn, gcu_context_t *ctx) { - memset(ctx, 0, offsetof(gcu_context_t, cursor)); - ctx->lifo = (txn->mt_env->me_flags & MDBX_LIFORECLAIM) != 0; -#if MDBX_ENABLE_BIGFOOT - ctx->bigfoot = txn->mt_txnid; -#endif /* MDBX_ENABLE_BIGFOOT */ - return cursor_init(&ctx->cursor, txn, FREE_DBI); -} + if (pagesize != (intptr_t)env->ps) { + rc = MDBX_EINVAL; + goto bailout; + } + const size_t usedbytes = pgno2bytes(env, mvcc_snapshot_largest(env, geo->first_unallocated)); + if ((size_t)size_upper < usedbytes) { + rc = MDBX_MAP_FULL; + goto bailout; + } + if ((size_t)size_now < usedbytes) + size_now = usedbytes; + } else { + /* env NOT yet mapped */ + if (unlikely(env->txn)) + return LOG_IFERR(MDBX_PANIC); -static __always_inline size_t gcu_backlog_size(MDBX_txn *txn) { - return MDBX_PNL_GETSIZE(txn->tw.relist) + txn->tw.loose_count; -} + /* is requested some auto-value for pagesize ? */ + if (pagesize >= INT_MAX /* maximal */) + pagesize = MDBX_MAX_PAGESIZE; + else if (pagesize <= 0) { + if (pagesize < 0 /* default */) { + pagesize = globals.sys_pagesize; + if ((uintptr_t)pagesize > MDBX_MAX_PAGESIZE) + pagesize = MDBX_MAX_PAGESIZE; + eASSERT(env, (uintptr_t)pagesize >= MDBX_MIN_PAGESIZE); + } else if (pagesize == 0 /* minimal */) + pagesize = MDBX_MIN_PAGESIZE; -static int gcu_clean_stored_retired(MDBX_txn *txn, gcu_context_t *ctx) { - int err = MDBX_SUCCESS; - if (ctx->retired_stored) { - MDBX_cursor *const gc = ptr_disp(txn, sizeof(MDBX_txn)); - tASSERT(txn, txn == txn->mt_env->me_txn0 && gc->mc_next == nullptr); - gc->mc_txn = txn; - gc->mc_flags = 0; - gc->mc_next = txn->mt_cursors[FREE_DBI]; - txn->mt_cursors[FREE_DBI] = gc; - do { - MDBX_val key, val; -#if MDBX_ENABLE_BIGFOOT - key.iov_base = &ctx->bigfoot; -#else - key.iov_base = &txn->mt_txnid; -#endif /* MDBX_ENABLE_BIGFOOT */ - key.iov_len = sizeof(txnid_t); - const struct cursor_set_result csr = cursor_set(gc, &key, &val, MDBX_SET); - if (csr.err == MDBX_SUCCESS && csr.exact) { - ctx->retired_stored = 0; - err = cursor_del(gc, 0); - TRACE("== clear-4linear, backlog %zu, err %d", gcu_backlog_size(txn), - err); - } + /* choose pagesize */ + intptr_t top = (size_now > size_lower) ? size_now : size_lower; + if (size_upper > top) + top = size_upper; + if (top < 0 /* default */) + top = reasonable_db_maxsize(); + else if (top == 0 /* minimal */) + top = MIN_MAPSIZE; + else if (top >= (intptr_t)MAX_MAPSIZE /* maximal */) + top = MAX_MAPSIZE; + + while (top > pagesize * (int64_t)(MAX_PAGENO + 1) && pagesize < MDBX_MAX_PAGESIZE) + pagesize <<= 1; } -#if MDBX_ENABLE_BIGFOOT - while (!err && --ctx->bigfoot >= txn->mt_txnid); -#else - while (0); -#endif /* MDBX_ENABLE_BIGFOOT */ - txn->mt_cursors[FREE_DBI] = gc->mc_next; - gc->mc_next = nullptr; } - return err; -} - -static int gcu_touch(gcu_context_t *ctx) { - MDBX_val key, val; - key.iov_base = val.iov_base = nullptr; - key.iov_len = sizeof(txnid_t); - val.iov_len = MDBX_PNL_SIZEOF(ctx->cursor.mc_txn->tw.retired_pages); - ctx->cursor.mc_flags |= C_GCU; - int err = cursor_touch(&ctx->cursor, &key, &val); - ctx->cursor.mc_flags -= C_GCU; - return err; -} -/* Prepare a backlog of pages to modify GC itself, while reclaiming is - * prohibited. It should be enough to prevent search in page_alloc_slowpath() - * during a deleting, when GC tree is unbalanced. */ -static int gcu_prepare_backlog(MDBX_txn *txn, gcu_context_t *ctx) { - const size_t for_cow = txn->mt_dbs[FREE_DBI].md_depth; - const size_t for_rebalance = for_cow + 1 + - (txn->mt_dbs[FREE_DBI].md_depth + 1ul >= - txn->mt_dbs[FREE_DBI].md_branch_pages); - size_t for_split = ctx->retired_stored == 0; + if (pagesize < (intptr_t)MDBX_MIN_PAGESIZE || pagesize > (intptr_t)MDBX_MAX_PAGESIZE || !is_powerof2(pagesize)) { + rc = MDBX_EINVAL; + goto bailout; + } - const intptr_t retired_left = - MDBX_PNL_SIZEOF(txn->tw.retired_pages) - ctx->retired_stored; - size_t for_relist = 0; - if (MDBX_ENABLE_BIGFOOT && retired_left > 0) { - for_relist = (retired_left + txn->mt_env->me_maxgc_ov1page - 1) / - txn->mt_env->me_maxgc_ov1page; - const size_t per_branch_page = txn->mt_env->me_maxgc_per_branch; - for (size_t entries = for_relist; entries > 1; for_split += entries) - entries = (entries + per_branch_page - 1) / per_branch_page; - } else if (!MDBX_ENABLE_BIGFOOT && retired_left != 0) { - for_relist = - number_of_ovpages(txn->mt_env, MDBX_PNL_SIZEOF(txn->tw.retired_pages)); + const bool size_lower_default = size_lower < 0; + if (size_lower <= 0) { + size_lower = (size_lower == 0) ? MIN_MAPSIZE : pagesize * MDBX_WORDBITS; + if (size_lower / pagesize < MIN_PAGENO) + size_lower = MIN_PAGENO * pagesize; + } + if (size_lower >= INTPTR_MAX) { + size_lower = reasonable_db_maxsize(); + if ((size_t)size_lower / pagesize > MAX_PAGENO + 1) + size_lower = pagesize * (MAX_PAGENO + 1); } - const size_t for_tree_before_touch = for_cow + for_rebalance + for_split; - const size_t for_tree_after_touch = for_rebalance + for_split; - const size_t for_all_before_touch = for_relist + for_tree_before_touch; - const size_t for_all_after_touch = for_relist + for_tree_after_touch; + if (size_now >= INTPTR_MAX) { + size_now = reasonable_db_maxsize(); + if ((size_t)size_now / pagesize > MAX_PAGENO + 1) + size_now = pagesize * (MAX_PAGENO + 1); + } - if (likely(for_relist < 2 && gcu_backlog_size(txn) > for_all_before_touch) && - (ctx->cursor.mc_snum == 0 || - IS_MODIFIABLE(txn, ctx->cursor.mc_pg[ctx->cursor.mc_top]))) - return MDBX_SUCCESS; + if (size_upper <= 0) { + if ((growth_step == 0 || size_upper == 0) && size_now >= size_lower) + size_upper = size_now; + else if (size_now <= 0 || size_now >= reasonable_db_maxsize() / 2) + size_upper = reasonable_db_maxsize(); + else if ((size_t)size_now >= MAX_MAPSIZE32 / 2 && (size_t)size_now <= MAX_MAPSIZE32 / 4 * 3) + size_upper = MAX_MAPSIZE32; + else { + size_upper = ceil_powerof2(((size_t)size_now < MAX_MAPSIZE / 4) ? size_now + size_now : size_now + size_now / 2, + MEGABYTE * MDBX_WORDBITS * MDBX_WORDBITS / 32); + if ((size_t)size_upper > MAX_MAPSIZE) + size_upper = MAX_MAPSIZE; + } + if ((size_t)size_upper / pagesize > (MAX_PAGENO + 1)) + size_upper = pagesize * (MAX_PAGENO + 1); + } else if (size_upper >= INTPTR_MAX) { + size_upper = reasonable_db_maxsize(); + if ((size_t)size_upper / pagesize > MAX_PAGENO + 1) + size_upper = pagesize * (MAX_PAGENO + 1); + } - TRACE(">> retired-stored %zu, left %zi, backlog %zu, need %zu (4list %zu, " - "4split %zu, " - "4cow %zu, 4tree %zu)", - ctx->retired_stored, retired_left, gcu_backlog_size(txn), - for_all_before_touch, for_relist, for_split, for_cow, - for_tree_before_touch); + if (unlikely(size_lower < (intptr_t)MIN_MAPSIZE || size_lower > size_upper)) { + /* паранойа на случай переполнения при невероятных значениях */ + rc = MDBX_EINVAL; + goto bailout; + } - int err = gcu_touch(ctx); - TRACE("== after-touch, backlog %zu, err %d", gcu_backlog_size(txn), err); + if (size_now <= 0) { + size_now = size_lower; + if (size_upper >= size_lower && size_now > size_upper) + size_now = size_upper; + } - if (!MDBX_ENABLE_BIGFOOT && unlikely(for_relist > 1) && - MDBX_PNL_GETSIZE(txn->tw.retired_pages) != ctx->retired_stored && - err == MDBX_SUCCESS) { - if (unlikely(ctx->retired_stored)) { - err = gcu_clean_stored_retired(txn, ctx); - if (unlikely(err != MDBX_SUCCESS)) - return err; - if (!ctx->retired_stored) - return /* restart by tail-recursion */ gcu_prepare_backlog(txn, ctx); + if ((uint64_t)size_lower / pagesize < MIN_PAGENO) { + size_lower = pagesize * MIN_PAGENO; + if (unlikely(size_lower > size_upper)) { + /* паранойа на случай переполнения при невероятных значениях */ + rc = MDBX_EINVAL; + goto bailout; } - err = page_alloc_slowpath(&ctx->cursor, for_relist, MDBX_ALLOC_RESERVE).err; - TRACE("== after-4linear, backlog %zu, err %d", gcu_backlog_size(txn), err); - cASSERT(&ctx->cursor, - gcu_backlog_size(txn) >= for_relist || err != MDBX_SUCCESS); + if (size_now < size_lower) + size_now = size_lower; } - while (gcu_backlog_size(txn) < for_all_after_touch && err == MDBX_SUCCESS) - err = page_alloc_slowpath(&ctx->cursor, 0, - MDBX_ALLOC_RESERVE | MDBX_ALLOC_UNIMPORTANT) - .err; - - TRACE("<< backlog %zu, err %d, gc: height %u, branch %zu, leaf %zu, large " - "%zu, entries %zu", - gcu_backlog_size(txn), err, txn->mt_dbs[FREE_DBI].md_depth, - (size_t)txn->mt_dbs[FREE_DBI].md_branch_pages, - (size_t)txn->mt_dbs[FREE_DBI].md_leaf_pages, - (size_t)txn->mt_dbs[FREE_DBI].md_overflow_pages, - (size_t)txn->mt_dbs[FREE_DBI].md_entries); - tASSERT(txn, - err != MDBX_NOTFOUND || (txn->mt_flags & MDBX_TXN_DRAINED_GC) != 0); - return (err != MDBX_NOTFOUND) ? err : MDBX_SUCCESS; -} + if (unlikely((size_t)size_upper > MAX_MAPSIZE || (uint64_t)size_upper / pagesize > MAX_PAGENO + 1)) { + rc = MDBX_TOO_LARGE; + goto bailout; + } -static __inline void gcu_clean_reserved(MDBX_env *env, MDBX_val pnl) { -#if MDBX_DEBUG && (defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__)) - /* Для предотвращения предупреждения Valgrind из mdbx_dump_val() - * вызванное через макрос DVAL_DEBUG() на выходе - * из cursor_set(MDBX_SET_KEY), которая вызывается ниже внутри update_gc() в - * цикле очистки и цикле заполнения зарезервированных элементов. */ - memset(pnl.iov_base, 0xBB, pnl.iov_len); -#endif /* MDBX_DEBUG && (MDBX_USE_VALGRIND || __SANITIZE_ADDRESS__) */ + const size_t unit = (globals.sys_pagesize > (size_t)pagesize) ? globals.sys_pagesize : (size_t)pagesize; + size_lower = ceil_powerof2(size_lower, unit); + size_upper = ceil_powerof2(size_upper, unit); + size_now = ceil_powerof2(size_now, unit); - /* PNL is initially empty, zero out at least the length */ - memset(pnl.iov_base, 0, sizeof(pgno_t)); - if ((env->me_flags & (MDBX_WRITEMAP | MDBX_NOMEMINIT)) == 0) - /* zero out to avoid leaking values from uninitialized malloc'ed memory - * to the file in non-writemap mode if length of the saving page-list - * was changed during space reservation. */ - memset(pnl.iov_base, 0, pnl.iov_len); -} + /* LY: подбираем значение size_upper: + * - кратное размеру страницы + * - без нарушения MAX_MAPSIZE и MAX_PAGENO */ + while (unlikely((size_t)size_upper > MAX_MAPSIZE || (uint64_t)size_upper / pagesize > MAX_PAGENO + 1)) { + if ((size_t)size_upper < unit + MIN_MAPSIZE || (size_t)size_upper < (size_t)pagesize * (MIN_PAGENO + 1)) { + /* паранойа на случай переполнения при невероятных значениях */ + rc = MDBX_EINVAL; + goto bailout; + } + size_upper -= unit; + if ((size_t)size_upper < (size_t)size_lower) + size_lower = size_upper; + } + eASSERT(env, (size_upper - size_lower) % globals.sys_pagesize == 0); -/* Cleanups reclaimed GC (aka freeDB) records, saves the retired-list (aka - * freelist) of current transaction to GC, puts back into GC leftover of the - * reclaimed pages with chunking. This recursive changes the reclaimed-list, - * loose-list and retired-list. Keep trying until it stabilizes. - * - * NOTE: This code is a consequence of many iterations of adding crutches (aka - * "checks and balances") to partially bypass the fundamental design problems - * inherited from LMDB. So do not try to understand it completely in order to - * avoid your madness. */ -static int update_gc(MDBX_txn *txn, gcu_context_t *ctx) { - TRACE("\n>>> @%" PRIaTXN, txn->mt_txnid); - MDBX_env *const env = txn->mt_env; - const char *const dbg_prefix_mode = ctx->lifo ? " lifo" : " fifo"; - (void)dbg_prefix_mode; - ctx->cursor.mc_next = txn->mt_cursors[FREE_DBI]; - txn->mt_cursors[FREE_DBI] = &ctx->cursor; - - /* txn->tw.relist[] can grow and shrink during this call. - * txn->tw.last_reclaimed and txn->tw.retired_pages[] can only grow. - * But page numbers cannot disappear from txn->tw.retired_pages[]. */ + if (size_now < size_lower) + size_now = size_lower; + if (size_now > size_upper) + size_now = size_upper; -retry: - if (ctx->loop++) - TRACE("%s", " >> restart"); - int rc = MDBX_SUCCESS; - tASSERT(txn, pnl_check_allocated(txn->tw.relist, - txn->mt_next_pgno - MDBX_ENABLE_REFUND)); - tASSERT(txn, dirtylist_check(txn)); - if (unlikely(/* paranoia */ ctx->loop > ((MDBX_DEBUG > 0) ? 12 : 42))) { - ERROR("too more loops %zu, bailout", ctx->loop); - rc = MDBX_PROBLEM; - goto bailout; + if (growth_step < 0) { + growth_step = ((size_t)(size_upper - size_lower)) / 42; + if (!size_lower_default && growth_step > size_lower && size_lower < (intptr_t)MEGABYTE) + growth_step = size_lower; + else if (growth_step / size_lower > 64) + growth_step = size_lower << 6; + if (growth_step < 65536) + growth_step = 65536; + if ((size_upper - size_lower) / growth_step > 65536) + growth_step = (size_upper - size_lower) >> 16; + const intptr_t growth_step_limit = MEGABYTE * ((MDBX_WORDBITS > 32) ? 4096 : 256); + if (growth_step > growth_step_limit) + growth_step = growth_step_limit; } + if (growth_step == 0 && shrink_threshold > 0) + growth_step = 1; + growth_step = ceil_powerof2(growth_step, unit); - if (unlikely(ctx->dense)) { - rc = gcu_clean_stored_retired(txn, ctx); - if (unlikely(rc != MDBX_SUCCESS)) - goto bailout; - } + if (shrink_threshold < 0) + shrink_threshold = growth_step + growth_step; + shrink_threshold = ceil_powerof2(shrink_threshold, unit); - ctx->settled = 0; - ctx->cleaned_slot = 0; - ctx->reused_slot = 0; - ctx->filled_slot = ~0u; - ctx->cleaned_id = 0; - ctx->rid = txn->tw.last_reclaimed; - while (true) { - /* Come back here after each Put() in case retired-list changed */ - TRACE("%s", " >> continue"); + //---------------------------------------------------------------------------- - if (ctx->retired_stored != MDBX_PNL_GETSIZE(txn->tw.retired_pages) && - (ctx->loop == 1 || ctx->retired_stored > env->me_maxgc_ov1page || - MDBX_PNL_GETSIZE(txn->tw.retired_pages) > env->me_maxgc_ov1page)) { - rc = gcu_prepare_backlog(txn, ctx); - if (unlikely(rc != MDBX_SUCCESS)) - goto bailout; - } + if (!env->dxb_mmap.base) { + /* save user's geo-params for future open/create */ + if (pagesize != (intptr_t)env->ps) + env_setup_pagesize(env, pagesize); + env->geo_in_bytes.lower = size_lower; + env->geo_in_bytes.now = size_now; + env->geo_in_bytes.upper = size_upper; + env->geo_in_bytes.grow = pgno2bytes(env, pv2pages(pages2pv(bytes2pgno(env, growth_step)))); + env->geo_in_bytes.shrink = pgno2bytes(env, pv2pages(pages2pv(bytes2pgno(env, shrink_threshold)))); + env_options_adjust_defaults(env); + + ENSURE(env, env->geo_in_bytes.lower >= MIN_MAPSIZE); + ENSURE(env, env->geo_in_bytes.lower / (unsigned)pagesize >= MIN_PAGENO); + ENSURE(env, env->geo_in_bytes.lower % (unsigned)pagesize == 0); + ENSURE(env, env->geo_in_bytes.lower % globals.sys_pagesize == 0); + + ENSURE(env, env->geo_in_bytes.upper <= MAX_MAPSIZE); + ENSURE(env, env->geo_in_bytes.upper / (unsigned)pagesize <= MAX_PAGENO + 1); + ENSURE(env, env->geo_in_bytes.upper % (unsigned)pagesize == 0); + ENSURE(env, env->geo_in_bytes.upper % globals.sys_pagesize == 0); + + ENSURE(env, env->geo_in_bytes.now >= env->geo_in_bytes.lower); + ENSURE(env, env->geo_in_bytes.now <= env->geo_in_bytes.upper); + ENSURE(env, env->geo_in_bytes.now % (unsigned)pagesize == 0); + ENSURE(env, env->geo_in_bytes.now % globals.sys_pagesize == 0); + + ENSURE(env, env->geo_in_bytes.grow % (unsigned)pagesize == 0); + ENSURE(env, env->geo_in_bytes.grow % globals.sys_pagesize == 0); + ENSURE(env, env->geo_in_bytes.shrink % (unsigned)pagesize == 0); + ENSURE(env, env->geo_in_bytes.shrink % globals.sys_pagesize == 0); - tASSERT(txn, pnl_check_allocated(txn->tw.relist, - txn->mt_next_pgno - MDBX_ENABLE_REFUND)); - MDBX_val key, data; - if (ctx->lifo) { - if (ctx->cleaned_slot < (txn->tw.lifo_reclaimed - ? MDBX_PNL_GETSIZE(txn->tw.lifo_reclaimed) - : 0)) { - ctx->settled = 0; - ctx->cleaned_slot = 0; - ctx->reused_slot = 0; - ctx->filled_slot = ~0u; - /* LY: cleanup reclaimed records. */ - do { - ctx->cleaned_id = txn->tw.lifo_reclaimed[++ctx->cleaned_slot]; - tASSERT(txn, - ctx->cleaned_slot > 0 && - ctx->cleaned_id <= env->me_lck->mti_oldest_reader.weak); - key.iov_base = &ctx->cleaned_id; - key.iov_len = sizeof(ctx->cleaned_id); - rc = cursor_set(&ctx->cursor, &key, NULL, MDBX_SET).err; - if (rc == MDBX_NOTFOUND) - continue; - if (unlikely(rc != MDBX_SUCCESS)) - goto bailout; - if (likely(!ctx->dense)) { - rc = gcu_prepare_backlog(txn, ctx); - if (unlikely(rc != MDBX_SUCCESS)) - goto bailout; - } - tASSERT(txn, ctx->cleaned_id <= env->me_lck->mti_oldest_reader.weak); - TRACE("%s: cleanup-reclaimed-id [%zu]%" PRIaTXN, dbg_prefix_mode, - ctx->cleaned_slot, ctx->cleaned_id); - tASSERT(txn, *txn->mt_cursors == &ctx->cursor); - rc = cursor_del(&ctx->cursor, 0); - if (unlikely(rc != MDBX_SUCCESS)) - goto bailout; - } while (ctx->cleaned_slot < MDBX_PNL_GETSIZE(txn->tw.lifo_reclaimed)); - txl_sort(txn->tw.lifo_reclaimed); - } - } else { - /* Удаляем оставшиеся вынутые из GC записи. */ - while (ctx->cleaned_id <= txn->tw.last_reclaimed) { - rc = cursor_first(&ctx->cursor, &key, NULL); - if (rc == MDBX_NOTFOUND) - break; - if (unlikely(rc != MDBX_SUCCESS)) - goto bailout; - if (!MDBX_DISABLE_VALIDATION && - unlikely(key.iov_len != sizeof(txnid_t))) { - rc = MDBX_CORRUPTED; - goto bailout; - } - ctx->rid = ctx->cleaned_id; - ctx->settled = 0; - ctx->reused_slot = 0; - ctx->cleaned_id = unaligned_peek_u64(4, key.iov_base); - if (ctx->cleaned_id > txn->tw.last_reclaimed) + rc = MDBX_SUCCESS; + } else { + /* apply new params to opened environment */ + ENSURE(env, pagesize == (intptr_t)env->ps); + meta_t meta; + memset(&meta, 0, sizeof(meta)); + if (!env->txn) { + const meta_ptr_t head = meta_recent(env, &env->basal_txn->tw.troika); + + uint64_t timestamp = 0; + while ("workaround for " + "https://libmdbx.dqdkfa.ru/dead-github/issues/269") { + rc = coherency_fetch_head(env->basal_txn, head, ×tamp); + if (likely(rc == MDBX_SUCCESS)) break; - if (likely(!ctx->dense)) { - rc = gcu_prepare_backlog(txn, ctx); - if (unlikely(rc != MDBX_SUCCESS)) - goto bailout; - } - tASSERT(txn, ctx->cleaned_id <= txn->tw.last_reclaimed); - tASSERT(txn, ctx->cleaned_id <= env->me_lck->mti_oldest_reader.weak); - TRACE("%s: cleanup-reclaimed-id %" PRIaTXN, dbg_prefix_mode, - ctx->cleaned_id); - tASSERT(txn, *txn->mt_cursors == &ctx->cursor); - rc = cursor_del(&ctx->cursor, 0); - if (unlikely(rc != MDBX_SUCCESS)) + if (unlikely(rc != MDBX_RESULT_TRUE)) goto bailout; } - } - - tASSERT(txn, pnl_check_allocated(txn->tw.relist, - txn->mt_next_pgno - MDBX_ENABLE_REFUND)); - tASSERT(txn, dirtylist_check(txn)); - if (AUDIT_ENABLED()) { - rc = audit_ex(txn, ctx->retired_stored, false); - if (unlikely(rc != MDBX_SUCCESS)) + meta = *head.ptr_c; + const txnid_t txnid = safe64_txnid_next(head.txnid); + if (unlikely(txnid > MAX_TXNID)) { + rc = MDBX_TXN_FULL; + ERROR("txnid overflow, raise %d", rc); goto bailout; - } - - /* return suitable into unallocated space */ - if (txn_refund(txn)) { - tASSERT(txn, pnl_check_allocated(txn->tw.relist, - txn->mt_next_pgno - MDBX_ENABLE_REFUND)); - if (AUDIT_ENABLED()) { - rc = audit_ex(txn, ctx->retired_stored, false); - if (unlikely(rc != MDBX_SUCCESS)) - goto bailout; } + meta_set_txnid(env, &meta, txnid); } - /* handle loose pages - put ones into the reclaimed- or retired-list */ - if (txn->tw.loose_pages) { - tASSERT(txn, txn->tw.loose_count > 0); - /* Return loose page numbers to tw.relist, - * though usually none are left at this point. - * The pages themselves remain in dirtylist. */ - if (unlikely(!txn->tw.lifo_reclaimed && txn->tw.last_reclaimed < 1)) { - TRACE("%s: try allocate gc-slot for %zu loose-pages", dbg_prefix_mode, - txn->tw.loose_count); - rc = page_alloc_slowpath(&ctx->cursor, 0, MDBX_ALLOC_RESERVE).err; - if (rc == MDBX_SUCCESS) { - TRACE("%s: retry since gc-slot for %zu loose-pages available", - dbg_prefix_mode, txn->tw.loose_count); - continue; - } + const geo_t *const current_geo = &(env->txn ? env->txn : env->basal_txn)->geo; + /* update env-geo to avoid influences */ + env->geo_in_bytes.now = pgno2bytes(env, current_geo->now); + env->geo_in_bytes.lower = pgno2bytes(env, current_geo->lower); + env->geo_in_bytes.upper = pgno2bytes(env, current_geo->upper); + env->geo_in_bytes.grow = pgno2bytes(env, pv2pages(current_geo->grow_pv)); + env->geo_in_bytes.shrink = pgno2bytes(env, pv2pages(current_geo->shrink_pv)); + + geo_t new_geo; + new_geo.lower = bytes2pgno(env, size_lower); + new_geo.now = bytes2pgno(env, size_now); + new_geo.upper = bytes2pgno(env, size_upper); + new_geo.grow_pv = pages2pv(bytes2pgno(env, growth_step)); + new_geo.shrink_pv = pages2pv(bytes2pgno(env, shrink_threshold)); + new_geo.first_unallocated = current_geo->first_unallocated; + + ENSURE(env, pgno_align2os_bytes(env, new_geo.lower) == (size_t)size_lower); + ENSURE(env, pgno_align2os_bytes(env, new_geo.upper) == (size_t)size_upper); + ENSURE(env, pgno_align2os_bytes(env, new_geo.now) == (size_t)size_now); + ENSURE(env, new_geo.grow_pv == pages2pv(pv2pages(new_geo.grow_pv))); + ENSURE(env, new_geo.shrink_pv == pages2pv(pv2pages(new_geo.shrink_pv))); + + ENSURE(env, (size_t)size_lower >= MIN_MAPSIZE); + ENSURE(env, new_geo.lower >= MIN_PAGENO); + ENSURE(env, (size_t)size_upper <= MAX_MAPSIZE); + ENSURE(env, new_geo.upper <= MAX_PAGENO + 1); + ENSURE(env, new_geo.now >= new_geo.first_unallocated); + ENSURE(env, new_geo.upper >= new_geo.now); + ENSURE(env, new_geo.now >= new_geo.lower); - /* Put loose page numbers in tw.retired_pages, - * since unable to return them to tw.relist. */ - if (unlikely((rc = pnl_need(&txn->tw.retired_pages, - txn->tw.loose_count)) != 0)) + if (memcmp(current_geo, &new_geo, sizeof(geo_t)) != 0) { +#if defined(_WIN32) || defined(_WIN64) + /* Was DB shrinking disabled before and now it will be enabled? */ + if (new_geo.lower < new_geo.upper && new_geo.shrink_pv && + !(current_geo->lower < current_geo->upper && current_geo->shrink_pv)) { + if (!env->lck_mmap.lck) { + rc = MDBX_EPERM; goto bailout; - for (MDBX_page *lp = txn->tw.loose_pages; lp; lp = mp_next(lp)) { - pnl_xappend(txn->tw.retired_pages, lp->mp_pgno); - MDBX_ASAN_UNPOISON_MEMORY_REGION(&mp_next(lp), sizeof(MDBX_page *)); - VALGRIND_MAKE_MEM_DEFINED(&mp_next(lp), sizeof(MDBX_page *)); } - TRACE("%s: append %zu loose-pages to retired-pages", dbg_prefix_mode, - txn->tw.loose_count); - } else { - /* Room for loose pages + temp PNL with same */ - rc = pnl_need(&txn->tw.relist, 2 * txn->tw.loose_count + 2); - if (unlikely(rc != MDBX_SUCCESS)) + int err = lck_rdt_lock(env); + if (unlikely(MDBX_IS_ERROR(err))) { + rc = err; goto bailout; - MDBX_PNL loose = txn->tw.relist + MDBX_PNL_ALLOCLEN(txn->tw.relist) - - txn->tw.loose_count - 1; - size_t count = 0; - for (MDBX_page *lp = txn->tw.loose_pages; lp; lp = mp_next(lp)) { - tASSERT(txn, lp->mp_flags == P_LOOSE); - loose[++count] = lp->mp_pgno; - MDBX_ASAN_UNPOISON_MEMORY_REGION(&mp_next(lp), sizeof(MDBX_page *)); - VALGRIND_MAKE_MEM_DEFINED(&mp_next(lp), sizeof(MDBX_page *)); } - tASSERT(txn, count == txn->tw.loose_count); - MDBX_PNL_SETSIZE(loose, count); - pnl_sort(loose, txn->mt_next_pgno); - pnl_merge(txn->tw.relist, loose); - TRACE("%s: append %zu loose-pages to reclaimed-pages", dbg_prefix_mode, - txn->tw.loose_count); - } - /* filter-out list of dirty-pages from loose-pages */ - MDBX_dpl *const dl = txn->tw.dirtylist; - if (dl) { - tASSERT(txn, (txn->mt_flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); - tASSERT(txn, dl->sorted <= dl->length); - size_t w = 0, sorted_out = 0; - for (size_t r = w; ++r <= dl->length;) { - MDBX_page *dp = dl->items[r].ptr; - tASSERT(txn, dp->mp_flags == P_LOOSE || IS_MODIFIABLE(txn, dp)); - tASSERT(txn, dpl_endpgno(dl, r) <= txn->mt_next_pgno); - if ((dp->mp_flags & P_LOOSE) == 0) { - if (++w != r) - dl->items[w] = dl->items[r]; - } else { - tASSERT(txn, dp->mp_flags == P_LOOSE); - sorted_out += dl->sorted >= r; - if (!MDBX_AVOID_MSYNC || !(env->me_flags & MDBX_WRITEMAP)) { - tASSERT(txn, (txn->mt_flags & MDBX_WRITEMAP) == 0); - dpage_free(env, dp, 1); - } + /* Check if there are any reading threads that do not use the SRWL */ + const size_t CurrentTid = GetCurrentThreadId(); + const reader_slot_t *const begin = env->lck_mmap.lck->rdt; + const reader_slot_t *const end = begin + atomic_load32(&env->lck_mmap.lck->rdt_length, mo_AcquireRelease); + for (const reader_slot_t *reader = begin; reader < end; ++reader) { + if (reader->pid.weak == env->pid && reader->tid.weak != CurrentTid) { + /* At least one thread may don't use SRWL */ + rc = MDBX_EPERM; + break; } } - TRACE("%s: filtered-out loose-pages from %zu -> %zu dirty-pages", - dbg_prefix_mode, dl->length, w); - tASSERT(txn, txn->tw.loose_count == dl->length - w); - dl->sorted -= sorted_out; - tASSERT(txn, dl->sorted <= w); - dpl_setlen(dl, w); - dl->pages_including_loose -= txn->tw.loose_count; - txn->tw.dirtyroom += txn->tw.loose_count; - tASSERT(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length == - (txn->mt_parent ? txn->mt_parent->tw.dirtyroom - : txn->mt_env->me_options.dp_limit)); - } else { - tASSERT(txn, (txn->mt_flags & MDBX_WRITEMAP) != 0 && !MDBX_AVOID_MSYNC); - } - txn->tw.loose_pages = NULL; - txn->tw.loose_count = 0; -#if MDBX_ENABLE_REFUND - txn->tw.loose_refund_wl = 0; -#endif /* MDBX_ENABLE_REFUND */ - } - const size_t amount = MDBX_PNL_GETSIZE(txn->tw.relist); - /* handle retired-list - store ones into single gc-record */ - if (ctx->retired_stored < MDBX_PNL_GETSIZE(txn->tw.retired_pages)) { - if (unlikely(!ctx->retired_stored)) { - /* Make sure last page of GC is touched and on retired-list */ - rc = cursor_last(&ctx->cursor, nullptr, nullptr); - if (likely(rc == MDBX_SUCCESS)) - rc = gcu_touch(ctx); - if (unlikely(rc != MDBX_SUCCESS) && rc != MDBX_NOTFOUND) + lck_rdt_unlock(env); + if (unlikely(rc != MDBX_SUCCESS)) goto bailout; } +#endif /* Windows */ -#if MDBX_ENABLE_BIGFOOT - size_t retired_pages_before; - do { - if (ctx->bigfoot > txn->mt_txnid) { - rc = gcu_clean_stored_retired(txn, ctx); - if (unlikely(rc != MDBX_SUCCESS)) - goto bailout; - tASSERT(txn, ctx->bigfoot <= txn->mt_txnid); - } - - retired_pages_before = MDBX_PNL_GETSIZE(txn->tw.retired_pages); - rc = gcu_prepare_backlog(txn, ctx); + if (new_geo.now != current_geo->now || new_geo.upper != current_geo->upper) { + rc = dxb_resize(env, current_geo->first_unallocated, new_geo.now, new_geo.upper, explicit_resize); if (unlikely(rc != MDBX_SUCCESS)) goto bailout; - if (retired_pages_before != MDBX_PNL_GETSIZE(txn->tw.retired_pages)) { - TRACE("%s: retired-list changed (%zu -> %zu), retry", dbg_prefix_mode, - retired_pages_before, MDBX_PNL_GETSIZE(txn->tw.retired_pages)); - break; + } + if (env->txn) { + env->txn->geo = new_geo; + env->txn->flags |= MDBX_TXN_DIRTY; + } else { + meta.geometry = new_geo; + rc = dxb_sync_locked(env, env->flags, &meta, &env->basal_txn->tw.troika); + if (likely(rc == MDBX_SUCCESS)) { + env->geo_in_bytes.now = pgno2bytes(env, new_geo.now = meta.geometry.now); + env->geo_in_bytes.upper = pgno2bytes(env, new_geo.upper = meta.geometry.upper); } + } + } + if (likely(rc == MDBX_SUCCESS)) { + /* update env-geo to avoid influences */ + eASSERT(env, env->geo_in_bytes.now == pgno2bytes(env, new_geo.now)); + env->geo_in_bytes.lower = pgno2bytes(env, new_geo.lower); + eASSERT(env, env->geo_in_bytes.upper == pgno2bytes(env, new_geo.upper)); + env->geo_in_bytes.grow = pgno2bytes(env, pv2pages(new_geo.grow_pv)); + env->geo_in_bytes.shrink = pgno2bytes(env, pv2pages(new_geo.shrink_pv)); + } + } - pnl_sort(txn->tw.retired_pages, txn->mt_next_pgno); - ctx->retired_stored = 0; - ctx->bigfoot = txn->mt_txnid; - do { - if (ctx->retired_stored) { - rc = gcu_prepare_backlog(txn, ctx); - if (unlikely(rc != MDBX_SUCCESS)) - goto bailout; - if (ctx->retired_stored >= - MDBX_PNL_GETSIZE(txn->tw.retired_pages)) { - TRACE("%s: retired-list changed (%zu -> %zu), retry", - dbg_prefix_mode, retired_pages_before, - MDBX_PNL_GETSIZE(txn->tw.retired_pages)); - break; - } - } - key.iov_len = sizeof(txnid_t); - key.iov_base = &ctx->bigfoot; - const size_t left = - MDBX_PNL_GETSIZE(txn->tw.retired_pages) - ctx->retired_stored; - const size_t chunk = - (left > env->me_maxgc_ov1page && ctx->bigfoot < MAX_TXNID) - ? env->me_maxgc_ov1page - : left; - data.iov_len = (chunk + 1) * sizeof(pgno_t); - rc = cursor_put_nochecklen(&ctx->cursor, &key, &data, MDBX_RESERVE); - if (unlikely(rc != MDBX_SUCCESS)) - goto bailout; +bailout: + if (should_unlock) + lck_txn_unlock(env); + return LOG_IFERR(rc); +} -#if MDBX_DEBUG && (defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__)) - /* Для предотвращения предупреждения Valgrind из mdbx_dump_val() - * вызванное через макрос DVAL_DEBUG() на выходе - * из cursor_set(MDBX_SET_KEY), которая вызывается как выше в цикле - * очистки, так и ниже в цикле заполнения зарезервированных элементов. - */ - memset(data.iov_base, 0xBB, data.iov_len); -#endif /* MDBX_DEBUG && (MDBX_USE_VALGRIND || __SANITIZE_ADDRESS__) */ - - if (retired_pages_before == MDBX_PNL_GETSIZE(txn->tw.retired_pages)) { - const size_t at = (ctx->lifo == MDBX_PNL_ASCENDING) - ? left - chunk - : ctx->retired_stored; - pgno_t *const begin = txn->tw.retired_pages + at; - /* MDBX_PNL_ASCENDING == false && LIFO == false: - * - the larger pgno is at the beginning of retired list - * and should be placed with the larger txnid. - * MDBX_PNL_ASCENDING == true && LIFO == true: - * - the larger pgno is at the ending of retired list - * and should be placed with the smaller txnid. - */ - const pgno_t save = *begin; - *begin = (pgno_t)chunk; - memcpy(data.iov_base, begin, data.iov_len); - *begin = save; - TRACE("%s: put-retired/bigfoot @ %" PRIaTXN - " (slice #%u) #%zu [%zu..%zu] of %zu", - dbg_prefix_mode, ctx->bigfoot, - (unsigned)(ctx->bigfoot - txn->mt_txnid), chunk, at, - at + chunk, retired_pages_before); - } - ctx->retired_stored += chunk; - } while (ctx->retired_stored < - MDBX_PNL_GETSIZE(txn->tw.retired_pages) && - (++ctx->bigfoot, true)); - } while (retired_pages_before != MDBX_PNL_GETSIZE(txn->tw.retired_pages)); -#else - /* Write to last page of GC */ - key.iov_len = sizeof(txnid_t); - key.iov_base = &txn->mt_txnid; - do { - gcu_prepare_backlog(txn, ctx); - data.iov_len = MDBX_PNL_SIZEOF(txn->tw.retired_pages); - rc = cursor_put_nochecklen(&ctx->cursor, &key, &data, MDBX_RESERVE); - if (unlikely(rc != MDBX_SUCCESS)) - goto bailout; +__cold int mdbx_env_sync_ex(MDBX_env *env, bool force, bool nonblock) { + int rc = check_env(env, true); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); -#if MDBX_DEBUG && (defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__)) - /* Для предотвращения предупреждения Valgrind из mdbx_dump_val() - * вызванное через макрос DVAL_DEBUG() на выходе - * из cursor_set(MDBX_SET_KEY), которая вызывается как выше в цикле - * очистки, так и ниже в цикле заполнения зарезервированных элементов. - */ - memset(data.iov_base, 0xBB, data.iov_len); -#endif /* MDBX_DEBUG && (MDBX_USE_VALGRIND || __SANITIZE_ADDRESS__) */ + return LOG_IFERR(env_sync(env, force, nonblock)); +} - /* Retry if tw.retired_pages[] grew during the Put() */ - } while (data.iov_len < MDBX_PNL_SIZEOF(txn->tw.retired_pages)); +/*----------------------------------------------------------------------------*/ - ctx->retired_stored = MDBX_PNL_GETSIZE(txn->tw.retired_pages); - pnl_sort(txn->tw.retired_pages, txn->mt_next_pgno); - eASSERT(env, data.iov_len == MDBX_PNL_SIZEOF(txn->tw.retired_pages)); - memcpy(data.iov_base, txn->tw.retired_pages, data.iov_len); +static void stat_add(const tree_t *db, MDBX_stat *const st, const size_t bytes) { + st->ms_depth += db->height; + st->ms_branch_pages += db->branch_pages; + st->ms_leaf_pages += db->leaf_pages; + st->ms_overflow_pages += db->large_pages; + st->ms_entries += db->items; + if (likely(bytes >= offsetof(MDBX_stat, ms_mod_txnid) + sizeof(st->ms_mod_txnid))) + st->ms_mod_txnid = (st->ms_mod_txnid > db->mod_txnid) ? st->ms_mod_txnid : db->mod_txnid; +} - TRACE("%s: put-retired #%zu @ %" PRIaTXN, dbg_prefix_mode, - ctx->retired_stored, txn->mt_txnid); -#endif /* MDBX_ENABLE_BIGFOOT */ - if (LOG_ENABLED(MDBX_LOG_EXTRA)) { - size_t i = ctx->retired_stored; - DEBUG_EXTRA("txn %" PRIaTXN " root %" PRIaPGNO " num %zu, retired-PNL", - txn->mt_txnid, txn->mt_dbs[FREE_DBI].md_root, i); - for (; i; i--) - DEBUG_EXTRA_PRINT(" %" PRIaPGNO, txn->tw.retired_pages[i]); - DEBUG_EXTRA_PRINT("%s\n", "."); - } - if (unlikely(amount != MDBX_PNL_GETSIZE(txn->tw.relist) && - ctx->settled)) { - TRACE("%s: reclaimed-list changed %zu -> %zu, retry", dbg_prefix_mode, - amount, MDBX_PNL_GETSIZE(txn->tw.relist)); - goto retry /* rare case, but avoids GC fragmentation - and one cycle. */ - ; - } - continue; - } +static int stat_acc(const MDBX_txn *txn, MDBX_stat *st, size_t bytes) { + memset(st, 0, bytes); - /* handle reclaimed and lost pages - merge and store both into gc */ - tASSERT(txn, pnl_check_allocated(txn->tw.relist, - txn->mt_next_pgno - MDBX_ENABLE_REFUND)); - tASSERT(txn, txn->tw.loose_count == 0); + int err = check_txn(txn, MDBX_TXN_BLOCKED); + if (unlikely(err != MDBX_SUCCESS)) + return err; - TRACE("%s", " >> reserving"); - if (AUDIT_ENABLED()) { - rc = audit_ex(txn, ctx->retired_stored, false); - if (unlikely(rc != MDBX_SUCCESS)) - goto bailout; - } - const size_t left = amount - ctx->settled; - TRACE("%s: amount %zu, settled %zd, left %zd, lifo-reclaimed-slots %zu, " - "reused-gc-slots %zu", - dbg_prefix_mode, amount, ctx->settled, left, - txn->tw.lifo_reclaimed ? MDBX_PNL_GETSIZE(txn->tw.lifo_reclaimed) : 0, - ctx->reused_slot); - if (0 >= (intptr_t)left) - break; + cursor_couple_t cx; + err = cursor_init(&cx.outer, (MDBX_txn *)txn, MAIN_DBI); + if (unlikely(err != MDBX_SUCCESS)) + return err; - const size_t prefer_max_scatter = MDBX_ENABLE_BIGFOOT ? MDBX_TXL_MAX : 257; - txnid_t reservation_gc_id; - if (ctx->lifo) { - if (txn->tw.lifo_reclaimed == nullptr) { - txn->tw.lifo_reclaimed = txl_alloc(); - if (unlikely(!txn->tw.lifo_reclaimed)) { - rc = MDBX_ENOMEM; - goto bailout; - } - } - if (MDBX_PNL_GETSIZE(txn->tw.lifo_reclaimed) < prefer_max_scatter && - left > (MDBX_PNL_GETSIZE(txn->tw.lifo_reclaimed) - ctx->reused_slot) * - env->me_maxgc_ov1page && - !ctx->dense) { - /* Hужен свободный для для сохранения списка страниц. */ - bool need_cleanup = false; - txnid_t snap_oldest = 0; - retry_rid: - do { - rc = page_alloc_slowpath(&ctx->cursor, 0, MDBX_ALLOC_RESERVE).err; - snap_oldest = env->me_lck->mti_oldest_reader.weak; - if (likely(rc == MDBX_SUCCESS)) { - TRACE("%s: took @%" PRIaTXN " from GC", dbg_prefix_mode, - MDBX_PNL_LAST(txn->tw.lifo_reclaimed)); - need_cleanup = true; - } - } while ( - rc == MDBX_SUCCESS && - MDBX_PNL_GETSIZE(txn->tw.lifo_reclaimed) < prefer_max_scatter && - left > - (MDBX_PNL_GETSIZE(txn->tw.lifo_reclaimed) - ctx->reused_slot) * - env->me_maxgc_ov1page); + const MDBX_env *const env = txn->env; + st->ms_psize = env->ps; + TXN_FOREACH_DBI_FROM(txn, dbi, + /* assuming GC is internal and not subject for accounting */ MAIN_DBI) { + if ((txn->dbi_state[dbi] & (DBI_VALID | DBI_STALE)) == DBI_VALID) + stat_add(txn->dbs + dbi, st, bytes); + } - if (likely(rc == MDBX_SUCCESS)) { - TRACE("%s: got enough from GC.", dbg_prefix_mode); - continue; - } else if (unlikely(rc != MDBX_NOTFOUND)) - /* LY: some troubles... */ - goto bailout; + if (!(txn->dbs[MAIN_DBI].flags & MDBX_DUPSORT) && txn->dbs[MAIN_DBI].items /* TODO: use `md_subs` field */) { - if (MDBX_PNL_GETSIZE(txn->tw.lifo_reclaimed)) { - if (need_cleanup) { - txl_sort(txn->tw.lifo_reclaimed); - ctx->cleaned_slot = 0; - } - ctx->rid = MDBX_PNL_LAST(txn->tw.lifo_reclaimed); - } else { - tASSERT(txn, txn->tw.last_reclaimed == 0); - if (unlikely(txn_oldest_reader(txn) != snap_oldest)) - /* should retry page_alloc_slowpath() - * if the oldest reader changes since the last attempt */ - goto retry_rid; - /* no reclaimable GC entries, - * therefore no entries with ID < mdbx_find_oldest(txn) */ - txn->tw.last_reclaimed = ctx->rid = snap_oldest; - TRACE("%s: none recycled yet, set rid to @%" PRIaTXN, dbg_prefix_mode, - ctx->rid); + /* scan and account not opened named tables */ + err = tree_search(&cx.outer, nullptr, Z_FIRST); + while (err == MDBX_SUCCESS) { + const page_t *mp = cx.outer.pg[cx.outer.top]; + for (size_t i = 0; i < page_numkeys(mp); i++) { + const node_t *node = page_node(mp, i); + if (node_flags(node) != N_TREE) + continue; + if (unlikely(node_ds(node) != sizeof(tree_t))) { + ERROR("%s/%d: %s %zu", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid table node size", node_ds(node)); + return MDBX_CORRUPTED; } - /* В GC нет годных к переработке записей, - * будем использовать свободные id в обратном порядке. */ - while (MDBX_PNL_GETSIZE(txn->tw.lifo_reclaimed) < prefer_max_scatter && - left > (MDBX_PNL_GETSIZE(txn->tw.lifo_reclaimed) - - ctx->reused_slot) * - env->me_maxgc_ov1page) { - if (unlikely(ctx->rid <= MIN_TXNID)) { - if (unlikely(MDBX_PNL_GETSIZE(txn->tw.lifo_reclaimed) <= - ctx->reused_slot)) { - VERBOSE("** restart: reserve depleted (reused_gc_slot %zu >= " - "lifo_reclaimed %zu" PRIaTXN, - ctx->reused_slot, - MDBX_PNL_GETSIZE(txn->tw.lifo_reclaimed)); - goto retry; - } + /* skip opened and already accounted */ + const MDBX_val name = {node_key(node), node_ks(node)}; + TXN_FOREACH_DBI_USER(txn, dbi) { + if ((txn->dbi_state[dbi] & (DBI_VALID | DBI_STALE)) == DBI_VALID && + env->kvs[MAIN_DBI].clc.k.cmp(&name, &env->kvs[dbi].name) == 0) { + node = nullptr; break; } - - tASSERT(txn, ctx->rid >= MIN_TXNID && ctx->rid <= MAX_TXNID); - ctx->rid -= 1; - key.iov_base = &ctx->rid; - key.iov_len = sizeof(ctx->rid); - rc = cursor_set(&ctx->cursor, &key, &data, MDBX_SET_KEY).err; - if (unlikely(rc == MDBX_SUCCESS)) { - DEBUG("%s: GC's id %" PRIaTXN " is present, going to first", - dbg_prefix_mode, ctx->rid); - rc = cursor_first(&ctx->cursor, &key, nullptr); - if (unlikely(rc != MDBX_SUCCESS || - key.iov_len != sizeof(txnid_t))) { - rc = MDBX_CORRUPTED; - goto bailout; - } - const txnid_t gc_first = unaligned_peek_u64(4, key.iov_base); - if (gc_first <= MIN_TXNID) { - DEBUG("%s: no free GC's id(s) less than %" PRIaTXN - " (going dense-mode)", - dbg_prefix_mode, ctx->rid); - ctx->dense = true; - break; - } - ctx->rid = gc_first - 1; - } - - eASSERT(env, !ctx->dense); - rc = txl_append(&txn->tw.lifo_reclaimed, ctx->rid); - if (unlikely(rc != MDBX_SUCCESS)) - goto bailout; - - if (ctx->reused_slot) - /* rare case, but it is better to clear and re-create GC entries - * with less fragmentation. */ - need_cleanup = true; - else - ctx->cleaned_slot += - 1 /* mark cleanup is not needed for added slot. */; - - TRACE("%s: append @%" PRIaTXN - " to lifo-reclaimed, cleaned-gc-slot = %zu", - dbg_prefix_mode, ctx->rid, ctx->cleaned_slot); } - if (need_cleanup || ctx->dense) { - if (ctx->cleaned_slot) { - TRACE("%s: restart to clear and re-create GC entries", - dbg_prefix_mode); - goto retry; - } - continue; + if (node) { + tree_t db; + memcpy(&db, node_data(node), sizeof(db)); + stat_add(&db, st, bytes); } } - - const size_t i = - MDBX_PNL_GETSIZE(txn->tw.lifo_reclaimed) - ctx->reused_slot; - tASSERT(txn, i > 0 && i <= MDBX_PNL_GETSIZE(txn->tw.lifo_reclaimed)); - reservation_gc_id = txn->tw.lifo_reclaimed[i]; - TRACE("%s: take @%" PRIaTXN " from lifo-reclaimed[%zu]", dbg_prefix_mode, - reservation_gc_id, i); - } else { - tASSERT(txn, txn->tw.lifo_reclaimed == NULL); - if (unlikely(ctx->rid == 0)) { - ctx->rid = txn_oldest_reader(txn); - rc = cursor_first(&ctx->cursor, &key, nullptr); - if (likely(rc == MDBX_SUCCESS)) { - if (unlikely(key.iov_len != sizeof(txnid_t))) { - rc = MDBX_CORRUPTED; - goto bailout; - } - const txnid_t gc_first = unaligned_peek_u64(4, key.iov_base); - if (ctx->rid >= gc_first) - ctx->rid = gc_first - 1; - if (unlikely(ctx->rid == 0)) { - ERROR("%s", "** no GC tail-space to store (going dense-mode)"); - ctx->dense = true; - goto retry; - } - } else if (rc != MDBX_NOTFOUND) - goto bailout; - txn->tw.last_reclaimed = ctx->rid; - ctx->cleaned_id = ctx->rid + 1; - } - reservation_gc_id = ctx->rid--; - TRACE("%s: take @%" PRIaTXN " from head-gc-id", dbg_prefix_mode, - reservation_gc_id); + err = cursor_sibling_right(&cx.outer); } - ++ctx->reused_slot; + if (unlikely(err != MDBX_NOTFOUND)) + return err; + } - size_t chunk = left; - if (unlikely(chunk > env->me_maxgc_ov1page)) { - const size_t avail_gc_slots = - txn->tw.lifo_reclaimed - ? MDBX_PNL_GETSIZE(txn->tw.lifo_reclaimed) - ctx->reused_slot + 1 - : (ctx->rid < INT16_MAX) ? (size_t)ctx->rid - : INT16_MAX; - if (avail_gc_slots > 1) { -#if MDBX_ENABLE_BIGFOOT - chunk = (chunk < env->me_maxgc_ov1page * (size_t)2) - ? chunk / 2 - : env->me_maxgc_ov1page; -#else - if (chunk < env->me_maxgc_ov1page * 2) - chunk /= 2; - else { - const size_t threshold = - env->me_maxgc_ov1page * ((avail_gc_slots < prefer_max_scatter) - ? avail_gc_slots - : prefer_max_scatter); - if (left < threshold) - chunk = env->me_maxgc_ov1page; - else { - const size_t tail = left - threshold + env->me_maxgc_ov1page + 1; - size_t span = 1; - size_t avail = ((pgno2bytes(env, span) - PAGEHDRSZ) / - sizeof(pgno_t)) /* - 1 + span */; - if (tail > avail) { - for (size_t i = amount - span; i > 0; --i) { - if (MDBX_PNL_ASCENDING ? (txn->tw.relist[i] + span) - : (txn->tw.relist[i] - span) == - txn->tw.relist[i + span]) { - span += 1; - avail = - ((pgno2bytes(env, span) - PAGEHDRSZ) / sizeof(pgno_t)) - - 1 + span; - if (avail >= tail) - break; - } - } - } + return MDBX_SUCCESS; +} - chunk = (avail >= tail) ? tail - span - : (avail_gc_slots > 3 && - ctx->reused_slot < prefer_max_scatter - 3) - ? avail - span - : tail; - } - } -#endif /* MDBX_ENABLE_BIGFOOT */ - } - } - tASSERT(txn, chunk > 0); +__cold int mdbx_env_stat_ex(const MDBX_env *env, const MDBX_txn *txn, MDBX_stat *dest, size_t bytes) { + if (unlikely(!dest)) + return LOG_IFERR(MDBX_EINVAL); + const size_t size_before_modtxnid = offsetof(MDBX_stat, ms_mod_txnid); + if (unlikely(bytes != sizeof(MDBX_stat)) && bytes != size_before_modtxnid) + return LOG_IFERR(MDBX_EINVAL); - TRACE("%s: gc_rid %" PRIaTXN ", reused_gc_slot %zu, reservation-id " - "%" PRIaTXN, - dbg_prefix_mode, ctx->rid, ctx->reused_slot, reservation_gc_id); - - TRACE("%s: chunk %zu, gc-per-ovpage %u", dbg_prefix_mode, chunk, - env->me_maxgc_ov1page); - - tASSERT(txn, reservation_gc_id <= env->me_lck->mti_oldest_reader.weak); - if (unlikely( - reservation_gc_id < MIN_TXNID || - reservation_gc_id > - atomic_load64(&env->me_lck->mti_oldest_reader, mo_Relaxed))) { - ERROR("** internal error (reservation_gc_id %" PRIaTXN ")", - reservation_gc_id); - rc = MDBX_PROBLEM; - goto bailout; - } + if (likely(txn)) { + if (env && unlikely(txn->env != env)) + return LOG_IFERR(MDBX_EINVAL); + return LOG_IFERR(stat_acc(txn, dest, bytes)); + } - key.iov_len = sizeof(reservation_gc_id); - key.iov_base = &reservation_gc_id; - data.iov_len = (chunk + 1) * sizeof(pgno_t); - TRACE("%s: reserve %zu [%zu...%zu) @%" PRIaTXN, dbg_prefix_mode, chunk, - ctx->settled + 1, ctx->settled + chunk + 1, reservation_gc_id); - gcu_prepare_backlog(txn, ctx); - rc = cursor_put_nochecklen(&ctx->cursor, &key, &data, - MDBX_RESERVE | MDBX_NOOVERWRITE); - tASSERT(txn, pnl_check_allocated(txn->tw.relist, - txn->mt_next_pgno - MDBX_ENABLE_REFUND)); - if (unlikely(rc != MDBX_SUCCESS)) - goto bailout; + int err = check_env(env, true); + if (unlikely(err != MDBX_SUCCESS)) + return LOG_IFERR(err); - gcu_clean_reserved(env, data); - ctx->settled += chunk; - TRACE("%s: settled %zu (+%zu), continue", dbg_prefix_mode, ctx->settled, - chunk); - - if (txn->tw.lifo_reclaimed && - unlikely(amount < MDBX_PNL_GETSIZE(txn->tw.relist)) && - (ctx->loop < 5 || - MDBX_PNL_GETSIZE(txn->tw.relist) - amount > env->me_maxgc_ov1page)) { - NOTICE("** restart: reclaimed-list growth %zu -> %zu", amount, - MDBX_PNL_GETSIZE(txn->tw.relist)); - goto retry; - } + MDBX_txn *txn_owned = env_owned_wrtxn(env); + if (txn_owned) + /* inside write-txn */ + return LOG_IFERR(stat_acc(txn_owned, dest, bytes)); - continue; - } + err = mdbx_txn_begin((MDBX_env *)env, nullptr, MDBX_TXN_RDONLY, &txn_owned); + if (unlikely(err != MDBX_SUCCESS)) + return LOG_IFERR(err); - tASSERT(txn, - ctx->cleaned_slot == (txn->tw.lifo_reclaimed - ? MDBX_PNL_GETSIZE(txn->tw.lifo_reclaimed) - : 0)); + const int rc = stat_acc(txn_owned, dest, bytes); + err = mdbx_txn_abort(txn_owned); + if (unlikely(err != MDBX_SUCCESS)) + return LOG_IFERR(err); + return LOG_IFERR(rc); +} +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 - TRACE("%s", " >> filling"); - /* Fill in the reserved records */ - ctx->filled_slot = - txn->tw.lifo_reclaimed - ? MDBX_PNL_GETSIZE(txn->tw.lifo_reclaimed) - ctx->reused_slot - : ctx->reused_slot; - rc = MDBX_SUCCESS; - tASSERT(txn, pnl_check_allocated(txn->tw.relist, - txn->mt_next_pgno - MDBX_ENABLE_REFUND)); - tASSERT(txn, dirtylist_check(txn)); - if (MDBX_PNL_GETSIZE(txn->tw.relist)) { - MDBX_val key, data; - key.iov_len = data.iov_len = 0; /* avoid MSVC warning */ - key.iov_base = data.iov_base = NULL; - - const size_t amount = MDBX_PNL_GETSIZE(txn->tw.relist); - size_t left = amount; - if (txn->tw.lifo_reclaimed == nullptr) { - tASSERT(txn, ctx->lifo == 0); - rc = cursor_first(&ctx->cursor, &key, &data); - if (unlikely(rc != MDBX_SUCCESS)) - goto bailout; - } else { - tASSERT(txn, ctx->lifo != 0); - } +/*------------------------------------------------------------------------------ + * Readers API */ - while (true) { - txnid_t fill_gc_id; - TRACE("%s: left %zu of %zu", dbg_prefix_mode, left, - MDBX_PNL_GETSIZE(txn->tw.relist)); - if (txn->tw.lifo_reclaimed == nullptr) { - tASSERT(txn, ctx->lifo == 0); - fill_gc_id = unaligned_peek_u64(4, key.iov_base); - if (ctx->filled_slot-- == 0 || fill_gc_id > txn->tw.last_reclaimed) { - VERBOSE( - "** restart: reserve depleted (filled_slot %zu, fill_id %" PRIaTXN - " > last_reclaimed %" PRIaTXN, - ctx->filled_slot, fill_gc_id, txn->tw.last_reclaimed); - goto retry; - } - } else { - tASSERT(txn, ctx->lifo != 0); - if (++ctx->filled_slot > MDBX_PNL_GETSIZE(txn->tw.lifo_reclaimed)) { - VERBOSE("** restart: reserve depleted (filled_gc_slot %zu > " - "lifo_reclaimed %zu" PRIaTXN, - ctx->filled_slot, MDBX_PNL_GETSIZE(txn->tw.lifo_reclaimed)); - goto retry; - } - fill_gc_id = txn->tw.lifo_reclaimed[ctx->filled_slot]; - TRACE("%s: seek-reservation @%" PRIaTXN " at lifo_reclaimed[%zu]", - dbg_prefix_mode, fill_gc_id, ctx->filled_slot); - key.iov_base = &fill_gc_id; - key.iov_len = sizeof(fill_gc_id); - rc = cursor_set(&ctx->cursor, &key, &data, MDBX_SET_KEY).err; - if (unlikely(rc != MDBX_SUCCESS)) - goto bailout; - } - tASSERT(txn, ctx->cleaned_slot == - (txn->tw.lifo_reclaimed - ? MDBX_PNL_GETSIZE(txn->tw.lifo_reclaimed) - : 0)); - tASSERT(txn, fill_gc_id > 0 && - fill_gc_id <= env->me_lck->mti_oldest_reader.weak); - key.iov_base = &fill_gc_id; - key.iov_len = sizeof(fill_gc_id); +__cold int mdbx_reader_list(const MDBX_env *env, MDBX_reader_list_func *func, void *ctx) { + int rc = check_env(env, true); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); - tASSERT(txn, data.iov_len >= sizeof(pgno_t) * 2); - size_t chunk = data.iov_len / sizeof(pgno_t) - 1; - if (unlikely(chunk > left)) { - TRACE("%s: chunk %zu > left %zu, @%" PRIaTXN, dbg_prefix_mode, chunk, - left, fill_gc_id); - if ((ctx->loop < 5 && chunk - left > ctx->loop / 2) || - chunk - left > env->me_maxgc_ov1page) { - data.iov_len = (left + 1) * sizeof(pgno_t); - } - chunk = left; - } - rc = cursor_put_nochecklen(&ctx->cursor, &key, &data, - MDBX_CURRENT | MDBX_RESERVE); - if (unlikely(rc != MDBX_SUCCESS)) - goto bailout; - gcu_clean_reserved(env, data); + if (unlikely(!func)) + return LOG_IFERR(MDBX_EINVAL); - if (unlikely(txn->tw.loose_count || - amount != MDBX_PNL_GETSIZE(txn->tw.relist))) { - NOTICE("** restart: reclaimed-list growth (%zu -> %zu, loose +%zu)", - amount, MDBX_PNL_GETSIZE(txn->tw.relist), txn->tw.loose_count); - goto retry; - } - if (unlikely(txn->tw.lifo_reclaimed - ? ctx->cleaned_slot < - MDBX_PNL_GETSIZE(txn->tw.lifo_reclaimed) - : ctx->cleaned_id < txn->tw.last_reclaimed)) { - NOTICE("%s", "** restart: reclaimed-slots changed"); - goto retry; - } - if (unlikely(ctx->retired_stored != - MDBX_PNL_GETSIZE(txn->tw.retired_pages))) { - tASSERT(txn, - ctx->retired_stored < MDBX_PNL_GETSIZE(txn->tw.retired_pages)); - NOTICE("** restart: retired-list growth (%zu -> %zu)", - ctx->retired_stored, MDBX_PNL_GETSIZE(txn->tw.retired_pages)); - goto retry; - } + rc = MDBX_RESULT_TRUE; + int serial = 0; + lck_t *const lck = env->lck_mmap.lck; + if (likely(lck)) { + const size_t snap_nreaders = atomic_load32(&lck->rdt_length, mo_AcquireRelease); + for (size_t i = 0; i < snap_nreaders; i++) { + const reader_slot_t *r = lck->rdt + i; + retry_reader:; + const uint32_t pid = atomic_load32(&r->pid, mo_AcquireRelease); + if (!pid) + continue; + txnid_t txnid = safe64_read(&r->txnid); + const uint64_t tid = atomic_load64(&r->tid, mo_Relaxed); + const pgno_t pages_used = atomic_load32(&r->snapshot_pages_used, mo_Relaxed); + const uint64_t reader_pages_retired = atomic_load64(&r->snapshot_pages_retired, mo_Relaxed); + if (unlikely(txnid != safe64_read(&r->txnid) || pid != atomic_load32(&r->pid, mo_AcquireRelease) || + tid != atomic_load64(&r->tid, mo_Relaxed) || + pages_used != atomic_load32(&r->snapshot_pages_used, mo_Relaxed) || + reader_pages_retired != atomic_load64(&r->snapshot_pages_retired, mo_Relaxed))) + goto retry_reader; - pgno_t *dst = data.iov_base; - *dst++ = (pgno_t)chunk; - pgno_t *src = MDBX_PNL_BEGIN(txn->tw.relist) + left - chunk; - memcpy(dst, src, chunk * sizeof(pgno_t)); - pgno_t *from = src, *to = src + chunk; - TRACE("%s: fill %zu [ %zu:%" PRIaPGNO "...%zu:%" PRIaPGNO "] @%" PRIaTXN, - dbg_prefix_mode, chunk, from - txn->tw.relist, from[0], - to - txn->tw.relist, to[-1], fill_gc_id); + eASSERT(env, txnid > 0); + if (txnid >= SAFE64_INVALID_THRESHOLD) + txnid = 0; - left -= chunk; - if (AUDIT_ENABLED()) { - rc = audit_ex(txn, ctx->retired_stored + amount - left, true); - if (unlikely(rc != MDBX_SUCCESS)) - goto bailout; - } - if (left == 0) { - rc = MDBX_SUCCESS; - break; - } + size_t bytes_used = 0; + size_t bytes_retained = 0; + uint64_t lag = 0; + if (txnid) { + troika_t troika = meta_tap(env); + retry_header:; + const meta_ptr_t head = meta_recent(env, &troika); + const uint64_t head_pages_retired = unaligned_peek_u64_volatile(4, head.ptr_v->pages_retired); + if (unlikely(meta_should_retry(env, &troika) || + head_pages_retired != unaligned_peek_u64_volatile(4, head.ptr_v->pages_retired))) + goto retry_header; - if (txn->tw.lifo_reclaimed == nullptr) { - tASSERT(txn, ctx->lifo == 0); - rc = cursor_next(&ctx->cursor, &key, &data, MDBX_NEXT); - if (unlikely(rc != MDBX_SUCCESS)) - goto bailout; - } else { - tASSERT(txn, ctx->lifo != 0); + lag = (head.txnid - txnid) / xMDBX_TXNID_STEP; + bytes_used = pgno2bytes(env, pages_used); + bytes_retained = (head_pages_retired > reader_pages_retired) + ? pgno2bytes(env, (pgno_t)(head_pages_retired - reader_pages_retired)) + : 0; } + rc = func(ctx, ++serial, (unsigned)i, pid, (mdbx_tid_t)((intptr_t)tid), txnid, lag, bytes_used, bytes_retained); + if (unlikely(rc != MDBX_SUCCESS)) + break; } } - tASSERT(txn, rc == MDBX_SUCCESS); - if (unlikely(txn->tw.loose_count != 0)) { - NOTICE("** restart: got %zu loose pages", txn->tw.loose_count); - goto retry; + return LOG_IFERR(rc); +} + +__cold int mdbx_reader_check(MDBX_env *env, int *dead) { + if (dead) + *dead = 0; + return LOG_IFERR(mvcc_cleanup_dead(env, false, dead)); +} + +__cold int mdbx_thread_register(const MDBX_env *env) { + int rc = check_env(env, true); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); + + if (unlikely(!env->lck_mmap.lck)) + return LOG_IFERR((env->flags & MDBX_EXCLUSIVE) ? MDBX_EINVAL : MDBX_EPERM); + + if (unlikely((env->flags & ENV_TXKEY) == 0)) { + eASSERT(env, env->flags & MDBX_NOSTICKYTHREADS); + return LOG_IFERR(MDBX_EINVAL) /* MDBX_NOSTICKYTHREADS mode */; } - if (unlikely(ctx->filled_slot != - (txn->tw.lifo_reclaimed - ? MDBX_PNL_GETSIZE(txn->tw.lifo_reclaimed) - : 0))) { - const bool will_retry = ctx->loop < 9; - NOTICE("** %s: reserve excess (filled-slot %zu, loop %zu)", - will_retry ? "restart" : "ignore", ctx->filled_slot, ctx->loop); - if (will_retry) - goto retry; + eASSERT(env, (env->flags & (MDBX_NOSTICKYTHREADS | ENV_TXKEY)) == ENV_TXKEY); + reader_slot_t *r = thread_rthc_get(env->me_txkey); + if (unlikely(r != nullptr)) { + eASSERT(env, r->pid.weak == env->pid); + eASSERT(env, r->tid.weak == osal_thread_self()); + if (unlikely(r->pid.weak != env->pid)) + return LOG_IFERR(MDBX_BAD_RSLOT); + return MDBX_RESULT_TRUE /* already registered */; } - tASSERT(txn, - txn->tw.lifo_reclaimed == NULL || - ctx->cleaned_slot == MDBX_PNL_GETSIZE(txn->tw.lifo_reclaimed)); + return LOG_IFERR(mvcc_bind_slot((MDBX_env *)env).err); +} -bailout: - txn->mt_cursors[FREE_DBI] = ctx->cursor.mc_next; +__cold int mdbx_thread_unregister(const MDBX_env *env) { + int rc = check_env(env, true); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); - MDBX_PNL_SETSIZE(txn->tw.relist, 0); -#if MDBX_ENABLE_PROFGC - env->me_lck->mti_pgop_stat.gc_prof.wloops += (uint32_t)ctx->loop; -#endif /* MDBX_ENABLE_PROFGC */ - TRACE("<<< %zu loops, rc = %d", ctx->loop, rc); - return rc; -} + if (unlikely(!env->lck_mmap.lck)) + return MDBX_RESULT_TRUE; -static int txn_write(MDBX_txn *txn, iov_ctx_t *ctx) { - tASSERT(txn, (txn->mt_flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); - MDBX_dpl *const dl = dpl_sort(txn); - int rc = MDBX_SUCCESS; - size_t r, w, total_npages = 0; - for (w = 0, r = 1; r <= dl->length; ++r) { - MDBX_page *dp = dl->items[r].ptr; - if (dp->mp_flags & P_LOOSE) { - dl->items[++w] = dl->items[r]; - continue; - } - unsigned npages = dpl_npages(dl, r); - total_npages += npages; - rc = iov_page(txn, ctx, dp, npages); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; + if (unlikely((env->flags & ENV_TXKEY) == 0)) { + eASSERT(env, env->flags & MDBX_NOSTICKYTHREADS); + return MDBX_RESULT_TRUE /* MDBX_NOSTICKYTHREADS mode */; } - if (!iov_empty(ctx)) { - tASSERT(txn, rc == MDBX_SUCCESS); - rc = iov_write(ctx); - } + eASSERT(env, (env->flags & (MDBX_NOSTICKYTHREADS | ENV_TXKEY)) == ENV_TXKEY); + reader_slot_t *r = thread_rthc_get(env->me_txkey); + if (unlikely(r == nullptr)) + return MDBX_RESULT_TRUE /* not registered */; - if (likely(rc == MDBX_SUCCESS) && ctx->fd == txn->mt_env->me_lazy_fd) { - txn->mt_env->me_lck->mti_unsynced_pages.weak += total_npages; - if (!txn->mt_env->me_lck->mti_eoos_timestamp.weak) - txn->mt_env->me_lck->mti_eoos_timestamp.weak = osal_monotime(); - } + eASSERT(env, r->pid.weak == env->pid); + if (unlikely(r->pid.weak != env->pid || r->tid.weak != osal_thread_self())) + return LOG_IFERR(MDBX_BAD_RSLOT); - txn->tw.dirtylist->pages_including_loose -= total_npages; - while (r <= dl->length) - dl->items[++w] = dl->items[r++]; + eASSERT(env, r->txnid.weak >= SAFE64_INVALID_THRESHOLD); + if (unlikely(r->txnid.weak < SAFE64_INVALID_THRESHOLD)) + return LOG_IFERR(MDBX_BUSY) /* transaction is still active */; - dl->sorted = dpl_setlen(dl, w); - txn->tw.dirtyroom += r - 1 - w; - tASSERT(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length == - (txn->mt_parent ? txn->mt_parent->tw.dirtyroom - : txn->mt_env->me_options.dp_limit)); - tASSERT(txn, txn->tw.dirtylist->length == txn->tw.loose_count); - tASSERT(txn, txn->tw.dirtylist->pages_including_loose == txn->tw.loose_count); - return rc; + atomic_store32(&r->pid, 0, mo_Relaxed); + atomic_store32(&env->lck->rdt_refresh_flag, true, mo_AcquireRelease); + thread_rthc_set(env->me_txkey, nullptr); + return MDBX_SUCCESS; } -/* Check txn and dbi arguments to a function */ -static __always_inline bool check_dbi(const MDBX_txn *txn, MDBX_dbi dbi, - unsigned validity) { - if (likely(dbi < txn->mt_numdbs)) { - if (likely(!dbi_changed(txn, dbi))) { - if (likely(txn->mt_dbistate[dbi] & validity)) - return true; - if (likely(dbi < CORE_DBS || - (txn->mt_env->me_dbflags[dbi] & DB_VALID) == 0)) - return false; - } - } - return dbi_import((MDBX_txn *)txn, dbi); +/*------------------------------------------------------------------------------ + * Locking API */ + +int mdbx_txn_lock(MDBX_env *env, bool dont_wait) { + int rc = check_env(env, true); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); + + if (unlikely(env->flags & MDBX_RDONLY)) + return LOG_IFERR(MDBX_EACCESS); + if (dont_wait && unlikely(env->basal_txn->owner || (env->basal_txn->flags & MDBX_TXN_FINISHED) == 0)) + return LOG_IFERR(MDBX_BUSY); + + return LOG_IFERR(lck_txn_lock(env, dont_wait)); } -/* Merge child txn into parent */ -static __inline void txn_merge(MDBX_txn *const parent, MDBX_txn *const txn, - const size_t parent_retired_len) { - tASSERT(txn, (txn->mt_flags & MDBX_WRITEMAP) == 0); - MDBX_dpl *const src = dpl_sort(txn); +int mdbx_txn_unlock(MDBX_env *env) { + int rc = check_env(env, true); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); - /* Remove refunded pages from parent's dirty list */ - MDBX_dpl *const dst = dpl_sort(parent); - if (MDBX_ENABLE_REFUND) { - size_t n = dst->length; - while (n && dst->items[n].pgno >= parent->mt_next_pgno) { - const unsigned npages = dpl_npages(dst, n); - dpage_free(txn->mt_env, dst->items[n].ptr, npages); - --n; - } - parent->tw.dirtyroom += dst->sorted - n; - dst->sorted = dpl_setlen(dst, n); - tASSERT(parent, - parent->tw.dirtyroom + parent->tw.dirtylist->length == - (parent->mt_parent ? parent->mt_parent->tw.dirtyroom - : parent->mt_env->me_options.dp_limit)); - } + if (unlikely(env->flags & MDBX_RDONLY)) + return LOG_IFERR(MDBX_EACCESS); +#if MDBX_TXN_CHECKOWNER + if (unlikely(env->basal_txn->owner != osal_thread_self())) + return LOG_IFERR(MDBX_THREAD_MISMATCH); +#endif /* MDBX_TXN_CHECKOWNER */ + if (unlikely((env->basal_txn->flags & MDBX_TXN_FINISHED) == 0)) + return LOG_IFERR(MDBX_BUSY); - /* Remove reclaimed pages from parent's dirty list */ - const MDBX_PNL reclaimed_list = parent->tw.relist; - dpl_sift(parent, reclaimed_list, false); + lck_txn_unlock(env); + return MDBX_SUCCESS; +} +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 - /* Move retired pages from parent's dirty & spilled list to reclaimed */ - size_t r, w, d, s, l; - for (r = w = parent_retired_len; - ++r <= MDBX_PNL_GETSIZE(parent->tw.retired_pages);) { - const pgno_t pgno = parent->tw.retired_pages[r]; - const size_t di = dpl_exist(parent, pgno); - const size_t si = !di ? search_spilled(parent, pgno) : 0; - unsigned npages; - const char *kind; - if (di) { - MDBX_page *dp = dst->items[di].ptr; - tASSERT(parent, (dp->mp_flags & ~(P_LEAF | P_LEAF2 | P_BRANCH | - P_OVERFLOW | P_SPILLED)) == 0); - npages = dpl_npages(dst, di); - page_wash(parent, di, dp, npages); - kind = "dirty"; - l = 1; - if (unlikely(npages > l)) { - /* OVERFLOW-страница могла быть переиспользована по частям. Тогда - * в retired-списке может быть только начало последовательности, - * а остаток растащен по dirty, spilled и reclaimed спискам. Поэтому - * переносим в reclaimed с проверкой на обрыв последовательности. - * В любом случае, все осколки будут учтены и отфильтрованы, т.е. если - * страница была разбита на части, то важно удалить dirty-элемент, - * а все осколки будут учтены отдельно. */ +static inline double key2double(const int64_t key) { + union { + uint64_t u; + double f; + } casting; - /* Список retired страниц не сортирован, но для ускорения сортировки - * дополняется в соответствии с MDBX_PNL_ASCENDING */ -#if MDBX_PNL_ASCENDING - const size_t len = MDBX_PNL_GETSIZE(parent->tw.retired_pages); - while (r < len && parent->tw.retired_pages[r + 1] == pgno + l) { - ++r; - if (++l == npages) - break; - } -#else - while (w > parent_retired_len && - parent->tw.retired_pages[w - 1] == pgno + l) { - --w; - if (++l == npages) - break; - } -#endif - } - } else if (unlikely(si)) { - l = npages = 1; - spill_remove(parent, si, 1); - kind = "spilled"; - } else { - parent->tw.retired_pages[++w] = pgno; - continue; - } + casting.u = (key < 0) ? key + UINT64_C(0x8000000000000000) : UINT64_C(0xffffFFFFffffFFFF) - key; + return casting.f; +} - DEBUG("reclaim retired parent's %u -> %zu %s page %" PRIaPGNO, npages, l, - kind, pgno); - int err = pnl_insert_range(&parent->tw.relist, pgno, l); - ENSURE(txn->mt_env, err == MDBX_SUCCESS); +static inline uint64_t double2key(const double *const ptr) { + STATIC_ASSERT(sizeof(double) == sizeof(int64_t)); + const int64_t i = *(const int64_t *)ptr; + const uint64_t u = (i < 0) ? UINT64_C(0xffffFFFFffffFFFF) - i : i + UINT64_C(0x8000000000000000); + if (ASSERT_ENABLED()) { + const double f = key2double(u); + assert(memcmp(&f, ptr, sizeof(double)) == 0); } - MDBX_PNL_SETSIZE(parent->tw.retired_pages, w); + return u; +} - /* Filter-out parent spill list */ - if (parent->tw.spilled.list && - MDBX_PNL_GETSIZE(parent->tw.spilled.list) > 0) { - const MDBX_PNL sl = spill_purge(parent); - size_t len = MDBX_PNL_GETSIZE(sl); - if (len) { - /* Remove refunded pages from parent's spill list */ - if (MDBX_ENABLE_REFUND && - MDBX_PNL_MOST(sl) >= (parent->mt_next_pgno << 1)) { -#if MDBX_PNL_ASCENDING - size_t i = MDBX_PNL_GETSIZE(sl); - assert(MDBX_PNL_MOST(sl) == MDBX_PNL_LAST(sl)); - do { - if ((sl[i] & 1) == 0) - DEBUG("refund parent's spilled page %" PRIaPGNO, sl[i] >> 1); - i -= 1; - } while (i && sl[i] >= (parent->mt_next_pgno << 1)); - MDBX_PNL_SETSIZE(sl, i); -#else - assert(MDBX_PNL_MOST(sl) == MDBX_PNL_FIRST(sl)); - size_t i = 0; - do { - ++i; - if ((sl[i] & 1) == 0) - DEBUG("refund parent's spilled page %" PRIaPGNO, sl[i] >> 1); - } while (i < len && sl[i + 1] >= (parent->mt_next_pgno << 1)); - MDBX_PNL_SETSIZE(sl, len -= i); - memmove(sl + 1, sl + 1 + i, len * sizeof(sl[0])); -#endif - } - tASSERT(txn, pnl_check_allocated(sl, (size_t)parent->mt_next_pgno << 1)); +static inline float key2float(const int32_t key) { + union { + uint32_t u; + float f; + } casting; - /* Remove reclaimed pages from parent's spill list */ - s = MDBX_PNL_GETSIZE(sl), r = MDBX_PNL_GETSIZE(reclaimed_list); - /* Scanning from end to begin */ - while (s && r) { - if (sl[s] & 1) { - --s; - continue; - } - const pgno_t spilled_pgno = sl[s] >> 1; - const pgno_t reclaimed_pgno = reclaimed_list[r]; - if (reclaimed_pgno != spilled_pgno) { - const bool cmp = MDBX_PNL_ORDERED(spilled_pgno, reclaimed_pgno); - s -= !cmp; - r -= cmp; - } else { - DEBUG("remove reclaimed parent's spilled page %" PRIaPGNO, - reclaimed_pgno); - spill_remove(parent, s, 1); - --s; - --r; - } - } + casting.u = (key < 0) ? key + UINT32_C(0x80000000) : UINT32_C(0xffffFFFF) - key; + return casting.f; +} - /* Remove anything in our dirty list from parent's spill list */ - /* Scanning spill list in descend order */ - const intptr_t step = MDBX_PNL_ASCENDING ? -1 : 1; - s = MDBX_PNL_ASCENDING ? MDBX_PNL_GETSIZE(sl) : 1; - d = src->length; - while (d && (MDBX_PNL_ASCENDING ? s > 0 : s <= MDBX_PNL_GETSIZE(sl))) { - if (sl[s] & 1) { - s += step; - continue; - } - const pgno_t spilled_pgno = sl[s] >> 1; - const pgno_t dirty_pgno_form = src->items[d].pgno; - const unsigned npages = dpl_npages(src, d); - const pgno_t dirty_pgno_to = dirty_pgno_form + npages; - if (dirty_pgno_form > spilled_pgno) { - --d; - continue; - } - if (dirty_pgno_to <= spilled_pgno) { - s += step; - continue; - } +static inline uint32_t float2key(const float *const ptr) { + STATIC_ASSERT(sizeof(float) == sizeof(int32_t)); + const int32_t i = *(const int32_t *)ptr; + const uint32_t u = (i < 0) ? UINT32_C(0xffffFFFF) - i : i + UINT32_C(0x80000000); + if (ASSERT_ENABLED()) { + const float f = key2float(u); + assert(memcmp(&f, ptr, sizeof(float)) == 0); + } + return u; +} - DEBUG("remove dirtied parent's spilled %u page %" PRIaPGNO, npages, - dirty_pgno_form); - spill_remove(parent, s, 1); - s += step; - } +uint64_t mdbx_key_from_double(const double ieee754_64bit) { return double2key(&ieee754_64bit); } - /* Squash deleted pagenums if we deleted any */ - spill_purge(parent); - } - } +uint64_t mdbx_key_from_ptrdouble(const double *const ieee754_64bit) { return double2key(ieee754_64bit); } - /* Remove anything in our spill list from parent's dirty list */ - if (txn->tw.spilled.list) { - tASSERT(txn, pnl_check_allocated(txn->tw.spilled.list, - (size_t)parent->mt_next_pgno << 1)); - dpl_sift(parent, txn->tw.spilled.list, true); - tASSERT(parent, - parent->tw.dirtyroom + parent->tw.dirtylist->length == - (parent->mt_parent ? parent->mt_parent->tw.dirtyroom - : parent->mt_env->me_options.dp_limit)); +uint32_t mdbx_key_from_float(const float ieee754_32bit) { return float2key(&ieee754_32bit); } + +uint32_t mdbx_key_from_ptrfloat(const float *const ieee754_32bit) { return float2key(ieee754_32bit); } + +#define IEEE754_DOUBLE_MANTISSA_SIZE 52 +#define IEEE754_DOUBLE_EXPONENTA_BIAS 0x3FF +#define IEEE754_DOUBLE_EXPONENTA_MAX 0x7FF +#define IEEE754_DOUBLE_IMPLICIT_LEAD UINT64_C(0x0010000000000000) +#define IEEE754_DOUBLE_MANTISSA_MASK UINT64_C(0x000FFFFFFFFFFFFF) +#define IEEE754_DOUBLE_MANTISSA_AMAX UINT64_C(0x001FFFFFFFFFFFFF) + +static inline int clz64(uint64_t value) { +#if __GNUC_PREREQ(4, 1) || __has_builtin(__builtin_clzl) + if (sizeof(value) == sizeof(int)) + return __builtin_clz(value); + if (sizeof(value) == sizeof(long)) + return __builtin_clzl(value); +#if (defined(__SIZEOF_LONG_LONG__) && __SIZEOF_LONG_LONG__ == 8) || __has_builtin(__builtin_clzll) + return __builtin_clzll(value); +#endif /* have(long long) && long long == uint64_t */ +#endif /* GNU C */ + +#if defined(_MSC_VER) + unsigned long index; +#if defined(_M_AMD64) || defined(_M_ARM64) || defined(_M_X64) + _BitScanReverse64(&index, value); + return 63 - index; +#else + if (value > UINT32_MAX) { + _BitScanReverse(&index, (uint32_t)(value >> 32)); + return 31 - index; } + _BitScanReverse(&index, (uint32_t)value); + return 63 - index; +#endif +#endif /* MSVC */ - /* Find length of merging our dirty list with parent's and release - * filter-out pages */ - for (l = 0, d = dst->length, s = src->length; d > 0 && s > 0;) { - MDBX_page *sp = src->items[s].ptr; - tASSERT(parent, (sp->mp_flags & ~(P_LEAF | P_LEAF2 | P_BRANCH | P_OVERFLOW | - P_LOOSE | P_SPILLED)) == 0); - const unsigned s_npages = dpl_npages(src, s); - const pgno_t s_pgno = src->items[s].pgno; + value |= value >> 1; + value |= value >> 2; + value |= value >> 4; + value |= value >> 8; + value |= value >> 16; + value |= value >> 32; + static const uint8_t debruijn_clz64[64] = {63, 16, 62, 7, 15, 36, 61, 3, 6, 14, 22, 26, 35, 47, 60, 2, + 9, 5, 28, 11, 13, 21, 42, 19, 25, 31, 34, 40, 46, 52, 59, 1, + 17, 8, 37, 4, 23, 27, 48, 10, 29, 12, 43, 20, 32, 41, 53, 18, + 38, 24, 49, 30, 44, 33, 54, 39, 50, 45, 55, 51, 56, 57, 58, 0}; + return debruijn_clz64[value * UINT64_C(0x03F79D71B4CB0A89) >> 58]; +} - MDBX_page *dp = dst->items[d].ptr; - tASSERT(parent, (dp->mp_flags & ~(P_LEAF | P_LEAF2 | P_BRANCH | P_OVERFLOW | - P_SPILLED)) == 0); - const unsigned d_npages = dpl_npages(dst, d); - const pgno_t d_pgno = dst->items[d].pgno; +static inline uint64_t round_mantissa(const uint64_t u64, int shift) { + assert(shift < 0 && u64 > 0); + shift = -shift; + const unsigned half = 1 << (shift - 1); + const unsigned lsb = 1 & (unsigned)(u64 >> shift); + const unsigned tie2even = 1 ^ lsb; + return (u64 + half - tie2even) >> shift; +} - if (d_pgno >= s_pgno + s_npages) { - --d; - ++l; - } else if (d_pgno + d_npages <= s_pgno) { - if (sp->mp_flags != P_LOOSE) { - sp->mp_txnid = parent->mt_front; - sp->mp_flags &= ~P_SPILLED; - } - --s; - ++l; - } else { - dst->items[d--].ptr = nullptr; - dpage_free(txn->mt_env, dp, d_npages); +uint64_t mdbx_key_from_jsonInteger(const int64_t json_integer) { + const uint64_t bias = UINT64_C(0x8000000000000000); + if (json_integer > 0) { + const uint64_t u64 = json_integer; + int shift = clz64(u64) - (64 - IEEE754_DOUBLE_MANTISSA_SIZE - 1); + uint64_t mantissa = u64 << shift; + if (unlikely(shift < 0)) { + mantissa = round_mantissa(u64, shift); + if (mantissa > IEEE754_DOUBLE_MANTISSA_AMAX) + mantissa = round_mantissa(u64, --shift); } + + assert(mantissa >= IEEE754_DOUBLE_IMPLICIT_LEAD && mantissa <= IEEE754_DOUBLE_MANTISSA_AMAX); + const uint64_t exponent = (uint64_t)IEEE754_DOUBLE_EXPONENTA_BIAS + IEEE754_DOUBLE_MANTISSA_SIZE - shift; + assert(exponent > 0 && exponent <= IEEE754_DOUBLE_EXPONENTA_MAX); + const uint64_t key = bias + (exponent << IEEE754_DOUBLE_MANTISSA_SIZE) + (mantissa - IEEE754_DOUBLE_IMPLICIT_LEAD); +#if !defined(_MSC_VER) || defined(_DEBUG) /* Workaround for MSVC error LNK2019: unresolved external \ + symbol __except1 referenced in function __ftol3_except */ + assert(key == mdbx_key_from_double((double)json_integer)); +#endif /* Workaround for MSVC */ + return key; } - assert(dst->sorted == dst->length); - tASSERT(parent, dst->detent >= l + d + s); - dst->sorted = l + d + s; /* the merged length */ - while (s > 0) { - MDBX_page *sp = src->items[s].ptr; - tASSERT(parent, (sp->mp_flags & ~(P_LEAF | P_LEAF2 | P_BRANCH | P_OVERFLOW | - P_LOOSE | P_SPILLED)) == 0); - if (sp->mp_flags != P_LOOSE) { - sp->mp_txnid = parent->mt_front; - sp->mp_flags &= ~P_SPILLED; + if (json_integer < 0) { + const uint64_t u64 = -json_integer; + int shift = clz64(u64) - (64 - IEEE754_DOUBLE_MANTISSA_SIZE - 1); + uint64_t mantissa = u64 << shift; + if (unlikely(shift < 0)) { + mantissa = round_mantissa(u64, shift); + if (mantissa > IEEE754_DOUBLE_MANTISSA_AMAX) + mantissa = round_mantissa(u64, --shift); } - --s; + + assert(mantissa >= IEEE754_DOUBLE_IMPLICIT_LEAD && mantissa <= IEEE754_DOUBLE_MANTISSA_AMAX); + const uint64_t exponent = (uint64_t)IEEE754_DOUBLE_EXPONENTA_BIAS + IEEE754_DOUBLE_MANTISSA_SIZE - shift; + assert(exponent > 0 && exponent <= IEEE754_DOUBLE_EXPONENTA_MAX); + const uint64_t key = + bias - 1 - (exponent << IEEE754_DOUBLE_MANTISSA_SIZE) - (mantissa - IEEE754_DOUBLE_IMPLICIT_LEAD); +#if !defined(_MSC_VER) || defined(_DEBUG) /* Workaround for MSVC error LNK2019: unresolved external \ + symbol __except1 referenced in function __ftol3_except */ + assert(key == mdbx_key_from_double((double)json_integer)); +#endif /* Workaround for MSVC */ + return key; } - /* Merge our dirty list into parent's, i.e. merge(dst, src) -> dst */ - if (dst->sorted >= dst->length) { - /* from end to begin with dst extending */ - for (l = dst->sorted, s = src->length, d = dst->length; s > 0 && d > 0;) { - if (unlikely(l <= d)) { - /* squash to get a gap of free space for merge */ - for (r = w = 1; r <= d; ++r) - if (dst->items[r].ptr) { - if (w != r) { - dst->items[w] = dst->items[r]; - dst->items[r].ptr = nullptr; - } - ++w; - } - VERBOSE("squash to begin for extending-merge %zu -> %zu", d, w - 1); - d = w - 1; - continue; - } - assert(l > d); - if (dst->items[d].ptr) { - dst->items[l--] = (dst->items[d].pgno > src->items[s].pgno) - ? dst->items[d--] - : src->items[s--]; - } else - --d; - } - if (s > 0) { - assert(l == s); - while (d > 0) { - assert(dst->items[d].ptr == nullptr); - --d; - } - do { - assert(l > 0); - dst->items[l--] = src->items[s--]; - } while (s > 0); - } else { - assert(l == d); - while (l > 0) { - assert(dst->items[l].ptr != nullptr); - --l; - } - } - } else { - /* from begin to end with shrinking (a lot of new large/overflow pages) */ - for (l = s = d = 1; s <= src->length && d <= dst->length;) { - if (unlikely(l >= d)) { - /* squash to get a gap of free space for merge */ - for (r = w = dst->length; r >= d; --r) - if (dst->items[r].ptr) { - if (w != r) { - dst->items[w] = dst->items[r]; - dst->items[r].ptr = nullptr; - } - --w; - } - VERBOSE("squash to end for shrinking-merge %zu -> %zu", d, w + 1); - d = w + 1; - continue; - } - assert(l < d); - if (dst->items[d].ptr) { - dst->items[l++] = (dst->items[d].pgno < src->items[s].pgno) - ? dst->items[d++] - : src->items[s++]; - } else - ++d; - } - if (s <= src->length) { - assert(dst->sorted - l == src->length - s); - while (d <= dst->length) { - assert(dst->items[d].ptr == nullptr); - --d; - } - do { - assert(l <= dst->sorted); - dst->items[l++] = src->items[s++]; - } while (s <= src->length); - } else { - assert(dst->sorted - l == dst->length - d); - while (l <= dst->sorted) { - assert(l <= d && d <= dst->length && dst->items[d].ptr); - dst->items[l++] = dst->items[d++]; - } - } - } - parent->tw.dirtyroom -= dst->sorted - dst->length; - assert(parent->tw.dirtyroom <= parent->mt_env->me_options.dp_limit); - dpl_setlen(dst, dst->sorted); - parent->tw.dirtylru = txn->tw.dirtylru; - - /* В текущем понимании выгоднее пересчитать кол-во страниц, - * чем подмешивать лишние ветвления и вычисления в циклы выше. */ - dst->pages_including_loose = 0; - for (r = 1; r <= dst->length; ++r) - dst->pages_including_loose += dpl_npages(dst, r); + return bias; +} - tASSERT(parent, dirtylist_check(parent)); - dpl_free(txn); +int64_t mdbx_jsonInteger_from_key(const MDBX_val v) { + assert(v.iov_len == 8); + const uint64_t key = unaligned_peek_u64(2, v.iov_base); + const uint64_t bias = UINT64_C(0x8000000000000000); + const uint64_t covalent = (key > bias) ? key - bias : bias - key - 1; + const int shift = IEEE754_DOUBLE_EXPONENTA_BIAS + 63 - + (IEEE754_DOUBLE_EXPONENTA_MAX & (int)(covalent >> IEEE754_DOUBLE_MANTISSA_SIZE)); + if (unlikely(shift < 1)) + return (key < bias) ? INT64_MIN : INT64_MAX; + if (unlikely(shift > 63)) + return 0; - if (txn->tw.spilled.list) { - if (parent->tw.spilled.list) { - /* Must not fail since space was preserved above. */ - pnl_merge(parent->tw.spilled.list, txn->tw.spilled.list); - pnl_free(txn->tw.spilled.list); - } else { - parent->tw.spilled.list = txn->tw.spilled.list; - parent->tw.spilled.least_removed = txn->tw.spilled.least_removed; - } - tASSERT(parent, dirtylist_check(parent)); - } + const uint64_t unscaled = ((covalent & IEEE754_DOUBLE_MANTISSA_MASK) << (63 - IEEE754_DOUBLE_MANTISSA_SIZE)) + bias; + const int64_t absolute = unscaled >> shift; + const int64_t value = (key < bias) ? -absolute : absolute; + assert(key == mdbx_key_from_jsonInteger(value) || + (mdbx_key_from_jsonInteger(value - 1) < key && key < mdbx_key_from_jsonInteger(value + 1))); + return value; +} - parent->mt_flags &= ~MDBX_TXN_HAS_CHILD; - if (parent->tw.spilled.list) { - assert(pnl_check_allocated(parent->tw.spilled.list, - (size_t)parent->mt_next_pgno << 1)); - if (MDBX_PNL_GETSIZE(parent->tw.spilled.list)) - parent->mt_flags |= MDBX_TXN_SPILLS; - } +double mdbx_double_from_key(const MDBX_val v) { + assert(v.iov_len == 8); + return key2double(unaligned_peek_u64(2, v.iov_base)); } -static void take_gcprof(MDBX_txn *txn, MDBX_commit_latency *latency) { - MDBX_env *const env = txn->mt_env; - if (MDBX_ENABLE_PROFGC) { - pgop_stat_t *const ptr = &env->me_lck->mti_pgop_stat; - latency->gc_prof.work_counter = ptr->gc_prof.work.spe_counter; - latency->gc_prof.work_rtime_monotonic = - osal_monotime_to_16dot16(ptr->gc_prof.work.rtime_monotonic); - latency->gc_prof.work_xtime_cpu = - osal_monotime_to_16dot16(ptr->gc_prof.work.xtime_cpu); - latency->gc_prof.work_rsteps = ptr->gc_prof.work.rsteps; - latency->gc_prof.work_xpages = ptr->gc_prof.work.xpages; - latency->gc_prof.work_majflt = ptr->gc_prof.work.majflt; +float mdbx_float_from_key(const MDBX_val v) { + assert(v.iov_len == 4); + return key2float(unaligned_peek_u32(2, v.iov_base)); +} - latency->gc_prof.self_counter = ptr->gc_prof.self.spe_counter; - latency->gc_prof.self_rtime_monotonic = - osal_monotime_to_16dot16(ptr->gc_prof.self.rtime_monotonic); - latency->gc_prof.self_xtime_cpu = - osal_monotime_to_16dot16(ptr->gc_prof.self.xtime_cpu); - latency->gc_prof.self_rsteps = ptr->gc_prof.self.rsteps; - latency->gc_prof.self_xpages = ptr->gc_prof.self.xpages; - latency->gc_prof.self_majflt = ptr->gc_prof.self.majflt; +int32_t mdbx_int32_from_key(const MDBX_val v) { + assert(v.iov_len == 4); + return (int32_t)(unaligned_peek_u32(2, v.iov_base) - UINT32_C(0x80000000)); +} - latency->gc_prof.wloops = ptr->gc_prof.wloops; - latency->gc_prof.coalescences = ptr->gc_prof.coalescences; - latency->gc_prof.wipes = ptr->gc_prof.wipes; - latency->gc_prof.flushes = ptr->gc_prof.flushes; - latency->gc_prof.kicks = ptr->gc_prof.kicks; - if (txn == env->me_txn0) - memset(&ptr->gc_prof, 0, sizeof(ptr->gc_prof)); - } else - memset(&latency->gc_prof, 0, sizeof(latency->gc_prof)); +int64_t mdbx_int64_from_key(const MDBX_val v) { + assert(v.iov_len == 8); + return (int64_t)(unaligned_peek_u64(2, v.iov_base) - UINT64_C(0x8000000000000000)); } +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 -int mdbx_txn_commit_ex(MDBX_txn *txn, MDBX_commit_latency *latency) { - STATIC_ASSERT(MDBX_TXN_FINISHED == - MDBX_TXN_BLOCKED - MDBX_TXN_HAS_CHILD - MDBX_TXN_ERROR); - const uint64_t ts_0 = latency ? osal_monotime() : 0; - uint64_t ts_1 = 0, ts_2 = 0, ts_3 = 0, ts_4 = 0, ts_5 = 0, gc_cputime = 0; +__cold int mdbx_is_readahead_reasonable(size_t volume, intptr_t redundancy) { + if (volume <= 1024 * 1024 * 4ul) + return MDBX_RESULT_TRUE; - int rc = check_txn(txn, MDBX_TXN_FINISHED); - if (unlikely(rc != MDBX_SUCCESS)) { - if (latency) - memset(latency, 0, sizeof(*latency)); - return rc; - } + intptr_t pagesize, total_ram_pages; + int err = mdbx_get_sysraminfo(&pagesize, &total_ram_pages, nullptr); + if (unlikely(err != MDBX_SUCCESS)) + return LOG_IFERR(err); - MDBX_env *const env = txn->mt_env; -#if MDBX_ENV_CHECKPID - if (unlikely(env->me_pid != osal_getpid())) { - env->me_flags |= MDBX_FATAL_ERROR; - if (latency) - memset(latency, 0, sizeof(*latency)); - return MDBX_PANIC; - } -#endif /* MDBX_ENV_CHECKPID */ + const int log2page = log2n_powerof2(pagesize); + const intptr_t volume_pages = (volume + pagesize - 1) >> log2page; + const intptr_t redundancy_pages = (redundancy < 0) ? -(intptr_t)((-redundancy + pagesize - 1) >> log2page) + : (intptr_t)(redundancy + pagesize - 1) >> log2page; + if (volume_pages >= total_ram_pages || volume_pages + redundancy_pages >= total_ram_pages) + return MDBX_RESULT_FALSE; - if (unlikely(txn->mt_flags & MDBX_TXN_ERROR)) { - rc = MDBX_RESULT_TRUE; - goto fail; - } + intptr_t avail_ram_pages; + err = mdbx_get_sysraminfo(nullptr, nullptr, &avail_ram_pages); + if (unlikely(err != MDBX_SUCCESS)) + return LOG_IFERR(err); - /* txn_end() mode for a commit which writes nothing */ - unsigned end_mode = - MDBX_END_PURE_COMMIT | MDBX_END_UPDATE | MDBX_END_SLOT | MDBX_END_FREE; - if (unlikely(txn->mt_flags & MDBX_TXN_RDONLY)) - goto done; + return (volume_pages + redundancy_pages >= avail_ram_pages) ? MDBX_RESULT_FALSE : MDBX_RESULT_TRUE; +} - if (txn->mt_child) { - rc = mdbx_txn_commit_ex(txn->mt_child, NULL); - tASSERT(txn, txn->mt_child == NULL); - if (unlikely(rc != MDBX_SUCCESS)) - goto fail; - } +int mdbx_dbi_sequence(MDBX_txn *txn, MDBX_dbi dbi, uint64_t *result, uint64_t increment) { + int rc = check_txn(txn, MDBX_TXN_BLOCKED); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); - if (unlikely(txn != env->me_txn)) { - DEBUG("%s", "attempt to commit unknown transaction"); - rc = MDBX_EINVAL; - goto fail; + rc = dbi_check(txn, dbi); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); + + if (unlikely(txn->dbi_state[dbi] & DBI_STALE)) { + rc = tbl_fetch(txn, dbi); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); } - if (txn->mt_parent) { - tASSERT(txn, audit_ex(txn, 0, false) == 0); - eASSERT(env, txn != env->me_txn0); - MDBX_txn *const parent = txn->mt_parent; - eASSERT(env, parent->mt_signature == MDBX_MT_SIGNATURE); - eASSERT(env, parent->mt_child == txn && - (parent->mt_flags & MDBX_TXN_HAS_CHILD) != 0); - eASSERT(env, dirtylist_check(txn)); - - if (txn->tw.dirtylist->length == 0 && !(txn->mt_flags & MDBX_TXN_DIRTY) && - parent->mt_numdbs == txn->mt_numdbs) { - for (int i = txn->mt_numdbs; --i >= 0;) { - tASSERT(txn, (txn->mt_dbistate[i] & DBI_DIRTY) == 0); - if ((txn->mt_dbistate[i] & DBI_STALE) && - !(parent->mt_dbistate[i] & DBI_STALE)) - tASSERT(txn, memcmp(&parent->mt_dbs[i], &txn->mt_dbs[i], - sizeof(MDBX_db)) == 0); - } - - tASSERT(txn, memcmp(&parent->mt_geo, &txn->mt_geo, - sizeof(parent->mt_geo)) == 0); - tASSERT(txn, memcmp(&parent->mt_canary, &txn->mt_canary, - sizeof(parent->mt_canary)) == 0); - tASSERT(txn, !txn->tw.spilled.list || - MDBX_PNL_GETSIZE(txn->tw.spilled.list) == 0); - tASSERT(txn, txn->tw.loose_count == 0); + tree_t *dbs = &txn->dbs[dbi]; + if (likely(result)) + *result = dbs->sequence; - /* fast completion of pure nested transaction */ - end_mode = MDBX_END_PURE_COMMIT | MDBX_END_SLOT | MDBX_END_FREE; - goto done; - } + if (likely(increment > 0)) { + if (unlikely(dbi == FREE_DBI || (txn->flags & MDBX_TXN_RDONLY) != 0)) + return MDBX_EACCESS; - /* Preserve space for spill list to avoid parent's state corruption - * if allocation fails. */ - const size_t parent_retired_len = (uintptr_t)parent->tw.retired_pages; - tASSERT(txn, parent_retired_len <= MDBX_PNL_GETSIZE(txn->tw.retired_pages)); - const size_t retired_delta = - MDBX_PNL_GETSIZE(txn->tw.retired_pages) - parent_retired_len; - if (retired_delta) { - rc = pnl_need(&txn->tw.relist, retired_delta); - if (unlikely(rc != MDBX_SUCCESS)) - goto fail; - } + uint64_t new = dbs->sequence + increment; + if (unlikely(new < increment)) + return MDBX_RESULT_TRUE; - if (txn->tw.spilled.list) { - if (parent->tw.spilled.list) { - rc = pnl_need(&parent->tw.spilled.list, - MDBX_PNL_GETSIZE(txn->tw.spilled.list)); + tASSERT(txn, new > dbs->sequence); + if ((txn->dbi_state[dbi] & DBI_DIRTY) == 0) { + txn->flags |= MDBX_TXN_DIRTY; + txn->dbi_state[dbi] |= DBI_DIRTY; + if (unlikely(dbi == MAIN_DBI) && txn->dbs[MAIN_DBI].root != P_INVALID) { + /* LY: Временная подпорка для coherency_check(), которую в перспективе + * следует заменить вместе с переделкой установки mod_txnid. + * + * Суть проблемы: + * - coherency_check() в качестве одного из критериев "когерентности" + * проверяет условие meta.maindb.mod_txnid == maindb.root->txnid; + * - при обновлении maindb.sequence высталяется DBI_DIRTY, что приведет + * к обновлению meta.maindb.mod_txnid = current_txnid; + * - однако, если в само дерево maindb обновление не вносились и оно + * не пустое, то корневая страницы останеться с прежним txnid и из-за + * этого ложно сработает coherency_check(). + * + * Временное (текущее) решение: Принудительно обновляем корневую + * страницу в описанной выше ситуации. Это устраняет проблему, но и + * не создает рисков регресса. + * + * FIXME: Итоговое решение, которое предстоит реализовать: + * - изменить семантику установки/обновления mod_txnid, привязав его + * строго к изменению b-tree, но не атрибутов; + * - обновлять mod_txnid при фиксации вложенных транзакций; + * - для dbi-хендлов пользовательских table (видимо) можно оставить + * DBI_DIRTY в качестве признака необходимости обновления записи + * table в MainDB, при этом взводить DBI_DIRTY вместе с обновлением + * mod_txnid, в том числе при обновлении sequence. + * - для MAIN_DBI при обновлении sequence не следует взводить DBI_DIRTY + * и/или обновлять mod_txnid, а только взводить MDBX_TXN_DIRTY. + * - альтернативно, можно перераспределить флажки-признаки dbi_state, + * чтобы различать состояние dirty-tree и dirty-attributes. */ + cursor_couple_t cx; + rc = cursor_init(&cx.outer, txn, MAIN_DBI); if (unlikely(rc != MDBX_SUCCESS)) - goto fail; + return LOG_IFERR(rc); + rc = tree_search(&cx.outer, nullptr, Z_MODIFY | Z_ROOTONLY); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); } - spill_purge(txn); - } - - if (unlikely(txn->tw.dirtylist->length + parent->tw.dirtylist->length > - parent->tw.dirtylist->detent && - !dpl_reserve(parent, txn->tw.dirtylist->length + - parent->tw.dirtylist->length))) { - rc = MDBX_ENOMEM; - goto fail; } + dbs->sequence = new; + } - //------------------------------------------------------------------------- + return MDBX_SUCCESS; +} - parent->tw.lifo_reclaimed = txn->tw.lifo_reclaimed; - txn->tw.lifo_reclaimed = NULL; +int mdbx_cmp(const MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *a, const MDBX_val *b) { + eASSERT(nullptr, txn->signature == txn_signature); + tASSERT(txn, (dbi_state(txn, dbi) & DBI_VALID) && !dbi_changed(txn, dbi)); + tASSERT(txn, dbi < txn->env->n_dbi && (txn->env->dbs_flags[dbi] & DB_VALID) != 0); + return txn->env->kvs[dbi].clc.k.cmp(a, b); +} - parent->tw.retired_pages = txn->tw.retired_pages; - txn->tw.retired_pages = NULL; +int mdbx_dcmp(const MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *a, const MDBX_val *b) { + eASSERT(nullptr, txn->signature == txn_signature); + tASSERT(txn, (dbi_state(txn, dbi) & DBI_VALID) && !dbi_changed(txn, dbi)); + tASSERT(txn, dbi < txn->env->n_dbi && (txn->env->dbs_flags[dbi] & DB_VALID)); + return txn->env->kvs[dbi].clc.v.cmp(a, b); +} - pnl_free(parent->tw.relist); - parent->tw.relist = txn->tw.relist; - txn->tw.relist = NULL; - parent->tw.last_reclaimed = txn->tw.last_reclaimed; +__cold MDBX_cmp_func *mdbx_get_keycmp(MDBX_db_flags_t flags) { return builtin_keycmp(flags); } - parent->mt_geo = txn->mt_geo; - parent->mt_canary = txn->mt_canary; - parent->mt_flags |= txn->mt_flags & MDBX_TXN_DIRTY; +__cold MDBX_cmp_func *mdbx_get_datacmp(MDBX_db_flags_t flags) { return builtin_datacmp(flags); } - /* Move loose pages to parent */ -#if MDBX_ENABLE_REFUND - parent->tw.loose_refund_wl = txn->tw.loose_refund_wl; -#endif /* MDBX_ENABLE_REFUND */ - parent->tw.loose_count = txn->tw.loose_count; - parent->tw.loose_pages = txn->tw.loose_pages; +/*----------------------------------------------------------------------------*/ - /* Merge our cursors into parent's and close them */ - cursors_eot(txn, true); - end_mode |= MDBX_END_EOTDONE; +__cold const char *mdbx_liberr2str(int errnum) { + /* Table of descriptions for MDBX errors */ + static const char *const tbl[] = { + "MDBX_KEYEXIST: Key/data pair already exists", + "MDBX_NOTFOUND: No matching key/data pair found", + "MDBX_PAGE_NOTFOUND: Requested page not found", + "MDBX_CORRUPTED: Database is corrupted", + "MDBX_PANIC: Environment had fatal error", + "MDBX_VERSION_MISMATCH: DB version mismatch libmdbx", + "MDBX_INVALID: File is not an MDBX file", + "MDBX_MAP_FULL: Environment mapsize limit reached", + "MDBX_DBS_FULL: Too many DBI-handles (maxdbs reached)", + "MDBX_READERS_FULL: Too many readers (maxreaders reached)", + nullptr /* MDBX_TLS_FULL (-30789): unused in MDBX */, + "MDBX_TXN_FULL: Transaction has too many dirty pages," + " i.e transaction is too big", + "MDBX_CURSOR_FULL: Cursor stack limit reachedn - this usually indicates" + " corruption, i.e branch-pages loop", + "MDBX_PAGE_FULL: Internal error - Page has no more space", + "MDBX_UNABLE_EXTEND_MAPSIZE: Database engine was unable to extend" + " mapping, e.g. since address space is unavailable or busy," + " or Operation system not supported such operations", + "MDBX_INCOMPATIBLE: Environment or database is not compatible" + " with the requested operation or the specified flags", + "MDBX_BAD_RSLOT: Invalid reuse of reader locktable slot," + " e.g. read-transaction already run for current thread", + "MDBX_BAD_TXN: Transaction is not valid for requested operation," + " e.g. had errored and be must aborted, has a child, or is invalid", + "MDBX_BAD_VALSIZE: Invalid size or alignment of key or data" + " for target database, either invalid table name", + "MDBX_BAD_DBI: The specified DBI-handle is invalid" + " or changed by another thread/transaction", + "MDBX_PROBLEM: Unexpected internal error, transaction should be aborted", + "MDBX_BUSY: Another write transaction is running," + " or environment is already used while opening with MDBX_EXCLUSIVE flag", + }; - /* Update parent's DBs array */ - memcpy(parent->mt_dbs, txn->mt_dbs, txn->mt_numdbs * sizeof(MDBX_db)); - parent->mt_numdbs = txn->mt_numdbs; - for (size_t i = 0; i < txn->mt_numdbs; i++) { - /* preserve parent's status */ - const uint8_t state = - txn->mt_dbistate[i] | - (parent->mt_dbistate[i] & (DBI_CREAT | DBI_FRESH | DBI_DIRTY)); - DEBUG("dbi %zu dbi-state %s 0x%02x -> 0x%02x", i, - (parent->mt_dbistate[i] != state) ? "update" : "still", - parent->mt_dbistate[i], state); - parent->mt_dbistate[i] = state; - } + if (errnum >= MDBX_KEYEXIST && errnum <= MDBX_BUSY) { + int i = errnum - MDBX_KEYEXIST; + return tbl[i]; + } - if (latency) { - ts_1 = osal_monotime(); - ts_2 = /* no gc-update */ ts_1; - ts_3 = /* no audit */ ts_2; - ts_4 = /* no write */ ts_3; - ts_5 = /* no sync */ ts_4; - } - txn_merge(parent, txn, parent_retired_len); - env->me_txn = parent; - parent->mt_child = NULL; - tASSERT(parent, dirtylist_check(parent)); + switch (errnum) { + case MDBX_SUCCESS: + return "MDBX_SUCCESS: Successful"; + case MDBX_EMULTIVAL: + return "MDBX_EMULTIVAL: The specified key has" + " more than one associated value"; + case MDBX_EBADSIGN: + return "MDBX_EBADSIGN: Wrong signature of a runtime object(s)," + " e.g. memory corruption or double-free"; + case MDBX_WANNA_RECOVERY: + return "MDBX_WANNA_RECOVERY: Database should be recovered," + " but this could NOT be done automatically for now" + " since it opened in read-only mode"; + case MDBX_EKEYMISMATCH: + return "MDBX_EKEYMISMATCH: The given key value is mismatched to the" + " current cursor position"; + case MDBX_TOO_LARGE: + return "MDBX_TOO_LARGE: Database is too large for current system," + " e.g. could NOT be mapped into RAM"; + case MDBX_THREAD_MISMATCH: + return "MDBX_THREAD_MISMATCH: A thread has attempted to use a not" + " owned object, e.g. a transaction that started by another thread"; + case MDBX_TXN_OVERLAPPING: + return "MDBX_TXN_OVERLAPPING: Overlapping read and write transactions for" + " the current thread"; + case MDBX_DUPLICATED_CLK: + return "MDBX_DUPLICATED_CLK: Alternative/Duplicate LCK-file is exists," + " please keep one and remove unused other"; + case MDBX_DANGLING_DBI: + return "MDBX_DANGLING_DBI: Some cursors and/or other resources should be" + " closed before table or corresponding DBI-handle could be (re)used"; + case MDBX_OUSTED: + return "MDBX_OUSTED: The parked read transaction was outed for the sake" + " of recycling old MVCC snapshots"; + case MDBX_MVCC_RETARDED: + return "MDBX_MVCC_RETARDED: MVCC snapshot used by parked transaction was bygone"; + default: + return nullptr; + } +} -#if MDBX_ENABLE_REFUND - txn_refund(parent); - if (ASSERT_ENABLED()) { - /* Check parent's loose pages not suitable for refund */ - for (MDBX_page *lp = parent->tw.loose_pages; lp; lp = mp_next(lp)) { - tASSERT(parent, lp->mp_pgno < parent->tw.loose_refund_wl && - lp->mp_pgno + 1 < parent->mt_next_pgno); - MDBX_ASAN_UNPOISON_MEMORY_REGION(&mp_next(lp), sizeof(MDBX_page *)); - VALGRIND_MAKE_MEM_DEFINED(&mp_next(lp), sizeof(MDBX_page *)); +__cold const char *mdbx_strerror_r(int errnum, char *buf, size_t buflen) { + const char *msg = mdbx_liberr2str(errnum); + if (!msg && buflen > 0 && buflen < INT_MAX) { +#if defined(_WIN32) || defined(_WIN64) + DWORD size = FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, nullptr, errnum, + MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), buf, (DWORD)buflen, nullptr); + while (size && buf[size - 1] <= ' ') + --size; + buf[size] = 0; + return size ? buf : "FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM) failed"; +#elif defined(_GNU_SOURCE) && defined(__GLIBC__) + /* GNU-specific */ + if (errnum > 0) + msg = strerror_r(errnum, buf, buflen); +#elif (_POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600) + /* XSI-compliant */ + if (errnum > 0 && strerror_r(errnum, buf, buflen) == 0) + msg = buf; +#else + if (errnum > 0) { + msg = strerror(errnum); + if (msg) { + strncpy(buf, msg, buflen); + msg = buf; } - /* Check parent's reclaimed pages not suitable for refund */ - if (MDBX_PNL_GETSIZE(parent->tw.relist)) - tASSERT(parent, - MDBX_PNL_MOST(parent->tw.relist) + 1 < parent->mt_next_pgno); } -#endif /* MDBX_ENABLE_REFUND */ - - txn->mt_signature = 0; - osal_free(txn); - tASSERT(parent, audit_ex(parent, 0, false) == 0); - rc = MDBX_SUCCESS; - goto provide_latency; +#endif + if (!msg) { + (void)snprintf(buf, buflen, "error %d", errnum); + msg = buf; + } + buf[buflen - 1] = '\0'; } + return msg; +} - if (!txn->tw.dirtylist) { - tASSERT(txn, (txn->mt_flags & MDBX_WRITEMAP) != 0 && !MDBX_AVOID_MSYNC); - } else { - tASSERT(txn, (txn->mt_flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); - tASSERT(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length == - (txn->mt_parent ? txn->mt_parent->tw.dirtyroom - : txn->mt_env->me_options.dp_limit)); +__cold const char *mdbx_strerror(int errnum) { +#if defined(_WIN32) || defined(_WIN64) + static char buf[1024]; + return mdbx_strerror_r(errnum, buf, sizeof(buf)); +#else + const char *msg = mdbx_liberr2str(errnum); + if (!msg) { + if (errnum > 0) + msg = strerror(errnum); + if (!msg) { + static char buf[32]; + (void)snprintf(buf, sizeof(buf) - 1, "error %d", errnum); + msg = buf; + } } - cursors_eot(txn, false); - end_mode |= MDBX_END_EOTDONE; + return msg; +#endif +} - if ((!txn->tw.dirtylist || txn->tw.dirtylist->length == 0) && - (txn->mt_flags & (MDBX_TXN_DIRTY | MDBX_TXN_SPILLS)) == 0) { - for (intptr_t i = txn->mt_numdbs; --i >= 0;) - tASSERT(txn, (txn->mt_dbistate[i] & DBI_DIRTY) == 0); -#if defined(MDBX_NOSUCCESS_EMPTY_COMMIT) && MDBX_NOSUCCESS_EMPTY_COMMIT - rc = txn_end(txn, end_mode); - if (unlikely(rc != MDBX_SUCCESS)) - goto fail; - rc = MDBX_RESULT_TRUE; - goto provide_latency; -#else - goto done; -#endif /* MDBX_NOSUCCESS_EMPTY_COMMIT */ +#if defined(_WIN32) || defined(_WIN64) /* Bit of madness for Windows */ +const char *mdbx_strerror_r_ANSI2OEM(int errnum, char *buf, size_t buflen) { + const char *msg = mdbx_liberr2str(errnum); + if (!msg && buflen > 0 && buflen < INT_MAX) { + DWORD size = FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, nullptr, errnum, + MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), buf, (DWORD)buflen, nullptr); + while (size && buf[size - 1] <= ' ') + --size; + buf[size] = 0; + if (!size) + msg = "FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM) failed"; + else if (!CharToOemBuffA(buf, buf, size)) + msg = "CharToOemBuffA() failed"; + else + msg = buf; } + return msg; +} - DEBUG("committing txn %" PRIaTXN " %p on mdbenv %p, root page %" PRIaPGNO - "/%" PRIaPGNO, - txn->mt_txnid, (void *)txn, (void *)env, txn->mt_dbs[MAIN_DBI].md_root, - txn->mt_dbs[FREE_DBI].md_root); +const char *mdbx_strerror_ANSI2OEM(int errnum) { + static char buf[1024]; + return mdbx_strerror_r_ANSI2OEM(errnum, buf, sizeof(buf)); +} +#endif /* Bit of madness for Windows */ +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 - /* Update DB root pointers */ - if (txn->mt_numdbs > CORE_DBS) { - MDBX_cursor_couple couple; - MDBX_val data; - data.iov_len = sizeof(MDBX_db); +static pgno_t env_max_pgno(const MDBX_env *env) { + return env->ps ? bytes2pgno(env, env->geo_in_bytes.upper ? env->geo_in_bytes.upper : MAX_MAPSIZE) : PAGELIST_LIMIT; +} - rc = cursor_init(&couple.outer, txn, MAIN_DBI); - if (unlikely(rc != MDBX_SUCCESS)) - goto fail; - for (MDBX_dbi i = CORE_DBS; i < txn->mt_numdbs; i++) { - if (txn->mt_dbistate[i] & DBI_DIRTY) { - MDBX_db *db = &txn->mt_dbs[i]; - DEBUG("update main's entry for sub-db %u, mod_txnid %" PRIaTXN - " -> %" PRIaTXN, - i, db->md_mod_txnid, txn->mt_txnid); - /* Может быть mod_txnid > front после коммита вложенных тразакций */ - db->md_mod_txnid = txn->mt_txnid; - data.iov_base = db; - WITH_CURSOR_TRACKING( - couple.outer, - rc = cursor_put_nochecklen(&couple.outer, &txn->mt_dbxs[i].md_name, - &data, F_SUBDATA)); - if (unlikely(rc != MDBX_SUCCESS)) - goto fail; - } +__cold pgno_t default_dp_limit(const MDBX_env *env) { + /* auto-setup dp_limit by "The42" ;-) */ + intptr_t total_ram_pages, avail_ram_pages; + int err = mdbx_get_sysraminfo(nullptr, &total_ram_pages, &avail_ram_pages); + pgno_t dp_limit = 1024; + if (unlikely(err != MDBX_SUCCESS)) + ERROR("mdbx_get_sysraminfo(), rc %d", err); + else { + size_t estimate = (size_t)(total_ram_pages + avail_ram_pages) / 42; + if (env->ps) { + if (env->ps > globals.sys_pagesize) + estimate /= env->ps / globals.sys_pagesize; + else if (env->ps < globals.sys_pagesize) + estimate *= globals.sys_pagesize / env->ps; } + dp_limit = (pgno_t)estimate; } - ts_1 = latency ? osal_monotime() : 0; + dp_limit = (dp_limit < PAGELIST_LIMIT) ? dp_limit : PAGELIST_LIMIT; + const pgno_t max_pgno = env_max_pgno(env); + if (dp_limit > max_pgno - NUM_METAS) + dp_limit = max_pgno - NUM_METAS; + dp_limit = (dp_limit > CURSOR_STACK_SIZE * 4) ? dp_limit : CURSOR_STACK_SIZE * 4; + return dp_limit; +} - gcu_context_t gcu_ctx; - gc_cputime = latency ? osal_cputime(nullptr) : 0; - rc = gcu_context_init(txn, &gcu_ctx); - if (unlikely(rc != MDBX_SUCCESS)) - goto fail; - rc = update_gc(txn, &gcu_ctx); - gc_cputime = latency ? osal_cputime(nullptr) - gc_cputime : 0; - if (unlikely(rc != MDBX_SUCCESS)) - goto fail; +__cold static pgno_t default_rp_augment_limit(const MDBX_env *env) { + const size_t timeframe = /* 16 секунд */ 16 << 16; + const size_t remain_1sec = + (env->options.gc_time_limit < timeframe) ? timeframe - (size_t)env->options.gc_time_limit : 0; + const size_t minimum = (env->maxgc_large1page * 2 > MDBX_PNL_INITIAL) ? env->maxgc_large1page * 2 : MDBX_PNL_INITIAL; + const size_t one_third = env->geo_in_bytes.now / 3 >> env->ps2ln; + const size_t augment_limit = + (one_third > minimum) ? minimum + (one_third - minimum) / timeframe * remain_1sec : minimum; + eASSERT(env, augment_limit < PAGELIST_LIMIT); + return pnl_bytes2size(pnl_size2bytes(augment_limit)); +} - tASSERT(txn, txn->tw.loose_count == 0); - txn->mt_dbs[FREE_DBI].md_mod_txnid = (txn->mt_dbistate[FREE_DBI] & DBI_DIRTY) - ? txn->mt_txnid - : txn->mt_dbs[FREE_DBI].md_mod_txnid; +static bool default_prefault_write(const MDBX_env *env) { + return !MDBX_MMAP_INCOHERENT_FILE_WRITE && !env->incore && + (env->flags & (MDBX_WRITEMAP | MDBX_RDONLY)) == MDBX_WRITEMAP; +} - txn->mt_dbs[MAIN_DBI].md_mod_txnid = (txn->mt_dbistate[MAIN_DBI] & DBI_DIRTY) - ? txn->mt_txnid - : txn->mt_dbs[MAIN_DBI].md_mod_txnid; +static bool default_prefer_waf_insteadof_balance(const MDBX_env *env) { + (void)env; + return false; +} - ts_2 = latency ? osal_monotime() : 0; - ts_3 = ts_2; - if (AUDIT_ENABLED()) { - rc = audit_ex(txn, MDBX_PNL_GETSIZE(txn->tw.retired_pages), true); - ts_3 = osal_monotime(); - if (unlikely(rc != MDBX_SUCCESS)) - goto fail; - } +static uint16_t default_subpage_limit(const MDBX_env *env) { + (void)env; + return 65535 /* 100% */; +} - bool need_flush_for_nometasync = false; - const meta_ptr_t head = meta_recent(env, &txn->tw.troika); - const uint32_t meta_sync_txnid = - atomic_load32(&env->me_lck->mti_meta_sync_txnid, mo_Relaxed); - /* sync prev meta */ - if (head.is_steady && meta_sync_txnid != (uint32_t)head.txnid) { - /* Исправление унаследованного от LMDB недочета: - * - * Всё хорошо, если все процессы работающие с БД не используют WRITEMAP. - * Тогда мета-страница (обновленная, но не сброшенная на диск) будет - * сохранена в результате fdatasync() при записи данных этой транзакции. - * - * Всё хорошо, если все процессы работающие с БД используют WRITEMAP - * без MDBX_AVOID_MSYNC. - * Тогда мета-страница (обновленная, но не сброшенная на диск) будет - * сохранена в результате msync() при записи данных этой транзакции. - * - * Если же в процессах работающих с БД используется оба метода, как sync() - * в режиме MDBX_WRITEMAP, так и записи через файловый дескриптор, то - * становится невозможным обеспечить фиксацию на диске мета-страницы - * предыдущей транзакции и данных текущей транзакции, за счет одной - * sync-операцией выполняемой после записи данных текущей транзакции. - * Соответственно, требуется явно обновлять мета-страницу, что полностью - * уничтожает выгоду от NOMETASYNC. */ - const uint32_t txnid_dist = - ((txn->mt_flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC) - ? MDBX_NOMETASYNC_LAZY_FD - : MDBX_NOMETASYNC_LAZY_WRITEMAP; - /* Смысл "магии" в том, чтобы избежать отдельного вызова fdatasync() - * или msync() для гарантированной фиксации на диске мета-страницы, - * которая была "лениво" отправлена на запись в предыдущей транзакции, - * но не сброшена на диск из-за активного режима MDBX_NOMETASYNC. */ - if ( -#if defined(_WIN32) || defined(_WIN64) - !env->me_overlapped_fd && -#endif - meta_sync_txnid == (uint32_t)head.txnid - txnid_dist) - need_flush_for_nometasync = true; - else { - rc = meta_sync(env, head); - if (unlikely(rc != MDBX_SUCCESS)) { - ERROR("txn-%s: error %d", "presync-meta", rc); - goto fail; - } - } - } +static uint16_t default_subpage_room_threshold(const MDBX_env *env) { + (void)env; + return 0 /* 0% */; +} - if (txn->tw.dirtylist) { - tASSERT(txn, (txn->mt_flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); - tASSERT(txn, txn->tw.loose_count == 0); +static uint16_t default_subpage_reserve_prereq(const MDBX_env *env) { + (void)env; + return 27525 /* 42% */; +} - mdbx_filehandle_t fd = -#if defined(_WIN32) || defined(_WIN64) - env->me_overlapped_fd ? env->me_overlapped_fd : env->me_lazy_fd; - (void)need_flush_for_nometasync; -#else -#define MDBX_WRITETHROUGH_THRESHOLD_DEFAULT 2 - (need_flush_for_nometasync || - env->me_dsync_fd == INVALID_HANDLE_VALUE || - txn->tw.dirtylist->length > env->me_options.writethrough_threshold || - atomic_load64(&env->me_lck->mti_unsynced_pages, mo_Relaxed)) - ? env->me_lazy_fd - : env->me_dsync_fd; -#endif /* Windows */ +static uint16_t default_subpage_reserve_limit(const MDBX_env *env) { + (void)env; + return 2753 /* 4.2% */; +} - iov_ctx_t write_ctx; - rc = iov_init(txn, &write_ctx, txn->tw.dirtylist->length, - txn->tw.dirtylist->pages_including_loose, fd, false); - if (unlikely(rc != MDBX_SUCCESS)) { - ERROR("txn-%s: error %d", "iov-init", rc); - goto fail; - } +static uint16_t default_merge_threshold_16dot16_percent(const MDBX_env *env) { + (void)env; + return 65536 / 4 /* 25% */; +} - rc = txn_write(txn, &write_ctx); - if (unlikely(rc != MDBX_SUCCESS)) { - ERROR("txn-%s: error %d", "write", rc); - goto fail; - } - } else { - tASSERT(txn, (txn->mt_flags & MDBX_WRITEMAP) != 0 && !MDBX_AVOID_MSYNC); - env->me_lck->mti_unsynced_pages.weak += txn->tw.writemap_dirty_npages; - if (!env->me_lck->mti_eoos_timestamp.weak) - env->me_lck->mti_eoos_timestamp.weak = osal_monotime(); - } +static pgno_t default_dp_reserve_limit(const MDBX_env *env) { + (void)env; + return MDBX_PNL_INITIAL; +} - /* TODO: use ctx.flush_begin & ctx.flush_end for range-sync */ - ts_4 = latency ? osal_monotime() : 0; +static pgno_t default_dp_initial(const MDBX_env *env) { + (void)env; + return MDBX_PNL_INITIAL; +} - MDBX_meta meta; - memcpy(meta.mm_magic_and_version, head.ptr_c->mm_magic_and_version, 8); - meta.mm_extra_flags = head.ptr_c->mm_extra_flags; - meta.mm_validator_id = head.ptr_c->mm_validator_id; - meta.mm_extra_pagehdr = head.ptr_c->mm_extra_pagehdr; - unaligned_poke_u64(4, meta.mm_pages_retired, - unaligned_peek_u64(4, head.ptr_c->mm_pages_retired) + - MDBX_PNL_GETSIZE(txn->tw.retired_pages)); - meta.mm_geo = txn->mt_geo; - meta.mm_dbs[FREE_DBI] = txn->mt_dbs[FREE_DBI]; - meta.mm_dbs[MAIN_DBI] = txn->mt_dbs[MAIN_DBI]; - meta.mm_canary = txn->mt_canary; - - txnid_t commit_txnid = txn->mt_txnid; -#if MDBX_ENABLE_BIGFOOT - if (gcu_ctx.bigfoot > txn->mt_txnid) { - commit_txnid = gcu_ctx.bigfoot; - TRACE("use @%" PRIaTXN " (+%zu) for commit bigfoot-txn", commit_txnid, - (size_t)(commit_txnid - txn->mt_txnid)); - } -#endif - meta.unsafe_sign = MDBX_DATASIGN_NONE; - meta_set_txnid(env, &meta, commit_txnid); +static uint8_t default_spill_max_denominator(const MDBX_env *env) { + (void)env; + return 8; +} - rc = sync_locked(env, env->me_flags | txn->mt_flags | MDBX_SHRINK_ALLOWED, - &meta, &txn->tw.troika); +static uint8_t default_spill_min_denominator(const MDBX_env *env) { + (void)env; + return 8; +} - ts_5 = latency ? osal_monotime() : 0; - if (unlikely(rc != MDBX_SUCCESS)) { - env->me_flags |= MDBX_FATAL_ERROR; - ERROR("txn-%s: error %d", "sync", rc); - goto fail; - } +static uint8_t default_spill_parent4child_denominator(const MDBX_env *env) { + (void)env; + return 0; +} - end_mode = MDBX_END_COMMITTED | MDBX_END_UPDATE | MDBX_END_EOTDONE; +static uint8_t default_dp_loose_limit(const MDBX_env *env) { + (void)env; + return 64; +} + +void env_options_init(MDBX_env *env) { + env->options.rp_augment_limit = default_rp_augment_limit(env); + env->options.dp_reserve_limit = default_dp_reserve_limit(env); + env->options.dp_initial = default_dp_initial(env); + env->options.dp_limit = default_dp_limit(env); + env->options.spill_max_denominator = default_spill_max_denominator(env); + env->options.spill_min_denominator = default_spill_min_denominator(env); + env->options.spill_parent4child_denominator = default_spill_parent4child_denominator(env); + env->options.dp_loose_limit = default_dp_loose_limit(env); + env->options.merge_threshold_16dot16_percent = default_merge_threshold_16dot16_percent(env); + if (default_prefer_waf_insteadof_balance(env)) + env->options.prefer_waf_insteadof_balance = true; -done: - if (latency) - take_gcprof(txn, latency); - rc = txn_end(txn, end_mode); - -provide_latency: - if (latency) { - latency->preparation = ts_1 ? osal_monotime_to_16dot16(ts_1 - ts_0) : 0; - latency->gc_wallclock = - (ts_2 > ts_1) ? osal_monotime_to_16dot16(ts_2 - ts_1) : 0; - latency->gc_cputime = gc_cputime ? osal_monotime_to_16dot16(gc_cputime) : 0; - latency->audit = (ts_3 > ts_2) ? osal_monotime_to_16dot16(ts_3 - ts_2) : 0; - latency->write = (ts_4 > ts_3) ? osal_monotime_to_16dot16(ts_4 - ts_3) : 0; - latency->sync = (ts_5 > ts_4) ? osal_monotime_to_16dot16(ts_5 - ts_4) : 0; - const uint64_t ts_6 = osal_monotime(); - latency->ending = ts_5 ? osal_monotime_to_16dot16(ts_6 - ts_5) : 0; - latency->whole = osal_monotime_to_16dot16_noUnderflow(ts_6 - ts_0); - } - return rc; +#if !(defined(_WIN32) || defined(_WIN64)) + env->options.writethrough_threshold = +#if defined(__linux__) || defined(__gnu_linux__) + globals.running_on_WSL1 ? MAX_PAGENO : +#endif /* Linux */ + MDBX_WRITETHROUGH_THRESHOLD_DEFAULT; +#endif /* Windows */ -fail: - txn->mt_flags |= MDBX_TXN_ERROR; - if (latency) - take_gcprof(txn, latency); - mdbx_txn_abort(txn); - goto provide_latency; + env->options.subpage.limit = default_subpage_limit(env); + env->options.subpage.room_threshold = default_subpage_room_threshold(env); + env->options.subpage.reserve_prereq = default_subpage_reserve_prereq(env); + env->options.subpage.reserve_limit = default_subpage_reserve_limit(env); } -static __always_inline int cmp_int_inline(const size_t expected_alignment, - const MDBX_val *a, - const MDBX_val *b) { - if (likely(a->iov_len == b->iov_len)) { - if (sizeof(size_t) > 7 && likely(a->iov_len == 8)) - return CMP2INT(unaligned_peek_u64(expected_alignment, a->iov_base), - unaligned_peek_u64(expected_alignment, b->iov_base)); - if (likely(a->iov_len == 4)) - return CMP2INT(unaligned_peek_u32(expected_alignment, a->iov_base), - unaligned_peek_u32(expected_alignment, b->iov_base)); - if (sizeof(size_t) < 8 && likely(a->iov_len == 8)) - return CMP2INT(unaligned_peek_u64(expected_alignment, a->iov_base), - unaligned_peek_u64(expected_alignment, b->iov_base)); +void env_options_adjust_dp_limit(MDBX_env *env) { + if (!env->options.flags.non_auto.dp_limit) + env->options.dp_limit = default_dp_limit(env); + else { + const pgno_t max_pgno = env_max_pgno(env); + if (env->options.dp_limit > max_pgno - NUM_METAS) + env->options.dp_limit = max_pgno - NUM_METAS; + if (env->options.dp_limit < CURSOR_STACK_SIZE * 4) + env->options.dp_limit = CURSOR_STACK_SIZE * 4; } - ERROR("mismatch and/or invalid size %p.%zu/%p.%zu for INTEGERKEY/INTEGERDUP", - a->iov_base, a->iov_len, b->iov_base, b->iov_len); - return 0; + if (env->options.dp_initial > env->options.dp_limit && env->options.dp_initial > default_dp_initial(env)) + env->options.dp_initial = env->options.dp_limit; + env->options.need_dp_limit_adjust = false; } -__hot static int cmp_int_unaligned(const MDBX_val *a, const MDBX_val *b) { - return cmp_int_inline(1, a, b); -} +void env_options_adjust_defaults(MDBX_env *env) { + if (!env->options.flags.non_auto.rp_augment_limit) + env->options.rp_augment_limit = default_rp_augment_limit(env); + if (!env->options.flags.non_auto.prefault_write) + env->options.prefault_write = default_prefault_write(env); -/* Compare two items pointing at 2-byte aligned unsigned int's. */ -#if MDBX_UNALIGNED_OK < 2 || \ - (MDBX_DEBUG || MDBX_FORCE_ASSERTIONS || !defined(NDEBUG)) -__hot static int cmp_int_align2(const MDBX_val *a, const MDBX_val *b) { - return cmp_int_inline(2, a, b); -} -#else -#define cmp_int_align2 cmp_int_unaligned -#endif /* !MDBX_UNALIGNED_OK || debug */ + env->options.need_dp_limit_adjust = true; + if (!env->txn) + env_options_adjust_dp_limit(env); -/* Compare two items pointing at aligned unsigned int's. */ -#if MDBX_UNALIGNED_OK < 4 || \ - (MDBX_DEBUG || MDBX_FORCE_ASSERTIONS || !defined(NDEBUG)) -__hot static int cmp_int_align4(const MDBX_val *a, const MDBX_val *b) { - return cmp_int_inline(4, a, b); + const size_t basis = env->geo_in_bytes.now; + /* TODO: use options? */ + const unsigned factor = 9; + size_t threshold = (basis < ((size_t)65536 << factor)) ? 65536 /* minimal threshold */ + : (basis > (MEGABYTE * 4 << factor)) ? MEGABYTE * 4 /* maximal threshold */ + : basis >> factor; + threshold = + (threshold < env->geo_in_bytes.shrink || !env->geo_in_bytes.shrink) ? threshold : env->geo_in_bytes.shrink; + env->madv_threshold = bytes2pgno(env, bytes_align2os_bytes(env, threshold)); } -#else -#define cmp_int_align4 cmp_int_unaligned -#endif /* !MDBX_UNALIGNED_OK || debug */ -/* Compare two items lexically */ -__hot static int cmp_lexical(const MDBX_val *a, const MDBX_val *b) { - if (a->iov_len == b->iov_len) - return a->iov_len ? memcmp(a->iov_base, b->iov_base, a->iov_len) : 0; - - const int diff_len = (a->iov_len < b->iov_len) ? -1 : 1; - const size_t shortest = (a->iov_len < b->iov_len) ? a->iov_len : b->iov_len; - int diff_data = shortest ? memcmp(a->iov_base, b->iov_base, shortest) : 0; - return likely(diff_data) ? diff_data : diff_len; -} +//------------------------------------------------------------------------------ -MDBX_NOTHROW_PURE_FUNCTION static __always_inline unsigned -tail3le(const uint8_t *p, size_t l) { - STATIC_ASSERT(sizeof(unsigned) > 2); - // 1: 0 0 0 - // 2: 0 1 1 - // 3: 0 1 2 - return p[0] | p[l >> 1] << 8 | p[l - 1] << 16; -} +__cold int mdbx_env_set_option(MDBX_env *env, const MDBX_option_t option, uint64_t value) { + int err = check_env(env, false); + if (unlikely(err != MDBX_SUCCESS)) + return LOG_IFERR(err); -/* Compare two items in reverse byte order */ -__hot static int cmp_reverse(const MDBX_val *a, const MDBX_val *b) { - size_t left = (a->iov_len < b->iov_len) ? a->iov_len : b->iov_len; - if (likely(left)) { - const uint8_t *pa = ptr_disp(a->iov_base, a->iov_len); - const uint8_t *pb = ptr_disp(b->iov_base, b->iov_len); - while (left >= sizeof(size_t)) { - pa -= sizeof(size_t); - pb -= sizeof(size_t); - left -= sizeof(size_t); - STATIC_ASSERT(sizeof(size_t) == 4 || sizeof(size_t) == 8); - if (sizeof(size_t) == 4) { - uint32_t xa = unaligned_peek_u32(1, pa); - uint32_t xb = unaligned_peek_u32(1, pb); -#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__ - xa = osal_bswap32(xa); - xb = osal_bswap32(xb); -#endif /* __BYTE_ORDER__ != __ORDER_BIG_ENDIAN__ */ - if (xa != xb) - return (xa < xb) ? -1 : 1; - } else { - uint64_t xa = unaligned_peek_u64(1, pa); - uint64_t xb = unaligned_peek_u64(1, pb); -#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__ - xa = osal_bswap64(xa); - xb = osal_bswap64(xb); -#endif /* __BYTE_ORDER__ != __ORDER_BIG_ENDIAN__ */ - if (xa != xb) - return (xa < xb) ? -1 : 1; - } - } - if (sizeof(size_t) == 8 && left >= 4) { - pa -= 4; - pb -= 4; - left -= 4; - uint32_t xa = unaligned_peek_u32(1, pa); - uint32_t xb = unaligned_peek_u32(1, pb); -#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__ - xa = osal_bswap32(xa); - xb = osal_bswap32(xb); -#endif /* __BYTE_ORDER__ != __ORDER_BIG_ENDIAN__ */ - if (xa != xb) - return (xa < xb) ? -1 : 1; - } - if (left) { - unsigned xa = tail3le(pa - left, left); - unsigned xb = tail3le(pb - left, left); - if (xa != xb) - return (xa < xb) ? -1 : 1; + const bool lock_needed = ((env->flags & ENV_ACTIVE) && env->basal_txn && !env_owned_wrtxn(env)); + bool should_unlock = false; + switch (option) { + case MDBX_opt_sync_bytes: + if (value == /* default */ UINT64_MAX) + value = MAX_WRITE; + if (unlikely(env->flags & MDBX_RDONLY)) + return LOG_IFERR(MDBX_EACCESS); + if (unlikely(!(env->flags & ENV_ACTIVE))) + return LOG_IFERR(MDBX_EPERM); + if (unlikely(value > SIZE_MAX - 65536)) + return LOG_IFERR(MDBX_EINVAL); + value = bytes2pgno(env, (size_t)value + env->ps - 1); + if ((uint32_t)value != atomic_load32(&env->lck->autosync_threshold, mo_AcquireRelease) && + atomic_store32(&env->lck->autosync_threshold, (uint32_t)value, mo_Relaxed) + /* Дергаем sync(force=off) только если задано новое не-нулевое значение + * и мы вне транзакции */ + && lock_needed) { + err = env_sync(env, false, false); + if (err == /* нечего сбрасывать на диск */ MDBX_RESULT_TRUE) + err = MDBX_SUCCESS; } - } - return CMP2INT(a->iov_len, b->iov_len); -} - -/* Fast non-lexically comparator */ -__hot static int cmp_lenfast(const MDBX_val *a, const MDBX_val *b) { - int diff = CMP2INT(a->iov_len, b->iov_len); - return (likely(diff) || a->iov_len == 0) - ? diff - : memcmp(a->iov_base, b->iov_base, a->iov_len); -} - -__hot static bool eq_fast_slowpath(const uint8_t *a, const uint8_t *b, - size_t l) { - if (likely(l > 3)) { - if (MDBX_UNALIGNED_OK >= 4 && likely(l < 9)) - return ((unaligned_peek_u32(1, a) - unaligned_peek_u32(1, b)) | - (unaligned_peek_u32(1, a + l - 4) - - unaligned_peek_u32(1, b + l - 4))) == 0; - if (MDBX_UNALIGNED_OK >= 8 && sizeof(size_t) > 7 && likely(l < 17)) - return ((unaligned_peek_u64(1, a) - unaligned_peek_u64(1, b)) | - (unaligned_peek_u64(1, a + l - 8) - - unaligned_peek_u64(1, b + l - 8))) == 0; - return memcmp(a, b, l) == 0; - } - if (likely(l)) - return tail3le(a, l) == tail3le(b, l); - return true; -} - -static __always_inline bool eq_fast(const MDBX_val *a, const MDBX_val *b) { - return unlikely(a->iov_len == b->iov_len) && - eq_fast_slowpath(a->iov_base, b->iov_base, a->iov_len); -} + break; -static int validate_meta(MDBX_env *env, MDBX_meta *const meta, - const MDBX_page *const page, - const unsigned meta_number, unsigned *guess_pagesize) { - const uint64_t magic_and_version = - unaligned_peek_u64(4, &meta->mm_magic_and_version); - if (unlikely(magic_and_version != MDBX_DATA_MAGIC && - magic_and_version != MDBX_DATA_MAGIC_LEGACY_COMPAT && - magic_and_version != MDBX_DATA_MAGIC_LEGACY_DEVEL)) { - ERROR("meta[%u] has invalid magic/version %" PRIx64, meta_number, - magic_and_version); - return ((magic_and_version >> 8) != MDBX_MAGIC) ? MDBX_INVALID - : MDBX_VERSION_MISMATCH; - } + case MDBX_opt_sync_period: + if (value == /* default */ UINT64_MAX) + value = 2780315 /* 42.42424 секунды */; + if (unlikely(env->flags & MDBX_RDONLY)) + return LOG_IFERR(MDBX_EACCESS); + if (unlikely(!(env->flags & ENV_ACTIVE))) + return LOG_IFERR(MDBX_EPERM); + if (unlikely(value > UINT32_MAX)) + return LOG_IFERR(MDBX_EINVAL); + value = osal_16dot16_to_monotime((uint32_t)value); + if (value != atomic_load64(&env->lck->autosync_period, mo_AcquireRelease) && + atomic_store64(&env->lck->autosync_period, value, mo_Relaxed) + /* Дергаем sync(force=off) только если задано новое не-нулевое значение + * и мы вне транзакции */ + && lock_needed) { + err = env_sync(env, false, false); + if (err == /* нечего сбрасывать на диск */ MDBX_RESULT_TRUE) + err = MDBX_SUCCESS; + } + break; - if (unlikely(page->mp_pgno != meta_number)) { - ERROR("meta[%u] has invalid pageno %" PRIaPGNO, meta_number, page->mp_pgno); - return MDBX_INVALID; - } + case MDBX_opt_max_db: + if (value == /* default */ UINT64_MAX) + value = 42; + if (unlikely(value > MDBX_MAX_DBI)) + return LOG_IFERR(MDBX_EINVAL); + if (unlikely(env->dxb_mmap.base)) + return LOG_IFERR(MDBX_EPERM); + env->max_dbi = (unsigned)value + CORE_DBS; + break; - if (unlikely(page->mp_flags != P_META)) { - ERROR("page #%u not a meta-page", meta_number); - return MDBX_INVALID; - } + case MDBX_opt_max_readers: + if (value == /* default */ UINT64_MAX) + value = MDBX_READERS_LIMIT; + if (unlikely(value < 1 || value > MDBX_READERS_LIMIT)) + return LOG_IFERR(MDBX_EINVAL); + if (unlikely(env->dxb_mmap.base)) + return LOG_IFERR(MDBX_EPERM); + env->max_readers = (unsigned)value; + break; - /* LY: check pagesize */ - if (unlikely(!is_powerof2(meta->mm_psize) || meta->mm_psize < MIN_PAGESIZE || - meta->mm_psize > MAX_PAGESIZE)) { - WARNING("meta[%u] has invalid pagesize (%u), skip it", meta_number, - meta->mm_psize); - return is_powerof2(meta->mm_psize) ? MDBX_VERSION_MISMATCH : MDBX_INVALID; - } + case MDBX_opt_dp_reserve_limit: + if (value == /* default */ UINT64_MAX) + value = default_dp_reserve_limit(env); + if (unlikely(value > INT_MAX)) + return LOG_IFERR(MDBX_EINVAL); + if (env->options.dp_reserve_limit != (unsigned)value) { + if (lock_needed) { + err = lck_txn_lock(env, false); + if (unlikely(err != MDBX_SUCCESS)) + return LOG_IFERR(err); + should_unlock = true; + } + env->options.dp_reserve_limit = (unsigned)value; + while (env->shadow_reserve_len > env->options.dp_reserve_limit) { + eASSERT(env, env->shadow_reserve != nullptr); + page_t *dp = env->shadow_reserve; + MDBX_ASAN_UNPOISON_MEMORY_REGION(dp, env->ps); + VALGRIND_MAKE_MEM_DEFINED(&page_next(dp), sizeof(page_t *)); + env->shadow_reserve = page_next(dp); + void *const ptr = ptr_disp(dp, -(ptrdiff_t)sizeof(size_t)); + osal_free(ptr); + env->shadow_reserve_len -= 1; + } + } + break; - if (guess_pagesize && *guess_pagesize != meta->mm_psize) { - *guess_pagesize = meta->mm_psize; - VERBOSE("meta[%u] took pagesize %u", meta_number, meta->mm_psize); - } + case MDBX_opt_rp_augment_limit: + if (value == /* default */ UINT64_MAX) { + env->options.flags.non_auto.rp_augment_limit = 0; + env->options.rp_augment_limit = default_rp_augment_limit(env); + } else if (unlikely(value > PAGELIST_LIMIT)) + return LOG_IFERR(MDBX_EINVAL); + else { + env->options.flags.non_auto.rp_augment_limit = 1; + env->options.rp_augment_limit = (unsigned)value; + } + break; - const txnid_t txnid = unaligned_peek_u64(4, &meta->mm_txnid_a); - if (unlikely(txnid != unaligned_peek_u64(4, &meta->mm_txnid_b))) { - WARNING("meta[%u] not completely updated, skip it", meta_number); - return MDBX_RESULT_TRUE; - } + case MDBX_opt_gc_time_limit: + if (value == /* default */ UINT64_MAX) + value = 0; + if (unlikely(value > UINT32_MAX)) + return LOG_IFERR(MDBX_EINVAL); + if (unlikely(env->flags & MDBX_RDONLY)) + return LOG_IFERR(MDBX_EACCESS); + value = osal_16dot16_to_monotime((uint32_t)value); + if (value != env->options.gc_time_limit) { + if (env->txn && lock_needed) + return LOG_IFERR(MDBX_EPERM); + env->options.gc_time_limit = value; + if (!env->options.flags.non_auto.rp_augment_limit) + env->options.rp_augment_limit = default_rp_augment_limit(env); + } + break; - if (unlikely(meta->mm_extra_flags != 0)) { - WARNING("meta[%u] has unsupported %s 0x%x, skip it", meta_number, - "extra-flags", meta->mm_extra_flags); - return MDBX_RESULT_TRUE; - } - if (unlikely(meta->mm_validator_id != 0)) { - WARNING("meta[%u] has unsupported %s 0x%x, skip it", meta_number, - "validator-id", meta->mm_validator_id); - return MDBX_RESULT_TRUE; - } - if (unlikely(meta->mm_extra_pagehdr != 0)) { - WARNING("meta[%u] has unsupported %s 0x%x, skip it", meta_number, - "extra-pageheader", meta->mm_extra_pagehdr); - return MDBX_RESULT_TRUE; - } + case MDBX_opt_txn_dp_limit: + case MDBX_opt_txn_dp_initial: + if (value != /* default */ UINT64_MAX && unlikely(value > PAGELIST_LIMIT || value < CURSOR_STACK_SIZE * 4)) + return LOG_IFERR(MDBX_EINVAL); + if (unlikely(env->flags & MDBX_RDONLY)) + return LOG_IFERR(MDBX_EACCESS); + if (lock_needed) { + err = lck_txn_lock(env, false); + if (unlikely(err != MDBX_SUCCESS)) + return LOG_IFERR(err); + should_unlock = true; + } + if (env->txn) + err = MDBX_EPERM /* unable change during transaction */; + else { + const pgno_t max_pgno = env_max_pgno(env); + if (option == MDBX_opt_txn_dp_initial) { + if (value == /* default */ UINT64_MAX) + env->options.dp_initial = default_dp_initial(env); + else { + env->options.dp_initial = (pgno_t)value; + if (env->options.dp_initial > max_pgno) + env->options.dp_initial = (max_pgno > CURSOR_STACK_SIZE * 4) ? max_pgno : CURSOR_STACK_SIZE * 4; + } + } + if (option == MDBX_opt_txn_dp_limit) { + if (value == /* default */ UINT64_MAX) { + env->options.flags.non_auto.dp_limit = 0; + } else { + env->options.flags.non_auto.dp_limit = 1; + env->options.dp_limit = (pgno_t)value; + } + env_options_adjust_dp_limit(env); + } + } + break; - /* LY: check signature as a checksum */ - if (META_IS_STEADY(meta) && - unlikely(unaligned_peek_u64(4, &meta->mm_sign) != meta_sign(meta))) { - WARNING("meta[%u] has invalid steady-checksum (0x%" PRIx64 " != 0x%" PRIx64 - "), skip it", - meta_number, unaligned_peek_u64(4, &meta->mm_sign), - meta_sign(meta)); - return MDBX_RESULT_TRUE; - } + case MDBX_opt_spill_max_denominator: + if (value == /* default */ UINT64_MAX) + value = default_spill_max_denominator(env); + if (unlikely(value > 255)) + return LOG_IFERR(MDBX_EINVAL); + env->options.spill_max_denominator = (uint8_t)value; + break; + case MDBX_opt_spill_min_denominator: + if (value == /* default */ UINT64_MAX) + value = default_spill_min_denominator(env); + if (unlikely(value > 255)) + return LOG_IFERR(MDBX_EINVAL); + env->options.spill_min_denominator = (uint8_t)value; + break; + case MDBX_opt_spill_parent4child_denominator: + if (value == /* default */ UINT64_MAX) + value = default_spill_parent4child_denominator(env); + if (unlikely(value > 255)) + return LOG_IFERR(MDBX_EINVAL); + env->options.spill_parent4child_denominator = (uint8_t)value; + break; - DEBUG("checking meta%" PRIaPGNO " = root %" PRIaPGNO "/%" PRIaPGNO - ", geo %" PRIaPGNO "/%" PRIaPGNO "-%" PRIaPGNO "/%" PRIaPGNO - " +%u -%u, txn_id %" PRIaTXN ", %s", - page->mp_pgno, meta->mm_dbs[MAIN_DBI].md_root, - meta->mm_dbs[FREE_DBI].md_root, meta->mm_geo.lower, meta->mm_geo.next, - meta->mm_geo.now, meta->mm_geo.upper, pv2pages(meta->mm_geo.grow_pv), - pv2pages(meta->mm_geo.shrink_pv), txnid, durable_caption(meta)); + case MDBX_opt_loose_limit: + if (value == /* default */ UINT64_MAX) + value = default_dp_loose_limit(env); + if (unlikely(value > 255)) + return LOG_IFERR(MDBX_EINVAL); + env->options.dp_loose_limit = (uint8_t)value; + break; - if (unlikely(txnid < MIN_TXNID || txnid > MAX_TXNID)) { - WARNING("meta[%u] has invalid txnid %" PRIaTXN ", skip it", meta_number, - txnid); - return MDBX_RESULT_TRUE; - } + case MDBX_opt_merge_threshold_16dot16_percent: + if (value == /* default */ UINT64_MAX) + value = default_merge_threshold_16dot16_percent(env); + if (unlikely(value < 8192 || value > 32768)) + return LOG_IFERR(MDBX_EINVAL); + env->options.merge_threshold_16dot16_percent = (unsigned)value; + recalculate_merge_thresholds(env); + break; - /* LY: check min-pages value */ - if (unlikely(meta->mm_geo.lower < MIN_PAGENO || - meta->mm_geo.lower > MAX_PAGENO + 1)) { - WARNING("meta[%u] has invalid min-pages (%" PRIaPGNO "), skip it", - meta_number, meta->mm_geo.lower); - return MDBX_INVALID; - } + case MDBX_opt_writethrough_threshold: +#if defined(_WIN32) || defined(_WIN64) + /* позволяем "установить" значение по-умолчанию и совпадающее + * с поведением соответствующим текущей установке MDBX_NOMETASYNC */ + if (value == /* default */ UINT64_MAX && value != ((env->flags & MDBX_NOMETASYNC) ? 0 : UINT_MAX)) + err = MDBX_EINVAL; +#else + if (value == /* default */ UINT64_MAX) + value = MDBX_WRITETHROUGH_THRESHOLD_DEFAULT; + if (value != (unsigned)value) + err = MDBX_EINVAL; + else + env->options.writethrough_threshold = (unsigned)value; +#endif + break; - /* LY: check max-pages value */ - if (unlikely(meta->mm_geo.upper < MIN_PAGENO || - meta->mm_geo.upper > MAX_PAGENO + 1 || - meta->mm_geo.upper < meta->mm_geo.lower)) { - WARNING("meta[%u] has invalid max-pages (%" PRIaPGNO "), skip it", - meta_number, meta->mm_geo.upper); - return MDBX_INVALID; - } + case MDBX_opt_prefault_write_enable: + if (value == /* default */ UINT64_MAX) { + env->options.prefault_write = default_prefault_write(env); + env->options.flags.non_auto.prefault_write = false; + } else if (value > 1) + err = MDBX_EINVAL; + else { + env->options.prefault_write = value != 0; + env->options.flags.non_auto.prefault_write = true; + } + break; - /* LY: check last_pgno */ - if (unlikely(meta->mm_geo.next < MIN_PAGENO || - meta->mm_geo.next - 1 > MAX_PAGENO)) { - WARNING("meta[%u] has invalid next-pageno (%" PRIaPGNO "), skip it", - meta_number, meta->mm_geo.next); - return MDBX_CORRUPTED; - } + case MDBX_opt_prefer_waf_insteadof_balance: + if (value == /* default */ UINT64_MAX) + env->options.prefer_waf_insteadof_balance = default_prefer_waf_insteadof_balance(env); + else if (value > 1) + err = MDBX_EINVAL; + else + env->options.prefer_waf_insteadof_balance = value != 0; + break; - /* LY: check filesize & used_bytes */ - const uint64_t used_bytes = meta->mm_geo.next * (uint64_t)meta->mm_psize; - if (unlikely(used_bytes > env->me_dxb_mmap.filesize)) { - /* Here could be a race with DB-shrinking performed by other process */ - int err = osal_filesize(env->me_lazy_fd, &env->me_dxb_mmap.filesize); - if (unlikely(err != MDBX_SUCCESS)) - return err; - if (unlikely(used_bytes > env->me_dxb_mmap.filesize)) { - WARNING("meta[%u] used-bytes (%" PRIu64 ") beyond filesize (%" PRIu64 - "), skip it", - meta_number, used_bytes, env->me_dxb_mmap.filesize); - return MDBX_CORRUPTED; + case MDBX_opt_subpage_limit: + if (value == /* default */ UINT64_MAX) { + env->options.subpage.limit = default_subpage_limit(env); + recalculate_subpage_thresholds(env); + } else if (value > 65535) + err = MDBX_EINVAL; + else { + env->options.subpage.limit = (uint16_t)value; + recalculate_subpage_thresholds(env); } - } - if (unlikely(meta->mm_geo.next - 1 > MAX_PAGENO || - used_bytes > MAX_MAPSIZE)) { - WARNING("meta[%u] has too large used-space (%" PRIu64 "), skip it", - meta_number, used_bytes); - return MDBX_TOO_LARGE; - } + break; - /* LY: check mapsize limits */ - pgno_t geo_lower = meta->mm_geo.lower; - uint64_t mapsize_min = geo_lower * (uint64_t)meta->mm_psize; - STATIC_ASSERT(MAX_MAPSIZE < PTRDIFF_MAX - MAX_PAGESIZE); - STATIC_ASSERT(MIN_MAPSIZE < MAX_MAPSIZE); - STATIC_ASSERT((uint64_t)(MAX_PAGENO + 1) * MIN_PAGESIZE % (4ul << 20) == 0); - if (unlikely(mapsize_min < MIN_MAPSIZE || mapsize_min > MAX_MAPSIZE)) { - if (MAX_MAPSIZE != MAX_MAPSIZE64 && mapsize_min > MAX_MAPSIZE && - mapsize_min <= MAX_MAPSIZE64) { - eASSERT(env, - meta->mm_geo.next - 1 <= MAX_PAGENO && used_bytes <= MAX_MAPSIZE); - WARNING("meta[%u] has too large min-mapsize (%" PRIu64 "), " - "but size of used space still acceptable (%" PRIu64 ")", - meta_number, mapsize_min, used_bytes); - geo_lower = (pgno_t)((mapsize_min = MAX_MAPSIZE) / meta->mm_psize); - if (geo_lower > MAX_PAGENO + 1) { - geo_lower = MAX_PAGENO + 1; - mapsize_min = geo_lower * (uint64_t)meta->mm_psize; - } - WARNING("meta[%u] consider get-%s pageno is %" PRIaPGNO - " instead of wrong %" PRIaPGNO - ", will be corrected on next commit(s)", - meta_number, "lower", geo_lower, meta->mm_geo.lower); - meta->mm_geo.lower = geo_lower; - } else { - WARNING("meta[%u] has invalid min-mapsize (%" PRIu64 "), skip it", - meta_number, mapsize_min); - return MDBX_VERSION_MISMATCH; + case MDBX_opt_subpage_room_threshold: + if (value == /* default */ UINT64_MAX) { + env->options.subpage.room_threshold = default_subpage_room_threshold(env); + recalculate_subpage_thresholds(env); + } else if (value > 65535) + err = MDBX_EINVAL; + else { + env->options.subpage.room_threshold = (uint16_t)value; + recalculate_subpage_thresholds(env); } - } + break; - pgno_t geo_upper = meta->mm_geo.upper; - uint64_t mapsize_max = geo_upper * (uint64_t)meta->mm_psize; - STATIC_ASSERT(MIN_MAPSIZE < MAX_MAPSIZE); - if (unlikely(mapsize_max > MAX_MAPSIZE || - (MAX_PAGENO + 1) < - ceil_powerof2((size_t)mapsize_max, env->me_os_psize) / - (size_t)meta->mm_psize)) { - if (mapsize_max > MAX_MAPSIZE64) { - WARNING("meta[%u] has invalid max-mapsize (%" PRIu64 "), skip it", - meta_number, mapsize_max); - return MDBX_VERSION_MISMATCH; + case MDBX_opt_subpage_reserve_prereq: + if (value == /* default */ UINT64_MAX) { + env->options.subpage.reserve_prereq = default_subpage_reserve_prereq(env); + recalculate_subpage_thresholds(env); + } else if (value > 65535) + err = MDBX_EINVAL; + else { + env->options.subpage.reserve_prereq = (uint16_t)value; + recalculate_subpage_thresholds(env); } - /* allow to open large DB from a 32-bit environment */ - eASSERT(env, - meta->mm_geo.next - 1 <= MAX_PAGENO && used_bytes <= MAX_MAPSIZE); - WARNING("meta[%u] has too large max-mapsize (%" PRIu64 "), " - "but size of used space still acceptable (%" PRIu64 ")", - meta_number, mapsize_max, used_bytes); - geo_upper = (pgno_t)((mapsize_max = MAX_MAPSIZE) / meta->mm_psize); - if (geo_upper > MAX_PAGENO + 1) { - geo_upper = MAX_PAGENO + 1; - mapsize_max = geo_upper * (uint64_t)meta->mm_psize; + break; + + case MDBX_opt_subpage_reserve_limit: + if (value == /* default */ UINT64_MAX) { + env->options.subpage.reserve_limit = default_subpage_reserve_limit(env); + recalculate_subpage_thresholds(env); + } else if (value > 65535) + err = MDBX_EINVAL; + else { + env->options.subpage.reserve_limit = (uint16_t)value; + recalculate_subpage_thresholds(env); } - WARNING("meta[%u] consider get-%s pageno is %" PRIaPGNO - " instead of wrong %" PRIaPGNO - ", will be corrected on next commit(s)", - meta_number, "upper", geo_upper, meta->mm_geo.upper); - meta->mm_geo.upper = geo_upper; + break; + + default: + return LOG_IFERR(MDBX_EINVAL); } - /* LY: check and silently put mm_geo.now into [geo.lower...geo.upper]. - * - * Copy-with-compaction by previous version of libmdbx could produce DB-file - * less than meta.geo.lower bound, in case actual filling is low or no data - * at all. This is not a problem as there is no damage or loss of data. - * Therefore it is better not to consider such situation as an error, but - * silently correct it. */ - pgno_t geo_now = meta->mm_geo.now; - if (geo_now < geo_lower) - geo_now = geo_lower; - if (geo_now > geo_upper && meta->mm_geo.next <= geo_upper) - geo_now = geo_upper; + if (should_unlock) + lck_txn_unlock(env); + return LOG_IFERR(err); +} - if (unlikely(meta->mm_geo.next > geo_now)) { - WARNING("meta[%u] next-pageno (%" PRIaPGNO - ") is beyond end-pgno (%" PRIaPGNO "), skip it", - meta_number, meta->mm_geo.next, geo_now); - return MDBX_CORRUPTED; - } - if (meta->mm_geo.now != geo_now) { - WARNING("meta[%u] consider geo-%s pageno is %" PRIaPGNO - " instead of wrong %" PRIaPGNO - ", will be corrected on next commit(s)", - meta_number, "now", geo_now, meta->mm_geo.now); - meta->mm_geo.now = geo_now; - } +__cold int mdbx_env_get_option(const MDBX_env *env, const MDBX_option_t option, uint64_t *pvalue) { + int err = check_env(env, false); + if (unlikely(err != MDBX_SUCCESS)) + return LOG_IFERR(err); + if (unlikely(!pvalue)) + return LOG_IFERR(MDBX_EINVAL); - /* GC */ - if (meta->mm_dbs[FREE_DBI].md_root == P_INVALID) { - if (unlikely(meta->mm_dbs[FREE_DBI].md_branch_pages || - meta->mm_dbs[FREE_DBI].md_depth || - meta->mm_dbs[FREE_DBI].md_entries || - meta->mm_dbs[FREE_DBI].md_leaf_pages || - meta->mm_dbs[FREE_DBI].md_overflow_pages)) { - WARNING("meta[%u] has false-empty %s, skip it", meta_number, "GC/FreeDB"); - return MDBX_CORRUPTED; - } - } else if (unlikely(meta->mm_dbs[FREE_DBI].md_root >= meta->mm_geo.next)) { - WARNING("meta[%u] has invalid %s-root %" PRIaPGNO ", skip it", meta_number, - "GC/FreeDB", meta->mm_dbs[FREE_DBI].md_root); - return MDBX_CORRUPTED; - } + switch (option) { + case MDBX_opt_sync_bytes: + if (unlikely(!(env->flags & ENV_ACTIVE))) + return LOG_IFERR(MDBX_EPERM); + *pvalue = pgno2bytes(env, atomic_load32(&env->lck->autosync_threshold, mo_Relaxed)); + break; - /* MainDB */ - if (meta->mm_dbs[MAIN_DBI].md_root == P_INVALID) { - if (unlikely(meta->mm_dbs[MAIN_DBI].md_branch_pages || - meta->mm_dbs[MAIN_DBI].md_depth || - meta->mm_dbs[MAIN_DBI].md_entries || - meta->mm_dbs[MAIN_DBI].md_leaf_pages || - meta->mm_dbs[MAIN_DBI].md_overflow_pages)) { - WARNING("meta[%u] has false-empty %s", meta_number, "MainDB"); - return MDBX_CORRUPTED; - } - } else if (unlikely(meta->mm_dbs[MAIN_DBI].md_root >= meta->mm_geo.next)) { - WARNING("meta[%u] has invalid %s-root %" PRIaPGNO ", skip it", meta_number, - "MainDB", meta->mm_dbs[MAIN_DBI].md_root); - return MDBX_CORRUPTED; - } + case MDBX_opt_sync_period: + if (unlikely(!(env->flags & ENV_ACTIVE))) + return LOG_IFERR(MDBX_EPERM); + *pvalue = osal_monotime_to_16dot16(atomic_load64(&env->lck->autosync_period, mo_Relaxed)); + break; - if (unlikely(meta->mm_dbs[FREE_DBI].md_mod_txnid > txnid)) { - WARNING("meta[%u] has wrong md_mod_txnid %" PRIaTXN " for %s, skip it", - meta_number, meta->mm_dbs[FREE_DBI].md_mod_txnid, "GC/FreeDB"); - return MDBX_CORRUPTED; - } + case MDBX_opt_max_db: + *pvalue = env->max_dbi - CORE_DBS; + break; - if (unlikely(meta->mm_dbs[MAIN_DBI].md_mod_txnid > txnid)) { - WARNING("meta[%u] has wrong md_mod_txnid %" PRIaTXN " for %s, skip it", - meta_number, meta->mm_dbs[MAIN_DBI].md_mod_txnid, "MainDB"); - return MDBX_CORRUPTED; - } + case MDBX_opt_max_readers: + *pvalue = env->max_readers; + break; - if (unlikely((meta->mm_dbs[FREE_DBI].md_flags & DB_PERSISTENT_FLAGS) != - MDBX_INTEGERKEY)) { - WARNING("meta[%u] has unexpected/invalid db-flags 0x%x for %s", meta_number, - meta->mm_dbs[FREE_DBI].md_flags, "GC/FreeDB"); - return MDBX_INCOMPATIBLE; - } + case MDBX_opt_dp_reserve_limit: + *pvalue = env->options.dp_reserve_limit; + break; - return MDBX_SUCCESS; -} + case MDBX_opt_rp_augment_limit: + *pvalue = env->options.rp_augment_limit; + break; -static int validate_meta_copy(MDBX_env *env, const MDBX_meta *meta, - MDBX_meta *dest) { - *dest = *meta; - return validate_meta(env, dest, data_page(meta), - bytes2pgno(env, ptr_dist(meta, env->me_map)), nullptr); -} + case MDBX_opt_gc_time_limit: + *pvalue = osal_monotime_to_16dot16(env->options.gc_time_limit); + break; -/* Read the environment parameters of a DB environment - * before mapping it into memory. */ -__cold static int read_header(MDBX_env *env, MDBX_meta *dest, - const int lck_exclusive, - const mdbx_mode_t mode_bits) { - memset(dest, 0, sizeof(MDBX_meta)); - int rc = osal_filesize(env->me_lazy_fd, &env->me_dxb_mmap.filesize); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; + case MDBX_opt_txn_dp_limit: + *pvalue = env->options.dp_limit; + break; + case MDBX_opt_txn_dp_initial: + *pvalue = env->options.dp_initial; + break; - unaligned_poke_u64(4, dest->mm_sign, MDBX_DATASIGN_WEAK); - rc = MDBX_CORRUPTED; + case MDBX_opt_spill_max_denominator: + *pvalue = env->options.spill_max_denominator; + break; + case MDBX_opt_spill_min_denominator: + *pvalue = env->options.spill_min_denominator; + break; + case MDBX_opt_spill_parent4child_denominator: + *pvalue = env->options.spill_parent4child_denominator; + break; - /* Read twice all meta pages so we can find the latest one. */ - unsigned loop_limit = NUM_METAS * 2; - /* We don't know the page size on first time. So, just guess it. */ - unsigned guess_pagesize = 0; - for (unsigned loop_count = 0; loop_count < loop_limit; ++loop_count) { - const unsigned meta_number = loop_count % NUM_METAS; - const unsigned offset = (guess_pagesize ? guess_pagesize - : (loop_count > NUM_METAS) ? env->me_psize - : env->me_os_psize) * - meta_number; + case MDBX_opt_loose_limit: + *pvalue = env->options.dp_loose_limit; + break; - char buffer[MIN_PAGESIZE]; - unsigned retryleft = 42; - while (1) { - TRACE("reading meta[%d]: offset %u, bytes %u, retry-left %u", meta_number, - offset, MIN_PAGESIZE, retryleft); - int err = osal_pread(env->me_lazy_fd, buffer, MIN_PAGESIZE, offset); - if (err == MDBX_ENODATA && offset == 0 && loop_count == 0 && - env->me_dxb_mmap.filesize == 0 && - mode_bits /* non-zero for DB creation */ != 0) { - NOTICE("read meta: empty file (%d, %s)", err, mdbx_strerror(err)); - return err; - } -#if defined(_WIN32) || defined(_WIN64) - if (err == ERROR_LOCK_VIOLATION) { - SleepEx(0, true); - err = osal_pread(env->me_lazy_fd, buffer, MIN_PAGESIZE, offset); - if (err == ERROR_LOCK_VIOLATION && --retryleft) { - WARNING("read meta[%u,%u]: %i, %s", offset, MIN_PAGESIZE, err, - mdbx_strerror(err)); - continue; - } - } -#endif /* Windows */ - if (err != MDBX_SUCCESS) { - ERROR("read meta[%u,%u]: %i, %s", offset, MIN_PAGESIZE, err, - mdbx_strerror(err)); - return err; - } + case MDBX_opt_merge_threshold_16dot16_percent: + *pvalue = env->options.merge_threshold_16dot16_percent; + break; - char again[MIN_PAGESIZE]; - err = osal_pread(env->me_lazy_fd, again, MIN_PAGESIZE, offset); + case MDBX_opt_writethrough_threshold: #if defined(_WIN32) || defined(_WIN64) - if (err == ERROR_LOCK_VIOLATION) { - SleepEx(0, true); - err = osal_pread(env->me_lazy_fd, again, MIN_PAGESIZE, offset); - if (err == ERROR_LOCK_VIOLATION && --retryleft) { - WARNING("read meta[%u,%u]: %i, %s", offset, MIN_PAGESIZE, err, - mdbx_strerror(err)); - continue; - } - } -#endif /* Windows */ - if (err != MDBX_SUCCESS) { - ERROR("read meta[%u,%u]: %i, %s", offset, MIN_PAGESIZE, err, - mdbx_strerror(err)); - return err; - } + *pvalue = (env->flags & MDBX_NOMETASYNC) ? 0 : INT_MAX; +#else + *pvalue = env->options.writethrough_threshold; +#endif + break; - if (memcmp(buffer, again, MIN_PAGESIZE) == 0 || --retryleft == 0) - break; + case MDBX_opt_prefault_write_enable: + *pvalue = env->options.prefault_write; + break; - VERBOSE("meta[%u] was updated, re-read it", meta_number); - } + case MDBX_opt_prefer_waf_insteadof_balance: + *pvalue = env->options.prefer_waf_insteadof_balance; + break; - if (!retryleft) { - ERROR("meta[%u] is too volatile, skip it", meta_number); - continue; - } + case MDBX_opt_subpage_limit: + *pvalue = env->options.subpage.limit; + break; - MDBX_page *const page = (MDBX_page *)buffer; - MDBX_meta *const meta = page_meta(page); - rc = validate_meta(env, meta, page, meta_number, &guess_pagesize); - if (rc != MDBX_SUCCESS) - continue; + case MDBX_opt_subpage_room_threshold: + *pvalue = env->options.subpage.room_threshold; + break; - bool latch; - if (env->me_stuck_meta >= 0) - latch = (meta_number == (unsigned)env->me_stuck_meta); - else if (meta_bootid_match(meta)) - latch = meta_choice_recent( - meta->unsafe_txnid, SIGN_IS_STEADY(meta->unsafe_sign), - dest->unsafe_txnid, SIGN_IS_STEADY(dest->unsafe_sign)); - else - latch = meta_choice_steady( - meta->unsafe_txnid, SIGN_IS_STEADY(meta->unsafe_sign), - dest->unsafe_txnid, SIGN_IS_STEADY(dest->unsafe_sign)); - if (latch) { - *dest = *meta; - if (!lck_exclusive && !META_IS_STEADY(dest)) - loop_limit += 1; /* LY: should re-read to hush race with update */ - VERBOSE("latch meta[%u]", meta_number); - } - } + case MDBX_opt_subpage_reserve_prereq: + *pvalue = env->options.subpage.reserve_prereq; + break; - if (dest->mm_psize == 0 || - (env->me_stuck_meta < 0 && - !(META_IS_STEADY(dest) || - meta_weak_acceptable(env, dest, lck_exclusive)))) { - ERROR("%s", "no usable meta-pages, database is corrupted"); - if (rc == MDBX_SUCCESS) { - /* TODO: try to restore the database by fully checking b-tree structure - * for the each meta page, if the corresponding option was given */ - return MDBX_CORRUPTED; - } - return rc; + case MDBX_opt_subpage_reserve_limit: + *pvalue = env->options.subpage.reserve_limit; + break; + + default: + return LOG_IFERR(MDBX_EINVAL); } return MDBX_SUCCESS; } +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 -__cold static MDBX_page *meta_model(const MDBX_env *env, MDBX_page *model, - size_t num) { - ENSURE(env, is_powerof2(env->me_psize)); - ENSURE(env, env->me_psize >= MIN_PAGESIZE); - ENSURE(env, env->me_psize <= MAX_PAGESIZE); - ENSURE(env, env->me_dbgeo.lower >= MIN_MAPSIZE); - ENSURE(env, env->me_dbgeo.upper <= MAX_MAPSIZE); - ENSURE(env, env->me_dbgeo.now >= env->me_dbgeo.lower); - ENSURE(env, env->me_dbgeo.now <= env->me_dbgeo.upper); - - memset(model, 0, env->me_psize); - model->mp_pgno = (pgno_t)num; - model->mp_flags = P_META; - MDBX_meta *const model_meta = page_meta(model); - unaligned_poke_u64(4, model_meta->mm_magic_and_version, MDBX_DATA_MAGIC); - - model_meta->mm_geo.lower = bytes2pgno(env, env->me_dbgeo.lower); - model_meta->mm_geo.upper = bytes2pgno(env, env->me_dbgeo.upper); - model_meta->mm_geo.grow_pv = pages2pv(bytes2pgno(env, env->me_dbgeo.grow)); - model_meta->mm_geo.shrink_pv = - pages2pv(bytes2pgno(env, env->me_dbgeo.shrink)); - model_meta->mm_geo.now = bytes2pgno(env, env->me_dbgeo.now); - model_meta->mm_geo.next = NUM_METAS; - - ENSURE(env, model_meta->mm_geo.lower >= MIN_PAGENO); - ENSURE(env, model_meta->mm_geo.upper <= MAX_PAGENO + 1); - ENSURE(env, model_meta->mm_geo.now >= model_meta->mm_geo.lower); - ENSURE(env, model_meta->mm_geo.now <= model_meta->mm_geo.upper); - ENSURE(env, model_meta->mm_geo.next >= MIN_PAGENO); - ENSURE(env, model_meta->mm_geo.next <= model_meta->mm_geo.now); - ENSURE(env, model_meta->mm_geo.grow_pv == - pages2pv(pv2pages(model_meta->mm_geo.grow_pv))); - ENSURE(env, model_meta->mm_geo.shrink_pv == - pages2pv(pv2pages(model_meta->mm_geo.shrink_pv))); - - model_meta->mm_psize = env->me_psize; - model_meta->mm_dbs[FREE_DBI].md_flags = MDBX_INTEGERKEY; - model_meta->mm_dbs[FREE_DBI].md_root = P_INVALID; - model_meta->mm_dbs[MAIN_DBI].md_root = P_INVALID; - meta_set_txnid(env, model_meta, MIN_TXNID + num); - unaligned_poke_u64(4, model_meta->mm_sign, meta_sign(model_meta)); - eASSERT(env, check_meta_coherency(env, model_meta, true)); - return ptr_disp(model, env->me_psize); -} - -/* Fill in most of the zeroed meta-pages for an empty database environment. - * Return pointer to recently (head) meta-page. */ -__cold static MDBX_meta *init_metas(const MDBX_env *env, void *buffer) { - MDBX_page *page0 = (MDBX_page *)buffer; - MDBX_page *page1 = meta_model(env, page0, 0); - MDBX_page *page2 = meta_model(env, page1, 1); - meta_model(env, page2, 2); - return page_meta(page2); -} +typedef struct diff_result { + ptrdiff_t diff; + intptr_t level; + ptrdiff_t root_nkeys; +} diff_t; -static int sync_locked(MDBX_env *env, unsigned flags, MDBX_meta *const pending, - meta_troika_t *const troika) { - eASSERT(env, ((env->me_flags ^ flags) & MDBX_WRITEMAP) == 0); - const MDBX_meta *const meta0 = METAPAGE(env, 0); - const MDBX_meta *const meta1 = METAPAGE(env, 1); - const MDBX_meta *const meta2 = METAPAGE(env, 2); - const meta_ptr_t head = meta_recent(env, troika); - int rc; +/* calculates: r = x - y */ +__hot static int cursor_diff(const MDBX_cursor *const __restrict x, const MDBX_cursor *const __restrict y, + diff_t *const __restrict r) { + r->diff = 0; + r->level = 0; + r->root_nkeys = 0; + + int rc = check_txn(x->txn, MDBX_TXN_BLOCKED); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; - eASSERT(env, - pending < METAPAGE(env, 0) || pending > METAPAGE(env, NUM_METAS)); - eASSERT(env, (env->me_flags & (MDBX_RDONLY | MDBX_FATAL_ERROR)) == 0); - eASSERT(env, pending->mm_geo.next <= pending->mm_geo.now); + if (unlikely(x->txn != y->txn)) + return MDBX_BAD_TXN; - if (flags & MDBX_SAFE_NOSYNC) { - /* Check auto-sync conditions */ - const pgno_t autosync_threshold = - atomic_load32(&env->me_lck->mti_autosync_threshold, mo_Relaxed); - const uint64_t autosync_period = - atomic_load64(&env->me_lck->mti_autosync_period, mo_Relaxed); - uint64_t eoos_timestamp; - if ((autosync_threshold && - atomic_load64(&env->me_lck->mti_unsynced_pages, mo_Relaxed) >= - autosync_threshold) || - (autosync_period && - (eoos_timestamp = - atomic_load64(&env->me_lck->mti_eoos_timestamp, mo_Relaxed)) && - osal_monotime() - eoos_timestamp >= autosync_period)) - flags &= MDBX_WRITEMAP | MDBX_SHRINK_ALLOWED; /* force steady */ + if (unlikely(y->dbi_state != x->dbi_state)) + return MDBX_EINVAL; + + const intptr_t depth = (x->top < y->top) ? x->top : y->top; + if (unlikely(depth < 0)) + return MDBX_ENODATA; + + r->root_nkeys = page_numkeys(x->pg[0]); + intptr_t nkeys = r->root_nkeys; + for (;;) { + if (unlikely(y->pg[r->level] != x->pg[r->level])) { + ERROR("Mismatch cursors's pages at %zu level", r->level); + return MDBX_PROBLEM; + } + r->diff = x->ki[r->level] - y->ki[r->level]; + if (r->diff) + break; + r->level += 1; + if (r->level > depth) { + r->diff = CMP2INT(x->flags & z_eof_hard, y->flags & z_eof_hard); + return MDBX_SUCCESS; + } + nkeys = page_numkeys(x->pg[r->level]); } - pgno_t shrink = 0; - if (flags & MDBX_SHRINK_ALLOWED) { - const size_t prev_discarded_pgno = - atomic_load32(&env->me_lck->mti_discarded_tail, mo_Relaxed); - if (prev_discarded_pgno < pending->mm_geo.next) - env->me_lck->mti_discarded_tail.weak = pending->mm_geo.next; - else if (prev_discarded_pgno >= - pending->mm_geo.next + env->me_madv_threshold) { - /* LY: check conditions to discard unused pages */ - const pgno_t largest_pgno = find_largest_snapshot( - env, (head.ptr_c->mm_geo.next > pending->mm_geo.next) - ? head.ptr_c->mm_geo.next - : pending->mm_geo.next); - eASSERT(env, largest_pgno >= NUM_METAS); + while (unlikely(r->diff == 1) && likely(r->level < depth)) { + r->level += 1; + /* DB'PAGEs: 0------------------>MAX + * + * CURSORs: y < x + * STACK[i ]: | + * STACK[+1]: ...y++N|0++x... + */ + nkeys = page_numkeys(y->pg[r->level]); + r->diff = (nkeys - y->ki[r->level]) + x->ki[r->level]; + assert(r->diff > 0); + } -#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__) - const pgno_t edge = env->me_poison_edge; - if (edge > largest_pgno) { - env->me_poison_edge = largest_pgno; - VALGRIND_MAKE_MEM_NOACCESS( - ptr_disp(env->me_map, pgno2bytes(env, largest_pgno)), - pgno2bytes(env, edge - largest_pgno)); - MDBX_ASAN_POISON_MEMORY_REGION( - ptr_disp(env->me_map, pgno2bytes(env, largest_pgno)), - pgno2bytes(env, edge - largest_pgno)); - } -#endif /* MDBX_USE_VALGRIND || __SANITIZE_ADDRESS__ */ - -#if MDBX_ENABLE_MADVISE && \ - (defined(MADV_DONTNEED) || defined(POSIX_MADV_DONTNEED)) - const size_t discard_edge_pgno = pgno_align2os_pgno(env, largest_pgno); - if (prev_discarded_pgno >= discard_edge_pgno + env->me_madv_threshold) { - const size_t prev_discarded_bytes = - pgno_align2os_bytes(env, prev_discarded_pgno); - const size_t discard_edge_bytes = pgno2bytes(env, discard_edge_pgno); - /* из-за выравнивания prev_discarded_bytes и discard_edge_bytes - * могут быть равны */ - if (prev_discarded_bytes > discard_edge_bytes) { - NOTICE("shrink-MADV_%s %zu..%zu", "DONTNEED", discard_edge_pgno, - prev_discarded_pgno); - munlock_after(env, discard_edge_pgno, - bytes_align2os_bytes(env, env->me_dxb_mmap.current)); - const uint32_t munlocks_before = - atomic_load32(&env->me_lck->mti_mlcnt[1], mo_Relaxed); -#if defined(MADV_DONTNEED) - int advise = MADV_DONTNEED; -#if defined(MADV_FREE) && \ - 0 /* MADV_FREE works for only anonymous vma at the moment */ - if ((env->me_flags & MDBX_WRITEMAP) && - linux_kernel_version > 0x04050000) - advise = MADV_FREE; -#endif /* MADV_FREE */ - int err = madvise(ptr_disp(env->me_map, discard_edge_bytes), - prev_discarded_bytes - discard_edge_bytes, advise) - ? ignore_enosys(errno) - : MDBX_SUCCESS; -#else - int err = ignore_enosys(posix_madvise( - ptr_disp(env->me_map, discard_edge_bytes), - prev_discarded_bytes - discard_edge_bytes, POSIX_MADV_DONTNEED)); -#endif - if (unlikely(MDBX_IS_ERROR(err))) { - const uint32_t mlocks_after = - atomic_load32(&env->me_lck->mti_mlcnt[0], mo_Relaxed); - if (err == MDBX_EINVAL) { - const int severity = (mlocks_after - munlocks_before) - ? MDBX_LOG_NOTICE - : MDBX_LOG_WARN; - if (LOG_ENABLED(severity)) - debug_log( - severity, __func__, __LINE__, - "%s-madvise: ignore EINVAL (%d) since some pages maybe " - "locked (%u/%u mlcnt-processes)", - "shrink", err, mlocks_after, munlocks_before); - } else { - ERROR("%s-madvise(%s, %zu, +%zu), %u/%u mlcnt-processes, err %d", - "shrink", "DONTNEED", discard_edge_bytes, - prev_discarded_bytes - discard_edge_bytes, mlocks_after, - munlocks_before, err); - return err; - } - } else - env->me_lck->mti_discarded_tail.weak = discard_edge_pgno; - } - } -#endif /* MDBX_ENABLE_MADVISE && (MADV_DONTNEED || POSIX_MADV_DONTNEED) */ - - /* LY: check conditions to shrink datafile */ - const pgno_t backlog_gap = 3 + pending->mm_dbs[FREE_DBI].md_depth * 3; - pgno_t shrink_step = 0; - if (pending->mm_geo.shrink_pv && - pending->mm_geo.now - pending->mm_geo.next > - (shrink_step = pv2pages(pending->mm_geo.shrink_pv)) + - backlog_gap) { - if (pending->mm_geo.now > largest_pgno && - pending->mm_geo.now - largest_pgno > shrink_step + backlog_gap) { - const pgno_t aligner = - pending->mm_geo.grow_pv - ? /* grow_step */ pv2pages(pending->mm_geo.grow_pv) - : shrink_step; - const pgno_t with_backlog_gap = largest_pgno + backlog_gap; - const pgno_t aligned = - pgno_align2os_pgno(env, (size_t)with_backlog_gap + aligner - - with_backlog_gap % aligner); - const pgno_t bottom = (aligned > pending->mm_geo.lower) - ? aligned - : pending->mm_geo.lower; - if (pending->mm_geo.now > bottom) { - if (TROIKA_HAVE_STEADY(troika)) - /* force steady, but only if steady-checkpoint is present */ - flags &= MDBX_WRITEMAP | MDBX_SHRINK_ALLOWED; - shrink = pending->mm_geo.now - bottom; - pending->mm_geo.now = bottom; - if (unlikely(head.txnid == pending->unsafe_txnid)) { - const txnid_t txnid = safe64_txnid_next(pending->unsafe_txnid); - NOTICE("force-forward pending-txn %" PRIaTXN " -> %" PRIaTXN, - pending->unsafe_txnid, txnid); - ENSURE(env, !env->me_txn0 || - (env->me_txn0->mt_owner != osal_thread_self() && - !env->me_txn)); - if (unlikely(txnid > MAX_TXNID)) { - rc = MDBX_TXN_FULL; - ERROR("txnid overflow, raise %d", rc); - goto fail; - } - meta_set_txnid(env, pending, txnid); - eASSERT(env, check_meta_coherency(env, pending, true)); - } - } - } - } - } - } - - /* LY: step#1 - sync previously written/updated data-pages */ - rc = MDBX_RESULT_FALSE /* carry steady */; - if (atomic_load64(&env->me_lck->mti_unsynced_pages, mo_Relaxed)) { - eASSERT(env, ((flags ^ env->me_flags) & MDBX_WRITEMAP) == 0); - enum osal_syncmode_bits mode_bits = MDBX_SYNC_NONE; - unsigned sync_op = 0; - if ((flags & MDBX_SAFE_NOSYNC) == 0) { - sync_op = 1; - mode_bits = MDBX_SYNC_DATA; - if (pending->mm_geo.next > - meta_prefer_steady(env, troika).ptr_c->mm_geo.now) - mode_bits |= MDBX_SYNC_SIZE; - if (flags & MDBX_NOMETASYNC) - mode_bits |= MDBX_SYNC_IODQ; - } else if (unlikely(env->me_incore)) - goto skip_incore_sync; - if (flags & MDBX_WRITEMAP) { -#if MDBX_ENABLE_PGOP_STAT - env->me_lck->mti_pgop_stat.msync.weak += sync_op; -#else - (void)sync_op; -#endif /* MDBX_ENABLE_PGOP_STAT */ - rc = - osal_msync(&env->me_dxb_mmap, 0, - pgno_align2os_bytes(env, pending->mm_geo.next), mode_bits); - } else { -#if MDBX_ENABLE_PGOP_STAT - env->me_lck->mti_pgop_stat.fsync.weak += sync_op; -#else - (void)sync_op; -#endif /* MDBX_ENABLE_PGOP_STAT */ - rc = osal_fsync(env->me_lazy_fd, mode_bits); - } - if (unlikely(rc != MDBX_SUCCESS)) - goto fail; - rc = (flags & MDBX_SAFE_NOSYNC) ? MDBX_RESULT_TRUE /* carry non-steady */ - : MDBX_RESULT_FALSE /* carry steady */; + while (unlikely(r->diff == -1) && likely(r->level < depth)) { + r->level += 1; + /* DB'PAGEs: 0------------------>MAX + * + * CURSORs: x < y + * STACK[i ]: | + * STACK[+1]: ...x--N|0--y... + */ + nkeys = page_numkeys(x->pg[r->level]); + r->diff = -(nkeys - x->ki[r->level]) - y->ki[r->level]; + assert(r->diff < 0); } - eASSERT(env, check_meta_coherency(env, pending, true)); - /* Steady or Weak */ - if (rc == MDBX_RESULT_FALSE /* carry steady */) { - unaligned_poke_u64(4, pending->mm_sign, meta_sign(pending)); - atomic_store64(&env->me_lck->mti_eoos_timestamp, 0, mo_Relaxed); - atomic_store64(&env->me_lck->mti_unsynced_pages, 0, mo_Relaxed); - } else { - assert(rc == MDBX_RESULT_TRUE /* carry non-steady */); - skip_incore_sync: - eASSERT(env, env->me_lck->mti_unsynced_pages.weak > 0); - /* Может быть нулевым если unsynced_pages > 0 в результате спиллинга. - * eASSERT(env, env->me_lck->mti_eoos_timestamp.weak != 0); */ - unaligned_poke_u64(4, pending->mm_sign, MDBX_DATASIGN_WEAK); - } - - const bool legal4overwrite = - head.txnid == pending->unsafe_txnid && - memcmp(&head.ptr_c->mm_dbs, &pending->mm_dbs, sizeof(pending->mm_dbs)) == - 0 && - memcmp(&head.ptr_c->mm_canary, &pending->mm_canary, - sizeof(pending->mm_canary)) == 0 && - memcmp(&head.ptr_c->mm_geo, &pending->mm_geo, sizeof(pending->mm_geo)) == - 0; - MDBX_meta *target = nullptr; - if (head.txnid == pending->unsafe_txnid) { - ENSURE(env, legal4overwrite); - if (!head.is_steady && META_IS_STEADY(pending)) - target = (MDBX_meta *)head.ptr_c; - else { - WARNING("%s", "skip update meta"); - return MDBX_SUCCESS; - } - } else { - const unsigned troika_tail = troika->tail_and_flags & 3; - ENSURE(env, troika_tail < NUM_METAS && troika_tail != troika->recent && - troika_tail != troika->prefer_steady); - target = (MDBX_meta *)meta_tail(env, troika).ptr_c; - } + return MDBX_SUCCESS; +} - /* LY: step#2 - update meta-page. */ - DEBUG("writing meta%" PRIaPGNO " = root %" PRIaPGNO "/%" PRIaPGNO - ", geo %" PRIaPGNO "/%" PRIaPGNO "-%" PRIaPGNO "/%" PRIaPGNO - " +%u -%u, txn_id %" PRIaTXN ", %s", - data_page(target)->mp_pgno, pending->mm_dbs[MAIN_DBI].md_root, - pending->mm_dbs[FREE_DBI].md_root, pending->mm_geo.lower, - pending->mm_geo.next, pending->mm_geo.now, pending->mm_geo.upper, - pv2pages(pending->mm_geo.grow_pv), pv2pages(pending->mm_geo.shrink_pv), - pending->unsafe_txnid, durable_caption(pending)); +__hot static ptrdiff_t estimate(const tree_t *tree, diff_t *const __restrict dr) { + /* root: branch-page => scale = leaf-factor * branch-factor^(N-1) + * level-1: branch-page(s) => scale = leaf-factor * branch-factor^2 + * level-2: branch-page(s) => scale = leaf-factor * branch-factor + * level-N: branch-page(s) => scale = leaf-factor + * leaf-level: leaf-page(s) => scale = 1 + */ + ptrdiff_t btree_power = (ptrdiff_t)tree->height - 2 - (ptrdiff_t)dr->level; + if (btree_power < 0) + return dr->diff; - DEBUG("meta0: %s, %s, txn_id %" PRIaTXN ", root %" PRIaPGNO "/%" PRIaPGNO, - (meta0 == head.ptr_c) ? "head" - : (meta0 == target) ? "tail" - : "stay", - durable_caption(meta0), constmeta_txnid(meta0), - meta0->mm_dbs[MAIN_DBI].md_root, meta0->mm_dbs[FREE_DBI].md_root); - DEBUG("meta1: %s, %s, txn_id %" PRIaTXN ", root %" PRIaPGNO "/%" PRIaPGNO, - (meta1 == head.ptr_c) ? "head" - : (meta1 == target) ? "tail" - : "stay", - durable_caption(meta1), constmeta_txnid(meta1), - meta1->mm_dbs[MAIN_DBI].md_root, meta1->mm_dbs[FREE_DBI].md_root); - DEBUG("meta2: %s, %s, txn_id %" PRIaTXN ", root %" PRIaPGNO "/%" PRIaPGNO, - (meta2 == head.ptr_c) ? "head" - : (meta2 == target) ? "tail" - : "stay", - durable_caption(meta2), constmeta_txnid(meta2), - meta2->mm_dbs[MAIN_DBI].md_root, meta2->mm_dbs[FREE_DBI].md_root); - - eASSERT(env, pending->unsafe_txnid != constmeta_txnid(meta0) || - (META_IS_STEADY(pending) && !META_IS_STEADY(meta0))); - eASSERT(env, pending->unsafe_txnid != constmeta_txnid(meta1) || - (META_IS_STEADY(pending) && !META_IS_STEADY(meta1))); - eASSERT(env, pending->unsafe_txnid != constmeta_txnid(meta2) || - (META_IS_STEADY(pending) && !META_IS_STEADY(meta2))); - - eASSERT(env, ((env->me_flags ^ flags) & MDBX_WRITEMAP) == 0); - ENSURE(env, target == head.ptr_c || - constmeta_txnid(target) < pending->unsafe_txnid); - if (flags & MDBX_WRITEMAP) { - jitter4testing(true); - if (likely(target != head.ptr_c)) { - /* LY: 'invalidate' the meta. */ - meta_update_begin(env, target, pending->unsafe_txnid); - unaligned_poke_u64(4, target->mm_sign, MDBX_DATASIGN_WEAK); -#ifndef NDEBUG - /* debug: provoke failure to catch a violators, but don't touch mm_psize - * to allow readers catch actual pagesize. */ - void *provoke_begin = &target->mm_dbs[FREE_DBI].md_root; - void *provoke_end = &target->mm_sign; - memset(provoke_begin, 0xCC, ptr_dist(provoke_end, provoke_begin)); - jitter4testing(false); -#endif + ptrdiff_t estimated = (ptrdiff_t)tree->items * dr->diff / (ptrdiff_t)tree->leaf_pages; + if (btree_power == 0) + return estimated; - /* LY: update info */ - target->mm_geo = pending->mm_geo; - target->mm_dbs[FREE_DBI] = pending->mm_dbs[FREE_DBI]; - target->mm_dbs[MAIN_DBI] = pending->mm_dbs[MAIN_DBI]; - target->mm_canary = pending->mm_canary; - memcpy(target->mm_pages_retired, pending->mm_pages_retired, 8); - jitter4testing(true); + if (tree->height < 4) { + assert(dr->level == 0 && btree_power == 1); + return (ptrdiff_t)tree->items * dr->diff / (ptrdiff_t)dr->root_nkeys; + } - /* LY: 'commit' the meta */ - meta_update_end(env, target, unaligned_peek_u64(4, pending->mm_txnid_b)); - jitter4testing(true); - eASSERT(env, check_meta_coherency(env, target, true)); - } else { - /* dangerous case (target == head), only mm_sign could - * me updated, check assertions once again */ - eASSERT(env, - legal4overwrite && !head.is_steady && META_IS_STEADY(pending)); - } - memcpy(target->mm_sign, pending->mm_sign, 8); - osal_flush_incoherent_cpu_writeback(); - jitter4testing(true); - if (!env->me_incore) { - if (!MDBX_AVOID_MSYNC) { - /* sync meta-pages */ -#if MDBX_ENABLE_PGOP_STAT - env->me_lck->mti_pgop_stat.msync.weak += 1; -#endif /* MDBX_ENABLE_PGOP_STAT */ - rc = osal_msync( - &env->me_dxb_mmap, 0, pgno_align2os_bytes(env, NUM_METAS), - (flags & MDBX_NOMETASYNC) ? MDBX_SYNC_NONE - : MDBX_SYNC_DATA | MDBX_SYNC_IODQ); - } else { -#if MDBX_ENABLE_PGOP_STAT - env->me_lck->mti_pgop_stat.wops.weak += 1; -#endif /* MDBX_ENABLE_PGOP_STAT */ - const MDBX_page *page = data_page(target); - rc = osal_pwrite(env->me_fd4meta, page, env->me_psize, - ptr_dist(page, env->me_map)); - if (likely(rc == MDBX_SUCCESS)) { - osal_flush_incoherent_mmap(target, sizeof(MDBX_meta), - env->me_os_psize); - if ((flags & MDBX_NOMETASYNC) == 0 && - env->me_fd4meta == env->me_lazy_fd) { -#if MDBX_ENABLE_PGOP_STAT - env->me_lck->mti_pgop_stat.fsync.weak += 1; -#endif /* MDBX_ENABLE_PGOP_STAT */ - rc = osal_fsync(env->me_lazy_fd, MDBX_SYNC_DATA | MDBX_SYNC_IODQ); - } - } - } - if (unlikely(rc != MDBX_SUCCESS)) - goto fail; - } - } else { -#if MDBX_ENABLE_PGOP_STAT - env->me_lck->mti_pgop_stat.wops.weak += 1; -#endif /* MDBX_ENABLE_PGOP_STAT */ - const MDBX_meta undo_meta = *target; - rc = osal_pwrite(env->me_fd4meta, pending, sizeof(MDBX_meta), - ptr_dist(target, env->me_map)); - if (unlikely(rc != MDBX_SUCCESS)) { - undo: - DEBUG("%s", "write failed, disk error?"); - /* On a failure, the pagecache still contains the new data. - * Try write some old data back, to prevent it from being used. */ - osal_pwrite(env->me_fd4meta, &undo_meta, sizeof(MDBX_meta), - ptr_dist(target, env->me_map)); - goto fail; + /* average_branchpage_fillfactor = total(branch_entries) / branch_pages + total(branch_entries) = leaf_pages + branch_pages - 1 (root page) */ + const size_t log2_fixedpoint = sizeof(size_t) - 1; + const size_t half = UINT64_C(1) << (log2_fixedpoint - 1); + const size_t factor = ((tree->leaf_pages + tree->branch_pages - 1) << log2_fixedpoint) / tree->branch_pages; + while (1) { + switch ((size_t)btree_power) { + default: { + const size_t square = (factor * factor + half) >> log2_fixedpoint; + const size_t quad = (square * square + half) >> log2_fixedpoint; + do { + estimated = estimated * quad + half; + estimated >>= log2_fixedpoint; + btree_power -= 4; + } while (btree_power >= 4); + continue; } - osal_flush_incoherent_mmap(target, sizeof(MDBX_meta), env->me_os_psize); - /* sync meta-pages */ - if ((flags & MDBX_NOMETASYNC) == 0 && env->me_fd4meta == env->me_lazy_fd && - !env->me_incore) { -#if MDBX_ENABLE_PGOP_STAT - env->me_lck->mti_pgop_stat.fsync.weak += 1; -#endif /* MDBX_ENABLE_PGOP_STAT */ - rc = osal_fsync(env->me_lazy_fd, MDBX_SYNC_DATA | MDBX_SYNC_IODQ); - if (rc != MDBX_SUCCESS) - goto undo; + case 3: + estimated = estimated * factor + half; + estimated >>= log2_fixedpoint; + __fallthrough /* fall through */; + case 2: + estimated = estimated * factor + half; + estimated >>= log2_fixedpoint; + __fallthrough /* fall through */; + case 1: + estimated = estimated * factor + half; + estimated >>= log2_fixedpoint; + __fallthrough /* fall through */; + case 0: + if (unlikely(estimated > (ptrdiff_t)tree->items)) + return (ptrdiff_t)tree->items; + if (unlikely(estimated < -(ptrdiff_t)tree->items)) + return -(ptrdiff_t)tree->items; + return estimated; } } +} - uint64_t timestamp = 0; - while ("workaround for https://libmdbx.dqdkfa.ru/dead-github/issues/269") { - rc = coherency_check_written(env, pending->unsafe_txnid, target, - bytes2pgno(env, ptr_dist(target, env->me_map)), - ×tamp); - if (likely(rc == MDBX_SUCCESS)) - break; - if (unlikely(rc != MDBX_RESULT_TRUE)) - goto fail; - } +/*------------------------------------------------------------------------------ + * Range-Estimation API */ - const uint32_t sync_txnid_dist = - ((flags & MDBX_NOMETASYNC) == 0) ? 0 - : ((flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC) - ? MDBX_NOMETASYNC_LAZY_FD - : MDBX_NOMETASYNC_LAZY_WRITEMAP; - env->me_lck->mti_meta_sync_txnid.weak = - pending->mm_txnid_a[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__].weak - - sync_txnid_dist; +__hot int mdbx_estimate_distance(const MDBX_cursor *first, const MDBX_cursor *last, ptrdiff_t *distance_items) { + if (unlikely(!distance_items)) + return LOG_IFERR(MDBX_EINVAL); - *troika = meta_tap(env); - for (MDBX_txn *txn = env->me_txn0; txn; txn = txn->mt_child) - if (troika != &txn->tw.troika) - txn->tw.troika = *troika; + int rc = cursor_check_pure(first); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); - /* LY: shrink datafile if needed */ - if (unlikely(shrink)) { - VERBOSE("shrink to %" PRIaPGNO " pages (-%" PRIaPGNO ")", - pending->mm_geo.now, shrink); - rc = dxb_resize(env, pending->mm_geo.next, pending->mm_geo.now, - pending->mm_geo.upper, impilict_shrink); - if (rc != MDBX_SUCCESS && rc != MDBX_EPERM) - goto fail; - eASSERT(env, check_meta_coherency(env, target, true)); + rc = cursor_check_pure(last); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); + + *distance_items = 0; + diff_t dr; + rc = cursor_diff(last, first, &dr); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); + + cASSERT(first, dr.diff || inner_pointed(first) == inner_pointed(last)); + if (unlikely(dr.diff == 0) && inner_pointed(first)) { + first = &first->subcur->cursor; + last = &last->subcur->cursor; + rc = cursor_diff(first, last, &dr); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); } - MDBX_lockinfo *const lck = env->me_lck_mmap.lck; - if (likely(lck)) - /* toggle oldest refresh */ - atomic_store32(&lck->mti_readers_refresh_flag, false, mo_Relaxed); + if (likely(dr.diff != 0)) + *distance_items = estimate(first->tree, &dr); return MDBX_SUCCESS; - -fail: - env->me_flags |= MDBX_FATAL_ERROR; - return rc; } -static void recalculate_merge_threshold(MDBX_env *env) { - const size_t bytes = page_space(env); - env->me_merge_threshold = - (uint16_t)(bytes - - (bytes * env->me_options.merge_threshold_16dot16_percent >> - 16)); - env->me_merge_threshold_gc = - (uint16_t)(bytes - - ((env->me_options.merge_threshold_16dot16_percent > 19005) - ? bytes / 3 /* 33 % */ - : bytes / 4 /* 25 % */)); -} - -__cold static void setup_pagesize(MDBX_env *env, const size_t pagesize) { - STATIC_ASSERT(PTRDIFF_MAX > MAX_MAPSIZE); - STATIC_ASSERT(MIN_PAGESIZE > sizeof(MDBX_page) + sizeof(MDBX_meta)); - ENSURE(env, is_powerof2(pagesize)); - ENSURE(env, pagesize >= MIN_PAGESIZE); - ENSURE(env, pagesize <= MAX_PAGESIZE); - env->me_psize = (unsigned)pagesize; - if (env->me_pbuf) { - osal_memalign_free(env->me_pbuf); - env->me_pbuf = nullptr; - } +__hot int mdbx_estimate_move(const MDBX_cursor *cursor, MDBX_val *key, MDBX_val *data, MDBX_cursor_op move_op, + ptrdiff_t *distance_items) { + if (unlikely(!distance_items || move_op == MDBX_GET_CURRENT || move_op == MDBX_GET_MULTIPLE)) + return LOG_IFERR(MDBX_EINVAL); - STATIC_ASSERT(MAX_GC1OVPAGE(MIN_PAGESIZE) > 4); - STATIC_ASSERT(MAX_GC1OVPAGE(MAX_PAGESIZE) < MDBX_PGL_LIMIT); - const intptr_t maxgc_ov1page = (pagesize - PAGEHDRSZ) / sizeof(pgno_t) - 1; - ENSURE(env, - maxgc_ov1page > 42 && maxgc_ov1page < (intptr_t)MDBX_PGL_LIMIT / 4); - env->me_maxgc_ov1page = (unsigned)maxgc_ov1page; - env->me_maxgc_per_branch = - (unsigned)((pagesize - PAGEHDRSZ) / - (sizeof(indx_t) + sizeof(MDBX_node) + sizeof(txnid_t))); - - STATIC_ASSERT(LEAF_NODE_MAX(MIN_PAGESIZE) > sizeof(MDBX_db) + NODESIZE + 42); - STATIC_ASSERT(LEAF_NODE_MAX(MAX_PAGESIZE) < UINT16_MAX); - STATIC_ASSERT(LEAF_NODE_MAX(MIN_PAGESIZE) >= BRANCH_NODE_MAX(MIN_PAGESIZE)); - STATIC_ASSERT(BRANCH_NODE_MAX(MAX_PAGESIZE) > NODESIZE + 42); - STATIC_ASSERT(BRANCH_NODE_MAX(MAX_PAGESIZE) < UINT16_MAX); - const intptr_t branch_nodemax = BRANCH_NODE_MAX(pagesize); - const intptr_t leaf_nodemax = LEAF_NODE_MAX(pagesize); - ENSURE(env, branch_nodemax > (intptr_t)(NODESIZE + 42) && - branch_nodemax % 2 == 0 && - leaf_nodemax > (intptr_t)(sizeof(MDBX_db) + NODESIZE + 42) && - leaf_nodemax >= branch_nodemax && - leaf_nodemax < (int)UINT16_MAX && leaf_nodemax % 2 == 0); - env->me_leaf_nodemax = (uint16_t)leaf_nodemax; - env->me_branch_nodemax = (uint16_t)branch_nodemax; - env->me_psize2log = (uint8_t)log2n_powerof2(pagesize); - eASSERT(env, pgno2bytes(env, 1) == pagesize); - eASSERT(env, bytes2pgno(env, pagesize + pagesize) == 2); - recalculate_merge_threshold(env); - - /* TODO: recalculate me_subpage_xyz values from MDBX_opt_subpage_xyz. */ - env->me_subpage_limit = env->me_leaf_nodemax - NODESIZE; - env->me_subpage_room_threshold = 0; - env->me_subpage_reserve_prereq = env->me_leaf_nodemax; - env->me_subpage_reserve_limit = env->me_subpage_limit / 42; - eASSERT(env, - env->me_subpage_reserve_prereq > - env->me_subpage_room_threshold + env->me_subpage_reserve_limit); - eASSERT(env, env->me_leaf_nodemax >= env->me_subpage_limit + NODESIZE); - - const pgno_t max_pgno = bytes2pgno(env, MAX_MAPSIZE); - if (!env->me_options.flags.non_auto.dp_limit) { - /* auto-setup dp_limit by "The42" ;-) */ - intptr_t total_ram_pages, avail_ram_pages; - int err = mdbx_get_sysraminfo(nullptr, &total_ram_pages, &avail_ram_pages); - if (unlikely(err != MDBX_SUCCESS)) - ERROR("mdbx_get_sysraminfo(), rc %d", err); - else { - size_t reasonable_dpl_limit = - (size_t)(total_ram_pages + avail_ram_pages) / 42; - if (pagesize > env->me_os_psize) - reasonable_dpl_limit /= pagesize / env->me_os_psize; - else if (pagesize < env->me_os_psize) - reasonable_dpl_limit *= env->me_os_psize / pagesize; - reasonable_dpl_limit = (reasonable_dpl_limit < MDBX_PGL_LIMIT) - ? reasonable_dpl_limit - : MDBX_PGL_LIMIT; - reasonable_dpl_limit = (reasonable_dpl_limit > CURSOR_STACK * 4) - ? reasonable_dpl_limit - : CURSOR_STACK * 4; - env->me_options.dp_limit = (unsigned)reasonable_dpl_limit; - } - } - if (env->me_options.dp_limit > max_pgno - NUM_METAS) - env->me_options.dp_limit = max_pgno - NUM_METAS; - if (env->me_options.dp_initial > env->me_options.dp_limit) - env->me_options.dp_initial = env->me_options.dp_limit; -} + int rc = cursor_check_ro(cursor); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); -__cold int mdbx_env_create(MDBX_env **penv) { - if (unlikely(!penv)) - return MDBX_EINVAL; - *penv = nullptr; + if (unlikely(!is_pointed(cursor))) + return LOG_IFERR(MDBX_ENODATA); -#ifdef MDBX_HAVE_C11ATOMICS - if (unlikely(!atomic_is_lock_free((const volatile uint32_t *)penv))) { - ERROR("lock-free atomic ops for %u-bit types is required", 32); - return MDBX_INCOMPATIBLE; - } -#if MDBX_64BIT_ATOMIC - if (unlikely(!atomic_is_lock_free((const volatile uint64_t *)penv))) { - ERROR("lock-free atomic ops for %u-bit types is required", 64); - return MDBX_INCOMPATIBLE; + cursor_couple_t next; + rc = cursor_init(&next.outer, cursor->txn, cursor_dbi(cursor)); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); + + cursor_cpstk(cursor, &next.outer); + if (cursor->tree->flags & MDBX_DUPSORT) { + subcur_t *mx = &container_of(cursor, cursor_couple_t, outer)->inner; + cursor_cpstk(&mx->cursor, &next.inner.cursor); } -#endif /* MDBX_64BIT_ATOMIC */ -#endif /* MDBX_HAVE_C11ATOMICS */ - const size_t os_psize = osal_syspagesize(); - if (unlikely(!is_powerof2(os_psize) || os_psize < MIN_PAGESIZE)) { - ERROR("unsuitable system pagesize %" PRIuPTR, os_psize); - return MDBX_INCOMPATIBLE; + MDBX_val stub_data; + if (data == nullptr) { + const unsigned mask = 1 << MDBX_GET_BOTH | 1 << MDBX_GET_BOTH_RANGE | 1 << MDBX_SET_KEY; + if (unlikely(mask & (1 << move_op))) + return LOG_IFERR(MDBX_EINVAL); + stub_data.iov_base = nullptr; + stub_data.iov_len = 0; + data = &stub_data; } -#if defined(__linux__) || defined(__gnu_linux__) - if (unlikely(linux_kernel_version < 0x04000000)) { - /* 2022-09-01: Прошло уже больше двух после окончания какой-либо поддержки - * самого "долгоиграющего" ядра 3.16.85 ветки 3.x */ - ERROR("too old linux kernel %u.%u.%u.%u, the >= 4.0.0 is required", - linux_kernel_version >> 24, (linux_kernel_version >> 16) & 255, - (linux_kernel_version >> 8) & 255, linux_kernel_version & 255); - return MDBX_INCOMPATIBLE; + MDBX_val stub_key; + if (key == nullptr) { + const unsigned mask = + 1 << MDBX_GET_BOTH | 1 << MDBX_GET_BOTH_RANGE | 1 << MDBX_SET_KEY | 1 << MDBX_SET | 1 << MDBX_SET_RANGE; + if (unlikely(mask & (1 << move_op))) + return LOG_IFERR(MDBX_EINVAL); + stub_key.iov_base = nullptr; + stub_key.iov_len = 0; + key = &stub_key; } -#endif /* Linux */ - - MDBX_env *env = osal_calloc(1, sizeof(MDBX_env)); - if (unlikely(!env)) - return MDBX_ENOMEM; - - env->me_maxreaders = DEFAULT_READERS; - env->me_maxdbs = env->me_numdbs = CORE_DBS; - env->me_lazy_fd = env->me_dsync_fd = env->me_fd4meta = env->me_lfd = - INVALID_HANDLE_VALUE; - env->me_pid = osal_getpid(); - env->me_stuck_meta = -1; - - env->me_options.rp_augment_limit = MDBX_PNL_INITIAL; - env->me_options.dp_reserve_limit = MDBX_PNL_INITIAL; - env->me_options.dp_initial = MDBX_PNL_INITIAL; - env->me_options.spill_max_denominator = 8; - env->me_options.spill_min_denominator = 8; - env->me_options.spill_parent4child_denominator = 0; - env->me_options.dp_loose_limit = 64; - env->me_options.merge_threshold_16dot16_percent = 65536 / 4 /* 25% */; - -#if !(defined(_WIN32) || defined(_WIN64)) - env->me_options.writethrough_threshold = -#if defined(__linux__) || defined(__gnu_linux__) - mdbx_RunningOnWSL1 ? MAX_PAGENO : -#endif /* Linux */ - MDBX_WRITETHROUGH_THRESHOLD_DEFAULT; -#endif /* Windows */ - - env->me_os_psize = (unsigned)os_psize; - setup_pagesize(env, (env->me_os_psize < MAX_PAGESIZE) ? env->me_os_psize - : MAX_PAGESIZE); - - int rc = osal_fastmutex_init(&env->me_dbi_lock); - if (unlikely(rc != MDBX_SUCCESS)) - goto bailout; -#if defined(_WIN32) || defined(_WIN64) - osal_srwlock_Init(&env->me_remap_guard); - InitializeCriticalSection(&env->me_windowsbug_lock); -#else - rc = osal_fastmutex_init(&env->me_remap_guard); - if (unlikely(rc != MDBX_SUCCESS)) { - osal_fastmutex_destroy(&env->me_dbi_lock); - goto bailout; - } + next.outer.signature = cur_signature_live; + rc = cursor_ops(&next.outer, key, data, move_op); + if (unlikely(rc != MDBX_SUCCESS && (rc != MDBX_NOTFOUND || !is_pointed(&next.outer)))) + return LOG_IFERR(rc); -#if MDBX_LOCKING > MDBX_LOCKING_SYSV - MDBX_lockinfo *const stub = lckless_stub(env); - rc = osal_ipclock_stub(&stub->mti_wlock); -#endif /* MDBX_LOCKING */ - if (unlikely(rc != MDBX_SUCCESS)) { - osal_fastmutex_destroy(&env->me_remap_guard); - osal_fastmutex_destroy(&env->me_dbi_lock); - goto bailout; + if (move_op == MDBX_LAST) { + next.outer.flags |= z_eof_hard; + next.inner.cursor.flags |= z_eof_hard; } -#endif /* Windows */ - - VALGRIND_CREATE_MEMPOOL(env, 0, 0); - env->me_signature.weak = MDBX_ME_SIGNATURE; - *penv = env; - return MDBX_SUCCESS; - -bailout: - osal_free(env); - return rc; + return mdbx_estimate_distance(cursor, &next.outer, distance_items); } -__cold static intptr_t get_reasonable_db_maxsize(intptr_t *cached_result) { - if (*cached_result == 0) { - intptr_t pagesize, total_ram_pages; - if (unlikely(mdbx_get_sysraminfo(&pagesize, &total_ram_pages, nullptr) != - MDBX_SUCCESS)) - return *cached_result = MAX_MAPSIZE32 /* the 32-bit limit is good enough - for fallback */ - ; +__hot int mdbx_estimate_range(const MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *begin_key, const MDBX_val *begin_data, + const MDBX_val *end_key, const MDBX_val *end_data, ptrdiff_t *size_items) { + if (unlikely(!size_items)) + return LOG_IFERR(MDBX_EINVAL); - if (unlikely((size_t)total_ram_pages * 2 > MAX_MAPSIZE / (size_t)pagesize)) - return *cached_result = MAX_MAPSIZE; - assert(MAX_MAPSIZE >= (size_t)(total_ram_pages * pagesize * 2)); + if (unlikely(begin_data && (begin_key == nullptr || begin_key == MDBX_EPSILON))) + return LOG_IFERR(MDBX_EINVAL); - /* Suggesting should not be more than golden ratio of the size of RAM. */ - *cached_result = (intptr_t)((size_t)total_ram_pages * 207 >> 7) * pagesize; + if (unlikely(end_data && (end_key == nullptr || end_key == MDBX_EPSILON))) + return LOG_IFERR(MDBX_EINVAL); - /* Round to the nearest human-readable granulation. */ - for (size_t unit = MEGABYTE; unit; unit <<= 5) { - const size_t floor = floor_powerof2(*cached_result, unit); - const size_t ceil = ceil_powerof2(*cached_result, unit); - const size_t threshold = (size_t)*cached_result >> 4; - const bool down = - *cached_result - floor < ceil - *cached_result || ceil > MAX_MAPSIZE; - if (threshold < (down ? *cached_result - floor : ceil - *cached_result)) - break; - *cached_result = down ? floor : ceil; - } - } - return *cached_result; -} + if (unlikely(begin_key == MDBX_EPSILON && end_key == MDBX_EPSILON)) + return LOG_IFERR(MDBX_EINVAL); -__cold int mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower, - intptr_t size_now, intptr_t size_upper, - intptr_t growth_step, - intptr_t shrink_threshold, intptr_t pagesize) { - int rc = check_env(env, false); + int rc = check_txn(txn, MDBX_TXN_BLOCKED); if (unlikely(rc != MDBX_SUCCESS)) - return rc; + return LOG_IFERR(rc); - const bool inside_txn = - (env->me_txn0 && env->me_txn0->mt_owner == osal_thread_self()); + cursor_couple_t begin; + /* LY: first, initialize cursor to refresh a DB in case it have DB_STALE */ + rc = cursor_init(&begin.outer, txn, dbi); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); -#if MDBX_DEBUG - if (growth_step < 0) { - growth_step = 1; - if (shrink_threshold < 0) - shrink_threshold = 1; + if (unlikely(begin.outer.tree->items == 0)) { + *size_items = 0; + return MDBX_SUCCESS; } -#endif /* MDBX_DEBUG */ - - intptr_t reasonable_maxsize = 0; - bool need_unlock = false; - if (env->me_map) { - /* env already mapped */ - if (unlikely(env->me_flags & MDBX_RDONLY)) - return MDBX_EACCESS; - - if (!inside_txn) { - int err = mdbx_txn_lock(env, false); - if (unlikely(err != MDBX_SUCCESS)) - return err; - need_unlock = true; - env->me_txn0->tw.troika = meta_tap(env); - eASSERT(env, !env->me_txn && !env->me_txn0->mt_child); - env->me_txn0->mt_txnid = - env->me_txn0->tw.troika.txnid[env->me_txn0->tw.troika.recent]; - txn_oldest_reader(env->me_txn0); - } - - /* get untouched params from current TXN or DB */ - if (pagesize <= 0 || pagesize >= INT_MAX) - pagesize = env->me_psize; - const MDBX_geo *const geo = - inside_txn ? &env->me_txn->mt_geo - : &meta_recent(env, &env->me_txn0->tw.troika).ptr_c->mm_geo; - if (size_lower < 0) - size_lower = pgno2bytes(env, geo->lower); - if (size_now < 0) - size_now = pgno2bytes(env, geo->now); - if (size_upper < 0) - size_upper = pgno2bytes(env, geo->upper); - if (growth_step < 0) - growth_step = pgno2bytes(env, pv2pages(geo->grow_pv)); - if (shrink_threshold < 0) - shrink_threshold = pgno2bytes(env, pv2pages(geo->shrink_pv)); - if (pagesize != (intptr_t)env->me_psize) { - rc = MDBX_EINVAL; - goto bailout; + if (!begin_key) { + if (unlikely(!end_key)) { + /* LY: FIRST..LAST case */ + *size_items = (ptrdiff_t)begin.outer.tree->items; + return MDBX_SUCCESS; } - const size_t usedbytes = - pgno2bytes(env, find_largest_snapshot(env, geo->next)); - if ((size_t)size_upper < usedbytes) { - rc = MDBX_MAP_FULL; - goto bailout; + rc = outer_first(&begin.outer, nullptr, nullptr); + if (unlikely(end_key == MDBX_EPSILON)) { + /* LY: FIRST..+epsilon case */ + return LOG_IFERR((rc == MDBX_SUCCESS) ? mdbx_cursor_count(&begin.outer, (size_t *)size_items) : rc); } - if ((size_t)size_now < usedbytes) - size_now = usedbytes; } else { - /* env NOT yet mapped */ - if (unlikely(inside_txn)) - return MDBX_PANIC; - - /* is requested some auto-value for pagesize ? */ - if (pagesize >= INT_MAX /* maximal */) - pagesize = MAX_PAGESIZE; - else if (pagesize <= 0) { - if (pagesize < 0 /* default */) { - pagesize = env->me_os_psize; - if ((uintptr_t)pagesize > MAX_PAGESIZE) - pagesize = MAX_PAGESIZE; - eASSERT(env, (uintptr_t)pagesize >= MIN_PAGESIZE); - } else if (pagesize == 0 /* minimal */) - pagesize = MIN_PAGESIZE; + if (unlikely(begin_key == MDBX_EPSILON)) { + if (end_key == nullptr) { + /* LY: -epsilon..LAST case */ + rc = outer_last(&begin.outer, nullptr, nullptr); + return LOG_IFERR((rc == MDBX_SUCCESS) ? mdbx_cursor_count(&begin.outer, (size_t *)size_items) : rc); + } + /* LY: -epsilon..value case */ + assert(end_key != MDBX_EPSILON); + begin_key = end_key; + } else if (unlikely(end_key == MDBX_EPSILON)) { + /* LY: value..+epsilon case */ + assert(begin_key != MDBX_EPSILON); + end_key = begin_key; + } + if (end_key && !begin_data && !end_data && + (begin_key == end_key || begin.outer.clc->k.cmp(begin_key, end_key) == 0)) { + /* LY: single key case */ + rc = cursor_seek(&begin.outer, (MDBX_val *)begin_key, nullptr, MDBX_SET).err; + if (unlikely(rc != MDBX_SUCCESS)) { + *size_items = 0; + return LOG_IFERR((rc == MDBX_NOTFOUND) ? MDBX_SUCCESS : rc); + } + *size_items = 1; + if (inner_pointed(&begin.outer)) + *size_items = (sizeof(*size_items) >= sizeof(begin.inner.nested_tree.items) || + begin.inner.nested_tree.items <= PTRDIFF_MAX) + ? (size_t)begin.inner.nested_tree.items + : PTRDIFF_MAX; - /* choose pagesize */ - intptr_t max_size = (size_now > size_lower) ? size_now : size_lower; - max_size = (size_upper > max_size) ? size_upper : max_size; - if (max_size < 0 /* default */) - max_size = DEFAULT_MAPSIZE; - else if (max_size == 0 /* minimal */) - max_size = MIN_MAPSIZE; - else if (max_size >= (intptr_t)MAX_MAPSIZE /* maximal */) - max_size = get_reasonable_db_maxsize(&reasonable_maxsize); - - while (max_size > pagesize * (int64_t)(MAX_PAGENO + 1) && - pagesize < MAX_PAGESIZE) - pagesize <<= 1; + return MDBX_SUCCESS; + } else { + MDBX_val proxy_key = *begin_key; + MDBX_val proxy_data = {nullptr, 0}; + if (begin_data) + proxy_data = *begin_data; + rc = LOG_IFERR(cursor_seek(&begin.outer, &proxy_key, &proxy_data, MDBX_SET_LOWERBOUND).err); } } - if (pagesize < (intptr_t)MIN_PAGESIZE || pagesize > (intptr_t)MAX_PAGESIZE || - !is_powerof2(pagesize)) { - rc = MDBX_EINVAL; - goto bailout; + if (unlikely(rc != MDBX_SUCCESS)) { + if (rc != MDBX_NOTFOUND || !is_pointed(&begin.outer)) + return LOG_IFERR(rc); } - if (size_lower <= 0) { - size_lower = MIN_MAPSIZE; - if (MIN_MAPSIZE / pagesize < MIN_PAGENO) - size_lower = MIN_PAGENO * pagesize; + cursor_couple_t end; + rc = cursor_init(&end.outer, txn, dbi); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); + if (!end_key) { + rc = outer_last(&end.outer, nullptr, nullptr); + end.outer.flags |= z_eof_hard; + end.inner.cursor.flags |= z_eof_hard; + } else { + MDBX_val proxy_key = *end_key; + MDBX_val proxy_data = {nullptr, 0}; + if (end_data) + proxy_data = *end_data; + rc = cursor_seek(&end.outer, &proxy_key, &proxy_data, MDBX_SET_LOWERBOUND).err; } - if (size_lower >= INTPTR_MAX) { - size_lower = get_reasonable_db_maxsize(&reasonable_maxsize); - if ((size_t)size_lower / pagesize > MAX_PAGENO + 1) - size_lower = pagesize * (MAX_PAGENO + 1); + if (unlikely(rc != MDBX_SUCCESS)) { + if (rc != MDBX_NOTFOUND || !is_pointed(&end.outer)) + return LOG_IFERR(rc); } - if (size_now <= 0) { - size_now = size_lower; - if (size_upper >= size_lower && size_now > size_upper) - size_now = size_upper; - } - if (size_now >= INTPTR_MAX) { - size_now = get_reasonable_db_maxsize(&reasonable_maxsize); - if ((size_t)size_now / pagesize > MAX_PAGENO + 1) - size_now = pagesize * (MAX_PAGENO + 1); - } + rc = mdbx_estimate_distance(&begin.outer, &end.outer, size_items); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); + assert(*size_items >= -(ptrdiff_t)begin.outer.tree->items && *size_items <= (ptrdiff_t)begin.outer.tree->items); - if (size_upper <= 0) { - if (size_now >= get_reasonable_db_maxsize(&reasonable_maxsize) / 2) - size_upper = get_reasonable_db_maxsize(&reasonable_maxsize); - else if (MAX_MAPSIZE != MAX_MAPSIZE32 && - (size_t)size_now >= MAX_MAPSIZE32 / 2 && - (size_t)size_now <= MAX_MAPSIZE32 / 4 * 3) - size_upper = MAX_MAPSIZE32; - else { - size_upper = size_now + size_now; - if ((size_t)size_upper < DEFAULT_MAPSIZE * 2) - size_upper = DEFAULT_MAPSIZE * 2; - } - if ((size_t)size_upper / pagesize > (MAX_PAGENO + 1)) - size_upper = pagesize * (MAX_PAGENO + 1); - } else if (size_upper >= INTPTR_MAX) { - size_upper = get_reasonable_db_maxsize(&reasonable_maxsize); - if ((size_t)size_upper / pagesize > MAX_PAGENO + 1) - size_upper = pagesize * (MAX_PAGENO + 1); - } +#if 0 /* LY: Was decided to returns as-is (i.e. negative) the estimation \ + * results for an inverted ranges. */ - if (unlikely(size_lower < (intptr_t)MIN_MAPSIZE || size_lower > size_upper)) { - rc = MDBX_EINVAL; - goto bailout; - } + /* Commit 8ddfd1f34ad7cf7a3c4aa75d2e248ca7e639ed63 + Change-Id: If59eccf7311123ab6384c4b93f9b1fed5a0a10d1 */ - if ((uint64_t)size_lower / pagesize < MIN_PAGENO) { - size_lower = pagesize * MIN_PAGENO; - if (unlikely(size_lower > size_upper)) { - rc = MDBX_EINVAL; - goto bailout; + if (*size_items < 0) { + /* LY: inverted range case */ + *size_items += (ptrdiff_t)begin.outer.tree->items; + } else if (*size_items == 0 && begin_key && end_key) { + int cmp = begin.outer.kvx->cmp(&origin_begin_key, &origin_end_key); + if (cmp == 0 && cursor_pointed(begin.inner.cursor.flags) && + begin_data && end_data) + cmp = begin.outer.kvx->v.cmp(&origin_begin_data, &origin_end_data); + if (cmp > 0) { + /* LY: inverted range case with empty scope */ + *size_items = (ptrdiff_t)begin.outer.tree->items; } - if (size_now < size_lower) - size_now = size_lower; } + assert(*size_items >= 0 && + *size_items <= (ptrdiff_t)begin.outer.tree->items); +#endif - if (unlikely((size_t)size_upper > MAX_MAPSIZE || - (uint64_t)size_upper / pagesize > MAX_PAGENO + 1)) { - rc = MDBX_TOO_LARGE; - goto bailout; - } + return MDBX_SUCCESS; +} +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 - const size_t unit = (env->me_os_psize > (size_t)pagesize) ? env->me_os_psize - : (size_t)pagesize; - size_lower = ceil_powerof2(size_lower, unit); - size_upper = ceil_powerof2(size_upper, unit); - size_now = ceil_powerof2(size_now, unit); +__cold int mdbx_dbi_dupsort_depthmask(const MDBX_txn *txn, MDBX_dbi dbi, uint32_t *mask) { + if (unlikely(!mask)) + return LOG_IFERR(MDBX_EINVAL); + *mask = 0; - /* LY: подбираем значение size_upper: - * - кратное размеру страницы - * - без нарушения MAX_MAPSIZE и MAX_PAGENO */ - while (unlikely((size_t)size_upper > MAX_MAPSIZE || - (uint64_t)size_upper / pagesize > MAX_PAGENO + 1)) { - if ((size_t)size_upper < unit + MIN_MAPSIZE || - (size_t)size_upper < (size_t)pagesize * (MIN_PAGENO + 1)) { - /* паранойа на случай переполнения при невероятных значениях */ - rc = MDBX_EINVAL; - goto bailout; + int rc = check_txn(txn, MDBX_TXN_BLOCKED); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); + + cursor_couple_t cx; + rc = cursor_init(&cx.outer, txn, dbi); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); + + if ((cx.outer.tree->flags & MDBX_DUPSORT) == 0) + return MDBX_RESULT_TRUE; + + MDBX_val key, data; + rc = outer_first(&cx.outer, &key, &data); + while (rc == MDBX_SUCCESS) { + const node_t *node = page_node(cx.outer.pg[cx.outer.top], cx.outer.ki[cx.outer.top]); + const tree_t *db = node_data(node); + const unsigned flags = node_flags(node); + switch (flags) { + case N_BIG: + case 0: + /* single-value entry, deep = 0 */ + *mask |= 1 << 0; + break; + case N_DUP: + /* single sub-page, deep = 1 */ + *mask |= 1 << 1; + break; + case N_DUP | N_TREE: + /* sub-tree */ + *mask |= 1 << UNALIGNED_PEEK_16(db, tree_t, height); + break; + default: + ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid node-size", flags); + return LOG_IFERR(MDBX_CORRUPTED); } - size_upper -= unit; - if ((size_t)size_upper < (size_t)size_lower) - size_lower = size_upper; + rc = outer_next(&cx.outer, &key, &data, MDBX_NEXT_NODUP); } - eASSERT(env, (size_upper - size_lower) % env->me_os_psize == 0); - if (size_now < size_lower) - size_now = size_lower; - if (size_now > size_upper) - size_now = size_upper; + return LOG_IFERR((rc == MDBX_NOTFOUND) ? MDBX_SUCCESS : rc); +} - if (growth_step < 0) { - growth_step = ((size_t)(size_upper - size_lower)) / 42; - if (growth_step > size_lower && size_lower < (intptr_t)MEGABYTE) - growth_step = size_lower; - if (growth_step < 65536) - growth_step = 65536; - if ((size_t)growth_step > MAX_MAPSIZE / 64) - growth_step = MAX_MAPSIZE / 64; +int mdbx_canary_get(const MDBX_txn *txn, MDBX_canary *canary) { + if (unlikely(canary == nullptr)) + return LOG_IFERR(MDBX_EINVAL); + + int rc = check_txn(txn, MDBX_TXN_BLOCKED - MDBX_TXN_PARKED); + if (unlikely(rc != MDBX_SUCCESS)) { + memset(canary, 0, sizeof(*canary)); + return LOG_IFERR(rc); } - if (growth_step == 0 && shrink_threshold > 0) - growth_step = 1; - growth_step = ceil_powerof2(growth_step, unit); - if (shrink_threshold < 0) - shrink_threshold = growth_step + growth_step; - shrink_threshold = ceil_powerof2(shrink_threshold, unit); + *canary = txn->canary; + return MDBX_SUCCESS; +} - //---------------------------------------------------------------------------- +int mdbx_get(const MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, MDBX_val *data) { + DKBUF_DEBUG; + DEBUG("===> get db %u key [%s]", dbi, DKEY_DEBUG(key)); - if (!env->me_map) { - /* save user's geo-params for future open/create */ - if (pagesize != (intptr_t)env->me_psize) - setup_pagesize(env, pagesize); - env->me_dbgeo.lower = size_lower; - env->me_dbgeo.now = size_now; - env->me_dbgeo.upper = size_upper; - env->me_dbgeo.grow = - pgno2bytes(env, pv2pages(pages2pv(bytes2pgno(env, growth_step)))); - env->me_dbgeo.shrink = - pgno2bytes(env, pv2pages(pages2pv(bytes2pgno(env, shrink_threshold)))); - adjust_defaults(env); - - ENSURE(env, env->me_dbgeo.lower >= MIN_MAPSIZE); - ENSURE(env, env->me_dbgeo.lower / (unsigned)pagesize >= MIN_PAGENO); - ENSURE(env, env->me_dbgeo.lower % (unsigned)pagesize == 0); - ENSURE(env, env->me_dbgeo.lower % env->me_os_psize == 0); - - ENSURE(env, env->me_dbgeo.upper <= MAX_MAPSIZE); - ENSURE(env, env->me_dbgeo.upper / (unsigned)pagesize <= MAX_PAGENO + 1); - ENSURE(env, env->me_dbgeo.upper % (unsigned)pagesize == 0); - ENSURE(env, env->me_dbgeo.upper % env->me_os_psize == 0); - - ENSURE(env, env->me_dbgeo.now >= env->me_dbgeo.lower); - ENSURE(env, env->me_dbgeo.now <= env->me_dbgeo.upper); - ENSURE(env, env->me_dbgeo.now % (unsigned)pagesize == 0); - ENSURE(env, env->me_dbgeo.now % env->me_os_psize == 0); - - ENSURE(env, env->me_dbgeo.grow % (unsigned)pagesize == 0); - ENSURE(env, env->me_dbgeo.grow % env->me_os_psize == 0); - ENSURE(env, env->me_dbgeo.shrink % (unsigned)pagesize == 0); - ENSURE(env, env->me_dbgeo.shrink % env->me_os_psize == 0); + if (unlikely(!key || !data)) + return LOG_IFERR(MDBX_EINVAL); - rc = MDBX_SUCCESS; - } else { - /* apply new params to opened environment */ - ENSURE(env, pagesize == (intptr_t)env->me_psize); - MDBX_meta meta; - memset(&meta, 0, sizeof(meta)); - if (!inside_txn) { - eASSERT(env, need_unlock); - const meta_ptr_t head = meta_recent(env, &env->me_txn0->tw.troika); + int rc = check_txn(txn, MDBX_TXN_BLOCKED); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); - uint64_t timestamp = 0; - while ("workaround for " - "https://libmdbx.dqdkfa.ru/dead-github/issues/269") { - rc = coherency_check_head(env->me_txn0, head, ×tamp); - if (likely(rc == MDBX_SUCCESS)) - break; - if (unlikely(rc != MDBX_RESULT_TRUE)) - goto bailout; - } - meta = *head.ptr_c; - const txnid_t txnid = safe64_txnid_next(head.txnid); - if (unlikely(txnid > MAX_TXNID)) { - rc = MDBX_TXN_FULL; - ERROR("txnid overflow, raise %d", rc); - goto bailout; - } - meta_set_txnid(env, &meta, txnid); - } + cursor_couple_t cx; + rc = cursor_init(&cx.outer, txn, dbi); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); - const MDBX_geo *const current_geo = - &(env->me_txn ? env->me_txn : env->me_txn0)->mt_geo; - /* update env-geo to avoid influences */ - env->me_dbgeo.now = pgno2bytes(env, current_geo->now); - env->me_dbgeo.lower = pgno2bytes(env, current_geo->lower); - env->me_dbgeo.upper = pgno2bytes(env, current_geo->upper); - env->me_dbgeo.grow = pgno2bytes(env, pv2pages(current_geo->grow_pv)); - env->me_dbgeo.shrink = pgno2bytes(env, pv2pages(current_geo->shrink_pv)); + return LOG_IFERR(cursor_seek(&cx.outer, (MDBX_val *)key, data, MDBX_SET).err); +} - MDBX_geo new_geo; - new_geo.lower = bytes2pgno(env, size_lower); - new_geo.now = bytes2pgno(env, size_now); - new_geo.upper = bytes2pgno(env, size_upper); - new_geo.grow_pv = pages2pv(bytes2pgno(env, growth_step)); - new_geo.shrink_pv = pages2pv(bytes2pgno(env, shrink_threshold)); - new_geo.next = current_geo->next; +int mdbx_get_equal_or_great(const MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, MDBX_val *data) { + if (unlikely(!key || !data)) + return LOG_IFERR(MDBX_EINVAL); - ENSURE(env, pgno_align2os_bytes(env, new_geo.lower) == (size_t)size_lower); - ENSURE(env, pgno_align2os_bytes(env, new_geo.upper) == (size_t)size_upper); - ENSURE(env, pgno_align2os_bytes(env, new_geo.now) == (size_t)size_now); - ENSURE(env, new_geo.grow_pv == pages2pv(pv2pages(new_geo.grow_pv))); - ENSURE(env, new_geo.shrink_pv == pages2pv(pv2pages(new_geo.shrink_pv))); + int rc = check_txn(txn, MDBX_TXN_BLOCKED); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); - ENSURE(env, (size_t)size_lower >= MIN_MAPSIZE); - ENSURE(env, new_geo.lower >= MIN_PAGENO); - ENSURE(env, (size_t)size_upper <= MAX_MAPSIZE); - ENSURE(env, new_geo.upper <= MAX_PAGENO + 1); - ENSURE(env, new_geo.now >= new_geo.next); - ENSURE(env, new_geo.upper >= new_geo.now); - ENSURE(env, new_geo.now >= new_geo.lower); + cursor_couple_t cx; + rc = cursor_init(&cx.outer, txn, dbi); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); - if (memcmp(current_geo, &new_geo, sizeof(MDBX_geo)) != 0) { -#if defined(_WIN32) || defined(_WIN64) - /* Was DB shrinking disabled before and now it will be enabled? */ - if (new_geo.lower < new_geo.upper && new_geo.shrink_pv && - !(current_geo->lower < current_geo->upper && - current_geo->shrink_pv)) { - if (!env->me_lck_mmap.lck) { - rc = MDBX_EPERM; - goto bailout; - } - int err = osal_rdt_lock(env); - if (unlikely(MDBX_IS_ERROR(err))) { - rc = err; - goto bailout; - } + return LOG_IFERR(cursor_ops(&cx.outer, key, data, MDBX_SET_LOWERBOUND)); +} - /* Check if there are any reading threads that do not use the SRWL */ - const size_t CurrentTid = GetCurrentThreadId(); - const MDBX_reader *const begin = env->me_lck_mmap.lck->mti_readers; - const MDBX_reader *const end = - begin + atomic_load32(&env->me_lck_mmap.lck->mti_numreaders, - mo_AcquireRelease); - for (const MDBX_reader *reader = begin; reader < end; ++reader) { - if (reader->mr_pid.weak == env->me_pid && reader->mr_tid.weak && - reader->mr_tid.weak != CurrentTid) { - /* At least one thread may don't use SRWL */ - rc = MDBX_EPERM; - break; - } - } +int mdbx_get_ex(const MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, MDBX_val *data, size_t *values_count) { + DKBUF_DEBUG; + DEBUG("===> get db %u key [%s]", dbi, DKEY_DEBUG(key)); - osal_rdt_unlock(env); - if (unlikely(rc != MDBX_SUCCESS)) - goto bailout; - } -#endif + if (unlikely(!key || !data)) + return LOG_IFERR(MDBX_EINVAL); - if (new_geo.now != current_geo->now || - new_geo.upper != current_geo->upper) { - rc = dxb_resize(env, current_geo->next, new_geo.now, new_geo.upper, - explicit_resize); - if (unlikely(rc != MDBX_SUCCESS)) - goto bailout; - } - if (inside_txn) { - env->me_txn->mt_geo = new_geo; - env->me_txn->mt_flags |= MDBX_TXN_DIRTY; - } else { - meta.mm_geo = new_geo; - rc = sync_locked(env, env->me_flags, &meta, &env->me_txn0->tw.troika); - if (likely(rc == MDBX_SUCCESS)) { - env->me_dbgeo.now = pgno2bytes(env, new_geo.now = meta.mm_geo.now); - env->me_dbgeo.upper = - pgno2bytes(env, new_geo.upper = meta.mm_geo.upper); - } + int rc = check_txn(txn, MDBX_TXN_BLOCKED); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); + + cursor_couple_t cx; + rc = cursor_init(&cx.outer, txn, dbi); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); + + rc = cursor_seek(&cx.outer, key, data, MDBX_SET_KEY).err; + if (unlikely(rc != MDBX_SUCCESS)) { + if (values_count) + *values_count = 0; + return LOG_IFERR(rc); + } + + if (values_count) { + *values_count = 1; + if (inner_pointed(&cx.outer)) + *values_count = + (sizeof(*values_count) >= sizeof(cx.inner.nested_tree.items) || cx.inner.nested_tree.items <= PTRDIFF_MAX) + ? (size_t)cx.inner.nested_tree.items + : PTRDIFF_MAX; + } + return MDBX_SUCCESS; +} + +/*----------------------------------------------------------------------------*/ + +int mdbx_canary_put(MDBX_txn *txn, const MDBX_canary *canary) { + int rc = check_txn_rw(txn, MDBX_TXN_BLOCKED); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); + + if (likely(canary)) { + if (txn->canary.x == canary->x && txn->canary.y == canary->y && txn->canary.z == canary->z) + return MDBX_SUCCESS; + txn->canary.x = canary->x; + txn->canary.y = canary->y; + txn->canary.z = canary->z; + } + txn->canary.v = txn->txnid; + txn->flags |= MDBX_TXN_DIRTY; + + return MDBX_SUCCESS; +} + +/* Функция сообщает находится ли указанный адрес в "грязной" странице у + * заданной пишущей транзакции. В конечном счете это позволяет избавиться от + * лишнего копирования данных из НЕ-грязных страниц. + * + * "Грязные" страницы - это те, которые уже были изменены в ходе пишущей + * транзакции. Соответственно, какие-либо дальнейшие изменения могут привести + * к перезаписи таких страниц. Поэтому все функции, выполняющие изменения, в + * качестве аргументов НЕ должны получать указатели на данные в таких + * страницах. В свою очередь "НЕ грязные" страницы перед модификацией будут + * скопированы. + * + * Другими словами, данные из "грязных" страниц должны быть либо скопированы + * перед передачей в качестве аргументов для дальнейших модификаций, либо + * отвергнуты на стадии проверки корректности аргументов. + * + * Таким образом, функция позволяет как избавится от лишнего копирования, + * так и выполнить более полную проверку аргументов. + * + * ВАЖНО: Передаваемый указатель должен указывать на начало данных. Только + * так гарантируется что актуальный заголовок страницы будет физически + * расположен в той-же странице памяти, в том числе для многостраничных + * P_LARGE страниц с длинными данными. */ +int mdbx_is_dirty(const MDBX_txn *txn, const void *ptr) { + int rc = check_txn(txn, MDBX_TXN_BLOCKED - MDBX_TXN_PARKED); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); + + const MDBX_env *env = txn->env; + const ptrdiff_t offset = ptr_dist(ptr, env->dxb_mmap.base); + if (offset >= 0) { + const pgno_t pgno = bytes2pgno(env, offset); + if (likely(pgno < txn->geo.first_unallocated)) { + const page_t *page = pgno2page(env, pgno); + if (unlikely(page->pgno != pgno || (page->flags & P_ILL_BITS) != 0)) { + /* The ptr pointed into middle of a large page, + * not to the beginning of a data. */ + return LOG_IFERR(MDBX_EINVAL); } + return ((txn->flags & MDBX_TXN_RDONLY) || !is_modifable(txn, page)) ? MDBX_RESULT_FALSE : MDBX_RESULT_TRUE; } - if (likely(rc == MDBX_SUCCESS)) { - /* update env-geo to avoid influences */ - eASSERT(env, env->me_dbgeo.now == pgno2bytes(env, new_geo.now)); - env->me_dbgeo.lower = pgno2bytes(env, new_geo.lower); - eASSERT(env, env->me_dbgeo.upper == pgno2bytes(env, new_geo.upper)); - env->me_dbgeo.grow = pgno2bytes(env, pv2pages(new_geo.grow_pv)); - env->me_dbgeo.shrink = pgno2bytes(env, pv2pages(new_geo.shrink_pv)); + if ((size_t)offset < env->dxb_mmap.limit) { + /* Указатель адресует что-то в пределах mmap, но за границей + * распределенных страниц. Такое может случится если mdbx_is_dirty() + * вызывается после операции, в ходе которой грязная страница была + * возвращена в нераспределенное пространство. */ + return (txn->flags & MDBX_TXN_RDONLY) ? LOG_IFERR(MDBX_EINVAL) : MDBX_RESULT_TRUE; } } -bailout: - if (need_unlock) - mdbx_txn_unlock(env); - return rc; + /* Страница вне используемого mmap-диапазона, т.е. либо в функцию был + * передан некорректный адрес, либо адрес в теневой странице, которая была + * выделена посредством malloc(). + * + * Для режима MDBX_WRITE_MAP режима страница однозначно "не грязная", + * а для режимов без MDBX_WRITE_MAP однозначно "не чистая". */ + return (txn->flags & (MDBX_WRITEMAP | MDBX_TXN_RDONLY)) ? LOG_IFERR(MDBX_EINVAL) : MDBX_RESULT_TRUE; } -__cold static int alloc_page_buf(MDBX_env *env) { - return env->me_pbuf ? MDBX_SUCCESS - : osal_memalign_alloc(env->me_os_psize, - env->me_psize * (size_t)NUM_METAS, - &env->me_pbuf); -} +int mdbx_del(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, const MDBX_val *data) { + if (unlikely(!key)) + return LOG_IFERR(MDBX_EINVAL); -/* Further setup required for opening an MDBX environment */ -__cold static int setup_dxb(MDBX_env *env, const int lck_rc, - const mdbx_mode_t mode_bits) { - MDBX_meta header; - int rc = MDBX_RESULT_FALSE; - int err = read_header(env, &header, lck_rc, mode_bits); - if (unlikely(err != MDBX_SUCCESS)) { - if (lck_rc != /* lck exclusive */ MDBX_RESULT_TRUE || err != MDBX_ENODATA || - (env->me_flags & MDBX_RDONLY) != 0 || - /* recovery mode */ env->me_stuck_meta >= 0) - return err; + if (unlikely(dbi <= FREE_DBI)) + return LOG_IFERR(MDBX_BAD_DBI); - DEBUG("%s", "create new database"); - rc = /* new database */ MDBX_RESULT_TRUE; + int rc = check_txn_rw(txn, MDBX_TXN_BLOCKED); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); - if (!env->me_dbgeo.now) { - /* set defaults if not configured */ - err = mdbx_env_set_geometry(env, 0, -1, DEFAULT_MAPSIZE, -1, -1, -1); - if (unlikely(err != MDBX_SUCCESS)) - return err; - } + cursor_couple_t cx; + rc = cursor_init(&cx.outer, txn, dbi); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); - err = alloc_page_buf(env); - if (unlikely(err != MDBX_SUCCESS)) - return err; + MDBX_val proxy; + MDBX_cursor_op op = MDBX_SET; + unsigned flags = MDBX_ALLDUPS; + if (data) { + proxy = *data; + data = &proxy; + op = MDBX_GET_BOTH; + flags = 0; + } + rc = cursor_seek(&cx.outer, (MDBX_val *)key, (MDBX_val *)data, op).err; + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); - header = *init_metas(env, env->me_pbuf); - err = osal_pwrite(env->me_lazy_fd, env->me_pbuf, - env->me_psize * (size_t)NUM_METAS, 0); - if (unlikely(err != MDBX_SUCCESS)) - return err; + cx.outer.next = txn->cursors[dbi]; + txn->cursors[dbi] = &cx.outer; + rc = cursor_del(&cx.outer, flags); + txn->cursors[dbi] = cx.outer.next; + return LOG_IFERR(rc); +} - err = osal_ftruncate(env->me_lazy_fd, env->me_dxb_mmap.filesize = - env->me_dxb_mmap.current = - env->me_dbgeo.now); - if (unlikely(err != MDBX_SUCCESS)) - return err; +int mdbx_put(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, MDBX_val *data, MDBX_put_flags_t flags) { + if (unlikely(!key || !data)) + return LOG_IFERR(MDBX_EINVAL); -#ifndef NDEBUG /* just for checking */ - err = read_header(env, &header, lck_rc, mode_bits); - if (unlikely(err != MDBX_SUCCESS)) - return err; -#endif - } + if (unlikely(dbi <= FREE_DBI)) + return LOG_IFERR(MDBX_BAD_DBI); - VERBOSE("header: root %" PRIaPGNO "/%" PRIaPGNO ", geo %" PRIaPGNO - "/%" PRIaPGNO "-%" PRIaPGNO "/%" PRIaPGNO " +%u -%u, txn_id %" PRIaTXN - ", %s", - header.mm_dbs[MAIN_DBI].md_root, header.mm_dbs[FREE_DBI].md_root, - header.mm_geo.lower, header.mm_geo.next, header.mm_geo.now, - header.mm_geo.upper, pv2pages(header.mm_geo.grow_pv), - pv2pages(header.mm_geo.shrink_pv), - unaligned_peek_u64(4, header.mm_txnid_a), durable_caption(&header)); + if (unlikely(flags & ~(MDBX_NOOVERWRITE | MDBX_NODUPDATA | MDBX_ALLDUPS | MDBX_ALLDUPS | MDBX_RESERVE | MDBX_APPEND | + MDBX_APPENDDUP | MDBX_CURRENT | MDBX_MULTIPLE))) + return LOG_IFERR(MDBX_EINVAL); - if (env->me_psize != header.mm_psize) - setup_pagesize(env, header.mm_psize); - const size_t used_bytes = pgno2bytes(env, header.mm_geo.next); - const size_t used_aligned2os_bytes = - ceil_powerof2(used_bytes, env->me_os_psize); - if ((env->me_flags & MDBX_RDONLY) /* readonly */ - || lck_rc != MDBX_RESULT_TRUE /* not exclusive */ - || /* recovery mode */ env->me_stuck_meta >= 0) { - /* use present params from db */ - const size_t pagesize = header.mm_psize; - err = mdbx_env_set_geometry( - env, header.mm_geo.lower * pagesize, header.mm_geo.now * pagesize, - header.mm_geo.upper * pagesize, - pv2pages(header.mm_geo.grow_pv) * pagesize, - pv2pages(header.mm_geo.shrink_pv) * pagesize, header.mm_psize); - if (unlikely(err != MDBX_SUCCESS)) { - ERROR("%s: err %d", "could not apply geometry from db", err); - return (err == MDBX_EINVAL) ? MDBX_INCOMPATIBLE : err; - } - } else if (env->me_dbgeo.now) { - /* silently growth to last used page */ - if (env->me_dbgeo.now < used_aligned2os_bytes) - env->me_dbgeo.now = used_aligned2os_bytes; - if (env->me_dbgeo.upper < used_aligned2os_bytes) - env->me_dbgeo.upper = used_aligned2os_bytes; + int rc = check_txn_rw(txn, MDBX_TXN_BLOCKED); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); - /* apply preconfigured params, but only if substantial changes: - * - upper or lower limit changes - * - shrink threshold or growth step - * But ignore change just a 'now/current' size. */ - if (bytes_align2os_bytes(env, env->me_dbgeo.upper) != - pgno2bytes(env, header.mm_geo.upper) || - bytes_align2os_bytes(env, env->me_dbgeo.lower) != - pgno2bytes(env, header.mm_geo.lower) || - bytes_align2os_bytes(env, env->me_dbgeo.shrink) != - pgno2bytes(env, pv2pages(header.mm_geo.shrink_pv)) || - bytes_align2os_bytes(env, env->me_dbgeo.grow) != - pgno2bytes(env, pv2pages(header.mm_geo.grow_pv))) { - - if (env->me_dbgeo.shrink && env->me_dbgeo.now > used_bytes) - /* pre-shrink if enabled */ - env->me_dbgeo.now = used_bytes + env->me_dbgeo.shrink - - used_bytes % env->me_dbgeo.shrink; + cursor_couple_t cx; + rc = cursor_init(&cx.outer, txn, dbi); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); - err = mdbx_env_set_geometry(env, env->me_dbgeo.lower, env->me_dbgeo.now, - env->me_dbgeo.upper, env->me_dbgeo.grow, - env->me_dbgeo.shrink, header.mm_psize); - if (unlikely(err != MDBX_SUCCESS)) { - ERROR("%s: err %d", "could not apply preconfigured db-geometry", err); - return (err == MDBX_EINVAL) ? MDBX_INCOMPATIBLE : err; - } + if (unlikely(flags & MDBX_MULTIPLE)) { + rc = cursor_check_multiple(&cx.outer, key, data, flags); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); + } - /* update meta fields */ - header.mm_geo.now = bytes2pgno(env, env->me_dbgeo.now); - header.mm_geo.lower = bytes2pgno(env, env->me_dbgeo.lower); - header.mm_geo.upper = bytes2pgno(env, env->me_dbgeo.upper); - header.mm_geo.grow_pv = pages2pv(bytes2pgno(env, env->me_dbgeo.grow)); - header.mm_geo.shrink_pv = pages2pv(bytes2pgno(env, env->me_dbgeo.shrink)); + if (flags & MDBX_RESERVE) { + if (unlikely(cx.outer.tree->flags & (MDBX_DUPSORT | MDBX_REVERSEDUP | MDBX_INTEGERDUP | MDBX_DUPFIXED))) + return LOG_IFERR(MDBX_INCOMPATIBLE); + data->iov_base = nullptr; + } - VERBOSE("amended: root %" PRIaPGNO "/%" PRIaPGNO ", geo %" PRIaPGNO - "/%" PRIaPGNO "-%" PRIaPGNO "/%" PRIaPGNO - " +%u -%u, txn_id %" PRIaTXN ", %s", - header.mm_dbs[MAIN_DBI].md_root, header.mm_dbs[FREE_DBI].md_root, - header.mm_geo.lower, header.mm_geo.next, header.mm_geo.now, - header.mm_geo.upper, pv2pages(header.mm_geo.grow_pv), - pv2pages(header.mm_geo.shrink_pv), - unaligned_peek_u64(4, header.mm_txnid_a), - durable_caption(&header)); - } else { - /* fetch back 'now/current' size, since it was ignored during comparison - * and may differ. */ - env->me_dbgeo.now = pgno_align2os_bytes(env, header.mm_geo.now); - } - ENSURE(env, header.mm_geo.now >= header.mm_geo.next); - } else { - /* geo-params are not pre-configured by user, - * get current values from the meta. */ - env->me_dbgeo.now = pgno2bytes(env, header.mm_geo.now); - env->me_dbgeo.lower = pgno2bytes(env, header.mm_geo.lower); - env->me_dbgeo.upper = pgno2bytes(env, header.mm_geo.upper); - env->me_dbgeo.grow = pgno2bytes(env, pv2pages(header.mm_geo.grow_pv)); - env->me_dbgeo.shrink = pgno2bytes(env, pv2pages(header.mm_geo.shrink_pv)); - } - - ENSURE(env, pgno_align2os_bytes(env, header.mm_geo.now) == env->me_dbgeo.now); - ENSURE(env, env->me_dbgeo.now >= used_bytes); - const uint64_t filesize_before = env->me_dxb_mmap.filesize; - if (unlikely(filesize_before != env->me_dbgeo.now)) { - if (lck_rc != /* lck exclusive */ MDBX_RESULT_TRUE) { - VERBOSE("filesize mismatch (expect %" PRIuPTR "b/%" PRIaPGNO - "p, have %" PRIu64 "b/%" PRIaPGNO "p), " - "assume other process working", - env->me_dbgeo.now, bytes2pgno(env, env->me_dbgeo.now), - filesize_before, bytes2pgno(env, (size_t)filesize_before)); - } else { - WARNING("filesize mismatch (expect %" PRIuSIZE "b/%" PRIaPGNO - "p, have %" PRIu64 "b/%" PRIaPGNO "p)", - env->me_dbgeo.now, bytes2pgno(env, env->me_dbgeo.now), - filesize_before, bytes2pgno(env, (size_t)filesize_before)); - if (filesize_before < used_bytes) { - ERROR("last-page beyond end-of-file (last %" PRIaPGNO - ", have %" PRIaPGNO ")", - header.mm_geo.next, bytes2pgno(env, (size_t)filesize_before)); - return MDBX_CORRUPTED; - } + cx.outer.next = txn->cursors[dbi]; + txn->cursors[dbi] = &cx.outer; - if (env->me_flags & MDBX_RDONLY) { - if (filesize_before & (env->me_os_psize - 1)) { - ERROR("%s", "filesize should be rounded-up to system page"); - return MDBX_WANNA_RECOVERY; + /* LY: support for update (explicit overwrite) */ + if (flags & MDBX_CURRENT) { + rc = cursor_seek(&cx.outer, (MDBX_val *)key, nullptr, MDBX_SET).err; + if (likely(rc == MDBX_SUCCESS) && (txn->dbs[dbi].flags & MDBX_DUPSORT) && (flags & MDBX_ALLDUPS) == 0) { + /* LY: allows update (explicit overwrite) only for unique keys */ + node_t *node = page_node(cx.outer.pg[cx.outer.top], cx.outer.ki[cx.outer.top]); + if (node_flags(node) & N_DUP) { + tASSERT(txn, inner_pointed(&cx.outer) && cx.outer.subcur->nested_tree.items > 1); + rc = MDBX_EMULTIVAL; + if ((flags & MDBX_NOOVERWRITE) == 0) { + flags -= MDBX_CURRENT; + rc = cursor_del(&cx.outer, MDBX_ALLDUPS); } - WARNING("%s", "ignore filesize mismatch in readonly-mode"); - } else { - VERBOSE("will resize datafile to %" PRIuSIZE " bytes, %" PRIaPGNO - " pages", - env->me_dbgeo.now, bytes2pgno(env, env->me_dbgeo.now)); } } } - VERBOSE("current boot-id %" PRIx64 "-%" PRIx64 " (%savailable)", bootid.x, - bootid.y, (bootid.x | bootid.y) ? "" : "not-"); + if (likely(rc == MDBX_SUCCESS)) + rc = cursor_put_checklen(&cx.outer, key, data, flags); + txn->cursors[dbi] = cx.outer.next; -#if MDBX_ENABLE_MADVISE - /* calculate readahead hint before mmap with zero redundant pages */ - const bool readahead = - !(env->me_flags & MDBX_NORDAHEAD) && - mdbx_is_readahead_reasonable(used_bytes, 0) == MDBX_RESULT_TRUE; -#endif /* MDBX_ENABLE_MADVISE */ + return LOG_IFERR(rc); +} - err = osal_mmap( - env->me_flags, &env->me_dxb_mmap, env->me_dbgeo.now, env->me_dbgeo.upper, - (lck_rc && env->me_stuck_meta < 0) ? MMAP_OPTION_TRUNCATE : 0); - if (unlikely(err != MDBX_SUCCESS)) - return err; +//------------------------------------------------------------------------------ -#if MDBX_ENABLE_MADVISE -#if defined(MADV_DONTDUMP) - err = madvise(env->me_map, env->me_dxb_mmap.limit, MADV_DONTDUMP) - ? ignore_enosys(errno) - : MDBX_SUCCESS; - if (unlikely(MDBX_IS_ERROR(err))) - return err; -#endif /* MADV_DONTDUMP */ -#if defined(MADV_DODUMP) - if (runtime_flags & MDBX_DBG_DUMP) { - const size_t meta_length_aligned2os = pgno_align2os_bytes(env, NUM_METAS); - err = madvise(env->me_map, meta_length_aligned2os, MADV_DODUMP) - ? ignore_enosys(errno) - : MDBX_SUCCESS; - if (unlikely(MDBX_IS_ERROR(err))) - return err; - } -#endif /* MADV_DODUMP */ -#endif /* MDBX_ENABLE_MADVISE */ - -#ifdef MDBX_USE_VALGRIND - env->me_valgrind_handle = - VALGRIND_CREATE_BLOCK(env->me_map, env->me_dxb_mmap.limit, "mdbx"); -#endif /* MDBX_USE_VALGRIND */ - - eASSERT(env, used_bytes >= pgno2bytes(env, NUM_METAS) && - used_bytes <= env->me_dxb_mmap.limit); -#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__) - if (env->me_dxb_mmap.filesize > used_bytes && - env->me_dxb_mmap.filesize < env->me_dxb_mmap.limit) { - VALGRIND_MAKE_MEM_NOACCESS(ptr_disp(env->me_map, used_bytes), - env->me_dxb_mmap.filesize - used_bytes); - MDBX_ASAN_POISON_MEMORY_REGION(ptr_disp(env->me_map, used_bytes), - env->me_dxb_mmap.filesize - used_bytes); - } - env->me_poison_edge = - bytes2pgno(env, (env->me_dxb_mmap.filesize < env->me_dxb_mmap.limit) - ? env->me_dxb_mmap.filesize - : env->me_dxb_mmap.limit); -#endif /* MDBX_USE_VALGRIND || __SANITIZE_ADDRESS__ */ - - meta_troika_t troika = meta_tap(env); -#if MDBX_DEBUG - meta_troika_dump(env, &troika); -#endif - eASSERT(env, !env->me_txn && !env->me_txn0); - //-------------------------------- validate/rollback head & steady meta-pages - if (unlikely(env->me_stuck_meta >= 0)) { - /* recovery mode */ - MDBX_meta clone; - MDBX_meta const *const target = METAPAGE(env, env->me_stuck_meta); - err = validate_meta_copy(env, target, &clone); - if (unlikely(err != MDBX_SUCCESS)) { - ERROR("target meta[%u] is corrupted", - bytes2pgno(env, ptr_dist(data_page(target), env->me_map))); - meta_troika_dump(env, &troika); - return MDBX_CORRUPTED; - } - } else /* not recovery mode */ - while (1) { - const unsigned meta_clash_mask = meta_eq_mask(&troika); - if (unlikely(meta_clash_mask)) { - ERROR("meta-pages are clashed: mask 0x%d", meta_clash_mask); - meta_troika_dump(env, &troika); - return MDBX_CORRUPTED; - } +/* Позволяет обновить или удалить существующую запись с получением + * в old_data предыдущего значения данных. При этом если new_data равен + * нулю, то выполняется удаление, иначе обновление/вставка. + * + * Текущее значение может находиться в уже измененной (грязной) странице. + * В этом случае страница будет перезаписана при обновлении, а само старое + * значение утрачено. Поэтому исходно в old_data должен быть передан + * дополнительный буфер для копирования старого значения. + * Если переданный буфер слишком мал, то функция вернет -1, установив + * old_data->iov_len в соответствующее значение. + * + * Для не-уникальных ключей также возможен второй сценарий использования, + * когда посредством old_data из записей с одинаковым ключом для + * удаления/обновления выбирается конкретная. Для выбора этого сценария + * во flags следует одновременно указать MDBX_CURRENT и MDBX_NOOVERWRITE. + * Именно эта комбинация выбрана, так как она лишена смысла, и этим позволяет + * идентифицировать запрос такого сценария. + * + * Функция может быть замещена соответствующими операциями с курсорами + * после двух доработок (TODO): + * - внешняя аллокация курсоров, в том числе на стеке (без malloc). + * - получения dirty-статуса страницы по адресу (знать о MUTABLE/WRITEABLE). + */ - if (lck_rc != /* lck exclusive */ MDBX_RESULT_TRUE) { - /* non-exclusive mode, - * meta-pages should be validated by a first process opened the DB */ - if (troika.recent == troika.prefer_steady) - break; +int mdbx_replace_ex(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, MDBX_val *new_data, MDBX_val *old_data, + MDBX_put_flags_t flags, MDBX_preserve_func preserver, void *preserver_context) { + if (unlikely(!key || !old_data || old_data == new_data)) + return LOG_IFERR(MDBX_EINVAL); - if (!env->me_lck_mmap.lck) { - /* LY: without-lck (read-only) mode, so it is impossible that other - * process made weak checkpoint. */ - ERROR("%s", "without-lck, unable recovery/rollback"); - meta_troika_dump(env, &troika); - return MDBX_WANNA_RECOVERY; - } + if (unlikely(old_data->iov_base == nullptr && old_data->iov_len)) + return LOG_IFERR(MDBX_EINVAL); - /* LY: assume just have a collision with other running process, - * or someone make a weak checkpoint */ - VERBOSE("%s", "assume collision or online weak checkpoint"); - break; - } - eASSERT(env, lck_rc == MDBX_RESULT_TRUE); - /* exclusive mode */ + if (unlikely(new_data == nullptr && (flags & (MDBX_CURRENT | MDBX_RESERVE)) != MDBX_CURRENT)) + return LOG_IFERR(MDBX_EINVAL); - const meta_ptr_t recent = meta_recent(env, &troika); - const meta_ptr_t prefer_steady = meta_prefer_steady(env, &troika); - MDBX_meta clone; - if (prefer_steady.is_steady) { - err = validate_meta_copy(env, prefer_steady.ptr_c, &clone); - if (unlikely(err != MDBX_SUCCESS)) { - ERROR("meta[%u] with %s txnid %" PRIaTXN " is corrupted, %s needed", - bytes2pgno(env, ptr_dist(prefer_steady.ptr_c, env->me_map)), - "steady", prefer_steady.txnid, "manual recovery"); - meta_troika_dump(env, &troika); - return MDBX_CORRUPTED; - } - if (prefer_steady.ptr_c == recent.ptr_c) - break; - } + if (unlikely(dbi <= FREE_DBI)) + return LOG_IFERR(MDBX_BAD_DBI); - const pgno_t pgno = bytes2pgno(env, ptr_dist(recent.ptr_c, env->me_map)); - const bool last_valid = - validate_meta_copy(env, recent.ptr_c, &clone) == MDBX_SUCCESS; - eASSERT(env, - !prefer_steady.is_steady || recent.txnid != prefer_steady.txnid); - if (unlikely(!last_valid)) { - if (unlikely(!prefer_steady.is_steady)) { - ERROR("%s for open or automatic rollback, %s", - "there are no suitable meta-pages", - "manual recovery is required"); - meta_troika_dump(env, &troika); - return MDBX_CORRUPTED; - } - WARNING("meta[%u] with last txnid %" PRIaTXN - " is corrupted, rollback needed", - pgno, recent.txnid); - meta_troika_dump(env, &troika); - goto purge_meta_head; - } + if (unlikely(flags & ~(MDBX_NOOVERWRITE | MDBX_NODUPDATA | MDBX_ALLDUPS | MDBX_RESERVE | MDBX_APPEND | + MDBX_APPENDDUP | MDBX_CURRENT))) + return LOG_IFERR(MDBX_EINVAL); - if (meta_bootid_match(recent.ptr_c)) { - if (env->me_flags & MDBX_RDONLY) { - ERROR("%s, but boot-id(%016" PRIx64 "-%016" PRIx64 ") is MATCH: " - "rollback NOT needed, steady-sync NEEDED%s", - "opening after an unclean shutdown", bootid.x, bootid.y, - ", but unable in read-only mode"); - meta_troika_dump(env, &troika); - return MDBX_WANNA_RECOVERY; - } - WARNING("%s, but boot-id(%016" PRIx64 "-%016" PRIx64 ") is MATCH: " - "rollback NOT needed, steady-sync NEEDED%s", - "opening after an unclean shutdown", bootid.x, bootid.y, ""); - header = clone; - env->me_lck->mti_unsynced_pages.weak = header.mm_geo.next; - if (!env->me_lck->mti_eoos_timestamp.weak) - env->me_lck->mti_eoos_timestamp.weak = osal_monotime(); - break; - } - if (unlikely(!prefer_steady.is_steady)) { - ERROR("%s, but %s for automatic rollback: %s", - "opening after an unclean shutdown", - "there are no suitable meta-pages", - "manual recovery is required"); - meta_troika_dump(env, &troika); - return MDBX_CORRUPTED; - } - if (env->me_flags & MDBX_RDONLY) { - ERROR("%s and rollback needed: (from head %" PRIaTXN - " to steady %" PRIaTXN ")%s", - "opening after an unclean shutdown", recent.txnid, - prefer_steady.txnid, ", but unable in read-only mode"); - meta_troika_dump(env, &troika); - return MDBX_WANNA_RECOVERY; - } + int rc = check_txn_rw(txn, MDBX_TXN_BLOCKED); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); - purge_meta_head: - NOTICE("%s and doing automatic rollback: " - "purge%s meta[%u] with%s txnid %" PRIaTXN, - "opening after an unclean shutdown", last_valid ? "" : " invalid", - pgno, last_valid ? " weak" : "", recent.txnid); - meta_troika_dump(env, &troika); - ENSURE(env, prefer_steady.is_steady); - err = override_meta(env, pgno, 0, - last_valid ? recent.ptr_c : prefer_steady.ptr_c); - if (err) { - ERROR("rollback: overwrite meta[%u] with txnid %" PRIaTXN ", error %d", - pgno, recent.txnid, err); - return err; - } - troika = meta_tap(env); - ENSURE(env, 0 == meta_txnid(recent.ptr_v)); - ENSURE(env, 0 == meta_eq_mask(&troika)); - } + cursor_couple_t cx; + rc = cursor_init(&cx.outer, txn, dbi); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); + cx.outer.next = txn->cursors[dbi]; + txn->cursors[dbi] = &cx.outer; - if (lck_rc == /* lck exclusive */ MDBX_RESULT_TRUE) { - //-------------------------------------------------- shrink DB & update geo - /* re-check size after mmap */ - if ((env->me_dxb_mmap.current & (env->me_os_psize - 1)) != 0 || - env->me_dxb_mmap.current < used_bytes) { - ERROR("unacceptable/unexpected datafile size %" PRIuPTR, - env->me_dxb_mmap.current); - return MDBX_PROBLEM; - } - if (env->me_dxb_mmap.current != env->me_dbgeo.now) { - header.mm_geo.now = bytes2pgno(env, env->me_dxb_mmap.current); - NOTICE("need update meta-geo to filesize %" PRIuPTR " bytes, %" PRIaPGNO - " pages", - env->me_dxb_mmap.current, header.mm_geo.now); + MDBX_val present_key = *key; + if (F_ISSET(flags, MDBX_CURRENT | MDBX_NOOVERWRITE)) { + /* в old_data значение для выбора конкретного дубликата */ + if (unlikely(!(txn->dbs[dbi].flags & MDBX_DUPSORT))) { + rc = MDBX_EINVAL; + goto bailout; } - const meta_ptr_t recent = meta_recent(env, &troika); - if (/* не учитываем различия в geo.next */ - header.mm_geo.grow_pv != recent.ptr_c->mm_geo.grow_pv || - header.mm_geo.shrink_pv != recent.ptr_c->mm_geo.shrink_pv || - header.mm_geo.lower != recent.ptr_c->mm_geo.lower || - header.mm_geo.upper != recent.ptr_c->mm_geo.upper || - header.mm_geo.now != recent.ptr_c->mm_geo.now) { - if ((env->me_flags & MDBX_RDONLY) != 0 || - /* recovery mode */ env->me_stuck_meta >= 0) { - WARNING("skipped update meta.geo in %s mode: from l%" PRIaPGNO - "-n%" PRIaPGNO "-u%" PRIaPGNO "/s%u-g%u, to l%" PRIaPGNO - "-n%" PRIaPGNO "-u%" PRIaPGNO "/s%u-g%u", - (env->me_stuck_meta < 0) ? "read-only" : "recovery", - recent.ptr_c->mm_geo.lower, recent.ptr_c->mm_geo.now, - recent.ptr_c->mm_geo.upper, - pv2pages(recent.ptr_c->mm_geo.shrink_pv), - pv2pages(recent.ptr_c->mm_geo.grow_pv), header.mm_geo.lower, - header.mm_geo.now, header.mm_geo.upper, - pv2pages(header.mm_geo.shrink_pv), - pv2pages(header.mm_geo.grow_pv)); - } else { - const txnid_t next_txnid = safe64_txnid_next(recent.txnid); - if (unlikely(next_txnid > MAX_TXNID)) { - ERROR("txnid overflow, raise %d", MDBX_TXN_FULL); - return MDBX_TXN_FULL; - } - NOTICE("updating meta.geo: " - "from l%" PRIaPGNO "-n%" PRIaPGNO "-u%" PRIaPGNO - "/s%u-g%u (txn#%" PRIaTXN "), " - "to l%" PRIaPGNO "-n%" PRIaPGNO "-u%" PRIaPGNO - "/s%u-g%u (txn#%" PRIaTXN ")", - recent.ptr_c->mm_geo.lower, recent.ptr_c->mm_geo.now, - recent.ptr_c->mm_geo.upper, - pv2pages(recent.ptr_c->mm_geo.shrink_pv), - pv2pages(recent.ptr_c->mm_geo.grow_pv), recent.txnid, - header.mm_geo.lower, header.mm_geo.now, header.mm_geo.upper, - pv2pages(header.mm_geo.shrink_pv), - pv2pages(header.mm_geo.grow_pv), next_txnid); + /* убираем лишний бит, он был признаком запрошенного режима */ + flags -= MDBX_NOOVERWRITE; - ENSURE(env, header.unsafe_txnid == recent.txnid); - meta_set_txnid(env, &header, next_txnid); - err = sync_locked(env, env->me_flags | MDBX_SHRINK_ALLOWED, &header, - &troika); - if (err) { - ERROR("error %d, while updating meta.geo: " - "from l%" PRIaPGNO "-n%" PRIaPGNO "-u%" PRIaPGNO - "/s%u-g%u (txn#%" PRIaTXN "), " - "to l%" PRIaPGNO "-n%" PRIaPGNO "-u%" PRIaPGNO - "/s%u-g%u (txn#%" PRIaTXN ")", - err, recent.ptr_c->mm_geo.lower, recent.ptr_c->mm_geo.now, - recent.ptr_c->mm_geo.upper, - pv2pages(recent.ptr_c->mm_geo.shrink_pv), - pv2pages(recent.ptr_c->mm_geo.grow_pv), recent.txnid, - header.mm_geo.lower, header.mm_geo.now, header.mm_geo.upper, - pv2pages(header.mm_geo.shrink_pv), - pv2pages(header.mm_geo.grow_pv), header.unsafe_txnid); - return err; + rc = cursor_seek(&cx.outer, &present_key, old_data, MDBX_GET_BOTH).err; + if (rc != MDBX_SUCCESS) + goto bailout; + } else { + /* в old_data буфер для сохранения предыдущего значения */ + if (unlikely(new_data && old_data->iov_base == new_data->iov_base)) + return LOG_IFERR(MDBX_EINVAL); + MDBX_val present_data; + rc = cursor_seek(&cx.outer, &present_key, &present_data, MDBX_SET_KEY).err; + if (unlikely(rc != MDBX_SUCCESS)) { + old_data->iov_base = nullptr; + old_data->iov_len = 0; + if (rc != MDBX_NOTFOUND || (flags & MDBX_CURRENT)) + goto bailout; + } else if (flags & MDBX_NOOVERWRITE) { + rc = MDBX_KEYEXIST; + *old_data = present_data; + goto bailout; + } else { + page_t *page = cx.outer.pg[cx.outer.top]; + if (txn->dbs[dbi].flags & MDBX_DUPSORT) { + if (flags & MDBX_CURRENT) { + /* disallow update/delete for multi-values */ + node_t *node = page_node(page, cx.outer.ki[cx.outer.top]); + if (node_flags(node) & N_DUP) { + tASSERT(txn, inner_pointed(&cx.outer) && cx.outer.subcur->nested_tree.items > 1); + if (cx.outer.subcur->nested_tree.items > 1) { + rc = MDBX_EMULTIVAL; + goto bailout; + } + } + /* В LMDB флажок MDBX_CURRENT здесь приведет + * к замене данных без учета MDBX_DUPSORT сортировки, + * но здесь это в любом случае допустимо, так как мы + * проверили что для ключа есть только одно значение. */ } } - } - - atomic_store32(&env->me_lck->mti_discarded_tail, - bytes2pgno(env, used_aligned2os_bytes), mo_Relaxed); - if ((env->me_flags & MDBX_RDONLY) == 0 && env->me_stuck_meta < 0 && - (runtime_flags & MDBX_DBG_DONT_UPGRADE) == 0) { - for (int n = 0; n < NUM_METAS; ++n) { - MDBX_meta *const meta = METAPAGE(env, n); - if (unlikely(unaligned_peek_u64(4, &meta->mm_magic_and_version) != - MDBX_DATA_MAGIC)) { - const txnid_t txnid = constmeta_txnid(meta); - NOTICE("%s %s" - "meta[%u], txnid %" PRIaTXN, - "updating db-format signature for", - META_IS_STEADY(meta) ? "stead-" : "weak-", n, txnid); - err = override_meta(env, n, txnid, meta); - if (unlikely(err != MDBX_SUCCESS) && - /* Just ignore the MDBX_PROBLEM error, since here it is - * returned only in case of the attempt to upgrade an obsolete - * meta-page that is invalid for current state of a DB, - * e.g. after shrinking DB file */ - err != MDBX_PROBLEM) { - ERROR("%s meta[%u], txnid %" PRIaTXN ", error %d", - "updating db-format signature for", n, txnid, err); - return err; - } - troika = meta_tap(env); + if (is_modifable(txn, page)) { + if (new_data && cmp_lenfast(&present_data, new_data) == 0) { + /* если данные совпадают, то ничего делать не надо */ + *old_data = *new_data; + goto bailout; } + rc = preserver ? preserver(preserver_context, old_data, present_data.iov_base, present_data.iov_len) + : MDBX_SUCCESS; + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + } else { + *old_data = present_data; } + flags |= MDBX_CURRENT; } - } /* lck exclusive, lck_rc == MDBX_RESULT_TRUE */ - - //---------------------------------------------------- setup madvise/readahead -#if MDBX_ENABLE_MADVISE - if (used_aligned2os_bytes < env->me_dxb_mmap.current) { -#if defined(MADV_REMOVE) - if (lck_rc && (env->me_flags & MDBX_WRITEMAP) != 0 && - /* not recovery mode */ env->me_stuck_meta < 0) { - NOTICE("open-MADV_%s %u..%u", "REMOVE (deallocate file space)", - env->me_lck->mti_discarded_tail.weak, - bytes2pgno(env, env->me_dxb_mmap.current)); - err = - madvise(ptr_disp(env->me_map, used_aligned2os_bytes), - env->me_dxb_mmap.current - used_aligned2os_bytes, MADV_REMOVE) - ? ignore_enosys(errno) - : MDBX_SUCCESS; - if (unlikely(MDBX_IS_ERROR(err))) - return err; - } -#endif /* MADV_REMOVE */ -#if defined(MADV_DONTNEED) - NOTICE("open-MADV_%s %u..%u", "DONTNEED", - env->me_lck->mti_discarded_tail.weak, - bytes2pgno(env, env->me_dxb_mmap.current)); - err = - madvise(ptr_disp(env->me_map, used_aligned2os_bytes), - env->me_dxb_mmap.current - used_aligned2os_bytes, MADV_DONTNEED) - ? ignore_enosys(errno) - : MDBX_SUCCESS; - if (unlikely(MDBX_IS_ERROR(err))) - return err; -#elif defined(POSIX_MADV_DONTNEED) - err = ignore_enosys(posix_madvise( - ptr_disp(env->me_map, used_aligned2os_bytes), - env->me_dxb_mmap.current - used_aligned2os_bytes, POSIX_MADV_DONTNEED)); - if (unlikely(MDBX_IS_ERROR(err))) - return err; -#elif defined(POSIX_FADV_DONTNEED) - err = ignore_enosys(posix_fadvise( - env->me_lazy_fd, used_aligned2os_bytes, - env->me_dxb_mmap.current - used_aligned2os_bytes, POSIX_FADV_DONTNEED)); - if (unlikely(MDBX_IS_ERROR(err))) - return err; -#endif /* MADV_DONTNEED */ } - err = set_readahead(env, bytes2pgno(env, used_bytes), readahead, true); - if (unlikely(err != MDBX_SUCCESS)) - return err; -#endif /* MDBX_ENABLE_MADVISE */ + if (likely(new_data)) + rc = cursor_put_checklen(&cx.outer, key, new_data, flags); + else + rc = cursor_del(&cx.outer, flags & MDBX_ALLDUPS); - return rc; +bailout: + txn->cursors[dbi] = cx.outer.next; + return LOG_IFERR(rc); } -/******************************************************************************/ +static int default_value_preserver(void *context, MDBX_val *target, const void *src, size_t bytes) { + (void)context; + if (unlikely(target->iov_len < bytes)) { + target->iov_base = nullptr; + target->iov_len = bytes; + return MDBX_RESULT_TRUE; + } + memcpy(target->iov_base, src, target->iov_len = bytes); + return MDBX_SUCCESS; +} -/* Open and/or initialize the lock region for the environment. */ -__cold static int setup_lck(MDBX_env *env, pathchar_t *lck_pathname, - mdbx_mode_t mode) { - eASSERT(env, env->me_lazy_fd != INVALID_HANDLE_VALUE); - eASSERT(env, env->me_lfd == INVALID_HANDLE_VALUE); +int mdbx_replace(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, MDBX_val *new_data, MDBX_val *old_data, + MDBX_put_flags_t flags) { + return mdbx_replace_ex(txn, dbi, key, new_data, old_data, flags, default_value_preserver, nullptr); +} +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 - int err = osal_openfile(MDBX_OPEN_LCK, env, lck_pathname, &env->me_lfd, mode); - if (err != MDBX_SUCCESS) { - switch (err) { - default: - return err; - case MDBX_ENOFILE: - case MDBX_EACCESS: - case MDBX_EPERM: - if (!F_ISSET(env->me_flags, MDBX_RDONLY | MDBX_EXCLUSIVE)) - return err; - break; - case MDBX_EROFS: - if ((env->me_flags & MDBX_RDONLY) == 0) - return err; - break; - } +#ifdef __SANITIZE_THREAD__ +/* LY: avoid tsan-trap by txn, mm_last_pg and geo.first_unallocated */ +__attribute__((__no_sanitize_thread__, __noinline__)) +#endif +int mdbx_txn_straggler(const MDBX_txn *txn, int *percent) +{ + int rc = check_txn(txn, MDBX_TXN_BLOCKED - MDBX_TXN_PARKED); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR((rc > 0) ? -rc : rc); - if (err != MDBX_ENOFILE) { - /* ENSURE the file system is read-only */ - err = osal_check_fs_rdonly(env->me_lazy_fd, lck_pathname, err); - if (err != MDBX_SUCCESS && - /* ignore ERROR_NOT_SUPPORTED for exclusive mode */ - !(err == MDBX_ENOSYS && (env->me_flags & MDBX_EXCLUSIVE))) - return err; - } + MDBX_env *env = txn->env; + if (unlikely((txn->flags & MDBX_TXN_RDONLY) == 0)) { + if (percent) + *percent = (int)((txn->geo.first_unallocated * UINT64_C(100) + txn->geo.end_pgno / 2) / txn->geo.end_pgno); + return 0; + } - /* LY: without-lck mode (e.g. exclusive or on read-only filesystem) */ - /* beginning of a locked section ---------------------------------------- */ - lcklist_lock(); - eASSERT(env, env->me_lcklist_next == nullptr); - env->me_lfd = INVALID_HANDLE_VALUE; - const int rc = osal_lck_seize(env); - if (MDBX_IS_ERROR(rc)) { - /* Calling lcklist_detach_locked() is required to restore POSIX-filelock - * and this job will be done by env_close(). */ - lcklist_unlock(); - return rc; + txnid_t lag; + troika_t troika = meta_tap(env); + do { + const meta_ptr_t head = meta_recent(env, &troika); + if (percent) { + const pgno_t maxpg = head.ptr_v->geometry.now; + *percent = (int)((head.ptr_v->geometry.first_unallocated * UINT64_C(100) + maxpg / 2) / maxpg); } - /* insert into inprocess lck-list */ - env->me_lcklist_next = inprocess_lcklist_head; - inprocess_lcklist_head = env; - lcklist_unlock(); - /* end of a locked section ---------------------------------------------- */ - - env->me_lck = lckless_stub(env); - env->me_maxreaders = UINT_MAX; - DEBUG("lck-setup:%s%s%s", " lck-less", - (env->me_flags & MDBX_RDONLY) ? " readonly" : "", - (rc == MDBX_RESULT_TRUE) ? " exclusive" : " cooperative"); - return rc; - } + lag = (head.txnid - txn->txnid) / xMDBX_TXNID_STEP; + } while (unlikely(meta_should_retry(env, &troika))); - /* beginning of a locked section ------------------------------------------ */ - lcklist_lock(); - eASSERT(env, env->me_lcklist_next == nullptr); + return (lag > INT_MAX) ? INT_MAX : (int)lag; +} - /* Try to get exclusive lock. If we succeed, then - * nobody is using the lock region and we should initialize it. */ - err = osal_lck_seize(env); - if (MDBX_IS_ERROR(err)) { - bailout: - /* Calling lcklist_detach_locked() is required to restore POSIX-filelock - * and this job will be done by env_close(). */ - lcklist_unlock(); - return err; - } +MDBX_env *mdbx_txn_env(const MDBX_txn *txn) { + if (unlikely(!txn || txn->signature != txn_signature || txn->env->signature.weak != env_signature)) + return nullptr; + return txn->env; +} - MDBX_env *inprocess_neighbor = nullptr; - if (err == MDBX_RESULT_TRUE) { - err = uniq_check(&env->me_lck_mmap, &inprocess_neighbor); - if (MDBX_IS_ERROR(err)) - goto bailout; - if (inprocess_neighbor && - ((runtime_flags & MDBX_DBG_LEGACY_MULTIOPEN) == 0 || - (inprocess_neighbor->me_flags & MDBX_EXCLUSIVE) != 0)) { - err = MDBX_BUSY; - goto bailout; - } - } - const int lck_seize_rc = err; - - DEBUG("lck-setup:%s%s%s", " with-lck", - (env->me_flags & MDBX_RDONLY) ? " readonly" : "", - (lck_seize_rc == MDBX_RESULT_TRUE) ? " exclusive" : " cooperative"); - - uint64_t size = 0; - err = osal_filesize(env->me_lfd, &size); - if (unlikely(err != MDBX_SUCCESS)) - goto bailout; - - if (lck_seize_rc == MDBX_RESULT_TRUE) { - size = ceil_powerof2(env->me_maxreaders * sizeof(MDBX_reader) + - sizeof(MDBX_lockinfo), - env->me_os_psize); - jitter4testing(false); - } else { - if (env->me_flags & MDBX_EXCLUSIVE) { - err = MDBX_BUSY; - goto bailout; - } - if (size > INT_MAX || (size & (env->me_os_psize - 1)) != 0 || - size < env->me_os_psize) { - ERROR("lck-file has invalid size %" PRIu64 " bytes", size); - err = MDBX_PROBLEM; - goto bailout; - } - } +uint64_t mdbx_txn_id(const MDBX_txn *txn) { + if (unlikely(!txn || txn->signature != txn_signature)) + return 0; + return txn->txnid; +} - const size_t maxreaders = - ((size_t)size - sizeof(MDBX_lockinfo)) / sizeof(MDBX_reader); - if (maxreaders < 4) { - ERROR("lck-size too small (up to %" PRIuPTR " readers)", maxreaders); - err = MDBX_PROBLEM; - goto bailout; - } - env->me_maxreaders = (maxreaders <= MDBX_READERS_LIMIT) - ? (unsigned)maxreaders - : (unsigned)MDBX_READERS_LIMIT; +MDBX_txn_flags_t mdbx_txn_flags(const MDBX_txn *txn) { + STATIC_ASSERT( + (MDBX_TXN_INVALID & (MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_DIRTY | MDBX_TXN_SPILLS | MDBX_TXN_HAS_CHILD | + txn_gc_drained | txn_shrink_allowed | txn_rw_begin_flags | txn_ro_begin_flags)) == 0); + if (unlikely(!txn || txn->signature != txn_signature)) + return MDBX_TXN_INVALID; + assert(0 == (int)(txn->flags & MDBX_TXN_INVALID)); - err = osal_mmap((env->me_flags & MDBX_EXCLUSIVE) | MDBX_WRITEMAP, - &env->me_lck_mmap, (size_t)size, (size_t)size, - lck_seize_rc ? MMAP_OPTION_TRUNCATE | MMAP_OPTION_SEMAPHORE - : MMAP_OPTION_SEMAPHORE); - if (unlikely(err != MDBX_SUCCESS)) - goto bailout; + MDBX_txn_flags_t flags = txn->flags; + if (F_ISSET(flags, MDBX_TXN_PARKED | MDBX_TXN_RDONLY) && txn->to.reader && + safe64_read(&txn->to.reader->tid) == MDBX_TID_TXN_OUSTED) + flags |= MDBX_TXN_OUSTED; + return flags; +} -#if MDBX_ENABLE_MADVISE -#ifdef MADV_DODUMP - err = madvise(env->me_lck_mmap.lck, size, MADV_DODUMP) ? ignore_enosys(errno) - : MDBX_SUCCESS; - if (unlikely(MDBX_IS_ERROR(err))) - goto bailout; -#endif /* MADV_DODUMP */ +int mdbx_txn_reset(MDBX_txn *txn) { + int rc = check_txn(txn, 0); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); -#ifdef MADV_WILLNEED - err = madvise(env->me_lck_mmap.lck, size, MADV_WILLNEED) - ? ignore_enosys(errno) - : MDBX_SUCCESS; - if (unlikely(MDBX_IS_ERROR(err))) - goto bailout; -#elif defined(POSIX_MADV_WILLNEED) - err = ignore_enosys( - posix_madvise(env->me_lck_mmap.lck, size, POSIX_MADV_WILLNEED)); - if (unlikely(MDBX_IS_ERROR(err))) - goto bailout; -#endif /* MADV_WILLNEED */ -#endif /* MDBX_ENABLE_MADVISE */ + /* This call is only valid for read-only txns */ + if (unlikely((txn->flags & MDBX_TXN_RDONLY) == 0)) + return LOG_IFERR(MDBX_EINVAL); - struct MDBX_lockinfo *const lck = env->me_lck_mmap.lck; - if (lck_seize_rc == MDBX_RESULT_TRUE) { - /* LY: exclusive mode, check and reset lck content */ - memset(lck, 0, (size_t)size); - jitter4testing(false); - lck->mti_magic_and_version = MDBX_LOCK_MAGIC; - lck->mti_os_and_format = MDBX_LOCK_FORMAT; -#if MDBX_ENABLE_PGOP_STAT - lck->mti_pgop_stat.wops.weak = 1; -#endif /* MDBX_ENABLE_PGOP_STAT */ - err = osal_msync(&env->me_lck_mmap, 0, (size_t)size, - MDBX_SYNC_DATA | MDBX_SYNC_SIZE); - if (unlikely(err != MDBX_SUCCESS)) { - ERROR("initial-%s for lck-file failed, err %d", "msync/fsync", err); - goto bailout; - } - } else { - if (lck->mti_magic_and_version != MDBX_LOCK_MAGIC) { - const bool invalid = (lck->mti_magic_and_version >> 8) != MDBX_MAGIC; - ERROR("lock region has %s", - invalid - ? "invalid magic" - : "incompatible version (only applications with nearly or the " - "same versions of libmdbx can share the same database)"); - err = invalid ? MDBX_INVALID : MDBX_VERSION_MISMATCH; - goto bailout; - } - if (lck->mti_os_and_format != MDBX_LOCK_FORMAT) { - ERROR("lock region has os/format signature 0x%" PRIx32 - ", expected 0x%" PRIx32, - lck->mti_os_and_format, MDBX_LOCK_FORMAT); - err = MDBX_VERSION_MISMATCH; - goto bailout; - } + /* LY: don't close DBI-handles */ + rc = txn_end(txn, TXN_END_RESET | TXN_END_UPDATE); + if (rc == MDBX_SUCCESS) { + tASSERT(txn, txn->signature == txn_signature); + tASSERT(txn, txn->owner == 0); } - - err = osal_lck_init(env, inprocess_neighbor, lck_seize_rc); - if (MDBX_IS_ERROR(err)) - goto bailout; - - ENSURE(env, env->me_lcklist_next == nullptr); - /* insert into inprocess lck-list */ - env->me_lcklist_next = inprocess_lcklist_head; - inprocess_lcklist_head = env; - lcklist_unlock(); - /* end of a locked section ------------------------------------------------ */ - - eASSERT(env, !MDBX_IS_ERROR(lck_seize_rc)); - env->me_lck = lck; - return lck_seize_rc; + return LOG_IFERR(rc); } -__cold int mdbx_is_readahead_reasonable(size_t volume, intptr_t redundancy) { - if (volume <= 1024 * 1024 * 4ul) - return MDBX_RESULT_TRUE; +int mdbx_txn_break(MDBX_txn *txn) { + do { + int rc = check_txn(txn, 0); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); + txn->flags |= MDBX_TXN_ERROR; + if (txn->flags & MDBX_TXN_RDONLY) + break; + txn = txn->nested; + } while (txn); + return MDBX_SUCCESS; +} - intptr_t pagesize, total_ram_pages; - int err = mdbx_get_sysraminfo(&pagesize, &total_ram_pages, nullptr); - if (unlikely(err != MDBX_SUCCESS)) - return err; +int mdbx_txn_abort(MDBX_txn *txn) { + int rc = check_txn(txn, 0); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); - const int log2page = log2n_powerof2(pagesize); - const intptr_t volume_pages = (volume + pagesize - 1) >> log2page; - const intptr_t redundancy_pages = - (redundancy < 0) ? -(intptr_t)((-redundancy + pagesize - 1) >> log2page) - : (intptr_t)(redundancy + pagesize - 1) >> log2page; - if (volume_pages >= total_ram_pages || - volume_pages + redundancy_pages >= total_ram_pages) - return MDBX_RESULT_FALSE; + rc = check_env(txn->env, true); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); - intptr_t avail_ram_pages; - err = mdbx_get_sysraminfo(nullptr, nullptr, &avail_ram_pages); - if (unlikely(err != MDBX_SUCCESS)) - return err; +#if MDBX_TXN_CHECKOWNER + if ((txn->flags & (MDBX_TXN_RDONLY | MDBX_NOSTICKYTHREADS)) == MDBX_NOSTICKYTHREADS && + unlikely(txn->owner != osal_thread_self())) { + mdbx_txn_break(txn); + return LOG_IFERR(MDBX_THREAD_MISMATCH); + } +#endif /* MDBX_TXN_CHECKOWNER */ - return (volume_pages + redundancy_pages >= avail_ram_pages) - ? MDBX_RESULT_FALSE - : MDBX_RESULT_TRUE; + return LOG_IFERR(txn_abort(txn)); } -/* Merge sync flags */ -static uint32_t merge_sync_flags(const uint32_t a, const uint32_t b) { - uint32_t r = a | b; - - /* avoid false MDBX_UTTERLY_NOSYNC */ - if (F_ISSET(r, MDBX_UTTERLY_NOSYNC) && !F_ISSET(a, MDBX_UTTERLY_NOSYNC) && - !F_ISSET(b, MDBX_UTTERLY_NOSYNC)) - r = (r - MDBX_UTTERLY_NOSYNC) | MDBX_SAFE_NOSYNC; - - /* convert MDBX_DEPRECATED_MAPASYNC to MDBX_SAFE_NOSYNC */ - if ((r & (MDBX_WRITEMAP | MDBX_DEPRECATED_MAPASYNC)) == - (MDBX_WRITEMAP | MDBX_DEPRECATED_MAPASYNC) && - !F_ISSET(r, MDBX_UTTERLY_NOSYNC)) - r = (r - MDBX_DEPRECATED_MAPASYNC) | MDBX_SAFE_NOSYNC; +int mdbx_txn_park(MDBX_txn *txn, bool autounpark) { + STATIC_ASSERT(MDBX_TXN_BLOCKED > MDBX_TXN_ERROR); + int rc = check_txn(txn, MDBX_TXN_BLOCKED - MDBX_TXN_ERROR); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); + if (unlikely((txn->flags & MDBX_TXN_RDONLY) == 0)) + return LOG_IFERR(MDBX_TXN_INVALID); - /* force MDBX_NOMETASYNC if NOSYNC enabled */ - if (r & (MDBX_SAFE_NOSYNC | MDBX_UTTERLY_NOSYNC)) - r |= MDBX_NOMETASYNC; + if (unlikely((txn->flags & MDBX_TXN_ERROR))) { + rc = txn_end(txn, TXN_END_RESET | TXN_END_UPDATE); + return LOG_IFERR(rc ? rc : MDBX_OUSTED); + } - assert(!(F_ISSET(r, MDBX_UTTERLY_NOSYNC) && - !F_ISSET(a, MDBX_UTTERLY_NOSYNC) && - !F_ISSET(b, MDBX_UTTERLY_NOSYNC))); - return r; + return LOG_IFERR(txn_park(txn, autounpark)); } -__cold static int __must_check_result override_meta(MDBX_env *env, - size_t target, - txnid_t txnid, - const MDBX_meta *shape) { - int rc = alloc_page_buf(env); +int mdbx_txn_unpark(MDBX_txn *txn, bool restart_if_ousted) { + STATIC_ASSERT(MDBX_TXN_BLOCKED > MDBX_TXN_PARKED + MDBX_TXN_ERROR); + int rc = check_txn(txn, MDBX_TXN_BLOCKED - MDBX_TXN_PARKED - MDBX_TXN_ERROR); if (unlikely(rc != MDBX_SUCCESS)) - return rc; - MDBX_page *const page = env->me_pbuf; - meta_model(env, page, target); - MDBX_meta *const model = page_meta(page); - meta_set_txnid(env, model, txnid); - if (txnid) - eASSERT(env, check_meta_coherency(env, model, true)); - if (shape) { - if (txnid && unlikely(!check_meta_coherency(env, shape, false))) { - ERROR("bailout overriding meta-%zu since model failed " - "freedb/maindb %s-check for txnid #%" PRIaTXN, - target, "pre", constmeta_txnid(shape)); - return MDBX_PROBLEM; - } - if (runtime_flags & MDBX_DBG_DONT_UPGRADE) - memcpy(&model->mm_magic_and_version, &shape->mm_magic_and_version, - sizeof(model->mm_magic_and_version)); - model->mm_extra_flags = shape->mm_extra_flags; - model->mm_validator_id = shape->mm_validator_id; - model->mm_extra_pagehdr = shape->mm_extra_pagehdr; - memcpy(&model->mm_geo, &shape->mm_geo, sizeof(model->mm_geo)); - memcpy(&model->mm_dbs, &shape->mm_dbs, sizeof(model->mm_dbs)); - memcpy(&model->mm_canary, &shape->mm_canary, sizeof(model->mm_canary)); - memcpy(&model->mm_pages_retired, &shape->mm_pages_retired, - sizeof(model->mm_pages_retired)); - if (txnid) { - if ((!model->mm_dbs[FREE_DBI].md_mod_txnid && - model->mm_dbs[FREE_DBI].md_root != P_INVALID) || - (!model->mm_dbs[MAIN_DBI].md_mod_txnid && - model->mm_dbs[MAIN_DBI].md_root != P_INVALID)) - memcpy(&model->mm_magic_and_version, &shape->mm_magic_and_version, - sizeof(model->mm_magic_and_version)); - if (unlikely(!check_meta_coherency(env, model, false))) { - ERROR("bailout overriding meta-%zu since model failed " - "freedb/maindb %s-check for txnid #%" PRIaTXN, - target, "post", txnid); - return MDBX_PROBLEM; - } - } - } - unaligned_poke_u64(4, model->mm_sign, meta_sign(model)); - rc = validate_meta(env, model, page, (pgno_t)target, nullptr); - if (unlikely(MDBX_IS_ERROR(rc))) - return MDBX_PROBLEM; - - if (shape && memcmp(model, shape, sizeof(MDBX_meta)) == 0) { - NOTICE("skip overriding meta-%zu since no changes " - "for txnid #%" PRIaTXN, - target, txnid); + return LOG_IFERR(rc); + if (unlikely(!F_ISSET(txn->flags, MDBX_TXN_RDONLY | MDBX_TXN_PARKED))) return MDBX_SUCCESS; - } - if (env->me_flags & MDBX_WRITEMAP) { -#if MDBX_ENABLE_PGOP_STAT - env->me_lck->mti_pgop_stat.msync.weak += 1; -#endif /* MDBX_ENABLE_PGOP_STAT */ - rc = osal_msync(&env->me_dxb_mmap, 0, - pgno_align2os_bytes(env, model->mm_geo.next), - MDBX_SYNC_DATA | MDBX_SYNC_IODQ); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - /* override_meta() called only while current process have exclusive - * lock of a DB file. So meta-page could be updated directly without - * clearing consistency flag by mdbx_meta_update_begin() */ - memcpy(pgno2page(env, target), page, env->me_psize); - osal_flush_incoherent_cpu_writeback(); -#if MDBX_ENABLE_PGOP_STAT - env->me_lck->mti_pgop_stat.msync.weak += 1; -#endif /* MDBX_ENABLE_PGOP_STAT */ - rc = osal_msync(&env->me_dxb_mmap, 0, pgno_align2os_bytes(env, target + 1), - MDBX_SYNC_DATA | MDBX_SYNC_IODQ); - } else { -#if MDBX_ENABLE_PGOP_STAT - env->me_lck->mti_pgop_stat.wops.weak += 1; -#endif /* MDBX_ENABLE_PGOP_STAT */ - rc = osal_pwrite(env->me_fd4meta, page, env->me_psize, - pgno2bytes(env, target)); - if (rc == MDBX_SUCCESS && env->me_fd4meta == env->me_lazy_fd) { -#if MDBX_ENABLE_PGOP_STAT - env->me_lck->mti_pgop_stat.fsync.weak += 1; -#endif /* MDBX_ENABLE_PGOP_STAT */ - rc = osal_fsync(env->me_lazy_fd, MDBX_SYNC_DATA | MDBX_SYNC_IODQ); - } - osal_flush_incoherent_mmap(env->me_map, pgno2bytes(env, NUM_METAS), - env->me_os_psize); - } - eASSERT(env, (!env->me_txn && !env->me_txn0) || - (env->me_stuck_meta == (int)target && - (env->me_flags & (MDBX_EXCLUSIVE | MDBX_RDONLY)) == - MDBX_EXCLUSIVE)); - return rc; + rc = txn_unpark(txn); + if (likely(rc != MDBX_OUSTED) || !restart_if_ousted) + return LOG_IFERR(rc); + + tASSERT(txn, txn->flags & MDBX_TXN_FINISHED); + rc = txn_renew(txn, MDBX_TXN_RDONLY); + return (rc == MDBX_SUCCESS) ? MDBX_RESULT_TRUE : LOG_IFERR(rc); } -__cold int mdbx_env_turn_for_recovery(MDBX_env *env, unsigned target) { - if (unlikely(target >= NUM_METAS)) - return MDBX_EINVAL; - int rc = check_env(env, true); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; +int mdbx_txn_renew(MDBX_txn *txn) { + if (unlikely(!txn)) + return LOG_IFERR(MDBX_EINVAL); - if (unlikely((env->me_flags & (MDBX_EXCLUSIVE | MDBX_RDONLY)) != - MDBX_EXCLUSIVE)) - return MDBX_EPERM; + if (unlikely(txn->signature != txn_signature)) + return LOG_IFERR(MDBX_EBADSIGN); - const MDBX_meta *const target_meta = METAPAGE(env, target); - txnid_t new_txnid = constmeta_txnid(target_meta); - if (new_txnid < MIN_TXNID) - new_txnid = MIN_TXNID; - for (unsigned n = 0; n < NUM_METAS; ++n) { - if (n == target) - continue; - MDBX_page *const page = pgno2page(env, n); - MDBX_meta meta = *page_meta(page); - if (validate_meta(env, &meta, page, n, nullptr) != MDBX_SUCCESS) { - int err = override_meta(env, n, 0, nullptr); - if (unlikely(err != MDBX_SUCCESS)) - return err; - } else { - txnid_t txnid = constmeta_txnid(&meta); - if (new_txnid <= txnid) - new_txnid = safe64_txnid_next(txnid); - } - } + if (unlikely((txn->flags & MDBX_TXN_RDONLY) == 0)) + return LOG_IFERR(MDBX_EINVAL); - if (unlikely(new_txnid > MAX_TXNID)) { - ERROR("txnid overflow, raise %d", MDBX_TXN_FULL); - return MDBX_TXN_FULL; + if (unlikely(txn->owner != 0 || !(txn->flags & MDBX_TXN_FINISHED))) { + int rc = mdbx_txn_reset(txn); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; } - return override_meta(env, target, new_txnid, target_meta); -} -__cold int mdbx_env_open_for_recovery(MDBX_env *env, const char *pathname, - unsigned target_meta, bool writeable) { -#if defined(_WIN32) || defined(_WIN64) - wchar_t *pathnameW = nullptr; - int rc = osal_mb2w(pathname, &pathnameW); - if (likely(rc == MDBX_SUCCESS)) { - rc = mdbx_env_open_for_recoveryW(env, pathnameW, target_meta, writeable); - osal_free(pathnameW); + int rc = txn_renew(txn, MDBX_TXN_RDONLY); + if (rc == MDBX_SUCCESS) { + tASSERT(txn, txn->owner == (txn->flags & MDBX_NOSTICKYTHREADS) ? 0 : osal_thread_self()); + DEBUG("renew txn %" PRIaTXN "%c %p on env %p, root page %" PRIaPGNO "/%" PRIaPGNO, txn->txnid, + (txn->flags & MDBX_TXN_RDONLY) ? 'r' : 'w', (void *)txn, (void *)txn->env, txn->dbs[MAIN_DBI].root, + txn->dbs[FREE_DBI].root); } - return rc; + return LOG_IFERR(rc); } -__cold int mdbx_env_open_for_recoveryW(MDBX_env *env, const wchar_t *pathname, - unsigned target_meta, bool writeable) { -#endif /* Windows */ - - if (unlikely(target_meta >= NUM_METAS)) - return MDBX_EINVAL; - int rc = check_env(env, false); +int mdbx_txn_set_userctx(MDBX_txn *txn, void *ctx) { + int rc = check_txn(txn, MDBX_TXN_FINISHED); if (unlikely(rc != MDBX_SUCCESS)) - return rc; - if (unlikely(env->me_map)) - return MDBX_EPERM; + return LOG_IFERR(rc); - env->me_stuck_meta = (int8_t)target_meta; - return -#if defined(_WIN32) || defined(_WIN64) - mdbx_env_openW -#else - mdbx_env_open -#endif /* Windows */ - (env, pathname, writeable ? MDBX_EXCLUSIVE : MDBX_EXCLUSIVE | MDBX_RDONLY, - 0); + txn->userctx = ctx; + return MDBX_SUCCESS; } -typedef struct { - void *buffer_for_free; - pathchar_t *lck, *dxb; - size_t ent_len; -} MDBX_handle_env_pathname; +void *mdbx_txn_get_userctx(const MDBX_txn *txn) { return check_txn(txn, MDBX_TXN_FINISHED) ? nullptr : txn->userctx; } -__cold static int check_alternative_lck_absent(const pathchar_t *lck_pathname) { - int err = osal_fileexists(lck_pathname); - if (unlikely(err != MDBX_RESULT_FALSE)) { - if (err == MDBX_RESULT_TRUE) - err = MDBX_DUPLICATED_CLK; - ERROR("Alternative/Duplicate LCK-file '%" MDBX_PRIsPATH "' error %d", - lck_pathname, err); - } - return err; -} +int mdbx_txn_begin_ex(MDBX_env *env, MDBX_txn *parent, MDBX_txn_flags_t flags, MDBX_txn **ret, void *context) { + if (unlikely(!ret)) + return LOG_IFERR(MDBX_EINVAL); + *ret = nullptr; -__cold static int handle_env_pathname(MDBX_handle_env_pathname *ctx, - const pathchar_t *pathname, - MDBX_env_flags_t *flags, - const mdbx_mode_t mode) { - memset(ctx, 0, sizeof(*ctx)); - if (unlikely(!pathname || !*pathname)) - return MDBX_EINVAL; + if (unlikely((flags & ~txn_rw_begin_flags) && (parent || (flags & ~txn_ro_begin_flags)))) + return LOG_IFERR(MDBX_EINVAL); - int rc; -#if defined(_WIN32) || defined(_WIN64) - const DWORD dwAttrib = GetFileAttributesW(pathname); - if (dwAttrib == INVALID_FILE_ATTRIBUTES) { - rc = GetLastError(); - if (rc != MDBX_ENOFILE) - return rc; - if (mode == 0 || (*flags & MDBX_RDONLY) != 0) - /* can't open existing */ - return rc; + int rc = check_env(env, true); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); - /* auto-create directory if requested */ - if ((*flags & MDBX_NOSUBDIR) == 0 && !CreateDirectoryW(pathname, nullptr)) { - rc = GetLastError(); - if (rc != ERROR_ALREADY_EXISTS) - return rc; + if (unlikely(env->flags & MDBX_RDONLY & ~flags)) /* write txn in RDONLY env */ + return LOG_IFERR(MDBX_EACCESS); + + MDBX_txn *txn = nullptr; + if (parent) { + /* Nested transactions: Max 1 child, write txns only, no writemap */ + rc = check_txn(parent, MDBX_TXN_BLOCKED - MDBX_TXN_PARKED); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); + + if (unlikely(parent->flags & (MDBX_TXN_RDONLY | MDBX_WRITEMAP))) { + rc = MDBX_BAD_TXN; + if ((parent->flags & MDBX_TXN_RDONLY) == 0) { + ERROR("%s mode is incompatible with nested transactions", "MDBX_WRITEMAP"); + rc = MDBX_INCOMPATIBLE; + } + return LOG_IFERR(rc); } - } else { - /* ignore passed MDBX_NOSUBDIR flag and set it automatically */ - *flags |= MDBX_NOSUBDIR; - if (dwAttrib & FILE_ATTRIBUTE_DIRECTORY) - *flags -= MDBX_NOSUBDIR; + + if (env->options.spill_parent4child_denominator) { + /* Spill dirty-pages of parent to provide dirtyroom for child txn */ + rc = txn_spill(parent, nullptr, parent->tw.dirtylist->length / env->options.spill_parent4child_denominator); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); + } + tASSERT(parent, audit_ex(parent, 0, false) == 0); + + flags |= parent->flags & (txn_rw_begin_flags | MDBX_TXN_SPILLS | MDBX_NOSTICKYTHREADS | MDBX_WRITEMAP); + } else if ((flags & MDBX_TXN_RDONLY) == 0) { + /* Reuse preallocated write txn. However, do not touch it until + * txn_renew() succeeds, since it currently may be active. */ + txn = env->basal_txn; + goto renew; } + + const intptr_t bitmap_bytes = +#if MDBX_ENABLE_DBI_SPARSE + ceil_powerof2(env->max_dbi, CHAR_BIT * sizeof(txn->dbi_sparse[0])) / CHAR_BIT; #else - struct stat st; - if (stat(pathname, &st) != 0) { - rc = errno; - if (rc != MDBX_ENOFILE) - return rc; - if (mode == 0 || (*flags & MDBX_RDONLY) != 0) - /* can't open non-existing */ - return rc /* MDBX_ENOFILE */; + 0; +#endif /* MDBX_ENABLE_DBI_SPARSE */ + STATIC_ASSERT(sizeof(txn->tw) > sizeof(txn->to)); + const size_t base = + (flags & MDBX_TXN_RDONLY) ? sizeof(MDBX_txn) - sizeof(txn->tw) + sizeof(txn->to) : sizeof(MDBX_txn); + const size_t size = base + + ((flags & MDBX_TXN_RDONLY) ? (size_t)bitmap_bytes + env->max_dbi * sizeof(txn->dbi_seqs[0]) : 0) + + env->max_dbi * (sizeof(txn->dbs[0]) + sizeof(txn->cursors[0]) + sizeof(txn->dbi_state[0])); + txn = osal_malloc(size); + if (unlikely(txn == nullptr)) + return LOG_IFERR(MDBX_ENOMEM); +#if MDBX_DEBUG + memset(txn, 0xCD, size); + VALGRIND_MAKE_MEM_UNDEFINED(txn, size); +#endif /* MDBX_DEBUG */ + MDBX_ANALYSIS_ASSUME(size > base); + memset(txn, 0, (MDBX_GOOFY_MSVC_STATIC_ANALYZER && base > size) ? size : base); + txn->dbs = ptr_disp(txn, base); + txn->cursors = ptr_disp(txn->dbs, env->max_dbi * sizeof(txn->dbs[0])); +#if MDBX_DEBUG + txn->cursors[FREE_DBI] = nullptr; /* avoid SIGSEGV in an assertion later */ +#endif + txn->dbi_state = ptr_disp(txn, size - env->max_dbi * sizeof(txn->dbi_state[0])); + txn->flags = flags; + txn->env = env; - /* auto-create directory if requested */ - const mdbx_mode_t dir_mode = - (/* inherit read/write permissions for group and others */ mode & - (S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)) | - /* always add read/write/search for owner */ S_IRWXU | - ((mode & S_IRGRP) ? /* +search if readable by group */ S_IXGRP : 0) | - ((mode & S_IROTH) ? /* +search if readable by others */ S_IXOTH : 0); - if ((*flags & MDBX_NOSUBDIR) == 0 && mkdir(pathname, dir_mode)) { - rc = errno; - if (rc != EEXIST) - return rc; + if (parent) { + tASSERT(parent, dpl_check(parent)); +#if MDBX_ENABLE_DBI_SPARSE + txn->dbi_sparse = parent->dbi_sparse; +#endif /* MDBX_ENABLE_DBI_SPARSE */ + txn->dbi_seqs = parent->dbi_seqs; + txn->geo = parent->geo; + rc = dpl_alloc(txn); + if (likely(rc == MDBX_SUCCESS)) { + const size_t len = MDBX_PNL_GETSIZE(parent->tw.repnl) + parent->tw.loose_count; + txn->tw.repnl = pnl_alloc((len > MDBX_PNL_INITIAL) ? len : MDBX_PNL_INITIAL); + if (unlikely(!txn->tw.repnl)) + rc = MDBX_ENOMEM; + } + if (unlikely(rc != MDBX_SUCCESS)) { + nested_failed: + pnl_free(txn->tw.repnl); + dpl_free(txn); + osal_free(txn); + return LOG_IFERR(rc); } - } else { - /* ignore passed MDBX_NOSUBDIR flag and set it automatically */ - *flags |= MDBX_NOSUBDIR; - if (S_ISDIR(st.st_mode)) - *flags -= MDBX_NOSUBDIR; - } -#endif - static const pathchar_t dxb_name[] = MDBX_DATANAME; - static const pathchar_t lck_name[] = MDBX_LOCKNAME; - static const pathchar_t lock_suffix[] = MDBX_LOCK_SUFFIX; + /* Move loose pages to reclaimed list */ + if (parent->tw.loose_count) { + do { + page_t *lp = parent->tw.loose_pages; + tASSERT(parent, lp->flags == P_LOOSE); + rc = pnl_insert_span(&parent->tw.repnl, lp->pgno, 1); + if (unlikely(rc != MDBX_SUCCESS)) + goto nested_failed; + MDBX_ASAN_UNPOISON_MEMORY_REGION(&page_next(lp), sizeof(page_t *)); + VALGRIND_MAKE_MEM_DEFINED(&page_next(lp), sizeof(page_t *)); + parent->tw.loose_pages = page_next(lp); + /* Remove from dirty list */ + page_wash(parent, dpl_exist(parent, lp->pgno), lp, 1); + } while (parent->tw.loose_pages); + parent->tw.loose_count = 0; +#if MDBX_ENABLE_REFUND + parent->tw.loose_refund_wl = 0; +#endif /* MDBX_ENABLE_REFUND */ + tASSERT(parent, dpl_check(parent)); + } + txn->tw.dirtyroom = parent->tw.dirtyroom; + txn->tw.dirtylru = parent->tw.dirtylru; -#if defined(_WIN32) || defined(_WIN64) - assert(dxb_name[0] == '\\' && lck_name[0] == '\\'); - const size_t pathname_len = wcslen(pathname); -#else - assert(dxb_name[0] == '/' && lck_name[0] == '/'); - const size_t pathname_len = strlen(pathname); -#endif - assert(!osal_isdirsep(lock_suffix[0])); - ctx->ent_len = pathname_len; - static const size_t dxb_name_len = ARRAY_LENGTH(dxb_name) - 1; - if (*flags & MDBX_NOSUBDIR) { - if (ctx->ent_len > dxb_name_len && - osal_pathequal(pathname + ctx->ent_len - dxb_name_len, dxb_name, - dxb_name_len)) { - *flags -= MDBX_NOSUBDIR; - ctx->ent_len -= dxb_name_len; - } else if (ctx->ent_len == dxb_name_len - 1 && osal_isdirsep(dxb_name[0]) && - osal_isdirsep(lck_name[0]) && - osal_pathequal(pathname + ctx->ent_len - dxb_name_len + 1, - dxb_name + 1, dxb_name_len - 1)) { - *flags -= MDBX_NOSUBDIR; - ctx->ent_len -= dxb_name_len - 1; + dpl_sort(parent); + if (parent->tw.spilled.list) + spill_purge(parent); + + tASSERT(txn, MDBX_PNL_ALLOCLEN(txn->tw.repnl) >= MDBX_PNL_GETSIZE(parent->tw.repnl)); + memcpy(txn->tw.repnl, parent->tw.repnl, MDBX_PNL_SIZEOF(parent->tw.repnl)); + eASSERT(env, pnl_check_allocated(txn->tw.repnl, (txn->geo.first_unallocated /* LY: intentional assignment + here, only for assertion */ + = parent->geo.first_unallocated) - + MDBX_ENABLE_REFUND)); + + txn->tw.gc.time_acc = parent->tw.gc.time_acc; + txn->tw.gc.last_reclaimed = parent->tw.gc.last_reclaimed; + if (parent->tw.gc.retxl) { + txn->tw.gc.retxl = parent->tw.gc.retxl; + parent->tw.gc.retxl = (void *)(intptr_t)MDBX_PNL_GETSIZE(parent->tw.gc.retxl); + } + + txn->tw.retired_pages = parent->tw.retired_pages; + parent->tw.retired_pages = (void *)(intptr_t)MDBX_PNL_GETSIZE(parent->tw.retired_pages); + + txn->txnid = parent->txnid; + txn->front_txnid = parent->front_txnid + 1; +#if MDBX_ENABLE_REFUND + txn->tw.loose_refund_wl = 0; +#endif /* MDBX_ENABLE_REFUND */ + txn->canary = parent->canary; + parent->flags |= MDBX_TXN_HAS_CHILD; + parent->nested = txn; + txn->parent = parent; + txn->owner = parent->owner; + txn->tw.troika = parent->tw.troika; + + txn->cursors[FREE_DBI] = nullptr; + txn->cursors[MAIN_DBI] = nullptr; + txn->dbi_state[FREE_DBI] = parent->dbi_state[FREE_DBI] & ~(DBI_FRESH | DBI_CREAT | DBI_DIRTY); + txn->dbi_state[MAIN_DBI] = parent->dbi_state[MAIN_DBI] & ~(DBI_FRESH | DBI_CREAT | DBI_DIRTY); + memset(txn->dbi_state + CORE_DBS, 0, (txn->n_dbi = parent->n_dbi) - CORE_DBS); + memcpy(txn->dbs, parent->dbs, sizeof(txn->dbs[0]) * CORE_DBS); + + tASSERT(parent, parent->tw.dirtyroom + parent->tw.dirtylist->length == + (parent->parent ? parent->parent->tw.dirtyroom : parent->env->options.dp_limit)); + tASSERT(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length == + (txn->parent ? txn->parent->tw.dirtyroom : txn->env->options.dp_limit)); + env->txn = txn; + tASSERT(parent, parent->cursors[FREE_DBI] == nullptr); + rc = parent->cursors[MAIN_DBI] ? cursor_shadow(parent->cursors[MAIN_DBI], txn, MAIN_DBI) : MDBX_SUCCESS; + if (AUDIT_ENABLED() && ASSERT_ENABLED()) { + txn->signature = txn_signature; + tASSERT(txn, audit_ex(txn, 0, false) == 0); } + if (unlikely(rc != MDBX_SUCCESS)) + txn_end(txn, TXN_END_FAIL_BEGINCHILD); + } else { /* MDBX_TXN_RDONLY */ + txn->dbi_seqs = ptr_disp(txn->cursors, env->max_dbi * sizeof(txn->cursors[0])); +#if MDBX_ENABLE_DBI_SPARSE + txn->dbi_sparse = ptr_disp(txn->dbi_state, -bitmap_bytes); +#endif /* MDBX_ENABLE_DBI_SPARSE */ + renew: + rc = txn_renew(txn, flags); } - const size_t suflen_with_NOSUBDIR = sizeof(lock_suffix) + sizeof(pathchar_t); - const size_t suflen_without_NOSUBDIR = sizeof(lck_name) + sizeof(dxb_name); - const size_t enogh4any = (suflen_with_NOSUBDIR > suflen_without_NOSUBDIR) - ? suflen_with_NOSUBDIR - : suflen_without_NOSUBDIR; - const size_t bytes_needed = sizeof(pathchar_t) * ctx->ent_len * 2 + enogh4any; - ctx->buffer_for_free = osal_malloc(bytes_needed); - if (!ctx->buffer_for_free) - return MDBX_ENOMEM; + if (unlikely(rc != MDBX_SUCCESS)) { + if (txn != env->basal_txn) + osal_free(txn); + } else { + if (flags & (MDBX_TXN_RDONLY_PREPARE - MDBX_TXN_RDONLY)) + eASSERT(env, txn->flags == (MDBX_TXN_RDONLY | MDBX_TXN_FINISHED)); + else if (flags & MDBX_TXN_RDONLY) + eASSERT(env, (txn->flags & ~(MDBX_NOSTICKYTHREADS | MDBX_TXN_RDONLY | MDBX_WRITEMAP | + /* Win32: SRWL flag */ txn_shrink_allowed)) == 0); + else { + eASSERT(env, (txn->flags & ~(MDBX_NOSTICKYTHREADS | MDBX_WRITEMAP | txn_shrink_allowed | MDBX_NOMETASYNC | + MDBX_SAFE_NOSYNC | MDBX_TXN_SPILLS)) == 0); + assert(!txn->tw.spilled.list && !txn->tw.spilled.least_removed); + } + txn->signature = txn_signature; + txn->userctx = context; + *ret = txn; + DEBUG("begin txn %" PRIaTXN "%c %p on env %p, root page %" PRIaPGNO "/%" PRIaPGNO, txn->txnid, + (flags & MDBX_TXN_RDONLY) ? 'r' : 'w', (void *)txn, (void *)env, txn->dbs[MAIN_DBI].root, + txn->dbs[FREE_DBI].root); + } - ctx->dxb = ctx->buffer_for_free; - ctx->lck = ctx->dxb + ctx->ent_len + dxb_name_len + 1; - pathchar_t *const buf = ctx->buffer_for_free; - rc = MDBX_SUCCESS; - if (ctx->ent_len) { - memcpy(buf + /* shutting up goofy MSVC static analyzer */ 0, pathname, - sizeof(pathchar_t) * pathname_len); - if (*flags & MDBX_NOSUBDIR) { - const pathchar_t *const lck_ext = - osal_fileext(lck_name, ARRAY_LENGTH(lck_name)); - if (lck_ext) { - pathchar_t *pathname_ext = osal_fileext(buf, pathname_len); - memcpy(pathname_ext ? pathname_ext : buf + pathname_len, lck_ext, - sizeof(pathchar_t) * (ARRAY_END(lck_name) - lck_ext)); - rc = check_alternative_lck_absent(buf); - } - } else { - memcpy(buf + ctx->ent_len, dxb_name, sizeof(dxb_name)); - memcpy(buf + ctx->ent_len + dxb_name_len, lock_suffix, - sizeof(lock_suffix)); - rc = check_alternative_lck_absent(buf); + return LOG_IFERR(rc); +} + +int mdbx_txn_commit_ex(MDBX_txn *txn, MDBX_commit_latency *latency) { + STATIC_ASSERT(MDBX_TXN_FINISHED == MDBX_TXN_BLOCKED - MDBX_TXN_HAS_CHILD - MDBX_TXN_ERROR - MDBX_TXN_PARKED); + const uint64_t ts_0 = latency ? osal_monotime() : 0; + uint64_t ts_1 = 0, ts_2 = 0, ts_3 = 0, ts_4 = 0, ts_5 = 0, gc_cputime = 0; + + /* txn_end() mode for a commit which writes nothing */ + unsigned end_mode = TXN_END_PURE_COMMIT | TXN_END_UPDATE | TXN_END_SLOT | TXN_END_FREE; + + int rc = check_txn(txn, MDBX_TXN_FINISHED); + if (unlikely(rc != MDBX_SUCCESS)) { + if (rc == MDBX_BAD_TXN && (txn->flags & MDBX_TXN_RDONLY)) { + rc = MDBX_RESULT_TRUE; + goto fail; } + bailout: + if (latency) + memset(latency, 0, sizeof(*latency)); + return LOG_IFERR(rc); + } - memcpy(ctx->dxb + /* shutting up goofy MSVC static analyzer */ 0, pathname, - sizeof(pathchar_t) * (ctx->ent_len + 1)); - memcpy(ctx->lck, pathname, sizeof(pathchar_t) * ctx->ent_len); - if (*flags & MDBX_NOSUBDIR) { - memcpy(ctx->lck + ctx->ent_len, lock_suffix, sizeof(lock_suffix)); - } else { - memcpy(ctx->dxb + ctx->ent_len, dxb_name, sizeof(dxb_name)); - memcpy(ctx->lck + ctx->ent_len, lck_name, sizeof(lck_name)); + MDBX_env *const env = txn->env; + if (MDBX_ENV_CHECKPID && unlikely(env->pid != osal_getpid())) { + env->flags |= ENV_FATAL_ERROR; + rc = MDBX_PANIC; + goto bailout; + } + + if (unlikely(txn->flags & MDBX_TXN_RDONLY)) { + if (txn->flags & MDBX_TXN_ERROR) { + rc = MDBX_RESULT_TRUE; + goto fail; } - } else { - assert(!(*flags & MDBX_NOSUBDIR)); - memcpy(buf + /* shutting up goofy MSVC static analyzer */ 0, dxb_name + 1, - sizeof(dxb_name) - sizeof(pathchar_t)); - memcpy(buf + dxb_name_len - 1, lock_suffix, sizeof(lock_suffix)); - rc = check_alternative_lck_absent(buf); + goto done; + } - memcpy(ctx->dxb + /* shutting up goofy MSVC static analyzer */ 0, - dxb_name + 1, sizeof(dxb_name) - sizeof(pathchar_t)); - memcpy(ctx->lck, lck_name + 1, sizeof(lck_name) - sizeof(pathchar_t)); +#if MDBX_TXN_CHECKOWNER + if ((txn->flags & MDBX_NOSTICKYTHREADS) && txn == env->basal_txn && unlikely(txn->owner != osal_thread_self())) { + txn->flags |= MDBX_TXN_ERROR; + rc = MDBX_THREAD_MISMATCH; + return LOG_IFERR(rc); } - return rc; -} +#endif /* MDBX_TXN_CHECKOWNER */ -__cold int mdbx_env_delete(const char *pathname, MDBX_env_delete_mode_t mode) { -#if defined(_WIN32) || defined(_WIN64) - wchar_t *pathnameW = nullptr; - int rc = osal_mb2w(pathname, &pathnameW); - if (likely(rc == MDBX_SUCCESS)) { - rc = mdbx_env_deleteW(pathnameW, mode); - osal_free(pathnameW); + if (unlikely(txn->flags & MDBX_TXN_ERROR)) { + rc = MDBX_RESULT_TRUE; + goto fail; } - return rc; -} -__cold int mdbx_env_deleteW(const wchar_t *pathname, - MDBX_env_delete_mode_t mode) { -#endif /* Windows */ + if (txn->nested) { + rc = mdbx_txn_commit_ex(txn->nested, nullptr); + tASSERT(txn, txn->nested == nullptr); + if (unlikely(rc != MDBX_SUCCESS)) + goto fail; + } - switch (mode) { - default: - return MDBX_EINVAL; - case MDBX_ENV_JUST_DELETE: - case MDBX_ENV_ENSURE_UNUSED: - case MDBX_ENV_WAIT_FOR_UNUSED: - break; + if (unlikely(txn != env->txn)) { + DEBUG("%s", "attempt to commit unknown transaction"); + rc = MDBX_EINVAL; + goto fail; } -#ifdef __e2k__ /* https://bugs.mcst.ru/bugzilla/show_bug.cgi?id=6011 */ - MDBX_env *const dummy_env = alloca(sizeof(MDBX_env)); -#else - MDBX_env dummy_env_silo, *const dummy_env = &dummy_env_silo; -#endif - memset(dummy_env, 0, sizeof(*dummy_env)); - dummy_env->me_flags = - (mode == MDBX_ENV_ENSURE_UNUSED) ? MDBX_EXCLUSIVE : MDBX_ENV_DEFAULTS; - dummy_env->me_os_psize = (unsigned)osal_syspagesize(); - dummy_env->me_psize = (unsigned)mdbx_default_pagesize(); - dummy_env->me_pathname = (pathchar_t *)pathname; - - MDBX_handle_env_pathname env_pathname; - STATIC_ASSERT(sizeof(dummy_env->me_flags) == sizeof(MDBX_env_flags_t)); - int rc = MDBX_RESULT_TRUE, - err = handle_env_pathname(&env_pathname, pathname, - (MDBX_env_flags_t *)&dummy_env->me_flags, 0); - if (likely(err == MDBX_SUCCESS)) { - mdbx_filehandle_t clk_handle = INVALID_HANDLE_VALUE, - dxb_handle = INVALID_HANDLE_VALUE; - if (mode > MDBX_ENV_JUST_DELETE) { - err = osal_openfile(MDBX_OPEN_DELETE, dummy_env, env_pathname.dxb, - &dxb_handle, 0); - err = (err == MDBX_ENOFILE) ? MDBX_SUCCESS : err; - if (err == MDBX_SUCCESS) { - err = osal_openfile(MDBX_OPEN_DELETE, dummy_env, env_pathname.lck, - &clk_handle, 0); - err = (err == MDBX_ENOFILE) ? MDBX_SUCCESS : err; + if (txn->parent) { + tASSERT(txn, audit_ex(txn, 0, false) == 0); + eASSERT(env, txn != env->basal_txn); + MDBX_txn *const parent = txn->parent; + eASSERT(env, parent->signature == txn_signature); + eASSERT(env, parent->nested == txn && (parent->flags & MDBX_TXN_HAS_CHILD) != 0); + eASSERT(env, dpl_check(txn)); + + if (txn->tw.dirtylist->length == 0 && !(txn->flags & MDBX_TXN_DIRTY) && parent->n_dbi == txn->n_dbi) { + /* fast completion of pure nested transaction */ + VERBOSE("fast-complete pure nested txn %" PRIaTXN, txn->txnid); + + tASSERT(txn, memcmp(&parent->geo, &txn->geo, sizeof(parent->geo)) == 0); + tASSERT(txn, memcmp(&parent->canary, &txn->canary, sizeof(parent->canary)) == 0); + tASSERT(txn, !txn->tw.spilled.list || MDBX_PNL_GETSIZE(txn->tw.spilled.list) == 0); + tASSERT(txn, txn->tw.loose_count == 0); + + /* Update parent's DBs array */ + eASSERT(env, parent->n_dbi == txn->n_dbi); + TXN_FOREACH_DBI_ALL(txn, dbi) { + tASSERT(txn, (txn->dbi_state[dbi] & (DBI_CREAT | DBI_DIRTY)) == 0); + if (txn->dbi_state[dbi] & DBI_FRESH) { + parent->dbs[dbi] = txn->dbs[dbi]; + /* preserve parent's status */ + const uint8_t state = txn->dbi_state[dbi] | DBI_FRESH; + DEBUG("dbi %zu dbi-state %s 0x%02x -> 0x%02x", dbi, (parent->dbi_state[dbi] != state) ? "update" : "still", + parent->dbi_state[dbi], state); + parent->dbi_state[dbi] = state; + } } - if (err == MDBX_SUCCESS && clk_handle != INVALID_HANDLE_VALUE) - err = osal_lockfile(clk_handle, mode == MDBX_ENV_WAIT_FOR_UNUSED); - if (err == MDBX_SUCCESS && dxb_handle != INVALID_HANDLE_VALUE) - err = osal_lockfile(dxb_handle, mode == MDBX_ENV_WAIT_FOR_UNUSED); + txn_done_cursors(txn, true); + end_mode = TXN_END_PURE_COMMIT | TXN_END_SLOT | TXN_END_FREE | TXN_END_EOTDONE; + goto done; } - if (err == MDBX_SUCCESS) { - err = osal_removefile(env_pathname.dxb); - if (err == MDBX_SUCCESS) - rc = MDBX_SUCCESS; - else if (err == MDBX_ENOFILE) - err = MDBX_SUCCESS; + /* Preserve space for spill list to avoid parent's state corruption + * if allocation fails. */ + const size_t parent_retired_len = (uintptr_t)parent->tw.retired_pages; + tASSERT(txn, parent_retired_len <= MDBX_PNL_GETSIZE(txn->tw.retired_pages)); + const size_t retired_delta = MDBX_PNL_GETSIZE(txn->tw.retired_pages) - parent_retired_len; + if (retired_delta) { + rc = pnl_need(&txn->tw.repnl, retired_delta); + if (unlikely(rc != MDBX_SUCCESS)) + goto fail; } - if (err == MDBX_SUCCESS) { - err = osal_removefile(env_pathname.lck); - if (err == MDBX_SUCCESS) - rc = MDBX_SUCCESS; - else if (err == MDBX_ENOFILE) - err = MDBX_SUCCESS; + if (txn->tw.spilled.list) { + if (parent->tw.spilled.list) { + rc = pnl_need(&parent->tw.spilled.list, MDBX_PNL_GETSIZE(txn->tw.spilled.list)); + if (unlikely(rc != MDBX_SUCCESS)) + goto fail; + } + spill_purge(txn); } - if (err == MDBX_SUCCESS && !(dummy_env->me_flags & MDBX_NOSUBDIR) && - (/* pathname != "." */ pathname[0] != '.' || pathname[1] != 0) && - (/* pathname != ".." */ pathname[0] != '.' || pathname[1] != '.' || - pathname[2] != 0)) { - err = osal_removedirectory(pathname); - if (err == MDBX_SUCCESS) - rc = MDBX_SUCCESS; - else if (err == MDBX_ENOFILE) - err = MDBX_SUCCESS; + if (unlikely(txn->tw.dirtylist->length + parent->tw.dirtylist->length > parent->tw.dirtylist->detent && + !dpl_reserve(parent, txn->tw.dirtylist->length + parent->tw.dirtylist->length))) { + rc = MDBX_ENOMEM; + goto fail; } - if (dxb_handle != INVALID_HANDLE_VALUE) - osal_closefile(dxb_handle); - if (clk_handle != INVALID_HANDLE_VALUE) - osal_closefile(clk_handle); - } else if (err == MDBX_ENOFILE) - err = MDBX_SUCCESS; + //------------------------------------------------------------------------- - osal_free(env_pathname.buffer_for_free); - return (err == MDBX_SUCCESS) ? rc : err; -} + parent->tw.gc.retxl = txn->tw.gc.retxl; + txn->tw.gc.retxl = nullptr; -__cold int mdbx_env_open(MDBX_env *env, const char *pathname, - MDBX_env_flags_t flags, mdbx_mode_t mode) { -#if defined(_WIN32) || defined(_WIN64) - wchar_t *pathnameW = nullptr; - int rc = osal_mb2w(pathname, &pathnameW); - if (likely(rc == MDBX_SUCCESS)) { - rc = mdbx_env_openW(env, pathnameW, flags, mode); - osal_free(pathnameW); - if (rc == MDBX_SUCCESS) - /* force to make cache of the multi-byte pathname representation */ - mdbx_env_get_path(env, &pathname); - } - return rc; -} + parent->tw.retired_pages = txn->tw.retired_pages; + txn->tw.retired_pages = nullptr; -__cold int mdbx_env_openW(MDBX_env *env, const wchar_t *pathname, - MDBX_env_flags_t flags, mdbx_mode_t mode) { -#endif /* Windows */ + pnl_free(parent->tw.repnl); + parent->tw.repnl = txn->tw.repnl; + txn->tw.repnl = nullptr; + parent->tw.gc.time_acc = txn->tw.gc.time_acc; + parent->tw.gc.last_reclaimed = txn->tw.gc.last_reclaimed; - int rc = check_env(env, false); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; + parent->geo = txn->geo; + parent->canary = txn->canary; + parent->flags |= txn->flags & MDBX_TXN_DIRTY; - if (unlikely(flags & ~ENV_USABLE_FLAGS)) - return MDBX_EINVAL; + /* Move loose pages to parent */ +#if MDBX_ENABLE_REFUND + parent->tw.loose_refund_wl = txn->tw.loose_refund_wl; +#endif /* MDBX_ENABLE_REFUND */ + parent->tw.loose_count = txn->tw.loose_count; + parent->tw.loose_pages = txn->tw.loose_pages; - if (unlikely(env->me_lazy_fd != INVALID_HANDLE_VALUE || - (env->me_flags & MDBX_ENV_ACTIVE) != 0 || env->me_map)) - return MDBX_EPERM; + /* Merge our cursors into parent's and close them */ + txn_done_cursors(txn, true); + end_mode |= TXN_END_EOTDONE; - /* Pickup previously mdbx_env_set_flags(), - * but avoid MDBX_UTTERLY_NOSYNC by disjunction */ - const uint32_t saved_me_flags = env->me_flags; - flags = merge_sync_flags(flags | MDBX_DEPRECATED_COALESCE, env->me_flags); + /* Update parent's DBs array */ + eASSERT(env, parent->n_dbi == txn->n_dbi); + TXN_FOREACH_DBI_ALL(txn, dbi) { + if (txn->dbi_state[dbi] != (parent->dbi_state[dbi] & ~(DBI_FRESH | DBI_CREAT | DBI_DIRTY))) { + eASSERT(env, (txn->dbi_state[dbi] & (DBI_CREAT | DBI_FRESH | DBI_DIRTY)) != 0 || + (txn->dbi_state[dbi] | DBI_STALE) == + (parent->dbi_state[dbi] & ~(DBI_FRESH | DBI_CREAT | DBI_DIRTY))); + parent->dbs[dbi] = txn->dbs[dbi]; + /* preserve parent's status */ + const uint8_t state = txn->dbi_state[dbi] | (parent->dbi_state[dbi] & (DBI_CREAT | DBI_FRESH | DBI_DIRTY)); + DEBUG("dbi %zu dbi-state %s 0x%02x -> 0x%02x", dbi, (parent->dbi_state[dbi] != state) ? "update" : "still", + parent->dbi_state[dbi], state); + parent->dbi_state[dbi] = state; + } + } - if (flags & MDBX_RDONLY) { - /* Silently ignore irrelevant flags when we're only getting read access */ - flags &= ~(MDBX_WRITEMAP | MDBX_DEPRECATED_MAPASYNC | MDBX_SAFE_NOSYNC | - MDBX_NOMETASYNC | MDBX_DEPRECATED_COALESCE | MDBX_LIFORECLAIM | - MDBX_NOMEMINIT | MDBX_ACCEDE); - mode = 0; - } else { -#if MDBX_MMAP_INCOHERENT_FILE_WRITE - /* Temporary `workaround` for OpenBSD kernel's flaw. - * See https://libmdbx.dqdkfa.ru/dead-github/issues/67 */ - if ((flags & MDBX_WRITEMAP) == 0) { - if (flags & MDBX_ACCEDE) - flags |= MDBX_WRITEMAP; - else { - debug_log(MDBX_LOG_ERROR, __func__, __LINE__, - "System (i.e. OpenBSD) requires MDBX_WRITEMAP because " - "of an internal flaw(s) in a file/buffer/page cache.\n"); - return 42 /* ENOPROTOOPT */; - } + if (latency) { + ts_1 = osal_monotime(); + ts_2 = /* no gc-update */ ts_1; + ts_3 = /* no audit */ ts_2; + ts_4 = /* no write */ ts_3; + ts_5 = /* no sync */ ts_4; } -#endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */ - } + txn_merge(parent, txn, parent_retired_len); + env->txn = parent; + parent->nested = nullptr; + tASSERT(parent, dpl_check(parent)); - MDBX_handle_env_pathname env_pathname; - rc = handle_env_pathname(&env_pathname, pathname, &flags, mode); - if (unlikely(rc != MDBX_SUCCESS)) - goto bailout; +#if MDBX_ENABLE_REFUND + txn_refund(parent); + if (ASSERT_ENABLED()) { + /* Check parent's loose pages not suitable for refund */ + for (page_t *lp = parent->tw.loose_pages; lp; lp = page_next(lp)) { + tASSERT(parent, lp->pgno < parent->tw.loose_refund_wl && lp->pgno + 1 < parent->geo.first_unallocated); + MDBX_ASAN_UNPOISON_MEMORY_REGION(&page_next(lp), sizeof(page_t *)); + VALGRIND_MAKE_MEM_DEFINED(&page_next(lp), sizeof(page_t *)); + } + /* Check parent's reclaimed pages not suitable for refund */ + if (MDBX_PNL_GETSIZE(parent->tw.repnl)) + tASSERT(parent, MDBX_PNL_MOST(parent->tw.repnl) + 1 < parent->geo.first_unallocated); + } +#endif /* MDBX_ENABLE_REFUND */ - env->me_flags = (flags & ~MDBX_FATAL_ERROR) | MDBX_ENV_ACTIVE; - env->me_pathname = osal_calloc(env_pathname.ent_len + 1, sizeof(pathchar_t)); - env->me_dbxs = osal_calloc(env->me_maxdbs, sizeof(MDBX_dbx)); - env->me_dbflags = osal_calloc(env->me_maxdbs, sizeof(env->me_dbflags[0])); - env->me_dbiseqs = osal_calloc(env->me_maxdbs, sizeof(env->me_dbiseqs[0])); - if (!(env->me_dbxs && env->me_pathname && env->me_dbflags && - env->me_dbiseqs)) { - rc = MDBX_ENOMEM; - goto bailout; + txn->signature = 0; + osal_free(txn); + tASSERT(parent, audit_ex(parent, 0, false) == 0); + rc = MDBX_SUCCESS; + goto provide_latency; } - memcpy(env->me_pathname, env_pathname.dxb, - env_pathname.ent_len * sizeof(pathchar_t)); - env->me_dbxs[FREE_DBI].md_cmp = cmp_int_align4; /* aligned MDBX_INTEGERKEY */ - env->me_dbxs[FREE_DBI].md_dcmp = cmp_lenfast; - env->me_dbxs[FREE_DBI].md_klen_max = env->me_dbxs[FREE_DBI].md_klen_min = 8; - env->me_dbxs[FREE_DBI].md_vlen_min = 4; - env->me_dbxs[FREE_DBI].md_vlen_max = - mdbx_env_get_maxvalsize_ex(env, MDBX_INTEGERKEY); - - /* Использование O_DSYNC или FILE_FLAG_WRITE_THROUGH: - * - * 0) Если размер страниц БД меньше системной страницы ОЗУ, то ядру ОС - * придется чаще обновлять страницы в unified page cache. - * - * Однако, O_DSYNC не предполагает отключение unified page cache, - * поэтому подобные затруднения будем считать проблемой ОС и/или - * ожидаемым пенальти из-за использования мелких страниц БД. - * - * 1) В режиме MDBX_SYNC_DURABLE - O_DSYNC для записи как данных, - * так и мета-страниц. Однако, на Linux отказ от O_DSYNC с последующим - * fdatasync() может быть выгоднее при использовании HDD, так как - * позволяет io-scheduler переупорядочить запись с учетом актуального - * расположения файла БД на носителе. - * - * 2) В режиме MDBX_NOMETASYNC - O_DSYNC можно использовать для данных, - * но в этом может не быть смысла, так как fdatasync() всё равно - * требуется для гарантии фиксации мета после предыдущей транзакции. - * - * В итоге на нормальных системах (не Windows) есть два варианта: - * - при возможности O_DIRECT и/или io_ring для данных, скорее всего, - * есть смысл вызвать fdatasync() перед записью данных, а затем - * использовать O_DSYNC; - * - не использовать O_DSYNC и вызывать fdatasync() после записи данных. - * - * На Windows же следует минимизировать использование FlushFileBuffers() - * из-за проблем с производительностью. Поэтому на Windows в режиме - * MDBX_NOMETASYNC: - * - мета обновляется через дескриптор без FILE_FLAG_WRITE_THROUGH; - * - перед началом записи данных вызывается FlushFileBuffers(), если - * mti_meta_sync_txnid не совпадает с последней записанной мета; - * - данные записываются через дескриптор с FILE_FLAG_WRITE_THROUGH. - * - * 3) В режиме MDBX_SAFE_NOSYNC - O_DSYNC нет смысла использовать, пока не - * будет реализована возможность полностью асинхронной "догоняющей" - * записи в выделенном процессе-сервере с io-ring очередями внутри. - * - * ----- - * - * Использование O_DIRECT или FILE_FLAG_NO_BUFFERING: - * - * Назначение этих флагов в отключении файлового дескриптора от - * unified page cache, т.е. от отображенных в память данных в случае - * libmdbx. - * - * Поэтому, использование direct i/o в libmdbx без MDBX_WRITEMAP лишено - * смысла и контр-продуктивно, ибо так мы провоцируем ядро ОС на - * не-когерентность отображения в память с содержимым файла на носителе, - * либо требуем дополнительных проверок и действий направленных на - * фактическое отключение O_DIRECT для отображенных в память данных. - * - * В режиме MDBX_WRITEMAP когерентность отображенных данных обеспечивается - * физически. Поэтому использование direct i/o может иметь смысл, если у - * ядра ОС есть какие-то проблемы с msync(), в том числе с - * производительностью: - * - использование io_ring или gather-write может быть дешевле, чем - * просмотр PTE ядром и запись измененных/грязных; - * - но проблема в том, что записываемые из user mode страницы либо не - * будут помечены чистыми (и соответственно будут записаны ядром - * еще раз), либо ядру необходимо искать и чистить PTE при получении - * запроса на запись. - * - * Поэтому O_DIRECT или FILE_FLAG_NO_BUFFERING используется: - * - только в режиме MDBX_SYNC_DURABLE с MDBX_WRITEMAP; - * - когда me_psize >= me_os_psize; - * - опция сборки MDBX_AVOID_MSYNC != 0, которая по-умолчанию включена - * только на Windows (см ниже). - * - * ----- - * - * Использование FILE_FLAG_OVERLAPPED на Windows: - * - * У Windows очень плохо с I/O (за исключением прямых постраничных - * scatter/gather, которые работают в обход проблемного unified page - * cache и поэтому почти бесполезны в libmdbx). - * - * При этом всё еще хуже при использовании FlushFileBuffers(), что также - * требуется после FlushViewOfFile() в режиме MDBX_WRITEMAP. Поэтому - * на Windows вместо FlushViewOfFile() и FlushFileBuffers() следует - * использовать запись через дескриптор с FILE_FLAG_WRITE_THROUGH. - * - * В свою очередь, запись с FILE_FLAG_WRITE_THROUGH дешевле/быстрее - * при использовании FILE_FLAG_OVERLAPPED. В результате, на Windows - * в durable-режимах запись данных всегда в overlapped-режиме, - * при этом для записи мета требуется отдельный не-overlapped дескриптор. - */ - - rc = osal_openfile((flags & MDBX_RDONLY) ? MDBX_OPEN_DXB_READ - : MDBX_OPEN_DXB_LAZY, - env, env_pathname.dxb, &env->me_lazy_fd, mode); - if (rc != MDBX_SUCCESS) - goto bailout; -#if MDBX_LOCKING == MDBX_LOCKING_SYSV - env->me_sysv_ipc.key = ftok(env_pathname.dxb, 42); - if (env->me_sysv_ipc.key == -1) { - rc = errno; - goto bailout; + if (!txn->tw.dirtylist) { + tASSERT(txn, (txn->flags & MDBX_WRITEMAP) != 0 && !MDBX_AVOID_MSYNC); + } else { + tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); + tASSERT(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length == + (txn->parent ? txn->parent->tw.dirtyroom : env->options.dp_limit)); } -#endif /* MDBX_LOCKING */ - - /* Set the position in files outside of the data to avoid corruption - * due to erroneous use of file descriptors in the application code. */ - const uint64_t safe_parking_lot_offset = UINT64_C(0x7fffFFFF80000000); - osal_fseek(env->me_lazy_fd, safe_parking_lot_offset); - - env->me_fd4meta = env->me_lazy_fd; -#if defined(_WIN32) || defined(_WIN64) - eASSERT(env, env->me_overlapped_fd == 0); - bool ior_direct = false; - if (!(flags & - (MDBX_RDONLY | MDBX_SAFE_NOSYNC | MDBX_NOMETASYNC | MDBX_EXCLUSIVE))) { - if (MDBX_AVOID_MSYNC && (flags & MDBX_WRITEMAP)) { - /* Запрошен режим MDBX_SYNC_DURABLE | MDBX_WRITEMAP при активной опции - * MDBX_AVOID_MSYNC. - * - * 1) В этой комбинации наиболее выгодно использовать WriteFileGather(), - * но для этого необходимо открыть файл с флагом FILE_FLAG_NO_BUFFERING и - * после обеспечивать выравнивание адресов и размера данных на границу - * системной страницы, что в свою очередь возможно если размер страницы БД - * не меньше размера системной страницы ОЗУ. Поэтому для открытия файла в - * нужном режиме требуется знать размер страницы БД. - * - * 2) Кроме этого, в Windows запись в заблокированный регион файла - * возможно только через тот-же дескриптор. Поэтому изначальный захват - * блокировок посредством osal_lck_seize(), захват/освобождение блокировок - * во время пишущих транзакций и запись данных должны выполнятся через - * один дескриптор. - * - * Таким образом, требуется прочитать волатильный заголовок БД, чтобы - * узнать размер страницы, чтобы открыть дескриптор файла в режиме нужном - * для записи данных, чтобы использовать именно этот дескриптор для - * изначального захвата блокировок. */ - MDBX_meta header; - uint64_t dxb_filesize; - int err = read_header(env, &header, MDBX_SUCCESS, true); - if ((err == MDBX_SUCCESS && header.mm_psize >= env->me_os_psize) || - (err == MDBX_ENODATA && mode && env->me_psize >= env->me_os_psize && - osal_filesize(env->me_lazy_fd, &dxb_filesize) == MDBX_SUCCESS && - dxb_filesize == 0)) - /* Может быть коллизия, если два процесса пытаются одновременно создать - * БД с разным размером страницы, который у одного меньше системной - * страницы, а у другого НЕ меньше. Эта допустимая, но очень странная - * ситуация. Поэтому считаем её ошибочной и не пытаемся разрешить. */ - ior_direct = true; - } + txn_done_cursors(txn, false); + end_mode |= TXN_END_EOTDONE; - rc = osal_openfile(ior_direct ? MDBX_OPEN_DXB_OVERLAPPED_DIRECT - : MDBX_OPEN_DXB_OVERLAPPED, - env, env_pathname.dxb, &env->me_overlapped_fd, 0); - if (rc != MDBX_SUCCESS) - goto bailout; - env->me_data_lock_event = CreateEventW(nullptr, true, false, nullptr); - if (!env->me_data_lock_event) { - rc = (int)GetLastError(); - goto bailout; - } - osal_fseek(env->me_overlapped_fd, safe_parking_lot_offset); - } + if ((!txn->tw.dirtylist || txn->tw.dirtylist->length == 0) && + (txn->flags & (MDBX_TXN_DIRTY | MDBX_TXN_SPILLS)) == 0) { + TXN_FOREACH_DBI_ALL(txn, i) { tASSERT(txn, !(txn->dbi_state[i] & DBI_DIRTY)); } +#if defined(MDBX_NOSUCCESS_EMPTY_COMMIT) && MDBX_NOSUCCESS_EMPTY_COMMIT + rc = txn_end(txn, end_mode); + if (unlikely(rc != MDBX_SUCCESS)) + goto fail; + rc = MDBX_RESULT_TRUE; + goto provide_latency; #else - if (mode == 0) { - /* pickup mode for lck-file */ - struct stat st; - if (fstat(env->me_lazy_fd, &st)) { - rc = errno; - goto bailout; - } - mode = st.st_mode; - } - mode = (/* inherit read permissions for group and others */ mode & - (S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)) | - /* always add read/write for owner */ S_IRUSR | S_IWUSR | - ((mode & S_IRGRP) ? /* +write if readable by group */ S_IWGRP : 0) | - ((mode & S_IROTH) ? /* +write if readable by others */ S_IWOTH : 0); -#endif /* !Windows */ - const int lck_rc = setup_lck(env, env_pathname.lck, mode); - if (MDBX_IS_ERROR(lck_rc)) { - rc = lck_rc; - goto bailout; + goto done; +#endif /* MDBX_NOSUCCESS_EMPTY_COMMIT */ } - osal_fseek(env->me_lfd, safe_parking_lot_offset); - eASSERT(env, env->me_dsync_fd == INVALID_HANDLE_VALUE); - if (!(flags & (MDBX_RDONLY | MDBX_SAFE_NOSYNC | MDBX_DEPRECATED_MAPASYNC -#if defined(_WIN32) || defined(_WIN64) - | MDBX_EXCLUSIVE -#endif /* !Windows */ - ))) { - rc = osal_openfile(MDBX_OPEN_DXB_DSYNC, env, env_pathname.dxb, - &env->me_dsync_fd, 0); - if (MDBX_IS_ERROR(rc)) - goto bailout; - if (env->me_dsync_fd != INVALID_HANDLE_VALUE) { - if ((flags & MDBX_NOMETASYNC) == 0) - env->me_fd4meta = env->me_dsync_fd; - osal_fseek(env->me_dsync_fd, safe_parking_lot_offset); + DEBUG("committing txn %" PRIaTXN " %p on env %p, root page %" PRIaPGNO "/%" PRIaPGNO, txn->txnid, (void *)txn, + (void *)env, txn->dbs[MAIN_DBI].root, txn->dbs[FREE_DBI].root); + + if (txn->n_dbi > CORE_DBS) { + /* Update table root pointers */ + cursor_couple_t cx; + rc = cursor_init(&cx.outer, txn, MAIN_DBI); + if (unlikely(rc != MDBX_SUCCESS)) + goto fail; + cx.outer.next = txn->cursors[MAIN_DBI]; + txn->cursors[MAIN_DBI] = &cx.outer; + TXN_FOREACH_DBI_USER(txn, i) { + if ((txn->dbi_state[i] & DBI_DIRTY) == 0) + continue; + tree_t *const db = &txn->dbs[i]; + DEBUG("update main's entry for sub-db %zu, mod_txnid %" PRIaTXN " -> %" PRIaTXN, i, db->mod_txnid, txn->txnid); + /* Может быть mod_txnid > front после коммита вложенных тразакций */ + db->mod_txnid = txn->txnid; + MDBX_val data = {db, sizeof(tree_t)}; + rc = cursor_put(&cx.outer, &env->kvs[i].name, &data, N_TREE); + if (unlikely(rc != MDBX_SUCCESS)) { + txn->cursors[MAIN_DBI] = cx.outer.next; + goto fail; + } } + txn->cursors[MAIN_DBI] = cx.outer.next; } - const MDBX_env_flags_t lazy_flags = - MDBX_SAFE_NOSYNC | MDBX_UTTERLY_NOSYNC | MDBX_NOMETASYNC; - const MDBX_env_flags_t mode_flags = lazy_flags | MDBX_LIFORECLAIM | - MDBX_NORDAHEAD | MDBX_RDONLY | - MDBX_WRITEMAP; + ts_1 = latency ? osal_monotime() : 0; - MDBX_lockinfo *const lck = env->me_lck_mmap.lck; - if (lck && lck_rc != MDBX_RESULT_TRUE && (env->me_flags & MDBX_RDONLY) == 0) { - MDBX_env_flags_t snap_flags; - while ((snap_flags = atomic_load32(&lck->mti_envmode, mo_AcquireRelease)) == - MDBX_RDONLY) { - if (atomic_cas32(&lck->mti_envmode, MDBX_RDONLY, - (snap_flags = (env->me_flags & mode_flags)))) { - /* The case: - * - let's assume that for some reason the DB file is smaller - * than it should be according to the geometry, - * but not smaller than the last page used; - * - the first process that opens the database (lck_rc == RESULT_TRUE) - * does this in readonly mode and therefore cannot bring - * the file size back to normal; - * - some next process (lck_rc != RESULT_TRUE) opens the DB in - * read-write mode and now is here. - * - * FIXME: Should we re-check and set the size of DB-file right here? */ - break; - } - atomic_yield(); - } + gcu_t gcu_ctx; + gc_cputime = latency ? osal_cputime(nullptr) : 0; + rc = gc_update_init(txn, &gcu_ctx); + if (unlikely(rc != MDBX_SUCCESS)) + goto fail; + rc = gc_update(txn, &gcu_ctx); + gc_cputime = latency ? osal_cputime(nullptr) - gc_cputime : 0; + if (unlikely(rc != MDBX_SUCCESS)) + goto fail; - if (env->me_flags & MDBX_ACCEDE) { - /* Pickup current mode-flags (MDBX_LIFORECLAIM, MDBX_NORDAHEAD, etc). */ - const MDBX_env_flags_t diff = - (snap_flags ^ env->me_flags) & - ((snap_flags & lazy_flags) ? mode_flags - : mode_flags & ~MDBX_WRITEMAP); - env->me_flags ^= diff; - NOTICE("accede mode-flags: 0x%X, 0x%X -> 0x%X", diff, - env->me_flags ^ diff, env->me_flags); - } + tASSERT(txn, txn->tw.loose_count == 0); + txn->dbs[FREE_DBI].mod_txnid = (txn->dbi_state[FREE_DBI] & DBI_DIRTY) ? txn->txnid : txn->dbs[FREE_DBI].mod_txnid; - /* Ранее упущенный не очевидный момент: При работе БД в режимах - * не-синхронной/отложенной фиксации на диске, все процессы-писатели должны - * иметь одинаковый режим MDBX_WRITEMAP. + txn->dbs[MAIN_DBI].mod_txnid = (txn->dbi_state[MAIN_DBI] & DBI_DIRTY) ? txn->txnid : txn->dbs[MAIN_DBI].mod_txnid; + + ts_2 = latency ? osal_monotime() : 0; + ts_3 = ts_2; + if (AUDIT_ENABLED()) { + rc = audit_ex(txn, MDBX_PNL_GETSIZE(txn->tw.retired_pages), true); + ts_3 = osal_monotime(); + if (unlikely(rc != MDBX_SUCCESS)) + goto fail; + } + + bool need_flush_for_nometasync = false; + const meta_ptr_t head = meta_recent(env, &txn->tw.troika); + const uint32_t meta_sync_txnid = atomic_load32(&env->lck->meta_sync_txnid, mo_Relaxed); + /* sync prev meta */ + if (head.is_steady && meta_sync_txnid != (uint32_t)head.txnid) { + /* Исправление унаследованного от LMDB недочета: * - * В противном случае, сброс на диск следует выполнять дважды: сначала - * msync(), затем fdatasync(). При этом msync() не обязан отрабатывать - * в процессах без MDBX_WRITEMAP, так как файл в память отображен только - * для чтения. Поэтому, в общем случае, различия по MDBX_WRITEMAP не - * позволяют выполнить фиксацию данных на диск, после их изменения в другом - * процессе. + * Всё хорошо, если все процессы работающие с БД не используют WRITEMAP. + * Тогда мета-страница (обновленная, но не сброшенная на диск) будет + * сохранена в результате fdatasync() при записи данных этой транзакции. * - * В режиме MDBX_UTTERLY_NOSYNC позволять совместную работу с MDBX_WRITEMAP - * также не следует, поскольку никакой процесс (в том числе последний) не - * может гарантированно сбросить данные на диск, а следовательно не должен - * помечать какую-либо транзакцию как steady. + * Всё хорошо, если все процессы работающие с БД используют WRITEMAP + * без MDBX_AVOID_MSYNC. + * Тогда мета-страница (обновленная, но не сброшенная на диск) будет + * сохранена в результате msync() при записи данных этой транзакции. * - * В результате, требуется либо запретить совместную работу процессам с - * разным MDBX_WRITEMAP в режиме отложенной записи, либо отслеживать такое - * смешивание и блокировать steady-пометки - что контрпродуктивно. */ - const MDBX_env_flags_t rigorous_flags = - (snap_flags & lazy_flags) - ? MDBX_SAFE_NOSYNC | MDBX_UTTERLY_NOSYNC | MDBX_WRITEMAP - : MDBX_SAFE_NOSYNC | MDBX_UTTERLY_NOSYNC; - const MDBX_env_flags_t rigorous_diff = - (snap_flags ^ env->me_flags) & rigorous_flags; - if (rigorous_diff) { - ERROR("current mode/flags 0x%X incompatible with requested 0x%X, " - "rigorous diff 0x%X", - env->me_flags, snap_flags, rigorous_diff); - rc = MDBX_INCOMPATIBLE; - goto bailout; + * Если же в процессах работающих с БД используется оба метода, как sync() + * в режиме MDBX_WRITEMAP, так и записи через файловый дескриптор, то + * становится невозможным обеспечить фиксацию на диске мета-страницы + * предыдущей транзакции и данных текущей транзакции, за счет одной + * sync-операцией выполняемой после записи данных текущей транзакции. + * Соответственно, требуется явно обновлять мета-страницу, что полностью + * уничтожает выгоду от NOMETASYNC. */ + const uint32_t txnid_dist = ((txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC) ? MDBX_NOMETASYNC_LAZY_FD + : MDBX_NOMETASYNC_LAZY_WRITEMAP; + /* Смысл "магии" в том, чтобы избежать отдельного вызова fdatasync() + * или msync() для гарантированной фиксации на диске мета-страницы, + * которая была "лениво" отправлена на запись в предыдущей транзакции, + * но не сброшена на диск из-за активного режима MDBX_NOMETASYNC. */ + if ( +#if defined(_WIN32) || defined(_WIN64) + !env->ioring.overlapped_fd && +#endif + meta_sync_txnid == (uint32_t)head.txnid - txnid_dist) + need_flush_for_nometasync = true; + else { + rc = meta_sync(env, head); + if (unlikely(rc != MDBX_SUCCESS)) { + ERROR("txn-%s: error %d", "presync-meta", rc); + goto fail; + } } } - mincore_clean_cache(env); - const int dxb_rc = setup_dxb(env, lck_rc, mode); - if (MDBX_IS_ERROR(dxb_rc)) { - rc = dxb_rc; - goto bailout; - } - - rc = osal_check_fs_incore(env->me_lazy_fd); - env->me_incore = false; - if (rc == MDBX_RESULT_TRUE) { - env->me_incore = true; - NOTICE("%s", "in-core database"); - rc = MDBX_SUCCESS; - } else if (unlikely(rc != MDBX_SUCCESS)) { - ERROR("check_fs_incore(), err %d", rc); - goto bailout; - } + if (txn->tw.dirtylist) { + tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); + tASSERT(txn, txn->tw.loose_count == 0); - if (unlikely(/* recovery mode */ env->me_stuck_meta >= 0) && - (lck_rc != /* exclusive */ MDBX_RESULT_TRUE || - (flags & MDBX_EXCLUSIVE) == 0)) { - ERROR("%s", "recovery requires exclusive mode"); - rc = MDBX_BUSY; - goto bailout; - } + mdbx_filehandle_t fd = +#if defined(_WIN32) || defined(_WIN64) + env->ioring.overlapped_fd ? env->ioring.overlapped_fd : env->lazy_fd; + (void)need_flush_for_nometasync; +#else + (need_flush_for_nometasync || env->dsync_fd == INVALID_HANDLE_VALUE || + txn->tw.dirtylist->length > env->options.writethrough_threshold || + atomic_load64(&env->lck->unsynced_pages, mo_Relaxed)) + ? env->lazy_fd + : env->dsync_fd; +#endif /* Windows */ - DEBUG("opened dbenv %p", (void *)env); - if (!lck || lck_rc == MDBX_RESULT_TRUE) { - env->me_lck->mti_envmode.weak = env->me_flags & mode_flags; - env->me_lck->mti_meta_sync_txnid.weak = - (uint32_t)recent_committed_txnid(env); - env->me_lck->mti_reader_check_timestamp.weak = osal_monotime(); - } - if (lck) { - if (lck_rc == MDBX_RESULT_TRUE) { - rc = osal_lck_downgrade(env); - DEBUG("lck-downgrade-%s: rc %i", - (env->me_flags & MDBX_EXCLUSIVE) ? "partial" : "full", rc); - if (rc != MDBX_SUCCESS) - goto bailout; - } else { - rc = cleanup_dead_readers(env, false, NULL); - if (MDBX_IS_ERROR(rc)) - goto bailout; + iov_ctx_t write_ctx; + rc = iov_init(txn, &write_ctx, txn->tw.dirtylist->length, txn->tw.dirtylist->pages_including_loose, fd, false); + if (unlikely(rc != MDBX_SUCCESS)) { + ERROR("txn-%s: error %d", "iov-init", rc); + goto fail; } - if ((env->me_flags & MDBX_NOTLS) == 0) { - rc = rthc_alloc(&env->me_txkey, &lck->mti_readers[0], - &lck->mti_readers[env->me_maxreaders]); - if (unlikely(rc != MDBX_SUCCESS)) - goto bailout; - env->me_flags |= MDBX_ENV_TXKEY; + rc = txn_write(txn, &write_ctx); + if (unlikely(rc != MDBX_SUCCESS)) { + ERROR("txn-%s: error %d", "write", rc); + goto fail; } + } else { + tASSERT(txn, (txn->flags & MDBX_WRITEMAP) != 0 && !MDBX_AVOID_MSYNC); + env->lck->unsynced_pages.weak += txn->tw.writemap_dirty_npages; + if (!env->lck->eoos_timestamp.weak) + env->lck->eoos_timestamp.weak = osal_monotime(); } - if ((flags & MDBX_RDONLY) == 0) { - const size_t tsize = sizeof(MDBX_txn) + sizeof(MDBX_cursor), - size = tsize + env->me_maxdbs * - (sizeof(MDBX_db) + sizeof(MDBX_cursor *) + - sizeof(MDBX_atomic_uint32_t) + 1); - rc = alloc_page_buf(env); - if (rc == MDBX_SUCCESS) { - memset(env->me_pbuf, -1, env->me_psize * (size_t)2); - memset(ptr_disp(env->me_pbuf, env->me_psize * (size_t)2), 0, - env->me_psize); - MDBX_txn *txn = osal_calloc(1, size); - if (txn) { - txn->mt_dbs = ptr_disp(txn, tsize); - txn->mt_cursors = - ptr_disp(txn->mt_dbs, sizeof(MDBX_db) * env->me_maxdbs); - txn->mt_dbiseqs = - ptr_disp(txn->mt_cursors, sizeof(MDBX_cursor *) * env->me_maxdbs); - txn->mt_dbistate = ptr_disp( - txn->mt_dbiseqs, sizeof(MDBX_atomic_uint32_t) * env->me_maxdbs); - txn->mt_env = env; - txn->mt_dbxs = env->me_dbxs; - txn->mt_flags = MDBX_TXN_FINISHED; - env->me_txn0 = txn; - txn->tw.retired_pages = pnl_alloc(MDBX_PNL_INITIAL); - txn->tw.relist = pnl_alloc(MDBX_PNL_INITIAL); - if (unlikely(!txn->tw.retired_pages || !txn->tw.relist)) - rc = MDBX_ENOMEM; - } else - rc = MDBX_ENOMEM; - } - if (rc == MDBX_SUCCESS) - rc = osal_ioring_create(&env->me_ioring -#if defined(_WIN32) || defined(_WIN64) - , - ior_direct, env->me_overlapped_fd -#endif /* Windows */ - ); - if (rc == MDBX_SUCCESS) - adjust_defaults(env); + /* TODO: use ctx.flush_begin & ctx.flush_end for range-sync */ + ts_4 = latency ? osal_monotime() : 0; + + meta_t meta; + memcpy(meta.magic_and_version, head.ptr_c->magic_and_version, 8); + meta.reserve16 = head.ptr_c->reserve16; + meta.validator_id = head.ptr_c->validator_id; + meta.extra_pagehdr = head.ptr_c->extra_pagehdr; + unaligned_poke_u64(4, meta.pages_retired, + unaligned_peek_u64(4, head.ptr_c->pages_retired) + MDBX_PNL_GETSIZE(txn->tw.retired_pages)); + meta.geometry = txn->geo; + meta.trees.gc = txn->dbs[FREE_DBI]; + meta.trees.main = txn->dbs[MAIN_DBI]; + meta.canary = txn->canary; + memcpy(&meta.dxbid, &head.ptr_c->dxbid, sizeof(meta.dxbid)); + + txnid_t commit_txnid = txn->txnid; +#if MDBX_ENABLE_BIGFOOT + if (gcu_ctx.bigfoot > txn->txnid) { + commit_txnid = gcu_ctx.bigfoot; + TRACE("use @%" PRIaTXN " (+%zu) for commit bigfoot-txn", commit_txnid, (size_t)(commit_txnid - txn->txnid)); } +#endif + meta.unsafe_sign = DATASIGN_NONE; + meta_set_txnid(env, &meta, commit_txnid); -#if MDBX_DEBUG - if (rc == MDBX_SUCCESS) { - const meta_troika_t troika = meta_tap(env); - const meta_ptr_t head = meta_recent(env, &troika); - const MDBX_db *db = &head.ptr_c->mm_dbs[MAIN_DBI]; + rc = dxb_sync_locked(env, env->flags | txn->flags | txn_shrink_allowed, &meta, &txn->tw.troika); - DEBUG("opened database version %u, pagesize %u", - (uint8_t)unaligned_peek_u64(4, head.ptr_c->mm_magic_and_version), - env->me_psize); - DEBUG("using meta page %" PRIaPGNO ", txn %" PRIaTXN, - data_page(head.ptr_c)->mp_pgno, head.txnid); - DEBUG("depth: %u", db->md_depth); - DEBUG("entries: %" PRIu64, db->md_entries); - DEBUG("branch pages: %" PRIaPGNO, db->md_branch_pages); - DEBUG("leaf pages: %" PRIaPGNO, db->md_leaf_pages); - DEBUG("large/overflow pages: %" PRIaPGNO, db->md_overflow_pages); - DEBUG("root: %" PRIaPGNO, db->md_root); - DEBUG("schema_altered: %" PRIaTXN, db->md_mod_txnid); + ts_5 = latency ? osal_monotime() : 0; + if (unlikely(rc != MDBX_SUCCESS)) { + env->flags |= ENV_FATAL_ERROR; + ERROR("txn-%s: error %d", "sync", rc); + goto fail; } -#endif -bailout: - if (rc != MDBX_SUCCESS) { - rc = env_close(env) ? MDBX_PANIC : rc; - env->me_flags = - saved_me_flags | ((rc != MDBX_PANIC) ? 0 : MDBX_FATAL_ERROR); - } else { -#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__) - txn_valgrind(env, nullptr); -#endif /* MDBX_USE_VALGRIND || __SANITIZE_ADDRESS__ */ + end_mode = TXN_END_COMMITTED | TXN_END_UPDATE | TXN_END_EOTDONE; + +done: + if (latency) + txn_take_gcprof(txn, latency); + rc = txn_end(txn, end_mode); + +provide_latency: + if (latency) { + latency->preparation = ts_1 ? osal_monotime_to_16dot16(ts_1 - ts_0) : 0; + latency->gc_wallclock = (ts_2 > ts_1) ? osal_monotime_to_16dot16(ts_2 - ts_1) : 0; + latency->gc_cputime = gc_cputime ? osal_monotime_to_16dot16(gc_cputime) : 0; + latency->audit = (ts_3 > ts_2) ? osal_monotime_to_16dot16(ts_3 - ts_2) : 0; + latency->write = (ts_4 > ts_3) ? osal_monotime_to_16dot16(ts_4 - ts_3) : 0; + latency->sync = (ts_5 > ts_4) ? osal_monotime_to_16dot16(ts_5 - ts_4) : 0; + const uint64_t ts_6 = osal_monotime(); + latency->ending = ts_5 ? osal_monotime_to_16dot16(ts_6 - ts_5) : 0; + latency->whole = osal_monotime_to_16dot16_noUnderflow(ts_6 - ts_0); } - osal_free(env_pathname.buffer_for_free); - return rc; + return LOG_IFERR(rc); + +fail: + txn->flags |= MDBX_TXN_ERROR; + if (latency) + txn_take_gcprof(txn, latency); + txn_abort(txn); + goto provide_latency; } -/* Destroy resources from mdbx_env_open(), clear our readers & DBIs */ -__cold static int env_close(MDBX_env *env) { - const unsigned flags = env->me_flags; - if (!(flags & MDBX_ENV_ACTIVE)) { - ENSURE(env, env->me_lcklist_next == nullptr); - return MDBX_SUCCESS; - } +int mdbx_txn_info(const MDBX_txn *txn, MDBX_txn_info *info, bool scan_rlt) { + int rc = check_txn(txn, MDBX_TXN_FINISHED); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); - env->me_flags &= ~ENV_INTERNAL_FLAGS; - if (flags & MDBX_ENV_TXKEY) { - rthc_remove(env->me_txkey); - env->me_txkey = (osal_thread_key_t)0; + if (unlikely(!info)) + return LOG_IFERR(MDBX_EINVAL); + + MDBX_env *const env = txn->env; +#if MDBX_ENV_CHECKPID + if (unlikely(env->pid != osal_getpid())) { + env->flags |= ENV_FATAL_ERROR; + return LOG_IFERR(MDBX_PANIC); } +#endif /* MDBX_ENV_CHECKPID */ - munlock_all(env); - if (!(env->me_flags & MDBX_RDONLY)) - osal_ioring_destroy(&env->me_ioring); + info->txn_id = txn->txnid; + info->txn_space_used = pgno2bytes(env, txn->geo.first_unallocated); - lcklist_lock(); - const int rc = lcklist_detach_locked(env); - lcklist_unlock(); + if (txn->flags & MDBX_TXN_RDONLY) { + meta_ptr_t head; + uint64_t head_retired; + troika_t troika = meta_tap(env); + do { + /* fetch info from volatile head */ + head = meta_recent(env, &troika); + head_retired = unaligned_peek_u64_volatile(4, head.ptr_v->pages_retired); + info->txn_space_limit_soft = pgno2bytes(env, head.ptr_v->geometry.now); + info->txn_space_limit_hard = pgno2bytes(env, head.ptr_v->geometry.upper); + info->txn_space_leftover = pgno2bytes(env, head.ptr_v->geometry.now - head.ptr_v->geometry.first_unallocated); + } while (unlikely(meta_should_retry(env, &troika))); - env->me_lck = nullptr; - if (env->me_lck_mmap.lck) - osal_munmap(&env->me_lck_mmap); + info->txn_reader_lag = head.txnid - info->txn_id; + info->txn_space_dirty = info->txn_space_retired = 0; + uint64_t reader_snapshot_pages_retired = 0; + if (txn->to.reader && + ((txn->flags & MDBX_TXN_PARKED) == 0 || safe64_read(&txn->to.reader->tid) != MDBX_TID_TXN_OUSTED) && + head_retired > + (reader_snapshot_pages_retired = atomic_load64(&txn->to.reader->snapshot_pages_retired, mo_Relaxed))) { + info->txn_space_dirty = info->txn_space_retired = + pgno2bytes(env, (pgno_t)(head_retired - reader_snapshot_pages_retired)); - if (env->me_map) { - osal_munmap(&env->me_dxb_mmap); -#ifdef MDBX_USE_VALGRIND - VALGRIND_DISCARD(env->me_valgrind_handle); - env->me_valgrind_handle = -1; -#endif + size_t retired_next_reader = 0; + lck_t *const lck = env->lck_mmap.lck; + if (scan_rlt && info->txn_reader_lag > 1 && lck) { + /* find next more recent reader */ + txnid_t next_reader = head.txnid; + const size_t snap_nreaders = atomic_load32(&lck->rdt_length, mo_AcquireRelease); + for (size_t i = 0; i < snap_nreaders; ++i) { + retry: + if (atomic_load32(&lck->rdt[i].pid, mo_AcquireRelease)) { + jitter4testing(true); + const uint64_t snap_tid = safe64_read(&lck->rdt[i].tid); + const txnid_t snap_txnid = safe64_read(&lck->rdt[i].txnid); + const uint64_t snap_retired = atomic_load64(&lck->rdt[i].snapshot_pages_retired, mo_AcquireRelease); + if (unlikely(snap_retired != atomic_load64(&lck->rdt[i].snapshot_pages_retired, mo_Relaxed)) || + snap_txnid != safe64_read(&lck->rdt[i].txnid) || snap_tid != safe64_read(&lck->rdt[i].tid)) + goto retry; + if (snap_txnid <= txn->txnid) { + retired_next_reader = 0; + break; + } + if (snap_txnid < next_reader && snap_tid >= MDBX_TID_TXN_OUSTED) { + next_reader = snap_txnid; + retired_next_reader = pgno2bytes( + env, (pgno_t)(snap_retired - atomic_load64(&txn->to.reader->snapshot_pages_retired, mo_Relaxed))); + } + } + } + } + info->txn_space_dirty = retired_next_reader; + } + } else { + info->txn_space_limit_soft = pgno2bytes(env, txn->geo.now); + info->txn_space_limit_hard = pgno2bytes(env, txn->geo.upper); + info->txn_space_retired = + pgno2bytes(env, txn->nested ? (size_t)txn->tw.retired_pages : MDBX_PNL_GETSIZE(txn->tw.retired_pages)); + info->txn_space_leftover = pgno2bytes(env, txn->tw.dirtyroom); + info->txn_space_dirty = + pgno2bytes(env, txn->tw.dirtylist ? txn->tw.dirtylist->pages_including_loose + : (txn->tw.writemap_dirty_npages + txn->tw.writemap_spilled_npages)); + info->txn_reader_lag = INT64_MAX; + lck_t *const lck = env->lck_mmap.lck; + if (scan_rlt && lck) { + txnid_t oldest_snapshot = txn->txnid; + const size_t snap_nreaders = atomic_load32(&lck->rdt_length, mo_AcquireRelease); + if (snap_nreaders) { + oldest_snapshot = txn_snapshot_oldest(txn); + if (oldest_snapshot == txn->txnid - 1) { + /* check if there is at least one reader */ + bool exists = false; + for (size_t i = 0; i < snap_nreaders; ++i) { + if (atomic_load32(&lck->rdt[i].pid, mo_Relaxed) && txn->txnid > safe64_read(&lck->rdt[i].txnid)) { + exists = true; + break; + } + } + oldest_snapshot += !exists; + } + } + info->txn_reader_lag = txn->txnid - oldest_snapshot; + } } -#if defined(_WIN32) || defined(_WIN64) - eASSERT(env, !env->me_overlapped_fd || - env->me_overlapped_fd == INVALID_HANDLE_VALUE); - if (env->me_data_lock_event != INVALID_HANDLE_VALUE) { - CloseHandle(env->me_data_lock_event); - env->me_data_lock_event = INVALID_HANDLE_VALUE; - } -#endif /* Windows */ + return MDBX_SUCCESS; +} +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 - if (env->me_dsync_fd != INVALID_HANDLE_VALUE) { - (void)osal_closefile(env->me_dsync_fd); - env->me_dsync_fd = INVALID_HANDLE_VALUE; - } +struct audit_ctx { + size_t used; + uint8_t *const done_bitmap; +}; - if (env->me_lazy_fd != INVALID_HANDLE_VALUE) { - (void)osal_closefile(env->me_lazy_fd); - env->me_lazy_fd = INVALID_HANDLE_VALUE; - } +static int audit_dbi(void *ctx, const MDBX_txn *txn, const MDBX_val *name, MDBX_db_flags_t flags, + const struct MDBX_stat *stat, MDBX_dbi dbi) { + struct audit_ctx *audit_ctx = ctx; + (void)name; + (void)txn; + (void)flags; + audit_ctx->used += (size_t)stat->ms_branch_pages + (size_t)stat->ms_leaf_pages + (size_t)stat->ms_overflow_pages; + if (dbi) + audit_ctx->done_bitmap[dbi / CHAR_BIT] |= 1 << dbi % CHAR_BIT; + return MDBX_SUCCESS; +} - if (env->me_lfd != INVALID_HANDLE_VALUE) { - (void)osal_closefile(env->me_lfd); - env->me_lfd = INVALID_HANDLE_VALUE; - } +static size_t audit_db_used(const tree_t *db) { + return db ? (size_t)db->branch_pages + (size_t)db->leaf_pages + (size_t)db->large_pages : 0; +} - if (env->me_dbxs) { - for (size_t i = CORE_DBS; i < env->me_numdbs; ++i) - if (env->me_dbxs[i].md_name.iov_len) - osal_free(env->me_dbxs[i].md_name.iov_base); - osal_free(env->me_dbxs); - env->me_numdbs = CORE_DBS; - env->me_dbxs = nullptr; - } - if (env->me_pbuf) { - osal_memalign_free(env->me_pbuf); - env->me_pbuf = nullptr; - } - if (env->me_dbiseqs) { - osal_free(env->me_dbiseqs); - env->me_dbiseqs = nullptr; +__cold static int audit_ex_locked(MDBX_txn *txn, size_t retired_stored, bool dont_filter_gc) { + const MDBX_env *const env = txn->env; + size_t pending = 0; + if ((txn->flags & MDBX_TXN_RDONLY) == 0) + pending = txn->tw.loose_count + MDBX_PNL_GETSIZE(txn->tw.repnl) + + (MDBX_PNL_GETSIZE(txn->tw.retired_pages) - retired_stored); + + cursor_couple_t cx; + int rc = cursor_init(&cx.outer, txn, FREE_DBI); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + size_t gc = 0; + MDBX_val key, data; + rc = outer_first(&cx.outer, &key, &data); + while (rc == MDBX_SUCCESS) { + if (!dont_filter_gc) { + if (unlikely(key.iov_len != sizeof(txnid_t))) { + ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid GC-key size", (unsigned)key.iov_len); + return MDBX_CORRUPTED; + } + txnid_t id = unaligned_peek_u64(4, key.iov_base); + if (txn->tw.gc.retxl ? txl_contain(txn->tw.gc.retxl, id) : (id <= txn->tw.gc.last_reclaimed)) + goto skip; + } + gc += *(pgno_t *)data.iov_base; + skip: + rc = outer_next(&cx.outer, &key, &data, MDBX_NEXT); } - if (env->me_dbflags) { - osal_free(env->me_dbflags); - env->me_dbflags = nullptr; + tASSERT(txn, rc == MDBX_NOTFOUND); + + const size_t done_bitmap_size = (txn->n_dbi + CHAR_BIT - 1) / CHAR_BIT; + if (txn->parent) { + tASSERT(txn, txn->n_dbi == txn->parent->n_dbi && txn->n_dbi == txn->env->txn->n_dbi); +#if MDBX_ENABLE_DBI_SPARSE + tASSERT(txn, txn->dbi_sparse == txn->parent->dbi_sparse && txn->dbi_sparse == txn->env->txn->dbi_sparse); +#endif /* MDBX_ENABLE_DBI_SPARSE */ } - if (env->me_pathname) { - osal_free(env->me_pathname); - env->me_pathname = nullptr; + + struct audit_ctx ctx = {0, alloca(done_bitmap_size)}; + memset(ctx.done_bitmap, 0, done_bitmap_size); + ctx.used = + NUM_METAS + audit_db_used(dbi_dig(txn, FREE_DBI, nullptr)) + audit_db_used(dbi_dig(txn, MAIN_DBI, nullptr)); + + rc = mdbx_enumerate_tables(txn, audit_dbi, &ctx); + tASSERT(txn, rc == MDBX_SUCCESS); + + for (size_t dbi = CORE_DBS; dbi < txn->n_dbi; ++dbi) { + if (ctx.done_bitmap[dbi / CHAR_BIT] & (1 << dbi % CHAR_BIT)) + continue; + const tree_t *db = dbi_dig(txn, dbi, nullptr); + if (db) + ctx.used += audit_db_used(db); + else if (dbi_state(txn, dbi)) + WARNING("audit %s@%" PRIaTXN ": unable account dbi %zd / \"%.*s\", state 0x%02x", txn->parent ? "nested-" : "", + txn->txnid, dbi, (int)env->kvs[dbi].name.iov_len, (const char *)env->kvs[dbi].name.iov_base, + dbi_state(txn, dbi)); } -#if defined(_WIN32) || defined(_WIN64) - if (env->me_pathname_char) { - osal_free(env->me_pathname_char); - env->me_pathname_char = nullptr; + + if (pending + gc + ctx.used == txn->geo.first_unallocated) + return MDBX_SUCCESS; + + if ((txn->flags & MDBX_TXN_RDONLY) == 0) + ERROR("audit @%" PRIaTXN ": %zu(pending) = %zu(loose) + " + "%zu(reclaimed) + %zu(retired-pending) - %zu(retired-stored)", + txn->txnid, pending, txn->tw.loose_count, MDBX_PNL_GETSIZE(txn->tw.repnl), + txn->tw.retired_pages ? MDBX_PNL_GETSIZE(txn->tw.retired_pages) : 0, retired_stored); + ERROR("audit @%" PRIaTXN ": %zu(pending) + %zu" + "(gc) + %zu(count) = %zu(total) <> %zu" + "(allocated)", + txn->txnid, pending, gc, ctx.used, pending + gc + ctx.used, (size_t)txn->geo.first_unallocated); + return MDBX_PROBLEM; +} + +__cold int audit_ex(MDBX_txn *txn, size_t retired_stored, bool dont_filter_gc) { + MDBX_env *const env = txn->env; + int rc = osal_fastmutex_acquire(&env->dbi_lock); + if (likely(rc == MDBX_SUCCESS)) { + rc = audit_ex_locked(txn, retired_stored, dont_filter_gc); + ENSURE(txn->env, osal_fastmutex_release(&env->dbi_lock) == MDBX_SUCCESS); } -#endif /* Windows */ - if (env->me_txn0) { - dpl_free(env->me_txn0); - txl_free(env->me_txn0->tw.lifo_reclaimed); - pnl_free(env->me_txn0->tw.retired_pages); - pnl_free(env->me_txn0->tw.spilled.list); - pnl_free(env->me_txn0->tw.relist); - osal_free(env->me_txn0); - env->me_txn0 = nullptr; - } - env->me_stuck_meta = -1; return rc; } +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 -__cold int mdbx_env_close_ex(MDBX_env *env, bool dont_sync) { - MDBX_page *dp; - int rc = MDBX_SUCCESS; - - if (unlikely(!env)) - return MDBX_EINVAL; +typedef struct MDBX_chk_internal { + MDBX_chk_context_t *usr; + const struct MDBX_chk_callbacks *cb; + uint64_t monotime_timeout; - if (unlikely(env->me_signature.weak != MDBX_ME_SIGNATURE)) - return MDBX_EBADSIGN; + size_t *problem_counter; + uint8_t flags; + bool got_break; + bool write_locked; + uint8_t scope_depth; -#if MDBX_ENV_CHECKPID || !(defined(_WIN32) || defined(_WIN64)) - /* Check the PID even if MDBX_ENV_CHECKPID=0 on non-Windows - * platforms (i.e. where fork() is available). - * This is required to legitimize a call after fork() - * from a child process, that should be allowed to free resources. */ - if (unlikely(env->me_pid != osal_getpid())) - env->me_flags |= MDBX_FATAL_ERROR; -#endif /* MDBX_ENV_CHECKPID */ + MDBX_chk_table_t table_gc, table_main; + int16_t *pagemap; + MDBX_chk_table_t *last_lookup; + const void *last_nested; + MDBX_chk_scope_t scope_stack[12]; + MDBX_chk_table_t *table[MDBX_MAX_DBI + CORE_DBS]; - if (env->me_map && (env->me_flags & (MDBX_RDONLY | MDBX_FATAL_ERROR)) == 0 && - env->me_txn0) { - if (env->me_txn0->mt_owner && env->me_txn0->mt_owner != osal_thread_self()) - return MDBX_BUSY; - } else - dont_sync = true; + MDBX_envinfo envinfo; + troika_t troika; + MDBX_val v2a_buf; +} MDBX_chk_internal_t; - if (!atomic_cas32(&env->me_signature, MDBX_ME_SIGNATURE, 0)) - return MDBX_EBADSIGN; +__cold static int chk_check_break(MDBX_chk_scope_t *const scope) { + MDBX_chk_internal_t *const chk = scope->internal; + return (chk->got_break || (chk->cb->check_break && (chk->got_break = chk->cb->check_break(chk->usr)))) + ? MDBX_RESULT_TRUE + : MDBX_RESULT_FALSE; +} - if (!dont_sync) { -#if defined(_WIN32) || defined(_WIN64) - /* On windows, without blocking is impossible to determine whether another - * process is running a writing transaction or not. - * Because in the "owner died" condition kernel don't release - * file lock immediately. */ - rc = env_sync(env, true, false); - rc = (rc == MDBX_RESULT_TRUE) ? MDBX_SUCCESS : rc; -#else - struct stat st; - if (unlikely(fstat(env->me_lazy_fd, &st))) - rc = errno; - else if (st.st_nlink > 0 /* don't sync deleted files */) { - rc = env_sync(env, true, true); - rc = (rc == MDBX_BUSY || rc == EAGAIN || rc == EACCES || rc == EBUSY || - rc == EWOULDBLOCK || rc == MDBX_RESULT_TRUE) - ? MDBX_SUCCESS - : rc; - } -#endif /* Windows */ +__cold static void chk_line_end(MDBX_chk_line_t *line) { + if (likely(line)) { + MDBX_chk_internal_t *chk = line->ctx->internal; + assert(line->begin <= line->end && line->begin <= line->out && line->out <= line->end); + if (likely(chk->cb->print_done)) + chk->cb->print_done(line); } +} - eASSERT(env, env->me_signature.weak == 0); - rc = env_close(env) ? MDBX_PANIC : rc; - ENSURE(env, osal_fastmutex_destroy(&env->me_dbi_lock) == MDBX_SUCCESS); -#if defined(_WIN32) || defined(_WIN64) - /* me_remap_guard don't have destructor (Slim Reader/Writer Lock) */ - DeleteCriticalSection(&env->me_windowsbug_lock); -#else - ENSURE(env, osal_fastmutex_destroy(&env->me_remap_guard) == MDBX_SUCCESS); -#endif /* Windows */ - -#if MDBX_LOCKING > MDBX_LOCKING_SYSV - MDBX_lockinfo *const stub = lckless_stub(env); - ENSURE(env, osal_ipclock_destroy(&stub->mti_wlock) == 0); -#endif /* MDBX_LOCKING */ - - while ((dp = env->me_dp_reserve) != NULL) { - MDBX_ASAN_UNPOISON_MEMORY_REGION(dp, env->me_psize); - VALGRIND_MAKE_MEM_DEFINED(&mp_next(dp), sizeof(MDBX_page *)); - env->me_dp_reserve = mp_next(dp); - void *const ptr = ptr_disp(dp, -(ptrdiff_t)sizeof(size_t)); - osal_free(ptr); +__cold __must_check_result static MDBX_chk_line_t *chk_line_begin(MDBX_chk_scope_t *const scope, + enum MDBX_chk_severity severity) { + MDBX_chk_internal_t *const chk = scope->internal; + if (severity < MDBX_chk_warning) + mdbx_env_chk_encount_problem(chk->usr); + MDBX_chk_line_t *line = nullptr; + if (likely(chk->cb->print_begin)) { + line = chk->cb->print_begin(chk->usr, severity); + if (likely(line)) { + assert(line->ctx == nullptr || (line->ctx == chk->usr && line->empty)); + assert(line->begin <= line->end && line->begin <= line->out && line->out <= line->end); + line->ctx = chk->usr; + } } - VALGRIND_DESTROY_MEMPOOL(env); - ENSURE(env, env->me_lcklist_next == nullptr); - env->me_pid = 0; - osal_free(env); - - return rc; + return line; } -/* Search for key within a page, using binary search. - * Returns the smallest entry larger or equal to the key. - * Updates the cursor index with the index of the found entry. - * If no entry larger or equal to the key is found, returns NULL. */ -__hot static struct node_result node_search(MDBX_cursor *mc, - const MDBX_val *key) { - MDBX_page *mp = mc->mc_pg[mc->mc_top]; - const intptr_t nkeys = page_numkeys(mp); - DKBUF_DEBUG; - - DEBUG("searching %zu keys in %s %spage %" PRIaPGNO, nkeys, - IS_LEAF(mp) ? "leaf" : "branch", IS_SUBP(mp) ? "sub-" : "", - mp->mp_pgno); - - struct node_result ret; - ret.exact = false; - STATIC_ASSERT(P_BRANCH == 1); - intptr_t low = mp->mp_flags & P_BRANCH; - intptr_t high = nkeys - 1; - if (unlikely(high < low)) { - mc->mc_ki[mc->mc_top] = 0; - ret.node = NULL; - return ret; +__cold static MDBX_chk_line_t *chk_line_feed(MDBX_chk_line_t *line) { + if (likely(line)) { + MDBX_chk_internal_t *chk = line->ctx->internal; + enum MDBX_chk_severity severity = line->severity; + chk_line_end(line); + line = chk_line_begin(chk->usr->scope, severity); } + return line; +} - intptr_t i; - MDBX_cmp_func *cmp = mc->mc_dbx->md_cmp; - MDBX_val nodekey; - if (unlikely(IS_LEAF2(mp))) { - cASSERT(mc, mp->mp_leaf2_ksize == mc->mc_db->md_xsize); - nodekey.iov_len = mp->mp_leaf2_ksize; - do { - i = (low + high) >> 1; - nodekey.iov_base = page_leaf2key(mp, i, nodekey.iov_len); - cASSERT(mc, ptr_disp(mp, mc->mc_txn->mt_env->me_psize) >= - ptr_disp(nodekey.iov_base, nodekey.iov_len)); - int cr = cmp(key, &nodekey); - DEBUG("found leaf index %zu [%s], rc = %i", i, DKEY_DEBUG(&nodekey), cr); - if (cr > 0) - /* Found entry is less than the key. */ - /* Skip to get the smallest entry larger than key. */ - low = ++i; - else if (cr < 0) - high = i - 1; - else { - ret.exact = true; - break; - } - } while (likely(low <= high)); - - /* store the key index */ - mc->mc_ki[mc->mc_top] = (indx_t)i; - ret.node = (i < nkeys) - ? /* fake for LEAF2 */ (MDBX_node *)(intptr_t)-1 - : /* There is no entry larger or equal to the key. */ NULL; - return ret; +__cold static MDBX_chk_line_t *chk_flush(MDBX_chk_line_t *line) { + if (likely(line)) { + MDBX_chk_internal_t *chk = line->ctx->internal; + assert(line->begin <= line->end && line->begin <= line->out && line->out <= line->end); + if (likely(chk->cb->print_flush)) { + chk->cb->print_flush(line); + assert(line->begin <= line->end && line->begin <= line->out && line->out <= line->end); + line->out = line->begin; + } } + return line; +} - if (IS_BRANCH(mp) && cmp == cmp_int_align2) - /* Branch pages have no data, so if using integer keys, - * alignment is guaranteed. Use faster cmp_int_align4(). */ - cmp = cmp_int_align4; - - MDBX_node *node; - do { - i = (low + high) >> 1; - node = page_node(mp, i); - nodekey.iov_len = node_ks(node); - nodekey.iov_base = node_key(node); - cASSERT(mc, ptr_disp(mp, mc->mc_txn->mt_env->me_psize) >= - ptr_disp(nodekey.iov_base, nodekey.iov_len)); - int cr = cmp(key, &nodekey); - if (IS_LEAF(mp)) - DEBUG("found leaf index %zu [%s], rc = %i", i, DKEY_DEBUG(&nodekey), cr); - else - DEBUG("found branch index %zu [%s -> %" PRIaPGNO "], rc = %i", i, - DKEY_DEBUG(&nodekey), node_pgno(node), cr); - if (cr > 0) - /* Found entry is less than the key. */ - /* Skip to get the smallest entry larger than key. */ - low = ++i; - else if (cr < 0) - high = i - 1; - else { - ret.exact = true; - break; +__cold static size_t chk_print_wanna(MDBX_chk_line_t *line, size_t need) { + if (likely(line && need)) { + size_t have = line->end - line->out; + assert(line->begin <= line->end && line->begin <= line->out && line->out <= line->end); + if (need > have) { + line = chk_flush(line); + have = line->end - line->out; } - } while (likely(low <= high)); - - /* store the key index */ - mc->mc_ki[mc->mc_top] = (indx_t)i; - ret.node = (i < nkeys) - ? page_node(mp, i) - : /* There is no entry larger or equal to the key. */ NULL; - return ret; + return (need < have) ? need : have; + } + return 0; } -/* Pop a page off the top of the cursor's stack. */ -static __inline void cursor_pop(MDBX_cursor *mc) { - if (likely(mc->mc_snum)) { - DEBUG("popped page %" PRIaPGNO " off db %d cursor %p", - mc->mc_pg[mc->mc_top]->mp_pgno, DDBI(mc), (void *)mc); - if (likely(--mc->mc_snum)) { - mc->mc_top--; +__cold static MDBX_chk_line_t *chk_puts(MDBX_chk_line_t *line, const char *str) { + if (likely(line && str && *str)) { + MDBX_chk_internal_t *chk = line->ctx->internal; + size_t left = strlen(str); + assert(line->begin <= line->end && line->begin <= line->out && line->out <= line->end); + if (chk->cb->print_chars) { + chk->cb->print_chars(line, str, left); + assert(line->begin <= line->end && line->begin <= line->out && line->out <= line->end); + } else + do { + size_t chunk = chk_print_wanna(line, left); + assert(chunk <= left); + if (unlikely(!chunk)) + break; + memcpy(line->out, str, chunk); + line->out += chunk; + assert(line->begin <= line->end && line->begin <= line->out && line->out <= line->end); + str += chunk; + left -= chunk; + } while (left); + line->empty = false; + } + return line; +} + +__cold static MDBX_chk_line_t *chk_print_va(MDBX_chk_line_t *line, const char *fmt, va_list args) { + if (likely(line)) { + MDBX_chk_internal_t *chk = line->ctx->internal; + assert(line->begin <= line->end && line->begin <= line->out && line->out <= line->end); + if (chk->cb->print_format) { + chk->cb->print_format(line, fmt, args); + assert(line->begin <= line->end && line->begin <= line->out && line->out <= line->end); } else { - mc->mc_flags &= ~C_INITIALIZED; + va_list ones; + va_copy(ones, args); + const int needed = vsnprintf(nullptr, 0, fmt, ones); + va_end(ones); + if (likely(needed > 0)) { + const size_t have = chk_print_wanna(line, needed); + if (likely(have > 0)) { + int written = vsnprintf(line->out, have, fmt, args); + if (likely(written > 0)) + line->out += written; + assert(line->begin <= line->end && line->begin <= line->out && line->out <= line->end); + } + } } + line->empty = false; } + return line; } -/* Push a page onto the top of the cursor's stack. - * Set MDBX_TXN_ERROR on failure. */ -static __inline int cursor_push(MDBX_cursor *mc, MDBX_page *mp) { - DEBUG("pushing page %" PRIaPGNO " on db %d cursor %p", mp->mp_pgno, DDBI(mc), - (void *)mc); +__cold static MDBX_chk_line_t *MDBX_PRINTF_ARGS(2, 3) chk_print(MDBX_chk_line_t *line, const char *fmt, ...) { + if (likely(line)) { + // MDBX_chk_internal_t *chk = line->ctx->internal; + va_list args; + va_start(args, fmt); + line = chk_print_va(line, fmt, args); + va_end(args); + line->empty = false; + } + return line; +} - if (unlikely(mc->mc_snum >= CURSOR_STACK)) { - mc->mc_txn->mt_flags |= MDBX_TXN_ERROR; - return MDBX_CURSOR_FULL; +__cold static MDBX_chk_line_t *chk_print_size(MDBX_chk_line_t *line, const char *prefix, const uint64_t value, + const char *suffix) { + static const char sf[] = "KMGTPEZY"; /* LY: Kilo, Mega, Giga, Tera, Peta, Exa, Zetta, Yotta! */ + if (likely(line)) { + MDBX_chk_internal_t *chk = line->ctx->internal; + prefix = prefix ? prefix : ""; + suffix = suffix ? suffix : ""; + if (chk->cb->print_size) + chk->cb->print_size(line, prefix, value, suffix); + else + for (unsigned i = 0;; ++i) { + const unsigned scale = 10 + i * 10; + const uint64_t rounded = value + (UINT64_C(5) << (scale - 10)); + const uint64_t integer = rounded >> scale; + const uint64_t fractional = (rounded - (integer << scale)) * 100u >> scale; + if ((rounded >> scale) <= 1000) + return chk_print(line, "%s%" PRIu64 " (%u.%02u %ciB)%s", prefix, value, (unsigned)integer, + (unsigned)fractional, sf[i], suffix); + } + line->empty = false; } + return line; +} - mc->mc_top = mc->mc_snum++; - mc->mc_pg[mc->mc_top] = mp; - mc->mc_ki[mc->mc_top] = 0; - return MDBX_SUCCESS; +__cold static int chk_error_rc(MDBX_chk_scope_t *const scope, int err, const char *subj) { + MDBX_chk_line_t *line = chk_line_begin(scope, MDBX_chk_error); + if (line) + chk_line_end(chk_flush(chk_print(line, "%s() failed, error %s (%d)", subj, mdbx_strerror(err), err))); + else + debug_log(MDBX_LOG_ERROR, "mdbx_env_chk", 0, "%s() failed, error %s (%d)", subj, mdbx_strerror(err), err); + return err; } -__hot static __always_inline int page_get_checker_lite(const uint16_t ILL, - const MDBX_page *page, - MDBX_txn *const txn, - const txnid_t front) { - if (unlikely(page->mp_flags & ILL)) { - if (ILL == P_ILL_BITS || (page->mp_flags & P_ILL_BITS)) - return bad_page(page, "invalid page's flags (%u)\n", page->mp_flags); - else if (ILL & P_OVERFLOW) { - assert((ILL & (P_BRANCH | P_LEAF | P_LEAF2)) == 0); - assert(page->mp_flags & (P_BRANCH | P_LEAF | P_LEAF2)); - return bad_page(page, "unexpected %s instead of %s (%u)\n", - "large/overflow", "branch/leaf/leaf2", page->mp_flags); - } else if (ILL & (P_BRANCH | P_LEAF | P_LEAF2)) { - assert((ILL & P_BRANCH) && (ILL & P_LEAF) && (ILL & P_LEAF2)); - assert(page->mp_flags & (P_BRANCH | P_LEAF | P_LEAF2)); - return bad_page(page, "unexpected %s instead of %s (%u)\n", - "branch/leaf/leaf2", "large/overflow", page->mp_flags); - } else { - assert(false); - } +__cold static void MDBX_PRINTF_ARGS(5, 6) + chk_object_issue(MDBX_chk_scope_t *const scope, const char *object, uint64_t entry_number, const char *caption, + const char *extra_fmt, ...) { + MDBX_chk_internal_t *const chk = scope->internal; + MDBX_chk_issue_t *issue = chk->usr->scope->issues; + while (issue) { + if (issue->caption == caption) { + issue->count += 1; + break; + } else + issue = issue->next; + } + const bool fresh = issue == nullptr; + if (fresh) { + issue = osal_malloc(sizeof(*issue)); + if (likely(issue)) { + issue->caption = caption; + issue->count = 1; + issue->next = chk->usr->scope->issues; + chk->usr->scope->issues = issue; + } else + chk_error_rc(scope, MDBX_ENOMEM, "adding issue"); } - if (unlikely(page->mp_txnid > front) && - unlikely(page->mp_txnid > txn->mt_front || front < txn->mt_txnid)) - return bad_page( - page, - "invalid page' txnid (%" PRIaTXN ") for %s' txnid (%" PRIaTXN ")\n", - page->mp_txnid, - (front == txn->mt_front && front != txn->mt_txnid) ? "front-txn" - : "parent-page", - front); - - if (((ILL & P_OVERFLOW) || !IS_OVERFLOW(page)) && - (ILL & (P_BRANCH | P_LEAF | P_LEAF2)) == 0) { - /* Контроль четности page->mp_upper тут либо приводит к ложным ошибкам, - * либо слишком дорог по количеству операций. Заковырка в том, что mp_upper - * может быть нечетным на LEAF2-страницах, при нечетном количестве элементов - * нечетной длины. Поэтому четность page->mp_upper здесь не проверяется, но - * соответствующие полные проверки есть в page_check(). */ - if (unlikely(page->mp_upper < page->mp_lower || (page->mp_lower & 1) || - PAGEHDRSZ + page->mp_upper > txn->mt_env->me_psize)) - return bad_page(page, - "invalid page' lower(%u)/upper(%u) with limit %zu\n", - page->mp_lower, page->mp_upper, page_space(txn->mt_env)); - - } else if ((ILL & P_OVERFLOW) == 0) { - const pgno_t npages = page->mp_pages; - if (unlikely(npages < 1) || unlikely(npages >= MAX_PAGENO / 2)) - return bad_page(page, "invalid n-pages (%u) for large-page\n", npages); - if (unlikely(page->mp_pgno + npages > txn->mt_next_pgno)) - return bad_page( - page, - "end of large-page beyond (%u) allocated space (%u next-pgno)\n", - page->mp_pgno + npages, txn->mt_next_pgno); + va_list args; + va_start(args, extra_fmt); + if (chk->cb->issue) { + mdbx_env_chk_encount_problem(chk->usr); + chk->cb->issue(chk->usr, object, entry_number, caption, extra_fmt, args); } else { - assert(false); + MDBX_chk_line_t *line = chk_line_begin(scope, MDBX_chk_error); + if (entry_number != UINT64_MAX) + chk_print(line, "%s #%" PRIu64 ": %s", object, entry_number, caption); + else + chk_print(line, "%s: %s", object, caption); + if (extra_fmt) + chk_puts(chk_print_va(chk_puts(line, " ("), extra_fmt, args), ")"); + chk_line_end(fresh ? chk_flush(line) : line); } - return MDBX_SUCCESS; + va_end(args); } -__cold static __noinline pgr_t -page_get_checker_full(const uint16_t ILL, MDBX_page *page, - const MDBX_cursor *const mc, const txnid_t front) { - pgr_t r = {page, page_get_checker_lite(ILL, page, mc->mc_txn, front)}; - if (likely(r.err == MDBX_SUCCESS)) - r.err = page_check(mc, page); - if (unlikely(r.err != MDBX_SUCCESS)) - mc->mc_txn->mt_flags |= MDBX_TXN_ERROR; - return r; +__cold static void MDBX_PRINTF_ARGS(2, 3) chk_scope_issue(MDBX_chk_scope_t *const scope, const char *fmt, ...) { + MDBX_chk_internal_t *const chk = scope->internal; + va_list args; + va_start(args, fmt); + if (likely(chk->cb->issue)) { + mdbx_env_chk_encount_problem(chk->usr); + chk->cb->issue(chk->usr, nullptr, 0, nullptr, fmt, args); + } else + chk_line_end(chk_print_va(chk_line_begin(scope, MDBX_chk_error), fmt, args)); + va_end(args); } -__hot static __always_inline pgr_t page_get_inline(const uint16_t ILL, - const MDBX_cursor *const mc, - const pgno_t pgno, - const txnid_t front) { - MDBX_txn *const txn = mc->mc_txn; - tASSERT(txn, front <= txn->mt_front); - - pgr_t r; - if (unlikely(pgno >= txn->mt_next_pgno)) { - ERROR("page #%" PRIaPGNO " beyond next-pgno", pgno); - r.page = nullptr; - r.err = MDBX_PAGE_NOTFOUND; - bailout: - txn->mt_flags |= MDBX_TXN_ERROR; - return r; - } - - eASSERT(txn->mt_env, - ((txn->mt_flags ^ txn->mt_env->me_flags) & MDBX_WRITEMAP) == 0); - r.page = pgno2page(txn->mt_env, pgno); - if ((txn->mt_flags & (MDBX_TXN_RDONLY | MDBX_WRITEMAP)) == 0) { - const MDBX_txn *spiller = txn; - do { - /* Spilled pages were dirtied in this txn and flushed - * because the dirty list got full. Bring this page - * back in from the map (but don't unspill it here, - * leave that unless page_touch happens again). */ - if (unlikely(spiller->mt_flags & MDBX_TXN_SPILLS) && - search_spilled(spiller, pgno)) - break; - - const size_t i = dpl_search(spiller, pgno); - tASSERT(txn, (intptr_t)i > 0); - if (spiller->tw.dirtylist->items[i].pgno == pgno) { - r.page = spiller->tw.dirtylist->items[i].ptr; - break; - } +__cold static int chk_scope_end(MDBX_chk_internal_t *chk, int err) { + assert(chk->scope_depth > 0); + MDBX_chk_scope_t *const inner = chk->scope_stack + chk->scope_depth; + MDBX_chk_scope_t *const outer = chk->scope_depth ? inner - 1 : nullptr; + if (!outer || outer->stage != inner->stage) { + if (err == MDBX_SUCCESS && *chk->problem_counter) + err = MDBX_PROBLEM; + else if (*chk->problem_counter == 0 && MDBX_IS_ERROR(err)) + *chk->problem_counter = 1; + if (chk->problem_counter != &chk->usr->result.total_problems) { + chk->usr->result.total_problems += *chk->problem_counter; + chk->problem_counter = &chk->usr->result.total_problems; + } + if (chk->cb->stage_end) + err = chk->cb->stage_end(chk->usr, inner->stage, err); + } + if (chk->cb->scope_conclude) + err = chk->cb->scope_conclude(chk->usr, outer, inner, err); + chk->usr->scope = outer; + chk->usr->scope_nesting = chk->scope_depth -= 1; + if (outer) + outer->subtotal_issues += inner->subtotal_issues; + if (chk->cb->scope_pop) + chk->cb->scope_pop(chk->usr, outer, inner); + + while (inner->issues) { + MDBX_chk_issue_t *next = inner->issues->next; + osal_free(inner->issues); + inner->issues = next; + } + memset(inner, -1, sizeof(*inner)); + return err; +} - spiller = spiller->mt_parent; - } while (spiller); +__cold static int chk_scope_begin_args(MDBX_chk_internal_t *chk, int verbosity_adjustment, enum MDBX_chk_stage stage, + const void *object, size_t *problems, const char *fmt, va_list args) { + if (unlikely(chk->scope_depth + 1u >= ARRAY_LENGTH(chk->scope_stack))) + return MDBX_BACKLOG_DEPLETED; + + MDBX_chk_scope_t *const outer = chk->scope_stack + chk->scope_depth; + const int verbosity = outer->verbosity + (verbosity_adjustment - 1) * (1 << MDBX_chk_severity_prio_shift); + MDBX_chk_scope_t *const inner = outer + 1; + memset(inner, 0, sizeof(*inner)); + inner->internal = outer->internal; + inner->stage = stage ? stage : (stage = outer->stage); + inner->object = object; + inner->verbosity = (verbosity < MDBX_chk_warning) ? MDBX_chk_warning : (enum MDBX_chk_severity)verbosity; + if (problems) + chk->problem_counter = problems; + else if (!chk->problem_counter || outer->stage != stage) + chk->problem_counter = &chk->usr->result.total_problems; + + if (chk->cb->scope_push) { + const int err = chk->cb->scope_push(chk->usr, outer, inner, fmt, args); + if (unlikely(err != MDBX_SUCCESS)) + return err; } + chk->usr->scope = inner; + chk->usr->scope_nesting = chk->scope_depth += 1; - if (unlikely(r.page->mp_pgno != pgno)) { - r.err = bad_page( - r.page, "pgno mismatch (%" PRIaPGNO ") != expected (%" PRIaPGNO ")\n", - r.page->mp_pgno, pgno); - goto bailout; + if (stage != outer->stage && chk->cb->stage_begin) { + int err = chk->cb->stage_begin(chk->usr, stage); + if (unlikely(err != MDBX_SUCCESS)) { + err = chk_scope_end(chk, err); + assert(err != MDBX_SUCCESS); + return err ? err : MDBX_RESULT_TRUE; + } } - - if (unlikely(mc->mc_checking & CC_PAGECHECK)) - return page_get_checker_full(ILL, r.page, mc, front); - -#if MDBX_DISABLE_VALIDATION - r.err = MDBX_SUCCESS; -#else - r.err = page_get_checker_lite(ILL, r.page, txn, front); - if (unlikely(r.err != MDBX_SUCCESS)) - goto bailout; -#endif /* MDBX_DISABLE_VALIDATION */ - return r; + return MDBX_SUCCESS; } -/* Finish mdbx_page_search() / mdbx_page_search_lowest(). - * The cursor is at the root page, set up the rest of it. */ -__hot __noinline static int page_search_root(MDBX_cursor *mc, - const MDBX_val *key, int flags) { - MDBX_page *mp = mc->mc_pg[mc->mc_top]; - int rc; - DKBUF_DEBUG; +__cold static int MDBX_PRINTF_ARGS(6, 7) + chk_scope_begin(MDBX_chk_internal_t *chk, int verbosity_adjustment, enum MDBX_chk_stage stage, const void *object, + size_t *problems, const char *fmt, ...) { + va_list args; + va_start(args, fmt); + int rc = chk_scope_begin_args(chk, verbosity_adjustment, stage, object, problems, fmt, args); + va_end(args); + return rc; +} - while (IS_BRANCH(mp)) { - MDBX_node *node; - intptr_t i; +__cold static int chk_scope_restore(MDBX_chk_scope_t *const target, int err) { + MDBX_chk_internal_t *const chk = target->internal; + assert(target <= chk->usr->scope); + while (chk->usr->scope > target) + err = chk_scope_end(chk, err); + return err; +} - DEBUG("branch page %" PRIaPGNO " has %zu keys", mp->mp_pgno, - page_numkeys(mp)); - /* Don't assert on branch pages in the GC. We can get here - * while in the process of rebalancing a GC branch page; we must - * let that proceed. ITS#8336 */ - cASSERT(mc, !mc->mc_dbi || page_numkeys(mp) > 1); - DEBUG("found index 0 to page %" PRIaPGNO, node_pgno(page_node(mp, 0))); +__cold void chk_scope_pop(MDBX_chk_scope_t *const inner) { + if (inner && inner > inner->internal->scope_stack) + chk_scope_restore(inner - 1, MDBX_SUCCESS); +} - if (flags & (MDBX_PS_FIRST | MDBX_PS_LAST)) { - i = 0; - if (flags & MDBX_PS_LAST) { - i = page_numkeys(mp) - 1; - /* if already init'd, see if we're already in right place */ - if (mc->mc_flags & C_INITIALIZED) { - if (mc->mc_ki[mc->mc_top] == i) { - mc->mc_top = mc->mc_snum++; - mp = mc->mc_pg[mc->mc_top]; - goto ready; - } - } +__cold static MDBX_chk_scope_t *MDBX_PRINTF_ARGS(3, 4) + chk_scope_push(MDBX_chk_scope_t *const scope, int verbosity_adjustment, const char *fmt, ...) { + chk_scope_restore(scope, MDBX_SUCCESS); + va_list args; + va_start(args, fmt); + int err = chk_scope_begin_args(scope->internal, verbosity_adjustment, scope->stage, nullptr, nullptr, fmt, args); + va_end(args); + return err ? nullptr : scope + 1; +} + +__cold static const char *chk_v2a(MDBX_chk_internal_t *chk, const MDBX_val *val) { + if (val == MDBX_CHK_MAIN) + return "@MAIN"; + if (val == MDBX_CHK_GC) + return "@GC"; + if (val == MDBX_CHK_META) + return "@META"; + + const unsigned char *const data = val->iov_base; + const size_t len = val->iov_len; + if (data == MDBX_CHK_MAIN) + return "@MAIN"; + if (data == MDBX_CHK_GC) + return "@GC"; + if (data == MDBX_CHK_META) + return "@META"; + + if (!len) + return ""; + if (!data) + return ""; + if (len > 65536) { + const size_t enough = 42; + if (chk->v2a_buf.iov_len < enough) { + void *ptr = osal_realloc(chk->v2a_buf.iov_base, enough); + if (unlikely(!ptr)) + return ""; + chk->v2a_buf.iov_base = ptr; + chk->v2a_buf.iov_len = enough; + } + snprintf(chk->v2a_buf.iov_base, chk->v2a_buf.iov_len, "", len); + return chk->v2a_buf.iov_base; + } + + bool printable = true; + bool quoting = false; + size_t xchars = 0; + for (size_t i = 0; i < len && printable; ++i) { + quoting = quoting || !(data[i] == '_' || isalnum(data[i])); + printable = isprint(data[i]) || (data[i] < ' ' && ++xchars < 4 && len > xchars * 4); + } + + size_t need = len + 1; + if (quoting || !printable) + need += len + /* quotes */ 2 + 2 * /* max xchars */ 4; + if (need > chk->v2a_buf.iov_len) { + void *ptr = osal_realloc(chk->v2a_buf.iov_base, need); + if (unlikely(!ptr)) + return ""; + chk->v2a_buf.iov_base = ptr; + chk->v2a_buf.iov_len = need; + } + + static const char hex[] = "0123456789abcdef"; + char *w = chk->v2a_buf.iov_base; + if (!quoting) { + memcpy(w, data, len); + w += len; + } else if (printable) { + *w++ = '\''; + for (size_t i = 0; i < len; ++i) { + if (data[i] < ' ') { + assert((char *)chk->v2a_buf.iov_base + chk->v2a_buf.iov_len > w + 4); + w[0] = '\\'; + w[1] = 'x'; + w[2] = hex[data[i] >> 4]; + w[3] = hex[data[i] & 15]; + w += 4; + } else if (strchr("\"'`\\", data[i])) { + assert((char *)chk->v2a_buf.iov_base + chk->v2a_buf.iov_len > w + 2); + w[0] = '\\'; + w[1] = data[i]; + w += 2; + } else { + assert((char *)chk->v2a_buf.iov_base + chk->v2a_buf.iov_len > w + 1); + *w++ = data[i]; } - } else { - const struct node_result nsr = node_search(mc, key); - if (likely(nsr.node)) - i = mc->mc_ki[mc->mc_top] + (intptr_t)nsr.exact - 1; - else - i = page_numkeys(mp) - 1; - DEBUG("following index %zu for key [%s]", i, DKEY_DEBUG(key)); } - - cASSERT(mc, i >= 0 && i < (int)page_numkeys(mp)); - node = page_node(mp, i); - - rc = page_get(mc, node_pgno(node), &mp, mp->mp_txnid); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - - mc->mc_ki[mc->mc_top] = (indx_t)i; - if (unlikely(rc = cursor_push(mc, mp))) - return rc; - - ready: - if (flags & MDBX_PS_MODIFY) { - rc = page_touch(mc); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - mp = mc->mc_pg[mc->mc_top]; + *w++ = '\''; + } else { + *w++ = '\\'; + *w++ = 'x'; + for (size_t i = 0; i < len; ++i) { + assert((char *)chk->v2a_buf.iov_base + chk->v2a_buf.iov_len > w + 2); + w[0] = hex[data[i] >> 4]; + w[1] = hex[data[i] & 15]; + w += 2; + } + } + assert((char *)chk->v2a_buf.iov_base + chk->v2a_buf.iov_len > w); + *w = 0; + return chk->v2a_buf.iov_base; +} + +__cold static void chk_dispose(MDBX_chk_internal_t *chk) { + assert(chk->table[FREE_DBI] == &chk->table_gc); + assert(chk->table[MAIN_DBI] == &chk->table_main); + for (size_t i = 0; i < ARRAY_LENGTH(chk->table); ++i) { + MDBX_chk_table_t *const tbl = chk->table[i]; + if (tbl) { + chk->table[i] = nullptr; + if (chk->cb->table_dispose && tbl->cookie) { + chk->cb->table_dispose(chk->usr, tbl); + tbl->cookie = nullptr; + } + if (tbl != &chk->table_gc && tbl != &chk->table_main) { + osal_free(tbl); + } + } + } + osal_free(chk->v2a_buf.iov_base); + osal_free(chk->pagemap); + chk->usr->internal = nullptr; + chk->usr->scope = nullptr; + chk->pagemap = nullptr; + memset(chk, 0xDD, sizeof(*chk)); + osal_free(chk); +} + +static size_t div_8s(size_t numerator, size_t divider) { + assert(numerator <= (SIZE_MAX >> 8)); + return (numerator << 8) / divider; +} + +static size_t mul_8s(size_t quotient, size_t multiplier) { + size_t hi = multiplier * (quotient >> 8); + size_t lo = multiplier * (quotient & 255) + 128; + return hi + (lo >> 8); +} + +static void histogram_reduce(struct MDBX_chk_histogram *p) { + const size_t size = ARRAY_LENGTH(p->ranges), last = size - 1; + // ищем пару для слияния с минимальной ошибкой + size_t min_err = SIZE_MAX, min_i = last - 1; + for (size_t i = 0; i < last; ++i) { + const size_t b1 = p->ranges[i].begin, e1 = p->ranges[i].end, s1 = p->ranges[i].amount; + const size_t b2 = p->ranges[i + 1].begin, e2 = p->ranges[i + 1].end, s2 = p->ranges[i + 1].amount; + const size_t l1 = e1 - b1, l2 = e2 - b2, lx = e2 - b1, sx = s1 + s2; + assert(s1 > 0 && b1 > 0 && b1 < e1); + assert(s2 > 0 && b2 > 0 && b2 < e2); + assert(e1 <= b2); + // за ошибку принимаем площадь изменений на гистограмме при слиянии + const size_t h1 = div_8s(s1, l1), h2 = div_8s(s2, l2), hx = div_8s(sx, lx); + const size_t d1 = mul_8s((h1 > hx) ? h1 - hx : hx - h1, l1); + const size_t d2 = mul_8s((h2 > hx) ? h2 - hx : hx - h2, l2); + const size_t dx = mul_8s(hx, b2 - e1); + const size_t err = d1 + d2 + dx; + if (min_err >= err) { + min_i = i; + min_err = err; + } + } + // объединяем + p->ranges[min_i].end = p->ranges[min_i + 1].end; + p->ranges[min_i].amount += p->ranges[min_i + 1].amount; + p->ranges[min_i].count += p->ranges[min_i + 1].count; + if (min_i < last) + // перемещаем хвост + memmove(p->ranges + min_i, p->ranges + min_i + 1, (last - min_i) * sizeof(p->ranges[0])); + // обнуляем последний элемент и продолжаем + p->ranges[last].count = 0; +} + +static void histogram_acc(const size_t n, struct MDBX_chk_histogram *p) { + STATIC_ASSERT(ARRAY_LENGTH(p->ranges) > 2); + p->amount += n; + p->count += 1; + if (likely(n < 2)) { + p->ones += n; + p->pad += 1; + } else + for (;;) { + const size_t size = ARRAY_LENGTH(p->ranges), last = size - 1; + size_t i = 0; + while (i < size && p->ranges[i].count && n >= p->ranges[i].begin) { + if (n < p->ranges[i].end) { + // значение попадает в существующий интервал + p->ranges[i].amount += n; + p->ranges[i].count += 1; + return; + } + ++i; + } + if (p->ranges[last].count == 0) { + // использованы еще не все слоты, добавляем интервал + assert(i < size); + if (p->ranges[i].count) { + // раздвигаем + assert(i < last); +#ifdef __COVERITY__ + if (i < last) /* avoid Coverity false-positive issue */ +#endif /* __COVERITY__ */ + memmove(p->ranges + i + 1, p->ranges + i, (last - i) * sizeof(p->ranges[0])); + } + p->ranges[i].begin = n; + p->ranges[i].end = n + 1; + p->ranges[i].amount = n; + p->ranges[i].count = 1; + return; + } + histogram_reduce(p); } - } +} - if (!MDBX_DISABLE_VALIDATION && unlikely(!CHECK_LEAF_TYPE(mc, mp))) { - ERROR("unexpected leaf-page #%" PRIaPGNO " type 0x%x seen by cursor", - mp->mp_pgno, mp->mp_flags); - return MDBX_CORRUPTED; +__cold static MDBX_chk_line_t *histogram_dist(MDBX_chk_line_t *line, const struct MDBX_chk_histogram *histogram, + const char *prefix, const char *first, bool amount) { + line = chk_print(line, "%s:", prefix); + const char *comma = ""; + const size_t first_val = amount ? histogram->ones : histogram->pad; + if (first_val) { + chk_print(line, " %s=%" PRIuSIZE, first, first_val); + comma = ","; } - - DEBUG("found leaf page %" PRIaPGNO " for key [%s]", mp->mp_pgno, - DKEY_DEBUG(key)); - mc->mc_flags |= C_INITIALIZED; - mc->mc_flags &= ~C_EOF; - - return MDBX_SUCCESS; + for (size_t n = 0; n < ARRAY_LENGTH(histogram->ranges); ++n) + if (histogram->ranges[n].count) { + chk_print(line, "%s %" PRIuSIZE, comma, histogram->ranges[n].begin); + if (histogram->ranges[n].begin != histogram->ranges[n].end - 1) + chk_print(line, "-%" PRIuSIZE, histogram->ranges[n].end - 1); + line = chk_print(line, "=%" PRIuSIZE, amount ? histogram->ranges[n].amount : histogram->ranges[n].count); + comma = ","; + } + return line; } -static int setup_dbx(MDBX_dbx *const dbx, const MDBX_db *const db, - const unsigned pagesize) { - if (unlikely(!dbx->md_cmp)) { - dbx->md_cmp = get_default_keycmp(db->md_flags); - dbx->md_dcmp = get_default_datacmp(db->md_flags); +__cold static MDBX_chk_line_t *histogram_print(MDBX_chk_scope_t *scope, MDBX_chk_line_t *line, + const struct MDBX_chk_histogram *histogram, const char *prefix, + const char *first, bool amount) { + if (histogram->count) { + line = chk_print(line, "%s %" PRIuSIZE, prefix, amount ? histogram->amount : histogram->count); + if (scope->verbosity > MDBX_chk_info) + line = chk_puts(histogram_dist(line, histogram, " (distribution", first, amount), ")"); } + return line; +} - dbx->md_klen_min = - (db->md_flags & MDBX_INTEGERKEY) ? 4 /* sizeof(uint32_t) */ : 0; - dbx->md_klen_max = keysize_max(pagesize, db->md_flags); - assert(dbx->md_klen_max != (unsigned)-1); +//----------------------------------------------------------------------------- - dbx->md_vlen_min = (db->md_flags & MDBX_INTEGERDUP) - ? 4 /* sizeof(uint32_t) */ - : ((db->md_flags & MDBX_DUPFIXED) ? 1 : 0); - dbx->md_vlen_max = valsize_max(pagesize, db->md_flags); - assert(dbx->md_vlen_max != (size_t)-1); +__cold static int chk_get_tbl(MDBX_chk_scope_t *const scope, const walk_tbl_t *in, MDBX_chk_table_t **out) { + MDBX_chk_internal_t *const chk = scope->internal; + if (chk->last_lookup && chk->last_lookup->name.iov_base == in->name.iov_base) { + *out = chk->last_lookup; + return MDBX_SUCCESS; + } - if ((db->md_flags & (MDBX_DUPFIXED | MDBX_INTEGERDUP)) != 0 && db->md_xsize) { - if (!MDBX_DISABLE_VALIDATION && unlikely(db->md_xsize < dbx->md_vlen_min || - db->md_xsize > dbx->md_vlen_max)) { - ERROR("db.md_xsize (%u) <> min/max value-length (%zu/%zu)", db->md_xsize, - dbx->md_vlen_min, dbx->md_vlen_max); - return MDBX_CORRUPTED; + for (size_t i = 0; i < ARRAY_LENGTH(chk->table); ++i) { + MDBX_chk_table_t *tbl = chk->table[i]; + if (!tbl) { + tbl = osal_calloc(1, sizeof(MDBX_chk_table_t)); + if (unlikely(!tbl)) { + *out = nullptr; + return chk_error_rc(scope, MDBX_ENOMEM, "alloc_table"); + } + chk->table[i] = tbl; + tbl->flags = in->internal->flags; + tbl->id = -1; + tbl->name = in->name; + } + if (tbl->name.iov_base == in->name.iov_base) { + if (tbl->id < 0) { + tbl->id = (int)i; + tbl->cookie = + chk->cb->table_filter ? chk->cb->table_filter(chk->usr, &tbl->name, tbl->flags) : (void *)(intptr_t)-1; + } + *out = (chk->last_lookup = tbl); + return MDBX_SUCCESS; } - dbx->md_vlen_min = dbx->md_vlen_max = db->md_xsize; } - return MDBX_SUCCESS; + chk_scope_issue(scope, "too many tables > %u", (unsigned)ARRAY_LENGTH(chk->table) - CORE_DBS - /* meta */ 1); + *out = nullptr; + return MDBX_PROBLEM; } -static int fetch_sdb(MDBX_txn *txn, size_t dbi) { - MDBX_cursor_couple couple; - if (unlikely(dbi_changed(txn, dbi))) { - NOTICE("dbi %zu was changed for txn %" PRIaTXN, dbi, txn->mt_txnid); - return MDBX_BAD_DBI; - } - int rc = cursor_init(&couple.outer, txn, MAIN_DBI); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - - MDBX_dbx *const dbx = &txn->mt_dbxs[dbi]; - rc = page_search(&couple.outer, &dbx->md_name, 0); - if (unlikely(rc != MDBX_SUCCESS)) { - notfound: - NOTICE("dbi %zu refs to inaccessible subDB `%*s` for txn %" PRIaTXN - " (err %d)", - dbi, (int)dbx->md_name.iov_len, (const char *)dbx->md_name.iov_base, - txn->mt_txnid, rc); - return (rc == MDBX_NOTFOUND) ? MDBX_BAD_DBI : rc; - } +//------------------------------------------------------------------------------ - MDBX_val data; - struct node_result nsr = node_search(&couple.outer, &dbx->md_name); - if (unlikely(!nsr.exact)) { - rc = MDBX_NOTFOUND; - goto notfound; - } - if (unlikely((node_flags(nsr.node) & (F_DUPDATA | F_SUBDATA)) != F_SUBDATA)) { - NOTICE("dbi %zu refs to not a named subDB `%*s` for txn %" PRIaTXN " (%s)", - dbi, (int)dbx->md_name.iov_len, (const char *)dbx->md_name.iov_base, - txn->mt_txnid, "wrong flags"); - return MDBX_INCOMPATIBLE; /* not a named DB */ - } +__cold static void chk_verbose_meta(MDBX_chk_scope_t *const scope, const unsigned num) { + MDBX_chk_line_t *line = chk_line_begin(scope, MDBX_chk_verbose); + MDBX_chk_internal_t *const chk = scope->internal; + if (line) { + MDBX_env *const env = chk->usr->env; + const bool have_bootid = (chk->envinfo.mi_bootid.current.x | chk->envinfo.mi_bootid.current.y) != 0; + const bool bootid_match = have_bootid && memcmp(&chk->envinfo.mi_bootid.meta[num], &chk->envinfo.mi_bootid.current, + sizeof(chk->envinfo.mi_bootid.current)) == 0; + + const char *status = "stay"; + if (num == chk->troika.recent) + status = "head"; + else if (num == TROIKA_TAIL(&chk->troika)) + status = "tail"; + line = chk_print(line, "meta-%u: %s, ", num, status); + + switch (chk->envinfo.mi_meta_sign[num]) { + case DATASIGN_NONE: + line = chk_puts(line, "no-sync/legacy"); + break; + case DATASIGN_WEAK: + line = chk_print(line, "weak-%s", + have_bootid ? (bootid_match ? "intact (same boot-id)" : "dead") : "unknown (no boot-id)"); + break; + default: + line = chk_puts(line, "steady"); + break; + } + const txnid_t meta_txnid = chk->envinfo.mi_meta_txnid[num]; + line = chk_print(line, " txn#%" PRIaTXN ", ", meta_txnid); + if (chk->envinfo.mi_bootid.meta[num].x | chk->envinfo.mi_bootid.meta[num].y) + line = chk_print(line, "boot-id %" PRIx64 "-%" PRIx64 " (%s)", chk->envinfo.mi_bootid.meta[num].x, + chk->envinfo.mi_bootid.meta[num].y, bootid_match ? "live" : "not match"); + else + line = chk_puts(line, "no boot-id"); + + if (env->stuck_meta >= 0) { + if (num == (unsigned)env->stuck_meta) + line = chk_print(line, ", %s", "forced for checking"); + } else if (meta_txnid > chk->envinfo.mi_recent_txnid && + (env->flags & (MDBX_EXCLUSIVE | MDBX_RDONLY)) == MDBX_EXCLUSIVE) + line = chk_print(line, ", rolled-back %" PRIu64 " commit(s) (%" PRIu64 " >>> %" PRIu64 ")", + meta_txnid - chk->envinfo.mi_recent_txnid, meta_txnid, chk->envinfo.mi_recent_txnid); + chk_line_end(line); + } +} + +__cold static int chk_pgvisitor(const size_t pgno, const unsigned npages, void *const ctx, const int deep, + const walk_tbl_t *tbl_info, const size_t page_size, const page_type_t pagetype, + const MDBX_error_t page_err, const size_t nentries, const size_t payload_bytes, + const size_t header_bytes, const size_t unused_bytes) { + MDBX_chk_scope_t *const scope = ctx; + MDBX_chk_internal_t *const chk = scope->internal; + MDBX_chk_context_t *const usr = chk->usr; + MDBX_env *const env = usr->env; + + MDBX_chk_table_t *tbl; + int err = chk_get_tbl(scope, tbl_info, &tbl); + if (unlikely(err)) + return err; - rc = node_read(&couple.outer, nsr.node, &data, - couple.outer.mc_pg[couple.outer.mc_top]); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; + if (deep > 42) { + chk_scope_issue(scope, "too deeply %u", deep); + return MDBX_CORRUPTED /* avoid infinite loop/recursion */; + } + histogram_acc(deep, &tbl->histogram.deep); + usr->result.processed_pages += npages; + const size_t page_bytes = payload_bytes + header_bytes + unused_bytes; + + int height = deep + 1; + if (tbl->id >= CORE_DBS) + height -= usr->txn->dbs[MAIN_DBI].height; + const tree_t *nested = tbl_info->nested; + if (nested) { + if (tbl->flags & MDBX_DUPSORT) + height -= tbl_info->internal->height; + else { + chk_object_issue(scope, "nested tree", pgno, "unexpected", "table %s flags 0x%x, deep %i", + chk_v2a(chk, &tbl->name), tbl->flags, deep); + nested = nullptr; + } + } else + chk->last_nested = nullptr; - if (unlikely(data.iov_len != sizeof(MDBX_db))) { - NOTICE("dbi %zu refs to not a named subDB `%*s` for txn %" PRIaTXN " (%s)", - dbi, (int)dbx->md_name.iov_len, (const char *)dbx->md_name.iov_base, - txn->mt_txnid, "wrong rec-size"); - return MDBX_INCOMPATIBLE; /* not a named DB */ - } - - uint16_t md_flags = UNALIGNED_PEEK_16(data.iov_base, MDBX_db, md_flags); - /* The txn may not know this DBI, or another process may - * have dropped and recreated the DB with other flags. */ - MDBX_db *const db = &txn->mt_dbs[dbi]; - if (unlikely((db->md_flags & DB_PERSISTENT_FLAGS) != md_flags)) { - NOTICE("dbi %zu refs to the re-created subDB `%*s` for txn %" PRIaTXN - " with different flags (present 0x%X != wanna 0x%X)", - dbi, (int)dbx->md_name.iov_len, (const char *)dbx->md_name.iov_base, - txn->mt_txnid, db->md_flags & DB_PERSISTENT_FLAGS, md_flags); - return MDBX_INCOMPATIBLE; - } - - memcpy(db, data.iov_base, sizeof(MDBX_db)); -#if !MDBX_DISABLE_VALIDATION - const txnid_t pp_txnid = couple.outer.mc_pg[couple.outer.mc_top]->mp_txnid; - tASSERT(txn, txn->mt_front >= pp_txnid); - if (unlikely(db->md_mod_txnid > pp_txnid)) { - ERROR("db.md_mod_txnid (%" PRIaTXN ") > page-txnid (%" PRIaTXN ")", - db->md_mod_txnid, pp_txnid); - return MDBX_CORRUPTED; - } -#endif /* !MDBX_DISABLE_VALIDATION */ - rc = setup_dbx(dbx, db, txn->mt_env->me_psize); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - - txn->mt_dbistate[dbi] &= ~DBI_STALE; - return MDBX_SUCCESS; -} - -/* Search for the lowest key under the current branch page. - * This just bypasses a numkeys check in the current page - * before calling mdbx_page_search_root(), because the callers - * are all in situations where the current page is known to - * be underfilled. */ -__hot static int page_search_lowest(MDBX_cursor *mc) { - MDBX_page *mp = mc->mc_pg[mc->mc_top]; - cASSERT(mc, IS_BRANCH(mp)); - MDBX_node *node = page_node(mp, 0); - - int rc = page_get(mc, node_pgno(node), &mp, mp->mp_txnid); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - - mc->mc_ki[mc->mc_top] = 0; - if (unlikely(rc = cursor_push(mc, mp))) - return rc; - return page_search_root(mc, NULL, MDBX_PS_FIRST); -} - -/* Search for the page a given key should be in. - * Push it and its parent pages on the cursor stack. - * - * [in,out] mc the cursor for this operation. - * [in] key the key to search for, or NULL for first/last page. - * [in] flags If MDBX_PS_MODIFY is set, visited pages in the DB - * are touched (updated with new page numbers). - * If MDBX_PS_FIRST or MDBX_PS_LAST is set, find first or last - * leaf. - * This is used by mdbx_cursor_first() and mdbx_cursor_last(). - * If MDBX_PS_ROOTONLY set, just fetch root node, no further - * lookups. - * - * Returns 0 on success, non-zero on failure. */ -__hot static int page_search(MDBX_cursor *mc, const MDBX_val *key, int flags) { - int rc; - pgno_t root; - - /* Make sure the txn is still viable, then find the root from - * the txn's db table and set it as the root of the cursor's stack. */ - if (unlikely(mc->mc_txn->mt_flags & MDBX_TXN_BLOCKED)) { - DEBUG("%s", "transaction has failed, must abort"); - return MDBX_BAD_TXN; + const char *pagetype_caption; + bool branch = false; + switch (pagetype) { + default: + chk_object_issue(scope, "page", pgno, "unknown page-type", "type %u, deep %i", (unsigned)pagetype, deep); + pagetype_caption = "unknown"; + tbl->pages.other += npages; + break; + case page_broken: + assert(page_err != MDBX_SUCCESS); + pagetype_caption = "broken"; + tbl->pages.other += npages; + break; + case page_sub_broken: + assert(page_err != MDBX_SUCCESS); + pagetype_caption = "broken-subpage"; + tbl->pages.other += npages; + break; + case page_large: + pagetype_caption = "large"; + histogram_acc(npages, &tbl->histogram.large_pages); + if (tbl->flags & MDBX_DUPSORT) + chk_object_issue(scope, "page", pgno, "unexpected", "type %u, table %s flags 0x%x, deep %i", (unsigned)pagetype, + chk_v2a(chk, &tbl->name), tbl->flags, deep); + break; + case page_branch: + branch = true; + if (!nested) { + pagetype_caption = "branch"; + tbl->pages.branch += 1; + } else { + pagetype_caption = "nested-branch"; + tbl->pages.nested_branch += 1; + } + break; + case page_dupfix_leaf: + if (!nested) + chk_object_issue(scope, "page", pgno, "unexpected", "type %u, table %s flags 0x%x, deep %i", (unsigned)pagetype, + chk_v2a(chk, &tbl->name), tbl->flags, deep); + /* fall through */ + __fallthrough; + case page_leaf: + if (!nested) { + pagetype_caption = "leaf"; + tbl->pages.leaf += 1; + if (height != tbl_info->internal->height) + chk_object_issue(scope, "page", pgno, "wrong tree height", "actual %i != %i table %s", height, + tbl_info->internal->height, chk_v2a(chk, &tbl->name)); + } else { + pagetype_caption = (pagetype == page_leaf) ? "nested-leaf" : "nested-leaf-dupfix"; + tbl->pages.nested_leaf += 1; + if (chk->last_nested != nested) { + histogram_acc(height, &tbl->histogram.nested_tree); + chk->last_nested = nested; + } + if (height != nested->height) + chk_object_issue(scope, "page", pgno, "wrong nested-tree height", "actual %i != %i dupsort-node %s", height, + nested->height, chk_v2a(chk, &tbl->name)); + } + break; + case page_sub_dupfix_leaf: + case page_sub_leaf: + pagetype_caption = (pagetype == page_sub_leaf) ? "subleaf-dupsort" : "subleaf-dupfix"; + tbl->pages.nested_subleaf += 1; + if ((tbl->flags & MDBX_DUPSORT) == 0 || nested) + chk_object_issue(scope, "page", pgno, "unexpected", "type %u, table %s flags 0x%x, deep %i", (unsigned)pagetype, + chk_v2a(chk, &tbl->name), tbl->flags, deep); + break; } - /* Make sure we're using an up-to-date root */ - if (unlikely(*mc->mc_dbistate & DBI_STALE)) { - rc = fetch_sdb(mc->mc_txn, mc->mc_dbi); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - } - root = mc->mc_db->md_root; + if (npages) { + if (tbl->cookie) { + MDBX_chk_line_t *line = chk_line_begin(scope, MDBX_chk_extra); + if (npages == 1) + chk_print(line, "%s-page %" PRIuSIZE, pagetype_caption, pgno); + else + chk_print(line, "%s-span %" PRIuSIZE "[%u]", pagetype_caption, pgno, npages); + chk_line_end(chk_print( + line, " of %s: header %" PRIiPTR ", %s %" PRIiPTR ", payload %" PRIiPTR ", unused %" PRIiPTR ", deep %i", + chk_v2a(chk, &tbl->name), header_bytes, (pagetype == page_branch) ? "keys" : "entries", nentries, + payload_bytes, unused_bytes, deep)); + } + + bool already_used = false; + for (unsigned n = 0; n < npages; ++n) { + const size_t spanpgno = pgno + n; + if (spanpgno >= usr->result.alloc_pages) { + chk_object_issue(scope, "page", spanpgno, "wrong page-no", "%s-page: %" PRIuSIZE " > %" PRIuSIZE ", deep %i", + pagetype_caption, spanpgno, usr->result.alloc_pages, deep); + tbl->pages.all += 1; + } else if (chk->pagemap[spanpgno]) { + const MDBX_chk_table_t *const rival = chk->table[chk->pagemap[spanpgno] - 1]; + chk_object_issue(scope, "page", spanpgno, (branch && rival == tbl) ? "loop" : "already used", + "%s-page: by %s, deep %i", pagetype_caption, chk_v2a(chk, &rival->name), deep); + already_used = true; + } else { + chk->pagemap[spanpgno] = (int16_t)tbl->id + 1; + tbl->pages.all += 1; + } + } - if (unlikely(root == P_INVALID)) { /* Tree is empty. */ - DEBUG("%s", "tree is empty"); - return MDBX_NOTFOUND; + if (already_used) + return branch ? MDBX_RESULT_TRUE /* avoid infinite loop/recursion */ + : MDBX_SUCCESS; } - cASSERT(mc, root >= NUM_METAS); - if (!mc->mc_snum || !(mc->mc_flags & C_INITIALIZED) || - mc->mc_pg[0]->mp_pgno != root) { - txnid_t pp_txnid = mc->mc_db->md_mod_txnid; - pp_txnid = /* mc->mc_db->md_mod_txnid maybe zero in a legacy DB */ pp_txnid - ? pp_txnid - : mc->mc_txn->mt_txnid; - if ((mc->mc_txn->mt_flags & MDBX_TXN_RDONLY) == 0) { - MDBX_txn *scan = mc->mc_txn; - do - if ((scan->mt_flags & MDBX_TXN_DIRTY) && - (mc->mc_dbi == MAIN_DBI || - (scan->mt_dbistate[mc->mc_dbi] & DBI_DIRTY))) { - /* После коммита вложенных тразакций может быть mod_txnid > front */ - pp_txnid = scan->mt_front; - break; - } - while (unlikely((scan = scan->mt_parent) != nullptr)); + if (MDBX_IS_ERROR(page_err)) { + chk_object_issue(scope, "page", pgno, "invalid/corrupted", "%s-page", pagetype_caption); + } else { + if (unused_bytes > page_size) + chk_object_issue(scope, "page", pgno, "illegal unused-bytes", "%s-page: %u < %" PRIuSIZE " < %u", + pagetype_caption, 0, unused_bytes, env->ps); + + if (header_bytes < (int)sizeof(long) || (size_t)header_bytes >= env->ps - sizeof(long)) { + chk_object_issue(scope, "page", pgno, "illegal header-length", + "%s-page: %" PRIuSIZE " < %" PRIuSIZE " < %" PRIuSIZE, pagetype_caption, sizeof(long), + header_bytes, env->ps - sizeof(long)); + } + if (nentries < 1 || (pagetype == page_branch && nentries < 2)) { + chk_object_issue(scope, "page", pgno, nentries ? "half-empty" : "empty", + "%s-page: payload %" PRIuSIZE " bytes, %" PRIuSIZE " entries, deep %i", pagetype_caption, + payload_bytes, nentries, deep); + tbl->pages.empty += 1; + } + + if (npages) { + if (page_bytes != page_size) { + chk_object_issue(scope, "page", pgno, "misused", + "%s-page: %" PRIuPTR " != %" PRIuPTR " (%" PRIuPTR "h + %" PRIuPTR "p + %" PRIuPTR + "u), deep %i", + pagetype_caption, page_size, page_bytes, header_bytes, payload_bytes, unused_bytes, deep); + if (page_size > page_bytes) + tbl->lost_bytes += page_size - page_bytes; + } else { + tbl->payload_bytes += payload_bytes + header_bytes; + usr->result.total_payload_bytes += payload_bytes + header_bytes; + } } - if (unlikely((rc = page_get(mc, root, &mc->mc_pg[0], pp_txnid)) != 0)) - return rc; - } - - mc->mc_snum = 1; - mc->mc_top = 0; - - DEBUG("db %d root page %" PRIaPGNO " has flags 0x%X", DDBI(mc), root, - mc->mc_pg[0]->mp_flags); - - if (flags & MDBX_PS_MODIFY) { - if (unlikely(rc = page_touch(mc))) - return rc; } - - if (flags & MDBX_PS_ROOTONLY) - return MDBX_SUCCESS; - - return page_search_root(mc, key, flags); + return chk_check_break(scope); } -/* Read large/overflow node data. */ -static __noinline int node_read_bigdata(MDBX_cursor *mc, const MDBX_node *node, - MDBX_val *data, const MDBX_page *mp) { - cASSERT(mc, node_flags(node) == F_BIGDATA && data->iov_len == node_ds(node)); +__cold static int chk_tree(MDBX_chk_scope_t *const scope) { + MDBX_chk_internal_t *const chk = scope->internal; + MDBX_chk_context_t *const usr = chk->usr; + MDBX_env *const env = usr->env; + MDBX_txn *const txn = usr->txn; - pgr_t lp = page_get_large(mc, node_largedata_pgno(node), mp->mp_txnid); - if (unlikely((lp.err != MDBX_SUCCESS))) { - DEBUG("read large/overflow page %" PRIaPGNO " failed", - node_largedata_pgno(node)); - return lp.err; - } +#if defined(_WIN32) || defined(_WIN64) + SetLastError(ERROR_SUCCESS); +#else + errno = 0; +#endif /* Windows */ + chk->pagemap = osal_calloc(usr->result.alloc_pages, sizeof(*chk->pagemap)); + if (!chk->pagemap) { + int err = osal_get_errno(); + return chk_error_rc(scope, err ? err : MDBX_ENOMEM, "calloc"); + } + + if (scope->verbosity > MDBX_chk_info) + chk_scope_push(scope, 0, "Walking pages..."); + /* always skip key ordering checking + * to avoid MDBX_CORRUPTED in case custom comparators were used */ + usr->result.processed_pages = NUM_METAS; + int err = walk_pages(txn, chk_pgvisitor, scope, dont_check_keys_ordering); + if (MDBX_IS_ERROR(err) && err != MDBX_EINTR) + chk_error_rc(scope, err, "walk_pages"); + + for (size_t n = NUM_METAS; n < usr->result.alloc_pages; ++n) + if (!chk->pagemap[n]) + usr->result.unused_pages += 1; + + MDBX_chk_table_t total; + memset(&total, 0, sizeof(total)); + total.pages.all = NUM_METAS; + for (size_t i = 0; i < ARRAY_LENGTH(chk->table) && chk->table[i]; ++i) { + MDBX_chk_table_t *const tbl = chk->table[i]; + total.payload_bytes += tbl->payload_bytes; + total.lost_bytes += tbl->lost_bytes; + total.pages.all += tbl->pages.all; + total.pages.empty += tbl->pages.empty; + total.pages.other += tbl->pages.other; + total.pages.branch += tbl->pages.branch; + total.pages.leaf += tbl->pages.leaf; + total.pages.nested_branch += tbl->pages.nested_branch; + total.pages.nested_leaf += tbl->pages.nested_leaf; + total.pages.nested_subleaf += tbl->pages.nested_subleaf; + } + assert(total.pages.all == usr->result.processed_pages); + + const size_t total_page_bytes = pgno2bytes(env, total.pages.all); + if (usr->scope->subtotal_issues || usr->scope->verbosity >= MDBX_chk_verbose) + chk_line_end(chk_print(chk_line_begin(usr->scope, MDBX_chk_resolution), + "walked %zu pages, left/unused %zu" + ", %" PRIuSIZE " problem(s)", + usr->result.processed_pages, usr->result.unused_pages, usr->scope->subtotal_issues)); + + err = chk_scope_restore(scope, err); + if (scope->verbosity > MDBX_chk_info) { + for (size_t i = 0; i < ARRAY_LENGTH(chk->table) && chk->table[i]; ++i) { + MDBX_chk_table_t *const tbl = chk->table[i]; + MDBX_chk_scope_t *inner = chk_scope_push(scope, 0, "tree %s:", chk_v2a(chk, &tbl->name)); + if (tbl->pages.all == 0) + chk_line_end(chk_print(chk_line_begin(inner, MDBX_chk_resolution), "empty")); + else { + MDBX_chk_line_t *line = chk_line_begin(inner, MDBX_chk_info); + if (line) { + line = chk_print(line, "page usage: subtotal %" PRIuSIZE, tbl->pages.all); + const size_t branch_pages = tbl->pages.branch + tbl->pages.nested_branch; + const size_t leaf_pages = tbl->pages.leaf + tbl->pages.nested_leaf + tbl->pages.nested_subleaf; + if (tbl->pages.other) + line = chk_print(line, ", other %" PRIuSIZE, tbl->pages.other); + if (tbl->pages.other == 0 || (branch_pages | leaf_pages | tbl->histogram.large_pages.count) != 0) { + line = chk_print(line, ", branch %" PRIuSIZE ", leaf %" PRIuSIZE, branch_pages, leaf_pages); + if (tbl->histogram.large_pages.count || (tbl->flags & MDBX_DUPSORT) == 0) { + line = chk_print(line, ", large %" PRIuSIZE, tbl->histogram.large_pages.count); + if (tbl->histogram.large_pages.amount | tbl->histogram.large_pages.count) + line = histogram_print(inner, line, &tbl->histogram.large_pages, " amount", "single", true); + } + } + line = histogram_dist(chk_line_feed(line), &tbl->histogram.deep, "tree deep density", "1", false); + if (tbl != &chk->table_gc && tbl->histogram.nested_tree.count) { + line = chk_print(chk_line_feed(line), "nested tree(s) %" PRIuSIZE, tbl->histogram.nested_tree.count); + line = histogram_dist(line, &tbl->histogram.nested_tree, " density", "1", false); + line = chk_print(chk_line_feed(line), + "nested tree(s) pages %" PRIuSIZE ": branch %" PRIuSIZE ", leaf %" PRIuSIZE + ", subleaf %" PRIuSIZE, + tbl->pages.nested_branch + tbl->pages.nested_leaf, tbl->pages.nested_branch, + tbl->pages.nested_leaf, tbl->pages.nested_subleaf); + } - cASSERT(mc, PAGETYPE_WHOLE(lp.page) == P_OVERFLOW); - data->iov_base = page_data(lp.page); - if (!MDBX_DISABLE_VALIDATION) { - const MDBX_env *env = mc->mc_txn->mt_env; - const size_t dsize = data->iov_len; - const unsigned npages = number_of_ovpages(env, dsize); - if (unlikely(lp.page->mp_pages < npages)) - return bad_page(lp.page, - "too less n-pages %u for bigdata-node (%zu bytes)", - lp.page->mp_pages, dsize); + const size_t bytes = pgno2bytes(env, tbl->pages.all); + line = + chk_print(chk_line_feed(line), + "page filling: subtotal %" PRIuSIZE " bytes (%.1f%%), payload %" PRIuSIZE + " (%.1f%%), unused %" PRIuSIZE " (%.1f%%)", + bytes, bytes * 100.0 / total_page_bytes, tbl->payload_bytes, tbl->payload_bytes * 100.0 / bytes, + bytes - tbl->payload_bytes, (bytes - tbl->payload_bytes) * 100.0 / bytes); + if (tbl->pages.empty) + line = chk_print(line, ", %" PRIuSIZE " empty pages", tbl->pages.empty); + if (tbl->lost_bytes) + line = chk_print(line, ", %" PRIuSIZE " bytes lost", tbl->lost_bytes); + chk_line_end(line); + } + } + chk_scope_restore(scope, 0); + } } - return MDBX_SUCCESS; -} -/* Return the data associated with a given node. */ -static __always_inline int node_read(MDBX_cursor *mc, const MDBX_node *node, - MDBX_val *data, const MDBX_page *mp) { - data->iov_len = node_ds(node); - data->iov_base = node_data(node); - if (likely(node_flags(node) != F_BIGDATA)) - return MDBX_SUCCESS; - return node_read_bigdata(mc, node, data, mp); + MDBX_chk_line_t *line = chk_line_begin(scope, MDBX_chk_resolution); + line = chk_print(line, + "summary: total %" PRIuSIZE " bytes, payload %" PRIuSIZE " (%.1f%%), unused %" PRIuSIZE " (%.1f%%)," + " average fill %.1f%%", + total_page_bytes, usr->result.total_payload_bytes, + usr->result.total_payload_bytes * 100.0 / total_page_bytes, + total_page_bytes - usr->result.total_payload_bytes, + (total_page_bytes - usr->result.total_payload_bytes) * 100.0 / total_page_bytes, + usr->result.total_payload_bytes * 100.0 / total_page_bytes); + if (total.pages.empty) + line = chk_print(line, ", %" PRIuSIZE " empty pages", total.pages.empty); + if (total.lost_bytes) + line = chk_print(line, ", %" PRIuSIZE " bytes lost", total.lost_bytes); + chk_line_end(line); + return err; } -int mdbx_get(const MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, - MDBX_val *data) { - DKBUF_DEBUG; - DEBUG("===> get db %u key [%s]", dbi, DKEY_DEBUG(key)); - - int rc = check_txn(txn, MDBX_TXN_BLOCKED); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - - if (unlikely(!key || !data)) - return MDBX_EINVAL; +typedef int(chk_kv_visitor)(MDBX_chk_scope_t *const scope, MDBX_chk_table_t *tbl, const size_t record_number, + const MDBX_val *key, const MDBX_val *data); - if (unlikely(!check_dbi(txn, dbi, DBI_USRVALID))) - return MDBX_BAD_DBI; +__cold static int chk_handle_kv(MDBX_chk_scope_t *const scope, MDBX_chk_table_t *tbl, const size_t record_number, + const MDBX_val *key, const MDBX_val *data) { + MDBX_chk_internal_t *const chk = scope->internal; + int err = MDBX_SUCCESS; + assert(tbl->cookie); + if (chk->cb->table_handle_kv) + err = chk->cb->table_handle_kv(chk->usr, tbl, record_number, key, data); + return err ? err : chk_check_break(scope); +} + +__cold static int chk_db(MDBX_chk_scope_t *const scope, MDBX_dbi dbi, MDBX_chk_table_t *tbl, chk_kv_visitor *handler) { + MDBX_chk_internal_t *const chk = scope->internal; + MDBX_chk_context_t *const usr = chk->usr; + MDBX_env *const env = usr->env; + MDBX_txn *const txn = usr->txn; + MDBX_cursor *cursor = nullptr; + size_t record_count = 0, dups = 0, sub_databases = 0; + int err; - MDBX_cursor_couple cx; - rc = cursor_init(&cx.outer, txn, dbi); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; + if ((MDBX_TXN_FINISHED | MDBX_TXN_ERROR) & txn->flags) { + chk_line_end(chk_flush(chk_print(chk_line_begin(scope, MDBX_chk_error), + "abort processing %s due to a previous error", chk_v2a(chk, &tbl->name)))); + err = MDBX_BAD_TXN; + goto bailout; + } - return cursor_set(&cx.outer, (MDBX_val *)key, data, MDBX_SET).err; -} + if (0 > (int)dbi) { + err = dbi_open(txn, &tbl->name, MDBX_DB_ACCEDE, &dbi, + (chk->flags & MDBX_CHK_IGNORE_ORDER) ? cmp_equal_or_greater : nullptr, + (chk->flags & MDBX_CHK_IGNORE_ORDER) ? cmp_equal_or_greater : nullptr); + if (unlikely(err)) { + tASSERT(txn, dbi >= txn->env->n_dbi || (txn->env->dbs_flags[dbi] & DB_VALID) == 0); + chk_error_rc(scope, err, "mdbx_dbi_open"); + goto bailout; + } + tASSERT(txn, dbi < txn->env->n_dbi && (txn->env->dbs_flags[dbi] & DB_VALID) != 0); + } -int mdbx_get_equal_or_great(const MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, - MDBX_val *data) { - int rc = check_txn(txn, MDBX_TXN_BLOCKED); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; + const tree_t *const db = txn->dbs + dbi; + if (handler) { + const char *key_mode = nullptr; + switch (tbl->flags & (MDBX_REVERSEKEY | MDBX_INTEGERKEY)) { + case 0: + key_mode = "usual"; + break; + case MDBX_REVERSEKEY: + key_mode = "reserve"; + break; + case MDBX_INTEGERKEY: + key_mode = "ordinal"; + break; + case MDBX_REVERSEKEY | MDBX_INTEGERKEY: + key_mode = "msgpack"; + break; + default: + key_mode = "inconsistent"; + chk_scope_issue(scope, "wrong key-mode (0x%x)", tbl->flags & (MDBX_REVERSEKEY | MDBX_INTEGERKEY)); + } - if (unlikely(!key || !data)) - return MDBX_EINVAL; + const char *value_mode = nullptr; + switch (tbl->flags & (MDBX_DUPSORT | MDBX_REVERSEDUP | MDBX_DUPFIXED | MDBX_INTEGERDUP)) { + case 0: + value_mode = "single"; + break; + case MDBX_DUPSORT: + value_mode = "multi"; + break; + case MDBX_DUPSORT | MDBX_REVERSEDUP: + value_mode = "multi-reverse"; + break; + case MDBX_DUPSORT | MDBX_DUPFIXED: + value_mode = "multi-samelength"; + break; + case MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_REVERSEDUP: + value_mode = "multi-reverse-samelength"; + break; + case MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERDUP: + value_mode = "multi-ordinal"; + break; + case MDBX_DUPSORT | MDBX_INTEGERDUP | MDBX_REVERSEDUP: + value_mode = "multi-msgpack"; + break; + case MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERDUP | MDBX_REVERSEDUP: + value_mode = "reserved"; + break; + default: + value_mode = "inconsistent"; + chk_scope_issue(scope, "wrong value-mode (0x%x)", + tbl->flags & (MDBX_DUPSORT | MDBX_REVERSEDUP | MDBX_DUPFIXED | MDBX_INTEGERDUP)); + } - if (unlikely(!check_dbi(txn, dbi, DBI_USRVALID))) - return MDBX_BAD_DBI; + MDBX_chk_line_t *line = chk_line_begin(scope, MDBX_chk_info); + line = chk_print(line, "key-value kind: %s-key => %s-value", key_mode, value_mode); + line = chk_print(line, ", flags:"); + if (!tbl->flags) + line = chk_print(line, " none"); + else { + const uint8_t f[] = { + MDBX_DUPSORT, MDBX_INTEGERKEY, MDBX_REVERSEKEY, MDBX_DUPFIXED, MDBX_REVERSEDUP, MDBX_INTEGERDUP, 0}; + const char *const t[] = {"dupsort", "integerkey", "reversekey", "dupfix", "reversedup", "integerdup"}; + for (size_t i = 0; f[i]; i++) + if (tbl->flags & f[i]) + line = chk_print(line, " %s", t[i]); + } + chk_line_end(chk_print(line, " (0x%02X)", tbl->flags)); + + line = chk_print(chk_line_begin(scope, MDBX_chk_verbose), "entries %" PRIu64 ", sequence %" PRIu64, db->items, + db->sequence); + if (db->mod_txnid) + line = chk_print(line, ", last modification txn#%" PRIaTXN, db->mod_txnid); + if (db->root != P_INVALID) + line = chk_print(line, ", root #%" PRIaPGNO, db->root); + chk_line_end(line); + chk_line_end(chk_print(chk_line_begin(scope, MDBX_chk_verbose), + "b-tree depth %u, pages: branch %" PRIaPGNO ", leaf %" PRIaPGNO ", large %" PRIaPGNO, + db->height, db->branch_pages, db->leaf_pages, db->large_pages)); + + if ((chk->flags & MDBX_CHK_SKIP_BTREE_TRAVERSAL) == 0) { + const size_t branch_pages = tbl->pages.branch + tbl->pages.nested_branch; + const size_t leaf_pages = tbl->pages.leaf + tbl->pages.nested_leaf; + const size_t subtotal_pages = db->branch_pages + db->leaf_pages + db->large_pages; + if (subtotal_pages != tbl->pages.all) + chk_scope_issue(scope, "%s pages mismatch (%" PRIuSIZE " != walked %" PRIuSIZE ")", "subtotal", subtotal_pages, + tbl->pages.all); + if (db->branch_pages != branch_pages) + chk_scope_issue(scope, "%s pages mismatch (%" PRIaPGNO " != walked %" PRIuSIZE ")", "branch", db->branch_pages, + branch_pages); + if (db->leaf_pages != leaf_pages) + chk_scope_issue(scope, "%s pages mismatch (%" PRIaPGNO " != walked %" PRIuSIZE ")", "all-leaf", db->leaf_pages, + leaf_pages); + if (db->large_pages != tbl->histogram.large_pages.amount) + chk_scope_issue(scope, "%s pages mismatch (%" PRIaPGNO " != walked %" PRIuSIZE ")", "large/overlow", + db->large_pages, tbl->histogram.large_pages.amount); + } + } + + err = mdbx_cursor_open(txn, dbi, &cursor); + if (unlikely(err)) { + chk_error_rc(scope, err, "mdbx_cursor_open"); + goto bailout; + } + if (chk->flags & MDBX_CHK_IGNORE_ORDER) { + cursor->checking |= z_ignord | z_pagecheck; + if (cursor->subcur) + cursor->subcur->cursor.checking |= z_ignord | z_pagecheck; + } - if (unlikely(txn->mt_flags & MDBX_TXN_BLOCKED)) - return MDBX_BAD_TXN; + const size_t maxkeysize = mdbx_env_get_maxkeysize_ex(env, tbl->flags); + MDBX_val prev_key = {nullptr, 0}, prev_data = {nullptr, 0}; + MDBX_val key, data; + err = mdbx_cursor_get(cursor, &key, &data, MDBX_FIRST); + while (err == MDBX_SUCCESS) { + err = chk_check_break(scope); + if (unlikely(err)) + goto bailout; - MDBX_cursor_couple cx; - rc = cursor_init(&cx.outer, txn, dbi); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; + bool bad_key = false; + if (key.iov_len > maxkeysize) { + chk_object_issue(scope, "entry", record_count, "key length exceeds max-key-size", "%" PRIuPTR " > %" PRIuPTR, + key.iov_len, maxkeysize); + bad_key = true; + } else if ((tbl->flags & MDBX_INTEGERKEY) && key.iov_len != 8 && key.iov_len != 4) { + chk_object_issue(scope, "entry", record_count, "wrong key length", "%" PRIuPTR " != 4or8", key.iov_len); + bad_key = true; + } - return cursor_get(&cx.outer, key, data, MDBX_SET_LOWERBOUND); -} + bool bad_data = false; + if ((tbl->flags & MDBX_INTEGERDUP) && data.iov_len != 8 && data.iov_len != 4) { + chk_object_issue(scope, "entry", record_count, "wrong data length", "%" PRIuPTR " != 4or8", data.iov_len); + bad_data = true; + } -int mdbx_get_ex(const MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, - MDBX_val *data, size_t *values_count) { - DKBUF_DEBUG; - DEBUG("===> get db %u key [%s]", dbi, DKEY_DEBUG(key)); + if (prev_key.iov_base) { + if (prev_data.iov_base && !bad_data && (tbl->flags & MDBX_DUPFIXED) && prev_data.iov_len != data.iov_len) { + chk_object_issue(scope, "entry", record_count, "different data length", "%" PRIuPTR " != %" PRIuPTR, + prev_data.iov_len, data.iov_len); + bad_data = true; + } - int rc = check_txn(txn, MDBX_TXN_BLOCKED); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; + if (!bad_key) { + int cmp = mdbx_cmp(txn, dbi, &key, &prev_key); + if (cmp == 0) { + ++dups; + if ((tbl->flags & MDBX_DUPSORT) == 0) { + chk_object_issue(scope, "entry", record_count, "duplicated entries", nullptr); + if (prev_data.iov_base && data.iov_len == prev_data.iov_len && + memcmp(data.iov_base, prev_data.iov_base, data.iov_len) == 0) + chk_object_issue(scope, "entry", record_count, "complete duplicate", nullptr); + } else if (!bad_data && prev_data.iov_base) { + cmp = mdbx_dcmp(txn, dbi, &data, &prev_data); + if (cmp == 0) + chk_object_issue(scope, "entry", record_count, "complete duplicate", nullptr); + else if (cmp < 0 && !(chk->flags & MDBX_CHK_IGNORE_ORDER)) + chk_object_issue(scope, "entry", record_count, "wrong order of multi-values", nullptr); + } + } else if (cmp < 0 && !(chk->flags & MDBX_CHK_IGNORE_ORDER)) + chk_object_issue(scope, "entry", record_count, "wrong order of entries", nullptr); + } + } - if (unlikely(!key || !data)) - return MDBX_EINVAL; + if (!bad_key) { + if (!prev_key.iov_base && (tbl->flags & MDBX_INTEGERKEY)) + chk_line_end(chk_print(chk_line_begin(scope, MDBX_chk_info), "fixed key-size %" PRIuSIZE, key.iov_len)); + prev_key = key; + } + if (!bad_data) { + if (!prev_data.iov_base && (tbl->flags & (MDBX_INTEGERDUP | MDBX_DUPFIXED))) + chk_line_end(chk_print(chk_line_begin(scope, MDBX_chk_info), "fixed data-size %" PRIuSIZE, data.iov_len)); + prev_data = data; + } - if (unlikely(!check_dbi(txn, dbi, DBI_USRVALID))) - return MDBX_BAD_DBI; + record_count++; + histogram_acc(key.iov_len, &tbl->histogram.key_len); + histogram_acc(data.iov_len, &tbl->histogram.val_len); - MDBX_cursor_couple cx; - rc = cursor_init(&cx.outer, txn, dbi); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; + const node_t *const node = page_node(cursor->pg[cursor->top], cursor->ki[cursor->top]); + if (node_flags(node) == N_TREE) { + if (dbi != MAIN_DBI || (tbl->flags & (MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_REVERSEDUP | MDBX_INTEGERDUP))) + chk_object_issue(scope, "entry", record_count, "unexpected table", "node-flags 0x%x", node_flags(node)); + else if (data.iov_len != sizeof(tree_t)) + chk_object_issue(scope, "entry", record_count, "wrong table node size", "node-size %" PRIuSIZE " != %" PRIuSIZE, + data.iov_len, sizeof(tree_t)); + else if (scope->stage == MDBX_chk_maindb) + /* подсчитываем table при первом проходе */ + sub_databases += 1; + else { + /* обработка table при втором проходе */ + tree_t aligned_db; + memcpy(&aligned_db, data.iov_base, sizeof(aligned_db)); + walk_tbl_t tbl_info = {.name = key}; + tbl_info.internal = &aligned_db; + MDBX_chk_table_t *table; + err = chk_get_tbl(scope, &tbl_info, &table); + if (unlikely(err)) + goto bailout; + if (table->cookie) { + err = chk_scope_begin(chk, 0, MDBX_chk_tables, table, &usr->result.problems_kv, "Processing table %s...", + chk_v2a(chk, &table->name)); + if (likely(!err)) { + err = chk_db(usr->scope, (MDBX_dbi)-1, table, chk_handle_kv); + if (err != MDBX_EINTR && err != MDBX_RESULT_TRUE) + usr->result.table_processed += 1; + } + err = chk_scope_restore(scope, err); + if (unlikely(err)) + goto bailout; + } else + chk_line_end(chk_flush(chk_print(chk_line_begin(scope, MDBX_chk_processing), "Skip processing %s...", + chk_v2a(chk, &table->name)))); + } + } else if (handler) { + err = handler(scope, tbl, record_count, &key, &data); + if (unlikely(err)) + goto bailout; + } - rc = cursor_set(&cx.outer, key, data, MDBX_SET_KEY).err; - if (unlikely(rc != MDBX_SUCCESS)) { - if (rc == MDBX_NOTFOUND && values_count) - *values_count = 0; - return rc; + err = mdbx_cursor_get(cursor, &key, &data, MDBX_NEXT); } - if (values_count) { - *values_count = 1; - if (cx.outer.mc_xcursor != NULL) { - MDBX_node *node = page_node(cx.outer.mc_pg[cx.outer.mc_top], - cx.outer.mc_ki[cx.outer.mc_top]); - if (node_flags(node) & F_DUPDATA) { - // coverity[uninit_use : FALSE] - tASSERT(txn, cx.outer.mc_xcursor == &cx.inner && - (cx.inner.mx_cursor.mc_flags & C_INITIALIZED)); - // coverity[uninit_use : FALSE] - *values_count = - (sizeof(*values_count) >= sizeof(cx.inner.mx_db.md_entries) || - cx.inner.mx_db.md_entries <= PTRDIFF_MAX) - ? (size_t)cx.inner.mx_db.md_entries - : PTRDIFF_MAX; - } - } + err = (err != MDBX_NOTFOUND) ? chk_error_rc(scope, err, "mdbx_cursor_get") : MDBX_SUCCESS; + if (err == MDBX_SUCCESS && record_count != db->items) + chk_scope_issue(scope, "different number of entries %" PRIuSIZE " != %" PRIu64, record_count, db->items); +bailout: + if (cursor) { + if (handler) { + if (tbl->histogram.key_len.count) { + MDBX_chk_line_t *line = chk_line_begin(scope, MDBX_chk_info); + line = histogram_dist(line, &tbl->histogram.key_len, "key length density", "0/1", false); + chk_line_feed(line); + line = histogram_dist(line, &tbl->histogram.val_len, "value length density", "0/1", false); + chk_line_end(line); + } + if (scope->stage == MDBX_chk_maindb) + usr->result.table_total = sub_databases; + if (chk->cb->table_conclude) + err = chk->cb->table_conclude(usr, tbl, cursor, err); + MDBX_chk_line_t *line = chk_line_begin(scope, MDBX_chk_resolution); + line = chk_print(line, "summary: %" PRIuSIZE " records,", record_count); + if (dups || (tbl->flags & (MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_REVERSEDUP | MDBX_INTEGERDUP))) + line = chk_print(line, " %" PRIuSIZE " dups,", dups); + if (sub_databases || dbi == MAIN_DBI) + line = chk_print(line, " %" PRIuSIZE " tables,", sub_databases); + line = chk_print(line, + " %" PRIuSIZE " key's bytes," + " %" PRIuSIZE " data's bytes," + " %" PRIuSIZE " problem(s)", + tbl->histogram.key_len.amount, tbl->histogram.val_len.amount, scope->subtotal_issues); + chk_line_end(chk_flush(line)); + } + + mdbx_cursor_close(cursor); + if (!txn->cursors[dbi] && (txn->dbi_state[dbi] & DBI_FRESH)) + mdbx_dbi_close(env, dbi); } - return MDBX_SUCCESS; + return err; } -/* Find a sibling for a page. - * Replaces the page at the top of the cursor's stack with the specified - * sibling, if one exists. - * - * [in] mc The cursor for this operation. - * [in] dir SIBLING_LEFT or SIBLING_RIGHT. - * - * Returns 0 on success, non-zero on failure. */ -static int cursor_sibling(MDBX_cursor *mc, int dir) { - int rc; - MDBX_node *node; - MDBX_page *mp; - assert(dir == SIBLING_LEFT || dir == SIBLING_RIGHT); - - if (unlikely(mc->mc_snum < 2)) - return MDBX_NOTFOUND; /* root has no siblings */ +__cold static int chk_handle_gc(MDBX_chk_scope_t *const scope, MDBX_chk_table_t *tbl, const size_t record_number, + const MDBX_val *key, const MDBX_val *data) { + MDBX_chk_internal_t *const chk = scope->internal; + MDBX_chk_context_t *const usr = chk->usr; + assert(tbl == &chk->table_gc); + (void)tbl; + const char *bad = ""; + pgno_t *iptr = data->iov_base; - cursor_pop(mc); - DEBUG("parent page is page %" PRIaPGNO ", index %u", - mc->mc_pg[mc->mc_top]->mp_pgno, mc->mc_ki[mc->mc_top]); - - if ((dir == SIBLING_RIGHT) ? (mc->mc_ki[mc->mc_top] + (size_t)1 >= - page_numkeys(mc->mc_pg[mc->mc_top])) - : (mc->mc_ki[mc->mc_top] == 0)) { - DEBUG("no more keys aside, moving to next %s sibling", - dir ? "right" : "left"); - if (unlikely((rc = cursor_sibling(mc, dir)) != MDBX_SUCCESS)) { - /* undo cursor_pop before returning */ - mc->mc_top++; - mc->mc_snum++; - return rc; + if (key->iov_len != sizeof(txnid_t)) + chk_object_issue(scope, "entry", record_number, "wrong txn-id size", "key-size %" PRIuSIZE, key->iov_len); + else { + txnid_t txnid; + memcpy(&txnid, key->iov_base, sizeof(txnid)); + if (txnid < 1 || txnid > usr->txn->txnid) + chk_object_issue(scope, "entry", record_number, "wrong txn-id", "%" PRIaTXN, txnid); + else { + if (data->iov_len < sizeof(pgno_t) || data->iov_len % sizeof(pgno_t)) + chk_object_issue(scope, "entry", txnid, "wrong idl size", "%" PRIuPTR, data->iov_len); + size_t number = (data->iov_len >= sizeof(pgno_t)) ? *iptr++ : 0; + if (number > PAGELIST_LIMIT) + chk_object_issue(scope, "entry", txnid, "wrong idl length", "%" PRIuPTR, number); + else if ((number + 1) * sizeof(pgno_t) > data->iov_len) { + chk_object_issue(scope, "entry", txnid, "trimmed idl", "%" PRIuSIZE " > %" PRIuSIZE " (corruption)", + (number + 1) * sizeof(pgno_t), data->iov_len); + number = data->iov_len / sizeof(pgno_t) - 1; + } else if (data->iov_len - (number + 1) * sizeof(pgno_t) >= + /* LY: allow gap up to one page. it is ok + * and better than shink-and-retry inside gc_update() */ + usr->env->ps) + chk_object_issue(scope, "entry", txnid, "extra idl space", + "%" PRIuSIZE " < %" PRIuSIZE " (minor, not a trouble)", (number + 1) * sizeof(pgno_t), + data->iov_len); + + usr->result.gc_pages += number; + if (chk->envinfo.mi_latter_reader_txnid > txnid) + usr->result.reclaimable_pages += number; + + size_t prev = MDBX_PNL_ASCENDING ? NUM_METAS - 1 : usr->txn->geo.first_unallocated; + size_t span = 1; + for (size_t i = 0; i < number; ++i) { + const size_t pgno = iptr[i]; + if (pgno < NUM_METAS) + chk_object_issue(scope, "entry", txnid, "wrong idl entry", "pgno %" PRIuSIZE " < meta-pages %u", pgno, + NUM_METAS); + else if (pgno >= usr->result.backed_pages) + chk_object_issue(scope, "entry", txnid, "wrong idl entry", "pgno %" PRIuSIZE " > backed-pages %" PRIuSIZE, + pgno, usr->result.backed_pages); + else if (pgno >= usr->result.alloc_pages) + chk_object_issue(scope, "entry", txnid, "wrong idl entry", "pgno %" PRIuSIZE " > alloc-pages %" PRIuSIZE, + pgno, usr->result.alloc_pages - 1); + else { + if (MDBX_PNL_DISORDERED(prev, pgno)) { + bad = " [bad sequence]"; + chk_object_issue(scope, "entry", txnid, "bad sequence", "%" PRIuSIZE " %c [%" PRIuSIZE "].%" PRIuSIZE, prev, + (prev == pgno) ? '=' : (MDBX_PNL_ASCENDING ? '>' : '<'), i, pgno); + } + if (chk->pagemap) { + const intptr_t id = chk->pagemap[pgno]; + if (id == 0) + chk->pagemap[pgno] = -1 /* mark the pgno listed in GC */; + else if (id > 0) { + assert(id - 1 <= (intptr_t)ARRAY_LENGTH(chk->table)); + chk_object_issue(scope, "page", pgno, "already used", "by %s", chk_v2a(chk, &chk->table[id - 1]->name)); + } else + chk_object_issue(scope, "page", pgno, "already listed in GC", nullptr); + } + } + prev = pgno; + while (i + span < number && + iptr[i + span] == (MDBX_PNL_ASCENDING ? pgno_add(pgno, span) : pgno_sub(pgno, span))) + ++span; + } + if (tbl->cookie) { + chk_line_end(chk_print(chk_line_begin(scope, MDBX_chk_details), + "transaction %" PRIaTXN ", %" PRIuSIZE " pages, maxspan %" PRIuSIZE "%s", txnid, number, + span, bad)); + for (size_t i = 0; i < number; i += span) { + const size_t pgno = iptr[i]; + for (span = 1; i + span < number && + iptr[i + span] == (MDBX_PNL_ASCENDING ? pgno_add(pgno, span) : pgno_sub(pgno, span)); + ++span) + ; + histogram_acc(span, &tbl->histogram.nested_tree); + MDBX_chk_line_t *line = chk_line_begin(scope, MDBX_chk_extra); + if (line) { + if (span > 1) + line = chk_print(line, "%9" PRIuSIZE "[%" PRIuSIZE "]", pgno, span); + else + line = chk_print(line, "%9" PRIuSIZE, pgno); + chk_line_end(line); + int err = chk_check_break(scope); + if (err) + return err; + } + } + } } - } else { - assert((dir - 1) == -1 || (dir - 1) == 1); - mc->mc_ki[mc->mc_top] += (indx_t)(dir - 1); - DEBUG("just moving to %s index key %u", - (dir == SIBLING_RIGHT) ? "right" : "left", mc->mc_ki[mc->mc_top]); - } - cASSERT(mc, IS_BRANCH(mc->mc_pg[mc->mc_top])); - - node = page_node(mp = mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); - rc = page_get(mc, node_pgno(node), &mp, mp->mp_txnid); - if (unlikely(rc != MDBX_SUCCESS)) { - /* mc will be inconsistent if caller does mc_snum++ as above */ - mc->mc_flags &= ~(C_INITIALIZED | C_EOF); - return rc; } - - rc = cursor_push(mc, mp); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - - mc->mc_ki[mc->mc_top] = - (dir == SIBLING_LEFT) ? (indx_t)page_numkeys(mp) - 1 : 0; - return MDBX_SUCCESS; + return chk_check_break(scope); } -/* Move the cursor to the next data item. */ -static int cursor_next(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, - MDBX_cursor_op op) { - MDBX_page *mp; - MDBX_node *node; - int rc; +__cold static int env_chk(MDBX_chk_scope_t *const scope) { + MDBX_chk_internal_t *const chk = scope->internal; + MDBX_chk_context_t *const usr = chk->usr; + MDBX_env *const env = usr->env; + MDBX_txn *const txn = usr->txn; + int err = env_info(env, txn, &chk->envinfo, sizeof(chk->envinfo), &chk->troika); + if (unlikely(err)) + return chk_error_rc(scope, err, "env_info"); - if (unlikely(mc->mc_flags & C_DEL) && op == MDBX_NEXT_DUP) - return MDBX_NOTFOUND; + MDBX_chk_line_t *line = + chk_puts(chk_line_begin(scope, MDBX_chk_info - (1 << MDBX_chk_severity_prio_shift)), "dxb-id "); + if (chk->envinfo.mi_dxbid.x | chk->envinfo.mi_dxbid.y) + line = chk_print(line, "%016" PRIx64 "-%016" PRIx64, chk->envinfo.mi_dxbid.x, chk->envinfo.mi_dxbid.y); + else + line = chk_puts(line, "is absent"); + chk_line_end(line); - if (unlikely(!(mc->mc_flags & C_INITIALIZED))) - return cursor_first(mc, key, data); + line = chk_puts(chk_line_begin(scope, MDBX_chk_info), "current boot-id "); + if (chk->envinfo.mi_bootid.current.x | chk->envinfo.mi_bootid.current.y) + line = chk_print(line, "%016" PRIx64 "-%016" PRIx64, chk->envinfo.mi_bootid.current.x, + chk->envinfo.mi_bootid.current.y); + else + line = chk_puts(line, "is unavailable"); + chk_line_end(line); - mp = mc->mc_pg[mc->mc_top]; - if (unlikely(mc->mc_flags & C_EOF)) { - if (mc->mc_ki[mc->mc_top] + (size_t)1 >= page_numkeys(mp)) - return MDBX_NOTFOUND; - mc->mc_flags ^= C_EOF; - } + err = osal_filesize(env->lazy_fd, &env->dxb_mmap.filesize); + if (unlikely(err)) + return chk_error_rc(scope, err, "osal_filesize"); + + //-------------------------------------------------------------------------- + + err = chk_scope_begin(chk, 1, MDBX_chk_meta, nullptr, &usr->result.problems_meta, "Peek the meta-pages..."); + if (likely(!err)) { + MDBX_chk_scope_t *const inner = usr->scope; + const uint64_t dxbfile_pages = env->dxb_mmap.filesize >> env->ps2ln; + usr->result.alloc_pages = txn->geo.first_unallocated; + usr->result.backed_pages = bytes2pgno(env, env->dxb_mmap.current); + if (unlikely(usr->result.backed_pages > dxbfile_pages)) + chk_scope_issue(inner, "backed-pages %zu > file-pages %" PRIu64, usr->result.backed_pages, dxbfile_pages); + if (unlikely(dxbfile_pages < NUM_METAS)) + chk_scope_issue(inner, "file-pages %" PRIu64 " < %u", dxbfile_pages, NUM_METAS); + if (unlikely(usr->result.backed_pages < NUM_METAS)) + chk_scope_issue(inner, "backed-pages %zu < %u", usr->result.backed_pages, NUM_METAS); + if (unlikely(usr->result.backed_pages < NUM_METAS)) { + chk_scope_issue(inner, "backed-pages %zu < num-metas %u", usr->result.backed_pages, NUM_METAS); + return MDBX_CORRUPTED; + } + if (unlikely(dxbfile_pages < NUM_METAS)) { + chk_scope_issue(inner, "backed-pages %zu < num-metas %u", usr->result.backed_pages, NUM_METAS); + return MDBX_CORRUPTED; + } + if (unlikely(usr->result.backed_pages > (size_t)MAX_PAGENO + 1)) { + chk_scope_issue(inner, "backed-pages %zu > max-pages %zu", usr->result.backed_pages, (size_t)MAX_PAGENO + 1); + usr->result.backed_pages = MAX_PAGENO + 1; + } - if (mc->mc_db->md_flags & MDBX_DUPSORT) { - node = page_node(mp, mc->mc_ki[mc->mc_top]); - if (node_flags(node) & F_DUPDATA) { - if (op == MDBX_NEXT || op == MDBX_NEXT_DUP) { - rc = cursor_next(&mc->mc_xcursor->mx_cursor, data, NULL, MDBX_NEXT); - if (op != MDBX_NEXT || rc != MDBX_NOTFOUND) { - if (likely(rc == MDBX_SUCCESS)) - get_key_optional(node, key); - return rc; - } + if ((env->flags & (MDBX_EXCLUSIVE | MDBX_RDONLY)) != MDBX_RDONLY) { + if (unlikely(usr->result.backed_pages > dxbfile_pages)) { + chk_scope_issue(inner, "backed-pages %zu > file-pages %" PRIu64, usr->result.backed_pages, dxbfile_pages); + usr->result.backed_pages = (size_t)dxbfile_pages; + } + if (unlikely(usr->result.alloc_pages > usr->result.backed_pages)) { + chk_scope_issue(scope, "alloc-pages %zu > backed-pages %zu", usr->result.alloc_pages, usr->result.backed_pages); + usr->result.alloc_pages = usr->result.backed_pages; } } else { - mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED | C_EOF); - if (op == MDBX_NEXT_DUP) - return MDBX_NOTFOUND; - } - } - - DEBUG("cursor_next: top page is %" PRIaPGNO " in cursor %p", mp->mp_pgno, - (void *)mc); - if (mc->mc_flags & C_DEL) { - mc->mc_flags ^= C_DEL; - goto skip; - } + /* DB may be shrunk by writer down to the allocated (but unused) pages. */ + if (unlikely(usr->result.alloc_pages > usr->result.backed_pages)) { + chk_scope_issue(inner, "alloc-pages %zu > backed-pages %zu", usr->result.alloc_pages, usr->result.backed_pages); + usr->result.alloc_pages = usr->result.backed_pages; + } + if (unlikely(usr->result.alloc_pages > dxbfile_pages)) { + chk_scope_issue(inner, "alloc-pages %zu > file-pages %" PRIu64, usr->result.alloc_pages, dxbfile_pages); + usr->result.alloc_pages = (size_t)dxbfile_pages; + } + if (unlikely(usr->result.backed_pages > dxbfile_pages)) + usr->result.backed_pages = (size_t)dxbfile_pages; + } + + line = chk_line_feed(chk_print(chk_line_begin(inner, MDBX_chk_info), + "pagesize %u (%u system), max keysize %u..%u" + ", max readers %u", + env->ps, globals.sys_pagesize, mdbx_env_get_maxkeysize_ex(env, MDBX_DUPSORT), + mdbx_env_get_maxkeysize_ex(env, MDBX_DB_DEFAULTS), env->max_readers)); + line = chk_line_feed(chk_print_size(line, "mapsize ", env->dxb_mmap.current, nullptr)); + if (txn->geo.lower == txn->geo.upper) + line = chk_print_size(line, "fixed datafile: ", chk->envinfo.mi_geo.current, nullptr); + else { + line = chk_print_size(line, "dynamic datafile: ", chk->envinfo.mi_geo.lower, nullptr); + line = chk_print_size(line, " .. ", chk->envinfo.mi_geo.upper, ", "); + line = chk_print_size(line, "+", chk->envinfo.mi_geo.grow, ", "); + + line = chk_line_feed(chk_print_size(line, "-", chk->envinfo.mi_geo.shrink, nullptr)); + line = chk_print_size(line, "current datafile: ", chk->envinfo.mi_geo.current, nullptr); + } + tASSERT(txn, txn->geo.now == chk->envinfo.mi_geo.current / chk->envinfo.mi_dxb_pagesize); + chk_line_end(chk_print(line, ", %u pages", txn->geo.now)); +#if defined(_WIN32) || defined(_WIN64) || MDBX_DEBUG + if (txn->geo.shrink_pv && txn->geo.now != txn->geo.upper && scope->verbosity >= MDBX_chk_verbose) { + line = chk_line_begin(inner, MDBX_chk_notice); + chk_line_feed(chk_print(line, " > WARNING: Due Windows system limitations a file couldn't")); + chk_line_feed(chk_print(line, " > be truncated while the database is opened. So, the size")); + chk_line_feed(chk_print(line, " > database file of may by large than the database itself,")); + chk_line_end(chk_print(line, " > until it will be closed or reopened in read-write mode.")); + } +#endif /* Windows || Debug */ + chk_verbose_meta(inner, 0); + chk_verbose_meta(inner, 1); + chk_verbose_meta(inner, 2); + + if (env->stuck_meta >= 0) { + chk_line_end(chk_print(chk_line_begin(inner, MDBX_chk_processing), + "skip checking meta-pages since the %u" + " is selected for verification", + env->stuck_meta)); + line = chk_line_feed(chk_print(chk_line_begin(inner, MDBX_chk_resolution), + "transactions: recent %" PRIu64 ", " + "selected for verification %" PRIu64 ", lag %" PRIi64, + chk->envinfo.mi_recent_txnid, chk->envinfo.mi_meta_txnid[env->stuck_meta], + chk->envinfo.mi_recent_txnid - chk->envinfo.mi_meta_txnid[env->stuck_meta])); + chk_line_end(line); + } else { + chk_line_end(chk_puts(chk_line_begin(inner, MDBX_chk_verbose), "performs check for meta-pages clashes")); + const unsigned meta_clash_mask = meta_eq_mask(&chk->troika); + if (meta_clash_mask & 1) + chk_scope_issue(inner, "meta-%d and meta-%d are clashed", 0, 1); + if (meta_clash_mask & 2) + chk_scope_issue(inner, "meta-%d and meta-%d are clashed", 1, 2); + if (meta_clash_mask & 4) + chk_scope_issue(inner, "meta-%d and meta-%d are clashed", 2, 0); + + const unsigned prefer_steady_metanum = chk->troika.prefer_steady; + const uint64_t prefer_steady_txnid = chk->troika.txnid[prefer_steady_metanum]; + const unsigned recent_metanum = chk->troika.recent; + const uint64_t recent_txnid = chk->troika.txnid[recent_metanum]; + if (env->flags & MDBX_EXCLUSIVE) { + chk_line_end( + chk_puts(chk_line_begin(inner, MDBX_chk_verbose), "performs full check recent-txn-id with meta-pages")); + eASSERT(env, recent_txnid == chk->envinfo.mi_recent_txnid); + if (prefer_steady_txnid != recent_txnid) { + if ((chk->flags & MDBX_CHK_READWRITE) != 0 && (env->flags & MDBX_RDONLY) == 0 && + recent_txnid > prefer_steady_txnid && + (chk->envinfo.mi_bootid.current.x | chk->envinfo.mi_bootid.current.y) != 0 && + chk->envinfo.mi_bootid.current.x == chk->envinfo.mi_bootid.meta[recent_metanum].x && + chk->envinfo.mi_bootid.current.y == chk->envinfo.mi_bootid.meta[recent_metanum].y) { + chk_line_end(chk_print(chk_line_begin(inner, MDBX_chk_verbose), + "recent meta-%u is weak, but boot-id match current" + " (will synced upon successful check)", + recent_metanum)); + } else + chk_scope_issue(inner, "steady meta-%d txn-id mismatch recent-txn-id (%" PRIi64 " != %" PRIi64 ")", + prefer_steady_metanum, prefer_steady_txnid, recent_txnid); + } + } else if (chk->write_locked) { + chk_line_end(chk_puts(chk_line_begin(inner, MDBX_chk_verbose), + "performs lite check recent-txn-id with meta-pages (not a " + "monopolistic mode)")); + if (recent_txnid != chk->envinfo.mi_recent_txnid) { + chk_scope_issue(inner, "weak meta-%d txn-id mismatch recent-txn-id (%" PRIi64 " != %" PRIi64 ")", + recent_metanum, recent_txnid, chk->envinfo.mi_recent_txnid); + } + } else { + chk_line_end(chk_puts(chk_line_begin(inner, MDBX_chk_verbose), + "skip check recent-txn-id with meta-pages (monopolistic or " + "read-write mode only)")); + } - intptr_t ki = mc->mc_ki[mc->mc_top]; - mc->mc_ki[mc->mc_top] = (indx_t)++ki; - const intptr_t numkeys = page_numkeys(mp); - if (unlikely(ki >= numkeys)) { - DEBUG("%s", "=====> move to next sibling page"); - mc->mc_ki[mc->mc_top] = (indx_t)(numkeys - 1); - rc = cursor_sibling(mc, SIBLING_RIGHT); - if (unlikely(rc != MDBX_SUCCESS)) { - mc->mc_flags |= C_EOF; - return rc; + chk_line_end(chk_print(chk_line_begin(inner, MDBX_chk_resolution), + "transactions: recent %" PRIu64 ", latter reader %" PRIu64 ", lag %" PRIi64, + chk->envinfo.mi_recent_txnid, chk->envinfo.mi_latter_reader_txnid, + chk->envinfo.mi_recent_txnid - chk->envinfo.mi_latter_reader_txnid)); } - mp = mc->mc_pg[mc->mc_top]; - DEBUG("next page is %" PRIaPGNO ", key index %u", mp->mp_pgno, - mc->mc_ki[mc->mc_top]); } + err = chk_scope_restore(scope, err); -skip: - DEBUG("==> cursor points to page %" PRIaPGNO " with %zu keys, key index %u", - mp->mp_pgno, page_numkeys(mp), mc->mc_ki[mc->mc_top]); + //-------------------------------------------------------------------------- - if (!MDBX_DISABLE_VALIDATION && unlikely(!CHECK_LEAF_TYPE(mc, mp))) { - ERROR("unexpected leaf-page #%" PRIaPGNO " type 0x%x seen by cursor", - mp->mp_pgno, mp->mp_flags); - return MDBX_CORRUPTED; - } - - if (IS_LEAF2(mp)) { - if (likely(key)) { - key->iov_len = mc->mc_db->md_xsize; - key->iov_base = page_leaf2key(mp, mc->mc_ki[mc->mc_top], key->iov_len); - } + const char *const subj_tree = "B-Trees"; + if (chk->flags & MDBX_CHK_SKIP_BTREE_TRAVERSAL) + chk_line_end(chk_print(chk_line_begin(scope, MDBX_chk_processing), "Skipping %s traversal...", subj_tree)); + else { + err = chk_scope_begin(chk, -1, MDBX_chk_tree, nullptr, &usr->result.tree_problems, + "Traversal %s by txn#%" PRIaTXN "...", subj_tree, txn->txnid); + if (likely(!err)) + err = chk_tree(usr->scope); + if (usr->result.tree_problems && usr->result.gc_tree_problems == 0) + usr->result.gc_tree_problems = usr->result.tree_problems; + if (usr->result.tree_problems && usr->result.kv_tree_problems == 0) + usr->result.kv_tree_problems = usr->result.tree_problems; + chk_scope_restore(scope, err); + } + + const char *const subj_gc = chk_v2a(chk, MDBX_CHK_GC); + if (usr->result.gc_tree_problems > 0) + chk_line_end(chk_print(chk_line_begin(scope, MDBX_chk_processing), + "Skip processing %s since %s is corrupted (%" PRIuSIZE " problem(s))", subj_gc, subj_tree, + usr->result.problems_gc = usr->result.gc_tree_problems)); + else { + err = chk_scope_begin(chk, -1, MDBX_chk_gc, &chk->table_gc, &usr->result.problems_gc, + "Processing %s by txn#%" PRIaTXN "...", subj_gc, txn->txnid); + if (likely(!err)) + err = chk_db(usr->scope, FREE_DBI, &chk->table_gc, chk_handle_gc); + line = chk_line_begin(scope, MDBX_chk_info); + if (line) { + histogram_print(scope, line, &chk->table_gc.histogram.nested_tree, "span(s)", "single", false); + chk_line_end(line); + } + if (usr->result.problems_gc == 0 && (chk->flags & MDBX_CHK_SKIP_BTREE_TRAVERSAL) == 0) { + const size_t used_pages = usr->result.alloc_pages - usr->result.gc_pages; + if (usr->result.processed_pages != used_pages) + chk_scope_issue(usr->scope, "used pages mismatch (%" PRIuSIZE "(walked) != %" PRIuSIZE "(allocated - GC))", + usr->result.processed_pages, used_pages); + if (usr->result.unused_pages != usr->result.gc_pages) + chk_scope_issue(usr->scope, "GC pages mismatch (%" PRIuSIZE "(expected) != %" PRIuSIZE "(GC))", + usr->result.unused_pages, usr->result.gc_pages); + } + } + chk_scope_restore(scope, err); + + //-------------------------------------------------------------------------- + + err = chk_scope_begin(chk, 1, MDBX_chk_space, nullptr, nullptr, "Page allocation:"); + const double percent_boundary_reciprocal = 100.0 / txn->geo.upper; + const double percent_backed_reciprocal = 100.0 / usr->result.backed_pages; + const size_t detained = usr->result.gc_pages - usr->result.reclaimable_pages; + const size_t available2boundary = txn->geo.upper - usr->result.alloc_pages + usr->result.reclaimable_pages; + const size_t available2backed = usr->result.backed_pages - usr->result.alloc_pages + usr->result.reclaimable_pages; + const size_t remained2boundary = txn->geo.upper - usr->result.alloc_pages; + const size_t remained2backed = usr->result.backed_pages - usr->result.alloc_pages; + + const size_t used = (chk->flags & MDBX_CHK_SKIP_BTREE_TRAVERSAL) ? usr->result.alloc_pages - usr->result.gc_pages + : usr->result.processed_pages; + + line = chk_line_begin(usr->scope, MDBX_chk_info); + line = chk_print(line, + "backed by file: %" PRIuSIZE " pages (%.1f%%)" + ", %" PRIuSIZE " left to boundary (%.1f%%)", + usr->result.backed_pages, usr->result.backed_pages * percent_boundary_reciprocal, + txn->geo.upper - usr->result.backed_pages, + (txn->geo.upper - usr->result.backed_pages) * percent_boundary_reciprocal); + line = chk_line_feed(line); + + line = chk_print(line, "%s: %" PRIuSIZE " page(s), %.1f%% of backed, %.1f%% of boundary", "used", used, + used * percent_backed_reciprocal, used * percent_boundary_reciprocal); + line = chk_line_feed(line); + + line = chk_print(line, "%s: %" PRIuSIZE " page(s) (%.1f%%) of backed, %" PRIuSIZE " to boundary (%.1f%% of boundary)", + "remained", remained2backed, remained2backed * percent_backed_reciprocal, remained2boundary, + remained2boundary * percent_boundary_reciprocal); + line = chk_line_feed(line); + + line = + chk_print(line, + "reclaimable: %" PRIuSIZE " (%.1f%% of backed, %.1f%% of boundary)" + ", GC %" PRIuSIZE " (%.1f%% of backed, %.1f%% of boundary)", + usr->result.reclaimable_pages, usr->result.reclaimable_pages * percent_backed_reciprocal, + usr->result.reclaimable_pages * percent_boundary_reciprocal, usr->result.gc_pages, + usr->result.gc_pages * percent_backed_reciprocal, usr->result.gc_pages * percent_boundary_reciprocal); + line = chk_line_feed(line); + + line = chk_print(line, + "detained by reader(s): %" PRIuSIZE " (%.1f%% of backed, %.1f%% of boundary)" + ", %u reader(s), lag %" PRIi64, + detained, detained * percent_backed_reciprocal, detained * percent_boundary_reciprocal, + chk->envinfo.mi_numreaders, chk->envinfo.mi_recent_txnid - chk->envinfo.mi_latter_reader_txnid); + line = chk_line_feed(line); + + line = chk_print(line, "%s: %" PRIuSIZE " page(s), %.1f%% of backed, %.1f%% of boundary", "allocated", + usr->result.alloc_pages, usr->result.alloc_pages * percent_backed_reciprocal, + usr->result.alloc_pages * percent_boundary_reciprocal); + line = chk_line_feed(line); + + line = chk_print(line, "%s: %" PRIuSIZE " page(s) (%.1f%%) of backed, %" PRIuSIZE " to boundary (%.1f%% of boundary)", + "available", available2backed, available2backed * percent_backed_reciprocal, available2boundary, + available2boundary * percent_boundary_reciprocal); + chk_line_end(line); + + line = chk_line_begin(usr->scope, MDBX_chk_resolution); + line = chk_print(line, "%s %" PRIaPGNO " pages", (txn->geo.upper == txn->geo.now) ? "total" : "upto", txn->geo.upper); + line = chk_print(line, ", backed %" PRIuSIZE " (%.1f%%)", usr->result.backed_pages, + usr->result.backed_pages * percent_boundary_reciprocal); + line = chk_print(line, ", allocated %" PRIuSIZE " (%.1f%%)", usr->result.alloc_pages, + usr->result.alloc_pages * percent_boundary_reciprocal); + line = chk_print(line, ", available %" PRIuSIZE " (%.1f%%)", available2boundary, + available2boundary * percent_boundary_reciprocal); + chk_line_end(line); + chk_scope_restore(scope, err); + + //-------------------------------------------------------------------------- + + const char *const subj_main = chk_v2a(chk, MDBX_CHK_MAIN); + if (chk->flags & MDBX_CHK_SKIP_KV_TRAVERSAL) + chk_line_end(chk_print(chk_line_begin(scope, MDBX_chk_processing), "Skip processing %s...", subj_main)); + else if ((usr->result.problems_kv = usr->result.kv_tree_problems) > 0) + chk_line_end(chk_print(chk_line_begin(scope, MDBX_chk_processing), + "Skip processing %s since %s is corrupted (%" PRIuSIZE " problem(s))", subj_main, subj_tree, + usr->result.problems_kv = usr->result.kv_tree_problems)); + else { + err = chk_scope_begin(chk, 0, MDBX_chk_maindb, &chk->table_main, &usr->result.problems_kv, "Processing %s...", + subj_main); + if (likely(!err)) + err = chk_db(usr->scope, MAIN_DBI, &chk->table_main, chk_handle_kv); + chk_scope_restore(scope, err); + + const char *const subj_tables = "table(s)"; + if (usr->result.problems_kv && usr->result.table_total) + chk_line_end(chk_print(chk_line_begin(scope, MDBX_chk_processing), "Skip processing %s", subj_tables)); + else if (usr->result.problems_kv == 0 && usr->result.table_total == 0) + chk_line_end(chk_print(chk_line_begin(scope, MDBX_chk_info), "No %s", subj_tables)); + else if (usr->result.problems_kv == 0 && usr->result.table_total) { + err = chk_scope_begin(chk, 1, MDBX_chk_tables, nullptr, &usr->result.problems_kv, + "Processing %s by txn#%" PRIaTXN "...", subj_tables, txn->txnid); + if (!err) + err = chk_db(usr->scope, MAIN_DBI, &chk->table_main, nullptr); + if (usr->scope->subtotal_issues) + chk_line_end(chk_print(chk_line_begin(usr->scope, MDBX_chk_resolution), + "processed %" PRIuSIZE " of %" PRIuSIZE " %s, %" PRIuSIZE " problems(s)", + usr->result.table_processed, usr->result.table_total, subj_tables, + usr->scope->subtotal_issues)); + } + chk_scope_restore(scope, err); + } + + return chk_scope_end(chk, chk_scope_begin(chk, 0, MDBX_chk_conclude, nullptr, nullptr, nullptr)); +} + +__cold int mdbx_env_chk_encount_problem(MDBX_chk_context_t *ctx) { + if (likely(ctx && ctx->internal && ctx->internal->usr == ctx && ctx->internal->problem_counter && ctx->scope)) { + *ctx->internal->problem_counter += 1; + ctx->scope->subtotal_issues += 1; return MDBX_SUCCESS; } + return MDBX_EINVAL; +} - node = page_node(mp, mc->mc_ki[mc->mc_top]); - if (node_flags(node) & F_DUPDATA) { - rc = cursor_xinit1(mc, node, mp); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - rc = cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - } else if (likely(data)) { - rc = node_read(mc, node, data, mp); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - } - - get_key_optional(node, key); - return MDBX_SUCCESS; -} - -/* Move the cursor to the previous data item. */ -static int cursor_prev(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, - MDBX_cursor_op op) { - MDBX_page *mp; - MDBX_node *node; - int rc; - - if (unlikely(mc->mc_flags & C_DEL) && op == MDBX_PREV_DUP) - return MDBX_NOTFOUND; - - if (unlikely(!(mc->mc_flags & C_INITIALIZED))) { - rc = cursor_last(mc, key, data); +__cold int mdbx_env_chk(MDBX_env *env, const struct MDBX_chk_callbacks *cb, MDBX_chk_context_t *ctx, + const MDBX_chk_flags_t flags, MDBX_chk_severity_t verbosity, unsigned timeout_seconds_16dot16) { + int err, rc = check_env(env, false); + if (unlikely(rc != MDBX_SUCCESS)) + return LOG_IFERR(rc); + if (unlikely(!cb || !ctx || ctx->internal)) + return LOG_IFERR(MDBX_EINVAL); + + MDBX_chk_internal_t *const chk = osal_calloc(1, sizeof(MDBX_chk_internal_t)); + if (unlikely(!chk)) + return LOG_IFERR(MDBX_ENOMEM); + + chk->cb = cb; + chk->usr = ctx; + chk->usr->internal = chk; + chk->usr->env = env; + chk->flags = flags; + + chk->table_gc.id = -1; + chk->table_gc.name.iov_base = MDBX_CHK_GC; + chk->table[FREE_DBI] = &chk->table_gc; + + chk->table_main.id = -1; + chk->table_main.name.iov_base = MDBX_CHK_MAIN; + chk->table[MAIN_DBI] = &chk->table_main; + + chk->monotime_timeout = + timeout_seconds_16dot16 ? osal_16dot16_to_monotime(timeout_seconds_16dot16) + osal_monotime() : 0; + chk->usr->scope_nesting = 0; + chk->usr->result.tables = (const void *)&chk->table; + + MDBX_chk_scope_t *const top = chk->scope_stack; + top->verbosity = verbosity; + top->internal = chk; + + // init + rc = chk_scope_end(chk, chk_scope_begin(chk, 0, MDBX_chk_init, nullptr, nullptr, nullptr)); + + // lock + if (likely(!rc)) + rc = chk_scope_begin(chk, 0, MDBX_chk_lock, nullptr, nullptr, "Taking %slock...", + (env->flags & (MDBX_RDONLY | MDBX_EXCLUSIVE)) ? "" : "read "); + if (likely(!rc) && (env->flags & (MDBX_RDONLY | MDBX_EXCLUSIVE)) == 0 && (flags & MDBX_CHK_READWRITE)) { + rc = mdbx_txn_lock(env, false); if (unlikely(rc)) - return rc; - mc->mc_ki[mc->mc_top]++; - } - - mp = mc->mc_pg[mc->mc_top]; - if ((mc->mc_db->md_flags & MDBX_DUPSORT) && - mc->mc_ki[mc->mc_top] < page_numkeys(mp)) { - node = page_node(mp, mc->mc_ki[mc->mc_top]); - if (node_flags(node) & F_DUPDATA) { - if (op == MDBX_PREV || op == MDBX_PREV_DUP) { - rc = cursor_prev(&mc->mc_xcursor->mx_cursor, data, NULL, MDBX_PREV); - if (op != MDBX_PREV || rc != MDBX_NOTFOUND) { - if (likely(rc == MDBX_SUCCESS)) { - get_key_optional(node, key); - mc->mc_flags &= ~C_EOF; - } - return rc; - } - } - } else { - mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED | C_EOF); - if (op == MDBX_PREV_DUP) - return MDBX_NOTFOUND; - } + chk_error_rc(ctx->scope, rc, "mdbx_txn_lock"); + else + chk->write_locked = true; } - - DEBUG("cursor_prev: top page is %" PRIaPGNO " in cursor %p", mp->mp_pgno, - (void *)mc); - - mc->mc_flags &= ~(C_EOF | C_DEL); - - int ki = mc->mc_ki[mc->mc_top]; - mc->mc_ki[mc->mc_top] = (indx_t)--ki; - if (unlikely(ki < 0)) { - mc->mc_ki[mc->mc_top] = 0; - DEBUG("%s", "=====> move to prev sibling page"); - if ((rc = cursor_sibling(mc, SIBLING_LEFT)) != MDBX_SUCCESS) - return rc; - mp = mc->mc_pg[mc->mc_top]; - DEBUG("prev page is %" PRIaPGNO ", key index %u", mp->mp_pgno, - mc->mc_ki[mc->mc_top]); + if (likely(!rc)) { + rc = mdbx_txn_begin(env, nullptr, MDBX_TXN_RDONLY, &ctx->txn); + if (unlikely(rc)) + chk_error_rc(ctx->scope, rc, "mdbx_txn_begin"); } - DEBUG("==> cursor points to page %" PRIaPGNO " with %zu keys, key index %u", - mp->mp_pgno, page_numkeys(mp), mc->mc_ki[mc->mc_top]); + chk_scope_end(chk, rc); - if (!MDBX_DISABLE_VALIDATION && unlikely(!CHECK_LEAF_TYPE(mc, mp))) { - ERROR("unexpected leaf-page #%" PRIaPGNO " type 0x%x seen by cursor", - mp->mp_pgno, mp->mp_flags); - return MDBX_CORRUPTED; + // doit + if (likely(!rc)) { + chk->table_gc.flags = ctx->txn->dbs[FREE_DBI].flags; + chk->table_main.flags = ctx->txn->dbs[MAIN_DBI].flags; + rc = env_chk(top); } - if (IS_LEAF2(mp)) { - if (likely(key)) { - key->iov_len = mc->mc_db->md_xsize; - key->iov_base = page_leaf2key(mp, mc->mc_ki[mc->mc_top], key->iov_len); + // unlock + if (ctx->txn || chk->write_locked) { + chk_scope_begin(chk, 0, MDBX_chk_unlock, nullptr, nullptr, nullptr); + if (ctx->txn) { + err = mdbx_txn_abort(ctx->txn); + if (err && !rc) + rc = err; + ctx->txn = nullptr; } - return MDBX_SUCCESS; + if (chk->write_locked) + mdbx_txn_unlock(env); + rc = chk_scope_end(chk, rc); } - node = page_node(mp, mc->mc_ki[mc->mc_top]); + // finalize + err = chk_scope_begin(chk, 0, MDBX_chk_finalize, nullptr, nullptr, nullptr); + rc = chk_scope_end(chk, err ? err : rc); + chk_dispose(chk); + return LOG_IFERR(rc); +} +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 - if (node_flags(node) & F_DUPDATA) { - rc = cursor_xinit1(mc, node, mp); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - rc = cursor_last(&mc->mc_xcursor->mx_cursor, data, NULL); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - } else if (likely(data)) { - rc = node_read(mc, node, data, mp); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - } +/*------------------------------------------------------------------------------ + * Pack/Unpack 16-bit values for Grow step & Shrink threshold */ - get_key_optional(node, key); - return MDBX_SUCCESS; +MDBX_NOTHROW_CONST_FUNCTION static inline pgno_t me2v(size_t m, size_t e) { + assert(m < 2048 && e < 8); + return (pgno_t)(32768 + ((m + 1) << (e + 8))); } -/* Set the cursor on a specific data item. */ -__hot static struct cursor_set_result -cursor_set(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, MDBX_cursor_op op) { - MDBX_page *mp; - MDBX_node *node = NULL; - DKBUF_DEBUG; +MDBX_NOTHROW_CONST_FUNCTION static inline uint16_t v2me(size_t v, size_t e) { + assert(v > (e ? me2v(2047, e - 1) : 32768)); + assert(v <= me2v(2047, e)); + size_t m = (v - 32768 + ((size_t)1 << (e + 8)) - 1) >> (e + 8); + m -= m > 0; + assert(m < 2048 && e < 8); + // f e d c b a 9 8 7 6 5 4 3 2 1 0 + // 1 e e e m m m m m m m m m m m 1 + const uint16_t pv = (uint16_t)(0x8001 + (e << 12) + (m << 1)); + assert(pv != 65535); + return pv; +} - struct cursor_set_result ret; - ret.exact = false; - if (unlikely(key->iov_len < mc->mc_dbx->md_klen_min || - (key->iov_len > mc->mc_dbx->md_klen_max && - (mc->mc_dbx->md_klen_min == mc->mc_dbx->md_klen_max || MDBX_DEBUG || MDBX_FORCE_ASSERTIONS)))) { - cASSERT(mc, !"Invalid key-size"); - ret.err = MDBX_BAD_VALSIZE; - return ret; - } +/* Convert 16-bit packed (exponential quantized) value to number of pages */ +pgno_t pv2pages(uint16_t pv) { + if ((pv & 0x8001) != 0x8001) + return pv; + if (pv == 65535) + return 65536; + // f e d c b a 9 8 7 6 5 4 3 2 1 0 + // 1 e e e m m m m m m m m m m m 1 + return me2v((pv >> 1) & 2047, (pv >> 12) & 7); +} - MDBX_val aligned_key = *key; - uint64_t aligned_keybytes; - if (mc->mc_db->md_flags & MDBX_INTEGERKEY) { - switch (aligned_key.iov_len) { - default: - cASSERT(mc, !"key-size is invalid for MDBX_INTEGERKEY"); - ret.err = MDBX_BAD_VALSIZE; - return ret; - case 4: - if (unlikely(3 & (uintptr_t)aligned_key.iov_base)) - /* copy instead of return error to avoid break compatibility */ - aligned_key.iov_base = - memcpy(&aligned_keybytes, aligned_key.iov_base, 4); - break; - case 8: - if (unlikely(7 & (uintptr_t)aligned_key.iov_base)) - /* copy instead of return error to avoid break compatibility */ - aligned_key.iov_base = - memcpy(&aligned_keybytes, aligned_key.iov_base, 8); - break; +/* Convert number of pages to 16-bit packed (exponential quantized) value */ +uint16_t pages2pv(size_t pages) { + if (pages < 32769 || (pages < 65536 && (pages & 1) == 0)) + return (uint16_t)pages; + if (pages <= me2v(2047, 0)) + return v2me(pages, 0); + if (pages <= me2v(2047, 1)) + return v2me(pages, 1); + if (pages <= me2v(2047, 2)) + return v2me(pages, 2); + if (pages <= me2v(2047, 3)) + return v2me(pages, 3); + if (pages <= me2v(2047, 4)) + return v2me(pages, 4); + if (pages <= me2v(2047, 5)) + return v2me(pages, 5); + if (pages <= me2v(2047, 6)) + return v2me(pages, 6); + return (pages < me2v(2046, 7)) ? v2me(pages, 7) : 65533; +} + +__cold bool pv2pages_verify(void) { + bool ok = true, dump_translation = false; + for (size_t i = 0; i < 65536; ++i) { + size_t pages = pv2pages(i); + size_t x = pages2pv(pages); + size_t xp = pv2pages(x); + if (pages != xp) { + ERROR("%zu => %zu => %zu => %zu\n", i, pages, x, xp); + ok = false; + } else if (dump_translation && !(x == i || (x % 2 == 0 && x < 65536))) { + DEBUG("%zu => %zu => %zu => %zu\n", i, pages, x, xp); } } + return ok; +} - if (mc->mc_xcursor) - mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED | C_EOF); +/*----------------------------------------------------------------------------*/ - /* See if we're already on the right page */ - if (mc->mc_flags & C_INITIALIZED) { - MDBX_val nodekey; +MDBX_NOTHROW_PURE_FUNCTION size_t bytes_align2os_bytes(const MDBX_env *env, size_t bytes) { + return ceil_powerof2(bytes, (env->ps > globals.sys_pagesize) ? env->ps : globals.sys_pagesize); +} - cASSERT(mc, IS_LEAF(mc->mc_pg[mc->mc_top])); - mp = mc->mc_pg[mc->mc_top]; - if (unlikely(!page_numkeys(mp))) { - mc->mc_ki[mc->mc_top] = 0; - mc->mc_flags |= C_EOF; - ret.err = MDBX_NOTFOUND; - return ret; - } - if (IS_LEAF2(mp)) { - nodekey.iov_len = mc->mc_db->md_xsize; - nodekey.iov_base = page_leaf2key(mp, 0, nodekey.iov_len); - } else { - node = page_node(mp, 0); - get_key(node, &nodekey); - } - int cmp = mc->mc_dbx->md_cmp(&aligned_key, &nodekey); - if (unlikely(cmp == 0)) { - /* Probably happens rarely, but first node on the page - * was the one we wanted. */ - mc->mc_ki[mc->mc_top] = 0; - ret.exact = true; - cASSERT(mc, mc->mc_ki[mc->mc_top] < page_numkeys(mc->mc_pg[mc->mc_top]) || - (mc->mc_flags & C_EOF)); - goto got_node; - } - if (cmp > 0) { - const size_t nkeys = page_numkeys(mp); - if (nkeys > 1) { - if (IS_LEAF2(mp)) { - nodekey.iov_base = page_leaf2key(mp, nkeys - 1, nodekey.iov_len); - } else { - node = page_node(mp, nkeys - 1); - get_key(node, &nodekey); - } - cmp = mc->mc_dbx->md_cmp(&aligned_key, &nodekey); - if (cmp == 0) { - /* last node was the one we wanted */ - cASSERT(mc, nkeys >= 1 && nkeys <= UINT16_MAX + 1); - mc->mc_ki[mc->mc_top] = (indx_t)(nkeys - 1); - ret.exact = true; - cASSERT(mc, - mc->mc_ki[mc->mc_top] < page_numkeys(mc->mc_pg[mc->mc_top]) || - (mc->mc_flags & C_EOF)); - goto got_node; - } - if (cmp < 0) { - if (mc->mc_ki[mc->mc_top] < page_numkeys(mp)) { - /* This is definitely the right page, skip search_page */ - if (IS_LEAF2(mp)) { - nodekey.iov_base = - page_leaf2key(mp, mc->mc_ki[mc->mc_top], nodekey.iov_len); - } else { - node = page_node(mp, mc->mc_ki[mc->mc_top]); - get_key(node, &nodekey); - } - cmp = mc->mc_dbx->md_cmp(&aligned_key, &nodekey); - if (cmp == 0) { - /* current node was the one we wanted */ - ret.exact = true; - cASSERT(mc, mc->mc_ki[mc->mc_top] < - page_numkeys(mc->mc_pg[mc->mc_top]) || - (mc->mc_flags & C_EOF)); - goto got_node; - } - } - mc->mc_flags &= ~C_EOF; - goto search_node; - } - } - /* If any parents have right-sibs, search. - * Otherwise, there's nothing further. */ - size_t i; - for (i = 0; i < mc->mc_top; i++) - if (mc->mc_ki[i] < page_numkeys(mc->mc_pg[i]) - 1) - break; - if (i == mc->mc_top) { - /* There are no other pages */ - cASSERT(mc, nkeys <= UINT16_MAX); - mc->mc_ki[mc->mc_top] = (uint16_t)nkeys; - mc->mc_flags |= C_EOF; - ret.err = MDBX_NOTFOUND; - return ret; - } - } - if (!mc->mc_top) { - /* There are no other pages */ - mc->mc_ki[mc->mc_top] = 0; - if (op == MDBX_SET_RANGE) - goto got_node; +MDBX_NOTHROW_PURE_FUNCTION size_t pgno_align2os_bytes(const MDBX_env *env, size_t pgno) { + return ceil_powerof2(pgno2bytes(env, pgno), globals.sys_pagesize); +} - cASSERT(mc, mc->mc_ki[mc->mc_top] < page_numkeys(mc->mc_pg[mc->mc_top]) || - (mc->mc_flags & C_EOF)); - ret.err = MDBX_NOTFOUND; - return ret; - } - } else { - mc->mc_pg[0] = nullptr; +MDBX_NOTHROW_PURE_FUNCTION pgno_t pgno_align2os_pgno(const MDBX_env *env, size_t pgno) { + return bytes2pgno(env, pgno_align2os_bytes(env, pgno)); +} + +/*----------------------------------------------------------------------------*/ + +MDBX_NOTHROW_PURE_FUNCTION static __always_inline int cmp_int_inline(const size_t expected_alignment, const MDBX_val *a, + const MDBX_val *b) { + if (likely(a->iov_len == b->iov_len)) { + if (sizeof(size_t) > 7 && likely(a->iov_len == 8)) + return CMP2INT(unaligned_peek_u64(expected_alignment, a->iov_base), + unaligned_peek_u64(expected_alignment, b->iov_base)); + if (likely(a->iov_len == 4)) + return CMP2INT(unaligned_peek_u32(expected_alignment, a->iov_base), + unaligned_peek_u32(expected_alignment, b->iov_base)); + if (sizeof(size_t) < 8 && likely(a->iov_len == 8)) + return CMP2INT(unaligned_peek_u64(expected_alignment, a->iov_base), + unaligned_peek_u64(expected_alignment, b->iov_base)); } + ERROR("mismatch and/or invalid size %p.%zu/%p.%zu for INTEGERKEY/INTEGERDUP", a->iov_base, a->iov_len, b->iov_base, + b->iov_len); + return 0; +} - ret.err = page_search(mc, &aligned_key, 0); - if (unlikely(ret.err != MDBX_SUCCESS)) - return ret; +MDBX_NOTHROW_PURE_FUNCTION __hot int cmp_int_unaligned(const MDBX_val *a, const MDBX_val *b) { + return cmp_int_inline(1, a, b); +} - mp = mc->mc_pg[mc->mc_top]; - MDBX_ANALYSIS_ASSUME(mp != nullptr); - cASSERT(mc, IS_LEAF(mp)); +#ifndef cmp_int_align2 +/* Compare two items pointing at 2-byte aligned unsigned int's. */ +MDBX_NOTHROW_PURE_FUNCTION __hot int cmp_int_align2(const MDBX_val *a, const MDBX_val *b) { + return cmp_int_inline(2, a, b); +} +#endif /* cmp_int_align2 */ -search_node:; - struct node_result nsr = node_search(mc, &aligned_key); - node = nsr.node; - ret.exact = nsr.exact; - if (!ret.exact) { - if (op != MDBX_SET_RANGE) { - /* MDBX_SET specified and not an exact match. */ - if (unlikely(mc->mc_ki[mc->mc_top] >= - page_numkeys(mc->mc_pg[mc->mc_top]))) - mc->mc_flags |= C_EOF; - ret.err = MDBX_NOTFOUND; - return ret; - } +#ifndef cmp_int_align4 +/* Compare two items pointing at 4-byte aligned unsigned int's. */ +MDBX_NOTHROW_PURE_FUNCTION __hot int cmp_int_align4(const MDBX_val *a, const MDBX_val *b) { + return cmp_int_inline(4, a, b); +} +#endif /* cmp_int_align4 */ - if (node == NULL) { - DEBUG("%s", "===> inexact leaf not found, goto sibling"); - ret.err = cursor_sibling(mc, SIBLING_RIGHT); - if (unlikely(ret.err != MDBX_SUCCESS)) { - mc->mc_flags |= C_EOF; - return ret; /* no entries matched */ - } - mp = mc->mc_pg[mc->mc_top]; - cASSERT(mc, IS_LEAF(mp)); - if (!IS_LEAF2(mp)) - node = page_node(mp, 0); - } - } - cASSERT(mc, mc->mc_ki[mc->mc_top] < page_numkeys(mc->mc_pg[mc->mc_top]) || - (mc->mc_flags & C_EOF)); +/* Compare two items lexically */ +MDBX_NOTHROW_PURE_FUNCTION __hot int cmp_lexical(const MDBX_val *a, const MDBX_val *b) { + if (a->iov_len == b->iov_len) + return a->iov_len ? memcmp(a->iov_base, b->iov_base, a->iov_len) : 0; -got_node: - mc->mc_flags |= C_INITIALIZED; - mc->mc_flags &= ~C_EOF; + const int diff_len = (a->iov_len < b->iov_len) ? -1 : 1; + const size_t shortest = (a->iov_len < b->iov_len) ? a->iov_len : b->iov_len; + int diff_data = shortest ? memcmp(a->iov_base, b->iov_base, shortest) : 0; + return likely(diff_data) ? diff_data : diff_len; +} - if (!MDBX_DISABLE_VALIDATION && unlikely(!CHECK_LEAF_TYPE(mc, mp))) { - ERROR("unexpected leaf-page #%" PRIaPGNO " type 0x%x seen by cursor", - mp->mp_pgno, mp->mp_flags); - ret.err = MDBX_CORRUPTED; - return ret; - } +MDBX_NOTHROW_PURE_FUNCTION static __always_inline unsigned tail3le(const uint8_t *p, size_t l) { + STATIC_ASSERT(sizeof(unsigned) > 2); + // 1: 0 0 0 + // 2: 0 1 1 + // 3: 0 1 2 + return p[0] | p[l >> 1] << 8 | p[l - 1] << 16; +} - if (IS_LEAF2(mp)) { - if (op == MDBX_SET_RANGE || op == MDBX_SET_KEY) { - key->iov_len = mc->mc_db->md_xsize; - key->iov_base = page_leaf2key(mp, mc->mc_ki[mc->mc_top], key->iov_len); +/* Compare two items in reverse byte order */ +MDBX_NOTHROW_PURE_FUNCTION __hot int cmp_reverse(const MDBX_val *a, const MDBX_val *b) { + size_t left = (a->iov_len < b->iov_len) ? a->iov_len : b->iov_len; + if (likely(left)) { + const uint8_t *pa = ptr_disp(a->iov_base, a->iov_len); + const uint8_t *pb = ptr_disp(b->iov_base, b->iov_len); + while (left >= sizeof(size_t)) { + pa -= sizeof(size_t); + pb -= sizeof(size_t); + left -= sizeof(size_t); + STATIC_ASSERT(sizeof(size_t) == 4 || sizeof(size_t) == 8); + if (sizeof(size_t) == 4) { + uint32_t xa = unaligned_peek_u32(1, pa); + uint32_t xb = unaligned_peek_u32(1, pb); +#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__ + xa = osal_bswap32(xa); + xb = osal_bswap32(xb); +#endif /* __BYTE_ORDER__ != __ORDER_BIG_ENDIAN__ */ + if (xa != xb) + return (xa < xb) ? -1 : 1; + } else { + uint64_t xa = unaligned_peek_u64(1, pa); + uint64_t xb = unaligned_peek_u64(1, pb); +#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__ + xa = osal_bswap64(xa); + xb = osal_bswap64(xb); +#endif /* __BYTE_ORDER__ != __ORDER_BIG_ENDIAN__ */ + if (xa != xb) + return (xa < xb) ? -1 : 1; + } } - ret.err = MDBX_SUCCESS; - return ret; - } - - if (node_flags(node) & F_DUPDATA) { - ret.err = cursor_xinit1(mc, node, mp); - if (unlikely(ret.err != MDBX_SUCCESS)) - return ret; - if (op == MDBX_SET || op == MDBX_SET_KEY || op == MDBX_SET_RANGE) { - MDBX_ANALYSIS_ASSUME(mc->mc_xcursor != nullptr); - ret.err = cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL); - if (unlikely(ret.err != MDBX_SUCCESS)) - return ret; - } else { - MDBX_ANALYSIS_ASSUME(mc->mc_xcursor != nullptr); - ret = cursor_set(&mc->mc_xcursor->mx_cursor, data, NULL, MDBX_SET_RANGE); - if (unlikely(ret.err != MDBX_SUCCESS)) - return ret; - if (op == MDBX_GET_BOTH && !ret.exact) { - ret.err = MDBX_NOTFOUND; - return ret; - } + if (sizeof(size_t) == 8 && left >= 4) { + pa -= 4; + pb -= 4; + left -= 4; + uint32_t xa = unaligned_peek_u32(1, pa); + uint32_t xb = unaligned_peek_u32(1, pb); +#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__ + xa = osal_bswap32(xa); + xb = osal_bswap32(xb); +#endif /* __BYTE_ORDER__ != __ORDER_BIG_ENDIAN__ */ + if (xa != xb) + return (xa < xb) ? -1 : 1; } - } else if (likely(data)) { - if (op == MDBX_GET_BOTH || op == MDBX_GET_BOTH_RANGE) { - if (unlikely(data->iov_len < mc->mc_dbx->md_vlen_min || - data->iov_len > mc->mc_dbx->md_vlen_max)) { - cASSERT(mc, !"Invalid data-size"); - ret.err = MDBX_BAD_VALSIZE; - return ret; - } - MDBX_val aligned_data = *data; - uint64_t aligned_databytes; - if (mc->mc_db->md_flags & MDBX_INTEGERDUP) { - switch (aligned_data.iov_len) { - default: - cASSERT(mc, !"data-size is invalid for MDBX_INTEGERDUP"); - ret.err = MDBX_BAD_VALSIZE; - return ret; - case 4: - if (unlikely(3 & (uintptr_t)aligned_data.iov_base)) - /* copy instead of return error to avoid break compatibility */ - aligned_data.iov_base = - memcpy(&aligned_databytes, aligned_data.iov_base, 4); - break; - case 8: - if (unlikely(7 & (uintptr_t)aligned_data.iov_base)) - /* copy instead of return error to avoid break compatibility */ - aligned_data.iov_base = - memcpy(&aligned_databytes, aligned_data.iov_base, 8); - break; - } - } - MDBX_val actual_data; - ret.err = node_read(mc, node, &actual_data, mc->mc_pg[mc->mc_top]); - if (unlikely(ret.err != MDBX_SUCCESS)) - return ret; - const int cmp = mc->mc_dbx->md_dcmp(&aligned_data, &actual_data); - if (cmp) { - cASSERT(mc, - mc->mc_ki[mc->mc_top] < page_numkeys(mc->mc_pg[mc->mc_top]) || - (mc->mc_flags & C_EOF)); - if (op != MDBX_GET_BOTH_RANGE || cmp > 0) { - ret.err = MDBX_NOTFOUND; - return ret; - } - } - *data = actual_data; - } else { - ret.err = node_read(mc, node, data, mc->mc_pg[mc->mc_top]); - if (unlikely(ret.err != MDBX_SUCCESS)) - return ret; + if (left) { + unsigned xa = tail3le(pa - left, left); + unsigned xb = tail3le(pb - left, left); + if (xa != xb) + return (xa < xb) ? -1 : 1; } } + return CMP2INT(a->iov_len, b->iov_len); +} - /* The key already matches in all other cases */ - if (op == MDBX_SET_RANGE || op == MDBX_SET_KEY) - get_key_optional(node, key); +/* Fast non-lexically comparator */ +MDBX_NOTHROW_PURE_FUNCTION __hot int cmp_lenfast(const MDBX_val *a, const MDBX_val *b) { + int diff = CMP2INT(a->iov_len, b->iov_len); + return (likely(diff) || a->iov_len == 0) ? diff : memcmp(a->iov_base, b->iov_base, a->iov_len); +} - DEBUG("==> cursor placed on key [%s], data [%s]", DKEY_DEBUG(key), - DVAL_DEBUG(data)); - ret.err = MDBX_SUCCESS; - return ret; +MDBX_NOTHROW_PURE_FUNCTION __hot bool eq_fast_slowpath(const uint8_t *a, const uint8_t *b, size_t l) { + if (likely(l > 3)) { + if (MDBX_UNALIGNED_OK >= 4 && likely(l < 9)) + return ((unaligned_peek_u32(1, a) - unaligned_peek_u32(1, b)) | + (unaligned_peek_u32(1, a + l - 4) - unaligned_peek_u32(1, b + l - 4))) == 0; + if (MDBX_UNALIGNED_OK >= 8 && sizeof(size_t) > 7 && likely(l < 17)) + return ((unaligned_peek_u64(1, a) - unaligned_peek_u64(1, b)) | + (unaligned_peek_u64(1, a + l - 8) - unaligned_peek_u64(1, b + l - 8))) == 0; + return memcmp(a, b, l) == 0; + } + if (likely(l)) + return tail3le(a, l) == tail3le(b, l); + return true; } -/* Move the cursor to the first item in the database. */ -static int cursor_first(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data) { - int rc; +int cmp_equal_or_greater(const MDBX_val *a, const MDBX_val *b) { return eq_fast(a, b) ? 0 : 1; } - if (mc->mc_xcursor) - mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED | C_EOF); +int cmp_equal_or_wrong(const MDBX_val *a, const MDBX_val *b) { return eq_fast(a, b) ? 0 : -1; } - if (!(mc->mc_flags & C_INITIALIZED) || mc->mc_top) { - rc = page_search(mc, NULL, MDBX_PS_FIRST); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - } +/*----------------------------------------------------------------------------*/ - const MDBX_page *mp = mc->mc_pg[mc->mc_top]; - if (!MDBX_DISABLE_VALIDATION && unlikely(!CHECK_LEAF_TYPE(mc, mp))) { - ERROR("unexpected leaf-page #%" PRIaPGNO " type 0x%x seen by cursor", - mp->mp_pgno, mp->mp_flags); - return MDBX_CORRUPTED; +__cold void update_mlcnt(const MDBX_env *env, const pgno_t new_aligned_mlocked_pgno, const bool lock_not_release) { + for (;;) { + const pgno_t mlock_pgno_before = atomic_load32(&env->mlocked_pgno, mo_AcquireRelease); + eASSERT(env, pgno_align2os_pgno(env, mlock_pgno_before) == mlock_pgno_before); + eASSERT(env, pgno_align2os_pgno(env, new_aligned_mlocked_pgno) == new_aligned_mlocked_pgno); + if (lock_not_release ? (mlock_pgno_before >= new_aligned_mlocked_pgno) + : (mlock_pgno_before <= new_aligned_mlocked_pgno)) + break; + if (likely(atomic_cas32(&((MDBX_env *)env)->mlocked_pgno, mlock_pgno_before, new_aligned_mlocked_pgno))) + for (;;) { + mdbx_atomic_uint32_t *const mlcnt = env->lck->mlcnt; + const int32_t snap_locked = atomic_load32(mlcnt + 0, mo_Relaxed); + const int32_t snap_unlocked = atomic_load32(mlcnt + 1, mo_Relaxed); + if (mlock_pgno_before == 0 && (snap_locked - snap_unlocked) < INT_MAX) { + eASSERT(env, lock_not_release); + if (unlikely(!atomic_cas32(mlcnt + 0, snap_locked, snap_locked + 1))) + continue; + } + if (new_aligned_mlocked_pgno == 0 && (snap_locked - snap_unlocked) > 0) { + eASSERT(env, !lock_not_release); + if (unlikely(!atomic_cas32(mlcnt + 1, snap_unlocked, snap_unlocked + 1))) + continue; + } + NOTICE("%s-pages %u..%u, mlocked-process(es) %u -> %u", lock_not_release ? "lock" : "unlock", + lock_not_release ? mlock_pgno_before : new_aligned_mlocked_pgno, + lock_not_release ? new_aligned_mlocked_pgno : mlock_pgno_before, snap_locked - snap_unlocked, + atomic_load32(mlcnt + 0, mo_Relaxed) - atomic_load32(mlcnt + 1, mo_Relaxed)); + return; + } } +} - mc->mc_flags |= C_INITIALIZED; - mc->mc_flags &= ~C_EOF; - mc->mc_ki[mc->mc_top] = 0; - - if (IS_LEAF2(mp)) { - if (likely(key)) { - key->iov_len = mc->mc_db->md_xsize; - key->iov_base = page_leaf2key(mp, 0, key->iov_len); +__cold void munlock_after(const MDBX_env *env, const pgno_t aligned_pgno, const size_t end_bytes) { + if (atomic_load32(&env->mlocked_pgno, mo_AcquireRelease) > aligned_pgno) { + int err = MDBX_ENOSYS; + const size_t munlock_begin = pgno2bytes(env, aligned_pgno); + const size_t munlock_size = end_bytes - munlock_begin; + eASSERT(env, end_bytes % globals.sys_pagesize == 0 && munlock_begin % globals.sys_pagesize == 0 && + munlock_size % globals.sys_pagesize == 0); +#if defined(_WIN32) || defined(_WIN64) + err = VirtualUnlock(ptr_disp(env->dxb_mmap.base, munlock_begin), munlock_size) ? MDBX_SUCCESS : (int)GetLastError(); + if (err == ERROR_NOT_LOCKED) + err = MDBX_SUCCESS; +#elif defined(_POSIX_MEMLOCK_RANGE) + err = munlock(ptr_disp(env->dxb_mmap.base, munlock_begin), munlock_size) ? errno : MDBX_SUCCESS; +#endif + if (likely(err == MDBX_SUCCESS)) + update_mlcnt(env, aligned_pgno, false); + else { +#if defined(_WIN32) || defined(_WIN64) + WARNING("VirtualUnlock(%zu, %zu) error %d", munlock_begin, munlock_size, err); +#else + WARNING("munlock(%zu, %zu) error %d", munlock_begin, munlock_size, err); +#endif } - return MDBX_SUCCESS; } +} - MDBX_node *node = page_node(mp, 0); - if (node_flags(node) & F_DUPDATA) { - rc = cursor_xinit1(mc, node, mp); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - MDBX_ANALYSIS_ASSUME(mc->mc_xcursor != nullptr); - rc = cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL); - if (unlikely(rc)) - return rc; - } else if (likely(data)) { - rc = node_read(mc, node, data, mp); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - } +__cold void munlock_all(const MDBX_env *env) { + munlock_after(env, 0, bytes_align2os_bytes(env, env->dxb_mmap.current)); +} - get_key_optional(node, key); - return MDBX_SUCCESS; +/*----------------------------------------------------------------------------*/ + +uint32_t combine_durability_flags(const uint32_t a, const uint32_t b) { + uint32_t r = a | b; + + /* avoid false MDBX_UTTERLY_NOSYNC */ + if (F_ISSET(r, MDBX_UTTERLY_NOSYNC) && !F_ISSET(a, MDBX_UTTERLY_NOSYNC) && !F_ISSET(b, MDBX_UTTERLY_NOSYNC)) + r = (r - MDBX_UTTERLY_NOSYNC) | MDBX_SAFE_NOSYNC; + + /* convert DEPRECATED_MAPASYNC to MDBX_SAFE_NOSYNC */ + if ((r & (MDBX_WRITEMAP | DEPRECATED_MAPASYNC)) == (MDBX_WRITEMAP | DEPRECATED_MAPASYNC) && + !F_ISSET(r, MDBX_UTTERLY_NOSYNC)) + r = (r - DEPRECATED_MAPASYNC) | MDBX_SAFE_NOSYNC; + + /* force MDBX_NOMETASYNC if NOSYNC enabled */ + if (r & (MDBX_SAFE_NOSYNC | MDBX_UTTERLY_NOSYNC)) + r |= MDBX_NOMETASYNC; + + assert(!(F_ISSET(r, MDBX_UTTERLY_NOSYNC) && !F_ISSET(a, MDBX_UTTERLY_NOSYNC) && !F_ISSET(b, MDBX_UTTERLY_NOSYNC))); + return r; } +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 -/* Move the cursor to the last item in the database. */ -static int cursor_last(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data) { - int rc; +/* check against https://libmdbx.dqdkfa.ru/dead-github/issues/269 */ +static bool coherency_check(const MDBX_env *env, const txnid_t txnid, const volatile tree_t *trees, + const volatile meta_t *meta, bool report) { + const txnid_t freedb_mod_txnid = trees[FREE_DBI].mod_txnid; + const txnid_t maindb_mod_txnid = trees[MAIN_DBI].mod_txnid; + const pgno_t last_pgno = meta->geometry.now; - if (mc->mc_xcursor) - mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED | C_EOF); + const pgno_t freedb_root_pgno = trees[FREE_DBI].root; + const page_t *freedb_root = + (env->dxb_mmap.base && freedb_root_pgno < last_pgno) ? pgno2page(env, freedb_root_pgno) : nullptr; - if (!(mc->mc_flags & C_INITIALIZED) || mc->mc_top) { - rc = page_search(mc, NULL, MDBX_PS_LAST); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; + const pgno_t maindb_root_pgno = trees[MAIN_DBI].root; + const page_t *maindb_root = + (env->dxb_mmap.base && maindb_root_pgno < last_pgno) ? pgno2page(env, maindb_root_pgno) : nullptr; + const uint64_t magic_and_version = unaligned_peek_u64_volatile(4, &meta->magic_and_version); + + bool ok = true; + if (freedb_root_pgno != P_INVALID && unlikely(freedb_root_pgno >= last_pgno)) { + if (report) + WARNING("catch invalid %s-db root %" PRIaPGNO " for meta_txnid %" PRIaTXN " %s", "free", freedb_root_pgno, txnid, + (env->stuck_meta < 0) ? "(workaround for incoherent flaw of unified page/buffer cache)" + : "(wagering meta)"); + ok = false; + } + if (maindb_root_pgno != P_INVALID && unlikely(maindb_root_pgno >= last_pgno)) { + if (report) + WARNING("catch invalid %s-db root %" PRIaPGNO " for meta_txnid %" PRIaTXN " %s", "main", maindb_root_pgno, txnid, + (env->stuck_meta < 0) ? "(workaround for incoherent flaw of unified page/buffer cache)" + : "(wagering meta)"); + ok = false; + } + if (unlikely(txnid < freedb_mod_txnid || + (!freedb_mod_txnid && freedb_root && likely(magic_and_version == MDBX_DATA_MAGIC)))) { + if (report) + WARNING( + "catch invalid %s-db.mod_txnid %" PRIaTXN " for meta_txnid %" PRIaTXN " %s", "free", freedb_mod_txnid, txnid, + (env->stuck_meta < 0) ? "(workaround for incoherent flaw of unified page/buffer cache)" : "(wagering meta)"); + ok = false; + } + if (unlikely(txnid < maindb_mod_txnid || + (!maindb_mod_txnid && maindb_root && likely(magic_and_version == MDBX_DATA_MAGIC)))) { + if (report) + WARNING( + "catch invalid %s-db.mod_txnid %" PRIaTXN " for meta_txnid %" PRIaTXN " %s", "main", maindb_mod_txnid, txnid, + (env->stuck_meta < 0) ? "(workaround for incoherent flaw of unified page/buffer cache)" : "(wagering meta)"); + ok = false; } - const MDBX_page *mp = mc->mc_pg[mc->mc_top]; - if (!MDBX_DISABLE_VALIDATION && unlikely(!CHECK_LEAF_TYPE(mc, mp))) { - ERROR("unexpected leaf-page #%" PRIaPGNO " type 0x%x seen by cursor", - mp->mp_pgno, mp->mp_flags); - return MDBX_CORRUPTED; + /* Проверяем отметки внутри корневых страниц только если сами страницы + * в пределах текущего отображения. Иначе возможны SIGSEGV до переноса + * вызова coherency_check_head() после dxb_resize() внутри txn_renew(). */ + if (likely(freedb_root && freedb_mod_txnid && + (size_t)ptr_dist(env->dxb_mmap.base, freedb_root) < env->dxb_mmap.limit)) { + VALGRIND_MAKE_MEM_DEFINED(freedb_root, sizeof(freedb_root->txnid)); + MDBX_ASAN_UNPOISON_MEMORY_REGION(freedb_root, sizeof(freedb_root->txnid)); + const txnid_t root_txnid = freedb_root->txnid; + if (unlikely(root_txnid != freedb_mod_txnid)) { + if (report) + WARNING("catch invalid root_page %" PRIaPGNO " mod_txnid %" PRIaTXN " for %s-db.mod_txnid %" PRIaTXN " %s", + freedb_root_pgno, root_txnid, "free", freedb_mod_txnid, + (env->stuck_meta < 0) ? "(workaround for incoherent flaw of " + "unified page/buffer cache)" + : "(wagering meta)"); + ok = false; + } + } + if (likely(maindb_root && maindb_mod_txnid && + (size_t)ptr_dist(env->dxb_mmap.base, maindb_root) < env->dxb_mmap.limit)) { + VALGRIND_MAKE_MEM_DEFINED(maindb_root, sizeof(maindb_root->txnid)); + MDBX_ASAN_UNPOISON_MEMORY_REGION(maindb_root, sizeof(maindb_root->txnid)); + const txnid_t root_txnid = maindb_root->txnid; + if (unlikely(root_txnid != maindb_mod_txnid)) { + if (report) + WARNING("catch invalid root_page %" PRIaPGNO " mod_txnid %" PRIaTXN " for %s-db.mod_txnid %" PRIaTXN " %s", + maindb_root_pgno, root_txnid, "main", maindb_mod_txnid, + (env->stuck_meta < 0) ? "(workaround for incoherent flaw of " + "unified page/buffer cache)" + : "(wagering meta)"); + ok = false; + } + } + if (unlikely(!ok) && report) + env->lck->pgops.incoherence.weak = + (env->lck->pgops.incoherence.weak >= INT32_MAX) ? INT32_MAX : env->lck->pgops.incoherence.weak + 1; + return ok; +} + +__cold int coherency_timeout(uint64_t *timestamp, intptr_t pgno, const MDBX_env *env) { + if (likely(timestamp && *timestamp == 0)) + *timestamp = osal_monotime(); + else if (unlikely(!timestamp || osal_monotime() - *timestamp > osal_16dot16_to_monotime(65536 / 10))) { + if (pgno >= 0 && pgno != env->stuck_meta) + ERROR("bailout waiting for %" PRIuSIZE " page arrival %s", pgno, + "(workaround for incoherent flaw of unified page/buffer cache)"); + else if (env->stuck_meta < 0) + ERROR("bailout waiting for valid snapshot (%s)", "workaround for incoherent flaw of unified page/buffer cache"); + return MDBX_PROBLEM; } - mc->mc_ki[mc->mc_top] = (indx_t)page_numkeys(mp) - 1; - mc->mc_flags |= C_INITIALIZED | C_EOF; + osal_memory_fence(mo_AcquireRelease, true); +#if defined(_WIN32) || defined(_WIN64) + SwitchToThread(); +#elif defined(__linux__) || defined(__gnu_linux__) || defined(_UNIX03_SOURCE) + sched_yield(); +#elif (defined(_GNU_SOURCE) && __GLIBC_PREREQ(2, 1)) || defined(_OPEN_THREADS) + pthread_yield(); +#else + usleep(42); +#endif + return MDBX_RESULT_TRUE; +} - if (IS_LEAF2(mp)) { - if (likely(key)) { - key->iov_len = mc->mc_db->md_xsize; - key->iov_base = page_leaf2key(mp, mc->mc_ki[mc->mc_top], key->iov_len); +/* check with timeout as the workaround + * for https://libmdbx.dqdkfa.ru/dead-github/issues/269 */ +__hot int coherency_fetch_head(MDBX_txn *txn, const meta_ptr_t head, uint64_t *timestamp) { + /* Copy the DB info and flags */ + txn->txnid = head.txnid; + txn->geo = head.ptr_c->geometry; + memcpy(txn->dbs, &head.ptr_c->trees, sizeof(head.ptr_c->trees)); + STATIC_ASSERT(sizeof(head.ptr_c->trees) == CORE_DBS * sizeof(tree_t)); + VALGRIND_MAKE_MEM_UNDEFINED(txn->dbs + CORE_DBS, txn->env->max_dbi - CORE_DBS); + txn->canary = head.ptr_c->canary; + + if (unlikely(!coherency_check(txn->env, head.txnid, txn->dbs, head.ptr_v, *timestamp == 0) || + txn->txnid != meta_txnid(head.ptr_v))) + return coherency_timeout(timestamp, -1, txn->env); + + if (unlikely(txn->dbs[FREE_DBI].flags != MDBX_INTEGERKEY)) { + if ((txn->dbs[FREE_DBI].flags & DB_PERSISTENT_FLAGS) != MDBX_INTEGERKEY || + unaligned_peek_u64(4, &head.ptr_c->magic_and_version) == MDBX_DATA_MAGIC) { + ERROR("unexpected/invalid db-flags 0x%x for %s", txn->dbs[FREE_DBI].flags, "GC/FreeDB"); + return MDBX_INCOMPATIBLE; } - return MDBX_SUCCESS; + txn->dbs[FREE_DBI].flags &= DB_PERSISTENT_FLAGS; } + tASSERT(txn, txn->dbs[FREE_DBI].flags == MDBX_INTEGERKEY); + tASSERT(txn, check_table_flags(txn->dbs[MAIN_DBI].flags)); + return MDBX_SUCCESS; +} - MDBX_node *node = page_node(mp, mc->mc_ki[mc->mc_top]); - if (node_flags(node) & F_DUPDATA) { - rc = cursor_xinit1(mc, node, mp); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - MDBX_ANALYSIS_ASSUME(mc->mc_xcursor != nullptr); - rc = cursor_last(&mc->mc_xcursor->mx_cursor, data, NULL); - if (unlikely(rc)) - return rc; - } else if (likely(data)) { - rc = node_read(mc, node, data, mp); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; +int coherency_check_written(const MDBX_env *env, const txnid_t txnid, const volatile meta_t *meta, const intptr_t pgno, + uint64_t *timestamp) { + const bool report = !(timestamp && *timestamp); + const txnid_t head_txnid = meta_txnid(meta); + if (likely(head_txnid >= MIN_TXNID && head_txnid >= txnid)) { + if (likely(coherency_check(env, head_txnid, &meta->trees.gc, meta, report))) { + eASSERT(env, meta->trees.gc.flags == MDBX_INTEGERKEY); + eASSERT(env, check_table_flags(meta->trees.main.flags)); + return MDBX_SUCCESS; + } + } else if (report) { + env->lck->pgops.incoherence.weak = + (env->lck->pgops.incoherence.weak >= INT32_MAX) ? INT32_MAX : env->lck->pgops.incoherence.weak + 1; + WARNING("catch %s txnid %" PRIaTXN " for meta_%" PRIaPGNO " %s", + (head_txnid < MIN_TXNID) ? "invalid" : "unexpected", head_txnid, + bytes2pgno(env, ptr_dist(meta, env->dxb_mmap.base)), + "(workaround for incoherent flaw of unified page/buffer cache)"); } + return coherency_timeout(timestamp, pgno, env); +} - get_key_optional(node, key); - return MDBX_SUCCESS; +bool coherency_check_meta(const MDBX_env *env, const volatile meta_t *meta, bool report) { + uint64_t timestamp = 0; + return coherency_check_written(env, 0, meta, -1, report ? ×tamp : nullptr) == MDBX_SUCCESS; } +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \note Please refer to the COPYRIGHT file for explanations license change, +/// credits and acknowledgments. +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 -static __hot int cursor_get(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, - MDBX_cursor_op op) { - int (*mfunc)(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data); - int rc; +__cold int cursor_validate(const MDBX_cursor *mc) { + if (!mc->txn->tw.dirtylist) { + cASSERT(mc, (mc->txn->flags & MDBX_WRITEMAP) != 0 && !MDBX_AVOID_MSYNC); + } else { + cASSERT(mc, (mc->txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); + cASSERT(mc, mc->txn->tw.dirtyroom + mc->txn->tw.dirtylist->length == + (mc->txn->parent ? mc->txn->parent->tw.dirtyroom : mc->txn->env->options.dp_limit)); + } - switch (op) { - case MDBX_GET_CURRENT: { - if (unlikely(!(mc->mc_flags & C_INITIALIZED))) - return MDBX_ENODATA; - const MDBX_page *mp = mc->mc_pg[mc->mc_top]; - if (!MDBX_DISABLE_VALIDATION && unlikely(!CHECK_LEAF_TYPE(mc, mp))) { - ERROR("unexpected leaf-page #%" PRIaPGNO " type 0x%x seen by cursor", - mp->mp_pgno, mp->mp_flags); - return MDBX_CORRUPTED; - } + cASSERT(mc, (mc->checking & z_updating) ? mc->top + 1 <= mc->tree->height : mc->top + 1 == mc->tree->height); + if (unlikely((mc->checking & z_updating) ? mc->top + 1 > mc->tree->height : mc->top + 1 != mc->tree->height)) + return MDBX_CURSOR_FULL; + + if (is_pointed(mc) && (mc->checking & z_updating) == 0) { + const page_t *mp = mc->pg[mc->top]; const size_t nkeys = page_numkeys(mp); - if (unlikely(mc->mc_ki[mc->mc_top] >= nkeys)) { - cASSERT(mc, nkeys <= UINT16_MAX); - if (mc->mc_flags & C_EOF) - return MDBX_ENODATA; - mc->mc_ki[mc->mc_top] = (uint16_t)nkeys; - mc->mc_flags |= C_EOF; - return MDBX_NOTFOUND; + if (!is_hollow(mc)) { + cASSERT(mc, mc->ki[mc->top] < nkeys); + if (mc->ki[mc->top] >= nkeys) + return MDBX_CURSOR_FULL; } - cASSERT(mc, nkeys > 0); - - rc = MDBX_SUCCESS; - if (IS_LEAF2(mp)) { - key->iov_len = mc->mc_db->md_xsize; - key->iov_base = page_leaf2key(mp, mc->mc_ki[mc->mc_top], key->iov_len); - } else { - MDBX_node *node = page_node(mp, mc->mc_ki[mc->mc_top]); - get_key_optional(node, key); - if (data) { - if (node_flags(node) & F_DUPDATA) { - if (unlikely(!(mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED))) { - rc = cursor_xinit1(mc, node, mp); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - rc = cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL); - if (unlikely(rc)) - return rc; - } else { - rc = cursor_get(&mc->mc_xcursor->mx_cursor, data, NULL, - MDBX_GET_CURRENT); - if (unlikely(rc)) - return rc; - } - } else { - rc = node_read(mc, node, data, mp); - if (unlikely(rc)) - return rc; - } - } + if (inner_pointed(mc)) { + cASSERT(mc, is_filled(mc)); + if (!is_filled(mc)) + return MDBX_CURSOR_FULL; } - break; } - case MDBX_GET_BOTH: - case MDBX_GET_BOTH_RANGE: - if (unlikely(data == NULL)) - return MDBX_EINVAL; - if (unlikely(mc->mc_xcursor == NULL)) - return MDBX_INCOMPATIBLE; - /* fall through */ - __fallthrough; - case MDBX_SET: - case MDBX_SET_KEY: - case MDBX_SET_RANGE: - if (unlikely(key == NULL)) - return MDBX_EINVAL; - rc = cursor_set(mc, key, data, op).err; - if (mc->mc_flags & C_INITIALIZED) { - cASSERT(mc, mc->mc_snum > 0 && mc->mc_top < mc->mc_snum); - cASSERT(mc, mc->mc_ki[mc->mc_top] < page_numkeys(mc->mc_pg[mc->mc_top]) || - (mc->mc_flags & C_EOF)); - } - break; - case MDBX_GET_MULTIPLE: - if (unlikely(!data)) - return MDBX_EINVAL; - if (unlikely((mc->mc_db->md_flags & MDBX_DUPFIXED) == 0)) - return MDBX_INCOMPATIBLE; - if ((mc->mc_flags & C_INITIALIZED) == 0) { - if (unlikely(!key)) - return MDBX_EINVAL; - rc = cursor_set(mc, key, data, MDBX_SET).err; - if (unlikely(rc != MDBX_SUCCESS)) - break; - } - rc = MDBX_SUCCESS; - if (unlikely(C_INITIALIZED != (mc->mc_xcursor->mx_cursor.mc_flags & - (C_INITIALIZED | C_EOF)))) - break; - goto fetch_multiple; - case MDBX_NEXT_MULTIPLE: - if (unlikely(!data)) - return MDBX_EINVAL; - if (unlikely(!(mc->mc_db->md_flags & MDBX_DUPFIXED))) - return MDBX_INCOMPATIBLE; - rc = cursor_next(mc, key, data, MDBX_NEXT_DUP); - if (rc == MDBX_SUCCESS) { - if (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) { - fetch_multiple:; - MDBX_cursor *mx = &mc->mc_xcursor->mx_cursor; - data->iov_len = - page_numkeys(mx->mc_pg[mx->mc_top]) * mx->mc_db->md_xsize; - data->iov_base = page_data(mx->mc_pg[mx->mc_top]); - mx->mc_ki[mx->mc_top] = (indx_t)page_numkeys(mx->mc_pg[mx->mc_top]) - 1; - } - } - break; - case MDBX_PREV_MULTIPLE: - if (unlikely(!data)) - return MDBX_EINVAL; - if (!(mc->mc_db->md_flags & MDBX_DUPFIXED)) - return MDBX_INCOMPATIBLE; - rc = MDBX_SUCCESS; - if ((mc->mc_flags & C_INITIALIZED) == 0) - rc = cursor_last(mc, key, data); - if (rc == MDBX_SUCCESS) { - MDBX_cursor *mx = &mc->mc_xcursor->mx_cursor; - rc = MDBX_NOTFOUND; - if (mx->mc_flags & C_INITIALIZED) { - rc = cursor_sibling(mx, SIBLING_LEFT); - if (rc == MDBX_SUCCESS) - goto fetch_multiple; - } - } - break; - case MDBX_NEXT: - case MDBX_NEXT_DUP: - case MDBX_NEXT_NODUP: - rc = cursor_next(mc, key, data, op); - break; - case MDBX_PREV: - case MDBX_PREV_DUP: - case MDBX_PREV_NODUP: - rc = cursor_prev(mc, key, data, op); - break; - case MDBX_FIRST: - rc = cursor_first(mc, key, data); - break; - case MDBX_FIRST_DUP: - mfunc = cursor_first; - move: - if (unlikely(data == NULL || !(mc->mc_flags & C_INITIALIZED))) - return MDBX_EINVAL; - if (unlikely(mc->mc_xcursor == NULL)) - return MDBX_INCOMPATIBLE; - if (mc->mc_ki[mc->mc_top] >= page_numkeys(mc->mc_pg[mc->mc_top])) { - mc->mc_ki[mc->mc_top] = (indx_t)page_numkeys(mc->mc_pg[mc->mc_top]); - mc->mc_flags |= C_EOF; - return MDBX_NOTFOUND; + + for (intptr_t n = 0; n <= mc->top; ++n) { + page_t *mp = mc->pg[n]; + const size_t nkeys = page_numkeys(mp); + const bool expect_branch = (n < mc->tree->height - 1) ? true : false; + const bool expect_nested_leaf = (n + 1 == mc->tree->height - 1) ? true : false; + const bool branch = is_branch(mp) ? true : false; + cASSERT(mc, branch == expect_branch); + if (unlikely(branch != expect_branch)) + return MDBX_CURSOR_FULL; + if ((mc->checking & z_updating) == 0) { + cASSERT(mc, nkeys > mc->ki[n] || (!branch && nkeys == mc->ki[n] && (mc->flags & z_hollow) != 0)); + if (unlikely(nkeys <= mc->ki[n] && !(!branch && nkeys == mc->ki[n] && (mc->flags & z_hollow) != 0))) + return MDBX_CURSOR_FULL; } else { - MDBX_node *node = page_node(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); - if (!(node_flags(node) & F_DUPDATA)) { - get_key_optional(node, key); - rc = node_read(mc, node, data, mc->mc_pg[mc->mc_top]); - break; - } - } - if (unlikely(!(mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED))) - return MDBX_EINVAL; - rc = mfunc(&mc->mc_xcursor->mx_cursor, data, NULL); - break; - case MDBX_LAST: - rc = cursor_last(mc, key, data); - break; - case MDBX_LAST_DUP: - mfunc = cursor_last; - goto move; - case MDBX_SET_UPPERBOUND: /* mostly same as MDBX_SET_LOWERBOUND */ - case MDBX_SET_LOWERBOUND: { - if (unlikely(key == NULL || data == NULL)) - return MDBX_EINVAL; - MDBX_val save_data = *data; - struct cursor_set_result csr = cursor_set(mc, key, data, MDBX_SET_RANGE); - rc = csr.err; - if (rc == MDBX_SUCCESS && csr.exact && mc->mc_xcursor) { - mc->mc_flags &= ~C_DEL; - csr.exact = false; - if (!save_data.iov_base && (mc->mc_db->md_flags & MDBX_DUPFIXED)) { - /* Avoiding search nested dupfixed hive if no data provided. - * This is changes the semantic of MDBX_SET_LOWERBOUND but avoid - * returning MDBX_BAD_VALSIZE. */ - } else if (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) { - *data = save_data; - csr = - cursor_set(&mc->mc_xcursor->mx_cursor, data, NULL, MDBX_SET_RANGE); - rc = csr.err; - if (rc == MDBX_NOTFOUND) { - cASSERT(mc, !csr.exact); - rc = cursor_next(mc, key, data, MDBX_NEXT_NODUP); - } - } else { - int cmp = mc->mc_dbx->md_dcmp(&save_data, data); - csr.exact = (cmp == 0); - if (cmp > 0) - rc = cursor_next(mc, key, data, MDBX_NEXT_NODUP); - } - } - if (rc == MDBX_SUCCESS && !csr.exact) - rc = MDBX_RESULT_TRUE; - if (unlikely(op == MDBX_SET_UPPERBOUND)) { - /* minor fixups for MDBX_SET_UPPERBOUND */ - if (rc == MDBX_RESULT_TRUE) - /* already at great-than by MDBX_SET_LOWERBOUND */ - rc = MDBX_SUCCESS; - else if (rc == MDBX_SUCCESS) - /* exactly match, going next */ - rc = cursor_next(mc, key, data, MDBX_NEXT); + cASSERT(mc, nkeys + 1 >= mc->ki[n]); + if (unlikely(nkeys + 1 < mc->ki[n])) + return MDBX_CURSOR_FULL; } - break; - } - default: - DEBUG("unhandled/unimplemented cursor operation %u", op); - return MDBX_EINVAL; - } - - mc->mc_flags &= ~C_DEL; - return rc; -} - -int mdbx_cursor_get(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, - MDBX_cursor_op op) { - if (unlikely(mc == NULL)) - return MDBX_EINVAL; - - if (unlikely(mc->mc_signature != MDBX_MC_LIVE)) - return (mc->mc_signature == MDBX_MC_READY4CLOSE) ? MDBX_EINVAL - : MDBX_EBADSIGN; - - int rc = check_txn(mc->mc_txn, MDBX_TXN_BLOCKED); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - - return cursor_get(mc, key, data, op); -} -static int cursor_first_batch(MDBX_cursor *mc) { - if (!(mc->mc_flags & C_INITIALIZED) || mc->mc_top) { - int err = page_search(mc, NULL, MDBX_PS_FIRST); + int err = page_check(mc, mp); if (unlikely(err != MDBX_SUCCESS)) return err; - } - cASSERT(mc, IS_LEAF(mc->mc_pg[mc->mc_top])); - - mc->mc_flags |= C_INITIALIZED; - mc->mc_flags &= ~C_EOF; - mc->mc_ki[mc->mc_top] = 0; - return MDBX_SUCCESS; -} - -static int cursor_next_batch(MDBX_cursor *mc) { - if (unlikely(!(mc->mc_flags & C_INITIALIZED))) - return cursor_first_batch(mc); - - MDBX_page *mp = mc->mc_pg[mc->mc_top]; - if (unlikely(mc->mc_flags & C_EOF)) { - if ((size_t)mc->mc_ki[mc->mc_top] + 1 >= page_numkeys(mp)) - return MDBX_NOTFOUND; - mc->mc_flags ^= C_EOF; - } - intptr_t ki = mc->mc_ki[mc->mc_top]; - mc->mc_ki[mc->mc_top] = (indx_t)++ki; - const intptr_t numkeys = page_numkeys(mp); - if (likely(ki >= numkeys)) { - DEBUG("%s", "=====> move to next sibling page"); - mc->mc_ki[mc->mc_top] = (indx_t)(numkeys - 1); - int err = cursor_sibling(mc, SIBLING_RIGHT); - if (unlikely(err != MDBX_SUCCESS)) { - mc->mc_flags |= C_EOF; - return err; - } - mp = mc->mc_pg[mc->mc_top]; - DEBUG("next page is %" PRIaPGNO ", key index %u", mp->mp_pgno, - mc->mc_ki[mc->mc_top]); - if (!MDBX_DISABLE_VALIDATION && unlikely(!CHECK_LEAF_TYPE(mc, mp))) { - ERROR("unexpected leaf-page #%" PRIaPGNO " type 0x%x seen by cursor", - mp->mp_pgno, mp->mp_flags); - return MDBX_CORRUPTED; + for (size_t i = 0; i < nkeys; ++i) { + if (branch) { + node_t *node = page_node(mp, i); + cASSERT(mc, node_flags(node) == 0); + if (unlikely(node_flags(node) != 0)) + return MDBX_CURSOR_FULL; + pgno_t pgno = node_pgno(node); + page_t *np; + err = page_get(mc, pgno, &np, mp->txnid); + cASSERT(mc, err == MDBX_SUCCESS); + if (unlikely(err != MDBX_SUCCESS)) + return err; + const bool nested_leaf = is_leaf(np) ? true : false; + cASSERT(mc, nested_leaf == expect_nested_leaf); + if (unlikely(nested_leaf != expect_nested_leaf)) + return MDBX_CURSOR_FULL; + err = page_check(mc, np); + if (unlikely(err != MDBX_SUCCESS)) + return err; + } } } return MDBX_SUCCESS; } -int mdbx_cursor_get_batch(MDBX_cursor *mc, size_t *count, MDBX_val *pairs, - size_t limit, MDBX_cursor_op op) { - if (unlikely(mc == NULL || count == NULL || limit < 4)) - return MDBX_EINVAL; - - if (unlikely(mc->mc_signature != MDBX_MC_LIVE)) - return (mc->mc_signature == MDBX_MC_READY4CLOSE) ? MDBX_EINVAL - : MDBX_EBADSIGN; - - int rc = check_txn(mc->mc_txn, MDBX_TXN_BLOCKED); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - - if (unlikely(mc->mc_db->md_flags & MDBX_DUPSORT)) - return MDBX_INCOMPATIBLE /* must be a non-dupsort subDB */; - - switch (op) { - case MDBX_FIRST: - rc = cursor_first_batch(mc); - break; - case MDBX_NEXT: - rc = cursor_next_batch(mc); - break; - case MDBX_GET_CURRENT: - rc = likely(mc->mc_flags & C_INITIALIZED) ? MDBX_SUCCESS : MDBX_ENODATA; - break; - default: - DEBUG("unhandled/unimplemented cursor operation %u", op); - rc = MDBX_EINVAL; - break; - } - - if (unlikely(rc != MDBX_SUCCESS)) { - *count = 0; - return rc; - } - - const MDBX_page *const mp = mc->mc_pg[mc->mc_top]; - if (!MDBX_DISABLE_VALIDATION && unlikely(!CHECK_LEAF_TYPE(mc, mp))) { - ERROR("unexpected leaf-page #%" PRIaPGNO " type 0x%x seen by cursor", - mp->mp_pgno, mp->mp_flags); - return MDBX_CORRUPTED; - } - const size_t nkeys = page_numkeys(mp); - size_t i = mc->mc_ki[mc->mc_top], n = 0; - if (unlikely(i >= nkeys)) { - cASSERT(mc, op == MDBX_GET_CURRENT); - cASSERT(mc, mdbx_cursor_on_last(mc) == MDBX_RESULT_TRUE); - *count = 0; - if (mc->mc_flags & C_EOF) { - cASSERT(mc, mdbx_cursor_on_last(mc) == MDBX_RESULT_TRUE); - return MDBX_ENODATA; - } - if (mdbx_cursor_on_last(mc) != MDBX_RESULT_TRUE) - return MDBX_EINVAL /* again MDBX_GET_CURRENT after MDBX_GET_CURRENT */; - mc->mc_flags |= C_EOF; - return MDBX_NOTFOUND; - } - - do { - if (unlikely(n + 2 > limit)) { - rc = MDBX_RESULT_TRUE; - break; - } - const MDBX_node *leaf = page_node(mp, i); - get_key(leaf, &pairs[n]); - rc = node_read(mc, leaf, &pairs[n + 1], mp); - if (unlikely(rc != MDBX_SUCCESS)) - break; - n += 2; - } while (++i < nkeys); - - mc->mc_ki[mc->mc_top] = (indx_t)i; - *count = n; +__cold int cursor_validate_updating(MDBX_cursor *mc) { + const uint8_t checking = mc->checking; + mc->checking |= z_updating; + const int rc = cursor_validate(mc); + mc->checking = checking; return rc; } +bool cursor_is_tracked(const MDBX_cursor *mc) { + for (MDBX_cursor *scan = mc->txn->cursors[cursor_dbi(mc)]; scan; scan = scan->next) + if (mc == ((mc->flags & z_inner) ? &scan->subcur->cursor : scan)) + return true; + return false; +} + +/*----------------------------------------------------------------------------*/ + static int touch_dbi(MDBX_cursor *mc) { - cASSERT(mc, (*mc->mc_dbistate & DBI_DIRTY) == 0); - *mc->mc_dbistate |= DBI_DIRTY; - mc->mc_txn->mt_flags |= MDBX_TXN_DIRTY; - if (mc->mc_dbi >= CORE_DBS) { + cASSERT(mc, (mc->flags & z_inner) == 0); + cASSERT(mc, (*cursor_dbi_state(mc) & DBI_DIRTY) == 0); + *cursor_dbi_state(mc) |= DBI_DIRTY; + mc->txn->flags |= MDBX_TXN_DIRTY; + + if (!cursor_is_core(mc)) { /* Touch DB record of named DB */ - MDBX_cursor_couple cx; - int rc = cursor_init(&cx.outer, mc->mc_txn, MAIN_DBI); + cursor_couple_t cx; + int rc = dbi_check(mc->txn, MAIN_DBI); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + rc = cursor_init(&cx.outer, mc->txn, MAIN_DBI); if (unlikely(rc != MDBX_SUCCESS)) return rc; - mc->mc_txn->mt_dbistate[MAIN_DBI] |= DBI_DIRTY; - rc = page_search(&cx.outer, &mc->mc_dbx->md_name, MDBX_PS_MODIFY); + mc->txn->dbi_state[MAIN_DBI] |= DBI_DIRTY; + rc = tree_search(&cx.outer, &container_of(mc->clc, kvx_t, clc)->name, Z_MODIFY); if (unlikely(rc != MDBX_SUCCESS)) return rc; } return MDBX_SUCCESS; } -static __hot int cursor_touch(MDBX_cursor *const mc, const MDBX_val *key, - const MDBX_val *data) { - cASSERT(mc, (mc->mc_txn->mt_flags & MDBX_TXN_RDONLY) == 0); - cASSERT(mc, (mc->mc_flags & C_INITIALIZED) || mc->mc_snum == 0); +__hot int cursor_touch(MDBX_cursor *const mc, const MDBX_val *key, const MDBX_val *data) { + cASSERT(mc, (mc->txn->flags & MDBX_TXN_RDONLY) == 0); + cASSERT(mc, is_pointed(mc) || mc->tree->height == 0); cASSERT(mc, cursor_is_tracked(mc)); - if ((mc->mc_flags & C_SUB) == 0) { - MDBX_txn *const txn = mc->mc_txn; - txn_lru_turn(txn); + cASSERT(mc, F_ISSET(dbi_state(mc->txn, FREE_DBI), DBI_LINDO | DBI_VALID)); + cASSERT(mc, F_ISSET(dbi_state(mc->txn, MAIN_DBI), DBI_LINDO | DBI_VALID)); + if ((mc->flags & z_inner) == 0) { + MDBX_txn *const txn = mc->txn; + dpl_lru_turn(txn); - if (unlikely((*mc->mc_dbistate & DBI_DIRTY) == 0)) { + if (unlikely((*cursor_dbi_state(mc) & DBI_DIRTY) == 0)) { int err = touch_dbi(mc); if (unlikely(err != MDBX_SUCCESS)) return err; @@ -21346,13 +16200,13 @@ static __hot int cursor_touch(MDBX_cursor *const mc, const MDBX_val *key, /* Estimate how much space this operation will take: */ /* 1) Max b-tree height, reasonable enough with including dups' sub-tree */ - size_t need = CURSOR_STACK + 3; + size_t need = CURSOR_STACK_SIZE + 3; /* 2) GC/FreeDB for any payload */ - if (mc->mc_dbi > FREE_DBI) { - need += txn->mt_dbs[FREE_DBI].md_depth + (size_t)3; + if (!cursor_is_gc(mc)) { + need += txn->dbs[FREE_DBI].height + (size_t)3; /* 3) Named DBs also dirty the main DB */ - if (mc->mc_dbi > MAIN_DBI) - need += txn->mt_dbs[MAIN_DBI].md_depth + (size_t)3; + if (!cursor_is_main(mc)) + need += txn->dbs[MAIN_DBI].height + (size_t)3; } #if xMDBX_DEBUG_SPILLING != 2 /* production mode */ @@ -21360,13 +16214,13 @@ static __hot int cursor_touch(MDBX_cursor *const mc, const MDBX_val *key, * for extensively splitting, rebalance and merging */ need += need; /* 5) Factor the key+data which to be put in */ - need += bytes2pgno(txn->mt_env, node_size(key, data)) + (size_t)1; + need += bytes2pgno(txn->env, node_size(key, data)) + (size_t)1; #else /* debug mode */ (void)key; (void)data; - txn->mt_env->debug_dirtied_est = ++need; - txn->mt_env->debug_dirtied_act = 0; + txn->env->debug_dirtied_est = ++need; + txn->env->debug_dirtied_act = 0; #endif /* xMDBX_DEBUG_SPILLING == 2 */ int err = txn_spill(txn, mc, need); @@ -21374,116 +16228,630 @@ static __hot int cursor_touch(MDBX_cursor *const mc, const MDBX_val *key, return err; } - int rc = MDBX_SUCCESS; - if (likely(mc->mc_snum)) { - mc->mc_top = 0; + if (likely(is_pointed(mc)) && ((mc->txn->flags & MDBX_TXN_SPILLS) || !is_modifable(mc->txn, mc->pg[mc->top]))) { + const int8_t top = mc->top; + mc->top = 0; do { - rc = page_touch(mc); - if (unlikely(rc != MDBX_SUCCESS)) - break; - mc->mc_top += 1; - } while (mc->mc_top < mc->mc_snum); - mc->mc_top = mc->mc_snum - 1; + int err = page_touch(mc); + if (unlikely(err != MDBX_SUCCESS)) + return err; + mc->top += 1; + } while (mc->top <= top); + mc->top = top; } - return rc; + return MDBX_SUCCESS; } -static size_t leaf2_reserve(const MDBX_env *const env, size_t host_page_room, - size_t subpage_len, size_t item_len) { - eASSERT(env, (subpage_len & 1) == 0); - eASSERT(env, - env->me_subpage_reserve_prereq > env->me_subpage_room_threshold + - env->me_subpage_reserve_limit && - env->me_leaf_nodemax >= env->me_subpage_limit + NODESIZE); - size_t reserve = 0; - for (size_t n = 0; - n < 5 && reserve + item_len <= env->me_subpage_reserve_limit && - EVEN(subpage_len + item_len) <= env->me_subpage_limit && - host_page_room >= - env->me_subpage_reserve_prereq + EVEN(subpage_len + item_len); - ++n) { - subpage_len += item_len; - reserve += item_len; +/*----------------------------------------------------------------------------*/ + +int cursor_shadow(MDBX_cursor *mc, MDBX_txn *nested, const size_t dbi) { + tASSERT(nested, dbi > FREE_DBI && dbi < nested->n_dbi); + const size_t size = mc->subcur ? sizeof(MDBX_cursor) + sizeof(subcur_t) : sizeof(MDBX_cursor); + for (MDBX_cursor *bk; mc; mc = bk->next) { + cASSERT(mc, mc != mc->next); + if (mc->signature != cur_signature_live) { + ENSURE(nested->env, mc->signature == cur_signature_wait4eot); + bk = mc; + continue; + } + bk = osal_malloc(size); + if (unlikely(!bk)) + return MDBX_ENOMEM; +#if MDBX_DEBUG + memset(bk, 0xCD, size); + VALGRIND_MAKE_MEM_UNDEFINED(bk, size); +#endif /* MDBX_DEBUG */ + *bk = *mc; + mc->backup = bk; + mc->txn = nested; + mc->tree = &nested->dbs[dbi]; + mc->dbi_state = &nested->dbi_state[dbi]; + subcur_t *mx = mc->subcur; + if (mx) { + *(subcur_t *)(bk + 1) = *mx; + mx->cursor.txn = nested; + mx->cursor.dbi_state = &nested->dbi_state[dbi]; + } + mc->next = nested->cursors[dbi]; + nested->cursors[dbi] = mc; } - return reserve + (subpage_len & 1); + return MDBX_SUCCESS; } -static __hot int cursor_put_nochecklen(MDBX_cursor *mc, const MDBX_val *key, - MDBX_val *data, unsigned flags) { - int err; - DKBUF_DEBUG; - MDBX_env *const env = mc->mc_txn->mt_env; - if (LOG_ENABLED(MDBX_LOG_DEBUG) && (flags & MDBX_RESERVE)) - data->iov_base = nullptr; - DEBUG("==> put db %d key [%s], size %" PRIuPTR ", data [%s] size %" PRIuPTR, - DDBI(mc), DKEY_DEBUG(key), key->iov_len, DVAL_DEBUG(data), - data->iov_len); +MDBX_cursor *cursor_eot(MDBX_cursor *mc, MDBX_txn *txn, const bool merge) { + MDBX_cursor *const next = mc->next; + const unsigned stage = mc->signature; + MDBX_cursor *const bk = mc->backup; + ENSURE(txn->env, stage == cur_signature_live || (stage == cur_signature_wait4eot && bk)); + tASSERT(txn, mc->txn == txn); + if (bk) { + subcur_t *mx = mc->subcur; + tASSERT(txn, mc->txn->parent != nullptr); + tASSERT(txn, bk->txn == txn->parent); + /* Zap: Using uninitialized memory '*mc->backup'. */ + MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(6001); + ENSURE(txn->env, bk->signature == cur_signature_live); + tASSERT(txn, mx == bk->subcur); + if (merge) { + /* Update pointers to parent txn */ + mc->next = bk->next; + mc->backup = bk->backup; + mc->txn = bk->txn; + mc->tree = bk->tree; + mc->dbi_state = bk->dbi_state; + if (mx) { + mx->cursor.txn = bk->txn; + mx->cursor.dbi_state = bk->dbi_state; + } + } else { + /* Restore from backup, i.e. rollback/abort nested txn */ + *mc = *bk; + mc->signature = stage /* Promote (cur_signature_wait4eot) state to parent txn */; + if (mx) + *mx = *(subcur_t *)(bk + 1); + } + bk->signature = 0; + osal_free(bk); + } else { + ENSURE(mc->txn->env, stage == cur_signature_live); + mc->signature = cur_signature_ready4dispose /* Cursor may be reused */; + mc->next = mc; + cursor_drown((cursor_couple_t *)mc); + } + return next; +} - if ((flags & MDBX_CURRENT) != 0 && (mc->mc_flags & C_SUB) == 0) { - if (unlikely(flags & (MDBX_APPEND | MDBX_NOOVERWRITE))) - return MDBX_EINVAL; - /* Опция MDBX_CURRENT означает, что запрошено обновление текущей записи, - * на которой сейчас стоит курсор. Проверяем что переданный ключ совпадает - * со значением в текущей позиции курсора. - * Здесь проще вызвать cursor_get(), так как для обслуживания таблиц - * с MDBX_DUPSORT также требуется текущий размер данных. */ - MDBX_val current_key, current_data; - err = cursor_get(mc, ¤t_key, ¤t_data, MDBX_GET_CURRENT); - if (unlikely(err != MDBX_SUCCESS)) - return err; - if (mc->mc_dbx->md_cmp(key, ¤t_key) != 0) - return MDBX_EKEYMISMATCH; +/*----------------------------------------------------------------------------*/ - if (unlikely((flags & MDBX_MULTIPLE))) - goto drop_current; +static __always_inline int couple_init(cursor_couple_t *couple, const MDBX_txn *const txn, tree_t *const tree, + kvx_t *const kvx, uint8_t *const dbi_state) { + + VALGRIND_MAKE_MEM_UNDEFINED(couple, sizeof(cursor_couple_t)); + tASSERT(txn, F_ISSET(*dbi_state, DBI_VALID | DBI_LINDO)); + + couple->outer.signature = cur_signature_live; + couple->outer.next = &couple->outer; + couple->outer.backup = nullptr; + couple->outer.txn = (MDBX_txn *)txn; + couple->outer.tree = tree; + couple->outer.clc = &kvx->clc; + couple->outer.dbi_state = dbi_state; + couple->outer.top_and_flags = z_fresh_mark; + STATIC_ASSERT((int)z_branch == P_BRANCH && (int)z_leaf == P_LEAF && (int)z_largepage == P_LARGE && + (int)z_dupfix == P_DUPFIX); + couple->outer.checking = (AUDIT_ENABLED() || (txn->env->flags & MDBX_VALIDATION)) ? z_pagecheck | z_leaf : z_leaf; + couple->outer.subcur = nullptr; + + if (tree->flags & MDBX_DUPSORT) { + couple->inner.cursor.signature = cur_signature_live; + subcur_t *const mx = couple->outer.subcur = &couple->inner; + mx->cursor.subcur = nullptr; + mx->cursor.next = &mx->cursor; + mx->cursor.txn = (MDBX_txn *)txn; + mx->cursor.tree = &mx->nested_tree; + mx->cursor.clc = ptr_disp(couple->outer.clc, sizeof(clc_t)); + tASSERT(txn, &mx->cursor.clc->k == &kvx->clc.v); + mx->cursor.dbi_state = dbi_state; + mx->cursor.top_and_flags = z_fresh_mark | z_inner; + STATIC_ASSERT(MDBX_DUPFIXED * 2 == P_DUPFIX); + mx->cursor.checking = couple->outer.checking + ((tree->flags & MDBX_DUPFIXED) << 1); + } + + if (unlikely(*dbi_state & DBI_STALE)) + return tbl_fetch(couple->outer.txn, cursor_dbi(&couple->outer)); + + return tbl_setup_ifneed(txn->env, kvx, tree); +} + +__cold int cursor_init4walk(cursor_couple_t *couple, const MDBX_txn *const txn, tree_t *const tree, kvx_t *const kvx) { + return couple_init(couple, txn, tree, kvx, txn->dbi_state); +} + +int cursor_init(MDBX_cursor *mc, const MDBX_txn *txn, size_t dbi) { + STATIC_ASSERT(offsetof(cursor_couple_t, outer) == 0); + int rc = dbi_check(txn, dbi); + if (likely(rc == MDBX_SUCCESS)) + rc = couple_init(container_of(mc, cursor_couple_t, outer), txn, &txn->dbs[dbi], &txn->env->kvs[dbi], + &txn->dbi_state[dbi]); + return rc; +} - if (mc->mc_db->md_flags & MDBX_DUPSORT) { - MDBX_node *node = page_node(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); - if (node_flags(node) & F_DUPDATA) { - cASSERT(mc, mc->mc_xcursor != NULL && - (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)); - /* Если за ключом более одного значения, либо если размер данных - * отличается, то вместо обновления требуется удаление и - * последующая вставка. */ - if (mc->mc_xcursor->mx_db.md_entries > 1 || - current_data.iov_len != data->iov_len) { - drop_current: - err = cursor_del(mc, flags & MDBX_ALLDUPS); - if (unlikely(err != MDBX_SUCCESS)) - return err; - flags -= MDBX_CURRENT; - goto skip_check_samedata; - } - } else if (unlikely(node_size(key, data) > env->me_leaf_nodemax)) { - err = cursor_del(mc, 0); - if (unlikely(err != MDBX_SUCCESS)) - return err; - flags -= MDBX_CURRENT; - goto skip_check_samedata; - } +__cold static int unexpected_dupsort(MDBX_cursor *mc) { + ERROR("unexpected dupsort-page/node for non-dupsort db/cursor (dbi %zu)", cursor_dbi(mc)); + mc->txn->flags |= MDBX_TXN_ERROR; + be_poor(mc); + return MDBX_CORRUPTED; +} + +int cursor_dupsort_setup(MDBX_cursor *mc, const node_t *node, const page_t *mp) { + cASSERT(mc, is_pointed(mc)); + subcur_t *mx = mc->subcur; + if (!MDBX_DISABLE_VALIDATION && unlikely(mx == nullptr)) + return unexpected_dupsort(mc); + + const uint8_t flags = node_flags(node); + switch (flags) { + default: + ERROR("invalid node flags %u", flags); + goto bailout; + case N_DUP | N_TREE: + if (!MDBX_DISABLE_VALIDATION && unlikely(node_ds(node) != sizeof(tree_t))) { + ERROR("invalid nested-db record size (%zu, expect %zu)", node_ds(node), sizeof(tree_t)); + goto bailout; } - if (!(flags & MDBX_RESERVE) && - unlikely(cmp_lenfast(¤t_data, data) == 0)) - return MDBX_SUCCESS /* the same data, nothing to update */; - skip_check_samedata:; + memcpy(&mx->nested_tree, node_data(node), sizeof(tree_t)); + const txnid_t pp_txnid = mp->txnid; + if (!MDBX_DISABLE_VALIDATION && unlikely(mx->nested_tree.mod_txnid > pp_txnid)) { + ERROR("nested-db.mod_txnid (%" PRIaTXN ") > page-txnid (%" PRIaTXN ")", mx->nested_tree.mod_txnid, pp_txnid); + goto bailout; + } + mx->cursor.top_and_flags = z_fresh_mark | z_inner; + break; + case N_DUP: + if (!MDBX_DISABLE_VALIDATION && unlikely(node_ds(node) <= PAGEHDRSZ)) { + ERROR("invalid nested-page size %zu", node_ds(node)); + goto bailout; + } + page_t *sp = node_data(node); + mx->nested_tree.height = 1; + mx->nested_tree.branch_pages = 0; + mx->nested_tree.leaf_pages = 1; + mx->nested_tree.large_pages = 0; + mx->nested_tree.items = page_numkeys(sp); + mx->nested_tree.root = 0; + mx->nested_tree.mod_txnid = mp->txnid; + mx->cursor.top_and_flags = z_inner; + mx->cursor.pg[0] = sp; + mx->cursor.ki[0] = 0; + mx->nested_tree.flags = flags_db2sub(mc->tree->flags); + mx->nested_tree.dupfix_size = (mc->tree->flags & MDBX_DUPFIXED) ? sp->dupfix_ksize : 0; + break; } - int rc = MDBX_SUCCESS; - if (mc->mc_db->md_root == P_INVALID) { - /* new database, cursor has nothing to point to */ - mc->mc_snum = 0; - mc->mc_top = 0; - mc->mc_flags &= ~C_INITIALIZED; - rc = MDBX_NO_ROOT; - } else if ((flags & MDBX_CURRENT) == 0) { - bool exact = false; - MDBX_val last_key, old_data; - if ((flags & MDBX_APPEND) && mc->mc_db->md_entries > 0) { - rc = cursor_last(mc, &last_key, &old_data); - if (likely(rc == MDBX_SUCCESS)) { - const int cmp = mc->mc_dbx->md_cmp(key, &last_key); - if (likely(cmp > 0)) { - mc->mc_ki[mc->mc_top]++; /* step forward for appending */ + if (unlikely(mx->nested_tree.dupfix_size != mc->tree->dupfix_size)) { + if (!MDBX_DISABLE_VALIDATION && unlikely(mc->tree->dupfix_size != 0)) { + ERROR("cursor mismatched nested-db dupfix_size %u", mc->tree->dupfix_size); + goto bailout; + } + if (!MDBX_DISABLE_VALIDATION && unlikely((mc->tree->flags & MDBX_DUPFIXED) == 0)) { + ERROR("mismatched nested-db flags %u", mc->tree->flags); + goto bailout; + } + if (!MDBX_DISABLE_VALIDATION && + unlikely(mx->nested_tree.dupfix_size < mc->clc->v.lmin || mx->nested_tree.dupfix_size > mc->clc->v.lmax)) { + ERROR("mismatched nested-db.dupfix_size (%u) <> min/max value-length " + "(%zu/%zu)", + mx->nested_tree.dupfix_size, mc->clc->v.lmin, mc->clc->v.lmax); + goto bailout; + } + mc->tree->dupfix_size = mx->nested_tree.dupfix_size; + mc->clc->v.lmin = mc->clc->v.lmax = mx->nested_tree.dupfix_size; + cASSERT(mc, mc->clc->v.lmax >= mc->clc->v.lmin); + } + + DEBUG("Sub-db dbi -%zu root page %" PRIaPGNO, cursor_dbi(&mx->cursor), mx->nested_tree.root); + return MDBX_SUCCESS; + +bailout: + mx->cursor.top_and_flags = z_poor_mark | z_inner; + return MDBX_CORRUPTED; +} + +/*----------------------------------------------------------------------------*/ + +MDBX_cursor *cursor_cpstk(const MDBX_cursor *csrc, MDBX_cursor *cdst) { + cASSERT(cdst, cdst->txn == csrc->txn); + cASSERT(cdst, cdst->tree == csrc->tree); + cASSERT(cdst, cdst->clc == csrc->clc); + cASSERT(cdst, cdst->dbi_state == csrc->dbi_state); + cdst->top_and_flags = csrc->top_and_flags; + + for (intptr_t i = 0; i <= csrc->top; i++) { + cdst->pg[i] = csrc->pg[i]; + cdst->ki[i] = csrc->ki[i]; + } + return cdst; +} + +static __always_inline int sibling(MDBX_cursor *mc, bool right) { + if (mc->top < 1) { + /* root has no siblings */ + return MDBX_NOTFOUND; + } + + cursor_pop(mc); + DEBUG("parent page is page %" PRIaPGNO ", index %u", mc->pg[mc->top]->pgno, mc->ki[mc->top]); + + int err; + if (right ? (mc->ki[mc->top] + (size_t)1 >= page_numkeys(mc->pg[mc->top])) : (mc->ki[mc->top] == 0)) { + DEBUG("no more keys aside, moving to next %s sibling", right ? "right" : "left"); + err = right ? cursor_sibling_right(mc) : cursor_sibling_left(mc); + if (err != MDBX_SUCCESS) { + if (likely(err == MDBX_NOTFOUND)) + /* undo cursor_pop before returning */ + mc->top += 1; + return err; + } + } else { + mc->ki[mc->top] += right ? 1 : -1; + DEBUG("just moving to %s index key %u", right ? "right" : "left", mc->ki[mc->top]); + } + cASSERT(mc, is_branch(mc->pg[mc->top])); + + page_t *mp = mc->pg[mc->top]; + const node_t *node = page_node(mp, mc->ki[mc->top]); + err = page_get(mc, node_pgno(node), &mp, mp->txnid); + if (likely(err == MDBX_SUCCESS)) { + err = cursor_push(mc, mp, right ? 0 : (indx_t)page_numkeys(mp) - 1); + if (likely(err == MDBX_SUCCESS)) + return err; + } + + be_poor(mc); + return err; +} + +__hot int cursor_sibling_left(MDBX_cursor *mc) { + int err = sibling(mc, false); + if (likely(err != MDBX_NOTFOUND)) + return err; + + cASSERT(mc, mc->top >= 0); + size_t nkeys = page_numkeys(mc->pg[mc->top]); + cASSERT(mc, nkeys > 0); + mc->ki[mc->top] = 0; + return MDBX_NOTFOUND; +} + +__hot int cursor_sibling_right(MDBX_cursor *mc) { + int err = sibling(mc, true); + if (likely(err != MDBX_NOTFOUND)) + return err; + + cASSERT(mc, mc->top >= 0); + size_t nkeys = page_numkeys(mc->pg[mc->top]); + cASSERT(mc, nkeys > 0); + mc->ki[mc->top] = (indx_t)nkeys - 1; + mc->flags = z_eof_soft | z_eof_hard | (mc->flags & z_clear_mask); + inner_gone(mc); + return MDBX_NOTFOUND; +} + +/*----------------------------------------------------------------------------*/ + +/* Функция-шаблон: Приземляет курсор на данные в текущей позиции. + * В том числе, загружает данные во вложенный курсор при его наличии. */ +static __always_inline int cursor_bring(const bool inner, const bool tend2first, MDBX_cursor *__restrict mc, + MDBX_val *__restrict key, MDBX_val *__restrict data, bool eof) { + if (inner) { + cASSERT(mc, !data && !mc->subcur && (mc->flags & z_inner) != 0); + } else { + cASSERT(mc, (mc->flags & z_inner) == 0); + } + + const page_t *mp = mc->pg[mc->top]; + if (!MDBX_DISABLE_VALIDATION && unlikely(!check_leaf_type(mc, mp))) { + ERROR("unexpected leaf-page #%" PRIaPGNO " type 0x%x seen by cursor", mp->pgno, mp->flags); + return MDBX_CORRUPTED; + } + + const size_t nkeys = page_numkeys(mp); + cASSERT(mc, nkeys > 0); + const size_t ki = mc->ki[mc->top]; + cASSERT(mc, nkeys > ki); + cASSERT(mc, !eof || ki == nkeys - 1); + + if (inner && is_dupfix_leaf(mp)) { + be_filled(mc); + if (eof) + mc->flags |= z_eof_soft; + if (likely(key)) + *key = page_dupfix_key(mp, ki, mc->tree->dupfix_size); + return MDBX_SUCCESS; + } + + const node_t *__restrict node = page_node(mp, ki); + if (!inner && (node_flags(node) & N_DUP)) { + int err = cursor_dupsort_setup(mc, node, mp); + if (unlikely(err != MDBX_SUCCESS)) + return err; + MDBX_ANALYSIS_ASSUME(mc->subcur != nullptr); + if (node_flags(node) & N_TREE) { + err = tend2first ? inner_first(&mc->subcur->cursor, data) : inner_last(&mc->subcur->cursor, data); + if (unlikely(err != MDBX_SUCCESS)) + return err; + } else { + if (!tend2first) { + mc->subcur->cursor.ki[0] = (indx_t)mc->subcur->nested_tree.items - 1; + mc->subcur->cursor.flags |= z_eof_soft; + } + if (data) { + const page_t *inner_mp = mc->subcur->cursor.pg[0]; + cASSERT(mc, is_subpage(inner_mp) && is_leaf(inner_mp)); + const size_t inner_ki = mc->subcur->cursor.ki[0]; + if (is_dupfix_leaf(inner_mp)) + *data = page_dupfix_key(inner_mp, inner_ki, mc->tree->dupfix_size); + else + *data = get_key(page_node(inner_mp, inner_ki)); + } + } + be_filled(mc); + } else { + if (!inner) + inner_gone(mc); + if (data) { + int err = node_read(mc, node, data, mp); + if (unlikely(err != MDBX_SUCCESS)) + return err; + } + be_filled(mc); + if (eof) + mc->flags |= z_eof_soft; + } + + get_key_optional(node, key); + return MDBX_SUCCESS; +} + +/* Функция-шаблон: Устанавливает курсор в начало или конец. */ +static __always_inline int cursor_brim(const bool inner, const bool tend2first, MDBX_cursor *__restrict mc, + MDBX_val *__restrict key, MDBX_val *__restrict data) { + if (mc->top != 0) { + int err = tree_search(mc, nullptr, tend2first ? Z_FIRST : Z_LAST); + if (unlikely(err != MDBX_SUCCESS)) + return err; + } + const size_t nkeys = page_numkeys(mc->pg[mc->top]); + cASSERT(mc, nkeys > 0); + mc->ki[mc->top] = tend2first ? 0 : nkeys - 1; + return cursor_bring(inner, tend2first, mc, key, data, !tend2first); +} + +__hot int inner_first(MDBX_cursor *mc, MDBX_val *data) { return cursor_brim(true, true, mc, data, nullptr); } + +__hot int inner_last(MDBX_cursor *mc, MDBX_val *data) { return cursor_brim(true, false, mc, data, nullptr); } + +__hot int outer_first(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data) { + return cursor_brim(false, true, mc, key, data); +} + +__hot int outer_last(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data) { + return cursor_brim(false, false, mc, key, data); +} + +/*----------------------------------------------------------------------------*/ + +/* Функция-шаблон: Передвигает курсор на одну позицию. + * При необходимости управляет вложенным курсором. */ +static __always_inline int cursor_step(const bool inner, const bool forward, MDBX_cursor *__restrict mc, + MDBX_val *__restrict key, MDBX_val *__restrict data, MDBX_cursor_op op) { + if (forward) { + if (inner) + cASSERT(mc, op == MDBX_NEXT); + else + cASSERT(mc, op == MDBX_NEXT || op == MDBX_NEXT_DUP || op == MDBX_NEXT_NODUP); + } else { + if (inner) + cASSERT(mc, op == MDBX_PREV); + else + cASSERT(mc, op == MDBX_PREV || op == MDBX_PREV_DUP || op == MDBX_PREV_NODUP); + } + if (inner) { + cASSERT(mc, !data && !mc->subcur && (mc->flags & z_inner) != 0); + } else { + cASSERT(mc, (mc->flags & z_inner) == 0); + } + + if (unlikely(is_poor(mc))) { + int state = mc->flags; + if (state & z_fresh) { + if (forward) + return inner ? inner_first(mc, key) : outer_first(mc, key, data); + else + return inner ? inner_last(mc, key) : outer_last(mc, key, data); + } + mc->flags = inner ? z_inner | z_poor_mark : z_poor_mark; + return (state & z_after_delete) ? MDBX_NOTFOUND : MDBX_ENODATA; + } + + const page_t *mp = mc->pg[mc->top]; + const intptr_t nkeys = page_numkeys(mp); + cASSERT(mc, nkeys > 0); + + intptr_t ki = mc->ki[mc->top]; + const uint8_t state = mc->flags & (z_after_delete | z_hollow | z_eof_hard | z_eof_soft); + if (likely(state == 0)) { + cASSERT(mc, ki < nkeys); + if (!inner && op != (forward ? MDBX_NEXT_NODUP : MDBX_PREV_NODUP)) { + int err = MDBX_NOTFOUND; + if (inner_pointed(mc)) { + err = forward ? inner_next(&mc->subcur->cursor, data) : inner_prev(&mc->subcur->cursor, data); + if (likely(err == MDBX_SUCCESS)) { + get_key_optional(page_node(mp, ki), key); + return MDBX_SUCCESS; + } + if (unlikely(err != MDBX_NOTFOUND && err != MDBX_ENODATA)) { + cASSERT(mc, !inner_pointed(mc)); + return err; + } + cASSERT(mc, !forward || (mc->subcur->cursor.flags & z_eof_soft)); + } + if (op == (forward ? MDBX_NEXT_DUP : MDBX_PREV_DUP)) + return err; + } + if (!inner) + inner_gone(mc); + } else { + if (mc->flags & z_hollow) { + cASSERT(mc, !inner_pointed(mc)); + return MDBX_ENODATA; + } + + if (!inner && op == (forward ? MDBX_NEXT_DUP : MDBX_PREV_DUP)) + return MDBX_NOTFOUND; + + if (forward) { + if (state & z_after_delete) { + if (ki < nkeys) + goto bring; + } else { + cASSERT(mc, state & (z_eof_soft | z_eof_hard)); + return MDBX_NOTFOUND; + } + } else if (state & z_eof_hard) { + mc->ki[mc->top] = (indx_t)nkeys - 1; + goto bring; + } + } + + DEBUG("turn-%s: top page was %" PRIaPGNO " in cursor %p, ki %zi of %zi", forward ? "next" : "prev", mp->pgno, + __Wpedantic_format_voidptr(mc), ki, nkeys); + if (forward) { + if (likely(++ki < nkeys)) + mc->ki[mc->top] = (indx_t)ki; + else { + DEBUG("%s", "=====> move to next sibling page"); + int err = cursor_sibling_right(mc); + if (unlikely(err != MDBX_SUCCESS)) + return err; + mp = mc->pg[mc->top]; + DEBUG("next page is %" PRIaPGNO ", key index %u", mp->pgno, mc->ki[mc->top]); + } + } else { + if (likely(--ki >= 0)) + mc->ki[mc->top] = (indx_t)ki; + else { + DEBUG("%s", "=====> move to prev sibling page"); + int err = cursor_sibling_left(mc); + if (unlikely(err != MDBX_SUCCESS)) + return err; + mp = mc->pg[mc->top]; + DEBUG("prev page is %" PRIaPGNO ", key index %u", mp->pgno, mc->ki[mc->top]); + } + } + DEBUG("==> cursor points to page %" PRIaPGNO " with %zu keys, key index %u", mp->pgno, page_numkeys(mp), + mc->ki[mc->top]); + +bring: + return cursor_bring(inner, forward, mc, key, data, false); +} + +__hot int inner_next(MDBX_cursor *mc, MDBX_val *data) { return cursor_step(true, true, mc, data, nullptr, MDBX_NEXT); } + +__hot int inner_prev(MDBX_cursor *mc, MDBX_val *data) { return cursor_step(true, false, mc, data, nullptr, MDBX_PREV); } + +__hot int outer_next(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, MDBX_cursor_op op) { + return cursor_step(false, true, mc, key, data, op); +} + +__hot int outer_prev(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, MDBX_cursor_op op) { + return cursor_step(false, false, mc, key, data, op); +} + +/*----------------------------------------------------------------------------*/ + +__hot int cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, unsigned flags) { + int err; + DKBUF_DEBUG; + MDBX_env *const env = mc->txn->env; + if (LOG_ENABLED(MDBX_LOG_DEBUG) && (flags & MDBX_RESERVE)) + data->iov_base = nullptr; + DEBUG("==> put db %d key [%s], size %" PRIuPTR ", data [%s] size %" PRIuPTR, cursor_dbi_dbg(mc), DKEY_DEBUG(key), + key->iov_len, DVAL_DEBUG(data), data->iov_len); + + if ((flags & MDBX_CURRENT) != 0 && (mc->flags & z_inner) == 0) { + if (unlikely(flags & (MDBX_APPEND | MDBX_NOOVERWRITE))) + return MDBX_EINVAL; + /* Запрошено обновление текущей записи, на которой сейчас стоит курсор. + * Проверяем что переданный ключ совпадает со значением в текущей позиции + * курсора. Здесь проще вызвать cursor_ops(), так как для обслуживания + * таблиц с MDBX_DUPSORT также требуется текущий размер данных. */ + MDBX_val current_key, current_data; + err = cursor_ops(mc, ¤t_key, ¤t_data, MDBX_GET_CURRENT); + if (unlikely(err != MDBX_SUCCESS)) + return err; + if (mc->clc->k.cmp(key, ¤t_key) != 0) + return MDBX_EKEYMISMATCH; + + if (unlikely((flags & MDBX_MULTIPLE))) { + if (unlikely(!mc->subcur)) + return MDBX_EINVAL; + err = cursor_del(mc, flags & MDBX_ALLDUPS); + if (unlikely(err != MDBX_SUCCESS)) + return err; + if (unlikely(data[1].iov_len == 0)) + return MDBX_SUCCESS; + flags -= MDBX_CURRENT; + goto skip_check_samedata; + } + + if (mc->subcur) { + node_t *node = page_node(mc->pg[mc->top], mc->ki[mc->top]); + if (node_flags(node) & N_DUP) { + cASSERT(mc, inner_pointed(mc)); + /* Если за ключом более одного значения, либо если размер данных + * отличается, то вместо обновления требуется удаление и + * последующая вставка. */ + if (mc->subcur->nested_tree.items > 1 || current_data.iov_len != data->iov_len) { + err = cursor_del(mc, flags & MDBX_ALLDUPS); + if (unlikely(err != MDBX_SUCCESS)) + return err; + flags -= MDBX_CURRENT; + goto skip_check_samedata; + } + } else if (unlikely(node_size(key, data) > env->leaf_nodemax)) { + /* Уже есть пара key-value хранящаяся в обычном узле. Новые данные + * слишком большие для размещения в обычном узле вместе с ключом, но + * могут быть размещены в вложенном дереве. Удаляем узел со старыми + * данными, чтобы при помещении новых создать вложенное дерево. */ + err = cursor_del(mc, 0); + if (unlikely(err != MDBX_SUCCESS)) + return err; + flags -= MDBX_CURRENT; + goto skip_check_samedata; + } + } + if (!(flags & MDBX_RESERVE) && unlikely(cmp_lenfast(¤t_data, data) == 0)) + return MDBX_SUCCESS /* the same data, nothing to update */; + skip_check_samedata:; + } + + int rc = MDBX_SUCCESS; + if (mc->tree->height == 0) { + /* new database, cursor has nothing to point to */ + cASSERT(mc, is_poor(mc)); + rc = MDBX_NO_ROOT; + } else if ((flags & MDBX_CURRENT) == 0) { + bool exact = false; + MDBX_val last_key, old_data; + if ((flags & MDBX_APPEND) && mc->tree->items > 0) { + old_data.iov_base = nullptr; + old_data.iov_len = 0; + rc = (mc->flags & z_inner) ? inner_last(mc, &last_key) : outer_last(mc, &last_key, &old_data); + if (likely(rc == MDBX_SUCCESS)) { + const int cmp = mc->clc->k.cmp(key, &last_key); + if (likely(cmp > 0)) { + mc->ki[mc->top]++; /* step forward for appending */ rc = MDBX_NOTFOUND; } else if (unlikely(cmp != 0)) { /* new-key < last-key */ @@ -21494,9 +16862,9 @@ static __hot int cursor_put_nochecklen(MDBX_cursor *mc, const MDBX_val *key, } } } else { - struct cursor_set_result csr = - /* olddata may not be updated in case LEAF2-page of dupfixed-subDB */ - cursor_set(mc, (MDBX_val *)key, &old_data, MDBX_SET); + csr_t csr = + /* olddata may not be updated in case DUPFIX-page of dupfix-table */ + cursor_seek(mc, (MDBX_val *)key, &old_data, MDBX_SET); rc = csr.err; exact = csr.exact; } @@ -21507,29 +16875,28 @@ static __hot int cursor_put_nochecklen(MDBX_cursor *mc, const MDBX_val *key, *data = old_data; return MDBX_KEYEXIST; } - if (unlikely(mc->mc_flags & C_SUB)) { + if (unlikely(mc->flags & z_inner)) { /* nested subtree of DUPSORT-database with the same key, * nothing to update */ - eASSERT(env, data->iov_len == 0 && - (old_data.iov_len == 0 || - /* olddata may not be updated in case LEAF2-page - of dupfixed-subDB */ - (mc->mc_db->md_flags & MDBX_DUPFIXED))); + eASSERT(env, data->iov_len == 0 && (old_data.iov_len == 0 || + /* olddata may not be updated in case + DUPFIX-page of dupfix-table */ + (mc->tree->flags & MDBX_DUPFIXED))); return MDBX_SUCCESS; } - if (unlikely(flags & MDBX_ALLDUPS) && mc->mc_xcursor && - (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)) { + if (unlikely(flags & MDBX_ALLDUPS) && inner_pointed(mc)) { err = cursor_del(mc, MDBX_ALLDUPS); if (unlikely(err != MDBX_SUCCESS)) return err; flags -= MDBX_ALLDUPS; - rc = mc->mc_snum ? MDBX_NOTFOUND : MDBX_NO_ROOT; + cASSERT(mc, mc->top + 1 == mc->tree->height); + rc = (mc->top >= 0) ? MDBX_NOTFOUND : MDBX_NO_ROOT; exact = false; } else if (!(flags & (MDBX_RESERVE | MDBX_MULTIPLE))) { /* checking for early exit without dirtying pages */ if (unlikely(eq_fast(data, &old_data))) { - cASSERT(mc, mc->mc_dbx->md_dcmp(data, &old_data) == 0); - if (mc->mc_xcursor) { + cASSERT(mc, mc->clc->v.cmp(data, &old_data) == 0); + if (mc->subcur) { if (flags & MDBX_NODUPDATA) return MDBX_KEYEXIST; if (flags & MDBX_APPENDDUP) @@ -21538,20 +16905,22 @@ static __hot int cursor_put_nochecklen(MDBX_cursor *mc, const MDBX_val *key, /* the same data, nothing to update */ return MDBX_SUCCESS; } - cASSERT(mc, mc->mc_dbx->md_dcmp(data, &old_data) != 0); + cASSERT(mc, mc->clc->v.cmp(data, &old_data) != 0); } } } else if (unlikely(rc != MDBX_NOTFOUND)) return rc; } - mc->mc_flags &= ~C_DEL; + mc->flags &= ~z_after_delete; MDBX_val xdata, *ref_data = data; - size_t *batch_dupfixed_done = nullptr, batch_dupfixed_given = 0; + size_t *batch_dupfix_done = nullptr, batch_dupfix_given = 0; if (unlikely(flags & MDBX_MULTIPLE)) { - batch_dupfixed_given = data[1].iov_len; - batch_dupfixed_done = &data[1].iov_len; - *batch_dupfixed_done = 0; + batch_dupfix_given = data[1].iov_len; + if (unlikely(data[1].iov_len == 0)) + return /* nothing todo */ MDBX_SUCCESS; + batch_dupfix_done = &data[1].iov_len; + *batch_dupfix_done = 0; } /* Cursor is positioned, check for room in the dirty list */ @@ -21565,152 +16934,141 @@ static __hot int cursor_put_nochecklen(MDBX_cursor *mc, const MDBX_val *key, pgr_t npr = page_new(mc, P_LEAF); if (unlikely(npr.err != MDBX_SUCCESS)) return npr.err; - npr.err = cursor_push(mc, npr.page); + npr.err = cursor_push(mc, npr.page, 0); if (unlikely(npr.err != MDBX_SUCCESS)) return npr.err; - mc->mc_db->md_root = npr.page->mp_pgno; - mc->mc_db->md_depth++; - if (mc->mc_db->md_flags & MDBX_INTEGERKEY) { - assert(key->iov_len >= mc->mc_dbx->md_klen_min && - key->iov_len <= mc->mc_dbx->md_klen_max); - mc->mc_dbx->md_klen_min = mc->mc_dbx->md_klen_max = key->iov_len; - } - if (mc->mc_db->md_flags & (MDBX_INTEGERDUP | MDBX_DUPFIXED)) { - assert(data->iov_len >= mc->mc_dbx->md_vlen_min && - data->iov_len <= mc->mc_dbx->md_vlen_max); - assert(mc->mc_xcursor != NULL); - mc->mc_db->md_xsize = mc->mc_xcursor->mx_db.md_xsize = - (unsigned)(mc->mc_dbx->md_vlen_min = mc->mc_dbx->md_vlen_max = - mc->mc_xcursor->mx_dbx.md_klen_min = - mc->mc_xcursor->mx_dbx.md_klen_max = - data->iov_len); - if (mc->mc_flags & C_SUB) - npr.page->mp_flags |= P_LEAF2; - } - mc->mc_flags |= C_INITIALIZED; + mc->tree->root = npr.page->pgno; + mc->tree->height++; + if (mc->tree->flags & MDBX_INTEGERKEY) { + assert(key->iov_len >= mc->clc->k.lmin && key->iov_len <= mc->clc->k.lmax); + mc->clc->k.lmin = mc->clc->k.lmax = key->iov_len; + } + if (mc->tree->flags & (MDBX_INTEGERDUP | MDBX_DUPFIXED)) { + assert(data->iov_len >= mc->clc->v.lmin && data->iov_len <= mc->clc->v.lmax); + assert(mc->subcur != nullptr); + mc->tree->dupfix_size = /* mc->subcur->nested_tree.dupfix_size = */ + (unsigned)(mc->clc->v.lmin = mc->clc->v.lmax = data->iov_len); + cASSERT(mc, mc->clc->v.lmin == mc->subcur->cursor.clc->k.lmin); + cASSERT(mc, mc->clc->v.lmax == mc->subcur->cursor.clc->k.lmax); + if (mc->flags & z_inner) + npr.page->flags |= P_DUPFIX; + } } MDBX_val old_singledup, old_data; - MDBX_db nested_dupdb; - MDBX_page *sub_root = nullptr; + tree_t nested_dupdb; + page_t *sub_root = nullptr; bool insert_key, insert_data; uint16_t fp_flags = P_LEAF; - MDBX_page *fp = env->me_pbuf; - fp->mp_txnid = mc->mc_txn->mt_front; + page_t *fp = env->page_auxbuf; + fp->txnid = mc->txn->front_txnid; insert_key = insert_data = (rc != MDBX_SUCCESS); old_singledup.iov_base = nullptr; + old_singledup.iov_len = 0; if (insert_key) { /* The key does not exist */ - DEBUG("inserting key at index %i", mc->mc_ki[mc->mc_top]); - if ((mc->mc_db->md_flags & MDBX_DUPSORT) && - node_size(key, data) > env->me_leaf_nodemax) { - /* Too big for a node, insert in sub-DB. Set up an empty - * "old sub-page" for convert_to_subtree to expand to a full page. */ - fp->mp_leaf2_ksize = - (mc->mc_db->md_flags & MDBX_DUPFIXED) ? (uint16_t)data->iov_len : 0; - fp->mp_lower = fp->mp_upper = 0; - old_data.iov_len = PAGEHDRSZ; - goto convert_to_subtree; + DEBUG("inserting key at index %i", mc->ki[mc->top]); + if (mc->tree->flags & MDBX_DUPSORT) { + inner_gone(mc); + if (node_size(key, data) > env->leaf_nodemax) { + /* Too big for a node, insert in sub-DB. Set up an empty + * "old sub-page" for convert_to_subtree to expand to a full page. */ + fp->dupfix_ksize = (mc->tree->flags & MDBX_DUPFIXED) ? (uint16_t)data->iov_len : 0; + fp->lower = fp->upper = 0; + old_data.iov_len = PAGEHDRSZ; + goto convert_to_subtree; + } } } else { /* there's only a key anyway, so this is a no-op */ - if (IS_LEAF2(mc->mc_pg[mc->mc_top])) { - size_t ksize = mc->mc_db->md_xsize; + if (is_dupfix_leaf(mc->pg[mc->top])) { + size_t ksize = mc->tree->dupfix_size; if (unlikely(key->iov_len != ksize)) return MDBX_BAD_VALSIZE; - void *ptr = - page_leaf2key(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top], ksize); + void *ptr = page_dupfix_ptr(mc->pg[mc->top], mc->ki[mc->top], ksize); memcpy(ptr, key->iov_base, ksize); fix_parent: /* if overwriting slot 0 of leaf, need to * update branch key if there is a parent page */ - if (mc->mc_top && !mc->mc_ki[mc->mc_top]) { + if (mc->top && !mc->ki[mc->top]) { size_t dtop = 1; - mc->mc_top--; + mc->top--; /* slot 0 is always an empty key, find real slot */ - while (mc->mc_top && !mc->mc_ki[mc->mc_top]) { - mc->mc_top--; + while (mc->top && !mc->ki[mc->top]) { + mc->top--; dtop++; } err = MDBX_SUCCESS; - if (mc->mc_ki[mc->mc_top]) - err = update_key(mc, key); - cASSERT(mc, mc->mc_top + dtop < UINT16_MAX); - mc->mc_top += (uint8_t)dtop; + if (mc->ki[mc->top]) + err = tree_propagate_key(mc, key); + cASSERT(mc, mc->top + dtop < UINT16_MAX); + mc->top += (uint8_t)dtop; if (unlikely(err != MDBX_SUCCESS)) return err; } if (AUDIT_ENABLED()) { - err = cursor_check(mc); + err = cursor_validate(mc); if (unlikely(err != MDBX_SUCCESS)) return err; } return MDBX_SUCCESS; } - more:; + more: if (AUDIT_ENABLED()) { - err = cursor_check(mc); + err = cursor_validate(mc); if (unlikely(err != MDBX_SUCCESS)) return err; } - MDBX_node *const node = - page_node(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + node_t *const node = page_node(mc->pg[mc->top], mc->ki[mc->top]); /* Large/Overflow page overwrites need special handling */ - if (unlikely(node_flags(node) & F_BIGDATA)) { - const size_t dpages = (node_size(key, data) > env->me_leaf_nodemax) - ? number_of_ovpages(env, data->iov_len) - : 0; + if (unlikely(node_flags(node) & N_BIG)) { + const size_t dpages = (node_size(key, data) > env->leaf_nodemax) ? largechunk_npages(env, data->iov_len) : 0; const pgno_t pgno = node_largedata_pgno(node); - pgr_t lp = page_get_large(mc, pgno, mc->mc_pg[mc->mc_top]->mp_txnid); + pgr_t lp = page_get_large(mc, pgno, mc->pg[mc->top]->txnid); if (unlikely(lp.err != MDBX_SUCCESS)) return lp.err; - cASSERT(mc, PAGETYPE_WHOLE(lp.page) == P_OVERFLOW); + cASSERT(mc, page_type(lp.page) == P_LARGE); /* Is the ov page from this txn (or a parent) and big enough? */ - const size_t ovpages = lp.page->mp_pages; + const size_t ovpages = lp.page->pages; const size_t extra_threshold = - (mc->mc_dbi == FREE_DBI) - ? 1 - : /* LY: add configurable threshold to keep reserve space */ 0; - if (!IS_FROZEN(mc->mc_txn, lp.page) && ovpages >= dpages && - ovpages <= dpages + extra_threshold) { + (mc->tree == &mc->txn->dbs[FREE_DBI]) ? 1 : /* LY: add configurable threshold to keep reserve space */ 0; + if (!is_frozen(mc->txn, lp.page) && ovpages >= dpages && ovpages <= dpages + extra_threshold) { /* yes, overwrite it. */ - if (!IS_MODIFIABLE(mc->mc_txn, lp.page)) { - if (IS_SPILLED(mc->mc_txn, lp.page)) { + if (!is_modifable(mc->txn, lp.page)) { + if (is_spilled(mc->txn, lp.page)) { lp = /* TODO: avoid search and get txn & spill-index from page_result */ - page_unspill(mc->mc_txn, lp.page); + page_unspill(mc->txn, lp.page); if (unlikely(lp.err)) return lp.err; } else { - if (unlikely(!mc->mc_txn->mt_parent)) { + if (unlikely(!mc->txn->parent)) { ERROR("Unexpected not frozen/modifiable/spilled but shadowed %s " "page %" PRIaPGNO " mod-txnid %" PRIaTXN "," - " without parent transaction, current txn %" PRIaTXN - " front %" PRIaTXN, - "overflow/large", pgno, lp.page->mp_txnid, - mc->mc_txn->mt_txnid, mc->mc_txn->mt_front); + " without parent transaction, current txn %" PRIaTXN " front %" PRIaTXN, + "large/overflow", pgno, lp.page->txnid, mc->txn->txnid, mc->txn->front_txnid); return MDBX_PROBLEM; } /* It is writable only in a parent txn */ - MDBX_page *np = page_malloc(mc->mc_txn, ovpages); + page_t *np = page_shadow_alloc(mc->txn, ovpages); if (unlikely(!np)) return MDBX_ENOMEM; memcpy(np, lp.page, PAGEHDRSZ); /* Copy header of page */ - err = page_dirty(mc->mc_txn, lp.page = np, ovpages); + err = page_dirty(mc->txn, lp.page = np, ovpages); if (unlikely(err != MDBX_SUCCESS)) return err; #if MDBX_ENABLE_PGOP_STAT - mc->mc_txn->mt_env->me_lck->mti_pgop_stat.clone.weak += ovpages; + mc->txn->env->lck->pgops.clone.weak += ovpages; #endif /* MDBX_ENABLE_PGOP_STAT */ - cASSERT(mc, dirtylist_check(mc->mc_txn)); + cASSERT(mc, dpl_check(mc->txn)); } } node_set_ds(node, data->iov_len); @@ -21720,7 +17078,7 @@ static __hot int cursor_put_nochecklen(MDBX_cursor *mc, const MDBX_val *key, memcpy(page_data(lp.page), data->iov_base, data->iov_len); if (AUDIT_ENABLED()) { - err = cursor_check(mc); + err = cursor_validate(mc); if (unlikely(err != MDBX_SUCCESS)) return err; } @@ -21732,83 +17090,78 @@ static __hot int cursor_put_nochecklen(MDBX_cursor *mc, const MDBX_val *key, } else { old_data.iov_len = node_ds(node); old_data.iov_base = node_data(node); - cASSERT(mc, ptr_disp(old_data.iov_base, old_data.iov_len) <= - ptr_disp(mc->mc_pg[mc->mc_top], env->me_psize)); + cASSERT(mc, ptr_disp(old_data.iov_base, old_data.iov_len) <= ptr_disp(mc->pg[mc->top], env->ps)); /* DB has dups? */ - if (mc->mc_db->md_flags & MDBX_DUPSORT) { + if (mc->tree->flags & MDBX_DUPSORT) { /* Prepare (sub-)page/sub-DB to accept the new item, if needed. * fp: old sub-page or a header faking it. * mp: new (sub-)page. * xdata: node data with new sub-page or sub-DB. */ size_t growth = 0; /* growth in page size.*/ - MDBX_page *mp = fp = xdata.iov_base = env->me_pbuf; - mp->mp_pgno = mc->mc_pg[mc->mc_top]->mp_pgno; + page_t *mp = fp = xdata.iov_base = env->page_auxbuf; + mp->pgno = mc->pg[mc->top]->pgno; /* Was a single item before, must convert now */ - if (!(node_flags(node) & F_DUPDATA)) { + if (!(node_flags(node) & N_DUP)) { /* does data match? */ if (flags & MDBX_APPENDDUP) { - const int cmp = mc->mc_dbx->md_dcmp(data, &old_data); + const int cmp = mc->clc->v.cmp(data, &old_data); cASSERT(mc, cmp != 0 || eq_fast(data, &old_data)); if (unlikely(cmp <= 0)) return MDBX_EKEYMISMATCH; } else if (eq_fast(data, &old_data)) { - cASSERT(mc, mc->mc_dbx->md_dcmp(data, &old_data) == 0); + cASSERT(mc, mc->clc->v.cmp(data, &old_data) == 0); if (flags & MDBX_NODUPDATA) return MDBX_KEYEXIST; /* data is match exactly byte-to-byte, nothing to update */ rc = MDBX_SUCCESS; - if (unlikely(batch_dupfixed_done)) - goto batch_dupfixed_continue; + if (unlikely(batch_dupfix_done)) + goto batch_dupfix_continue; return rc; } /* Just overwrite the current item */ if (flags & MDBX_CURRENT) { - cASSERT(mc, node_size(key, data) <= env->me_leaf_nodemax); + cASSERT(mc, node_size(key, data) <= env->leaf_nodemax); goto current; } /* Back up original data item */ - memcpy(old_singledup.iov_base = fp + 1, old_data.iov_base, - old_singledup.iov_len = old_data.iov_len); + memcpy(old_singledup.iov_base = fp + 1, old_data.iov_base, old_singledup.iov_len = old_data.iov_len); /* Make sub-page header for the dup items, with dummy body */ - fp->mp_flags = P_LEAF | P_SUBP; - fp->mp_lower = 0; + fp->flags = P_LEAF | P_SUBP; + fp->lower = 0; xdata.iov_len = PAGEHDRSZ + old_data.iov_len + data->iov_len; - if (mc->mc_db->md_flags & MDBX_DUPFIXED) { - fp->mp_flags |= P_LEAF2; - fp->mp_leaf2_ksize = (uint16_t)data->iov_len; - /* Будем создавать LEAF2-страницу, как минимум с двумя элементами. + if (mc->tree->flags & MDBX_DUPFIXED) { + fp->flags |= P_DUPFIX; + fp->dupfix_ksize = (uint16_t)data->iov_len; + /* Будем создавать DUPFIX-страницу, как минимум с двумя элементами. * При коротких значениях и наличии свободного места можно сделать * некоторое резервирование места, чтобы при последующих добавлениях * не сразу расширять созданную под-страницу. * Резервирование в целом сомнительно (см ниже), но может сработать * в плюс (а если в минус то несущественный) при коротких ключах. */ - xdata.iov_len += leaf2_reserve( - env, page_room(mc->mc_pg[mc->mc_top]) + old_data.iov_len, - xdata.iov_len, data->iov_len); + xdata.iov_len += + page_subleaf2_reserve(env, page_room(mc->pg[mc->top]) + old_data.iov_len, xdata.iov_len, data->iov_len); cASSERT(mc, (xdata.iov_len & 1) == 0); } else { - xdata.iov_len += 2 * (sizeof(indx_t) + NODESIZE) + - (old_data.iov_len & 1) + (data->iov_len & 1); + xdata.iov_len += 2 * (sizeof(indx_t) + NODESIZE) + (old_data.iov_len & 1) + (data->iov_len & 1); } cASSERT(mc, (xdata.iov_len & 1) == 0); - fp->mp_upper = (uint16_t)(xdata.iov_len - PAGEHDRSZ); + fp->upper = (uint16_t)(xdata.iov_len - PAGEHDRSZ); old_data.iov_len = xdata.iov_len; /* pretend olddata is fp */ - } else if (node_flags(node) & F_SUBDATA) { + } else if (node_flags(node) & N_TREE) { /* Data is on sub-DB, just store it */ - flags |= F_DUPDATA | F_SUBDATA; + flags |= N_DUP | N_TREE; goto dupsort_put; } else { /* Data is on sub-page */ fp = old_data.iov_base; switch (flags) { default: - growth = IS_LEAF2(fp) ? fp->mp_leaf2_ksize - : (node_size(data, nullptr) + sizeof(indx_t)); + growth = is_dupfix_leaf(fp) ? fp->dupfix_ksize : (node_size(data, nullptr) + sizeof(indx_t)); if (page_room(fp) >= growth) { /* На текущей под-странице есть место для добавления элемента. * Оптимальнее продолжить использовать эту страницу, ибо @@ -21821,10 +17174,10 @@ static __hot int cursor_put_nochecklen(MDBX_cursor *mc, const MDBX_val *key, * * Продолжать использовать текущую под-страницу возможно * только пока и если размер после добавления элемента будет - * меньше me_leaf_nodemax. Соответственно, при превышении + * меньше leaf_nodemax. Соответственно, при превышении * просто сразу переходим на вложенное дерево. */ xdata.iov_len = old_data.iov_len + (growth += growth & 1); - if (xdata.iov_len > env->me_subpage_limit) + if (xdata.iov_len > env->subpage_limit) goto convert_to_subtree; /* Можно либо увеличить под-страницу, в том числе с некоторым @@ -21841,7 +17194,7 @@ static __hot int cursor_put_nochecklen(MDBX_cursor *mc, const MDBX_val *key, * размера под-страницы, её тело будет примыкать * к неиспользуемому месту на основной/гнездовой странице, * поэтому последующие последовательные добавления потребуют - * только передвижения в mp_ptrs[]. + * только передвижения в entries[]. * * Соответственно, более важным/определяющим представляется * своевременный переход к вложеному дереву, но тут достаточно @@ -21864,100 +17217,95 @@ static __hot int cursor_put_nochecklen(MDBX_cursor *mc, const MDBX_val *key, * Суммарно наиболее рациональным представляется такая тактика: * - Вводим три порога subpage_limit, subpage_room_threshold * и subpage_reserve_prereq, которые могут быть - * заданы/скорректированы пользователем в ‰ от me_leaf_nodemax; + * заданы/скорректированы пользователем в ‰ от leaf_nodemax; * - Используем под-страницу пока её размер меньше subpage_limit * и на основной/гнездовой странице не-менее * subpage_room_threshold свободного места; - * - Резервируем место только для 1-3 коротких dupfixed-элементов, + * - Резервируем место только для 1-3 коротких dupfix-элементов, * расширяя размер под-страницы на размер кэш-линии ЦПУ, но * только если на странице не менее subpage_reserve_prereq * свободного места. * - По-умолчанию устанавливаем: - * subpage_limit = me_leaf_nodemax (1000‰); + * subpage_limit = leaf_nodemax (1000‰); * subpage_room_threshold = 0; - * subpage_reserve_prereq = me_leaf_nodemax (1000‰). + * subpage_reserve_prereq = leaf_nodemax (1000‰). */ - if (IS_LEAF2(fp)) - growth += leaf2_reserve( - env, page_room(mc->mc_pg[mc->mc_top]) + old_data.iov_len, - xdata.iov_len, data->iov_len); + if (is_dupfix_leaf(fp)) + growth += page_subleaf2_reserve(env, page_room(mc->pg[mc->top]) + old_data.iov_len, xdata.iov_len, + data->iov_len); + else { + /* TODO: Если добавить возможность для пользователя задавать + * min/max размеров ключей/данных, то здесь разумно реализовать + * тактику резервирования подобную dupfixed. */ + } break; case MDBX_CURRENT | MDBX_NODUPDATA: case MDBX_CURRENT: continue_subpage: - fp->mp_txnid = mc->mc_txn->mt_front; - fp->mp_pgno = mp->mp_pgno; - mc->mc_xcursor->mx_cursor.mc_pg[0] = fp; - flags |= F_DUPDATA; + fp->txnid = mc->txn->front_txnid; + fp->pgno = mp->pgno; + mc->subcur->cursor.pg[0] = fp; + flags |= N_DUP; goto dupsort_put; } xdata.iov_len = old_data.iov_len + growth; cASSERT(mc, (xdata.iov_len & 1) == 0); } - fp_flags = fp->mp_flags; - if (xdata.iov_len > env->me_subpage_limit || - node_size_len(node_ks(node), xdata.iov_len) > - env->me_leaf_nodemax || - (env->me_subpage_room_threshold && - page_room(mc->mc_pg[mc->mc_top]) + - node_size_len(node_ks(node), old_data.iov_len) < - env->me_subpage_room_threshold + - node_size_len(node_ks(node), xdata.iov_len))) { + fp_flags = fp->flags; + if (xdata.iov_len > env->subpage_limit || node_size_len(node_ks(node), xdata.iov_len) > env->leaf_nodemax || + (env->subpage_room_threshold && + page_room(mc->pg[mc->top]) + node_size_len(node_ks(node), old_data.iov_len) < + env->subpage_room_threshold + node_size_len(node_ks(node), xdata.iov_len))) { /* Too big for a sub-page, convert to sub-DB */ convert_to_subtree: fp_flags &= ~P_SUBP; - nested_dupdb.md_xsize = 0; - nested_dupdb.md_flags = flags_db2sub(mc->mc_db->md_flags); - if (mc->mc_db->md_flags & MDBX_DUPFIXED) { - fp_flags |= P_LEAF2; - nested_dupdb.md_xsize = fp->mp_leaf2_ksize; + nested_dupdb.dupfix_size = 0; + nested_dupdb.flags = flags_db2sub(mc->tree->flags); + if (mc->tree->flags & MDBX_DUPFIXED) { + fp_flags |= P_DUPFIX; + nested_dupdb.dupfix_size = fp->dupfix_ksize; } - nested_dupdb.md_depth = 1; - nested_dupdb.md_branch_pages = 0; - nested_dupdb.md_leaf_pages = 1; - nested_dupdb.md_overflow_pages = 0; - nested_dupdb.md_entries = page_numkeys(fp); + nested_dupdb.height = 1; + nested_dupdb.branch_pages = 0; + nested_dupdb.leaf_pages = 1; + nested_dupdb.large_pages = 0; + nested_dupdb.items = page_numkeys(fp); xdata.iov_len = sizeof(nested_dupdb); xdata.iov_base = &nested_dupdb; - const pgr_t par = page_alloc(mc); + const pgr_t par = gc_alloc_single(mc); mp = par.page; if (unlikely(par.err != MDBX_SUCCESS)) return par.err; - mc->mc_db->md_leaf_pages += 1; - cASSERT(mc, env->me_psize > old_data.iov_len); - growth = env->me_psize - (unsigned)old_data.iov_len; + mc->tree->leaf_pages += 1; + cASSERT(mc, env->ps > old_data.iov_len); + growth = env->ps - (unsigned)old_data.iov_len; cASSERT(mc, (growth & 1) == 0); - flags |= F_DUPDATA | F_SUBDATA; - nested_dupdb.md_root = mp->mp_pgno; - nested_dupdb.md_seq = 0; - nested_dupdb.md_mod_txnid = mc->mc_txn->mt_txnid; + flags |= N_DUP | N_TREE; + nested_dupdb.root = mp->pgno; + nested_dupdb.sequence = 0; + nested_dupdb.mod_txnid = mc->txn->txnid; sub_root = mp; } if (mp != fp) { - mp->mp_flags = fp_flags; - mp->mp_txnid = mc->mc_txn->mt_front; - mp->mp_leaf2_ksize = fp->mp_leaf2_ksize; - mp->mp_lower = fp->mp_lower; - cASSERT(mc, fp->mp_upper + growth < UINT16_MAX); - mp->mp_upper = fp->mp_upper + (indx_t)growth; - if (unlikely(fp_flags & P_LEAF2)) { - memcpy(page_data(mp), page_data(fp), - page_numkeys(fp) * fp->mp_leaf2_ksize); - cASSERT(mc, - (((mp->mp_leaf2_ksize & page_numkeys(mp)) ^ mp->mp_upper) & - 1) == 0); + mp->flags = fp_flags; + mp->txnid = mc->txn->front_txnid; + mp->dupfix_ksize = fp->dupfix_ksize; + mp->lower = fp->lower; + cASSERT(mc, fp->upper + growth < UINT16_MAX); + mp->upper = fp->upper + (indx_t)growth; + if (unlikely(fp_flags & P_DUPFIX)) { + memcpy(page_data(mp), page_data(fp), page_numkeys(fp) * fp->dupfix_ksize); + cASSERT(mc, (((mp->dupfix_ksize & page_numkeys(mp)) ^ mp->upper) & 1) == 0); } else { - cASSERT(mc, (mp->mp_upper & 1) == 0); - memcpy(ptr_disp(mp, mp->mp_upper + PAGEHDRSZ), - ptr_disp(fp, fp->mp_upper + PAGEHDRSZ), - old_data.iov_len - fp->mp_upper - PAGEHDRSZ); - memcpy(mp->mp_ptrs, fp->mp_ptrs, - page_numkeys(fp) * sizeof(mp->mp_ptrs[0])); + cASSERT(mc, (mp->upper & 1) == 0); + memcpy(ptr_disp(mp, mp->upper + PAGEHDRSZ), ptr_disp(fp, fp->upper + PAGEHDRSZ), + old_data.iov_len - fp->upper - PAGEHDRSZ); + memcpy(mp->entries, fp->entries, page_numkeys(fp) * sizeof(mp->entries[0])); for (size_t i = 0; i < page_numkeys(fp); i++) { - cASSERT(mc, mp->mp_ptrs[i] + growth <= UINT16_MAX); - mp->mp_ptrs[i] += (indx_t)growth; + cASSERT(mc, mp->entries[i] + growth <= UINT16_MAX); + mp->entries[i] += (indx_t)growth; } } } @@ -21965,39 +17313,38 @@ static __hot int cursor_put_nochecklen(MDBX_cursor *mc, const MDBX_val *key, if (!insert_key) node_del(mc, 0); ref_data = &xdata; - flags |= F_DUPDATA; + flags |= N_DUP; goto insert_node; } - /* MDBX passes F_SUBDATA in 'flags' to write a DB record */ - if (unlikely((node_flags(node) ^ flags) & F_SUBDATA)) + /* MDBX passes N_TREE in 'flags' to write a DB record */ + if (unlikely((node_flags(node) ^ flags) & N_TREE)) return MDBX_INCOMPATIBLE; current: if (data->iov_len == old_data.iov_len) { - cASSERT(mc, EVEN(key->iov_len) == EVEN(node_ks(node))); + cASSERT(mc, EVEN_CEIL(key->iov_len) == EVEN_CEIL(node_ks(node))); /* same size, just replace it. Note that we could * also reuse this node if the new data is smaller, * but instead we opt to shrink the node in that case. */ if (flags & MDBX_RESERVE) data->iov_base = old_data.iov_base; - else if (!(mc->mc_flags & C_SUB)) + else if (!(mc->flags & z_inner)) memcpy(old_data.iov_base, data->iov_base, data->iov_len); else { - cASSERT(mc, page_numkeys(mc->mc_pg[mc->mc_top]) == 1); - cASSERT(mc, PAGETYPE_COMPAT(mc->mc_pg[mc->mc_top]) == P_LEAF); + cASSERT(mc, page_numkeys(mc->pg[mc->top]) == 1); + cASSERT(mc, page_type_compat(mc->pg[mc->top]) == P_LEAF); cASSERT(mc, node_ds(node) == 0); cASSERT(mc, node_flags(node) == 0); cASSERT(mc, key->iov_len < UINT16_MAX); node_set_ks(node, key->iov_len); memcpy(node_key(node), key->iov_base, key->iov_len); - cASSERT(mc, ptr_disp(node_key(node), node_ds(node)) < - ptr_disp(mc->mc_pg[mc->mc_top], env->me_psize)); + cASSERT(mc, ptr_disp(node_key(node), node_ds(node)) < ptr_disp(mc->pg[mc->top], env->ps)); goto fix_parent; } if (AUDIT_ENABLED()) { - err = cursor_check(mc); + err = cursor_validate(mc); if (unlikely(err != MDBX_SUCCESS)) return err; } @@ -22011,37 +17358,30 @@ static __hot int cursor_put_nochecklen(MDBX_cursor *mc, const MDBX_val *key, insert_node:; const unsigned naf = flags & NODE_ADD_FLAGS; - size_t nsize = IS_LEAF2(mc->mc_pg[mc->mc_top]) - ? key->iov_len - : leaf_size(env, key, ref_data); - if (page_room(mc->mc_pg[mc->mc_top]) < nsize) { - rc = page_split(mc, key, ref_data, P_INVALID, - insert_key ? naf : naf | MDBX_SPLIT_REPLACE); + size_t nsize = is_dupfix_leaf(mc->pg[mc->top]) ? key->iov_len : leaf_size(env, key, ref_data); + if (page_room(mc->pg[mc->top]) < nsize) { + rc = page_split(mc, key, ref_data, P_INVALID, insert_key ? naf : naf | MDBX_SPLIT_REPLACE); if (rc == MDBX_SUCCESS && AUDIT_ENABLED()) - rc = insert_key ? cursor_check(mc) : cursor_check_updating(mc); + rc = insert_key ? cursor_validate(mc) : cursor_validate_updating(mc); } else { /* There is room already in this leaf page. */ - if (IS_LEAF2(mc->mc_pg[mc->mc_top])) { - cASSERT(mc, !(naf & (F_BIGDATA | F_SUBDATA | F_DUPDATA)) && - ref_data->iov_len == 0); - rc = node_add_leaf2(mc, mc->mc_ki[mc->mc_top], key); + if (is_dupfix_leaf(mc->pg[mc->top])) { + cASSERT(mc, !(naf & (N_BIG | N_TREE | N_DUP)) && ref_data->iov_len == 0); + rc = node_add_dupfix(mc, mc->ki[mc->top], key); } else - rc = node_add_leaf(mc, mc->mc_ki[mc->mc_top], key, ref_data, naf); + rc = node_add_leaf(mc, mc->ki[mc->top], key, ref_data, naf); if (likely(rc == 0)) { /* Adjust other cursors pointing to mp */ - const MDBX_dbi dbi = mc->mc_dbi; - const size_t top = mc->mc_top; - MDBX_page *const mp = mc->mc_pg[top]; - for (MDBX_cursor *m2 = mc->mc_txn->mt_cursors[dbi]; m2; - m2 = m2->mc_next) { - MDBX_cursor *m3 = - (mc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2; - if (m3 == mc || m3->mc_snum < mc->mc_snum || m3->mc_pg[top] != mp) + page_t *const mp = mc->pg[mc->top]; + const size_t dbi = cursor_dbi(mc); + for (MDBX_cursor *m2 = mc->txn->cursors[dbi]; m2; m2 = m2->next) { + MDBX_cursor *m3 = (mc->flags & z_inner) ? &m2->subcur->cursor : m2; + if (!is_related(mc, m3) || m3->pg[mc->top] != mp) continue; - if (m3->mc_ki[top] >= mc->mc_ki[top]) - m3->mc_ki[top] += insert_key; - if (XCURSOR_INITED(m3)) - XCURSOR_REFRESH(m3, mp, m3->mc_ki[top]); + if (m3->ki[mc->top] >= mc->ki[mc->top]) + m3->ki[mc->top] += insert_key; + if (inner_pointed(m3)) + cursor_inner_refresh(m3, mp, m3->ki[mc->top]); } } } @@ -22051,96 +17391,101 @@ insert_node:; * storing the user data in the keys field, so there are strict * size limits on dupdata. The actual data fields of the child * DB are all zero size. */ - if (flags & F_DUPDATA) { + if (flags & N_DUP) { MDBX_val empty; dupsort_put: empty.iov_len = 0; empty.iov_base = nullptr; - MDBX_node *node = page_node(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + node_t *node = page_node(mc->pg[mc->top], mc->ki[mc->top]); #define SHIFT_MDBX_NODUPDATA_TO_MDBX_NOOVERWRITE 1 - STATIC_ASSERT( - (MDBX_NODUPDATA >> SHIFT_MDBX_NODUPDATA_TO_MDBX_NOOVERWRITE) == - MDBX_NOOVERWRITE); - unsigned xflags = - MDBX_CURRENT | ((flags & MDBX_NODUPDATA) >> - SHIFT_MDBX_NODUPDATA_TO_MDBX_NOOVERWRITE); + STATIC_ASSERT((MDBX_NODUPDATA >> SHIFT_MDBX_NODUPDATA_TO_MDBX_NOOVERWRITE) == MDBX_NOOVERWRITE); + unsigned inner_flags = MDBX_CURRENT | ((flags & MDBX_NODUPDATA) >> SHIFT_MDBX_NODUPDATA_TO_MDBX_NOOVERWRITE); if ((flags & MDBX_CURRENT) == 0) { - xflags -= MDBX_CURRENT; - err = cursor_xinit1(mc, node, mc->mc_pg[mc->mc_top]); - if (unlikely(err != MDBX_SUCCESS)) - return err; + inner_flags -= MDBX_CURRENT; + rc = cursor_dupsort_setup(mc, node, mc->pg[mc->top]); + if (unlikely(rc != MDBX_SUCCESS)) + goto dupsort_error; + } + subcur_t *const mx = mc->subcur; + if (sub_root) { + cASSERT(mc, mx->nested_tree.height == 1 && mx->nested_tree.root == sub_root->pgno); + mx->cursor.flags = z_inner; + mx->cursor.top = 0; + mx->cursor.pg[0] = sub_root; + mx->cursor.ki[0] = 0; } - if (sub_root) - mc->mc_xcursor->mx_cursor.mc_pg[0] = sub_root; - /* converted, write the original data first */ if (old_singledup.iov_base) { - rc = cursor_put_nochecklen(&mc->mc_xcursor->mx_cursor, &old_singledup, - &empty, xflags); - if (unlikely(rc)) + /* converted, write the original data first */ + if (is_dupfix_leaf(mx->cursor.pg[0])) + rc = node_add_dupfix(&mx->cursor, 0, &old_singledup); + else + rc = node_add_leaf(&mx->cursor, 0, &old_singledup, &empty, 0); + if (unlikely(rc != MDBX_SUCCESS)) goto dupsort_error; + mx->cursor.tree->items = 1; } - if (!(node_flags(node) & F_SUBDATA) || sub_root) { - /* Adjust other cursors pointing to mp */ - MDBX_xcursor *const mx = mc->mc_xcursor; - const size_t top = mc->mc_top; - MDBX_page *const mp = mc->mc_pg[top]; + if (!(node_flags(node) & N_TREE) || sub_root) { + page_t *const mp = mc->pg[mc->top]; const intptr_t nkeys = page_numkeys(mp); + const size_t dbi = cursor_dbi(mc); - for (MDBX_cursor *m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; - m2 = m2->mc_next) { - if (m2 == mc || m2->mc_snum < mc->mc_snum) - continue; - if (!(m2->mc_flags & C_INITIALIZED)) + for (MDBX_cursor *m2 = mc->txn->cursors[dbi]; m2; m2 = m2->next) { + if (!is_related(mc, m2) || m2->pg[mc->top] != mp) continue; - if (m2->mc_pg[top] == mp) { - if (m2->mc_ki[top] == mc->mc_ki[top]) { - err = cursor_xinit2(m2, mx, old_singledup.iov_base != nullptr); - if (unlikely(err != MDBX_SUCCESS)) - return err; - } else if (!insert_key && m2->mc_ki[top] < nkeys) { - XCURSOR_REFRESH(m2, mp, m2->mc_ki[top]); + if (/* пропускаем незаполненные курсоры, иначе получится что у такого + курсора будет инициализирован вложенный, + что антилогично и бесполезно. */ + is_filled(m2) && m2->ki[mc->top] == mc->ki[mc->top]) { + cASSERT(m2, m2->subcur->cursor.clc == mx->cursor.clc); + m2->subcur->nested_tree = mx->nested_tree; + m2->subcur->cursor.pg[0] = mx->cursor.pg[0]; + if (old_singledup.iov_base) { + m2->subcur->cursor.top_and_flags = z_inner; + m2->subcur->cursor.ki[0] = 0; } - } + DEBUG("Sub-dbi -%zu root page %" PRIaPGNO, cursor_dbi(&m2->subcur->cursor), m2->subcur->nested_tree.root); + } else if (!insert_key && m2->ki[mc->top] < nkeys) + cursor_inner_refresh(m2, mp, m2->ki[mc->top]); } } - cASSERT(mc, mc->mc_xcursor->mx_db.md_entries < PTRDIFF_MAX); - const size_t probe = (size_t)mc->mc_xcursor->mx_db.md_entries; + cASSERT(mc, mc->subcur->nested_tree.items < PTRDIFF_MAX); + const size_t probe = (size_t)mc->subcur->nested_tree.items; #define SHIFT_MDBX_APPENDDUP_TO_MDBX_APPEND 1 - STATIC_ASSERT((MDBX_APPENDDUP >> SHIFT_MDBX_APPENDDUP_TO_MDBX_APPEND) == - MDBX_APPEND); - xflags |= (flags & MDBX_APPENDDUP) >> SHIFT_MDBX_APPENDDUP_TO_MDBX_APPEND; - rc = cursor_put_nochecklen(&mc->mc_xcursor->mx_cursor, data, &empty, - xflags); - if (flags & F_SUBDATA) { + STATIC_ASSERT((MDBX_APPENDDUP >> SHIFT_MDBX_APPENDDUP_TO_MDBX_APPEND) == MDBX_APPEND); + inner_flags |= (flags & MDBX_APPENDDUP) >> SHIFT_MDBX_APPENDDUP_TO_MDBX_APPEND; + rc = cursor_put(&mc->subcur->cursor, data, &empty, inner_flags); + if (flags & N_TREE) { void *db = node_data(node); - mc->mc_xcursor->mx_db.md_mod_txnid = mc->mc_txn->mt_txnid; - memcpy(db, &mc->mc_xcursor->mx_db, sizeof(MDBX_db)); + mc->subcur->nested_tree.mod_txnid = mc->txn->txnid; + memcpy(db, &mc->subcur->nested_tree, sizeof(tree_t)); } - insert_data = (probe != (size_t)mc->mc_xcursor->mx_db.md_entries); + insert_data = (probe != (size_t)mc->subcur->nested_tree.items); } /* Increment count unless we just replaced an existing item. */ if (insert_data) - mc->mc_db->md_entries++; + mc->tree->items++; if (insert_key) { if (unlikely(rc != MDBX_SUCCESS)) goto dupsort_error; /* If we succeeded and the key didn't exist before, * make sure the cursor is marked valid. */ - mc->mc_flags |= C_INITIALIZED; + be_filled(mc); } if (likely(rc == MDBX_SUCCESS)) { - if (unlikely(batch_dupfixed_done)) { - batch_dupfixed_continue: + cASSERT(mc, is_filled(mc)); + if (unlikely(batch_dupfix_done)) { + batch_dupfix_continue: /* let caller know how many succeeded, if any */ - if ((*batch_dupfixed_done += 1) < batch_dupfixed_given) { + if ((*batch_dupfix_done += 1) < batch_dupfix_given) { data[0].iov_base = ptr_disp(data[0].iov_base, data[0].iov_len); insert_key = insert_data = false; old_singledup.iov_base = nullptr; + sub_root = nullptr; goto more; } } if (AUDIT_ENABLED()) - rc = cursor_check(mc); + rc = cursor_validate(mc); } return rc; @@ -22151,13422 +17496,19977 @@ insert_node:; rc = MDBX_PROBLEM; } } - mc->mc_txn->mt_flags |= MDBX_TXN_ERROR; + mc->txn->flags |= MDBX_TXN_ERROR; return rc; } -static __hot int cursor_put_checklen(MDBX_cursor *mc, const MDBX_val *key, - MDBX_val *data, unsigned flags) { - cASSERT(mc, (mc->mc_flags & C_SUB) == 0); - uint64_t aligned_keybytes, aligned_databytes; - MDBX_val aligned_key, aligned_data; - if (unlikely(key->iov_len < mc->mc_dbx->md_klen_min || - key->iov_len > mc->mc_dbx->md_klen_max)) { +int cursor_check_multiple(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, unsigned flags) { + (void)key; + if (unlikely(flags & MDBX_RESERVE)) + return MDBX_EINVAL; + if (unlikely(!(mc->tree->flags & MDBX_DUPFIXED))) + return MDBX_INCOMPATIBLE; + const size_t number = data[1].iov_len; + if (unlikely(number > MAX_MAPSIZE / 2 / (BRANCH_NODE_MAX(MDBX_MAX_PAGESIZE) - NODESIZE))) { + /* checking for multiplication overflow */ + if (unlikely(number > MAX_MAPSIZE / 2 / data->iov_len)) + return MDBX_TOO_LARGE; + } + return MDBX_SUCCESS; +} + +__hot int cursor_put_checklen(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, unsigned flags) { + cASSERT(mc, (mc->flags & z_inner) == 0); + if (unlikely(key->iov_len > mc->clc->k.lmax || key->iov_len < mc->clc->k.lmin)) { cASSERT(mc, !"Invalid key-size"); return MDBX_BAD_VALSIZE; } - if (unlikely(data->iov_len < mc->mc_dbx->md_vlen_min || - data->iov_len > mc->mc_dbx->md_vlen_max)) { + if (unlikely(data->iov_len > mc->clc->v.lmax || data->iov_len < mc->clc->v.lmin)) { cASSERT(mc, !"Invalid data-size"); return MDBX_BAD_VALSIZE; } - if (mc->mc_db->md_flags & MDBX_INTEGERKEY) { - switch (key->iov_len) { - default: - cASSERT(mc, !"key-size is invalid for MDBX_INTEGERKEY"); - return MDBX_BAD_VALSIZE; - case 4: - if (unlikely(3 & (uintptr_t)key->iov_base)) { + uint64_t aligned_keybytes, aligned_databytes; + MDBX_val aligned_key, aligned_data; + if (mc->tree->flags & MDBX_INTEGERKEY) { + if (key->iov_len == 8) { + if (unlikely(7 & (uintptr_t)key->iov_base)) { /* copy instead of return error to avoid break compatibility */ - aligned_key.iov_base = - memcpy(&aligned_keybytes, key->iov_base, aligned_key.iov_len = 4); + aligned_key.iov_base = bcopy_8(&aligned_keybytes, key->iov_base); + aligned_key.iov_len = key->iov_len; key = &aligned_key; } - break; - case 8: - if (unlikely(7 & (uintptr_t)key->iov_base)) { + } else if (key->iov_len == 4) { + if (unlikely(3 & (uintptr_t)key->iov_base)) { /* copy instead of return error to avoid break compatibility */ - aligned_key.iov_base = - memcpy(&aligned_keybytes, key->iov_base, aligned_key.iov_len = 8); + aligned_key.iov_base = bcopy_4(&aligned_keybytes, key->iov_base); + aligned_key.iov_len = key->iov_len; key = &aligned_key; } - break; + } else { + cASSERT(mc, !"key-size is invalid for MDBX_INTEGERKEY"); + return MDBX_BAD_VALSIZE; } } - if (mc->mc_db->md_flags & MDBX_INTEGERDUP) { - switch (data->iov_len) { - default: - cASSERT(mc, !"data-size is invalid for MDBX_INTEGERKEY"); - return MDBX_BAD_VALSIZE; - case 4: - if (unlikely(3 & (uintptr_t)data->iov_base)) { - if (unlikely(flags & MDBX_MULTIPLE)) - return MDBX_BAD_VALSIZE; - /* copy instead of return error to avoid break compatibility */ - aligned_data.iov_base = memcpy(&aligned_databytes, data->iov_base, - aligned_data.iov_len = 4); - data = &aligned_data; - } - break; - case 8: + if (mc->tree->flags & MDBX_INTEGERDUP) { + if (data->iov_len == 8) { if (unlikely(7 & (uintptr_t)data->iov_base)) { + if (unlikely(flags & MDBX_MULTIPLE)) { + /* LY: использование alignof(uint64_t) тут не подходил из-за ошибок + * MSVC и некоторых других компиляторов, когда для элементов + * массивов/векторов обеспечивает выравнивание только на 4-х байтовых + * границу и одновременно alignof(uint64_t) == 8. */ + if (MDBX_WORDBITS > 32 || (3 & (uintptr_t)data->iov_base) != 0) + return MDBX_BAD_VALSIZE; + } else { + /* copy instead of return error to avoid break compatibility */ + aligned_data.iov_base = bcopy_8(&aligned_databytes, data->iov_base); + aligned_data.iov_len = data->iov_len; + data = &aligned_data; + } + } + } else if (data->iov_len == 4) { + if (unlikely(3 & (uintptr_t)data->iov_base)) { if (unlikely(flags & MDBX_MULTIPLE)) return MDBX_BAD_VALSIZE; /* copy instead of return error to avoid break compatibility */ - aligned_data.iov_base = memcpy(&aligned_databytes, data->iov_base, - aligned_data.iov_len = 8); + aligned_data.iov_base = bcopy_4(&aligned_databytes, data->iov_base); + aligned_data.iov_len = data->iov_len; data = &aligned_data; } - break; + } else { + cASSERT(mc, !"data-size is invalid for MDBX_INTEGERKEY"); + return MDBX_BAD_VALSIZE; } } - return cursor_put_nochecklen(mc, key, data, flags); + return cursor_put(mc, key, data, flags); } -int mdbx_cursor_put(MDBX_cursor *mc, const MDBX_val *key, MDBX_val *data, - MDBX_put_flags_t flags) { - if (unlikely(mc == NULL || key == NULL || data == NULL)) - return MDBX_EINVAL; - - if (unlikely(mc->mc_signature != MDBX_MC_LIVE)) - return (mc->mc_signature == MDBX_MC_READY4CLOSE) ? MDBX_EINVAL - : MDBX_EBADSIGN; +__hot int cursor_del(MDBX_cursor *mc, unsigned flags) { + if (unlikely(!is_filled(mc))) + return MDBX_ENODATA; - int rc = check_txn_rw(mc->mc_txn, MDBX_TXN_BLOCKED); + int rc = cursor_touch(mc, nullptr, nullptr); if (unlikely(rc != MDBX_SUCCESS)) return rc; - if (unlikely(dbi_changed(mc->mc_txn, mc->mc_dbi))) - return MDBX_BAD_DBI; - - cASSERT(mc, cursor_is_tracked(mc)); - - /* Check this first so counter will always be zero on any early failures. */ - if (unlikely(flags & MDBX_MULTIPLE)) { - if (unlikely(flags & MDBX_RESERVE)) - return MDBX_EINVAL; - if (unlikely(!(mc->mc_db->md_flags & MDBX_DUPFIXED))) - return MDBX_INCOMPATIBLE; - const size_t dcount = data[1].iov_len; - if (unlikely(dcount < 2 || data->iov_len == 0)) - return MDBX_BAD_VALSIZE; - if (unlikely(mc->mc_db->md_xsize != data->iov_len) && mc->mc_db->md_xsize) - return MDBX_BAD_VALSIZE; - if (unlikely(dcount > MAX_MAPSIZE / 2 / - (BRANCH_NODE_MAX(MAX_PAGESIZE) - NODESIZE))) { - /* checking for multiplication overflow */ - if (unlikely(dcount > MAX_MAPSIZE / 2 / data->iov_len)) - return MDBX_TOO_LARGE; - } - } - - if (flags & MDBX_RESERVE) { - if (unlikely(mc->mc_db->md_flags & (MDBX_DUPSORT | MDBX_REVERSEDUP | - MDBX_INTEGERDUP | MDBX_DUPFIXED))) - return MDBX_INCOMPATIBLE; - data->iov_base = nullptr; - } - - if (unlikely(mc->mc_txn->mt_flags & (MDBX_TXN_RDONLY | MDBX_TXN_BLOCKED))) - return (mc->mc_txn->mt_flags & MDBX_TXN_RDONLY) ? MDBX_EACCESS - : MDBX_BAD_TXN; - - return cursor_put_checklen(mc, key, data, flags); -} - -int mdbx_cursor_del(MDBX_cursor *mc, MDBX_put_flags_t flags) { - if (unlikely(!mc)) - return MDBX_EINVAL; - - if (unlikely(mc->mc_signature != MDBX_MC_LIVE)) - return (mc->mc_signature == MDBX_MC_READY4CLOSE) ? MDBX_EINVAL - : MDBX_EBADSIGN; - - int rc = check_txn_rw(mc->mc_txn, MDBX_TXN_BLOCKED); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - - if (unlikely(dbi_changed(mc->mc_txn, mc->mc_dbi))) - return MDBX_BAD_DBI; - - if (unlikely(!(mc->mc_flags & C_INITIALIZED))) - return MDBX_ENODATA; - - if (unlikely(mc->mc_ki[mc->mc_top] >= page_numkeys(mc->mc_pg[mc->mc_top]))) - return MDBX_NOTFOUND; - - return cursor_del(mc, flags); -} - -static __hot int cursor_del(MDBX_cursor *mc, MDBX_put_flags_t flags) { - cASSERT(mc, mc->mc_flags & C_INITIALIZED); - cASSERT(mc, mc->mc_ki[mc->mc_top] < page_numkeys(mc->mc_pg[mc->mc_top])); - - int rc = cursor_touch(mc, nullptr, nullptr); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - - MDBX_page *mp = mc->mc_pg[mc->mc_top]; - cASSERT(mc, IS_MODIFIABLE(mc->mc_txn, mp)); - if (!MDBX_DISABLE_VALIDATION && unlikely(!CHECK_LEAF_TYPE(mc, mp))) { - ERROR("unexpected leaf-page #%" PRIaPGNO " type 0x%x seen by cursor", - mp->mp_pgno, mp->mp_flags); + page_t *mp = mc->pg[mc->top]; + cASSERT(mc, is_modifable(mc->txn, mp)); + if (!MDBX_DISABLE_VALIDATION && unlikely(!check_leaf_type(mc, mp))) { + ERROR("unexpected leaf-page #%" PRIaPGNO " type 0x%x seen by cursor", mp->pgno, mp->flags); return MDBX_CORRUPTED; } - if (IS_LEAF2(mp)) + if (is_dupfix_leaf(mp)) goto del_key; - MDBX_node *node = page_node(mp, mc->mc_ki[mc->mc_top]); - if (node_flags(node) & F_DUPDATA) { + node_t *node = page_node(mp, mc->ki[mc->top]); + if (node_flags(node) & N_DUP) { if (flags & (MDBX_ALLDUPS | /* for compatibility */ MDBX_NODUPDATA)) { /* will subtract the final entry later */ - mc->mc_db->md_entries -= mc->mc_xcursor->mx_db.md_entries - 1; - mc->mc_xcursor->mx_cursor.mc_flags &= ~C_INITIALIZED; + mc->tree->items -= mc->subcur->nested_tree.items - 1; } else { - if (!(node_flags(node) & F_SUBDATA)) - mc->mc_xcursor->mx_cursor.mc_pg[0] = node_data(node); - rc = cursor_del(&mc->mc_xcursor->mx_cursor, 0); + if (!(node_flags(node) & N_TREE)) { + page_t *sp = node_data(node); + cASSERT(mc, is_subpage(sp)); + sp->txnid = mp->txnid; + mc->subcur->cursor.pg[0] = sp; + } + rc = cursor_del(&mc->subcur->cursor, 0); if (unlikely(rc != MDBX_SUCCESS)) return rc; /* If sub-DB still has entries, we're done */ - if (mc->mc_xcursor->mx_db.md_entries) { - if (node_flags(node) & F_SUBDATA) { - /* update subDB info */ - mc->mc_xcursor->mx_db.md_mod_txnid = mc->mc_txn->mt_txnid; - memcpy(node_data(node), &mc->mc_xcursor->mx_db, sizeof(MDBX_db)); + if (mc->subcur->nested_tree.items) { + if (node_flags(node) & N_TREE) { + /* update table info */ + mc->subcur->nested_tree.mod_txnid = mc->txn->txnid; + memcpy(node_data(node), &mc->subcur->nested_tree, sizeof(tree_t)); } else { /* shrink sub-page */ - node = node_shrink(mp, mc->mc_ki[mc->mc_top], node); - mc->mc_xcursor->mx_cursor.mc_pg[0] = node_data(node); + node = node_shrink(mp, mc->ki[mc->top], node); + mc->subcur->cursor.pg[0] = node_data(node); /* fix other sub-DB cursors pointed at sub-pages on this page */ - for (MDBX_cursor *m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; - m2 = m2->mc_next) { - if (m2 == mc || m2->mc_snum < mc->mc_snum) + for (MDBX_cursor *m2 = mc->txn->cursors[cursor_dbi(mc)]; m2; m2 = m2->next) { + if (!is_related(mc, m2) || m2->pg[mc->top] != mp) continue; - if (!(m2->mc_flags & C_INITIALIZED)) + const node_t *inner = node; + if (unlikely(m2->ki[mc->top] >= page_numkeys(mp))) { + m2->flags = z_poor_mark; + m2->subcur->nested_tree.root = 0; + m2->subcur->cursor.top_and_flags = z_inner | z_poor_mark; continue; - if (m2->mc_pg[mc->mc_top] == mp) { - MDBX_node *inner = node; - if (m2->mc_ki[mc->mc_top] >= page_numkeys(mp)) + } + if (m2->ki[mc->top] != mc->ki[mc->top]) { + inner = page_node(mp, m2->ki[mc->top]); + if (node_flags(inner) & N_TREE) continue; - if (m2->mc_ki[mc->mc_top] != mc->mc_ki[mc->mc_top]) { - inner = page_node(mp, m2->mc_ki[mc->mc_top]); - if (node_flags(inner) & F_SUBDATA) - continue; - } - m2->mc_xcursor->mx_cursor.mc_pg[0] = node_data(inner); } + m2->subcur->cursor.pg[0] = node_data(inner); } } - mc->mc_db->md_entries--; - cASSERT(mc, mc->mc_db->md_entries > 0 && mc->mc_db->md_depth > 0 && - mc->mc_db->md_root != P_INVALID); + mc->tree->items -= 1; + cASSERT(mc, mc->tree->items > 0 && mc->tree->height > 0 && mc->tree->root != P_INVALID); return rc; - } else { - mc->mc_xcursor->mx_cursor.mc_flags &= ~C_INITIALIZED; } /* otherwise fall thru and delete the sub-DB */ } - if (node_flags(node) & F_SUBDATA) { + if ((node_flags(node) & N_TREE) && mc->subcur->cursor.tree->height) { /* add all the child DB's pages to the free list */ - rc = drop_tree(&mc->mc_xcursor->mx_cursor, false); - if (unlikely(rc)) + rc = tree_drop(&mc->subcur->cursor, false); + if (unlikely(rc != MDBX_SUCCESS)) goto fail; } + inner_gone(mc); + } else { + cASSERT(mc, !inner_pointed(mc)); + /* MDBX passes N_TREE in 'flags' to delete a DB record */ + if (unlikely((node_flags(node) ^ flags) & N_TREE)) + return MDBX_INCOMPATIBLE; } - /* MDBX passes F_SUBDATA in 'flags' to delete a DB record */ - else if (unlikely((node_flags(node) ^ flags) & F_SUBDATA)) - return MDBX_INCOMPATIBLE; /* add large/overflow pages to free list */ - if (node_flags(node) & F_BIGDATA) { - pgr_t lp = page_get_large(mc, node_largedata_pgno(node), mp->mp_txnid); + if (node_flags(node) & N_BIG) { + pgr_t lp = page_get_large(mc, node_largedata_pgno(node), mp->txnid); if (unlikely((rc = lp.err) || (rc = page_retire(mc, lp.page)))) goto fail; } del_key: - mc->mc_db->md_entries--; - const MDBX_dbi dbi = mc->mc_dbi; - indx_t ki = mc->mc_ki[mc->mc_top]; - mp = mc->mc_pg[mc->mc_top]; - cASSERT(mc, IS_LEAF(mp)); - node_del(mc, mc->mc_db->md_xsize); + mc->tree->items -= 1; + const MDBX_dbi dbi = cursor_dbi(mc); + indx_t ki = mc->ki[mc->top]; + mp = mc->pg[mc->top]; + cASSERT(mc, is_leaf(mp)); + node_del(mc, mc->tree->dupfix_size); /* Adjust other cursors pointing to mp */ - for (MDBX_cursor *m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2 = m2->mc_next) { - MDBX_cursor *m3 = (mc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2; - if (m3 == mc || !(m2->mc_flags & m3->mc_flags & C_INITIALIZED)) + for (MDBX_cursor *m2 = mc->txn->cursors[dbi]; m2; m2 = m2->next) { + MDBX_cursor *m3 = (mc->flags & z_inner) ? &m2->subcur->cursor : m2; + if (!is_related(mc, m3) || m3->pg[mc->top] != mp) continue; - if (m3->mc_snum < mc->mc_snum) - continue; - if (m3->mc_pg[mc->mc_top] == mp) { - if (m3->mc_ki[mc->mc_top] == ki) { - m3->mc_flags |= C_DEL; - if (mc->mc_db->md_flags & MDBX_DUPSORT) { - /* Sub-cursor referred into dataset which is gone */ - m3->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED | C_EOF); - } - continue; - } else if (m3->mc_ki[mc->mc_top] > ki) { - m3->mc_ki[mc->mc_top]--; - } - if (XCURSOR_INITED(m3)) - XCURSOR_REFRESH(m3, m3->mc_pg[mc->mc_top], m3->mc_ki[mc->mc_top]); + if (m3->ki[mc->top] == ki) { + m3->flags |= z_after_delete; + inner_gone(m3); + } else { + m3->ki[mc->top] -= m3->ki[mc->top] > ki; + if (inner_pointed(m3)) + cursor_inner_refresh(m3, m3->pg[mc->top], m3->ki[mc->top]); } } - rc = rebalance(mc); + rc = tree_rebalance(mc); if (unlikely(rc != MDBX_SUCCESS)) goto fail; - if (unlikely(!mc->mc_snum)) { + mc->flags |= z_after_delete; + inner_gone(mc); + if (unlikely(mc->top < 0)) { /* DB is totally empty now, just bail out. * Other cursors adjustments were already done * by rebalance and aren't needed here. */ - cASSERT(mc, mc->mc_db->md_entries == 0 && mc->mc_db->md_depth == 0 && - mc->mc_db->md_root == P_INVALID); - mc->mc_flags |= C_EOF; + cASSERT(mc, mc->tree->items == 0 && (mc->tree->root == P_INVALID || (is_inner(mc) && !mc->tree->root)) && + mc->flags < 0); return MDBX_SUCCESS; } - ki = mc->mc_ki[mc->mc_top]; - mp = mc->mc_pg[mc->mc_top]; - cASSERT(mc, IS_LEAF(mc->mc_pg[mc->mc_top])); + ki = mc->ki[mc->top]; + mp = mc->pg[mc->top]; + cASSERT(mc, is_leaf(mc->pg[mc->top])); size_t nkeys = page_numkeys(mp); - cASSERT(mc, (mc->mc_db->md_entries > 0 && nkeys > 0) || - ((mc->mc_flags & C_SUB) && mc->mc_db->md_entries == 0 && - nkeys == 0)); + cASSERT(mc, (mc->tree->items > 0 && nkeys > 0) || ((mc->flags & z_inner) && mc->tree->items == 0 && nkeys == 0)); /* Adjust this and other cursors pointing to mp */ - for (MDBX_cursor *m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2 = m2->mc_next) { - MDBX_cursor *m3 = (mc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2; - if (!(m2->mc_flags & m3->mc_flags & C_INITIALIZED)) - continue; - if (m3->mc_snum < mc->mc_snum) + const intptr_t top = /* может быть сброшен в -1 */ mc->top; + for (MDBX_cursor *m2 = mc->txn->cursors[dbi]; m2; m2 = m2->next) { + MDBX_cursor *m3 = (mc->flags & z_inner) ? &m2->subcur->cursor : m2; + if (top > m3->top || m3->pg[top] != mp) continue; - if (m3->mc_pg[mc->mc_top] == mp) { - /* if m3 points past last node in page, find next sibling */ - if (m3->mc_ki[mc->mc_top] >= nkeys) { - rc = cursor_sibling(m3, SIBLING_RIGHT); - if (rc == MDBX_NOTFOUND) { - m3->mc_flags |= C_EOF; - rc = MDBX_SUCCESS; - continue; - } - if (unlikely(rc != MDBX_SUCCESS)) - goto fail; + /* if m3 points past last node in page, find next sibling */ + if (m3->ki[top] >= nkeys) { + rc = cursor_sibling_right(m3); + if (rc == MDBX_NOTFOUND) { + rc = MDBX_SUCCESS; + continue; } - if (m3->mc_ki[mc->mc_top] >= ki || - /* moved to right sibling */ m3->mc_pg[mc->mc_top] != mp) { - if (m3->mc_xcursor && !(m3->mc_flags & C_EOF)) { - node = page_node(m3->mc_pg[m3->mc_top], m3->mc_ki[m3->mc_top]); - /* If this node has dupdata, it may need to be reinited - * because its data has moved. - * If the xcursor was not inited it must be reinited. - * Else if node points to a subDB, nothing is needed. */ - if (node_flags(node) & F_DUPDATA) { - if (m3->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) { - if (!(node_flags(node) & F_SUBDATA)) - m3->mc_xcursor->mx_cursor.mc_pg[0] = node_data(node); - } else { - rc = cursor_xinit1(m3, node, m3->mc_pg[m3->mc_top]); - if (unlikely(rc != MDBX_SUCCESS)) - goto fail; - rc = cursor_first(&m3->mc_xcursor->mx_cursor, NULL, NULL); - if (unlikely(rc != MDBX_SUCCESS)) - goto fail; - } + if (unlikely(rc != MDBX_SUCCESS)) + goto fail; + } + if (/* пропускаем незаполненные курсоры, иначе получится что у такого + курсора будет инициализирован вложенный, + что антилогично и бесполезно. */ + is_filled(m3) && m3->subcur && + (m3->ki[top] >= ki || + /* уже переместились вправо */ m3->pg[top] != mp)) { + node = page_node(m3->pg[m3->top], m3->ki[m3->top]); + /* Если это dupsort-узел, то должен быть валидный вложенный курсор. */ + if (node_flags(node) & N_DUP) { + /* Тут три варианта событий: + * 1) Вложенный курсор уже инициализирован, у узла есть флаг N_TREE, + * соответственно дубликаты вынесены в отдельное дерево с корнем + * в отдельной странице = ничего корректировать не требуется. + * 2) Вложенный курсор уже инициализирован, у узла нет флага N_TREE, + * соответственно дубликаты размещены на вложенной sub-странице. + * 3) Курсор стоял на удалённом элементе, который имел одно значение, + * а после удаления переместился на следующий элемент с дубликатами. + * В этом случае вложенный курсор не инициализирован и тепеь его + * нужно установить на первый дубликат. */ + if (is_pointed(&m3->subcur->cursor)) { + if ((node_flags(node) & N_TREE) == 0) { + cASSERT(m3, m3->subcur->cursor.top == 0 && m3->subcur->nested_tree.height == 1); + m3->subcur->cursor.pg[0] = node_data(node); + } + } else { + rc = cursor_dupsort_setup(m3, node, m3->pg[m3->top]); + if (unlikely(rc != MDBX_SUCCESS)) + goto fail; + if (node_flags(node) & N_TREE) { + rc = inner_first(&m3->subcur->cursor, nullptr); + if (unlikely(rc != MDBX_SUCCESS)) + goto fail; } - m3->mc_xcursor->mx_cursor.mc_flags |= C_DEL; } - m3->mc_flags |= C_DEL; - } + } else + inner_gone(m3); } } cASSERT(mc, rc == MDBX_SUCCESS); if (AUDIT_ENABLED()) - rc = cursor_check(mc); + rc = cursor_validate(mc); return rc; fail: - mc->mc_txn->mt_flags |= MDBX_TXN_ERROR; + mc->txn->flags |= MDBX_TXN_ERROR; return rc; } -/* Allocate and initialize new pages for a database. - * Set MDBX_TXN_ERROR on failure. */ -static pgr_t page_new(MDBX_cursor *mc, const unsigned flags) { - cASSERT(mc, (flags & P_OVERFLOW) == 0); - pgr_t ret = page_alloc(mc); - if (unlikely(ret.err != MDBX_SUCCESS)) - return ret; - - DEBUG("db %u allocated new page %" PRIaPGNO, mc->mc_dbi, ret.page->mp_pgno); - ret.page->mp_flags = (uint16_t)flags; - cASSERT(mc, *mc->mc_dbistate & DBI_DIRTY); - cASSERT(mc, mc->mc_txn->mt_flags & MDBX_TXN_DIRTY); -#if MDBX_ENABLE_PGOP_STAT - mc->mc_txn->mt_env->me_lck->mti_pgop_stat.newly.weak += 1; -#endif /* MDBX_ENABLE_PGOP_STAT */ - - STATIC_ASSERT(P_BRANCH == 1); - const unsigned is_branch = flags & P_BRANCH; - - ret.page->mp_lower = 0; - ret.page->mp_upper = (indx_t)(mc->mc_txn->mt_env->me_psize - PAGEHDRSZ); - mc->mc_db->md_branch_pages += is_branch; - mc->mc_db->md_leaf_pages += 1 - is_branch; - if (unlikely(mc->mc_flags & C_SUB)) { - MDBX_db *outer = outer_db(mc); - outer->md_branch_pages += is_branch; - outer->md_leaf_pages += 1 - is_branch; - } - return ret; -} - -static pgr_t page_new_large(MDBX_cursor *mc, const size_t npages) { - pgr_t ret = likely(npages == 1) - ? page_alloc(mc) - : page_alloc_slowpath(mc, npages, MDBX_ALLOC_DEFAULT); - if (unlikely(ret.err != MDBX_SUCCESS)) - return ret; - - DEBUG("db %u allocated new large-page %" PRIaPGNO ", num %zu", mc->mc_dbi, - ret.page->mp_pgno, npages); - ret.page->mp_flags = P_OVERFLOW; - cASSERT(mc, *mc->mc_dbistate & DBI_DIRTY); - cASSERT(mc, mc->mc_txn->mt_flags & MDBX_TXN_DIRTY); -#if MDBX_ENABLE_PGOP_STAT - mc->mc_txn->mt_env->me_lck->mti_pgop_stat.newly.weak += npages; -#endif /* MDBX_ENABLE_PGOP_STAT */ - - mc->mc_db->md_overflow_pages += (pgno_t)npages; - ret.page->mp_pages = (pgno_t)npages; - cASSERT(mc, !(mc->mc_flags & C_SUB)); - return ret; -} +/*----------------------------------------------------------------------------*/ -__hot static int __must_check_result node_add_leaf2(MDBX_cursor *mc, - size_t indx, - const MDBX_val *key) { - MDBX_page *mp = mc->mc_pg[mc->mc_top]; - MDBX_ANALYSIS_ASSUME(key != nullptr); +__hot csr_t cursor_seek(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, MDBX_cursor_op op) { DKBUF_DEBUG; - DEBUG("add to leaf2-%spage %" PRIaPGNO " index %zi, " - " key size %" PRIuPTR " [%s]", - IS_SUBP(mp) ? "sub-" : "", mp->mp_pgno, indx, key ? key->iov_len : 0, - DKEY_DEBUG(key)); - - cASSERT(mc, key); - cASSERT(mc, PAGETYPE_COMPAT(mp) == (P_LEAF | P_LEAF2)); - const size_t ksize = mc->mc_db->md_xsize; - cASSERT(mc, ksize == key->iov_len); - const size_t nkeys = page_numkeys(mp); - cASSERT(mc, (((ksize & page_numkeys(mp)) ^ mp->mp_upper) & 1) == 0); - /* Just using these for counting */ - const intptr_t lower = mp->mp_lower + sizeof(indx_t); - const intptr_t upper = mp->mp_upper - (ksize - sizeof(indx_t)); - if (unlikely(lower > upper)) { - mc->mc_txn->mt_flags |= MDBX_TXN_ERROR; - return MDBX_PAGE_FULL; + csr_t ret; + ret.exact = false; + if (unlikely(key->iov_len < mc->clc->k.lmin || + (key->iov_len > mc->clc->k.lmax && + (mc->clc->k.lmin == mc->clc->k.lmax || MDBX_DEBUG || MDBX_FORCE_ASSERTIONS)))) { + cASSERT(mc, !"Invalid key-size"); + ret.err = MDBX_BAD_VALSIZE; + return ret; } - mp->mp_lower = (indx_t)lower; - mp->mp_upper = (indx_t)upper; - void *const ptr = page_leaf2key(mp, indx, ksize); - cASSERT(mc, nkeys >= indx); - const size_t diff = nkeys - indx; - if (likely(diff > 0)) - /* Move higher keys up one slot. */ - memmove(ptr_disp(ptr, ksize), ptr, diff * ksize); - /* insert new key */ - memcpy(ptr, key->iov_base, ksize); + MDBX_val aligned_key = *key; + uint64_t aligned_key_buf; + if (mc->tree->flags & MDBX_INTEGERKEY) { + if (aligned_key.iov_len == 8) { + if (unlikely(7 & (uintptr_t)aligned_key.iov_base)) + /* copy instead of return error to avoid break compatibility */ + aligned_key.iov_base = bcopy_8(&aligned_key_buf, aligned_key.iov_base); + } else if (aligned_key.iov_len == 4) { + if (unlikely(3 & (uintptr_t)aligned_key.iov_base)) + /* copy instead of return error to avoid break compatibility */ + aligned_key.iov_base = bcopy_4(&aligned_key_buf, aligned_key.iov_base); + } else { + cASSERT(mc, !"key-size is invalid for MDBX_INTEGERKEY"); + ret.err = MDBX_BAD_VALSIZE; + return ret; + } + } - cASSERT(mc, (((ksize & page_numkeys(mp)) ^ mp->mp_upper) & 1) == 0); - return MDBX_SUCCESS; -} + page_t *mp; + node_t *node = nullptr; + /* See if we're already on the right page */ + if (is_pointed(mc)) { + mp = mc->pg[mc->top]; + cASSERT(mc, is_leaf(mp)); + const size_t nkeys = page_numkeys(mp); + if (unlikely(nkeys == 0)) { + /* при создании первой листовой страницы */ + cASSERT(mc, mc->top == 0 && mc->tree->height == 1 && mc->tree->branch_pages == 0 && mc->tree->leaf_pages == 1 && + mc->ki[0] == 0); + /* Логически верно, но нет смысла, ибо это мимолетная/временная + * ситуация до добавления элемента выше по стеку вызовов: + mc->flags |= z_eof_soft | z_hollow; */ + ret.err = MDBX_NOTFOUND; + return ret; + } -static int __must_check_result node_add_branch(MDBX_cursor *mc, size_t indx, - const MDBX_val *key, - pgno_t pgno) { - MDBX_page *mp = mc->mc_pg[mc->mc_top]; - DKBUF_DEBUG; - DEBUG("add to branch-%spage %" PRIaPGNO " index %zi, node-pgno %" PRIaPGNO - " key size %" PRIuPTR " [%s]", - IS_SUBP(mp) ? "sub-" : "", mp->mp_pgno, indx, pgno, - key ? key->iov_len : 0, DKEY_DEBUG(key)); + MDBX_val nodekey; + if (is_dupfix_leaf(mp)) + nodekey = page_dupfix_key(mp, 0, mc->tree->dupfix_size); + else { + node = page_node(mp, 0); + nodekey = get_key(node); + inner_gone(mc); + } + int cmp = mc->clc->k.cmp(&aligned_key, &nodekey); + if (unlikely(cmp == 0)) { + /* Probably happens rarely, but first node on the page + * was the one we wanted. */ + mc->ki[mc->top] = 0; + ret.exact = true; + goto got_node; + } - cASSERT(mc, PAGETYPE_WHOLE(mp) == P_BRANCH); - STATIC_ASSERT(NODESIZE % 2 == 0); + if (cmp > 0) { + /* Искомый ключ больше первого на этой странице, + * целевая позиция на этой странице либо правее (ближе к концу). */ + if (likely(nkeys > 1)) { + if (is_dupfix_leaf(mp)) { + nodekey.iov_base = page_dupfix_ptr(mp, nkeys - 1, nodekey.iov_len); + } else { + node = page_node(mp, nkeys - 1); + nodekey = get_key(node); + } + cmp = mc->clc->k.cmp(&aligned_key, &nodekey); + if (cmp == 0) { + /* last node was the one we wanted */ + mc->ki[mc->top] = (indx_t)(nkeys - 1); + ret.exact = true; + goto got_node; + } + if (cmp < 0) { + /* Искомый ключ между первым и последним на этой страницы, + * поэтому пропускаем поиск по дереву и продолжаем только на текущей + * странице. */ + /* Сравниваем с текущей позицией, ибо частным сценарием является такое + * совпадение, но не делаем проверку если текущая позиция является + * первой/последний и соответственно такое сравнение было выше. */ + if (mc->ki[mc->top] > 0 && mc->ki[mc->top] < nkeys - 1) { + if (is_dupfix_leaf(mp)) { + nodekey.iov_base = page_dupfix_ptr(mp, mc->ki[mc->top], nodekey.iov_len); + } else { + node = page_node(mp, mc->ki[mc->top]); + nodekey = get_key(node); + } + cmp = mc->clc->k.cmp(&aligned_key, &nodekey); + if (cmp == 0) { + /* current node was the one we wanted */ + ret.exact = true; + goto got_node; + } + } + goto search_node; + } + } - /* Move higher pointers up one slot. */ - const size_t nkeys = page_numkeys(mp); - cASSERT(mc, nkeys >= indx); - for (size_t i = nkeys; i > indx; --i) - mp->mp_ptrs[i] = mp->mp_ptrs[i - 1]; + /* Если в стеке курсора есть страницы справа, то продолжим искать там. */ + cASSERT(mc, mc->tree->height > mc->top); + for (intptr_t i = 0; i < mc->top; i++) + if ((size_t)mc->ki[i] + 1 < page_numkeys(mc->pg[i])) + goto continue_other_pages; + + /* Ключ больше последнего. */ + mc->ki[mc->top] = (indx_t)nkeys; + if (op < MDBX_SET_RANGE) { + target_not_found: + cASSERT(mc, op == MDBX_SET || op == MDBX_SET_KEY || op == MDBX_GET_BOTH || op == MDBX_GET_BOTH_RANGE); + /* Операция предполагает поиск конкретного ключа, который не найден. + * Поэтому переводим курсор в неустановленное состояние, но без сброса + * top, что позволяет работать fastpath при последующем поиске по дереву + * страниц. */ + mc->flags = z_hollow | (mc->flags & z_clear_mask); + inner_gone(mc); + ret.err = MDBX_NOTFOUND; + return ret; + } + cASSERT(mc, op == MDBX_SET_RANGE); + mc->flags = z_eof_soft | z_eof_hard | (mc->flags & z_clear_mask); + ret.err = MDBX_NOTFOUND; + return ret; + } - /* Adjust free space offsets. */ - const size_t branch_bytes = branch_size(mc->mc_txn->mt_env, key); - const intptr_t lower = mp->mp_lower + sizeof(indx_t); - const intptr_t upper = mp->mp_upper - (branch_bytes - sizeof(indx_t)); - if (unlikely(lower > upper)) { - mc->mc_txn->mt_flags |= MDBX_TXN_ERROR; - return MDBX_PAGE_FULL; + if (mc->top == 0) { + /* There are no other pages */ + mc->ki[mc->top] = 0; + if (op >= MDBX_SET_RANGE) + goto got_node; + else + goto target_not_found; + } } - mp->mp_lower = (indx_t)lower; - mp->mp_ptrs[indx] = mp->mp_upper = (indx_t)upper; + cASSERT(mc, !inner_pointed(mc)); - /* Write the node data. */ - MDBX_node *node = page_node(mp, indx); - node_set_pgno(node, pgno); - node_set_flags(node, 0); - UNALIGNED_POKE_8(node, MDBX_node, mn_extra, 0); - node_set_ks(node, 0); - if (likely(key != NULL)) { - node_set_ks(node, key->iov_len); - memcpy(node_key(node), key->iov_base, key->iov_len); - } - return MDBX_SUCCESS; -} +continue_other_pages: + ret.err = tree_search(mc, &aligned_key, 0); + if (unlikely(ret.err != MDBX_SUCCESS)) + return ret; -__hot static int __must_check_result node_add_leaf(MDBX_cursor *mc, size_t indx, - const MDBX_val *key, - MDBX_val *data, - unsigned flags) { - MDBX_ANALYSIS_ASSUME(key != nullptr); - MDBX_ANALYSIS_ASSUME(data != nullptr); - MDBX_page *mp = mc->mc_pg[mc->mc_top]; - DKBUF_DEBUG; - DEBUG("add to leaf-%spage %" PRIaPGNO " index %zi, data size %" PRIuPTR - " key size %" PRIuPTR " [%s]", - IS_SUBP(mp) ? "sub-" : "", mp->mp_pgno, indx, data ? data->iov_len : 0, - key ? key->iov_len : 0, DKEY_DEBUG(key)); - cASSERT(mc, key != NULL && data != NULL); - cASSERT(mc, PAGETYPE_COMPAT(mp) == P_LEAF); - MDBX_page *largepage = NULL; + cASSERT(mc, is_pointed(mc) && !inner_pointed(mc)); + mp = mc->pg[mc->top]; + MDBX_ANALYSIS_ASSUME(mp != nullptr); + cASSERT(mc, is_leaf(mp)); - size_t node_bytes; - if (unlikely(flags & F_BIGDATA)) { - /* Data already on large/overflow page. */ - STATIC_ASSERT(sizeof(pgno_t) % 2 == 0); - node_bytes = - node_size_len(key->iov_len, 0) + sizeof(pgno_t) + sizeof(indx_t); - cASSERT(mc, page_room(mp) >= node_bytes); - } else if (unlikely(node_size(key, data) > - mc->mc_txn->mt_env->me_leaf_nodemax)) { - /* Put data on large/overflow page. */ - if (unlikely(mc->mc_db->md_flags & MDBX_DUPSORT)) { - ERROR("Unexpected target %s flags 0x%x for large data-item", "dupsort-db", - mc->mc_db->md_flags); - return MDBX_PROBLEM; - } - if (unlikely(flags & (F_DUPDATA | F_SUBDATA))) { - ERROR("Unexpected target %s flags 0x%x for large data-item", "node", - flags); - return MDBX_PROBLEM; +search_node: + cASSERT(mc, is_pointed(mc) && !inner_pointed(mc)); + struct node_search_result nsr = node_search(mc, &aligned_key); + node = nsr.node; + ret.exact = nsr.exact; + if (!ret.exact) { + if (op < MDBX_SET_RANGE) + goto target_not_found; + + if (node == nullptr) { + DEBUG("%s", "===> inexact leaf not found, goto sibling"); + ret.err = cursor_sibling_right(mc); + if (unlikely(ret.err != MDBX_SUCCESS)) + return ret; /* no entries matched */ + mp = mc->pg[mc->top]; + cASSERT(mc, is_leaf(mp)); + if (!is_dupfix_leaf(mp)) + node = page_node(mp, 0); } - cASSERT(mc, page_room(mp) >= leaf_size(mc->mc_txn->mt_env, key, data)); - const pgno_t ovpages = number_of_ovpages(mc->mc_txn->mt_env, data->iov_len); - const pgr_t npr = page_new_large(mc, ovpages); - if (unlikely(npr.err != MDBX_SUCCESS)) - return npr.err; - largepage = npr.page; - DEBUG("allocated %u large/overflow page(s) %" PRIaPGNO "for %" PRIuPTR - " data bytes", - largepage->mp_pages, largepage->mp_pgno, data->iov_len); - flags |= F_BIGDATA; - node_bytes = - node_size_len(key->iov_len, 0) + sizeof(pgno_t) + sizeof(indx_t); - cASSERT(mc, node_bytes == leaf_size(mc->mc_txn->mt_env, key, data)); - } else { - cASSERT(mc, page_room(mp) >= leaf_size(mc->mc_txn->mt_env, key, data)); - node_bytes = node_size(key, data) + sizeof(indx_t); - cASSERT(mc, node_bytes == leaf_size(mc->mc_txn->mt_env, key, data)); } - /* Move higher pointers up one slot. */ - const size_t nkeys = page_numkeys(mp); - cASSERT(mc, nkeys >= indx); - for (size_t i = nkeys; i > indx; --i) - mp->mp_ptrs[i] = mp->mp_ptrs[i - 1]; - - /* Adjust free space offsets. */ - const intptr_t lower = mp->mp_lower + sizeof(indx_t); - const intptr_t upper = mp->mp_upper - (node_bytes - sizeof(indx_t)); - if (unlikely(lower > upper)) { - mc->mc_txn->mt_flags |= MDBX_TXN_ERROR; - return MDBX_PAGE_FULL; +got_node: + cASSERT(mc, is_pointed(mc) && !inner_pointed(mc)); + cASSERT(mc, mc->ki[mc->top] < page_numkeys(mc->pg[mc->top])); + if (!MDBX_DISABLE_VALIDATION && unlikely(!check_leaf_type(mc, mp))) { + ERROR("unexpected leaf-page #%" PRIaPGNO " type 0x%x seen by cursor", mp->pgno, mp->flags); + ret.err = MDBX_CORRUPTED; + return ret; } - mp->mp_lower = (indx_t)lower; - mp->mp_ptrs[indx] = mp->mp_upper = (indx_t)upper; - - /* Write the node data. */ - MDBX_node *node = page_node(mp, indx); - node_set_ks(node, key->iov_len); - node_set_flags(node, (uint8_t)flags); - UNALIGNED_POKE_8(node, MDBX_node, mn_extra, 0); - node_set_ds(node, data->iov_len); - memcpy(node_key(node), key->iov_base, key->iov_len); - void *nodedata = node_data(node); - if (likely(largepage == NULL)) { - if (unlikely(flags & F_BIGDATA)) { - memcpy(nodedata, data->iov_base, sizeof(pgno_t)); - return MDBX_SUCCESS; - } - } else { - poke_pgno(nodedata, largepage->mp_pgno); - nodedata = page_data(largepage); + if (is_dupfix_leaf(mp)) { + if (op >= MDBX_SET_KEY) + *key = page_dupfix_key(mp, mc->ki[mc->top], mc->tree->dupfix_size); + be_filled(mc); + ret.err = MDBX_SUCCESS; + return ret; } - if (unlikely(flags & MDBX_RESERVE)) - data->iov_base = nodedata; - else if (likely(nodedata != data->iov_base && - data->iov_len /* to avoid UBSAN traps*/ != 0)) - memcpy(nodedata, data->iov_base, data->iov_len); - return MDBX_SUCCESS; -} - -/* Delete the specified node from a page. - * [in] mc Cursor pointing to the node to delete. - * [in] ksize The size of a node. Only used if the page is - * part of a MDBX_DUPFIXED database. */ -__hot static void node_del(MDBX_cursor *mc, size_t ksize) { - MDBX_page *mp = mc->mc_pg[mc->mc_top]; - const size_t hole = mc->mc_ki[mc->mc_top]; - const size_t nkeys = page_numkeys(mp); - DEBUG("delete node %zu on %s page %" PRIaPGNO, hole, - IS_LEAF(mp) ? "leaf" : "branch", mp->mp_pgno); - cASSERT(mc, hole < nkeys); - - if (IS_LEAF2(mp)) { - cASSERT(mc, ksize >= sizeof(indx_t)); - size_t diff = nkeys - 1 - hole; - void *const base = page_leaf2key(mp, hole, ksize); - if (diff) - memmove(base, ptr_disp(base, ksize), diff * ksize); - cASSERT(mc, mp->mp_lower >= sizeof(indx_t)); - mp->mp_lower -= sizeof(indx_t); - cASSERT(mc, (size_t)UINT16_MAX - mp->mp_upper >= ksize - sizeof(indx_t)); - mp->mp_upper += (indx_t)(ksize - sizeof(indx_t)); - cASSERT(mc, (((ksize & page_numkeys(mp)) ^ mp->mp_upper) & 1) == 0); - return; + if (node_flags(node) & N_DUP) { + ret.err = cursor_dupsort_setup(mc, node, mp); + if (unlikely(ret.err != MDBX_SUCCESS)) + return ret; + if (op >= MDBX_SET) { + MDBX_ANALYSIS_ASSUME(mc->subcur != nullptr); + if (node_flags(node) & N_TREE) { + ret.err = inner_first(&mc->subcur->cursor, data); + if (unlikely(ret.err != MDBX_SUCCESS)) + return ret; + } else if (data) { + const page_t *inner_mp = mc->subcur->cursor.pg[0]; + cASSERT(mc, is_subpage(inner_mp) && is_leaf(inner_mp)); + const size_t inner_ki = mc->subcur->cursor.ki[0]; + if (is_dupfix_leaf(inner_mp)) + *data = page_dupfix_key(inner_mp, inner_ki, mc->tree->dupfix_size); + else + *data = get_key(page_node(inner_mp, inner_ki)); + } + } else { + MDBX_ANALYSIS_ASSUME(mc->subcur != nullptr); + ret = cursor_seek(&mc->subcur->cursor, data, nullptr, MDBX_SET_RANGE); + if (unlikely(ret.err != MDBX_SUCCESS)) { + if (ret.err == MDBX_NOTFOUND && op < MDBX_SET_RANGE) + goto target_not_found; + return ret; + } + if (op == MDBX_GET_BOTH && !ret.exact) + goto target_not_found; + } + } else if (likely(data)) { + if (op <= MDBX_GET_BOTH_RANGE) { + if (unlikely(data->iov_len < mc->clc->v.lmin || data->iov_len > mc->clc->v.lmax)) { + cASSERT(mc, !"Invalid data-size"); + ret.err = MDBX_BAD_VALSIZE; + return ret; + } + MDBX_val aligned_data = *data; + uint64_t aligned_databytes; + if (mc->tree->flags & MDBX_INTEGERDUP) { + if (aligned_data.iov_len == 8) { + if (unlikely(7 & (uintptr_t)aligned_data.iov_base)) + /* copy instead of return error to avoid break compatibility */ + aligned_data.iov_base = bcopy_8(&aligned_databytes, aligned_data.iov_base); + } else if (aligned_data.iov_len == 4) { + if (unlikely(3 & (uintptr_t)aligned_data.iov_base)) + /* copy instead of return error to avoid break compatibility */ + aligned_data.iov_base = bcopy_4(&aligned_databytes, aligned_data.iov_base); + } else { + cASSERT(mc, !"data-size is invalid for MDBX_INTEGERDUP"); + ret.err = MDBX_BAD_VALSIZE; + return ret; + } + } + MDBX_val actual_data; + ret.err = node_read(mc, node, &actual_data, mc->pg[mc->top]); + if (unlikely(ret.err != MDBX_SUCCESS)) + return ret; + const int cmp = mc->clc->v.cmp(&aligned_data, &actual_data); + if (cmp) { + if (op != MDBX_GET_BOTH_RANGE) { + cASSERT(mc, op == MDBX_GET_BOTH); + goto target_not_found; + } + if (cmp > 0) { + ret.err = MDBX_NOTFOUND; + return ret; + } + } + *data = actual_data; + } else { + ret.err = node_read(mc, node, data, mc->pg[mc->top]); + if (unlikely(ret.err != MDBX_SUCCESS)) + return ret; + } } - MDBX_node *node = page_node(mp, hole); - cASSERT(mc, !IS_BRANCH(mp) || hole || node_ks(node) == 0); - size_t hole_size = NODESIZE + node_ks(node); - if (IS_LEAF(mp)) - hole_size += - (node_flags(node) & F_BIGDATA) ? sizeof(pgno_t) : node_ds(node); - hole_size = EVEN(hole_size); - - const indx_t hole_offset = mp->mp_ptrs[hole]; - size_t r, w; - for (r = w = 0; r < nkeys; r++) - if (r != hole) - mp->mp_ptrs[w++] = (mp->mp_ptrs[r] < hole_offset) - ? mp->mp_ptrs[r] + (indx_t)hole_size - : mp->mp_ptrs[r]; - - void *const base = ptr_disp(mp, mp->mp_upper + PAGEHDRSZ); - memmove(ptr_disp(base, hole_size), base, hole_offset - mp->mp_upper); - - cASSERT(mc, mp->mp_lower >= sizeof(indx_t)); - mp->mp_lower -= sizeof(indx_t); - cASSERT(mc, (size_t)UINT16_MAX - mp->mp_upper >= hole_size); - mp->mp_upper += (indx_t)hole_size; + /* The key already matches in all other cases */ + if (op >= MDBX_SET_KEY) + get_key_optional(node, key); - if (AUDIT_ENABLED()) { - const uint8_t checking = mc->mc_checking; - mc->mc_checking |= CC_UPDATING; - const int page_check_err = page_check(mc, mp); - mc->mc_checking = checking; - cASSERT(mc, page_check_err == MDBX_SUCCESS); - } + DEBUG("==> cursor placed on key [%s], data [%s]", DKEY_DEBUG(key), DVAL_DEBUG(data)); + ret.err = MDBX_SUCCESS; + be_filled(mc); + return ret; } -/* Compact the main page after deleting a node on a subpage. - * [in] mp The main page to operate on. - * [in] indx The index of the subpage on the main page. */ -static MDBX_node *node_shrink(MDBX_page *mp, size_t indx, MDBX_node *node) { - assert(node = page_node(mp, indx)); - MDBX_page *sp = (MDBX_page *)node_data(node); - assert(IS_SUBP(sp) && page_numkeys(sp) > 0); - const size_t delta = - EVEN_FLOOR(page_room(sp) /* avoid the node uneven-sized */); - if (unlikely(delta) == 0) - return node; +__hot int cursor_ops(MDBX_cursor *mc, MDBX_val *key, MDBX_val *data, const MDBX_cursor_op op) { + if (op != MDBX_GET_CURRENT) + DEBUG(">> cursor %p(0x%x), ops %u, key %p, value %p", __Wpedantic_format_voidptr(mc), mc->flags, op, + __Wpedantic_format_voidptr(key), __Wpedantic_format_voidptr(data)); + int rc; - /* Prepare to shift upward, set len = length(subpage part to shift) */ - size_t nsize = node_ds(node) - delta, len = nsize; - assert(nsize % 1 == 0); - if (!IS_LEAF2(sp)) { - len = PAGEHDRSZ; - MDBX_page *xp = ptr_disp(sp, delta); /* destination subpage */ - for (intptr_t i = page_numkeys(sp); --i >= 0;) { - assert(sp->mp_ptrs[i] >= delta); - xp->mp_ptrs[i] = (indx_t)(sp->mp_ptrs[i] - delta); + switch (op) { + case MDBX_GET_CURRENT: + cASSERT(mc, (mc->flags & z_inner) == 0); + if (unlikely(!is_filled(mc))) { + if (is_hollow(mc)) + return MDBX_ENODATA; + if (mc->ki[mc->top] >= page_numkeys(mc->pg[mc->top])) + return MDBX_NOTFOUND; + } + if (mc->flags & z_after_delete) + return outer_next(mc, key, data, MDBX_NEXT_NODUP); + else if (inner_pointed(mc) && (mc->subcur->cursor.flags & z_after_delete)) + return outer_next(mc, key, data, MDBX_NEXT_DUP); + else { + const page_t *mp = mc->pg[mc->top]; + const node_t *node = page_node(mp, mc->ki[mc->top]); + get_key_optional(node, key); + if (!data) + return MDBX_SUCCESS; + if (node_flags(node) & N_DUP) { + if (!MDBX_DISABLE_VALIDATION && unlikely(!mc->subcur)) + return unexpected_dupsort(mc); + mc = &mc->subcur->cursor; + if (unlikely(!is_filled(mc))) { + if (is_hollow(mc)) + return MDBX_ENODATA; + if (mc->ki[mc->top] >= page_numkeys(mc->pg[mc->top])) + return MDBX_NOTFOUND; + } + mp = mc->pg[mc->top]; + if (is_dupfix_leaf(mp)) + *data = page_dupfix_key(mp, mc->ki[mc->top], mc->tree->dupfix_size); + else + *data = get_key(page_node(mp, mc->ki[mc->top])); + return MDBX_SUCCESS; + } else { + cASSERT(mc, !inner_pointed(mc)); + return node_read(mc, node, data, mc->pg[mc->top]); + } } - } - assert(sp->mp_upper >= sp->mp_lower + delta); - sp->mp_upper -= (indx_t)delta; - sp->mp_pgno = mp->mp_pgno; - node_set_ds(node, nsize); - /* Shift upward */ - void *const base = ptr_disp(mp, mp->mp_upper + PAGEHDRSZ); - memmove(ptr_disp(base, delta), base, ptr_dist(sp, base) + len); + case MDBX_GET_BOTH: + case MDBX_GET_BOTH_RANGE: + if (unlikely(data == nullptr)) + return MDBX_EINVAL; + if (unlikely(mc->subcur == nullptr)) + return MDBX_INCOMPATIBLE; + /* fall through */ + __fallthrough; + case MDBX_SET: + case MDBX_SET_KEY: + case MDBX_SET_RANGE: + if (unlikely(key == nullptr)) + return MDBX_EINVAL; + rc = cursor_seek(mc, key, data, op).err; + if (rc == MDBX_SUCCESS) + cASSERT(mc, is_filled(mc)); + else if (rc == MDBX_NOTFOUND && mc->tree->items) { + cASSERT(mc, is_pointed(mc)); + cASSERT(mc, op == MDBX_SET_RANGE || op == MDBX_GET_BOTH_RANGE || is_hollow(mc)); + cASSERT(mc, op == MDBX_GET_BOTH_RANGE || inner_hollow(mc)); + } else + cASSERT(mc, is_poor(mc) && !is_filled(mc)); + return rc; - const size_t pivot = mp->mp_ptrs[indx]; - for (intptr_t i = page_numkeys(mp); --i >= 0;) { - if (mp->mp_ptrs[i] <= pivot) { - assert((size_t)UINT16_MAX - mp->mp_ptrs[i] >= delta); - mp->mp_ptrs[i] += (indx_t)delta; + case MDBX_SEEK_AND_GET_MULTIPLE: + if (unlikely(!key)) + return MDBX_EINVAL; + rc = cursor_seek(mc, key, data, MDBX_SET).err; + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + __fallthrough /* fall through */; + case MDBX_GET_MULTIPLE: + if (unlikely(!data)) + return MDBX_EINVAL; + if (unlikely((mc->tree->flags & MDBX_DUPFIXED) == 0)) + return MDBX_INCOMPATIBLE; + if (unlikely(!is_filled(mc))) + return MDBX_ENODATA; + if (key) { + const page_t *mp = mc->pg[mc->top]; + const node_t *node = page_node(mp, mc->ki[mc->top]); + *key = get_key(node); + } + cASSERT(mc, is_filled(mc)); + if (unlikely(!inner_filled(mc))) { + if (inner_pointed(mc)) + return MDBX_ENODATA; + const page_t *mp = mc->pg[mc->top]; + const node_t *node = page_node(mp, mc->ki[mc->top]); + return node_read(mc, node, data, mp); } - } - assert((size_t)UINT16_MAX - mp->mp_upper >= delta); - mp->mp_upper += (indx_t)delta; + goto fetch_multiple; - return ptr_disp(node, delta); -} + case MDBX_NEXT_MULTIPLE: + if (unlikely(!data)) + return MDBX_EINVAL; + if (unlikely(mc->subcur == nullptr)) + return MDBX_INCOMPATIBLE; + rc = outer_next(mc, key, data, MDBX_NEXT_DUP); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + else { + fetch_multiple: + cASSERT(mc, is_filled(mc) && inner_filled(mc)); + MDBX_cursor *mx = &mc->subcur->cursor; + data->iov_len = page_numkeys(mx->pg[mx->top]) * mx->tree->dupfix_size; + data->iov_base = page_data(mx->pg[mx->top]); + mx->ki[mx->top] = (indx_t)page_numkeys(mx->pg[mx->top]) - 1; + return MDBX_SUCCESS; + } -/* Initial setup of a sorted-dups cursor. - * - * Sorted duplicates are implemented as a sub-database for the given key. - * The duplicate data items are actually keys of the sub-database. - * Operations on the duplicate data items are performed using a sub-cursor - * initialized when the sub-database is first accessed. This function does - * the preliminary setup of the sub-cursor, filling in the fields that - * depend only on the parent DB. - * - * [in] mc The main cursor whose sorted-dups cursor is to be initialized. */ -static int cursor_xinit0(MDBX_cursor *mc) { - MDBX_xcursor *mx = mc->mc_xcursor; - if (!MDBX_DISABLE_VALIDATION && unlikely(mx == nullptr)) { - ERROR("unexpected dupsort-page for non-dupsort db/cursor (dbi %u)", - mc->mc_dbi); - return MDBX_CORRUPTED; - } + case MDBX_PREV_MULTIPLE: + if (unlikely(!data)) + return MDBX_EINVAL; + if (unlikely(mc->subcur == nullptr)) + return MDBX_INCOMPATIBLE; + if (unlikely(!is_filled(mc) || !inner_filled(mc))) + return MDBX_ENODATA; + rc = cursor_sibling_left(&mc->subcur->cursor); + if (likely(rc == MDBX_SUCCESS)) + goto fetch_multiple; + return rc; - mx->mx_cursor.mc_xcursor = NULL; - mx->mx_cursor.mc_next = NULL; - mx->mx_cursor.mc_txn = mc->mc_txn; - mx->mx_cursor.mc_db = &mx->mx_db; - mx->mx_cursor.mc_dbx = &mx->mx_dbx; - mx->mx_cursor.mc_dbi = mc->mc_dbi; - mx->mx_cursor.mc_dbistate = mc->mc_dbistate; - mx->mx_cursor.mc_snum = 0; - mx->mx_cursor.mc_top = 0; - mx->mx_cursor.mc_flags = C_SUB; - STATIC_ASSERT(MDBX_DUPFIXED * 2 == P_LEAF2); - cASSERT(mc, (mc->mc_checking & (P_BRANCH | P_LEAF | P_LEAF2)) == P_LEAF); - mx->mx_cursor.mc_checking = - mc->mc_checking + ((mc->mc_db->md_flags & MDBX_DUPFIXED) << 1); - mx->mx_dbx.md_name.iov_len = 0; - mx->mx_dbx.md_name.iov_base = NULL; - mx->mx_dbx.md_cmp = mc->mc_dbx->md_dcmp; - mx->mx_dbx.md_dcmp = NULL; - mx->mx_dbx.md_klen_min = INT_MAX; - mx->mx_dbx.md_vlen_min = mx->mx_dbx.md_klen_max = mx->mx_dbx.md_vlen_max = 0; - return MDBX_SUCCESS; -} + case MDBX_NEXT_DUP: + case MDBX_NEXT: + case MDBX_NEXT_NODUP: + rc = outer_next(mc, key, data, op); + mc->flags &= ~z_eof_hard; + ((cursor_couple_t *)mc)->inner.cursor.flags &= ~z_eof_hard; + return rc; -/* Final setup of a sorted-dups cursor. - * Sets up the fields that depend on the data from the main cursor. - * [in] mc The main cursor whose sorted-dups cursor is to be initialized. - * [in] node The data containing the MDBX_db record for the sorted-dup database. - */ -static int cursor_xinit1(MDBX_cursor *mc, MDBX_node *node, - const MDBX_page *mp) { - MDBX_xcursor *mx = mc->mc_xcursor; - if (!MDBX_DISABLE_VALIDATION && unlikely(mx == nullptr)) { - ERROR("unexpected dupsort-page for non-dupsort db/cursor (dbi %u)", - mc->mc_dbi); - return MDBX_CORRUPTED; - } + case MDBX_PREV_DUP: + case MDBX_PREV: + case MDBX_PREV_NODUP: + return outer_prev(mc, key, data, op); - const uint8_t flags = node_flags(node); - switch (flags) { - default: - ERROR("invalid node flags %u", flags); - return MDBX_CORRUPTED; - case F_DUPDATA | F_SUBDATA: - if (!MDBX_DISABLE_VALIDATION && - unlikely(node_ds(node) != sizeof(MDBX_db))) { - ERROR("invalid nested-db record size %zu", node_ds(node)); - return MDBX_CORRUPTED; - } - memcpy(&mx->mx_db, node_data(node), sizeof(MDBX_db)); - const txnid_t pp_txnid = mp->mp_txnid; - if (!MDBX_DISABLE_VALIDATION && - unlikely(mx->mx_db.md_mod_txnid > pp_txnid)) { - ERROR("nested-db.md_mod_txnid (%" PRIaTXN ") > page-txnid (%" PRIaTXN ")", - mx->mx_db.md_mod_txnid, pp_txnid); - return MDBX_CORRUPTED; + case MDBX_FIRST: + return outer_first(mc, key, data); + case MDBX_LAST: + return outer_last(mc, key, data); + + case MDBX_LAST_DUP: + case MDBX_FIRST_DUP: + if (unlikely(data == nullptr)) + return MDBX_EINVAL; + if (unlikely(!is_filled(mc))) + return MDBX_ENODATA; + else { + node_t *node = page_node(mc->pg[mc->top], mc->ki[mc->top]); + get_key_optional(node, key); + if ((node_flags(node) & N_DUP) == 0) + return node_read(mc, node, data, mc->pg[mc->top]); + else if (MDBX_DISABLE_VALIDATION || likely(mc->subcur)) + return ((op == MDBX_FIRST_DUP) ? inner_first : inner_last)(&mc->subcur->cursor, data); + else + return unexpected_dupsort(mc); } - mx->mx_cursor.mc_pg[0] = 0; - mx->mx_cursor.mc_snum = 0; - mx->mx_cursor.mc_top = 0; - mx->mx_cursor.mc_flags = C_SUB; break; - case F_DUPDATA: - if (!MDBX_DISABLE_VALIDATION && unlikely(node_ds(node) <= PAGEHDRSZ)) { - ERROR("invalid nested-page size %zu", node_ds(node)); - return MDBX_CORRUPTED; + + case MDBX_SET_UPPERBOUND: + case MDBX_SET_LOWERBOUND: + if (unlikely(key == nullptr || data == nullptr)) + return MDBX_EINVAL; + else { + MDBX_val save_data = *data; + csr_t csr = cursor_seek(mc, key, data, MDBX_SET_RANGE); + rc = csr.err; + if (rc == MDBX_SUCCESS && csr.exact && mc->subcur) { + csr.exact = false; + if (!save_data.iov_base) { + /* Avoiding search nested dupfix hive if no data provided. + * This is changes the semantic of MDBX_SET_LOWERBOUND but avoid + * returning MDBX_BAD_VALSIZE. */ + } else if (is_pointed(&mc->subcur->cursor)) { + *data = save_data; + csr = cursor_seek(&mc->subcur->cursor, data, nullptr, MDBX_SET_RANGE); + rc = csr.err; + if (rc == MDBX_NOTFOUND) { + cASSERT(mc, !csr.exact); + rc = outer_next(mc, key, data, MDBX_NEXT_NODUP); + } + } else { + int cmp = mc->clc->v.cmp(&save_data, data); + csr.exact = (cmp == 0); + if (cmp > 0) + rc = outer_next(mc, key, data, MDBX_NEXT_NODUP); + } + } + if (rc == MDBX_SUCCESS && !csr.exact) + rc = MDBX_RESULT_TRUE; + if (unlikely(op == MDBX_SET_UPPERBOUND)) { + /* minor fixups for MDBX_SET_UPPERBOUND */ + if (rc == MDBX_RESULT_TRUE) + /* already at great-than by MDBX_SET_LOWERBOUND */ + rc = MDBX_SUCCESS; + else if (rc == MDBX_SUCCESS) + /* exactly match, going next */ + rc = outer_next(mc, key, data, MDBX_NEXT); + } } - MDBX_page *fp = node_data(node); - mx->mx_db.md_depth = 1; - mx->mx_db.md_branch_pages = 0; - mx->mx_db.md_leaf_pages = 1; - mx->mx_db.md_overflow_pages = 0; - mx->mx_db.md_entries = page_numkeys(fp); - mx->mx_db.md_root = fp->mp_pgno; - mx->mx_db.md_mod_txnid = mp->mp_txnid; - mx->mx_cursor.mc_snum = 1; - mx->mx_cursor.mc_top = 0; - mx->mx_cursor.mc_flags = C_SUB | C_INITIALIZED; - mx->mx_cursor.mc_pg[0] = fp; - mx->mx_cursor.mc_ki[0] = 0; - mx->mx_db.md_flags = flags_db2sub(mc->mc_db->md_flags); - mx->mx_db.md_xsize = - (mc->mc_db->md_flags & MDBX_DUPFIXED) ? fp->mp_leaf2_ksize : 0; - break; - } + return rc; - if (unlikely(mx->mx_db.md_xsize != mc->mc_db->md_xsize)) { - if (!MDBX_DISABLE_VALIDATION && unlikely(mc->mc_db->md_xsize != 0)) { - ERROR("cursor mismatched nested-db md_xsize %u", mc->mc_db->md_xsize); - return MDBX_CORRUPTED; + /* Doubtless API to positioning of the cursor at a specified key. */ + case MDBX_TO_KEY_LESSER_THAN: + case MDBX_TO_KEY_LESSER_OR_EQUAL: + case MDBX_TO_KEY_EQUAL: + case MDBX_TO_KEY_GREATER_OR_EQUAL: + case MDBX_TO_KEY_GREATER_THAN: + if (unlikely(key == nullptr)) + return MDBX_EINVAL; + else { + csr_t csr = cursor_seek(mc, key, data, MDBX_SET_RANGE); + rc = csr.err; + if (csr.exact) { + cASSERT(mc, csr.err == MDBX_SUCCESS); + if (op == MDBX_TO_KEY_LESSER_THAN) + rc = outer_prev(mc, key, data, MDBX_PREV_NODUP); + else if (op == MDBX_TO_KEY_GREATER_THAN) + rc = outer_next(mc, key, data, MDBX_NEXT_NODUP); + } else if (op < MDBX_TO_KEY_EQUAL && (rc == MDBX_NOTFOUND || rc == MDBX_SUCCESS)) + rc = outer_prev(mc, key, data, MDBX_PREV_NODUP); + else if (op == MDBX_TO_KEY_EQUAL && rc == MDBX_SUCCESS) + rc = MDBX_NOTFOUND; } - if (!MDBX_DISABLE_VALIDATION && - unlikely((mc->mc_db->md_flags & MDBX_DUPFIXED) == 0)) { - ERROR("mismatched nested-db md_flags %u", mc->mc_db->md_flags); - return MDBX_CORRUPTED; + return rc; + + /* Doubtless API to positioning of the cursor at a specified key-value pair + * for multi-value hives. */ + case MDBX_TO_EXACT_KEY_VALUE_LESSER_THAN: + case MDBX_TO_EXACT_KEY_VALUE_LESSER_OR_EQUAL: + case MDBX_TO_EXACT_KEY_VALUE_EQUAL: + case MDBX_TO_EXACT_KEY_VALUE_GREATER_OR_EQUAL: + case MDBX_TO_EXACT_KEY_VALUE_GREATER_THAN: + if (unlikely(key == nullptr || data == nullptr)) + return MDBX_EINVAL; + else { + MDBX_val save_data = *data; + csr_t csr = cursor_seek(mc, key, data, MDBX_SET_KEY); + rc = csr.err; + if (rc == MDBX_SUCCESS) { + cASSERT(mc, csr.exact); + if (inner_pointed(mc)) { + MDBX_cursor *const mx = &mc->subcur->cursor; + csr = cursor_seek(mx, &save_data, nullptr, MDBX_SET_RANGE); + rc = csr.err; + if (csr.exact) { + cASSERT(mc, csr.err == MDBX_SUCCESS); + if (op == MDBX_TO_EXACT_KEY_VALUE_LESSER_THAN) + rc = inner_prev(mx, data); + else if (op == MDBX_TO_EXACT_KEY_VALUE_GREATER_THAN) + rc = inner_next(mx, data); + } else if (op < MDBX_TO_EXACT_KEY_VALUE_EQUAL && (rc == MDBX_NOTFOUND || rc == MDBX_SUCCESS)) + rc = inner_prev(mx, data); + else if (op == MDBX_TO_EXACT_KEY_VALUE_EQUAL && rc == MDBX_SUCCESS) + rc = MDBX_NOTFOUND; + } else { + int cmp = mc->clc->v.cmp(data, &save_data); + switch (op) { + default: + __unreachable(); + case MDBX_TO_EXACT_KEY_VALUE_LESSER_THAN: + rc = (cmp < 0) ? MDBX_SUCCESS : MDBX_NOTFOUND; + break; + case MDBX_TO_EXACT_KEY_VALUE_LESSER_OR_EQUAL: + rc = (cmp <= 0) ? MDBX_SUCCESS : MDBX_NOTFOUND; + break; + case MDBX_TO_EXACT_KEY_VALUE_EQUAL: + rc = (cmp == 0) ? MDBX_SUCCESS : MDBX_NOTFOUND; + break; + case MDBX_TO_EXACT_KEY_VALUE_GREATER_OR_EQUAL: + rc = (cmp >= 0) ? MDBX_SUCCESS : MDBX_NOTFOUND; + break; + case MDBX_TO_EXACT_KEY_VALUE_GREATER_THAN: + rc = (cmp > 0) ? MDBX_SUCCESS : MDBX_NOTFOUND; + break; + } + } + } } - if (!MDBX_DISABLE_VALIDATION && - unlikely(mx->mx_db.md_xsize < mc->mc_dbx->md_vlen_min || - mx->mx_db.md_xsize > mc->mc_dbx->md_vlen_max)) { - ERROR("mismatched nested-db.md_xsize (%u) <> min/max value-length " - "(%zu/%zu)", - mx->mx_db.md_xsize, mc->mc_dbx->md_vlen_min, - mc->mc_dbx->md_vlen_max); - return MDBX_CORRUPTED; + return rc; + + case MDBX_TO_PAIR_LESSER_THAN: + case MDBX_TO_PAIR_LESSER_OR_EQUAL: + case MDBX_TO_PAIR_EQUAL: + case MDBX_TO_PAIR_GREATER_OR_EQUAL: + case MDBX_TO_PAIR_GREATER_THAN: + if (unlikely(key == nullptr || data == nullptr)) + return MDBX_EINVAL; + else { + MDBX_val save_data = *data; + csr_t csr = cursor_seek(mc, key, data, MDBX_SET_RANGE); + rc = csr.err; + if (csr.exact) { + cASSERT(mc, csr.err == MDBX_SUCCESS); + if (inner_pointed(mc)) { + MDBX_cursor *const mx = &mc->subcur->cursor; + csr = cursor_seek(mx, &save_data, nullptr, MDBX_SET_RANGE); + rc = csr.err; + if (csr.exact) { + cASSERT(mc, csr.err == MDBX_SUCCESS); + if (op == MDBX_TO_PAIR_LESSER_THAN) + rc = outer_prev(mc, key, data, MDBX_PREV); + else if (op == MDBX_TO_PAIR_GREATER_THAN) + rc = outer_next(mc, key, data, MDBX_NEXT); + } else if (op < MDBX_TO_PAIR_EQUAL && (rc == MDBX_NOTFOUND || rc == MDBX_SUCCESS)) + rc = outer_prev(mc, key, data, MDBX_PREV); + else if (op == MDBX_TO_PAIR_EQUAL && rc == MDBX_SUCCESS) + rc = MDBX_NOTFOUND; + else if (op > MDBX_TO_PAIR_EQUAL && rc == MDBX_NOTFOUND) + rc = outer_next(mc, key, data, MDBX_NEXT); + } else { + int cmp = mc->clc->v.cmp(data, &save_data); + switch (op) { + default: + __unreachable(); + case MDBX_TO_PAIR_LESSER_THAN: + if (cmp >= 0) + rc = outer_prev(mc, key, data, MDBX_PREV); + break; + case MDBX_TO_PAIR_LESSER_OR_EQUAL: + if (cmp > 0) + rc = outer_prev(mc, key, data, MDBX_PREV); + break; + case MDBX_TO_PAIR_EQUAL: + rc = (cmp == 0) ? MDBX_SUCCESS : MDBX_NOTFOUND; + break; + case MDBX_TO_PAIR_GREATER_OR_EQUAL: + if (cmp < 0) + rc = outer_next(mc, key, data, MDBX_NEXT); + break; + case MDBX_TO_PAIR_GREATER_THAN: + if (cmp <= 0) + rc = outer_next(mc, key, data, MDBX_NEXT); + break; + } + } + } else if (op < MDBX_TO_PAIR_EQUAL && (rc == MDBX_NOTFOUND || rc == MDBX_SUCCESS)) + rc = outer_prev(mc, key, data, MDBX_PREV_NODUP); + else if (op == MDBX_TO_PAIR_EQUAL && rc == MDBX_SUCCESS) + rc = MDBX_NOTFOUND; } - mc->mc_db->md_xsize = mx->mx_db.md_xsize; - mc->mc_dbx->md_vlen_min = mc->mc_dbx->md_vlen_max = mx->mx_db.md_xsize; - } - mx->mx_dbx.md_klen_min = mc->mc_dbx->md_vlen_min; - mx->mx_dbx.md_klen_max = mc->mc_dbx->md_vlen_max; + return rc; - DEBUG("Sub-db -%u root page %" PRIaPGNO, mx->mx_cursor.mc_dbi, - mx->mx_db.md_root); - return MDBX_SUCCESS; + default: + DEBUG("unhandled/unimplemented cursor operation %u", op); + return MDBX_EINVAL; + } } -/* Fixup a sorted-dups cursor due to underlying update. - * Sets up some fields that depend on the data from the main cursor. - * Almost the same as init1, but skips initialization steps if the - * xcursor had already been used. - * [in] mc The main cursor whose sorted-dups cursor is to be fixed up. - * [in] src_mx The xcursor of an up-to-date cursor. - * [in] new_dupdata True if converting from a non-F_DUPDATA item. */ -static int cursor_xinit2(MDBX_cursor *mc, MDBX_xcursor *src_mx, - bool new_dupdata) { - MDBX_xcursor *mx = mc->mc_xcursor; - if (!MDBX_DISABLE_VALIDATION && unlikely(mx == nullptr)) { - ERROR("unexpected dupsort-page for non-dupsort db/cursor (dbi %u)", - mc->mc_dbi); - return MDBX_CORRUPTED; - } +int cursor_check(const MDBX_cursor *mc, int txn_bad_bits) { + if (unlikely(mc == nullptr)) + return MDBX_EINVAL; - if (new_dupdata) { - mx->mx_cursor.mc_snum = 1; - mx->mx_cursor.mc_top = 0; - mx->mx_cursor.mc_flags = C_SUB | C_INITIALIZED; - mx->mx_cursor.mc_ki[0] = 0; + if (unlikely(mc->signature != cur_signature_live)) { + if (mc->signature != cur_signature_ready4dispose) + return MDBX_EBADSIGN; + return (txn_bad_bits > MDBX_TXN_FINISHED) ? MDBX_EINVAL : MDBX_SUCCESS; } - mx->mx_dbx.md_klen_min = src_mx->mx_dbx.md_klen_min; - mx->mx_dbx.md_klen_max = src_mx->mx_dbx.md_klen_max; - mx->mx_dbx.md_cmp = src_mx->mx_dbx.md_cmp; - mx->mx_db = src_mx->mx_db; - mx->mx_cursor.mc_pg[0] = src_mx->mx_cursor.mc_pg[0]; - if (mx->mx_cursor.mc_flags & C_INITIALIZED) { - DEBUG("Sub-db -%u root page %" PRIaPGNO, mx->mx_cursor.mc_dbi, - mx->mx_db.md_root); - } - return MDBX_SUCCESS; -} + /* проверяем что курсор в связном списке для отслеживания, исключение допускается только для read-only операций для + * служебных/временных курсоров на стеке. */ + MDBX_MAYBE_UNUSED char stack_top[sizeof(void *)]; + cASSERT(mc, cursor_is_tracked(mc) || (!(txn_bad_bits & MDBX_TXN_RDONLY) && stack_top < (char *)mc && + (char *)mc - stack_top < (ptrdiff_t)globals.sys_pagesize * 4)); -static __inline int couple_init(MDBX_cursor_couple *couple, const size_t dbi, - const MDBX_txn *const txn, MDBX_db *const db, - MDBX_dbx *const dbx, uint8_t *const dbstate) { - couple->outer.mc_signature = MDBX_MC_LIVE; - couple->outer.mc_next = NULL; - couple->outer.mc_backup = NULL; - couple->outer.mc_dbi = (MDBX_dbi)dbi; - couple->outer.mc_txn = (MDBX_txn *)txn; - couple->outer.mc_db = db; - couple->outer.mc_dbx = dbx; - couple->outer.mc_dbistate = dbstate; - couple->outer.mc_snum = 0; - couple->outer.mc_top = 0; - couple->outer.mc_pg[0] = 0; - couple->outer.mc_flags = 0; - STATIC_ASSERT(CC_BRANCH == P_BRANCH && CC_LEAF == P_LEAF && - CC_OVERFLOW == P_OVERFLOW && CC_LEAF2 == P_LEAF2); - couple->outer.mc_checking = - (AUDIT_ENABLED() || (txn->mt_env->me_flags & MDBX_VALIDATION)) - ? CC_PAGECHECK | CC_LEAF - : CC_LEAF; - couple->outer.mc_ki[0] = 0; - couple->outer.mc_xcursor = NULL; + if (txn_bad_bits) { + int rc = check_txn(mc->txn, txn_bad_bits & ~MDBX_TXN_HAS_CHILD); + if (unlikely(rc != MDBX_SUCCESS)) { + cASSERT(mc, rc != MDBX_RESULT_TRUE); + return rc; + } - int rc = MDBX_SUCCESS; - if (unlikely(*couple->outer.mc_dbistate & DBI_STALE)) { - rc = page_search(&couple->outer, NULL, MDBX_PS_ROOTONLY); - rc = (rc != MDBX_NOTFOUND) ? rc : MDBX_SUCCESS; - } else if (unlikely(dbx->md_klen_max == 0)) { - rc = setup_dbx(dbx, db, txn->mt_env->me_psize); - } + if (likely((mc->txn->flags & MDBX_TXN_HAS_CHILD) == 0)) + return likely(!cursor_dbi_changed(mc)) ? MDBX_SUCCESS : MDBX_BAD_DBI; - if (couple->outer.mc_db->md_flags & MDBX_DUPSORT) { - couple->inner.mx_cursor.mc_signature = MDBX_MC_LIVE; - couple->outer.mc_xcursor = &couple->inner; - rc = cursor_xinit0(&couple->outer); + cASSERT(mc, (mc->txn->flags & MDBX_TXN_RDONLY) == 0 && mc->txn != mc->txn->env->txn && mc->txn->env->txn); + rc = dbi_check(mc->txn->env->txn, cursor_dbi(mc)); if (unlikely(rc != MDBX_SUCCESS)) return rc; - couple->inner.mx_dbx.md_klen_min = couple->outer.mc_dbx->md_vlen_min; - couple->inner.mx_dbx.md_klen_max = couple->outer.mc_dbx->md_vlen_max; + + cASSERT(mc, (mc->txn->flags & MDBX_TXN_RDONLY) == 0 && mc->txn == mc->txn->env->txn); } - return rc; -} -/* Initialize a cursor for a given transaction and database. */ -static int cursor_init(MDBX_cursor *mc, const MDBX_txn *txn, size_t dbi) { - STATIC_ASSERT(offsetof(MDBX_cursor_couple, outer) == 0); - return couple_init(container_of(mc, MDBX_cursor_couple, outer), dbi, txn, - &txn->mt_dbs[dbi], &txn->mt_dbxs[dbi], - &txn->mt_dbistate[dbi]); + return MDBX_SUCCESS; } +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 + +#if MDBX_ENABLE_DBI_SPARSE +size_t dbi_bitmap_ctz_fallback(const MDBX_txn *txn, intptr_t bmi) { + tASSERT(txn, bmi > 0); + bmi &= -bmi; + if (sizeof(txn->dbi_sparse[0]) > 4) { + static const uint8_t debruijn_ctz64[64] = {0, 1, 2, 53, 3, 7, 54, 27, 4, 38, 41, 8, 34, 55, 48, 28, + 62, 5, 39, 46, 44, 42, 22, 9, 24, 35, 59, 56, 49, 18, 29, 11, + 63, 52, 6, 26, 37, 40, 33, 47, 61, 45, 43, 21, 23, 58, 17, 10, + 51, 25, 36, 32, 60, 20, 57, 16, 50, 31, 19, 15, 30, 14, 13, 12}; + return debruijn_ctz64[(UINT64_C(0x022FDD63CC95386D) * (uint64_t)bmi) >> 58]; + } else { + static const uint8_t debruijn_ctz32[32] = {0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8, + 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9}; + return debruijn_ctz32[(UINT32_C(0x077CB531) * (uint32_t)bmi) >> 27]; + } +} +#endif /* MDBX_ENABLE_DBI_SPARSE */ -MDBX_cursor *mdbx_cursor_create(void *context) { - MDBX_cursor_couple *couple = osal_calloc(1, sizeof(MDBX_cursor_couple)); - if (unlikely(!couple)) - return nullptr; - - couple->outer.mc_signature = MDBX_MC_READY4CLOSE; - couple->outer.mc_dbi = UINT_MAX; - couple->mc_userctx = context; - return &couple->outer; +struct dbi_snap_result dbi_snap(const MDBX_env *env, const size_t dbi) { + eASSERT(env, dbi < env->n_dbi); + struct dbi_snap_result r; + uint32_t snap = atomic_load32(&env->dbi_seqs[dbi], mo_AcquireRelease); + do { + r.sequence = snap; + r.flags = env->dbs_flags[dbi]; + snap = atomic_load32(&env->dbi_seqs[dbi], mo_AcquireRelease); + } while (unlikely(snap != r.sequence)); + return r; } -int mdbx_cursor_set_userctx(MDBX_cursor *mc, void *ctx) { - if (unlikely(!mc)) - return MDBX_EINVAL; - - if (unlikely(mc->mc_signature != MDBX_MC_READY4CLOSE && - mc->mc_signature != MDBX_MC_LIVE)) - return MDBX_EBADSIGN; +__noinline int dbi_import(MDBX_txn *txn, const size_t dbi) { + const MDBX_env *const env = txn->env; + if (dbi >= env->n_dbi || !env->dbs_flags[dbi]) + return MDBX_BAD_DBI; - MDBX_cursor_couple *couple = container_of(mc, MDBX_cursor_couple, outer); - couple->mc_userctx = ctx; - return MDBX_SUCCESS; +#if MDBX_ENABLE_DBI_SPARSE + const size_t bitmap_chunk = CHAR_BIT * sizeof(txn->dbi_sparse[0]); + const size_t bitmap_indx = dbi / bitmap_chunk; + const size_t bitmap_mask = (size_t)1 << dbi % bitmap_chunk; + if (dbi >= txn->n_dbi) { + for (size_t i = (txn->n_dbi + bitmap_chunk - 1) / bitmap_chunk; bitmap_indx >= i; ++i) + txn->dbi_sparse[i] = 0; + eASSERT(env, (txn->dbi_sparse[bitmap_indx] & bitmap_mask) == 0); + MDBX_txn *scan = txn; + do { + eASSERT(env, scan->dbi_sparse == txn->dbi_sparse); + eASSERT(env, scan->n_dbi < dbi + 1); + scan->n_dbi = (unsigned)dbi + 1; + scan->dbi_state[dbi] = 0; + scan = scan->parent; + } while (scan /* && scan->dbi_sparse == txn->dbi_sparse */); + txn->dbi_sparse[bitmap_indx] |= bitmap_mask; + goto lindo; + } + if ((txn->dbi_sparse[bitmap_indx] & bitmap_mask) == 0) { + MDBX_txn *scan = txn; + do { + eASSERT(env, scan->dbi_sparse == txn->dbi_sparse); + eASSERT(env, scan->n_dbi == txn->n_dbi); + scan->dbi_state[dbi] = 0; + scan = scan->parent; + } while (scan /* && scan->dbi_sparse == txn->dbi_sparse */); + txn->dbi_sparse[bitmap_indx] |= bitmap_mask; + goto lindo; + } +#else + if (dbi >= txn->n_dbi) { + size_t i = txn->n_dbi; + do + txn->dbi_state[i] = 0; + while (dbi >= ++i); + txn->n_dbi = i; + goto lindo; + } +#endif /* MDBX_ENABLE_DBI_SPARSE */ + + if (!txn->dbi_state[dbi]) { + lindo: + /* dbi-слот еще не инициализирован в транзакции, а хендл не использовался */ + txn->cursors[dbi] = nullptr; + MDBX_txn *const parent = txn->parent; + if (unlikely(parent)) { + /* вложенная пишущая транзакция */ + int rc = dbi_check(parent, dbi); + /* копируем состояние table очищая new-флаги. */ + eASSERT(env, txn->dbi_seqs == parent->dbi_seqs); + txn->dbi_state[dbi] = parent->dbi_state[dbi] & ~(DBI_FRESH | DBI_CREAT | DBI_DIRTY); + if (likely(rc == MDBX_SUCCESS)) { + txn->dbs[dbi] = parent->dbs[dbi]; + if (parent->cursors[dbi]) { + rc = cursor_shadow(parent->cursors[dbi], txn, dbi); + if (unlikely(rc != MDBX_SUCCESS)) { + /* не получилось забекапить курсоры */ + txn->dbi_state[dbi] = DBI_OLDEN | DBI_LINDO | DBI_STALE; + txn->flags |= MDBX_TXN_ERROR; + } + } + } + return rc; + } + txn->dbi_seqs[dbi] = 0; + txn->dbi_state[dbi] = DBI_LINDO; + } else { + eASSERT(env, txn->dbi_seqs[dbi] != env->dbi_seqs[dbi].weak); + if (unlikely(txn->cursors[dbi])) { + /* хендл уже использовался в транзакции и остались висячие курсоры */ + txn->dbi_seqs[dbi] = env->dbi_seqs[dbi].weak; + txn->dbi_state[dbi] = DBI_OLDEN | DBI_LINDO; + return MDBX_DANGLING_DBI; + } + if (unlikely(txn->dbi_state[dbi] & (DBI_OLDEN | DBI_VALID))) { + /* хендл уже использовался в транзакции, но был закрыт или переоткрыт, + * висячих курсоров нет */ + txn->dbi_seqs[dbi] = env->dbi_seqs[dbi].weak; + txn->dbi_state[dbi] = DBI_OLDEN | DBI_LINDO; + return MDBX_BAD_DBI; + } + } + + /* хендл не использовался в транзакции, либо явно пере-отрывается при + * отсутствии висячих курсоров */ + eASSERT(env, (txn->dbi_state[dbi] & (DBI_LINDO | DBI_VALID)) == DBI_LINDO && !txn->cursors[dbi]); + + /* читаем актуальные флаги и sequence */ + struct dbi_snap_result snap = dbi_snap(env, dbi); + txn->dbi_seqs[dbi] = snap.sequence; + if (snap.flags & DB_VALID) { + txn->dbs[dbi].flags = snap.flags & DB_PERSISTENT_FLAGS; + txn->dbi_state[dbi] = (dbi >= CORE_DBS) ? DBI_LINDO | DBI_VALID | DBI_STALE : DBI_LINDO | DBI_VALID; + return MDBX_SUCCESS; + } + return MDBX_BAD_DBI; } -void *mdbx_cursor_get_userctx(const MDBX_cursor *mc) { - if (unlikely(!mc)) - return nullptr; +int dbi_defer_release(MDBX_env *const env, defer_free_item_t *const chain) { + size_t length = 0; + defer_free_item_t *obsolete_chain = nullptr; +#if MDBX_ENABLE_DBI_LOCKFREE + const uint64_t now = osal_monotime(); + defer_free_item_t **scan = &env->defer_free; + if (env->defer_free) { + const uint64_t threshold_1second = osal_16dot16_to_monotime(1 * 65536); + do { + defer_free_item_t *item = *scan; + if (now - item->timestamp < threshold_1second) { + scan = &item->next; + length += 1; + } else { + *scan = item->next; + item->next = obsolete_chain; + obsolete_chain = item; + } + } while (*scan); + } - if (unlikely(mc->mc_signature != MDBX_MC_READY4CLOSE && - mc->mc_signature != MDBX_MC_LIVE)) - return nullptr; + eASSERT(env, *scan == nullptr); + if (chain) { + defer_free_item_t *item = chain; + do { + item->timestamp = now; + item = item->next; + } while (item); + *scan = chain; + } +#else /* MDBX_ENABLE_DBI_LOCKFREE */ + obsolete_chain = chain; +#endif /* MDBX_ENABLE_DBI_LOCKFREE */ - MDBX_cursor_couple *couple = container_of(mc, MDBX_cursor_couple, outer); - return couple->mc_userctx; + ENSURE(env, osal_fastmutex_release(&env->dbi_lock) == MDBX_SUCCESS); + if (length > 42) { +#if defined(_WIN32) || defined(_WIN64) + SwitchToThread(); +#else + sched_yield(); +#endif /* Windows */ + } + while (obsolete_chain) { + defer_free_item_t *item = obsolete_chain; + obsolete_chain = obsolete_chain->next; + osal_free(item); + } + return chain ? MDBX_SUCCESS : MDBX_BAD_DBI; } -int mdbx_cursor_bind(const MDBX_txn *txn, MDBX_cursor *mc, MDBX_dbi dbi) { - if (unlikely(!mc)) - return MDBX_EINVAL; - - if (unlikely(mc->mc_signature != MDBX_MC_READY4CLOSE && - mc->mc_signature != MDBX_MC_LIVE)) - return MDBX_EBADSIGN; +/* Export or close DBI handles opened in this txn. */ +int dbi_update(MDBX_txn *txn, int keep) { + MDBX_env *const env = txn->env; + tASSERT(txn, !txn->parent && txn == env->basal_txn); + bool locked = false; + defer_free_item_t *defer_chain = nullptr; + TXN_FOREACH_DBI_USER(txn, dbi) { + if (likely((txn->dbi_state[dbi] & DBI_CREAT) == 0)) + continue; + if (!locked) { + int err = osal_fastmutex_acquire(&env->dbi_lock); + if (unlikely(err != MDBX_SUCCESS)) + return err; + locked = true; + if (dbi >= env->n_dbi) + /* хендл был закрыт из другого потока пока захватывали блокировку */ + continue; + } + tASSERT(txn, dbi < env->n_dbi); + if (keep) { + env->dbs_flags[dbi] = txn->dbs[dbi].flags | DB_VALID; + } else { + uint32_t seq = dbi_seq_next(env, dbi); + defer_free_item_t *item = env->kvs[dbi].name.iov_base; + if (item) { + env->dbs_flags[dbi] = 0; + env->kvs[dbi].name.iov_len = 0; + env->kvs[dbi].name.iov_base = nullptr; + atomic_store32(&env->dbi_seqs[dbi], seq, mo_AcquireRelease); + osal_flush_incoherent_cpu_writeback(); + item->next = defer_chain; + defer_chain = item; + } else { + eASSERT(env, env->kvs[dbi].name.iov_len == 0); + eASSERT(env, env->dbs_flags[dbi] == 0); + } + } + } - int rc = check_txn(txn, MDBX_TXN_BLOCKED); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; + if (locked) { + size_t i = env->n_dbi; + while ((env->dbs_flags[i - 1] & DB_VALID) == 0) { + --i; + eASSERT(env, i >= CORE_DBS); + eASSERT(env, !env->dbs_flags[i] && !env->kvs[i].name.iov_len && !env->kvs[i].name.iov_base); + } + env->n_dbi = (unsigned)i; + dbi_defer_release(env, defer_chain); + } + return MDBX_SUCCESS; +} - if (unlikely(!check_dbi(txn, dbi, DBI_VALID))) - return MDBX_BAD_DBI; +int dbi_bind(MDBX_txn *txn, const size_t dbi, unsigned user_flags, MDBX_cmp_func *keycmp, MDBX_cmp_func *datacmp) { + const MDBX_env *const env = txn->env; + eASSERT(env, dbi < txn->n_dbi && dbi < env->n_dbi); + eASSERT(env, dbi_state(txn, dbi) & DBI_LINDO); + eASSERT(env, env->dbs_flags[dbi] != DB_POISON); + if ((env->dbs_flags[dbi] & DB_VALID) == 0) { + eASSERT(env, !env->kvs[dbi].clc.k.cmp && !env->kvs[dbi].clc.v.cmp && !env->kvs[dbi].name.iov_len && + !env->kvs[dbi].name.iov_base && !env->kvs[dbi].clc.k.lmax && !env->kvs[dbi].clc.k.lmin && + !env->kvs[dbi].clc.v.lmax && !env->kvs[dbi].clc.v.lmin); + } else { + eASSERT(env, !(txn->dbi_state[dbi] & DBI_VALID) || (txn->dbs[dbi].flags | DB_VALID) == env->dbs_flags[dbi]); + eASSERT(env, env->kvs[dbi].name.iov_base || dbi < CORE_DBS); + } + + /* Если dbi уже использовался, то корректными считаем четыре варианта: + * 1) user_flags равны MDBX_DB_ACCEDE + * = предполагаем что пользователь открывает существующую table, + * при этом код проверки не позволит установить другие компараторы. + * 2) user_flags нулевые, а оба компаратора пустые/нулевые или равны текущим + * = предполагаем что пользователь открывает существующую table + * старым способом с нулевыми с флагами по-умолчанию. + * 3) user_flags совпадают, а компараторы не заданы или те же + * = предполагаем что пользователь открывает table указывая все параметры; + * 4) user_flags отличаются, но table пустая и задан флаг MDBX_CREATE + * = предполагаем что пользователь пересоздает table; + */ + if ((user_flags & ~MDBX_CREATE) != (unsigned)(env->dbs_flags[dbi] & DB_PERSISTENT_FLAGS)) { + /* flags are differs, check other conditions */ + if ((!user_flags && (!keycmp || keycmp == env->kvs[dbi].clc.k.cmp) && + (!datacmp || datacmp == env->kvs[dbi].clc.v.cmp)) || + user_flags == MDBX_DB_ACCEDE) { + user_flags = env->dbs_flags[dbi] & DB_PERSISTENT_FLAGS; + } else if ((user_flags & MDBX_CREATE) == 0) + return /* FIXME: return extended info */ MDBX_INCOMPATIBLE; + else { + if (txn->dbi_state[dbi] & DBI_STALE) { + eASSERT(env, env->dbs_flags[dbi] & DB_VALID); + int err = tbl_fetch(txn, dbi); + if (unlikely(err == MDBX_SUCCESS)) + return err; + } + eASSERT(env, ((env->dbs_flags[dbi] ^ txn->dbs[dbi].flags) & DB_PERSISTENT_FLAGS) == 0); + eASSERT(env, (txn->dbi_state[dbi] & (DBI_LINDO | DBI_VALID | DBI_STALE)) == (DBI_LINDO | DBI_VALID)); + if (unlikely(txn->dbs[dbi].leaf_pages)) + return /* FIXME: return extended info */ MDBX_INCOMPATIBLE; + + /* Пересоздаём table если там пусто */ + if (unlikely(txn->cursors[dbi])) + return MDBX_DANGLING_DBI; + env->dbs_flags[dbi] = DB_POISON; + atomic_store32(&env->dbi_seqs[dbi], dbi_seq_next(env, dbi), mo_AcquireRelease); + + const uint32_t seq = dbi_seq_next(env, dbi); + const uint16_t db_flags = user_flags & DB_PERSISTENT_FLAGS; + eASSERT(env, txn->dbs[dbi].height == 0 && txn->dbs[dbi].items == 0 && txn->dbs[dbi].root == P_INVALID); + env->kvs[dbi].clc.k.cmp = keycmp ? keycmp : builtin_keycmp(user_flags); + env->kvs[dbi].clc.v.cmp = datacmp ? datacmp : builtin_datacmp(user_flags); + txn->dbs[dbi].flags = db_flags; + txn->dbs[dbi].dupfix_size = 0; + if (unlikely(tbl_setup(env, &env->kvs[dbi], &txn->dbs[dbi]))) { + txn->dbi_state[dbi] = DBI_LINDO; + txn->flags |= MDBX_TXN_ERROR; + return MDBX_PROBLEM; + } - if (unlikely(dbi == FREE_DBI && !(txn->mt_flags & MDBX_TXN_RDONLY))) - return MDBX_EACCESS; + env->dbs_flags[dbi] = db_flags | DB_VALID; + atomic_store32(&env->dbi_seqs[dbi], seq, mo_AcquireRelease); + txn->dbi_seqs[dbi] = seq; + txn->dbi_state[dbi] = DBI_LINDO | DBI_VALID | DBI_CREAT | DBI_DIRTY; + txn->flags |= MDBX_TXN_DIRTY; + } + } - if (unlikely(mc->mc_backup)) /* Cursor from parent transaction */ { - cASSERT(mc, mc->mc_signature == MDBX_MC_LIVE); - if (unlikely(mc->mc_dbi != dbi || - /* paranoia */ mc->mc_signature != MDBX_MC_LIVE || - mc->mc_txn != txn)) + if (!keycmp) + keycmp = (env->dbs_flags[dbi] & DB_VALID) ? env->kvs[dbi].clc.k.cmp : builtin_keycmp(user_flags); + if (env->kvs[dbi].clc.k.cmp != keycmp) { + if (env->dbs_flags[dbi] & DB_VALID) return MDBX_EINVAL; - - assert(mc->mc_db == &txn->mt_dbs[dbi]); - assert(mc->mc_dbx == &txn->mt_dbxs[dbi]); - assert(mc->mc_dbi == dbi); - assert(mc->mc_dbistate == &txn->mt_dbistate[dbi]); - return likely(mc->mc_dbi == dbi && - /* paranoia */ mc->mc_signature == MDBX_MC_LIVE && - mc->mc_txn == txn) - ? MDBX_SUCCESS - : MDBX_EINVAL /* Disallow change DBI in nested transactions */; + env->kvs[dbi].clc.k.cmp = keycmp; } - if (mc->mc_signature == MDBX_MC_LIVE) { - if (unlikely(!mc->mc_txn || - mc->mc_txn->mt_signature != MDBX_MT_SIGNATURE)) { - ERROR("Wrong cursor's transaction %p 0x%x", - __Wpedantic_format_voidptr(mc->mc_txn), - mc->mc_txn ? mc->mc_txn->mt_signature : 0); - return MDBX_PROBLEM; - } - if (mc->mc_flags & C_UNTRACK) { - MDBX_cursor **prev = &mc->mc_txn->mt_cursors[mc->mc_dbi]; - while (*prev && *prev != mc) - prev = &(*prev)->mc_next; - cASSERT(mc, *prev == mc); - *prev = mc->mc_next; - } - mc->mc_signature = MDBX_MC_READY4CLOSE; - mc->mc_flags = 0; - mc->mc_dbi = UINT_MAX; - mc->mc_next = NULL; - mc->mc_db = NULL; - mc->mc_dbx = NULL; - mc->mc_dbistate = NULL; + if (!datacmp) + datacmp = (env->dbs_flags[dbi] & DB_VALID) ? env->kvs[dbi].clc.v.cmp : builtin_datacmp(user_flags); + if (env->kvs[dbi].clc.v.cmp != datacmp) { + if (env->dbs_flags[dbi] & DB_VALID) + return MDBX_EINVAL; + env->kvs[dbi].clc.v.cmp = datacmp; } - cASSERT(mc, !(mc->mc_flags & C_UNTRACK)); - - rc = cursor_init(mc, txn, dbi); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - - mc->mc_next = txn->mt_cursors[dbi]; - txn->mt_cursors[dbi] = mc; - mc->mc_flags |= C_UNTRACK; return MDBX_SUCCESS; } -int mdbx_cursor_open(const MDBX_txn *txn, MDBX_dbi dbi, MDBX_cursor **ret) { - if (unlikely(!ret)) - return MDBX_EINVAL; - *ret = NULL; +static inline size_t dbi_namelen(const MDBX_val name) { + return (name.iov_len > sizeof(defer_free_item_t)) ? name.iov_len : sizeof(defer_free_item_t); +} - MDBX_cursor *const mc = mdbx_cursor_create(nullptr); - if (unlikely(!mc)) - return MDBX_ENOMEM; +static int dbi_open_locked(MDBX_txn *txn, unsigned user_flags, MDBX_dbi *dbi, MDBX_cmp_func *keycmp, + MDBX_cmp_func *datacmp, MDBX_val name) { + MDBX_env *const env = txn->env; - int rc = mdbx_cursor_bind(txn, mc, dbi); - if (unlikely(rc != MDBX_SUCCESS)) { - mdbx_cursor_close(mc); - return rc; + /* Cannot mix named table(s) with DUPSORT flags */ + tASSERT(txn, (txn->dbi_state[MAIN_DBI] & (DBI_LINDO | DBI_VALID | DBI_STALE)) == (DBI_LINDO | DBI_VALID)); + if (unlikely(txn->dbs[MAIN_DBI].flags & MDBX_DUPSORT)) { + if (unlikely((user_flags & MDBX_CREATE) == 0)) + return MDBX_NOTFOUND; + if (unlikely(txn->dbs[MAIN_DBI].leaf_pages)) + /* В MainDB есть записи, либо она уже использовалась. */ + return MDBX_INCOMPATIBLE; + + /* Пересоздаём MainDB когда там пусто. */ + tASSERT(txn, + txn->dbs[MAIN_DBI].height == 0 && txn->dbs[MAIN_DBI].items == 0 && txn->dbs[MAIN_DBI].root == P_INVALID); + if (unlikely(txn->cursors[MAIN_DBI])) + return MDBX_DANGLING_DBI; + env->dbs_flags[MAIN_DBI] = DB_POISON; + atomic_store32(&env->dbi_seqs[MAIN_DBI], dbi_seq_next(env, MAIN_DBI), mo_AcquireRelease); + + const uint32_t seq = dbi_seq_next(env, MAIN_DBI); + const uint16_t main_flags = txn->dbs[MAIN_DBI].flags & (MDBX_REVERSEKEY | MDBX_INTEGERKEY); + env->kvs[MAIN_DBI].clc.k.cmp = builtin_keycmp(main_flags); + env->kvs[MAIN_DBI].clc.v.cmp = builtin_datacmp(main_flags); + txn->dbs[MAIN_DBI].flags = main_flags; + txn->dbs[MAIN_DBI].dupfix_size = 0; + int err = tbl_setup(env, &env->kvs[MAIN_DBI], &txn->dbs[MAIN_DBI]); + if (unlikely(err != MDBX_SUCCESS)) { + txn->dbi_state[MAIN_DBI] = DBI_LINDO; + txn->flags |= MDBX_TXN_ERROR; + env->flags |= ENV_FATAL_ERROR; + return err; + } + env->dbs_flags[MAIN_DBI] = main_flags | DB_VALID; + txn->dbi_seqs[MAIN_DBI] = atomic_store32(&env->dbi_seqs[MAIN_DBI], seq, mo_AcquireRelease); + txn->dbi_state[MAIN_DBI] |= DBI_DIRTY; + txn->flags |= MDBX_TXN_DIRTY; } - *ret = mc; - return MDBX_SUCCESS; -} + tASSERT(txn, env->kvs[MAIN_DBI].clc.k.cmp); -int mdbx_cursor_renew(const MDBX_txn *txn, MDBX_cursor *mc) { - return likely(mc) ? mdbx_cursor_bind(txn, mc, mc->mc_dbi) : MDBX_EINVAL; -} + /* Is the DB already open? */ + size_t slot = env->n_dbi; + for (size_t scan = CORE_DBS; scan < env->n_dbi; ++scan) { + if ((env->dbs_flags[scan] & DB_VALID) == 0) { + /* Remember this free slot */ + slot = (slot < scan) ? slot : scan; + continue; + } + if (env->kvs[MAIN_DBI].clc.k.cmp(&name, &env->kvs[scan].name) == 0) { + slot = scan; + int err = dbi_check(txn, slot); + if (err == MDBX_BAD_DBI && txn->dbi_state[slot] == (DBI_OLDEN | DBI_LINDO)) { + /* хендл использовался, стал невалидным, + * но теперь явно пере-открывается в этой транзакци */ + eASSERT(env, !txn->cursors[slot]); + txn->dbi_state[slot] = DBI_LINDO; + err = dbi_check(txn, slot); + } + if (err == MDBX_SUCCESS) { + err = dbi_bind(txn, slot, user_flags, keycmp, datacmp); + if (likely(err == MDBX_SUCCESS)) { + goto done; + } + } + return err; + } + } -int mdbx_cursor_copy(const MDBX_cursor *src, MDBX_cursor *dest) { - if (unlikely(!src)) - return MDBX_EINVAL; - if (unlikely(src->mc_signature != MDBX_MC_LIVE)) - return (src->mc_signature == MDBX_MC_READY4CLOSE) ? MDBX_EINVAL - : MDBX_EBADSIGN; + /* Fail, if no free slot and max hit */ + if (unlikely(slot >= env->max_dbi)) + return MDBX_DBS_FULL; + + if (env->n_dbi == slot) + eASSERT(env, !env->dbs_flags[slot] && !env->kvs[slot].name.iov_len && !env->kvs[slot].name.iov_base); + + env->dbs_flags[slot] = DB_POISON; + atomic_store32(&env->dbi_seqs[slot], dbi_seq_next(env, slot), mo_AcquireRelease); + memset(&env->kvs[slot], 0, sizeof(env->kvs[slot])); + if (env->n_dbi == slot) + env->n_dbi = (unsigned)slot + 1; + eASSERT(env, slot < env->n_dbi); + + int err = dbi_check(txn, slot); + eASSERT(env, err == MDBX_BAD_DBI); + if (err != MDBX_BAD_DBI) + return MDBX_PROBLEM; - int rc = mdbx_cursor_bind(src->mc_txn, dest, src->mc_dbi); + /* Find the DB info */ + MDBX_val body; + cursor_couple_t cx; + int rc = cursor_init(&cx.outer, txn, MAIN_DBI); if (unlikely(rc != MDBX_SUCCESS)) return rc; - - assert(dest->mc_db == src->mc_db); - assert(dest->mc_dbi == src->mc_dbi); - assert(dest->mc_dbx == src->mc_dbx); - assert(dest->mc_dbistate == src->mc_dbistate); -again: - assert(dest->mc_txn == src->mc_txn); - dest->mc_flags ^= (dest->mc_flags ^ src->mc_flags) & ~C_UNTRACK; - dest->mc_top = src->mc_top; - dest->mc_snum = src->mc_snum; - for (size_t i = 0; i < src->mc_snum; ++i) { - dest->mc_ki[i] = src->mc_ki[i]; - dest->mc_pg[i] = src->mc_pg[i]; - } - - if (src->mc_xcursor) { - dest->mc_xcursor->mx_db = src->mc_xcursor->mx_db; - dest->mc_xcursor->mx_dbx = src->mc_xcursor->mx_dbx; - src = &src->mc_xcursor->mx_cursor; - dest = &dest->mc_xcursor->mx_cursor; - goto again; + rc = cursor_seek(&cx.outer, &name, &body, MDBX_SET).err; + if (unlikely(rc != MDBX_SUCCESS)) { + if (rc != MDBX_NOTFOUND || !(user_flags & MDBX_CREATE)) + return rc; + } else { + /* make sure this is actually a table */ + node_t *node = page_node(cx.outer.pg[cx.outer.top], cx.outer.ki[cx.outer.top]); + if (unlikely((node_flags(node) & (N_DUP | N_TREE)) != N_TREE)) + return MDBX_INCOMPATIBLE; + if (!MDBX_DISABLE_VALIDATION && unlikely(body.iov_len != sizeof(tree_t))) { + ERROR("%s/%d: %s %zu", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid table node size", body.iov_len); + return MDBX_CORRUPTED; + } + memcpy(&txn->dbs[slot], body.iov_base, sizeof(tree_t)); } - return MDBX_SUCCESS; -} + /* Done here so we cannot fail after creating a new DB */ + defer_free_item_t *const clone = osal_malloc(dbi_namelen(name)); + if (unlikely(!clone)) + return MDBX_ENOMEM; + memcpy(clone, name.iov_base, name.iov_len); + name.iov_base = clone; -void mdbx_cursor_close(MDBX_cursor *mc) { - if (likely(mc)) { - ENSURE(NULL, mc->mc_signature == MDBX_MC_LIVE || - mc->mc_signature == MDBX_MC_READY4CLOSE); - MDBX_txn *const txn = mc->mc_txn; - if (!mc->mc_backup) { - mc->mc_txn = NULL; - /* Unlink from txn, if tracked. */ - if (mc->mc_flags & C_UNTRACK) { - ENSURE(txn->mt_env, check_txn(txn, 0) == MDBX_SUCCESS); - MDBX_cursor **prev = &txn->mt_cursors[mc->mc_dbi]; - while (*prev && *prev != mc) - prev = &(*prev)->mc_next; - tASSERT(txn, *prev == mc); - *prev = mc->mc_next; - } - mc->mc_signature = 0; - mc->mc_next = mc; - osal_free(mc); - } else { - /* Cursor closed before nested txn ends */ - tASSERT(txn, mc->mc_signature == MDBX_MC_LIVE); - ENSURE(txn->mt_env, check_txn_rw(txn, 0) == MDBX_SUCCESS); - mc->mc_signature = MDBX_MC_WAIT4EOT; - } + uint8_t dbi_state = DBI_LINDO | DBI_VALID | DBI_FRESH; + if (unlikely(rc)) { + /* MDBX_NOTFOUND and MDBX_CREATE: Create new DB */ + tASSERT(txn, rc == MDBX_NOTFOUND); + body.iov_base = memset(&txn->dbs[slot], 0, body.iov_len = sizeof(tree_t)); + txn->dbs[slot].root = P_INVALID; + txn->dbs[slot].mod_txnid = txn->txnid; + txn->dbs[slot].flags = user_flags & DB_PERSISTENT_FLAGS; + cx.outer.next = txn->cursors[MAIN_DBI]; + txn->cursors[MAIN_DBI] = &cx.outer; + rc = cursor_put_checklen(&cx.outer, &name, &body, N_TREE | MDBX_NOOVERWRITE); + txn->cursors[MAIN_DBI] = cx.outer.next; + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + + dbi_state |= DBI_DIRTY | DBI_CREAT; + txn->flags |= MDBX_TXN_DIRTY; + tASSERT(txn, (txn->dbi_state[MAIN_DBI] & DBI_DIRTY) != 0); } -} -MDBX_txn *mdbx_cursor_txn(const MDBX_cursor *mc) { - if (unlikely(!mc || mc->mc_signature != MDBX_MC_LIVE)) - return NULL; - MDBX_txn *txn = mc->mc_txn; - if (unlikely(!txn || txn->mt_signature != MDBX_MT_SIGNATURE)) - return NULL; - if (unlikely(txn->mt_flags & MDBX_TXN_FINISHED)) - return NULL; - return txn; -} + /* Got info, register DBI in this txn */ + const uint32_t seq = dbi_seq_next(env, slot); + eASSERT(env, env->dbs_flags[slot] == DB_POISON && !txn->cursors[slot] && + (txn->dbi_state[slot] & (DBI_LINDO | DBI_VALID)) == DBI_LINDO); + txn->dbi_state[slot] = dbi_state; + memcpy(&txn->dbs[slot], body.iov_base, sizeof(txn->dbs[slot])); + env->dbs_flags[slot] = txn->dbs[slot].flags; + rc = dbi_bind(txn, slot, user_flags, keycmp, datacmp); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; -MDBX_dbi mdbx_cursor_dbi(const MDBX_cursor *mc) { - if (unlikely(!mc || mc->mc_signature != MDBX_MC_LIVE)) - return UINT_MAX; - return mc->mc_dbi; + env->kvs[slot].name = name; + env->dbs_flags[slot] = txn->dbs[slot].flags | DB_VALID; + txn->dbi_seqs[slot] = atomic_store32(&env->dbi_seqs[slot], seq, mo_AcquireRelease); + +done: + *dbi = (MDBX_dbi)slot; + tASSERT(txn, slot < txn->n_dbi && (env->dbs_flags[slot] & DB_VALID) != 0); + eASSERT(env, dbi_check(txn, slot) == MDBX_SUCCESS); + return MDBX_SUCCESS; + +bailout: + eASSERT(env, !txn->cursors[slot] && !env->kvs[slot].name.iov_len && !env->kvs[slot].name.iov_base); + txn->dbi_state[slot] &= DBI_LINDO | DBI_OLDEN; + env->dbs_flags[slot] = 0; + osal_free(clone); + if (slot + 1 == env->n_dbi) + txn->n_dbi = env->n_dbi = (unsigned)slot; + return rc; } -/* Return the count of duplicate data items for the current key */ -int mdbx_cursor_count(const MDBX_cursor *mc, size_t *countp) { - if (unlikely(mc == NULL)) +int dbi_open(MDBX_txn *txn, const MDBX_val *const name, unsigned user_flags, MDBX_dbi *dbi, MDBX_cmp_func *keycmp, + MDBX_cmp_func *datacmp) { + if (unlikely(!dbi)) return MDBX_EINVAL; + *dbi = 0; - if (unlikely(mc->mc_signature != MDBX_MC_LIVE)) - return (mc->mc_signature == MDBX_MC_READY4CLOSE) ? MDBX_EINVAL - : MDBX_EBADSIGN; + if (user_flags != MDBX_ACCEDE && unlikely(!check_table_flags(user_flags & ~MDBX_CREATE))) + return MDBX_EINVAL; - int rc = check_txn(mc->mc_txn, MDBX_TXN_BLOCKED); + int rc = check_txn(txn, MDBX_TXN_BLOCKED); if (unlikely(rc != MDBX_SUCCESS)) return rc; - if (unlikely(countp == NULL || !(mc->mc_flags & C_INITIALIZED))) - return MDBX_EINVAL; + if ((user_flags & MDBX_CREATE) && unlikely(txn->flags & MDBX_TXN_RDONLY)) + return MDBX_EACCESS; - if (!mc->mc_snum) { - *countp = 0; - return MDBX_NOTFOUND; + /* main table? */ + if (unlikely(name == MDBX_CHK_MAIN || name->iov_base == MDBX_CHK_MAIN)) { + rc = dbi_bind(txn, MAIN_DBI, user_flags, keycmp, datacmp); + if (likely(rc == MDBX_SUCCESS)) + *dbi = MAIN_DBI; + return rc; } - - MDBX_page *mp = mc->mc_pg[mc->mc_top]; - if ((mc->mc_flags & C_EOF) && mc->mc_ki[mc->mc_top] >= page_numkeys(mp)) { - *countp = 0; - return MDBX_NOTFOUND; + if (unlikely(name == MDBX_CHK_GC || name->iov_base == MDBX_CHK_GC)) { + rc = dbi_bind(txn, FREE_DBI, user_flags, keycmp, datacmp); + if (likely(rc == MDBX_SUCCESS)) + *dbi = FREE_DBI; + return rc; } + if (unlikely(name == MDBX_CHK_META || name->iov_base == MDBX_CHK_META)) + return MDBX_EINVAL; + if (unlikely(name->iov_len > txn->env->leaf_nodemax - NODESIZE - sizeof(tree_t))) + return MDBX_EINVAL; - *countp = 1; - if (mc->mc_xcursor != NULL) { - MDBX_node *node = page_node(mp, mc->mc_ki[mc->mc_top]); - if (node_flags(node) & F_DUPDATA) { - cASSERT(mc, mc->mc_xcursor && - (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)); - *countp = unlikely(mc->mc_xcursor->mx_db.md_entries > PTRDIFF_MAX) - ? PTRDIFF_MAX - : (size_t)mc->mc_xcursor->mx_db.md_entries; +#if MDBX_ENABLE_DBI_LOCKFREE + /* Is the DB already open? */ + const MDBX_env *const env = txn->env; + bool have_free_slot = env->n_dbi < env->max_dbi; + for (size_t i = CORE_DBS; i < env->n_dbi; ++i) { + if ((env->dbs_flags[i] & DB_VALID) == 0) { + have_free_slot = true; + continue; } - } - return MDBX_SUCCESS; -} -/* Replace the key for a branch node with a new key. - * Set MDBX_TXN_ERROR on failure. - * [in] mc Cursor pointing to the node to operate on. - * [in] key The new key to use. - * Returns 0 on success, non-zero on failure. */ -static int update_key(MDBX_cursor *mc, const MDBX_val *key) { - MDBX_page *mp; - MDBX_node *node; - size_t len; - ptrdiff_t delta, ksize, oksize; - intptr_t ptr, i, nkeys, indx; - DKBUF_DEBUG; + struct dbi_snap_result snap = dbi_snap(env, i); + const MDBX_val snap_name = env->kvs[i].name; + const uint32_t main_seq = atomic_load32(&env->dbi_seqs[MAIN_DBI], mo_AcquireRelease); + MDBX_cmp_func *const snap_cmp = env->kvs[MAIN_DBI].clc.k.cmp; + if (unlikely(!(snap.flags & DB_VALID) || !snap_name.iov_base || !snap_name.iov_len || !snap_cmp)) + /* похоже на столкновение с параллельно работающим обновлением */ + goto slowpath_locking; - cASSERT(mc, cursor_is_tracked(mc)); - indx = mc->mc_ki[mc->mc_top]; - mp = mc->mc_pg[mc->mc_top]; - node = page_node(mp, indx); - ptr = mp->mp_ptrs[indx]; -#if MDBX_DEBUG - MDBX_val k2; - k2.iov_base = node_key(node); - k2.iov_len = node_ks(node); - DEBUG("update key %zi (offset %zu) [%s] to [%s] on page %" PRIaPGNO, indx, - ptr, DVAL_DEBUG(&k2), DKEY_DEBUG(key), mp->mp_pgno); -#endif /* MDBX_DEBUG */ + const bool name_match = snap_cmp(&snap_name, name) == 0; + if (unlikely(snap.sequence != atomic_load32(&env->dbi_seqs[i], mo_AcquireRelease) || + main_seq != atomic_load32(&env->dbi_seqs[MAIN_DBI], mo_AcquireRelease) || + snap.flags != env->dbs_flags[i] || snap_name.iov_base != env->kvs[i].name.iov_base || + snap_name.iov_len != env->kvs[i].name.iov_len)) + /* похоже на столкновение с параллельно работающим обновлением */ + goto slowpath_locking; - /* Sizes must be 2-byte aligned. */ - ksize = EVEN(key->iov_len); - oksize = EVEN(node_ks(node)); - delta = ksize - oksize; + if (!name_match) + continue; - /* Shift node contents if EVEN(key length) changed. */ - if (delta) { - if (delta > (int)page_room(mp)) { - /* not enough space left, do a delete and split */ - DEBUG("Not enough room, delta = %zd, splitting...", delta); - pgno_t pgno = node_pgno(node); - node_del(mc, 0); - int err = page_split(mc, key, NULL, pgno, MDBX_SPLIT_REPLACE); - if (err == MDBX_SUCCESS && AUDIT_ENABLED()) - err = cursor_check_updating(mc); - return err; + osal_flush_incoherent_cpu_writeback(); + if (user_flags != MDBX_ACCEDE && + (((user_flags ^ snap.flags) & DB_PERSISTENT_FLAGS) || (keycmp && keycmp != env->kvs[i].clc.k.cmp) || + (datacmp && datacmp != env->kvs[i].clc.v.cmp))) + /* есть подозрение что пользователь открывает таблицу с другими флагами/атрибутами + * или другими компараторами, поэтому уходим в безопасный режим */ + goto slowpath_locking; + + rc = dbi_check(txn, i); + if (rc == MDBX_BAD_DBI && txn->dbi_state[i] == (DBI_OLDEN | DBI_LINDO)) { + /* хендл использовался, стал невалидным, + * но теперь явно пере-открывается в этой транзакци */ + eASSERT(env, !txn->cursors[i]); + txn->dbi_state[i] = DBI_LINDO; + rc = dbi_check(txn, i); } - - nkeys = page_numkeys(mp); - for (i = 0; i < nkeys; i++) { - if (mp->mp_ptrs[i] <= ptr) { - cASSERT(mc, mp->mp_ptrs[i] >= delta); - mp->mp_ptrs[i] -= (indx_t)delta; - } + if (likely(rc == MDBX_SUCCESS)) { + if (unlikely(snap.sequence != atomic_load32(&env->dbi_seqs[i], mo_AcquireRelease) || + main_seq != atomic_load32(&env->dbi_seqs[MAIN_DBI], mo_AcquireRelease) || + snap.flags != env->dbs_flags[i] || snap_name.iov_base != env->kvs[i].name.iov_base || + snap_name.iov_len != env->kvs[i].name.iov_len)) + /* похоже на столкновение с параллельно работающим обновлением */ + goto slowpath_locking; + rc = dbi_bind(txn, i, user_flags, keycmp, datacmp); + if (likely(rc == MDBX_SUCCESS)) + *dbi = (MDBX_dbi)i; } + return rc; + } - void *const base = ptr_disp(mp, mp->mp_upper + PAGEHDRSZ); - len = ptr - mp->mp_upper + NODESIZE; - memmove(ptr_disp(base, -delta), base, len); - cASSERT(mc, mp->mp_upper >= delta); - mp->mp_upper -= (indx_t)delta; + /* Fail, if no free slot and max hit */ + if (unlikely(!have_free_slot)) + return MDBX_DBS_FULL; - node = page_node(mp, indx); +slowpath_locking: + +#endif /* MDBX_ENABLE_DBI_LOCKFREE */ + + rc = osal_fastmutex_acquire(&txn->env->dbi_lock); + if (likely(rc == MDBX_SUCCESS)) { + rc = dbi_open_locked(txn, user_flags, dbi, keycmp, datacmp, *name); + ENSURE(txn->env, osal_fastmutex_release(&txn->env->dbi_lock) == MDBX_SUCCESS); } + return rc; +} - /* But even if no shift was needed, update ksize */ - node_set_ks(node, key->iov_len); +__cold struct dbi_rename_result dbi_rename_locked(MDBX_txn *txn, MDBX_dbi dbi, MDBX_val new_name) { + struct dbi_rename_result pair; + pair.defer = nullptr; + pair.err = dbi_check(txn, dbi); + if (unlikely(pair.err != MDBX_SUCCESS)) + return pair; + + MDBX_env *const env = txn->env; + MDBX_val old_name = env->kvs[dbi].name; + if (env->kvs[MAIN_DBI].clc.k.cmp(&new_name, &old_name) == 0 && MDBX_DEBUG == 0) + return pair; + + cursor_couple_t cx; + pair.err = cursor_init(&cx.outer, txn, MAIN_DBI); + if (unlikely(pair.err != MDBX_SUCCESS)) + return pair; + pair.err = cursor_seek(&cx.outer, &new_name, nullptr, MDBX_SET).err; + if (unlikely(pair.err != MDBX_NOTFOUND)) { + pair.err = (pair.err == MDBX_SUCCESS) ? MDBX_KEYEXIST : pair.err; + return pair; + } + + pair.defer = osal_malloc(dbi_namelen(new_name)); + if (unlikely(!pair.defer)) { + pair.err = MDBX_ENOMEM; + return pair; + } + new_name.iov_base = memcpy(pair.defer, new_name.iov_base, new_name.iov_len); + + cx.outer.next = txn->cursors[MAIN_DBI]; + txn->cursors[MAIN_DBI] = &cx.outer; + + MDBX_val data = {&txn->dbs[dbi], sizeof(tree_t)}; + pair.err = cursor_put_checklen(&cx.outer, &new_name, &data, N_TREE | MDBX_NOOVERWRITE); + if (likely(pair.err == MDBX_SUCCESS)) { + pair.err = cursor_seek(&cx.outer, &old_name, nullptr, MDBX_SET).err; + if (likely(pair.err == MDBX_SUCCESS)) + pair.err = cursor_del(&cx.outer, N_TREE); + if (likely(pair.err == MDBX_SUCCESS)) { + pair.defer = env->kvs[dbi].name.iov_base; + env->kvs[dbi].name = new_name; + } else + txn->flags |= MDBX_TXN_ERROR; + } - if (likely(key->iov_len /* to avoid UBSAN traps*/ != 0)) - memcpy(node_key(node), key->iov_base, key->iov_len); - return MDBX_SUCCESS; + txn->cursors[MAIN_DBI] = cx.outer.next; + return pair; } -/* Move a node from csrc to cdst. */ -static int node_move(MDBX_cursor *csrc, MDBX_cursor *cdst, bool fromleft) { - int rc; - DKBUF_DEBUG; +static defer_free_item_t *dbi_close_locked(MDBX_env *env, MDBX_dbi dbi) { + eASSERT(env, dbi >= CORE_DBS); + if (unlikely(dbi >= env->n_dbi)) + return nullptr; - MDBX_page *psrc = csrc->mc_pg[csrc->mc_top]; - MDBX_page *pdst = cdst->mc_pg[cdst->mc_top]; - cASSERT(csrc, PAGETYPE_WHOLE(psrc) == PAGETYPE_WHOLE(pdst)); - cASSERT(csrc, csrc->mc_dbi == cdst->mc_dbi); - cASSERT(csrc, csrc->mc_top == cdst->mc_top); - if (unlikely(PAGETYPE_WHOLE(psrc) != PAGETYPE_WHOLE(pdst))) { - bailout: - ERROR("Wrong or mismatch pages's types (src %d, dst %d) to move node", - PAGETYPE_WHOLE(psrc), PAGETYPE_WHOLE(pdst)); - csrc->mc_txn->mt_flags |= MDBX_TXN_ERROR; - return MDBX_PROBLEM; + const uint32_t seq = dbi_seq_next(env, dbi); + defer_free_item_t *defer_item = env->kvs[dbi].name.iov_base; + if (likely(defer_item)) { + env->dbs_flags[dbi] = 0; + env->kvs[dbi].name.iov_len = 0; + env->kvs[dbi].name.iov_base = nullptr; + atomic_store32(&env->dbi_seqs[dbi], seq, mo_AcquireRelease); + osal_flush_incoherent_cpu_writeback(); + defer_item->next = nullptr; + + if (env->n_dbi == dbi + 1) { + size_t i = env->n_dbi; + do { + --i; + eASSERT(env, i >= CORE_DBS); + eASSERT(env, !env->dbs_flags[i] && !env->kvs[i].name.iov_len && !env->kvs[i].name.iov_base); + } while (i > CORE_DBS && !env->kvs[i - 1].name.iov_base); + env->n_dbi = (unsigned)i; + } } - MDBX_val key4move; - switch (PAGETYPE_WHOLE(psrc)) { - case P_BRANCH: { - const MDBX_node *srcnode = page_node(psrc, csrc->mc_ki[csrc->mc_top]); - cASSERT(csrc, node_flags(srcnode) == 0); - const pgno_t srcpg = node_pgno(srcnode); - key4move.iov_len = node_ks(srcnode); - key4move.iov_base = node_key(srcnode); + return defer_item; +} - if (csrc->mc_ki[csrc->mc_top] == 0) { - const size_t snum = csrc->mc_snum; - cASSERT(csrc, snum > 0); - /* must find the lowest key below src */ - rc = page_search_lowest(csrc); - MDBX_page *lowest_page = csrc->mc_pg[csrc->mc_top]; - if (unlikely(rc)) - return rc; - cASSERT(csrc, IS_LEAF(lowest_page)); - if (unlikely(!IS_LEAF(lowest_page))) - goto bailout; - if (IS_LEAF2(lowest_page)) { - key4move.iov_len = csrc->mc_db->md_xsize; - key4move.iov_base = page_leaf2key(lowest_page, 0, key4move.iov_len); - } else { - const MDBX_node *lowest_node = page_node(lowest_page, 0); - key4move.iov_len = node_ks(lowest_node); - key4move.iov_base = node_key(lowest_node); +__cold const tree_t *dbi_dig(const MDBX_txn *txn, const size_t dbi, tree_t *fallback) { + const MDBX_txn *dig = txn; + do { + tASSERT(txn, txn->n_dbi == dig->n_dbi); + const uint8_t state = dbi_state(dig, dbi); + if (state & DBI_LINDO) + switch (state & (DBI_VALID | DBI_STALE | DBI_OLDEN)) { + case DBI_VALID: + case DBI_OLDEN: + return dig->dbs + dbi; + case 0: + return fallback; + case DBI_VALID | DBI_STALE: + case DBI_OLDEN | DBI_STALE: + break; + default: + tASSERT(txn, !!"unexpected dig->dbi_state[dbi]"); } + dig = dig->parent; + } while (dig); + return fallback; +} - /* restore cursor after mdbx_page_search_lowest() */ - csrc->mc_snum = (uint8_t)snum; - csrc->mc_top = (uint8_t)snum - 1; - csrc->mc_ki[csrc->mc_top] = 0; - - /* paranoia */ - cASSERT(csrc, psrc == csrc->mc_pg[csrc->mc_top]); - cASSERT(csrc, IS_BRANCH(psrc)); - if (unlikely(!IS_BRANCH(psrc))) - goto bailout; - } - - if (cdst->mc_ki[cdst->mc_top] == 0) { - const size_t snum = cdst->mc_snum; - cASSERT(csrc, snum > 0); - MDBX_cursor mn; - cursor_copy(cdst, &mn); - /* must find the lowest key below dst */ - rc = page_search_lowest(&mn); - if (unlikely(rc)) - return rc; - MDBX_page *const lowest_page = mn.mc_pg[mn.mc_top]; - cASSERT(cdst, IS_LEAF(lowest_page)); - if (unlikely(!IS_LEAF(lowest_page))) - goto bailout; - MDBX_val key; - if (IS_LEAF2(lowest_page)) { - key.iov_len = mn.mc_db->md_xsize; - key.iov_base = page_leaf2key(lowest_page, 0, key.iov_len); - } else { - MDBX_node *lowest_node = page_node(lowest_page, 0); - key.iov_len = node_ks(lowest_node); - key.iov_base = node_key(lowest_node); - } +int dbi_close_release(MDBX_env *env, MDBX_dbi dbi) { return dbi_defer_release(env, dbi_close_locked(env, dbi)); } +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 - /* restore cursor after mdbx_page_search_lowest() */ - mn.mc_snum = (uint8_t)snum; - mn.mc_top = (uint8_t)snum - 1; - mn.mc_ki[mn.mc_top] = 0; - - const intptr_t delta = - EVEN(key.iov_len) - EVEN(node_ks(page_node(mn.mc_pg[mn.mc_top], 0))); - const intptr_t needed = - branch_size(cdst->mc_txn->mt_env, &key4move) + delta; - const intptr_t have = page_room(pdst); - if (unlikely(needed > have)) - return MDBX_RESULT_TRUE; +static inline size_t dpl_size2bytes(ptrdiff_t size) { + assert(size > CURSOR_STACK_SIZE && (size_t)size <= PAGELIST_LIMIT); +#if MDBX_DPL_PREALLOC_FOR_RADIXSORT + size += size; +#endif /* MDBX_DPL_PREALLOC_FOR_RADIXSORT */ + STATIC_ASSERT(MDBX_ASSUME_MALLOC_OVERHEAD + sizeof(dpl_t) + + (PAGELIST_LIMIT * (MDBX_DPL_PREALLOC_FOR_RADIXSORT + 1)) * sizeof(dp_t) + + MDBX_PNL_GRANULATE * sizeof(void *) * 2 < + SIZE_MAX / 4 * 3); + size_t bytes = ceil_powerof2(MDBX_ASSUME_MALLOC_OVERHEAD + sizeof(dpl_t) + size * sizeof(dp_t), + MDBX_PNL_GRANULATE * sizeof(void *) * 2) - + MDBX_ASSUME_MALLOC_OVERHEAD; + return bytes; +} - if (unlikely((rc = page_touch(csrc)) || (rc = page_touch(cdst)))) - return rc; - psrc = csrc->mc_pg[csrc->mc_top]; - pdst = cdst->mc_pg[cdst->mc_top]; +static inline size_t dpl_bytes2size(const ptrdiff_t bytes) { + size_t size = (bytes - sizeof(dpl_t)) / sizeof(dp_t); +#if MDBX_DPL_PREALLOC_FOR_RADIXSORT + size >>= 1; +#endif /* MDBX_DPL_PREALLOC_FOR_RADIXSORT */ + assert(size > CURSOR_STACK_SIZE && size <= PAGELIST_LIMIT + MDBX_PNL_GRANULATE); + return size; +} - WITH_CURSOR_TRACKING(mn, rc = update_key(&mn, &key)); - if (unlikely(rc)) - return rc; - } else { - const size_t needed = branch_size(cdst->mc_txn->mt_env, &key4move); - const size_t have = page_room(pdst); - if (unlikely(needed > have)) - return MDBX_RESULT_TRUE; +void dpl_free(MDBX_txn *txn) { + if (likely(txn->tw.dirtylist)) { + osal_free(txn->tw.dirtylist); + txn->tw.dirtylist = nullptr; + } +} - if (unlikely((rc = page_touch(csrc)) || (rc = page_touch(cdst)))) - return rc; - psrc = csrc->mc_pg[csrc->mc_top]; - pdst = cdst->mc_pg[cdst->mc_top]; - } +dpl_t *dpl_reserve(MDBX_txn *txn, size_t size) { + tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0); + tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); - DEBUG("moving %s-node %u [%s] on page %" PRIaPGNO - " to node %u on page %" PRIaPGNO, - "branch", csrc->mc_ki[csrc->mc_top], DKEY_DEBUG(&key4move), - psrc->mp_pgno, cdst->mc_ki[cdst->mc_top], pdst->mp_pgno); - /* Add the node to the destination page. */ - rc = node_add_branch(cdst, cdst->mc_ki[cdst->mc_top], &key4move, srcpg); - } break; + size_t bytes = dpl_size2bytes((size < PAGELIST_LIMIT) ? size : PAGELIST_LIMIT); + dpl_t *const dl = osal_realloc(txn->tw.dirtylist, bytes); + if (likely(dl)) { +#ifdef osal_malloc_usable_size + bytes = osal_malloc_usable_size(dl); +#endif /* osal_malloc_usable_size */ + dl->detent = dpl_bytes2size(bytes); + tASSERT(txn, txn->tw.dirtylist == nullptr || dl->length <= dl->detent); + txn->tw.dirtylist = dl; + } + return dl; +} - case P_LEAF: { - /* Mark src and dst as dirty. */ - if (unlikely((rc = page_touch(csrc)) || (rc = page_touch(cdst)))) - return rc; - psrc = csrc->mc_pg[csrc->mc_top]; - pdst = cdst->mc_pg[cdst->mc_top]; - const MDBX_node *srcnode = page_node(psrc, csrc->mc_ki[csrc->mc_top]); - MDBX_val data; - data.iov_len = node_ds(srcnode); - data.iov_base = node_data(srcnode); - key4move.iov_len = node_ks(srcnode); - key4move.iov_base = node_key(srcnode); - DEBUG("moving %s-node %u [%s] on page %" PRIaPGNO - " to node %u on page %" PRIaPGNO, - "leaf", csrc->mc_ki[csrc->mc_top], DKEY_DEBUG(&key4move), - psrc->mp_pgno, cdst->mc_ki[cdst->mc_top], pdst->mp_pgno); - /* Add the node to the destination page. */ - rc = node_add_leaf(cdst, cdst->mc_ki[cdst->mc_top], &key4move, &data, - node_flags(srcnode)); - } break; +int dpl_alloc(MDBX_txn *txn) { + tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0); + tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); - case P_LEAF | P_LEAF2: { - /* Mark src and dst as dirty. */ - if (unlikely((rc = page_touch(csrc)) || (rc = page_touch(cdst)))) - return rc; - psrc = csrc->mc_pg[csrc->mc_top]; - pdst = cdst->mc_pg[cdst->mc_top]; - key4move.iov_len = csrc->mc_db->md_xsize; - key4move.iov_base = - page_leaf2key(psrc, csrc->mc_ki[csrc->mc_top], key4move.iov_len); - DEBUG("moving %s-node %u [%s] on page %" PRIaPGNO - " to node %u on page %" PRIaPGNO, - "leaf2", csrc->mc_ki[csrc->mc_top], DKEY_DEBUG(&key4move), - psrc->mp_pgno, cdst->mc_ki[cdst->mc_top], pdst->mp_pgno); - /* Add the node to the destination page. */ - rc = node_add_leaf2(cdst, cdst->mc_ki[cdst->mc_top], &key4move); - } break; + const size_t wanna = (txn->env->options.dp_initial < txn->geo.upper) ? txn->env->options.dp_initial : txn->geo.upper; +#if MDBX_FORCE_ASSERTIONS || MDBX_DEBUG + if (txn->tw.dirtylist) + /* обнуляем чтобы не сработал ассерт внутри dpl_reserve() */ + txn->tw.dirtylist->sorted = txn->tw.dirtylist->length = 0; +#endif /* asertions enabled */ + if (unlikely(!txn->tw.dirtylist || txn->tw.dirtylist->detent < wanna || txn->tw.dirtylist->detent > wanna + wanna) && + unlikely(!dpl_reserve(txn, wanna))) + return MDBX_ENOMEM; - default: - assert(false); - goto bailout; - } + dpl_clear(txn->tw.dirtylist); + return MDBX_SUCCESS; +} - if (unlikely(rc != MDBX_SUCCESS)) - return rc; +#define MDBX_DPL_EXTRACT_KEY(ptr) ((ptr)->pgno) +RADIXSORT_IMPL(dp, dp_t, MDBX_DPL_EXTRACT_KEY, MDBX_DPL_PREALLOC_FOR_RADIXSORT, 1) - /* Delete the node from the source page. */ - node_del(csrc, key4move.iov_len); +#define DP_SORT_CMP(first, last) ((first).pgno < (last).pgno) +SORT_IMPL(dp_sort, false, dp_t, DP_SORT_CMP) - cASSERT(csrc, psrc == csrc->mc_pg[csrc->mc_top]); - cASSERT(cdst, pdst == cdst->mc_pg[cdst->mc_top]); - cASSERT(csrc, PAGETYPE_WHOLE(psrc) == PAGETYPE_WHOLE(pdst)); +__hot __noinline dpl_t *dpl_sort_slowpath(const MDBX_txn *txn) { + tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0); + tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); - { - /* Adjust other cursors pointing to mp */ - MDBX_cursor *m2, *m3; - const MDBX_dbi dbi = csrc->mc_dbi; - cASSERT(csrc, csrc->mc_top == cdst->mc_top); - if (fromleft) { - /* If we're adding on the left, bump others up */ - for (m2 = csrc->mc_txn->mt_cursors[dbi]; m2; m2 = m2->mc_next) { - m3 = (csrc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2; - if (!(m3->mc_flags & C_INITIALIZED) || m3->mc_top < csrc->mc_top) - continue; - if (m3 != cdst && m3->mc_pg[csrc->mc_top] == pdst && - m3->mc_ki[csrc->mc_top] >= cdst->mc_ki[csrc->mc_top]) { - m3->mc_ki[csrc->mc_top]++; - } - if (m3 != csrc && m3->mc_pg[csrc->mc_top] == psrc && - m3->mc_ki[csrc->mc_top] == csrc->mc_ki[csrc->mc_top]) { - m3->mc_pg[csrc->mc_top] = pdst; - m3->mc_ki[csrc->mc_top] = cdst->mc_ki[cdst->mc_top]; - cASSERT(csrc, csrc->mc_top > 0); - m3->mc_ki[csrc->mc_top - 1]++; - } - if (XCURSOR_INITED(m3) && IS_LEAF(psrc)) - XCURSOR_REFRESH(m3, m3->mc_pg[csrc->mc_top], m3->mc_ki[csrc->mc_top]); - } + dpl_t *dl = txn->tw.dirtylist; + assert(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID); + const size_t unsorted = dl->length - dl->sorted; + if (likely(unsorted < MDBX_RADIXSORT_THRESHOLD) || unlikely(!dp_radixsort(dl->items + 1, dl->length))) { + if (dl->sorted > unsorted / 4 + 4 && + (MDBX_DPL_PREALLOC_FOR_RADIXSORT || dl->length + unsorted < dl->detent + dpl_gap_mergesort)) { + dp_t *const sorted_begin = dl->items + 1; + dp_t *const sorted_end = sorted_begin + dl->sorted; + dp_t *const end = + dl->items + (MDBX_DPL_PREALLOC_FOR_RADIXSORT ? dl->length + dl->length + 1 : dl->detent + dpl_reserve_gap); + dp_t *const tmp = end - unsorted; + assert(dl->items + dl->length + 1 < tmp); + /* copy unsorted to the end of allocated space and sort it */ + memcpy(tmp, sorted_end, unsorted * sizeof(dp_t)); + dp_sort(tmp, tmp + unsorted); + /* merge two parts from end to begin */ + dp_t *__restrict w = dl->items + dl->length; + dp_t *__restrict l = dl->items + dl->sorted; + dp_t *__restrict r = end - 1; + do { + const bool cmp = expect_with_probability(l->pgno > r->pgno, 0, .5); +#if defined(__LCC__) || __CLANG_PREREQ(13, 0) || !MDBX_HAVE_CMOV + *w = cmp ? *l-- : *r--; +#else + *w = cmp ? *l : *r; + l -= cmp; + r += (ptrdiff_t)cmp - 1; +#endif + } while (likely(--w > l)); + assert(r == tmp - 1); + assert(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID); + if (ASSERT_ENABLED()) + for (size_t i = 0; i <= dl->length; ++i) + assert(dl->items[i].pgno < dl->items[i + 1].pgno); } else { - /* Adding on the right, bump others down */ - for (m2 = csrc->mc_txn->mt_cursors[dbi]; m2; m2 = m2->mc_next) { - m3 = (csrc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2; - if (m3 == csrc) - continue; - if (!(m3->mc_flags & C_INITIALIZED) || m3->mc_top < csrc->mc_top) - continue; - if (m3->mc_pg[csrc->mc_top] == psrc) { - if (!m3->mc_ki[csrc->mc_top]) { - m3->mc_pg[csrc->mc_top] = pdst; - m3->mc_ki[csrc->mc_top] = cdst->mc_ki[cdst->mc_top]; - cASSERT(csrc, csrc->mc_top > 0); - m3->mc_ki[csrc->mc_top - 1]--; - } else { - m3->mc_ki[csrc->mc_top]--; - } - if (XCURSOR_INITED(m3) && IS_LEAF(psrc)) - XCURSOR_REFRESH(m3, m3->mc_pg[csrc->mc_top], - m3->mc_ki[csrc->mc_top]); - } - } + dp_sort(dl->items + 1, dl->items + dl->length + 1); + assert(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID); } + } else { + assert(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID); } + dl->sorted = dl->length; + return dl; +} - /* Update the parent separators. */ - if (csrc->mc_ki[csrc->mc_top] == 0) { - cASSERT(csrc, csrc->mc_top > 0); - if (csrc->mc_ki[csrc->mc_top - 1] != 0) { - MDBX_val key; - if (IS_LEAF2(psrc)) { - key.iov_len = psrc->mp_leaf2_ksize; - key.iov_base = page_leaf2key(psrc, 0, key.iov_len); - } else { - MDBX_node *srcnode = page_node(psrc, 0); - key.iov_len = node_ks(srcnode); - key.iov_base = node_key(srcnode); - } - DEBUG("update separator for source page %" PRIaPGNO " to [%s]", - psrc->mp_pgno, DKEY_DEBUG(&key)); - MDBX_cursor mn; - cursor_copy(csrc, &mn); - cASSERT(csrc, mn.mc_snum > 0); - mn.mc_snum--; - mn.mc_top--; - /* We want rebalance to find mn when doing fixups */ - WITH_CURSOR_TRACKING(mn, rc = update_key(&mn, &key)); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - } - if (IS_BRANCH(psrc)) { - const MDBX_val nullkey = {0, 0}; - const indx_t ix = csrc->mc_ki[csrc->mc_top]; - csrc->mc_ki[csrc->mc_top] = 0; - rc = update_key(csrc, &nullkey); - csrc->mc_ki[csrc->mc_top] = ix; - cASSERT(csrc, rc == MDBX_SUCCESS); - } - } +/* Returns the index of the first dirty-page whose pgno + * member is greater than or equal to id. */ +#define DP_SEARCH_CMP(dp, id) ((dp).pgno < (id)) +SEARCH_IMPL(dp_bsearch, dp_t, pgno_t, DP_SEARCH_CMP) - if (cdst->mc_ki[cdst->mc_top] == 0) { - cASSERT(cdst, cdst->mc_top > 0); - if (cdst->mc_ki[cdst->mc_top - 1] != 0) { - MDBX_val key; - if (IS_LEAF2(pdst)) { - key.iov_len = pdst->mp_leaf2_ksize; - key.iov_base = page_leaf2key(pdst, 0, key.iov_len); - } else { - MDBX_node *srcnode = page_node(pdst, 0); - key.iov_len = node_ks(srcnode); - key.iov_base = node_key(srcnode); - } - DEBUG("update separator for destination page %" PRIaPGNO " to [%s]", - pdst->mp_pgno, DKEY_DEBUG(&key)); - MDBX_cursor mn; - cursor_copy(cdst, &mn); - cASSERT(cdst, mn.mc_snum > 0); - mn.mc_snum--; - mn.mc_top--; - /* We want rebalance to find mn when doing fixups */ - WITH_CURSOR_TRACKING(mn, rc = update_key(&mn, &key)); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - } - if (IS_BRANCH(pdst)) { - const MDBX_val nullkey = {0, 0}; - const indx_t ix = cdst->mc_ki[cdst->mc_top]; - cdst->mc_ki[cdst->mc_top] = 0; - rc = update_key(cdst, &nullkey); - cdst->mc_ki[cdst->mc_top] = ix; - cASSERT(cdst, rc == MDBX_SUCCESS); +__hot __noinline size_t dpl_search(const MDBX_txn *txn, pgno_t pgno) { + tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0); + tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); + + dpl_t *dl = txn->tw.dirtylist; + assert(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID); + if (AUDIT_ENABLED()) { + for (const dp_t *ptr = dl->items + dl->sorted; --ptr > dl->items;) { + assert(ptr[0].pgno < ptr[1].pgno); + assert(ptr[0].pgno >= NUM_METAS); } } - return MDBX_SUCCESS; + switch (dl->length - dl->sorted) { + default: + /* sort a whole */ + dpl_sort_slowpath(txn); + break; + case 0: + /* whole sorted cases */ + break; + +#define LINEAR_SEARCH_CASE(N) \ + case N: \ + if (dl->items[dl->length - N + 1].pgno == pgno) \ + return dl->length - N + 1; \ + __fallthrough + + /* use linear scan until the threshold */ + LINEAR_SEARCH_CASE(7); /* fall through */ + LINEAR_SEARCH_CASE(6); /* fall through */ + LINEAR_SEARCH_CASE(5); /* fall through */ + LINEAR_SEARCH_CASE(4); /* fall through */ + LINEAR_SEARCH_CASE(3); /* fall through */ + LINEAR_SEARCH_CASE(2); /* fall through */ + case 1: + if (dl->items[dl->length].pgno == pgno) + return dl->length; + /* continue bsearch on the sorted part */ + break; + } + return dp_bsearch(dl->items + 1, dl->sorted, pgno) - dl->items; } -/* Merge one page into another. - * - * The nodes from the page pointed to by csrc will be copied to the page - * pointed to by cdst and then the csrc page will be freed. - * - * [in] csrc Cursor pointing to the source page. - * [in] cdst Cursor pointing to the destination page. - * - * Returns 0 on success, non-zero on failure. */ -static int page_merge(MDBX_cursor *csrc, MDBX_cursor *cdst) { - MDBX_val key; - int rc; +const page_t *debug_dpl_find(const MDBX_txn *txn, const pgno_t pgno) { + tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0); + const dpl_t *dl = txn->tw.dirtylist; + if (dl) { + tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); + assert(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID); + for (size_t i = dl->length; i > dl->sorted; --i) + if (dl->items[i].pgno == pgno) + return dl->items[i].ptr; - cASSERT(csrc, csrc != cdst); - cASSERT(csrc, cursor_is_tracked(csrc)); - cASSERT(cdst, cursor_is_tracked(cdst)); - const MDBX_page *const psrc = csrc->mc_pg[csrc->mc_top]; - MDBX_page *pdst = cdst->mc_pg[cdst->mc_top]; - DEBUG("merging page %" PRIaPGNO " into %" PRIaPGNO, psrc->mp_pgno, - pdst->mp_pgno); - - cASSERT(csrc, PAGETYPE_WHOLE(psrc) == PAGETYPE_WHOLE(pdst)); - cASSERT(csrc, csrc->mc_dbi == cdst->mc_dbi && csrc->mc_db == cdst->mc_db); - cASSERT(csrc, csrc->mc_snum > 1); /* can't merge root page */ - cASSERT(cdst, cdst->mc_snum > 1); - cASSERT(cdst, cdst->mc_snum < cdst->mc_db->md_depth || - IS_LEAF(cdst->mc_pg[cdst->mc_db->md_depth - 1])); - cASSERT(csrc, csrc->mc_snum < csrc->mc_db->md_depth || - IS_LEAF(csrc->mc_pg[csrc->mc_db->md_depth - 1])); - const int pagetype = PAGETYPE_WHOLE(psrc); + if (dl->sorted) { + const size_t i = dp_bsearch(dl->items + 1, dl->sorted, pgno) - dl->items; + if (dl->items[i].pgno == pgno) + return dl->items[i].ptr; + } + } else { + tASSERT(txn, (txn->flags & MDBX_WRITEMAP) != 0 && !MDBX_AVOID_MSYNC); + } + return nullptr; +} - /* Move all nodes from src to dst */ - const size_t dst_nkeys = page_numkeys(pdst); - const size_t src_nkeys = page_numkeys(psrc); - cASSERT(cdst, dst_nkeys + src_nkeys >= (IS_LEAF(psrc) ? 1u : 2u)); - if (likely(src_nkeys)) { - size_t j = dst_nkeys; - if (unlikely(pagetype & P_LEAF2)) { - /* Mark dst as dirty. */ - rc = page_touch(cdst); - cASSERT(cdst, rc != MDBX_RESULT_TRUE); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; +void dpl_remove_ex(const MDBX_txn *txn, size_t i, size_t npages) { + tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0); + tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); - key.iov_len = csrc->mc_db->md_xsize; - key.iov_base = page_data(psrc); - size_t i = 0; - do { - rc = node_add_leaf2(cdst, j++, &key); - cASSERT(cdst, rc != MDBX_RESULT_TRUE); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - key.iov_base = ptr_disp(key.iov_base, key.iov_len); - } while (++i != src_nkeys); - } else { - MDBX_node *srcnode = page_node(psrc, 0); - key.iov_len = node_ks(srcnode); - key.iov_base = node_key(srcnode); - if (pagetype & P_BRANCH) { - MDBX_cursor mn; - cursor_copy(csrc, &mn); - /* must find the lowest key below src */ - rc = page_search_lowest(&mn); - cASSERT(csrc, rc != MDBX_RESULT_TRUE); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; + dpl_t *dl = txn->tw.dirtylist; + assert((intptr_t)i > 0 && i <= dl->length); + assert(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID); + dl->pages_including_loose -= npages; + dl->sorted -= dl->sorted >= i; + dl->length -= 1; + memmove(dl->items + i, dl->items + i + 1, (dl->length - i + 2) * sizeof(dl->items[0])); + assert(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID); +} - const MDBX_page *mp = mn.mc_pg[mn.mc_top]; - if (likely(!IS_LEAF2(mp))) { - cASSERT(&mn, IS_LEAF(mp)); - const MDBX_node *lowest = page_node(mp, 0); - key.iov_len = node_ks(lowest); - key.iov_base = node_key(lowest); - } else { - cASSERT(&mn, mn.mc_top > csrc->mc_top); - key.iov_len = mp->mp_leaf2_ksize; - key.iov_base = page_leaf2key(mp, mn.mc_ki[mn.mc_top], key.iov_len); - } - cASSERT(&mn, key.iov_len >= csrc->mc_dbx->md_klen_min); - cASSERT(&mn, key.iov_len <= csrc->mc_dbx->md_klen_max); +int __must_check_result dpl_append(MDBX_txn *txn, pgno_t pgno, page_t *page, size_t npages) { + tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0); + tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); + const dp_t dp = {page, pgno, (pgno_t)npages}; + if ((txn->flags & MDBX_WRITEMAP) == 0) { + size_t *const ptr = ptr_disp(page, -(ptrdiff_t)sizeof(size_t)); + *ptr = txn->tw.dirtylru; + } - const size_t dst_room = page_room(pdst); - const size_t src_used = page_used(cdst->mc_txn->mt_env, psrc); - const size_t space_needed = src_used - node_ks(srcnode) + key.iov_len; - if (unlikely(space_needed > dst_room)) - return MDBX_RESULT_TRUE; - } - - /* Mark dst as dirty. */ - rc = page_touch(cdst); - cASSERT(cdst, rc != MDBX_RESULT_TRUE); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - - size_t i = 0; - while (true) { - if (pagetype & P_LEAF) { - MDBX_val data; - data.iov_len = node_ds(srcnode); - data.iov_base = node_data(srcnode); - rc = node_add_leaf(cdst, j++, &key, &data, node_flags(srcnode)); - } else { - cASSERT(csrc, node_flags(srcnode) == 0); - rc = node_add_branch(cdst, j++, &key, node_pgno(srcnode)); - } - cASSERT(cdst, rc != MDBX_RESULT_TRUE); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - - if (++i == src_nkeys) - break; - srcnode = page_node(psrc, i); - key.iov_len = node_ks(srcnode); - key.iov_base = node_key(srcnode); + dpl_t *dl = txn->tw.dirtylist; + tASSERT(txn, dl->length <= PAGELIST_LIMIT + MDBX_PNL_GRANULATE); + tASSERT(txn, dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID); + if (AUDIT_ENABLED()) { + for (size_t i = dl->length; i > 0; --i) { + assert(dl->items[i].pgno != dp.pgno); + if (unlikely(dl->items[i].pgno == dp.pgno)) { + ERROR("Page %u already exist in the DPL at %zu", dp.pgno, i); + return MDBX_PROBLEM; } } - - pdst = cdst->mc_pg[cdst->mc_top]; - DEBUG("dst page %" PRIaPGNO " now has %zu keys (%.1f%% filled)", - pdst->mp_pgno, page_numkeys(pdst), - page_fill(cdst->mc_txn->mt_env, pdst)); - - cASSERT(csrc, psrc == csrc->mc_pg[csrc->mc_top]); - cASSERT(cdst, pdst == cdst->mc_pg[cdst->mc_top]); - } - - /* Unlink the src page from parent and add to free list. */ - csrc->mc_top--; - node_del(csrc, 0); - if (csrc->mc_ki[csrc->mc_top] == 0) { - const MDBX_val nullkey = {0, 0}; - rc = update_key(csrc, &nullkey); - cASSERT(csrc, rc != MDBX_RESULT_TRUE); - if (unlikely(rc != MDBX_SUCCESS)) { - csrc->mc_top++; - return rc; - } } - csrc->mc_top++; - - cASSERT(csrc, psrc == csrc->mc_pg[csrc->mc_top]); - cASSERT(cdst, pdst == cdst->mc_pg[cdst->mc_top]); - - { - /* Adjust other cursors pointing to mp */ - MDBX_cursor *m2, *m3; - const MDBX_dbi dbi = csrc->mc_dbi; - const size_t top = csrc->mc_top; - for (m2 = csrc->mc_txn->mt_cursors[dbi]; m2; m2 = m2->mc_next) { - m3 = (csrc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2; - if (m3 == csrc || top >= m3->mc_snum) - continue; - if (m3->mc_pg[top] == psrc) { - m3->mc_pg[top] = pdst; - cASSERT(m3, dst_nkeys + m3->mc_ki[top] <= UINT16_MAX); - m3->mc_ki[top] += (indx_t)dst_nkeys; - m3->mc_ki[top - 1] = cdst->mc_ki[top - 1]; - } else if (m3->mc_pg[top - 1] == csrc->mc_pg[top - 1] && - m3->mc_ki[top - 1] > csrc->mc_ki[top - 1]) { - m3->mc_ki[top - 1]--; - } - if (XCURSOR_INITED(m3) && IS_LEAF(psrc)) - XCURSOR_REFRESH(m3, m3->mc_pg[top], m3->mc_ki[top]); + if (unlikely(dl->length == dl->detent)) { + if (unlikely(dl->detent >= PAGELIST_LIMIT)) { + ERROR("DPL is full (PAGELIST_LIMIT %zu)", PAGELIST_LIMIT); + return MDBX_TXN_FULL; } + const size_t size = (dl->detent < MDBX_PNL_INITIAL * 42) ? dl->detent + dl->detent : dl->detent + dl->detent / 2; + dl = dpl_reserve(txn, size); + if (unlikely(!dl)) + return MDBX_ENOMEM; + tASSERT(txn, dl->length < dl->detent); } - rc = page_retire(csrc, (MDBX_page *)psrc); - cASSERT(csrc, rc != MDBX_RESULT_TRUE); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - - cASSERT(cdst, cdst->mc_db->md_entries > 0); - cASSERT(cdst, cdst->mc_snum <= cdst->mc_db->md_depth); - cASSERT(cdst, cdst->mc_top > 0); - cASSERT(cdst, cdst->mc_snum == cdst->mc_top + 1); - MDBX_page *const top_page = cdst->mc_pg[cdst->mc_top]; - const indx_t top_indx = cdst->mc_ki[cdst->mc_top]; - const unsigned save_snum = cdst->mc_snum; - const uint16_t save_depth = cdst->mc_db->md_depth; - cursor_pop(cdst); - rc = rebalance(cdst); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - - cASSERT(cdst, cdst->mc_db->md_entries > 0); - cASSERT(cdst, cdst->mc_snum <= cdst->mc_db->md_depth); - cASSERT(cdst, cdst->mc_snum == cdst->mc_top + 1); + /* Сортировка нужна для быстрого поиска, используем несколько тактик: + * 1) Сохраняем упорядоченность при естественной вставке в нужном порядке. + * 2) Добавляем в не-сортированный хвост, который сортируем и сливаем + * с отсортированной головой по необходимости, а пока хвост короткий + * ищем в нём сканированием, избегая большой пересортировки. + * 3) Если не-сортированный хвост короткий, а добавляемый элемент близок + * к концу отсортированной головы, то выгоднее сразу вставить элемент + * в нужное место. + * + * Алгоритмически: + * - добавлять в не-сортированный хвост следует только если вставка сильно + * дорогая, т.е. если целевая позиция элемента сильно далека от конца; + * - для быстрой проверки достаточно сравнить добавляемый элемент с отстоящим + * от конца на максимально-приемлемое расстояние; + * - если список короче, либо элемент в этой позиции меньше вставляемого, + * то следует перемещать элементы и вставлять в отсортированную голову; + * - если не-сортированный хвост длиннее, либо элемент в этой позиции больше, + * то следует добавлять в не-сортированный хвост. */ -#if MDBX_ENABLE_PGOP_STAT - cdst->mc_txn->mt_env->me_lck->mti_pgop_stat.merge.weak += 1; -#endif /* MDBX_ENABLE_PGOP_STAT */ + dl->pages_including_loose += npages; + dp_t *i = dl->items + dl->length; - if (IS_LEAF(cdst->mc_pg[cdst->mc_top])) { - /* LY: don't touch cursor if top-page is a LEAF */ - cASSERT(cdst, IS_LEAF(cdst->mc_pg[cdst->mc_top]) || - PAGETYPE_WHOLE(cdst->mc_pg[cdst->mc_top]) == pagetype); - return MDBX_SUCCESS; - } + const ptrdiff_t pivot = (ptrdiff_t)dl->length - dpl_insertion_threshold; +#if MDBX_HAVE_CMOV + const pgno_t pivot_pgno = + dl->items[(dl->length < dpl_insertion_threshold) ? 0 : dl->length - dpl_insertion_threshold].pgno; +#endif /* MDBX_HAVE_CMOV */ - cASSERT(cdst, page_numkeys(top_page) == dst_nkeys + src_nkeys); + /* copy the stub beyond the end */ + i[2] = i[1]; + dl->length += 1; - if (unlikely(pagetype != PAGETYPE_WHOLE(top_page))) { - /* LY: LEAF-page becomes BRANCH, unable restore cursor's stack */ - goto bailout; - } + if (likely(pivot <= (ptrdiff_t)dl->sorted) && +#if MDBX_HAVE_CMOV + pivot_pgno < dp.pgno) { +#else + (pivot <= 0 || dl->items[pivot].pgno < dp.pgno)) { +#endif /* MDBX_HAVE_CMOV */ + dl->sorted += 1; - if (top_page == cdst->mc_pg[cdst->mc_top]) { - /* LY: don't touch cursor if prev top-page already on the top */ - cASSERT(cdst, cdst->mc_ki[cdst->mc_top] == top_indx); - cASSERT(cdst, IS_LEAF(cdst->mc_pg[cdst->mc_top]) || - PAGETYPE_WHOLE(cdst->mc_pg[cdst->mc_top]) == pagetype); - return MDBX_SUCCESS; + /* сдвигаем несортированный хвост */ + while (i >= dl->items + dl->sorted) { +#if !defined(__GNUC__) /* пытаемся избежать вызова memmove() */ + i[1] = *i; +#elif MDBX_WORDBITS == 64 && (defined(__SIZEOF_INT128__) || (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128)) + STATIC_ASSERT(sizeof(dp) == sizeof(__uint128_t)); + ((__uint128_t *)i)[1] = *(volatile __uint128_t *)i; +#else + i[1].ptr = i->ptr; + i[1].pgno = i->pgno; + i[1].npages = i->npages; +#endif + --i; + } + /* ищем нужную позицию сдвигая отсортированные элементы */ + while (i->pgno > pgno) { + tASSERT(txn, i > dl->items); + i[1] = *i; + --i; + } + tASSERT(txn, i->pgno < dp.pgno); } - const int new_snum = save_snum - save_depth + cdst->mc_db->md_depth; - if (unlikely(new_snum < 1 || new_snum > cdst->mc_db->md_depth)) { - /* LY: out of range, unable restore cursor's stack */ - goto bailout; - } + i[1] = dp; + assert(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID); + assert(dl->sorted <= dl->length); + return MDBX_SUCCESS; +} - if (top_page == cdst->mc_pg[new_snum - 1]) { - cASSERT(cdst, cdst->mc_ki[new_snum - 1] == top_indx); - /* LY: restore cursor stack */ - cdst->mc_snum = (uint8_t)new_snum; - cdst->mc_top = (uint8_t)new_snum - 1; - cASSERT(cdst, cdst->mc_snum < cdst->mc_db->md_depth || - IS_LEAF(cdst->mc_pg[cdst->mc_db->md_depth - 1])); - cASSERT(cdst, IS_LEAF(cdst->mc_pg[cdst->mc_top]) || - PAGETYPE_WHOLE(cdst->mc_pg[cdst->mc_top]) == pagetype); - return MDBX_SUCCESS; +__cold bool dpl_check(MDBX_txn *txn) { + tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0); + const dpl_t *const dl = txn->tw.dirtylist; + if (!dl) { + tASSERT(txn, (txn->flags & MDBX_WRITEMAP) != 0 && !MDBX_AVOID_MSYNC); + return true; } + tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); - MDBX_page *const stub_page = (MDBX_page *)(~(uintptr_t)top_page); - const indx_t stub_indx = top_indx; - if (save_depth > cdst->mc_db->md_depth && - ((cdst->mc_pg[save_snum - 1] == top_page && - cdst->mc_ki[save_snum - 1] == top_indx) || - (cdst->mc_pg[save_snum - 1] == stub_page && - cdst->mc_ki[save_snum - 1] == stub_indx))) { - /* LY: restore cursor stack */ - cdst->mc_pg[new_snum - 1] = top_page; - cdst->mc_ki[new_snum - 1] = top_indx; - cdst->mc_pg[new_snum] = (MDBX_page *)(~(uintptr_t)cdst->mc_pg[new_snum]); - cdst->mc_ki[new_snum] = ~cdst->mc_ki[new_snum]; - cdst->mc_snum = (uint8_t)new_snum; - cdst->mc_top = (uint8_t)new_snum - 1; - cASSERT(cdst, cdst->mc_snum < cdst->mc_db->md_depth || - IS_LEAF(cdst->mc_pg[cdst->mc_db->md_depth - 1])); - cASSERT(cdst, IS_LEAF(cdst->mc_pg[cdst->mc_top]) || - PAGETYPE_WHOLE(cdst->mc_pg[cdst->mc_top]) == pagetype); - return MDBX_SUCCESS; - } + assert(dl->items[0].pgno == 0 && dl->items[dl->length + 1].pgno == P_INVALID); + tASSERT(txn, + txn->tw.dirtyroom + dl->length == (txn->parent ? txn->parent->tw.dirtyroom : txn->env->options.dp_limit)); -bailout: - /* LY: unable restore cursor's stack */ - cdst->mc_flags &= ~C_INITIALIZED; - return MDBX_CURSOR_FULL; -} + if (!AUDIT_ENABLED()) + return true; -static void cursor_restore(const MDBX_cursor *csrc, MDBX_cursor *cdst) { - cASSERT(cdst, cdst->mc_dbi == csrc->mc_dbi); - cASSERT(cdst, cdst->mc_txn == csrc->mc_txn); - cASSERT(cdst, cdst->mc_db == csrc->mc_db); - cASSERT(cdst, cdst->mc_dbx == csrc->mc_dbx); - cASSERT(cdst, cdst->mc_dbistate == csrc->mc_dbistate); - cdst->mc_snum = csrc->mc_snum; - cdst->mc_top = csrc->mc_top; - cdst->mc_flags = csrc->mc_flags; - cdst->mc_checking = csrc->mc_checking; - - for (size_t i = 0; i < csrc->mc_snum; i++) { - cdst->mc_pg[i] = csrc->mc_pg[i]; - cdst->mc_ki[i] = csrc->mc_ki[i]; - } -} - -/* Copy the contents of a cursor. - * [in] csrc The cursor to copy from. - * [out] cdst The cursor to copy to. */ -static void cursor_copy(const MDBX_cursor *csrc, MDBX_cursor *cdst) { - cASSERT(csrc, csrc->mc_txn->mt_txnid >= - csrc->mc_txn->mt_env->me_lck->mti_oldest_reader.weak); - cdst->mc_dbi = csrc->mc_dbi; - cdst->mc_next = NULL; - cdst->mc_backup = NULL; - cdst->mc_xcursor = NULL; - cdst->mc_txn = csrc->mc_txn; - cdst->mc_db = csrc->mc_db; - cdst->mc_dbx = csrc->mc_dbx; - cdst->mc_dbistate = csrc->mc_dbistate; - cursor_restore(csrc, cdst); -} - -/* Rebalance the tree after a delete operation. - * [in] mc Cursor pointing to the page where rebalancing should begin. - * Returns 0 on success, non-zero on failure. */ -static int rebalance(MDBX_cursor *mc) { - cASSERT(mc, cursor_is_tracked(mc)); - cASSERT(mc, mc->mc_snum > 0); - cASSERT(mc, mc->mc_snum < mc->mc_db->md_depth || - IS_LEAF(mc->mc_pg[mc->mc_db->md_depth - 1])); - const int pagetype = PAGETYPE_WHOLE(mc->mc_pg[mc->mc_top]); + size_t loose = 0, pages = 0; + for (size_t i = dl->length; i > 0; --i) { + const page_t *const dp = dl->items[i].ptr; + if (!dp) + continue; - STATIC_ASSERT(P_BRANCH == 1); - const size_t minkeys = (pagetype & P_BRANCH) + (size_t)1; + tASSERT(txn, dp->pgno == dl->items[i].pgno); + if (unlikely(dp->pgno != dl->items[i].pgno)) + return false; - /* Pages emptier than this are candidates for merging. */ - size_t room_threshold = likely(mc->mc_dbi != FREE_DBI) - ? mc->mc_txn->mt_env->me_merge_threshold - : mc->mc_txn->mt_env->me_merge_threshold_gc; + if ((txn->flags & MDBX_WRITEMAP) == 0) { + const uint32_t age = dpl_age(txn, i); + tASSERT(txn, age < UINT32_MAX / 3); + if (unlikely(age > UINT32_MAX / 3)) + return false; + } - const MDBX_page *const tp = mc->mc_pg[mc->mc_top]; - const size_t numkeys = page_numkeys(tp); - const size_t room = page_room(tp); - DEBUG("rebalancing %s page %" PRIaPGNO - " (has %zu keys, full %.1f%%, used %zu, room %zu bytes )", - (pagetype & P_LEAF) ? "leaf" : "branch", tp->mp_pgno, numkeys, - page_fill(mc->mc_txn->mt_env, tp), page_used(mc->mc_txn->mt_env, tp), - room); - cASSERT(mc, IS_MODIFIABLE(mc->mc_txn, tp)); + tASSERT(txn, dp->flags == P_LOOSE || is_modifable(txn, dp)); + if (dp->flags == P_LOOSE) { + loose += 1; + } else if (unlikely(!is_modifable(txn, dp))) + return false; - if (unlikely(numkeys < minkeys)) { - DEBUG("page %" PRIaPGNO " must be merged due keys < %zu threshold", - tp->mp_pgno, minkeys); - } else if (unlikely(room > room_threshold)) { - DEBUG("page %" PRIaPGNO " should be merged due room %zu > %zu threshold", - tp->mp_pgno, room, room_threshold); - } else { - DEBUG("no need to rebalance page %" PRIaPGNO ", room %zu < %zu threshold", - tp->mp_pgno, room, room_threshold); - cASSERT(mc, mc->mc_db->md_entries > 0); - return MDBX_SUCCESS; - } + const unsigned num = dpl_npages(dl, i); + pages += num; + tASSERT(txn, txn->geo.first_unallocated >= dp->pgno + num); + if (unlikely(txn->geo.first_unallocated < dp->pgno + num)) + return false; - int rc; - if (mc->mc_snum < 2) { - MDBX_page *const mp = mc->mc_pg[0]; - const size_t nkeys = page_numkeys(mp); - cASSERT(mc, (mc->mc_db->md_entries == 0) == (nkeys == 0)); - if (IS_SUBP(mp)) { - DEBUG("%s", "Can't rebalance a subpage, ignoring"); - cASSERT(mc, pagetype & P_LEAF); - return MDBX_SUCCESS; - } - if (nkeys == 0) { - cASSERT(mc, IS_LEAF(mp)); - DEBUG("%s", "tree is completely empty"); - cASSERT(mc, (*mc->mc_dbistate & DBI_DIRTY) != 0); - mc->mc_db->md_root = P_INVALID; - mc->mc_db->md_depth = 0; - cASSERT(mc, mc->mc_db->md_branch_pages == 0 && - mc->mc_db->md_overflow_pages == 0 && - mc->mc_db->md_leaf_pages == 1); - /* Adjust cursors pointing to mp */ - for (MDBX_cursor *m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; - m2 = m2->mc_next) { - MDBX_cursor *m3 = - (mc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2; - if (m3 == mc || !(m3->mc_flags & C_INITIALIZED)) - continue; - if (m3->mc_pg[0] == mp) { - m3->mc_snum = 0; - m3->mc_top = 0; - m3->mc_flags &= ~C_INITIALIZED; - } - } - mc->mc_snum = 0; - mc->mc_top = 0; - mc->mc_flags &= ~C_INITIALIZED; - return page_retire(mc, mp); + if (i < dl->sorted) { + tASSERT(txn, dl->items[i + 1].pgno >= dp->pgno + num); + if (unlikely(dl->items[i + 1].pgno < dp->pgno + num)) + return false; } - if (IS_BRANCH(mp) && nkeys == 1) { - DEBUG("%s", "collapsing root page!"); - mc->mc_db->md_root = node_pgno(page_node(mp, 0)); - rc = page_get(mc, mc->mc_db->md_root, &mc->mc_pg[0], mp->mp_txnid); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - mc->mc_db->md_depth--; - mc->mc_ki[0] = mc->mc_ki[1]; - for (int i = 1; i < mc->mc_db->md_depth; i++) { - mc->mc_pg[i] = mc->mc_pg[i + 1]; - mc->mc_ki[i] = mc->mc_ki[i + 1]; - } - /* Adjust other cursors pointing to mp */ - for (MDBX_cursor *m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; - m2 = m2->mc_next) { - MDBX_cursor *m3 = - (mc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2; - if (m3 == mc || !(m3->mc_flags & C_INITIALIZED)) - continue; - if (m3->mc_pg[0] == mp) { - for (int i = 0; i < mc->mc_db->md_depth; i++) { - m3->mc_pg[i] = m3->mc_pg[i + 1]; - m3->mc_ki[i] = m3->mc_ki[i + 1]; - } - m3->mc_snum--; - m3->mc_top--; - } - } - cASSERT(mc, IS_LEAF(mc->mc_pg[mc->mc_top]) || - PAGETYPE_WHOLE(mc->mc_pg[mc->mc_top]) == pagetype); - cASSERT(mc, mc->mc_snum < mc->mc_db->md_depth || - IS_LEAF(mc->mc_pg[mc->mc_db->md_depth - 1])); - return page_retire(mc, mp); + const size_t rpa = pnl_search(txn->tw.repnl, dp->pgno, txn->geo.first_unallocated); + tASSERT(txn, rpa > MDBX_PNL_GETSIZE(txn->tw.repnl) || txn->tw.repnl[rpa] != dp->pgno); + if (rpa <= MDBX_PNL_GETSIZE(txn->tw.repnl) && unlikely(txn->tw.repnl[rpa] == dp->pgno)) + return false; + if (num > 1) { + const size_t rpb = pnl_search(txn->tw.repnl, dp->pgno + num - 1, txn->geo.first_unallocated); + tASSERT(txn, rpa == rpb); + if (unlikely(rpa != rpb)) + return false; } - DEBUG("root page %" PRIaPGNO " doesn't need rebalancing (flags 0x%x)", - mp->mp_pgno, mp->mp_flags); - return MDBX_SUCCESS; } - /* The parent (branch page) must have at least 2 pointers, - * otherwise the tree is invalid. */ - const size_t pre_top = mc->mc_top - 1; - cASSERT(mc, IS_BRANCH(mc->mc_pg[pre_top])); - cASSERT(mc, !IS_SUBP(mc->mc_pg[0])); - cASSERT(mc, page_numkeys(mc->mc_pg[pre_top]) > 1); + tASSERT(txn, loose == txn->tw.loose_count); + if (unlikely(loose != txn->tw.loose_count)) + return false; - /* Leaf page fill factor is below the threshold. - * Try to move keys from left or right neighbor, or - * merge with a neighbor page. */ + tASSERT(txn, pages == dl->pages_including_loose); + if (unlikely(pages != dl->pages_including_loose)) + return false; - /* Find neighbors. */ - MDBX_cursor mn; - cursor_copy(mc, &mn); - - MDBX_page *left = nullptr, *right = nullptr; - if (mn.mc_ki[pre_top] > 0) { - rc = page_get( - &mn, node_pgno(page_node(mn.mc_pg[pre_top], mn.mc_ki[pre_top] - 1)), - &left, mc->mc_pg[mc->mc_top]->mp_txnid); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - cASSERT(mc, PAGETYPE_WHOLE(left) == PAGETYPE_WHOLE(mc->mc_pg[mc->mc_top])); - } - if (mn.mc_ki[pre_top] + (size_t)1 < page_numkeys(mn.mc_pg[pre_top])) { - rc = page_get( - &mn, - node_pgno(page_node(mn.mc_pg[pre_top], mn.mc_ki[pre_top] + (size_t)1)), - &right, mc->mc_pg[mc->mc_top]->mp_txnid); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - cASSERT(mc, PAGETYPE_WHOLE(right) == PAGETYPE_WHOLE(mc->mc_pg[mc->mc_top])); + for (size_t i = 1; i <= MDBX_PNL_GETSIZE(txn->tw.retired_pages); ++i) { + const page_t *const dp = debug_dpl_find(txn, txn->tw.retired_pages[i]); + tASSERT(txn, !dp); + if (unlikely(dp)) + return false; } - cASSERT(mc, left || right); - const size_t ki_top = mc->mc_ki[mc->mc_top]; - const size_t ki_pre_top = mn.mc_ki[pre_top]; - const size_t nkeys = page_numkeys(mn.mc_pg[mn.mc_top]); + return true; +} - const size_t left_room = left ? page_room(left) : 0; - const size_t right_room = right ? page_room(right) : 0; - const size_t left_nkeys = left ? page_numkeys(left) : 0; - const size_t right_nkeys = right ? page_numkeys(right) : 0; - bool involve = false; -retry: - cASSERT(mc, mc->mc_snum > 1); - if (left_room > room_threshold && left_room >= right_room && - (IS_MODIFIABLE(mc->mc_txn, left) || involve)) { - /* try merge with left */ - cASSERT(mc, left_nkeys >= minkeys); - mn.mc_pg[mn.mc_top] = left; - mn.mc_ki[mn.mc_top - 1] = (indx_t)(ki_pre_top - 1); - mn.mc_ki[mn.mc_top] = (indx_t)(left_nkeys - 1); - mc->mc_ki[mc->mc_top] = 0; - const size_t new_ki = ki_top + left_nkeys; - mn.mc_ki[mn.mc_top] += mc->mc_ki[mn.mc_top] + 1; - /* We want rebalance to find mn when doing fixups */ - WITH_CURSOR_TRACKING(mn, rc = page_merge(mc, &mn)); - if (likely(rc != MDBX_RESULT_TRUE)) { - cursor_restore(&mn, mc); - mc->mc_ki[mc->mc_top] = (indx_t)new_ki; - cASSERT(mc, rc || page_numkeys(mc->mc_pg[mc->mc_top]) >= minkeys); - return rc; - } - } - if (right_room > room_threshold && - (IS_MODIFIABLE(mc->mc_txn, right) || involve)) { - /* try merge with right */ - cASSERT(mc, right_nkeys >= minkeys); - mn.mc_pg[mn.mc_top] = right; - mn.mc_ki[mn.mc_top - 1] = (indx_t)(ki_pre_top + 1); - mn.mc_ki[mn.mc_top] = 0; - mc->mc_ki[mc->mc_top] = (indx_t)nkeys; - WITH_CURSOR_TRACKING(mn, rc = page_merge(&mn, mc)); - if (likely(rc != MDBX_RESULT_TRUE)) { - mc->mc_ki[mc->mc_top] = (indx_t)ki_top; - cASSERT(mc, rc || page_numkeys(mc->mc_pg[mc->mc_top]) >= minkeys); - return rc; - } - } +/*----------------------------------------------------------------------------*/ - if (left_nkeys > minkeys && - (right_nkeys <= left_nkeys || right_room >= left_room) && - (IS_MODIFIABLE(mc->mc_txn, left) || involve)) { - /* try move from left */ - mn.mc_pg[mn.mc_top] = left; - mn.mc_ki[mn.mc_top - 1] = (indx_t)(ki_pre_top - 1); - mn.mc_ki[mn.mc_top] = (indx_t)(left_nkeys - 1); - mc->mc_ki[mc->mc_top] = 0; - WITH_CURSOR_TRACKING(mn, rc = node_move(&mn, mc, true)); - if (likely(rc != MDBX_RESULT_TRUE)) { - mc->mc_ki[mc->mc_top] = (indx_t)(ki_top + 1); - cASSERT(mc, rc || page_numkeys(mc->mc_pg[mc->mc_top]) >= minkeys); - return rc; - } - } - if (right_nkeys > minkeys && (IS_MODIFIABLE(mc->mc_txn, right) || involve)) { - /* try move from right */ - mn.mc_pg[mn.mc_top] = right; - mn.mc_ki[mn.mc_top - 1] = (indx_t)(ki_pre_top + 1); - mn.mc_ki[mn.mc_top] = 0; - mc->mc_ki[mc->mc_top] = (indx_t)nkeys; - WITH_CURSOR_TRACKING(mn, rc = node_move(&mn, mc, false)); - if (likely(rc != MDBX_RESULT_TRUE)) { - mc->mc_ki[mc->mc_top] = (indx_t)ki_top; - cASSERT(mc, rc || page_numkeys(mc->mc_pg[mc->mc_top]) >= minkeys); - return rc; +__noinline void dpl_lru_reduce(MDBX_txn *txn) { + NOTICE("lru-reduce %u -> %u", txn->tw.dirtylru, txn->tw.dirtylru >> 1); + tASSERT(txn, (txn->flags & (MDBX_TXN_RDONLY | MDBX_WRITEMAP)) == 0); + do { + txn->tw.dirtylru >>= 1; + dpl_t *dl = txn->tw.dirtylist; + for (size_t i = 1; i <= dl->length; ++i) { + size_t *const ptr = ptr_disp(dl->items[i].ptr, -(ptrdiff_t)sizeof(size_t)); + *ptr >>= 1; } - } + txn = txn->parent; + } while (txn); +} - if (nkeys >= minkeys) { - mc->mc_ki[mc->mc_top] = (indx_t)ki_top; - if (AUDIT_ENABLED()) - return cursor_check_updating(mc); - return MDBX_SUCCESS; - } +void dpl_sift(MDBX_txn *const txn, pnl_t pl, const bool spilled) { + tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0); + tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); + if (MDBX_PNL_GETSIZE(pl) && txn->tw.dirtylist->length) { + tASSERT(txn, pnl_check_allocated(pl, (size_t)txn->geo.first_unallocated << spilled)); + dpl_t *dl = dpl_sort(txn); - /* Заглушено в ветке v0.12.x, будет работать в v0.13.1 и далее. - * - * if (mc->mc_txn->mt_env->me_options.prefer_waf_insteadof_balance && - * likely(room_threshold > 0)) { - * room_threshold = 0; - * goto retry; - * } - */ - if (likely(!involve) && - (likely(mc->mc_dbi != FREE_DBI) || mc->mc_txn->tw.loose_pages || - MDBX_PNL_GETSIZE(mc->mc_txn->tw.relist) || (mc->mc_flags & C_GCU) || - (mc->mc_txn->mt_flags & MDBX_TXN_DRAINED_GC) || room_threshold)) { - involve = true; - goto retry; - } - if (likely(room_threshold > 0)) { - room_threshold = 0; - goto retry; + /* Scanning in ascend order */ + const intptr_t step = MDBX_PNL_ASCENDING ? 1 : -1; + const intptr_t begin = MDBX_PNL_ASCENDING ? 1 : MDBX_PNL_GETSIZE(pl); + const intptr_t end = MDBX_PNL_ASCENDING ? MDBX_PNL_GETSIZE(pl) + 1 : 0; + tASSERT(txn, pl[begin] <= pl[end - step]); + + size_t w, r = dpl_search(txn, pl[begin] >> spilled); + tASSERT(txn, dl->sorted == dl->length); + for (intptr_t i = begin; r <= dl->length;) { /* scan loop */ + assert(i != end); + tASSERT(txn, !spilled || (pl[i] & 1) == 0); + pgno_t pl_pgno = pl[i] >> spilled; + pgno_t dp_pgno = dl->items[r].pgno; + if (likely(dp_pgno != pl_pgno)) { + const bool cmp = dp_pgno < pl_pgno; + r += cmp; + i += cmp ? 0 : step; + if (likely(i != end)) + continue; + return; + } + + /* update loop */ + unsigned npages; + w = r; + remove_dl: + npages = dpl_npages(dl, r); + dl->pages_including_loose -= npages; + if (!MDBX_AVOID_MSYNC || !(txn->flags & MDBX_WRITEMAP)) + page_shadow_release(txn->env, dl->items[r].ptr, npages); + ++r; + next_i: + i += step; + if (unlikely(i == end)) { + while (r <= dl->length) + dl->items[w++] = dl->items[r++]; + } else { + while (r <= dl->length) { + assert(i != end); + tASSERT(txn, !spilled || (pl[i] & 1) == 0); + pl_pgno = pl[i] >> spilled; + dp_pgno = dl->items[r].pgno; + if (dp_pgno < pl_pgno) + dl->items[w++] = dl->items[r++]; + else if (dp_pgno > pl_pgno) + goto next_i; + else + goto remove_dl; + } + } + dl->sorted = dpl_setlen(dl, w - 1); + txn->tw.dirtyroom += r - w; + tASSERT(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length == + (txn->parent ? txn->parent->tw.dirtyroom : txn->env->options.dp_limit)); + return; + } } - ERROR("Unable to merge/rebalance %s page %" PRIaPGNO - " (has %zu keys, full %.1f%%, used %zu, room %zu bytes )", - (pagetype & P_LEAF) ? "leaf" : "branch", tp->mp_pgno, numkeys, - page_fill(mc->mc_txn->mt_env, tp), page_used(mc->mc_txn->mt_env, tp), - room); - return MDBX_PROBLEM; } -__cold static int page_check(const MDBX_cursor *const mc, - const MDBX_page *const mp) { - DKBUF; - int rc = MDBX_SUCCESS; - if (unlikely(mp->mp_pgno < MIN_PAGENO || mp->mp_pgno > MAX_PAGENO)) - rc = bad_page(mp, "invalid pgno (%u)\n", mp->mp_pgno); - - MDBX_env *const env = mc->mc_txn->mt_env; - const ptrdiff_t offset = ptr_dist(mp, env->me_map); - unsigned flags_mask = P_ILL_BITS; - unsigned flags_expected = 0; - if (offset < 0 || - offset > (ptrdiff_t)(pgno2bytes(env, mc->mc_txn->mt_next_pgno) - - ((mp->mp_flags & P_SUBP) ? PAGEHDRSZ + 1 - : env->me_psize))) { - /* should be dirty page without MDBX_WRITEMAP, or a subpage of. */ - flags_mask -= P_SUBP; - if ((env->me_flags & MDBX_WRITEMAP) != 0 || - (!IS_SHADOWED(mc->mc_txn, mp) && !(mp->mp_flags & P_SUBP))) - rc = bad_page(mp, "invalid page-address %p, offset %zi\n", - __Wpedantic_format_voidptr(mp), offset); - } else if (offset & (env->me_psize - 1)) - flags_expected = P_SUBP; +void dpl_release_shadows(MDBX_txn *txn) { + tASSERT(txn, (txn->flags & (MDBX_TXN_RDONLY | MDBX_WRITEMAP)) == 0); + MDBX_env *env = txn->env; + dpl_t *const dl = txn->tw.dirtylist; - if (unlikely((mp->mp_flags & flags_mask) != flags_expected)) - rc = bad_page(mp, "unknown/extra page-flags (have 0x%x, expect 0x%x)\n", - mp->mp_flags & flags_mask, flags_expected); + for (size_t i = 1; i <= dl->length; i++) + page_shadow_release(env, dl->items[i].ptr, dpl_npages(dl, i)); - cASSERT(mc, (mc->mc_checking & CC_LEAF2) == 0 || (mc->mc_flags & C_SUB) != 0); - const uint8_t type = PAGETYPE_WHOLE(mp); - switch (type) { - default: - return bad_page(mp, "invalid type (%u)\n", type); - case P_OVERFLOW: - if (unlikely(mc->mc_flags & C_SUB)) - rc = bad_page(mp, "unexpected %s-page for %s (db-flags 0x%x)\n", "large", - "nested dupsort tree", mc->mc_db->md_flags); - const pgno_t npages = mp->mp_pages; - if (unlikely(npages < 1 || npages >= MAX_PAGENO / 2)) - rc = bad_page(mp, "invalid n-pages (%u) for large-page\n", npages); - if (unlikely(mp->mp_pgno + npages > mc->mc_txn->mt_next_pgno)) - rc = bad_page( - mp, "end of large-page beyond (%u) allocated space (%u next-pgno)\n", - mp->mp_pgno + npages, mc->mc_txn->mt_next_pgno); - return rc; //-------------------------- end of large/overflow page handling - case P_LEAF | P_SUBP: - if (unlikely(mc->mc_db->md_depth != 1)) - rc = bad_page(mp, "unexpected %s-page for %s (db-flags 0x%x)\n", - "leaf-sub", "nested dupsort db", mc->mc_db->md_flags); - /* fall through */ - __fallthrough; - case P_LEAF: - if (unlikely((mc->mc_checking & CC_LEAF2) != 0)) - rc = bad_page( - mp, "unexpected leaf-page for dupfixed subtree (db-lags 0x%x)\n", - mc->mc_db->md_flags); - break; - case P_LEAF | P_LEAF2 | P_SUBP: - if (unlikely(mc->mc_db->md_depth != 1)) - rc = bad_page(mp, "unexpected %s-page for %s (db-flags 0x%x)\n", - "leaf2-sub", "nested dupsort db", mc->mc_db->md_flags); - /* fall through */ - __fallthrough; - case P_LEAF | P_LEAF2: - if (unlikely((mc->mc_checking & CC_LEAF2) == 0)) - rc = bad_page( - mp, - "unexpected leaf2-page for non-dupfixed (sub)tree (db-flags 0x%x)\n", - mc->mc_db->md_flags); - break; - case P_BRANCH: - break; - } + dpl_clear(dl); +} +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 - if (unlikely(mp->mp_upper < mp->mp_lower || (mp->mp_lower & 1) || - PAGEHDRSZ + mp->mp_upper > env->me_psize)) - rc = bad_page(mp, "invalid page lower(%u)/upper(%u) with limit %zu\n", - mp->mp_lower, mp->mp_upper, page_space(env)); +__cold int dxb_read_header(MDBX_env *env, meta_t *dest, const int lck_exclusive, const mdbx_mode_t mode_bits) { + memset(dest, 0, sizeof(meta_t)); + int rc = osal_filesize(env->lazy_fd, &env->dxb_mmap.filesize); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; - const char *const end_of_page = ptr_disp(mp, env->me_psize); - const size_t nkeys = page_numkeys(mp); - STATIC_ASSERT(P_BRANCH == 1); - if (unlikely(nkeys <= (uint8_t)(mp->mp_flags & P_BRANCH))) { - if ((!(mc->mc_flags & C_SUB) || mc->mc_db->md_entries) && - (!(mc->mc_checking & CC_UPDATING) || - !(IS_MODIFIABLE(mc->mc_txn, mp) || (mp->mp_flags & P_SUBP)))) - rc = - bad_page(mp, "%s-page nkeys (%zu) < %u\n", - IS_BRANCH(mp) ? "branch" : "leaf", nkeys, 1 + IS_BRANCH(mp)); - } - - const size_t ksize_max = keysize_max(env->me_psize, 0); - const size_t leaf2_ksize = mp->mp_leaf2_ksize; - if (IS_LEAF2(mp)) { - if (unlikely((mc->mc_flags & C_SUB) == 0 || - (mc->mc_db->md_flags & MDBX_DUPFIXED) == 0)) - rc = bad_page(mp, "unexpected leaf2-page (db-flags 0x%x)\n", - mc->mc_db->md_flags); - else if (unlikely(leaf2_ksize != mc->mc_db->md_xsize)) - rc = bad_page(mp, "invalid leaf2_ksize %zu\n", leaf2_ksize); - else if (unlikely(((leaf2_ksize & nkeys) ^ mp->mp_upper) & 1)) - rc = bad_page( - mp, "invalid page upper (%u) for nkeys %zu with leaf2-length %zu\n", - mp->mp_upper, nkeys, leaf2_ksize); - } else { - if (unlikely((mp->mp_upper & 1) || PAGEHDRSZ + mp->mp_upper + - nkeys * sizeof(MDBX_node) + - nkeys - 1 > - env->me_psize)) - rc = - bad_page(mp, "invalid page upper (%u) for nkeys %zu with limit %zu\n", - mp->mp_upper, nkeys, page_space(env)); - } + unaligned_poke_u64(4, dest->sign, DATASIGN_WEAK); + rc = MDBX_CORRUPTED; - MDBX_val here, prev = {0, 0}; - for (size_t i = 0; i < nkeys; ++i) { - if (IS_LEAF2(mp)) { - const char *const key = page_leaf2key(mp, i, leaf2_ksize); - if (unlikely(end_of_page < key + leaf2_ksize)) { - rc = bad_page(mp, "leaf2-item beyond (%zu) page-end\n", - key + leaf2_ksize - end_of_page); - continue; - } + /* Read twice all meta pages so we can find the latest one. */ + unsigned loop_limit = NUM_METAS * 2; + /* We don't know the page size on first time. So, just guess it. */ + unsigned guess_pagesize = 0; + for (unsigned loop_count = 0; loop_count < loop_limit; ++loop_count) { + const unsigned meta_number = loop_count % NUM_METAS; + const unsigned offset = (guess_pagesize ? guess_pagesize + : (loop_count > NUM_METAS) ? env->ps + : globals.sys_pagesize) * + meta_number; - if (unlikely(leaf2_ksize != mc->mc_dbx->md_klen_min)) { - if (unlikely(leaf2_ksize < mc->mc_dbx->md_klen_min || - leaf2_ksize > mc->mc_dbx->md_klen_max)) - rc = bad_page( - mp, "leaf2-item size (%zu) <> min/max length (%zu/%zu)\n", - leaf2_ksize, mc->mc_dbx->md_klen_min, mc->mc_dbx->md_klen_max); - else - mc->mc_dbx->md_klen_min = mc->mc_dbx->md_klen_max = leaf2_ksize; - } - if ((mc->mc_checking & CC_SKIPORD) == 0) { - here.iov_base = (void *)key; - here.iov_len = leaf2_ksize; - if (prev.iov_base && unlikely(mc->mc_dbx->md_cmp(&prev, &here) >= 0)) - rc = bad_page(mp, "leaf2-item #%zu wrong order (%s >= %s)\n", i, - DKEY(&prev), DVAL(&here)); - prev = here; - } - } else { - const MDBX_node *const node = page_node(mp, i); - const char *const node_end = ptr_disp(node, NODESIZE); - if (unlikely(node_end > end_of_page)) { - rc = bad_page(mp, "node[%zu] (%zu) beyond page-end\n", i, - node_end - end_of_page); - continue; - } - const size_t ksize = node_ks(node); - if (unlikely(ksize > ksize_max)) - rc = bad_page(mp, "node[%zu] too long key (%zu)\n", i, ksize); - const char *const key = node_key(node); - if (unlikely(end_of_page < key + ksize)) { - rc = bad_page(mp, "node[%zu] key (%zu) beyond page-end\n", i, - key + ksize - end_of_page); - continue; + char buffer[MDBX_MIN_PAGESIZE]; + unsigned retryleft = 42; + while (1) { + TRACE("reading meta[%d]: offset %u, bytes %u, retry-left %u", meta_number, offset, MDBX_MIN_PAGESIZE, retryleft); + int err = osal_pread(env->lazy_fd, buffer, MDBX_MIN_PAGESIZE, offset); + if (err == MDBX_ENODATA && offset == 0 && loop_count == 0 && env->dxb_mmap.filesize == 0 && + mode_bits /* non-zero for DB creation */ != 0) { + NOTICE("read meta: empty file (%d, %s)", err, mdbx_strerror(err)); + return err; } - if ((IS_LEAF(mp) || i > 0)) { - if (unlikely(ksize < mc->mc_dbx->md_klen_min || - ksize > mc->mc_dbx->md_klen_max)) - rc = bad_page( - mp, "node[%zu] key size (%zu) <> min/max key-length (%zu/%zu)\n", - i, ksize, mc->mc_dbx->md_klen_min, mc->mc_dbx->md_klen_max); - if ((mc->mc_checking & CC_SKIPORD) == 0) { - here.iov_base = (void *)key; - here.iov_len = ksize; - if (prev.iov_base && unlikely(mc->mc_dbx->md_cmp(&prev, &here) >= 0)) - rc = bad_page(mp, "node[%zu] key wrong order (%s >= %s)\n", i, - DKEY(&prev), DVAL(&here)); - prev = here; +#if defined(_WIN32) || defined(_WIN64) + if (err == ERROR_LOCK_VIOLATION) { + SleepEx(0, true); + err = osal_pread(env->lazy_fd, buffer, MDBX_MIN_PAGESIZE, offset); + if (err == ERROR_LOCK_VIOLATION && --retryleft) { + WARNING("read meta[%u,%u]: %i, %s", offset, MDBX_MIN_PAGESIZE, err, mdbx_strerror(err)); + continue; } } - if (IS_BRANCH(mp)) { - if ((mc->mc_checking & CC_UPDATING) == 0 && i == 0 && - unlikely(ksize != 0)) - rc = bad_page(mp, "branch-node[%zu] wrong 0-node key-length (%zu)\n", - i, ksize); - const pgno_t ref = node_pgno(node); - if (unlikely(ref < MIN_PAGENO) || - (unlikely(ref >= mc->mc_txn->mt_next_pgno) && - (unlikely(ref >= mc->mc_txn->mt_geo.now) || - !(mc->mc_checking & CC_RETIRING)))) - rc = bad_page(mp, "branch-node[%zu] wrong pgno (%u)\n", i, ref); - if (unlikely(node_flags(node))) - rc = bad_page(mp, "branch-node[%zu] wrong flags (%u)\n", i, - node_flags(node)); - continue; - } - - switch (node_flags(node)) { - default: - rc = - bad_page(mp, "invalid node[%zu] flags (%u)\n", i, node_flags(node)); - break; - case F_BIGDATA /* data on large-page */: - case 0 /* usual */: - case F_SUBDATA /* sub-db */: - case F_SUBDATA | F_DUPDATA /* dupsorted sub-tree */: - case F_DUPDATA /* short sub-page */: - break; +#endif /* Windows */ + if (err != MDBX_SUCCESS) { + ERROR("read meta[%u,%u]: %i, %s", offset, MDBX_MIN_PAGESIZE, err, mdbx_strerror(err)); + return err; } - const size_t dsize = node_ds(node); - const char *const data = node_data(node); - if (node_flags(node) & F_BIGDATA) { - if (unlikely(end_of_page < data + sizeof(pgno_t))) { - rc = bad_page( - mp, "node-%s(%zu of %zu, %zu bytes) beyond (%zu) page-end\n", - "bigdata-pgno", i, nkeys, dsize, data + dsize - end_of_page); + char again[MDBX_MIN_PAGESIZE]; + err = osal_pread(env->lazy_fd, again, MDBX_MIN_PAGESIZE, offset); +#if defined(_WIN32) || defined(_WIN64) + if (err == ERROR_LOCK_VIOLATION) { + SleepEx(0, true); + err = osal_pread(env->lazy_fd, again, MDBX_MIN_PAGESIZE, offset); + if (err == ERROR_LOCK_VIOLATION && --retryleft) { + WARNING("read meta[%u,%u]: %i, %s", offset, MDBX_MIN_PAGESIZE, err, mdbx_strerror(err)); continue; } - if (unlikely(dsize <= mc->mc_dbx->md_vlen_min || - dsize > mc->mc_dbx->md_vlen_max)) - rc = bad_page( - mp, - "big-node data size (%zu) <> min/max value-length (%zu/%zu)\n", - dsize, mc->mc_dbx->md_vlen_min, mc->mc_dbx->md_vlen_max); - if (unlikely(node_size_len(node_ks(node), dsize) <= - mc->mc_txn->mt_env->me_leaf_nodemax) && - mc->mc_dbi != FREE_DBI) - poor_page(mp, "too small data (%zu bytes) for bigdata-node", dsize); - - if ((mc->mc_checking & CC_RETIRING) == 0) { - const pgr_t lp = - page_get_large(mc, node_largedata_pgno(node), mp->mp_txnid); - if (unlikely(lp.err != MDBX_SUCCESS)) - return lp.err; - cASSERT(mc, PAGETYPE_WHOLE(lp.page) == P_OVERFLOW); - const unsigned npages = number_of_ovpages(env, dsize); - if (unlikely(lp.page->mp_pages != npages)) { - if (lp.page->mp_pages < npages) - rc = bad_page(lp.page, - "too less n-pages %u for bigdata-node (%zu bytes)", - lp.page->mp_pages, dsize); - else if (mc->mc_dbi != FREE_DBI) - poor_page(lp.page, - "extra n-pages %u for bigdata-node (%zu bytes)", - lp.page->mp_pages, dsize); - } - } - continue; } - - if (unlikely(end_of_page < data + dsize)) { - rc = bad_page(mp, - "node-%s(%zu of %zu, %zu bytes) beyond (%zu) page-end\n", - "data", i, nkeys, dsize, data + dsize - end_of_page); - continue; +#endif /* Windows */ + if (err != MDBX_SUCCESS) { + ERROR("read meta[%u,%u]: %i, %s", offset, MDBX_MIN_PAGESIZE, err, mdbx_strerror(err)); + return err; } - switch (node_flags(node)) { - default: - /* wrong, but already handled */ - continue; - case 0 /* usual */: - if (unlikely(dsize < mc->mc_dbx->md_vlen_min || - dsize > mc->mc_dbx->md_vlen_max)) { - rc = bad_page( - mp, "node-data size (%zu) <> min/max value-length (%zu/%zu)\n", - dsize, mc->mc_dbx->md_vlen_min, mc->mc_dbx->md_vlen_max); - continue; - } - break; - case F_SUBDATA /* sub-db */: - if (unlikely(dsize != sizeof(MDBX_db))) { - rc = bad_page(mp, "invalid sub-db record size (%zu)\n", dsize); - continue; - } - break; - case F_SUBDATA | F_DUPDATA /* dupsorted sub-tree */: - if (unlikely(dsize != sizeof(MDBX_db))) { - rc = bad_page(mp, "invalid nested-db record size (%zu)\n", dsize); - continue; - } + if (memcmp(buffer, again, MDBX_MIN_PAGESIZE) == 0 || --retryleft == 0) break; - case F_DUPDATA /* short sub-page */: - if (unlikely(dsize <= PAGEHDRSZ)) { - rc = bad_page(mp, "invalid nested/sub-page record size (%zu)\n", - dsize); - continue; - } else { - const MDBX_page *const sp = (MDBX_page *)data; - switch (sp->mp_flags & - /* ignore legacy P_DIRTY flag */ ~P_LEGACY_DIRTY) { - case P_LEAF | P_SUBP: - case P_LEAF | P_LEAF2 | P_SUBP: - break; - default: - rc = bad_page(mp, "invalid nested/sub-page flags (0x%02x)\n", - sp->mp_flags); - continue; - } - - const char *const end_of_subpage = data + dsize; - const intptr_t nsubkeys = page_numkeys(sp); - if (unlikely(nsubkeys == 0) && !(mc->mc_checking & CC_UPDATING) && - mc->mc_db->md_entries) - rc = bad_page(mp, "no keys on a %s-page\n", - IS_LEAF2(sp) ? "leaf2-sub" : "leaf-sub"); - MDBX_val sub_here, sub_prev = {0, 0}; - for (int j = 0; j < nsubkeys; j++) { - if (IS_LEAF2(sp)) { - /* LEAF2 pages have no mp_ptrs[] or node headers */ - const size_t sub_ksize = sp->mp_leaf2_ksize; - const char *const sub_key = page_leaf2key(sp, j, sub_ksize); - if (unlikely(end_of_subpage < sub_key + sub_ksize)) { - rc = bad_page(mp, "nested-leaf2-key beyond (%zu) nested-page\n", - sub_key + sub_ksize - end_of_subpage); - continue; - } + VERBOSE("meta[%u] was updated, re-read it", meta_number); + } - if (unlikely(sub_ksize != mc->mc_dbx->md_vlen_min)) { - if (unlikely(sub_ksize < mc->mc_dbx->md_vlen_min || - sub_ksize > mc->mc_dbx->md_vlen_max)) - rc = bad_page(mp, - "nested-leaf2-key size (%zu) <> min/max " - "value-length (%zu/%zu)\n", - sub_ksize, mc->mc_dbx->md_vlen_min, - mc->mc_dbx->md_vlen_max); - else - mc->mc_dbx->md_vlen_min = mc->mc_dbx->md_vlen_max = sub_ksize; - } - if ((mc->mc_checking & CC_SKIPORD) == 0) { - sub_here.iov_base = (void *)sub_key; - sub_here.iov_len = sub_ksize; - if (sub_prev.iov_base && - unlikely(mc->mc_dbx->md_dcmp(&sub_prev, &sub_here) >= 0)) - rc = bad_page(mp, - "nested-leaf2-key #%u wrong order (%s >= %s)\n", - j, DKEY(&sub_prev), DVAL(&sub_here)); - sub_prev = sub_here; - } - } else { - const MDBX_node *const sub_node = page_node(sp, j); - const char *const sub_node_end = ptr_disp(sub_node, NODESIZE); - if (unlikely(sub_node_end > end_of_subpage)) { - rc = bad_page(mp, "nested-node beyond (%zu) nested-page\n", - end_of_subpage - sub_node_end); - continue; - } - if (unlikely(node_flags(sub_node) != 0)) - rc = bad_page(mp, "nested-node invalid flags (%u)\n", - node_flags(sub_node)); + if (!retryleft) { + ERROR("meta[%u] is too volatile, skip it", meta_number); + continue; + } - const size_t sub_ksize = node_ks(sub_node); - const char *const sub_key = node_key(sub_node); - const size_t sub_dsize = node_ds(sub_node); - /* char *sub_data = node_data(sub_node); */ + page_t *const page = (page_t *)buffer; + meta_t *const meta = page_meta(page); + rc = meta_validate(env, meta, page, meta_number, &guess_pagesize); + if (rc != MDBX_SUCCESS) + continue; - if (unlikely(sub_ksize < mc->mc_dbx->md_vlen_min || - sub_ksize > mc->mc_dbx->md_vlen_max)) - rc = bad_page(mp, - "nested-node-key size (%zu) <> min/max " - "value-length (%zu/%zu)\n", - sub_ksize, mc->mc_dbx->md_vlen_min, - mc->mc_dbx->md_vlen_max); - if ((mc->mc_checking & CC_SKIPORD) == 0) { - sub_here.iov_base = (void *)sub_key; - sub_here.iov_len = sub_ksize; - if (sub_prev.iov_base && - unlikely(mc->mc_dbx->md_dcmp(&sub_prev, &sub_here) >= 0)) - rc = bad_page(mp, - "nested-node-key #%u wrong order (%s >= %s)\n", - j, DKEY(&sub_prev), DVAL(&sub_here)); - sub_prev = sub_here; - } - if (unlikely(sub_dsize != 0)) - rc = bad_page(mp, "nested-node non-empty data size (%zu)\n", - sub_dsize); - if (unlikely(end_of_subpage < sub_key + sub_ksize)) - rc = bad_page(mp, "nested-node-key beyond (%zu) nested-page\n", - sub_key + sub_ksize - end_of_subpage); - } - } - } - break; - } + bool latch; + if (env->stuck_meta >= 0) + latch = (meta_number == (unsigned)env->stuck_meta); + else if (meta_bootid_match(meta)) + latch = meta_choice_recent(meta->unsafe_txnid, SIGN_IS_STEADY(meta->unsafe_sign), dest->unsafe_txnid, + SIGN_IS_STEADY(dest->unsafe_sign)); + else + latch = meta_choice_steady(meta->unsafe_txnid, SIGN_IS_STEADY(meta->unsafe_sign), dest->unsafe_txnid, + SIGN_IS_STEADY(dest->unsafe_sign)); + if (latch) { + *dest = *meta; + if (!lck_exclusive && !meta_is_steady(dest)) + loop_limit += 1; /* LY: should re-read to hush race with update */ + VERBOSE("latch meta[%u]", meta_number); } } - return rc; -} - -__cold static int cursor_check(const MDBX_cursor *mc) { - if (!mc->mc_txn->tw.dirtylist) { - cASSERT(mc, - (mc->mc_txn->mt_flags & MDBX_WRITEMAP) != 0 && !MDBX_AVOID_MSYNC); - } else { - cASSERT(mc, - (mc->mc_txn->mt_flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); - cASSERT(mc, mc->mc_txn->tw.dirtyroom + mc->mc_txn->tw.dirtylist->length == - (mc->mc_txn->mt_parent - ? mc->mc_txn->mt_parent->tw.dirtyroom - : mc->mc_txn->mt_env->me_options.dp_limit)); - } - cASSERT(mc, mc->mc_top == mc->mc_snum - 1 || (mc->mc_checking & CC_UPDATING)); - if (unlikely(mc->mc_top != mc->mc_snum - 1) && - (mc->mc_checking & CC_UPDATING) == 0) - return MDBX_CURSOR_FULL; - cASSERT(mc, (mc->mc_checking & CC_UPDATING) - ? mc->mc_snum <= mc->mc_db->md_depth - : mc->mc_snum == mc->mc_db->md_depth); - if (unlikely((mc->mc_checking & CC_UPDATING) - ? mc->mc_snum > mc->mc_db->md_depth - : mc->mc_snum != mc->mc_db->md_depth)) - return MDBX_CURSOR_FULL; - - for (int n = 0; n < (int)mc->mc_snum; ++n) { - MDBX_page *mp = mc->mc_pg[n]; - const size_t nkeys = page_numkeys(mp); - const bool expect_branch = (n < mc->mc_db->md_depth - 1) ? true : false; - const bool expect_nested_leaf = - (n + 1 == mc->mc_db->md_depth - 1) ? true : false; - const bool branch = IS_BRANCH(mp) ? true : false; - cASSERT(mc, branch == expect_branch); - if (unlikely(branch != expect_branch)) - return MDBX_CURSOR_FULL; - if ((mc->mc_checking & CC_UPDATING) == 0) { - cASSERT(mc, nkeys > mc->mc_ki[n] || (!branch && nkeys == mc->mc_ki[n] && - (mc->mc_flags & C_EOF) != 0)); - if (unlikely(nkeys <= mc->mc_ki[n] && - !(!branch && nkeys == mc->mc_ki[n] && - (mc->mc_flags & C_EOF) != 0))) - return MDBX_CURSOR_FULL; - } else { - cASSERT(mc, nkeys + 1 >= mc->mc_ki[n]); - if (unlikely(nkeys + 1 < mc->mc_ki[n])) - return MDBX_CURSOR_FULL; - } - - int err = page_check(mc, mp); - if (unlikely(err != MDBX_SUCCESS)) - return err; - for (size_t i = 0; i < nkeys; ++i) { - if (branch) { - MDBX_node *node = page_node(mp, i); - cASSERT(mc, node_flags(node) == 0); - if (unlikely(node_flags(node) != 0)) - return MDBX_CURSOR_FULL; - pgno_t pgno = node_pgno(node); - MDBX_page *np; - err = page_get(mc, pgno, &np, mp->mp_txnid); - cASSERT(mc, err == MDBX_SUCCESS); - if (unlikely(err != MDBX_SUCCESS)) - return err; - const bool nested_leaf = IS_LEAF(np) ? true : false; - cASSERT(mc, nested_leaf == expect_nested_leaf); - if (unlikely(nested_leaf != expect_nested_leaf)) - return MDBX_CURSOR_FULL; - err = page_check(mc, np); - if (unlikely(err != MDBX_SUCCESS)) - return err; - } + if (dest->pagesize == 0 || + (env->stuck_meta < 0 && !(meta_is_steady(dest) || meta_weak_acceptable(env, dest, lck_exclusive)))) { + ERROR("%s", "no usable meta-pages, database is corrupted"); + if (rc == MDBX_SUCCESS) { + /* TODO: try to restore the database by fully checking b-tree structure + * for the each meta page, if the corresponding option was given */ + return MDBX_CORRUPTED; } + return rc; } - return MDBX_SUCCESS; -} -__cold static int cursor_check_updating(MDBX_cursor *mc) { - const uint8_t checking = mc->mc_checking; - mc->mc_checking |= CC_UPDATING; - const int rc = cursor_check(mc); - mc->mc_checking = checking; - return rc; + return MDBX_SUCCESS; } -int mdbx_del(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, - const MDBX_val *data) { - int rc = check_txn_rw(txn, MDBX_TXN_BLOCKED); +__cold int dxb_resize(MDBX_env *const env, const pgno_t used_pgno, const pgno_t size_pgno, pgno_t limit_pgno, + const enum resize_mode mode) { + /* Acquire guard to avoid collision between read and write txns + * around geo_in_bytes and dxb_mmap */ +#if defined(_WIN32) || defined(_WIN64) + imports.srwl_AcquireExclusive(&env->remap_guard); + int rc = MDBX_SUCCESS; + mdbx_handle_array_t *suspended = nullptr; + mdbx_handle_array_t array_onstack; +#else + int rc = osal_fastmutex_acquire(&env->remap_guard); if (unlikely(rc != MDBX_SUCCESS)) return rc; +#endif - if (unlikely(!key)) - return MDBX_EINVAL; - - if (unlikely(!check_dbi(txn, dbi, DBI_USRVALID))) - return MDBX_BAD_DBI; + const size_t prev_size = env->dxb_mmap.current; + const size_t prev_limit = env->dxb_mmap.limit; + const pgno_t prev_limit_pgno = bytes2pgno(env, prev_limit); + eASSERT(env, limit_pgno >= size_pgno); + eASSERT(env, size_pgno >= used_pgno); + if (mode < explicit_resize && size_pgno <= prev_limit_pgno) { + /* The actual mapsize may be less since the geo.upper may be changed + * by other process. Avoids remapping until it necessary. */ + limit_pgno = prev_limit_pgno; + } + const size_t limit_bytes = pgno_align2os_bytes(env, limit_pgno); + const size_t size_bytes = pgno_align2os_bytes(env, size_pgno); + const void *const prev_map = env->dxb_mmap.base; - if (unlikely(txn->mt_flags & (MDBX_TXN_RDONLY | MDBX_TXN_BLOCKED))) - return (txn->mt_flags & MDBX_TXN_RDONLY) ? MDBX_EACCESS : MDBX_BAD_TXN; + VERBOSE("resize(env-flags 0x%x, mode %d) datafile/mapping: " + "present %" PRIuPTR " -> %" PRIuPTR ", " + "limit %" PRIuPTR " -> %" PRIuPTR, + env->flags, mode, prev_size, size_bytes, prev_limit, limit_bytes); - return delete (txn, dbi, key, data, 0); -} + eASSERT(env, limit_bytes >= size_bytes); + eASSERT(env, bytes2pgno(env, size_bytes) >= size_pgno); + eASSERT(env, bytes2pgno(env, limit_bytes) >= limit_pgno); -static int delete(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, - const MDBX_val *data, unsigned flags) { - MDBX_cursor_couple cx; - MDBX_cursor_op op; - MDBX_val rdata; - int rc; - DKBUF_DEBUG; + unsigned mresize_flags = env->flags & (MDBX_RDONLY | MDBX_WRITEMAP | MDBX_UTTERLY_NOSYNC); + if (mode >= impilict_shrink) + mresize_flags |= txn_shrink_allowed; - DEBUG("====> delete db %u key [%s], data [%s]", dbi, DKEY_DEBUG(key), - DVAL_DEBUG(data)); + if (limit_bytes == env->dxb_mmap.limit && size_bytes == env->dxb_mmap.current && size_bytes == env->dxb_mmap.filesize) + goto bailout; - rc = cursor_init(&cx.outer, txn, dbi); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; + /* При использовании MDBX_NOSTICKYTHREADS с транзакциями могут работать любые + * потоки и у нас нет информации о том, какие именно. Поэтому нет возможности + * выполнить remap-действия требующие приостановки работающих с БД потоков. */ + if ((env->flags & MDBX_NOSTICKYTHREADS) == 0) { +#if defined(_WIN32) || defined(_WIN64) + if ((size_bytes < env->dxb_mmap.current && mode > implicit_grow) || limit_bytes != env->dxb_mmap.limit) { + /* 1) Windows allows only extending a read-write section, but not a + * corresponding mapped view. Therefore in other cases we must suspend + * the local threads for safe remap. + * 2) At least on Windows 10 1803 the entire mapped section is unavailable + * for short time during NtExtendSection() or VirtualAlloc() execution. + * 3) Under Wine runtime environment on Linux a section extending is not + * supported. + * + * THEREFORE LOCAL THREADS SUSPENDING IS ALWAYS REQUIRED! */ + array_onstack.limit = ARRAY_LENGTH(array_onstack.handles); + array_onstack.count = 0; + suspended = &array_onstack; + rc = osal_suspend_threads_before_remap(env, &suspended); + if (rc != MDBX_SUCCESS) { + ERROR("failed suspend-for-remap: errcode %d", rc); + goto bailout; + } + mresize_flags |= + (mode < explicit_resize) ? MDBX_MRESIZE_MAY_UNMAP : MDBX_MRESIZE_MAY_UNMAP | MDBX_MRESIZE_MAY_MOVE; + } +#else /* Windows */ + lck_t *const lck = env->lck_mmap.lck; + if (mode == explicit_resize && limit_bytes != env->dxb_mmap.limit) { + mresize_flags |= MDBX_MRESIZE_MAY_UNMAP | MDBX_MRESIZE_MAY_MOVE; + if (lck) { + int err = lck_rdt_lock(env) /* lock readers table until remap done */; + if (unlikely(MDBX_IS_ERROR(err))) { + rc = err; + goto bailout; + } - if (data) { - op = MDBX_GET_BOTH; - rdata = *data; - data = &rdata; - } else { - op = MDBX_SET; - flags |= MDBX_ALLDUPS; - } - rc = cursor_set(&cx.outer, (MDBX_val *)key, (MDBX_val *)data, op).err; - if (likely(rc == MDBX_SUCCESS)) { - /* let mdbx_page_split know about this cursor if needed: - * delete will trigger a rebalance; if it needs to move - * a node from one page to another, it will have to - * update the parent's separator key(s). If the new sepkey - * is larger than the current one, the parent page may - * run out of space, triggering a split. We need this - * cursor to be consistent until the end of the rebalance. */ - cx.outer.mc_next = txn->mt_cursors[dbi]; - txn->mt_cursors[dbi] = &cx.outer; - rc = cursor_del(&cx.outer, flags); - txn->mt_cursors[dbi] = cx.outer.mc_next; + /* looking for readers from this process */ + const size_t snap_nreaders = atomic_load32(&lck->rdt_length, mo_AcquireRelease); + eASSERT(env, mode == explicit_resize); + for (size_t i = 0; i < snap_nreaders; ++i) { + if (lck->rdt[i].pid.weak == env->pid && lck->rdt[i].tid.weak != osal_thread_self()) { + /* the base address of the mapping can't be changed since + * the other reader thread from this process exists. */ + lck_rdt_unlock(env); + mresize_flags &= ~(MDBX_MRESIZE_MAY_UNMAP | MDBX_MRESIZE_MAY_MOVE); + break; + } + } + } + } +#endif /* ! Windows */ } - return rc; -} - -/* Split a page and insert a new node. - * Set MDBX_TXN_ERROR on failure. - * [in,out] mc Cursor pointing to the page and desired insertion index. - * The cursor will be updated to point to the actual page and index where - * the node got inserted after the split. - * [in] newkey The key for the newly inserted node. - * [in] newdata The data for the newly inserted node. - * [in] newpgno The page number, if the new node is a branch node. - * [in] naf The NODE_ADD_FLAGS for the new node. - * Returns 0 on success, non-zero on failure. */ -static int page_split(MDBX_cursor *mc, const MDBX_val *const newkey, - MDBX_val *const newdata, pgno_t newpgno, - const unsigned naf) { - unsigned flags; - int rc = MDBX_SUCCESS, foliage = 0; - size_t i, ptop; - MDBX_env *const env = mc->mc_txn->mt_env; - MDBX_val rkey, xdata; - MDBX_page *tmp_ki_copy = NULL; - DKBUF; - MDBX_page *const mp = mc->mc_pg[mc->mc_top]; - cASSERT(mc, (mp->mp_flags & P_ILL_BITS) == 0); - - const size_t newindx = mc->mc_ki[mc->mc_top]; - size_t nkeys = page_numkeys(mp); - if (AUDIT_ENABLED()) { - rc = cursor_check_updating(mc); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; + const pgno_t aligned_munlock_pgno = + (mresize_flags & (MDBX_MRESIZE_MAY_UNMAP | MDBX_MRESIZE_MAY_MOVE)) ? 0 : bytes2pgno(env, size_bytes); + if (mresize_flags & (MDBX_MRESIZE_MAY_UNMAP | MDBX_MRESIZE_MAY_MOVE)) { + mincore_clean_cache(env); + if ((env->flags & MDBX_WRITEMAP) && env->lck->unsynced_pages.weak) { +#if MDBX_ENABLE_PGOP_STAT + env->lck->pgops.msync.weak += 1; +#endif /* MDBX_ENABLE_PGOP_STAT */ + rc = osal_msync(&env->dxb_mmap, 0, pgno_align2os_bytes(env, used_pgno), MDBX_SYNC_NONE); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + } } - STATIC_ASSERT(P_BRANCH == 1); - const size_t minkeys = (mp->mp_flags & P_BRANCH) + (size_t)1; + munlock_after(env, aligned_munlock_pgno, size_bytes); - DEBUG(">> splitting %s-page %" PRIaPGNO - " and adding %zu+%zu [%s] at %i, nkeys %zi", - IS_LEAF(mp) ? "leaf" : "branch", mp->mp_pgno, newkey->iov_len, - newdata ? newdata->iov_len : 0, DKEY_DEBUG(newkey), - mc->mc_ki[mc->mc_top], nkeys); - cASSERT(mc, nkeys + 1 >= minkeys * 2); + if (size_bytes < prev_size && mode > implicit_grow) { + NOTICE("resize-MADV_%s %u..%u", (env->flags & MDBX_WRITEMAP) ? "REMOVE" : "DONTNEED", size_pgno, + bytes2pgno(env, prev_size)); + const uint32_t munlocks_before = atomic_load32(&env->lck->mlcnt[1], mo_Relaxed); + rc = MDBX_RESULT_TRUE; +#if defined(MADV_REMOVE) + if (env->flags & MDBX_WRITEMAP) + rc = madvise(ptr_disp(env->dxb_mmap.base, size_bytes), prev_size - size_bytes, MADV_REMOVE) + ? ignore_enosys_and_eagain(errno) + : MDBX_SUCCESS; +#endif /* MADV_REMOVE */ +#if defined(MADV_DONTNEED) + if (rc == MDBX_RESULT_TRUE) + rc = madvise(ptr_disp(env->dxb_mmap.base, size_bytes), prev_size - size_bytes, MADV_DONTNEED) + ? ignore_enosys_and_eagain(errno) + : MDBX_SUCCESS; +#elif defined(POSIX_MADV_DONTNEED) + if (rc == MDBX_RESULT_TRUE) + rc = ignore_enosys( + posix_madvise(ptr_disp(env->dxb_mmap.base, size_bytes), prev_size - size_bytes, POSIX_MADV_DONTNEED)); +#elif defined(POSIX_FADV_DONTNEED) + if (rc == MDBX_RESULT_TRUE) + rc = ignore_enosys(posix_fadvise(env->lazy_fd, size_bytes, prev_size - size_bytes, POSIX_FADV_DONTNEED)); +#endif /* MADV_DONTNEED */ + if (unlikely(MDBX_IS_ERROR(rc))) { + const uint32_t mlocks_after = atomic_load32(&env->lck->mlcnt[0], mo_Relaxed); + if (rc == MDBX_EINVAL) { + const int severity = (mlocks_after - munlocks_before) ? MDBX_LOG_NOTICE : MDBX_LOG_WARN; + if (LOG_ENABLED(severity)) + debug_log(severity, __func__, __LINE__, + "%s-madvise: ignore EINVAL (%d) since some pages maybe " + "locked (%u/%u mlcnt-processes)", + "resize", rc, mlocks_after, munlocks_before); + } else { + ERROR("%s-madvise(%s, %zu, +%zu), %u/%u mlcnt-processes, err %d", "mresize", "DONTNEED", size_bytes, + prev_size - size_bytes, mlocks_after, munlocks_before, rc); + goto bailout; + } + } else + env->lck->discarded_tail.weak = size_pgno; + } - /* Create a new sibling page. */ - pgr_t npr = page_new(mc, mp->mp_flags); - if (unlikely(npr.err != MDBX_SUCCESS)) - return npr.err; - MDBX_page *const sister = npr.page; - sister->mp_leaf2_ksize = mp->mp_leaf2_ksize; - DEBUG("new sibling: page %" PRIaPGNO, sister->mp_pgno); + rc = osal_mresize(mresize_flags, &env->dxb_mmap, size_bytes, limit_bytes); + eASSERT(env, env->dxb_mmap.limit >= env->dxb_mmap.current); - /* Usually when splitting the root page, the cursor - * height is 1. But when called from update_key, - * the cursor height may be greater because it walks - * up the stack while finding the branch slot to update. */ - if (mc->mc_top < 1) { - npr = page_new(mc, P_BRANCH); - rc = npr.err; - if (unlikely(rc != MDBX_SUCCESS)) - goto done; - MDBX_page *const pp = npr.page; - /* shift current top to make room for new parent */ - cASSERT(mc, mc->mc_snum < 2 && mc->mc_db->md_depth > 0); -#if MDBX_DEBUG - memset(mc->mc_pg + 3, 0, sizeof(mc->mc_pg) - sizeof(mc->mc_pg[0]) * 3); - memset(mc->mc_ki + 3, -1, sizeof(mc->mc_ki) - sizeof(mc->mc_ki[0]) * 3); -#endif - mc->mc_pg[2] = mc->mc_pg[1]; - mc->mc_ki[2] = mc->mc_ki[1]; - mc->mc_pg[1] = mc->mc_pg[0]; - mc->mc_ki[1] = mc->mc_ki[0]; - mc->mc_pg[0] = pp; - mc->mc_ki[0] = 0; - mc->mc_db->md_root = pp->mp_pgno; - DEBUG("root split! new root = %" PRIaPGNO, pp->mp_pgno); - foliage = mc->mc_db->md_depth++; + if (rc == MDBX_SUCCESS) { + eASSERT(env, limit_bytes == env->dxb_mmap.limit); + eASSERT(env, size_bytes <= env->dxb_mmap.filesize); + if (mode == explicit_resize) + eASSERT(env, size_bytes == env->dxb_mmap.current); + else + eASSERT(env, size_bytes <= env->dxb_mmap.current); + env->lck->discarded_tail.weak = size_pgno; + const bool readahead = + !(env->flags & MDBX_NORDAHEAD) && mdbx_is_readahead_reasonable(size_bytes, -(intptr_t)prev_size); + const bool force = limit_bytes != prev_limit || env->dxb_mmap.base != prev_map +#if defined(_WIN32) || defined(_WIN64) + || prev_size > size_bytes +#endif /* Windows */ + ; + rc = dxb_set_readahead(env, size_pgno, readahead, force); + } - /* Add left (implicit) pointer. */ - rc = node_add_branch(mc, 0, NULL, mp->mp_pgno); - if (unlikely(rc != MDBX_SUCCESS)) { - /* undo the pre-push */ - mc->mc_pg[0] = mc->mc_pg[1]; - mc->mc_ki[0] = mc->mc_ki[1]; - mc->mc_db->md_root = mp->mp_pgno; - mc->mc_db->md_depth--; - goto done; +bailout: + if (rc == MDBX_SUCCESS) { + eASSERT(env, env->dxb_mmap.limit >= env->dxb_mmap.current); + eASSERT(env, limit_bytes == env->dxb_mmap.limit); + eASSERT(env, size_bytes <= env->dxb_mmap.filesize); + if (mode == explicit_resize) + eASSERT(env, size_bytes == env->dxb_mmap.current); + else + eASSERT(env, size_bytes <= env->dxb_mmap.current); + /* update env-geo to avoid influences */ + env->geo_in_bytes.now = env->dxb_mmap.current; + env->geo_in_bytes.upper = env->dxb_mmap.limit; + env_options_adjust_defaults(env); +#ifdef ENABLE_MEMCHECK + if (prev_limit != env->dxb_mmap.limit || prev_map != env->dxb_mmap.base) { + VALGRIND_DISCARD(env->valgrind_handle); + env->valgrind_handle = 0; + if (env->dxb_mmap.limit) + env->valgrind_handle = VALGRIND_CREATE_BLOCK(env->dxb_mmap.base, env->dxb_mmap.limit, "mdbx"); + } +#endif /* ENABLE_MEMCHECK */ + } else { + if (rc != MDBX_UNABLE_EXTEND_MAPSIZE && rc != MDBX_EPERM) { + ERROR("failed resize datafile/mapping: " + "present %" PRIuPTR " -> %" PRIuPTR ", " + "limit %" PRIuPTR " -> %" PRIuPTR ", errcode %d", + prev_size, size_bytes, prev_limit, limit_bytes, rc); + } else { + WARNING("unable resize datafile/mapping: " + "present %" PRIuPTR " -> %" PRIuPTR ", " + "limit %" PRIuPTR " -> %" PRIuPTR ", errcode %d", + prev_size, size_bytes, prev_limit, limit_bytes, rc); + eASSERT(env, env->dxb_mmap.limit >= env->dxb_mmap.current); } - mc->mc_snum++; - mc->mc_top++; - ptop = 0; - if (AUDIT_ENABLED()) { - rc = cursor_check_updating(mc); - if (unlikely(rc != MDBX_SUCCESS)) - goto done; + if (!env->dxb_mmap.base) { + env->flags |= ENV_FATAL_ERROR; + if (env->txn) + env->txn->flags |= MDBX_TXN_ERROR; + rc = MDBX_PANIC; } - } else { - ptop = mc->mc_top - 1; - DEBUG("parent branch page is %" PRIaPGNO, mc->mc_pg[ptop]->mp_pgno); } - MDBX_cursor mn; - cursor_copy(mc, &mn); - mn.mc_pg[mn.mc_top] = sister; - mn.mc_ki[mn.mc_top] = 0; - mn.mc_ki[ptop] = mc->mc_ki[ptop] + 1; - - size_t split_indx = - (newindx < nkeys) - ? /* split at the middle */ (nkeys + 1) >> 1 - : /* split at the end (i.e. like append-mode ) */ nkeys - minkeys + 1; - eASSERT(env, split_indx >= minkeys && split_indx <= nkeys - minkeys + 1); - - cASSERT(mc, !IS_BRANCH(mp) || newindx > 0); - MDBX_val sepkey = {nullptr, 0}; - /* It is reasonable and possible to split the page at the begin */ - if (unlikely(newindx < minkeys)) { - split_indx = minkeys; - if (newindx == 0 && !(naf & MDBX_SPLIT_REPLACE)) { - split_indx = 0; - /* Checking for ability of splitting by the left-side insertion - * of a pure page with the new key */ - for (i = 0; i < mc->mc_top; ++i) - if (mc->mc_ki[i]) { - get_key(page_node(mc->mc_pg[i], mc->mc_ki[i]), &sepkey); - if (mc->mc_dbx->md_cmp(newkey, &sepkey) >= 0) - split_indx = minkeys; - break; - } - if (split_indx == 0) { - /* Save the current first key which was omitted on the parent branch - * page and should be updated if the new first entry will be added */ - if (IS_LEAF2(mp)) { - sepkey.iov_len = mp->mp_leaf2_ksize; - sepkey.iov_base = page_leaf2key(mp, 0, sepkey.iov_len); - } else - get_key(page_node(mp, 0), &sepkey); - cASSERT(mc, mc->mc_dbx->md_cmp(newkey, &sepkey) < 0); - /* Avoiding rare complex cases of nested split the parent page(s) */ - if (page_room(mc->mc_pg[ptop]) < branch_size(env, &sepkey)) - split_indx = minkeys; - } - if (foliage) { - TRACE("pure-left: foliage %u, top %i, ptop %zu, split_indx %zi, " - "minkeys %zi, sepkey %s, parent-room %zu, need4split %zu", - foliage, mc->mc_top, ptop, split_indx, minkeys, - DKEY_DEBUG(&sepkey), page_room(mc->mc_pg[ptop]), - branch_size(env, &sepkey)); - TRACE("pure-left: newkey %s, newdata %s, newindx %zu", - DKEY_DEBUG(newkey), DVAL_DEBUG(newdata), newindx); - } - } +#if defined(_WIN32) || defined(_WIN64) + int err = MDBX_SUCCESS; + imports.srwl_ReleaseExclusive(&env->remap_guard); + if (suspended) { + err = osal_resume_threads_after_remap(suspended); + if (suspended != &array_onstack) + osal_free(suspended); } - - const bool pure_right = split_indx == nkeys; - const bool pure_left = split_indx == 0; - if (unlikely(pure_right)) { - /* newindx == split_indx == nkeys */ - TRACE("no-split, but add new pure page at the %s", "right/after"); - cASSERT(mc, newindx == nkeys && split_indx == nkeys && minkeys == 1); - sepkey = *newkey; - } else if (unlikely(pure_left)) { - /* newindx == split_indx == 0 */ - TRACE("pure-left: no-split, but add new pure page at the %s", - "left/before"); - cASSERT(mc, newindx == 0 && split_indx == 0 && minkeys == 1); - TRACE("pure-left: old-first-key is %s", DKEY_DEBUG(&sepkey)); - } else { - if (IS_LEAF2(sister)) { - /* Move half of the keys to the right sibling */ - const intptr_t distance = mc->mc_ki[mc->mc_top] - split_indx; - size_t ksize = mc->mc_db->md_xsize; - void *const split = page_leaf2key(mp, split_indx, ksize); - size_t rsize = (nkeys - split_indx) * ksize; - size_t lsize = (nkeys - split_indx) * sizeof(indx_t); - cASSERT(mc, mp->mp_lower >= lsize); - mp->mp_lower -= (indx_t)lsize; - cASSERT(mc, sister->mp_lower + lsize <= UINT16_MAX); - sister->mp_lower += (indx_t)lsize; - cASSERT(mc, mp->mp_upper + rsize - lsize <= UINT16_MAX); - mp->mp_upper += (indx_t)(rsize - lsize); - cASSERT(mc, sister->mp_upper >= rsize - lsize); - sister->mp_upper -= (indx_t)(rsize - lsize); - sepkey.iov_len = ksize; - sepkey.iov_base = (newindx != split_indx) ? split : newkey->iov_base; - if (distance < 0) { - cASSERT(mc, ksize >= sizeof(indx_t)); - void *const ins = page_leaf2key(mp, mc->mc_ki[mc->mc_top], ksize); - memcpy(sister->mp_ptrs, split, rsize); - sepkey.iov_base = sister->mp_ptrs; - memmove(ptr_disp(ins, ksize), ins, - (split_indx - mc->mc_ki[mc->mc_top]) * ksize); - memcpy(ins, newkey->iov_base, ksize); - cASSERT(mc, UINT16_MAX - mp->mp_lower >= (int)sizeof(indx_t)); - mp->mp_lower += sizeof(indx_t); - cASSERT(mc, mp->mp_upper >= ksize - sizeof(indx_t)); - mp->mp_upper -= (indx_t)(ksize - sizeof(indx_t)); - cASSERT(mc, (((ksize & page_numkeys(mp)) ^ mp->mp_upper) & 1) == 0); - } else { - memcpy(sister->mp_ptrs, split, distance * ksize); - void *const ins = page_leaf2key(sister, distance, ksize); - memcpy(ins, newkey->iov_base, ksize); - memcpy(ptr_disp(ins, ksize), ptr_disp(split, distance * ksize), - rsize - distance * ksize); - cASSERT(mc, UINT16_MAX - sister->mp_lower >= (int)sizeof(indx_t)); - sister->mp_lower += sizeof(indx_t); - cASSERT(mc, sister->mp_upper >= ksize - sizeof(indx_t)); - sister->mp_upper -= (indx_t)(ksize - sizeof(indx_t)); - cASSERT(mc, distance <= (int)UINT16_MAX); - mc->mc_ki[mc->mc_top] = (indx_t)distance; - cASSERT(mc, - (((ksize & page_numkeys(sister)) ^ sister->mp_upper) & 1) == 0); - } - - if (AUDIT_ENABLED()) { - rc = cursor_check_updating(mc); - if (unlikely(rc != MDBX_SUCCESS)) - goto done; - rc = cursor_check_updating(&mn); - if (unlikely(rc != MDBX_SUCCESS)) - goto done; - } +#else + if (env->lck_mmap.lck && (mresize_flags & (MDBX_MRESIZE_MAY_UNMAP | MDBX_MRESIZE_MAY_MOVE)) != 0) + lck_rdt_unlock(env); + int err = osal_fastmutex_release(&env->remap_guard); +#endif /* Windows */ + if (err != MDBX_SUCCESS) { + FATAL("failed resume-after-remap: errcode %d", err); + return MDBX_PANIC; + } + return rc; +} +#if defined(ENABLE_MEMCHECK) || defined(__SANITIZE_ADDRESS__) +void dxb_sanitize_tail(MDBX_env *env, MDBX_txn *txn) { +#if !defined(__SANITIZE_ADDRESS__) + if (!RUNNING_ON_VALGRIND) + return; +#endif + if (txn) { /* transaction start */ + if (env->poison_edge < txn->geo.first_unallocated) + env->poison_edge = txn->geo.first_unallocated; + VALGRIND_MAKE_MEM_DEFINED(env->dxb_mmap.base, pgno2bytes(env, txn->geo.first_unallocated)); + MDBX_ASAN_UNPOISON_MEMORY_REGION(env->dxb_mmap.base, pgno2bytes(env, txn->geo.first_unallocated)); + /* don't touch more, it should be already poisoned */ + } else { /* transaction end */ + bool should_unlock = false; + pgno_t last = MAX_PAGENO + 1; + if (env->pid != osal_getpid()) { + /* resurrect after fork */ + return; + } else if (env_owned_wrtxn(env)) { + /* inside write-txn */ + last = meta_recent(env, &env->basal_txn->tw.troika).ptr_v->geometry.first_unallocated; + } else if (env->flags & MDBX_RDONLY) { + /* read-only mode, no write-txn, no wlock mutex */ + last = NUM_METAS; + } else if (lck_txn_lock(env, true) == MDBX_SUCCESS) { + /* no write-txn */ + last = NUM_METAS; + should_unlock = true; } else { - /* grab a page to hold a temporary copy */ - tmp_ki_copy = page_malloc(mc->mc_txn, 1); - if (unlikely(tmp_ki_copy == NULL)) { - rc = MDBX_ENOMEM; - goto done; - } - - const size_t max_space = page_space(env); - const size_t new_size = IS_LEAF(mp) ? leaf_size(env, newkey, newdata) - : branch_size(env, newkey); - - /* prepare to insert */ - for (i = 0; i < newindx; ++i) - tmp_ki_copy->mp_ptrs[i] = mp->mp_ptrs[i]; - tmp_ki_copy->mp_ptrs[i] = (indx_t)-1; - while (++i <= nkeys) - tmp_ki_copy->mp_ptrs[i] = mp->mp_ptrs[i - 1]; - tmp_ki_copy->mp_pgno = mp->mp_pgno; - tmp_ki_copy->mp_flags = mp->mp_flags; - tmp_ki_copy->mp_txnid = INVALID_TXNID; - tmp_ki_copy->mp_lower = 0; - tmp_ki_copy->mp_upper = (indx_t)max_space; + /* write txn is running, therefore shouldn't poison any memory range */ + return; + } - /* Добавляемый узел может не поместиться в страницу-половину вместе - * с количественной половиной узлов из исходной страницы. В худшем случае, - * в страницу-половину с добавляемым узлом могут попасть самые больше узлы - * из исходной страницы, а другую половину только узлы с самыми короткими - * ключами и с пустыми данными. Поэтому, чтобы найти подходящую границу - * разреза требуется итерировать узлы и считая их объем. - * - * Однако, при простом количественном делении (без учета размера ключей - * и данных) на страницах-половинах будет примерно вдвое меньше узлов. - * Поэтому добавляемый узел точно поместится, если его размер не больше - * чем место "освобождающееся" от заголовков узлов, которые переедут - * в другую страницу-половину. Кроме этого, как минимум по одному байту - * будет в каждом ключе, в худшем случае кроме одного, который может быть - * нулевого размера. */ + last = mvcc_largest_this(env, last); + const pgno_t edge = env->poison_edge; + if (edge > last) { + eASSERT(env, last >= NUM_METAS); + env->poison_edge = last; + VALGRIND_MAKE_MEM_NOACCESS(ptr_disp(env->dxb_mmap.base, pgno2bytes(env, last)), pgno2bytes(env, edge - last)); + MDBX_ASAN_POISON_MEMORY_REGION(ptr_disp(env->dxb_mmap.base, pgno2bytes(env, last)), pgno2bytes(env, edge - last)); + } + if (should_unlock) + lck_txn_unlock(env); + } +} +#endif /* ENABLE_MEMCHECK || __SANITIZE_ADDRESS__ */ - if (newindx == split_indx && nkeys >= 5) { - STATIC_ASSERT(P_BRANCH == 1); - split_indx += mp->mp_flags & P_BRANCH; - } - eASSERT(env, split_indx >= minkeys && split_indx <= nkeys + 1 - minkeys); - const size_t dim_nodes = - (newindx >= split_indx) ? split_indx : nkeys - split_indx; - const size_t dim_used = (sizeof(indx_t) + NODESIZE + 1) * dim_nodes; - if (new_size >= dim_used) { - /* Search for best acceptable split point */ - i = (newindx < split_indx) ? 0 : nkeys; - intptr_t dir = (newindx < split_indx) ? 1 : -1; - size_t before = 0, after = new_size + page_used(env, mp); - size_t best_split = split_indx; - size_t best_shift = INT_MAX; +/* Turn on/off readahead. It's harmful when the DB is larger than RAM. */ +__cold int dxb_set_readahead(const MDBX_env *env, const pgno_t edge, const bool enable, const bool force_whole) { + eASSERT(env, edge >= NUM_METAS && edge <= MAX_PAGENO + 1); + eASSERT(env, (enable & 1) == (enable != 0)); + const bool toggle = force_whole || ((enable ^ env->lck->readahead_anchor) & 1) || !env->lck->readahead_anchor; + const pgno_t prev_edge = env->lck->readahead_anchor >> 1; + const size_t limit = env->dxb_mmap.limit; + size_t offset = toggle ? 0 : pgno_align2os_bytes(env, (prev_edge < edge) ? prev_edge : edge); + offset = (offset < limit) ? offset : limit; - TRACE("seek separator from %zu, step %zi, default %zu, new-idx %zu, " - "new-size %zu", - i, dir, split_indx, newindx, new_size); - do { - cASSERT(mc, i <= nkeys); - size_t size = new_size; - if (i != newindx) { - MDBX_node *node = ptr_disp(mp, tmp_ki_copy->mp_ptrs[i] + PAGEHDRSZ); - size = NODESIZE + node_ks(node) + sizeof(indx_t); - if (IS_LEAF(mp)) - size += (node_flags(node) & F_BIGDATA) ? sizeof(pgno_t) - : node_ds(node); - size = EVEN(size); - } + size_t length = pgno_align2os_bytes(env, (prev_edge < edge) ? edge : prev_edge); + length = (length < limit) ? length : limit; + length -= offset; - before += size; - after -= size; - TRACE("step %zu, size %zu, before %zu, after %zu, max %zu", i, size, - before, after, max_space); + eASSERT(env, 0 <= (intptr_t)length); + if (length == 0) + return MDBX_SUCCESS; - if (before <= max_space && after <= max_space) { - const size_t split = i + (dir > 0); - if (split >= minkeys && split <= nkeys + 1 - minkeys) { - const size_t shift = branchless_abs(split_indx - split); - if (shift >= best_shift) - break; - best_shift = shift; - best_split = split; - if (!best_shift) - break; - } - } - i += dir; - } while (i < nkeys); + NOTICE("readahead %s %u..%u", enable ? "ON" : "OFF", bytes2pgno(env, offset), bytes2pgno(env, offset + length)); - split_indx = best_split; - TRACE("chosen %zu", split_indx); - } - eASSERT(env, split_indx >= minkeys && split_indx <= nkeys + 1 - minkeys); +#if defined(F_RDAHEAD) + if (toggle && unlikely(fcntl(env->lazy_fd, F_RDAHEAD, enable) == -1)) + return errno; +#endif /* F_RDAHEAD */ - sepkey = *newkey; - if (split_indx != newindx) { - MDBX_node *node = - ptr_disp(mp, tmp_ki_copy->mp_ptrs[split_indx] + PAGEHDRSZ); - sepkey.iov_len = node_ks(node); - sepkey.iov_base = node_key(node); + int err; + void *const ptr = ptr_disp(env->dxb_mmap.base, offset); + if (enable) { +#if defined(MADV_NORMAL) + err = madvise(ptr, length, MADV_NORMAL) ? ignore_enosys_and_eagain(errno) : MDBX_SUCCESS; + if (unlikely(MDBX_IS_ERROR(err))) + return err; +#elif defined(POSIX_MADV_NORMAL) + err = ignore_enosys(posix_madvise(ptr, length, POSIX_MADV_NORMAL)); + if (unlikely(MDBX_IS_ERROR(err))) + return err; +#elif defined(POSIX_FADV_NORMAL) && defined(POSIX_FADV_WILLNEED) + err = ignore_enosys(posix_fadvise(env->lazy_fd, offset, length, POSIX_FADV_NORMAL)); + if (unlikely(MDBX_IS_ERROR(err))) + return err; +#elif defined(_WIN32) || defined(_WIN64) + /* no madvise on Windows */ +#else +#warning "FIXME" +#endif + if (toggle) { + /* NOTE: Seems there is a bug in the Mach/Darwin/OSX kernel, + * because MADV_WILLNEED with offset != 0 may cause SIGBUS + * on following access to the hinted region. + * 19.6.0 Darwin Kernel Version 19.6.0: Tue Jan 12 22:13:05 PST 2021; + * root:xnu-6153.141.16~1/RELEASE_X86_64 x86_64 */ +#if defined(F_RDADVISE) + struct radvisory hint; + hint.ra_offset = offset; + hint.ra_count = unlikely(length > INT_MAX && sizeof(length) > sizeof(hint.ra_count)) ? INT_MAX : (int)length; + (void)/* Ignore ENOTTY for DB on the ram-disk and so on */ fcntl(env->lazy_fd, F_RDADVISE, &hint); +#elif defined(MADV_WILLNEED) + err = madvise(ptr, length, MADV_WILLNEED) ? ignore_enosys_and_eagain(errno) : MDBX_SUCCESS; + if (unlikely(MDBX_IS_ERROR(err))) + return err; +#elif defined(POSIX_MADV_WILLNEED) + err = ignore_enosys(posix_madvise(ptr, length, POSIX_MADV_WILLNEED)); + if (unlikely(MDBX_IS_ERROR(err))) + return err; +#elif defined(_WIN32) || defined(_WIN64) + if (imports.PrefetchVirtualMemory) { + WIN32_MEMORY_RANGE_ENTRY hint; + hint.VirtualAddress = ptr; + hint.NumberOfBytes = length; + (void)imports.PrefetchVirtualMemory(GetCurrentProcess(), 1, &hint, 0); } +#elif defined(POSIX_FADV_WILLNEED) + err = ignore_enosys(posix_fadvise(env->lazy_fd, offset, length, POSIX_FADV_WILLNEED)); + if (unlikely(MDBX_IS_ERROR(err))) + return err; +#else +#warning "FIXME" +#endif } + } else { + mincore_clean_cache(env); +#if defined(MADV_RANDOM) + err = madvise(ptr, length, MADV_RANDOM) ? ignore_enosys_and_eagain(errno) : MDBX_SUCCESS; + if (unlikely(MDBX_IS_ERROR(err))) + return err; +#elif defined(POSIX_MADV_RANDOM) + err = ignore_enosys(posix_madvise(ptr, length, POSIX_MADV_RANDOM)); + if (unlikely(MDBX_IS_ERROR(err))) + return err; +#elif defined(POSIX_FADV_RANDOM) + err = ignore_enosys(posix_fadvise(env->lazy_fd, offset, length, POSIX_FADV_RANDOM)); + if (unlikely(MDBX_IS_ERROR(err))) + return err; +#elif defined(_WIN32) || defined(_WIN64) + /* no madvise on Windows */ +#else +#warning "FIXME" +#endif /* MADV_RANDOM */ } - DEBUG("separator is %zd [%s]", split_indx, DKEY_DEBUG(&sepkey)); - bool did_split_parent = false; - /* Copy separator key to the parent. */ - if (page_room(mn.mc_pg[ptop]) < branch_size(env, &sepkey)) { - TRACE("need split parent branch-page for key %s", DKEY_DEBUG(&sepkey)); - cASSERT(mc, page_numkeys(mn.mc_pg[ptop]) > 2); - cASSERT(mc, !pure_left); - const int snum = mc->mc_snum; - const int depth = mc->mc_db->md_depth; - mn.mc_snum--; - mn.mc_top--; - did_split_parent = true; - /* We want other splits to find mn when doing fixups */ - WITH_CURSOR_TRACKING( - mn, rc = page_split(&mn, &sepkey, NULL, sister->mp_pgno, 0)); - if (unlikely(rc != MDBX_SUCCESS)) - goto done; - cASSERT(mc, (int)mc->mc_snum - snum == mc->mc_db->md_depth - depth); - if (AUDIT_ENABLED()) { - rc = cursor_check_updating(mc); - if (unlikely(rc != MDBX_SUCCESS)) - goto done; - } + env->lck->readahead_anchor = (enable & 1) + (edge << 1); + err = MDBX_SUCCESS; + return err; +} - /* root split? */ - ptop += mc->mc_snum - (size_t)snum; +__cold int dxb_setup(MDBX_env *env, const int lck_rc, const mdbx_mode_t mode_bits) { + meta_t header; + eASSERT(env, !(env->flags & ENV_ACTIVE)); + int rc = MDBX_RESULT_FALSE; + int err = dxb_read_header(env, &header, lck_rc, mode_bits); + if (unlikely(err != MDBX_SUCCESS)) { + if (lck_rc != /* lck exclusive */ MDBX_RESULT_TRUE || err != MDBX_ENODATA || (env->flags & MDBX_RDONLY) != 0 || + /* recovery mode */ env->stuck_meta >= 0) + return err; - /* Right page might now have changed parent. - * Check if left page also changed parent. */ - if (mn.mc_pg[ptop] != mc->mc_pg[ptop] && - mc->mc_ki[ptop] >= page_numkeys(mc->mc_pg[ptop])) { - for (i = 0; i < ptop; i++) { - mc->mc_pg[i] = mn.mc_pg[i]; - mc->mc_ki[i] = mn.mc_ki[i]; - } - mc->mc_pg[ptop] = mn.mc_pg[ptop]; - if (mn.mc_ki[ptop]) { - mc->mc_ki[ptop] = mn.mc_ki[ptop] - 1; - } else { - /* find right page's left sibling */ - mc->mc_ki[ptop] = mn.mc_ki[ptop]; - rc = cursor_sibling(mc, SIBLING_LEFT); - if (unlikely(rc != MDBX_SUCCESS)) { - if (rc == MDBX_NOTFOUND) /* improper mdbx_cursor_sibling() result */ { - ERROR("unexpected %i error going left sibling", rc); - rc = MDBX_PROBLEM; - } - goto done; - } - } - } - } else if (unlikely(pure_left)) { - MDBX_page *ptop_page = mc->mc_pg[ptop]; - TRACE("pure-left: adding to parent page %u node[%u] left-leaf page #%u key " - "%s", - ptop_page->mp_pgno, mc->mc_ki[ptop], sister->mp_pgno, - DKEY(mc->mc_ki[ptop] ? newkey : NULL)); - assert(mc->mc_top == ptop + 1); - mc->mc_top = (uint8_t)ptop; - rc = node_add_branch(mc, mc->mc_ki[ptop], mc->mc_ki[ptop] ? newkey : NULL, - sister->mp_pgno); - cASSERT(mc, mp == mc->mc_pg[ptop + 1] && newindx == mc->mc_ki[ptop + 1] && - ptop == mc->mc_top); - - if (likely(rc == MDBX_SUCCESS) && mc->mc_ki[ptop] == 0) { - MDBX_node *node = page_node(mc->mc_pg[ptop], 1); - TRACE("pure-left: update prev-first key on parent to %s", DKEY(&sepkey)); - cASSERT(mc, node_ks(node) == 0 && node_pgno(node) == mp->mp_pgno); - cASSERT(mc, mc->mc_top == ptop && mc->mc_ki[ptop] == 0); - mc->mc_ki[ptop] = 1; - rc = update_key(mc, &sepkey); - cASSERT(mc, mc->mc_top == ptop && mc->mc_ki[ptop] == 1); - cASSERT(mc, mp == mc->mc_pg[ptop + 1] && newindx == mc->mc_ki[ptop + 1]); - mc->mc_ki[ptop] = 0; - } else { - TRACE("pure-left: no-need-update prev-first key on parent %s", - DKEY(&sepkey)); + DEBUG("%s", "create new database"); + rc = /* new database */ MDBX_RESULT_TRUE; + + if (!env->geo_in_bytes.now) { + /* set defaults if not configured */ + err = mdbx_env_set_geometry(env, 0, -1, -1, -1, -1, -1); + if (unlikely(err != MDBX_SUCCESS)) + return err; } - mc->mc_top++; - if (unlikely(rc != MDBX_SUCCESS)) - goto done; + err = env_page_auxbuffer(env); + if (unlikely(err != MDBX_SUCCESS)) + return err; - MDBX_node *node = page_node(mc->mc_pg[ptop], mc->mc_ki[ptop] + (size_t)1); - cASSERT(mc, node_pgno(node) == mp->mp_pgno && mc->mc_pg[ptop] == ptop_page); - } else { - mn.mc_top--; - TRACE("add-to-parent the right-entry[%u] for new sibling-page", - mn.mc_ki[ptop]); - rc = node_add_branch(&mn, mn.mc_ki[ptop], &sepkey, sister->mp_pgno); - mn.mc_top++; - if (unlikely(rc != MDBX_SUCCESS)) - goto done; - } + header = *meta_init_triplet(env, env->page_auxbuf); + err = osal_pwrite(env->lazy_fd, env->page_auxbuf, env->ps * (size_t)NUM_METAS, 0); + if (unlikely(err != MDBX_SUCCESS)) + return err; - if (unlikely(pure_left | pure_right)) { - mc->mc_pg[mc->mc_top] = sister; - mc->mc_ki[mc->mc_top] = 0; - switch (PAGETYPE_WHOLE(sister)) { - case P_LEAF: { - cASSERT(mc, newpgno == 0 || newpgno == P_INVALID); - rc = node_add_leaf(mc, 0, newkey, newdata, naf); - } break; - case P_LEAF | P_LEAF2: { - cASSERT(mc, (naf & (F_BIGDATA | F_SUBDATA | F_DUPDATA)) == 0); - cASSERT(mc, newpgno == 0 || newpgno == P_INVALID); - rc = node_add_leaf2(mc, 0, newkey); - } break; - default: - rc = bad_page(sister, "wrong page-type %u\n", PAGETYPE_WHOLE(sister)); - } - if (unlikely(rc != MDBX_SUCCESS)) - goto done; + err = osal_ftruncate(env->lazy_fd, env->dxb_mmap.filesize = env->dxb_mmap.current = env->geo_in_bytes.now); + if (unlikely(err != MDBX_SUCCESS)) + return err; - if (pure_right) { - for (i = 0; i < mc->mc_top; i++) - mc->mc_ki[i] = mn.mc_ki[i]; - } else if (mc->mc_ki[mc->mc_top - 1] == 0) { - for (i = 2; i <= mc->mc_top; ++i) - if (mc->mc_ki[mc->mc_top - i]) { - get_key( - page_node(mc->mc_pg[mc->mc_top - i], mc->mc_ki[mc->mc_top - i]), - &sepkey); - if (mc->mc_dbx->md_cmp(newkey, &sepkey) < 0) { - mc->mc_top -= (uint8_t)i; - DEBUG("pure-left: update new-first on parent [%i] page %u key %s", - mc->mc_ki[mc->mc_top], mc->mc_pg[mc->mc_top]->mp_pgno, - DKEY(newkey)); - rc = update_key(mc, newkey); - mc->mc_top += (uint8_t)i; - if (unlikely(rc != MDBX_SUCCESS)) - goto done; - } - break; - } +#ifndef NDEBUG /* just for checking */ + err = dxb_read_header(env, &header, lck_rc, mode_bits); + if (unlikely(err != MDBX_SUCCESS)) + return err; +#endif + } + + VERBOSE("header: root %" PRIaPGNO "/%" PRIaPGNO ", geo %" PRIaPGNO "/%" PRIaPGNO "-%" PRIaPGNO "/%" PRIaPGNO + " +%u -%u, txn_id %" PRIaTXN ", %s", + header.trees.main.root, header.trees.gc.root, header.geometry.lower, header.geometry.first_unallocated, + header.geometry.now, header.geometry.upper, pv2pages(header.geometry.grow_pv), + pv2pages(header.geometry.shrink_pv), unaligned_peek_u64(4, header.txnid_a), durable_caption(&header)); + + if (unlikely((header.trees.gc.flags & DB_PERSISTENT_FLAGS) != MDBX_INTEGERKEY)) { + ERROR("unexpected/invalid db-flags 0x%x for %s", header.trees.gc.flags, "GC/FreeDB"); + return MDBX_INCOMPATIBLE; + } + env->dbs_flags[FREE_DBI] = DB_VALID | MDBX_INTEGERKEY; + env->kvs[FREE_DBI].clc.k.cmp = cmp_int_align4; /* aligned MDBX_INTEGERKEY */ + env->kvs[FREE_DBI].clc.k.lmax = env->kvs[FREE_DBI].clc.k.lmin = 8; + env->kvs[FREE_DBI].clc.v.cmp = cmp_lenfast; + env->kvs[FREE_DBI].clc.v.lmin = 4; + env->kvs[FREE_DBI].clc.v.lmax = mdbx_env_get_maxvalsize_ex(env, MDBX_INTEGERKEY); + + if (env->ps != header.pagesize) + env_setup_pagesize(env, header.pagesize); + if ((env->flags & MDBX_RDONLY) == 0) { + err = env_page_auxbuffer(env); + if (unlikely(err != MDBX_SUCCESS)) + return err; + } + + size_t expected_filesize = 0; + const size_t used_bytes = pgno2bytes(env, header.geometry.first_unallocated); + const size_t used_aligned2os_bytes = ceil_powerof2(used_bytes, globals.sys_pagesize); + if ((env->flags & MDBX_RDONLY) /* readonly */ + || lck_rc != MDBX_RESULT_TRUE /* not exclusive */ + || /* recovery mode */ env->stuck_meta >= 0) { + /* use present params from db */ + const size_t pagesize = header.pagesize; + err = mdbx_env_set_geometry(env, header.geometry.lower * pagesize, header.geometry.now * pagesize, + header.geometry.upper * pagesize, pv2pages(header.geometry.grow_pv) * pagesize, + pv2pages(header.geometry.shrink_pv) * pagesize, header.pagesize); + if (unlikely(err != MDBX_SUCCESS)) { + ERROR("%s: err %d", "could not apply geometry from db", err); + return (err == MDBX_EINVAL) ? MDBX_INCOMPATIBLE : err; } - } else if (tmp_ki_copy) { /* !IS_LEAF2(mp) */ - /* Move nodes */ - mc->mc_pg[mc->mc_top] = sister; - i = split_indx; - size_t n = 0; - do { - TRACE("i %zu, nkeys %zu => n %zu, rp #%u", i, nkeys, n, sister->mp_pgno); - pgno_t pgno = 0; - MDBX_val *rdata = NULL; - if (i == newindx) { - rkey = *newkey; - if (IS_LEAF(mp)) - rdata = newdata; - else - pgno = newpgno; - flags = naf; - /* Update index for the new key. */ - mc->mc_ki[mc->mc_top] = (indx_t)n; - } else { - MDBX_node *node = ptr_disp(mp, tmp_ki_copy->mp_ptrs[i] + PAGEHDRSZ); - rkey.iov_base = node_key(node); - rkey.iov_len = node_ks(node); - if (IS_LEAF(mp)) { - xdata.iov_base = node_data(node); - xdata.iov_len = node_ds(node); - rdata = &xdata; - } else - pgno = node_pgno(node); - flags = node_flags(node); - } + } else if (env->geo_in_bytes.now) { + /* silently growth to last used page */ + if (env->geo_in_bytes.now < used_aligned2os_bytes) + env->geo_in_bytes.now = used_aligned2os_bytes; + if (env->geo_in_bytes.upper < used_aligned2os_bytes) + env->geo_in_bytes.upper = used_aligned2os_bytes; - switch (PAGETYPE_WHOLE(sister)) { - case P_BRANCH: { - cASSERT(mc, 0 == (uint16_t)flags); - /* First branch index doesn't need key data. */ - rc = node_add_branch(mc, n, n ? &rkey : NULL, pgno); - } break; - case P_LEAF: { - cASSERT(mc, pgno == 0); - cASSERT(mc, rdata != NULL); - rc = node_add_leaf(mc, n, &rkey, rdata, flags); - } break; - /* case P_LEAF | P_LEAF2: { - cASSERT(mc, (nflags & (F_BIGDATA | F_SUBDATA | F_DUPDATA)) == 0); - cASSERT(mc, gno == 0); - rc = mdbx_node_add_leaf2(mc, n, &rkey); - } break; */ - default: - rc = bad_page(sister, "wrong page-type %u\n", PAGETYPE_WHOLE(sister)); - } - if (unlikely(rc != MDBX_SUCCESS)) - goto done; + /* apply preconfigured params, but only if substantial changes: + * - upper or lower limit changes + * - shrink threshold or growth step + * But ignore change just a 'now/current' size. */ + if (bytes_align2os_bytes(env, env->geo_in_bytes.upper) != pgno2bytes(env, header.geometry.upper) || + bytes_align2os_bytes(env, env->geo_in_bytes.lower) != pgno2bytes(env, header.geometry.lower) || + bytes_align2os_bytes(env, env->geo_in_bytes.shrink) != pgno2bytes(env, pv2pages(header.geometry.shrink_pv)) || + bytes_align2os_bytes(env, env->geo_in_bytes.grow) != pgno2bytes(env, pv2pages(header.geometry.grow_pv))) { - ++n; - if (++i > nkeys) { - i = 0; - n = 0; - mc->mc_pg[mc->mc_top] = tmp_ki_copy; - TRACE("switch to mp #%u", tmp_ki_copy->mp_pgno); - } - } while (i != split_indx); + if (env->geo_in_bytes.shrink && env->geo_in_bytes.now > used_bytes) + /* pre-shrink if enabled */ + env->geo_in_bytes.now = used_bytes + env->geo_in_bytes.shrink - used_bytes % env->geo_in_bytes.shrink; - TRACE("i %zu, nkeys %zu, n %zu, pgno #%u", i, nkeys, n, - mc->mc_pg[mc->mc_top]->mp_pgno); + /* сейчас БД еще не открыта, поэтому этот вызов не изменит геометрию, но проверит и скорректирует параметры + * с учетом реального размера страницы. */ + err = mdbx_env_set_geometry(env, env->geo_in_bytes.lower, env->geo_in_bytes.now, env->geo_in_bytes.upper, + env->geo_in_bytes.grow, env->geo_in_bytes.shrink, header.pagesize); + if (unlikely(err != MDBX_SUCCESS)) { + ERROR("%s: err %d", "could not apply preconfigured db-geometry", err); + return (err == MDBX_EINVAL) ? MDBX_INCOMPATIBLE : err; + } - nkeys = page_numkeys(tmp_ki_copy); - for (i = 0; i < nkeys; i++) - mp->mp_ptrs[i] = tmp_ki_copy->mp_ptrs[i]; - mp->mp_lower = tmp_ki_copy->mp_lower; - mp->mp_upper = tmp_ki_copy->mp_upper; - memcpy(page_node(mp, nkeys - 1), page_node(tmp_ki_copy, nkeys - 1), - env->me_psize - tmp_ki_copy->mp_upper - PAGEHDRSZ); + /* altering fields to match geometry given from user */ + expected_filesize = pgno_align2os_bytes(env, header.geometry.now); + header.geometry.now = bytes2pgno(env, env->geo_in_bytes.now); + header.geometry.lower = bytes2pgno(env, env->geo_in_bytes.lower); + header.geometry.upper = bytes2pgno(env, env->geo_in_bytes.upper); + header.geometry.grow_pv = pages2pv(bytes2pgno(env, env->geo_in_bytes.grow)); + header.geometry.shrink_pv = pages2pv(bytes2pgno(env, env->geo_in_bytes.shrink)); - /* reset back to original page */ - if (newindx < split_indx) { - mc->mc_pg[mc->mc_top] = mp; + VERBOSE("amending: root %" PRIaPGNO "/%" PRIaPGNO ", geo %" PRIaPGNO "/%" PRIaPGNO "-%" PRIaPGNO "/%" PRIaPGNO + " +%u -%u, txn_id %" PRIaTXN ", %s", + header.trees.main.root, header.trees.gc.root, header.geometry.lower, header.geometry.first_unallocated, + header.geometry.now, header.geometry.upper, pv2pages(header.geometry.grow_pv), + pv2pages(header.geometry.shrink_pv), unaligned_peek_u64(4, header.txnid_a), durable_caption(&header)); } else { - mc->mc_pg[mc->mc_top] = sister; - mc->mc_ki[ptop]++; - /* Make sure mc_ki is still valid. */ - if (mn.mc_pg[ptop] != mc->mc_pg[ptop] && - mc->mc_ki[ptop] >= page_numkeys(mc->mc_pg[ptop])) { - for (i = 0; i <= ptop; i++) { - mc->mc_pg[i] = mn.mc_pg[i]; - mc->mc_ki[i] = mn.mc_ki[i]; - } - } + /* fetch back 'now/current' size, since it was ignored during comparison and may differ. */ + env->geo_in_bytes.now = pgno_align2os_bytes(env, header.geometry.now); } - } else if (newindx >= split_indx) { - mc->mc_pg[mc->mc_top] = sister; - mc->mc_ki[ptop]++; - /* Make sure mc_ki is still valid. */ - if (mn.mc_pg[ptop] != mc->mc_pg[ptop] && - mc->mc_ki[ptop] >= page_numkeys(mc->mc_pg[ptop])) { - for (i = 0; i <= ptop; i++) { - mc->mc_pg[i] = mn.mc_pg[i]; - mc->mc_ki[i] = mn.mc_ki[i]; + ENSURE(env, header.geometry.now >= header.geometry.first_unallocated); + } else { + /* geo-params are not pre-configured by user, get current values from the meta. */ + env->geo_in_bytes.now = pgno2bytes(env, header.geometry.now); + env->geo_in_bytes.lower = pgno2bytes(env, header.geometry.lower); + env->geo_in_bytes.upper = pgno2bytes(env, header.geometry.upper); + env->geo_in_bytes.grow = pgno2bytes(env, pv2pages(header.geometry.grow_pv)); + env->geo_in_bytes.shrink = pgno2bytes(env, pv2pages(header.geometry.shrink_pv)); + } + + ENSURE(env, pgno_align2os_bytes(env, header.geometry.now) == env->geo_in_bytes.now); + ENSURE(env, env->geo_in_bytes.now >= used_bytes); + if (!expected_filesize) + expected_filesize = env->geo_in_bytes.now; + const uint64_t filesize_before = env->dxb_mmap.filesize; + if (unlikely(filesize_before != env->geo_in_bytes.now)) { + if (lck_rc != /* lck exclusive */ MDBX_RESULT_TRUE) { + VERBOSE("filesize mismatch (expect %" PRIuPTR "b/%" PRIaPGNO "p, have %" PRIu64 "b/%" PRIu64 + "p), assume other process working", + env->geo_in_bytes.now, bytes2pgno(env, env->geo_in_bytes.now), filesize_before, + filesize_before >> env->ps2ln); + } else { + if (filesize_before != expected_filesize) + WARNING("filesize mismatch (expect %" PRIuSIZE "b/%" PRIaPGNO "p, have %" PRIu64 "b/%" PRIu64 "p)", + expected_filesize, bytes2pgno(env, expected_filesize), filesize_before, filesize_before >> env->ps2ln); + if (filesize_before < used_bytes) { + ERROR("last-page beyond end-of-file (last %" PRIaPGNO ", have %" PRIaPGNO ")", + header.geometry.first_unallocated, bytes2pgno(env, (size_t)filesize_before)); + return MDBX_CORRUPTED; } - } - } - /* Adjust other cursors pointing to mp and/or to parent page */ - nkeys = page_numkeys(mp); - for (MDBX_cursor *m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; - m2 = m2->mc_next) { - MDBX_cursor *m3 = (mc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2; - if (m3 == mc) - continue; - if (!(m2->mc_flags & m3->mc_flags & C_INITIALIZED)) - continue; - if (foliage) { - /* sub cursors may be on different DB */ - if (m3->mc_pg[0] != mp) - continue; - /* root split */ - for (int k = foliage; k >= 0; k--) { - m3->mc_ki[k + 1] = m3->mc_ki[k]; - m3->mc_pg[k + 1] = m3->mc_pg[k]; - } - m3->mc_ki[0] = m3->mc_ki[0] >= nkeys + pure_left; - m3->mc_pg[0] = mc->mc_pg[0]; - m3->mc_snum++; - m3->mc_top++; - } - - if (m3->mc_top >= mc->mc_top && m3->mc_pg[mc->mc_top] == mp && !pure_left) { - if (m3->mc_ki[mc->mc_top] >= newindx && !(naf & MDBX_SPLIT_REPLACE)) - m3->mc_ki[mc->mc_top]++; - if (m3->mc_ki[mc->mc_top] >= nkeys) { - m3->mc_pg[mc->mc_top] = sister; - cASSERT(mc, m3->mc_ki[mc->mc_top] >= nkeys); - m3->mc_ki[mc->mc_top] -= (indx_t)nkeys; - for (i = 0; i < mc->mc_top; i++) { - m3->mc_ki[i] = mn.mc_ki[i]; - m3->mc_pg[i] = mn.mc_pg[i]; + if (env->flags & MDBX_RDONLY) { + if (filesize_before & (globals.sys_allocation_granularity - 1)) { + ERROR("filesize should be rounded-up to system allocation granularity %u", + globals.sys_allocation_granularity); + return MDBX_WANNA_RECOVERY; } + WARNING("%s", "ignore filesize mismatch in readonly-mode"); + } else { + VERBOSE("will resize datafile to %" PRIuSIZE " bytes, %" PRIaPGNO " pages", env->geo_in_bytes.now, + bytes2pgno(env, env->geo_in_bytes.now)); } - } else if (!did_split_parent && m3->mc_top >= ptop && - m3->mc_pg[ptop] == mc->mc_pg[ptop] && - m3->mc_ki[ptop] >= mc->mc_ki[ptop]) { - m3->mc_ki[ptop]++; /* also for the `pure-left` case */ } - if (XCURSOR_INITED(m3) && IS_LEAF(mp)) - XCURSOR_REFRESH(m3, m3->mc_pg[mc->mc_top], m3->mc_ki[mc->mc_top]); } - TRACE("mp #%u left: %zd, sister #%u left: %zd", mp->mp_pgno, page_room(mp), - sister->mp_pgno, page_room(sister)); -done: - if (tmp_ki_copy) - dpage_free(env, tmp_ki_copy, 1); + VERBOSE("current boot-id %" PRIx64 "-%" PRIx64 " (%savailable)", globals.bootid.x, globals.bootid.y, + (globals.bootid.x | globals.bootid.y) ? "" : "not-"); - if (unlikely(rc != MDBX_SUCCESS)) - mc->mc_txn->mt_flags |= MDBX_TXN_ERROR; - else { - if (AUDIT_ENABLED()) - rc = cursor_check_updating(mc); - if (unlikely(naf & MDBX_RESERVE)) { - MDBX_node *node = page_node(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); - if (!(node_flags(node) & F_BIGDATA)) - newdata->iov_base = node_data(node); - } -#if MDBX_ENABLE_PGOP_STAT - env->me_lck->mti_pgop_stat.split.weak += 1; -#endif /* MDBX_ENABLE_PGOP_STAT */ + /* calculate readahead hint before mmap with zero redundant pages */ + const bool readahead = + !(env->flags & MDBX_NORDAHEAD) && mdbx_is_readahead_reasonable(used_bytes, 0) == MDBX_RESULT_TRUE; + + err = osal_mmap(env->flags, &env->dxb_mmap, env->geo_in_bytes.now, env->geo_in_bytes.upper, + (lck_rc && env->stuck_meta < 0) ? MMAP_OPTION_TRUNCATE : 0, env->pathname.dxb); + if (unlikely(err != MDBX_SUCCESS)) + return err; + +#if defined(MADV_DONTDUMP) + err = + madvise(env->dxb_mmap.base, env->dxb_mmap.limit, MADV_DONTDUMP) ? ignore_enosys_and_eagain(errno) : MDBX_SUCCESS; + if (unlikely(MDBX_IS_ERROR(err))) + return err; +#endif /* MADV_DONTDUMP */ +#if defined(MADV_DODUMP) + if (globals.runtime_flags & MDBX_DBG_DUMP) { + const size_t meta_length_aligned2os = pgno_align2os_bytes(env, NUM_METAS); + err = madvise(env->dxb_mmap.base, meta_length_aligned2os, MADV_DODUMP) ? ignore_enosys_and_eagain(errno) + : MDBX_SUCCESS; + if (unlikely(MDBX_IS_ERROR(err))) + return err; } +#endif /* MADV_DODUMP */ - DEBUG("<< mp #%u, rc %d", mp->mp_pgno, rc); - return rc; -} +#ifdef ENABLE_MEMCHECK + env->valgrind_handle = VALGRIND_CREATE_BLOCK(env->dxb_mmap.base, env->dxb_mmap.limit, "mdbx"); +#endif /* ENABLE_MEMCHECK */ -int mdbx_put(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, MDBX_val *data, - MDBX_put_flags_t flags) { - int rc = check_txn_rw(txn, MDBX_TXN_BLOCKED); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; + eASSERT(env, used_bytes >= pgno2bytes(env, NUM_METAS) && used_bytes <= env->dxb_mmap.limit); +#if defined(ENABLE_MEMCHECK) || defined(__SANITIZE_ADDRESS__) + if (env->dxb_mmap.filesize > used_bytes && env->dxb_mmap.filesize < env->dxb_mmap.limit) { + VALGRIND_MAKE_MEM_NOACCESS(ptr_disp(env->dxb_mmap.base, used_bytes), env->dxb_mmap.filesize - used_bytes); + MDBX_ASAN_POISON_MEMORY_REGION(ptr_disp(env->dxb_mmap.base, used_bytes), env->dxb_mmap.filesize - used_bytes); + } + env->poison_edge = + bytes2pgno(env, (env->dxb_mmap.filesize < env->dxb_mmap.limit) ? env->dxb_mmap.filesize : env->dxb_mmap.limit); +#endif /* ENABLE_MEMCHECK || __SANITIZE_ADDRESS__ */ - if (unlikely(!key || !data)) - return MDBX_EINVAL; + troika_t troika = meta_tap(env); +#if MDBX_DEBUG + meta_troika_dump(env, &troika); +#endif + //-------------------------------- validate/rollback head & steady meta-pages + if (unlikely(env->stuck_meta >= 0)) { + /* recovery mode */ + meta_t clone; + meta_t const *const target = METAPAGE(env, env->stuck_meta); + err = meta_validate_copy(env, target, &clone); + if (unlikely(err != MDBX_SUCCESS)) { + ERROR("target meta[%u] is corrupted", bytes2pgno(env, ptr_dist(data_page(target), env->dxb_mmap.base))); + meta_troika_dump(env, &troika); + return MDBX_CORRUPTED; + } + } else /* not recovery mode */ + while (1) { + const unsigned meta_clash_mask = meta_eq_mask(&troika); + if (unlikely(meta_clash_mask)) { + ERROR("meta-pages are clashed: mask 0x%d", meta_clash_mask); + meta_troika_dump(env, &troika); + return MDBX_CORRUPTED; + } - if (unlikely(!check_dbi(txn, dbi, DBI_USRVALID))) - return MDBX_BAD_DBI; + if (lck_rc != /* lck exclusive */ MDBX_RESULT_TRUE) { + /* non-exclusive mode, + * meta-pages should be validated by a first process opened the DB */ + if (troika.recent == troika.prefer_steady) + break; - if (unlikely(flags & ~(MDBX_NOOVERWRITE | MDBX_NODUPDATA | MDBX_ALLDUPS | - MDBX_ALLDUPS | MDBX_RESERVE | MDBX_APPEND | - MDBX_APPENDDUP | MDBX_CURRENT | MDBX_MULTIPLE))) - return MDBX_EINVAL; + if (!env->lck_mmap.lck) { + /* LY: without-lck (read-only) mode, so it is impossible that other + * process made weak checkpoint. */ + ERROR("%s", "without-lck, unable recovery/rollback"); + meta_troika_dump(env, &troika); + return MDBX_WANNA_RECOVERY; + } - if (unlikely(txn->mt_flags & (MDBX_TXN_RDONLY | MDBX_TXN_BLOCKED))) - return (txn->mt_flags & MDBX_TXN_RDONLY) ? MDBX_EACCESS : MDBX_BAD_TXN; + /* LY: assume just have a collision with other running process, + * or someone make a weak checkpoint */ + VERBOSE("%s", "assume collision or online weak checkpoint"); + break; + } + eASSERT(env, lck_rc == MDBX_RESULT_TRUE); + /* exclusive mode */ - MDBX_cursor_couple cx; - rc = cursor_init(&cx.outer, txn, dbi); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - cx.outer.mc_next = txn->mt_cursors[dbi]; - txn->mt_cursors[dbi] = &cx.outer; + const meta_ptr_t recent = meta_recent(env, &troika); + const meta_ptr_t prefer_steady = meta_prefer_steady(env, &troika); + meta_t clone; + if (prefer_steady.is_steady) { + err = meta_validate_copy(env, prefer_steady.ptr_c, &clone); + if (unlikely(err != MDBX_SUCCESS)) { + ERROR("meta[%u] with %s txnid %" PRIaTXN " is corrupted, %s needed", + bytes2pgno(env, ptr_dist(prefer_steady.ptr_c, env->dxb_mmap.base)), "steady", prefer_steady.txnid, + "manual recovery"); + meta_troika_dump(env, &troika); + return MDBX_CORRUPTED; + } + if (prefer_steady.ptr_c == recent.ptr_c) + break; + } - /* LY: support for update (explicit overwrite) */ - if (flags & MDBX_CURRENT) { - rc = cursor_set(&cx.outer, (MDBX_val *)key, NULL, MDBX_SET).err; - if (likely(rc == MDBX_SUCCESS) && - (txn->mt_dbs[dbi].md_flags & MDBX_DUPSORT) && - (flags & MDBX_ALLDUPS) == 0) { - /* LY: allows update (explicit overwrite) only for unique keys */ - MDBX_node *node = page_node(cx.outer.mc_pg[cx.outer.mc_top], - cx.outer.mc_ki[cx.outer.mc_top]); - if (node_flags(node) & F_DUPDATA) { - tASSERT(txn, XCURSOR_INITED(&cx.outer) && - cx.outer.mc_xcursor->mx_db.md_entries > 1); - rc = MDBX_EMULTIVAL; - if ((flags & MDBX_NOOVERWRITE) == 0) { - flags -= MDBX_CURRENT; - rc = cursor_del(&cx.outer, MDBX_ALLDUPS); + const pgno_t pgno = bytes2pgno(env, ptr_dist(recent.ptr_c, env->dxb_mmap.base)); + const bool last_valid = meta_validate_copy(env, recent.ptr_c, &clone) == MDBX_SUCCESS; + eASSERT(env, !prefer_steady.is_steady || recent.txnid != prefer_steady.txnid); + if (unlikely(!last_valid)) { + if (unlikely(!prefer_steady.is_steady)) { + ERROR("%s for open or automatic rollback, %s", "there are no suitable meta-pages", + "manual recovery is required"); + meta_troika_dump(env, &troika); + return MDBX_CORRUPTED; } + WARNING("meta[%u] with last txnid %" PRIaTXN " is corrupted, rollback needed", pgno, recent.txnid); + meta_troika_dump(env, &troika); + goto purge_meta_head; } - } - } - if (likely(rc == MDBX_SUCCESS)) - rc = cursor_put_checklen(&cx.outer, key, data, flags); - txn->mt_cursors[dbi] = cx.outer.mc_next; + if (meta_bootid_match(recent.ptr_c)) { + if (env->flags & MDBX_RDONLY) { + ERROR("%s, but boot-id(%016" PRIx64 "-%016" PRIx64 ") is MATCH: " + "rollback NOT needed, steady-sync NEEDED%s", + "opening after an unclean shutdown", globals.bootid.x, globals.bootid.y, + ", but unable in read-only mode"); + meta_troika_dump(env, &troika); + return MDBX_WANNA_RECOVERY; + } + WARNING("%s, but boot-id(%016" PRIx64 "-%016" PRIx64 ") is MATCH: " + "rollback NOT needed, steady-sync NEEDED%s", + "opening after an unclean shutdown", globals.bootid.x, globals.bootid.y, ""); + header = clone; + env->lck->unsynced_pages.weak = header.geometry.first_unallocated; + if (!env->lck->eoos_timestamp.weak) + env->lck->eoos_timestamp.weak = osal_monotime(); + break; + } + if (unlikely(!prefer_steady.is_steady)) { + ERROR("%s, but %s for automatic rollback: %s", "opening after an unclean shutdown", + "there are no suitable meta-pages", "manual recovery is required"); + meta_troika_dump(env, &troika); + return MDBX_CORRUPTED; + } + if (env->flags & MDBX_RDONLY) { + ERROR("%s and rollback needed: (from head %" PRIaTXN " to steady %" PRIaTXN ")%s", + "opening after an unclean shutdown", recent.txnid, prefer_steady.txnid, ", but unable in read-only mode"); + meta_troika_dump(env, &troika); + return MDBX_WANNA_RECOVERY; + } - return rc; -} + purge_meta_head: + NOTICE("%s and doing automatic rollback: " + "purge%s meta[%u] with%s txnid %" PRIaTXN, + "opening after an unclean shutdown", last_valid ? "" : " invalid", pgno, last_valid ? " weak" : "", + recent.txnid); + meta_troika_dump(env, &troika); + ENSURE(env, prefer_steady.is_steady); + err = meta_override(env, pgno, 0, last_valid ? recent.ptr_c : prefer_steady.ptr_c); + if (err) { + ERROR("rollback: overwrite meta[%u] with txnid %" PRIaTXN ", error %d", pgno, recent.txnid, err); + return err; + } + troika = meta_tap(env); + ENSURE(env, 0 == meta_txnid(recent.ptr_v)); + ENSURE(env, 0 == meta_eq_mask(&troika)); + } -/**** COPYING *****************************************************************/ + if (lck_rc == /* lck exclusive */ MDBX_RESULT_TRUE) { + //-------------------------------------------------- shrink DB & update geo + /* re-check size after mmap */ + if ((env->dxb_mmap.current & (globals.sys_pagesize - 1)) != 0 || env->dxb_mmap.current < used_bytes) { + ERROR("unacceptable/unexpected datafile size %" PRIuPTR, env->dxb_mmap.current); + return MDBX_PROBLEM; + } + if (env->dxb_mmap.current != env->geo_in_bytes.now) { + header.geometry.now = bytes2pgno(env, env->dxb_mmap.current); + NOTICE("need update meta-geo to filesize %" PRIuPTR " bytes, %" PRIaPGNO " pages", env->dxb_mmap.current, + header.geometry.now); + } -/* State needed for a double-buffering compacting copy. */ -typedef struct mdbx_compacting_ctx { - MDBX_env *mc_env; - MDBX_txn *mc_txn; - osal_condpair_t mc_condpair; - uint8_t *mc_wbuf[2]; - size_t mc_wlen[2]; - mdbx_filehandle_t mc_fd; - /* Error code. Never cleared if set. Both threads can set nonzero - * to fail the copy. Not mutex-protected, MDBX expects atomic int. */ - volatile int mc_error; - pgno_t mc_next_pgno; - volatile unsigned mc_head; - volatile unsigned mc_tail; -} mdbx_compacting_ctx; + const meta_ptr_t recent = meta_recent(env, &troika); + if (/* не учитываем различия в geo.first_unallocated */ + header.geometry.grow_pv != recent.ptr_c->geometry.grow_pv || + header.geometry.shrink_pv != recent.ptr_c->geometry.shrink_pv || + header.geometry.lower != recent.ptr_c->geometry.lower || + header.geometry.upper != recent.ptr_c->geometry.upper || header.geometry.now != recent.ptr_c->geometry.now) { + if ((env->flags & MDBX_RDONLY) != 0 || + /* recovery mode */ env->stuck_meta >= 0) { + WARNING("skipped update meta.geo in %s mode: from l%" PRIaPGNO "-n%" PRIaPGNO "-u%" PRIaPGNO + "/s%u-g%u, to l%" PRIaPGNO "-n%" PRIaPGNO "-u%" PRIaPGNO "/s%u-g%u", + (env->stuck_meta < 0) ? "read-only" : "recovery", recent.ptr_c->geometry.lower, + recent.ptr_c->geometry.now, recent.ptr_c->geometry.upper, pv2pages(recent.ptr_c->geometry.shrink_pv), + pv2pages(recent.ptr_c->geometry.grow_pv), header.geometry.lower, header.geometry.now, + header.geometry.upper, pv2pages(header.geometry.shrink_pv), pv2pages(header.geometry.grow_pv)); + } else { + const txnid_t next_txnid = safe64_txnid_next(recent.txnid); + if (unlikely(next_txnid > MAX_TXNID)) { + ERROR("txnid overflow, raise %d", MDBX_TXN_FULL); + return MDBX_TXN_FULL; + } + NOTICE("updating meta.geo: " + "from l%" PRIaPGNO "-n%" PRIaPGNO "-u%" PRIaPGNO "/s%u-g%u (txn#%" PRIaTXN "), " + "to l%" PRIaPGNO "-n%" PRIaPGNO "-u%" PRIaPGNO "/s%u-g%u (txn#%" PRIaTXN ")", + recent.ptr_c->geometry.lower, recent.ptr_c->geometry.now, recent.ptr_c->geometry.upper, + pv2pages(recent.ptr_c->geometry.shrink_pv), pv2pages(recent.ptr_c->geometry.grow_pv), recent.txnid, + header.geometry.lower, header.geometry.now, header.geometry.upper, pv2pages(header.geometry.shrink_pv), + pv2pages(header.geometry.grow_pv), next_txnid); -/* Dedicated writer thread for compacting copy. */ -__cold static THREAD_RESULT THREAD_CALL compacting_write_thread(void *arg) { - mdbx_compacting_ctx *const ctx = arg; - -#if defined(EPIPE) && !(defined(_WIN32) || defined(_WIN64)) - sigset_t sigset; - sigemptyset(&sigset); - sigaddset(&sigset, SIGPIPE); - ctx->mc_error = pthread_sigmask(SIG_BLOCK, &sigset, NULL); -#endif /* EPIPE */ - - osal_condpair_lock(&ctx->mc_condpair); - while (!ctx->mc_error) { - while (ctx->mc_tail == ctx->mc_head && !ctx->mc_error) { - int err = osal_condpair_wait(&ctx->mc_condpair, true); - if (err != MDBX_SUCCESS) { - ctx->mc_error = err; - goto bailout; - } - } - const unsigned toggle = ctx->mc_tail & 1; - size_t wsize = ctx->mc_wlen[toggle]; - if (wsize == 0) { - ctx->mc_tail += 1; - break /* EOF */; - } - ctx->mc_wlen[toggle] = 0; - uint8_t *ptr = ctx->mc_wbuf[toggle]; - if (!ctx->mc_error) { - int err = osal_write(ctx->mc_fd, ptr, wsize); - if (err != MDBX_SUCCESS) { -#if defined(EPIPE) && !(defined(_WIN32) || defined(_WIN64)) - if (err == EPIPE) { - /* Collect the pending SIGPIPE, - * otherwise at least OS X gives it to the process on thread-exit. */ - int unused; - sigwait(&sigset, &unused); + ENSURE(env, header.unsafe_txnid == recent.txnid); + meta_set_txnid(env, &header, next_txnid); + err = dxb_sync_locked(env, env->flags | txn_shrink_allowed, &header, &troika); + if (err) { + ERROR("error %d, while updating meta.geo: " + "from l%" PRIaPGNO "-n%" PRIaPGNO "-u%" PRIaPGNO "/s%u-g%u (txn#%" PRIaTXN "), " + "to l%" PRIaPGNO "-n%" PRIaPGNO "-u%" PRIaPGNO "/s%u-g%u (txn#%" PRIaTXN ")", + err, recent.ptr_c->geometry.lower, recent.ptr_c->geometry.now, recent.ptr_c->geometry.upper, + pv2pages(recent.ptr_c->geometry.shrink_pv), pv2pages(recent.ptr_c->geometry.grow_pv), recent.txnid, + header.geometry.lower, header.geometry.now, header.geometry.upper, pv2pages(header.geometry.shrink_pv), + pv2pages(header.geometry.grow_pv), header.unsafe_txnid); + return err; } -#endif /* EPIPE */ - ctx->mc_error = err; - goto bailout; } } - ctx->mc_tail += 1; - osal_condpair_signal(&ctx->mc_condpair, false); - } -bailout: - osal_condpair_unlock(&ctx->mc_condpair); - return (THREAD_RESULT)0; -} - -/* Give buffer and/or MDBX_EOF to writer thread, await unused buffer. */ -__cold static int compacting_toggle_write_buffers(mdbx_compacting_ctx *ctx) { - osal_condpair_lock(&ctx->mc_condpair); - eASSERT(ctx->mc_env, ctx->mc_head - ctx->mc_tail < 2 || ctx->mc_error); - ctx->mc_head += 1; - osal_condpair_signal(&ctx->mc_condpair, true); - while (!ctx->mc_error && - ctx->mc_head - ctx->mc_tail == 2 /* both buffers in use */) { - int err = osal_condpair_wait(&ctx->mc_condpair, false); - if (err != MDBX_SUCCESS) - ctx->mc_error = err; - } - osal_condpair_unlock(&ctx->mc_condpair); - return ctx->mc_error; -} -__cold static int compacting_walk_sdb(mdbx_compacting_ctx *ctx, MDBX_db *sdb); + atomic_store32(&env->lck->discarded_tail, bytes2pgno(env, used_aligned2os_bytes), mo_Relaxed); -static int compacting_put_bytes(mdbx_compacting_ctx *ctx, const void *src, - size_t bytes, pgno_t pgno, pgno_t npages) { - assert(pgno == 0 || bytes > PAGEHDRSZ); - while (bytes > 0) { - const size_t side = ctx->mc_head & 1; - const size_t left = MDBX_ENVCOPY_WRITEBUF - ctx->mc_wlen[side]; - if (left < (pgno ? PAGEHDRSZ : 1)) { - int err = compacting_toggle_write_buffers(ctx); - if (unlikely(err != MDBX_SUCCESS)) - return err; - continue; - } - const size_t chunk = (bytes < left) ? bytes : left; - void *const dst = ctx->mc_wbuf[side] + ctx->mc_wlen[side]; - if (src) { - memcpy(dst, src, chunk); - if (pgno) { - assert(chunk > PAGEHDRSZ); - MDBX_page *mp = dst; - mp->mp_pgno = pgno; - if (mp->mp_txnid == 0) - mp->mp_txnid = ctx->mc_txn->mt_txnid; - if (mp->mp_flags == P_OVERFLOW) { - assert(bytes <= pgno2bytes(ctx->mc_env, npages)); - mp->mp_pages = npages; + if ((env->flags & MDBX_RDONLY) == 0 && env->stuck_meta < 0 && + (globals.runtime_flags & MDBX_DBG_DONT_UPGRADE) == 0) { + for (unsigned n = 0; n < NUM_METAS; ++n) { + meta_t *const meta = METAPAGE(env, n); + if (unlikely(unaligned_peek_u64(4, &meta->magic_and_version) != MDBX_DATA_MAGIC) || + (meta->dxbid.x | meta->dxbid.y) == 0 || (meta->gc_flags & ~DB_PERSISTENT_FLAGS)) { + const txnid_t txnid = meta_is_used(&troika, n) ? constmeta_txnid(meta) : 0; + NOTICE("%s %s" + "meta[%u], txnid %" PRIaTXN, + "updating db-format/guid signature for", meta_is_steady(meta) ? "stead-" : "weak-", n, txnid); + err = meta_override(env, n, txnid, meta); + if (unlikely(err != MDBX_SUCCESS) && + /* Just ignore the MDBX_PROBLEM error, since here it is + * returned only in case of the attempt to upgrade an obsolete + * meta-page that is invalid for current state of a DB, + * e.g. after shrinking DB file */ + err != MDBX_PROBLEM) { + ERROR("%s meta[%u], txnid %" PRIaTXN ", error %d", "updating db-format signature for", n, txnid, err); + return err; + } + troika = meta_tap(env); } - pgno = 0; } - src = ptr_disp(src, chunk); - } else - memset(dst, 0, chunk); - bytes -= chunk; - ctx->mc_wlen[side] += chunk; - } - return MDBX_SUCCESS; -} + } + } /* lck exclusive, lck_rc == MDBX_RESULT_TRUE */ -static int compacting_put_page(mdbx_compacting_ctx *ctx, const MDBX_page *mp, - const size_t head_bytes, const size_t tail_bytes, - const pgno_t npages) { - if (tail_bytes) { - assert(head_bytes + tail_bytes <= ctx->mc_env->me_psize); - assert(npages == 1 && - (PAGETYPE_WHOLE(mp) == P_BRANCH || PAGETYPE_WHOLE(mp) == P_LEAF)); - } else { - assert(head_bytes <= pgno2bytes(ctx->mc_env, npages)); - assert((npages == 1 && PAGETYPE_WHOLE(mp) == (P_LEAF | P_LEAF2)) || - PAGETYPE_WHOLE(mp) == P_OVERFLOW); + //---------------------------------------------------- setup madvise/readahead + if (used_aligned2os_bytes < env->dxb_mmap.current) { +#if defined(MADV_REMOVE) + if (lck_rc && (env->flags & MDBX_WRITEMAP) != 0 && + /* not recovery mode */ env->stuck_meta < 0) { + NOTICE("open-MADV_%s %u..%u", "REMOVE (deallocate file space)", env->lck->discarded_tail.weak, + bytes2pgno(env, env->dxb_mmap.current)); + err = madvise(ptr_disp(env->dxb_mmap.base, used_aligned2os_bytes), env->dxb_mmap.current - used_aligned2os_bytes, + MADV_REMOVE) + ? ignore_enosys_and_eagain(errno) + : MDBX_SUCCESS; + if (unlikely(MDBX_IS_ERROR(err))) + return err; + } +#endif /* MADV_REMOVE */ +#if defined(MADV_DONTNEED) + NOTICE("open-MADV_%s %u..%u", "DONTNEED", env->lck->discarded_tail.weak, bytes2pgno(env, env->dxb_mmap.current)); + err = madvise(ptr_disp(env->dxb_mmap.base, used_aligned2os_bytes), env->dxb_mmap.current - used_aligned2os_bytes, + MADV_DONTNEED) + ? ignore_enosys_and_eagain(errno) + : MDBX_SUCCESS; + if (unlikely(MDBX_IS_ERROR(err))) + return err; +#elif defined(POSIX_MADV_DONTNEED) + err = ignore_enosys(posix_madvise(ptr_disp(env->dxb_mmap.base, used_aligned2os_bytes), + env->dxb_mmap.current - used_aligned2os_bytes, POSIX_MADV_DONTNEED)); + if (unlikely(MDBX_IS_ERROR(err))) + return err; +#elif defined(POSIX_FADV_DONTNEED) + err = ignore_enosys(posix_fadvise(env->lazy_fd, used_aligned2os_bytes, + env->dxb_mmap.current - used_aligned2os_bytes, POSIX_FADV_DONTNEED)); + if (unlikely(MDBX_IS_ERROR(err))) + return err; +#endif /* MADV_DONTNEED */ } - const pgno_t pgno = ctx->mc_next_pgno; - ctx->mc_next_pgno += npages; - int err = compacting_put_bytes(ctx, mp, head_bytes, pgno, npages); - if (unlikely(err != MDBX_SUCCESS)) - return err; - err = compacting_put_bytes( - ctx, nullptr, pgno2bytes(ctx->mc_env, npages) - (head_bytes + tail_bytes), - 0, 0); + err = dxb_set_readahead(env, bytes2pgno(env, used_bytes), readahead, true); if (unlikely(err != MDBX_SUCCESS)) return err; - return compacting_put_bytes( - ctx, ptr_disp(mp, ctx->mc_env->me_psize - tail_bytes), tail_bytes, 0, 0); -} -__cold static int compacting_walk_tree(mdbx_compacting_ctx *ctx, - MDBX_cursor *mc, pgno_t *root, - txnid_t parent_txnid) { - mc->mc_snum = 1; - int rc = page_get(mc, *root, &mc->mc_pg[0], parent_txnid); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; + return rc; +} - rc = page_search_root(mc, nullptr, MDBX_PS_FIRST); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; +int dxb_sync_locked(MDBX_env *env, unsigned flags, meta_t *const pending, troika_t *const troika) { + eASSERT(env, ((env->flags ^ flags) & MDBX_WRITEMAP) == 0); + eASSERT(env, pending->trees.gc.flags == MDBX_INTEGERKEY); + eASSERT(env, check_table_flags(pending->trees.main.flags)); + const meta_t *const meta0 = METAPAGE(env, 0); + const meta_t *const meta1 = METAPAGE(env, 1); + const meta_t *const meta2 = METAPAGE(env, 2); + const meta_ptr_t head = meta_recent(env, troika); + int rc; - /* Make cursor pages writable */ - void *const buf = osal_malloc(pgno2bytes(ctx->mc_env, mc->mc_snum)); - if (buf == NULL) - return MDBX_ENOMEM; + eASSERT(env, pending < METAPAGE(env, 0) || pending > METAPAGE(env, NUM_METAS)); + eASSERT(env, (env->flags & (MDBX_RDONLY | ENV_FATAL_ERROR)) == 0); + eASSERT(env, pending->geometry.first_unallocated <= pending->geometry.now); - void *ptr = buf; - for (size_t i = 0; i < mc->mc_top; i++) { - page_copy(ptr, mc->mc_pg[i], ctx->mc_env->me_psize); - mc->mc_pg[i] = ptr; - ptr = ptr_disp(ptr, ctx->mc_env->me_psize); + if (flags & MDBX_SAFE_NOSYNC) { + /* Check auto-sync conditions */ + const pgno_t autosync_threshold = atomic_load32(&env->lck->autosync_threshold, mo_Relaxed); + const uint64_t autosync_period = atomic_load64(&env->lck->autosync_period, mo_Relaxed); + uint64_t eoos_timestamp; + if ((autosync_threshold && atomic_load64(&env->lck->unsynced_pages, mo_Relaxed) >= autosync_threshold) || + (autosync_period && (eoos_timestamp = atomic_load64(&env->lck->eoos_timestamp, mo_Relaxed)) && + osal_monotime() - eoos_timestamp >= autosync_period)) + flags &= MDBX_WRITEMAP | txn_shrink_allowed; /* force steady */ } - /* This is writable space for a leaf page. Usually not needed. */ - MDBX_page *const leaf = ptr; - - while (mc->mc_snum > 0) { - MDBX_page *mp = mc->mc_pg[mc->mc_top]; - size_t n = page_numkeys(mp); - - if (IS_LEAF(mp)) { - if (!(mc->mc_flags & - C_SUB) /* may have nested F_SUBDATA or F_BIGDATA nodes */) { - for (size_t i = 0; i < n; i++) { - MDBX_node *node = page_node(mp, i); - if (node_flags(node) == F_BIGDATA) { - /* Need writable leaf */ - if (mp != leaf) { - mc->mc_pg[mc->mc_top] = leaf; - page_copy(leaf, mp, ctx->mc_env->me_psize); - mp = leaf; - node = page_node(mp, i); - } - const pgr_t lp = - page_get_large(mc, node_largedata_pgno(node), mp->mp_txnid); - if (unlikely((rc = lp.err) != MDBX_SUCCESS)) - goto done; - const size_t datasize = node_ds(node); - const pgno_t npages = number_of_ovpages(ctx->mc_env, datasize); - poke_pgno(node_data(node), ctx->mc_next_pgno); - rc = compacting_put_page(ctx, lp.page, PAGEHDRSZ + datasize, 0, - npages); - if (unlikely(rc != MDBX_SUCCESS)) - goto done; - } else if (node_flags(node) & F_SUBDATA) { - if (!MDBX_DISABLE_VALIDATION && - unlikely(node_ds(node) != sizeof(MDBX_db))) { - rc = MDBX_CORRUPTED; - goto done; - } + pgno_t shrink = 0; + if (flags & txn_shrink_allowed) { + const size_t prev_discarded_pgno = atomic_load32(&env->lck->discarded_tail, mo_Relaxed); + if (prev_discarded_pgno < pending->geometry.first_unallocated) + env->lck->discarded_tail.weak = pending->geometry.first_unallocated; + else if (prev_discarded_pgno >= pending->geometry.first_unallocated + env->madv_threshold) { + /* LY: check conditions to discard unused pages */ + const pgno_t largest_pgno = + mvcc_snapshot_largest(env, (head.ptr_c->geometry.first_unallocated > pending->geometry.first_unallocated) + ? head.ptr_c->geometry.first_unallocated + : pending->geometry.first_unallocated); + eASSERT(env, largest_pgno >= NUM_METAS); - /* Need writable leaf */ - if (mp != leaf) { - mc->mc_pg[mc->mc_top] = leaf; - page_copy(leaf, mp, ctx->mc_env->me_psize); - mp = leaf; - node = page_node(mp, i); - } +#if defined(ENABLE_MEMCHECK) || defined(__SANITIZE_ADDRESS__) + const pgno_t edge = env->poison_edge; + if (edge > largest_pgno) { + env->poison_edge = largest_pgno; + VALGRIND_MAKE_MEM_NOACCESS(ptr_disp(env->dxb_mmap.base, pgno2bytes(env, largest_pgno)), + pgno2bytes(env, edge - largest_pgno)); + MDBX_ASAN_POISON_MEMORY_REGION(ptr_disp(env->dxb_mmap.base, pgno2bytes(env, largest_pgno)), + pgno2bytes(env, edge - largest_pgno)); + } +#endif /* ENABLE_MEMCHECK || __SANITIZE_ADDRESS__ */ - MDBX_db *nested = nullptr; - if (node_flags(node) & F_DUPDATA) { - rc = cursor_xinit1(mc, node, mp); - if (likely(rc == MDBX_SUCCESS)) { - nested = &mc->mc_xcursor->mx_db; - rc = compacting_walk_tree(ctx, &mc->mc_xcursor->mx_cursor, - &nested->md_root, mp->mp_txnid); - } +#if defined(MADV_DONTNEED) || defined(POSIX_MADV_DONTNEED) + const size_t discard_edge_pgno = pgno_align2os_pgno(env, largest_pgno); + if (prev_discarded_pgno >= discard_edge_pgno + env->madv_threshold) { + const size_t prev_discarded_bytes = pgno_align2os_bytes(env, prev_discarded_pgno); + const size_t discard_edge_bytes = pgno2bytes(env, discard_edge_pgno); + /* из-за выравнивания prev_discarded_bytes и discard_edge_bytes + * могут быть равны */ + if (prev_discarded_bytes > discard_edge_bytes) { + NOTICE("shrink-MADV_%s %zu..%zu", "DONTNEED", discard_edge_pgno, prev_discarded_pgno); + munlock_after(env, discard_edge_pgno, bytes_align2os_bytes(env, env->dxb_mmap.current)); + const uint32_t munlocks_before = atomic_load32(&env->lck->mlcnt[1], mo_Relaxed); +#if defined(MADV_DONTNEED) + int advise = MADV_DONTNEED; +#if defined(MADV_FREE) && 0 /* MADV_FREE works for only anonymous vma at the moment */ + if ((env->flags & MDBX_WRITEMAP) && global.linux_kernel_version > 0x04050000) + advise = MADV_FREE; +#endif /* MADV_FREE */ + int err = madvise(ptr_disp(env->dxb_mmap.base, discard_edge_bytes), prev_discarded_bytes - discard_edge_bytes, + advise) + ? ignore_enosys_and_eagain(errno) + : MDBX_SUCCESS; +#else + int err = ignore_enosys(posix_madvise(ptr_disp(env->dxb_mmap.base, discard_edge_bytes), + prev_discarded_bytes - discard_edge_bytes, POSIX_MADV_DONTNEED)); +#endif + if (unlikely(MDBX_IS_ERROR(err))) { + const uint32_t mlocks_after = atomic_load32(&env->lck->mlcnt[0], mo_Relaxed); + if (err == MDBX_EINVAL) { + const int severity = (mlocks_after - munlocks_before) ? MDBX_LOG_NOTICE : MDBX_LOG_WARN; + if (LOG_ENABLED(severity)) + debug_log(severity, __func__, __LINE__, + "%s-madvise: ignore EINVAL (%d) since some pages maybe " + "locked (%u/%u mlcnt-processes)", + "shrink", err, mlocks_after, munlocks_before); } else { - cASSERT(mc, (mc->mc_flags & C_SUB) == 0 && mc->mc_xcursor == 0); - MDBX_cursor_couple *couple = - container_of(mc, MDBX_cursor_couple, outer); - cASSERT(mc, - couple->inner.mx_cursor.mc_signature == ~MDBX_MC_LIVE && - !couple->inner.mx_cursor.mc_flags && - !couple->inner.mx_cursor.mc_db && - !couple->inner.mx_cursor.mc_dbx); - nested = &couple->inner.mx_db; - memcpy(nested, node_data(node), sizeof(MDBX_db)); - rc = compacting_walk_sdb(ctx, nested); + ERROR("%s-madvise(%s, %zu, +%zu), %u/%u mlcnt-processes, err %d", "shrink", "DONTNEED", + discard_edge_bytes, prev_discarded_bytes - discard_edge_bytes, mlocks_after, munlocks_before, err); + return err; } - if (unlikely(rc != MDBX_SUCCESS)) - goto done; - memcpy(node_data(node), nested, sizeof(MDBX_db)); - } + } else + env->lck->discarded_tail.weak = discard_edge_pgno; } } - } else { - mc->mc_ki[mc->mc_top]++; - if (mc->mc_ki[mc->mc_top] < n) { - while (1) { - const MDBX_node *node = page_node(mp, mc->mc_ki[mc->mc_top]); - rc = page_get(mc, node_pgno(node), &mp, mp->mp_txnid); - if (unlikely(rc != MDBX_SUCCESS)) - goto done; - mc->mc_top++; - mc->mc_snum++; - mc->mc_ki[mc->mc_top] = 0; - if (!IS_BRANCH(mp)) { - mc->mc_pg[mc->mc_top] = mp; - break; +#endif /* MADV_DONTNEED || POSIX_MADV_DONTNEED */ + + /* LY: check conditions to shrink datafile */ + const pgno_t backlog_gap = 3 + pending->trees.gc.height * 3; + pgno_t shrink_step = 0; + if (pending->geometry.shrink_pv && pending->geometry.now - pending->geometry.first_unallocated > + (shrink_step = pv2pages(pending->geometry.shrink_pv)) + backlog_gap) { + if (pending->geometry.now > largest_pgno && pending->geometry.now - largest_pgno > shrink_step + backlog_gap) { + const pgno_t aligner = + pending->geometry.grow_pv ? /* grow_step */ pv2pages(pending->geometry.grow_pv) : shrink_step; + const pgno_t with_backlog_gap = largest_pgno + backlog_gap; + const pgno_t aligned = + pgno_align2os_pgno(env, (size_t)with_backlog_gap + aligner - with_backlog_gap % aligner); + const pgno_t bottom = (aligned > pending->geometry.lower) ? aligned : pending->geometry.lower; + if (pending->geometry.now > bottom) { + if (TROIKA_HAVE_STEADY(troika)) + /* force steady, but only if steady-checkpoint is present */ + flags &= MDBX_WRITEMAP | txn_shrink_allowed; + shrink = pending->geometry.now - bottom; + pending->geometry.now = bottom; + if (unlikely(head.txnid == pending->unsafe_txnid)) { + const txnid_t txnid = safe64_txnid_next(pending->unsafe_txnid); + NOTICE("force-forward pending-txn %" PRIaTXN " -> %" PRIaTXN, pending->unsafe_txnid, txnid); + ENSURE(env, !env->basal_txn || !env->txn); + if (unlikely(txnid > MAX_TXNID)) { + rc = MDBX_TXN_FULL; + ERROR("txnid overflow, raise %d", rc); + goto fail; + } + meta_set_txnid(env, pending, txnid); + eASSERT(env, coherency_check_meta(env, pending, true)); + } } - /* Whenever we advance to a sibling branch page, - * we must proceed all the way down to its first leaf. */ - page_copy(mc->mc_pg[mc->mc_top], mp, ctx->mc_env->me_psize); } - continue; } } + } - const pgno_t pgno = ctx->mc_next_pgno; - if (likely(!IS_LEAF2(mp))) { - rc = compacting_put_page( - ctx, mp, PAGEHDRSZ + mp->mp_lower, - ctx->mc_env->me_psize - (PAGEHDRSZ + mp->mp_upper), 1); + /* LY: step#1 - sync previously written/updated data-pages */ + rc = MDBX_RESULT_FALSE /* carry steady */; + if (atomic_load64(&env->lck->unsynced_pages, mo_Relaxed)) { + eASSERT(env, ((flags ^ env->flags) & MDBX_WRITEMAP) == 0); + enum osal_syncmode_bits mode_bits = MDBX_SYNC_NONE; + unsigned sync_op = 0; + if ((flags & MDBX_SAFE_NOSYNC) == 0) { + sync_op = 1; + mode_bits = MDBX_SYNC_DATA; + if (pending->geometry.first_unallocated > meta_prefer_steady(env, troika).ptr_c->geometry.now) + mode_bits |= MDBX_SYNC_SIZE; + if (flags & MDBX_NOMETASYNC) + mode_bits |= MDBX_SYNC_IODQ; + } else if (unlikely(env->incore)) + goto skip_incore_sync; + if (flags & MDBX_WRITEMAP) { +#if MDBX_ENABLE_PGOP_STAT + env->lck->pgops.msync.weak += sync_op; +#else + (void)sync_op; +#endif /* MDBX_ENABLE_PGOP_STAT */ + rc = osal_msync(&env->dxb_mmap, 0, pgno_align2os_bytes(env, pending->geometry.first_unallocated), mode_bits); } else { - rc = compacting_put_page( - ctx, mp, PAGEHDRSZ + page_numkeys(mp) * mp->mp_leaf2_ksize, 0, 1); +#if MDBX_ENABLE_PGOP_STAT + env->lck->pgops.fsync.weak += sync_op; +#else + (void)sync_op; +#endif /* MDBX_ENABLE_PGOP_STAT */ + rc = osal_fsync(env->lazy_fd, mode_bits); } if (unlikely(rc != MDBX_SUCCESS)) - goto done; - - if (mc->mc_top) { - /* Update parent if there is one */ - node_set_pgno( - page_node(mc->mc_pg[mc->mc_top - 1], mc->mc_ki[mc->mc_top - 1]), - pgno); - cursor_pop(mc); - } else { - /* Otherwise we're done */ - *root = pgno; - break; - } + goto fail; + rc = (flags & MDBX_SAFE_NOSYNC) ? MDBX_RESULT_TRUE /* carry non-steady */ + : MDBX_RESULT_FALSE /* carry steady */; } -done: - osal_free(buf); - return rc; -} - -__cold static int compacting_walk_sdb(mdbx_compacting_ctx *ctx, MDBX_db *sdb) { - if (unlikely(sdb->md_root == P_INVALID)) - return MDBX_SUCCESS; /* empty db */ - - MDBX_cursor_couple couple; - memset(&couple, 0, sizeof(couple)); - couple.inner.mx_cursor.mc_signature = ~MDBX_MC_LIVE; - MDBX_dbx dbx = {.md_klen_min = INT_MAX}; - uint8_t dbistate = DBI_VALID | DBI_AUDITED; - int rc = couple_init(&couple, ~0u, ctx->mc_txn, sdb, &dbx, &dbistate); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - - couple.outer.mc_checking |= CC_SKIPORD | CC_PAGECHECK; - couple.inner.mx_cursor.mc_checking |= CC_SKIPORD | CC_PAGECHECK; - if (!sdb->md_mod_txnid) - sdb->md_mod_txnid = ctx->mc_txn->mt_txnid; - return compacting_walk_tree(ctx, &couple.outer, &sdb->md_root, - sdb->md_mod_txnid); -} + eASSERT(env, coherency_check_meta(env, pending, true)); -__cold static void compacting_fixup_meta(MDBX_env *env, MDBX_meta *meta) { - eASSERT(env, meta->mm_dbs[FREE_DBI].md_mod_txnid || - meta->mm_dbs[FREE_DBI].md_root == P_INVALID); - eASSERT(env, meta->mm_dbs[MAIN_DBI].md_mod_txnid || - meta->mm_dbs[MAIN_DBI].md_root == P_INVALID); + /* Steady or Weak */ + if (rc == MDBX_RESULT_FALSE /* carry steady */) { + meta_sign_as_steady(pending); + atomic_store64(&env->lck->eoos_timestamp, 0, mo_Relaxed); + atomic_store64(&env->lck->unsynced_pages, 0, mo_Relaxed); + } else { + assert(rc == MDBX_RESULT_TRUE /* carry non-steady */); + skip_incore_sync: + eASSERT(env, env->lck->unsynced_pages.weak > 0); + /* Может быть нулевым если unsynced_pages > 0 в результате спиллинга. + * eASSERT(env, env->lck->eoos_timestamp.weak != 0); */ + unaligned_poke_u64(4, pending->sign, DATASIGN_WEAK); + } - /* Calculate filesize taking in account shrink/growing thresholds */ - if (meta->mm_geo.next != meta->mm_geo.now) { - meta->mm_geo.now = meta->mm_geo.next; - const size_t aligner = pv2pages( - meta->mm_geo.grow_pv ? meta->mm_geo.grow_pv : meta->mm_geo.shrink_pv); - if (aligner) { - const pgno_t aligned = pgno_align2os_pgno( - env, meta->mm_geo.next + aligner - meta->mm_geo.next % aligner); - meta->mm_geo.now = aligned; + const bool legal4overwrite = head.txnid == pending->unsafe_txnid && + !memcmp(&head.ptr_c->trees, &pending->trees, sizeof(pending->trees)) && + !memcmp(&head.ptr_c->canary, &pending->canary, sizeof(pending->canary)) && + !memcmp(&head.ptr_c->geometry, &pending->geometry, sizeof(pending->geometry)); + meta_t *target = nullptr; + if (head.txnid == pending->unsafe_txnid) { + ENSURE(env, legal4overwrite); + if (!head.is_steady && meta_is_steady(pending)) + target = (meta_t *)head.ptr_c; + else { + NOTICE("skip update meta%" PRIaPGNO " for txn#%" PRIaTXN ", since it is already steady", + data_page(head.ptr_c)->pgno, head.txnid); + return MDBX_SUCCESS; } + } else { + const unsigned troika_tail = troika->tail_and_flags & 3; + ENSURE(env, troika_tail < NUM_METAS && troika_tail != troika->recent && troika_tail != troika->prefer_steady); + target = (meta_t *)meta_tail(env, troika).ptr_c; } - if (meta->mm_geo.now < meta->mm_geo.lower) - meta->mm_geo.now = meta->mm_geo.lower; - if (meta->mm_geo.now > meta->mm_geo.upper) - meta->mm_geo.now = meta->mm_geo.upper; + /* LY: step#2 - update meta-page. */ + DEBUG("writing meta%" PRIaPGNO " = root %" PRIaPGNO "/%" PRIaPGNO ", geo %" PRIaPGNO "/%" PRIaPGNO "-%" PRIaPGNO + "/%" PRIaPGNO " +%u -%u, txn_id %" PRIaTXN ", %s", + data_page(target)->pgno, pending->trees.main.root, pending->trees.gc.root, pending->geometry.lower, + pending->geometry.first_unallocated, pending->geometry.now, pending->geometry.upper, + pv2pages(pending->geometry.grow_pv), pv2pages(pending->geometry.shrink_pv), pending->unsafe_txnid, + durable_caption(pending)); - /* Update signature */ - assert(meta->mm_geo.now >= meta->mm_geo.next); - unaligned_poke_u64(4, meta->mm_sign, meta_sign(meta)); -} + DEBUG("meta0: %s, %s, txn_id %" PRIaTXN ", root %" PRIaPGNO "/%" PRIaPGNO, + (meta0 == head.ptr_c) ? "head" + : (meta0 == target) ? "tail" + : "stay", + durable_caption(meta0), constmeta_txnid(meta0), meta0->trees.main.root, meta0->trees.gc.root); + DEBUG("meta1: %s, %s, txn_id %" PRIaTXN ", root %" PRIaPGNO "/%" PRIaPGNO, + (meta1 == head.ptr_c) ? "head" + : (meta1 == target) ? "tail" + : "stay", + durable_caption(meta1), constmeta_txnid(meta1), meta1->trees.main.root, meta1->trees.gc.root); + DEBUG("meta2: %s, %s, txn_id %" PRIaTXN ", root %" PRIaPGNO "/%" PRIaPGNO, + (meta2 == head.ptr_c) ? "head" + : (meta2 == target) ? "tail" + : "stay", + durable_caption(meta2), constmeta_txnid(meta2), meta2->trees.main.root, meta2->trees.gc.root); -/* Make resizable */ -__cold static void meta_make_sizeable(MDBX_meta *meta) { - meta->mm_geo.lower = MIN_PAGENO; - if (meta->mm_geo.grow_pv == 0) { - const pgno_t step = 1 + (meta->mm_geo.upper - meta->mm_geo.lower) / 42; - meta->mm_geo.grow_pv = pages2pv(step); - } - if (meta->mm_geo.shrink_pv == 0) { - const pgno_t step = pv2pages(meta->mm_geo.grow_pv) << 1; - meta->mm_geo.shrink_pv = pages2pv(step); - } -} - -/* Copy environment with compaction. */ -__cold static int env_compact(MDBX_env *env, MDBX_txn *read_txn, - mdbx_filehandle_t fd, uint8_t *buffer, - const bool dest_is_pipe, - const MDBX_copy_flags_t flags) { - const size_t meta_bytes = pgno2bytes(env, NUM_METAS); - uint8_t *const data_buffer = - buffer + ceil_powerof2(meta_bytes, env->me_os_psize); - MDBX_meta *const meta = init_metas(env, buffer); - meta_set_txnid(env, meta, read_txn->mt_txnid); + eASSERT(env, pending->unsafe_txnid != constmeta_txnid(meta0) || (meta_is_steady(pending) && !meta_is_steady(meta0))); + eASSERT(env, pending->unsafe_txnid != constmeta_txnid(meta1) || (meta_is_steady(pending) && !meta_is_steady(meta1))); + eASSERT(env, pending->unsafe_txnid != constmeta_txnid(meta2) || (meta_is_steady(pending) && !meta_is_steady(meta2))); - if (flags & MDBX_CP_FORCE_DYNAMIC_SIZE) - meta_make_sizeable(meta); + eASSERT(env, ((env->flags ^ flags) & MDBX_WRITEMAP) == 0); + ENSURE(env, target == head.ptr_c || constmeta_txnid(target) < pending->unsafe_txnid); + if (flags & MDBX_WRITEMAP) { + jitter4testing(true); + if (likely(target != head.ptr_c)) { + /* LY: 'invalidate' the meta. */ + meta_update_begin(env, target, pending->unsafe_txnid); + unaligned_poke_u64(4, target->sign, DATASIGN_WEAK); +#ifndef NDEBUG + /* debug: provoke failure to catch a violators, but don't touch pagesize + * to allow readers catch actual pagesize. */ + void *provoke_begin = &target->trees.gc.root; + void *provoke_end = &target->sign; + memset(provoke_begin, 0xCC, ptr_dist(provoke_end, provoke_begin)); + jitter4testing(false); +#endif - /* copy canary sequences if present */ - if (read_txn->mt_canary.v) { - meta->mm_canary = read_txn->mt_canary; - meta->mm_canary.v = constmeta_txnid(meta); - } + /* LY: update info */ + target->geometry = pending->geometry; + target->trees.gc = pending->trees.gc; + target->trees.main = pending->trees.main; + eASSERT(env, target->trees.gc.flags == MDBX_INTEGERKEY); + eASSERT(env, check_table_flags(target->trees.main.flags)); + target->canary = pending->canary; + memcpy(target->pages_retired, pending->pages_retired, 8); + jitter4testing(true); - if (read_txn->mt_dbs[MAIN_DBI].md_root == P_INVALID) { - /* When the DB is empty, handle it specially to - * fix any breakage like page leaks from ITS#8174. */ - meta->mm_dbs[MAIN_DBI].md_flags = read_txn->mt_dbs[MAIN_DBI].md_flags; - compacting_fixup_meta(env, meta); - if (dest_is_pipe) { - int rc = osal_write(fd, buffer, meta_bytes); + /* LY: 'commit' the meta */ + meta_update_end(env, target, unaligned_peek_u64(4, pending->txnid_b)); + jitter4testing(true); + eASSERT(env, coherency_check_meta(env, target, true)); + } else { + /* dangerous case (target == head), only sign could + * me updated, check assertions once again */ + eASSERT(env, legal4overwrite && !head.is_steady && meta_is_steady(pending)); + } + memcpy(target->sign, pending->sign, 8); + osal_flush_incoherent_cpu_writeback(); + jitter4testing(true); + if (!env->incore) { + if (!MDBX_AVOID_MSYNC) { + /* sync meta-pages */ +#if MDBX_ENABLE_PGOP_STAT + env->lck->pgops.msync.weak += 1; +#endif /* MDBX_ENABLE_PGOP_STAT */ + rc = osal_msync(&env->dxb_mmap, 0, pgno_align2os_bytes(env, NUM_METAS), + (flags & MDBX_NOMETASYNC) ? MDBX_SYNC_NONE : MDBX_SYNC_DATA | MDBX_SYNC_IODQ); + } else { +#if MDBX_ENABLE_PGOP_STAT + env->lck->pgops.wops.weak += 1; +#endif /* MDBX_ENABLE_PGOP_STAT */ + const page_t *page = data_page(target); + rc = osal_pwrite(env->fd4meta, page, env->ps, ptr_dist(page, env->dxb_mmap.base)); + if (likely(rc == MDBX_SUCCESS)) { + osal_flush_incoherent_mmap(target, sizeof(meta_t), globals.sys_pagesize); + if ((flags & MDBX_NOMETASYNC) == 0 && env->fd4meta == env->lazy_fd) { +#if MDBX_ENABLE_PGOP_STAT + env->lck->pgops.fsync.weak += 1; +#endif /* MDBX_ENABLE_PGOP_STAT */ + rc = osal_fsync(env->lazy_fd, MDBX_SYNC_DATA | MDBX_SYNC_IODQ); + } + } + } if (unlikely(rc != MDBX_SUCCESS)) - return rc; + goto fail; } } else { - /* Count free pages + GC pages. */ - MDBX_cursor_couple couple; - int rc = cursor_init(&couple.outer, read_txn, FREE_DBI); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - pgno_t gc = read_txn->mt_dbs[FREE_DBI].md_branch_pages + - read_txn->mt_dbs[FREE_DBI].md_leaf_pages + - read_txn->mt_dbs[FREE_DBI].md_overflow_pages; - MDBX_val key, data; - while ((rc = cursor_get(&couple.outer, &key, &data, MDBX_NEXT)) == - MDBX_SUCCESS) { - const MDBX_PNL pnl = data.iov_base; - if (unlikely(data.iov_len % sizeof(pgno_t) || - data.iov_len < MDBX_PNL_SIZEOF(pnl) || - !(pnl_check(pnl, read_txn->mt_next_pgno)))) - return MDBX_CORRUPTED; - gc += MDBX_PNL_GETSIZE(pnl); +#if MDBX_ENABLE_PGOP_STAT + env->lck->pgops.wops.weak += 1; +#endif /* MDBX_ENABLE_PGOP_STAT */ + const meta_t undo_meta = *target; + eASSERT(env, pending->trees.gc.flags == MDBX_INTEGERKEY); + eASSERT(env, check_table_flags(pending->trees.main.flags)); + rc = osal_pwrite(env->fd4meta, pending, sizeof(meta_t), ptr_dist(target, env->dxb_mmap.base)); + if (unlikely(rc != MDBX_SUCCESS)) { + undo: + DEBUG("%s", "write failed, disk error?"); + /* On a failure, the pagecache still contains the new data. + * Try write some old data back, to prevent it from being used. */ + osal_pwrite(env->fd4meta, &undo_meta, sizeof(meta_t), ptr_dist(target, env->dxb_mmap.base)); + goto fail; } - if (unlikely(rc != MDBX_NOTFOUND)) - return rc; - - /* Substract GC-pages from mt_next_pgno to find the new mt_next_pgno. */ - meta->mm_geo.next = read_txn->mt_next_pgno - gc; - /* Set with current main DB */ - meta->mm_dbs[MAIN_DBI] = read_txn->mt_dbs[MAIN_DBI]; - - mdbx_compacting_ctx ctx; - memset(&ctx, 0, sizeof(ctx)); - rc = osal_condpair_init(&ctx.mc_condpair); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; + osal_flush_incoherent_mmap(target, sizeof(meta_t), globals.sys_pagesize); + /* sync meta-pages */ + if ((flags & MDBX_NOMETASYNC) == 0 && env->fd4meta == env->lazy_fd && !env->incore) { +#if MDBX_ENABLE_PGOP_STAT + env->lck->pgops.fsync.weak += 1; +#endif /* MDBX_ENABLE_PGOP_STAT */ + rc = osal_fsync(env->lazy_fd, MDBX_SYNC_DATA | MDBX_SYNC_IODQ); + if (rc != MDBX_SUCCESS) + goto undo; + } + } - memset(data_buffer, 0, 2 * (size_t)MDBX_ENVCOPY_WRITEBUF); - ctx.mc_wbuf[0] = data_buffer; - ctx.mc_wbuf[1] = data_buffer + (size_t)MDBX_ENVCOPY_WRITEBUF; - ctx.mc_next_pgno = NUM_METAS; - ctx.mc_env = env; - ctx.mc_fd = fd; - ctx.mc_txn = read_txn; + uint64_t timestamp = 0; + while ("workaround for https://libmdbx.dqdkfa.ru/dead-github/issues/269") { + rc = coherency_check_written(env, pending->unsafe_txnid, target, + bytes2pgno(env, ptr_dist(target, env->dxb_mmap.base)), ×tamp); + if (likely(rc == MDBX_SUCCESS)) + break; + if (unlikely(rc != MDBX_RESULT_TRUE)) + goto fail; + } - osal_thread_t thread; - int thread_err = osal_thread_create(&thread, compacting_write_thread, &ctx); - if (likely(thread_err == MDBX_SUCCESS)) { - if (dest_is_pipe) { - if (!meta->mm_dbs[MAIN_DBI].md_mod_txnid) - meta->mm_dbs[MAIN_DBI].md_mod_txnid = read_txn->mt_txnid; - compacting_fixup_meta(env, meta); - rc = osal_write(fd, buffer, meta_bytes); - } - if (likely(rc == MDBX_SUCCESS)) - rc = compacting_walk_sdb(&ctx, &meta->mm_dbs[MAIN_DBI]); - if (ctx.mc_wlen[ctx.mc_head & 1]) - /* toggle to flush non-empty buffers */ - compacting_toggle_write_buffers(&ctx); + const uint32_t sync_txnid_dist = ((flags & MDBX_NOMETASYNC) == 0) ? 0 + : ((flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC) ? MDBX_NOMETASYNC_LAZY_FD + : MDBX_NOMETASYNC_LAZY_WRITEMAP; + env->lck->meta_sync_txnid.weak = pending->txnid_a[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__].weak - sync_txnid_dist; - if (likely(rc == MDBX_SUCCESS) && - unlikely(meta->mm_geo.next != ctx.mc_next_pgno)) { - if (ctx.mc_next_pgno > meta->mm_geo.next) { - ERROR("the source DB %s: post-compactification used pages %" PRIaPGNO - " %c expected %" PRIaPGNO, - "has double-used pages or other corruption", ctx.mc_next_pgno, - '>', meta->mm_geo.next); - rc = MDBX_CORRUPTED; /* corrupted DB */ - } - if (ctx.mc_next_pgno < meta->mm_geo.next) { - WARNING( - "the source DB %s: post-compactification used pages %" PRIaPGNO - " %c expected %" PRIaPGNO, - "has page leak(s)", ctx.mc_next_pgno, '<', meta->mm_geo.next); - if (dest_is_pipe) - /* the root within already written meta-pages is wrong */ - rc = MDBX_CORRUPTED; - } - /* fixup meta */ - meta->mm_geo.next = ctx.mc_next_pgno; - } + *troika = meta_tap(env); + for (MDBX_txn *txn = env->basal_txn; txn; txn = txn->nested) + if (troika != &txn->tw.troika) + txn->tw.troika = *troika; - /* toggle with empty buffers to exit thread's loop */ - eASSERT(env, (ctx.mc_wlen[ctx.mc_head & 1]) == 0); - compacting_toggle_write_buffers(&ctx); - thread_err = osal_thread_join(thread); - eASSERT(env, (ctx.mc_tail == ctx.mc_head && - ctx.mc_wlen[ctx.mc_head & 1] == 0) || - ctx.mc_error); - osal_condpair_destroy(&ctx.mc_condpair); - } - if (unlikely(thread_err != MDBX_SUCCESS)) - return thread_err; - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - if (unlikely(ctx.mc_error != MDBX_SUCCESS)) - return ctx.mc_error; - if (!dest_is_pipe) - compacting_fixup_meta(env, meta); + /* LY: shrink datafile if needed */ + if (unlikely(shrink)) { + VERBOSE("shrink to %" PRIaPGNO " pages (-%" PRIaPGNO ")", pending->geometry.now, shrink); + rc = dxb_resize(env, pending->geometry.first_unallocated, pending->geometry.now, pending->geometry.upper, + impilict_shrink); + if (rc != MDBX_SUCCESS && rc != MDBX_EPERM) + goto fail; + eASSERT(env, coherency_check_meta(env, target, true)); } - /* Extend file if required */ - if (meta->mm_geo.now != meta->mm_geo.next) { - const size_t whole_size = pgno2bytes(env, meta->mm_geo.now); - if (!dest_is_pipe) - return osal_ftruncate(fd, whole_size); + lck_t *const lck = env->lck_mmap.lck; + if (likely(lck)) + /* toggle oldest refresh */ + atomic_store32(&lck->rdt_refresh_flag, false, mo_Relaxed); - const size_t used_size = pgno2bytes(env, meta->mm_geo.next); - memset(data_buffer, 0, (size_t)MDBX_ENVCOPY_WRITEBUF); - for (size_t offset = used_size; offset < whole_size;) { - const size_t chunk = ((size_t)MDBX_ENVCOPY_WRITEBUF < whole_size - offset) - ? (size_t)MDBX_ENVCOPY_WRITEBUF - : whole_size - offset; - int rc = osal_write(fd, data_buffer, chunk); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - offset += chunk; - } - } return MDBX_SUCCESS; -} -/* Copy environment as-is. */ -__cold static int env_copy_asis(MDBX_env *env, MDBX_txn *read_txn, - mdbx_filehandle_t fd, uint8_t *buffer, - const bool dest_is_pipe, - const MDBX_copy_flags_t flags) { - /* We must start the actual read txn after blocking writers */ - int rc = txn_end(read_txn, MDBX_END_RESET_TMP); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - - /* Temporarily block writers until we snapshot the meta pages */ - rc = mdbx_txn_lock(env, false); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; +fail: + env->flags |= ENV_FATAL_ERROR; + return rc; +} +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 - rc = txn_renew(read_txn, MDBX_TXN_RDONLY); - if (unlikely(rc != MDBX_SUCCESS)) { - mdbx_txn_unlock(env); - return rc; +MDBX_txn *env_owned_wrtxn(const MDBX_env *env) { + if (likely(env->basal_txn)) { + const bool is_owned = (env->flags & MDBX_NOSTICKYTHREADS) ? (env->basal_txn->owner != 0) + : (env->basal_txn->owner == osal_thread_self()); + if (is_owned) + return env->txn ? env->txn : env->basal_txn; } + return nullptr; +} - jitter4testing(false); - const size_t meta_bytes = pgno2bytes(env, NUM_METAS); - const meta_troika_t troika = meta_tap(env); - /* Make a snapshot of meta-pages, - * but writing ones after the data was flushed */ - memcpy(buffer, env->me_map, meta_bytes); - MDBX_meta *const headcopy = /* LY: get pointer to the snapshot copy */ - ptr_disp(buffer, ptr_dist(meta_recent(env, &troika).ptr_c, env->me_map)); - mdbx_txn_unlock(env); +int env_page_auxbuffer(MDBX_env *env) { + const int err = env->page_auxbuf + ? MDBX_SUCCESS + : osal_memalign_alloc(globals.sys_pagesize, env->ps * (size_t)NUM_METAS, &env->page_auxbuf); + if (likely(err == MDBX_SUCCESS)) { + memset(env->page_auxbuf, -1, env->ps * (size_t)2); + memset(ptr_disp(env->page_auxbuf, env->ps * (size_t)2), 0, env->ps); + } + return err; +} - if (flags & MDBX_CP_FORCE_DYNAMIC_SIZE) - meta_make_sizeable(headcopy); - /* Update signature to steady */ - unaligned_poke_u64(4, headcopy->mm_sign, meta_sign(headcopy)); +__cold unsigned env_setup_pagesize(MDBX_env *env, const size_t pagesize) { + STATIC_ASSERT(PTRDIFF_MAX > MAX_MAPSIZE); + STATIC_ASSERT(MDBX_MIN_PAGESIZE > sizeof(page_t) + sizeof(meta_t)); + ENSURE(env, is_powerof2(pagesize)); + ENSURE(env, pagesize >= MDBX_MIN_PAGESIZE); + ENSURE(env, pagesize <= MDBX_MAX_PAGESIZE); + ENSURE(env, !env->page_auxbuf && env->ps != pagesize); + env->ps = (unsigned)pagesize; - /* Copy the data */ - const size_t whole_size = pgno_align2os_bytes(env, read_txn->mt_end_pgno); - const size_t used_size = pgno2bytes(env, read_txn->mt_next_pgno); - jitter4testing(false); + STATIC_ASSERT(MAX_GC1OVPAGE(MDBX_MIN_PAGESIZE) > 4); + STATIC_ASSERT(MAX_GC1OVPAGE(MDBX_MAX_PAGESIZE) < PAGELIST_LIMIT); + const intptr_t maxgc_ov1page = (pagesize - PAGEHDRSZ) / sizeof(pgno_t) - 1; + ENSURE(env, maxgc_ov1page > 42 && maxgc_ov1page < (intptr_t)PAGELIST_LIMIT / 4); + env->maxgc_large1page = (unsigned)maxgc_ov1page; + env->maxgc_per_branch = (unsigned)((pagesize - PAGEHDRSZ) / (sizeof(indx_t) + sizeof(node_t) + sizeof(txnid_t))); + + STATIC_ASSERT(LEAF_NODE_MAX(MDBX_MIN_PAGESIZE) > sizeof(tree_t) + NODESIZE + 42); + STATIC_ASSERT(LEAF_NODE_MAX(MDBX_MAX_PAGESIZE) < UINT16_MAX); + STATIC_ASSERT(LEAF_NODE_MAX(MDBX_MIN_PAGESIZE) >= BRANCH_NODE_MAX(MDBX_MIN_PAGESIZE)); + STATIC_ASSERT(BRANCH_NODE_MAX(MDBX_MAX_PAGESIZE) > NODESIZE + 42); + STATIC_ASSERT(BRANCH_NODE_MAX(MDBX_MAX_PAGESIZE) < UINT16_MAX); + const intptr_t branch_nodemax = BRANCH_NODE_MAX(pagesize); + const intptr_t leaf_nodemax = LEAF_NODE_MAX(pagesize); + ENSURE(env, branch_nodemax > (intptr_t)(NODESIZE + 42) && branch_nodemax % 2 == 0 && + leaf_nodemax > (intptr_t)(sizeof(tree_t) + NODESIZE + 42) && leaf_nodemax >= branch_nodemax && + leaf_nodemax < (int)UINT16_MAX && leaf_nodemax % 2 == 0); + env->leaf_nodemax = (uint16_t)leaf_nodemax; + env->branch_nodemax = (uint16_t)branch_nodemax; + env->ps2ln = (uint8_t)log2n_powerof2(pagesize); + eASSERT(env, pgno2bytes(env, 1) == pagesize); + eASSERT(env, bytes2pgno(env, pagesize + pagesize) == 2); + recalculate_merge_thresholds(env); + recalculate_subpage_thresholds(env); + env_options_adjust_dp_limit(env); + return env->ps; +} - if (dest_is_pipe) - rc = osal_write(fd, buffer, meta_bytes); +__cold int env_sync(MDBX_env *env, bool force, bool nonblock) { + if (unlikely(env->flags & MDBX_RDONLY)) + return MDBX_EACCESS; - uint8_t *const data_buffer = - buffer + ceil_powerof2(meta_bytes, env->me_os_psize); -#if MDBX_USE_COPYFILERANGE - static bool copyfilerange_unavailable; - bool not_the_same_filesystem = false; - struct statfs statfs_info; - if (fstatfs(fd, &statfs_info) || - statfs_info.f_type == /* ECRYPTFS_SUPER_MAGIC */ 0xf15f) - /* avoid use copyfilerange_unavailable() to ecryptfs due bugs */ - not_the_same_filesystem = true; -#endif /* MDBX_USE_COPYFILERANGE */ - for (size_t offset = meta_bytes; rc == MDBX_SUCCESS && offset < used_size;) { -#if MDBX_USE_SENDFILE - static bool sendfile_unavailable; - if (dest_is_pipe && likely(!sendfile_unavailable)) { - off_t in_offset = offset; - const ssize_t written = - sendfile(fd, env->me_lazy_fd, &in_offset, used_size - offset); - if (likely(written > 0)) { - offset = in_offset; - continue; - } - rc = MDBX_ENODATA; - if (written == 0 || ignore_enosys(rc = errno) != MDBX_RESULT_TRUE) - break; - sendfile_unavailable = true; - } -#endif /* MDBX_USE_SENDFILE */ + MDBX_txn *const txn_owned = env_owned_wrtxn(env); + bool should_unlock = false; + int rc = MDBX_RESULT_TRUE /* means "nothing to sync" */; -#if MDBX_USE_COPYFILERANGE - if (!dest_is_pipe && !not_the_same_filesystem && - likely(!copyfilerange_unavailable)) { - off_t in_offset = offset, out_offset = offset; - ssize_t bytes_copied = copy_file_range( - env->me_lazy_fd, &in_offset, fd, &out_offset, used_size - offset, 0); - if (likely(bytes_copied > 0)) { - offset = in_offset; - continue; - } - rc = MDBX_ENODATA; - if (bytes_copied == 0) - break; - rc = errno; - if (rc == EXDEV || rc == /* workaround for ecryptfs bug(s), - maybe useful for others FS */ - EINVAL) - not_the_same_filesystem = true; - else if (ignore_enosys(rc) == MDBX_RESULT_TRUE) - copyfilerange_unavailable = true; - else - break; - } -#endif /* MDBX_USE_COPYFILERANGE */ +retry:; + unsigned flags = env->flags & ~(MDBX_NOMETASYNC | txn_shrink_allowed); + if (unlikely((flags & (ENV_FATAL_ERROR | ENV_ACTIVE)) != ENV_ACTIVE)) { + rc = (flags & ENV_FATAL_ERROR) ? MDBX_PANIC : MDBX_EPERM; + goto bailout; + } - /* fallback to portable */ - const size_t chunk = ((size_t)MDBX_ENVCOPY_WRITEBUF < used_size - offset) - ? (size_t)MDBX_ENVCOPY_WRITEBUF - : used_size - offset; - /* copy to avoid EFAULT in case swapped-out */ - memcpy(data_buffer, ptr_disp(env->me_map, offset), chunk); - rc = osal_write(fd, data_buffer, chunk); - offset += chunk; + const troika_t troika = (txn_owned || should_unlock) ? env->basal_txn->tw.troika : meta_tap(env); + const meta_ptr_t head = meta_recent(env, &troika); + const uint64_t unsynced_pages = atomic_load64(&env->lck->unsynced_pages, mo_Relaxed); + if (unsynced_pages == 0) { + const uint32_t synched_meta_txnid_u32 = atomic_load32(&env->lck->meta_sync_txnid, mo_Relaxed); + if (synched_meta_txnid_u32 == (uint32_t)head.txnid && head.is_steady) + goto bailout; } - /* Extend file if required */ - if (likely(rc == MDBX_SUCCESS) && whole_size != used_size) { - if (!dest_is_pipe) - rc = osal_ftruncate(fd, whole_size); - else { - memset(data_buffer, 0, (size_t)MDBX_ENVCOPY_WRITEBUF); - for (size_t offset = used_size; - rc == MDBX_SUCCESS && offset < whole_size;) { - const size_t chunk = - ((size_t)MDBX_ENVCOPY_WRITEBUF < whole_size - offset) - ? (size_t)MDBX_ENVCOPY_WRITEBUF - : whole_size - offset; - rc = osal_write(fd, data_buffer, chunk); - offset += chunk; - } + if (should_unlock && (env->flags & MDBX_WRITEMAP) && + unlikely(head.ptr_c->geometry.first_unallocated > bytes2pgno(env, env->dxb_mmap.current))) { + + if (unlikely(env->stuck_meta >= 0) && troika.recent != (uint8_t)env->stuck_meta) { + NOTICE("skip %s since wagering meta-page (%u) is mispatch the recent " + "meta-page (%u)", + "sync datafile", env->stuck_meta, troika.recent); + rc = MDBX_RESULT_TRUE; + } else { + rc = dxb_resize(env, head.ptr_c->geometry.first_unallocated, head.ptr_c->geometry.now, head.ptr_c->geometry.upper, + implicit_grow); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; } } - return rc; -} - -__cold int mdbx_env_copy2fd(MDBX_env *env, mdbx_filehandle_t fd, - MDBX_copy_flags_t flags) { - int rc = check_env(env, true); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; + const size_t autosync_threshold = atomic_load32(&env->lck->autosync_threshold, mo_Relaxed); + const uint64_t autosync_period = atomic_load64(&env->lck->autosync_period, mo_Relaxed); + uint64_t eoos_timestamp; + if (force || (autosync_threshold && unsynced_pages >= autosync_threshold) || + (autosync_period && (eoos_timestamp = atomic_load64(&env->lck->eoos_timestamp, mo_Relaxed)) && + osal_monotime() - eoos_timestamp >= autosync_period)) + flags &= MDBX_WRITEMAP /* clear flags for full steady sync */; - const int dest_is_pipe = osal_is_pipe(fd); - if (MDBX_IS_ERROR(dest_is_pipe)) - return dest_is_pipe; + if (!txn_owned) { + if (!should_unlock) { +#if MDBX_ENABLE_PGOP_STAT + unsigned wops = 0; +#endif /* MDBX_ENABLE_PGOP_STAT */ - if (!dest_is_pipe) { - rc = osal_fseek(fd, 0); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - } - - const size_t buffer_size = - pgno_align2os_bytes(env, NUM_METAS) + - ceil_powerof2(((flags & MDBX_CP_COMPACT) - ? 2 * (size_t)MDBX_ENVCOPY_WRITEBUF - : (size_t)MDBX_ENVCOPY_WRITEBUF), - env->me_os_psize); + int err; + /* pre-sync to avoid latency for writer */ + if (unsynced_pages > /* FIXME: define threshold */ 42 && (flags & MDBX_SAFE_NOSYNC) == 0) { + eASSERT(env, ((flags ^ env->flags) & MDBX_WRITEMAP) == 0); + if (flags & MDBX_WRITEMAP) { + /* Acquire guard to avoid collision with remap */ +#if defined(_WIN32) || defined(_WIN64) + imports.srwl_AcquireShared(&env->remap_guard); +#else + err = osal_fastmutex_acquire(&env->remap_guard); + if (unlikely(err != MDBX_SUCCESS)) + return err; +#endif + const size_t usedbytes = pgno_align2os_bytes(env, head.ptr_c->geometry.first_unallocated); + err = osal_msync(&env->dxb_mmap, 0, usedbytes, MDBX_SYNC_DATA); +#if defined(_WIN32) || defined(_WIN64) + imports.srwl_ReleaseShared(&env->remap_guard); +#else + int unlock_err = osal_fastmutex_release(&env->remap_guard); + if (unlikely(unlock_err != MDBX_SUCCESS) && err == MDBX_SUCCESS) + err = unlock_err; +#endif + } else + err = osal_fsync(env->lazy_fd, MDBX_SYNC_DATA); - uint8_t *buffer = NULL; - rc = osal_memalign_alloc(env->me_os_psize, buffer_size, (void **)&buffer); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; + if (unlikely(err != MDBX_SUCCESS)) + return err; - MDBX_txn *read_txn = NULL; - /* Do the lock/unlock of the reader mutex before starting the - * write txn. Otherwise other read txns could block writers. */ - rc = mdbx_txn_begin(env, NULL, MDBX_TXN_RDONLY, &read_txn); - if (unlikely(rc != MDBX_SUCCESS)) { - osal_memalign_free(buffer); - return rc; - } +#if MDBX_ENABLE_PGOP_STAT + wops = 1; +#endif /* MDBX_ENABLE_PGOP_STAT */ + /* pre-sync done */ + rc = MDBX_SUCCESS /* means "some data was synced" */; + } - if (!dest_is_pipe) { - /* Firstly write a stub to meta-pages. - * Now we sure to incomplete copy will not be used. */ - memset(buffer, -1, pgno2bytes(env, NUM_METAS)); - rc = osal_write(fd, buffer, pgno2bytes(env, NUM_METAS)); - } + err = lck_txn_lock(env, nonblock); + if (unlikely(err != MDBX_SUCCESS)) + return err; - if (likely(rc == MDBX_SUCCESS)) { - memset(buffer, 0, pgno2bytes(env, NUM_METAS)); - rc = ((flags & MDBX_CP_COMPACT) ? env_compact : env_copy_asis)( - env, read_txn, fd, buffer, dest_is_pipe, flags); + should_unlock = true; +#if MDBX_ENABLE_PGOP_STAT + env->lck->pgops.wops.weak += wops; +#endif /* MDBX_ENABLE_PGOP_STAT */ + env->basal_txn->tw.troika = meta_tap(env); + eASSERT(env, !env->txn && !env->basal_txn->nested); + goto retry; + } + eASSERT(env, head.txnid == recent_committed_txnid(env)); + env->basal_txn->txnid = head.txnid; + txn_snapshot_oldest(env->basal_txn); + flags |= txn_shrink_allowed; } - mdbx_txn_abort(read_txn); - - if (!dest_is_pipe) { - if (likely(rc == MDBX_SUCCESS)) - rc = osal_fsync(fd, MDBX_SYNC_DATA | MDBX_SYNC_SIZE); - /* Write actual meta */ - if (likely(rc == MDBX_SUCCESS)) - rc = osal_pwrite(fd, buffer, pgno2bytes(env, NUM_METAS), 0); + eASSERT(env, txn_owned || should_unlock); + eASSERT(env, !txn_owned || (flags & txn_shrink_allowed) == 0); - if (likely(rc == MDBX_SUCCESS)) - rc = osal_fsync(fd, MDBX_SYNC_DATA | MDBX_SYNC_IODQ); + if (!head.is_steady && unlikely(env->stuck_meta >= 0) && troika.recent != (uint8_t)env->stuck_meta) { + NOTICE("skip %s since wagering meta-page (%u) is mispatch the recent " + "meta-page (%u)", + "sync datafile", env->stuck_meta, troika.recent); + rc = MDBX_RESULT_TRUE; + goto bailout; + } + if (!head.is_steady || ((flags & MDBX_SAFE_NOSYNC) == 0 && unsynced_pages)) { + DEBUG("meta-head %" PRIaPGNO ", %s, sync_pending %" PRIu64, data_page(head.ptr_c)->pgno, + durable_caption(head.ptr_c), unsynced_pages); + meta_t meta = *head.ptr_c; + rc = dxb_sync_locked(env, flags, &meta, &env->basal_txn->tw.troika); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; } - osal_memalign_free(buffer); - return rc; -} + /* LY: sync meta-pages if MDBX_NOMETASYNC enabled + * and someone was not synced above. */ + if (atomic_load32(&env->lck->meta_sync_txnid, mo_Relaxed) != (uint32_t)head.txnid) + rc = meta_sync(env, head); -__cold int mdbx_env_copy(MDBX_env *env, const char *dest_path, - MDBX_copy_flags_t flags) { -#if defined(_WIN32) || defined(_WIN64) - wchar_t *dest_pathW = nullptr; - int rc = osal_mb2w(dest_path, &dest_pathW); - if (likely(rc == MDBX_SUCCESS)) { - rc = mdbx_env_copyW(env, dest_pathW, flags); - osal_free(dest_pathW); - } +bailout: + if (should_unlock) + lck_txn_unlock(env); return rc; } -__cold int mdbx_env_copyW(MDBX_env *env, const wchar_t *dest_path, - MDBX_copy_flags_t flags) { -#endif /* Windows */ +__cold int env_open(MDBX_env *env, mdbx_mode_t mode) { + /* Использование O_DSYNC или FILE_FLAG_WRITE_THROUGH: + * + * 0) Если размер страниц БД меньше системной страницы ОЗУ, то ядру ОС + * придется чаще обновлять страницы в unified page cache. + * + * Однако, O_DSYNC не предполагает отключение unified page cache, + * поэтому подобные затруднения будем считать проблемой ОС и/или + * ожидаемым пенальти из-за использования мелких страниц БД. + * + * 1) В режиме MDBX_SYNC_DURABLE - O_DSYNC для записи как данных, + * так и мета-страниц. Однако, на Linux отказ от O_DSYNC с последующим + * fdatasync() может быть выгоднее при использовании HDD, так как + * позволяет io-scheduler переупорядочить запись с учетом актуального + * расположения файла БД на носителе. + * + * 2) В режиме MDBX_NOMETASYNC - O_DSYNC можно использовать для данных, + * но в этом может не быть смысла, так как fdatasync() всё равно + * требуется для гарантии фиксации мета после предыдущей транзакции. + * + * В итоге на нормальных системах (не Windows) есть два варианта: + * - при возможности O_DIRECT и/или io_ring для данных, скорее всего, + * есть смысл вызвать fdatasync() перед записью данных, а затем + * использовать O_DSYNC; + * - не использовать O_DSYNC и вызывать fdatasync() после записи данных. + * + * На Windows же следует минимизировать использование FlushFileBuffers() + * из-за проблем с производительностью. Поэтому на Windows в режиме + * MDBX_NOMETASYNC: + * - мета обновляется через дескриптор без FILE_FLAG_WRITE_THROUGH; + * - перед началом записи данных вызывается FlushFileBuffers(), если + * meta_sync_txnid не совпадает с последней записанной мета; + * - данные записываются через дескриптор с FILE_FLAG_WRITE_THROUGH. + * + * 3) В режиме MDBX_SAFE_NOSYNC - O_DSYNC нет смысла использовать, пока не + * будет реализована возможность полностью асинхронной "догоняющей" + * записи в выделенном процессе-сервере с io-ring очередями внутри. + * + * ----- + * + * Использование O_DIRECT или FILE_FLAG_NO_BUFFERING: + * + * Назначение этих флагов в отключении файлового дескриптора от + * unified page cache, т.е. от отображенных в память данных в случае + * libmdbx. + * + * Поэтому, использование direct i/o в libmdbx без MDBX_WRITEMAP лишено + * смысла и контр-продуктивно, ибо так мы провоцируем ядро ОС на + * не-когерентность отображения в память с содержимым файла на носителе, + * либо требуем дополнительных проверок и действий направленных на + * фактическое отключение O_DIRECT для отображенных в память данных. + * + * В режиме MDBX_WRITEMAP когерентность отображенных данных обеспечивается + * физически. Поэтому использование direct i/o может иметь смысл, если у + * ядра ОС есть какие-то проблемы с msync(), в том числе с + * производительностью: + * - использование io_ring или gather-write может быть дешевле, чем + * просмотр PTE ядром и запись измененных/грязных; + * - но проблема в том, что записываемые из user mode страницы либо не + * будут помечены чистыми (и соответственно будут записаны ядром + * еще раз), либо ядру необходимо искать и чистить PTE при получении + * запроса на запись. + * + * Поэтому O_DIRECT или FILE_FLAG_NO_BUFFERING используется: + * - только в режиме MDBX_SYNC_DURABLE с MDBX_WRITEMAP; + * - когда ps >= me_os_psize; + * - опция сборки MDBX_AVOID_MSYNC != 0, которая по-умолчанию включена + * только на Windows (см ниже). + * + * ----- + * + * Использование FILE_FLAG_OVERLAPPED на Windows: + * + * У Windows очень плохо с I/O (за исключением прямых постраничных + * scatter/gather, которые работают в обход проблемного unified page + * cache и поэтому почти бесполезны в libmdbx). + * + * При этом всё еще хуже при использовании FlushFileBuffers(), что также + * требуется после FlushViewOfFile() в режиме MDBX_WRITEMAP. Поэтому + * на Windows вместо FlushViewOfFile() и FlushFileBuffers() следует + * использовать запись через дескриптор с FILE_FLAG_WRITE_THROUGH. + * + * В свою очередь, запись с FILE_FLAG_WRITE_THROUGH дешевле/быстрее + * при использовании FILE_FLAG_OVERLAPPED. В результате, на Windows + * в durable-режимах запись данных всегда в overlapped-режиме, + * при этом для записи мета требуется отдельный не-overlapped дескриптор. + */ - int rc = check_env(env, true); + env->pid = osal_getpid(); + int rc = osal_openfile((env->flags & MDBX_RDONLY) ? MDBX_OPEN_DXB_READ : MDBX_OPEN_DXB_LAZY, env, env->pathname.dxb, + &env->lazy_fd, mode); if (unlikely(rc != MDBX_SUCCESS)) return rc; - if (unlikely(!dest_path)) - return MDBX_EINVAL; +#if MDBX_LOCKING == MDBX_LOCKING_SYSV + env->me_sysv_ipc.key = ftok(env->pathname.dxb, 42); + if (unlikely(env->me_sysv_ipc.key == -1)) + return errno; +#endif /* MDBX_LOCKING */ - /* The destination path must exist, but the destination file must not. - * We don't want the OS to cache the writes, since the source data is - * already in the OS cache. */ - mdbx_filehandle_t newfd; - rc = osal_openfile(MDBX_OPEN_COPY, env, dest_path, &newfd, -#if defined(_WIN32) || defined(_WIN64) - (mdbx_mode_t)-1 -#else - S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP -#endif - ); + /* Set the position in files outside of the data to avoid corruption + * due to erroneous use of file descriptors in the application code. */ + const uint64_t safe_parking_lot_offset = UINT64_C(0x7fffFFFF80000000); + osal_fseek(env->lazy_fd, safe_parking_lot_offset); + env->fd4meta = env->lazy_fd; #if defined(_WIN32) || defined(_WIN64) - /* no locking required since the file opened with ShareMode == 0 */ + eASSERT(env, env->ioring.overlapped_fd == 0); + bool ior_direct = false; + if (!(env->flags & (MDBX_RDONLY | MDBX_SAFE_NOSYNC | MDBX_NOMETASYNC | MDBX_EXCLUSIVE))) { + if (MDBX_AVOID_MSYNC && (env->flags & MDBX_WRITEMAP)) { + /* Запрошен режим MDBX_SYNC_DURABLE | MDBX_WRITEMAP при активной опции + * MDBX_AVOID_MSYNC. + * + * 1) В этой комбинации наиболее выгодно использовать WriteFileGather(), + * но для этого необходимо открыть файл с флагом FILE_FLAG_NO_BUFFERING и + * после обеспечивать выравнивание адресов и размера данных на границу + * системной страницы, что в свою очередь возможно если размер страницы БД + * не меньше размера системной страницы ОЗУ. Поэтому для открытия файла в + * нужном режиме требуется знать размер страницы БД. + * + * 2) Кроме этого, в Windows запись в заблокированный регион файла + * возможно только через тот-же дескриптор. Поэтому изначальный захват + * блокировок посредством lck_seize(), захват/освобождение блокировок + * во время пишущих транзакций и запись данных должны выполнятся через + * один дескриптор. + * + * Таким образом, требуется прочитать волатильный заголовок БД, чтобы + * узнать размер страницы, чтобы открыть дескриптор файла в режиме нужном + * для записи данных, чтобы использовать именно этот дескриптор для + * изначального захвата блокировок. */ + meta_t header; + uint64_t dxb_filesize; + int err = dxb_read_header(env, &header, MDBX_SUCCESS, true); + if ((err == MDBX_SUCCESS && header.pagesize >= globals.sys_pagesize) || + (err == MDBX_ENODATA && mode && env->ps >= globals.sys_pagesize && + osal_filesize(env->lazy_fd, &dxb_filesize) == MDBX_SUCCESS && dxb_filesize == 0)) + /* Может быть коллизия, если два процесса пытаются одновременно создать + * БД с разным размером страницы, который у одного меньше системной + * страницы, а у другого НЕ меньше. Эта допустимая, но очень странная + * ситуация. Поэтому считаем её ошибочной и не пытаемся разрешить. */ + ior_direct = true; + } + + rc = osal_openfile(ior_direct ? MDBX_OPEN_DXB_OVERLAPPED_DIRECT : MDBX_OPEN_DXB_OVERLAPPED, env, env->pathname.dxb, + &env->ioring.overlapped_fd, 0); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + env->dxb_lock_event = CreateEventW(nullptr, true, false, nullptr); + if (unlikely(!env->dxb_lock_event)) + return (int)GetLastError(); + osal_fseek(env->ioring.overlapped_fd, safe_parking_lot_offset); + } #else - if (rc == MDBX_SUCCESS) { - MDBX_STRUCT_FLOCK lock_op; - memset(&lock_op, 0, sizeof(lock_op)); - lock_op.l_type = F_WRLCK; - lock_op.l_whence = SEEK_SET; - lock_op.l_start = 0; - lock_op.l_len = OFF_T_MAX; - if (MDBX_FCNTL(newfd, MDBX_F_SETLK, &lock_op) -#if (defined(__linux__) || defined(__gnu_linux__)) && defined(LOCK_EX) && \ - (!defined(__ANDROID_API__) || __ANDROID_API__ >= 24) - || flock(newfd, LOCK_EX | LOCK_NB) -#endif /* Linux */ - ) - rc = errno; + if (mode == 0) { + /* pickup mode for lck-file */ + struct stat st; + if (unlikely(fstat(env->lazy_fd, &st))) + return errno; + mode = st.st_mode; } -#endif /* Windows / POSIX */ - - if (rc == MDBX_SUCCESS) - rc = mdbx_env_copy2fd(env, newfd, flags); - - if (newfd != INVALID_HANDLE_VALUE) { - int err = osal_closefile(newfd); - if (rc == MDBX_SUCCESS && err != rc) - rc = err; - if (rc != MDBX_SUCCESS) - (void)osal_removefile(dest_path); + mode = (/* inherit read permissions for group and others */ mode & (S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH)) | + /* always add read/write for owner */ S_IRUSR | S_IWUSR | + ((mode & S_IRGRP) ? /* +write if readable by group */ S_IWGRP : 0) | + ((mode & S_IROTH) ? /* +write if readable by others */ S_IWOTH : 0); +#endif /* !Windows */ + const int lck_rc = lck_setup(env, mode); + if (unlikely(MDBX_IS_ERROR(lck_rc))) + return lck_rc; + if (env->lck_mmap.fd != INVALID_HANDLE_VALUE) + osal_fseek(env->lck_mmap.fd, safe_parking_lot_offset); + + eASSERT(env, env->dsync_fd == INVALID_HANDLE_VALUE); + if (!(env->flags & (MDBX_RDONLY | MDBX_SAFE_NOSYNC | DEPRECATED_MAPASYNC +#if defined(_WIN32) || defined(_WIN64) + | MDBX_EXCLUSIVE +#endif /* !Windows */ + ))) { + rc = osal_openfile(MDBX_OPEN_DXB_DSYNC, env, env->pathname.dxb, &env->dsync_fd, 0); + if (unlikely(MDBX_IS_ERROR(rc))) + return rc; + if (env->dsync_fd != INVALID_HANDLE_VALUE) { + if ((env->flags & MDBX_NOMETASYNC) == 0) + env->fd4meta = env->dsync_fd; + osal_fseek(env->dsync_fd, safe_parking_lot_offset); + } } - return rc; -} - -/******************************************************************************/ - -__cold int mdbx_env_set_flags(MDBX_env *env, MDBX_env_flags_t flags, - bool onoff) { - int rc = check_env(env, false); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - - if (unlikely(flags & - ((env->me_flags & MDBX_ENV_ACTIVE) ? ~ENV_CHANGEABLE_FLAGS - : ~ENV_USABLE_FLAGS))) - return MDBX_EPERM; - - if (unlikely(env->me_flags & MDBX_RDONLY)) - return MDBX_EACCESS; + const MDBX_env_flags_t lazy_flags = MDBX_SAFE_NOSYNC | MDBX_UTTERLY_NOSYNC | MDBX_NOMETASYNC; + const MDBX_env_flags_t mode_flags = lazy_flags | MDBX_LIFORECLAIM | MDBX_NORDAHEAD | MDBX_RDONLY | MDBX_WRITEMAP; - if ((env->me_flags & MDBX_ENV_ACTIVE) && - unlikely(env->me_txn0->mt_owner == osal_thread_self())) - return MDBX_BUSY; + lck_t *const lck = env->lck_mmap.lck; + if (lck && lck_rc != MDBX_RESULT_TRUE && (env->flags & MDBX_RDONLY) == 0) { + MDBX_env_flags_t snap_flags; + while ((snap_flags = atomic_load32(&lck->envmode, mo_AcquireRelease)) == MDBX_RDONLY) { + if (atomic_cas32(&lck->envmode, MDBX_RDONLY, (snap_flags = (env->flags & mode_flags)))) { + /* The case: + * - let's assume that for some reason the DB file is smaller + * than it should be according to the geometry, + * but not smaller than the last page used; + * - the first process that opens the database (lck_rc == RESULT_TRUE) + * does this in readonly mode and therefore cannot bring + * the file size back to normal; + * - some next process (lck_rc != RESULT_TRUE) opens the DB in + * read-write mode and now is here. + * + * FIXME: Should we re-check and set the size of DB-file right here? */ + break; + } + atomic_yield(); + } - const bool lock_needed = (env->me_flags & MDBX_ENV_ACTIVE) && - env->me_txn0->mt_owner != osal_thread_self(); - bool should_unlock = false; - if (lock_needed) { - rc = mdbx_txn_lock(env, false); - if (unlikely(rc)) - return rc; - should_unlock = true; - } + if (env->flags & MDBX_ACCEDE) { + /* Pickup current mode-flags (MDBX_LIFORECLAIM, MDBX_NORDAHEAD, etc). */ + const MDBX_env_flags_t diff = + (snap_flags ^ env->flags) & ((snap_flags & lazy_flags) ? mode_flags : mode_flags & ~MDBX_WRITEMAP); + env->flags ^= diff; + NOTICE("accede mode-flags: 0x%X, 0x%X -> 0x%X", diff, env->flags ^ diff, env->flags); + } - if (onoff) - env->me_flags = merge_sync_flags(env->me_flags, flags); - else - env->me_flags &= ~flags; + /* Ранее упущенный не очевидный момент: При работе БД в режимах + * не-синхронной/отложенной фиксации на диске, все процессы-писатели должны + * иметь одинаковый режим MDBX_WRITEMAP. + * + * В противном случае, сброс на диск следует выполнять дважды: сначала + * msync(), затем fdatasync(). При этом msync() не обязан отрабатывать + * в процессах без MDBX_WRITEMAP, так как файл в память отображен только + * для чтения. Поэтому, в общем случае, различия по MDBX_WRITEMAP не + * позволяют выполнить фиксацию данных на диск, после их изменения в другом + * процессе. + * + * В режиме MDBX_UTTERLY_NOSYNC позволять совместную работу с MDBX_WRITEMAP + * также не следует, поскольку никакой процесс (в том числе последний) не + * может гарантированно сбросить данные на диск, а следовательно не должен + * помечать какую-либо транзакцию как steady. + * + * В результате, требуется либо запретить совместную работу процессам с + * разным MDBX_WRITEMAP в режиме отложенной записи, либо отслеживать такое + * смешивание и блокировать steady-пометки - что контрпродуктивно. */ + const MDBX_env_flags_t rigorous_flags = (snap_flags & lazy_flags) + ? MDBX_SAFE_NOSYNC | MDBX_UTTERLY_NOSYNC | MDBX_WRITEMAP + : MDBX_SAFE_NOSYNC | MDBX_UTTERLY_NOSYNC; + const MDBX_env_flags_t rigorous_diff = (snap_flags ^ env->flags) & rigorous_flags; + if (rigorous_diff) { + ERROR("current mode/flags 0x%X incompatible with requested 0x%X, " + "rigorous diff 0x%X", + env->flags, snap_flags, rigorous_diff); + return MDBX_INCOMPATIBLE; + } + } - if (should_unlock) - mdbx_txn_unlock(env); - return MDBX_SUCCESS; -} + mincore_clean_cache(env); + const int dxb_rc = dxb_setup(env, lck_rc, mode); + if (MDBX_IS_ERROR(dxb_rc)) + return dxb_rc; -__cold int mdbx_env_get_flags(const MDBX_env *env, unsigned *arg) { - int rc = check_env(env, false); - if (unlikely(rc != MDBX_SUCCESS)) + rc = osal_check_fs_incore(env->lazy_fd); + env->incore = false; + if (rc == MDBX_RESULT_TRUE) { + env->incore = true; + NOTICE("%s", "in-core database"); + rc = MDBX_SUCCESS; + } else if (unlikely(rc != MDBX_SUCCESS)) { + ERROR("check_fs_incore(), err %d", rc); return rc; + } - if (unlikely(!arg)) - return MDBX_EINVAL; - - *arg = env->me_flags & ENV_USABLE_FLAGS; - return MDBX_SUCCESS; -} - -__cold int mdbx_env_set_userctx(MDBX_env *env, void *ctx) { - int rc = check_env(env, false); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; + if (unlikely(/* recovery mode */ env->stuck_meta >= 0) && + (lck_rc != /* exclusive */ MDBX_RESULT_TRUE || (env->flags & MDBX_EXCLUSIVE) == 0)) { + ERROR("%s", "recovery requires exclusive mode"); + return MDBX_BUSY; + } - env->me_userctx = ctx; - return MDBX_SUCCESS; -} + DEBUG("opened dbenv %p", (void *)env); + env->flags |= ENV_ACTIVE; + if (!lck || lck_rc == MDBX_RESULT_TRUE) { + env->lck->envmode.weak = env->flags & mode_flags; + env->lck->meta_sync_txnid.weak = (uint32_t)recent_committed_txnid(env); + env->lck->readers_check_timestamp.weak = osal_monotime(); + } + if (lck) { + if (lck_rc == MDBX_RESULT_TRUE) { + rc = lck_downgrade(env); + DEBUG("lck-downgrade-%s: rc %i", (env->flags & MDBX_EXCLUSIVE) ? "partial" : "full", rc); + if (rc != MDBX_SUCCESS) + return rc; + } else { + rc = mvcc_cleanup_dead(env, false, nullptr); + if (MDBX_IS_ERROR(rc)) + return rc; + } + } -__cold void *mdbx_env_get_userctx(const MDBX_env *env) { - return env ? env->me_userctx : NULL; + rc = (env->flags & MDBX_RDONLY) ? MDBX_SUCCESS + : osal_ioring_create(&env->ioring +#if defined(_WIN32) || defined(_WIN64) + , + ior_direct, env->ioring.overlapped_fd +#endif /* Windows */ + ); + return rc; } -__cold int mdbx_env_set_assert(MDBX_env *env, MDBX_assert_func *func) { - int rc = check_env(env, false); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; +__cold int env_close(MDBX_env *env, bool resurrect_after_fork) { + const unsigned flags = env->flags; + env->flags &= ~ENV_INTERNAL_FLAGS; + if (flags & ENV_TXKEY) { + thread_key_delete(env->me_txkey); + env->me_txkey = 0; + } -#if MDBX_DEBUG - env->me_assert_func = func; - return MDBX_SUCCESS; -#else - (void)func; - return MDBX_ENOSYS; -#endif -} + if (env->lck) + munlock_all(env); -#if defined(_WIN32) || defined(_WIN64) -__cold int mdbx_env_get_pathW(const MDBX_env *env, const wchar_t **arg) { - int rc = check_env(env, true); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; + rthc_lock(); + int rc = rthc_remove(env); + rthc_unlock(); - if (unlikely(!arg)) - return MDBX_EINVAL; +#if MDBX_ENABLE_DBI_LOCKFREE + for (defer_free_item_t *next, *ptr = env->defer_free; ptr; ptr = next) { + next = ptr->next; + osal_free(ptr); + } + env->defer_free = nullptr; +#endif /* MDBX_ENABLE_DBI_LOCKFREE */ - *arg = env->me_pathname; - return MDBX_SUCCESS; -} -#endif /* Windows */ + if (!(env->flags & MDBX_RDONLY)) + osal_ioring_destroy(&env->ioring); -__cold int mdbx_env_get_path(const MDBX_env *env, const char **arg) { - int rc = check_env(env, true); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; + env->lck = nullptr; + if (env->lck_mmap.lck) + osal_munmap(&env->lck_mmap); - if (unlikely(!arg)) - return MDBX_EINVAL; + if (env->dxb_mmap.base) { + osal_munmap(&env->dxb_mmap); +#ifdef ENABLE_MEMCHECK + VALGRIND_DISCARD(env->valgrind_handle); + env->valgrind_handle = -1; +#endif /* ENABLE_MEMCHECK */ + } #if defined(_WIN32) || defined(_WIN64) - if (!env->me_pathname_char) { - *arg = nullptr; - DWORD flags = /* WC_ERR_INVALID_CHARS */ 0x80; - size_t mb_len = WideCharToMultiByte(CP_THREAD_ACP, flags, env->me_pathname, - -1, nullptr, 0, nullptr, nullptr); - rc = mb_len ? MDBX_SUCCESS : (int)GetLastError(); - if (rc == ERROR_INVALID_FLAGS) { - mb_len = WideCharToMultiByte(CP_THREAD_ACP, flags = 0, env->me_pathname, - -1, nullptr, 0, nullptr, nullptr); - rc = mb_len ? MDBX_SUCCESS : (int)GetLastError(); - } - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - - char *const mb_pathname = osal_malloc(mb_len); - if (!mb_pathname) - return MDBX_ENOMEM; - if (mb_len != (size_t)WideCharToMultiByte(CP_THREAD_ACP, flags, - env->me_pathname, -1, mb_pathname, - (int)mb_len, nullptr, nullptr)) { - rc = (int)GetLastError(); - osal_free(mb_pathname); - return rc; - } - if (env->me_pathname_char || - InterlockedCompareExchangePointer( - (PVOID volatile *)&env->me_pathname_char, mb_pathname, nullptr)) - osal_free(mb_pathname); + eASSERT(env, !env->ioring.overlapped_fd || env->ioring.overlapped_fd == INVALID_HANDLE_VALUE); + if (env->dxb_lock_event != INVALID_HANDLE_VALUE) { + CloseHandle(env->dxb_lock_event); + env->dxb_lock_event = INVALID_HANDLE_VALUE; + } + eASSERT(env, !resurrect_after_fork); + if (env->pathname_char) { + osal_free(env->pathname_char); + env->pathname_char = nullptr; } - *arg = env->me_pathname_char; -#else - *arg = env->me_pathname; #endif /* Windows */ - return MDBX_SUCCESS; -} - -__cold int mdbx_env_get_fd(const MDBX_env *env, mdbx_filehandle_t *arg) { - int rc = check_env(env, true); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - if (unlikely(!arg)) - return MDBX_EINVAL; + if (env->dsync_fd != INVALID_HANDLE_VALUE) { + (void)osal_closefile(env->dsync_fd); + env->dsync_fd = INVALID_HANDLE_VALUE; + } - *arg = env->me_lazy_fd; - return MDBX_SUCCESS; -} + if (env->lazy_fd != INVALID_HANDLE_VALUE) { + (void)osal_closefile(env->lazy_fd); + env->lazy_fd = INVALID_HANDLE_VALUE; + } -static void stat_get(const MDBX_db *db, MDBX_stat *st, size_t bytes) { - st->ms_depth = db->md_depth; - st->ms_branch_pages = db->md_branch_pages; - st->ms_leaf_pages = db->md_leaf_pages; - st->ms_overflow_pages = db->md_overflow_pages; - st->ms_entries = db->md_entries; - if (likely(bytes >= - offsetof(MDBX_stat, ms_mod_txnid) + sizeof(st->ms_mod_txnid))) - st->ms_mod_txnid = db->md_mod_txnid; -} + if (env->lck_mmap.fd != INVALID_HANDLE_VALUE) { + (void)osal_closefile(env->lck_mmap.fd); + env->lck_mmap.fd = INVALID_HANDLE_VALUE; + } -static void stat_add(const MDBX_db *db, MDBX_stat *const st, - const size_t bytes) { - st->ms_depth += db->md_depth; - st->ms_branch_pages += db->md_branch_pages; - st->ms_leaf_pages += db->md_leaf_pages; - st->ms_overflow_pages += db->md_overflow_pages; - st->ms_entries += db->md_entries; - if (likely(bytes >= - offsetof(MDBX_stat, ms_mod_txnid) + sizeof(st->ms_mod_txnid))) - st->ms_mod_txnid = (st->ms_mod_txnid > db->md_mod_txnid) ? st->ms_mod_txnid - : db->md_mod_txnid; + if (!resurrect_after_fork) { + if (env->kvs) { + for (size_t i = CORE_DBS; i < env->n_dbi; ++i) + if (env->kvs[i].name.iov_len) + osal_free(env->kvs[i].name.iov_base); + osal_free(env->kvs); + env->n_dbi = CORE_DBS; + env->kvs = nullptr; + } + if (env->page_auxbuf) { + osal_memalign_free(env->page_auxbuf); + env->page_auxbuf = nullptr; + } + if (env->dbi_seqs) { + osal_free(env->dbi_seqs); + env->dbi_seqs = nullptr; + } + if (env->dbs_flags) { + osal_free(env->dbs_flags); + env->dbs_flags = nullptr; + } + if (env->pathname.buffer) { + osal_free(env->pathname.buffer); + env->pathname.buffer = nullptr; + } + if (env->basal_txn) { + dpl_free(env->basal_txn); + txl_free(env->basal_txn->tw.gc.retxl); + pnl_free(env->basal_txn->tw.retired_pages); + pnl_free(env->basal_txn->tw.spilled.list); + pnl_free(env->basal_txn->tw.repnl); + osal_free(env->basal_txn); + env->basal_txn = nullptr; + } + } + env->stuck_meta = -1; + return rc; } +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 -__cold static int stat_acc(const MDBX_txn *txn, MDBX_stat *st, size_t bytes) { - int err = check_txn(txn, MDBX_TXN_BLOCKED); - if (unlikely(err != MDBX_SUCCESS)) - return err; - - st->ms_psize = txn->mt_env->me_psize; -#if 1 - /* assuming GC is internal and not subject for accounting */ - stat_get(&txn->mt_dbs[MAIN_DBI], st, bytes); -#else - stat_get(&txn->mt_dbs[FREE_DBI], st, bytes); - stat_add(&txn->mt_dbs[MAIN_DBI], st, bytes); -#endif - - /* account opened named subDBs */ - for (MDBX_dbi dbi = CORE_DBS; dbi < txn->mt_numdbs; dbi++) - if ((txn->mt_dbistate[dbi] & (DBI_VALID | DBI_STALE)) == DBI_VALID) - stat_add(txn->mt_dbs + dbi, st, bytes); - - if (!(txn->mt_dbs[MAIN_DBI].md_flags & (MDBX_DUPSORT | MDBX_INTEGERKEY)) && - txn->mt_dbs[MAIN_DBI].md_entries /* TODO: use `md_subs` field */) { - MDBX_cursor_couple cx; - err = cursor_init(&cx.outer, (MDBX_txn *)txn, MAIN_DBI); - if (unlikely(err != MDBX_SUCCESS)) - return err; - - /* scan and account not opened named subDBs */ - err = page_search(&cx.outer, NULL, MDBX_PS_FIRST); - while (err == MDBX_SUCCESS) { - const MDBX_page *mp = cx.outer.mc_pg[cx.outer.mc_top]; - for (size_t i = 0; i < page_numkeys(mp); i++) { - const MDBX_node *node = page_node(mp, i); - if (node_flags(node) != F_SUBDATA) - continue; - if (unlikely(node_ds(node) != sizeof(MDBX_db))) - return MDBX_CORRUPTED; +#if MDBX_USE_MINCORE +/*------------------------------------------------------------------------------ + * Проверка размещения/расположения отображенных страниц БД в ОЗУ (mem-in-core), + * с кешированием этой информации. */ - /* skip opened and already accounted */ - for (MDBX_dbi dbi = CORE_DBS; dbi < txn->mt_numdbs; dbi++) - if ((txn->mt_dbistate[dbi] & (DBI_VALID | DBI_STALE)) == DBI_VALID && - node_ks(node) == txn->mt_dbxs[dbi].md_name.iov_len && - memcmp(node_key(node), txn->mt_dbxs[dbi].md_name.iov_base, - node_ks(node)) == 0) { - node = NULL; - break; - } +static inline bool bit_tas(uint64_t *field, char bit) { + const uint64_t m = UINT64_C(1) << bit; + const bool r = (*field & m) != 0; + *field |= m; + return r; +} - if (node) { - MDBX_db db; - memcpy(&db, node_data(node), sizeof(db)); - stat_add(&db, st, bytes); - } - } - err = cursor_sibling(&cx.outer, SIBLING_RIGHT); +static bool mincore_fetch(MDBX_env *const env, const size_t unit_begin) { + lck_t *const lck = env->lck; + for (size_t i = 1; i < ARRAY_LENGTH(lck->mincore_cache.begin); ++i) { + const ptrdiff_t dist = unit_begin - lck->mincore_cache.begin[i]; + if (likely(dist >= 0 && dist < 64)) { + const pgno_t tmp_begin = lck->mincore_cache.begin[i]; + const uint64_t tmp_mask = lck->mincore_cache.mask[i]; + do { + lck->mincore_cache.begin[i] = lck->mincore_cache.begin[i - 1]; + lck->mincore_cache.mask[i] = lck->mincore_cache.mask[i - 1]; + } while (--i); + lck->mincore_cache.begin[0] = tmp_begin; + lck->mincore_cache.mask[0] = tmp_mask; + return bit_tas(lck->mincore_cache.mask, (char)dist); } - if (unlikely(err != MDBX_NOTFOUND)) - return err; } - return MDBX_SUCCESS; -} + size_t pages = 64; + unsigned unit_log = globals.sys_pagesize_ln2; + unsigned shift = 0; + if (env->ps > globals.sys_pagesize) { + unit_log = env->ps2ln; + shift = env->ps2ln - globals.sys_pagesize_ln2; + pages <<= shift; + } -__cold int mdbx_env_stat_ex(const MDBX_env *env, const MDBX_txn *txn, - MDBX_stat *dest, size_t bytes) { - if (unlikely(!dest)) - return MDBX_EINVAL; - const size_t size_before_modtxnid = offsetof(MDBX_stat, ms_mod_txnid); - if (unlikely(bytes != sizeof(MDBX_stat)) && bytes != size_before_modtxnid) - return MDBX_EINVAL; + const size_t offset = unit_begin << unit_log; + size_t length = pages << globals.sys_pagesize_ln2; + if (offset + length > env->dxb_mmap.current) { + length = env->dxb_mmap.current - offset; + pages = length >> globals.sys_pagesize_ln2; + } - if (likely(txn)) { - if (env && unlikely(txn->mt_env != env)) - return MDBX_EINVAL; - return stat_acc(txn, dest, bytes); +#if MDBX_ENABLE_PGOP_STAT + env->lck->pgops.mincore.weak += 1; +#endif /* MDBX_ENABLE_PGOP_STAT */ + uint8_t *const vector = alloca(pages); + if (unlikely(mincore(ptr_disp(env->dxb_mmap.base, offset), length, (void *)vector))) { + NOTICE("mincore(+%zu, %zu), err %d", offset, length, errno); + return false; } - int err = check_env(env, true); - if (unlikely(err != MDBX_SUCCESS)) - return err; + for (size_t i = 1; i < ARRAY_LENGTH(lck->mincore_cache.begin); ++i) { + lck->mincore_cache.begin[i] = lck->mincore_cache.begin[i - 1]; + lck->mincore_cache.mask[i] = lck->mincore_cache.mask[i - 1]; + } + lck->mincore_cache.begin[0] = unit_begin; - if (env->me_txn0 && env->me_txn0->mt_owner == osal_thread_self()) - /* inside write-txn */ - return stat_acc(env->me_txn, dest, bytes); + uint64_t mask = 0; +#ifdef MINCORE_INCORE + STATIC_ASSERT(MINCORE_INCORE == 1); +#endif + for (size_t i = 0; i < pages; ++i) { + uint64_t bit = (vector[i] & 1) == 0; + bit <<= i >> shift; + mask |= bit; + } - MDBX_txn *tmp_txn; - err = mdbx_txn_begin((MDBX_env *)env, NULL, MDBX_TXN_RDONLY, &tmp_txn); - if (unlikely(err != MDBX_SUCCESS)) - return err; + lck->mincore_cache.mask[0] = ~mask; + return bit_tas(lck->mincore_cache.mask, 0); +} +#endif /* MDBX_USE_MINCORE */ - const int rc = stat_acc(tmp_txn, dest, bytes); - err = mdbx_txn_abort(tmp_txn); - if (unlikely(err != MDBX_SUCCESS)) - return err; - return rc; +MDBX_MAYBE_UNUSED static inline bool mincore_probe(MDBX_env *const env, const pgno_t pgno) { +#if MDBX_USE_MINCORE + const size_t offset_aligned = floor_powerof2(pgno2bytes(env, pgno), globals.sys_pagesize); + const unsigned unit_log2 = (env->ps2ln > globals.sys_pagesize_ln2) ? env->ps2ln : globals.sys_pagesize_ln2; + const size_t unit_begin = offset_aligned >> unit_log2; + eASSERT(env, (unit_begin << unit_log2) == offset_aligned); + const ptrdiff_t dist = unit_begin - env->lck->mincore_cache.begin[0]; + if (likely(dist >= 0 && dist < 64)) + return bit_tas(env->lck->mincore_cache.mask, (char)dist); + return mincore_fetch(env, unit_begin); +#else + (void)env; + (void)pgno; + return false; +#endif /* MDBX_USE_MINCORE */ } -__cold int mdbx_dbi_dupsort_depthmask(const MDBX_txn *txn, MDBX_dbi dbi, - uint32_t *mask) { - int rc = check_txn(txn, MDBX_TXN_BLOCKED); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - - if (unlikely(!mask)) - return MDBX_EINVAL; - - if (unlikely(!check_dbi(txn, dbi, DBI_VALID))) - return MDBX_BAD_DBI; - - MDBX_cursor_couple cx; - rc = cursor_init(&cx.outer, txn, dbi); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - if ((cx.outer.mc_db->md_flags & MDBX_DUPSORT) == 0) - return MDBX_RESULT_TRUE; +/*----------------------------------------------------------------------------*/ - MDBX_val key, data; - rc = cursor_first(&cx.outer, &key, &data); - *mask = 0; - while (rc == MDBX_SUCCESS) { - const MDBX_node *node = page_node(cx.outer.mc_pg[cx.outer.mc_top], - cx.outer.mc_ki[cx.outer.mc_top]); - const MDBX_db *db = node_data(node); - const unsigned flags = node_flags(node); - switch (flags) { - case F_BIGDATA: - case 0: - /* single-value entry, deep = 0 */ - *mask |= 1 << 0; - break; - case F_DUPDATA: - /* single sub-page, deep = 1 */ - *mask |= 1 << 1; - break; - case F_DUPDATA | F_SUBDATA: - /* sub-tree */ - *mask |= 1 << UNALIGNED_PEEK_16(db, MDBX_db, md_depth); - break; - default: - ERROR("wrong node-flags %u", flags); - return MDBX_CORRUPTED; - } - rc = cursor_next(&cx.outer, &key, &data, MDBX_NEXT_NODUP); +MDBX_MAYBE_UNUSED __hot static pgno_t *scan4seq_fallback(pgno_t *range, const size_t len, const size_t seq) { + assert(seq > 0 && len > seq); +#if MDBX_PNL_ASCENDING + assert(range[-1] == len); + const pgno_t *const detent = range + len - seq; + const ptrdiff_t offset = (ptrdiff_t)seq; + const pgno_t target = (pgno_t)offset; + if (likely(len > seq + 3)) { + do { + const pgno_t diff0 = range[offset + 0] - range[0]; + const pgno_t diff1 = range[offset + 1] - range[1]; + const pgno_t diff2 = range[offset + 2] - range[2]; + const pgno_t diff3 = range[offset + 3] - range[3]; + if (diff0 == target) + return range + 0; + if (diff1 == target) + return range + 1; + if (diff2 == target) + return range + 2; + if (diff3 == target) + return range + 3; + range += 4; + } while (range + 3 < detent); + if (range == detent) + return nullptr; } - - return (rc == MDBX_NOTFOUND) ? MDBX_SUCCESS : rc; + do + if (range[offset] - *range == target) + return range; + while (++range < detent); +#else + assert(range[-(ptrdiff_t)len] == len); + const pgno_t *const detent = range - len + seq; + const ptrdiff_t offset = -(ptrdiff_t)seq; + const pgno_t target = (pgno_t)offset; + if (likely(len > seq + 3)) { + do { + const pgno_t diff0 = range[-0] - range[offset - 0]; + const pgno_t diff1 = range[-1] - range[offset - 1]; + const pgno_t diff2 = range[-2] - range[offset - 2]; + const pgno_t diff3 = range[-3] - range[offset - 3]; + /* Смысл вычислений до ветвлений в том, чтобы позволить компилятору + * загружать и вычислять все значения параллельно. */ + if (diff0 == target) + return range - 0; + if (diff1 == target) + return range - 1; + if (diff2 == target) + return range - 2; + if (diff3 == target) + return range - 3; + range -= 4; + } while (range > detent + 3); + if (range == detent) + return nullptr; + } + do + if (*range - range[offset] == target) + return range; + while (--range > detent); +#endif /* pnl_t sort-order */ + return nullptr; } -__cold static int fetch_envinfo_ex(const MDBX_env *env, const MDBX_txn *txn, - MDBX_envinfo *arg, const size_t bytes) { +MDBX_MAYBE_UNUSED static const pgno_t *scan4range_checker(const pnl_t pnl, const size_t seq) { + size_t begin = MDBX_PNL_ASCENDING ? 1 : MDBX_PNL_GETSIZE(pnl); +#if MDBX_PNL_ASCENDING + while (seq <= MDBX_PNL_GETSIZE(pnl) - begin) { + if (pnl[begin + seq] - pnl[begin] == seq) + return pnl + begin; + ++begin; + } +#else + while (begin > seq) { + if (pnl[begin - seq] - pnl[begin] == seq) + return pnl + begin; + --begin; + } +#endif /* pnl_t sort-order */ + return nullptr; +} - const size_t size_before_bootid = offsetof(MDBX_envinfo, mi_bootid); - const size_t size_before_pgop_stat = offsetof(MDBX_envinfo, mi_pgop_stat); +#if defined(_MSC_VER) && !defined(__builtin_clz) && !__has_builtin(__builtin_clz) +MDBX_MAYBE_UNUSED static __always_inline size_t __builtin_clz(uint32_t value) { + unsigned long index; + _BitScanReverse(&index, value); + return 31 - index; +} +#endif /* _MSC_VER */ - /* is the environment open? - * (https://libmdbx.dqdkfa.ru/dead-github/issues/171) */ - if (unlikely(!env->me_map)) { - /* environment not yet opened */ -#if 1 - /* default behavior: returns the available info but zeroed the rest */ - memset(arg, 0, bytes); - arg->mi_geo.lower = env->me_dbgeo.lower; - arg->mi_geo.upper = env->me_dbgeo.upper; - arg->mi_geo.shrink = env->me_dbgeo.shrink; - arg->mi_geo.grow = env->me_dbgeo.grow; - arg->mi_geo.current = env->me_dbgeo.now; - arg->mi_maxreaders = env->me_maxreaders; - arg->mi_dxb_pagesize = env->me_psize; - arg->mi_sys_pagesize = env->me_os_psize; - if (likely(bytes > size_before_bootid)) { - arg->mi_bootid.current.x = bootid.x; - arg->mi_bootid.current.y = bootid.y; - } - return MDBX_SUCCESS; +#if defined(_MSC_VER) && !defined(__builtin_clzl) && !__has_builtin(__builtin_clzl) +MDBX_MAYBE_UNUSED static __always_inline size_t __builtin_clzl(size_t value) { + unsigned long index; +#ifdef _WIN64 + assert(sizeof(value) == 8); + _BitScanReverse64(&index, value); + return 63 - index; #else - /* some users may prefer this behavior: return appropriate error */ - return MDBX_EPERM; + assert(sizeof(value) == 4); + _BitScanReverse(&index, value); + return 31 - index; #endif - } - - const MDBX_meta *const meta0 = METAPAGE(env, 0); - const MDBX_meta *const meta1 = METAPAGE(env, 1); - const MDBX_meta *const meta2 = METAPAGE(env, 2); - if (unlikely(env->me_flags & MDBX_FATAL_ERROR)) - return MDBX_PANIC; +} +#endif /* _MSC_VER */ - meta_troika_t holder; - meta_troika_t const *troika; - if (txn && !(txn->mt_flags & MDBX_TXN_RDONLY)) - troika = &txn->tw.troika; - else { - holder = meta_tap(env); - troika = &holder; - } +#if !MDBX_PNL_ASCENDING - const meta_ptr_t head = meta_recent(env, troika); - arg->mi_recent_txnid = head.txnid; - arg->mi_meta0_txnid = troika->txnid[0]; - arg->mi_meta0_sign = unaligned_peek_u64(4, meta0->mm_sign); - arg->mi_meta1_txnid = troika->txnid[1]; - arg->mi_meta1_sign = unaligned_peek_u64(4, meta1->mm_sign); - arg->mi_meta2_txnid = troika->txnid[2]; - arg->mi_meta2_sign = unaligned_peek_u64(4, meta2->mm_sign); - if (likely(bytes > size_before_bootid)) { - memcpy(&arg->mi_bootid.meta0, &meta0->mm_bootid, 16); - memcpy(&arg->mi_bootid.meta1, &meta1->mm_bootid, 16); - memcpy(&arg->mi_bootid.meta2, &meta2->mm_bootid, 16); - } +#if !defined(MDBX_ATTRIBUTE_TARGET) && (__has_attribute(__target__) || __GNUC_PREREQ(5, 0)) +#define MDBX_ATTRIBUTE_TARGET(target) __attribute__((__target__(target))) +#endif /* MDBX_ATTRIBUTE_TARGET */ - const volatile MDBX_meta *txn_meta = head.ptr_v; - arg->mi_last_pgno = txn_meta->mm_geo.next - 1; - arg->mi_geo.current = pgno2bytes(env, txn_meta->mm_geo.now); - if (txn) { - arg->mi_last_pgno = txn->mt_next_pgno - 1; - arg->mi_geo.current = pgno2bytes(env, txn->mt_end_pgno); - - const txnid_t wanna_meta_txnid = (txn->mt_flags & MDBX_TXN_RDONLY) - ? txn->mt_txnid - : txn->mt_txnid - xMDBX_TXNID_STEP; - txn_meta = (arg->mi_meta0_txnid == wanna_meta_txnid) ? meta0 : txn_meta; - txn_meta = (arg->mi_meta1_txnid == wanna_meta_txnid) ? meta1 : txn_meta; - txn_meta = (arg->mi_meta2_txnid == wanna_meta_txnid) ? meta2 : txn_meta; - } - arg->mi_geo.lower = pgno2bytes(env, txn_meta->mm_geo.lower); - arg->mi_geo.upper = pgno2bytes(env, txn_meta->mm_geo.upper); - arg->mi_geo.shrink = pgno2bytes(env, pv2pages(txn_meta->mm_geo.shrink_pv)); - arg->mi_geo.grow = pgno2bytes(env, pv2pages(txn_meta->mm_geo.grow_pv)); - const uint64_t unsynced_pages = - atomic_load64(&env->me_lck->mti_unsynced_pages, mo_Relaxed) + - (atomic_load32(&env->me_lck->mti_meta_sync_txnid, mo_Relaxed) != - (uint32_t)arg->mi_recent_txnid); - - arg->mi_mapsize = env->me_dxb_mmap.limit; - - const MDBX_lockinfo *const lck = env->me_lck; - arg->mi_maxreaders = env->me_maxreaders; - arg->mi_numreaders = env->me_lck_mmap.lck - ? atomic_load32(&lck->mti_numreaders, mo_Relaxed) - : INT32_MAX; - arg->mi_dxb_pagesize = env->me_psize; - arg->mi_sys_pagesize = env->me_os_psize; +#ifndef MDBX_GCC_FASTMATH_i686_SIMD_WORKAROUND +/* Workaround for GCC's bug with `-m32 -march=i686 -Ofast` + * gcc/i686-buildroot-linux-gnu/12.2.0/include/xmmintrin.h:814:1: + * error: inlining failed in call to 'always_inline' '_mm_movemask_ps': + * target specific option mismatch */ +#if !defined(__FAST_MATH__) || !__FAST_MATH__ || !defined(__GNUC__) || defined(__e2k__) || defined(__clang__) || \ + defined(__amd64__) || defined(__SSE2__) +#define MDBX_GCC_FASTMATH_i686_SIMD_WORKAROUND 0 +#else +#define MDBX_GCC_FASTMATH_i686_SIMD_WORKAROUND 1 +#endif +#endif /* MDBX_GCC_FASTMATH_i686_SIMD_WORKAROUND */ - if (likely(bytes > size_before_bootid)) { - arg->mi_unsync_volume = pgno2bytes(env, (size_t)unsynced_pages); - const uint64_t monotime_now = osal_monotime(); - uint64_t ts = atomic_load64(&lck->mti_eoos_timestamp, mo_Relaxed); - arg->mi_since_sync_seconds16dot16 = - ts ? osal_monotime_to_16dot16_noUnderflow(monotime_now - ts) : 0; - ts = atomic_load64(&lck->mti_reader_check_timestamp, mo_Relaxed); - arg->mi_since_reader_check_seconds16dot16 = - ts ? osal_monotime_to_16dot16_noUnderflow(monotime_now - ts) : 0; - arg->mi_autosync_threshold = pgno2bytes( - env, atomic_load32(&lck->mti_autosync_threshold, mo_Relaxed)); - arg->mi_autosync_period_seconds16dot16 = - osal_monotime_to_16dot16_noUnderflow( - atomic_load64(&lck->mti_autosync_period, mo_Relaxed)); - arg->mi_bootid.current.x = bootid.x; - arg->mi_bootid.current.y = bootid.y; - arg->mi_mode = env->me_lck_mmap.lck ? lck->mti_envmode.weak : env->me_flags; - } +#if defined(__SSE2__) && defined(__SSE__) +#define MDBX_ATTRIBUTE_TARGET_SSE2 /* nope */ +#elif (defined(_M_IX86_FP) && _M_IX86_FP >= 2) || defined(__amd64__) +#define __SSE2__ +#define MDBX_ATTRIBUTE_TARGET_SSE2 /* nope */ +#elif defined(MDBX_ATTRIBUTE_TARGET) && defined(__ia32__) && !MDBX_GCC_FASTMATH_i686_SIMD_WORKAROUND +#define MDBX_ATTRIBUTE_TARGET_SSE2 MDBX_ATTRIBUTE_TARGET("sse,sse2") +#endif /* __SSE2__ */ - if (likely(bytes > size_before_pgop_stat)) { -#if MDBX_ENABLE_PGOP_STAT - arg->mi_pgop_stat.newly = - atomic_load64(&lck->mti_pgop_stat.newly, mo_Relaxed); - arg->mi_pgop_stat.cow = atomic_load64(&lck->mti_pgop_stat.cow, mo_Relaxed); - arg->mi_pgop_stat.clone = - atomic_load64(&lck->mti_pgop_stat.clone, mo_Relaxed); - arg->mi_pgop_stat.split = - atomic_load64(&lck->mti_pgop_stat.split, mo_Relaxed); - arg->mi_pgop_stat.merge = - atomic_load64(&lck->mti_pgop_stat.merge, mo_Relaxed); - arg->mi_pgop_stat.spill = - atomic_load64(&lck->mti_pgop_stat.spill, mo_Relaxed); - arg->mi_pgop_stat.unspill = - atomic_load64(&lck->mti_pgop_stat.unspill, mo_Relaxed); - arg->mi_pgop_stat.wops = - atomic_load64(&lck->mti_pgop_stat.wops, mo_Relaxed); - arg->mi_pgop_stat.prefault = - atomic_load64(&lck->mti_pgop_stat.prefault, mo_Relaxed); - arg->mi_pgop_stat.mincore = - atomic_load64(&lck->mti_pgop_stat.mincore, mo_Relaxed); - arg->mi_pgop_stat.msync = - atomic_load64(&lck->mti_pgop_stat.msync, mo_Relaxed); - arg->mi_pgop_stat.fsync = - atomic_load64(&lck->mti_pgop_stat.fsync, mo_Relaxed); -#else - memset(&arg->mi_pgop_stat, 0, sizeof(arg->mi_pgop_stat)); -#endif /* MDBX_ENABLE_PGOP_STAT*/ - } +#if defined(__AVX2__) +#define MDBX_ATTRIBUTE_TARGET_AVX2 /* nope */ +#elif defined(MDBX_ATTRIBUTE_TARGET) && defined(__ia32__) && !MDBX_GCC_FASTMATH_i686_SIMD_WORKAROUND +#define MDBX_ATTRIBUTE_TARGET_AVX2 MDBX_ATTRIBUTE_TARGET("sse,sse2,avx,avx2") +#endif /* __AVX2__ */ - arg->mi_self_latter_reader_txnid = arg->mi_latter_reader_txnid = - arg->mi_recent_txnid; - if (env->me_lck_mmap.lck) { - for (size_t i = 0; i < arg->mi_numreaders; ++i) { - const uint32_t pid = - atomic_load32(&lck->mti_readers[i].mr_pid, mo_AcquireRelease); - if (pid) { - const txnid_t txnid = safe64_read(&lck->mti_readers[i].mr_txnid); - if (arg->mi_latter_reader_txnid > txnid) - arg->mi_latter_reader_txnid = txnid; - if (pid == env->me_pid && arg->mi_self_latter_reader_txnid > txnid) - arg->mi_self_latter_reader_txnid = txnid; - } - } - } +#if defined(MDBX_ATTRIBUTE_TARGET_AVX2) +#if defined(__AVX512BW__) +#define MDBX_ATTRIBUTE_TARGET_AVX512BW /* nope */ +#elif defined(MDBX_ATTRIBUTE_TARGET) && defined(__ia32__) && !MDBX_GCC_FASTMATH_i686_SIMD_WORKAROUND && \ + (__GNUC_PREREQ(6, 0) || __CLANG_PREREQ(5, 0)) +#define MDBX_ATTRIBUTE_TARGET_AVX512BW MDBX_ATTRIBUTE_TARGET("sse,sse2,avx,avx2,avx512bw") +#endif /* __AVX512BW__ */ +#endif /* MDBX_ATTRIBUTE_TARGET_AVX2 for MDBX_ATTRIBUTE_TARGET_AVX512BW */ - osal_compiler_barrier(); - return MDBX_SUCCESS; +#ifdef MDBX_ATTRIBUTE_TARGET_SSE2 +MDBX_ATTRIBUTE_TARGET_SSE2 static __always_inline unsigned +diffcmp2mask_sse2(const pgno_t *const ptr, const ptrdiff_t offset, const __m128i pattern) { + const __m128i f = _mm_loadu_si128((const __m128i *)ptr); + const __m128i l = _mm_loadu_si128((const __m128i *)(ptr + offset)); + const __m128i cmp = _mm_cmpeq_epi32(_mm_sub_epi32(f, l), pattern); + return _mm_movemask_ps(*(const __m128 *)&cmp); } -__cold int mdbx_env_info_ex(const MDBX_env *env, const MDBX_txn *txn, - MDBX_envinfo *arg, size_t bytes) { - if (unlikely((env == NULL && txn == NULL) || arg == NULL)) - return MDBX_EINVAL; - - if (txn) { - int err = check_txn(txn, MDBX_TXN_BLOCKED - MDBX_TXN_ERROR); - if (unlikely(err != MDBX_SUCCESS)) - return err; - } - if (env) { - int err = check_env(env, false); - if (unlikely(err != MDBX_SUCCESS)) - return err; - if (txn && unlikely(txn->mt_env != env)) - return MDBX_EINVAL; - } else { - env = txn->mt_env; +MDBX_MAYBE_UNUSED __hot MDBX_ATTRIBUTE_TARGET_SSE2 static pgno_t *scan4seq_sse2(pgno_t *range, const size_t len, + const size_t seq) { + assert(seq > 0 && len > seq); +#if MDBX_PNL_ASCENDING +#error "FIXME: Not implemented" +#endif /* MDBX_PNL_ASCENDING */ + assert(range[-(ptrdiff_t)len] == len); + pgno_t *const detent = range - len + seq; + const ptrdiff_t offset = -(ptrdiff_t)seq; + const pgno_t target = (pgno_t)offset; + const __m128i pattern = _mm_set1_epi32(target); + uint8_t mask; + if (likely(len > seq + 3)) { + do { + mask = (uint8_t)diffcmp2mask_sse2(range - 3, offset, pattern); + if (mask) { +#if !defined(ENABLE_MEMCHECK) && !defined(__SANITIZE_ADDRESS__) + found: +#endif /* !ENABLE_MEMCHECK && !__SANITIZE_ADDRESS__ */ + return range + 28 - __builtin_clz(mask); + } + range -= 4; + } while (range > detent + 3); + if (range == detent) + return nullptr; } - const size_t size_before_bootid = offsetof(MDBX_envinfo, mi_bootid); - const size_t size_before_pgop_stat = offsetof(MDBX_envinfo, mi_pgop_stat); - if (unlikely(bytes != sizeof(MDBX_envinfo)) && bytes != size_before_bootid && - bytes != size_before_pgop_stat) - return MDBX_EINVAL; - - MDBX_envinfo snap; - int rc = fetch_envinfo_ex(env, txn, &snap, sizeof(snap)); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - - while (1) { - rc = fetch_envinfo_ex(env, txn, arg, bytes); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - snap.mi_since_sync_seconds16dot16 = arg->mi_since_sync_seconds16dot16; - snap.mi_since_reader_check_seconds16dot16 = - arg->mi_since_reader_check_seconds16dot16; - if (likely(memcmp(&snap, arg, bytes) == 0)) - return MDBX_SUCCESS; - memcpy(&snap, arg, bytes); + /* Далее происходит чтение от 4 до 12 лишних байт, которые могут быть не + * только за пределами региона выделенного под PNL, но и пересекать границу + * страницы памяти. Что может приводить как к ошибкам ASAN, так и к падению. + * Поэтому проверяем смещение на странице, а с ASAN всегда страхуемся. */ +#if !defined(ENABLE_MEMCHECK) && !defined(__SANITIZE_ADDRESS__) + const unsigned on_page_safe_mask = 0xff0 /* enough for '-15' bytes offset */; + if (likely(on_page_safe_mask & (uintptr_t)(range + offset)) && !RUNNING_ON_VALGRIND) { + const unsigned extra = (unsigned)(detent + 4 - range); + assert(extra > 0 && extra < 4); + mask = 0xF << extra; + mask &= diffcmp2mask_sse2(range - 3, offset, pattern); + if (mask) + goto found; + return nullptr; } +#endif /* !ENABLE_MEMCHECK && !__SANITIZE_ADDRESS__ */ + do + if (*range - range[offset] == target) + return range; + while (--range != detent); + return nullptr; } +#endif /* MDBX_ATTRIBUTE_TARGET_SSE2 */ -static __inline MDBX_cmp_func *get_default_keycmp(MDBX_db_flags_t flags) { - return (flags & MDBX_REVERSEKEY) ? cmp_reverse - : (flags & MDBX_INTEGERKEY) ? cmp_int_align2 - : cmp_lexical; +#ifdef MDBX_ATTRIBUTE_TARGET_AVX2 +MDBX_ATTRIBUTE_TARGET_AVX2 static __always_inline unsigned +diffcmp2mask_avx2(const pgno_t *const ptr, const ptrdiff_t offset, const __m256i pattern) { + const __m256i f = _mm256_loadu_si256((const __m256i *)ptr); + const __m256i l = _mm256_loadu_si256((const __m256i *)(ptr + offset)); + const __m256i cmp = _mm256_cmpeq_epi32(_mm256_sub_epi32(f, l), pattern); + return _mm256_movemask_ps(*(const __m256 *)&cmp); } -static __inline MDBX_cmp_func *get_default_datacmp(MDBX_db_flags_t flags) { - return !(flags & MDBX_DUPSORT) - ? cmp_lenfast - : ((flags & MDBX_INTEGERDUP) - ? cmp_int_unaligned - : ((flags & MDBX_REVERSEDUP) ? cmp_reverse : cmp_lexical)); -} - -static int dbi_bind(MDBX_txn *txn, const MDBX_dbi dbi, unsigned user_flags, - MDBX_cmp_func *keycmp, MDBX_cmp_func *datacmp) { - /* Accepting only three cases: - * 1) user_flags and both comparators are zero - * = assume that a by-default mode/flags is requested for reading; - * 2) user_flags exactly the same - * = assume that the target mode/flags are requested properly; - * 3) user_flags differs, but table is empty and MDBX_CREATE is provided - * = assume that a properly create request with custom flags; - */ - if ((user_flags ^ txn->mt_dbs[dbi].md_flags) & DB_PERSISTENT_FLAGS) { - /* flags are differs, check other conditions */ - if ((!user_flags && (!keycmp || keycmp == txn->mt_dbxs[dbi].md_cmp) && - (!datacmp || datacmp == txn->mt_dbxs[dbi].md_dcmp)) || - user_flags == MDBX_ACCEDE) { - /* no comparators were provided and flags are zero, - * seems that is case #1 above */ - user_flags = txn->mt_dbs[dbi].md_flags; - } else if ((user_flags & MDBX_CREATE) && txn->mt_dbs[dbi].md_entries == 0) { - if (txn->mt_flags & MDBX_TXN_RDONLY) - return /* FIXME: return extended info */ MDBX_EACCESS; - /* make sure flags changes get committed */ - txn->mt_dbs[dbi].md_flags = user_flags & DB_PERSISTENT_FLAGS; - txn->mt_flags |= MDBX_TXN_DIRTY; - /* обнуляем компараторы для установки в соответствии с флагами, - * либо заданных пользователем */ - txn->mt_dbxs[dbi].md_cmp = nullptr; - txn->mt_dbxs[dbi].md_dcmp = nullptr; - } else { - return /* FIXME: return extended info */ MDBX_INCOMPATIBLE; - } - } +MDBX_ATTRIBUTE_TARGET_AVX2 static __always_inline unsigned +diffcmp2mask_sse2avx(const pgno_t *const ptr, const ptrdiff_t offset, const __m128i pattern) { + const __m128i f = _mm_loadu_si128((const __m128i *)ptr); + const __m128i l = _mm_loadu_si128((const __m128i *)(ptr + offset)); + const __m128i cmp = _mm_cmpeq_epi32(_mm_sub_epi32(f, l), pattern); + return _mm_movemask_ps(*(const __m128 *)&cmp); +} - if (!keycmp) - keycmp = txn->mt_dbxs[dbi].md_cmp ? txn->mt_dbxs[dbi].md_cmp - : get_default_keycmp(user_flags); - if (txn->mt_dbxs[dbi].md_cmp != keycmp) { - if (txn->mt_dbxs[dbi].md_cmp) - return MDBX_EINVAL; - txn->mt_dbxs[dbi].md_cmp = keycmp; +MDBX_MAYBE_UNUSED __hot MDBX_ATTRIBUTE_TARGET_AVX2 static pgno_t *scan4seq_avx2(pgno_t *range, const size_t len, + const size_t seq) { + assert(seq > 0 && len > seq); +#if MDBX_PNL_ASCENDING +#error "FIXME: Not implemented" +#endif /* MDBX_PNL_ASCENDING */ + assert(range[-(ptrdiff_t)len] == len); + pgno_t *const detent = range - len + seq; + const ptrdiff_t offset = -(ptrdiff_t)seq; + const pgno_t target = (pgno_t)offset; + const __m256i pattern = _mm256_set1_epi32(target); + uint8_t mask; + if (likely(len > seq + 7)) { + do { + mask = (uint8_t)diffcmp2mask_avx2(range - 7, offset, pattern); + if (mask) { +#if !defined(ENABLE_MEMCHECK) && !defined(__SANITIZE_ADDRESS__) + found: +#endif /* !ENABLE_MEMCHECK && !__SANITIZE_ADDRESS__ */ + return range + 24 - __builtin_clz(mask); + } + range -= 8; + } while (range > detent + 7); + if (range == detent) + return nullptr; } - if (!datacmp) - datacmp = txn->mt_dbxs[dbi].md_dcmp ? txn->mt_dbxs[dbi].md_dcmp - : get_default_datacmp(user_flags); - if (txn->mt_dbxs[dbi].md_dcmp != datacmp) { - if (txn->mt_dbxs[dbi].md_dcmp) - return MDBX_EINVAL; - txn->mt_dbxs[dbi].md_dcmp = datacmp; + /* Далее происходит чтение от 4 до 28 лишних байт, которые могут быть не + * только за пределами региона выделенного под PNL, но и пересекать границу + * страницы памяти. Что может приводить как к ошибкам ASAN, так и к падению. + * Поэтому проверяем смещение на странице, а с ASAN всегда страхуемся. */ +#if !defined(ENABLE_MEMCHECK) && !defined(__SANITIZE_ADDRESS__) + const unsigned on_page_safe_mask = 0xfe0 /* enough for '-31' bytes offset */; + if (likely(on_page_safe_mask & (uintptr_t)(range + offset)) && !RUNNING_ON_VALGRIND) { + const unsigned extra = (unsigned)(detent + 8 - range); + assert(extra > 0 && extra < 8); + mask = 0xFF << extra; + mask &= diffcmp2mask_avx2(range - 7, offset, pattern); + if (mask) + goto found; + return nullptr; } - - return MDBX_SUCCESS; -} - -static int dbi_open(MDBX_txn *txn, const MDBX_val *const table_name, - unsigned user_flags, MDBX_dbi *dbi, MDBX_cmp_func *keycmp, - MDBX_cmp_func *datacmp) { - int rc = MDBX_EINVAL; - if (unlikely(!dbi)) - return rc; - - void *clone = nullptr; - bool locked = false; - if (unlikely((user_flags & ~DB_USABLE_FLAGS) != 0)) { - bailout: - tASSERT(txn, MDBX_IS_ERROR(rc)); - *dbi = 0; - if (locked) - ENSURE(txn->mt_env, - osal_fastmutex_release(&txn->mt_env->me_dbi_lock) == MDBX_SUCCESS); - osal_free(clone); - return rc; +#endif /* !ENABLE_MEMCHECK && !__SANITIZE_ADDRESS__ */ + if (range - 3 > detent) { + mask = diffcmp2mask_sse2avx(range - 3, offset, *(const __m128i *)&pattern); + if (mask) + return range + 28 - __builtin_clz(mask); + range -= 4; } - - rc = check_txn(txn, MDBX_TXN_BLOCKED); - if (unlikely(rc != MDBX_SUCCESS)) - goto bailout; - - if ((user_flags & MDBX_CREATE) && unlikely(txn->mt_flags & MDBX_TXN_RDONLY)) { - rc = MDBX_EACCESS; - goto bailout; + while (range > detent) { + if (*range - range[offset] == target) + return range; + --range; } + return nullptr; +} +#endif /* MDBX_ATTRIBUTE_TARGET_AVX2 */ - switch (user_flags & (MDBX_INTEGERDUP | MDBX_DUPFIXED | MDBX_DUPSORT | - MDBX_REVERSEDUP | MDBX_ACCEDE)) { - case MDBX_ACCEDE: - if ((user_flags & MDBX_CREATE) == 0) - break; - __fallthrough /* fall through */; - default: - rc = MDBX_EINVAL; - goto bailout; +#ifdef MDBX_ATTRIBUTE_TARGET_AVX512BW +MDBX_ATTRIBUTE_TARGET_AVX512BW static __always_inline unsigned +diffcmp2mask_avx512bw(const pgno_t *const ptr, const ptrdiff_t offset, const __m512i pattern) { + const __m512i f = _mm512_loadu_si512((const __m512i *)ptr); + const __m512i l = _mm512_loadu_si512((const __m512i *)(ptr + offset)); + return _mm512_cmpeq_epi32_mask(_mm512_sub_epi32(f, l), pattern); +} - case MDBX_DUPSORT: - case MDBX_DUPSORT | MDBX_REVERSEDUP: - case MDBX_DUPSORT | MDBX_DUPFIXED: - case MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_REVERSEDUP: - case MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERDUP: - case MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERDUP | MDBX_REVERSEDUP: - case 0: - break; +MDBX_MAYBE_UNUSED __hot MDBX_ATTRIBUTE_TARGET_AVX512BW static pgno_t *scan4seq_avx512bw(pgno_t *range, const size_t len, + const size_t seq) { + assert(seq > 0 && len > seq); +#if MDBX_PNL_ASCENDING +#error "FIXME: Not implemented" +#endif /* MDBX_PNL_ASCENDING */ + assert(range[-(ptrdiff_t)len] == len); + pgno_t *const detent = range - len + seq; + const ptrdiff_t offset = -(ptrdiff_t)seq; + const pgno_t target = (pgno_t)offset; + const __m512i pattern = _mm512_set1_epi32(target); + unsigned mask; + if (likely(len > seq + 15)) { + do { + mask = diffcmp2mask_avx512bw(range - 15, offset, pattern); + if (mask) { +#if !defined(ENABLE_MEMCHECK) && !defined(__SANITIZE_ADDRESS__) + found: +#endif /* !ENABLE_MEMCHECK && !__SANITIZE_ADDRESS__ */ + return range + 16 - __builtin_clz(mask); + } + range -= 16; + } while (range > detent + 15); + if (range == detent) + return nullptr; } - /* main table? */ - if (table_name == MDBX_PGWALK_MAIN || - table_name->iov_base == MDBX_PGWALK_MAIN) { - rc = dbi_bind(txn, MAIN_DBI, user_flags, keycmp, datacmp); - if (unlikely(rc != MDBX_SUCCESS)) - goto bailout; - *dbi = MAIN_DBI; - return rc; + /* Далее происходит чтение от 4 до 60 лишних байт, которые могут быть не + * только за пределами региона выделенного под PNL, но и пересекать границу + * страницы памяти. Что может приводить как к ошибкам ASAN, так и к падению. + * Поэтому проверяем смещение на странице, а с ASAN всегда страхуемся. */ +#if !defined(ENABLE_MEMCHECK) && !defined(__SANITIZE_ADDRESS__) + const unsigned on_page_safe_mask = 0xfc0 /* enough for '-63' bytes offset */; + if (likely(on_page_safe_mask & (uintptr_t)(range + offset)) && !RUNNING_ON_VALGRIND) { + const unsigned extra = (unsigned)(detent + 16 - range); + assert(extra > 0 && extra < 16); + mask = 0xFFFF << extra; + mask &= diffcmp2mask_avx512bw(range - 15, offset, pattern); + if (mask) + goto found; + return nullptr; } - if (table_name == MDBX_PGWALK_GC || table_name->iov_base == MDBX_PGWALK_GC) { - rc = dbi_bind(txn, FREE_DBI, user_flags, keycmp, datacmp); - if (unlikely(rc != MDBX_SUCCESS)) - goto bailout; - *dbi = FREE_DBI; - return rc; +#endif /* !ENABLE_MEMCHECK && !__SANITIZE_ADDRESS__ */ + if (range - 7 > detent) { + mask = diffcmp2mask_avx2(range - 7, offset, *(const __m256i *)&pattern); + if (mask) + return range + 24 - __builtin_clz(mask); + range -= 8; } - if (table_name == MDBX_PGWALK_META || - table_name->iov_base == MDBX_PGWALK_META) { - rc = MDBX_EINVAL; - goto bailout; + if (range - 3 > detent) { + mask = diffcmp2mask_sse2avx(range - 3, offset, *(const __m128i *)&pattern); + if (mask) + return range + 28 - __builtin_clz(mask); + range -= 4; } - - MDBX_val key = *table_name; - MDBX_env *const env = txn->mt_env; - if (key.iov_len > env->me_leaf_nodemax - NODESIZE - sizeof(MDBX_db)) - return MDBX_EINVAL; - - /* Cannot mix named table(s) with DUPSORT flags */ - if (unlikely(txn->mt_dbs[MAIN_DBI].md_flags & MDBX_DUPSORT)) { - if ((user_flags & MDBX_CREATE) == 0) { - rc = MDBX_NOTFOUND; - goto bailout; - } - if (txn->mt_dbs[MAIN_DBI].md_leaf_pages || txn->mt_dbxs[MAIN_DBI].md_cmp) { - /* В MAIN_DBI есть записи либо она уже использовалась. */ - rc = MDBX_INCOMPATIBLE; - goto bailout; - } - /* Пересоздаём MAIN_DBI если там пусто. */ - atomic_store32(&txn->mt_dbiseqs[MAIN_DBI], dbi_seq(env, MAIN_DBI), - mo_AcquireRelease); - tASSERT(txn, txn->mt_dbs[MAIN_DBI].md_depth == 0 && - txn->mt_dbs[MAIN_DBI].md_entries == 0 && - txn->mt_dbs[MAIN_DBI].md_root == P_INVALID); - txn->mt_dbs[MAIN_DBI].md_flags &= MDBX_REVERSEKEY | MDBX_INTEGERKEY; - txn->mt_dbistate[MAIN_DBI] |= DBI_DIRTY; - txn->mt_flags |= MDBX_TXN_DIRTY; - txn->mt_dbxs[MAIN_DBI].md_cmp = - get_default_keycmp(txn->mt_dbs[MAIN_DBI].md_flags); - txn->mt_dbxs[MAIN_DBI].md_dcmp = - get_default_datacmp(txn->mt_dbs[MAIN_DBI].md_flags); + while (range > detent) { + if (*range - range[offset] == target) + return range; + --range; } + return nullptr; +} +#endif /* MDBX_ATTRIBUTE_TARGET_AVX512BW */ - tASSERT(txn, txn->mt_dbxs[MAIN_DBI].md_cmp); +#if (defined(__ARM_NEON) || defined(__ARM_NEON__)) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) +static __always_inline size_t diffcmp2mask_neon(const pgno_t *const ptr, const ptrdiff_t offset, + const uint32x4_t pattern) { + const uint32x4_t f = vld1q_u32(ptr); + const uint32x4_t l = vld1q_u32(ptr + offset); + const uint16x4_t cmp = vmovn_u32(vceqq_u32(vsubq_u32(f, l), pattern)); + if (sizeof(size_t) > 7) + return vget_lane_u64(vreinterpret_u64_u16(cmp), 0); + else + return vget_lane_u32(vreinterpret_u32_u8(vmovn_u16(vcombine_u16(cmp, cmp))), 0); +} - /* Is the DB already open? */ - MDBX_dbi scan, slot; - for (slot = scan = txn->mt_numdbs; --scan >= CORE_DBS;) { - if (!txn->mt_dbxs[scan].md_name.iov_base) { - /* Remember this free slot */ - slot = scan; - continue; - } - if (key.iov_len == txn->mt_dbxs[scan].md_name.iov_len && - !memcmp(key.iov_base, txn->mt_dbxs[scan].md_name.iov_base, - key.iov_len)) { - rc = dbi_bind(txn, scan, user_flags, keycmp, datacmp); - if (unlikely(rc != MDBX_SUCCESS)) - goto bailout; - *dbi = scan; - return rc; - } +__hot static pgno_t *scan4seq_neon(pgno_t *range, const size_t len, const size_t seq) { + assert(seq > 0 && len > seq); +#if MDBX_PNL_ASCENDING +#error "FIXME: Not implemented" +#endif /* MDBX_PNL_ASCENDING */ + assert(range[-(ptrdiff_t)len] == len); + pgno_t *const detent = range - len + seq; + const ptrdiff_t offset = -(ptrdiff_t)seq; + const pgno_t target = (pgno_t)offset; + const uint32x4_t pattern = vmovq_n_u32(target); + size_t mask; + if (likely(len > seq + 3)) { + do { + mask = diffcmp2mask_neon(range - 3, offset, pattern); + if (mask) { +#if !defined(ENABLE_MEMCHECK) && !defined(__SANITIZE_ADDRESS__) + found: +#endif /* !ENABLE_MEMCHECK && !__SANITIZE_ADDRESS__ */ + return ptr_disp(range, -(__builtin_clzl(mask) >> sizeof(size_t) / 4)); + } + range -= 4; + } while (range > detent + 3); + if (range == detent) + return nullptr; } - /* Fail, if no free slot and max hit */ - if (unlikely(slot >= env->me_maxdbs)) { - rc = MDBX_DBS_FULL; - goto bailout; + /* Далее происходит чтение от 4 до 12 лишних байт, которые могут быть не + * только за пределами региона выделенного под PNL, но и пересекать границу + * страницы памяти. Что может приводить как к ошибкам ASAN, так и к падению. + * Поэтому проверяем смещение на странице, а с ASAN всегда страхуемся. */ +#if !defined(ENABLE_MEMCHECK) && !defined(__SANITIZE_ADDRESS__) + const unsigned on_page_safe_mask = 0xff0 /* enough for '-15' bytes offset */; + if (likely(on_page_safe_mask & (uintptr_t)(range + offset)) && !RUNNING_ON_VALGRIND) { + const unsigned extra = (unsigned)(detent + 4 - range); + assert(extra > 0 && extra < 4); + mask = (~(size_t)0) << (extra * sizeof(size_t) * 2); + mask &= diffcmp2mask_neon(range - 3, offset, pattern); + if (mask) + goto found; + return nullptr; } +#endif /* !ENABLE_MEMCHECK && !__SANITIZE_ADDRESS__ */ + do + if (*range - range[offset] == target) + return range; + while (--range != detent); + return nullptr; +} +#endif /* __ARM_NEON || __ARM_NEON__ */ - /* Find the DB info */ - MDBX_val data; - MDBX_cursor_couple couple; - rc = cursor_init(&couple.outer, txn, MAIN_DBI); - if (unlikely(rc != MDBX_SUCCESS)) - goto bailout; - rc = cursor_set(&couple.outer, &key, &data, MDBX_SET).err; - if (unlikely(rc != MDBX_SUCCESS)) { - if (rc != MDBX_NOTFOUND || !(user_flags & MDBX_CREATE)) - goto bailout; - } else { - /* make sure this is actually a table */ - MDBX_node *node = page_node(couple.outer.mc_pg[couple.outer.mc_top], - couple.outer.mc_ki[couple.outer.mc_top]); - if (unlikely((node_flags(node) & (F_DUPDATA | F_SUBDATA)) != F_SUBDATA)) { - rc = MDBX_INCOMPATIBLE; - goto bailout; - } - if (!MDBX_DISABLE_VALIDATION && unlikely(data.iov_len != sizeof(MDBX_db))) { - rc = MDBX_CORRUPTED; - goto bailout; - } - } +#if defined(__AVX512BW__) && defined(MDBX_ATTRIBUTE_TARGET_AVX512BW) +#define scan4seq_default scan4seq_avx512bw +#define scan4seq_impl scan4seq_default +#elif defined(__AVX2__) && defined(MDBX_ATTRIBUTE_TARGET_AVX2) +#define scan4seq_default scan4seq_avx2 +#elif defined(__SSE2__) && defined(MDBX_ATTRIBUTE_TARGET_SSE2) +#define scan4seq_default scan4seq_sse2 +#elif (defined(__ARM_NEON) || defined(__ARM_NEON__)) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) +#define scan4seq_default scan4seq_neon +/* Choosing of another variants should be added here. */ +#endif /* scan4seq_default */ - if (rc != MDBX_SUCCESS && unlikely(txn->mt_flags & MDBX_TXN_RDONLY)) { - rc = MDBX_EACCESS; - goto bailout; - } - - /* Done here so we cannot fail after creating a new DB */ - if (key.iov_len) { - clone = osal_malloc(key.iov_len); - if (unlikely(!clone)) { - rc = MDBX_ENOMEM; - goto bailout; - } - key.iov_base = memcpy(clone, key.iov_base, key.iov_len); - } else - key.iov_base = ""; - - int err = osal_fastmutex_acquire(&env->me_dbi_lock); - if (unlikely(err != MDBX_SUCCESS)) { - rc = err; - goto bailout; - } - locked = true; - - /* Import handles from env */ - dbi_import_locked(txn); - - /* Rescan after mutex acquisition & import handles */ - for (slot = scan = txn->mt_numdbs; --scan >= CORE_DBS;) { - if (!txn->mt_dbxs[scan].md_name.iov_base) { - /* Remember this free slot */ - slot = scan; - continue; - } - if (key.iov_len == txn->mt_dbxs[scan].md_name.iov_len && - !memcmp(key.iov_base, txn->mt_dbxs[scan].md_name.iov_base, - key.iov_len)) { - rc = dbi_bind(txn, scan, user_flags, keycmp, datacmp); - if (unlikely(rc != MDBX_SUCCESS)) - goto bailout; - slot = scan; - goto done; - } - } - - if (unlikely(slot >= env->me_maxdbs)) { - rc = MDBX_DBS_FULL; - goto bailout; - } - - unsigned dbiflags = DBI_FRESH | DBI_VALID | DBI_USRVALID; - MDBX_db db_dummy; - if (unlikely(rc)) { - /* MDBX_NOTFOUND and MDBX_CREATE: Create new DB */ - tASSERT(txn, rc == MDBX_NOTFOUND); - memset(&db_dummy, 0, sizeof(db_dummy)); - db_dummy.md_root = P_INVALID; - db_dummy.md_mod_txnid = txn->mt_txnid; - db_dummy.md_flags = user_flags & DB_PERSISTENT_FLAGS; - data.iov_len = sizeof(db_dummy); - data.iov_base = &db_dummy; - WITH_CURSOR_TRACKING( - couple.outer, rc = cursor_put_checklen(&couple.outer, &key, &data, - F_SUBDATA | MDBX_NOOVERWRITE)); - if (unlikely(rc != MDBX_SUCCESS)) - goto bailout; - - dbiflags |= DBI_DIRTY | DBI_CREAT; - txn->mt_flags |= MDBX_TXN_DIRTY; - tASSERT(txn, (txn->mt_dbistate[MAIN_DBI] & DBI_DIRTY) != 0); - } - - /* Got info, register DBI in this txn */ - memset(txn->mt_dbxs + slot, 0, sizeof(MDBX_dbx)); - memcpy(&txn->mt_dbs[slot], data.iov_base, sizeof(MDBX_db)); - env->me_dbflags[slot] = 0; - rc = dbi_bind(txn, slot, user_flags, keycmp, datacmp); - if (unlikely(rc != MDBX_SUCCESS)) { - tASSERT(txn, (dbiflags & DBI_CREAT) == 0); - goto bailout; - } - - txn->mt_dbistate[slot] = (uint8_t)dbiflags; - txn->mt_dbxs[slot].md_name = key; - txn->mt_dbiseqs[slot].weak = env->me_dbiseqs[slot].weak = dbi_seq(env, slot); - if (!(dbiflags & DBI_CREAT)) - env->me_dbflags[slot] = txn->mt_dbs[slot].md_flags | DB_VALID; - if (txn->mt_numdbs == slot) { - txn->mt_cursors[slot] = NULL; - osal_compiler_barrier(); - txn->mt_numdbs = slot + 1; - } - if (env->me_numdbs <= slot) { - osal_memory_fence(mo_AcquireRelease, true); - env->me_numdbs = slot + 1; - } - -done: - *dbi = slot; - ENSURE(env, osal_fastmutex_release(&env->me_dbi_lock) == MDBX_SUCCESS); - return MDBX_SUCCESS; -} - -static int dbi_open_cstr(MDBX_txn *txn, const char *name_cstr, - MDBX_db_flags_t flags, MDBX_dbi *dbi, - MDBX_cmp_func *keycmp, MDBX_cmp_func *datacmp) { - MDBX_val thunk, *name; - if (name_cstr == MDBX_PGWALK_MAIN || name_cstr == MDBX_PGWALK_GC || - name_cstr == MDBX_PGWALK_META) - name = (void *)name_cstr; - else { - thunk.iov_len = strlen(name_cstr); - thunk.iov_base = (void *)name_cstr; - name = &thunk; - } - return dbi_open(txn, name, flags, dbi, keycmp, datacmp); -} - -int mdbx_dbi_open(MDBX_txn *txn, const char *name, MDBX_db_flags_t flags, - MDBX_dbi *dbi) { - return dbi_open_cstr(txn, name, flags, dbi, nullptr, nullptr); -} +#endif /* MDBX_PNL_ASCENDING */ -int mdbx_dbi_open2(MDBX_txn *txn, const MDBX_val *name, MDBX_db_flags_t flags, - MDBX_dbi *dbi) { - return dbi_open(txn, name, flags, dbi, nullptr, nullptr); -} +#ifndef scan4seq_default +#define scan4seq_default scan4seq_fallback +#endif /* scan4seq_default */ -int mdbx_dbi_open_ex(MDBX_txn *txn, const char *name, MDBX_db_flags_t flags, - MDBX_dbi *dbi, MDBX_cmp_func *keycmp, - MDBX_cmp_func *datacmp) { - return dbi_open_cstr(txn, name, flags, dbi, keycmp, datacmp); -} +#ifdef scan4seq_impl +/* The scan4seq_impl() is the best or no alternatives */ +#elif !MDBX_HAVE_BUILTIN_CPU_SUPPORTS +/* The scan4seq_default() will be used since no cpu-features detection support + * from compiler. Please don't ask to implement cpuid-based detection and don't + * make such PRs. */ +#define scan4seq_impl scan4seq_default +#else +/* Selecting the most appropriate implementation at runtime, + * depending on the available CPU features. */ +static pgno_t *scan4seq_resolver(pgno_t *range, const size_t len, const size_t seq); +static pgno_t *(*scan4seq_impl)(pgno_t *range, const size_t len, const size_t seq) = scan4seq_resolver; -int mdbx_dbi_open_ex2(MDBX_txn *txn, const MDBX_val *name, - MDBX_db_flags_t flags, MDBX_dbi *dbi, - MDBX_cmp_func *keycmp, MDBX_cmp_func *datacmp) { - return dbi_open(txn, name, flags, dbi, keycmp, datacmp); +static pgno_t *scan4seq_resolver(pgno_t *range, const size_t len, const size_t seq) { + pgno_t *(*choice)(pgno_t *range, const size_t len, const size_t seq) = nullptr; +#if __has_builtin(__builtin_cpu_init) || defined(__BUILTIN_CPU_INIT__) || __GNUC_PREREQ(4, 8) + __builtin_cpu_init(); +#endif /* __builtin_cpu_init() */ +#ifdef MDBX_ATTRIBUTE_TARGET_SSE2 + if (__builtin_cpu_supports("sse2")) + choice = scan4seq_sse2; +#endif /* MDBX_ATTRIBUTE_TARGET_SSE2 */ +#ifdef MDBX_ATTRIBUTE_TARGET_AVX2 + if (__builtin_cpu_supports("avx2")) + choice = scan4seq_avx2; +#endif /* MDBX_ATTRIBUTE_TARGET_AVX2 */ +#ifdef MDBX_ATTRIBUTE_TARGET_AVX512BW + if (__builtin_cpu_supports("avx512bw")) + choice = scan4seq_avx512bw; +#endif /* MDBX_ATTRIBUTE_TARGET_AVX512BW */ + /* Choosing of another variants should be added here. */ + scan4seq_impl = choice ? choice : scan4seq_default; + return scan4seq_impl(range, len, seq); } +#endif /* scan4seq_impl */ -__cold int mdbx_dbi_stat(const MDBX_txn *txn, MDBX_dbi dbi, MDBX_stat *dest, - size_t bytes) { - int rc = check_txn(txn, MDBX_TXN_BLOCKED); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - - if (unlikely(!dest)) - return MDBX_EINVAL; - - if (unlikely(!check_dbi(txn, dbi, DBI_VALID))) - return MDBX_BAD_DBI; +/*----------------------------------------------------------------------------*/ - const size_t size_before_modtxnid = offsetof(MDBX_stat, ms_mod_txnid); - if (unlikely(bytes != sizeof(MDBX_stat)) && bytes != size_before_modtxnid) - return MDBX_EINVAL; +#define ALLOC_COALESCE 4 /* внутреннее состояние */ +#define ALLOC_SHOULD_SCAN 8 /* внутреннее состояние */ +#define ALLOC_LIFO 16 /* внутреннее состояние */ - if (unlikely(txn->mt_flags & MDBX_TXN_BLOCKED)) - return MDBX_BAD_TXN; +static inline bool is_gc_usable(MDBX_txn *txn, const MDBX_cursor *mc, const uint8_t flags) { + /* If txn is updating the GC, then the retired-list cannot play catch-up with + * itself by growing while trying to save it. */ + if (mc->tree == &txn->dbs[FREE_DBI] && !(flags & ALLOC_RESERVE) && !(mc->flags & z_gcu_preparation)) + return false; - if (unlikely(txn->mt_dbistate[dbi] & DBI_STALE)) { - rc = fetch_sdb((MDBX_txn *)txn, dbi); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; + /* avoid search inside empty tree and while tree is updating, + https://libmdbx.dqdkfa.ru/dead-github/issues/31 */ + if (unlikely(txn->dbs[FREE_DBI].items == 0)) { + txn->flags |= txn_gc_drained; + return false; } - dest->ms_psize = txn->mt_env->me_psize; - stat_get(&txn->mt_dbs[dbi], dest, bytes); - return MDBX_SUCCESS; + return true; } -static int dbi_close_locked(MDBX_env *env, MDBX_dbi dbi) { - eASSERT(env, dbi >= CORE_DBS); - if (unlikely(dbi >= env->me_numdbs)) - return MDBX_BAD_DBI; +static inline bool is_already_reclaimed(const MDBX_txn *txn, txnid_t id) { return txl_contain(txn->tw.gc.retxl, id); } - char *const ptr = env->me_dbxs[dbi].md_name.iov_base; - /* If there was no name, this was already closed */ - if (unlikely(!ptr)) - return MDBX_BAD_DBI; +__hot static pgno_t repnl_get_single(MDBX_txn *txn) { + const size_t len = MDBX_PNL_GETSIZE(txn->tw.repnl); + assert(len > 0); + pgno_t *target = MDBX_PNL_EDGE(txn->tw.repnl); + const ptrdiff_t dir = MDBX_PNL_ASCENDING ? 1 : -1; - env->me_dbflags[dbi] = 0; - env->me_dbxs[dbi].md_name.iov_len = 0; - osal_memory_fence(mo_AcquireRelease, true); - env->me_dbxs[dbi].md_name.iov_base = NULL; - osal_free(ptr); + /* Есть ТРИ потенциально выигрышные, но противо-направленные тактики: + * + * 1. Стараться использовать страницы с наименьшими номерами. Так обмен с + * диском будет более кучным, а у страниц ближе к концу БД будет больше шансов + * попасть под авто-компактификацию. Частично эта тактика уже реализована, но + * для её эффективности требуется явно приоритезировать выделение страниц: + * - поддерживать два repnl, для ближних и для дальних страниц; + * - использовать страницы из дальнего списка, если первый пуст, + * а второй слишком большой, либо при пустой GC. + * + * 2. Стараться выделять страницы последовательно. Так записываемые на диск + * регионы будут линейными, что принципиально ускоряет запись на HDD. + * Одновременно, в среднем это не повлияет на чтение, точнее говоря, если + * порядок чтения не совпадает с порядком изменения (иначе говоря, если + * чтение не коррелирует с обновлениями и/или вставками) то не повлияет, иначе + * может ускорить. Однако, последовательности в среднем достаточно редки. + * Поэтому для эффективности требуется аккумулировать и поддерживать в ОЗУ + * огромные списки страниц, а затем сохранять их обратно в БД. Текущий формат + * БД (без сжатых битовых карт) для этого крайне не удачен. Поэтому эта тактика не + * имеет шансов быть успешной без смены формата БД (Mithril). + * + * 3. Стараться экономить последовательности страниц. Это позволяет избегать + * лишнего чтения/поиска в GC при более-менее постоянном размещении и/или + * обновлении данных требующих более одной страницы. Проблема в том, что без + * информации от приложения библиотека не может знать насколько + * востребованными будут последовательности в ближайшей перспективе, а + * экономия последовательностей "на всякий случай" не только затратна + * сама-по-себе, но и работает во вред (добавляет хаоса). + * + * Поэтому: + * - в TODO добавляется разделение repnl на «ближние» и «дальние» страницы, + * с последующей реализацией первой тактики; + * - преимущественное использование последовательностей отправляется + * в MithrilDB как составляющая "HDD frendly" feature; + * - реализованная в 3757eb72f7c6b46862f8f17881ac88e8cecc1979 экономия + * последовательностей отключается через MDBX_ENABLE_SAVING_SEQUENCES=0. + * + * В качестве альтернативы для безусловной «экономии» последовательностей, + * в следующих версиях libmdbx, вероятно, будет предложено + * API для взаимодействия с GC: + * - получение размера GC, включая гистограммы размеров последовательностей + * и близости к концу БД; + * - включение формирования "линейного запаса" для последующего использования + * в рамках текущей транзакции; + * - намеренная загрузка GC в память для коагуляции и "выпрямления"; + * - намеренное копирование данных из страниц в конце БД для последующего + * из освобождения, т.е. контролируемая компактификация по запросу. */ - if (env->me_numdbs == dbi + 1) { - size_t i = env->me_numdbs; - do - --i; - while (i > CORE_DBS && !env->me_dbxs[i - 1].md_name.iov_base); - env->me_numdbs = (MDBX_dbi)i; +#ifndef MDBX_ENABLE_SAVING_SEQUENCES +#define MDBX_ENABLE_SAVING_SEQUENCES 0 +#endif + if (MDBX_ENABLE_SAVING_SEQUENCES && unlikely(target[dir] == *target + 1) && len > 2) { + /* Пытаемся пропускать последовательности при наличии одиночных элементов. + * TODO: необходимо кэшировать пропускаемые последовательности + * чтобы не сканировать список сначала при каждом выделении. */ + pgno_t *scan = target + dir + dir; + size_t left = len; + do { + if (likely(scan[-dir] != *scan - 1 && *scan + 1 != scan[dir])) { +#if MDBX_PNL_ASCENDING + target = scan; + break; +#else + /* вырезаем элемент с перемещением хвоста */ + const pgno_t pgno = *scan; + MDBX_PNL_SETSIZE(txn->tw.repnl, len - 1); + while (++scan <= target) + scan[-1] = *scan; + return pgno; +#endif + } + scan += dir; + } while (--left > 2); } - return MDBX_SUCCESS; + const pgno_t pgno = *target; +#if MDBX_PNL_ASCENDING + /* вырезаем элемент с перемещением хвоста */ + MDBX_PNL_SETSIZE(txn->tw.repnl, len - 1); + for (const pgno_t *const end = txn->tw.repnl + len - 1; target <= end; ++target) + *target = target[1]; +#else + /* перемещать хвост не нужно, просто усекам список */ + MDBX_PNL_SETSIZE(txn->tw.repnl, len - 1); +#endif + return pgno; } -int mdbx_dbi_close(MDBX_env *env, MDBX_dbi dbi) { - int rc = check_env(env, true); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - - if (unlikely(dbi < CORE_DBS)) - return (dbi == MAIN_DBI) ? MDBX_SUCCESS : MDBX_BAD_DBI; - - if (unlikely(dbi >= env->me_maxdbs)) - return MDBX_BAD_DBI; - - rc = osal_fastmutex_acquire(&env->me_dbi_lock); - if (likely(rc == MDBX_SUCCESS)) { - retry: - rc = MDBX_BAD_DBI; - if (dbi < env->me_maxdbs && (env->me_dbflags[dbi] & DB_VALID)) { - const MDBX_txn *const hazard = env->me_txn; - osal_compiler_barrier(); - if (env->me_txn0 && (env->me_txn0->mt_flags & MDBX_TXN_FINISHED) == 0) { - if (env->me_txn0->mt_dbistate[dbi] & (DBI_DIRTY | DBI_CREAT)) - goto bailout_dirty_dbi; - osal_memory_barrier(); - if (unlikely(hazard != env->me_txn)) - goto retry; - if (hazard != env->me_txn0 && hazard && - (hazard->mt_flags & MDBX_TXN_FINISHED) == 0 && - hazard->mt_signature == MDBX_MT_SIGNATURE && - (hazard->mt_dbistate[dbi] & (DBI_DIRTY | DBI_CREAT))) - goto bailout_dirty_dbi; - osal_compiler_barrier(); - if (unlikely(hazard != env->me_txn)) - goto retry; - } - rc = dbi_close_locked(env, dbi); - } - bailout_dirty_dbi: - ENSURE(env, osal_fastmutex_release(&env->me_dbi_lock) == MDBX_SUCCESS); +__hot static pgno_t repnl_get_sequence(MDBX_txn *txn, const size_t num, uint8_t flags) { + const size_t len = MDBX_PNL_GETSIZE(txn->tw.repnl); + pgno_t *edge = MDBX_PNL_EDGE(txn->tw.repnl); + assert(len >= num && num > 1); + const size_t seq = num - 1; +#if !MDBX_PNL_ASCENDING + if (edge[-(ptrdiff_t)seq] - *edge == seq) { + if (unlikely(flags & ALLOC_RESERVE)) + return P_INVALID; + assert(edge == scan4range_checker(txn->tw.repnl, seq)); + /* перемещать хвост не нужно, просто усекам список */ + MDBX_PNL_SETSIZE(txn->tw.repnl, len - num); + return *edge; } - return rc; +#endif + pgno_t *target = scan4seq_impl(edge, len, seq); + assert(target == scan4range_checker(txn->tw.repnl, seq)); + if (target) { + if (unlikely(flags & ALLOC_RESERVE)) + return P_INVALID; + const pgno_t pgno = *target; + /* вырезаем найденную последовательность с перемещением хвоста */ + MDBX_PNL_SETSIZE(txn->tw.repnl, len - num); +#if MDBX_PNL_ASCENDING + for (const pgno_t *const end = txn->tw.repnl + len - num; target <= end; ++target) + *target = target[num]; +#else + for (const pgno_t *const end = txn->tw.repnl + len; ++target <= end;) + target[-(ptrdiff_t)num] = *target; +#endif + return pgno; + } + return 0; } -int mdbx_dbi_flags_ex(const MDBX_txn *txn, MDBX_dbi dbi, unsigned *flags, - unsigned *state) { - int rc = check_txn(txn, MDBX_TXN_BLOCKED - MDBX_TXN_ERROR); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; +static inline pgr_t page_alloc_finalize(MDBX_env *const env, MDBX_txn *const txn, const MDBX_cursor *const mc, + const pgno_t pgno, const size_t num) { +#if MDBX_ENABLE_PROFGC + size_t majflt_before; + const uint64_t cputime_before = osal_cputime(&majflt_before); + gc_prof_stat_t *const prof = + (cursor_dbi(mc) == FREE_DBI) ? &env->lck->pgops.gc_prof.self : &env->lck->pgops.gc_prof.work; +#else + (void)mc; +#endif /* MDBX_ENABLE_PROFGC */ + ENSURE(env, pgno >= NUM_METAS); - if (unlikely(!flags || !state)) - return MDBX_EINVAL; - - if (unlikely(!check_dbi(txn, dbi, DBI_VALID))) - return MDBX_BAD_DBI; - - *flags = txn->mt_dbs[dbi].md_flags & DB_PERSISTENT_FLAGS; - *state = - txn->mt_dbistate[dbi] & (DBI_FRESH | DBI_CREAT | DBI_DIRTY | DBI_STALE); - - return MDBX_SUCCESS; -} - -static int drop_tree(MDBX_cursor *mc, const bool may_have_subDBs) { - int rc = page_search(mc, NULL, MDBX_PS_FIRST); - if (likely(rc == MDBX_SUCCESS)) { - MDBX_txn *txn = mc->mc_txn; - - /* DUPSORT sub-DBs have no ovpages/DBs. Omit scanning leaves. - * This also avoids any P_LEAF2 pages, which have no nodes. - * Also if the DB doesn't have sub-DBs and has no large/overflow - * pages, omit scanning leaves. */ - if (!(may_have_subDBs | mc->mc_db->md_overflow_pages)) - cursor_pop(mc); + pgr_t ret; + bool need_clean = (env->flags & MDBX_PAGEPERTURB) != 0; + if (env->flags & MDBX_WRITEMAP) { + ret.page = pgno2page(env, pgno); + MDBX_ASAN_UNPOISON_MEMORY_REGION(ret.page, pgno2bytes(env, num)); + VALGRIND_MAKE_MEM_UNDEFINED(ret.page, pgno2bytes(env, num)); - rc = pnl_need(&txn->tw.retired_pages, - (size_t)mc->mc_db->md_branch_pages + - (size_t)mc->mc_db->md_leaf_pages + - (size_t)mc->mc_db->md_overflow_pages); - if (unlikely(rc != MDBX_SUCCESS)) - goto bailout; + /* Содержимое выделенной страницы не нужно, но если страница отсутствует + * в ОЗУ (что весьма вероятно), то любое обращение к ней приведет + * к page-fault: + * - прерыванию по отсутствию страницы; + * - переключение контекста в режим ядра с засыпанием процесса; + * - чтение страницы с диска; + * - обновление PTE и пробуждением процесса; + * - переключение контекста по доступности ЦПУ. + * + * Пытаемся минимизировать накладные расходы записывая страницу, что при + * наличии unified page cache приведет к появлению страницы в ОЗУ без чтения + * с диска. При этом запись на диск должна быть отложена адекватным ядром, + * так как страница отображена в память в режиме чтения-записи и следом в + * неё пишет ЦПУ. */ - MDBX_cursor mx; - cursor_copy(mc, &mx); - while (mc->mc_snum > 0) { - MDBX_page *const mp = mc->mc_pg[mc->mc_top]; - const size_t nkeys = page_numkeys(mp); - if (IS_LEAF(mp)) { - cASSERT(mc, mc->mc_snum == mc->mc_db->md_depth); - for (size_t i = 0; i < nkeys; i++) { - MDBX_node *node = page_node(mp, i); - if (node_flags(node) & F_BIGDATA) { - rc = page_retire_ex(mc, node_largedata_pgno(node), nullptr, 0); - if (unlikely(rc != MDBX_SUCCESS)) - goto bailout; - if (!(may_have_subDBs | mc->mc_db->md_overflow_pages)) - goto pop; - } else if (node_flags(node) & F_SUBDATA) { - if (unlikely((node_flags(node) & F_DUPDATA) == 0)) { - rc = /* disallowing implicit subDB deletion */ MDBX_INCOMPATIBLE; - goto bailout; - } - rc = cursor_xinit1(mc, node, mp); - if (unlikely(rc != MDBX_SUCCESS)) - goto bailout; - rc = drop_tree(&mc->mc_xcursor->mx_cursor, false); - if (unlikely(rc != MDBX_SUCCESS)) - goto bailout; - } + /* В случае если страница в памяти процесса, то излишняя запись может быть + * достаточно дорогой. Кроме системного вызова и копирования данных, в особо + * одаренных ОС при этом могут включаться файловая система, выделяться + * временная страница, пополняться очереди асинхронного выполнения, + * обновляться PTE с последующей генерацией page-fault и чтением данных из + * грязной I/O очереди. Из-за этого штраф за лишнюю запись может быть + * сравним с избегаемым ненужным чтением. */ + if (txn->tw.prefault_write_activated) { + void *const pattern = ptr_disp(env->page_auxbuf, need_clean ? env->ps : env->ps * 2); + size_t file_offset = pgno2bytes(env, pgno); + if (likely(num == 1)) { + if (!mincore_probe(env, pgno)) { + osal_pwrite(env->lazy_fd, pattern, env->ps, file_offset); +#if MDBX_ENABLE_PGOP_STAT + env->lck->pgops.prefault.weak += 1; +#endif /* MDBX_ENABLE_PGOP_STAT */ + need_clean = false; } } else { - cASSERT(mc, mc->mc_snum < mc->mc_db->md_depth); - mc->mc_checking |= CC_RETIRING; - const unsigned pagetype = (IS_FROZEN(txn, mp) ? P_FROZEN : 0) + - ((mc->mc_snum + 1 == mc->mc_db->md_depth) - ? (mc->mc_checking & (P_LEAF | P_LEAF2)) - : P_BRANCH); - for (size_t i = 0; i < nkeys; i++) { - MDBX_node *node = page_node(mp, i); - tASSERT(txn, (node_flags(node) & - (F_BIGDATA | F_SUBDATA | F_DUPDATA)) == 0); - const pgno_t pgno = node_pgno(node); - rc = page_retire_ex(mc, pgno, nullptr, pagetype); - if (unlikely(rc != MDBX_SUCCESS)) - goto bailout; + struct iovec iov[MDBX_AUXILARY_IOV_MAX]; + size_t n = 0, cleared = 0; + for (size_t i = 0; i < num; ++i) { + if (!mincore_probe(env, pgno + (pgno_t)i)) { + ++cleared; + iov[n].iov_len = env->ps; + iov[n].iov_base = pattern; + if (unlikely(++n == MDBX_AUXILARY_IOV_MAX)) { + osal_pwritev(env->lazy_fd, iov, MDBX_AUXILARY_IOV_MAX, file_offset); +#if MDBX_ENABLE_PGOP_STAT + env->lck->pgops.prefault.weak += 1; +#endif /* MDBX_ENABLE_PGOP_STAT */ + file_offset += pgno2bytes(env, MDBX_AUXILARY_IOV_MAX); + n = 0; + } + } } - mc->mc_checking -= CC_RETIRING; - } - if (!mc->mc_top) - break; - cASSERT(mc, nkeys > 0); - mc->mc_ki[mc->mc_top] = (indx_t)nkeys; - rc = cursor_sibling(mc, SIBLING_RIGHT); - if (unlikely(rc != MDBX_SUCCESS)) { - if (unlikely(rc != MDBX_NOTFOUND)) - goto bailout; - /* no more siblings, go back to beginning - * of previous level. */ - pop: - cursor_pop(mc); - mc->mc_ki[0] = 0; - for (size_t i = 1; i < mc->mc_snum; i++) { - mc->mc_ki[i] = 0; - mc->mc_pg[i] = mx.mc_pg[i]; + if (likely(n > 0)) { + osal_pwritev(env->lazy_fd, iov, n, file_offset); +#if MDBX_ENABLE_PGOP_STAT + env->lck->pgops.prefault.weak += 1; +#endif /* MDBX_ENABLE_PGOP_STAT */ } + if (cleared == num) + need_clean = false; } } - rc = page_retire(mc, mc->mc_pg[0]); - bailout: - if (unlikely(rc != MDBX_SUCCESS)) - txn->mt_flags |= MDBX_TXN_ERROR; - } else if (rc == MDBX_NOTFOUND) { - rc = MDBX_SUCCESS; + } else { + ret.page = page_shadow_alloc(txn, num); + if (unlikely(!ret.page)) { + ret.err = MDBX_ENOMEM; + goto bailout; + } } - mc->mc_flags &= ~C_INITIALIZED; - return rc; -} -int mdbx_drop(MDBX_txn *txn, MDBX_dbi dbi, bool del) { - int rc = check_txn_rw(txn, MDBX_TXN_BLOCKED); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - - MDBX_cursor *mc; - rc = mdbx_cursor_open(txn, dbi, &mc); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - - rc = drop_tree(mc, - dbi == MAIN_DBI || (mc->mc_db->md_flags & MDBX_DUPSORT) != 0); - /* Invalidate the dropped DB's cursors */ - for (MDBX_cursor *m2 = txn->mt_cursors[dbi]; m2; m2 = m2->mc_next) - m2->mc_flags &= ~(C_INITIALIZED | C_EOF); - if (unlikely(rc)) - goto bailout; + if (unlikely(need_clean)) + memset(ret.page, -1, pgno2bytes(env, num)); - /* Can't delete the main DB */ - if (del && dbi >= CORE_DBS) { - rc = delete (txn, MAIN_DBI, &mc->mc_dbx->md_name, NULL, F_SUBDATA); - if (likely(rc == MDBX_SUCCESS)) { - tASSERT(txn, txn->mt_dbistate[MAIN_DBI] & DBI_DIRTY); - tASSERT(txn, txn->mt_flags & MDBX_TXN_DIRTY); - txn->mt_dbistate[dbi] = DBI_STALE; - MDBX_env *env = txn->mt_env; - rc = osal_fastmutex_acquire(&env->me_dbi_lock); - if (unlikely(rc != MDBX_SUCCESS)) { - txn->mt_flags |= MDBX_TXN_ERROR; - goto bailout; - } - dbi_close_locked(env, dbi); - ENSURE(env, osal_fastmutex_release(&env->me_dbi_lock) == MDBX_SUCCESS); - } else { - txn->mt_flags |= MDBX_TXN_ERROR; - } - } else { - /* reset the DB record, mark it dirty */ - txn->mt_dbistate[dbi] |= DBI_DIRTY; - txn->mt_dbs[dbi].md_depth = 0; - txn->mt_dbs[dbi].md_branch_pages = 0; - txn->mt_dbs[dbi].md_leaf_pages = 0; - txn->mt_dbs[dbi].md_overflow_pages = 0; - txn->mt_dbs[dbi].md_entries = 0; - txn->mt_dbs[dbi].md_root = P_INVALID; - txn->mt_dbs[dbi].md_seq = 0; - txn->mt_flags |= MDBX_TXN_DIRTY; + VALGRIND_MAKE_MEM_UNDEFINED(ret.page, pgno2bytes(env, num)); + ret.page->pgno = pgno; + ret.page->dupfix_ksize = 0; + ret.page->flags = 0; + if ((ASSERT_ENABLED() || AUDIT_ENABLED()) && num > 1) { + ret.page->pages = (pgno_t)num; + ret.page->flags = P_LARGE; } + ret.err = page_dirty(txn, ret.page, (pgno_t)num); bailout: - mdbx_cursor_close(mc); - return rc; + tASSERT(txn, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); +#if MDBX_ENABLE_PROFGC + size_t majflt_after; + prof->xtime_cpu += osal_cputime(&majflt_after) - cputime_before; + prof->majflt += (uint32_t)(majflt_after - majflt_before); +#endif /* MDBX_ENABLE_PROFGC */ + return ret; } -__cold int mdbx_reader_list(const MDBX_env *env, MDBX_reader_list_func *func, - void *ctx) { - int rc = check_env(env, true); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - - if (unlikely(!func)) - return MDBX_EINVAL; - - rc = MDBX_RESULT_TRUE; - int serial = 0; - MDBX_lockinfo *const lck = env->me_lck_mmap.lck; - if (likely(lck)) { - const size_t snap_nreaders = - atomic_load32(&lck->mti_numreaders, mo_AcquireRelease); - for (size_t i = 0; i < snap_nreaders; i++) { - const MDBX_reader *r = lck->mti_readers + i; - retry_reader:; - const uint32_t pid = atomic_load32(&r->mr_pid, mo_AcquireRelease); - if (!pid) - continue; - txnid_t txnid = safe64_read(&r->mr_txnid); - const uint64_t tid = atomic_load64(&r->mr_tid, mo_Relaxed); - const pgno_t pages_used = - atomic_load32(&r->mr_snapshot_pages_used, mo_Relaxed); - const uint64_t reader_pages_retired = - atomic_load64(&r->mr_snapshot_pages_retired, mo_Relaxed); - if (unlikely( - txnid != safe64_read(&r->mr_txnid) || - pid != atomic_load32(&r->mr_pid, mo_AcquireRelease) || - tid != atomic_load64(&r->mr_tid, mo_Relaxed) || - pages_used != - atomic_load32(&r->mr_snapshot_pages_used, mo_Relaxed) || - reader_pages_retired != - atomic_load64(&r->mr_snapshot_pages_retired, mo_Relaxed))) - goto retry_reader; - - eASSERT(env, txnid > 0); - if (txnid >= SAFE64_INVALID_THRESHOLD) - txnid = 0; +pgr_t gc_alloc_ex(const MDBX_cursor *const mc, const size_t num, uint8_t flags) { + pgr_t ret; + MDBX_txn *const txn = mc->txn; + MDBX_env *const env = txn->env; +#if MDBX_ENABLE_PROFGC + gc_prof_stat_t *const prof = + (cursor_dbi(mc) == FREE_DBI) ? &env->lck->pgops.gc_prof.self : &env->lck->pgops.gc_prof.work; + prof->spe_counter += 1; +#endif /* MDBX_ENABLE_PROFGC */ - size_t bytes_used = 0; - size_t bytes_retained = 0; - uint64_t lag = 0; - if (txnid) { - meta_troika_t troika = meta_tap(env); - retry_header:; - const meta_ptr_t head = meta_recent(env, &troika); - const uint64_t head_pages_retired = - unaligned_peek_u64_volatile(4, head.ptr_v->mm_pages_retired); - if (unlikely(meta_should_retry(env, &troika) || - head_pages_retired != - unaligned_peek_u64_volatile( - 4, head.ptr_v->mm_pages_retired))) - goto retry_header; + eASSERT(env, num > 0 || (flags & ALLOC_RESERVE)); + eASSERT(env, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); - lag = (head.txnid - txnid) / xMDBX_TXNID_STEP; - bytes_used = pgno2bytes(env, pages_used); - bytes_retained = (head_pages_retired > reader_pages_retired) - ? pgno2bytes(env, (pgno_t)(head_pages_retired - - reader_pages_retired)) - : 0; - } - rc = func(ctx, ++serial, (unsigned)i, pid, (mdbx_tid_t)((intptr_t)tid), - txnid, lag, bytes_used, bytes_retained); - if (unlikely(rc != MDBX_SUCCESS)) - break; + size_t newnext; + const uint64_t monotime_begin = (MDBX_ENABLE_PROFGC || (num > 1 && env->options.gc_time_limit)) ? osal_monotime() : 0; + struct monotime_cache now_cache; + now_cache.expire_countdown = 1 /* старт с 1 позволяет избавиться как от лишних системных вызовов когда + лимит времени задан нулевой или уже исчерпан, так и от подсчета + времени при не-достижении rp_augment_limit */ + ; + now_cache.value = monotime_begin; + pgno_t pgno = 0; + if (num > 1) { +#if MDBX_ENABLE_PROFGC + prof->xpages += 1; +#endif /* MDBX_ENABLE_PROFGC */ + if (MDBX_PNL_GETSIZE(txn->tw.repnl) >= num) { + eASSERT(env, MDBX_PNL_LAST(txn->tw.repnl) < txn->geo.first_unallocated && + MDBX_PNL_FIRST(txn->tw.repnl) < txn->geo.first_unallocated); + pgno = repnl_get_sequence(txn, num, flags); + if (likely(pgno)) + goto done; } + } else { + eASSERT(env, num == 0 || MDBX_PNL_GETSIZE(txn->tw.repnl) == 0); + eASSERT(env, !(flags & ALLOC_RESERVE) || num == 0); } - return rc; -} + //--------------------------------------------------------------------------- -/* Insert pid into list if not already present. - * return -1 if already present. */ -__cold static bool pid_insert(uint32_t *ids, uint32_t pid) { - /* binary search of pid in list */ - size_t base = 0; - size_t cursor = 1; - int val = 0; - size_t n = ids[0]; + if (unlikely(!is_gc_usable(txn, mc, flags))) + goto no_gc; - while (n > 0) { - size_t pivot = n >> 1; - cursor = base + pivot + 1; - val = pid - ids[cursor]; + eASSERT(env, (flags & (ALLOC_COALESCE | ALLOC_LIFO | ALLOC_SHOULD_SCAN)) == 0); + flags += (env->flags & MDBX_LIFORECLAIM) ? ALLOC_LIFO : 0; - if (val < 0) { - n = pivot; - } else if (val > 0) { - base = cursor; - n -= pivot + 1; - } else { - /* found, so it's a duplicate */ - return false; - } + if (/* Не коагулируем записи при подготовке резерва для обновления GC. + * Иначе попытка увеличить резерв может приводить к необходимости ещё + * большего резерва из-за увеличения списка переработанных страниц. */ + (flags & ALLOC_RESERVE) == 0) { + if (txn->dbs[FREE_DBI].branch_pages && MDBX_PNL_GETSIZE(txn->tw.repnl) < env->maxgc_large1page / 2) + flags += ALLOC_COALESCE; } - if (val > 0) - ++cursor; - - ids[0]++; - for (n = ids[0]; n > cursor; n--) - ids[n] = ids[n - 1]; - ids[n] = pid; - return true; -} + MDBX_cursor *const gc = ptr_disp(env->basal_txn, sizeof(MDBX_txn)); + eASSERT(env, mc != gc && gc->next == gc); + gc->txn = txn; + gc->dbi_state = txn->dbi_state; + gc->top_and_flags = z_fresh_mark; -__cold int mdbx_reader_check(MDBX_env *env, int *dead) { - if (dead) - *dead = 0; - return cleanup_dead_readers(env, false, dead); -} + txn->tw.prefault_write_activated = env->options.prefault_write; + if (txn->tw.prefault_write_activated) { + /* Проверка посредством minicore() существенно снижает затраты, но в + * простейших случаях (тривиальный бенчмарк) интегральная производительность + * становится вдвое меньше. А на платформах без mincore() и с проблемной + * подсистемой виртуальной памяти ситуация может быть многократно хуже. + * Поэтому избегаем затрат в ситуациях когда prefault-write скорее всего не + * нужна. */ + const bool readahead_enabled = env->lck->readahead_anchor & 1; + const pgno_t readahead_edge = env->lck->readahead_anchor >> 1; + if (/* Не суетимся если GC почти пустая и БД маленькая */ + (txn->dbs[FREE_DBI].branch_pages == 0 && txn->geo.now < 1234) || + /* Не суетимся если страница в зоне включенного упреждающего чтения */ + (readahead_enabled && pgno + num < readahead_edge)) + txn->tw.prefault_write_activated = false; + } -/* Return: - * MDBX_RESULT_TRUE - done and mutex recovered - * MDBX_SUCCESS - done - * Otherwise errcode. */ -__cold MDBX_INTERNAL_FUNC int cleanup_dead_readers(MDBX_env *env, - int rdt_locked, int *dead) { - int rc = check_env(env, true); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; +retry_gc_refresh_oldest:; + txnid_t oldest = txn_snapshot_oldest(txn); +retry_gc_have_oldest: + if (unlikely(oldest >= txn->txnid)) { + ERROR("unexpected/invalid oldest-readed txnid %" PRIaTXN " for current-txnid %" PRIaTXN, oldest, txn->txnid); + ret.err = MDBX_PROBLEM; + goto fail; + } + const txnid_t detent = oldest + 1; - eASSERT(env, rdt_locked >= 0); - MDBX_lockinfo *const lck = env->me_lck_mmap.lck; - if (unlikely(lck == NULL)) { - /* exclusive mode */ - if (dead) - *dead = 0; - return MDBX_SUCCESS; + txnid_t id = 0; + MDBX_cursor_op op = MDBX_FIRST; + if (flags & ALLOC_LIFO) { + if (!txn->tw.gc.retxl) { + txn->tw.gc.retxl = txl_alloc(); + if (unlikely(!txn->tw.gc.retxl)) { + ret.err = MDBX_ENOMEM; + goto fail; + } + } + /* Begin lookup backward from oldest reader */ + id = detent - 1; + op = MDBX_SET_RANGE; + } else if (txn->tw.gc.last_reclaimed) { + /* Continue lookup forward from last-reclaimed */ + id = txn->tw.gc.last_reclaimed + 1; + if (id >= detent) + goto depleted_gc; + op = MDBX_SET_RANGE; } - const size_t snap_nreaders = - atomic_load32(&lck->mti_numreaders, mo_AcquireRelease); - uint32_t pidsbuf_onstask[142]; - uint32_t *const pids = - (snap_nreaders < ARRAY_LENGTH(pidsbuf_onstask)) - ? pidsbuf_onstask - : osal_malloc((snap_nreaders + 1) * sizeof(uint32_t)); - if (unlikely(!pids)) - return MDBX_ENOMEM; +next_gc:; + MDBX_val key; + key.iov_base = &id; + key.iov_len = sizeof(id); - pids[0] = 0; - int count = 0; - for (size_t i = 0; i < snap_nreaders; i++) { - const uint32_t pid = - atomic_load32(&lck->mti_readers[i].mr_pid, mo_AcquireRelease); - if (pid == 0) - continue /* skip empty */; - if (pid == env->me_pid) - continue /* skip self */; - if (!pid_insert(pids, pid)) - continue /* such pid already processed */; - - int err = osal_rpid_check(env, pid); - if (err == MDBX_RESULT_TRUE) - continue /* reader is live */; +#if MDBX_ENABLE_PROFGC + prof->rsteps += 1; +#endif /* MDBX_ENABLE_PROFGC */ - if (err != MDBX_SUCCESS) { - rc = err; - break /* osal_rpid_check() failed */; + /* Seek first/next GC record */ + ret.err = cursor_ops(gc, &key, nullptr, op); + if (unlikely(ret.err != MDBX_SUCCESS)) { + if (unlikely(ret.err != MDBX_NOTFOUND)) + goto fail; + if ((flags & ALLOC_LIFO) && op == MDBX_SET_RANGE) { + op = MDBX_PREV; + goto next_gc; } + goto depleted_gc; + } + if (unlikely(key.iov_len != sizeof(txnid_t))) { + ERROR("%s/%d: %s", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid GC key-length"); + ret.err = MDBX_CORRUPTED; + goto fail; + } + id = unaligned_peek_u64(4, key.iov_base); + if (flags & ALLOC_LIFO) { + op = MDBX_PREV; + if (id >= detent || is_already_reclaimed(txn, id)) + goto next_gc; + } else { + op = MDBX_NEXT; + if (unlikely(id >= detent)) + goto depleted_gc; + } + txn->flags &= ~txn_gc_drained; - /* stale reader found */ - if (!rdt_locked) { - err = osal_rdt_lock(env); - if (MDBX_IS_ERROR(err)) { - rc = err; - break; - } + /* Reading next GC record */ + MDBX_val data; + page_t *const mp = gc->pg[gc->top]; + if (unlikely((ret.err = node_read(gc, page_node(mp, gc->ki[gc->top]), &data, mp)) != MDBX_SUCCESS)) + goto fail; - rdt_locked = -1; - if (err == MDBX_RESULT_TRUE) { - /* mutex recovered, the mdbx_ipclock_failed() checked all readers */ - rc = MDBX_RESULT_TRUE; - break; - } + pgno_t *gc_pnl = (pgno_t *)data.iov_base; + if (unlikely(data.iov_len % sizeof(pgno_t) || data.iov_len < MDBX_PNL_SIZEOF(gc_pnl) || + !pnl_check(gc_pnl, txn->geo.first_unallocated))) { + ERROR("%s/%d: %s", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid GC value-length"); + ret.err = MDBX_CORRUPTED; + goto fail; + } - /* a other process may have clean and reused slot, recheck */ - if (lck->mti_readers[i].mr_pid.weak != pid) - continue; + const size_t gc_len = MDBX_PNL_GETSIZE(gc_pnl); + TRACE("gc-read: id #%" PRIaTXN " len %zu, re-list will %zu ", id, gc_len, gc_len + MDBX_PNL_GETSIZE(txn->tw.repnl)); - err = osal_rpid_check(env, pid); - if (MDBX_IS_ERROR(err)) { - rc = err; - break; + if (unlikely(gc_len + MDBX_PNL_GETSIZE(txn->tw.repnl) >= env->maxgc_large1page)) { + /* Don't try to coalesce too much. */ + if (flags & ALLOC_SHOULD_SCAN) { + eASSERT(env, flags & ALLOC_COALESCE); + eASSERT(env, !(flags & ALLOC_RESERVE)); + eASSERT(env, num > 0); +#if MDBX_ENABLE_PROFGC + env->lck->pgops.gc_prof.coalescences += 1; +#endif /* MDBX_ENABLE_PROFGC */ + TRACE("clear %s %s", "ALLOC_COALESCE", "since got threshold"); + if (MDBX_PNL_GETSIZE(txn->tw.repnl) >= num) { + eASSERT(env, MDBX_PNL_LAST(txn->tw.repnl) < txn->geo.first_unallocated && + MDBX_PNL_FIRST(txn->tw.repnl) < txn->geo.first_unallocated); + if (likely(num == 1)) { + pgno = repnl_get_single(txn); + goto done; + } + pgno = repnl_get_sequence(txn, num, flags); + if (likely(pgno)) + goto done; } - - if (err != MDBX_SUCCESS) - continue /* the race with other process, slot reused */; + flags -= ALLOC_COALESCE | ALLOC_SHOULD_SCAN; } - - /* clean it */ - for (size_t j = i; j < snap_nreaders; j++) { - if (lck->mti_readers[j].mr_pid.weak == pid) { - DEBUG("clear stale reader pid %" PRIuPTR " txn %" PRIaTXN, (size_t)pid, - lck->mti_readers[j].mr_txnid.weak); - atomic_store32(&lck->mti_readers[j].mr_pid, 0, mo_Relaxed); - atomic_store32(&lck->mti_readers_refresh_flag, true, mo_AcquireRelease); - count++; - } + if (unlikely(/* list is too long already */ MDBX_PNL_GETSIZE(txn->tw.repnl) >= env->options.rp_augment_limit) && + ((/* not a slot-request from gc-update */ num && + /* have enough unallocated space */ txn->geo.upper >= txn->geo.first_unallocated + num && + monotime_since_cached(monotime_begin, &now_cache) + txn->tw.gc.time_acc >= env->options.gc_time_limit) || + gc_len + MDBX_PNL_GETSIZE(txn->tw.repnl) >= PAGELIST_LIMIT)) { + /* Stop reclaiming to avoid large/overflow the page list. This is a rare + * case while search for a continuously multi-page region in a + * large database, see https://libmdbx.dqdkfa.ru/dead-github/issues/123 */ + NOTICE("stop reclaiming %s: %zu (current) + %zu " + "(chunk) -> %zu, rp_augment_limit %u", + likely(gc_len + MDBX_PNL_GETSIZE(txn->tw.repnl) < PAGELIST_LIMIT) ? "since rp_augment_limit was reached" + : "to avoid PNL overflow", + MDBX_PNL_GETSIZE(txn->tw.repnl), gc_len, gc_len + MDBX_PNL_GETSIZE(txn->tw.repnl), + env->options.rp_augment_limit); + goto depleted_gc; } } - if (likely(!MDBX_IS_ERROR(rc))) - atomic_store64(&lck->mti_reader_check_timestamp, osal_monotime(), - mo_Relaxed); - - if (rdt_locked < 0) - osal_rdt_unlock(env); - - if (pids != pidsbuf_onstask) - osal_free(pids); - - if (dead) - *dead = count; - return rc; -} + /* Remember ID of readed GC record */ + txn->tw.gc.last_reclaimed = id; + if (flags & ALLOC_LIFO) { + ret.err = txl_append(&txn->tw.gc.retxl, id); + if (unlikely(ret.err != MDBX_SUCCESS)) + goto fail; + } -__cold int mdbx_setup_debug(MDBX_log_level_t level, MDBX_debug_flags_t flags, - MDBX_debug_func *logger) { - const int rc = runtime_flags | (loglevel << 16); + /* Append PNL from GC record to tw.repnl */ + ret.err = pnl_need(&txn->tw.repnl, gc_len); + if (unlikely(ret.err != MDBX_SUCCESS)) + goto fail; - if (level != MDBX_LOG_DONTCHANGE) - loglevel = (uint8_t)level; + if (LOG_ENABLED(MDBX_LOG_EXTRA)) { + DEBUG_EXTRA("readed GC-pnl txn %" PRIaTXN " root %" PRIaPGNO " len %zu, PNL", id, txn->dbs[FREE_DBI].root, gc_len); + for (size_t i = gc_len; i; i--) + DEBUG_EXTRA_PRINT(" %" PRIaPGNO, gc_pnl[i]); + DEBUG_EXTRA_PRINT(", first_unallocated %u\n", txn->geo.first_unallocated); + } - if (flags != MDBX_DBG_DONTCHANGE) { - flags &= -#if MDBX_DEBUG - MDBX_DBG_ASSERT | MDBX_DBG_AUDIT | MDBX_DBG_JITTER | -#endif - MDBX_DBG_DUMP | MDBX_DBG_LEGACY_MULTIOPEN | MDBX_DBG_LEGACY_OVERLAP | - MDBX_DBG_DONT_UPGRADE; - runtime_flags = (uint8_t)flags; + /* Merge in descending sorted order */ +#if MDBX_ENABLE_PROFGC + const uint64_t merge_begin = osal_monotime(); +#endif /* MDBX_ENABLE_PROFGC */ + pnl_merge(txn->tw.repnl, gc_pnl); +#if MDBX_ENABLE_PROFGC + prof->pnl_merge.calls += 1; + prof->pnl_merge.volume += MDBX_PNL_GETSIZE(txn->tw.repnl); + prof->pnl_merge.time += osal_monotime() - merge_begin; +#endif /* MDBX_ENABLE_PROFGC */ + flags |= ALLOC_SHOULD_SCAN; + if (AUDIT_ENABLED()) { + if (unlikely(!pnl_check(txn->tw.repnl, txn->geo.first_unallocated))) { + ERROR("%s/%d: %s", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid txn retired-list"); + ret.err = MDBX_CORRUPTED; + goto fail; + } + } else { + eASSERT(env, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated)); } + eASSERT(env, dpl_check(txn)); - if (logger != MDBX_LOGGER_DONTCHANGE) - debug_logger = logger; - return rc; -} + eASSERT(env, MDBX_PNL_GETSIZE(txn->tw.repnl) == 0 || MDBX_PNL_MOST(txn->tw.repnl) < txn->geo.first_unallocated); + if (MDBX_ENABLE_REFUND && MDBX_PNL_GETSIZE(txn->tw.repnl) && + unlikely(MDBX_PNL_MOST(txn->tw.repnl) == txn->geo.first_unallocated - 1)) { + /* Refund suitable pages into "unallocated" space */ + txn_refund(txn); + } + eASSERT(env, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); -__cold static txnid_t kick_longlived_readers(MDBX_env *env, - const txnid_t laggard) { - DEBUG("DB size maxed out by reading #%" PRIaTXN, laggard); - osal_memory_fence(mo_AcquireRelease, false); - MDBX_hsr_func *const callback = env->me_hsr_callback; - txnid_t oldest = 0; - bool notify_eof_of_loop = false; - int retry = 0; - do { - const txnid_t steady = - env->me_txn->tw.troika.txnid[env->me_txn->tw.troika.prefer_steady]; - env->me_lck->mti_readers_refresh_flag.weak = /* force refresh */ true; - oldest = find_oldest_reader(env, steady); - eASSERT(env, oldest < env->me_txn0->mt_txnid); - eASSERT(env, oldest >= laggard); - eASSERT(env, oldest >= env->me_lck->mti_oldest_reader.weak); - - MDBX_lockinfo *const lck = env->me_lck_mmap.lck; - if (oldest == steady || oldest > laggard || /* without-LCK mode */ !lck) - break; + /* Done for a kick-reclaim mode, actually no page needed */ + if (unlikely(num == 0)) { + eASSERT(env, ret.err == MDBX_SUCCESS); + TRACE("%s: last id #%" PRIaTXN ", re-len %zu", "early-exit for slot", id, MDBX_PNL_GETSIZE(txn->tw.repnl)); + goto early_exit; + } - if (MDBX_IS_ERROR(cleanup_dead_readers(env, false, NULL))) - break; + /* TODO: delete reclaimed records */ - if (!callback) - break; + eASSERT(env, op == MDBX_PREV || op == MDBX_NEXT); + if (flags & ALLOC_COALESCE) { + TRACE("%s: last id #%" PRIaTXN ", re-len %zu", "coalesce-continue", id, MDBX_PNL_GETSIZE(txn->tw.repnl)); + goto next_gc; + } - MDBX_reader *stucked = nullptr; - uint64_t hold_retired = 0; - for (size_t i = 0; i < lck->mti_numreaders.weak; ++i) { - const uint64_t snap_retired = atomic_load64( - &lck->mti_readers[i].mr_snapshot_pages_retired, mo_Relaxed); - const txnid_t rtxn = safe64_read(&lck->mti_readers[i].mr_txnid); - if (rtxn == laggard && - atomic_load32(&lck->mti_readers[i].mr_pid, mo_AcquireRelease)) { - hold_retired = snap_retired; - stucked = &lck->mti_readers[i]; - } +scan: + eASSERT(env, flags & ALLOC_SHOULD_SCAN); + eASSERT(env, num > 0); + if (MDBX_PNL_GETSIZE(txn->tw.repnl) >= num) { + eASSERT(env, MDBX_PNL_LAST(txn->tw.repnl) < txn->geo.first_unallocated && + MDBX_PNL_FIRST(txn->tw.repnl) < txn->geo.first_unallocated); + if (likely(num == 1)) { + eASSERT(env, !(flags & ALLOC_RESERVE)); + pgno = repnl_get_single(txn); + goto done; } + pgno = repnl_get_sequence(txn, num, flags); + if (likely(pgno)) + goto done; + } + flags -= ALLOC_SHOULD_SCAN; + if (ret.err == MDBX_SUCCESS) { + TRACE("%s: last id #%" PRIaTXN ", re-len %zu", "continue-search", id, MDBX_PNL_GETSIZE(txn->tw.repnl)); + goto next_gc; + } - if (!stucked) - break; +depleted_gc: + TRACE("%s: last id #%" PRIaTXN ", re-len %zu", "gc-depleted", id, MDBX_PNL_GETSIZE(txn->tw.repnl)); + ret.err = MDBX_NOTFOUND; + if (flags & ALLOC_SHOULD_SCAN) + goto scan; + txn->flags |= txn_gc_drained; - uint32_t pid = atomic_load32(&stucked->mr_pid, mo_AcquireRelease); - uint64_t tid = atomic_load64(&stucked->mr_tid, mo_AcquireRelease); - if (safe64_read(&stucked->mr_txnid) != laggard || !pid || - stucked->mr_snapshot_pages_retired.weak != hold_retired) - continue; + //------------------------------------------------------------------------- - const meta_ptr_t head = meta_recent(env, &env->me_txn->tw.troika); - const txnid_t gap = (head.txnid - laggard) / xMDBX_TXNID_STEP; - const uint64_t head_retired = - unaligned_peek_u64(4, head.ptr_c->mm_pages_retired); - const size_t space = - (head_retired > hold_retired) - ? pgno2bytes(env, (pgno_t)(head_retired - hold_retired)) - : 0; - int rc = - callback(env, env->me_txn, pid, (mdbx_tid_t)((intptr_t)tid), laggard, - (gap < UINT_MAX) ? (unsigned)gap : UINT_MAX, space, retry); - if (rc < 0) - /* hsr returned error and/or agree MDBX_MAP_FULL error */ - break; + /* There is no suitable pages in the GC and to be able to allocate + * we should CHOICE one of: + * - make a new steady checkpoint if reclaiming was stopped by + * the last steady-sync, or wipe it in the MDBX_UTTERLY_NOSYNC mode; + * - kick lagging reader(s) if reclaiming was stopped by ones of it. + * - extend the database file. */ - if (rc > 0) { - if (rc == 1) { - /* hsr reported transaction (will be) aborted asynchronous */ - safe64_reset_compare(&stucked->mr_txnid, laggard); - } else { - /* hsr reported reader process was killed and slot should be cleared */ - safe64_reset(&stucked->mr_txnid, true); - atomic_store64(&stucked->mr_tid, 0, mo_Relaxed); - atomic_store32(&stucked->mr_pid, 0, mo_AcquireRelease); - } - } else if (!notify_eof_of_loop) { + /* Will use new pages from the map if nothing is suitable in the GC. */ + newnext = txn->geo.first_unallocated + num; + + /* Does reclaiming stopped at the last steady point? */ + const meta_ptr_t recent = meta_recent(env, &txn->tw.troika); + const meta_ptr_t prefer_steady = meta_prefer_steady(env, &txn->tw.troika); + if (recent.ptr_c != prefer_steady.ptr_c && prefer_steady.is_steady && detent == prefer_steady.txnid + 1) { + DEBUG("gc-kick-steady: recent %" PRIaTXN "-%s, steady %" PRIaTXN "-%s, detent %" PRIaTXN, recent.txnid, + durable_caption(recent.ptr_c), prefer_steady.txnid, durable_caption(prefer_steady.ptr_c), detent); + const pgno_t autosync_threshold = atomic_load32(&env->lck->autosync_threshold, mo_Relaxed); + const uint64_t autosync_period = atomic_load64(&env->lck->autosync_period, mo_Relaxed); + uint64_t eoos_timestamp; + /* wipe the last steady-point if one of: + * - UTTERLY_NOSYNC mode AND auto-sync threshold is NOT specified + * - UTTERLY_NOSYNC mode AND free space at steady-point is exhausted + * otherwise, make a new steady-point if one of: + * - auto-sync threshold is specified and reached; + * - upper limit of database size is reached; + * - database is full (with the current file size) + * AND auto-sync threshold it NOT specified */ + if (F_ISSET(env->flags, MDBX_UTTERLY_NOSYNC) && + ((autosync_threshold | autosync_period) == 0 || newnext >= prefer_steady.ptr_c->geometry.now)) { + /* wipe steady checkpoint in MDBX_UTTERLY_NOSYNC mode + * without any auto-sync threshold(s). */ #if MDBX_ENABLE_PROFGC - env->me_lck->mti_pgop_stat.gc_prof.kicks += 1; + env->lck->pgops.gc_prof.wipes += 1; #endif /* MDBX_ENABLE_PROFGC */ - notify_eof_of_loop = true; + ret.err = meta_wipe_steady(env, detent); + DEBUG("gc-wipe-steady, rc %d", ret.err); + if (unlikely(ret.err != MDBX_SUCCESS)) + goto fail; + eASSERT(env, prefer_steady.ptr_c != meta_prefer_steady(env, &txn->tw.troika).ptr_c); + goto retry_gc_refresh_oldest; + } + if ((autosync_threshold && atomic_load64(&env->lck->unsynced_pages, mo_Relaxed) >= autosync_threshold) || + (autosync_period && (eoos_timestamp = atomic_load64(&env->lck->eoos_timestamp, mo_Relaxed)) && + osal_monotime() - eoos_timestamp >= autosync_period) || + newnext >= txn->geo.upper || + ((num == 0 || newnext >= txn->geo.end_pgno) && (autosync_threshold | autosync_period) == 0)) { + /* make steady checkpoint. */ +#if MDBX_ENABLE_PROFGC + env->lck->pgops.gc_prof.flushes += 1; +#endif /* MDBX_ENABLE_PROFGC */ + meta_t meta = *recent.ptr_c; + ret.err = dxb_sync_locked(env, env->flags & MDBX_WRITEMAP, &meta, &txn->tw.troika); + DEBUG("gc-make-steady, rc %d", ret.err); + eASSERT(env, ret.err != MDBX_RESULT_TRUE); + if (unlikely(ret.err != MDBX_SUCCESS)) + goto fail; + eASSERT(env, prefer_steady.ptr_c != meta_prefer_steady(env, &txn->tw.troika).ptr_c); + goto retry_gc_refresh_oldest; } - - } while (++retry < INT_MAX); - - if (notify_eof_of_loop) { - /* notify end of hsr-loop */ - const txnid_t turn = oldest - laggard; - if (turn) - NOTICE("hsr-kick: done turn %" PRIaTXN " -> %" PRIaTXN " +%" PRIaTXN, - laggard, oldest, turn); - callback(env, env->me_txn, 0, 0, laggard, - (turn < UINT_MAX) ? (unsigned)turn : UINT_MAX, 0, -retry); } - return oldest; -} -__cold int mdbx_env_set_hsr(MDBX_env *env, MDBX_hsr_func *hsr) { - int rc = check_env(env, false); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; + if (unlikely(true == atomic_load32(&env->lck->rdt_refresh_flag, mo_AcquireRelease))) { + oldest = txn_snapshot_oldest(txn); + if (oldest >= detent) + goto retry_gc_have_oldest; + } - env->me_hsr_callback = hsr; - return MDBX_SUCCESS; -} + /* Avoid kick lagging reader(s) if is enough unallocated space + * at the end of database file. */ + if (!(flags & ALLOC_RESERVE) && newnext <= txn->geo.end_pgno) { + eASSERT(env, pgno == 0); + goto done; + } -__cold MDBX_hsr_func *mdbx_env_get_hsr(const MDBX_env *env) { - return likely(env && env->me_signature.weak == MDBX_ME_SIGNATURE) - ? env->me_hsr_callback - : NULL; -} + if (oldest < txn->txnid - xMDBX_TXNID_STEP) { + oldest = mvcc_kick_laggards(env, oldest); + if (oldest >= detent) + goto retry_gc_have_oldest; + } -#ifdef __SANITIZE_THREAD__ -/* LY: avoid tsan-trap by me_txn, mm_last_pg and mt_next_pgno */ -__attribute__((__no_sanitize_thread__, __noinline__)) -#endif -int mdbx_txn_straggler(const MDBX_txn *txn, int *percent) -{ - int rc = check_txn(txn, MDBX_TXN_BLOCKED); - if (unlikely(rc != MDBX_SUCCESS)) - return (rc > 0) ? -rc : rc; + //--------------------------------------------------------------------------- - MDBX_env *env = txn->mt_env; - if (unlikely((txn->mt_flags & MDBX_TXN_RDONLY) == 0)) { - if (percent) - *percent = - (int)((txn->mt_next_pgno * UINT64_C(100) + txn->mt_end_pgno / 2) / - txn->mt_end_pgno); - return 0; +no_gc: + eASSERT(env, pgno == 0); +#ifndef MDBX_ENABLE_BACKLOG_DEPLETED +#define MDBX_ENABLE_BACKLOG_DEPLETED 0 +#endif /* MDBX_ENABLE_BACKLOG_DEPLETED*/ + if (MDBX_ENABLE_BACKLOG_DEPLETED && unlikely(!(txn->flags & txn_gc_drained))) { + ret.err = MDBX_BACKLOG_DEPLETED; + goto fail; + } + if (flags & ALLOC_RESERVE) { + ret.err = MDBX_NOTFOUND; + goto fail; } - txnid_t lag; - meta_troika_t troika = meta_tap(env); - do { - const meta_ptr_t head = meta_recent(env, &troika); - if (percent) { - const pgno_t maxpg = head.ptr_v->mm_geo.now; - *percent = - (int)((head.ptr_v->mm_geo.next * UINT64_C(100) + maxpg / 2) / maxpg); - } - lag = (head.txnid - txn->mt_txnid) / xMDBX_TXNID_STEP; - } while (unlikely(meta_should_retry(env, &troika))); - - return (lag > INT_MAX) ? INT_MAX : (int)lag; -} - -typedef struct mdbx_walk_ctx { - void *mw_user; - MDBX_pgvisitor_func *mw_visitor; - MDBX_txn *mw_txn; - MDBX_cursor *mw_cursor; - bool mw_dont_check_keys_ordering; -} mdbx_walk_ctx_t; - -__cold static int walk_sdb(mdbx_walk_ctx_t *ctx, MDBX_db *const sdb, - const MDBX_val *name, int deep); + /* Will use new pages from the map if nothing is suitable in the GC. */ + newnext = txn->geo.first_unallocated + num; + if (newnext <= txn->geo.end_pgno) + goto done; -static MDBX_page_type_t walk_page_type(const MDBX_page *mp) { - if (mp) - switch (mp->mp_flags) { - case P_BRANCH: - return MDBX_page_branch; - case P_LEAF: - return MDBX_page_leaf; - case P_LEAF | P_LEAF2: - return MDBX_page_dupfixed_leaf; - case P_OVERFLOW: - return MDBX_page_large; - case P_META: - return MDBX_page_meta; - } - return MDBX_page_broken; -} + if (newnext > txn->geo.upper || !txn->geo.grow_pv) { + NOTICE("gc-alloc: next %zu > upper %" PRIaPGNO, newnext, txn->geo.upper); + ret.err = MDBX_MAP_FULL; + goto fail; + } -/* Depth-first tree traversal. */ -__cold static int walk_tree(mdbx_walk_ctx_t *ctx, const pgno_t pgno, - const MDBX_val *name, int deep, - txnid_t parent_txnid) { - assert(pgno != P_INVALID); - MDBX_page *mp = nullptr; - int err = page_get(ctx->mw_cursor, pgno, &mp, parent_txnid); + eASSERT(env, newnext > txn->geo.end_pgno); + const size_t grow_step = pv2pages(txn->geo.grow_pv); + size_t aligned = pgno_align2os_pgno(env, (pgno_t)(newnext + grow_step - newnext % grow_step)); - MDBX_page_type_t type = walk_page_type(mp); - const size_t nentries = mp ? page_numkeys(mp) : 0; - unsigned npages = 1; - size_t pagesize = pgno2bytes(ctx->mw_txn->mt_env, npages); - size_t header_size = - (mp && !IS_LEAF2(mp)) ? PAGEHDRSZ + mp->mp_lower : PAGEHDRSZ; - size_t payload_size = 0; - size_t unused_size = - (mp ? page_room(mp) : pagesize - header_size) - payload_size; - size_t align_bytes = 0; + if (aligned > txn->geo.upper) + aligned = txn->geo.upper; + eASSERT(env, aligned >= newnext); - for (size_t i = 0; err == MDBX_SUCCESS && i < nentries; ++i) { - if (type == MDBX_page_dupfixed_leaf) { - /* LEAF2 pages have no mp_ptrs[] or node headers */ - payload_size += mp->mp_leaf2_ksize; - continue; - } + VERBOSE("try growth datafile to %zu pages (+%zu)", aligned, aligned - txn->geo.end_pgno); + ret.err = dxb_resize(env, txn->geo.first_unallocated, (pgno_t)aligned, txn->geo.upper, implicit_grow); + if (ret.err != MDBX_SUCCESS) { + ERROR("unable growth datafile to %zu pages (+%zu), errcode %d", aligned, aligned - txn->geo.end_pgno, ret.err); + goto fail; + } + env->txn->geo.end_pgno = (pgno_t)aligned; + eASSERT(env, pgno == 0); - MDBX_node *node = page_node(mp, i); - const size_t node_key_size = node_ks(node); - payload_size += NODESIZE + node_key_size; + //--------------------------------------------------------------------------- - if (type == MDBX_page_branch) { - assert(i > 0 || node_ks(node) == 0); - align_bytes += node_key_size & 1; - continue; +done: + ret.err = MDBX_SUCCESS; + if (likely((flags & ALLOC_RESERVE) == 0)) { + if (pgno) { + eASSERT(env, pgno + num <= txn->geo.first_unallocated && pgno >= NUM_METAS); + eASSERT(env, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); + } else { + pgno = txn->geo.first_unallocated; + txn->geo.first_unallocated += (pgno_t)num; + eASSERT(env, txn->geo.first_unallocated <= txn->geo.end_pgno); + eASSERT(env, pgno >= NUM_METAS && pgno + num <= txn->geo.first_unallocated); } - const size_t node_data_size = node_ds(node); - assert(type == MDBX_page_leaf); - switch (node_flags(node)) { - case 0 /* usual node */: - payload_size += node_data_size; - align_bytes += (node_key_size + node_data_size) & 1; - break; - - case F_BIGDATA /* long data on the large/overflow page */: { - const pgno_t large_pgno = node_largedata_pgno(node); - const size_t over_payload = node_data_size; - const size_t over_header = PAGEHDRSZ; - npages = 1; - - assert(err == MDBX_SUCCESS); - pgr_t lp = page_get_large(ctx->mw_cursor, large_pgno, mp->mp_txnid); - err = lp.err; - if (err == MDBX_SUCCESS) { - cASSERT(ctx->mw_cursor, PAGETYPE_WHOLE(lp.page) == P_OVERFLOW); - npages = lp.page->mp_pages; - } - - pagesize = pgno2bytes(ctx->mw_txn->mt_env, npages); - const size_t over_unused = pagesize - over_payload - over_header; - const int rc = ctx->mw_visitor(large_pgno, npages, ctx->mw_user, deep, - name, pagesize, MDBX_page_large, err, 1, - over_payload, over_header, over_unused); - if (unlikely(rc != MDBX_SUCCESS)) - return (rc == MDBX_RESULT_TRUE) ? MDBX_SUCCESS : rc; - payload_size += sizeof(pgno_t); - align_bytes += node_key_size & 1; - } break; - - case F_SUBDATA /* sub-db */: { - const size_t namelen = node_key_size; - if (unlikely(namelen == 0 || node_data_size != sizeof(MDBX_db))) { - assert(err == MDBX_CORRUPTED); - err = MDBX_CORRUPTED; - } - header_size += node_data_size; - align_bytes += (node_key_size + node_data_size) & 1; - } break; - - case F_SUBDATA | F_DUPDATA /* dupsorted sub-tree */: - if (unlikely(node_data_size != sizeof(MDBX_db))) { - assert(err == MDBX_CORRUPTED); - err = MDBX_CORRUPTED; - } - header_size += node_data_size; - align_bytes += (node_key_size + node_data_size) & 1; - break; - - case F_DUPDATA /* short sub-page */: { - if (unlikely(node_data_size <= PAGEHDRSZ || (node_data_size & 1))) { - assert(err == MDBX_CORRUPTED); - err = MDBX_CORRUPTED; - break; - } - - MDBX_page *sp = node_data(node); - const size_t nsubkeys = page_numkeys(sp); - size_t subheader_size = - IS_LEAF2(sp) ? PAGEHDRSZ : PAGEHDRSZ + sp->mp_lower; - size_t subunused_size = page_room(sp); - size_t subpayload_size = 0; - size_t subalign_bytes = 0; - MDBX_page_type_t subtype; - - switch (sp->mp_flags & /* ignore legacy P_DIRTY flag */ ~P_LEGACY_DIRTY) { - case P_LEAF | P_SUBP: - subtype = MDBX_subpage_leaf; - break; - case P_LEAF | P_LEAF2 | P_SUBP: - subtype = MDBX_subpage_dupfixed_leaf; - break; - default: - assert(err == MDBX_CORRUPTED); - subtype = MDBX_subpage_broken; - err = MDBX_CORRUPTED; - } - - for (size_t j = 0; err == MDBX_SUCCESS && j < nsubkeys; ++j) { - if (subtype == MDBX_subpage_dupfixed_leaf) { - /* LEAF2 pages have no mp_ptrs[] or node headers */ - subpayload_size += sp->mp_leaf2_ksize; - } else { - assert(subtype == MDBX_subpage_leaf); - const MDBX_node *subnode = page_node(sp, j); - const size_t subnode_size = node_ks(subnode) + node_ds(subnode); - subheader_size += NODESIZE; - subpayload_size += subnode_size; - subalign_bytes += subnode_size & 1; - if (unlikely(node_flags(subnode) != 0)) { - assert(err == MDBX_CORRUPTED); - err = MDBX_CORRUPTED; - } - } + ret = page_alloc_finalize(env, txn, mc, pgno, num); + if (unlikely(ret.err != MDBX_SUCCESS)) { + fail: + eASSERT(env, ret.err != MDBX_SUCCESS); + eASSERT(env, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); + int level; + const char *what; + if (flags & ALLOC_RESERVE) { + level = (flags & ALLOC_UNIMPORTANT) ? MDBX_LOG_DEBUG : MDBX_LOG_NOTICE; + what = num ? "reserve-pages" : "fetch-slot"; + } else { + txn->flags |= MDBX_TXN_ERROR; + level = MDBX_LOG_ERROR; + what = "pages"; } - - const int rc = - ctx->mw_visitor(pgno, 0, ctx->mw_user, deep + 1, name, node_data_size, - subtype, err, nsubkeys, subpayload_size, - subheader_size, subunused_size + subalign_bytes); - if (unlikely(rc != MDBX_SUCCESS)) - return (rc == MDBX_RESULT_TRUE) ? MDBX_SUCCESS : rc; - header_size += subheader_size; - unused_size += subunused_size; - payload_size += subpayload_size; - align_bytes += subalign_bytes + (node_key_size & 1); - } break; - - default: - assert(err == MDBX_CORRUPTED); - err = MDBX_CORRUPTED; + if (LOG_ENABLED(level)) + debug_log(level, __func__, __LINE__, + "unable alloc %zu %s, alloc-flags 0x%x, err %d, txn-flags " + "0x%x, re-list-len %zu, loose-count %zu, gc: height %u, " + "branch %zu, leaf %zu, large %zu, entries %zu\n", + num, what, flags, ret.err, txn->flags, MDBX_PNL_GETSIZE(txn->tw.repnl), txn->tw.loose_count, + txn->dbs[FREE_DBI].height, (size_t)txn->dbs[FREE_DBI].branch_pages, + (size_t)txn->dbs[FREE_DBI].leaf_pages, (size_t)txn->dbs[FREE_DBI].large_pages, + (size_t)txn->dbs[FREE_DBI].items); + ret.page = nullptr; } + if (num > 1) + txn->tw.gc.time_acc += monotime_since_cached(monotime_begin, &now_cache); + } else { + early_exit: + DEBUG("return nullptr for %zu pages for ALLOC_%s, rc %d", num, num ? "RESERVE" : "SLOT", ret.err); + ret.page = nullptr; } - const int rc = ctx->mw_visitor( - pgno, 1, ctx->mw_user, deep, name, ctx->mw_txn->mt_env->me_psize, type, - err, nentries, payload_size, header_size, unused_size + align_bytes); - if (unlikely(rc != MDBX_SUCCESS)) - return (rc == MDBX_RESULT_TRUE) ? MDBX_SUCCESS : rc; +#if MDBX_ENABLE_PROFGC + prof->rtime_monotonic += osal_monotime() - monotime_begin; +#endif /* MDBX_ENABLE_PROFGC */ + return ret; +} - for (size_t i = 0; err == MDBX_SUCCESS && i < nentries; ++i) { - if (type == MDBX_page_dupfixed_leaf) - continue; +__hot pgr_t gc_alloc_single(const MDBX_cursor *const mc) { + MDBX_txn *const txn = mc->txn; + tASSERT(txn, mc->txn->flags & MDBX_TXN_DIRTY); + tASSERT(txn, F_ISSET(*cursor_dbi_state(mc), DBI_LINDO | DBI_VALID | DBI_DIRTY)); - MDBX_node *node = page_node(mp, i); - if (type == MDBX_page_branch) { - assert(err == MDBX_SUCCESS); - err = walk_tree(ctx, node_pgno(node), name, deep + 1, mp->mp_txnid); - if (unlikely(err != MDBX_SUCCESS)) { - if (err == MDBX_RESULT_TRUE) - break; - return err; - } - continue; + /* If there are any loose pages, just use them */ + while (likely(txn->tw.loose_pages)) { +#if MDBX_ENABLE_REFUND + if (unlikely(txn->tw.loose_refund_wl > txn->geo.first_unallocated)) { + txn_refund(txn); + if (!txn->tw.loose_pages) + break; } +#endif /* MDBX_ENABLE_REFUND */ - assert(type == MDBX_page_leaf); - switch (node_flags(node)) { - default: - continue; - - case F_SUBDATA /* sub-db */: - if (unlikely(node_ds(node) != sizeof(MDBX_db))) { - assert(err == MDBX_CORRUPTED); - err = MDBX_CORRUPTED; - } else { - MDBX_db db; - memcpy(&db, node_data(node), sizeof(db)); - const MDBX_val subdb_name = {node_key(node), node_ks(node)}; - assert(err == MDBX_SUCCESS); - err = walk_sdb(ctx, &db, &subdb_name, deep + 1); - } - break; - - case F_SUBDATA | F_DUPDATA /* dupsorted sub-tree */: - if (unlikely(node_ds(node) != sizeof(MDBX_db) || - ctx->mw_cursor->mc_xcursor == NULL)) { - assert(err == MDBX_CORRUPTED); - err = MDBX_CORRUPTED; - } else { - MDBX_db db; - memcpy(&db, node_data(node), sizeof(db)); - assert(ctx->mw_cursor->mc_xcursor == - &container_of(ctx->mw_cursor, MDBX_cursor_couple, outer)->inner); - assert(err == MDBX_SUCCESS); - err = cursor_xinit1(ctx->mw_cursor, node, mp); - if (likely(err == MDBX_SUCCESS)) { - ctx->mw_cursor = &ctx->mw_cursor->mc_xcursor->mx_cursor; - err = walk_tree(ctx, db.md_root, name, deep + 1, mp->mp_txnid); - MDBX_xcursor *inner_xcursor = - container_of(ctx->mw_cursor, MDBX_xcursor, mx_cursor); - MDBX_cursor_couple *couple = - container_of(inner_xcursor, MDBX_cursor_couple, inner); - ctx->mw_cursor = &couple->outer; - } - } - break; - } + page_t *lp = txn->tw.loose_pages; + MDBX_ASAN_UNPOISON_MEMORY_REGION(lp, txn->env->ps); + VALGRIND_MAKE_MEM_DEFINED(&page_next(lp), sizeof(page_t *)); + txn->tw.loose_pages = page_next(lp); + txn->tw.loose_count--; + DEBUG_EXTRA("db %d use loose page %" PRIaPGNO, cursor_dbi_dbg(mc), lp->pgno); + tASSERT(txn, lp->pgno < txn->geo.first_unallocated); + tASSERT(txn, lp->pgno >= NUM_METAS); + VALGRIND_MAKE_MEM_UNDEFINED(page_data(lp), page_space(txn->env)); + lp->txnid = txn->front_txnid; + pgr_t ret = {lp, MDBX_SUCCESS}; + return ret; } - return MDBX_SUCCESS; -} - -__cold static int walk_sdb(mdbx_walk_ctx_t *ctx, MDBX_db *const sdb, - const MDBX_val *name, int deep) { - if (unlikely(sdb->md_root == P_INVALID)) - return MDBX_SUCCESS; /* empty db */ - - MDBX_cursor_couple couple; - MDBX_dbx dbx = {.md_klen_min = INT_MAX}; - uint8_t dbistate = DBI_VALID | DBI_AUDITED; - int rc = couple_init(&couple, ~0u, ctx->mw_txn, sdb, &dbx, &dbistate); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; + if (likely(MDBX_PNL_GETSIZE(txn->tw.repnl) > 0)) + return page_alloc_finalize(txn->env, txn, mc, repnl_get_single(txn), 1); - couple.outer.mc_checking |= ctx->mw_dont_check_keys_ordering - ? CC_SKIPORD | CC_PAGECHECK - : CC_PAGECHECK; - couple.inner.mx_cursor.mc_checking |= ctx->mw_dont_check_keys_ordering - ? CC_SKIPORD | CC_PAGECHECK - : CC_PAGECHECK; - couple.outer.mc_next = ctx->mw_cursor; - ctx->mw_cursor = &couple.outer; - rc = walk_tree(ctx, sdb->md_root, name, deep, - sdb->md_mod_txnid ? sdb->md_mod_txnid : ctx->mw_txn->mt_txnid); - ctx->mw_cursor = couple.outer.mc_next; - return rc; + return gc_alloc_ex(mc, 1, ALLOC_DEFAULT); } +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 -__cold int mdbx_env_pgwalk(MDBX_txn *txn, MDBX_pgvisitor_func *visitor, - void *user, bool dont_check_keys_ordering) { - int rc = check_txn(txn, MDBX_TXN_BLOCKED); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; +MDBX_NOTHROW_PURE_FUNCTION static bool is_lifo(const MDBX_txn *txn) { + return (txn->env->flags & MDBX_LIFORECLAIM) != 0; +} - mdbx_walk_ctx_t ctx; - memset(&ctx, 0, sizeof(ctx)); - ctx.mw_txn = txn; - ctx.mw_user = user; - ctx.mw_visitor = visitor; - ctx.mw_dont_check_keys_ordering = dont_check_keys_ordering; - - rc = visitor(0, NUM_METAS, user, 0, MDBX_PGWALK_META, - pgno2bytes(txn->mt_env, NUM_METAS), MDBX_page_meta, MDBX_SUCCESS, - NUM_METAS, sizeof(MDBX_meta) * NUM_METAS, PAGEHDRSZ * NUM_METAS, - (txn->mt_env->me_psize - sizeof(MDBX_meta) - PAGEHDRSZ) * - NUM_METAS); - if (!MDBX_IS_ERROR(rc)) - rc = walk_sdb(&ctx, &txn->mt_dbs[FREE_DBI], MDBX_PGWALK_GC, 0); - if (!MDBX_IS_ERROR(rc)) - rc = walk_sdb(&ctx, &txn->mt_dbs[MAIN_DBI], MDBX_PGWALK_MAIN, 0); - return rc; +MDBX_MAYBE_UNUSED static inline const char *dbg_prefix(const gcu_t *ctx) { + return is_lifo(ctx->cursor.txn) ? " lifo" : " fifo"; } -int mdbx_canary_put(MDBX_txn *txn, const MDBX_canary *canary) { - int rc = check_txn_rw(txn, MDBX_TXN_BLOCKED); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; +static inline size_t backlog_size(MDBX_txn *txn) { return MDBX_PNL_GETSIZE(txn->tw.repnl) + txn->tw.loose_count; } - if (likely(canary)) { - if (txn->mt_canary.x == canary->x && txn->mt_canary.y == canary->y && - txn->mt_canary.z == canary->z) - return MDBX_SUCCESS; - txn->mt_canary.x = canary->x; - txn->mt_canary.y = canary->y; - txn->mt_canary.z = canary->z; +static int clean_stored_retired(MDBX_txn *txn, gcu_t *ctx) { + int err = MDBX_SUCCESS; + if (ctx->retired_stored) { + MDBX_cursor *const gc = ptr_disp(txn, sizeof(MDBX_txn)); + tASSERT(txn, txn == txn->env->basal_txn && gc->next == gc); + gc->txn = txn; + gc->dbi_state = txn->dbi_state; + gc->top_and_flags = z_fresh_mark; + gc->next = txn->cursors[FREE_DBI]; + txn->cursors[FREE_DBI] = gc; + do { + MDBX_val key, val; +#if MDBX_ENABLE_BIGFOOT + key.iov_base = &ctx->bigfoot; +#else + key.iov_base = &txn->txnid; +#endif /* MDBX_ENABLE_BIGFOOT */ + key.iov_len = sizeof(txnid_t); + const csr_t csr = cursor_seek(gc, &key, &val, MDBX_SET); + if (csr.err == MDBX_SUCCESS && csr.exact) { + ctx->retired_stored = 0; + err = cursor_del(gc, 0); + TRACE("== clear-4linear, backlog %zu, err %d", backlog_size(txn), err); + } else + err = (csr.err == MDBX_NOTFOUND) ? MDBX_SUCCESS : csr.err; + } +#if MDBX_ENABLE_BIGFOOT + while (!err && --ctx->bigfoot >= txn->txnid); +#else + while (0); +#endif /* MDBX_ENABLE_BIGFOOT */ + txn->cursors[FREE_DBI] = gc->next; + gc->next = gc; } - txn->mt_canary.v = txn->mt_txnid; - txn->mt_flags |= MDBX_TXN_DIRTY; + return err; +} - return MDBX_SUCCESS; +static int touch_gc(gcu_t *ctx) { + tASSERT(ctx->cursor.txn, is_pointed(&ctx->cursor) || ctx->cursor.txn->dbs[FREE_DBI].leaf_pages == 0); + MDBX_val key, val; + key.iov_base = val.iov_base = nullptr; + key.iov_len = sizeof(txnid_t); + val.iov_len = MDBX_PNL_SIZEOF(ctx->cursor.txn->tw.retired_pages); + ctx->cursor.flags |= z_gcu_preparation; + int err = cursor_touch(&ctx->cursor, &key, &val); + ctx->cursor.flags -= z_gcu_preparation; + return err; } -int mdbx_canary_get(const MDBX_txn *txn, MDBX_canary *canary) { - int rc = check_txn(txn, MDBX_TXN_BLOCKED); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; +/* Prepare a backlog of pages to modify GC itself, while reclaiming is + * prohibited. It should be enough to prevent search in gc_alloc_ex() + * during a deleting, when GC tree is unbalanced. */ +static int prepare_backlog(MDBX_txn *txn, gcu_t *ctx) { + const size_t for_cow = txn->dbs[FREE_DBI].height; + const size_t for_rebalance = for_cow + 1 + (txn->dbs[FREE_DBI].height + 1ul >= txn->dbs[FREE_DBI].branch_pages); + size_t for_split = ctx->retired_stored == 0; + tASSERT(txn, is_pointed(&ctx->cursor) || txn->dbs[FREE_DBI].leaf_pages == 0); - if (unlikely(canary == NULL)) - return MDBX_EINVAL; + const intptr_t retired_left = MDBX_PNL_SIZEOF(txn->tw.retired_pages) - ctx->retired_stored; + size_t for_repnl = 0; + if (MDBX_ENABLE_BIGFOOT && retired_left > 0) { + for_repnl = (retired_left + txn->env->maxgc_large1page - 1) / txn->env->maxgc_large1page; + const size_t per_branch_page = txn->env->maxgc_per_branch; + for (size_t entries = for_repnl; entries > 1; for_split += entries) + entries = (entries + per_branch_page - 1) / per_branch_page; + } else if (!MDBX_ENABLE_BIGFOOT && retired_left != 0) { + for_repnl = largechunk_npages(txn->env, MDBX_PNL_SIZEOF(txn->tw.retired_pages)); + } - *canary = txn->mt_canary; - return MDBX_SUCCESS; -} + const size_t for_tree_before_touch = for_cow + for_rebalance + for_split; + const size_t for_tree_after_touch = for_rebalance + for_split; + const size_t for_all_before_touch = for_repnl + for_tree_before_touch; + const size_t for_all_after_touch = for_repnl + for_tree_after_touch; -int mdbx_cursor_on_first(const MDBX_cursor *mc) { - if (unlikely(mc == NULL)) - return MDBX_EINVAL; + if (likely(for_repnl < 2 && backlog_size(txn) > for_all_before_touch) && + (ctx->cursor.top < 0 || is_modifable(txn, ctx->cursor.pg[ctx->cursor.top]))) + return MDBX_SUCCESS; - if (unlikely(mc->mc_signature != MDBX_MC_LIVE)) - return (mc->mc_signature == MDBX_MC_READY4CLOSE) ? MDBX_EINVAL - : MDBX_EBADSIGN; + TRACE(">> retired-stored %zu, left %zi, backlog %zu, need %zu (4list %zu, " + "4split %zu, " + "4cow %zu, 4tree %zu)", + ctx->retired_stored, retired_left, backlog_size(txn), for_all_before_touch, for_repnl, for_split, for_cow, + for_tree_before_touch); - if (!(mc->mc_flags & C_INITIALIZED)) - return mc->mc_db->md_entries ? MDBX_RESULT_FALSE : MDBX_RESULT_TRUE; + int err = touch_gc(ctx); + TRACE("== after-touch, backlog %zu, err %d", backlog_size(txn), err); - for (size_t i = 0; i < mc->mc_snum; ++i) { - if (mc->mc_ki[i]) - return MDBX_RESULT_FALSE; + if (!MDBX_ENABLE_BIGFOOT && unlikely(for_repnl > 1) && + MDBX_PNL_GETSIZE(txn->tw.retired_pages) != ctx->retired_stored && err == MDBX_SUCCESS) { + if (unlikely(ctx->retired_stored)) { + err = clean_stored_retired(txn, ctx); + if (unlikely(err != MDBX_SUCCESS)) + return err; + if (!ctx->retired_stored) + return /* restart by tail-recursion */ prepare_backlog(txn, ctx); + } + err = gc_alloc_ex(&ctx->cursor, for_repnl, ALLOC_RESERVE).err; + TRACE("== after-4linear, backlog %zu, err %d", backlog_size(txn), err); + cASSERT(&ctx->cursor, backlog_size(txn) >= for_repnl || err != MDBX_SUCCESS); } - return MDBX_RESULT_TRUE; -} + while (backlog_size(txn) < for_all_after_touch && err == MDBX_SUCCESS) + err = gc_alloc_ex(&ctx->cursor, 0, ALLOC_RESERVE | ALLOC_UNIMPORTANT).err; -int mdbx_cursor_on_last(const MDBX_cursor *mc) { - if (unlikely(mc == NULL)) - return MDBX_EINVAL; + TRACE("<< backlog %zu, err %d, gc: height %u, branch %zu, leaf %zu, large " + "%zu, entries %zu", + backlog_size(txn), err, txn->dbs[FREE_DBI].height, (size_t)txn->dbs[FREE_DBI].branch_pages, + (size_t)txn->dbs[FREE_DBI].leaf_pages, (size_t)txn->dbs[FREE_DBI].large_pages, + (size_t)txn->dbs[FREE_DBI].items); + tASSERT(txn, err != MDBX_NOTFOUND || (txn->flags & txn_gc_drained) != 0); + return (err != MDBX_NOTFOUND) ? err : MDBX_SUCCESS; +} - if (unlikely(mc->mc_signature != MDBX_MC_LIVE)) - return (mc->mc_signature == MDBX_MC_READY4CLOSE) ? MDBX_EINVAL - : MDBX_EBADSIGN; +static inline void zeroize_reserved(const MDBX_env *env, MDBX_val pnl) { +#if MDBX_DEBUG && (defined(ENABLE_MEMCHECK) || defined(__SANITIZE_ADDRESS__)) + /* Для предотвращения предупреждения Valgrind из mdbx_dump_val() + * вызванное через макрос DVAL_DEBUG() на выходе + * из cursor_seek(MDBX_SET_KEY), которая вызывается ниже внутри gc_update() в + * цикле очистки и цикле заполнения зарезервированных элементов. */ + memset(pnl.iov_base, 0xBB, pnl.iov_len); +#endif /* MDBX_DEBUG && (ENABLE_MEMCHECK || __SANITIZE_ADDRESS__) */ - if (!(mc->mc_flags & C_INITIALIZED)) - return mc->mc_db->md_entries ? MDBX_RESULT_FALSE : MDBX_RESULT_TRUE; + /* PNL is initially empty, zero out at least the length */ + memset(pnl.iov_base, 0, sizeof(pgno_t)); + if ((env->flags & (MDBX_WRITEMAP | MDBX_NOMEMINIT)) == 0) + /* zero out to avoid leaking values from uninitialized malloc'ed memory + * to the file in non-writemap mode if length of the saving page-list + * was changed during space reservation. */ + memset(pnl.iov_base, 0, pnl.iov_len); +} - for (size_t i = 0; i < mc->mc_snum; ++i) { - size_t nkeys = page_numkeys(mc->mc_pg[i]); - if (mc->mc_ki[i] < nkeys - 1) - return MDBX_RESULT_FALSE; +static int gcu_loose(MDBX_txn *txn, gcu_t *ctx) { + tASSERT(txn, txn->tw.loose_count > 0); + /* Return loose page numbers to tw.repnl, though usually none are left at this point. + * The pages themselves remain in dirtylist. */ + if (unlikely(!txn->tw.gc.retxl && txn->tw.gc.last_reclaimed < 1)) { + /* Put loose page numbers in tw.retired_pages, since unable to return ones to tw.repnl. */ + TRACE("%s: merge %zu loose-pages into %s-pages", dbg_prefix(ctx), txn->tw.loose_count, "retired"); + int err = pnl_need(&txn->tw.retired_pages, txn->tw.loose_count); + if (unlikely(err != MDBX_SUCCESS)) + return err; + for (page_t *lp = txn->tw.loose_pages; lp; lp = page_next(lp)) { + pnl_append_prereserved(txn->tw.retired_pages, lp->pgno); + MDBX_ASAN_UNPOISON_MEMORY_REGION(&page_next(lp), sizeof(page_t *)); + VALGRIND_MAKE_MEM_DEFINED(&page_next(lp), sizeof(page_t *)); + } + } else { + /* Room for loose pages + temp PNL with same */ + TRACE("%s: merge %zu loose-pages into %s-pages", dbg_prefix(ctx), txn->tw.loose_count, "reclaimed"); + int err = pnl_need(&txn->tw.repnl, 2 * txn->tw.loose_count + 2); + if (unlikely(err != MDBX_SUCCESS)) + return err; + pnl_t loose = txn->tw.repnl + MDBX_PNL_ALLOCLEN(txn->tw.repnl) - txn->tw.loose_count - 1; + size_t count = 0; + for (page_t *lp = txn->tw.loose_pages; lp; lp = page_next(lp)) { + tASSERT(txn, lp->flags == P_LOOSE); + loose[++count] = lp->pgno; + MDBX_ASAN_UNPOISON_MEMORY_REGION(&page_next(lp), sizeof(page_t *)); + VALGRIND_MAKE_MEM_DEFINED(&page_next(lp), sizeof(page_t *)); + } + tASSERT(txn, count == txn->tw.loose_count); + MDBX_PNL_SETSIZE(loose, count); + pnl_sort(loose, txn->geo.first_unallocated); + pnl_merge(txn->tw.repnl, loose); + } + + /* filter-out list of dirty-pages from loose-pages */ + dpl_t *const dl = txn->tw.dirtylist; + if (dl) { + tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); + tASSERT(txn, dl->sorted <= dl->length); + size_t w = 0, sorted_out = 0; + for (size_t r = w; ++r <= dl->length;) { + page_t *dp = dl->items[r].ptr; + tASSERT(txn, dp->flags == P_LOOSE || is_modifable(txn, dp)); + tASSERT(txn, dpl_endpgno(dl, r) <= txn->geo.first_unallocated); + if ((dp->flags & P_LOOSE) == 0) { + if (++w != r) + dl->items[w] = dl->items[r]; + } else { + tASSERT(txn, dp->flags == P_LOOSE); + sorted_out += dl->sorted >= r; + if (!MDBX_AVOID_MSYNC || !(txn->flags & MDBX_WRITEMAP)) + page_shadow_release(txn->env, dp, 1); + } + } + TRACE("%s: filtered-out loose-pages from %zu -> %zu dirty-pages", dbg_prefix(ctx), dl->length, w); + tASSERT(txn, txn->tw.loose_count == dl->length - w); + dl->sorted -= sorted_out; + tASSERT(txn, dl->sorted <= w); + dpl_setlen(dl, w); + dl->pages_including_loose -= txn->tw.loose_count; + txn->tw.dirtyroom += txn->tw.loose_count; + tASSERT(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length == + (txn->parent ? txn->parent->tw.dirtyroom : txn->env->options.dp_limit)); + } else { + tASSERT(txn, (txn->flags & MDBX_WRITEMAP) != 0 && !MDBX_AVOID_MSYNC); } - - return MDBX_RESULT_TRUE; + txn->tw.loose_pages = nullptr; + txn->tw.loose_count = 0; +#if MDBX_ENABLE_REFUND + txn->tw.loose_refund_wl = 0; +#endif /* MDBX_ENABLE_REFUND */ + return MDBX_SUCCESS; } -int mdbx_cursor_eof(const MDBX_cursor *mc) { - if (unlikely(mc == NULL)) - return MDBX_EINVAL; +static int gcu_retired(MDBX_txn *txn, gcu_t *ctx) { + int err; + if (unlikely(!ctx->retired_stored)) { + /* Make sure last page of GC is touched and on retired-list */ + err = outer_last(&ctx->cursor, nullptr, nullptr); + if (likely(err == MDBX_SUCCESS)) + err = touch_gc(ctx); + if (unlikely(err != MDBX_SUCCESS) && err != MDBX_NOTFOUND) + return err; + } - if (unlikely(mc->mc_signature != MDBX_MC_LIVE)) - return (mc->mc_signature == MDBX_MC_READY4CLOSE) ? MDBX_EINVAL - : MDBX_EBADSIGN; + MDBX_val key, data; +#if MDBX_ENABLE_BIGFOOT + size_t retired_pages_before; + do { + if (ctx->bigfoot > txn->txnid) { + err = clean_stored_retired(txn, ctx); + if (unlikely(err != MDBX_SUCCESS)) + return err; + tASSERT(txn, ctx->bigfoot <= txn->txnid); + } - return ((mc->mc_flags & (C_INITIALIZED | C_EOF)) == C_INITIALIZED && - mc->mc_snum && - mc->mc_ki[mc->mc_top] < page_numkeys(mc->mc_pg[mc->mc_top])) - ? MDBX_RESULT_FALSE - : MDBX_RESULT_TRUE; -} + retired_pages_before = MDBX_PNL_GETSIZE(txn->tw.retired_pages); + err = prepare_backlog(txn, ctx); + if (unlikely(err != MDBX_SUCCESS)) + return err; + if (retired_pages_before != MDBX_PNL_GETSIZE(txn->tw.retired_pages)) { + TRACE("%s: retired-list changed (%zu -> %zu), retry", dbg_prefix(ctx), retired_pages_before, + MDBX_PNL_GETSIZE(txn->tw.retired_pages)); + break; + } -//------------------------------------------------------------------------------ + pnl_sort(txn->tw.retired_pages, txn->geo.first_unallocated); + ctx->retired_stored = 0; + ctx->bigfoot = txn->txnid; + do { + if (ctx->retired_stored) { + err = prepare_backlog(txn, ctx); + if (unlikely(err != MDBX_SUCCESS)) + return err; + if (ctx->retired_stored >= MDBX_PNL_GETSIZE(txn->tw.retired_pages)) { + TRACE("%s: retired-list changed (%zu -> %zu), retry", dbg_prefix(ctx), retired_pages_before, + MDBX_PNL_GETSIZE(txn->tw.retired_pages)); + break; + } + } + key.iov_len = sizeof(txnid_t); + key.iov_base = &ctx->bigfoot; + const size_t left = MDBX_PNL_GETSIZE(txn->tw.retired_pages) - ctx->retired_stored; + const size_t chunk = + (left > txn->env->maxgc_large1page && ctx->bigfoot < MAX_TXNID) ? txn->env->maxgc_large1page : left; + data.iov_len = (chunk + 1) * sizeof(pgno_t); + err = cursor_put(&ctx->cursor, &key, &data, MDBX_RESERVE); + if (unlikely(err != MDBX_SUCCESS)) + return err; -struct diff_result { - ptrdiff_t diff; - size_t level; - ptrdiff_t root_nkeys; -}; +#if MDBX_DEBUG && (defined(ENABLE_MEMCHECK) || defined(__SANITIZE_ADDRESS__)) + /* Для предотвращения предупреждения Valgrind из mdbx_dump_val() + * вызванное через макрос DVAL_DEBUG() на выходе + * из cursor_seek(MDBX_SET_KEY), которая вызывается как выше в цикле + * очистки, так и ниже в цикле заполнения зарезервированных элементов. + */ + memset(data.iov_base, 0xBB, data.iov_len); +#endif /* MDBX_DEBUG && (ENABLE_MEMCHECK || __SANITIZE_ADDRESS__) */ + + if (retired_pages_before == MDBX_PNL_GETSIZE(txn->tw.retired_pages)) { + const size_t at = (is_lifo(txn) == MDBX_PNL_ASCENDING) ? left - chunk : ctx->retired_stored; + pgno_t *const begin = txn->tw.retired_pages + at; + /* MDBX_PNL_ASCENDING == false && LIFO == false: + * - the larger pgno is at the beginning of retired list + * and should be placed with the larger txnid. + * MDBX_PNL_ASCENDING == true && LIFO == true: + * - the larger pgno is at the ending of retired list + * and should be placed with the smaller txnid. */ + const pgno_t save = *begin; + *begin = (pgno_t)chunk; + memcpy(data.iov_base, begin, data.iov_len); + *begin = save; + TRACE("%s: put-retired/bigfoot @ %" PRIaTXN " (slice #%u) #%zu [%zu..%zu] of %zu", dbg_prefix(ctx), + ctx->bigfoot, (unsigned)(ctx->bigfoot - txn->txnid), chunk, at, at + chunk, retired_pages_before); + } + ctx->retired_stored += chunk; + } while (ctx->retired_stored < MDBX_PNL_GETSIZE(txn->tw.retired_pages) && (++ctx->bigfoot, true)); + } while (retired_pages_before != MDBX_PNL_GETSIZE(txn->tw.retired_pages)); +#else + /* Write to last page of GC */ + key.iov_len = sizeof(txnid_t); + key.iov_base = &txn->txnid; + do { + prepare_backlog(txn, ctx); + data.iov_len = MDBX_PNL_SIZEOF(txn->tw.retired_pages); + err = cursor_put(&ctx->cursor, &key, &data, MDBX_RESERVE); + if (unlikely(err != MDBX_SUCCESS)) + return err; -/* calculates: r = x - y */ -__hot static int cursor_diff(const MDBX_cursor *const __restrict x, - const MDBX_cursor *const __restrict y, - struct diff_result *const __restrict r) { - r->diff = 0; - r->level = 0; - r->root_nkeys = 0; +#if MDBX_DEBUG && (defined(ENABLE_MEMCHECK) || defined(__SANITIZE_ADDRESS__)) + /* Для предотвращения предупреждения Valgrind из mdbx_dump_val() + * вызванное через макрос DVAL_DEBUG() на выходе + * из cursor_seek(MDBX_SET_KEY), которая вызывается как выше в цикле + * очистки, так и ниже в цикле заполнения зарезервированных элементов. */ + memset(data.iov_base, 0xBB, data.iov_len); +#endif /* MDBX_DEBUG && (ENABLE_MEMCHECK || __SANITIZE_ADDRESS__) */ - if (unlikely(x->mc_signature != MDBX_MC_LIVE)) - return (x->mc_signature == MDBX_MC_READY4CLOSE) ? MDBX_EINVAL - : MDBX_EBADSIGN; + /* Retry if tw.retired_pages[] grew during the Put() */ + } while (data.iov_len < MDBX_PNL_SIZEOF(txn->tw.retired_pages)); - if (unlikely(y->mc_signature != MDBX_MC_LIVE)) - return (y->mc_signature == MDBX_MC_READY4CLOSE) ? MDBX_EINVAL - : MDBX_EBADSIGN; + ctx->retired_stored = MDBX_PNL_GETSIZE(txn->tw.retired_pages); + pnl_sort(txn->tw.retired_pages, txn->geo.first_unallocated); + tASSERT(txn, data.iov_len == MDBX_PNL_SIZEOF(txn->tw.retired_pages)); + memcpy(data.iov_base, txn->tw.retired_pages, data.iov_len); - int rc = check_txn(x->mc_txn, MDBX_TXN_BLOCKED); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; + TRACE("%s: put-retired #%zu @ %" PRIaTXN, dbg_prefix(ctx), ctx->retired_stored, txn->txnid); +#endif /* MDBX_ENABLE_BIGFOOT */ + if (LOG_ENABLED(MDBX_LOG_EXTRA)) { + size_t i = ctx->retired_stored; + DEBUG_EXTRA("txn %" PRIaTXN " root %" PRIaPGNO " num %zu, retired-PNL", txn->txnid, txn->dbs[FREE_DBI].root, i); + for (; i; i--) + DEBUG_EXTRA_PRINT(" %" PRIaPGNO, txn->tw.retired_pages[i]); + DEBUG_EXTRA_PRINT("%s\n", "."); + } + return MDBX_SUCCESS; +} - if (unlikely(x->mc_txn != y->mc_txn)) - return MDBX_BAD_TXN; +typedef struct gcu_rid_result { + int err; + txnid_t rid; +} rid_t; + +static rid_t get_rid_for_reclaimed(MDBX_txn *txn, gcu_t *ctx, const size_t left) { + rid_t r; + if (is_lifo(txn)) { + if (txn->tw.gc.retxl == nullptr) { + txn->tw.gc.retxl = txl_alloc(); + if (unlikely(!txn->tw.gc.retxl)) { + r.err = MDBX_ENOMEM; + goto return_error; + } + } + if (MDBX_PNL_GETSIZE(txn->tw.gc.retxl) < txl_max && + left > (MDBX_PNL_GETSIZE(txn->tw.gc.retxl) - ctx->reused_slot) * txn->env->maxgc_large1page && !ctx->dense) { + /* Hужен свободный для для сохранения списка страниц. */ + bool need_cleanup = false; + txnid_t snap_oldest = 0; + retry_rid: + do { + r.err = gc_alloc_ex(&ctx->cursor, 0, ALLOC_RESERVE).err; + snap_oldest = txn->env->lck->cached_oldest.weak; + if (likely(r.err == MDBX_SUCCESS)) { + TRACE("%s: took @%" PRIaTXN " from GC", dbg_prefix(ctx), MDBX_PNL_LAST(txn->tw.gc.retxl)); + need_cleanup = true; + } + } while (r.err == MDBX_SUCCESS && MDBX_PNL_GETSIZE(txn->tw.gc.retxl) < txl_max && + left > (MDBX_PNL_GETSIZE(txn->tw.gc.retxl) - ctx->reused_slot) * txn->env->maxgc_large1page); + + if (likely(r.err == MDBX_SUCCESS)) { + TRACE("%s: got enough from GC.", dbg_prefix(ctx)); + goto return_continue; + } else if (unlikely(r.err != MDBX_NOTFOUND)) + /* LY: some troubles... */ + goto return_error; + + if (MDBX_PNL_GETSIZE(txn->tw.gc.retxl)) { + if (need_cleanup) { + txl_sort(txn->tw.gc.retxl); + ctx->cleaned_slot = 0; + } + ctx->rid = MDBX_PNL_LAST(txn->tw.gc.retxl); + } else { + tASSERT(txn, txn->tw.gc.last_reclaimed == 0); + if (unlikely(txn_snapshot_oldest(txn) != snap_oldest)) + /* should retry gc_alloc_ex() + * if the oldest reader changes since the last attempt */ + goto retry_rid; + /* no reclaimable GC entries, + * therefore no entries with ID < mdbx_find_oldest(txn) */ + txn->tw.gc.last_reclaimed = ctx->rid = snap_oldest; + TRACE("%s: none recycled yet, set rid to @%" PRIaTXN, dbg_prefix(ctx), ctx->rid); + } + + /* В GC нет годных к переработке записей, + * будем использовать свободные id в обратном порядке. */ + while (MDBX_PNL_GETSIZE(txn->tw.gc.retxl) < txl_max && + left > (MDBX_PNL_GETSIZE(txn->tw.gc.retxl) - ctx->reused_slot) * txn->env->maxgc_large1page) { + if (unlikely(ctx->rid <= MIN_TXNID)) { + ctx->dense = true; + if (unlikely(MDBX_PNL_GETSIZE(txn->tw.gc.retxl) <= ctx->reused_slot)) { + VERBOSE("** restart: reserve depleted (reused_gc_slot %zu >= " + "gc.reclaimed %zu)", + ctx->reused_slot, MDBX_PNL_GETSIZE(txn->tw.gc.retxl)); + goto return_restart; + } + break; + } - if (unlikely(y->mc_dbi != x->mc_dbi)) - return MDBX_EINVAL; + tASSERT(txn, ctx->rid >= MIN_TXNID && ctx->rid <= MAX_TXNID); + ctx->rid -= 1; + MDBX_val key = {&ctx->rid, sizeof(ctx->rid)}, data; + r.err = cursor_seek(&ctx->cursor, &key, &data, MDBX_SET_KEY).err; + if (unlikely(r.err == MDBX_SUCCESS)) { + DEBUG("%s: GC's id %" PRIaTXN " is present, going to first", dbg_prefix(ctx), ctx->rid); + r.err = outer_first(&ctx->cursor, &key, nullptr); + if (unlikely(r.err != MDBX_SUCCESS || key.iov_len != sizeof(txnid_t))) { + ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid GC-key size", (unsigned)key.iov_len); + r.err = MDBX_CORRUPTED; + goto return_error; + } + const txnid_t gc_first = unaligned_peek_u64(4, key.iov_base); + if (unlikely(gc_first <= INITIAL_TXNID)) { + NOTICE("%s: no free GC's id(s) less than %" PRIaTXN " (going dense-mode)", dbg_prefix(ctx), ctx->rid); + ctx->dense = true; + goto return_restart; + } + ctx->rid = gc_first - 1; + } - if (unlikely(!(y->mc_flags & x->mc_flags & C_INITIALIZED))) - return MDBX_ENODATA; + tASSERT(txn, !ctx->dense); + r.err = txl_append(&txn->tw.gc.retxl, ctx->rid); + if (unlikely(r.err != MDBX_SUCCESS)) + goto return_error; - while (likely(r->level < y->mc_snum && r->level < x->mc_snum)) { - if (unlikely(y->mc_pg[r->level] != x->mc_pg[r->level])) { - ERROR("Mismatch cursors's pages at %zu level", r->level); - return MDBX_PROBLEM; - } + if (ctx->reused_slot) + /* rare case, but it is better to clear and re-create GC entries + * with less fragmentation. */ + need_cleanup = true; + else + ctx->cleaned_slot += 1 /* mark cleanup is not needed for added slot. */; - intptr_t nkeys = page_numkeys(y->mc_pg[r->level]); - assert(nkeys > 0); - if (r->level == 0) - r->root_nkeys = nkeys; + TRACE("%s: append @%" PRIaTXN " to lifo-reclaimed, cleaned-gc-slot = %zu", dbg_prefix(ctx), ctx->rid, + ctx->cleaned_slot); + } - const intptr_t limit_ki = nkeys - 1; - const intptr_t x_ki = x->mc_ki[r->level]; - const intptr_t y_ki = y->mc_ki[r->level]; - r->diff = ((x_ki < limit_ki) ? x_ki : limit_ki) - - ((y_ki < limit_ki) ? y_ki : limit_ki); - if (r->diff == 0) { - r->level += 1; - continue; + if (need_cleanup) { + if (ctx->cleaned_slot) { + TRACE("%s: restart to clear and re-create GC entries", dbg_prefix(ctx)); + goto return_restart; + } + goto return_continue; + } } - while (unlikely(r->diff == 1) && - likely(r->level + 1 < y->mc_snum && r->level + 1 < x->mc_snum)) { - r->level += 1; - /* DB'PAGEs: 0------------------>MAX - * - * CURSORs: y < x - * STACK[i ]: | - * STACK[+1]: ...y++N|0++x... - */ - nkeys = page_numkeys(y->mc_pg[r->level]); - r->diff = (nkeys - y->mc_ki[r->level]) + x->mc_ki[r->level]; - assert(r->diff > 0); + const size_t i = MDBX_PNL_GETSIZE(txn->tw.gc.retxl) - ctx->reused_slot; + tASSERT(txn, i > 0 && i <= MDBX_PNL_GETSIZE(txn->tw.gc.retxl)); + r.rid = txn->tw.gc.retxl[i]; + TRACE("%s: take @%" PRIaTXN " from lifo-reclaimed[%zu]", dbg_prefix(ctx), r.rid, i); + } else { + tASSERT(txn, txn->tw.gc.retxl == nullptr); + if (unlikely(ctx->rid == 0)) { + ctx->rid = txn_snapshot_oldest(txn); + MDBX_val key; + r.err = outer_first(&ctx->cursor, &key, nullptr); + if (likely(r.err == MDBX_SUCCESS)) { + if (unlikely(key.iov_len != sizeof(txnid_t))) { + ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid GC-key size", (unsigned)key.iov_len); + r.err = MDBX_CORRUPTED; + goto return_error; + } + const txnid_t gc_first = unaligned_peek_u64(4, key.iov_base); + if (ctx->rid >= gc_first && gc_first) + ctx->rid = gc_first - 1; + if (unlikely(ctx->rid <= MIN_TXNID)) { + ERROR("%s", "** no GC tail-space to store (going dense-mode)"); + ctx->dense = true; + goto return_restart; + } + } else if (r.err != MDBX_NOTFOUND) { + r.rid = 0; + return r; + } + txn->tw.gc.last_reclaimed = ctx->rid; + ctx->cleaned_id = ctx->rid + 1; } + r.rid = ctx->rid--; + TRACE("%s: take @%" PRIaTXN " from GC", dbg_prefix(ctx), r.rid); + } + ++ctx->reused_slot; + r.err = MDBX_SUCCESS; + return r; - while (unlikely(r->diff == -1) && - likely(r->level + 1 < y->mc_snum && r->level + 1 < x->mc_snum)) { - r->level += 1; - /* DB'PAGEs: 0------------------>MAX - * - * CURSORs: x < y - * STACK[i ]: | - * STACK[+1]: ...x--N|0--y... - */ - nkeys = page_numkeys(x->mc_pg[r->level]); - r->diff = -(nkeys - x->mc_ki[r->level]) - y->mc_ki[r->level]; - assert(r->diff < 0); - } +return_continue: + r.err = MDBX_SUCCESS; + r.rid = 0; + return r; - return MDBX_SUCCESS; - } +return_restart: + r.err = MDBX_RESULT_TRUE; + r.rid = 0; + return r; - r->diff = CMP2INT(x->mc_flags & C_EOF, y->mc_flags & C_EOF); - return MDBX_SUCCESS; +return_error: + tASSERT(txn, r.err != MDBX_SUCCESS); + r.rid = 0; + return r; } -__hot static ptrdiff_t estimate(const MDBX_db *db, - struct diff_result *const __restrict dr) { - /* root: branch-page => scale = leaf-factor * branch-factor^(N-1) - * level-1: branch-page(s) => scale = leaf-factor * branch-factor^2 - * level-2: branch-page(s) => scale = leaf-factor * branch-factor - * level-N: branch-page(s) => scale = leaf-factor - * leaf-level: leaf-page(s) => scale = 1 - */ - ptrdiff_t btree_power = (ptrdiff_t)db->md_depth - 2 - (ptrdiff_t)dr->level; - if (btree_power < 0) - return dr->diff; - - ptrdiff_t estimated = - (ptrdiff_t)db->md_entries * dr->diff / (ptrdiff_t)db->md_leaf_pages; - if (btree_power == 0) - return estimated; +/* Cleanups retxl GC (aka freeDB) records, saves the retired-list (aka + * freelist) of current transaction to GC, puts back into GC leftover of the + * retxl pages with chunking. This recursive changes the retxl-list, + * loose-list and retired-list. Keep trying until it stabilizes. + * + * NOTE: This code is a consequence of many iterations of adding crutches (aka + * "checks and balances") to partially bypass the fundamental design problems + * inherited from LMDB. So do not try to understand it completely in order to + * avoid your madness. */ +int gc_update(MDBX_txn *txn, gcu_t *ctx) { + TRACE("\n>>> @%" PRIaTXN, txn->txnid); + MDBX_env *const env = txn->env; + ctx->cursor.next = txn->cursors[FREE_DBI]; + txn->cursors[FREE_DBI] = &ctx->cursor; + int rc; - if (db->md_depth < 4) { - assert(dr->level == 0 && btree_power == 1); - return (ptrdiff_t)db->md_entries * dr->diff / (ptrdiff_t)dr->root_nkeys; - } + /* txn->tw.repnl[] can grow and shrink during this call. + * txn->tw.gc.last_reclaimed and txn->tw.retired_pages[] can only grow. + * But page numbers cannot disappear from txn->tw.retired_pages[]. */ +retry_clean_adj: + ctx->reserve_adj = 0; +retry: + ctx->loop += !(ctx->prev_first_unallocated > txn->geo.first_unallocated); + TRACE(">> restart, loop %u", ctx->loop); - /* average_branchpage_fillfactor = total(branch_entries) / branch_pages - total(branch_entries) = leaf_pages + branch_pages - 1 (root page) */ - const size_t log2_fixedpoint = sizeof(size_t) - 1; - const size_t half = UINT64_C(1) << (log2_fixedpoint - 1); - const size_t factor = - ((db->md_leaf_pages + db->md_branch_pages - 1) << log2_fixedpoint) / - db->md_branch_pages; - while (1) { - switch ((size_t)btree_power) { - default: { - const size_t square = (factor * factor + half) >> log2_fixedpoint; - const size_t quad = (square * square + half) >> log2_fixedpoint; - do { - estimated = estimated * quad + half; - estimated >>= log2_fixedpoint; - btree_power -= 4; - } while (btree_power >= 4); - continue; - } - case 3: - estimated = estimated * factor + half; - estimated >>= log2_fixedpoint; - __fallthrough /* fall through */; - case 2: - estimated = estimated * factor + half; - estimated >>= log2_fixedpoint; - __fallthrough /* fall through */; - case 1: - estimated = estimated * factor + half; - estimated >>= log2_fixedpoint; - __fallthrough /* fall through */; - case 0: - if (unlikely(estimated > (ptrdiff_t)db->md_entries)) - return (ptrdiff_t)db->md_entries; - if (unlikely(estimated < -(ptrdiff_t)db->md_entries)) - return -(ptrdiff_t)db->md_entries; - return estimated; - } + tASSERT(txn, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); + tASSERT(txn, dpl_check(txn)); + if (unlikely(/* paranoia */ ctx->loop > ((MDBX_DEBUG > 0) ? 12 : 42))) { + ERROR("txn #%" PRIaTXN " too more loops %u, bailout", txn->txnid, ctx->loop); + rc = MDBX_PROBLEM; + goto bailout; } -} - -int mdbx_estimate_distance(const MDBX_cursor *first, const MDBX_cursor *last, - ptrdiff_t *distance_items) { - if (unlikely(first == NULL || last == NULL || distance_items == NULL)) - return MDBX_EINVAL; - - *distance_items = 0; - struct diff_result dr; - int rc = cursor_diff(last, first, &dr); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - if (unlikely(dr.diff == 0) && - F_ISSET(first->mc_db->md_flags & last->mc_db->md_flags, - MDBX_DUPSORT | C_INITIALIZED)) { - first = &first->mc_xcursor->mx_cursor; - last = &last->mc_xcursor->mx_cursor; - rc = cursor_diff(first, last, &dr); + if (unlikely(ctx->dense || ctx->prev_first_unallocated > txn->geo.first_unallocated)) { + rc = clean_stored_retired(txn, ctx); if (unlikely(rc != MDBX_SUCCESS)) - return rc; + goto bailout; } - if (likely(dr.diff != 0)) - *distance_items = estimate(first->mc_db, &dr); + ctx->prev_first_unallocated = txn->geo.first_unallocated; + rc = MDBX_SUCCESS; + ctx->reserved = 0; + ctx->cleaned_slot = 0; + ctx->reused_slot = 0; + ctx->amount = 0; + ctx->fill_idx = ~0u; + ctx->cleaned_id = 0; + ctx->rid = txn->tw.gc.last_reclaimed; + while (true) { + /* Come back here after each Put() in case retired-list changed */ + TRACE("%s", " >> continue"); - return MDBX_SUCCESS; -} + tASSERT(txn, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); + MDBX_val key, data; + if (is_lifo(txn)) { + if (ctx->cleaned_slot < (txn->tw.gc.retxl ? MDBX_PNL_GETSIZE(txn->tw.gc.retxl) : 0)) { + ctx->reserved = 0; + ctx->cleaned_slot = 0; + ctx->reused_slot = 0; + ctx->fill_idx = ~0u; + /* LY: cleanup reclaimed records. */ + do { + ctx->cleaned_id = txn->tw.gc.retxl[++ctx->cleaned_slot]; + tASSERT(txn, ctx->cleaned_slot > 0 && ctx->cleaned_id <= env->lck->cached_oldest.weak); + key.iov_base = &ctx->cleaned_id; + key.iov_len = sizeof(ctx->cleaned_id); + rc = cursor_seek(&ctx->cursor, &key, nullptr, MDBX_SET).err; + if (rc == MDBX_NOTFOUND) + continue; + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + rc = prepare_backlog(txn, ctx); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + tASSERT(txn, ctx->cleaned_id <= env->lck->cached_oldest.weak); + TRACE("%s: cleanup-reclaimed-id [%zu]%" PRIaTXN, dbg_prefix(ctx), ctx->cleaned_slot, ctx->cleaned_id); + tASSERT(txn, *txn->cursors == &ctx->cursor); + rc = cursor_del(&ctx->cursor, 0); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + } while (ctx->cleaned_slot < MDBX_PNL_GETSIZE(txn->tw.gc.retxl)); + txl_sort(txn->tw.gc.retxl); + } + } else { + /* Удаляем оставшиеся вынутые из GC записи. */ + while (txn->tw.gc.last_reclaimed && ctx->cleaned_id <= txn->tw.gc.last_reclaimed) { + rc = outer_first(&ctx->cursor, &key, nullptr); + if (rc == MDBX_NOTFOUND) { + ctx->cleaned_id = txn->tw.gc.last_reclaimed + 1; + ctx->rid = txn->tw.gc.last_reclaimed; + ctx->reserved = 0; + ctx->reused_slot = 0; + break; + } + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + if (!MDBX_DISABLE_VALIDATION && unlikely(key.iov_len != sizeof(txnid_t))) { + ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid GC-key size", (unsigned)key.iov_len); + rc = MDBX_CORRUPTED; + goto bailout; + } + if (ctx->rid != ctx->cleaned_id) { + ctx->rid = ctx->cleaned_id; + ctx->reserved = 0; + ctx->reused_slot = 0; + } + ctx->cleaned_id = unaligned_peek_u64(4, key.iov_base); + if (ctx->cleaned_id > txn->tw.gc.last_reclaimed) + break; + rc = prepare_backlog(txn, ctx); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + tASSERT(txn, ctx->cleaned_id <= txn->tw.gc.last_reclaimed); + tASSERT(txn, ctx->cleaned_id <= env->lck->cached_oldest.weak); + TRACE("%s: cleanup-reclaimed-id %" PRIaTXN, dbg_prefix(ctx), ctx->cleaned_id); + tASSERT(txn, *txn->cursors == &ctx->cursor); + rc = cursor_del(&ctx->cursor, 0); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + } + } -int mdbx_estimate_move(const MDBX_cursor *cursor, MDBX_val *key, MDBX_val *data, - MDBX_cursor_op move_op, ptrdiff_t *distance_items) { - if (unlikely(cursor == NULL || distance_items == NULL || - move_op == MDBX_GET_CURRENT || move_op == MDBX_GET_MULTIPLE)) - return MDBX_EINVAL; + tASSERT(txn, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); + tASSERT(txn, dpl_check(txn)); + if (AUDIT_ENABLED()) { + rc = audit_ex(txn, ctx->retired_stored, false); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + } - if (unlikely(cursor->mc_signature != MDBX_MC_LIVE)) - return (cursor->mc_signature == MDBX_MC_READY4CLOSE) ? MDBX_EINVAL - : MDBX_EBADSIGN; + /* return suitable into unallocated space */ + if (txn_refund(txn)) { + tASSERT(txn, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); + if (AUDIT_ENABLED()) { + rc = audit_ex(txn, ctx->retired_stored, false); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + } + } - int rc = check_txn(cursor->mc_txn, MDBX_TXN_BLOCKED); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - - if (!(cursor->mc_flags & C_INITIALIZED)) - return MDBX_ENODATA; + if (txn->tw.loose_pages) { + /* put loose pages into the reclaimed- or retired-list */ + rc = gcu_loose(txn, ctx); + if (unlikely(rc != MDBX_SUCCESS)) { + if (rc == MDBX_RESULT_TRUE) + continue; + goto bailout; + } + tASSERT(txn, txn->tw.loose_pages == 0); + } - MDBX_cursor_couple next; - cursor_copy(cursor, &next.outer); - if (cursor->mc_db->md_flags & MDBX_DUPSORT) { - next.outer.mc_xcursor = &next.inner; - rc = cursor_xinit0(&next.outer); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - MDBX_xcursor *mx = &container_of(cursor, MDBX_cursor_couple, outer)->inner; - cursor_copy(&mx->mx_cursor, &next.inner.mx_cursor); - } + if (unlikely(ctx->reserved > MDBX_PNL_GETSIZE(txn->tw.repnl)) && + (ctx->loop < 5 || ctx->reserved - MDBX_PNL_GETSIZE(txn->tw.repnl) > env->maxgc_large1page / 2)) { + TRACE("%s: reclaimed-list changed %zu -> %zu, retry", dbg_prefix(ctx), ctx->amount, + MDBX_PNL_GETSIZE(txn->tw.repnl)); + ctx->reserve_adj += ctx->reserved - MDBX_PNL_GETSIZE(txn->tw.repnl); + goto retry; + } + ctx->amount = MDBX_PNL_GETSIZE(txn->tw.repnl); - MDBX_val stub = {0, 0}; - if (data == NULL) { - const unsigned mask = - 1 << MDBX_GET_BOTH | 1 << MDBX_GET_BOTH_RANGE | 1 << MDBX_SET_KEY; - if (unlikely(mask & (1 << move_op))) - return MDBX_EINVAL; - data = &stub; - } + if (ctx->retired_stored < MDBX_PNL_GETSIZE(txn->tw.retired_pages)) { + /* store retired-list into GC */ + rc = gcu_retired(txn, ctx); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + continue; + } - if (key == NULL) { - const unsigned mask = 1 << MDBX_GET_BOTH | 1 << MDBX_GET_BOTH_RANGE | - 1 << MDBX_SET_KEY | 1 << MDBX_SET | - 1 << MDBX_SET_RANGE; - if (unlikely(mask & (1 << move_op))) - return MDBX_EINVAL; - key = &stub; - } + tASSERT(txn, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); + tASSERT(txn, txn->tw.loose_count == 0); - next.outer.mc_signature = MDBX_MC_LIVE; - rc = cursor_get(&next.outer, key, data, move_op); - if (unlikely(rc != MDBX_SUCCESS && - (rc != MDBX_NOTFOUND || !(next.outer.mc_flags & C_INITIALIZED)))) - return rc; + TRACE("%s", " >> reserving"); + if (AUDIT_ENABLED()) { + rc = audit_ex(txn, ctx->retired_stored, false); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + } + const size_t left = ctx->amount - ctx->reserved - ctx->reserve_adj; + TRACE("%s: amount %zu, reserved %zd, reserve_adj %zu, left %zd, " + "lifo-reclaimed-slots %zu, " + "reused-gc-slots %zu", + dbg_prefix(ctx), ctx->amount, ctx->reserved, ctx->reserve_adj, left, + txn->tw.gc.retxl ? MDBX_PNL_GETSIZE(txn->tw.gc.retxl) : 0, ctx->reused_slot); + if (0 >= (intptr_t)left) + break; - return mdbx_estimate_distance(cursor, &next.outer, distance_items); -} + const rid_t rid_result = get_rid_for_reclaimed(txn, ctx, left); + if (unlikely(!rid_result.rid)) { + rc = rid_result.err; + if (likely(rc == MDBX_SUCCESS)) + continue; + if (likely(rc == MDBX_RESULT_TRUE)) + goto retry; + goto bailout; + } + tASSERT(txn, rid_result.err == MDBX_SUCCESS); + const txnid_t reservation_gc_id = rid_result.rid; -int mdbx_estimate_range(const MDBX_txn *txn, MDBX_dbi dbi, - const MDBX_val *begin_key, const MDBX_val *begin_data, - const MDBX_val *end_key, const MDBX_val *end_data, - ptrdiff_t *size_items) { - int rc = check_txn(txn, MDBX_TXN_BLOCKED); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; + size_t chunk = left; + if (unlikely(left > env->maxgc_large1page)) { + const size_t avail_gc_slots = txn->tw.gc.retxl ? MDBX_PNL_GETSIZE(txn->tw.gc.retxl) - ctx->reused_slot + 1 + : (ctx->rid < INT16_MAX) ? (size_t)ctx->rid + : INT16_MAX; + if (likely(avail_gc_slots > 1)) { +#if MDBX_ENABLE_BIGFOOT + chunk = env->maxgc_large1page; + if (avail_gc_slots < INT16_MAX && unlikely(left > env->maxgc_large1page * avail_gc_slots)) + /* TODO: Можно смотреть последовательности какой длины есть в repnl + * и пробовать нарезать куски соответствующего размера. + * Смысл в том, чтобы не дробить последовательности страниц, + * а использовать целиком. */ + chunk = env->maxgc_large1page + left / (env->maxgc_large1page * avail_gc_slots) * env->maxgc_large1page; +#else + if (chunk < env->maxgc_large1page * 2) + chunk /= 2; + else { + const size_t prefer_max_scatter = 257; + const size_t threshold = + env->maxgc_large1page * ((avail_gc_slots < prefer_max_scatter) ? avail_gc_slots : prefer_max_scatter); + if (left < threshold) + chunk = env->maxgc_large1page; + else { + const size_t tail = left - threshold + env->maxgc_large1page + 1; + size_t span = 1; + size_t avail = ((pgno2bytes(env, span) - PAGEHDRSZ) / sizeof(pgno_t)) /* - 1 + span */; + if (tail > avail) { + for (size_t i = ctx->amount - span; i > 0; --i) { + if (MDBX_PNL_ASCENDING ? (txn->tw.repnl[i] + span) + : (txn->tw.repnl[i] - span) == txn->tw.repnl[i + span]) { + span += 1; + avail = ((pgno2bytes(env, span) - PAGEHDRSZ) / sizeof(pgno_t)) - 1 + span; + if (avail >= tail) + break; + } + } + } - if (unlikely(!size_items)) - return MDBX_EINVAL; + chunk = (avail >= tail) ? tail - span + : (avail_gc_slots > 3 && ctx->reused_slot < prefer_max_scatter - 3) ? avail - span + : tail; + } + } +#endif /* MDBX_ENABLE_BIGFOOT */ + } + } + tASSERT(txn, chunk > 0); - if (unlikely(begin_data && (begin_key == NULL || begin_key == MDBX_EPSILON))) - return MDBX_EINVAL; + TRACE("%s: gc_rid %" PRIaTXN ", reused_gc_slot %zu, reservation-id " + "%" PRIaTXN, + dbg_prefix(ctx), ctx->rid, ctx->reused_slot, reservation_gc_id); - if (unlikely(end_data && (end_key == NULL || end_key == MDBX_EPSILON))) - return MDBX_EINVAL; + TRACE("%s: chunk %zu, gc-per-ovpage %u", dbg_prefix(ctx), chunk, env->maxgc_large1page); - if (unlikely(begin_key == MDBX_EPSILON && end_key == MDBX_EPSILON)) - return MDBX_EINVAL; + tASSERT(txn, reservation_gc_id <= env->lck->cached_oldest.weak); + if (unlikely(reservation_gc_id < MIN_TXNID || + reservation_gc_id > atomic_load64(&env->lck->cached_oldest, mo_Relaxed))) { + ERROR("** internal error (reservation_gc_id %" PRIaTXN ")", reservation_gc_id); + rc = MDBX_PROBLEM; + goto bailout; + } - if (unlikely(!check_dbi(txn, dbi, DBI_USRVALID))) - return MDBX_BAD_DBI; + tASSERT(txn, reservation_gc_id >= MIN_TXNID && reservation_gc_id <= MAX_TXNID); + key.iov_len = sizeof(reservation_gc_id); + key.iov_base = (void *)&reservation_gc_id; + data.iov_len = (chunk + 1) * sizeof(pgno_t); + TRACE("%s: reserve %zu [%zu...%zu) @%" PRIaTXN, dbg_prefix(ctx), chunk, ctx->reserved + 1, + ctx->reserved + chunk + 1, reservation_gc_id); + prepare_backlog(txn, ctx); + rc = cursor_put(&ctx->cursor, &key, &data, MDBX_RESERVE | MDBX_NOOVERWRITE); + tASSERT(txn, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; - MDBX_cursor_couple begin; - /* LY: first, initialize cursor to refresh a DB in case it have DB_STALE */ - rc = cursor_init(&begin.outer, txn, dbi); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; + zeroize_reserved(env, data); + ctx->reserved += chunk; + TRACE("%s: reserved %zu (+%zu), continue", dbg_prefix(ctx), ctx->reserved, chunk); - if (unlikely(begin.outer.mc_db->md_entries == 0)) { - *size_items = 0; - return MDBX_SUCCESS; + continue; } - MDBX_val stub; - if (!begin_key) { - if (unlikely(!end_key)) { - /* LY: FIRST..LAST case */ - *size_items = (ptrdiff_t)begin.outer.mc_db->md_entries; - return MDBX_SUCCESS; - } - rc = cursor_first(&begin.outer, &stub, &stub); - if (unlikely(end_key == MDBX_EPSILON)) { - /* LY: FIRST..+epsilon case */ - return (rc == MDBX_SUCCESS) - ? mdbx_cursor_count(&begin.outer, (size_t *)size_items) - : rc; - } - } else { - if (unlikely(begin_key == MDBX_EPSILON)) { - if (end_key == NULL) { - /* LY: -epsilon..LAST case */ - rc = cursor_last(&begin.outer, &stub, &stub); - return (rc == MDBX_SUCCESS) - ? mdbx_cursor_count(&begin.outer, (size_t *)size_items) - : rc; + tASSERT(txn, ctx->cleaned_slot == (txn->tw.gc.retxl ? MDBX_PNL_GETSIZE(txn->tw.gc.retxl) : 0)); + + TRACE("%s", " >> filling"); + /* Fill in the reserved records */ + size_t excess_slots = 0; + ctx->fill_idx = txn->tw.gc.retxl ? MDBX_PNL_GETSIZE(txn->tw.gc.retxl) - ctx->reused_slot : ctx->reused_slot; + rc = MDBX_SUCCESS; + tASSERT(txn, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); + tASSERT(txn, dpl_check(txn)); + if (ctx->amount) { + MDBX_val key, data; + key.iov_len = data.iov_len = 0; + key.iov_base = data.iov_base = nullptr; + + size_t left = ctx->amount, excess = 0; + if (txn->tw.gc.retxl == nullptr) { + tASSERT(txn, is_lifo(txn) == 0); + rc = outer_first(&ctx->cursor, &key, &data); + if (unlikely(rc != MDBX_SUCCESS)) { + if (rc != MDBX_NOTFOUND) + goto bailout; } - /* LY: -epsilon..value case */ - assert(end_key != MDBX_EPSILON); - begin_key = end_key; - } else if (unlikely(end_key == MDBX_EPSILON)) { - /* LY: value..+epsilon case */ - assert(begin_key != MDBX_EPSILON); - end_key = begin_key; + } else { + tASSERT(txn, is_lifo(txn) != 0); } - if (end_key && !begin_data && !end_data && - (begin_key == end_key || - begin.outer.mc_dbx->md_cmp(begin_key, end_key) == 0)) { - /* LY: single key case */ - rc = cursor_set(&begin.outer, (MDBX_val *)begin_key, NULL, MDBX_SET).err; - if (unlikely(rc != MDBX_SUCCESS)) { - *size_items = 0; - return (rc == MDBX_NOTFOUND) ? MDBX_SUCCESS : rc; + + while (true) { + txnid_t fill_gc_id; + TRACE("%s: left %zu of %zu", dbg_prefix(ctx), left, MDBX_PNL_GETSIZE(txn->tw.repnl)); + if (txn->tw.gc.retxl == nullptr) { + tASSERT(txn, is_lifo(txn) == 0); + fill_gc_id = key.iov_base ? unaligned_peek_u64(4, key.iov_base) : MIN_TXNID; + if (ctx->fill_idx == 0 || fill_gc_id > txn->tw.gc.last_reclaimed) { + if (!left) + break; + VERBOSE("** restart: reserve depleted (fill_idx %zu, fill_id %" PRIaTXN " > last_reclaimed %" PRIaTXN + ", left %zu", + ctx->fill_idx, fill_gc_id, txn->tw.gc.last_reclaimed, left); + ctx->reserve_adj = (ctx->reserve_adj > left) ? ctx->reserve_adj - left : 0; + goto retry; + } + ctx->fill_idx -= 1; + } else { + tASSERT(txn, is_lifo(txn) != 0); + if (ctx->fill_idx >= MDBX_PNL_GETSIZE(txn->tw.gc.retxl)) { + if (!left) + break; + VERBOSE("** restart: reserve depleted (fill_idx %zu >= " + "gc.reclaimed %zu, left %zu", + ctx->fill_idx, MDBX_PNL_GETSIZE(txn->tw.gc.retxl), left); + ctx->reserve_adj = (ctx->reserve_adj > left) ? ctx->reserve_adj - left : 0; + goto retry; + } + ctx->fill_idx += 1; + fill_gc_id = txn->tw.gc.retxl[ctx->fill_idx]; + TRACE("%s: seek-reservation @%" PRIaTXN " at gc.reclaimed[%zu]", dbg_prefix(ctx), fill_gc_id, ctx->fill_idx); + key.iov_base = &fill_gc_id; + key.iov_len = sizeof(fill_gc_id); + rc = cursor_seek(&ctx->cursor, &key, &data, MDBX_SET_KEY).err; + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; } - *size_items = 1; - if (begin.outer.mc_xcursor != NULL) { - MDBX_node *node = page_node(begin.outer.mc_pg[begin.outer.mc_top], - begin.outer.mc_ki[begin.outer.mc_top]); - if (node_flags(node) & F_DUPDATA) { - /* LY: return the number of duplicates for given key */ - tASSERT(txn, begin.outer.mc_xcursor == &begin.inner && - (begin.inner.mx_cursor.mc_flags & C_INITIALIZED)); - *size_items = - (sizeof(*size_items) >= sizeof(begin.inner.mx_db.md_entries) || - begin.inner.mx_db.md_entries <= PTRDIFF_MAX) - ? (size_t)begin.inner.mx_db.md_entries - : PTRDIFF_MAX; + tASSERT(txn, ctx->cleaned_slot == (txn->tw.gc.retxl ? MDBX_PNL_GETSIZE(txn->tw.gc.retxl) : 0)); + tASSERT(txn, fill_gc_id > 0 && fill_gc_id <= env->lck->cached_oldest.weak); + key.iov_base = &fill_gc_id; + key.iov_len = sizeof(fill_gc_id); + + tASSERT(txn, data.iov_len >= sizeof(pgno_t) * 2); + size_t chunk = data.iov_len / sizeof(pgno_t) - 1; + if (unlikely(chunk > left)) { + const size_t delta = chunk - left; + excess += delta; + TRACE("%s: chunk %zu > left %zu, @%" PRIaTXN, dbg_prefix(ctx), chunk, left, fill_gc_id); + if (!left) { + excess_slots += 1; + goto next; } + if ((ctx->loop < 5 && delta > (ctx->loop / 2)) || delta > env->maxgc_large1page) + data.iov_len = (left + 1) * sizeof(pgno_t); + chunk = left; } - return MDBX_SUCCESS; - } else if (begin_data) { - stub = *begin_data; - rc = cursor_set(&begin.outer, (MDBX_val *)begin_key, &stub, - MDBX_GET_BOTH_RANGE) - .err; - } else { - stub = *begin_key; - rc = cursor_set(&begin.outer, &stub, nullptr, MDBX_SET_RANGE).err; - } - } + rc = cursor_put(&ctx->cursor, &key, &data, MDBX_CURRENT | MDBX_RESERVE); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + zeroize_reserved(env, data); - if (unlikely(rc != MDBX_SUCCESS)) { - if (rc != MDBX_NOTFOUND || !(begin.outer.mc_flags & C_INITIALIZED)) - return rc; - } + if (unlikely(txn->tw.loose_count || ctx->amount != MDBX_PNL_GETSIZE(txn->tw.repnl))) { + NOTICE("** restart: reclaimed-list changed (%zu -> %zu, loose +%zu)", ctx->amount, + MDBX_PNL_GETSIZE(txn->tw.repnl), txn->tw.loose_count); + if (ctx->loop < 5 || (ctx->loop > 10 && (ctx->loop & 1))) + goto retry_clean_adj; + goto retry; + } - MDBX_cursor_couple end; - rc = cursor_init(&end.outer, txn, dbi); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - if (!end_key) - rc = cursor_last(&end.outer, &stub, &stub); - else if (end_data) { - stub = *end_data; - rc = cursor_set(&end.outer, (MDBX_val *)end_key, &stub, MDBX_GET_BOTH_RANGE) - .err; - } else { - stub = *end_key; - rc = cursor_set(&end.outer, &stub, nullptr, MDBX_SET_RANGE).err; - } - if (unlikely(rc != MDBX_SUCCESS)) { - if (rc != MDBX_NOTFOUND || !(end.outer.mc_flags & C_INITIALIZED)) - return rc; - } + if (unlikely(txn->tw.gc.retxl ? ctx->cleaned_slot < MDBX_PNL_GETSIZE(txn->tw.gc.retxl) + : ctx->cleaned_id < txn->tw.gc.last_reclaimed)) { + NOTICE("%s", "** restart: reclaimed-slots changed"); + goto retry; + } + if (unlikely(ctx->retired_stored != MDBX_PNL_GETSIZE(txn->tw.retired_pages))) { + tASSERT(txn, ctx->retired_stored < MDBX_PNL_GETSIZE(txn->tw.retired_pages)); + NOTICE("** restart: retired-list growth (%zu -> %zu)", ctx->retired_stored, + MDBX_PNL_GETSIZE(txn->tw.retired_pages)); + goto retry; + } - rc = mdbx_estimate_distance(&begin.outer, &end.outer, size_items); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - assert(*size_items >= -(ptrdiff_t)begin.outer.mc_db->md_entries && - *size_items <= (ptrdiff_t)begin.outer.mc_db->md_entries); + pgno_t *dst = data.iov_base; + *dst++ = (pgno_t)chunk; + pgno_t *src = MDBX_PNL_BEGIN(txn->tw.repnl) + left - chunk; + memcpy(dst, src, chunk * sizeof(pgno_t)); + pgno_t *from = src, *to = src + chunk; + TRACE("%s: fill %zu [ %zu:%" PRIaPGNO "...%zu:%" PRIaPGNO "] @%" PRIaTXN, dbg_prefix(ctx), chunk, + from - txn->tw.repnl, from[0], to - txn->tw.repnl, to[-1], fill_gc_id); -#if 0 /* LY: Was decided to returns as-is (i.e. negative) the estimation \ - * results for an inverted ranges. */ + left -= chunk; + if (AUDIT_ENABLED()) { + rc = audit_ex(txn, ctx->retired_stored + ctx->amount - left, true); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + } - /* Commit 8ddfd1f34ad7cf7a3c4aa75d2e248ca7e639ed63 - Change-Id: If59eccf7311123ab6384c4b93f9b1fed5a0a10d1 */ + next: - if (*size_items < 0) { - /* LY: inverted range case */ - *size_items += (ptrdiff_t)begin.outer.mc_db->md_entries; - } else if (*size_items == 0 && begin_key && end_key) { - int cmp = begin.outer.mc_dbx->md_cmp(&origin_begin_key, &origin_end_key); - if (cmp == 0 && (begin.inner.mx_cursor.mc_flags & C_INITIALIZED) && - begin_data && end_data) - cmp = begin.outer.mc_dbx->md_dcmp(&origin_begin_data, &origin_end_data); - if (cmp > 0) { - /* LY: inverted range case with empty scope */ - *size_items = (ptrdiff_t)begin.outer.mc_db->md_entries; + if (txn->tw.gc.retxl == nullptr) { + tASSERT(txn, is_lifo(txn) == 0); + rc = outer_next(&ctx->cursor, &key, &data, MDBX_NEXT); + if (unlikely(rc != MDBX_SUCCESS)) { + if (rc == MDBX_NOTFOUND && !left) { + rc = MDBX_SUCCESS; + break; + } + goto bailout; + } + } else { + tASSERT(txn, is_lifo(txn) != 0); + } } - } - assert(*size_items >= 0 && - *size_items <= (ptrdiff_t)begin.outer.mc_db->md_entries); -#endif - return MDBX_SUCCESS; -} + if (excess) { + size_t n = excess, adj = excess; + while (n >= env->maxgc_large1page) + adj -= n /= env->maxgc_large1page; + ctx->reserve_adj += adj; + TRACE("%s: extra %zu reserved space, adj +%zu (%zu)", dbg_prefix(ctx), excess, adj, ctx->reserve_adj); + } + } -//------------------------------------------------------------------------------ + tASSERT(txn, rc == MDBX_SUCCESS); + if (unlikely(txn->tw.loose_count != 0 || ctx->amount != MDBX_PNL_GETSIZE(txn->tw.repnl))) { + NOTICE("** restart: got %zu loose pages (reclaimed-list %zu -> %zu)", txn->tw.loose_count, ctx->amount, + MDBX_PNL_GETSIZE(txn->tw.repnl)); + goto retry; + } -/* Позволяет обновить или удалить существующую запись с получением - * в old_data предыдущего значения данных. При этом если new_data равен - * нулю, то выполняется удаление, иначе обновление/вставка. - * - * Текущее значение может находиться в уже измененной (грязной) странице. - * В этом случае страница будет перезаписана при обновлении, а само старое - * значение утрачено. Поэтому исходно в old_data должен быть передан - * дополнительный буфер для копирования старого значения. - * Если переданный буфер слишком мал, то функция вернет -1, установив - * old_data->iov_len в соответствующее значение. - * - * Для не-уникальных ключей также возможен второй сценарий использования, - * когда посредством old_data из записей с одинаковым ключом для - * удаления/обновления выбирается конкретная. Для выбора этого сценария - * во flags следует одновременно указать MDBX_CURRENT и MDBX_NOOVERWRITE. - * Именно эта комбинация выбрана, так как она лишена смысла, и этим позволяет - * идентифицировать запрос такого сценария. - * - * Функция может быть замещена соответствующими операциями с курсорами - * после двух доработок (TODO): - * - внешняя аллокация курсоров, в том числе на стеке (без malloc). - * - получения dirty-статуса страницы по адресу (знать о MUTABLE/WRITEABLE). - */ + if (unlikely(excess_slots)) { + const bool will_retry = ctx->loop < 5 || excess_slots > 1; + NOTICE("** %s: reserve excess (excess-slots %zu, filled-slot %zu, adj %zu, " + "loop %u)", + will_retry ? "restart" : "ignore", excess_slots, ctx->fill_idx, ctx->reserve_adj, ctx->loop); + if (will_retry) + goto retry; + } -int mdbx_replace_ex(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, - MDBX_val *new_data, MDBX_val *old_data, - MDBX_put_flags_t flags, MDBX_preserve_func preserver, - void *preserver_context) { - int rc = check_txn_rw(txn, MDBX_TXN_BLOCKED); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; + tASSERT(txn, txn->tw.gc.retxl == nullptr || ctx->cleaned_slot == MDBX_PNL_GETSIZE(txn->tw.gc.retxl)); - if (unlikely(!key || !old_data || old_data == new_data)) - return MDBX_EINVAL; +bailout: + txn->cursors[FREE_DBI] = ctx->cursor.next; - if (unlikely(old_data->iov_base == NULL && old_data->iov_len)) - return MDBX_EINVAL; + MDBX_PNL_SETSIZE(txn->tw.repnl, 0); +#if MDBX_ENABLE_PROFGC + env->lck->pgops.gc_prof.wloops += (uint32_t)ctx->loop; +#endif /* MDBX_ENABLE_PROFGC */ + TRACE("<<< %u loops, rc = %d", ctx->loop, rc); + return rc; +} +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 - if (unlikely(new_data == NULL && - (flags & (MDBX_CURRENT | MDBX_RESERVE)) != MDBX_CURRENT)) - return MDBX_EINVAL; +static void mdbx_init(void); +static void mdbx_fini(void); - if (unlikely(!check_dbi(txn, dbi, DBI_USRVALID))) - return MDBX_BAD_DBI; +/*----------------------------------------------------------------------------*/ +/* mdbx constructor/destructor */ - if (unlikely(flags & - ~(MDBX_NOOVERWRITE | MDBX_NODUPDATA | MDBX_ALLDUPS | - MDBX_RESERVE | MDBX_APPEND | MDBX_APPENDDUP | MDBX_CURRENT))) - return MDBX_EINVAL; +#if defined(_WIN32) || defined(_WIN64) - MDBX_cursor_couple cx; - rc = cursor_init(&cx.outer, txn, dbi); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - cx.outer.mc_next = txn->mt_cursors[dbi]; - txn->mt_cursors[dbi] = &cx.outer; +#if MDBX_BUILD_SHARED_LIBRARY +#if MDBX_WITHOUT_MSVC_CRT && defined(NDEBUG) +/* DEBUG/CHECKED builds still require MSVC's CRT for runtime checks. + * + * Define dll's entry point only for Release build when NDEBUG is defined and + * MDBX_WITHOUT_MSVC_CRT=ON. if the entry point isn't defined then MSVC's will + * automatically use DllMainCRTStartup() from CRT library, which also + * automatically call DllMain() from our mdbx.dll */ +#pragma comment(linker, "/ENTRY:DllMain") +#endif /* MDBX_WITHOUT_MSVC_CRT */ - MDBX_val present_key = *key; - if (F_ISSET(flags, MDBX_CURRENT | MDBX_NOOVERWRITE)) { - /* в old_data значение для выбора конкретного дубликата */ - if (unlikely(!(txn->mt_dbs[dbi].md_flags & MDBX_DUPSORT))) { - rc = MDBX_EINVAL; - goto bailout; - } +BOOL APIENTRY DllMain(HANDLE module, DWORD reason, LPVOID reserved) +#else +#if !MDBX_MANUAL_MODULE_HANDLER +static +#endif /* !MDBX_MANUAL_MODULE_HANDLER */ + void NTAPI + mdbx_module_handler(PVOID module, DWORD reason, PVOID reserved) +#endif /* MDBX_BUILD_SHARED_LIBRARY */ +{ + (void)reserved; + switch (reason) { + case DLL_PROCESS_ATTACH: + windows_import(); + mdbx_init(); + break; + case DLL_PROCESS_DETACH: + mdbx_fini(); + break; - /* убираем лишний бит, он был признаком запрошенного режима */ - flags -= MDBX_NOOVERWRITE; + case DLL_THREAD_ATTACH: + break; + case DLL_THREAD_DETACH: + rthc_thread_dtor(module); + break; + } +#if MDBX_BUILD_SHARED_LIBRARY + return TRUE; +#endif +} - rc = cursor_set(&cx.outer, &present_key, old_data, MDBX_GET_BOTH).err; - if (rc != MDBX_SUCCESS) - goto bailout; - } else { - /* в old_data буфер для сохранения предыдущего значения */ - if (unlikely(new_data && old_data->iov_base == new_data->iov_base)) - return MDBX_EINVAL; - MDBX_val present_data; - rc = cursor_set(&cx.outer, &present_key, &present_data, MDBX_SET_KEY).err; - if (unlikely(rc != MDBX_SUCCESS)) { - old_data->iov_base = NULL; - old_data->iov_len = 0; - if (rc != MDBX_NOTFOUND || (flags & MDBX_CURRENT)) - goto bailout; - } else if (flags & MDBX_NOOVERWRITE) { - rc = MDBX_KEYEXIST; - *old_data = present_data; - goto bailout; - } else { - MDBX_page *page = cx.outer.mc_pg[cx.outer.mc_top]; - if (txn->mt_dbs[dbi].md_flags & MDBX_DUPSORT) { - if (flags & MDBX_CURRENT) { - /* disallow update/delete for multi-values */ - MDBX_node *node = page_node(page, cx.outer.mc_ki[cx.outer.mc_top]); - if (node_flags(node) & F_DUPDATA) { - tASSERT(txn, XCURSOR_INITED(&cx.outer) && - cx.outer.mc_xcursor->mx_db.md_entries > 1); - if (cx.outer.mc_xcursor->mx_db.md_entries > 1) { - rc = MDBX_EMULTIVAL; - goto bailout; - } - } - /* В оригинальной LMDB флажок MDBX_CURRENT здесь приведет - * к замене данных без учета MDBX_DUPSORT сортировки, - * но здесь это в любом случае допустимо, так как мы - * проверили что для ключа есть только одно значение. */ - } - } +#if !MDBX_BUILD_SHARED_LIBRARY && !MDBX_MANUAL_MODULE_HANDLER +#if defined(_MSC_VER) +# pragma const_seg(push) +# pragma data_seg(push) - if (IS_MODIFIABLE(txn, page)) { - if (new_data && cmp_lenfast(&present_data, new_data) == 0) { - /* если данные совпадают, то ничего делать не надо */ - *old_data = *new_data; - goto bailout; - } - rc = preserver ? preserver(preserver_context, old_data, - present_data.iov_base, present_data.iov_len) - : MDBX_SUCCESS; - if (unlikely(rc != MDBX_SUCCESS)) - goto bailout; - } else { - *old_data = present_data; - } - flags |= MDBX_CURRENT; - } - } +# ifndef _M_IX86 + /* kick a linker to create the TLS directory if not already done */ +# pragma comment(linker, "/INCLUDE:_tls_used") + /* Force some symbol references. */ +# pragma comment(linker, "/INCLUDE:mdbx_tls_anchor") + /* specific const-segment for WIN64 */ +# pragma const_seg(".CRT$XLB") + const +# else + /* kick a linker to create the TLS directory if not already done */ +# pragma comment(linker, "/INCLUDE:__tls_used") + /* Force some symbol references. */ +# pragma comment(linker, "/INCLUDE:_mdbx_tls_anchor") + /* specific data-segment for WIN32 */ +# pragma data_seg(".CRT$XLB") +# endif - if (likely(new_data)) - rc = cursor_put_checklen(&cx.outer, key, new_data, flags); - else - rc = cursor_del(&cx.outer, flags & MDBX_ALLDUPS); + __declspec(allocate(".CRT$XLB")) PIMAGE_TLS_CALLBACK mdbx_tls_anchor = mdbx_module_handler; +# pragma data_seg(pop) +# pragma const_seg(pop) -bailout: - txn->mt_cursors[dbi] = cx.outer.mc_next; - return rc; -} +#elif defined(__GNUC__) +# ifndef _M_IX86 + const +# endif + PIMAGE_TLS_CALLBACK mdbx_tls_anchor __attribute__((__section__(".CRT$XLB"), used)) = mdbx_module_handler; +#else +# error FIXME +#endif +#endif /* !MDBX_BUILD_SHARED_LIBRARY && !MDBX_MANUAL_MODULE_HANDLER */ -static int default_value_preserver(void *context, MDBX_val *target, - const void *src, size_t bytes) { - (void)context; - if (unlikely(target->iov_len < bytes)) { - target->iov_base = nullptr; - target->iov_len = bytes; - return MDBX_RESULT_TRUE; - } - memcpy(target->iov_base, src, target->iov_len = bytes); - return MDBX_SUCCESS; -} +#else -int mdbx_replace(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, - MDBX_val *new_data, MDBX_val *old_data, - MDBX_put_flags_t flags) { - return mdbx_replace_ex(txn, dbi, key, new_data, old_data, flags, - default_value_preserver, nullptr); +#if defined(__linux__) || defined(__gnu_linux__) +#include + +MDBX_EXCLUDE_FOR_GPROF +__cold static uint8_t probe_for_WSL(const char *tag) { + const char *const WSL = strstr(tag, "WSL"); + if (WSL && WSL[3] >= '2' && WSL[3] <= '9') + return WSL[3] - '0'; + const char *const wsl = strstr(tag, "wsl"); + if (wsl && wsl[3] >= '2' && wsl[3] <= '9') + return wsl[3] - '0'; + if (WSL || wsl || strcasestr(tag, "Microsoft")) + /* Expecting no new kernel within WSL1, either it will explicitly + * marked by an appropriate WSL-version hint. */ + return (globals.linux_kernel_version < /* 4.19.x */ 0x04130000) ? 1 : 2; + return 0; } +#endif /* Linux */ -/* Функция сообщает находится ли указанный адрес в "грязной" странице у - * заданной пишущей транзакции. В конечном счете это позволяет избавиться от - * лишнего копирования данных из НЕ-грязных страниц. - * - * "Грязные" страницы - это те, которые уже были изменены в ходе пишущей - * транзакции. Соответственно, какие-либо дальнейшие изменения могут привести - * к перезаписи таких страниц. Поэтому все функции, выполняющие изменения, в - * качестве аргументов НЕ должны получать указатели на данные в таких - * страницах. В свою очередь "НЕ грязные" страницы перед модификацией будут - * скопированы. - * - * Другими словами, данные из "грязных" страниц должны быть либо скопированы - * перед передачей в качестве аргументов для дальнейших модификаций, либо - * отвергнуты на стадии проверки корректности аргументов. - * - * Таким образом, функция позволяет как избавится от лишнего копирования, - * так и выполнить более полную проверку аргументов. - * - * ВАЖНО: Передаваемый указатель должен указывать на начало данных. Только - * так гарантируется что актуальный заголовок страницы будет физически - * расположен в той-же странице памяти, в том числе для многостраничных - * P_OVERFLOW страниц с длинными данными. */ -int mdbx_is_dirty(const MDBX_txn *txn, const void *ptr) { - int rc = check_txn(txn, MDBX_TXN_BLOCKED); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; +#ifdef ENABLE_GPROF +extern void _mcleanup(void); +extern void monstartup(unsigned long, unsigned long); +extern void _init(void); +extern void _fini(void); +extern void __gmon_start__(void) __attribute__((__weak__)); +#endif /* ENABLE_GPROF */ - const MDBX_env *env = txn->mt_env; - const ptrdiff_t offset = ptr_dist(ptr, env->me_map); - if (offset >= 0) { - const pgno_t pgno = bytes2pgno(env, offset); - if (likely(pgno < txn->mt_next_pgno)) { - const MDBX_page *page = pgno2page(env, pgno); - if (unlikely(page->mp_pgno != pgno || - (page->mp_flags & P_ILL_BITS) != 0)) { - /* The ptr pointed into middle of a large page, - * not to the beginning of a data. */ - return MDBX_EINVAL; +MDBX_EXCLUDE_FOR_GPROF +__cold static __attribute__((__constructor__)) void mdbx_global_constructor(void) { +#ifdef ENABLE_GPROF + if (!&__gmon_start__) + monstartup((uintptr_t)&_init, (uintptr_t)&_fini); +#endif /* ENABLE_GPROF */ + +#if defined(__linux__) || defined(__gnu_linux__) + struct utsname buffer; + if (uname(&buffer) == 0) { + int i = 0; + char *p = buffer.release; + while (*p && i < 4) { + if (*p >= '0' && *p <= '9') { + long number = strtol(p, &p, 10); + if (number > 0) { + if (number > 255) + number = 255; + globals.linux_kernel_version += number << (24 - i * 8); + } + ++i; + } else { + ++p; } - return ((txn->mt_flags & MDBX_TXN_RDONLY) || !IS_MODIFIABLE(txn, page)) - ? MDBX_RESULT_FALSE - : MDBX_RESULT_TRUE; - } - if ((size_t)offset < env->me_dxb_mmap.limit) { - /* Указатель адресует что-то в пределах mmap, но за границей - * распределенных страниц. Такое может случится если mdbx_is_dirty() - * вызывается после операции, в ходе которой грязная страница была - * возвращена в нераспределенное пространство. */ - return (txn->mt_flags & MDBX_TXN_RDONLY) ? MDBX_EINVAL : MDBX_RESULT_TRUE; } + /* "Official" way of detecting WSL1 but not WSL2 + * https://github.com/Microsoft/WSL/issues/423#issuecomment-221627364 + * + * WARNING: False negative detection of WSL1 will result in DATA LOSS! + * So, the REQUIREMENTS for this code: + * 1. MUST detect WSL1 without false-negatives. + * 2. DESIRABLE detect WSL2 but without the risk of violating the first. */ + globals.running_on_WSL1 = + probe_for_WSL(buffer.version) == 1 || probe_for_WSL(buffer.sysname) == 1 || probe_for_WSL(buffer.release) == 1; } +#endif /* Linux */ - /* Страница вне используемого mmap-диапазона, т.е. либо в функцию был - * передан некорректный адрес, либо адрес в теневой странице, которая была - * выделена посредством malloc(). - * - * Для режима MDBX_WRITE_MAP режима страница однозначно "не грязная", - * а для режимов без MDBX_WRITE_MAP однозначно "не чистая". */ - return (txn->mt_flags & (MDBX_WRITEMAP | MDBX_TXN_RDONLY)) ? MDBX_EINVAL - : MDBX_RESULT_TRUE; + mdbx_init(); } -int mdbx_dbi_sequence(MDBX_txn *txn, MDBX_dbi dbi, uint64_t *result, - uint64_t increment) { - int rc = check_txn(txn, MDBX_TXN_BLOCKED); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - - if (unlikely(!check_dbi(txn, dbi, DBI_USRVALID))) - return MDBX_BAD_DBI; - - if (unlikely(txn->mt_dbistate[dbi] & DBI_STALE)) { - rc = fetch_sdb(txn, dbi); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - } +MDBX_EXCLUDE_FOR_GPROF +__cold static __attribute__((__destructor__)) void mdbx_global_destructor(void) { + mdbx_fini(); +#ifdef ENABLE_GPROF + if (!&__gmon_start__) + _mcleanup(); +#endif /* ENABLE_GPROF */ +} - MDBX_db *dbs = &txn->mt_dbs[dbi]; - if (likely(result)) - *result = dbs->md_seq; +#endif /* ! Windows */ - if (likely(increment > 0)) { - if (unlikely(txn->mt_flags & MDBX_TXN_RDONLY)) - return MDBX_EACCESS; +/******************************************************************************/ - uint64_t new = dbs->md_seq + increment; - if (unlikely(new < increment)) - return MDBX_RESULT_TRUE; +struct libmdbx_globals globals; - tASSERT(txn, new > dbs->md_seq); - dbs->md_seq = new; - txn->mt_flags |= MDBX_TXN_DIRTY; - txn->mt_dbistate[dbi] |= DBI_DIRTY; - } +__cold static void mdbx_init(void) { + globals.runtime_flags = ((MDBX_DEBUG) > 0) * MDBX_DBG_ASSERT + ((MDBX_DEBUG) > 1) * MDBX_DBG_AUDIT; + globals.loglevel = MDBX_LOG_FATAL; + ENSURE(nullptr, osal_fastmutex_init(&globals.debug_lock) == 0); + osal_ctor(); + assert(globals.sys_pagesize > 0 && (globals.sys_pagesize & (globals.sys_pagesize - 1)) == 0); + rthc_ctor(); +#if MDBX_DEBUG + ENSURE(nullptr, troika_verify_fsm()); + ENSURE(nullptr, pv2pages_verify()); +#endif /* MDBX_DEBUG*/ +} - return MDBX_SUCCESS; +MDBX_EXCLUDE_FOR_GPROF +__cold static void mdbx_fini(void) { + const uint32_t current_pid = osal_getpid(); + TRACE(">> pid %d", current_pid); + rthc_dtor(current_pid); + osal_dtor(); + TRACE("<< pid %d\n", current_pid); + ENSURE(nullptr, osal_fastmutex_destroy(&globals.debug_lock) == 0); +} + +/******************************************************************************/ + +__dll_export +#ifdef __attribute_used__ + __attribute_used__ +#elif defined(__GNUC__) || __has_attribute(__used__) + __attribute__((__used__)) +#endif +#ifdef __attribute_externally_visible__ + __attribute_externally_visible__ +#elif (defined(__GNUC__) && !defined(__clang__)) || \ + __has_attribute(__externally_visible__) + __attribute__((__externally_visible__)) +#endif + const struct MDBX_build_info mdbx_build = { +#ifdef MDBX_BUILD_TIMESTAMP + MDBX_BUILD_TIMESTAMP +#else + "\"" __DATE__ " " __TIME__ "\"" +#endif /* MDBX_BUILD_TIMESTAMP */ + + , +#ifdef MDBX_BUILD_TARGET + MDBX_BUILD_TARGET +#else + #if defined(__ANDROID_API__) + "Android" MDBX_STRINGIFY(__ANDROID_API__) + #elif defined(__linux__) || defined(__gnu_linux__) + "Linux" + #elif defined(EMSCRIPTEN) || defined(__EMSCRIPTEN__) + "webassembly" + #elif defined(__CYGWIN__) + "CYGWIN" + #elif defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) \ + || defined(__WINDOWS__) + "Windows" + #elif defined(__APPLE__) + #if (defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE) \ + || (defined(TARGET_IPHONE_SIMULATOR) && TARGET_IPHONE_SIMULATOR) + "iOS" + #else + "MacOS" + #endif + #elif defined(__FreeBSD__) + "FreeBSD" + #elif defined(__DragonFly__) + "DragonFlyBSD" + #elif defined(__NetBSD__) + "NetBSD" + #elif defined(__OpenBSD__) + "OpenBSD" + #elif defined(__bsdi__) + "UnixBSDI" + #elif defined(__MACH__) + "MACH" + #elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) + "HPUX" + #elif defined(_AIX) + "AIX" + #elif defined(__sun) && defined(__SVR4) + "Solaris" + #elif defined(__BSD__) || defined(BSD) + "UnixBSD" + #elif defined(__unix__) || defined(UNIX) || defined(__unix) \ + || defined(__UNIX) || defined(__UNIX__) + "UNIX" + #elif defined(_POSIX_VERSION) + "POSIX" MDBX_STRINGIFY(_POSIX_VERSION) + #else + "UnknownOS" + #endif /* Target OS */ + + "-" + + #if defined(__amd64__) + "AMD64" + #elif defined(__ia32__) + "IA32" + #elif defined(__e2k__) || defined(__elbrus__) + "Elbrus" + #elif defined(__alpha__) || defined(__alpha) || defined(_M_ALPHA) + "Alpha" + #elif defined(__aarch64__) || defined(_M_ARM64) + "ARM64" + #elif defined(__arm__) || defined(__thumb__) || defined(__TARGET_ARCH_ARM) \ + || defined(__TARGET_ARCH_THUMB) || defined(_ARM) || defined(_M_ARM) \ + || defined(_M_ARMT) || defined(__arm) + "ARM" + #elif defined(__mips64) || defined(__mips64__) || (defined(__mips) && (__mips >= 64)) + "MIPS64" + #elif defined(__mips__) || defined(__mips) || defined(_R4000) || defined(__MIPS__) + "MIPS" + #elif defined(__hppa64__) || defined(__HPPA64__) || defined(__hppa64) + "PARISC64" + #elif defined(__hppa__) || defined(__HPPA__) || defined(__hppa) + "PARISC" + #elif defined(__ia64__) || defined(__ia64) || defined(_IA64) \ + || defined(__IA64__) || defined(_M_IA64) || defined(__itanium__) + "Itanium" + #elif defined(__powerpc64__) || defined(__ppc64__) || defined(__ppc64) \ + || defined(__powerpc64) || defined(_ARCH_PPC64) + "PowerPC64" + #elif defined(__powerpc__) || defined(__ppc__) || defined(__powerpc) \ + || defined(__ppc) || defined(_ARCH_PPC) || defined(__PPC__) || defined(__POWERPC__) + "PowerPC" + #elif defined(__sparc64__) || defined(__sparc64) + "SPARC64" + #elif defined(__sparc__) || defined(__sparc) + "SPARC" + #elif defined(__s390__) || defined(__s390) || defined(__zarch__) || defined(__zarch) + "S390" + #else + "UnknownARCH" + #endif +#endif /* MDBX_BUILD_TARGET */ + +#ifdef MDBX_BUILD_TYPE +# if defined(_MSC_VER) +# pragma message("Configuration-depended MDBX_BUILD_TYPE: " MDBX_BUILD_TYPE) +# endif + "-" MDBX_BUILD_TYPE +#endif /* MDBX_BUILD_TYPE */ + , + "MDBX_DEBUG=" MDBX_STRINGIFY(MDBX_DEBUG) +#ifdef ENABLE_GPROF + " ENABLE_GPROF" +#endif /* ENABLE_GPROF */ + " MDBX_WORDBITS=" MDBX_STRINGIFY(MDBX_WORDBITS) + " BYTE_ORDER=" +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + "LITTLE_ENDIAN" +#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + "BIG_ENDIAN" +#else + #error "FIXME: Unsupported byte order" +#endif /* __BYTE_ORDER__ */ + " MDBX_ENABLE_BIGFOOT=" MDBX_STRINGIFY(MDBX_ENABLE_BIGFOOT) + " MDBX_ENV_CHECKPID=" MDBX_ENV_CHECKPID_CONFIG + " MDBX_TXN_CHECKOWNER=" MDBX_TXN_CHECKOWNER_CONFIG + " MDBX_64BIT_ATOMIC=" MDBX_64BIT_ATOMIC_CONFIG + " MDBX_64BIT_CAS=" MDBX_64BIT_CAS_CONFIG + " MDBX_TRUST_RTC=" MDBX_TRUST_RTC_CONFIG + " MDBX_AVOID_MSYNC=" MDBX_STRINGIFY(MDBX_AVOID_MSYNC) + " MDBX_ENABLE_REFUND=" MDBX_STRINGIFY(MDBX_ENABLE_REFUND) + " MDBX_USE_MINCORE=" MDBX_STRINGIFY(MDBX_USE_MINCORE) + " MDBX_ENABLE_PGOP_STAT=" MDBX_STRINGIFY(MDBX_ENABLE_PGOP_STAT) + " MDBX_ENABLE_PROFGC=" MDBX_STRINGIFY(MDBX_ENABLE_PROFGC) +#if MDBX_DISABLE_VALIDATION + " MDBX_DISABLE_VALIDATION=YES" +#endif /* MDBX_DISABLE_VALIDATION */ +#ifdef __SANITIZE_ADDRESS__ + " SANITIZE_ADDRESS=YES" +#endif /* __SANITIZE_ADDRESS__ */ +#ifdef ENABLE_MEMCHECK + " ENABLE_MEMCHECK=YES" +#endif /* ENABLE_MEMCHECK */ +#if MDBX_FORCE_ASSERTIONS + " MDBX_FORCE_ASSERTIONS=YES" +#endif /* MDBX_FORCE_ASSERTIONS */ +#ifdef _GNU_SOURCE + " _GNU_SOURCE=YES" +#else + " _GNU_SOURCE=NO" +#endif /* _GNU_SOURCE */ +#ifdef __APPLE__ + " MDBX_APPLE_SPEED_INSTEADOF_DURABILITY=" MDBX_STRINGIFY(MDBX_APPLE_SPEED_INSTEADOF_DURABILITY) +#endif /* MacOS */ +#if defined(_WIN32) || defined(_WIN64) + " MDBX_WITHOUT_MSVC_CRT=" MDBX_STRINGIFY(MDBX_WITHOUT_MSVC_CRT) + " MDBX_BUILD_SHARED_LIBRARY=" MDBX_STRINGIFY(MDBX_BUILD_SHARED_LIBRARY) +#if !MDBX_BUILD_SHARED_LIBRARY + " MDBX_MANUAL_MODULE_HANDLER=" MDBX_STRINGIFY(MDBX_MANUAL_MODULE_HANDLER) +#endif + " WINVER=" MDBX_STRINGIFY(WINVER) +#else /* Windows */ + " MDBX_LOCKING=" MDBX_LOCKING_CONFIG + " MDBX_USE_OFDLOCKS=" MDBX_USE_OFDLOCKS_CONFIG +#endif /* !Windows */ + " MDBX_CACHELINE_SIZE=" MDBX_STRINGIFY(MDBX_CACHELINE_SIZE) + " MDBX_CPU_WRITEBACK_INCOHERENT=" MDBX_STRINGIFY(MDBX_CPU_WRITEBACK_INCOHERENT) + " MDBX_MMAP_INCOHERENT_CPU_CACHE=" MDBX_STRINGIFY(MDBX_MMAP_INCOHERENT_CPU_CACHE) + " MDBX_MMAP_INCOHERENT_FILE_WRITE=" MDBX_STRINGIFY(MDBX_MMAP_INCOHERENT_FILE_WRITE) + " MDBX_UNALIGNED_OK=" MDBX_STRINGIFY(MDBX_UNALIGNED_OK) + " MDBX_PNL_ASCENDING=" MDBX_STRINGIFY(MDBX_PNL_ASCENDING) + , +#ifdef MDBX_BUILD_COMPILER + MDBX_BUILD_COMPILER +#else + #ifdef __INTEL_COMPILER + "Intel C/C++ " MDBX_STRINGIFY(__INTEL_COMPILER) + #elif defined(__apple_build_version__) + "Apple clang " MDBX_STRINGIFY(__apple_build_version__) + #elif defined(__ibmxl__) + "IBM clang C " MDBX_STRINGIFY(__ibmxl_version__) "." MDBX_STRINGIFY(__ibmxl_release__) + "." MDBX_STRINGIFY(__ibmxl_modification__) "." MDBX_STRINGIFY(__ibmxl_ptf_fix_level__) + #elif defined(__clang__) + "clang " MDBX_STRINGIFY(__clang_version__) + #elif defined(__MINGW64__) + "MINGW-64 " MDBX_STRINGIFY(__MINGW64_MAJOR_VERSION) "." MDBX_STRINGIFY(__MINGW64_MINOR_VERSION) + #elif defined(__MINGW32__) + "MINGW-32 " MDBX_STRINGIFY(__MINGW32_MAJOR_VERSION) "." MDBX_STRINGIFY(__MINGW32_MINOR_VERSION) + #elif defined(__MINGW__) + "MINGW " MDBX_STRINGIFY(__MINGW_MAJOR_VERSION) "." MDBX_STRINGIFY(__MINGW_MINOR_VERSION) + #elif defined(__IBMC__) + "IBM C " MDBX_STRINGIFY(__IBMC__) + #elif defined(__GNUC__) + "GNU C/C++ " + #ifdef __VERSION__ + __VERSION__ + #else + MDBX_STRINGIFY(__GNUC__) "." MDBX_STRINGIFY(__GNUC_MINOR__) "." MDBX_STRINGIFY(__GNUC_PATCHLEVEL__) + #endif + #elif defined(_MSC_VER) + "MSVC " MDBX_STRINGIFY(_MSC_FULL_VER) "-" MDBX_STRINGIFY(_MSC_BUILD) + #else + "Unknown compiler" + #endif +#endif /* MDBX_BUILD_COMPILER */ + , +#ifdef MDBX_BUILD_FLAGS_CONFIG + MDBX_BUILD_FLAGS_CONFIG +#endif /* MDBX_BUILD_FLAGS_CONFIG */ +#if defined(MDBX_BUILD_FLAGS_CONFIG) && defined(MDBX_BUILD_FLAGS) + " " +#endif +#ifdef MDBX_BUILD_FLAGS + MDBX_BUILD_FLAGS +#endif /* MDBX_BUILD_FLAGS */ +#if !(defined(MDBX_BUILD_FLAGS_CONFIG) || defined(MDBX_BUILD_FLAGS)) + "undefined (please use correct build script)" +#ifdef _MSC_VER +#pragma message("warning: Build flags undefined. Please use correct build script") +#else +#warning "Build flags undefined. Please use correct build script" +#endif // _MSC_VER +#endif + , MDBX_BUILD_METADATA +}; + +#ifdef __SANITIZE_ADDRESS__ +#if !defined(_MSC_VER) || __has_attribute(weak) +LIBMDBX_API __attribute__((__weak__)) +#endif +const char *__asan_default_options(void) { + return "symbolize=1:allow_addr2line=1:" +#if MDBX_DEBUG + "debug=1:" + "verbosity=2:" +#endif /* MDBX_DEBUG */ + "log_threads=1:" + "report_globals=1:" + "replace_str=1:replace_intrin=1:" + "malloc_context_size=9:" +#if !defined(__APPLE__) + "detect_leaks=1:" +#endif + "check_printf=1:" + "detect_deadlocks=1:" +#ifndef LTO_ENABLED + "check_initialization_order=1:" +#endif + "detect_stack_use_after_return=1:" + "intercept_tls_get_addr=1:" + "decorate_proc_maps=1:" + "abort_on_error=1"; +} +#endif /* __SANITIZE_ADDRESS__ */ + +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 + +#if !(defined(_WIN32) || defined(_WIN64)) +/*----------------------------------------------------------------------------* + * POSIX/non-Windows LCK-implementation */ + +#if MDBX_LOCKING == MDBX_LOCKING_SYSV +#include +#endif /* MDBX_LOCKING == MDBX_LOCKING_SYSV */ + +/* Описание реализации блокировок для POSIX & Linux: + * + * lck-файл отображается в память, в нём организуется таблица читателей и + * размещаются совместно используемые posix-мьютексы (futex). Посредством + * этих мьютексов (см struct lck_t) реализуются: + * - Блокировка таблицы читателей для регистрации, + * т.е. функции lck_rdt_lock() и lck_rdt_unlock(). + * - Блокировка БД для пишущих транзакций, + * т.е. функции lck_txn_lock() и lck_txn_unlock(). + * + * Остальной функционал реализуется отдельно посредством файловых блокировок: + * - Первоначальный захват БД в режиме exclusive/shared и последующий перевод + * в операционный режим, функции lck_seize() и lck_downgrade(). + * - Проверка присутствие процессов-читателей, + * т.е. функции lck_rpid_set(), lck_rpid_clear() и lck_rpid_check(). + * + * Для блокировки файлов используется fcntl(F_SETLK), так как: + * - lockf() оперирует только эксклюзивной блокировкой и требует + * открытия файла в RW-режиме. + * - flock() не гарантирует атомарности при смене блокировок + * и оперирует только всем файлом целиком. + * - Для контроля процессов-читателей используются однобайтовые + * range-блокировки lck-файла посредством fcntl(F_SETLK). При этом + * в качестве позиции используется pid процесса-читателя. + * - Для первоначального захвата и shared/exclusive выполняется блокировка + * основного файла БД и при успехе lck-файла. + * + * ---------------------------------------------------------------------------- + * УДЕРЖИВАЕМЫЕ БЛОКИРОВКИ В ЗАВИСИМОСТИ ОТ РЕЖИМА И СОСТОЯНИЯ + * + * Эксклюзивный режим без lck-файла: + * = заблокирован весь dxb-файл посредством F_RDLCK или F_WRLCK, + * в зависимости от MDBX_RDONLY. + * + * Не-операционный режим на время пере-инициализации и разрушении lck-файла: + * = F_WRLCK блокировка первого байта lck-файла, другие процессы ждут её + * снятия при получении F_RDLCK через F_SETLKW. + * - блокировки dxb-файла могут меняться до снятие эксклюзивной блокировки + * lck-файла: + * + для НЕ-эксклюзивного режима блокировка pid-байта в dxb-файле + * посредством F_RDLCK или F_WRLCK, в зависимости от MDBX_RDONLY. + * + для ЭКСКЛЮЗИВНОГО режима блокировка всего dxb-файла + * посредством F_RDLCK или F_WRLCK, в зависимости от MDBX_RDONLY. + * + * ОПЕРАЦИОННЫЙ режим с lck-файлом: + * = F_RDLCK блокировка первого байта lck-файла, другие процессы не могут + * получить F_WRLCK и таким образом видят что БД используется. + * + F_WRLCK блокировка pid-байта в clk-файле после первой транзакции чтения. + * + для НЕ-эксклюзивного режима блокировка pid-байта в dxb-файле + * посредством F_RDLCK или F_WRLCK, в зависимости от MDBX_RDONLY. + * + для ЭКСКЛЮЗИВНОГО режима блокировка pid-байта всего dxb-файла + * посредством F_RDLCK или F_WRLCK, в зависимости от MDBX_RDONLY. + */ + +#if MDBX_USE_OFDLOCKS +static int op_setlk, op_setlkw, op_getlk; +__cold static void choice_fcntl(void) { + assert(!op_setlk && !op_setlkw && !op_getlk); + if ((globals.runtime_flags & MDBX_DBG_LEGACY_MULTIOPEN) == 0 +#if defined(__linux__) || defined(__gnu_linux__) + && globals.linux_kernel_version > 0x030f0000 /* OFD locks are available since 3.15, but engages here + only for 3.16 and later kernels (i.e. LTS) because + of reliability reasons */ +#endif /* linux */ + ) { + op_setlk = MDBX_F_OFD_SETLK; + op_setlkw = MDBX_F_OFD_SETLKW; + op_getlk = MDBX_F_OFD_GETLK; + return; + } + op_setlk = MDBX_F_SETLK; + op_setlkw = MDBX_F_SETLKW; + op_getlk = MDBX_F_GETLK; +} +#else +#define op_setlk MDBX_F_SETLK +#define op_setlkw MDBX_F_SETLKW +#define op_getlk MDBX_F_GETLK +#endif /* MDBX_USE_OFDLOCKS */ + +static int lck_op(const mdbx_filehandle_t fd, int cmd, const int lck, const off_t offset, off_t len) { + STATIC_ASSERT(sizeof(off_t) >= sizeof(void *) && sizeof(off_t) >= sizeof(size_t)); +#if defined(__ANDROID_API__) && __ANDROID_API__ < 24 + STATIC_ASSERT_MSG((sizeof(off_t) * CHAR_BIT == MDBX_WORDBITS), + "The bitness of system `off_t` type is mismatch. Please " + "fix build and/or NDK configuration."); +#endif /* Android && API < 24 */ + assert(offset >= 0 && len > 0); + assert((uint64_t)offset < (uint64_t)INT64_MAX && (uint64_t)len < (uint64_t)INT64_MAX && + (uint64_t)(offset + len) > (uint64_t)offset); + + assert((uint64_t)offset < (uint64_t)OFF_T_MAX && (uint64_t)len <= (uint64_t)OFF_T_MAX && + (uint64_t)(offset + len) <= (uint64_t)OFF_T_MAX); + + assert((uint64_t)((off_t)((uint64_t)offset + (uint64_t)len)) == ((uint64_t)offset + (uint64_t)len)); + + jitter4testing(true); + for (;;) { + MDBX_STRUCT_FLOCK lock_op; + STATIC_ASSERT_MSG(sizeof(off_t) <= sizeof(lock_op.l_start) && sizeof(off_t) <= sizeof(lock_op.l_len) && + OFF_T_MAX == (off_t)OFF_T_MAX, + "Support for large/64-bit-sized files is misconfigured " + "for the target system and/or toolchain. " + "Please fix it or at least disable it completely."); + memset(&lock_op, 0, sizeof(lock_op)); + lock_op.l_type = lck; + lock_op.l_whence = SEEK_SET; + lock_op.l_start = offset; + lock_op.l_len = len; + int rc = MDBX_FCNTL(fd, cmd, &lock_op); + jitter4testing(true); + if (rc != -1) { + if (cmd == op_getlk) { + /* Checks reader by pid. Returns: + * MDBX_RESULT_TRUE - if pid is live (reader holds a lock). + * MDBX_RESULT_FALSE - if pid is dead (a lock could be placed). */ + return (lock_op.l_type == F_UNLCK) ? MDBX_RESULT_FALSE : MDBX_RESULT_TRUE; + } + return MDBX_SUCCESS; + } + rc = errno; +#if MDBX_USE_OFDLOCKS + if (ignore_enosys_and_einval(rc) == MDBX_RESULT_TRUE && + (cmd == MDBX_F_OFD_SETLK || cmd == MDBX_F_OFD_SETLKW || cmd == MDBX_F_OFD_GETLK)) { + /* fallback to non-OFD locks */ + if (cmd == MDBX_F_OFD_SETLK) + cmd = MDBX_F_SETLK; + else if (cmd == MDBX_F_OFD_SETLKW) + cmd = MDBX_F_SETLKW; + else + cmd = MDBX_F_GETLK; + op_setlk = MDBX_F_SETLK; + op_setlkw = MDBX_F_SETLKW; + op_getlk = MDBX_F_GETLK; + continue; + } +#endif /* MDBX_USE_OFDLOCKS */ + if (rc != EINTR || cmd == op_setlkw) { + assert(MDBX_IS_ERROR(rc)); + return rc; + } + } +} + +MDBX_INTERNAL int osal_lockfile(mdbx_filehandle_t fd, bool wait) { +#if MDBX_USE_OFDLOCKS + if (unlikely(op_setlk == 0)) + choice_fcntl(); +#endif /* MDBX_USE_OFDLOCKS */ + return lck_op(fd, wait ? op_setlkw : op_setlk, F_WRLCK, 0, OFF_T_MAX); +} + +MDBX_INTERNAL int lck_rpid_set(MDBX_env *env) { + assert(env->lck_mmap.fd != INVALID_HANDLE_VALUE); + assert(env->pid > 0); + if (unlikely(osal_getpid() != env->pid)) + return MDBX_PANIC; + return lck_op(env->lck_mmap.fd, op_setlk, F_WRLCK, env->pid, 1); +} + +MDBX_INTERNAL int lck_rpid_clear(MDBX_env *env) { + assert(env->lck_mmap.fd != INVALID_HANDLE_VALUE); + assert(env->pid > 0); + return lck_op(env->lck_mmap.fd, op_setlk, F_UNLCK, env->pid, 1); +} + +MDBX_INTERNAL int lck_rpid_check(MDBX_env *env, uint32_t pid) { + assert(env->lck_mmap.fd != INVALID_HANDLE_VALUE); + assert(pid > 0); + return lck_op(env->lck_mmap.fd, op_getlk, F_WRLCK, pid, 1); +} + +/*---------------------------------------------------------------------------*/ + +#if MDBX_LOCKING > MDBX_LOCKING_SYSV +MDBX_INTERNAL int lck_ipclock_stubinit(osal_ipclock_t *ipc) { +#if MDBX_LOCKING == MDBX_LOCKING_POSIX1988 + return sem_init(ipc, false, 1) ? errno : 0; +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || MDBX_LOCKING == MDBX_LOCKING_POSIX2008 + return pthread_mutex_init(ipc, nullptr); +#else +#error "FIXME" +#endif +} + +MDBX_INTERNAL int lck_ipclock_destroy(osal_ipclock_t *ipc) { +#if MDBX_LOCKING == MDBX_LOCKING_POSIX1988 + return sem_destroy(ipc) ? errno : 0; +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || MDBX_LOCKING == MDBX_LOCKING_POSIX2008 + return pthread_mutex_destroy(ipc); +#else +#error "FIXME" +#endif +} +#endif /* MDBX_LOCKING > MDBX_LOCKING_SYSV */ + +static int check_fstat(MDBX_env *env) { + struct stat st; + + int rc = MDBX_SUCCESS; + if (fstat(env->lazy_fd, &st)) { + rc = errno; + ERROR("fstat(%s), err %d", "DXB", rc); + return rc; + } + + if (!S_ISREG(st.st_mode) || st.st_nlink < 1) { +#ifdef EBADFD + rc = EBADFD; +#else + rc = EPERM; +#endif + ERROR("%s %s, err %d", "DXB", (st.st_nlink < 1) ? "file was removed" : "not a regular file", rc); + return rc; + } + + if (st.st_size < (off_t)(MDBX_MIN_PAGESIZE * NUM_METAS)) { + VERBOSE("dxb-file is too short (%u), exclusive-lock needed", (unsigned)st.st_size); + rc = MDBX_RESULT_TRUE; + } + + //---------------------------------------------------------------------------- + + if (fstat(env->lck_mmap.fd, &st)) { + rc = errno; + ERROR("fstat(%s), err %d", "LCK", rc); + return rc; + } + + if (!S_ISREG(st.st_mode) || st.st_nlink < 1) { +#ifdef EBADFD + rc = EBADFD; +#else + rc = EPERM; +#endif + ERROR("%s %s, err %d", "LCK", (st.st_nlink < 1) ? "file was removed" : "not a regular file", rc); + return rc; + } + + /* Checking file size for detect the situation when we got the shared lock + * immediately after lck_destroy(). */ + if (st.st_size < (off_t)(sizeof(lck_t) + sizeof(reader_slot_t))) { + VERBOSE("lck-file is too short (%u), exclusive-lock needed", (unsigned)st.st_size); + rc = MDBX_RESULT_TRUE; + } + + return rc; +} + +__cold MDBX_INTERNAL int lck_seize(MDBX_env *env) { + assert(env->lazy_fd != INVALID_HANDLE_VALUE); + if (unlikely(osal_getpid() != env->pid)) + return MDBX_PANIC; + + int rc = MDBX_SUCCESS; +#if defined(__linux__) || defined(__gnu_linux__) + if (unlikely(globals.running_on_WSL1)) { + rc = ENOLCK /* No record locks available */; + ERROR("%s, err %u", + "WSL1 (Windows Subsystem for Linux) is mad and trouble-full, " + "injecting failure to avoid data loss", + rc); + return rc; + } +#endif /* Linux */ + +#if MDBX_USE_OFDLOCKS + if (unlikely(op_setlk == 0)) + choice_fcntl(); +#endif /* MDBX_USE_OFDLOCKS */ + + if (env->lck_mmap.fd == INVALID_HANDLE_VALUE) { + /* LY: without-lck mode (e.g. exclusive or on read-only filesystem) */ + rc = lck_op(env->lazy_fd, op_setlk, (env->flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, 0, OFF_T_MAX); + if (rc != MDBX_SUCCESS) { + ERROR("%s, err %u", "without-lck", rc); + eASSERT(env, MDBX_IS_ERROR(rc)); + return rc; + } + return MDBX_RESULT_TRUE /* Done: return with exclusive locking. */; + } +#if defined(_POSIX_PRIORITY_SCHEDULING) && _POSIX_PRIORITY_SCHEDULING > 0 + sched_yield(); +#endif + +retry: + if (rc == MDBX_RESULT_TRUE) { + rc = lck_op(env->lck_mmap.fd, op_setlk, F_UNLCK, 0, 1); + if (rc != MDBX_SUCCESS) { + ERROR("%s, err %u", "unlock-before-retry", rc); + eASSERT(env, MDBX_IS_ERROR(rc)); + return rc; + } + } + + /* Firstly try to get exclusive locking. */ + rc = lck_op(env->lck_mmap.fd, op_setlk, F_WRLCK, 0, 1); + if (rc == MDBX_SUCCESS) { + rc = check_fstat(env); + if (MDBX_IS_ERROR(rc)) + return rc; + + continue_dxb_exclusive: + rc = lck_op(env->lazy_fd, op_setlk, (env->flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, 0, OFF_T_MAX); + if (rc == MDBX_SUCCESS) + return MDBX_RESULT_TRUE /* Done: return with exclusive locking. */; + + int err = check_fstat(env); + if (MDBX_IS_ERROR(err)) + return err; + + /* the cause may be a collision with POSIX's file-lock recovery. */ + if (!(rc == EAGAIN || rc == EACCES || rc == EBUSY || rc == EWOULDBLOCK || rc == EDEADLK)) { + ERROR("%s, err %u", "dxb-exclusive", rc); + eASSERT(env, MDBX_IS_ERROR(rc)); + return rc; + } + + /* Fallback to lck-shared */ + } else if (!(rc == EAGAIN || rc == EACCES || rc == EBUSY || rc == EWOULDBLOCK || rc == EDEADLK)) { + ERROR("%s, err %u", "try-exclusive", rc); + eASSERT(env, MDBX_IS_ERROR(rc)); + return rc; + } + + /* Here could be one of two: + * - lck_destroy() from the another process was hold the lock + * during a destruction. + * - either lck_seize() from the another process was got the exclusive + * lock and doing initialization. + * For distinguish these cases will use size of the lck-file later. */ + + /* Wait for lck-shared now. */ + /* Here may be await during transient processes, for instance until another + * competing process doesn't call lck_downgrade(). */ + rc = lck_op(env->lck_mmap.fd, op_setlkw, F_RDLCK, 0, 1); + if (rc != MDBX_SUCCESS) { + ERROR("%s, err %u", "try-shared", rc); + eASSERT(env, MDBX_IS_ERROR(rc)); + return rc; + } + + rc = check_fstat(env); + if (rc == MDBX_RESULT_TRUE) + goto retry; + if (rc != MDBX_SUCCESS) { + ERROR("%s, err %u", "lck_fstat", rc); + return rc; + } + + /* got shared, retry exclusive */ + rc = lck_op(env->lck_mmap.fd, op_setlk, F_WRLCK, 0, 1); + if (rc == MDBX_SUCCESS) + goto continue_dxb_exclusive; + + if (!(rc == EAGAIN || rc == EACCES || rc == EBUSY || rc == EWOULDBLOCK || rc == EDEADLK)) { + ERROR("%s, err %u", "try-exclusive", rc); + eASSERT(env, MDBX_IS_ERROR(rc)); + return rc; + } + + /* Lock against another process operating in without-lck or exclusive mode. */ + rc = lck_op(env->lazy_fd, op_setlk, (env->flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, env->pid, 1); + if (rc != MDBX_SUCCESS) { + ERROR("%s, err %u", "lock-against-without-lck", rc); + eASSERT(env, MDBX_IS_ERROR(rc)); + return rc; + } + + /* Done: return with shared locking. */ + return MDBX_RESULT_FALSE; +} + +MDBX_INTERNAL int lck_downgrade(MDBX_env *env) { + assert(env->lck_mmap.fd != INVALID_HANDLE_VALUE); + if (unlikely(osal_getpid() != env->pid)) + return MDBX_PANIC; + + int rc = MDBX_SUCCESS; + if ((env->flags & MDBX_EXCLUSIVE) == 0) { + rc = lck_op(env->lazy_fd, op_setlk, F_UNLCK, 0, env->pid); + if (rc == MDBX_SUCCESS) + rc = lck_op(env->lazy_fd, op_setlk, F_UNLCK, env->pid + 1, OFF_T_MAX - env->pid - 1); + } + if (rc == MDBX_SUCCESS) + rc = lck_op(env->lck_mmap.fd, op_setlk, F_RDLCK, 0, 1); + if (unlikely(rc != 0)) { + ERROR("%s, err %u", "lck", rc); + assert(MDBX_IS_ERROR(rc)); + } + return rc; +} + +MDBX_INTERNAL int lck_upgrade(MDBX_env *env, bool dont_wait) { + assert(env->lck_mmap.fd != INVALID_HANDLE_VALUE); + if (unlikely(osal_getpid() != env->pid)) + return MDBX_PANIC; + + const int cmd = dont_wait ? op_setlk : op_setlkw; + int rc = lck_op(env->lck_mmap.fd, cmd, F_WRLCK, 0, 1); + if (rc == MDBX_SUCCESS && (env->flags & MDBX_EXCLUSIVE) == 0) { + rc = (env->pid > 1) ? lck_op(env->lazy_fd, cmd, F_WRLCK, 0, env->pid - 1) : MDBX_SUCCESS; + if (rc == MDBX_SUCCESS) { + rc = lck_op(env->lazy_fd, cmd, F_WRLCK, env->pid + 1, OFF_T_MAX - env->pid - 1); + if (rc != MDBX_SUCCESS && env->pid > 1 && lck_op(env->lazy_fd, op_setlk, F_UNLCK, 0, env->pid - 1)) + rc = MDBX_PANIC; + } + if (rc != MDBX_SUCCESS && lck_op(env->lck_mmap.fd, op_setlk, F_RDLCK, 0, 1)) + rc = MDBX_PANIC; + } + if (unlikely(rc != 0)) { + ERROR("%s, err %u", "lck", rc); + assert(MDBX_IS_ERROR(rc)); + } + return rc; +} + +__cold MDBX_INTERNAL int lck_destroy(MDBX_env *env, MDBX_env *inprocess_neighbor, const uint32_t current_pid) { + eASSERT(env, osal_getpid() == current_pid); + int rc = MDBX_SUCCESS; + struct stat lck_info; + lck_t *lck = env->lck; + if (lck && lck == env->lck_mmap.lck && !inprocess_neighbor && + /* try get exclusive access */ + lck_op(env->lck_mmap.fd, op_setlk, F_WRLCK, 0, OFF_T_MAX) == 0 && + /* if LCK was not removed */ + fstat(env->lck_mmap.fd, &lck_info) == 0 && lck_info.st_nlink > 0 && + lck_op(env->lazy_fd, op_setlk, (env->flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, 0, OFF_T_MAX) == 0) { + + VERBOSE("%p got exclusive, drown ipc-locks", (void *)env); + eASSERT(env, current_pid == env->pid); +#if MDBX_LOCKING == MDBX_LOCKING_SYSV + if (env->me_sysv_ipc.semid != -1) + rc = semctl(env->me_sysv_ipc.semid, 2, IPC_RMID) ? errno : 0; +#else + rc = lck_ipclock_destroy(&lck->rdt_lock); + if (rc == 0) + rc = lck_ipclock_destroy(&lck->wrt_lock); +#endif /* MDBX_LOCKING */ + + eASSERT(env, rc == 0); + if (rc == 0) { + const bool synced = lck->unsynced_pages.weak == 0; + osal_munmap(&env->lck_mmap); + if (synced && env->lck_mmap.fd != INVALID_HANDLE_VALUE) + rc = ftruncate(env->lck_mmap.fd, 0) ? errno : 0; + } + + jitter4testing(false); + } + +#if MDBX_LOCKING == MDBX_LOCKING_SYSV + env->me_sysv_ipc.semid = -1; +#endif /* MDBX_LOCKING */ + + if (current_pid != env->pid) { + eASSERT(env, !inprocess_neighbor); + NOTICE("drown env %p after-fork pid %d -> %d", __Wpedantic_format_voidptr(env), env->pid, current_pid); + inprocess_neighbor = nullptr; + } + + /* 1) POSIX's fcntl() locks (i.e. when op_setlk == F_SETLK) should be restored + * after file was closed. + * + * 2) File locks would be released (by kernel) while the file-descriptors will + * be closed. But to avoid false-positive EACCESS and EDEADLK from the kernel, + * locks should be released here explicitly with properly order. */ + + /* close dxb and restore lock */ + if (env->dsync_fd != INVALID_HANDLE_VALUE) { + if (unlikely(close(env->dsync_fd) != 0) && rc == MDBX_SUCCESS) + rc = errno; + env->dsync_fd = INVALID_HANDLE_VALUE; + } + if (env->lazy_fd != INVALID_HANDLE_VALUE) { + if (unlikely(close(env->lazy_fd) != 0) && rc == MDBX_SUCCESS) + rc = errno; + env->lazy_fd = INVALID_HANDLE_VALUE; + if (op_setlk == F_SETLK && inprocess_neighbor && rc == MDBX_SUCCESS) { + /* restore file-lock */ + rc = lck_op(inprocess_neighbor->lazy_fd, F_SETLKW, (inprocess_neighbor->flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, + (inprocess_neighbor->flags & MDBX_EXCLUSIVE) ? 0 : inprocess_neighbor->pid, + (inprocess_neighbor->flags & MDBX_EXCLUSIVE) ? OFF_T_MAX : 1); + } + } + + /* close clk and restore locks */ + if (env->lck_mmap.fd != INVALID_HANDLE_VALUE) { + if (unlikely(close(env->lck_mmap.fd) != 0) && rc == MDBX_SUCCESS) + rc = errno; + env->lck_mmap.fd = INVALID_HANDLE_VALUE; + if (op_setlk == F_SETLK && inprocess_neighbor && rc == MDBX_SUCCESS) { + /* restore file-locks */ + rc = lck_op(inprocess_neighbor->lck_mmap.fd, F_SETLKW, F_RDLCK, 0, 1); + if (rc == MDBX_SUCCESS && inprocess_neighbor->registered_reader_pid) + rc = lck_rpid_set(inprocess_neighbor); + } + } + + if (inprocess_neighbor && rc != MDBX_SUCCESS) + inprocess_neighbor->flags |= ENV_FATAL_ERROR; + return rc; +} + +/*---------------------------------------------------------------------------*/ + +__cold MDBX_INTERNAL int lck_init(MDBX_env *env, MDBX_env *inprocess_neighbor, int global_uniqueness_flag) { +#if MDBX_LOCKING == MDBX_LOCKING_SYSV + int semid = -1; + /* don't initialize semaphores twice */ + (void)inprocess_neighbor; + if (global_uniqueness_flag == MDBX_RESULT_TRUE) { + struct stat st; + if (fstat(env->lazy_fd, &st)) + return errno; + sysv_retry_create: + semid = semget(env->me_sysv_ipc.key, 2, IPC_CREAT | IPC_EXCL | (st.st_mode & (S_IRWXU | S_IRWXG | S_IRWXO))); + if (unlikely(semid == -1)) { + int err = errno; + if (err != EEXIST) + return err; + + /* remove and re-create semaphore set */ + semid = semget(env->me_sysv_ipc.key, 2, 0); + if (semid == -1) { + err = errno; + if (err != ENOENT) + return err; + goto sysv_retry_create; + } + if (semctl(semid, 2, IPC_RMID)) { + err = errno; + if (err != EIDRM) + return err; + } + goto sysv_retry_create; + } + + unsigned short val_array[2] = {1, 1}; + if (semctl(semid, 2, SETALL, val_array)) + return errno; + } else { + semid = semget(env->me_sysv_ipc.key, 2, 0); + if (semid == -1) + return errno; + + /* check read & write access */ + struct semid_ds data[2]; + if (semctl(semid, 2, IPC_STAT, data) || semctl(semid, 2, IPC_SET, data)) + return errno; + } + + env->me_sysv_ipc.semid = semid; + return MDBX_SUCCESS; + +#elif MDBX_LOCKING == MDBX_LOCKING_FUTEX + (void)inprocess_neighbor; + if (global_uniqueness_flag != MDBX_RESULT_TRUE) + return MDBX_SUCCESS; +#error "FIXME: Not implemented" +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988 + + /* don't initialize semaphores twice */ + (void)inprocess_neighbor; + if (global_uniqueness_flag == MDBX_RESULT_TRUE) { + if (sem_init(&env->lck_mmap.lck->rdt_lock, true, 1)) + return errno; + if (sem_init(&env->lck_mmap.lck->wrt_lock, true, 1)) + return errno; + } + return MDBX_SUCCESS; + +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || MDBX_LOCKING == MDBX_LOCKING_POSIX2008 + if (inprocess_neighbor) + return MDBX_SUCCESS /* don't need any initialization for mutexes + if LCK already opened/used inside current process */ + ; + + /* FIXME: Unfortunately, there is no other reliable way but to long testing + * on each platform. On the other hand, behavior like FreeBSD is incorrect + * and we can expect it to be rare. Moreover, even on FreeBSD without + * additional in-process initialization, the probability of an problem + * occurring is vanishingly small, and the symptom is a return of EINVAL + * while locking a mutex. In other words, in the worst case, the problem + * results in an EINVAL error at the start of the transaction, but NOT data + * loss, nor database corruption, nor other fatal troubles. Thus, the code + * below I am inclined to think the workaround for erroneous platforms (like + * FreeBSD), rather than a defect of libmdbx. */ +#if defined(__FreeBSD__) + /* seems that shared mutexes on FreeBSD required in-process initialization */ + (void)global_uniqueness_flag; +#else + /* shared mutexes on many other platforms (including Darwin and Linux's + * futexes) doesn't need any addition in-process initialization */ + if (global_uniqueness_flag != MDBX_RESULT_TRUE) + return MDBX_SUCCESS; +#endif + + pthread_mutexattr_t ma; + int rc = pthread_mutexattr_init(&ma); + if (rc) + return rc; + + rc = pthread_mutexattr_setpshared(&ma, PTHREAD_PROCESS_SHARED); + if (rc) + goto bailout; + +#if MDBX_LOCKING == MDBX_LOCKING_POSIX2008 +#if defined(PTHREAD_MUTEX_ROBUST) || defined(pthread_mutexattr_setrobust) + rc = pthread_mutexattr_setrobust(&ma, PTHREAD_MUTEX_ROBUST); +#elif defined(PTHREAD_MUTEX_ROBUST_NP) || defined(pthread_mutexattr_setrobust_np) + rc = pthread_mutexattr_setrobust_np(&ma, PTHREAD_MUTEX_ROBUST_NP); +#elif _POSIX_THREAD_PROCESS_SHARED < 200809L + rc = pthread_mutexattr_setrobust_np(&ma, PTHREAD_MUTEX_ROBUST_NP); +#else + rc = pthread_mutexattr_setrobust(&ma, PTHREAD_MUTEX_ROBUST); +#endif + if (rc) + goto bailout; +#endif /* MDBX_LOCKING == MDBX_LOCKING_POSIX2008 */ + +#if defined(_POSIX_THREAD_PRIO_INHERIT) && _POSIX_THREAD_PRIO_INHERIT >= 0 && !defined(MDBX_SAFE4QEMU) + rc = pthread_mutexattr_setprotocol(&ma, PTHREAD_PRIO_INHERIT); + if (rc == ENOTSUP) + rc = pthread_mutexattr_setprotocol(&ma, PTHREAD_PRIO_NONE); + if (rc && rc != ENOTSUP) + goto bailout; +#endif /* PTHREAD_PRIO_INHERIT */ + + rc = pthread_mutexattr_settype(&ma, PTHREAD_MUTEX_ERRORCHECK); + if (rc && rc != ENOTSUP) + goto bailout; + + rc = pthread_mutex_init(&env->lck_mmap.lck->rdt_lock, &ma); + if (rc) + goto bailout; + rc = pthread_mutex_init(&env->lck_mmap.lck->wrt_lock, &ma); + +bailout: + pthread_mutexattr_destroy(&ma); + return rc; +#else +#error "FIXME" +#endif /* MDBX_LOCKING > 0 */ +} + +__cold static int osal_ipclock_failed(MDBX_env *env, osal_ipclock_t *ipc, const int err) { + int rc = err; +#if MDBX_LOCKING == MDBX_LOCKING_POSIX2008 || MDBX_LOCKING == MDBX_LOCKING_SYSV + +#ifndef EOWNERDEAD +#define EOWNERDEAD MDBX_RESULT_TRUE +#endif /* EOWNERDEAD */ + + if (err == EOWNERDEAD) { + /* We own the mutex. Clean up after dead previous owner. */ + const bool rlocked = ipc == &env->lck->rdt_lock; + rc = MDBX_SUCCESS; + if (!rlocked) { + if (unlikely(env->txn)) { + /* env is hosed if the dead thread was ours */ + env->flags |= ENV_FATAL_ERROR; + env->txn = nullptr; + rc = MDBX_PANIC; + } + } + WARNING("%clock owner died, %s", (rlocked ? 'r' : 'w'), (rc ? "this process' env is hosed" : "recovering")); + + int check_rc = mvcc_cleanup_dead(env, rlocked, nullptr); + check_rc = (check_rc == MDBX_SUCCESS) ? MDBX_RESULT_TRUE : check_rc; + +#if MDBX_LOCKING == MDBX_LOCKING_SYSV + rc = (rc == MDBX_SUCCESS) ? check_rc : rc; +#else +#if defined(PTHREAD_MUTEX_ROBUST) || defined(pthread_mutex_consistent) + int mreco_rc = pthread_mutex_consistent(ipc); +#elif defined(PTHREAD_MUTEX_ROBUST_NP) || defined(pthread_mutex_consistent_np) + int mreco_rc = pthread_mutex_consistent_np(ipc); +#elif _POSIX_THREAD_PROCESS_SHARED < 200809L + int mreco_rc = pthread_mutex_consistent_np(ipc); +#else + int mreco_rc = pthread_mutex_consistent(ipc); +#endif + check_rc = (mreco_rc == 0) ? check_rc : mreco_rc; + + if (unlikely(mreco_rc)) + ERROR("lock recovery failed, %s", mdbx_strerror(mreco_rc)); + + rc = (rc == MDBX_SUCCESS) ? check_rc : rc; + if (MDBX_IS_ERROR(rc)) + pthread_mutex_unlock(ipc); +#endif /* MDBX_LOCKING == MDBX_LOCKING_POSIX2008 */ + return rc; + } +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 + (void)ipc; +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988 + (void)ipc; +#elif MDBX_LOCKING == MDBX_LOCKING_FUTEX +#ifdef _MSC_VER +#pragma message("warning: TODO") +#else +#warning "TODO" +#endif + (void)ipc; +#else +#error "FIXME" +#endif /* MDBX_LOCKING */ + + ERROR("mutex (un)lock failed, %s", mdbx_strerror(err)); + if (rc != EDEADLK) + env->flags |= ENV_FATAL_ERROR; + return rc; +} + +#if defined(__ANDROID_API__) || defined(ANDROID) || defined(BIONIC) +MDBX_INTERNAL int osal_check_tid4bionic(void) { + /* avoid 32-bit Bionic bug/hang with 32-pit TID */ + if (sizeof(pthread_mutex_t) < sizeof(pid_t) + sizeof(unsigned)) { + pid_t tid = gettid(); + if (unlikely(tid > 0xffff)) { + FATAL("Raise the ENOSYS(%d) error to avoid hang due " + "the 32-bit Bionic/Android bug with tid/thread_id 0x%08x(%i) " + "that don’t fit in 16 bits, see " + "https://android.googlesource.com/platform/bionic/+/master/" + "docs/32-bit-abi.md#is-too-small-for-large-pids", + ENOSYS, tid, tid); + return ENOSYS; + } + } + return 0; +} +#endif /* __ANDROID_API__ || ANDROID) || BIONIC */ + +static int osal_ipclock_lock(MDBX_env *env, osal_ipclock_t *ipc, const bool dont_wait) { +#if MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || MDBX_LOCKING == MDBX_LOCKING_POSIX2008 + int rc = osal_check_tid4bionic(); + if (likely(rc == 0)) + rc = dont_wait ? pthread_mutex_trylock(ipc) : pthread_mutex_lock(ipc); + rc = (rc == EBUSY && dont_wait) ? MDBX_BUSY : rc; +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988 + int rc = MDBX_SUCCESS; + if (dont_wait) { + if (sem_trywait(ipc)) { + rc = errno; + if (rc == EAGAIN) + rc = MDBX_BUSY; + } + } else if (sem_wait(ipc)) + rc = errno; +#elif MDBX_LOCKING == MDBX_LOCKING_SYSV + struct sembuf op = { + .sem_num = (ipc != &env->lck->wrt_lock), .sem_op = -1, .sem_flg = dont_wait ? IPC_NOWAIT | SEM_UNDO : SEM_UNDO}; + int rc; + if (semop(env->me_sysv_ipc.semid, &op, 1)) { + rc = errno; + if (dont_wait && rc == EAGAIN) + rc = MDBX_BUSY; + } else { + rc = *ipc ? EOWNERDEAD : MDBX_SUCCESS; + *ipc = env->pid; + } +#else +#error "FIXME" +#endif /* MDBX_LOCKING */ + + if (unlikely(rc != MDBX_SUCCESS && rc != MDBX_BUSY)) + rc = osal_ipclock_failed(env, ipc, rc); + return rc; +} + +static int osal_ipclock_unlock(MDBX_env *env, osal_ipclock_t *ipc) { + int err = MDBX_ENOSYS; +#if MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || MDBX_LOCKING == MDBX_LOCKING_POSIX2008 + err = pthread_mutex_unlock(ipc); +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988 + err = sem_post(ipc) ? errno : MDBX_SUCCESS; +#elif MDBX_LOCKING == MDBX_LOCKING_SYSV + if (unlikely(*ipc != (pid_t)env->pid || env->me_sysv_ipc.key == -1)) + err = EPERM; + else { + *ipc = 0; + struct sembuf op = {.sem_num = (ipc != &env->lck->wrt_lock), .sem_op = 1, .sem_flg = SEM_UNDO}; + err = semop(env->me_sysv_ipc.semid, &op, 1) ? errno : MDBX_SUCCESS; + } +#else +#error "FIXME" +#endif /* MDBX_LOCKING */ + int rc = err; + if (unlikely(rc != MDBX_SUCCESS)) { + const uint32_t current_pid = osal_getpid(); + if (current_pid == env->pid || LOG_ENABLED(MDBX_LOG_NOTICE)) + debug_log((current_pid == env->pid) ? MDBX_LOG_FATAL : (rc = MDBX_SUCCESS, MDBX_LOG_NOTICE), "ipc-unlock()", + __LINE__, "failed: env %p, lck-%s %p, err %d\n", __Wpedantic_format_voidptr(env), + (env->lck == env->lck_mmap.lck) ? "mmap" : "stub", __Wpedantic_format_voidptr(env->lck), err); + } + return rc; +} + +MDBX_INTERNAL int lck_rdt_lock(MDBX_env *env) { + TRACE("%s", ">>"); + jitter4testing(true); + int rc = osal_ipclock_lock(env, &env->lck->rdt_lock, false); + TRACE("<< rc %d", rc); + return rc; +} + +MDBX_INTERNAL void lck_rdt_unlock(MDBX_env *env) { + TRACE("%s", ">>"); + int err = osal_ipclock_unlock(env, &env->lck->rdt_lock); + TRACE("<< err %d", err); + if (unlikely(err != MDBX_SUCCESS)) + mdbx_panic("%s() failed: err %d\n", __func__, err); + jitter4testing(true); +} + +int lck_txn_lock(MDBX_env *env, bool dont_wait) { + TRACE("%swait %s", dont_wait ? "dont-" : "", ">>"); + jitter4testing(true); + const int err = osal_ipclock_lock(env, &env->lck->wrt_lock, dont_wait); + int rc = err; + if (likely(env->basal_txn && !MDBX_IS_ERROR(err))) { + eASSERT(env, !env->basal_txn->owner || err == /* если другой поток в этом-же процессе завершился + не освободив блокировку */ + MDBX_RESULT_TRUE); + env->basal_txn->owner = osal_thread_self(); + rc = MDBX_SUCCESS; + } + TRACE("<< err %d, rc %d", err, rc); + return rc; +} + +void lck_txn_unlock(MDBX_env *env) { + TRACE("%s", ">>"); + if (env->basal_txn) { + eASSERT(env, env->basal_txn->owner == osal_thread_self()); + env->basal_txn->owner = 0; + } + int err = osal_ipclock_unlock(env, &env->lck->wrt_lock); + TRACE("<< err %d", err); + if (unlikely(err != MDBX_SUCCESS)) + mdbx_panic("%s() failed: err %d\n", __func__, err); + jitter4testing(true); +} + +#endif /* !Windows LCK-implementation */ +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 + +#if defined(_WIN32) || defined(_WIN64) + +/* PREAMBLE FOR WINDOWS: + * + * We are not concerned for performance here. + * If you are running Windows a performance could NOT be the goal. + * Otherwise please use Linux. */ + +#define LCK_SHARED 0 +#define LCK_EXCLUSIVE LOCKFILE_EXCLUSIVE_LOCK +#define LCK_WAITFOR 0 +#define LCK_DONTWAIT LOCKFILE_FAIL_IMMEDIATELY + +static int flock_with_event(HANDLE fd, HANDLE event, unsigned flags, size_t offset, size_t bytes) { + TRACE("lock>>: fd %p, event %p, flags 0x%x offset %zu, bytes %zu >>", fd, event, flags, offset, bytes); + OVERLAPPED ov; + ov.Internal = 0; + ov.InternalHigh = 0; + ov.hEvent = event; + ov.Offset = (DWORD)offset; + ov.OffsetHigh = HIGH_DWORD(offset); + if (LockFileEx(fd, flags, 0, (DWORD)bytes, HIGH_DWORD(bytes), &ov)) { + TRACE("lock<<: fd %p, event %p, flags 0x%x offset %zu, bytes %zu << %s", fd, event, flags, offset, bytes, "done"); + return MDBX_SUCCESS; + } + + DWORD rc = GetLastError(); + if (rc == ERROR_IO_PENDING) { + if (event) { + if (GetOverlappedResult(fd, &ov, &rc, true)) { + TRACE("lock<<: fd %p, event %p, flags 0x%x offset %zu, bytes %zu << %s", fd, event, flags, offset, bytes, + "overlapped-done"); + return MDBX_SUCCESS; + } + rc = GetLastError(); + } else + CancelIo(fd); + } + TRACE("lock<<: fd %p, event %p, flags 0x%x offset %zu, bytes %zu << err %d", fd, event, flags, offset, bytes, + (int)rc); + return (int)rc; +} + +static inline int flock(HANDLE fd, unsigned flags, size_t offset, size_t bytes) { + return flock_with_event(fd, 0, flags, offset, bytes); +} + +static inline int flock_data(const MDBX_env *env, unsigned flags, size_t offset, size_t bytes) { + const HANDLE fd4data = env->ioring.overlapped_fd ? env->ioring.overlapped_fd : env->lazy_fd; + return flock_with_event(fd4data, env->dxb_lock_event, flags, offset, bytes); +} + +static int funlock(mdbx_filehandle_t fd, size_t offset, size_t bytes) { + TRACE("unlock: fd %p, offset %zu, bytes %zu", fd, offset, bytes); + return UnlockFile(fd, (DWORD)offset, HIGH_DWORD(offset), (DWORD)bytes, HIGH_DWORD(bytes)) ? MDBX_SUCCESS + : (int)GetLastError(); +} + +/*----------------------------------------------------------------------------*/ +/* global `write` lock for write-txt processing, + * exclusive locking both meta-pages) */ + +#ifdef _WIN64 +#define DXB_MAXLEN UINT64_C(0x7fffFFFFfff00000) +#else +#define DXB_MAXLEN UINT32_C(0x7ff00000) +#endif +#define DXB_BODY (env->ps * (size_t)NUM_METAS), DXB_MAXLEN +#define DXB_WHOLE 0, DXB_MAXLEN + +int lck_txn_lock(MDBX_env *env, bool dontwait) { + if (dontwait) { + if (!TryEnterCriticalSection(&env->windowsbug_lock)) + return MDBX_BUSY; + } else { + __try { + EnterCriticalSection(&env->windowsbug_lock); + } __except ((GetExceptionCode() == 0xC0000194 /* STATUS_POSSIBLE_DEADLOCK / EXCEPTION_POSSIBLE_DEADLOCK */) + ? EXCEPTION_EXECUTE_HANDLER + : EXCEPTION_CONTINUE_SEARCH) { + return MDBX_EDEADLK; + } + } + + eASSERT(env, !env->basal_txn || !env->basal_txn->owner); + if (env->flags & MDBX_EXCLUSIVE) + goto done; + + const HANDLE fd4data = env->ioring.overlapped_fd ? env->ioring.overlapped_fd : env->lazy_fd; + int rc = flock_with_event(fd4data, env->dxb_lock_event, + dontwait ? (LCK_EXCLUSIVE | LCK_DONTWAIT) : (LCK_EXCLUSIVE | LCK_WAITFOR), DXB_BODY); + if (rc == ERROR_LOCK_VIOLATION && dontwait) { + SleepEx(0, true); + rc = flock_with_event(fd4data, env->dxb_lock_event, LCK_EXCLUSIVE | LCK_DONTWAIT, DXB_BODY); + if (rc == ERROR_LOCK_VIOLATION) { + SleepEx(0, true); + rc = flock_with_event(fd4data, env->dxb_lock_event, LCK_EXCLUSIVE | LCK_DONTWAIT, DXB_BODY); + } + } + if (rc == MDBX_SUCCESS) { + done: + if (env->basal_txn) + env->basal_txn->owner = osal_thread_self(); + /* Zap: Failing to release lock 'env->windowsbug_lock' + * in function 'mdbx_txn_lock' */ + MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(26115); + return MDBX_SUCCESS; + } + + LeaveCriticalSection(&env->windowsbug_lock); + return (!dontwait || rc != ERROR_LOCK_VIOLATION) ? rc : MDBX_BUSY; +} + +void lck_txn_unlock(MDBX_env *env) { + eASSERT(env, !env->basal_txn || env->basal_txn->owner == osal_thread_self()); + if ((env->flags & MDBX_EXCLUSIVE) == 0) { + const HANDLE fd4data = env->ioring.overlapped_fd ? env->ioring.overlapped_fd : env->lazy_fd; + int err = funlock(fd4data, DXB_BODY); + if (err != MDBX_SUCCESS) + mdbx_panic("%s failed: err %u", __func__, err); + } + if (env->basal_txn) + env->basal_txn->owner = 0; + LeaveCriticalSection(&env->windowsbug_lock); +} + +/*----------------------------------------------------------------------------*/ +/* global `read` lock for readers registration, + * exclusive locking `rdt_length` (second) cacheline */ + +#define LCK_LO_OFFSET 0 +#define LCK_LO_LEN offsetof(lck_t, rdt_length) +#define LCK_UP_OFFSET LCK_LO_LEN +#define LCK_UP_LEN (sizeof(lck_t) - LCK_UP_OFFSET) +#define LCK_LOWER LCK_LO_OFFSET, LCK_LO_LEN +#define LCK_UPPER LCK_UP_OFFSET, LCK_UP_LEN + +MDBX_INTERNAL int lck_rdt_lock(MDBX_env *env) { + imports.srwl_AcquireShared(&env->remap_guard); + if (env->lck_mmap.fd == INVALID_HANDLE_VALUE) + return MDBX_SUCCESS; /* readonly database in readonly filesystem */ + + /* transition from S-? (used) to S-E (locked), + * e.g. exclusive lock upper-part */ + if (env->flags & MDBX_EXCLUSIVE) + return MDBX_SUCCESS; + + int rc = flock(env->lck_mmap.fd, LCK_EXCLUSIVE | LCK_WAITFOR, LCK_UPPER); + if (rc == MDBX_SUCCESS) + return MDBX_SUCCESS; + + imports.srwl_ReleaseShared(&env->remap_guard); + return rc; +} + +MDBX_INTERNAL void lck_rdt_unlock(MDBX_env *env) { + if (env->lck_mmap.fd != INVALID_HANDLE_VALUE && (env->flags & MDBX_EXCLUSIVE) == 0) { + /* transition from S-E (locked) to S-? (used), e.g. unlock upper-part */ + int err = funlock(env->lck_mmap.fd, LCK_UPPER); + if (err != MDBX_SUCCESS) + mdbx_panic("%s failed: err %u", __func__, err); + } + imports.srwl_ReleaseShared(&env->remap_guard); +} + +MDBX_INTERNAL int osal_lockfile(mdbx_filehandle_t fd, bool wait) { + return flock(fd, wait ? LCK_EXCLUSIVE | LCK_WAITFOR : LCK_EXCLUSIVE | LCK_DONTWAIT, 0, DXB_MAXLEN); +} + +static int suspend_and_append(mdbx_handle_array_t **array, const DWORD ThreadId) { + const unsigned limit = (*array)->limit; + if ((*array)->count == limit) { + mdbx_handle_array_t *const ptr = osal_realloc( + (limit > ARRAY_LENGTH((*array)->handles)) ? *array : /* don't free initial array on the stack */ nullptr, + sizeof(mdbx_handle_array_t) + sizeof(HANDLE) * (limit * (size_t)2 - ARRAY_LENGTH((*array)->handles))); + if (!ptr) + return MDBX_ENOMEM; + if (limit == ARRAY_LENGTH((*array)->handles)) + *ptr = **array; + *array = ptr; + (*array)->limit = limit * 2; + } + + HANDLE hThread = OpenThread(THREAD_SUSPEND_RESUME | THREAD_QUERY_INFORMATION, FALSE, ThreadId); + if (hThread == nullptr) + return (int)GetLastError(); + + if (SuspendThread(hThread) == (DWORD)-1) { + int err = (int)GetLastError(); + DWORD ExitCode; + if (err == /* workaround for Win10 UCRT bug */ ERROR_ACCESS_DENIED || !GetExitCodeThread(hThread, &ExitCode) || + ExitCode != STILL_ACTIVE) + err = MDBX_SUCCESS; + CloseHandle(hThread); + return err; + } + + (*array)->handles[(*array)->count++] = hThread; + return MDBX_SUCCESS; +} + +MDBX_INTERNAL int osal_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array) { + eASSERT(env, (env->flags & MDBX_NOSTICKYTHREADS) == 0); + const uintptr_t CurrentTid = GetCurrentThreadId(); + int rc; + if (env->lck_mmap.lck) { + /* Scan LCK for threads of the current process */ + const reader_slot_t *const begin = env->lck_mmap.lck->rdt; + const reader_slot_t *const end = begin + atomic_load32(&env->lck_mmap.lck->rdt_length, mo_AcquireRelease); + const uintptr_t WriteTxnOwner = env->basal_txn ? env->basal_txn->owner : 0; + for (const reader_slot_t *reader = begin; reader < end; ++reader) { + if (reader->pid.weak != env->pid || !reader->tid.weak || reader->tid.weak >= MDBX_TID_TXN_OUSTED) { + skip_lck: + continue; + } + if (reader->tid.weak == CurrentTid || reader->tid.weak == WriteTxnOwner) + goto skip_lck; + + rc = suspend_and_append(array, (mdbx_tid_t)reader->tid.weak); + if (rc != MDBX_SUCCESS) { + bailout_lck: + (void)osal_resume_threads_after_remap(*array); + return rc; + } + } + if (WriteTxnOwner && WriteTxnOwner != CurrentTid) { + rc = suspend_and_append(array, (mdbx_tid_t)WriteTxnOwner); + if (rc != MDBX_SUCCESS) + goto bailout_lck; + } + } else { + /* Without LCK (i.e. read-only mode). + * Walk through a snapshot of all running threads */ + eASSERT(env, env->flags & (MDBX_EXCLUSIVE | MDBX_RDONLY)); + const HANDLE hSnapshot = CreateToolhelp32Snapshot(TH32CS_SNAPTHREAD, 0); + if (hSnapshot == INVALID_HANDLE_VALUE) + return (int)GetLastError(); + + THREADENTRY32 entry; + entry.dwSize = sizeof(THREADENTRY32); + + if (!Thread32First(hSnapshot, &entry)) { + rc = (int)GetLastError(); + bailout_toolhelp: + CloseHandle(hSnapshot); + (void)osal_resume_threads_after_remap(*array); + return rc; + } + + do { + if (entry.th32OwnerProcessID != env->pid || entry.th32ThreadID == CurrentTid) + continue; + + rc = suspend_and_append(array, entry.th32ThreadID); + if (rc != MDBX_SUCCESS) + goto bailout_toolhelp; + + } while (Thread32Next(hSnapshot, &entry)); + + rc = (int)GetLastError(); + if (rc != ERROR_NO_MORE_FILES) + goto bailout_toolhelp; + CloseHandle(hSnapshot); + } + + return MDBX_SUCCESS; +} + +MDBX_INTERNAL int osal_resume_threads_after_remap(mdbx_handle_array_t *array) { + int rc = MDBX_SUCCESS; + for (unsigned i = 0; i < array->count; ++i) { + const HANDLE hThread = array->handles[i]; + if (ResumeThread(hThread) == (DWORD)-1) { + const int err = (int)GetLastError(); + DWORD ExitCode; + if (err != /* workaround for Win10 UCRT bug */ ERROR_ACCESS_DENIED && GetExitCodeThread(hThread, &ExitCode) && + ExitCode == STILL_ACTIVE) + rc = err; + } + CloseHandle(hThread); + } + return rc; +} + +/*----------------------------------------------------------------------------*/ +/* global `initial` lock for lockfile initialization, + * exclusive/shared locking first cacheline */ + +/* Briefly description of locking schema/algorithm: + * - Windows does not support upgrading or downgrading for file locking. + * - Therefore upgrading/downgrading is emulated by shared and exclusive + * locking of upper and lower halves. + * - In other words, we have FSM with possible 9 states, + * i.e. free/shared/exclusive x free/shared/exclusive == 9. + * Only 6 states of FSM are used, which 2 of ones are transitive. + * + * States: + * LO HI + * ?-? = free, i.e. unlocked + * S-? = used, i.e. shared lock + * E-? = exclusive-read, i.e. operational exclusive + * ?-S + * ?-E = middle (transitive state) + * S-S + * S-E = locked (transitive state) + * E-S + * E-E = exclusive-write, i.e. exclusive due (re)initialization + * + * The lck_seize() moves the locking-FSM from the initial free/unlocked + * state to the "exclusive write" (and returns MDBX_RESULT_TRUE) if possible, + * or to the "used" (and returns MDBX_RESULT_FALSE). + * + * The lck_downgrade() moves the locking-FSM from "exclusive write" + * state to the "used" (i.e. shared) state. + * + * The lck_upgrade() moves the locking-FSM from "used" (i.e. shared) + * state to the "exclusive write" state. + */ + +static void lck_unlock(MDBX_env *env) { + int err; + + if (env->lck_mmap.fd != INVALID_HANDLE_VALUE) { + /* double `unlock` for robustly remove overlapped shared/exclusive locks */ + do + err = funlock(env->lck_mmap.fd, LCK_LOWER); + while (err == MDBX_SUCCESS); + assert(err == ERROR_NOT_LOCKED || (globals.running_under_Wine && err == ERROR_LOCK_VIOLATION)); + SetLastError(ERROR_SUCCESS); + + do + err = funlock(env->lck_mmap.fd, LCK_UPPER); + while (err == MDBX_SUCCESS); + assert(err == ERROR_NOT_LOCKED || (globals.running_under_Wine && err == ERROR_LOCK_VIOLATION)); + SetLastError(ERROR_SUCCESS); + } + + const HANDLE fd4data = env->ioring.overlapped_fd ? env->ioring.overlapped_fd : env->lazy_fd; + if (fd4data != INVALID_HANDLE_VALUE) { + /* explicitly unlock to avoid latency for other processes (windows kernel + * releases such locks via deferred queues) */ + do + err = funlock(fd4data, DXB_BODY); + while (err == MDBX_SUCCESS); + assert(err == ERROR_NOT_LOCKED || (globals.running_under_Wine && err == ERROR_LOCK_VIOLATION)); + SetLastError(ERROR_SUCCESS); + + do + err = funlock(fd4data, DXB_WHOLE); + while (err == MDBX_SUCCESS); + assert(err == ERROR_NOT_LOCKED || (globals.running_under_Wine && err == ERROR_LOCK_VIOLATION)); + SetLastError(ERROR_SUCCESS); + } +} + +/* Seize state as 'exclusive-write' (E-E and returns MDBX_RESULT_TRUE) + * or as 'used' (S-? and returns MDBX_RESULT_FALSE). + * Otherwise returns an error. */ +static int internal_seize_lck(HANDLE lfd) { + assert(lfd != INVALID_HANDLE_VALUE); + + /* 1) now on ?-? (free), get ?-E (middle) */ + jitter4testing(false); + int rc = flock(lfd, LCK_EXCLUSIVE | LCK_WAITFOR, LCK_UPPER); + if (rc != MDBX_SUCCESS) { + /* 2) something went wrong, give up */; + ERROR("%s, err %u", "?-?(free) >> ?-E(middle)", rc); + return rc; + } + + /* 3) now on ?-E (middle), try E-E (exclusive-write) */ + jitter4testing(false); + rc = flock(lfd, LCK_EXCLUSIVE | LCK_DONTWAIT, LCK_LOWER); + if (rc == MDBX_SUCCESS) + return MDBX_RESULT_TRUE /* 4) got E-E (exclusive-write), done */; + + /* 5) still on ?-E (middle) */ + jitter4testing(false); + if (rc != ERROR_SHARING_VIOLATION && rc != ERROR_LOCK_VIOLATION) { + /* 6) something went wrong, give up */ + rc = funlock(lfd, LCK_UPPER); + if (rc != MDBX_SUCCESS) + mdbx_panic("%s(%s) failed: err %u", __func__, "?-E(middle) >> ?-?(free)", rc); + return rc; + } + + /* 7) still on ?-E (middle), try S-E (locked) */ + jitter4testing(false); + rc = flock(lfd, LCK_SHARED | LCK_DONTWAIT, LCK_LOWER); + + jitter4testing(false); + if (rc != MDBX_SUCCESS) + ERROR("%s, err %u", "?-E(middle) >> S-E(locked)", rc); + + /* 8) now on S-E (locked) or still on ?-E (middle), + * transition to S-? (used) or ?-? (free) */ + int err = funlock(lfd, LCK_UPPER); + if (err != MDBX_SUCCESS) + mdbx_panic("%s(%s) failed: err %u", __func__, "X-E(locked/middle) >> X-?(used/free)", err); + + /* 9) now on S-? (used, DONE) or ?-? (free, FAILURE) */ + return rc; +} + +MDBX_INTERNAL int lck_seize(MDBX_env *env) { + const HANDLE fd4data = env->ioring.overlapped_fd ? env->ioring.overlapped_fd : env->lazy_fd; + assert(fd4data != INVALID_HANDLE_VALUE); + if (env->flags & MDBX_EXCLUSIVE) + return MDBX_RESULT_TRUE /* nope since files were must be opened + non-shareable */ + ; + + if (env->lck_mmap.fd == INVALID_HANDLE_VALUE) { + /* LY: without-lck mode (e.g. on read-only filesystem) */ + jitter4testing(false); + int rc = flock_data(env, LCK_SHARED | LCK_DONTWAIT, DXB_WHOLE); + if (rc != MDBX_SUCCESS) + ERROR("%s, err %u", "without-lck", rc); + return rc; + } + + int rc = internal_seize_lck(env->lck_mmap.fd); + jitter4testing(false); + if (rc == MDBX_RESULT_TRUE && (env->flags & MDBX_RDONLY) == 0) { + /* Check that another process don't operates in without-lck mode. + * Doing such check by exclusive locking the body-part of db. Should be + * noted: + * - we need an exclusive lock for do so; + * - we can't lock meta-pages, otherwise other process could get an error + * while opening db in valid (non-conflict) mode. */ + int err = flock_data(env, LCK_EXCLUSIVE | LCK_DONTWAIT, DXB_WHOLE); + if (err != MDBX_SUCCESS) { + ERROR("%s, err %u", "lock-against-without-lck", err); + jitter4testing(false); + lck_unlock(env); + return err; + } + jitter4testing(false); + err = funlock(fd4data, DXB_WHOLE); + if (err != MDBX_SUCCESS) + mdbx_panic("%s(%s) failed: err %u", __func__, "unlock-against-without-lck", err); + } + + return rc; +} + +MDBX_INTERNAL int lck_downgrade(MDBX_env *env) { + const HANDLE fd4data = env->ioring.overlapped_fd ? env->ioring.overlapped_fd : env->lazy_fd; + /* Transite from exclusive-write state (E-E) to used (S-?) */ + assert(fd4data != INVALID_HANDLE_VALUE); + assert(env->lck_mmap.fd != INVALID_HANDLE_VALUE); + + if (env->flags & MDBX_EXCLUSIVE) + return MDBX_SUCCESS /* nope since files were must be opened non-shareable */ + ; + /* 1) now at E-E (exclusive-write), transition to ?_E (middle) */ + int rc = funlock(env->lck_mmap.fd, LCK_LOWER); + if (rc != MDBX_SUCCESS) + mdbx_panic("%s(%s) failed: err %u", __func__, "E-E(exclusive-write) >> ?-E(middle)", rc); + + /* 2) now at ?-E (middle), transition to S-E (locked) */ + rc = flock(env->lck_mmap.fd, LCK_SHARED | LCK_DONTWAIT, LCK_LOWER); + if (rc != MDBX_SUCCESS) { + /* 3) something went wrong, give up */; + ERROR("%s, err %u", "?-E(middle) >> S-E(locked)", rc); + return rc; + } + + /* 4) got S-E (locked), continue transition to S-? (used) */ + rc = funlock(env->lck_mmap.fd, LCK_UPPER); + if (rc != MDBX_SUCCESS) + mdbx_panic("%s(%s) failed: err %u", __func__, "S-E(locked) >> S-?(used)", rc); + + return MDBX_SUCCESS /* 5) now at S-? (used), done */; +} + +MDBX_INTERNAL int lck_upgrade(MDBX_env *env, bool dont_wait) { + /* Transite from used state (S-?) to exclusive-write (E-E) */ + assert(env->lck_mmap.fd != INVALID_HANDLE_VALUE); + + if (env->flags & MDBX_EXCLUSIVE) + return MDBX_SUCCESS /* nope since files were must be opened non-shareable */ + ; + + /* 1) now on S-? (used), try S-E (locked) */ + jitter4testing(false); + int rc = flock(env->lck_mmap.fd, dont_wait ? LCK_EXCLUSIVE | LCK_DONTWAIT : LCK_EXCLUSIVE, LCK_UPPER); + if (rc != MDBX_SUCCESS) { + /* 2) something went wrong, give up */; + VERBOSE("%s, err %u", "S-?(used) >> S-E(locked)", rc); + return rc; + } + + /* 3) now on S-E (locked), transition to ?-E (middle) */ + rc = funlock(env->lck_mmap.fd, LCK_LOWER); + if (rc != MDBX_SUCCESS) + mdbx_panic("%s(%s) failed: err %u", __func__, "S-E(locked) >> ?-E(middle)", rc); + + /* 4) now on ?-E (middle), try E-E (exclusive-write) */ + jitter4testing(false); + rc = flock(env->lck_mmap.fd, dont_wait ? LCK_EXCLUSIVE | LCK_DONTWAIT : LCK_EXCLUSIVE, LCK_LOWER); + if (rc != MDBX_SUCCESS) { + /* 5) something went wrong, give up */; + VERBOSE("%s, err %u", "?-E(middle) >> E-E(exclusive-write)", rc); + return rc; + } + + return MDBX_SUCCESS /* 6) now at E-E (exclusive-write), done */; +} + +MDBX_INTERNAL int lck_init(MDBX_env *env, MDBX_env *inprocess_neighbor, int global_uniqueness_flag) { + (void)env; + (void)inprocess_neighbor; + (void)global_uniqueness_flag; + if (imports.SetFileIoOverlappedRange && !(env->flags & MDBX_RDONLY)) { + HANDLE token = INVALID_HANDLE_VALUE; + TOKEN_PRIVILEGES privileges; + privileges.PrivilegeCount = 1; + privileges.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED; + if (!OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES, &token) || + !LookupPrivilegeValue(nullptr, SE_LOCK_MEMORY_NAME, &privileges.Privileges[0].Luid) || + !AdjustTokenPrivileges(token, FALSE, &privileges, sizeof(privileges), nullptr, nullptr) || + GetLastError() != ERROR_SUCCESS) + imports.SetFileIoOverlappedRange = nullptr; + + if (token != INVALID_HANDLE_VALUE) + CloseHandle(token); + } + return MDBX_SUCCESS; +} + +MDBX_INTERNAL int lck_destroy(MDBX_env *env, MDBX_env *inprocess_neighbor, const uint32_t current_pid) { + (void)current_pid; + /* LY: should unmap before releasing the locks to avoid race condition and + * STATUS_USER_MAPPED_FILE/ERROR_USER_MAPPED_FILE */ + if (env->dxb_mmap.base) + osal_munmap(&env->dxb_mmap); + if (env->lck_mmap.lck) { + const bool synced = env->lck_mmap.lck->unsynced_pages.weak == 0; + osal_munmap(&env->lck_mmap); + if (synced && !inprocess_neighbor && env->lck_mmap.fd != INVALID_HANDLE_VALUE && + lck_upgrade(env, true) == MDBX_SUCCESS) + /* this will fail if LCK is used/mmapped by other process(es) */ + osal_ftruncate(env->lck_mmap.fd, 0); + } + lck_unlock(env); + return MDBX_SUCCESS; +} + +/*----------------------------------------------------------------------------*/ +/* reader checking (by pid) */ + +MDBX_INTERNAL int lck_rpid_set(MDBX_env *env) { + (void)env; + return MDBX_SUCCESS; +} + +MDBX_INTERNAL int lck_rpid_clear(MDBX_env *env) { + (void)env; + return MDBX_SUCCESS; +} + +/* Checks reader by pid. + * + * Returns: + * MDBX_RESULT_TRUE, if pid is live (unable to acquire lock) + * MDBX_RESULT_FALSE, if pid is dead (lock acquired) + * or otherwise the errcode. */ +MDBX_INTERNAL int lck_rpid_check(MDBX_env *env, uint32_t pid) { + (void)env; + HANDLE hProcess = OpenProcess(SYNCHRONIZE, FALSE, pid); + int rc; + if (likely(hProcess)) { + rc = WaitForSingleObject(hProcess, 0); + if (unlikely(rc == (int)WAIT_FAILED)) + rc = (int)GetLastError(); + CloseHandle(hProcess); + } else { + rc = (int)GetLastError(); + } + + switch (rc) { + case ERROR_INVALID_PARAMETER: + /* pid seems invalid */ + return MDBX_RESULT_FALSE; + case WAIT_OBJECT_0: + /* process just exited */ + return MDBX_RESULT_FALSE; + case ERROR_ACCESS_DENIED: + /* The ERROR_ACCESS_DENIED would be returned for CSRSS-processes, etc. + * assume pid exists */ + return MDBX_RESULT_TRUE; + case WAIT_TIMEOUT: + /* pid running */ + return MDBX_RESULT_TRUE; + default: + /* failure */ + return rc; + } +} + +#endif /* Windows */ +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 + +__cold static int lck_setup_locked(MDBX_env *env) { + int err = rthc_register(env); + if (unlikely(err != MDBX_SUCCESS)) + return err; + + int lck_seize_rc = lck_seize(env); + if (unlikely(MDBX_IS_ERROR(lck_seize_rc))) + return lck_seize_rc; + + if (env->lck_mmap.fd == INVALID_HANDLE_VALUE) { + env->lck = lckless_stub(env); + env->max_readers = UINT_MAX; + DEBUG("lck-setup:%s%s%s", " lck-less", (env->flags & MDBX_RDONLY) ? " readonly" : "", + (lck_seize_rc == MDBX_RESULT_TRUE) ? " exclusive" : " cooperative"); + return lck_seize_rc; + } + + DEBUG("lck-setup:%s%s%s", " with-lck", (env->flags & MDBX_RDONLY) ? " readonly" : "", + (lck_seize_rc == MDBX_RESULT_TRUE) ? " exclusive" : " cooperative"); + + MDBX_env *inprocess_neighbor = nullptr; + err = rthc_uniq_check(&env->lck_mmap, &inprocess_neighbor); + if (unlikely(MDBX_IS_ERROR(err))) + return err; + if (inprocess_neighbor) { + if ((globals.runtime_flags & MDBX_DBG_LEGACY_MULTIOPEN) == 0 || (inprocess_neighbor->flags & MDBX_EXCLUSIVE) != 0) + return MDBX_BUSY; + if (lck_seize_rc == MDBX_RESULT_TRUE) { + err = lck_downgrade(env); + if (unlikely(err != MDBX_SUCCESS)) + return err; + lck_seize_rc = MDBX_RESULT_FALSE; + } + } + + uint64_t size = 0; + err = osal_filesize(env->lck_mmap.fd, &size); + if (unlikely(err != MDBX_SUCCESS)) + return err; + + if (lck_seize_rc == MDBX_RESULT_TRUE) { + size = ceil_powerof2(env->max_readers * sizeof(reader_slot_t) + sizeof(lck_t), globals.sys_pagesize); + jitter4testing(false); + } else { + if (env->flags & MDBX_EXCLUSIVE) + return MDBX_BUSY; + if (size > INT_MAX || (size & (globals.sys_pagesize - 1)) != 0 || size < globals.sys_pagesize) { + ERROR("lck-file has invalid size %" PRIu64 " bytes", size); + return MDBX_PROBLEM; + } + } + + const size_t maxreaders = ((size_t)size - sizeof(lck_t)) / sizeof(reader_slot_t); + if (maxreaders < 4) { + ERROR("lck-size too small (up to %" PRIuPTR " readers)", maxreaders); + return MDBX_PROBLEM; + } + env->max_readers = (maxreaders <= MDBX_READERS_LIMIT) ? (unsigned)maxreaders : (unsigned)MDBX_READERS_LIMIT; + + err = + osal_mmap((env->flags & MDBX_EXCLUSIVE) | MDBX_WRITEMAP, &env->lck_mmap, (size_t)size, (size_t)size, + lck_seize_rc ? MMAP_OPTION_TRUNCATE | MMAP_OPTION_SEMAPHORE : MMAP_OPTION_SEMAPHORE, env->pathname.lck); + if (unlikely(err != MDBX_SUCCESS)) + return err; + +#ifdef MADV_DODUMP + err = madvise(env->lck_mmap.lck, size, MADV_DODUMP) ? ignore_enosys_and_eagain(errno) : MDBX_SUCCESS; + if (unlikely(MDBX_IS_ERROR(err))) + return err; +#endif /* MADV_DODUMP */ + +#ifdef MADV_WILLNEED + err = madvise(env->lck_mmap.lck, size, MADV_WILLNEED) ? ignore_enosys_and_eagain(errno) : MDBX_SUCCESS; + if (unlikely(MDBX_IS_ERROR(err))) + return err; +#elif defined(POSIX_MADV_WILLNEED) + err = ignore_enosys(posix_madvise(env->lck_mmap.lck, size, POSIX_MADV_WILLNEED)); + if (unlikely(MDBX_IS_ERROR(err))) + return err; +#endif /* MADV_WILLNEED */ + + lck_t *lck = env->lck_mmap.lck; + if (lck_seize_rc == MDBX_RESULT_TRUE) { + /* If we succeed got exclusive lock, then nobody is using the lock region + * and we should initialize it. */ + memset(lck, 0, (size_t)size); + jitter4testing(false); + lck->magic_and_version = MDBX_LOCK_MAGIC; + lck->os_and_format = MDBX_LOCK_FORMAT; +#if MDBX_ENABLE_PGOP_STAT + lck->pgops.wops.weak = 1; +#endif /* MDBX_ENABLE_PGOP_STAT */ + err = osal_msync(&env->lck_mmap, 0, (size_t)size, MDBX_SYNC_DATA | MDBX_SYNC_SIZE); + if (unlikely(err != MDBX_SUCCESS)) { + ERROR("initial-%s for lck-file failed, err %d", "msync/fsync", err); + eASSERT(env, MDBX_IS_ERROR(err)); + return err; + } + } else { + if (lck->magic_and_version != MDBX_LOCK_MAGIC) { + const bool invalid = (lck->magic_and_version >> 8) != MDBX_MAGIC; + ERROR("lock region has %s", invalid ? "invalid magic" + : "incompatible version (only applications with nearly or the " + "same versions of libmdbx can share the same database)"); + return invalid ? MDBX_INVALID : MDBX_VERSION_MISMATCH; + } + if (lck->os_and_format != MDBX_LOCK_FORMAT) { + ERROR("lock region has os/format signature 0x%" PRIx32 ", expected 0x%" PRIx32, lck->os_and_format, + MDBX_LOCK_FORMAT); + return MDBX_VERSION_MISMATCH; + } + } + + err = lck_init(env, inprocess_neighbor, lck_seize_rc); + if (unlikely(err != MDBX_SUCCESS)) { + eASSERT(env, MDBX_IS_ERROR(err)); + return err; + } + + env->lck = lck; + eASSERT(env, !MDBX_IS_ERROR(lck_seize_rc)); + return lck_seize_rc; +} + +__cold int lck_setup(MDBX_env *env, mdbx_mode_t mode) { + eASSERT(env, env->lazy_fd != INVALID_HANDLE_VALUE); + eASSERT(env, env->lck_mmap.fd == INVALID_HANDLE_VALUE); + + int err = osal_openfile(MDBX_OPEN_LCK, env, env->pathname.lck, &env->lck_mmap.fd, mode); + if (err != MDBX_SUCCESS) { + switch (err) { + case MDBX_EACCESS: + case MDBX_EPERM: + if (F_ISSET(env->flags, MDBX_RDONLY | MDBX_EXCLUSIVE)) + break; + __fallthrough /* fall through */; + case MDBX_ENOFILE: + case MDBX_EROFS: + if (env->flags & MDBX_RDONLY) { + /* ENSURE the file system is read-only */ + int err_rofs = osal_check_fs_rdonly(env->lazy_fd, env->pathname.lck, err); + if (err_rofs == MDBX_SUCCESS || + /* ignore ERROR_NOT_SUPPORTED for exclusive mode */ + (err_rofs == MDBX_ENOSYS && (env->flags & MDBX_EXCLUSIVE))) + break; + if (err_rofs != MDBX_ENOSYS) + err = err_rofs; + } + __fallthrough /* fall through */; + default: + ERROR("unable to open lck-file %" MDBX_PRIsPATH ", env-flags 0x%X, err %d", env->pathname.lck, env->flags, err); + return err; + } + + /* LY: without-lck mode (e.g. exclusive or on read-only filesystem) */ + env->lck_mmap.fd = INVALID_HANDLE_VALUE; + NOTICE("continue %" MDBX_PRIsPATH " within without-lck mode, env-flags 0x%X, lck-error %d", env->pathname.dxb, + env->flags, err); + } + + rthc_lock(); + err = lck_setup_locked(env); + rthc_unlock(); + return err; +} + +void mincore_clean_cache(const MDBX_env *const env) { + memset(env->lck->mincore_cache.begin, -1, sizeof(env->lck->mincore_cache.begin)); +} +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 + +__cold void debug_log_va(int level, const char *function, int line, const char *fmt, va_list args) { + ENSURE(nullptr, osal_fastmutex_acquire(&globals.debug_lock) == 0); + if (globals.logger.ptr) { + if (globals.logger_buffer == nullptr) + globals.logger.fmt(level, function, line, fmt, args); + else { + const int len = vsnprintf(globals.logger_buffer, globals.logger_buffer_size, fmt, args); + if (len > 0) + globals.logger.nofmt(level, function, line, globals.logger_buffer, len); + } + } else { +#if defined(_WIN32) || defined(_WIN64) + if (IsDebuggerPresent()) { + int prefix_len = 0; + char *prefix = nullptr; + if (function && line > 0) + prefix_len = osal_asprintf(&prefix, "%s:%d ", function, line); + else if (function) + prefix_len = osal_asprintf(&prefix, "%s: ", function); + else if (line > 0) + prefix_len = osal_asprintf(&prefix, "%d: ", line); + if (prefix_len > 0 && prefix) { + OutputDebugStringA(prefix); + osal_free(prefix); + } + char *msg = nullptr; + int msg_len = osal_vasprintf(&msg, fmt, args); + if (msg_len > 0 && msg) { + OutputDebugStringA(msg); + osal_free(msg); + } + } +#else + if (function && line > 0) + fprintf(stderr, "%s:%d ", function, line); + else if (function) + fprintf(stderr, "%s: ", function); + else if (line > 0) + fprintf(stderr, "%d: ", line); + vfprintf(stderr, fmt, args); + fflush(stderr); +#endif + } + ENSURE(nullptr, osal_fastmutex_release(&globals.debug_lock) == 0); +} + +__cold void debug_log(int level, const char *function, int line, const char *fmt, ...) { + va_list args; + va_start(args, fmt); + debug_log_va(level, function, line, fmt, args); + va_end(args); +} + +__cold void log_error(const int err, const char *func, unsigned line) { + assert(err != MDBX_SUCCESS); + if (unlikely(globals.loglevel >= MDBX_LOG_DEBUG)) { + const bool is_error = err != MDBX_RESULT_TRUE && err != MDBX_NOTFOUND; + char buf[256]; + debug_log(is_error ? MDBX_LOG_ERROR : MDBX_LOG_VERBOSE, func, line, "%s %d (%s)\n", + is_error ? "error" : "condition", err, mdbx_strerror_r(err, buf, sizeof(buf))); + } +} + +/* Dump a val in ascii or hexadecimal. */ +__cold const char *mdbx_dump_val(const MDBX_val *val, char *const buf, const size_t bufsize) { + if (!val) + return ""; + if (!val->iov_len) + return ""; + if (!buf || bufsize < 4) + return nullptr; + + if (!val->iov_base) { + int len = snprintf(buf, bufsize, "", val->iov_len); + assert(len > 0 && (size_t)len < bufsize); + (void)len; + return buf; + } + + bool is_ascii = true; + const uint8_t *const data = val->iov_base; + for (size_t i = 0; i < val->iov_len; i++) + if (data[i] < ' ' || data[i] > '~') { + is_ascii = false; + break; + } + + if (is_ascii) { + int len = snprintf(buf, bufsize, "%.*s", (val->iov_len > INT_MAX) ? INT_MAX : (int)val->iov_len, data); + assert(len > 0 && (size_t)len < bufsize); + (void)len; + } else { + char *const detent = buf + bufsize - 2; + char *ptr = buf; + *ptr++ = '<'; + for (size_t i = 0; i < val->iov_len && ptr < detent; i++) { + const char hex[16] = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'}; + *ptr++ = hex[data[i] >> 4]; + *ptr++ = hex[data[i] & 15]; + } + if (ptr < detent) + *ptr++ = '>'; + *ptr = '\0'; + } + return buf; +} + +/*------------------------------------------------------------------------------ + LY: debug stuff */ + +__cold const char *pagetype_caption(const uint8_t type, char buf4unknown[16]) { + switch (type) { + case P_BRANCH: + return "branch"; + case P_LEAF: + return "leaf"; + case P_LEAF | P_SUBP: + return "subleaf"; + case P_LEAF | P_DUPFIX: + return "dupfix-leaf"; + case P_LEAF | P_DUPFIX | P_SUBP: + return "dupfix-subleaf"; + case P_LEAF | P_DUPFIX | P_SUBP | P_LEGACY_DIRTY: + return "dupfix-subleaf.legacy-dirty"; + case P_LARGE: + return "large"; + default: + snprintf(buf4unknown, 16, "unknown_0x%x", type); + return buf4unknown; + } +} + +__cold static const char *leafnode_type(node_t *n) { + static const char *const tp[2][2] = {{"", ": DB"}, {": sub-page", ": sub-DB"}}; + return (node_flags(n) & N_BIG) ? ": large page" : tp[!!(node_flags(n) & N_DUP)][!!(node_flags(n) & N_TREE)]; +} + +/* Display all the keys in the page. */ +__cold void page_list(page_t *mp) { + pgno_t pgno = mp->pgno; + const char *type; + node_t *node; + size_t i, nkeys, nsize, total = 0; + MDBX_val key; + DKBUF; + + switch (page_type(mp)) { + case P_BRANCH: + type = "Branch page"; + break; + case P_LEAF: + type = "Leaf page"; + break; + case P_LEAF | P_SUBP: + type = "Leaf sub-page"; + break; + case P_LEAF | P_DUPFIX: + type = "Leaf2 page"; + break; + case P_LEAF | P_DUPFIX | P_SUBP: + type = "Leaf2 sub-page"; + break; + case P_LARGE: + VERBOSE("Overflow page %" PRIaPGNO " pages %u\n", pgno, mp->pages); + return; + case P_META: + VERBOSE("Meta-page %" PRIaPGNO " txnid %" PRIu64 "\n", pgno, unaligned_peek_u64(4, page_meta(mp)->txnid_a)); + return; + default: + VERBOSE("Bad page %" PRIaPGNO " flags 0x%X\n", pgno, mp->flags); + return; + } + + nkeys = page_numkeys(mp); + VERBOSE("%s %" PRIaPGNO " numkeys %zu\n", type, pgno, nkeys); + + for (i = 0; i < nkeys; i++) { + if (is_dupfix_leaf(mp)) { /* DUPFIX pages have no entries[] or node headers */ + key = page_dupfix_key(mp, i, nsize = mp->dupfix_ksize); + total += nsize; + VERBOSE("key %zu: nsize %zu, %s\n", i, nsize, DKEY(&key)); + continue; + } + node = page_node(mp, i); + key.iov_len = node_ks(node); + key.iov_base = node->payload; + nsize = NODESIZE + key.iov_len; + if (is_branch(mp)) { + VERBOSE("key %zu: page %" PRIaPGNO ", %s\n", i, node_pgno(node), DKEY(&key)); + total += nsize; + } else { + if (node_flags(node) & N_BIG) + nsize += sizeof(pgno_t); + else + nsize += node_ds(node); + total += nsize; + nsize += sizeof(indx_t); + VERBOSE("key %zu: nsize %zu, %s%s\n", i, nsize, DKEY(&key), leafnode_type(node)); + } + total = EVEN_CEIL(total); + } + VERBOSE("Total: header %u + contents %zu + unused %zu\n", is_dupfix_leaf(mp) ? PAGEHDRSZ : PAGEHDRSZ + mp->lower, + total, page_room(mp)); +} + +__cold static int setup_debug(MDBX_log_level_t level, MDBX_debug_flags_t flags, union logger_union logger, char *buffer, + size_t buffer_size) { + ENSURE(nullptr, osal_fastmutex_acquire(&globals.debug_lock) == 0); + + const int rc = globals.runtime_flags | (globals.loglevel << 16); + if (level != MDBX_LOG_DONTCHANGE) + globals.loglevel = (uint8_t)level; + + if (flags != MDBX_DBG_DONTCHANGE) { + flags &= +#if MDBX_DEBUG + MDBX_DBG_ASSERT | MDBX_DBG_AUDIT | MDBX_DBG_JITTER | +#endif + MDBX_DBG_DUMP | MDBX_DBG_LEGACY_MULTIOPEN | MDBX_DBG_LEGACY_OVERLAP | MDBX_DBG_DONT_UPGRADE; + globals.runtime_flags = (uint8_t)flags; + } + + assert(MDBX_LOGGER_DONTCHANGE == ((MDBX_debug_func *)(intptr_t)-1)); + if (logger.ptr != (void *)((intptr_t)-1)) { + globals.logger.ptr = logger.ptr; + globals.logger_buffer = buffer; + globals.logger_buffer_size = buffer_size; + } + + ENSURE(nullptr, osal_fastmutex_release(&globals.debug_lock) == 0); + return rc; +} + +__cold int mdbx_setup_debug_nofmt(MDBX_log_level_t level, MDBX_debug_flags_t flags, MDBX_debug_func_nofmt *logger, + char *buffer, size_t buffer_size) { + union logger_union thunk; + thunk.nofmt = (logger && buffer && buffer_size) ? logger : MDBX_LOGGER_NOFMT_DONTCHANGE; + return setup_debug(level, flags, thunk, buffer, buffer_size); +} + +__cold int mdbx_setup_debug(MDBX_log_level_t level, MDBX_debug_flags_t flags, MDBX_debug_func *logger) { + union logger_union thunk; + thunk.fmt = logger; + return setup_debug(level, flags, thunk, nullptr, 0); +} +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 + +typedef struct meta_snap { + uint64_t txnid; + size_t is_steady; +} meta_snap_t; + +static inline txnid_t fetch_txnid(const volatile mdbx_atomic_uint32_t *ptr) { +#if (defined(__amd64__) || defined(__e2k__)) && !defined(ENABLE_UBSAN) && MDBX_UNALIGNED_OK >= 8 + return atomic_load64((const volatile mdbx_atomic_uint64_t *)ptr, mo_AcquireRelease); +#else + const uint32_t l = atomic_load32(&ptr[__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__], mo_AcquireRelease); + const uint32_t h = atomic_load32(&ptr[__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__], mo_AcquireRelease); + return (uint64_t)h << 32 | l; +#endif +} + +static inline meta_snap_t meta_snap(const volatile meta_t *meta) { + txnid_t txnid = fetch_txnid(meta->txnid_a); + jitter4testing(true); + size_t is_steady = meta_is_steady(meta) && txnid >= MIN_TXNID; + jitter4testing(true); + if (unlikely(txnid != fetch_txnid(meta->txnid_b))) + txnid = is_steady = 0; + meta_snap_t r = {txnid, is_steady}; + return r; +} + +txnid_t meta_txnid(const volatile meta_t *meta) { return meta_snap(meta).txnid; } + +meta_ptr_t meta_ptr(const MDBX_env *env, unsigned n) { + eASSERT(env, n < NUM_METAS); + meta_ptr_t r; + meta_snap_t snap = meta_snap(r.ptr_v = METAPAGE(env, n)); + r.txnid = snap.txnid; + r.is_steady = snap.is_steady; + return r; +} + +static uint8_t meta_cmp2pack(uint8_t c01, uint8_t c02, uint8_t c12, bool s0, bool s1, bool s2) { + assert(c01 < 3 && c02 < 3 && c12 < 3); + /* assert(s0 < 2 && s1 < 2 && s2 < 2); */ + const uint8_t recent = + meta_cmp2recent(c01, s0, s1) ? (meta_cmp2recent(c02, s0, s2) ? 0 : 2) : (meta_cmp2recent(c12, s1, s2) ? 1 : 2); + const uint8_t prefer_steady = + meta_cmp2steady(c01, s0, s1) ? (meta_cmp2steady(c02, s0, s2) ? 0 : 2) : (meta_cmp2steady(c12, s1, s2) ? 1 : 2); + + uint8_t tail; + if (recent == 0) + tail = meta_cmp2steady(c12, s1, s2) ? 2 : 1; + else if (recent == 1) + tail = meta_cmp2steady(c02, s0, s2) ? 2 : 0; + else + tail = meta_cmp2steady(c01, s0, s1) ? 1 : 0; + + const bool valid = c01 != 1 || s0 != s1 || c02 != 1 || s0 != s2 || c12 != 1 || s1 != s2; + const bool strict = (c01 != 1 || s0 != s1) && (c02 != 1 || s0 != s2) && (c12 != 1 || s1 != s2); + return tail | recent << 2 | prefer_steady << 4 | strict << 6 | valid << 7; +} + +static inline void meta_troika_unpack(troika_t *troika, const uint8_t packed) { + troika->recent = (packed >> 2) & 3; + troika->prefer_steady = (packed >> 4) & 3; + troika->tail_and_flags = packed & 0xC3; +#if MDBX_WORDBITS > 32 /* Workaround for false-positives from Valgrind */ + troika->unused_pad = 0; +#endif +} + +static const uint8_t troika_fsm_map[2 * 2 * 2 * 3 * 3 * 3] = { + 232, 201, 216, 216, 232, 233, 232, 232, 168, 201, 216, 152, 168, 233, 232, 168, 233, 201, 216, 201, 233, 233, + 232, 233, 168, 201, 152, 216, 232, 169, 232, 168, 168, 193, 152, 152, 168, 169, 232, 168, 169, 193, 152, 194, + 233, 169, 232, 169, 232, 201, 216, 216, 232, 201, 232, 232, 168, 193, 216, 152, 168, 193, 232, 168, 193, 193, + 210, 194, 225, 193, 225, 193, 168, 137, 212, 214, 232, 233, 168, 168, 168, 137, 212, 150, 168, 233, 168, 168, + 169, 137, 216, 201, 233, 233, 168, 169, 168, 137, 148, 214, 232, 169, 168, 168, 40, 129, 148, 150, 168, 169, + 168, 40, 169, 129, 152, 194, 233, 169, 168, 169, 168, 137, 214, 214, 232, 201, 168, 168, 168, 129, 214, 150, + 168, 193, 168, 168, 129, 129, 210, 194, 225, 193, 161, 129, 212, 198, 212, 214, 228, 228, 212, 212, 148, 201, + 212, 150, 164, 233, 212, 148, 233, 201, 216, 201, 233, 233, 216, 233, 148, 198, 148, 214, 228, 164, 212, 148, + 148, 194, 148, 150, 164, 169, 212, 148, 169, 194, 152, 194, 233, 169, 216, 169, 214, 198, 214, 214, 228, 198, + 212, 214, 150, 194, 214, 150, 164, 193, 212, 150, 194, 194, 210, 194, 225, 193, 210, 194}; + +__cold bool troika_verify_fsm(void) { + bool ok = true; + for (size_t i = 0; i < 2 * 2 * 2 * 3 * 3 * 3; ++i) { + const bool s0 = (i >> 0) & 1; + const bool s1 = (i >> 1) & 1; + const bool s2 = (i >> 2) & 1; + const uint8_t c01 = (i / (8 * 1)) % 3; + const uint8_t c02 = (i / (8 * 3)) % 3; + const uint8_t c12 = (i / (8 * 9)) % 3; + + const uint8_t packed = meta_cmp2pack(c01, c02, c12, s0, s1, s2); + troika_t troika; + troika.fsm = (uint8_t)i; + meta_troika_unpack(&troika, packed); + + const uint8_t tail = TROIKA_TAIL(&troika); + const bool strict = TROIKA_STRICT_VALID(&troika); + const bool valid = TROIKA_VALID(&troika); + + const uint8_t recent_chk = + meta_cmp2recent(c01, s0, s1) ? (meta_cmp2recent(c02, s0, s2) ? 0 : 2) : (meta_cmp2recent(c12, s1, s2) ? 1 : 2); + const uint8_t prefer_steady_chk = + meta_cmp2steady(c01, s0, s1) ? (meta_cmp2steady(c02, s0, s2) ? 0 : 2) : (meta_cmp2steady(c12, s1, s2) ? 1 : 2); + + uint8_t tail_chk; + if (recent_chk == 0) + tail_chk = meta_cmp2steady(c12, s1, s2) ? 2 : 1; + else if (recent_chk == 1) + tail_chk = meta_cmp2steady(c02, s0, s2) ? 2 : 0; + else + tail_chk = meta_cmp2steady(c01, s0, s1) ? 1 : 0; + + const bool valid_chk = c01 != 1 || s0 != s1 || c02 != 1 || s0 != s2 || c12 != 1 || s1 != s2; + const bool strict_chk = (c01 != 1 || s0 != s1) && (c02 != 1 || s0 != s2) && (c12 != 1 || s1 != s2); + assert(troika.recent == recent_chk); + assert(troika.prefer_steady == prefer_steady_chk); + assert(tail == tail_chk); + assert(valid == valid_chk); + assert(strict == strict_chk); + assert(troika_fsm_map[troika.fsm] == packed); + if (troika.recent != recent_chk || troika.prefer_steady != prefer_steady_chk || tail != tail_chk || + valid != valid_chk || strict != strict_chk || troika_fsm_map[troika.fsm] != packed) { + ok = false; + } + } + return ok; +} + +__hot troika_t meta_tap(const MDBX_env *env) { + meta_snap_t snap; + troika_t troika; + snap = meta_snap(METAPAGE(env, 0)); + troika.txnid[0] = snap.txnid; + troika.fsm = (uint8_t)snap.is_steady << 0; + snap = meta_snap(METAPAGE(env, 1)); + troika.txnid[1] = snap.txnid; + troika.fsm += (uint8_t)snap.is_steady << 1; + troika.fsm += meta_cmp2int(troika.txnid[0], troika.txnid[1], 8); + snap = meta_snap(METAPAGE(env, 2)); + troika.txnid[2] = snap.txnid; + troika.fsm += (uint8_t)snap.is_steady << 2; + troika.fsm += meta_cmp2int(troika.txnid[0], troika.txnid[2], 8 * 3); + troika.fsm += meta_cmp2int(troika.txnid[1], troika.txnid[2], 8 * 3 * 3); + + meta_troika_unpack(&troika, troika_fsm_map[troika.fsm]); + return troika; +} + +txnid_t recent_committed_txnid(const MDBX_env *env) { + const txnid_t m0 = meta_txnid(METAPAGE(env, 0)); + const txnid_t m1 = meta_txnid(METAPAGE(env, 1)); + const txnid_t m2 = meta_txnid(METAPAGE(env, 2)); + return (m0 > m1) ? ((m0 > m2) ? m0 : m2) : ((m1 > m2) ? m1 : m2); +} + +static inline bool meta_eq(const troika_t *troika, size_t a, size_t b) { + assert(a < NUM_METAS && b < NUM_METAS); + return troika->txnid[a] == troika->txnid[b] && (((troika->fsm >> a) ^ (troika->fsm >> b)) & 1) == 0 && + troika->txnid[a]; +} + +unsigned meta_eq_mask(const troika_t *troika) { + return meta_eq(troika, 0, 1) | meta_eq(troika, 1, 2) << 1 | meta_eq(troika, 2, 0) << 2; +} + +__hot bool meta_should_retry(const MDBX_env *env, troika_t *troika) { + const troika_t prev = *troika; + *troika = meta_tap(env); + return prev.fsm != troika->fsm || prev.txnid[0] != troika->txnid[0] || prev.txnid[1] != troika->txnid[1] || + prev.txnid[2] != troika->txnid[2]; +} + +const char *durable_caption(const meta_t *const meta) { + if (meta_is_steady(meta)) + return (meta_sign_get(meta) == meta_sign_calculate(meta)) ? "Steady" : "Tainted"; + return "Weak"; +} + +__cold void meta_troika_dump(const MDBX_env *env, const troika_t *troika) { + const meta_ptr_t recent = meta_recent(env, troika); + const meta_ptr_t prefer_steady = meta_prefer_steady(env, troika); + const meta_ptr_t tail = meta_tail(env, troika); + NOTICE("troika: %" PRIaTXN ".%c:%" PRIaTXN ".%c:%" PRIaTXN ".%c, fsm=0x%02x, " + "head=%d-%" PRIaTXN ".%c, " + "base=%d-%" PRIaTXN ".%c, " + "tail=%d-%" PRIaTXN ".%c, " + "valid %c, strict %c", + troika->txnid[0], (troika->fsm & 1) ? 's' : 'w', troika->txnid[1], (troika->fsm & 2) ? 's' : 'w', + troika->txnid[2], (troika->fsm & 4) ? 's' : 'w', troika->fsm, troika->recent, recent.txnid, + recent.is_steady ? 's' : 'w', troika->prefer_steady, prefer_steady.txnid, prefer_steady.is_steady ? 's' : 'w', + troika->tail_and_flags % NUM_METAS, tail.txnid, tail.is_steady ? 's' : 'w', TROIKA_VALID(troika) ? 'Y' : 'N', + TROIKA_STRICT_VALID(troika) ? 'Y' : 'N'); +} + +/*----------------------------------------------------------------------------*/ + +static int meta_unsteady(MDBX_env *env, const txnid_t inclusive_upto, const pgno_t pgno) { + meta_t *const meta = METAPAGE(env, pgno); + const txnid_t txnid = constmeta_txnid(meta); + if (!meta_is_steady(meta) || txnid > inclusive_upto) + return MDBX_RESULT_FALSE; + + WARNING("wipe txn #%" PRIaTXN ", meta %" PRIaPGNO, txnid, pgno); + const uint64_t wipe = DATASIGN_NONE; + const void *ptr = &wipe; + size_t bytes = sizeof(meta->sign), offset = ptr_dist(&meta->sign, env->dxb_mmap.base); + if (env->flags & MDBX_WRITEMAP) { + unaligned_poke_u64(4, meta->sign, wipe); + osal_flush_incoherent_cpu_writeback(); + if (!MDBX_AVOID_MSYNC) + return MDBX_RESULT_TRUE; + ptr = data_page(meta); + offset = ptr_dist(ptr, env->dxb_mmap.base); + bytes = env->ps; + } + +#if MDBX_ENABLE_PGOP_STAT + env->lck->pgops.wops.weak += 1; +#endif /* MDBX_ENABLE_PGOP_STAT */ + int err = osal_pwrite(env->fd4meta, ptr, bytes, offset); + return likely(err == MDBX_SUCCESS) ? MDBX_RESULT_TRUE : err; +} + +__cold int meta_wipe_steady(MDBX_env *env, txnid_t inclusive_upto) { + int err = meta_unsteady(env, inclusive_upto, 0); + if (likely(!MDBX_IS_ERROR(err))) + err = meta_unsteady(env, inclusive_upto, 1); + if (likely(!MDBX_IS_ERROR(err))) + err = meta_unsteady(env, inclusive_upto, 2); + + if (err == MDBX_RESULT_TRUE) { + err = MDBX_SUCCESS; + if (!MDBX_AVOID_MSYNC && (env->flags & MDBX_WRITEMAP)) { + err = osal_msync(&env->dxb_mmap, 0, pgno_align2os_bytes(env, NUM_METAS), MDBX_SYNC_DATA | MDBX_SYNC_IODQ); +#if MDBX_ENABLE_PGOP_STAT + env->lck->pgops.msync.weak += 1; +#endif /* MDBX_ENABLE_PGOP_STAT */ + } else if (env->fd4meta == env->lazy_fd) { + err = osal_fsync(env->lazy_fd, MDBX_SYNC_DATA | MDBX_SYNC_IODQ); +#if MDBX_ENABLE_PGOP_STAT + env->lck->pgops.fsync.weak += 1; +#endif /* MDBX_ENABLE_PGOP_STAT */ + } + } + + osal_flush_incoherent_mmap(env->dxb_mmap.base, pgno2bytes(env, NUM_METAS), globals.sys_pagesize); + + /* force oldest refresh */ + atomic_store32(&env->lck->rdt_refresh_flag, true, mo_Relaxed); + + env->basal_txn->tw.troika = meta_tap(env); + for (MDBX_txn *scan = env->basal_txn->nested; scan; scan = scan->nested) + scan->tw.troika = env->basal_txn->tw.troika; + return err; +} + +int meta_sync(const MDBX_env *env, const meta_ptr_t head) { + eASSERT(env, atomic_load32(&env->lck->meta_sync_txnid, mo_Relaxed) != (uint32_t)head.txnid); + /* Функция может вызываться (в том числе) при (env->flags & + * MDBX_NOMETASYNC) == 0 и env->fd4meta == env->dsync_fd, например если + * предыдущая транзакция была выполненна с флагом MDBX_NOMETASYNC. */ + + int rc = MDBX_RESULT_TRUE; + if (env->flags & MDBX_WRITEMAP) { + if (!MDBX_AVOID_MSYNC) { + rc = osal_msync(&env->dxb_mmap, 0, pgno_align2os_bytes(env, NUM_METAS), MDBX_SYNC_DATA | MDBX_SYNC_IODQ); +#if MDBX_ENABLE_PGOP_STAT + env->lck->pgops.msync.weak += 1; +#endif /* MDBX_ENABLE_PGOP_STAT */ + } else { +#if MDBX_ENABLE_PGOP_ST + env->lck->pgops.wops.weak += 1; +#endif /* MDBX_ENABLE_PGOP_STAT */ + const page_t *page = data_page(head.ptr_c); + rc = osal_pwrite(env->fd4meta, page, env->ps, ptr_dist(page, env->dxb_mmap.base)); + + if (likely(rc == MDBX_SUCCESS) && env->fd4meta == env->lazy_fd) { + rc = osal_fsync(env->lazy_fd, MDBX_SYNC_DATA | MDBX_SYNC_IODQ); +#if MDBX_ENABLE_PGOP_STAT + env->lck->pgops.fsync.weak += 1; +#endif /* MDBX_ENABLE_PGOP_STAT */ + } + } + } else { + rc = osal_fsync(env->lazy_fd, MDBX_SYNC_DATA | MDBX_SYNC_IODQ); +#if MDBX_ENABLE_PGOP_STAT + env->lck->pgops.fsync.weak += 1; +#endif /* MDBX_ENABLE_PGOP_STAT */ + } + + if (likely(rc == MDBX_SUCCESS)) + env->lck->meta_sync_txnid.weak = (uint32_t)head.txnid; + return rc; +} + +__cold static page_t *meta_model(const MDBX_env *env, page_t *model, size_t num, const bin128_t *guid) { + ENSURE(env, is_powerof2(env->ps)); + ENSURE(env, env->ps >= MDBX_MIN_PAGESIZE); + ENSURE(env, env->ps <= MDBX_MAX_PAGESIZE); + ENSURE(env, env->geo_in_bytes.lower >= MIN_MAPSIZE); + ENSURE(env, env->geo_in_bytes.upper <= MAX_MAPSIZE); + ENSURE(env, env->geo_in_bytes.now >= env->geo_in_bytes.lower); + ENSURE(env, env->geo_in_bytes.now <= env->geo_in_bytes.upper); + + memset(model, 0, env->ps); + model->pgno = (pgno_t)num; + model->flags = P_META; + meta_t *const model_meta = page_meta(model); + unaligned_poke_u64(4, model_meta->magic_and_version, MDBX_DATA_MAGIC); + + model_meta->geometry.lower = bytes2pgno(env, env->geo_in_bytes.lower); + model_meta->geometry.upper = bytes2pgno(env, env->geo_in_bytes.upper); + model_meta->geometry.grow_pv = pages2pv(bytes2pgno(env, env->geo_in_bytes.grow)); + model_meta->geometry.shrink_pv = pages2pv(bytes2pgno(env, env->geo_in_bytes.shrink)); + model_meta->geometry.now = bytes2pgno(env, env->geo_in_bytes.now); + model_meta->geometry.first_unallocated = NUM_METAS; + + ENSURE(env, model_meta->geometry.lower >= MIN_PAGENO); + ENSURE(env, model_meta->geometry.upper <= MAX_PAGENO + 1); + ENSURE(env, model_meta->geometry.now >= model_meta->geometry.lower); + ENSURE(env, model_meta->geometry.now <= model_meta->geometry.upper); + ENSURE(env, model_meta->geometry.first_unallocated >= MIN_PAGENO); + ENSURE(env, model_meta->geometry.first_unallocated <= model_meta->geometry.now); + ENSURE(env, model_meta->geometry.grow_pv == pages2pv(pv2pages(model_meta->geometry.grow_pv))); + ENSURE(env, model_meta->geometry.shrink_pv == pages2pv(pv2pages(model_meta->geometry.shrink_pv))); + + model_meta->pagesize = env->ps; + model_meta->trees.gc.flags = MDBX_INTEGERKEY; + model_meta->trees.gc.root = P_INVALID; + model_meta->trees.main.root = P_INVALID; + memcpy(&model_meta->dxbid, guid, sizeof(model_meta->dxbid)); + meta_set_txnid(env, model_meta, MIN_TXNID + num); + unaligned_poke_u64(4, model_meta->sign, meta_sign_calculate(model_meta)); + eASSERT(env, coherency_check_meta(env, model_meta, true)); + return ptr_disp(model, env->ps); +} + +__cold meta_t *meta_init_triplet(const MDBX_env *env, void *buffer) { + const bin128_t guid = osal_guid(env); + page_t *page0 = (page_t *)buffer; + page_t *page1 = meta_model(env, page0, 0, &guid); + page_t *page2 = meta_model(env, page1, 1, &guid); + meta_model(env, page2, 2, &guid); + return page_meta(page2); +} + +__cold int __must_check_result meta_override(MDBX_env *env, size_t target, txnid_t txnid, const meta_t *shape) { + page_t *const page = env->page_auxbuf; + meta_model(env, page, target, &((target == 0 && shape) ? shape : METAPAGE(env, 0))->dxbid); + meta_t *const model = page_meta(page); + meta_set_txnid(env, model, txnid); + if (txnid) + eASSERT(env, coherency_check_meta(env, model, true)); + if (shape) { + if (txnid && unlikely(!coherency_check_meta(env, shape, false))) { + ERROR("bailout overriding meta-%zu since model failed " + "FreeDB/MainDB %s-check for txnid #%" PRIaTXN, + target, "pre", constmeta_txnid(shape)); + return MDBX_PROBLEM; + } + if (globals.runtime_flags & MDBX_DBG_DONT_UPGRADE) + memcpy(&model->magic_and_version, &shape->magic_and_version, sizeof(model->magic_and_version)); + model->reserve16 = shape->reserve16; + model->validator_id = shape->validator_id; + model->extra_pagehdr = shape->extra_pagehdr; + memcpy(&model->geometry, &shape->geometry, sizeof(model->geometry)); + memcpy(&model->trees, &shape->trees, sizeof(model->trees)); + memcpy(&model->canary, &shape->canary, sizeof(model->canary)); + memcpy(&model->pages_retired, &shape->pages_retired, sizeof(model->pages_retired)); + if (txnid) { + if ((!model->trees.gc.mod_txnid && model->trees.gc.root != P_INVALID) || + (!model->trees.main.mod_txnid && model->trees.main.root != P_INVALID)) + memcpy(&model->magic_and_version, &shape->magic_and_version, sizeof(model->magic_and_version)); + if (unlikely(!coherency_check_meta(env, model, false))) { + ERROR("bailout overriding meta-%zu since model failed " + "FreeDB/MainDB %s-check for txnid #%" PRIaTXN, + target, "post", txnid); + return MDBX_PROBLEM; + } + } + } + + if (target == 0 && (model->dxbid.x | model->dxbid.y) == 0) { + const bin128_t guid = osal_guid(env); + memcpy(&model->dxbid, &guid, sizeof(model->dxbid)); + } + + meta_sign_as_steady(model); + int rc = meta_validate(env, model, page, (pgno_t)target, nullptr); + if (unlikely(MDBX_IS_ERROR(rc))) + return MDBX_PROBLEM; + + if (shape && memcmp(model, shape, sizeof(meta_t)) == 0) { + NOTICE("skip overriding meta-%zu since no changes " + "for txnid #%" PRIaTXN, + target, txnid); + return MDBX_SUCCESS; + } + + if (env->flags & MDBX_WRITEMAP) { +#if MDBX_ENABLE_PGOP_STAT + env->lck->pgops.msync.weak += 1; +#endif /* MDBX_ENABLE_PGOP_STAT */ + rc = osal_msync(&env->dxb_mmap, 0, pgno_align2os_bytes(env, model->geometry.first_unallocated), + MDBX_SYNC_DATA | MDBX_SYNC_IODQ); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + /* meta_override() called only while current process have exclusive + * lock of a DB file. So meta-page could be updated directly without + * clearing consistency flag by mdbx_meta_update_begin() */ + memcpy(pgno2page(env, target), page, env->ps); + osal_flush_incoherent_cpu_writeback(); +#if MDBX_ENABLE_PGOP_STAT + env->lck->pgops.msync.weak += 1; +#endif /* MDBX_ENABLE_PGOP_STAT */ + rc = osal_msync(&env->dxb_mmap, 0, pgno_align2os_bytes(env, target + 1), MDBX_SYNC_DATA | MDBX_SYNC_IODQ); + } else { +#if MDBX_ENABLE_PGOP_STAT + env->lck->pgops.wops.weak += 1; +#endif /* MDBX_ENABLE_PGOP_STAT */ + rc = osal_pwrite(env->fd4meta, page, env->ps, pgno2bytes(env, target)); + if (rc == MDBX_SUCCESS && env->fd4meta == env->lazy_fd) { +#if MDBX_ENABLE_PGOP_STAT + env->lck->pgops.fsync.weak += 1; +#endif /* MDBX_ENABLE_PGOP_STAT */ + rc = osal_fsync(env->lazy_fd, MDBX_SYNC_DATA | MDBX_SYNC_IODQ); + } + osal_flush_incoherent_mmap(env->dxb_mmap.base, pgno2bytes(env, NUM_METAS), globals.sys_pagesize); + } + eASSERT(env, (!env->txn && (env->flags & ENV_ACTIVE) == 0) || + (env->stuck_meta == (int)target && (env->flags & (MDBX_EXCLUSIVE | MDBX_RDONLY)) == MDBX_EXCLUSIVE)); + return rc; +} + +__cold int meta_validate(MDBX_env *env, meta_t *const meta, const page_t *const page, const unsigned meta_number, + unsigned *guess_pagesize) { + const uint64_t magic_and_version = unaligned_peek_u64(4, &meta->magic_and_version); + if (unlikely(magic_and_version != MDBX_DATA_MAGIC && magic_and_version != MDBX_DATA_MAGIC_LEGACY_COMPAT && + magic_and_version != MDBX_DATA_MAGIC_LEGACY_DEVEL)) { + ERROR("meta[%u] has invalid magic/version %" PRIx64, meta_number, magic_and_version); + return ((magic_and_version >> 8) != MDBX_MAGIC) ? MDBX_INVALID : MDBX_VERSION_MISMATCH; + } + + if (unlikely(page->pgno != meta_number)) { + ERROR("meta[%u] has invalid pageno %" PRIaPGNO, meta_number, page->pgno); + return MDBX_INVALID; + } + + if (unlikely(page->flags != P_META)) { + ERROR("page #%u not a meta-page", meta_number); + return MDBX_INVALID; + } + + if (unlikely(!is_powerof2(meta->pagesize) || meta->pagesize < MDBX_MIN_PAGESIZE || + meta->pagesize > MDBX_MAX_PAGESIZE)) { + WARNING("meta[%u] has invalid pagesize (%u), skip it", meta_number, meta->pagesize); + return is_powerof2(meta->pagesize) ? MDBX_VERSION_MISMATCH : MDBX_INVALID; + } + + if (guess_pagesize && *guess_pagesize != meta->pagesize) { + *guess_pagesize = meta->pagesize; + VERBOSE("meta[%u] took pagesize %u", meta_number, meta->pagesize); + } + + const txnid_t txnid = unaligned_peek_u64(4, &meta->txnid_a); + if (unlikely(txnid != unaligned_peek_u64(4, &meta->txnid_b))) { + WARNING("meta[%u] not completely updated, skip it", meta_number); + return MDBX_RESULT_TRUE; + } + + /* LY: check signature as a checksum */ + const uint64_t sign = meta_sign_get(meta); + const uint64_t sign_stready = meta_sign_calculate(meta); + if (SIGN_IS_STEADY(sign) && unlikely(sign != sign_stready)) { + WARNING("meta[%u] has invalid steady-checksum (0x%" PRIx64 " != 0x%" PRIx64 "), skip it", meta_number, sign, + sign_stready); + return MDBX_RESULT_TRUE; + } + + if (unlikely(meta->trees.gc.flags != MDBX_INTEGERKEY) && + ((meta->trees.gc.flags & DB_PERSISTENT_FLAGS) != MDBX_INTEGERKEY || magic_and_version == MDBX_DATA_MAGIC)) { + WARNING("meta[%u] has invalid %s flags 0x%x, skip it", meta_number, "GC/FreeDB", meta->trees.gc.flags); + return MDBX_INCOMPATIBLE; + } + + if (unlikely(!check_table_flags(meta->trees.main.flags))) { + WARNING("meta[%u] has invalid %s flags 0x%x, skip it", meta_number, "MainDB", meta->trees.main.flags); + return MDBX_INCOMPATIBLE; + } + + DEBUG("checking meta%" PRIaPGNO " = root %" PRIaPGNO "/%" PRIaPGNO ", geo %" PRIaPGNO "/%" PRIaPGNO "-%" PRIaPGNO + "/%" PRIaPGNO " +%u -%u, txn_id %" PRIaTXN ", %s", + page->pgno, meta->trees.main.root, meta->trees.gc.root, meta->geometry.lower, meta->geometry.first_unallocated, + meta->geometry.now, meta->geometry.upper, pv2pages(meta->geometry.grow_pv), pv2pages(meta->geometry.shrink_pv), + txnid, durable_caption(meta)); + + if (unlikely(txnid < MIN_TXNID || txnid > MAX_TXNID)) { + WARNING("meta[%u] has invalid txnid %" PRIaTXN ", skip it", meta_number, txnid); + return MDBX_RESULT_TRUE; + } + + if (unlikely(meta->geometry.lower < MIN_PAGENO || meta->geometry.lower > MAX_PAGENO + 1)) { + WARNING("meta[%u] has invalid min-pages (%" PRIaPGNO "), skip it", meta_number, meta->geometry.lower); + return MDBX_INVALID; + } + + if (unlikely(meta->geometry.upper < MIN_PAGENO || meta->geometry.upper > MAX_PAGENO + 1 || + meta->geometry.upper < meta->geometry.lower)) { + WARNING("meta[%u] has invalid max-pages (%" PRIaPGNO "), skip it", meta_number, meta->geometry.upper); + return MDBX_INVALID; + } + + if (unlikely(meta->geometry.first_unallocated < MIN_PAGENO || meta->geometry.first_unallocated - 1 > MAX_PAGENO)) { + WARNING("meta[%u] has invalid next-pageno (%" PRIaPGNO "), skip it", meta_number, meta->geometry.first_unallocated); + return MDBX_CORRUPTED; + } + + const uint64_t used_bytes = meta->geometry.first_unallocated * (uint64_t)meta->pagesize; + if (unlikely(used_bytes > env->dxb_mmap.filesize)) { + /* Here could be a race with DB-shrinking performed by other process */ + int err = osal_filesize(env->lazy_fd, &env->dxb_mmap.filesize); + if (unlikely(err != MDBX_SUCCESS)) + return err; + if (unlikely(used_bytes > env->dxb_mmap.filesize)) { + WARNING("meta[%u] used-bytes (%" PRIu64 ") beyond filesize (%" PRIu64 "), skip it", meta_number, used_bytes, + env->dxb_mmap.filesize); + return MDBX_CORRUPTED; + } + } + if (unlikely(meta->geometry.first_unallocated - 1 > MAX_PAGENO || used_bytes > MAX_MAPSIZE)) { + WARNING("meta[%u] has too large used-space (%" PRIu64 "), skip it", meta_number, used_bytes); + return MDBX_TOO_LARGE; + } + + pgno_t geo_lower = meta->geometry.lower; + uint64_t mapsize_min = geo_lower * (uint64_t)meta->pagesize; + STATIC_ASSERT(MAX_MAPSIZE < PTRDIFF_MAX - MDBX_MAX_PAGESIZE); + STATIC_ASSERT(MIN_MAPSIZE < MAX_MAPSIZE); + STATIC_ASSERT((uint64_t)(MAX_PAGENO + 1) * MDBX_MIN_PAGESIZE % (4ul << 20) == 0); + if (unlikely(mapsize_min < MIN_MAPSIZE || mapsize_min > MAX_MAPSIZE)) { + if (MAX_MAPSIZE != MAX_MAPSIZE64 && mapsize_min > MAX_MAPSIZE && mapsize_min <= MAX_MAPSIZE64) { + eASSERT(env, meta->geometry.first_unallocated - 1 <= MAX_PAGENO && used_bytes <= MAX_MAPSIZE); + WARNING("meta[%u] has too large min-mapsize (%" PRIu64 "), " + "but size of used space still acceptable (%" PRIu64 ")", + meta_number, mapsize_min, used_bytes); + geo_lower = (pgno_t)((mapsize_min = MAX_MAPSIZE) / meta->pagesize); + if (geo_lower > MAX_PAGENO + 1) { + geo_lower = MAX_PAGENO + 1; + mapsize_min = geo_lower * (uint64_t)meta->pagesize; + } + WARNING("meta[%u] consider get-%s pageno is %" PRIaPGNO " instead of wrong %" PRIaPGNO + ", will be corrected on next commit(s)", + meta_number, "lower", geo_lower, meta->geometry.lower); + meta->geometry.lower = geo_lower; + } else { + WARNING("meta[%u] has invalid min-mapsize (%" PRIu64 "), skip it", meta_number, mapsize_min); + return MDBX_VERSION_MISMATCH; + } + } + + pgno_t geo_upper = meta->geometry.upper; + uint64_t mapsize_max = geo_upper * (uint64_t)meta->pagesize; + STATIC_ASSERT(MIN_MAPSIZE < MAX_MAPSIZE); + if (unlikely(mapsize_max > MAX_MAPSIZE || + (MAX_PAGENO + 1) < ceil_powerof2((size_t)mapsize_max, globals.sys_pagesize) / (size_t)meta->pagesize)) { + if (mapsize_max > MAX_MAPSIZE64) { + WARNING("meta[%u] has invalid max-mapsize (%" PRIu64 "), skip it", meta_number, mapsize_max); + return MDBX_VERSION_MISMATCH; + } + /* allow to open large DB from a 32-bit environment */ + eASSERT(env, meta->geometry.first_unallocated - 1 <= MAX_PAGENO && used_bytes <= MAX_MAPSIZE); + WARNING("meta[%u] has too large max-mapsize (%" PRIu64 "), " + "but size of used space still acceptable (%" PRIu64 ")", + meta_number, mapsize_max, used_bytes); + geo_upper = (pgno_t)((mapsize_max = MAX_MAPSIZE) / meta->pagesize); + if (geo_upper > MAX_PAGENO + 1) { + geo_upper = MAX_PAGENO + 1; + mapsize_max = geo_upper * (uint64_t)meta->pagesize; + } + WARNING("meta[%u] consider get-%s pageno is %" PRIaPGNO " instead of wrong %" PRIaPGNO + ", will be corrected on next commit(s)", + meta_number, "upper", geo_upper, meta->geometry.upper); + meta->geometry.upper = geo_upper; + } + + /* LY: check and silently put geometry.now into [geo.lower...geo.upper]. + * + * Copy-with-compaction by old version of libmdbx could produce DB-file + * less than meta.geo.lower bound, in case actual filling is low or no data + * at all. This is not a problem as there is no damage or loss of data. + * Therefore it is better not to consider such situation as an error, but + * silently correct it. */ + pgno_t geo_now = meta->geometry.now; + if (geo_now < geo_lower) + geo_now = geo_lower; + if (geo_now > geo_upper && meta->geometry.first_unallocated <= geo_upper) + geo_now = geo_upper; + + if (unlikely(meta->geometry.first_unallocated > geo_now)) { + WARNING("meta[%u] next-pageno (%" PRIaPGNO ") is beyond end-pgno (%" PRIaPGNO "), skip it", meta_number, + meta->geometry.first_unallocated, geo_now); + return MDBX_CORRUPTED; + } + if (meta->geometry.now != geo_now) { + WARNING("meta[%u] consider geo-%s pageno is %" PRIaPGNO " instead of wrong %" PRIaPGNO + ", will be corrected on next commit(s)", + meta_number, "now", geo_now, meta->geometry.now); + meta->geometry.now = geo_now; + } + + /* GC */ + if (meta->trees.gc.root == P_INVALID) { + if (unlikely(meta->trees.gc.branch_pages || meta->trees.gc.height || meta->trees.gc.items || + meta->trees.gc.leaf_pages || meta->trees.gc.large_pages)) { + WARNING("meta[%u] has false-empty %s, skip it", meta_number, "GC"); + return MDBX_CORRUPTED; + } + } else if (unlikely(meta->trees.gc.root >= meta->geometry.first_unallocated)) { + WARNING("meta[%u] has invalid %s-root %" PRIaPGNO ", skip it", meta_number, "GC", meta->trees.gc.root); + return MDBX_CORRUPTED; + } + + /* MainDB */ + if (meta->trees.main.root == P_INVALID) { + if (unlikely(meta->trees.main.branch_pages || meta->trees.main.height || meta->trees.main.items || + meta->trees.main.leaf_pages || meta->trees.main.large_pages)) { + WARNING("meta[%u] has false-empty %s", meta_number, "MainDB"); + return MDBX_CORRUPTED; + } + } else if (unlikely(meta->trees.main.root >= meta->geometry.first_unallocated)) { + WARNING("meta[%u] has invalid %s-root %" PRIaPGNO ", skip it", meta_number, "MainDB", meta->trees.main.root); + return MDBX_CORRUPTED; + } + + if (unlikely(meta->trees.gc.mod_txnid > txnid)) { + WARNING("meta[%u] has wrong mod_txnid %" PRIaTXN " for %s, skip it", meta_number, meta->trees.gc.mod_txnid, "GC"); + return MDBX_CORRUPTED; + } + + if (unlikely(meta->trees.main.mod_txnid > txnid)) { + WARNING("meta[%u] has wrong mod_txnid %" PRIaTXN " for %s, skip it", meta_number, meta->trees.main.mod_txnid, + "MainDB"); + return MDBX_CORRUPTED; + } + + return MDBX_SUCCESS; +} + +__cold int meta_validate_copy(MDBX_env *env, const meta_t *meta, meta_t *dest) { + *dest = *meta; + return meta_validate(env, dest, data_page(meta), bytes2pgno(env, ptr_dist(meta, env->dxb_mmap.base)), nullptr); +} +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 + +bsr_t mvcc_bind_slot(MDBX_env *env) { + eASSERT(env, env->lck_mmap.lck); + eASSERT(env, env->lck->magic_and_version == MDBX_LOCK_MAGIC); + eASSERT(env, env->lck->os_and_format == MDBX_LOCK_FORMAT); + + bsr_t result = {lck_rdt_lock(env), nullptr}; + if (unlikely(MDBX_IS_ERROR(result.err))) + return result; + if (unlikely(env->flags & ENV_FATAL_ERROR)) { + lck_rdt_unlock(env); + result.err = MDBX_PANIC; + return result; + } + if (unlikely(!env->dxb_mmap.base)) { + lck_rdt_unlock(env); + result.err = MDBX_EPERM; + return result; + } + + if (unlikely(env->registered_reader_pid != env->pid)) { + result.err = lck_rpid_set(env); + if (unlikely(result.err != MDBX_SUCCESS)) { + lck_rdt_unlock(env); + return result; + } + env->registered_reader_pid = env->pid; + } + + result.err = MDBX_SUCCESS; + size_t slot, nreaders; + while (1) { + nreaders = env->lck->rdt_length.weak; + for (slot = 0; slot < nreaders; slot++) + if (!atomic_load32(&env->lck->rdt[slot].pid, mo_AcquireRelease)) + break; + + if (likely(slot < env->max_readers)) + break; + + result.err = mvcc_cleanup_dead(env, true, nullptr); + if (result.err != MDBX_RESULT_TRUE) { + lck_rdt_unlock(env); + result.err = (result.err == MDBX_SUCCESS) ? MDBX_READERS_FULL : result.err; + return result; + } + } + + result.rslot = &env->lck->rdt[slot]; + /* Claim the reader slot, carefully since other code + * uses the reader table un-mutexed: First reset the + * slot, next publish it in lck->rdt_length. After + * that, it is safe for mdbx_env_close() to touch it. + * When it will be closed, we can finally claim it. */ + atomic_store32(&result.rslot->pid, 0, mo_AcquireRelease); + safe64_reset(&result.rslot->txnid, true); + if (slot == nreaders) + env->lck->rdt_length.weak = (uint32_t)++nreaders; + result.rslot->tid.weak = (env->flags & MDBX_NOSTICKYTHREADS) ? 0 : osal_thread_self(); + atomic_store32(&result.rslot->pid, env->pid, mo_AcquireRelease); + lck_rdt_unlock(env); + + if (likely(env->flags & ENV_TXKEY)) { + eASSERT(env, env->registered_reader_pid == env->pid); + thread_rthc_set(env->me_txkey, result.rslot); + } + return result; +} + +__hot txnid_t mvcc_shapshot_oldest(MDBX_env *const env, const txnid_t steady) { + const uint32_t nothing_changed = MDBX_STRING_TETRAD("None"); + eASSERT(env, steady <= env->basal_txn->txnid); + + lck_t *const lck = env->lck_mmap.lck; + if (unlikely(lck == nullptr /* exclusive without-lck mode */)) { + eASSERT(env, env->lck == lckless_stub(env)); + env->lck->rdt_refresh_flag.weak = nothing_changed; + return env->lck->cached_oldest.weak = steady; + } + + const txnid_t prev_oldest = atomic_load64(&lck->cached_oldest, mo_AcquireRelease); + eASSERT(env, steady >= prev_oldest); + + txnid_t new_oldest = prev_oldest; + while (nothing_changed != atomic_load32(&lck->rdt_refresh_flag, mo_AcquireRelease)) { + lck->rdt_refresh_flag.weak = nothing_changed; + jitter4testing(false); + const size_t snap_nreaders = atomic_load32(&lck->rdt_length, mo_AcquireRelease); + new_oldest = steady; + + for (size_t i = 0; i < snap_nreaders; ++i) { + const uint32_t pid = atomic_load32(&lck->rdt[i].pid, mo_AcquireRelease); + if (!pid) + continue; + jitter4testing(true); + + const txnid_t rtxn = safe64_read(&lck->rdt[i].txnid); + if (unlikely(rtxn < prev_oldest)) { + if (unlikely(nothing_changed == atomic_load32(&lck->rdt_refresh_flag, mo_AcquireRelease)) && + safe64_reset_compare(&lck->rdt[i].txnid, rtxn)) { + NOTICE("kick stuck reader[%zu of %zu].pid_%u %" PRIaTXN " < prev-oldest %" PRIaTXN ", steady-txn %" PRIaTXN, + i, snap_nreaders, pid, rtxn, prev_oldest, steady); + } + continue; + } + + if (rtxn < new_oldest) { + new_oldest = rtxn; + if (!MDBX_DEBUG && !MDBX_FORCE_ASSERTIONS && new_oldest == prev_oldest) + break; + } + } + } + + if (new_oldest != prev_oldest) { + VERBOSE("update oldest %" PRIaTXN " -> %" PRIaTXN, prev_oldest, new_oldest); + eASSERT(env, new_oldest >= lck->cached_oldest.weak); + atomic_store64(&lck->cached_oldest, new_oldest, mo_Relaxed); + } + return new_oldest; +} + +pgno_t mvcc_snapshot_largest(const MDBX_env *env, pgno_t last_used_page) { + lck_t *const lck = env->lck_mmap.lck; + if (likely(lck != nullptr /* check for exclusive without-lck mode */)) { + retry:; + const size_t snap_nreaders = atomic_load32(&lck->rdt_length, mo_AcquireRelease); + for (size_t i = 0; i < snap_nreaders; ++i) { + if (atomic_load32(&lck->rdt[i].pid, mo_AcquireRelease)) { + /* jitter4testing(true); */ + const pgno_t snap_pages = atomic_load32(&lck->rdt[i].snapshot_pages_used, mo_Relaxed); + const txnid_t snap_txnid = safe64_read(&lck->rdt[i].txnid); + if (unlikely(snap_pages != atomic_load32(&lck->rdt[i].snapshot_pages_used, mo_AcquireRelease) || + snap_txnid != safe64_read(&lck->rdt[i].txnid))) + goto retry; + if (last_used_page < snap_pages && snap_txnid <= env->basal_txn->txnid) + last_used_page = snap_pages; + } + } + } + + return last_used_page; +} + +/* Find largest mvcc-snapshot still referenced by this process. */ +pgno_t mvcc_largest_this(MDBX_env *env, pgno_t largest) { + lck_t *const lck = env->lck_mmap.lck; + if (likely(lck != nullptr /* exclusive mode */)) { + const size_t snap_nreaders = atomic_load32(&lck->rdt_length, mo_AcquireRelease); + for (size_t i = 0; i < snap_nreaders; ++i) { + retry: + if (atomic_load32(&lck->rdt[i].pid, mo_AcquireRelease) == env->pid) { + /* jitter4testing(true); */ + const pgno_t snap_pages = atomic_load32(&lck->rdt[i].snapshot_pages_used, mo_Relaxed); + const txnid_t snap_txnid = safe64_read(&lck->rdt[i].txnid); + if (unlikely(snap_pages != atomic_load32(&lck->rdt[i].snapshot_pages_used, mo_AcquireRelease) || + snap_txnid != safe64_read(&lck->rdt[i].txnid))) + goto retry; + if (largest < snap_pages && + atomic_load64(&lck->cached_oldest, mo_AcquireRelease) <= + /* ignore pending updates */ snap_txnid && + snap_txnid <= MAX_TXNID) + largest = snap_pages; + } + } + } + return largest; +} + +static bool pid_insert(uint32_t *list, uint32_t pid) { + /* binary search of pid in list */ + size_t base = 0; + size_t cursor = 1; + int32_t val = 0; + size_t n = /* length */ list[0]; + + while (n > 0) { + size_t pivot = n >> 1; + cursor = base + pivot + 1; + val = pid - list[cursor]; + + if (val < 0) { + n = pivot; + } else if (val > 0) { + base = cursor; + n -= pivot + 1; + } else { + /* found, so it's a duplicate */ + return false; + } + } + + if (val > 0) + ++cursor; + + list[0]++; + for (n = list[0]; n > cursor; n--) + list[n] = list[n - 1]; + list[n] = pid; + return true; +} + +__cold MDBX_INTERNAL int mvcc_cleanup_dead(MDBX_env *env, int rdt_locked, int *dead) { + int rc = check_env(env, true); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + eASSERT(env, rdt_locked >= 0); + lck_t *const lck = env->lck_mmap.lck; + if (unlikely(lck == nullptr)) { + /* exclusive mode */ + if (dead) + *dead = 0; + return MDBX_SUCCESS; + } + + const size_t snap_nreaders = atomic_load32(&lck->rdt_length, mo_AcquireRelease); + uint32_t pidsbuf_onstask[142]; + uint32_t *const pids = (snap_nreaders < ARRAY_LENGTH(pidsbuf_onstask)) + ? pidsbuf_onstask + : osal_malloc((snap_nreaders + 1) * sizeof(uint32_t)); + if (unlikely(!pids)) + return MDBX_ENOMEM; + + pids[0] = 0; + int count = 0; + for (size_t i = 0; i < snap_nreaders; i++) { + const uint32_t pid = atomic_load32(&lck->rdt[i].pid, mo_AcquireRelease); + if (pid == 0) + continue /* skip empty */; + if (pid == env->pid) + continue /* skip self */; + if (!pid_insert(pids, pid)) + continue /* such pid already processed */; + + int err = lck_rpid_check(env, pid); + if (err == MDBX_RESULT_TRUE) + continue /* reader is live */; + + if (err != MDBX_SUCCESS) { + rc = err; + break /* lck_rpid_check() failed */; + } + + /* stale reader found */ + if (!rdt_locked) { + err = lck_rdt_lock(env); + if (MDBX_IS_ERROR(err)) { + rc = err; + break; + } + + rdt_locked = -1; + if (err == MDBX_RESULT_TRUE) { + /* mutex recovered, the mdbx_ipclock_failed() checked all readers */ + rc = MDBX_RESULT_TRUE; + break; + } + + /* a other process may have clean and reused slot, recheck */ + if (lck->rdt[i].pid.weak != pid) + continue; + + err = lck_rpid_check(env, pid); + if (MDBX_IS_ERROR(err)) { + rc = err; + break; + } + + if (err != MDBX_SUCCESS) + continue /* the race with other process, slot reused */; + } + + /* clean it */ + for (size_t ii = i; ii < snap_nreaders; ii++) { + if (lck->rdt[ii].pid.weak == pid) { + DEBUG("clear stale reader pid %" PRIuPTR " txn %" PRIaTXN, (size_t)pid, lck->rdt[ii].txnid.weak); + atomic_store32(&lck->rdt[ii].pid, 0, mo_Relaxed); + atomic_store32(&lck->rdt_refresh_flag, true, mo_AcquireRelease); + count++; + } + } + } + + if (likely(!MDBX_IS_ERROR(rc))) + atomic_store64(&lck->readers_check_timestamp, osal_monotime(), mo_Relaxed); + + if (rdt_locked < 0) + lck_rdt_unlock(env); + + if (pids != pidsbuf_onstask) + osal_free(pids); + + if (dead) + *dead = count; + return rc; +} + +__cold txnid_t mvcc_kick_laggards(MDBX_env *env, const txnid_t straggler) { + DEBUG("DB size maxed out by reading #%" PRIaTXN, straggler); + osal_memory_fence(mo_AcquireRelease, false); + MDBX_hsr_func *const callback = env->hsr_callback; + txnid_t oldest = 0; + bool notify_eof_of_loop = false; + int retry = 0; + do { + const txnid_t steady = env->txn->tw.troika.txnid[env->txn->tw.troika.prefer_steady]; + env->lck->rdt_refresh_flag.weak = /* force refresh */ true; + oldest = mvcc_shapshot_oldest(env, steady); + eASSERT(env, oldest < env->basal_txn->txnid); + eASSERT(env, oldest >= straggler); + eASSERT(env, oldest >= env->lck->cached_oldest.weak); + + lck_t *const lck = env->lck_mmap.lck; + if (oldest == steady || oldest > straggler || /* without-LCK mode */ !lck) + break; + + if (MDBX_IS_ERROR(mvcc_cleanup_dead(env, false, nullptr))) + break; + + reader_slot_t *stucked = nullptr; + uint64_t hold_retired = 0; + for (size_t i = 0; i < lck->rdt_length.weak; ++i) { + uint32_t pid; + reader_slot_t *const rslot = &lck->rdt[i]; + txnid_t rtxn = safe64_read(&rslot->txnid); + retry: + if (rtxn == straggler && (pid = atomic_load32(&rslot->pid, mo_AcquireRelease)) != 0) { + const uint64_t tid = safe64_read(&rslot->tid); + if (tid == MDBX_TID_TXN_PARKED) { + /* Читающая транзакция была помечена владельцем как "припаркованная", + * т.е. подлежащая асинхронному прерыванию, либо восстановлению + * по активности читателя. + * + * Если первый CAS(slot->tid) будет успешным, то + * safe64_reset_compare() безопасно очистит txnid, либо откажется + * из-за того что читатель сбросил и/или перезапустил транзакцию. + * При этом читатеть может не заметить вытестения, если приступит + * к завершению транзакции. Все эти исходы нас устраивют. + * + * Если первый CAS(slot->tid) будет НЕ успешным, то значит читатеть + * восстановил транзакцию, либо завершил её, либо даже освободил слот. + */ + bool ousted = +#if MDBX_64BIT_CAS + atomic_cas64(&rslot->tid, MDBX_TID_TXN_PARKED, MDBX_TID_TXN_OUSTED); +#else + atomic_cas32(&rslot->tid.low, (uint32_t)MDBX_TID_TXN_PARKED, (uint32_t)MDBX_TID_TXN_OUSTED); +#endif + if (likely(ousted)) { + ousted = safe64_reset_compare(&rslot->txnid, rtxn); + NOTICE("ousted-%s parked read-txn %" PRIaTXN ", pid %u, tid 0x%" PRIx64, ousted ? "complete" : "half", rtxn, + pid, tid); + eASSERT(env, ousted || safe64_read(&rslot->txnid) > straggler); + continue; + } + rtxn = safe64_read(&rslot->txnid); + goto retry; + } + hold_retired = atomic_load64(&lck->rdt[i].snapshot_pages_retired, mo_Relaxed); + stucked = rslot; + } + } + + if (!callback || !stucked) + break; + + uint32_t pid = atomic_load32(&stucked->pid, mo_AcquireRelease); + uint64_t tid = safe64_read(&stucked->tid); + if (safe64_read(&stucked->txnid) != straggler || !pid) + continue; + + const meta_ptr_t head = meta_recent(env, &env->txn->tw.troika); + const txnid_t gap = (head.txnid - straggler) / xMDBX_TXNID_STEP; + const uint64_t head_retired = unaligned_peek_u64(4, head.ptr_c->pages_retired); + const size_t space = (head_retired > hold_retired) ? pgno2bytes(env, (pgno_t)(head_retired - hold_retired)) : 0; + int rc = callback(env, env->txn, pid, (mdbx_tid_t)((intptr_t)tid), straggler, + (gap < UINT_MAX) ? (unsigned)gap : UINT_MAX, space, retry); + if (rc < 0) + /* hsr returned error and/or agree MDBX_MAP_FULL error */ + break; + + if (rc > 0) { + if (rc == 1) { + /* hsr reported transaction (will be) aborted asynchronous */ + safe64_reset_compare(&stucked->txnid, straggler); + } else { + /* hsr reported reader process was killed and slot should be cleared */ + safe64_reset(&stucked->txnid, true); + atomic_store64(&stucked->tid, 0, mo_Relaxed); + atomic_store32(&stucked->pid, 0, mo_AcquireRelease); + } + } else if (!notify_eof_of_loop) { +#if MDBX_ENABLE_PROFGC + env->lck->pgops.gc_prof.kicks += 1; +#endif /* MDBX_ENABLE_PROFGC */ + notify_eof_of_loop = true; + } + + } while (++retry < INT_MAX); + + if (notify_eof_of_loop) { + /* notify end of hsr-loop */ + const txnid_t turn = oldest - straggler; + if (turn) + NOTICE("hsr-kick: done turn %" PRIaTXN " -> %" PRIaTXN " +%" PRIaTXN, straggler, oldest, turn); + callback(env, env->txn, 0, 0, straggler, (turn < UINT_MAX) ? (unsigned)turn : UINT_MAX, 0, -retry); + } + return oldest; +} +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \note Please refer to the COPYRIGHT file for explanations license change, +/// credits and acknowledgments. +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 + +__hot int __must_check_result node_add_dupfix(MDBX_cursor *mc, size_t indx, const MDBX_val *key) { + page_t *mp = mc->pg[mc->top]; + MDBX_ANALYSIS_ASSUME(key != nullptr); + DKBUF_DEBUG; + DEBUG("add to leaf2-%spage %" PRIaPGNO " index %zi, " + " key size %" PRIuPTR " [%s]", + is_subpage(mp) ? "sub-" : "", mp->pgno, indx, key ? key->iov_len : 0, DKEY_DEBUG(key)); + + cASSERT(mc, key); + cASSERT(mc, page_type_compat(mp) == (P_LEAF | P_DUPFIX)); + const size_t ksize = mc->tree->dupfix_size; + cASSERT(mc, ksize == key->iov_len); + const size_t nkeys = page_numkeys(mp); + cASSERT(mc, (((ksize & page_numkeys(mp)) ^ mp->upper) & 1) == 0); + + /* Just using these for counting */ + const intptr_t lower = mp->lower + sizeof(indx_t); + const intptr_t upper = mp->upper - (ksize - sizeof(indx_t)); + if (unlikely(lower > upper)) { + mc->txn->flags |= MDBX_TXN_ERROR; + return MDBX_PAGE_FULL; + } + mp->lower = (indx_t)lower; + mp->upper = (indx_t)upper; + + void *const ptr = page_dupfix_ptr(mp, indx, ksize); + cASSERT(mc, nkeys >= indx); + const size_t diff = nkeys - indx; + if (likely(diff > 0)) + /* Move higher keys up one slot. */ + memmove(ptr_disp(ptr, ksize), ptr, diff * ksize); + /* insert new key */ + memcpy(ptr, key->iov_base, ksize); + + cASSERT(mc, (((ksize & page_numkeys(mp)) ^ mp->upper) & 1) == 0); + return MDBX_SUCCESS; +} + +int __must_check_result node_add_branch(MDBX_cursor *mc, size_t indx, const MDBX_val *key, pgno_t pgno) { + page_t *mp = mc->pg[mc->top]; + DKBUF_DEBUG; + DEBUG("add to branch-%spage %" PRIaPGNO " index %zi, node-pgno %" PRIaPGNO " key size %" PRIuPTR " [%s]", + is_subpage(mp) ? "sub-" : "", mp->pgno, indx, pgno, key ? key->iov_len : 0, DKEY_DEBUG(key)); + + cASSERT(mc, page_type(mp) == P_BRANCH); + STATIC_ASSERT(NODESIZE % 2 == 0); + + /* Move higher pointers up one slot. */ + const size_t nkeys = page_numkeys(mp); + cASSERT(mc, nkeys >= indx); + for (size_t i = nkeys; i > indx; --i) + mp->entries[i] = mp->entries[i - 1]; + + /* Adjust free space offsets. */ + const size_t branch_bytes = branch_size(mc->txn->env, key); + const intptr_t lower = mp->lower + sizeof(indx_t); + const intptr_t upper = mp->upper - (branch_bytes - sizeof(indx_t)); + if (unlikely(lower > upper)) { + mc->txn->flags |= MDBX_TXN_ERROR; + return MDBX_PAGE_FULL; + } + mp->lower = (indx_t)lower; + mp->entries[indx] = mp->upper = (indx_t)upper; + + /* Write the node data. */ + node_t *node = page_node(mp, indx); + node_set_pgno(node, pgno); + node_set_flags(node, 0); + UNALIGNED_POKE_8(node, node_t, extra, 0); + node_set_ks(node, 0); + if (likely(key != nullptr)) { + node_set_ks(node, key->iov_len); + memcpy(node_key(node), key->iov_base, key->iov_len); + } + return MDBX_SUCCESS; +} + +__hot int __must_check_result node_add_leaf(MDBX_cursor *mc, size_t indx, const MDBX_val *key, MDBX_val *data, + unsigned flags) { + MDBX_ANALYSIS_ASSUME(key != nullptr); + MDBX_ANALYSIS_ASSUME(data != nullptr); + page_t *mp = mc->pg[mc->top]; + DKBUF_DEBUG; + DEBUG("add to leaf-%spage %" PRIaPGNO " index %zi, data size %" PRIuPTR " key size %" PRIuPTR " [%s]", + is_subpage(mp) ? "sub-" : "", mp->pgno, indx, data ? data->iov_len : 0, key ? key->iov_len : 0, + DKEY_DEBUG(key)); + cASSERT(mc, key != nullptr && data != nullptr); + cASSERT(mc, page_type_compat(mp) == P_LEAF); + page_t *largepage = nullptr; + + size_t node_bytes; + if (unlikely(flags & N_BIG)) { + /* Data already on large/overflow page. */ + STATIC_ASSERT(sizeof(pgno_t) % 2 == 0); + node_bytes = node_size_len(key->iov_len, 0) + sizeof(pgno_t) + sizeof(indx_t); + cASSERT(mc, page_room(mp) >= node_bytes); + } else if (unlikely(node_size(key, data) > mc->txn->env->leaf_nodemax)) { + /* Put data on large/overflow page. */ + if (unlikely(mc->tree->flags & MDBX_DUPSORT)) { + ERROR("Unexpected target %s flags 0x%x for large data-item", "dupsort-db", mc->tree->flags); + return MDBX_PROBLEM; + } + if (unlikely(flags & (N_DUP | N_TREE))) { + ERROR("Unexpected target %s flags 0x%x for large data-item", "node", flags); + return MDBX_PROBLEM; + } + cASSERT(mc, page_room(mp) >= leaf_size(mc->txn->env, key, data)); + const pgno_t ovpages = largechunk_npages(mc->txn->env, data->iov_len); + const pgr_t npr = page_new_large(mc, ovpages); + if (unlikely(npr.err != MDBX_SUCCESS)) + return npr.err; + largepage = npr.page; + DEBUG("allocated %u large/overflow page(s) %" PRIaPGNO "for %" PRIuPTR " data bytes", largepage->pages, + largepage->pgno, data->iov_len); + flags |= N_BIG; + node_bytes = node_size_len(key->iov_len, 0) + sizeof(pgno_t) + sizeof(indx_t); + cASSERT(mc, node_bytes == leaf_size(mc->txn->env, key, data)); + } else { + cASSERT(mc, page_room(mp) >= leaf_size(mc->txn->env, key, data)); + node_bytes = node_size(key, data) + sizeof(indx_t); + cASSERT(mc, node_bytes == leaf_size(mc->txn->env, key, data)); + } + + /* Move higher pointers up one slot. */ + const size_t nkeys = page_numkeys(mp); + cASSERT(mc, nkeys >= indx); + for (size_t i = nkeys; i > indx; --i) + mp->entries[i] = mp->entries[i - 1]; + + /* Adjust free space offsets. */ + const intptr_t lower = mp->lower + sizeof(indx_t); + const intptr_t upper = mp->upper - (node_bytes - sizeof(indx_t)); + if (unlikely(lower > upper)) { + mc->txn->flags |= MDBX_TXN_ERROR; + return MDBX_PAGE_FULL; + } + mp->lower = (indx_t)lower; + mp->entries[indx] = mp->upper = (indx_t)upper; + + /* Write the node data. */ + node_t *node = page_node(mp, indx); + node_set_ks(node, key->iov_len); + node_set_flags(node, (uint8_t)flags); + UNALIGNED_POKE_8(node, node_t, extra, 0); + node_set_ds(node, data->iov_len); + memcpy(node_key(node), key->iov_base, key->iov_len); + + void *nodedata = node_data(node); + if (likely(largepage == nullptr)) { + if (unlikely(flags & N_BIG)) { + memcpy(nodedata, data->iov_base, sizeof(pgno_t)); + return MDBX_SUCCESS; + } + } else { + poke_pgno(nodedata, largepage->pgno); + nodedata = page_data(largepage); + } + if (unlikely(flags & MDBX_RESERVE)) + data->iov_base = nodedata; + else if (likely(data->iov_len /* to avoid UBSAN traps */)) + memcpy(nodedata, data->iov_base, data->iov_len); + return MDBX_SUCCESS; +} + +__hot void node_del(MDBX_cursor *mc, size_t ksize) { + page_t *mp = mc->pg[mc->top]; + const size_t hole = mc->ki[mc->top]; + const size_t nkeys = page_numkeys(mp); + + DEBUG("delete node %zu on %s page %" PRIaPGNO, hole, is_leaf(mp) ? "leaf" : "branch", mp->pgno); + cASSERT(mc, hole < nkeys); + + if (is_dupfix_leaf(mp)) { + cASSERT(mc, ksize >= sizeof(indx_t)); + size_t diff = nkeys - 1 - hole; + void *const base = page_dupfix_ptr(mp, hole, ksize); + if (diff) + memmove(base, ptr_disp(base, ksize), diff * ksize); + cASSERT(mc, mp->lower >= sizeof(indx_t)); + mp->lower -= sizeof(indx_t); + cASSERT(mc, (size_t)UINT16_MAX - mp->upper >= ksize - sizeof(indx_t)); + mp->upper += (indx_t)(ksize - sizeof(indx_t)); + cASSERT(mc, (((ksize & page_numkeys(mp)) ^ mp->upper) & 1) == 0); + return; + } + + node_t *node = page_node(mp, hole); + cASSERT(mc, !is_branch(mp) || hole || node_ks(node) == 0); + size_t hole_size = NODESIZE + node_ks(node); + if (is_leaf(mp)) + hole_size += (node_flags(node) & N_BIG) ? sizeof(pgno_t) : node_ds(node); + hole_size = EVEN_CEIL(hole_size); + + const indx_t hole_offset = mp->entries[hole]; + size_t r, w; + for (r = w = 0; r < nkeys; r++) + if (r != hole) + mp->entries[w++] = (mp->entries[r] < hole_offset) ? mp->entries[r] + (indx_t)hole_size : mp->entries[r]; + + void *const base = ptr_disp(mp, mp->upper + PAGEHDRSZ); + memmove(ptr_disp(base, hole_size), base, hole_offset - mp->upper); + + cASSERT(mc, mp->lower >= sizeof(indx_t)); + mp->lower -= sizeof(indx_t); + cASSERT(mc, (size_t)UINT16_MAX - mp->upper >= hole_size); + mp->upper += (indx_t)hole_size; + + if (AUDIT_ENABLED()) { + const uint8_t checking = mc->checking; + mc->checking |= z_updating; + const int page_check_err = page_check(mc, mp); + mc->checking = checking; + cASSERT(mc, page_check_err == MDBX_SUCCESS); + } +} + +__noinline int node_read_bigdata(MDBX_cursor *mc, const node_t *node, MDBX_val *data, const page_t *mp) { + cASSERT(mc, node_flags(node) == N_BIG && data->iov_len == node_ds(node)); + + pgr_t lp = page_get_large(mc, node_largedata_pgno(node), mp->txnid); + if (unlikely((lp.err != MDBX_SUCCESS))) { + DEBUG("read large/overflow page %" PRIaPGNO " failed", node_largedata_pgno(node)); + return lp.err; + } + + cASSERT(mc, page_type(lp.page) == P_LARGE); + data->iov_base = page_data(lp.page); + if (!MDBX_DISABLE_VALIDATION) { + const MDBX_env *env = mc->txn->env; + const size_t dsize = data->iov_len; + const unsigned npages = largechunk_npages(env, dsize); + if (unlikely(lp.page->pages < npages)) + return bad_page(lp.page, "too less n-pages %u for bigdata-node (%zu bytes)", lp.page->pages, dsize); + } + return MDBX_SUCCESS; +} + +node_t *node_shrink(page_t *mp, size_t indx, node_t *node) { + assert(node == page_node(mp, indx)); + page_t *sp = (page_t *)node_data(node); + assert(is_subpage(sp) && page_numkeys(sp) > 0); + const size_t delta = EVEN_FLOOR(page_room(sp) /* avoid the node uneven-sized */); + if (unlikely(delta) == 0) + return node; + + /* Prepare to shift upward, set len = length(subpage part to shift) */ + size_t nsize = node_ds(node) - delta, len = nsize; + assert(nsize % 1 == 0); + if (!is_dupfix_leaf(sp)) { + len = PAGEHDRSZ; + page_t *xp = ptr_disp(sp, delta); /* destination subpage */ + for (intptr_t i = page_numkeys(sp); --i >= 0;) { + assert(sp->entries[i] >= delta); + xp->entries[i] = (indx_t)(sp->entries[i] - delta); + } + } + assert(sp->upper >= sp->lower + delta); + sp->upper -= (indx_t)delta; + sp->pgno = mp->pgno; + node_set_ds(node, nsize); + + /* Shift upward */ + void *const base = ptr_disp(mp, mp->upper + PAGEHDRSZ); + memmove(ptr_disp(base, delta), base, ptr_dist(sp, base) + len); + + const size_t pivot = mp->entries[indx]; + for (intptr_t i = page_numkeys(mp); --i >= 0;) { + if (mp->entries[i] <= pivot) { + assert((size_t)UINT16_MAX - mp->entries[i] >= delta); + mp->entries[i] += (indx_t)delta; + } + } + assert((size_t)UINT16_MAX - mp->upper >= delta); + mp->upper += (indx_t)delta; + + return ptr_disp(node, delta); +} + +__hot struct node_search_result node_search(MDBX_cursor *mc, const MDBX_val *key) { + page_t *mp = mc->pg[mc->top]; + const intptr_t nkeys = page_numkeys(mp); + DKBUF_DEBUG; + + DEBUG("searching %zu keys in %s %spage %" PRIaPGNO, nkeys, is_leaf(mp) ? "leaf" : "branch", + is_subpage(mp) ? "sub-" : "", mp->pgno); + + struct node_search_result ret; + ret.exact = false; + STATIC_ASSERT(P_BRANCH == 1); + intptr_t low = mp->flags & P_BRANCH; + intptr_t high = nkeys - 1; + if (unlikely(high < low)) { + mc->ki[mc->top] = 0; + ret.node = nullptr; + return ret; + } + + intptr_t i; + MDBX_cmp_func *cmp = mc->clc->k.cmp; + MDBX_val nodekey; + if (unlikely(is_dupfix_leaf(mp))) { + cASSERT(mc, mp->dupfix_ksize == mc->tree->dupfix_size); + nodekey.iov_len = mp->dupfix_ksize; + do { + i = (low + high) >> 1; + nodekey.iov_base = page_dupfix_ptr(mp, i, nodekey.iov_len); + cASSERT(mc, ptr_disp(mp, mc->txn->env->ps) >= ptr_disp(nodekey.iov_base, nodekey.iov_len)); + int cr = cmp(key, &nodekey); + DEBUG("found leaf index %zu [%s], rc = %i", i, DKEY_DEBUG(&nodekey), cr); + if (cr > 0) + low = ++i; + else if (cr < 0) + high = i - 1; + else { + ret.exact = true; + break; + } + } while (likely(low <= high)); + + /* store the key index */ + mc->ki[mc->top] = (indx_t)i; + ret.node = (i < nkeys) ? /* fake for DUPFIX */ (node_t *)(intptr_t)-1 + : /* There is no entry larger or equal to the key. */ nullptr; + return ret; + } + + if (MDBX_UNALIGNED_OK < 4 && is_branch(mp) && cmp == cmp_int_align2) + /* Branch pages have no data, so if using integer keys, + * alignment is guaranteed. Use faster cmp_int_align4(). */ + cmp = cmp_int_align4; + + node_t *node; + do { + i = (low + high) >> 1; + node = page_node(mp, i); + nodekey.iov_len = node_ks(node); + nodekey.iov_base = node_key(node); + cASSERT(mc, ptr_disp(mp, mc->txn->env->ps) >= ptr_disp(nodekey.iov_base, nodekey.iov_len)); + int cr = cmp(key, &nodekey); + if (is_leaf(mp)) + DEBUG("found leaf index %zu [%s], rc = %i", i, DKEY_DEBUG(&nodekey), cr); + else + DEBUG("found branch index %zu [%s -> %" PRIaPGNO "], rc = %i", i, DKEY_DEBUG(&nodekey), node_pgno(node), cr); + if (cr > 0) + low = ++i; + else if (cr < 0) + high = i - 1; + else { + ret.exact = true; + break; + } + } while (likely(low <= high)); + + /* store the key index */ + mc->ki[mc->top] = (indx_t)i; + ret.node = (i < nkeys) ? page_node(mp, i) : /* There is no entry larger or equal to the key. */ nullptr; + return ret; +} +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 +/// +/// https://en.wikipedia.org/wiki/Operating_system_abstraction_layer + +#if defined(_WIN32) || defined(_WIN64) + +#include +#include + +#if !MDBX_WITHOUT_MSVC_CRT && defined(_DEBUG) +#include +#endif + +static int waitstatus2errcode(DWORD result) { + switch (result) { + case WAIT_OBJECT_0: + return MDBX_SUCCESS; + case WAIT_FAILED: + return (int)GetLastError(); + case WAIT_ABANDONED: + return ERROR_ABANDONED_WAIT_0; + case WAIT_IO_COMPLETION: + return ERROR_USER_APC; + case WAIT_TIMEOUT: + return ERROR_TIMEOUT; + default: + return ERROR_UNHANDLED_ERROR; + } +} + +/* Map a result from an NTAPI call to WIN32 error code. */ +static int ntstatus2errcode(NTSTATUS status) { + DWORD dummy; + OVERLAPPED ov; + memset(&ov, 0, sizeof(ov)); + ov.Internal = status; + /* Zap: '_Param_(1)' could be '0' */ + MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(6387); + return GetOverlappedResult(nullptr, &ov, &dummy, FALSE) ? MDBX_SUCCESS : (int)GetLastError(); +} + +/* We use native NT APIs to setup the memory map, so that we can + * let the DB file grow incrementally instead of always preallocating + * the full size. These APIs are defined in and + * but those headers are meant for driver-level development and + * conflict with the regular user-level headers, so we explicitly + * declare them here. Using these APIs also means we must link to + * ntdll.dll, which is not linked by default in user code. */ + +extern NTSTATUS NTAPI NtCreateSection(OUT PHANDLE SectionHandle, IN ACCESS_MASK DesiredAccess, + IN OPTIONAL POBJECT_ATTRIBUTES ObjectAttributes, + IN OPTIONAL PLARGE_INTEGER MaximumSize, IN ULONG SectionPageProtection, + IN ULONG AllocationAttributes, IN OPTIONAL HANDLE FileHandle); + +typedef struct _SECTION_BASIC_INFORMATION { + ULONG Unknown; + ULONG SectionAttributes; + LARGE_INTEGER SectionSize; +} SECTION_BASIC_INFORMATION, *PSECTION_BASIC_INFORMATION; + +extern NTSTATUS NTAPI NtMapViewOfSection(IN HANDLE SectionHandle, IN HANDLE ProcessHandle, IN OUT PVOID *BaseAddress, + IN ULONG_PTR ZeroBits, IN SIZE_T CommitSize, + IN OUT OPTIONAL PLARGE_INTEGER SectionOffset, IN OUT PSIZE_T ViewSize, + IN SECTION_INHERIT InheritDisposition, IN ULONG AllocationType, + IN ULONG Win32Protect); + +extern NTSTATUS NTAPI NtUnmapViewOfSection(IN HANDLE ProcessHandle, IN OPTIONAL PVOID BaseAddress); + +/* Zap: Inconsistent annotation for 'NtClose'... */ +MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(28251) +extern NTSTATUS NTAPI NtClose(HANDLE Handle); + +extern NTSTATUS NTAPI NtAllocateVirtualMemory(IN HANDLE ProcessHandle, IN OUT PVOID *BaseAddress, IN ULONG_PTR ZeroBits, + IN OUT PSIZE_T RegionSize, IN ULONG AllocationType, IN ULONG Protect); + +extern NTSTATUS NTAPI NtFreeVirtualMemory(IN HANDLE ProcessHandle, IN PVOID *BaseAddress, IN OUT PSIZE_T RegionSize, + IN ULONG FreeType); + +#ifndef WOF_CURRENT_VERSION +typedef struct _WOF_EXTERNAL_INFO { + DWORD Version; + DWORD Provider; +} WOF_EXTERNAL_INFO, *PWOF_EXTERNAL_INFO; +#endif /* WOF_CURRENT_VERSION */ + +#ifndef WIM_PROVIDER_CURRENT_VERSION +#define WIM_PROVIDER_HASH_SIZE 20 + +typedef struct _WIM_PROVIDER_EXTERNAL_INFO { + DWORD Version; + DWORD Flags; + LARGE_INTEGER DataSourceId; + BYTE ResourceHash[WIM_PROVIDER_HASH_SIZE]; +} WIM_PROVIDER_EXTERNAL_INFO, *PWIM_PROVIDER_EXTERNAL_INFO; +#endif /* WIM_PROVIDER_CURRENT_VERSION */ + +#ifndef FILE_PROVIDER_CURRENT_VERSION +typedef struct _FILE_PROVIDER_EXTERNAL_INFO_V1 { + ULONG Version; + ULONG Algorithm; + ULONG Flags; +} FILE_PROVIDER_EXTERNAL_INFO_V1, *PFILE_PROVIDER_EXTERNAL_INFO_V1; +#endif /* FILE_PROVIDER_CURRENT_VERSION */ + +#ifndef STATUS_OBJECT_NOT_EXTERNALLY_BACKED +#define STATUS_OBJECT_NOT_EXTERNALLY_BACKED ((NTSTATUS)0xC000046DL) +#endif +#ifndef STATUS_INVALID_DEVICE_REQUEST +#define STATUS_INVALID_DEVICE_REQUEST ((NTSTATUS)0xC0000010L) +#endif +#ifndef STATUS_NOT_SUPPORTED +#define STATUS_NOT_SUPPORTED ((NTSTATUS)0xC00000BBL) +#endif + +#ifndef FILE_DEVICE_FILE_SYSTEM +#define FILE_DEVICE_FILE_SYSTEM 0x00000009 +#endif + +#ifndef FSCTL_GET_EXTERNAL_BACKING +#define FSCTL_GET_EXTERNAL_BACKING CTL_CODE(FILE_DEVICE_FILE_SYSTEM, 196, METHOD_BUFFERED, FILE_ANY_ACCESS) +#endif + +#ifndef ERROR_NOT_CAPABLE +#define ERROR_NOT_CAPABLE 775L +#endif + +#endif /* _WIN32 || _WIN64 */ + +/*----------------------------------------------------------------------------*/ + +#if defined(__ANDROID_API__) +__extern_C void __assert2(const char *file, int line, const char *function, const char *msg) __noreturn; +#define __assert_fail(assertion, file, line, function) __assert2(file, line, function, assertion) + +#elif defined(__UCLIBC__) +MDBX_NORETURN __extern_C void __assert(const char *, const char *, unsigned, const char *) +#ifdef __THROW + __THROW +#else + __nothrow +#endif /* __THROW */ + ; +#define __assert_fail(assertion, file, line, function) __assert(assertion, file, line, function) + +#elif _POSIX_C_SOURCE > 200212 && \ + /* workaround for avoid musl libc wrong prototype */ (defined(__GLIBC__) || defined(__GNU_LIBRARY__)) +/* Prototype should match libc runtime. ISO POSIX (2003) & LSB 1.x-3.x */ +MDBX_NORETURN __extern_C void __assert_fail(const char *assertion, const char *file, unsigned line, + const char *function) +#ifdef __THROW + __THROW +#else + __nothrow +#endif /* __THROW */ + ; + +#elif defined(__APPLE__) || defined(__MACH__) +__extern_C void __assert_rtn(const char *function, const char *file, int line, const char *assertion) /* __nothrow */ +#ifdef __dead2 + __dead2 +#else + MDBX_NORETURN +#endif /* __dead2 */ +#ifdef __disable_tail_calls + __disable_tail_calls +#endif /* __disable_tail_calls */ + ; + +#define __assert_fail(assertion, file, line, function) __assert_rtn(function, file, line, assertion) +#elif defined(__sun) || defined(__SVR4) || defined(__svr4__) +MDBX_NORETURN __extern_C void __assert_c99(const char *assection, const char *file, int line, const char *function); +#define __assert_fail(assertion, file, line, function) __assert_c99(assertion, file, line, function) +#elif defined(__OpenBSD__) +__extern_C __dead void __assert2(const char *file, int line, const char *function, + const char *assertion) /* __nothrow */; +#define __assert_fail(assertion, file, line, function) __assert2(file, line, function, assertion) +#elif defined(__NetBSD__) +__extern_C __dead void __assert13(const char *file, int line, const char *function, + const char *assertion) /* __nothrow */; +#define __assert_fail(assertion, file, line, function) __assert13(file, line, function, assertion) +#elif defined(__FreeBSD__) || defined(__BSD__) || defined(__bsdi__) || defined(__DragonFly__) +__extern_C void __assert(const char *function, const char *file, int line, const char *assertion) /* __nothrow */ +#ifdef __dead2 + __dead2 +#else + MDBX_NORETURN +#endif /* __dead2 */ +#ifdef __disable_tail_calls + __disable_tail_calls +#endif /* __disable_tail_calls */ + ; +#define __assert_fail(assertion, file, line, function) __assert(function, file, line, assertion) + +#endif /* __assert_fail */ + +__cold void mdbx_assert_fail(const MDBX_env *env, const char *msg, const char *func, unsigned line) { +#if MDBX_DEBUG + if (env && env->assert_func) + env->assert_func(env, msg, func, line); +#else + (void)env; + assert_fail(msg, func, line); +} + +MDBX_NORETURN __cold void assert_fail(const char *msg, const char *func, unsigned line) { +#endif /* MDBX_DEBUG */ + + if (globals.logger.ptr) + debug_log(MDBX_LOG_FATAL, func, line, "assert: %s\n", msg); + else { +#if defined(_WIN32) || defined(_WIN64) + char *message = nullptr; + const int num = osal_asprintf(&message, "\r\nMDBX-ASSERTION: %s, %s:%u", msg, func ? func : "unknown", line); + if (num < 1 || !message) + message = ""; + OutputDebugStringA(message); +#else + __assert_fail(msg, "mdbx", line, func); +#endif + } + + while (1) { +#if defined(_WIN32) || defined(_WIN64) +#if !MDBX_WITHOUT_MSVC_CRT && defined(_DEBUG) + _CrtDbgReport(_CRT_ASSERT, func ? func : "unknown", line, "libmdbx", "assertion failed: %s", msg); +#else + if (IsDebuggerPresent()) + DebugBreak(); +#endif + FatalExit(STATUS_ASSERTION_FAILURE); +#else + abort(); +#endif + } +} + +__cold void mdbx_panic(const char *fmt, ...) { + va_list ap; + va_start(ap, fmt); + + char *message = nullptr; + const int num = osal_vasprintf(&message, fmt, ap); + va_end(ap); + const char *const const_message = + unlikely(num < 1 || !message) ? "" : message; + + if (globals.logger.ptr) + debug_log(MDBX_LOG_FATAL, "panic", 0, "%s", const_message); + + while (1) { +#if defined(_WIN32) || defined(_WIN64) +#if !MDBX_WITHOUT_MSVC_CRT && defined(_DEBUG) + _CrtDbgReport(_CRT_ASSERT, "mdbx.c", 0, "libmdbx", "panic: %s", const_message); +#else + OutputDebugStringA("\r\nMDBX-PANIC: "); + OutputDebugStringA(const_message); + if (IsDebuggerPresent()) + DebugBreak(); +#endif + FatalExit(ERROR_UNHANDLED_ERROR); +#else + __assert_fail(const_message, "mdbx", 0, "panic"); + abort(); +#endif + } +} + +/*----------------------------------------------------------------------------*/ + +#ifndef osal_vasprintf +MDBX_INTERNAL int osal_vasprintf(char **strp, const char *fmt, va_list ap) { + va_list ones; + va_copy(ones, ap); + const int needed = vsnprintf(nullptr, 0, fmt, ones); + va_end(ones); + + if (unlikely(needed < 0 || needed >= INT_MAX)) { + *strp = nullptr; + return needed; + } + + *strp = osal_malloc(needed + (size_t)1); + if (unlikely(*strp == nullptr)) { +#if defined(_WIN32) || defined(_WIN64) + SetLastError(MDBX_ENOMEM); +#else + errno = MDBX_ENOMEM; +#endif + return -1; + } + + const int actual = vsnprintf(*strp, needed + (size_t)1, fmt, ap); + assert(actual == needed); + if (unlikely(actual < 0)) { + osal_free(*strp); + *strp = nullptr; + } + return actual; +} +#endif /* osal_vasprintf */ + +#ifndef osal_asprintf +MDBX_INTERNAL int osal_asprintf(char **strp, const char *fmt, ...) { + va_list ap; + va_start(ap, fmt); + const int rc = osal_vasprintf(strp, fmt, ap); + va_end(ap); + return rc; +} +#endif /* osal_asprintf */ + +#ifndef osal_memalign_alloc +MDBX_INTERNAL int osal_memalign_alloc(size_t alignment, size_t bytes, void **result) { + assert(is_powerof2(alignment) && alignment >= sizeof(void *)); +#if defined(_WIN32) || defined(_WIN64) + (void)alignment; + *result = VirtualAlloc(nullptr, bytes, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE); + return *result ? MDBX_SUCCESS : MDBX_ENOMEM /* ERROR_OUTOFMEMORY */; +#elif defined(_ISOC11_SOURCE) + *result = aligned_alloc(alignment, ceil_powerof2(bytes, alignment)); + return *result ? MDBX_SUCCESS : errno; +#elif _POSIX_VERSION >= 200112L && (!defined(__ANDROID_API__) || __ANDROID_API__ >= 17) + *result = nullptr; + return posix_memalign(result, alignment, bytes); +#elif __GLIBC_PREREQ(2, 16) || __STDC_VERSION__ >= 201112L + *result = memalign(alignment, bytes); + return *result ? MDBX_SUCCESS : errno; +#else +#error FIXME +#endif +} +#endif /* osal_memalign_alloc */ + +#ifndef osal_memalign_free +MDBX_INTERNAL void osal_memalign_free(void *ptr) { +#if defined(_WIN32) || defined(_WIN64) + VirtualFree(ptr, 0, MEM_RELEASE); +#else + osal_free(ptr); +#endif +} +#endif /* osal_memalign_free */ + +#ifndef osal_strdup +char *osal_strdup(const char *str) { + if (!str) + return nullptr; + size_t bytes = strlen(str) + 1; + char *dup = osal_malloc(bytes); + if (dup) + memcpy(dup, str, bytes); + return dup; +} +#endif /* osal_strdup */ + +/*----------------------------------------------------------------------------*/ + +MDBX_INTERNAL int osal_condpair_init(osal_condpair_t *condpair) { + int rc; + memset(condpair, 0, sizeof(osal_condpair_t)); +#if defined(_WIN32) || defined(_WIN64) + if (!(condpair->mutex = CreateMutexW(nullptr, FALSE, nullptr))) { + rc = (int)GetLastError(); + goto bailout_mutex; + } + if (!(condpair->event[0] = CreateEventW(nullptr, FALSE, FALSE, nullptr))) { + rc = (int)GetLastError(); + goto bailout_event; + } + if ((condpair->event[1] = CreateEventW(nullptr, FALSE, FALSE, nullptr))) + return MDBX_SUCCESS; + + rc = (int)GetLastError(); + (void)CloseHandle(condpair->event[0]); +bailout_event: + (void)CloseHandle(condpair->mutex); +#else + rc = pthread_mutex_init(&condpair->mutex, nullptr); + if (unlikely(rc != 0)) + goto bailout_mutex; + rc = pthread_cond_init(&condpair->cond[0], nullptr); + if (unlikely(rc != 0)) + goto bailout_cond; + rc = pthread_cond_init(&condpair->cond[1], nullptr); + if (likely(rc == 0)) + return MDBX_SUCCESS; + + (void)pthread_cond_destroy(&condpair->cond[0]); +bailout_cond: + (void)pthread_mutex_destroy(&condpair->mutex); +#endif +bailout_mutex: + memset(condpair, 0, sizeof(osal_condpair_t)); + return rc; +} + +MDBX_INTERNAL int osal_condpair_destroy(osal_condpair_t *condpair) { +#if defined(_WIN32) || defined(_WIN64) + int rc = CloseHandle(condpair->mutex) ? MDBX_SUCCESS : (int)GetLastError(); + rc = CloseHandle(condpair->event[0]) ? rc : (int)GetLastError(); + rc = CloseHandle(condpair->event[1]) ? rc : (int)GetLastError(); +#else + int err, rc = pthread_mutex_destroy(&condpair->mutex); + rc = (err = pthread_cond_destroy(&condpair->cond[0])) ? err : rc; + rc = (err = pthread_cond_destroy(&condpair->cond[1])) ? err : rc; +#endif + memset(condpair, 0, sizeof(osal_condpair_t)); + return rc; +} + +MDBX_INTERNAL int osal_condpair_lock(osal_condpair_t *condpair) { +#if defined(_WIN32) || defined(_WIN64) + DWORD code = WaitForSingleObject(condpair->mutex, INFINITE); + return waitstatus2errcode(code); +#else + return osal_pthread_mutex_lock(&condpair->mutex); +#endif +} + +MDBX_INTERNAL int osal_condpair_unlock(osal_condpair_t *condpair) { +#if defined(_WIN32) || defined(_WIN64) + return ReleaseMutex(condpair->mutex) ? MDBX_SUCCESS : (int)GetLastError(); +#else + return pthread_mutex_unlock(&condpair->mutex); +#endif +} + +MDBX_INTERNAL int osal_condpair_signal(osal_condpair_t *condpair, bool part) { +#if defined(_WIN32) || defined(_WIN64) + return SetEvent(condpair->event[part]) ? MDBX_SUCCESS : (int)GetLastError(); +#else + return pthread_cond_signal(&condpair->cond[part]); +#endif +} + +MDBX_INTERNAL int osal_condpair_wait(osal_condpair_t *condpair, bool part) { +#if defined(_WIN32) || defined(_WIN64) + DWORD code = SignalObjectAndWait(condpair->mutex, condpair->event[part], INFINITE, FALSE); + if (code == WAIT_OBJECT_0) { + code = WaitForSingleObject(condpair->mutex, INFINITE); + if (code == WAIT_OBJECT_0) + return MDBX_SUCCESS; + } + return waitstatus2errcode(code); +#else + return pthread_cond_wait(&condpair->cond[part], &condpair->mutex); +#endif +} + +/*----------------------------------------------------------------------------*/ + +MDBX_INTERNAL int osal_fastmutex_init(osal_fastmutex_t *fastmutex) { +#if defined(_WIN32) || defined(_WIN64) + InitializeCriticalSection(fastmutex); + return MDBX_SUCCESS; +#elif MDBX_DEBUG + pthread_mutexattr_t ma; + int rc = pthread_mutexattr_init(&ma); + if (likely(!rc)) { + rc = pthread_mutexattr_settype(&ma, PTHREAD_MUTEX_ERRORCHECK); + if (likely(!rc) || rc == ENOTSUP) + rc = pthread_mutex_init(fastmutex, &ma); + pthread_mutexattr_destroy(&ma); + } + return rc; +#else + return pthread_mutex_init(fastmutex, nullptr); +#endif +} + +MDBX_INTERNAL int osal_fastmutex_destroy(osal_fastmutex_t *fastmutex) { +#if defined(_WIN32) || defined(_WIN64) + DeleteCriticalSection(fastmutex); + return MDBX_SUCCESS; +#else + return pthread_mutex_destroy(fastmutex); +#endif +} + +MDBX_INTERNAL int osal_fastmutex_acquire(osal_fastmutex_t *fastmutex) { +#if defined(_WIN32) || defined(_WIN64) + __try { + EnterCriticalSection(fastmutex); + } __except ((GetExceptionCode() == 0xC0000194 /* STATUS_POSSIBLE_DEADLOCK / EXCEPTION_POSSIBLE_DEADLOCK */) + ? EXCEPTION_EXECUTE_HANDLER + : EXCEPTION_CONTINUE_SEARCH) { + return MDBX_EDEADLK; + } + return MDBX_SUCCESS; +#else + return osal_pthread_mutex_lock(fastmutex); +#endif +} + +MDBX_INTERNAL int osal_fastmutex_release(osal_fastmutex_t *fastmutex) { +#if defined(_WIN32) || defined(_WIN64) + LeaveCriticalSection(fastmutex); + return MDBX_SUCCESS; +#else + return pthread_mutex_unlock(fastmutex); +#endif +} + +/*----------------------------------------------------------------------------*/ + +#if defined(_WIN32) || defined(_WIN64) + +MDBX_INTERNAL int osal_mb2w(const char *const src, wchar_t **const pdst) { + const size_t dst_wlen = MultiByteToWideChar(CP_THREAD_ACP, MB_ERR_INVALID_CHARS, src, -1, nullptr, 0); + wchar_t *dst = *pdst; + int rc = ERROR_INVALID_NAME; + if (unlikely(dst_wlen < 2 || dst_wlen > /* MAX_PATH */ INT16_MAX)) + goto bailout; + + dst = osal_realloc(dst, dst_wlen * sizeof(wchar_t)); + rc = MDBX_ENOMEM; + if (unlikely(!dst)) + goto bailout; + + *pdst = dst; + if (likely(dst_wlen == (size_t)MultiByteToWideChar(CP_THREAD_ACP, MB_ERR_INVALID_CHARS, src, -1, dst, (int)dst_wlen))) + return MDBX_SUCCESS; + + rc = ERROR_INVALID_NAME; +bailout: + if (*pdst) { + osal_free(*pdst); + *pdst = nullptr; + } + return rc; +} + +#endif /* Windows */ + +/*----------------------------------------------------------------------------*/ + +#if defined(_WIN32) || defined(_WIN64) +#define ior_alignment_mask (ior->pagesize - 1) +#define ior_WriteFile_flag 1 +#define OSAL_IOV_MAX (4096 / sizeof(ior_sgv_element)) + +static void ior_put_event(osal_ioring_t *ior, HANDLE event) { + assert(event && event != INVALID_HANDLE_VALUE && event != ior); + assert(ior->event_stack < ior->allocated); + ior->event_pool[ior->event_stack] = event; + ior->event_stack += 1; +} + +static HANDLE ior_get_event(osal_ioring_t *ior) { + assert(ior->event_stack <= ior->allocated); + if (ior->event_stack > 0) { + ior->event_stack -= 1; + assert(ior->event_pool[ior->event_stack] != 0); + return ior->event_pool[ior->event_stack]; + } + return CreateEventW(nullptr, true, false, nullptr); +} + +static void WINAPI ior_wocr(DWORD err, DWORD bytes, OVERLAPPED *ov) { + osal_ioring_t *ior = ov->hEvent; + ov->Internal = err; + ov->InternalHigh = bytes; + if (++ior->async_completed >= ior->async_waiting) + SetEvent(ior->async_done); +} + +#elif MDBX_HAVE_PWRITEV +#if defined(_SC_IOV_MAX) +static size_t osal_iov_max; +#define OSAL_IOV_MAX osal_iov_max +#else +#define OSAL_IOV_MAX IOV_MAX +#endif +#else +#undef OSAL_IOV_MAX +#endif /* OSAL_IOV_MAX */ + +MDBX_INTERNAL int osal_ioring_create(osal_ioring_t *ior +#if defined(_WIN32) || defined(_WIN64) + , + bool enable_direct, mdbx_filehandle_t overlapped_fd +#endif /* Windows */ +) { + memset(ior, 0, sizeof(osal_ioring_t)); + +#if defined(_WIN32) || defined(_WIN64) + ior->overlapped_fd = overlapped_fd; + ior->direct = enable_direct && overlapped_fd; + ior->pagesize = globals.sys_pagesize; + ior->pagesize_ln2 = globals.sys_pagesize_ln2; + ior->async_done = ior_get_event(ior); + if (!ior->async_done) + return GetLastError(); +#endif /* !Windows */ + +#if MDBX_HAVE_PWRITEV && defined(_SC_IOV_MAX) + assert(osal_iov_max > 0); +#endif /* MDBX_HAVE_PWRITEV && _SC_IOV_MAX */ + + ior->boundary = ptr_disp(ior->pool, ior->allocated); + return MDBX_SUCCESS; +} + +static inline size_t ior_offset(const ior_item_t *item) { +#if defined(_WIN32) || defined(_WIN64) + return item->ov.Offset | + (size_t)((sizeof(size_t) > sizeof(item->ov.Offset)) ? (uint64_t)item->ov.OffsetHigh << 32 : 0); +#else + return item->offset; +#endif /* !Windows */ +} + +static inline ior_item_t *ior_next(ior_item_t *item, size_t sgvcnt) { +#if defined(ior_sgv_element) + assert(sgvcnt > 0); + return (ior_item_t *)ptr_disp(item, sizeof(ior_item_t) - sizeof(ior_sgv_element) + sizeof(ior_sgv_element) * sgvcnt); +#else + assert(sgvcnt == 1); + (void)sgvcnt; + return item + 1; +#endif +} + +MDBX_INTERNAL int osal_ioring_add(osal_ioring_t *ior, const size_t offset, void *data, const size_t bytes) { + assert(bytes && data); + assert(bytes % MDBX_MIN_PAGESIZE == 0 && bytes <= MAX_WRITE); + assert(offset % MDBX_MIN_PAGESIZE == 0 && offset + (uint64_t)bytes <= MAX_MAPSIZE); + +#if defined(_WIN32) || defined(_WIN64) + const unsigned segments = (unsigned)(bytes >> ior->pagesize_ln2); + const bool use_gather = ior->direct && ior->overlapped_fd && ior->slots_left >= segments; +#endif /* Windows */ + + ior_item_t *item = ior->pool; + if (likely(ior->last)) { + item = ior->last; + if (unlikely(ior_offset(item) + ior_last_bytes(ior, item) == offset) && + likely(ior_last_bytes(ior, item) + bytes <= MAX_WRITE)) { +#if defined(_WIN32) || defined(_WIN64) + if (use_gather && + ((bytes | (uintptr_t)data | ior->last_bytes | (uintptr_t)(uint64_t)item->sgv[0].Buffer) & + ior_alignment_mask) == 0 && + ior->last_sgvcnt + (size_t)segments < OSAL_IOV_MAX) { + assert(ior->overlapped_fd); + assert((item->single.iov_len & ior_WriteFile_flag) == 0); + assert(item->sgv[ior->last_sgvcnt].Buffer == 0); + ior->last_bytes += bytes; + size_t i = 0; + do { + item->sgv[ior->last_sgvcnt + i].Buffer = PtrToPtr64(data); + data = ptr_disp(data, ior->pagesize); + } while (++i < segments); + ior->slots_left -= segments; + item->sgv[ior->last_sgvcnt += segments].Buffer = 0; + assert((item->single.iov_len & ior_WriteFile_flag) == 0); + return MDBX_SUCCESS; + } + const void *end = ptr_disp(item->single.iov_base, item->single.iov_len - ior_WriteFile_flag); + if (unlikely(end == data)) { + assert((item->single.iov_len & ior_WriteFile_flag) != 0); + item->single.iov_len += bytes; + return MDBX_SUCCESS; + } +#elif MDBX_HAVE_PWRITEV + assert((int)item->sgvcnt > 0); + const void *end = ptr_disp(item->sgv[item->sgvcnt - 1].iov_base, item->sgv[item->sgvcnt - 1].iov_len); + if (unlikely(end == data)) { + item->sgv[item->sgvcnt - 1].iov_len += bytes; + ior->last_bytes += bytes; + return MDBX_SUCCESS; + } + if (likely(item->sgvcnt < OSAL_IOV_MAX)) { + if (unlikely(ior->slots_left < 1)) + return MDBX_RESULT_TRUE; + item->sgv[item->sgvcnt].iov_base = data; + item->sgv[item->sgvcnt].iov_len = bytes; + ior->last_bytes += bytes; + item->sgvcnt += 1; + ior->slots_left -= 1; + return MDBX_SUCCESS; + } +#else + const void *end = ptr_disp(item->single.iov_base, item->single.iov_len); + if (unlikely(end == data)) { + item->single.iov_len += bytes; + return MDBX_SUCCESS; + } +#endif + } + item = ior_next(item, ior_last_sgvcnt(ior, item)); + } + + if (unlikely(ior->slots_left < 1)) + return MDBX_RESULT_TRUE; + + unsigned slots_used = 1; +#if defined(_WIN32) || defined(_WIN64) + item->ov.Internal = item->ov.InternalHigh = 0; + item->ov.Offset = (DWORD)offset; + item->ov.OffsetHigh = HIGH_DWORD(offset); + item->ov.hEvent = 0; + if (!use_gather || ((bytes | (uintptr_t)(data)) & ior_alignment_mask) != 0 || segments > OSAL_IOV_MAX) { + /* WriteFile() */ + item->single.iov_base = data; + item->single.iov_len = bytes + ior_WriteFile_flag; + assert((item->single.iov_len & ior_WriteFile_flag) != 0); + } else { + /* WriteFileGather() */ + assert(ior->overlapped_fd); + item->sgv[0].Buffer = PtrToPtr64(data); + for (size_t i = 1; i < segments; ++i) { + data = ptr_disp(data, ior->pagesize); + item->sgv[i].Buffer = PtrToPtr64(data); + } + item->sgv[slots_used = segments].Buffer = 0; + assert((item->single.iov_len & ior_WriteFile_flag) == 0); + } + ior->last_bytes = bytes; + ior_last_sgvcnt(ior, item) = slots_used; +#elif MDBX_HAVE_PWRITEV + item->offset = offset; + item->sgv[0].iov_base = data; + item->sgv[0].iov_len = bytes; + ior->last_bytes = bytes; + ior_last_sgvcnt(ior, item) = slots_used; +#else + item->offset = offset; + item->single.iov_base = data; + item->single.iov_len = bytes; +#endif /* !Windows */ + ior->slots_left -= slots_used; + ior->last = item; + return MDBX_SUCCESS; +} + +MDBX_INTERNAL void osal_ioring_walk(osal_ioring_t *ior, iov_ctx_t *ctx, + void (*callback)(iov_ctx_t *ctx, size_t offset, void *data, size_t bytes)) { + for (ior_item_t *item = ior->pool; item <= ior->last;) { +#if defined(_WIN32) || defined(_WIN64) + size_t offset = ior_offset(item); + char *data = item->single.iov_base; + size_t bytes = item->single.iov_len - ior_WriteFile_flag; + size_t i = 1; + if (bytes & ior_WriteFile_flag) { + data = Ptr64ToPtr(item->sgv[0].Buffer); + bytes = ior->pagesize; + /* Zap: Reading invalid data from 'item->sgv' */ + MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(6385); + while (item->sgv[i].Buffer) { + if (data + ior->pagesize != item->sgv[i].Buffer) { + callback(ctx, offset, data, bytes); + offset += bytes; + data = Ptr64ToPtr(item->sgv[i].Buffer); + bytes = 0; + } + bytes += ior->pagesize; + ++i; + } + } + assert(bytes < MAX_WRITE); + callback(ctx, offset, data, bytes); +#elif MDBX_HAVE_PWRITEV + assert(item->sgvcnt > 0); + size_t offset = item->offset; + size_t i = 0; + do { + callback(ctx, offset, item->sgv[i].iov_base, item->sgv[i].iov_len); + offset += item->sgv[i].iov_len; + } while (++i != item->sgvcnt); +#else + const size_t i = 1; + callback(ctx, item->offset, item->single.iov_base, item->single.iov_len); +#endif + item = ior_next(item, i); + } +} + +MDBX_INTERNAL osal_ioring_write_result_t osal_ioring_write(osal_ioring_t *ior, mdbx_filehandle_t fd) { + osal_ioring_write_result_t r = {MDBX_SUCCESS, 0}; + +#if defined(_WIN32) || defined(_WIN64) + HANDLE *const end_wait_for = ior->event_pool + ior->allocated + + /* был выделен один дополнительный элемент для async_done */ 1; + HANDLE *wait_for = end_wait_for; + LONG async_started = 0; + for (ior_item_t *item = ior->pool; item <= ior->last;) { + item->ov.Internal = STATUS_PENDING; + size_t i = 1, bytes = item->single.iov_len - ior_WriteFile_flag; + r.wops += 1; + if (bytes & ior_WriteFile_flag) { + assert(ior->overlapped_fd && fd == ior->overlapped_fd); + bytes = ior->pagesize; + /* Zap: Reading invalid data from 'item->sgv' */ + MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(6385); + while (item->sgv[i].Buffer) { + bytes += ior->pagesize; + ++i; + } + assert(bytes < MAX_WRITE); + item->ov.hEvent = ior_get_event(ior); + if (unlikely(!item->ov.hEvent)) { + bailout_geterr: + r.err = GetLastError(); + bailout_rc: + assert(r.err != MDBX_SUCCESS); + CancelIo(fd); + return r; + } + if (WriteFileGather(fd, item->sgv, (DWORD)bytes, nullptr, &item->ov)) { + assert(item->ov.Internal == 0 && WaitForSingleObject(item->ov.hEvent, 0) == WAIT_OBJECT_0); + ior_put_event(ior, item->ov.hEvent); + item->ov.hEvent = 0; + } else { + r.err = (int)GetLastError(); + if (unlikely(r.err != ERROR_IO_PENDING)) { + void *data = Ptr64ToPtr(item->sgv[0].Buffer); + ERROR("%s: fd %p, item %p (%zu), addr %p pgno %u, bytes %zu," + " offset %" PRId64 ", err %d", + "WriteFileGather", fd, __Wpedantic_format_voidptr(item), item - ior->pool, data, ((page_t *)data)->pgno, + bytes, item->ov.Offset + ((uint64_t)item->ov.OffsetHigh << 32), r.err); + goto bailout_rc; + } + assert(wait_for > ior->event_pool + ior->event_stack); + *--wait_for = item->ov.hEvent; + } + } else if (fd == ior->overlapped_fd) { + assert(bytes < MAX_WRITE); + retry: + item->ov.hEvent = ior; + if (WriteFileEx(fd, item->single.iov_base, (DWORD)bytes, &item->ov, ior_wocr)) { + async_started += 1; + } else { + r.err = (int)GetLastError(); + switch (r.err) { + default: + ERROR("%s: fd %p, item %p (%zu), addr %p pgno %u, bytes %zu," + " offset %" PRId64 ", err %d", + "WriteFileEx", fd, __Wpedantic_format_voidptr(item), item - ior->pool, item->single.iov_base, + ((page_t *)item->single.iov_base)->pgno, bytes, item->ov.Offset + ((uint64_t)item->ov.OffsetHigh << 32), + r.err); + goto bailout_rc; + case ERROR_NOT_FOUND: + case ERROR_USER_MAPPED_FILE: + case ERROR_LOCK_VIOLATION: + WARNING("%s: fd %p, item %p (%zu), addr %p pgno %u, bytes %zu," + " offset %" PRId64 ", err %d", + "WriteFileEx", fd, __Wpedantic_format_voidptr(item), item - ior->pool, item->single.iov_base, + ((page_t *)item->single.iov_base)->pgno, bytes, + item->ov.Offset + ((uint64_t)item->ov.OffsetHigh << 32), r.err); + SleepEx(0, true); + goto retry; + case ERROR_INVALID_USER_BUFFER: + case ERROR_NOT_ENOUGH_MEMORY: + if (SleepEx(0, true) == WAIT_IO_COMPLETION) + goto retry; + goto bailout_rc; + case ERROR_IO_PENDING: + async_started += 1; + } + } + } else { + assert(bytes < MAX_WRITE); + DWORD written = 0; + if (!WriteFile(fd, item->single.iov_base, (DWORD)bytes, &written, &item->ov)) { + r.err = (int)GetLastError(); + ERROR("%s: fd %p, item %p (%zu), addr %p pgno %u, bytes %zu," + " offset %" PRId64 ", err %d", + "WriteFile", fd, __Wpedantic_format_voidptr(item), item - ior->pool, item->single.iov_base, + ((page_t *)item->single.iov_base)->pgno, bytes, item->ov.Offset + ((uint64_t)item->ov.OffsetHigh << 32), + r.err); + goto bailout_rc; + } else if (unlikely(written != bytes)) { + r.err = ERROR_WRITE_FAULT; + goto bailout_rc; + } + } + item = ior_next(item, i); + } + + assert(ior->async_waiting > ior->async_completed && ior->async_waiting == INT_MAX); + ior->async_waiting = async_started; + if (async_started > ior->async_completed && end_wait_for == wait_for) { + assert(wait_for > ior->event_pool + ior->event_stack); + *--wait_for = ior->async_done; + } + + const size_t pending_count = end_wait_for - wait_for; + if (pending_count) { + /* Ждем до MAXIMUM_WAIT_OBJECTS (64) последних хендлов, а после избирательно + * ждем посредством GetOverlappedResult(), если какие-то более ранние + * элементы еще не завершены. В целом, так получается меньше системных + * вызовов, т.е. меньше накладных расходов. Однако, не факт что эта экономия + * не будет перекрыта неэффективностью реализации + * WaitForMultipleObjectsEx(), но тогда это проблемы на стороне M$. */ + DWORD madness; + do + madness = WaitForMultipleObjectsEx( + (pending_count < MAXIMUM_WAIT_OBJECTS) ? (DWORD)pending_count : MAXIMUM_WAIT_OBJECTS, wait_for, true, + /* сутки */ 86400000ul, true); + while (madness == WAIT_IO_COMPLETION); + STATIC_ASSERT(WAIT_OBJECT_0 == 0); + if (/* madness >= WAIT_OBJECT_0 && */ + madness < WAIT_OBJECT_0 + MAXIMUM_WAIT_OBJECTS) + r.err = MDBX_SUCCESS; + else if (madness >= WAIT_ABANDONED_0 && madness < WAIT_ABANDONED_0 + MAXIMUM_WAIT_OBJECTS) { + r.err = ERROR_ABANDONED_WAIT_0; + goto bailout_rc; + } else if (madness == WAIT_TIMEOUT) { + r.err = WAIT_TIMEOUT; + goto bailout_rc; + } else { + r.err = /* madness == WAIT_FAILED */ MDBX_PROBLEM; + goto bailout_rc; + } + + assert(ior->async_waiting == ior->async_completed); + for (ior_item_t *item = ior->pool; item <= ior->last;) { + size_t i = 1, bytes = item->single.iov_len - ior_WriteFile_flag; + void *data = item->single.iov_base; + if (bytes & ior_WriteFile_flag) { + data = Ptr64ToPtr(item->sgv[0].Buffer); + bytes = ior->pagesize; + /* Zap: Reading invalid data from 'item->sgv' */ + MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(6385); + while (item->sgv[i].Buffer) { + bytes += ior->pagesize; + ++i; + } + if (!HasOverlappedIoCompleted(&item->ov)) { + DWORD written = 0; + if (unlikely(!GetOverlappedResult(fd, &item->ov, &written, true))) { + ERROR("%s: item %p (%zu), addr %p pgno %u, bytes %zu," + " offset %" PRId64 ", err %d", + "GetOverlappedResult", __Wpedantic_format_voidptr(item), item - ior->pool, data, + ((page_t *)data)->pgno, bytes, item->ov.Offset + ((uint64_t)item->ov.OffsetHigh << 32), + (int)GetLastError()); + goto bailout_geterr; + } + assert(MDBX_SUCCESS == item->ov.Internal); + assert(written == item->ov.InternalHigh); + } + } else { + assert(HasOverlappedIoCompleted(&item->ov)); + } + assert(item->ov.Internal != ERROR_IO_PENDING); + if (unlikely(item->ov.Internal != MDBX_SUCCESS)) { + DWORD written = 0; + r.err = (int)item->ov.Internal; + if ((r.err & 0x80000000) && GetOverlappedResult(nullptr, &item->ov, &written, true)) + r.err = (int)GetLastError(); + ERROR("%s: item %p (%zu), addr %p pgno %u, bytes %zu," + " offset %" PRId64 ", err %d", + "Result", __Wpedantic_format_voidptr(item), item - ior->pool, data, ((page_t *)data)->pgno, bytes, + item->ov.Offset + ((uint64_t)item->ov.OffsetHigh << 32), (int)GetLastError()); + goto bailout_rc; + } + if (unlikely(item->ov.InternalHigh != bytes)) { + r.err = ERROR_WRITE_FAULT; + goto bailout_rc; + } + item = ior_next(item, i); + } + assert(ior->async_waiting == ior->async_completed); + } else { + assert(r.err == MDBX_SUCCESS); + } + assert(ior->async_waiting == ior->async_completed); + +#else + STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), "libmdbx requires 64-bit file I/O on 64-bit systems"); + for (ior_item_t *item = ior->pool; item <= ior->last;) { +#if MDBX_HAVE_PWRITEV + assert(item->sgvcnt > 0); + if (item->sgvcnt == 1) + r.err = osal_pwrite(fd, item->sgv[0].iov_base, item->sgv[0].iov_len, item->offset); + else + r.err = osal_pwritev(fd, item->sgv, item->sgvcnt, item->offset); + + // TODO: io_uring_prep_write(sqe, fd, ...); + + item = ior_next(item, item->sgvcnt); +#else + r.err = osal_pwrite(fd, item->single.iov_base, item->single.iov_len, item->offset); + item = ior_next(item, 1); +#endif + r.wops += 1; + if (unlikely(r.err != MDBX_SUCCESS)) + break; + } + + // TODO: io_uring_submit(&ring) + // TODO: err = io_uring_wait_cqe(&ring, &cqe); + // TODO: io_uring_cqe_seen(&ring, cqe); + +#endif /* !Windows */ + return r; +} + +MDBX_INTERNAL void osal_ioring_reset(osal_ioring_t *ior) { +#if defined(_WIN32) || defined(_WIN64) + if (ior->last) { + for (ior_item_t *item = ior->pool; item <= ior->last;) { + if (!HasOverlappedIoCompleted(&item->ov)) { + assert(ior->overlapped_fd); + CancelIoEx(ior->overlapped_fd, &item->ov); + } + if (item->ov.hEvent && item->ov.hEvent != ior) + ior_put_event(ior, item->ov.hEvent); + size_t i = 1; + if ((item->single.iov_len & ior_WriteFile_flag) == 0) { + /* Zap: Reading invalid data from 'item->sgv' */ + MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(6385); + while (item->sgv[i].Buffer) + ++i; + } + item = ior_next(item, i); + } + } + ior->async_waiting = INT_MAX; + ior->async_completed = 0; + ResetEvent(ior->async_done); +#endif /* !Windows */ + ior->slots_left = ior->allocated; + ior->last = nullptr; +} + +static void ior_cleanup(osal_ioring_t *ior, const size_t since) { + osal_ioring_reset(ior); +#if defined(_WIN32) || defined(_WIN64) + for (size_t i = since; i < ior->event_stack; ++i) { + /* Zap: Using uninitialized memory '**ior.event_pool' */ + MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(6001); + CloseHandle(ior->event_pool[i]); + } + ior->event_stack = 0; +#else + (void)since; +#endif /* Windows */ +} + +MDBX_INTERNAL int osal_ioring_resize(osal_ioring_t *ior, size_t items) { + assert(items > 0 && items < INT_MAX / sizeof(ior_item_t)); +#if defined(_WIN32) || defined(_WIN64) + if (ior->state & IOR_STATE_LOCKED) + return MDBX_SUCCESS; + const bool useSetFileIoOverlappedRange = ior->overlapped_fd && imports.SetFileIoOverlappedRange && items > 42; + const size_t ceiling = + useSetFileIoOverlappedRange ? ((items < 65536 / 2 / sizeof(ior_item_t)) ? 65536 : 65536 * 4) : 1024; + const size_t bytes = ceil_powerof2(sizeof(ior_item_t) * items, ceiling); + items = bytes / sizeof(ior_item_t); +#endif /* Windows */ + + if (items != ior->allocated) { + assert(items >= osal_ioring_used(ior)); + if (items < ior->allocated) + ior_cleanup(ior, items); +#if defined(_WIN32) || defined(_WIN64) + void *ptr = osal_realloc(ior->event_pool, (items + /* extra for waiting the async_done */ 1) * sizeof(HANDLE)); + if (unlikely(!ptr)) + return MDBX_ENOMEM; + ior->event_pool = ptr; + + int err = osal_memalign_alloc(ceiling, bytes, &ptr); + if (unlikely(err != MDBX_SUCCESS)) + return err; + if (ior->pool) { + memcpy(ptr, ior->pool, ior->allocated * sizeof(ior_item_t)); + osal_memalign_free(ior->pool); + } +#else + void *ptr = osal_realloc(ior->pool, sizeof(ior_item_t) * items); + if (unlikely(!ptr)) + return MDBX_ENOMEM; +#endif + ior->pool = ptr; + + if (items > ior->allocated) + memset(ior->pool + ior->allocated, 0, sizeof(ior_item_t) * (items - ior->allocated)); + ior->allocated = (unsigned)items; + ior->boundary = ptr_disp(ior->pool, ior->allocated); +#if defined(_WIN32) || defined(_WIN64) + if (useSetFileIoOverlappedRange) { + if (imports.SetFileIoOverlappedRange(ior->overlapped_fd, ptr, (ULONG)bytes)) + ior->state += IOR_STATE_LOCKED; + else + return GetLastError(); + } +#endif /* Windows */ + } + return MDBX_SUCCESS; +} + +MDBX_INTERNAL void osal_ioring_destroy(osal_ioring_t *ior) { + if (ior->allocated) + ior_cleanup(ior, 0); +#if defined(_WIN32) || defined(_WIN64) + osal_memalign_free(ior->pool); + osal_free(ior->event_pool); + CloseHandle(ior->async_done); + if (ior->overlapped_fd) + CloseHandle(ior->overlapped_fd); +#else + osal_free(ior->pool); +#endif + memset(ior, 0, sizeof(osal_ioring_t)); } /*----------------------------------------------------------------------------*/ -__cold intptr_t mdbx_limits_dbsize_min(intptr_t pagesize) { - if (pagesize < 1) - pagesize = (intptr_t)mdbx_default_pagesize(); - else if (unlikely(pagesize < (intptr_t)MIN_PAGESIZE || - pagesize > (intptr_t)MAX_PAGESIZE || - !is_powerof2((size_t)pagesize))) - return -1; +MDBX_INTERNAL int osal_removefile(const pathchar_t *pathname) { +#if defined(_WIN32) || defined(_WIN64) + return DeleteFileW(pathname) ? MDBX_SUCCESS : (int)GetLastError(); +#else + return unlink(pathname) ? errno : MDBX_SUCCESS; +#endif +} + +#if !(defined(_WIN32) || defined(_WIN64)) +static bool is_valid_fd(int fd) { return !(isatty(fd) < 0 && errno == EBADF); } +#endif /*! Windows */ + +MDBX_INTERNAL int osal_removedirectory(const pathchar_t *pathname) { +#if defined(_WIN32) || defined(_WIN64) + return RemoveDirectoryW(pathname) ? MDBX_SUCCESS : (int)GetLastError(); +#else + return rmdir(pathname) ? errno : MDBX_SUCCESS; +#endif +} + +MDBX_INTERNAL int osal_fileexists(const pathchar_t *pathname) { +#if defined(_WIN32) || defined(_WIN64) + if (GetFileAttributesW(pathname) != INVALID_FILE_ATTRIBUTES) + return MDBX_RESULT_TRUE; + int err = GetLastError(); + return (err == ERROR_FILE_NOT_FOUND || err == ERROR_PATH_NOT_FOUND) ? MDBX_RESULT_FALSE : err; +#else + if (access(pathname, F_OK) == 0) + return MDBX_RESULT_TRUE; + int err = errno; + return (err == ENOENT || err == ENOTDIR) ? MDBX_RESULT_FALSE : err; +#endif +} + +MDBX_INTERNAL pathchar_t *osal_fileext(const pathchar_t *pathname, size_t len) { + const pathchar_t *ext = nullptr; + for (size_t i = 0; i < len && pathname[i]; i++) + if (pathname[i] == '.') + ext = pathname + i; + else if (osal_isdirsep(pathname[i])) + ext = nullptr; + return (pathchar_t *)ext; +} + +MDBX_INTERNAL bool osal_pathequal(const pathchar_t *l, const pathchar_t *r, size_t len) { +#if defined(_WIN32) || defined(_WIN64) + for (size_t i = 0; i < len; ++i) { + pathchar_t a = l[i]; + pathchar_t b = r[i]; + a = (a == '\\') ? '/' : a; + b = (b == '\\') ? '/' : b; + if (a != b) + return false; + } + return true; +#else + return memcmp(l, r, len * sizeof(pathchar_t)) == 0; +#endif +} + +MDBX_INTERNAL int osal_openfile(const enum osal_openfile_purpose purpose, const MDBX_env *env, + const pathchar_t *pathname, mdbx_filehandle_t *fd, mdbx_mode_t unix_mode_bits) { + *fd = INVALID_HANDLE_VALUE; + +#if defined(_WIN32) || defined(_WIN64) + DWORD CreationDisposition = unix_mode_bits ? OPEN_ALWAYS : OPEN_EXISTING; + DWORD FlagsAndAttributes = FILE_FLAG_POSIX_SEMANTICS | FILE_ATTRIBUTE_NOT_CONTENT_INDEXED; + DWORD DesiredAccess = FILE_READ_ATTRIBUTES; + DWORD ShareMode = (env->flags & MDBX_EXCLUSIVE) ? 0 : (FILE_SHARE_READ | FILE_SHARE_WRITE); + + switch (purpose) { + default: + return ERROR_INVALID_PARAMETER; + case MDBX_OPEN_LCK: + CreationDisposition = OPEN_ALWAYS; + DesiredAccess |= GENERIC_READ | GENERIC_WRITE; + FlagsAndAttributes |= FILE_ATTRIBUTE_HIDDEN | FILE_ATTRIBUTE_TEMPORARY; + break; + case MDBX_OPEN_DXB_READ: + CreationDisposition = OPEN_EXISTING; + DesiredAccess |= GENERIC_READ; + ShareMode |= FILE_SHARE_READ; + break; + case MDBX_OPEN_DXB_LAZY: + DesiredAccess |= GENERIC_READ | GENERIC_WRITE; + break; + case MDBX_OPEN_DXB_OVERLAPPED_DIRECT: + FlagsAndAttributes |= FILE_FLAG_NO_BUFFERING; + /* fall through */ + __fallthrough; + case MDBX_OPEN_DXB_OVERLAPPED: + FlagsAndAttributes |= FILE_FLAG_OVERLAPPED; + /* fall through */ + __fallthrough; + case MDBX_OPEN_DXB_DSYNC: + CreationDisposition = OPEN_EXISTING; + DesiredAccess |= GENERIC_WRITE | GENERIC_READ; + FlagsAndAttributes |= FILE_FLAG_WRITE_THROUGH; + break; + case MDBX_OPEN_COPY: + CreationDisposition = CREATE_NEW; + ShareMode = 0; + DesiredAccess |= GENERIC_WRITE; + if (env->ps >= globals.sys_pagesize) + FlagsAndAttributes |= FILE_FLAG_NO_BUFFERING; + break; + case MDBX_OPEN_DELETE: + CreationDisposition = OPEN_EXISTING; + ShareMode |= FILE_SHARE_DELETE; + DesiredAccess = FILE_READ_ATTRIBUTES | FILE_WRITE_ATTRIBUTES | DELETE | SYNCHRONIZE; + break; + } + + *fd = CreateFileW(pathname, DesiredAccess, ShareMode, nullptr, CreationDisposition, FlagsAndAttributes, nullptr); + if (*fd == INVALID_HANDLE_VALUE) { + int err = (int)GetLastError(); + if (err == ERROR_ACCESS_DENIED && purpose == MDBX_OPEN_LCK) { + if (GetFileAttributesW(pathname) == INVALID_FILE_ATTRIBUTES && GetLastError() == ERROR_FILE_NOT_FOUND) + err = ERROR_FILE_NOT_FOUND; + } + return err; + } + + BY_HANDLE_FILE_INFORMATION info; + if (!GetFileInformationByHandle(*fd, &info)) { + int err = (int)GetLastError(); + CloseHandle(*fd); + *fd = INVALID_HANDLE_VALUE; + return err; + } + const DWORD AttributesDiff = + (info.dwFileAttributes ^ FlagsAndAttributes) & (FILE_ATTRIBUTE_HIDDEN | FILE_ATTRIBUTE_NOT_CONTENT_INDEXED | + FILE_ATTRIBUTE_TEMPORARY | FILE_ATTRIBUTE_COMPRESSED); + if (AttributesDiff) + (void)SetFileAttributesW(pathname, info.dwFileAttributes ^ AttributesDiff); - return MIN_PAGENO * pagesize; -} +#else + int flags = unix_mode_bits ? O_CREAT : 0; + switch (purpose) { + default: + return EINVAL; + case MDBX_OPEN_LCK: + flags |= O_RDWR; + break; + case MDBX_OPEN_DXB_READ: + flags = O_RDONLY; + break; + case MDBX_OPEN_DXB_LAZY: + flags |= O_RDWR; + break; + case MDBX_OPEN_COPY: + flags = O_CREAT | O_WRONLY | O_EXCL; + break; + case MDBX_OPEN_DXB_DSYNC: + flags |= O_WRONLY; +#if defined(O_DSYNC) + flags |= O_DSYNC; +#elif defined(O_SYNC) + flags |= O_SYNC; +#elif defined(O_FSYNC) + flags |= O_FSYNC; +#endif + break; + case MDBX_OPEN_DELETE: + flags = O_RDWR; + break; + } -__cold intptr_t mdbx_limits_dbsize_max(intptr_t pagesize) { - if (pagesize < 1) - pagesize = (intptr_t)mdbx_default_pagesize(); - else if (unlikely(pagesize < (intptr_t)MIN_PAGESIZE || - pagesize > (intptr_t)MAX_PAGESIZE || - !is_powerof2((size_t)pagesize))) - return -1; + const bool direct_nocache_for_copy = env->ps >= globals.sys_pagesize && purpose == MDBX_OPEN_COPY; + if (direct_nocache_for_copy) { +#if defined(O_DIRECT) + flags |= O_DIRECT; +#endif /* O_DIRECT */ +#if defined(O_NOCACHE) + flags |= O_NOCACHE; +#endif /* O_NOCACHE */ + } - STATIC_ASSERT(MAX_MAPSIZE < INTPTR_MAX); - const uint64_t limit = (1 + (uint64_t)MAX_PAGENO) * pagesize; - return (limit < MAX_MAPSIZE) ? (intptr_t)limit : (intptr_t)MAX_MAPSIZE; +#ifdef O_CLOEXEC + flags |= O_CLOEXEC; +#endif /* O_CLOEXEC */ + + /* Safeguard for https://libmdbx.dqdkfa.ru/dead-github/issues/144 */ +#if STDIN_FILENO == 0 && STDOUT_FILENO == 1 && STDERR_FILENO == 2 + int stub_fd0 = -1, stub_fd1 = -1, stub_fd2 = -1; + static const char dev_null[] = "/dev/null"; + if (!is_valid_fd(STDIN_FILENO)) { + WARNING("STD%s_FILENO/%d is invalid, open %s for temporary stub", "IN", STDIN_FILENO, dev_null); + stub_fd0 = open(dev_null, O_RDONLY | O_NOCTTY); + } + if (!is_valid_fd(STDOUT_FILENO)) { + WARNING("STD%s_FILENO/%d is invalid, open %s for temporary stub", "OUT", STDOUT_FILENO, dev_null); + stub_fd1 = open(dev_null, O_WRONLY | O_NOCTTY); + } + if (!is_valid_fd(STDERR_FILENO)) { + WARNING("STD%s_FILENO/%d is invalid, open %s for temporary stub", "ERR", STDERR_FILENO, dev_null); + stub_fd2 = open(dev_null, O_WRONLY | O_NOCTTY); + } +#else +#error "Unexpected or unsupported UNIX or POSIX system" +#endif /* STDIN_FILENO == 0 && STDERR_FILENO == 2 */ + + *fd = open(pathname, flags, unix_mode_bits); +#if defined(O_DIRECT) + if (*fd < 0 && (flags & O_DIRECT) && (errno == EINVAL || errno == EAFNOSUPPORT)) { + flags &= ~(O_DIRECT | O_EXCL); + *fd = open(pathname, flags, unix_mode_bits); + } +#endif /* O_DIRECT */ + + if (*fd < 0 && errno == EACCES && purpose == MDBX_OPEN_LCK) { + struct stat unused; + if (stat(pathname, &unused) == 0 || errno != ENOENT) + errno = EACCES /* restore errno if file exists */; + } + + /* Safeguard for https://libmdbx.dqdkfa.ru/dead-github/issues/144 */ +#if STDIN_FILENO == 0 && STDOUT_FILENO == 1 && STDERR_FILENO == 2 + if (*fd == STDIN_FILENO) { + WARNING("Got STD%s_FILENO/%d, avoid using it by dup(fd)", "IN", STDIN_FILENO); + assert(stub_fd0 == -1); + *fd = dup(stub_fd0 = *fd); + } + if (*fd == STDOUT_FILENO) { + WARNING("Got STD%s_FILENO/%d, avoid using it by dup(fd)", "OUT", STDOUT_FILENO); + assert(stub_fd1 == -1); + *fd = dup(stub_fd1 = *fd); + } + if (*fd == STDERR_FILENO) { + WARNING("Got STD%s_FILENO/%d, avoid using it by dup(fd)", "ERR", STDERR_FILENO); + assert(stub_fd2 == -1); + *fd = dup(stub_fd2 = *fd); + } + const int err = errno; + if (stub_fd0 != -1) + close(stub_fd0); + if (stub_fd1 != -1) + close(stub_fd1); + if (stub_fd2 != -1) + close(stub_fd2); + if (*fd >= STDIN_FILENO && *fd <= STDERR_FILENO) { + ERROR("Rejecting the use of a FD in the range " + "STDIN_FILENO/%d..STDERR_FILENO/%d to prevent database corruption", + STDIN_FILENO, STDERR_FILENO); + close(*fd); + return EBADF; + } +#else +#error "Unexpected or unsupported UNIX or POSIX system" +#endif /* STDIN_FILENO == 0 && STDERR_FILENO == 2 */ + + if (*fd < 0) + return err; + +#if defined(FD_CLOEXEC) && !defined(O_CLOEXEC) + const int fd_flags = fcntl(*fd, F_GETFD); + if (fd_flags != -1) + (void)fcntl(*fd, F_SETFD, fd_flags | FD_CLOEXEC); +#endif /* FD_CLOEXEC && !O_CLOEXEC */ + + if (direct_nocache_for_copy) { +#if defined(F_NOCACHE) && !defined(O_NOCACHE) + (void)fcntl(*fd, F_NOCACHE, 1); +#endif /* F_NOCACHE */ + } + +#endif + return MDBX_SUCCESS; } -__cold intptr_t mdbx_limits_txnsize_max(intptr_t pagesize) { - if (pagesize < 1) - pagesize = (intptr_t)mdbx_default_pagesize(); - else if (unlikely(pagesize < (intptr_t)MIN_PAGESIZE || - pagesize > (intptr_t)MAX_PAGESIZE || - !is_powerof2((size_t)pagesize))) - return -1; +MDBX_INTERNAL int osal_closefile(mdbx_filehandle_t fd) { +#if defined(_WIN32) || defined(_WIN64) + return CloseHandle(fd) ? MDBX_SUCCESS : (int)GetLastError(); +#else + assert(fd > STDERR_FILENO); + return (close(fd) == 0) ? MDBX_SUCCESS : errno; +#endif +} - STATIC_ASSERT(MAX_MAPSIZE < INTPTR_MAX); - const uint64_t pgl_limit = - pagesize * (uint64_t)(MDBX_PGL_LIMIT / MDBX_GOLD_RATIO_DBL); - const uint64_t map_limit = (uint64_t)(MAX_MAPSIZE / MDBX_GOLD_RATIO_DBL); - return (pgl_limit < map_limit) ? (intptr_t)pgl_limit : (intptr_t)map_limit; +MDBX_INTERNAL int osal_pread(mdbx_filehandle_t fd, void *buf, size_t bytes, uint64_t offset) { + if (bytes > MAX_WRITE) + return MDBX_EINVAL; +#if defined(_WIN32) || defined(_WIN64) + OVERLAPPED ov; + ov.hEvent = 0; + ov.Offset = (DWORD)offset; + ov.OffsetHigh = HIGH_DWORD(offset); + + DWORD read = 0; + if (unlikely(!ReadFile(fd, buf, (DWORD)bytes, &read, &ov))) { + int rc = (int)GetLastError(); + return (rc == MDBX_SUCCESS) ? /* paranoia */ ERROR_READ_FAULT : rc; + } +#else + STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), "libmdbx requires 64-bit file I/O on 64-bit systems"); + intptr_t read = pread(fd, buf, bytes, offset); + if (read < 0) { + int rc = errno; + return (rc == MDBX_SUCCESS) ? /* paranoia */ MDBX_EIO : rc; + } +#endif + return (bytes == (size_t)read) ? MDBX_SUCCESS : MDBX_ENODATA; } -/*** Key-making functions to avoid custom comparators *************************/ +MDBX_INTERNAL int osal_pwrite(mdbx_filehandle_t fd, const void *buf, size_t bytes, uint64_t offset) { + while (true) { +#if defined(_WIN32) || defined(_WIN64) + OVERLAPPED ov; + ov.hEvent = 0; + ov.Offset = (DWORD)offset; + ov.OffsetHigh = HIGH_DWORD(offset); -static __always_inline double key2double(const int64_t key) { - union { - uint64_t u; - double f; - } casting; + DWORD written; + if (unlikely(!WriteFile(fd, buf, likely(bytes <= MAX_WRITE) ? (DWORD)bytes : MAX_WRITE, &written, &ov))) + return (int)GetLastError(); + if (likely(bytes == written)) + return MDBX_SUCCESS; +#else + STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), "libmdbx requires 64-bit file I/O on 64-bit systems"); + const intptr_t written = pwrite(fd, buf, likely(bytes <= MAX_WRITE) ? bytes : MAX_WRITE, offset); + if (likely(bytes == (size_t)written)) + return MDBX_SUCCESS; + if (written < 0) { + const int rc = errno; + if (rc != EINTR) + return rc; + continue; + } +#endif + bytes -= written; + offset += written; + buf = ptr_disp(buf, written); + } +} - casting.u = (key < 0) ? key + UINT64_C(0x8000000000000000) - : UINT64_C(0xffffFFFFffffFFFF) - key; - return casting.f; +MDBX_INTERNAL int osal_write(mdbx_filehandle_t fd, const void *buf, size_t bytes) { + while (true) { +#if defined(_WIN32) || defined(_WIN64) + DWORD written; + if (unlikely(!WriteFile(fd, buf, likely(bytes <= MAX_WRITE) ? (DWORD)bytes : MAX_WRITE, &written, nullptr))) + return (int)GetLastError(); + if (likely(bytes == written)) + return MDBX_SUCCESS; +#else + STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), "libmdbx requires 64-bit file I/O on 64-bit systems"); + const intptr_t written = write(fd, buf, likely(bytes <= MAX_WRITE) ? bytes : MAX_WRITE); + if (likely(bytes == (size_t)written)) + return MDBX_SUCCESS; + if (written < 0) { + const int rc = errno; + if (rc != EINTR) + return rc; + continue; + } +#endif + bytes -= written; + buf = ptr_disp(buf, written); + } } -static __always_inline uint64_t double2key(const double *const ptr) { - STATIC_ASSERT(sizeof(double) == sizeof(int64_t)); - const int64_t i = *(const int64_t *)ptr; - const uint64_t u = (i < 0) ? UINT64_C(0xffffFFFFffffFFFF) - i - : i + UINT64_C(0x8000000000000000); - if (ASSERT_ENABLED()) { - const double f = key2double(u); - assert(memcmp(&f, ptr, 8) == 0); +int osal_pwritev(mdbx_filehandle_t fd, struct iovec *iov, size_t sgvcnt, uint64_t offset) { + size_t expected = 0; + for (size_t i = 0; i < sgvcnt; ++i) + expected += iov[i].iov_len; +#if !MDBX_HAVE_PWRITEV + size_t written = 0; + for (size_t i = 0; i < sgvcnt; ++i) { + int rc = osal_pwrite(fd, iov[i].iov_base, iov[i].iov_len, offset); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + written += iov[i].iov_len; + offset += iov[i].iov_len; } - return u; + return (expected == written) ? MDBX_SUCCESS : MDBX_EIO /* ERROR_WRITE_FAULT */; +#else + int rc; + intptr_t written; + do { + STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), "libmdbx requires 64-bit file I/O on 64-bit systems"); + written = pwritev(fd, iov, sgvcnt, offset); + if (likely(expected == (size_t)written)) + return MDBX_SUCCESS; + rc = errno; + } while (rc == EINTR); + return (written < 0) ? rc : MDBX_EIO /* Use which error code? */; +#endif } -static __always_inline float key2float(const int32_t key) { - union { - uint32_t u; - float f; - } casting; +MDBX_INTERNAL int osal_fsync(mdbx_filehandle_t fd, enum osal_syncmode_bits mode_bits) { +#if defined(_WIN32) || defined(_WIN64) + if ((mode_bits & (MDBX_SYNC_DATA | MDBX_SYNC_IODQ)) && !FlushFileBuffers(fd)) + return (int)GetLastError(); + return MDBX_SUCCESS; +#else - casting.u = - (key < 0) ? key + UINT32_C(0x80000000) : UINT32_C(0xffffFFFF) - key; - return casting.f; -} +#if defined(__APPLE__) && MDBX_APPLE_SPEED_INSTEADOF_DURABILITY == MDBX_OSX_WANNA_DURABILITY + if (mode_bits & MDBX_SYNC_IODQ) + return likely(fcntl(fd, F_FULLFSYNC) != -1) ? MDBX_SUCCESS : errno; +#endif /* MacOS */ -static __always_inline uint32_t float2key(const float *const ptr) { - STATIC_ASSERT(sizeof(float) == sizeof(int32_t)); - const int32_t i = *(const int32_t *)ptr; - const uint32_t u = - (i < 0) ? UINT32_C(0xffffFFFF) - i : i + UINT32_C(0x80000000); - if (ASSERT_ENABLED()) { - const float f = key2float(u); - assert(memcmp(&f, ptr, 4) == 0); - } - return u; -} + /* LY: This approach is always safe and without appreciable performance + * degradation, even on a kernel with fdatasync's bug. + * + * For more info about of a corresponding fdatasync() bug + * see http://www.spinics.net/lists/linux-ext4/msg33714.html */ + while (1) { + switch (mode_bits & (MDBX_SYNC_DATA | MDBX_SYNC_SIZE)) { + case MDBX_SYNC_NONE: + case MDBX_SYNC_KICK: + return MDBX_SUCCESS /* nothing to do */; +#if defined(_POSIX_SYNCHRONIZED_IO) && _POSIX_SYNCHRONIZED_IO > 0 + case MDBX_SYNC_DATA: + if (likely(fdatasync(fd) == 0)) + return MDBX_SUCCESS; + break /* error */; +#if defined(__linux__) || defined(__gnu_linux__) + case MDBX_SYNC_SIZE: + assert(globals.linux_kernel_version >= 0x03060000); + return MDBX_SUCCESS; +#endif /* Linux */ +#endif /* _POSIX_SYNCHRONIZED_IO > 0 */ + default: + if (likely(fsync(fd) == 0)) + return MDBX_SUCCESS; + } -uint64_t mdbx_key_from_double(const double ieee754_64bit) { - return double2key(&ieee754_64bit); + int rc = errno; + if (rc != EINTR) + return rc; + } +#endif } -uint64_t mdbx_key_from_ptrdouble(const double *const ieee754_64bit) { - return double2key(ieee754_64bit); -} +int osal_filesize(mdbx_filehandle_t fd, uint64_t *length) { +#if defined(_WIN32) || defined(_WIN64) + BY_HANDLE_FILE_INFORMATION info; + if (!GetFileInformationByHandle(fd, &info)) + return (int)GetLastError(); + *length = info.nFileSizeLow | (uint64_t)info.nFileSizeHigh << 32; +#else + struct stat st; -uint32_t mdbx_key_from_float(const float ieee754_32bit) { - return float2key(&ieee754_32bit); -} + STATIC_ASSERT_MSG(sizeof(off_t) <= sizeof(uint64_t), "libmdbx requires 64-bit file I/O on 64-bit systems"); + if (fstat(fd, &st)) + return errno; -uint32_t mdbx_key_from_ptrfloat(const float *const ieee754_32bit) { - return float2key(ieee754_32bit); + *length = st.st_size; +#endif + return MDBX_SUCCESS; } -#define IEEE754_DOUBLE_MANTISSA_SIZE 52 -#define IEEE754_DOUBLE_EXPONENTA_BIAS 0x3FF -#define IEEE754_DOUBLE_EXPONENTA_MAX 0x7FF -#define IEEE754_DOUBLE_IMPLICIT_LEAD UINT64_C(0x0010000000000000) -#define IEEE754_DOUBLE_MANTISSA_MASK UINT64_C(0x000FFFFFFFFFFFFF) -#define IEEE754_DOUBLE_MANTISSA_AMAX UINT64_C(0x001FFFFFFFFFFFFF) - -static __inline int clz64(uint64_t value) { -#if __GNUC_PREREQ(4, 1) || __has_builtin(__builtin_clzl) - if (sizeof(value) == sizeof(int)) - return __builtin_clz(value); - if (sizeof(value) == sizeof(long)) - return __builtin_clzl(value); -#if (defined(__SIZEOF_LONG_LONG__) && __SIZEOF_LONG_LONG__ == 8) || \ - __has_builtin(__builtin_clzll) - return __builtin_clzll(value); -#endif /* have(long long) && long long == uint64_t */ -#endif /* GNU C */ - -#if defined(_MSC_VER) - unsigned long index; -#if defined(_M_AMD64) || defined(_M_ARM64) || defined(_M_X64) - _BitScanReverse64(&index, value); - return 63 - index; +MDBX_INTERNAL int osal_is_pipe(mdbx_filehandle_t fd) { +#if defined(_WIN32) || defined(_WIN64) + switch (GetFileType(fd)) { + case FILE_TYPE_DISK: + return MDBX_RESULT_FALSE; + case FILE_TYPE_CHAR: + case FILE_TYPE_PIPE: + return MDBX_RESULT_TRUE; + default: + return (int)GetLastError(); + } #else - if (value > UINT32_MAX) { - _BitScanReverse(&index, (uint32_t)(value >> 32)); - return 31 - index; + struct stat info; + if (fstat(fd, &info)) + return errno; + switch (info.st_mode & S_IFMT) { + case S_IFBLK: + case S_IFREG: + return MDBX_RESULT_FALSE; + case S_IFCHR: + case S_IFIFO: + case S_IFSOCK: + return MDBX_RESULT_TRUE; + case S_IFDIR: + case S_IFLNK: + default: + return MDBX_INCOMPATIBLE; } - _BitScanReverse(&index, (uint32_t)value); - return 63 - index; #endif -#endif /* MSVC */ - - value |= value >> 1; - value |= value >> 2; - value |= value >> 4; - value |= value >> 8; - value |= value >> 16; - value |= value >> 32; - static const uint8_t debruijn_clz64[64] = { - 63, 16, 62, 7, 15, 36, 61, 3, 6, 14, 22, 26, 35, 47, 60, 2, - 9, 5, 28, 11, 13, 21, 42, 19, 25, 31, 34, 40, 46, 52, 59, 1, - 17, 8, 37, 4, 23, 27, 48, 10, 29, 12, 43, 20, 32, 41, 53, 18, - 38, 24, 49, 30, 44, 33, 54, 39, 50, 45, 55, 51, 56, 57, 58, 0}; - return debruijn_clz64[value * UINT64_C(0x03F79D71B4CB0A89) >> 58]; } -static __inline uint64_t round_mantissa(const uint64_t u64, int shift) { - assert(shift < 0 && u64 > 0); - shift = -shift; - const unsigned half = 1 << (shift - 1); - const unsigned lsb = 1 & (unsigned)(u64 >> shift); - const unsigned tie2even = 1 ^ lsb; - return (u64 + half - tie2even) >> shift; +MDBX_INTERNAL int osal_ftruncate(mdbx_filehandle_t fd, uint64_t length) { +#if defined(_WIN32) || defined(_WIN64) + if (imports.SetFileInformationByHandle) { + FILE_END_OF_FILE_INFO EndOfFileInfo; + EndOfFileInfo.EndOfFile.QuadPart = length; + return imports.SetFileInformationByHandle(fd, FileEndOfFileInfo, &EndOfFileInfo, sizeof(FILE_END_OF_FILE_INFO)) + ? MDBX_SUCCESS + : (int)GetLastError(); + } else { + LARGE_INTEGER li; + li.QuadPart = length; + return (SetFilePointerEx(fd, li, nullptr, FILE_BEGIN) && SetEndOfFile(fd)) ? MDBX_SUCCESS : (int)GetLastError(); + } +#else + STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), "libmdbx requires 64-bit file I/O on 64-bit systems"); + return ftruncate(fd, length) == 0 ? MDBX_SUCCESS : errno; +#endif } -uint64_t mdbx_key_from_jsonInteger(const int64_t json_integer) { - const uint64_t bias = UINT64_C(0x8000000000000000); - if (json_integer > 0) { - const uint64_t u64 = json_integer; - int shift = clz64(u64) - (64 - IEEE754_DOUBLE_MANTISSA_SIZE - 1); - uint64_t mantissa = u64 << shift; - if (unlikely(shift < 0)) { - mantissa = round_mantissa(u64, shift); - if (mantissa > IEEE754_DOUBLE_MANTISSA_AMAX) - mantissa = round_mantissa(u64, --shift); - } - - assert(mantissa >= IEEE754_DOUBLE_IMPLICIT_LEAD && - mantissa <= IEEE754_DOUBLE_MANTISSA_AMAX); - const uint64_t exponent = (uint64_t)IEEE754_DOUBLE_EXPONENTA_BIAS + - IEEE754_DOUBLE_MANTISSA_SIZE - shift; - assert(exponent > 0 && exponent <= IEEE754_DOUBLE_EXPONENTA_MAX); - const uint64_t key = bias + (exponent << IEEE754_DOUBLE_MANTISSA_SIZE) + - (mantissa - IEEE754_DOUBLE_IMPLICIT_LEAD); -#if !defined(_MSC_VER) || \ - defined( \ - _DEBUG) /* Workaround for MSVC error LNK2019: unresolved external \ - symbol __except1 referenced in function __ftol3_except */ - assert(key == mdbx_key_from_double((double)json_integer)); -#endif /* Workaround for MSVC */ - return key; - } +MDBX_INTERNAL int osal_fseek(mdbx_filehandle_t fd, uint64_t pos) { +#if defined(_WIN32) || defined(_WIN64) + LARGE_INTEGER li; + li.QuadPart = pos; + return SetFilePointerEx(fd, li, nullptr, FILE_BEGIN) ? MDBX_SUCCESS : (int)GetLastError(); +#else + STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), "libmdbx requires 64-bit file I/O on 64-bit systems"); + return (lseek(fd, pos, SEEK_SET) < 0) ? errno : MDBX_SUCCESS; +#endif +} - if (json_integer < 0) { - const uint64_t u64 = -json_integer; - int shift = clz64(u64) - (64 - IEEE754_DOUBLE_MANTISSA_SIZE - 1); - uint64_t mantissa = u64 << shift; - if (unlikely(shift < 0)) { - mantissa = round_mantissa(u64, shift); - if (mantissa > IEEE754_DOUBLE_MANTISSA_AMAX) - mantissa = round_mantissa(u64, --shift); - } +/*----------------------------------------------------------------------------*/ - assert(mantissa >= IEEE754_DOUBLE_IMPLICIT_LEAD && - mantissa <= IEEE754_DOUBLE_MANTISSA_AMAX); - const uint64_t exponent = (uint64_t)IEEE754_DOUBLE_EXPONENTA_BIAS + - IEEE754_DOUBLE_MANTISSA_SIZE - shift; - assert(exponent > 0 && exponent <= IEEE754_DOUBLE_EXPONENTA_MAX); - const uint64_t key = bias - 1 - (exponent << IEEE754_DOUBLE_MANTISSA_SIZE) - - (mantissa - IEEE754_DOUBLE_IMPLICIT_LEAD); -#if !defined(_MSC_VER) || \ - defined( \ - _DEBUG) /* Workaround for MSVC error LNK2019: unresolved external \ - symbol __except1 referenced in function __ftol3_except */ - assert(key == mdbx_key_from_double((double)json_integer)); -#endif /* Workaround for MSVC */ - return key; - } +MDBX_INTERNAL int osal_thread_create(osal_thread_t *thread, THREAD_RESULT(THREAD_CALL *start_routine)(void *), + void *arg) { +#if defined(_WIN32) || defined(_WIN64) + *thread = CreateThread(nullptr, 0, start_routine, arg, 0, nullptr); + return *thread ? MDBX_SUCCESS : (int)GetLastError(); +#else + return pthread_create(thread, nullptr, start_routine, arg); +#endif +} - return bias; +MDBX_INTERNAL int osal_thread_join(osal_thread_t thread) { +#if defined(_WIN32) || defined(_WIN64) + DWORD code = WaitForSingleObject(thread, INFINITE); + return waitstatus2errcode(code); +#else + void *unused_retval = &unused_retval; + return pthread_join(thread, &unused_retval); +#endif } -int64_t mdbx_jsonInteger_from_key(const MDBX_val v) { - assert(v.iov_len == 8); - const uint64_t key = unaligned_peek_u64(2, v.iov_base); - const uint64_t bias = UINT64_C(0x8000000000000000); - const uint64_t covalent = (key > bias) ? key - bias : bias - key - 1; - const int shift = IEEE754_DOUBLE_EXPONENTA_BIAS + 63 - - (IEEE754_DOUBLE_EXPONENTA_MAX & - (int)(covalent >> IEEE754_DOUBLE_MANTISSA_SIZE)); - if (unlikely(shift < 1)) - return (key < bias) ? INT64_MIN : INT64_MAX; - if (unlikely(shift > 63)) - return 0; +/*----------------------------------------------------------------------------*/ - const uint64_t unscaled = ((covalent & IEEE754_DOUBLE_MANTISSA_MASK) - << (63 - IEEE754_DOUBLE_MANTISSA_SIZE)) + - bias; - const int64_t absolute = unscaled >> shift; - const int64_t value = (key < bias) ? -absolute : absolute; - assert(key == mdbx_key_from_jsonInteger(value) || - (mdbx_key_from_jsonInteger(value - 1) < key && - key < mdbx_key_from_jsonInteger(value + 1))); - return value; +MDBX_INTERNAL int osal_msync(const osal_mmap_t *map, size_t offset, size_t length, enum osal_syncmode_bits mode_bits) { + if (!MDBX_MMAP_NEEDS_JOLT && mode_bits == MDBX_SYNC_NONE) + return MDBX_SUCCESS; + + void *ptr = ptr_disp(map->base, offset); +#if defined(_WIN32) || defined(_WIN64) + if (!FlushViewOfFile(ptr, length)) + return (int)GetLastError(); + if ((mode_bits & (MDBX_SYNC_DATA | MDBX_SYNC_IODQ)) && !FlushFileBuffers(map->fd)) + return (int)GetLastError(); +#else +#if defined(__linux__) || defined(__gnu_linux__) + /* Since Linux 2.6.19, MS_ASYNC is in fact a no-op. The kernel properly + * tracks dirty pages and flushes ones as necessary. */ + // + // However, this behavior may be changed in custom kernels, + // so just leave such optimization to the libc discretion. + // NOTE: The MDBX_MMAP_NEEDS_JOLT must be defined to 1 for such cases. + // + // assert(mdbx.linux_kernel_version > 0x02061300); + // if (mode_bits <= MDBX_SYNC_KICK) + // return MDBX_SUCCESS; +#endif /* Linux */ + if (msync(ptr, length, (mode_bits & MDBX_SYNC_DATA) ? MS_SYNC : MS_ASYNC)) + return errno; + if ((mode_bits & MDBX_SYNC_SIZE) && fsync(map->fd)) + return errno; +#endif + return MDBX_SUCCESS; } -double mdbx_double_from_key(const MDBX_val v) { - assert(v.iov_len == 8); - return key2double(unaligned_peek_u64(2, v.iov_base)); +MDBX_INTERNAL int osal_check_fs_rdonly(mdbx_filehandle_t handle, const pathchar_t *pathname, int err) { +#if defined(_WIN32) || defined(_WIN64) + (void)pathname; + (void)err; + if (!imports.GetVolumeInformationByHandleW) + return MDBX_ENOSYS; + DWORD unused, flags; + if (!imports.GetVolumeInformationByHandleW(handle, nullptr, 0, nullptr, &unused, &flags, nullptr, 0)) + return (int)GetLastError(); + if ((flags & FILE_READ_ONLY_VOLUME) == 0) + return MDBX_EACCESS; +#else + struct statvfs info; + if (err != MDBX_ENOFILE) { + if (statvfs(pathname, &info) == 0) + return (info.f_flag & ST_RDONLY) ? MDBX_SUCCESS : err; + if (errno != MDBX_ENOFILE) + return errno; + } + if (fstatvfs(handle, &info)) + return errno; + if ((info.f_flag & ST_RDONLY) == 0) + return (err == MDBX_ENOFILE) ? MDBX_EACCESS : err; +#endif /* !Windows */ + return MDBX_SUCCESS; } -float mdbx_float_from_key(const MDBX_val v) { - assert(v.iov_len == 4); - return key2float(unaligned_peek_u32(2, v.iov_base)); -} +MDBX_INTERNAL int osal_check_fs_incore(mdbx_filehandle_t handle) { +#if defined(_WIN32) || defined(_WIN64) + (void)handle; +#else + struct statfs statfs_info; + if (fstatfs(handle, &statfs_info)) + return errno; -int32_t mdbx_int32_from_key(const MDBX_val v) { - assert(v.iov_len == 4); - return (int32_t)(unaligned_peek_u32(2, v.iov_base) - UINT32_C(0x80000000)); -} +#if defined(__OpenBSD__) + const unsigned type = 0; +#else + const unsigned type = statfs_info.f_type; +#endif + switch (type) { + case 0x28cd3d45 /* CRAMFS_MAGIC */: + case 0x858458f6 /* RAMFS_MAGIC */: + case 0x01021994 /* TMPFS_MAGIC */: + case 0x73717368 /* SQUASHFS_MAGIC */: + case 0x7275 /* ROMFS_MAGIC */: + return MDBX_RESULT_TRUE; + } -int64_t mdbx_int64_from_key(const MDBX_val v) { - assert(v.iov_len == 8); - return (int64_t)(unaligned_peek_u64(2, v.iov_base) - - UINT64_C(0x8000000000000000)); -} +#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__BSD__) || defined(__bsdi__) || \ + defined(__DragonFly__) || defined(__APPLE__) || defined(__MACH__) || defined(MFSNAMELEN) || \ + defined(MFSTYPENAMELEN) || defined(VFS_NAMELEN) + const char *const name = statfs_info.f_fstypename; + const size_t name_len = sizeof(statfs_info.f_fstypename); +#else + const char *const name = ""; + const size_t name_len = 0; +#endif + if (name_len) { + if (strncasecmp("tmpfs", name, 6) == 0 || strncasecmp("mfs", name, 4) == 0 || strncasecmp("ramfs", name, 6) == 0 || + strncasecmp("romfs", name, 6) == 0) + return MDBX_RESULT_TRUE; + } +#endif /* !Windows */ -__cold MDBX_cmp_func *mdbx_get_keycmp(MDBX_db_flags_t flags) { - return get_default_keycmp(flags); + return MDBX_RESULT_FALSE; } -__cold MDBX_cmp_func *mdbx_get_datacmp(MDBX_db_flags_t flags) { - return get_default_datacmp(flags); -} +MDBX_INTERNAL int osal_check_fs_local(mdbx_filehandle_t handle, int flags) { +#if defined(_WIN32) || defined(_WIN64) + if (globals.running_under_Wine && !(flags & MDBX_EXCLUSIVE)) + return ERROR_NOT_CAPABLE /* workaround for Wine */; -__cold int mdbx_env_set_option(MDBX_env *env, const MDBX_option_t option, - uint64_t value) { - int err = check_env(env, false); - if (unlikely(err != MDBX_SUCCESS)) - return err; + if (GetFileType(handle) != FILE_TYPE_DISK) + return ERROR_FILE_OFFLINE; - const bool lock_needed = ((env->me_flags & MDBX_ENV_ACTIVE) && env->me_txn0 && - env->me_txn0->mt_owner != osal_thread_self()); - bool should_unlock = false; - switch (option) { - case MDBX_opt_sync_bytes: - if (value == /* default */ UINT64_MAX) - value = MAX_WRITE; - if (unlikely(env->me_flags & MDBX_RDONLY)) - return MDBX_EACCESS; - if (unlikely(!(env->me_flags & MDBX_ENV_ACTIVE))) - return MDBX_EPERM; - if (unlikely(value > SIZE_MAX - 65536)) - return MDBX_EINVAL; - value = bytes2pgno(env, (size_t)value + env->me_psize - 1); - if ((uint32_t)value != atomic_load32(&env->me_lck->mti_autosync_threshold, - mo_AcquireRelease) && - atomic_store32(&env->me_lck->mti_autosync_threshold, (uint32_t)value, - mo_Relaxed) - /* Дергаем sync(force=off) только если задано новое не-нулевое значение - * и мы вне транзакции */ - && lock_needed) { - err = env_sync(env, false, false); - if (err == /* нечего сбрасывать на диск */ MDBX_RESULT_TRUE) - err = MDBX_SUCCESS; + if (imports.GetFileInformationByHandleEx) { + FILE_REMOTE_PROTOCOL_INFO RemoteProtocolInfo; + if (imports.GetFileInformationByHandleEx(handle, FileRemoteProtocolInfo, &RemoteProtocolInfo, + sizeof(RemoteProtocolInfo))) { + if ((RemoteProtocolInfo.Flags & REMOTE_PROTOCOL_INFO_FLAG_OFFLINE) && !(flags & MDBX_RDONLY)) + return ERROR_FILE_OFFLINE; + if (!(RemoteProtocolInfo.Flags & REMOTE_PROTOCOL_INFO_FLAG_LOOPBACK) && !(flags & MDBX_EXCLUSIVE)) + return MDBX_EREMOTE; } - break; + } - case MDBX_opt_sync_period: - if (value == /* default */ UINT64_MAX) - value = 2780315 /* 42.42424 секунды */; - if (unlikely(env->me_flags & MDBX_RDONLY)) - return MDBX_EACCESS; - if (unlikely(!(env->me_flags & MDBX_ENV_ACTIVE))) - return MDBX_EPERM; - if (unlikely(value > UINT32_MAX)) - return MDBX_EINVAL; - value = osal_16dot16_to_monotime((uint32_t)value); - if (value != atomic_load64(&env->me_lck->mti_autosync_period, - mo_AcquireRelease) && - atomic_store64(&env->me_lck->mti_autosync_period, value, mo_Relaxed) - /* Дергаем sync(force=off) только если задано новое не-нулевое значение - * и мы вне транзакции */ - && lock_needed) { - err = env_sync(env, false, false); - if (err == /* нечего сбрасывать на диск */ MDBX_RESULT_TRUE) - err = MDBX_SUCCESS; - } - break; + if (imports.NtFsControlFile) { + NTSTATUS rc; + struct { + WOF_EXTERNAL_INFO wof_info; + union { + WIM_PROVIDER_EXTERNAL_INFO wim_info; + FILE_PROVIDER_EXTERNAL_INFO_V1 file_info; + }; + size_t reserved_for_microsoft_madness[42]; + } GetExternalBacking_OutputBuffer; + IO_STATUS_BLOCK StatusBlock; + rc = imports.NtFsControlFile(handle, nullptr, nullptr, nullptr, &StatusBlock, FSCTL_GET_EXTERNAL_BACKING, nullptr, + 0, &GetExternalBacking_OutputBuffer, sizeof(GetExternalBacking_OutputBuffer)); + if (NT_SUCCESS(rc)) { + if (!(flags & MDBX_EXCLUSIVE)) + return MDBX_EREMOTE; + } else if (rc != STATUS_OBJECT_NOT_EXTERNALLY_BACKED && rc != STATUS_INVALID_DEVICE_REQUEST && + rc != STATUS_NOT_SUPPORTED) + return ntstatus2errcode(rc); + } - case MDBX_opt_max_db: - if (value == /* default */ UINT64_MAX) - value = 42; - if (unlikely(value > MDBX_MAX_DBI)) - return MDBX_EINVAL; - if (unlikely(env->me_map)) - return MDBX_EPERM; - env->me_maxdbs = (unsigned)value + CORE_DBS; - break; + if (imports.GetVolumeInformationByHandleW && imports.GetFinalPathNameByHandleW) { + WCHAR *PathBuffer = osal_malloc(sizeof(WCHAR) * INT16_MAX); + if (!PathBuffer) + return MDBX_ENOMEM; - case MDBX_opt_max_readers: - if (value == /* default */ UINT64_MAX) - value = MDBX_READERS_LIMIT; - if (unlikely(value < 1 || value > MDBX_READERS_LIMIT)) - return MDBX_EINVAL; - if (unlikely(env->me_map)) - return MDBX_EPERM; - env->me_maxreaders = (unsigned)value; - break; + int rc = MDBX_SUCCESS; + DWORD VolumeSerialNumber, FileSystemFlags; + if (!imports.GetVolumeInformationByHandleW(handle, PathBuffer, INT16_MAX, &VolumeSerialNumber, nullptr, + &FileSystemFlags, nullptr, 0)) { + rc = (int)GetLastError(); + goto bailout; + } - case MDBX_opt_dp_reserve_limit: - if (value == /* default */ UINT64_MAX) - value = INT_MAX; - if (unlikely(value > INT_MAX)) - return MDBX_EINVAL; - if (env->me_options.dp_reserve_limit != (unsigned)value) { - if (lock_needed) { - err = mdbx_txn_lock(env, false); - if (unlikely(err != MDBX_SUCCESS)) - return err; - should_unlock = true; - } - env->me_options.dp_reserve_limit = (unsigned)value; - while (env->me_dp_reserve_len > env->me_options.dp_reserve_limit) { - eASSERT(env, env->me_dp_reserve != NULL); - MDBX_page *dp = env->me_dp_reserve; - MDBX_ASAN_UNPOISON_MEMORY_REGION(dp, env->me_psize); - VALGRIND_MAKE_MEM_DEFINED(&mp_next(dp), sizeof(MDBX_page *)); - env->me_dp_reserve = mp_next(dp); - void *const ptr = ptr_disp(dp, -(ptrdiff_t)sizeof(size_t)); - osal_free(ptr); - env->me_dp_reserve_len -= 1; + if ((flags & MDBX_RDONLY) == 0) { + if (FileSystemFlags & (FILE_SEQUENTIAL_WRITE_ONCE | FILE_READ_ONLY_VOLUME | FILE_VOLUME_IS_COMPRESSED)) { + rc = MDBX_EREMOTE; + goto bailout; } } - break; - case MDBX_opt_rp_augment_limit: - if (value == /* default */ UINT64_MAX) { - env->me_options.flags.non_auto.rp_augment_limit = 0; - env->me_options.rp_augment_limit = default_rp_augment_limit(env); - } else if (unlikely(value > MDBX_PGL_LIMIT)) - return MDBX_EINVAL; - else { - env->me_options.flags.non_auto.rp_augment_limit = 1; - env->me_options.rp_augment_limit = (unsigned)value; + if (imports.GetFinalPathNameByHandleW(handle, PathBuffer, INT16_MAX, FILE_NAME_NORMALIZED | VOLUME_NAME_NT)) { + if (_wcsnicmp(PathBuffer, L"\\Device\\Mup\\", 12) == 0) { + if (!(flags & MDBX_EXCLUSIVE)) { + rc = MDBX_EREMOTE; + goto bailout; + } + } } - break; - case MDBX_opt_txn_dp_limit: - case MDBX_opt_txn_dp_initial: - if (value == /* default */ UINT64_MAX) - value = MDBX_PGL_LIMIT; - if (unlikely(value > MDBX_PGL_LIMIT || value < CURSOR_STACK * 4)) - return MDBX_EINVAL; - if (unlikely(env->me_flags & MDBX_RDONLY)) - return MDBX_EACCESS; - if (lock_needed) { - err = mdbx_txn_lock(env, false); - if (unlikely(err != MDBX_SUCCESS)) - return err; - should_unlock = true; + if (F_ISSET(flags, MDBX_RDONLY | MDBX_EXCLUSIVE) && (FileSystemFlags & FILE_READ_ONLY_VOLUME)) { + /* without-LCK (exclusive readonly) mode for DB on a read-only volume */ + goto bailout; } - if (env->me_txn) - err = MDBX_EPERM /* unable change during transaction */; - else { - const pgno_t value32 = (pgno_t)value; - if (option == MDBX_opt_txn_dp_initial && - env->me_options.dp_initial != value32) { - env->me_options.dp_initial = value32; - if (env->me_options.dp_limit < value32) { - env->me_options.dp_limit = value32; - env->me_options.flags.non_auto.dp_limit = 1; - } + + if (imports.GetFinalPathNameByHandleW(handle, PathBuffer, INT16_MAX, FILE_NAME_NORMALIZED | VOLUME_NAME_DOS)) { + UINT DriveType = GetDriveTypeW(PathBuffer); + if (DriveType == DRIVE_NO_ROOT_DIR && _wcsnicmp(PathBuffer, L"\\\\?\\", 4) == 0 && + _wcsnicmp(PathBuffer + 5, L":\\", 2) == 0) { + PathBuffer[7] = 0; + DriveType = GetDriveTypeW(PathBuffer + 4); } - if (option == MDBX_opt_txn_dp_limit && - env->me_options.dp_limit != value32) { - env->me_options.dp_limit = value32; - env->me_options.flags.non_auto.dp_limit = 1; - if (env->me_options.dp_initial > value32) - env->me_options.dp_initial = value32; + switch (DriveType) { + case DRIVE_CDROM: + if (flags & MDBX_RDONLY) + break; + // fall through + case DRIVE_UNKNOWN: + case DRIVE_NO_ROOT_DIR: + case DRIVE_REMOTE: + default: + if (!(flags & MDBX_EXCLUSIVE)) + rc = MDBX_EREMOTE; + // fall through + case DRIVE_REMOVABLE: + case DRIVE_FIXED: + case DRIVE_RAMDISK: + break; } } - break; - case MDBX_opt_spill_max_denominator: - if (value == /* default */ UINT64_MAX) - value = 8; - if (unlikely(value > 255)) - return MDBX_EINVAL; - env->me_options.spill_max_denominator = (uint8_t)value; - break; - case MDBX_opt_spill_min_denominator: - if (value == /* default */ UINT64_MAX) - value = 8; - if (unlikely(value > 255)) - return MDBX_EINVAL; - env->me_options.spill_min_denominator = (uint8_t)value; - break; - case MDBX_opt_spill_parent4child_denominator: - if (value == /* default */ UINT64_MAX) - value = 0; - if (unlikely(value > 255)) - return MDBX_EINVAL; - env->me_options.spill_parent4child_denominator = (uint8_t)value; - break; + bailout: + osal_free(PathBuffer); + return rc; + } + +#else + + struct statvfs statvfs_info; + if (fstatvfs(handle, &statvfs_info)) + return errno; +#if defined(ST_LOCAL) || defined(ST_EXPORTED) + const unsigned long st_flags = statvfs_info.f_flag; +#endif /* ST_LOCAL || ST_EXPORTED */ + +#if defined(__NetBSD__) + const unsigned type = 0; + const char *const name = statvfs_info.f_fstypename; + const size_t name_len = VFS_NAMELEN; +#elif defined(_AIX) || defined(__OS400__) + const char *const name = statvfs_info.f_basetype; + const size_t name_len = sizeof(statvfs_info.f_basetype); + struct stat st; + if (fstat(handle, &st)) + return errno; + const unsigned type = st.st_vfstype; + if ((st.st_flag & FS_REMOTE) != 0 && !(flags & MDBX_EXCLUSIVE)) + return MDBX_EREMOTE; +#elif defined(FSTYPSZ) || defined(_FSTYPSZ) + const unsigned type = 0; + const char *const name = statvfs_info.f_basetype; + const size_t name_len = sizeof(statvfs_info.f_basetype); +#elif defined(__sun) || defined(__SVR4) || defined(__svr4__) || defined(ST_FSTYPSZ) || defined(_ST_FSTYPSZ) + const unsigned type = 0; + struct stat st; + if (fstat(handle, &st)) + return errno; + const char *const name = st.st_fstype; + const size_t name_len = strlen(name); +#else + struct statfs statfs_info; + if (fstatfs(handle, &statfs_info)) + return errno; +#if defined(__OpenBSD__) + const unsigned type = 0; +#else + const unsigned type = statfs_info.f_type; +#endif +#if defined(MNT_LOCAL) || defined(MNT_EXPORTED) + const unsigned long mnt_flags = statfs_info.f_flags; +#endif /* MNT_LOCAL || MNT_EXPORTED */ +#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__BSD__) || defined(__bsdi__) || \ + defined(__DragonFly__) || defined(__APPLE__) || defined(__MACH__) || defined(MFSNAMELEN) || \ + defined(MFSTYPENAMELEN) || defined(VFS_NAMELEN) + const char *const name = statfs_info.f_fstypename; + const size_t name_len = sizeof(statfs_info.f_fstypename); +#elif defined(__ANDROID_API__) && __ANDROID_API__ < 21 + const char *const name = ""; + const unsigned name_len = 0; +#else - case MDBX_opt_loose_limit: - if (value == /* default */ UINT64_MAX) - value = 64; - if (unlikely(value > 255)) - return MDBX_EINVAL; - env->me_options.dp_loose_limit = (uint8_t)value; - break; + const char *name = ""; + unsigned name_len = 0; - case MDBX_opt_merge_threshold_16dot16_percent: - if (value == /* default */ UINT64_MAX) - value = 65536 / 4 /* 25% */; - if (unlikely(value < 8192 || value > 32768)) - return MDBX_EINVAL; - env->me_options.merge_threshold_16dot16_percent = (unsigned)value; - recalculate_merge_threshold(env); - break; + struct stat st; + if (fstat(handle, &st)) + return errno; - case MDBX_opt_writethrough_threshold: -#if defined(_WIN32) || defined(_WIN64) - /* позволяем "установить" значение по-умолчанию и совпадающее - * с поведением соответствующим текущей установке MDBX_NOMETASYNC */ - if (value == /* default */ UINT64_MAX && - value != ((env->me_flags & MDBX_NOMETASYNC) ? 0 : UINT_MAX)) - err = MDBX_EINVAL; + char pathbuf[PATH_MAX]; + FILE *mounted = nullptr; +#if defined(__linux__) || defined(__gnu_linux__) + mounted = setmntent("/proc/mounts", "r"); +#endif /* Linux */ + if (!mounted) + mounted = setmntent("/etc/mtab", "r"); + if (mounted) { + const struct mntent *ent; +#if defined(_BSD_SOURCE) || defined(_SVID_SOURCE) || defined(__BIONIC__) || \ + (defined(_DEFAULT_SOURCE) && __GLIBC_PREREQ(2, 19)) + struct mntent entbuf; + const bool should_copy = false; + while (nullptr != (ent = getmntent_r(mounted, &entbuf, pathbuf, sizeof(pathbuf)))) #else - if (value == /* default */ UINT64_MAX) - value = MDBX_WRITETHROUGH_THRESHOLD_DEFAULT; - if (value != (unsigned)value) - err = MDBX_EINVAL; - else - env->me_options.writethrough_threshold = (unsigned)value; + const bool should_copy = true; + while (nullptr != (ent = getmntent(mounted))) #endif - break; - - case MDBX_opt_prefault_write_enable: - if (value == /* default */ UINT64_MAX) { - env->me_options.prefault_write = default_prefault_write(env); - env->me_options.flags.non_auto.prefault_write = false; - } else if (value > 1) - err = MDBX_EINVAL; - else { - env->me_options.prefault_write = value != 0; - env->me_options.flags.non_auto.prefault_write = true; + { + struct stat mnt; + if (!stat(ent->mnt_dir, &mnt) && mnt.st_dev == st.st_dev) { + if (should_copy) { + name = strncpy(pathbuf, ent->mnt_fsname, name_len = sizeof(pathbuf) - 1); + pathbuf[name_len] = 0; + } else { + name = ent->mnt_fsname; + name_len = strlen(name); + } + break; + } } - break; - - default: - return MDBX_EINVAL; + endmntent(mounted); } +#endif /* !xBSD && !Android/Bionic */ +#endif - if (should_unlock) - mdbx_txn_unlock(env); - return err; -} - -__cold int mdbx_env_get_option(const MDBX_env *env, const MDBX_option_t option, - uint64_t *pvalue) { - int err = check_env(env, false); - if (unlikely(err != MDBX_SUCCESS)) - return err; - if (unlikely(!pvalue)) - return MDBX_EINVAL; - - switch (option) { - case MDBX_opt_sync_bytes: - if (unlikely(!(env->me_flags & MDBX_ENV_ACTIVE))) - return MDBX_EPERM; - *pvalue = pgno2bytes( - env, atomic_load32(&env->me_lck->mti_autosync_threshold, mo_Relaxed)); - break; - - case MDBX_opt_sync_period: - if (unlikely(!(env->me_flags & MDBX_ENV_ACTIVE))) - return MDBX_EPERM; - *pvalue = osal_monotime_to_16dot16( - atomic_load64(&env->me_lck->mti_autosync_period, mo_Relaxed)); - break; - - case MDBX_opt_max_db: - *pvalue = env->me_maxdbs - CORE_DBS; - break; - - case MDBX_opt_max_readers: - *pvalue = env->me_maxreaders; - break; - - case MDBX_opt_dp_reserve_limit: - *pvalue = env->me_options.dp_reserve_limit; - break; - - case MDBX_opt_rp_augment_limit: - *pvalue = env->me_options.rp_augment_limit; - break; - - case MDBX_opt_txn_dp_limit: - *pvalue = env->me_options.dp_limit; - break; - case MDBX_opt_txn_dp_initial: - *pvalue = env->me_options.dp_initial; - break; - - case MDBX_opt_spill_max_denominator: - *pvalue = env->me_options.spill_max_denominator; - break; - case MDBX_opt_spill_min_denominator: - *pvalue = env->me_options.spill_min_denominator; - break; - case MDBX_opt_spill_parent4child_denominator: - *pvalue = env->me_options.spill_parent4child_denominator; - break; + if (name_len) { + if (((name_len > 2 && strncasecmp("nfs", name, 3) == 0) || strncasecmp("cifs", name, name_len) == 0 || + strncasecmp("ncpfs", name, name_len) == 0 || strncasecmp("smbfs", name, name_len) == 0 || + strcasecmp("9P" /* WSL2 */, name) == 0 || + ((name_len > 3 && strncasecmp("fuse", name, 4) == 0) && strncasecmp("fuseblk", name, name_len) != 0)) && + !(flags & MDBX_EXCLUSIVE)) + return MDBX_EREMOTE; + if (strcasecmp("ftp", name) == 0 || strcasecmp("http", name) == 0 || strcasecmp("sshfs", name) == 0) + return MDBX_EREMOTE; + } - case MDBX_opt_loose_limit: - *pvalue = env->me_options.dp_loose_limit; - break; +#ifdef ST_LOCAL + if ((st_flags & ST_LOCAL) == 0 && !(flags & MDBX_EXCLUSIVE)) + return MDBX_EREMOTE; +#elif defined(MNT_LOCAL) + if ((mnt_flags & MNT_LOCAL) == 0 && !(flags & MDBX_EXCLUSIVE)) + return MDBX_EREMOTE; +#endif /* ST/MNT_LOCAL */ - case MDBX_opt_merge_threshold_16dot16_percent: - *pvalue = env->me_options.merge_threshold_16dot16_percent; - break; +#ifdef ST_EXPORTED + if ((st_flags & ST_EXPORTED) != 0 && !(flags & (MDBX_RDONLY | MDBX_EXCLUSIVE)))) + return MDBX_RESULT_TRUE; +#elif defined(MNT_EXPORTED) + if ((mnt_flags & MNT_EXPORTED) != 0 && !(flags & (MDBX_RDONLY | MDBX_EXCLUSIVE))) + return MDBX_RESULT_TRUE; +#endif /* ST/MNT_EXPORTED */ - case MDBX_opt_writethrough_threshold: -#if defined(_WIN32) || defined(_WIN64) - *pvalue = (env->me_flags & MDBX_NOMETASYNC) ? 0 : INT_MAX; -#else - *pvalue = env->me_options.writethrough_threshold; + switch (type) { + case 0xFF534D42 /* CIFS_MAGIC_NUMBER */: + case 0x6969 /* NFS_SUPER_MAGIC */: + case 0x564c /* NCP_SUPER_MAGIC */: + case 0x517B /* SMB_SUPER_MAGIC */: +#if defined(__digital__) || defined(__osf__) || defined(__osf) + case 0x0E /* Tru64 NFS */: #endif - break; - - case MDBX_opt_prefault_write_enable: - *pvalue = env->me_options.prefault_write; - break; - +#ifdef ST_FST_NFS + case ST_FST_NFS: +#endif + if ((flags & MDBX_EXCLUSIVE) == 0) + return MDBX_EREMOTE; + case 0: default: - return MDBX_EINVAL; + break; } +#endif /* Unix */ return MDBX_SUCCESS; } -static size_t estimate_rss(size_t database_bytes) { - return database_bytes + database_bytes / 64 + - (512 + MDBX_WORDBITS * 16) * MEGABYTE; -} - -__cold int mdbx_env_warmup(const MDBX_env *env, const MDBX_txn *txn, - MDBX_warmup_flags_t flags, - unsigned timeout_seconds_16dot16) { - if (unlikely(env == NULL && txn == NULL)) - return MDBX_EINVAL; - if (unlikely(flags > - (MDBX_warmup_force | MDBX_warmup_oomsafe | MDBX_warmup_lock | - MDBX_warmup_touchlimit | MDBX_warmup_release))) - return MDBX_EINVAL; +static int check_mmap_limit(const size_t limit) { + const bool should_check = +#if defined(__SANITIZE_ADDRESS__) + true; +#else + RUNNING_ON_VALGRIND; +#endif /* __SANITIZE_ADDRESS__ */ - if (txn) { - int err = check_txn(txn, MDBX_TXN_BLOCKED - MDBX_TXN_ERROR); - if (unlikely(err != MDBX_SUCCESS)) - return err; - } - if (env) { - int err = check_env(env, false); + if (should_check) { + intptr_t pagesize, total_ram_pages, avail_ram_pages; + int err = mdbx_get_sysraminfo(&pagesize, &total_ram_pages, &avail_ram_pages); if (unlikely(err != MDBX_SUCCESS)) return err; - if (txn && unlikely(txn->mt_env != env)) - return MDBX_EINVAL; - } else { - env = txn->mt_env; - } - - const uint64_t timeout_monotime = - (timeout_seconds_16dot16 && (flags & MDBX_warmup_force)) - ? osal_monotime() + osal_16dot16_to_monotime(timeout_seconds_16dot16) - : 0; - if (flags & MDBX_warmup_release) - munlock_all(env); - - pgno_t used_pgno; - if (txn) { - used_pgno = txn->mt_geo.next; - } else { - const meta_troika_t troika = meta_tap(env); - used_pgno = meta_recent(env, &troika).ptr_v->mm_geo.next; + const int log2page = log2n_powerof2(pagesize); + if ((limit >> (log2page + 7)) > (size_t)total_ram_pages || (limit >> (log2page + 6)) > (size_t)avail_ram_pages) { + ERROR("%s (%zu pages) is too large for available (%zu pages) or total " + "(%zu pages) system RAM", + "database upper size limit", limit >> log2page, avail_ram_pages, total_ram_pages); + return MDBX_TOO_LARGE; + } } - const size_t used_range = pgno_align2os_bytes(env, used_pgno); - const pgno_t mlock_pgno = bytes2pgno(env, used_range); - int rc = MDBX_SUCCESS; - if (flags & MDBX_warmup_touchlimit) { - const size_t estimated_rss = estimate_rss(used_range); + return MDBX_SUCCESS; +} + +MDBX_INTERNAL int osal_mmap(const int flags, osal_mmap_t *map, size_t size, const size_t limit, const unsigned options, + const pathchar_t *pathname4logging) { + assert(size <= limit); + map->limit = 0; + map->current = 0; + map->base = nullptr; + map->filesize = 0; #if defined(_WIN32) || defined(_WIN64) - SIZE_T current_ws_lower, current_ws_upper; - if (GetProcessWorkingSetSize(GetCurrentProcess(), ¤t_ws_lower, - ¤t_ws_upper) && - current_ws_lower < estimated_rss) { - const SIZE_T ws_lower = estimated_rss; - const SIZE_T ws_upper = - (MDBX_WORDBITS == 32 && ws_lower > MEGABYTE * 2048) - ? ws_lower - : ws_lower + MDBX_WORDBITS * MEGABYTE * 32; - if (!SetProcessWorkingSetSize(GetCurrentProcess(), ws_lower, ws_upper)) { - rc = (int)GetLastError(); - WARNING("SetProcessWorkingSetSize(%zu, %zu) error %d", ws_lower, - ws_upper, rc); - } - } + map->section = nullptr; #endif /* Windows */ -#ifdef RLIMIT_RSS - struct rlimit rss; - if (getrlimit(RLIMIT_RSS, &rss) == 0 && rss.rlim_cur < estimated_rss) { - rss.rlim_cur = estimated_rss; - if (rss.rlim_max < estimated_rss) - rss.rlim_max = estimated_rss; - if (setrlimit(RLIMIT_RSS, &rss)) { - rc = errno; - WARNING("setrlimit(%s, {%zu, %zu}) error %d", "RLIMIT_RSS", - (size_t)rss.rlim_cur, (size_t)rss.rlim_max, rc); - } - } -#endif /* RLIMIT_RSS */ -#ifdef RLIMIT_MEMLOCK - if (flags & MDBX_warmup_lock) { - struct rlimit memlock; - if (getrlimit(RLIMIT_MEMLOCK, &memlock) == 0 && - memlock.rlim_cur < estimated_rss) { - memlock.rlim_cur = estimated_rss; - if (memlock.rlim_max < estimated_rss) - memlock.rlim_max = estimated_rss; - if (setrlimit(RLIMIT_MEMLOCK, &memlock)) { - rc = errno; - WARNING("setrlimit(%s, {%zu, %zu}) error %d", "RLIMIT_MEMLOCK", - (size_t)memlock.rlim_cur, (size_t)memlock.rlim_max, rc); - } - } - } -#endif /* RLIMIT_MEMLOCK */ - (void)estimated_rss; - } -#if defined(MLOCK_ONFAULT) && \ - ((defined(_GNU_SOURCE) && __GLIBC_PREREQ(2, 27)) || \ - (defined(__ANDROID_API__) && __ANDROID_API__ >= 30)) && \ - (defined(__linux__) || defined(__gnu_linux__)) - if ((flags & MDBX_warmup_lock) != 0 && linux_kernel_version >= 0x04040000 && - atomic_load32(&env->me_mlocked_pgno, mo_AcquireRelease) < mlock_pgno) { - if (mlock2(env->me_map, used_range, MLOCK_ONFAULT)) { - rc = errno; - WARNING("mlock2(%zu, %s) error %d", used_range, "MLOCK_ONFAULT", rc); - } else { - update_mlcnt(env, mlock_pgno, true); - rc = MDBX_SUCCESS; + int err = osal_check_fs_local(map->fd, flags); + if (unlikely(err != MDBX_SUCCESS)) { +#if defined(_WIN32) || defined(_WIN64) + if (globals.running_under_Wine) + NOTICE("%s", "Please use native Linux application or WSL at least, instead of trouble-full Wine!"); +#endif /* Windows */ + switch (err) { + case MDBX_RESULT_TRUE: +#if MDBX_ENABLE_NON_READONLY_EXPORT + WARNING("%" MDBX_PRIsPATH " is exported via NFS, avoid using the file on a remote side!", pathname4logging); + break; +#else + ERROR("%" MDBX_PRIsPATH " is exported via NFS", pathname4logging); + return MDBX_EREMOTE; +#endif /* MDBX_PROHIBIT_NON_READONLY_EXPORT */ + case MDBX_EREMOTE: + ERROR("%" MDBX_PRIsPATH " is on a remote file system, the %s is required", pathname4logging, "MDBX_EXCLUSIVE"); + __fallthrough /* fall through */; + default: + return err; } - if (rc != EINVAL) - flags -= MDBX_warmup_lock; } -#endif /* MLOCK_ONFAULT */ - int err = MDBX_ENOSYS; -#if MDBX_ENABLE_MADVISE - err = set_readahead(env, used_pgno, true, true); -#else + err = check_mmap_limit(limit); + if (unlikely(err != MDBX_SUCCESS)) + return err; + + if ((flags & MDBX_RDONLY) == 0 && (options & MMAP_OPTION_TRUNCATE) != 0) { + err = osal_ftruncate(map->fd, size); + VERBOSE("ftruncate %zu, err %d", size, err); + if (err != MDBX_SUCCESS) + return err; + map->filesize = size; +#if !(defined(_WIN32) || defined(_WIN64)) + map->current = size; +#endif /* !Windows */ + } else { + err = osal_filesize(map->fd, &map->filesize); + VERBOSE("filesize %" PRIu64 ", err %d", map->filesize, err); + if (err != MDBX_SUCCESS) + return err; #if defined(_WIN32) || defined(_WIN64) - if (mdbx_PrefetchVirtualMemory) { - WIN32_MEMORY_RANGE_ENTRY hint; - hint.VirtualAddress = env->me_map; - hint.NumberOfBytes = used_range; - if (mdbx_PrefetchVirtualMemory(GetCurrentProcess(), 1, &hint, 0)) - err = MDBX_SUCCESS; - else { - err = (int)GetLastError(); - ERROR("%s(%zu) error %d", "PrefetchVirtualMemory", used_range, err); + if (map->filesize < size) { + WARNING("file size (%zu) less than requested for mapping (%zu)", (size_t)map->filesize, size); + size = (size_t)map->filesize; } +#else + map->current = (map->filesize > limit) ? limit : (size_t)map->filesize; +#endif /* !Windows */ } -#endif /* Windows */ -#if defined(POSIX_MADV_WILLNEED) - err = posix_madvise(env->me_map, used_range, POSIX_MADV_WILLNEED) - ? ignore_enosys(errno) - : MDBX_SUCCESS; -#elif defined(MADV_WILLNEED) - err = madvise(env->me_map, used_range, MADV_WILLNEED) ? ignore_enosys(errno) - : MDBX_SUCCESS; +#if defined(_WIN32) || defined(_WIN64) + LARGE_INTEGER SectionSize; + SectionSize.QuadPart = size; + err = NtCreateSection(&map->section, + /* DesiredAccess */ + (flags & MDBX_WRITEMAP) + ? SECTION_QUERY | SECTION_MAP_READ | SECTION_EXTEND_SIZE | SECTION_MAP_WRITE + : SECTION_QUERY | SECTION_MAP_READ | SECTION_EXTEND_SIZE, + /* ObjectAttributes */ nullptr, + /* MaximumSize (InitialSize) */ &SectionSize, + /* SectionPageProtection */ + (flags & MDBX_RDONLY) ? PAGE_READONLY : PAGE_READWRITE, + /* AllocationAttributes */ SEC_RESERVE, map->fd); + if (!NT_SUCCESS(err)) + return ntstatus2errcode(err); + + SIZE_T ViewSize = (flags & MDBX_RDONLY) ? 0 : globals.running_under_Wine ? size : limit; + err = NtMapViewOfSection(map->section, GetCurrentProcess(), &map->base, + /* ZeroBits */ 0, + /* CommitSize */ 0, + /* SectionOffset */ nullptr, &ViewSize, + /* InheritDisposition */ ViewUnmap, + /* AllocationType */ (flags & MDBX_RDONLY) ? 0 : MEM_RESERVE, + /* Win32Protect */ + (flags & MDBX_WRITEMAP) ? PAGE_READWRITE : PAGE_READONLY); + if (!NT_SUCCESS(err)) { + NtClose(map->section); + map->section = 0; + map->base = nullptr; + return ntstatus2errcode(err); + } + assert(map->base != MAP_FAILED); + + map->current = (size_t)SectionSize.QuadPart; + map->limit = ViewSize; + +#else /* Windows */ + +#ifndef MAP_TRYFIXED +#define MAP_TRYFIXED 0 #endif -#if defined(F_RDADVISE) - if (err) { - fcntl(env->me_lazy_fd, F_RDAHEAD, true); - struct radvisory hint; - hint.ra_offset = 0; - hint.ra_count = unlikely(used_range > INT_MAX && - sizeof(used_range) > sizeof(hint.ra_count)) - ? INT_MAX - : (int)used_range; - err = fcntl(env->me_lazy_fd, F_RDADVISE, &hint) ? ignore_enosys(errno) - : MDBX_SUCCESS; - if (err == ENOTTY) - err = MDBX_SUCCESS /* Ignore ENOTTY for DB on the ram-disk */; - } -#endif /* F_RDADVISE */ -#endif /* MDBX_ENABLE_MADVISE */ - if (err != MDBX_SUCCESS && rc == MDBX_SUCCESS) - rc = err; +#ifndef MAP_HASSEMAPHORE +#define MAP_HASSEMAPHORE 0 +#endif - if ((flags & MDBX_warmup_force) != 0 && - (rc == MDBX_SUCCESS || rc == MDBX_ENOSYS)) { - const volatile uint8_t *ptr = env->me_map; - size_t offset = 0, unused = 42; -#if !(defined(_WIN32) || defined(_WIN64)) - if (flags & MDBX_warmup_oomsafe) { - const int null_fd = open("/dev/null", O_WRONLY); - if (unlikely(null_fd < 0)) - rc = errno; - else { - struct iovec iov[MDBX_AUXILARY_IOV_MAX]; - for (;;) { - unsigned i; - for (i = 0; i < MDBX_AUXILARY_IOV_MAX && offset < used_range; ++i) { - iov[i].iov_base = (void *)(ptr + offset); - iov[i].iov_len = 1; - offset += env->me_os_psize; - } - if (unlikely(writev(null_fd, iov, i) < 0)) { - rc = errno; - if (rc == EFAULT) - rc = ENOMEM; - break; - } - if (offset >= used_range) { - rc = MDBX_SUCCESS; - break; - } - if (timeout_seconds_16dot16 && osal_monotime() > timeout_monotime) { - rc = MDBX_RESULT_TRUE; - break; - } - } - close(null_fd); - } - } else -#endif /* Windows */ - for (;;) { - unused += ptr[offset]; - offset += env->me_os_psize; - if (offset >= used_range) { - rc = MDBX_SUCCESS; - break; - } - if (timeout_seconds_16dot16 && osal_monotime() > timeout_monotime) { - rc = MDBX_RESULT_TRUE; - break; - } - } - (void)unused; - } +#ifndef MAP_CONCEAL +#define MAP_CONCEAL 0 +#endif - if ((flags & MDBX_warmup_lock) != 0 && - (rc == MDBX_SUCCESS || rc == MDBX_ENOSYS) && - atomic_load32(&env->me_mlocked_pgno, mo_AcquireRelease) < mlock_pgno) { -#if defined(_WIN32) || defined(_WIN64) - if (VirtualLock(env->me_map, used_range)) { - update_mlcnt(env, mlock_pgno, true); - rc = MDBX_SUCCESS; - } else { - rc = (int)GetLastError(); - WARNING("%s(%zu) error %d", "VirtualLock", used_range, rc); - } -#elif defined(_POSIX_MEMLOCK_RANGE) - if (mlock(env->me_map, used_range) == 0) { - update_mlcnt(env, mlock_pgno, true); - rc = MDBX_SUCCESS; - } else { - rc = errno; - WARNING("%s(%zu) error %d", "mlock", used_range, rc); - } -#else - rc = MDBX_ENOSYS; +#ifndef MAP_NOSYNC +#define MAP_NOSYNC 0 #endif + +#ifndef MAP_FIXED_NOREPLACE +#define MAP_FIXED_NOREPLACE 0 +#endif + +#ifndef MAP_NORESERVE +#define MAP_NORESERVE 0 +#endif + + map->base = mmap(nullptr, limit, (flags & MDBX_WRITEMAP) ? PROT_READ | PROT_WRITE : PROT_READ, + MAP_SHARED | MAP_FILE | MAP_NORESERVE | (F_ISSET(flags, MDBX_UTTERLY_NOSYNC) ? MAP_NOSYNC : 0) | + ((options & MMAP_OPTION_SEMAPHORE) ? MAP_HASSEMAPHORE | MAP_NOSYNC : MAP_CONCEAL), + map->fd, 0); + + if (unlikely(map->base == MAP_FAILED)) { + map->limit = 0; + map->current = 0; + map->base = nullptr; + assert(errno != 0); + return errno; } + map->limit = limit; - return rc; +#ifdef MADV_DONTFORK + if (unlikely(madvise(map->base, map->limit, MADV_DONTFORK) != 0)) + return errno; +#endif /* MADV_DONTFORK */ +#ifdef MADV_NOHUGEPAGE + (void)madvise(map->base, map->limit, MADV_NOHUGEPAGE); +#endif /* MADV_NOHUGEPAGE */ + +#endif /* ! Windows */ + + VALGRIND_MAKE_MEM_DEFINED(map->base, map->current); + MDBX_ASAN_UNPOISON_MEMORY_REGION(map->base, map->current); + return MDBX_SUCCESS; } -__cold void global_ctor(void) { - osal_ctor(); - rthc_limit = RTHC_INITIAL_LIMIT; - rthc_table = rthc_table_static; +MDBX_INTERNAL int osal_munmap(osal_mmap_t *map) { + VALGRIND_MAKE_MEM_NOACCESS(map->base, map->current); + /* Unpoisoning is required for ASAN to avoid false-positive diagnostic + * when this memory will re-used by malloc or another mmapping. + * See https://libmdbx.dqdkfa.ru/dead-github/pull/93#issuecomment-613687203 */ + MDBX_ASAN_UNPOISON_MEMORY_REGION(map->base, + (map->filesize && map->filesize < map->limit) ? map->filesize : map->limit); #if defined(_WIN32) || defined(_WIN64) - InitializeCriticalSection(&rthc_critical_section); - InitializeCriticalSection(&lcklist_critical_section); + if (map->section) + NtClose(map->section); + NTSTATUS rc = NtUnmapViewOfSection(GetCurrentProcess(), map->base); + if (!NT_SUCCESS(rc)) + ntstatus2errcode(rc); #else - ENSURE(nullptr, pthread_key_create(&rthc_key, thread_dtor) == 0); - TRACE("pid %d, &mdbx_rthc_key = %p, value 0x%x", osal_getpid(), - __Wpedantic_format_voidptr(&rthc_key), (unsigned)rthc_key); -#endif - /* checking time conversion, this also avoids racing on 32-bit architectures - * during storing calculated 64-bit ratio(s) into memory. */ - uint32_t proba = UINT32_MAX; - while (true) { - unsigned time_conversion_checkup = - osal_monotime_to_16dot16(osal_16dot16_to_monotime(proba)); - unsigned one_more = (proba < UINT32_MAX) ? proba + 1 : proba; - unsigned one_less = (proba > 0) ? proba - 1 : proba; - ENSURE(nullptr, time_conversion_checkup >= one_less && - time_conversion_checkup <= one_more); - if (proba == 0) - break; - proba >>= 1; + if (unlikely(munmap(map->base, map->limit))) { + assert(errno != 0); + return errno; } +#endif /* ! Windows */ - bootid = osal_bootid(); + map->limit = 0; + map->current = 0; + map->base = nullptr; + return MDBX_SUCCESS; +} -#if MDBX_DEBUG - for (size_t i = 0; i < 2 * 2 * 2 * 3 * 3 * 3; ++i) { - const bool s0 = (i >> 0) & 1; - const bool s1 = (i >> 1) & 1; - const bool s2 = (i >> 2) & 1; - const uint8_t c01 = (i / (8 * 1)) % 3; - const uint8_t c02 = (i / (8 * 3)) % 3; - const uint8_t c12 = (i / (8 * 9)) % 3; +MDBX_INTERNAL int osal_mresize(const int flags, osal_mmap_t *map, size_t size, size_t limit) { + int rc = osal_filesize(map->fd, &map->filesize); + VERBOSE("flags 0x%x, size %zu, limit %zu, filesize %" PRIu64, flags, size, limit, map->filesize); + assert(size <= limit); + if (rc != MDBX_SUCCESS) { + map->filesize = 0; + return rc; + } - const uint8_t packed = meta_cmp2pack(c01, c02, c12, s0, s1, s2); - meta_troika_t troika; - troika.fsm = (uint8_t)i; - meta_troika_unpack(&troika, packed); +#if defined(_WIN32) || defined(_WIN64) + assert(size != map->current || limit != map->limit || size < map->filesize); - const uint8_t tail = TROIKA_TAIL(&troika); - const bool strict = TROIKA_STRICT_VALID(&troika); - const bool valid = TROIKA_VALID(&troika); + NTSTATUS status; + LARGE_INTEGER SectionSize; + int err; - const uint8_t recent_chk = meta_cmp2recent(c01, s0, s1) - ? (meta_cmp2recent(c02, s0, s2) ? 0 : 2) - : (meta_cmp2recent(c12, s1, s2) ? 1 : 2); - const uint8_t prefer_steady_chk = - meta_cmp2steady(c01, s0, s1) ? (meta_cmp2steady(c02, s0, s2) ? 0 : 2) - : (meta_cmp2steady(c12, s1, s2) ? 1 : 2); + if (limit == map->limit && size > map->current) { + if ((flags & MDBX_RDONLY) && map->filesize >= size) { + map->current = size; + return MDBX_SUCCESS; + } else if (!(flags & MDBX_RDONLY) && + /* workaround for Wine */ imports.NtExtendSection) { + /* growth rw-section */ + SectionSize.QuadPart = size; + status = imports.NtExtendSection(map->section, &SectionSize); + if (!NT_SUCCESS(status)) + return ntstatus2errcode(status); + map->current = size; + if (map->filesize < size) + map->filesize = size; + return MDBX_SUCCESS; + } + } + + if (limit > map->limit) { + err = check_mmap_limit(limit); + if (unlikely(err != MDBX_SUCCESS)) + return err; + + /* check ability of address space for growth before unmap */ + PVOID BaseAddress = (PBYTE)map->base + map->limit; + SIZE_T RegionSize = limit - map->limit; + status = NtAllocateVirtualMemory(GetCurrentProcess(), &BaseAddress, 0, &RegionSize, MEM_RESERVE, PAGE_NOACCESS); + if (status == (NTSTATUS) /* STATUS_CONFLICTING_ADDRESSES */ 0xC0000018) + return MDBX_UNABLE_EXTEND_MAPSIZE; + if (!NT_SUCCESS(status)) + return ntstatus2errcode(status); + + status = NtFreeVirtualMemory(GetCurrentProcess(), &BaseAddress, &RegionSize, MEM_RELEASE); + if (!NT_SUCCESS(status)) + return ntstatus2errcode(status); + } + + /* Windows unable: + * - shrink a mapped file; + * - change size of mapped view; + * - extend read-only mapping; + * Therefore we should unmap/map entire section. */ + if ((flags & MDBX_MRESIZE_MAY_UNMAP) == 0) { + if (size <= map->current && limit == map->limit) + return MDBX_SUCCESS; + return MDBX_EPERM; + } - uint8_t tail_chk; - if (recent_chk == 0) - tail_chk = meta_cmp2steady(c12, s1, s2) ? 2 : 1; - else if (recent_chk == 1) - tail_chk = meta_cmp2steady(c02, s0, s2) ? 2 : 0; - else - tail_chk = meta_cmp2steady(c01, s0, s1) ? 1 : 0; + /* Unpoisoning is required for ASAN to avoid false-positive diagnostic + * when this memory will re-used by malloc or another mmapping. + * See https://libmdbx.dqdkfa.ru/dead-github/pull/93#issuecomment-613687203 */ + MDBX_ASAN_UNPOISON_MEMORY_REGION(map->base, map->limit); + status = NtUnmapViewOfSection(GetCurrentProcess(), map->base); + if (!NT_SUCCESS(status)) + return ntstatus2errcode(status); + status = NtClose(map->section); + map->section = nullptr; + PVOID ReservedAddress = nullptr; + SIZE_T ReservedSize = limit; - const bool valid_chk = - c01 != 1 || s0 != s1 || c02 != 1 || s0 != s2 || c12 != 1 || s1 != s2; - const bool strict_chk = (c01 != 1 || s0 != s1) && (c02 != 1 || s0 != s2) && - (c12 != 1 || s1 != s2); - assert(troika.recent == recent_chk); - assert(troika.prefer_steady == prefer_steady_chk); - assert(tail == tail_chk); - assert(valid == valid_chk); - assert(strict == strict_chk); - // printf(" %d, ", packed); - assert(troika_fsm_map[troika.fsm] == packed); + if (!NT_SUCCESS(status)) { + bailout_ntstatus: + err = ntstatus2errcode(status); + map->base = nullptr; + map->current = map->limit = 0; + if (ReservedAddress) { + ReservedSize = 0; + status = NtFreeVirtualMemory(GetCurrentProcess(), &ReservedAddress, &ReservedSize, MEM_RELEASE); + assert(NT_SUCCESS(status)); + (void)status; + } + return err; } -#endif /* MDBX_DEBUG*/ -#if 0 /* debug */ - for (size_t i = 0; i < 65536; ++i) { - size_t pages = pv2pages(i); - size_t x = pages2pv(pages); - size_t xp = pv2pages(x); - if (!(x == i || (x % 2 == 0 && x < 65536)) || pages != xp) - printf("%u => %zu => %u => %zu\n", i, pages, x, xp); - assert(pages == xp); +retry_file_and_section: + /* resizing of the file may take a while, + * therefore we reserve address space to avoid occupy it by other threads */ + ReservedAddress = map->base; + status = NtAllocateVirtualMemory(GetCurrentProcess(), &ReservedAddress, 0, &ReservedSize, MEM_RESERVE, PAGE_NOACCESS); + if (!NT_SUCCESS(status)) { + ReservedAddress = nullptr; + if (status != (NTSTATUS) /* STATUS_CONFLICTING_ADDRESSES */ 0xC0000018) + goto bailout_ntstatus /* no way to recovery */; + + if (flags & MDBX_MRESIZE_MAY_MOVE) + /* the base address could be changed */ + map->base = nullptr; } - fflush(stdout); -#endif /* #if 0 */ -} -/*------------------------------------------------------------------------------ - * Legacy API */ + if ((flags & MDBX_RDONLY) == 0 && map->filesize != size) { + err = osal_ftruncate(map->fd, size); + if (err == MDBX_SUCCESS) + map->filesize = size; + /* ignore error, because Windows unable shrink file + * that already mapped (by another process) */ + } -#ifndef LIBMDBX_NO_EXPORTS_LEGACY_API + SectionSize.QuadPart = size; + status = NtCreateSection(&map->section, + /* DesiredAccess */ + (flags & MDBX_WRITEMAP) + ? SECTION_QUERY | SECTION_MAP_READ | SECTION_EXTEND_SIZE | SECTION_MAP_WRITE + : SECTION_QUERY | SECTION_MAP_READ | SECTION_EXTEND_SIZE, + /* ObjectAttributes */ nullptr, + /* MaximumSize (InitialSize) */ &SectionSize, + /* SectionPageProtection */ + (flags & MDBX_RDONLY) ? PAGE_READONLY : PAGE_READWRITE, + /* AllocationAttributes */ SEC_RESERVE, map->fd); -LIBMDBX_API int mdbx_txn_begin(MDBX_env *env, MDBX_txn *parent, - MDBX_txn_flags_t flags, MDBX_txn **ret) { - return __inline_mdbx_txn_begin(env, parent, flags, ret); -} + if (!NT_SUCCESS(status)) + goto bailout_ntstatus; -LIBMDBX_API int mdbx_txn_commit(MDBX_txn *txn) { - return __inline_mdbx_txn_commit(txn); -} + if (ReservedAddress) { + /* release reserved address space */ + ReservedSize = 0; + status = NtFreeVirtualMemory(GetCurrentProcess(), &ReservedAddress, &ReservedSize, MEM_RELEASE); + ReservedAddress = nullptr; + if (!NT_SUCCESS(status)) + goto bailout_ntstatus; + } -LIBMDBX_API __cold int mdbx_env_stat(const MDBX_env *env, MDBX_stat *stat, - size_t bytes) { - return __inline_mdbx_env_stat(env, stat, bytes); -} +retry_mapview:; + SIZE_T ViewSize = (flags & MDBX_RDONLY) ? size : limit; + status = NtMapViewOfSection(map->section, GetCurrentProcess(), &map->base, + /* ZeroBits */ 0, + /* CommitSize */ 0, + /* SectionOffset */ nullptr, &ViewSize, + /* InheritDisposition */ ViewUnmap, + /* AllocationType */ (flags & MDBX_RDONLY) ? 0 : MEM_RESERVE, + /* Win32Protect */ + (flags & MDBX_WRITEMAP) ? PAGE_READWRITE : PAGE_READONLY); -LIBMDBX_API __cold int mdbx_env_info(const MDBX_env *env, MDBX_envinfo *info, - size_t bytes) { - return __inline_mdbx_env_info(env, info, bytes); -} + if (!NT_SUCCESS(status)) { + if (status == (NTSTATUS) /* STATUS_CONFLICTING_ADDRESSES */ 0xC0000018 && map->base && + (flags & MDBX_MRESIZE_MAY_MOVE) != 0) { + /* try remap at another base address */ + map->base = nullptr; + goto retry_mapview; + } + NtClose(map->section); + map->section = nullptr; -LIBMDBX_API int mdbx_dbi_flags(const MDBX_txn *txn, MDBX_dbi dbi, - unsigned *flags) { - return __inline_mdbx_dbi_flags(txn, dbi, flags); -} + if (map->base && (size != map->current || limit != map->limit)) { + /* try remap with previously size and limit, + * but will return MDBX_UNABLE_EXTEND_MAPSIZE on success */ + rc = (limit > map->limit) ? MDBX_UNABLE_EXTEND_MAPSIZE : MDBX_EPERM; + size = map->current; + ReservedSize = limit = map->limit; + goto retry_file_and_section; + } -LIBMDBX_API __cold int mdbx_env_sync(MDBX_env *env) { - return __inline_mdbx_env_sync(env); -} + /* no way to recovery */ + goto bailout_ntstatus; + } + assert(map->base != MAP_FAILED); -LIBMDBX_API __cold int mdbx_env_sync_poll(MDBX_env *env) { - return __inline_mdbx_env_sync_poll(env); -} + map->current = (size_t)SectionSize.QuadPart; + map->limit = ViewSize; -LIBMDBX_API __cold int mdbx_env_close(MDBX_env *env) { - return __inline_mdbx_env_close(env); -} +#else /* Windows */ -LIBMDBX_API __cold int mdbx_env_set_mapsize(MDBX_env *env, size_t size) { - return __inline_mdbx_env_set_mapsize(env, size); -} + if (flags & MDBX_RDONLY) { + if (size > map->filesize) + rc = MDBX_UNABLE_EXTEND_MAPSIZE; + else if (size < map->filesize && map->filesize > limit) + rc = MDBX_EPERM; + map->current = (map->filesize > limit) ? limit : (size_t)map->filesize; + } else { + if (size > map->filesize || (size < map->filesize && (flags & txn_shrink_allowed))) { + rc = osal_ftruncate(map->fd, size); + VERBOSE("ftruncate %zu, err %d", size, rc); + if (rc != MDBX_SUCCESS) + return rc; + map->filesize = size; + } -LIBMDBX_API __cold int mdbx_env_set_maxdbs(MDBX_env *env, MDBX_dbi dbs) { - return __inline_mdbx_env_set_maxdbs(env, dbs); -} + if (map->current > size) { + /* Clearing asan's bitmask for the region which released in shrinking, + * since: + * - after the shrinking we will get an exception when accessing + * this region and (therefore) do not need the help of ASAN. + * - this allows us to clear the mask only within the file size + * when closing the mapping. */ + MDBX_ASAN_UNPOISON_MEMORY_REGION(ptr_disp(map->base, size), + ((map->current < map->limit) ? map->current : map->limit) - size); + } + map->current = (size < map->limit) ? size : map->limit; + } -LIBMDBX_API __cold int mdbx_env_get_maxdbs(const MDBX_env *env, MDBX_dbi *dbs) { - return __inline_mdbx_env_get_maxdbs(env, dbs); -} + if (limit == map->limit) + return rc; -LIBMDBX_API __cold int mdbx_env_set_maxreaders(MDBX_env *env, - unsigned readers) { - return __inline_mdbx_env_set_maxreaders(env, readers); -} + if (limit < map->limit) { + /* unmap an excess at end of mapping. */ + // coverity[offset_free : FALSE] + if (unlikely(munmap(ptr_disp(map->base, limit), map->limit - limit))) { + assert(errno != 0); + return errno; + } + map->limit = limit; + return rc; + } -LIBMDBX_API __cold int mdbx_env_get_maxreaders(const MDBX_env *env, - unsigned *readers) { - return __inline_mdbx_env_get_maxreaders(env, readers); -} + int err = check_mmap_limit(limit); + if (unlikely(err != MDBX_SUCCESS)) + return err; -LIBMDBX_API __cold int mdbx_env_set_syncbytes(MDBX_env *env, size_t threshold) { - return __inline_mdbx_env_set_syncbytes(env, threshold); -} + assert(limit > map->limit); + void *ptr = MAP_FAILED; -LIBMDBX_API __cold int mdbx_env_get_syncbytes(const MDBX_env *env, - size_t *threshold) { - return __inline_mdbx_env_get_syncbytes(env, threshold); -} +#if (defined(__linux__) || defined(__gnu_linux__)) && defined(_GNU_SOURCE) + ptr = mremap(map->base, map->limit, limit, +#if defined(MREMAP_MAYMOVE) + (flags & MDBX_MRESIZE_MAY_MOVE) ? MREMAP_MAYMOVE : +#endif /* MREMAP_MAYMOVE */ + 0); + if (ptr == MAP_FAILED) { + err = errno; + assert(err != 0); + switch (err) { + default: + return err; + case 0 /* paranoia */: + case EAGAIN: + case ENOMEM: + return MDBX_UNABLE_EXTEND_MAPSIZE; + case EFAULT /* MADV_DODUMP / MADV_DONTDUMP are mixed for mmap-range */: + break; + } + } +#endif /* Linux & _GNU_SOURCE */ -LIBMDBX_API __cold int mdbx_env_set_syncperiod(MDBX_env *env, - unsigned seconds_16dot16) { - return __inline_mdbx_env_set_syncperiod(env, seconds_16dot16); -} + const unsigned mmap_flags = + MAP_CONCEAL | MAP_SHARED | MAP_FILE | MAP_NORESERVE | (F_ISSET(flags, MDBX_UTTERLY_NOSYNC) ? MAP_NOSYNC : 0); + const unsigned mmap_prot = (flags & MDBX_WRITEMAP) ? PROT_READ | PROT_WRITE : PROT_READ; -LIBMDBX_API __cold int mdbx_env_get_syncperiod(const MDBX_env *env, - unsigned *seconds_16dot16) { - return __inline_mdbx_env_get_syncperiod(env, seconds_16dot16); -} + if (ptr == MAP_FAILED) { + /* Try to mmap additional space beyond the end of mapping. */ + ptr = mmap(ptr_disp(map->base, map->limit), limit - map->limit, mmap_prot, mmap_flags | MAP_FIXED_NOREPLACE, + map->fd, map->limit); + if (ptr == ptr_disp(map->base, map->limit)) + /* успешно прилепили отображение в конец */ + ptr = map->base; + else if (ptr != MAP_FAILED) { + /* the desired address is busy, unmap unsuitable one */ + if (unlikely(munmap(ptr, limit - map->limit))) { + assert(errno != 0); + return errno; + } + ptr = MAP_FAILED; + } else { + err = errno; + assert(err != 0); + switch (err) { + default: + return err; + case 0 /* paranoia */: + case EAGAIN: + case ENOMEM: + return MDBX_UNABLE_EXTEND_MAPSIZE; + case EEXIST: /* address busy */ + case EINVAL: /* kernel don't support MAP_FIXED_NOREPLACE */ + break; + } + } + } -LIBMDBX_API __cold MDBX_NOTHROW_CONST_FUNCTION intptr_t -mdbx_limits_pgsize_min(void) { - return __inline_mdbx_limits_pgsize_min(); -} + if (ptr == MAP_FAILED) { + /* unmap and map again whole region */ + if ((flags & MDBX_MRESIZE_MAY_UNMAP) == 0) { + /* TODO: Perhaps here it is worth to implement suspend/resume threads + * and perform unmap/map as like for Windows. */ + return MDBX_UNABLE_EXTEND_MAPSIZE; + } -LIBMDBX_API __cold MDBX_NOTHROW_CONST_FUNCTION intptr_t -mdbx_limits_pgsize_max(void) { - return __inline_mdbx_limits_pgsize_max(); -} + if (unlikely(munmap(map->base, map->limit))) { + assert(errno != 0); + return errno; + } -LIBMDBX_API MDBX_NOTHROW_CONST_FUNCTION uint64_t -mdbx_key_from_int64(const int64_t i64) { - return __inline_mdbx_key_from_int64(i64); -} + // coverity[pass_freed_arg : FALSE] + ptr = mmap(map->base, limit, mmap_prot, + (flags & MDBX_MRESIZE_MAY_MOVE) ? mmap_flags + : mmap_flags | (MAP_FIXED_NOREPLACE ? MAP_FIXED_NOREPLACE : MAP_FIXED), + map->fd, 0); + if (MAP_FIXED_NOREPLACE != 0 && MAP_FIXED_NOREPLACE != MAP_FIXED && unlikely(ptr == MAP_FAILED) && + !(flags & MDBX_MRESIZE_MAY_MOVE) && errno == /* kernel don't support MAP_FIXED_NOREPLACE */ EINVAL) + // coverity[pass_freed_arg : FALSE] + ptr = mmap(map->base, limit, mmap_prot, mmap_flags | MAP_FIXED, map->fd, 0); -LIBMDBX_API MDBX_NOTHROW_CONST_FUNCTION uint32_t -mdbx_key_from_int32(const int32_t i32) { - return __inline_mdbx_key_from_int32(i32); -} + if (unlikely(ptr == MAP_FAILED)) { + /* try to restore prev mapping */ + // coverity[pass_freed_arg : FALSE] + ptr = mmap(map->base, map->limit, mmap_prot, + (flags & MDBX_MRESIZE_MAY_MOVE) ? mmap_flags + : mmap_flags | (MAP_FIXED_NOREPLACE ? MAP_FIXED_NOREPLACE : MAP_FIXED), + map->fd, 0); + if (MAP_FIXED_NOREPLACE != 0 && MAP_FIXED_NOREPLACE != MAP_FIXED && unlikely(ptr == MAP_FAILED) && + !(flags & MDBX_MRESIZE_MAY_MOVE) && errno == /* kernel don't support MAP_FIXED_NOREPLACE */ EINVAL) + // coverity[pass_freed_arg : FALSE] + ptr = mmap(map->base, map->limit, mmap_prot, mmap_flags | MAP_FIXED, map->fd, 0); + if (unlikely(ptr == MAP_FAILED)) { + VALGRIND_MAKE_MEM_NOACCESS(map->base, map->current); + /* Unpoisoning is required for ASAN to avoid false-positive diagnostic + * when this memory will re-used by malloc or another mmapping. + * See + * https://libmdbx.dqdkfa.ru/dead-github/pull/93#issuecomment-613687203 + */ + MDBX_ASAN_UNPOISON_MEMORY_REGION(map->base, (map->current < map->limit) ? map->current : map->limit); + map->limit = 0; + map->current = 0; + map->base = nullptr; + assert(errno != 0); + return errno; + } + rc = MDBX_UNABLE_EXTEND_MAPSIZE; + limit = map->limit; + } + } -#endif /* LIBMDBX_NO_EXPORTS_LEGACY_API */ + assert(ptr && ptr != MAP_FAILED); + if (map->base != ptr) { + VALGRIND_MAKE_MEM_NOACCESS(map->base, map->current); + /* Unpoisoning is required for ASAN to avoid false-positive diagnostic + * when this memory will re-used by malloc or another mmapping. + * See + * https://libmdbx.dqdkfa.ru/dead-github/pull/93#issuecomment-613687203 + */ + MDBX_ASAN_UNPOISON_MEMORY_REGION(map->base, (map->current < map->limit) ? map->current : map->limit); -/******************************************************************************/ + VALGRIND_MAKE_MEM_DEFINED(ptr, map->current); + MDBX_ASAN_UNPOISON_MEMORY_REGION(ptr, map->current); + map->base = ptr; + } + map->limit = limit; + map->current = size; -__dll_export -#ifdef __attribute_used__ - __attribute_used__ -#elif defined(__GNUC__) || __has_attribute(__used__) - __attribute__((__used__)) -#endif -#ifdef __attribute_externally_visible__ - __attribute_externally_visible__ -#elif (defined(__GNUC__) && !defined(__clang__)) || \ - __has_attribute(__externally_visible__) - __attribute__((__externally_visible__)) -#endif - const struct MDBX_build_info mdbx_build = { -#ifdef MDBX_BUILD_TIMESTAMP - MDBX_BUILD_TIMESTAMP -#else - "\"" __DATE__ " " __TIME__ "\"" -#endif /* MDBX_BUILD_TIMESTAMP */ +#ifdef MADV_DONTFORK + if (unlikely(madvise(map->base, map->limit, MADV_DONTFORK) != 0)) { + assert(errno != 0); + return errno; + } +#endif /* MADV_DONTFORK */ +#ifdef MADV_NOHUGEPAGE + (void)madvise(map->base, map->limit, MADV_NOHUGEPAGE); +#endif /* MADV_NOHUGEPAGE */ - , -#ifdef MDBX_BUILD_TARGET - MDBX_BUILD_TARGET -#else - #if defined(__ANDROID_API__) - "Android" MDBX_STRINGIFY(__ANDROID_API__) - #elif defined(__linux__) || defined(__gnu_linux__) - "Linux" - #elif defined(EMSCRIPTEN) || defined(__EMSCRIPTEN__) - "webassembly" - #elif defined(__CYGWIN__) - "CYGWIN" - #elif defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) \ - || defined(__WINDOWS__) - "Windows" - #elif defined(__APPLE__) - #if (defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE) \ - || (defined(TARGET_IPHONE_SIMULATOR) && TARGET_IPHONE_SIMULATOR) - "iOS" - #else - "MacOS" - #endif - #elif defined(__FreeBSD__) - "FreeBSD" - #elif defined(__DragonFly__) - "DragonFlyBSD" - #elif defined(__NetBSD__) - "NetBSD" - #elif defined(__OpenBSD__) - "OpenBSD" - #elif defined(__bsdi__) - "UnixBSDI" - #elif defined(__MACH__) - "MACH" - #elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) - "HPUX" - #elif defined(_AIX) - "AIX" - #elif defined(__sun) && defined(__SVR4) - "Solaris" - #elif defined(__BSD__) || defined(BSD) - "UnixBSD" - #elif defined(__unix__) || defined(UNIX) || defined(__unix) \ - || defined(__UNIX) || defined(__UNIX__) - "UNIX" - #elif defined(_POSIX_VERSION) - "POSIX" MDBX_STRINGIFY(_POSIX_VERSION) - #else - "UnknownOS" - #endif /* Target OS */ +#endif /* POSIX / Windows */ - "-" + /* Zap: Redundant code */ + MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(6287); + assert(rc != MDBX_SUCCESS || (map->base != nullptr && map->base != MAP_FAILED && map->current == size && + map->limit == limit && map->filesize >= size)); + return rc; +} - #if defined(__amd64__) - "AMD64" - #elif defined(__ia32__) - "IA32" - #elif defined(__e2k__) || defined(__elbrus__) - "Elbrus" - #elif defined(__alpha__) || defined(__alpha) || defined(_M_ALPHA) - "Alpha" - #elif defined(__aarch64__) || defined(_M_ARM64) - "ARM64" - #elif defined(__arm__) || defined(__thumb__) || defined(__TARGET_ARCH_ARM) \ - || defined(__TARGET_ARCH_THUMB) || defined(_ARM) || defined(_M_ARM) \ - || defined(_M_ARMT) || defined(__arm) - "ARM" - #elif defined(__mips64) || defined(__mips64__) || (defined(__mips) && (__mips >= 64)) - "MIPS64" - #elif defined(__mips__) || defined(__mips) || defined(_R4000) || defined(__MIPS__) - "MIPS" - #elif defined(__hppa64__) || defined(__HPPA64__) || defined(__hppa64) - "PARISC64" - #elif defined(__hppa__) || defined(__HPPA__) || defined(__hppa) - "PARISC" - #elif defined(__ia64__) || defined(__ia64) || defined(_IA64) \ - || defined(__IA64__) || defined(_M_IA64) || defined(__itanium__) - "Itanium" - #elif defined(__powerpc64__) || defined(__ppc64__) || defined(__ppc64) \ - || defined(__powerpc64) || defined(_ARCH_PPC64) - "PowerPC64" - #elif defined(__powerpc__) || defined(__ppc__) || defined(__powerpc) \ - || defined(__ppc) || defined(_ARCH_PPC) || defined(__PPC__) || defined(__POWERPC__) - "PowerPC" - #elif defined(__sparc64__) || defined(__sparc64) - "SPARC64" - #elif defined(__sparc__) || defined(__sparc) - "SPARC" - #elif defined(__s390__) || defined(__s390) || defined(__zarch__) || defined(__zarch) - "S390" - #else - "UnknownARCH" - #endif -#endif /* MDBX_BUILD_TARGET */ +/*----------------------------------------------------------------------------*/ -#ifdef MDBX_BUILD_TYPE -# if defined(_MSC_VER) -# pragma message("Configuration-depended MDBX_BUILD_TYPE: " MDBX_BUILD_TYPE) -# endif - "-" MDBX_BUILD_TYPE -#endif /* MDBX_BUILD_TYPE */ - , - "MDBX_DEBUG=" MDBX_STRINGIFY(MDBX_DEBUG) -#ifdef ENABLE_GPROF - " ENABLE_GPROF" -#endif /* ENABLE_GPROF */ - " MDBX_WORDBITS=" MDBX_STRINGIFY(MDBX_WORDBITS) - " BYTE_ORDER=" -#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ - "LITTLE_ENDIAN" -#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - "BIG_ENDIAN" -#else - #error "FIXME: Unsupported byte order" -#endif /* __BYTE_ORDER__ */ - " MDBX_ENABLE_BIGFOOT=" MDBX_STRINGIFY(MDBX_ENABLE_BIGFOOT) - " MDBX_ENV_CHECKPID=" MDBX_ENV_CHECKPID_CONFIG - " MDBX_TXN_CHECKOWNER=" MDBX_TXN_CHECKOWNER_CONFIG - " MDBX_64BIT_ATOMIC=" MDBX_64BIT_ATOMIC_CONFIG - " MDBX_64BIT_CAS=" MDBX_64BIT_CAS_CONFIG - " MDBX_TRUST_RTC=" MDBX_TRUST_RTC_CONFIG - " MDBX_AVOID_MSYNC=" MDBX_STRINGIFY(MDBX_AVOID_MSYNC) - " MDBX_ENABLE_REFUND=" MDBX_STRINGIFY(MDBX_ENABLE_REFUND) - " MDBX_ENABLE_MADVISE=" MDBX_STRINGIFY(MDBX_ENABLE_MADVISE) - " MDBX_ENABLE_MINCORE=" MDBX_STRINGIFY(MDBX_ENABLE_MINCORE) - " MDBX_ENABLE_PGOP_STAT=" MDBX_STRINGIFY(MDBX_ENABLE_PGOP_STAT) - " MDBX_ENABLE_PROFGC=" MDBX_STRINGIFY(MDBX_ENABLE_PROFGC) -#if MDBX_DISABLE_VALIDATION - " MDBX_DISABLE_VALIDATION=YES" -#endif /* MDBX_DISABLE_VALIDATION */ -#ifdef __SANITIZE_ADDRESS__ - " SANITIZE_ADDRESS=YES" -#endif /* __SANITIZE_ADDRESS__ */ -#ifdef MDBX_USE_VALGRIND - " MDBX_USE_VALGRIND=YES" -#endif /* MDBX_USE_VALGRIND */ -#if MDBX_FORCE_ASSERTIONS - " MDBX_FORCE_ASSERTIONS=YES" -#endif /* MDBX_FORCE_ASSERTIONS */ -#ifdef _GNU_SOURCE - " _GNU_SOURCE=YES" -#else - " _GNU_SOURCE=NO" -#endif /* _GNU_SOURCE */ -#ifdef __APPLE__ - " MDBX_OSX_SPEED_INSTEADOF_DURABILITY=" MDBX_STRINGIFY(MDBX_OSX_SPEED_INSTEADOF_DURABILITY) -#endif /* MacOS */ -#if defined(_WIN32) || defined(_WIN64) - " MDBX_WITHOUT_MSVC_CRT=" MDBX_STRINGIFY(MDBX_WITHOUT_MSVC_CRT) - " MDBX_BUILD_SHARED_LIBRARY=" MDBX_STRINGIFY(MDBX_BUILD_SHARED_LIBRARY) -#if !MDBX_BUILD_SHARED_LIBRARY - " MDBX_MANUAL_MODULE_HANDLER=" MDBX_STRINGIFY(MDBX_MANUAL_MODULE_HANDLER) -#endif - " WINVER=" MDBX_STRINGIFY(WINVER) -#else /* Windows */ - " MDBX_LOCKING=" MDBX_LOCKING_CONFIG - " MDBX_USE_OFDLOCKS=" MDBX_USE_OFDLOCKS_CONFIG -#endif /* !Windows */ - " MDBX_CACHELINE_SIZE=" MDBX_STRINGIFY(MDBX_CACHELINE_SIZE) - " MDBX_CPU_WRITEBACK_INCOHERENT=" MDBX_STRINGIFY(MDBX_CPU_WRITEBACK_INCOHERENT) - " MDBX_MMAP_INCOHERENT_CPU_CACHE=" MDBX_STRINGIFY(MDBX_MMAP_INCOHERENT_CPU_CACHE) - " MDBX_MMAP_INCOHERENT_FILE_WRITE=" MDBX_STRINGIFY(MDBX_MMAP_INCOHERENT_FILE_WRITE) - " MDBX_UNALIGNED_OK=" MDBX_STRINGIFY(MDBX_UNALIGNED_OK) - " MDBX_PNL_ASCENDING=" MDBX_STRINGIFY(MDBX_PNL_ASCENDING) - , -#ifdef MDBX_BUILD_COMPILER - MDBX_BUILD_COMPILER -#else - #ifdef __INTEL_COMPILER - "Intel C/C++ " MDBX_STRINGIFY(__INTEL_COMPILER) - #elif defined(__apple_build_version__) - "Apple clang " MDBX_STRINGIFY(__apple_build_version__) - #elif defined(__ibmxl__) - "IBM clang C " MDBX_STRINGIFY(__ibmxl_version__) "." MDBX_STRINGIFY(__ibmxl_release__) - "." MDBX_STRINGIFY(__ibmxl_modification__) "." MDBX_STRINGIFY(__ibmxl_ptf_fix_level__) - #elif defined(__clang__) - "clang " MDBX_STRINGIFY(__clang_version__) - #elif defined(__MINGW64__) - "MINGW-64 " MDBX_STRINGIFY(__MINGW64_MAJOR_VERSION) "." MDBX_STRINGIFY(__MINGW64_MINOR_VERSION) - #elif defined(__MINGW32__) - "MINGW-32 " MDBX_STRINGIFY(__MINGW32_MAJOR_VERSION) "." MDBX_STRINGIFY(__MINGW32_MINOR_VERSION) - #elif defined(__MINGW__) - "MINGW " MDBX_STRINGIFY(__MINGW_MAJOR_VERSION) "." MDBX_STRINGIFY(__MINGW_MINOR_VERSION) - #elif defined(__IBMC__) - "IBM C " MDBX_STRINGIFY(__IBMC__) - #elif defined(__GNUC__) - "GNU C/C++ " - #ifdef __VERSION__ - __VERSION__ - #else - MDBX_STRINGIFY(__GNUC__) "." MDBX_STRINGIFY(__GNUC_MINOR__) "." MDBX_STRINGIFY(__GNUC_PATCHLEVEL__) - #endif - #elif defined(_MSC_VER) - "MSVC " MDBX_STRINGIFY(_MSC_FULL_VER) "-" MDBX_STRINGIFY(_MSC_BUILD) - #else - "Unknown compiler" - #endif -#endif /* MDBX_BUILD_COMPILER */ - , -#ifdef MDBX_BUILD_FLAGS_CONFIG - MDBX_BUILD_FLAGS_CONFIG -#endif /* MDBX_BUILD_FLAGS_CONFIG */ -#ifdef MDBX_BUILD_FLAGS - MDBX_BUILD_FLAGS -#endif /* MDBX_BUILD_FLAGS */ -#if !(defined(MDBX_BUILD_FLAGS_CONFIG) || defined(MDBX_BUILD_FLAGS)) - "undefined (please use correct build script)" -#ifdef _MSC_VER -#pragma message("warning: Build flags undefined. Please use correct build script") +__cold MDBX_INTERNAL void osal_jitter(bool tiny) { + for (;;) { +#if defined(_M_IX86) || defined(_M_X64) || defined(__i386__) || defined(__x86_64__) + unsigned salt = 5296013u * (unsigned)__rdtsc(); + salt ^= salt >> 11; + salt *= 25810541u; +#elif (defined(_WIN32) || defined(_WIN64)) && MDBX_WITHOUT_MSVC_CRT + static ULONG state; + const unsigned salt = (unsigned)RtlRandomEx(&state); #else -#warning "Build flags undefined. Please use correct build script" -#endif // _MSC_VER + const unsigned salt = rand(); #endif -}; -#ifdef __SANITIZE_ADDRESS__ -#if !defined(_MSC_VER) || __has_attribute(weak) -LIBMDBX_API __attribute__((__weak__)) + const int coin = salt % (tiny ? 29u : 43u); + if (coin < 43 / 3) + break; +#if defined(_WIN32) || defined(_WIN64) + if (coin < 43 * 2 / 3) + SwitchToThread(); + else { + static HANDLE timer; + if (!timer) + timer = CreateWaitableTimer(NULL, TRUE, NULL); + + LARGE_INTEGER ft; + ft.QuadPart = coin * (int64_t)-10; // Convert to 100 nanosecond interval, + // negative value indicates relative time. + SetWaitableTimer(timer, &ft, 0, NULL, NULL, 0); + WaitForSingleObject(timer, INFINITE); + // CloseHandle(timer); + break; + } +#else + sched_yield(); + if (coin > 43 * 2 / 3) + usleep(coin); #endif -const char *__asan_default_options(void) { - return "symbolize=1:allow_addr2line=1:" -#if MDBX_DEBUG - "debug=1:" - "verbosity=2:" -#endif /* MDBX_DEBUG */ - "log_threads=1:" - "report_globals=1:" - "replace_str=1:replace_intrin=1:" - "malloc_context_size=9:" -#if !defined(__APPLE__) - "detect_leaks=1:" + } +} + +/*----------------------------------------------------------------------------*/ + +#if defined(_WIN32) || defined(_WIN64) +static LARGE_INTEGER performance_frequency; +#elif defined(__APPLE__) || defined(__MACH__) +#include +static uint64_t ratio_16dot16_to_monotine; +#elif defined(__linux__) || defined(__gnu_linux__) +static clockid_t posix_clockid; +__cold static clockid_t choice_monoclock(void) { + struct timespec probe; +#if defined(CLOCK_BOOTTIME) + if (clock_gettime(CLOCK_BOOTTIME, &probe) == 0) + return CLOCK_BOOTTIME; +#elif defined(CLOCK_MONOTONIC_RAW) + if (clock_gettime(CLOCK_MONOTONIC_RAW, &probe) == 0) + return CLOCK_MONOTONIC_RAW; +#elif defined(CLOCK_MONOTONIC_COARSE) + if (clock_gettime(CLOCK_MONOTONIC_COARSE, &probe) == 0) + return CLOCK_MONOTONIC_COARSE; #endif - "check_printf=1:" - "detect_deadlocks=1:" -#ifndef LTO_ENABLED - "check_initialization_order=1:" + return CLOCK_MONOTONIC; +} +#elif defined(CLOCK_MONOTONIC) +#define posix_clockid CLOCK_MONOTONIC +#else +#define posix_clockid CLOCK_REALTIME #endif - "detect_stack_use_after_return=1:" - "intercept_tls_get_addr=1:" - "decorate_proc_maps=1:" - "abort_on_error=1"; + +MDBX_INTERNAL uint64_t osal_16dot16_to_monotime(uint32_t seconds_16dot16) { +#if defined(_WIN32) || defined(_WIN64) + const uint64_t ratio = performance_frequency.QuadPart; +#elif defined(__APPLE__) || defined(__MACH__) + const uint64_t ratio = ratio_16dot16_to_monotine; +#else + const uint64_t ratio = UINT64_C(1000000000); +#endif + const uint64_t ret = (ratio * seconds_16dot16 + 32768) >> 16; + return likely(ret || seconds_16dot16 == 0) ? ret : /* fix underflow */ 1; } -#endif /* __SANITIZE_ADDRESS__ */ -/* https://en.wikipedia.org/wiki/Operating_system_abstraction_layer */ +static uint64_t monotime_limit; +MDBX_INTERNAL uint32_t osal_monotime_to_16dot16(uint64_t monotime) { + if (unlikely(monotime > monotime_limit)) + return UINT32_MAX; -/* - * Copyright 2015-2024 Leonid Yuriev - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - */ + const uint32_t ret = +#if defined(_WIN32) || defined(_WIN64) + (uint32_t)((monotime << 16) / performance_frequency.QuadPart); +#elif defined(__APPLE__) || defined(__MACH__) + (uint32_t)((monotime << 16) / ratio_16dot16_to_monotine); +#else + (uint32_t)((monotime << 7) / 1953125); +#endif + return ret; +} +MDBX_INTERNAL uint64_t osal_monotime(void) { +#if defined(_WIN32) || defined(_WIN64) + LARGE_INTEGER counter; + if (QueryPerformanceCounter(&counter)) + return counter.QuadPart; +#elif defined(__APPLE__) || defined(__MACH__) + return mach_absolute_time(); +#else + struct timespec ts; + if (likely(clock_gettime(posix_clockid, &ts) == 0)) + return ts.tv_sec * UINT64_C(1000000000) + ts.tv_nsec; +#endif + return 0; +} +MDBX_INTERNAL uint64_t osal_cputime(size_t *optional_page_faults) { #if defined(_WIN32) || defined(_WIN64) + if (optional_page_faults) { + PROCESS_MEMORY_COUNTERS pmc; + *optional_page_faults = GetProcessMemoryInfo(GetCurrentProcess(), &pmc, sizeof(pmc)) ? pmc.PageFaultCount : 0; + } + FILETIME unused, usermode; + if (GetThreadTimes(GetCurrentThread(), + /* CreationTime */ &unused, + /* ExitTime */ &unused, + /* KernelTime */ &unused, + /* UserTime */ &usermode)) { + /* one second = 10_000_000 * 100ns = 78125 * (1 << 7) * 100ns; + * result = (h * f / 10_000_000) << 32) + l * f / 10_000_000 = + * = ((h * f) >> 7) / 78125) << 32) + ((l * f) >> 7) / 78125; + * 1) {h, l} *= f; + * 2) {h, l} >>= 7; + * 3) result = ((h / 78125) << 32) + l / 78125; */ + uint64_t l = usermode.dwLowDateTime * performance_frequency.QuadPart; + uint64_t h = usermode.dwHighDateTime * performance_frequency.QuadPart; + l = h << (64 - 7) | l >> 7; + h = h >> 7; + return ((h / 78125) << 32) + l / 78125; + } +#elif defined(RUSAGE_THREAD) || defined(RUSAGE_LWP) +#ifndef RUSAGE_THREAD +#define RUSAGE_THREAD RUSAGE_LWP /* Solaris */ +#endif + struct rusage usage; + if (getrusage(RUSAGE_THREAD, &usage) == 0) { + if (optional_page_faults) + *optional_page_faults = usage.ru_majflt; + return usage.ru_utime.tv_sec * UINT64_C(1000000000) + usage.ru_utime.tv_usec * 1000u; + } + if (optional_page_faults) + *optional_page_faults = 0; +#elif defined(CLOCK_THREAD_CPUTIME_ID) + if (optional_page_faults) + *optional_page_faults = 0; + struct timespec ts; + if (likely(clock_gettime(CLOCK_THREAD_CPUTIME_ID, &ts) == 0)) + return ts.tv_sec * UINT64_C(1000000000) + ts.tv_nsec; +#else + /* FIXME */ + if (optional_page_faults) + *optional_page_faults = 0; +#endif + return 0; +} -#include -#include +/*----------------------------------------------------------------------------*/ -#if !MDBX_WITHOUT_MSVC_CRT && defined(_DEBUG) -#include -#endif +static void bootid_shake(bin128_t *p) { + /* Bob Jenkins's PRNG: https://burtleburtle.net/bob/rand/smallprng.html */ + const uint32_t e = p->a - (p->b << 23 | p->b >> 9); + p->a = p->b ^ (p->c << 16 | p->c >> 16); + p->b = p->c + (p->d << 11 | p->d >> 21); + p->c = p->d + e; + p->d = e + p->a; +} -static int waitstatus2errcode(DWORD result) { - switch (result) { - case WAIT_OBJECT_0: - return MDBX_SUCCESS; - case WAIT_FAILED: - return (int)GetLastError(); - case WAIT_ABANDONED: - return ERROR_ABANDONED_WAIT_0; - case WAIT_IO_COMPLETION: - return ERROR_USER_APC; - case WAIT_TIMEOUT: - return ERROR_TIMEOUT; - default: - return ERROR_UNHANDLED_ERROR; +__cold static void bootid_collect(bin128_t *p, const void *s, size_t n) { + p->y += UINT64_C(64526882297375213); + bootid_shake(p); + for (size_t i = 0; i < n; ++i) { + bootid_shake(p); + p->y ^= UINT64_C(48797879452804441) * ((const uint8_t *)s)[i]; + bootid_shake(p); + p->y += 14621231; } + bootid_shake(p); + + /* minor non-linear tomfoolery */ + const unsigned z = p->x % 61 + 1; + p->y = p->y << z | p->y >> (64 - z); + bootid_shake(p); + bootid_shake(p); + const unsigned q = p->x % 59 + 1; + p->y = p->y << q | p->y >> (64 - q); + bootid_shake(p); + bootid_shake(p); + bootid_shake(p); } -/* Map a result from an NTAPI call to WIN32 error code. */ -static int ntstatus2errcode(NTSTATUS status) { - DWORD dummy; - OVERLAPPED ov; - memset(&ov, 0, sizeof(ov)); - ov.Internal = status; - /* Zap: '_Param_(1)' could be '0' */ - MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(6387); - return GetOverlappedResult(NULL, &ov, &dummy, FALSE) ? MDBX_SUCCESS - : (int)GetLastError(); +static size_t hamming_weight(size_t v) { + const size_t m1 = (size_t)UINT64_C(0x5555555555555555); + const size_t m2 = (size_t)UINT64_C(0x3333333333333333); + const size_t m4 = (size_t)UINT64_C(0x0f0f0f0f0f0f0f0f); + const size_t h01 = (size_t)UINT64_C(0x0101010101010101); + v -= (v >> 1) & m1; + v = (v & m2) + ((v >> 2) & m2); + v = (v + (v >> 4)) & m4; + return (v * h01) >> (sizeof(v) * 8 - 8); } -/* We use native NT APIs to setup the memory map, so that we can - * let the DB file grow incrementally instead of always preallocating - * the full size. These APIs are defined in and - * but those headers are meant for driver-level development and - * conflict with the regular user-level headers, so we explicitly - * declare them here. Using these APIs also means we must link to - * ntdll.dll, which is not linked by default in user code. */ +static inline size_t hw64(uint64_t v) { + size_t r = hamming_weight((size_t)v); + if (sizeof(v) > sizeof(r)) + r += hamming_weight((size_t)(v >> sizeof(r) * 4 >> sizeof(r) * 4)); + return r; +} -extern NTSTATUS NTAPI NtCreateSection( - OUT PHANDLE SectionHandle, IN ACCESS_MASK DesiredAccess, - IN OPTIONAL POBJECT_ATTRIBUTES ObjectAttributes, - IN OPTIONAL PLARGE_INTEGER MaximumSize, IN ULONG SectionPageProtection, - IN ULONG AllocationAttributes, IN OPTIONAL HANDLE FileHandle); +static bool check_uuid(bin128_t uuid) { + size_t hw = hw64(uuid.x) + hw64(uuid.y) + hw64(uuid.x ^ uuid.y); + return (hw >> 6) == 1; +} -typedef struct _SECTION_BASIC_INFORMATION { - ULONG Unknown; - ULONG SectionAttributes; - LARGE_INTEGER SectionSize; -} SECTION_BASIC_INFORMATION, *PSECTION_BASIC_INFORMATION; +#if defined(_WIN32) || defined(_WIN64) -extern NTSTATUS NTAPI NtMapViewOfSection( - IN HANDLE SectionHandle, IN HANDLE ProcessHandle, IN OUT PVOID *BaseAddress, - IN ULONG_PTR ZeroBits, IN SIZE_T CommitSize, - IN OUT OPTIONAL PLARGE_INTEGER SectionOffset, IN OUT PSIZE_T ViewSize, - IN SECTION_INHERIT InheritDisposition, IN ULONG AllocationType, - IN ULONG Win32Protect); +__cold static uint64_t windows_systemtime_ms() { + FILETIME ft; + GetSystemTimeAsFileTime(&ft); + return ((uint64_t)ft.dwHighDateTime << 32 | ft.dwLowDateTime) / 10000ul; +} -extern NTSTATUS NTAPI NtUnmapViewOfSection(IN HANDLE ProcessHandle, - IN OPTIONAL PVOID BaseAddress); +__cold static uint64_t windows_bootime(void) { + unsigned confirmed = 0; + uint64_t boottime = 0; + uint64_t up0 = imports.GetTickCount64(); + uint64_t st0 = windows_systemtime_ms(); + for (uint64_t fuse = st0; up0 && st0 < fuse + 1000 * 1000u / 42;) { + YieldProcessor(); + const uint64_t up1 = imports.GetTickCount64(); + const uint64_t st1 = windows_systemtime_ms(); + if (st1 > fuse && st1 == st0 && up1 == up0) { + uint64_t diff = st1 - up1; + if (boottime == diff) { + if (++confirmed > 4) + return boottime; + } else { + confirmed = 0; + boottime = diff; + } + fuse = st1; + Sleep(1); + } + st0 = st1; + up0 = up1; + } + return 0; +} -/* Zap: Inconsistent annotation for 'NtClose'... */ -MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(28251) -extern NTSTATUS NTAPI NtClose(HANDLE Handle); +__cold static LSTATUS mdbx_RegGetValue(HKEY hKey, LPCSTR lpSubKey, LPCSTR lpValue, PVOID pvData, LPDWORD pcbData) { + LSTATUS rc; + if (!imports.RegGetValueA) { + /* an old Windows 2000/XP */ + HKEY hSubKey; + rc = RegOpenKeyA(hKey, lpSubKey, &hSubKey); + if (rc == ERROR_SUCCESS) { + rc = RegQueryValueExA(hSubKey, lpValue, nullptr, nullptr, pvData, pcbData); + RegCloseKey(hSubKey); + } + return rc; + } -extern NTSTATUS NTAPI NtAllocateVirtualMemory( - IN HANDLE ProcessHandle, IN OUT PVOID *BaseAddress, IN ULONG_PTR ZeroBits, - IN OUT PSIZE_T RegionSize, IN ULONG AllocationType, IN ULONG Protect); + rc = imports.RegGetValueA(hKey, lpSubKey, lpValue, RRF_RT_ANY, nullptr, pvData, pcbData); + if (rc != ERROR_FILE_NOT_FOUND) + return rc; -extern NTSTATUS NTAPI NtFreeVirtualMemory(IN HANDLE ProcessHandle, - IN PVOID *BaseAddress, - IN OUT PSIZE_T RegionSize, - IN ULONG FreeType); + rc = imports.RegGetValueA(hKey, lpSubKey, lpValue, RRF_RT_ANY | 0x00010000 /* RRF_SUBKEY_WOW6464KEY */, nullptr, + pvData, pcbData); + if (rc != ERROR_FILE_NOT_FOUND) + return rc; + return imports.RegGetValueA(hKey, lpSubKey, lpValue, RRF_RT_ANY | 0x00020000 /* RRF_SUBKEY_WOW6432KEY */, nullptr, + pvData, pcbData); +} +#endif -#ifndef WOF_CURRENT_VERSION -typedef struct _WOF_EXTERNAL_INFO { - DWORD Version; - DWORD Provider; -} WOF_EXTERNAL_INFO, *PWOF_EXTERNAL_INFO; -#endif /* WOF_CURRENT_VERSION */ +MDBX_MAYBE_UNUSED __cold static bool bootid_parse_uuid(bin128_t *s, const void *p, const size_t n) { + if (n > 31) { + unsigned bits = 0; + for (unsigned i = 0; i < n; ++i) /* try parse an UUID in text form */ { + uint8_t c = ((const uint8_t *)p)[i]; + if (c >= '0' && c <= '9') + c -= '0'; + else if (c >= 'a' && c <= 'f') + c -= 'a' - 10; + else if (c >= 'A' && c <= 'F') + c -= 'A' - 10; + else + continue; + assert(c <= 15); + c ^= s->y >> 60; + s->y = s->y << 4 | s->x >> 60; + s->x = s->x << 4 | c; + bits += 4; + } + if (bits > 42 * 3) + /* UUID parsed successfully */ + return true; + } -#ifndef WIM_PROVIDER_CURRENT_VERSION -#define WIM_PROVIDER_HASH_SIZE 20 + if (n > 15) /* is enough handle it as a binary? */ { + if (n == sizeof(bin128_t)) { + bin128_t aligned; + memcpy(&aligned, p, sizeof(bin128_t)); + s->x += aligned.x; + s->y += aligned.y; + } else + bootid_collect(s, p, n); + return check_uuid(*s); + } -typedef struct _WIM_PROVIDER_EXTERNAL_INFO { - DWORD Version; - DWORD Flags; - LARGE_INTEGER DataSourceId; - BYTE ResourceHash[WIM_PROVIDER_HASH_SIZE]; -} WIM_PROVIDER_EXTERNAL_INFO, *PWIM_PROVIDER_EXTERNAL_INFO; -#endif /* WIM_PROVIDER_CURRENT_VERSION */ + if (n) + bootid_collect(s, p, n); + return false; +} -#ifndef FILE_PROVIDER_CURRENT_VERSION -typedef struct _FILE_PROVIDER_EXTERNAL_INFO_V1 { - ULONG Version; - ULONG Algorithm; - ULONG Flags; -} FILE_PROVIDER_EXTERNAL_INFO_V1, *PFILE_PROVIDER_EXTERNAL_INFO_V1; -#endif /* FILE_PROVIDER_CURRENT_VERSION */ +#if defined(__linux__) || defined(__gnu_linux__) -#ifndef STATUS_OBJECT_NOT_EXTERNALLY_BACKED -#define STATUS_OBJECT_NOT_EXTERNALLY_BACKED ((NTSTATUS)0xC000046DL) -#endif -#ifndef STATUS_INVALID_DEVICE_REQUEST -#define STATUS_INVALID_DEVICE_REQUEST ((NTSTATUS)0xC0000010L) -#endif -#ifndef STATUS_NOT_SUPPORTED -#define STATUS_NOT_SUPPORTED ((NTSTATUS)0xC00000BBL) -#endif +__cold static bool is_inside_lxc(void) { + bool inside_lxc = false; + FILE *mounted = setmntent("/proc/mounts", "r"); + if (mounted) { + const struct mntent *ent; + while (nullptr != (ent = getmntent(mounted))) { + if (strcmp(ent->mnt_fsname, "lxcfs") == 0 && strncmp(ent->mnt_dir, "/proc/", 6) == 0) { + inside_lxc = true; + break; + } + } + endmntent(mounted); + } + return inside_lxc; +} -#ifndef FILE_DEVICE_FILE_SYSTEM -#define FILE_DEVICE_FILE_SYSTEM 0x00000009 -#endif +__cold static bool proc_read_uuid(const char *path, bin128_t *target) { + const int fd = open(path, O_RDONLY | O_NOFOLLOW); + if (fd != -1) { + struct statfs fs; + char buf[42]; + const ssize_t len = (fstatfs(fd, &fs) == 0 && + (fs.f_type == /* procfs */ 0x9FA0 || (fs.f_type == /* tmpfs */ 0x1021994 && is_inside_lxc()))) + ? read(fd, buf, sizeof(buf)) + : -1; + const int err = close(fd); + assert(err == 0); + (void)err; + if (len > 0) + return bootid_parse_uuid(target, buf, len); + } + return false; +} +#endif /* Linux */ -#ifndef FSCTL_GET_EXTERNAL_BACKING -#define FSCTL_GET_EXTERNAL_BACKING \ - CTL_CODE(FILE_DEVICE_FILE_SYSTEM, 196, METHOD_BUFFERED, FILE_ANY_ACCESS) -#endif +__cold static bin128_t osal_bootid(void) { + bin128_t uuid = {{0, 0}}; + bool got_machineid = false, got_boottime = false, got_bootseq = false; -#ifndef ERROR_NOT_CAPABLE -#define ERROR_NOT_CAPABLE 775L -#endif +#if defined(__linux__) || defined(__gnu_linux__) + if (proc_read_uuid("/proc/sys/kernel/random/boot_id", &uuid)) + return uuid; +#endif /* Linux */ -#endif /* _WIN32 || _WIN64 */ +#if defined(__APPLE__) || defined(__MACH__) + { + char buf[42]; + size_t len = sizeof(buf); + if (!sysctlbyname("kern.bootsessionuuid", buf, &len, nullptr, 0) && bootid_parse_uuid(&uuid, buf, len)) + return uuid; -/*----------------------------------------------------------------------------*/ +#if defined(__MAC_OS_X_VERSION_MIN_REQUIRED) && __MAC_OS_X_VERSION_MIN_REQUIRED > 1050 + uuid_t hostuuid; + struct timespec wait = {0, 1000000000u / 42}; + if (!gethostuuid(hostuuid, &wait)) + got_machineid = bootid_parse_uuid(&uuid, hostuuid, sizeof(hostuuid)); +#endif /* > 10.5 */ -#if defined(__ANDROID_API__) -__extern_C void __assert2(const char *file, int line, const char *function, - const char *msg) __noreturn; -#define __assert_fail(assertion, file, line, function) \ - __assert2(file, line, function, assertion) + struct timeval boottime; + len = sizeof(boottime); + if (!sysctlbyname("kern.boottime", &boottime, &len, nullptr, 0) && len == sizeof(boottime) && boottime.tv_sec) + got_boottime = true; + } +#endif /* Apple/Darwin */ -#elif defined(__UCLIBC__) -__extern_C void __assert(const char *, const char *, unsigned int, const char *) -#ifdef __THROW - __THROW -#else - __nothrow -#endif /* __THROW */ - MDBX_NORETURN; -#define __assert_fail(assertion, file, line, function) \ - __assert(assertion, file, line, function) +#if defined(_WIN32) || defined(_WIN64) + { + union buf { + DWORD BootId; + DWORD BaseTime; + SYSTEM_TIMEOFDAY_INFORMATION SysTimeOfDayInfo; + struct { + LARGE_INTEGER BootTime; + LARGE_INTEGER CurrentTime; + LARGE_INTEGER TimeZoneBias; + ULONG TimeZoneId; + ULONG Reserved; + ULONGLONG BootTimeBias; + ULONGLONG SleepTimeBias; + } SysTimeOfDayInfoHacked; + wchar_t MachineGuid[42]; + char DigitalProductId[248]; + } buf; -#elif _POSIX_C_SOURCE > 200212 && \ - /* workaround for avoid musl libc wrong prototype */ ( \ - defined(__GLIBC__) || defined(__GNU_LIBRARY__)) -/* Prototype should match libc runtime. ISO POSIX (2003) & LSB 1.x-3.x */ -__extern_C void __assert_fail(const char *assertion, const char *file, - unsigned line, const char *function) -#ifdef __THROW - __THROW -#else - __nothrow -#endif /* __THROW */ - MDBX_NORETURN; + static const char HKLM_MicrosoftCryptography[] = "SOFTWARE\\Microsoft\\Cryptography"; + DWORD len = sizeof(buf); + /* Windows is madness and must die */ + if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_MicrosoftCryptography, "MachineGuid", &buf.MachineGuid, &len) == + ERROR_SUCCESS && + len < sizeof(buf)) + got_machineid = bootid_parse_uuid(&uuid, &buf.MachineGuid, len); -#elif defined(__APPLE__) || defined(__MACH__) -__extern_C void __assert_rtn(const char *function, const char *file, int line, - const char *assertion) /* __nothrow */ -#ifdef __dead2 - __dead2 -#else - MDBX_NORETURN -#endif /* __dead2 */ -#ifdef __disable_tail_calls - __disable_tail_calls -#endif /* __disable_tail_calls */ - ; + if (!got_machineid) { + /* again, Windows is madness */ + static const char HKLM_WindowsNT[] = "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion"; + static const char HKLM_WindowsNT_DPK[] = "SOFTWARE\\Microsoft\\Windows " + "NT\\CurrentVersion\\DefaultProductKey"; + static const char HKLM_WindowsNT_DPK2[] = "SOFTWARE\\Microsoft\\Windows " + "NT\\CurrentVersion\\DefaultProductKey2"; -#define __assert_fail(assertion, file, line, function) \ - __assert_rtn(function, file, line, assertion) -#elif defined(__sun) || defined(__SVR4) || defined(__svr4__) -__extern_C void __assert_c99(const char *assection, const char *file, int line, - const char *function) MDBX_NORETURN; -#define __assert_fail(assertion, file, line, function) \ - __assert_c99(assertion, file, line, function) -#elif defined(__OpenBSD__) -__extern_C __dead void __assert2(const char *file, int line, - const char *function, - const char *assertion) /* __nothrow */; -#define __assert_fail(assertion, file, line, function) \ - __assert2(file, line, function, assertion) -#elif defined(__NetBSD__) -__extern_C __dead void __assert13(const char *file, int line, - const char *function, - const char *assertion) /* __nothrow */; -#define __assert_fail(assertion, file, line, function) \ - __assert13(file, line, function, assertion) -#elif defined(__FreeBSD__) || defined(__BSD__) || defined(__bsdi__) || \ - defined(__DragonFly__) -__extern_C void __assert(const char *function, const char *file, int line, - const char *assertion) /* __nothrow */ -#ifdef __dead2 - __dead2 -#else - MDBX_NORETURN -#endif /* __dead2 */ -#ifdef __disable_tail_calls - __disable_tail_calls -#endif /* __disable_tail_calls */ - ; -#define __assert_fail(assertion, file, line, function) \ - __assert(function, file, line, assertion) + len = sizeof(buf); + if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_WindowsNT, "DigitalProductId", &buf.DigitalProductId, &len) == + ERROR_SUCCESS && + len > 42 && len < sizeof(buf)) { + bootid_collect(&uuid, &buf.DigitalProductId, len); + got_machineid = true; + } + len = sizeof(buf); + if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_WindowsNT_DPK, "DigitalProductId", &buf.DigitalProductId, &len) == + ERROR_SUCCESS && + len > 42 && len < sizeof(buf)) { + bootid_collect(&uuid, &buf.DigitalProductId, len); + got_machineid = true; + } + len = sizeof(buf); + if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_WindowsNT_DPK2, "DigitalProductId", &buf.DigitalProductId, &len) == + ERROR_SUCCESS && + len > 42 && len < sizeof(buf)) { + bootid_collect(&uuid, &buf.DigitalProductId, len); + got_machineid = true; + } + } -#endif /* __assert_fail */ + static const char HKLM_PrefetcherParams[] = "SYSTEM\\CurrentControlSet\\Control\\Session Manager\\Memory " + "Management\\PrefetchParameters"; + len = sizeof(buf); + if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_PrefetcherParams, "BootId", &buf.BootId, &len) == ERROR_SUCCESS && + len > 1 && len < sizeof(buf)) { + bootid_collect(&uuid, &buf.BootId, len); + got_bootseq = true; + } -__cold void mdbx_assert_fail(const MDBX_env *env, const char *msg, - const char *func, unsigned line) { -#if MDBX_DEBUG - if (env && env->me_assert_func) - env->me_assert_func(env, msg, func, line); -#else - (void)env; - assert_fail(msg, func, line); -} + len = sizeof(buf); + if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_PrefetcherParams, "BaseTime", &buf.BaseTime, &len) == ERROR_SUCCESS && + len >= sizeof(buf.BaseTime) && buf.BaseTime) { + bootid_collect(&uuid, &buf.BaseTime, len); + got_boottime = true; + } -MDBX_NORETURN __cold void assert_fail(const char *msg, const char *func, - unsigned line) { -#endif /* MDBX_DEBUG */ + /* BootTime from SYSTEM_TIMEOFDAY_INFORMATION */ + NTSTATUS status = NtQuerySystemInformation(0x03 /* SystemTmeOfDayInformation */, &buf.SysTimeOfDayInfo, + sizeof(buf.SysTimeOfDayInfo), &len); + if (NT_SUCCESS(status) && + len >= offsetof(union buf, SysTimeOfDayInfoHacked.BootTimeBias) + + sizeof(buf.SysTimeOfDayInfoHacked.BootTimeBias) && + buf.SysTimeOfDayInfoHacked.BootTime.QuadPart) { + const uint64_t UnbiasedBootTime = + buf.SysTimeOfDayInfoHacked.BootTime.QuadPart - buf.SysTimeOfDayInfoHacked.BootTimeBias; + if (UnbiasedBootTime) { + bootid_collect(&uuid, &UnbiasedBootTime, sizeof(UnbiasedBootTime)); + got_boottime = true; + } + } - if (debug_logger) - debug_log(MDBX_LOG_FATAL, func, line, "assert: %s\n", msg); - else { -#if defined(_WIN32) || defined(_WIN64) - char *message = nullptr; - const int num = osal_asprintf(&message, "\r\nMDBX-ASSERTION: %s, %s:%u", - msg, func ? func : "unknown", line); - if (num < 1 || !message) - message = ""; - OutputDebugStringA(message); -#else - __assert_fail(msg, "mdbx", line, func); -#endif + if (!got_boottime) { + uint64_t boottime = windows_bootime(); + if (boottime) { + bootid_collect(&uuid, &boottime, sizeof(boottime)); + got_boottime = true; + } + } } +#endif /* Windows */ - while (1) { -#if defined(_WIN32) || defined(_WIN64) -#if !MDBX_WITHOUT_MSVC_CRT && defined(_DEBUG) - _CrtDbgReport(_CRT_ASSERT, func ? func : "unknown", line, "libmdbx", - "assertion failed: %s", msg); -#else - if (IsDebuggerPresent()) - DebugBreak(); -#endif - FatalExit(STATUS_ASSERTION_FAILURE); -#else - abort(); +#if defined(CTL_HW) && defined(HW_UUID) + if (!got_machineid) { + static const int mib[] = {CTL_HW, HW_UUID}; + char buf[42]; + size_t len = sizeof(buf); + if (sysctl( +#ifdef SYSCTL_LEGACY_NONCONST_MIB + (int *) #endif + mib, + ARRAY_LENGTH(mib), &buf, &len, nullptr, 0) == 0) + got_machineid = bootid_parse_uuid(&uuid, buf, len); } -} - -__cold void mdbx_panic(const char *fmt, ...) { - va_list ap; - va_start(ap, fmt); - - char *message = nullptr; - const int num = osal_vasprintf(&message, fmt, ap); - va_end(ap); - const char *const const_message = - unlikely(num < 1 || !message) - ? "" - : message; - - if (debug_logger) - debug_log(MDBX_LOG_FATAL, "panic", 0, "%s", const_message); +#endif /* CTL_HW && HW_UUID */ - while (1) { -#if defined(_WIN32) || defined(_WIN64) -#if !MDBX_WITHOUT_MSVC_CRT && defined(_DEBUG) - _CrtDbgReport(_CRT_ASSERT, "mdbx.c", 0, "libmdbx", "panic: %s", - const_message); -#else - OutputDebugStringA("\r\nMDBX-PANIC: "); - OutputDebugStringA(const_message); - if (IsDebuggerPresent()) - DebugBreak(); -#endif - FatalExit(ERROR_UNHANDLED_ERROR); -#else - __assert_fail(const_message, "mdbx", 0, "panic"); - abort(); +#if defined(CTL_KERN) && defined(KERN_HOSTUUID) + if (!got_machineid) { + static const int mib[] = {CTL_KERN, KERN_HOSTUUID}; + char buf[42]; + size_t len = sizeof(buf); + if (sysctl( +#ifdef SYSCTL_LEGACY_NONCONST_MIB + (int *) #endif + mib, + ARRAY_LENGTH(mib), &buf, &len, nullptr, 0) == 0) + got_machineid = bootid_parse_uuid(&uuid, buf, len); } -} - -/*----------------------------------------------------------------------------*/ - -#ifndef osal_vasprintf -MDBX_INTERNAL_FUNC int osal_vasprintf(char **strp, const char *fmt, - va_list ap) { - va_list ones; - va_copy(ones, ap); - const int needed = vsnprintf(nullptr, 0, fmt, ones); - va_end(ones); +#endif /* CTL_KERN && KERN_HOSTUUID */ - if (unlikely(needed < 0 || needed >= INT_MAX)) { - *strp = nullptr; - return needed; +#if defined(__NetBSD__) + if (!got_machineid) { + char buf[42]; + size_t len = sizeof(buf); + if (sysctlbyname("machdep.dmi.system-uuid", buf, &len, nullptr, 0) == 0) + got_machineid = bootid_parse_uuid(&uuid, buf, len); } +#endif /* __NetBSD__ */ - *strp = osal_malloc(needed + (size_t)1); - if (unlikely(*strp == nullptr)) { -#if defined(_WIN32) || defined(_WIN64) - SetLastError(MDBX_ENOMEM); -#else - errno = MDBX_ENOMEM; -#endif - return -1; +#if !(defined(_WIN32) || defined(_WIN64)) + if (!got_machineid) { + int fd = open("/etc/machine-id", O_RDONLY); + if (fd == -1) + fd = open("/var/lib/dbus/machine-id", O_RDONLY); + if (fd != -1) { + char buf[42]; + const ssize_t len = read(fd, buf, sizeof(buf)); + const int err = close(fd); + assert(err == 0); + (void)err; + if (len > 0) + got_machineid = bootid_parse_uuid(&uuid, buf, len); + } } +#endif /* !Windows */ - const int actual = vsnprintf(*strp, needed + (size_t)1, fmt, ap); - assert(actual == needed); - if (unlikely(actual < 0)) { - osal_free(*strp); - *strp = nullptr; +#if _XOPEN_SOURCE_EXTENDED + if (!got_machineid) { + const long hostid = gethostid(); + if (hostid != 0 && hostid != -1) { + bootid_collect(&uuid, &hostid, sizeof(hostid)); + got_machineid = true; + } } - return actual; -} -#endif /* osal_vasprintf */ +#endif /* _XOPEN_SOURCE_EXTENDED */ -#ifndef osal_asprintf -MDBX_INTERNAL_FUNC int osal_asprintf(char **strp, const char *fmt, ...) { - va_list ap; - va_start(ap, fmt); - const int rc = osal_vasprintf(strp, fmt, ap); - va_end(ap); - return rc; -} -#endif /* osal_asprintf */ + if (!got_machineid) { + lack: + uuid.x = uuid.y = 0; + return uuid; + } -#ifndef osal_memalign_alloc -MDBX_INTERNAL_FUNC int osal_memalign_alloc(size_t alignment, size_t bytes, - void **result) { - assert(is_powerof2(alignment) && alignment >= sizeof(void *)); -#if defined(_WIN32) || defined(_WIN64) - (void)alignment; - *result = VirtualAlloc(NULL, bytes, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE); - return *result ? MDBX_SUCCESS : MDBX_ENOMEM /* ERROR_OUTOFMEMORY */; -#elif defined(_ISOC11_SOURCE) - *result = aligned_alloc(alignment, ceil_powerof2(bytes, alignment)); - return *result ? MDBX_SUCCESS : errno; -#elif _POSIX_VERSION >= 200112L && \ - (!defined(__ANDROID_API__) || __ANDROID_API__ >= 17) - *result = nullptr; - return posix_memalign(result, alignment, bytes); -#elif __GLIBC_PREREQ(2, 16) || __STDC_VERSION__ >= 201112L - *result = memalign(alignment, bytes); - return *result ? MDBX_SUCCESS : errno; -#else -#error FIXME -#endif -} -#endif /* osal_memalign_alloc */ + /*--------------------------------------------------------------------------*/ -#ifndef osal_memalign_free -MDBX_INTERNAL_FUNC void osal_memalign_free(void *ptr) { -#if defined(_WIN32) || defined(_WIN64) - VirtualFree(ptr, 0, MEM_RELEASE); -#else - osal_free(ptr); +#if defined(CTL_KERN) && defined(KERN_BOOTTIME) + if (!got_boottime) { + static const int mib[] = {CTL_KERN, KERN_BOOTTIME}; + struct timeval boottime; + size_t len = sizeof(boottime); + if (sysctl( +#ifdef SYSCTL_LEGACY_NONCONST_MIB + (int *) #endif -} -#endif /* osal_memalign_free */ - -#ifndef osal_strdup -char *osal_strdup(const char *str) { - if (!str) - return NULL; - size_t bytes = strlen(str) + 1; - char *dup = osal_malloc(bytes); - if (dup) - memcpy(dup, str, bytes); - return dup; -} -#endif /* osal_strdup */ - -/*----------------------------------------------------------------------------*/ + mib, + ARRAY_LENGTH(mib), &boottime, &len, nullptr, 0) == 0 && + len == sizeof(boottime) && boottime.tv_sec) { + bootid_collect(&uuid, &boottime, len); + got_boottime = true; + } + } +#endif /* CTL_KERN && KERN_BOOTTIME */ -MDBX_INTERNAL_FUNC int osal_condpair_init(osal_condpair_t *condpair) { - int rc; - memset(condpair, 0, sizeof(osal_condpair_t)); -#if defined(_WIN32) || defined(_WIN64) - if ((condpair->mutex = CreateMutexW(NULL, FALSE, NULL)) == NULL) { - rc = (int)GetLastError(); - goto bailout_mutex; +#if defined(__sun) || defined(__SVR4) || defined(__svr4__) + if (!got_boottime) { + kstat_ctl_t *kc = kstat_open(); + if (kc) { + kstat_t *kp = kstat_lookup(kc, "unix", 0, "system_misc"); + if (kp && kstat_read(kc, kp, 0) != -1) { + kstat_named_t *kn = (kstat_named_t *)kstat_data_lookup(kp, "boot_time"); + if (kn) { + switch (kn->data_type) { + case KSTAT_DATA_INT32: + case KSTAT_DATA_UINT32: + bootid_collect(&uuid, &kn->value, sizeof(int32_t)); + got_boottime = true; + case KSTAT_DATA_INT64: + case KSTAT_DATA_UINT64: + bootid_collect(&uuid, &kn->value, sizeof(int64_t)); + got_boottime = true; + } + } + } + kstat_close(kc); + } } - if ((condpair->event[0] = CreateEventW(NULL, FALSE, FALSE, NULL)) == NULL) { - rc = (int)GetLastError(); - goto bailout_event; +#endif /* SunOS / Solaris */ + +#if _XOPEN_SOURCE_EXTENDED && defined(BOOT_TIME) + if (!got_boottime) { + setutxent(); + const struct utmpx id = {.ut_type = BOOT_TIME}; + const struct utmpx *entry = getutxid(&id); + if (entry) { + bootid_collect(&uuid, entry, sizeof(*entry)); + got_boottime = true; + while (unlikely((entry = getutxid(&id)) != nullptr)) { + /* have multiple reboot records, assuming we can distinguish next + * bootsession even if RTC is wrong or absent */ + bootid_collect(&uuid, entry, sizeof(*entry)); + got_bootseq = true; + } + } + endutxent(); } - if ((condpair->event[1] = CreateEventW(NULL, FALSE, FALSE, NULL)) != NULL) - return MDBX_SUCCESS; - - rc = (int)GetLastError(); - (void)CloseHandle(condpair->event[0]); -bailout_event: - (void)CloseHandle(condpair->mutex); -#else - rc = pthread_mutex_init(&condpair->mutex, NULL); - if (unlikely(rc != 0)) - goto bailout_mutex; - rc = pthread_cond_init(&condpair->cond[0], NULL); - if (unlikely(rc != 0)) - goto bailout_cond; - rc = pthread_cond_init(&condpair->cond[1], NULL); - if (likely(rc == 0)) - return MDBX_SUCCESS; - - (void)pthread_cond_destroy(&condpair->cond[0]); -bailout_cond: - (void)pthread_mutex_destroy(&condpair->mutex); -#endif -bailout_mutex: - memset(condpair, 0, sizeof(osal_condpair_t)); - return rc; -} +#endif /* _XOPEN_SOURCE_EXTENDED && BOOT_TIME */ -MDBX_INTERNAL_FUNC int osal_condpair_destroy(osal_condpair_t *condpair) { -#if defined(_WIN32) || defined(_WIN64) - int rc = CloseHandle(condpair->mutex) ? MDBX_SUCCESS : (int)GetLastError(); - rc = CloseHandle(condpair->event[0]) ? rc : (int)GetLastError(); - rc = CloseHandle(condpair->event[1]) ? rc : (int)GetLastError(); -#else - int err, rc = pthread_mutex_destroy(&condpair->mutex); - rc = (err = pthread_cond_destroy(&condpair->cond[0])) ? err : rc; - rc = (err = pthread_cond_destroy(&condpair->cond[1])) ? err : rc; -#endif - memset(condpair, 0, sizeof(osal_condpair_t)); - return rc; -} + if (!got_bootseq) { + if (!got_boottime || !MDBX_TRUST_RTC) + goto lack; -MDBX_INTERNAL_FUNC int osal_condpair_lock(osal_condpair_t *condpair) { #if defined(_WIN32) || defined(_WIN64) - DWORD code = WaitForSingleObject(condpair->mutex, INFINITE); - return waitstatus2errcode(code); + FILETIME now; + GetSystemTimeAsFileTime(&now); + if (0x1CCCCCC > now.dwHighDateTime) #else - return osal_pthread_mutex_lock(&condpair->mutex); + struct timespec mono, real; + if (clock_gettime(CLOCK_MONOTONIC, &mono) || clock_gettime(CLOCK_REALTIME, &real) || + /* wrong time, RTC is mad or absent */ + 1555555555l > real.tv_sec || + /* seems no adjustment by RTC/NTP, i.e. a fake time */ + real.tv_sec < mono.tv_sec || 1234567890l > real.tv_sec - mono.tv_sec || (real.tv_sec - mono.tv_sec) % 900u == 0) #endif -} + goto lack; + } -MDBX_INTERNAL_FUNC int osal_condpair_unlock(osal_condpair_t *condpair) { -#if defined(_WIN32) || defined(_WIN64) - return ReleaseMutex(condpair->mutex) ? MDBX_SUCCESS : (int)GetLastError(); -#else - return pthread_mutex_unlock(&condpair->mutex); -#endif + return uuid; } -MDBX_INTERNAL_FUNC int osal_condpair_signal(osal_condpair_t *condpair, - bool part) { -#if defined(_WIN32) || defined(_WIN64) - return SetEvent(condpair->event[part]) ? MDBX_SUCCESS : (int)GetLastError(); -#else - return pthread_cond_signal(&condpair->cond[part]); -#endif -} +__cold int mdbx_get_sysraminfo(intptr_t *page_size, intptr_t *total_pages, intptr_t *avail_pages) { + if (!page_size && !total_pages && !avail_pages) + return LOG_IFERR(MDBX_EINVAL); + if (total_pages) + *total_pages = -1; + if (avail_pages) + *avail_pages = -1; -MDBX_INTERNAL_FUNC int osal_condpair_wait(osal_condpair_t *condpair, - bool part) { -#if defined(_WIN32) || defined(_WIN64) - DWORD code = SignalObjectAndWait(condpair->mutex, condpair->event[part], - INFINITE, FALSE); - if (code == WAIT_OBJECT_0) { - code = WaitForSingleObject(condpair->mutex, INFINITE); - if (code == WAIT_OBJECT_0) - return MDBX_SUCCESS; - } - return waitstatus2errcode(code); -#else - return pthread_cond_wait(&condpair->cond[part], &condpair->mutex); -#endif -} + const intptr_t pagesize = globals.sys_pagesize; + if (page_size) + *page_size = pagesize; + if (unlikely(pagesize < MDBX_MIN_PAGESIZE || !is_powerof2(pagesize))) + return LOG_IFERR(MDBX_INCOMPATIBLE); -/*----------------------------------------------------------------------------*/ + MDBX_MAYBE_UNUSED const int log2page = log2n_powerof2(pagesize); + assert(pagesize == (INT64_C(1) << log2page)); + (void)log2page; -MDBX_INTERNAL_FUNC int osal_fastmutex_init(osal_fastmutex_t *fastmutex) { #if defined(_WIN32) || defined(_WIN64) - InitializeCriticalSection(fastmutex); - return MDBX_SUCCESS; -#else - return pthread_mutex_init(fastmutex, NULL); + MEMORYSTATUSEX info; + memset(&info, 0, sizeof(info)); + info.dwLength = sizeof(info); + if (!GlobalMemoryStatusEx(&info)) + return LOG_IFERR((int)GetLastError()); #endif -} -MDBX_INTERNAL_FUNC int osal_fastmutex_destroy(osal_fastmutex_t *fastmutex) { + if (total_pages) { #if defined(_WIN32) || defined(_WIN64) - DeleteCriticalSection(fastmutex); - return MDBX_SUCCESS; + const intptr_t total_ram_pages = (intptr_t)(info.ullTotalPhys >> log2page); +#elif defined(_SC_PHYS_PAGES) + const intptr_t total_ram_pages = sysconf(_SC_PHYS_PAGES); + if (total_ram_pages == -1) + return LOG_IFERR(errno); +#elif defined(_SC_AIX_REALMEM) + const intptr_t total_ram_Kb = sysconf(_SC_AIX_REALMEM); + if (total_ram_Kb == -1) + return LOG_IFERR(errno); + const intptr_t total_ram_pages = (total_ram_Kb << 10) >> log2page; +#elif defined(HW_USERMEM) || defined(HW_PHYSMEM64) || defined(HW_MEMSIZE) || defined(HW_PHYSMEM) + size_t ram, len = sizeof(ram); + static const int mib[] = {CTL_HW, +#if defined(HW_USERMEM) + HW_USERMEM +#elif defined(HW_PHYSMEM64) + HW_PHYSMEM64 +#elif defined(HW_MEMSIZE) + HW_MEMSIZE #else - return pthread_mutex_destroy(fastmutex); + HW_PHYSMEM #endif -} - -MDBX_INTERNAL_FUNC int osal_fastmutex_acquire(osal_fastmutex_t *fastmutex) { -#if defined(_WIN32) || defined(_WIN64) - __try { - EnterCriticalSection(fastmutex); - } __except ( - (GetExceptionCode() == - 0xC0000194 /* STATUS_POSSIBLE_DEADLOCK / EXCEPTION_POSSIBLE_DEADLOCK */) - ? EXCEPTION_EXECUTE_HANDLER - : EXCEPTION_CONTINUE_SEARCH) { - return ERROR_POSSIBLE_DEADLOCK; - } - return MDBX_SUCCESS; -#else - return osal_pthread_mutex_lock(fastmutex); + }; + if (sysctl( +#ifdef SYSCTL_LEGACY_NONCONST_MIB + (int *) #endif -} - -MDBX_INTERNAL_FUNC int osal_fastmutex_release(osal_fastmutex_t *fastmutex) { -#if defined(_WIN32) || defined(_WIN64) - LeaveCriticalSection(fastmutex); - return MDBX_SUCCESS; + mib, + ARRAY_LENGTH(mib), &ram, &len, nullptr, 0) != 0) + return LOG_IFERR(errno); + if (len != sizeof(ram)) + return LOG_IFERR(MDBX_ENOSYS); + const intptr_t total_ram_pages = (intptr_t)(ram >> log2page); #else - return pthread_mutex_unlock(fastmutex); +#error "FIXME: Get User-accessible or physical RAM" #endif -} - -/*----------------------------------------------------------------------------*/ - -#if defined(_WIN32) || defined(_WIN64) - -MDBX_INTERNAL_FUNC int osal_mb2w(const char *const src, wchar_t **const pdst) { - const size_t dst_wlen = MultiByteToWideChar( - CP_THREAD_ACP, MB_ERR_INVALID_CHARS, src, -1, nullptr, 0); - wchar_t *dst = *pdst; - int rc = ERROR_INVALID_NAME; - if (unlikely(dst_wlen < 2 || dst_wlen > /* MAX_PATH */ INT16_MAX)) - goto bailout; - - dst = osal_realloc(dst, dst_wlen * sizeof(wchar_t)); - rc = MDBX_ENOMEM; - if (unlikely(!dst)) - goto bailout; - - *pdst = dst; - if (likely(dst_wlen == (size_t)MultiByteToWideChar(CP_THREAD_ACP, - MB_ERR_INVALID_CHARS, src, - -1, dst, (int)dst_wlen))) - return MDBX_SUCCESS; - - rc = ERROR_INVALID_NAME; -bailout: - if (*pdst) { - osal_free(*pdst); - *pdst = nullptr; + *total_pages = total_ram_pages; + if (total_ram_pages < 1) + return LOG_IFERR(MDBX_ENOSYS); } - return rc; -} - -#endif /* Windows */ - -/*----------------------------------------------------------------------------*/ + if (avail_pages) { #if defined(_WIN32) || defined(_WIN64) -#define ior_alignment_mask (ior->pagesize - 1) -#define ior_WriteFile_flag 1 -#define OSAL_IOV_MAX (4096 / sizeof(ior_sgv_element)) - -static void ior_put_event(osal_ioring_t *ior, HANDLE event) { - assert(event && event != INVALID_HANDLE_VALUE && event != ior); - assert(ior->event_stack < ior->allocated); - ior->event_pool[ior->event_stack] = event; - ior->event_stack += 1; -} - -static HANDLE ior_get_event(osal_ioring_t *ior) { - assert(ior->event_stack <= ior->allocated); - if (ior->event_stack > 0) { - ior->event_stack -= 1; - assert(ior->event_pool[ior->event_stack] != 0); - return ior->event_pool[ior->event_stack]; - } - return CreateEventW(nullptr, true, false, nullptr); -} - -static void WINAPI ior_wocr(DWORD err, DWORD bytes, OVERLAPPED *ov) { - osal_ioring_t *ior = ov->hEvent; - ov->Internal = err; - ov->InternalHigh = bytes; - if (++ior->async_completed >= ior->async_waiting) - SetEvent(ior->async_done); -} - -#elif MDBX_HAVE_PWRITEV -#if defined(_SC_IOV_MAX) -static size_t osal_iov_max; -#define OSAL_IOV_MAX osal_iov_max -#else -#define OSAL_IOV_MAX IOV_MAX + const intptr_t avail_ram_pages = (intptr_t)(info.ullAvailPhys >> log2page); +#elif defined(_SC_AVPHYS_PAGES) + const intptr_t avail_ram_pages = sysconf(_SC_AVPHYS_PAGES); + if (avail_ram_pages == -1) + return LOG_IFERR(errno); +#elif defined(__MACH__) + mach_msg_type_number_t count = HOST_VM_INFO_COUNT; + vm_statistics_data_t vmstat; + mach_port_t mport = mach_host_self(); + kern_return_t kerr = host_statistics(mach_host_self(), HOST_VM_INFO, (host_info_t)&vmstat, &count); + mach_port_deallocate(mach_task_self(), mport); + if (unlikely(kerr != KERN_SUCCESS)) + return LOG_IFERR(MDBX_ENOSYS); + const intptr_t avail_ram_pages = vmstat.free_count; +#elif defined(VM_TOTAL) || defined(VM_METER) + struct vmtotal info; + size_t len = sizeof(info); + static const int mib[] = {CTL_VM, +#if defined(VM_TOTAL) + VM_TOTAL +#elif defined(VM_METER) + VM_METER +#endif + }; + if (sysctl( +#ifdef SYSCTL_LEGACY_NONCONST_MIB + (int *) #endif + mib, + ARRAY_LENGTH(mib), &info, &len, nullptr, 0) != 0) + return LOG_IFERR(errno); + if (len != sizeof(info)) + return LOG_IFERR(MDBX_ENOSYS); + const intptr_t avail_ram_pages = info.t_free; #else -#undef OSAL_IOV_MAX -#endif /* OSAL_IOV_MAX */ - -MDBX_INTERNAL_FUNC int osal_ioring_create(osal_ioring_t *ior -#if defined(_WIN32) || defined(_WIN64) - , - bool enable_direct, - mdbx_filehandle_t overlapped_fd -#endif /* Windows */ -) { - memset(ior, 0, sizeof(osal_ioring_t)); - -#if defined(_WIN32) || defined(_WIN64) - ior->overlapped_fd = overlapped_fd; - ior->direct = enable_direct && overlapped_fd; - const unsigned pagesize = (unsigned)osal_syspagesize(); - ior->pagesize = pagesize; - ior->pagesize_ln2 = (uint8_t)log2n_powerof2(pagesize); - ior->async_done = ior_get_event(ior); - if (!ior->async_done) - return GetLastError(); -#endif /* !Windows */ - -#if MDBX_HAVE_PWRITEV && defined(_SC_IOV_MAX) - assert(osal_iov_max > 0); -#endif /* MDBX_HAVE_PWRITEV && _SC_IOV_MAX */ +#error "FIXME: Get Available RAM" +#endif + *avail_pages = avail_ram_pages; + if (avail_ram_pages < 1) + return LOG_IFERR(MDBX_ENOSYS); + } - ior->boundary = ptr_disp(ior->pool, ior->allocated); return MDBX_SUCCESS; } -static __inline size_t ior_offset(const ior_item_t *item) { +/*----------------------------------------------------------------------------*/ + +#ifdef __FreeBSD__ +#include +#endif /* FreeBSD */ + +#ifdef __IPHONE_OS_VERSION_MIN_REQUIRED +#include +#elif __GLIBC_PREREQ(2, 25) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__BSD__) || defined(__bsdi__) || \ + defined(__DragonFly__) || defined(__APPLE__) || __has_include() +#include +#endif /* sys/random.h */ + +#if defined(_WIN32) || defined(_WIN64) +#include +#endif /* Windows */ + +MDBX_INTERNAL bin128_t osal_guid(const MDBX_env *env) { + struct { + uint64_t begin, end, cputime; + uintptr_t thread, pid; + const void *x, *y; + bin128_t (*z)(const MDBX_env *env); + } salt; + + salt.begin = osal_monotime(); + bin128_t uuid = {{0, 0}}; + +#if defined(__linux__) || defined(__gnu_linux__) + if (proc_read_uuid("/proc/sys/kernel/random/uuid", &uuid) && check_uuid(uuid)) + return uuid; +#endif /* Linux */ + +#ifdef __FreeBSD__ + STATIC_ASSERT(sizeof(uuid) == sizeof(struct uuid)); + if (uuidgen((struct uuid *)&uuid, 1) == 0 && check_uuid(uuid)) + return uuid; +#endif /* FreeBSD */ + #if defined(_WIN32) || defined(_WIN64) - return item->ov.Offset | (size_t)((sizeof(size_t) > sizeof(item->ov.Offset)) - ? (uint64_t)item->ov.OffsetHigh << 32 - : 0); + if (imports.CoCreateGuid && imports.CoCreateGuid(&uuid) == 0 && check_uuid(uuid)) + return uuid; + + HCRYPTPROV hCryptProv = 0; + if (CryptAcquireContextW(&hCryptProv, nullptr, nullptr, PROV_RSA_FULL, CRYPT_VERIFYCONTEXT | CRYPT_SILENT)) { + const BOOL ok = CryptGenRandom(hCryptProv, sizeof(uuid), (unsigned char *)&uuid); + CryptReleaseContext(hCryptProv, 0); + if (ok && check_uuid(uuid)) + return uuid; + } +#elif defined(__IPHONE_OS_VERSION_MIN_REQUIRED) && defined(__IPHONE_8_0) +#if __IPHONE_OS_VERSION_MIN_REQUIRED >= __IPHONE_8_0 + if (CCRandomGenerateBytes(&uuid, sizeof(uuid)) == kCCSuccess && check_uuid(uuid)) + return uuid; +#endif /* iOS >= 8.x */ #else - return item->offset; + const int fd = open("/dev/urandom", O_RDONLY); + if (fd != -1) { + const ssize_t len = read(fd, &uuid, sizeof(uuid)); + const int err = close(fd); + assert(err == 0); + (void)err; + if (len == sizeof(uuid) && check_uuid(uuid)) + return uuid; + } +#if (__GLIBC_PREREQ(2, 25) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__BSD__) || defined(__bsdi__) || \ + defined(__DragonFly__)) && \ + !defined(__APPLE__) && !defined(__ANDROID_API__) + if (getrandom(&uuid, sizeof(uuid), 0) == sizeof(uuid) && check_uuid(uuid)) + return uuid; +#elif defined(__OpenBSD__) || (defined(__sun) && defined(__SVR4)) || \ + (defined(__MAC_OS_X_VERSION_MIN_REQUIRED) && __MAC_OS_X_VERSION_MIN_REQUIRED >= 101200) + if (getentropy(&uuid, sizeof(uuid)) == 0 && check_uuid(uuid)) + return uuid; +#endif /* getrandom() / getentropy() */ #endif /* !Windows */ -} -static __inline ior_item_t *ior_next(ior_item_t *item, size_t sgvcnt) { -#if defined(ior_sgv_element) - assert(sgvcnt > 0); - return (ior_item_t *)ptr_disp(item, sizeof(ior_item_t) - - sizeof(ior_sgv_element) + - sizeof(ior_sgv_element) * sgvcnt); -#else - assert(sgvcnt == 1); - (void)sgvcnt; - return item + 1; -#endif + uuid = globals.bootid; + bootid_collect(&uuid, env, sizeof(*env)); + salt.thread = osal_thread_self(); + salt.pid = osal_getpid(); + salt.x = &salt; + salt.y = env; + salt.z = &osal_guid; + do { + salt.cputime = osal_cputime(nullptr); + salt.end = osal_monotime(); + bootid_collect(&uuid, &salt, sizeof(salt)); + } while (!check_uuid(uuid)); + return uuid; } -MDBX_INTERNAL_FUNC int osal_ioring_add(osal_ioring_t *ior, const size_t offset, - void *data, const size_t bytes) { - assert(bytes && data); - assert(bytes % MIN_PAGESIZE == 0 && bytes <= MAX_WRITE); - assert(offset % MIN_PAGESIZE == 0 && offset + (uint64_t)bytes <= MAX_MAPSIZE); +/*--------------------------------------------------------------------------*/ -#if defined(_WIN32) || defined(_WIN64) - const unsigned segments = (unsigned)(bytes >> ior->pagesize_ln2); - const bool use_gather = - ior->direct && ior->overlapped_fd && ior->slots_left >= segments; -#endif /* Windows */ +void osal_ctor(void) { +#if MDBX_HAVE_PWRITEV && defined(_SC_IOV_MAX) + osal_iov_max = sysconf(_SC_IOV_MAX); + if (RUNNING_ON_VALGRIND && osal_iov_max > 64) + /* чтобы не описывать все 1024 исключения в valgrind_suppress.txt */ + osal_iov_max = 64; +#endif /* MDBX_HAVE_PWRITEV && _SC_IOV_MAX */ - ior_item_t *item = ior->pool; - if (likely(ior->last)) { - item = ior->last; - if (unlikely(ior_offset(item) + ior_last_bytes(ior, item) == offset) && - likely(ior_last_bytes(ior, item) + bytes <= MAX_WRITE)) { #if defined(_WIN32) || defined(_WIN64) - if (use_gather && - ((bytes | (uintptr_t)data | ior->last_bytes | - (uintptr_t)(uint64_t)item->sgv[0].Buffer) & - ior_alignment_mask) == 0 && - ior->last_sgvcnt + (size_t)segments < OSAL_IOV_MAX) { - assert(ior->overlapped_fd); - assert((item->single.iov_len & ior_WriteFile_flag) == 0); - assert(item->sgv[ior->last_sgvcnt].Buffer == 0); - ior->last_bytes += bytes; - size_t i = 0; - do { - item->sgv[ior->last_sgvcnt + i].Buffer = PtrToPtr64(data); - data = ptr_disp(data, ior->pagesize); - } while (++i < segments); - ior->slots_left -= segments; - item->sgv[ior->last_sgvcnt += segments].Buffer = 0; - assert((item->single.iov_len & ior_WriteFile_flag) == 0); - return MDBX_SUCCESS; - } - const void *end = ptr_disp(item->single.iov_base, - item->single.iov_len - ior_WriteFile_flag); - if (unlikely(end == data)) { - assert((item->single.iov_len & ior_WriteFile_flag) != 0); - item->single.iov_len += bytes; - return MDBX_SUCCESS; - } -#elif MDBX_HAVE_PWRITEV - assert((int)item->sgvcnt > 0); - const void *end = ptr_disp(item->sgv[item->sgvcnt - 1].iov_base, - item->sgv[item->sgvcnt - 1].iov_len); - if (unlikely(end == data)) { - item->sgv[item->sgvcnt - 1].iov_len += bytes; - ior->last_bytes += bytes; - return MDBX_SUCCESS; - } - if (likely(item->sgvcnt < OSAL_IOV_MAX)) { - if (unlikely(ior->slots_left < 1)) - return MDBX_RESULT_TRUE; - item->sgv[item->sgvcnt].iov_base = data; - item->sgv[item->sgvcnt].iov_len = bytes; - ior->last_bytes += bytes; - item->sgvcnt += 1; - ior->slots_left -= 1; - return MDBX_SUCCESS; - } + SYSTEM_INFO si; + GetSystemInfo(&si); + globals.sys_pagesize = si.dwPageSize; + globals.sys_allocation_granularity = si.dwAllocationGranularity; #else - const void *end = ptr_disp(item->single.iov_base, item->single.iov_len); - if (unlikely(end == data)) { - item->single.iov_len += bytes; - return MDBX_SUCCESS; - } + globals.sys_pagesize = sysconf(_SC_PAGE_SIZE); + globals.sys_allocation_granularity = (MDBX_WORDBITS > 32) ? 65536 : 4096; + globals.sys_allocation_granularity = (globals.sys_allocation_granularity > globals.sys_pagesize) + ? globals.sys_allocation_granularity + : globals.sys_pagesize; #endif - } - item = ior_next(item, ior_last_sgvcnt(ior, item)); - } + assert(globals.sys_pagesize > 0 && (globals.sys_pagesize & (globals.sys_pagesize - 1)) == 0); + assert(globals.sys_allocation_granularity >= globals.sys_pagesize && + globals.sys_allocation_granularity % globals.sys_pagesize == 0); + globals.sys_pagesize_ln2 = log2n_powerof2(globals.sys_pagesize); - if (unlikely(ior->slots_left < 1)) - return MDBX_RESULT_TRUE; +#if defined(__linux__) || defined(__gnu_linux__) + posix_clockid = choice_monoclock(); +#endif - unsigned slots_used = 1; #if defined(_WIN32) || defined(_WIN64) - item->ov.Internal = item->ov.InternalHigh = 0; - item->ov.Offset = (DWORD)offset; - item->ov.OffsetHigh = HIGH_DWORD(offset); - item->ov.hEvent = 0; - if (!use_gather || ((bytes | (uintptr_t)(data)) & ior_alignment_mask) != 0 || - segments > OSAL_IOV_MAX) { - /* WriteFile() */ - item->single.iov_base = data; - item->single.iov_len = bytes + ior_WriteFile_flag; - assert((item->single.iov_len & ior_WriteFile_flag) != 0); - } else { - /* WriteFileGather() */ - assert(ior->overlapped_fd); - item->sgv[0].Buffer = PtrToPtr64(data); - for (size_t i = 1; i < segments; ++i) { - data = ptr_disp(data, ior->pagesize); - item->sgv[i].Buffer = PtrToPtr64(data); + QueryPerformanceFrequency(&performance_frequency); +#elif defined(__APPLE__) || defined(__MACH__) + mach_timebase_info_data_t ti; + mach_timebase_info(&ti); + ratio_16dot16_to_monotine = UINT64_C(1000000000) * ti.denom / ti.numer; +#endif + monotime_limit = osal_16dot16_to_monotime(UINT32_MAX - 1); + + uint32_t proba = UINT32_MAX; + while (true) { + unsigned time_conversion_checkup = osal_monotime_to_16dot16(osal_16dot16_to_monotime(proba)); + unsigned one_more = (proba < UINT32_MAX) ? proba + 1 : proba; + unsigned one_less = (proba > 0) ? proba - 1 : proba; + ENSURE(nullptr, time_conversion_checkup >= one_less && time_conversion_checkup <= one_more); + if (proba == 0) + break; + proba >>= 1; + } + + globals.bootid = osal_bootid(); +} + +void osal_dtor(void) {} +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 + +__cold int MDBX_PRINTF_ARGS(2, 3) bad_page(const page_t *mp, const char *fmt, ...) { + if (LOG_ENABLED(MDBX_LOG_ERROR)) { + static const page_t *prev; + if (prev != mp) { + char buf4unknown[16]; + prev = mp; + debug_log(MDBX_LOG_ERROR, "badpage", 0, "corrupted %s-page #%u, mod-txnid %" PRIaTXN "\n", + pagetype_caption(page_type(mp), buf4unknown), mp->pgno, mp->txnid); } - item->sgv[slots_used = segments].Buffer = 0; - assert((item->single.iov_len & ior_WriteFile_flag) == 0); + + va_list args; + va_start(args, fmt); + debug_log_va(MDBX_LOG_ERROR, "badpage", 0, fmt, args); + va_end(args); } - ior->last_bytes = bytes; - ior_last_sgvcnt(ior, item) = slots_used; -#elif MDBX_HAVE_PWRITEV - item->offset = offset; - item->sgv[0].iov_base = data; - item->sgv[0].iov_len = bytes; - ior->last_bytes = bytes; - ior_last_sgvcnt(ior, item) = slots_used; -#else - item->offset = offset; - item->single.iov_base = data; - item->single.iov_len = bytes; -#endif /* !Windows */ - ior->slots_left -= slots_used; - ior->last = item; - return MDBX_SUCCESS; + return MDBX_CORRUPTED; } -MDBX_INTERNAL_FUNC void osal_ioring_walk( - osal_ioring_t *ior, iov_ctx_t *ctx, - void (*callback)(iov_ctx_t *ctx, size_t offset, void *data, size_t bytes)) { - for (ior_item_t *item = ior->pool; item <= ior->last;) { -#if defined(_WIN32) || defined(_WIN64) - size_t offset = ior_offset(item); - char *data = item->single.iov_base; - size_t bytes = item->single.iov_len - ior_WriteFile_flag; - size_t i = 1; - if (bytes & ior_WriteFile_flag) { - data = Ptr64ToPtr(item->sgv[0].Buffer); - bytes = ior->pagesize; - /* Zap: Reading invalid data from 'item->sgv' */ - MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(6385); - while (item->sgv[i].Buffer) { - if (data + ior->pagesize != item->sgv[i].Buffer) { - callback(ctx, offset, data, bytes); - offset += bytes; - data = Ptr64ToPtr(item->sgv[i].Buffer); - bytes = 0; - } - bytes += ior->pagesize; - ++i; - } +__cold void MDBX_PRINTF_ARGS(2, 3) poor_page(const page_t *mp, const char *fmt, ...) { + if (LOG_ENABLED(MDBX_LOG_NOTICE)) { + static const page_t *prev; + if (prev != mp) { + char buf4unknown[16]; + prev = mp; + debug_log(MDBX_LOG_NOTICE, "poorpage", 0, "suboptimal %s-page #%u, mod-txnid %" PRIaTXN "\n", + pagetype_caption(page_type(mp), buf4unknown), mp->pgno, mp->txnid); } - assert(bytes < MAX_WRITE); - callback(ctx, offset, data, bytes); -#elif MDBX_HAVE_PWRITEV - assert(item->sgvcnt > 0); - size_t offset = item->offset; - size_t i = 0; - do { - callback(ctx, offset, item->sgv[i].iov_base, item->sgv[i].iov_len); - offset += item->sgv[i].iov_len; - } while (++i != item->sgvcnt); -#else - const size_t i = 1; - callback(ctx, item->offset, item->single.iov_base, item->single.iov_len); -#endif - item = ior_next(item, i); + + va_list args; + va_start(args, fmt); + debug_log_va(MDBX_LOG_NOTICE, "poorpage", 0, fmt, args); + va_end(args); + } +} + +MDBX_CONST_FUNCTION static clc_t value_clc(const MDBX_cursor *mc) { + if (likely((mc->flags & z_inner) == 0)) + return mc->clc->v; + else { + clc_t stub = {.cmp = cmp_equal_or_wrong, .lmin = 0, .lmax = 0}; + return stub; + } +} + +__cold int page_check(const MDBX_cursor *const mc, const page_t *const mp) { + DKBUF; + int rc = MDBX_SUCCESS; + if (unlikely(mp->pgno < MIN_PAGENO || mp->pgno > MAX_PAGENO)) + rc = bad_page(mp, "invalid pgno (%u)\n", mp->pgno); + + MDBX_env *const env = mc->txn->env; + const ptrdiff_t offset = ptr_dist(mp, env->dxb_mmap.base); + unsigned flags_mask = P_ILL_BITS; + unsigned flags_expected = 0; + if (offset < 0 || offset > (ptrdiff_t)(pgno2bytes(env, mc->txn->geo.first_unallocated) - + ((mp->flags & P_SUBP) ? PAGEHDRSZ + 1 : env->ps))) { + /* should be dirty page without MDBX_WRITEMAP, or a subpage of. */ + flags_mask -= P_SUBP; + if ((env->flags & MDBX_WRITEMAP) != 0 || (!is_shadowed(mc->txn, mp) && !(mp->flags & P_SUBP))) + rc = bad_page(mp, "invalid page-address %p, offset %zi\n", __Wpedantic_format_voidptr(mp), offset); + } else if (offset & (env->ps - 1)) + flags_expected = P_SUBP; + + if (unlikely((mp->flags & flags_mask) != flags_expected)) + rc = bad_page(mp, "unknown/extra page-flags (have 0x%x, expect 0x%x)\n", mp->flags & flags_mask, flags_expected); + + cASSERT(mc, (mc->checking & z_dupfix) == 0 || (mc->flags & z_inner) != 0); + const uint8_t type = page_type(mp); + switch (type) { + default: + return bad_page(mp, "invalid type (%u)\n", type); + case P_LARGE: + if (unlikely(mc->flags & z_inner)) + rc = bad_page(mp, "unexpected %s-page for %s (db-flags 0x%x)\n", "large", "nested dupsort tree", mc->tree->flags); + const pgno_t npages = mp->pages; + if (unlikely(npages < 1 || npages >= MAX_PAGENO / 2)) + rc = bad_page(mp, "invalid n-pages (%u) for large-page\n", npages); + if (unlikely(mp->pgno + npages > mc->txn->geo.first_unallocated)) + rc = bad_page(mp, "end of large-page beyond (%u) allocated space (%u next-pgno)\n", mp->pgno + npages, + mc->txn->geo.first_unallocated); + return rc; //-------------------------- end of large/overflow page handling + case P_LEAF | P_SUBP: + if (unlikely(mc->tree->height != 1)) + rc = + bad_page(mp, "unexpected %s-page for %s (db-flags 0x%x)\n", "leaf-sub", "nested dupsort db", mc->tree->flags); + /* fall through */ + __fallthrough; + case P_LEAF: + if (unlikely((mc->checking & z_dupfix) != 0)) + rc = bad_page(mp, "unexpected leaf-page for dupfix subtree (db-lags 0x%x)\n", mc->tree->flags); + break; + case P_LEAF | P_DUPFIX | P_SUBP: + if (unlikely(mc->tree->height != 1)) + rc = bad_page(mp, "unexpected %s-page for %s (db-flags 0x%x)\n", "leaf2-sub", "nested dupsort db", + mc->tree->flags); + /* fall through */ + __fallthrough; + case P_LEAF | P_DUPFIX: + if (unlikely((mc->checking & z_dupfix) == 0)) + rc = bad_page(mp, "unexpected leaf2-page for non-dupfix (sub)tree (db-flags 0x%x)\n", mc->tree->flags); + break; + case P_BRANCH: + break; } -} -MDBX_INTERNAL_FUNC osal_ioring_write_result_t -osal_ioring_write(osal_ioring_t *ior, mdbx_filehandle_t fd) { - osal_ioring_write_result_t r = {MDBX_SUCCESS, 0}; + if (unlikely(mp->upper < mp->lower || (mp->lower & 1) || PAGEHDRSZ + mp->upper > env->ps)) + rc = bad_page(mp, "invalid page lower(%u)/upper(%u) with limit %zu\n", mp->lower, mp->upper, page_space(env)); -#if defined(_WIN32) || defined(_WIN64) - HANDLE *const end_wait_for = - ior->event_pool + ior->allocated + - /* был выделен один дополнительный элемент для async_done */ 1; - HANDLE *wait_for = end_wait_for; - LONG async_started = 0; - for (ior_item_t *item = ior->pool; item <= ior->last;) { - item->ov.Internal = STATUS_PENDING; - size_t i = 1, bytes = item->single.iov_len - ior_WriteFile_flag; - r.wops += 1; - if (bytes & ior_WriteFile_flag) { - assert(ior->overlapped_fd && fd == ior->overlapped_fd); - bytes = ior->pagesize; - /* Zap: Reading invalid data from 'item->sgv' */ - MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(6385); - while (item->sgv[i].Buffer) { - bytes += ior->pagesize; - ++i; + const char *const end_of_page = ptr_disp(mp, env->ps); + const size_t nkeys = page_numkeys(mp); + STATIC_ASSERT(P_BRANCH == 1); + if (unlikely(nkeys <= (uint8_t)(mp->flags & P_BRANCH))) { + if ((!(mc->flags & z_inner) || mc->tree->items) && + (!(mc->checking & z_updating) || !(is_modifable(mc->txn, mp) || (mp->flags & P_SUBP)))) + rc = bad_page(mp, "%s-page nkeys (%zu) < %u\n", is_branch(mp) ? "branch" : "leaf", nkeys, 1 + is_branch(mp)); + } + + const size_t ksize_max = keysize_max(env->ps, 0); + const size_t leaf2_ksize = mp->dupfix_ksize; + if (is_dupfix_leaf(mp)) { + if (unlikely((mc->flags & z_inner) == 0 || (mc->tree->flags & MDBX_DUPFIXED) == 0)) + rc = bad_page(mp, "unexpected leaf2-page (db-flags 0x%x)\n", mc->tree->flags); + else if (unlikely(leaf2_ksize != mc->tree->dupfix_size)) + rc = bad_page(mp, "invalid leaf2_ksize %zu\n", leaf2_ksize); + else if (unlikely(((leaf2_ksize & nkeys) ^ mp->upper) & 1)) + rc = bad_page(mp, "invalid page upper (%u) for nkeys %zu with leaf2-length %zu\n", mp->upper, nkeys, leaf2_ksize); + } else { + if (unlikely((mp->upper & 1) || PAGEHDRSZ + mp->upper + nkeys * sizeof(node_t) + nkeys - 1 > env->ps)) + rc = bad_page(mp, "invalid page upper (%u) for nkeys %zu with limit %zu\n", mp->upper, nkeys, page_space(env)); + } + + MDBX_val here, prev = {0, 0}; + clc_t v_clc = value_clc(mc); + for (size_t i = 0; i < nkeys; ++i) { + if (is_dupfix_leaf(mp)) { + const char *const key = page_dupfix_ptr(mp, i, mc->tree->dupfix_size); + if (unlikely(end_of_page < key + leaf2_ksize)) { + rc = bad_page(mp, "leaf2-item beyond (%zu) page-end\n", key + leaf2_ksize - end_of_page); + continue; } - assert(bytes < MAX_WRITE); - item->ov.hEvent = ior_get_event(ior); - if (unlikely(!item->ov.hEvent)) { - bailout_geterr: - r.err = GetLastError(); - bailout_rc: - assert(r.err != MDBX_SUCCESS); - CancelIo(fd); - return r; + + if (unlikely(leaf2_ksize != mc->clc->k.lmin)) { + if (unlikely(leaf2_ksize < mc->clc->k.lmin || leaf2_ksize > mc->clc->k.lmax)) + rc = bad_page(mp, "leaf2-item size (%zu) <> min/max length (%zu/%zu)\n", leaf2_ksize, mc->clc->k.lmin, + mc->clc->k.lmax); + else + mc->clc->k.lmin = mc->clc->k.lmax = leaf2_ksize; } - if (WriteFileGather(fd, item->sgv, (DWORD)bytes, nullptr, &item->ov)) { - assert(item->ov.Internal == 0 && - WaitForSingleObject(item->ov.hEvent, 0) == WAIT_OBJECT_0); - ior_put_event(ior, item->ov.hEvent); - item->ov.hEvent = 0; - } else { - r.err = (int)GetLastError(); - if (unlikely(r.err != ERROR_IO_PENDING)) { - ERROR("%s: fd %p, item %p (%zu), pgno %u, bytes %zu, offset %" PRId64 - ", err %d", - "WriteFileGather", fd, __Wpedantic_format_voidptr(item), - item - ior->pool, ((MDBX_page *)item->single.iov_base)->mp_pgno, - bytes, item->ov.Offset + ((uint64_t)item->ov.OffsetHigh << 32), - r.err); - goto bailout_rc; - } - assert(wait_for > ior->event_pool + ior->event_stack); - *--wait_for = item->ov.hEvent; + if ((mc->checking & z_ignord) == 0) { + here.iov_base = (void *)key; + here.iov_len = leaf2_ksize; + if (prev.iov_base && unlikely(mc->clc->k.cmp(&prev, &here) >= 0)) + rc = bad_page(mp, "leaf2-item #%zu wrong order (%s >= %s)\n", i, DKEY(&prev), DVAL(&here)); + prev = here; } - } else if (fd == ior->overlapped_fd) { - assert(bytes < MAX_WRITE); - retry: - item->ov.hEvent = ior; - if (WriteFileEx(fd, item->single.iov_base, (DWORD)bytes, &item->ov, - ior_wocr)) { - async_started += 1; - } else { - r.err = (int)GetLastError(); - switch (r.err) { - default: - ERROR("%s: fd %p, item %p (%zu), pgno %u, bytes %zu, offset %" PRId64 - ", err %d", - "WriteFileEx", fd, __Wpedantic_format_voidptr(item), - item - ior->pool, ((MDBX_page *)item->single.iov_base)->mp_pgno, - bytes, item->ov.Offset + ((uint64_t)item->ov.OffsetHigh << 32), - r.err); - goto bailout_rc; - case ERROR_NOT_FOUND: - case ERROR_USER_MAPPED_FILE: - case ERROR_LOCK_VIOLATION: - WARNING( - "%s: fd %p, item %p (%zu), pgno %u, bytes %zu, offset %" PRId64 - ", err %d", - "WriteFileEx", fd, __Wpedantic_format_voidptr(item), - item - ior->pool, ((MDBX_page *)item->single.iov_base)->mp_pgno, - bytes, item->ov.Offset + ((uint64_t)item->ov.OffsetHigh << 32), - r.err); - SleepEx(0, true); - goto retry; - case ERROR_INVALID_USER_BUFFER: - case ERROR_NOT_ENOUGH_MEMORY: - if (SleepEx(0, true) == WAIT_IO_COMPLETION) - goto retry; - goto bailout_rc; - case ERROR_IO_PENDING: - async_started += 1; + } else { + const node_t *const node = page_node(mp, i); + const char *const node_end = ptr_disp(node, NODESIZE); + if (unlikely(node_end > end_of_page)) { + rc = bad_page(mp, "node[%zu] (%zu) beyond page-end\n", i, node_end - end_of_page); + continue; + } + const size_t ksize = node_ks(node); + if (unlikely(ksize > ksize_max)) + rc = bad_page(mp, "node[%zu] too long key (%zu)\n", i, ksize); + const char *const key = node_key(node); + if (unlikely(end_of_page < key + ksize)) { + rc = bad_page(mp, "node[%zu] key (%zu) beyond page-end\n", i, key + ksize - end_of_page); + continue; + } + if ((is_leaf(mp) || i > 0)) { + if (unlikely(ksize < mc->clc->k.lmin || ksize > mc->clc->k.lmax)) + rc = bad_page(mp, "node[%zu] key size (%zu) <> min/max key-length (%zu/%zu)\n", i, ksize, mc->clc->k.lmin, + mc->clc->k.lmax); + if ((mc->checking & z_ignord) == 0) { + here.iov_base = (void *)key; + here.iov_len = ksize; + if (prev.iov_base && unlikely(mc->clc->k.cmp(&prev, &here) >= 0)) + rc = bad_page(mp, "node[%zu] key wrong order (%s >= %s)\n", i, DKEY(&prev), DVAL(&here)); + prev = here; } } - } else { - assert(bytes < MAX_WRITE); - DWORD written = 0; - if (!WriteFile(fd, item->single.iov_base, (DWORD)bytes, &written, - &item->ov)) { - r.err = (int)GetLastError(); - ERROR("%s: fd %p, item %p (%zu), pgno %u, bytes %zu, offset %" PRId64 - ", err %d", - "WriteFile", fd, __Wpedantic_format_voidptr(item), - item - ior->pool, ((MDBX_page *)item->single.iov_base)->mp_pgno, - bytes, item->ov.Offset + ((uint64_t)item->ov.OffsetHigh << 32), - r.err); - goto bailout_rc; - } else if (unlikely(written != bytes)) { - r.err = ERROR_WRITE_FAULT; - goto bailout_rc; + if (is_branch(mp)) { + if ((mc->checking & z_updating) == 0 && i == 0 && unlikely(ksize != 0)) + rc = bad_page(mp, "branch-node[%zu] wrong 0-node key-length (%zu)\n", i, ksize); + const pgno_t ref = node_pgno(node); + if (unlikely(ref < MIN_PAGENO) || (unlikely(ref >= mc->txn->geo.first_unallocated) && + (unlikely(ref >= mc->txn->geo.now) || !(mc->checking & z_retiring)))) + rc = bad_page(mp, "branch-node[%zu] wrong pgno (%u)\n", i, ref); + if (unlikely(node_flags(node))) + rc = bad_page(mp, "branch-node[%zu] wrong flags (%u)\n", i, node_flags(node)); + continue; } - } - item = ior_next(item, i); - } - - assert(ior->async_waiting > ior->async_completed && - ior->async_waiting == INT_MAX); - ior->async_waiting = async_started; - if (async_started > ior->async_completed && end_wait_for == wait_for) { - assert(wait_for > ior->event_pool + ior->event_stack); - *--wait_for = ior->async_done; - } - const size_t pending_count = end_wait_for - wait_for; - if (pending_count) { - /* Ждем до MAXIMUM_WAIT_OBJECTS (64) последних хендлов, а после избирательно - * ждем посредством GetOverlappedResult(), если какие-то более ранние - * элементы еще не завершены. В целом, так получается меньше системных - * вызовов, т.е. меньше накладных расходов. Однако, не факт что эта экономия - * не будет перекрыта неэффективностью реализации - * WaitForMultipleObjectsEx(), но тогда это проблемы на стороне M$. */ - DWORD madness; - do - madness = WaitForMultipleObjectsEx((pending_count < MAXIMUM_WAIT_OBJECTS) - ? (DWORD)pending_count - : MAXIMUM_WAIT_OBJECTS, - wait_for, true, - /* сутки */ 86400000ul, true); - while (madness == WAIT_IO_COMPLETION); - STATIC_ASSERT(WAIT_OBJECT_0 == 0); - if (/* madness >= WAIT_OBJECT_0 && */ - madness < WAIT_OBJECT_0 + MAXIMUM_WAIT_OBJECTS) - r.err = MDBX_SUCCESS; - else if (madness >= WAIT_ABANDONED_0 && - madness < WAIT_ABANDONED_0 + MAXIMUM_WAIT_OBJECTS) { - r.err = ERROR_ABANDONED_WAIT_0; - goto bailout_rc; - } else if (madness == WAIT_TIMEOUT) { - r.err = WAIT_TIMEOUT; - goto bailout_rc; - } else { - r.err = /* madness == WAIT_FAILED */ MDBX_PROBLEM; - goto bailout_rc; - } + switch (node_flags(node)) { + default: + rc = bad_page(mp, "invalid node[%zu] flags (%u)\n", i, node_flags(node)); + break; + case N_BIG /* data on large-page */: + case 0 /* usual */: + case N_TREE /* sub-db */: + case N_TREE | N_DUP /* dupsorted sub-tree */: + case N_DUP /* short sub-page */: + break; + } - assert(ior->async_waiting == ior->async_completed); - for (ior_item_t *item = ior->pool; item <= ior->last;) { - size_t i = 1, bytes = item->single.iov_len - ior_WriteFile_flag; - if (bytes & ior_WriteFile_flag) { - bytes = ior->pagesize; - /* Zap: Reading invalid data from 'item->sgv' */ - MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(6385); - while (item->sgv[i].Buffer) { - bytes += ior->pagesize; - ++i; + const size_t dsize = node_ds(node); + const char *const data = node_data(node); + if (node_flags(node) & N_BIG) { + if (unlikely(end_of_page < data + sizeof(pgno_t))) { + rc = bad_page(mp, "node-%s(%zu of %zu, %zu bytes) beyond (%zu) page-end\n", "bigdata-pgno", i, nkeys, dsize, + data + dsize - end_of_page); + continue; } - if (!HasOverlappedIoCompleted(&item->ov)) { - DWORD written = 0; - if (unlikely(!GetOverlappedResult(fd, &item->ov, &written, true))) { - ERROR("%s: item %p (%zu), pgno %u, bytes %zu, offset %" PRId64 - ", err %d", - "GetOverlappedResult", __Wpedantic_format_voidptr(item), - item - ior->pool, - ((MDBX_page *)item->single.iov_base)->mp_pgno, bytes, - item->ov.Offset + ((uint64_t)item->ov.OffsetHigh << 32), - (int)GetLastError()); - goto bailout_geterr; + if (unlikely(dsize <= v_clc.lmin || dsize > v_clc.lmax)) + rc = bad_page(mp, "big-node data size (%zu) <> min/max value-length (%zu/%zu)\n", dsize, v_clc.lmin, + v_clc.lmax); + if (unlikely(node_size_len(node_ks(node), dsize) <= mc->txn->env->leaf_nodemax) && + mc->tree != &mc->txn->dbs[FREE_DBI]) + poor_page(mp, "too small data (%zu bytes) for bigdata-node", dsize); + + if ((mc->checking & z_retiring) == 0) { + const pgr_t lp = page_get_large(mc, node_largedata_pgno(node), mp->txnid); + if (unlikely(lp.err != MDBX_SUCCESS)) + return lp.err; + cASSERT(mc, page_type(lp.page) == P_LARGE); + const unsigned npages = largechunk_npages(env, dsize); + if (unlikely(lp.page->pages != npages)) { + if (lp.page->pages < npages) + rc = bad_page(lp.page, "too less n-pages %u for bigdata-node (%zu bytes)", lp.page->pages, dsize); + else if (mc->tree != &mc->txn->dbs[FREE_DBI]) + poor_page(lp.page, "extra n-pages %u for bigdata-node (%zu bytes)", lp.page->pages, dsize); } - assert(MDBX_SUCCESS == item->ov.Internal); - assert(written == item->ov.InternalHigh); } - } else { - assert(HasOverlappedIoCompleted(&item->ov)); + continue; } - assert(item->ov.Internal != ERROR_IO_PENDING); - if (unlikely(item->ov.Internal != MDBX_SUCCESS)) { - DWORD written = 0; - r.err = (int)item->ov.Internal; - if ((r.err & 0x80000000) && - GetOverlappedResult(NULL, &item->ov, &written, true)) - r.err = (int)GetLastError(); - ERROR("%s: item %p (%zu), pgno %u, bytes %zu, offset %" PRId64 - ", err %d", - "Result", __Wpedantic_format_voidptr(item), item - ior->pool, - ((MDBX_page *)item->single.iov_base)->mp_pgno, bytes, - item->ov.Offset + ((uint64_t)item->ov.OffsetHigh << 32), - (int)GetLastError()); - goto bailout_rc; + + if (unlikely(end_of_page < data + dsize)) { + rc = bad_page(mp, "node-%s(%zu of %zu, %zu bytes) beyond (%zu) page-end\n", "data", i, nkeys, dsize, + data + dsize - end_of_page); + continue; } - if (unlikely(item->ov.InternalHigh != bytes)) { - r.err = ERROR_WRITE_FAULT; - goto bailout_rc; + + switch (node_flags(node)) { + default: + /* wrong, but already handled */ + continue; + case 0 /* usual */: + if (unlikely(dsize < v_clc.lmin || dsize > v_clc.lmax)) { + rc = bad_page(mp, "node-data size (%zu) <> min/max value-length (%zu/%zu)\n", dsize, v_clc.lmin, v_clc.lmax); + continue; + } + break; + case N_TREE /* sub-db */: + if (unlikely(dsize != sizeof(tree_t))) { + rc = bad_page(mp, "invalid sub-db record size (%zu)\n", dsize); + continue; + } + break; + case N_TREE | N_DUP /* dupsorted sub-tree */: + if (unlikely(dsize != sizeof(tree_t))) { + rc = bad_page(mp, "invalid nested-db record size (%zu, expect %zu)\n", dsize, sizeof(tree_t)); + continue; + } + break; + case N_DUP /* short sub-page */: + if (unlikely(dsize <= PAGEHDRSZ)) { + rc = bad_page(mp, "invalid nested/sub-page record size (%zu)\n", dsize); + continue; + } else { + const page_t *const sp = (page_t *)data; + switch (sp->flags & + /* ignore legacy P_DIRTY flag */ ~P_LEGACY_DIRTY) { + case P_LEAF | P_SUBP: + case P_LEAF | P_DUPFIX | P_SUBP: + break; + default: + rc = bad_page(mp, "invalid nested/sub-page flags (0x%02x)\n", sp->flags); + continue; + } + + const char *const end_of_subpage = data + dsize; + const intptr_t nsubkeys = page_numkeys(sp); + if (unlikely(nsubkeys == 0) && !(mc->checking & z_updating) && mc->tree->items) + rc = bad_page(mp, "no keys on a %s-page\n", is_dupfix_leaf(sp) ? "leaf2-sub" : "leaf-sub"); + + MDBX_val sub_here, sub_prev = {0, 0}; + for (int ii = 0; ii < nsubkeys; ii++) { + if (is_dupfix_leaf(sp)) { + /* DUPFIX pages have no entries[] or node headers */ + const size_t sub_ksize = sp->dupfix_ksize; + const char *const sub_key = page_dupfix_ptr(sp, ii, mc->tree->dupfix_size); + if (unlikely(end_of_subpage < sub_key + sub_ksize)) { + rc = bad_page(mp, "nested-leaf2-key beyond (%zu) nested-page\n", sub_key + sub_ksize - end_of_subpage); + continue; + } + + if (unlikely(sub_ksize != v_clc.lmin)) { + if (unlikely(sub_ksize < v_clc.lmin || sub_ksize > v_clc.lmax)) + rc = bad_page(mp, + "nested-leaf2-key size (%zu) <> min/max " + "value-length (%zu/%zu)\n", + sub_ksize, v_clc.lmin, v_clc.lmax); + else + v_clc.lmin = v_clc.lmax = sub_ksize; + } + if ((mc->checking & z_ignord) == 0) { + sub_here.iov_base = (void *)sub_key; + sub_here.iov_len = sub_ksize; + if (sub_prev.iov_base && unlikely(v_clc.cmp(&sub_prev, &sub_here) >= 0)) + rc = bad_page(mp, "nested-leaf2-key #%u wrong order (%s >= %s)\n", ii, DKEY(&sub_prev), + DVAL(&sub_here)); + sub_prev = sub_here; + } + } else { + const node_t *const sub_node = page_node(sp, ii); + const char *const sub_node_end = ptr_disp(sub_node, NODESIZE); + if (unlikely(sub_node_end > end_of_subpage)) { + rc = bad_page(mp, "nested-node beyond (%zu) nested-page\n", end_of_subpage - sub_node_end); + continue; + } + if (unlikely(node_flags(sub_node) != 0)) + rc = bad_page(mp, "nested-node invalid flags (%u)\n", node_flags(sub_node)); + + const size_t sub_ksize = node_ks(sub_node); + const char *const sub_key = node_key(sub_node); + const size_t sub_dsize = node_ds(sub_node); + /* char *sub_data = node_data(sub_node); */ + + if (unlikely(sub_ksize < v_clc.lmin || sub_ksize > v_clc.lmax)) + rc = bad_page(mp, + "nested-node-key size (%zu) <> min/max " + "value-length (%zu/%zu)\n", + sub_ksize, v_clc.lmin, v_clc.lmax); + if ((mc->checking & z_ignord) == 0) { + sub_here.iov_base = (void *)sub_key; + sub_here.iov_len = sub_ksize; + if (sub_prev.iov_base && unlikely(v_clc.cmp(&sub_prev, &sub_here) >= 0)) + rc = bad_page(mp, "nested-node-key #%u wrong order (%s >= %s)\n", ii, DKEY(&sub_prev), + DVAL(&sub_here)); + sub_prev = sub_here; + } + if (unlikely(sub_dsize != 0)) + rc = bad_page(mp, "nested-node non-empty data size (%zu)\n", sub_dsize); + if (unlikely(end_of_subpage < sub_key + sub_ksize)) + rc = bad_page(mp, "nested-node-key beyond (%zu) nested-page\n", sub_key + sub_ksize - end_of_subpage); + } + } + } + break; } - item = ior_next(item, i); } - assert(ior->async_waiting == ior->async_completed); - } else { - assert(r.err == MDBX_SUCCESS); } - assert(ior->async_waiting == ior->async_completed); + return rc; +} + +static __always_inline int check_page_header(const uint16_t ILL, const page_t *page, MDBX_txn *const txn, + const txnid_t front) { + if (unlikely(page->flags & ILL)) { + if (ILL == P_ILL_BITS || (page->flags & P_ILL_BITS)) + return bad_page(page, "invalid page's flags (%u)\n", page->flags); + else if (ILL & P_LARGE) { + assert((ILL & (P_BRANCH | P_LEAF | P_DUPFIX)) == 0); + assert(page->flags & (P_BRANCH | P_LEAF | P_DUPFIX)); + return bad_page(page, "unexpected %s instead of %s (%u)\n", "large/overflow", "branch/leaf/leaf2", page->flags); + } else if (ILL & (P_BRANCH | P_LEAF | P_DUPFIX)) { + assert((ILL & P_BRANCH) && (ILL & P_LEAF) && (ILL & P_DUPFIX)); + assert(page->flags & (P_BRANCH | P_LEAF | P_DUPFIX)); + return bad_page(page, "unexpected %s instead of %s (%u)\n", "branch/leaf/leaf2", "large/overflow", page->flags); + } else { + assert(false); + } + } -#else - STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), - "libmdbx requires 64-bit file I/O on 64-bit systems"); - for (ior_item_t *item = ior->pool; item <= ior->last;) { -#if MDBX_HAVE_PWRITEV - assert(item->sgvcnt > 0); - if (item->sgvcnt == 1) - r.err = osal_pwrite(fd, item->sgv[0].iov_base, item->sgv[0].iov_len, - item->offset); - else - r.err = osal_pwritev(fd, item->sgv, item->sgvcnt, item->offset); + if (unlikely(page->txnid > front) && unlikely(page->txnid > txn->front_txnid || front < txn->txnid)) + return bad_page(page, "invalid page' txnid (%" PRIaTXN ") for %s' txnid (%" PRIaTXN ")\n", page->txnid, + (front == txn->front_txnid && front != txn->txnid) ? "front-txn" : "parent-page", front); - // TODO: io_uring_prep_write(sqe, fd, ...); + if (((ILL & P_LARGE) || !is_largepage(page)) && (ILL & (P_BRANCH | P_LEAF | P_DUPFIX)) == 0) { + /* Контроль четности page->upper тут либо приводит к ложным ошибкам, + * либо слишком дорог по количеству операций. Заковырка в том, что upper + * может быть нечетным на DUPFIX-страницах, при нечетном количестве + * элементов нечетной длины. Поэтому четность page->upper здесь не + * проверяется, но соответствующие полные проверки есть в page_check(). */ + if (unlikely(page->upper < page->lower || (page->lower & 1) || PAGEHDRSZ + page->upper > txn->env->ps)) + return bad_page(page, "invalid page' lower(%u)/upper(%u) with limit %zu\n", page->lower, page->upper, + page_space(txn->env)); - item = ior_next(item, item->sgvcnt); -#else - r.err = osal_pwrite(fd, item->single.iov_base, item->single.iov_len, - item->offset); - item = ior_next(item, 1); -#endif - r.wops += 1; - if (unlikely(r.err != MDBX_SUCCESS)) - break; + } else if ((ILL & P_LARGE) == 0) { + const pgno_t npages = page->pages; + if (unlikely(npages < 1) || unlikely(npages >= MAX_PAGENO / 2)) + return bad_page(page, "invalid n-pages (%u) for large-page\n", npages); + if (unlikely(page->pgno + npages > txn->geo.first_unallocated)) + return bad_page(page, "end of large-page beyond (%u) allocated space (%u next-pgno)\n", page->pgno + npages, + txn->geo.first_unallocated); + } else { + assert(false); } + return MDBX_SUCCESS; +} - // TODO: io_uring_submit(&ring) - // TODO: err = io_uring_wait_cqe(&ring, &cqe); - // TODO: io_uring_cqe_seen(&ring, cqe); - -#endif /* !Windows */ +__cold static __noinline pgr_t check_page_complete(const uint16_t ILL, page_t *page, const MDBX_cursor *const mc, + const txnid_t front) { + pgr_t r = {page, check_page_header(ILL, page, mc->txn, front)}; + if (likely(r.err == MDBX_SUCCESS)) + r.err = page_check(mc, page); + if (unlikely(r.err != MDBX_SUCCESS)) + mc->txn->flags |= MDBX_TXN_ERROR; return r; } -MDBX_INTERNAL_FUNC void osal_ioring_reset(osal_ioring_t *ior) { -#if defined(_WIN32) || defined(_WIN64) - if (ior->last) { - for (ior_item_t *item = ior->pool; item <= ior->last;) { - if (!HasOverlappedIoCompleted(&item->ov)) { - assert(ior->overlapped_fd); - CancelIoEx(ior->overlapped_fd, &item->ov); - } - if (item->ov.hEvent && item->ov.hEvent != ior) - ior_put_event(ior, item->ov.hEvent); - size_t i = 1; - if ((item->single.iov_len & ior_WriteFile_flag) == 0) { - /* Zap: Reading invalid data from 'item->sgv' */ - MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(6385); - while (item->sgv[i].Buffer) - ++i; +static __always_inline pgr_t page_get_inline(const uint16_t ILL, const MDBX_cursor *const mc, const pgno_t pgno, + const txnid_t front) { + MDBX_txn *const txn = mc->txn; + tASSERT(txn, front <= txn->front_txnid); + + pgr_t r; + if (unlikely(pgno >= txn->geo.first_unallocated)) { + ERROR("page #%" PRIaPGNO " beyond next-pgno", pgno); + r.page = nullptr; + r.err = MDBX_PAGE_NOTFOUND; + bailout: + txn->flags |= MDBX_TXN_ERROR; + return r; + } + + eASSERT(txn->env, ((txn->flags ^ txn->env->flags) & MDBX_WRITEMAP) == 0); + r.page = pgno2page(txn->env, pgno); + if ((txn->flags & (MDBX_TXN_RDONLY | MDBX_WRITEMAP)) == 0) { + const MDBX_txn *spiller = txn; + do { + /* Spilled pages were dirtied in this txn and flushed + * because the dirty list got full. Bring this page + * back in from the map (but don't unspill it here, + * leave that unless page_touch happens again). */ + if (unlikely(spiller->flags & MDBX_TXN_SPILLS) && spill_search(spiller, pgno)) + break; + + const size_t i = dpl_search(spiller, pgno); + tASSERT(txn, (intptr_t)i > 0); + if (spiller->tw.dirtylist->items[i].pgno == pgno) { + r.page = spiller->tw.dirtylist->items[i].ptr; + break; } - item = ior_next(item, i); - } + + spiller = spiller->parent; + } while (unlikely(spiller)); } - ior->async_waiting = INT_MAX; - ior->async_completed = 0; - ResetEvent(ior->async_done); -#endif /* !Windows */ - ior->slots_left = ior->allocated; - ior->last = nullptr; -} -static void ior_cleanup(osal_ioring_t *ior, const size_t since) { - osal_ioring_reset(ior); -#if defined(_WIN32) || defined(_WIN64) - for (size_t i = since; i < ior->event_stack; ++i) { - /* Zap: Using uninitialized memory '**ior.event_pool' */ - MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(6001); - CloseHandle(ior->event_pool[i]); + if (unlikely(r.page->pgno != pgno)) { + r.err = bad_page(r.page, "pgno mismatch (%" PRIaPGNO ") != expected (%" PRIaPGNO ")\n", r.page->pgno, pgno); + goto bailout; } - ior->event_stack = 0; + + if (unlikely(mc->checking & z_pagecheck)) + return check_page_complete(ILL, r.page, mc, front); + +#if MDBX_DISABLE_VALIDATION + r.err = MDBX_SUCCESS; #else - (void)since; -#endif /* Windows */ + r.err = check_page_header(ILL, r.page, txn, front); + if (unlikely(r.err != MDBX_SUCCESS)) + goto bailout; +#endif /* MDBX_DISABLE_VALIDATION */ + return r; } -MDBX_INTERNAL_FUNC int osal_ioring_resize(osal_ioring_t *ior, size_t items) { - assert(items > 0 && items < INT_MAX / sizeof(ior_item_t)); -#if defined(_WIN32) || defined(_WIN64) - if (ior->state & IOR_STATE_LOCKED) - return MDBX_SUCCESS; - const bool useSetFileIoOverlappedRange = - ior->overlapped_fd && mdbx_SetFileIoOverlappedRange && items > 42; - const size_t ceiling = - useSetFileIoOverlappedRange - ? ((items < 65536 / 2 / sizeof(ior_item_t)) ? 65536 : 65536 * 4) - : 1024; - const size_t bytes = ceil_powerof2(sizeof(ior_item_t) * items, ceiling); - items = bytes / sizeof(ior_item_t); -#endif /* Windows */ +pgr_t page_get_any(const MDBX_cursor *const mc, const pgno_t pgno, const txnid_t front) { + return page_get_inline(P_ILL_BITS, mc, pgno, front); +} - if (items != ior->allocated) { - assert(items >= osal_ioring_used(ior)); - if (items < ior->allocated) - ior_cleanup(ior, items); -#if defined(_WIN32) || defined(_WIN64) - void *ptr = osal_realloc( - ior->event_pool, - (items + /* extra for waiting the async_done */ 1) * sizeof(HANDLE)); - if (unlikely(!ptr)) - return MDBX_ENOMEM; - ior->event_pool = ptr; +__hot pgr_t page_get_three(const MDBX_cursor *const mc, const pgno_t pgno, const txnid_t front) { + return page_get_inline(P_ILL_BITS | P_LARGE, mc, pgno, front); +} - int err = osal_memalign_alloc(ceiling, bytes, &ptr); - if (unlikely(err != MDBX_SUCCESS)) - return err; - if (ior->pool) { - memcpy(ptr, ior->pool, ior->allocated * sizeof(ior_item_t)); - osal_memalign_free(ior->pool); +pgr_t page_get_large(const MDBX_cursor *const mc, const pgno_t pgno, const txnid_t front) { + return page_get_inline(P_ILL_BITS | P_BRANCH | P_LEAF | P_DUPFIX, mc, pgno, front); +} +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 + +int iov_init(MDBX_txn *const txn, iov_ctx_t *ctx, size_t items, size_t npages, mdbx_filehandle_t fd, + bool check_coherence) { + ctx->env = txn->env; + ctx->ior = &txn->env->ioring; + ctx->fd = fd; + ctx->coherency_timestamp = + (check_coherence || txn->env->lck->pgops.incoherence.weak) ? 0 : UINT64_MAX /* не выполнять сверку */; + ctx->err = osal_ioring_prepare(ctx->ior, items, pgno_align2os_bytes(txn->env, npages)); + if (likely(ctx->err == MDBX_SUCCESS)) { +#if MDBX_NEED_WRITTEN_RANGE + ctx->flush_begin = MAX_PAGENO; + ctx->flush_end = MIN_PAGENO; +#endif /* MDBX_NEED_WRITTEN_RANGE */ + osal_ioring_reset(ctx->ior); + } + return ctx->err; +} + +static void iov_callback4dirtypages(iov_ctx_t *ctx, size_t offset, void *data, size_t bytes) { + MDBX_env *const env = ctx->env; + eASSERT(env, (env->flags & MDBX_WRITEMAP) == 0); + + page_t *wp = (page_t *)data; + eASSERT(env, wp->pgno == bytes2pgno(env, offset)); + eASSERT(env, bytes2pgno(env, bytes) >= (is_largepage(wp) ? wp->pages : 1u)); + eASSERT(env, (wp->flags & P_ILL_BITS) == 0); + + if (likely(ctx->err == MDBX_SUCCESS)) { + const page_t *const rp = ptr_disp(env->dxb_mmap.base, offset); + VALGRIND_MAKE_MEM_DEFINED(rp, bytes); + MDBX_ASAN_UNPOISON_MEMORY_REGION(rp, bytes); + osal_flush_incoherent_mmap(rp, bytes, globals.sys_pagesize); + /* check with timeout as the workaround + * for https://libmdbx.dqdkfa.ru/dead-github/issues/269 + * + * Проблема проявляется только при неупорядоченности: если записанная + * последней мета-страница "обгоняет" ранее записанные, т.е. когда + * записанное в файл позже становится видимым в отображении раньше, + * чем записанное ранее. + * + * Исходно здесь всегда выполнялась полная сверка. Это давало полную + * гарантию защиты от проявления проблемы, но порождало накладные расходы. + * В некоторых сценариях наблюдалось снижение производительности до 10-15%, + * а в синтетических тестах до 30%. Конечно никто не вникал в причины, + * а просто останавливался на мнении "libmdbx не быстрее LMDB", + * например: https://clck.ru/3386er + * + * Поэтому после серии экспериментов и тестов реализовано следующее: + * 0. Посредством опции сборки MDBX_FORCE_CHECK_MMAP_COHERENCY=1 + * можно включить полную сверку после записи. + * Остальные пункты являются взвешенным компромиссом между полной + * гарантией обнаружения проблемы и бесполезными затратами на системах + * без этого недостатка. + * 1. При старте транзакций проверяется соответствие выбранной мета-страницы + * корневым страницам b-tree проверяется. Эта проверка показала себя + * достаточной без сверки после записи. При обнаружении "некогерентности" + * эти случаи подсчитываются, а при их ненулевом счетчике выполняется + * полная сверка. Таким образом, произойдет переключение в режим полной + * сверки, если показавшая себя достаточной проверка заметит проявление + * проблемы хоты-бы раз. + * 2. Сверка не выполняется при фиксации транзакции, так как: + * - при наличии проблемы "не-когерентности" (при отложенном копировании + * или обновлении PTE, после возврата из write-syscall), проверка + * в этом процессе не гарантирует актуальность данных в другом + * процессе, который может запустить транзакцию сразу после коммита; + * - сверка только последнего блока позволяет почти восстановить + * производительность в больших транзакциях, но одновременно размывает + * уверенность в отсутствии сбоев, чем обесценивает всю затею; + * - после записи данных будет записана мета-страница, соответствие + * которой корневым страницам b-tree проверяется при старте + * транзакций, и только эта проверка показала себя достаточной; + * 3. При спиллинге производится полная сверка записанных страниц. Тут был + * соблазн сверять не полностью, а например начало и конец каждого блока. + * Но при спиллинге возможна ситуация повторного вытеснения страниц, в + * том числе large/overflow. При этом возникает риск прочитать в текущей + * транзакции старую версию страницы, до повторной записи. В этом случае + * могут возникать крайне редкие невоспроизводимые ошибки. С учетом того + * что спиллинг выполняет крайне редко, решено отказаться от экономии + * в пользу надежности. */ +#ifndef MDBX_FORCE_CHECK_MMAP_COHERENCY +#define MDBX_FORCE_CHECK_MMAP_COHERENCY 0 +#endif /* MDBX_FORCE_CHECK_MMAP_COHERENCY */ + if ((MDBX_FORCE_CHECK_MMAP_COHERENCY || ctx->coherency_timestamp != UINT64_MAX) && + unlikely(memcmp(wp, rp, bytes))) { + ctx->coherency_timestamp = 0; + env->lck->pgops.incoherence.weak = + (env->lck->pgops.incoherence.weak >= INT32_MAX) ? INT32_MAX : env->lck->pgops.incoherence.weak + 1; + WARNING("catch delayed/non-arrived page %" PRIaPGNO " %s", wp->pgno, + "(workaround for incoherent flaw of unified page/buffer cache)"); + do + if (coherency_timeout(&ctx->coherency_timestamp, wp->pgno, env) != MDBX_RESULT_TRUE) { + ctx->err = MDBX_PROBLEM; + break; + } + while (unlikely(memcmp(wp, rp, bytes))); } -#else - void *ptr = osal_realloc(ior->pool, sizeof(ior_item_t) * items); - if (unlikely(!ptr)) - return MDBX_ENOMEM; -#endif - ior->pool = ptr; + } - if (items > ior->allocated) - memset(ior->pool + ior->allocated, 0, - sizeof(ior_item_t) * (items - ior->allocated)); - ior->allocated = (unsigned)items; - ior->boundary = ptr_disp(ior->pool, ior->allocated); -#if defined(_WIN32) || defined(_WIN64) - if (useSetFileIoOverlappedRange) { - if (mdbx_SetFileIoOverlappedRange(ior->overlapped_fd, ptr, (ULONG)bytes)) - ior->state += IOR_STATE_LOCKED; - else - return GetLastError(); + if (likely(bytes == env->ps)) + page_shadow_release(env, wp, 1); + else { + do { + eASSERT(env, wp->pgno == bytes2pgno(env, offset)); + eASSERT(env, (wp->flags & P_ILL_BITS) == 0); + size_t npages = is_largepage(wp) ? wp->pages : 1u; + size_t chunk = pgno2bytes(env, npages); + eASSERT(env, bytes >= chunk); + page_t *next = ptr_disp(wp, chunk); + page_shadow_release(env, wp, npages); + wp = next; + offset += chunk; + bytes -= chunk; + } while (bytes); + } +} + +static void iov_complete(iov_ctx_t *ctx) { + if ((ctx->env->flags & MDBX_WRITEMAP) == 0) + osal_ioring_walk(ctx->ior, ctx, iov_callback4dirtypages); + osal_ioring_reset(ctx->ior); +} + +int iov_write(iov_ctx_t *ctx) { + eASSERT(ctx->env, !iov_empty(ctx)); + osal_ioring_write_result_t r = osal_ioring_write(ctx->ior, ctx->fd); +#if MDBX_ENABLE_PGOP_STAT + ctx->env->lck->pgops.wops.weak += r.wops; +#endif /* MDBX_ENABLE_PGOP_STAT */ + ctx->err = r.err; + if (unlikely(ctx->err != MDBX_SUCCESS)) + ERROR("Write error: %s", mdbx_strerror(ctx->err)); + iov_complete(ctx); + return ctx->err; +} + +int iov_page(MDBX_txn *txn, iov_ctx_t *ctx, page_t *dp, size_t npages) { + MDBX_env *const env = txn->env; + tASSERT(txn, ctx->err == MDBX_SUCCESS); + tASSERT(txn, dp->pgno >= MIN_PAGENO && dp->pgno < txn->geo.first_unallocated); + tASSERT(txn, is_modifable(txn, dp)); + tASSERT(txn, !(dp->flags & ~(P_BRANCH | P_LEAF | P_DUPFIX | P_LARGE))); + + if (is_shadowed(txn, dp)) { + tASSERT(txn, !(txn->flags & MDBX_WRITEMAP)); + dp->txnid = txn->txnid; + tASSERT(txn, is_spilled(txn, dp)); +#if MDBX_AVOID_MSYNC + doit:; +#endif /* MDBX_AVOID_MSYNC */ + int err = osal_ioring_add(ctx->ior, pgno2bytes(env, dp->pgno), dp, pgno2bytes(env, npages)); + if (unlikely(err != MDBX_SUCCESS)) { + ctx->err = err; + if (unlikely(err != MDBX_RESULT_TRUE)) { + iov_complete(ctx); + return err; + } + err = iov_write(ctx); + tASSERT(txn, iov_empty(ctx)); + if (likely(err == MDBX_SUCCESS)) { + err = osal_ioring_add(ctx->ior, pgno2bytes(env, dp->pgno), dp, pgno2bytes(env, npages)); + if (unlikely(err != MDBX_SUCCESS)) { + iov_complete(ctx); + return ctx->err = err; + } + } + tASSERT(txn, ctx->err == MDBX_SUCCESS); } -#endif /* Windows */ + } else { + tASSERT(txn, txn->flags & MDBX_WRITEMAP); +#if MDBX_AVOID_MSYNC + goto doit; +#endif /* MDBX_AVOID_MSYNC */ } + +#if MDBX_NEED_WRITTEN_RANGE + ctx->flush_begin = (ctx->flush_begin < dp->pgno) ? ctx->flush_begin : dp->pgno; + ctx->flush_end = (ctx->flush_end > dp->pgno + (pgno_t)npages) ? ctx->flush_end : dp->pgno + (pgno_t)npages; +#endif /* MDBX_NEED_WRITTEN_RANGE */ return MDBX_SUCCESS; } +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 -MDBX_INTERNAL_FUNC void osal_ioring_destroy(osal_ioring_t *ior) { - if (ior->allocated) - ior_cleanup(ior, 0); -#if defined(_WIN32) || defined(_WIN64) - osal_memalign_free(ior->pool); - osal_free(ior->event_pool); - CloseHandle(ior->async_done); - if (ior->overlapped_fd) - CloseHandle(ior->overlapped_fd); -#else - osal_free(ior->pool); -#endif - memset(ior, 0, sizeof(osal_ioring_t)); +static inline tree_t *outer_tree(MDBX_cursor *mc) { + cASSERT(mc, (mc->flags & z_inner) != 0); + subcur_t *mx = container_of(mc->tree, subcur_t, nested_tree); + cursor_couple_t *couple = container_of(mx, cursor_couple_t, inner); + cASSERT(mc, mc->tree == &couple->outer.subcur->nested_tree); + cASSERT(mc, &mc->clc->k == &couple->outer.clc->v); + return couple->outer.tree; } -/*----------------------------------------------------------------------------*/ +pgr_t page_new(MDBX_cursor *mc, const unsigned flags) { + cASSERT(mc, (flags & P_LARGE) == 0); + pgr_t ret = gc_alloc_single(mc); + if (unlikely(ret.err != MDBX_SUCCESS)) + return ret; -MDBX_INTERNAL_FUNC int osal_removefile(const pathchar_t *pathname) { -#if defined(_WIN32) || defined(_WIN64) - return DeleteFileW(pathname) ? MDBX_SUCCESS : (int)GetLastError(); -#else - return unlink(pathname) ? errno : MDBX_SUCCESS; -#endif + DEBUG("db %zu allocated new page %" PRIaPGNO, cursor_dbi(mc), ret.page->pgno); + ret.page->flags = (uint16_t)flags; + cASSERT(mc, *cursor_dbi_state(mc) & DBI_DIRTY); + cASSERT(mc, mc->txn->flags & MDBX_TXN_DIRTY); +#if MDBX_ENABLE_PGOP_STAT + mc->txn->env->lck->pgops.newly.weak += 1; +#endif /* MDBX_ENABLE_PGOP_STAT */ + + STATIC_ASSERT(P_BRANCH == 1); + const unsigned is_branch = flags & P_BRANCH; + + ret.page->lower = 0; + ret.page->upper = (indx_t)(mc->txn->env->ps - PAGEHDRSZ); + mc->tree->branch_pages += is_branch; + mc->tree->leaf_pages += 1 - is_branch; + if (unlikely(mc->flags & z_inner)) { + tree_t *outer = outer_tree(mc); + outer->branch_pages += is_branch; + outer->leaf_pages += 1 - is_branch; + } + return ret; } -#if !(defined(_WIN32) || defined(_WIN64)) -static bool is_valid_fd(int fd) { return !(isatty(fd) < 0 && errno == EBADF); } -#endif /*! Windows */ +pgr_t page_new_large(MDBX_cursor *mc, const size_t npages) { + pgr_t ret = likely(npages == 1) ? gc_alloc_single(mc) : gc_alloc_ex(mc, npages, ALLOC_DEFAULT); + if (unlikely(ret.err != MDBX_SUCCESS)) + return ret; -MDBX_INTERNAL_FUNC int osal_removedirectory(const pathchar_t *pathname) { -#if defined(_WIN32) || defined(_WIN64) - return RemoveDirectoryW(pathname) ? MDBX_SUCCESS : (int)GetLastError(); -#else - return rmdir(pathname) ? errno : MDBX_SUCCESS; -#endif + DEBUG("dbi %zu allocated new large-page %" PRIaPGNO ", num %zu", cursor_dbi(mc), ret.page->pgno, npages); + ret.page->flags = P_LARGE; + cASSERT(mc, *cursor_dbi_state(mc) & DBI_DIRTY); + cASSERT(mc, mc->txn->flags & MDBX_TXN_DIRTY); +#if MDBX_ENABLE_PGOP_STAT + mc->txn->env->lck->pgops.newly.weak += npages; +#endif /* MDBX_ENABLE_PGOP_STAT */ + + mc->tree->large_pages += (pgno_t)npages; + ret.page->pages = (pgno_t)npages; + cASSERT(mc, !(mc->flags & z_inner)); + return ret; } -MDBX_INTERNAL_FUNC int osal_fileexists(const pathchar_t *pathname) { -#if defined(_WIN32) || defined(_WIN64) - if (GetFileAttributesW(pathname) != INVALID_FILE_ATTRIBUTES) - return MDBX_RESULT_TRUE; - int err = GetLastError(); - return (err == ERROR_FILE_NOT_FOUND || err == ERROR_PATH_NOT_FOUND) - ? MDBX_RESULT_FALSE - : err; -#else - if (access(pathname, F_OK) == 0) - return MDBX_RESULT_TRUE; - int err = errno; - return (err == ENOENT || err == ENOTDIR) ? MDBX_RESULT_FALSE : err; -#endif +__hot void page_copy(page_t *const dst, const page_t *const src, const size_t size) { + STATIC_ASSERT(UINT16_MAX > MDBX_MAX_PAGESIZE - PAGEHDRSZ); + STATIC_ASSERT(MDBX_MIN_PAGESIZE > PAGEHDRSZ + NODESIZE * 4); + void *copy_dst = dst; + const void *copy_src = src; + size_t copy_len = size; + if (src->flags & P_DUPFIX) { + copy_len = PAGEHDRSZ + src->dupfix_ksize * page_numkeys(src); + if (unlikely(copy_len > size)) + goto bailout; + } else if ((src->flags & P_LARGE) == 0) { + size_t upper = src->upper, lower = src->lower; + intptr_t unused = upper - lower; + /* If page isn't full, just copy the used portion. Adjust + * alignment so memcpy may copy words instead of bytes. */ + if (unused > MDBX_CACHELINE_SIZE * 3) { + lower = ceil_powerof2(lower + PAGEHDRSZ, sizeof(void *)); + upper = floor_powerof2(upper + PAGEHDRSZ, sizeof(void *)); + if (unlikely(upper > copy_len)) + goto bailout; + memcpy(copy_dst, copy_src, lower); + copy_dst = ptr_disp(copy_dst, upper); + copy_src = ptr_disp(copy_src, upper); + copy_len -= upper; + } + } + memcpy(copy_dst, copy_src, copy_len); + return; + +bailout: + if (src->flags & P_DUPFIX) + bad_page(src, "%s addr %p, n-keys %zu, ksize %u", "invalid/corrupted source page", __Wpedantic_format_voidptr(src), + page_numkeys(src), src->dupfix_ksize); + else + bad_page(src, "%s addr %p, upper %u", "invalid/corrupted source page", __Wpedantic_format_voidptr(src), src->upper); + memset(dst, -1, size); } -MDBX_INTERNAL_FUNC pathchar_t *osal_fileext(const pathchar_t *pathname, - size_t len) { - const pathchar_t *ext = nullptr; - for (size_t i = 0; i < len && pathname[i]; i++) - if (pathname[i] == '.') - ext = pathname + i; - else if (osal_isdirsep(pathname[i])) - ext = nullptr; - return (pathchar_t *)ext; +__cold pgr_t __must_check_result page_unspill(MDBX_txn *const txn, const page_t *const mp) { + VERBOSE("unspill page %" PRIaPGNO, mp->pgno); + tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0); + tASSERT(txn, is_spilled(txn, mp)); + const MDBX_txn *scan = txn; + pgr_t ret; + do { + tASSERT(txn, (scan->flags & MDBX_TXN_SPILLS) != 0); + const size_t si = spill_search(scan, mp->pgno); + if (!si) + continue; + const unsigned npages = is_largepage(mp) ? mp->pages : 1; + ret.page = page_shadow_alloc(txn, npages); + if (unlikely(!ret.page)) { + ret.err = MDBX_ENOMEM; + return ret; + } + page_copy(ret.page, mp, pgno2bytes(txn->env, npages)); + if (scan == txn) { + /* If in current txn, this page is no longer spilled. + * If it happens to be the last page, truncate the spill list. + * Otherwise mark it as deleted by setting the LSB. */ + spill_remove(txn, si, npages); + } /* otherwise, if belonging to a parent txn, the + * page remains spilled until child commits */ + + ret.err = page_dirty(txn, ret.page, npages); + if (unlikely(ret.err != MDBX_SUCCESS)) + return ret; +#if MDBX_ENABLE_PGOP_STAT + txn->env->lck->pgops.unspill.weak += npages; +#endif /* MDBX_ENABLE_PGOP_STAT */ + ret.page->flags |= (scan == txn) ? 0 : P_SPILLED; + ret.err = MDBX_SUCCESS; + return ret; + } while (likely((scan = scan->parent) != nullptr && (scan->flags & MDBX_TXN_SPILLS) != 0)); + ERROR("Page %" PRIaPGNO " mod-txnid %" PRIaTXN " not found in the spill-list(s), current txn %" PRIaTXN + " front %" PRIaTXN ", root txn %" PRIaTXN " front %" PRIaTXN, + mp->pgno, mp->txnid, txn->txnid, txn->front_txnid, txn->env->basal_txn->txnid, + txn->env->basal_txn->front_txnid); + ret.err = MDBX_PROBLEM; + ret.page = nullptr; + return ret; } -MDBX_INTERNAL_FUNC bool osal_pathequal(const pathchar_t *l, const pathchar_t *r, - size_t len) { -#if defined(_WIN32) || defined(_WIN64) - for (size_t i = 0; i < len; ++i) { - pathchar_t a = l[i]; - pathchar_t b = r[i]; - a = (a == '\\') ? '/' : a; - b = (b == '\\') ? '/' : b; - if (a != b) - return false; +__hot int page_touch_modifable(MDBX_txn *txn, const page_t *const mp) { + tASSERT(txn, is_modifable(txn, mp) && txn->tw.dirtylist); + tASSERT(txn, !is_largepage(mp) && !is_subpage(mp)); + tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); + + const size_t n = dpl_search(txn, mp->pgno); + if (MDBX_AVOID_MSYNC && unlikely(txn->tw.dirtylist->items[n].pgno != mp->pgno)) { + tASSERT(txn, (txn->flags & MDBX_WRITEMAP)); + tASSERT(txn, n > 0 && n <= txn->tw.dirtylist->length + 1); + VERBOSE("unspill page %" PRIaPGNO, mp->pgno); +#if MDBX_ENABLE_PGOP_STAT + txn->env->lck->pgops.unspill.weak += 1; +#endif /* MDBX_ENABLE_PGOP_STAT */ + return page_dirty(txn, (page_t *)mp, 1); } - return true; -#else - return memcmp(l, r, len * sizeof(pathchar_t)) == 0; -#endif + + tASSERT(txn, n > 0 && n <= txn->tw.dirtylist->length); + tASSERT(txn, txn->tw.dirtylist->items[n].pgno == mp->pgno && txn->tw.dirtylist->items[n].ptr == mp); + if (!MDBX_AVOID_MSYNC || (txn->flags & MDBX_WRITEMAP) == 0) { + size_t *const ptr = ptr_disp(txn->tw.dirtylist->items[n].ptr, -(ptrdiff_t)sizeof(size_t)); + *ptr = txn->tw.dirtylru; + } + return MDBX_SUCCESS; } -MDBX_INTERNAL_FUNC int osal_openfile(const enum osal_openfile_purpose purpose, - const MDBX_env *env, - const pathchar_t *pathname, - mdbx_filehandle_t *fd, - mdbx_mode_t unix_mode_bits) { - *fd = INVALID_HANDLE_VALUE; +__hot int page_touch_unmodifable(MDBX_txn *txn, MDBX_cursor *mc, const page_t *const mp) { + tASSERT(txn, !is_modifable(txn, mp) && !is_largepage(mp)); + if (is_subpage(mp)) { + ((page_t *)mp)->txnid = txn->front_txnid; + return MDBX_SUCCESS; + } -#if defined(_WIN32) || defined(_WIN64) - DWORD CreationDisposition = unix_mode_bits ? OPEN_ALWAYS : OPEN_EXISTING; - DWORD FlagsAndAttributes = - FILE_FLAG_POSIX_SEMANTICS | FILE_ATTRIBUTE_NOT_CONTENT_INDEXED; - DWORD DesiredAccess = FILE_READ_ATTRIBUTES; - DWORD ShareMode = (env->me_flags & MDBX_EXCLUSIVE) - ? 0 - : (FILE_SHARE_READ | FILE_SHARE_WRITE); + int rc; + page_t *np; + if (is_frozen(txn, mp)) { + /* CoW the page */ + rc = pnl_need(&txn->tw.retired_pages, 1); + if (unlikely(rc != MDBX_SUCCESS)) + goto fail; + const pgr_t par = gc_alloc_single(mc); + rc = par.err; + np = par.page; + if (unlikely(rc != MDBX_SUCCESS)) + goto fail; - switch (purpose) { - default: - return ERROR_INVALID_PARAMETER; - case MDBX_OPEN_LCK: - CreationDisposition = OPEN_ALWAYS; - DesiredAccess |= GENERIC_READ | GENERIC_WRITE; - FlagsAndAttributes |= FILE_ATTRIBUTE_HIDDEN | FILE_ATTRIBUTE_TEMPORARY; - break; - case MDBX_OPEN_DXB_READ: - CreationDisposition = OPEN_EXISTING; - DesiredAccess |= GENERIC_READ; - ShareMode |= FILE_SHARE_READ; - break; - case MDBX_OPEN_DXB_LAZY: - DesiredAccess |= GENERIC_READ | GENERIC_WRITE; - break; - case MDBX_OPEN_DXB_OVERLAPPED_DIRECT: - FlagsAndAttributes |= FILE_FLAG_NO_BUFFERING; - /* fall through */ - __fallthrough; - case MDBX_OPEN_DXB_OVERLAPPED: - FlagsAndAttributes |= FILE_FLAG_OVERLAPPED; - /* fall through */ - __fallthrough; - case MDBX_OPEN_DXB_DSYNC: - CreationDisposition = OPEN_EXISTING; - DesiredAccess |= GENERIC_WRITE | GENERIC_READ; - FlagsAndAttributes |= FILE_FLAG_WRITE_THROUGH; - break; - case MDBX_OPEN_COPY: - CreationDisposition = CREATE_NEW; - ShareMode = 0; - DesiredAccess |= GENERIC_WRITE; - if (env->me_psize >= env->me_os_psize) - FlagsAndAttributes |= FILE_FLAG_NO_BUFFERING; - break; - case MDBX_OPEN_DELETE: - CreationDisposition = OPEN_EXISTING; - ShareMode |= FILE_SHARE_DELETE; - DesiredAccess = - FILE_READ_ATTRIBUTES | FILE_WRITE_ATTRIBUTES | DELETE | SYNCHRONIZE; - break; + const pgno_t pgno = np->pgno; + DEBUG("touched db %d page %" PRIaPGNO " -> %" PRIaPGNO, cursor_dbi_dbg(mc), mp->pgno, pgno); + tASSERT(txn, mp->pgno != pgno); + pnl_append_prereserved(txn->tw.retired_pages, mp->pgno); + /* Update the parent page, if any, to point to the new page */ + if (likely(mc->top)) { + page_t *parent = mc->pg[mc->top - 1]; + node_t *node = page_node(parent, mc->ki[mc->top - 1]); + node_set_pgno(node, pgno); + } else { + mc->tree->root = pgno; + } + +#if MDBX_ENABLE_PGOP_STAT + txn->env->lck->pgops.cow.weak += 1; +#endif /* MDBX_ENABLE_PGOP_STAT */ + page_copy(np, mp, txn->env->ps); + np->pgno = pgno; + np->txnid = txn->front_txnid; + } else if (is_spilled(txn, mp)) { + pgr_t pur = page_unspill(txn, mp); + np = pur.page; + rc = pur.err; + if (likely(rc == MDBX_SUCCESS)) { + tASSERT(txn, np != nullptr); + goto done; + } + goto fail; + } else { + if (unlikely(!txn->parent)) { + ERROR("Unexpected not frozen/modifiable/spilled but shadowed %s " + "page %" PRIaPGNO " mod-txnid %" PRIaTXN "," + " without parent transaction, current txn %" PRIaTXN " front %" PRIaTXN, + is_branch(mp) ? "branch" : "leaf", mp->pgno, mp->txnid, mc->txn->txnid, mc->txn->front_txnid); + rc = MDBX_PROBLEM; + goto fail; + } + + DEBUG("clone db %d page %" PRIaPGNO, cursor_dbi_dbg(mc), mp->pgno); + tASSERT(txn, txn->tw.dirtylist->length <= PAGELIST_LIMIT + MDBX_PNL_GRANULATE); + /* No - copy it */ + np = page_shadow_alloc(txn, 1); + if (unlikely(!np)) { + rc = MDBX_ENOMEM; + goto fail; + } + page_copy(np, mp, txn->env->ps); + + /* insert a clone of parent's dirty page, so don't touch dirtyroom */ + rc = page_dirty(txn, np, 1); + if (unlikely(rc != MDBX_SUCCESS)) + goto fail; + +#if MDBX_ENABLE_PGOP_STAT + txn->env->lck->pgops.clone.weak += 1; +#endif /* MDBX_ENABLE_PGOP_STAT */ } - *fd = CreateFileW(pathname, DesiredAccess, ShareMode, NULL, - CreationDisposition, FlagsAndAttributes, NULL); - if (*fd == INVALID_HANDLE_VALUE) { - int err = (int)GetLastError(); - if (err == ERROR_ACCESS_DENIED && purpose == MDBX_OPEN_LCK) { - if (GetFileAttributesW(pathname) == INVALID_FILE_ATTRIBUTES && - GetLastError() == ERROR_FILE_NOT_FOUND) - err = ERROR_FILE_NOT_FOUND; +done: + /* Adjust cursors pointing to mp */ + mc->pg[mc->top] = np; + MDBX_cursor *m2 = txn->cursors[cursor_dbi(mc)]; + if (mc->flags & z_inner) { + for (; m2; m2 = m2->next) { + MDBX_cursor *m3 = &m2->subcur->cursor; + if (m3->top < mc->top) + continue; + if (m3->pg[mc->top] == mp) + m3->pg[mc->top] = np; + } + } else { + for (; m2; m2 = m2->next) { + if (m2->top < mc->top) + continue; + if (m2->pg[mc->top] == mp) { + m2->pg[mc->top] = np; + if (is_leaf(np) && inner_pointed(m2)) + cursor_inner_refresh(m2, np, m2->ki[mc->top]); + } + } + } + return MDBX_SUCCESS; + +fail: + txn->flags |= MDBX_TXN_ERROR; + return rc; +} + +page_t *page_shadow_alloc(MDBX_txn *txn, size_t num) { + MDBX_env *env = txn->env; + page_t *np = env->shadow_reserve; + size_t size = env->ps; + if (likely(num == 1 && np)) { + eASSERT(env, env->shadow_reserve_len > 0); + MDBX_ASAN_UNPOISON_MEMORY_REGION(np, size); + VALGRIND_MEMPOOL_ALLOC(env, ptr_disp(np, -(ptrdiff_t)sizeof(size_t)), size + sizeof(size_t)); + VALGRIND_MAKE_MEM_DEFINED(&page_next(np), sizeof(page_t *)); + env->shadow_reserve = page_next(np); + env->shadow_reserve_len -= 1; + } else { + size = pgno2bytes(env, num); + void *const ptr = osal_malloc(size + sizeof(size_t)); + if (unlikely(!ptr)) { + txn->flags |= MDBX_TXN_ERROR; + return nullptr; } - return err; + VALGRIND_MEMPOOL_ALLOC(env, ptr, size + sizeof(size_t)); + np = ptr_disp(ptr, sizeof(size_t)); } - BY_HANDLE_FILE_INFORMATION info; - if (!GetFileInformationByHandle(*fd, &info)) { - int err = (int)GetLastError(); - CloseHandle(*fd); - *fd = INVALID_HANDLE_VALUE; - return err; + if ((env->flags & MDBX_NOMEMINIT) == 0) { + /* For a single page alloc, we init everything after the page header. + * For multi-page, we init the final page; if the caller needed that + * many pages they will be filling in at least up to the last page. */ + size_t skip = PAGEHDRSZ; + if (num > 1) + skip += pgno2bytes(env, num - 1); + memset(ptr_disp(np, skip), 0, size - skip); } - const DWORD AttributesDiff = - (info.dwFileAttributes ^ FlagsAndAttributes) & - (FILE_ATTRIBUTE_HIDDEN | FILE_ATTRIBUTE_NOT_CONTENT_INDEXED | - FILE_ATTRIBUTE_TEMPORARY | FILE_ATTRIBUTE_COMPRESSED); - if (AttributesDiff) - (void)SetFileAttributesW(pathname, info.dwFileAttributes ^ AttributesDiff); - -#else - int flags = unix_mode_bits ? O_CREAT : 0; - switch (purpose) { - default: - return EINVAL; - case MDBX_OPEN_LCK: - flags |= O_RDWR; - break; - case MDBX_OPEN_DXB_READ: - flags = O_RDONLY; - break; - case MDBX_OPEN_DXB_LAZY: - flags |= O_RDWR; - break; - case MDBX_OPEN_COPY: - flags = O_CREAT | O_WRONLY | O_EXCL; - break; - case MDBX_OPEN_DXB_DSYNC: - flags |= O_WRONLY; -#if defined(O_DSYNC) - flags |= O_DSYNC; -#elif defined(O_SYNC) - flags |= O_SYNC; -#elif defined(O_FSYNC) - flags |= O_FSYNC; +#if MDBX_DEBUG + np->pgno = 0; #endif - break; - case MDBX_OPEN_DELETE: - flags = O_RDWR; - break; + VALGRIND_MAKE_MEM_UNDEFINED(np, size); + np->flags = 0; + np->pages = (pgno_t)num; + return np; +} + +void page_shadow_release(MDBX_env *env, page_t *dp, size_t npages) { + VALGRIND_MAKE_MEM_UNDEFINED(dp, pgno2bytes(env, npages)); + MDBX_ASAN_UNPOISON_MEMORY_REGION(dp, pgno2bytes(env, npages)); + if (unlikely(env->flags & MDBX_PAGEPERTURB)) + memset(dp, -1, pgno2bytes(env, npages)); + if (likely(npages == 1 && env->shadow_reserve_len < env->options.dp_reserve_limit)) { + MDBX_ASAN_POISON_MEMORY_REGION(dp, env->ps); + MDBX_ASAN_UNPOISON_MEMORY_REGION(&page_next(dp), sizeof(page_t *)); + page_next(dp) = env->shadow_reserve; + VALGRIND_MEMPOOL_FREE(env, ptr_disp(dp, -(ptrdiff_t)sizeof(size_t))); + env->shadow_reserve = dp; + env->shadow_reserve_len += 1; + } else { + /* large pages just get freed directly */ + void *const ptr = ptr_disp(dp, -(ptrdiff_t)sizeof(size_t)); + VALGRIND_MEMPOOL_FREE(env, ptr); + osal_free(ptr); } +} - const bool direct_nocache_for_copy = - env->me_psize >= env->me_os_psize && purpose == MDBX_OPEN_COPY; - if (direct_nocache_for_copy) { -#if defined(O_DIRECT) - flags |= O_DIRECT; -#endif /* O_DIRECT */ -#if defined(O_NOCACHE) - flags |= O_NOCACHE; -#endif /* O_NOCACHE */ +__cold static void page_kill(MDBX_txn *txn, page_t *mp, pgno_t pgno, size_t npages) { + MDBX_env *const env = txn->env; + DEBUG("kill %zu page(s) %" PRIaPGNO, npages, pgno); + eASSERT(env, pgno >= NUM_METAS && npages); + if (!is_frozen(txn, mp)) { + const size_t bytes = pgno2bytes(env, npages); + memset(mp, -1, bytes); + mp->pgno = pgno; + if ((txn->flags & MDBX_WRITEMAP) == 0) + osal_pwrite(env->lazy_fd, mp, bytes, pgno2bytes(env, pgno)); + } else { + struct iovec iov[MDBX_AUXILARY_IOV_MAX]; + iov[0].iov_len = env->ps; + iov[0].iov_base = ptr_disp(env->page_auxbuf, env->ps); + size_t iov_off = pgno2bytes(env, pgno), n = 1; + while (--npages) { + iov[n] = iov[0]; + if (++n == MDBX_AUXILARY_IOV_MAX) { + osal_pwritev(env->lazy_fd, iov, MDBX_AUXILARY_IOV_MAX, iov_off); + iov_off += pgno2bytes(env, MDBX_AUXILARY_IOV_MAX); + n = 0; + } + } + osal_pwritev(env->lazy_fd, iov, n, iov_off); } +} -#ifdef O_CLOEXEC - flags |= O_CLOEXEC; -#endif /* O_CLOEXEC */ +static inline bool suitable4loose(const MDBX_txn *txn, pgno_t pgno) { + /* TODO: + * 1) при включенной "экономии последовательностей" проверить, что + * страница не примыкает к какой-либо из уже находящийся в reclaimed. + * 2) стоит подумать над тем, чтобы при большом loose-списке отбрасывать + половину в reclaimed. */ + return txn->tw.loose_count < txn->env->options.dp_loose_limit && + (!MDBX_ENABLE_REFUND || + /* skip pages near to the end in favor of compactification */ + txn->geo.first_unallocated > pgno + txn->env->options.dp_loose_limit || + txn->geo.first_unallocated <= txn->env->options.dp_loose_limit); +} - /* Safeguard for https://libmdbx.dqdkfa.ru/dead-github/issues/144 */ -#if STDIN_FILENO == 0 && STDOUT_FILENO == 1 && STDERR_FILENO == 2 - int stub_fd0 = -1, stub_fd1 = -1, stub_fd2 = -1; - static const char dev_null[] = "/dev/null"; - if (!is_valid_fd(STDIN_FILENO)) { - WARNING("STD%s_FILENO/%d is invalid, open %s for temporary stub", "IN", - STDIN_FILENO, dev_null); - stub_fd0 = open(dev_null, O_RDONLY | O_NOCTTY); - } - if (!is_valid_fd(STDOUT_FILENO)) { - WARNING("STD%s_FILENO/%d is invalid, open %s for temporary stub", "OUT", - STDOUT_FILENO, dev_null); - stub_fd1 = open(dev_null, O_WRONLY | O_NOCTTY); - } - if (!is_valid_fd(STDERR_FILENO)) { - WARNING("STD%s_FILENO/%d is invalid, open %s for temporary stub", "ERR", - STDERR_FILENO, dev_null); - stub_fd2 = open(dev_null, O_WRONLY | O_NOCTTY); - } -#else -#error "Unexpected or unsupported UNIX or POSIX system" -#endif /* STDIN_FILENO == 0 && STDERR_FILENO == 2 */ +/* Retire, loosen or free a single page. + * + * For dirty pages, saves single pages to a list for future reuse in this same + * txn. It has been pulled from the GC and already resides on the dirty list, + * but has been deleted. Use these pages first before pulling again from the GC. + * + * If the page wasn't dirtied in this txn, just add it + * to this txn's free list. */ +int page_retire_ex(MDBX_cursor *mc, const pgno_t pgno, page_t *mp /* maybe null */, + unsigned pageflags /* maybe unknown/zero */) { + int rc; + MDBX_txn *const txn = mc->txn; + tASSERT(txn, !mp || (mp->pgno == pgno && mp->flags == pageflags)); - *fd = open(pathname, flags, unix_mode_bits); -#if defined(O_DIRECT) - if (*fd < 0 && (flags & O_DIRECT) && - (errno == EINVAL || errno == EAFNOSUPPORT)) { - flags &= ~(O_DIRECT | O_EXCL); - *fd = open(pathname, flags, unix_mode_bits); - } -#endif /* O_DIRECT */ + /* During deleting entire subtrees, it is reasonable and possible to avoid + * reading leaf pages, i.e. significantly reduce hard page-faults & IOPs: + * - mp is null, i.e. the page has not yet been read; + * - pagetype is known and the P_LEAF bit is set; + * - we can determine the page status via scanning the lists + * of dirty and spilled pages. + * + * On the other hand, this could be suboptimal for WRITEMAP mode, since + * requires support the list of dirty pages and avoid explicit spilling. + * So for flexibility and avoid extra internal dependencies we just + * fallback to reading if dirty list was not allocated yet. */ + size_t di = 0, si = 0, npages = 1; + enum page_status { unknown, frozen, spilled, shadowed, modifable } status = unknown; - if (*fd < 0 && errno == EACCES && purpose == MDBX_OPEN_LCK) { - struct stat unused; - if (stat(pathname, &unused) == 0 || errno != ENOENT) - errno = EACCES /* restore errno if file exists */; + if (unlikely(!mp)) { + if (ASSERT_ENABLED() && pageflags) { + pgr_t check; + check = page_get_any(mc, pgno, txn->front_txnid); + if (unlikely(check.err != MDBX_SUCCESS)) + return check.err; + tASSERT(txn, ((unsigned)check.page->flags & ~P_SPILLED) == (pageflags & ~P_FROZEN)); + tASSERT(txn, !(pageflags & P_FROZEN) || is_frozen(txn, check.page)); + } + if (pageflags & P_FROZEN) { + status = frozen; + if (ASSERT_ENABLED()) { + for (MDBX_txn *scan = txn; scan; scan = scan->parent) { + tASSERT(txn, !txn->tw.spilled.list || !spill_search(scan, pgno)); + tASSERT(txn, !scan->tw.dirtylist || !debug_dpl_find(scan, pgno)); + } + } + goto status_done; + } else if (pageflags && txn->tw.dirtylist) { + if ((di = dpl_exist(txn, pgno)) != 0) { + mp = txn->tw.dirtylist->items[di].ptr; + tASSERT(txn, is_modifable(txn, mp)); + status = modifable; + goto status_done; + } + if ((si = spill_search(txn, pgno)) != 0) { + status = spilled; + goto status_done; + } + for (MDBX_txn *parent = txn->parent; parent; parent = parent->parent) { + if (dpl_exist(parent, pgno)) { + status = shadowed; + goto status_done; + } + if (spill_search(parent, pgno)) { + status = spilled; + goto status_done; + } + } + status = frozen; + goto status_done; + } + + pgr_t pg = page_get_any(mc, pgno, txn->front_txnid); + if (unlikely(pg.err != MDBX_SUCCESS)) + return pg.err; + mp = pg.page; + tASSERT(txn, !pageflags || mp->flags == pageflags); + pageflags = mp->flags; } - /* Safeguard for https://libmdbx.dqdkfa.ru/dead-github/issues/144 */ -#if STDIN_FILENO == 0 && STDOUT_FILENO == 1 && STDERR_FILENO == 2 - if (*fd == STDIN_FILENO) { - WARNING("Got STD%s_FILENO/%d, avoid using it by dup(fd)", "IN", - STDIN_FILENO); - assert(stub_fd0 == -1); - *fd = dup(stub_fd0 = *fd); + if (is_frozen(txn, mp)) { + status = frozen; + tASSERT(txn, !is_modifable(txn, mp)); + tASSERT(txn, !is_spilled(txn, mp)); + tASSERT(txn, !is_shadowed(txn, mp)); + tASSERT(txn, !debug_dpl_find(txn, pgno)); + tASSERT(txn, !txn->tw.spilled.list || !spill_search(txn, pgno)); + } else if (is_modifable(txn, mp)) { + status = modifable; + if (txn->tw.dirtylist) + di = dpl_exist(txn, pgno); + tASSERT(txn, (txn->flags & MDBX_WRITEMAP) || !is_spilled(txn, mp)); + tASSERT(txn, !txn->tw.spilled.list || !spill_search(txn, pgno)); + } else if (is_shadowed(txn, mp)) { + status = shadowed; + tASSERT(txn, !txn->tw.spilled.list || !spill_search(txn, pgno)); + tASSERT(txn, !debug_dpl_find(txn, pgno)); + } else { + tASSERT(txn, is_spilled(txn, mp)); + status = spilled; + si = spill_search(txn, pgno); + tASSERT(txn, !debug_dpl_find(txn, pgno)); } - if (*fd == STDOUT_FILENO) { - WARNING("Got STD%s_FILENO/%d, avoid using it by dup(fd)", "OUT", - STDOUT_FILENO); - assert(stub_fd1 == -1); - *fd = dup(stub_fd1 = *fd); + +status_done: + if (likely((pageflags & P_LARGE) == 0)) { + STATIC_ASSERT(P_BRANCH == 1); + const bool is_branch = pageflags & P_BRANCH; + cASSERT(mc, ((pageflags & P_LEAF) == 0) == is_branch); + if (unlikely(mc->flags & z_inner)) { + tree_t *outer = outer_tree(mc); + cASSERT(mc, !is_branch || outer->branch_pages > 0); + outer->branch_pages -= is_branch; + cASSERT(mc, is_branch || outer->leaf_pages > 0); + outer->leaf_pages -= 1 - is_branch; + } + cASSERT(mc, !is_branch || mc->tree->branch_pages > 0); + mc->tree->branch_pages -= is_branch; + cASSERT(mc, is_branch || mc->tree->leaf_pages > 0); + mc->tree->leaf_pages -= 1 - is_branch; + } else { + npages = mp->pages; + cASSERT(mc, mc->tree->large_pages >= npages); + mc->tree->large_pages -= (pgno_t)npages; } - if (*fd == STDERR_FILENO) { - WARNING("Got STD%s_FILENO/%d, avoid using it by dup(fd)", "ERR", - STDERR_FILENO); - assert(stub_fd2 == -1); - *fd = dup(stub_fd2 = *fd); + + if (status == frozen) { + retire: + DEBUG("retire %zu page %" PRIaPGNO, npages, pgno); + rc = pnl_append_span(&txn->tw.retired_pages, pgno, npages); + tASSERT(txn, dpl_check(txn)); + return rc; } - if (stub_fd0 != -1) - close(stub_fd0); - if (stub_fd1 != -1) - close(stub_fd1); - if (stub_fd2 != -1) - close(stub_fd2); - if (*fd >= STDIN_FILENO && *fd <= STDERR_FILENO) { - ERROR("Rejecting the use of a FD in the range " - "STDIN_FILENO/%d..STDERR_FILENO/%d to prevent database corruption", - STDIN_FILENO, STDERR_FILENO); - close(*fd); - return EBADF; + + /* Возврат страниц в нераспределенный "хвост" БД. + * Содержимое страниц не уничтожается, а для вложенных транзакций граница + * нераспределенного "хвоста" БД сдвигается только при их коммите. */ + if (MDBX_ENABLE_REFUND && unlikely(pgno + npages == txn->geo.first_unallocated)) { + const char *kind = nullptr; + if (status == modifable) { + /* Страница испачкана в этой транзакции, но до этого могла быть + * аллоцирована, испачкана и пролита в одной из родительских транзакций. + * Её МОЖНО вытолкнуть в нераспределенный хвост. */ + kind = "dirty"; + /* Remove from dirty list */ + page_wash(txn, di, mp, npages); + } else if (si) { + /* Страница пролита в этой транзакции, т.е. она аллоцирована + * и запачкана в этой или одной из родительских транзакций. + * Её МОЖНО вытолкнуть в нераспределенный хвост. */ + kind = "spilled"; + tASSERT(txn, status == spilled); + spill_remove(txn, si, npages); + } else { + /* Страница аллоцирована, запачкана и возможно пролита в одной + * из родительских транзакций. + * Её МОЖНО вытолкнуть в нераспределенный хвост. */ + kind = "parent's"; + if (ASSERT_ENABLED() && mp) { + kind = nullptr; + for (MDBX_txn *parent = txn->parent; parent; parent = parent->parent) { + if (spill_search(parent, pgno)) { + kind = "parent-spilled"; + tASSERT(txn, status == spilled); + break; + } + if (mp == debug_dpl_find(parent, pgno)) { + kind = "parent-dirty"; + tASSERT(txn, status == shadowed); + break; + } + } + tASSERT(txn, kind != nullptr); + } + tASSERT(txn, status == spilled || status == shadowed); + } + DEBUG("refunded %zu %s page %" PRIaPGNO, npages, kind, pgno); + txn->geo.first_unallocated = pgno; + txn_refund(txn); + return MDBX_SUCCESS; } -#else -#error "Unexpected or unsupported UNIX or POSIX system" -#endif /* STDIN_FILENO == 0 && STDERR_FILENO == 2 */ - - if (*fd < 0) - return errno; - -#if defined(FD_CLOEXEC) && !defined(O_CLOEXEC) - const int fd_flags = fcntl(*fd, F_GETFD); - if (fd_flags != -1) - (void)fcntl(*fd, F_SETFD, fd_flags | FD_CLOEXEC); -#endif /* FD_CLOEXEC && !O_CLOEXEC */ - if (direct_nocache_for_copy) { -#if defined(F_NOCACHE) && !defined(O_NOCACHE) - (void)fcntl(*fd, F_NOCACHE, 1); -#endif /* F_NOCACHE */ - } + if (status == modifable) { + /* Dirty page from this transaction */ + /* If suitable we can reuse it through loose list */ + if (likely(npages == 1 && suitable4loose(txn, pgno)) && (di || !txn->tw.dirtylist)) { + DEBUG("loosen dirty page %" PRIaPGNO, pgno); + if (MDBX_DEBUG != 0 || unlikely(txn->env->flags & MDBX_PAGEPERTURB)) + memset(page_data(mp), -1, txn->env->ps - PAGEHDRSZ); + mp->txnid = INVALID_TXNID; + mp->flags = P_LOOSE; + page_next(mp) = txn->tw.loose_pages; + txn->tw.loose_pages = mp; + txn->tw.loose_count++; +#if MDBX_ENABLE_REFUND + txn->tw.loose_refund_wl = (pgno + 2 > txn->tw.loose_refund_wl) ? pgno + 2 : txn->tw.loose_refund_wl; +#endif /* MDBX_ENABLE_REFUND */ + VALGRIND_MAKE_MEM_NOACCESS(page_data(mp), txn->env->ps - PAGEHDRSZ); + MDBX_ASAN_POISON_MEMORY_REGION(page_data(mp), txn->env->ps - PAGEHDRSZ); + return MDBX_SUCCESS; + } +#if !MDBX_DEBUG && !defined(ENABLE_MEMCHECK) && !defined(__SANITIZE_ADDRESS__) + if (unlikely(txn->env->flags & MDBX_PAGEPERTURB)) #endif - return MDBX_SUCCESS; -} + { + /* Страница могла быть изменена в одной из родительских транзакций, + * в том числе, позже выгружена и затем снова загружена и изменена. + * В обоих случаях её нельзя затирать на диске и помечать недоступной + * в asan и/или valgrind */ + for (MDBX_txn *parent = txn->parent; parent && (parent->flags & MDBX_TXN_SPILLS); parent = parent->parent) { + if (spill_intersect(parent, pgno, npages)) + goto skip_invalidate; + if (dpl_intersect(parent, pgno, npages)) + goto skip_invalidate; + } -MDBX_INTERNAL_FUNC int osal_closefile(mdbx_filehandle_t fd) { -#if defined(_WIN32) || defined(_WIN64) - return CloseHandle(fd) ? MDBX_SUCCESS : (int)GetLastError(); -#else - assert(fd > STDERR_FILENO); - return (close(fd) == 0) ? MDBX_SUCCESS : errno; +#if defined(ENABLE_MEMCHECK) || defined(__SANITIZE_ADDRESS__) + if (MDBX_DEBUG != 0 || unlikely(txn->env->flags & MDBX_PAGEPERTURB)) #endif -} + page_kill(txn, mp, pgno, npages); + if ((txn->flags & MDBX_WRITEMAP) == 0) { + VALGRIND_MAKE_MEM_NOACCESS(page_data(pgno2page(txn->env, pgno)), pgno2bytes(txn->env, npages) - PAGEHDRSZ); + MDBX_ASAN_POISON_MEMORY_REGION(page_data(pgno2page(txn->env, pgno)), pgno2bytes(txn->env, npages) - PAGEHDRSZ); + } + } + skip_invalidate: -MDBX_INTERNAL_FUNC int osal_pread(mdbx_filehandle_t fd, void *buf, size_t bytes, - uint64_t offset) { - if (bytes > MAX_WRITE) - return MDBX_EINVAL; -#if defined(_WIN32) || defined(_WIN64) - OVERLAPPED ov; - ov.hEvent = 0; - ov.Offset = (DWORD)offset; - ov.OffsetHigh = HIGH_DWORD(offset); + /* wash dirty page */ + page_wash(txn, di, mp, npages); - DWORD read = 0; - if (unlikely(!ReadFile(fd, buf, (DWORD)bytes, &read, &ov))) { - int rc = (int)GetLastError(); - return (rc == MDBX_SUCCESS) ? /* paranoia */ ERROR_READ_FAULT : rc; - } -#else - STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), - "libmdbx requires 64-bit file I/O on 64-bit systems"); - intptr_t read = pread(fd, buf, bytes, offset); - if (read < 0) { - int rc = errno; - return (rc == MDBX_SUCCESS) ? /* paranoia */ MDBX_EIO : rc; + reclaim: + DEBUG("reclaim %zu %s page %" PRIaPGNO, npages, "dirty", pgno); + rc = pnl_insert_span(&txn->tw.repnl, pgno, npages); + tASSERT(txn, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); + tASSERT(txn, dpl_check(txn)); + return rc; } -#endif - return (bytes == (size_t)read) ? MDBX_SUCCESS : MDBX_ENODATA; -} - -MDBX_INTERNAL_FUNC int osal_pwrite(mdbx_filehandle_t fd, const void *buf, - size_t bytes, uint64_t offset) { - while (true) { -#if defined(_WIN32) || defined(_WIN64) - OVERLAPPED ov; - ov.hEvent = 0; - ov.Offset = (DWORD)offset; - ov.OffsetHigh = HIGH_DWORD(offset); - DWORD written; - if (unlikely(!WriteFile( - fd, buf, likely(bytes <= MAX_WRITE) ? (DWORD)bytes : MAX_WRITE, - &written, &ov))) - return (int)GetLastError(); - if (likely(bytes == written)) - return MDBX_SUCCESS; -#else - STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), - "libmdbx requires 64-bit file I/O on 64-bit systems"); - const intptr_t written = - pwrite(fd, buf, likely(bytes <= MAX_WRITE) ? bytes : MAX_WRITE, offset); - if (likely(bytes == (size_t)written)) - return MDBX_SUCCESS; - if (written < 0) { - const int rc = errno; - if (rc != EINTR) - return rc; - continue; + if (si) { + /* Page ws spilled in this txn */ + spill_remove(txn, si, npages); + /* Страница могла быть выделена и затем пролита в этой транзакции, + * тогда её необходимо поместить в reclaimed-список. + * Либо она могла быть выделена в одной из родительских транзакций и затем + * пролита в этой транзакции, тогда её необходимо поместить в + * retired-список для последующей фильтрации при коммите. */ + for (MDBX_txn *parent = txn->parent; parent; parent = parent->parent) { + if (dpl_exist(parent, pgno)) + goto retire; } -#endif - bytes -= written; - offset += written; - buf = ptr_disp(buf, written); + /* Страница точно была выделена в этой транзакции + * и теперь может быть использована повторно. */ + goto reclaim; } -} -MDBX_INTERNAL_FUNC int osal_write(mdbx_filehandle_t fd, const void *buf, - size_t bytes) { - while (true) { -#if defined(_WIN32) || defined(_WIN64) - DWORD written; - if (unlikely(!WriteFile( - fd, buf, likely(bytes <= MAX_WRITE) ? (DWORD)bytes : MAX_WRITE, - &written, nullptr))) - return (int)GetLastError(); - if (likely(bytes == written)) - return MDBX_SUCCESS; -#else - STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), - "libmdbx requires 64-bit file I/O on 64-bit systems"); - const intptr_t written = - write(fd, buf, likely(bytes <= MAX_WRITE) ? bytes : MAX_WRITE); - if (likely(bytes == (size_t)written)) - return MDBX_SUCCESS; - if (written < 0) { - const int rc = errno; - if (rc != EINTR) - return rc; - continue; + if (status == shadowed) { + /* Dirty page MUST BE a clone from (one of) parent transaction(s). */ + if (ASSERT_ENABLED()) { + const page_t *parent_dp = nullptr; + /* Check parent(s)'s dirty lists. */ + for (MDBX_txn *parent = txn->parent; parent && !parent_dp; parent = parent->parent) { + tASSERT(txn, !spill_search(parent, pgno)); + parent_dp = debug_dpl_find(parent, pgno); + } + tASSERT(txn, parent_dp && (!mp || parent_dp == mp)); } -#endif - bytes -= written; - buf = ptr_disp(buf, written); + /* Страница была выделена в родительской транзакции и теперь может быть + * использована повторно, но только внутри этой транзакции, либо дочерних. + */ + goto reclaim; } + + /* Страница может входить в доступный читателям MVCC-снимок, либо же она + * могла быть выделена, а затем пролита в одной из родительских + * транзакций. Поэтому пока помещаем её в retired-список, который будет + * фильтроваться относительно dirty- и spilled-списков родительских + * транзакций при коммите дочерних транзакций, либо же будет записан + * в GC в неизменном виде. */ + goto retire; } -int osal_pwritev(mdbx_filehandle_t fd, struct iovec *iov, size_t sgvcnt, - uint64_t offset) { - size_t expected = 0; - for (size_t i = 0; i < sgvcnt; ++i) - expected += iov[i].iov_len; -#if !MDBX_HAVE_PWRITEV - size_t written = 0; - for (size_t i = 0; i < sgvcnt; ++i) { - int rc = osal_pwrite(fd, iov[i].iov_base, iov[i].iov_len, offset); - if (unlikely(rc != MDBX_SUCCESS)) - return rc; - written += iov[i].iov_len; - offset += iov[i].iov_len; +__hot int __must_check_result page_dirty(MDBX_txn *txn, page_t *mp, size_t npages) { + tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0); + mp->txnid = txn->front_txnid; + if (!txn->tw.dirtylist) { + tASSERT(txn, (txn->flags & MDBX_WRITEMAP) != 0 && !MDBX_AVOID_MSYNC); + txn->tw.writemap_dirty_npages += npages; + tASSERT(txn, txn->tw.spilled.list == nullptr); + return MDBX_SUCCESS; } - return (expected == written) ? MDBX_SUCCESS - : MDBX_EIO /* ERROR_WRITE_FAULT */; -#else + tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); + +#if xMDBX_DEBUG_SPILLING == 2 + txn->env->debug_dirtied_act += 1; + ENSURE(txn->env, txn->env->debug_dirtied_act < txn->env->debug_dirtied_est); + ENSURE(txn->env, txn->tw.dirtyroom + txn->tw.loose_count > 0); +#endif /* xMDBX_DEBUG_SPILLING == 2 */ + int rc; - intptr_t written; - do { - STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), - "libmdbx requires 64-bit file I/O on 64-bit systems"); - written = pwritev(fd, iov, sgvcnt, offset); - if (likely(expected == (size_t)written)) - return MDBX_SUCCESS; - rc = errno; - } while (rc == EINTR); - return (written < 0) ? rc : MDBX_EIO /* Use which error code? */; -#endif -} + if (unlikely(txn->tw.dirtyroom == 0)) { + if (txn->tw.loose_count) { + page_t *lp = txn->tw.loose_pages; + DEBUG("purge-and-reclaim loose page %" PRIaPGNO, lp->pgno); + rc = pnl_insert_span(&txn->tw.repnl, lp->pgno, 1); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + size_t di = dpl_search(txn, lp->pgno); + tASSERT(txn, txn->tw.dirtylist->items[di].ptr == lp); + dpl_remove(txn, di); + MDBX_ASAN_UNPOISON_MEMORY_REGION(&page_next(lp), sizeof(page_t *)); + VALGRIND_MAKE_MEM_DEFINED(&page_next(lp), sizeof(page_t *)); + txn->tw.loose_pages = page_next(lp); + txn->tw.loose_count--; + txn->tw.dirtyroom++; + if (!MDBX_AVOID_MSYNC || !(txn->flags & MDBX_WRITEMAP)) + page_shadow_release(txn->env, lp, 1); + } else { + ERROR("Dirtyroom is depleted, DPL length %zu", txn->tw.dirtylist->length); + if (!MDBX_AVOID_MSYNC || !(txn->flags & MDBX_WRITEMAP)) + page_shadow_release(txn->env, mp, npages); + return MDBX_TXN_FULL; + } + } -MDBX_INTERNAL_FUNC int osal_fsync(mdbx_filehandle_t fd, - enum osal_syncmode_bits mode_bits) { -#if defined(_WIN32) || defined(_WIN64) - if ((mode_bits & (MDBX_SYNC_DATA | MDBX_SYNC_IODQ)) && !FlushFileBuffers(fd)) - return (int)GetLastError(); + rc = dpl_append(txn, mp->pgno, mp, npages); + if (unlikely(rc != MDBX_SUCCESS)) { + bailout: + txn->flags |= MDBX_TXN_ERROR; + return rc; + } + txn->tw.dirtyroom--; + tASSERT(txn, dpl_check(txn)); return MDBX_SUCCESS; -#else +} -#if defined(__APPLE__) && \ - MDBX_OSX_SPEED_INSTEADOF_DURABILITY == MDBX_OSX_WANNA_DURABILITY - if (mode_bits & MDBX_SYNC_IODQ) - return likely(fcntl(fd, F_FULLFSYNC) != -1) ? MDBX_SUCCESS : errno; -#endif /* MacOS */ +void recalculate_subpage_thresholds(MDBX_env *env) { + size_t whole = env->leaf_nodemax - NODESIZE; + env->subpage_limit = (whole * env->options.subpage.limit + 32767) >> 16; + whole = env->subpage_limit; + env->subpage_reserve_limit = (whole * env->options.subpage.reserve_limit + 32767) >> 16; + eASSERT(env, env->leaf_nodemax >= env->subpage_limit + NODESIZE); + eASSERT(env, env->subpage_limit >= env->subpage_reserve_limit); - /* LY: This approach is always safe and without appreciable performance - * degradation, even on a kernel with fdatasync's bug. - * - * For more info about of a corresponding fdatasync() bug - * see http://www.spinics.net/lists/linux-ext4/msg33714.html */ - while (1) { - switch (mode_bits & (MDBX_SYNC_DATA | MDBX_SYNC_SIZE)) { - case MDBX_SYNC_NONE: - case MDBX_SYNC_KICK: - return MDBX_SUCCESS /* nothing to do */; -#if defined(_POSIX_SYNCHRONIZED_IO) && _POSIX_SYNCHRONIZED_IO > 0 - case MDBX_SYNC_DATA: - if (likely(fdatasync(fd) == 0)) - return MDBX_SUCCESS; - break /* error */; -#if defined(__linux__) || defined(__gnu_linux__) - case MDBX_SYNC_SIZE: - assert(linux_kernel_version >= 0x03060000); - return MDBX_SUCCESS; -#endif /* Linux */ -#endif /* _POSIX_SYNCHRONIZED_IO > 0 */ - default: - if (likely(fsync(fd) == 0)) - return MDBX_SUCCESS; - } + whole = env->leaf_nodemax; + env->subpage_room_threshold = (whole * env->options.subpage.room_threshold + 32767) >> 16; + env->subpage_reserve_prereq = (whole * env->options.subpage.reserve_prereq + 32767) >> 16; + if (env->subpage_room_threshold + env->subpage_reserve_limit > (intptr_t)page_space(env)) + env->subpage_reserve_prereq = page_space(env); + else if (env->subpage_reserve_prereq < env->subpage_room_threshold + env->subpage_reserve_limit) + env->subpage_reserve_prereq = env->subpage_room_threshold + env->subpage_reserve_limit; + eASSERT(env, env->subpage_reserve_prereq >= env->subpage_room_threshold + env->subpage_reserve_limit); +} - int rc = errno; - if (rc != EINTR) - return rc; +size_t page_subleaf2_reserve(const MDBX_env *env, size_t host_page_room, size_t subpage_len, size_t item_len) { + eASSERT(env, (subpage_len & 1) == 0); + eASSERT(env, env->leaf_nodemax >= env->subpage_limit + NODESIZE); + size_t reserve = 0; + for (size_t n = 0; n < 5 && reserve + item_len <= env->subpage_reserve_limit && + EVEN_CEIL(subpage_len + item_len) <= env->subpage_limit && + host_page_room >= env->subpage_reserve_prereq + EVEN_CEIL(subpage_len + item_len); + ++n) { + subpage_len += item_len; + reserve += item_len; } -#endif + return reserve + (subpage_len & 1); } +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 -int osal_filesize(mdbx_filehandle_t fd, uint64_t *length) { -#if defined(_WIN32) || defined(_WIN64) - BY_HANDLE_FILE_INFORMATION info; - if (!GetFileInformationByHandle(fd, &info)) - return (int)GetLastError(); - *length = info.nFileSizeLow | (uint64_t)info.nFileSizeHigh << 32; -#else - struct stat st; +pnl_t pnl_alloc(size_t size) { + size_t bytes = pnl_size2bytes(size); + pnl_t pnl = osal_malloc(bytes); + if (likely(pnl)) { +#ifdef osal_malloc_usable_size + bytes = osal_malloc_usable_size(pnl); +#endif /* osal_malloc_usable_size */ + pnl[0] = pnl_bytes2size(bytes); + assert(pnl[0] >= size); + pnl += 1; + *pnl = 0; + } + return pnl; +} - STATIC_ASSERT_MSG(sizeof(off_t) <= sizeof(uint64_t), - "libmdbx requires 64-bit file I/O on 64-bit systems"); - if (fstat(fd, &st)) - return errno; +void pnl_free(pnl_t pnl) { + if (likely(pnl)) + osal_free(pnl - 1); +} - *length = st.st_size; -#endif - return MDBX_SUCCESS; +void pnl_shrink(pnl_t __restrict *__restrict ppnl) { + assert(pnl_bytes2size(pnl_size2bytes(MDBX_PNL_INITIAL)) >= MDBX_PNL_INITIAL && + pnl_bytes2size(pnl_size2bytes(MDBX_PNL_INITIAL)) < MDBX_PNL_INITIAL * 3 / 2); + assert(MDBX_PNL_GETSIZE(*ppnl) <= PAGELIST_LIMIT && MDBX_PNL_ALLOCLEN(*ppnl) >= MDBX_PNL_GETSIZE(*ppnl)); + MDBX_PNL_SETSIZE(*ppnl, 0); + if (unlikely(MDBX_PNL_ALLOCLEN(*ppnl) > + MDBX_PNL_INITIAL * (MDBX_PNL_PREALLOC_FOR_RADIXSORT ? 8 : 4) - MDBX_CACHELINE_SIZE / sizeof(pgno_t))) { + size_t bytes = pnl_size2bytes(MDBX_PNL_INITIAL * 2); + pnl_t pnl = osal_realloc(*ppnl - 1, bytes); + if (likely(pnl)) { +#ifdef osal_malloc_usable_size + bytes = osal_malloc_usable_size(pnl); +#endif /* osal_malloc_usable_size */ + *pnl = pnl_bytes2size(bytes); + *ppnl = pnl + 1; + } + } } -MDBX_INTERNAL_FUNC int osal_is_pipe(mdbx_filehandle_t fd) { -#if defined(_WIN32) || defined(_WIN64) - switch (GetFileType(fd)) { - case FILE_TYPE_DISK: - return MDBX_RESULT_FALSE; - case FILE_TYPE_CHAR: - case FILE_TYPE_PIPE: - return MDBX_RESULT_TRUE; - default: - return (int)GetLastError(); +int pnl_reserve(pnl_t __restrict *__restrict ppnl, const size_t wanna) { + const size_t allocated = MDBX_PNL_ALLOCLEN(*ppnl); + assert(MDBX_PNL_GETSIZE(*ppnl) <= PAGELIST_LIMIT && MDBX_PNL_ALLOCLEN(*ppnl) >= MDBX_PNL_GETSIZE(*ppnl)); + if (likely(allocated >= wanna)) + return MDBX_SUCCESS; + + if (unlikely(wanna > /* paranoia */ PAGELIST_LIMIT)) { + ERROR("PNL too long (%zu > %zu)", wanna, (size_t)PAGELIST_LIMIT); + return MDBX_TXN_FULL; } -#else - struct stat info; - if (fstat(fd, &info)) - return errno; - switch (info.st_mode & S_IFMT) { - case S_IFBLK: - case S_IFREG: - return MDBX_RESULT_FALSE; - case S_IFCHR: - case S_IFIFO: - case S_IFSOCK: - return MDBX_RESULT_TRUE; - case S_IFDIR: - case S_IFLNK: - default: - return MDBX_INCOMPATIBLE; + + const size_t size = (wanna + wanna - allocated < PAGELIST_LIMIT) ? wanna + wanna - allocated : PAGELIST_LIMIT; + size_t bytes = pnl_size2bytes(size); + pnl_t pnl = osal_realloc(*ppnl - 1, bytes); + if (likely(pnl)) { +#ifdef osal_malloc_usable_size + bytes = osal_malloc_usable_size(pnl); +#endif /* osal_malloc_usable_size */ + *pnl = pnl_bytes2size(bytes); + assert(*pnl >= wanna); + *ppnl = pnl + 1; + return MDBX_SUCCESS; } -#endif + return MDBX_ENOMEM; } -MDBX_INTERNAL_FUNC int osal_ftruncate(mdbx_filehandle_t fd, uint64_t length) { -#if defined(_WIN32) || defined(_WIN64) - if (mdbx_SetFileInformationByHandle) { - FILE_END_OF_FILE_INFO EndOfFileInfo; - EndOfFileInfo.EndOfFile.QuadPart = length; - return mdbx_SetFileInformationByHandle(fd, FileEndOfFileInfo, - &EndOfFileInfo, - sizeof(FILE_END_OF_FILE_INFO)) - ? MDBX_SUCCESS - : (int)GetLastError(); - } else { - LARGE_INTEGER li; - li.QuadPart = length; - return (SetFilePointerEx(fd, li, NULL, FILE_BEGIN) && SetEndOfFile(fd)) - ? MDBX_SUCCESS - : (int)GetLastError(); +static __always_inline int __must_check_result pnl_append_stepped(unsigned step, __restrict pnl_t *ppnl, pgno_t pgno, + size_t n) { + assert(n > 0); + int rc = pnl_need(ppnl, n); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + + const pnl_t pnl = *ppnl; + if (likely(n == 1)) { + pnl_append_prereserved(pnl, pgno); + return MDBX_SUCCESS; } -#else - STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), - "libmdbx requires 64-bit file I/O on 64-bit systems"); - return ftruncate(fd, length) == 0 ? MDBX_SUCCESS : errno; -#endif -} -MDBX_INTERNAL_FUNC int osal_fseek(mdbx_filehandle_t fd, uint64_t pos) { -#if defined(_WIN32) || defined(_WIN64) - LARGE_INTEGER li; - li.QuadPart = pos; - return SetFilePointerEx(fd, li, NULL, FILE_BEGIN) ? MDBX_SUCCESS - : (int)GetLastError(); +#if MDBX_PNL_ASCENDING + size_t w = MDBX_PNL_GETSIZE(pnl); + do { + pnl[++w] = pgno; + pgno += step; + } while (--n); + MDBX_PNL_SETSIZE(pnl, w); #else - STATIC_ASSERT_MSG(sizeof(off_t) >= sizeof(size_t), - "libmdbx requires 64-bit file I/O on 64-bit systems"); - return (lseek(fd, pos, SEEK_SET) < 0) ? errno : MDBX_SUCCESS; + size_t w = MDBX_PNL_GETSIZE(pnl) + n; + MDBX_PNL_SETSIZE(pnl, w); + do { + pnl[w--] = pgno; + pgno += step; + } while (--n); #endif + return MDBX_SUCCESS; } -/*----------------------------------------------------------------------------*/ - -MDBX_INTERNAL_FUNC int -osal_thread_create(osal_thread_t *thread, - THREAD_RESULT(THREAD_CALL *start_routine)(void *), - void *arg) { -#if defined(_WIN32) || defined(_WIN64) - *thread = CreateThread(NULL, 0, start_routine, arg, 0, NULL); - return *thread ? MDBX_SUCCESS : (int)GetLastError(); -#else - return pthread_create(thread, NULL, start_routine, arg); -#endif +__hot int __must_check_result spill_append_span(__restrict pnl_t *ppnl, pgno_t pgno, size_t n) { + return pnl_append_stepped(2, ppnl, pgno << 1, n); } -MDBX_INTERNAL_FUNC int osal_thread_join(osal_thread_t thread) { -#if defined(_WIN32) || defined(_WIN64) - DWORD code = WaitForSingleObject(thread, INFINITE); - return waitstatus2errcode(code); -#else - void *unused_retval = &unused_retval; - return pthread_join(thread, &unused_retval); -#endif +__hot int __must_check_result pnl_append_span(__restrict pnl_t *ppnl, pgno_t pgno, size_t n) { + return pnl_append_stepped(1, ppnl, pgno, n); } -/*----------------------------------------------------------------------------*/ +__hot int __must_check_result pnl_insert_span(__restrict pnl_t *ppnl, pgno_t pgno, size_t n) { + assert(n > 0); + int rc = pnl_need(ppnl, n); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; -MDBX_INTERNAL_FUNC int osal_msync(const osal_mmap_t *map, size_t offset, - size_t length, - enum osal_syncmode_bits mode_bits) { - if (!MDBX_MMAP_USE_MS_ASYNC && mode_bits == MDBX_SYNC_NONE) - return MDBX_SUCCESS; + const pnl_t pnl = *ppnl; + size_t r = MDBX_PNL_GETSIZE(pnl), w = r + n; + MDBX_PNL_SETSIZE(pnl, w); + while (r && MDBX_PNL_DISORDERED(pnl[r], pgno)) + pnl[w--] = pnl[r--]; + + for (pgno_t fill = MDBX_PNL_ASCENDING ? pgno + n : pgno; w > r; --w) + pnl[w] = MDBX_PNL_ASCENDING ? --fill : fill++; - void *ptr = ptr_disp(map->base, offset); -#if defined(_WIN32) || defined(_WIN64) - if (!FlushViewOfFile(ptr, length)) - return (int)GetLastError(); - if ((mode_bits & (MDBX_SYNC_DATA | MDBX_SYNC_IODQ)) && - !FlushFileBuffers(map->fd)) - return (int)GetLastError(); -#else -#if defined(__linux__) || defined(__gnu_linux__) - /* Since Linux 2.6.19, MS_ASYNC is in fact a no-op. The kernel properly - * tracks dirty pages and flushes ones as necessary. */ - // - // However, this behavior may be changed in custom kernels, - // so just leave such optimization to the libc discretion. - // NOTE: The MDBX_MMAP_USE_MS_ASYNC must be defined to 1 for such cases. - // - // assert(linux_kernel_version > 0x02061300); - // if (mode_bits <= MDBX_SYNC_KICK) - // return MDBX_SUCCESS; -#endif /* Linux */ - if (msync(ptr, length, (mode_bits & MDBX_SYNC_DATA) ? MS_SYNC : MS_ASYNC)) - return errno; - if ((mode_bits & MDBX_SYNC_SIZE) && fsync(map->fd)) - return errno; -#endif return MDBX_SUCCESS; } -MDBX_INTERNAL_FUNC int osal_check_fs_rdonly(mdbx_filehandle_t handle, - const pathchar_t *pathname, - int err) { -#if defined(_WIN32) || defined(_WIN64) - (void)pathname; - (void)err; - if (!mdbx_GetVolumeInformationByHandleW) - return MDBX_ENOSYS; - DWORD unused, flags; - if (!mdbx_GetVolumeInformationByHandleW(handle, nullptr, 0, nullptr, &unused, - &flags, nullptr, 0)) - return (int)GetLastError(); - if ((flags & FILE_READ_ONLY_VOLUME) == 0) - return MDBX_EACCESS; -#else - struct statvfs info; - if (err != MDBX_ENOFILE) { - if (statvfs(pathname, &info) == 0) - return (info.f_flag & ST_RDONLY) ? MDBX_SUCCESS : err; - if (errno != MDBX_ENOFILE) - return errno; +__hot __noinline bool pnl_check(const const_pnl_t pnl, const size_t limit) { + assert(limit >= MIN_PAGENO - MDBX_ENABLE_REFUND); + if (likely(MDBX_PNL_GETSIZE(pnl))) { + if (unlikely(MDBX_PNL_GETSIZE(pnl) > PAGELIST_LIMIT)) + return false; + if (unlikely(MDBX_PNL_LEAST(pnl) < MIN_PAGENO)) + return false; + if (unlikely(MDBX_PNL_MOST(pnl) >= limit)) + return false; + + if ((!MDBX_DISABLE_VALIDATION || AUDIT_ENABLED()) && likely(MDBX_PNL_GETSIZE(pnl) > 1)) { + const pgno_t *scan = MDBX_PNL_BEGIN(pnl); + const pgno_t *const end = MDBX_PNL_END(pnl); + pgno_t prev = *scan++; + do { + if (unlikely(!MDBX_PNL_ORDERED(prev, *scan))) + return false; + prev = *scan; + } while (likely(++scan != end)); + } } - if (fstatvfs(handle, &info)) - return errno; - if ((info.f_flag & ST_RDONLY) == 0) - return (err == MDBX_ENOFILE) ? MDBX_EACCESS : err; -#endif /* !Windows */ - return MDBX_SUCCESS; + return true; } -MDBX_INTERNAL_FUNC int osal_check_fs_incore(mdbx_filehandle_t handle) { -#if defined(_WIN32) || defined(_WIN64) - (void)handle; +static __always_inline void pnl_merge_inner(pgno_t *__restrict dst, const pgno_t *__restrict src_a, + const pgno_t *__restrict src_b, + const pgno_t *__restrict const src_b_detent) { + do { +#if MDBX_HAVE_CMOV + const bool flag = MDBX_PNL_ORDERED(*src_b, *src_a); +#if defined(__LCC__) || __CLANG_PREREQ(13, 0) + // lcc 1.26: 13ШК (подготовка и первая итерация) + 7ШК (цикл), БЕЗ loop-mode + // gcc>=7: cmp+jmp с возвратом в тело цикла (WTF?) + // gcc<=6: cmov×3 + // clang<=12: cmov×3 + // clang>=13: cmov, set+add/sub + *dst = flag ? *src_a-- : *src_b--; #else - struct statfs statfs_info; - if (fstatfs(handle, &statfs_info)) - return errno; + // gcc: cmov, cmp+set+add/sub + // clang<=5: cmov×2, set+add/sub + // clang>=6: cmov, set+add/sub + *dst = flag ? *src_a : *src_b; + src_b += (ptrdiff_t)flag - 1; + src_a -= flag; +#endif + --dst; +#else /* MDBX_HAVE_CMOV */ + while (MDBX_PNL_ORDERED(*src_b, *src_a)) + *dst-- = *src_a--; + *dst-- = *src_b--; +#endif /* !MDBX_HAVE_CMOV */ + } while (likely(src_b > src_b_detent)); +} -#if defined(__OpenBSD__) - const unsigned type = 0; +__hot size_t pnl_merge(pnl_t dst, const pnl_t src) { + assert(pnl_check_allocated(dst, MAX_PAGENO + 1)); + assert(pnl_check(src, MAX_PAGENO + 1)); + const size_t src_len = MDBX_PNL_GETSIZE(src); + const size_t dst_len = MDBX_PNL_GETSIZE(dst); + size_t total = dst_len; + assert(MDBX_PNL_ALLOCLEN(dst) >= total); + if (likely(src_len > 0)) { + total += src_len; + if (!MDBX_DEBUG && total < (MDBX_HAVE_CMOV ? 21 : 12)) + goto avoid_call_libc_for_short_cases; + if (dst_len == 0 || MDBX_PNL_ORDERED(MDBX_PNL_LAST(dst), MDBX_PNL_FIRST(src))) + memcpy(MDBX_PNL_END(dst), MDBX_PNL_BEGIN(src), src_len * sizeof(pgno_t)); + else if (MDBX_PNL_ORDERED(MDBX_PNL_LAST(src), MDBX_PNL_FIRST(dst))) { + memmove(MDBX_PNL_BEGIN(dst) + src_len, MDBX_PNL_BEGIN(dst), dst_len * sizeof(pgno_t)); + memcpy(MDBX_PNL_BEGIN(dst), MDBX_PNL_BEGIN(src), src_len * sizeof(pgno_t)); + } else { + avoid_call_libc_for_short_cases: + dst[0] = /* the detent */ (MDBX_PNL_ASCENDING ? 0 : P_INVALID); + pnl_merge_inner(dst + total, dst + dst_len, src + src_len, src); + } + MDBX_PNL_SETSIZE(dst, total); + } + assert(pnl_check_allocated(dst, MAX_PAGENO + 1)); + return total; +} + +#if MDBX_PNL_ASCENDING +#define MDBX_PNL_EXTRACT_KEY(ptr) (*(ptr)) #else - const unsigned type = statfs_info.f_type; +#define MDBX_PNL_EXTRACT_KEY(ptr) (P_INVALID - *(ptr)) #endif - switch (type) { - case 0x28cd3d45 /* CRAMFS_MAGIC */: - case 0x858458f6 /* RAMFS_MAGIC */: - case 0x01021994 /* TMPFS_MAGIC */: - case 0x73717368 /* SQUASHFS_MAGIC */: - case 0x7275 /* ROMFS_MAGIC */: - return MDBX_RESULT_TRUE; - } +RADIXSORT_IMPL(pgno, pgno_t, MDBX_PNL_EXTRACT_KEY, MDBX_PNL_PREALLOC_FOR_RADIXSORT, 0) + +SORT_IMPL(pgno_sort, false, pgno_t, MDBX_PNL_ORDERED) + +__hot __noinline void pnl_sort_nochk(pnl_t pnl) { + if (likely(MDBX_PNL_GETSIZE(pnl) < MDBX_RADIXSORT_THRESHOLD) || + unlikely(!pgno_radixsort(&MDBX_PNL_FIRST(pnl), MDBX_PNL_GETSIZE(pnl)))) + pgno_sort(MDBX_PNL_BEGIN(pnl), MDBX_PNL_END(pnl)); +} + +SEARCH_IMPL(pgno_bsearch, pgno_t, pgno_t, MDBX_PNL_ORDERED) + +__hot __noinline size_t pnl_search_nochk(const pnl_t pnl, pgno_t pgno) { + const pgno_t *begin = MDBX_PNL_BEGIN(pnl); + const pgno_t *it = pgno_bsearch(begin, MDBX_PNL_GETSIZE(pnl), pgno); + const pgno_t *end = begin + MDBX_PNL_GETSIZE(pnl); + assert(it >= begin && it <= end); + if (it != begin) + assert(MDBX_PNL_ORDERED(it[-1], pgno)); + if (it != end) + assert(!MDBX_PNL_ORDERED(it[0], pgno)); + return it - begin + 1; +} +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 -#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \ - defined(__BSD__) || defined(__bsdi__) || defined(__DragonFly__) || \ - defined(__APPLE__) || defined(__MACH__) || defined(MFSNAMELEN) || \ - defined(MFSTYPENAMELEN) || defined(VFS_NAMELEN) - const char *const name = statfs_info.f_fstypename; - const size_t name_len = sizeof(statfs_info.f_fstypename); +#if MDBX_ENABLE_REFUND +static void refund_reclaimed(MDBX_txn *txn) { + /* Scanning in descend order */ + pgno_t first_unallocated = txn->geo.first_unallocated; + const pnl_t pnl = txn->tw.repnl; + tASSERT(txn, MDBX_PNL_GETSIZE(pnl) && MDBX_PNL_MOST(pnl) == first_unallocated - 1); +#if MDBX_PNL_ASCENDING + size_t i = MDBX_PNL_GETSIZE(pnl); + tASSERT(txn, pnl[i] == first_unallocated - 1); + while (--first_unallocated, --i > 0 && pnl[i] == first_unallocated - 1) + ; + MDBX_PNL_SETSIZE(pnl, i); #else - const char *const name = ""; - const size_t name_len = 0; + size_t i = 1; + tASSERT(txn, pnl[i] == first_unallocated - 1); + size_t len = MDBX_PNL_GETSIZE(pnl); + while (--first_unallocated, ++i <= len && pnl[i] == first_unallocated - 1) + ; + MDBX_PNL_SETSIZE(pnl, len -= i - 1); + for (size_t move = 0; move < len; ++move) + pnl[1 + move] = pnl[i + move]; #endif - if (name_len) { - if (strncasecmp("tmpfs", name, 6) == 0 || - strncasecmp("mfs", name, 4) == 0 || - strncasecmp("ramfs", name, 6) == 0 || - strncasecmp("romfs", name, 6) == 0) - return MDBX_RESULT_TRUE; - } -#endif /* !Windows */ - - return MDBX_RESULT_FALSE; + VERBOSE("refunded %" PRIaPGNO " pages: %" PRIaPGNO " -> %" PRIaPGNO, txn->geo.first_unallocated - first_unallocated, + txn->geo.first_unallocated, first_unallocated); + txn->geo.first_unallocated = first_unallocated; + tASSERT(txn, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated - 1)); } -static int osal_check_fs_local(mdbx_filehandle_t handle, int flags) { -#if defined(_WIN32) || defined(_WIN64) - if (mdbx_RunningUnderWine() && !(flags & MDBX_EXCLUSIVE)) - return ERROR_NOT_CAPABLE /* workaround for Wine */; - - if (GetFileType(handle) != FILE_TYPE_DISK) - return ERROR_FILE_OFFLINE; - - if (mdbx_GetFileInformationByHandleEx) { - FILE_REMOTE_PROTOCOL_INFO RemoteProtocolInfo; - if (mdbx_GetFileInformationByHandleEx(handle, FileRemoteProtocolInfo, - &RemoteProtocolInfo, - sizeof(RemoteProtocolInfo))) { - if ((RemoteProtocolInfo.Flags & REMOTE_PROTOCOL_INFO_FLAG_OFFLINE) && - !(flags & MDBX_RDONLY)) - return ERROR_FILE_OFFLINE; - if (!(RemoteProtocolInfo.Flags & REMOTE_PROTOCOL_INFO_FLAG_LOOPBACK) && - !(flags & MDBX_EXCLUSIVE)) - return ERROR_REMOTE_STORAGE_MEDIA_ERROR; - } - } +static void refund_loose(MDBX_txn *txn) { + tASSERT(txn, txn->tw.loose_pages != nullptr); + tASSERT(txn, txn->tw.loose_count > 0); - if (mdbx_NtFsControlFile) { - NTSTATUS rc; - struct { - WOF_EXTERNAL_INFO wof_info; - union { - WIM_PROVIDER_EXTERNAL_INFO wim_info; - FILE_PROVIDER_EXTERNAL_INFO_V1 file_info; - }; - size_t reserved_for_microsoft_madness[42]; - } GetExternalBacking_OutputBuffer; - IO_STATUS_BLOCK StatusBlock; - rc = mdbx_NtFsControlFile(handle, NULL, NULL, NULL, &StatusBlock, - FSCTL_GET_EXTERNAL_BACKING, NULL, 0, - &GetExternalBacking_OutputBuffer, - sizeof(GetExternalBacking_OutputBuffer)); - if (NT_SUCCESS(rc)) { - if (!(flags & MDBX_EXCLUSIVE)) - return ERROR_REMOTE_STORAGE_MEDIA_ERROR; - } else if (rc != STATUS_OBJECT_NOT_EXTERNALLY_BACKED && - rc != STATUS_INVALID_DEVICE_REQUEST && - rc != STATUS_NOT_SUPPORTED) - return ntstatus2errcode(rc); + dpl_t *const dl = txn->tw.dirtylist; + if (dl) { + tASSERT(txn, dl->length >= txn->tw.loose_count); + tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); + } else { + tASSERT(txn, (txn->flags & MDBX_WRITEMAP) != 0 && !MDBX_AVOID_MSYNC); } - if (mdbx_GetVolumeInformationByHandleW && mdbx_GetFinalPathNameByHandleW) { - WCHAR *PathBuffer = osal_malloc(sizeof(WCHAR) * INT16_MAX); - if (!PathBuffer) - return MDBX_ENOMEM; + pgno_t onstack[MDBX_CACHELINE_SIZE * 8 / sizeof(pgno_t)]; + pnl_t suitable = onstack; - int rc = MDBX_SUCCESS; - DWORD VolumeSerialNumber, FileSystemFlags; - if (!mdbx_GetVolumeInformationByHandleW(handle, PathBuffer, INT16_MAX, - &VolumeSerialNumber, NULL, - &FileSystemFlags, NULL, 0)) { - rc = (int)GetLastError(); - goto bailout; + if (!dl || dl->length - dl->sorted > txn->tw.loose_count) { + /* Dirty list is useless since unsorted. */ + if (pnl_bytes2size(sizeof(onstack)) < txn->tw.loose_count) { + suitable = pnl_alloc(txn->tw.loose_count); + if (unlikely(!suitable)) + return /* this is not a reason for transaction fail */; } - if ((flags & MDBX_RDONLY) == 0) { - if (FileSystemFlags & - (FILE_SEQUENTIAL_WRITE_ONCE | FILE_READ_ONLY_VOLUME | - FILE_VOLUME_IS_COMPRESSED)) { - rc = ERROR_REMOTE_STORAGE_MEDIA_ERROR; - goto bailout; + /* Collect loose-pages which may be refunded. */ + tASSERT(txn, txn->geo.first_unallocated >= MIN_PAGENO + txn->tw.loose_count); + pgno_t most = MIN_PAGENO; + size_t w = 0; + for (const page_t *lp = txn->tw.loose_pages; lp; lp = page_next(lp)) { + tASSERT(txn, lp->flags == P_LOOSE); + tASSERT(txn, txn->geo.first_unallocated > lp->pgno); + if (likely(txn->geo.first_unallocated - txn->tw.loose_count <= lp->pgno)) { + tASSERT(txn, w < ((suitable == onstack) ? pnl_bytes2size(sizeof(onstack)) : MDBX_PNL_ALLOCLEN(suitable))); + suitable[++w] = lp->pgno; + most = (lp->pgno > most) ? lp->pgno : most; } + MDBX_ASAN_UNPOISON_MEMORY_REGION(&page_next(lp), sizeof(page_t *)); + VALGRIND_MAKE_MEM_DEFINED(&page_next(lp), sizeof(page_t *)); } - if (mdbx_GetFinalPathNameByHandleW(handle, PathBuffer, INT16_MAX, - FILE_NAME_NORMALIZED | VOLUME_NAME_NT)) { - if (_wcsnicmp(PathBuffer, L"\\Device\\Mup\\", 12) == 0) { - if (!(flags & MDBX_EXCLUSIVE)) { - rc = ERROR_REMOTE_STORAGE_MEDIA_ERROR; - goto bailout; - } - } - } + if (most + 1 == txn->geo.first_unallocated) { + /* Sort suitable list and refund pages at the tail. */ + MDBX_PNL_SETSIZE(suitable, w); + pnl_sort(suitable, MAX_PAGENO + 1); - if (F_ISSET(flags, MDBX_RDONLY | MDBX_EXCLUSIVE) && - (FileSystemFlags & FILE_READ_ONLY_VOLUME)) { - /* without-LCK (exclusive readonly) mode for DB on a read-only volume */ - goto bailout; - } + /* Scanning in descend order */ + const intptr_t step = MDBX_PNL_ASCENDING ? -1 : 1; + const intptr_t begin = MDBX_PNL_ASCENDING ? MDBX_PNL_GETSIZE(suitable) : 1; + const intptr_t end = MDBX_PNL_ASCENDING ? 0 : MDBX_PNL_GETSIZE(suitable) + 1; + tASSERT(txn, suitable[begin] >= suitable[end - step]); + tASSERT(txn, most == suitable[begin]); - if (mdbx_GetFinalPathNameByHandleW(handle, PathBuffer, INT16_MAX, - FILE_NAME_NORMALIZED | - VOLUME_NAME_DOS)) { - UINT DriveType = GetDriveTypeW(PathBuffer); - if (DriveType == DRIVE_NO_ROOT_DIR && - _wcsnicmp(PathBuffer, L"\\\\?\\", 4) == 0 && - _wcsnicmp(PathBuffer + 5, L":\\", 2) == 0) { - PathBuffer[7] = 0; - DriveType = GetDriveTypeW(PathBuffer + 4); - } - switch (DriveType) { - case DRIVE_CDROM: - if (flags & MDBX_RDONLY) + for (intptr_t i = begin + step; i != end; i += step) { + if (suitable[i] != most - 1) break; - // fall through - case DRIVE_UNKNOWN: - case DRIVE_NO_ROOT_DIR: - case DRIVE_REMOTE: - default: - if (!(flags & MDBX_EXCLUSIVE)) - rc = ERROR_REMOTE_STORAGE_MEDIA_ERROR; - // fall through - case DRIVE_REMOVABLE: - case DRIVE_FIXED: - case DRIVE_RAMDISK: - break; + most -= 1; } - } - - bailout: - osal_free(PathBuffer); - return rc; - } - -#else - - struct statvfs statvfs_info; - if (fstatvfs(handle, &statvfs_info)) - return errno; -#if defined(ST_LOCAL) || defined(ST_EXPORTED) - const unsigned long st_flags = statvfs_info.f_flag; -#endif /* ST_LOCAL || ST_EXPORTED */ - -#if defined(__NetBSD__) - const unsigned type = 0; - const char *const name = statvfs_info.f_fstypename; - const size_t name_len = VFS_NAMELEN; -#elif defined(_AIX) || defined(__OS400__) - const char *const name = statvfs_info.f_basetype; - const size_t name_len = sizeof(statvfs_info.f_basetype); - struct stat st; - if (fstat(handle, &st)) - return errno; - const unsigned type = st.st_vfstype; - if ((st.st_flag & FS_REMOTE) != 0 && !(flags & MDBX_EXCLUSIVE)) - return MDBX_EREMOTE; -#elif defined(FSTYPSZ) || defined(_FSTYPSZ) - const unsigned type = 0; - const char *const name = statvfs_info.f_basetype; - const size_t name_len = sizeof(statvfs_info.f_basetype); -#elif defined(__sun) || defined(__SVR4) || defined(__svr4__) || \ - defined(ST_FSTYPSZ) || defined(_ST_FSTYPSZ) - const unsigned type = 0; - struct stat st; - if (fstat(handle, &st)) - return errno; - const char *const name = st.st_fstype; - const size_t name_len = strlen(name); -#else - struct statfs statfs_info; - if (fstatfs(handle, &statfs_info)) - return errno; -#if defined(__OpenBSD__) - const unsigned type = 0; -#else - const unsigned type = statfs_info.f_type; -#endif -#if defined(MNT_LOCAL) || defined(MNT_EXPORTED) - const unsigned long mnt_flags = statfs_info.f_flags; -#endif /* MNT_LOCAL || MNT_EXPORTED */ -#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \ - defined(__BSD__) || defined(__bsdi__) || defined(__DragonFly__) || \ - defined(__APPLE__) || defined(__MACH__) || defined(MFSNAMELEN) || \ - defined(MFSTYPENAMELEN) || defined(VFS_NAMELEN) - const char *const name = statfs_info.f_fstypename; - const size_t name_len = sizeof(statfs_info.f_fstypename); -#elif defined(__ANDROID_API__) && __ANDROID_API__ < 21 - const char *const name = ""; - const unsigned name_len = 0; -#else - - const char *name = ""; - unsigned name_len = 0; - - struct stat st; - if (fstat(handle, &st)) - return errno; + const size_t refunded = txn->geo.first_unallocated - most; + DEBUG("refund-suitable %zu pages %" PRIaPGNO " -> %" PRIaPGNO, refunded, most, txn->geo.first_unallocated); + txn->geo.first_unallocated = most; + txn->tw.loose_count -= refunded; + if (dl) { + txn->tw.dirtyroom += refunded; + dl->pages_including_loose -= refunded; + assert(txn->tw.dirtyroom <= txn->env->options.dp_limit); - char pathbuf[PATH_MAX]; - FILE *mounted = nullptr; -#if defined(__linux__) || defined(__gnu_linux__) - mounted = setmntent("/proc/mounts", "r"); -#endif /* Linux */ - if (!mounted) - mounted = setmntent("/etc/mtab", "r"); - if (mounted) { - const struct mntent *ent; -#if defined(_BSD_SOURCE) || defined(_SVID_SOURCE) || defined(__BIONIC__) || \ - (defined(_DEFAULT_SOURCE) && __GLIBC_PREREQ(2, 19)) - struct mntent entbuf; - const bool should_copy = false; - while (nullptr != - (ent = getmntent_r(mounted, &entbuf, pathbuf, sizeof(pathbuf)))) -#else - const bool should_copy = true; - while (nullptr != (ent = getmntent(mounted))) -#endif - { - struct stat mnt; - if (!stat(ent->mnt_dir, &mnt) && mnt.st_dev == st.st_dev) { - if (should_copy) { - name = - strncpy(pathbuf, ent->mnt_fsname, name_len = sizeof(pathbuf) - 1); - pathbuf[name_len] = 0; - } else { - name = ent->mnt_fsname; - name_len = strlen(name); + /* Filter-out dirty list */ + size_t r = 0; + w = 0; + if (dl->sorted) { + do { + if (dl->items[++r].pgno < most) { + if (++w != r) + dl->items[w] = dl->items[r]; + } + } while (r < dl->sorted); + dl->sorted = w; } - break; + while (r < dl->length) { + if (dl->items[++r].pgno < most) { + if (++w != r) + dl->items[w] = dl->items[r]; + } + } + dpl_setlen(dl, w); + tASSERT(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length == + (txn->parent ? txn->parent->tw.dirtyroom : txn->env->options.dp_limit)); } + goto unlink_loose; } - endmntent(mounted); - } -#endif /* !xBSD && !Android/Bionic */ -#endif + } else { + /* Dirtylist is mostly sorted, just refund loose pages at the end. */ + dpl_sort(txn); + tASSERT(txn, dl->length < 2 || dl->items[1].pgno < dl->items[dl->length].pgno); + tASSERT(txn, dl->sorted == dl->length); - if (name_len) { - if (((name_len > 2 && strncasecmp("nfs", name, 3) == 0) || - strncasecmp("cifs", name, name_len) == 0 || - strncasecmp("ncpfs", name, name_len) == 0 || - strncasecmp("smbfs", name, name_len) == 0 || - strcasecmp("9P" /* WSL2 */, name) == 0 || - ((name_len > 3 && strncasecmp("fuse", name, 4) == 0) && - strncasecmp("fuseblk", name, name_len) != 0)) && - !(flags & MDBX_EXCLUSIVE)) - return MDBX_EREMOTE; - if (strcasecmp("ftp", name) == 0 || strcasecmp("http", name) == 0 || - strcasecmp("sshfs", name) == 0) - return MDBX_EREMOTE; + /* Scan dirtylist tail-forward and cutoff suitable pages. */ + size_t n; + for (n = dl->length; dl->items[n].pgno == txn->geo.first_unallocated - 1 && dl->items[n].ptr->flags == P_LOOSE; + --n) { + tASSERT(txn, n > 0); + page_t *dp = dl->items[n].ptr; + DEBUG("refund-sorted page %" PRIaPGNO, dp->pgno); + tASSERT(txn, dp->pgno == dl->items[n].pgno); + txn->geo.first_unallocated -= 1; + } + dpl_setlen(dl, n); + + if (dl->sorted != dl->length) { + const size_t refunded = dl->sorted - dl->length; + dl->sorted = dl->length; + txn->tw.loose_count -= refunded; + txn->tw.dirtyroom += refunded; + dl->pages_including_loose -= refunded; + tASSERT(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length == + (txn->parent ? txn->parent->tw.dirtyroom : txn->env->options.dp_limit)); + + /* Filter-out loose chain & dispose refunded pages. */ + unlink_loose: + for (page_t *__restrict *__restrict link = &txn->tw.loose_pages; *link;) { + page_t *dp = *link; + tASSERT(txn, dp->flags == P_LOOSE); + MDBX_ASAN_UNPOISON_MEMORY_REGION(&page_next(dp), sizeof(page_t *)); + VALGRIND_MAKE_MEM_DEFINED(&page_next(dp), sizeof(page_t *)); + if (txn->geo.first_unallocated > dp->pgno) { + link = &page_next(dp); + } else { + *link = page_next(dp); + if ((txn->flags & MDBX_WRITEMAP) == 0) + page_shadow_release(txn->env, dp, 1); + } + } + } } -#ifdef ST_LOCAL - if ((st_flags & ST_LOCAL) == 0 && !(flags & MDBX_EXCLUSIVE)) - return MDBX_EREMOTE; -#elif defined(MNT_LOCAL) - if ((mnt_flags & MNT_LOCAL) == 0 && !(flags & MDBX_EXCLUSIVE)) - return MDBX_EREMOTE; -#endif /* ST/MNT_LOCAL */ + tASSERT(txn, dpl_check(txn)); + if (suitable != onstack) + pnl_free(suitable); + txn->tw.loose_refund_wl = txn->geo.first_unallocated; +} -#ifdef ST_EXPORTED - if ((st_flags & ST_EXPORTED) != 0 && !(flags & MDBX_RDONLY)) - return MDBX_EREMOTE; -#elif defined(MNT_EXPORTED) - if ((mnt_flags & MNT_EXPORTED) != 0 && !(flags & MDBX_RDONLY)) - return MDBX_EREMOTE; -#endif /* ST/MNT_EXPORTED */ +bool txn_refund(MDBX_txn *txn) { + const pgno_t before = txn->geo.first_unallocated; - switch (type) { - case 0xFF534D42 /* CIFS_MAGIC_NUMBER */: - case 0x6969 /* NFS_SUPER_MAGIC */: - case 0x564c /* NCP_SUPER_MAGIC */: - case 0x517B /* SMB_SUPER_MAGIC */: -#if defined(__digital__) || defined(__osf__) || defined(__osf) - case 0x0E /* Tru64 NFS */: -#endif -#ifdef ST_FST_NFS - case ST_FST_NFS: -#endif - if ((flags & MDBX_EXCLUSIVE) == 0) - return MDBX_EREMOTE; - case 0: - default: - break; + if (txn->tw.loose_pages && txn->tw.loose_refund_wl > txn->geo.first_unallocated) + refund_loose(txn); + + while (true) { + if (MDBX_PNL_GETSIZE(txn->tw.repnl) == 0 || MDBX_PNL_MOST(txn->tw.repnl) != txn->geo.first_unallocated - 1) + break; + + refund_reclaimed(txn); + if (!txn->tw.loose_pages || txn->tw.loose_refund_wl <= txn->geo.first_unallocated) + break; + + const pgno_t memo = txn->geo.first_unallocated; + refund_loose(txn); + if (memo == txn->geo.first_unallocated) + break; } -#endif /* Unix */ - return MDBX_SUCCESS; -} + if (before == txn->geo.first_unallocated) + return false; -static int check_mmap_limit(const size_t limit) { - const bool should_check = -#if defined(__SANITIZE_ADDRESS__) - true; -#else - RUNNING_ON_VALGRIND; -#endif /* __SANITIZE_ADDRESS__ */ + if (txn->tw.spilled.list) + /* Squash deleted pagenums if we refunded any */ + spill_purge(txn); - if (should_check) { - intptr_t pagesize, total_ram_pages, avail_ram_pages; - int err = - mdbx_get_sysraminfo(&pagesize, &total_ram_pages, &avail_ram_pages); - if (unlikely(err != MDBX_SUCCESS)) - return err; + return true; +} - const int log2page = log2n_powerof2(pagesize); - if ((limit >> (log2page + 7)) > (size_t)total_ram_pages || - (limit >> (log2page + 6)) > (size_t)avail_ram_pages) { - ERROR("%s (%zu pages) is too large for available (%zu pages) or total " - "(%zu pages) system RAM", - "database upper size limit", limit >> log2page, avail_ram_pages, - total_ram_pages); - return MDBX_TOO_LARGE; - } - } +#else /* MDBX_ENABLE_REFUND */ - return MDBX_SUCCESS; +bool txn_refund(MDBX_txn *txn) { + (void)txn; + /* No online auto-compactification. */ + return false; } -MDBX_INTERNAL_FUNC int osal_mmap(const int flags, osal_mmap_t *map, size_t size, - const size_t limit, const unsigned options) { - assert(size <= limit); - map->limit = 0; - map->current = 0; - map->base = nullptr; - map->filesize = 0; -#if defined(_WIN32) || defined(_WIN64) - map->section = NULL; -#endif /* Windows */ +#endif /* MDBX_ENABLE_REFUND */ +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 - int err = osal_check_fs_local(map->fd, flags); - if (unlikely(err != MDBX_SUCCESS)) - return err; +void spill_remove(MDBX_txn *txn, size_t idx, size_t npages) { + tASSERT(txn, idx > 0 && idx <= MDBX_PNL_GETSIZE(txn->tw.spilled.list) && txn->tw.spilled.least_removed > 0); + txn->tw.spilled.least_removed = (idx < txn->tw.spilled.least_removed) ? idx : txn->tw.spilled.least_removed; + txn->tw.spilled.list[idx] |= 1; + MDBX_PNL_SETSIZE(txn->tw.spilled.list, + MDBX_PNL_GETSIZE(txn->tw.spilled.list) - (idx == MDBX_PNL_GETSIZE(txn->tw.spilled.list))); - err = check_mmap_limit(limit); - if (unlikely(err != MDBX_SUCCESS)) - return err; + while (unlikely(npages > 1)) { + const pgno_t pgno = (txn->tw.spilled.list[idx] >> 1) + 1; + if (MDBX_PNL_ASCENDING) { + if (++idx > MDBX_PNL_GETSIZE(txn->tw.spilled.list) || (txn->tw.spilled.list[idx] >> 1) != pgno) + return; + } else { + if (--idx < 1 || (txn->tw.spilled.list[idx] >> 1) != pgno) + return; + txn->tw.spilled.least_removed = (idx < txn->tw.spilled.least_removed) ? idx : txn->tw.spilled.least_removed; + } + txn->tw.spilled.list[idx] |= 1; + MDBX_PNL_SETSIZE(txn->tw.spilled.list, + MDBX_PNL_GETSIZE(txn->tw.spilled.list) - (idx == MDBX_PNL_GETSIZE(txn->tw.spilled.list))); + --npages; + } +} - if ((flags & MDBX_RDONLY) == 0 && (options & MMAP_OPTION_TRUNCATE) != 0) { - err = osal_ftruncate(map->fd, size); - VERBOSE("ftruncate %zu, err %d", size, err); - if (err != MDBX_SUCCESS) - return err; - map->filesize = size; -#if !(defined(_WIN32) || defined(_WIN64)) - map->current = size; -#endif /* !Windows */ - } else { - err = osal_filesize(map->fd, &map->filesize); - VERBOSE("filesize %" PRIu64 ", err %d", map->filesize, err); - if (err != MDBX_SUCCESS) - return err; -#if defined(_WIN32) || defined(_WIN64) - if (map->filesize < size) { - WARNING("file size (%zu) less than requested for mapping (%zu)", - (size_t)map->filesize, size); - size = (size_t)map->filesize; +pnl_t spill_purge(MDBX_txn *txn) { + tASSERT(txn, txn->tw.spilled.least_removed > 0); + const pnl_t sl = txn->tw.spilled.list; + if (txn->tw.spilled.least_removed != INT_MAX) { + size_t len = MDBX_PNL_GETSIZE(sl), r, w; + for (w = r = txn->tw.spilled.least_removed; r <= len; ++r) { + sl[w] = sl[r]; + w += 1 - (sl[r] & 1); } -#else - map->current = (map->filesize > limit) ? limit : (size_t)map->filesize; -#endif /* !Windows */ + for (size_t i = 1; i < w; ++i) + tASSERT(txn, (sl[i] & 1) == 0); + MDBX_PNL_SETSIZE(sl, w - 1); + txn->tw.spilled.least_removed = INT_MAX; + } else { + for (size_t i = 1; i <= MDBX_PNL_GETSIZE(sl); ++i) + tASSERT(txn, (sl[i] & 1) == 0); } + return sl; +} -#if defined(_WIN32) || defined(_WIN64) - LARGE_INTEGER SectionSize; - SectionSize.QuadPart = size; - err = NtCreateSection( - &map->section, - /* DesiredAccess */ - (flags & MDBX_WRITEMAP) - ? SECTION_QUERY | SECTION_MAP_READ | SECTION_EXTEND_SIZE | - SECTION_MAP_WRITE - : SECTION_QUERY | SECTION_MAP_READ | SECTION_EXTEND_SIZE, - /* ObjectAttributes */ NULL, /* MaximumSize (InitialSize) */ &SectionSize, - /* SectionPageProtection */ - (flags & MDBX_RDONLY) ? PAGE_READONLY : PAGE_READWRITE, - /* AllocationAttributes */ SEC_RESERVE, map->fd); - if (!NT_SUCCESS(err)) - return ntstatus2errcode(err); +/*----------------------------------------------------------------------------*/ - SIZE_T ViewSize = (flags & MDBX_RDONLY) ? 0 - : mdbx_RunningUnderWine() ? size - : limit; - err = NtMapViewOfSection( - map->section, GetCurrentProcess(), &map->base, - /* ZeroBits */ 0, - /* CommitSize */ 0, - /* SectionOffset */ NULL, &ViewSize, - /* InheritDisposition */ ViewUnmap, - /* AllocationType */ (flags & MDBX_RDONLY) ? 0 : MEM_RESERVE, - /* Win32Protect */ - (flags & MDBX_WRITEMAP) ? PAGE_READWRITE : PAGE_READONLY); - if (!NT_SUCCESS(err)) { - NtClose(map->section); - map->section = 0; - map->base = nullptr; - return ntstatus2errcode(err); +static int spill_page(MDBX_txn *txn, iov_ctx_t *ctx, page_t *dp, const size_t npages) { + tASSERT(txn, !(txn->flags & MDBX_WRITEMAP)); +#if MDBX_ENABLE_PGOP_STAT + txn->env->lck->pgops.spill.weak += npages; +#endif /* MDBX_ENABLE_PGOP_STAT */ + const pgno_t pgno = dp->pgno; + int err = iov_page(txn, ctx, dp, npages); + if (likely(err == MDBX_SUCCESS)) + err = spill_append_span(&txn->tw.spilled.list, pgno, npages); + return err; +} + +/* Set unspillable LRU-label for dirty pages watched by txn. + * Returns the number of pages marked as unspillable. */ +static size_t spill_cursor_keep(const MDBX_txn *const txn, const MDBX_cursor *mc) { + tASSERT(txn, (txn->flags & (MDBX_TXN_RDONLY | MDBX_WRITEMAP)) == 0); + size_t keep = 0; + while (!is_poor(mc)) { + tASSERT(txn, mc->top >= 0); + const page_t *mp; + intptr_t i = 0; + do { + mp = mc->pg[i]; + tASSERT(txn, !is_subpage(mp)); + if (is_modifable(txn, mp)) { + size_t const n = dpl_search(txn, mp->pgno); + if (txn->tw.dirtylist->items[n].pgno == mp->pgno && + /* не считаем дважды */ dpl_age(txn, n)) { + size_t *const ptr = ptr_disp(txn->tw.dirtylist->items[n].ptr, -(ptrdiff_t)sizeof(size_t)); + *ptr = txn->tw.dirtylru; + tASSERT(txn, dpl_age(txn, n) == 0); + ++keep; + } + } + } while (++i <= mc->top); + + tASSERT(txn, is_leaf(mp)); + if (!mc->subcur || mc->ki[mc->top] >= page_numkeys(mp)) + break; + if (!(node_flags(page_node(mp, mc->ki[mc->top])) & N_TREE)) + break; + mc = &mc->subcur->cursor; } - assert(map->base != MAP_FAILED); + return keep; +} - map->current = (size_t)SectionSize.QuadPart; - map->limit = ViewSize; +static size_t spill_txn_keep(MDBX_txn *txn, MDBX_cursor *m0) { + tASSERT(txn, (txn->flags & (MDBX_TXN_RDONLY | MDBX_WRITEMAP)) == 0); + dpl_lru_turn(txn); + size_t keep = m0 ? spill_cursor_keep(txn, m0) : 0; -#else /* Windows */ + TXN_FOREACH_DBI_ALL(txn, dbi) { + if (F_ISSET(txn->dbi_state[dbi], DBI_DIRTY | DBI_VALID) && txn->dbs[dbi].root != P_INVALID) + for (MDBX_cursor *mc = txn->cursors[dbi]; mc; mc = mc->next) + if (mc != m0) + keep += spill_cursor_keep(txn, mc); + } -#ifndef MAP_TRYFIXED -#define MAP_TRYFIXED 0 -#endif + return keep; +} -#ifndef MAP_HASSEMAPHORE -#define MAP_HASSEMAPHORE 0 -#endif +/* Returns the spilling priority (0..255) for a dirty page: + * 0 = should be spilled; + * ... + * > 255 = must not be spilled. */ +MDBX_NOTHROW_PURE_FUNCTION static unsigned spill_prio(const MDBX_txn *txn, const size_t i, const uint32_t reciprocal) { + dpl_t *const dl = txn->tw.dirtylist; + const uint32_t age = dpl_age(txn, i); + const size_t npages = dpl_npages(dl, i); + const pgno_t pgno = dl->items[i].pgno; + if (age == 0) { + DEBUG("skip %s %zu page %" PRIaPGNO, "keep", npages, pgno); + return 256; + } -#ifndef MAP_CONCEAL -#define MAP_CONCEAL 0 -#endif + page_t *const dp = dl->items[i].ptr; + if (dp->flags & (P_LOOSE | P_SPILLED)) { + DEBUG("skip %s %zu page %" PRIaPGNO, (dp->flags & P_LOOSE) ? "loose" : "parent-spilled", npages, pgno); + return 256; + } -#ifndef MAP_NOSYNC -#define MAP_NOSYNC 0 -#endif + /* Can't spill twice, + * make sure it's not already in a parent's spill list(s). */ + MDBX_txn *parent = txn->parent; + if (parent && (parent->flags & MDBX_TXN_SPILLS)) { + do + if (spill_intersect(parent, pgno, npages)) { + DEBUG("skip-2 parent-spilled %zu page %" PRIaPGNO, npages, pgno); + dp->flags |= P_SPILLED; + return 256; + } + while ((parent = parent->parent) != nullptr); + } -#ifndef MAP_FIXED_NOREPLACE -#define MAP_FIXED_NOREPLACE 0 -#endif + tASSERT(txn, age * (uint64_t)reciprocal < UINT32_MAX); + unsigned prio = age * reciprocal >> 24; + tASSERT(txn, prio < 256); + if (likely(npages == 1)) + return prio = 256 - prio; -#ifndef MAP_NORESERVE -#define MAP_NORESERVE 0 -#endif + /* make a large/overflow pages be likely to spill */ + size_t factor = npages | npages >> 1; + factor |= factor >> 2; + factor |= factor >> 4; + factor |= factor >> 8; + factor |= factor >> 16; + factor = (size_t)prio * log2n_powerof2(factor + 1) + /* golden ratio */ 157; + factor = (factor < 256) ? 255 - factor : 0; + tASSERT(txn, factor < 256 && factor < (256 - prio)); + return prio = (unsigned)factor; +} - map->base = mmap( - NULL, limit, (flags & MDBX_WRITEMAP) ? PROT_READ | PROT_WRITE : PROT_READ, - MAP_SHARED | MAP_FILE | MAP_NORESERVE | - (F_ISSET(flags, MDBX_UTTERLY_NOSYNC) ? MAP_NOSYNC : 0) | - ((options & MMAP_OPTION_SEMAPHORE) ? MAP_HASSEMAPHORE | MAP_NOSYNC - : MAP_CONCEAL), - map->fd, 0); +static size_t spill_gate(const MDBX_env *env, intptr_t part, const size_t total) { + const intptr_t spill_min = env->options.spill_min_denominator + ? (total + env->options.spill_min_denominator - 1) / env->options.spill_min_denominator + : 1; + const intptr_t spill_max = + total - (env->options.spill_max_denominator ? total / env->options.spill_max_denominator : 0); + part = (part < spill_max) ? part : spill_max; + part = (part > spill_min) ? part : spill_min; + eASSERT(env, part >= 0 && (size_t)part <= total); + return (size_t)part; +} - if (unlikely(map->base == MAP_FAILED)) { - map->limit = 0; - map->current = 0; - map->base = nullptr; - assert(errno != 0); - return errno; - } - map->limit = limit; +__cold int spill_slowpath(MDBX_txn *const txn, MDBX_cursor *const m0, const intptr_t wanna_spill_entries, + const intptr_t wanna_spill_npages, const size_t need) { + tASSERT(txn, (txn->flags & MDBX_TXN_RDONLY) == 0); -#if MDBX_ENABLE_MADVISE -#ifdef MADV_DONTFORK - if (unlikely(madvise(map->base, map->limit, MADV_DONTFORK) != 0)) - return errno; -#endif /* MADV_DONTFORK */ -#ifdef MADV_NOHUGEPAGE - (void)madvise(map->base, map->limit, MADV_NOHUGEPAGE); -#endif /* MADV_NOHUGEPAGE */ -#endif /* MDBX_ENABLE_MADVISE */ + int rc = MDBX_SUCCESS; + if (unlikely(txn->tw.loose_count >= + (txn->tw.dirtylist ? txn->tw.dirtylist->pages_including_loose : txn->tw.writemap_dirty_npages))) + goto done; -#endif /* ! Windows */ + const size_t dirty_entries = txn->tw.dirtylist ? (txn->tw.dirtylist->length - txn->tw.loose_count) : 1; + const size_t dirty_npages = + (txn->tw.dirtylist ? txn->tw.dirtylist->pages_including_loose : txn->tw.writemap_dirty_npages) - + txn->tw.loose_count; + const size_t need_spill_entries = spill_gate(txn->env, wanna_spill_entries, dirty_entries); + const size_t need_spill_npages = spill_gate(txn->env, wanna_spill_npages, dirty_npages); - VALGRIND_MAKE_MEM_DEFINED(map->base, map->current); - MDBX_ASAN_UNPOISON_MEMORY_REGION(map->base, map->current); - return MDBX_SUCCESS; -} + const size_t need_spill = (need_spill_entries > need_spill_npages) ? need_spill_entries : need_spill_npages; + if (!need_spill) + goto done; -MDBX_INTERNAL_FUNC int osal_munmap(osal_mmap_t *map) { - VALGRIND_MAKE_MEM_NOACCESS(map->base, map->current); - /* Unpoisoning is required for ASAN to avoid false-positive diagnostic - * when this memory will re-used by malloc or another mmapping. - * See https://libmdbx.dqdkfa.ru/dead-github/pull/93#issuecomment-613687203 */ - MDBX_ASAN_UNPOISON_MEMORY_REGION( - map->base, (map->filesize && map->filesize < map->limit) ? map->filesize - : map->limit); -#if defined(_WIN32) || defined(_WIN64) - if (map->section) - NtClose(map->section); - NTSTATUS rc = NtUnmapViewOfSection(GetCurrentProcess(), map->base); - if (!NT_SUCCESS(rc)) - ntstatus2errcode(rc); + if (txn->flags & MDBX_WRITEMAP) { + NOTICE("%s-spilling %zu dirty-entries, %zu dirty-npages", "msync", dirty_entries, dirty_npages); + const MDBX_env *env = txn->env; + tASSERT(txn, txn->tw.spilled.list == nullptr); + rc = osal_msync(&txn->env->dxb_mmap, 0, pgno_align2os_bytes(env, txn->geo.first_unallocated), MDBX_SYNC_KICK); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; +#if MDBX_AVOID_MSYNC + MDBX_ANALYSIS_ASSUME(txn->tw.dirtylist != nullptr); + tASSERT(txn, dpl_check(txn)); + env->lck->unsynced_pages.weak += txn->tw.dirtylist->pages_including_loose - txn->tw.loose_count; + dpl_clear(txn->tw.dirtylist); + txn->tw.dirtyroom = env->options.dp_limit - txn->tw.loose_count; + for (page_t *lp = txn->tw.loose_pages; lp != nullptr; lp = page_next(lp)) { + tASSERT(txn, lp->flags == P_LOOSE); + rc = dpl_append(txn, lp->pgno, lp, 1); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + MDBX_ASAN_UNPOISON_MEMORY_REGION(&page_next(lp), sizeof(page_t *)); + VALGRIND_MAKE_MEM_DEFINED(&page_next(lp), sizeof(page_t *)); + } + tASSERT(txn, dpl_check(txn)); #else - if (unlikely(munmap(map->base, map->limit))) { - assert(errno != 0); - return errno; + tASSERT(txn, txn->tw.dirtylist == nullptr); + env->lck->unsynced_pages.weak += txn->tw.writemap_dirty_npages; + txn->tw.writemap_spilled_npages += txn->tw.writemap_dirty_npages; + txn->tw.writemap_dirty_npages = 0; +#endif /* MDBX_AVOID_MSYNC */ + goto done; } -#endif /* ! Windows */ - map->limit = 0; - map->current = 0; - map->base = nullptr; - return MDBX_SUCCESS; -} + NOTICE("%s-spilling %zu dirty-entries, %zu dirty-npages", "write", need_spill_entries, need_spill_npages); + MDBX_ANALYSIS_ASSUME(txn->tw.dirtylist != nullptr); + tASSERT(txn, txn->tw.dirtylist->length - txn->tw.loose_count >= 1); + tASSERT(txn, txn->tw.dirtylist->pages_including_loose - txn->tw.loose_count >= need_spill_npages); + if (!txn->tw.spilled.list) { + txn->tw.spilled.least_removed = INT_MAX; + txn->tw.spilled.list = pnl_alloc(need_spill); + if (unlikely(!txn->tw.spilled.list)) { + rc = MDBX_ENOMEM; + bailout: + txn->flags |= MDBX_TXN_ERROR; + return rc; + } + } else { + /* purge deleted slots */ + spill_purge(txn); + rc = pnl_reserve(&txn->tw.spilled.list, need_spill); + (void)rc /* ignore since the resulting list may be shorter + and pnl_append() will increase pnl on demand */ + ; + } -MDBX_INTERNAL_FUNC int osal_mresize(const int flags, osal_mmap_t *map, - size_t size, size_t limit) { - int rc = osal_filesize(map->fd, &map->filesize); - VERBOSE("flags 0x%x, size %zu, limit %zu, filesize %" PRIu64, flags, size, - limit, map->filesize); - assert(size <= limit); - if (rc != MDBX_SUCCESS) { - map->filesize = 0; - return rc; + /* Сортируем чтобы запись на диск была полее последовательна */ + dpl_t *const dl = dpl_sort(txn); + + /* Preserve pages which may soon be dirtied again */ + const size_t unspillable = spill_txn_keep(txn, m0); + if (unspillable + txn->tw.loose_count >= dl->length) { +#if xMDBX_DEBUG_SPILLING == 1 /* avoid false failure in debug mode */ + if (likely(txn->tw.dirtyroom + txn->tw.loose_count >= need)) + return MDBX_SUCCESS; +#endif /* xMDBX_DEBUG_SPILLING */ + ERROR("all %zu dirty pages are unspillable since referenced " + "by a cursor(s), use fewer cursors or increase " + "MDBX_opt_txn_dp_limit", + unspillable); + goto done; } -#if defined(_WIN32) || defined(_WIN64) - assert(size != map->current || limit != map->limit || size < map->filesize); + /* Подзадача: Вытолкнуть часть страниц на диск в соответствии с LRU, + * но при этом учесть важные поправки: + * - лучше выталкивать старые large/overflow страницы, так будет освобождено + * больше памяти, а также так как они (в текущем понимании) гораздо реже + * повторно изменяются; + * - при прочих равных лучше выталкивать смежные страницы, так будет + * меньше I/O операций; + * - желательно потратить на это меньше времени чем std::partial_sort_copy; + * + * Решение: + * - Квантуем весь диапазон lru-меток до 256 значений и задействуем один + * проход 8-битного radix-sort. В результате получаем 256 уровней + * "свежести", в том числе значение lru-метки, старее которой страницы + * должны быть выгружены; + * - Двигаемся последовательно в сторону увеличения номеров страниц + * и выталкиваем страницы с lru-меткой старее отсекающего значения, + * пока не вытолкнем достаточно; + * - Встречая страницы смежные с выталкиваемыми для уменьшения кол-ва + * I/O операций выталкиваем и их, если они попадают в первую половину + * между выталкиваемыми и самыми свежими lru-метками; + * - дополнительно при сортировке умышленно старим large/overflow страницы, + * тем самым повышая их шансы на выталкивание. */ - NTSTATUS status; - LARGE_INTEGER SectionSize; - int err; + /* get min/max of LRU-labels */ + uint32_t age_max = 0; + for (size_t i = 1; i <= dl->length; ++i) { + const uint32_t age = dpl_age(txn, i); + age_max = (age_max >= age) ? age_max : age; + } - if (limit == map->limit && size > map->current) { - if ((flags & MDBX_RDONLY) && map->filesize >= size) { - map->current = size; - return MDBX_SUCCESS; - } else if (!(flags & MDBX_RDONLY) && - /* workaround for Wine */ mdbx_NtExtendSection) { - /* growth rw-section */ - SectionSize.QuadPart = size; - status = mdbx_NtExtendSection(map->section, &SectionSize); - if (!NT_SUCCESS(status)) - return ntstatus2errcode(status); - map->current = size; - if (map->filesize < size) - map->filesize = size; - return MDBX_SUCCESS; + VERBOSE("lru-head %u, age-max %u", txn->tw.dirtylru, age_max); + + /* half of 8-bit radix-sort */ + pgno_t radix_entries[256], radix_npages[256]; + memset(&radix_entries, 0, sizeof(radix_entries)); + memset(&radix_npages, 0, sizeof(radix_npages)); + size_t spillable_entries = 0, spillable_npages = 0; + const uint32_t reciprocal = (UINT32_C(255) << 24) / (age_max + 1); + for (size_t i = 1; i <= dl->length; ++i) { + const unsigned prio = spill_prio(txn, i, reciprocal); + size_t *const ptr = ptr_disp(dl->items[i].ptr, -(ptrdiff_t)sizeof(size_t)); + TRACE("page %" PRIaPGNO ", lru %zu, is_multi %c, npages %u, age %u of %u, prio %u", dl->items[i].pgno, *ptr, + (dl->items[i].npages > 1) ? 'Y' : 'N', dpl_npages(dl, i), dpl_age(txn, i), age_max, prio); + if (prio < 256) { + radix_entries[prio] += 1; + spillable_entries += 1; + const pgno_t npages = dpl_npages(dl, i); + radix_npages[prio] += npages; + spillable_npages += npages; } } - if (limit > map->limit) { - err = check_mmap_limit(limit); - if (unlikely(err != MDBX_SUCCESS)) - return err; + tASSERT(txn, spillable_npages >= spillable_entries); + pgno_t spilled_entries = 0, spilled_npages = 0; + if (likely(spillable_entries > 0)) { + size_t prio2spill = 0, prio2adjacent = 128, amount_entries = radix_entries[0], amount_npages = radix_npages[0]; + for (size_t i = 1; i < 256; i++) { + if (amount_entries < need_spill_entries || amount_npages < need_spill_npages) { + prio2spill = i; + prio2adjacent = i + (257 - i) / 2; + amount_entries += radix_entries[i]; + amount_npages += radix_npages[i]; + } else if (amount_entries + amount_entries < spillable_entries + need_spill_entries + /* РАВНОЗНАЧНО: amount - need_spill < spillable - amount */ + || amount_npages + amount_npages < spillable_npages + need_spill_npages) { + prio2adjacent = i; + amount_entries += radix_entries[i]; + amount_npages += radix_npages[i]; + } else + break; + } - /* check ability of address space for growth before unmap */ - PVOID BaseAddress = (PBYTE)map->base + map->limit; - SIZE_T RegionSize = limit - map->limit; - status = NtAllocateVirtualMemory(GetCurrentProcess(), &BaseAddress, 0, - &RegionSize, MEM_RESERVE, PAGE_NOACCESS); - if (status == (NTSTATUS) /* STATUS_CONFLICTING_ADDRESSES */ 0xC0000018) - return MDBX_UNABLE_EXTEND_MAPSIZE; - if (!NT_SUCCESS(status)) - return ntstatus2errcode(status); + VERBOSE("prio2spill %zu, prio2adjacent %zu, spillable %zu/%zu," + " wanna-spill %zu/%zu, amount %zu/%zu", + prio2spill, prio2adjacent, spillable_entries, spillable_npages, need_spill_entries, need_spill_npages, + amount_entries, amount_npages); + tASSERT(txn, prio2spill < prio2adjacent && prio2adjacent <= 256); - status = NtFreeVirtualMemory(GetCurrentProcess(), &BaseAddress, &RegionSize, - MEM_RELEASE); - if (!NT_SUCCESS(status)) - return ntstatus2errcode(status); - } + iov_ctx_t ctx; + rc = iov_init(txn, &ctx, amount_entries, amount_npages, +#if defined(_WIN32) || defined(_WIN64) + txn->env->ioring.overlapped_fd ? txn->env->ioring.overlapped_fd : +#endif + txn->env->lazy_fd, + true); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; - /* Windows unable: - * - shrink a mapped file; - * - change size of mapped view; - * - extend read-only mapping; - * Therefore we should unmap/map entire section. */ - if ((flags & MDBX_MRESIZE_MAY_UNMAP) == 0) { - if (size <= map->current && limit == map->limit) - return MDBX_SUCCESS; - return MDBX_EPERM; - } + size_t r = 0, w = 0; + pgno_t last = 0; + while (r < dl->length && (spilled_entries < need_spill_entries || spilled_npages < need_spill_npages)) { + dl->items[++w] = dl->items[++r]; + unsigned prio = spill_prio(txn, w, reciprocal); + if (prio > prio2spill && (prio >= prio2adjacent || last != dl->items[w].pgno)) + continue; - /* Unpoisoning is required for ASAN to avoid false-positive diagnostic - * when this memory will re-used by malloc or another mmapping. - * See https://libmdbx.dqdkfa.ru/dead-github/pull/93#issuecomment-613687203 */ - MDBX_ASAN_UNPOISON_MEMORY_REGION(map->base, map->limit); - status = NtUnmapViewOfSection(GetCurrentProcess(), map->base); - if (!NT_SUCCESS(status)) - return ntstatus2errcode(status); - status = NtClose(map->section); - map->section = NULL; - PVOID ReservedAddress = NULL; - SIZE_T ReservedSize = limit; + const size_t e = w; + last = dpl_endpgno(dl, w); + while (--w && dpl_endpgno(dl, w) == dl->items[w + 1].pgno && spill_prio(txn, w, reciprocal) < prio2adjacent) + ; - if (!NT_SUCCESS(status)) { - bailout_ntstatus: - err = ntstatus2errcode(status); - map->base = NULL; - map->current = map->limit = 0; - if (ReservedAddress) { - ReservedSize = 0; - status = NtFreeVirtualMemory(GetCurrentProcess(), &ReservedAddress, - &ReservedSize, MEM_RELEASE); - assert(NT_SUCCESS(status)); - (void)status; + for (size_t i = w; ++i <= e;) { + const unsigned npages = dpl_npages(dl, i); + prio = spill_prio(txn, i, reciprocal); + DEBUG("%sspill[%zu] %u page %" PRIaPGNO " (age %d, prio %u)", (prio > prio2spill) ? "co-" : "", i, npages, + dl->items[i].pgno, dpl_age(txn, i), prio); + tASSERT(txn, prio < 256); + ++spilled_entries; + spilled_npages += npages; + rc = spill_page(txn, &ctx, dl->items[i].ptr, npages); + if (unlikely(rc != MDBX_SUCCESS)) + goto failed; + } } - return err; - } -retry_file_and_section: - /* resizing of the file may take a while, - * therefore we reserve address space to avoid occupy it by other threads */ - ReservedAddress = map->base; - status = NtAllocateVirtualMemory(GetCurrentProcess(), &ReservedAddress, 0, - &ReservedSize, MEM_RESERVE, PAGE_NOACCESS); - if (!NT_SUCCESS(status)) { - ReservedAddress = NULL; - if (status != (NTSTATUS) /* STATUS_CONFLICTING_ADDRESSES */ 0xC0000018) - goto bailout_ntstatus /* no way to recovery */; + VERBOSE("spilled entries %u, spilled npages %u", spilled_entries, spilled_npages); + tASSERT(txn, spillable_entries == 0 || spilled_entries > 0); + tASSERT(txn, spilled_npages >= spilled_entries); - if (flags & MDBX_MRESIZE_MAY_MOVE) - /* the base address could be changed */ - map->base = NULL; + failed: + while (r < dl->length) + dl->items[++w] = dl->items[++r]; + tASSERT(txn, r - w == spilled_entries || rc != MDBX_SUCCESS); + + dl->sorted = dpl_setlen(dl, w); + txn->tw.dirtyroom += spilled_entries; + txn->tw.dirtylist->pages_including_loose -= spilled_npages; + tASSERT(txn, dpl_check(txn)); + + if (!iov_empty(&ctx)) { + tASSERT(txn, rc == MDBX_SUCCESS); + rc = iov_write(&ctx); + } + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + + txn->env->lck->unsynced_pages.weak += spilled_npages; + pnl_sort(txn->tw.spilled.list, (size_t)txn->geo.first_unallocated << 1); + txn->flags |= MDBX_TXN_SPILLS; + NOTICE("spilled %u dirty-entries, %u dirty-npages, now have %zu dirty-room", spilled_entries, spilled_npages, + txn->tw.dirtyroom); + } else { + tASSERT(txn, rc == MDBX_SUCCESS); + for (size_t i = 1; i <= dl->length; ++i) { + page_t *dp = dl->items[i].ptr; + VERBOSE("unspillable[%zu]: pgno %u, npages %u, flags 0x%04X, age %u, prio %u", i, dp->pgno, dpl_npages(dl, i), + dp->flags, dpl_age(txn, i), spill_prio(txn, i, reciprocal)); + } } - if ((flags & MDBX_RDONLY) == 0 && map->filesize != size) { - err = osal_ftruncate(map->fd, size); - if (err == MDBX_SUCCESS) - map->filesize = size; - /* ignore error, because Windows unable shrink file - * that already mapped (by another process) */ - } +#if xMDBX_DEBUG_SPILLING == 2 + if (txn->tw.loose_count + txn->tw.dirtyroom <= need / 2 + 1) + ERROR("dirty-list length: before %zu, after %zu, parent %zi, loose %zu; " + "needed %zu, spillable %zu; " + "spilled %u dirty-entries, now have %zu dirty-room", + dl->length + spilled_entries, dl->length, + (txn->parent && txn->parent->tw.dirtylist) ? (intptr_t)txn->parent->tw.dirtylist->length : -1, + txn->tw.loose_count, need, spillable_entries, spilled_entries, txn->tw.dirtyroom); + ENSURE(txn->env, txn->tw.loose_count + txn->tw.dirtyroom > need / 2); +#endif /* xMDBX_DEBUG_SPILLING */ - SectionSize.QuadPart = size; - status = NtCreateSection( - &map->section, - /* DesiredAccess */ - (flags & MDBX_WRITEMAP) - ? SECTION_QUERY | SECTION_MAP_READ | SECTION_EXTEND_SIZE | - SECTION_MAP_WRITE - : SECTION_QUERY | SECTION_MAP_READ | SECTION_EXTEND_SIZE, - /* ObjectAttributes */ NULL, - /* MaximumSize (InitialSize) */ &SectionSize, - /* SectionPageProtection */ - (flags & MDBX_RDONLY) ? PAGE_READONLY : PAGE_READWRITE, - /* AllocationAttributes */ SEC_RESERVE, map->fd); +done: + return likely(txn->tw.dirtyroom + txn->tw.loose_count > ((need > CURSOR_STACK_SIZE) ? CURSOR_STACK_SIZE : need)) + ? MDBX_SUCCESS + : MDBX_TXN_FULL; +} +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 - if (!NT_SUCCESS(status)) - goto bailout_ntstatus; +int tbl_setup(const MDBX_env *env, volatile kvx_t *const kvx, const tree_t *const db) { + osal_memory_fence(mo_AcquireRelease, false); - if (ReservedAddress) { - /* release reserved address space */ - ReservedSize = 0; - status = NtFreeVirtualMemory(GetCurrentProcess(), &ReservedAddress, - &ReservedSize, MEM_RELEASE); - ReservedAddress = NULL; - if (!NT_SUCCESS(status)) - goto bailout_ntstatus; + if (unlikely(!check_table_flags(db->flags))) { + ERROR("incompatible or invalid db.flags (0x%x) ", db->flags); + return MDBX_INCOMPATIBLE; } -retry_mapview:; - SIZE_T ViewSize = (flags & MDBX_RDONLY) ? size : limit; - status = NtMapViewOfSection( - map->section, GetCurrentProcess(), &map->base, - /* ZeroBits */ 0, - /* CommitSize */ 0, - /* SectionOffset */ NULL, &ViewSize, - /* InheritDisposition */ ViewUnmap, - /* AllocationType */ (flags & MDBX_RDONLY) ? 0 : MEM_RESERVE, - /* Win32Protect */ - (flags & MDBX_WRITEMAP) ? PAGE_READWRITE : PAGE_READONLY); - - if (!NT_SUCCESS(status)) { - if (status == (NTSTATUS) /* STATUS_CONFLICTING_ADDRESSES */ 0xC0000018 && - map->base && (flags & MDBX_MRESIZE_MAY_MOVE) != 0) { - /* try remap at another base address */ - map->base = NULL; - goto retry_mapview; - } - NtClose(map->section); - map->section = NULL; - - if (map->base && (size != map->current || limit != map->limit)) { - /* try remap with previously size and limit, - * but will return MDBX_UNABLE_EXTEND_MAPSIZE on success */ - rc = (limit > map->limit) ? MDBX_UNABLE_EXTEND_MAPSIZE : MDBX_EPERM; - size = map->current; - ReservedSize = limit = map->limit; - goto retry_file_and_section; + size_t v_lmin = valsize_min(db->flags); + size_t v_lmax = env_valsize_max(env, db->flags); + if ((db->flags & (MDBX_DUPFIXED | MDBX_INTEGERDUP)) != 0 && db->dupfix_size) { + if (!MDBX_DISABLE_VALIDATION && unlikely(db->dupfix_size < v_lmin || db->dupfix_size > v_lmax)) { + ERROR("db.dupfix_size (%u) <> min/max value-length (%zu/%zu)", db->dupfix_size, v_lmin, v_lmax); + return MDBX_CORRUPTED; } + v_lmin = v_lmax = db->dupfix_size; + } - /* no way to recovery */ - goto bailout_ntstatus; + kvx->clc.k.lmin = keysize_min(db->flags); + kvx->clc.k.lmax = env_keysize_max(env, db->flags); + if (unlikely(!kvx->clc.k.cmp)) { + kvx->clc.v.cmp = builtin_datacmp(db->flags); + kvx->clc.k.cmp = builtin_keycmp(db->flags); } - assert(map->base != MAP_FAILED); + kvx->clc.v.lmin = v_lmin; + osal_memory_fence(mo_Relaxed, true); + kvx->clc.v.lmax = v_lmax; + osal_memory_fence(mo_AcquireRelease, true); - map->current = (size_t)SectionSize.QuadPart; - map->limit = ViewSize; + eASSERT(env, kvx->clc.k.lmax >= kvx->clc.k.lmin); + eASSERT(env, kvx->clc.v.lmax >= kvx->clc.v.lmin); + return MDBX_SUCCESS; +} -#else /* Windows */ +int tbl_fetch(MDBX_txn *txn, size_t dbi) { + cursor_couple_t couple; + int rc = cursor_init(&couple.outer, txn, MAIN_DBI); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; - if (flags & MDBX_RDONLY) { - if (size > map->filesize) - rc = MDBX_UNABLE_EXTEND_MAPSIZE; - else if (size < map->filesize && map->filesize > limit) - rc = MDBX_EPERM; - map->current = (map->filesize > limit) ? limit : (size_t)map->filesize; - } else { - if (size > map->filesize || - (size < map->filesize && (flags & MDBX_SHRINK_ALLOWED))) { - rc = osal_ftruncate(map->fd, size); - VERBOSE("ftruncate %zu, err %d", size, rc); - if (rc != MDBX_SUCCESS) - return rc; - map->filesize = size; - } + kvx_t *const kvx = &txn->env->kvs[dbi]; + rc = tree_search(&couple.outer, &kvx->name, 0); + if (unlikely(rc != MDBX_SUCCESS)) { + bailout: + NOTICE("dbi %zu refs to inaccessible table `%.*s` for txn %" PRIaTXN " (err %d)", dbi, (int)kvx->name.iov_len, + (const char *)kvx->name.iov_base, txn->txnid, rc); + return (rc == MDBX_NOTFOUND) ? MDBX_BAD_DBI : rc; + } - if (map->current > size) { - /* Clearing asan's bitmask for the region which released in shrinking, - * since: - * - after the shrinking we will get an exception when accessing - * this region and (therefore) do not need the help of ASAN. - * - this allows us to clear the mask only within the file size - * when closing the mapping. */ - MDBX_ASAN_UNPOISON_MEMORY_REGION( - ptr_disp(map->base, size), - ((map->current < map->limit) ? map->current : map->limit) - size); - } - map->current = (size < map->limit) ? size : map->limit; + MDBX_val data; + struct node_search_result nsr = node_search(&couple.outer, &kvx->name); + if (unlikely(!nsr.exact)) { + rc = MDBX_NOTFOUND; + goto bailout; + } + if (unlikely((node_flags(nsr.node) & (N_DUP | N_TREE)) != N_TREE)) { + NOTICE("dbi %zu refs to not a named table `%.*s` for txn %" PRIaTXN " (%s)", dbi, (int)kvx->name.iov_len, + (const char *)kvx->name.iov_base, txn->txnid, "wrong flags"); + return MDBX_INCOMPATIBLE; /* not a named DB */ } - if (limit == map->limit) + rc = node_read(&couple.outer, nsr.node, &data, couple.outer.pg[couple.outer.top]); + if (unlikely(rc != MDBX_SUCCESS)) return rc; - if (limit < map->limit) { - /* unmap an excess at end of mapping. */ - // coverity[offset_free : FALSE] - if (unlikely(munmap(ptr_disp(map->base, limit), map->limit - limit))) { - assert(errno != 0); - return errno; - } - map->limit = limit; - return rc; + if (unlikely(data.iov_len != sizeof(tree_t))) { + NOTICE("dbi %zu refs to not a named table `%.*s` for txn %" PRIaTXN " (%s)", dbi, (int)kvx->name.iov_len, + (const char *)kvx->name.iov_base, txn->txnid, "wrong rec-size"); + return MDBX_INCOMPATIBLE; /* not a named DB */ } - int err = check_mmap_limit(limit); - if (unlikely(err != MDBX_SUCCESS)) - return err; - - assert(limit > map->limit); - void *ptr = MAP_FAILED; + uint16_t flags = UNALIGNED_PEEK_16(data.iov_base, tree_t, flags); + /* The txn may not know this DBI, or another process may + * have dropped and recreated the DB with other flags. */ + tree_t *const db = &txn->dbs[dbi]; + if (unlikely((db->flags & DB_PERSISTENT_FLAGS) != flags)) { + NOTICE("dbi %zu refs to the re-created table `%.*s` for txn %" PRIaTXN + " with different flags (present 0x%X != wanna 0x%X)", + dbi, (int)kvx->name.iov_len, (const char *)kvx->name.iov_base, txn->txnid, db->flags & DB_PERSISTENT_FLAGS, + flags); + return MDBX_INCOMPATIBLE; + } -#if (defined(__linux__) || defined(__gnu_linux__)) && defined(_GNU_SOURCE) - ptr = mremap(map->base, map->limit, limit, -#if defined(MREMAP_MAYMOVE) - (flags & MDBX_MRESIZE_MAY_MOVE) ? MREMAP_MAYMOVE : -#endif /* MREMAP_MAYMOVE */ - 0); - if (ptr == MAP_FAILED) { - err = errno; - assert(err != 0); - switch (err) { - default: - return err; - case 0 /* paranoia */: - case EAGAIN: - case ENOMEM: - return MDBX_UNABLE_EXTEND_MAPSIZE; - case EFAULT /* MADV_DODUMP / MADV_DONTDUMP are mixed for mmap-range */: - break; - } + memcpy(db, data.iov_base, sizeof(tree_t)); +#if !MDBX_DISABLE_VALIDATION + const txnid_t pp_txnid = couple.outer.pg[couple.outer.top]->txnid; + tASSERT(txn, txn->front_txnid >= pp_txnid); + if (unlikely(db->mod_txnid > pp_txnid)) { + ERROR("db.mod_txnid (%" PRIaTXN ") > page-txnid (%" PRIaTXN ")", db->mod_txnid, pp_txnid); + return MDBX_CORRUPTED; } -#endif /* Linux & _GNU_SOURCE */ +#endif /* !MDBX_DISABLE_VALIDATION */ + rc = tbl_setup_ifneed(txn->env, kvx, db); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; - const unsigned mmap_flags = - MAP_CONCEAL | MAP_SHARED | MAP_FILE | MAP_NORESERVE | - (F_ISSET(flags, MDBX_UTTERLY_NOSYNC) ? MAP_NOSYNC : 0); - const unsigned mmap_prot = - (flags & MDBX_WRITEMAP) ? PROT_READ | PROT_WRITE : PROT_READ; + if (unlikely(dbi_changed(txn, dbi))) + return MDBX_BAD_DBI; - if (ptr == MAP_FAILED) { - /* Try to mmap additional space beyond the end of mapping. */ - ptr = mmap(ptr_disp(map->base, map->limit), limit - map->limit, mmap_prot, - mmap_flags | MAP_FIXED_NOREPLACE, map->fd, map->limit); - if (ptr == ptr_disp(map->base, map->limit)) - /* успешно прилепили отображение в конец */ - ptr = map->base; - else if (ptr != MAP_FAILED) { - /* the desired address is busy, unmap unsuitable one */ - if (unlikely(munmap(ptr, limit - map->limit))) { - assert(errno != 0); - return errno; - } - ptr = MAP_FAILED; - } else { - err = errno; - assert(err != 0); - switch (err) { - default: - return err; - case 0 /* paranoia */: - case EAGAIN: - case ENOMEM: - return MDBX_UNABLE_EXTEND_MAPSIZE; - case EEXIST: /* address busy */ - case EINVAL: /* kernel don't support MAP_FIXED_NOREPLACE */ - break; - } - } - } + txn->dbi_state[dbi] &= ~DBI_STALE; + return MDBX_SUCCESS; +} +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 - if (ptr == MAP_FAILED) { - /* unmap and map again whole region */ - if ((flags & MDBX_MRESIZE_MAY_UNMAP) == 0) { - /* TODO: Perhaps here it is worth to implement suspend/resume threads - * and perform unmap/map as like for Windows. */ - return MDBX_UNABLE_EXTEND_MAPSIZE; - } +typedef struct rthc_entry { + MDBX_env *env; +} rthc_entry_t; - if (unlikely(munmap(map->base, map->limit))) { - assert(errno != 0); - return errno; - } +#if MDBX_DEBUG +#define RTHC_INITIAL_LIMIT 1 +#else +#define RTHC_INITIAL_LIMIT 16 +#endif - // coverity[pass_freed_arg : FALSE] - ptr = mmap(map->base, limit, mmap_prot, - (flags & MDBX_MRESIZE_MAY_MOVE) - ? mmap_flags - : mmap_flags | (MAP_FIXED_NOREPLACE ? MAP_FIXED_NOREPLACE - : MAP_FIXED), - map->fd, 0); - if (MAP_FIXED_NOREPLACE != 0 && MAP_FIXED_NOREPLACE != MAP_FIXED && - unlikely(ptr == MAP_FAILED) && !(flags & MDBX_MRESIZE_MAY_MOVE) && - errno == /* kernel don't support MAP_FIXED_NOREPLACE */ EINVAL) - // coverity[pass_freed_arg : FALSE] - ptr = - mmap(map->base, limit, mmap_prot, mmap_flags | MAP_FIXED, map->fd, 0); +static unsigned rthc_count, rthc_limit = RTHC_INITIAL_LIMIT; +static rthc_entry_t rthc_table_static[RTHC_INITIAL_LIMIT]; +static rthc_entry_t *rthc_table = rthc_table_static; - if (unlikely(ptr == MAP_FAILED)) { - /* try to restore prev mapping */ - // coverity[pass_freed_arg : FALSE] - ptr = mmap(map->base, map->limit, mmap_prot, - (flags & MDBX_MRESIZE_MAY_MOVE) - ? mmap_flags - : mmap_flags | (MAP_FIXED_NOREPLACE ? MAP_FIXED_NOREPLACE - : MAP_FIXED), - map->fd, 0); - if (MAP_FIXED_NOREPLACE != 0 && MAP_FIXED_NOREPLACE != MAP_FIXED && - unlikely(ptr == MAP_FAILED) && !(flags & MDBX_MRESIZE_MAY_MOVE) && - errno == /* kernel don't support MAP_FIXED_NOREPLACE */ EINVAL) - // coverity[pass_freed_arg : FALSE] - ptr = mmap(map->base, map->limit, mmap_prot, mmap_flags | MAP_FIXED, - map->fd, 0); - if (unlikely(ptr == MAP_FAILED)) { - VALGRIND_MAKE_MEM_NOACCESS(map->base, map->current); - /* Unpoisoning is required for ASAN to avoid false-positive diagnostic - * when this memory will re-used by malloc or another mmapping. - * See - * https://libmdbx.dqdkfa.ru/dead-github/pull/93#issuecomment-613687203 - */ - MDBX_ASAN_UNPOISON_MEMORY_REGION( - map->base, (map->current < map->limit) ? map->current : map->limit); - map->limit = 0; - map->current = 0; - map->base = nullptr; - assert(errno != 0); - return errno; +static int uniq_peek(const osal_mmap_t *pending, osal_mmap_t *scan) { + int rc; + uint64_t bait; + lck_t *const pending_lck = pending->lck; + lck_t *const scan_lck = scan->lck; + if (pending_lck) { + bait = atomic_load64(&pending_lck->bait_uniqueness, mo_AcquireRelease); + rc = MDBX_SUCCESS; + } else { + bait = 0 /* hush MSVC warning */; + rc = osal_msync(scan, 0, sizeof(lck_t), MDBX_SYNC_DATA); + if (rc == MDBX_SUCCESS) + rc = osal_pread(pending->fd, &bait, sizeof(scan_lck->bait_uniqueness), offsetof(lck_t, bait_uniqueness)); + } + if (likely(rc == MDBX_SUCCESS) && bait == atomic_load64(&scan_lck->bait_uniqueness, mo_AcquireRelease)) + rc = MDBX_RESULT_TRUE; + + TRACE("uniq-peek: %s, bait 0x%016" PRIx64 ",%s rc %d", pending_lck ? "mem" : "file", bait, + (rc == MDBX_RESULT_TRUE) ? " found," : (rc ? " FAILED," : ""), rc); + return rc; +} + +static int uniq_poke(const osal_mmap_t *pending, osal_mmap_t *scan, uint64_t *abra) { + if (*abra == 0) { + const uintptr_t tid = osal_thread_self(); + uintptr_t uit = 0; + memcpy(&uit, &tid, (sizeof(tid) < sizeof(uit)) ? sizeof(tid) : sizeof(uit)); + *abra = rrxmrrxmsx_0(osal_monotime() + UINT64_C(5873865991930747) * uit); + } + const uint64_t cadabra = + rrxmrrxmsx_0(*abra + UINT64_C(7680760450171793) * (unsigned)osal_getpid()) << 24 | *abra >> 40; + lck_t *const scan_lck = scan->lck; + atomic_store64(&scan_lck->bait_uniqueness, cadabra, mo_AcquireRelease); + *abra = *abra * UINT64_C(6364136223846793005) + 1; + return uniq_peek(pending, scan); +} + +__cold int rthc_uniq_check(const osal_mmap_t *pending, MDBX_env **found) { + *found = nullptr; + uint64_t salt = 0; + for (size_t i = 0; i < rthc_count; ++i) { + MDBX_env *const scan = rthc_table[i].env; + if (!scan->lck_mmap.lck || &scan->lck_mmap == pending) + continue; + int err = atomic_load64(&scan->lck_mmap.lck->bait_uniqueness, mo_AcquireRelease) + ? uniq_peek(pending, &scan->lck_mmap) + : uniq_poke(pending, &scan->lck_mmap, &salt); + if (err == MDBX_ENODATA) { + uint64_t length = 0; + if (likely(osal_filesize(pending->fd, &length) == MDBX_SUCCESS && length == 0)) { + /* LY: skip checking since LCK-file is empty, i.e. just created. */ + DEBUG("%s", "unique (new/empty lck)"); + return MDBX_SUCCESS; } - rc = MDBX_UNABLE_EXTEND_MAPSIZE; - limit = map->limit; + } + if (err == MDBX_RESULT_TRUE) + err = uniq_poke(pending, &scan->lck_mmap, &salt); + if (err == MDBX_RESULT_TRUE) { + (void)osal_msync(&scan->lck_mmap, 0, sizeof(lck_t), MDBX_SYNC_KICK); + err = uniq_poke(pending, &scan->lck_mmap, &salt); + } + if (err == MDBX_RESULT_TRUE) { + err = uniq_poke(pending, &scan->lck_mmap, &salt); + *found = scan; + DEBUG("found %p", __Wpedantic_format_voidptr(*found)); + return MDBX_SUCCESS; + } + if (unlikely(err != MDBX_SUCCESS)) { + DEBUG("failed rc %d", err); + return err; } } - assert(ptr && ptr != MAP_FAILED); - if (map->base != ptr) { - VALGRIND_MAKE_MEM_NOACCESS(map->base, map->current); - /* Unpoisoning is required for ASAN to avoid false-positive diagnostic - * when this memory will re-used by malloc or another mmapping. - * See - * https://libmdbx.dqdkfa.ru/dead-github/pull/93#issuecomment-613687203 - */ - MDBX_ASAN_UNPOISON_MEMORY_REGION( - map->base, (map->current < map->limit) ? map->current : map->limit); + DEBUG("%s", "unique"); + return MDBX_SUCCESS; +} - VALGRIND_MAKE_MEM_DEFINED(ptr, map->current); - MDBX_ASAN_UNPOISON_MEMORY_REGION(ptr, map->current); - map->base = ptr; - } - map->limit = limit; - map->current = size; +//------------------------------------------------------------------------------ -#if MDBX_ENABLE_MADVISE -#ifdef MADV_DONTFORK - if (unlikely(madvise(map->base, map->limit, MADV_DONTFORK) != 0)) { - assert(errno != 0); - return errno; - } -#endif /* MADV_DONTFORK */ -#ifdef MADV_NOHUGEPAGE - (void)madvise(map->base, map->limit, MADV_NOHUGEPAGE); -#endif /* MADV_NOHUGEPAGE */ -#endif /* MDBX_ENABLE_MADVISE */ +#if defined(_WIN32) || defined(_WIN64) +static CRITICAL_SECTION rthc_critical_section; +#else -#endif /* POSIX / Windows */ +static pthread_mutex_t rthc_mutex = PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t rthc_cond = PTHREAD_COND_INITIALIZER; +static osal_thread_key_t rthc_key; +static mdbx_atomic_uint32_t rthc_pending; - /* Zap: Redundant code */ - MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(6287); - assert(rc != MDBX_SUCCESS || - (map->base != nullptr && map->base != MAP_FAILED && - map->current == size && map->limit == limit && - map->filesize >= size)); - return rc; +static inline uint64_t rthc_signature(const void *addr, uint8_t kind) { + uint64_t salt = osal_thread_self() * UINT64_C(0xA2F0EEC059629A17) ^ UINT64_C(0x01E07C6FDB596497) * (uintptr_t)(addr); +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + return salt << 8 | kind; +#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + return (uint64_t)kind << 56 | salt >> 8; +#else +#error "FIXME: Unsupported byte order" +#endif /* __BYTE_ORDER__ */ } -/*----------------------------------------------------------------------------*/ +#define MDBX_THREAD_RTHC_REGISTERED(addr) rthc_signature(addr, 0x0D) +#define MDBX_THREAD_RTHC_COUNTED(addr) rthc_signature(addr, 0xC0) +static __thread uint64_t rthc_thread_state +#if __has_attribute(tls_model) && (defined(__PIC__) || defined(__pic__) || MDBX_BUILD_SHARED_LIBRARY) + __attribute__((tls_model("local-dynamic"))) +#endif + ; -__cold MDBX_INTERNAL_FUNC void osal_jitter(bool tiny) { - for (;;) { -#if defined(_M_IX86) || defined(_M_X64) || defined(__i386__) || \ - defined(__x86_64__) - const unsigned salt = 277u * (unsigned)__rdtsc(); -#elif (defined(_WIN32) || defined(_WIN64)) && MDBX_WITHOUT_MSVC_CRT - static ULONG state; - const unsigned salt = (unsigned)RtlRandomEx(&state); +#if defined(__APPLE__) && defined(__SANITIZE_ADDRESS__) && !defined(MDBX_ATTRIBUTE_NO_SANITIZE_ADDRESS) +/* Avoid ASAN-trap due the target TLS-variable feed by Darwin's tlv_free() */ +#define MDBX_ATTRIBUTE_NO_SANITIZE_ADDRESS __attribute__((__no_sanitize_address__, __noinline__)) #else - const unsigned salt = rand(); +#define MDBX_ATTRIBUTE_NO_SANITIZE_ADDRESS inline #endif - const unsigned coin = salt % (tiny ? 29u : 43u); - if (coin < 43 / 3) - break; -#if defined(_WIN32) || defined(_WIN64) - SwitchToThread(); - if (coin > 43 * 2 / 3) - Sleep(1); +MDBX_ATTRIBUTE_NO_SANITIZE_ADDRESS static uint64_t rthc_read(const void *rthc) { return *(volatile uint64_t *)rthc; } + +MDBX_ATTRIBUTE_NO_SANITIZE_ADDRESS static uint64_t rthc_compare_and_clean(const void *rthc, const uint64_t signature) { +#if MDBX_64BIT_CAS + return atomic_cas64((mdbx_atomic_uint64_t *)rthc, signature, 0); +#elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + return atomic_cas32((mdbx_atomic_uint32_t *)rthc, (uint32_t)signature, 0); +#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + return atomic_cas32((mdbx_atomic_uint32_t *)rthc, (uint32_t)(signature >> 32), 0); #else - sched_yield(); - if (coin > 43 * 2 / 3) - usleep(coin); +#error "FIXME: Unsupported byte order" #endif - } } -/*----------------------------------------------------------------------------*/ +static inline int rthc_atexit(void (*dtor)(void *), void *obj, void *dso_symbol) { +#ifndef MDBX_HAVE_CXA_THREAD_ATEXIT_IMPL +#if defined(LIBCXXABI_HAS_CXA_THREAD_ATEXIT_IMPL) || defined(HAVE___CXA_THREAD_ATEXIT_IMPL) || \ + __GLIBC_PREREQ(2, 18) || defined(BIONIC) +#define MDBX_HAVE_CXA_THREAD_ATEXIT_IMPL 1 +#else +#define MDBX_HAVE_CXA_THREAD_ATEXIT_IMPL 0 +#endif +#endif /* MDBX_HAVE_CXA_THREAD_ATEXIT_IMPL */ -#if defined(_WIN32) || defined(_WIN64) -static LARGE_INTEGER performance_frequency; -#elif defined(__APPLE__) || defined(__MACH__) -#include -static uint64_t ratio_16dot16_to_monotine; -#elif defined(__linux__) || defined(__gnu_linux__) -static clockid_t posix_clockid; -__cold static clockid_t choice_monoclock(void) { - struct timespec probe; -#if defined(CLOCK_BOOTTIME) - if (clock_gettime(CLOCK_BOOTTIME, &probe) == 0) - return CLOCK_BOOTTIME; -#elif defined(CLOCK_MONOTONIC_RAW) - if (clock_gettime(CLOCK_MONOTONIC_RAW, &probe) == 0) - return CLOCK_MONOTONIC_RAW; -#elif defined(CLOCK_MONOTONIC_COARSE) - if (clock_gettime(CLOCK_MONOTONIC_COARSE, &probe) == 0) - return CLOCK_MONOTONIC_COARSE; +#ifndef MDBX_HAVE_CXA_THREAD_ATEXIT +#if defined(LIBCXXABI_HAS_CXA_THREAD_ATEXIT) || defined(HAVE___CXA_THREAD_ATEXIT) +#define MDBX_HAVE_CXA_THREAD_ATEXIT 1 +#elif !MDBX_HAVE_CXA_THREAD_ATEXIT_IMPL && (defined(__linux__) || defined(__gnu_linux__)) +#define MDBX_HAVE_CXA_THREAD_ATEXIT 1 +#else +#define MDBX_HAVE_CXA_THREAD_ATEXIT 0 #endif - return CLOCK_MONOTONIC; -} -#elif defined(CLOCK_MONOTONIC) -#define posix_clockid CLOCK_MONOTONIC +#endif /* MDBX_HAVE_CXA_THREAD_ATEXIT */ + + int rc = MDBX_ENOSYS; +#if MDBX_HAVE_CXA_THREAD_ATEXIT_IMPL && !MDBX_HAVE_CXA_THREAD_ATEXIT +#define __cxa_thread_atexit __cxa_thread_atexit_impl +#endif +#if MDBX_HAVE_CXA_THREAD_ATEXIT || defined(__cxa_thread_atexit) + extern int __cxa_thread_atexit(void (*dtor)(void *), void *obj, void *dso_symbol) MDBX_WEAK_IMPORT_ATTRIBUTE; + if (&__cxa_thread_atexit) + rc = __cxa_thread_atexit(dtor, obj, dso_symbol); +#elif defined(__APPLE__) || defined(_DARWIN_C_SOURCE) + extern void _tlv_atexit(void (*termfunc)(void *objAddr), void *objAddr) MDBX_WEAK_IMPORT_ATTRIBUTE; + if (&_tlv_atexit) { + (void)dso_symbol; + _tlv_atexit(dtor, obj); + rc = 0; + } #else -#define posix_clockid CLOCK_REALTIME + (void)dtor; + (void)obj; + (void)dso_symbol; #endif + return rc; +} + +__cold void workaround_glibc_bug21031(void) { + /* Workaround for https://sourceware.org/bugzilla/show_bug.cgi?id=21031 + * + * Due race between pthread_key_delete() and __nptl_deallocate_tsd() + * The destructor(s) of thread-local-storage object(s) may be running + * in another thread(s) and be blocked or not finished yet. + * In such case we get a SEGFAULT after unload this library DSO. + * + * So just by yielding a few timeslices we give a chance + * to such destructor(s) for completion and avoids segfault. */ + sched_yield(); + sched_yield(); + sched_yield(); +} +#endif /* !Windows */ -MDBX_INTERNAL_FUNC uint64_t osal_16dot16_to_monotime(uint32_t seconds_16dot16) { +void rthc_lock(void) { #if defined(_WIN32) || defined(_WIN64) - const uint64_t ratio = performance_frequency.QuadPart; -#elif defined(__APPLE__) || defined(__MACH__) - const uint64_t ratio = ratio_16dot16_to_monotine; + EnterCriticalSection(&rthc_critical_section); #else - const uint64_t ratio = UINT64_C(1000000000); + ENSURE(nullptr, osal_pthread_mutex_lock(&rthc_mutex) == 0); #endif - const uint64_t ret = (ratio * seconds_16dot16 + 32768) >> 16; - return likely(ret || seconds_16dot16 == 0) ? ret : /* fix underflow */ 1; } -static uint64_t monotime_limit; -MDBX_INTERNAL_FUNC uint32_t osal_monotime_to_16dot16(uint64_t monotime) { - if (unlikely(monotime > monotime_limit)) - return UINT32_MAX; - - const uint32_t ret = +void rthc_unlock(void) { #if defined(_WIN32) || defined(_WIN64) - (uint32_t)((monotime << 16) / performance_frequency.QuadPart); -#elif defined(__APPLE__) || defined(__MACH__) - (uint32_t)((monotime << 16) / ratio_16dot16_to_monotine); + LeaveCriticalSection(&rthc_critical_section); #else - (uint32_t)((monotime << 7) / 1953125); + ENSURE(nullptr, pthread_mutex_unlock(&rthc_mutex) == 0); #endif - return ret; } -MDBX_INTERNAL_FUNC uint64_t osal_monotime(void) { +static inline int thread_key_create(osal_thread_key_t *key) { + int rc; #if defined(_WIN32) || defined(_WIN64) - LARGE_INTEGER counter; - if (QueryPerformanceCounter(&counter)) - return counter.QuadPart; -#elif defined(__APPLE__) || defined(__MACH__) - return mach_absolute_time(); + *key = TlsAlloc(); + rc = (*key != TLS_OUT_OF_INDEXES) ? MDBX_SUCCESS : GetLastError(); #else - struct timespec ts; - if (likely(clock_gettime(posix_clockid, &ts) == 0)) - return ts.tv_sec * UINT64_C(1000000000) + ts.tv_nsec; + rc = pthread_key_create(key, nullptr); #endif - return 0; + TRACE("&key = %p, value %" PRIuPTR ", rc %d", __Wpedantic_format_voidptr(key), (uintptr_t)*key, rc); + return rc; } -MDBX_INTERNAL_FUNC uint64_t osal_cputime(size_t *optional_page_faults) { +void thread_rthc_set(osal_thread_key_t key, const void *value) { #if defined(_WIN32) || defined(_WIN64) - if (optional_page_faults) { - PROCESS_MEMORY_COUNTERS pmc; - *optional_page_faults = - GetProcessMemoryInfo(GetCurrentProcess(), &pmc, sizeof(pmc)) - ? pmc.PageFaultCount - : 0; - } - FILETIME unused, usermode; - if (GetThreadTimes(GetCurrentThread(), - /* CreationTime */ &unused, - /* ExitTime */ &unused, - /* KernelTime */ &unused, - /* UserTime */ &usermode)) { - /* one second = 10_000_000 * 100ns = 78125 * (1 << 7) * 100ns; - * result = (h * f / 10_000_000) << 32) + l * f / 10_000_000 = - * = ((h * f) >> 7) / 78125) << 32) + ((l * f) >> 7) / 78125; - * 1) {h, l} *= f; - * 2) {h, l} >>= 7; - * 3) result = ((h / 78125) << 32) + l / 78125; */ - uint64_t l = usermode.dwLowDateTime * performance_frequency.QuadPart; - uint64_t h = usermode.dwHighDateTime * performance_frequency.QuadPart; - l = h << (64 - 7) | l >> 7; - h = h >> 7; - return ((h / 78125) << 32) + l / 78125; + ENSURE(nullptr, TlsSetValue(key, (void *)value)); +#else + const uint64_t sign_registered = MDBX_THREAD_RTHC_REGISTERED(&rthc_thread_state); + const uint64_t sign_counted = MDBX_THREAD_RTHC_COUNTED(&rthc_thread_state); + if (value && unlikely(rthc_thread_state != sign_registered && rthc_thread_state != sign_counted)) { + rthc_thread_state = sign_registered; + TRACE("thread registered 0x%" PRIxPTR, osal_thread_self()); + if (rthc_atexit(rthc_thread_dtor, &rthc_thread_state, (void *)&mdbx_version /* dso_anchor */)) { + ENSURE(nullptr, pthread_setspecific(rthc_key, &rthc_thread_state) == 0); + rthc_thread_state = sign_counted; + const unsigned count_before = atomic_add32(&rthc_pending, 1); + ENSURE(nullptr, count_before < INT_MAX); + NOTICE("fallback to pthreads' tsd, key %" PRIuPTR ", count %u", (uintptr_t)rthc_key, count_before); + (void)count_before; + } } -#elif defined(RUSAGE_THREAD) || defined(RUSAGE_LWP) -#ifndef RUSAGE_THREAD -#define RUSAGE_THREAD RUSAGE_LWP /* Solaris */ + ENSURE(nullptr, pthread_setspecific(key, value) == 0); #endif - struct rusage usage; - if (getrusage(RUSAGE_THREAD, &usage) == 0) { - if (optional_page_faults) - *optional_page_faults = usage.ru_majflt; - return usage.ru_utime.tv_sec * UINT64_C(1000000000) + - usage.ru_utime.tv_usec * 1000u; +} + +/* dtor called for thread, i.e. for all mdbx's environment objects */ +__cold void rthc_thread_dtor(void *rthc) { + rthc_lock(); + const uint32_t current_pid = osal_getpid(); +#if defined(_WIN32) || defined(_WIN64) + TRACE(">> pid %d, thread 0x%" PRIxPTR ", module %p", current_pid, osal_thread_self(), rthc); +#else + TRACE(">> pid %d, thread 0x%" PRIxPTR ", rthc %p", current_pid, osal_thread_self(), rthc); +#endif + + for (size_t i = 0; i < rthc_count; ++i) { + MDBX_env *const env = rthc_table[i].env; + if (env->pid != current_pid) + continue; + if (!(env->flags & ENV_TXKEY)) + continue; + reader_slot_t *const reader = thread_rthc_get(env->me_txkey); + reader_slot_t *const begin = &env->lck_mmap.lck->rdt[0]; + reader_slot_t *const end = &env->lck_mmap.lck->rdt[env->max_readers]; + if (reader < begin || reader >= end) + continue; +#if !defined(_WIN32) && !defined(_WIN64) + if (pthread_setspecific(env->me_txkey, nullptr) != 0) { + TRACE("== thread 0x%" PRIxPTR ", rthc %p: ignore race with tsd-key deletion", osal_thread_self(), + __Wpedantic_format_voidptr(reader)); + continue /* ignore race with tsd-key deletion by mdbx_env_close() */; + } +#endif + + TRACE("== thread 0x%" PRIxPTR ", rthc %p, [%zi], %p ... %p (%+i), rtch-pid %i, " + "current-pid %i", + osal_thread_self(), __Wpedantic_format_voidptr(reader), i, __Wpedantic_format_voidptr(begin), + __Wpedantic_format_voidptr(end), (int)(reader - begin), reader->pid.weak, current_pid); + if (atomic_load32(&reader->pid, mo_Relaxed) == current_pid) { + TRACE("==== thread 0x%" PRIxPTR ", rthc %p, cleanup", osal_thread_self(), __Wpedantic_format_voidptr(reader)); + (void)atomic_cas32(&reader->pid, current_pid, 0); + atomic_store32(&env->lck->rdt_refresh_flag, true, mo_Relaxed); + } } - if (optional_page_faults) - *optional_page_faults = 0; -#elif defined(CLOCK_THREAD_CPUTIME_ID) - if (optional_page_faults) - *optional_page_faults = 0; - struct timespec ts; - if (likely(clock_gettime(CLOCK_THREAD_CPUTIME_ID, &ts) == 0)) - return ts.tv_sec * UINT64_C(1000000000) + ts.tv_nsec; + +#if defined(_WIN32) || defined(_WIN64) + TRACE("<< thread 0x%" PRIxPTR ", module %p", osal_thread_self(), rthc); + rthc_unlock(); #else - /* FIXME */ - if (optional_page_faults) - *optional_page_faults = 0; + const uint64_t sign_registered = MDBX_THREAD_RTHC_REGISTERED(rthc); + const uint64_t sign_counted = MDBX_THREAD_RTHC_COUNTED(rthc); + const uint64_t state = rthc_read(rthc); + if (state == sign_registered && rthc_compare_and_clean(rthc, sign_registered)) { + TRACE("== thread 0x%" PRIxPTR ", rthc %p, pid %d, self-status %s (0x%08" PRIx64 ")", osal_thread_self(), rthc, + osal_getpid(), "registered", state); + } else if (state == sign_counted && rthc_compare_and_clean(rthc, sign_counted)) { + TRACE("== thread 0x%" PRIxPTR ", rthc %p, pid %d, self-status %s (0x%08" PRIx64 ")", osal_thread_self(), rthc, + osal_getpid(), "counted", state); + ENSURE(nullptr, atomic_sub32(&rthc_pending, 1) > 0); + } else { + WARNING("thread 0x%" PRIxPTR ", rthc %p, pid %d, self-status %s (0x%08" PRIx64 ")", osal_thread_self(), rthc, + osal_getpid(), "wrong", state); + } + + if (atomic_load32(&rthc_pending, mo_AcquireRelease) == 0) { + TRACE("== thread 0x%" PRIxPTR ", rthc %p, pid %d, wake", osal_thread_self(), rthc, osal_getpid()); + ENSURE(nullptr, pthread_cond_broadcast(&rthc_cond) == 0); + } + + TRACE("<< thread 0x%" PRIxPTR ", rthc %p", osal_thread_self(), rthc); + /* Allow tail call optimization, i.e. gcc should generate the jmp instruction + * instead of a call for pthread_mutex_unlock() and therefore CPU could not + * return to current DSO's code section, which may be unloaded immediately + * after the mutex got released. */ + pthread_mutex_unlock(&rthc_mutex); #endif - return 0; } -/*----------------------------------------------------------------------------*/ +__cold int rthc_register(MDBX_env *const env) { + TRACE(">> env %p, rthc_count %u, rthc_limit %u", __Wpedantic_format_voidptr(env), rthc_count, rthc_limit); -static void bootid_shake(bin128_t *p) { - /* Bob Jenkins's PRNG: https://burtleburtle.net/bob/rand/smallprng.html */ - const uint32_t e = p->a - (p->b << 23 | p->b >> 9); - p->a = p->b ^ (p->c << 16 | p->c >> 16); - p->b = p->c + (p->d << 11 | p->d >> 21); - p->c = p->d + e; - p->d = e + p->a; -} + int rc = MDBX_SUCCESS; + for (size_t i = 0; i < rthc_count; ++i) + if (unlikely(rthc_table[i].env == env)) { + rc = MDBX_PANIC; + goto bailout; + } -__cold static void bootid_collect(bin128_t *p, const void *s, size_t n) { - p->y += UINT64_C(64526882297375213); - bootid_shake(p); - for (size_t i = 0; i < n; ++i) { - bootid_shake(p); - p->y ^= UINT64_C(48797879452804441) * ((const uint8_t *)s)[i]; - bootid_shake(p); - p->y += 14621231; + env->me_txkey = 0; + if (unlikely(rthc_count == rthc_limit)) { + rthc_entry_t *new_table = + osal_realloc((rthc_table == rthc_table_static) ? nullptr : rthc_table, sizeof(rthc_entry_t) * rthc_limit * 2); + if (unlikely(new_table == nullptr)) { + rc = MDBX_ENOMEM; + goto bailout; + } + if (rthc_table == rthc_table_static) + memcpy(new_table, rthc_table, sizeof(rthc_entry_t) * rthc_limit); + rthc_table = new_table; + rthc_limit *= 2; } - bootid_shake(p); - /* minor non-linear tomfoolery */ - const unsigned z = p->x % 61; - p->y = p->y << z | p->y >> (64 - z); - bootid_shake(p); - bootid_shake(p); - const unsigned q = p->x % 59; - p->y = p->y << q | p->y >> (64 - q); - bootid_shake(p); - bootid_shake(p); - bootid_shake(p); -} + if ((env->flags & MDBX_NOSTICKYTHREADS) == 0) { + rc = thread_key_create(&env->me_txkey); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + env->flags |= ENV_TXKEY; + } -#if defined(_WIN32) || defined(_WIN64) + rthc_table[rthc_count].env = env; + TRACE("== [%i] = env %p, key %" PRIuPTR, rthc_count, __Wpedantic_format_voidptr(env), (uintptr_t)env->me_txkey); + ++rthc_count; -__cold static uint64_t windows_systemtime_ms() { - FILETIME ft; - GetSystemTimeAsFileTime(&ft); - return ((uint64_t)ft.dwHighDateTime << 32 | ft.dwLowDateTime) / 10000ul; +bailout: + TRACE("<< env %p, key %" PRIuPTR ", rthc_count %u, rthc_limit %u, rc %d", __Wpedantic_format_voidptr(env), + (uintptr_t)env->me_txkey, rthc_count, rthc_limit, rc); + return rc; } -__cold static uint64_t windows_bootime(void) { - unsigned confirmed = 0; - uint64_t boottime = 0; - uint64_t up0 = mdbx_GetTickCount64(); - uint64_t st0 = windows_systemtime_ms(); - for (uint64_t fuse = st0; up0 && st0 < fuse + 1000 * 1000u / 42;) { - YieldProcessor(); - const uint64_t up1 = mdbx_GetTickCount64(); - const uint64_t st1 = windows_systemtime_ms(); - if (st1 > fuse && st1 == st0 && up1 == up0) { - uint64_t diff = st1 - up1; - if (boottime == diff) { - if (++confirmed > 4) - return boottime; - } else { - confirmed = 0; - boottime = diff; +__cold static int rthc_drown(MDBX_env *const env) { + const uint32_t current_pid = osal_getpid(); + int rc = MDBX_SUCCESS; + MDBX_env *inprocess_neighbor = nullptr; + if (likely(env->lck_mmap.lck && current_pid == env->pid)) { + reader_slot_t *const begin = &env->lck_mmap.lck->rdt[0]; + reader_slot_t *const end = &env->lck_mmap.lck->rdt[env->max_readers]; + TRACE("== %s env %p pid %d, readers %p ...%p, current-pid %d", (current_pid == env->pid) ? "cleanup" : "skip", + __Wpedantic_format_voidptr(env), env->pid, __Wpedantic_format_voidptr(begin), __Wpedantic_format_voidptr(end), + current_pid); + bool cleaned = false; + for (reader_slot_t *r = begin; r < end; ++r) { + if (atomic_load32(&r->pid, mo_Relaxed) == current_pid) { + atomic_store32(&r->pid, 0, mo_AcquireRelease); + TRACE("== cleanup %p", __Wpedantic_format_voidptr(r)); + cleaned = true; + } + } + if (cleaned) + atomic_store32(&env->lck_mmap.lck->rdt_refresh_flag, true, mo_Relaxed); + rc = rthc_uniq_check(&env->lck_mmap, &inprocess_neighbor); + if (!inprocess_neighbor && env->registered_reader_pid && env->lck_mmap.fd != INVALID_HANDLE_VALUE) { + int err = lck_rpid_clear(env); + rc = rc ? rc : err; + } + } + int err = lck_destroy(env, inprocess_neighbor, current_pid); + env->pid = 0; + return rc ? rc : err; +} + +__cold int rthc_remove(MDBX_env *const env) { + TRACE(">>> env %p, key %zu, rthc_count %u, rthc_limit %u", __Wpedantic_format_voidptr(env), (uintptr_t)env->me_txkey, + rthc_count, rthc_limit); + + int rc = MDBX_SUCCESS; + if (likely(env->pid)) + rc = rthc_drown(env); + + for (size_t i = 0; i < rthc_count; ++i) { + if (rthc_table[i].env == env) { + if (--rthc_count > 0) + rthc_table[i] = rthc_table[rthc_count]; + else if (rthc_table != rthc_table_static) { + void *tmp = rthc_table; + rthc_table = rthc_table_static; + rthc_limit = RTHC_INITIAL_LIMIT; + osal_memory_barrier(); + osal_free(tmp); } - fuse = st1; - Sleep(1); + break; } - st0 = st1; - up0 = up1; } - return 0; + + TRACE("<<< %p, key %zu, rthc_count %u, rthc_limit %u", __Wpedantic_format_voidptr(env), (uintptr_t)env->me_txkey, + rthc_count, rthc_limit); + return rc; } -__cold static LSTATUS mdbx_RegGetValue(HKEY hKey, LPCSTR lpSubKey, - LPCSTR lpValue, PVOID pvData, - LPDWORD pcbData) { - LSTATUS rc; - if (!mdbx_RegGetValueA) { - /* an old Windows 2000/XP */ - HKEY hSubKey; - rc = RegOpenKeyA(hKey, lpSubKey, &hSubKey); - if (rc == ERROR_SUCCESS) { - rc = RegQueryValueExA(hSubKey, lpValue, NULL, NULL, pvData, pcbData); - RegCloseKey(hSubKey); - } - return rc; +#if !defined(_WIN32) && !defined(_WIN64) +__cold void rthc_afterfork(void) { + NOTICE("drown %d rthc entries", rthc_count); + for (size_t i = 0; i < rthc_count; ++i) { + MDBX_env *const env = rthc_table[i].env; + NOTICE("drown env %p", __Wpedantic_format_voidptr(env)); + if (env->lck_mmap.lck) + osal_munmap(&env->lck_mmap); + if (env->dxb_mmap.base) { + osal_munmap(&env->dxb_mmap); +#ifdef ENABLE_MEMCHECK + VALGRIND_DISCARD(env->valgrind_handle); + env->valgrind_handle = -1; +#endif /* ENABLE_MEMCHECK */ + } + env->lck = lckless_stub(env); + rthc_drown(env); } - - rc = mdbx_RegGetValueA(hKey, lpSubKey, lpValue, RRF_RT_ANY, NULL, pvData, - pcbData); - if (rc != ERROR_FILE_NOT_FOUND) - return rc; - - rc = mdbx_RegGetValueA(hKey, lpSubKey, lpValue, - RRF_RT_ANY | 0x00010000 /* RRF_SUBKEY_WOW6464KEY */, - NULL, pvData, pcbData); - if (rc != ERROR_FILE_NOT_FOUND) - return rc; - return mdbx_RegGetValueA(hKey, lpSubKey, lpValue, - RRF_RT_ANY | 0x00020000 /* RRF_SUBKEY_WOW6432KEY */, - NULL, pvData, pcbData); + if (rthc_table != rthc_table_static) + osal_free(rthc_table); + rthc_count = 0; + rthc_table = rthc_table_static; + rthc_limit = RTHC_INITIAL_LIMIT; + rthc_pending.weak = 0; } -#endif +#endif /* ! Windows */ -static size_t hamming_weight(size_t v) { - const size_t m1 = (size_t)UINT64_C(0x5555555555555555); - const size_t m2 = (size_t)UINT64_C(0x3333333333333333); - const size_t m4 = (size_t)UINT64_C(0x0f0f0f0f0f0f0f0f); - const size_t h01 = (size_t)UINT64_C(0x0101010101010101); - v -= (v >> 1) & m1; - v = (v & m2) + ((v >> 2) & m2); - v = (v + (v >> 4)) & m4; - return (v * h01) >> (sizeof(v) * 8 - 8); +__cold void rthc_ctor(void) { +#if defined(_WIN32) || defined(_WIN64) + InitializeCriticalSection(&rthc_critical_section); +#else + ENSURE(nullptr, pthread_atfork(nullptr, nullptr, rthc_afterfork) == 0); + ENSURE(nullptr, pthread_key_create(&rthc_key, rthc_thread_dtor) == 0); + TRACE("pid %d, &mdbx_rthc_key = %p, value 0x%x", osal_getpid(), __Wpedantic_format_voidptr(&rthc_key), + (unsigned)rthc_key); +#endif } -static inline size_t hw64(uint64_t v) { - size_t r = hamming_weight((size_t)v); - if (sizeof(v) > sizeof(r)) - r += hamming_weight((size_t)(v >> sizeof(r) * 4 >> sizeof(r) * 4)); - return r; -} +__cold void rthc_dtor(const uint32_t current_pid) { + rthc_lock(); +#if !defined(_WIN32) && !defined(_WIN64) + uint64_t *rthc = pthread_getspecific(rthc_key); + TRACE("== thread 0x%" PRIxPTR ", rthc %p, pid %d, self-status 0x%08" PRIx64 ", left %d", osal_thread_self(), + __Wpedantic_format_voidptr(rthc), current_pid, rthc ? rthc_read(rthc) : ~UINT64_C(0), + atomic_load32(&rthc_pending, mo_Relaxed)); + if (rthc) { + const uint64_t sign_registered = MDBX_THREAD_RTHC_REGISTERED(rthc); + const uint64_t sign_counted = MDBX_THREAD_RTHC_COUNTED(rthc); + const uint64_t state = rthc_read(rthc); + if (state == sign_registered && rthc_compare_and_clean(rthc, sign_registered)) { + TRACE("== thread 0x%" PRIxPTR ", rthc %p, pid %d, self-status %s (0x%08" PRIx64 ")", osal_thread_self(), + __Wpedantic_format_voidptr(rthc), current_pid, "registered", state); + } else if (state == sign_counted && rthc_compare_and_clean(rthc, sign_counted)) { + TRACE("== thread 0x%" PRIxPTR ", rthc %p, pid %d, self-status %s (0x%08" PRIx64 ")", osal_thread_self(), + __Wpedantic_format_voidptr(rthc), current_pid, "counted", state); + ENSURE(nullptr, atomic_sub32(&rthc_pending, 1) > 0); + } else { + WARNING("thread 0x%" PRIxPTR ", rthc %p, pid %d, self-status %s (0x%08" PRIx64 ")", osal_thread_self(), + __Wpedantic_format_voidptr(rthc), current_pid, "wrong", state); + } + } -static bool check_uuid(bin128_t uuid) { - size_t hw = hw64(uuid.x) + hw64(uuid.y) + hw64(uuid.x ^ uuid.y); - return (hw >> 6) == 1; -} + struct timespec abstime; + ENSURE(nullptr, clock_gettime(CLOCK_REALTIME, &abstime) == 0); + abstime.tv_nsec += 1000000000l / 10; + if (abstime.tv_nsec >= 1000000000l) { + abstime.tv_nsec -= 1000000000l; + abstime.tv_sec += 1; + } +#if MDBX_DEBUG > 0 + abstime.tv_sec += 600; +#endif -__cold MDBX_MAYBE_UNUSED static bool -bootid_parse_uuid(bin128_t *s, const void *p, const size_t n) { - if (n > 31) { - unsigned bits = 0; - for (unsigned i = 0; i < n; ++i) /* try parse an UUID in text form */ { - uint8_t c = ((const uint8_t *)p)[i]; - if (c >= '0' && c <= '9') - c -= '0'; - else if (c >= 'a' && c <= 'f') - c -= 'a' - 10; - else if (c >= 'A' && c <= 'F') - c -= 'A' - 10; - else - continue; - assert(c <= 15); - c ^= s->y >> 60; - s->y = s->y << 4 | s->x >> 60; - s->x = s->x << 4 | c; - bits += 4; - } - if (bits > 42 * 3) - /* UUID parsed successfully */ - return true; + for (unsigned left; (left = atomic_load32(&rthc_pending, mo_AcquireRelease)) > 0;) { + NOTICE("tls-cleanup: pid %d, pending %u, wait for...", current_pid, left); + const int rc = pthread_cond_timedwait(&rthc_cond, &rthc_mutex, &abstime); + if (rc && rc != EINTR) + break; } + thread_key_delete(rthc_key); +#endif - if (n > 15) /* is enough handle it as a binary? */ { - if (n == sizeof(bin128_t)) { - bin128_t aligned; - memcpy(&aligned, p, sizeof(bin128_t)); - s->x += aligned.x; - s->y += aligned.y; - } else - bootid_collect(s, p, n); - return check_uuid(*s); + for (size_t i = 0; i < rthc_count; ++i) { + MDBX_env *const env = rthc_table[i].env; + if (env->pid != current_pid) + continue; + if (!(env->flags & ENV_TXKEY)) + continue; + env->flags -= ENV_TXKEY; + reader_slot_t *const begin = &env->lck_mmap.lck->rdt[0]; + reader_slot_t *const end = &env->lck_mmap.lck->rdt[env->max_readers]; + thread_key_delete(env->me_txkey); + bool cleaned = false; + for (reader_slot_t *reader = begin; reader < end; ++reader) { + TRACE("== [%zi] = key %" PRIuPTR ", %p ... %p, rthc %p (%+i), " + "rthc-pid %i, current-pid %i", + i, (uintptr_t)env->me_txkey, __Wpedantic_format_voidptr(begin), __Wpedantic_format_voidptr(end), + __Wpedantic_format_voidptr(reader), (int)(reader - begin), reader->pid.weak, current_pid); + if (atomic_load32(&reader->pid, mo_Relaxed) == current_pid) { + (void)atomic_cas32(&reader->pid, current_pid, 0); + TRACE("== cleanup %p", __Wpedantic_format_voidptr(reader)); + cleaned = true; + } + } + if (cleaned) + atomic_store32(&env->lck->rdt_refresh_flag, true, mo_Relaxed); } - if (n) - bootid_collect(s, p, n); - return false; + rthc_limit = rthc_count = 0; + if (rthc_table != rthc_table_static) + osal_free(rthc_table); + rthc_table = nullptr; + rthc_unlock(); + +#if defined(_WIN32) || defined(_WIN64) + DeleteCriticalSection(&rthc_critical_section); +#else + /* LY: yielding a few timeslices to give a more chance + * to racing destructor(s) for completion. */ + workaround_glibc_bug21031(); +#endif +} +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \note Please refer to the COPYRIGHT file for explanations license change, +/// credits and acknowledgments. +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 + +static MDBX_cursor *cursor_clone(const MDBX_cursor *csrc, cursor_couple_t *couple) { + cASSERT(csrc, csrc->txn->txnid >= csrc->txn->env->lck->cached_oldest.weak); + couple->outer.next = nullptr; + couple->outer.backup = nullptr; + couple->outer.subcur = nullptr; + couple->outer.clc = nullptr; + couple->outer.txn = csrc->txn; + couple->outer.dbi_state = csrc->dbi_state; + couple->outer.checking = z_pagecheck; + couple->outer.tree = nullptr; + couple->outer.top_and_flags = 0; + + MDBX_cursor *cdst = &couple->outer; + if (is_inner(csrc)) { + couple->inner.cursor.next = nullptr; + couple->inner.cursor.backup = nullptr; + couple->inner.cursor.subcur = nullptr; + couple->inner.cursor.txn = csrc->txn; + couple->inner.cursor.dbi_state = csrc->dbi_state; + couple->outer.subcur = &couple->inner; + cdst = &couple->inner.cursor; + } + + cdst->checking = csrc->checking; + cdst->tree = csrc->tree; + cdst->clc = csrc->clc; + cursor_cpstk(csrc, cdst); + return cdst; } -#if defined(__linux__) || defined(__gnu_linux__) +/*----------------------------------------------------------------------------*/ -__cold static bool is_inside_lxc(void) { - bool inside_lxc = false; - FILE *mounted = setmntent("/proc/mounts", "r"); - if (mounted) { - const struct mntent *ent; - while (nullptr != (ent = getmntent(mounted))) { - if (strcmp(ent->mnt_fsname, "lxcfs") == 0 && - strncmp(ent->mnt_dir, "/proc/", 6) == 0) { - inside_lxc = true; +void recalculate_merge_thresholds(MDBX_env *env) { + const size_t bytes = page_space(env); + env->merge_threshold = (uint16_t)(bytes - (bytes * env->options.merge_threshold_16dot16_percent >> 16)); + env->merge_threshold_gc = + (uint16_t)(bytes - ((env->options.merge_threshold_16dot16_percent > 19005) ? bytes / 3 /* 33 % */ + : bytes / 4 /* 25 % */)); +} + +int tree_drop(MDBX_cursor *mc, const bool may_have_tables) { + MDBX_txn *txn = mc->txn; + int rc = tree_search(mc, nullptr, Z_FIRST); + if (likely(rc == MDBX_SUCCESS)) { + /* DUPSORT sub-DBs have no large-pages/tables. Omit scanning leaves. + * This also avoids any P_DUPFIX pages, which have no nodes. + * Also if the DB doesn't have sub-DBs and has no large/overflow + * pages, omit scanning leaves. */ + if (!(may_have_tables | mc->tree->large_pages)) + cursor_pop(mc); + + rc = pnl_need(&txn->tw.retired_pages, + (size_t)mc->tree->branch_pages + (size_t)mc->tree->leaf_pages + (size_t)mc->tree->large_pages); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + + page_t *stack[CURSOR_STACK_SIZE]; + for (intptr_t i = 0; i <= mc->top; ++i) + stack[i] = mc->pg[i]; + + while (mc->top >= 0) { + page_t *const mp = mc->pg[mc->top]; + const size_t nkeys = page_numkeys(mp); + if (is_leaf(mp)) { + cASSERT(mc, mc->top + 1 == mc->tree->height); + for (size_t i = 0; i < nkeys; i++) { + node_t *node = page_node(mp, i); + if (node_flags(node) & N_BIG) { + rc = page_retire_ex(mc, node_largedata_pgno(node), nullptr, 0); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + if (!(may_have_tables | mc->tree->large_pages)) + goto pop; + } else if (node_flags(node) & N_TREE) { + if (unlikely((node_flags(node) & N_DUP) == 0)) { + rc = /* disallowing implicit table deletion */ MDBX_INCOMPATIBLE; + goto bailout; + } + rc = cursor_dupsort_setup(mc, node, mp); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + rc = tree_drop(&mc->subcur->cursor, false); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + } + } + } else { + cASSERT(mc, mc->top + 1 < mc->tree->height); + mc->checking |= z_retiring; + const unsigned pagetype = (is_frozen(txn, mp) ? P_FROZEN : 0) + + ((mc->top + 2 == mc->tree->height) ? (mc->checking & (P_LEAF | P_DUPFIX)) : P_BRANCH); + for (size_t i = 0; i < nkeys; i++) { + node_t *node = page_node(mp, i); + tASSERT(txn, (node_flags(node) & (N_BIG | N_TREE | N_DUP)) == 0); + const pgno_t pgno = node_pgno(node); + rc = page_retire_ex(mc, pgno, nullptr, pagetype); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + } + mc->checking -= z_retiring; + } + if (!mc->top) break; + cASSERT(mc, nkeys > 0); + mc->ki[mc->top] = (indx_t)nkeys; + rc = cursor_sibling_right(mc); + if (unlikely(rc != MDBX_SUCCESS)) { + if (unlikely(rc != MDBX_NOTFOUND)) + goto bailout; + /* no more siblings, go back to beginning + * of previous level. */ + pop: + cursor_pop(mc); + mc->ki[0] = 0; + for (intptr_t i = 1; i <= mc->top; i++) { + mc->pg[i] = stack[i]; + mc->ki[i] = 0; + } } } - endmntent(mounted); + rc = page_retire(mc, mc->pg[0]); } - return inside_lxc; -} -__cold static bool proc_read_uuid(const char *path, bin128_t *target) { - const int fd = open(path, O_RDONLY | O_NOFOLLOW); - if (fd != -1) { - struct statfs fs; - char buf[42]; - const ssize_t len = - (fstatfs(fd, &fs) == 0 && - (fs.f_type == /* procfs */ 0x9FA0 || - (fs.f_type == /* tmpfs */ 0x1021994 && is_inside_lxc()))) - ? read(fd, buf, sizeof(buf)) - : -1; - const int err = close(fd); - assert(err == 0); - (void)err; - if (len > 0) - return bootid_parse_uuid(target, buf, len); - } - return false; +bailout: + be_poor(mc); + if (unlikely(rc != MDBX_SUCCESS)) + txn->flags |= MDBX_TXN_ERROR; + return rc; } -#endif /* Linux */ - -__cold bin128_t osal_bootid(void) { - bin128_t uuid = {{0, 0}}; - bool got_machineid = false, got_boottime = false, got_bootseq = false; - -#if defined(__linux__) || defined(__gnu_linux__) - if (proc_read_uuid("/proc/sys/kernel/random/boot_id", &uuid)) - return uuid; -#endif /* Linux */ - -#if defined(__APPLE__) || defined(__MACH__) - { - char buf[42]; - size_t len = sizeof(buf); - if (!sysctlbyname("kern.bootsessionuuid", buf, &len, nullptr, 0) && - bootid_parse_uuid(&uuid, buf, len)) - return uuid; -#if defined(__MAC_OS_X_VERSION_MIN_REQUIRED) && \ - __MAC_OS_X_VERSION_MIN_REQUIRED > 1050 - uuid_t hostuuid; - struct timespec wait = {0, 1000000000u / 42}; - if (!gethostuuid(hostuuid, &wait)) - got_machineid = bootid_parse_uuid(&uuid, hostuuid, sizeof(hostuuid)); -#endif /* > 10.5 */ +static int node_move(MDBX_cursor *csrc, MDBX_cursor *cdst, bool fromleft) { + int rc; + DKBUF_DEBUG; - struct timeval boottime; - len = sizeof(boottime); - if (!sysctlbyname("kern.boottime", &boottime, &len, nullptr, 0) && - len == sizeof(boottime) && boottime.tv_sec) - got_boottime = true; + page_t *psrc = csrc->pg[csrc->top]; + page_t *pdst = cdst->pg[cdst->top]; + cASSERT(csrc, page_type(psrc) == page_type(pdst)); + cASSERT(csrc, csrc->tree == cdst->tree); + cASSERT(csrc, csrc->top == cdst->top); + if (unlikely(page_type(psrc) != page_type(pdst))) { + bailout: + ERROR("Wrong or mismatch pages's types (src %d, dst %d) to move node", page_type(psrc), page_type(pdst)); + csrc->txn->flags |= MDBX_TXN_ERROR; + return MDBX_PROBLEM; } -#endif /* Apple/Darwin */ - -#if defined(_WIN32) || defined(_WIN64) - { - union buf { - DWORD BootId; - DWORD BaseTime; - SYSTEM_TIMEOFDAY_INFORMATION SysTimeOfDayInfo; - struct { - LARGE_INTEGER BootTime; - LARGE_INTEGER CurrentTime; - LARGE_INTEGER TimeZoneBias; - ULONG TimeZoneId; - ULONG Reserved; - ULONGLONG BootTimeBias; - ULONGLONG SleepTimeBias; - } SysTimeOfDayInfoHacked; - wchar_t MachineGuid[42]; - char DigitalProductId[248]; - } buf; - - static const char HKLM_MicrosoftCryptography[] = - "SOFTWARE\\Microsoft\\Cryptography"; - DWORD len = sizeof(buf); - /* Windows is madness and must die */ - if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_MicrosoftCryptography, - "MachineGuid", &buf.MachineGuid, - &len) == ERROR_SUCCESS && - len < sizeof(buf)) - got_machineid = bootid_parse_uuid(&uuid, &buf.MachineGuid, len); - if (!got_machineid) { - /* again, Windows is madness */ - static const char HKLM_WindowsNT[] = - "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion"; - static const char HKLM_WindowsNT_DPK[] = - "SOFTWARE\\Microsoft\\Windows " - "NT\\CurrentVersion\\DefaultProductKey"; - static const char HKLM_WindowsNT_DPK2[] = - "SOFTWARE\\Microsoft\\Windows " - "NT\\CurrentVersion\\DefaultProductKey2"; + MDBX_val key4move; + switch (page_type(psrc)) { + case P_BRANCH: { + const node_t *srcnode = page_node(psrc, csrc->ki[csrc->top]); + cASSERT(csrc, node_flags(srcnode) == 0); + const pgno_t srcpg = node_pgno(srcnode); + key4move.iov_len = node_ks(srcnode); + key4move.iov_base = node_key(srcnode); - len = sizeof(buf); - if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_WindowsNT, - "DigitalProductId", &buf.DigitalProductId, - &len) == ERROR_SUCCESS && - len > 42 && len < sizeof(buf)) { - bootid_collect(&uuid, &buf.DigitalProductId, len); - got_machineid = true; - } - len = sizeof(buf); - if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_WindowsNT_DPK, - "DigitalProductId", &buf.DigitalProductId, - &len) == ERROR_SUCCESS && - len > 42 && len < sizeof(buf)) { - bootid_collect(&uuid, &buf.DigitalProductId, len); - got_machineid = true; - } - len = sizeof(buf); - if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_WindowsNT_DPK2, - "DigitalProductId", &buf.DigitalProductId, - &len) == ERROR_SUCCESS && - len > 42 && len < sizeof(buf)) { - bootid_collect(&uuid, &buf.DigitalProductId, len); - got_machineid = true; + if (csrc->ki[csrc->top] == 0) { + const int8_t top = csrc->top; + cASSERT(csrc, top >= 0); + /* must find the lowest key below src */ + rc = tree_search_lowest(csrc); + page_t *lowest_page = csrc->pg[csrc->top]; + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + cASSERT(csrc, is_leaf(lowest_page)); + if (unlikely(!is_leaf(lowest_page))) + goto bailout; + if (is_dupfix_leaf(lowest_page)) + key4move = page_dupfix_key(lowest_page, 0, csrc->tree->dupfix_size); + else { + const node_t *lowest_node = page_node(lowest_page, 0); + key4move.iov_len = node_ks(lowest_node); + key4move.iov_base = node_key(lowest_node); } - } - static const char HKLM_PrefetcherParams[] = - "SYSTEM\\CurrentControlSet\\Control\\Session Manager\\Memory " - "Management\\PrefetchParameters"; - len = sizeof(buf); - if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_PrefetcherParams, "BootId", - &buf.BootId, &len) == ERROR_SUCCESS && - len > 1 && len < sizeof(buf)) { - bootid_collect(&uuid, &buf.BootId, len); - got_bootseq = true; - } + /* restore cursor after mdbx_page_search_lowest() */ + csrc->top = top; + csrc->ki[csrc->top] = 0; - len = sizeof(buf); - if (mdbx_RegGetValue(HKEY_LOCAL_MACHINE, HKLM_PrefetcherParams, "BaseTime", - &buf.BaseTime, &len) == ERROR_SUCCESS && - len >= sizeof(buf.BaseTime) && buf.BaseTime) { - bootid_collect(&uuid, &buf.BaseTime, len); - got_boottime = true; + /* paranoia */ + cASSERT(csrc, psrc == csrc->pg[csrc->top]); + cASSERT(csrc, is_branch(psrc)); + if (unlikely(!is_branch(psrc))) + goto bailout; } - /* BootTime from SYSTEM_TIMEOFDAY_INFORMATION */ - NTSTATUS status = NtQuerySystemInformation( - 0x03 /* SystemTmeOfDayInformation */, &buf.SysTimeOfDayInfo, - sizeof(buf.SysTimeOfDayInfo), &len); - if (NT_SUCCESS(status) && - len >= offsetof(union buf, SysTimeOfDayInfoHacked.BootTimeBias) + - sizeof(buf.SysTimeOfDayInfoHacked.BootTimeBias) && - buf.SysTimeOfDayInfoHacked.BootTime.QuadPart) { - const uint64_t UnbiasedBootTime = - buf.SysTimeOfDayInfoHacked.BootTime.QuadPart - - buf.SysTimeOfDayInfoHacked.BootTimeBias; - if (UnbiasedBootTime) { - bootid_collect(&uuid, &UnbiasedBootTime, sizeof(UnbiasedBootTime)); - got_boottime = true; - } - } + if (cdst->ki[cdst->top] == 0) { + cursor_couple_t couple; + MDBX_cursor *const mn = cursor_clone(cdst, &couple); + const int8_t top = cdst->top; + cASSERT(csrc, top >= 0); - if (!got_boottime) { - uint64_t boottime = windows_bootime(); - if (boottime) { - bootid_collect(&uuid, &boottime, sizeof(boottime)); - got_boottime = true; + /* must find the lowest key below dst */ + rc = tree_search_lowest(mn); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + page_t *const lowest_page = mn->pg[mn->top]; + cASSERT(cdst, is_leaf(lowest_page)); + if (unlikely(!is_leaf(lowest_page))) + goto bailout; + MDBX_val key; + if (is_dupfix_leaf(lowest_page)) + key = page_dupfix_key(lowest_page, 0, mn->tree->dupfix_size); + else { + node_t *lowest_node = page_node(lowest_page, 0); + key.iov_len = node_ks(lowest_node); + key.iov_base = node_key(lowest_node); } - } - } -#endif /* Windows */ -#if defined(CTL_HW) && defined(HW_UUID) - if (!got_machineid) { - static const int mib[] = {CTL_HW, HW_UUID}; - char buf[42]; - size_t len = sizeof(buf); - if (sysctl( -#ifdef SYSCTL_LEGACY_NONCONST_MIB - (int *) -#endif - mib, - ARRAY_LENGTH(mib), &buf, &len, nullptr, 0) == 0) - got_machineid = bootid_parse_uuid(&uuid, buf, len); - } -#endif /* CTL_HW && HW_UUID */ + /* restore cursor after mdbx_page_search_lowest() */ + mn->top = top; + mn->ki[mn->top] = 0; -#if defined(CTL_KERN) && defined(KERN_HOSTUUID) - if (!got_machineid) { - static const int mib[] = {CTL_KERN, KERN_HOSTUUID}; - char buf[42]; - size_t len = sizeof(buf); - if (sysctl( -#ifdef SYSCTL_LEGACY_NONCONST_MIB - (int *) -#endif - mib, - ARRAY_LENGTH(mib), &buf, &len, nullptr, 0) == 0) - got_machineid = bootid_parse_uuid(&uuid, buf, len); - } -#endif /* CTL_KERN && KERN_HOSTUUID */ + const intptr_t delta = EVEN_CEIL(key.iov_len) - EVEN_CEIL(node_ks(page_node(mn->pg[mn->top], 0))); + const intptr_t needed = branch_size(cdst->txn->env, &key4move) + delta; + const intptr_t have = page_room(pdst); + if (unlikely(needed > have)) + return MDBX_RESULT_TRUE; -#if defined(__NetBSD__) - if (!got_machineid) { - char buf[42]; - size_t len = sizeof(buf); - if (sysctlbyname("machdep.dmi.system-uuid", buf, &len, nullptr, 0) == 0) - got_machineid = bootid_parse_uuid(&uuid, buf, len); - } -#endif /* __NetBSD__ */ + if (unlikely((rc = page_touch(csrc)) || (rc = page_touch(cdst)))) + return rc; + psrc = csrc->pg[csrc->top]; + pdst = cdst->pg[cdst->top]; -#if !(defined(_WIN32) || defined(_WIN64)) - if (!got_machineid) { - int fd = open("/etc/machine-id", O_RDONLY); - if (fd == -1) - fd = open("/var/lib/dbus/machine-id", O_RDONLY); - if (fd != -1) { - char buf[42]; - const ssize_t len = read(fd, buf, sizeof(buf)); - const int err = close(fd); - assert(err == 0); - (void)err; - if (len > 0) - got_machineid = bootid_parse_uuid(&uuid, buf, len); - } - } -#endif /* !Windows */ + couple.outer.next = mn->txn->cursors[cursor_dbi(mn)]; + mn->txn->cursors[cursor_dbi(mn)] = &couple.outer; + rc = tree_propagate_key(mn, &key); + mn->txn->cursors[cursor_dbi(mn)] = couple.outer.next; + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + } else { + const size_t needed = branch_size(cdst->txn->env, &key4move); + const size_t have = page_room(pdst); + if (unlikely(needed > have)) + return MDBX_RESULT_TRUE; -#if _XOPEN_SOURCE_EXTENDED - if (!got_machineid) { - const long hostid = gethostid(); - if (hostid != 0 && hostid != -1) { - bootid_collect(&uuid, &hostid, sizeof(hostid)); - got_machineid = true; + if (unlikely((rc = page_touch(csrc)) || (rc = page_touch(cdst)))) + return rc; + psrc = csrc->pg[csrc->top]; + pdst = cdst->pg[cdst->top]; } - } -#endif /* _XOPEN_SOURCE_EXTENDED */ - if (!got_machineid) { - lack: - uuid.x = uuid.y = 0; - return uuid; + DEBUG("moving %s-node %u [%s] on page %" PRIaPGNO " to node %u on page %" PRIaPGNO, "branch", csrc->ki[csrc->top], + DKEY_DEBUG(&key4move), psrc->pgno, cdst->ki[cdst->top], pdst->pgno); + /* Add the node to the destination page. */ + rc = node_add_branch(cdst, cdst->ki[cdst->top], &key4move, srcpg); + } break; + + case P_LEAF: { + /* Mark src and dst as dirty. */ + if (unlikely((rc = page_touch(csrc)) || (rc = page_touch(cdst)))) + return rc; + psrc = csrc->pg[csrc->top]; + pdst = cdst->pg[cdst->top]; + const node_t *srcnode = page_node(psrc, csrc->ki[csrc->top]); + MDBX_val data; + data.iov_len = node_ds(srcnode); + data.iov_base = node_data(srcnode); + key4move.iov_len = node_ks(srcnode); + key4move.iov_base = node_key(srcnode); + DEBUG("moving %s-node %u [%s] on page %" PRIaPGNO " to node %u on page %" PRIaPGNO, "leaf", csrc->ki[csrc->top], + DKEY_DEBUG(&key4move), psrc->pgno, cdst->ki[cdst->top], pdst->pgno); + /* Add the node to the destination page. */ + rc = node_add_leaf(cdst, cdst->ki[cdst->top], &key4move, &data, node_flags(srcnode)); + } break; + + case P_LEAF | P_DUPFIX: { + /* Mark src and dst as dirty. */ + if (unlikely((rc = page_touch(csrc)) || (rc = page_touch(cdst)))) + return rc; + psrc = csrc->pg[csrc->top]; + pdst = cdst->pg[cdst->top]; + key4move = page_dupfix_key(psrc, csrc->ki[csrc->top], csrc->tree->dupfix_size); + DEBUG("moving %s-node %u [%s] on page %" PRIaPGNO " to node %u on page %" PRIaPGNO, "leaf2", csrc->ki[csrc->top], + DKEY_DEBUG(&key4move), psrc->pgno, cdst->ki[cdst->top], pdst->pgno); + /* Add the node to the destination page. */ + rc = node_add_dupfix(cdst, cdst->ki[cdst->top], &key4move); + } break; + + default: + assert(false); + goto bailout; } - /*--------------------------------------------------------------------------*/ + if (unlikely(rc != MDBX_SUCCESS)) + return rc; -#if defined(CTL_KERN) && defined(KERN_BOOTTIME) - if (!got_boottime) { - static const int mib[] = {CTL_KERN, KERN_BOOTTIME}; - struct timeval boottime; - size_t len = sizeof(boottime); - if (sysctl( -#ifdef SYSCTL_LEGACY_NONCONST_MIB - (int *) -#endif - mib, - ARRAY_LENGTH(mib), &boottime, &len, nullptr, 0) == 0 && - len == sizeof(boottime) && boottime.tv_sec) { - bootid_collect(&uuid, &boottime, len); - got_boottime = true; - } - } -#endif /* CTL_KERN && KERN_BOOTTIME */ + /* Delete the node from the source page. */ + node_del(csrc, key4move.iov_len); -#if defined(__sun) || defined(__SVR4) || defined(__svr4__) - if (!got_boottime) { - kstat_ctl_t *kc = kstat_open(); - if (kc) { - kstat_t *kp = kstat_lookup(kc, "unix", 0, "system_misc"); - if (kp && kstat_read(kc, kp, 0) != -1) { - kstat_named_t *kn = (kstat_named_t *)kstat_data_lookup(kp, "boot_time"); - if (kn) { - switch (kn->data_type) { - case KSTAT_DATA_INT32: - case KSTAT_DATA_UINT32: - bootid_collect(&uuid, &kn->value, sizeof(int32_t)); - got_boottime = true; - case KSTAT_DATA_INT64: - case KSTAT_DATA_UINT64: - bootid_collect(&uuid, &kn->value, sizeof(int64_t)); - got_boottime = true; + cASSERT(csrc, psrc == csrc->pg[csrc->top]); + cASSERT(cdst, pdst == cdst->pg[cdst->top]); + cASSERT(csrc, page_type(psrc) == page_type(pdst)); + + /* csrc курсор тут всегда временный, на стеке внутри tree_rebalance(), + * и его нет необходимости корректировать. */ + { + /* Adjust other cursors pointing to mp */ + MDBX_cursor *m2, *m3; + const size_t dbi = cursor_dbi(csrc); + cASSERT(csrc, csrc->top == cdst->top); + if (fromleft) { + /* Перемещаем с левой страницы нв правую, нужно сдвинуть ki на +1 */ + for (m2 = csrc->txn->cursors[dbi]; m2; m2 = m2->next) { + m3 = (csrc->flags & z_inner) ? &m2->subcur->cursor : m2; + if (!is_related(csrc, m3)) + continue; + + if (m3 != cdst && m3->pg[csrc->top] == pdst && m3->ki[csrc->top] >= cdst->ki[csrc->top]) { + m3->ki[csrc->top] += 1; + } + + if (/* m3 != csrc && */ m3->pg[csrc->top] == psrc && m3->ki[csrc->top] == csrc->ki[csrc->top]) { + m3->pg[csrc->top] = pdst; + m3->ki[csrc->top] = cdst->ki[cdst->top]; + cASSERT(csrc, csrc->top > 0); + m3->ki[csrc->top - 1] += 1; + } + + if (is_leaf(psrc) && inner_pointed(m3)) { + cASSERT(csrc, csrc->top == m3->top); + size_t nkeys = page_numkeys(m3->pg[csrc->top]); + if (likely(nkeys > m3->ki[csrc->top])) + cursor_inner_refresh(m3, m3->pg[csrc->top], m3->ki[csrc->top]); + } + } + } else { + /* Перемещаем с правой страницы на левую, нужно сдвинуть ki на -1 */ + for (m2 = csrc->txn->cursors[dbi]; m2; m2 = m2->next) { + m3 = (csrc->flags & z_inner) ? &m2->subcur->cursor : m2; + if (!is_related(csrc, m3)) + continue; + if (m3->pg[csrc->top] == psrc) { + if (!m3->ki[csrc->top]) { + m3->pg[csrc->top] = pdst; + m3->ki[csrc->top] = cdst->ki[cdst->top]; + cASSERT(csrc, csrc->top > 0 && m3->ki[csrc->top - 1] > 0); + m3->ki[csrc->top - 1] -= 1; + } else + m3->ki[csrc->top] -= 1; + + if (is_leaf(psrc) && inner_pointed(m3)) { + cASSERT(csrc, csrc->top == m3->top); + size_t nkeys = page_numkeys(m3->pg[csrc->top]); + if (likely(nkeys > m3->ki[csrc->top])) + cursor_inner_refresh(m3, m3->pg[csrc->top], m3->ki[csrc->top]); } } } - kstat_close(kc); } } -#endif /* SunOS / Solaris */ -#if _XOPEN_SOURCE_EXTENDED && defined(BOOT_TIME) - if (!got_boottime) { - setutxent(); - const struct utmpx id = {.ut_type = BOOT_TIME}; - const struct utmpx *entry = getutxid(&id); - if (entry) { - bootid_collect(&uuid, entry, sizeof(*entry)); - got_boottime = true; - while (unlikely((entry = getutxid(&id)) != nullptr)) { - /* have multiple reboot records, assuming we can distinguish next - * bootsession even if RTC is wrong or absent */ - bootid_collect(&uuid, entry, sizeof(*entry)); - got_bootseq = true; + /* Update the parent separators. */ + if (csrc->ki[csrc->top] == 0) { + cASSERT(csrc, csrc->top > 0); + if (csrc->ki[csrc->top - 1] != 0) { + MDBX_val key; + if (is_dupfix_leaf(psrc)) + key = page_dupfix_key(psrc, 0, csrc->tree->dupfix_size); + else { + node_t *srcnode = page_node(psrc, 0); + key.iov_len = node_ks(srcnode); + key.iov_base = node_key(srcnode); } + DEBUG("update separator for source page %" PRIaPGNO " to [%s]", psrc->pgno, DKEY_DEBUG(&key)); + + cursor_couple_t couple; + MDBX_cursor *const mn = cursor_clone(csrc, &couple); + cASSERT(csrc, mn->top > 0); + mn->top -= 1; + + couple.outer.next = mn->txn->cursors[cursor_dbi(mn)]; + mn->txn->cursors[cursor_dbi(mn)] = &couple.outer; + rc = tree_propagate_key(mn, &key); + mn->txn->cursors[cursor_dbi(mn)] = couple.outer.next; + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + } + if (is_branch(psrc)) { + const MDBX_val nullkey = {0, 0}; + const indx_t ix = csrc->ki[csrc->top]; + csrc->ki[csrc->top] = 0; + rc = tree_propagate_key(csrc, &nullkey); + csrc->ki[csrc->top] = ix; + cASSERT(csrc, rc == MDBX_SUCCESS); } - endutxent(); } -#endif /* _XOPEN_SOURCE_EXTENDED && BOOT_TIME */ - if (!got_bootseq) { - if (!got_boottime || !MDBX_TRUST_RTC) - goto lack; + if (cdst->ki[cdst->top] == 0) { + cASSERT(cdst, cdst->top > 0); + if (cdst->ki[cdst->top - 1] != 0) { + MDBX_val key; + if (is_dupfix_leaf(pdst)) + key = page_dupfix_key(pdst, 0, cdst->tree->dupfix_size); + else { + node_t *srcnode = page_node(pdst, 0); + key.iov_len = node_ks(srcnode); + key.iov_base = node_key(srcnode); + } + DEBUG("update separator for destination page %" PRIaPGNO " to [%s]", pdst->pgno, DKEY_DEBUG(&key)); + cursor_couple_t couple; + MDBX_cursor *const mn = cursor_clone(cdst, &couple); + cASSERT(cdst, mn->top > 0); + mn->top -= 1; -#if defined(_WIN32) || defined(_WIN64) - FILETIME now; - GetSystemTimeAsFileTime(&now); - if (0x1CCCCCC > now.dwHighDateTime) -#else - struct timespec mono, real; - if (clock_gettime(CLOCK_MONOTONIC, &mono) || - clock_gettime(CLOCK_REALTIME, &real) || - /* wrong time, RTC is mad or absent */ - 1555555555l > real.tv_sec || - /* seems no adjustment by RTC/NTP, i.e. a fake time */ - real.tv_sec < mono.tv_sec || 1234567890l > real.tv_sec - mono.tv_sec || - (real.tv_sec - mono.tv_sec) % 900u == 0) -#endif - goto lack; + couple.outer.next = mn->txn->cursors[cursor_dbi(mn)]; + mn->txn->cursors[cursor_dbi(mn)] = &couple.outer; + rc = tree_propagate_key(mn, &key); + mn->txn->cursors[cursor_dbi(mn)] = couple.outer.next; + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + } + if (is_branch(pdst)) { + const MDBX_val nullkey = {0, 0}; + const indx_t ix = cdst->ki[cdst->top]; + cdst->ki[cdst->top] = 0; + rc = tree_propagate_key(cdst, &nullkey); + cdst->ki[cdst->top] = ix; + cASSERT(cdst, rc == MDBX_SUCCESS); + } } - return uuid; + return MDBX_SUCCESS; } -__cold int mdbx_get_sysraminfo(intptr_t *page_size, intptr_t *total_pages, - intptr_t *avail_pages) { - if (!page_size && !total_pages && !avail_pages) - return MDBX_EINVAL; - if (total_pages) - *total_pages = -1; - if (avail_pages) - *avail_pages = -1; +static int page_merge(MDBX_cursor *csrc, MDBX_cursor *cdst) { + MDBX_val key; + int rc; - const intptr_t pagesize = osal_syspagesize(); - if (page_size) - *page_size = pagesize; - if (unlikely(pagesize < MIN_PAGESIZE || !is_powerof2(pagesize))) - return MDBX_INCOMPATIBLE; + cASSERT(csrc, csrc != cdst); + cASSERT(csrc, cursor_is_tracked(csrc)); + cASSERT(cdst, cursor_is_tracked(cdst)); + const page_t *const psrc = csrc->pg[csrc->top]; + page_t *pdst = cdst->pg[cdst->top]; + DEBUG("merging page %" PRIaPGNO " into %" PRIaPGNO, psrc->pgno, pdst->pgno); + + cASSERT(csrc, page_type(psrc) == page_type(pdst)); + cASSERT(csrc, csrc->clc == cdst->clc && csrc->tree == cdst->tree); + cASSERT(csrc, csrc->top > 0); /* can't merge root page */ + cASSERT(cdst, cdst->top > 0); + cASSERT(cdst, cdst->top + 1 < cdst->tree->height || is_leaf(cdst->pg[cdst->tree->height - 1])); + cASSERT(csrc, csrc->top + 1 < csrc->tree->height || is_leaf(csrc->pg[csrc->tree->height - 1])); + cASSERT(cdst, + csrc->txn->env->options.prefer_waf_insteadof_balance || page_room(pdst) >= page_used(cdst->txn->env, psrc)); + const int pagetype = page_type(psrc); - MDBX_MAYBE_UNUSED const int log2page = log2n_powerof2(pagesize); - assert(pagesize == (INT64_C(1) << log2page)); - (void)log2page; + /* Move all nodes from src to dst */ + const size_t dst_nkeys = page_numkeys(pdst); + const size_t src_nkeys = page_numkeys(psrc); + cASSERT(cdst, dst_nkeys + src_nkeys >= (is_leaf(psrc) ? 1u : 2u)); + if (likely(src_nkeys)) { + size_t ii = dst_nkeys; + if (unlikely(pagetype & P_DUPFIX)) { + /* Mark dst as dirty. */ + rc = page_touch(cdst); + cASSERT(cdst, rc != MDBX_RESULT_TRUE); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; -#if defined(_WIN32) || defined(_WIN64) - MEMORYSTATUSEX info; - memset(&info, 0, sizeof(info)); - info.dwLength = sizeof(info); - if (!GlobalMemoryStatusEx(&info)) - return (int)GetLastError(); -#endif + key.iov_len = csrc->tree->dupfix_size; + key.iov_base = page_data(psrc); + size_t i = 0; + do { + rc = node_add_dupfix(cdst, ii++, &key); + cASSERT(cdst, rc != MDBX_RESULT_TRUE); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + key.iov_base = ptr_disp(key.iov_base, key.iov_len); + } while (++i != src_nkeys); + } else { + node_t *srcnode = page_node(psrc, 0); + key.iov_len = node_ks(srcnode); + key.iov_base = node_key(srcnode); + if (pagetype & P_BRANCH) { + cursor_couple_t couple; + MDBX_cursor *const mn = cursor_clone(csrc, &couple); - if (total_pages) { -#if defined(_WIN32) || defined(_WIN64) - const intptr_t total_ram_pages = (intptr_t)(info.ullTotalPhys >> log2page); -#elif defined(_SC_PHYS_PAGES) - const intptr_t total_ram_pages = sysconf(_SC_PHYS_PAGES); - if (total_ram_pages == -1) - return errno; -#elif defined(_SC_AIX_REALMEM) - const intptr_t total_ram_Kb = sysconf(_SC_AIX_REALMEM); - if (total_ram_Kb == -1) - return errno; - const intptr_t total_ram_pages = (total_ram_Kb << 10) >> log2page; -#elif defined(HW_USERMEM) || defined(HW_PHYSMEM64) || defined(HW_MEMSIZE) || \ - defined(HW_PHYSMEM) - size_t ram, len = sizeof(ram); - static const int mib[] = {CTL_HW, -#if defined(HW_USERMEM) - HW_USERMEM -#elif defined(HW_PHYSMEM64) - HW_PHYSMEM64 -#elif defined(HW_MEMSIZE) - HW_MEMSIZE -#else - HW_PHYSMEM -#endif - }; - if (sysctl( -#ifdef SYSCTL_LEGACY_NONCONST_MIB - (int *) -#endif - mib, - ARRAY_LENGTH(mib), &ram, &len, NULL, 0) != 0) - return errno; - if (len != sizeof(ram)) - return MDBX_ENOSYS; - const intptr_t total_ram_pages = (intptr_t)(ram >> log2page); -#else -#error "FIXME: Get User-accessible or physical RAM" -#endif - *total_pages = total_ram_pages; - if (total_ram_pages < 1) - return MDBX_ENOSYS; - } + /* must find the lowest key below src */ + rc = tree_search_lowest(mn); + cASSERT(csrc, rc != MDBX_RESULT_TRUE); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; - if (avail_pages) { -#if defined(_WIN32) || defined(_WIN64) - const intptr_t avail_ram_pages = (intptr_t)(info.ullAvailPhys >> log2page); -#elif defined(_SC_AVPHYS_PAGES) - const intptr_t avail_ram_pages = sysconf(_SC_AVPHYS_PAGES); - if (avail_ram_pages == -1) - return errno; -#elif defined(__MACH__) - mach_msg_type_number_t count = HOST_VM_INFO_COUNT; - vm_statistics_data_t vmstat; - mach_port_t mport = mach_host_self(); - kern_return_t kerr = host_statistics(mach_host_self(), HOST_VM_INFO, - (host_info_t)&vmstat, &count); - mach_port_deallocate(mach_task_self(), mport); - if (unlikely(kerr != KERN_SUCCESS)) - return MDBX_ENOSYS; - const intptr_t avail_ram_pages = vmstat.free_count; -#elif defined(VM_TOTAL) || defined(VM_METER) - struct vmtotal info; - size_t len = sizeof(info); - static const int mib[] = {CTL_VM, -#if defined(VM_TOTAL) - VM_TOTAL -#elif defined(VM_METER) - VM_METER -#endif - }; - if (sysctl( -#ifdef SYSCTL_LEGACY_NONCONST_MIB - (int *) -#endif - mib, - ARRAY_LENGTH(mib), &info, &len, NULL, 0) != 0) - return errno; - if (len != sizeof(info)) - return MDBX_ENOSYS; - const intptr_t avail_ram_pages = info.t_free; -#else -#error "FIXME: Get Available RAM" -#endif - *avail_pages = avail_ram_pages; - if (avail_ram_pages < 1) - return MDBX_ENOSYS; - } + const page_t *mp = mn->pg[mn->top]; + if (likely(!is_dupfix_leaf(mp))) { + cASSERT(mn, is_leaf(mp)); + const node_t *lowest = page_node(mp, 0); + key.iov_len = node_ks(lowest); + key.iov_base = node_key(lowest); + } else { + cASSERT(mn, mn->top > csrc->top); + key = page_dupfix_key(mp, mn->ki[mn->top], csrc->tree->dupfix_size); + } + cASSERT(mn, key.iov_len >= csrc->clc->k.lmin); + cASSERT(mn, key.iov_len <= csrc->clc->k.lmax); + + const size_t dst_room = page_room(pdst); + const size_t src_used = page_used(cdst->txn->env, psrc); + const size_t space_needed = src_used - node_ks(srcnode) + key.iov_len; + if (unlikely(space_needed > dst_room)) + return MDBX_RESULT_TRUE; + } - return MDBX_SUCCESS; -} + /* Mark dst as dirty. */ + rc = page_touch(cdst); + cASSERT(cdst, rc != MDBX_RESULT_TRUE); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; -#ifndef xMDBX_ALLOY -unsigned sys_pagesize; -MDBX_MAYBE_UNUSED unsigned sys_pagesize_ln2, sys_allocation_granularity; -#endif /* xMDBX_ALLOY */ + size_t i = 0; + while (true) { + if (pagetype & P_LEAF) { + MDBX_val data; + data.iov_len = node_ds(srcnode); + data.iov_base = node_data(srcnode); + rc = node_add_leaf(cdst, ii++, &key, &data, node_flags(srcnode)); + } else { + cASSERT(csrc, node_flags(srcnode) == 0); + rc = node_add_branch(cdst, ii++, &key, node_pgno(srcnode)); + } + cASSERT(cdst, rc != MDBX_RESULT_TRUE); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; -void osal_ctor(void) { -#if MDBX_HAVE_PWRITEV && defined(_SC_IOV_MAX) - osal_iov_max = sysconf(_SC_IOV_MAX); - if (RUNNING_ON_VALGRIND && osal_iov_max > 64) - /* чтобы не описывать все 1024 исключения в valgrind_suppress.txt */ - osal_iov_max = 64; -#endif /* MDBX_HAVE_PWRITEV && _SC_IOV_MAX */ + if (++i == src_nkeys) + break; + srcnode = page_node(psrc, i); + key.iov_len = node_ks(srcnode); + key.iov_base = node_key(srcnode); + } + } -#if defined(_WIN32) || defined(_WIN64) - SYSTEM_INFO si; - GetSystemInfo(&si); - sys_pagesize = si.dwPageSize; - sys_allocation_granularity = si.dwAllocationGranularity; -#else - sys_pagesize = sysconf(_SC_PAGE_SIZE); - sys_allocation_granularity = (MDBX_WORDBITS > 32) ? 65536 : 4096; - sys_allocation_granularity = (sys_allocation_granularity > sys_pagesize) - ? sys_allocation_granularity - : sys_pagesize; -#endif - assert(sys_pagesize > 0 && (sys_pagesize & (sys_pagesize - 1)) == 0); - assert(sys_allocation_granularity >= sys_pagesize && - sys_allocation_granularity % sys_pagesize == 0); - sys_pagesize_ln2 = log2n_powerof2(sys_pagesize); + pdst = cdst->pg[cdst->top]; + DEBUG("dst page %" PRIaPGNO " now has %zu keys (%u.%u%% filled)", pdst->pgno, page_numkeys(pdst), + page_fill_percentum_x10(cdst->txn->env, pdst) / 10, page_fill_percentum_x10(cdst->txn->env, pdst) % 10); -#if defined(__linux__) || defined(__gnu_linux__) - posix_clockid = choice_monoclock(); -#endif + cASSERT(csrc, psrc == csrc->pg[csrc->top]); + cASSERT(cdst, pdst == cdst->pg[cdst->top]); + } -#if defined(_WIN32) || defined(_WIN64) - QueryPerformanceFrequency(&performance_frequency); -#elif defined(__APPLE__) || defined(__MACH__) - mach_timebase_info_data_t ti; - mach_timebase_info(&ti); - ratio_16dot16_to_monotine = UINT64_C(1000000000) * ti.denom / ti.numer; -#endif - monotime_limit = osal_16dot16_to_monotime(UINT32_MAX - 1); -} + /* Unlink the src page from parent and add to free list. */ + csrc->top -= 1; + node_del(csrc, 0); + if (csrc->ki[csrc->top] == 0) { + const MDBX_val nullkey = {0, 0}; + rc = tree_propagate_key(csrc, &nullkey); + cASSERT(csrc, rc != MDBX_RESULT_TRUE); + if (unlikely(rc != MDBX_SUCCESS)) { + csrc->top += 1; + return rc; + } + } + csrc->top += 1; -void osal_dtor(void) {} -/* This is CMake-template for libmdbx's version.c - ******************************************************************************/ + cASSERT(csrc, psrc == csrc->pg[csrc->top]); + cASSERT(cdst, pdst == cdst->pg[cdst->top]); + { + /* Adjust other cursors pointing to mp */ + MDBX_cursor *m2, *m3; + const size_t dbi = cursor_dbi(csrc); + for (m2 = csrc->txn->cursors[dbi]; m2; m2 = m2->next) { + m3 = (csrc->flags & z_inner) ? &m2->subcur->cursor : m2; + if (!is_related(csrc, m3)) + continue; + if (m3->pg[csrc->top] == psrc) { + m3->pg[csrc->top] = pdst; + m3->ki[csrc->top] += (indx_t)dst_nkeys; + m3->ki[csrc->top - 1] = cdst->ki[csrc->top - 1]; + } else if (m3->pg[csrc->top - 1] == csrc->pg[csrc->top - 1] && m3->ki[csrc->top - 1] > csrc->ki[csrc->top - 1]) { + cASSERT(m3, m3->ki[csrc->top - 1] > 0 && m3->ki[csrc->top - 1] <= page_numkeys(m3->pg[csrc->top - 1])); + m3->ki[csrc->top - 1] -= 1; + } -#if MDBX_VERSION_MAJOR != 0 || \ - MDBX_VERSION_MINOR != 12 -#error "API version mismatch! Had `git fetch --tags` done?" -#endif + if (is_leaf(psrc) && inner_pointed(m3)) { + cASSERT(csrc, csrc->top == m3->top); + size_t nkeys = page_numkeys(m3->pg[csrc->top]); + if (likely(nkeys > m3->ki[csrc->top])) + cursor_inner_refresh(m3, m3->pg[csrc->top], m3->ki[csrc->top]); + } + } + } -static const char sourcery[] = MDBX_STRINGIFY(MDBX_BUILD_SOURCERY); + rc = page_retire(csrc, (page_t *)psrc); + cASSERT(csrc, rc != MDBX_RESULT_TRUE); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; -__dll_export -#ifdef __attribute_used__ - __attribute_used__ -#elif defined(__GNUC__) || __has_attribute(__used__) - __attribute__((__used__)) -#endif -#ifdef __attribute_externally_visible__ - __attribute_externally_visible__ -#elif (defined(__GNUC__) && !defined(__clang__)) || \ - __has_attribute(__externally_visible__) - __attribute__((__externally_visible__)) -#endif - const struct MDBX_version_info mdbx_version = { - 0, - 12, - 13, - 0, - {"2025-02-28T23:34:52+03:00", "240ba36a2661369aae042378919f1a185aac9705", "1fff1f67d5862b4f5c129b0d735913ac3ee1aaec", - "v0.12.13-0-g1fff1f67"}, - sourcery}; + cASSERT(cdst, cdst->tree->items > 0); + cASSERT(cdst, cdst->top + 1 <= cdst->tree->height); + cASSERT(cdst, cdst->top > 0); + page_t *const top_page = cdst->pg[cdst->top]; + const indx_t top_indx = cdst->ki[cdst->top]; + const int save_top = cdst->top; + const uint16_t save_height = cdst->tree->height; + cursor_pop(cdst); + rc = tree_rebalance(cdst); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; -__dll_export -#ifdef __attribute_used__ - __attribute_used__ -#elif defined(__GNUC__) || __has_attribute(__used__) - __attribute__((__used__)) -#endif -#ifdef __attribute_externally_visible__ - __attribute_externally_visible__ -#elif (defined(__GNUC__) && !defined(__clang__)) || \ - __has_attribute(__externally_visible__) - __attribute__((__externally_visible__)) -#endif - const char *const mdbx_sourcery_anchor = sourcery; -/* - * Copyright 2015-2024 Leonid Yuriev - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - */ + cASSERT(cdst, cdst->tree->items > 0); + cASSERT(cdst, cdst->top + 1 <= cdst->tree->height); -#if defined(_WIN32) || defined(_WIN64) /* Windows LCK-implementation */ +#if MDBX_ENABLE_PGOP_STAT + cdst->txn->env->lck->pgops.merge.weak += 1; +#endif /* MDBX_ENABLE_PGOP_STAT */ -/* PREAMBLE FOR WINDOWS: - * - * We are not concerned for performance here. - * If you are running Windows a performance could NOT be the goal. - * Otherwise please use Linux. */ + if (is_leaf(cdst->pg[cdst->top])) { + /* LY: don't touch cursor if top-page is a LEAF */ + cASSERT(cdst, is_leaf(cdst->pg[cdst->top]) || page_type(cdst->pg[cdst->top]) == pagetype); + return MDBX_SUCCESS; + } + cASSERT(cdst, page_numkeys(top_page) == dst_nkeys + src_nkeys); -static void mdbx_winnt_import(void); + if (unlikely(pagetype != page_type(top_page))) { + /* LY: LEAF-page becomes BRANCH, unable restore cursor's stack */ + goto bailout; + } -#if MDBX_BUILD_SHARED_LIBRARY -#if MDBX_WITHOUT_MSVC_CRT && defined(NDEBUG) -/* DEBUG/CHECKED builds still require MSVC's CRT for runtime checks. - * - * Define dll's entry point only for Release build when NDEBUG is defined and - * MDBX_WITHOUT_MSVC_CRT=ON. if the entry point isn't defined then MSVC's will - * automatically use DllMainCRTStartup() from CRT library, which also - * automatically call DllMain() from our mdbx.dll */ -#pragma comment(linker, "/ENTRY:DllMain") -#endif /* MDBX_WITHOUT_MSVC_CRT */ + if (top_page == cdst->pg[cdst->top]) { + /* LY: don't touch cursor if prev top-page already on the top */ + cASSERT(cdst, cdst->ki[cdst->top] == top_indx); + cASSERT(cdst, is_leaf(cdst->pg[cdst->top]) || page_type(cdst->pg[cdst->top]) == pagetype); + return MDBX_SUCCESS; + } -BOOL APIENTRY DllMain(HANDLE module, DWORD reason, LPVOID reserved) -#else -#if !MDBX_MANUAL_MODULE_HANDLER -static -#endif /* !MDBX_MANUAL_MODULE_HANDLER */ - void NTAPI - mdbx_module_handler(PVOID module, DWORD reason, PVOID reserved) -#endif /* MDBX_BUILD_SHARED_LIBRARY */ -{ - (void)reserved; - switch (reason) { - case DLL_PROCESS_ATTACH: - mdbx_winnt_import(); - global_ctor(); - break; - case DLL_PROCESS_DETACH: - global_dtor(); - break; + const int new_top = save_top - save_height + cdst->tree->height; + if (unlikely(new_top < 0 || new_top >= cdst->tree->height)) { + /* LY: out of range, unable restore cursor's stack */ + goto bailout; + } - case DLL_THREAD_ATTACH: - break; - case DLL_THREAD_DETACH: - thread_dtor(module); - break; + if (top_page == cdst->pg[new_top]) { + cASSERT(cdst, cdst->ki[new_top] == top_indx); + /* LY: restore cursor stack */ + cdst->top = (int8_t)new_top; + cASSERT(cdst, cdst->top + 1 < cdst->tree->height || is_leaf(cdst->pg[cdst->tree->height - 1])); + cASSERT(cdst, is_leaf(cdst->pg[cdst->top]) || page_type(cdst->pg[cdst->top]) == pagetype); + return MDBX_SUCCESS; } -#if MDBX_BUILD_SHARED_LIBRARY - return TRUE; + + page_t *const stub_page = (page_t *)(~(uintptr_t)top_page); + const indx_t stub_indx = top_indx; + if (save_height > cdst->tree->height && ((cdst->pg[save_top] == top_page && cdst->ki[save_top] == top_indx) || + (cdst->pg[save_top] == stub_page && cdst->ki[save_top] == stub_indx))) { + /* LY: restore cursor stack */ + cdst->pg[new_top] = top_page; + cdst->ki[new_top] = top_indx; +#if MDBX_DEBUG + cdst->pg[new_top + 1] = nullptr; + cdst->ki[new_top + 1] = INT16_MAX; #endif + cdst->top = (int8_t)new_top; + cASSERT(cdst, cdst->top + 1 < cdst->tree->height || is_leaf(cdst->pg[cdst->tree->height - 1])); + cASSERT(cdst, is_leaf(cdst->pg[cdst->top]) || page_type(cdst->pg[cdst->top]) == pagetype); + return MDBX_SUCCESS; + } + +bailout: + /* LY: unable restore cursor's stack */ + be_poor(cdst); + return MDBX_CURSOR_FULL; } -#if !MDBX_BUILD_SHARED_LIBRARY && !MDBX_MANUAL_MODULE_HANDLER -#if defined(_MSC_VER) -# pragma const_seg(push) -# pragma data_seg(push) +int tree_rebalance(MDBX_cursor *mc) { + cASSERT(mc, cursor_is_tracked(mc)); + cASSERT(mc, mc->top >= 0); + cASSERT(mc, mc->top + 1 < mc->tree->height || is_leaf(mc->pg[mc->tree->height - 1])); + const page_t *const tp = mc->pg[mc->top]; + const uint8_t pagetype = page_type(tp); -# ifndef _M_IX86 - /* kick a linker to create the TLS directory if not already done */ -# pragma comment(linker, "/INCLUDE:_tls_used") - /* Force some symbol references. */ -# pragma comment(linker, "/INCLUDE:mdbx_tls_anchor") - /* specific const-segment for WIN64 */ -# pragma const_seg(".CRT$XLB") - const -# else - /* kick a linker to create the TLS directory if not already done */ -# pragma comment(linker, "/INCLUDE:__tls_used") - /* Force some symbol references. */ -# pragma comment(linker, "/INCLUDE:_mdbx_tls_anchor") - /* specific data-segment for WIN32 */ -# pragma data_seg(".CRT$XLB") -# endif + STATIC_ASSERT(P_BRANCH == 1); + const size_t minkeys = (pagetype & P_BRANCH) + (size_t)1; - __declspec(allocate(".CRT$XLB")) PIMAGE_TLS_CALLBACK mdbx_tls_anchor = mdbx_module_handler; -# pragma data_seg(pop) -# pragma const_seg(pop) + /* Pages emptier than this are candidates for merging. */ + size_t room_threshold = + likely(mc->tree != &mc->txn->dbs[FREE_DBI]) ? mc->txn->env->merge_threshold : mc->txn->env->merge_threshold_gc; -#elif defined(__GNUC__) -# ifndef _M_IX86 - const -# endif - PIMAGE_TLS_CALLBACK mdbx_tls_anchor __attribute__((__section__(".CRT$XLB"), used)) = mdbx_module_handler; -#else -# error FIXME -#endif -#endif /* !MDBX_BUILD_SHARED_LIBRARY && !MDBX_MANUAL_MODULE_HANDLER */ + const size_t numkeys = page_numkeys(tp); + const size_t room = page_room(tp); + DEBUG("rebalancing %s page %" PRIaPGNO " (has %zu keys, fill %u.%u%%, used %zu, room %zu bytes)", + is_leaf(tp) ? "leaf" : "branch", tp->pgno, numkeys, page_fill_percentum_x10(mc->txn->env, tp) / 10, + page_fill_percentum_x10(mc->txn->env, tp) % 10, page_used(mc->txn->env, tp), room); + cASSERT(mc, is_modifable(mc->txn, tp)); -/*----------------------------------------------------------------------------*/ + if (unlikely(numkeys < minkeys)) { + DEBUG("page %" PRIaPGNO " must be merged due keys < %zu threshold", tp->pgno, minkeys); + } else if (unlikely(room > room_threshold)) { + DEBUG("page %" PRIaPGNO " should be merged due room %zu > %zu threshold", tp->pgno, room, room_threshold); + } else { + DEBUG("no need to rebalance page %" PRIaPGNO ", room %zu < %zu threshold", tp->pgno, room, room_threshold); + cASSERT(mc, mc->tree->items > 0); + return MDBX_SUCCESS; + } -#define LCK_SHARED 0 -#define LCK_EXCLUSIVE LOCKFILE_EXCLUSIVE_LOCK -#define LCK_WAITFOR 0 -#define LCK_DONTWAIT LOCKFILE_FAIL_IMMEDIATELY + int rc; + if (mc->top == 0) { + page_t *const mp = mc->pg[0]; + const size_t nkeys = page_numkeys(mp); + cASSERT(mc, (mc->tree->items == 0) == (nkeys == 0)); + if (nkeys == 0) { + DEBUG("%s", "tree is completely empty"); + cASSERT(mc, is_leaf(mp)); + cASSERT(mc, (*cursor_dbi_state(mc) & DBI_DIRTY) != 0); + cASSERT(mc, mc->tree->branch_pages == 0 && mc->tree->large_pages == 0 && mc->tree->leaf_pages == 1); + /* Adjust cursors pointing to mp */ + for (MDBX_cursor *m2 = mc->txn->cursors[cursor_dbi(mc)]; m2; m2 = m2->next) { + MDBX_cursor *m3 = (mc->flags & z_inner) ? &m2->subcur->cursor : m2; + if (!is_poor(m3) && m3->pg[0] == mp) { + be_poor(m3); + m3->flags |= z_after_delete; + } + } + if (is_subpage(mp)) { + return MDBX_SUCCESS; + } else { + mc->tree->root = P_INVALID; + mc->tree->height = 0; + return page_retire(mc, mp); + } + } + if (is_subpage(mp)) { + DEBUG("%s", "Can't rebalance a subpage, ignoring"); + cASSERT(mc, is_leaf(tp)); + return MDBX_SUCCESS; + } + if (is_branch(mp) && nkeys == 1) { + DEBUG("%s", "collapsing root page!"); + mc->tree->root = node_pgno(page_node(mp, 0)); + rc = page_get(mc, mc->tree->root, &mc->pg[0], mp->txnid); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + mc->tree->height--; + mc->ki[0] = mc->ki[1]; + for (intptr_t i = 1; i < mc->tree->height; i++) { + mc->pg[i] = mc->pg[i + 1]; + mc->ki[i] = mc->ki[i + 1]; + } -static int flock_with_event(HANDLE fd, HANDLE event, unsigned flags, - size_t offset, size_t bytes) { - TRACE("lock>>: fd %p, event %p, flags 0x%x offset %zu, bytes %zu >>", fd, - event, flags, offset, bytes); - OVERLAPPED ov; - ov.Internal = 0; - ov.InternalHigh = 0; - ov.hEvent = event; - ov.Offset = (DWORD)offset; - ov.OffsetHigh = HIGH_DWORD(offset); - if (LockFileEx(fd, flags, 0, (DWORD)bytes, HIGH_DWORD(bytes), &ov)) { - TRACE("lock<<: fd %p, event %p, flags 0x%x offset %zu, bytes %zu << %s", fd, - event, flags, offset, bytes, "done"); + /* Adjust other cursors pointing to mp */ + for (MDBX_cursor *m2 = mc->txn->cursors[cursor_dbi(mc)]; m2; m2 = m2->next) { + MDBX_cursor *m3 = (mc->flags & z_inner) ? &m2->subcur->cursor : m2; + if (is_related(mc, m3) && m3->pg[0] == mp) { + for (intptr_t i = 0; i < mc->tree->height; i++) { + m3->pg[i] = m3->pg[i + 1]; + m3->ki[i] = m3->ki[i + 1]; + } + m3->top -= 1; + } + } + cASSERT(mc, is_leaf(mc->pg[mc->top]) || page_type(mc->pg[mc->top]) == pagetype); + cASSERT(mc, mc->top + 1 < mc->tree->height || is_leaf(mc->pg[mc->tree->height - 1])); + return page_retire(mc, mp); + } + DEBUG("root page %" PRIaPGNO " doesn't need rebalancing (flags 0x%x)", mp->pgno, mp->flags); return MDBX_SUCCESS; } - DWORD rc = GetLastError(); - if (rc == ERROR_IO_PENDING) { - if (event) { - if (GetOverlappedResult(fd, &ov, &rc, true)) { - TRACE("lock<<: fd %p, event %p, flags 0x%x offset %zu, bytes %zu << %s", - fd, event, flags, offset, bytes, "overlapped-done"); - return MDBX_SUCCESS; - } - rc = GetLastError(); - } else - CancelIo(fd); - } - TRACE("lock<<: fd %p, event %p, flags 0x%x offset %zu, bytes %zu << err %d", - fd, event, flags, offset, bytes, (int)rc); - return (int)rc; -} + /* The parent (branch page) must have at least 2 pointers, + * otherwise the tree is invalid. */ + const size_t pre_top = mc->top - 1; + cASSERT(mc, is_branch(mc->pg[pre_top])); + cASSERT(mc, !is_subpage(mc->pg[0])); + cASSERT(mc, page_numkeys(mc->pg[pre_top]) > 1); -static __inline int flock(HANDLE fd, unsigned flags, size_t offset, - size_t bytes) { - return flock_with_event(fd, 0, flags, offset, bytes); -} + /* Leaf page fill factor is below the threshold. + * Try to move keys from left or right neighbor, or + * merge with a neighbor page. */ -static __inline int flock_data(const MDBX_env *env, unsigned flags, - size_t offset, size_t bytes) { - const HANDLE fd4data = - env->me_overlapped_fd ? env->me_overlapped_fd : env->me_lazy_fd; - return flock_with_event(fd4data, env->me_data_lock_event, flags, offset, - bytes); -} + /* Find neighbors. */ + cursor_couple_t couple; + MDBX_cursor *const mn = cursor_clone(mc, &couple); -static int funlock(mdbx_filehandle_t fd, size_t offset, size_t bytes) { - TRACE("unlock: fd %p, offset %zu, bytes %zu", fd, offset, bytes); - return UnlockFile(fd, (DWORD)offset, HIGH_DWORD(offset), (DWORD)bytes, - HIGH_DWORD(bytes)) - ? MDBX_SUCCESS - : (int)GetLastError(); -} + page_t *left = nullptr, *right = nullptr; + if (mn->ki[pre_top] > 0) { + rc = page_get(mn, node_pgno(page_node(mn->pg[pre_top], mn->ki[pre_top] - 1)), &left, mc->pg[mc->top]->txnid); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + cASSERT(mc, page_type(left) == page_type(mc->pg[mc->top])); + } + if (mn->ki[pre_top] + (size_t)1 < page_numkeys(mn->pg[pre_top])) { + rc = page_get(mn, node_pgno(page_node(mn->pg[pre_top], mn->ki[pre_top] + (size_t)1)), &right, + mc->pg[mc->top]->txnid); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + cASSERT(mc, page_type(right) == page_type(mc->pg[mc->top])); + } + cASSERT(mc, left || right); -/*----------------------------------------------------------------------------*/ -/* global `write` lock for write-txt processing, - * exclusive locking both meta-pages) */ + const size_t ki_top = mc->ki[mc->top]; + const size_t ki_pre_top = mn->ki[pre_top]; + const size_t nkeys = page_numkeys(mn->pg[mn->top]); -#ifdef _WIN64 -#define DXB_MAXLEN UINT64_C(0x7fffFFFFfff00000) -#else -#define DXB_MAXLEN UINT32_C(0x7ff00000) -#endif -#define DXB_BODY (env->me_psize * (size_t)NUM_METAS), DXB_MAXLEN -#define DXB_WHOLE 0, DXB_MAXLEN + const size_t left_room = left ? page_room(left) : 0; + const size_t right_room = right ? page_room(right) : 0; + const size_t left_nkeys = left ? page_numkeys(left) : 0; + const size_t right_nkeys = right ? page_numkeys(right) : 0; + bool involve = !(left && right); +retry: + cASSERT(mc, mc->top > 0); + if (left_room > room_threshold && left_room >= right_room && (is_modifable(mc->txn, left) || involve)) { + /* try merge with left */ + cASSERT(mc, left_nkeys >= minkeys); + mn->pg[mn->top] = left; + mn->ki[mn->top - 1] = (indx_t)(ki_pre_top - 1); + mn->ki[mn->top] = (indx_t)(left_nkeys - 1); + mc->ki[mc->top] = 0; + const size_t new_ki = ki_top + left_nkeys; + mn->ki[mn->top] += mc->ki[mn->top] + 1; + couple.outer.next = mn->txn->cursors[cursor_dbi(mn)]; + mn->txn->cursors[cursor_dbi(mn)] = &couple.outer; + rc = page_merge(mc, mn); + mn->txn->cursors[cursor_dbi(mn)] = couple.outer.next; + if (likely(rc != MDBX_RESULT_TRUE)) { + cursor_cpstk(mn, mc); + mc->ki[mc->top] = (indx_t)new_ki; + cASSERT(mc, rc || page_numkeys(mc->pg[mc->top]) >= minkeys); + return rc; + } + } + if (right_room > room_threshold && (is_modifable(mc->txn, right) || involve)) { + /* try merge with right */ + cASSERT(mc, right_nkeys >= minkeys); + mn->pg[mn->top] = right; + mn->ki[mn->top - 1] = (indx_t)(ki_pre_top + 1); + mn->ki[mn->top] = 0; + mc->ki[mc->top] = (indx_t)nkeys; + couple.outer.next = mn->txn->cursors[cursor_dbi(mn)]; + mn->txn->cursors[cursor_dbi(mn)] = &couple.outer; + rc = page_merge(mn, mc); + mn->txn->cursors[cursor_dbi(mn)] = couple.outer.next; + if (likely(rc != MDBX_RESULT_TRUE)) { + mc->ki[mc->top] = (indx_t)ki_top; + cASSERT(mc, rc || page_numkeys(mc->pg[mc->top]) >= minkeys); + return rc; + } + } -int mdbx_txn_lock(MDBX_env *env, bool dontwait) { - if (dontwait) { - if (!TryEnterCriticalSection(&env->me_windowsbug_lock)) - return MDBX_BUSY; - } else { - __try { - EnterCriticalSection(&env->me_windowsbug_lock); + if (left_nkeys > minkeys && (right_nkeys <= left_nkeys || right_room >= left_room) && + (is_modifable(mc->txn, left) || involve)) { + /* try move from left */ + mn->pg[mn->top] = left; + mn->ki[mn->top - 1] = (indx_t)(ki_pre_top - 1); + mn->ki[mn->top] = (indx_t)(left_nkeys - 1); + mc->ki[mc->top] = 0; + couple.outer.next = mn->txn->cursors[cursor_dbi(mn)]; + mn->txn->cursors[cursor_dbi(mn)] = &couple.outer; + rc = node_move(mn, mc, true); + mn->txn->cursors[cursor_dbi(mn)] = couple.outer.next; + if (likely(rc != MDBX_RESULT_TRUE)) { + mc->ki[mc->top] = (indx_t)(ki_top + 1); + cASSERT(mc, rc || page_numkeys(mc->pg[mc->top]) >= minkeys); + return rc; } - __except ((GetExceptionCode() == - 0xC0000194 /* STATUS_POSSIBLE_DEADLOCK / EXCEPTION_POSSIBLE_DEADLOCK */) - ? EXCEPTION_EXECUTE_HANDLER - : EXCEPTION_CONTINUE_SEARCH) { - return ERROR_POSSIBLE_DEADLOCK; + } + if (right_nkeys > minkeys && (is_modifable(mc->txn, right) || involve)) { + /* try move from right */ + mn->pg[mn->top] = right; + mn->ki[mn->top - 1] = (indx_t)(ki_pre_top + 1); + mn->ki[mn->top] = 0; + mc->ki[mc->top] = (indx_t)nkeys; + couple.outer.next = mn->txn->cursors[cursor_dbi(mn)]; + mn->txn->cursors[cursor_dbi(mn)] = &couple.outer; + rc = node_move(mn, mc, false); + mn->txn->cursors[cursor_dbi(mn)] = couple.outer.next; + if (likely(rc != MDBX_RESULT_TRUE)) { + mc->ki[mc->top] = (indx_t)ki_top; + cASSERT(mc, rc || page_numkeys(mc->pg[mc->top]) >= minkeys); + return rc; } } - if (env->me_flags & MDBX_EXCLUSIVE) { - /* Zap: Failing to release lock 'env->me_windowsbug_lock' - * in function 'mdbx_txn_lock' */ - MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(26115); + if (nkeys >= minkeys) { + mc->ki[mc->top] = (indx_t)ki_top; + if (AUDIT_ENABLED()) + return cursor_validate_updating(mc); return MDBX_SUCCESS; } - const HANDLE fd4data = - env->me_overlapped_fd ? env->me_overlapped_fd : env->me_lazy_fd; - int rc = flock_with_event(fd4data, env->me_data_lock_event, - dontwait ? (LCK_EXCLUSIVE | LCK_DONTWAIT) - : (LCK_EXCLUSIVE | LCK_WAITFOR), - DXB_BODY); - if (rc == ERROR_LOCK_VIOLATION && dontwait) { - SleepEx(0, true); - rc = flock_with_event(fd4data, env->me_data_lock_event, - LCK_EXCLUSIVE | LCK_DONTWAIT, DXB_BODY); - if (rc == ERROR_LOCK_VIOLATION) { - SleepEx(0, true); - rc = flock_with_event(fd4data, env->me_data_lock_event, - LCK_EXCLUSIVE | LCK_DONTWAIT, DXB_BODY); - } + if (mc->txn->env->options.prefer_waf_insteadof_balance && likely(room_threshold > 0)) { + room_threshold = 0; + goto retry; } - if (rc == MDBX_SUCCESS) { - /* Zap: Failing to release lock 'env->me_windowsbug_lock' - * in function 'mdbx_txn_lock' */ - MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(26115); - return rc; + if (likely(!involve) && + (likely(mc->tree != &mc->txn->dbs[FREE_DBI]) || mc->txn->tw.loose_pages || MDBX_PNL_GETSIZE(mc->txn->tw.repnl) || + (mc->flags & z_gcu_preparation) || (mc->txn->flags & txn_gc_drained) || room_threshold)) { + involve = true; + goto retry; + } + if (likely(room_threshold > 0)) { + room_threshold = 0; + goto retry; } - LeaveCriticalSection(&env->me_windowsbug_lock); - return (!dontwait || rc != ERROR_LOCK_VIOLATION) ? rc : MDBX_BUSY; + ERROR("Unable to merge/rebalance %s page %" PRIaPGNO " (has %zu keys, fill %u.%u%%, used %zu, room %zu bytes)", + is_leaf(tp) ? "leaf" : "branch", tp->pgno, numkeys, page_fill_percentum_x10(mc->txn->env, tp) / 10, + page_fill_percentum_x10(mc->txn->env, tp) % 10, page_used(mc->txn->env, tp), room); + return MDBX_PROBLEM; } -void mdbx_txn_unlock(MDBX_env *env) { - if ((env->me_flags & MDBX_EXCLUSIVE) == 0) { - const HANDLE fd4data = - env->me_overlapped_fd ? env->me_overlapped_fd : env->me_lazy_fd; - int err = funlock(fd4data, DXB_BODY); - if (err != MDBX_SUCCESS) - mdbx_panic("%s failed: err %u", __func__, err); +int page_split(MDBX_cursor *mc, const MDBX_val *const newkey, MDBX_val *const newdata, pgno_t newpgno, + const unsigned naf) { + unsigned flags; + int rc = MDBX_SUCCESS, foliage = 0; + MDBX_env *const env = mc->txn->env; + MDBX_val rkey, xdata; + page_t *tmp_ki_copy = nullptr; + DKBUF; + + page_t *const mp = mc->pg[mc->top]; + cASSERT(mc, (mp->flags & P_ILL_BITS) == 0); + + const size_t newindx = mc->ki[mc->top]; + size_t nkeys = page_numkeys(mp); + if (AUDIT_ENABLED()) { + rc = cursor_validate_updating(mc); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; } - LeaveCriticalSection(&env->me_windowsbug_lock); -} + STATIC_ASSERT(P_BRANCH == 1); + const size_t minkeys = (mp->flags & P_BRANCH) + (size_t)1; -/*----------------------------------------------------------------------------*/ -/* global `read` lock for readers registration, - * exclusive locking `mti_numreaders` (second) cacheline */ + DEBUG(">> splitting %s-page %" PRIaPGNO " and adding %zu+%zu [%s] at %i, nkeys %zi", is_leaf(mp) ? "leaf" : "branch", + mp->pgno, newkey->iov_len, newdata ? newdata->iov_len : 0, DKEY_DEBUG(newkey), mc->ki[mc->top], nkeys); + cASSERT(mc, nkeys + 1 >= minkeys * 2); -#define LCK_LO_OFFSET 0 -#define LCK_LO_LEN offsetof(MDBX_lockinfo, mti_numreaders) -#define LCK_UP_OFFSET LCK_LO_LEN -#define LCK_UP_LEN (sizeof(MDBX_lockinfo) - LCK_UP_OFFSET) -#define LCK_LOWER LCK_LO_OFFSET, LCK_LO_LEN -#define LCK_UPPER LCK_UP_OFFSET, LCK_UP_LEN + /* Create a new sibling page. */ + pgr_t npr = page_new(mc, mp->flags); + if (unlikely(npr.err != MDBX_SUCCESS)) + return npr.err; + page_t *const sister = npr.page; + sister->dupfix_ksize = mp->dupfix_ksize; + DEBUG("new sibling: page %" PRIaPGNO, sister->pgno); -MDBX_INTERNAL_FUNC int osal_rdt_lock(MDBX_env *env) { - osal_srwlock_AcquireShared(&env->me_remap_guard); - if (env->me_lfd == INVALID_HANDLE_VALUE) - return MDBX_SUCCESS; /* readonly database in readonly filesystem */ + /* Usually when splitting the root page, the cursor + * height is 1. But when called from tree_propagate_key, + * the cursor height may be greater because it walks + * up the stack while finding the branch slot to update. */ + intptr_t prev_top = mc->top - 1; + if (mc->top == 0) { + npr = page_new(mc, P_BRANCH); + rc = npr.err; + if (unlikely(rc != MDBX_SUCCESS)) + goto done; + page_t *const pp = npr.page; + /* shift current top to make room for new parent */ + cASSERT(mc, mc->tree->height > 0); +#if MDBX_DEBUG + memset(mc->pg + 3, 0, sizeof(mc->pg) - sizeof(mc->pg[0]) * 3); + memset(mc->ki + 3, -1, sizeof(mc->ki) - sizeof(mc->ki[0]) * 3); +#endif + mc->pg[2] = mc->pg[1]; + mc->ki[2] = mc->ki[1]; + mc->pg[1] = mc->pg[0]; + mc->ki[1] = mc->ki[0]; + mc->pg[0] = pp; + mc->ki[0] = 0; + mc->tree->root = pp->pgno; + DEBUG("root split! new root = %" PRIaPGNO, pp->pgno); + foliage = mc->tree->height++; - /* transition from S-? (used) to S-E (locked), - * e.g. exclusive lock upper-part */ - if (env->me_flags & MDBX_EXCLUSIVE) - return MDBX_SUCCESS; + /* Add left (implicit) pointer. */ + rc = node_add_branch(mc, 0, nullptr, mp->pgno); + if (unlikely(rc != MDBX_SUCCESS)) { + /* undo the pre-push */ + mc->pg[0] = mc->pg[1]; + mc->ki[0] = mc->ki[1]; + mc->tree->root = mp->pgno; + mc->tree->height--; + goto done; + } + mc->top = 1; + prev_top = 0; + if (AUDIT_ENABLED()) { + rc = cursor_validate_updating(mc); + if (unlikely(rc != MDBX_SUCCESS)) + goto done; + } + } else { + DEBUG("parent branch page is %" PRIaPGNO, mc->pg[prev_top]->pgno); + } - int rc = flock(env->me_lfd, LCK_EXCLUSIVE | LCK_WAITFOR, LCK_UPPER); - if (rc == MDBX_SUCCESS) - return MDBX_SUCCESS; + cursor_couple_t couple; + MDBX_cursor *const mn = cursor_clone(mc, &couple); + mn->pg[mn->top] = sister; + mn->ki[mn->top] = 0; + mn->ki[prev_top] = mc->ki[prev_top] + 1; - osal_srwlock_ReleaseShared(&env->me_remap_guard); - return rc; -} + size_t split_indx = (newindx < nkeys) ? /* split at the middle */ (nkeys + 1) >> 1 + : /* split at the end (i.e. like append-mode ) */ nkeys - minkeys + 1; + eASSERT(env, split_indx >= minkeys && split_indx <= nkeys - minkeys + 1); -MDBX_INTERNAL_FUNC void osal_rdt_unlock(MDBX_env *env) { - if (env->me_lfd != INVALID_HANDLE_VALUE && - (env->me_flags & MDBX_EXCLUSIVE) == 0) { - /* transition from S-E (locked) to S-? (used), e.g. unlock upper-part */ - int err = funlock(env->me_lfd, LCK_UPPER); - if (err != MDBX_SUCCESS) - mdbx_panic("%s failed: err %u", __func__, err); + cASSERT(mc, !is_branch(mp) || newindx > 0); + MDBX_val sepkey = {nullptr, 0}; + /* It is reasonable and possible to split the page at the begin */ + if (unlikely(newindx < minkeys)) { + split_indx = minkeys; + if (newindx == 0 && !(naf & MDBX_SPLIT_REPLACE)) { + split_indx = 0; + /* Checking for ability of splitting by the left-side insertion + * of a pure page with the new key */ + for (intptr_t i = 0; i < mc->top; ++i) + if (mc->ki[i]) { + sepkey = get_key(page_node(mc->pg[i], mc->ki[i])); + if (mc->clc->k.cmp(newkey, &sepkey) >= 0) + split_indx = minkeys; + break; + } + if (split_indx == 0) { + /* Save the current first key which was omitted on the parent branch + * page and should be updated if the new first entry will be added */ + if (is_dupfix_leaf(mp)) + sepkey = page_dupfix_key(mp, 0, mc->tree->dupfix_size); + else + sepkey = get_key(page_node(mp, 0)); + cASSERT(mc, mc->clc->k.cmp(newkey, &sepkey) < 0); + /* Avoiding rare complex cases of nested split the parent page(s) */ + if (page_room(mc->pg[prev_top]) < branch_size(env, &sepkey)) + split_indx = minkeys; + } + if (foliage) { + TRACE("pure-left: foliage %u, top %i, ptop %zu, split_indx %zi, " + "minkeys %zi, sepkey %s, parent-room %zu, need4split %zu", + foliage, mc->top, prev_top, split_indx, minkeys, DKEY_DEBUG(&sepkey), page_room(mc->pg[prev_top]), + branch_size(env, &sepkey)); + TRACE("pure-left: newkey %s, newdata %s, newindx %zu", DKEY_DEBUG(newkey), DVAL_DEBUG(newdata), newindx); + } + } } - osal_srwlock_ReleaseShared(&env->me_remap_guard); -} -MDBX_INTERNAL_FUNC int osal_lockfile(mdbx_filehandle_t fd, bool wait) { - return flock( - fd, wait ? LCK_EXCLUSIVE | LCK_WAITFOR : LCK_EXCLUSIVE | LCK_DONTWAIT, 0, - DXB_MAXLEN); -} + const bool pure_right = split_indx == nkeys; + const bool pure_left = split_indx == 0; + if (unlikely(pure_right)) { + /* newindx == split_indx == nkeys */ + TRACE("no-split, but add new pure page at the %s", "right/after"); + cASSERT(mc, newindx == nkeys && split_indx == nkeys && minkeys == 1); + sepkey = *newkey; + } else if (unlikely(pure_left)) { + /* newindx == split_indx == 0 */ + TRACE("pure-left: no-split, but add new pure page at the %s", "left/before"); + cASSERT(mc, newindx == 0 && split_indx == 0 && minkeys == 1); + TRACE("pure-left: old-first-key is %s", DKEY_DEBUG(&sepkey)); + } else { + if (is_dupfix_leaf(sister)) { + /* Move half of the keys to the right sibling */ + const intptr_t distance = mc->ki[mc->top] - split_indx; + size_t ksize = mc->tree->dupfix_size; + void *const split = page_dupfix_ptr(mp, split_indx, ksize); + size_t rsize = (nkeys - split_indx) * ksize; + size_t lsize = (nkeys - split_indx) * sizeof(indx_t); + cASSERT(mc, mp->lower >= lsize); + mp->lower -= (indx_t)lsize; + cASSERT(mc, sister->lower + lsize <= UINT16_MAX); + sister->lower += (indx_t)lsize; + cASSERT(mc, mp->upper + rsize - lsize <= UINT16_MAX); + mp->upper += (indx_t)(rsize - lsize); + cASSERT(mc, sister->upper >= rsize - lsize); + sister->upper -= (indx_t)(rsize - lsize); + sepkey.iov_len = ksize; + sepkey.iov_base = (newindx != split_indx) ? split : newkey->iov_base; + if (distance < 0) { + cASSERT(mc, ksize >= sizeof(indx_t)); + void *const ins = page_dupfix_ptr(mp, mc->ki[mc->top], ksize); + memcpy(sister->entries, split, rsize); + sepkey.iov_base = sister->entries; + memmove(ptr_disp(ins, ksize), ins, (split_indx - mc->ki[mc->top]) * ksize); + memcpy(ins, newkey->iov_base, ksize); + cASSERT(mc, UINT16_MAX - mp->lower >= (int)sizeof(indx_t)); + mp->lower += sizeof(indx_t); + cASSERT(mc, mp->upper >= ksize - sizeof(indx_t)); + mp->upper -= (indx_t)(ksize - sizeof(indx_t)); + cASSERT(mc, (((ksize & page_numkeys(mp)) ^ mp->upper) & 1) == 0); + } else { + memcpy(sister->entries, split, distance * ksize); + void *const ins = page_dupfix_ptr(sister, distance, ksize); + memcpy(ins, newkey->iov_base, ksize); + memcpy(ptr_disp(ins, ksize), ptr_disp(split, distance * ksize), rsize - distance * ksize); + cASSERT(mc, UINT16_MAX - sister->lower >= (int)sizeof(indx_t)); + sister->lower += sizeof(indx_t); + cASSERT(mc, sister->upper >= ksize - sizeof(indx_t)); + sister->upper -= (indx_t)(ksize - sizeof(indx_t)); + cASSERT(mc, distance <= (int)UINT16_MAX); + mc->ki[mc->top] = (indx_t)distance; + cASSERT(mc, (((ksize & page_numkeys(sister)) ^ sister->upper) & 1) == 0); + } -static int suspend_and_append(mdbx_handle_array_t **array, - const DWORD ThreadId) { - const unsigned limit = (*array)->limit; - if ((*array)->count == limit) { - mdbx_handle_array_t *const ptr = - osal_realloc((limit > ARRAY_LENGTH((*array)->handles)) - ? *array - : /* don't free initial array on the stack */ NULL, - sizeof(mdbx_handle_array_t) + - sizeof(HANDLE) * (limit * (size_t)2 - - ARRAY_LENGTH((*array)->handles))); - if (!ptr) - return MDBX_ENOMEM; - if (limit == ARRAY_LENGTH((*array)->handles)) - *ptr = **array; - *array = ptr; - (*array)->limit = limit * 2; - } + if (AUDIT_ENABLED()) { + rc = cursor_validate_updating(mc); + if (unlikely(rc != MDBX_SUCCESS)) + goto done; + rc = cursor_validate_updating(mn); + if (unlikely(rc != MDBX_SUCCESS)) + goto done; + } + } else { + /* grab a page to hold a temporary copy */ + tmp_ki_copy = page_shadow_alloc(mc->txn, 1); + if (unlikely(tmp_ki_copy == nullptr)) { + rc = MDBX_ENOMEM; + goto done; + } + + const size_t max_space = page_space(env); + const size_t new_size = is_leaf(mp) ? leaf_size(env, newkey, newdata) : branch_size(env, newkey); + + /* prepare to insert */ + size_t i = 0; + while (i < newindx) { + tmp_ki_copy->entries[i] = mp->entries[i]; + ++i; + } + tmp_ki_copy->entries[i] = (indx_t)-1; + while (++i <= nkeys) + tmp_ki_copy->entries[i] = mp->entries[i - 1]; + tmp_ki_copy->pgno = mp->pgno; + tmp_ki_copy->flags = mp->flags; + tmp_ki_copy->txnid = INVALID_TXNID; + tmp_ki_copy->lower = 0; + tmp_ki_copy->upper = (indx_t)max_space; + + /* Добавляемый узел может не поместиться в страницу-половину вместе + * с количественной половиной узлов из исходной страницы. В худшем случае, + * в страницу-половину с добавляемым узлом могут попасть самые больше узлы + * из исходной страницы, а другую половину только узлы с самыми короткими + * ключами и с пустыми данными. Поэтому, чтобы найти подходящую границу + * разреза требуется итерировать узлы и считая их объем. + * + * Однако, при простом количественном делении (без учета размера ключей + * и данных) на страницах-половинах будет примерно вдвое меньше узлов. + * Поэтому добавляемый узел точно поместится, если его размер не больше + * чем место "освобождающееся" от заголовков узлов, которые переедут + * в другую страницу-половину. Кроме этого, как минимум по одному байту + * будет в каждом ключе, в худшем случае кроме одного, который может быть + * нулевого размера. */ + + if (newindx == split_indx && nkeys >= 5) { + STATIC_ASSERT(P_BRANCH == 1); + split_indx += mp->flags & P_BRANCH; + } + eASSERT(env, split_indx >= minkeys && split_indx <= nkeys + 1 - minkeys); + const size_t dim_nodes = (newindx >= split_indx) ? split_indx : nkeys - split_indx; + const size_t dim_used = (sizeof(indx_t) + NODESIZE + 1) * dim_nodes; + if (new_size >= dim_used) { + /* Search for best acceptable split point */ + i = (newindx < split_indx) ? 0 : nkeys; + intptr_t dir = (newindx < split_indx) ? 1 : -1; + size_t before = 0, after = new_size + page_used(env, mp); + size_t best_split = split_indx; + size_t best_shift = INT_MAX; - HANDLE hThread = OpenThread(THREAD_SUSPEND_RESUME | THREAD_QUERY_INFORMATION, - FALSE, ThreadId); - if (hThread == NULL) - return (int)GetLastError(); + TRACE("seek separator from %zu, step %zi, default %zu, new-idx %zu, " + "new-size %zu", + i, dir, split_indx, newindx, new_size); + do { + cASSERT(mc, i <= nkeys); + size_t size = new_size; + if (i != newindx) { + node_t *node = ptr_disp(mp, tmp_ki_copy->entries[i] + PAGEHDRSZ); + size = NODESIZE + node_ks(node) + sizeof(indx_t); + if (is_leaf(mp)) + size += (node_flags(node) & N_BIG) ? sizeof(pgno_t) : node_ds(node); + size = EVEN_CEIL(size); + } - if (SuspendThread(hThread) == (DWORD)-1) { - int err = (int)GetLastError(); - DWORD ExitCode; - if (err == /* workaround for Win10 UCRT bug */ ERROR_ACCESS_DENIED || - !GetExitCodeThread(hThread, &ExitCode) || ExitCode != STILL_ACTIVE) - err = MDBX_SUCCESS; - CloseHandle(hThread); - return err; - } + before += size; + after -= size; + TRACE("step %zu, size %zu, before %zu, after %zu, max %zu", i, size, before, after, max_space); - (*array)->handles[(*array)->count++] = hThread; - return MDBX_SUCCESS; -} + if (before <= max_space && after <= max_space) { + const size_t split = i + (dir > 0); + if (split >= minkeys && split <= nkeys + 1 - minkeys) { + const size_t shift = branchless_abs(split_indx - split); + if (shift >= best_shift) + break; + best_shift = shift; + best_split = split; + if (!best_shift) + break; + } + } + i += dir; + } while (i < nkeys); -MDBX_INTERNAL_FUNC int -osal_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array) { - eASSERT(env, (env->me_flags & MDBX_NOTLS) == 0); - const uintptr_t CurrentTid = GetCurrentThreadId(); - int rc; - if (env->me_lck_mmap.lck) { - /* Scan LCK for threads of the current process */ - const MDBX_reader *const begin = env->me_lck_mmap.lck->mti_readers; - const MDBX_reader *const end = - begin + - atomic_load32(&env->me_lck_mmap.lck->mti_numreaders, mo_AcquireRelease); - const uintptr_t WriteTxnOwner = env->me_txn0 ? env->me_txn0->mt_owner : 0; - for (const MDBX_reader *reader = begin; reader < end; ++reader) { - if (reader->mr_pid.weak != env->me_pid || !reader->mr_tid.weak) { - skip_lck: - continue; + split_indx = best_split; + TRACE("chosen %zu", split_indx); } - if (reader->mr_tid.weak == CurrentTid || - reader->mr_tid.weak == WriteTxnOwner) - goto skip_lck; + eASSERT(env, split_indx >= minkeys && split_indx <= nkeys + 1 - minkeys); - rc = suspend_and_append(array, (mdbx_tid_t)reader->mr_tid.weak); - if (rc != MDBX_SUCCESS) { - bailout_lck: - (void)osal_resume_threads_after_remap(*array); - return rc; + sepkey = *newkey; + if (split_indx != newindx) { + node_t *node = ptr_disp(mp, tmp_ki_copy->entries[split_indx] + PAGEHDRSZ); + sepkey.iov_len = node_ks(node); + sepkey.iov_base = node_key(node); } } - if (WriteTxnOwner && WriteTxnOwner != CurrentTid) { - rc = suspend_and_append(array, (mdbx_tid_t)WriteTxnOwner); - if (rc != MDBX_SUCCESS) - goto bailout_lck; - } - } else { - /* Without LCK (i.e. read-only mode). - * Walk through a snapshot of all running threads */ - eASSERT(env, env->me_flags & (MDBX_EXCLUSIVE | MDBX_RDONLY)); - const HANDLE hSnapshot = CreateToolhelp32Snapshot(TH32CS_SNAPTHREAD, 0); - if (hSnapshot == INVALID_HANDLE_VALUE) - return (int)GetLastError(); - - THREADENTRY32 entry; - entry.dwSize = sizeof(THREADENTRY32); + } + DEBUG("separator is %zd [%s]", split_indx, DKEY_DEBUG(&sepkey)); - if (!Thread32First(hSnapshot, &entry)) { - rc = (int)GetLastError(); - bailout_toolhelp: - CloseHandle(hSnapshot); - (void)osal_resume_threads_after_remap(*array); - return rc; + bool did_split_parent = false; + /* Copy separator key to the parent. */ + if (page_room(mn->pg[prev_top]) < branch_size(env, &sepkey)) { + TRACE("need split parent branch-page for key %s", DKEY_DEBUG(&sepkey)); + cASSERT(mc, page_numkeys(mn->pg[prev_top]) > 2); + cASSERT(mc, !pure_left); + const int top = mc->top; + const int height = mc->tree->height; + mn->top -= 1; + did_split_parent = true; + couple.outer.next = mn->txn->cursors[cursor_dbi(mn)]; + mn->txn->cursors[cursor_dbi(mn)] = &couple.outer; + rc = page_split(mn, &sepkey, nullptr, sister->pgno, 0); + mn->txn->cursors[cursor_dbi(mn)] = couple.outer.next; + if (unlikely(rc != MDBX_SUCCESS)) + goto done; + cASSERT(mc, mc->top - top == mc->tree->height - height); + if (AUDIT_ENABLED()) { + rc = cursor_validate_updating(mc); + if (unlikely(rc != MDBX_SUCCESS)) + goto done; } - do { - if (entry.th32OwnerProcessID != env->me_pid || - entry.th32ThreadID == CurrentTid) - continue; + /* root split? */ + prev_top += mc->top - top; - rc = suspend_and_append(array, entry.th32ThreadID); - if (rc != MDBX_SUCCESS) - goto bailout_toolhelp; + /* Right page might now have changed parent. + * Check if left page also changed parent. */ + if (mn->pg[prev_top] != mc->pg[prev_top] && mc->ki[prev_top] >= page_numkeys(mc->pg[prev_top])) { + for (intptr_t i = 0; i < prev_top; i++) { + mc->pg[i] = mn->pg[i]; + mc->ki[i] = mn->ki[i]; + } + mc->pg[prev_top] = mn->pg[prev_top]; + if (mn->ki[prev_top]) { + mc->ki[prev_top] = mn->ki[prev_top] - 1; + } else { + /* find right page's left sibling */ + mc->ki[prev_top] = mn->ki[prev_top]; + rc = cursor_sibling_left(mc); + if (unlikely(rc != MDBX_SUCCESS)) { + if (rc == MDBX_NOTFOUND) /* improper mdbx_cursor_sibling() result */ { + ERROR("unexpected %i error going left sibling", rc); + rc = MDBX_PROBLEM; + } + goto done; + } + } + } + } else if (unlikely(pure_left)) { + page_t *ptop_page = mc->pg[prev_top]; + TRACE("pure-left: adding to parent page %u node[%u] left-leaf page #%u key " + "%s", + ptop_page->pgno, mc->ki[prev_top], sister->pgno, DKEY(mc->ki[prev_top] ? newkey : nullptr)); + assert(mc->top == prev_top + 1); + mc->top = (uint8_t)prev_top; + rc = node_add_branch(mc, mc->ki[prev_top], mc->ki[prev_top] ? newkey : nullptr, sister->pgno); + cASSERT(mc, mp == mc->pg[prev_top + 1] && newindx == mc->ki[prev_top + 1] && prev_top == mc->top); + + if (likely(rc == MDBX_SUCCESS) && mc->ki[prev_top] == 0) { + node_t *node = page_node(mc->pg[prev_top], 1); + TRACE("pure-left: update prev-first key on parent to %s", DKEY(&sepkey)); + cASSERT(mc, node_ks(node) == 0 && node_pgno(node) == mp->pgno); + cASSERT(mc, mc->top == prev_top && mc->ki[prev_top] == 0); + mc->ki[prev_top] = 1; + rc = tree_propagate_key(mc, &sepkey); + cASSERT(mc, mc->top == prev_top && mc->ki[prev_top] == 1); + cASSERT(mc, mp == mc->pg[prev_top + 1] && newindx == mc->ki[prev_top + 1]); + mc->ki[prev_top] = 0; + } else { + TRACE("pure-left: no-need-update prev-first key on parent %s", DKEY(&sepkey)); + } - } while (Thread32Next(hSnapshot, &entry)); + mc->top++; + if (unlikely(rc != MDBX_SUCCESS)) + goto done; - rc = (int)GetLastError(); - if (rc != ERROR_NO_MORE_FILES) - goto bailout_toolhelp; - CloseHandle(hSnapshot); + node_t *node = page_node(mc->pg[prev_top], mc->ki[prev_top] + (size_t)1); + cASSERT(mc, node_pgno(node) == mp->pgno && mc->pg[prev_top] == ptop_page); + } else { + mn->top -= 1; + TRACE("add-to-parent the right-entry[%u] for new sibling-page", mn->ki[prev_top]); + rc = node_add_branch(mn, mn->ki[prev_top], &sepkey, sister->pgno); + mn->top += 1; + if (unlikely(rc != MDBX_SUCCESS)) + goto done; } - return MDBX_SUCCESS; -} - -MDBX_INTERNAL_FUNC int -osal_resume_threads_after_remap(mdbx_handle_array_t *array) { - int rc = MDBX_SUCCESS; - for (unsigned i = 0; i < array->count; ++i) { - const HANDLE hThread = array->handles[i]; - if (ResumeThread(hThread) == (DWORD)-1) { - const int err = (int)GetLastError(); - DWORD ExitCode; - if (err != /* workaround for Win10 UCRT bug */ ERROR_ACCESS_DENIED && - GetExitCodeThread(hThread, &ExitCode) && ExitCode == STILL_ACTIVE) - rc = err; + if (unlikely(pure_left | pure_right)) { + mc->pg[mc->top] = sister; + mc->ki[mc->top] = 0; + switch (page_type(sister)) { + case P_LEAF: { + cASSERT(mc, newpgno == 0 || newpgno == P_INVALID); + rc = node_add_leaf(mc, 0, newkey, newdata, naf); + } break; + case P_LEAF | P_DUPFIX: { + cASSERT(mc, (naf & (N_BIG | N_TREE | N_DUP)) == 0); + cASSERT(mc, newpgno == 0 || newpgno == P_INVALID); + rc = node_add_dupfix(mc, 0, newkey); + } break; + default: + rc = bad_page(sister, "wrong page-type %u\n", page_type(sister)); } - CloseHandle(hThread); - } - return rc; -} - -/*----------------------------------------------------------------------------*/ -/* global `initial` lock for lockfile initialization, - * exclusive/shared locking first cacheline */ - -/* Briefly description of locking schema/algorithm: - * - Windows does not support upgrading or downgrading for file locking. - * - Therefore upgrading/downgrading is emulated by shared and exclusive - * locking of upper and lower halves. - * - In other words, we have FSM with possible 9 states, - * i.e. free/shared/exclusive x free/shared/exclusive == 9. - * Only 6 states of FSM are used, which 2 of ones are transitive. - * - * States: - * ?-? = free, i.e. unlocked - * S-? = used, i.e. shared lock - * E-? = exclusive-read, i.e. operational exclusive - * ?-S - * ?-E = middle (transitive state) - * S-S - * S-E = locked (transitive state) - * E-S - * E-E = exclusive-write, i.e. exclusive due (re)initialization - * - * The osal_lck_seize() moves the locking-FSM from the initial free/unlocked - * state to the "exclusive write" (and returns MDBX_RESULT_TRUE) if possible, - * or to the "used" (and returns MDBX_RESULT_FALSE). - * - * The osal_lck_downgrade() moves the locking-FSM from "exclusive write" - * state to the "used" (i.e. shared) state. - * - * The mdbx_lck_upgrade() moves the locking-FSM from "used" (i.e. shared) - * state to the "exclusive write" state. - */ - -static void lck_unlock(MDBX_env *env) { - int err; - - if (env->me_lfd != INVALID_HANDLE_VALUE) { - /* double `unlock` for robustly remove overlapped shared/exclusive locks */ - do - err = funlock(env->me_lfd, LCK_LOWER); - while (err == MDBX_SUCCESS); - assert(err == ERROR_NOT_LOCKED || - (mdbx_RunningUnderWine() && err == ERROR_LOCK_VIOLATION)); - SetLastError(ERROR_SUCCESS); - - do - err = funlock(env->me_lfd, LCK_UPPER); - while (err == MDBX_SUCCESS); - assert(err == ERROR_NOT_LOCKED || - (mdbx_RunningUnderWine() && err == ERROR_LOCK_VIOLATION)); - SetLastError(ERROR_SUCCESS); - } + if (unlikely(rc != MDBX_SUCCESS)) + goto done; - const HANDLE fd4data = - env->me_overlapped_fd ? env->me_overlapped_fd : env->me_lazy_fd; - if (fd4data != INVALID_HANDLE_VALUE) { - /* explicitly unlock to avoid latency for other processes (windows kernel - * releases such locks via deferred queues) */ - do - err = funlock(fd4data, DXB_BODY); - while (err == MDBX_SUCCESS); - assert(err == ERROR_NOT_LOCKED || - (mdbx_RunningUnderWine() && err == ERROR_LOCK_VIOLATION)); - SetLastError(ERROR_SUCCESS); + if (pure_right) { + for (intptr_t i = 0; i < mc->top; i++) + mc->ki[i] = mn->ki[i]; + } else if (mc->ki[mc->top - 1] == 0) { + for (intptr_t i = 2; i <= mc->top; ++i) + if (mc->ki[mc->top - i]) { + sepkey = get_key(page_node(mc->pg[mc->top - i], mc->ki[mc->top - i])); + if (mc->clc->k.cmp(newkey, &sepkey) < 0) { + mc->top -= (int8_t)i; + DEBUG("pure-left: update new-first on parent [%i] page %u key %s", mc->ki[mc->top], mc->pg[mc->top]->pgno, + DKEY(newkey)); + rc = tree_propagate_key(mc, newkey); + mc->top += (int8_t)i; + if (unlikely(rc != MDBX_SUCCESS)) + goto done; + } + break; + } + } + } else if (tmp_ki_copy) { /* !is_dupfix_leaf(mp) */ + /* Move nodes */ + mc->pg[mc->top] = sister; + size_t n = 0, ii = split_indx; + do { + TRACE("i %zu, nkeys %zu => n %zu, rp #%u", ii, nkeys, n, sister->pgno); + pgno_t pgno = 0; + MDBX_val *rdata = nullptr; + if (ii == newindx) { + rkey = *newkey; + if (is_leaf(mp)) + rdata = newdata; + else + pgno = newpgno; + flags = naf; + /* Update index for the new key. */ + mc->ki[mc->top] = (indx_t)n; + } else { + node_t *node = ptr_disp(mp, tmp_ki_copy->entries[ii] + PAGEHDRSZ); + rkey.iov_base = node_key(node); + rkey.iov_len = node_ks(node); + if (is_leaf(mp)) { + xdata.iov_base = node_data(node); + xdata.iov_len = node_ds(node); + rdata = &xdata; + } else + pgno = node_pgno(node); + flags = node_flags(node); + } - do - err = funlock(fd4data, DXB_WHOLE); - while (err == MDBX_SUCCESS); - assert(err == ERROR_NOT_LOCKED || - (mdbx_RunningUnderWine() && err == ERROR_LOCK_VIOLATION)); - SetLastError(ERROR_SUCCESS); - } -} + switch (page_type(sister)) { + case P_BRANCH: { + cASSERT(mc, 0 == (uint16_t)flags); + /* First branch index doesn't need key data. */ + rc = node_add_branch(mc, n, n ? &rkey : nullptr, pgno); + } break; + case P_LEAF: { + cASSERT(mc, pgno == 0); + cASSERT(mc, rdata != nullptr); + rc = node_add_leaf(mc, n, &rkey, rdata, flags); + } break; + /* case P_LEAF | P_DUPFIX: { + cASSERT(mc, (nflags & (N_BIG | N_TREE | N_DUP)) == 0); + cASSERT(mc, gno == 0); + rc = mdbx_node_add_dupfix(mc, n, &rkey); + } break; */ + default: + rc = bad_page(sister, "wrong page-type %u\n", page_type(sister)); + } + if (unlikely(rc != MDBX_SUCCESS)) + goto done; -/* Seize state as 'exclusive-write' (E-E and returns MDBX_RESULT_TRUE) - * or as 'used' (S-? and returns MDBX_RESULT_FALSE). - * Otherwise returns an error. */ -static int internal_seize_lck(HANDLE lfd) { - assert(lfd != INVALID_HANDLE_VALUE); + ++n; + if (++ii > nkeys) { + ii = 0; + n = 0; + mc->pg[mc->top] = tmp_ki_copy; + TRACE("switch to mp #%u", tmp_ki_copy->pgno); + } + } while (ii != split_indx); - /* 1) now on ?-? (free), get ?-E (middle) */ - jitter4testing(false); - int rc = flock(lfd, LCK_EXCLUSIVE | LCK_WAITFOR, LCK_UPPER); - if (rc != MDBX_SUCCESS) { - /* 2) something went wrong, give up */; - ERROR("%s, err %u", "?-?(free) >> ?-E(middle)", rc); - return rc; - } + TRACE("ii %zu, nkeys %zu, n %zu, pgno #%u", ii, nkeys, n, mc->pg[mc->top]->pgno); - /* 3) now on ?-E (middle), try E-E (exclusive-write) */ - jitter4testing(false); - rc = flock(lfd, LCK_EXCLUSIVE | LCK_DONTWAIT, LCK_LOWER); - if (rc == MDBX_SUCCESS) - return MDBX_RESULT_TRUE /* 4) got E-E (exclusive-write), done */; + nkeys = page_numkeys(tmp_ki_copy); + for (size_t i = 0; i < nkeys; i++) + mp->entries[i] = tmp_ki_copy->entries[i]; + mp->lower = tmp_ki_copy->lower; + mp->upper = tmp_ki_copy->upper; + memcpy(page_node(mp, nkeys - 1), page_node(tmp_ki_copy, nkeys - 1), env->ps - tmp_ki_copy->upper - PAGEHDRSZ); - /* 5) still on ?-E (middle) */ - jitter4testing(false); - if (rc != ERROR_SHARING_VIOLATION && rc != ERROR_LOCK_VIOLATION) { - /* 6) something went wrong, give up */ - rc = funlock(lfd, LCK_UPPER); - if (rc != MDBX_SUCCESS) - mdbx_panic("%s(%s) failed: err %u", __func__, "?-E(middle) >> ?-?(free)", - rc); - return rc; + /* reset back to original page */ + if (newindx < split_indx) { + mc->pg[mc->top] = mp; + } else { + mc->pg[mc->top] = sister; + mc->ki[prev_top]++; + /* Make sure ki is still valid. */ + if (mn->pg[prev_top] != mc->pg[prev_top] && mc->ki[prev_top] >= page_numkeys(mc->pg[prev_top])) { + for (intptr_t i = 0; i <= prev_top; i++) { + mc->pg[i] = mn->pg[i]; + mc->ki[i] = mn->ki[i]; + } + } + } + } else if (newindx >= split_indx) { + mc->pg[mc->top] = sister; + mc->ki[prev_top]++; + /* Make sure ki is still valid. */ + if (mn->pg[prev_top] != mc->pg[prev_top] && mc->ki[prev_top] >= page_numkeys(mc->pg[prev_top])) { + for (intptr_t i = 0; i <= prev_top; i++) { + mc->pg[i] = mn->pg[i]; + mc->ki[i] = mn->ki[i]; + } + } } - /* 7) still on ?-E (middle), try S-E (locked) */ - jitter4testing(false); - rc = flock(lfd, LCK_SHARED | LCK_DONTWAIT, LCK_LOWER); + /* Adjust other cursors pointing to mp and/or to parent page */ + nkeys = page_numkeys(mp); + for (MDBX_cursor *m2 = mc->txn->cursors[cursor_dbi(mc)]; m2; m2 = m2->next) { + MDBX_cursor *m3 = (mc->flags & z_inner) ? &m2->subcur->cursor : m2; + if (!is_pointed(m3) || m3 == mc) + continue; + if (foliage) { + /* sub cursors may be on different DB */ + if (m3->pg[0] != mp) + continue; + /* root split */ + for (intptr_t k = foliage; k >= 0; k--) { + m3->ki[k + 1] = m3->ki[k]; + m3->pg[k + 1] = m3->pg[k]; + } + m3->ki[0] = m3->ki[0] >= nkeys + pure_left; + m3->pg[0] = mc->pg[0]; + m3->top += 1; + } + + if (m3->top >= mc->top && m3->pg[mc->top] == mp && !pure_left) { + if (m3->ki[mc->top] >= newindx) + m3->ki[mc->top] += !(naf & MDBX_SPLIT_REPLACE); + if (m3->ki[mc->top] >= nkeys) { + m3->pg[mc->top] = sister; + cASSERT(mc, m3->ki[mc->top] >= nkeys); + m3->ki[mc->top] -= (indx_t)nkeys; + for (intptr_t i = 0; i < mc->top; i++) { + m3->ki[i] = mn->ki[i]; + m3->pg[i] = mn->pg[i]; + } + } + } else if (!did_split_parent && m3->top >= prev_top && m3->pg[prev_top] == mc->pg[prev_top] && + m3->ki[prev_top] >= mc->ki[prev_top]) { + m3->ki[prev_top]++; /* also for the `pure-left` case */ + } + if (inner_pointed(m3) && is_leaf(mp)) + cursor_inner_refresh(m3, m3->pg[mc->top], m3->ki[mc->top]); + } + TRACE("mp #%u left: %zd, sister #%u left: %zd", mp->pgno, page_room(mp), sister->pgno, page_room(sister)); - jitter4testing(false); - if (rc != MDBX_SUCCESS) - ERROR("%s, err %u", "?-E(middle) >> S-E(locked)", rc); +done: + if (tmp_ki_copy) + page_shadow_release(env, tmp_ki_copy, 1); - /* 8) now on S-E (locked) or still on ?-E (middle), - * transition to S-? (used) or ?-? (free) */ - int err = funlock(lfd, LCK_UPPER); - if (err != MDBX_SUCCESS) - mdbx_panic("%s(%s) failed: err %u", __func__, - "X-E(locked/middle) >> X-?(used/free)", err); + if (unlikely(rc != MDBX_SUCCESS)) + mc->txn->flags |= MDBX_TXN_ERROR; + else { + if (AUDIT_ENABLED()) + rc = cursor_validate_updating(mc); + if (unlikely(naf & MDBX_RESERVE)) { + node_t *node = page_node(mc->pg[mc->top], mc->ki[mc->top]); + if (!(node_flags(node) & N_BIG)) + newdata->iov_base = node_data(node); + } +#if MDBX_ENABLE_PGOP_STAT + env->lck->pgops.split.weak += 1; +#endif /* MDBX_ENABLE_PGOP_STAT */ + } - /* 9) now on S-? (used, DONE) or ?-? (free, FAILURE) */ + DEBUG("<< mp #%u, rc %d", mp->pgno, rc); return rc; } -MDBX_INTERNAL_FUNC int osal_lck_seize(MDBX_env *env) { - const HANDLE fd4data = - env->me_overlapped_fd ? env->me_overlapped_fd : env->me_lazy_fd; - assert(fd4data != INVALID_HANDLE_VALUE); - if (env->me_flags & MDBX_EXCLUSIVE) - return MDBX_RESULT_TRUE /* nope since files were must be opened - non-shareable */ - ; +int tree_propagate_key(MDBX_cursor *mc, const MDBX_val *key) { + page_t *mp; + node_t *node; + size_t len; + ptrdiff_t delta, ksize, oksize; + intptr_t ptr, i, nkeys, indx; + DKBUF_DEBUG; - if (env->me_lfd == INVALID_HANDLE_VALUE) { - /* LY: without-lck mode (e.g. on read-only filesystem) */ - jitter4testing(false); - int rc = flock_data(env, LCK_SHARED | LCK_DONTWAIT, DXB_WHOLE); - if (rc != MDBX_SUCCESS) - ERROR("%s, err %u", "without-lck", rc); - return rc; - } + cASSERT(mc, cursor_is_tracked(mc)); + indx = mc->ki[mc->top]; + mp = mc->pg[mc->top]; + node = page_node(mp, indx); + ptr = mp->entries[indx]; +#if MDBX_DEBUG + MDBX_val k2; + k2.iov_base = node_key(node); + k2.iov_len = node_ks(node); + DEBUG("update key %zi (offset %zu) [%s] to [%s] on page %" PRIaPGNO, indx, ptr, DVAL_DEBUG(&k2), DKEY_DEBUG(key), + mp->pgno); +#endif /* MDBX_DEBUG */ - int rc = internal_seize_lck(env->me_lfd); - jitter4testing(false); - if (rc == MDBX_RESULT_TRUE && (env->me_flags & MDBX_RDONLY) == 0) { - /* Check that another process don't operates in without-lck mode. - * Doing such check by exclusive locking the body-part of db. Should be - * noted: - * - we need an exclusive lock for do so; - * - we can't lock meta-pages, otherwise other process could get an error - * while opening db in valid (non-conflict) mode. */ - int err = flock_data(env, LCK_EXCLUSIVE | LCK_DONTWAIT, DXB_WHOLE); - if (err != MDBX_SUCCESS) { - ERROR("%s, err %u", "lock-against-without-lck", err); - jitter4testing(false); - lck_unlock(env); + /* Sizes must be 2-byte aligned. */ + ksize = EVEN_CEIL(key->iov_len); + oksize = EVEN_CEIL(node_ks(node)); + delta = ksize - oksize; + + /* Shift node contents if EVEN_CEIL(key length) changed. */ + if (delta) { + if (delta > (int)page_room(mp)) { + /* not enough space left, do a delete and split */ + DEBUG("Not enough room, delta = %zd, splitting...", delta); + pgno_t pgno = node_pgno(node); + node_del(mc, 0); + int err = page_split(mc, key, nullptr, pgno, MDBX_SPLIT_REPLACE); + if (err == MDBX_SUCCESS && AUDIT_ENABLED()) + err = cursor_validate_updating(mc); return err; } - jitter4testing(false); - err = funlock(fd4data, DXB_WHOLE); - if (err != MDBX_SUCCESS) - mdbx_panic("%s(%s) failed: err %u", __func__, - "unlock-against-without-lck", err); - } - - return rc; -} -MDBX_INTERNAL_FUNC int osal_lck_downgrade(MDBX_env *env) { - const HANDLE fd4data = - env->me_overlapped_fd ? env->me_overlapped_fd : env->me_lazy_fd; - /* Transite from exclusive-write state (E-E) to used (S-?) */ - assert(fd4data != INVALID_HANDLE_VALUE); - assert(env->me_lfd != INVALID_HANDLE_VALUE); + nkeys = page_numkeys(mp); + for (i = 0; i < nkeys; i++) { + if (mp->entries[i] <= ptr) { + cASSERT(mc, mp->entries[i] >= delta); + mp->entries[i] -= (indx_t)delta; + } + } - if (env->me_flags & MDBX_EXCLUSIVE) - return MDBX_SUCCESS /* nope since files were must be opened non-shareable */ - ; - /* 1) now at E-E (exclusive-write), transition to ?_E (middle) */ - int rc = funlock(env->me_lfd, LCK_LOWER); - if (rc != MDBX_SUCCESS) - mdbx_panic("%s(%s) failed: err %u", __func__, - "E-E(exclusive-write) >> ?-E(middle)", rc); + void *const base = ptr_disp(mp, mp->upper + PAGEHDRSZ); + len = ptr - mp->upper + NODESIZE; + memmove(ptr_disp(base, -delta), base, len); + cASSERT(mc, mp->upper >= delta); + mp->upper -= (indx_t)delta; - /* 2) now at ?-E (middle), transition to S-E (locked) */ - rc = flock(env->me_lfd, LCK_SHARED | LCK_DONTWAIT, LCK_LOWER); - if (rc != MDBX_SUCCESS) { - /* 3) something went wrong, give up */; - ERROR("%s, err %u", "?-E(middle) >> S-E(locked)", rc); - return rc; + node = page_node(mp, indx); } - /* 4) got S-E (locked), continue transition to S-? (used) */ - rc = funlock(env->me_lfd, LCK_UPPER); - if (rc != MDBX_SUCCESS) - mdbx_panic("%s(%s) failed: err %u", __func__, "S-E(locked) >> S-?(used)", - rc); + /* But even if no shift was needed, update ksize */ + node_set_ks(node, key->iov_len); - return MDBX_SUCCESS /* 5) now at S-? (used), done */; + if (likely(key->iov_len /* to avoid UBSAN traps*/ != 0)) + memcpy(node_key(node), key->iov_base, key->iov_len); + return MDBX_SUCCESS; } +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \note Please refer to the COPYRIGHT file for explanations license change, +/// credits and acknowledgments. +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 -MDBX_INTERNAL_FUNC int mdbx_lck_upgrade(MDBX_env *env) { - /* Transite from used state (S-?) to exclusive-write (E-E) */ - assert(env->me_lfd != INVALID_HANDLE_VALUE); - - if (env->me_flags & MDBX_EXCLUSIVE) - return MDBX_SUCCESS /* nope since files were must be opened non-shareable */ - ; - - /* 1) now on S-? (used), try S-E (locked) */ - jitter4testing(false); - int rc = flock(env->me_lfd, LCK_EXCLUSIVE | LCK_DONTWAIT, LCK_UPPER); - if (rc != MDBX_SUCCESS) { - /* 2) something went wrong, give up */; - VERBOSE("%s, err %u", "S-?(used) >> S-E(locked)", rc); - return rc; - } - - /* 3) now on S-E (locked), transition to ?-E (middle) */ - rc = funlock(env->me_lfd, LCK_LOWER); - if (rc != MDBX_SUCCESS) - mdbx_panic("%s(%s) failed: err %u", __func__, "S-E(locked) >> ?-E(middle)", - rc); +/* Search for the lowest key under the current branch page. + * This just bypasses a numkeys check in the current page + * before calling tree_search_finalize(), because the callers + * are all in situations where the current page is known to + * be underfilled. */ +__hot int tree_search_lowest(MDBX_cursor *mc) { + cASSERT(mc, mc->top >= 0); + page_t *mp = mc->pg[mc->top]; + cASSERT(mc, is_branch(mp)); - /* 4) now on ?-E (middle), try E-E (exclusive-write) */ - jitter4testing(false); - rc = flock(env->me_lfd, LCK_EXCLUSIVE | LCK_DONTWAIT, LCK_LOWER); - if (rc != MDBX_SUCCESS) { - /* 5) something went wrong, give up */; - VERBOSE("%s, err %u", "?-E(middle) >> E-E(exclusive-write)", rc); - return rc; - } + node_t *node = page_node(mp, 0); + int err = page_get(mc, node_pgno(node), &mp, mp->txnid); + if (unlikely(err != MDBX_SUCCESS)) + return err; - return MDBX_SUCCESS /* 6) now at E-E (exclusive-write), done */; + mc->ki[mc->top] = 0; + err = cursor_push(mc, mp, 0); + if (unlikely(err != MDBX_SUCCESS)) + return err; + return tree_search_finalize(mc, nullptr, Z_FIRST); } -MDBX_INTERNAL_FUNC int osal_lck_init(MDBX_env *env, - MDBX_env *inprocess_neighbor, - int global_uniqueness_flag) { - (void)env; - (void)inprocess_neighbor; - (void)global_uniqueness_flag; - if (mdbx_SetFileIoOverlappedRange && !(env->me_flags & MDBX_RDONLY)) { - HANDLE token = INVALID_HANDLE_VALUE; - TOKEN_PRIVILEGES privileges; - privileges.PrivilegeCount = 1; - privileges.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED; - if (!OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES, - &token) || - !LookupPrivilegeValue(NULL, SE_LOCK_MEMORY_NAME, - &privileges.Privileges[0].Luid) || - !AdjustTokenPrivileges(token, FALSE, &privileges, sizeof(privileges), - nullptr, nullptr) || - GetLastError() != ERROR_SUCCESS) - mdbx_SetFileIoOverlappedRange = NULL; - - if (token != INVALID_HANDLE_VALUE) - CloseHandle(token); +__hot int tree_search(MDBX_cursor *mc, const MDBX_val *key, int flags) { + int err; + if (unlikely(mc->txn->flags & MDBX_TXN_BLOCKED)) { + DEBUG("%s", "transaction has failed, must abort"); + err = MDBX_BAD_TXN; + bailout: + be_poor(mc); + return err; } - return MDBX_SUCCESS; -} -MDBX_INTERNAL_FUNC int osal_lck_destroy(MDBX_env *env, - MDBX_env *inprocess_neighbor) { - /* LY: should unmap before releasing the locks to avoid race condition and - * STATUS_USER_MAPPED_FILE/ERROR_USER_MAPPED_FILE */ - if (env->me_map) - osal_munmap(&env->me_dxb_mmap); - if (env->me_lck_mmap.lck) { - const bool synced = env->me_lck_mmap.lck->mti_unsynced_pages.weak == 0; - osal_munmap(&env->me_lck_mmap); - if (synced && !inprocess_neighbor && env->me_lfd != INVALID_HANDLE_VALUE && - mdbx_lck_upgrade(env) == MDBX_SUCCESS) - /* this will fail if LCK is used/mmapped by other process(es) */ - osal_ftruncate(env->me_lfd, 0); + const size_t dbi = cursor_dbi(mc); + if (unlikely(*cursor_dbi_state(mc) & DBI_STALE)) { + err = tbl_fetch(mc->txn, dbi); + if (unlikely(err != MDBX_SUCCESS)) + goto bailout; } - lck_unlock(env); - return MDBX_SUCCESS; -} -/*----------------------------------------------------------------------------*/ -/* reader checking (by pid) */ + const pgno_t root = mc->tree->root; + if (unlikely(root == P_INVALID)) { + DEBUG("%s", "tree is empty"); + cASSERT(mc, is_poor(mc)); + return MDBX_NOTFOUND; + } -MDBX_INTERNAL_FUNC int osal_rpid_set(MDBX_env *env) { - (void)env; - return MDBX_SUCCESS; -} + cASSERT(mc, root >= NUM_METAS && root < mc->txn->geo.first_unallocated); + if (mc->top < 0 || mc->pg[0]->pgno != root) { + txnid_t pp_txnid = mc->tree->mod_txnid; + pp_txnid = /* tree->mod_txnid maybe zero in a legacy DB */ pp_txnid ? pp_txnid : mc->txn->txnid; + if ((mc->txn->flags & MDBX_TXN_RDONLY) == 0) { + MDBX_txn *scan = mc->txn; + do + if ((scan->flags & MDBX_TXN_DIRTY) && (dbi == MAIN_DBI || (scan->dbi_state[dbi] & DBI_DIRTY))) { + /* После коммита вложенных тразакций может быть mod_txnid > front */ + pp_txnid = scan->front_txnid; + break; + } + while (unlikely((scan = scan->parent) != nullptr)); + } + err = page_get(mc, root, &mc->pg[0], pp_txnid); + if (unlikely(err != MDBX_SUCCESS)) + goto bailout; + } -MDBX_INTERNAL_FUNC int osal_rpid_clear(MDBX_env *env) { - (void)env; - return MDBX_SUCCESS; -} + mc->top = 0; + mc->ki[0] = (flags & Z_LAST) ? page_numkeys(mc->pg[0]) - 1 : 0; + DEBUG("db %d root page %" PRIaPGNO " has flags 0x%X", cursor_dbi_dbg(mc), root, mc->pg[0]->flags); -/* Checks reader by pid. - * - * Returns: - * MDBX_RESULT_TRUE, if pid is live (unable to acquire lock) - * MDBX_RESULT_FALSE, if pid is dead (lock acquired) - * or otherwise the errcode. */ -MDBX_INTERNAL_FUNC int osal_rpid_check(MDBX_env *env, uint32_t pid) { - (void)env; - HANDLE hProcess = OpenProcess(SYNCHRONIZE, FALSE, pid); - int rc; - if (likely(hProcess)) { - rc = WaitForSingleObject(hProcess, 0); - if (unlikely(rc == (int)WAIT_FAILED)) - rc = (int)GetLastError(); - CloseHandle(hProcess); - } else { - rc = (int)GetLastError(); + if (flags & Z_MODIFY) { + err = page_touch(mc); + if (unlikely(err != MDBX_SUCCESS)) + goto bailout; } - switch (rc) { - case ERROR_INVALID_PARAMETER: - /* pid seems invalid */ - return MDBX_RESULT_FALSE; - case WAIT_OBJECT_0: - /* process just exited */ - return MDBX_RESULT_FALSE; - case ERROR_ACCESS_DENIED: - /* The ERROR_ACCESS_DENIED would be returned for CSRSS-processes, etc. - * assume pid exists */ - return MDBX_RESULT_TRUE; - case WAIT_TIMEOUT: - /* pid running */ - return MDBX_RESULT_TRUE; - default: - /* failure */ - return rc; - } + if (flags & Z_ROOTONLY) + return MDBX_SUCCESS; + + return tree_search_finalize(mc, key, flags); } -//---------------------------------------------------------------------------- -// Stub for slim read-write lock -// Copyright (C) 1995-2002 Brad Wilson +__hot __noinline int tree_search_finalize(MDBX_cursor *mc, const MDBX_val *key, int flags) { + cASSERT(mc, !is_poor(mc)); + DKBUF_DEBUG; + int err; + page_t *mp = mc->pg[mc->top]; + intptr_t ki = (flags & Z_FIRST) ? 0 : page_numkeys(mp) - 1; + while (is_branch(mp)) { + DEBUG("branch page %" PRIaPGNO " has %zu keys", mp->pgno, page_numkeys(mp)); + cASSERT(mc, page_numkeys(mp) > 1); + DEBUG("found index 0 to page %" PRIaPGNO, node_pgno(page_node(mp, 0))); + + if ((flags & (Z_FIRST | Z_LAST)) == 0) { + const struct node_search_result nsr = node_search(mc, key); + if (likely(nsr.node)) + ki = mc->ki[mc->top] + (intptr_t)nsr.exact - 1; + DEBUG("following index %zu for key [%s]", ki, DKEY_DEBUG(key)); + } -static void WINAPI stub_srwlock_Init(osal_srwlock_t *srwl) { - srwl->readerCount = srwl->writerCount = 0; -} + err = page_get(mc, node_pgno(page_node(mp, ki)), &mp, mp->txnid); + if (unlikely(err != MDBX_SUCCESS)) + goto bailout; -static void WINAPI stub_srwlock_AcquireShared(osal_srwlock_t *srwl) { - while (true) { - assert(srwl->writerCount >= 0 && srwl->readerCount >= 0); + mc->ki[mc->top] = (indx_t)ki; + ki = (flags & Z_FIRST) ? 0 : page_numkeys(mp) - 1; + err = cursor_push(mc, mp, ki); + if (unlikely(err != MDBX_SUCCESS)) + goto bailout; - // If there's a writer already, spin without unnecessarily - // interlocking the CPUs - if (srwl->writerCount != 0) { - SwitchToThread(); - continue; + if (flags & Z_MODIFY) { + err = page_touch(mc); + if (unlikely(err != MDBX_SUCCESS)) + goto bailout; + mp = mc->pg[mc->top]; } + } - // Add to the readers list - _InterlockedIncrement(&srwl->readerCount); + if (!MDBX_DISABLE_VALIDATION && unlikely(!check_leaf_type(mc, mp))) { + ERROR("unexpected leaf-page #%" PRIaPGNO " type 0x%x seen by cursor", mp->pgno, mp->flags); + err = MDBX_CORRUPTED; + bailout: + be_poor(mc); + return err; + } - // Check for writers again (we may have been preempted). If - // there are no writers writing or waiting, then we're done. - if (srwl->writerCount == 0) - break; + DEBUG("found leaf page %" PRIaPGNO " for key [%s]", mp->pgno, DKEY_DEBUG(key)); + /* Логически верно, но (в текущем понимании) нет необходимости. + Однако, стоит ещё по-проверять/по-тестировать. + Возможно есть сценарий, в котором очистка флагов всё-таки требуется. - // Remove from the readers list, spin, try again - _InterlockedDecrement(&srwl->readerCount); - SwitchToThread(); - } + be_filled(mc); */ + return MDBX_SUCCESS; } +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 -static void WINAPI stub_srwlock_ReleaseShared(osal_srwlock_t *srwl) { - assert(srwl->readerCount > 0); - _InterlockedDecrement(&srwl->readerCount); +static inline size_t txl_size2bytes(const size_t size) { + assert(size > 0 && size <= txl_max * 2); + size_t bytes = + ceil_powerof2(MDBX_ASSUME_MALLOC_OVERHEAD + sizeof(txnid_t) * (size + 2), txl_granulate * sizeof(txnid_t)) - + MDBX_ASSUME_MALLOC_OVERHEAD; + return bytes; } -static void WINAPI stub_srwlock_AcquireExclusive(osal_srwlock_t *srwl) { - while (true) { - assert(srwl->writerCount >= 0 && srwl->readerCount >= 0); +static inline size_t txl_bytes2size(const size_t bytes) { + size_t size = bytes / sizeof(txnid_t); + assert(size > 2 && size <= txl_max * 2); + return size - 2; +} - // If there's a writer already, spin without unnecessarily - // interlocking the CPUs - if (srwl->writerCount != 0) { - SwitchToThread(); - continue; - } +txl_t txl_alloc(void) { + size_t bytes = txl_size2bytes(txl_initial); + txl_t txl = osal_malloc(bytes); + if (likely(txl)) { +#ifdef osal_malloc_usable_size + bytes = osal_malloc_usable_size(txl); +#endif /* osal_malloc_usable_size */ + txl[0] = txl_bytes2size(bytes); + assert(txl[0] >= txl_initial); + txl += 1; + *txl = 0; + } + return txl; +} - // See if we can become the writer (expensive, because it inter- - // locks the CPUs, so writing should be an infrequent process) - if (_InterlockedExchange(&srwl->writerCount, 1) == 0) - break; +void txl_free(txl_t txl) { + if (likely(txl)) + osal_free(txl - 1); +} + +static int txl_reserve(txl_t __restrict *__restrict ptxl, const size_t wanna) { + const size_t allocated = (size_t)MDBX_PNL_ALLOCLEN(*ptxl); + assert(MDBX_PNL_GETSIZE(*ptxl) <= txl_max && MDBX_PNL_ALLOCLEN(*ptxl) >= MDBX_PNL_GETSIZE(*ptxl)); + if (likely(allocated >= wanna)) + return MDBX_SUCCESS; + + if (unlikely(wanna > /* paranoia */ txl_max)) { + ERROR("TXL too long (%zu > %zu)", wanna, (size_t)txl_max); + return MDBX_TXN_FULL; } - // Now we're the writer, but there may be outstanding readers. - // Spin until there aren't any more; new readers will wait now - // that we're the writer. - while (srwl->readerCount != 0) { - assert(srwl->writerCount >= 0 && srwl->readerCount >= 0); - SwitchToThread(); + const size_t size = (wanna + wanna - allocated < txl_max) ? wanna + wanna - allocated : txl_max; + size_t bytes = txl_size2bytes(size); + txl_t txl = osal_realloc(*ptxl - 1, bytes); + if (likely(txl)) { +#ifdef osal_malloc_usable_size + bytes = osal_malloc_usable_size(txl); +#endif /* osal_malloc_usable_size */ + *txl = txl_bytes2size(bytes); + assert(*txl >= wanna); + *ptxl = txl + 1; + return MDBX_SUCCESS; } + return MDBX_ENOMEM; } -static void WINAPI stub_srwlock_ReleaseExclusive(osal_srwlock_t *srwl) { - assert(srwl->writerCount == 1 && srwl->readerCount >= 0); - srwl->writerCount = 0; +static __always_inline int __must_check_result txl_need(txl_t __restrict *__restrict ptxl, size_t num) { + assert(MDBX_PNL_GETSIZE(*ptxl) <= txl_max && MDBX_PNL_ALLOCLEN(*ptxl) >= MDBX_PNL_GETSIZE(*ptxl)); + assert(num <= PAGELIST_LIMIT); + const size_t wanna = (size_t)MDBX_PNL_GETSIZE(*ptxl) + num; + return likely(MDBX_PNL_ALLOCLEN(*ptxl) >= wanna) ? MDBX_SUCCESS : txl_reserve(ptxl, wanna); } -static uint64_t WINAPI stub_GetTickCount64(void) { - LARGE_INTEGER Counter, Frequency; - return (QueryPerformanceFrequency(&Frequency) && - QueryPerformanceCounter(&Counter)) - ? Counter.QuadPart * 1000ul / Frequency.QuadPart - : 0; +static __always_inline void txl_xappend(txl_t __restrict txl, txnid_t id) { + assert(MDBX_PNL_GETSIZE(txl) < MDBX_PNL_ALLOCLEN(txl)); + txl[0] += 1; + MDBX_PNL_LAST(txl) = id; } -/*----------------------------------------------------------------------------*/ +#define TXNID_SORT_CMP(first, last) ((first) > (last)) +SORT_IMPL(txnid_sort, false, txnid_t, TXNID_SORT_CMP) +void txl_sort(txl_t txl) { txnid_sort(MDBX_PNL_BEGIN(txl), MDBX_PNL_END(txl)); } -#ifndef xMDBX_ALLOY -osal_srwlock_t_function osal_srwlock_Init, osal_srwlock_AcquireShared, - osal_srwlock_ReleaseShared, osal_srwlock_AcquireExclusive, - osal_srwlock_ReleaseExclusive; - -MDBX_NtExtendSection mdbx_NtExtendSection; -MDBX_GetFileInformationByHandleEx mdbx_GetFileInformationByHandleEx; -MDBX_GetVolumeInformationByHandleW mdbx_GetVolumeInformationByHandleW; -MDBX_GetFinalPathNameByHandleW mdbx_GetFinalPathNameByHandleW; -MDBX_SetFileInformationByHandle mdbx_SetFileInformationByHandle; -MDBX_NtFsControlFile mdbx_NtFsControlFile; -MDBX_PrefetchVirtualMemory mdbx_PrefetchVirtualMemory; -MDBX_GetTickCount64 mdbx_GetTickCount64; -MDBX_RegGetValueA mdbx_RegGetValueA; -MDBX_SetFileIoOverlappedRange mdbx_SetFileIoOverlappedRange; -#endif /* xMDBX_ALLOY */ +int __must_check_result txl_append(txl_t __restrict *ptxl, txnid_t id) { + if (unlikely(MDBX_PNL_GETSIZE(*ptxl) == MDBX_PNL_ALLOCLEN(*ptxl))) { + int rc = txl_need(ptxl, txl_granulate); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; + } + txl_xappend(*ptxl, id); + return MDBX_SUCCESS; +} -#if __GNUC_PREREQ(8, 0) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wcast-function-type" -#endif /* GCC/MINGW */ +__hot bool txl_contain(const txl_t txl, txnid_t id) { + const size_t len = MDBX_PNL_GETSIZE(txl); + for (size_t i = 1; i <= len; ++i) + if (txl[i] == id) + return true; + return false; +} +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 -static void mdbx_winnt_import(void) { -#define GET_PROC_ADDR(dll, ENTRY) \ - mdbx_##ENTRY = (MDBX_##ENTRY)GetProcAddress(dll, #ENTRY) +__hot txnid_t txn_snapshot_oldest(const MDBX_txn *const txn) { + return mvcc_shapshot_oldest(txn->env, txn->tw.troika.txnid[txn->tw.troika.prefer_steady]); +} - const HINSTANCE hNtdll = GetModuleHandleA("ntdll.dll"); - if (hNtdll) { - if (GetProcAddress(hNtdll, "wine_get_version")) { - assert(mdbx_RunningUnderWine()); - } else { - GET_PROC_ADDR(hNtdll, NtFsControlFile); - GET_PROC_ADDR(hNtdll, NtExtendSection); - assert(!mdbx_RunningUnderWine()); +void txn_done_cursors(MDBX_txn *txn, const bool merge) { + TXN_FOREACH_DBI_ALL(txn, i) { + MDBX_cursor *mc = txn->cursors[i]; + if (mc) { + txn->cursors[i] = nullptr; + do + mc = cursor_eot(mc, txn, merge); + while (mc); } } +} - const HINSTANCE hKernel32dll = GetModuleHandleA("kernel32.dll"); - if (hKernel32dll) { - GET_PROC_ADDR(hKernel32dll, GetFileInformationByHandleEx); - GET_PROC_ADDR(hKernel32dll, GetTickCount64); - if (!mdbx_GetTickCount64) - mdbx_GetTickCount64 = stub_GetTickCount64; - if (!mdbx_RunningUnderWine()) { - GET_PROC_ADDR(hKernel32dll, SetFileInformationByHandle); - GET_PROC_ADDR(hKernel32dll, GetVolumeInformationByHandleW); - GET_PROC_ADDR(hKernel32dll, GetFinalPathNameByHandleW); - GET_PROC_ADDR(hKernel32dll, PrefetchVirtualMemory); - GET_PROC_ADDR(hKernel32dll, SetFileIoOverlappedRange); - } - } - - const osal_srwlock_t_function init = - (osal_srwlock_t_function)(hKernel32dll - ? GetProcAddress(hKernel32dll, - "InitializeSRWLock") - : nullptr); - if (init != NULL) { - osal_srwlock_Init = init; - osal_srwlock_AcquireShared = (osal_srwlock_t_function)GetProcAddress( - hKernel32dll, "AcquireSRWLockShared"); - osal_srwlock_ReleaseShared = (osal_srwlock_t_function)GetProcAddress( - hKernel32dll, "ReleaseSRWLockShared"); - osal_srwlock_AcquireExclusive = (osal_srwlock_t_function)GetProcAddress( - hKernel32dll, "AcquireSRWLockExclusive"); - osal_srwlock_ReleaseExclusive = (osal_srwlock_t_function)GetProcAddress( - hKernel32dll, "ReleaseSRWLockExclusive"); - } else { - osal_srwlock_Init = stub_srwlock_Init; - osal_srwlock_AcquireShared = stub_srwlock_AcquireShared; - osal_srwlock_ReleaseShared = stub_srwlock_ReleaseShared; - osal_srwlock_AcquireExclusive = stub_srwlock_AcquireExclusive; - osal_srwlock_ReleaseExclusive = stub_srwlock_ReleaseExclusive; +int txn_write(MDBX_txn *txn, iov_ctx_t *ctx) { + tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC); + dpl_t *const dl = dpl_sort(txn); + int rc = MDBX_SUCCESS; + size_t r, w, total_npages = 0; + for (w = 0, r = 1; r <= dl->length; ++r) { + page_t *dp = dl->items[r].ptr; + if (dp->flags & P_LOOSE) { + dl->items[++w] = dl->items[r]; + continue; + } + unsigned npages = dpl_npages(dl, r); + total_npages += npages; + rc = iov_page(txn, ctx, dp, npages); + if (unlikely(rc != MDBX_SUCCESS)) + return rc; } - const HINSTANCE hAdvapi32dll = GetModuleHandleA("advapi32.dll"); - if (hAdvapi32dll) { - GET_PROC_ADDR(hAdvapi32dll, RegGetValueA); + if (!iov_empty(ctx)) { + tASSERT(txn, rc == MDBX_SUCCESS); + rc = iov_write(ctx); + } + + if (likely(rc == MDBX_SUCCESS) && ctx->fd == txn->env->lazy_fd) { + txn->env->lck->unsynced_pages.weak += total_npages; + if (!txn->env->lck->eoos_timestamp.weak) + txn->env->lck->eoos_timestamp.weak = osal_monotime(); } -#undef GET_PROC_ADDR + + txn->tw.dirtylist->pages_including_loose -= total_npages; + while (r <= dl->length) + dl->items[++w] = dl->items[r++]; + + dl->sorted = dpl_setlen(dl, w); + txn->tw.dirtyroom += r - 1 - w; + tASSERT(txn, txn->tw.dirtyroom + txn->tw.dirtylist->length == + (txn->parent ? txn->parent->tw.dirtyroom : txn->env->options.dp_limit)); + tASSERT(txn, txn->tw.dirtylist->length == txn->tw.loose_count); + tASSERT(txn, txn->tw.dirtylist->pages_including_loose == txn->tw.loose_count); + return rc; } -#if __GNUC_PREREQ(8, 0) -#pragma GCC diagnostic pop -#endif /* GCC/MINGW */ +/* Merge child txn into parent */ +void txn_merge(MDBX_txn *const parent, MDBX_txn *const txn, const size_t parent_retired_len) { + tASSERT(txn, (txn->flags & MDBX_WRITEMAP) == 0); + dpl_t *const src = dpl_sort(txn); -#endif /* Windows LCK-implementation */ -/* - * Copyright 2015-2024 Leonid Yuriev - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - */ + /* Remove refunded pages from parent's dirty list */ + dpl_t *const dst = dpl_sort(parent); + if (MDBX_ENABLE_REFUND) { + size_t n = dst->length; + while (n && dst->items[n].pgno >= parent->geo.first_unallocated) { + const unsigned npages = dpl_npages(dst, n); + page_shadow_release(txn->env, dst->items[n].ptr, npages); + --n; + } + parent->tw.dirtyroom += dst->sorted - n; + dst->sorted = dpl_setlen(dst, n); + tASSERT(parent, parent->tw.dirtyroom + parent->tw.dirtylist->length == + (parent->parent ? parent->parent->tw.dirtyroom : parent->env->options.dp_limit)); + } -#if !(defined(_WIN32) || defined(_WIN64)) /* !Windows LCK-implementation */ + /* Remove reclaimed pages from parent's dirty list */ + const pnl_t reclaimed_list = parent->tw.repnl; + dpl_sift(parent, reclaimed_list, false); + /* Move retired pages from parent's dirty & spilled list to reclaimed */ + size_t r, w, d, s, l; + for (r = w = parent_retired_len; ++r <= MDBX_PNL_GETSIZE(parent->tw.retired_pages);) { + const pgno_t pgno = parent->tw.retired_pages[r]; + const size_t di = dpl_exist(parent, pgno); + const size_t si = !di ? spill_search(parent, pgno) : 0; + unsigned npages; + const char *kind; + if (di) { + page_t *dp = dst->items[di].ptr; + tASSERT(parent, (dp->flags & ~(P_LEAF | P_DUPFIX | P_BRANCH | P_LARGE | P_SPILLED)) == 0); + npages = dpl_npages(dst, di); + page_wash(parent, di, dp, npages); + kind = "dirty"; + l = 1; + if (unlikely(npages > l)) { + /* OVERFLOW-страница могла быть переиспользована по частям. Тогда + * в retired-списке может быть только начало последовательности, + * а остаток растащен по dirty, spilled и reclaimed спискам. Поэтому + * переносим в reclaimed с проверкой на обрыв последовательности. + * В любом случае, все осколки будут учтены и отфильтрованы, т.е. если + * страница была разбита на части, то важно удалить dirty-элемент, + * а все осколки будут учтены отдельно. */ -#if MDBX_LOCKING == MDBX_LOCKING_SYSV -#include -#endif /* MDBX_LOCKING == MDBX_LOCKING_SYSV */ + /* Список retired страниц не сортирован, но для ускорения сортировки + * дополняется в соответствии с MDBX_PNL_ASCENDING */ +#if MDBX_PNL_ASCENDING + const size_t len = MDBX_PNL_GETSIZE(parent->tw.retired_pages); + while (r < len && parent->tw.retired_pages[r + 1] == pgno + l) { + ++r; + if (++l == npages) + break; + } +#else + while (w > parent_retired_len && parent->tw.retired_pages[w - 1] == pgno + l) { + --w; + if (++l == npages) + break; + } +#endif + } + } else if (unlikely(si)) { + l = npages = 1; + spill_remove(parent, si, 1); + kind = "spilled"; + } else { + parent->tw.retired_pages[++w] = pgno; + continue; + } -/*----------------------------------------------------------------------------*/ -/* global constructor/destructor */ + DEBUG("reclaim retired parent's %u -> %zu %s page %" PRIaPGNO, npages, l, kind, pgno); + int err = pnl_insert_span(&parent->tw.repnl, pgno, l); + ENSURE(txn->env, err == MDBX_SUCCESS); + } + MDBX_PNL_SETSIZE(parent->tw.retired_pages, w); -#if defined(__linux__) || defined(__gnu_linux__) + /* Filter-out parent spill list */ + if (parent->tw.spilled.list && MDBX_PNL_GETSIZE(parent->tw.spilled.list) > 0) { + const pnl_t sl = spill_purge(parent); + size_t len = MDBX_PNL_GETSIZE(sl); + if (len) { + /* Remove refunded pages from parent's spill list */ + if (MDBX_ENABLE_REFUND && MDBX_PNL_MOST(sl) >= (parent->geo.first_unallocated << 1)) { +#if MDBX_PNL_ASCENDING + size_t i = MDBX_PNL_GETSIZE(sl); + assert(MDBX_PNL_MOST(sl) == MDBX_PNL_LAST(sl)); + do { + if ((sl[i] & 1) == 0) + DEBUG("refund parent's spilled page %" PRIaPGNO, sl[i] >> 1); + i -= 1; + } while (i && sl[i] >= (parent->geo.first_unallocated << 1)); + MDBX_PNL_SETSIZE(sl, i); +#else + assert(MDBX_PNL_MOST(sl) == MDBX_PNL_FIRST(sl)); + size_t i = 0; + do { + ++i; + if ((sl[i] & 1) == 0) + DEBUG("refund parent's spilled page %" PRIaPGNO, sl[i] >> 1); + } while (i < len && sl[i + 1] >= (parent->geo.first_unallocated << 1)); + MDBX_PNL_SETSIZE(sl, len -= i); + memmove(sl + 1, sl + 1 + i, len * sizeof(sl[0])); +#endif + } + tASSERT(txn, pnl_check_allocated(sl, (size_t)parent->geo.first_unallocated << 1)); -#include + /* Remove reclaimed pages from parent's spill list */ + s = MDBX_PNL_GETSIZE(sl), r = MDBX_PNL_GETSIZE(reclaimed_list); + /* Scanning from end to begin */ + while (s && r) { + if (sl[s] & 1) { + --s; + continue; + } + const pgno_t spilled_pgno = sl[s] >> 1; + const pgno_t reclaimed_pgno = reclaimed_list[r]; + if (reclaimed_pgno != spilled_pgno) { + const bool cmp = MDBX_PNL_ORDERED(spilled_pgno, reclaimed_pgno); + s -= !cmp; + r -= cmp; + } else { + DEBUG("remove reclaimed parent's spilled page %" PRIaPGNO, reclaimed_pgno); + spill_remove(parent, s, 1); + --s; + --r; + } + } + + /* Remove anything in our dirty list from parent's spill list */ + /* Scanning spill list in descend order */ + const intptr_t step = MDBX_PNL_ASCENDING ? -1 : 1; + s = MDBX_PNL_ASCENDING ? MDBX_PNL_GETSIZE(sl) : 1; + d = src->length; + while (d && (MDBX_PNL_ASCENDING ? s > 0 : s <= MDBX_PNL_GETSIZE(sl))) { + if (sl[s] & 1) { + s += step; + continue; + } + const pgno_t spilled_pgno = sl[s] >> 1; + const pgno_t dirty_pgno_form = src->items[d].pgno; + const unsigned npages = dpl_npages(src, d); + const pgno_t dirty_pgno_to = dirty_pgno_form + npages; + if (dirty_pgno_form > spilled_pgno) { + --d; + continue; + } + if (dirty_pgno_to <= spilled_pgno) { + s += step; + continue; + } -#ifndef xMDBX_ALLOY -uint32_t linux_kernel_version; -bool mdbx_RunningOnWSL1; -#endif /* xMDBX_ALLOY */ + DEBUG("remove dirtied parent's spilled %u page %" PRIaPGNO, npages, dirty_pgno_form); + spill_remove(parent, s, 1); + s += step; + } -MDBX_EXCLUDE_FOR_GPROF -__cold static uint8_t probe_for_WSL(const char *tag) { - const char *const WSL = strstr(tag, "WSL"); - if (WSL && WSL[3] >= '2' && WSL[3] <= '9') - return WSL[3] - '0'; - const char *const wsl = strstr(tag, "wsl"); - if (wsl && wsl[3] >= '2' && wsl[3] <= '9') - return wsl[3] - '0'; - if (WSL || wsl || strcasestr(tag, "Microsoft")) - /* Expecting no new kernel within WSL1, either it will explicitly - * marked by an appropriate WSL-version hint. */ - return (linux_kernel_version < /* 4.19.x */ 0x04130000) ? 1 : 2; - return 0; -} + /* Squash deleted pagenums if we deleted any */ + spill_purge(parent); + } + } -#endif /* Linux */ + /* Remove anything in our spill list from parent's dirty list */ + if (txn->tw.spilled.list) { + tASSERT(txn, pnl_check_allocated(txn->tw.spilled.list, (size_t)parent->geo.first_unallocated << 1)); + dpl_sift(parent, txn->tw.spilled.list, true); + tASSERT(parent, parent->tw.dirtyroom + parent->tw.dirtylist->length == + (parent->parent ? parent->parent->tw.dirtyroom : parent->env->options.dp_limit)); + } -#ifdef ENABLE_GPROF -extern void _mcleanup(void); -extern void monstartup(unsigned long, unsigned long); -extern void _init(void); -extern void _fini(void); -extern void __gmon_start__(void) __attribute__((__weak__)); -#endif /* ENABLE_GPROF */ + /* Find length of merging our dirty list with parent's and release + * filter-out pages */ + for (l = 0, d = dst->length, s = src->length; d > 0 && s > 0;) { + page_t *sp = src->items[s].ptr; + tASSERT(parent, (sp->flags & ~(P_LEAF | P_DUPFIX | P_BRANCH | P_LARGE | P_LOOSE | P_SPILLED)) == 0); + const unsigned s_npages = dpl_npages(src, s); + const pgno_t s_pgno = src->items[s].pgno; -MDBX_EXCLUDE_FOR_GPROF -__cold static __attribute__((__constructor__)) void -mdbx_global_constructor(void) { -#ifdef ENABLE_GPROF - if (!&__gmon_start__) - monstartup((uintptr_t)&_init, (uintptr_t)&_fini); -#endif /* ENABLE_GPROF */ + page_t *dp = dst->items[d].ptr; + tASSERT(parent, (dp->flags & ~(P_LEAF | P_DUPFIX | P_BRANCH | P_LARGE | P_SPILLED)) == 0); + const unsigned d_npages = dpl_npages(dst, d); + const pgno_t d_pgno = dst->items[d].pgno; -#if defined(__linux__) || defined(__gnu_linux__) - struct utsname buffer; - if (uname(&buffer) == 0) { - int i = 0; - char *p = buffer.release; - while (*p && i < 4) { - if (*p >= '0' && *p <= '9') { - long number = strtol(p, &p, 10); - if (number > 0) { - if (number > 255) - number = 255; - linux_kernel_version += number << (24 - i * 8); - } - ++i; - } else { - ++p; + if (d_pgno >= s_pgno + s_npages) { + --d; + ++l; + } else if (d_pgno + d_npages <= s_pgno) { + if (sp->flags != P_LOOSE) { + sp->txnid = parent->front_txnid; + sp->flags &= ~P_SPILLED; } + --s; + ++l; + } else { + dst->items[d--].ptr = nullptr; + page_shadow_release(txn->env, dp, d_npages); } - /* "Official" way of detecting WSL1 but not WSL2 - * https://github.com/Microsoft/WSL/issues/423#issuecomment-221627364 - * - * WARNING: False negative detection of WSL1 will result in DATA LOSS! - * So, the REQUIREMENTS for this code: - * 1. MUST detect WSL1 without false-negatives. - * 2. DESIRABLE detect WSL2 but without the risk of violating the first. */ - mdbx_RunningOnWSL1 = probe_for_WSL(buffer.version) == 1 || - probe_for_WSL(buffer.sysname) == 1 || - probe_for_WSL(buffer.release) == 1; } -#endif /* Linux */ + assert(dst->sorted == dst->length); + tASSERT(parent, dst->detent >= l + d + s); + dst->sorted = l + d + s; /* the merged length */ - global_ctor(); -} + while (s > 0) { + page_t *sp = src->items[s].ptr; + tASSERT(parent, (sp->flags & ~(P_LEAF | P_DUPFIX | P_BRANCH | P_LARGE | P_LOOSE | P_SPILLED)) == 0); + if (sp->flags != P_LOOSE) { + sp->txnid = parent->front_txnid; + sp->flags &= ~P_SPILLED; + } + --s; + } -MDBX_EXCLUDE_FOR_GPROF -__cold static __attribute__((__destructor__)) void -mdbx_global_destructor(void) { - global_dtor(); -#ifdef ENABLE_GPROF - if (!&__gmon_start__) - _mcleanup(); -#endif /* ENABLE_GPROF */ -} + /* Merge our dirty list into parent's, i.e. merge(dst, src) -> dst */ + if (dst->sorted >= dst->length) { + /* from end to begin with dst extending */ + for (l = dst->sorted, s = src->length, d = dst->length; s > 0 && d > 0;) { + if (unlikely(l <= d)) { + /* squash to get a gap of free space for merge */ + for (r = w = 1; r <= d; ++r) + if (dst->items[r].ptr) { + if (w != r) { + dst->items[w] = dst->items[r]; + dst->items[r].ptr = nullptr; + } + ++w; + } + VERBOSE("squash to begin for extending-merge %zu -> %zu", d, w - 1); + d = w - 1; + continue; + } + assert(l > d); + if (dst->items[d].ptr) { + dst->items[l--] = (dst->items[d].pgno > src->items[s].pgno) ? dst->items[d--] : src->items[s--]; + } else + --d; + } + if (s > 0) { + assert(l == s); + while (d > 0) { + assert(dst->items[d].ptr == nullptr); + --d; + } + do { + assert(l > 0); + dst->items[l--] = src->items[s--]; + } while (s > 0); + } else { + assert(l == d); + while (l > 0) { + assert(dst->items[l].ptr != nullptr); + --l; + } + } + } else { + /* from begin to end with shrinking (a lot of new large/overflow pages) */ + for (l = s = d = 1; s <= src->length && d <= dst->length;) { + if (unlikely(l >= d)) { + /* squash to get a gap of free space for merge */ + for (r = w = dst->length; r >= d; --r) + if (dst->items[r].ptr) { + if (w != r) { + dst->items[w] = dst->items[r]; + dst->items[r].ptr = nullptr; + } + --w; + } + VERBOSE("squash to end for shrinking-merge %zu -> %zu", d, w + 1); + d = w + 1; + continue; + } + assert(l < d); + if (dst->items[d].ptr) { + dst->items[l++] = (dst->items[d].pgno < src->items[s].pgno) ? dst->items[d++] : src->items[s++]; + } else + ++d; + } + if (s <= src->length) { + assert(dst->sorted - l == src->length - s); + while (d <= dst->length) { + assert(dst->items[d].ptr == nullptr); + --d; + } + do { + assert(l <= dst->sorted); + dst->items[l++] = src->items[s++]; + } while (s <= src->length); + } else { + assert(dst->sorted - l == dst->length - d); + while (l <= dst->sorted) { + assert(l <= d && d <= dst->length && dst->items[d].ptr); + dst->items[l++] = dst->items[d++]; + } + } + } + parent->tw.dirtyroom -= dst->sorted - dst->length; + assert(parent->tw.dirtyroom <= parent->env->options.dp_limit); + dpl_setlen(dst, dst->sorted); + parent->tw.dirtylru = txn->tw.dirtylru; -/*----------------------------------------------------------------------------*/ -/* lck */ + /* В текущем понимании выгоднее пересчитать кол-во страниц, + * чем подмешивать лишние ветвления и вычисления в циклы выше. */ + dst->pages_including_loose = 0; + for (r = 1; r <= dst->length; ++r) + dst->pages_including_loose += dpl_npages(dst, r); -/* Описание реализации блокировок для POSIX & Linux: - * - * lck-файл отображается в память, в нём организуется таблица читателей и - * размещаются совместно используемые posix-мьютексы (futex). Посредством - * этих мьютексов (см struct MDBX_lockinfo) реализуются: - * - Блокировка таблицы читателей для регистрации, - * т.е. функции osal_rdt_lock() и osal_rdt_unlock(). - * - Блокировка БД для пишущих транзакций, - * т.е. функции mdbx_txn_lock() и mdbx_txn_unlock(). - * - * Остальной функционал реализуется отдельно посредством файловых блокировок: - * - Первоначальный захват БД в режиме exclusive/shared и последующий перевод - * в операционный режим, функции osal_lck_seize() и osal_lck_downgrade(). - * - Проверка присутствие процессов-читателей, - * т.е. функции osal_rpid_set(), osal_rpid_clear() и osal_rpid_check(). - * - * Для блокировки файлов используется fcntl(F_SETLK), так как: - * - lockf() оперирует только эксклюзивной блокировкой и требует - * открытия файла в RW-режиме. - * - flock() не гарантирует атомарности при смене блокировок - * и оперирует только всем файлом целиком. - * - Для контроля процессов-читателей используются однобайтовые - * range-блокировки lck-файла посредством fcntl(F_SETLK). При этом - * в качестве позиции используется pid процесса-читателя. - * - Для первоначального захвата и shared/exclusive выполняется блокировка - * основного файла БД и при успехе lck-файла. - * - * ---------------------------------------------------------------------------- - * УДЕРЖИВАЕМЫЕ БЛОКИРОВКИ В ЗАВИСИМОСТИ ОТ РЕЖИМА И СОСТОЯНИЯ - * - * Эксклюзивный режим без lck-файла: - * = заблокирован весь dxb-файл посредством F_RDLCK или F_WRLCK, - * в зависимости от MDBX_RDONLY. - * - * Не-операционный режим на время пере-инициализации и разрушении lck-файла: - * = F_WRLCK блокировка первого байта lck-файла, другие процессы ждут её - * снятия при получении F_RDLCK через F_SETLKW. - * - блокировки dxb-файла могут меняться до снятие эксклюзивной блокировки - * lck-файла: - * + для НЕ-эксклюзивного режима блокировка pid-байта в dxb-файле - * посредством F_RDLCK или F_WRLCK, в зависимости от MDBX_RDONLY. - * + для ЭКСКЛЮЗИВНОГО режима блокировка всего dxb-файла - * посредством F_RDLCK или F_WRLCK, в зависимости от MDBX_RDONLY. - * - * ОПЕРАЦИОННЫЙ режим с lck-файлом: - * = F_RDLCK блокировка первого байта lck-файла, другие процессы не могут - * получить F_WRLCK и таким образом видят что БД используется. - * + F_WRLCK блокировка pid-байта в clk-файле после первой транзакции чтения. - * + для НЕ-эксклюзивного режима блокировка pid-байта в dxb-файле - * посредством F_RDLCK или F_WRLCK, в зависимости от MDBX_RDONLY. - * + для ЭКСКЛЮЗИВНОГО режима блокировка pid-байта всего dxb-файла - * посредством F_RDLCK или F_WRLCK, в зависимости от MDBX_RDONLY. - */ + tASSERT(parent, dpl_check(parent)); + dpl_free(txn); -#if MDBX_USE_OFDLOCKS -static int op_setlk, op_setlkw, op_getlk; -__cold static void choice_fcntl(void) { - assert(!op_setlk && !op_setlkw && !op_getlk); - if ((runtime_flags & MDBX_DBG_LEGACY_MULTIOPEN) == 0 -#if defined(__linux__) || defined(__gnu_linux__) - && linux_kernel_version > - 0x030f0000 /* OFD locks are available since 3.15, but engages here - only for 3.16 and later kernels (i.e. LTS) because - of reliability reasons */ -#endif /* linux */ - ) { - op_setlk = MDBX_F_OFD_SETLK; - op_setlkw = MDBX_F_OFD_SETLKW; - op_getlk = MDBX_F_OFD_GETLK; - return; + if (txn->tw.spilled.list) { + if (parent->tw.spilled.list) { + /* Must not fail since space was preserved above. */ + pnl_merge(parent->tw.spilled.list, txn->tw.spilled.list); + pnl_free(txn->tw.spilled.list); + } else { + parent->tw.spilled.list = txn->tw.spilled.list; + parent->tw.spilled.least_removed = txn->tw.spilled.least_removed; + } + tASSERT(parent, dpl_check(parent)); + } + + parent->flags &= ~MDBX_TXN_HAS_CHILD; + if (parent->tw.spilled.list) { + assert(pnl_check_allocated(parent->tw.spilled.list, (size_t)parent->geo.first_unallocated << 1)); + if (MDBX_PNL_GETSIZE(parent->tw.spilled.list)) + parent->flags |= MDBX_TXN_SPILLS; } - op_setlk = MDBX_F_SETLK; - op_setlkw = MDBX_F_SETLKW; - op_getlk = MDBX_F_GETLK; } -#else -#define op_setlk MDBX_F_SETLK -#define op_setlkw MDBX_F_SETLKW -#define op_getlk MDBX_F_GETLK -#endif /* MDBX_USE_OFDLOCKS */ -static int lck_op(const mdbx_filehandle_t fd, int cmd, const int lck, - const off_t offset, off_t len) { - STATIC_ASSERT(sizeof(off_t) >= sizeof(void *) && - sizeof(off_t) >= sizeof(size_t)); -#ifdef __ANDROID_API__ - STATIC_ASSERT_MSG((sizeof(off_t) * 8 == MDBX_WORDBITS), - "The bitness of system `off_t` type is mismatch. Please " - "fix build and/or NDK configuration."); -#endif /* Android */ - jitter4testing(true); - assert(offset >= 0 && len > 0); - assert((uint64_t)offset < (uint64_t)INT64_MAX && - (uint64_t)len < (uint64_t)INT64_MAX && - (uint64_t)(offset + len) > (uint64_t)offset); +void txn_take_gcprof(const MDBX_txn *txn, MDBX_commit_latency *latency) { + MDBX_env *const env = txn->env; + if (MDBX_ENABLE_PROFGC) { + pgop_stat_t *const ptr = &env->lck->pgops; + latency->gc_prof.work_counter = ptr->gc_prof.work.spe_counter; + latency->gc_prof.work_rtime_monotonic = osal_monotime_to_16dot16(ptr->gc_prof.work.rtime_monotonic); + latency->gc_prof.work_xtime_cpu = osal_monotime_to_16dot16(ptr->gc_prof.work.xtime_cpu); + latency->gc_prof.work_rsteps = ptr->gc_prof.work.rsteps; + latency->gc_prof.work_xpages = ptr->gc_prof.work.xpages; + latency->gc_prof.work_majflt = ptr->gc_prof.work.majflt; - assert((uint64_t)offset < (uint64_t)OFF_T_MAX && - (uint64_t)len <= (uint64_t)OFF_T_MAX && - (uint64_t)(offset + len) <= (uint64_t)OFF_T_MAX); + latency->gc_prof.self_counter = ptr->gc_prof.self.spe_counter; + latency->gc_prof.self_rtime_monotonic = osal_monotime_to_16dot16(ptr->gc_prof.self.rtime_monotonic); + latency->gc_prof.self_xtime_cpu = osal_monotime_to_16dot16(ptr->gc_prof.self.xtime_cpu); + latency->gc_prof.self_rsteps = ptr->gc_prof.self.rsteps; + latency->gc_prof.self_xpages = ptr->gc_prof.self.xpages; + latency->gc_prof.self_majflt = ptr->gc_prof.self.majflt; - assert((uint64_t)((off_t)((uint64_t)offset + (uint64_t)len)) == - ((uint64_t)offset + (uint64_t)len)); - for (;;) { - MDBX_STRUCT_FLOCK lock_op; - STATIC_ASSERT_MSG(sizeof(off_t) <= sizeof(lock_op.l_start) && - sizeof(off_t) <= sizeof(lock_op.l_len) && - OFF_T_MAX == (off_t)OFF_T_MAX, - "Support for large/64-bit-sized files is misconfigured " - "for the target system and/or toolchain. " - "Please fix it or at least disable it completely."); - memset(&lock_op, 0, sizeof(lock_op)); - lock_op.l_type = lck; - lock_op.l_whence = SEEK_SET; - lock_op.l_start = offset; - lock_op.l_len = len; - int rc = MDBX_FCNTL(fd, cmd, &lock_op); - jitter4testing(true); - if (rc != -1) { - if (cmd == op_getlk) { - /* Checks reader by pid. Returns: - * MDBX_RESULT_TRUE - if pid is live (reader holds a lock). - * MDBX_RESULT_FALSE - if pid is dead (a lock could be placed). */ - return (lock_op.l_type == F_UNLCK) ? MDBX_RESULT_FALSE - : MDBX_RESULT_TRUE; + latency->gc_prof.wloops = ptr->gc_prof.wloops; + latency->gc_prof.coalescences = ptr->gc_prof.coalescences; + latency->gc_prof.wipes = ptr->gc_prof.wipes; + latency->gc_prof.flushes = ptr->gc_prof.flushes; + latency->gc_prof.kicks = ptr->gc_prof.kicks; + + latency->gc_prof.pnl_merge_work.time = osal_monotime_to_16dot16(ptr->gc_prof.work.pnl_merge.time); + latency->gc_prof.pnl_merge_work.calls = ptr->gc_prof.work.pnl_merge.calls; + latency->gc_prof.pnl_merge_work.volume = ptr->gc_prof.work.pnl_merge.volume; + latency->gc_prof.pnl_merge_self.time = osal_monotime_to_16dot16(ptr->gc_prof.self.pnl_merge.time); + latency->gc_prof.pnl_merge_self.calls = ptr->gc_prof.self.pnl_merge.calls; + latency->gc_prof.pnl_merge_self.volume = ptr->gc_prof.self.pnl_merge.volume; + + if (txn == env->basal_txn) + memset(&ptr->gc_prof, 0, sizeof(ptr->gc_prof)); + } else + memset(&latency->gc_prof, 0, sizeof(latency->gc_prof)); +} + +int txn_abort(MDBX_txn *txn) { + if (txn->flags & MDBX_TXN_RDONLY) + /* LY: don't close DBI-handles */ + return txn_end(txn, TXN_END_ABORT | TXN_END_UPDATE | TXN_END_SLOT | TXN_END_FREE); + + if (unlikely(txn->flags & MDBX_TXN_FINISHED)) + return MDBX_BAD_TXN; + + if (txn->nested) + txn_abort(txn->nested); + + tASSERT(txn, (txn->flags & MDBX_TXN_ERROR) || dpl_check(txn)); + return txn_end(txn, TXN_END_ABORT | TXN_END_SLOT | TXN_END_FREE); +} + +int txn_renew(MDBX_txn *txn, unsigned flags) { + MDBX_env *const env = txn->env; + int rc; + +#if MDBX_ENV_CHECKPID + if (unlikely(env->pid != osal_getpid())) { + env->flags |= ENV_FATAL_ERROR; + return MDBX_PANIC; + } +#endif /* MDBX_ENV_CHECKPID */ + + flags |= env->flags & (MDBX_NOSTICKYTHREADS | MDBX_WRITEMAP); + if (flags & MDBX_TXN_RDONLY) { + eASSERT(env, (flags & ~(txn_ro_begin_flags | MDBX_WRITEMAP | MDBX_NOSTICKYTHREADS)) == 0); + txn->flags = flags; + reader_slot_t *r = txn->to.reader; + STATIC_ASSERT(sizeof(uintptr_t) <= sizeof(r->tid)); + if (likely(env->flags & ENV_TXKEY)) { + eASSERT(env, !(env->flags & MDBX_NOSTICKYTHREADS)); + r = thread_rthc_get(env->me_txkey); + if (likely(r)) { + if (unlikely(!r->pid.weak) && (globals.runtime_flags & MDBX_DBG_LEGACY_MULTIOPEN)) { + thread_rthc_set(env->me_txkey, nullptr); + r = nullptr; + } else { + eASSERT(env, r->pid.weak == env->pid); + eASSERT(env, r->tid.weak == osal_thread_self()); + } + } + } else { + eASSERT(env, !env->lck_mmap.lck || (env->flags & MDBX_NOSTICKYTHREADS)); + } + + if (likely(r)) { + if (unlikely(r->pid.weak != env->pid || r->txnid.weak < SAFE64_INVALID_THRESHOLD)) + return MDBX_BAD_RSLOT; + } else if (env->lck_mmap.lck) { + bsr_t brs = mvcc_bind_slot(env); + if (unlikely(brs.err != MDBX_SUCCESS)) + return brs.err; + r = brs.rslot; + } + txn->to.reader = r; + STATIC_ASSERT(MDBX_TXN_RDONLY_PREPARE > MDBX_TXN_RDONLY); + if (flags & (MDBX_TXN_RDONLY_PREPARE - MDBX_TXN_RDONLY)) { + eASSERT(env, txn->txnid == 0); + eASSERT(env, txn->owner == 0); + eASSERT(env, txn->n_dbi == 0); + if (likely(r)) { + eASSERT(env, r->snapshot_pages_used.weak == 0); + eASSERT(env, r->txnid.weak >= SAFE64_INVALID_THRESHOLD); + atomic_store32(&r->snapshot_pages_used, 0, mo_Relaxed); } + txn->flags = MDBX_TXN_RDONLY | MDBX_TXN_FINISHED; return MDBX_SUCCESS; } - rc = errno; -#if MDBX_USE_OFDLOCKS - if (rc == EINVAL && (cmd == MDBX_F_OFD_SETLK || cmd == MDBX_F_OFD_SETLKW || - cmd == MDBX_F_OFD_GETLK)) { - /* fallback to non-OFD locks */ - if (cmd == MDBX_F_OFD_SETLK) - cmd = MDBX_F_SETLK; - else if (cmd == MDBX_F_OFD_SETLKW) - cmd = MDBX_F_SETLKW; - else - cmd = MDBX_F_GETLK; - op_setlk = MDBX_F_SETLK; - op_setlkw = MDBX_F_SETLKW; - op_getlk = MDBX_F_GETLK; - continue; + txn->owner = likely(r) ? (uintptr_t)r->tid.weak : ((env->flags & MDBX_NOSTICKYTHREADS) ? 0 : osal_thread_self()); + if ((env->flags & MDBX_NOSTICKYTHREADS) == 0 && env->txn && unlikely(env->basal_txn->owner == txn->owner) && + (globals.runtime_flags & MDBX_DBG_LEGACY_OVERLAP) == 0) + return MDBX_TXN_OVERLAPPING; + + /* Seek & fetch the last meta */ + uint64_t timestamp = 0; + size_t loop = 0; + troika_t troika = meta_tap(env); + while (1) { + const meta_ptr_t head = likely(env->stuck_meta < 0) ? /* regular */ meta_recent(env, &troika) + : /* recovery mode */ meta_ptr(env, env->stuck_meta); + if (likely(r != nullptr)) { + safe64_reset(&r->txnid, true); + atomic_store32(&r->snapshot_pages_used, head.ptr_v->geometry.first_unallocated, mo_Relaxed); + atomic_store64(&r->snapshot_pages_retired, unaligned_peek_u64_volatile(4, head.ptr_v->pages_retired), + mo_Relaxed); + safe64_write(&r->txnid, head.txnid); + eASSERT(env, r->pid.weak == osal_getpid()); + eASSERT(env, r->tid.weak == ((env->flags & MDBX_NOSTICKYTHREADS) ? 0 : osal_thread_self())); + eASSERT(env, r->txnid.weak == head.txnid || + (r->txnid.weak >= SAFE64_INVALID_THRESHOLD && head.txnid < env->lck->cached_oldest.weak)); + atomic_store32(&env->lck->rdt_refresh_flag, true, mo_AcquireRelease); + } else { + /* exclusive mode without lck */ + eASSERT(env, !env->lck_mmap.lck && env->lck == lckless_stub(env)); + } + jitter4testing(true); + + if (unlikely(meta_should_retry(env, &troika))) { + retry: + if (likely(++loop < 42)) { + timestamp = 0; + continue; + } + ERROR("bailout waiting for valid snapshot (%s)", "meta-pages are too volatile"); + rc = MDBX_PROBLEM; + goto read_failed; + } + + /* Snap the state from current meta-head */ + rc = coherency_fetch_head(txn, head, ×tamp); + jitter4testing(false); + if (unlikely(rc != MDBX_SUCCESS)) { + if (rc == MDBX_RESULT_TRUE) + goto retry; + else + goto read_failed; + } + + const uint64_t snap_oldest = atomic_load64(&env->lck->cached_oldest, mo_AcquireRelease); + if (unlikely(txn->txnid < snap_oldest)) { + if (env->stuck_meta < 0) + goto retry; + ERROR("target meta-page %i is referenced to an obsolete MVCC-snapshot " + "%" PRIaTXN " < cached-oldest %" PRIaTXN, + env->stuck_meta, txn->txnid, snap_oldest); + rc = MDBX_MVCC_RETARDED; + goto read_failed; + } + + if (likely(r != nullptr) && unlikely(txn->txnid != atomic_load64(&r->txnid, mo_Relaxed))) + goto retry; + break; } -#endif /* MDBX_USE_OFDLOCKS */ - if (rc != EINTR || cmd == op_setlkw) { - assert(MDBX_IS_ERROR(rc)); + + if (unlikely(txn->txnid < MIN_TXNID || txn->txnid > MAX_TXNID)) { + ERROR("%s", "environment corrupted by died writer, must shutdown!"); + rc = MDBX_CORRUPTED; + read_failed: + txn->txnid = INVALID_TXNID; + if (likely(r != nullptr)) + safe64_reset(&r->txnid, true); + goto bailout; + } + + tASSERT(txn, rc == MDBX_SUCCESS); + ENSURE(env, txn->txnid >= + /* paranoia is appropriate here */ env->lck->cached_oldest.weak); + tASSERT(txn, txn->dbs[FREE_DBI].flags == MDBX_INTEGERKEY); + tASSERT(txn, check_table_flags(txn->dbs[MAIN_DBI].flags)); + } else { + eASSERT(env, (flags & ~(txn_rw_begin_flags | MDBX_TXN_SPILLS | MDBX_WRITEMAP | MDBX_NOSTICKYTHREADS)) == 0); + const uintptr_t tid = osal_thread_self(); + if (unlikely(txn->owner == tid || + /* not recovery mode */ env->stuck_meta >= 0)) + return MDBX_BUSY; + lck_t *const lck = env->lck_mmap.lck; + if (lck && (env->flags & MDBX_NOSTICKYTHREADS) == 0 && (globals.runtime_flags & MDBX_DBG_LEGACY_OVERLAP) == 0) { + const size_t snap_nreaders = atomic_load32(&lck->rdt_length, mo_AcquireRelease); + for (size_t i = 0; i < snap_nreaders; ++i) { + if (atomic_load32(&lck->rdt[i].pid, mo_Relaxed) == env->pid && + unlikely(atomic_load64(&lck->rdt[i].tid, mo_Relaxed) == tid)) { + const txnid_t txnid = safe64_read(&lck->rdt[i].txnid); + if (txnid >= MIN_TXNID && txnid <= MAX_TXNID) + return MDBX_TXN_OVERLAPPING; + } + } + } + + /* Not yet touching txn == env->basal_txn, it may be active */ + jitter4testing(false); + rc = lck_txn_lock(env, !!(flags & MDBX_TXN_TRY)); + if (unlikely(rc)) return rc; + if (unlikely(env->flags & ENV_FATAL_ERROR)) { + lck_txn_unlock(env); + return MDBX_PANIC; } - } -} +#if defined(_WIN32) || defined(_WIN64) + if (unlikely(!env->dxb_mmap.base)) { + lck_txn_unlock(env); + return MDBX_EPERM; + } +#endif /* Windows */ -MDBX_INTERNAL_FUNC int osal_lockfile(mdbx_filehandle_t fd, bool wait) { -#if MDBX_USE_OFDLOCKS - if (unlikely(op_setlk == 0)) - choice_fcntl(); -#endif /* MDBX_USE_OFDLOCKS */ - return lck_op(fd, wait ? op_setlkw : op_setlk, F_WRLCK, 0, OFF_T_MAX); -} + txn->tw.troika = meta_tap(env); + const meta_ptr_t head = meta_recent(env, &txn->tw.troika); + uint64_t timestamp = 0; + while ("workaround for https://libmdbx.dqdkfa.ru/dead-github/issues/269") { + rc = coherency_fetch_head(txn, head, ×tamp); + if (likely(rc == MDBX_SUCCESS)) + break; + if (unlikely(rc != MDBX_RESULT_TRUE)) + goto bailout; + } + eASSERT(env, meta_txnid(head.ptr_v) == txn->txnid); + txn->txnid = safe64_txnid_next(txn->txnid); + if (unlikely(txn->txnid > MAX_TXNID)) { + rc = MDBX_TXN_FULL; + ERROR("txnid overflow, raise %d", rc); + goto bailout; + } -MDBX_INTERNAL_FUNC int osal_rpid_set(MDBX_env *env) { - assert(env->me_lfd != INVALID_HANDLE_VALUE); - assert(env->me_pid > 0); - if (unlikely(osal_getpid() != env->me_pid)) - return MDBX_PANIC; - return lck_op(env->me_lfd, op_setlk, F_WRLCK, env->me_pid, 1); -} + tASSERT(txn, txn->dbs[FREE_DBI].flags == MDBX_INTEGERKEY); + tASSERT(txn, check_table_flags(txn->dbs[MAIN_DBI].flags)); + txn->flags = flags; + txn->nested = nullptr; + txn->tw.loose_pages = nullptr; + txn->tw.loose_count = 0; +#if MDBX_ENABLE_REFUND + txn->tw.loose_refund_wl = 0; +#endif /* MDBX_ENABLE_REFUND */ + MDBX_PNL_SETSIZE(txn->tw.retired_pages, 0); + txn->tw.spilled.list = nullptr; + txn->tw.spilled.least_removed = 0; + txn->tw.gc.time_acc = 0; + txn->tw.gc.last_reclaimed = 0; + if (txn->tw.gc.retxl) + MDBX_PNL_SETSIZE(txn->tw.gc.retxl, 0); + env->txn = txn; + } -MDBX_INTERNAL_FUNC int osal_rpid_clear(MDBX_env *env) { - assert(env->me_lfd != INVALID_HANDLE_VALUE); - assert(env->me_pid > 0); - return lck_op(env->me_lfd, op_setlk, F_UNLCK, env->me_pid, 1); -} + txn->front_txnid = txn->txnid + ((flags & (MDBX_WRITEMAP | MDBX_RDONLY)) == 0); -MDBX_INTERNAL_FUNC int osal_rpid_check(MDBX_env *env, uint32_t pid) { - assert(env->me_lfd != INVALID_HANDLE_VALUE); - assert(pid > 0); - return lck_op(env->me_lfd, op_getlk, F_WRLCK, pid, 1); -} + /* Setup db info */ + tASSERT(txn, txn->dbs[FREE_DBI].flags == MDBX_INTEGERKEY); + tASSERT(txn, check_table_flags(txn->dbs[MAIN_DBI].flags)); + VALGRIND_MAKE_MEM_UNDEFINED(txn->dbi_state, env->max_dbi); +#if MDBX_ENABLE_DBI_SPARSE + txn->n_dbi = CORE_DBS; + VALGRIND_MAKE_MEM_UNDEFINED(txn->dbi_sparse, + ceil_powerof2(env->max_dbi, CHAR_BIT * sizeof(txn->dbi_sparse[0])) / CHAR_BIT); + txn->dbi_sparse[0] = (1 << CORE_DBS) - 1; +#else + txn->n_dbi = (env->n_dbi < 8) ? env->n_dbi : 8; + if (txn->n_dbi > CORE_DBS) + memset(txn->dbi_state + CORE_DBS, 0, txn->n_dbi - CORE_DBS); +#endif /* MDBX_ENABLE_DBI_SPARSE */ + txn->dbi_state[FREE_DBI] = DBI_LINDO | DBI_VALID; + txn->dbi_state[MAIN_DBI] = DBI_LINDO | DBI_VALID; + txn->cursors[FREE_DBI] = nullptr; + txn->cursors[MAIN_DBI] = nullptr; + txn->dbi_seqs[FREE_DBI] = 0; + txn->dbi_seqs[MAIN_DBI] = atomic_load32(&env->dbi_seqs[MAIN_DBI], mo_AcquireRelease); + + if (unlikely(env->dbs_flags[MAIN_DBI] != (DB_VALID | txn->dbs[MAIN_DBI].flags) || !txn->dbi_seqs[MAIN_DBI])) { + const bool need_txn_lock = env->basal_txn && env->basal_txn->owner != osal_thread_self(); + bool should_unlock = false; + if (need_txn_lock) { + rc = lck_txn_lock(env, true); + if (rc == MDBX_SUCCESS) + should_unlock = true; + else if (rc != MDBX_BUSY && rc != MDBX_EDEADLK) + goto bailout; + } + rc = osal_fastmutex_acquire(&env->dbi_lock); + if (likely(rc == MDBX_SUCCESS)) { + /* проверяем повторно после захвата блокировки */ + uint32_t seq = atomic_load32(&env->dbi_seqs[MAIN_DBI], mo_AcquireRelease); + if (env->dbs_flags[MAIN_DBI] != (DB_VALID | txn->dbs[MAIN_DBI].flags)) { + if (!(env->dbs_flags[MAIN_DBI] & DB_VALID) || !need_txn_lock || should_unlock || + /* если нет активной пишущей транзакции, * то следующая будет ждать на dbi_lock */ !env->txn) { + if (env->dbs_flags[MAIN_DBI] & DB_VALID) { + NOTICE("renew MainDB for %s-txn %" PRIaTXN " since db-flags changes 0x%x -> 0x%x", + (txn->flags & MDBX_TXN_RDONLY) ? "ro" : "rw", txn->txnid, env->dbs_flags[MAIN_DBI] & ~DB_VALID, + txn->dbs[MAIN_DBI].flags); + seq = dbi_seq_next(env, MAIN_DBI); + env->dbs_flags[MAIN_DBI] = DB_POISON; + atomic_store32(&env->dbi_seqs[MAIN_DBI], seq, mo_AcquireRelease); + } + rc = tbl_setup(env, &env->kvs[MAIN_DBI], &txn->dbs[MAIN_DBI]); + if (likely(rc == MDBX_SUCCESS)) { + seq = dbi_seq_next(env, MAIN_DBI); + env->dbs_flags[MAIN_DBI] = DB_VALID | txn->dbs[MAIN_DBI].flags; + atomic_store32(&env->dbi_seqs[MAIN_DBI], seq, mo_AcquireRelease); + } + } else { + ERROR("MainDB db-flags changes 0x%x -> 0x%x ahead of read-txn " + "%" PRIaTXN, + txn->dbs[MAIN_DBI].flags, env->dbs_flags[MAIN_DBI] & ~DB_VALID, txn->txnid); + rc = MDBX_INCOMPATIBLE; + } + } + txn->dbi_seqs[MAIN_DBI] = seq; + ENSURE(env, osal_fastmutex_release(&env->dbi_lock) == MDBX_SUCCESS); + } else { + DEBUG("dbi_lock failed, err %d", rc); + } + if (should_unlock) + lck_txn_unlock(env); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + } -/*---------------------------------------------------------------------------*/ + if (unlikely(txn->dbs[FREE_DBI].flags != MDBX_INTEGERKEY)) { + ERROR("unexpected/invalid db-flags 0x%x for %s", txn->dbs[FREE_DBI].flags, "GC/FreeDB"); + rc = MDBX_INCOMPATIBLE; + goto bailout; + } -#if MDBX_LOCKING > MDBX_LOCKING_SYSV -MDBX_INTERNAL_FUNC int osal_ipclock_stub(osal_ipclock_t *ipc) { -#if MDBX_LOCKING == MDBX_LOCKING_POSIX1988 - return sem_init(ipc, false, 1) ? errno : 0; -#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ - MDBX_LOCKING == MDBX_LOCKING_POSIX2008 - return pthread_mutex_init(ipc, nullptr); + tASSERT(txn, txn->dbs[FREE_DBI].flags == MDBX_INTEGERKEY); + tASSERT(txn, check_table_flags(txn->dbs[MAIN_DBI].flags)); + if (unlikely(env->flags & ENV_FATAL_ERROR)) { + WARNING("%s", "environment had fatal error, must shutdown!"); + rc = MDBX_PANIC; + } else { + const size_t size_bytes = pgno2bytes(env, txn->geo.end_pgno); + const size_t used_bytes = pgno2bytes(env, txn->geo.first_unallocated); + const size_t required_bytes = (txn->flags & MDBX_TXN_RDONLY) ? used_bytes : size_bytes; + eASSERT(env, env->dxb_mmap.limit >= env->dxb_mmap.current); + if (unlikely(required_bytes > env->dxb_mmap.current)) { + /* Размер БД (для пишущих транзакций) или используемых данных (для + * читающих транзакций) больше предыдущего/текущего размера внутри + * процесса, увеличиваем. Сюда также попадает случай увеличения верхней + * границы размера БД и отображения. В читающих транзакциях нельзя + * изменять размер файла, который может быть больше необходимого этой + * транзакции. */ + if (txn->geo.upper > MAX_PAGENO + 1 || bytes2pgno(env, pgno2bytes(env, txn->geo.upper)) != txn->geo.upper) { + rc = MDBX_UNABLE_EXTEND_MAPSIZE; + goto bailout; + } + rc = dxb_resize(env, txn->geo.first_unallocated, txn->geo.end_pgno, txn->geo.upper, implicit_grow); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + eASSERT(env, env->dxb_mmap.limit >= env->dxb_mmap.current); + } else if (unlikely(size_bytes < env->dxb_mmap.current)) { + /* Размер БД меньше предыдущего/текущего размера внутри процесса, можно + * уменьшить, но всё сложнее: + * - размер файла согласован со всеми читаемыми снимками на момент + * коммита последней транзакции; + * - в читающей транзакции размер файла может быть больше и него нельзя + * изменять, в том числе менять madvise (меньша размера файла нельзя, + * а за размером нет смысла). + * - в пишущей транзакции уменьшать размер файла можно только после + * проверки размера читаемых снимков, но в этом нет смысла, так как + * это будет сделано при фиксации транзакции. + * + * В сухом остатке, можно только установить dxb_mmap.current равным + * размеру файла, а это проще сделать без вызова dxb_resize() и усложения + * внутренней логики. + * + * В этой тактике есть недостаток: если пишущите транзакции не регулярны, + * и при завершении такой транзакции файл БД остаётся не-уменьшеным из-за + * читающих транзакций использующих предыдущие снимки. */ +#if defined(_WIN32) || defined(_WIN64) + imports.srwl_AcquireShared(&env->remap_guard); #else -#error "FIXME" + rc = osal_fastmutex_acquire(&env->remap_guard); #endif -} - -MDBX_INTERNAL_FUNC int osal_ipclock_destroy(osal_ipclock_t *ipc) { -#if MDBX_LOCKING == MDBX_LOCKING_POSIX1988 - return sem_destroy(ipc) ? errno : 0; -#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ - MDBX_LOCKING == MDBX_LOCKING_POSIX2008 - return pthread_mutex_destroy(ipc); + if (likely(rc == MDBX_SUCCESS)) { + eASSERT(env, env->dxb_mmap.limit >= env->dxb_mmap.current); + rc = osal_filesize(env->dxb_mmap.fd, &env->dxb_mmap.filesize); + if (likely(rc == MDBX_SUCCESS)) { + eASSERT(env, env->dxb_mmap.filesize >= required_bytes); + if (env->dxb_mmap.current > env->dxb_mmap.filesize) + env->dxb_mmap.current = + (env->dxb_mmap.limit < env->dxb_mmap.filesize) ? env->dxb_mmap.limit : (size_t)env->dxb_mmap.filesize; + } +#if defined(_WIN32) || defined(_WIN64) + imports.srwl_ReleaseShared(&env->remap_guard); #else -#error "FIXME" + int err = osal_fastmutex_release(&env->remap_guard); + if (unlikely(err) && likely(rc == MDBX_SUCCESS)) + rc = err; #endif -} -#endif /* MDBX_LOCKING > MDBX_LOCKING_SYSV */ - -static int check_fstat(MDBX_env *env) { - struct stat st; - - int rc = MDBX_SUCCESS; - if (fstat(env->me_lazy_fd, &st)) { - rc = errno; - ERROR("fstat(%s), err %d", "DXB", rc); - return rc; - } + } + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + } + eASSERT(env, pgno2bytes(env, txn->geo.first_unallocated) <= env->dxb_mmap.current); + eASSERT(env, env->dxb_mmap.limit >= env->dxb_mmap.current); + if (txn->flags & MDBX_TXN_RDONLY) { +#if defined(_WIN32) || defined(_WIN64) + if (((used_bytes > env->geo_in_bytes.lower && env->geo_in_bytes.shrink) || + (globals.running_under_Wine && + /* under Wine acquisition of remap_guard is always required, + * since Wine don't support section extending, + * i.e. in both cases unmap+map are required. */ + used_bytes < env->geo_in_bytes.upper && env->geo_in_bytes.grow)) && + /* avoid recursive use SRW */ (txn->flags & MDBX_NOSTICKYTHREADS) == 0) { + txn->flags |= txn_shrink_allowed; + imports.srwl_AcquireShared(&env->remap_guard); + } +#endif /* Windows */ + } else { + tASSERT(txn, txn == env->basal_txn); - if (!S_ISREG(st.st_mode) || st.st_nlink < 1) { -#ifdef EBADFD - rc = EBADFD; -#else - rc = EPERM; -#endif - ERROR("%s %s, err %d", "DXB", - (st.st_nlink < 1) ? "file was removed" : "not a regular file", rc); - return rc; - } + if (env->options.need_dp_limit_adjust) + env_options_adjust_dp_limit(env); + if ((txn->flags & MDBX_WRITEMAP) == 0 || MDBX_AVOID_MSYNC) { + rc = dpl_alloc(txn); + if (unlikely(rc != MDBX_SUCCESS)) + goto bailout; + txn->tw.dirtyroom = txn->env->options.dp_limit; + txn->tw.dirtylru = MDBX_DEBUG ? UINT32_MAX / 3 - 42 : 0; + } else { + tASSERT(txn, txn->tw.dirtylist == nullptr); + txn->tw.dirtylist = nullptr; + txn->tw.dirtyroom = MAX_PAGENO; + txn->tw.dirtylru = 0; + } + eASSERT(env, txn->tw.writemap_dirty_npages == 0); + eASSERT(env, txn->tw.writemap_spilled_npages == 0); - if (st.st_size < (off_t)(MDBX_MIN_PAGESIZE * NUM_METAS)) { - VERBOSE("dxb-file is too short (%u), exclusive-lock needed", - (unsigned)st.st_size); - rc = MDBX_RESULT_TRUE; + MDBX_cursor *const gc = ptr_disp(txn, sizeof(MDBX_txn)); + rc = cursor_init(gc, txn, FREE_DBI); + if (rc != MDBX_SUCCESS) + goto bailout; + } + dxb_sanitize_tail(env, txn); + return MDBX_SUCCESS; } +bailout: + tASSERT(txn, rc != MDBX_SUCCESS); + txn_end(txn, TXN_END_SLOT | TXN_END_EOTDONE | TXN_END_FAIL_BEGIN); + return rc; +} - //---------------------------------------------------------------------------- +int txn_end(MDBX_txn *txn, unsigned mode) { + MDBX_env *env = txn->env; + static const char *const names[] = TXN_END_NAMES; - if (fstat(env->me_lfd, &st)) { - rc = errno; - ERROR("fstat(%s), err %d", "LCK", rc); - return rc; - } + DEBUG("%s txn %" PRIaTXN "%c-0x%X %p on env %p, root page %" PRIaPGNO "/%" PRIaPGNO, names[mode & TXN_END_OPMASK], + txn->txnid, (txn->flags & MDBX_TXN_RDONLY) ? 'r' : 'w', txn->flags, (void *)txn, (void *)env, + txn->dbs[MAIN_DBI].root, txn->dbs[FREE_DBI].root); - if (!S_ISREG(st.st_mode) || st.st_nlink < 1) { -#ifdef EBADFD - rc = EBADFD; -#else - rc = EPERM; -#endif - ERROR("%s %s, err %d", "LCK", - (st.st_nlink < 1) ? "file was removed" : "not a regular file", rc); - return rc; - } + if (!(mode & TXN_END_EOTDONE)) /* !(already closed cursors) */ + txn_done_cursors(txn, false); - /* Checking file size for detect the situation when we got the shared lock - * immediately after osal_lck_destroy(). */ - if (st.st_size < (off_t)(sizeof(MDBX_lockinfo) + sizeof(MDBX_reader))) { - VERBOSE("lck-file is too short (%u), exclusive-lock needed", - (unsigned)st.st_size); - rc = MDBX_RESULT_TRUE; - } + int rc = MDBX_SUCCESS; + if (txn->flags & MDBX_TXN_RDONLY) { + if (txn->to.reader) { + reader_slot_t *slot = txn->to.reader; + eASSERT(env, slot->pid.weak == env->pid); + if (likely(!(txn->flags & MDBX_TXN_FINISHED))) { + if (likely((txn->flags & MDBX_TXN_PARKED) == 0)) { + ENSURE(env, txn->txnid >= + /* paranoia is appropriate here */ env->lck->cached_oldest.weak); + eASSERT(env, txn->txnid == slot->txnid.weak && slot->txnid.weak >= env->lck->cached_oldest.weak); + } else { + if ((mode & TXN_END_OPMASK) != TXN_END_OUSTED && safe64_read(&slot->tid) == MDBX_TID_TXN_OUSTED) + mode = (mode & ~TXN_END_OPMASK) | TXN_END_OUSTED; + do { + safe64_reset(&slot->txnid, false); + atomic_store64(&slot->tid, txn->owner, mo_AcquireRelease); + atomic_yield(); + } while ( + unlikely(safe64_read(&slot->txnid) < SAFE64_INVALID_THRESHOLD || safe64_read(&slot->tid) != txn->owner)); + } + dxb_sanitize_tail(env, nullptr); + atomic_store32(&slot->snapshot_pages_used, 0, mo_Relaxed); + safe64_reset(&slot->txnid, true); + atomic_store32(&env->lck->rdt_refresh_flag, true, mo_Relaxed); + } else { + eASSERT(env, slot->pid.weak == env->pid); + eASSERT(env, slot->txnid.weak >= SAFE64_INVALID_THRESHOLD); + } + if (mode & TXN_END_SLOT) { + if ((env->flags & ENV_TXKEY) == 0) + atomic_store32(&slot->pid, 0, mo_Relaxed); + txn->to.reader = nullptr; + } + } +#if defined(_WIN32) || defined(_WIN64) + if (txn->flags & txn_shrink_allowed) + imports.srwl_ReleaseShared(&env->remap_guard); +#endif + txn->n_dbi = 0; /* prevent further DBI activity */ + txn->flags = ((mode & TXN_END_OPMASK) != TXN_END_OUSTED) ? MDBX_TXN_RDONLY | MDBX_TXN_FINISHED + : MDBX_TXN_RDONLY | MDBX_TXN_FINISHED | MDBX_TXN_OUSTED; + txn->owner = 0; + } else if (!(txn->flags & MDBX_TXN_FINISHED)) { + ENSURE(env, txn->txnid >= + /* paranoia is appropriate here */ env->lck->cached_oldest.weak); + if (txn == env->basal_txn) + dxb_sanitize_tail(env, nullptr); + + txn->flags = MDBX_TXN_FINISHED; + env->txn = txn->parent; + pnl_free(txn->tw.spilled.list); + txn->tw.spilled.list = nullptr; + if (txn == env->basal_txn) { + eASSERT(env, txn->parent == nullptr); + /* Export or close DBI handles created in this txn */ + rc = dbi_update(txn, mode & TXN_END_UPDATE); + pnl_shrink(&txn->tw.retired_pages); + pnl_shrink(&txn->tw.repnl); + if (!(env->flags & MDBX_WRITEMAP)) + dpl_release_shadows(txn); + /* The writer mutex was locked in mdbx_txn_begin. */ + lck_txn_unlock(env); + } else { + eASSERT(env, txn->parent != nullptr); + MDBX_txn *const parent = txn->parent; + eASSERT(env, parent->signature == txn_signature); + eASSERT(env, parent->nested == txn && (parent->flags & MDBX_TXN_HAS_CHILD) != 0); + eASSERT(env, pnl_check_allocated(txn->tw.repnl, txn->geo.first_unallocated - MDBX_ENABLE_REFUND)); + eASSERT(env, memcmp(&txn->tw.troika, &parent->tw.troika, sizeof(troika_t)) == 0); - return rc; -} + txn->owner = 0; + if (txn->tw.gc.retxl) { + eASSERT(env, MDBX_PNL_GETSIZE(txn->tw.gc.retxl) >= (uintptr_t)parent->tw.gc.retxl); + MDBX_PNL_SETSIZE(txn->tw.gc.retxl, (uintptr_t)parent->tw.gc.retxl); + parent->tw.gc.retxl = txn->tw.gc.retxl; + } -__cold MDBX_INTERNAL_FUNC int osal_lck_seize(MDBX_env *env) { - assert(env->me_lazy_fd != INVALID_HANDLE_VALUE); - if (unlikely(osal_getpid() != env->me_pid)) - return MDBX_PANIC; -#if MDBX_USE_OFDLOCKS - if (unlikely(op_setlk == 0)) - choice_fcntl(); -#endif /* MDBX_USE_OFDLOCKS */ + if (txn->tw.retired_pages) { + eASSERT(env, MDBX_PNL_GETSIZE(txn->tw.retired_pages) >= (uintptr_t)parent->tw.retired_pages); + MDBX_PNL_SETSIZE(txn->tw.retired_pages, (uintptr_t)parent->tw.retired_pages); + parent->tw.retired_pages = txn->tw.retired_pages; + } - int rc = MDBX_SUCCESS; -#if defined(__linux__) || defined(__gnu_linux__) - if (unlikely(mdbx_RunningOnWSL1)) { - rc = ENOLCK /* No record locks available */; - ERROR("%s, err %u", - "WSL1 (Windows Subsystem for Linux) is mad and trouble-full, " - "injecting failure to avoid data loss", - rc); - return rc; - } -#endif /* Linux */ + parent->nested = nullptr; + parent->flags &= ~MDBX_TXN_HAS_CHILD; + parent->tw.dirtylru = txn->tw.dirtylru; + tASSERT(parent, dpl_check(parent)); + tASSERT(parent, audit_ex(parent, 0, false) == 0); + dpl_release_shadows(txn); + dpl_free(txn); + pnl_free(txn->tw.repnl); - if (env->me_lfd == INVALID_HANDLE_VALUE) { - /* LY: without-lck mode (e.g. exclusive or on read-only filesystem) */ - rc = - lck_op(env->me_lazy_fd, op_setlk, - (env->me_flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, 0, OFF_T_MAX); - if (rc != MDBX_SUCCESS) { - ERROR("%s, err %u", "without-lck", rc); - eASSERT(env, MDBX_IS_ERROR(rc)); - return rc; + if (parent->geo.upper != txn->geo.upper || parent->geo.now != txn->geo.now) { + /* undo resize performed by child txn */ + rc = dxb_resize(env, parent->geo.first_unallocated, parent->geo.now, parent->geo.upper, impilict_shrink); + if (rc == MDBX_EPERM) { + /* unable undo resize (it is regular for Windows), + * therefore promote size changes from child to the parent txn */ + WARNING("unable undo resize performed by child txn, promote to " + "the parent (%u->%u, %u->%u)", + txn->geo.now, parent->geo.now, txn->geo.upper, parent->geo.upper); + parent->geo.now = txn->geo.now; + parent->geo.upper = txn->geo.upper; + parent->flags |= MDBX_TXN_DIRTY; + rc = MDBX_SUCCESS; + } else if (unlikely(rc != MDBX_SUCCESS)) { + ERROR("error %d while undo resize performed by child txn, fail " + "the parent", + rc); + parent->flags |= MDBX_TXN_ERROR; + if (!env->dxb_mmap.base) + env->flags |= ENV_FATAL_ERROR; + } + } } - return MDBX_RESULT_TRUE /* Done: return with exclusive locking. */; } -#if defined(_POSIX_PRIORITY_SCHEDULING) && _POSIX_PRIORITY_SCHEDULING > 0 - sched_yield(); -#endif -retry: - if (rc == MDBX_RESULT_TRUE) { - rc = lck_op(env->me_lfd, op_setlk, F_UNLCK, 0, 1); - if (rc != MDBX_SUCCESS) { - ERROR("%s, err %u", "unlock-before-retry", rc); - eASSERT(env, MDBX_IS_ERROR(rc)); - return rc; - } + eASSERT(env, txn == env->basal_txn || txn->owner == 0); + if ((mode & TXN_END_FREE) != 0 && txn != env->basal_txn) { + txn->signature = 0; + osal_free(txn); } - /* Firstly try to get exclusive locking. */ - rc = lck_op(env->me_lfd, op_setlk, F_WRLCK, 0, 1); - if (rc == MDBX_SUCCESS) { - rc = check_fstat(env); - if (MDBX_IS_ERROR(rc)) - return rc; + return rc; +} - continue_dxb_exclusive: - rc = - lck_op(env->me_lazy_fd, op_setlk, - (env->me_flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, 0, OFF_T_MAX); - if (rc == MDBX_SUCCESS) - return MDBX_RESULT_TRUE /* Done: return with exclusive locking. */; +int txn_check_badbits_parked(const MDBX_txn *txn, int bad_bits) { + tASSERT(txn, (bad_bits & MDBX_TXN_PARKED) && (txn->flags & bad_bits)); + /* Здесь осознано заложено отличие в поведении припаркованных транзакций: + * - некоторые функции (например mdbx_env_info_ex()), допускают + * использование поломанных транзакций (с флагом MDBX_TXN_ERROR), но + * не могут работать с припаркованными транзакциями (требуют распарковки). + * - но при распарковке поломанные транзакции завершаются. + * - получается что транзакцию можно припарковать, потом поломать вызвав + * mdbx_txn_break(), но далее любое её использование приведет к завершению + * при распарковке. + * + * Поэтому для припаркованных транзакций возвращается ошибка если не-включена + * авто-распарковка, либо есть другие плохие биты. */ + if ((txn->flags & (bad_bits | MDBX_TXN_AUTOUNPARK)) != (MDBX_TXN_PARKED | MDBX_TXN_AUTOUNPARK)) + return LOG_IFERR(MDBX_BAD_TXN); - int err = check_fstat(env); - if (MDBX_IS_ERROR(err)) - return err; + tASSERT(txn, bad_bits == MDBX_TXN_BLOCKED || bad_bits == MDBX_TXN_BLOCKED - MDBX_TXN_ERROR); + return mdbx_txn_unpark((MDBX_txn *)txn, false); +} - /* the cause may be a collision with POSIX's file-lock recovery. */ - if (!(rc == EAGAIN || rc == EACCES || rc == EBUSY || rc == EWOULDBLOCK || - rc == EDEADLK)) { - ERROR("%s, err %u", "dxb-exclusive", rc); - eASSERT(env, MDBX_IS_ERROR(rc)); - return rc; - } +int txn_park(MDBX_txn *txn, bool autounpark) { + reader_slot_t *const rslot = txn->to.reader; + tASSERT(txn, (txn->flags & (MDBX_TXN_FINISHED | MDBX_TXN_RDONLY | MDBX_TXN_PARKED)) == MDBX_TXN_RDONLY); + tASSERT(txn, txn->to.reader->tid.weak < MDBX_TID_TXN_OUSTED); + if (unlikely((txn->flags & (MDBX_TXN_FINISHED | MDBX_TXN_RDONLY | MDBX_TXN_PARKED)) != MDBX_TXN_RDONLY)) + return MDBX_BAD_TXN; - /* Fallback to lck-shared */ - } else if (!(rc == EAGAIN || rc == EACCES || rc == EBUSY || - rc == EWOULDBLOCK || rc == EDEADLK)) { - ERROR("%s, err %u", "try-exclusive", rc); - eASSERT(env, MDBX_IS_ERROR(rc)); - return rc; + const uint32_t pid = atomic_load32(&rslot->pid, mo_Relaxed); + const uint64_t tid = atomic_load64(&rslot->tid, mo_Relaxed); + const uint64_t txnid = atomic_load64(&rslot->txnid, mo_Relaxed); + if (unlikely(pid != txn->env->pid)) { + ERROR("unexpected pid %u%s%u", pid, " != must ", txn->env->pid); + return MDBX_PROBLEM; } - - /* Here could be one of two: - * - osal_lck_destroy() from the another process was hold the lock - * during a destruction. - * - either osal_lck_seize() from the another process was got the exclusive - * lock and doing initialization. - * For distinguish these cases will use size of the lck-file later. */ - - /* Wait for lck-shared now. */ - /* Here may be await during transient processes, for instance until another - * competing process doesn't call lck_downgrade(). */ - rc = lck_op(env->me_lfd, op_setlkw, F_RDLCK, 0, 1); - if (rc != MDBX_SUCCESS) { - ERROR("%s, err %u", "try-shared", rc); - eASSERT(env, MDBX_IS_ERROR(rc)); - return rc; + if (unlikely(tid != txn->owner || txnid != txn->txnid)) { + ERROR("unexpected thread-id 0x%" PRIx64 "%s0x%0zx" + " and/or txn-id %" PRIaTXN "%s%" PRIaTXN, + tid, " != must ", txn->owner, txnid, " != must ", txn->txnid); + return MDBX_BAD_RSLOT; } - rc = check_fstat(env); - if (rc == MDBX_RESULT_TRUE) - goto retry; - if (rc != MDBX_SUCCESS) { - ERROR("%s, err %u", "lck_fstat", rc); - return rc; - } + atomic_store64(&rslot->tid, MDBX_TID_TXN_PARKED, mo_AcquireRelease); + atomic_store32(&txn->env->lck->rdt_refresh_flag, true, mo_Relaxed); + txn->flags += autounpark ? MDBX_TXN_PARKED | MDBX_TXN_AUTOUNPARK : MDBX_TXN_PARKED; + return MDBX_SUCCESS; +} - /* got shared, retry exclusive */ - rc = lck_op(env->me_lfd, op_setlk, F_WRLCK, 0, 1); - if (rc == MDBX_SUCCESS) - goto continue_dxb_exclusive; +int txn_unpark(MDBX_txn *txn) { + if (unlikely((txn->flags & (MDBX_TXN_FINISHED | MDBX_TXN_HAS_CHILD | MDBX_TXN_RDONLY | MDBX_TXN_PARKED)) != + (MDBX_TXN_RDONLY | MDBX_TXN_PARKED))) + return MDBX_BAD_TXN; - if (!(rc == EAGAIN || rc == EACCES || rc == EBUSY || rc == EWOULDBLOCK || - rc == EDEADLK)) { - ERROR("%s, err %u", "try-exclusive", rc); - eASSERT(env, MDBX_IS_ERROR(rc)); - return rc; - } + for (reader_slot_t *const rslot = txn->to.reader; rslot; atomic_yield()) { + const uint32_t pid = atomic_load32(&rslot->pid, mo_Relaxed); + uint64_t tid = safe64_read(&rslot->tid); + uint64_t txnid = safe64_read(&rslot->txnid); + if (unlikely(pid != txn->env->pid)) { + ERROR("unexpected pid %u%s%u", pid, " != expected ", txn->env->pid); + return MDBX_PROBLEM; + } + if (unlikely(tid == MDBX_TID_TXN_OUSTED || txnid >= SAFE64_INVALID_THRESHOLD)) + break; + if (unlikely(tid != MDBX_TID_TXN_PARKED || txnid != txn->txnid)) { + ERROR("unexpected thread-id 0x%" PRIx64 "%s0x%" PRIx64 " and/or txn-id %" PRIaTXN "%s%" PRIaTXN, tid, " != must ", + MDBX_TID_TXN_OUSTED, txnid, " != must ", txn->txnid); + break; + } + if (unlikely((txn->flags & MDBX_TXN_ERROR))) + break; - /* Lock against another process operating in without-lck or exclusive mode. */ - rc = - lck_op(env->me_lazy_fd, op_setlk, - (env->me_flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, env->me_pid, 1); - if (rc != MDBX_SUCCESS) { - ERROR("%s, err %u", "lock-against-without-lck", rc); - eASSERT(env, MDBX_IS_ERROR(rc)); - return rc; +#if MDBX_64BIT_CAS + if (unlikely(!atomic_cas64(&rslot->tid, MDBX_TID_TXN_PARKED, txn->owner))) + continue; +#else + atomic_store32(&rslot->tid.high, (uint32_t)((uint64_t)txn->owner >> 32), mo_Relaxed); + if (unlikely(!atomic_cas32(&rslot->tid.low, (uint32_t)MDBX_TID_TXN_PARKED, (uint32_t)txn->owner))) { + atomic_store32(&rslot->tid.high, (uint32_t)(MDBX_TID_TXN_PARKED >> 32), mo_AcquireRelease); + continue; + } +#endif + txnid = safe64_read(&rslot->txnid); + tid = safe64_read(&rslot->tid); + if (unlikely(txnid != txn->txnid || tid != txn->owner)) { + ERROR("unexpected thread-id 0x%" PRIx64 "%s0x%zx" + " and/or txn-id %" PRIaTXN "%s%" PRIaTXN, + tid, " != must ", txn->owner, txnid, " != must ", txn->txnid); + break; + } + txn->flags &= ~(MDBX_TXN_PARKED | MDBX_TXN_AUTOUNPARK); + return MDBX_SUCCESS; } - /* Done: return with shared locking. */ - return MDBX_RESULT_FALSE; + int err = txn_end(txn, TXN_END_OUSTED | TXN_END_RESET | TXN_END_UPDATE); + return err ? err : MDBX_OUSTED; } +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 -MDBX_INTERNAL_FUNC int osal_lck_downgrade(MDBX_env *env) { - assert(env->me_lfd != INVALID_HANDLE_VALUE); - if (unlikely(osal_getpid() != env->me_pid)) - return MDBX_PANIC; +MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION MDBX_INTERNAL unsigned log2n_powerof2(size_t value_uintptr) { + assert(value_uintptr > 0 && value_uintptr < INT32_MAX && is_powerof2(value_uintptr)); + assert((value_uintptr & -(intptr_t)value_uintptr) == value_uintptr); + const uint32_t value_uint32 = (uint32_t)value_uintptr; +#if __GNUC_PREREQ(4, 1) || __has_builtin(__builtin_ctz) + STATIC_ASSERT(sizeof(value_uint32) <= sizeof(unsigned)); + return __builtin_ctz(value_uint32); +#elif defined(_MSC_VER) + unsigned long index; + STATIC_ASSERT(sizeof(value_uint32) <= sizeof(long)); + _BitScanForward(&index, value_uint32); + return index; +#else + static const uint8_t debruijn_ctz32[32] = {0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8, + 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9}; + return debruijn_ctz32[(uint32_t)(value_uint32 * 0x077CB531ul) >> 27]; +#endif +} - int rc = MDBX_SUCCESS; - if ((env->me_flags & MDBX_EXCLUSIVE) == 0) { - rc = lck_op(env->me_lazy_fd, op_setlk, F_UNLCK, 0, env->me_pid); - if (rc == MDBX_SUCCESS) - rc = lck_op(env->me_lazy_fd, op_setlk, F_UNLCK, env->me_pid + 1, - OFF_T_MAX - env->me_pid - 1); - } - if (rc == MDBX_SUCCESS) - rc = lck_op(env->me_lfd, op_setlk, F_RDLCK, 0, 1); - if (unlikely(rc != 0)) { - ERROR("%s, err %u", "lck", rc); - assert(MDBX_IS_ERROR(rc)); - } - return rc; +MDBX_NOTHROW_CONST_FUNCTION MDBX_INTERNAL uint64_t rrxmrrxmsx_0(uint64_t v) { + /* Pelle Evensen's mixer, https://bit.ly/2HOfynt */ + v ^= (v << 39 | v >> 25) ^ (v << 14 | v >> 50); + v *= UINT64_C(0xA24BAED4963EE407); + v ^= (v << 40 | v >> 24) ^ (v << 15 | v >> 49); + v *= UINT64_C(0x9FB21C651E98DF25); + return v ^ v >> 28; } +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 -__cold MDBX_INTERNAL_FUNC int osal_lck_destroy(MDBX_env *env, - MDBX_env *inprocess_neighbor) { - if (unlikely(osal_getpid() != env->me_pid)) - return MDBX_PANIC; - - int rc = MDBX_SUCCESS; - struct stat lck_info; - MDBX_lockinfo *lck = env->me_lck_mmap.lck; - if (env->me_lfd != INVALID_HANDLE_VALUE && !inprocess_neighbor && lck && - /* try get exclusive access */ - lck_op(env->me_lfd, op_setlk, F_WRLCK, 0, OFF_T_MAX) == 0 && - /* if LCK was not removed */ - fstat(env->me_lfd, &lck_info) == 0 && lck_info.st_nlink > 0 && - lck_op(env->me_lazy_fd, op_setlk, - (env->me_flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, 0, - OFF_T_MAX) == 0) { +typedef struct walk_ctx { + void *userctx; + walk_options_t options; + int deep; + walk_func *visitor; + MDBX_txn *txn; + MDBX_cursor *cursor; +} walk_ctx_t; - VERBOSE("%p got exclusive, drown locks", (void *)env); -#if MDBX_LOCKING == MDBX_LOCKING_SYSV - if (env->me_sysv_ipc.semid != -1) - rc = semctl(env->me_sysv_ipc.semid, 2, IPC_RMID) ? errno : 0; -#else - rc = osal_ipclock_destroy(&lck->mti_rlock); - if (rc == 0) - rc = osal_ipclock_destroy(&lck->mti_wlock); -#endif /* MDBX_LOCKING */ +__cold static int walk_tbl(walk_ctx_t *ctx, walk_tbl_t *tbl); - eASSERT(env, rc == 0); - if (rc == 0) { - const bool synced = lck->mti_unsynced_pages.weak == 0; - osal_munmap(&env->me_lck_mmap); - if (synced) - rc = ftruncate(env->me_lfd, 0) ? errno : 0; +static page_type_t walk_page_type(const page_t *mp) { + if (mp) + switch (mp->flags & ~P_SPILLED) { + case P_BRANCH: + return page_branch; + case P_LEAF: + return page_leaf; + case P_LEAF | P_DUPFIX: + return page_dupfix_leaf; + case P_LARGE: + return page_large; } + return page_broken; +} - jitter4testing(false); +static page_type_t walk_subpage_type(const page_t *sp) { + switch (sp->flags & /* ignore legacy P_DIRTY flag */ ~P_LEGACY_DIRTY) { + case P_LEAF | P_SUBP: + return page_sub_leaf; + case P_LEAF | P_DUPFIX | P_SUBP: + return page_sub_dupfix_leaf; + default: + return page_sub_broken; } +} - /* 1) POSIX's fcntl() locks (i.e. when op_setlk == F_SETLK) should be restored - * after file was closed. - * - * 2) File locks would be released (by kernel) while the file-descriptors will - * be closed. But to avoid false-positive EACCESS and EDEADLK from the kernel, - * locks should be released here explicitly with properly order. */ +/* Depth-first tree traversal. */ +__cold static int walk_pgno(walk_ctx_t *ctx, walk_tbl_t *tbl, const pgno_t pgno, txnid_t parent_txnid) { + assert(pgno != P_INVALID); + page_t *mp = nullptr; + int err = page_get(ctx->cursor, pgno, &mp, parent_txnid); - /* close dxb and restore lock */ - if (env->me_dsync_fd != INVALID_HANDLE_VALUE) { - if (unlikely(close(env->me_dsync_fd) != 0) && rc == MDBX_SUCCESS) - rc = errno; - env->me_dsync_fd = INVALID_HANDLE_VALUE; - } - if (env->me_lazy_fd != INVALID_HANDLE_VALUE) { - if (unlikely(close(env->me_lazy_fd) != 0) && rc == MDBX_SUCCESS) - rc = errno; - env->me_lazy_fd = INVALID_HANDLE_VALUE; - if (op_setlk == F_SETLK && inprocess_neighbor && rc == MDBX_SUCCESS) { - /* restore file-lock */ - rc = lck_op( - inprocess_neighbor->me_lazy_fd, F_SETLKW, - (inprocess_neighbor->me_flags & MDBX_RDONLY) ? F_RDLCK : F_WRLCK, - (inprocess_neighbor->me_flags & MDBX_EXCLUSIVE) - ? 0 - : inprocess_neighbor->me_pid, - (inprocess_neighbor->me_flags & MDBX_EXCLUSIVE) ? OFF_T_MAX : 1); - } - } + const page_type_t type = walk_page_type(mp); + const size_t nentries = mp ? page_numkeys(mp) : 0; + size_t header_size = (mp && !is_dupfix_leaf(mp)) ? PAGEHDRSZ + mp->lower : PAGEHDRSZ; + size_t payload_size = 0; + size_t unused_size = (mp ? page_room(mp) : ctx->txn->env->ps - header_size) - payload_size; + size_t align_bytes = 0; - /* close clk and restore locks */ - if (env->me_lfd != INVALID_HANDLE_VALUE) { - if (unlikely(close(env->me_lfd) != 0) && rc == MDBX_SUCCESS) - rc = errno; - env->me_lfd = INVALID_HANDLE_VALUE; - if (op_setlk == F_SETLK && inprocess_neighbor && rc == MDBX_SUCCESS) { - /* restore file-locks */ - rc = lck_op(inprocess_neighbor->me_lfd, F_SETLKW, F_RDLCK, 0, 1); - if (rc == MDBX_SUCCESS && inprocess_neighbor->me_live_reader) - rc = osal_rpid_set(inprocess_neighbor); + for (size_t i = 0; err == MDBX_SUCCESS && i < nentries; ++i) { + if (type == page_dupfix_leaf) { + /* DUPFIX pages have no entries[] or node headers */ + payload_size += mp->dupfix_ksize; + continue; } - } - if (inprocess_neighbor && rc != MDBX_SUCCESS) - inprocess_neighbor->me_flags |= MDBX_FATAL_ERROR; - return rc; -} - -/*---------------------------------------------------------------------------*/ - -__cold MDBX_INTERNAL_FUNC int osal_lck_init(MDBX_env *env, - MDBX_env *inprocess_neighbor, - int global_uniqueness_flag) { -#if MDBX_LOCKING == MDBX_LOCKING_SYSV - int semid = -1; - /* don't initialize semaphores twice */ - (void)inprocess_neighbor; - if (global_uniqueness_flag == MDBX_RESULT_TRUE) { - struct stat st; - if (fstat(env->me_lazy_fd, &st)) - return errno; - sysv_retry_create: - semid = semget(env->me_sysv_ipc.key, 2, - IPC_CREAT | IPC_EXCL | - (st.st_mode & (S_IRWXU | S_IRWXG | S_IRWXO))); - if (unlikely(semid == -1)) { - int err = errno; - if (err != EEXIST) - return err; + const node_t *node = page_node(mp, i); + header_size += NODESIZE; + const size_t node_key_size = node_ks(node); + payload_size += node_key_size; - /* remove and re-create semaphore set */ - semid = semget(env->me_sysv_ipc.key, 2, 0); - if (semid == -1) { - err = errno; - if (err != ENOENT) - return err; - goto sysv_retry_create; - } - if (semctl(semid, 2, IPC_RMID)) { - err = errno; - if (err != EIDRM) - return err; - } - goto sysv_retry_create; + if (type == page_branch) { + assert(i > 0 || node_ks(node) == 0); + align_bytes += node_key_size & 1; + continue; } - unsigned short val_array[2] = {1, 1}; - if (semctl(semid, 2, SETALL, val_array)) - return errno; - } else { - semid = semget(env->me_sysv_ipc.key, 2, 0); - if (semid == -1) - return errno; + const size_t node_data_size = node_ds(node); + assert(type == page_leaf); + switch (node_flags(node)) { + case 0 /* usual node */: + payload_size += node_data_size; + align_bytes += (node_key_size + node_data_size) & 1; + break; - /* check read & write access */ - struct semid_ds data[2]; - if (semctl(semid, 2, IPC_STAT, data) || semctl(semid, 2, IPC_SET, data)) - return errno; - } + case N_BIG /* long data on the large/overflow page */: { + const pgno_t large_pgno = node_largedata_pgno(node); + const size_t over_payload = node_data_size; + const size_t over_header = PAGEHDRSZ; - env->me_sysv_ipc.semid = semid; - return MDBX_SUCCESS; + assert(err == MDBX_SUCCESS); + pgr_t lp = page_get_large(ctx->cursor, large_pgno, mp->txnid); + const size_t npages = ((err = lp.err) == MDBX_SUCCESS) ? lp.page->pages : 1; + const size_t pagesize = pgno2bytes(ctx->txn->env, npages); + const size_t over_unused = pagesize - over_payload - over_header; + const int rc = ctx->visitor(large_pgno, npages, ctx->userctx, ctx->deep, tbl, pagesize, page_large, err, 1, + over_payload, over_header, over_unused); + if (unlikely(rc != MDBX_SUCCESS)) + return (rc == MDBX_RESULT_TRUE) ? MDBX_SUCCESS : rc; + payload_size += sizeof(pgno_t); + align_bytes += node_key_size & 1; + } break; -#elif MDBX_LOCKING == MDBX_LOCKING_FUTEX - (void)inprocess_neighbor; - if (global_uniqueness_flag != MDBX_RESULT_TRUE) - return MDBX_SUCCESS; -#error "FIXME: Not implemented" -#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988 + case N_TREE /* sub-db */: { + if (unlikely(node_data_size != sizeof(tree_t))) { + ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid table node size", (unsigned)node_data_size); + assert(err == MDBX_CORRUPTED); + err = MDBX_CORRUPTED; + } + header_size += node_data_size; + align_bytes += (node_key_size + node_data_size) & 1; + } break; - /* don't initialize semaphores twice */ - (void)inprocess_neighbor; - if (global_uniqueness_flag == MDBX_RESULT_TRUE) { - if (sem_init(&env->me_lck_mmap.lck->mti_rlock, true, 1)) - return errno; - if (sem_init(&env->me_lck_mmap.lck->mti_wlock, true, 1)) - return errno; - } - return MDBX_SUCCESS; + case N_TREE | N_DUP /* dupsorted sub-tree */: + if (unlikely(node_data_size != sizeof(tree_t))) { + ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid sub-tree node size", (unsigned)node_data_size); + assert(err == MDBX_CORRUPTED); + err = MDBX_CORRUPTED; + } + header_size += node_data_size; + align_bytes += (node_key_size + node_data_size) & 1; + break; -#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ - MDBX_LOCKING == MDBX_LOCKING_POSIX2008 - if (inprocess_neighbor) - return MDBX_SUCCESS /* don't need any initialization for mutexes - if LCK already opened/used inside current process */ - ; + case N_DUP /* short sub-page */: { + if (unlikely(node_data_size <= PAGEHDRSZ || (node_data_size & 1))) { + ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid sub-page node size", (unsigned)node_data_size); + assert(err == MDBX_CORRUPTED); + err = MDBX_CORRUPTED; + break; + } - /* FIXME: Unfortunately, there is no other reliable way but to long testing - * on each platform. On the other hand, behavior like FreeBSD is incorrect - * and we can expect it to be rare. Moreover, even on FreeBSD without - * additional in-process initialization, the probability of an problem - * occurring is vanishingly small, and the symptom is a return of EINVAL - * while locking a mutex. In other words, in the worst case, the problem - * results in an EINVAL error at the start of the transaction, but NOT data - * loss, nor database corruption, nor other fatal troubles. Thus, the code - * below I am inclined to think the workaround for erroneous platforms (like - * FreeBSD), rather than a defect of libmdbx. */ -#if defined(__FreeBSD__) - /* seems that shared mutexes on FreeBSD required in-process initialization */ - (void)global_uniqueness_flag; -#else - /* shared mutexes on many other platforms (including Darwin and Linux's - * futexes) doesn't need any addition in-process initialization */ - if (global_uniqueness_flag != MDBX_RESULT_TRUE) - return MDBX_SUCCESS; -#endif + const page_t *const sp = node_data(node); + const page_type_t subtype = walk_subpage_type(sp); + const size_t nsubkeys = page_numkeys(sp); + if (unlikely(subtype == page_sub_broken)) { + ERROR("%s/%d: %s 0x%x", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid sub-page flags", sp->flags); + assert(err == MDBX_CORRUPTED); + err = MDBX_CORRUPTED; + } - pthread_mutexattr_t ma; - int rc = pthread_mutexattr_init(&ma); - if (rc) - return rc; + size_t subheader_size = is_dupfix_leaf(sp) ? PAGEHDRSZ : PAGEHDRSZ + sp->lower; + size_t subunused_size = page_room(sp); + size_t subpayload_size = 0; + size_t subalign_bytes = 0; - rc = pthread_mutexattr_setpshared(&ma, PTHREAD_PROCESS_SHARED); - if (rc) - goto bailout; + for (size_t ii = 0; err == MDBX_SUCCESS && ii < nsubkeys; ++ii) { + if (subtype == page_sub_dupfix_leaf) { + /* DUPFIX pages have no entries[] or node headers */ + subpayload_size += sp->dupfix_ksize; + } else { + assert(subtype == page_sub_leaf); + const node_t *subnode = page_node(sp, ii); + const size_t subnode_size = node_ks(subnode) + node_ds(subnode); + subheader_size += NODESIZE; + subpayload_size += subnode_size; + subalign_bytes += subnode_size & 1; + if (unlikely(node_flags(subnode) != 0)) { + ERROR("%s/%d: %s 0x%x", "MDBX_CORRUPTED", MDBX_CORRUPTED, "unexpected sub-node flags", node_flags(subnode)); + assert(err == MDBX_CORRUPTED); + err = MDBX_CORRUPTED; + } + } + } -#if MDBX_LOCKING == MDBX_LOCKING_POSIX2008 -#if defined(PTHREAD_MUTEX_ROBUST) || defined(pthread_mutexattr_setrobust) - rc = pthread_mutexattr_setrobust(&ma, PTHREAD_MUTEX_ROBUST); -#elif defined(PTHREAD_MUTEX_ROBUST_NP) || \ - defined(pthread_mutexattr_setrobust_np) - rc = pthread_mutexattr_setrobust_np(&ma, PTHREAD_MUTEX_ROBUST_NP); -#elif _POSIX_THREAD_PROCESS_SHARED < 200809L - rc = pthread_mutexattr_setrobust_np(&ma, PTHREAD_MUTEX_ROBUST_NP); -#else - rc = pthread_mutexattr_setrobust(&ma, PTHREAD_MUTEX_ROBUST); -#endif - if (rc) - goto bailout; -#endif /* MDBX_LOCKING == MDBX_LOCKING_POSIX2008 */ + const int rc = ctx->visitor(pgno, 0, ctx->userctx, ctx->deep + 1, tbl, node_data_size, subtype, err, nsubkeys, + subpayload_size, subheader_size, subunused_size + subalign_bytes); + if (unlikely(rc != MDBX_SUCCESS)) + return (rc == MDBX_RESULT_TRUE) ? MDBX_SUCCESS : rc; + header_size += subheader_size; + unused_size += subunused_size; + payload_size += subpayload_size; + align_bytes += subalign_bytes + (node_key_size & 1); + } break; -#if defined(_POSIX_THREAD_PRIO_INHERIT) && _POSIX_THREAD_PRIO_INHERIT >= 0 && \ - !defined(MDBX_SAFE4QEMU) - rc = pthread_mutexattr_setprotocol(&ma, PTHREAD_PRIO_INHERIT); - if (rc == ENOTSUP) - rc = pthread_mutexattr_setprotocol(&ma, PTHREAD_PRIO_NONE); - if (rc && rc != ENOTSUP) - goto bailout; -#endif /* PTHREAD_PRIO_INHERIT */ + default: + ERROR("%s/%d: %s 0x%x", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid node flags", node_flags(node)); + assert(err == MDBX_CORRUPTED); + err = MDBX_CORRUPTED; + } + } - rc = pthread_mutexattr_settype(&ma, PTHREAD_MUTEX_ERRORCHECK); - if (rc && rc != ENOTSUP) - goto bailout; + const int rc = ctx->visitor(pgno, 1, ctx->userctx, ctx->deep, tbl, ctx->txn->env->ps, type, err, nentries, + payload_size, header_size, unused_size + align_bytes); + if (unlikely(rc != MDBX_SUCCESS)) + return (rc == MDBX_RESULT_TRUE) ? MDBX_SUCCESS : rc; - rc = pthread_mutex_init(&env->me_lck_mmap.lck->mti_rlock, &ma); - if (rc) - goto bailout; - rc = pthread_mutex_init(&env->me_lck_mmap.lck->mti_wlock, &ma); + for (size_t i = 0; err == MDBX_SUCCESS && i < nentries; ++i) { + if (type == page_dupfix_leaf) + continue; -bailout: - pthread_mutexattr_destroy(&ma); - return rc; -#else -#error "FIXME" -#endif /* MDBX_LOCKING > 0 */ -} + node_t *node = page_node(mp, i); + if (type == page_branch) { + assert(err == MDBX_SUCCESS); + ctx->deep += 1; + err = walk_pgno(ctx, tbl, node_pgno(node), mp->txnid); + ctx->deep -= 1; + if (unlikely(err != MDBX_SUCCESS)) { + if (err == MDBX_RESULT_TRUE) + break; + return err; + } + continue; + } -__cold static int mdbx_ipclock_failed(MDBX_env *env, osal_ipclock_t *ipc, - const int err) { - int rc = err; -#if MDBX_LOCKING == MDBX_LOCKING_POSIX2008 || MDBX_LOCKING == MDBX_LOCKING_SYSV - if (err == EOWNERDEAD) { - /* We own the mutex. Clean up after dead previous owner. */ + assert(type == page_leaf); + switch (node_flags(node)) { + default: + continue; + + case N_TREE /* sub-db */: + if (unlikely(node_ds(node) != sizeof(tree_t))) { + ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid sub-tree node size", (unsigned)node_ds(node)); + assert(err == MDBX_CORRUPTED); + err = MDBX_CORRUPTED; + } else { + tree_t aligned_db; + memcpy(&aligned_db, node_data(node), sizeof(aligned_db)); + walk_tbl_t table = {{node_key(node), node_ks(node)}, nullptr, nullptr}; + table.internal = &aligned_db; + assert(err == MDBX_SUCCESS); + ctx->deep += 1; + err = walk_tbl(ctx, &table); + ctx->deep -= 1; + } + break; - const bool rlocked = ipc == &env->me_lck->mti_rlock; - rc = MDBX_SUCCESS; - if (!rlocked) { - if (unlikely(env->me_txn)) { - /* env is hosed if the dead thread was ours */ - env->me_flags |= MDBX_FATAL_ERROR; - env->me_txn = NULL; - rc = MDBX_PANIC; + case N_TREE | N_DUP /* dupsorted sub-tree */: + if (unlikely(node_ds(node) != sizeof(tree_t))) { + ERROR("%s/%d: %s %u", "MDBX_CORRUPTED", MDBX_CORRUPTED, "invalid dupsort sub-tree node size", + (unsigned)node_ds(node)); + assert(err == MDBX_CORRUPTED); + err = MDBX_CORRUPTED; + } else { + tree_t aligned_db; + memcpy(&aligned_db, node_data(node), sizeof(aligned_db)); + assert(err == MDBX_SUCCESS); + err = cursor_dupsort_setup(ctx->cursor, node, mp); + if (likely(err == MDBX_SUCCESS)) { + assert(ctx->cursor->subcur == &container_of(ctx->cursor, cursor_couple_t, outer)->inner); + ctx->cursor = &ctx->cursor->subcur->cursor; + ctx->deep += 1; + tbl->nested = &aligned_db; + err = walk_pgno(ctx, tbl, aligned_db.root, mp->txnid); + tbl->nested = nullptr; + ctx->deep -= 1; + subcur_t *inner_xcursor = container_of(ctx->cursor, subcur_t, cursor); + cursor_couple_t *couple = container_of(inner_xcursor, cursor_couple_t, inner); + ctx->cursor = &couple->outer; + } } + break; } - WARNING("%clock owner died, %s", (rlocked ? 'r' : 'w'), - (rc ? "this process' env is hosed" : "recovering")); - - int check_rc = cleanup_dead_readers(env, rlocked, NULL); - check_rc = (check_rc == MDBX_SUCCESS) ? MDBX_RESULT_TRUE : check_rc; + } -#if MDBX_LOCKING == MDBX_LOCKING_SYSV - rc = (rc == MDBX_SUCCESS) ? check_rc : rc; -#else -#if defined(PTHREAD_MUTEX_ROBUST) || defined(pthread_mutex_consistent) - int mreco_rc = pthread_mutex_consistent(ipc); -#elif defined(PTHREAD_MUTEX_ROBUST_NP) || defined(pthread_mutex_consistent_np) - int mreco_rc = pthread_mutex_consistent_np(ipc); -#elif _POSIX_THREAD_PROCESS_SHARED < 200809L - int mreco_rc = pthread_mutex_consistent_np(ipc); -#else - int mreco_rc = pthread_mutex_consistent(ipc); -#endif - check_rc = (mreco_rc == 0) ? check_rc : mreco_rc; + return MDBX_SUCCESS; +} - if (unlikely(mreco_rc)) - ERROR("lock recovery failed, %s", mdbx_strerror(mreco_rc)); +__cold static int walk_tbl(walk_ctx_t *ctx, walk_tbl_t *tbl) { + tree_t *const db = tbl->internal; + if (unlikely(db->root == P_INVALID)) + return MDBX_SUCCESS; /* empty db */ - rc = (rc == MDBX_SUCCESS) ? check_rc : rc; - if (MDBX_IS_ERROR(rc)) - pthread_mutex_unlock(ipc); -#endif /* MDBX_LOCKING == MDBX_LOCKING_POSIX2008 */ + kvx_t kvx = {.clc = {.k = {.lmin = INT_MAX}, .v = {.lmin = INT_MAX}}}; + cursor_couple_t couple; + int rc = cursor_init4walk(&couple, ctx->txn, db, &kvx); + if (unlikely(rc != MDBX_SUCCESS)) return rc; - } -#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 - (void)ipc; -#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988 - (void)ipc; -#elif MDBX_LOCKING == MDBX_LOCKING_FUTEX -#ifdef _MSC_VER -#pragma message("warning: TODO") -#else -#warning "TODO" -#endif - (void)ipc; -#else -#error "FIXME" -#endif /* MDBX_LOCKING */ -#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__) - if (rc == EDEADLK && atomic_load32(&env->me_ignore_EDEADLK, mo_Relaxed) > 0) + const uint8_t cursor_checking = (ctx->options & dont_check_keys_ordering) ? z_pagecheck | z_ignord : z_pagecheck; + couple.outer.checking |= cursor_checking; + couple.inner.cursor.checking |= cursor_checking; + couple.outer.next = ctx->cursor; + couple.outer.top_and_flags = z_disable_tree_search_fastpath; + ctx->cursor = &couple.outer; + rc = walk_pgno(ctx, tbl, db->root, db->mod_txnid ? db->mod_txnid : ctx->txn->txnid); + ctx->cursor = couple.outer.next; + return rc; +} + +__cold int walk_pages(MDBX_txn *txn, walk_func *visitor, void *user, walk_options_t options) { + int rc = check_txn(txn, MDBX_TXN_BLOCKED); + if (unlikely(rc != MDBX_SUCCESS)) return rc; -#endif /* MDBX_USE_VALGRIND || __SANITIZE_ADDRESS__ */ - ERROR("mutex (un)lock failed, %s", mdbx_strerror(err)); - if (rc != EDEADLK) - env->me_flags |= MDBX_FATAL_ERROR; + walk_ctx_t ctx = {.txn = txn, .userctx = user, .visitor = visitor, .options = options}; + walk_tbl_t tbl = {.name = {.iov_base = MDBX_CHK_GC}, .internal = &txn->dbs[FREE_DBI]}; + rc = walk_tbl(&ctx, &tbl); + if (!MDBX_IS_ERROR(rc)) { + tbl.name.iov_base = MDBX_CHK_MAIN; + tbl.internal = &txn->dbs[MAIN_DBI]; + rc = walk_tbl(&ctx, &tbl); + } return rc; } +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 -#if defined(__ANDROID_API__) || defined(ANDROID) || defined(BIONIC) -MDBX_INTERNAL_FUNC int osal_check_tid4bionic(void) { - /* avoid 32-bit Bionic bug/hang with 32-pit TID */ - if (sizeof(pthread_mutex_t) < sizeof(pid_t) + sizeof(unsigned)) { - pid_t tid = gettid(); - if (unlikely(tid > 0xffff)) { - FATAL("Raise the ENOSYS(%d) error to avoid hang due " - "the 32-bit Bionic/Android bug with tid/thread_id 0x%08x(%i) " - "that don’t fit in 16 bits, see " - "https://android.googlesource.com/platform/bionic/+/master/" - "docs/32-bit-abi.md#is-too-small-for-large-pids", - ENOSYS, tid, tid); - return ENOSYS; +#if defined(_WIN32) || defined(_WIN64) + +//------------------------------------------------------------------------------ +// Stub for slim read-write lock +// Portion Copyright (C) 1995-2002 Brad Wilson + +static void WINAPI stub_srwlock_Init(osal_srwlock_t *srwl) { srwl->readerCount = srwl->writerCount = 0; } + +static void WINAPI stub_srwlock_AcquireShared(osal_srwlock_t *srwl) { + while (true) { + assert(srwl->writerCount >= 0 && srwl->readerCount >= 0); + + // If there's a writer already, spin without unnecessarily + // interlocking the CPUs + if (srwl->writerCount != 0) { + SwitchToThread(); + continue; } + + // Add to the readers list + _InterlockedIncrement(&srwl->readerCount); + + // Check for writers again (we may have been preempted). If + // there are no writers writing or waiting, then we're done. + if (srwl->writerCount == 0) + break; + + // Remove from the readers list, spin, try again + _InterlockedDecrement(&srwl->readerCount); + SwitchToThread(); } - return 0; } -#endif /* __ANDROID_API__ || ANDROID) || BIONIC */ -static int mdbx_ipclock_lock(MDBX_env *env, osal_ipclock_t *ipc, - const bool dont_wait) { -#if MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ - MDBX_LOCKING == MDBX_LOCKING_POSIX2008 - int rc = osal_check_tid4bionic(); - if (likely(rc == 0)) - rc = dont_wait ? pthread_mutex_trylock(ipc) : pthread_mutex_lock(ipc); - rc = (rc == EBUSY && dont_wait) ? MDBX_BUSY : rc; -#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988 - int rc = MDBX_SUCCESS; - if (dont_wait) { - if (sem_trywait(ipc)) { - rc = errno; - if (rc == EAGAIN) - rc = MDBX_BUSY; +static void WINAPI stub_srwlock_ReleaseShared(osal_srwlock_t *srwl) { + assert(srwl->readerCount > 0); + _InterlockedDecrement(&srwl->readerCount); +} + +static void WINAPI stub_srwlock_AcquireExclusive(osal_srwlock_t *srwl) { + while (true) { + assert(srwl->writerCount >= 0 && srwl->readerCount >= 0); + + // If there's a writer already, spin without unnecessarily + // interlocking the CPUs + if (srwl->writerCount != 0) { + SwitchToThread(); + continue; } - } else if (sem_wait(ipc)) - rc = errno; -#elif MDBX_LOCKING == MDBX_LOCKING_SYSV - struct sembuf op = {.sem_num = (ipc != &env->me_lck->mti_wlock), - .sem_op = -1, - .sem_flg = dont_wait ? IPC_NOWAIT | SEM_UNDO : SEM_UNDO}; - int rc; - if (semop(env->me_sysv_ipc.semid, &op, 1)) { - rc = errno; - if (dont_wait && rc == EAGAIN) - rc = MDBX_BUSY; - } else { - rc = *ipc ? EOWNERDEAD : MDBX_SUCCESS; - *ipc = env->me_pid; + + // See if we can become the writer (expensive, because it inter- + // locks the CPUs, so writing should be an infrequent process) + if (_InterlockedExchange(&srwl->writerCount, 1) == 0) + break; } -#else -#error "FIXME" -#endif /* MDBX_LOCKING */ - if (unlikely(rc != MDBX_SUCCESS && rc != MDBX_BUSY)) - rc = mdbx_ipclock_failed(env, ipc, rc); - return rc; + // Now we're the writer, but there may be outstanding readers. + // Spin until there aren't any more; new readers will wait now + // that we're the writer. + while (srwl->readerCount != 0) { + assert(srwl->writerCount >= 0 && srwl->readerCount >= 0); + SwitchToThread(); + } } -static int mdbx_ipclock_unlock(MDBX_env *env, osal_ipclock_t *ipc) { -#if MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ - MDBX_LOCKING == MDBX_LOCKING_POSIX2008 - int rc = pthread_mutex_unlock(ipc); - (void)env; -#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988 - int rc = sem_post(ipc) ? errno : MDBX_SUCCESS; - (void)env; -#elif MDBX_LOCKING == MDBX_LOCKING_SYSV - if (unlikely(*ipc != (pid_t)env->me_pid)) - return EPERM; - *ipc = 0; - struct sembuf op = {.sem_num = (ipc != &env->me_lck->mti_wlock), - .sem_op = 1, - .sem_flg = SEM_UNDO}; - int rc = semop(env->me_sysv_ipc.semid, &op, 1) ? errno : MDBX_SUCCESS; -#else -#error "FIXME" -#endif /* MDBX_LOCKING */ - return rc; +static void WINAPI stub_srwlock_ReleaseExclusive(osal_srwlock_t *srwl) { + assert(srwl->writerCount == 1 && srwl->readerCount >= 0); + srwl->writerCount = 0; } -MDBX_INTERNAL_FUNC int osal_rdt_lock(MDBX_env *env) { - TRACE("%s", ">>"); - jitter4testing(true); - int rc = mdbx_ipclock_lock(env, &env->me_lck->mti_rlock, false); - TRACE("<< rc %d", rc); - return rc; +static uint64_t WINAPI stub_GetTickCount64(void) { + LARGE_INTEGER Counter, Frequency; + return (QueryPerformanceFrequency(&Frequency) && QueryPerformanceCounter(&Counter)) + ? Counter.QuadPart * 1000ul / Frequency.QuadPart + : 0; } -MDBX_INTERNAL_FUNC void osal_rdt_unlock(MDBX_env *env) { - TRACE("%s", ">>"); - int rc = mdbx_ipclock_unlock(env, &env->me_lck->mti_rlock); - TRACE("<< rc %d", rc); - if (unlikely(rc != MDBX_SUCCESS)) - mdbx_panic("%s() failed: err %d\n", __func__, rc); - jitter4testing(true); -} +//------------------------------------------------------------------------------ -int mdbx_txn_lock(MDBX_env *env, bool dont_wait) { - TRACE("%swait %s", dont_wait ? "dont-" : "", ">>"); - jitter4testing(true); - int rc = mdbx_ipclock_lock(env, &env->me_lck->mti_wlock, dont_wait); - TRACE("<< rc %d", rc); - return MDBX_IS_ERROR(rc) ? rc : MDBX_SUCCESS; -} +struct libmdbx_imports imports; -void mdbx_txn_unlock(MDBX_env *env) { - TRACE("%s", ">>"); - int rc = mdbx_ipclock_unlock(env, &env->me_lck->mti_wlock); - TRACE("<< rc %d", rc); - if (unlikely(rc != MDBX_SUCCESS)) - mdbx_panic("%s() failed: err %d\n", __func__, rc); - jitter4testing(true); +#if __GNUC_PREREQ(8, 0) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wcast-function-type" +#endif /* GCC/MINGW */ + +#define MDBX_IMPORT(HANDLE, ENTRY) imports.ENTRY = (MDBX_##ENTRY)GetProcAddress(HANDLE, #ENTRY) + +void windows_import(void) { + const HINSTANCE hNtdll = GetModuleHandleA("ntdll.dll"); + if (hNtdll) { + globals.running_under_Wine = !!GetProcAddress(hNtdll, "wine_get_version"); + if (!globals.running_under_Wine) { + MDBX_IMPORT(hNtdll, NtFsControlFile); + MDBX_IMPORT(hNtdll, NtExtendSection); + ENSURE(nullptr, imports.NtExtendSection); + } + } + + const HINSTANCE hKernel32dll = GetModuleHandleA("kernel32.dll"); + if (hKernel32dll) { + MDBX_IMPORT(hKernel32dll, GetFileInformationByHandleEx); + MDBX_IMPORT(hKernel32dll, GetTickCount64); + if (!imports.GetTickCount64) + imports.GetTickCount64 = stub_GetTickCount64; + if (!globals.running_under_Wine) { + MDBX_IMPORT(hKernel32dll, SetFileInformationByHandle); + MDBX_IMPORT(hKernel32dll, GetVolumeInformationByHandleW); + MDBX_IMPORT(hKernel32dll, GetFinalPathNameByHandleW); + MDBX_IMPORT(hKernel32dll, PrefetchVirtualMemory); + MDBX_IMPORT(hKernel32dll, SetFileIoOverlappedRange); + } + } + + const osal_srwlock_t_function srwlock_init = + (osal_srwlock_t_function)(hKernel32dll ? GetProcAddress(hKernel32dll, "InitializeSRWLock") : nullptr); + if (srwlock_init) { + imports.srwl_Init = srwlock_init; + imports.srwl_AcquireShared = (osal_srwlock_t_function)GetProcAddress(hKernel32dll, "AcquireSRWLockShared"); + imports.srwl_ReleaseShared = (osal_srwlock_t_function)GetProcAddress(hKernel32dll, "ReleaseSRWLockShared"); + imports.srwl_AcquireExclusive = (osal_srwlock_t_function)GetProcAddress(hKernel32dll, "AcquireSRWLockExclusive"); + imports.srwl_ReleaseExclusive = (osal_srwlock_t_function)GetProcAddress(hKernel32dll, "ReleaseSRWLockExclusive"); + } else { + imports.srwl_Init = stub_srwlock_Init; + imports.srwl_AcquireShared = stub_srwlock_AcquireShared; + imports.srwl_ReleaseShared = stub_srwlock_ReleaseShared; + imports.srwl_AcquireExclusive = stub_srwlock_AcquireExclusive; + imports.srwl_ReleaseExclusive = stub_srwlock_ReleaseExclusive; + } + + const HINSTANCE hAdvapi32dll = GetModuleHandleA("advapi32.dll"); + if (hAdvapi32dll) { + MDBX_IMPORT(hAdvapi32dll, RegGetValueA); + } + + const HINSTANCE hOle32dll = GetModuleHandleA("ole32.dll"); + if (hOle32dll) { + MDBX_IMPORT(hOle32dll, CoCreateGuid); + } } -#else -#ifdef _MSC_VER -#pragma warning(disable : 4206) /* nonstandard extension used: translation \ - unit is empty */ -#endif /* _MSC_VER (warnings) */ -#endif /* !Windows LCK-implementation */ +#undef MDBX_IMPORT + +#if __GNUC_PREREQ(8, 0) +#pragma GCC diagnostic pop +#endif /* GCC/MINGW */ + +#endif /* Windows */ +/* This is CMake-template for libmdbx's version.c + ******************************************************************************/ + +#if MDBX_VERSION_MAJOR != 0 || MDBX_VERSION_MINOR != 13 +#error "API version mismatch! Had `git fetch --tags` done?" +#endif + +static const char sourcery[] = MDBX_STRINGIFY(MDBX_BUILD_SOURCERY); + +__dll_export +#ifdef __attribute_used__ + __attribute_used__ +#elif defined(__GNUC__) || __has_attribute(__used__) + __attribute__((__used__)) +#endif +#ifdef __attribute_externally_visible__ + __attribute_externally_visible__ +#elif (defined(__GNUC__) && !defined(__clang__)) || __has_attribute(__externally_visible__) + __attribute__((__externally_visible__)) +#endif + const struct MDBX_version_info mdbx_version = { + 0, + 13, + 7, + 0, + "", /* pre-release suffix of SemVer + 0.13.7 */ + {"2025-07-30T11:44:04+03:00", "7777cbdf5aa4c1ce85ff902a4c3e6170edd42495", "566b0f93c7c9a3bdffb8fb3dc0ce8ca42641bd72", "v0.13.7-0-g566b0f93"}, + sourcery}; + +__dll_export +#ifdef __attribute_used__ + __attribute_used__ +#elif defined(__GNUC__) || __has_attribute(__used__) + __attribute__((__used__)) +#endif +#ifdef __attribute_externally_visible__ + __attribute_externally_visible__ +#elif (defined(__GNUC__) && !defined(__clang__)) || __has_attribute(__externally_visible__) + __attribute__((__externally_visible__)) +#endif + const char *const mdbx_sourcery_anchor = sourcery; diff --git a/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx.c++ b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx.c++ index acf99dcbfd7..27220d53088 100644 --- a/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx.c++ +++ b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx.c++ @@ -1,36 +1,24 @@ -/* - * Copyright 2015-2024 Leonid Yuriev - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . */ +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 +/* clang-format off */ -#define xMDBX_ALLOY 1 -#define MDBX_BUILD_SOURCERY e156c1a97c017ce89d6541cd9464ae5a9761d76b3fd2f1696521f5f3792904fc_v0_12_13_0_g1fff1f67 -#ifdef MDBX_CONFIG_H -#include MDBX_CONFIG_H -#endif +#define MDBX_BUILD_SOURCERY 6b5df6869d2bf5419e3a8189d9cc849cc9911b9c8a951b9750ed0a261ce43724_v0_13_7_0_g566b0f93 #define LIBMDBX_INTERNALS -#ifdef xMDBX_TOOLS #define MDBX_DEPRECATED -#endif /* xMDBX_TOOLS */ -#ifdef xMDBX_ALLOY -/* Amalgamated build */ -#define MDBX_INTERNAL_FUNC static -#define MDBX_INTERNAL_VAR static -#else -/* Non-amalgamated build */ -#define MDBX_INTERNAL_FUNC -#define MDBX_INTERNAL_VAR extern -#endif /* xMDBX_ALLOY */ +#ifdef MDBX_CONFIG_H +#include MDBX_CONFIG_H +#endif + +/* Undefine the NDEBUG if debugging is enforced by MDBX_DEBUG */ +#if (defined(MDBX_DEBUG) && MDBX_DEBUG > 0) || (defined(MDBX_FORCE_ASSERTIONS) && MDBX_FORCE_ASSERTIONS) +#undef NDEBUG +#ifndef MDBX_DEBUG +/* Чтобы избежать включения отладки только из-за включения assert-проверок */ +#define MDBX_DEBUG 0 +#endif +#endif /*----------------------------------------------------------------------------*/ @@ -48,14 +36,59 @@ #endif /* MDBX_DISABLE_GNU_SOURCE */ /* Should be defined before any includes */ -#if !defined(_FILE_OFFSET_BITS) && !defined(__ANDROID_API__) && \ - !defined(ANDROID) +#if !defined(_FILE_OFFSET_BITS) && !defined(__ANDROID_API__) && !defined(ANDROID) #define _FILE_OFFSET_BITS 64 -#endif +#endif /* _FILE_OFFSET_BITS */ -#ifdef __APPLE__ +#if defined(__APPLE__) && !defined(_DARWIN_C_SOURCE) #define _DARWIN_C_SOURCE -#endif +#endif /* _DARWIN_C_SOURCE */ + +#if (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) && !defined(__USE_MINGW_ANSI_STDIO) +#define __USE_MINGW_ANSI_STDIO 1 +#endif /* MinGW */ + +#if defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS) + +#ifndef _WIN32_WINNT +#define _WIN32_WINNT 0x0601 /* Windows 7 */ +#endif /* _WIN32_WINNT */ + +#if !defined(_CRT_SECURE_NO_WARNINGS) +#define _CRT_SECURE_NO_WARNINGS +#endif /* _CRT_SECURE_NO_WARNINGS */ +#if !defined(UNICODE) +#define UNICODE +#endif /* UNICODE */ + +#if !defined(_NO_CRT_STDIO_INLINE) && MDBX_BUILD_SHARED_LIBRARY && !defined(xMDBX_TOOLS) && MDBX_WITHOUT_MSVC_CRT +#define _NO_CRT_STDIO_INLINE +#endif /* _NO_CRT_STDIO_INLINE */ + +#elif !defined(_POSIX_C_SOURCE) +#define _POSIX_C_SOURCE 200809L +#endif /* Windows */ + +#ifdef __cplusplus + +#ifndef NOMINMAX +#define NOMINMAX +#endif /* NOMINMAX */ + +/* Workaround for modern libstdc++ with CLANG < 4.x */ +#if defined(__SIZEOF_INT128__) && !defined(__GLIBCXX_TYPE_INT_N_0) && defined(__clang__) && __clang_major__ < 4 +#define __GLIBCXX_BITSIZE_INT_N_0 128 +#define __GLIBCXX_TYPE_INT_N_0 __int128 +#endif /* Workaround for modern libstdc++ with CLANG < 4.x */ + +#ifdef _MSC_VER +/* Workaround for MSVC' header `extern "C"` vs `std::` redefinition bug */ +#if defined(__SANITIZE_ADDRESS__) && !defined(_DISABLE_VECTOR_ANNOTATION) +#define _DISABLE_VECTOR_ANNOTATION +#endif /* _DISABLE_VECTOR_ANNOTATION */ +#endif /* _MSC_VER */ + +#endif /* __cplusplus */ #ifdef _MSC_VER #if _MSC_FULL_VER < 190024234 @@ -77,12 +110,8 @@ * and how to and where you can obtain the latest "Visual Studio 2015" build * with all fixes. */ -#error \ - "At least \"Microsoft C/C++ Compiler\" version 19.00.24234 (Visual Studio 2015 Update 3) is required." +#error "At least \"Microsoft C/C++ Compiler\" version 19.00.24234 (Visual Studio 2015 Update 3) is required." #endif -#ifndef _CRT_SECURE_NO_WARNINGS -#define _CRT_SECURE_NO_WARNINGS -#endif /* _CRT_SECURE_NO_WARNINGS */ #if _MSC_VER > 1800 #pragma warning(disable : 4464) /* relative include path contains '..' */ #endif @@ -90,124 +119,78 @@ #pragma warning(disable : 5045) /* will insert Spectre mitigation... */ #endif #if _MSC_VER > 1914 -#pragma warning( \ - disable : 5105) /* winbase.h(9531): warning C5105: macro expansion \ - producing 'defined' has undefined behavior */ +#pragma warning(disable : 5105) /* winbase.h(9531): warning C5105: macro expansion \ + producing 'defined' has undefined behavior */ +#endif +#if _MSC_VER < 1920 +/* avoid "error C2219: syntax error: type qualifier must be after '*'" */ +#define __restrict #endif #if _MSC_VER > 1930 #pragma warning(disable : 6235) /* is always a constant */ -#pragma warning(disable : 6237) /* is never evaluated and might \ +#pragma warning(disable : 6237) /* is never evaluated and might \ have side effects */ +#pragma warning(disable : 5286) /* implicit conversion from enum type 'type 1' to enum type 'type 2' */ +#pragma warning(disable : 5287) /* operands are different enum types 'type 1' and 'type 2' */ #endif #pragma warning(disable : 4710) /* 'xyz': function not inlined */ -#pragma warning(disable : 4711) /* function 'xyz' selected for automatic \ +#pragma warning(disable : 4711) /* function 'xyz' selected for automatic \ inline expansion */ -#pragma warning(disable : 4201) /* nonstandard extension used: nameless \ +#pragma warning(disable : 4201) /* nonstandard extension used: nameless \ struct/union */ #pragma warning(disable : 4702) /* unreachable code */ #pragma warning(disable : 4706) /* assignment within conditional expression */ #pragma warning(disable : 4127) /* conditional expression is constant */ -#pragma warning(disable : 4324) /* 'xyz': structure was padded due to \ +#pragma warning(disable : 4324) /* 'xyz': structure was padded due to \ alignment specifier */ #pragma warning(disable : 4310) /* cast truncates constant value */ -#pragma warning(disable : 4820) /* bytes padding added after data member for \ +#pragma warning(disable : 4820) /* bytes padding added after data member for \ alignment */ -#pragma warning(disable : 4548) /* expression before comma has no effect; \ +#pragma warning(disable : 4548) /* expression before comma has no effect; \ expected expression with side - effect */ -#pragma warning(disable : 4366) /* the result of the unary '&' operator may be \ +#pragma warning(disable : 4366) /* the result of the unary '&' operator may be \ unaligned */ -#pragma warning(disable : 4200) /* nonstandard extension used: zero-sized \ +#pragma warning(disable : 4200) /* nonstandard extension used: zero-sized \ array in struct/union */ -#pragma warning(disable : 4204) /* nonstandard extension used: non-constant \ +#pragma warning(disable : 4204) /* nonstandard extension used: non-constant \ aggregate initializer */ -#pragma warning( \ - disable : 4505) /* unreferenced local function has been removed */ -#endif /* _MSC_VER (warnings) */ +#pragma warning(disable : 4505) /* unreferenced local function has been removed */ +#endif /* _MSC_VER (warnings) */ #if defined(__GNUC__) && __GNUC__ < 9 #pragma GCC diagnostic ignored "-Wattributes" #endif /* GCC < 9 */ -#if (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) && \ - !defined(__USE_MINGW_ANSI_STDIO) -#define __USE_MINGW_ANSI_STDIO 1 -#endif /* MinGW */ - -#if (defined(_WIN32) || defined(_WIN64)) && !defined(UNICODE) -#define UNICODE -#endif /* UNICODE */ - -#include "mdbx.h++" -/* - * Copyright 2015-2024 Leonid Yuriev - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - */ - - /*----------------------------------------------------------------------------*/ /* Microsoft compiler generates a lot of warning for self includes... */ #ifdef _MSC_VER #pragma warning(push, 1) -#pragma warning(disable : 4548) /* expression before comma has no effect; \ +#pragma warning(disable : 4548) /* expression before comma has no effect; \ expected expression with side - effect */ -#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ +#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ * semantics are not enabled. Specify /EHsc */ -#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ - * mode specified; termination on exception is \ +#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ + * mode specified; termination on exception is \ * not guaranteed. Specify /EHsc */ #endif /* _MSC_VER (warnings) */ -#if defined(_WIN32) || defined(_WIN64) -#if !defined(_CRT_SECURE_NO_WARNINGS) -#define _CRT_SECURE_NO_WARNINGS -#endif /* _CRT_SECURE_NO_WARNINGS */ -#if !defined(_NO_CRT_STDIO_INLINE) && MDBX_BUILD_SHARED_LIBRARY && \ - !defined(xMDBX_TOOLS) && MDBX_WITHOUT_MSVC_CRT -#define _NO_CRT_STDIO_INLINE -#endif -#elif !defined(_POSIX_C_SOURCE) -#define _POSIX_C_SOURCE 200809L -#endif /* Windows */ - /*----------------------------------------------------------------------------*/ /* basic C99 includes */ + #include #include #include #include #include +#include #include #include #include #include #include -#if (-6 & 5) || CHAR_BIT != 8 || UINT_MAX < 0xffffffff || ULONG_MAX % 0xFFFF -#error \ - "Sanity checking failed: Two's complement, reasonably sized integer types" -#endif - -#ifndef SSIZE_MAX -#define SSIZE_MAX INTPTR_MAX -#endif - -#if UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul || defined(_WIN64) -#define MDBX_WORDBITS 64 -#else -#define MDBX_WORDBITS 32 -#endif /* MDBX_WORDBITS */ - /*----------------------------------------------------------------------------*/ /* feature testing */ @@ -219,6 +202,14 @@ #define __has_include(x) (0) #endif +#ifndef __has_attribute +#define __has_attribute(x) (0) +#endif + +#ifndef __has_cpp_attribute +#define __has_cpp_attribute(x) 0 +#endif + #ifndef __has_feature #define __has_feature(x) (0) #endif @@ -241,8 +232,7 @@ #ifndef __GNUC_PREREQ #if defined(__GNUC__) && defined(__GNUC_MINOR__) -#define __GNUC_PREREQ(maj, min) \ - ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min)) +#define __GNUC_PREREQ(maj, min) ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min)) #else #define __GNUC_PREREQ(maj, min) (0) #endif @@ -250,8 +240,7 @@ #ifndef __CLANG_PREREQ #ifdef __clang__ -#define __CLANG_PREREQ(maj, min) \ - ((__clang_major__ << 16) + __clang_minor__ >= ((maj) << 16) + (min)) +#define __CLANG_PREREQ(maj, min) ((__clang_major__ << 16) + __clang_minor__ >= ((maj) << 16) + (min)) #else #define __CLANG_PREREQ(maj, min) (0) #endif @@ -259,13 +248,51 @@ #ifndef __GLIBC_PREREQ #if defined(__GLIBC__) && defined(__GLIBC_MINOR__) -#define __GLIBC_PREREQ(maj, min) \ - ((__GLIBC__ << 16) + __GLIBC_MINOR__ >= ((maj) << 16) + (min)) +#define __GLIBC_PREREQ(maj, min) ((__GLIBC__ << 16) + __GLIBC_MINOR__ >= ((maj) << 16) + (min)) #else #define __GLIBC_PREREQ(maj, min) (0) #endif #endif /* __GLIBC_PREREQ */ +/*----------------------------------------------------------------------------*/ +/* pre-requirements */ + +#if (-6 & 5) || CHAR_BIT != 8 || UINT_MAX < 0xffffffff || ULONG_MAX % 0xFFFF +#error "Sanity checking failed: Two's complement, reasonably sized integer types" +#endif + +#ifndef SSIZE_MAX +#define SSIZE_MAX INTPTR_MAX +#endif + +#if defined(__GNUC__) && !__GNUC_PREREQ(4, 2) +/* Actually libmdbx was not tested with compilers older than GCC 4.2. + * But you could ignore this warning at your own risk. + * In such case please don't rise up an issues related ONLY to old compilers. + */ +#warning "libmdbx required GCC >= 4.2" +#endif + +#if defined(__clang__) && !__CLANG_PREREQ(3, 8) +/* Actually libmdbx was not tested with CLANG older than 3.8. + * But you could ignore this warning at your own risk. + * In such case please don't rise up an issues related ONLY to old compilers. + */ +#warning "libmdbx required CLANG >= 3.8" +#endif + +#if defined(__GLIBC__) && !__GLIBC_PREREQ(2, 12) +/* Actually libmdbx was not tested with something older than glibc 2.12. + * But you could ignore this warning at your own risk. + * In such case please don't rise up an issues related ONLY to old systems. + */ +#warning "libmdbx was only tested with GLIBC >= 2.12." +#endif + +#ifdef __SANITIZE_THREAD__ +#warning "libmdbx don't compatible with ThreadSanitizer, you will get a lot of false-positive issues." +#endif /* __SANITIZE_THREAD__ */ + /*----------------------------------------------------------------------------*/ /* C11' alignas() */ @@ -295,8 +322,7 @@ #endif #endif /* __extern_C */ -#if !defined(nullptr) && !defined(__cplusplus) || \ - (__cplusplus < 201103L && !defined(_MSC_VER)) +#if !defined(nullptr) && !defined(__cplusplus) || (__cplusplus < 201103L && !defined(_MSC_VER)) #define nullptr NULL #endif @@ -308,9 +334,8 @@ #endif #endif /* Apple OSX & iOS */ -#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \ - defined(__BSD__) || defined(__bsdi__) || defined(__DragonFly__) || \ - defined(__APPLE__) || defined(__MACH__) +#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__BSD__) || defined(__bsdi__) || \ + defined(__DragonFly__) || defined(__APPLE__) || defined(__MACH__) #include #include #include @@ -327,8 +352,7 @@ #endif #else #include -#if !(defined(__sun) || defined(__SVR4) || defined(__svr4__) || \ - defined(_WIN32) || defined(_WIN64)) +#if !(defined(__sun) || defined(__SVR4) || defined(__svr4__) || defined(_WIN32) || defined(_WIN64)) #include #endif /* !Solaris */ #endif /* !xBSD */ @@ -382,12 +406,14 @@ __extern_C key_t ftok(const char *, int); #ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN #endif /* WIN32_LEAN_AND_MEAN */ -#include -#include #include #include #include +/* После подгрузки windows.h, чтобы избежать проблем со сборкой MINGW и т.п. */ +#include +#include + #else /*----------------------------------------------------------------------*/ #include @@ -415,11 +441,6 @@ __extern_C key_t ftok(const char *, int); #if __ANDROID_API__ >= 21 #include #endif -#if defined(_FILE_OFFSET_BITS) && _FILE_OFFSET_BITS != MDBX_WORDBITS -#error "_FILE_OFFSET_BITS != MDBX_WORDBITS" (_FILE_OFFSET_BITS != MDBX_WORDBITS) -#elif defined(__FILE_OFFSET_BITS) && __FILE_OFFSET_BITS != MDBX_WORDBITS -#error "__FILE_OFFSET_BITS != MDBX_WORDBITS" (__FILE_OFFSET_BITS != MDBX_WORDBITS) -#endif #endif /* Android */ #if defined(HAVE_SYS_STAT_H) || __has_include() @@ -435,43 +456,38 @@ __extern_C key_t ftok(const char *, int); /*----------------------------------------------------------------------------*/ /* Byteorder */ -#if defined(i386) || defined(__386) || defined(__i386) || defined(__i386__) || \ - defined(i486) || defined(__i486) || defined(__i486__) || defined(i586) || \ - defined(__i586) || defined(__i586__) || defined(i686) || \ - defined(__i686) || defined(__i686__) || defined(_M_IX86) || \ - defined(_X86_) || defined(__THW_INTEL__) || defined(__I86__) || \ - defined(__INTEL__) || defined(__x86_64) || defined(__x86_64__) || \ - defined(__amd64__) || defined(__amd64) || defined(_M_X64) || \ - defined(_M_AMD64) || defined(__IA32__) || defined(__INTEL__) +#if defined(i386) || defined(__386) || defined(__i386) || defined(__i386__) || defined(i486) || defined(__i486) || \ + defined(__i486__) || defined(i586) || defined(__i586) || defined(__i586__) || defined(i686) || defined(__i686) || \ + defined(__i686__) || defined(_M_IX86) || defined(_X86_) || defined(__THW_INTEL__) || defined(__I86__) || \ + defined(__INTEL__) || defined(__x86_64) || defined(__x86_64__) || defined(__amd64__) || defined(__amd64) || \ + defined(_M_X64) || defined(_M_AMD64) || defined(__IA32__) || defined(__INTEL__) #ifndef __ia32__ /* LY: define neutral __ia32__ for x86 and x86-64 */ #define __ia32__ 1 #endif /* __ia32__ */ -#if !defined(__amd64__) && \ - (defined(__x86_64) || defined(__x86_64__) || defined(__amd64) || \ - defined(_M_X64) || defined(_M_AMD64)) +#if !defined(__amd64__) && \ + (defined(__x86_64) || defined(__x86_64__) || defined(__amd64) || defined(_M_X64) || defined(_M_AMD64)) /* LY: define trusty __amd64__ for all AMD64/x86-64 arch */ #define __amd64__ 1 #endif /* __amd64__ */ #endif /* all x86 */ -#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || \ - !defined(__ORDER_BIG_ENDIAN__) +#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || !defined(__ORDER_BIG_ENDIAN__) -#if defined(__GLIBC__) || defined(__GNU_LIBRARY__) || \ - defined(__ANDROID_API__) || defined(HAVE_ENDIAN_H) || __has_include() +#if defined(__GLIBC__) || defined(__GNU_LIBRARY__) || defined(__ANDROID_API__) || defined(HAVE_ENDIAN_H) || \ + __has_include() #include -#elif defined(__APPLE__) || defined(__MACH__) || defined(__OpenBSD__) || \ - defined(HAVE_MACHINE_ENDIAN_H) || __has_include() +#elif defined(__APPLE__) || defined(__MACH__) || defined(__OpenBSD__) || defined(HAVE_MACHINE_ENDIAN_H) || \ + __has_include() #include #elif defined(HAVE_SYS_ISA_DEFS_H) || __has_include() #include -#elif (defined(HAVE_SYS_TYPES_H) && defined(HAVE_SYS_ENDIAN_H)) || \ +#elif (defined(HAVE_SYS_TYPES_H) && defined(HAVE_SYS_ENDIAN_H)) || \ (__has_include() && __has_include()) #include #include -#elif defined(__bsdi__) || defined(__DragonFly__) || defined(__FreeBSD__) || \ - defined(__NetBSD__) || defined(HAVE_SYS_PARAM_H) || __has_include() +#elif defined(__bsdi__) || defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || \ + defined(HAVE_SYS_PARAM_H) || __has_include() #include #endif /* OS */ @@ -487,27 +503,19 @@ __extern_C key_t ftok(const char *, int); #define __ORDER_LITTLE_ENDIAN__ 1234 #define __ORDER_BIG_ENDIAN__ 4321 -#if defined(__LITTLE_ENDIAN__) || \ - (defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)) || \ - defined(__ARMEL__) || defined(__THUMBEL__) || defined(__AARCH64EL__) || \ - defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \ - defined(_M_ARM) || defined(_M_ARM64) || defined(__e2k__) || \ - defined(__elbrus_4c__) || defined(__elbrus_8c__) || defined(__bfin__) || \ - defined(__BFIN__) || defined(__ia64__) || defined(_IA64) || \ - defined(__IA64__) || defined(__ia64) || defined(_M_IA64) || \ - defined(__itanium__) || defined(__ia32__) || defined(__CYGWIN__) || \ - defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) || \ - defined(__WINDOWS__) +#if defined(__LITTLE_ENDIAN__) || (defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)) || defined(__ARMEL__) || \ + defined(__THUMBEL__) || defined(__AARCH64EL__) || defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \ + defined(_M_ARM) || defined(_M_ARM64) || defined(__e2k__) || defined(__elbrus_4c__) || defined(__elbrus_8c__) || \ + defined(__bfin__) || defined(__BFIN__) || defined(__ia64__) || defined(_IA64) || defined(__IA64__) || \ + defined(__ia64) || defined(_M_IA64) || defined(__itanium__) || defined(__ia32__) || defined(__CYGWIN__) || \ + defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) || defined(__WINDOWS__) #define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__ -#elif defined(__BIG_ENDIAN__) || \ - (defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) || \ - defined(__ARMEB__) || defined(__THUMBEB__) || defined(__AARCH64EB__) || \ - defined(__MIPSEB__) || defined(_MIPSEB) || defined(__MIPSEB) || \ - defined(__m68k__) || defined(M68000) || defined(__hppa__) || \ - defined(__hppa) || defined(__HPPA__) || defined(__sparc__) || \ - defined(__sparc) || defined(__370__) || defined(__THW_370__) || \ - defined(__s390__) || defined(__s390x__) || defined(__SYSC_ZARCH__) +#elif defined(__BIG_ENDIAN__) || (defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) || defined(__ARMEB__) || \ + defined(__THUMBEB__) || defined(__AARCH64EB__) || defined(__MIPSEB__) || defined(_MIPSEB) || defined(__MIPSEB) || \ + defined(__m68k__) || defined(M68000) || defined(__hppa__) || defined(__hppa) || defined(__HPPA__) || \ + defined(__sparc__) || defined(__sparc) || defined(__370__) || defined(__THW_370__) || defined(__s390__) || \ + defined(__s390x__) || defined(__SYSC_ZARCH__) #define __BYTE_ORDER__ __ORDER_BIG_ENDIAN__ #else @@ -517,6 +525,12 @@ __extern_C key_t ftok(const char *, int); #endif #endif /* __BYTE_ORDER__ || __ORDER_LITTLE_ENDIAN__ || __ORDER_BIG_ENDIAN__ */ +#if UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul || defined(_WIN64) +#define MDBX_WORDBITS 64 +#else +#define MDBX_WORDBITS 32 +#endif /* MDBX_WORDBITS */ + /*----------------------------------------------------------------------------*/ /* Availability of CMOV or equivalent */ @@ -527,17 +541,14 @@ __extern_C key_t ftok(const char *, int); #define MDBX_HAVE_CMOV 1 #elif defined(__thumb__) || defined(__thumb) || defined(__TARGET_ARCH_THUMB) #define MDBX_HAVE_CMOV 0 -#elif defined(_M_ARM) || defined(_M_ARM64) || defined(__aarch64__) || \ - defined(__aarch64) || defined(__arm__) || defined(__arm) || \ - defined(__CC_ARM) +#elif defined(_M_ARM) || defined(_M_ARM64) || defined(__aarch64__) || defined(__aarch64) || defined(__arm__) || \ + defined(__arm) || defined(__CC_ARM) #define MDBX_HAVE_CMOV 1 -#elif (defined(__riscv__) || defined(__riscv64)) && \ - (defined(__riscv_b) || defined(__riscv_bitmanip)) +#elif (defined(__riscv__) || defined(__riscv64)) && (defined(__riscv_b) || defined(__riscv_bitmanip)) #define MDBX_HAVE_CMOV 1 -#elif defined(i686) || defined(__i686) || defined(__i686__) || \ - (defined(_M_IX86) && _M_IX86 > 600) || defined(__x86_64) || \ - defined(__x86_64__) || defined(__amd64__) || defined(__amd64) || \ - defined(_M_X64) || defined(_M_AMD64) +#elif defined(i686) || defined(__i686) || defined(__i686__) || (defined(_M_IX86) && _M_IX86 > 600) || \ + defined(__x86_64) || defined(__x86_64__) || defined(__amd64__) || defined(__amd64) || defined(_M_X64) || \ + defined(_M_AMD64) #define MDBX_HAVE_CMOV 1 #else #define MDBX_HAVE_CMOV 0 @@ -563,8 +574,7 @@ __extern_C key_t ftok(const char *, int); #endif #elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) #include -#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ - (defined(HP_IA64) || defined(__ia64)) +#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && (defined(HP_IA64) || defined(__ia64)) #include #elif defined(__IBMC__) && defined(__powerpc) #include @@ -586,29 +596,26 @@ __extern_C key_t ftok(const char *, int); #endif /* Compiler */ #if !defined(__noop) && !defined(_MSC_VER) -#define __noop \ - do { \ +#define __noop \ + do { \ } while (0) #endif /* __noop */ -#if defined(__fallthrough) && \ - (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) +#if defined(__fallthrough) && (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) #undef __fallthrough #endif /* __fallthrough workaround for MinGW */ #ifndef __fallthrough -#if defined(__cplusplus) && (__has_cpp_attribute(fallthrough) && \ - (!defined(__clang__) || __clang__ > 4)) || \ +#if defined(__cplusplus) && (__has_cpp_attribute(fallthrough) && (!defined(__clang__) || __clang__ > 4)) || \ __cplusplus >= 201703L #define __fallthrough [[fallthrough]] #elif __GNUC_PREREQ(8, 0) && defined(__cplusplus) && __cplusplus >= 201103L #define __fallthrough [[fallthrough]] -#elif __GNUC_PREREQ(7, 0) && \ - (!defined(__LCC__) || (__LCC__ == 124 && __LCC_MINOR__ >= 12) || \ - (__LCC__ == 125 && __LCC_MINOR__ >= 5) || (__LCC__ >= 126)) +#elif __GNUC_PREREQ(7, 0) && (!defined(__LCC__) || (__LCC__ == 124 && __LCC_MINOR__ >= 12) || \ + (__LCC__ == 125 && __LCC_MINOR__ >= 5) || (__LCC__ >= 126)) #define __fallthrough __attribute__((__fallthrough__)) -#elif defined(__clang__) && defined(__cplusplus) && __cplusplus >= 201103L && \ - __has_feature(cxx_attributes) && __has_warning("-Wimplicit-fallthrough") +#elif defined(__clang__) && defined(__cplusplus) && __cplusplus >= 201103L && __has_feature(cxx_attributes) && \ + __has_warning("-Wimplicit-fallthrough") #define __fallthrough [[clang::fallthrough]] #else #define __fallthrough @@ -621,8 +628,8 @@ __extern_C key_t ftok(const char *, int); #elif defined(_MSC_VER) #define __unreachable() __assume(0) #else -#define __unreachable() \ - do { \ +#define __unreachable() \ + do { \ } while (1) #endif #endif /* __unreachable */ @@ -631,9 +638,9 @@ __extern_C key_t ftok(const char *, int); #if defined(__GNUC__) || defined(__clang__) || __has_builtin(__builtin_prefetch) #define __prefetch(ptr) __builtin_prefetch(ptr) #else -#define __prefetch(ptr) \ - do { \ - (void)(ptr); \ +#define __prefetch(ptr) \ + do { \ + (void)(ptr); \ } while (0) #endif #endif /* __prefetch */ @@ -643,11 +650,11 @@ __extern_C key_t ftok(const char *, int); #endif /* offsetof */ #ifndef container_of -#define container_of(ptr, type, member) \ - ((type *)((char *)(ptr) - offsetof(type, member))) +#define container_of(ptr, type, member) ((type *)((char *)(ptr) - offsetof(type, member))) #endif /* container_of */ /*----------------------------------------------------------------------------*/ +/* useful attributes */ #ifndef __always_inline #if defined(__GNUC__) || __has_attribute(__always_inline__) @@ -715,8 +722,7 @@ __extern_C key_t ftok(const char *, int); #ifndef __hot #if defined(__OPTIMIZE__) -#if defined(__clang__) && !__has_attribute(__hot__) && \ - __has_attribute(__section__) && \ +#if defined(__clang__) && !__has_attribute(__hot__) && __has_attribute(__section__) && \ (defined(__linux__) || defined(__gnu_linux__)) /* just put frequently used functions in separate section */ #define __hot __attribute__((__section__("text.hot"))) __optimize("O3") @@ -732,8 +738,7 @@ __extern_C key_t ftok(const char *, int); #ifndef __cold #if defined(__OPTIMIZE__) -#if defined(__clang__) && !__has_attribute(__cold__) && \ - __has_attribute(__section__) && \ +#if defined(__clang__) && !__has_attribute(__cold__) && __has_attribute(__section__) && \ (defined(__linux__) || defined(__gnu_linux__)) /* just put infrequently used functions in separate section */ #define __cold __attribute__((__section__("text.unlikely"))) __optimize("Os") @@ -756,8 +761,7 @@ __extern_C key_t ftok(const char *, int); #endif /* __flatten */ #ifndef likely -#if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && \ - !defined(__COVERITY__) +#if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__) #define likely(cond) __builtin_expect(!!(cond), 1) #else #define likely(x) (!!(x)) @@ -765,8 +769,7 @@ __extern_C key_t ftok(const char *, int); #endif /* likely */ #ifndef unlikely -#if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && \ - !defined(__COVERITY__) +#if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__) #define unlikely(cond) __builtin_expect(!!(cond), 0) #else #define unlikely(x) (!!(x)) @@ -781,29 +784,41 @@ __extern_C key_t ftok(const char *, int); #endif #endif /* __anonymous_struct_extension__ */ -#ifndef expect_with_probability -#if defined(__builtin_expect_with_probability) || \ - __has_builtin(__builtin_expect_with_probability) || __GNUC_PREREQ(9, 0) -#define expect_with_probability(expr, value, prob) \ - __builtin_expect_with_probability(expr, value, prob) -#else -#define expect_with_probability(expr, value, prob) (expr) -#endif -#endif /* expect_with_probability */ - #ifndef MDBX_WEAK_IMPORT_ATTRIBUTE #ifdef WEAK_IMPORT_ATTRIBUTE #define MDBX_WEAK_IMPORT_ATTRIBUTE WEAK_IMPORT_ATTRIBUTE #elif __has_attribute(__weak__) && __has_attribute(__weak_import__) #define MDBX_WEAK_IMPORT_ATTRIBUTE __attribute__((__weak__, __weak_import__)) -#elif __has_attribute(__weak__) || \ - (defined(__GNUC__) && __GNUC__ >= 4 && defined(__ELF__)) +#elif __has_attribute(__weak__) || (defined(__GNUC__) && __GNUC__ >= 4 && defined(__ELF__)) #define MDBX_WEAK_IMPORT_ATTRIBUTE __attribute__((__weak__)) #else #define MDBX_WEAK_IMPORT_ATTRIBUTE #endif #endif /* MDBX_WEAK_IMPORT_ATTRIBUTE */ +#if !defined(__thread) && (defined(_MSC_VER) || defined(__DMC__)) +#define __thread __declspec(thread) +#endif /* __thread */ + +#ifndef MDBX_EXCLUDE_FOR_GPROF +#ifdef ENABLE_GPROF +#define MDBX_EXCLUDE_FOR_GPROF __attribute__((__no_instrument_function__, __no_profile_instrument_function__)) +#else +#define MDBX_EXCLUDE_FOR_GPROF +#endif /* ENABLE_GPROF */ +#endif /* MDBX_EXCLUDE_FOR_GPROF */ + +/*----------------------------------------------------------------------------*/ + +#ifndef expect_with_probability +#if defined(__builtin_expect_with_probability) || __has_builtin(__builtin_expect_with_probability) || \ + __GNUC_PREREQ(9, 0) +#define expect_with_probability(expr, value, prob) __builtin_expect_with_probability(expr, value, prob) +#else +#define expect_with_probability(expr, value, prob) (expr) +#endif +#endif /* expect_with_probability */ + #ifndef MDBX_GOOFY_MSVC_STATIC_ANALYZER #ifdef _PREFAST_ #define MDBX_GOOFY_MSVC_STATIC_ANALYZER 1 @@ -815,20 +830,27 @@ __extern_C key_t ftok(const char *, int); #if MDBX_GOOFY_MSVC_STATIC_ANALYZER || (defined(_MSC_VER) && _MSC_VER > 1919) #define MDBX_ANALYSIS_ASSUME(expr) __analysis_assume(expr) #ifdef _PREFAST_ -#define MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(warn_id) \ - __pragma(prefast(suppress : warn_id)) +#define MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(warn_id) __pragma(prefast(suppress : warn_id)) #else -#define MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(warn_id) \ - __pragma(warning(suppress : warn_id)) +#define MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(warn_id) __pragma(warning(suppress : warn_id)) #endif #else #define MDBX_ANALYSIS_ASSUME(expr) assert(expr) #define MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(warn_id) #endif /* MDBX_GOOFY_MSVC_STATIC_ANALYZER */ +#ifndef FLEXIBLE_ARRAY_MEMBERS +#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || (!defined(__cplusplus) && defined(_MSC_VER)) +#define FLEXIBLE_ARRAY_MEMBERS 1 +#else +#define FLEXIBLE_ARRAY_MEMBERS 0 +#endif +#endif /* FLEXIBLE_ARRAY_MEMBERS */ + /*----------------------------------------------------------------------------*/ +/* Valgrind and Address Sanitizer */ -#if defined(MDBX_USE_VALGRIND) +#if defined(ENABLE_MEMCHECK) #include #ifndef VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE /* LY: available since Valgrind 3.10 */ @@ -850,7 +872,7 @@ __extern_C key_t ftok(const char *, int); #define VALGRIND_CHECK_MEM_IS_ADDRESSABLE(a, s) (0) #define VALGRIND_CHECK_MEM_IS_DEFINED(a, s) (0) #define RUNNING_ON_VALGRIND (0) -#endif /* MDBX_USE_VALGRIND */ +#endif /* ENABLE_MEMCHECK */ #ifdef __SANITIZE_ADDRESS__ #include @@ -877,8 +899,7 @@ template char (&__ArraySizeHelper(T (&array)[N]))[N]; #define CONCAT(a, b) a##b #define XCONCAT(a, b) CONCAT(a, b) -#define MDBX_TETRAD(a, b, c, d) \ - ((uint32_t)(a) << 24 | (uint32_t)(b) << 16 | (uint32_t)(c) << 8 | (d)) +#define MDBX_TETRAD(a, b, c, d) ((uint32_t)(a) << 24 | (uint32_t)(b) << 16 | (uint32_t)(c) << 8 | (d)) #define MDBX_STRING_TETRAD(str) MDBX_TETRAD(str[0], str[1], str[2], str[3]) @@ -892,14 +913,13 @@ template char (&__ArraySizeHelper(T (&array)[N]))[N]; #elif defined(_MSC_VER) #include #define STATIC_ASSERT_MSG(expr, msg) _STATIC_ASSERT(expr) -#elif (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) || \ - __has_feature(c_static_assert) +#elif (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) || __has_feature(c_static_assert) #define STATIC_ASSERT_MSG(expr, msg) _Static_assert(expr, msg) #else -#define STATIC_ASSERT_MSG(expr, msg) \ - switch (0) { \ - case 0: \ - case (expr):; \ +#define STATIC_ASSERT_MSG(expr, msg) \ + switch (0) { \ + case 0: \ + case (expr):; \ } #endif #endif /* STATIC_ASSERT */ @@ -908,42 +928,37 @@ template char (&__ArraySizeHelper(T (&array)[N]))[N]; #define STATIC_ASSERT(expr) STATIC_ASSERT_MSG(expr, #expr) #endif -#ifndef __Wpedantic_format_voidptr -MDBX_MAYBE_UNUSED MDBX_PURE_FUNCTION static __inline const void * -__Wpedantic_format_voidptr(const void *ptr) { - return ptr; -} -#define __Wpedantic_format_voidptr(ARG) __Wpedantic_format_voidptr(ARG) -#endif /* __Wpedantic_format_voidptr */ +/*----------------------------------------------------------------------------*/ -#if defined(__GNUC__) && !__GNUC_PREREQ(4, 2) -/* Actually libmdbx was not tested with compilers older than GCC 4.2. - * But you could ignore this warning at your own risk. - * In such case please don't rise up an issues related ONLY to old compilers. - */ -#warning "libmdbx required GCC >= 4.2" -#endif +#if defined(_MSC_VER) && _MSC_VER >= 1900 +/* LY: MSVC 2015/2017/2019 has buggy/inconsistent PRIuPTR/PRIxPTR macros + * for internal format-args checker. */ +#undef PRIuPTR +#undef PRIiPTR +#undef PRIdPTR +#undef PRIxPTR +#define PRIuPTR "Iu" +#define PRIiPTR "Ii" +#define PRIdPTR "Id" +#define PRIxPTR "Ix" +#define PRIuSIZE "zu" +#define PRIiSIZE "zi" +#define PRIdSIZE "zd" +#define PRIxSIZE "zx" +#endif /* fix PRI*PTR for _MSC_VER */ -#if defined(__clang__) && !__CLANG_PREREQ(3, 8) -/* Actually libmdbx was not tested with CLANG older than 3.8. - * But you could ignore this warning at your own risk. - * In such case please don't rise up an issues related ONLY to old compilers. - */ -#warning "libmdbx required CLANG >= 3.8" -#endif +#ifndef PRIuSIZE +#define PRIuSIZE PRIuPTR +#define PRIiSIZE PRIiPTR +#define PRIdSIZE PRIdPTR +#define PRIxSIZE PRIxPTR +#endif /* PRI*SIZE macros for MSVC */ -#if defined(__GLIBC__) && !__GLIBC_PREREQ(2, 12) -/* Actually libmdbx was not tested with something older than glibc 2.12. - * But you could ignore this warning at your own risk. - * In such case please don't rise up an issues related ONLY to old systems. - */ -#warning "libmdbx was only tested with GLIBC >= 2.12." +#ifdef _MSC_VER +#pragma warning(pop) #endif -#ifdef __SANITIZE_THREAD__ -#warning \ - "libmdbx don't compatible with ThreadSanitizer, you will get a lot of false-positive issues." -#endif /* __SANITIZE_THREAD__ */ +/*----------------------------------------------------------------------------*/ #if __has_warning("-Wnested-anon-types") #if defined(__clang__) @@ -980,80 +995,34 @@ __Wpedantic_format_voidptr(const void *ptr) { #endif #endif /* -Walignment-reduction-ignored */ -#ifndef MDBX_EXCLUDE_FOR_GPROF -#ifdef ENABLE_GPROF -#define MDBX_EXCLUDE_FOR_GPROF \ - __attribute__((__no_instrument_function__, \ - __no_profile_instrument_function__)) +#ifdef xMDBX_ALLOY +/* Amalgamated build */ +#define MDBX_INTERNAL static #else -#define MDBX_EXCLUDE_FOR_GPROF -#endif /* ENABLE_GPROF */ -#endif /* MDBX_EXCLUDE_FOR_GPROF */ - -#ifdef __cplusplus -extern "C" { -#endif +/* Non-amalgamated build */ +#define MDBX_INTERNAL +#endif /* xMDBX_ALLOY */ -/* https://en.wikipedia.org/wiki/Operating_system_abstraction_layer */ +#include "mdbx.h++" -/* - * Copyright 2015-2024 Leonid Yuriev - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - */ +/*----------------------------------------------------------------------------*/ +/* Basic constants and types */ +typedef struct iov_ctx iov_ctx_t; +/// /*----------------------------------------------------------------------------*/ -/* C11 Atomics */ - -#if defined(__cplusplus) && !defined(__STDC_NO_ATOMICS__) && __has_include() -#include -#define MDBX_HAVE_C11ATOMICS -#elif !defined(__cplusplus) && \ - (__STDC_VERSION__ >= 201112L || __has_extension(c_atomic)) && \ - !defined(__STDC_NO_ATOMICS__) && \ - (__GNUC_PREREQ(4, 9) || __CLANG_PREREQ(3, 8) || \ - !(defined(__GNUC__) || defined(__clang__))) -#include -#define MDBX_HAVE_C11ATOMICS -#elif defined(__GNUC__) || defined(__clang__) -#elif defined(_MSC_VER) -#pragma warning(disable : 4163) /* 'xyz': not available as an intrinsic */ -#pragma warning(disable : 4133) /* 'function': incompatible types - from \ - 'size_t' to 'LONGLONG' */ -#pragma warning(disable : 4244) /* 'return': conversion from 'LONGLONG' to \ - 'std::size_t', possible loss of data */ -#pragma warning(disable : 4267) /* 'function': conversion from 'size_t' to \ - 'long', possible loss of data */ -#pragma intrinsic(_InterlockedExchangeAdd, _InterlockedCompareExchange) -#pragma intrinsic(_InterlockedExchangeAdd64, _InterlockedCompareExchange64) -#elif defined(__APPLE__) -#include -#else -#error FIXME atomic-ops -#endif - -/*----------------------------------------------------------------------------*/ -/* Memory/Compiler barriers, cache coherence */ +/* Memory/Compiler barriers, cache coherence */ #if __has_include() #include -#elif defined(__mips) || defined(__mips__) || defined(__mips64) || \ - defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \ - defined(__MWERKS__) || defined(__sgi) +#elif defined(__mips) || defined(__mips__) || defined(__mips64) || defined(__mips64__) || defined(_M_MRX000) || \ + defined(_MIPS_) || defined(__MWERKS__) || defined(__sgi) /* MIPS should have explicit cache control */ #include #endif -MDBX_MAYBE_UNUSED static __inline void osal_compiler_barrier(void) { +MDBX_MAYBE_UNUSED static inline void osal_compiler_barrier(void) { #if defined(__clang__) || defined(__GNUC__) __asm__ __volatile__("" ::: "memory"); #elif defined(_MSC_VER) @@ -1062,18 +1031,16 @@ MDBX_MAYBE_UNUSED static __inline void osal_compiler_barrier(void) { __memory_barrier(); #elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) __compiler_barrier(); -#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ - (defined(HP_IA64) || defined(__ia64)) +#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && (defined(HP_IA64) || defined(__ia64)) _Asm_sched_fence(/* LY: no-arg meaning 'all expect ALU', e.g. 0x3D3D */); -#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \ - defined(__ppc64__) || defined(__powerpc64__) +#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || defined(__ppc64__) || defined(__powerpc64__) __fence(); #else #error "Could not guess the kind of compiler, please report to us." #endif } -MDBX_MAYBE_UNUSED static __inline void osal_memory_barrier(void) { +MDBX_MAYBE_UNUSED static inline void osal_memory_barrier(void) { #ifdef MDBX_HAVE_C11ATOMICS atomic_thread_fence(memory_order_seq_cst); #elif defined(__ATOMIC_SEQ_CST) @@ -1094,11 +1061,9 @@ MDBX_MAYBE_UNUSED static __inline void osal_memory_barrier(void) { #endif #elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) __machine_rw_barrier(); -#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ - (defined(HP_IA64) || defined(__ia64)) +#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && (defined(HP_IA64) || defined(__ia64)) _Asm_mf(); -#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \ - defined(__ppc64__) || defined(__powerpc64__) +#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || defined(__ppc64__) || defined(__powerpc64__) __lwsync(); #else #error "Could not guess the kind of compiler, please report to us." @@ -1113,7 +1078,7 @@ MDBX_MAYBE_UNUSED static __inline void osal_memory_barrier(void) { #define HAVE_SYS_TYPES_H typedef HANDLE osal_thread_t; typedef unsigned osal_thread_key_t; -#define MAP_FAILED NULL +#define MAP_FAILED nullptr #define HIGH_DWORD(v) ((DWORD)((sizeof(v) > 4) ? ((uint64_t)(v) >> 32) : 0)) #define THREAD_CALL WINAPI #define THREAD_RESULT DWORD @@ -1125,15 +1090,13 @@ typedef CRITICAL_SECTION osal_fastmutex_t; #if !defined(_MSC_VER) && !defined(__try) #define __try -#define __except(COND) if (false) +#define __except(COND) if (/* (void)(COND), */ false) #endif /* stub for MSVC's __try/__except */ #if MDBX_WITHOUT_MSVC_CRT #ifndef osal_malloc -static inline void *osal_malloc(size_t bytes) { - return HeapAlloc(GetProcessHeap(), 0, bytes); -} +static inline void *osal_malloc(size_t bytes) { return HeapAlloc(GetProcessHeap(), 0, bytes); } #endif /* osal_malloc */ #ifndef osal_calloc @@ -1144,8 +1107,7 @@ static inline void *osal_calloc(size_t nelem, size_t size) { #ifndef osal_realloc static inline void *osal_realloc(void *ptr, size_t bytes) { - return ptr ? HeapReAlloc(GetProcessHeap(), 0, ptr, bytes) - : HeapAlloc(GetProcessHeap(), 0, bytes); + return ptr ? HeapReAlloc(GetProcessHeap(), 0, ptr, bytes) : HeapAlloc(GetProcessHeap(), 0, bytes); } #endif /* osal_realloc */ @@ -1191,29 +1153,16 @@ typedef pthread_mutex_t osal_fastmutex_t; #endif /* Platform */ #if __GLIBC_PREREQ(2, 12) || defined(__FreeBSD__) || defined(malloc_usable_size) -/* malloc_usable_size() already provided */ +#define osal_malloc_usable_size(ptr) malloc_usable_size(ptr) #elif defined(__APPLE__) -#define malloc_usable_size(ptr) malloc_size(ptr) +#define osal_malloc_usable_size(ptr) malloc_size(ptr) #elif defined(_MSC_VER) && !MDBX_WITHOUT_MSVC_CRT -#define malloc_usable_size(ptr) _msize(ptr) -#endif /* malloc_usable_size */ +#define osal_malloc_usable_size(ptr) _msize(ptr) +#endif /* osal_malloc_usable_size */ /*----------------------------------------------------------------------------*/ /* OS abstraction layer stuff */ -MDBX_INTERNAL_VAR unsigned sys_pagesize; -MDBX_MAYBE_UNUSED MDBX_INTERNAL_VAR unsigned sys_pagesize_ln2, - sys_allocation_granularity; - -/* Get the size of a memory page for the system. - * This is the basic size that the platform's memory manager uses, and is - * fundamental to the use of memory-mapped files. */ -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline size_t -osal_syspagesize(void) { - assert(sys_pagesize > 0 && (sys_pagesize & (sys_pagesize - 1)) == 0); - return sys_pagesize; -} - #if defined(_WIN32) || defined(_WIN64) typedef wchar_t pathchar_t; #define MDBX_PRIsPATH "ls" @@ -1225,7 +1174,7 @@ typedef char pathchar_t; typedef struct osal_mmap { union { void *base; - struct MDBX_lockinfo *lck; + struct shared_lck *lck; }; mdbx_filehandle_t fd; size_t limit; /* mapping length, but NOT a size of file nor DB */ @@ -1236,25 +1185,6 @@ typedef struct osal_mmap { #endif } osal_mmap_t; -typedef union bin128 { - __anonymous_struct_extension__ struct { - uint64_t x, y; - }; - __anonymous_struct_extension__ struct { - uint32_t a, b, c, d; - }; -} bin128_t; - -#if defined(_WIN32) || defined(_WIN64) -typedef union osal_srwlock { - __anonymous_struct_extension__ struct { - long volatile readerCount; - long volatile writerCount; - }; - RTL_SRWLOCK native; -} osal_srwlock_t; -#endif /* Windows */ - #ifndef MDBX_HAVE_PWRITEV #if defined(_WIN32) || defined(_WIN64) @@ -1263,14 +1193,21 @@ typedef union osal_srwlock { #elif defined(__ANDROID_API__) #if __ANDROID_API__ < 24 +/* https://android-developers.googleblog.com/2017/09/introducing-android-native-development.html + * https://android.googlesource.com/platform/bionic/+/master/docs/32-bit-abi.md */ #define MDBX_HAVE_PWRITEV 0 +#if defined(_FILE_OFFSET_BITS) && _FILE_OFFSET_BITS != MDBX_WORDBITS +#error "_FILE_OFFSET_BITS != MDBX_WORDBITS and __ANDROID_API__ < 24" (_FILE_OFFSET_BITS != MDBX_WORDBITS) +#elif defined(__FILE_OFFSET_BITS) && __FILE_OFFSET_BITS != MDBX_WORDBITS +#error "__FILE_OFFSET_BITS != MDBX_WORDBITS and __ANDROID_API__ < 24" (__FILE_OFFSET_BITS != MDBX_WORDBITS) +#endif #else #define MDBX_HAVE_PWRITEV 1 #endif #elif defined(__APPLE__) || defined(__MACH__) || defined(_DARWIN_C_SOURCE) -#if defined(MAC_OS_X_VERSION_MIN_REQUIRED) && defined(MAC_OS_VERSION_11_0) && \ +#if defined(MAC_OS_X_VERSION_MIN_REQUIRED) && defined(MAC_OS_VERSION_11_0) && \ MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_VERSION_11_0 /* FIXME: add checks for IOS versions, etc */ #define MDBX_HAVE_PWRITEV 1 @@ -1288,20 +1225,20 @@ typedef union osal_srwlock { typedef struct ior_item { #if defined(_WIN32) || defined(_WIN64) OVERLAPPED ov; -#define ior_svg_gap4terminator 1 +#define ior_sgv_gap4terminator 1 #define ior_sgv_element FILE_SEGMENT_ELEMENT #else size_t offset; #if MDBX_HAVE_PWRITEV size_t sgvcnt; -#define ior_svg_gap4terminator 0 +#define ior_sgv_gap4terminator 0 #define ior_sgv_element struct iovec #endif /* MDBX_HAVE_PWRITEV */ #endif /* !Windows */ union { MDBX_val single; #if defined(ior_sgv_element) - ior_sgv_element sgv[1 + ior_svg_gap4terminator]; + ior_sgv_element sgv[1 + ior_sgv_gap4terminator]; #endif /* ior_sgv_element */ }; } ior_item_t; @@ -1337,45 +1274,33 @@ typedef struct osal_ioring { char *boundary; } osal_ioring_t; -#ifndef __cplusplus - /* Actually this is not ioring for now, but on the way. */ -MDBX_INTERNAL_FUNC int osal_ioring_create(osal_ioring_t * +MDBX_INTERNAL int osal_ioring_create(osal_ioring_t * #if defined(_WIN32) || defined(_WIN64) - , - bool enable_direct, - mdbx_filehandle_t overlapped_fd + , + bool enable_direct, mdbx_filehandle_t overlapped_fd #endif /* Windows */ ); -MDBX_INTERNAL_FUNC int osal_ioring_resize(osal_ioring_t *, size_t items); -MDBX_INTERNAL_FUNC void osal_ioring_destroy(osal_ioring_t *); -MDBX_INTERNAL_FUNC void osal_ioring_reset(osal_ioring_t *); -MDBX_INTERNAL_FUNC int osal_ioring_add(osal_ioring_t *ctx, const size_t offset, - void *data, const size_t bytes); +MDBX_INTERNAL int osal_ioring_resize(osal_ioring_t *, size_t items); +MDBX_INTERNAL void osal_ioring_destroy(osal_ioring_t *); +MDBX_INTERNAL void osal_ioring_reset(osal_ioring_t *); +MDBX_INTERNAL int osal_ioring_add(osal_ioring_t *ctx, const size_t offset, void *data, const size_t bytes); typedef struct osal_ioring_write_result { int err; unsigned wops; } osal_ioring_write_result_t; -MDBX_INTERNAL_FUNC osal_ioring_write_result_t -osal_ioring_write(osal_ioring_t *ior, mdbx_filehandle_t fd); +MDBX_INTERNAL osal_ioring_write_result_t osal_ioring_write(osal_ioring_t *ior, mdbx_filehandle_t fd); -typedef struct iov_ctx iov_ctx_t; -MDBX_INTERNAL_FUNC void osal_ioring_walk( - osal_ioring_t *ior, iov_ctx_t *ctx, - void (*callback)(iov_ctx_t *ctx, size_t offset, void *data, size_t bytes)); +MDBX_INTERNAL void osal_ioring_walk(osal_ioring_t *ior, iov_ctx_t *ctx, + void (*callback)(iov_ctx_t *ctx, size_t offset, void *data, size_t bytes)); -MDBX_MAYBE_UNUSED static inline unsigned -osal_ioring_left(const osal_ioring_t *ior) { - return ior->slots_left; -} +MDBX_MAYBE_UNUSED static inline unsigned osal_ioring_left(const osal_ioring_t *ior) { return ior->slots_left; } -MDBX_MAYBE_UNUSED static inline unsigned -osal_ioring_used(const osal_ioring_t *ior) { +MDBX_MAYBE_UNUSED static inline unsigned osal_ioring_used(const osal_ioring_t *ior) { return ior->allocated - ior->slots_left; } -MDBX_MAYBE_UNUSED static inline int -osal_ioring_prepare(osal_ioring_t *ior, size_t items, size_t bytes) { +MDBX_MAYBE_UNUSED static inline int osal_ioring_prepare(osal_ioring_t *ior, size_t items, size_t bytes) { items = (items > 32) ? items : 32; #if defined(_WIN32) || defined(_WIN64) if (ior->direct) { @@ -1394,14 +1319,12 @@ osal_ioring_prepare(osal_ioring_t *ior, size_t items, size_t bytes) { /*----------------------------------------------------------------------------*/ /* libc compatibility stuff */ -#if (!defined(__GLIBC__) && __GLIBC_PREREQ(2, 1)) && \ - (defined(_GNU_SOURCE) || defined(_BSD_SOURCE)) +#if (!defined(__GLIBC__) && __GLIBC_PREREQ(2, 1)) && (defined(_GNU_SOURCE) || defined(_BSD_SOURCE)) #define osal_asprintf asprintf #define osal_vasprintf vasprintf #else -MDBX_MAYBE_UNUSED MDBX_INTERNAL_FUNC - MDBX_PRINTF_ARGS(2, 3) int osal_asprintf(char **strp, const char *fmt, ...); -MDBX_INTERNAL_FUNC int osal_vasprintf(char **strp, const char *fmt, va_list ap); +MDBX_MAYBE_UNUSED MDBX_INTERNAL MDBX_PRINTF_ARGS(2, 3) int osal_asprintf(char **strp, const char *fmt, ...); +MDBX_INTERNAL int osal_vasprintf(char **strp, const char *fmt, va_list ap); #endif #if !defined(MADV_DODUMP) && defined(MADV_CORE) @@ -1412,8 +1335,7 @@ MDBX_INTERNAL_FUNC int osal_vasprintf(char **strp, const char *fmt, va_list ap); #define MADV_DONTDUMP MADV_NOCORE #endif /* MADV_NOCORE -> MADV_DONTDUMP */ -MDBX_MAYBE_UNUSED MDBX_INTERNAL_FUNC void osal_jitter(bool tiny); -MDBX_MAYBE_UNUSED static __inline void jitter4testing(bool tiny); +MDBX_MAYBE_UNUSED MDBX_INTERNAL void osal_jitter(bool tiny); /* max bytes to write in one call */ #if defined(_WIN64) @@ -1423,14 +1345,12 @@ MDBX_MAYBE_UNUSED static __inline void jitter4testing(bool tiny); #else #define MAX_WRITE UINT32_C(0x3f000000) -#if defined(F_GETLK64) && defined(F_SETLK64) && defined(F_SETLKW64) && \ - !defined(__ANDROID_API__) +#if defined(F_GETLK64) && defined(F_SETLK64) && defined(F_SETLKW64) && !defined(__ANDROID_API__) #define MDBX_F_SETLK F_SETLK64 #define MDBX_F_SETLKW F_SETLKW64 #define MDBX_F_GETLK F_GETLK64 -#if (__GLIBC_PREREQ(2, 28) && \ - (defined(__USE_LARGEFILE64) || defined(__LARGEFILE64_SOURCE) || \ - defined(_USE_LARGEFILE64) || defined(_LARGEFILE64_SOURCE))) || \ +#if (__GLIBC_PREREQ(2, 28) && (defined(__USE_LARGEFILE64) || defined(__LARGEFILE64_SOURCE) || \ + defined(_USE_LARGEFILE64) || defined(_LARGEFILE64_SOURCE))) || \ defined(fcntl64) #define MDBX_FCNTL fcntl64 #else @@ -1448,8 +1368,7 @@ MDBX_MAYBE_UNUSED static __inline void jitter4testing(bool tiny); #define MDBX_STRUCT_FLOCK struct flock #endif /* MDBX_F_SETLK, MDBX_F_SETLKW, MDBX_F_GETLK */ -#if defined(F_OFD_SETLK64) && defined(F_OFD_SETLKW64) && \ - defined(F_OFD_GETLK64) && !defined(__ANDROID_API__) +#if defined(F_OFD_SETLK64) && defined(F_OFD_SETLKW64) && defined(F_OFD_GETLK64) && !defined(__ANDROID_API__) #define MDBX_F_OFD_SETLK F_OFD_SETLK64 #define MDBX_F_OFD_SETLKW F_OFD_SETLKW64 #define MDBX_F_OFD_GETLK F_OFD_GETLK64 @@ -1458,23 +1377,17 @@ MDBX_MAYBE_UNUSED static __inline void jitter4testing(bool tiny); #define MDBX_F_OFD_SETLKW F_OFD_SETLKW #define MDBX_F_OFD_GETLK F_OFD_GETLK #ifndef OFF_T_MAX -#define OFF_T_MAX \ - (((sizeof(off_t) > 4) ? INT64_MAX : INT32_MAX) & ~(size_t)0xFffff) +#define OFF_T_MAX (((sizeof(off_t) > 4) ? INT64_MAX : INT32_MAX) & ~(size_t)0xFffff) #endif /* OFF_T_MAX */ #endif /* MDBX_F_OFD_SETLK64, MDBX_F_OFD_SETLKW64, MDBX_F_OFD_GETLK64 */ -#endif - -#if defined(__linux__) || defined(__gnu_linux__) -MDBX_INTERNAL_VAR uint32_t linux_kernel_version; -MDBX_INTERNAL_VAR bool mdbx_RunningOnWSL1 /* Windows Subsystem 1 for Linux */; -#endif /* Linux */ +#endif /* !Windows */ #ifndef osal_strdup LIBMDBX_API char *osal_strdup(const char *str); #endif -MDBX_MAYBE_UNUSED static __inline int osal_get_errno(void) { +MDBX_MAYBE_UNUSED static inline int osal_get_errno(void) { #if defined(_WIN32) || defined(_WIN64) DWORD rc = GetLastError(); #else @@ -1484,40 +1397,32 @@ MDBX_MAYBE_UNUSED static __inline int osal_get_errno(void) { } #ifndef osal_memalign_alloc -MDBX_INTERNAL_FUNC int osal_memalign_alloc(size_t alignment, size_t bytes, - void **result); +MDBX_INTERNAL int osal_memalign_alloc(size_t alignment, size_t bytes, void **result); #endif #ifndef osal_memalign_free -MDBX_INTERNAL_FUNC void osal_memalign_free(void *ptr); -#endif - -MDBX_INTERNAL_FUNC int osal_condpair_init(osal_condpair_t *condpair); -MDBX_INTERNAL_FUNC int osal_condpair_lock(osal_condpair_t *condpair); -MDBX_INTERNAL_FUNC int osal_condpair_unlock(osal_condpair_t *condpair); -MDBX_INTERNAL_FUNC int osal_condpair_signal(osal_condpair_t *condpair, - bool part); -MDBX_INTERNAL_FUNC int osal_condpair_wait(osal_condpair_t *condpair, bool part); -MDBX_INTERNAL_FUNC int osal_condpair_destroy(osal_condpair_t *condpair); - -MDBX_INTERNAL_FUNC int osal_fastmutex_init(osal_fastmutex_t *fastmutex); -MDBX_INTERNAL_FUNC int osal_fastmutex_acquire(osal_fastmutex_t *fastmutex); -MDBX_INTERNAL_FUNC int osal_fastmutex_release(osal_fastmutex_t *fastmutex); -MDBX_INTERNAL_FUNC int osal_fastmutex_destroy(osal_fastmutex_t *fastmutex); - -MDBX_INTERNAL_FUNC int osal_pwritev(mdbx_filehandle_t fd, struct iovec *iov, - size_t sgvcnt, uint64_t offset); -MDBX_INTERNAL_FUNC int osal_pread(mdbx_filehandle_t fd, void *buf, size_t count, - uint64_t offset); -MDBX_INTERNAL_FUNC int osal_pwrite(mdbx_filehandle_t fd, const void *buf, - size_t count, uint64_t offset); -MDBX_INTERNAL_FUNC int osal_write(mdbx_filehandle_t fd, const void *buf, - size_t count); - -MDBX_INTERNAL_FUNC int -osal_thread_create(osal_thread_t *thread, - THREAD_RESULT(THREAD_CALL *start_routine)(void *), - void *arg); -MDBX_INTERNAL_FUNC int osal_thread_join(osal_thread_t thread); +MDBX_INTERNAL void osal_memalign_free(void *ptr); +#endif + +MDBX_INTERNAL int osal_condpair_init(osal_condpair_t *condpair); +MDBX_INTERNAL int osal_condpair_lock(osal_condpair_t *condpair); +MDBX_INTERNAL int osal_condpair_unlock(osal_condpair_t *condpair); +MDBX_INTERNAL int osal_condpair_signal(osal_condpair_t *condpair, bool part); +MDBX_INTERNAL int osal_condpair_wait(osal_condpair_t *condpair, bool part); +MDBX_INTERNAL int osal_condpair_destroy(osal_condpair_t *condpair); + +MDBX_INTERNAL int osal_fastmutex_init(osal_fastmutex_t *fastmutex); +MDBX_INTERNAL int osal_fastmutex_acquire(osal_fastmutex_t *fastmutex); +MDBX_INTERNAL int osal_fastmutex_release(osal_fastmutex_t *fastmutex); +MDBX_INTERNAL int osal_fastmutex_destroy(osal_fastmutex_t *fastmutex); + +MDBX_INTERNAL int osal_pwritev(mdbx_filehandle_t fd, struct iovec *iov, size_t sgvcnt, uint64_t offset); +MDBX_INTERNAL int osal_pread(mdbx_filehandle_t fd, void *buf, size_t count, uint64_t offset); +MDBX_INTERNAL int osal_pwrite(mdbx_filehandle_t fd, const void *buf, size_t count, uint64_t offset); +MDBX_INTERNAL int osal_write(mdbx_filehandle_t fd, const void *buf, size_t count); + +MDBX_INTERNAL int osal_thread_create(osal_thread_t *thread, THREAD_RESULT(THREAD_CALL *start_routine)(void *), + void *arg); +MDBX_INTERNAL int osal_thread_join(osal_thread_t thread); enum osal_syncmode_bits { MDBX_SYNC_NONE = 0, @@ -1527,11 +1432,10 @@ enum osal_syncmode_bits { MDBX_SYNC_IODQ = 8 }; -MDBX_INTERNAL_FUNC int osal_fsync(mdbx_filehandle_t fd, - const enum osal_syncmode_bits mode_bits); -MDBX_INTERNAL_FUNC int osal_ftruncate(mdbx_filehandle_t fd, uint64_t length); -MDBX_INTERNAL_FUNC int osal_fseek(mdbx_filehandle_t fd, uint64_t pos); -MDBX_INTERNAL_FUNC int osal_filesize(mdbx_filehandle_t fd, uint64_t *length); +MDBX_INTERNAL int osal_fsync(mdbx_filehandle_t fd, const enum osal_syncmode_bits mode_bits); +MDBX_INTERNAL int osal_ftruncate(mdbx_filehandle_t fd, uint64_t length); +MDBX_INTERNAL int osal_fseek(mdbx_filehandle_t fd, uint64_t pos); +MDBX_INTERNAL int osal_filesize(mdbx_filehandle_t fd, uint64_t *length); enum osal_openfile_purpose { MDBX_OPEN_DXB_READ, @@ -1546,7 +1450,7 @@ enum osal_openfile_purpose { MDBX_OPEN_DELETE }; -MDBX_MAYBE_UNUSED static __inline bool osal_isdirsep(pathchar_t c) { +MDBX_MAYBE_UNUSED static inline bool osal_isdirsep(pathchar_t c) { return #if defined(_WIN32) || defined(_WIN64) c == '\\' || @@ -1554,50 +1458,39 @@ MDBX_MAYBE_UNUSED static __inline bool osal_isdirsep(pathchar_t c) { c == '/'; } -MDBX_INTERNAL_FUNC bool osal_pathequal(const pathchar_t *l, const pathchar_t *r, - size_t len); -MDBX_INTERNAL_FUNC pathchar_t *osal_fileext(const pathchar_t *pathname, - size_t len); -MDBX_INTERNAL_FUNC int osal_fileexists(const pathchar_t *pathname); -MDBX_INTERNAL_FUNC int osal_openfile(const enum osal_openfile_purpose purpose, - const MDBX_env *env, - const pathchar_t *pathname, - mdbx_filehandle_t *fd, - mdbx_mode_t unix_mode_bits); -MDBX_INTERNAL_FUNC int osal_closefile(mdbx_filehandle_t fd); -MDBX_INTERNAL_FUNC int osal_removefile(const pathchar_t *pathname); -MDBX_INTERNAL_FUNC int osal_removedirectory(const pathchar_t *pathname); -MDBX_INTERNAL_FUNC int osal_is_pipe(mdbx_filehandle_t fd); -MDBX_INTERNAL_FUNC int osal_lockfile(mdbx_filehandle_t fd, bool wait); +MDBX_INTERNAL bool osal_pathequal(const pathchar_t *l, const pathchar_t *r, size_t len); +MDBX_INTERNAL pathchar_t *osal_fileext(const pathchar_t *pathname, size_t len); +MDBX_INTERNAL int osal_fileexists(const pathchar_t *pathname); +MDBX_INTERNAL int osal_openfile(const enum osal_openfile_purpose purpose, const MDBX_env *env, + const pathchar_t *pathname, mdbx_filehandle_t *fd, mdbx_mode_t unix_mode_bits); +MDBX_INTERNAL int osal_closefile(mdbx_filehandle_t fd); +MDBX_INTERNAL int osal_removefile(const pathchar_t *pathname); +MDBX_INTERNAL int osal_removedirectory(const pathchar_t *pathname); +MDBX_INTERNAL int osal_is_pipe(mdbx_filehandle_t fd); +MDBX_INTERNAL int osal_lockfile(mdbx_filehandle_t fd, bool wait); #define MMAP_OPTION_TRUNCATE 1 #define MMAP_OPTION_SEMAPHORE 2 -MDBX_INTERNAL_FUNC int osal_mmap(const int flags, osal_mmap_t *map, size_t size, - const size_t limit, const unsigned options); -MDBX_INTERNAL_FUNC int osal_munmap(osal_mmap_t *map); +MDBX_INTERNAL int osal_mmap(const int flags, osal_mmap_t *map, size_t size, const size_t limit, const unsigned options, + const pathchar_t *pathname4logging); +MDBX_INTERNAL int osal_munmap(osal_mmap_t *map); #define MDBX_MRESIZE_MAY_MOVE 0x00000100 #define MDBX_MRESIZE_MAY_UNMAP 0x00000200 -MDBX_INTERNAL_FUNC int osal_mresize(const int flags, osal_mmap_t *map, - size_t size, size_t limit); +MDBX_INTERNAL int osal_mresize(const int flags, osal_mmap_t *map, size_t size, size_t limit); #if defined(_WIN32) || defined(_WIN64) typedef struct { unsigned limit, count; HANDLE handles[31]; } mdbx_handle_array_t; -MDBX_INTERNAL_FUNC int -osal_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array); -MDBX_INTERNAL_FUNC int -osal_resume_threads_after_remap(mdbx_handle_array_t *array); +MDBX_INTERNAL int osal_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array); +MDBX_INTERNAL int osal_resume_threads_after_remap(mdbx_handle_array_t *array); #endif /* Windows */ -MDBX_INTERNAL_FUNC int osal_msync(const osal_mmap_t *map, size_t offset, - size_t length, - enum osal_syncmode_bits mode_bits); -MDBX_INTERNAL_FUNC int osal_check_fs_rdonly(mdbx_filehandle_t handle, - const pathchar_t *pathname, - int err); -MDBX_INTERNAL_FUNC int osal_check_fs_incore(mdbx_filehandle_t handle); - -MDBX_MAYBE_UNUSED static __inline uint32_t osal_getpid(void) { +MDBX_INTERNAL int osal_msync(const osal_mmap_t *map, size_t offset, size_t length, enum osal_syncmode_bits mode_bits); +MDBX_INTERNAL int osal_check_fs_rdonly(mdbx_filehandle_t handle, const pathchar_t *pathname, int err); +MDBX_INTERNAL int osal_check_fs_incore(mdbx_filehandle_t handle); +MDBX_INTERNAL int osal_check_fs_local(mdbx_filehandle_t handle, int flags); + +MDBX_MAYBE_UNUSED static inline uint32_t osal_getpid(void) { STATIC_ASSERT(sizeof(mdbx_pid_t) <= sizeof(uint32_t)); #if defined(_WIN32) || defined(_WIN64) return GetCurrentProcessId(); @@ -1607,7 +1500,7 @@ MDBX_MAYBE_UNUSED static __inline uint32_t osal_getpid(void) { #endif } -MDBX_MAYBE_UNUSED static __inline uintptr_t osal_thread_self(void) { +MDBX_MAYBE_UNUSED static inline uintptr_t osal_thread_self(void) { mdbx_tid_t thunk; STATIC_ASSERT(sizeof(uintptr_t) >= sizeof(thunk)); #if defined(_WIN32) || defined(_WIN64) @@ -1620,274 +1513,51 @@ MDBX_MAYBE_UNUSED static __inline uintptr_t osal_thread_self(void) { #if !defined(_WIN32) && !defined(_WIN64) #if defined(__ANDROID_API__) || defined(ANDROID) || defined(BIONIC) -MDBX_INTERNAL_FUNC int osal_check_tid4bionic(void); +MDBX_INTERNAL int osal_check_tid4bionic(void); #else -static __inline int osal_check_tid4bionic(void) { return 0; } +static inline int osal_check_tid4bionic(void) { return 0; } #endif /* __ANDROID_API__ || ANDROID) || BIONIC */ -MDBX_MAYBE_UNUSED static __inline int -osal_pthread_mutex_lock(pthread_mutex_t *mutex) { +MDBX_MAYBE_UNUSED static inline int osal_pthread_mutex_lock(pthread_mutex_t *mutex) { int err = osal_check_tid4bionic(); return unlikely(err) ? err : pthread_mutex_lock(mutex); } #endif /* !Windows */ -MDBX_INTERNAL_FUNC uint64_t osal_monotime(void); -MDBX_INTERNAL_FUNC uint64_t osal_cputime(size_t *optional_page_faults); -MDBX_INTERNAL_FUNC uint64_t osal_16dot16_to_monotime(uint32_t seconds_16dot16); -MDBX_INTERNAL_FUNC uint32_t osal_monotime_to_16dot16(uint64_t monotime); +MDBX_INTERNAL uint64_t osal_monotime(void); +MDBX_INTERNAL uint64_t osal_cputime(size_t *optional_page_faults); +MDBX_INTERNAL uint64_t osal_16dot16_to_monotime(uint32_t seconds_16dot16); +MDBX_INTERNAL uint32_t osal_monotime_to_16dot16(uint64_t monotime); -MDBX_MAYBE_UNUSED static inline uint32_t -osal_monotime_to_16dot16_noUnderflow(uint64_t monotime) { +MDBX_MAYBE_UNUSED static inline uint32_t osal_monotime_to_16dot16_noUnderflow(uint64_t monotime) { uint32_t seconds_16dot16 = osal_monotime_to_16dot16(monotime); return seconds_16dot16 ? seconds_16dot16 : /* fix underflow */ (monotime > 0); } -MDBX_INTERNAL_FUNC bin128_t osal_bootid(void); /*----------------------------------------------------------------------------*/ -/* lck stuff */ - -/// \brief Initialization of synchronization primitives linked with MDBX_env -/// instance both in LCK-file and within the current process. -/// \param -/// global_uniqueness_flag = true - denotes that there are no other processes -/// working with DB and LCK-file. Thus the function MUST initialize -/// shared synchronization objects in memory-mapped LCK-file. -/// global_uniqueness_flag = false - denotes that at least one process is -/// already working with DB and LCK-file, including the case when DB -/// has already been opened in the current process. Thus the function -/// MUST NOT initialize shared synchronization objects in memory-mapped -/// LCK-file that are already in use. -/// \return Error code or zero on success. -MDBX_INTERNAL_FUNC int osal_lck_init(MDBX_env *env, - MDBX_env *inprocess_neighbor, - int global_uniqueness_flag); - -/// \brief Disconnects from shared interprocess objects and destructs -/// synchronization objects linked with MDBX_env instance -/// within the current process. -/// \param -/// inprocess_neighbor = NULL - if the current process does not have other -/// instances of MDBX_env linked with the DB being closed. -/// Thus the function MUST check for other processes working with DB or -/// LCK-file, and keep or destroy shared synchronization objects in -/// memory-mapped LCK-file depending on the result. -/// inprocess_neighbor = not-NULL - pointer to another instance of MDBX_env -/// (anyone of there is several) working with DB or LCK-file within the -/// current process. Thus the function MUST NOT try to acquire exclusive -/// lock and/or try to destruct shared synchronization objects linked with -/// DB or LCK-file. Moreover, the implementation MUST ensure correct work -/// of other instances of MDBX_env within the current process, e.g. -/// restore POSIX-fcntl locks after the closing of file descriptors. -/// \return Error code (MDBX_PANIC) or zero on success. -MDBX_INTERNAL_FUNC int osal_lck_destroy(MDBX_env *env, - MDBX_env *inprocess_neighbor); - -/// \brief Connects to shared interprocess locking objects and tries to acquire -/// the maximum lock level (shared if exclusive is not available) -/// Depending on implementation or/and platform (Windows) this function may -/// acquire the non-OS super-level lock (e.g. for shared synchronization -/// objects initialization), which will be downgraded to OS-exclusive or -/// shared via explicit calling of osal_lck_downgrade(). -/// \return -/// MDBX_RESULT_TRUE (-1) - if an exclusive lock was acquired and thus -/// the current process is the first and only after the last use of DB. -/// MDBX_RESULT_FALSE (0) - if a shared lock was acquired and thus -/// DB has already been opened and now is used by other processes. -/// Otherwise (not 0 and not -1) - error code. -MDBX_INTERNAL_FUNC int osal_lck_seize(MDBX_env *env); - -/// \brief Downgrades the level of initially acquired lock to -/// operational level specified by argument. The reason for such downgrade: -/// - unblocking of other processes that are waiting for access, i.e. -/// if (env->me_flags & MDBX_EXCLUSIVE) != 0, then other processes -/// should be made aware that access is unavailable rather than -/// wait for it. -/// - freeing locks that interfere file operation (especially for Windows) -/// (env->me_flags & MDBX_EXCLUSIVE) == 0 - downgrade to shared lock. -/// (env->me_flags & MDBX_EXCLUSIVE) != 0 - downgrade to exclusive -/// operational lock. -/// \return Error code or zero on success -MDBX_INTERNAL_FUNC int osal_lck_downgrade(MDBX_env *env); - -/// \brief Locks LCK-file or/and table of readers for (de)registering. -/// \return Error code or zero on success -MDBX_INTERNAL_FUNC int osal_rdt_lock(MDBX_env *env); - -/// \brief Unlocks LCK-file or/and table of readers after (de)registering. -MDBX_INTERNAL_FUNC void osal_rdt_unlock(MDBX_env *env); - -/// \brief Acquires lock for DB change (on writing transaction start) -/// Reading transactions will not be blocked. -/// Declared as LIBMDBX_API because it is used in mdbx_chk. -/// \return Error code or zero on success -LIBMDBX_API int mdbx_txn_lock(MDBX_env *env, bool dont_wait); - -/// \brief Releases lock once DB changes is made (after writing transaction -/// has finished). -/// Declared as LIBMDBX_API because it is used in mdbx_chk. -LIBMDBX_API void mdbx_txn_unlock(MDBX_env *env); - -/// \brief Sets alive-flag of reader presence (indicative lock) for PID of -/// the current process. The function does no more than needed for -/// the correct working of osal_rpid_check() in other processes. -/// \return Error code or zero on success -MDBX_INTERNAL_FUNC int osal_rpid_set(MDBX_env *env); - -/// \brief Resets alive-flag of reader presence (indicative lock) -/// for PID of the current process. The function does no more than needed -/// for the correct working of osal_rpid_check() in other processes. -/// \return Error code or zero on success -MDBX_INTERNAL_FUNC int osal_rpid_clear(MDBX_env *env); - -/// \brief Checks for reading process status with the given pid with help of -/// alive-flag of presence (indicative lock) or using another way. -/// \return -/// MDBX_RESULT_TRUE (-1) - if the reader process with the given PID is alive -/// and working with DB (indicative lock is present). -/// MDBX_RESULT_FALSE (0) - if the reader process with the given PID is absent -/// or not working with DB (indicative lock is not present). -/// Otherwise (not 0 and not -1) - error code. -MDBX_INTERNAL_FUNC int osal_rpid_check(MDBX_env *env, uint32_t pid); -#if defined(_WIN32) || defined(_WIN64) - -MDBX_INTERNAL_FUNC int osal_mb2w(const char *const src, wchar_t **const pdst); - -typedef void(WINAPI *osal_srwlock_t_function)(osal_srwlock_t *); -MDBX_INTERNAL_VAR osal_srwlock_t_function osal_srwlock_Init, - osal_srwlock_AcquireShared, osal_srwlock_ReleaseShared, - osal_srwlock_AcquireExclusive, osal_srwlock_ReleaseExclusive; - -#if _WIN32_WINNT < 0x0600 /* prior to Windows Vista */ -typedef enum _FILE_INFO_BY_HANDLE_CLASS { - FileBasicInfo, - FileStandardInfo, - FileNameInfo, - FileRenameInfo, - FileDispositionInfo, - FileAllocationInfo, - FileEndOfFileInfo, - FileStreamInfo, - FileCompressionInfo, - FileAttributeTagInfo, - FileIdBothDirectoryInfo, - FileIdBothDirectoryRestartInfo, - FileIoPriorityHintInfo, - FileRemoteProtocolInfo, - MaximumFileInfoByHandleClass -} FILE_INFO_BY_HANDLE_CLASS, - *PFILE_INFO_BY_HANDLE_CLASS; - -typedef struct _FILE_END_OF_FILE_INFO { - LARGE_INTEGER EndOfFile; -} FILE_END_OF_FILE_INFO, *PFILE_END_OF_FILE_INFO; - -#define REMOTE_PROTOCOL_INFO_FLAG_LOOPBACK 0x00000001 -#define REMOTE_PROTOCOL_INFO_FLAG_OFFLINE 0x00000002 - -typedef struct _FILE_REMOTE_PROTOCOL_INFO { - USHORT StructureVersion; - USHORT StructureSize; - DWORD Protocol; - USHORT ProtocolMajorVersion; - USHORT ProtocolMinorVersion; - USHORT ProtocolRevision; - USHORT Reserved; - DWORD Flags; - struct { - DWORD Reserved[8]; - } GenericReserved; - struct { - DWORD Reserved[16]; - } ProtocolSpecificReserved; -} FILE_REMOTE_PROTOCOL_INFO, *PFILE_REMOTE_PROTOCOL_INFO; - -#endif /* _WIN32_WINNT < 0x0600 (prior to Windows Vista) */ - -typedef BOOL(WINAPI *MDBX_GetFileInformationByHandleEx)( - _In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass, - _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize); -MDBX_INTERNAL_VAR MDBX_GetFileInformationByHandleEx - mdbx_GetFileInformationByHandleEx; - -typedef BOOL(WINAPI *MDBX_GetVolumeInformationByHandleW)( - _In_ HANDLE hFile, _Out_opt_ LPWSTR lpVolumeNameBuffer, - _In_ DWORD nVolumeNameSize, _Out_opt_ LPDWORD lpVolumeSerialNumber, - _Out_opt_ LPDWORD lpMaximumComponentLength, - _Out_opt_ LPDWORD lpFileSystemFlags, - _Out_opt_ LPWSTR lpFileSystemNameBuffer, _In_ DWORD nFileSystemNameSize); -MDBX_INTERNAL_VAR MDBX_GetVolumeInformationByHandleW - mdbx_GetVolumeInformationByHandleW; - -typedef DWORD(WINAPI *MDBX_GetFinalPathNameByHandleW)(_In_ HANDLE hFile, - _Out_ LPWSTR lpszFilePath, - _In_ DWORD cchFilePath, - _In_ DWORD dwFlags); -MDBX_INTERNAL_VAR MDBX_GetFinalPathNameByHandleW mdbx_GetFinalPathNameByHandleW; - -typedef BOOL(WINAPI *MDBX_SetFileInformationByHandle)( - _In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass, - _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize); -MDBX_INTERNAL_VAR MDBX_SetFileInformationByHandle - mdbx_SetFileInformationByHandle; - -typedef NTSTATUS(NTAPI *MDBX_NtFsControlFile)( - IN HANDLE FileHandle, IN OUT HANDLE Event, - IN OUT PVOID /* PIO_APC_ROUTINE */ ApcRoutine, IN OUT PVOID ApcContext, - OUT PIO_STATUS_BLOCK IoStatusBlock, IN ULONG FsControlCode, - IN OUT PVOID InputBuffer, IN ULONG InputBufferLength, - OUT OPTIONAL PVOID OutputBuffer, IN ULONG OutputBufferLength); -MDBX_INTERNAL_VAR MDBX_NtFsControlFile mdbx_NtFsControlFile; - -typedef uint64_t(WINAPI *MDBX_GetTickCount64)(void); -MDBX_INTERNAL_VAR MDBX_GetTickCount64 mdbx_GetTickCount64; - -#if !defined(_WIN32_WINNT_WIN8) || _WIN32_WINNT < _WIN32_WINNT_WIN8 -typedef struct _WIN32_MEMORY_RANGE_ENTRY { - PVOID VirtualAddress; - SIZE_T NumberOfBytes; -} WIN32_MEMORY_RANGE_ENTRY, *PWIN32_MEMORY_RANGE_ENTRY; -#endif /* Windows 8.x */ - -typedef BOOL(WINAPI *MDBX_PrefetchVirtualMemory)( - HANDLE hProcess, ULONG_PTR NumberOfEntries, - PWIN32_MEMORY_RANGE_ENTRY VirtualAddresses, ULONG Flags); -MDBX_INTERNAL_VAR MDBX_PrefetchVirtualMemory mdbx_PrefetchVirtualMemory; - -typedef enum _SECTION_INHERIT { ViewShare = 1, ViewUnmap = 2 } SECTION_INHERIT; - -typedef NTSTATUS(NTAPI *MDBX_NtExtendSection)(IN HANDLE SectionHandle, - IN PLARGE_INTEGER NewSectionSize); -MDBX_INTERNAL_VAR MDBX_NtExtendSection mdbx_NtExtendSection; - -static __inline bool mdbx_RunningUnderWine(void) { - return !mdbx_NtExtendSection; -} - -typedef LSTATUS(WINAPI *MDBX_RegGetValueA)(HKEY hkey, LPCSTR lpSubKey, - LPCSTR lpValue, DWORD dwFlags, - LPDWORD pdwType, PVOID pvData, - LPDWORD pcbData); -MDBX_INTERNAL_VAR MDBX_RegGetValueA mdbx_RegGetValueA; - -NTSYSAPI ULONG RtlRandomEx(PULONG Seed); - -typedef BOOL(WINAPI *MDBX_SetFileIoOverlappedRange)(HANDLE FileHandle, - PUCHAR OverlappedRangeStart, - ULONG Length); -MDBX_INTERNAL_VAR MDBX_SetFileIoOverlappedRange mdbx_SetFileIoOverlappedRange; +MDBX_INTERNAL void osal_ctor(void); +MDBX_INTERNAL void osal_dtor(void); +#if defined(_WIN32) || defined(_WIN64) +MDBX_INTERNAL int osal_mb2w(const char *const src, wchar_t **const pdst); #endif /* Windows */ -#endif /* !__cplusplus */ +typedef union bin128 { + __anonymous_struct_extension__ struct { + uint64_t x, y; + }; + __anonymous_struct_extension__ struct { + uint32_t a, b, c, d; + }; +} bin128_t; + +MDBX_INTERNAL bin128_t osal_guid(const MDBX_env *); /*----------------------------------------------------------------------------*/ -MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static __always_inline uint64_t -osal_bswap64(uint64_t v) { -#if __GNUC_PREREQ(4, 4) || __CLANG_PREREQ(4, 0) || \ - __has_builtin(__builtin_bswap64) +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint64_t osal_bswap64(uint64_t v) { +#if __GNUC_PREREQ(4, 4) || __CLANG_PREREQ(4, 0) || __has_builtin(__builtin_bswap64) return __builtin_bswap64(v); #elif defined(_MSC_VER) && !defined(__clang__) return _byteswap_uint64(v); @@ -1896,19 +1566,14 @@ osal_bswap64(uint64_t v) { #elif defined(bswap_64) return bswap_64(v); #else - return v << 56 | v >> 56 | ((v << 40) & UINT64_C(0x00ff000000000000)) | - ((v << 24) & UINT64_C(0x0000ff0000000000)) | - ((v << 8) & UINT64_C(0x000000ff00000000)) | - ((v >> 8) & UINT64_C(0x00000000ff000000)) | - ((v >> 24) & UINT64_C(0x0000000000ff0000)) | - ((v >> 40) & UINT64_C(0x000000000000ff00)); + return v << 56 | v >> 56 | ((v << 40) & UINT64_C(0x00ff000000000000)) | ((v << 24) & UINT64_C(0x0000ff0000000000)) | + ((v << 8) & UINT64_C(0x000000ff00000000)) | ((v >> 8) & UINT64_C(0x00000000ff000000)) | + ((v >> 24) & UINT64_C(0x0000000000ff0000)) | ((v >> 40) & UINT64_C(0x000000000000ff00)); #endif } -MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static __always_inline uint32_t -osal_bswap32(uint32_t v) { -#if __GNUC_PREREQ(4, 4) || __CLANG_PREREQ(4, 0) || \ - __has_builtin(__builtin_bswap32) +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint32_t osal_bswap32(uint32_t v) { +#if __GNUC_PREREQ(4, 4) || __CLANG_PREREQ(4, 0) || __has_builtin(__builtin_bswap32) return __builtin_bswap32(v); #elif defined(_MSC_VER) && !defined(__clang__) return _byteswap_ulong(v); @@ -1917,50 +1582,14 @@ osal_bswap32(uint32_t v) { #elif defined(bswap_32) return bswap_32(v); #else - return v << 24 | v >> 24 | ((v << 8) & UINT32_C(0x00ff0000)) | - ((v >> 8) & UINT32_C(0x0000ff00)); + return v << 24 | v >> 24 | ((v << 8) & UINT32_C(0x00ff0000)) | ((v >> 8) & UINT32_C(0x0000ff00)); #endif } -/*----------------------------------------------------------------------------*/ - -#if defined(_MSC_VER) && _MSC_VER >= 1900 -/* LY: MSVC 2015/2017/2019 has buggy/inconsistent PRIuPTR/PRIxPTR macros - * for internal format-args checker. */ -#undef PRIuPTR -#undef PRIiPTR -#undef PRIdPTR -#undef PRIxPTR -#define PRIuPTR "Iu" -#define PRIiPTR "Ii" -#define PRIdPTR "Id" -#define PRIxPTR "Ix" -#define PRIuSIZE "zu" -#define PRIiSIZE "zi" -#define PRIdSIZE "zd" -#define PRIxSIZE "zx" -#endif /* fix PRI*PTR for _MSC_VER */ - -#ifndef PRIuSIZE -#define PRIuSIZE PRIuPTR -#define PRIiSIZE PRIiPTR -#define PRIdSIZE PRIdPTR -#define PRIxSIZE PRIxPTR -#endif /* PRI*SIZE macros for MSVC */ - -#ifdef _MSC_VER -#pragma warning(pop) -#endif - -#define mdbx_sourcery_anchor XCONCAT(mdbx_sourcery_, MDBX_BUILD_SOURCERY) -#if defined(xMDBX_TOOLS) -extern LIBMDBX_API const char *const mdbx_sourcery_anchor; -#endif - /******************************************************************************* - ******************************************************************************* ******************************************************************************* * + * BUILD TIME * * #### ##### ##### # #### # # #### * # # # # # # # # ## # # @@ -1981,23 +1610,15 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Using fsync() with chance of data lost on power failure */ #define MDBX_OSX_WANNA_SPEED 1 -#ifndef MDBX_OSX_SPEED_INSTEADOF_DURABILITY +#ifndef MDBX_APPLE_SPEED_INSTEADOF_DURABILITY /** Choices \ref MDBX_OSX_WANNA_DURABILITY or \ref MDBX_OSX_WANNA_SPEED * for OSX & iOS */ -#define MDBX_OSX_SPEED_INSTEADOF_DURABILITY MDBX_OSX_WANNA_DURABILITY -#endif /* MDBX_OSX_SPEED_INSTEADOF_DURABILITY */ - -/** Controls using of POSIX' madvise() and/or similar hints. */ -#ifndef MDBX_ENABLE_MADVISE -#define MDBX_ENABLE_MADVISE 1 -#elif !(MDBX_ENABLE_MADVISE == 0 || MDBX_ENABLE_MADVISE == 1) -#error MDBX_ENABLE_MADVISE must be defined as 0 or 1 -#endif /* MDBX_ENABLE_MADVISE */ +#define MDBX_APPLE_SPEED_INSTEADOF_DURABILITY MDBX_OSX_WANNA_DURABILITY +#endif /* MDBX_APPLE_SPEED_INSTEADOF_DURABILITY */ /** Controls checking PID against reuse DB environment after the fork() */ #ifndef MDBX_ENV_CHECKPID -#if (defined(MADV_DONTFORK) && MDBX_ENABLE_MADVISE) || defined(_WIN32) || \ - defined(_WIN64) +#if defined(MADV_DONTFORK) || defined(_WIN32) || defined(_WIN64) /* PID check could be omitted: * - on Linux when madvise(MADV_DONTFORK) is available, i.e. after the fork() * mapped pages will not be available for child process. @@ -2026,8 +1647,7 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Does a system have battery-backed Real-Time Clock or just a fake. */ #ifndef MDBX_TRUST_RTC -#if defined(__linux__) || defined(__gnu_linux__) || defined(__NetBSD__) || \ - defined(__OpenBSD__) +#if defined(__linux__) || defined(__gnu_linux__) || defined(__NetBSD__) || defined(__OpenBSD__) #define MDBX_TRUST_RTC 0 /* a lot of embedded systems have a fake RTC */ #else #define MDBX_TRUST_RTC 1 @@ -2062,24 +1682,21 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Controls using Unix' mincore() to determine whether DB-pages * are resident in memory. */ -#ifndef MDBX_ENABLE_MINCORE +#ifndef MDBX_USE_MINCORE #if defined(MINCORE_INCORE) || !(defined(_WIN32) || defined(_WIN64)) -#define MDBX_ENABLE_MINCORE 1 +#define MDBX_USE_MINCORE 1 #else -#define MDBX_ENABLE_MINCORE 0 +#define MDBX_USE_MINCORE 0 #endif -#elif !(MDBX_ENABLE_MINCORE == 0 || MDBX_ENABLE_MINCORE == 1) -#error MDBX_ENABLE_MINCORE must be defined as 0 or 1 -#endif /* MDBX_ENABLE_MINCORE */ +#define MDBX_USE_MINCORE_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_USE_MINCORE) +#elif !(MDBX_USE_MINCORE == 0 || MDBX_USE_MINCORE == 1) +#error MDBX_USE_MINCORE must be defined as 0 or 1 +#endif /* MDBX_USE_MINCORE */ /** Enables chunking long list of retired pages during huge transactions commit * to avoid use sequences of pages. */ #ifndef MDBX_ENABLE_BIGFOOT -#if MDBX_WORDBITS >= 64 || defined(DOXYGEN) #define MDBX_ENABLE_BIGFOOT 1 -#else -#define MDBX_ENABLE_BIGFOOT 0 -#endif #elif !(MDBX_ENABLE_BIGFOOT == 0 || MDBX_ENABLE_BIGFOOT == 1) #error MDBX_ENABLE_BIGFOOT must be defined as 0 or 1 #endif /* MDBX_ENABLE_BIGFOOT */ @@ -2094,25 +1711,27 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #ifndef MDBX_PNL_PREALLOC_FOR_RADIXSORT #define MDBX_PNL_PREALLOC_FOR_RADIXSORT 1 -#elif !(MDBX_PNL_PREALLOC_FOR_RADIXSORT == 0 || \ - MDBX_PNL_PREALLOC_FOR_RADIXSORT == 1) +#elif !(MDBX_PNL_PREALLOC_FOR_RADIXSORT == 0 || MDBX_PNL_PREALLOC_FOR_RADIXSORT == 1) #error MDBX_PNL_PREALLOC_FOR_RADIXSORT must be defined as 0 or 1 #endif /* MDBX_PNL_PREALLOC_FOR_RADIXSORT */ #ifndef MDBX_DPL_PREALLOC_FOR_RADIXSORT #define MDBX_DPL_PREALLOC_FOR_RADIXSORT 1 -#elif !(MDBX_DPL_PREALLOC_FOR_RADIXSORT == 0 || \ - MDBX_DPL_PREALLOC_FOR_RADIXSORT == 1) +#elif !(MDBX_DPL_PREALLOC_FOR_RADIXSORT == 0 || MDBX_DPL_PREALLOC_FOR_RADIXSORT == 1) #error MDBX_DPL_PREALLOC_FOR_RADIXSORT must be defined as 0 or 1 #endif /* MDBX_DPL_PREALLOC_FOR_RADIXSORT */ -/** Controls dirty pages tracking, spilling and persisting in MDBX_WRITEMAP - * mode. 0/OFF = Don't track dirty pages at all, don't spill ones, and use - * msync() to persist data. This is by-default on Linux and other systems where - * kernel provides properly LRU tracking and effective flushing on-demand. 1/ON - * = Tracking of dirty pages but with LRU labels for spilling and explicit - * persist ones by write(). This may be reasonable for systems which low - * performance of msync() and/or LRU tracking. */ +/** Controls dirty pages tracking, spilling and persisting in `MDBX_WRITEMAP` + * mode, i.e. disables in-memory database updating with consequent + * flush-to-disk/msync syscall. + * + * 0/OFF = Don't track dirty pages at all, don't spill ones, and use msync() to + * persist data. This is by-default on Linux and other systems where kernel + * provides properly LRU tracking and effective flushing on-demand. + * + * 1/ON = Tracking of dirty pages but with LRU labels for spilling and explicit + * persist ones by write(). This may be reasonable for goofy systems (Windows) + * which low performance of msync() and/or zany LRU tracking. */ #ifndef MDBX_AVOID_MSYNC #if defined(_WIN32) || defined(_WIN64) #define MDBX_AVOID_MSYNC 1 @@ -2123,6 +1742,22 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #error MDBX_AVOID_MSYNC must be defined as 0 or 1 #endif /* MDBX_AVOID_MSYNC */ +/** Управляет механизмом поддержки разреженных наборов DBI-хендлов для снижения + * накладных расходов при запуске и обработке транзакций. */ +#ifndef MDBX_ENABLE_DBI_SPARSE +#define MDBX_ENABLE_DBI_SPARSE 1 +#elif !(MDBX_ENABLE_DBI_SPARSE == 0 || MDBX_ENABLE_DBI_SPARSE == 1) +#error MDBX_ENABLE_DBI_SPARSE must be defined as 0 or 1 +#endif /* MDBX_ENABLE_DBI_SPARSE */ + +/** Управляет механизмом отложенного освобождения и поддержки пути быстрого + * открытия DBI-хендлов без захвата блокировок. */ +#ifndef MDBX_ENABLE_DBI_LOCKFREE +#define MDBX_ENABLE_DBI_LOCKFREE 1 +#elif !(MDBX_ENABLE_DBI_LOCKFREE == 0 || MDBX_ENABLE_DBI_LOCKFREE == 1) +#error MDBX_ENABLE_DBI_LOCKFREE must be defined as 0 or 1 +#endif /* MDBX_ENABLE_DBI_LOCKFREE */ + /** Controls sort order of internal page number lists. * This mostly experimental/advanced option with not for regular MDBX users. * \warning The database format depend on this option and libmdbx built with @@ -2135,7 +1770,11 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Avoid dependence from MSVC CRT and use ntdll.dll instead. */ #ifndef MDBX_WITHOUT_MSVC_CRT +#if defined(MDBX_BUILD_CXX) && !MDBX_BUILD_CXX #define MDBX_WITHOUT_MSVC_CRT 1 +#else +#define MDBX_WITHOUT_MSVC_CRT 0 +#endif #elif !(MDBX_WITHOUT_MSVC_CRT == 0 || MDBX_WITHOUT_MSVC_CRT == 1) #error MDBX_WITHOUT_MSVC_CRT must be defined as 0 or 1 #endif /* MDBX_WITHOUT_MSVC_CRT */ @@ -2143,12 +1782,11 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Size of buffer used during copying a environment/database file. */ #ifndef MDBX_ENVCOPY_WRITEBUF #define MDBX_ENVCOPY_WRITEBUF 1048576u -#elif MDBX_ENVCOPY_WRITEBUF < 65536u || MDBX_ENVCOPY_WRITEBUF > 1073741824u || \ - MDBX_ENVCOPY_WRITEBUF % 65536u +#elif MDBX_ENVCOPY_WRITEBUF < 65536u || MDBX_ENVCOPY_WRITEBUF > 1073741824u || MDBX_ENVCOPY_WRITEBUF % 65536u #error MDBX_ENVCOPY_WRITEBUF must be defined in range 65536..1073741824 and be multiple of 65536 #endif /* MDBX_ENVCOPY_WRITEBUF */ -/** Forces assertion checking */ +/** Forces assertion checking. */ #ifndef MDBX_FORCE_ASSERTIONS #define MDBX_FORCE_ASSERTIONS 0 #elif !(MDBX_FORCE_ASSERTIONS == 0 || MDBX_FORCE_ASSERTIONS == 1) @@ -2163,15 +1801,14 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #else #define MDBX_ASSUME_MALLOC_OVERHEAD (sizeof(void *) * 2u) #endif -#elif MDBX_ASSUME_MALLOC_OVERHEAD < 0 || MDBX_ASSUME_MALLOC_OVERHEAD > 64 || \ - MDBX_ASSUME_MALLOC_OVERHEAD % 4 +#elif MDBX_ASSUME_MALLOC_OVERHEAD < 0 || MDBX_ASSUME_MALLOC_OVERHEAD > 64 || MDBX_ASSUME_MALLOC_OVERHEAD % 4 #error MDBX_ASSUME_MALLOC_OVERHEAD must be defined in range 0..64 and be multiple of 4 #endif /* MDBX_ASSUME_MALLOC_OVERHEAD */ /** If defined then enables integration with Valgrind, * a memory analyzing tool. */ -#ifndef MDBX_USE_VALGRIND -#endif /* MDBX_USE_VALGRIND */ +#ifndef ENABLE_MEMCHECK +#endif /* ENABLE_MEMCHECK */ /** If defined then enables use C11 atomics, * otherwise detects ones availability automatically. */ @@ -2191,18 +1828,24 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #define MDBX_HAVE_BUILTIN_CPU_SUPPORTS 0 #elif defined(__e2k__) #define MDBX_HAVE_BUILTIN_CPU_SUPPORTS 0 -#elif __has_builtin(__builtin_cpu_supports) || \ - defined(__BUILTIN_CPU_SUPPORTS__) || \ +#elif __has_builtin(__builtin_cpu_supports) || defined(__BUILTIN_CPU_SUPPORTS__) || \ (defined(__ia32__) && __GNUC_PREREQ(4, 8) && __GLIBC_PREREQ(2, 23)) #define MDBX_HAVE_BUILTIN_CPU_SUPPORTS 1 #else #define MDBX_HAVE_BUILTIN_CPU_SUPPORTS 0 #endif -#elif !(MDBX_HAVE_BUILTIN_CPU_SUPPORTS == 0 || \ - MDBX_HAVE_BUILTIN_CPU_SUPPORTS == 1) +#elif !(MDBX_HAVE_BUILTIN_CPU_SUPPORTS == 0 || MDBX_HAVE_BUILTIN_CPU_SUPPORTS == 1) #error MDBX_HAVE_BUILTIN_CPU_SUPPORTS must be defined as 0 or 1 #endif /* MDBX_HAVE_BUILTIN_CPU_SUPPORTS */ +/** if enabled then instead of the returned error `MDBX_REMOTE`, only a warning is issued, when + * the database being opened in non-read-only mode is located in a file system exported via NFS. */ +#ifndef MDBX_ENABLE_NON_READONLY_EXPORT +#define MDBX_ENABLE_NON_READONLY_EXPORT 0 +#elif !(MDBX_ENABLE_NON_READONLY_EXPORT == 0 || MDBX_ENABLE_NON_READONLY_EXPORT == 1) +#error MDBX_ENABLE_NON_READONLY_EXPORT must be defined as 0 or 1 +#endif /* MDBX_ENABLE_NON_READONLY_EXPORT */ + //------------------------------------------------------------------------------ /** Win32 File Locking API for \ref MDBX_LOCKING */ @@ -2220,27 +1863,20 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** POSIX-2008 Robust Mutexes for \ref MDBX_LOCKING */ #define MDBX_LOCKING_POSIX2008 2008 -/** BeOS Benaphores, aka Futexes for \ref MDBX_LOCKING */ -#define MDBX_LOCKING_BENAPHORE 1995 - /** Advanced: Choices the locking implementation (autodetection by default). */ #if defined(_WIN32) || defined(_WIN64) #define MDBX_LOCKING MDBX_LOCKING_WIN32FILES #else #ifndef MDBX_LOCKING -#if defined(_POSIX_THREAD_PROCESS_SHARED) && \ - _POSIX_THREAD_PROCESS_SHARED >= 200112L && !defined(__FreeBSD__) +#if defined(_POSIX_THREAD_PROCESS_SHARED) && _POSIX_THREAD_PROCESS_SHARED >= 200112L && !defined(__FreeBSD__) /* Some platforms define the EOWNERDEAD error code even though they * don't support Robust Mutexes. If doubt compile with -MDBX_LOCKING=2001. */ -#if defined(EOWNERDEAD) && _POSIX_THREAD_PROCESS_SHARED >= 200809L && \ - ((defined(_POSIX_THREAD_ROBUST_PRIO_INHERIT) && \ - _POSIX_THREAD_ROBUST_PRIO_INHERIT > 0) || \ - (defined(_POSIX_THREAD_ROBUST_PRIO_PROTECT) && \ - _POSIX_THREAD_ROBUST_PRIO_PROTECT > 0) || \ - defined(PTHREAD_MUTEX_ROBUST) || defined(PTHREAD_MUTEX_ROBUST_NP)) && \ - (!defined(__GLIBC__) || \ - __GLIBC_PREREQ(2, 10) /* troubles with Robust mutexes before 2.10 */) +#if defined(EOWNERDEAD) && _POSIX_THREAD_PROCESS_SHARED >= 200809L && \ + ((defined(_POSIX_THREAD_ROBUST_PRIO_INHERIT) && _POSIX_THREAD_ROBUST_PRIO_INHERIT > 0) || \ + (defined(_POSIX_THREAD_ROBUST_PRIO_PROTECT) && _POSIX_THREAD_ROBUST_PRIO_PROTECT > 0) || \ + defined(PTHREAD_MUTEX_ROBUST) || defined(PTHREAD_MUTEX_ROBUST_NP)) && \ + (!defined(__GLIBC__) || __GLIBC_PREREQ(2, 10) /* troubles with Robust mutexes before 2.10 */) #define MDBX_LOCKING MDBX_LOCKING_POSIX2008 #else #define MDBX_LOCKING MDBX_LOCKING_POSIX2001 @@ -2258,12 +1894,9 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Advanced: Using POSIX OFD-locks (autodetection by default). */ #ifndef MDBX_USE_OFDLOCKS -#if ((defined(F_OFD_SETLK) && defined(F_OFD_SETLKW) && \ - defined(F_OFD_GETLK)) || \ - (defined(F_OFD_SETLK64) && defined(F_OFD_SETLKW64) && \ - defined(F_OFD_GETLK64))) && \ - !defined(MDBX_SAFE4QEMU) && \ - !defined(__sun) /* OFD-lock are broken on Solaris */ +#if ((defined(F_OFD_SETLK) && defined(F_OFD_SETLKW) && defined(F_OFD_GETLK)) || \ + (defined(F_OFD_SETLK64) && defined(F_OFD_SETLKW64) && defined(F_OFD_GETLK64))) && \ + !defined(MDBX_SAFE4QEMU) && !defined(__sun) /* OFD-lock are broken on Solaris */ #define MDBX_USE_OFDLOCKS 1 #else #define MDBX_USE_OFDLOCKS 0 @@ -2277,8 +1910,7 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Advanced: Using sendfile() syscall (autodetection by default). */ #ifndef MDBX_USE_SENDFILE -#if ((defined(__linux__) || defined(__gnu_linux__)) && \ - !defined(__ANDROID_API__)) || \ +#if ((defined(__linux__) || defined(__gnu_linux__)) && !defined(__ANDROID_API__)) || \ (defined(__ANDROID_API__) && __ANDROID_API__ >= 21) #define MDBX_USE_SENDFILE 1 #else @@ -2299,30 +1931,15 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #error MDBX_USE_COPYFILERANGE must be defined as 0 or 1 #endif /* MDBX_USE_COPYFILERANGE */ -/** Advanced: Using sync_file_range() syscall (autodetection by default). */ -#ifndef MDBX_USE_SYNCFILERANGE -#if ((defined(__linux__) || defined(__gnu_linux__)) && \ - defined(SYNC_FILE_RANGE_WRITE) && !defined(__ANDROID_API__)) || \ - (defined(__ANDROID_API__) && __ANDROID_API__ >= 26) -#define MDBX_USE_SYNCFILERANGE 1 -#else -#define MDBX_USE_SYNCFILERANGE 0 -#endif -#elif !(MDBX_USE_SYNCFILERANGE == 0 || MDBX_USE_SYNCFILERANGE == 1) -#error MDBX_USE_SYNCFILERANGE must be defined as 0 or 1 -#endif /* MDBX_USE_SYNCFILERANGE */ - //------------------------------------------------------------------------------ #ifndef MDBX_CPU_WRITEBACK_INCOHERENT -#if defined(__ia32__) || defined(__e2k__) || defined(__hppa) || \ - defined(__hppa__) || defined(DOXYGEN) +#if defined(__ia32__) || defined(__e2k__) || defined(__hppa) || defined(__hppa__) || defined(DOXYGEN) #define MDBX_CPU_WRITEBACK_INCOHERENT 0 #else #define MDBX_CPU_WRITEBACK_INCOHERENT 1 #endif -#elif !(MDBX_CPU_WRITEBACK_INCOHERENT == 0 || \ - MDBX_CPU_WRITEBACK_INCOHERENT == 1) +#elif !(MDBX_CPU_WRITEBACK_INCOHERENT == 0 || MDBX_CPU_WRITEBACK_INCOHERENT == 1) #error MDBX_CPU_WRITEBACK_INCOHERENT must be defined as 0 or 1 #endif /* MDBX_CPU_WRITEBACK_INCOHERENT */ @@ -2332,35 +1949,35 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #else #define MDBX_MMAP_INCOHERENT_FILE_WRITE 0 #endif -#elif !(MDBX_MMAP_INCOHERENT_FILE_WRITE == 0 || \ - MDBX_MMAP_INCOHERENT_FILE_WRITE == 1) +#elif !(MDBX_MMAP_INCOHERENT_FILE_WRITE == 0 || MDBX_MMAP_INCOHERENT_FILE_WRITE == 1) #error MDBX_MMAP_INCOHERENT_FILE_WRITE must be defined as 0 or 1 #endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */ #ifndef MDBX_MMAP_INCOHERENT_CPU_CACHE -#if defined(__mips) || defined(__mips__) || defined(__mips64) || \ - defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \ - defined(__MWERKS__) || defined(__sgi) +#if defined(__mips) || defined(__mips__) || defined(__mips64) || defined(__mips64__) || defined(_M_MRX000) || \ + defined(_MIPS_) || defined(__MWERKS__) || defined(__sgi) /* MIPS has cache coherency issues. */ #define MDBX_MMAP_INCOHERENT_CPU_CACHE 1 #else /* LY: assume no relevant mmap/dcache issues. */ #define MDBX_MMAP_INCOHERENT_CPU_CACHE 0 #endif -#elif !(MDBX_MMAP_INCOHERENT_CPU_CACHE == 0 || \ - MDBX_MMAP_INCOHERENT_CPU_CACHE == 1) +#elif !(MDBX_MMAP_INCOHERENT_CPU_CACHE == 0 || MDBX_MMAP_INCOHERENT_CPU_CACHE == 1) #error MDBX_MMAP_INCOHERENT_CPU_CACHE must be defined as 0 or 1 #endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */ -#ifndef MDBX_MMAP_USE_MS_ASYNC -#if MDBX_MMAP_INCOHERENT_FILE_WRITE || MDBX_MMAP_INCOHERENT_CPU_CACHE -#define MDBX_MMAP_USE_MS_ASYNC 1 +/** Assume system needs explicit syscall to sync/flush/write modified mapped + * memory. */ +#ifndef MDBX_MMAP_NEEDS_JOLT +#if MDBX_MMAP_INCOHERENT_FILE_WRITE || MDBX_MMAP_INCOHERENT_CPU_CACHE || !(defined(__linux__) || defined(__gnu_linux__)) +#define MDBX_MMAP_NEEDS_JOLT 1 #else -#define MDBX_MMAP_USE_MS_ASYNC 0 +#define MDBX_MMAP_NEEDS_JOLT 0 #endif -#elif !(MDBX_MMAP_USE_MS_ASYNC == 0 || MDBX_MMAP_USE_MS_ASYNC == 1) -#error MDBX_MMAP_USE_MS_ASYNC must be defined as 0 or 1 -#endif /* MDBX_MMAP_USE_MS_ASYNC */ +#define MDBX_MMAP_NEEDS_JOLT_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_MMAP_NEEDS_JOLT) +#elif !(MDBX_MMAP_NEEDS_JOLT == 0 || MDBX_MMAP_NEEDS_JOLT == 1) +#error MDBX_MMAP_NEEDS_JOLT must be defined as 0 or 1 +#endif /* MDBX_MMAP_NEEDS_JOLT */ #ifndef MDBX_64BIT_ATOMIC #if MDBX_WORDBITS >= 64 || defined(DOXYGEN) @@ -2407,8 +2024,7 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #endif /* MDBX_64BIT_CAS */ #ifndef MDBX_UNALIGNED_OK -#if defined(__ALIGNED__) || defined(__SANITIZE_UNDEFINED__) || \ - defined(ENABLE_UBSAN) +#if defined(__ALIGNED__) || defined(__SANITIZE_UNDEFINED__) || defined(ENABLE_UBSAN) #define MDBX_UNALIGNED_OK 0 /* no unaligned access allowed */ #elif defined(__ARM_FEATURE_UNALIGNED) #define MDBX_UNALIGNED_OK 4 /* ok unaligned for 32-bit words */ @@ -2442,6 +2058,19 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #endif #endif /* MDBX_CACHELINE_SIZE */ +/* Max length of iov-vector passed to writev() call, used for auxilary writes */ +#ifndef MDBX_AUXILARY_IOV_MAX +#define MDBX_AUXILARY_IOV_MAX 64 +#endif +#if defined(IOV_MAX) && IOV_MAX < MDBX_AUXILARY_IOV_MAX +#undef MDBX_AUXILARY_IOV_MAX +#define MDBX_AUXILARY_IOV_MAX IOV_MAX +#endif /* MDBX_AUXILARY_IOV_MAX */ + +/* An extra/custom information provided during library build */ +#ifndef MDBX_BUILD_METADATA +#define MDBX_BUILD_METADATA "" +#endif /* MDBX_BUILD_METADATA */ /** @} end of build options */ /******************************************************************************* ******************************************************************************* @@ -2456,6 +2085,9 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #else #define MDBX_DEBUG 1 #endif +#endif +#if MDBX_DEBUG < 0 || MDBX_DEBUG > 2 +#error "The MDBX_DEBUG must be defined to 0, 1 or 2" #endif /* MDBX_DEBUG */ #else @@ -2475,169 +2107,58 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; * Also enables \ref MDBX_DBG_AUDIT if `MDBX_DEBUG >= 2`. * * \ingroup build_option */ -#define MDBX_DEBUG 0...7 +#define MDBX_DEBUG 0...2 /** Disables using of GNU libc extensions. */ #define MDBX_DISABLE_GNU_SOURCE 0 or 1 #endif /* DOXYGEN */ -/* Undefine the NDEBUG if debugging is enforced by MDBX_DEBUG */ -#if MDBX_DEBUG -#undef NDEBUG -#endif - -#ifndef __cplusplus -/*----------------------------------------------------------------------------*/ -/* Debug and Logging stuff */ - -#define MDBX_RUNTIME_FLAGS_INIT \ - ((MDBX_DEBUG) > 0) * MDBX_DBG_ASSERT + ((MDBX_DEBUG) > 1) * MDBX_DBG_AUDIT - -extern uint8_t runtime_flags; -extern uint8_t loglevel; -extern MDBX_debug_func *debug_logger; - -MDBX_MAYBE_UNUSED static __inline void jitter4testing(bool tiny) { -#if MDBX_DEBUG - if (MDBX_DBG_JITTER & runtime_flags) - osal_jitter(tiny); -#else - (void)tiny; -#endif -} - -MDBX_INTERNAL_FUNC void MDBX_PRINTF_ARGS(4, 5) - debug_log(int level, const char *function, int line, const char *fmt, ...) - MDBX_PRINTF_ARGS(4, 5); -MDBX_INTERNAL_FUNC void debug_log_va(int level, const char *function, int line, - const char *fmt, va_list args); +#ifndef MDBX_64BIT_ATOMIC +#error "The MDBX_64BIT_ATOMIC must be defined before" +#endif /* MDBX_64BIT_ATOMIC */ -#if MDBX_DEBUG -#define LOG_ENABLED(msg) unlikely(msg <= loglevel) -#define AUDIT_ENABLED() unlikely((runtime_flags & MDBX_DBG_AUDIT)) -#else /* MDBX_DEBUG */ -#define LOG_ENABLED(msg) (msg < MDBX_LOG_VERBOSE && msg <= loglevel) -#define AUDIT_ENABLED() (0) -#endif /* MDBX_DEBUG */ +#ifndef MDBX_64BIT_CAS +#error "The MDBX_64BIT_CAS must be defined before" +#endif /* MDBX_64BIT_CAS */ -#if MDBX_FORCE_ASSERTIONS -#define ASSERT_ENABLED() (1) -#elif MDBX_DEBUG -#define ASSERT_ENABLED() likely((runtime_flags & MDBX_DBG_ASSERT)) +#if defined(__cplusplus) && !defined(__STDC_NO_ATOMICS__) && __has_include() +#include +#define MDBX_HAVE_C11ATOMICS +#elif !defined(__cplusplus) && (__STDC_VERSION__ >= 201112L || __has_extension(c_atomic)) && \ + !defined(__STDC_NO_ATOMICS__) && \ + (__GNUC_PREREQ(4, 9) || __CLANG_PREREQ(3, 8) || !(defined(__GNUC__) || defined(__clang__))) +#include +#define MDBX_HAVE_C11ATOMICS +#elif defined(__GNUC__) || defined(__clang__) +#elif defined(_MSC_VER) +#pragma warning(disable : 4163) /* 'xyz': not available as an intrinsic */ +#pragma warning(disable : 4133) /* 'function': incompatible types - from \ + 'size_t' to 'LONGLONG' */ +#pragma warning(disable : 4244) /* 'return': conversion from 'LONGLONG' to \ + 'std::size_t', possible loss of data */ +#pragma warning(disable : 4267) /* 'function': conversion from 'size_t' to \ + 'long', possible loss of data */ +#pragma intrinsic(_InterlockedExchangeAdd, _InterlockedCompareExchange) +#pragma intrinsic(_InterlockedExchangeAdd64, _InterlockedCompareExchange64) +#elif defined(__APPLE__) +#include #else -#define ASSERT_ENABLED() (0) -#endif /* assertions */ - -#define DEBUG_EXTRA(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_EXTRA)) \ - debug_log(MDBX_LOG_EXTRA, __func__, __LINE__, fmt, __VA_ARGS__); \ - } while (0) - -#define DEBUG_EXTRA_PRINT(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_EXTRA)) \ - debug_log(MDBX_LOG_EXTRA, NULL, 0, fmt, __VA_ARGS__); \ - } while (0) - -#define TRACE(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_TRACE)) \ - debug_log(MDBX_LOG_TRACE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ - } while (0) - -#define DEBUG(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_DEBUG)) \ - debug_log(MDBX_LOG_DEBUG, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ - } while (0) - -#define VERBOSE(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_VERBOSE)) \ - debug_log(MDBX_LOG_VERBOSE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ - } while (0) - -#define NOTICE(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_NOTICE)) \ - debug_log(MDBX_LOG_NOTICE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ - } while (0) - -#define WARNING(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_WARN)) \ - debug_log(MDBX_LOG_WARN, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ - } while (0) - -#undef ERROR /* wingdi.h \ - Yeah, morons from M$ put such definition to the public header. */ - -#define ERROR(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_ERROR)) \ - debug_log(MDBX_LOG_ERROR, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ - } while (0) - -#define FATAL(fmt, ...) \ - debug_log(MDBX_LOG_FATAL, __func__, __LINE__, fmt "\n", __VA_ARGS__); - -#if MDBX_DEBUG -#define ASSERT_FAIL(env, msg, func, line) mdbx_assert_fail(env, msg, func, line) -#else /* MDBX_DEBUG */ -MDBX_NORETURN __cold void assert_fail(const char *msg, const char *func, - unsigned line); -#define ASSERT_FAIL(env, msg, func, line) \ - do { \ - (void)(env); \ - assert_fail(msg, func, line); \ - } while (0) -#endif /* MDBX_DEBUG */ - -#define ENSURE_MSG(env, expr, msg) \ - do { \ - if (unlikely(!(expr))) \ - ASSERT_FAIL(env, msg, __func__, __LINE__); \ - } while (0) - -#define ENSURE(env, expr) ENSURE_MSG(env, expr, #expr) - -/* assert(3) variant in environment context */ -#define eASSERT(env, expr) \ - do { \ - if (ASSERT_ENABLED()) \ - ENSURE(env, expr); \ - } while (0) - -/* assert(3) variant in cursor context */ -#define cASSERT(mc, expr) eASSERT((mc)->mc_txn->mt_env, expr) - -/* assert(3) variant in transaction context */ -#define tASSERT(txn, expr) eASSERT((txn)->mt_env, expr) - -#ifndef xMDBX_TOOLS /* Avoid using internal eASSERT() */ -#undef assert -#define assert(expr) eASSERT(NULL, expr) +#error FIXME atomic-ops #endif -#endif /* __cplusplus */ - -/*----------------------------------------------------------------------------*/ -/* Atomics */ - -enum MDBX_memory_order { +typedef enum mdbx_memory_order { mo_Relaxed, mo_AcquireRelease /* , mo_SequentialConsistency */ -}; +} mdbx_memory_order_t; typedef union { volatile uint32_t weak; #ifdef MDBX_HAVE_C11ATOMICS volatile _Atomic uint32_t c11a; #endif /* MDBX_HAVE_C11ATOMICS */ -} MDBX_atomic_uint32_t; +} mdbx_atomic_uint32_t; typedef union { volatile uint64_t weak; @@ -2647,15 +2168,15 @@ typedef union { #if !defined(MDBX_HAVE_C11ATOMICS) || !MDBX_64BIT_CAS || !MDBX_64BIT_ATOMIC __anonymous_struct_extension__ struct { #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ - MDBX_atomic_uint32_t low, high; + mdbx_atomic_uint32_t low, high; #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - MDBX_atomic_uint32_t high, low; + mdbx_atomic_uint32_t high, low; #else #error "FIXME: Unsupported byte order" #endif /* __BYTE_ORDER__ */ }; #endif -} MDBX_atomic_uint64_t; +} mdbx_atomic_uint64_t; #ifdef MDBX_HAVE_C11ATOMICS @@ -2671,92 +2192,20 @@ typedef union { #define MDBX_c11a_rw(type, ptr) (&(ptr)->c11a) #endif /* Crutches for C11 atomic compiler's bugs */ -#define mo_c11_store(fence) \ - (((fence) == mo_Relaxed) ? memory_order_relaxed \ - : ((fence) == mo_AcquireRelease) ? memory_order_release \ +#define mo_c11_store(fence) \ + (((fence) == mo_Relaxed) ? memory_order_relaxed \ + : ((fence) == mo_AcquireRelease) ? memory_order_release \ : memory_order_seq_cst) -#define mo_c11_load(fence) \ - (((fence) == mo_Relaxed) ? memory_order_relaxed \ - : ((fence) == mo_AcquireRelease) ? memory_order_acquire \ +#define mo_c11_load(fence) \ + (((fence) == mo_Relaxed) ? memory_order_relaxed \ + : ((fence) == mo_AcquireRelease) ? memory_order_acquire \ : memory_order_seq_cst) #endif /* MDBX_HAVE_C11ATOMICS */ -#ifndef __cplusplus - -#ifdef MDBX_HAVE_C11ATOMICS -#define osal_memory_fence(order, write) \ - atomic_thread_fence((write) ? mo_c11_store(order) : mo_c11_load(order)) -#else /* MDBX_HAVE_C11ATOMICS */ -#define osal_memory_fence(order, write) \ - do { \ - osal_compiler_barrier(); \ - if (write && order > (MDBX_CPU_WRITEBACK_INCOHERENT ? mo_Relaxed \ - : mo_AcquireRelease)) \ - osal_memory_barrier(); \ - } while (0) -#endif /* MDBX_HAVE_C11ATOMICS */ - -#if defined(MDBX_HAVE_C11ATOMICS) && defined(__LCC__) -#define atomic_store32(p, value, order) \ - ({ \ - const uint32_t value_to_store = (value); \ - atomic_store_explicit(MDBX_c11a_rw(uint32_t, p), value_to_store, \ - mo_c11_store(order)); \ - value_to_store; \ - }) -#define atomic_load32(p, order) \ - atomic_load_explicit(MDBX_c11a_ro(uint32_t, p), mo_c11_load(order)) -#define atomic_store64(p, value, order) \ - ({ \ - const uint64_t value_to_store = (value); \ - atomic_store_explicit(MDBX_c11a_rw(uint64_t, p), value_to_store, \ - mo_c11_store(order)); \ - value_to_store; \ - }) -#define atomic_load64(p, order) \ - atomic_load_explicit(MDBX_c11a_ro(uint64_t, p), mo_c11_load(order)) -#endif /* LCC && MDBX_HAVE_C11ATOMICS */ - -#ifndef atomic_store32 -MDBX_MAYBE_UNUSED static __always_inline uint32_t -atomic_store32(MDBX_atomic_uint32_t *p, const uint32_t value, - enum MDBX_memory_order order) { - STATIC_ASSERT(sizeof(MDBX_atomic_uint32_t) == 4); -#ifdef MDBX_HAVE_C11ATOMICS - assert(atomic_is_lock_free(MDBX_c11a_rw(uint32_t, p))); - atomic_store_explicit(MDBX_c11a_rw(uint32_t, p), value, mo_c11_store(order)); -#else /* MDBX_HAVE_C11ATOMICS */ - if (order != mo_Relaxed) - osal_compiler_barrier(); - p->weak = value; - osal_memory_fence(order, true); -#endif /* MDBX_HAVE_C11ATOMICS */ - return value; -} -#endif /* atomic_store32 */ - -#ifndef atomic_load32 -MDBX_MAYBE_UNUSED static __always_inline uint32_t atomic_load32( - const volatile MDBX_atomic_uint32_t *p, enum MDBX_memory_order order) { - STATIC_ASSERT(sizeof(MDBX_atomic_uint32_t) == 4); -#ifdef MDBX_HAVE_C11ATOMICS - assert(atomic_is_lock_free(MDBX_c11a_ro(uint32_t, p))); - return atomic_load_explicit(MDBX_c11a_ro(uint32_t, p), mo_c11_load(order)); -#else /* MDBX_HAVE_C11ATOMICS */ - osal_memory_fence(order, false); - const uint32_t value = p->weak; - if (order != mo_Relaxed) - osal_compiler_barrier(); - return value; -#endif /* MDBX_HAVE_C11ATOMICS */ -} -#endif /* atomic_load32 */ - -#endif /* !__cplusplus */ +#define SAFE64_INVALID_THRESHOLD UINT64_C(0xffffFFFF00000000) -/*----------------------------------------------------------------------------*/ -/* Basic constants and types */ +#pragma pack(push, 4) /* A stamp that identifies a file as an MDBX file. * There's nothing special about this value other than that it is easily @@ -2765,8 +2214,10 @@ MDBX_MAYBE_UNUSED static __always_inline uint32_t atomic_load32( /* FROZEN: The version number for a database's datafile format. */ #define MDBX_DATA_VERSION 3 -/* The version number for a database's lockfile format. */ -#define MDBX_LOCK_VERSION 5 + +#define MDBX_DATA_MAGIC ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + MDBX_DATA_VERSION) +#define MDBX_DATA_MAGIC_LEGACY_COMPAT ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + 2) +#define MDBX_DATA_MAGIC_LEGACY_DEVEL ((MDBX_MAGIC << 8) + 255) /* handle for the DB used to track free pages. */ #define FREE_DBI 0 @@ -2783,203 +2234,285 @@ MDBX_MAYBE_UNUSED static __always_inline uint32_t atomic_load32( * MDBX uses 32 bit for page numbers. This limits database * size up to 2^44 bytes, in case of 4K pages. */ typedef uint32_t pgno_t; -typedef MDBX_atomic_uint32_t atomic_pgno_t; +typedef mdbx_atomic_uint32_t atomic_pgno_t; #define PRIaPGNO PRIu32 #define MAX_PAGENO UINT32_C(0x7FFFffff) #define MIN_PAGENO NUM_METAS -#define SAFE64_INVALID_THRESHOLD UINT64_C(0xffffFFFF00000000) +/* An invalid page number. + * Mainly used to denote an empty tree. */ +#define P_INVALID (~(pgno_t)0) /* A transaction ID. */ typedef uint64_t txnid_t; -typedef MDBX_atomic_uint64_t atomic_txnid_t; +typedef mdbx_atomic_uint64_t atomic_txnid_t; #define PRIaTXN PRIi64 #define MIN_TXNID UINT64_C(1) #define MAX_TXNID (SAFE64_INVALID_THRESHOLD - 1) #define INITIAL_TXNID (MIN_TXNID + NUM_METAS - 1) #define INVALID_TXNID UINT64_MAX -/* LY: for testing non-atomic 64-bit txnid on 32-bit arches. - * #define xMDBX_TXNID_STEP (UINT32_MAX / 3) */ -#ifndef xMDBX_TXNID_STEP -#if MDBX_64BIT_CAS -#define xMDBX_TXNID_STEP 1u -#else -#define xMDBX_TXNID_STEP 2u -#endif -#endif /* xMDBX_TXNID_STEP */ -/* Used for offsets within a single page. - * Since memory pages are typically 4 or 8KB in size, 12-13 bits, - * this is plenty. */ +/* Used for offsets within a single page. */ typedef uint16_t indx_t; -#define MEGABYTE ((size_t)1 << 20) - -/*----------------------------------------------------------------------------*/ -/* Core structures for database and shared memory (i.e. format definition) */ -#pragma pack(push, 4) - -/* Information about a single database in the environment. */ -typedef struct MDBX_db { - uint16_t md_flags; /* see mdbx_dbi_open */ - uint16_t md_depth; /* depth of this tree */ - uint32_t md_xsize; /* key-size for MDBX_DUPFIXED (LEAF2 pages) */ - pgno_t md_root; /* the root page of this tree */ - pgno_t md_branch_pages; /* number of internal pages */ - pgno_t md_leaf_pages; /* number of leaf pages */ - pgno_t md_overflow_pages; /* number of overflow pages */ - uint64_t md_seq; /* table sequence counter */ - uint64_t md_entries; /* number of data items */ - uint64_t md_mod_txnid; /* txnid of last committed modification */ -} MDBX_db; +typedef struct tree { + uint16_t flags; /* see mdbx_dbi_open */ + uint16_t height; /* height of this tree */ + uint32_t dupfix_size; /* key-size for MDBX_DUPFIXED (DUPFIX pages) */ + pgno_t root; /* the root page of this tree */ + pgno_t branch_pages; /* number of branch pages */ + pgno_t leaf_pages; /* number of leaf pages */ + pgno_t large_pages; /* number of large pages */ + uint64_t sequence; /* table sequence counter */ + uint64_t items; /* number of data items */ + uint64_t mod_txnid; /* txnid of last committed modification */ +} tree_t; /* database size-related parameters */ -typedef struct MDBX_geo { +typedef struct geo { uint16_t grow_pv; /* datafile growth step as a 16-bit packed (exponential quantized) value */ uint16_t shrink_pv; /* datafile shrink threshold as a 16-bit packed (exponential quantized) value */ pgno_t lower; /* minimal size of datafile in pages */ pgno_t upper; /* maximal size of datafile in pages */ - pgno_t now; /* current size of datafile in pages */ - pgno_t next; /* first unused page in the datafile, + union { + pgno_t now; /* current size of datafile in pages */ + pgno_t end_pgno; + }; + union { + pgno_t first_unallocated; /* first unused page in the datafile, but actually the file may be shorter. */ -} MDBX_geo; + pgno_t next_pgno; + }; +} geo_t; /* Meta page content. * A meta page is the start point for accessing a database snapshot. - * Pages 0-1 are meta pages. Transaction N writes meta page (N % 2). */ -typedef struct MDBX_meta { + * Pages 0-2 are meta pages. */ +typedef struct meta { /* Stamp identifying this as an MDBX file. * It must be set to MDBX_MAGIC with MDBX_DATA_VERSION. */ - uint32_t mm_magic_and_version[2]; + uint32_t magic_and_version[2]; - /* txnid that committed this page, the first of a two-phase-update pair */ + /* txnid that committed this meta, the first of a two-phase-update pair */ union { - MDBX_atomic_uint32_t mm_txnid_a[2]; + mdbx_atomic_uint32_t txnid_a[2]; uint64_t unsafe_txnid; }; - uint16_t mm_extra_flags; /* extra DB flags, zero (nothing) for now */ - uint8_t mm_validator_id; /* ID of checksum and page validation method, - * zero (nothing) for now */ - uint8_t mm_extra_pagehdr; /* extra bytes in the page header, - * zero (nothing) for now */ + uint16_t reserve16; /* extra flags, zero (nothing) for now */ + uint8_t validator_id; /* ID of checksum and page validation method, + * zero (nothing) for now */ + int8_t extra_pagehdr; /* extra bytes in the page header, + * zero (nothing) for now */ + + geo_t geometry; /* database size-related parameters */ - MDBX_geo mm_geo; /* database size-related parameters */ + union { + struct { + tree_t gc, main; + } trees; + __anonymous_struct_extension__ struct { + uint16_t gc_flags; + uint16_t gc_height; + uint32_t pagesize; + }; + }; - MDBX_db mm_dbs[CORE_DBS]; /* first is free space, 2nd is main db */ - /* The size of pages used in this DB */ -#define mm_psize mm_dbs[FREE_DBI].md_xsize - MDBX_canary mm_canary; + MDBX_canary canary; -#define MDBX_DATASIGN_NONE 0u -#define MDBX_DATASIGN_WEAK 1u -#define SIGN_IS_STEADY(sign) ((sign) > MDBX_DATASIGN_WEAK) -#define META_IS_STEADY(meta) \ - SIGN_IS_STEADY(unaligned_peek_u64_volatile(4, (meta)->mm_sign)) +#define DATASIGN_NONE 0u +#define DATASIGN_WEAK 1u +#define SIGN_IS_STEADY(sign) ((sign) > DATASIGN_WEAK) union { - uint32_t mm_sign[2]; + uint32_t sign[2]; uint64_t unsafe_sign; }; - /* txnid that committed this page, the second of a two-phase-update pair */ - MDBX_atomic_uint32_t mm_txnid_b[2]; + /* txnid that committed this meta, the second of a two-phase-update pair */ + mdbx_atomic_uint32_t txnid_b[2]; /* Number of non-meta pages which were put in GC after COW. May be 0 in case * DB was previously handled by libmdbx without corresponding feature. - * This value in couple with mr_snapshot_pages_retired allows fast estimation - * of "how much reader is restraining GC recycling". */ - uint32_t mm_pages_retired[2]; + * This value in couple with reader.snapshot_pages_retired allows fast + * estimation of "how much reader is restraining GC recycling". */ + uint32_t pages_retired[2]; /* The analogue /proc/sys/kernel/random/boot_id or similar to determine * whether the system was rebooted after the last use of the database files. * If there was no reboot, but there is no need to rollback to the last * steady sync point. Zeros mean that no relevant information is available * from the system. */ - bin128_t mm_bootid; + bin128_t bootid; -} MDBX_meta; + /* GUID базы данных, начиная с v0.13.1 */ + bin128_t dxbid; +} meta_t; #pragma pack(1) -/* Common header for all page types. The page type depends on mp_flags. +typedef enum page_type { + P_BRANCH = 0x01u /* branch page */, + P_LEAF = 0x02u /* leaf page */, + P_LARGE = 0x04u /* large/overflow page */, + P_META = 0x08u /* meta page */, + P_LEGACY_DIRTY = 0x10u /* legacy P_DIRTY flag prior to v0.10 958fd5b9 */, + P_BAD = P_LEGACY_DIRTY /* explicit flag for invalid/bad page */, + P_DUPFIX = 0x20u /* for MDBX_DUPFIXED records */, + P_SUBP = 0x40u /* for MDBX_DUPSORT sub-pages */, + P_SPILLED = 0x2000u /* spilled in parent txn */, + P_LOOSE = 0x4000u /* page was dirtied then freed, can be reused */, + P_FROZEN = 0x8000u /* used for retire page with known status */, + P_ILL_BITS = (uint16_t)~(P_BRANCH | P_LEAF | P_DUPFIX | P_LARGE | P_SPILLED), + + page_broken = 0, + page_large = P_LARGE, + page_branch = P_BRANCH, + page_leaf = P_LEAF, + page_dupfix_leaf = P_DUPFIX, + page_sub_leaf = P_SUBP | P_LEAF, + page_sub_dupfix_leaf = P_SUBP | P_DUPFIX, + page_sub_broken = P_SUBP, +} page_type_t; + +/* Common header for all page types. The page type depends on flags. * - * P_BRANCH and P_LEAF pages have unsorted 'MDBX_node's at the end, with - * sorted mp_ptrs[] entries referring to them. Exception: P_LEAF2 pages - * omit mp_ptrs and pack sorted MDBX_DUPFIXED values after the page header. + * P_BRANCH and P_LEAF pages have unsorted 'node_t's at the end, with + * sorted entries[] entries referring to them. Exception: P_DUPFIX pages + * omit entries and pack sorted MDBX_DUPFIXED values after the page header. * - * P_OVERFLOW records occupy one or more contiguous pages where only the - * first has a page header. They hold the real data of F_BIGDATA nodes. + * P_LARGE records occupy one or more contiguous pages where only the + * first has a page header. They hold the real data of N_BIG nodes. * * P_SUBP sub-pages are small leaf "pages" with duplicate data. - * A node with flag F_DUPDATA but not F_SUBDATA contains a sub-page. - * (Duplicate data can also go in sub-databases, which use normal pages.) + * A node with flag N_DUP but not N_TREE contains a sub-page. + * (Duplicate data can also go in tables, which use normal pages.) * - * P_META pages contain MDBX_meta, the start point of an MDBX snapshot. + * P_META pages contain meta_t, the start point of an MDBX snapshot. * - * Each non-metapage up to MDBX_meta.mm_last_pg is reachable exactly once + * Each non-metapage up to meta_t.mm_last_pg is reachable exactly once * in the snapshot: Either used by a database or listed in a GC record. */ -typedef struct MDBX_page { -#define IS_FROZEN(txn, p) ((p)->mp_txnid < (txn)->mt_txnid) -#define IS_SPILLED(txn, p) ((p)->mp_txnid == (txn)->mt_txnid) -#define IS_SHADOWED(txn, p) ((p)->mp_txnid > (txn)->mt_txnid) -#define IS_VALID(txn, p) ((p)->mp_txnid <= (txn)->mt_front) -#define IS_MODIFIABLE(txn, p) ((p)->mp_txnid == (txn)->mt_front) - uint64_t mp_txnid; /* txnid which created page, maybe zero in legacy DB */ - uint16_t mp_leaf2_ksize; /* key size if this is a LEAF2 page */ -#define P_BRANCH 0x01u /* branch page */ -#define P_LEAF 0x02u /* leaf page */ -#define P_OVERFLOW 0x04u /* overflow page */ -#define P_META 0x08u /* meta page */ -#define P_LEGACY_DIRTY 0x10u /* legacy P_DIRTY flag prior to v0.10 958fd5b9 */ -#define P_BAD P_LEGACY_DIRTY /* explicit flag for invalid/bad page */ -#define P_LEAF2 0x20u /* for MDBX_DUPFIXED records */ -#define P_SUBP 0x40u /* for MDBX_DUPSORT sub-pages */ -#define P_SPILLED 0x2000u /* spilled in parent txn */ -#define P_LOOSE 0x4000u /* page was dirtied then freed, can be reused */ -#define P_FROZEN 0x8000u /* used for retire page with known status */ -#define P_ILL_BITS \ - ((uint16_t)~(P_BRANCH | P_LEAF | P_LEAF2 | P_OVERFLOW | P_SPILLED)) - uint16_t mp_flags; +typedef struct page { + uint64_t txnid; /* txnid which created page, maybe zero in legacy DB */ + uint16_t dupfix_ksize; /* key size if this is a DUPFIX page */ + uint16_t flags; union { - uint32_t mp_pages; /* number of overflow pages */ + uint32_t pages; /* number of overflow pages */ __anonymous_struct_extension__ struct { - indx_t mp_lower; /* lower bound of free space */ - indx_t mp_upper; /* upper bound of free space */ + indx_t lower; /* lower bound of free space */ + indx_t upper; /* upper bound of free space */ }; }; - pgno_t mp_pgno; /* page number */ + pgno_t pgno; /* page number */ -#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ - (!defined(__cplusplus) && defined(_MSC_VER)) - indx_t mp_ptrs[] /* dynamic size */; -#endif /* C99 */ -} MDBX_page; - -#define PAGETYPE_WHOLE(p) ((uint8_t)(p)->mp_flags) - -/* Drop legacy P_DIRTY flag for sub-pages for compatilibity */ -#define PAGETYPE_COMPAT(p) \ - (unlikely(PAGETYPE_WHOLE(p) & P_SUBP) \ - ? PAGETYPE_WHOLE(p) & ~(P_SUBP | P_LEGACY_DIRTY) \ - : PAGETYPE_WHOLE(p)) +#if FLEXIBLE_ARRAY_MEMBERS + indx_t entries[] /* dynamic size */; +#endif /* FLEXIBLE_ARRAY_MEMBERS */ +} page_t; /* Size of the page header, excluding dynamic data at the end */ -#define PAGEHDRSZ offsetof(MDBX_page, mp_ptrs) +#define PAGEHDRSZ 20u -/* Pointer displacement without casting to char* to avoid pointer-aliasing */ -#define ptr_disp(ptr, disp) ((void *)(((intptr_t)(ptr)) + ((intptr_t)(disp)))) +/* Header for a single key/data pair within a page. + * Used in pages of type P_BRANCH and P_LEAF without P_DUPFIX. + * We guarantee 2-byte alignment for 'node_t's. + * + * Leaf node flags describe node contents. N_BIG says the node's + * data part is the page number of an overflow page with actual data. + * N_DUP and N_TREE can be combined giving duplicate data in + * a sub-page/table, and named databases (just N_TREE). */ +typedef struct node { +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + union { + uint32_t dsize; + uint32_t child_pgno; + }; + uint8_t flags; /* see node_flags */ + uint8_t extra; + uint16_t ksize; /* key size */ +#else + uint16_t ksize; /* key size */ + uint8_t extra; + uint8_t flags; /* see node_flags */ + union { + uint32_t child_pgno; + uint32_t dsize; + }; +#endif /* __BYTE_ORDER__ */ -/* Pointer distance as signed number of bytes */ -#define ptr_dist(more, less) (((intptr_t)(more)) - ((intptr_t)(less))) +#if FLEXIBLE_ARRAY_MEMBERS + uint8_t payload[] /* key and data are appended here */; +#endif /* FLEXIBLE_ARRAY_MEMBERS */ +} node_t; + +/* Size of the node header, excluding dynamic data at the end */ +#define NODESIZE 8u -#define mp_next(mp) \ - (*(MDBX_page **)ptr_disp((mp)->mp_ptrs, sizeof(void *) - sizeof(uint32_t))) +typedef enum node_flags { + N_BIG = 0x01 /* data put on large page */, + N_TREE = 0x02 /* data is a b-tree */, + N_DUP = 0x04 /* data has duplicates */ +} node_flags_t; #pragma pack(pop) -typedef struct profgc_stat { +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint8_t page_type(const page_t *mp) { return mp->flags; } + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint8_t page_type_compat(const page_t *mp) { + /* Drop legacy P_DIRTY flag for sub-pages for compatilibity, + * for assertions only. */ + return unlikely(mp->flags & P_SUBP) ? mp->flags & ~(P_SUBP | P_LEGACY_DIRTY) : mp->flags; +} + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_leaf(const page_t *mp) { + return (mp->flags & P_LEAF) != 0; +} + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_dupfix_leaf(const page_t *mp) { + return (mp->flags & P_DUPFIX) != 0; +} + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_branch(const page_t *mp) { + return (mp->flags & P_BRANCH) != 0; +} + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_largepage(const page_t *mp) { + return (mp->flags & P_LARGE) != 0; +} + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_subpage(const page_t *mp) { + return (mp->flags & P_SUBP) != 0; +} + +/* The version number for a database's lockfile format. */ +#define MDBX_LOCK_VERSION 6 + +#if MDBX_LOCKING == MDBX_LOCKING_WIN32FILES + +#define MDBX_LCK_SIGN UINT32_C(0xF10C) +typedef void osal_ipclock_t; +#elif MDBX_LOCKING == MDBX_LOCKING_SYSV + +#define MDBX_LCK_SIGN UINT32_C(0xF18D) +typedef mdbx_pid_t osal_ipclock_t; + +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || MDBX_LOCKING == MDBX_LOCKING_POSIX2008 + +#define MDBX_LCK_SIGN UINT32_C(0x8017) +typedef pthread_mutex_t osal_ipclock_t; + +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988 + +#define MDBX_LCK_SIGN UINT32_C(0xFC29) +typedef sem_t osal_ipclock_t; + +#else +#error "FIXME" +#endif /* MDBX_LOCKING */ + +/* Статистика профилирования работы GC */ +typedef struct gc_prof_stat { /* Монотонное время по "настенным часам" * затраченное на чтение и поиск внутри GC */ uint64_t rtime_monotonic; @@ -2995,42 +2528,44 @@ typedef struct profgc_stat { uint32_t spe_counter; /* page faults (hard page faults) */ uint32_t majflt; -} profgc_stat_t; - -/* Statistics of page operations overall of all (running, completed and aborted) - * transactions */ -typedef struct pgop_stat { - MDBX_atomic_uint64_t newly; /* Quantity of a new pages added */ - MDBX_atomic_uint64_t cow; /* Quantity of pages copied for update */ - MDBX_atomic_uint64_t clone; /* Quantity of parent's dirty pages clones + /* Для разборок с pnl_merge() */ + struct { + uint64_t time; + uint64_t volume; + uint32_t calls; + } pnl_merge; +} gc_prof_stat_t; + +/* Statistics of pages operations for all transactions, + * including incomplete and aborted. */ +typedef struct pgops { + mdbx_atomic_uint64_t newly; /* Quantity of a new pages added */ + mdbx_atomic_uint64_t cow; /* Quantity of pages copied for update */ + mdbx_atomic_uint64_t clone; /* Quantity of parent's dirty pages clones for nested transactions */ - MDBX_atomic_uint64_t split; /* Page splits */ - MDBX_atomic_uint64_t merge; /* Page merges */ - MDBX_atomic_uint64_t spill; /* Quantity of spilled dirty pages */ - MDBX_atomic_uint64_t unspill; /* Quantity of unspilled/reloaded pages */ - MDBX_atomic_uint64_t - wops; /* Number of explicit write operations (not a pages) to a disk */ - MDBX_atomic_uint64_t - msync; /* Number of explicit msync/flush-to-disk operations */ - MDBX_atomic_uint64_t - fsync; /* Number of explicit fsync/flush-to-disk operations */ - - MDBX_atomic_uint64_t prefault; /* Number of prefault write operations */ - MDBX_atomic_uint64_t mincore; /* Number of mincore() calls */ - - MDBX_atomic_uint32_t - incoherence; /* number of https://libmdbx.dqdkfa.ru/dead-github/issues/269 - caught */ - MDBX_atomic_uint32_t reserved; + mdbx_atomic_uint64_t split; /* Page splits */ + mdbx_atomic_uint64_t merge; /* Page merges */ + mdbx_atomic_uint64_t spill; /* Quantity of spilled dirty pages */ + mdbx_atomic_uint64_t unspill; /* Quantity of unspilled/reloaded pages */ + mdbx_atomic_uint64_t wops; /* Number of explicit write operations (not a pages) to a disk */ + mdbx_atomic_uint64_t msync; /* Number of explicit msync/flush-to-disk operations */ + mdbx_atomic_uint64_t fsync; /* Number of explicit fsync/flush-to-disk operations */ + + mdbx_atomic_uint64_t prefault; /* Number of prefault write operations */ + mdbx_atomic_uint64_t mincore; /* Number of mincore() calls */ + + mdbx_atomic_uint32_t incoherence; /* number of https://libmdbx.dqdkfa.ru/dead-github/issues/269 + caught */ + mdbx_atomic_uint32_t reserved; /* Статистика для профилирования GC. - * Логически эти данные может быть стоит вынести в другую структуру, + * Логически эти данные, возможно, стоит вынести в другую структуру, * но разница будет сугубо косметическая. */ struct { /* Затраты на поддержку данных пользователя */ - profgc_stat_t work; + gc_prof_stat_t work; /* Затраты на поддержку и обновления самой GC */ - profgc_stat_t self; + gc_prof_stat_t self; /* Итераций обновления GC, * больше 1 если были повторы/перезапуски */ uint32_t wloops; @@ -3045,33 +2580,6 @@ typedef struct pgop_stat { } gc_prof; } pgop_stat_t; -#if MDBX_LOCKING == MDBX_LOCKING_WIN32FILES -#define MDBX_CLOCK_SIGN UINT32_C(0xF10C) -typedef void osal_ipclock_t; -#elif MDBX_LOCKING == MDBX_LOCKING_SYSV - -#define MDBX_CLOCK_SIGN UINT32_C(0xF18D) -typedef mdbx_pid_t osal_ipclock_t; -#ifndef EOWNERDEAD -#define EOWNERDEAD MDBX_RESULT_TRUE -#endif - -#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ - MDBX_LOCKING == MDBX_LOCKING_POSIX2008 -#define MDBX_CLOCK_SIGN UINT32_C(0x8017) -typedef pthread_mutex_t osal_ipclock_t; -#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988 -#define MDBX_CLOCK_SIGN UINT32_C(0xFC29) -typedef sem_t osal_ipclock_t; -#else -#error "FIXME" -#endif /* MDBX_LOCKING */ - -#if MDBX_LOCKING > MDBX_LOCKING_SYSV && !defined(__cplusplus) -MDBX_INTERNAL_FUNC int osal_ipclock_stub(osal_ipclock_t *ipc); -MDBX_INTERNAL_FUNC int osal_ipclock_destroy(osal_ipclock_t *ipc); -#endif /* MDBX_LOCKING */ - /* Reader Lock Table * * Readers don't acquire any locks for their data access. Instead, they @@ -3081,8 +2589,9 @@ MDBX_INTERNAL_FUNC int osal_ipclock_destroy(osal_ipclock_t *ipc); * read transactions started by the same thread need no further locking to * proceed. * - * If MDBX_NOTLS is set, the slot address is not saved in thread-specific data. - * No reader table is used if the database is on a read-only filesystem. + * If MDBX_NOSTICKYTHREADS is set, the slot address is not saved in + * thread-specific data. No reader table is used if the database is on a + * read-only filesystem. * * Since the database uses multi-version concurrency control, readers don't * actually need any locking. This table is used to keep track of which @@ -3111,14 +2620,14 @@ MDBX_INTERNAL_FUNC int osal_ipclock_destroy(osal_ipclock_t *ipc); * many old transactions together. */ /* The actual reader record, with cacheline padding. */ -typedef struct MDBX_reader { - /* Current Transaction ID when this transaction began, or (txnid_t)-1. +typedef struct reader_slot { + /* Current Transaction ID when this transaction began, or INVALID_TXNID. * Multiple readers that start at the same time will probably have the * same ID here. Again, it's not important to exclude them from * anything; all we need to know is which version of the DB they * started from so we can avoid overwriting any data used in that * particular version. */ - MDBX_atomic_uint64_t /* txnid_t */ mr_txnid; + atomic_txnid_t txnid; /* The information we store in a single slot of the reader table. * In addition to a transaction ID, we also record the process and @@ -3129,181 +2638,421 @@ typedef struct MDBX_reader { * We simply re-init the table when we know that we're the only process * opening the lock file. */ + /* Псевдо thread_id для пометки вытесненных читающих транзакций. */ +#define MDBX_TID_TXN_OUSTED (UINT64_MAX - 1) + + /* Псевдо thread_id для пометки припаркованных читающих транзакций. */ +#define MDBX_TID_TXN_PARKED UINT64_MAX + /* The thread ID of the thread owning this txn. */ - MDBX_atomic_uint64_t mr_tid; + mdbx_atomic_uint64_t tid; /* The process ID of the process owning this reader txn. */ - MDBX_atomic_uint32_t mr_pid; + mdbx_atomic_uint32_t pid; /* The number of pages used in the reader's MVCC snapshot, - * i.e. the value of meta->mm_geo.next and txn->mt_next_pgno */ - atomic_pgno_t mr_snapshot_pages_used; + * i.e. the value of meta->geometry.first_unallocated and + * txn->geo.first_unallocated */ + atomic_pgno_t snapshot_pages_used; /* Number of retired pages at the time this reader starts transaction. So, - * at any time the difference mm_pages_retired - mr_snapshot_pages_retired - * will give the number of pages which this reader restraining from reuse. */ - MDBX_atomic_uint64_t mr_snapshot_pages_retired; -} MDBX_reader; + * at any time the difference meta.pages_retired - + * reader.snapshot_pages_retired will give the number of pages which this + * reader restraining from reuse. */ + mdbx_atomic_uint64_t snapshot_pages_retired; +} reader_slot_t; /* The header for the reader table (a memory-mapped lock file). */ -typedef struct MDBX_lockinfo { +typedef struct shared_lck { /* Stamp identifying this as an MDBX file. * It must be set to MDBX_MAGIC with with MDBX_LOCK_VERSION. */ - uint64_t mti_magic_and_version; + uint64_t magic_and_version; /* Format of this lock file. Must be set to MDBX_LOCK_FORMAT. */ - uint32_t mti_os_and_format; + uint32_t os_and_format; /* Flags which environment was opened. */ - MDBX_atomic_uint32_t mti_envmode; + mdbx_atomic_uint32_t envmode; /* Threshold of un-synced-with-disk pages for auto-sync feature, * zero means no-threshold, i.e. auto-sync is disabled. */ - atomic_pgno_t mti_autosync_threshold; + atomic_pgno_t autosync_threshold; /* Low 32-bit of txnid with which meta-pages was synced, * i.e. for sync-polling in the MDBX_NOMETASYNC mode. */ #define MDBX_NOMETASYNC_LAZY_UNK (UINT32_MAX / 3) #define MDBX_NOMETASYNC_LAZY_FD (MDBX_NOMETASYNC_LAZY_UNK + UINT32_MAX / 8) -#define MDBX_NOMETASYNC_LAZY_WRITEMAP \ - (MDBX_NOMETASYNC_LAZY_UNK - UINT32_MAX / 8) - MDBX_atomic_uint32_t mti_meta_sync_txnid; +#define MDBX_NOMETASYNC_LAZY_WRITEMAP (MDBX_NOMETASYNC_LAZY_UNK - UINT32_MAX / 8) + mdbx_atomic_uint32_t meta_sync_txnid; /* Period for timed auto-sync feature, i.e. at the every steady checkpoint - * the mti_unsynced_timeout sets to the current_time + mti_autosync_period. + * the mti_unsynced_timeout sets to the current_time + autosync_period. * The time value is represented in a suitable system-dependent form, for * example clock_gettime(CLOCK_BOOTTIME) or clock_gettime(CLOCK_MONOTONIC). * Zero means timed auto-sync is disabled. */ - MDBX_atomic_uint64_t mti_autosync_period; + mdbx_atomic_uint64_t autosync_period; /* Marker to distinguish uniqueness of DB/CLK. */ - MDBX_atomic_uint64_t mti_bait_uniqueness; + mdbx_atomic_uint64_t bait_uniqueness; /* Paired counter of processes that have mlock()ed part of mmapped DB. - * The (mti_mlcnt[0] - mti_mlcnt[1]) > 0 means at least one process + * The (mlcnt[0] - mlcnt[1]) > 0 means at least one process * lock at least one page, so therefore madvise() could return EINVAL. */ - MDBX_atomic_uint32_t mti_mlcnt[2]; + mdbx_atomic_uint32_t mlcnt[2]; MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/ /* Statistics of costly ops of all (running, completed and aborted) * transactions */ - pgop_stat_t mti_pgop_stat; + pgop_stat_t pgops; MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/ - /* Write transaction lock. */ #if MDBX_LOCKING > 0 - osal_ipclock_t mti_wlock; + /* Write transaction lock. */ + osal_ipclock_t wrt_lock; #endif /* MDBX_LOCKING > 0 */ - atomic_txnid_t mti_oldest_reader; + atomic_txnid_t cached_oldest; /* Timestamp of entering an out-of-sync state. Value is represented in a * suitable system-dependent form, for example clock_gettime(CLOCK_BOOTTIME) * or clock_gettime(CLOCK_MONOTONIC). */ - MDBX_atomic_uint64_t mti_eoos_timestamp; + mdbx_atomic_uint64_t eoos_timestamp; /* Number un-synced-with-disk pages for auto-sync feature. */ - MDBX_atomic_uint64_t mti_unsynced_pages; + mdbx_atomic_uint64_t unsynced_pages; /* Timestamp of the last readers check. */ - MDBX_atomic_uint64_t mti_reader_check_timestamp; + mdbx_atomic_uint64_t readers_check_timestamp; /* Number of page which was discarded last time by madvise(DONTNEED). */ - atomic_pgno_t mti_discarded_tail; + atomic_pgno_t discarded_tail; /* Shared anchor for tracking readahead edge and enabled/disabled status. */ - pgno_t mti_readahead_anchor; + pgno_t readahead_anchor; /* Shared cache for mincore() results */ struct { pgno_t begin[4]; uint64_t mask[4]; - } mti_mincore_cache; + } mincore_cache; MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/ - /* Readeaders registration lock. */ #if MDBX_LOCKING > 0 - osal_ipclock_t mti_rlock; + /* Readeaders table lock. */ + osal_ipclock_t rdt_lock; #endif /* MDBX_LOCKING > 0 */ /* The number of slots that have been used in the reader table. * This always records the maximum count, it is not decremented * when readers release their slots. */ - MDBX_atomic_uint32_t mti_numreaders; - MDBX_atomic_uint32_t mti_readers_refresh_flag; + mdbx_atomic_uint32_t rdt_length; + mdbx_atomic_uint32_t rdt_refresh_flag; + +#if FLEXIBLE_ARRAY_MEMBERS + MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/ + reader_slot_t rdt[] /* dynamic size */; + +/* Lockfile format signature: version, features and field layout */ +#define MDBX_LOCK_FORMAT \ + (MDBX_LCK_SIGN * 27733 + (unsigned)sizeof(reader_slot_t) * 13 + \ + (unsigned)offsetof(reader_slot_t, snapshot_pages_used) * 251 + (unsigned)offsetof(lck_t, cached_oldest) * 83 + \ + (unsigned)offsetof(lck_t, rdt_length) * 37 + (unsigned)offsetof(lck_t, rdt) * 29) +#endif /* FLEXIBLE_ARRAY_MEMBERS */ +} lck_t; + +#define MDBX_LOCK_MAGIC ((MDBX_MAGIC << 8) + MDBX_LOCK_VERSION) + +#define MDBX_READERS_LIMIT 32767 + +#define MIN_MAPSIZE (MDBX_MIN_PAGESIZE * MIN_PAGENO) +#if defined(_WIN32) || defined(_WIN64) +#define MAX_MAPSIZE32 UINT32_C(0x38000000) +#else +#define MAX_MAPSIZE32 UINT32_C(0x7f000000) +#endif +#define MAX_MAPSIZE64 ((MAX_PAGENO + 1) * (uint64_t)MDBX_MAX_PAGESIZE) + +#if MDBX_WORDBITS >= 64 +#define MAX_MAPSIZE MAX_MAPSIZE64 +#define PAGELIST_LIMIT ((size_t)MAX_PAGENO) +#else +#define MAX_MAPSIZE MAX_MAPSIZE32 +#define PAGELIST_LIMIT (MAX_MAPSIZE32 / MDBX_MIN_PAGESIZE) +#endif /* MDBX_WORDBITS */ + +#define MDBX_GOLD_RATIO_DBL 1.6180339887498948482 +#define MEGABYTE ((size_t)1 << 20) + +/*----------------------------------------------------------------------------*/ + +union logger_union { + void *ptr; + MDBX_debug_func *fmt; + MDBX_debug_func_nofmt *nofmt; +}; + +struct libmdbx_globals { + bin128_t bootid; + unsigned sys_pagesize, sys_allocation_granularity; + uint8_t sys_pagesize_ln2; + uint8_t runtime_flags; + uint8_t loglevel; +#if defined(_WIN32) || defined(_WIN64) + bool running_under_Wine; +#elif defined(__linux__) || defined(__gnu_linux__) + bool running_on_WSL1 /* Windows Subsystem 1 for Linux */; + uint32_t linux_kernel_version; +#endif /* Linux */ + union logger_union logger; + osal_fastmutex_t debug_lock; + size_t logger_buffer_size; + char *logger_buffer; +}; + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +extern struct libmdbx_globals globals; +#if defined(_WIN32) || defined(_WIN64) +extern struct libmdbx_imports imports; +#endif /* Windows */ + +#ifndef __Wpedantic_format_voidptr +MDBX_MAYBE_UNUSED static inline const void *__Wpedantic_format_voidptr(const void *ptr) { return ptr; } +#define __Wpedantic_format_voidptr(ARG) __Wpedantic_format_voidptr(ARG) +#endif /* __Wpedantic_format_voidptr */ + +MDBX_INTERNAL void MDBX_PRINTF_ARGS(4, 5) debug_log(int level, const char *function, int line, const char *fmt, ...) + MDBX_PRINTF_ARGS(4, 5); +MDBX_INTERNAL void debug_log_va(int level, const char *function, int line, const char *fmt, va_list args); + +#if MDBX_DEBUG +#define LOG_ENABLED(LVL) unlikely(LVL <= globals.loglevel) +#define AUDIT_ENABLED() unlikely((globals.runtime_flags & (unsigned)MDBX_DBG_AUDIT)) +#else /* MDBX_DEBUG */ +#define LOG_ENABLED(LVL) (LVL < MDBX_LOG_VERBOSE && LVL <= globals.loglevel) +#define AUDIT_ENABLED() (0) +#endif /* LOG_ENABLED() & AUDIT_ENABLED() */ + +#if MDBX_FORCE_ASSERTIONS +#define ASSERT_ENABLED() (1) +#elif MDBX_DEBUG +#define ASSERT_ENABLED() likely((globals.runtime_flags & (unsigned)MDBX_DBG_ASSERT)) +#else +#define ASSERT_ENABLED() (0) +#endif /* ASSERT_ENABLED() */ + +#define DEBUG_EXTRA(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_EXTRA)) \ + debug_log(MDBX_LOG_EXTRA, __func__, __LINE__, fmt, __VA_ARGS__); \ + } while (0) + +#define DEBUG_EXTRA_PRINT(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_EXTRA)) \ + debug_log(MDBX_LOG_EXTRA, nullptr, 0, fmt, __VA_ARGS__); \ + } while (0) + +#define TRACE(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_TRACE)) \ + debug_log(MDBX_LOG_TRACE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ + } while (0) + +#define DEBUG(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_DEBUG)) \ + debug_log(MDBX_LOG_DEBUG, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ + } while (0) + +#define VERBOSE(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_VERBOSE)) \ + debug_log(MDBX_LOG_VERBOSE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ + } while (0) + +#define NOTICE(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_NOTICE)) \ + debug_log(MDBX_LOG_NOTICE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ + } while (0) + +#define WARNING(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_WARN)) \ + debug_log(MDBX_LOG_WARN, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ + } while (0) + +#undef ERROR /* wingdi.h \ + Yeah, morons from M$ put such definition to the public header. */ + +#define ERROR(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_ERROR)) \ + debug_log(MDBX_LOG_ERROR, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ + } while (0) + +#define FATAL(fmt, ...) debug_log(MDBX_LOG_FATAL, __func__, __LINE__, fmt "\n", __VA_ARGS__); + +#if MDBX_DEBUG +#define ASSERT_FAIL(env, msg, func, line) mdbx_assert_fail(env, msg, func, line) +#else /* MDBX_DEBUG */ +MDBX_NORETURN __cold void assert_fail(const char *msg, const char *func, unsigned line); +#define ASSERT_FAIL(env, msg, func, line) \ + do { \ + (void)(env); \ + assert_fail(msg, func, line); \ + } while (0) +#endif /* MDBX_DEBUG */ + +#define ENSURE_MSG(env, expr, msg) \ + do { \ + if (unlikely(!(expr))) \ + ASSERT_FAIL(env, msg, __func__, __LINE__); \ + } while (0) + +#define ENSURE(env, expr) ENSURE_MSG(env, expr, #expr) + +/* assert(3) variant in environment context */ +#define eASSERT(env, expr) \ + do { \ + if (ASSERT_ENABLED()) \ + ENSURE(env, expr); \ + } while (0) + +/* assert(3) variant in cursor context */ +#define cASSERT(mc, expr) eASSERT((mc)->txn->env, expr) + +/* assert(3) variant in transaction context */ +#define tASSERT(txn, expr) eASSERT((txn)->env, expr) + +#ifndef xMDBX_TOOLS /* Avoid using internal eASSERT() */ +#undef assert +#define assert(expr) eASSERT(nullptr, expr) +#endif + +MDBX_MAYBE_UNUSED static inline void jitter4testing(bool tiny) { +#if MDBX_DEBUG + if (globals.runtime_flags & (unsigned)MDBX_DBG_JITTER) + osal_jitter(tiny); +#else + (void)tiny; +#endif +} + +MDBX_MAYBE_UNUSED MDBX_INTERNAL void page_list(page_t *mp); + +MDBX_INTERNAL const char *pagetype_caption(const uint8_t type, char buf4unknown[16]); +/* Key size which fits in a DKBUF (debug key buffer). */ +#define DKBUF_MAX 127 +#define DKBUF char dbg_kbuf[DKBUF_MAX * 4 + 2] +#define DKEY(x) mdbx_dump_val(x, dbg_kbuf, DKBUF_MAX * 2 + 1) +#define DVAL(x) mdbx_dump_val(x, dbg_kbuf + DKBUF_MAX * 2 + 1, DKBUF_MAX * 2 + 1) + +#if MDBX_DEBUG +#define DKBUF_DEBUG DKBUF +#define DKEY_DEBUG(x) DKEY(x) +#define DVAL_DEBUG(x) DVAL(x) +#else +#define DKBUF_DEBUG ((void)(0)) +#define DKEY_DEBUG(x) ("-") +#define DVAL_DEBUG(x) ("-") +#endif + +MDBX_INTERNAL void log_error(const int err, const char *func, unsigned line); + +MDBX_MAYBE_UNUSED static inline int log_if_error(const int err, const char *func, unsigned line) { + if (unlikely(err != MDBX_SUCCESS)) + log_error(err, func, line); + return err; +} + +#define LOG_IFERR(err) log_if_error((err), __func__, __LINE__) + +/* Test if the flags f are set in a flag word w. */ +#define F_ISSET(w, f) (((w) & (f)) == (f)) + +/* Round n up to an even number. */ +#define EVEN_CEIL(n) (((n) + 1UL) & -2L) /* sign-extending -2 to match n+1U */ + +/* Round n down to an even number. */ +#define EVEN_FLOOR(n) ((n) & ~(size_t)1) + +/* + * / + * | -1, a < b + * CMP2INT(a,b) = < 0, a == b + * | 1, a > b + * \ + */ +#define CMP2INT(a, b) (((a) != (b)) ? (((a) < (b)) ? -1 : 1) : 0) + +/* Pointer displacement without casting to char* to avoid pointer-aliasing */ +#define ptr_disp(ptr, disp) ((void *)(((intptr_t)(ptr)) + ((intptr_t)(disp)))) -#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ - (!defined(__cplusplus) && defined(_MSC_VER)) - MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/ - MDBX_reader mti_readers[] /* dynamic size */; -#endif /* C99 */ -} MDBX_lockinfo; +/* Pointer distance as signed number of bytes */ +#define ptr_dist(more, less) (((intptr_t)(more)) - ((intptr_t)(less))) -/* Lockfile format signature: version, features and field layout */ -#define MDBX_LOCK_FORMAT \ - (MDBX_CLOCK_SIGN * 27733 + (unsigned)sizeof(MDBX_reader) * 13 + \ - (unsigned)offsetof(MDBX_reader, mr_snapshot_pages_used) * 251 + \ - (unsigned)offsetof(MDBX_lockinfo, mti_oldest_reader) * 83 + \ - (unsigned)offsetof(MDBX_lockinfo, mti_numreaders) * 37 + \ - (unsigned)offsetof(MDBX_lockinfo, mti_readers) * 29) +#define MDBX_ASAN_POISON_MEMORY_REGION(addr, size) \ + do { \ + TRACE("POISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), (size_t)(size), __LINE__); \ + ASAN_POISON_MEMORY_REGION(addr, size); \ + } while (0) -#define MDBX_DATA_MAGIC \ - ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + MDBX_DATA_VERSION) +#define MDBX_ASAN_UNPOISON_MEMORY_REGION(addr, size) \ + do { \ + TRACE("UNPOISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), (size_t)(size), __LINE__); \ + ASAN_UNPOISON_MEMORY_REGION(addr, size); \ + } while (0) -#define MDBX_DATA_MAGIC_LEGACY_COMPAT \ - ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + 2) +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline size_t branchless_abs(intptr_t value) { + assert(value > INT_MIN); + const size_t expanded_sign = (size_t)(value >> (sizeof(value) * CHAR_BIT - 1)); + return ((size_t)value + expanded_sign) ^ expanded_sign; +} -#define MDBX_DATA_MAGIC_LEGACY_DEVEL ((MDBX_MAGIC << 8) + 255) +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline bool is_powerof2(size_t x) { return (x & (x - 1)) == 0; } -#define MDBX_LOCK_MAGIC ((MDBX_MAGIC << 8) + MDBX_LOCK_VERSION) +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline size_t floor_powerof2(size_t value, size_t granularity) { + assert(is_powerof2(granularity)); + return value & ~(granularity - 1); +} -/* The maximum size of a database page. - * - * It is 64K, but value-PAGEHDRSZ must fit in MDBX_page.mp_upper. - * - * MDBX will use database pages < OS pages if needed. - * That causes more I/O in write transactions: The OS must - * know (read) the whole page before writing a partial page. - * - * Note that we don't currently support Huge pages. On Linux, - * regular data files cannot use Huge pages, and in general - * Huge pages aren't actually pageable. We rely on the OS - * demand-pager to read our data and page it out when memory - * pressure from other processes is high. So until OSs have - * actual paging support for Huge pages, they're not viable. */ -#define MAX_PAGESIZE MDBX_MAX_PAGESIZE -#define MIN_PAGESIZE MDBX_MIN_PAGESIZE - -#define MIN_MAPSIZE (MIN_PAGESIZE * MIN_PAGENO) -#if defined(_WIN32) || defined(_WIN64) -#define MAX_MAPSIZE32 UINT32_C(0x38000000) -#else -#define MAX_MAPSIZE32 UINT32_C(0x7f000000) -#endif -#define MAX_MAPSIZE64 ((MAX_PAGENO + 1) * (uint64_t)MAX_PAGESIZE) +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline size_t ceil_powerof2(size_t value, size_t granularity) { + return floor_powerof2(value + granularity - 1, granularity); +} -#if MDBX_WORDBITS >= 64 -#define MAX_MAPSIZE MAX_MAPSIZE64 -#define MDBX_PGL_LIMIT ((size_t)MAX_PAGENO) -#else -#define MAX_MAPSIZE MAX_MAPSIZE32 -#define MDBX_PGL_LIMIT (MAX_MAPSIZE32 / MIN_PAGESIZE) -#endif /* MDBX_WORDBITS */ +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED MDBX_INTERNAL unsigned log2n_powerof2(size_t value_uintptr); -#define MDBX_READERS_LIMIT 32767 -#define MDBX_RADIXSORT_THRESHOLD 142 -#define MDBX_GOLD_RATIO_DBL 1.6180339887498948482 +MDBX_NOTHROW_CONST_FUNCTION MDBX_INTERNAL uint64_t rrxmrrxmsx_0(uint64_t v); -/*----------------------------------------------------------------------------*/ +struct monotime_cache { + uint64_t value; + int expire_countdown; +}; + +MDBX_MAYBE_UNUSED static inline uint64_t monotime_since_cached(uint64_t begin_timestamp, struct monotime_cache *cache) { + if (cache->expire_countdown) + cache->expire_countdown -= 1; + else { + cache->value = osal_monotime(); + cache->expire_countdown = 42 / 3; + } + return cache->value - begin_timestamp; +} /* An PNL is an Page Number List, a sorted array of IDs. + * * The first element of the array is a counter for how many actual page-numbers * are in the list. By default PNLs are sorted in descending order, this allow * cut off a page with lowest pgno (at the tail) just truncating the list. The * sort order of PNLs is controlled by the MDBX_PNL_ASCENDING build option. */ -typedef pgno_t *MDBX_PNL; +typedef pgno_t *pnl_t; +typedef const pgno_t *const_pnl_t; #if MDBX_PNL_ASCENDING #define MDBX_PNL_ORDERED(first, last) ((first) < (last)) @@ -3313,46 +3062,17 @@ typedef pgno_t *MDBX_PNL; #define MDBX_PNL_DISORDERED(first, last) ((first) <= (last)) #endif -/* List of txnid, only for MDBX_txn.tw.lifo_reclaimed */ -typedef txnid_t *MDBX_TXL; - -/* An Dirty-Page list item is an pgno/pointer pair. */ -typedef struct MDBX_dp { - MDBX_page *ptr; - pgno_t pgno, npages; -} MDBX_dp; - -/* An DPL (dirty-page list) is a sorted array of MDBX_DPs. */ -typedef struct MDBX_dpl { - size_t sorted; - size_t length; - size_t pages_including_loose; /* number of pages, but not an entries. */ - size_t detent; /* allocated size excluding the MDBX_DPL_RESERVE_GAP */ -#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ - (!defined(__cplusplus) && defined(_MSC_VER)) - MDBX_dp items[] /* dynamic size with holes at zero and after the last */; -#endif -} MDBX_dpl; - -/* PNL sizes */ #define MDBX_PNL_GRANULATE_LOG2 10 #define MDBX_PNL_GRANULATE (1 << MDBX_PNL_GRANULATE_LOG2) -#define MDBX_PNL_INITIAL \ - (MDBX_PNL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t)) - -#define MDBX_TXL_GRANULATE 32 -#define MDBX_TXL_INITIAL \ - (MDBX_TXL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t)) -#define MDBX_TXL_MAX \ - ((1u << 26) - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t)) +#define MDBX_PNL_INITIAL (MDBX_PNL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t)) #define MDBX_PNL_ALLOCLEN(pl) ((pl)[-1]) #define MDBX_PNL_GETSIZE(pl) ((size_t)((pl)[0])) -#define MDBX_PNL_SETSIZE(pl, size) \ - do { \ - const size_t __size = size; \ - assert(__size < INT_MAX); \ - (pl)[0] = (pgno_t)__size; \ +#define MDBX_PNL_SETSIZE(pl, size) \ + do { \ + const size_t __size = size; \ + assert(__size < INT_MAX); \ + (pl)[0] = (pgno_t)__size; \ } while (0) #define MDBX_PNL_FIRST(pl) ((pl)[1]) #define MDBX_PNL_LAST(pl) ((pl)[MDBX_PNL_GETSIZE(pl)]) @@ -3372,697 +3092,134 @@ typedef struct MDBX_dpl { #define MDBX_PNL_SIZEOF(pl) ((MDBX_PNL_GETSIZE(pl) + 1) * sizeof(pgno_t)) #define MDBX_PNL_IS_EMPTY(pl) (MDBX_PNL_GETSIZE(pl) == 0) -/*----------------------------------------------------------------------------*/ -/* Internal structures */ - -/* Auxiliary DB info. - * The information here is mostly static/read-only. There is - * only a single copy of this record in the environment. */ -typedef struct MDBX_dbx { - MDBX_val md_name; /* name of the database */ - MDBX_cmp_func *md_cmp; /* function for comparing keys */ - MDBX_cmp_func *md_dcmp; /* function for comparing data items */ - size_t md_klen_min, md_klen_max; /* min/max key length for the database */ - size_t md_vlen_min, - md_vlen_max; /* min/max value/data length for the database */ -} MDBX_dbx; - -typedef struct troika { - uint8_t fsm, recent, prefer_steady, tail_and_flags; -#if MDBX_WORDBITS > 32 /* Workaround for false-positives from Valgrind */ - uint32_t unused_pad; -#endif -#define TROIKA_HAVE_STEADY(troika) ((troika)->fsm & 7) -#define TROIKA_STRICT_VALID(troika) ((troika)->tail_and_flags & 64) -#define TROIKA_VALID(troika) ((troika)->tail_and_flags & 128) -#define TROIKA_TAIL(troika) ((troika)->tail_and_flags & 3) - txnid_t txnid[NUM_METAS]; -} meta_troika_t; - -/* A database transaction. - * Every operation requires a transaction handle. */ -struct MDBX_txn { -#define MDBX_MT_SIGNATURE UINT32_C(0x93D53A31) - uint32_t mt_signature; - - /* Transaction Flags */ - /* mdbx_txn_begin() flags */ -#define MDBX_TXN_RO_BEGIN_FLAGS (MDBX_TXN_RDONLY | MDBX_TXN_RDONLY_PREPARE) -#define MDBX_TXN_RW_BEGIN_FLAGS \ - (MDBX_TXN_NOMETASYNC | MDBX_TXN_NOSYNC | MDBX_TXN_TRY) - /* Additional flag for sync_locked() */ -#define MDBX_SHRINK_ALLOWED UINT32_C(0x40000000) - -#define MDBX_TXN_DRAINED_GC 0x20 /* GC was depleted up to oldest reader */ - -#define TXN_FLAGS \ - (MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_DIRTY | MDBX_TXN_SPILLS | \ - MDBX_TXN_HAS_CHILD | MDBX_TXN_INVALID | MDBX_TXN_DRAINED_GC) - -#if (TXN_FLAGS & (MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_RO_BEGIN_FLAGS)) || \ - ((MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_RO_BEGIN_FLAGS | TXN_FLAGS) & \ - MDBX_SHRINK_ALLOWED) -#error "Oops, some txn flags overlapped or wrong" -#endif - uint32_t mt_flags; - - MDBX_txn *mt_parent; /* parent of a nested txn */ - /* Nested txn under this txn, set together with flag MDBX_TXN_HAS_CHILD */ - MDBX_txn *mt_child; - MDBX_geo mt_geo; - /* next unallocated page */ -#define mt_next_pgno mt_geo.next - /* corresponding to the current size of datafile */ -#define mt_end_pgno mt_geo.now - - /* The ID of this transaction. IDs are integers incrementing from - * INITIAL_TXNID. Only committed write transactions increment the ID. If a - * transaction aborts, the ID may be re-used by the next writer. */ - txnid_t mt_txnid; - txnid_t mt_front; - - MDBX_env *mt_env; /* the DB environment */ - /* Array of records for each DB known in the environment. */ - MDBX_dbx *mt_dbxs; - /* Array of MDBX_db records for each known DB */ - MDBX_db *mt_dbs; - /* Array of sequence numbers for each DB handle */ - MDBX_atomic_uint32_t *mt_dbiseqs; - - /* Transaction DBI Flags */ -#define DBI_DIRTY MDBX_DBI_DIRTY /* DB was written in this txn */ -#define DBI_STALE MDBX_DBI_STALE /* Named-DB record is older than txnID */ -#define DBI_FRESH MDBX_DBI_FRESH /* Named-DB handle opened in this txn */ -#define DBI_CREAT MDBX_DBI_CREAT /* Named-DB handle created in this txn */ -#define DBI_VALID 0x10 /* DB handle is valid, see also DB_VALID */ -#define DBI_USRVALID 0x20 /* As DB_VALID, but not set for FREE_DBI */ -#define DBI_AUDITED 0x40 /* Internal flag for accounting during audit */ - /* Array of flags for each DB */ - uint8_t *mt_dbistate; - /* Number of DB records in use, or 0 when the txn is finished. - * This number only ever increments until the txn finishes; we - * don't decrement it when individual DB handles are closed. */ - MDBX_dbi mt_numdbs; - size_t mt_owner; /* thread ID that owns this transaction */ - MDBX_canary mt_canary; - void *mt_userctx; /* User-settable context */ - MDBX_cursor **mt_cursors; - - union { - struct { - /* For read txns: This thread/txn's reader table slot, or NULL. */ - MDBX_reader *reader; - } to; - struct { - meta_troika_t troika; - /* In write txns, array of cursors for each DB */ - MDBX_PNL relist; /* Reclaimed GC pages */ - txnid_t last_reclaimed; /* ID of last used record */ -#if MDBX_ENABLE_REFUND - pgno_t loose_refund_wl /* FIXME: describe */; -#endif /* MDBX_ENABLE_REFUND */ - /* a sequence to spilling dirty page with LRU policy */ - unsigned dirtylru; - /* dirtylist room: Dirty array size - dirty pages visible to this txn. - * Includes ancestor txns' dirty pages not hidden by other txns' - * dirty/spilled pages. Thus commit(nested txn) has room to merge - * dirtylist into mt_parent after freeing hidden mt_parent pages. */ - size_t dirtyroom; - /* For write txns: Modified pages. Sorted when not MDBX_WRITEMAP. */ - MDBX_dpl *dirtylist; - /* The list of reclaimed txns from GC */ - MDBX_TXL lifo_reclaimed; - /* The list of pages that became unused during this transaction. */ - MDBX_PNL retired_pages; - /* The list of loose pages that became unused and may be reused - * in this transaction, linked through `mp_next`. */ - MDBX_page *loose_pages; - /* Number of loose pages (tw.loose_pages) */ - size_t loose_count; - union { - struct { - size_t least_removed; - /* The sorted list of dirty pages we temporarily wrote to disk - * because the dirty list was full. page numbers in here are - * shifted left by 1, deleted slots have the LSB set. */ - MDBX_PNL list; - } spilled; - size_t writemap_dirty_npages; - size_t writemap_spilled_npages; - }; - } tw; - }; -}; - -#if MDBX_WORDBITS >= 64 -#define CURSOR_STACK 32 -#else -#define CURSOR_STACK 24 -#endif - -struct MDBX_xcursor; - -/* Cursors are used for all DB operations. - * A cursor holds a path of (page pointer, key index) from the DB - * root to a position in the DB, plus other state. MDBX_DUPSORT - * cursors include an xcursor to the current data item. Write txns - * track their cursors and keep them up to date when data moves. - * Exception: An xcursor's pointer to a P_SUBP page can be stale. - * (A node with F_DUPDATA but no F_SUBDATA contains a subpage). */ -struct MDBX_cursor { -#define MDBX_MC_LIVE UINT32_C(0xFE05D5B1) -#define MDBX_MC_READY4CLOSE UINT32_C(0x2817A047) -#define MDBX_MC_WAIT4EOT UINT32_C(0x90E297A7) - uint32_t mc_signature; - /* The database handle this cursor operates on */ - MDBX_dbi mc_dbi; - /* Next cursor on this DB in this txn */ - MDBX_cursor *mc_next; - /* Backup of the original cursor if this cursor is a shadow */ - MDBX_cursor *mc_backup; - /* Context used for databases with MDBX_DUPSORT, otherwise NULL */ - struct MDBX_xcursor *mc_xcursor; - /* The transaction that owns this cursor */ - MDBX_txn *mc_txn; - /* The database record for this cursor */ - MDBX_db *mc_db; - /* The database auxiliary record for this cursor */ - MDBX_dbx *mc_dbx; - /* The mt_dbistate for this database */ - uint8_t *mc_dbistate; - uint8_t mc_snum; /* number of pushed pages */ - uint8_t mc_top; /* index of top page, normally mc_snum-1 */ - - /* Cursor state flags. */ -#define C_INITIALIZED 0x01 /* cursor has been initialized and is valid */ -#define C_EOF 0x02 /* No more data */ -#define C_SUB 0x04 /* Cursor is a sub-cursor */ -#define C_DEL 0x08 /* last op was a cursor_del */ -#define C_UNTRACK 0x10 /* Un-track cursor when closing */ -#define C_GCU \ - 0x20 /* Происходит подготовка к обновлению GC, поэтому \ - * можно брать страницы из GC даже для FREE_DBI */ - uint8_t mc_flags; - - /* Cursor checking flags. */ -#define CC_BRANCH 0x01 /* same as P_BRANCH for CHECK_LEAF_TYPE() */ -#define CC_LEAF 0x02 /* same as P_LEAF for CHECK_LEAF_TYPE() */ -#define CC_OVERFLOW 0x04 /* same as P_OVERFLOW for CHECK_LEAF_TYPE() */ -#define CC_UPDATING 0x08 /* update/rebalance pending */ -#define CC_SKIPORD 0x10 /* don't check keys ordering */ -#define CC_LEAF2 0x20 /* same as P_LEAF2 for CHECK_LEAF_TYPE() */ -#define CC_RETIRING 0x40 /* refs to child pages may be invalid */ -#define CC_PAGECHECK 0x80 /* perform page checking, see MDBX_VALIDATION */ - uint8_t mc_checking; - - MDBX_page *mc_pg[CURSOR_STACK]; /* stack of pushed pages */ - indx_t mc_ki[CURSOR_STACK]; /* stack of page indices */ -}; - -#define CHECK_LEAF_TYPE(mc, mp) \ - (((PAGETYPE_WHOLE(mp) ^ (mc)->mc_checking) & \ - (CC_BRANCH | CC_LEAF | CC_OVERFLOW | CC_LEAF2)) == 0) - -/* Context for sorted-dup records. - * We could have gone to a fully recursive design, with arbitrarily - * deep nesting of sub-databases. But for now we only handle these - * levels - main DB, optional sub-DB, sorted-duplicate DB. */ -typedef struct MDBX_xcursor { - /* A sub-cursor for traversing the Dup DB */ - MDBX_cursor mx_cursor; - /* The database record for this Dup DB */ - MDBX_db mx_db; - /* The auxiliary DB record for this Dup DB */ - MDBX_dbx mx_dbx; -} MDBX_xcursor; - -typedef struct MDBX_cursor_couple { - MDBX_cursor outer; - void *mc_userctx; /* User-settable context */ - MDBX_xcursor inner; -} MDBX_cursor_couple; - -/* The database environment. */ -struct MDBX_env { - /* ----------------------------------------------------- mostly static part */ -#define MDBX_ME_SIGNATURE UINT32_C(0x9A899641) - MDBX_atomic_uint32_t me_signature; - /* Failed to update the meta page. Probably an I/O error. */ -#define MDBX_FATAL_ERROR UINT32_C(0x80000000) - /* Some fields are initialized. */ -#define MDBX_ENV_ACTIVE UINT32_C(0x20000000) - /* me_txkey is set */ -#define MDBX_ENV_TXKEY UINT32_C(0x10000000) - /* Legacy MDBX_MAPASYNC (prior v0.9) */ -#define MDBX_DEPRECATED_MAPASYNC UINT32_C(0x100000) - /* Legacy MDBX_COALESCE (prior v0.12) */ -#define MDBX_DEPRECATED_COALESCE UINT32_C(0x2000000) -#define ENV_INTERNAL_FLAGS (MDBX_FATAL_ERROR | MDBX_ENV_ACTIVE | MDBX_ENV_TXKEY) - uint32_t me_flags; - osal_mmap_t me_dxb_mmap; /* The main data file */ -#define me_map me_dxb_mmap.base -#define me_lazy_fd me_dxb_mmap.fd - mdbx_filehandle_t me_dsync_fd, me_fd4meta; -#if defined(_WIN32) || defined(_WIN64) -#define me_overlapped_fd me_ioring.overlapped_fd - HANDLE me_data_lock_event; -#endif /* Windows */ - osal_mmap_t me_lck_mmap; /* The lock file */ -#define me_lfd me_lck_mmap.fd - struct MDBX_lockinfo *me_lck; - - unsigned me_psize; /* DB page size, initialized from me_os_psize */ - uint16_t me_leaf_nodemax; /* max size of a leaf-node */ - uint16_t me_branch_nodemax; /* max size of a branch-node */ - uint16_t me_subpage_limit; - uint16_t me_subpage_room_threshold; - uint16_t me_subpage_reserve_prereq; - uint16_t me_subpage_reserve_limit; - atomic_pgno_t me_mlocked_pgno; - uint8_t me_psize2log; /* log2 of DB page size */ - int8_t me_stuck_meta; /* recovery-only: target meta page or less that zero */ - uint16_t me_merge_threshold, - me_merge_threshold_gc; /* pages emptier than this are candidates for - merging */ - unsigned me_os_psize; /* OS page size, from osal_syspagesize() */ - unsigned me_maxreaders; /* size of the reader table */ - MDBX_dbi me_maxdbs; /* size of the DB table */ - uint32_t me_pid; /* process ID of this env */ - osal_thread_key_t me_txkey; /* thread-key for readers */ - pathchar_t *me_pathname; /* path to the DB files */ - void *me_pbuf; /* scratch area for DUPSORT put() */ - MDBX_txn *me_txn0; /* preallocated write transaction */ - - MDBX_dbx *me_dbxs; /* array of static DB info */ - uint16_t *me_dbflags; /* array of flags from MDBX_db.md_flags */ - MDBX_atomic_uint32_t *me_dbiseqs; /* array of dbi sequence numbers */ - unsigned - me_maxgc_ov1page; /* Number of pgno_t fit in a single overflow page */ - unsigned me_maxgc_per_branch; - uint32_t me_live_reader; /* have liveness lock in reader table */ - void *me_userctx; /* User-settable context */ - MDBX_hsr_func *me_hsr_callback; /* Callback for kicking laggard readers */ - size_t me_madv_threshold; - - struct { - unsigned dp_reserve_limit; - unsigned rp_augment_limit; - unsigned dp_limit; - unsigned dp_initial; - uint8_t dp_loose_limit; - uint8_t spill_max_denominator; - uint8_t spill_min_denominator; - uint8_t spill_parent4child_denominator; - unsigned merge_threshold_16dot16_percent; -#if !(defined(_WIN32) || defined(_WIN64)) - unsigned writethrough_threshold; -#endif /* Windows */ - bool prefault_write; - union { - unsigned all; - /* tracks options with non-auto values but tuned by user */ - struct { - unsigned dp_limit : 1; - unsigned rp_augment_limit : 1; - unsigned prefault_write : 1; - } non_auto; - } flags; - } me_options; - - /* struct me_dbgeo used for accepting db-geo params from user for the new - * database creation, i.e. when mdbx_env_set_geometry() was called before - * mdbx_env_open(). */ - struct { - size_t lower; /* minimal size of datafile */ - size_t upper; /* maximal size of datafile */ - size_t now; /* current size of datafile */ - size_t grow; /* step to grow datafile */ - size_t shrink; /* threshold to shrink datafile */ - } me_dbgeo; - -#if MDBX_LOCKING == MDBX_LOCKING_SYSV - union { - key_t key; - int semid; - } me_sysv_ipc; -#endif /* MDBX_LOCKING == MDBX_LOCKING_SYSV */ - bool me_incore; - - MDBX_env *me_lcklist_next; - - /* --------------------------------------------------- mostly volatile part */ - - MDBX_txn *me_txn; /* current write transaction */ - osal_fastmutex_t me_dbi_lock; - MDBX_dbi me_numdbs; /* number of DBs opened */ - bool me_prefault_write; - - MDBX_page *me_dp_reserve; /* list of malloc'ed blocks for re-use */ - unsigned me_dp_reserve_len; - /* PNL of pages that became unused in a write txn */ - MDBX_PNL me_retired_pages; - osal_ioring_t me_ioring; +MDBX_MAYBE_UNUSED static inline size_t pnl_size2bytes(size_t size) { + assert(size > 0 && size <= PAGELIST_LIMIT); +#if MDBX_PNL_PREALLOC_FOR_RADIXSORT -#if defined(_WIN32) || defined(_WIN64) - osal_srwlock_t me_remap_guard; - /* Workaround for LockFileEx and WriteFile multithread bug */ - CRITICAL_SECTION me_windowsbug_lock; - char *me_pathname_char; /* cache of multi-byte representation of pathname - to the DB files */ -#else - osal_fastmutex_t me_remap_guard; -#endif - - /* -------------------------------------------------------------- debugging */ - -#if MDBX_DEBUG - MDBX_assert_func *me_assert_func; /* Callback for assertion failures */ -#endif -#ifdef MDBX_USE_VALGRIND - int me_valgrind_handle; -#endif -#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__) - MDBX_atomic_uint32_t me_ignore_EDEADLK; - pgno_t me_poison_edge; -#endif /* MDBX_USE_VALGRIND || __SANITIZE_ADDRESS__ */ - -#ifndef xMDBX_DEBUG_SPILLING -#define xMDBX_DEBUG_SPILLING 0 -#endif -#if xMDBX_DEBUG_SPILLING == 2 - size_t debug_dirtied_est, debug_dirtied_act; -#endif /* xMDBX_DEBUG_SPILLING */ - - /* ------------------------------------------------- stub for lck-less mode */ - MDBX_atomic_uint64_t - x_lckless_stub[(sizeof(MDBX_lockinfo) + MDBX_CACHELINE_SIZE - 1) / - sizeof(MDBX_atomic_uint64_t)]; -}; - -#ifndef __cplusplus -/*----------------------------------------------------------------------------*/ -/* Cache coherence and mmap invalidation */ - -#if MDBX_CPU_WRITEBACK_INCOHERENT -#define osal_flush_incoherent_cpu_writeback() osal_memory_barrier() -#else -#define osal_flush_incoherent_cpu_writeback() osal_compiler_barrier() -#endif /* MDBX_CPU_WRITEBACK_INCOHERENT */ - -MDBX_MAYBE_UNUSED static __inline void -osal_flush_incoherent_mmap(const void *addr, size_t nbytes, - const intptr_t pagesize) { -#if MDBX_MMAP_INCOHERENT_FILE_WRITE - char *const begin = (char *)(-pagesize & (intptr_t)addr); - char *const end = - (char *)(-pagesize & (intptr_t)((char *)addr + nbytes + pagesize - 1)); - int err = msync(begin, end - begin, MS_SYNC | MS_INVALIDATE) ? errno : 0; - eASSERT(nullptr, err == 0); - (void)err; -#else - (void)pagesize; -#endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */ - -#if MDBX_MMAP_INCOHERENT_CPU_CACHE -#ifdef DCACHE - /* MIPS has cache coherency issues. - * Note: for any nbytes >= on-chip cache size, entire is flushed. */ - cacheflush((void *)addr, nbytes, DCACHE); -#else -#error "Oops, cacheflush() not available" -#endif /* DCACHE */ -#endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */ - -#if !MDBX_MMAP_INCOHERENT_FILE_WRITE && !MDBX_MMAP_INCOHERENT_CPU_CACHE - (void)addr; - (void)nbytes; -#endif + size += size; +#endif /* MDBX_PNL_PREALLOC_FOR_RADIXSORT */ + STATIC_ASSERT(MDBX_ASSUME_MALLOC_OVERHEAD + + (PAGELIST_LIMIT * (MDBX_PNL_PREALLOC_FOR_RADIXSORT + 1) + MDBX_PNL_GRANULATE + 3) * sizeof(pgno_t) < + SIZE_MAX / 4 * 3); + size_t bytes = + ceil_powerof2(MDBX_ASSUME_MALLOC_OVERHEAD + sizeof(pgno_t) * (size + 3), MDBX_PNL_GRANULATE * sizeof(pgno_t)) - + MDBX_ASSUME_MALLOC_OVERHEAD; + return bytes; } -/*----------------------------------------------------------------------------*/ -/* Internal prototypes */ - -MDBX_INTERNAL_FUNC int cleanup_dead_readers(MDBX_env *env, int rlocked, - int *dead); -MDBX_INTERNAL_FUNC int rthc_alloc(osal_thread_key_t *key, MDBX_reader *begin, - MDBX_reader *end); -MDBX_INTERNAL_FUNC void rthc_remove(const osal_thread_key_t key); +MDBX_MAYBE_UNUSED static inline pgno_t pnl_bytes2size(const size_t bytes) { + size_t size = bytes / sizeof(pgno_t); + assert(size > 3 && size <= PAGELIST_LIMIT + /* alignment gap */ 65536); + size -= 3; +#if MDBX_PNL_PREALLOC_FOR_RADIXSORT + size >>= 1; +#endif /* MDBX_PNL_PREALLOC_FOR_RADIXSORT */ + return (pgno_t)size; +} -MDBX_INTERNAL_FUNC void global_ctor(void); -MDBX_INTERNAL_FUNC void osal_ctor(void); -MDBX_INTERNAL_FUNC void global_dtor(void); -MDBX_INTERNAL_FUNC void osal_dtor(void); -MDBX_INTERNAL_FUNC void thread_dtor(void *ptr); +MDBX_INTERNAL pnl_t pnl_alloc(size_t size); -#endif /* !__cplusplus */ +MDBX_INTERNAL void pnl_free(pnl_t pnl); -#define MDBX_IS_ERROR(rc) \ - ((rc) != MDBX_RESULT_TRUE && (rc) != MDBX_RESULT_FALSE) +MDBX_INTERNAL int pnl_reserve(pnl_t __restrict *__restrict ppnl, const size_t wanna); -/* Internal error codes, not exposed outside libmdbx */ -#define MDBX_NO_ROOT (MDBX_LAST_ADDED_ERRCODE + 10) +MDBX_MAYBE_UNUSED static inline int __must_check_result pnl_need(pnl_t __restrict *__restrict ppnl, size_t num) { + assert(MDBX_PNL_GETSIZE(*ppnl) <= PAGELIST_LIMIT && MDBX_PNL_ALLOCLEN(*ppnl) >= MDBX_PNL_GETSIZE(*ppnl)); + assert(num <= PAGELIST_LIMIT); + const size_t wanna = MDBX_PNL_GETSIZE(*ppnl) + num; + return likely(MDBX_PNL_ALLOCLEN(*ppnl) >= wanna) ? MDBX_SUCCESS : pnl_reserve(ppnl, wanna); +} -/* Debugging output value of a cursor DBI: Negative in a sub-cursor. */ -#define DDBI(mc) \ - (((mc)->mc_flags & C_SUB) ? -(int)(mc)->mc_dbi : (int)(mc)->mc_dbi) +MDBX_MAYBE_UNUSED static inline void pnl_append_prereserved(__restrict pnl_t pnl, pgno_t pgno) { + assert(MDBX_PNL_GETSIZE(pnl) < MDBX_PNL_ALLOCLEN(pnl)); + if (AUDIT_ENABLED()) { + for (size_t i = MDBX_PNL_GETSIZE(pnl); i > 0; --i) + assert(pgno != pnl[i]); + } + *pnl += 1; + MDBX_PNL_LAST(pnl) = pgno; +} -/* Key size which fits in a DKBUF (debug key buffer). */ -#define DKBUF_MAX 511 -#define DKBUF char _kbuf[DKBUF_MAX * 4 + 2] -#define DKEY(x) mdbx_dump_val(x, _kbuf, DKBUF_MAX * 2 + 1) -#define DVAL(x) mdbx_dump_val(x, _kbuf + DKBUF_MAX * 2 + 1, DKBUF_MAX * 2 + 1) +MDBX_INTERNAL void pnl_shrink(pnl_t __restrict *__restrict ppnl); -#if MDBX_DEBUG -#define DKBUF_DEBUG DKBUF -#define DKEY_DEBUG(x) DKEY(x) -#define DVAL_DEBUG(x) DVAL(x) -#else -#define DKBUF_DEBUG ((void)(0)) -#define DKEY_DEBUG(x) ("-") -#define DVAL_DEBUG(x) ("-") -#endif +MDBX_INTERNAL int __must_check_result spill_append_span(__restrict pnl_t *ppnl, pgno_t pgno, size_t n); -/* An invalid page number. - * Mainly used to denote an empty tree. */ -#define P_INVALID (~(pgno_t)0) +MDBX_INTERNAL int __must_check_result pnl_append_span(__restrict pnl_t *ppnl, pgno_t pgno, size_t n); -/* Test if the flags f are set in a flag word w. */ -#define F_ISSET(w, f) (((w) & (f)) == (f)) +MDBX_INTERNAL int __must_check_result pnl_insert_span(__restrict pnl_t *ppnl, pgno_t pgno, size_t n); -/* Round n up to an even number. */ -#define EVEN(n) (((n) + 1UL) & -2L) /* sign-extending -2 to match n+1U */ - -/* Default size of memory map. - * This is certainly too small for any actual applications. Apps should - * always set the size explicitly using mdbx_env_set_geometry(). */ -#define DEFAULT_MAPSIZE MEGABYTE - -/* Number of slots in the reader table. - * This value was chosen somewhat arbitrarily. The 61 is a prime number, - * and such readers plus a couple mutexes fit into single 4KB page. - * Applications should set the table size using mdbx_env_set_maxreaders(). */ -#define DEFAULT_READERS 61 - -/* Test if a page is a leaf page */ -#define IS_LEAF(p) (((p)->mp_flags & P_LEAF) != 0) -/* Test if a page is a LEAF2 page */ -#define IS_LEAF2(p) unlikely(((p)->mp_flags & P_LEAF2) != 0) -/* Test if a page is a branch page */ -#define IS_BRANCH(p) (((p)->mp_flags & P_BRANCH) != 0) -/* Test if a page is an overflow page */ -#define IS_OVERFLOW(p) unlikely(((p)->mp_flags & P_OVERFLOW) != 0) -/* Test if a page is a sub page */ -#define IS_SUBP(p) (((p)->mp_flags & P_SUBP) != 0) +MDBX_INTERNAL size_t pnl_search_nochk(const pnl_t pnl, pgno_t pgno); -/* Header for a single key/data pair within a page. - * Used in pages of type P_BRANCH and P_LEAF without P_LEAF2. - * We guarantee 2-byte alignment for 'MDBX_node's. - * - * Leaf node flags describe node contents. F_BIGDATA says the node's - * data part is the page number of an overflow page with actual data. - * F_DUPDATA and F_SUBDATA can be combined giving duplicate data in - * a sub-page/sub-database, and named databases (just F_SUBDATA). */ -typedef struct MDBX_node { -#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ - union { - uint32_t mn_dsize; - uint32_t mn_pgno32; - }; - uint8_t mn_flags; /* see mdbx_node flags */ - uint8_t mn_extra; - uint16_t mn_ksize; /* key size */ -#else - uint16_t mn_ksize; /* key size */ - uint8_t mn_extra; - uint8_t mn_flags; /* see mdbx_node flags */ - union { - uint32_t mn_pgno32; - uint32_t mn_dsize; - }; -#endif /* __BYTE_ORDER__ */ +MDBX_INTERNAL void pnl_sort_nochk(pnl_t pnl); - /* mdbx_node Flags */ -#define F_BIGDATA 0x01 /* data put on overflow page */ -#define F_SUBDATA 0x02 /* data is a sub-database */ -#define F_DUPDATA 0x04 /* data has duplicates */ +MDBX_INTERNAL bool pnl_check(const const_pnl_t pnl, const size_t limit); - /* valid flags for mdbx_node_add() */ -#define NODE_ADD_FLAGS (F_DUPDATA | F_SUBDATA | MDBX_RESERVE | MDBX_APPEND) +MDBX_MAYBE_UNUSED static inline bool pnl_check_allocated(const const_pnl_t pnl, const size_t limit) { + return pnl == nullptr || (MDBX_PNL_ALLOCLEN(pnl) >= MDBX_PNL_GETSIZE(pnl) && pnl_check(pnl, limit)); +} -#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ - (!defined(__cplusplus) && defined(_MSC_VER)) - uint8_t mn_data[] /* key and data are appended here */; -#endif /* C99 */ -} MDBX_node; +MDBX_MAYBE_UNUSED static inline void pnl_sort(pnl_t pnl, size_t limit4check) { + pnl_sort_nochk(pnl); + assert(pnl_check(pnl, limit4check)); + (void)limit4check; +} -#define DB_PERSISTENT_FLAGS \ - (MDBX_REVERSEKEY | MDBX_DUPSORT | MDBX_INTEGERKEY | MDBX_DUPFIXED | \ - MDBX_INTEGERDUP | MDBX_REVERSEDUP) +MDBX_MAYBE_UNUSED static inline size_t pnl_search(const pnl_t pnl, pgno_t pgno, size_t limit) { + assert(pnl_check_allocated(pnl, limit)); + if (MDBX_HAVE_CMOV) { + /* cmov-ускоренный бинарный поиск может читать (но не использовать) один + * элемент за концом данных, этот элемент в пределах выделенного участка + * памяти, но не инициализирован. */ + VALGRIND_MAKE_MEM_DEFINED(MDBX_PNL_END(pnl), sizeof(pgno_t)); + } + assert(pgno < limit); + (void)limit; + size_t n = pnl_search_nochk(pnl, pgno); + if (MDBX_HAVE_CMOV) { + VALGRIND_MAKE_MEM_UNDEFINED(MDBX_PNL_END(pnl), sizeof(pgno_t)); + } + return n; +} -/* mdbx_dbi_open() flags */ -#define DB_USABLE_FLAGS (DB_PERSISTENT_FLAGS | MDBX_CREATE | MDBX_DB_ACCEDE) +MDBX_INTERNAL size_t pnl_merge(pnl_t dst, const pnl_t src); -#define DB_VALID 0x8000 /* DB handle is valid, for me_dbflags */ -#define DB_INTERNAL_FLAGS DB_VALID +#ifdef __cplusplus +} +#endif /* __cplusplus */ -#if DB_INTERNAL_FLAGS & DB_USABLE_FLAGS -#error "Oops, some flags overlapped or wrong" -#endif -#if DB_PERSISTENT_FLAGS & ~DB_USABLE_FLAGS -#error "Oops, some flags overlapped or wrong" +#define mdbx_sourcery_anchor XCONCAT(mdbx_sourcery_, MDBX_BUILD_SOURCERY) +#if defined(xMDBX_TOOLS) +extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #endif -/* Max length of iov-vector passed to writev() call, used for auxilary writes */ -#define MDBX_AUXILARY_IOV_MAX 64 -#if defined(IOV_MAX) && IOV_MAX < MDBX_AUXILARY_IOV_MAX -#undef MDBX_AUXILARY_IOV_MAX -#define MDBX_AUXILARY_IOV_MAX IOV_MAX -#endif /* MDBX_AUXILARY_IOV_MAX */ +#define MDBX_IS_ERROR(rc) ((rc) != MDBX_RESULT_TRUE && (rc) != MDBX_RESULT_FALSE) -/* - * / - * | -1, a < b - * CMP2INT(a,b) = < 0, a == b - * | 1, a > b - * \ - */ -#define CMP2INT(a, b) (((a) != (b)) ? (((a) < (b)) ? -1 : 1) : 0) +/*----------------------------------------------------------------------------*/ -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline pgno_t -int64pgno(int64_t i64) { +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline pgno_t int64pgno(int64_t i64) { if (likely(i64 >= (int64_t)MIN_PAGENO && i64 <= (int64_t)MAX_PAGENO + 1)) return (pgno_t)i64; return (i64 < (int64_t)MIN_PAGENO) ? MIN_PAGENO : MAX_PAGENO; } -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline pgno_t -pgno_add(size_t base, size_t augend) { +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline pgno_t pgno_add(size_t base, size_t augend) { assert(base <= MAX_PAGENO + 1 && augend < MAX_PAGENO); return int64pgno((int64_t)base + (int64_t)augend); } -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline pgno_t -pgno_sub(size_t base, size_t subtrahend) { - assert(base >= MIN_PAGENO && base <= MAX_PAGENO + 1 && - subtrahend < MAX_PAGENO); +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline pgno_t pgno_sub(size_t base, size_t subtrahend) { + assert(base >= MIN_PAGENO && base <= MAX_PAGENO + 1 && subtrahend < MAX_PAGENO); return int64pgno((int64_t)base - (int64_t)subtrahend); } +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \author Леонид Юрьев aka Leonid Yuriev \date 2020-2025 +/// +/// \brief Non-inline part of the libmdbx C++ API +/// -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __always_inline bool -is_powerof2(size_t x) { - return (x & (x - 1)) == 0; -} - -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __always_inline size_t -floor_powerof2(size_t value, size_t granularity) { - assert(is_powerof2(granularity)); - return value & ~(granularity - 1); -} - -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __always_inline size_t -ceil_powerof2(size_t value, size_t granularity) { - return floor_powerof2(value + granularity - 1, granularity); -} - -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static unsigned -log2n_powerof2(size_t value_uintptr) { - assert(value_uintptr > 0 && value_uintptr < INT32_MAX && - is_powerof2(value_uintptr)); - assert((value_uintptr & -(intptr_t)value_uintptr) == value_uintptr); - const uint32_t value_uint32 = (uint32_t)value_uintptr; -#if __GNUC_PREREQ(4, 1) || __has_builtin(__builtin_ctz) - STATIC_ASSERT(sizeof(value_uint32) <= sizeof(unsigned)); - return __builtin_ctz(value_uint32); -#elif defined(_MSC_VER) - unsigned long index; - STATIC_ASSERT(sizeof(value_uint32) <= sizeof(long)); - _BitScanForward(&index, value_uint32); - return index; -#else - static const uint8_t debruijn_ctz32[32] = { - 0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8, - 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9}; - return debruijn_ctz32[(uint32_t)(value_uint32 * 0x077CB531ul) >> 27]; -#endif -} - -/* Only a subset of the mdbx_env flags can be changed - * at runtime. Changing other flags requires closing the - * environment and re-opening it with the new flags. */ -#define ENV_CHANGEABLE_FLAGS \ - (MDBX_SAFE_NOSYNC | MDBX_NOMETASYNC | MDBX_DEPRECATED_MAPASYNC | \ - MDBX_NOMEMINIT | MDBX_COALESCE | MDBX_PAGEPERTURB | MDBX_ACCEDE | \ - MDBX_VALIDATION) -#define ENV_CHANGELESS_FLAGS \ - (MDBX_NOSUBDIR | MDBX_RDONLY | MDBX_WRITEMAP | MDBX_NOTLS | MDBX_NORDAHEAD | \ - MDBX_LIFORECLAIM | MDBX_EXCLUSIVE) -#define ENV_USABLE_FLAGS (ENV_CHANGEABLE_FLAGS | ENV_CHANGELESS_FLAGS) - -#if !defined(__cplusplus) || CONSTEXPR_ENUM_FLAGS_OPERATIONS -MDBX_MAYBE_UNUSED static void static_checks(void) { - STATIC_ASSERT_MSG(INT16_MAX - CORE_DBS == MDBX_MAX_DBI, - "Oops, MDBX_MAX_DBI or CORE_DBS?"); - STATIC_ASSERT_MSG((unsigned)(MDBX_DB_ACCEDE | MDBX_CREATE) == - ((DB_USABLE_FLAGS | DB_INTERNAL_FLAGS) & - (ENV_USABLE_FLAGS | ENV_INTERNAL_FLAGS)), - "Oops, some flags overlapped or wrong"); - STATIC_ASSERT_MSG((ENV_INTERNAL_FLAGS & ENV_USABLE_FLAGS) == 0, - "Oops, some flags overlapped or wrong"); -} -#endif /* Disabled for MSVC 19.0 (VisualStudio 2015) */ - -#ifdef __cplusplus -} -#endif - -#define MDBX_ASAN_POISON_MEMORY_REGION(addr, size) \ - do { \ - TRACE("POISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), \ - (size_t)(size), __LINE__); \ - ASAN_POISON_MEMORY_REGION(addr, size); \ - } while (0) - -#define MDBX_ASAN_UNPOISON_MEMORY_REGION(addr, size) \ - do { \ - TRACE("UNPOISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), \ - (size_t)(size), __LINE__); \ - ASAN_UNPOISON_MEMORY_REGION(addr, size); \ - } while (0) -// -// Copyright (c) 2020-2024, Leonid Yuriev . -// SPDX-License-Identifier: Apache-2.0 -// -// Non-inline part of the libmdbx C++ API -// - -#if defined(_MSC_VER) && !defined(_CRT_SECURE_NO_WARNINGS) -#define _CRT_SECURE_NO_WARNINGS -#endif /* _CRT_SECURE_NO_WARNINGS */ - -#if (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) && \ - !defined(__USE_MINGW_ANSI_STDIO) -#define __USE_MINGW_ANSI_STDIO 1 -#endif /* MinGW */ +#if !defined(MDBX_BUILD_CXX) || MDBX_BUILD_CXX != 1 +#error "Build is misconfigured! Expecting MDBX_BUILD_CXX=1 for C++ API." +#endif /* MDBX_BUILD_CXX*/ /* Workaround for MSVC' header `extern "C"` vs `std::` redefinition bug */ #if defined(_MSC_VER) @@ -4074,8 +3231,6 @@ MDBX_MAYBE_UNUSED static void static_checks(void) { #endif /* #define _SILENCE_EXPERIMENTAL_FILESYSTEM_DEPRECATION_WARNING */ #endif /* _MSC_VER */ - - #include #include #include // for isxdigit(), etc @@ -4117,8 +3272,8 @@ class trouble_location { #endif public: - MDBX_CXX11_CONSTEXPR trouble_location(unsigned line, const char *condition, - const char *function, const char *filename) + MDBX_CXX11_CONSTEXPR trouble_location(unsigned line, const char *condition, const char *function, + const char *filename) : #if TROUBLE_PROVIDE_LINENO line_(line) @@ -4187,7 +3342,7 @@ public: //------------------------------------------------------------------------------ -__cold std::string format_va(const char *fmt, va_list ap) { +__cold std::string format_va(const char *fmt, va_list ap) { va_list ones; va_copy(ones, ap); #ifdef _MSC_VER @@ -4200,15 +3355,14 @@ __cold std::string format_va(const char *fmt, va_list ap) { result.reserve(size_t(needed + 1)); result.resize(size_t(needed), '\0'); assert(int(result.capacity()) > needed); - int actual = vsnprintf(const_cast(result.data()), result.capacity(), - fmt, ones); + int actual = vsnprintf(const_cast(result.data()), result.capacity(), fmt, ones); assert(actual == needed); (void)actual; va_end(ones); return result; } -__cold std::string format(const char *fmt, ...) { +__cold std::string format(const char *fmt, ...) { va_list ap; va_start(ap, fmt); std::string result = format_va(fmt, ap); @@ -4229,33 +3383,29 @@ public: virtual ~bug() noexcept; }; -__cold bug::bug(const trouble_location &location) noexcept - : std::runtime_error(format("mdbx.bug: %s.%s at %s:%u", location.function(), - location.condition(), location.filename(), - location.line())), +__cold bug::bug(const trouble_location &location) noexcept + : std::runtime_error(format("mdbx.bug: %s.%s at %s:%u", location.function(), location.condition(), + location.filename(), location.line())), location_(location) {} -__cold bug::~bug() noexcept {} +__cold bug::~bug() noexcept {} -[[noreturn]] __cold void raise_bug(const trouble_location &what_and_where) { - throw bug(what_and_where); -} +[[maybe_unused, noreturn]] __cold void raise_bug(const trouble_location &what_and_where) { throw bug(what_and_where); } -#define RAISE_BUG(line, condition, function, file) \ - do { \ - static MDBX_CXX11_CONSTEXPR_VAR trouble_location bug(line, condition, \ - function, file); \ - raise_bug(bug); \ +#define RAISE_BUG(line, condition, function, file) \ + do { \ + static MDBX_CXX11_CONSTEXPR_VAR trouble_location bug(line, condition, function, file); \ + raise_bug(bug); \ } while (0) -#define ENSURE(condition) \ - do \ - if (MDBX_UNLIKELY(!(condition))) \ - MDBX_CXX20_UNLIKELY RAISE_BUG(__LINE__, #condition, __func__, __FILE__); \ +#undef ENSURE +#define ENSURE(condition) \ + do \ + if (MDBX_UNLIKELY(!(condition))) \ + MDBX_CXX20_UNLIKELY RAISE_BUG(__LINE__, #condition, __func__, __FILE__); \ while (0) -#define NOT_IMPLEMENTED() \ - RAISE_BUG(__LINE__, "not_implemented", __func__, __FILE__); +#define NOT_IMPLEMENTED() RAISE_BUG(__LINE__, "not_implemented", __func__, __FILE__); #endif /* Unused*/ @@ -4280,14 +3430,12 @@ struct line_wrapper { } }; -template -struct temp_buffer { +template struct temp_buffer { TYPE inplace[(INPLACE_BYTES + sizeof(TYPE) - 1) / sizeof(TYPE)]; const size_t size; TYPE *const area; temp_buffer(size_t bytes) - : size((bytes + sizeof(TYPE) - 1) / sizeof(TYPE)), - area((bytes > sizeof(inplace)) ? new TYPE[size] : inplace) { + : size((bytes + sizeof(TYPE) - 1) / sizeof(TYPE)), area((bytes > sizeof(inplace)) ? new TYPE[size] : inplace) { memset(area, 0, sizeof(TYPE) * size); } ~temp_buffer() { @@ -4319,8 +3467,7 @@ struct temp_buffer { namespace mdbx { [[noreturn]] __cold void throw_max_length_exceeded() { - throw std::length_error( - "mdbx:: Exceeded the maximal length of data/slice/buffer."); + throw std::length_error("mdbx:: Exceeded the maximal length of data/slice/buffer."); } [[noreturn]] __cold void throw_too_small_target_buffer() { @@ -4333,33 +3480,31 @@ namespace mdbx { } [[noreturn]] __cold void throw_allocators_mismatch() { - throw std::logic_error( - "mdbx:: An allocators mismatch, so an object could not be transferred " - "into an incompatible memory allocation scheme."); + throw std::logic_error("mdbx:: An allocators mismatch, so an object could not be transferred " + "into an incompatible memory allocation scheme."); } -[[noreturn]] __cold void throw_bad_value_size() { - throw bad_value_size(MDBX_BAD_VALSIZE); +[[noreturn]] __cold void throw_incomparable_cursors() { + throw std::logic_error("mdbx:: incomparable and/or invalid cursors to compare positions."); } -__cold exception::exception(const ::mdbx::error &error) noexcept - : base(error.what()), error_(error) {} +[[noreturn]] __cold void throw_bad_value_size() { throw bad_value_size(MDBX_BAD_VALSIZE); } + +__cold exception::exception(const ::mdbx::error &error) noexcept : base(error.what()), error_(error) {} __cold exception::~exception() noexcept {} static std::atomic_int fatal_countdown; -__cold fatal::fatal(const ::mdbx::error &error) noexcept : base(error) { - ++fatal_countdown; -} +__cold fatal::fatal(const ::mdbx::error &error) noexcept : base(error) { ++fatal_countdown; } __cold fatal::~fatal() noexcept { if (--fatal_countdown == 0) std::terminate(); } -#define DEFINE_EXCEPTION(NAME) \ - __cold NAME::NAME(const ::mdbx::error &rc) : exception(rc) {} \ +#define DEFINE_EXCEPTION(NAME) \ + __cold NAME::NAME(const ::mdbx::error &rc) : exception(rc) {} \ __cold NAME::~NAME() noexcept {} DEFINE_EXCEPTION(bad_map_id) @@ -4391,6 +3536,9 @@ DEFINE_EXCEPTION(thread_mismatch) DEFINE_EXCEPTION(transaction_full) DEFINE_EXCEPTION(transaction_overlapping) DEFINE_EXCEPTION(duplicated_lck_file) +DEFINE_EXCEPTION(dangling_map_id) +DEFINE_EXCEPTION(transaction_ousted) +DEFINE_EXCEPTION(mvcc_retarded) #undef DEFINE_EXCEPTION __cold const char *error::what() const noexcept { @@ -4398,8 +3546,8 @@ __cold const char *error::what() const noexcept { return mdbx_liberr2str(code()); switch (code()) { -#define ERROR_CASE(CODE) \ - case CODE: \ +#define ERROR_CASE(CODE) \ + case CODE: \ return MDBX_STRINGIFY(CODE) ERROR_CASE(MDBX_ENODATA); ERROR_CASE(MDBX_EINVAL); @@ -4412,6 +3560,7 @@ __cold const char *error::what() const noexcept { ERROR_CASE(MDBX_EINTR); ERROR_CASE(MDBX_ENOFILE); ERROR_CASE(MDBX_EREMOTE); + ERROR_CASE(MDBX_EDEADLK); #undef ERROR_CASE default: return "SYSTEM"; @@ -4424,8 +3573,7 @@ __cold std::string error::message() const { return std::string(msg ? msg : "unknown"); } -[[noreturn]] __cold void error::panic(const char *context, - const char *func) const noexcept { +[[noreturn]] __cold void error::panic(const char *context, const char *func) const noexcept { assert(code() != MDBX_SUCCESS); ::mdbx_panic("mdbx::%s.%s(): \"%s\" (%d)", context, func, what(), code()); std::terminate(); @@ -4434,7 +3582,7 @@ __cold std::string error::message() const { __cold void error::throw_exception() const { switch (code()) { case MDBX_EINVAL: - throw std::invalid_argument("mdbx"); + throw std::invalid_argument("MDBX_EINVAL"); case MDBX_ENOMEM: throw std::bad_alloc(); case MDBX_SUCCESS: @@ -4442,8 +3590,8 @@ __cold void error::throw_exception() const { throw std::logic_error("MDBX_SUCCESS (MDBX_RESULT_FALSE)"); case MDBX_RESULT_TRUE: throw std::logic_error("MDBX_RESULT_TRUE"); -#define CASE_EXCEPTION(NAME, CODE) \ - case CODE: \ +#define CASE_EXCEPTION(NAME, CODE) \ + case CODE: \ throw NAME(code()) CASE_EXCEPTION(bad_map_id, MDBX_BAD_DBI); CASE_EXCEPTION(bad_transaction, MDBX_BAD_TXN); @@ -4478,6 +3626,9 @@ __cold void error::throw_exception() const { CASE_EXCEPTION(transaction_full, MDBX_TXN_FULL); CASE_EXCEPTION(transaction_overlapping, MDBX_TXN_OVERLAPPING); CASE_EXCEPTION(duplicated_lck_file, MDBX_DUPLICATED_CLK); + CASE_EXCEPTION(dangling_map_id, MDBX_DANGLING_DBI); + CASE_EXCEPTION(transaction_ousted, MDBX_OUSTED); + CASE_EXCEPTION(mvcc_retarded, MDBX_MVCC_RETARDED); #undef CASE_EXCEPTION default: if (is_mdbx_error()) @@ -4595,48 +3746,48 @@ bool slice::is_printable(bool disable_utf8) const noexcept { } #ifdef MDBX_U128_TYPE -MDBX_U128_TYPE slice::as_uint128() const { +MDBX_U128_TYPE slice::as_uint128_adapt() const { static_assert(sizeof(MDBX_U128_TYPE) == 16, "WTF?"); if (size() == 16) { MDBX_U128_TYPE r; memcpy(&r, data(), sizeof(r)); return r; } else - return as_uint64(); + return as_uint64_adapt(); } #endif /* MDBX_U128_TYPE */ -uint64_t slice::as_uint64() const { +uint64_t slice::as_uint64_adapt() const { static_assert(sizeof(uint64_t) == 8, "WTF?"); if (size() == 8) { uint64_t r; memcpy(&r, data(), sizeof(r)); return r; } else - return as_uint32(); + return as_uint32_adapt(); } -uint32_t slice::as_uint32() const { +uint32_t slice::as_uint32_adapt() const { static_assert(sizeof(uint32_t) == 4, "WTF?"); if (size() == 4) { uint32_t r; memcpy(&r, data(), sizeof(r)); return r; } else - return as_uint16(); + return as_uint16_adapt(); } -uint16_t slice::as_uint16() const { +uint16_t slice::as_uint16_adapt() const { static_assert(sizeof(uint16_t) == 2, "WTF?"); if (size() == 2) { uint16_t r; memcpy(&r, data(), sizeof(r)); return r; } else - return as_uint8(); + return as_uint8_adapt(); } -uint8_t slice::as_uint8() const { +uint8_t slice::as_uint8_adapt() const { static_assert(sizeof(uint8_t) == 1, "WTF?"); if (size() == 1) return *static_cast(data()); @@ -4647,48 +3798,48 @@ uint8_t slice::as_uint8() const { } #ifdef MDBX_I128_TYPE -MDBX_I128_TYPE slice::as_int128() const { +MDBX_I128_TYPE slice::as_int128_adapt() const { static_assert(sizeof(MDBX_I128_TYPE) == 16, "WTF?"); if (size() == 16) { MDBX_I128_TYPE r; memcpy(&r, data(), sizeof(r)); return r; } else - return as_int64(); + return as_int64_adapt(); } #endif /* MDBX_I128_TYPE */ -int64_t slice::as_int64() const { +int64_t slice::as_int64_adapt() const { static_assert(sizeof(int64_t) == 8, "WTF?"); if (size() == 8) { uint64_t r; memcpy(&r, data(), sizeof(r)); return r; } else - return as_int32(); + return as_int32_adapt(); } -int32_t slice::as_int32() const { +int32_t slice::as_int32_adapt() const { static_assert(sizeof(int32_t) == 4, "WTF?"); if (size() == 4) { int32_t r; memcpy(&r, data(), sizeof(r)); return r; } else - return as_int16(); + return as_int16_adapt(); } -int16_t slice::as_int16() const { +int16_t slice::as_int16_adapt() const { static_assert(sizeof(int16_t) == 2, "WTF?"); if (size() == 2) { int16_t r; memcpy(&r, data(), sizeof(r)); return r; } else - return as_int8(); + return as_int8_adapt(); } -int8_t slice::as_int8() const { +int8_t slice::as_int8_adapt() const { if (size() == 1) return *static_cast(data()); else if (size() == 0) @@ -4744,27 +3895,23 @@ char *to_hex::write_bytes(char *__restrict const dest, size_t dest_size) const { return out; } -char *from_hex::write_bytes(char *__restrict const dest, - size_t dest_size) const { +char *from_hex::write_bytes(char *__restrict const dest, size_t dest_size) const { if (MDBX_UNLIKELY(source.length() % 2 && !ignore_spaces)) - MDBX_CXX20_UNLIKELY throw std::domain_error( - "mdbx::from_hex:: odd length of hexadecimal string"); + MDBX_CXX20_UNLIKELY throw std::domain_error("mdbx::from_hex:: odd length of hexadecimal string"); if (MDBX_UNLIKELY(envisage_result_length() > dest_size)) MDBX_CXX20_UNLIKELY throw_too_small_target_buffer(); auto ptr = dest; auto src = source.byte_ptr(); for (auto left = source.length(); left > 0;) { - if (MDBX_UNLIKELY(*src <= ' ') && - MDBX_LIKELY(ignore_spaces && isspace(*src))) { + if (MDBX_UNLIKELY(*src <= ' ') && MDBX_LIKELY(ignore_spaces && isspace(*src))) { ++src; --left; continue; } if (MDBX_UNLIKELY(left < 1 || !isxdigit(src[0]) || !isxdigit(src[1]))) - MDBX_CXX20_UNLIKELY throw std::domain_error( - "mdbx::from_hex:: invalid hexadecimal string"); + MDBX_CXX20_UNLIKELY throw std::domain_error("mdbx::from_hex:: invalid hexadecimal string"); int8_t hi = src[0]; hi = (hi | 0x20) - 'a'; @@ -4789,8 +3936,7 @@ bool from_hex::is_erroneous() const noexcept { bool got = false; auto src = source.byte_ptr(); for (auto left = source.length(); left > 0;) { - if (MDBX_UNLIKELY(*src <= ' ') && - MDBX_LIKELY(ignore_spaces && isspace(*src))) { + if (MDBX_UNLIKELY(*src <= ' ') && MDBX_LIKELY(ignore_spaces && isspace(*src))) { ++src; --left; continue; @@ -4822,25 +3968,21 @@ using b58_uint = uint_fast32_t; #endif struct b58_buffer : public temp_buffer { - b58_buffer(size_t bytes, size_t estimation_ratio_numerator, - size_t estimation_ratio_denominator, size_t extra = 0) - : temp_buffer((/* пересчитываем по указанной пропорции */ - bytes = (bytes * estimation_ratio_numerator + - estimation_ratio_denominator - 1) / - estimation_ratio_denominator, - /* учитываем резервный старший байт в каждом слове */ - ((bytes + sizeof(b58_uint) - 2) / (sizeof(b58_uint) - 1) * - sizeof(b58_uint) + - extra) * - sizeof(b58_uint))) {} + b58_buffer(size_t bytes, size_t estimation_ratio_numerator, size_t estimation_ratio_denominator, size_t extra = 0) + : temp_buffer( + (/* пересчитываем по указанной пропорции */ + bytes = + (bytes * estimation_ratio_numerator + estimation_ratio_denominator - 1) / estimation_ratio_denominator, + /* учитываем резервный старший байт в каждом слове */ + ((bytes + sizeof(b58_uint) - 2) / (sizeof(b58_uint) - 1) * sizeof(b58_uint) + extra) * sizeof(b58_uint))) { + } }; static byte b58_8to11(b58_uint &v) noexcept { - static const char b58_alphabet[58] = { - '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F', - 'G', 'H', 'J', 'K', 'L', 'M', 'N', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', - 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'm', - 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z'}; + static const char b58_alphabet[58] = {'1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F', + 'G', 'H', 'J', 'K', 'L', 'M', 'N', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', + 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'm', + 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z'}; const auto i = size_t(v % 58); v /= 58; @@ -4849,9 +3991,8 @@ static byte b58_8to11(b58_uint &v) noexcept { static slice b58_encode(b58_buffer &buf, const byte *begin, const byte *end) { auto high = buf.end(); - const auto modulo = - b58_uint((sizeof(b58_uint) > 4) ? UINT64_C(0x1A636A90B07A00) /* 58^9 */ - : UINT32_C(0xACAD10) /* 58^4 */); + const auto modulo = b58_uint((sizeof(b58_uint) > 4) ? UINT64_C(0x1A636A90B07A00) /* 58^9 */ + : UINT32_C(0xACAD10) /* 58^4 */); static_assert(sizeof(modulo) == 4 || sizeof(modulo) == 8, "WTF?"); while (begin < end) { b58_uint carry = *begin++; @@ -4897,8 +4038,7 @@ static slice b58_encode(b58_buffer &buf, const byte *begin, const byte *end) { return slice(output, ptr); } -char *to_base58::write_bytes(char *__restrict const dest, - size_t dest_size) const { +char *to_base58::write_bytes(char *__restrict const dest, size_t dest_size) const { if (MDBX_UNLIKELY(envisage_result_length() > dest_size)) MDBX_CXX20_UNLIKELY throw_too_small_target_buffer(); @@ -4969,8 +4109,7 @@ const signed char b58_map[256] = { IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL // f0 }; -static slice b58_decode(b58_buffer &buf, const byte *begin, const byte *end, - bool ignore_spaces) { +static slice b58_decode(b58_buffer &buf, const byte *begin, const byte *end, bool ignore_spaces) { auto high = buf.end(); while (begin < end) { const auto c = b58_map[*begin++]; @@ -5011,8 +4150,7 @@ static slice b58_decode(b58_buffer &buf, const byte *begin, const byte *end, return slice(output, ptr); } -char *from_base58::write_bytes(char *__restrict const dest, - size_t dest_size) const { +char *from_base58::write_bytes(char *__restrict const dest, size_t dest_size) const { if (MDBX_UNLIKELY(envisage_result_length() > dest_size)) MDBX_CXX20_UNLIKELY throw_too_small_target_buffer(); @@ -5038,8 +4176,7 @@ bool from_base58::is_erroneous() const noexcept { auto begin = source.byte_ptr(); auto const end = source.end_byte_ptr(); while (begin < end) { - if (MDBX_UNLIKELY(b58_map[*begin] < 0 && - !(ignore_spaces && isspace(*begin)))) + if (MDBX_UNLIKELY(b58_map[*begin] < 0 && !(ignore_spaces && isspace(*begin)))) return true; ++begin; } @@ -5048,22 +4185,18 @@ bool from_base58::is_erroneous() const noexcept { //------------------------------------------------------------------------------ -static inline void b64_3to4(const byte x, const byte y, const byte z, - char *__restrict dest) noexcept { - static const byte alphabet[64] = { - 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', - 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', - 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', - 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', - '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/'}; +static inline void b64_3to4(const byte x, const byte y, const byte z, char *__restrict dest) noexcept { + static const byte alphabet[64] = {'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', + 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', + 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', + 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/'}; dest[0] = alphabet[(x & 0xfc) >> 2]; dest[1] = alphabet[((x & 0x03) << 4) + ((y & 0xf0) >> 4)]; dest[2] = alphabet[((y & 0x0f) << 2) + ((z & 0xc0) >> 6)]; dest[3] = alphabet[z & 0x3f]; } -char *to_base64::write_bytes(char *__restrict const dest, - size_t dest_size) const { +char *to_base64::write_bytes(char *__restrict const dest, size_t dest_size) const { if (MDBX_UNLIKELY(envisage_result_length() > dest_size)) MDBX_CXX20_UNLIKELY throw_too_small_target_buffer(); @@ -5157,8 +4290,7 @@ static const signed char b64_map[256] = { IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL, IL // f0 }; -static inline signed char b64_4to3(signed char a, signed char b, signed char c, - signed char d, +static inline signed char b64_4to3(signed char a, signed char b, signed char c, signed char d, char *__restrict dest) noexcept { dest[0] = byte((a << 2) + ((b & 0x30) >> 4)); dest[1] = byte(((b & 0xf) << 4) + ((c & 0x3c) >> 2)); @@ -5166,19 +4298,16 @@ static inline signed char b64_4to3(signed char a, signed char b, signed char c, return a | b | c | d; } -char *from_base64::write_bytes(char *__restrict const dest, - size_t dest_size) const { +char *from_base64::write_bytes(char *__restrict const dest, size_t dest_size) const { if (MDBX_UNLIKELY(source.length() % 4 && !ignore_spaces)) - MDBX_CXX20_UNLIKELY throw std::domain_error( - "mdbx::from_base64:: odd length of base64 string"); + MDBX_CXX20_UNLIKELY throw std::domain_error("mdbx::from_base64:: odd length of base64 string"); if (MDBX_UNLIKELY(envisage_result_length() > dest_size)) MDBX_CXX20_UNLIKELY throw_too_small_target_buffer(); auto ptr = dest; auto src = source.byte_ptr(); for (auto left = source.length(); left > 0;) { - if (MDBX_UNLIKELY(*src <= ' ') && - MDBX_LIKELY(ignore_spaces && isspace(*src))) { + if (MDBX_UNLIKELY(*src <= ' ') && MDBX_LIKELY(ignore_spaces && isspace(*src))) { ++src; --left; continue; @@ -5189,8 +4318,7 @@ char *from_base64::write_bytes(char *__restrict const dest, bailout: throw std::domain_error("mdbx::from_base64:: invalid base64 string"); } - const signed char a = b64_map[src[0]], b = b64_map[src[1]], - c = b64_map[src[2]], d = b64_map[src[3]]; + const signed char a = b64_map[src[0]], b = b64_map[src[1]], c = b64_map[src[2]], d = b64_map[src[3]]; if (MDBX_UNLIKELY(b64_4to3(a, b, c, d, ptr) < 0)) { if (left == 4 && (a | b) >= 0 && d == EQ) { if (c >= 0) { @@ -5219,8 +4347,7 @@ bool from_base64::is_erroneous() const noexcept { bool got = false; auto src = source.byte_ptr(); for (auto left = source.length(); left > 0;) { - if (MDBX_UNLIKELY(*src <= ' ') && - MDBX_LIKELY(ignore_spaces && isspace(*src))) { + if (MDBX_UNLIKELY(*src <= ' ') && MDBX_LIKELY(ignore_spaces && isspace(*src))) { ++src; --left; continue; @@ -5228,8 +4355,7 @@ bool from_base64::is_erroneous() const noexcept { if (MDBX_UNLIKELY(left < 3)) MDBX_CXX20_UNLIKELY return false; - const signed char a = b64_map[src[0]], b = b64_map[src[1]], - c = b64_map[src[2]], d = b64_map[src[3]]; + const signed char a = b64_map[src[0]], b = b64_map[src[1]], c = b64_map[src[2]], d = b64_map[src[3]]; if (MDBX_UNLIKELY((a | b | c | d) < 0)) MDBX_CXX20_UNLIKELY { if (left == 4 && (a | b) >= 0 && d == EQ && (c >= 0 || c == d)) @@ -5245,13 +4371,32 @@ bool from_base64::is_erroneous() const noexcept { //------------------------------------------------------------------------------ -template class LIBMDBX_API_TYPE buffer; +#if defined(_MSC_VER) +#pragma warning(push) +/* warning C4251: 'mdbx::buffer<...>::silo_': + * struct 'mdbx::buffer<..>::silo' needs to have dll-interface to be used by clients of class 'mdbx::buffer<...>' + * + * Microsoft не хочет признавать ошибки и пересматривать приятные решения, поэтому MSVC продолжает кошмарить + * и стращать разработчиков предупреждениями, тем самым перекладывая ответственность на их плечи. + * + * В данном случае предупреждение выдаётся из-за инстанцирования std::string::allocator_type::pointer и + * std::pmr::string::allocator_type::pointer внутри mdbx::buffer<..>::silo. А так как эти типы являются частью + * стандартной библиотеки C++ они всегда будут доступны и без необходимости их инстанцирования и экспорта из libmdbx. + * + * Поэтому нет других вариантов как заглушить это предупреждение и еще раз плюнуть в сторону microsoft. */ +#pragma warning(disable : 4251) +#endif /* MSVC */ + +MDBX_INSTALL_API_TEMPLATE(LIBMDBX_API_TYPE, buffer); -#if defined(__cpp_lib_memory_resource) && \ - __cpp_lib_memory_resource >= 201603L && _GLIBCXX_USE_CXX11_ABI -template class LIBMDBX_API_TYPE buffer; +#if defined(__cpp_lib_memory_resource) && __cpp_lib_memory_resource >= 201603L && _GLIBCXX_USE_CXX11_ABI +MDBX_INSTALL_API_TEMPLATE(LIBMDBX_API_TYPE, buffer); #endif /* __cpp_lib_memory_resource >= 201603L */ +#if defined(_MSC_VER) +#pragma warning(pop) +#endif /* MSVC */ + //------------------------------------------------------------------------------ static inline MDBX_env_flags_t mode2flags(env::mode mode) { @@ -5267,8 +4412,7 @@ static inline MDBX_env_flags_t mode2flags(env::mode mode) { } } -__cold MDBX_env_flags_t -env::operate_parameters::make_flags(bool accede, bool use_subdirectory) const { +__cold MDBX_env_flags_t env::operate_parameters::make_flags(bool accede, bool use_subdirectory) const { MDBX_env_flags_t flags = mode2flags(mode); if (accede) flags |= MDBX_ACCEDE; @@ -5276,12 +4420,14 @@ env::operate_parameters::make_flags(bool accede, bool use_subdirectory) const { flags |= MDBX_NOSUBDIR; if (options.exclusive) flags |= MDBX_EXCLUSIVE; - if (options.orphan_read_transactions) - flags |= MDBX_NOTLS; + if (options.no_sticky_threads) + flags |= MDBX_NOSTICKYTHREADS; if (options.disable_readahead) flags |= MDBX_NORDAHEAD; if (options.disable_clear_memory) flags |= MDBX_NOMEMINIT; + if (options.enable_validation) + flags |= MDBX_VALIDATION; if (mode != readonly) { if (options.nested_write_transactions) @@ -5292,8 +4438,7 @@ env::operate_parameters::make_flags(bool accede, bool use_subdirectory) const { flags |= MDBX_LIFORECLAIM; switch (durability) { default: - MDBX_CXX20_UNLIKELY throw std::invalid_argument( - "db::durability is invalid"); + MDBX_CXX20_UNLIKELY throw std::invalid_argument("db::durability is invalid"); case env::durability::robust_synchronous: break; case env::durability::half_synchronous_weak_last: @@ -5311,16 +4456,13 @@ env::operate_parameters::make_flags(bool accede, bool use_subdirectory) const { return flags; } -env::mode -env::operate_parameters::mode_from_flags(MDBX_env_flags_t flags) noexcept { +env::mode env::operate_parameters::mode_from_flags(MDBX_env_flags_t flags) noexcept { if (flags & MDBX_RDONLY) return env::mode::readonly; - return (flags & MDBX_WRITEMAP) ? env::mode::write_mapped_io - : env::mode::write_file_io; + return (flags & MDBX_WRITEMAP) ? env::mode::write_mapped_io : env::mode::write_file_io; } -env::durability env::operate_parameters::durability_from_flags( - MDBX_env_flags_t flags) noexcept { +env::durability env::operate_parameters::durability_from_flags(MDBX_env_flags_t flags) noexcept { if ((flags & MDBX_UTTERLY_NOSYNC) == MDBX_UTTERLY_NOSYNC) return env::durability::whole_fragile; if (flags & MDBX_SAFE_NOSYNC) @@ -5331,70 +4473,51 @@ env::durability env::operate_parameters::durability_from_flags( } env::reclaiming_options::reclaiming_options(MDBX_env_flags_t flags) noexcept - : lifo((flags & MDBX_LIFORECLAIM) ? true : false), - coalesce((flags & MDBX_COALESCE) ? true : false) {} + : lifo((flags & MDBX_LIFORECLAIM) ? true : false), coalesce((flags & MDBX_COALESCE) ? true : false) {} env::operate_options::operate_options(MDBX_env_flags_t flags) noexcept - : orphan_read_transactions( - ((flags & (MDBX_NOTLS | MDBX_EXCLUSIVE)) == MDBX_NOTLS) ? true - : false), - nested_write_transactions((flags & (MDBX_WRITEMAP | MDBX_RDONLY)) ? false - : true), - exclusive((flags & MDBX_EXCLUSIVE) ? true : false), - disable_readahead((flags & MDBX_NORDAHEAD) ? true : false), + : no_sticky_threads(((flags & (MDBX_NOSTICKYTHREADS | MDBX_EXCLUSIVE)) == MDBX_NOSTICKYTHREADS) ? true : false), + nested_write_transactions((flags & (MDBX_WRITEMAP | MDBX_RDONLY)) ? false : true), + exclusive((flags & MDBX_EXCLUSIVE) ? true : false), disable_readahead((flags & MDBX_NORDAHEAD) ? true : false), disable_clear_memory((flags & MDBX_NOMEMINIT) ? true : false) {} -bool env::is_pristine() const { - return get_stat().ms_mod_txnid == 0 && - get_info().mi_recent_txnid == INITIAL_TXNID; -} +bool env::is_pristine() const { return get_stat().ms_mod_txnid == 0 && get_info().mi_recent_txnid == INITIAL_TXNID; } bool env::is_empty() const { return get_stat().ms_leaf_pages == 0; } __cold env &env::copy(filehandle fd, bool compactify, bool force_dynamic_size) { - error::success_or_throw( - ::mdbx_env_copy2fd(handle_, fd, - (compactify ? MDBX_CP_COMPACT : MDBX_CP_DEFAULTS) | - (force_dynamic_size ? MDBX_CP_FORCE_DYNAMIC_SIZE - : MDBX_CP_DEFAULTS))); + error::success_or_throw(::mdbx_env_copy2fd(handle_, fd, + (compactify ? MDBX_CP_COMPACT : MDBX_CP_DEFAULTS) | + (force_dynamic_size ? MDBX_CP_FORCE_DYNAMIC_SIZE : MDBX_CP_DEFAULTS))); return *this; } -__cold env &env::copy(const char *destination, bool compactify, - bool force_dynamic_size) { - error::success_or_throw( - ::mdbx_env_copy(handle_, destination, - (compactify ? MDBX_CP_COMPACT : MDBX_CP_DEFAULTS) | - (force_dynamic_size ? MDBX_CP_FORCE_DYNAMIC_SIZE - : MDBX_CP_DEFAULTS))); +__cold env &env::copy(const char *destination, bool compactify, bool force_dynamic_size) { + error::success_or_throw(::mdbx_env_copy(handle_, destination, + (compactify ? MDBX_CP_COMPACT : MDBX_CP_DEFAULTS) | + (force_dynamic_size ? MDBX_CP_FORCE_DYNAMIC_SIZE : MDBX_CP_DEFAULTS))); return *this; } -__cold env &env::copy(const ::std::string &destination, bool compactify, - bool force_dynamic_size) { +__cold env &env::copy(const ::std::string &destination, bool compactify, bool force_dynamic_size) { return copy(destination.c_str(), compactify, force_dynamic_size); } #if defined(_WIN32) || defined(_WIN64) -__cold env &env::copy(const wchar_t *destination, bool compactify, - bool force_dynamic_size) { - error::success_or_throw( - ::mdbx_env_copyW(handle_, destination, - (compactify ? MDBX_CP_COMPACT : MDBX_CP_DEFAULTS) | - (force_dynamic_size ? MDBX_CP_FORCE_DYNAMIC_SIZE - : MDBX_CP_DEFAULTS))); +__cold env &env::copy(const wchar_t *destination, bool compactify, bool force_dynamic_size) { + error::success_or_throw(::mdbx_env_copyW(handle_, destination, + (compactify ? MDBX_CP_COMPACT : MDBX_CP_DEFAULTS) | + (force_dynamic_size ? MDBX_CP_FORCE_DYNAMIC_SIZE : MDBX_CP_DEFAULTS))); return *this; } -env &env::copy(const ::std::wstring &destination, bool compactify, - bool force_dynamic_size) { +env &env::copy(const ::std::wstring &destination, bool compactify, bool force_dynamic_size) { return copy(destination.c_str(), compactify, force_dynamic_size); } #endif /* Windows */ #ifdef MDBX_STD_FILESYSTEM_PATH -__cold env &env::copy(const MDBX_STD_FILESYSTEM_PATH &destination, - bool compactify, bool force_dynamic_size) { +__cold env &env::copy(const MDBX_STD_FILESYSTEM_PATH &destination, bool compactify, bool force_dynamic_size) { return copy(destination.native(), compactify, force_dynamic_size); } #endif /* MDBX_STD_FILESYSTEM_PATH */ @@ -5414,8 +4537,7 @@ __cold path env::get_path() const { } __cold bool env::remove(const char *pathname, const remove_mode mode) { - return !error::boolean_or_throw( - ::mdbx_env_delete(pathname, MDBX_env_delete_mode_t(mode))); + return !error::boolean_or_throw(::mdbx_env_delete(pathname, MDBX_env_delete_mode_t(mode))); } __cold bool env::remove(const ::std::string &pathname, const remove_mode mode) { @@ -5424,19 +4546,16 @@ __cold bool env::remove(const ::std::string &pathname, const remove_mode mode) { #if defined(_WIN32) || defined(_WIN64) __cold bool env::remove(const wchar_t *pathname, const remove_mode mode) { - return !error::boolean_or_throw( - ::mdbx_env_deleteW(pathname, MDBX_env_delete_mode_t(mode))); + return !error::boolean_or_throw(::mdbx_env_deleteW(pathname, MDBX_env_delete_mode_t(mode))); } -__cold bool env::remove(const ::std::wstring &pathname, - const remove_mode mode) { +__cold bool env::remove(const ::std::wstring &pathname, const remove_mode mode) { return remove(pathname.c_str(), mode); } #endif /* Windows */ #ifdef MDBX_STD_FILESYSTEM_PATH -__cold bool env::remove(const MDBX_STD_FILESYSTEM_PATH &pathname, - const remove_mode mode) { +__cold bool env::remove(const MDBX_STD_FILESYSTEM_PATH &pathname, const remove_mode mode) { return remove(pathname.native(), mode); } #endif /* MDBX_STD_FILESYSTEM_PATH */ @@ -5452,13 +4571,11 @@ static inline MDBX_env *create_env() { __cold env_managed::~env_managed() noexcept { if (MDBX_UNLIKELY(handle_)) - MDBX_CXX20_UNLIKELY error::success_or_panic( - ::mdbx_env_close(handle_), "mdbx::~env()", "mdbx_env_close"); + MDBX_CXX20_UNLIKELY error::success_or_panic(::mdbx_env_close(handle_), "mdbx::~env()", "mdbx_env_close"); } __cold void env_managed::close(bool dont_sync) { - const error rc = - static_cast(::mdbx_env_close_ex(handle_, dont_sync)); + const error rc = static_cast(::mdbx_env_close_ex(handle_, dont_sync)); switch (rc.code()) { case MDBX_EBADSIGN: MDBX_CXX20_UNLIKELY handle_ = nullptr; @@ -5477,87 +4594,69 @@ __cold void env_managed::setup(unsigned max_maps, unsigned max_readers) { error::success_or_throw(::mdbx_env_set_maxdbs(handle_, max_maps)); } -__cold env_managed::env_managed(const char *pathname, - const operate_parameters &op, bool accede) +__cold env_managed::env_managed(const char *pathname, const operate_parameters &op, bool accede) : env_managed(create_env()) { setup(op.max_maps, op.max_readers); - error::success_or_throw( - ::mdbx_env_open(handle_, pathname, op.make_flags(accede), 0)); + error::success_or_throw(::mdbx_env_open(handle_, pathname, op.make_flags(accede), 0)); - if (op.options.nested_write_transactions && - !get_options().nested_write_transactions) + if (op.options.nested_write_transactions && !get_options().nested_write_transactions) MDBX_CXX20_UNLIKELY error::throw_exception(MDBX_INCOMPATIBLE); } -__cold env_managed::env_managed(const char *pathname, - const env_managed::create_parameters &cp, +__cold env_managed::env_managed(const char *pathname, const env_managed::create_parameters &cp, const env::operate_parameters &op, bool accede) : env_managed(create_env()) { setup(op.max_maps, op.max_readers); set_geometry(cp.geometry); - error::success_or_throw(::mdbx_env_open( - handle_, pathname, op.make_flags(accede, cp.use_subdirectory), - cp.file_mode_bits)); + error::success_or_throw( + ::mdbx_env_open(handle_, pathname, op.make_flags(accede, cp.use_subdirectory), cp.file_mode_bits)); - if (op.options.nested_write_transactions && - !get_options().nested_write_transactions) + if (op.options.nested_write_transactions && !get_options().nested_write_transactions) MDBX_CXX20_UNLIKELY error::throw_exception(MDBX_INCOMPATIBLE); } -__cold env_managed::env_managed(const ::std::string &pathname, - const operate_parameters &op, bool accede) +__cold env_managed::env_managed(const ::std::string &pathname, const operate_parameters &op, bool accede) : env_managed(pathname.c_str(), op, accede) {} -__cold env_managed::env_managed(const ::std::string &pathname, - const env_managed::create_parameters &cp, +__cold env_managed::env_managed(const ::std::string &pathname, const env_managed::create_parameters &cp, const env::operate_parameters &op, bool accede) : env_managed(pathname.c_str(), cp, op, accede) {} #if defined(_WIN32) || defined(_WIN64) -__cold env_managed::env_managed(const wchar_t *pathname, - const operate_parameters &op, bool accede) +__cold env_managed::env_managed(const wchar_t *pathname, const operate_parameters &op, bool accede) : env_managed(create_env()) { setup(op.max_maps, op.max_readers); - error::success_or_throw( - ::mdbx_env_openW(handle_, pathname, op.make_flags(accede), 0)); + error::success_or_throw(::mdbx_env_openW(handle_, pathname, op.make_flags(accede), 0)); - if (op.options.nested_write_transactions && - !get_options().nested_write_transactions) + if (op.options.nested_write_transactions && !get_options().nested_write_transactions) MDBX_CXX20_UNLIKELY error::throw_exception(MDBX_INCOMPATIBLE); } -__cold env_managed::env_managed(const wchar_t *pathname, - const env_managed::create_parameters &cp, +__cold env_managed::env_managed(const wchar_t *pathname, const env_managed::create_parameters &cp, const env::operate_parameters &op, bool accede) : env_managed(create_env()) { setup(op.max_maps, op.max_readers); set_geometry(cp.geometry); - error::success_or_throw(::mdbx_env_openW( - handle_, pathname, op.make_flags(accede, cp.use_subdirectory), - cp.file_mode_bits)); + error::success_or_throw( + ::mdbx_env_openW(handle_, pathname, op.make_flags(accede, cp.use_subdirectory), cp.file_mode_bits)); - if (op.options.nested_write_transactions && - !get_options().nested_write_transactions) + if (op.options.nested_write_transactions && !get_options().nested_write_transactions) MDBX_CXX20_UNLIKELY error::throw_exception(MDBX_INCOMPATIBLE); } -__cold env_managed::env_managed(const ::std::wstring &pathname, - const operate_parameters &op, bool accede) +__cold env_managed::env_managed(const ::std::wstring &pathname, const operate_parameters &op, bool accede) : env_managed(pathname.c_str(), op, accede) {} -__cold env_managed::env_managed(const ::std::wstring &pathname, - const env_managed::create_parameters &cp, +__cold env_managed::env_managed(const ::std::wstring &pathname, const env_managed::create_parameters &cp, const env::operate_parameters &op, bool accede) : env_managed(pathname.c_str(), cp, op, accede) {} #endif /* Windows */ #ifdef MDBX_STD_FILESYSTEM_PATH -__cold env_managed::env_managed(const MDBX_STD_FILESYSTEM_PATH &pathname, - const operate_parameters &op, bool accede) +__cold env_managed::env_managed(const MDBX_STD_FILESYSTEM_PATH &pathname, const operate_parameters &op, bool accede) : env_managed(pathname.native(), op, accede) {} -__cold env_managed::env_managed(const MDBX_STD_FILESYSTEM_PATH &pathname, - const env_managed::create_parameters &cp, +__cold env_managed::env_managed(const MDBX_STD_FILESYSTEM_PATH &pathname, const env_managed::create_parameters &cp, const env::operate_parameters &op, bool accede) : env_managed(pathname.native(), cp, op, accede) {} #endif /* MDBX_STD_FILESYSTEM_PATH */ @@ -5567,16 +4666,14 @@ __cold env_managed::env_managed(const MDBX_STD_FILESYSTEM_PATH &pathname, txn_managed txn::start_nested() { MDBX_txn *nested; error::throw_on_nullptr(handle_, MDBX_BAD_TXN); - error::success_or_throw(::mdbx_txn_begin(mdbx_txn_env(handle_), handle_, - MDBX_TXN_READWRITE, &nested)); + error::success_or_throw(::mdbx_txn_begin(mdbx_txn_env(handle_), handle_, MDBX_TXN_READWRITE, &nested)); assert(nested != nullptr); return txn_managed(nested); } txn_managed::~txn_managed() noexcept { if (MDBX_UNLIKELY(handle_)) - MDBX_CXX20_UNLIKELY error::success_or_panic(::mdbx_txn_abort(handle_), - "mdbx::~txn", "mdbx_txn_abort"); + MDBX_CXX20_UNLIKELY error::success_or_panic(::mdbx_txn_abort(handle_), "mdbx::~txn", "mdbx_txn_abort"); } void txn_managed::abort() { @@ -5596,14 +4693,19 @@ void txn_managed::commit() { } void txn_managed::commit(commit_latency *latency) { - const error err = - static_cast(::mdbx_txn_commit_ex(handle_, latency)); + const error err = static_cast(::mdbx_txn_commit_ex(handle_, latency)); if (MDBX_LIKELY(err.code() != MDBX_THREAD_MISMATCH)) MDBX_CXX20_LIKELY handle_ = nullptr; if (MDBX_UNLIKELY(err.code() != MDBX_SUCCESS)) MDBX_CXX20_UNLIKELY err.throw_exception(); } +void txn_managed::commit_embark_read() { + auto env = this->env(); + commit(); + error::success_or_throw(::mdbx_txn_begin(env, nullptr, MDBX_TXN_RDONLY, &handle_)); +} + //------------------------------------------------------------------------------ __cold bool txn::drop_map(const char *name, bool throw_if_absent) { @@ -5640,13 +4742,76 @@ __cold bool txn::clear_map(const char *name, bool throw_if_absent) { } } -//------------------------------------------------------------------------------ +__cold bool txn::rename_map(const char *old_name, const char *new_name, bool throw_if_absent) { + map_handle map; + const int err = ::mdbx_dbi_open(handle_, old_name, MDBX_DB_ACCEDE, &map.dbi); + switch (err) { + case MDBX_SUCCESS: + rename_map(map, new_name); + return true; + case MDBX_NOTFOUND: + case MDBX_BAD_DBI: + if (!throw_if_absent) + return false; + MDBX_CXX17_FALLTHROUGH /* fallthrough */; + default: + MDBX_CXX20_UNLIKELY error::throw_exception(err); + } +} + +__cold bool txn::drop_map(const ::mdbx::slice &name, bool throw_if_absent) { + map_handle map; + const int err = ::mdbx_dbi_open2(handle_, name, MDBX_DB_ACCEDE, &map.dbi); + switch (err) { + case MDBX_SUCCESS: + drop_map(map); + return true; + case MDBX_NOTFOUND: + case MDBX_BAD_DBI: + if (!throw_if_absent) + return false; + MDBX_CXX17_FALLTHROUGH /* fallthrough */; + default: + MDBX_CXX20_UNLIKELY error::throw_exception(err); + } +} + +__cold bool txn::clear_map(const ::mdbx::slice &name, bool throw_if_absent) { + map_handle map; + const int err = ::mdbx_dbi_open2(handle_, name, MDBX_DB_ACCEDE, &map.dbi); + switch (err) { + case MDBX_SUCCESS: + clear_map(map); + return true; + case MDBX_NOTFOUND: + case MDBX_BAD_DBI: + if (!throw_if_absent) + return false; + MDBX_CXX17_FALLTHROUGH /* fallthrough */; + default: + MDBX_CXX20_UNLIKELY error::throw_exception(err); + } +} + +__cold bool txn::rename_map(const ::mdbx::slice &old_name, const ::mdbx::slice &new_name, bool throw_if_absent) { + map_handle map; + const int err = ::mdbx_dbi_open2(handle_, old_name, MDBX_DB_ACCEDE, &map.dbi); + switch (err) { + case MDBX_SUCCESS: + rename_map(map, new_name); + return true; + case MDBX_NOTFOUND: + case MDBX_BAD_DBI: + if (!throw_if_absent) + return false; + MDBX_CXX17_FALLTHROUGH /* fallthrough */; + default: + MDBX_CXX20_UNLIKELY error::throw_exception(err); + } +} -void cursor_managed::close() { - if (MDBX_UNLIKELY(!handle_)) - MDBX_CXX20_UNLIKELY error::throw_exception(MDBX_EINVAL); - ::mdbx_cursor_close(handle_); - handle_ = nullptr; +__cold bool txn::rename_map(const ::std::string &old_name, const ::std::string &new_name, bool throw_if_absent) { + return rename_map(::mdbx::slice(old_name), ::mdbx::slice(new_name), throw_if_absent); } //------------------------------------------------------------------------------ @@ -5677,12 +4842,10 @@ __cold ::std::ostream &operator<<(::std::ostream &out, const pair &it) { } __cold ::std::ostream &operator<<(::std::ostream &out, const pair_result &it) { - return out << "{" << (it.done ? "done: " : "non-done: ") << it.key << " => " - << it.value << "}"; + return out << "{" << (it.done ? "done: " : "non-done: ") << it.key << " => " << it.value << "}"; } -__cold ::std::ostream &operator<<(::std::ostream &out, - const ::mdbx::env::geometry::size &it) { +__cold ::std::ostream &operator<<(::std::ostream &out, const ::mdbx::env::geometry::size &it) { switch (it.bytes) { case ::mdbx::env::geometry::default_value: return out << "default"; @@ -5692,8 +4855,7 @@ __cold ::std::ostream &operator<<(::std::ostream &out, return out << "maximal"; } - const auto bytes = (it.bytes < 0) ? out << "-", - size_t(-it.bytes) : size_t(it.bytes); + const auto bytes = (it.bytes < 0) ? out << "-", size_t(-it.bytes) : size_t(it.bytes); struct { size_t one; const char *suffix; @@ -5723,8 +4885,7 @@ __cold ::std::ostream &operator<<(::std::ostream &out, return out; } -__cold ::std::ostream &operator<<(::std::ostream &out, - const env::geometry &it) { +__cold ::std::ostream &operator<<(::std::ostream &out, const env::geometry &it) { return // out << "\tlower " << env::geometry::size(it.size_lower) // << ",\n\tnow " << env::geometry::size(it.size_now) // @@ -5734,8 +4895,7 @@ __cold ::std::ostream &operator<<(::std::ostream &out, << ",\n\tpagesize " << env::geometry::size(it.pagesize) << "\n"; } -__cold ::std::ostream &operator<<(::std::ostream &out, - const env::operate_parameters &it) { +__cold ::std::ostream &operator<<(::std::ostream &out, const env::operate_parameters &it) { return out << "{\n" // << "\tmax_maps " << it.max_maps // << ",\n\tmax_readers " << it.max_readers // @@ -5759,8 +4919,7 @@ __cold ::std::ostream &operator<<(::std::ostream &out, const env::mode &it) { } } -__cold ::std::ostream &operator<<(::std::ostream &out, - const env::durability &it) { +__cold ::std::ostream &operator<<(::std::ostream &out, const env::durability &it) { switch (it) { case env::durability::robust_synchronous: return out << "robust_synchronous"; @@ -5775,21 +4934,19 @@ __cold ::std::ostream &operator<<(::std::ostream &out, } } -__cold ::std::ostream &operator<<(::std::ostream &out, - const env::reclaiming_options &it) { +__cold ::std::ostream &operator<<(::std::ostream &out, const env::reclaiming_options &it) { return out << "{" // << "lifo: " << (it.lifo ? "yes" : "no") // << ", coalesce: " << (it.coalesce ? "yes" : "no") // << "}"; } -__cold ::std::ostream &operator<<(::std::ostream &out, - const env::operate_options &it) { +__cold ::std::ostream &operator<<(::std::ostream &out, const env::operate_options &it) { static const char comma[] = ", "; const char *delimiter = ""; out << "{"; - if (it.orphan_read_transactions) { - out << delimiter << "orphan_read_transactions"; + if (it.no_sticky_threads) { + out << delimiter << "no_sticky_threads"; delimiter = comma; } if (it.nested_write_transactions) { @@ -5813,8 +4970,7 @@ __cold ::std::ostream &operator<<(::std::ostream &out, return out << "}"; } -__cold ::std::ostream &operator<<(::std::ostream &out, - const env_managed::create_parameters &it) { +__cold ::std::ostream &operator<<(::std::ostream &out, const env_managed::create_parameters &it) { return out << "{\n" // << "\tfile_mode " << std::oct << it.file_mode_bits << std::dec // << ",\n\tsubdirectory " << (it.use_subdirectory ? "yes" : "no") // @@ -5822,8 +4978,7 @@ __cold ::std::ostream &operator<<(::std::ostream &out, << it.geometry << "}"; } -__cold ::std::ostream &operator<<(::std::ostream &out, - const MDBX_log_level_t &it) { +__cold ::std::ostream &operator<<(::std::ostream &out, const MDBX_log_level_t &it) { switch (it) { case MDBX_LOG_FATAL: return out << "LOG_FATAL"; @@ -5848,8 +5003,7 @@ __cold ::std::ostream &operator<<(::std::ostream &out, } } -__cold ::std::ostream &operator<<(::std::ostream &out, - const MDBX_debug_flags_t &it) { +__cold ::std::ostream &operator<<(::std::ostream &out, const MDBX_debug_flags_t &it) { if (it == MDBX_DBG_DONTCHANGE) return out << "DBG_DONTCHANGE"; @@ -5885,8 +5039,7 @@ __cold ::std::ostream &operator<<(::std::ostream &out, return out << "}"; } -__cold ::std::ostream &operator<<(::std::ostream &out, - const ::mdbx::error &err) { +__cold ::std::ostream &operator<<(::std::ostream &out, const ::mdbx::error &err) { return out << err.what() << " (" << long(err.code()) << ")"; } diff --git a/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx.h b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx.h index 14b7513d16c..90835d1b9e9 100644 --- a/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx.h +++ b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx.h @@ -1,11 +1,10 @@ /** -_libmdbx_ is an extremely fast, compact, powerful, embedded, +_libmdbx_ (aka MDBX) is an extremely fast, compact, powerful, embeddable, transactional [key-value -store](https://en.wikipedia.org/wiki/Key-value_database) database, with -[permissive license](./LICENSE). _MDBX_ has a specific set of properties and -capabilities, focused on creating unique lightweight solutions with -extraordinary performance. +store](https://en.wikipedia.org/wiki/Key-value_database), with [Apache 2.0 +license](./LICENSE). _MDBX_ has a specific set of properties and capabilities, +focused on creating unique lightweight solutions with extraordinary performance. _libmdbx_ is superior to [LMDB](https://bit.ly/26ts7tL) in terms of features and reliability, not inferior in performance. In comparison to LMDB, _libmdbx_ @@ -14,60 +13,25 @@ break down. _libmdbx_ supports Linux, Windows, MacOS, OSX, iOS, Android, FreeBSD, DragonFly, Solaris, OpenSolaris, OpenIndiana, NetBSD, OpenBSD and other systems compliant with POSIX.1-2008. -The origin has been migrated to -[GitFlic](https://gitflic.ru/project/erthink/libmdbx) since on 2022-04-15 -the Github administration, without any warning nor explanation, deleted libmdbx -along with a lot of other projects, simultaneously blocking access for many -developers. For the same reason ~~Github~~ is blacklisted forever. +Please visit https://libmdbx.dqdkfa.ru for more information, documentation, +C++ API description and links to the origin git repo with the source code. +Questions, feedback and suggestions are welcome to the Telegram' group +https://t.me/libmdbx. -_The Future will (be) [Positive](https://www.ptsecurity.com). Всё будет хорошо._ +Donations are welcome to ETH `0xD104d8f8B2dC312aaD74899F83EBf3EEBDC1EA3A`. +Всё будет хорошо! +\note The origin has been migrated to +[GitFlic](https://gitflic.ru/project/erthink/libmdbx) since on 2022-04-15 the +Github administration, without any warning nor explanation, deleted libmdbx +along with a lot of other projects, simultaneously blocking access for many +developers. For the same reason ~~Github~~ is blacklisted forever. \section copyright LICENSE & COPYRIGHT - -\authors Copyright (c) 2015-2024, Leonid Yuriev -and other _libmdbx_ authors: please see [AUTHORS](./AUTHORS) file. - -\copyright Redistribution and use in source and binary forms, with or without -modification, are permitted only as authorized by the OpenLDAP Public License. - -A copy of this license is available in the file LICENSE in the -top-level directory of the distribution or, alternatively, at -. - - --- - -This code is derived from "LMDB engine" written by -Howard Chu (Symas Corporation), which itself derived from btree.c -written by Martin Hedenfalk. - - --- - -Portions Copyright 2011-2015 Howard Chu, Symas Corp. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted only as authorized by the OpenLDAP -Public License. - -A copy of this license is available in the file LICENSE in the -top-level directory of the distribution or, alternatively, at -. - - --- - -Portions Copyright (c) 2009, 2010 Martin Hedenfalk - -Permission to use, copy, modify, and distribute this software for any -purpose with or without fee is hereby granted, provided that the above -copyright notice and this permission notice appear in all copies. - -THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +\copyright SPDX-License-Identifier: Apache-2.0 +\note Please refer to the COPYRIGHT file for explanations license change, +credits and acknowledgments. +\author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 *******************************************************************************/ @@ -75,8 +39,7 @@ OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. #ifndef LIBMDBX_H #define LIBMDBX_H -#if defined(__riscv) || defined(__riscv__) || defined(__RISCV) || \ - defined(__RISCV__) +#if defined(__riscv) || defined(__riscv__) || defined(__RISCV) || defined(__RISCV__) #warning "The RISC-V architecture is intentionally insecure by design. \ Please delete this admonition at your own risk, \ if you make such decision informed and consciously. \ @@ -85,12 +48,12 @@ OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. #ifdef _MSC_VER #pragma warning(push, 1) -#pragma warning(disable : 4548) /* expression before comma has no effect; \ +#pragma warning(disable : 4548) /* expression before comma has no effect; \ expected expression with side - effect */ -#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ +#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ * semantics are not enabled. Specify /EHsc */ -#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ - * mode specified; termination on exception is \ +#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ + * mode specified; termination on exception is \ * not guaranteed. Specify /EHsc */ #endif /* _MSC_VER (warnings) */ @@ -98,14 +61,14 @@ OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. /* clang-format off */ /** \file mdbx.h - \brief The libmdbx C API header file + \brief The libmdbx C API header file. \defgroup c_api C API @{ \defgroup c_err Error handling \defgroup c_opening Opening & Closing \defgroup c_transactions Transactions - \defgroup c_dbi Databases + \defgroup c_dbi Tables \defgroup c_crud Create/Read/Update/Delete (see Quick Reference in details) \details @@ -116,9 +79,9 @@ Historically, libmdbx inherits the API basis from LMDB, where it is often difficult to select flags/options and functions for the desired operation. So it is recommend using this hints. -## Databases with UNIQUE keys +## Tables with UNIQUE keys -In databases created without the \ref MDBX_DUPSORT option, keys are always +In tables created without the \ref MDBX_DUPSORT option, keys are always unique. Thus always a single value corresponds to the each key, and so there are only a few cases of changing data. @@ -136,18 +99,16 @@ are only a few cases of changing data. | _DELETING_||| |Key is absent → Error since no such key |\ref mdbx_del() or \ref mdbx_replace()|Error \ref MDBX_NOTFOUND| |Key exist → Delete by key |\ref mdbx_del() with the parameter `data = NULL`|Deletion| -|Key exist → Delete by key with with data matching check|\ref mdbx_del() with the parameter `data` filled with the value which should be match for deletion|Deletion or \ref MDBX_NOTFOUND if the value does not match| +|Key exist → Delete by key with data matching check|\ref mdbx_del() with the parameter `data` filled with the value which should be match for deletion|Deletion or \ref MDBX_NOTFOUND if the value does not match| |Delete at the current cursor position |\ref mdbx_cursor_del() with \ref MDBX_CURRENT flag|Deletion| |Extract (read & delete) value by the key |\ref mdbx_replace() with zero flag and parameter `new_data = NULL`|Returning a deleted value| +## Tables with NON-UNIQUE keys -## Databases with NON-UNIQUE keys - -In databases created with the \ref MDBX_DUPSORT (Sorted Duplicates) option, keys -may be non unique. Such non-unique keys in a key-value database may be treated +In tables created with the \ref MDBX_DUPSORT (Sorted Duplicates) option, keys +may be non unique. Such non-unique keys in a key-value table may be treated as a duplicates or as like a multiple values corresponds to keys. - | Case | Flags to use | Result | |---------------------------------------------|---------------------|------------------------| | _INSERTING_||| @@ -169,7 +130,7 @@ as a duplicates or as like a multiple values corresponds to keys. |Key exist → Delete all values corresponds given key|\ref mdbx_del() with the parameter `data = NULL`|Deletion| |Key exist → Delete particular value corresponds given key|\ref mdbx_del() with the parameter `data` filled with the value that wanna to delete, or \ref mdbx_replace() with \ref MDBX_CURRENT + \ref MDBX_NOOVERWRITE and the `old_value` parameter filled with the value that wanna to delete and `new_data = NULL`| Deletion or \ref MDBX_NOTFOUND if no such key-value pair| |Delete one value at the current cursor position|\ref mdbx_cursor_del() with \ref MDBX_CURRENT flag|Deletion only the current entry| -|Delete all values of key at the current cursor position|\ref mdbx_cursor_del() with with \ref MDBX_ALLDUPS flag|Deletion all duplicates of key (all multi-values) at the current cursor position| +|Delete all values of key at the current cursor position|\ref mdbx_cursor_del() with \ref MDBX_ALLDUPS flag|Deletion all duplicates of key (all multi-values) at the current cursor position| \defgroup c_cursors Cursors \defgroup c_statinfo Statistics & Information @@ -226,16 +187,45 @@ typedef mode_t mdbx_mode_t; #define __has_attribute(x) (0) #endif /* __has_attribute */ +#ifndef __has_c_attribute +#define __has_c_attribute(x) (0) +#define __has_c_attribute_qualified(x) 0 +#elif !defined(__STDC_VERSION__) || __STDC_VERSION__ < 202311L +#define __has_c_attribute_qualified(x) 0 +#elif defined(_MSC_VER) +/* MSVC don't support `namespace::attr` syntax */ +#define __has_c_attribute_qualified(x) 0 +#else +#define __has_c_attribute_qualified(x) __has_c_attribute(x) +#endif /* __has_c_attribute */ + #ifndef __has_cpp_attribute #define __has_cpp_attribute(x) 0 +#define __has_cpp_attribute_qualified(x) 0 +#elif defined(_MSC_VER) || (__clang__ && __clang__ < 14) +/* MSVC don't support `namespace::attr` syntax */ +#define __has_cpp_attribute_qualified(x) 0 +#else +#define __has_cpp_attribute_qualified(x) __has_cpp_attribute(x) #endif /* __has_cpp_attribute */ +#ifndef __has_C23_or_CXX_attribute +#if defined(__cplusplus) +#define __has_C23_or_CXX_attribute(x) __has_cpp_attribute_qualified(x) +#else +#define __has_C23_or_CXX_attribute(x) __has_c_attribute_qualified(x) +#endif +#endif /* __has_C23_or_CXX_attribute */ + #ifndef __has_feature #define __has_feature(x) (0) +#define __has_exceptions_disabled (0) +#elif !defined(__has_exceptions_disabled) +#define __has_exceptions_disabled (__has_feature(cxx_noexcept) && !__has_feature(cxx_exceptions)) #endif /* __has_feature */ #ifndef __has_extension -#define __has_extension(x) (0) +#define __has_extension(x) __has_feature(x) #endif /* __has_extension */ #ifndef __has_builtin @@ -250,15 +240,12 @@ typedef mode_t mdbx_mode_t; * These functions should be declared with the attribute pure. */ #if defined(DOXYGEN) #define MDBX_PURE_FUNCTION [[gnu::pure]] -#elif (defined(__GNUC__) || __has_attribute(__pure__)) && \ - (!defined(__clang__) /* https://bugs.llvm.org/show_bug.cgi?id=43275 */ \ - || !defined(__cplusplus) || !__has_feature(cxx_exceptions)) -#define MDBX_PURE_FUNCTION __attribute__((__pure__)) -#elif defined(_MSC_VER) && !defined(__clang__) && _MSC_VER >= 1920 -#define MDBX_PURE_FUNCTION -#elif defined(__cplusplus) && __has_cpp_attribute(gnu::pure) && \ - (!defined(__clang__) || !__has_feature(cxx_exceptions)) +#elif __has_C23_or_CXX_attribute(gnu::pure) #define MDBX_PURE_FUNCTION [[gnu::pure]] +#elif (defined(__GNUC__) || __has_attribute(__pure__)) && \ + (!defined(__clang__) /* https://bugs.llvm.org/show_bug.cgi?id=43275 */ || !defined(__cplusplus) || \ + __has_exceptions_disabled) +#define MDBX_PURE_FUNCTION __attribute__((__pure__)) #else #define MDBX_PURE_FUNCTION #endif /* MDBX_PURE_FUNCTION */ @@ -268,22 +255,15 @@ typedef mode_t mdbx_mode_t; * that is compatible to CLANG and proposed [[pure]]. */ #if defined(DOXYGEN) #define MDBX_NOTHROW_PURE_FUNCTION [[gnu::pure, gnu::nothrow]] -#elif defined(__GNUC__) || \ - (__has_attribute(__pure__) && __has_attribute(__nothrow__)) -#define MDBX_NOTHROW_PURE_FUNCTION __attribute__((__pure__, __nothrow__)) -#elif defined(_MSC_VER) && !defined(__clang__) && _MSC_VER >= 1920 -#if __has_cpp_attribute(pure) -#define MDBX_NOTHROW_PURE_FUNCTION [[pure]] -#else -#define MDBX_NOTHROW_PURE_FUNCTION -#endif -#elif defined(__cplusplus) && __has_cpp_attribute(gnu::pure) -#if __has_cpp_attribute(gnu::nothrow) +#elif __has_C23_or_CXX_attribute(gnu::pure) +#if __has_C23_or_CXX_attribute(gnu::nothrow) #define MDBX_NOTHROW_PURE_FUNCTION [[gnu::pure, gnu::nothrow]] #else #define MDBX_NOTHROW_PURE_FUNCTION [[gnu::pure]] #endif -#elif defined(__cplusplus) && __has_cpp_attribute(pure) +#elif defined(__GNUC__) || (__has_attribute(__pure__) && __has_attribute(__nothrow__)) +#define MDBX_NOTHROW_PURE_FUNCTION __attribute__((__pure__, __nothrow__)) +#elif __has_cpp_attribute(pure) #define MDBX_NOTHROW_PURE_FUNCTION [[pure]] #else #define MDBX_NOTHROW_PURE_FUNCTION @@ -301,15 +281,12 @@ typedef mode_t mdbx_mode_t; * It does not make sense for a const function to return void. */ #if defined(DOXYGEN) #define MDBX_CONST_FUNCTION [[gnu::const]] -#elif (defined(__GNUC__) || __has_attribute(__pure__)) && \ - (!defined(__clang__) /* https://bugs.llvm.org/show_bug.cgi?id=43275 */ \ - || !defined(__cplusplus) || !__has_feature(cxx_exceptions)) -#define MDBX_CONST_FUNCTION __attribute__((__const__)) -#elif defined(_MSC_VER) && !defined(__clang__) && _MSC_VER >= 1920 -#define MDBX_CONST_FUNCTION MDBX_PURE_FUNCTION -#elif defined(__cplusplus) && __has_cpp_attribute(gnu::const) && \ - (!defined(__clang__) || !__has_feature(cxx_exceptions)) +#elif __has_C23_or_CXX_attribute(gnu::const) #define MDBX_CONST_FUNCTION [[gnu::const]] +#elif (defined(__GNUC__) || __has_attribute(__const__)) && \ + (!defined(__clang__) /* https://bugs.llvm.org/show_bug.cgi?id=43275 */ || !defined(__cplusplus) || \ + __has_exceptions_disabled) +#define MDBX_CONST_FUNCTION __attribute__((__const__)) #else #define MDBX_CONST_FUNCTION MDBX_PURE_FUNCTION #endif /* MDBX_CONST_FUNCTION */ @@ -319,18 +296,15 @@ typedef mode_t mdbx_mode_t; * that is compatible to CLANG and future [[const]]. */ #if defined(DOXYGEN) #define MDBX_NOTHROW_CONST_FUNCTION [[gnu::const, gnu::nothrow]] -#elif defined(__GNUC__) || \ - (__has_attribute(__const__) && __has_attribute(__nothrow__)) -#define MDBX_NOTHROW_CONST_FUNCTION __attribute__((__const__, __nothrow__)) -#elif defined(_MSC_VER) && !defined(__clang__) && _MSC_VER >= 1920 -#define MDBX_NOTHROW_CONST_FUNCTION MDBX_NOTHROW_PURE_FUNCTION -#elif defined(__cplusplus) && __has_cpp_attribute(gnu::const) -#if __has_cpp_attribute(gnu::nothrow) -#define MDBX_NOTHROW_PURE_FUNCTION [[gnu::const, gnu::nothrow]] +#elif __has_C23_or_CXX_attribute(gnu::const) +#if __has_C23_or_CXX_attribute(gnu::nothrow) +#define MDBX_NOTHROW_CONST_FUNCTION [[gnu::const, gnu::nothrow]] #else -#define MDBX_NOTHROW_PURE_FUNCTION [[gnu::const]] +#define MDBX_NOTHROW_CONST_FUNCTION [[gnu::const]] #endif -#elif defined(__cplusplus) && __has_cpp_attribute(const) +#elif defined(__GNUC__) || (__has_attribute(__const__) && __has_attribute(__nothrow__)) +#define MDBX_NOTHROW_CONST_FUNCTION __attribute__((__const__, __nothrow__)) +#elif __has_cpp_attribute_qualified(const) #define MDBX_NOTHROW_CONST_FUNCTION [[const]] #else #define MDBX_NOTHROW_CONST_FUNCTION MDBX_NOTHROW_PURE_FUNCTION @@ -342,15 +316,13 @@ typedef mode_t mdbx_mode_t; #ifndef MDBX_DEPRECATED #ifdef __deprecated #define MDBX_DEPRECATED __deprecated -#elif defined(DOXYGEN) || \ - (defined(__cplusplus) && __cplusplus >= 201403L && \ - __has_cpp_attribute(deprecated) && \ - __has_cpp_attribute(deprecated) >= 201309L) || \ - (!defined(__cplusplus) && defined(__STDC_VERSION__) && \ - __STDC_VERSION__ >= 202304L) +#elif defined(DOXYGEN) || ((!defined(__GNUC__) || (defined(__clang__) && __clang__ > 19) || __GNUC__ > 5) && \ + ((defined(__cplusplus) && __cplusplus >= 201403L && __has_cpp_attribute(deprecated) && \ + __has_cpp_attribute(deprecated) >= 201309L) || \ + (!defined(__cplusplus) && defined(__STDC_VERSION__) && __STDC_VERSION__ >= 202304L))) #define MDBX_DEPRECATED [[deprecated]] -#elif (defined(__GNUC__) && __GNUC__ > 5) || \ - (__has_attribute(__deprecated__) && !defined(__GNUC__)) +#elif (defined(__GNUC__) && __GNUC__ > 5) || \ + (__has_attribute(__deprecated__) && (!defined(__GNUC__) || defined(__clang__) || __GNUC__ > 5)) #define MDBX_DEPRECATED __attribute__((__deprecated__)) #elif defined(_MSC_VER) #define MDBX_DEPRECATED __declspec(deprecated) @@ -359,9 +331,21 @@ typedef mode_t mdbx_mode_t; #endif #endif /* MDBX_DEPRECATED */ +#ifndef MDBX_DEPRECATED_ENUM +#ifdef __deprecated_enum +#define MDBX_DEPRECATED_ENUM __deprecated_enum +#elif defined(DOXYGEN) || \ + (!defined(_MSC_VER) || (defined(__cplusplus) && __cplusplus >= 201403L && __has_cpp_attribute(deprecated) && \ + __has_cpp_attribute(deprecated) >= 201309L)) +#define MDBX_DEPRECATED_ENUM MDBX_DEPRECATED +#else +#define MDBX_DEPRECATED_ENUM /* avoid madness MSVC */ +#endif +#endif /* MDBX_DEPRECATED_ENUM */ + #ifndef __dll_export -#if defined(_WIN32) || defined(_WIN64) || defined(__CYGWIN__) || \ - defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__) +#if defined(_WIN32) || defined(_WIN64) || defined(__CYGWIN__) || defined(__MINGW__) || defined(__MINGW32__) || \ + defined(__MINGW64__) #if defined(__GNUC__) || __has_attribute(__dllexport__) #define __dll_export __attribute__((__dllexport__)) #elif defined(_MSC_VER) @@ -377,8 +361,8 @@ typedef mode_t mdbx_mode_t; #endif /* __dll_export */ #ifndef __dll_import -#if defined(_WIN32) || defined(_WIN64) || defined(__CYGWIN__) || \ - defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__) +#if defined(_WIN32) || defined(_WIN64) || defined(__CYGWIN__) || defined(__MINGW__) || defined(__MINGW32__) || \ + defined(__MINGW64__) #if defined(__GNUC__) || __has_attribute(__dllimport__) #define __dll_import __attribute__((__dllimport__)) #elif defined(_MSC_VER) @@ -393,10 +377,11 @@ typedef mode_t mdbx_mode_t; /** \brief Auxiliary macro for robustly define the both inline version of API * function and non-inline fallback dll-exported version for applications linked - * with old version of libmdbx, with a strictly ODR-common implementation. */ + * with old version of libmdbx, with a strictly ODR-common implementation. Thus, + * we emulate __extern_inline for all compilers, including non-GNU ones. */ #if defined(LIBMDBX_INTERNALS) && !defined(LIBMDBX_NO_EXPORTS_LEGACY_API) -#define LIBMDBX_INLINE_API(TYPE, NAME, ARGS) \ - /* proto of exported which uses common impl */ LIBMDBX_API TYPE NAME ARGS; \ +#define LIBMDBX_INLINE_API(TYPE, NAME, ARGS) \ + /* proto of exported which uses common impl */ LIBMDBX_API TYPE NAME ARGS; \ /* definition of common impl */ static __inline TYPE __inline_##NAME ARGS #else #define LIBMDBX_INLINE_API(TYPE, NAME, ARGS) static __inline TYPE NAME ARGS @@ -425,8 +410,7 @@ typedef mode_t mdbx_mode_t; /** Workaround for old compilers without support for C++17 `noexcept`. */ #if defined(DOXYGEN) #define MDBX_CXX17_NOEXCEPT noexcept -#elif !defined(__cpp_noexcept_function_type) || \ - __cpp_noexcept_function_type < 201510L +#elif !defined(__cpp_noexcept_function_type) || __cpp_noexcept_function_type < 201510L #define MDBX_CXX17_NOEXCEPT #else #define MDBX_CXX17_NOEXCEPT noexcept @@ -439,14 +423,11 @@ typedef mode_t mdbx_mode_t; #elif !defined(__cplusplus) #define MDBX_CXX01_CONSTEXPR __inline #define MDBX_CXX01_CONSTEXPR_VAR const -#elif !defined(DOXYGEN) && \ - ((__cplusplus < 201103L && defined(__cpp_constexpr) && \ - __cpp_constexpr < 200704L) || \ - (defined(__LCC__) && __LCC__ < 124) || \ - (defined(__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ < 407) && \ - !defined(__clang__) && !defined(__LCC__)) || \ - (defined(_MSC_VER) && _MSC_VER < 1910) || \ - (defined(__clang__) && __clang_major__ < 4)) +#elif !defined(DOXYGEN) && \ + ((__cplusplus < 201103L && defined(__cpp_constexpr) && __cpp_constexpr < 200704L) || \ + (defined(__LCC__) && __LCC__ < 124) || \ + (defined(__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ < 407) && !defined(__clang__) && !defined(__LCC__)) || \ + (defined(_MSC_VER) && _MSC_VER < 1910) || (defined(__clang__) && __clang_major__ < 4)) #define MDBX_CXX01_CONSTEXPR inline #define MDBX_CXX01_CONSTEXPR_VAR const #else @@ -462,13 +443,10 @@ typedef mode_t mdbx_mode_t; #elif !defined(__cplusplus) #define MDBX_CXX11_CONSTEXPR __inline #define MDBX_CXX11_CONSTEXPR_VAR const -#elif !defined(DOXYGEN) && \ - (!defined(__cpp_constexpr) || __cpp_constexpr < 201304L || \ - (defined(__LCC__) && __LCC__ < 124) || \ - (defined(__GNUC__) && __GNUC__ < 6 && !defined(__clang__) && \ - !defined(__LCC__)) || \ - (defined(_MSC_VER) && _MSC_VER < 1910) || \ - (defined(__clang__) && __clang_major__ < 5)) +#elif !defined(DOXYGEN) && \ + (!defined(__cpp_constexpr) || __cpp_constexpr < 201304L || (defined(__LCC__) && __LCC__ < 124) || \ + (defined(__GNUC__) && __GNUC__ < 6 && !defined(__clang__) && !defined(__LCC__)) || \ + (defined(_MSC_VER) && _MSC_VER < 1910) || (defined(__clang__) && __clang_major__ < 5)) #define MDBX_CXX11_CONSTEXPR inline #define MDBX_CXX11_CONSTEXPR_VAR const #else @@ -484,12 +462,10 @@ typedef mode_t mdbx_mode_t; #elif !defined(__cplusplus) #define MDBX_CXX14_CONSTEXPR __inline #define MDBX_CXX14_CONSTEXPR_VAR const -#elif defined(DOXYGEN) || \ - defined(__cpp_constexpr) && __cpp_constexpr >= 201304L && \ - ((defined(_MSC_VER) && _MSC_VER >= 1910) || \ - (defined(__clang__) && __clang_major__ > 4) || \ - (defined(__GNUC__) && __GNUC__ > 6) || \ - (!defined(__GNUC__) && !defined(__clang__) && !defined(_MSC_VER))) +#elif defined(DOXYGEN) || \ + defined(__cpp_constexpr) && __cpp_constexpr >= 201304L && \ + ((defined(_MSC_VER) && _MSC_VER >= 1910) || (defined(__clang__) && __clang_major__ > 4) || \ + (defined(__GNUC__) && __GNUC__ > 6) || (!defined(__GNUC__) && !defined(__clang__) && !defined(_MSC_VER))) #define MDBX_CXX14_CONSTEXPR constexpr #define MDBX_CXX14_CONSTEXPR_VAR constexpr #else @@ -501,9 +477,8 @@ typedef mode_t mdbx_mode_t; #define MDBX_NORETURN __noreturn #elif defined(_Noreturn) #define MDBX_NORETURN _Noreturn -#elif defined(DOXYGEN) || (defined(__cplusplus) && __cplusplus >= 201103L) || \ - (!defined(__cplusplus) && defined(__STDC_VERSION__) && \ - __STDC_VERSION__ > 202005L) +#elif defined(DOXYGEN) || (defined(__cplusplus) && __cplusplus >= 201103L) || \ + (!defined(__cplusplus) && defined(__STDC_VERSION__) && __STDC_VERSION__ > 202005L) #define MDBX_NORETURN [[noreturn]] #elif defined(__GNUC__) || __has_attribute(__noreturn__) #define MDBX_NORETURN __attribute__((__noreturn__)) @@ -516,23 +491,19 @@ typedef mode_t mdbx_mode_t; #ifndef MDBX_PRINTF_ARGS #if defined(__GNUC__) || __has_attribute(__format__) || defined(DOXYGEN) #if defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__) -#define MDBX_PRINTF_ARGS(format_index, first_arg) \ - __attribute__((__format__(__gnu_printf__, format_index, first_arg))) +#define MDBX_PRINTF_ARGS(format_index, first_arg) __attribute__((__format__(__gnu_printf__, format_index, first_arg))) #else -#define MDBX_PRINTF_ARGS(format_index, first_arg) \ - __attribute__((__format__(__printf__, format_index, first_arg))) +#define MDBX_PRINTF_ARGS(format_index, first_arg) __attribute__((__format__(__printf__, format_index, first_arg))) #endif /* MinGW */ #else #define MDBX_PRINTF_ARGS(format_index, first_arg) #endif #endif /* MDBX_PRINTF_ARGS */ -#if defined(DOXYGEN) || \ - (defined(__cplusplus) && __cplusplus >= 201603L && \ - __has_cpp_attribute(maybe_unused) && \ - __has_cpp_attribute(maybe_unused) >= 201603L) || \ - (!defined(__cplusplus) && defined(__STDC_VERSION__) && \ - __STDC_VERSION__ > 202005L) +#if defined(DOXYGEN) || \ + (defined(__cplusplus) && __cplusplus >= 201603L && __has_cpp_attribute(maybe_unused) && \ + __has_cpp_attribute(maybe_unused) >= 201603L && (!defined(__clang__) || __clang__ > 19)) || \ + (!defined(__cplusplus) && defined(__STDC_VERSION__) && __STDC_VERSION__ > 202005L) #define MDBX_MAYBE_UNUSED [[maybe_unused]] #elif defined(__GNUC__) || __has_attribute(__unused__) #define MDBX_MAYBE_UNUSED __attribute__((__unused__)) @@ -551,15 +522,12 @@ typedef mode_t mdbx_mode_t; * - the proper implementation of DEFINE_ENUM_FLAG_OPERATORS for C++ required * the constexpr feature which is broken in most old compilers; * - DEFINE_ENUM_FLAG_OPERATORS may be defined broken as in the Windows SDK. */ -#ifndef DEFINE_ENUM_FLAG_OPERATORS +#if !defined(DEFINE_ENUM_FLAG_OPERATORS) && !defined(DOXYGEN) #ifdef __cplusplus -#if !defined(__cpp_constexpr) || __cpp_constexpr < 200704L || \ - (defined(__LCC__) && __LCC__ < 124) || \ - (defined(__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ < 407) && \ - !defined(__clang__) && !defined(__LCC__)) || \ - (defined(_MSC_VER) && _MSC_VER < 1910) || \ - (defined(__clang__) && __clang_major__ < 4) +#if !defined(__cpp_constexpr) || __cpp_constexpr < 200704L || (defined(__LCC__) && __LCC__ < 124) || \ + (defined(__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ < 407) && !defined(__clang__) && !defined(__LCC__)) || \ + (defined(_MSC_VER) && _MSC_VER < 1910) || (defined(__clang__) && __clang_major__ < 4) /* The constexpr feature is not available or (may be) broken */ #define CONSTEXPR_ENUM_FLAGS_OPERATIONS 0 #else @@ -569,42 +537,18 @@ typedef mode_t mdbx_mode_t; /// Define operator overloads to enable bit operations on enum values that are /// used to define flags (based on Microsoft's DEFINE_ENUM_FLAG_OPERATORS). -#define DEFINE_ENUM_FLAG_OPERATORS(ENUM) \ - extern "C++" { \ - MDBX_NOSANITIZE_ENUM MDBX_CXX01_CONSTEXPR ENUM operator|(ENUM a, ENUM b) { \ - return ENUM(unsigned(a) | unsigned(b)); \ - } \ - MDBX_NOSANITIZE_ENUM MDBX_CXX14_CONSTEXPR ENUM &operator|=(ENUM &a, \ - ENUM b) { \ - return a = a | b; \ - } \ - MDBX_NOSANITIZE_ENUM MDBX_CXX01_CONSTEXPR ENUM operator&(ENUM a, ENUM b) { \ - return ENUM(unsigned(a) & unsigned(b)); \ - } \ - MDBX_NOSANITIZE_ENUM MDBX_CXX01_CONSTEXPR ENUM operator&(ENUM a, \ - unsigned b) { \ - return ENUM(unsigned(a) & b); \ - } \ - MDBX_NOSANITIZE_ENUM MDBX_CXX01_CONSTEXPR ENUM operator&(unsigned a, \ - ENUM b) { \ - return ENUM(a & unsigned(b)); \ - } \ - MDBX_NOSANITIZE_ENUM MDBX_CXX14_CONSTEXPR ENUM &operator&=(ENUM &a, \ - ENUM b) { \ - return a = a & b; \ - } \ - MDBX_NOSANITIZE_ENUM MDBX_CXX14_CONSTEXPR ENUM &operator&=(ENUM &a, \ - unsigned b) { \ - return a = a & b; \ - } \ - MDBX_CXX01_CONSTEXPR unsigned operator~(ENUM a) { return ~unsigned(a); } \ - MDBX_NOSANITIZE_ENUM MDBX_CXX01_CONSTEXPR ENUM operator^(ENUM a, ENUM b) { \ - return ENUM(unsigned(a) ^ unsigned(b)); \ - } \ - MDBX_NOSANITIZE_ENUM MDBX_CXX14_CONSTEXPR ENUM &operator^=(ENUM &a, \ - ENUM b) { \ - return a = a ^ b; \ - } \ +#define DEFINE_ENUM_FLAG_OPERATORS(ENUM) \ + extern "C++" { \ + MDBX_NOSANITIZE_ENUM MDBX_CXX01_CONSTEXPR ENUM operator|(ENUM a, ENUM b) { return ENUM(unsigned(a) | unsigned(b)); } \ + MDBX_NOSANITIZE_ENUM MDBX_CXX14_CONSTEXPR ENUM &operator|=(ENUM &a, ENUM b) { return a = a | b; } \ + MDBX_NOSANITIZE_ENUM MDBX_CXX01_CONSTEXPR ENUM operator&(ENUM a, ENUM b) { return ENUM(unsigned(a) & unsigned(b)); } \ + MDBX_NOSANITIZE_ENUM MDBX_CXX01_CONSTEXPR ENUM operator&(ENUM a, unsigned b) { return ENUM(unsigned(a) & b); } \ + MDBX_NOSANITIZE_ENUM MDBX_CXX01_CONSTEXPR ENUM operator&(unsigned a, ENUM b) { return ENUM(a & unsigned(b)); } \ + MDBX_NOSANITIZE_ENUM MDBX_CXX14_CONSTEXPR ENUM &operator&=(ENUM &a, ENUM b) { return a = a & b; } \ + MDBX_NOSANITIZE_ENUM MDBX_CXX14_CONSTEXPR ENUM &operator&=(ENUM &a, unsigned b) { return a = a & b; } \ + MDBX_CXX01_CONSTEXPR unsigned operator~(ENUM a) { return ~unsigned(a); } \ + MDBX_NOSANITIZE_ENUM MDBX_CXX01_CONSTEXPR ENUM operator^(ENUM a, ENUM b) { return ENUM(unsigned(a) ^ unsigned(b)); } \ + MDBX_NOSANITIZE_ENUM MDBX_CXX14_CONSTEXPR ENUM &operator^=(ENUM &a, ENUM b) { return a = a ^ b; } \ } #else /* __cplusplus */ /* nope for C since it always allows these operators for enums */ @@ -635,12 +579,12 @@ typedef mode_t mdbx_mode_t; extern "C" { #endif -/* MDBX version 0.12.x */ +/* MDBX version 0.13.x */ #define MDBX_VERSION_MAJOR 0 -#define MDBX_VERSION_MINOR 12 +#define MDBX_VERSION_MINOR 13 #ifndef LIBMDBX_API -#if defined(LIBMDBX_EXPORTS) +#if defined(LIBMDBX_EXPORTS) || defined(DOXYGEN) #define LIBMDBX_API __dll_export #elif defined(LIBMDBX_IMPORTS) #define LIBMDBX_API __dll_import @@ -650,7 +594,7 @@ extern "C" { #endif /* LIBMDBX_API */ #ifdef __cplusplus -#if defined(__clang__) || __has_attribute(type_visibility) +#if defined(__clang__) || __has_attribute(type_visibility) || defined(DOXYGEN) #define LIBMDBX_API_TYPE LIBMDBX_API __attribute__((type_visibility("default"))) #else #define LIBMDBX_API_TYPE LIBMDBX_API @@ -665,12 +609,13 @@ extern "C" { #define LIBMDBX_VERINFO_API __dll_export #endif /* LIBMDBX_VERINFO_API */ -/** \brief libmdbx version information */ +/** \brief libmdbx version information, \see https://semver.org/ */ extern LIBMDBX_VERINFO_API const struct MDBX_version_info { - uint8_t major; /**< Major version number */ - uint8_t minor; /**< Minor version number */ - uint16_t release; /**< Release number of Major.Minor */ - uint32_t revision; /**< Revision number of Release */ + uint16_t major; /**< Major version number */ + uint16_t minor; /**< Minor version number */ + uint16_t patch; /**< Patch number */ + uint16_t tweak; /**< Tweak number */ + const char *semver_prerelease; /**< Semantic Versioning `pre-release` */ struct { const char *datetime; /**< committer date, strict ISO-8601 format */ const char *tree; /**< commit hash (hexadecimal digits) */ @@ -689,6 +634,9 @@ extern LIBMDBX_VERINFO_API const struct MDBX_build_info { const char *options; /**< mdbx-related options */ const char *compiler; /**< compiler */ const char *flags; /**< CFLAGS and CXXFLAGS */ + const char *metadata; /**< an extra/custom information provided via + the MDBX_BUILD_METADATA definition + during library build */ } /** \brief libmdbx build information */ mdbx_build; #if (defined(_WIN32) || defined(_WIN64)) && !MDBX_BUILD_SHARED_LIBRARY @@ -732,8 +680,7 @@ extern LIBMDBX_VERINFO_API const struct MDBX_build_info { /* As described above mdbx_module_handler() IS REQUIRED for Windows versions * prior to Windows Vista. */ #define MDBX_MANUAL_MODULE_HANDLER 1 -void LIBMDBX_API NTAPI mdbx_module_handler(PVOID module, DWORD reason, - PVOID reserved); +void LIBMDBX_API NTAPI mdbx_module_handler(PVOID module, DWORD reason, PVOID reserved); #endif #endif /* Windows && !DLL && MDBX_MANUAL_MODULE_HANDLER */ @@ -741,8 +688,8 @@ void LIBMDBX_API NTAPI mdbx_module_handler(PVOID module, DWORD reason, /* OPACITY STRUCTURES *********************************************************/ /** \brief Opaque structure for a database environment. - * \details An environment supports multiple key-value sub-databases (aka - * key-value spaces or tables), all residing in the same shared-memory map. + * \details An environment supports multiple key-value tables (aka key-value + * maps, spaces or sub-databases), all residing in the same shared-memory map. * \see mdbx_env_create() \see mdbx_env_close() */ #ifndef __cplusplus typedef struct MDBX_env MDBX_env; @@ -752,7 +699,7 @@ struct MDBX_env; /** \brief Opaque structure for a transaction handle. * \ingroup c_transactions - * \details All database operations require a transaction handle. Transactions + * \details All table operations require a transaction handle. Transactions * may be read-only or read-write. * \see mdbx_txn_begin() \see mdbx_txn_commit() \see mdbx_txn_abort() */ #ifndef __cplusplus @@ -761,16 +708,16 @@ typedef struct MDBX_txn MDBX_txn; struct MDBX_txn; #endif -/** \brief A handle for an individual database (key-value spaces) in the +/** \brief A handle for an individual table (key-value spaces) in the * environment. * \ingroup c_dbi - * \details Zero handle is used internally (hidden Garbage Collection subDB). + * \details Zero handle is used internally (hidden Garbage Collection table). * So, any valid DBI-handle great than 0 and less than or equal * \ref MDBX_MAX_DBI. * \see mdbx_dbi_open() \see mdbx_dbi_close() */ typedef uint32_t MDBX_dbi; -/** \brief Opaque structure for navigating through a database +/** \brief Opaque structure for navigating through a table * \ingroup c_cursors * \see mdbx_cursor_create() \see mdbx_cursor_bind() \see mdbx_cursor_close() */ @@ -781,15 +728,15 @@ struct MDBX_cursor; #endif /** \brief Generic structure used for passing keys and data in and out of the - * database. + * table. * \anchor MDBX_val \see mdbx::slice \see mdbx::buffer * - * \details Values returned from the database are valid only until a subsequent + * \details Values returned from the table are valid only until a subsequent * update operation, or the end of the transaction. Do not modify or * free them, they commonly point into the database itself. * * Key sizes must be between 0 and \ref mdbx_env_get_maxkeysize() inclusive. - * The same applies to data sizes in databases with the \ref MDBX_DUPSORT flag. + * The same applies to data sizes in tables with the \ref MDBX_DUPSORT flag. * Other data items can in theory be from 0 to \ref MDBX_MAXDATASIZE bytes long. * * \note The notable difference between MDBX and LMDB is that MDBX support zero @@ -817,7 +764,7 @@ typedef struct iovec MDBX_val; #endif /* ! SunOS */ enum MDBX_constants { - /** The hard limit for DBI handles */ + /** The hard limit for DBI handles. */ MDBX_MAX_DBI = UINT32_C(32765), /** The maximum size of a data item. */ @@ -888,7 +835,7 @@ enum MDBX_constants { /** Log level * \note Levels detailed than (great than) \ref MDBX_LOG_NOTICE * requires build libmdbx with \ref MDBX_DEBUG option. */ -enum MDBX_log_level_t { +typedef enum MDBX_log_level { /** Critical conditions, i.e. assertion failures. * \note libmdbx always produces such messages regardless * of \ref MDBX_DEBUG build option. */ @@ -938,17 +885,14 @@ enum MDBX_log_level_t { /** for \ref mdbx_setup_debug() only: Don't change current settings */ MDBX_LOG_DONTCHANGE = -1 -}; -#ifndef __cplusplus -typedef enum MDBX_log_level_t MDBX_log_level_t; -#endif +} MDBX_log_level_t; /** \brief Runtime debug flags * * \details `MDBX_DBG_DUMP` and `MDBX_DBG_LEGACY_MULTIOPEN` always have an * effect, but `MDBX_DBG_ASSERT`, `MDBX_DBG_AUDIT` and `MDBX_DBG_JITTER` only if * libmdbx built with \ref MDBX_DEBUG. */ -enum MDBX_debug_flags_t { +typedef enum MDBX_debug_flags { MDBX_DBG_NONE = 0, /** Enable assertion checks. @@ -980,18 +924,13 @@ enum MDBX_debug_flags_t { MDBX_DBG_DONT_UPGRADE = 64, #ifdef ENABLE_UBSAN - MDBX_DBG_MAX = ((unsigned)MDBX_LOG_MAX) << 16 | - 127 /* avoid UBSAN false-positive trap by a tests */, + MDBX_DBG_MAX = ((unsigned)MDBX_LOG_MAX) << 16 | 127 /* avoid UBSAN false-positive trap by a tests */, #endif /* ENABLE_UBSAN */ /** for mdbx_setup_debug() only: Don't change current settings */ MDBX_DBG_DONTCHANGE = -1 -}; -#ifndef __cplusplus -typedef enum MDBX_debug_flags_t MDBX_debug_flags_t; -#else -DEFINE_ENUM_FLAG_OPERATORS(MDBX_debug_flags_t) -#endif +} MDBX_debug_flags_t; +DEFINE_ENUM_FLAG_OPERATORS(MDBX_debug_flags) /** \brief A debug-logger callback function, * called before printing the message and aborting. @@ -1007,19 +946,23 @@ DEFINE_ENUM_FLAG_OPERATORS(MDBX_debug_flags_t) * format-message string passed by `fmt` argument. * Maybe NULL or invalid if the format-message string * don't contain `%`-specification of arguments. */ -typedef void MDBX_debug_func(MDBX_log_level_t loglevel, const char *function, - int line, const char *fmt, +typedef void MDBX_debug_func(MDBX_log_level_t loglevel, const char *function, int line, const char *fmt, va_list args) MDBX_CXX17_NOEXCEPT; /** \brief The "don't change `logger`" value for mdbx_setup_debug() */ #define MDBX_LOGGER_DONTCHANGE ((MDBX_debug_func *)(intptr_t)-1) +#define MDBX_LOGGER_NOFMT_DONTCHANGE ((MDBX_debug_func_nofmt *)(intptr_t)-1) /** \brief Setup global log-level, debug options and debug logger. * \returns The previously `debug_flags` in the 0-15 bits * and `log_level` in the 16-31 bits. */ -LIBMDBX_API int mdbx_setup_debug(MDBX_log_level_t log_level, - MDBX_debug_flags_t debug_flags, - MDBX_debug_func *logger); +LIBMDBX_API int mdbx_setup_debug(MDBX_log_level_t log_level, MDBX_debug_flags_t debug_flags, MDBX_debug_func *logger); + +typedef void MDBX_debug_func_nofmt(MDBX_log_level_t loglevel, const char *function, int line, const char *msg, + unsigned length) MDBX_CXX17_NOEXCEPT; + +LIBMDBX_API int mdbx_setup_debug_nofmt(MDBX_log_level_t log_level, MDBX_debug_flags_t debug_flags, + MDBX_debug_func_nofmt *logger, char *logger_buffer, size_t logger_buffer_size); /** \brief A callback function for most MDBX assert() failures, * called before printing the message and aborting. @@ -1031,8 +974,7 @@ LIBMDBX_API int mdbx_setup_debug(MDBX_log_level_t log_level, * may be NULL. * \param [in] line The line number in the source file * where the assertion check failed, may be zero. */ -typedef void MDBX_assert_func(const MDBX_env *env, const char *msg, - const char *function, +typedef void MDBX_assert_func(const MDBX_env *env, const char *msg, const char *function, unsigned line) MDBX_CXX17_NOEXCEPT; /** \brief Set or reset the assert() callback of the environment. @@ -1055,26 +997,21 @@ LIBMDBX_API int mdbx_env_set_assert(MDBX_env *env, MDBX_assert_func *func); * - NULL if given buffer size less than 4 bytes; * - pointer to constant string if given value NULL or empty; * - otherwise pointer to given buffer. */ -LIBMDBX_API const char *mdbx_dump_val(const MDBX_val *key, char *const buf, - const size_t bufsize); +LIBMDBX_API const char *mdbx_dump_val(const MDBX_val *key, char *const buf, const size_t bufsize); /** \brief Panics with message and causes abnormal process termination. */ -MDBX_NORETURN LIBMDBX_API void mdbx_panic(const char *fmt, ...) - MDBX_PRINTF_ARGS(1, 2); +MDBX_NORETURN LIBMDBX_API void mdbx_panic(const char *fmt, ...) MDBX_PRINTF_ARGS(1, 2); /** \brief Panics with asserton failed message and causes abnormal process * termination. */ -MDBX_NORETURN LIBMDBX_API void mdbx_assert_fail(const MDBX_env *env, - const char *msg, - const char *func, - unsigned line); +MDBX_NORETURN LIBMDBX_API void mdbx_assert_fail(const MDBX_env *env, const char *msg, const char *func, unsigned line); /** end of c_debug @} */ /** \brief Environment flags * \ingroup c_opening * \anchor env_flags * \see mdbx_env_open() \see mdbx_env_set_flags() */ -enum MDBX_env_flags_t { +typedef enum MDBX_env_flags { MDBX_ENV_DEFAULTS = 0, /** Extra validation of DB structure and pages content. @@ -1196,28 +1133,79 @@ enum MDBX_env_flags_t { */ MDBX_WRITEMAP = UINT32_C(0x80000), - /** Tie reader locktable slots to read-only transactions - * instead of to threads. + /** Отвязывает транзакции от потоков/threads насколько это возможно. * - * Don't use Thread-Local Storage, instead tie reader locktable slots to - * \ref MDBX_txn objects instead of to threads. So, \ref mdbx_txn_reset() - * keeps the slot reserved for the \ref MDBX_txn object. A thread may use - * parallel read-only transactions. And a read-only transaction may span - * threads if you synchronizes its use. + * Опция предназначена для приложений, которые мультиплексируют множество + * пользовательских легковесных потоков выполнения по отдельным потокам + * операционной системы, например как это происходит в средах выполнения + * GoLang и Rust. Таким приложениям также рекомендуется сериализовать + * транзакции записи в одном потоке операционной системы, поскольку блокировка + * записи MDBX использует базовые системные примитивы синхронизации и ничего + * не знает о пользовательских потоках и/или легковесных потоков среды + * выполнения. Как минимум, обязательно требуется обеспечить завершение каждой + * пишущей транзакции строго в том же потоке операционной системы где она была + * запущена. * - * Applications that multiplex many user threads over individual OS threads - * need this option. Such an application must also serialize the write - * transactions in an OS thread, since MDBX's write locking is unaware of - * the user threads. + * \note Начиная с версии v0.13 опция `MDBX_NOSTICKYTHREADS` полностью + * заменяет опцию \ref MDBX_NOTLS. * - * \note Regardless to `MDBX_NOTLS` flag a write transaction entirely should - * always be used in one thread from start to finish. MDBX checks this in a - * reasonable manner and return the \ref MDBX_THREAD_MISMATCH error in rules - * violation. + * При использовании `MDBX_NOSTICKYTHREADS` транзакции становятся не + * ассоциированными с создавшими их потоками выполнения. Поэтому в функциях + * API не выполняется проверка соответствия транзакции и текущего потока + * выполнения. Большинство функций работающих с транзакциями и курсорами + * становится возможным вызывать из любых потоков выполнения. Однако, также + * становится невозможно обнаружить ошибки одновременного использования + * транзакций и/или курсоров в разных потоках. * - * This flag affects only at environment opening but can't be changed after. + * Использование `MDBX_NOSTICKYTHREADS` также сужает возможности по изменению + * размера БД, так как теряется возможность отслеживать работающие с БД потоки + * выполнения и приостанавливать их на время снятия отображения БД в ОЗУ. В + * частности, по этой причине на Windows уменьшение файла БД не возможно до + * закрытия БД последним работающим с ней процессом или до последующего + * открытия БД в режиме чтения-записи. + * + * \warning Вне зависимости от \ref MDBX_NOSTICKYTHREADS и \ref MDBX_NOTLS не + * допускается одновременно использование объектов API из разных потоков + * выполнения! Обеспечение всех мер для исключения одновременного + * использования объектов API из разных потоков выполнения целиком ложится на + * вас! + * + * \warning Транзакции записи могут быть завершены только в том же потоке + * выполнения где они были запущены. Это ограничение следует из требований + * большинства операционных систем о том, что захваченный примитив + * синхронизации (мьютекс, семафор, критическая секция) должен освобождаться + * только захватившим его потоком выполнения. + * + * \warning Создание курсора в контексте транзакции, привязка курсора к + * транзакции, отвязка курсора от транзакции и закрытие привязанного к + * транзакции курсора, являются операциями использующими как сам курсор так и + * соответствующую транзакцию. Аналогично, завершение или прерывание + * транзакции является операцией использующей как саму транзакцию, так и все + * привязанные к ней курсоры. Во избежание повреждения внутренних структур + * данных, непредсказуемого поведения, разрушение БД и потери данных следует + * не допускать возможности одновременного использования каких-либо курсора + * или транзакций из разных потоков выполнения. + * + * Читающие транзакции при использовании `MDBX_NOSTICKYTHREADS` перестают + * использовать TLS (Thread Local Storage), а слоты блокировок MVCC-снимков в + * таблице читателей привязываются только к транзакциям. Завершение каких-либо + * потоков не приводит к снятию блокировок MVCC-снимков до явного завершения + * транзакций, либо до завершения соответствующего процесса в целом. + * + * Для пишущих транзакций не выполняется проверка соответствия текущего потока + * выполнения и потока создавшего транзакцию. Однако, фиксация или прерывание + * пишущих транзакций должны выполняться строго в потоке запустившим + * транзакцию, так как эти операции связаны с захватом и освобождением + * примитивов синхронизации (мьютексов, критических секций), для которых + * большинство операционных систем требует освобождение только потоком + * захватившим ресурс. + * + * Этот флаг вступает в силу при открытии среды и не может быть изменен после. */ - MDBX_NOTLS = UINT32_C(0x200000), + MDBX_NOSTICKYTHREADS = UINT32_C(0x200000), + + /** \deprecated Please use \ref MDBX_NOSTICKYTHREADS instead. */ + MDBX_NOTLS MDBX_DEPRECATED_ENUM = MDBX_NOSTICKYTHREADS, /** Don't do readahead. * @@ -1264,7 +1252,7 @@ enum MDBX_env_flags_t { MDBX_NOMEMINIT = UINT32_C(0x1000000), /** Aims to coalesce a Garbage Collection items. - * \note Always enabled since v0.12 + * \deprecated Always enabled since v0.12 and deprecated since v0.13. * * With `MDBX_COALESCE` flag MDBX will aims to coalesce items while recycling * a Garbage Collection. Technically, when possible short lists of pages @@ -1274,7 +1262,7 @@ enum MDBX_env_flags_t { * Unallocated space and reducing the database file. * * This flag may be changed at any time using mdbx_env_set_flags(). */ - MDBX_COALESCE = UINT32_C(0x2000000), + MDBX_COALESCE MDBX_DEPRECATED_ENUM = UINT32_C(0x2000000), /** LIFO policy for recycling a Garbage Collection items. * @@ -1414,7 +1402,7 @@ enum MDBX_env_flags_t { * \ref mdbx_env_set_syncbytes() and \ref mdbx_env_set_syncperiod() functions * could be very useful with `MDBX_SAFE_NOSYNC` flag. * - * The number and volume of of disk IOPs with MDBX_SAFE_NOSYNC flag will + * The number and volume of disk IOPs with MDBX_SAFE_NOSYNC flag will * exactly the as without any no-sync flags. However, you should expect a * larger process's [work set](https://bit.ly/2kA2tFX) and significantly worse * a [locality of reference](https://bit.ly/2mbYq2J), due to the more @@ -1477,19 +1465,14 @@ enum MDBX_env_flags_t { MDBX_UTTERLY_NOSYNC = MDBX_SAFE_NOSYNC | UINT32_C(0x100000), /** end of sync_modes @} */ -}; -#ifndef __cplusplus -/** \ingroup c_opening */ -typedef enum MDBX_env_flags_t MDBX_env_flags_t; -#else -DEFINE_ENUM_FLAG_OPERATORS(MDBX_env_flags_t) -#endif +} MDBX_env_flags_t; +DEFINE_ENUM_FLAG_OPERATORS(MDBX_env_flags) /** Transaction flags * \ingroup c_transactions * \anchor txn_flags * \see mdbx_txn_begin() \see mdbx_txn_flags() */ -enum MDBX_txn_flags_t { +typedef enum MDBX_txn_flags { /** Start read-write transaction. * * Only one write transaction may be active at a time. Writes are fully @@ -1533,46 +1516,61 @@ enum MDBX_txn_flags_t { MDBX_TXN_INVALID = INT32_MIN, /** Transaction is finished or never began. - * \note Transaction state flag. Returned from \ref mdbx_txn_flags() + * \note This is a transaction state flag. Returned from \ref mdbx_txn_flags() * but can't be used with \ref mdbx_txn_begin(). */ MDBX_TXN_FINISHED = 0x01, /** Transaction is unusable after an error. - * \note Transaction state flag. Returned from \ref mdbx_txn_flags() + * \note This is a transaction state flag. Returned from \ref mdbx_txn_flags() * but can't be used with \ref mdbx_txn_begin(). */ MDBX_TXN_ERROR = 0x02, /** Transaction must write, even if dirty list is empty. - * \note Transaction state flag. Returned from \ref mdbx_txn_flags() + * \note This is a transaction state flag. Returned from \ref mdbx_txn_flags() * but can't be used with \ref mdbx_txn_begin(). */ MDBX_TXN_DIRTY = 0x04, /** Transaction or a parent has spilled pages. - * \note Transaction state flag. Returned from \ref mdbx_txn_flags() + * \note This is a transaction state flag. Returned from \ref mdbx_txn_flags() * but can't be used with \ref mdbx_txn_begin(). */ MDBX_TXN_SPILLS = 0x08, /** Transaction has a nested child transaction. - * \note Transaction state flag. Returned from \ref mdbx_txn_flags() + * \note This is a transaction state flag. Returned from \ref mdbx_txn_flags() * but can't be used with \ref mdbx_txn_begin(). */ MDBX_TXN_HAS_CHILD = 0x10, + /** Transaction is parked by \ref mdbx_txn_park(). + * \note This is a transaction state flag. Returned from \ref mdbx_txn_flags() + * but can't be used with \ref mdbx_txn_begin(). */ + MDBX_TXN_PARKED = 0x20, + + /** Transaction is parked by \ref mdbx_txn_park() with `autounpark=true`, + * and therefore it can be used without explicitly calling + * \ref mdbx_txn_unpark() first. + * \note This is a transaction state flag. Returned from \ref mdbx_txn_flags() + * but can't be used with \ref mdbx_txn_begin(). */ + MDBX_TXN_AUTOUNPARK = 0x40, + + /** The transaction was blocked using the \ref mdbx_txn_park() function, + * and then ousted by a write transaction because + * this transaction was interfered with garbage recycling. + * \note This is a transaction state flag. Returned from \ref mdbx_txn_flags() + * but can't be used with \ref mdbx_txn_begin(). */ + MDBX_TXN_OUSTED = 0x80, + /** Most operations on the transaction are currently illegal. - * \note Transaction state flag. Returned from \ref mdbx_txn_flags() + * \note This is a transaction state flag. Returned from \ref mdbx_txn_flags() * but can't be used with \ref mdbx_txn_begin(). */ - MDBX_TXN_BLOCKED = MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_HAS_CHILD -}; -#ifndef __cplusplus -typedef enum MDBX_txn_flags_t MDBX_txn_flags_t; -#else -DEFINE_ENUM_FLAG_OPERATORS(MDBX_txn_flags_t) -#endif + MDBX_TXN_BLOCKED = MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_HAS_CHILD | MDBX_TXN_PARKED +} MDBX_txn_flags_t; +DEFINE_ENUM_FLAG_OPERATORS(MDBX_txn_flags) -/** \brief Database flags +/** \brief Table flags * \ingroup c_dbi * \anchor db_flags * \see mdbx_dbi_open() */ -enum MDBX_db_flags_t { +typedef enum MDBX_db_flags { /** Variable length unique keys with usual byte-by-byte string comparison. */ MDBX_DB_DEFAULTS = 0, @@ -1604,37 +1602,32 @@ enum MDBX_db_flags_t { /** Create DB if not already existing. */ MDBX_CREATE = UINT32_C(0x40000), - /** Opens an existing sub-database created with unknown flags. + /** Opens an existing table created with unknown flags. * - * The `MDBX_DB_ACCEDE` flag is intend to open a existing sub-database which + * The `MDBX_DB_ACCEDE` flag is intend to open a existing table which * was created with unknown flags (\ref MDBX_REVERSEKEY, \ref MDBX_DUPSORT, * \ref MDBX_INTEGERKEY, \ref MDBX_DUPFIXED, \ref MDBX_INTEGERDUP and * \ref MDBX_REVERSEDUP). * * In such cases, instead of returning the \ref MDBX_INCOMPATIBLE error, the - * sub-database will be opened with flags which it was created, and then an + * table will be opened with flags which it was created, and then an * application could determine the actual flags by \ref mdbx_dbi_flags(). */ MDBX_DB_ACCEDE = MDBX_ACCEDE -}; -#ifndef __cplusplus -/** \ingroup c_dbi */ -typedef enum MDBX_db_flags_t MDBX_db_flags_t; -#else -DEFINE_ENUM_FLAG_OPERATORS(MDBX_db_flags_t) -#endif +} MDBX_db_flags_t; +DEFINE_ENUM_FLAG_OPERATORS(MDBX_db_flags) /** \brief Data changing flags * \ingroup c_crud * \see \ref c_crud_hints "Quick reference for Insert/Update/Delete operations" * \see mdbx_put() \see mdbx_cursor_put() \see mdbx_replace() */ -enum MDBX_put_flags_t { +typedef enum MDBX_put_flags { /** Upsertion by default (without any other flags) */ MDBX_UPSERT = 0, /** For insertion: Don't write if the key already exists. */ MDBX_NOOVERWRITE = UINT32_C(0x10), - /** Has effect only for \ref MDBX_DUPSORT databases. + /** Has effect only for \ref MDBX_DUPSORT tables. * For upsertion: don't write if the key-value pair already exist. */ MDBX_NODUPDATA = UINT32_C(0x20), @@ -1644,7 +1637,7 @@ enum MDBX_put_flags_t { * For deletion: remove only single entry at the current cursor position. */ MDBX_CURRENT = UINT32_C(0x40), - /** Has effect only for \ref MDBX_DUPSORT databases. + /** Has effect only for \ref MDBX_DUPSORT tables. * For deletion: remove all multi-values (aka duplicates) for given key. * For upsertion: replace all multi-values for given key with a new one. */ MDBX_ALLDUPS = UINT32_C(0x80), @@ -1657,7 +1650,7 @@ enum MDBX_put_flags_t { * Don't split full pages, continue on a new instead. */ MDBX_APPEND = UINT32_C(0x20000), - /** Has effect only for \ref MDBX_DUPSORT databases. + /** Has effect only for \ref MDBX_DUPSORT tables. * Duplicate data is being appended. * Don't split full pages, continue on a new instead. */ MDBX_APPENDDUP = UINT32_C(0x40000), @@ -1665,18 +1658,13 @@ enum MDBX_put_flags_t { /** Only for \ref MDBX_DUPFIXED. * Store multiple data items in one call. */ MDBX_MULTIPLE = UINT32_C(0x80000) -}; -#ifndef __cplusplus -/** \ingroup c_crud */ -typedef enum MDBX_put_flags_t MDBX_put_flags_t; -#else -DEFINE_ENUM_FLAG_OPERATORS(MDBX_put_flags_t) -#endif +} MDBX_put_flags_t; +DEFINE_ENUM_FLAG_OPERATORS(MDBX_put_flags) /** \brief Environment copy flags * \ingroup c_extra * \see mdbx_env_copy() \see mdbx_env_copy2fd() */ -enum MDBX_copy_flags_t { +typedef enum MDBX_copy_flags { MDBX_CP_DEFAULTS = 0, /** Copy with compactification: Omit free space from copy and renumber all @@ -1684,20 +1672,32 @@ enum MDBX_copy_flags_t { MDBX_CP_COMPACT = 1u, /** Force to make resizable copy, i.e. dynamic size instead of fixed */ - MDBX_CP_FORCE_DYNAMIC_SIZE = 2u -}; -#ifndef __cplusplus -/** \ingroup c_extra */ -typedef enum MDBX_copy_flags_t MDBX_copy_flags_t; -#else -DEFINE_ENUM_FLAG_OPERATORS(MDBX_copy_flags_t) -#endif + MDBX_CP_FORCE_DYNAMIC_SIZE = 2u, + + /** Don't explicitly flush the written data to an output media */ + MDBX_CP_DONT_FLUSH = 4u, + + /** Use read transaction parking during copying MVCC-snapshot + * \see mdbx_txn_park() */ + MDBX_CP_THROTTLE_MVCC = 8u, + + /** Abort/dispose passed transaction after copy + * \see mdbx_txn_copy2fd() \see mdbx_txn_copy2pathname() */ + MDBX_CP_DISPOSE_TXN = 16u, + + /** Enable renew/restart read transaction in case it use outdated + * MVCC shapshot, otherwise the \ref MDBX_MVCC_RETARDED will be returned + * \see mdbx_txn_copy2fd() \see mdbx_txn_copy2pathname() */ + MDBX_CP_RENEW_TXN = 32u + +} MDBX_copy_flags_t; +DEFINE_ENUM_FLAG_OPERATORS(MDBX_copy_flags) /** \brief Cursor operations * \ingroup c_cursors * This is the set of all operations for retrieving data using a cursor. * \see mdbx_cursor_get() */ -enum MDBX_cursor_op { +typedef enum MDBX_cursor_op { /** Position at first key/data item */ MDBX_FIRST, @@ -1716,7 +1716,7 @@ enum MDBX_cursor_op { /** \ref MDBX_DUPFIXED -only: Return up to a page of duplicate data items * from current cursor position. Move cursor to prepare - * for \ref MDBX_NEXT_MULTIPLE. */ + * for \ref MDBX_NEXT_MULTIPLE. \see MDBX_SEEK_AND_GET_MULTIPLE */ MDBX_GET_MULTIPLE, /** Position at last key/data item */ @@ -1732,8 +1732,8 @@ enum MDBX_cursor_op { MDBX_NEXT_DUP, /** \ref MDBX_DUPFIXED -only: Return up to a page of duplicate data items - * from next cursor position. Move cursor to prepare - * for `MDBX_NEXT_MULTIPLE`. */ + * from next cursor position. Move cursor to prepare for `MDBX_NEXT_MULTIPLE`. + * \see MDBX_SEEK_AND_GET_MULTIPLE \see MDBX_GET_MULTIPLE */ MDBX_NEXT_MULTIPLE, /** Position at first data item of next key */ @@ -1758,7 +1758,8 @@ enum MDBX_cursor_op { MDBX_SET_RANGE, /** \ref MDBX_DUPFIXED -only: Position at previous page and return up to - * a page of duplicate data items. */ + * a page of duplicate data items. + * \see MDBX_SEEK_AND_GET_MULTIPLE \see MDBX_GET_MULTIPLE */ MDBX_PREV_MULTIPLE, /** Positions cursor at first key-value pair greater than or equal to @@ -1779,7 +1780,7 @@ enum MDBX_cursor_op { * return both key and data, and the return code depends on whether a * upper-bound was found. * - * For non DUPSORT-ed collections this work the same to \ref MDBX_SET_RANGE, + * For non DUPSORT-ed collections this work like \ref MDBX_SET_RANGE, * but returns \ref MDBX_SUCCESS if the greater key was found or * \ref MDBX_NOTFOUND otherwise. * @@ -1787,19 +1788,43 @@ enum MDBX_cursor_op { * i.e. for a pairs/tuples of a key and an each data value of duplicates. * Returns \ref MDBX_SUCCESS if the greater pair was returned or * \ref MDBX_NOTFOUND otherwise. */ - MDBX_SET_UPPERBOUND -}; -#ifndef __cplusplus -/** \ingroup c_cursors */ -typedef enum MDBX_cursor_op MDBX_cursor_op; -#endif + MDBX_SET_UPPERBOUND, + + /** Doubtless cursor positioning at a specified key. */ + MDBX_TO_KEY_LESSER_THAN, + MDBX_TO_KEY_LESSER_OR_EQUAL /** \copydoc MDBX_TO_KEY_LESSER_THAN */, + MDBX_TO_KEY_EQUAL /** \copydoc MDBX_TO_KEY_LESSER_THAN */, + MDBX_TO_KEY_GREATER_OR_EQUAL /** \copydoc MDBX_TO_KEY_LESSER_THAN */, + MDBX_TO_KEY_GREATER_THAN /** \copydoc MDBX_TO_KEY_LESSER_THAN */, + + /** Doubtless cursor positioning at a specified key-value pair + * for dupsort/multi-value hives. */ + MDBX_TO_EXACT_KEY_VALUE_LESSER_THAN, + MDBX_TO_EXACT_KEY_VALUE_LESSER_OR_EQUAL /** \copydoc MDBX_TO_EXACT_KEY_VALUE_LESSER_THAN */, + MDBX_TO_EXACT_KEY_VALUE_EQUAL /** \copydoc MDBX_TO_EXACT_KEY_VALUE_LESSER_THAN */, + MDBX_TO_EXACT_KEY_VALUE_GREATER_OR_EQUAL /** \copydoc MDBX_TO_EXACT_KEY_VALUE_LESSER_THAN */, + MDBX_TO_EXACT_KEY_VALUE_GREATER_THAN /** \copydoc MDBX_TO_EXACT_KEY_VALUE_LESSER_THAN */, + + /** Doubtless cursor positioning at a specified key-value pair + * for dupsort/multi-value hives. */ + MDBX_TO_PAIR_LESSER_THAN, + MDBX_TO_PAIR_LESSER_OR_EQUAL /** \copydoc MDBX_TO_PAIR_LESSER_THAN */, + MDBX_TO_PAIR_EQUAL /** \copydoc MDBX_TO_PAIR_LESSER_THAN */, + MDBX_TO_PAIR_GREATER_OR_EQUAL /** \copydoc MDBX_TO_PAIR_LESSER_THAN */, + MDBX_TO_PAIR_GREATER_THAN /** \copydoc MDBX_TO_PAIR_LESSER_THAN */, + + /** \ref MDBX_DUPFIXED -only: Seek to given key and return up to a page of + * duplicate data items from current cursor position. Move cursor to prepare + * for \ref MDBX_NEXT_MULTIPLE. \see MDBX_GET_MULTIPLE */ + MDBX_SEEK_AND_GET_MULTIPLE +} MDBX_cursor_op; /** \brief Errors and return codes * \ingroup c_err * * BerkeleyDB uses -30800 to -30999, we'll go under them * \see mdbx_strerror() \see mdbx_strerror_r() \see mdbx_liberr2str() */ -enum MDBX_error_t { +typedef enum MDBX_error { /** Successful result */ MDBX_SUCCESS = 0, @@ -1862,14 +1887,14 @@ enum MDBX_error_t { * or explicit call of \ref mdbx_env_set_geometry(). */ MDBX_UNABLE_EXTEND_MAPSIZE = -30785, - /** Environment or database is not compatible with the requested operation + /** Environment or table is not compatible with the requested operation * or the specified flags. This can mean: * - The operation expects an \ref MDBX_DUPSORT / \ref MDBX_DUPFIXED - * database. + * table. * - Opening a named DB when the unnamed DB has \ref MDBX_DUPSORT / * \ref MDBX_INTEGERKEY. - * - Accessing a data record as a database, or vice versa. - * - The database was dropped and recreated with different flags. */ + * - Accessing a data record as a named table, or vice versa. + * - The table was dropped and recreated with different flags. */ MDBX_INCOMPATIBLE = -30784, /** Invalid reuse of reader locktable slot, @@ -1881,8 +1906,8 @@ enum MDBX_error_t { * or is invalid */ MDBX_BAD_TXN = -30782, - /** Invalid size or alignment of key or data for target database, - * either invalid subDB name */ + /** Invalid size or alignment of key or data for target table, + * either invalid table name */ MDBX_BAD_VALSIZE = -30781, /** The specified DBI-handle is invalid @@ -1922,7 +1947,7 @@ enum MDBX_error_t { MDBX_TOO_LARGE = -30417, /** A thread has attempted to use a not owned object, - * e.g. a transaction that started by another thread. */ + * e.g. a transaction that started by another thread */ MDBX_THREAD_MISMATCH = -30416, /** Overlapping read and write transactions for the current thread */ @@ -1937,8 +1962,19 @@ enum MDBX_error_t { /** Alternative/Duplicate LCK-file is exists and should be removed manually */ MDBX_DUPLICATED_CLK = -30413, + /** Some cursors and/or other resources should be closed before table or + * corresponding DBI-handle could be (re)used and/or closed. */ + MDBX_DANGLING_DBI = -30412, + + /** The parked read transaction was outed for the sake of + * recycling old MVCC snapshots. */ + MDBX_OUSTED = -30411, + + /** MVCC snapshot used by parked transaction was bygone. */ + MDBX_MVCC_RETARDED = -30410, + /* The last of MDBX-added error codes */ - MDBX_LAST_ADDED_ERRCODE = MDBX_DUPLICATED_CLK, + MDBX_LAST_ADDED_ERRCODE = MDBX_MVCC_RETARDED, #if defined(_WIN32) || defined(_WIN64) MDBX_ENODATA = ERROR_HANDLE_EOF, @@ -1951,7 +1987,8 @@ enum MDBX_error_t { MDBX_EPERM = ERROR_INVALID_FUNCTION, MDBX_EINTR = ERROR_CANCELLED, MDBX_ENOFILE = ERROR_FILE_NOT_FOUND, - MDBX_EREMOTE = ERROR_REMOTE_STORAGE_MEDIA_ERROR + MDBX_EREMOTE = ERROR_REMOTE_STORAGE_MEDIA_ERROR, + MDBX_EDEADLK = ERROR_POSSIBLE_DEADLOCK #else /* Windows */ #ifdef ENODATA MDBX_ENODATA = ENODATA, @@ -1967,21 +2004,21 @@ enum MDBX_error_t { MDBX_EPERM = EPERM, MDBX_EINTR = EINTR, MDBX_ENOFILE = ENOENT, - MDBX_EREMOTE = ENOTBLK +#if defined(EREMOTEIO) || defined(DOXYGEN) + /** Cannot use the database on a network file system or when exporting it via NFS. */ + MDBX_EREMOTE = EREMOTEIO, +#else + MDBX_EREMOTE = ENOTBLK, +#endif /* EREMOTEIO */ + MDBX_EDEADLK = EDEADLK #endif /* !Windows */ -}; -#ifndef __cplusplus -/** \ingroup c_err */ -typedef enum MDBX_error_t MDBX_error_t; -#endif +} MDBX_error_t; /** MDBX_MAP_RESIZED * \ingroup c_err * \deprecated Please review your code to use MDBX_UNABLE_EXTEND_MAPSIZE * instead. */ -MDBX_DEPRECATED static __inline int MDBX_MAP_RESIZED_is_deprecated(void) { - return MDBX_UNABLE_EXTEND_MAPSIZE; -} +MDBX_DEPRECATED static __inline int MDBX_MAP_RESIZED_is_deprecated(void) { return MDBX_UNABLE_EXTEND_MAPSIZE; } #define MDBX_MAP_RESIZED MDBX_MAP_RESIZED_is_deprecated() /** \brief Return a string describing a given error code. @@ -2042,8 +2079,7 @@ LIBMDBX_API const char *mdbx_strerror_ANSI2OEM(int errnum); * Windows error-messages in the OEM-encoding for console utilities. * \ingroup c_err * \see mdbx_strerror_ANSI2OEM() */ -LIBMDBX_API const char *mdbx_strerror_r_ANSI2OEM(int errnum, char *buf, - size_t buflen); +LIBMDBX_API const char *mdbx_strerror_r_ANSI2OEM(int errnum, char *buf, size_t buflen); #endif /* Bit of Windows' madness */ /** \brief Create an MDBX environment instance. @@ -2065,12 +2101,12 @@ LIBMDBX_API int mdbx_env_create(MDBX_env **penv); /** \brief MDBX environment extra runtime options. * \ingroup c_settings * \see mdbx_env_set_option() \see mdbx_env_get_option() */ -enum MDBX_option_t { - /** \brief Controls the maximum number of named databases for the environment. +typedef enum MDBX_option { + /** \brief Controls the maximum number of named tables for the environment. * - * \details By default only unnamed key-value database could used and + * \details By default only unnamed key-value table could used and * appropriate value should set by `MDBX_opt_max_db` to using any more named - * subDB(s). To reduce overhead, use the minimum sufficient value. This option + * table(s). To reduce overhead, use the minimum sufficient value. This option * may only set after \ref mdbx_env_create() and before \ref mdbx_env_open(). * * \see mdbx_env_set_maxdbs() \see mdbx_env_get_maxdbs() */ @@ -2080,14 +2116,15 @@ enum MDBX_option_t { * for all processes interacting with the database. * * \details This defines the number of slots in the lock table that is used to - * track readers in the the environment. The default is about 100 for 4K + * track readers in the environment. The default is about 100 for 4K * system page size. Starting a read-only transaction normally ties a lock * table slot to the current thread until the environment closes or the thread - * exits. If \ref MDBX_NOTLS is in use, \ref mdbx_txn_begin() instead ties the - * slot to the \ref MDBX_txn object until it or the \ref MDBX_env object is - * destroyed. This option may only set after \ref mdbx_env_create() and before - * \ref mdbx_env_open(), and has an effect only when the database is opened by - * the first process interacts with the database. + * exits. If \ref MDBX_NOSTICKYTHREADS is in use, \ref mdbx_txn_begin() + * instead ties the slot to the \ref MDBX_txn object until it or the \ref + * MDBX_env object is destroyed. This option may only set after \ref + * mdbx_env_create() and before \ref mdbx_env_open(), and has an effect only + * when the database is opened by the first process interacts with the + * database. * * \see mdbx_env_set_maxreaders() \see mdbx_env_get_maxreaders() */ MDBX_opt_max_readers, @@ -2107,6 +2144,7 @@ enum MDBX_option_t { /** \brief Controls the in-process limit to grow a list of reclaimed/recycled * page's numbers for finding a sequence of contiguous pages for large data * items. + * \see MDBX_opt_gc_time_limit * * \details A long values requires allocation of contiguous database pages. * To find such sequences, it may be necessary to accumulate very large lists, @@ -2226,14 +2264,15 @@ enum MDBX_option_t { MDBX_opt_spill_parent4child_denominator, /** \brief Controls the in-process threshold of semi-empty pages merge. - * \warning This is experimental option and subject for change or removal. * \details This option controls the in-process threshold of minimum page * fill, as used space of percentage of a page. Neighbour pages emptier than * this value are candidates for merging. The threshold value is specified - * in 1/65536 of percent, which is equivalent to the 16-dot-16 fixed point - * format. The specified value must be in the range from 12.5% (almost empty) - * to 50% (half empty) which corresponds to the range from 8192 and to 32768 - * in units respectively. */ + * in 1/65536 points of a whole page, which is equivalent to the 16-dot-16 + * fixed point format. + * The specified value must be in the range from 12.5% (almost empty page) + * to 50% (half empty page) which corresponds to the range from 8192 and + * to 32768 in units respectively. + * \see MDBX_opt_prefer_waf_insteadof_balance */ MDBX_opt_merge_threshold_16dot16_percent, /** \brief Controls the choosing between use write-through disk writes and @@ -2268,11 +2307,101 @@ enum MDBX_option_t { * in the \ref MDBX_WRITEMAP mode by clearing ones through file handle before * touching. */ MDBX_opt_prefault_write_enable, -}; -#ifndef __cplusplus -/** \ingroup c_settings */ -typedef enum MDBX_option_t MDBX_option_t; -#endif + + /** \brief Controls the in-process spending time limit of searching + * consecutive pages inside GC. + * \see MDBX_opt_rp_augment_limit + * + * \details Задаёт ограничение времени в 1/65536 долях секунды, которое может + * быть потрачено в ходе пишущей транзакции на поиск последовательностей + * страниц внутри GC/freelist после достижения ограничения задаваемого опцией + * \ref MDBX_opt_rp_augment_limit. Контроль по времени не выполняется при + * поиске/выделении одиночных страниц и выделении страниц под нужды GC (при + * обновлении GC в ходе фиксации транзакции). + * + * Задаваемый лимит времени исчисляется по "настенным часам" и контролируется + * в рамках транзакции, наследуется для вложенных транзакций и с + * аккумулированием в родительской при их фиксации. Контроль по времени + * производится только при достижении ограничения задаваемого опцией \ref + * MDBX_opt_rp_augment_limit. Это позволяет гибко управлять поведением + * используя обе опции. + * + * По умолчанию ограничение устанавливается в 0, что приводит к + * незамедлительной остановке поиска в GC при достижении \ref + * MDBX_opt_rp_augment_limit во внутреннем состоянии транзакции и + * соответствует поведению до появления опции `MDBX_opt_gc_time_limit`. + * С другой стороны, при минимальном значении (включая 0) + * `MDBX_opt_rp_augment_limit` переработка GC будет ограничиваться + * преимущественно затраченным временем. */ + MDBX_opt_gc_time_limit, + + /** \brief Управляет выбором между стремлением к равномерности наполнения + * страниц, либо уменьшением количества измененных и записанных страниц. + * + * \details После операций удаления страницы содержащие меньше минимума + * ключей, либо опустошенные до \ref MDBX_opt_merge_threshold_16dot16_percent + * подлежат слиянию с одной из соседних. Если страницы справа и слева от + * текущей обе «грязные» (были изменены в ходе транзакции и должны быть + * записаны на диск), либо обе «чисты» (не изменялись в текущей транзакции), + * то целью для слияния всегда выбирается менее заполненная страница. + * Когда же только одна из соседствующих является «грязной», а другая + * «чистой», то возможны две тактики выбора цели для слияния: + * + * - Если `MDBX_opt_prefer_waf_insteadof_balance = True`, то будет выбрана + * уже измененная страница, что НЕ УВЕЛИЧИТ количество измененных страниц + * и объем записи на диск при фиксации текущей транзакции, но в среднем + * будет УВЕЛИЧИВАТЬ неравномерность заполнения страниц. + * + * - Если `MDBX_opt_prefer_waf_insteadof_balance = False`, то будет выбрана + * менее заполненная страница, что УВЕЛИЧИТ количество измененных страниц + * и объем записи на диск при фиксации текущей транзакции, но в среднем + * будет УМЕНЬШАТЬ неравномерность заполнения страниц. + * + * \see MDBX_opt_merge_threshold_16dot16_percent */ + MDBX_opt_prefer_waf_insteadof_balance, + + /** \brief Задаёт в % максимальный размер вложенных страниц, используемых для + * размещения небольшого количества мульти-значений связанных с одном ключем. + * + * Использование вложенных страниц, вместо выноса значений на отдельные + * страницы вложенного дерева, позволяет уменьшить объем неиспользуемого места + * и этим увеличить плотность размещения данных. + * + * Но с увеличением размера вложенных страниц требуется больше листовых + * страниц основного дерева, что также увеличивает высоту основного дерева. + * Кроме этого, изменение данных на вложенных страницах требует дополнительных + * копирований, поэтому стоимость может быть больше во многих сценариях. + * + * min 12.5% (8192), max 100% (65535), default = 100% */ + MDBX_opt_subpage_limit, + + /** \brief Задаёт в % минимальный объём свободного места на основной странице, + * при отсутствии которого вложенные страницы выносятся в отдельное дерево. + * + * min 0, max 100% (65535), default = 0 */ + MDBX_opt_subpage_room_threshold, + + /** \brief Задаёт в % минимальный объём свободного места на основной странице, + * при наличии которого, производится резервирование места во вложенной. + * + * Если на основной странице свободного места недостаточно, то вложенная + * страница будет минимального размера. В свою очередь, при отсутствии резерва + * во вложенной странице, каждое добавлении в неё элементов будет требовать + * переформирования основной страниц с переносом всех узлов данных. + * + * Поэтому резервирование места, как правило, выгодно в сценариях с + * интенсивным добавлением коротких мульти-значений, например при + * индексировании. Но уменьшает плотность размещения данных, соответственно + * увеличивает объем БД и операций ввода-вывода. + * + * min 0, max 100% (65535), default = 42% (27525) */ + MDBX_opt_subpage_reserve_prereq, + + /** \brief Задаёт в % ограничение резервирования места на вложенных страницах. + * + * min 0, max 100% (65535), default = 4.2% (2753) */ + MDBX_opt_subpage_reserve_limit +} MDBX_option_t; /** \brief Sets the value of a extra runtime options for an environment. * \ingroup c_settings @@ -2284,8 +2413,7 @@ typedef enum MDBX_option_t MDBX_option_t; * \see MDBX_option_t * \see mdbx_env_get_option() * \returns A non-zero error value on failure and 0 on success. */ -LIBMDBX_API int mdbx_env_set_option(MDBX_env *env, const MDBX_option_t option, - uint64_t value); +LIBMDBX_API int mdbx_env_set_option(MDBX_env *env, const MDBX_option_t option, uint64_t value); /** \brief Gets the value of extra runtime options from an environment. * \ingroup c_settings @@ -2297,9 +2425,7 @@ LIBMDBX_API int mdbx_env_set_option(MDBX_env *env, const MDBX_option_t option, * \see MDBX_option_t * \see mdbx_env_get_option() * \returns A non-zero error value on failure and 0 on success. */ -LIBMDBX_API int mdbx_env_get_option(const MDBX_env *env, - const MDBX_option_t option, - uint64_t *pvalue); +LIBMDBX_API int mdbx_env_get_option(const MDBX_env *env, const MDBX_option_t option, uint64_t *pvalue); /** \brief Open an environment instance. * \ingroup c_opening @@ -2324,7 +2450,7 @@ LIBMDBX_API int mdbx_env_get_option(const MDBX_env *env, * * Flags set by mdbx_env_set_flags() are also used: * - \ref MDBX_ENV_DEFAULTS, \ref MDBX_NOSUBDIR, \ref MDBX_RDONLY, - * \ref MDBX_EXCLUSIVE, \ref MDBX_WRITEMAP, \ref MDBX_NOTLS, + * \ref MDBX_EXCLUSIVE, \ref MDBX_WRITEMAP, \ref MDBX_NOSTICKYTHREADS, * \ref MDBX_NORDAHEAD, \ref MDBX_NOMEMINIT, \ref MDBX_COALESCE, * \ref MDBX_LIFORECLAIM. See \ref env_flags section. * @@ -2373,21 +2499,22 @@ LIBMDBX_API int mdbx_env_get_option(const MDBX_env *env, * \retval MDBX_TOO_LARGE Database is too large for this process, * i.e. 32-bit process tries to open >4Gb database. */ -LIBMDBX_API int mdbx_env_open(MDBX_env *env, const char *pathname, - MDBX_env_flags_t flags, mdbx_mode_t mode); +LIBMDBX_API int mdbx_env_open(MDBX_env *env, const char *pathname, MDBX_env_flags_t flags, mdbx_mode_t mode); #if defined(_WIN32) || defined(_WIN64) || defined(DOXYGEN) /** \copydoc mdbx_env_open() * \note Available only on Windows. * \see mdbx_env_open() */ -LIBMDBX_API int mdbx_env_openW(MDBX_env *env, const wchar_t *pathname, - MDBX_env_flags_t flags, mdbx_mode_t mode); +LIBMDBX_API int mdbx_env_openW(MDBX_env *env, const wchar_t *pathname, MDBX_env_flags_t flags, mdbx_mode_t mode); +#define mdbx_env_openT(env, pathname, flags, mode) mdbx_env_openW(env, pathname, flags, mode) +#else +#define mdbx_env_openT(env, pathname, flags, mode) mdbx_env_open(env, pathname, flags, mode) #endif /* Windows */ /** \brief Deletion modes for \ref mdbx_env_delete(). * \ingroup c_extra * \see mdbx_env_delete() */ -enum MDBX_env_delete_mode_t { +typedef enum MDBX_env_delete_mode { /** \brief Just delete the environment's files and directory if any. * \note On POSIX systems, processes already working with the database will * continue to work without interference until it close the environment. @@ -2401,11 +2528,7 @@ enum MDBX_env_delete_mode_t { /** \brief Wait until other processes closes the environment before deletion. */ MDBX_ENV_WAIT_FOR_UNUSED = 2, -}; -#ifndef __cplusplus -/** \ingroup c_extra */ -typedef enum MDBX_env_delete_mode_t MDBX_env_delete_mode_t; -#endif +} MDBX_env_delete_mode_t; /** \brief Delete the environment's files in a proper and multiprocess-safe way. * \ingroup c_extra @@ -2426,15 +2549,17 @@ typedef enum MDBX_env_delete_mode_t MDBX_env_delete_mode_t; * some possible errors are: * \retval MDBX_RESULT_TRUE No corresponding files or directories were found, * so no deletion was performed. */ -LIBMDBX_API int mdbx_env_delete(const char *pathname, - MDBX_env_delete_mode_t mode); +LIBMDBX_API int mdbx_env_delete(const char *pathname, MDBX_env_delete_mode_t mode); #if defined(_WIN32) || defined(_WIN64) || defined(DOXYGEN) /** \copydoc mdbx_env_delete() + * \ingroup c_extra * \note Available only on Windows. * \see mdbx_env_delete() */ -LIBMDBX_API int mdbx_env_deleteW(const wchar_t *pathname, - MDBX_env_delete_mode_t mode); +LIBMDBX_API int mdbx_env_deleteW(const wchar_t *pathname, MDBX_env_delete_mode_t mode); +#define mdbx_env_deleteT(pathname, mode) mdbx_env_deleteW(pathname, mode) +#else +#define mdbx_env_deleteT(pathname, mode) mdbx_env_delete(pathname, mode) #endif /* Windows */ /** \brief Copy an MDBX environment to the specified path, with options. @@ -2447,6 +2572,8 @@ LIBMDBX_API int mdbx_env_deleteW(const wchar_t *pathname, * transaction. See long-lived transactions under \ref restrictions section. * * \note On Windows the \ref mdbx_env_copyW() is recommended to use. + * \see mdbx_env_copy2fd() + * \see mdbx_txn_copy2pathname() * * \param [in] env An environment handle returned by mdbx_env_create(). * It must have already been opened successfully. @@ -2469,16 +2596,99 @@ LIBMDBX_API int mdbx_env_deleteW(const wchar_t *pathname, * - \ref MDBX_CP_FORCE_DYNAMIC_SIZE * Force to make resizable copy, i.e. dynamic size instead of fixed. * + * - \ref MDBX_CP_DONT_FLUSH + * Don't explicitly flush the written data to an output media to reduce + * the time of the operation and the duration of the transaction. + * + * - \ref MDBX_CP_THROTTLE_MVCC + * Use read transaction parking during copying MVCC-snapshot + * to avoid stopping recycling and overflowing the database. + * This allows the writing transaction to oust the read + * transaction used to copy the database if copying takes so long + * that it will interfere with the recycling old MVCC snapshots + * and may lead to an overflow of the database. + * However, if the reading transaction is ousted the copy will + * be aborted until successful completion. Thus, this option + * allows copy the database without interfering with write + * transactions and a threat of database overflow, but at the cost + * that copying will be aborted to prevent such conditions. + * \see mdbx_txn_park() + * * \returns A non-zero error value on failure and 0 on success. */ -LIBMDBX_API int mdbx_env_copy(MDBX_env *env, const char *dest, - MDBX_copy_flags_t flags); +LIBMDBX_API int mdbx_env_copy(MDBX_env *env, const char *dest, MDBX_copy_flags_t flags); + +/** \brief Copy an MDBX environment by given read transaction to the specified + * path, with options. + * \ingroup c_extra + * + * This function may be used to make a backup of an existing environment. + * No lockfile is created, since it gets recreated at need. + * \note This call can trigger significant file size growth if run in + * parallel with write transactions, because it employs a read-only + * transaction. See long-lived transactions under \ref restrictions section. + * + * \note On Windows the \ref mdbx_txn_copy2pathnameW() is recommended to use. + * \see mdbx_txn_copy2fd() + * \see mdbx_env_copy() + * + * \param [in] txn A transaction handle returned by \ref mdbx_txn_begin(). + * \param [in] dest The pathname of a file in which the copy will reside. + * This file must not be already exist, but parent directory + * must be writable. + * \param [in] flags Specifies options for this operation. This parameter + * must be bitwise OR'ing together any of the constants + * described here: + * + * - \ref MDBX_CP_DEFAULTS + * Perform copy as-is without compaction, etc. + * + * - \ref MDBX_CP_COMPACT + * Perform compaction while copying: omit free pages and sequentially + * renumber all pages in output. This option consumes little bit more + * CPU for processing, but may running quickly than the default, on + * account skipping free pages. + * + * - \ref MDBX_CP_FORCE_DYNAMIC_SIZE + * Force to make resizable copy, i.e. dynamic size instead of fixed. + * + * - \ref MDBX_CP_DONT_FLUSH + * Don't explicitly flush the written data to an output media to reduce + * the time of the operation and the duration of the transaction. + * + * - \ref MDBX_CP_THROTTLE_MVCC + * Use read transaction parking during copying MVCC-snapshot + * to avoid stopping recycling and overflowing the database. + * This allows the writing transaction to oust the read + * transaction used to copy the database if copying takes so long + * that it will interfere with the recycling old MVCC snapshots + * and may lead to an overflow of the database. + * However, if the reading transaction is ousted the copy will + * be aborted until successful completion. Thus, this option + * allows copy the database without interfering with write + * transactions and a threat of database overflow, but at the cost + * that copying will be aborted to prevent such conditions. + * \see mdbx_txn_park() + * + * \returns A non-zero error value on failure and 0 on success. */ +LIBMDBX_API int mdbx_txn_copy2pathname(MDBX_txn *txn, const char *dest, MDBX_copy_flags_t flags); #if defined(_WIN32) || defined(_WIN64) || defined(DOXYGEN) /** \copydoc mdbx_env_copy() + * \ingroup c_extra * \note Available only on Windows. * \see mdbx_env_copy() */ -LIBMDBX_API int mdbx_env_copyW(MDBX_env *env, const wchar_t *dest, - MDBX_copy_flags_t flags); +LIBMDBX_API int mdbx_env_copyW(MDBX_env *env, const wchar_t *dest, MDBX_copy_flags_t flags); +#define mdbx_env_copyT(env, dest, flags) mdbx_env_copyW(env, dest, flags) + +/** \copydoc mdbx_txn_copy2pathname() + * \ingroup c_extra + * \note Available only on Windows. + * \see mdbx_txn_copy2pathname() */ +LIBMDBX_API int mdbx_txn_copy2pathnameW(MDBX_txn *txn, const wchar_t *dest, MDBX_copy_flags_t flags); +#define mdbx_txn_copy2pathnameT(txn, dest, flags) mdbx_txn_copy2pathnameW(txn, dest, path) +#else +#define mdbx_env_copyT(env, dest, flags) mdbx_env_copy(env, dest, flags) +#define mdbx_txn_copy2pathnameT(txn, dest, flags) mdbx_txn_copy2pathname(txn, dest, path) #endif /* Windows */ /** \brief Copy an environment to the specified file descriptor, with @@ -2488,6 +2698,7 @@ LIBMDBX_API int mdbx_env_copyW(MDBX_env *env, const wchar_t *dest, * This function may be used to make a backup of an existing environment. * No lockfile is created, since it gets recreated at need. * \see mdbx_env_copy() + * \see mdbx_txn_copy2fd() * * \note This call can trigger significant file size growth if run in * parallel with write transactions, because it employs a read-only @@ -2504,21 +2715,45 @@ LIBMDBX_API int mdbx_env_copyW(MDBX_env *env, const wchar_t *dest, * \param [in] flags Special options for this operation. \see mdbx_env_copy() * * \returns A non-zero error value on failure and 0 on success. */ -LIBMDBX_API int mdbx_env_copy2fd(MDBX_env *env, mdbx_filehandle_t fd, - MDBX_copy_flags_t flags); +LIBMDBX_API int mdbx_env_copy2fd(MDBX_env *env, mdbx_filehandle_t fd, MDBX_copy_flags_t flags); -/** \brief Statistics for a database in the environment +/** \brief Copy an environment by given read transaction to the specified file + * descriptor, with options. + * \ingroup c_extra + * + * This function may be used to make a backup of an existing environment. + * No lockfile is created, since it gets recreated at need. + * \see mdbx_txn_copy2pathname() + * \see mdbx_env_copy2fd() + * + * \note This call can trigger significant file size growth if run in + * parallel with write transactions, because it employs a read-only + * transaction. See long-lived transactions under \ref restrictions + * section. + * + * \note Fails if the environment has suffered a page leak and the destination + * file descriptor is associated with a pipe, socket, or FIFO. + * + * \param [in] txn A transaction handle returned by \ref mdbx_txn_begin(). + * \param [in] fd The file descriptor to write the copy to. It must have + * already been opened for Write access. + * \param [in] flags Special options for this operation. \see mdbx_env_copy() + * + * \returns A non-zero error value on failure and 0 on success. */ +LIBMDBX_API int mdbx_txn_copy2fd(MDBX_txn *txn, mdbx_filehandle_t fd, MDBX_copy_flags_t flags); + +/** \brief Statistics for a table in the environment * \ingroup c_statinfo * \see mdbx_env_stat_ex() \see mdbx_dbi_stat() */ struct MDBX_stat { - uint32_t ms_psize; /**< Size of a database page. This is the same for all - databases. */ - uint32_t ms_depth; /**< Depth (height) of the B-tree */ + uint32_t ms_psize; /**< Size of a table page. This is the same for all tables + in a database. */ + uint32_t ms_depth; /**< Depth (height) of the B-tree */ uint64_t ms_branch_pages; /**< Number of internal (non-leaf) pages */ uint64_t ms_leaf_pages; /**< Number of leaf pages */ - uint64_t ms_overflow_pages; /**< Number of overflow pages */ + uint64_t ms_overflow_pages; /**< Number of large/overflow pages */ uint64_t ms_entries; /**< Number of data items */ - uint64_t ms_mod_txnid; /**< Transaction ID of committed last modification */ + uint64_t ms_mod_txnid; /**< Transaction ID of committed last modification */ }; #ifndef __cplusplus /** \ingroup c_statinfo */ @@ -2544,15 +2779,12 @@ typedef struct MDBX_stat MDBX_stat; * \param [in] bytes The size of \ref MDBX_stat. * * \returns A non-zero error value on failure and 0 on success. */ -LIBMDBX_API int mdbx_env_stat_ex(const MDBX_env *env, const MDBX_txn *txn, - MDBX_stat *stat, size_t bytes); +LIBMDBX_API int mdbx_env_stat_ex(const MDBX_env *env, const MDBX_txn *txn, MDBX_stat *stat, size_t bytes); /** \brief Return statistics about the MDBX environment. * \ingroup c_statinfo * \deprecated Please use mdbx_env_stat_ex() instead. */ -MDBX_DEPRECATED LIBMDBX_INLINE_API(int, mdbx_env_stat, - (const MDBX_env *env, MDBX_stat *stat, - size_t bytes)) { +MDBX_DEPRECATED LIBMDBX_INLINE_API(int, mdbx_env_stat, (const MDBX_env *env, MDBX_stat *stat, size_t bytes)) { return mdbx_env_stat_ex(env, NULL, stat, bytes); } @@ -2567,15 +2799,13 @@ struct MDBX_envinfo { uint64_t shrink; /**< Shrink threshold for datafile */ uint64_t grow; /**< Growth step for datafile */ } mi_geo; - uint64_t mi_mapsize; /**< Size of the data memory map */ - uint64_t mi_last_pgno; /**< Number of the last used page */ - uint64_t mi_recent_txnid; /**< ID of the last committed transaction */ - uint64_t mi_latter_reader_txnid; /**< ID of the last reader transaction */ + uint64_t mi_mapsize; /**< Size of the data memory map */ + uint64_t mi_last_pgno; /**< Number of the last used page */ + uint64_t mi_recent_txnid; /**< ID of the last committed transaction */ + uint64_t mi_latter_reader_txnid; /**< ID of the last reader transaction */ uint64_t mi_self_latter_reader_txnid; /**< ID of the last reader transaction of caller process */ - uint64_t mi_meta0_txnid, mi_meta0_sign; - uint64_t mi_meta1_txnid, mi_meta1_sign; - uint64_t mi_meta2_txnid, mi_meta2_sign; + uint64_t mi_meta_txnid[3], mi_meta_sign[3]; uint32_t mi_maxreaders; /**< Total reader slots in the environment */ uint32_t mi_numreaders; /**< Max reader slots used in the environment */ uint32_t mi_dxb_pagesize; /**< Database pagesize */ @@ -2592,7 +2822,7 @@ struct MDBX_envinfo { struct { struct { uint64_t x, y; - } current, meta0, meta1, meta2; + } current, meta[3]; } mi_bootid; /** Bytes not explicitly synchronized to disk */ @@ -2631,11 +2861,14 @@ struct MDBX_envinfo { to a disk */ uint64_t prefault; /**< Number of prefault write operations (not a pages) */ uint64_t mincore; /**< Number of mincore() calls */ - uint64_t - msync; /**< Number of explicit msync-to-disk operations (not a pages) */ - uint64_t - fsync; /**< Number of explicit fsync-to-disk operations (not a pages) */ + uint64_t msync; /**< Number of explicit msync-to-disk operations (not a pages) */ + uint64_t fsync; /**< Number of explicit fsync-to-disk operations (not a pages) */ } mi_pgop_stat; + + /* GUID of the database DXB file. */ + struct { + uint64_t x, y; + } mi_dxbid; }; #ifndef __cplusplus /** \ingroup c_statinfo */ @@ -2658,17 +2891,15 @@ typedef struct MDBX_envinfo MDBX_envinfo; * \param [in] txn A transaction handle returned by \ref mdbx_txn_begin() * \param [out] info The address of an \ref MDBX_envinfo structure * where the information will be copied - * \param [in] bytes The size of \ref MDBX_envinfo. + * \param [in] bytes The actual size of \ref MDBX_envinfo, + * this value is used to provide ABI compatibility. * * \returns A non-zero error value on failure and 0 on success. */ -LIBMDBX_API int mdbx_env_info_ex(const MDBX_env *env, const MDBX_txn *txn, - MDBX_envinfo *info, size_t bytes); +LIBMDBX_API int mdbx_env_info_ex(const MDBX_env *env, const MDBX_txn *txn, MDBX_envinfo *info, size_t bytes); /** \brief Return information about the MDBX environment. * \ingroup c_statinfo * \deprecated Please use mdbx_env_info_ex() instead. */ -MDBX_DEPRECATED LIBMDBX_INLINE_API(int, mdbx_env_info, - (const MDBX_env *env, MDBX_envinfo *info, - size_t bytes)) { +MDBX_DEPRECATED LIBMDBX_INLINE_API(int, mdbx_env_info, (const MDBX_env *env, MDBX_envinfo *info, size_t bytes)) { return mdbx_env_info_ex(env, NULL, info, bytes); } @@ -2713,16 +2944,12 @@ LIBMDBX_API int mdbx_env_sync_ex(MDBX_env *env, bool force, bool nonblock); /** \brief The shortcut to calling \ref mdbx_env_sync_ex() with * the `force=true` and `nonblock=false` arguments. * \ingroup c_extra */ -LIBMDBX_INLINE_API(int, mdbx_env_sync, (MDBX_env * env)) { - return mdbx_env_sync_ex(env, true, false); -} +LIBMDBX_INLINE_API(int, mdbx_env_sync, (MDBX_env * env)) { return mdbx_env_sync_ex(env, true, false); } /** \brief The shortcut to calling \ref mdbx_env_sync_ex() with * the `force=false` and `nonblock=true` arguments. * \ingroup c_extra */ -LIBMDBX_INLINE_API(int, mdbx_env_sync_poll, (MDBX_env * env)) { - return mdbx_env_sync_ex(env, false, true); -} +LIBMDBX_INLINE_API(int, mdbx_env_sync_poll, (MDBX_env * env)) { return mdbx_env_sync_ex(env, false, true); } /** \brief Sets threshold to force flush the data buffers to disk, even any of * \ref MDBX_SAFE_NOSYNC flag in the environment. @@ -2747,8 +2974,7 @@ LIBMDBX_INLINE_API(int, mdbx_env_sync_poll, (MDBX_env * env)) { * a synchronous flush would be made. * * \returns A non-zero error value on failure and 0 on success. */ -LIBMDBX_INLINE_API(int, mdbx_env_set_syncbytes, - (MDBX_env * env, size_t threshold)) { +LIBMDBX_INLINE_API(int, mdbx_env_set_syncbytes, (MDBX_env * env, size_t threshold)) { return mdbx_env_set_option(env, MDBX_opt_sync_bytes, threshold); } @@ -2766,8 +2992,7 @@ LIBMDBX_INLINE_API(int, mdbx_env_set_syncbytes, * \returns A non-zero error value on failure and 0 on success, * some possible errors are: * \retval MDBX_EINVAL An invalid parameter was specified. */ -LIBMDBX_INLINE_API(int, mdbx_env_get_syncbytes, - (const MDBX_env *env, size_t *threshold)) { +LIBMDBX_INLINE_API(int, mdbx_env_get_syncbytes, (const MDBX_env *env, size_t *threshold)) { int rc = MDBX_EINVAL; if (threshold) { uint64_t proxy = 0; @@ -2810,8 +3035,7 @@ LIBMDBX_INLINE_API(int, mdbx_env_get_syncbytes, * the last unsteady commit. * * \returns A non-zero error value on failure and 0 on success. */ -LIBMDBX_INLINE_API(int, mdbx_env_set_syncperiod, - (MDBX_env * env, unsigned seconds_16dot16)) { +LIBMDBX_INLINE_API(int, mdbx_env_set_syncperiod, (MDBX_env * env, unsigned seconds_16dot16)) { return mdbx_env_set_option(env, MDBX_opt_sync_period, seconds_16dot16); } @@ -2831,8 +3055,7 @@ LIBMDBX_INLINE_API(int, mdbx_env_set_syncperiod, * \returns A non-zero error value on failure and 0 on success, * some possible errors are: * \retval MDBX_EINVAL An invalid parameter was specified. */ -LIBMDBX_INLINE_API(int, mdbx_env_get_syncperiod, - (const MDBX_env *env, unsigned *period_seconds_16dot16)) { +LIBMDBX_INLINE_API(int, mdbx_env_get_syncperiod, (const MDBX_env *env, unsigned *period_seconds_16dot16)) { int rc = MDBX_EINVAL; if (period_seconds_16dot16) { uint64_t proxy = 0; @@ -2848,7 +3071,7 @@ LIBMDBX_INLINE_API(int, mdbx_env_get_syncperiod, /** \brief Close the environment and release the memory map. * \ingroup c_opening * - * Only a single thread may call this function. All transactions, databases, + * Only a single thread may call this function. All transactions, tables, * and cursors must already be closed before calling this function. Attempts * to use any such handles after calling this function is UB and would cause * a `SIGSEGV`. The environment handle will be freed and must not be used again @@ -2889,15 +3112,93 @@ LIBMDBX_API int mdbx_env_close_ex(MDBX_env *env, bool dont_sync); /** \brief The shortcut to calling \ref mdbx_env_close_ex() with * the `dont_sync=false` argument. * \ingroup c_opening */ -LIBMDBX_INLINE_API(int, mdbx_env_close, (MDBX_env * env)) { - return mdbx_env_close_ex(env, false); -} +LIBMDBX_INLINE_API(int, mdbx_env_close, (MDBX_env * env)) { return mdbx_env_close_ex(env, false); } + +#if defined(DOXYGEN) || !(defined(_WIN32) || defined(_WIN64)) +/** \brief Восстанавливает экземпляр среды в дочернем процессе после ветвления + * родительского процесса посредством `fork()` и родственных системных вызовов. + * \ingroup c_extra + * + * Без вызова \ref mdbx_env_resurrect_after_fork() использование открытого + * экземпляра среды в дочернем процессе не возможно, включая все выполняющиеся + * на момент ветвления транзакции. + * + * Выполняемые функцией действия можно рассматривать как повторное открытие БД + * в дочернем процессе, с сохранением заданных опций и адресов уже созданных + * экземпляров объектов связанных с API. + * + * \note Функция не доступна в ОС семейства Windows по причине отсутствия + * функционала ветвления процесса в API операционной системы. + * + * Ветвление не оказывает влияния на состояние MDBX-среды в родительском + * процессе. Все транзакции, которые были в родительском процессе на момент + * ветвления, после ветвления в родительском процессе продолжат выполняться без + * помех. Но в дочернем процессе все соответствующие транзакции безальтернативно + * перестают быть валидными, а попытка их использования приведет к возврату + * ошибки или отправке `SIGSEGV`. + * + * Использование экземпляра среды в дочернем процессе не возможно до вызова + * \ref mdbx_env_resurrect_after_fork(), так как в результате ветвления у + * процесса меняется PID, значение которого используется для организации + * совместно работы с БД, в том числе, для отслеживания процессов/потоков + * выполняющих читающие транзакции связанные с соответствующими снимками данных. + * Все активные на момент ветвления транзакции не могут продолжаться в дочернем + * процессе, так как не владеют какими-либо блокировками или каким-либо снимком + * данных и не удерживает его от переработки при сборке мусора. + * + * Функция \ref mdbx_env_resurrect_after_fork() восстанавливает переданный + * экземпляр среды в дочернем процессе после ветвления, а именно: обновляет + * используемые системные идентификаторы, повторно открывает дескрипторы файлов, + * производит захват необходимых блокировок связанных с LCK- и DXB-файлами БД, + * восстанавливает отображения в память страницы БД, таблицы читателей и + * служебных/вспомогательных данных в память. Однако унаследованные от + * родительского процесса транзакции не восстанавливаются, прием пишущие и + * читающие транзакции обрабатываются по-разному: + * + * - Пишущая транзакция, если таковая была на момент ветвления, + * прерывается в дочернем процессе с освобождение связанных с ней ресурсов, + * включая все вложенные транзакции. + * + * - Читающие же транзакции, если таковые были в родительском процессе, + * в дочернем процессе логически прерываются, но без освобождения ресурсов. + * Поэтому необходимо обеспечить вызов \ref mdbx_txn_abort() для каждой + * такой читающей транзакций в дочернем процессе, либо смириться с утечкой + * ресурсов до завершения дочернего процесса. + * + * Причина не-освобождения ресурсов читающих транзакций в том, что исторически + * MDBX не ведет какой-либо общий список экземпляров читающих, так как это не + * требуется для штатных режимов работы, но требует использования атомарных + * операций или дополнительных объектов синхронизации при создании/разрушении + * экземпляров \ref MDBX_txn. + * + * Вызов \ref mdbx_env_resurrect_after_fork() без ветвления, не в дочернем + * процессе, либо повторные вызовы не приводят к каким-либо действиям или + * изменениям. + * + * \param [in,out] env Экземпляр среды созданный функцией + * \ref mdbx_env_create(). + * + * \returns Ненулевое значение кода ошибки, либо 0 при успешном выполнении. + * Некоторые возможные ошибки таковы: + * + * \retval MDBX_BUSY В родительском процессе БД была открыта + * в режиме \ref MDBX_EXCLUSIVE. + * + * \retval MDBX_EBADSIGN При повреждении сигнатуры экземпляра объекта, а также + * в случае одновременного вызова \ref + * mdbx_env_resurrect_after_fork() из разных потоков. + * + * \retval MDBX_PANIC Произошла критическая ошибка при восстановлении + * экземпляра среды, либо такая ошибка уже была + * до вызова функции. */ +LIBMDBX_API int mdbx_env_resurrect_after_fork(MDBX_env *env); +#endif /* Windows */ /** \brief Warming up options * \ingroup c_settings * \anchor warmup_flags * \see mdbx_env_warmup() */ -enum MDBX_warmup_flags_t { +typedef enum MDBX_warmup_flags { /** By default \ref mdbx_env_warmup() just ask OS kernel to asynchronously * prefetch database pages. */ MDBX_warmup_default = 0, @@ -2927,7 +3228,7 @@ enum MDBX_warmup_flags_t { * On successful, all currently allocated pages, both unused in GC and * containing payload, will be locked in memory until the environment closes, * or explicitly unblocked by using \ref MDBX_warmup_release, or the - * database geomenry will changed, including its auto-shrinking. */ + * database geometry will changed, including its auto-shrinking. */ MDBX_warmup_lock = 4, /** Alters corresponding current resource limits to be enough for lock pages @@ -2940,15 +3241,12 @@ enum MDBX_warmup_flags_t { /** Release the lock that was performed before by \ref MDBX_warmup_lock. */ MDBX_warmup_release = 16, -}; -#ifndef __cplusplus -typedef enum MDBX_warmup_flags_t MDBX_warmup_flags_t; -#else -DEFINE_ENUM_FLAG_OPERATORS(MDBX_warmup_flags_t) -#endif +} MDBX_warmup_flags_t; +DEFINE_ENUM_FLAG_OPERATORS(MDBX_warmup_flags) -/** \brief Warms up the database by loading pages into memory, optionally lock - * ones. \ingroup c_settings +/** \brief Warms up the database by loading pages into memory, + * optionally lock ones. + * \ingroup c_settings * * Depending on the specified flags, notifies OS kernel about following access, * force loads the database pages, including locks ones in memory or releases @@ -2977,8 +3275,7 @@ DEFINE_ENUM_FLAG_OPERATORS(MDBX_warmup_flags_t) * * \retval MDBX_RESULT_TRUE The specified timeout is reached during load * data into memory. */ -LIBMDBX_API int mdbx_env_warmup(const MDBX_env *env, const MDBX_txn *txn, - MDBX_warmup_flags_t flags, +LIBMDBX_API int mdbx_env_warmup(const MDBX_env *env, const MDBX_txn *txn, MDBX_warmup_flags_t flags, unsigned timeout_seconds_16dot16); /** \brief Set environment flags. @@ -3001,8 +3298,7 @@ LIBMDBX_API int mdbx_env_warmup(const MDBX_env *env, const MDBX_txn *txn, * \returns A non-zero error value on failure and 0 on success, * some possible errors are: * \retval MDBX_EINVAL An invalid parameter was specified. */ -LIBMDBX_API int mdbx_env_set_flags(MDBX_env *env, MDBX_env_flags_t flags, - bool onoff); +LIBMDBX_API int mdbx_env_set_flags(MDBX_env *env, MDBX_env_flags_t flags, bool onoff); /** \brief Get environment flags. * \ingroup c_statinfo @@ -3033,9 +3329,13 @@ LIBMDBX_API int mdbx_env_get_path(const MDBX_env *env, const char **dest); #if defined(_WIN32) || defined(_WIN64) || defined(DOXYGEN) /** \copydoc mdbx_env_get_path() + * \ingroup c_statinfo * \note Available only on Windows. * \see mdbx_env_get_path() */ LIBMDBX_API int mdbx_env_get_pathW(const MDBX_env *env, const wchar_t **dest); +#define mdbx_env_get_pathT(env, dest) mdbx_env_get_pathW(env, dest) +#else +#define mdbx_env_get_pathT(env, dest) mdbx_env_get_path(env, dest) #endif /* Windows */ /** \brief Return the file descriptor for the given environment. @@ -3241,23 +3541,19 @@ LIBMDBX_API int mdbx_env_get_fd(const MDBX_env *env, mdbx_filehandle_t *fd); * 2) Temporary close memory mapped is required to change * geometry, but there read transaction(s) is running * and no corresponding thread(s) could be suspended - * since the \ref MDBX_NOTLS mode is used. + * since the \ref MDBX_NOSTICKYTHREADS mode is used. * \retval MDBX_EACCESS The environment opened in read-only. * \retval MDBX_MAP_FULL Specified size smaller than the space already * consumed by the environment. * \retval MDBX_TOO_LARGE Specified size is too large, i.e. too many pages for * given size, or a 32-bit process requests too much * bytes for the 32-bit address space. */ -LIBMDBX_API int mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower, - intptr_t size_now, intptr_t size_upper, - intptr_t growth_step, - intptr_t shrink_threshold, - intptr_t pagesize); +LIBMDBX_API int mdbx_env_set_geometry(MDBX_env *env, intptr_t size_lower, intptr_t size_now, intptr_t size_upper, + intptr_t growth_step, intptr_t shrink_threshold, intptr_t pagesize); /** \deprecated Please use \ref mdbx_env_set_geometry() instead. * \ingroup c_settings */ -MDBX_DEPRECATED LIBMDBX_INLINE_API(int, mdbx_env_set_mapsize, - (MDBX_env * env, size_t size)) { +MDBX_DEPRECATED LIBMDBX_INLINE_API(int, mdbx_env_set_mapsize, (MDBX_env * env, size_t size)) { return mdbx_env_set_geometry(env, size, size, size, -1, -1, -1); } @@ -3270,89 +3566,86 @@ MDBX_DEPRECATED LIBMDBX_INLINE_API(int, mdbx_env_set_mapsize, * value. * * \returns A \ref MDBX_RESULT_TRUE or \ref MDBX_RESULT_FALSE value, - * otherwise the error code: + * otherwise the error code. * \retval MDBX_RESULT_TRUE Readahead is reasonable. * \retval MDBX_RESULT_FALSE Readahead is NOT reasonable, * i.e. \ref MDBX_NORDAHEAD is useful to * open environment by \ref mdbx_env_open(). * \retval Otherwise the error code. */ -LIBMDBX_API int mdbx_is_readahead_reasonable(size_t volume, - intptr_t redundancy); +LIBMDBX_API int mdbx_is_readahead_reasonable(size_t volume, intptr_t redundancy); /** \brief Returns the minimal database page size in bytes. * \ingroup c_statinfo */ -MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_INLINE_API(intptr_t, mdbx_limits_pgsize_min, - (void)) { - return MDBX_MIN_PAGESIZE; -} +MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_INLINE_API(intptr_t, mdbx_limits_pgsize_min, (void)) { return MDBX_MIN_PAGESIZE; } /** \brief Returns the maximal database page size in bytes. * \ingroup c_statinfo */ -MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_INLINE_API(intptr_t, mdbx_limits_pgsize_max, - (void)) { - return MDBX_MAX_PAGESIZE; -} +MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_INLINE_API(intptr_t, mdbx_limits_pgsize_max, (void)) { return MDBX_MAX_PAGESIZE; } /** \brief Returns minimal database size in bytes for given page size, * or -1 if pagesize is invalid. * \ingroup c_statinfo */ -MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API intptr_t -mdbx_limits_dbsize_min(intptr_t pagesize); +MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API intptr_t mdbx_limits_dbsize_min(intptr_t pagesize); /** \brief Returns maximal database size in bytes for given page size, * or -1 if pagesize is invalid. * \ingroup c_statinfo */ -MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API intptr_t -mdbx_limits_dbsize_max(intptr_t pagesize); +MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API intptr_t mdbx_limits_dbsize_max(intptr_t pagesize); /** \brief Returns maximal key size in bytes for given page size - * and database flags, or -1 if pagesize is invalid. + * and table flags, or -1 if pagesize is invalid. * \ingroup c_statinfo * \see db_flags */ -MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API intptr_t -mdbx_limits_keysize_max(intptr_t pagesize, MDBX_db_flags_t flags); +MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API intptr_t mdbx_limits_keysize_max(intptr_t pagesize, MDBX_db_flags_t flags); + +/** \brief Returns minimal key size in bytes for given table flags. + * \ingroup c_statinfo + * \see db_flags */ +MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API intptr_t mdbx_limits_keysize_min(MDBX_db_flags_t flags); /** \brief Returns maximal data size in bytes for given page size - * and database flags, or -1 if pagesize is invalid. + * and table flags, or -1 if pagesize is invalid. * \ingroup c_statinfo * \see db_flags */ -MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API intptr_t -mdbx_limits_valsize_max(intptr_t pagesize, MDBX_db_flags_t flags); +MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API intptr_t mdbx_limits_valsize_max(intptr_t pagesize, MDBX_db_flags_t flags); + +/** \brief Returns minimal data size in bytes for given table flags. + * \ingroup c_statinfo + * \see db_flags */ +MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API intptr_t mdbx_limits_valsize_min(MDBX_db_flags_t flags); /** \brief Returns maximal size of key-value pair to fit in a single page with - * the given size and database flags, or -1 if pagesize is invalid. + * the given size and table flags, or -1 if pagesize is invalid. * \ingroup c_statinfo * \see db_flags */ -MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API intptr_t -mdbx_limits_pairsize4page_max(intptr_t pagesize, MDBX_db_flags_t flags); +MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API intptr_t mdbx_limits_pairsize4page_max(intptr_t pagesize, + MDBX_db_flags_t flags); /** \brief Returns maximal data size in bytes to fit in a leaf-page or - * single overflow/large-page with the given page size and database flags, + * single large/overflow-page with the given page size and table flags, * or -1 if pagesize is invalid. * \ingroup c_statinfo * \see db_flags */ -MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API intptr_t -mdbx_limits_valsize4page_max(intptr_t pagesize, MDBX_db_flags_t flags); +MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API intptr_t mdbx_limits_valsize4page_max(intptr_t pagesize, MDBX_db_flags_t flags); /** \brief Returns maximal write transaction size (i.e. limit for summary volume * of dirty pages) in bytes for given page size, or -1 if pagesize is invalid. * \ingroup c_statinfo */ -MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API intptr_t -mdbx_limits_txnsize_max(intptr_t pagesize); +MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API intptr_t mdbx_limits_txnsize_max(intptr_t pagesize); /** \brief Set the maximum number of threads/reader slots for for all processes * interacts with the database. * \ingroup c_settings * * \details This defines the number of slots in the lock table that is used to - * track readers in the the environment. The default is about 100 for 4K system + * track readers in the environment. The default is about 100 for 4K system * page size. Starting a read-only transaction normally ties a lock table slot * to the current thread until the environment closes or the thread exits. If - * \ref MDBX_NOTLS is in use, \ref mdbx_txn_begin() instead ties the slot to the - * \ref MDBX_txn object until it or the \ref MDBX_env object is destroyed. - * This function may only be called after \ref mdbx_env_create() and before - * \ref mdbx_env_open(), and has an effect only when the database is opened by - * the first process interacts with the database. + * \ref MDBX_NOSTICKYTHREADS is in use, \ref mdbx_txn_begin() instead ties the + * slot to the \ref MDBX_txn object until it or the \ref MDBX_env object is + * destroyed. This function may only be called after \ref mdbx_env_create() and + * before \ref mdbx_env_open(), and has an effect only when the database is + * opened by the first process interacts with the database. * \see mdbx_env_get_maxreaders() * * \param [in] env An environment handle returned @@ -3363,8 +3656,7 @@ mdbx_limits_txnsize_max(intptr_t pagesize); * some possible errors are: * \retval MDBX_EINVAL An invalid parameter was specified. * \retval MDBX_EPERM The environment is already open. */ -LIBMDBX_INLINE_API(int, mdbx_env_set_maxreaders, - (MDBX_env * env, unsigned readers)) { +LIBMDBX_INLINE_API(int, mdbx_env_set_maxreaders, (MDBX_env * env, unsigned readers)) { return mdbx_env_set_option(env, MDBX_opt_max_readers, readers); } @@ -3379,8 +3671,7 @@ LIBMDBX_INLINE_API(int, mdbx_env_set_maxreaders, * \returns A non-zero error value on failure and 0 on success, * some possible errors are: * \retval MDBX_EINVAL An invalid parameter was specified. */ -LIBMDBX_INLINE_API(int, mdbx_env_get_maxreaders, - (const MDBX_env *env, unsigned *readers)) { +LIBMDBX_INLINE_API(int, mdbx_env_get_maxreaders, (const MDBX_env *env, unsigned *readers)) { int rc = MDBX_EINVAL; if (readers) { uint64_t proxy = 0; @@ -3390,12 +3681,12 @@ LIBMDBX_INLINE_API(int, mdbx_env_get_maxreaders, return rc; } -/** \brief Set the maximum number of named databases for the environment. +/** \brief Set the maximum number of named tables for the environment. * \ingroup c_settings * - * This function is only needed if multiple databases will be used in the + * This function is only needed if multiple tables will be used in the * environment. Simpler applications that use the environment as a single - * unnamed database can ignore this option. + * unnamed table can ignore this option. * This function may only be called after \ref mdbx_env_create() and before * \ref mdbx_env_open(). * @@ -3405,7 +3696,7 @@ LIBMDBX_INLINE_API(int, mdbx_env_get_maxreaders, * \see mdbx_env_get_maxdbs() * * \param [in] env An environment handle returned by \ref mdbx_env_create(). - * \param [in] dbs The maximum number of databases. + * \param [in] dbs The maximum number of tables. * * \returns A non-zero error value on failure and 0 on success, * some possible errors are: @@ -3415,18 +3706,17 @@ LIBMDBX_INLINE_API(int, mdbx_env_set_maxdbs, (MDBX_env * env, MDBX_dbi dbs)) { return mdbx_env_set_option(env, MDBX_opt_max_db, dbs); } -/** \brief Get the maximum number of named databases for the environment. +/** \brief Get the maximum number of named tables for the environment. * \ingroup c_statinfo * \see mdbx_env_set_maxdbs() * * \param [in] env An environment handle returned by \ref mdbx_env_create(). - * \param [out] dbs Address to store the maximum number of databases. + * \param [out] dbs Address to store the maximum number of tables. * * \returns A non-zero error value on failure and 0 on success, * some possible errors are: * \retval MDBX_EINVAL An invalid parameter was specified. */ -LIBMDBX_INLINE_API(int, mdbx_env_get_maxdbs, - (const MDBX_env *env, MDBX_dbi *dbs)) { +LIBMDBX_INLINE_API(int, mdbx_env_get_maxdbs, (const MDBX_env *env, MDBX_dbi *dbs)) { int rc = MDBX_EINVAL; if (dbs) { uint64_t proxy = 0; @@ -3456,64 +3746,58 @@ MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API size_t mdbx_default_pagesize(void); * available/free RAM pages will be stored. * * \returns A non-zero error value on failure and 0 on success. */ -LIBMDBX_API int mdbx_get_sysraminfo(intptr_t *page_size, intptr_t *total_pages, - intptr_t *avail_pages); +LIBMDBX_API int mdbx_get_sysraminfo(intptr_t *page_size, intptr_t *total_pages, intptr_t *avail_pages); /** \brief Returns the maximum size of keys can put. * \ingroup c_statinfo * * \param [in] env An environment handle returned by \ref mdbx_env_create(). - * \param [in] flags Database options (\ref MDBX_DUPSORT, \ref MDBX_INTEGERKEY + * \param [in] flags Table options (\ref MDBX_DUPSORT, \ref MDBX_INTEGERKEY * and so on). \see db_flags * * \returns The maximum size of a key can write, * or -1 if something is wrong. */ -MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int -mdbx_env_get_maxkeysize_ex(const MDBX_env *env, MDBX_db_flags_t flags); +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int mdbx_env_get_maxkeysize_ex(const MDBX_env *env, MDBX_db_flags_t flags); /** \brief Returns the maximum size of data we can put. * \ingroup c_statinfo * * \param [in] env An environment handle returned by \ref mdbx_env_create(). - * \param [in] flags Database options (\ref MDBX_DUPSORT, \ref MDBX_INTEGERKEY + * \param [in] flags Table options (\ref MDBX_DUPSORT, \ref MDBX_INTEGERKEY * and so on). \see db_flags * * \returns The maximum size of a data can write, * or -1 if something is wrong. */ -MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int -mdbx_env_get_maxvalsize_ex(const MDBX_env *env, MDBX_db_flags_t flags); +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int mdbx_env_get_maxvalsize_ex(const MDBX_env *env, MDBX_db_flags_t flags); /** \deprecated Please use \ref mdbx_env_get_maxkeysize_ex() * and/or \ref mdbx_env_get_maxvalsize_ex() * \ingroup c_statinfo */ -MDBX_DEPRECATED MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int -mdbx_env_get_maxkeysize(const MDBX_env *env); +MDBX_NOTHROW_PURE_FUNCTION MDBX_DEPRECATED LIBMDBX_API int mdbx_env_get_maxkeysize(const MDBX_env *env); /** \brief Returns maximal size of key-value pair to fit in a single page - * for specified database flags. + * for specified table flags. * \ingroup c_statinfo * * \param [in] env An environment handle returned by \ref mdbx_env_create(). - * \param [in] flags Database options (\ref MDBX_DUPSORT, \ref MDBX_INTEGERKEY + * \param [in] flags Table options (\ref MDBX_DUPSORT, \ref MDBX_INTEGERKEY * and so on). \see db_flags * * \returns The maximum size of a data can write, * or -1 if something is wrong. */ -MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int -mdbx_env_get_pairsize4page_max(const MDBX_env *env, MDBX_db_flags_t flags); +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int mdbx_env_get_pairsize4page_max(const MDBX_env *env, MDBX_db_flags_t flags); /** \brief Returns maximal data size in bytes to fit in a leaf-page or - * single overflow/large-page for specified database flags. + * single large/overflow-page for specified table flags. * \ingroup c_statinfo * * \param [in] env An environment handle returned by \ref mdbx_env_create(). - * \param [in] flags Database options (\ref MDBX_DUPSORT, \ref MDBX_INTEGERKEY + * \param [in] flags Table options (\ref MDBX_DUPSORT, \ref MDBX_INTEGERKEY * and so on). \see db_flags * * \returns The maximum size of a data can write, * or -1 if something is wrong. */ -MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int -mdbx_env_get_valsize4page_max(const MDBX_env *env, MDBX_db_flags_t flags); +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int mdbx_env_get_valsize4page_max(const MDBX_env *env, MDBX_db_flags_t flags); /** \brief Sets application information (a context pointer) associated with * the environment. @@ -3534,8 +3818,7 @@ LIBMDBX_API int mdbx_env_set_userctx(MDBX_env *env, void *ctx); * \param [in] env An environment handle returned by \ref mdbx_env_create() * \returns The pointer set by \ref mdbx_env_set_userctx() * or `NULL` if something wrong. */ -MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API void * -mdbx_env_get_userctx(const MDBX_env *env); +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API void *mdbx_env_get_userctx(const MDBX_env *env); /** \brief Create a transaction with a user provided context pointer * for use with the environment. @@ -3546,8 +3829,8 @@ mdbx_env_get_userctx(const MDBX_env *env); * \see mdbx_txn_begin() * * \note A transaction and its cursors must only be used by a single thread, - * and a thread may only have a single transaction at a time. If \ref MDBX_NOTLS - * is in use, this does not apply to read-only transactions. + * and a thread may only have a single transaction at a time unless + * the \ref MDBX_NOSTICKYTHREADS is used. * * \note Cursors may not span transactions. * @@ -3596,8 +3879,7 @@ mdbx_env_get_userctx(const MDBX_env *env); * \retval MDBX_ENOMEM Out of memory. * \retval MDBX_BUSY The write transaction is already started by the * current thread. */ -LIBMDBX_API int mdbx_txn_begin_ex(MDBX_env *env, MDBX_txn *parent, - MDBX_txn_flags_t flags, MDBX_txn **txn, +LIBMDBX_API int mdbx_txn_begin_ex(MDBX_env *env, MDBX_txn *parent, MDBX_txn_flags_t flags, MDBX_txn **txn, void *context); /** \brief Create a transaction for use with the environment. @@ -3608,8 +3890,8 @@ LIBMDBX_API int mdbx_txn_begin_ex(MDBX_env *env, MDBX_txn *parent, * \see mdbx_txn_begin_ex() * * \note A transaction and its cursors must only be used by a single thread, - * and a thread may only have a single transaction at a time. If \ref MDBX_NOTLS - * is in use, this does not apply to read-only transactions. + * and a thread may only have a single transaction at a time unless + * the \ref MDBX_NOSTICKYTHREADS is used. * * \note Cursors may not span transactions. * @@ -3654,9 +3936,7 @@ LIBMDBX_API int mdbx_txn_begin_ex(MDBX_env *env, MDBX_txn *parent, * \retval MDBX_ENOMEM Out of memory. * \retval MDBX_BUSY The write transaction is already started by the * current thread. */ -LIBMDBX_INLINE_API(int, mdbx_txn_begin, - (MDBX_env * env, MDBX_txn *parent, MDBX_txn_flags_t flags, - MDBX_txn **txn)) { +LIBMDBX_INLINE_API(int, mdbx_txn_begin, (MDBX_env * env, MDBX_txn *parent, MDBX_txn_flags_t flags, MDBX_txn **txn)) { return mdbx_txn_begin_ex(env, parent, flags, txn, NULL); } @@ -3682,8 +3962,7 @@ LIBMDBX_API int mdbx_txn_set_userctx(MDBX_txn *txn, void *ctx); * \returns The pointer which was passed via the `context` parameter * of `mdbx_txn_begin_ex()` or set by \ref mdbx_txn_set_userctx(), * or `NULL` if something wrong. */ -MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API void * -mdbx_txn_get_userctx(const MDBX_txn *txn); +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API void *mdbx_txn_get_userctx(const MDBX_txn *txn); /** \brief Information about the transaction * \ingroup c_statinfo @@ -3750,15 +4029,13 @@ typedef struct MDBX_txn_info MDBX_txn_info; * See description of \ref MDBX_txn_info. * * \returns A non-zero error value on failure and 0 on success. */ -LIBMDBX_API int mdbx_txn_info(const MDBX_txn *txn, MDBX_txn_info *info, - bool scan_rlt); +LIBMDBX_API int mdbx_txn_info(const MDBX_txn *txn, MDBX_txn_info *info, bool scan_rlt); /** \brief Returns the transaction's MDBX_env. * \ingroup c_transactions * * \param [in] txn A transaction handle returned by \ref mdbx_txn_begin() */ -MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API MDBX_env * -mdbx_txn_env(const MDBX_txn *txn); +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API MDBX_env *mdbx_txn_env(const MDBX_txn *txn); /** \brief Return the transaction's flags. * \ingroup c_transactions @@ -3768,8 +4045,8 @@ mdbx_txn_env(const MDBX_txn *txn); * \param [in] txn A transaction handle returned by \ref mdbx_txn_begin(). * * \returns A transaction flags, valid if input is an valid transaction, - * otherwise -1. */ -MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int mdbx_txn_flags(const MDBX_txn *txn); + * otherwise \ref MDBX_TXN_INVALID. */ +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API MDBX_txn_flags_t mdbx_txn_flags(const MDBX_txn *txn); /** \brief Return the transaction's ID. * \ingroup c_statinfo @@ -3782,8 +4059,7 @@ MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int mdbx_txn_flags(const MDBX_txn *txn); * * \returns A transaction ID, valid if input is an active transaction, * otherwise 0. */ -MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API uint64_t -mdbx_txn_id(const MDBX_txn *txn); +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API uint64_t mdbx_txn_id(const MDBX_txn *txn); /** \brief Latency of commit stages in 1/65536 of seconds units. * \warning This structure may be changed in future releases. @@ -3791,7 +4067,7 @@ mdbx_txn_id(const MDBX_txn *txn); * \see mdbx_txn_commit_ex() */ struct MDBX_commit_latency { /** \brief Duration of preparation (commit child transactions, update - * sub-databases records and cursors destroying). */ + * table's records and cursors destroying). */ uint32_t preparation; /** \brief Duration of GC update by wall clock. */ uint32_t gc_wallclock; @@ -3874,6 +4150,12 @@ struct MDBX_commit_latency { /** \brief Количество страничных промахов (page faults) внутри GC * при выделении и подготовки страниц для самой GC. */ uint32_t self_majflt; + /* Для разборок с pnl_merge() */ + struct { + uint32_t time; + uint64_t volume; + uint32_t calls; + } pnl_merge_work, pnl_merge_self; } gc_prof; }; #ifndef __cplusplus @@ -3926,9 +4208,7 @@ LIBMDBX_API int mdbx_txn_commit_ex(MDBX_txn *txn, MDBX_commit_latency *latency); * \retval MDBX_EIO An error occurred during the flushing/writing * data to a storage medium/disk. * \retval MDBX_ENOMEM Out of memory. */ -LIBMDBX_INLINE_API(int, mdbx_txn_commit, (MDBX_txn * txn)) { - return mdbx_txn_commit_ex(txn, NULL); -} +LIBMDBX_INLINE_API(int, mdbx_txn_commit, (MDBX_txn * txn)) { return mdbx_txn_commit_ex(txn, NULL); } /** \brief Abandon all the operations of the transaction instead of saving them. * \ingroup c_transactions @@ -3965,7 +4245,7 @@ LIBMDBX_INLINE_API(int, mdbx_txn_commit, (MDBX_txn * txn)) { * \retval MDBX_EINVAL Transaction handle is NULL. */ LIBMDBX_API int mdbx_txn_abort(MDBX_txn *txn); -/** \brief Marks transaction as broken. +/** \brief Marks transaction as broken to prevent further operations. * \ingroup c_transactions * * Function keeps the transaction handle and corresponding locks, but makes @@ -3984,10 +4264,11 @@ LIBMDBX_API int mdbx_txn_break(MDBX_txn *txn); * Abort the read-only transaction like \ref mdbx_txn_abort(), but keep the * transaction handle. Therefore \ref mdbx_txn_renew() may reuse the handle. * This saves allocation overhead if the process will start a new read-only - * transaction soon, and also locking overhead if \ref MDBX_NOTLS is in use. The - * reader table lock is released, but the table slot stays tied to its thread - * or \ref MDBX_txn. Use \ref mdbx_txn_abort() to discard a reset handle, and to - * free its lock table slot if \ref MDBX_NOTLS is in use. + * transaction soon, and also locking overhead if \ref MDBX_NOSTICKYTHREADS is + * in use. The reader table lock is released, but the table slot stays tied to + * its thread or \ref MDBX_txn. Use \ref mdbx_txn_abort() to discard a reset + * handle, and to free its lock table slot if \ref MDBX_NOSTICKYTHREADS + * is in use. * * Cursors opened within the transaction must not be used again after this * call, except with \ref mdbx_cursor_renew() and \ref mdbx_cursor_close(). @@ -4012,6 +4293,94 @@ LIBMDBX_API int mdbx_txn_break(MDBX_txn *txn); * \retval MDBX_EINVAL Transaction handle is NULL. */ LIBMDBX_API int mdbx_txn_reset(MDBX_txn *txn); +/** \brief Переводит читающую транзакцию в "припаркованное" состояние. + * \ingroup c_transactions + * + * Выполняющиеся читающие транзакции не позволяют перерабатывать старые + * MVCC-снимки данных, начиная с самой старой используемой/читаемой версии и все + * последующие. Припаркованная же транзакция может быть вытеснена транзакцией + * записи, если будет мешать переработке мусора (старых MVCC-снимков данных). + * А если вытеснения не произойдет, то восстановление (перевод в рабочее + * состояние и продолжение выполнение) читающей транзакции будет существенно + * дешевле. Таким образом, парковка транзакций позволяет предотвратить + * негативные последствия связанные с остановкой переработки мусора, + * одновременно сохранив накладные расходы на минимальном уровне. + * + * Для продолжения выполнения (чтения и/или использования данных) припаркованная + * транзакция должна быть восстановлена посредством \ref mdbx_txn_unpark(). + * Для удобства использования и предотвращения лишних вызовов API, посредством + * параметра `autounpark`, предусмотрена возможность автоматической + * «распарковки» при использовании припаркованной транзакции в функциях API + * предполагающих чтение данных. + * + * \warning До восстановления/распарковки транзакции, вне зависимости от + * аргумента `autounpark`, нельзя допускать разыменования указателей полученных + * ранее при чтении данных в рамках припаркованной транзакции, так как + * MVCC-снимок в котором размещены эти данные не удерживается и может + * переработан в любой момент. + * + * Припаркованная транзакция без "распарковки" может быть прервана, сброшена + * или перезапущена в любой момент посредством \ref mdbx_txn_abort(), + * \ref mdbx_txn_reset() и \ref mdbx_txn_renew(), соответственно. + * + * \see mdbx_txn_unpark() + * \see mdbx_txn_flags() + * \see mdbx_env_set_hsr() + * \see Long-lived read transactions + * + * \param [in] txn Транзакция чтения запущенная посредством + * \ref mdbx_txn_begin(). + * + * \param [in] autounpark Позволяет включить автоматическую + * распарковку/восстановление транзакции при вызове + * функций API предполагающих чтение данных. + * + * \returns Ненулевое значение кода ошибки, либо 0 при успешном выполнении. */ +LIBMDBX_API int mdbx_txn_park(MDBX_txn *txn, bool autounpark); + +/** \brief Распарковывает ранее припаркованную читающую транзакцию. + * \ingroup c_transactions + * + * Функция пытается восстановить ранее припаркованную транзакцию. Если + * припаркованная транзакция была вытеснена ради переработки старых + * MVCC-снимков, то в зависимости от аргумента `restart_if_ousted` выполняется + * её перезапуск аналогично \ref mdbx_txn_renew(), либо транзакция сбрасывается + * и возвращается код ошибки \ref MDBX_OUSTED. + * + * \see mdbx_txn_park() + * \see mdbx_txn_flags() + * \see Long-lived read transactions + * + * \param [in] txn Транзакция чтения запущенная посредством + * \ref mdbx_txn_begin() и затем припаркованная + * посредством \ref mdbx_txn_park. + * + * \param [in] restart_if_ousted Позволяет сразу выполнить перезапуск + * транзакции, если она была вынестена. + * + * \returns Ненулевое значение кода ошибки, либо 0 при успешном выполнении. + * Некоторые специфичекие коды результата: + * + * \retval MDBX_SUCCESS Припаркованная транзакция успешно восстановлена, + * либо она не была припаркована. + * + * \retval MDBX_OUSTED Читающая транзакция была вытеснена пишущей + * транзакцией ради переработки старых MVCC-снимков, + * а аргумент `restart_if_ousted` был задан `false`. + * Транзакция сбрасывается в состояние аналогичное + * после вызова \ref mdbx_txn_reset(), но экземпляр + * (хендл) не освобождается и может быть использован + * повторно посредством \ref mdbx_txn_renew(), либо + * освобожден посредством \ref mdbx_txn_abort(). + * + * \retval MDBX_RESULT_TRUE Читающая транзакция была вынеснена, но теперь + * перезапущена для чтения другого (последнего) + * MVCC-снимка, так как restart_if_ousted` был задан + * `true`. + * + * \retval MDBX_BAD_TXN Транзакция уже завершена, либо не была запущена. */ +LIBMDBX_API int mdbx_txn_unpark(MDBX_txn *txn, bool restart_if_ousted); + /** \brief Renew a read-only transaction. * \ingroup c_transactions * @@ -4084,7 +4453,7 @@ LIBMDBX_API int mdbx_canary_put(MDBX_txn *txn, const MDBX_canary *canary); * \returns A non-zero error value on failure and 0 on success. */ LIBMDBX_API int mdbx_canary_get(const MDBX_txn *txn, MDBX_canary *canary); -/** \brief A callback function used to compare two keys in a database +/** \brief A callback function used to compare two keys in a table * \ingroup c_crud * \see mdbx_cmp() \see mdbx_get_keycmp() * \see mdbx_get_datacmp \see mdbx_dcmp() @@ -4104,26 +4473,25 @@ LIBMDBX_API int mdbx_canary_get(const MDBX_txn *txn, MDBX_canary *canary); * You have been warned but still can use custom comparators knowing * about the issues noted above. In this case you should ignore `deprecated` * warnings or define `MDBX_DEPRECATED` macro to empty to avoid ones. */ -typedef int(MDBX_cmp_func)(const MDBX_val *a, - const MDBX_val *b) MDBX_CXX17_NOEXCEPT; +typedef int(MDBX_cmp_func)(const MDBX_val *a, const MDBX_val *b) MDBX_CXX17_NOEXCEPT; -/** \brief Open or Create a database in the environment. +/** \brief Open or Create a named table in the environment. * \ingroup c_dbi * - * A database handle denotes the name and parameters of a database, - * independently of whether such a database exists. The database handle may be - * discarded by calling \ref mdbx_dbi_close(). The old database handle is - * returned if the database was already open. The handle may only be closed + * A table handle denotes the name and parameters of a table, + * independently of whether such a table exists. The table handle may be + * discarded by calling \ref mdbx_dbi_close(). The old table handle is + * returned if the table was already open. The handle may only be closed * once. * * \note A notable difference between MDBX and LMDB is that MDBX make handles - * opened for existing databases immediately available for other transactions, + * opened for existing tables immediately available for other transactions, * regardless this transaction will be aborted or reset. The REASON for this is * to avoiding the requirement for multiple opening a same handles in * concurrent read transactions, and tracking of such open but hidden handles * until the completion of read transactions which opened them. * - * Nevertheless, the handle for the NEWLY CREATED database will be invisible + * Nevertheless, the handle for the NEWLY CREATED table will be invisible * for other transactions until the this write transaction is successfully * committed. If the write transaction is aborted the handle will be closed * automatically. After a successful commit the such handle will reside in the @@ -4132,15 +4500,15 @@ typedef int(MDBX_cmp_func)(const MDBX_val *a, * In contrast to LMDB, the MDBX allow this function to be called from multiple * concurrent transactions or threads in the same process. * - * To use named database (with name != NULL), \ref mdbx_env_set_maxdbs() + * To use named table (with name != NULL), \ref mdbx_env_set_maxdbs() * must be called before opening the environment. Table names are - * keys in the internal unnamed database, and may be read but not written. + * keys in the internal unnamed table, and may be read but not written. * * \param [in] txn transaction handle returned by \ref mdbx_txn_begin(). - * \param [in] name The name of the database to open. If only a single - * database is needed in the environment, + * \param [in] name The name of the table to open. If only a single + * table is needed in the environment, * this value may be NULL. - * \param [in] flags Special options for this database. This parameter must + * \param [in] flags Special options for this table. This parameter must * be bitwise OR'ing together any of the constants * described here: * @@ -4154,12 +4522,12 @@ typedef int(MDBX_cmp_func)(const MDBX_val *a, * uint64_t, and will be sorted as such. The keys must all be of the * same size and must be aligned while passing as arguments. * - \ref MDBX_DUPSORT - * Duplicate keys may be used in the database. Or, from another point of + * Duplicate keys may be used in the table. Or, from another point of * view, keys may have multiple data items, stored in sorted order. By * default keys must be unique and may have only a single data item. * - \ref MDBX_DUPFIXED * This flag may only be used in combination with \ref MDBX_DUPSORT. This - * option tells the library that the data items for this database are + * option tells the library that the data items for this table are * all the same size, which allows further optimizations in storage and * retrieval. When all data items are the same size, the * \ref MDBX_GET_MULTIPLE, \ref MDBX_NEXT_MULTIPLE and @@ -4174,7 +4542,7 @@ typedef int(MDBX_cmp_func)(const MDBX_val *a, * strings in reverse order (the comparison is performed in the direction * from the last byte to the first). * - \ref MDBX_CREATE - * Create the named database if it doesn't exist. This option is not + * Create the named table if it doesn't exist. This option is not * allowed in a read-only transaction or a read-only environment. * * \param [out] dbi Address where the new \ref MDBX_dbi handle @@ -4186,42 +4554,107 @@ typedef int(MDBX_cmp_func)(const MDBX_val *a, * * \returns A non-zero error value on failure and 0 on success, * some possible errors are: - * \retval MDBX_NOTFOUND The specified database doesn't exist in the + * \retval MDBX_NOTFOUND The specified table doesn't exist in the * environment and \ref MDBX_CREATE was not specified. - * \retval MDBX_DBS_FULL Too many databases have been opened. + * \retval MDBX_DBS_FULL Too many tables have been opened. * \see mdbx_env_set_maxdbs() - * \retval MDBX_INCOMPATIBLE Database is incompatible with given flags, + * \retval MDBX_INCOMPATIBLE Table is incompatible with given flags, * i.e. the passed flags is different with which the - * database was created, or the database was already + * table was created, or the table was already * opened with a different comparison function(s). * \retval MDBX_THREAD_MISMATCH Given transaction is not owned * by current thread. */ -LIBMDBX_API int mdbx_dbi_open(MDBX_txn *txn, const char *name, - MDBX_db_flags_t flags, MDBX_dbi *dbi); -LIBMDBX_API int mdbx_dbi_open2(MDBX_txn *txn, const MDBX_val *name, - MDBX_db_flags_t flags, MDBX_dbi *dbi); +LIBMDBX_API int mdbx_dbi_open(MDBX_txn *txn, const char *name, MDBX_db_flags_t flags, MDBX_dbi *dbi); +/** \copydoc mdbx_dbi_open() + * \ingroup c_dbi */ +LIBMDBX_API int mdbx_dbi_open2(MDBX_txn *txn, const MDBX_val *name, MDBX_db_flags_t flags, MDBX_dbi *dbi); -/** \deprecated Please - * \ref avoid_custom_comparators "avoid using custom comparators" and use - * \ref mdbx_dbi_open() instead. - * +/** \brief Open or Create a named table in the environment + * with using custom comparison functions. * \ingroup c_dbi * + * \deprecated Please \ref avoid_custom_comparators + * "avoid using custom comparators" and use \ref mdbx_dbi_open() instead. + * * \param [in] txn transaction handle returned by \ref mdbx_txn_begin(). - * \param [in] name The name of the database to open. If only a single - * database is needed in the environment, + * \param [in] name The name of the table to open. If only a single + * table is needed in the environment, * this value may be NULL. - * \param [in] flags Special options for this database. - * \param [in] keycmp Optional custom key comparison function for a database. - * \param [in] datacmp Optional custom data comparison function for a database. + * \param [in] flags Special options for this table. + * \param [in] keycmp Optional custom key comparison function for a table. + * \param [in] datacmp Optional custom data comparison function for a table. * \param [out] dbi Address where the new MDBX_dbi handle will be stored. * \returns A non-zero error value on failure and 0 on success. */ -MDBX_DEPRECATED LIBMDBX_API int -mdbx_dbi_open_ex(MDBX_txn *txn, const char *name, MDBX_db_flags_t flags, - MDBX_dbi *dbi, MDBX_cmp_func *keycmp, MDBX_cmp_func *datacmp); -MDBX_DEPRECATED LIBMDBX_API int -mdbx_dbi_open_ex2(MDBX_txn *txn, const MDBX_val *name, MDBX_db_flags_t flags, - MDBX_dbi *dbi, MDBX_cmp_func *keycmp, MDBX_cmp_func *datacmp); +MDBX_DEPRECATED LIBMDBX_API int mdbx_dbi_open_ex(MDBX_txn *txn, const char *name, MDBX_db_flags_t flags, MDBX_dbi *dbi, + MDBX_cmp_func *keycmp, MDBX_cmp_func *datacmp); +/** \copydoc mdbx_dbi_open_ex() + * \ingroup c_dbi */ +MDBX_DEPRECATED LIBMDBX_API int mdbx_dbi_open_ex2(MDBX_txn *txn, const MDBX_val *name, MDBX_db_flags_t flags, + MDBX_dbi *dbi, MDBX_cmp_func *keycmp, MDBX_cmp_func *datacmp); + +/** \brief Переименовает таблицу по DBI-дескриптору + * + * \ingroup c_dbi + * + * Переименовывает пользовательскую именованную таблицу связанную с передаваемым + * DBI-дескриптором. + * + * \param [in,out] txn Пишущая транзакция запущенная посредством + * \ref mdbx_txn_begin(). + * \param [in] dbi Дескриптор таблицы + * открытый посредством \ref mdbx_dbi_open(). + * + * \param [in] name Новое имя для переименования. + * + * \returns Ненулевое значение кода ошибки, либо 0 при успешном выполнении. */ +LIBMDBX_API int mdbx_dbi_rename(MDBX_txn *txn, MDBX_dbi dbi, const char *name); +/** \copydoc mdbx_dbi_rename() + * \ingroup c_dbi */ +LIBMDBX_API int mdbx_dbi_rename2(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *name); + +/** \brief Функция обратного вызова для перечисления + * пользовательских именованных таблиц. + * + * \ingroup c_statinfo + * \see mdbx_enumerate_tables() + * + * \param [in] ctx Указатель на контекст переданный аналогичным + * параметром в \ref mdbx_enumerate_tables(). + * \param [in] txn Транзазакция. + * \param [in] name Имя таблицы. + * \param [in] flags Флаги \ref MDBX_db_flags_t. + * \param [in] stat Базовая информация \ref MDBX_stat о таблице. + * \param [in] dbi Отличное от 0 значение DBI-дескриптора, + * если таковой был открыт для этой таблицы. + * Либо 0 если такого открытого дескриптора нет. + * + * \returns Ноль при успехе и продолжении перечисления, при возвращении другого + * значения оно будет немедленно возвращено вызывающему + * без продолжения перечисления. */ +typedef int(MDBX_table_enum_func)(void *ctx, const MDBX_txn *txn, const MDBX_val *name, MDBX_db_flags_t flags, + const struct MDBX_stat *stat, MDBX_dbi dbi) MDBX_CXX17_NOEXCEPT; + +/** \brief Перечисляет пользовательские именнованные таблицы. + * + * Производит перечисление пользовательских именнованных таблиц, вызывая + * специфицируемую пользователем функцию-визитер для каждой именованной таблицы. + * Перечисление продолжается до исчерпания именованных таблиц, либо до возврата + * отличного от нуля результата из заданной пользователем функции, которое будет + * сразу возвращено в качестве результата. + * + * \ingroup c_statinfo + * \see MDBX_table_enum_func + * + * \param [in] txn Транзакция запущенная посредством + * \ref mdbx_txn_begin(). + * \param [in] func Указатель на пользовательскую функцию + * с сигнатурой \ref MDBX_table_enum_func, + * которая будет вызвана для каждой таблицы. + * \param [in] ctx Указатель на некоторый контект, который будет передан + * в функцию `func()` как есть. + * + * \returns Ненулевое значение кода ошибки, либо 0 при успешном выполнении. */ +LIBMDBX_API int mdbx_enumerate_tables(const MDBX_txn *txn, MDBX_table_enum_func *func, void *ctx); /** \defgroup value2key Value-to-Key functions * \brief Value-to-Key functions to @@ -4234,28 +4667,21 @@ mdbx_dbi_open_ex2(MDBX_txn *txn, const MDBX_val *name, MDBX_db_flags_t flags, * and IEEE754 double values in one index for JSON-numbers with restriction for * integer numbers range corresponding to RFC-7159, i.e. \f$[-2^{53}+1, * 2^{53}-1]\f$. See bottom of page 6 at https://tools.ietf.org/html/rfc7159 */ -MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API uint64_t -mdbx_key_from_jsonInteger(const int64_t json_integer); +MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API uint64_t mdbx_key_from_jsonInteger(const int64_t json_integer); -MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API uint64_t -mdbx_key_from_double(const double ieee754_64bit); +MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API uint64_t mdbx_key_from_double(const double ieee754_64bit); -MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API uint64_t -mdbx_key_from_ptrdouble(const double *const ieee754_64bit); +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API uint64_t mdbx_key_from_ptrdouble(const double *const ieee754_64bit); -MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API uint32_t -mdbx_key_from_float(const float ieee754_32bit); +MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API uint32_t mdbx_key_from_float(const float ieee754_32bit); -MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API uint32_t -mdbx_key_from_ptrfloat(const float *const ieee754_32bit); +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API uint32_t mdbx_key_from_ptrfloat(const float *const ieee754_32bit); -MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_INLINE_API(uint64_t, mdbx_key_from_int64, - (const int64_t i64)) { +MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_INLINE_API(uint64_t, mdbx_key_from_int64, (const int64_t i64)) { return UINT64_C(0x8000000000000000) + i64; } -MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_INLINE_API(uint32_t, mdbx_key_from_int32, - (const int32_t i32)) { +MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_INLINE_API(uint32_t, mdbx_key_from_int32, (const int32_t i32)) { return UINT32_C(0x80000000) + i32; } /** end of value2key @} */ @@ -4265,27 +4691,22 @@ MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_INLINE_API(uint32_t, mdbx_key_from_int32, * \ref avoid_custom_comparators "avoid using custom comparators" * \see value2key * @{ */ -MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int64_t -mdbx_jsonInteger_from_key(const MDBX_val); +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int64_t mdbx_jsonInteger_from_key(const MDBX_val); -MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API double -mdbx_double_from_key(const MDBX_val); +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API double mdbx_double_from_key(const MDBX_val); -MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API float -mdbx_float_from_key(const MDBX_val); +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API float mdbx_float_from_key(const MDBX_val); -MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int32_t -mdbx_int32_from_key(const MDBX_val); +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int32_t mdbx_int32_from_key(const MDBX_val); -MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int64_t -mdbx_int64_from_key(const MDBX_val); +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int64_t mdbx_int64_from_key(const MDBX_val); /** end of value2key @} */ -/** \brief Retrieve statistics for a database. +/** \brief Retrieve statistics for a table. * \ingroup c_statinfo * * \param [in] txn A transaction handle returned by \ref mdbx_txn_begin(). - * \param [in] dbi A database handle returned by \ref mdbx_dbi_open(). + * \param [in] dbi A table handle returned by \ref mdbx_dbi_open(). * \param [out] stat The address of an \ref MDBX_stat structure where * the statistics will be copied. * \param [in] bytes The size of \ref MDBX_stat. @@ -4295,15 +4716,14 @@ mdbx_int64_from_key(const MDBX_val); * \retval MDBX_THREAD_MISMATCH Given transaction is not owned * by current thread. * \retval MDBX_EINVAL An invalid parameter was specified. */ -LIBMDBX_API int mdbx_dbi_stat(const MDBX_txn *txn, MDBX_dbi dbi, - MDBX_stat *stat, size_t bytes); +LIBMDBX_API int mdbx_dbi_stat(const MDBX_txn *txn, MDBX_dbi dbi, MDBX_stat *stat, size_t bytes); /** \brief Retrieve depth (bitmask) information of nested dupsort (multi-value) - * B+trees for given database. + * B+trees for given table. * \ingroup c_statinfo * * \param [in] txn A transaction handle returned by \ref mdbx_txn_begin(). - * \param [in] dbi A database handle returned by \ref mdbx_dbi_open(). + * \param [in] dbi A table handle returned by \ref mdbx_dbi_open(). * \param [out] mask The address of an uint32_t value where the bitmask * will be stored. * @@ -4312,14 +4732,13 @@ LIBMDBX_API int mdbx_dbi_stat(const MDBX_txn *txn, MDBX_dbi dbi, * \retval MDBX_THREAD_MISMATCH Given transaction is not owned * by current thread. * \retval MDBX_EINVAL An invalid parameter was specified. - * \retval MDBX_RESULT_TRUE The dbi isn't a dupsort (multi-value) database. */ -LIBMDBX_API int mdbx_dbi_dupsort_depthmask(const MDBX_txn *txn, MDBX_dbi dbi, - uint32_t *mask); + * \retval MDBX_RESULT_TRUE The dbi isn't a dupsort (multi-value) table. */ +LIBMDBX_API int mdbx_dbi_dupsort_depthmask(const MDBX_txn *txn, MDBX_dbi dbi, uint32_t *mask); /** \brief DBI state bits returted by \ref mdbx_dbi_flags_ex() * \ingroup c_statinfo * \see mdbx_dbi_flags_ex() */ -enum MDBX_dbi_state_t { +typedef enum MDBX_dbi_state { /** DB was written in this txn */ MDBX_DBI_DIRTY = 0x01, /** Cached Named-DB record is older than txnID */ @@ -4328,109 +4747,107 @@ enum MDBX_dbi_state_t { MDBX_DBI_FRESH = 0x04, /** Named-DB handle created in this txn */ MDBX_DBI_CREAT = 0x08, -}; -#ifndef __cplusplus -/** \ingroup c_statinfo */ -typedef enum MDBX_dbi_state_t MDBX_dbi_state_t; -#else -DEFINE_ENUM_FLAG_OPERATORS(MDBX_dbi_state_t) -#endif +} MDBX_dbi_state_t; +DEFINE_ENUM_FLAG_OPERATORS(MDBX_dbi_state) -/** \brief Retrieve the DB flags and status for a database handle. +/** \brief Retrieve the DB flags and status for a table handle. * \ingroup c_statinfo + * \see MDBX_db_flags_t + * \see MDBX_dbi_state_t * * \param [in] txn A transaction handle returned by \ref mdbx_txn_begin(). - * \param [in] dbi A database handle returned by \ref mdbx_dbi_open(). + * \param [in] dbi A table handle returned by \ref mdbx_dbi_open(). * \param [out] flags Address where the flags will be returned. * \param [out] state Address where the state will be returned. * * \returns A non-zero error value on failure and 0 on success. */ -LIBMDBX_API int mdbx_dbi_flags_ex(const MDBX_txn *txn, MDBX_dbi dbi, - unsigned *flags, unsigned *state); +LIBMDBX_API int mdbx_dbi_flags_ex(const MDBX_txn *txn, MDBX_dbi dbi, unsigned *flags, unsigned *state); /** \brief The shortcut to calling \ref mdbx_dbi_flags_ex() with `state=NULL` * for discarding it result. - * \ingroup c_statinfo */ -LIBMDBX_INLINE_API(int, mdbx_dbi_flags, - (const MDBX_txn *txn, MDBX_dbi dbi, unsigned *flags)) { + * \ingroup c_statinfo + * \see MDBX_db_flags_t */ +LIBMDBX_INLINE_API(int, mdbx_dbi_flags, (const MDBX_txn *txn, MDBX_dbi dbi, unsigned *flags)) { unsigned state; return mdbx_dbi_flags_ex(txn, dbi, flags, &state); } -/** \brief Close a database handle. Normally unnecessary. +/** \brief Close a table handle. Normally unnecessary. * \ingroup c_dbi * - * Closing a database handle is not necessary, but lets \ref mdbx_dbi_open() + * Closing a table handle is not necessary, but lets \ref mdbx_dbi_open() * reuse the handle value. Usually it's better to set a bigger * \ref mdbx_env_set_maxdbs(), unless that value would be large. * * \note Use with care. - * This call is synchronized via mutex with \ref mdbx_dbi_close(), but NOT with - * other transactions running by other threads. The "next" version of libmdbx - * (\ref MithrilDB) will solve this issue. + * This call is synchronized via mutex with \ref mdbx_dbi_open(), but NOT with + * any transaction(s) running by other thread(s). + * So the `mdbx_dbi_close()` MUST NOT be called in-parallel/concurrently + * with any transactions using the closing dbi-handle, nor during other thread + * commit/abort a write transacton(s). The "next" version of libmdbx (\ref + * MithrilDB) will solve this issue. * * Handles should only be closed if no other threads are going to reference - * the database handle or one of its cursors any further. Do not close a handle - * if an existing transaction has modified its database. Doing so can cause - * misbehavior from database corruption to errors like \ref MDBX_BAD_DBI + * the table handle or one of its cursors any further. Do not close a handle + * if an existing transaction has modified its table. Doing so can cause + * misbehavior from table corruption to errors like \ref MDBX_BAD_DBI * (since the DB name is gone). * * \param [in] env An environment handle returned by \ref mdbx_env_create(). - * \param [in] dbi A database handle returned by \ref mdbx_dbi_open(). + * \param [in] dbi A table handle returned by \ref mdbx_dbi_open(). * * \returns A non-zero error value on failure and 0 on success. */ LIBMDBX_API int mdbx_dbi_close(MDBX_env *env, MDBX_dbi dbi); -/** \brief Empty or delete and close a database. +/** \brief Empty or delete and close a table. * \ingroup c_crud * * \see mdbx_dbi_close() \see mdbx_dbi_open() * * \param [in] txn A transaction handle returned by \ref mdbx_txn_begin(). - * \param [in] dbi A database handle returned by \ref mdbx_dbi_open(). + * \param [in] dbi A table handle returned by \ref mdbx_dbi_open(). * \param [in] del `false` to empty the DB, `true` to delete it * from the environment and close the DB handle. * * \returns A non-zero error value on failure and 0 on success. */ LIBMDBX_API int mdbx_drop(MDBX_txn *txn, MDBX_dbi dbi, bool del); -/** \brief Get items from a database. +/** \brief Get items from a table. * \ingroup c_crud * - * This function retrieves key/data pairs from the database. The address + * This function retrieves key/data pairs from the table. The address * and length of the data associated with the specified key are returned * in the structure to which data refers. - * If the database supports duplicate keys (\ref MDBX_DUPSORT) then the + * If the table supports duplicate keys (\ref MDBX_DUPSORT) then the * first data item for the key will be returned. Retrieval of other * items requires the use of \ref mdbx_cursor_get(). * * \note The memory pointed to by the returned values is owned by the - * database. The caller MUST not dispose of the memory, and MUST not modify it + * table. The caller MUST not dispose of the memory, and MUST not modify it * in any way regardless in a read-only nor read-write transactions! - * For case a database opened without the \ref MDBX_WRITEMAP modification - * attempts likely will cause a `SIGSEGV`. However, when a database opened with + * For case a table opened without the \ref MDBX_WRITEMAP modification + * attempts likely will cause a `SIGSEGV`. However, when a table opened with * the \ref MDBX_WRITEMAP or in case values returned inside read-write * transaction are located on a "dirty" (modified and pending to commit) pages, * such modification will silently accepted and likely will lead to DB and/or * data corruption. * - * \note Values returned from the database are valid only until a + * \note Values returned from the table are valid only until a * subsequent update operation, or the end of the transaction. * * \param [in] txn A transaction handle returned by \ref mdbx_txn_begin(). - * \param [in] dbi A database handle returned by \ref mdbx_dbi_open(). - * \param [in] key The key to search for in the database. + * \param [in] dbi A table handle returned by \ref mdbx_dbi_open(). + * \param [in] key The key to search for in the table. * \param [in,out] data The data corresponding to the key. * * \returns A non-zero error value on failure and 0 on success, * some possible errors are: * \retval MDBX_THREAD_MISMATCH Given transaction is not owned * by current thread. - * \retval MDBX_NOTFOUND The key was not in the database. + * \retval MDBX_NOTFOUND The key was not in the table. * \retval MDBX_EINVAL An invalid parameter was specified. */ -LIBMDBX_API int mdbx_get(const MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, - MDBX_val *data); +LIBMDBX_API int mdbx_get(const MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, MDBX_val *data); -/** \brief Get items from a database +/** \brief Get items from a table * and optionally number of data items for a given key. * * \ingroup c_crud @@ -4440,30 +4857,29 @@ LIBMDBX_API int mdbx_get(const MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, * 1. If values_count is NOT NULL, then returns the count * of multi-values/duplicates for a given key. * 2. Updates BOTH the key and the data for pointing to the actual key-value - * pair inside the database. + * pair inside the table. * * \param [in] txn A transaction handle returned * by \ref mdbx_txn_begin(). - * \param [in] dbi A database handle returned by \ref mdbx_dbi_open(). - * \param [in,out] key The key to search for in the database. + * \param [in] dbi A table handle returned by \ref mdbx_dbi_open(). + * \param [in,out] key The key to search for in the table. * \param [in,out] data The data corresponding to the key. * \param [out] values_count The optional address to return number of values * associated with given key: * = 0 - in case \ref MDBX_NOTFOUND error; - * = 1 - exactly for databases + * = 1 - exactly for tables * WITHOUT \ref MDBX_DUPSORT; - * >= 1 for databases WITH \ref MDBX_DUPSORT. + * >= 1 for tables WITH \ref MDBX_DUPSORT. * * \returns A non-zero error value on failure and 0 on success, * some possible errors are: * \retval MDBX_THREAD_MISMATCH Given transaction is not owned * by current thread. - * \retval MDBX_NOTFOUND The key was not in the database. + * \retval MDBX_NOTFOUND The key was not in the table. * \retval MDBX_EINVAL An invalid parameter was specified. */ -LIBMDBX_API int mdbx_get_ex(const MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, - MDBX_val *data, size_t *values_count); +LIBMDBX_API int mdbx_get_ex(const MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, MDBX_val *data, size_t *values_count); -/** \brief Get equal or great item from a database. +/** \brief Get equal or great item from a table. * \ingroup c_crud * * Briefly this function does the same as \ref mdbx_get() with a few @@ -4471,17 +4887,17 @@ LIBMDBX_API int mdbx_get_ex(const MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, * 1. Return equal or great (due comparison function) key-value * pair, but not only exactly matching with the key. * 2. On success return \ref MDBX_SUCCESS if key found exactly, - * and \ref MDBX_RESULT_TRUE otherwise. Moreover, for databases with + * and \ref MDBX_RESULT_TRUE otherwise. Moreover, for tables with * \ref MDBX_DUPSORT flag the data argument also will be used to match over * multi-value/duplicates, and \ref MDBX_SUCCESS will be returned only when * BOTH the key and the data match exactly. * 3. Updates BOTH the key and the data for pointing to the actual key-value - * pair inside the database. + * pair inside the table. * * \param [in] txn A transaction handle returned * by \ref mdbx_txn_begin(). - * \param [in] dbi A database handle returned by \ref mdbx_dbi_open(). - * \param [in,out] key The key to search for in the database. + * \param [in] dbi A table handle returned by \ref mdbx_dbi_open(). + * \param [in,out] key The key to search for in the table. * \param [in,out] data The data corresponding to the key. * * \returns A non-zero error value on failure and \ref MDBX_RESULT_FALSE @@ -4489,43 +4905,42 @@ LIBMDBX_API int mdbx_get_ex(const MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, * Some possible errors are: * \retval MDBX_THREAD_MISMATCH Given transaction is not owned * by current thread. - * \retval MDBX_NOTFOUND The key was not in the database. + * \retval MDBX_NOTFOUND The key was not in the table. * \retval MDBX_EINVAL An invalid parameter was specified. */ -LIBMDBX_API int mdbx_get_equal_or_great(const MDBX_txn *txn, MDBX_dbi dbi, - MDBX_val *key, MDBX_val *data); +LIBMDBX_API int mdbx_get_equal_or_great(const MDBX_txn *txn, MDBX_dbi dbi, MDBX_val *key, MDBX_val *data); -/** \brief Store items into a database. +/** \brief Store items into a table. * \ingroup c_crud * - * This function stores key/data pairs in the database. The default behavior + * This function stores key/data pairs in the table. The default behavior * is to enter the new key/data pair, replacing any previously existing key * if duplicates are disallowed, or adding a duplicate data item if * duplicates are allowed (see \ref MDBX_DUPSORT). * * \param [in] txn A transaction handle returned * by \ref mdbx_txn_begin(). - * \param [in] dbi A database handle returned by \ref mdbx_dbi_open(). - * \param [in] key The key to store in the database. + * \param [in] dbi A table handle returned by \ref mdbx_dbi_open(). + * \param [in] key The key to store in the table. * \param [in,out] data The data to store. * \param [in] flags Special options for this operation. * This parameter must be set to 0 or by bitwise OR'ing * together one or more of the values described here: * - \ref MDBX_NODUPDATA * Enter the new key-value pair only if it does not already appear - * in the database. This flag may only be specified if the database + * in the table. This flag may only be specified if the table * was opened with \ref MDBX_DUPSORT. The function will return - * \ref MDBX_KEYEXIST if the key/data pair already appears in the database. + * \ref MDBX_KEYEXIST if the key/data pair already appears in the table. * * - \ref MDBX_NOOVERWRITE * Enter the new key/data pair only if the key does not already appear - * in the database. The function will return \ref MDBX_KEYEXIST if the key - * already appears in the database, even if the database supports + * in the table. The function will return \ref MDBX_KEYEXIST if the key + * already appears in the table, even if the table supports * duplicates (see \ref MDBX_DUPSORT). The data parameter will be set * to point to the existing item. * * - \ref MDBX_CURRENT * Update an single existing entry, but not add new ones. The function will - * return \ref MDBX_NOTFOUND if the given key not exist in the database. + * return \ref MDBX_NOTFOUND if the given key not exist in the table. * In case multi-values for the given key, with combination of * the \ref MDBX_ALLDUPS will replace all multi-values, * otherwise return the \ref MDBX_EMULTIVAL. @@ -4537,10 +4952,10 @@ LIBMDBX_API int mdbx_get_equal_or_great(const MDBX_txn *txn, MDBX_dbi dbi, * transaction ends. This saves an extra memcpy if the data is being * generated later. MDBX does nothing else with this memory, the caller * is expected to modify all of the space requested. This flag must not - * be specified if the database was opened with \ref MDBX_DUPSORT. + * be specified if the table was opened with \ref MDBX_DUPSORT. * * - \ref MDBX_APPEND - * Append the given key/data pair to the end of the database. This option + * Append the given key/data pair to the end of the table. This option * allows fast bulk loading when keys are already known to be in the * correct order. Loading unsorted keys with this flag will cause * a \ref MDBX_EKEYMISMATCH error. @@ -4550,14 +4965,14 @@ LIBMDBX_API int mdbx_get_equal_or_great(const MDBX_txn *txn, MDBX_dbi dbi, * * - \ref MDBX_MULTIPLE * Store multiple contiguous data elements in a single request. This flag - * may only be specified if the database was opened with + * may only be specified if the table was opened with * \ref MDBX_DUPFIXED. With combination the \ref MDBX_ALLDUPS * will replace all multi-values. * The data argument must be an array of two \ref MDBX_val. The `iov_len` * of the first \ref MDBX_val must be the size of a single data element. * The `iov_base` of the first \ref MDBX_val must point to the beginning * of the array of contiguous data elements which must be properly aligned - * in case of database with \ref MDBX_INTEGERDUP flag. + * in case of table with \ref MDBX_INTEGERDUP flag. * The `iov_len` of the second \ref MDBX_val must be the count of the * number of data elements to store. On return this field will be set to * the count of the number of elements actually written. The `iov_base` of @@ -4569,16 +4984,15 @@ LIBMDBX_API int mdbx_get_equal_or_great(const MDBX_txn *txn, MDBX_dbi dbi, * some possible errors are: * \retval MDBX_THREAD_MISMATCH Given transaction is not owned * by current thread. - * \retval MDBX_KEYEXIST The key/value pair already exists in the database. + * \retval MDBX_KEYEXIST The key/value pair already exists in the table. * \retval MDBX_MAP_FULL The database is full, see \ref mdbx_env_set_mapsize(). * \retval MDBX_TXN_FULL The transaction has too many dirty pages. * \retval MDBX_EACCES An attempt was made to write * in a read-only transaction. * \retval MDBX_EINVAL An invalid parameter was specified. */ -LIBMDBX_API int mdbx_put(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, - MDBX_val *data, MDBX_put_flags_t flags); +LIBMDBX_API int mdbx_put(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, MDBX_val *data, MDBX_put_flags_t flags); -/** \brief Replace items in a database. +/** \brief Replace items in a table. * \ingroup c_crud * * This function allows to update or delete an existing value at the same time @@ -4593,7 +5007,7 @@ LIBMDBX_API int mdbx_put(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, * field pointed by old_data argument to the appropriate value, without * performing any changes. * - * For databases with non-unique keys (i.e. with \ref MDBX_DUPSORT flag), + * For tables with non-unique keys (i.e. with \ref MDBX_DUPSORT flag), * another use case is also possible, when by old_data argument selects a * specific item from multi-value/duplicates with the same key for deletion or * update. To select this scenario in flags should simultaneously specify @@ -4603,8 +5017,8 @@ LIBMDBX_API int mdbx_put(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, * * \param [in] txn A transaction handle returned * by \ref mdbx_txn_begin(). - * \param [in] dbi A database handle returned by \ref mdbx_dbi_open(). - * \param [in] key The key to store in the database. + * \param [in] dbi A table handle returned by \ref mdbx_dbi_open(). + * \param [in] key The key to store in the table. * \param [in] new_data The data to store, if NULL then deletion will * be performed. * \param [in,out] old_data The buffer for retrieve previous value as describe @@ -4621,36 +5035,32 @@ LIBMDBX_API int mdbx_put(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, * \see \ref c_crud_hints "Quick reference for Insert/Update/Delete operations" * * \returns A non-zero error value on failure and 0 on success. */ -LIBMDBX_API int mdbx_replace(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, - MDBX_val *new_data, MDBX_val *old_data, +LIBMDBX_API int mdbx_replace(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, MDBX_val *new_data, MDBX_val *old_data, MDBX_put_flags_t flags); -typedef int (*MDBX_preserve_func)(void *context, MDBX_val *target, - const void *src, size_t bytes); -LIBMDBX_API int mdbx_replace_ex(MDBX_txn *txn, MDBX_dbi dbi, - const MDBX_val *key, MDBX_val *new_data, - MDBX_val *old_data, MDBX_put_flags_t flags, - MDBX_preserve_func preserver, +typedef int (*MDBX_preserve_func)(void *context, MDBX_val *target, const void *src, size_t bytes); +LIBMDBX_API int mdbx_replace_ex(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, MDBX_val *new_data, + MDBX_val *old_data, MDBX_put_flags_t flags, MDBX_preserve_func preserver, void *preserver_context); -/** \brief Delete items from a database. +/** \brief Delete items from a table. * \ingroup c_crud * - * This function removes key/data pairs from the database. + * This function removes key/data pairs from the table. * - * \note The data parameter is NOT ignored regardless the database does + * \note The data parameter is NOT ignored regardless the table does * support sorted duplicate data items or not. If the data parameter * is non-NULL only the matching data item will be deleted. Otherwise, if data * parameter is NULL, any/all value(s) for specified key will be deleted. * * This function will return \ref MDBX_NOTFOUND if the specified key/data - * pair is not in the database. + * pair is not in the table. * * \see \ref c_crud_hints "Quick reference for Insert/Update/Delete operations" * * \param [in] txn A transaction handle returned by \ref mdbx_txn_begin(). - * \param [in] dbi A database handle returned by \ref mdbx_dbi_open(). - * \param [in] key The key to delete from the database. + * \param [in] dbi A table handle returned by \ref mdbx_dbi_open(). + * \param [in] key The key to delete from the table. * \param [in] data The data to delete. * * \returns A non-zero error value on failure and 0 on success, @@ -4658,13 +5068,12 @@ LIBMDBX_API int mdbx_replace_ex(MDBX_txn *txn, MDBX_dbi dbi, * \retval MDBX_EACCES An attempt was made to write * in a read-only transaction. * \retval MDBX_EINVAL An invalid parameter was specified. */ -LIBMDBX_API int mdbx_del(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, - const MDBX_val *data); +LIBMDBX_API int mdbx_del(MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *key, const MDBX_val *data); /** \brief Create a cursor handle but not bind it to transaction nor DBI-handle. * \ingroup c_cursors * - * A cursor cannot be used when its database handle is closed. Nor when its + * A cursor cannot be used when its table handle is closed. Nor when its * transaction has ended, except with \ref mdbx_cursor_bind() and \ref * mdbx_cursor_renew(). Also it can be discarded with \ref mdbx_cursor_close(). * @@ -4705,8 +5114,7 @@ LIBMDBX_API int mdbx_cursor_set_userctx(MDBX_cursor *cursor, void *ctx); * \returns The pointer which was passed via the `context` parameter * of `mdbx_cursor_create()` or set by \ref mdbx_cursor_set_userctx(), * or `NULL` if something wrong. */ -MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API void * -mdbx_cursor_get_userctx(const MDBX_cursor *cursor); +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API void *mdbx_cursor_get_userctx(const MDBX_cursor *cursor); /** \brief Bind cursor to specified transaction and DBI-handle. * \ingroup c_cursors @@ -4715,9 +5123,13 @@ mdbx_cursor_get_userctx(const MDBX_cursor *cursor); * \ref mdbx_cursor_renew() but with specifying an arbitrary DBI-handle. * * A cursor may be associated with a new transaction, and referencing a new or - * the same database handle as it was created with. This may be done whether the + * the same table handle as it was created with. This may be done whether the * previous transaction is live or dead. * + * If the transaction is nested, then the cursor should not be used in its parent transaction. + * Otherwise it is no way to restore state if this nested transaction will be aborted, + * nor impossible to define the expected behavior. + * * \note In contrast to LMDB, the MDBX required that any opened cursors can be * reused and must be freed explicitly, regardless ones was opened in a * read-only or write transaction. The REASON for this is eliminates ambiguity @@ -4725,7 +5137,7 @@ mdbx_cursor_get_userctx(const MDBX_cursor *cursor); * memory corruption and segfaults. * * \param [in] txn A transaction handle returned by \ref mdbx_txn_begin(). - * \param [in] dbi A database handle returned by \ref mdbx_dbi_open(). + * \param [in] dbi A table handle returned by \ref mdbx_dbi_open(). * \param [in] cursor A cursor handle returned by \ref mdbx_cursor_create(). * * \returns A non-zero error value on failure and 0 on success, @@ -4733,8 +5145,48 @@ mdbx_cursor_get_userctx(const MDBX_cursor *cursor); * \retval MDBX_THREAD_MISMATCH Given transaction is not owned * by current thread. * \retval MDBX_EINVAL An invalid parameter was specified. */ -LIBMDBX_API int mdbx_cursor_bind(const MDBX_txn *txn, MDBX_cursor *cursor, - MDBX_dbi dbi); +LIBMDBX_API int mdbx_cursor_bind(MDBX_txn *txn, MDBX_cursor *cursor, MDBX_dbi dbi); + +/** \brief Unbind cursor from a transaction. + * \ingroup c_cursors + * + * Unbinded cursor is disassociated with any transactions but still holds + * the original DBI-handle internally. Thus it could be renewed with any running + * transaction or closed. + * + * If the transaction is nested, then the cursor should not be used in its parent transaction. + * Otherwise it is no way to restore state if this nested transaction will be aborted, + * nor impossible to define the expected behavior. + * + * \see mdbx_cursor_renew() + * \see mdbx_cursor_bind() + * \see mdbx_cursor_close() + * \see mdbx_cursor_reset() + * + * \note In contrast to LMDB, the MDBX required that any opened cursors can be + * reused and must be freed explicitly, regardless ones was opened in a + * read-only or write transaction. The REASON for this is eliminates ambiguity + * which helps to avoid errors such as: use-after-free, double-free, i.e. + * memory corruption and segfaults. + * + * \param [in] cursor A cursor handle returned by \ref mdbx_cursor_open(). + * + * \returns A non-zero error value on failure and 0 on success. */ +LIBMDBX_API int mdbx_cursor_unbind(MDBX_cursor *cursor); + +/** \brief Сбрасывает состояние курсора. + * \ingroup c_cursors + * + * В результате сброса курсор становится неустановленным и не позволяет + * выполнять операции относительного позиционирования, получения или изменения + * данных, до установки на позицию не зависящую от текущей. Что позволяет + * приложению пресекать дальнейшие операции без предварительного + * позиционирования курсора. + * + * \param [in] cursor Указатель на курсор. + * + * \returns Результат операции сканирования, либо код ошибки. */ +LIBMDBX_API int mdbx_cursor_reset(MDBX_cursor *cursor); /** \brief Create a cursor handle for the specified transaction and DBI handle. * \ingroup c_cursors @@ -4742,7 +5194,7 @@ LIBMDBX_API int mdbx_cursor_bind(const MDBX_txn *txn, MDBX_cursor *cursor, * Using of the `mdbx_cursor_open()` is equivalent to calling * \ref mdbx_cursor_create() and then \ref mdbx_cursor_bind() functions. * - * A cursor cannot be used when its database handle is closed. Nor when its + * A cursor cannot be used when its table handle is closed. Nor when its * transaction has ended, except with \ref mdbx_cursor_bind() and \ref * mdbx_cursor_renew(). Also it can be discarded with \ref mdbx_cursor_close(). * @@ -4757,7 +5209,7 @@ LIBMDBX_API int mdbx_cursor_bind(const MDBX_txn *txn, MDBX_cursor *cursor, * memory corruption and segfaults. * * \param [in] txn A transaction handle returned by \ref mdbx_txn_begin(). - * \param [in] dbi A database handle returned by \ref mdbx_dbi_open(). + * \param [in] dbi A table handle returned by \ref mdbx_dbi_open(). * \param [out] cursor Address where the new \ref MDBX_cursor handle will be * stored. * @@ -4766,15 +5218,19 @@ LIBMDBX_API int mdbx_cursor_bind(const MDBX_txn *txn, MDBX_cursor *cursor, * \retval MDBX_THREAD_MISMATCH Given transaction is not owned * by current thread. * \retval MDBX_EINVAL An invalid parameter was specified. */ -LIBMDBX_API int mdbx_cursor_open(const MDBX_txn *txn, MDBX_dbi dbi, - MDBX_cursor **cursor); +LIBMDBX_API int mdbx_cursor_open(MDBX_txn *txn, MDBX_dbi dbi, MDBX_cursor **cursor); -/** \brief Close a cursor handle. +/** \brief Closes a cursor handle without returning error code. * \ingroup c_cursors * * The cursor handle will be freed and must not be used again after this call, * but its transaction may still be live. * + * This function returns `void` but panic in case of error. Use \ref mdbx_cursor_close2() + * if you need to receive an error code instead of an app crash. + * + * \see mdbx_cursor_close2 + * * \note In contrast to LMDB, the MDBX required that any opened cursors can be * reused and must be freed explicitly, regardless ones was opened in a * read-only or write transaction. The REASON for this is eliminates ambiguity @@ -4785,6 +5241,77 @@ LIBMDBX_API int mdbx_cursor_open(const MDBX_txn *txn, MDBX_dbi dbi, * or \ref mdbx_cursor_create(). */ LIBMDBX_API void mdbx_cursor_close(MDBX_cursor *cursor); +/** \brief Closes a cursor handle with returning error code. + * \ingroup c_cursors + * + * The cursor handle will be freed and must not be used again after this call, + * but its transaction may still be live. + * + * \see mdbx_cursor_close + * + * \note In contrast to LMDB, the MDBX required that any opened cursors can be + * reused and must be freed explicitly, regardless ones was opened in a + * read-only or write transaction. The REASON for this is eliminates ambiguity + * which helps to avoid errors such as: use-after-free, double-free, i.e. + * memory corruption and segfaults. + * + * \param [in] cursor A cursor handle returned by \ref mdbx_cursor_open() + * or \ref mdbx_cursor_create(). + * \returns A non-zero error value on failure and 0 on success, + * some possible errors are: + * \retval MDBX_THREAD_MISMATCH Given transaction is not owned + * by current thread. + * \retval MDBX_EINVAL An invalid parameter was specified. */ +LIBMDBX_API int mdbx_cursor_close2(MDBX_cursor *cursor); + +/** \brief Unbind or closes all cursors of a given transaction and of all + * its parent transactions if ones are. + * \ingroup c_cursors + * + * Unbinds either closes all cursors associated (opened, renewed or binded) with + * the given transaction in a bulk with minimal overhead. + * + * \see mdbx_cursor_unbind() + * \see mdbx_cursor_close() + * + * \param [in] txn A transaction handle returned by \ref mdbx_txn_begin(). + * \param [in] unbind If non-zero, unbinds cursors and leaves ones reusable. + * Otherwise close and dispose cursors. + * \param [in,out] count An optional pointer to return the number of cursors + * processed by the requested operation. + * + * \returns A non-zero error value on failure and 0 on success, + * some possible errors are: + * \retval MDBX_THREAD_MISMATCH Given transaction is not owned + * by current thread. + * \retval MDBX_BAD_TXN Given transaction is invalid or has + * a child/nested transaction transaction. */ +LIBMDBX_API int mdbx_txn_release_all_cursors_ex(const MDBX_txn *txn, bool unbind, size_t *count); + +/** \brief Unbind or closes all cursors of a given transaction and of all + * its parent transactions if ones are. + * \ingroup c_cursors + * + * Unbinds either closes all cursors associated (opened, renewed or binded) with + * the given transaction in a bulk with minimal overhead. + * + * \see mdbx_cursor_unbind() + * \see mdbx_cursor_close() + * + * \param [in] txn A transaction handle returned by \ref mdbx_txn_begin(). + * \param [in] unbind If non-zero, unbinds cursors and leaves ones reusable. + * Otherwise close and dispose cursors. + * + * \returns A non-zero error value on failure and 0 on success, + * some possible errors are: + * \retval MDBX_THREAD_MISMATCH Given transaction is not owned + * by current thread. + * \retval MDBX_BAD_TXN Given transaction is invalid or has + * a child/nested transaction transaction. */ +LIBMDBX_INLINE_API(int, mdbx_txn_release_all_cursors, (const MDBX_txn *txn, bool unbind)) { + return mdbx_txn_release_all_cursors_ex(txn, unbind, NULL); +} + /** \brief Renew a cursor handle for use within the given transaction. * \ingroup c_cursors * @@ -4809,16 +5336,15 @@ LIBMDBX_API void mdbx_cursor_close(MDBX_cursor *cursor); * \retval MDBX_EINVAL An invalid parameter was specified. * \retval MDBX_BAD_DBI The cursor was not bound to a DBI-handle * or such a handle became invalid. */ -LIBMDBX_API int mdbx_cursor_renew(const MDBX_txn *txn, MDBX_cursor *cursor); +LIBMDBX_API int mdbx_cursor_renew(MDBX_txn *txn, MDBX_cursor *cursor); /** \brief Return the cursor's transaction handle. * \ingroup c_cursors * * \param [in] cursor A cursor handle returned by \ref mdbx_cursor_open(). */ -MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API MDBX_txn * -mdbx_cursor_txn(const MDBX_cursor *cursor); +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API MDBX_txn *mdbx_cursor_txn(const MDBX_cursor *cursor); -/** \brief Return the cursor's database handle. +/** \brief Return the cursor's table handle. * \ingroup c_cursors * * \param [in] cursor A cursor handle returned by \ref mdbx_cursor_open(). */ @@ -4836,10 +5362,33 @@ LIBMDBX_API MDBX_dbi mdbx_cursor_dbi(const MDBX_cursor *cursor); * \returns A non-zero error value on failure and 0 on success. */ LIBMDBX_API int mdbx_cursor_copy(const MDBX_cursor *src, MDBX_cursor *dest); +/** \brief Сравнивает позицию курсоров. + * \ingroup c_cursors + * + * Функция предназначена для сравнения позиций двух + * инициализированных/установленных курсоров, связанных с одной транзакцией и + * одной таблицей (DBI-дескриптором). + * Если же курсоры связаны с разными транзакциями, либо с разными таблицами, + * либо один из них не инициализирован, то результат сравнения не определен + * (поведением может быть изменено в последующих версиях). + * + * \param [in] left Левый курсор для сравнения позиций. + * \param [in] right Правый курсор для сравнения позиций. + * \param [in] ignore_multival Булевой флаг, влияющий на результат только при + * сравнении курсоров для таблиц с мульти-значениями, т.е. с флагом + * \ref MDBX_DUPSORT. В случае `true`, позиции курсоров сравниваются + * только по ключам, без учета позиционирования среди мульти-значений. + * Иначе, в случае `false`, при совпадении позиций по ключам, + * сравниваются также позиции по мульти-значениям. + * + * \retval Значение со знаком в семантике оператора `<=>` (меньше нуля, ноль, + * либо больше нуля) как результат сравнения позиций курсоров. */ +LIBMDBX_API int mdbx_cursor_compare(const MDBX_cursor *left, const MDBX_cursor *right, bool ignore_multival); + /** \brief Retrieve by cursor. * \ingroup c_crud * - * This function retrieves key/data pairs from the database. The address and + * This function retrieves key/data pairs from the table. The address and * length of the key are returned in the object to which key refers (except * for the case of the \ref MDBX_SET option, in which the key object is * unchanged), and the address and length of the data are returned in the object @@ -4867,14 +5416,218 @@ LIBMDBX_API int mdbx_cursor_copy(const MDBX_cursor *src, MDBX_cursor *dest); * by current thread. * \retval MDBX_NOTFOUND No matching key found. * \retval MDBX_EINVAL An invalid parameter was specified. */ -LIBMDBX_API int mdbx_cursor_get(MDBX_cursor *cursor, MDBX_val *key, - MDBX_val *data, MDBX_cursor_op op); +LIBMDBX_API int mdbx_cursor_get(MDBX_cursor *cursor, MDBX_val *key, MDBX_val *data, MDBX_cursor_op op); + +/** \brief Служебная функция для использования в утилитах. + * \ingroup c_extra + * + * При использовании определяемых пользователем функций сравнения (aka custom + * comparison functions) проверка порядка ключей может приводить к неверным + * результатам и возврате ошибки \ref MDBX_CORRUPTED. + * + * Эта функция отключает контроль порядка следования ключей на страницах при + * чтении страниц БД для этого курсора, и таким образом, позволяет прочитать + * данные при отсутствии/недоступности использованных функций сравнения. + * \see avoid_custom_comparators + * + * \returns Результат операции сканирования, либо код ошибки. */ +LIBMDBX_API int mdbx_cursor_ignord(MDBX_cursor *cursor); + +/** \brief Тип предикативных функций обратного вызова используемых + * \ref mdbx_cursor_scan() и \ref mdbx_cursor_scan_from() для пробирования + * пар ключ-значения. + * \ingroup c_crud + * + * \param [in,out] context Указатель на контекст с необходимой для оценки + * информацией, который полностью подготавливается + * и контролируется вами. + * \param [in] key Ключ для оценки пользовательской функцией. + * \param [in] value Значение для оценки пользовательской функцией. + * \param [in,out] arg Дополнительный аргумент предикативной функции, + * который полностью подготавливается + * и контролируется вами. + * + * \returns Результат проверки соответствия переданной пары ключ-значения + * искомой цели. Иначе код ошибки, который прерывает сканирование и возвращается + * без изменения в качестве результата из функций \ref mdbx_cursor_scan() + * или \ref mdbx_cursor_scan_from(). + * + * \retval MDBX_RESULT_TRUE если переданная пара ключ-значение соответствует + * искомой и следует завершить сканирование. + * \retval MDBX_RESULT_FALSE если переданная пара ключ-значение НЕ соответствует + * искомой и следует продолжать сканирование. + * \retval ИНАЧЕ любое другое значение, отличное от \ref MDBX_RESULT_TRUE + * и \ref MDBX_RESULT_FALSE, считается индикатором ошибки + * и возвращается без изменений в качестве результата сканирования. + * + * \see mdbx_cursor_scan() + * \see mdbx_cursor_scan_from() */ +typedef int(MDBX_predicate_func)(void *context, MDBX_val *key, MDBX_val *value, void *arg) MDBX_CXX17_NOEXCEPT; + +/** \brief Сканирует таблицу с использованием передаваемого предиката, + * с уменьшением сопутствующих накладных расходов. + * \ingroup c_crud + * + * Реализует функционал сходный с шаблоном `std::find_if<>()` с использованием + * курсора и пользовательской предикативной функции, экономя при этом + * на сопутствующих накладных расходах, в том числе, не выполняя часть проверок + * внутри цикла итерации записей и потенциально уменьшая количество + * DSO-трансграничных вызовов. + * + * Функция принимает курсор, который должен быть привязан к некоторой транзакции + * и DBI-дескриптору таблицы, выполняет первоначальное позиционирование курсора + * определяемое аргументом `start_op`. Далее, производится оценка каждой пары + * ключ-значения посредством предоставляемой вами предикативной функции + * `predicate` и затем, при необходимости, переход к следующему элементу + * посредством операции `turn_op`, до наступления одного из четырех событий: + * - достигается конец данных; + * - возникнет ошибка при позиционировании курсора; + * - оценочная функция вернет \ref MDBX_RESULT_TRUE, сигнализируя + * о необходимости остановить дальнейшее сканирование; + * - оценочная функция возвратит значение отличное от \ref MDBX_RESULT_FALSE + * и \ref MDBX_RESULT_TRUE сигнализируя об ошибке. + * + * \param [in,out] cursor Курсор для выполнения операции сканирования, + * связанный с активной транзакцией и DBI-дескриптором + * таблицы. Например, курсор созданный + * посредством \ref mdbx_cursor_open(). + * \param [in] predicate Предикативная функция для оценки итерируемых + * пар ключ-значения, + * более подробно смотрите \ref MDBX_predicate_func. + * \param [in,out] context Указатель на контекст с необходимой для оценки + * информацией, который полностью подготавливается + * и контролируется вами. + * \param [in] start_op Стартовая операция позиционирования курсора, + * более подробно смотрите \ref MDBX_cursor_op. + * Для сканирования без изменения исходной позиции + * курсора используйте \ref MDBX_GET_CURRENT. + * Допустимые значения \ref MDBX_FIRST, + * \ref MDBX_FIRST_DUP, \ref MDBX_LAST, + * \ref MDBX_LAST_DUP, \ref MDBX_GET_CURRENT, + * а также \ref MDBX_GET_MULTIPLE. + * \param [in] turn_op Операция позиционирования курсора для перехода + * к следующему элементу. Допустимые значения + * \ref MDBX_NEXT, \ref MDBX_NEXT_DUP, + * \ref MDBX_NEXT_NODUP, \ref MDBX_PREV, + * \ref MDBX_PREV_DUP, \ref MDBX_PREV_NODUP, а также + * \ref MDBX_NEXT_MULTIPLE и \ref MDBX_PREV_MULTIPLE. + * \param [in,out] arg Дополнительный аргумент предикативной функции, + * который полностью подготавливается + * и контролируется вами. + * + * \note При использовании \ref MDBX_GET_MULTIPLE, \ref MDBX_NEXT_MULTIPLE + * или \ref MDBX_PREV_MULTIPLE внимательно учитывайте пакетную специфику + * передачи значений через параметры предикативной функции. + * + * \see MDBX_predicate_func + * \see mdbx_cursor_scan_from + * + * \returns Результат операции сканирования, либо код ошибки. + * + * \retval MDBX_RESULT_TRUE если найдена пара ключ-значение, для которой + * предикативная функция вернула \ref MDBX_RESULT_TRUE. + * \retval MDBX_RESULT_FALSE если если подходящая пара ключ-значения НЕ найдена, + * в процессе поиска достигнут конец данных, либо нет данных для поиска. + * \retval ИНАЧЕ любое другое значение, отличное от \ref MDBX_RESULT_TRUE + * и \ref MDBX_RESULT_FALSE, является кодом ошибки при позиционировании + * курса, либо определяемым пользователем кодом остановки поиска + * или ошибочной ситуации. */ +LIBMDBX_API int mdbx_cursor_scan(MDBX_cursor *cursor, MDBX_predicate_func *predicate, void *context, + MDBX_cursor_op start_op, MDBX_cursor_op turn_op, void *arg); + +/** Сканирует таблицу с использованием передаваемого предиката, + * начиная с передаваемой пары ключ-значение, + * с уменьшением сопутствующих накладных расходов. + * \ingroup c_crud + * + * Функция принимает курсор, который должен быть привязан к некоторой транзакции + * и DBI-дескриптору таблицы, выполняет первоначальное позиционирование курсора + * определяемое аргументом `from_op`. а также аргументами `from_key` и + * `from_value`. Далее, производится оценка каждой пары ключ-значения + * посредством предоставляемой вами предикативной функции `predicate` и затем, + * при необходимости, переход к следующему элементу посредством операции + * `turn_op`, до наступления одного из четырех событий: + * - достигается конец данных; + * - возникнет ошибка при позиционировании курсора; + * - оценочная функция вернет \ref MDBX_RESULT_TRUE, сигнализируя + * о необходимости остановить дальнейшее сканирование; + * - оценочная функция возвратит значение отличное от \ref MDBX_RESULT_FALSE + * и \ref MDBX_RESULT_TRUE сигнализируя об ошибке. + * + * \param [in,out] cursor Курсор для выполнения операции сканирования, + * связанный с активной транзакцией и DBI-дескриптором + * таблицы. Например, курсор созданный + * посредством \ref mdbx_cursor_open(). + * \param [in] predicate Предикативная функция для оценки итерируемых + * пар ключ-значения, + * более подробно смотрите \ref MDBX_predicate_func. + * \param [in,out] context Указатель на контекст с необходимой для оценки + * информацией, который полностью подготавливается + * и контролируется вами. + * \param [in] from_op Операция позиционирования курсора к исходной + * позиции, более подробно смотрите + * \ref MDBX_cursor_op. + * Допустимые значения \ref MDBX_GET_BOTH, + * \ref MDBX_GET_BOTH_RANGE, \ref MDBX_SET_KEY, + * \ref MDBX_SET_LOWERBOUND, \ref MDBX_SET_UPPERBOUND, + * \ref MDBX_TO_KEY_LESSER_THAN, + * \ref MDBX_TO_KEY_LESSER_OR_EQUAL, + * \ref MDBX_TO_KEY_EQUAL, + * \ref MDBX_TO_KEY_GREATER_OR_EQUAL, + * \ref MDBX_TO_KEY_GREATER_THAN, + * \ref MDBX_TO_EXACT_KEY_VALUE_LESSER_THAN, + * \ref MDBX_TO_EXACT_KEY_VALUE_LESSER_OR_EQUAL, + * \ref MDBX_TO_EXACT_KEY_VALUE_EQUAL, + * \ref MDBX_TO_EXACT_KEY_VALUE_GREATER_OR_EQUAL, + * \ref MDBX_TO_EXACT_KEY_VALUE_GREATER_THAN, + * \ref MDBX_TO_PAIR_LESSER_THAN, + * \ref MDBX_TO_PAIR_LESSER_OR_EQUAL, + * \ref MDBX_TO_PAIR_EQUAL, + * \ref MDBX_TO_PAIR_GREATER_OR_EQUAL, + * \ref MDBX_TO_PAIR_GREATER_THAN, + * а также \ref MDBX_GET_MULTIPLE. + * \param [in,out] from_key Указатель на ключ используемый как для исходного + * позиционирования, так и для последующих итераций + * перехода. + * \param [in,out] from_value Указатель на значние используемое как для + * исходного позиционирования, так и для последующих + * итераций перехода. + * \param [in] turn_op Операция позиционирования курсора для перехода + * к следующему элементу. Допустимые значения + * \ref MDBX_NEXT, \ref MDBX_NEXT_DUP, + * \ref MDBX_NEXT_NODUP, \ref MDBX_PREV, + * \ref MDBX_PREV_DUP, \ref MDBX_PREV_NODUP, а также + * \ref MDBX_NEXT_MULTIPLE и \ref MDBX_PREV_MULTIPLE. + * \param [in,out] arg Дополнительный аргумент предикативной функции, + * который полностью подготавливается + * и контролируется вами. + * + * \note При использовании \ref MDBX_GET_MULTIPLE, \ref MDBX_NEXT_MULTIPLE + * или \ref MDBX_PREV_MULTIPLE внимательно учитывайте пакетную специфику + * передачи значений через параметры предикативной функции. + * + * \see MDBX_predicate_func + * \see mdbx_cursor_scan + * + * \returns Результат операции сканирования, либо код ошибки. + * + * \retval MDBX_RESULT_TRUE если найдена пара ключ-значение, для которой + * предикативная функция вернула \ref MDBX_RESULT_TRUE. + * \retval MDBX_RESULT_FALSE если если подходящая пара ключ-значения НЕ найдена, + * в процессе поиска достигнут конец данных, либо нет данных для поиска. + * \retval ИНАЧЕ любое другое значение, отличное от \ref MDBX_RESULT_TRUE + * и \ref MDBX_RESULT_FALSE, является кодом ошибки при позиционировании + * курса, либо определяемым пользователем кодом остановки поиска + * или ошибочной ситуации. */ +LIBMDBX_API int mdbx_cursor_scan_from(MDBX_cursor *cursor, MDBX_predicate_func *predicate, void *context, + MDBX_cursor_op from_op, MDBX_val *from_key, MDBX_val *from_value, + MDBX_cursor_op turn_op, void *arg); /** \brief Retrieve multiple non-dupsort key/value pairs by cursor. * \ingroup c_crud * - * This function retrieves multiple key/data pairs from the database without - * \ref MDBX_DUPSORT option. For `MDBX_DUPSORT` databases please + * This function retrieves multiple key/data pairs from the table without + * \ref MDBX_DUPSORT option. For `MDBX_DUPSORT` tables please * use \ref MDBX_GET_MULTIPLE and \ref MDBX_NEXT_MULTIPLE. * * The number of key and value items is returned in the `size_t count` @@ -4900,27 +5653,24 @@ LIBMDBX_API int mdbx_cursor_get(MDBX_cursor *cursor, MDBX_val *key, * \param [in] limit The size of pairs buffer as the number of items, * but not a pairs. * \param [in] op A cursor operation \ref MDBX_cursor_op (only - * \ref MDBX_FIRST, \ref MDBX_NEXT, \ref MDBX_GET_CURRENT - * are supported). + * \ref MDBX_FIRST and \ref MDBX_NEXT are supported). * * \returns A non-zero error value on failure and 0 on success, * some possible errors are: * \retval MDBX_THREAD_MISMATCH Given transaction is not owned * by current thread. - * \retval MDBX_NOTFOUND No more key-value pairs are available. + * \retval MDBX_NOTFOUND No any key-value pairs are available. * \retval MDBX_ENODATA The cursor is already at the end of data. - * \retval MDBX_RESULT_TRUE The specified limit is less than the available - * key-value pairs on the current page/position - * that the cursor points to. + * \retval MDBX_RESULT_TRUE The returned chunk is the last one, + * and there are no pairs left. * \retval MDBX_EINVAL An invalid parameter was specified. */ -LIBMDBX_API int mdbx_cursor_get_batch(MDBX_cursor *cursor, size_t *count, - MDBX_val *pairs, size_t limit, +LIBMDBX_API int mdbx_cursor_get_batch(MDBX_cursor *cursor, size_t *count, MDBX_val *pairs, size_t limit, MDBX_cursor_op op); /** \brief Store by cursor. * \ingroup c_crud * - * This function stores key/data pairs into the database. The cursor is + * This function stores key/data pairs into the table. The cursor is * positioned at the new item, or on failure usually near it. * * \param [in] cursor A cursor handle returned by \ref mdbx_cursor_open(). @@ -4941,14 +5691,14 @@ LIBMDBX_API int mdbx_cursor_get_batch(MDBX_cursor *cursor, size_t *count, * * - \ref MDBX_NODUPDATA * Enter the new key-value pair only if it does not already appear in the - * database. This flag may only be specified if the database was opened + * table. This flag may only be specified if the table was opened * with \ref MDBX_DUPSORT. The function will return \ref MDBX_KEYEXIST - * if the key/data pair already appears in the database. + * if the key/data pair already appears in the table. * * - \ref MDBX_NOOVERWRITE * Enter the new key/data pair only if the key does not already appear - * in the database. The function will return \ref MDBX_KEYEXIST if the key - * already appears in the database, even if the database supports + * in the table. The function will return \ref MDBX_KEYEXIST if the key + * already appears in the table, even if the table supports * duplicates (\ref MDBX_DUPSORT). * * - \ref MDBX_RESERVE @@ -4956,11 +5706,11 @@ LIBMDBX_API int mdbx_cursor_get_batch(MDBX_cursor *cursor, size_t *count, * data. Instead, return a pointer to the reserved space, which the * caller can fill in later - before the next update operation or the * transaction ends. This saves an extra memcpy if the data is being - * generated later. This flag must not be specified if the database + * generated later. This flag must not be specified if the table * was opened with \ref MDBX_DUPSORT. * * - \ref MDBX_APPEND - * Append the given key/data pair to the end of the database. No key + * Append the given key/data pair to the end of the table. No key * comparisons are performed. This option allows fast bulk loading when * keys are already known to be in the correct order. Loading unsorted * keys with this flag will cause a \ref MDBX_KEYEXIST error. @@ -4970,14 +5720,14 @@ LIBMDBX_API int mdbx_cursor_get_batch(MDBX_cursor *cursor, size_t *count, * * - \ref MDBX_MULTIPLE * Store multiple contiguous data elements in a single request. This flag - * may only be specified if the database was opened with + * may only be specified if the table was opened with * \ref MDBX_DUPFIXED. With combination the \ref MDBX_ALLDUPS * will replace all multi-values. * The data argument must be an array of two \ref MDBX_val. The `iov_len` * of the first \ref MDBX_val must be the size of a single data element. * The `iov_base` of the first \ref MDBX_val must point to the beginning * of the array of contiguous data elements which must be properly aligned - * in case of database with \ref MDBX_INTEGERDUP flag. + * in case of table with \ref MDBX_INTEGERDUP flag. * The `iov_len` of the second \ref MDBX_val must be the count of the * number of data elements to store. On return this field will be set to * the count of the number of elements actually written. The `iov_base` of @@ -4997,8 +5747,7 @@ LIBMDBX_API int mdbx_cursor_get_batch(MDBX_cursor *cursor, size_t *count, * \retval MDBX_EACCES An attempt was made to write in a read-only * transaction. * \retval MDBX_EINVAL An invalid parameter was specified. */ -LIBMDBX_API int mdbx_cursor_put(MDBX_cursor *cursor, const MDBX_val *key, - MDBX_val *data, MDBX_put_flags_t flags); +LIBMDBX_API int mdbx_cursor_put(MDBX_cursor *cursor, const MDBX_val *key, MDBX_val *data, MDBX_put_flags_t flags); /** \brief Delete current key/data pair. * \ingroup c_crud @@ -5016,7 +5765,7 @@ LIBMDBX_API int mdbx_cursor_put(MDBX_cursor *cursor, const MDBX_val *key, * - \ref MDBX_ALLDUPS * or \ref MDBX_NODUPDATA (supported for compatibility) * Delete all of the data items for the current key. This flag has effect - * only for database(s) was created with \ref MDBX_DUPSORT. + * only for table(s) was created with \ref MDBX_DUPSORT. * * \see \ref c_crud_hints "Quick reference for Insert/Update/Delete operations" * @@ -5032,10 +5781,12 @@ LIBMDBX_API int mdbx_cursor_put(MDBX_cursor *cursor, const MDBX_val *key, * \retval MDBX_EINVAL An invalid parameter was specified. */ LIBMDBX_API int mdbx_cursor_del(MDBX_cursor *cursor, MDBX_put_flags_t flags); -/** \brief Return count of duplicates for current key. +/** \brief Return count values (aka duplicates) for current key. * \ingroup c_crud * - * This call is valid for all databases, but reasonable only for that support + * \see mdbx_cursor_count_ex + * + * This call is valid for all tables, but reasonable only for that support * sorted duplicate data items \ref MDBX_DUPSORT. * * \param [in] cursor A cursor handle returned by \ref mdbx_cursor_open(). @@ -5049,6 +5800,30 @@ LIBMDBX_API int mdbx_cursor_del(MDBX_cursor *cursor, MDBX_put_flags_t flags); * was specified. */ LIBMDBX_API int mdbx_cursor_count(const MDBX_cursor *cursor, size_t *count); +/** \brief Return count values (aka duplicates) and nested b-tree statistics for current key. + * \ingroup c_crud + * + * \see mdbx_dbi_stat + * \see mdbx_dbi_dupsort_depthmask + * \see mdbx_cursor_count + * + * This call is valid for all tables, but reasonable only for that support + * sorted duplicate data items \ref MDBX_DUPSORT. + * + * \param [in] cursor A cursor handle returned by \ref mdbx_cursor_open(). + * \param [out] count Address where the count will be stored. + * \param [out] stat The address of an \ref MDBX_stat structure where + * the statistics of a nested b-tree will be copied. + * \param [in] bytes The size of \ref MDBX_stat. + * + * \returns A non-zero error value on failure and 0 on success, + * some possible errors are: + * \retval MDBX_THREAD_MISMATCH Given transaction is not owned + * by current thread. + * \retval MDBX_EINVAL Cursor is not initialized, or an invalid parameter + * was specified. */ +LIBMDBX_API int mdbx_cursor_count_ex(const MDBX_cursor *cursor, size_t *count, MDBX_stat *stat, size_t bytes); + /** \brief Determines whether the cursor is pointed to a key-value pair or not, * i.e. was not positioned or points to the end of data. * \ingroup c_cursors @@ -5056,13 +5831,12 @@ LIBMDBX_API int mdbx_cursor_count(const MDBX_cursor *cursor, size_t *count); * \param [in] cursor A cursor handle returned by \ref mdbx_cursor_open(). * * \returns A \ref MDBX_RESULT_TRUE or \ref MDBX_RESULT_FALSE value, - * otherwise the error code: + * otherwise the error code. * \retval MDBX_RESULT_TRUE No more data available or cursor not * positioned * \retval MDBX_RESULT_FALSE A data is available * \retval Otherwise the error code */ -MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int -mdbx_cursor_eof(const MDBX_cursor *cursor); +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int mdbx_cursor_eof(const MDBX_cursor *cursor); /** \brief Determines whether the cursor is pointed to the first key-value pair * or not. @@ -5071,12 +5845,24 @@ mdbx_cursor_eof(const MDBX_cursor *cursor); * \param [in] cursor A cursor handle returned by \ref mdbx_cursor_open(). * * \returns A MDBX_RESULT_TRUE or MDBX_RESULT_FALSE value, - * otherwise the error code: + * otherwise the error code. * \retval MDBX_RESULT_TRUE Cursor positioned to the first key-value pair * \retval MDBX_RESULT_FALSE Cursor NOT positioned to the first key-value * pair \retval Otherwise the error code */ -MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int -mdbx_cursor_on_first(const MDBX_cursor *cursor); +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int mdbx_cursor_on_first(const MDBX_cursor *cursor); + +/** \brief Определяет стоит ли курсор на первом или единственном + * мульти-значении соответствующем ключу. + * \ingroup c_cursors + * \param [in] cursor Курсор созданный посредством \ref mdbx_cursor_open(). + * \returns Значание \ref MDBX_RESULT_TRUE, либо \ref MDBX_RESULT_FALSE, + * иначе код ошибки. + * \retval MDBX_RESULT_TRUE курсор установлен на первом или единственном + * мульти-значении соответствующем ключу. + * \retval MDBX_RESULT_FALSE курсор НЕ установлен на первом или единственном + * мульти-значении соответствующем ключу. + * \retval ИНАЧЕ код ошибки. */ +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int mdbx_cursor_on_first_dup(const MDBX_cursor *cursor); /** \brief Determines whether the cursor is pointed to the last key-value pair * or not. @@ -5085,12 +5871,24 @@ mdbx_cursor_on_first(const MDBX_cursor *cursor); * \param [in] cursor A cursor handle returned by \ref mdbx_cursor_open(). * * \returns A \ref MDBX_RESULT_TRUE or \ref MDBX_RESULT_FALSE value, - * otherwise the error code: + * otherwise the error code. * \retval MDBX_RESULT_TRUE Cursor positioned to the last key-value pair * \retval MDBX_RESULT_FALSE Cursor NOT positioned to the last key-value pair * \retval Otherwise the error code */ -MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int -mdbx_cursor_on_last(const MDBX_cursor *cursor); +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int mdbx_cursor_on_last(const MDBX_cursor *cursor); + +/** \brief Определяет стоит ли курсор на последнем или единственном + * мульти-значении соответствующем ключу. + * \ingroup c_cursors + * \param [in] cursor Курсор созданный посредством \ref mdbx_cursor_open(). + * \returns Значание \ref MDBX_RESULT_TRUE, либо \ref MDBX_RESULT_FALSE, + * иначе код ошибки. + * \retval MDBX_RESULT_TRUE курсор установлен на последнем или единственном + * мульти-значении соответствующем ключу. + * \retval MDBX_RESULT_FALSE курсор НЕ установлен на последнем или единственном + * мульти-значении соответствующем ключу. + * \retval ИНАЧЕ код ошибки. */ +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int mdbx_cursor_on_last_dup(const MDBX_cursor *cursor); /** \addtogroup c_rqest * \details \note The estimation result varies greatly depending on the filling @@ -5127,7 +5925,7 @@ mdbx_cursor_on_last(const MDBX_cursor *cursor); * Please see notes on accuracy of the result in the details * of \ref c_rqest section. * - * Both cursors must be initialized for the same database and the same + * Both cursors must be initialized for the same table and the same * transaction. * * \param [in] first The first cursor for estimation. @@ -5136,9 +5934,7 @@ mdbx_cursor_on_last(const MDBX_cursor *cursor); * i.e. `*distance_items = distance(first, last)`. * * \returns A non-zero error value on failure and 0 on success. */ -LIBMDBX_API int mdbx_estimate_distance(const MDBX_cursor *first, - const MDBX_cursor *last, - ptrdiff_t *distance_items); +LIBMDBX_API int mdbx_estimate_distance(const MDBX_cursor *first, const MDBX_cursor *last, ptrdiff_t *distance_items); /** \brief Estimates the move distance. * \ingroup c_rqest @@ -5160,8 +5956,7 @@ LIBMDBX_API int mdbx_estimate_distance(const MDBX_cursor *first, * as the number of elements. * * \returns A non-zero error value on failure and 0 on success. */ -LIBMDBX_API int mdbx_estimate_move(const MDBX_cursor *cursor, MDBX_val *key, - MDBX_val *data, MDBX_cursor_op move_op, +LIBMDBX_API int mdbx_estimate_move(const MDBX_cursor *cursor, MDBX_val *key, MDBX_val *data, MDBX_cursor_op move_op, ptrdiff_t *distance_items); /** \brief Estimates the size of a range as a number of elements. @@ -5176,7 +5971,7 @@ LIBMDBX_API int mdbx_estimate_move(const MDBX_cursor *cursor, MDBX_val *key, * * \param [in] txn A transaction handle returned * by \ref mdbx_txn_begin(). - * \param [in] dbi A database handle returned by \ref mdbx_dbi_open(). + * \param [in] dbi A table handle returned by \ref mdbx_dbi_open(). * \param [in] begin_key The key of range beginning or NULL for explicit FIRST. * \param [in] begin_data Optional additional data to seeking among sorted * duplicates. @@ -5188,11 +5983,8 @@ LIBMDBX_API int mdbx_estimate_move(const MDBX_cursor *cursor, MDBX_val *key, * \param [out] distance_items A pointer to store range estimation result. * * \returns A non-zero error value on failure and 0 on success. */ -LIBMDBX_API int mdbx_estimate_range(const MDBX_txn *txn, MDBX_dbi dbi, - const MDBX_val *begin_key, - const MDBX_val *begin_data, - const MDBX_val *end_key, - const MDBX_val *end_data, +LIBMDBX_API int mdbx_estimate_range(const MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *begin_key, + const MDBX_val *begin_data, const MDBX_val *end_key, const MDBX_val *end_data, ptrdiff_t *distance_items); /** \brief The EPSILON value for mdbx_estimate_range() @@ -5228,25 +6020,24 @@ LIBMDBX_API int mdbx_estimate_range(const MDBX_txn *txn, MDBX_dbi dbi, * \param [in] ptr The address of data to check. * * \returns A MDBX_RESULT_TRUE or MDBX_RESULT_FALSE value, - * otherwise the error code: + * otherwise the error code. * \retval MDBX_RESULT_TRUE Given address is on the dirty page. * \retval MDBX_RESULT_FALSE Given address is NOT on the dirty page. * \retval Otherwise the error code. */ -MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int mdbx_is_dirty(const MDBX_txn *txn, - const void *ptr); +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int mdbx_is_dirty(const MDBX_txn *txn, const void *ptr); -/** \brief Sequence generation for a database. +/** \brief Sequence generation for a table. * \ingroup c_crud * * The function allows to create a linear sequence of unique positive integers - * for each database. The function can be called for a read transaction to + * for each table. The function can be called for a read transaction to * retrieve the current sequence value, and the increment must be zero. * Sequence changes become visible outside the current write transaction after * it is committed, and discarded on abort. * * \param [in] txn A transaction handle returned * by \ref mdbx_txn_begin(). - * \param [in] dbi A database handle returned by \ref mdbx_dbi_open(). + * \param [in] dbi A table handle returned by \ref mdbx_dbi_open(). * \param [out] result The optional address where the value of sequence * before the change will be stored. * \param [in] increment Value to increase the sequence, @@ -5256,58 +6047,51 @@ MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int mdbx_is_dirty(const MDBX_txn *txn, * some possible errors are: * \retval MDBX_RESULT_TRUE Increasing the sequence has resulted in an * overflow and therefore cannot be executed. */ -LIBMDBX_API int mdbx_dbi_sequence(MDBX_txn *txn, MDBX_dbi dbi, uint64_t *result, - uint64_t increment); +LIBMDBX_API int mdbx_dbi_sequence(MDBX_txn *txn, MDBX_dbi dbi, uint64_t *result, uint64_t increment); -/** \brief Compare two keys according to a particular database. +/** \brief Compare two keys according to a particular table. * \ingroup c_crud * \see MDBX_cmp_func * * This returns a comparison as if the two data items were keys in the - * specified database. + * specified table. * - * \warning There ss a Undefined behavior if one of arguments is invalid. + * \warning There is a Undefined behavior if one of arguments is invalid. * * \param [in] txn A transaction handle returned by \ref mdbx_txn_begin(). - * \param [in] dbi A database handle returned by \ref mdbx_dbi_open(). + * \param [in] dbi A table handle returned by \ref mdbx_dbi_open(). * \param [in] a The first item to compare. * \param [in] b The second item to compare. * * \returns < 0 if a < b, 0 if a == b, > 0 if a > b */ -MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int mdbx_cmp(const MDBX_txn *txn, - MDBX_dbi dbi, - const MDBX_val *a, +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int mdbx_cmp(const MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *a, const MDBX_val *b); -/** \brief Returns default internal key's comparator for given database flags. +/** \brief Returns default internal key's comparator for given table flags. * \ingroup c_extra */ -MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API MDBX_cmp_func * -mdbx_get_keycmp(MDBX_db_flags_t flags); +MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API MDBX_cmp_func *mdbx_get_keycmp(MDBX_db_flags_t flags); -/** \brief Compare two data items according to a particular database. +/** \brief Compare two data items according to a particular table. * \ingroup c_crud * \see MDBX_cmp_func * * This returns a comparison as if the two items were data items of the - * specified database. + * specified table. * - * \warning There ss a Undefined behavior if one of arguments is invalid. + * \warning There is a Undefined behavior if one of arguments is invalid. * * \param [in] txn A transaction handle returned by \ref mdbx_txn_begin(). - * \param [in] dbi A database handle returned by \ref mdbx_dbi_open(). + * \param [in] dbi A table handle returned by \ref mdbx_dbi_open(). * \param [in] a The first item to compare. * \param [in] b The second item to compare. * * \returns < 0 if a < b, 0 if a == b, > 0 if a > b */ -MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int mdbx_dcmp(const MDBX_txn *txn, - MDBX_dbi dbi, - const MDBX_val *a, +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API int mdbx_dcmp(const MDBX_txn *txn, MDBX_dbi dbi, const MDBX_val *a, const MDBX_val *b); -/** \brief Returns default internal data's comparator for given database flags +/** \brief Returns default internal data's comparator for given table flags * \ingroup c_extra */ -MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API MDBX_cmp_func * -mdbx_get_datacmp(MDBX_db_flags_t flags); +MDBX_NOTHROW_CONST_FUNCTION LIBMDBX_API MDBX_cmp_func *mdbx_get_datacmp(MDBX_db_flags_t flags); /** \brief A callback function used to enumerate the reader lock table. * \ingroup c_statinfo @@ -5334,10 +6118,8 @@ mdbx_get_datacmp(MDBX_db_flags_t flags); * for reuse by completion read transaction. * * \returns < 0 on failure, >= 0 on success. \see mdbx_reader_list() */ -typedef int(MDBX_reader_list_func)(void *ctx, int num, int slot, mdbx_pid_t pid, - mdbx_tid_t thread, uint64_t txnid, - uint64_t lag, size_t bytes_used, - size_t bytes_retained) MDBX_CXX17_NOEXCEPT; +typedef int(MDBX_reader_list_func)(void *ctx, int num, int slot, mdbx_pid_t pid, mdbx_tid_t thread, uint64_t txnid, + uint64_t lag, size_t bytes_used, size_t bytes_retained) MDBX_CXX17_NOEXCEPT; /** \brief Enumerate the entries in the reader lock table. * @@ -5350,8 +6132,7 @@ typedef int(MDBX_reader_list_func)(void *ctx, int num, int slot, mdbx_pid_t pid, * * \returns A non-zero error value on failure and 0 on success, * or \ref MDBX_RESULT_TRUE if the reader lock table is empty. */ -LIBMDBX_API int mdbx_reader_list(const MDBX_env *env, - MDBX_reader_list_func *func, void *ctx); +LIBMDBX_API int mdbx_reader_list(const MDBX_env *env, MDBX_reader_list_func *func, void *ctx); /** \brief Check for stale entries in the reader lock table. * \ingroup c_extra @@ -5375,8 +6156,7 @@ LIBMDBX_API int mdbx_reader_check(MDBX_env *env, int *dead); * * \returns Number of transactions committed after the given was started for * read, or negative value on failure. */ -MDBX_DEPRECATED LIBMDBX_API int mdbx_txn_straggler(const MDBX_txn *txn, - int *percent); +MDBX_DEPRECATED LIBMDBX_API int mdbx_txn_straggler(const MDBX_txn *txn, int *percent); /** \brief Registers the current thread as a reader for the environment. * \ingroup c_extra @@ -5425,6 +6205,7 @@ LIBMDBX_API int mdbx_thread_unregister(const MDBX_env *env); * with a "long-lived" read transactions. * \see mdbx_env_set_hsr() * \see mdbx_env_get_hsr() + * \see mdbx_txn_park() * \see Long-lived read transactions * * Using this callback you can choose how to resolve the situation: @@ -5485,10 +6266,8 @@ LIBMDBX_API int mdbx_thread_unregister(const MDBX_env *env); * \retval 2 or great The reader process was terminated or killed, * and libmdbx should entirely reset reader registration. */ -typedef int(MDBX_hsr_func)(const MDBX_env *env, const MDBX_txn *txn, - mdbx_pid_t pid, mdbx_tid_t tid, uint64_t laggard, - unsigned gap, size_t space, - int retry) MDBX_CXX17_NOEXCEPT; +typedef int(MDBX_hsr_func)(const MDBX_env *env, const MDBX_txn *txn, mdbx_pid_t pid, mdbx_tid_t tid, uint64_t laggard, + unsigned gap, size_t space, int retry) MDBX_CXX17_NOEXCEPT; /** \brief Sets a Handle-Slow-Readers callback to resolve database full/overflow * issue due to a reader(s) which prevents the old data from being recycled. @@ -5499,6 +6278,7 @@ typedef int(MDBX_hsr_func)(const MDBX_env *env, const MDBX_txn *txn, * * \see MDBX_hsr_func * \see mdbx_env_get_hsr() + * \see mdbx_txn_park() * \see Long-lived read transactions * * \param [in] env An environment handle returned @@ -5514,57 +6294,30 @@ LIBMDBX_API int mdbx_env_set_hsr(MDBX_env *env, MDBX_hsr_func *hsr_callback); * recycled. * \see MDBX_hsr_func * \see mdbx_env_set_hsr() + * \see mdbx_txn_park() * \see Long-lived read transactions * * \param [in] env An environment handle returned by \ref mdbx_env_create(). * * \returns A MDBX_hsr_func function or NULL if disabled * or something wrong. */ -MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API MDBX_hsr_func * -mdbx_env_get_hsr(const MDBX_env *env); +MDBX_NOTHROW_PURE_FUNCTION LIBMDBX_API MDBX_hsr_func *mdbx_env_get_hsr(const MDBX_env *env); -/** \defgroup btree_traversal B-tree Traversal - * This is internal API for mdbx_chk tool. You should avoid to use it, except - * some extremal special cases. +/** \defgroup chk Checking and Recovery + * Basically this is internal API for `mdbx_chk` tool, etc. + * You should avoid to use it, except some extremal special cases. * \ingroup c_extra * @{ */ -/** \brief Page types for traverse the b-tree. - * \see mdbx_env_pgwalk() \see MDBX_pgvisitor_func */ -enum MDBX_page_type_t { - MDBX_page_broken, - MDBX_page_meta, - MDBX_page_large, - MDBX_page_branch, - MDBX_page_leaf, - MDBX_page_dupfixed_leaf, - MDBX_subpage_leaf, - MDBX_subpage_dupfixed_leaf, - MDBX_subpage_broken, -}; -#ifndef __cplusplus -typedef enum MDBX_page_type_t MDBX_page_type_t; -#endif - -/** \brief Pseudo-name for MainDB */ -#define MDBX_PGWALK_MAIN ((void *)((ptrdiff_t)0)) -/** \brief Pseudo-name for GarbageCollectorDB */ -#define MDBX_PGWALK_GC ((void *)((ptrdiff_t)-1)) -/** \brief Pseudo-name for MetaPages */ -#define MDBX_PGWALK_META ((void *)((ptrdiff_t)-2)) - -/** \brief Callback function for traverse the b-tree. \see mdbx_env_pgwalk() */ -typedef int -MDBX_pgvisitor_func(const uint64_t pgno, const unsigned number, void *const ctx, - const int deep, const MDBX_val *dbi_name, - const size_t page_size, const MDBX_page_type_t type, - const MDBX_error_t err, const size_t nentries, - const size_t payload_bytes, const size_t header_bytes, - const size_t unused_bytes) MDBX_CXX17_NOEXCEPT; +/** \brief Acquires write-transaction lock. + * Provided for custom and/or complex locking scenarios. + * \returns A non-zero error value on failure and 0 on success. */ +LIBMDBX_API int mdbx_txn_lock(MDBX_env *env, bool dont_wait); -/** \brief B-tree traversal function. */ -LIBMDBX_API int mdbx_env_pgwalk(MDBX_txn *txn, MDBX_pgvisitor_func *visitor, - void *ctx, bool dont_check_keys_ordering); +/** \brief Releases write-transaction lock. + * Provided for custom and/or complex locking scenarios. + * \returns A non-zero error value on failure and 0 on success. */ +LIBMDBX_API int mdbx_txn_unlock(MDBX_env *env); /** \brief Open an environment instance using specific meta-page * for checking and recovery. @@ -5575,18 +6328,20 @@ LIBMDBX_API int mdbx_env_pgwalk(MDBX_txn *txn, MDBX_pgvisitor_func *visitor, * * \note On Windows the \ref mdbx_env_open_for_recoveryW() is recommended * to use. */ -LIBMDBX_API int mdbx_env_open_for_recovery(MDBX_env *env, const char *pathname, - unsigned target_meta, - bool writeable); +LIBMDBX_API int mdbx_env_open_for_recovery(MDBX_env *env, const char *pathname, unsigned target_meta, bool writeable); #if defined(_WIN32) || defined(_WIN64) || defined(DOXYGEN) /** \copydoc mdbx_env_open_for_recovery() + * \ingroup c_extra * \note Available only on Windows. * \see mdbx_env_open_for_recovery() */ -LIBMDBX_API int mdbx_env_open_for_recoveryW(MDBX_env *env, - const wchar_t *pathname, - unsigned target_meta, +LIBMDBX_API int mdbx_env_open_for_recoveryW(MDBX_env *env, const wchar_t *pathname, unsigned target_meta, bool writeable); +#define mdbx_env_open_for_recoveryT(env, pathname, target_mets, writeable) \ + mdbx_env_open_for_recoveryW(env, pathname, target_mets, writeable) +#else +#define mdbx_env_open_for_recoveryT(env, pathname, target_mets, writeable) \ + mdbx_env_open_for_recovery(env, pathname, target_mets, writeable) #endif /* Windows */ /** \brief Turn database to the specified meta-page. @@ -5596,7 +6351,298 @@ LIBMDBX_API int mdbx_env_open_for_recoveryW(MDBX_env *env, * leg(s). */ LIBMDBX_API int mdbx_env_turn_for_recovery(MDBX_env *env, unsigned target_meta); -/** end of btree_traversal @} */ +/** \brief Получает базовую информацию о БД не открывая её. + * \ingroup c_opening + * + * Назначение функции в получении базовой информации без открытия БД и + * отображения данных в память (что может быть достаточно затратным действием + * для ядра ОС). Полученная таким образом информация может быть полезной для + * подстройки опций работы с БД перед её открытием, а также в сценариях файловых + * менеджерах и прочих вспомогательных утилитах. + * + * \todo Добавить в API возможность установки обратного вызова для ревизии опций + * работы с БД в процессе её открытия (при удержании блокировок). + * + * \param [in] pathname Путь к директории или файлу БД. + * \param [out] info Указатель на структуру \ref MDBX_envinfo + * для получения информации. + * \param [in] bytes Актуальный размер структуры \ref MDBX_envinfo, это + * значение используется для обеспечения совместимости + * ABI. + * + * \note Заполняется только некоторые поля структуры \ref MDBX_envinfo, значения + * которых возможно получить без отображения файлов БД в память и без захвата + * блокировок: размер страницы БД, геометрия БД, размер распределенного места + * (номер последней распределенной страницы), номер последней транзакции и + * boot-id. + * + * \warning Полученная информация является снимком на время выполнения функции и + * может быть в любой момент изменена работающим с БД процессом. В том числе, + * нет препятствий к тому, чтобы другой процесс удалил БД и создал её заново с + * другим размером страницы и/или изменением любых других параметров. + * + * \returns Ненулевое значение кода ошибки, либо 0 при успешном выполнении. */ +LIBMDBX_API int mdbx_preopen_snapinfo(const char *pathname, MDBX_envinfo *info, size_t bytes); +#if defined(_WIN32) || defined(_WIN64) || defined(DOXYGEN) +/** \copydoc mdbx_preopen_snapinfo() + * \ingroup c_opening + * \note Available only on Windows. + * \see mdbx_preopen_snapinfo() */ +LIBMDBX_API int mdbx_preopen_snapinfoW(const wchar_t *pathname, MDBX_envinfo *info, size_t bytes); +#define mdbx_preopen_snapinfoT(pathname, info, bytes) mdbx_preopen_snapinfoW(pathname, info, bytes) +#else +#define mdbx_preopen_snapinfoT(pathname, info, bytes) mdbx_preopen_snapinfo(pathname, info, bytes) +#endif /* Windows */ + +/** \brief Флаги/опции для проверки целостности базы данных. + * \note Данный API еще не зафиксирован, в последующих версиях могут быть + * незначительные доработки и изменения. + * \see mdbx_env_chk() */ +typedef enum MDBX_chk_flags { + /** Режим проверки по-умолчанию, в том числе в режиме только-чтения. */ + MDBX_CHK_DEFAULTS = 0, + + /** Проверка в режиме чтения-записи, с захватом блокировки и приостановки + * пишущих транзакций. */ + MDBX_CHK_READWRITE = 1, + + /** Пропустить обход дерева страниц. */ + MDBX_CHK_SKIP_BTREE_TRAVERSAL = 2, + + /** Пропустить просмотр записей ключ-значение. */ + MDBX_CHK_SKIP_KV_TRAVERSAL = 4, + + /** Игнорировать порядок ключей и записей. + * \note Требуется при проверке унаследованных БД созданных с использованием + * нестандартных (пользовательских) функций сравнения ключей или значений. */ + MDBX_CHK_IGNORE_ORDER = 8 +} MDBX_chk_flags_t; +DEFINE_ENUM_FLAG_OPERATORS(MDBX_chk_flags) + +/** \brief Уровни логирование/детализации информации, + * поставляемой через обратные вызовы при проверке целостности базы данных. + * \see mdbx_env_chk() */ +typedef enum MDBX_chk_severity { + MDBX_chk_severity_prio_shift = 4, + MDBX_chk_severity_kind_mask = 0xF, + MDBX_chk_fatal = 0x00u, + MDBX_chk_error = 0x11u, + MDBX_chk_warning = 0x22u, + MDBX_chk_notice = 0x33u, + MDBX_chk_result = 0x44u, + MDBX_chk_resolution = 0x55u, + MDBX_chk_processing = 0x56u, + MDBX_chk_info = 0x67u, + MDBX_chk_verbose = 0x78u, + MDBX_chk_details = 0x89u, + MDBX_chk_extra = 0x9Au +} MDBX_chk_severity_t; + +/** \brief Стадии проверки, + * сообщаемые через обратные вызовы при проверке целостности базы данных. + * \see mdbx_env_chk() */ +typedef enum MDBX_chk_stage { + MDBX_chk_none, + MDBX_chk_init, + MDBX_chk_lock, + MDBX_chk_meta, + MDBX_chk_tree, + MDBX_chk_gc, + MDBX_chk_space, + MDBX_chk_maindb, + MDBX_chk_tables, + MDBX_chk_conclude, + MDBX_chk_unlock, + MDBX_chk_finalize +} MDBX_chk_stage_t; + +/** \brief Виртуальная строка отчета, формируемого при проверке целостности базы + * данных. \see mdbx_env_chk() */ +typedef struct MDBX_chk_line { + struct MDBX_chk_context *ctx; + uint8_t severity, scope_depth, empty; + char *begin, *end, *out; +} MDBX_chk_line_t; + +/** \brief Проблема обнаруженная при проверке целостности базы данных. + * \see mdbx_env_chk() */ +typedef struct MDBX_chk_issue { + struct MDBX_chk_issue *next; + size_t count; + const char *caption; +} MDBX_chk_issue_t; + +/** \brief Иерархический контекст при проверке целостности базы данных. + * \see mdbx_env_chk() */ +typedef struct MDBX_chk_scope { + MDBX_chk_issue_t *issues; + struct MDBX_chk_internal *internal; + const void *object; + MDBX_chk_stage_t stage; + MDBX_chk_severity_t verbosity; + size_t subtotal_issues; + union { + void *ptr; + size_t number; + } usr_z, usr_v, usr_o; +} MDBX_chk_scope_t; + +/** \brief Пользовательский тип для привязки дополнительных данных, + * связанных с некоторой таблицей ключ-значение, при проверке целостности базы + * данных. \see mdbx_env_chk() */ +typedef struct MDBX_chk_user_table_cookie MDBX_chk_user_table_cookie_t; + +/** \brief Гистограмма с некоторой статистической информацией, + * собираемой при проверке целостности БД. + * \see mdbx_env_chk() */ +struct MDBX_chk_histogram { + size_t amount, count, ones, pad; + struct { + size_t begin, end, amount, count; + } ranges[9]; +}; + +/** \brief Информация о некоторой таблицей ключ-значение, + * при проверке целостности базы данных. + * \see mdbx_env_chk() */ +typedef struct MDBX_chk_table { + MDBX_chk_user_table_cookie_t *cookie; + +/** \brief Pseudo-name for MainDB */ +#define MDBX_CHK_MAIN ((void *)((ptrdiff_t)0)) +/** \brief Pseudo-name for GarbageCollectorDB */ +#define MDBX_CHK_GC ((void *)((ptrdiff_t)-1)) +/** \brief Pseudo-name for MetaPages */ +#define MDBX_CHK_META ((void *)((ptrdiff_t)-2)) + + MDBX_val name; + MDBX_db_flags_t flags; + int id; + + size_t payload_bytes, lost_bytes; + struct { + size_t all, empty, other; + size_t branch, leaf; + size_t nested_branch, nested_leaf, nested_subleaf; + } pages; + struct { + /// Tree deep histogram + struct MDBX_chk_histogram deep; + /// Histogram of large/overflow pages length + struct MDBX_chk_histogram large_pages; + /// Histogram of nested trees height, span length for GC + struct MDBX_chk_histogram nested_tree; + /// Keys length histogram + struct MDBX_chk_histogram key_len; + /// Values length histogram + struct MDBX_chk_histogram val_len; + } histogram; +} MDBX_chk_table_t; + +/** \brief Контекст проверки целостности базы данных. + * \see mdbx_env_chk() */ +typedef struct MDBX_chk_context { + struct MDBX_chk_internal *internal; + MDBX_env *env; + MDBX_txn *txn; + MDBX_chk_scope_t *scope; + uint8_t scope_nesting; + struct { + size_t total_payload_bytes; + size_t table_total, table_processed; + size_t total_unused_bytes, unused_pages; + size_t processed_pages, reclaimable_pages, gc_pages, alloc_pages, backed_pages; + size_t problems_meta, tree_problems, gc_tree_problems, kv_tree_problems, problems_gc, problems_kv, total_problems; + uint64_t steady_txnid, recent_txnid; + /** Указатель на массив размером table_total с указателями на экземпляры + * структур MDBX_chk_table_t с информацией о всех таблицах ключ-значение, + * включая MainDB и GC/FreeDB. */ + const MDBX_chk_table_t *const *tables; + } result; +} MDBX_chk_context_t; + +/** \brief Набор функций обратного вызова используемых при проверке целостности + * базы данных. + * + * Функции обратного вызова предназначены для организации взаимодействия с кодом + * приложения. В том числе, для интеграции логики приложения проверяющей + * целостность стуктуры данных выше уровня ключ-значение, подготовки и + * структурированного вывода информации как о ходе, так и результатов проверки. + * + * Все функции обратного вызова опциональны, неиспользуемые указатели должны + * быть установлены в `nullptr`. + * + * \note Данный API еще не зафиксирован, в последующих версиях могут быть + * незначительные доработки и изменения. + * + * \see mdbx_env_chk() */ +typedef struct MDBX_chk_callbacks { + bool (*check_break)(MDBX_chk_context_t *ctx); + int (*scope_push)(MDBX_chk_context_t *ctx, MDBX_chk_scope_t *outer, MDBX_chk_scope_t *inner, const char *fmt, + va_list args); + int (*scope_conclude)(MDBX_chk_context_t *ctx, MDBX_chk_scope_t *outer, MDBX_chk_scope_t *inner, int err); + void (*scope_pop)(MDBX_chk_context_t *ctx, MDBX_chk_scope_t *outer, MDBX_chk_scope_t *inner); + void (*issue)(MDBX_chk_context_t *ctx, const char *object, uint64_t entry_number, const char *issue, + const char *extra_fmt, va_list extra_args); + MDBX_chk_user_table_cookie_t *(*table_filter)(MDBX_chk_context_t *ctx, const MDBX_val *name, MDBX_db_flags_t flags); + int (*table_conclude)(MDBX_chk_context_t *ctx, const MDBX_chk_table_t *table, MDBX_cursor *cursor, int err); + void (*table_dispose)(MDBX_chk_context_t *ctx, const MDBX_chk_table_t *table); + + int (*table_handle_kv)(MDBX_chk_context_t *ctx, const MDBX_chk_table_t *table, size_t entry_number, + const MDBX_val *key, const MDBX_val *value); + + int (*stage_begin)(MDBX_chk_context_t *ctx, MDBX_chk_stage_t); + int (*stage_end)(MDBX_chk_context_t *ctx, MDBX_chk_stage_t, int err); + + MDBX_chk_line_t *(*print_begin)(MDBX_chk_context_t *ctx, MDBX_chk_severity_t severity); + void (*print_flush)(MDBX_chk_line_t *); + void (*print_done)(MDBX_chk_line_t *); + void (*print_chars)(MDBX_chk_line_t *, const char *str, size_t len); + void (*print_format)(MDBX_chk_line_t *, const char *fmt, va_list args); + void (*print_size)(MDBX_chk_line_t *, const char *prefix, const uint64_t value, const char *suffix); +} MDBX_chk_callbacks_t; + +/** \brief Проверяет целостность базы данных. + * + * Взаимодействие с кодом приложения реализуется через функции обратного вызова, + * предоставляемые приложением посредством параметра `cb`. В ходе такого + * взаимодействия приложение может контролировать ход проверки, в том числе, + * пропускать/фильтровать обработку отдельных элементов, а также реализовать + * дополнительную верификацию структуры и/или информации с учетом назначения и + * семантической значимости для приложения. Например, приложение может выполнить + * проверку собственных индексов и корректность записей в БД. Именно с этой + * целью функционал проверки целостности был доработан для интенсивного + * использования обратных вызовов и перенесен из утилиты `mdbx_chk` в основную + * библиотеку. + * + * Проверка выполняется в несколько стадий, начиная с инициализации и до + * завершения, более подробно см \ref MDBX_chk_stage_t. О начале и завершении + * каждой стадии код приложения уведомляется через соответствующие функции + * обратного вызова, более подробно см \ref MDBX_chk_callbacks_t. + * + * \param [in] env Указатель на экземпляр среды. + * \param [in] cb Набор функций обратного вызова. + * \param [in,out] ctx Контекст проверки целостности базы данных, + * где будут формироваться результаты проверки. + * \param [in] flags Флаги/опции проверки целостности базы данных. + * \param [in] verbosity Необходимый уровень детализации информации о ходе + * и результатах проверки. + * \param [in] timeout_seconds_16dot16 Ограничение длительности в 1/65536 долях + * секунды для выполнения проверки, + * либо 0 при отсутствии ограничения. + * \returns Нулевое значение в случае успеха, иначе код ошибки. */ +LIBMDBX_API int mdbx_env_chk(MDBX_env *env, const MDBX_chk_callbacks_t *cb, MDBX_chk_context_t *ctx, + const MDBX_chk_flags_t flags, MDBX_chk_severity_t verbosity, + unsigned timeout_seconds_16dot16); + +/** \brief Вспомогательная функция для подсчета проблем детектируемых + * приложением, в том числе, поступающим к приложению через логирование. + * \see mdbx_env_chk() + * \see MDBX_debug_func + * \returns Нулевое значение в случае успеха, иначе код ошибки. */ +LIBMDBX_API int mdbx_env_chk_encount_problem(MDBX_chk_context_t *ctx); + +/** end of chk @} */ /** end of c_api @} */ diff --git a/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx.h++ b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx.h++ index f8c8db6100c..2d5f62b17d5 100644 --- a/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx.h++ +++ b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx.h++ @@ -1,8 +1,11 @@ -/// \file mdbx.h++ -/// \brief The libmdbx C++ API header file. +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \author Леонид Юрьев aka Leonid Yuriev \date 2020-2025 +/// +/// Donations are welcome to ETH `0xD104d8f8B2dC312aaD74899F83EBf3EEBDC1EA3A`. +/// Всё будет хорошо! /// -/// \author Copyright (c) 2020-2024, Leonid Yuriev . -/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \file mdbx.h++ +/// \brief The libmdbx C++ API header file. /// /// Tested with: /// - Elbrus LCC >= 1.23 (http://www.mcst.ru/lcc); @@ -24,8 +27,7 @@ #pragma once /* Workaround for modern libstdc++ with CLANG < 4.x */ -#if defined(__SIZEOF_INT128__) && !defined(__GLIBCXX_TYPE_INT_N_0) && \ - defined(__clang__) && __clang_major__ < 4 +#if defined(__SIZEOF_INT128__) && !defined(__GLIBCXX_TYPE_INT_N_0) && defined(__clang__) && __clang_major__ < 4 #define __GLIBCXX_BITSIZE_INT_N_0 128 #define __GLIBCXX_TYPE_INT_N_0 __int128 #endif /* Workaround for modern libstdc++ with CLANG < 4.x */ @@ -34,14 +36,12 @@ #if !defined(_MSC_VER) || _MSC_VER < 1900 #error "C++11 compiler or better is required" #elif _MSC_VER >= 1910 -#error \ - "Please add `/Zc:__cplusplus` to MSVC compiler options to enforce it conform ISO C++" +#error "Please add `/Zc:__cplusplus` to MSVC compiler options to enforce it conform ISO C++" #endif /* MSVC is mad and don't define __cplusplus properly */ #endif /* __cplusplus < 201103L */ #if (defined(_WIN32) || defined(_WIN64)) && MDBX_WITHOUT_MSVC_CRT -#error \ - "CRT is required for C++ API, the MDBX_WITHOUT_MSVC_CRT option must be disabled" +#error "CRT is required for C++ API, the MDBX_WITHOUT_MSVC_CRT option must be disabled" #endif /* Windows */ #ifndef __has_include @@ -81,18 +81,13 @@ #ifndef MDBX_USING_CXX_EXPERIMETAL_FILESYSTEM #ifdef INCLUDE_STD_FILESYSTEM_EXPERIMENTAL #define MDBX_USING_CXX_EXPERIMETAL_FILESYSTEM 1 -#elif defined(__cpp_lib_filesystem) && __cpp_lib_filesystem >= 201703L && \ - __cplusplus >= 201703L +#elif defined(__cpp_lib_filesystem) && __cpp_lib_filesystem >= 201703L && __cplusplus >= 201703L #define MDBX_USING_CXX_EXPERIMETAL_FILESYSTEM 0 -#elif (!defined(_MSC_VER) || __cplusplus >= 201403L || \ - (defined(_MSC_VER) && \ - defined(_SILENCE_EXPERIMENTAL_FILESYSTEM_DEPRECATION_WARNING) && \ - __cplusplus >= 201403L)) -#if defined(__cpp_lib_experimental_filesystem) && \ - __cpp_lib_experimental_filesystem >= 201406L +#elif (!defined(_MSC_VER) || __cplusplus >= 201403L || \ + (defined(_MSC_VER) && defined(_SILENCE_EXPERIMENTAL_FILESYSTEM_DEPRECATION_WARNING) && __cplusplus >= 201403L)) +#if defined(__cpp_lib_experimental_filesystem) && __cpp_lib_experimental_filesystem >= 201406L #define MDBX_USING_CXX_EXPERIMETAL_FILESYSTEM 1 -#elif defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L && \ - __has_include() +#elif defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L && __has_include() #define MDBX_USING_CXX_EXPERIMETAL_FILESYSTEM 1 #else #define MDBX_USING_CXX_EXPERIMETAL_FILESYSTEM 0 @@ -108,6 +103,20 @@ #include #endif +#if defined(__cpp_lib_span) && __cpp_lib_span >= 202002L +#include +#endif + +#if !defined(_MSC_VER) || defined(__clang__) +/* adequate compilers */ +#define MDBX_EXTERN_API_TEMPLATE(API_ATTRIBUTES, API_TYPENAME) extern template class API_ATTRIBUTES API_TYPENAME +#define MDBX_INSTALL_API_TEMPLATE(API_ATTRIBUTES, API_TYPENAME) template class API_TYPENAME +#else +/* stupid microsoft showing off */ +#define MDBX_EXTERN_API_TEMPLATE(API_ATTRIBUTES, API_TYPENAME) extern template class API_TYPENAME +#define MDBX_INSTALL_API_TEMPLATE(API_ATTRIBUTES, API_TYPENAME) template class API_ATTRIBUTES API_TYPENAME +#endif + #if __cplusplus >= 201103L #include #include @@ -115,13 +124,12 @@ #include "mdbx.h" -#if (defined(__cpp_lib_bit_cast) && __cpp_lib_bit_cast >= 201806L) || \ - (defined(__cpp_lib_endian) && __cpp_lib_endian >= 201907L) || \ - (defined(__cpp_lib_bitops) && __cpp_lib_bitops >= 201907L) || \ +#if (defined(__cpp_lib_bit_cast) && __cpp_lib_bit_cast >= 201806L) || \ + (defined(__cpp_lib_endian) && __cpp_lib_endian >= 201907L) || \ + (defined(__cpp_lib_bitops) && __cpp_lib_bitops >= 201907L) || \ (defined(__cpp_lib_int_pow2) && __cpp_lib_int_pow2 >= 202002L) #include -#elif !(defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && \ - defined(__ORDER_BIG_ENDIAN__)) +#elif !(defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && defined(__ORDER_BIG_ENDIAN__)) #if defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && defined(__BIG_ENDIAN) #define __ORDER_LITTLE_ENDIAN__ __LITTLE_ENDIAN #define __ORDER_BIG_ENDIAN__ __BIG_ENDIAN @@ -133,73 +141,69 @@ #else #define __ORDER_LITTLE_ENDIAN__ 1234 #define __ORDER_BIG_ENDIAN__ 4321 -#if defined(__LITTLE_ENDIAN__) || \ - (defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)) || \ - defined(__ARMEL__) || defined(__THUMBEL__) || defined(__AARCH64EL__) || \ - defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \ - defined(_M_ARM) || defined(_M_ARM64) || defined(__e2k__) || \ - defined(__elbrus_4c__) || defined(__elbrus_8c__) || defined(__bfin__) || \ - defined(__BFIN__) || defined(__ia64__) || defined(_IA64) || \ - defined(__IA64__) || defined(__ia64) || defined(_M_IA64) || \ - defined(__itanium__) || defined(__ia32__) || defined(__CYGWIN__) || \ - defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) || \ - defined(__WINDOWS__) +#if defined(__LITTLE_ENDIAN__) || (defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)) || defined(__ARMEL__) || \ + defined(__THUMBEL__) || defined(__AARCH64EL__) || defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \ + defined(_M_ARM) || defined(_M_ARM64) || defined(__e2k__) || defined(__elbrus_4c__) || defined(__elbrus_8c__) || \ + defined(__bfin__) || defined(__BFIN__) || defined(__ia64__) || defined(_IA64) || defined(__IA64__) || \ + defined(__ia64) || defined(_M_IA64) || defined(__itanium__) || defined(__ia32__) || defined(__CYGWIN__) || \ + defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) || defined(__WINDOWS__) #define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__ -#elif defined(__BIG_ENDIAN__) || \ - (defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) || \ - defined(__ARMEB__) || defined(__THUMBEB__) || defined(__AARCH64EB__) || \ - defined(__MIPSEB__) || defined(_MIPSEB) || defined(__MIPSEB) || \ - defined(__m68k__) || defined(M68000) || defined(__hppa__) || \ - defined(__hppa) || defined(__HPPA__) || defined(__sparc__) || \ - defined(__sparc) || defined(__370__) || defined(__THW_370__) || \ - defined(__s390__) || defined(__s390x__) || defined(__SYSC_ZARCH__) +#elif defined(__BIG_ENDIAN__) || (defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) || defined(__ARMEB__) || \ + defined(__THUMBEB__) || defined(__AARCH64EB__) || defined(__MIPSEB__) || defined(_MIPSEB) || defined(__MIPSEB) || \ + defined(__m68k__) || defined(M68000) || defined(__hppa__) || defined(__hppa) || defined(__HPPA__) || \ + defined(__sparc__) || defined(__sparc) || defined(__370__) || defined(__THW_370__) || defined(__s390__) || \ + defined(__s390x__) || defined(__SYSC_ZARCH__) #define __BYTE_ORDER__ __ORDER_BIG_ENDIAN__ #endif #endif #endif /* Byte Order */ -/** Workaround for old compilers without properly support for `C++17 constexpr`. - */ +/** Workaround for old compilers without properly support for `C++17 constexpr` */ #if defined(DOXYGEN) #define MDBX_CXX17_CONSTEXPR constexpr -#elif defined(__cpp_constexpr) && __cpp_constexpr >= 201603L && \ - ((defined(_MSC_VER) && _MSC_VER >= 1915) || \ - (defined(__clang__) && __clang_major__ > 5) || \ - (defined(__GNUC__) && __GNUC__ > 7) || \ - (!defined(__GNUC__) && !defined(__clang__) && !defined(_MSC_VER))) +#elif defined(__cpp_constexpr) && __cpp_constexpr >= 201603L && \ + ((defined(_MSC_VER) && _MSC_VER >= 1915) || (defined(__clang__) && __clang_major__ > 5) || \ + (defined(__GNUC__) && __GNUC__ > 7) || (!defined(__GNUC__) && !defined(__clang__) && !defined(_MSC_VER))) #define MDBX_CXX17_CONSTEXPR constexpr #else #define MDBX_CXX17_CONSTEXPR inline #endif /* MDBX_CXX17_CONSTEXPR */ -/** Workaround for old compilers without properly support for C++20 `constexpr`. - */ +/** Workaround for old compilers without properly support for C++20 `constexpr`. */ #if defined(DOXYGEN) #define MDBX_CXX20_CONSTEXPR constexpr -#elif defined(__cpp_lib_is_constant_evaluated) && \ - __cpp_lib_is_constant_evaluated >= 201811L && \ - defined(__cpp_lib_constexpr_string) && \ - __cpp_lib_constexpr_string >= 201907L +#elif defined(__cpp_lib_is_constant_evaluated) && __cpp_lib_is_constant_evaluated >= 201811L && \ + defined(__cpp_lib_constexpr_string) && __cpp_lib_constexpr_string >= 201907L #define MDBX_CXX20_CONSTEXPR constexpr #else #define MDBX_CXX20_CONSTEXPR inline #endif /* MDBX_CXX20_CONSTEXPR */ -/** Workaround for old compilers without support assertion inside `constexpr` - * functions. */ +#if CONSTEXPR_ENUM_FLAGS_OPERATIONS || defined(DOXYGEN) +#define MDBX_CXX01_CONSTEXPR_ENUM MDBX_CXX01_CONSTEXPR +#define MDBX_CXX11_CONSTEXPR_ENUM MDBX_CXX11_CONSTEXPR +#define MDBX_CXX14_CONSTEXPR_ENUM MDBX_CXX14_CONSTEXPR +#define MDBX_CXX17_CONSTEXPR_ENUM MDBX_CXX17_CONSTEXPR +#define MDBX_CXX20_CONSTEXPR_ENUM MDBX_CXX20_CONSTEXPR +#else +#define MDBX_CXX01_CONSTEXPR_ENUM inline +#define MDBX_CXX11_CONSTEXPR_ENUM inline +#define MDBX_CXX14_CONSTEXPR_ENUM inline +#define MDBX_CXX17_CONSTEXPR_ENUM inline +#define MDBX_CXX20_CONSTEXPR_ENUM inline +#endif /* CONSTEXPR_ENUM_FLAGS_OPERATIONS */ + +/** Workaround for old compilers without support assertion inside `constexpr` functions. */ #if defined(CONSTEXPR_ASSERT) #define MDBX_CONSTEXPR_ASSERT(expr) CONSTEXPR_ASSERT(expr) #elif defined NDEBUG #define MDBX_CONSTEXPR_ASSERT(expr) void(0) #else -#define MDBX_CONSTEXPR_ASSERT(expr) \ - ((expr) ? void(0) : [] { assert(!#expr); }()) +#define MDBX_CONSTEXPR_ASSERT(expr) ((expr) ? void(0) : [] { assert(!#expr); }()) #endif /* MDBX_CONSTEXPR_ASSERT */ #ifndef MDBX_LIKELY -#if defined(DOXYGEN) || \ - (defined(__GNUC__) || __has_builtin(__builtin_expect)) && \ - !defined(__COVERITY__) +#if defined(DOXYGEN) || (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__) #define MDBX_LIKELY(cond) __builtin_expect(!!(cond), 1) #else #define MDBX_LIKELY(x) (x) @@ -207,17 +211,14 @@ #endif /* MDBX_LIKELY */ #ifndef MDBX_UNLIKELY -#if defined(DOXYGEN) || \ - (defined(__GNUC__) || __has_builtin(__builtin_expect)) && \ - !defined(__COVERITY__) +#if defined(DOXYGEN) || (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__) #define MDBX_UNLIKELY(cond) __builtin_expect(!!(cond), 0) #else #define MDBX_UNLIKELY(x) (x) #endif #endif /* MDBX_UNLIKELY */ -/** Workaround for old compilers without properly support for C++20 `if - * constexpr`. */ +/** Workaround for old compilers without properly support for C++20 `if constexpr`. */ #if defined(DOXYGEN) #define MDBX_IF_CONSTEXPR constexpr #elif defined(__cpp_if_constexpr) && __cpp_if_constexpr >= 201606L @@ -226,25 +227,21 @@ #define MDBX_IF_CONSTEXPR #endif /* MDBX_IF_CONSTEXPR */ -#if defined(DOXYGEN) || \ - (__has_cpp_attribute(fallthrough) && \ - (!defined(__clang__) || __clang__ > 4)) || \ +#if defined(DOXYGEN) || (__has_cpp_attribute(fallthrough) && (!defined(__clang__) || __clang__ > 4)) || \ __cplusplus >= 201703L #define MDBX_CXX17_FALLTHROUGH [[fallthrough]] #else #define MDBX_CXX17_FALLTHROUGH #endif /* MDBX_CXX17_FALLTHROUGH */ -#if defined(DOXYGEN) || (__has_cpp_attribute(likely) >= 201803L && \ - (!defined(__GNUC__) || __GNUC__ > 9)) +#if defined(DOXYGEN) || (__has_cpp_attribute(likely) >= 201803L && (!defined(__GNUC__) || __GNUC__ > 9)) #define MDBX_CXX20_LIKELY [[likely]] #else #define MDBX_CXX20_LIKELY #endif /* MDBX_CXX20_LIKELY */ #ifndef MDBX_CXX20_UNLIKELY -#if defined(DOXYGEN) || (__has_cpp_attribute(unlikely) >= 201803L && \ - (!defined(__GNUC__) || __GNUC__ > 9)) +#if defined(DOXYGEN) || (__has_cpp_attribute(unlikely) >= 201803L && (!defined(__GNUC__) || __GNUC__ > 9)) #define MDBX_CXX20_UNLIKELY [[unlikely]] #else #define MDBX_CXX20_UNLIKELY @@ -252,7 +249,7 @@ #endif /* MDBX_CXX20_UNLIKELY */ #ifndef MDBX_HAVE_CXX20_CONCEPTS -#if defined(__cpp_lib_concepts) && __cpp_lib_concepts >= 202002L +#if defined(__cpp_concepts) && __cpp_concepts >= 202002L && defined(__cpp_lib_concepts) && __cpp_lib_concepts >= 202002L #include #define MDBX_HAVE_CXX20_CONCEPTS 1 #elif defined(DOXYGEN) @@ -272,10 +269,9 @@ #ifndef MDBX_ASSERT_CXX20_CONCEPT_SATISFIED #if MDBX_HAVE_CXX20_CONCEPTS || defined(DOXYGEN) -#define MDBX_ASSERT_CXX20_CONCEPT_SATISFIED(CONCEPT, TYPE) \ - static_assert(CONCEPT) +#define MDBX_ASSERT_CXX20_CONCEPT_SATISFIED(CONCEPT, TYPE) static_assert(CONCEPT) #else -#define MDBX_ASSERT_CXX20_CONCEPT_SATISFIED(CONCEPT, NAME) \ +#define MDBX_ASSERT_CXX20_CONCEPT_SATISFIED(CONCEPT, NAME) \ static_assert(true, MDBX_STRINGIFY(CONCEPT) "<" MDBX_STRINGIFY(TYPE) ">") #endif #endif /* MDBX_ASSERT_CXX20_CONCEPT_SATISFIED */ @@ -283,9 +279,9 @@ #ifdef _MSC_VER #pragma warning(push, 4) #pragma warning(disable : 4127) /* conditional expression is constant */ -#pragma warning(disable : 4251) /* 'std::FOO' needs to have dll-interface to \ +#pragma warning(disable : 4251) /* 'std::FOO' needs to have dll-interface to \ be used by clients of 'mdbx::BAR' */ -#pragma warning(disable : 4275) /* non dll-interface 'std::FOO' used as \ +#pragma warning(disable : 4275) /* non dll-interface 'std::FOO' used as \ base for dll-interface 'mdbx::BAR' */ /* MSVC is mad and can generate this warning for its own intermediate * automatically generated code, which becomes unreachable after some kinds of @@ -296,9 +292,9 @@ #if defined(__LCC__) && __LCC__ >= 126 #pragma diagnostic push #if __LCC__ < 127 -#pragma diag_suppress 3058 /* workaround: call to is_constant_evaluated() \ +#pragma diag_suppress 3058 /* workaround: call to is_constant_evaluated() \ appearing in a constant expression `true` */ -#pragma diag_suppress 3060 /* workaround: call to is_constant_evaluated() \ +#pragma diag_suppress 3060 /* workaround: call to is_constant_evaluated() \ appearing in a constant expression `false` */ #endif #endif /* E2K LCC (warnings) */ @@ -328,16 +324,10 @@ using byte = unsigned char; #if defined(__cpp_lib_endian) && __cpp_lib_endian >= 201907L using endian = ::std::endian; -#elif defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && \ - defined(__ORDER_BIG_ENDIAN__) -enum class endian { - little = __ORDER_LITTLE_ENDIAN__, - big = __ORDER_BIG_ENDIAN__, - native = __BYTE_ORDER__ -}; +#elif defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && defined(__ORDER_BIG_ENDIAN__) +enum class endian { little = __ORDER_LITTLE_ENDIAN__, big = __ORDER_BIG_ENDIAN__, native = __BYTE_ORDER__ }; #else -#error \ - "Please use a C++ compiler provides byte order information or C++20 support" +#error "Please use a C++ compiler provides byte order information or C++20 support" #endif /* Byte Order enum */ /// \copydoc MDBX_version_info @@ -353,21 +343,26 @@ MDBX_CXX11_CONSTEXPR const build_info &get_build() noexcept; static MDBX_CXX17_CONSTEXPR size_t strlen(const char *c_str) noexcept; /// \brief constexpr-enabled memcpy(). -static MDBX_CXX20_CONSTEXPR void *memcpy(void *dest, const void *src, - size_t bytes) noexcept; +static MDBX_CXX20_CONSTEXPR void *memcpy(void *dest, const void *src, size_t bytes) noexcept; /// \brief constexpr-enabled memcmp(). -static MDBX_CXX20_CONSTEXPR int memcmp(const void *a, const void *b, - size_t bytes) noexcept; +static MDBX_CXX20_CONSTEXPR int memcmp(const void *a, const void *b, size_t bytes) noexcept; -/// \brief Legacy default allocator +/// \brief Legacy allocator /// but it is recommended to use \ref polymorphic_allocator. using legacy_allocator = ::std::string::allocator_type; +#if defined(DOXYGEN) || \ + (defined(__cpp_lib_memory_resource) && __cpp_lib_memory_resource >= 201603L && _GLIBCXX_USE_CXX11_ABI) +/// \brief Default polymorphic allocator for modern code. +using polymorphic_allocator = ::std::pmr::string::allocator_type; +using default_allocator = polymorphic_allocator; +#else +using default_allocator = legacy_allocator; +#endif /* __cpp_lib_memory_resource >= 201603L */ + struct slice; struct default_capacity_policy; -template -class buffer; +template class buffer; class env; class env_managed; class txn; @@ -375,16 +370,6 @@ class txn_managed; class cursor; class cursor_managed; -#if defined(DOXYGEN) || \ - (defined(__cpp_lib_memory_resource) && \ - __cpp_lib_memory_resource >= 201603L && _GLIBCXX_USE_CXX11_ABI) -/// \brief Default polymorphic allocator for modern code. -using polymorphic_allocator = ::std::pmr::string::allocator_type; -using default_allocator = polymorphic_allocator; -#else -using default_allocator = legacy_allocator; -#endif /* __cpp_lib_memory_resource >= 201603L */ - /// \brief Default buffer. using default_buffer = buffer; @@ -400,19 +385,16 @@ namespace filesystem = ::std::experimental::filesystem::v1; namespace filesystem = ::std::experimental::filesystem; #endif #define MDBX_STD_FILESYSTEM_PATH ::mdbx::filesystem::path -#elif defined(DOXYGEN) || \ - (defined(__cpp_lib_filesystem) && __cpp_lib_filesystem >= 201703L && \ - defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L && \ - (!defined(__MAC_OS_X_VERSION_MIN_REQUIRED) || \ - __MAC_OS_X_VERSION_MIN_REQUIRED >= 101500) && \ - (!defined(__IPHONE_OS_VERSION_MIN_REQUIRED) || \ - __IPHONE_OS_VERSION_MIN_REQUIRED >= 130100)) && \ +#elif defined(DOXYGEN) || \ + (defined(__cpp_lib_filesystem) && __cpp_lib_filesystem >= 201703L && defined(__cpp_lib_string_view) && \ + __cpp_lib_string_view >= 201606L && \ + (!defined(__MAC_OS_X_VERSION_MIN_REQUIRED) || __MAC_OS_X_VERSION_MIN_REQUIRED >= 101500) && \ + (!defined(__IPHONE_OS_VERSION_MIN_REQUIRED) || __IPHONE_OS_VERSION_MIN_REQUIRED >= 130100)) && \ (!defined(_MSC_VER) || __cplusplus >= 201703L) namespace filesystem = ::std::filesystem; /// \brief Defined if `mdbx::filesystem::path` is available. /// \details If defined, it is always `mdbx::filesystem::path`, -/// which in turn can be refs to `std::filesystem::path` -/// or `std::experimental::filesystem::path`. +/// which in turn can be refs to `std::filesystem::path` or `std::experimental::filesystem::path`. /// Nonetheless `MDBX_STD_FILESYSTEM_PATH` not defined if the `::mdbx::path` /// is fallbacked to c `std::string` or `std::wstring`. #define MDBX_STD_FILESYSTEM_PATH ::mdbx::filesystem::path @@ -426,8 +408,7 @@ using path = ::std::wstring; using path = ::std::string; #endif /* mdbx::path */ -#if defined(__SIZEOF_INT128__) || \ - (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) +#if defined(__SIZEOF_INT128__) || (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) #ifndef MDBX_U128_TYPE #define MDBX_U128_TYPE __uint128_t #endif /* MDBX_U128_TYPE */ @@ -474,10 +455,8 @@ public: error &operator=(const error &) = default; error &operator=(error &&) = default; - MDBX_CXX11_CONSTEXPR friend bool operator==(const error &a, - const error &b) noexcept; - MDBX_CXX11_CONSTEXPR friend bool operator!=(const error &a, - const error &b) noexcept; + MDBX_CXX11_CONSTEXPR friend bool operator==(const error &a, const error &b) noexcept; + MDBX_CXX11_CONSTEXPR friend bool operator!=(const error &a, const error &b) noexcept; MDBX_CXX11_CONSTEXPR bool is_success() const noexcept; MDBX_CXX11_CONSTEXPR bool is_result_true() const noexcept; @@ -496,34 +475,26 @@ public: /// \brief Returns true for MDBX's errors. MDBX_CXX11_CONSTEXPR bool is_mdbx_error() const noexcept; /// \brief Panics on unrecoverable errors inside destructors etc. - [[noreturn]] void panic(const char *context_where_when, - const char *func_who_what) const noexcept; + [[noreturn]] void panic(const char *context_where_when, const char *func_who_what) const noexcept; [[noreturn]] void throw_exception() const; [[noreturn]] static inline void throw_exception(int error_code); inline void throw_on_failure() const; inline void success_or_throw() const; inline void success_or_throw(const exception_thunk &) const; - inline void panic_on_failure(const char *context_where, - const char *func_who) const noexcept; - inline void success_or_panic(const char *context_where, - const char *func_who) const noexcept; + inline void panic_on_failure(const char *context_where, const char *func_who) const noexcept; + inline void success_or_panic(const char *context_where, const char *func_who) const noexcept; static inline void throw_on_nullptr(const void *ptr, MDBX_error_t error_code); static inline void success_or_throw(MDBX_error_t error_code); - static void success_or_throw(int error_code) { - success_or_throw(static_cast(error_code)); - } + static void success_or_throw(int error_code) { success_or_throw(static_cast(error_code)); } static inline void throw_on_failure(int error_code); static inline bool boolean_or_throw(int error_code); static inline void success_or_throw(int error_code, const exception_thunk &); - static inline void panic_on_failure(int error_code, const char *context_where, - const char *func_who) noexcept; - static inline void success_or_panic(int error_code, const char *context_where, - const char *func_who) noexcept; + static inline bool boolean_or_throw(int error_code, const exception_thunk &); + static inline void panic_on_failure(int error_code, const char *context_where, const char *func_who) noexcept; + static inline void success_or_panic(int error_code, const char *context_where, const char *func_who) noexcept; }; -/// \brief Base class for all libmdbx's exceptions that are corresponds -/// to libmdbx errors. -/// +/// \brief Base class for all libmdbx's exceptions that are corresponds to libmdbx errors. /// \see MDBX_error_t class LIBMDBX_API_TYPE exception : public ::std::runtime_error { using base = ::std::runtime_error; @@ -539,8 +510,7 @@ public: const ::mdbx::error error() const noexcept { return error_; } }; -/// \brief Fatal exception that lead termination anyway -/// in dangerous unrecoverable cases. +/// \brief Fatal exception that lead termination anyway in dangerous unrecoverable cases. class LIBMDBX_API_TYPE fatal : public exception { using base = exception; @@ -555,10 +525,10 @@ public: virtual ~fatal() noexcept; }; -#define MDBX_DECLARE_EXCEPTION(NAME) \ - struct LIBMDBX_API_TYPE NAME : public exception { \ - NAME(const ::mdbx::error &); \ - virtual ~NAME() noexcept; \ +#define MDBX_DECLARE_EXCEPTION(NAME) \ + struct LIBMDBX_API_TYPE NAME : public exception { \ + NAME(const ::mdbx::error &); \ + virtual ~NAME() noexcept; \ } MDBX_DECLARE_EXCEPTION(bad_map_id); MDBX_DECLARE_EXCEPTION(bad_transaction); @@ -589,6 +559,9 @@ MDBX_DECLARE_EXCEPTION(thread_mismatch); MDBX_DECLARE_EXCEPTION(transaction_full); MDBX_DECLARE_EXCEPTION(transaction_overlapping); MDBX_DECLARE_EXCEPTION(duplicated_lck_file); +MDBX_DECLARE_EXCEPTION(dangling_map_id); +MDBX_DECLARE_EXCEPTION(transaction_ousted); +MDBX_DECLARE_EXCEPTION(mvcc_retarded); #undef MDBX_DECLARE_EXCEPTION [[noreturn]] LIBMDBX_API void throw_too_small_target_buffer(); @@ -596,11 +569,10 @@ MDBX_DECLARE_EXCEPTION(duplicated_lck_file); [[noreturn]] LIBMDBX_API void throw_out_range(); [[noreturn]] LIBMDBX_API void throw_allocators_mismatch(); [[noreturn]] LIBMDBX_API void throw_bad_value_size(); +[[noreturn]] LIBMDBX_API void throw_incomparable_cursors(); static MDBX_CXX14_CONSTEXPR size_t check_length(size_t bytes); -static MDBX_CXX14_CONSTEXPR size_t check_length(size_t headroom, - size_t payload); -static MDBX_CXX14_CONSTEXPR size_t check_length(size_t headroom, size_t payload, - size_t tailroom); +static MDBX_CXX14_CONSTEXPR size_t check_length(size_t headroom, size_t payload); +static MDBX_CXX14_CONSTEXPR size_t check_length(size_t headroom, size_t payload, size_t tailroom); /// end of cxx_exceptions @} @@ -635,35 +607,27 @@ concept ImmutableByteProducer = requires(const T &a, char array[42]) { * \interface SliceTranscoder * \brief SliceTranscoder C++20 concept */ template -concept SliceTranscoder = - ImmutableByteProducer && requires(const slice &source, const T &a) { - T(source); - { a.is_erroneous() } -> std::same_as; - }; +concept SliceTranscoder = ImmutableByteProducer && requires(const slice &source, const T &a) { + T(source); + { a.is_erroneous() } -> std::same_as; +}; #endif /* MDBX_HAVE_CXX20_CONCEPTS */ -template -inline buffer -make_buffer(PRODUCER &producer, const ALLOCATOR &allocator = ALLOCATOR()); +inline buffer make_buffer(PRODUCER &producer, const ALLOCATOR &allocator = ALLOCATOR()); -template -inline buffer -make_buffer(const PRODUCER &producer, const ALLOCATOR &allocator = ALLOCATOR()); +inline buffer make_buffer(const PRODUCER &producer, + const ALLOCATOR &allocator = ALLOCATOR()); -template -inline string make_string(PRODUCER &producer, - const ALLOCATOR &allocator = ALLOCATOR()); +template +inline string make_string(PRODUCER &producer, const ALLOCATOR &allocator = ALLOCATOR()); -template -inline string make_string(const PRODUCER &producer, - const ALLOCATOR &allocator = ALLOCATOR()); +template +inline string make_string(const PRODUCER &producer, const ALLOCATOR &allocator = ALLOCATOR()); /// \brief References a data located outside the slice. /// @@ -682,16 +646,14 @@ struct LIBMDBX_API_TYPE slice : public ::MDBX_val { /// \brief Create an empty slice. MDBX_CXX11_CONSTEXPR slice() noexcept; - /// \brief Create a slice that refers to [0,bytes-1] of memory bytes pointed - /// by ptr. + /// \brief Create a slice that refers to [0,bytes-1] of memory bytes pointed by ptr. MDBX_CXX14_CONSTEXPR slice(const void *ptr, size_t bytes); /// \brief Create a slice that refers to [begin,end] of memory bytes. MDBX_CXX14_CONSTEXPR slice(const void *begin, const void *end); /// \brief Create a slice that refers to text[0,strlen(text)-1]. - template - MDBX_CXX14_CONSTEXPR slice(const char (&text)[SIZE]) : slice(text, SIZE - 1) { + template MDBX_CXX14_CONSTEXPR slice(const char (&text)[SIZE]) : slice(text, SIZE - 1) { MDBX_CONSTEXPR_ASSERT(SIZE > 0 && text[SIZE - 1] == '\0'); } /// \brief Create a slice that refers to c_str[0,strlen(c_str)-1]. @@ -700,8 +662,7 @@ struct LIBMDBX_API_TYPE slice : public ::MDBX_val { /// \brief Create a slice that refers to the contents of "str". /// \note 'explicit' to avoid reference to the temporary std::string instance. template - explicit MDBX_CXX20_CONSTEXPR - slice(const ::std::basic_string &str) + explicit MDBX_CXX20_CONSTEXPR slice(const ::std::basic_string &str) : slice(str.data(), str.length() * sizeof(CHAR)) {} MDBX_CXX14_CONSTEXPR slice(const MDBX_val &src); @@ -709,28 +670,48 @@ struct LIBMDBX_API_TYPE slice : public ::MDBX_val { MDBX_CXX14_CONSTEXPR slice(MDBX_val &&src); MDBX_CXX14_CONSTEXPR slice(slice &&src) noexcept; -#if defined(DOXYGEN) || \ - (defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L) +#if defined(DOXYGEN) || (defined(__cpp_lib_span) && __cpp_lib_span >= 202002L) + template MDBX_CXX14_CONSTEXPR slice(const ::std::span &span) : slice(span.begin(), span.end()) { + static_assert(::std::is_standard_layout::value && !::std::is_pointer::value, + "Must be a standard layout type!"); + } + + template MDBX_CXX14_CONSTEXPR ::std::span as_span() const { + static_assert(::std::is_standard_layout::value && !::std::is_pointer::value, + "Must be a standard layout type!"); + if (MDBX_LIKELY(size() % sizeof(POD) == 0)) + MDBX_CXX20_LIKELY + return ::std::span(static_cast(data()), size() / sizeof(POD)); + throw_bad_value_size(); + } + + template MDBX_CXX14_CONSTEXPR ::std::span as_span() { + static_assert(::std::is_standard_layout::value && !::std::is_pointer::value, + "Must be a standard layout type!"); + if (MDBX_LIKELY(size() % sizeof(POD) == 0)) + MDBX_CXX20_LIKELY + return ::std::span(static_cast(data()), size() / sizeof(POD)); + throw_bad_value_size(); + } + + MDBX_CXX14_CONSTEXPR ::std::span bytes() const { return as_span(); } + MDBX_CXX14_CONSTEXPR ::std::span bytes() { return as_span(); } + MDBX_CXX14_CONSTEXPR ::std::span chars() const { return as_span(); } + MDBX_CXX14_CONSTEXPR ::std::span chars() { return as_span(); } +#endif /* __cpp_lib_span >= 202002L */ + +#if defined(DOXYGEN) || (defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L) /// \brief Create a slice that refers to the same contents as "string_view" template - MDBX_CXX14_CONSTEXPR slice(const ::std::basic_string_view &sv) - : slice(sv.data(), sv.data() + sv.length()) {} + MDBX_CXX14_CONSTEXPR slice(const ::std::basic_string_view &sv) : slice(sv.data(), sv.data() + sv.length()) {} - template - slice(::std::basic_string_view &&sv) : slice(sv) { - sv = {}; - } + template slice(::std::basic_string_view &&sv) : slice(sv) { sv = {}; } #endif /* __cpp_lib_string_view >= 201606L */ - template - static MDBX_CXX14_CONSTEXPR slice wrap(const char (&text)[SIZE]) { - return slice(text); - } + template static MDBX_CXX14_CONSTEXPR slice wrap(const char (&text)[SIZE]) { return slice(text); } - template - MDBX_CXX14_CONSTEXPR static slice wrap(const POD &pod) { - static_assert(::std::is_standard_layout::value && - !::std::is_pointer::value, + template MDBX_CXX14_CONSTEXPR static slice wrap(const POD &pod) { + static_assert(::std::is_standard_layout::value && !::std::is_pointer::value, "Must be a standard layout type!"); return slice(&pod, sizeof(pod)); } @@ -741,19 +722,15 @@ struct LIBMDBX_API_TYPE slice : public ::MDBX_val { inline slice &assign(slice &&src) noexcept; inline slice &assign(::MDBX_val &&src); inline slice &assign(const void *begin, const void *end); - template - slice &assign(const ::std::basic_string &str) { + template slice &assign(const ::std::basic_string &str) { return assign(str.data(), str.length() * sizeof(CHAR)); } inline slice &assign(const char *c_str); -#if defined(DOXYGEN) || \ - (defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L) - template - slice &assign(const ::std::basic_string_view &view) { +#if defined(DOXYGEN) || (defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L) + template slice &assign(const ::std::basic_string_view &view) { return assign(view.begin(), view.end()); } - template - slice &assign(::std::basic_string_view &&view) { + template slice &assign(::std::basic_string_view &&view) { assign(view); view = {}; return *this; @@ -766,152 +743,119 @@ struct LIBMDBX_API_TYPE slice : public ::MDBX_val { operator MDBX_val *() noexcept { return this; } operator const MDBX_val *() const noexcept { return this; } -#if defined(DOXYGEN) || \ - (defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L) - template - slice &operator=(const ::std::basic_string_view &view) { +#if defined(DOXYGEN) || (defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L) + template slice &operator=(const ::std::basic_string_view &view) { return assign(view); } - template - slice &operator=(::std::basic_string_view &&view) { - return assign(view); - } + template slice &operator=(::std::basic_string_view &&view) { return assign(view); } /// \brief Return a string_view that references the same data as this slice. template > - MDBX_CXX11_CONSTEXPR ::std::basic_string_view - string_view() const noexcept { + MDBX_CXX11_CONSTEXPR ::std::basic_string_view string_view() const noexcept { static_assert(sizeof(CHAR) == 1, "Must be single byte characters"); return ::std::basic_string_view(char_ptr(), length()); } /// \brief Return a string_view that references the same data as this slice. template - MDBX_CXX11_CONSTEXPR explicit - operator ::std::basic_string_view() const noexcept { + MDBX_CXX11_CONSTEXPR explicit operator ::std::basic_string_view() const noexcept { return this->string_view(); } #endif /* __cpp_lib_string_view >= 201606L */ - template , - class ALLOCATOR = legacy_allocator> + template , class ALLOCATOR = default_allocator> MDBX_CXX20_CONSTEXPR ::std::basic_string as_string(const ALLOCATOR &allocator = ALLOCATOR()) const { static_assert(sizeof(CHAR) == 1, "Must be single byte characters"); - return ::std::basic_string(char_ptr(), length(), - allocator); + return ::std::basic_string(char_ptr(), length(), allocator); } template - MDBX_CXX20_CONSTEXPR explicit - operator ::std::basic_string() const { + MDBX_CXX20_CONSTEXPR explicit operator ::std::basic_string() const { return as_string(); } /// \brief Returns a string with a hexadecimal dump of the slice content. - template - inline string - as_hex_string(bool uppercase = false, unsigned wrap_width = 0, - const ALLOCATOR &allocator = ALLOCATOR()) const; + template + inline string as_hex_string(bool uppercase = false, unsigned wrap_width = 0, + const ALLOCATOR &allocator = ALLOCATOR()) const; /// \brief Returns a string with a /// [Base58](https://en.wikipedia.org/wiki/Base58) dump of the slice content. - template - inline string - as_base58_string(unsigned wrap_width = 0, - const ALLOCATOR &allocator = ALLOCATOR()) const; + template + inline string as_base58_string(unsigned wrap_width = 0, const ALLOCATOR &allocator = ALLOCATOR()) const; /// \brief Returns a string with a /// [Base58](https://en.wikipedia.org/wiki/Base64) dump of the slice content. - template - inline string - as_base64_string(unsigned wrap_width = 0, - const ALLOCATOR &allocator = ALLOCATOR()) const; + template + inline string as_base64_string(unsigned wrap_width = 0, const ALLOCATOR &allocator = ALLOCATOR()) const; /// \brief Returns a buffer with a hexadecimal dump of the slice content. - template - inline buffer - encode_hex(bool uppercase = false, unsigned wrap_width = 0, - const ALLOCATOR &allocator = ALLOCATOR()) const; + template + inline buffer encode_hex(bool uppercase = false, unsigned wrap_width = 0, + const ALLOCATOR &allocator = ALLOCATOR()) const; /// \brief Returns a buffer with a /// [Base58](https://en.wikipedia.org/wiki/Base58) dump of the slice content. - template - inline buffer - encode_base58(unsigned wrap_width = 0, - const ALLOCATOR &allocator = ALLOCATOR()) const; + template + inline buffer encode_base58(unsigned wrap_width = 0, + const ALLOCATOR &allocator = ALLOCATOR()) const; /// \brief Returns a buffer with a /// [Base64](https://en.wikipedia.org/wiki/Base64) dump of the slice content. - template - inline buffer - encode_base64(unsigned wrap_width = 0, - const ALLOCATOR &allocator = ALLOCATOR()) const; + template + inline buffer encode_base64(unsigned wrap_width = 0, + const ALLOCATOR &allocator = ALLOCATOR()) const; /// \brief Decodes hexadecimal dump from the slice content to returned buffer. - template - inline buffer - hex_decode(bool ignore_spaces = false, - const ALLOCATOR &allocator = ALLOCATOR()) const; + template + inline buffer hex_decode(bool ignore_spaces = false, + const ALLOCATOR &allocator = ALLOCATOR()) const; /// \brief Decodes [Base58](https://en.wikipedia.org/wiki/Base58) dump /// from the slice content to returned buffer. - template - inline buffer - base58_decode(bool ignore_spaces = false, - const ALLOCATOR &allocator = ALLOCATOR()) const; + template + inline buffer base58_decode(bool ignore_spaces = false, + const ALLOCATOR &allocator = ALLOCATOR()) const; /// \brief Decodes [Base64](https://en.wikipedia.org/wiki/Base64) dump /// from the slice content to returned buffer. - template - inline buffer - base64_decode(bool ignore_spaces = false, - const ALLOCATOR &allocator = ALLOCATOR()) const; + template + inline buffer base64_decode(bool ignore_spaces = false, + const ALLOCATOR &allocator = ALLOCATOR()) const; /// \brief Checks whether the content of the slice is printable. /// \param [in] disable_utf8 By default if `disable_utf8` is `false` function /// checks that content bytes are printable ASCII-7 characters or a valid UTF8 - /// sequences. Otherwise, if if `disable_utf8` is `true` function checks that + /// sequences. Otherwise, if `disable_utf8` is `true` function checks that /// content bytes are printable extended 8-bit ASCII codes. - MDBX_NOTHROW_PURE_FUNCTION bool - is_printable(bool disable_utf8 = false) const noexcept; + MDBX_NOTHROW_PURE_FUNCTION bool is_printable(bool disable_utf8 = false) const noexcept; /// \brief Checks whether the content of the slice is a hexadecimal dump. /// \param [in] ignore_spaces If `true` function will skips spaces surrounding /// (before, between and after) a encoded bytes. However, spaces should not /// break a pair of characters encoding a single byte. - inline MDBX_NOTHROW_PURE_FUNCTION bool - is_hex(bool ignore_spaces = false) const noexcept; + MDBX_NOTHROW_PURE_FUNCTION inline bool is_hex(bool ignore_spaces = false) const noexcept; /// \brief Checks whether the content of the slice is a /// [Base58](https://en.wikipedia.org/wiki/Base58) dump. /// \param [in] ignore_spaces If `true` function will skips spaces surrounding /// (before, between and after) a encoded bytes. However, spaces should not /// break a code group of characters. - inline MDBX_NOTHROW_PURE_FUNCTION bool - is_base58(bool ignore_spaces = false) const noexcept; + MDBX_NOTHROW_PURE_FUNCTION inline bool is_base58(bool ignore_spaces = false) const noexcept; /// \brief Checks whether the content of the slice is a /// [Base64](https://en.wikipedia.org/wiki/Base64) dump. /// \param [in] ignore_spaces If `true` function will skips spaces surrounding /// (before, between and after) a encoded bytes. However, spaces should not /// break a code group of characters. - inline MDBX_NOTHROW_PURE_FUNCTION bool - is_base64(bool ignore_spaces = false) const noexcept; + MDBX_NOTHROW_PURE_FUNCTION inline bool is_base64(bool ignore_spaces = false) const noexcept; inline void swap(slice &other) noexcept; -#if defined(DOXYGEN) || \ - (defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L) - template - void swap(::std::basic_string_view &view) noexcept { +#if defined(DOXYGEN) || (defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L) + template void swap(::std::basic_string_view &view) noexcept { static_assert(sizeof(CHAR) == 1, "Must be single byte characters"); const auto temp = ::std::basic_string_view(*this); *this = view; @@ -987,12 +931,10 @@ struct LIBMDBX_API_TYPE slice : public ::MDBX_val { inline void safe_remove_suffix(size_t n); /// \brief Checks if the data starts with the given prefix. - MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR bool - starts_with(const slice &prefix) const noexcept; + MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR bool starts_with(const slice &prefix) const noexcept; /// \brief Checks if the data ends with the given suffix. - MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR bool - ends_with(const slice &suffix) const noexcept; + MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR bool ends_with(const slice &suffix) const noexcept; /// \brief Returns the nth byte in the referenced data. /// \pre REQUIRES: `n < size()` @@ -1030,8 +972,7 @@ struct LIBMDBX_API_TYPE slice : public ::MDBX_val { /// \attention Function implementation and returned hash values may changed /// version to version, and in future the t1ha3 will be used here. Therefore /// values obtained from this function shouldn't be persisted anywhere. - MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR size_t - hash_value() const noexcept; + MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR size_t hash_value() const noexcept; /// \brief Three-way fast non-lexicographically length-based comparison. /// \details Firstly compares length and if it equal then compare content @@ -1041,43 +982,30 @@ struct LIBMDBX_API_TYPE slice : public ::MDBX_val { /// or the same length and lexicographically less than `b`; /// `> 0` if `a` longer than `b`, /// or the same length and lexicographically great than `b`. - MDBX_NOTHROW_PURE_FUNCTION static MDBX_CXX14_CONSTEXPR intptr_t - compare_fast(const slice &a, const slice &b) noexcept; + MDBX_NOTHROW_PURE_FUNCTION static MDBX_CXX14_CONSTEXPR intptr_t compare_fast(const slice &a, const slice &b) noexcept; /// \brief Three-way lexicographically comparison. /// \return value: /// `== 0` if `a` lexicographically equal `b`; /// `< 0` if `a` lexicographically less than `b`; /// `> 0` if `a` lexicographically great than `b`. - MDBX_NOTHROW_PURE_FUNCTION static MDBX_CXX14_CONSTEXPR intptr_t - compare_lexicographically(const slice &a, const slice &b) noexcept; - friend MDBX_CXX14_CONSTEXPR bool operator==(const slice &a, - const slice &b) noexcept; - friend MDBX_CXX14_CONSTEXPR bool operator<(const slice &a, - const slice &b) noexcept; - friend MDBX_CXX14_CONSTEXPR bool operator>(const slice &a, - const slice &b) noexcept; - friend MDBX_CXX14_CONSTEXPR bool operator<=(const slice &a, - const slice &b) noexcept; - friend MDBX_CXX14_CONSTEXPR bool operator>=(const slice &a, - const slice &b) noexcept; - friend MDBX_CXX14_CONSTEXPR bool operator!=(const slice &a, - const slice &b) noexcept; + MDBX_NOTHROW_PURE_FUNCTION static MDBX_CXX14_CONSTEXPR intptr_t compare_lexicographically(const slice &a, + const slice &b) noexcept; + friend MDBX_CXX14_CONSTEXPR bool operator==(const slice &a, const slice &b) noexcept; + friend MDBX_CXX14_CONSTEXPR bool operator<(const slice &a, const slice &b) noexcept; + friend MDBX_CXX14_CONSTEXPR bool operator>(const slice &a, const slice &b) noexcept; + friend MDBX_CXX14_CONSTEXPR bool operator<=(const slice &a, const slice &b) noexcept; + friend MDBX_CXX14_CONSTEXPR bool operator>=(const slice &a, const slice &b) noexcept; + friend MDBX_CXX14_CONSTEXPR bool operator!=(const slice &a, const slice &b) noexcept; /// \brief Checks the slice is not refers to null address or has zero length. - MDBX_CXX11_CONSTEXPR bool is_valid() const noexcept { - return !(iov_base == nullptr && iov_len != 0); - } + MDBX_CXX11_CONSTEXPR bool is_valid() const noexcept { return !(iov_base == nullptr && iov_len != 0); } - /// \brief Build an invalid slice which non-zero length and refers to null - /// address. - MDBX_CXX14_CONSTEXPR static slice invalid() noexcept { - return slice(size_t(-1)); - } + /// \brief Build an invalid slice which non-zero length and refers to null address. + MDBX_CXX14_CONSTEXPR static slice invalid() noexcept { return slice(size_t(-1)); } template MDBX_CXX14_CONSTEXPR POD as_pod() const { - static_assert(::std::is_standard_layout::value && - !::std::is_pointer::value, + static_assert(::std::is_standard_layout::value && !::std::is_pointer::value, "Must be a standard layout type!"); if (MDBX_LIKELY(size() == sizeof(POD))) MDBX_CXX20_LIKELY { @@ -1089,24 +1017,39 @@ struct LIBMDBX_API_TYPE slice : public ::MDBX_val { } #ifdef MDBX_U128_TYPE - MDBX_U128_TYPE as_uint128() const; + MDBX_CXX14_CONSTEXPR MDBX_U128_TYPE as_uint128() const { return as_pod(); } +#endif /* MDBX_U128_TYPE */ + MDBX_CXX14_CONSTEXPR uint64_t as_uint64() const { return as_pod(); } + MDBX_CXX14_CONSTEXPR uint32_t as_uint32() const { return as_pod(); } + MDBX_CXX14_CONSTEXPR uint16_t as_uint16() const { return as_pod(); } + MDBX_CXX14_CONSTEXPR uint8_t as_uint8() const { return as_pod(); } + +#ifdef MDBX_I128_TYPE + MDBX_CXX14_CONSTEXPR MDBX_I128_TYPE as_int128() const { return as_pod(); } +#endif /* MDBX_I128_TYPE */ + MDBX_CXX14_CONSTEXPR int64_t as_int64() const { return as_pod(); } + MDBX_CXX14_CONSTEXPR int32_t as_int32() const { return as_pod(); } + MDBX_CXX14_CONSTEXPR int16_t as_int16() const { return as_pod(); } + MDBX_CXX14_CONSTEXPR int8_t as_int8() const { return as_pod(); } + +#ifdef MDBX_U128_TYPE + MDBX_U128_TYPE as_uint128_adapt() const; #endif /* MDBX_U128_TYPE */ - uint64_t as_uint64() const; - uint32_t as_uint32() const; - uint16_t as_uint16() const; - uint8_t as_uint8() const; + uint64_t as_uint64_adapt() const; + uint32_t as_uint32_adapt() const; + uint16_t as_uint16_adapt() const; + uint8_t as_uint8_adapt() const; #ifdef MDBX_I128_TYPE - MDBX_I128_TYPE as_int128() const; + MDBX_I128_TYPE as_int128_adapt() const; #endif /* MDBX_I128_TYPE */ - int64_t as_int64() const; - int32_t as_int32() const; - int16_t as_int16() const; - int8_t as_int8() const; + int64_t as_int64_adapt() const; + int32_t as_int32_adapt() const; + int16_t as_int16_adapt() const; + int8_t as_int8_adapt() const; protected: - MDBX_CXX11_CONSTEXPR slice(size_t invalid_length) noexcept - : ::MDBX_val({nullptr, invalid_length}) {} + MDBX_CXX11_CONSTEXPR slice(size_t invalid_length) noexcept : ::MDBX_val({nullptr, invalid_length}) {} }; //------------------------------------------------------------------------------ @@ -1114,8 +1057,7 @@ protected: namespace allocation_aware_details { template constexpr bool allocator_is_always_equal() noexcept { -#if defined(__cpp_lib_allocator_traits_is_always_equal) && \ - __cpp_lib_allocator_traits_is_always_equal >= 201411L +#if defined(__cpp_lib_allocator_traits_is_always_equal) && __cpp_lib_allocator_traits_is_always_equal >= 201411L return ::std::allocator_traits::is_always_equal::value; #else return ::std::is_empty::value; @@ -1123,17 +1065,13 @@ template constexpr bool allocator_is_always_equal() noexcept { } template ::propagate_on_container_move_assignment::value> + bool PoCMA = ::std::allocator_traits::propagate_on_container_move_assignment::value> struct move_assign_alloc; template struct move_assign_alloc { - static constexpr bool is_nothrow() noexcept { - return allocator_is_always_equal(); - } + static constexpr bool is_nothrow() noexcept { return allocator_is_always_equal(); } static MDBX_CXX20_CONSTEXPR bool is_moveable(T *target, T &source) noexcept { - return allocator_is_always_equal() || - target->get_allocator() == source.get_allocator(); + return allocator_is_always_equal() || target->get_allocator() == source.get_allocator(); } static MDBX_CXX20_CONSTEXPR void propagate(T *target, T &source) noexcept { assert(target->get_allocator() != source.get_allocator()); @@ -1144,8 +1082,7 @@ template struct move_assign_alloc { template struct move_assign_alloc { static constexpr bool is_nothrow() noexcept { - return allocator_is_always_equal() || - ::std::is_nothrow_move_assignable::value; + return allocator_is_always_equal() || ::std::is_nothrow_move_assignable::value; } static constexpr bool is_moveable(T *, T &) noexcept { return true; } static MDBX_CXX20_CONSTEXPR void propagate(T *target, T &source) { @@ -1155,14 +1092,12 @@ template struct move_assign_alloc { }; template ::propagate_on_container_copy_assignment::value> + bool PoCCA = ::std::allocator_traits::propagate_on_container_copy_assignment::value> struct copy_assign_alloc; template struct copy_assign_alloc { static constexpr bool is_nothrow() noexcept { return false; } - static MDBX_CXX20_CONSTEXPR void propagate(T *target, - const T &source) noexcept { + static MDBX_CXX20_CONSTEXPR void propagate(T *target, const T &source) noexcept { assert(target->get_allocator() != source.get_allocator()); (void)target; (void)source; @@ -1171,16 +1106,13 @@ template struct copy_assign_alloc { template struct copy_assign_alloc { static constexpr bool is_nothrow() noexcept { - return allocator_is_always_equal() || - ::std::is_nothrow_copy_assignable::value; + return allocator_is_always_equal() || ::std::is_nothrow_copy_assignable::value; } - static MDBX_CXX20_CONSTEXPR void - propagate(T *target, const T &source) noexcept(is_nothrow()) { + static MDBX_CXX20_CONSTEXPR void propagate(T *target, const T &source) noexcept(is_nothrow()) { if MDBX_IF_CONSTEXPR (!allocator_is_always_equal()) { if (MDBX_UNLIKELY(target->get_allocator() != source.get_allocator())) MDBX_CXX20_UNLIKELY target->get_allocator() = - ::std::allocator_traits::select_on_container_copy_construction( - source.get_allocator()); + ::std::allocator_traits::select_on_container_copy_construction(source.get_allocator()); } else { /* gag for buggy compilers */ (void)target; @@ -1190,16 +1122,12 @@ template struct copy_assign_alloc { }; template ::propagate_on_container_swap::value> + bool PoCS = ::std::allocator_traits::propagate_on_container_swap::value> struct swap_alloc; template struct swap_alloc { - static constexpr bool is_nothrow() noexcept { - return allocator_is_always_equal(); - } - static MDBX_CXX20_CONSTEXPR void propagate(T *target, - T &source) noexcept(is_nothrow()) { + static constexpr bool is_nothrow() noexcept { return allocator_is_always_equal(); } + static MDBX_CXX20_CONSTEXPR void propagate(T *target, T &source) noexcept(is_nothrow()) { if MDBX_IF_CONSTEXPR (!allocator_is_always_equal()) { if (MDBX_UNLIKELY(target->get_allocator() != source.get_allocator())) MDBX_CXX20_UNLIKELY throw_allocators_mismatch(); @@ -1217,11 +1145,9 @@ template struct swap_alloc { #if defined(__cpp_lib_is_swappable) && __cpp_lib_is_swappable >= 201603L ::std::is_nothrow_swappable() || #endif /* __cpp_lib_is_swappable >= 201603L */ - (::std::is_nothrow_move_constructible::value && - ::std::is_nothrow_move_assignable::value); + (::std::is_nothrow_move_constructible::value && ::std::is_nothrow_move_assignable::value); } - static MDBX_CXX20_CONSTEXPR void propagate(T *target, - T &source) noexcept(is_nothrow()) { + static MDBX_CXX20_CONSTEXPR void propagate(T *target, T &source) noexcept(is_nothrow()) { if MDBX_IF_CONSTEXPR (!allocator_is_always_equal()) { if (MDBX_UNLIKELY(target->get_allocator() != source.get_allocator())) MDBX_CXX20_UNLIKELY ::std::swap(*target, source); @@ -1238,27 +1164,22 @@ template struct swap_alloc { struct default_capacity_policy { enum : size_t { extra_inplace_storage = 0, + inplace_storage_size_rounding = 16, pettiness_threshold = 64, max_reserve = 65536 }; static MDBX_CXX11_CONSTEXPR size_t round(const size_t value) { - static_assert((pettiness_threshold & (pettiness_threshold - 1)) == 0, - "pettiness_threshold must be a power of 2"); - static_assert(pettiness_threshold % 2 == 0, - "pettiness_threshold must be even"); - static_assert(pettiness_threshold >= sizeof(uint64_t), - "pettiness_threshold must be > 7"); + static_assert((pettiness_threshold & (pettiness_threshold - 1)) == 0, "pettiness_threshold must be a power of 2"); + static_assert(pettiness_threshold % 2 == 0, "pettiness_threshold must be even"); + static_assert(pettiness_threshold >= sizeof(uint64_t), "pettiness_threshold must be > 7"); constexpr const auto pettiness_mask = ~size_t(pettiness_threshold - 1); return (value + pettiness_threshold - 1) & pettiness_mask; } - static MDBX_CXX11_CONSTEXPR size_t advise(const size_t current, - const size_t wanna) { - static_assert(max_reserve % pettiness_threshold == 0, - "max_reserve must be a multiple of pettiness_threshold"); - static_assert(max_reserve / 3 > pettiness_threshold, - "max_reserve must be > pettiness_threshold * 3"); + static MDBX_CXX11_CONSTEXPR size_t advise(const size_t current, const size_t wanna) { + static_assert(max_reserve % pettiness_threshold == 0, "max_reserve must be a multiple of pettiness_threshold"); + static_assert(max_reserve / 3 > pettiness_threshold, "max_reserve must be > pettiness_threshold * 3"); if (wanna > current) /* doubling capacity, but don't made reserve more than max_reserve */ return round(wanna + ::std::min(size_t(max_reserve), current)); @@ -1279,23 +1200,20 @@ struct LIBMDBX_API to_hex { const slice source; const bool uppercase = false; const unsigned wrap_width = 0; - MDBX_CXX11_CONSTEXPR to_hex(const slice &source, bool uppercase = false, - unsigned wrap_width = 0) noexcept + MDBX_CXX11_CONSTEXPR to_hex(const slice &source, bool uppercase = false, unsigned wrap_width = 0) noexcept : source(source), uppercase(uppercase), wrap_width(wrap_width) { MDBX_ASSERT_CXX20_CONCEPT_SATISFIED(SliceTranscoder, to_hex); } /// \brief Returns a string with a hexadecimal dump of a passed slice. - template + template string as_string(const ALLOCATOR &allocator = ALLOCATOR()) const { return make_string(*this, allocator); } /// \brief Returns a buffer with a hexadecimal dump of a passed slice. - template - buffer - as_buffer(const ALLOCATOR &allocator = ALLOCATOR()) const { + template + buffer as_buffer(const ALLOCATOR &allocator = ALLOCATOR()) const { return make_buffer(*this, allocator); } @@ -1311,8 +1229,7 @@ struct LIBMDBX_API to_hex { char *write_bytes(char *dest, size_t dest_size) const; /// \brief Output hexadecimal dump of passed slice to the std::ostream. - /// \throws std::ios_base::failure corresponding to std::ostream::write() - /// behaviour. + /// \throws std::ios_base::failure corresponding to std::ostream::write() behaviour. ::std::ostream &output(::std::ostream &out) const; /// \brief Checks whether a passed slice is empty, @@ -1330,24 +1247,21 @@ struct LIBMDBX_API to_base58 { const slice source; const unsigned wrap_width = 0; MDBX_CXX11_CONSTEXPR - to_base58(const slice &source, unsigned wrap_width = 0) noexcept - : source(source), wrap_width(wrap_width) { + to_base58(const slice &source, unsigned wrap_width = 0) noexcept : source(source), wrap_width(wrap_width) { MDBX_ASSERT_CXX20_CONCEPT_SATISFIED(SliceTranscoder, to_base58); } /// \brief Returns a string with a /// [Base58](https://en.wikipedia.org/wiki/Base58) dump of a passed slice. - template + template string as_string(const ALLOCATOR &allocator = ALLOCATOR()) const { return make_string(*this, allocator); } /// \brief Returns a buffer with a /// [Base58](https://en.wikipedia.org/wiki/Base58) dump of a passed slice. - template - buffer - as_buffer(const ALLOCATOR &allocator = ALLOCATOR()) const { + template + buffer as_buffer(const ALLOCATOR &allocator = ALLOCATOR()) const { return make_buffer(*this, allocator); } @@ -1358,23 +1272,18 @@ struct LIBMDBX_API to_base58 { return wrap_width ? bytes + bytes / wrap_width : bytes; } - /// \brief Fills the buffer by [Base58](https://en.wikipedia.org/wiki/Base58) - /// dump of passed slice. + /// \brief Fills the buffer by [Base58](https://en.wikipedia.org/wiki/Base58) dump of passed slice. /// \throws std::length_error if given buffer is too small. char *write_bytes(char *dest, size_t dest_size) const; - /// \brief Output [Base58](https://en.wikipedia.org/wiki/Base58) - /// dump of passed slice to the std::ostream. - /// \throws std::ios_base::failure corresponding to std::ostream::write() - /// behaviour. + /// \brief Output [Base58](https://en.wikipedia.org/wiki/Base58) dump of passed slice to the std::ostream. + /// \throws std::ios_base::failure corresponding to std::ostream::write() behaviour. ::std::ostream &output(::std::ostream &out) const; - /// \brief Checks whether a passed slice is empty, - /// and therefore there will be no output bytes. + /// \brief Checks whether a passed slice is empty, and therefore there will be no output bytes. bool is_empty() const noexcept { return source.empty(); } - /// \brief Checks whether the content of a passed slice is a valid data - /// and could be encoded or unexpectedly not. + /// \brief Checks whether the content of a passed slice is a valid data and could be encoded or unexpectedly not. bool is_erroneous() const noexcept { return false; } }; @@ -1384,24 +1293,21 @@ struct LIBMDBX_API to_base64 { const slice source; const unsigned wrap_width = 0; MDBX_CXX11_CONSTEXPR - to_base64(const slice &source, unsigned wrap_width = 0) noexcept - : source(source), wrap_width(wrap_width) { + to_base64(const slice &source, unsigned wrap_width = 0) noexcept : source(source), wrap_width(wrap_width) { MDBX_ASSERT_CXX20_CONCEPT_SATISFIED(SliceTranscoder, to_base64); } /// \brief Returns a string with a /// [Base64](https://en.wikipedia.org/wiki/Base64) dump of a passed slice. - template + template string as_string(const ALLOCATOR &allocator = ALLOCATOR()) const { return make_string(*this, allocator); } /// \brief Returns a buffer with a /// [Base64](https://en.wikipedia.org/wiki/Base64) dump of a passed slice. - template - buffer - as_buffer(const ALLOCATOR &allocator = ALLOCATOR()) const { + template + buffer as_buffer(const ALLOCATOR &allocator = ALLOCATOR()) const { return make_buffer(*this, allocator); } @@ -1419,8 +1325,7 @@ struct LIBMDBX_API to_base64 { /// \brief Output [Base64](https://en.wikipedia.org/wiki/Base64) /// dump of passed slice to the std::ostream. - /// \throws std::ios_base::failure corresponding to std::ostream::write() - /// behaviour. + /// \throws std::ios_base::failure corresponding to std::ostream::write() behaviour. ::std::ostream &output(::std::ostream &out) const; /// \brief Checks whether a passed slice is empty, @@ -1432,55 +1337,40 @@ struct LIBMDBX_API to_base64 { bool is_erroneous() const noexcept { return false; } }; -inline ::std::ostream &operator<<(::std::ostream &out, const to_hex &wrapper) { - return wrapper.output(out); -} -inline ::std::ostream &operator<<(::std::ostream &out, - const to_base58 &wrapper) { - return wrapper.output(out); -} -inline ::std::ostream &operator<<(::std::ostream &out, - const to_base64 &wrapper) { - return wrapper.output(out); -} +inline ::std::ostream &operator<<(::std::ostream &out, const to_hex &wrapper) { return wrapper.output(out); } +inline ::std::ostream &operator<<(::std::ostream &out, const to_base58 &wrapper) { return wrapper.output(out); } +inline ::std::ostream &operator<<(::std::ostream &out, const to_base64 &wrapper) { return wrapper.output(out); } /// \brief Hexadecimal decoder which satisfy \ref SliceTranscoder concept. struct LIBMDBX_API from_hex { const slice source; const bool ignore_spaces = false; - MDBX_CXX11_CONSTEXPR from_hex(const slice &source, - bool ignore_spaces = false) noexcept + MDBX_CXX11_CONSTEXPR from_hex(const slice &source, bool ignore_spaces = false) noexcept : source(source), ignore_spaces(ignore_spaces) { MDBX_ASSERT_CXX20_CONCEPT_SATISFIED(SliceTranscoder, from_hex); } /// \brief Decodes hexadecimal dump from a passed slice to returned string. - template + template string as_string(const ALLOCATOR &allocator = ALLOCATOR()) const { return make_string(*this, allocator); } /// \brief Decodes hexadecimal dump from a passed slice to returned buffer. - template - buffer - as_buffer(const ALLOCATOR &allocator = ALLOCATOR()) const { + template + buffer as_buffer(const ALLOCATOR &allocator = ALLOCATOR()) const { return make_buffer(*this, allocator); } /// \brief Returns the number of bytes needed for conversion /// hexadecimal dump from a passed slice to decoded data. - MDBX_CXX11_CONSTEXPR size_t envisage_result_length() const noexcept { - return source.length() >> 1; - } + MDBX_CXX11_CONSTEXPR size_t envisage_result_length() const noexcept { return source.length() >> 1; } - /// \brief Fills the destination with data decoded from hexadecimal dump - /// from a passed slice. + /// \brief Fills the destination with data decoded from hexadecimal dump from a passed slice. /// \throws std::length_error if given buffer is too small. char *write_bytes(char *dest, size_t dest_size) const; - /// \brief Checks whether a passed slice is empty, - /// and therefore there will be no output bytes. + /// \brief Checks whether a passed slice is empty, and therefore there will be no output bytes. bool is_empty() const noexcept { return source.empty(); } /// \brief Checks whether the content of a passed slice is a valid hexadecimal @@ -1493,31 +1383,27 @@ struct LIBMDBX_API from_hex { struct LIBMDBX_API from_base58 { const slice source; const bool ignore_spaces = false; - MDBX_CXX11_CONSTEXPR from_base58(const slice &source, - bool ignore_spaces = false) noexcept + MDBX_CXX11_CONSTEXPR from_base58(const slice &source, bool ignore_spaces = false) noexcept : source(source), ignore_spaces(ignore_spaces) { MDBX_ASSERT_CXX20_CONCEPT_SATISFIED(SliceTranscoder, from_base58); } /// \brief Decodes [Base58](https://en.wikipedia.org/wiki/Base58) dump from a /// passed slice to returned string. - template + template string as_string(const ALLOCATOR &allocator = ALLOCATOR()) const { return make_string(*this, allocator); } /// \brief Decodes [Base58](https://en.wikipedia.org/wiki/Base58) dump from a /// passed slice to returned buffer. - template - buffer - as_buffer(const ALLOCATOR &allocator = ALLOCATOR()) const { + template + buffer as_buffer(const ALLOCATOR &allocator = ALLOCATOR()) const { return make_buffer(*this, allocator); } /// \brief Returns the number of bytes needed for conversion - /// [Base58](https://en.wikipedia.org/wiki/Base58) dump from a passed slice to - /// decoded data. + /// [Base58](https://en.wikipedia.org/wiki/Base58) dump from a passed slice to decoded data. MDBX_CXX11_CONSTEXPR size_t envisage_result_length() const noexcept { return source.length() /* могут быть все нули кодируемые один-к-одному */; } @@ -1527,13 +1413,11 @@ struct LIBMDBX_API from_base58 { /// \throws std::length_error if given buffer is too small. char *write_bytes(char *dest, size_t dest_size) const; - /// \brief Checks whether a passed slice is empty, - /// and therefore there will be no output bytes. + /// \brief Checks whether a passed slice is empty, and therefore there will be no output bytes. bool is_empty() const noexcept { return source.empty(); } /// \brief Checks whether the content of a passed slice is a valid - /// [Base58](https://en.wikipedia.org/wiki/Base58) dump, and therefore there - /// could be decoded or not. + /// [Base58](https://en.wikipedia.org/wiki/Base58) dump, and therefore there could be decoded or not. bool is_erroneous() const noexcept; }; @@ -1542,34 +1426,28 @@ struct LIBMDBX_API from_base58 { struct LIBMDBX_API from_base64 { const slice source; const bool ignore_spaces = false; - MDBX_CXX11_CONSTEXPR from_base64(const slice &source, - bool ignore_spaces = false) noexcept + MDBX_CXX11_CONSTEXPR from_base64(const slice &source, bool ignore_spaces = false) noexcept : source(source), ignore_spaces(ignore_spaces) { MDBX_ASSERT_CXX20_CONCEPT_SATISFIED(SliceTranscoder, from_base64); } /// \brief Decodes [Base64](https://en.wikipedia.org/wiki/Base64) dump from a /// passed slice to returned string. - template + template string as_string(const ALLOCATOR &allocator = ALLOCATOR()) const { return make_string(*this, allocator); } /// \brief Decodes [Base64](https://en.wikipedia.org/wiki/Base64) dump from a /// passed slice to returned buffer. - template - buffer - as_buffer(const ALLOCATOR &allocator = ALLOCATOR()) const { + template + buffer as_buffer(const ALLOCATOR &allocator = ALLOCATOR()) const { return make_buffer(*this, allocator); } /// \brief Returns the number of bytes needed for conversion - /// [Base64](https://en.wikipedia.org/wiki/Base64) dump from a passed slice to - /// decoded data. - MDBX_CXX11_CONSTEXPR size_t envisage_result_length() const noexcept { - return (source.length() + 3) / 4 * 3; - } + /// [Base64](https://en.wikipedia.org/wiki/Base64) dump from a passed slice to decoded data. + MDBX_CXX11_CONSTEXPR size_t envisage_result_length() const noexcept { return (source.length() + 3) / 4 * 3; } /// \brief Fills the destination with data decoded from /// [Base64](https://en.wikipedia.org/wiki/Base64) dump from a passed slice. @@ -1590,8 +1468,7 @@ struct LIBMDBX_API from_base64 { template class buffer { public: #if !defined(_MSC_VER) || _MSC_VER > 1900 - using allocator_type = typename ::std::allocator_traits< - ALLOCATOR>::template rebind_alloc; + using allocator_type = typename ::std::allocator_traits::template rebind_alloc; #else using allocator_type = typename ALLOCATOR::template rebind::other; #endif /* MSVC is mad */ @@ -1601,52 +1478,45 @@ public: max_length = MDBX_MAXDATASIZE, max_capacity = (max_length / 3u * 4u + 1023u) & ~size_t(1023), extra_inplace_storage = reservation_policy::extra_inplace_storage, + inplace_storage_size_rounding = + (alignof(max_align_t) * 2 > size_t(reservation_policy::inplace_storage_size_rounding)) + ? alignof(max_align_t) * 2 + : size_t(reservation_policy::inplace_storage_size_rounding), pettiness_threshold = reservation_policy::pettiness_threshold }; private: friend class txn; - struct silo; - using swap_alloc = allocation_aware_details::swap_alloc; + using swap_alloc = allocation_aware_details::swap_alloc; struct silo /* Empty Base Class Optimization */ : public allocator_type { - MDBX_CXX20_CONSTEXPR const allocator_type &get_allocator() const noexcept { - return *this; - } - MDBX_CXX20_CONSTEXPR allocator_type &get_allocator() noexcept { - return *this; - } + MDBX_CXX20_CONSTEXPR const allocator_type &get_allocator() const noexcept { return *this; } + MDBX_CXX20_CONSTEXPR allocator_type &get_allocator() noexcept { return *this; } using allocator_pointer = typename allocator_traits::pointer; using allocator_const_pointer = typename allocator_traits::const_pointer; - MDBX_CXX20_CONSTEXPR ::std::pair - allocate_storage(size_t bytes) { + MDBX_CXX20_CONSTEXPR ::std::pair allocate_storage(size_t bytes) { assert(bytes >= sizeof(bin)); constexpr size_t unit = sizeof(typename allocator_type::value_type); - static_assert((unit & (unit - 1)) == 0, - "size of ALLOCATOR::value_type should be a power of 2"); + static_assert((unit & (unit - 1)) == 0, "size of ALLOCATOR::value_type should be a power of 2"); static_assert(unit > 0, "size of ALLOCATOR::value_type must be > 0"); const size_t n = (bytes + unit - 1) / unit; - return ::std::make_pair(allocator_traits::allocate(get_allocator(), n), - n * unit); + return ::std::make_pair(allocator_traits::allocate(get_allocator(), n), n * unit); } - MDBX_CXX20_CONSTEXPR void deallocate_storage(allocator_pointer ptr, - size_t bytes) { + MDBX_CXX20_CONSTEXPR void deallocate_storage(allocator_pointer ptr, size_t bytes) { constexpr size_t unit = sizeof(typename allocator_type::value_type); assert(ptr && bytes >= sizeof(bin) && bytes >= unit && bytes % unit == 0); allocator_traits::deallocate(get_allocator(), ptr, bytes / unit); } - static MDBX_CXX17_CONSTEXPR void * - to_address(allocator_pointer ptr) noexcept { + static MDBX_CXX17_CONSTEXPR void *to_address(allocator_pointer ptr) noexcept { #if defined(__cpp_lib_to_address) && __cpp_lib_to_address >= 201711L return static_cast(::std::to_address(ptr)); #else return static_cast(::std::addressof(*ptr)); #endif /* __cpp_lib_to_address */ } - static MDBX_CXX17_CONSTEXPR const void * - to_address(allocator_const_pointer ptr) noexcept { + static MDBX_CXX17_CONSTEXPR const void *to_address(allocator_const_pointer ptr) noexcept { #if defined(__cpp_lib_to_address) && __cpp_lib_to_address >= 201711L return static_cast(::std::to_address(ptr)); #else @@ -1654,95 +1524,81 @@ private: #endif /* __cpp_lib_to_address */ } - union bin { - struct allocated { + union alignas(max_align_t) bin { + struct stub_allocated_holder /* используется только для вычисления (минимального необходимого) размера, + с учетом выравнивания */ + { allocator_pointer ptr_; - size_t capacity_bytes_; - constexpr allocated(allocator_pointer ptr, size_t bytes) noexcept - : ptr_(ptr), capacity_bytes_(bytes) {} - constexpr allocated(const allocated &) noexcept = default; - constexpr allocated(allocated &&) noexcept = default; - MDBX_CXX17_CONSTEXPR allocated & - operator=(const allocated &) noexcept = default; - MDBX_CXX17_CONSTEXPR allocated & - operator=(allocated &&) noexcept = default; + size_t stub_capacity_bytes_; }; - allocated allocated_; - uint64_t align_hint_; - byte inplace_[(sizeof(allocated) + extra_inplace_storage + 7u) & - ~size_t(7)]; + enum : byte { lastbyte_poison = 0, lastbyte_inplace_signature = byte(~byte(lastbyte_poison)) }; + enum : size_t { + inplace_signature_limit = size_t(lastbyte_inplace_signature) + << (sizeof(size_t /* allocated::capacity_bytes_ */) - 1) * CHAR_BIT, + inplace_size_rounding = size_t(inplace_storage_size_rounding) - 1, + inplace_size = + (sizeof(stub_allocated_holder) + extra_inplace_storage + inplace_size_rounding) & ~inplace_size_rounding + }; - static constexpr bool - is_suitable_for_inplace(size_t capacity_bytes) noexcept { - static_assert(sizeof(bin) == sizeof(inplace_), "WTF?"); - return capacity_bytes < sizeof(bin); - } + struct capacity_holder { + byte pad_[inplace_size - sizeof(allocator_pointer)]; + size_t bytes_; + }; - enum : byte { - /* Little Endian: - * last byte is the most significant byte of u_.allocated.cap, - * so use higher bit of capacity as the inplace-flag */ - le_lastbyte_mask = 0x80, - /* Big Endian: - * last byte is the least significant byte of u_.allocated.cap, - * so use lower bit of capacity as the inplace-flag. */ - be_lastbyte_mask = 0x01 + struct inplace_flag_holder { + byte buffer_[inplace_size - sizeof(byte)]; + byte lastbyte_; }; - static constexpr byte inplace_lastbyte_mask() noexcept { - static_assert( - endian::native == endian::little || endian::native == endian::big, - "Only the little-endian or big-endian bytes order are supported"); - return (endian::native == endian::little) ? le_lastbyte_mask - : be_lastbyte_mask; - } - constexpr byte lastbyte() const noexcept { - return inplace_[sizeof(bin) - 1]; - } - MDBX_CXX17_CONSTEXPR byte &lastbyte() noexcept { - return inplace_[sizeof(bin) - 1]; + allocator_pointer allocated_ptr_; + capacity_holder capacity_; + inplace_flag_holder inplace_; + + static constexpr bool is_suitable_for_inplace(size_t capacity_bytes) noexcept { + static_assert((size_t(reservation_policy::inplace_storage_size_rounding) & + (size_t(reservation_policy::inplace_storage_size_rounding) - 1)) == 0, + "CAPACITY_POLICY::inplace_storage_size_rounding must be power of 2"); + static_assert(sizeof(bin) == sizeof(inplace_) && sizeof(bin) == sizeof(capacity_), "WTF?"); + return capacity_bytes < sizeof(bin); } constexpr bool is_inplace() const noexcept { - return (lastbyte() & inplace_lastbyte_mask()) != 0; + static_assert(size_t(inplace_signature_limit) > size_t(max_capacity), "WTF?"); + static_assert(std::numeric_limits::max() - (std::numeric_limits::max() >> CHAR_BIT) == + inplace_signature_limit, + "WTF?"); + return inplace_.lastbyte_ == lastbyte_inplace_signature; } constexpr bool is_allocated() const noexcept { return !is_inplace(); } - template - MDBX_CXX17_CONSTEXPR byte *make_inplace() noexcept { + template MDBX_CXX17_CONSTEXPR byte *make_inplace() noexcept { if (destroy_ptr) { MDBX_CONSTEXPR_ASSERT(is_allocated()); /* properly destroy allocator::pointer */ - allocated_.~allocated(); + allocated_ptr_.~allocator_pointer(); } if (::std::is_trivial::value) /* workaround for "uninitialized" warning from some compilers */ - ::std::memset(&allocated_.ptr_, 0, sizeof(allocated_.ptr_)); - lastbyte() = inplace_lastbyte_mask(); - MDBX_CONSTEXPR_ASSERT(is_inplace() && address() == inplace_ && - is_suitable_for_inplace(capacity())); + memset(&allocated_ptr_, 0, sizeof(allocated_ptr_)); + inplace_.lastbyte_ = lastbyte_inplace_signature; + MDBX_CONSTEXPR_ASSERT(is_inplace() && address() == inplace_.buffer_ && is_suitable_for_inplace(capacity())); return address(); } template - MDBX_CXX17_CONSTEXPR byte * - make_allocated(allocator_pointer ptr, size_t capacity_bytes) noexcept { - MDBX_CONSTEXPR_ASSERT( - (capacity_bytes & be_lastbyte_mask) == 0 && - ((capacity_bytes >> - (sizeof(allocated_.capacity_bytes_) - 1) * CHAR_BIT) & - le_lastbyte_mask) == 0); - if (construct_ptr) + MDBX_CXX17_CONSTEXPR byte *make_allocated(allocator_pointer ptr, size_t capacity_bytes) noexcept { + MDBX_CONSTEXPR_ASSERT(inplace_signature_limit > capacity_bytes); + if (construct_ptr) { /* properly construct allocator::pointer */ - new (&allocated_) allocated(ptr, capacity_bytes); - else { + new (&allocated_ptr_) allocator_pointer(ptr); + capacity_.bytes_ = capacity_bytes; + } else { MDBX_CONSTEXPR_ASSERT(is_allocated()); - allocated_.ptr_ = ptr; - allocated_.capacity_bytes_ = capacity_bytes; + allocated_ptr_ = ptr; + capacity_.bytes_ = capacity_bytes; } - MDBX_CONSTEXPR_ASSERT(is_allocated() && address() == to_address(ptr) && - capacity() == capacity_bytes); + MDBX_CONSTEXPR_ASSERT(is_allocated() && address() == to_address(ptr) && capacity() == capacity_bytes); return address(); } @@ -1751,24 +1607,24 @@ private: make_inplace(); (void)capacity_bytes; } - MDBX_CXX20_CONSTEXPR bin(allocator_pointer ptr, - size_t capacity_bytes) noexcept { + MDBX_CXX20_CONSTEXPR bin(allocator_pointer ptr, size_t capacity_bytes) noexcept { MDBX_CONSTEXPR_ASSERT(!is_suitable_for_inplace(capacity_bytes)); make_allocated(ptr, capacity_bytes); } MDBX_CXX20_CONSTEXPR ~bin() { if (is_allocated()) /* properly destroy allocator::pointer */ - allocated_.~allocated(); + allocated_ptr_.~allocator_pointer(); } MDBX_CXX20_CONSTEXPR bin(bin &&ditto) noexcept { if (ditto.is_inplace()) { // micro-optimization: don't use make_inplace<> here // since memcpy() will copy the flag. - memcpy(inplace_, ditto.inplace_, sizeof(inplace_)); + memcpy(&inplace_, &ditto.inplace_, sizeof(inplace_)); MDBX_CONSTEXPR_ASSERT(is_inplace()); } else { - new (&allocated_) allocated(::std::move(ditto.allocated_)); + new (&allocated_ptr_) allocator_pointer(::std::move(ditto.allocated_ptr_)); + capacity_.bytes_ = ditto.capacity_.bytes_; ditto.make_inplace(); MDBX_CONSTEXPR_ASSERT(is_allocated()); } @@ -1780,15 +1636,13 @@ private: // since memcpy() will copy the flag. if (is_allocated()) /* properly destroy allocator::pointer */ - allocated_.~allocated(); - memcpy(inplace_, ditto.inplace_, sizeof(inplace_)); + allocated_ptr_.~allocator_pointer(); + memcpy(&inplace_, &ditto.inplace_, sizeof(inplace_)); MDBX_CONSTEXPR_ASSERT(is_inplace()); } else if (is_inplace()) - make_allocated(ditto.allocated_.ptr_, - ditto.allocated_.capacity_bytes_); + make_allocated(ditto.allocated_ptr_, ditto.capacity_.bytes_); else - make_allocated(ditto.allocated_.ptr_, - ditto.allocated_.capacity_bytes_); + make_allocated(ditto.allocated_ptr_, ditto.capacity_.bytes_); return *this; } @@ -1799,29 +1653,22 @@ private: return *this; } - static MDBX_CXX20_CONSTEXPR size_t advise_capacity(const size_t current, - const size_t wanna) { + static MDBX_CXX20_CONSTEXPR size_t advise_capacity(const size_t current, const size_t wanna) { if (MDBX_UNLIKELY(wanna > max_capacity)) MDBX_CXX20_UNLIKELY throw_max_length_exceeded(); const size_t advised = reservation_policy::advise(current, wanna); assert(advised >= wanna); - return ::std::min(size_t(max_capacity), - ::std::max(sizeof(bin) - 1, advised)); + return ::std::min(size_t(max_capacity), ::std::max(sizeof(bin) - 1, advised)); } constexpr const byte *address() const noexcept { - return is_inplace() - ? inplace_ - : static_cast(to_address(allocated_.ptr_)); + return is_inplace() ? inplace_.buffer_ : static_cast(to_address(allocated_ptr_)); } MDBX_CXX17_CONSTEXPR byte *address() noexcept { - return is_inplace() ? inplace_ - : static_cast(to_address(allocated_.ptr_)); - } - constexpr size_t capacity() const noexcept { - return is_inplace() ? sizeof(bin) - 1 : allocated_.capacity_bytes_; + return is_inplace() ? inplace_.buffer_ : static_cast(to_address(allocated_ptr_)); } + constexpr size_t capacity() const noexcept { return is_inplace() ? sizeof(bin) - 1 : capacity_.bytes_; } } bin_; MDBX_CXX20_CONSTEXPR void *init(size_t capacity) { @@ -1838,36 +1685,30 @@ private: MDBX_CXX20_CONSTEXPR void release() noexcept { if (bin_.is_allocated()) { - deallocate_storage(bin_.allocated_.ptr_, - bin_.allocated_.capacity_bytes_); + deallocate_storage(bin_.allocated_ptr_, bin_.capacity_.bytes_); bin_.template make_inplace(); } } template - MDBX_CXX20_CONSTEXPR void * - reshape(const size_t wanna_capacity, const size_t wanna_headroom, - const void *const content, const size_t length) { + MDBX_CXX20_CONSTEXPR void *reshape(const size_t wanna_capacity, const size_t wanna_headroom, + const void *const content, const size_t length) { assert(wanna_capacity >= wanna_headroom + length); const size_t old_capacity = bin_.capacity(); - const size_t new_capacity = - bin::advise_capacity(old_capacity, wanna_capacity); + const size_t new_capacity = bin::advise_capacity(old_capacity, wanna_capacity); if (MDBX_LIKELY(new_capacity == old_capacity)) MDBX_CXX20_LIKELY { - assert(bin_.is_inplace() == - bin::is_suitable_for_inplace(new_capacity)); + assert(bin_.is_inplace() == bin::is_suitable_for_inplace(new_capacity)); byte *const new_place = bin_.address() + wanna_headroom; if (MDBX_LIKELY(length)) MDBX_CXX20_LIKELY { if (external_content) memcpy(new_place, content, length); else { - const size_t old_headroom = - bin_.address() - static_cast(content); + const size_t old_headroom = bin_.address() - static_cast(content); assert(old_capacity >= old_headroom + length); if (MDBX_UNLIKELY(old_headroom != wanna_headroom)) - MDBX_CXX20_UNLIKELY ::std::memmove(new_place, content, - length); + MDBX_CXX20_UNLIKELY ::std::memmove(new_place, content, length); } } return new_place; @@ -1875,9 +1716,8 @@ private: if (bin::is_suitable_for_inplace(new_capacity)) { assert(bin_.is_allocated()); - const auto old_allocated = ::std::move(bin_.allocated_.ptr_); - byte *const new_place = - bin_.template make_inplace() + wanna_headroom; + const auto old_allocated = ::std::move(bin_.allocated_ptr_); + byte *const new_place = bin_.template make_inplace() + wanna_headroom; if (MDBX_LIKELY(length)) MDBX_CXX20_LIKELY memcpy(new_place, content, length); deallocate_storage(old_allocated, old_capacity); @@ -1887,22 +1727,19 @@ private: if (!bin_.is_allocated()) { const auto pair = allocate_storage(new_capacity); assert(pair.second >= new_capacity); - byte *const new_place = - static_cast(to_address(pair.first)) + wanna_headroom; + byte *const new_place = static_cast(to_address(pair.first)) + wanna_headroom; if (MDBX_LIKELY(length)) MDBX_CXX20_LIKELY memcpy(new_place, content, length); bin_.template make_allocated(pair.first, pair.second); return new_place; } - const auto old_allocated = ::std::move(bin_.allocated_.ptr_); + const auto old_allocated = ::std::move(bin_.allocated_ptr_); if (external_content) deallocate_storage(old_allocated, old_capacity); const auto pair = allocate_storage(new_capacity); assert(pair.second >= new_capacity); - byte *const new_place = - bin_.template make_allocated(pair.first, pair.second) + - wanna_headroom; + byte *const new_place = bin_.template make_allocated(pair.first, pair.second) + wanna_headroom; if (MDBX_LIKELY(length)) MDBX_CXX20_LIKELY memcpy(new_place, content, length); if (!external_content) @@ -1918,8 +1755,7 @@ private: assert(capacity() >= offset); return bin_.address() + offset; } - MDBX_CXX20_CONSTEXPR byte *put(size_t offset, const void *ptr, - size_t length) { + MDBX_CXX20_CONSTEXPR byte *put(size_t offset, const void *ptr, size_t length) { assert(capacity() >= offset + length); return static_cast(memcpy(get(offset), ptr, length)); } @@ -1929,128 +1765,92 @@ private: MDBX_CXX20_CONSTEXPR silo() noexcept : allocator_type() { init(0); } MDBX_CXX20_CONSTEXPR - silo(const allocator_type &alloc) noexcept : allocator_type(alloc) { - init(0); - } + silo(const allocator_type &alloc) noexcept : allocator_type(alloc) { init(0); } MDBX_CXX20_CONSTEXPR silo(size_t capacity) { init(capacity); } - MDBX_CXX20_CONSTEXPR silo(size_t capacity, const allocator_type &alloc) - : silo(alloc) { - init(capacity); - } + MDBX_CXX20_CONSTEXPR silo(size_t capacity, const allocator_type &alloc) : silo(alloc) { init(capacity); } - MDBX_CXX20_CONSTEXPR silo(silo &&ditto) noexcept( - ::std::is_nothrow_move_constructible::value) - : allocator_type(::std::move(ditto.get_allocator())), - bin_(::std::move(ditto.bin_)) {} + MDBX_CXX20_CONSTEXPR silo(silo &&ditto) noexcept(::std::is_nothrow_move_constructible::value) + : allocator_type(::std::move(ditto.get_allocator())), bin_(::std::move(ditto.bin_)) {} - MDBX_CXX20_CONSTEXPR silo(size_t capacity, size_t headroom, const void *ptr, - size_t length) - : silo(capacity) { + MDBX_CXX20_CONSTEXPR silo(size_t capacity, size_t headroom, const void *ptr, size_t length) : silo(capacity) { assert(capacity >= headroom + length); if (length) put(headroom, ptr, length); } // select_on_container_copy_construction() - MDBX_CXX20_CONSTEXPR silo(size_t capacity, size_t headroom, const void *ptr, - size_t length, const allocator_type &alloc) + MDBX_CXX20_CONSTEXPR silo(size_t capacity, size_t headroom, const void *ptr, size_t length, + const allocator_type &alloc) : silo(capacity, alloc) { assert(capacity >= headroom + length); if (length) put(headroom, ptr, length); } - MDBX_CXX20_CONSTEXPR silo(const void *ptr, size_t length) - : silo(length, 0, ptr, length) {} - MDBX_CXX20_CONSTEXPR silo(const void *ptr, size_t length, - const allocator_type &alloc) + MDBX_CXX20_CONSTEXPR silo(const void *ptr, size_t length) : silo(length, 0, ptr, length) {} + MDBX_CXX20_CONSTEXPR silo(const void *ptr, size_t length, const allocator_type &alloc) : silo(length, 0, ptr, length, alloc) {} ~silo() { release(); } //-------------------------------------------------------------------------- - MDBX_CXX20_CONSTEXPR void *assign(size_t headroom, const void *ptr, - size_t length, size_t tailroom) { + MDBX_CXX20_CONSTEXPR void *assign(size_t headroom, const void *ptr, size_t length, size_t tailroom) { return reshape(headroom + length + tailroom, headroom, ptr, length); } - MDBX_CXX20_CONSTEXPR void *assign(const void *ptr, size_t length) { - return assign(0, ptr, length, 0); - } + MDBX_CXX20_CONSTEXPR void *assign(const void *ptr, size_t length) { return assign(0, ptr, length, 0); } - MDBX_CXX20_CONSTEXPR silo &assign(const silo &ditto, size_t headroom, - slice &content) { + MDBX_CXX20_CONSTEXPR silo &assign(const silo &ditto, size_t headroom, slice &content) { assert(ditto.get() + headroom == content.byte_ptr()); - if MDBX_IF_CONSTEXPR (!allocation_aware_details:: - allocator_is_always_equal()) { + if MDBX_IF_CONSTEXPR (!allocation_aware_details::allocator_is_always_equal()) { if (MDBX_UNLIKELY(get_allocator() != ditto.get_allocator())) MDBX_CXX20_UNLIKELY { release(); - allocation_aware_details::copy_assign_alloc< - silo, allocator_type>::propagate(this, ditto); + allocation_aware_details::copy_assign_alloc::propagate(this, ditto); } } - content.iov_base = reshape(ditto.capacity(), headroom, - content.data(), content.length()); + content.iov_base = reshape(ditto.capacity(), headroom, content.data(), content.length()); return *this; } MDBX_CXX20_CONSTEXPR silo & - assign(silo &&ditto, size_t headroom, slice &content) noexcept( - allocation_aware_details::move_assign_alloc< - silo, allocator_type>::is_nothrow()) { + assign(silo &&ditto, size_t headroom, + slice &content) noexcept(allocation_aware_details::move_assign_alloc::is_nothrow()) { assert(ditto.get() + headroom == content.byte_ptr()); - if (allocation_aware_details::move_assign_alloc< - silo, allocator_type>::is_moveable(this, ditto)) { + if (allocation_aware_details::move_assign_alloc::is_moveable(this, ditto)) { release(); - allocation_aware_details::move_assign_alloc< - silo, allocator_type>::propagate(this, ditto); + allocation_aware_details::move_assign_alloc::propagate(this, ditto); /* no reallocation nor copying required */ bin_ = ::std::move(ditto.bin_); assert(get() + headroom == content.byte_ptr()); } else { /* copy content since allocators are different */ - content.iov_base = reshape(ditto.capacity(), headroom, - content.data(), content.length()); + content.iov_base = reshape(ditto.capacity(), headroom, content.data(), content.length()); ditto.release(); } return *this; } - MDBX_CXX20_CONSTEXPR void *clear() { - return reshape(0, 0, nullptr, 0); - } - MDBX_CXX20_CONSTEXPR void *clear_and_reserve(size_t whole_capacity, - size_t headroom) { + MDBX_CXX20_CONSTEXPR void *clear() { return reshape(0, 0, nullptr, 0); } + MDBX_CXX20_CONSTEXPR void *clear_and_reserve(size_t whole_capacity, size_t headroom) { return reshape(whole_capacity, headroom, nullptr, 0); } - MDBX_CXX20_CONSTEXPR void resize(size_t capacity, size_t headroom, - slice &content) { - content.iov_base = - reshape(capacity, headroom, content.iov_base, content.iov_len); + MDBX_CXX20_CONSTEXPR void resize(size_t capacity, size_t headroom, slice &content) { + content.iov_base = reshape(capacity, headroom, content.iov_base, content.iov_len); } - MDBX_CXX20_CONSTEXPR void swap(silo &ditto) noexcept( - allocation_aware_details::swap_alloc::is_nothrow()) { - allocation_aware_details::swap_alloc::propagate( - this, ditto); + MDBX_CXX20_CONSTEXPR void + swap(silo &ditto) noexcept(allocation_aware_details::swap_alloc::is_nothrow()) { + allocation_aware_details::swap_alloc::propagate(this, ditto); ::std::swap(bin_, ditto.bin_); } /* MDBX_CXX20_CONSTEXPR void shrink_to_fit() { TODO } */ - MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX11_CONSTEXPR size_t - capacity() const noexcept { - return bin_.capacity(); - } - MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX11_CONSTEXPR const void * - data(size_t offset = 0) const noexcept { - return get(offset); - } - MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX11_CONSTEXPR void * - data(size_t offset = 0) noexcept { + MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX11_CONSTEXPR size_t capacity() const noexcept { return bin_.capacity(); } + MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX11_CONSTEXPR const void *data(size_t offset = 0) const noexcept { return get(offset); } + MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX11_CONSTEXPR void *data(size_t offset = 0) noexcept { return get(offset); } }; silo silo_; @@ -2062,21 +1862,18 @@ private: slice_.iov_base = silo_.data(); } - MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX20_CONSTEXPR const byte * - silo_begin() const noexcept { + MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX20_CONSTEXPR const byte *silo_begin() const noexcept { return static_cast(silo_.data()); } - MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX20_CONSTEXPR const byte * - silo_end() const noexcept { + MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX20_CONSTEXPR const byte *silo_end() const noexcept { return silo_begin() + silo_.capacity(); } struct data_preserver : public exception_thunk { buffer data; data_preserver(allocator_type &allocator) : data(allocator) {} - static int callback(void *context, MDBX_val *target, const void *src, - size_t bytes) noexcept { + static int callback(void *context, MDBX_val *target, const void *src, size_t bytes) noexcept { auto self = static_cast(context); assert(self->is_clean()); assert(&self->data.slice_ == target); @@ -2089,12 +1886,8 @@ private: return MDBX_RESULT_TRUE; } } - MDBX_CXX11_CONSTEXPR operator MDBX_preserve_func() const noexcept { - return callback; - } - MDBX_CXX11_CONSTEXPR operator const buffer &() const noexcept { - return data; - } + MDBX_CXX11_CONSTEXPR operator MDBX_preserve_func() const noexcept { return callback; } + MDBX_CXX11_CONSTEXPR operator const buffer &() const noexcept { return data; } MDBX_CXX11_CONSTEXPR operator buffer &() noexcept { return data; } }; @@ -2103,141 +1896,108 @@ public: /// \todo buffer& operator>>(buffer&, ...) for reading (delegated to slice) /// \todo template key(X) for encoding keys while writing - using move_assign_alloc = - allocation_aware_details::move_assign_alloc; - using copy_assign_alloc = - allocation_aware_details::copy_assign_alloc; + using move_assign_alloc = allocation_aware_details::move_assign_alloc; + using copy_assign_alloc = allocation_aware_details::copy_assign_alloc; /// \brief Returns the associated allocator. - MDBX_CXX20_CONSTEXPR allocator_type get_allocator() const { - return silo_.get_allocator(); - } + MDBX_CXX20_CONSTEXPR allocator_type get_allocator() const { return silo_.get_allocator(); } /// \brief Checks whether data chunk stored inside the buffer, otherwise /// buffer just refers to data located outside the buffer. - MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX20_CONSTEXPR bool - is_freestanding() const noexcept { + MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX20_CONSTEXPR bool is_freestanding() const noexcept { static_assert(size_t(-long(max_length)) > max_length, "WTF?"); return size_t(byte_ptr() - silo_begin()) < silo_.capacity(); } /// \brief Checks whether the buffer just refers to data located outside /// the buffer, rather than stores it. - MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX20_CONSTEXPR bool - is_reference() const noexcept { - return !is_freestanding(); - } + MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX20_CONSTEXPR bool is_reference() const noexcept { return !is_freestanding(); } - /// \brief Returns the number of bytes that can be held in currently allocated - /// storage. - MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX20_CONSTEXPR size_t - capacity() const noexcept { + /// \brief Returns the number of bytes that can be held in currently allocated storage. + MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX20_CONSTEXPR size_t capacity() const noexcept { return is_freestanding() ? silo_.capacity() : 0; } /// \brief Returns the number of bytes that available in currently allocated /// storage ahead the currently beginning of data. - MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX20_CONSTEXPR size_t - headroom() const noexcept { + MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX20_CONSTEXPR size_t headroom() const noexcept { return is_freestanding() ? slice_.byte_ptr() - silo_begin() : 0; } /// \brief Returns the number of bytes that available in currently allocated /// storage after the currently data end. - MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX20_CONSTEXPR size_t - tailroom() const noexcept { + MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX20_CONSTEXPR size_t tailroom() const noexcept { return is_freestanding() ? capacity() - headroom() - slice_.length() : 0; } /// \brief Returns casted to const pointer to byte an address of data. - MDBX_CXX11_CONSTEXPR const byte *byte_ptr() const noexcept { - return slice_.byte_ptr(); - } + MDBX_CXX11_CONSTEXPR const byte *byte_ptr() const noexcept { return slice_.byte_ptr(); } /// \brief Returns casted to const pointer to byte an end of data. - MDBX_CXX11_CONSTEXPR const byte *end_byte_ptr() const noexcept { - return slice_.end_byte_ptr(); - } + MDBX_CXX11_CONSTEXPR const byte *end_byte_ptr() const noexcept { return slice_.end_byte_ptr(); } /// \brief Returns casted to pointer to byte an address of data. - /// \pre REQUIRES: The buffer should store data chunk, but not referenced to - /// an external one. + /// \pre REQUIRES: The buffer should store data chunk, but not referenced to an external one. MDBX_CXX11_CONSTEXPR byte *byte_ptr() noexcept { MDBX_CONSTEXPR_ASSERT(is_freestanding()); return const_cast(slice_.byte_ptr()); } /// \brief Returns casted to pointer to byte an end of data. - /// \pre REQUIRES: The buffer should store data chunk, but not referenced to - /// an external one. + /// \pre REQUIRES: The buffer should store data chunk, but not referenced to an external one. MDBX_CXX11_CONSTEXPR byte *end_byte_ptr() noexcept { MDBX_CONSTEXPR_ASSERT(is_freestanding()); return const_cast(slice_.end_byte_ptr()); } /// \brief Returns casted to const pointer to char an address of data. - MDBX_CXX11_CONSTEXPR const char *char_ptr() const noexcept { - return slice_.char_ptr(); - } + MDBX_CXX11_CONSTEXPR const char *char_ptr() const noexcept { return slice_.char_ptr(); } /// \brief Returns casted to const pointer to char an end of data. - MDBX_CXX11_CONSTEXPR const char *end_char_ptr() const noexcept { - return slice_.end_char_ptr(); - } + MDBX_CXX11_CONSTEXPR const char *end_char_ptr() const noexcept { return slice_.end_char_ptr(); } /// \brief Returns casted to pointer to char an address of data. - /// \pre REQUIRES: The buffer should store data chunk, but not referenced to - /// an external one. + /// \pre REQUIRES: The buffer should store data chunk, but not referenced to an external one. MDBX_CXX11_CONSTEXPR char *char_ptr() noexcept { MDBX_CONSTEXPR_ASSERT(is_freestanding()); return const_cast(slice_.char_ptr()); } /// \brief Returns casted to pointer to char an end of data. - /// \pre REQUIRES: The buffer should store data chunk, but not referenced to - /// an external one. + /// \pre REQUIRES: The buffer should store data chunk, but not referenced to an external one. MDBX_CXX11_CONSTEXPR char *end_char_ptr() noexcept { MDBX_CONSTEXPR_ASSERT(is_freestanding()); return const_cast(slice_.end_char_ptr()); } /// \brief Return a const pointer to the beginning of the referenced data. - MDBX_CXX11_CONSTEXPR const void *data() const noexcept { - return slice_.data(); - } + MDBX_CXX11_CONSTEXPR const void *data() const noexcept { return slice_.data(); } /// \brief Return a const pointer to the end of the referenced data. MDBX_CXX11_CONSTEXPR const void *end() const noexcept { return slice_.end(); } /// \brief Return a pointer to the beginning of the referenced data. - /// \pre REQUIRES: The buffer should store data chunk, but not referenced to - /// an external one. + /// \pre REQUIRES: The buffer should store data chunk, but not referenced to an external one. MDBX_CXX11_CONSTEXPR void *data() noexcept { MDBX_CONSTEXPR_ASSERT(is_freestanding()); return const_cast(slice_.data()); } /// \brief Return a pointer to the end of the referenced data. - /// \pre REQUIRES: The buffer should store data chunk, but not referenced to - /// an external one. + /// \pre REQUIRES: The buffer should store data chunk, but not referenced to an external one. MDBX_CXX11_CONSTEXPR void *end() noexcept { MDBX_CONSTEXPR_ASSERT(is_freestanding()); return const_cast(slice_.end()); } /// \brief Returns the number of bytes. - MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR size_t - length() const noexcept { - return MDBX_CONSTEXPR_ASSERT(is_reference() || - slice_.length() + headroom() <= - silo_.capacity()), - slice_.length(); + MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR size_t length() const noexcept { + return MDBX_CONSTEXPR_ASSERT(is_reference() || slice_.length() + headroom() <= silo_.capacity()), slice_.length(); } /// \brief Set length of data. MDBX_CXX14_CONSTEXPR buffer &set_length(size_t bytes) { - MDBX_CONSTEXPR_ASSERT(is_reference() || - bytes + headroom() <= silo_.capacity()); + MDBX_CONSTEXPR_ASSERT(is_reference() || bytes + headroom() <= silo_.capacity()); slice_.set_length(bytes); return *this; } @@ -2257,87 +2017,66 @@ public: } MDBX_CXX20_CONSTEXPR buffer() noexcept = default; - MDBX_CXX20_CONSTEXPR buffer(const allocator_type &allocator) noexcept - : silo_(allocator) {} + MDBX_CXX20_CONSTEXPR buffer(const allocator_type &allocator) noexcept : silo_(allocator) {} - buffer(const struct slice &src, bool make_reference, - const allocator_type &allocator = allocator_type()) + buffer(const struct slice &src, bool make_reference, const allocator_type &allocator = allocator_type()) : silo_(allocator), slice_(src) { if (!make_reference) insulate(); } - buffer(const buffer &src, bool make_reference, - const allocator_type &allocator = allocator_type()) + buffer(const buffer &src, bool make_reference, const allocator_type &allocator = allocator_type()) : buffer(src.slice_, make_reference, allocator) {} - buffer(const void *ptr, size_t bytes, bool make_reference, - const allocator_type &allocator = allocator_type()) + buffer(const void *ptr, size_t bytes, bool make_reference, const allocator_type &allocator = allocator_type()) : buffer(::mdbx::slice(ptr, bytes), make_reference, allocator) {} - template - buffer(const ::std::basic_string &) = delete; - template - buffer(const ::std::basic_string &&) = delete; + template buffer(const ::std::basic_string &) = delete; + template buffer(const ::std::basic_string &&) = delete; - buffer(const char *c_str, bool make_reference, - const allocator_type &allocator = allocator_type()) - : buffer(::mdbx::slice(c_str), make_reference, allocator){} + buffer(const char *c_str, bool make_reference, const allocator_type &allocator = allocator_type()) + : buffer(::mdbx::slice(c_str), make_reference, allocator) {} -#if defined(DOXYGEN) || \ - (defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L) - template - buffer(const ::std::basic_string_view &view, - bool make_reference, - const allocator_type &allocator = allocator_type()) - : buffer(::mdbx::slice(view), make_reference, allocator) { - } +#if defined(DOXYGEN) || (defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L) + template + buffer(const ::std::basic_string_view &view, bool make_reference, + const allocator_type &allocator = allocator_type()) + : buffer(::mdbx::slice(view), make_reference, allocator) {} #endif /* __cpp_lib_string_view >= 201606L */ MDBX_CXX20_CONSTEXPR - buffer(const struct slice &src, - const allocator_type &allocator = allocator_type()) - : silo_(src.data(), src.length(), allocator), - slice_(silo_.data(), src.length()) {} + buffer(const struct slice &src, const allocator_type &allocator = allocator_type()) + : silo_(src.data(), src.length(), allocator), slice_(silo_.data(), src.length()) {} MDBX_CXX20_CONSTEXPR - buffer(const buffer &src, const allocator_type &allocator = allocator_type()) - : buffer(src.slice_, allocator) {} + buffer(const buffer &src, const allocator_type &allocator = allocator_type()) : buffer(src.slice_, allocator) {} MDBX_CXX20_CONSTEXPR - buffer(const void *ptr, size_t bytes, - const allocator_type &allocator = allocator_type()) + buffer(const void *ptr, size_t bytes, const allocator_type &allocator = allocator_type()) : buffer(::mdbx::slice(ptr, bytes), allocator) {} template - MDBX_CXX20_CONSTEXPR - buffer(const ::std::basic_string &str, - const allocator_type &allocator = allocator_type()) + MDBX_CXX20_CONSTEXPR buffer(const ::std::basic_string &str, + const allocator_type &allocator = allocator_type()) : buffer(::mdbx::slice(str), allocator) {} MDBX_CXX20_CONSTEXPR buffer(const char *c_str, const allocator_type &allocator = allocator_type()) - : buffer(::mdbx::slice(c_str), allocator){} + : buffer(::mdbx::slice(c_str), allocator) {} -#if defined(DOXYGEN) || \ - (defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L) - template - MDBX_CXX20_CONSTEXPR - buffer(const ::std::basic_string_view &view, - const allocator_type &allocator = allocator_type()) - : buffer(::mdbx::slice(view), allocator) { - } +#if defined(DOXYGEN) || (defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L) + template + MDBX_CXX20_CONSTEXPR buffer(const ::std::basic_string_view &view, + const allocator_type &allocator = allocator_type()) + : buffer(::mdbx::slice(view), allocator) {} #endif /* __cpp_lib_string_view >= 201606L */ - buffer(size_t head_room, size_t tail_room, - const allocator_type &allocator = allocator_type()) - : silo_(allocator) { + buffer(size_t head_room, size_t tail_room, const allocator_type &allocator = allocator_type()) : silo_(allocator) { slice_.iov_base = silo_.init(check_length(head_room, tail_room)); assert(slice_.iov_len == 0); } - buffer(size_t capacity, const allocator_type &allocator = allocator_type()) - : silo_(allocator) { + buffer(size_t capacity, const allocator_type &allocator = allocator_type()) : silo_(allocator) { slice_.iov_base = silo_.init(check_length(capacity)); assert(slice_.iov_len == 0); } @@ -2345,67 +2084,101 @@ public: buffer(size_t head_room, const struct slice &src, size_t tail_room, const allocator_type &allocator = allocator_type()) : silo_(allocator) { - slice_.iov_base = - silo_.init(check_length(head_room, src.length(), tail_room)); + slice_.iov_base = silo_.init(check_length(head_room, src.length(), tail_room)); slice_.iov_len = src.length(); memcpy(slice_.iov_base, src.data(), src.length()); } - buffer(size_t head_room, const buffer &src, size_t tail_room, - const allocator_type &allocator = allocator_type()) + buffer(size_t head_room, const buffer &src, size_t tail_room, const allocator_type &allocator = allocator_type()) : buffer(head_room, src.slice_, tail_room, allocator) {} - inline buffer(const ::mdbx::txn &txn, const struct slice &src, - const allocator_type &allocator = allocator_type()); + inline buffer(const ::mdbx::txn &txn, const struct slice &src, const allocator_type &allocator = allocator_type()); buffer(buffer &&src) noexcept(move_assign_alloc::is_nothrow()) : silo_(::std::move(src.silo_)), slice_(::std::move(src.slice_)) {} - MDBX_CXX11_CONSTEXPR const struct slice &slice() const noexcept { - return slice_; + MDBX_CXX11_CONSTEXPR const struct slice &slice() const noexcept { return slice_; } + + MDBX_CXX11_CONSTEXPR operator const struct slice &() const noexcept { return slice_; } + +#if defined(DOXYGEN) || (defined(__cpp_lib_span) && __cpp_lib_span >= 202002L) + template MDBX_CXX14_CONSTEXPR buffer(const ::std::span &span) : buffer(span.begin(), span.end()) { + static_assert(::std::is_standard_layout::value && !::std::is_pointer::value, + "Must be a standard layout type!"); } - MDBX_CXX11_CONSTEXPR operator const struct slice &() const noexcept { - return slice_; + template MDBX_CXX14_CONSTEXPR ::std::span as_span() const { + return slice_.template as_span(); } + template MDBX_CXX14_CONSTEXPR ::std::span as_span() { return slice_.template as_span(); } + + MDBX_CXX14_CONSTEXPR ::std::span bytes() const { return as_span(); } + MDBX_CXX14_CONSTEXPR ::std::span bytes() { return as_span(); } + MDBX_CXX14_CONSTEXPR ::std::span chars() const { return as_span(); } + MDBX_CXX14_CONSTEXPR ::std::span chars() { return as_span(); } +#endif /* __cpp_lib_span >= 202002L */ template - static buffer wrap(const POD &pod, bool make_reference = false, - const allocator_type &allocator = allocator_type()) { + static buffer wrap(const POD &pod, bool make_reference = false, const allocator_type &allocator = allocator_type()) { return buffer(::mdbx::slice::wrap(pod), make_reference, allocator); } - template MDBX_CXX14_CONSTEXPR POD as_pod() const { - return slice_.as_pod(); - } + template MDBX_CXX14_CONSTEXPR POD as_pod() const { return slice_.as_pod(); } + +#ifdef MDBX_U128_TYPE + MDBX_CXX14_CONSTEXPR MDBX_U128_TYPE as_uint128() const { return slice().as_uint128(); } +#endif /* MDBX_U128_TYPE */ + MDBX_CXX14_CONSTEXPR uint64_t as_uint64() const { return slice().as_uint64(); } + MDBX_CXX14_CONSTEXPR uint32_t as_uint32() const { return slice().as_uint32(); } + MDBX_CXX14_CONSTEXPR uint16_t as_uint16() const { return slice().as_uint16(); } + MDBX_CXX14_CONSTEXPR uint8_t as_uint8() const { return slice().as_uint8(); } + +#ifdef MDBX_I128_TYPE + MDBX_CXX14_CONSTEXPR MDBX_I128_TYPE as_int128() const { return slice().as_int128(); } +#endif /* MDBX_I128_TYPE */ + MDBX_CXX14_CONSTEXPR int64_t as_int64() const { return slice().as_int64(); } + MDBX_CXX14_CONSTEXPR int32_t as_int32() const { return slice().as_int32(); } + MDBX_CXX14_CONSTEXPR int16_t as_int16() const { return slice().as_int16(); } + MDBX_CXX14_CONSTEXPR int8_t as_int8() const { return slice().as_int8(); } + +#ifdef MDBX_U128_TYPE + MDBX_U128_TYPE as_uint128_adapt() const { return slice().as_uint128_adapt(); } +#endif /* MDBX_U128_TYPE */ + uint64_t as_uint64_adapt() const { return slice().as_uint64_adapt(); } + uint32_t as_uint32_adapt() const { return slice().as_uint32_adapt(); } + uint16_t as_uint16_adapt() const { return slice().as_uint16_adapt(); } + uint8_t as_uint8_adapt() const { return slice().as_uint8_adapt(); } + +#ifdef MDBX_I128_TYPE + MDBX_I128_TYPE as_int128_adapt() const { return slice().as_int128_adapt(); } +#endif /* MDBX_I128_TYPE */ + int64_t as_int64_adapt() const { return slice().as_int64_adapt(); } + int32_t as_int32_adapt() const { return slice().as_int32_adapt(); } + int16_t as_int16_adapt() const { return slice().as_int16_adapt(); } + int8_t as_int8_adapt() const { return slice().as_int8_adapt(); } /// \brief Returns a new buffer with a hexadecimal dump of the slice content. - static buffer hex(const ::mdbx::slice &source, bool uppercase = false, - unsigned wrap_width = 0, + static buffer hex(const ::mdbx::slice &source, bool uppercase = false, unsigned wrap_width = 0, const allocator_type &allocator = allocator_type()) { - return source.template encode_hex( - uppercase, wrap_width, allocator); + return source.template encode_hex(uppercase, wrap_width, allocator); } /// \brief Returns a new buffer with a /// [Base58](https://en.wikipedia.org/wiki/Base58) dump of the slice content. static buffer base58(const ::mdbx::slice &source, unsigned wrap_width = 0, const allocator_type &allocator = allocator_type()) { - return source.template encode_base58(wrap_width, - allocator); + return source.template encode_base58(wrap_width, allocator); } /// \brief Returns a new buffer with a /// [Base64](https://en.wikipedia.org/wiki/Base64) dump of the slice content. static buffer base64(const ::mdbx::slice &source, unsigned wrap_width = 0, const allocator_type &allocator = allocator_type()) { - return source.template encode_base64(wrap_width, - allocator); + return source.template encode_base64(wrap_width, allocator); } /// \brief Returns a new buffer with a hexadecimal dump of the given pod. template - static buffer hex(const POD &pod, bool uppercase = false, - unsigned wrap_width = 0, + static buffer hex(const POD &pod, bool uppercase = false, unsigned wrap_width = 0, const allocator_type &allocator = allocator_type()) { return hex(mdbx::slice::wrap(pod), uppercase, wrap_width, allocator); } @@ -2413,105 +2186,80 @@ public: /// \brief Returns a new buffer with a /// [Base58](https://en.wikipedia.org/wiki/Base58) dump of the given pod. template - static buffer base58(const POD &pod, unsigned wrap_width = 0, - const allocator_type &allocator = allocator_type()) { + static buffer base58(const POD &pod, unsigned wrap_width = 0, const allocator_type &allocator = allocator_type()) { return base58(mdbx::slice::wrap(pod), wrap_width, allocator); } /// \brief Returns a new buffer with a /// [Base64](https://en.wikipedia.org/wiki/Base64) dump of the given pod. template - static buffer base64(const POD &pod, unsigned wrap_width = 0, - const allocator_type &allocator = allocator_type()) { + static buffer base64(const POD &pod, unsigned wrap_width = 0, const allocator_type &allocator = allocator_type()) { return base64(mdbx::slice::wrap(pod), wrap_width, allocator); } /// \brief Returns a new buffer with a hexadecimal dump of the slice content. buffer encode_hex(bool uppercase = false, unsigned wrap_width = 0, const allocator_type &allocator = allocator_type()) const { - return slice().template encode_hex( - uppercase, wrap_width, allocator); + return slice().template encode_hex(uppercase, wrap_width, allocator); } /// \brief Returns a new buffer with a /// [Base58](https://en.wikipedia.org/wiki/Base58) dump of the slice content. - buffer - encode_base58(unsigned wrap_width = 0, - const allocator_type &allocator = allocator_type()) const { - return slice().template encode_base58( - wrap_width, allocator); + buffer encode_base58(unsigned wrap_width = 0, const allocator_type &allocator = allocator_type()) const { + return slice().template encode_base58(wrap_width, allocator); } /// \brief Returns a new buffer with a /// [Base64](https://en.wikipedia.org/wiki/Base64) dump of the slice content. - buffer - encode_base64(unsigned wrap_width = 0, - const allocator_type &allocator = allocator_type()) const { - return slice().template encode_base64( - wrap_width, allocator); + buffer encode_base64(unsigned wrap_width = 0, const allocator_type &allocator = allocator_type()) const { + return slice().template encode_base64(wrap_width, allocator); } /// \brief Decodes hexadecimal dump from the slice content to returned buffer. - static buffer hex_decode(const ::mdbx::slice &source, - bool ignore_spaces = false, + static buffer hex_decode(const ::mdbx::slice &source, bool ignore_spaces = false, const allocator_type &allocator = allocator_type()) { - return source.template hex_decode(ignore_spaces, - allocator); + return source.template hex_decode(ignore_spaces, allocator); } /// \brief Decodes [Base58](https://en.wikipedia.org/wiki/Base58) dump /// from the slice content to returned buffer. - static buffer - base58_decode(const ::mdbx::slice &source, bool ignore_spaces = false, - const allocator_type &allocator = allocator_type()) { - return source.template base58_decode( - ignore_spaces, allocator); + static buffer base58_decode(const ::mdbx::slice &source, bool ignore_spaces = false, + const allocator_type &allocator = allocator_type()) { + return source.template base58_decode(ignore_spaces, allocator); } /// \brief Decodes [Base64](https://en.wikipedia.org/wiki/Base64) dump /// from the slice content to returned buffer. - static buffer - base64_decode(const ::mdbx::slice &source, bool ignore_spaces = false, - const allocator_type &allocator = allocator_type()) { - return source.template base64_decode( - ignore_spaces, allocator); + static buffer base64_decode(const ::mdbx::slice &source, bool ignore_spaces = false, + const allocator_type &allocator = allocator_type()) { + return source.template base64_decode(ignore_spaces, allocator); } /// \brief Decodes hexadecimal dump /// from the buffer content to new returned buffer. - buffer hex_decode(bool ignore_spaces = false, - const allocator_type &allocator = allocator_type()) const { + buffer hex_decode(bool ignore_spaces = false, const allocator_type &allocator = allocator_type()) const { return hex_decode(slice(), ignore_spaces, allocator); } /// \brief Decodes [Base58](https://en.wikipedia.org/wiki/Base58) dump /// from the buffer content to new returned buffer. - buffer - base58_decode(bool ignore_spaces = false, - const allocator_type &allocator = allocator_type()) const { + buffer base58_decode(bool ignore_spaces = false, const allocator_type &allocator = allocator_type()) const { return base58_decode(slice(), ignore_spaces, allocator); } /// \brief Decodes [Base64](https://en.wikipedia.org/wiki/Base64) dump /// from the buffer content to new returned buffer. - buffer - base64_decode(bool ignore_spaces = false, - const allocator_type &allocator = allocator_type()) const { + buffer base64_decode(bool ignore_spaces = false, const allocator_type &allocator = allocator_type()) const { return base64_decode(slice(), ignore_spaces, allocator); } /// \brief Reserves storage space. void reserve(size_t wanna_headroom, size_t wanna_tailroom) { - wanna_headroom = ::std::min(::std::max(headroom(), wanna_headroom), - wanna_headroom + pettiness_threshold); - wanna_tailroom = ::std::min(::std::max(tailroom(), wanna_tailroom), - wanna_tailroom + pettiness_threshold); - const size_t wanna_capacity = - check_length(wanna_headroom, slice_.length(), wanna_tailroom); + wanna_headroom = ::std::min(::std::max(headroom(), wanna_headroom), wanna_headroom + pettiness_threshold); + wanna_tailroom = ::std::min(::std::max(tailroom(), wanna_tailroom), wanna_tailroom + pettiness_threshold); + const size_t wanna_capacity = check_length(wanna_headroom, slice_.length(), wanna_tailroom); silo_.resize(wanna_capacity, wanna_headroom, slice_); - assert(headroom() >= wanna_headroom && - headroom() <= wanna_headroom + pettiness_threshold); - assert(tailroom() >= wanna_tailroom && - tailroom() <= wanna_tailroom + pettiness_threshold); + assert(headroom() >= wanna_headroom && headroom() <= wanna_headroom + pettiness_threshold); + assert(tailroom() >= wanna_tailroom && tailroom() <= wanna_tailroom + pettiness_threshold); } /// \brief Reserves space before the payload. @@ -2527,30 +2275,24 @@ public: } buffer &assign_freestanding(const void *ptr, size_t bytes) { - silo_.assign(static_cast(ptr), - check_length(bytes)); + silo_.assign(static_cast(ptr), check_length(bytes)); slice_.assign(silo_.data(), bytes); return *this; } - MDBX_CXX20_CONSTEXPR void - swap(buffer &other) noexcept(swap_alloc::is_nothrow()) { + MDBX_CXX20_CONSTEXPR void swap(buffer &other) noexcept(swap_alloc::is_nothrow()) { silo_.swap(other.silo_); slice_.swap(other.slice_); } - static buffer clone(const buffer &src, - const allocator_type &allocator = allocator_type()) { + static buffer clone(const buffer &src, const allocator_type &allocator = allocator_type()) { return buffer(src.headroom(), src.slice_, src.tailroom(), allocator); } - buffer &assign(const buffer &src, bool make_reference = false) { - return assign(src.slice_, make_reference); - } + buffer &assign(const buffer &src, bool make_reference = false) { return assign(src.slice_, make_reference); } buffer &assign(const void *ptr, size_t bytes, bool make_reference = false) { - return make_reference ? assign_reference(ptr, bytes) - : assign_freestanding(ptr, bytes); + return make_reference ? assign_reference(ptr, bytes) : assign_freestanding(ptr, bytes); } buffer &assign(const struct slice &src, bool make_reference = false) { @@ -2573,17 +2315,12 @@ public: return *this; } - buffer &assign(const void *begin, const void *end, - bool make_reference = false) { - return assign(begin, - static_cast(end) - - static_cast(begin), - make_reference); + buffer &assign(const void *begin, const void *end, bool make_reference = false) { + return assign(begin, static_cast(end) - static_cast(begin), make_reference); } template - buffer &assign(const ::std::basic_string &str, - bool make_reference = false) { + buffer &assign(const ::std::basic_string &str, bool make_reference = false) { return assign(str.data(), str.length(), make_reference); } @@ -2593,14 +2330,11 @@ public: #if defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L template - buffer &assign(const ::std::basic_string_view &view, - bool make_reference = false) { + buffer &assign(const ::std::basic_string_view &view, bool make_reference = false) { return assign(view.data(), view.length(), make_reference); } - template - buffer &assign(::std::basic_string_view &&view, - bool make_reference = false) { + template buffer &assign(::std::basic_string_view &&view, bool make_reference = false) { assign(view.data(), view.length(), make_reference); view = {}; return *this; @@ -2609,18 +2343,14 @@ public: buffer &operator=(const buffer &src) { return assign(src); } - buffer &operator=(buffer &&src) noexcept(move_assign_alloc::is_nothrow()) { - return assign(::std::move(src)); - } + buffer &operator=(buffer &&src) noexcept(move_assign_alloc::is_nothrow()) { return assign(::std::move(src)); } buffer &operator=(const struct slice &src) { return assign(src); } buffer &operator=(struct slice &&src) { return assign(::std::move(src)); } -#if defined(DOXYGEN) || \ - (defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L) - template - buffer &operator=(const ::std::basic_string_view &view) noexcept { +#if defined(DOXYGEN) || (defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L) + template buffer &operator=(const ::std::basic_string_view &view) noexcept { return assign(view); } @@ -2631,58 +2361,43 @@ public: } /// \brief Return a string_view that references the data of this buffer. - template - operator ::std::basic_string_view() const noexcept { + template operator ::std::basic_string_view() const noexcept { return string_view(); } #endif /* __cpp_lib_string_view >= 201606L */ /// \brief Checks whether the string is empty. - MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX20_CONSTEXPR bool empty() const noexcept { - return length() == 0; - } + MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX20_CONSTEXPR bool empty() const noexcept { return length() == 0; } /// \brief Checks whether the data pointer of the buffer is nullptr. - MDBX_CXX11_CONSTEXPR bool is_null() const noexcept { - return data() == nullptr; - } + MDBX_CXX11_CONSTEXPR bool is_null() const noexcept { return data() == nullptr; } /// \brief Returns the number of bytes. - MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX20_CONSTEXPR size_t size() const noexcept { - return length(); - } + MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX20_CONSTEXPR size_t size() const noexcept { return length(); } /// \brief Returns the hash value of the data. /// \attention Function implementation and returned hash values may changed /// version to version, and in future the t1ha3 will be used here. Therefore /// values obtained from this function shouldn't be persisted anywhere. - MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR size_t - hash_value() const noexcept { - return slice_.hash_value(); - } + MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR size_t hash_value() const noexcept { return slice_.hash_value(); } - template , - class A = legacy_allocator> - MDBX_CXX20_CONSTEXPR ::std::basic_string - as_string(const A &allocator = A()) const { + template , class A = legacy_allocator> + MDBX_CXX20_CONSTEXPR ::std::basic_string as_string(const A &allocator = A()) const { return slice_.as_string(allocator); } template - MDBX_CXX20_CONSTEXPR explicit - operator ::std::basic_string() const { + MDBX_CXX20_CONSTEXPR explicit operator ::std::basic_string() const { return as_string(); } /// \brief Checks if the data starts with the given prefix. - MDBX_NOTHROW_PURE_FUNCTION bool - starts_with(const struct slice &prefix) const noexcept { + MDBX_NOTHROW_PURE_FUNCTION bool starts_with(const struct slice &prefix) const noexcept { return slice_.starts_with(prefix); } /// \brief Checks if the data ends with the given suffix. - MDBX_NOTHROW_PURE_FUNCTION bool - ends_with(const struct slice &suffix) const noexcept { + MDBX_NOTHROW_PURE_FUNCTION bool ends_with(const struct slice &suffix) const noexcept { return slice_.ends_with(suffix); } @@ -2741,40 +2456,27 @@ public: /// \brief Returns the first "n" bytes of the data chunk. /// \pre REQUIRES: `n <= size()` - MDBX_CXX14_CONSTEXPR struct slice head(size_t n) const noexcept { - return slice_.head(n); - } + MDBX_CXX14_CONSTEXPR struct slice head(size_t n) const noexcept { return slice_.head(n); } /// \brief Returns the last "n" bytes of the data chunk. /// \pre REQUIRES: `n <= size()` - MDBX_CXX14_CONSTEXPR struct slice tail(size_t n) const noexcept { - return slice_.tail(n); - } + MDBX_CXX14_CONSTEXPR struct slice tail(size_t n) const noexcept { return slice_.tail(n); } /// \brief Returns the middle "n" bytes of the data chunk. /// \pre REQUIRES: `from + n <= size()` - MDBX_CXX14_CONSTEXPR struct slice middle(size_t from, - size_t n) const noexcept { - return slice_.middle(from, n); - } + MDBX_CXX14_CONSTEXPR struct slice middle(size_t from, size_t n) const noexcept { return slice_.middle(from, n); } /// \brief Returns the first "n" bytes of the data chunk. /// \throws std::out_of_range if `n >= size()` - MDBX_CXX14_CONSTEXPR struct slice safe_head(size_t n) const { - return slice_.safe_head(n); - } + MDBX_CXX14_CONSTEXPR struct slice safe_head(size_t n) const { return slice_.safe_head(n); } /// \brief Returns the last "n" bytes of the data chunk. /// \throws std::out_of_range if `n >= size()` - MDBX_CXX14_CONSTEXPR struct slice safe_tail(size_t n) const { - return slice_.safe_tail(n); - } + MDBX_CXX14_CONSTEXPR struct slice safe_tail(size_t n) const { return slice_.safe_tail(n); } /// \brief Returns the middle "n" bytes of the data chunk. /// \throws std::out_of_range if `from + n >= size()` - MDBX_CXX14_CONSTEXPR struct slice safe_middle(size_t from, size_t n) const { - return slice_.safe_middle(from, n); - } + MDBX_CXX14_CONSTEXPR struct slice safe_middle(size_t from, size_t n) const { return slice_.safe_middle(from, n); } buffer &append(const void *src, size_t bytes) { if (MDBX_UNLIKELY(tailroom() < check_length(bytes))) @@ -2784,41 +2486,33 @@ public: return *this; } - buffer &append(const struct slice &chunk) { - return append(chunk.data(), chunk.size()); - } + buffer &append(const struct slice &chunk) { return append(chunk.data(), chunk.size()); } buffer &add_header(const void *src, size_t bytes) { if (MDBX_UNLIKELY(headroom() < check_length(bytes))) MDBX_CXX20_UNLIKELY reserve_headroom(bytes); - slice_.iov_base = - memcpy(static_cast(slice_.iov_base) - bytes, src, bytes); + slice_.iov_base = memcpy(static_cast(slice_.iov_base) - bytes, src, bytes); slice_.iov_len += bytes; return *this; } - buffer &add_header(const struct slice &chunk) { - return add_header(chunk.data(), chunk.size()); - } + buffer &add_header(const struct slice &chunk) { return add_header(chunk.data(), chunk.size()); } - template - buffer &append_producer(PRODUCER &producer) { + template buffer &append_producer(PRODUCER &producer) { const size_t wanna_bytes = producer.envisage_result_length(); if (MDBX_UNLIKELY(tailroom() < check_length(wanna_bytes))) MDBX_CXX20_UNLIKELY reserve_tailroom(wanna_bytes); return set_end(producer.write_bytes(end_char_ptr(), tailroom())); } - template - buffer &append_producer(const PRODUCER &producer) { + template buffer &append_producer(const PRODUCER &producer) { const size_t wanna_bytes = producer.envisage_result_length(); if (MDBX_UNLIKELY(tailroom() < check_length(wanna_bytes))) MDBX_CXX20_UNLIKELY reserve_tailroom(wanna_bytes); return set_end(producer.write_bytes(end_char_ptr(), tailroom())); } - buffer &append_hex(const struct slice &data, bool uppercase = false, - unsigned wrap_width = 0) { + buffer &append_hex(const struct slice &data, bool uppercase = false, unsigned wrap_width = 0) { return append_producer(to_hex(data, uppercase, wrap_width)); } @@ -2830,18 +2524,15 @@ public: return append_producer(to_base64(data, wrap_width)); } - buffer &append_decoded_hex(const struct slice &data, - bool ignore_spaces = false) { + buffer &append_decoded_hex(const struct slice &data, bool ignore_spaces = false) { return append_producer(from_hex(data, ignore_spaces)); } - buffer &append_decoded_base58(const struct slice &data, - bool ignore_spaces = false) { + buffer &append_decoded_base58(const struct slice &data, bool ignore_spaces = false) { return append_producer(from_base58(data, ignore_spaces)); } - buffer &append_decoded_base64(const struct slice &data, - bool ignore_spaces = false) { + buffer &append_decoded_base64(const struct slice &data, bool ignore_spaces = false) { return append_producer(from_base64(data, ignore_spaces)); } @@ -2920,160 +2611,114 @@ public: //---------------------------------------------------------------------------- - template - static buffer key_from(const char (&text)[SIZE], bool make_reference = true) { + template static buffer key_from(const char (&text)[SIZE], bool make_reference = true) { return buffer(::mdbx::slice(text), make_reference); } -#if defined(DOXYGEN) || \ - (defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L) +#if defined(DOXYGEN) || (defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L) template - static buffer key_from(const ::std::basic_string_view &src, - bool make_reference = false) { + static buffer key_from(const ::std::basic_string_view &src, bool make_reference = false) { return buffer(src, make_reference); } #endif /* __cpp_lib_string_view >= 201606L */ - static buffer key_from(const char *src, bool make_reference = false) { - return buffer(src, make_reference); - } + static buffer key_from(const char *src, bool make_reference = false) { return buffer(src, make_reference); } template - static buffer key_from(const ::std::basic_string &src, - bool make_reference = false) { + static buffer key_from(const ::std::basic_string &src, bool make_reference = false) { return buffer(src, make_reference); } - static buffer key_from(silo &&src) noexcept { - return buffer(::std::move(src)); - } + static buffer key_from(silo &&src) noexcept { return buffer(::std::move(src)); } - static buffer key_from_double(const double ieee754_64bit) { - return wrap(::mdbx_key_from_double(ieee754_64bit)); - } + static buffer key_from_double(const double ieee754_64bit) { return wrap(::mdbx_key_from_double(ieee754_64bit)); } - static buffer key_from(const double ieee754_64bit) { - return key_from_double(ieee754_64bit); - } + static buffer key_from(const double ieee754_64bit) { return key_from_double(ieee754_64bit); } - static buffer key_from(const double *ieee754_64bit) { - return wrap(::mdbx_key_from_ptrdouble(ieee754_64bit)); - } + static buffer key_from(const double *ieee754_64bit) { return wrap(::mdbx_key_from_ptrdouble(ieee754_64bit)); } - static buffer key_from_u64(const uint64_t unsigned_int64) { - return wrap(unsigned_int64); - } + static buffer key_from_u64(const uint64_t unsigned_int64) { return wrap(unsigned_int64); } - static buffer key_from(const uint64_t unsigned_int64) { - return key_from_u64(unsigned_int64); - } + static buffer key_from(const uint64_t unsigned_int64) { return key_from_u64(unsigned_int64); } - static buffer key_from_i64(const int64_t signed_int64) { - return wrap(::mdbx_key_from_int64(signed_int64)); - } + static buffer key_from_i64(const int64_t signed_int64) { return wrap(::mdbx_key_from_int64(signed_int64)); } - static buffer key_from(const int64_t signed_int64) { - return key_from_i64(signed_int64); - } + static buffer key_from(const int64_t signed_int64) { return key_from_i64(signed_int64); } static buffer key_from_jsonInteger(const int64_t json_integer) { return wrap(::mdbx_key_from_jsonInteger(json_integer)); } - static buffer key_from_float(const float ieee754_32bit) { - return wrap(::mdbx_key_from_float(ieee754_32bit)); - } + static buffer key_from_float(const float ieee754_32bit) { return wrap(::mdbx_key_from_float(ieee754_32bit)); } - static buffer key_from(const float ieee754_32bit) { - return key_from_float(ieee754_32bit); - } + static buffer key_from(const float ieee754_32bit) { return key_from_float(ieee754_32bit); } - static buffer key_from(const float *ieee754_32bit) { - return wrap(::mdbx_key_from_ptrfloat(ieee754_32bit)); - } + static buffer key_from(const float *ieee754_32bit) { return wrap(::mdbx_key_from_ptrfloat(ieee754_32bit)); } - static buffer key_from_u32(const uint32_t unsigned_int32) { - return wrap(unsigned_int32); - } + static buffer key_from_u32(const uint32_t unsigned_int32) { return wrap(unsigned_int32); } - static buffer key_from(const uint32_t unsigned_int32) { - return key_from_u32(unsigned_int32); - } + static buffer key_from(const uint32_t unsigned_int32) { return key_from_u32(unsigned_int32); } - static buffer key_from_i32(const int32_t signed_int32) { - return wrap(::mdbx_key_from_int32(signed_int32)); - } + static buffer key_from_i32(const int32_t signed_int32) { return wrap(::mdbx_key_from_int32(signed_int32)); } - static buffer key_from(const int32_t signed_int32) { - return key_from_i32(signed_int32); - } + static buffer key_from(const int32_t signed_int32) { return key_from_i32(signed_int32); } }; -template -inline buffer -make_buffer(PRODUCER &producer, const ALLOCATOR &allocator) { +template +inline buffer make_buffer(PRODUCER &producer, const ALLOCATOR &allocator) { if (MDBX_LIKELY(!producer.is_empty())) MDBX_CXX20_LIKELY { - buffer result( - producer.envisage_result_length(), allocator); - result.set_end( - producer.write_bytes(result.end_char_ptr(), result.tailroom())); + buffer result(producer.envisage_result_length(), allocator); + result.set_end(producer.write_bytes(result.end_char_ptr(), result.tailroom())); return result; } return buffer(allocator); } -template -inline buffer -make_buffer(const PRODUCER &producer, const ALLOCATOR &allocator) { +template +inline buffer make_buffer(const PRODUCER &producer, const ALLOCATOR &allocator) { if (MDBX_LIKELY(!producer.is_empty())) MDBX_CXX20_LIKELY { - buffer result( - producer.envisage_result_length(), allocator); - result.set_end( - producer.write_bytes(result.end_char_ptr(), result.tailroom())); + buffer result(producer.envisage_result_length(), allocator); + result.set_end(producer.write_bytes(result.end_char_ptr(), result.tailroom())); return result; } return buffer(allocator); } template -inline string make_string(PRODUCER &producer, - const ALLOCATOR &allocator) { +inline string make_string(PRODUCER &producer, const ALLOCATOR &allocator) { string result(allocator); if (MDBX_LIKELY(!producer.is_empty())) MDBX_CXX20_LIKELY { result.resize(producer.envisage_result_length()); - result.resize(producer.write_bytes(const_cast(result.data()), - result.capacity()) - - result.data()); + result.resize(producer.write_bytes(const_cast(result.data()), result.capacity()) - result.data()); } return result; } template -inline string make_string(const PRODUCER &producer, - const ALLOCATOR &allocator) { +inline string make_string(const PRODUCER &producer, const ALLOCATOR &allocator) { string result(allocator); if (MDBX_LIKELY(!producer.is_empty())) MDBX_CXX20_LIKELY { result.resize(producer.envisage_result_length()); - result.resize(producer.write_bytes(const_cast(result.data()), - result.capacity()) - - result.data()); + result.resize(producer.write_bytes(const_cast(result.data()), result.capacity()) - result.data()); } return result; } -/// \brief Combines data slice with boolean flag to represent result of certain -/// operations. +MDBX_EXTERN_API_TEMPLATE(LIBMDBX_API_TYPE, buffer); + +#if defined(__cpp_lib_memory_resource) && __cpp_lib_memory_resource >= 201603L && _GLIBCXX_USE_CXX11_ABI +MDBX_EXTERN_API_TEMPLATE(LIBMDBX_API_TYPE, buffer); +#endif /* __cpp_lib_memory_resource >= 201603L */ + +/// \brief Combines data slice with boolean flag to represent result of certain operations. struct value_result { slice value; bool done; - value_result(const slice &value, bool done) noexcept - : value(value), done(done) {} + value_result(const slice &value, bool done) noexcept : value(value), done(done) {} value_result(const value_result &) noexcept = default; value_result &operator=(const value_result &) noexcept = default; MDBX_CXX14_CONSTEXPR operator bool() const noexcept { @@ -3082,25 +2727,41 @@ struct value_result { } }; -/// \brief Combines pair of slices for key and value to represent result of -/// certain operations. +/// \brief Combines pair of slices for key and value to represent result of certain operations. struct pair { + using stl_pair = std::pair; slice key, value; - pair(const slice &key, const slice &value) noexcept - : key(key), value(value) {} + MDBX_CXX11_CONSTEXPR pair(const slice &key, const slice &value) noexcept : key(key), value(value) {} + MDBX_CXX11_CONSTEXPR pair(const stl_pair &couple) noexcept : key(couple.first), value(couple.second) {} + MDBX_CXX11_CONSTEXPR operator stl_pair() const noexcept { return stl_pair(key, value); } pair(const pair &) noexcept = default; pair &operator=(const pair &) noexcept = default; MDBX_CXX14_CONSTEXPR operator bool() const noexcept { assert(bool(key) == bool(value)); return key; } + MDBX_CXX14_CONSTEXPR static pair invalid() noexcept { return pair(slice::invalid(), slice::invalid()); } + + /// \brief Three-way fast non-lexicographically length-based comparison. + MDBX_NOTHROW_PURE_FUNCTION static MDBX_CXX14_CONSTEXPR intptr_t compare_fast(const pair &a, const pair &b) noexcept; + + /// \brief Three-way lexicographically comparison. + MDBX_NOTHROW_PURE_FUNCTION static MDBX_CXX14_CONSTEXPR intptr_t compare_lexicographically(const pair &a, + const pair &b) noexcept; + friend MDBX_CXX14_CONSTEXPR bool operator==(const pair &a, const pair &b) noexcept; + friend MDBX_CXX14_CONSTEXPR bool operator<(const pair &a, const pair &b) noexcept; + friend MDBX_CXX14_CONSTEXPR bool operator>(const pair &a, const pair &b) noexcept; + friend MDBX_CXX14_CONSTEXPR bool operator<=(const pair &a, const pair &b) noexcept; + friend MDBX_CXX14_CONSTEXPR bool operator>=(const pair &a, const pair &b) noexcept; + friend MDBX_CXX14_CONSTEXPR bool operator!=(const pair &a, const pair &b) noexcept; }; /// \brief Combines pair of slices for key and value with boolean flag to /// represent result of certain operations. struct pair_result : public pair { bool done; - pair_result(const slice &key, const slice &value, bool done) noexcept + MDBX_CXX11_CONSTEXPR pair_result() noexcept : pair(pair::invalid()), done(false) {} + MDBX_CXX11_CONSTEXPR pair_result(const slice &key, const slice &value, bool done) noexcept : pair(key, value), done(done) {} pair_result(const pair_result &) noexcept = default; pair_result &operator=(const pair_result &) noexcept = default; @@ -3110,6 +2771,79 @@ struct pair_result : public pair { } }; +template struct buffer_pair_spec { + using buffer_type = buffer; + using allocator_type = typename buffer_type::allocator_type; + using allocator_traits = typename buffer_type::allocator_traits; + using reservation_policy = CAPACITY_POLICY; + using stl_pair = ::std::pair; + buffer_type key, value; + + MDBX_CXX20_CONSTEXPR buffer_pair_spec() noexcept = default; + MDBX_CXX20_CONSTEXPR + buffer_pair_spec(const allocator_type &allocator) noexcept : key(allocator), value(allocator) {} + + buffer_pair_spec(const buffer_type &key, const buffer_type &value, const allocator_type &allocator = allocator_type()) + : key(key, allocator), value(value, allocator) {} + buffer_pair_spec(const buffer_type &key, const buffer_type &value, bool make_reference, + const allocator_type &allocator = allocator_type()) + : key(key, make_reference, allocator), value(value, make_reference, allocator) {} + + buffer_pair_spec(const stl_pair &pair, const allocator_type &allocator = allocator_type()) + : buffer_pair_spec(pair.first, pair.second, allocator) {} + buffer_pair_spec(const stl_pair &pair, bool make_reference, const allocator_type &allocator = allocator_type()) + : buffer_pair_spec(pair.first, pair.second, make_reference, allocator) {} + + buffer_pair_spec(const slice &key, const slice &value, const allocator_type &allocator = allocator_type()) + : key(key, allocator), value(value, allocator) {} + buffer_pair_spec(const slice &key, const slice &value, bool make_reference, + const allocator_type &allocator = allocator_type()) + : key(key, make_reference, allocator), value(value, make_reference, allocator) {} + + buffer_pair_spec(const pair &pair, const allocator_type &allocator = allocator_type()) + : buffer_pair_spec(pair.key, pair.value, allocator) {} + buffer_pair_spec(const pair &pair, bool make_reference, const allocator_type &allocator = allocator_type()) + : buffer_pair_spec(pair.key, pair.value, make_reference, allocator) {} + + buffer_pair_spec(const txn &txn, const slice &key, const slice &value, + const allocator_type &allocator = allocator_type()) + : key(txn, key, allocator), value(txn, value, allocator) {} + buffer_pair_spec(const txn &txn, const pair &pair, const allocator_type &allocator = allocator_type()) + : buffer_pair_spec(txn, pair.key, pair.value, allocator) {} + + buffer_pair_spec(buffer_type &&key, buffer_type &&value) noexcept(buffer_type::move_assign_alloc::is_nothrow()) + : key(::std::move(key)), value(::std::move(value)) {} + buffer_pair_spec(buffer_pair_spec &&pair) noexcept(buffer_type::move_assign_alloc::is_nothrow()) + : buffer_pair_spec(::std::move(pair.key), ::std::move(pair.value)) {} + + /// \brief Checks whether data chunk stored inside the buffers both, otherwise + /// at least one of buffers just refers to data located outside. + MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX20_CONSTEXPR bool is_freestanding() const noexcept { + return key.is_freestanding() && value.is_freestanding(); + } + /// \brief Checks whether one of the buffers just refers to data located + /// outside the buffer, rather than stores it. + MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX20_CONSTEXPR bool is_reference() const noexcept { + return key.is_reference() || value.is_reference(); + } + /// \brief Makes buffers owning the data. + /// \details If buffer refers to an external data, then makes it the owner + /// of clone by allocating storage and copying the data. + void make_freestanding() { + key.make_freestanding(); + value.make_freestanding(); + } + + operator pair() const noexcept { return pair(key, value); } +}; + +/// \brief Combines pair of buffers for key and value to hold an operands for certain operations. +template +using buffer_pair = buffer_pair_spec; + +/// \brief Default pair of buffers. +using default_buffer_pair = buffer_pair; + /// end of cxx_data @} //------------------------------------------------------------------------------ @@ -3130,63 +2864,78 @@ enum class key_mode { ///< sorted as such. The keys must all be of the ///< same size and must be aligned while passing ///< as arguments. - msgpack = -1 ///< Keys are in [MessagePack](https://msgpack.org/) - ///< format with appropriate comparison. - ///< \note Not yet implemented and PRs are welcome. + msgpack = -1 ///< Keys are in [MessagePack](https://msgpack.org/) + ///< format with appropriate comparison. + ///< \note Not yet implemented and PRs are welcome. }; -/// \brief Kind of the values and sorted multi-values with corresponding -/// comparison. +MDBX_CXX01_CONSTEXPR_ENUM bool is_usual(key_mode mode) noexcept { + return (MDBX_db_flags_t(mode) & (MDBX_REVERSEKEY | MDBX_INTEGERKEY)) == 0; +} + +MDBX_CXX01_CONSTEXPR_ENUM bool is_ordinal(key_mode mode) noexcept { + return (MDBX_db_flags_t(mode) & MDBX_INTEGERKEY) != 0; +} + +MDBX_CXX01_CONSTEXPR_ENUM bool is_samelength(key_mode mode) noexcept { + return (MDBX_db_flags_t(mode) & MDBX_INTEGERKEY) != 0; +} + +MDBX_CXX01_CONSTEXPR_ENUM bool is_reverse(key_mode mode) noexcept { + return (MDBX_db_flags_t(mode) & MDBX_REVERSEKEY) != 0; +} + +MDBX_CXX01_CONSTEXPR_ENUM bool is_msgpack(key_mode mode) noexcept { return mode == key_mode::msgpack; } + +/// \brief Kind of the values and sorted multi-values with corresponding comparison. enum class value_mode { single = MDBX_DB_DEFAULTS, ///< Usual single value for each key. In terms of ///< keys, they are unique. - multi = - MDBX_DUPSORT, ///< A more than one data value could be associated with - ///< each key. Internally each key is stored once, and the - ///< corresponding data values are sorted by byte-by-byte - ///< lexicographic comparison like `std::memcmp()`. - ///< In terms of keys, they are not unique, i.e. has - ///< duplicates which are sorted by associated data values. + multi = MDBX_DUPSORT, ///< A more than one data value could be associated with + ///< each key. Internally each key is stored once, and the + ///< corresponding data values are sorted by byte-by-byte + ///< lexicographic comparison like `std::memcmp()`. + ///< In terms of keys, they are not unique, i.e. has + ///< duplicates which are sorted by associated data values. #if CONSTEXPR_ENUM_FLAGS_OPERATIONS || defined(DOXYGEN) - multi_reverse = - MDBX_DUPSORT | - MDBX_REVERSEDUP, ///< A more than one data value could be associated with - ///< each key. Internally each key is stored once, and - ///< the corresponding data values are sorted by - ///< byte-by-byte lexicographic comparison in reverse - ///< order, from the end of the keys to the beginning. - ///< In terms of keys, they are not unique, i.e. has - ///< duplicates which are sorted by associated data - ///< values. - multi_samelength = - MDBX_DUPSORT | - MDBX_DUPFIXED, ///< A more than one data value could be associated with - ///< each key, and all data values must be same length. - ///< Internally each key is stored once, and the - ///< corresponding data values are sorted by byte-by-byte - ///< lexicographic comparison like `std::memcmp()`. In - ///< terms of keys, they are not unique, i.e. has - ///< duplicates which are sorted by associated data values. - multi_ordinal = - MDBX_DUPSORT | MDBX_DUPFIXED | - MDBX_INTEGERDUP, ///< A more than one data value could be associated with - ///< each key, and all data values are binary integers in - ///< native byte order, either `uint32_t` or `uint64_t`, - ///< and will be sorted as such. Internally each key is - ///< stored once, and the corresponding data values are - ///< sorted. In terms of keys, they are not unique, i.e. - ///< has duplicates which are sorted by associated data - ///< values. + multi_reverse = MDBX_DUPSORT | MDBX_REVERSEDUP, ///< A more than one data value could be associated with + ///< each key. Internally each key is stored once, and + ///< the corresponding data values are sorted by + ///< byte-by-byte lexicographic comparison in reverse + ///< order, from the end of the keys to the beginning. + ///< In terms of keys, they are not unique, i.e. has + ///< duplicates which are sorted by associated data + ///< values. + multi_samelength = MDBX_DUPSORT | MDBX_DUPFIXED, ///< A more than one data value could be associated with + ///< each key, and all data values must be same length. + ///< Internally each key is stored once, and the + ///< corresponding data values are sorted by byte-by-byte + ///< lexicographic comparison like `std::memcmp()`. In + ///< terms of keys, they are not unique, i.e. has + ///< duplicates which are sorted by associated data values. + multi_ordinal = MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERDUP, ///< A more than one data value could be associated + ///< with each key, and all data values are binary + ///< integers in native byte order, either `uint32_t` + ///< or `uint64_t`, and will be sorted as such. + ///< Internally each key is stored once, and the + ///< corresponding data values are sorted. In terms of + ///< keys, they are not unique, i.e. has duplicates + ///< which are sorted by associated data values. multi_reverse_samelength = - MDBX_DUPSORT | MDBX_REVERSEDUP | - MDBX_DUPFIXED, ///< A more than one data value could be associated with - ///< each key, and all data values must be same length. - ///< Internally each key is stored once, and the - ///< corresponding data values are sorted by byte-by-byte - ///< lexicographic comparison in reverse order, from the - ///< end of the keys to the beginning. In terms of keys, - ///< they are not unique, i.e. has duplicates which are - ///< sorted by associated data values. + MDBX_DUPSORT | MDBX_REVERSEDUP | MDBX_DUPFIXED, ///< A more than one data value could be associated with + ///< each key, and all data values must be same length. + ///< Internally each key is stored once, and the + ///< corresponding data values are sorted by byte-by-byte + ///< lexicographic comparison in reverse order, from the + ///< end of the keys to the beginning. In terms of keys, + ///< they are not unique, i.e. has duplicates which are + ///< sorted by associated data values. +#else + multi_reverse = uint32_t(MDBX_DUPSORT) | uint32_t(MDBX_REVERSEDUP), + multi_samelength = uint32_t(MDBX_DUPSORT) | uint32_t(MDBX_DUPFIXED), + multi_ordinal = uint32_t(MDBX_DUPSORT) | uint32_t(MDBX_DUPFIXED) | uint32_t(MDBX_INTEGERDUP), + multi_reverse_samelength = uint32_t(MDBX_DUPSORT) | uint32_t(MDBX_REVERSEDUP) | uint32_t(MDBX_DUPFIXED), +#endif msgpack = -1 ///< A more than one data value could be associated with each ///< key. Values are in [MessagePack](https://msgpack.org/) ///< format with appropriate comparison. Internally each key is @@ -3194,18 +2943,31 @@ enum class value_mode { ///< In terms of keys, they are not unique, i.e. has duplicates ///< which are sorted by associated data values. ///< \note Not yet implemented and PRs are welcome. -#else - multi_reverse = uint32_t(MDBX_DUPSORT) | uint32_t(MDBX_REVERSEDUP), - multi_samelength = uint32_t(MDBX_DUPSORT) | uint32_t(MDBX_DUPFIXED), - multi_ordinal = uint32_t(MDBX_DUPSORT) | uint32_t(MDBX_DUPFIXED) | - uint32_t(MDBX_INTEGERDUP), - multi_reverse_samelength = uint32_t(MDBX_DUPSORT) | - uint32_t(MDBX_REVERSEDUP) | uint32_t(MDBX_DUPFIXED) -#endif }; -/// \brief A handle for an individual database (key-value spaces) in the -/// environment. +MDBX_CXX01_CONSTEXPR_ENUM bool is_usual(value_mode mode) noexcept { + return (MDBX_db_flags_t(mode) & (MDBX_DUPSORT | MDBX_INTEGERDUP | MDBX_DUPFIXED | MDBX_REVERSEDUP)) == 0; +} + +MDBX_CXX01_CONSTEXPR_ENUM bool is_multi(value_mode mode) noexcept { + return (MDBX_db_flags_t(mode) & MDBX_DUPSORT) != 0; +} + +MDBX_CXX01_CONSTEXPR_ENUM bool is_ordinal(value_mode mode) noexcept { + return (MDBX_db_flags_t(mode) & MDBX_INTEGERDUP) != 0; +} + +MDBX_CXX01_CONSTEXPR_ENUM bool is_samelength(value_mode mode) noexcept { + return (MDBX_db_flags_t(mode) & MDBX_DUPFIXED) != 0; +} + +MDBX_CXX01_CONSTEXPR_ENUM bool is_reverse(value_mode mode) noexcept { + return (MDBX_db_flags_t(mode) & MDBX_REVERSEDUP) != 0; +} + +MDBX_CXX01_CONSTEXPR_ENUM bool is_msgpack(value_mode mode) noexcept { return mode == value_mode::msgpack; } + +/// \brief A handle for an individual table (aka key-value space, maps or sub-database) in the environment. /// \see txn::open_map() \see txn::create_map() /// \see txn::clear_map() \see txn::drop_map() /// \see txn::get_handle_info() \see txn::get_map_stat() @@ -3225,22 +2987,11 @@ struct LIBMDBX_API_TYPE map_handle { struct LIBMDBX_API_TYPE info { map_handle::flags flags; map_handle::state state; - MDBX_CXX11_CONSTEXPR info(map_handle::flags flags, - map_handle::state state) noexcept; + MDBX_CXX11_CONSTEXPR info(map_handle::flags flags, map_handle::state state) noexcept; info(const info &) noexcept = default; info &operator=(const info &) noexcept = default; -#if CONSTEXPR_ENUM_FLAGS_OPERATIONS - MDBX_CXX11_CONSTEXPR -#else - inline -#endif - ::mdbx::key_mode key_mode() const noexcept; -#if CONSTEXPR_ENUM_FLAGS_OPERATIONS - MDBX_CXX11_CONSTEXPR -#else - inline -#endif - ::mdbx::value_mode value_mode() const noexcept; + MDBX_CXX11_CONSTEXPR_ENUM mdbx::key_mode key_mode() const noexcept; + MDBX_CXX11_CONSTEXPR_ENUM mdbx::value_mode value_mode() const noexcept; }; }; @@ -3265,8 +3016,9 @@ enum put_mode { /// instances, but does not destroys the represented underlying object from the /// own class destructor. /// -/// An environment supports multiple key-value sub-databases (aka key-value -/// spaces or tables), all residing in the same shared-memory map. +/// An environment supports multiple key-value tables (aka key-value maps, +/// spaces or sub-databases), all residing in the same shared-memory mapped +/// file. class LIBMDBX_API_TYPE env { friend class txn; @@ -3284,10 +3036,8 @@ public: MDBX_CXX14_CONSTEXPR operator bool() const noexcept; MDBX_CXX14_CONSTEXPR operator const MDBX_env *() const; MDBX_CXX14_CONSTEXPR operator MDBX_env *(); - friend MDBX_CXX11_CONSTEXPR bool operator==(const env &a, - const env &b) noexcept; - friend MDBX_CXX11_CONSTEXPR bool operator!=(const env &a, - const env &b) noexcept; + friend MDBX_CXX11_CONSTEXPR bool operator==(const env &a, const env &b) noexcept; + friend MDBX_CXX11_CONSTEXPR bool operator!=(const env &a, const env &b) noexcept; //---------------------------------------------------------------------------- @@ -3297,22 +3047,26 @@ public: /// create_parameters &, const operate_parameters &, bool accede) struct LIBMDBX_API_TYPE geometry { - enum : int64_t { + enum : intptr_t { default_value = -1, ///< Means "keep current or use default" minimal_value = 0, ///< Means "minimal acceptable" maximal_value = INTPTR_MAX, ///< Means "maximal acceptable" - kB = 1000, ///< \f$10^{3}\f$ bytes - MB = kB * 1000, ///< \f$10^{6}\f$ bytes - GB = MB * 1000, ///< \f$10^{9}\f$ bytes - TB = GB * 1000, ///< \f$10^{12}\f$ bytes - PB = TB * 1000, ///< \f$10^{15}\f$ bytes - EB = PB * 1000, ///< \f$10^{18}\f$ bytes - KiB = 1024, ///< \f$2^{10}\f$ bytes - MiB = KiB << 10, ///< \f$2^{20}\f$ bytes - GiB = MiB << 10, ///< \f$2^{30}\f$ bytes - TiB = GiB << 10, ///< \f$2^{40}\f$ bytes - PiB = TiB << 10, ///< \f$2^{50}\f$ bytes - EiB = PiB << 10, ///< \f$2^{60}\f$ bytes + kB = 1000, ///< \f$10^{3}\f$ bytes (0x03E8) + MB = kB * 1000, ///< \f$10^{6}\f$ bytes (0x000F_4240) + GB = MB * 1000, ///< \f$10^{9}\f$ bytes (0x3B9A_CA00) +#if INTPTR_MAX > 0x7fffFFFFl + TB = GB * 1000, ///< \f$10^{12}\f$ bytes (0x0000_00E8_D4A5_1000) + PB = TB * 1000, ///< \f$10^{15}\f$ bytes (0x0003_8D7E_A4C6_8000) + EB = PB * 1000, ///< \f$10^{18}\f$ bytes (0x0DE0_B6B3_A764_0000) +#endif /* 64-bit intptr_t */ + KiB = 1024, ///< \f$2^{10}\f$ bytes (0x0400) + MiB = KiB << 10, ///< \f$2^{20}\f$ bytes (0x0010_0000) + GiB = MiB << 10, ///< \f$2^{30}\f$ bytes (0x4000_0000) +#if INTPTR_MAX > 0x7fffFFFFl + TiB = GiB << 10, ///< \f$2^{40}\f$ bytes (0x0000_0100_0000_0000) + PiB = TiB << 10, ///< \f$2^{50}\f$ bytes (0x0004_0000_0000_0000) + EiB = PiB << 10, ///< \f$2^{60}\f$ bytes (0x1000_0000_0000_0000) +#endif /* 64-bit intptr_t */ }; /// \brief Tagged type for output to std::ostream @@ -3323,7 +3077,7 @@ public: }; /// \brief The lower bound of database size in bytes. - intptr_t size_lower{minimal_value}; + intptr_t size_lower{default_value}; /// \brief The size in bytes to setup the database size for now. /// \details It is recommended always pass \ref default_value in this @@ -3340,14 +3094,12 @@ public: /// robustly because there may be a lack of appropriate system resources /// (which are extremely volatile in a multi-process multi-threaded /// environment). - intptr_t size_upper{maximal_value}; + intptr_t size_upper{default_value}; - /// \brief The growth step in bytes, must be greater than zero to allow the - /// database to grow. + /// \brief The growth step in bytes, must be greater than zero to allow the database to grow. intptr_t growth_step{default_value}; - /// \brief The shrink threshold in bytes, must be greater than zero to allow - /// the database to shrink. + /// \brief The shrink threshold in bytes, must be greater than zero to allow the database to shrink. intptr_t shrink_threshold{default_value}; /// \brief The database page size for new database creation @@ -3357,27 +3109,23 @@ public: intptr_t pagesize{default_value}; inline geometry &make_fixed(intptr_t size) noexcept; - inline geometry &make_dynamic(intptr_t lower = minimal_value, - intptr_t upper = maximal_value) noexcept; + inline geometry &make_dynamic(intptr_t lower = default_value, intptr_t upper = default_value) noexcept; MDBX_CXX11_CONSTEXPR geometry() noexcept {} MDBX_CXX11_CONSTEXPR geometry(const geometry &) noexcept = default; - MDBX_CXX11_CONSTEXPR geometry(intptr_t size_lower, - intptr_t size_now = default_value, - intptr_t size_upper = maximal_value, - intptr_t growth_step = default_value, - intptr_t shrink_threshold = default_value, - intptr_t pagesize = default_value) noexcept - : size_lower(size_lower), size_now(size_now), size_upper(size_upper), - growth_step(growth_step), shrink_threshold(shrink_threshold), - pagesize(pagesize) {} + MDBX_CXX11_CONSTEXPR geometry(intptr_t size_lower, intptr_t size_now = default_value, + intptr_t size_upper = default_value, intptr_t growth_step = default_value, + intptr_t shrink_threshold = default_value, intptr_t pagesize = default_value) noexcept + : size_lower(size_lower), size_now(size_now), size_upper(size_upper), growth_step(growth_step), + shrink_threshold(shrink_threshold), pagesize(pagesize) {} }; /// \brief Operation mode. enum mode { - readonly, ///< \copydoc MDBX_RDONLY - write_file_io, // don't available on OpenBSD - write_mapped_io ///< \copydoc MDBX_WRITEMAP + readonly, ///< \copydoc MDBX_RDONLY + write_file_io, // don't available on OpenBSD + write_mapped_io, ///< \copydoc MDBX_WRITEMAP + nested_transactions = write_file_io }; /// \brief Durability level. @@ -3397,15 +3145,16 @@ public: MDBX_CXX11_CONSTEXPR reclaiming_options() noexcept {} MDBX_CXX11_CONSTEXPR reclaiming_options(const reclaiming_options &) noexcept = default; - MDBX_CXX14_CONSTEXPR reclaiming_options & - operator=(const reclaiming_options &) noexcept = default; + MDBX_CXX14_CONSTEXPR reclaiming_options &operator=(const reclaiming_options &) noexcept = default; reclaiming_options(MDBX_env_flags_t) noexcept; }; /// \brief Operate options. struct LIBMDBX_API_TYPE operate_options { - /// \copydoc MDBX_NOTLS - bool orphan_read_transactions{false}; + /// \copydoc MDBX_NOSTICKYTHREADS + bool no_sticky_threads{false}; + /// \brief Разрешает вложенные транзакции ценой отключения + /// \ref MDBX_WRITEMAP и увеличением накладных расходов. bool nested_write_transactions{false}; /// \copydoc MDBX_EXCLUSIVE bool exclusive{false}; @@ -3413,17 +3162,18 @@ public: bool disable_readahead{false}; /// \copydoc MDBX_NOMEMINIT bool disable_clear_memory{false}; + /// \copydoc MDBX_VALIDATION + bool enable_validation{false}; MDBX_CXX11_CONSTEXPR operate_options() noexcept {} MDBX_CXX11_CONSTEXPR operate_options(const operate_options &) noexcept = default; - MDBX_CXX14_CONSTEXPR operate_options & - operator=(const operate_options &) noexcept = default; + MDBX_CXX14_CONSTEXPR operate_options &operator=(const operate_options &) noexcept = default; operate_options(MDBX_env_flags_t) noexcept; }; /// \brief Operate parameters. struct LIBMDBX_API_TYPE operate_parameters { - /// \brief The maximum number of named databases for the environment. + /// \brief The maximum number of named tables/maps for the environment. /// Zero means default value. unsigned max_maps{0}; /// \brief The maximum number of threads/reader slots for the environment. @@ -3436,31 +3186,25 @@ public: MDBX_CXX11_CONSTEXPR operate_parameters() noexcept {} MDBX_CXX11_CONSTEXPR - operate_parameters( - const unsigned max_maps, const unsigned max_readers = 0, - const env::mode mode = env::mode::write_mapped_io, - env::durability durability = env::durability::robust_synchronous, - const env::reclaiming_options &reclaiming = env::reclaiming_options(), - const env::operate_options &options = env::operate_options()) noexcept - : max_maps(max_maps), max_readers(max_readers), mode(mode), - durability(durability), reclaiming(reclaiming), options(options) {} + operate_parameters(const unsigned max_maps, const unsigned max_readers = 0, + const env::mode mode = env::mode::write_mapped_io, + env::durability durability = env::durability::robust_synchronous, + const env::reclaiming_options &reclaiming = env::reclaiming_options(), + const env::operate_options &options = env::operate_options()) noexcept + : max_maps(max_maps), max_readers(max_readers), mode(mode), durability(durability), reclaiming(reclaiming), + options(options) {} MDBX_CXX11_CONSTEXPR operate_parameters(const operate_parameters &) noexcept = default; - MDBX_CXX14_CONSTEXPR operate_parameters & - operator=(const operate_parameters &) noexcept = default; - MDBX_env_flags_t make_flags( - bool accede = true, ///< Allows accepting incompatible operating options - ///< in case the database is already being used by - ///< another process(es) \see MDBX_ACCEDE - bool use_subdirectory = - false ///< use subdirectory to place the DB files + MDBX_CXX14_CONSTEXPR operate_parameters &operator=(const operate_parameters &) noexcept = default; + MDBX_env_flags_t make_flags(bool accede = true, ///< Allows accepting incompatible operating options + ///< in case the database is already being used by + ///< another process(es) \see MDBX_ACCEDE + bool use_subdirectory = false ///< use subdirectory to place the DB files ) const; static env::mode mode_from_flags(MDBX_env_flags_t) noexcept; static env::durability durability_from_flags(MDBX_env_flags_t) noexcept; - inline static env::reclaiming_options - reclaiming_from_flags(MDBX_env_flags_t flags) noexcept; - inline static env::operate_options - options_from_flags(MDBX_env_flags_t flags) noexcept; + inline static env::reclaiming_options reclaiming_from_flags(MDBX_env_flags_t flags) noexcept; + inline static env::operate_options options_from_flags(MDBX_env_flags_t flags) noexcept; }; /// \brief Returns current operation parameters. @@ -3482,9 +3226,7 @@ public: bool is_empty() const; /// \brief Returns default page size for current system/platform. - static size_t default_pagesize() noexcept { - return ::mdbx_default_pagesize(); - } + static size_t default_pagesize() noexcept { return ::mdbx_default_pagesize(); } struct limits { limits() = delete; @@ -3492,80 +3234,68 @@ public: static inline size_t pagesize_min() noexcept; /// \brief Returns the maximal database page size in bytes. static inline size_t pagesize_max() noexcept; - /// \brief Returns the minimal database size in bytes for specified page - /// size. + /// \brief Returns the minimal database size in bytes for specified page size. static inline size_t dbsize_min(intptr_t pagesize); - /// \brief Returns the maximal database size in bytes for specified page - /// size. + /// \brief Returns the maximal database size in bytes for specified page size. static inline size_t dbsize_max(intptr_t pagesize); - /// \brief Returns the minimal key size in bytes for specified database - /// flags. + /// \brief Returns the minimal key size in bytes for specified table flags. static inline size_t key_min(MDBX_db_flags_t flags) noexcept; /// \brief Returns the minimal key size in bytes for specified keys mode. static inline size_t key_min(key_mode mode) noexcept; - /// \brief Returns the maximal key size in bytes for specified page size and - /// database flags. + /// \brief Returns the maximal key size in bytes for specified page size and table flags. static inline size_t key_max(intptr_t pagesize, MDBX_db_flags_t flags); - /// \brief Returns the maximal key size in bytes for specified page size and - /// keys mode. + /// \brief Returns the maximal key size in bytes for specified page size and keys mode. static inline size_t key_max(intptr_t pagesize, key_mode mode); - /// \brief Returns the maximal key size in bytes for given environment and - /// database flags. + /// \brief Returns the maximal key size in bytes for given environment and table flags. static inline size_t key_max(const env &, MDBX_db_flags_t flags); - /// \brief Returns the maximal key size in bytes for given environment and - /// keys mode. + /// \brief Returns the maximal key size in bytes for given environment and keys mode. static inline size_t key_max(const env &, key_mode mode); - /// \brief Returns the minimal values size in bytes for specified database - /// flags. + /// \brief Returns the minimal values size in bytes for specified table flags. static inline size_t value_min(MDBX_db_flags_t flags) noexcept; - /// \brief Returns the minimal values size in bytes for specified values - /// mode. + /// \brief Returns the minimal values size in bytes for specified values mode. static inline size_t value_min(value_mode) noexcept; - /// \brief Returns the maximal value size in bytes for specified page size - /// and database flags. + /// \brief Returns the maximal value size in bytes for specified page size and table flags. static inline size_t value_max(intptr_t pagesize, MDBX_db_flags_t flags); - /// \brief Returns the maximal value size in bytes for specified page size - /// and values mode. + /// \brief Returns the maximal value size in bytes for specified page size and values mode. static inline size_t value_max(intptr_t pagesize, value_mode); - /// \brief Returns the maximal value size in bytes for given environment and - /// database flags. + /// \brief Returns the maximal value size in bytes for given environment and table flags. static inline size_t value_max(const env &, MDBX_db_flags_t flags); - /// \brief Returns the maximal value size in bytes for specified page size - /// and values mode. + /// \brief Returns the maximal value size in bytes for specified page size and values mode. static inline size_t value_max(const env &, value_mode); /// \brief Returns maximal size of key-value pair to fit in a single page - /// for specified size and database flags. - static inline size_t pairsize4page_max(intptr_t pagesize, - MDBX_db_flags_t flags); + /// for specified size and table flags. + static inline size_t pairsize4page_max(intptr_t pagesize, MDBX_db_flags_t flags); /// \brief Returns maximal size of key-value pair to fit in a single page /// for specified page size and values mode. static inline size_t pairsize4page_max(intptr_t pagesize, value_mode); /// \brief Returns maximal size of key-value pair to fit in a single page - /// for given environment and database flags. + /// for given environment and table flags. static inline size_t pairsize4page_max(const env &, MDBX_db_flags_t flags); /// \brief Returns maximal size of key-value pair to fit in a single page /// for specified page size and values mode. static inline size_t pairsize4page_max(const env &, value_mode); /// \brief Returns maximal data size in bytes to fit in a leaf-page or - /// single overflow/large-page for specified size and database flags. - static inline size_t valsize4page_max(intptr_t pagesize, - MDBX_db_flags_t flags); + /// single large/overflow-page for specified size and table flags. + static inline size_t valsize4page_max(intptr_t pagesize, MDBX_db_flags_t flags); /// \brief Returns maximal data size in bytes to fit in a leaf-page or - /// single overflow/large-page for specified page size and values mode. + /// single large/overflow-page for specified page size and values mode. static inline size_t valsize4page_max(intptr_t pagesize, value_mode); /// \brief Returns maximal data size in bytes to fit in a leaf-page or - /// single overflow/large-page for given environment and database flags. + /// single large/overflow-page for given environment and table flags. static inline size_t valsize4page_max(const env &, MDBX_db_flags_t flags); /// \brief Returns maximal data size in bytes to fit in a leaf-page or - /// single overflow/large-page for specified page size and values mode. + /// single large/overflow-page for specified page size and values mode. static inline size_t valsize4page_max(const env &, value_mode); /// \brief Returns the maximal write transaction size (i.e. limit for /// summary volume of dirty pages) in bytes for specified page size. static inline size_t transaction_size_max(intptr_t pagesize); + + /// \brief Returns the maximum opened map handles, aka DBI-handles. + static inline size_t max_map_handles(void); }; /// \brief Returns the minimal database size in bytes for the environment. @@ -3577,35 +3307,24 @@ public: /// \brief Returns the maximal key size in bytes for specified keys mode. size_t key_max(key_mode mode) const { return limits::key_max(*this, mode); } /// \brief Returns the minimal value size in bytes for specified values mode. - size_t value_min(value_mode mode) const noexcept { - return limits::value_min(mode); - } + size_t value_min(value_mode mode) const noexcept { return limits::value_min(mode); } /// \brief Returns the maximal value size in bytes for specified values mode. - size_t value_max(value_mode mode) const { - return limits::value_max(*this, mode); - } + size_t value_max(value_mode mode) const { return limits::value_max(*this, mode); } /// \brief Returns the maximal write transaction size (i.e. limit for summary /// volume of dirty pages) in bytes. - size_t transaction_size_max() const { - return limits::transaction_size_max(this->get_pagesize()); - } + size_t transaction_size_max() const { return limits::transaction_size_max(this->get_pagesize()); } /// \brief Make a copy (backup) of an existing environment to the specified /// path. #ifdef MDBX_STD_FILESYSTEM_PATH - env ©(const MDBX_STD_FILESYSTEM_PATH &destination, bool compactify, - bool force_dynamic_size = false); + env ©(const MDBX_STD_FILESYSTEM_PATH &destination, bool compactify, bool force_dynamic_size = false); #endif /* MDBX_STD_FILESYSTEM_PATH */ #if defined(_WIN32) || defined(_WIN64) || defined(DOXYGEN) - env ©(const ::std::wstring &destination, bool compactify, - bool force_dynamic_size = false); - env ©(const wchar_t *destination, bool compactify, - bool force_dynamic_size = false); + env ©(const ::std::wstring &destination, bool compactify, bool force_dynamic_size = false); + env ©(const wchar_t *destination, bool compactify, bool force_dynamic_size = false); #endif /* Windows */ - env ©(const ::std::string &destination, bool compactify, - bool force_dynamic_size = false); - env ©(const char *destination, bool compactify, - bool force_dynamic_size = false); + env ©(const ::std::string &destination, bool compactify, bool force_dynamic_size = false); + env ©(const char *destination, bool compactify, bool force_dynamic_size = false); /// \brief Copy an environment to the specified file descriptor. env ©(filehandle fd, bool compactify, bool force_dynamic_size = false); @@ -3622,27 +3341,20 @@ public: /// \brief Make sure that the environment is not being used by other /// processes, or return an error otherwise. ensure_unused = MDBX_ENV_ENSURE_UNUSED, - /// \brief Wait until other processes closes the environment before - /// deletion. + /// \brief Wait until other processes closes the environment before deletion. wait_for_unused = MDBX_ENV_WAIT_FOR_UNUSED }; - /// \brief Removes the environment's files in a proper and multiprocess-safe - /// way. + /// \brief Removes the environment's files in a proper and multiprocess-safe way. #ifdef MDBX_STD_FILESYSTEM_PATH - static bool remove(const MDBX_STD_FILESYSTEM_PATH &pathname, - const remove_mode mode = just_remove); + static bool remove(const MDBX_STD_FILESYSTEM_PATH &pathname, const remove_mode mode = just_remove); #endif /* MDBX_STD_FILESYSTEM_PATH */ #if defined(_WIN32) || defined(_WIN64) || defined(DOXYGEN) - static bool remove(const ::std::wstring &pathname, - const remove_mode mode = just_remove); - static bool remove(const wchar_t *pathname, - const remove_mode mode = just_remove); + static bool remove(const ::std::wstring &pathname, const remove_mode mode = just_remove); + static bool remove(const wchar_t *pathname, const remove_mode mode = just_remove); #endif /* Windows */ - static bool remove(const ::std::string &pathname, - const remove_mode mode = just_remove); - static bool remove(const char *pathname, - const remove_mode mode = just_remove); + static bool remove(const ::std::string &pathname, const remove_mode mode = just_remove); + static bool remove(const char *pathname, const remove_mode mode = just_remove); /// \brief Statistics for a database in the MDBX environment. using stat = ::MDBX_stat; @@ -3659,12 +3371,10 @@ public: /// \brief Return snapshot information about the MDBX environment. inline info get_info() const; - /// \brief Return statistics about the MDBX environment accordingly to the - /// specified transaction. + /// \brief Return statistics about the MDBX environment accordingly to the specified transaction. inline stat get_stat(const txn &) const; - /// \brief Return information about the MDBX environment accordingly to the - /// specified transaction. + /// \brief Return information about the MDBX environment accordingly to the specified transaction. inline info get_info(const txn &) const; /// \brief Returns the file descriptor for the DXB file of MDBX environment. @@ -3676,12 +3386,11 @@ public: /// Returns environment flags. inline MDBX_env_flags_t get_flags() const; - /// \brief Returns the maximum number of threads/reader slots for the - /// environment. + /// \brief Returns the maximum number of threads/reader slots for the environment. /// \see extra_runtime_option::max_readers inline unsigned max_readers() const; - /// \brief Returns the maximum number of named databases for the environment. + /// \brief Returns the maximum number of named tables for the environment. /// \see extra_runtime_option::max_maps inline unsigned max_maps() const; @@ -3822,10 +3531,10 @@ public: /// environment is busy by other thread or none of the thresholds are reached. bool poll_sync_to_disk() { return sync_to_disk(false, true); } - /// \brief Close a key-value map (aka sub-database) handle. Normally + /// \brief Close a key-value map (aka table) handle. Normally /// unnecessary. /// - /// Closing a database handle is not necessary, but lets \ref txn::open_map() + /// Closing a table handle is not necessary, but lets \ref txn::open_map() /// reuse the handle value. Usually it's better to set a bigger /// \ref env::operate_parameters::max_maps, unless that value would be /// large. @@ -3836,8 +3545,8 @@ public: /// of libmdbx (\ref MithrilDB) will solve this issue. /// /// Handles should only be closed if no other threads are going to reference - /// the database handle or one of its cursors any further. Do not close a - /// handle if an existing transaction has modified its database. Doing so can + /// the table handle or one of its cursors any further. Do not close a + /// handle if an existing transaction has modified its table. Doing so can /// cause misbehavior from database corruption to errors like /// \ref MDBX_BAD_DBI (since the DB name is gone). inline void close_map(const map_handle &); @@ -3853,20 +3562,18 @@ public: ///< i.e. the number of committed write /// transactions since the current read /// transaction started. - size_t bytes_used; ///< The number of last used page in the MVCC-snapshot - ///< which being read, i.e. database file can't be shrunk - ///< beyond this. - size_t bytes_retained; ///< The total size of the database pages that - ///< were retired by committed write transactions - ///< after the reader's MVCC-snapshot, i.e. the space - ///< which would be freed after the Reader releases - ///< the MVCC-snapshot for reuse by completion read - ///< transaction. - - MDBX_CXX11_CONSTEXPR reader_info(int slot, mdbx_pid_t pid, - mdbx_tid_t thread, uint64_t txnid, - uint64_t lag, size_t used, - size_t retained) noexcept; + size_t bytes_used; ///< The number of last used page in the MVCC-snapshot + ///< which being read, i.e. database file can't be shrunk + ///< beyond this. + size_t bytes_retained; ///< The total size of the database pages that + ///< were retired by committed write transactions + ///< after the reader's MVCC-snapshot, i.e. the space + ///< which would be freed after the Reader releases + ///< the MVCC-snapshot for reuse by completion read + ///< transaction. + + MDBX_CXX11_CONSTEXPR reader_info(int slot, mdbx_pid_t pid, mdbx_tid_t thread, uint64_t txnid, uint64_t lag, + size_t used, size_t retained) noexcept; }; /// \brief Enumerate readers. @@ -3913,6 +3620,9 @@ public: /// \brief Creates but not start read transaction. inline txn_managed prepare_read() const; + /// \brief Starts write (read-write) transaction. + inline txn_managed start_write(txn &parent); + /// \brief Starts write (read-write) transaction. inline txn_managed start_write(bool dont_wait = false); @@ -3926,8 +3636,8 @@ public: /// object from the own class destructor, but disallows copying and assignment /// for instances. /// -/// An environment supports multiple key-value databases (aka key-value spaces -/// or tables), all residing in the same shared-memory map. +/// An environment supports multiple key-value tables (aka key-value spaces +/// or maps), all residing in the same shared-memory mapped file. class LIBMDBX_API_TYPE env_managed : public env { using inherited = env; /// delegated constructor for RAII @@ -3939,19 +3649,14 @@ public: /// \brief Open existing database. #ifdef MDBX_STD_FILESYSTEM_PATH - env_managed(const MDBX_STD_FILESYSTEM_PATH &pathname, - const operate_parameters &, bool accede = true); + env_managed(const MDBX_STD_FILESYSTEM_PATH &pathname, const operate_parameters &, bool accede = true); #endif /* MDBX_STD_FILESYSTEM_PATH */ #if defined(_WIN32) || defined(_WIN64) || defined(DOXYGEN) - env_managed(const ::std::wstring &pathname, const operate_parameters &, - bool accede = true); - explicit env_managed(const wchar_t *pathname, const operate_parameters &, - bool accede = true); + env_managed(const ::std::wstring &pathname, const operate_parameters &, bool accede = true); + explicit env_managed(const wchar_t *pathname, const operate_parameters &, bool accede = true); #endif /* Windows */ - env_managed(const ::std::string &pathname, const operate_parameters &, - bool accede = true); - explicit env_managed(const char *pathname, const operate_parameters &, - bool accede = true); + env_managed(const ::std::string &pathname, const operate_parameters &, bool accede = true); + explicit env_managed(const char *pathname, const operate_parameters &, bool accede = true); /// \brief Additional parameters for creating a new database. /// \see env_managed(const ::std::string &pathname, const create_parameters &, @@ -3966,24 +3671,21 @@ public: /// \brief Create new or open existing database. #ifdef MDBX_STD_FILESYSTEM_PATH - env_managed(const MDBX_STD_FILESYSTEM_PATH &pathname, - const create_parameters &, const operate_parameters &, + env_managed(const MDBX_STD_FILESYSTEM_PATH &pathname, const create_parameters &, const operate_parameters &, bool accede = true); #endif /* MDBX_STD_FILESYSTEM_PATH */ #if defined(_WIN32) || defined(_WIN64) || defined(DOXYGEN) - env_managed(const ::std::wstring &pathname, const create_parameters &, - const operate_parameters &, bool accede = true); - explicit env_managed(const wchar_t *pathname, const create_parameters &, - const operate_parameters &, bool accede = true); + env_managed(const ::std::wstring &pathname, const create_parameters &, const operate_parameters &, + bool accede = true); + explicit env_managed(const wchar_t *pathname, const create_parameters &, const operate_parameters &, + bool accede = true); #endif /* Windows */ - env_managed(const ::std::string &pathname, const create_parameters &, - const operate_parameters &, bool accede = true); - explicit env_managed(const char *pathname, const create_parameters &, - const operate_parameters &, bool accede = true); + env_managed(const ::std::string &pathname, const create_parameters &, const operate_parameters &, bool accede = true); + explicit env_managed(const char *pathname, const create_parameters &, const operate_parameters &, bool accede = true); /// \brief Explicitly closes the environment and release the memory map. /// - /// Only a single thread may call this function. All transactions, databases, + /// Only a single thread may call this function. All transactions, tables, /// and cursors must already be closed before calling this function. Attempts /// to use any such handles after calling this function will cause a /// `SIGSEGV`. The environment handle will be freed and must not be used again @@ -4035,10 +3737,8 @@ public: MDBX_CXX14_CONSTEXPR operator bool() const noexcept; MDBX_CXX14_CONSTEXPR operator const MDBX_txn *() const; MDBX_CXX14_CONSTEXPR operator MDBX_txn *(); - friend MDBX_CXX11_CONSTEXPR bool operator==(const txn &a, - const txn &b) noexcept; - friend MDBX_CXX11_CONSTEXPR bool operator!=(const txn &a, - const txn &b) noexcept; + friend MDBX_CXX11_CONSTEXPR bool operator==(const txn &a, const txn &b) noexcept; + friend MDBX_CXX11_CONSTEXPR bool operator!=(const txn &a, const txn &b) noexcept; /// \brief Returns the transaction's environment. inline ::mdbx::env env() const noexcept; @@ -4070,8 +3770,7 @@ public: /// volume of dirty pages) in bytes. size_t size_max() const { return env().transaction_size_max(); } - /// \brief Returns current write transaction size (i.e.summary volume of dirty - /// pages) in bytes. + /// \brief Returns current write transaction size (i.e.summary volume of dirty pages) in bytes. size_t size_current() const { assert(is_readwrite()); return size_t(get_info().txn_space_dirty); @@ -4079,39 +3778,63 @@ public: //---------------------------------------------------------------------------- - /// \brief Reset a read-only transaction. + /// \brief Reset read-only transaction. inline void reset_reading(); - /// \brief Renew a read-only transaction. + /// \brief Renew read-only transaction. inline void renew_reading(); + /// \brief Marks transaction as broken to prevent further operations. + inline void make_broken(); + + /// \brief Park read-only transaction. + inline void park_reading(bool autounpark = true); + + /// \brief Resume parked read-only transaction. + /// \returns True if transaction was restarted while `restart_if_ousted=true`. + inline bool unpark_reading(bool restart_if_ousted = true); + /// \brief Start nested write transaction. txn_managed start_nested(); /// \brief Opens cursor for specified key-value map handle. inline cursor_managed open_cursor(map_handle map) const; + /// \brief Unbind or close all cursors. + inline size_t release_all_cursors(bool unbind) const; + + /// \brief Close all cursors. + inline size_t close_all_cursors() const { return release_all_cursors(false); } + + /// \brief Unbind all cursors. + inline size_t unbind_all_cursors() const { return release_all_cursors(true); } + /// \brief Open existing key-value map. - inline map_handle open_map( - const char *name, - const ::mdbx::key_mode key_mode = ::mdbx::key_mode::usual, - const ::mdbx::value_mode value_mode = ::mdbx::value_mode::single) const; + inline map_handle open_map(const char *name, const ::mdbx::key_mode key_mode = ::mdbx::key_mode::usual, + const ::mdbx::value_mode value_mode = ::mdbx::value_mode::single) const; /// \brief Open existing key-value map. - inline map_handle open_map( - const ::std::string &name, - const ::mdbx::key_mode key_mode = ::mdbx::key_mode::usual, - const ::mdbx::value_mode value_mode = ::mdbx::value_mode::single) const; + inline map_handle open_map(const ::std::string &name, const ::mdbx::key_mode key_mode = ::mdbx::key_mode::usual, + const ::mdbx::value_mode value_mode = ::mdbx::value_mode::single) const; + /// \brief Open existing key-value map. + inline map_handle open_map(const ::mdbx::slice &name, const ::mdbx::key_mode key_mode = ::mdbx::key_mode::usual, + const ::mdbx::value_mode value_mode = ::mdbx::value_mode::single) const; + /// \brief Open existing key-value map. + inline map_handle open_map_accede(const char *name) const; + /// \brief Open existing key-value map. + inline map_handle open_map_accede(const ::std::string &name) const; + /// \brief Open existing key-value map. + inline map_handle open_map_accede(const ::mdbx::slice &name) const; + + /// \brief Create new or open existing key-value map. + inline map_handle create_map(const char *name, const ::mdbx::key_mode key_mode = ::mdbx::key_mode::usual, + const ::mdbx::value_mode value_mode = ::mdbx::value_mode::single); /// \brief Create new or open existing key-value map. - inline map_handle - create_map(const char *name, - const ::mdbx::key_mode key_mode = ::mdbx::key_mode::usual, - const ::mdbx::value_mode value_mode = ::mdbx::value_mode::single); + inline map_handle create_map(const ::std::string &name, const ::mdbx::key_mode key_mode = ::mdbx::key_mode::usual, + const ::mdbx::value_mode value_mode = ::mdbx::value_mode::single); /// \brief Create new or open existing key-value map. - inline map_handle - create_map(const ::std::string &name, - const ::mdbx::key_mode key_mode = ::mdbx::key_mode::usual, - const ::mdbx::value_mode value_mode = ::mdbx::value_mode::single); + inline map_handle create_map(const ::mdbx::slice &name, const ::mdbx::key_mode key_mode = ::mdbx::key_mode::usual, + const ::mdbx::value_mode value_mode = ::mdbx::value_mode::single); /// \brief Drops key-value map using handle. inline void drop_map(map_handle map); @@ -4123,6 +3846,10 @@ public: /// \return `True` if the key-value map existed and was deleted, either /// `false` if the key-value map did not exist and there is nothing to delete. inline bool drop_map(const ::std::string &name, bool throw_if_absent = false); + /// \brief Drop key-value map. + /// \return `True` if the key-value map existed and was deleted, either + /// `false` if the key-value map did not exist and there is nothing to delete. + bool drop_map(const ::mdbx::slice &name, bool throw_if_absent = false); /// \brief Clear key-value map. inline void clear_map(map_handle map); @@ -4131,92 +3858,132 @@ public: bool clear_map(const char *name, bool throw_if_absent = false); /// \return `True` if the key-value map existed and was cleared, either /// `false` if the key-value map did not exist and there is nothing to clear. - inline bool clear_map(const ::std::string &name, - bool throw_if_absent = false); + inline bool clear_map(const ::std::string &name, bool throw_if_absent = false); + /// \return `True` if the key-value map existed and was cleared, either + /// `false` if the key-value map did not exist and there is nothing to clear. + bool clear_map(const ::mdbx::slice &name, bool throw_if_absent = false); + + /// \brief Переименовывает таблицу ключ-значение. + inline void rename_map(map_handle map, const char *new_name); + /// \brief Переименовывает таблицу ключ-значение. + inline void rename_map(map_handle map, const ::std::string &new_name); + /// \brief Переименовывает таблицу ключ-значение. + inline void rename_map(map_handle map, const ::mdbx::slice &new_name); + /// \brief Переименовывает таблицу ключ-значение. + /// \return `True` если таблица существует и была переименована, либо + /// `false` в случае отсутствия исходной таблицы. + bool rename_map(const char *old_name, const char *new_name, bool throw_if_absent = false); + /// \brief Переименовывает таблицу ключ-значение. + /// \return `True` если таблица существует и была переименована, либо + /// `false` в случае отсутствия исходной таблицы. + bool rename_map(const ::std::string &old_name, const ::std::string &new_name, bool throw_if_absent = false); + /// \brief Переименовывает таблицу ключ-значение. + /// \return `True` если таблица существует и была переименована, либо + /// `false` в случае отсутствия исходной таблицы. + bool rename_map(const ::mdbx::slice &old_name, const ::mdbx::slice &new_name, bool throw_if_absent = false); + +#if defined(DOXYGEN) || (defined(__cpp_lib_string_view) && __cpp_lib_string_view >= 201606L) + + /// \brief Open existing key-value map. + inline map_handle open_map(const ::std::string_view &name, const ::mdbx::key_mode key_mode = ::mdbx::key_mode::usual, + const ::mdbx::value_mode value_mode = ::mdbx::value_mode::single) const { + return open_map(::mdbx::slice(name), key_mode, value_mode); + } + /// \brief Open existing key-value map. + inline map_handle open_map_accede(const ::std::string_view &name) const; + /// \brief Create new or open existing key-value map. + inline map_handle create_map(const ::std::string_view &name, + const ::mdbx::key_mode key_mode = ::mdbx::key_mode::usual, + const ::mdbx::value_mode value_mode = ::mdbx::value_mode::single) { + return create_map(::mdbx::slice(name), key_mode, value_mode); + } + /// \brief Drop key-value map. + /// \return `True` if the key-value map existed and was deleted, either + /// `false` if the key-value map did not exist and there is nothing to delete. + bool drop_map(const ::std::string_view &name, bool throw_if_absent = false) { + return drop_map(::mdbx::slice(name), throw_if_absent); + } + /// \return `True` if the key-value map existed and was cleared, either + /// `false` if the key-value map did not exist and there is nothing to clear. + bool clear_map(const ::std::string_view &name, bool throw_if_absent = false) { + return clear_map(::mdbx::slice(name), throw_if_absent); + } + /// \brief Переименовывает таблицу ключ-значение. + inline void rename_map(map_handle map, const ::std::string_view &new_name); + /// \brief Переименовывает таблицу ключ-значение. + /// \return `True` если таблица существует и была переименована, либо + /// `false` в случае отсутствия исходной таблицы. + bool rename_map(const ::std::string_view &old_name, const ::std::string_view &new_name, + bool throw_if_absent = false) { + return rename_map(::mdbx::slice(old_name), ::mdbx::slice(new_name), throw_if_absent); + } +#endif /* __cpp_lib_string_view >= 201606L */ using map_stat = ::MDBX_stat; - /// \brief Returns statistics for a sub-database. + /// \brief Returns statistics for a table. inline map_stat get_map_stat(map_handle map) const; /// \brief Returns depth (bitmask) information of nested dupsort (multi-value) - /// B+trees for given database. + /// B+trees for given table. inline uint32_t get_tree_deepmask(map_handle map) const; - /// \brief Returns information about key-value map (aka sub-database) handle. + /// \brief Returns information about key-value map (aka table) handle. inline map_handle::info get_handle_info(map_handle map) const; using canary = ::MDBX_canary; - /// \brief Set integers markers (aka "canary") associated with the - /// environment. + /// \brief Set integers markers (aka "canary") associated with the environment. inline txn &put_canary(const canary &); - /// \brief Returns fours integers markers (aka "canary") associated with the - /// environment. + /// \brief Returns fours integers markers (aka "canary") associated with the environment. inline canary get_canary() const; - /// Reads sequence generator associated with a key-value map (aka - /// sub-database). + /// Reads sequence generator associated with a key-value map (aka table). inline uint64_t sequence(map_handle map) const; - /// \brief Reads and increment sequence generator associated with a key-value - /// map (aka sub-database). + /// \brief Reads and increment sequence generator associated with a key-value map (aka table). inline uint64_t sequence(map_handle map, uint64_t increment); - /// \brief Compare two keys according to a particular key-value map (aka - /// sub-database). - inline int compare_keys(map_handle map, const slice &a, - const slice &b) const noexcept; - /// \brief Compare two values according to a particular key-value map (aka - /// sub-database). - inline int compare_values(map_handle map, const slice &a, - const slice &b) const noexcept; - /// \brief Compare keys of two pairs according to a particular key-value map - /// (aka sub-database). - inline int compare_keys(map_handle map, const pair &a, - const pair &b) const noexcept; - /// \brief Compare values of two pairs according to a particular key-value map - /// (aka sub-database). - inline int compare_values(map_handle map, const pair &a, - const pair &b) const noexcept; - - /// \brief Get value by key from a key-value map (aka sub-database). + /// \brief Compare two keys according to a particular key-value map (aka table). + inline int compare_keys(map_handle map, const slice &a, const slice &b) const noexcept; + /// \brief Compare two values according to a particular key-value map (aka table). + inline int compare_values(map_handle map, const slice &a, const slice &b) const noexcept; + /// \brief Compare keys of two pairs according to a particular key-value map (aka table). + inline int compare_keys(map_handle map, const pair &a, const pair &b) const noexcept; + /// \brief Compare values of two pairs according to a particular key-value map(aka table). + inline int compare_values(map_handle map, const pair &a, const pair &b) const noexcept; + + /// \brief Get value by key from a key-value map (aka table). inline slice get(map_handle map, const slice &key) const; - /// \brief Get first of multi-value and values count by key from a key-value - /// multimap (aka sub-database). + /// \brief Get first of multi-value and values count by key from a key-value multimap (aka table). inline slice get(map_handle map, slice key, size_t &values_count) const; - /// \brief Get value by key from a key-value map (aka sub-database). - inline slice get(map_handle map, const slice &key, - const slice &value_at_absence) const; - /// \brief Get first of multi-value and values count by key from a key-value - /// multimap (aka sub-database). - inline slice get(map_handle map, slice key, size_t &values_count, - const slice &value_at_absence) const; - /// \brief Get value for equal or great key from a database. + /// \brief Get value by key from a key-value map (aka table). + inline slice get(map_handle map, const slice &key, const slice &value_at_absence) const; + /// \brief Get first of multi-value and values count by key from a key-value multimap (aka table). + inline slice get(map_handle map, slice key, size_t &values_count, const slice &value_at_absence) const; + /// \brief Get value for equal or great key from a table. /// \return Bundle of key-value pair and boolean flag, /// which will be `true` if the exact key was found and `false` otherwise. inline pair_result get_equal_or_great(map_handle map, const slice &key) const; - /// \brief Get value for equal or great key from a database. + /// \brief Get value for equal or great key from a table. /// \return Bundle of key-value pair and boolean flag, /// which will be `true` if the exact key was found and `false` otherwise. - inline pair_result get_equal_or_great(map_handle map, const slice &key, - const slice &value_at_absence) const; + inline pair_result get_equal_or_great(map_handle map, const slice &key, const slice &value_at_absence) const; - inline MDBX_error_t put(map_handle map, const slice &key, slice *value, - MDBX_put_flags_t flags) noexcept; + inline MDBX_error_t put(map_handle map, const slice &key, slice *value, MDBX_put_flags_t flags) noexcept; inline void put(map_handle map, const slice &key, slice value, put_mode mode); inline void insert(map_handle map, const slice &key, slice value); inline value_result try_insert(map_handle map, const slice &key, slice value); - inline slice insert_reserve(map_handle map, const slice &key, - size_t value_length); - inline value_result try_insert_reserve(map_handle map, const slice &key, - size_t value_length); + inline slice insert_reserve(map_handle map, const slice &key, size_t value_length); + inline value_result try_insert_reserve(map_handle map, const slice &key, size_t value_length); inline void upsert(map_handle map, const slice &key, const slice &value); - inline slice upsert_reserve(map_handle map, const slice &key, - size_t value_length); + inline slice upsert_reserve(map_handle map, const slice &key, size_t value_length); inline void update(map_handle map, const slice &key, const slice &value); inline bool try_update(map_handle map, const slice &key, const slice &value); - inline slice update_reserve(map_handle map, const slice &key, - size_t value_length); - inline value_result try_update_reserve(map_handle map, const slice &key, - size_t value_length); + inline slice update_reserve(map_handle map, const slice &key, size_t value_length); + inline value_result try_update_reserve(map_handle map, const slice &key, size_t value_length); + + void put(map_handle map, const pair &kv, put_mode mode) { return put(map, kv.key, kv.value, mode); } + void insert(map_handle map, const pair &kv) { return insert(map, kv.key, kv.value); } + value_result try_insert(map_handle map, const pair &kv) { return try_insert(map, kv.key, kv.value); } + void upsert(map_handle map, const pair &kv) { return upsert(map, kv.key, kv.value); } /// \brief Removes all values for given key. inline bool erase(map_handle map, const slice &key); @@ -4225,28 +3992,27 @@ public: inline bool erase(map_handle map, const slice &key, const slice &value); /// \brief Replaces the particular multi-value of the key with a new value. - inline void replace(map_handle map, const slice &key, slice old_value, - const slice &new_value); + inline void replace(map_handle map, const slice &key, slice old_value, const slice &new_value); /// \brief Removes and return a value of the key. template inline buffer extract(map_handle map, const slice &key, - const typename buffer::allocator_type & - allocator = buffer::allocator_type()); + const typename buffer::allocator_type &allocator = + buffer::allocator_type()); /// \brief Replaces and returns a value of the key with new one. template inline buffer replace(map_handle map, const slice &key, const slice &new_value, - const typename buffer::allocator_type & - allocator = buffer::allocator_type()); + const typename buffer::allocator_type &allocator = + buffer::allocator_type()); template - inline buffer replace_reserve( - map_handle map, const slice &key, slice &new_value, - const typename buffer::allocator_type - &allocator = buffer::allocator_type()); + inline buffer + replace_reserve(map_handle map, const slice &key, slice &new_value, + const typename buffer::allocator_type &allocator = + buffer::allocator_type()); /// \brief Adding a key-value pair, provided that ascending order of the keys /// and (optionally) values are preserved. @@ -4264,34 +4030,29 @@ public: /// \param [in] multivalue_order_preserved /// If `multivalue_order_preserved == true` then the same rules applied for /// to pages of nested b+tree of multimap's values. - inline void append(map_handle map, const slice &key, const slice &value, - bool multivalue_order_preserved = true); + inline void append(map_handle map, const slice &key, const slice &value, bool multivalue_order_preserved = true); + inline void append(map_handle map, const pair &kv, bool multivalue_order_preserved = true) { + return append(map, kv.key, kv.value, multivalue_order_preserved); + } - size_t put_multiple(map_handle map, const slice &key, - const size_t value_length, const void *values_array, - size_t values_count, put_mode mode, - bool allow_partial = false); + inline size_t put_multiple_samelength(map_handle map, const slice &key, const size_t value_length, + const void *values_array, size_t values_count, put_mode mode, + bool allow_partial = false); template - size_t put_multiple(map_handle map, const slice &key, - const VALUE *values_array, size_t values_count, - put_mode mode, bool allow_partial = false) { - static_assert(::std::is_standard_layout::value && - !::std::is_pointer::value && + size_t put_multiple_samelength(map_handle map, const slice &key, const VALUE *values_array, size_t values_count, + put_mode mode, bool allow_partial = false) { + static_assert(::std::is_standard_layout::value && !::std::is_pointer::value && !::std::is_array::value, "Must be a standard layout type!"); - return put_multiple(map, key, sizeof(VALUE), values_array, values_count, - mode, allow_partial); + return put_multiple_samelength(map, key, sizeof(VALUE), values_array, values_count, mode, allow_partial); } template - void put_multiple(map_handle map, const slice &key, - const ::std::vector &vector, put_mode mode) { - put_multiple(map, key, vector.data(), vector.size(), mode); + void put_multiple_samelength(map_handle map, const slice &key, const ::std::vector &vector, put_mode mode) { + put_multiple_samelength(map, key, vector.data(), vector.size(), mode); } - inline ptrdiff_t estimate(map_handle map, const pair &from, - const pair &to) const; - inline ptrdiff_t estimate(map_handle map, const slice &from, - const slice &to) const; + inline ptrdiff_t estimate(map_handle map, const pair &from, const pair &to) const; + inline ptrdiff_t estimate(map_handle map, const slice &from, const slice &to) const; inline ptrdiff_t estimate_from_first(map_handle map, const slice &to) const; inline ptrdiff_t estimate_to_last(map_handle map, const slice &from) const; }; @@ -4336,6 +4097,10 @@ public: /// \brief Commit all the operations of a transaction into the database. void commit(); + /// \brief Commit all the operations of a transaction into the database + /// and then start read transaction. + void commit_embark_read(); + using commit_latency = MDBX_commit_latency; /// \brief Commit all the operations of a transaction into the database @@ -4366,9 +4131,9 @@ public: class LIBMDBX_API_TYPE cursor { protected: MDBX_cursor *handle_{nullptr}; - MDBX_CXX11_CONSTEXPR cursor(MDBX_cursor *ptr) noexcept; public: + MDBX_CXX11_CONSTEXPR cursor(MDBX_cursor *ptr) noexcept; MDBX_CXX11_CONSTEXPR cursor() noexcept = default; cursor(const cursor &) noexcept = default; inline cursor &operator=(cursor &&other) noexcept; @@ -4378,10 +4143,31 @@ public: MDBX_CXX14_CONSTEXPR operator bool() const noexcept; MDBX_CXX14_CONSTEXPR operator const MDBX_cursor *() const; MDBX_CXX14_CONSTEXPR operator MDBX_cursor *(); - friend MDBX_CXX11_CONSTEXPR bool operator==(const cursor &a, - const cursor &b) noexcept; - friend MDBX_CXX11_CONSTEXPR bool operator!=(const cursor &a, - const cursor &b) noexcept; + friend MDBX_CXX11_CONSTEXPR bool operator==(const cursor &a, const cursor &b) noexcept; + friend MDBX_CXX11_CONSTEXPR bool operator!=(const cursor &a, const cursor &b) noexcept; + + friend inline int compare_position_nothrow(const cursor &left, const cursor &right, bool ignore_nested) noexcept; + friend inline int compare_position(const cursor &left, const cursor &right, bool ignore_nested); + + bool is_before_than(const cursor &other, bool ignore_nested = false) const { + return compare_position(*this, other, ignore_nested) < 0; + } + + bool is_same_or_before_than(const cursor &other, bool ignore_nested = false) const { + return compare_position(*this, other, ignore_nested) <= 0; + } + + bool is_same_position(const cursor &other, bool ignore_nested = false) const { + return compare_position(*this, other, ignore_nested) == 0; + } + + bool is_after_than(const cursor &other, bool ignore_nested = false) const { + return compare_position(*this, other, ignore_nested) > 0; + } + + bool is_same_or_after_than(const cursor &other, bool ignore_nested = false) const { + return compare_position(*this, other, ignore_nested) >= 0; + } /// \brief Returns the application context associated with the cursor. inline void *get_context() const noexcept; @@ -4406,22 +4192,49 @@ public: multi_find_pair = MDBX_GET_BOTH, multi_exactkey_lowerboundvalue = MDBX_GET_BOTH_RANGE, - find_key = MDBX_SET, + seek_key = MDBX_SET, key_exact = MDBX_SET_KEY, - key_lowerbound = MDBX_SET_RANGE + key_lowerbound = MDBX_SET_RANGE, + + /* Doubtless cursor positioning at a specified key. */ + key_lesser_than = MDBX_TO_KEY_LESSER_THAN, + key_lesser_or_equal = MDBX_TO_KEY_LESSER_OR_EQUAL, + key_equal = MDBX_TO_KEY_EQUAL, + key_greater_or_equal = MDBX_TO_KEY_GREATER_OR_EQUAL, + key_greater_than = MDBX_TO_KEY_GREATER_THAN, + + /* Doubtless cursor positioning at a specified key-value pair + * for dupsort/multi-value hives. */ + multi_exactkey_value_lesser_than = MDBX_TO_EXACT_KEY_VALUE_LESSER_THAN, + multi_exactkey_value_lesser_or_equal = MDBX_TO_EXACT_KEY_VALUE_LESSER_OR_EQUAL, + multi_exactkey_value_equal = MDBX_TO_EXACT_KEY_VALUE_EQUAL, + multi_exactkey_value_greater_or_equal = MDBX_TO_EXACT_KEY_VALUE_GREATER_OR_EQUAL, + multi_exactkey_value_greater = MDBX_TO_EXACT_KEY_VALUE_GREATER_THAN, + + pair_lesser_than = MDBX_TO_PAIR_LESSER_THAN, + pair_lesser_or_equal = MDBX_TO_PAIR_LESSER_OR_EQUAL, + pair_equal = MDBX_TO_PAIR_EQUAL, + pair_exact = pair_equal, + pair_greater_or_equal = MDBX_TO_PAIR_GREATER_OR_EQUAL, + pair_greater_than = MDBX_TO_PAIR_GREATER_THAN, + + batch_samelength = MDBX_GET_MULTIPLE, + batch_samelength_next = MDBX_NEXT_MULTIPLE, + batch_samelength_previous = MDBX_PREV_MULTIPLE, + seek_and_batch_samelength = MDBX_SEEK_AND_GET_MULTIPLE }; + // TODO: добавить легковесный proxy-класс для замещения параметра throw_notfound более сложным набором опций, + // в том числе с explicit-конструктором из bool, чтобы защититься от неявной конвертации ключей поиска + // и других параметров в bool-throw_notfound. + struct move_result : public pair_result { inline move_result(const cursor &cursor, bool throw_notfound); move_result(cursor &cursor, move_operation operation, bool throw_notfound) - : move_result(cursor, operation, slice::invalid(), slice::invalid(), - throw_notfound) {} - move_result(cursor &cursor, move_operation operation, const slice &key, - bool throw_notfound) - : move_result(cursor, operation, key, slice::invalid(), - throw_notfound) {} - inline move_result(cursor &cursor, move_operation operation, - const slice &key, const slice &value, + : move_result(cursor, operation, slice::invalid(), slice::invalid(), throw_notfound) {} + move_result(cursor &cursor, move_operation operation, const slice &key, bool throw_notfound) + : move_result(cursor, operation, key, slice::invalid(), throw_notfound) {} + inline move_result(cursor &cursor, move_operation operation, const slice &key, const slice &value, bool throw_notfound); move_result(const move_result &) noexcept = default; move_result &operator=(const move_result &) noexcept = default; @@ -4430,61 +4243,209 @@ public: struct estimate_result : public pair { ptrdiff_t approximate_quantity; estimate_result(const cursor &cursor, move_operation operation) - : estimate_result(cursor, operation, slice::invalid(), - slice::invalid()) {} - estimate_result(const cursor &cursor, move_operation operation, - const slice &key) + : estimate_result(cursor, operation, slice::invalid(), slice::invalid()) {} + estimate_result(const cursor &cursor, move_operation operation, const slice &key) : estimate_result(cursor, operation, key, slice::invalid()) {} - inline estimate_result(const cursor &cursor, move_operation operation, - const slice &key, const slice &value); + inline estimate_result(const cursor &cursor, move_operation operation, const slice &key, const slice &value); estimate_result(const estimate_result &) noexcept = default; estimate_result &operator=(const estimate_result &) noexcept = default; }; protected: - inline bool move(move_operation operation, MDBX_val *key, MDBX_val *value, - bool throw_notfound) const - /* fake const, i.e. for some operations */; - inline ptrdiff_t estimate(move_operation operation, MDBX_val *key, - MDBX_val *value) const; + /* fake const, i.e. for some move/get operations */ + inline bool move(move_operation operation, MDBX_val *key, MDBX_val *value, bool throw_notfound) const; + + inline ptrdiff_t estimate(move_operation operation, MDBX_val *key, MDBX_val *value) const; public: - inline move_result move(move_operation operation, bool throw_notfound); - inline move_result to_first(bool throw_notfound = true); - inline move_result to_previous(bool throw_notfound = true); - inline move_result to_previous_last_multi(bool throw_notfound = true); - inline move_result to_current_first_multi(bool throw_notfound = true); - inline move_result to_current_prev_multi(bool throw_notfound = true); - inline move_result current(bool throw_notfound = true) const; - inline move_result to_current_next_multi(bool throw_notfound = true); - inline move_result to_current_last_multi(bool throw_notfound = true); - inline move_result to_next_first_multi(bool throw_notfound = true); - inline move_result to_next(bool throw_notfound = true); - inline move_result to_last(bool throw_notfound = true); - - inline move_result move(move_operation operation, const slice &key, - bool throw_notfound); - inline move_result find(const slice &key, bool throw_notfound = true); - inline move_result lower_bound(const slice &key, bool throw_notfound = true); + template + bool scan(CALLABLE_PREDICATE predicate, move_operation start = first, move_operation turn = next) { + struct wrapper : public exception_thunk { + static int probe(void *context, MDBX_val *key, MDBX_val *value, void *arg) noexcept { + auto thunk = static_cast(context); + assert(thunk->is_clean()); + auto &predicate = *static_cast(arg); + try { + return predicate(pair(*key, *value)) ? MDBX_RESULT_TRUE : MDBX_RESULT_FALSE; + } catch (... /* capture any exception to rethrow it over C code */) { + thunk->capture(); + return MDBX_RESULT_TRUE; + } + } + } thunk; + return error::boolean_or_throw( + ::mdbx_cursor_scan(handle_, wrapper::probe, &thunk, MDBX_cursor_op(start), MDBX_cursor_op(turn), &predicate), + thunk); + } + + template bool fullscan(CALLABLE_PREDICATE predicate, bool backward = false) { + return scan(std::move(predicate), backward ? last : first, backward ? previous : next); + } + + template + bool scan_from(CALLABLE_PREDICATE predicate, slice &from, move_operation start = key_greater_or_equal, + move_operation turn = next) { + struct wrapper : public exception_thunk { + static int probe(void *context, MDBX_val *key, MDBX_val *value, void *arg) noexcept { + auto thunk = static_cast(context); + assert(thunk->is_clean()); + auto &predicate = *static_cast(arg); + try { + return predicate(pair(*key, *value)) ? MDBX_RESULT_TRUE : MDBX_RESULT_FALSE; + } catch (... /* capture any exception to rethrow it over C code */) { + thunk->capture(); + return MDBX_RESULT_TRUE; + } + } + } thunk; + return error::boolean_or_throw(::mdbx_cursor_scan_from(handle_, wrapper::probe, &thunk, MDBX_cursor_op(start), + &from, nullptr, MDBX_cursor_op(turn), &predicate), + thunk); + } + + template + bool scan_from(CALLABLE_PREDICATE predicate, pair &from, move_operation start = pair_greater_or_equal, + move_operation turn = next) { + struct wrapper : public exception_thunk { + static int probe(void *context, MDBX_val *key, MDBX_val *value, void *arg) noexcept { + auto thunk = static_cast(context); + assert(thunk->is_clean()); + auto &predicate = *static_cast(arg); + try { + return predicate(pair(*key, *value)) ? MDBX_RESULT_TRUE : MDBX_RESULT_FALSE; + } catch (... /* capture any exception to rethrow it over C code */) { + thunk->capture(); + return MDBX_RESULT_TRUE; + } + } + } thunk; + return error::boolean_or_throw(::mdbx_cursor_scan_from(handle_, wrapper::probe, &thunk, MDBX_cursor_op(start), + &from.key, &from.value, MDBX_cursor_op(turn), &predicate), + thunk); + } - inline move_result move(move_operation operation, const slice &key, - const slice &value, bool throw_notfound); - inline move_result find_multivalue(const slice &key, const slice &value, - bool throw_notfound = true); - inline move_result lower_bound_multivalue(const slice &key, - const slice &value, - bool throw_notfound = false); + move_result move(move_operation operation, bool throw_notfound) { + return move_result(*this, operation, throw_notfound); + } + move_result move(move_operation operation, const slice &key, bool throw_notfound) { + return move_result(*this, operation, key, slice::invalid(), throw_notfound); + } + move_result move(move_operation operation, const slice &key, const slice &value, bool throw_notfound) { + return move_result(*this, operation, key, value, throw_notfound); + } + bool move(move_operation operation, slice &key, slice &value, bool throw_notfound) { + return move(operation, &key, &value, throw_notfound); + } + + move_result to_first(bool throw_notfound = true) { return move(first, throw_notfound); } + move_result to_previous(bool throw_notfound = true) { return move(previous, throw_notfound); } + move_result to_previous_last_multi(bool throw_notfound = true) { + return move(multi_prevkey_lastvalue, throw_notfound); + } + move_result to_current_first_multi(bool throw_notfound = true) { + return move(multi_currentkey_firstvalue, throw_notfound); + } + move_result to_current_prev_multi(bool throw_notfound = true) { + return move(multi_currentkey_prevvalue, throw_notfound); + } + move_result current(bool throw_notfound = true) const { return move_result(*this, throw_notfound); } + move_result to_current_next_multi(bool throw_notfound = true) { + return move(multi_currentkey_nextvalue, throw_notfound); + } + move_result to_current_last_multi(bool throw_notfound = true) { + return move(multi_currentkey_lastvalue, throw_notfound); + } + move_result to_next_first_multi(bool throw_notfound = true) { return move(multi_nextkey_firstvalue, throw_notfound); } + move_result to_next(bool throw_notfound = true) { return move(next, throw_notfound); } + move_result to_last(bool throw_notfound = true) { return move(last, throw_notfound); } + + move_result to_key_lesser_than(const slice &key, bool throw_notfound = true) { + return move(key_lesser_than, key, throw_notfound); + } + move_result to_key_lesser_or_equal(const slice &key, bool throw_notfound = true) { + return move(key_lesser_or_equal, key, throw_notfound); + } + move_result to_key_equal(const slice &key, bool throw_notfound = true) { + return move(key_equal, key, throw_notfound); + } + move_result to_key_exact(const slice &key, bool throw_notfound = true) { + return move(key_exact, key, throw_notfound); + } + move_result to_key_greater_or_equal(const slice &key, bool throw_notfound = true) { + return move(key_greater_or_equal, key, throw_notfound); + } + move_result to_key_greater_than(const slice &key, bool throw_notfound = true) { + return move(key_greater_than, key, throw_notfound); + } + + move_result to_exact_key_value_lesser_than(const slice &key, const slice &value, bool throw_notfound = true) { + return move(multi_exactkey_value_lesser_than, key, value, throw_notfound); + } + move_result to_exact_key_value_lesser_or_equal(const slice &key, const slice &value, bool throw_notfound = true) { + return move(multi_exactkey_value_lesser_or_equal, key, value, throw_notfound); + } + move_result to_exact_key_value_equal(const slice &key, const slice &value, bool throw_notfound = true) { + return move(multi_exactkey_value_equal, key, value, throw_notfound); + } + move_result to_exact_key_value_greater_or_equal(const slice &key, const slice &value, bool throw_notfound = true) { + return move(multi_exactkey_value_greater_or_equal, key, value, throw_notfound); + } + move_result to_exact_key_value_greater_than(const slice &key, const slice &value, bool throw_notfound = true) { + return move(multi_exactkey_value_greater, key, value, throw_notfound); + } + + move_result to_pair_lesser_than(const slice &key, const slice &value, bool throw_notfound = true) { + return move(pair_lesser_than, key, value, throw_notfound); + } + move_result to_pair_lesser_or_equal(const slice &key, const slice &value, bool throw_notfound = true) { + return move(pair_lesser_or_equal, key, value, throw_notfound); + } + move_result to_pair_equal(const slice &key, const slice &value, bool throw_notfound = true) { + return move(pair_equal, key, value, throw_notfound); + } + move_result to_pair_exact(const slice &key, const slice &value, bool throw_notfound = true) { + return move(pair_exact, key, value, throw_notfound); + } + move_result to_pair_greater_or_equal(const slice &key, const slice &value, bool throw_notfound = true) { + return move(pair_greater_or_equal, key, value, throw_notfound); + } + move_result to_pair_greater_than(const slice &key, const slice &value, bool throw_notfound = true) { + return move(pair_greater_than, key, value, throw_notfound); + } inline bool seek(const slice &key); - inline bool move(move_operation operation, slice &key, slice &value, - bool throw_notfound); + inline move_result find(const slice &key, bool throw_notfound = true); + inline move_result lower_bound(const slice &key, bool throw_notfound = false); + inline move_result upper_bound(const slice &key, bool throw_notfound = false); /// \brief Return count of duplicates for current key. inline size_t count_multivalue() const; + inline move_result find_multivalue(const slice &key, const slice &value, bool throw_notfound = true); + inline move_result lower_bound_multivalue(const slice &key, const slice &value, bool throw_notfound = false); + inline move_result upper_bound_multivalue(const slice &key, const slice &value, bool throw_notfound = false); + + inline move_result seek_multiple_samelength(const slice &key, bool throw_notfound = true) { + return move(seek_and_batch_samelength, key, throw_notfound); + } + + inline move_result get_multiple_samelength(bool throw_notfound = false) { + return move(batch_samelength, throw_notfound); + } + + inline move_result next_multiple_samelength(bool throw_notfound = false) { + return move(batch_samelength_next, throw_notfound); + } + + inline move_result previous_multiple_samelength(bool throw_notfound = false) { + return move(batch_samelength_previous, throw_notfound); + } + inline bool eof() const; inline bool on_first() const; inline bool on_last() const; + inline bool on_first_multival() const; + inline bool on_last_multival() const; inline estimate_result estimate(const slice &key, const slice &value) const; inline estimate_result estimate(const slice &key) const; inline estimate_result estimate(move_operation operation) const; @@ -4492,13 +4453,14 @@ public: //---------------------------------------------------------------------------- - /// \brief Renew/bind a cursor with a new transaction and previously used - /// key-value map handle. - inline void renew(const ::mdbx::txn &txn); + /// \brief Renew/bind a cursor with a new transaction and previously used key-value map handle. + inline void renew(::mdbx::txn &txn); - /// \brief Bind/renew a cursor with a new transaction and specified key-value - /// map handle. - inline void bind(const ::mdbx::txn &txn, ::mdbx::map_handle map_handle); + /// \brief Bind/renew a cursor with a new transaction and specified key-value map handle. + inline void bind(::mdbx::txn &txn, ::mdbx::map_handle map_handle); + + /// \brief Unbind cursor from a transaction. + inline void unbind(); /// \brief Returns the cursor's transaction. inline ::mdbx::txn txn() const; @@ -4507,8 +4469,8 @@ public: inline operator ::mdbx::txn() const { return txn(); } inline operator ::mdbx::map_handle() const { return map(); } - inline MDBX_error_t put(const slice &key, slice *value, - MDBX_put_flags_t flags) noexcept; + inline MDBX_error_t put(const slice &key, slice *value, MDBX_put_flags_t flags) noexcept; + inline void put(const slice &key, slice value, put_mode mode); inline void insert(const slice &key, slice value); inline value_result try_insert(const slice &key, slice value); inline slice insert_reserve(const slice &key, size_t value_length); @@ -4522,18 +4484,36 @@ public: inline slice update_reserve(const slice &key, size_t value_length); inline value_result try_update_reserve(const slice &key, size_t value_length); - /// \brief Removes single key-value pair or all multi-values at the current - /// cursor position. + void put(const pair &kv, put_mode mode) { return put(kv.key, kv.value, mode); } + void insert(const pair &kv) { return insert(kv.key, kv.value); } + value_result try_insert(const pair &kv) { return try_insert(kv.key, kv.value); } + void upsert(const pair &kv) { return upsert(kv.key, kv.value); } + + /// \brief Removes single key-value pair or all multi-values at the current cursor position. inline bool erase(bool whole_multivalue = false); - /// \brief Seeks and removes first value or whole multi-value of the given - /// key. + /// \brief Seeks and removes first value or whole multi-value of the given key. /// \return `True` if the key is found and a value(s) is removed. inline bool erase(const slice &key, bool whole_multivalue = true); /// \brief Seeks and removes the particular multi-value entry of the key. /// \return `True` if the given key-value pair is found and removed. inline bool erase(const slice &key, const slice &value); + + inline size_t put_multiple_samelength(const slice &key, const size_t value_length, const void *values_array, + size_t values_count, put_mode mode, bool allow_partial = false); + template + size_t put_multiple_samelength(const slice &key, const VALUE *values_array, size_t values_count, put_mode mode, + bool allow_partial = false) { + static_assert(::std::is_standard_layout::value && !::std::is_pointer::value && + !::std::is_array::value, + "Must be a standard layout type!"); + return put_multiple_samelength(key, sizeof(VALUE), values_array, values_count, mode, allow_partial); + } + template + void put_multiple_samelength(const slice &key, const ::std::vector &vector, put_mode mode) { + put_multiple_samelength(key, vector.data(), vector.size(), mode); + } }; /// \brief Managed cursor. @@ -4547,19 +4527,20 @@ class LIBMDBX_API_TYPE cursor_managed : public cursor { using inherited = cursor; friend class txn; /// delegated constructor for RAII - MDBX_CXX11_CONSTEXPR cursor_managed(MDBX_cursor *ptr) noexcept - : inherited(ptr) {} + MDBX_CXX11_CONSTEXPR cursor_managed(MDBX_cursor *ptr) noexcept : inherited(ptr) {} public: /// \brief Creates a new managed cursor with underlying object. - cursor_managed(void *your_context = nullptr) - : cursor_managed(::mdbx_cursor_create(your_context)) { + cursor_managed(void *your_context = nullptr) : cursor_managed(::mdbx_cursor_create(your_context)) { if (MDBX_UNLIKELY(!handle_)) MDBX_CXX20_UNLIKELY error::throw_exception(MDBX_ENOMEM); } /// \brief Explicitly closes the cursor. - void close(); + inline void close() { + error::success_or_throw(::mdbx_cursor_close2(handle_)); + handle_ = nullptr; + } cursor_managed(cursor_managed &&) = default; cursor_managed &operator=(cursor_managed &&other) noexcept { @@ -4572,6 +4553,12 @@ public: return *this; } + inline MDBX_cursor *withdraw_handle() noexcept { + MDBX_cursor *handle = handle_; + handle_ = nullptr; + return handle; + } + cursor_managed(const cursor_managed &) = delete; cursor_managed &operator=(const cursor_managed &) = delete; ~cursor_managed() noexcept { ::mdbx_cursor_close(handle_); } @@ -4583,53 +4570,33 @@ LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, const slice &); LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, const pair &); LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, const pair_result &); template -inline ::std::ostream & -operator<<(::std::ostream &out, const buffer &it) { - return (it.is_freestanding() - ? out << "buf-" << it.headroom() << "." << it.tailroom() - : out << "ref-") - << it.slice(); -} -LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, - const env::geometry::size &); +inline ::std::ostream &operator<<(::std::ostream &out, const buffer &it) { + return (it.is_freestanding() ? out << "buf-" << it.headroom() << "." << it.tailroom() : out << "ref-") << it.slice(); +} +LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, const env::geometry::size &); LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, const env::geometry &); -LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, - const env::operate_parameters &); +LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, const env::operate_parameters &); LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, const env::mode &); -LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, - const env::durability &); -LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, - const env::reclaiming_options &); -LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, - const env::operate_options &); -LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, - const env_managed::create_parameters &); - -LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, - const MDBX_log_level_t &); -LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, - const MDBX_debug_flags_t &); +LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, const env::durability &); +LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, const env::reclaiming_options &); +LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, const env::operate_options &); +LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, const env_managed::create_parameters &); + +LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, const MDBX_log_level_t &); +LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, const MDBX_debug_flags_t &); LIBMDBX_API ::std::ostream &operator<<(::std::ostream &, const error &); -inline ::std::ostream &operator<<(::std::ostream &out, - const MDBX_error_t &errcode) { - return out << error(errcode); -} +inline ::std::ostream &operator<<(::std::ostream &out, const MDBX_error_t &errcode) { return out << error(errcode); } //============================================================================== // // Inline body of the libmdbx C++ API // -MDBX_CXX11_CONSTEXPR const version_info &get_version() noexcept { - return ::mdbx_version; -} -MDBX_CXX11_CONSTEXPR const build_info &get_build() noexcept { - return ::mdbx_build; -} +MDBX_CXX11_CONSTEXPR const version_info &get_version() noexcept { return ::mdbx_version; } +MDBX_CXX11_CONSTEXPR const build_info &get_build() noexcept { return ::mdbx_build; } static MDBX_CXX17_CONSTEXPR size_t strlen(const char *c_str) noexcept { -#if defined(__cpp_lib_is_constant_evaluated) && \ - __cpp_lib_is_constant_evaluated >= 201811L +#if defined(__cpp_lib_is_constant_evaluated) && __cpp_lib_is_constant_evaluated >= 201811L if (::std::is_constant_evaluated()) { for (size_t i = 0; c_str; ++i) if (!c_str[i]) @@ -4644,10 +4611,8 @@ static MDBX_CXX17_CONSTEXPR size_t strlen(const char *c_str) noexcept { #endif } -MDBX_MAYBE_UNUSED static MDBX_CXX20_CONSTEXPR void * -memcpy(void *dest, const void *src, size_t bytes) noexcept { -#if defined(__cpp_lib_is_constant_evaluated) && \ - __cpp_lib_is_constant_evaluated >= 201811L +MDBX_MAYBE_UNUSED static MDBX_CXX20_CONSTEXPR void *memcpy(void *dest, const void *src, size_t bytes) noexcept { +#if defined(__cpp_lib_is_constant_evaluated) && __cpp_lib_is_constant_evaluated >= 201811L if (::std::is_constant_evaluated()) { for (size_t i = 0; i < bytes; ++i) static_cast(dest)[i] = static_cast(src)[i]; @@ -4657,14 +4622,11 @@ memcpy(void *dest, const void *src, size_t bytes) noexcept { return ::std::memcpy(dest, src, bytes); } -static MDBX_CXX20_CONSTEXPR int memcmp(const void *a, const void *b, - size_t bytes) noexcept { -#if defined(__cpp_lib_is_constant_evaluated) && \ - __cpp_lib_is_constant_evaluated >= 201811L +static MDBX_CXX20_CONSTEXPR int memcmp(const void *a, const void *b, size_t bytes) noexcept { +#if defined(__cpp_lib_is_constant_evaluated) && __cpp_lib_is_constant_evaluated >= 201811L if (::std::is_constant_evaluated()) { for (size_t i = 0; i < bytes; ++i) { - const int diff = - static_cast(a)[i] - static_cast(b)[i]; + const int diff = int(static_cast(a)[i]) - int(static_cast(b)[i]); if (diff) return diff; } @@ -4680,13 +4642,11 @@ static MDBX_CXX14_CONSTEXPR size_t check_length(size_t bytes) { return bytes; } -static MDBX_CXX14_CONSTEXPR size_t check_length(size_t headroom, - size_t payload) { +static MDBX_CXX14_CONSTEXPR size_t check_length(size_t headroom, size_t payload) { return check_length(check_length(headroom) + check_length(payload)); } -MDBX_MAYBE_UNUSED static MDBX_CXX14_CONSTEXPR size_t -check_length(size_t headroom, size_t payload, size_t tailroom) { +MDBX_MAYBE_UNUSED static MDBX_CXX14_CONSTEXPR size_t check_length(size_t headroom, size_t payload, size_t tailroom) { return check_length(check_length(headroom, payload) + check_length(tailroom)); } @@ -4704,33 +4664,22 @@ inline void exception_thunk::rethrow_captured() const { //------------------------------------------------------------------------------ -MDBX_CXX11_CONSTEXPR error::error(MDBX_error_t error_code) noexcept - : code_(error_code) {} +MDBX_CXX11_CONSTEXPR error::error(MDBX_error_t error_code) noexcept : code_(error_code) {} inline error &error::operator=(MDBX_error_t error_code) noexcept { code_ = error_code; return *this; } -MDBX_CXX11_CONSTEXPR bool operator==(const error &a, const error &b) noexcept { - return a.code_ == b.code_; -} +MDBX_CXX11_CONSTEXPR bool operator==(const error &a, const error &b) noexcept { return a.code_ == b.code_; } -MDBX_CXX11_CONSTEXPR bool operator!=(const error &a, const error &b) noexcept { - return !(a == b); -} +MDBX_CXX11_CONSTEXPR bool operator!=(const error &a, const error &b) noexcept { return !(a == b); } -MDBX_CXX11_CONSTEXPR bool error::is_success() const noexcept { - return code_ == MDBX_SUCCESS; -} +MDBX_CXX11_CONSTEXPR bool error::is_success() const noexcept { return code_ == MDBX_SUCCESS; } -MDBX_CXX11_CONSTEXPR bool error::is_result_true() const noexcept { - return code_ == MDBX_RESULT_FALSE; -} +MDBX_CXX11_CONSTEXPR bool error::is_result_true() const noexcept { return code_ == MDBX_RESULT_FALSE; } -MDBX_CXX11_CONSTEXPR bool error::is_result_false() const noexcept { - return code_ == MDBX_RESULT_TRUE; -} +MDBX_CXX11_CONSTEXPR bool error::is_result_false() const noexcept { return code_ == MDBX_RESULT_TRUE; } MDBX_CXX11_CONSTEXPR bool error::is_failure() const noexcept { return code_ != MDBX_SUCCESS && code_ != MDBX_RESULT_TRUE; @@ -4739,10 +4688,8 @@ MDBX_CXX11_CONSTEXPR bool error::is_failure() const noexcept { MDBX_CXX11_CONSTEXPR MDBX_error_t error::code() const noexcept { return code_; } MDBX_CXX11_CONSTEXPR bool error::is_mdbx_error() const noexcept { - return (code() >= MDBX_FIRST_LMDB_ERRCODE && - code() <= MDBX_LAST_LMDB_ERRCODE) || - (code() >= MDBX_FIRST_ADDED_ERRCODE && - code() <= MDBX_LAST_ADDED_ERRCODE); + return (code() >= MDBX_FIRST_LMDB_ERRCODE && code() <= MDBX_LAST_LMDB_ERRCODE) || + (code() >= MDBX_FIRST_ADDED_ERRCODE && code() <= MDBX_LAST_ADDED_ERRCODE); } inline void error::throw_exception(int error_code) { @@ -4763,19 +4710,17 @@ inline void error::success_or_throw() const { inline void error::success_or_throw(const exception_thunk &thunk) const { assert(thunk.is_clean() || code() != MDBX_SUCCESS); if (MDBX_UNLIKELY(!is_success())) { - MDBX_CXX20_UNLIKELY if (!thunk.is_clean()) thunk.rethrow_captured(); + MDBX_CXX20_UNLIKELY if (MDBX_UNLIKELY(!thunk.is_clean())) thunk.rethrow_captured(); else throw_exception(); } } -inline void error::panic_on_failure(const char *context_where, - const char *func_who) const noexcept { +inline void error::panic_on_failure(const char *context_where, const char *func_who) const noexcept { if (MDBX_UNLIKELY(is_failure())) MDBX_CXX20_UNLIKELY panic(context_where, func_who); } -inline void error::success_or_panic(const char *context_where, - const char *func_who) const noexcept { +inline void error::success_or_panic(const char *context_where, const char *func_who) const noexcept { if (MDBX_UNLIKELY(!is_success())) MDBX_CXX20_UNLIKELY panic(context_where, func_who); } @@ -4806,24 +4751,27 @@ inline bool error::boolean_or_throw(int error_code) { } } -inline void error::success_or_throw(int error_code, - const exception_thunk &thunk) { +inline void error::success_or_throw(int error_code, const exception_thunk &thunk) { error rc(static_cast(error_code)); rc.success_or_throw(thunk); } -inline void error::panic_on_failure(int error_code, const char *context_where, - const char *func_who) noexcept { +inline void error::panic_on_failure(int error_code, const char *context_where, const char *func_who) noexcept { error rc(static_cast(error_code)); rc.panic_on_failure(context_where, func_who); } -inline void error::success_or_panic(int error_code, const char *context_where, - const char *func_who) noexcept { +inline void error::success_or_panic(int error_code, const char *context_where, const char *func_who) noexcept { error rc(static_cast(error_code)); rc.success_or_panic(context_where, func_who); } +inline bool error::boolean_or_throw(int error_code, const exception_thunk &thunk) { + if (MDBX_UNLIKELY(!thunk.is_clean())) + MDBX_CXX20_UNLIKELY thunk.rethrow_captured(); + return boolean_or_throw(error_code); +} + //------------------------------------------------------------------------------ MDBX_CXX11_CONSTEXPR slice::slice() noexcept : ::MDBX_val({nullptr, 0}) {} @@ -4832,22 +4780,15 @@ MDBX_CXX14_CONSTEXPR slice::slice(const void *ptr, size_t bytes) : ::MDBX_val({const_cast(ptr), check_length(bytes)}) {} MDBX_CXX14_CONSTEXPR slice::slice(const void *begin, const void *end) - : slice(begin, static_cast(end) - - static_cast(begin)) {} + : slice(begin, static_cast(end) - static_cast(begin)) {} -MDBX_CXX17_CONSTEXPR slice::slice(const char *c_str) - : slice(c_str, ::mdbx::strlen(c_str)) {} +MDBX_CXX17_CONSTEXPR slice::slice(const char *c_str) : slice(c_str, ::mdbx::strlen(c_str)) {} -MDBX_CXX14_CONSTEXPR slice::slice(const MDBX_val &src) - : slice(src.iov_base, src.iov_len) {} +MDBX_CXX14_CONSTEXPR slice::slice(const MDBX_val &src) : slice(src.iov_base, src.iov_len) {} -MDBX_CXX14_CONSTEXPR slice::slice(MDBX_val &&src) : slice(src) { - src.iov_base = nullptr; -} +MDBX_CXX14_CONSTEXPR slice::slice(MDBX_val &&src) : slice(src) { src.iov_base = nullptr; } -MDBX_CXX14_CONSTEXPR slice::slice(slice &&src) noexcept : slice(src) { - src.invalidate(); -} +MDBX_CXX14_CONSTEXPR slice::slice(slice &&src) noexcept : slice(src) { src.invalidate(); } inline slice &slice::assign(const void *ptr, size_t bytes) { iov_base = const_cast(ptr); @@ -4861,9 +4802,7 @@ inline slice &slice::assign(const slice &src) noexcept { return *this; } -inline slice &slice::assign(const ::MDBX_val &src) { - return assign(src.iov_base, src.iov_len); -} +inline slice &slice::assign(const ::MDBX_val &src) { return assign(src.iov_base, src.iov_len); } slice &slice::assign(slice &&src) noexcept { assign(src); @@ -4878,21 +4817,14 @@ inline slice &slice::assign(::MDBX_val &&src) { } inline slice &slice::assign(const void *begin, const void *end) { - return assign(begin, static_cast(end) - - static_cast(begin)); + return assign(begin, static_cast(end) - static_cast(begin)); } -inline slice &slice::assign(const char *c_str) { - return assign(c_str, ::mdbx::strlen(c_str)); -} +inline slice &slice::assign(const char *c_str) { return assign(c_str, ::mdbx::strlen(c_str)); } -inline slice &slice::operator=(slice &&src) noexcept { - return assign(::std::move(src)); -} +inline slice &slice::operator=(slice &&src) noexcept { return assign(::std::move(src)); } -inline slice &slice::operator=(::MDBX_val &&src) { - return assign(::std::move(src)); -} +inline slice &slice::operator=(::MDBX_val &&src) { return assign(::std::move(src)); } inline void slice::swap(slice &other) noexcept { const auto temp = *this; @@ -4904,47 +4836,27 @@ MDBX_CXX11_CONSTEXPR const ::mdbx::byte *slice::byte_ptr() const noexcept { return static_cast(iov_base); } -MDBX_CXX11_CONSTEXPR const ::mdbx::byte *slice::end_byte_ptr() const noexcept { - return byte_ptr() + length(); -} +MDBX_CXX11_CONSTEXPR const ::mdbx::byte *slice::end_byte_ptr() const noexcept { return byte_ptr() + length(); } -MDBX_CXX11_CONSTEXPR ::mdbx::byte *slice::byte_ptr() noexcept { - return static_cast(iov_base); -} +MDBX_CXX11_CONSTEXPR ::mdbx::byte *slice::byte_ptr() noexcept { return static_cast(iov_base); } -MDBX_CXX11_CONSTEXPR ::mdbx::byte *slice::end_byte_ptr() noexcept { - return byte_ptr() + length(); -} +MDBX_CXX11_CONSTEXPR ::mdbx::byte *slice::end_byte_ptr() noexcept { return byte_ptr() + length(); } -MDBX_CXX11_CONSTEXPR const char *slice::char_ptr() const noexcept { - return static_cast(iov_base); -} +MDBX_CXX11_CONSTEXPR const char *slice::char_ptr() const noexcept { return static_cast(iov_base); } -MDBX_CXX11_CONSTEXPR const char *slice::end_char_ptr() const noexcept { - return char_ptr() + length(); -} +MDBX_CXX11_CONSTEXPR const char *slice::end_char_ptr() const noexcept { return char_ptr() + length(); } -MDBX_CXX11_CONSTEXPR char *slice::char_ptr() noexcept { - return static_cast(iov_base); -} +MDBX_CXX11_CONSTEXPR char *slice::char_ptr() noexcept { return static_cast(iov_base); } -MDBX_CXX11_CONSTEXPR char *slice::end_char_ptr() noexcept { - return char_ptr() + length(); -} +MDBX_CXX11_CONSTEXPR char *slice::end_char_ptr() noexcept { return char_ptr() + length(); } -MDBX_CXX11_CONSTEXPR const void *slice::data() const noexcept { - return iov_base; -} +MDBX_CXX11_CONSTEXPR const void *slice::data() const noexcept { return iov_base; } -MDBX_CXX11_CONSTEXPR const void *slice::end() const noexcept { - return static_cast(end_byte_ptr()); -} +MDBX_CXX11_CONSTEXPR const void *slice::end() const noexcept { return static_cast(end_byte_ptr()); } MDBX_CXX11_CONSTEXPR void *slice::data() noexcept { return iov_base; } -MDBX_CXX11_CONSTEXPR void *slice::end() noexcept { - return static_cast(end_byte_ptr()); -} +MDBX_CXX11_CONSTEXPR void *slice::end() noexcept { return static_cast(end_byte_ptr()); } MDBX_CXX11_CONSTEXPR size_t slice::length() const noexcept { return iov_len; } @@ -4958,19 +4870,13 @@ MDBX_CXX14_CONSTEXPR slice &slice::set_end(const void *ptr) { return set_length(static_cast(ptr) - char_ptr()); } -MDBX_CXX11_CONSTEXPR bool slice::empty() const noexcept { - return length() == 0; -} +MDBX_CXX11_CONSTEXPR bool slice::empty() const noexcept { return length() == 0; } -MDBX_CXX11_CONSTEXPR bool slice::is_null() const noexcept { - return data() == nullptr; -} +MDBX_CXX11_CONSTEXPR bool slice::is_null() const noexcept { return data() == nullptr; } MDBX_CXX11_CONSTEXPR size_t slice::size() const noexcept { return length(); } -MDBX_CXX11_CONSTEXPR slice::operator bool() const noexcept { - return !is_null(); -} +MDBX_CXX11_CONSTEXPR slice::operator bool() const noexcept { return !is_null(); } MDBX_CXX14_CONSTEXPR void slice::invalidate() noexcept { iov_base = nullptr; } @@ -5002,20 +4908,16 @@ inline void slice::safe_remove_suffix(size_t n) { remove_suffix(n); } -MDBX_CXX14_CONSTEXPR bool -slice::starts_with(const slice &prefix) const noexcept { - return length() >= prefix.length() && - memcmp(data(), prefix.data(), prefix.length()) == 0; +MDBX_CXX14_CONSTEXPR bool slice::starts_with(const slice &prefix) const noexcept { + return length() >= prefix.length() && memcmp(data(), prefix.data(), prefix.length()) == 0; } MDBX_CXX14_CONSTEXPR bool slice::ends_with(const slice &suffix) const noexcept { return length() >= suffix.length() && - memcmp(byte_ptr() + length() - suffix.length(), suffix.data(), - suffix.length()) == 0; + memcmp(byte_ptr() + length() - suffix.length(), suffix.data(), suffix.length()) == 0; } -MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR size_t -slice::hash_value() const noexcept { +MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR size_t slice::hash_value() const noexcept { size_t h = length() * 3977471; for (size_t i = 0; i < length(); ++i) h = (h ^ static_cast(data())[i]) * 1664525 + 1013904223; @@ -5068,17 +4970,14 @@ MDBX_CXX14_CONSTEXPR slice slice::safe_middle(size_t from, size_t n) const { return middle(from, n); } -MDBX_CXX14_CONSTEXPR intptr_t slice::compare_fast(const slice &a, - const slice &b) noexcept { +MDBX_CXX14_CONSTEXPR intptr_t slice::compare_fast(const slice &a, const slice &b) noexcept { const intptr_t diff = intptr_t(a.length()) - intptr_t(b.length()); - return diff ? diff - : MDBX_UNLIKELY(a.length() == 0 || a.data() == b.data()) - ? 0 - : memcmp(a.data(), b.data(), a.length()); + return diff ? diff + : MDBX_UNLIKELY(a.length() == 0 || a.data() == b.data()) ? 0 + : memcmp(a.data(), b.data(), a.length()); } -MDBX_CXX14_CONSTEXPR intptr_t -slice::compare_lexicographically(const slice &a, const slice &b) noexcept { +MDBX_CXX14_CONSTEXPR intptr_t slice::compare_lexicographically(const slice &a, const slice &b) noexcept { const size_t shortest = ::std::min(a.length(), b.length()); if (MDBX_LIKELY(shortest > 0)) MDBX_CXX20_LIKELY { @@ -5089,139 +4988,146 @@ slice::compare_lexicographically(const slice &a, const slice &b) noexcept { return intptr_t(a.length()) - intptr_t(b.length()); } -MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR bool -operator==(const slice &a, const slice &b) noexcept { +MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR bool operator==(const slice &a, const slice &b) noexcept { return slice::compare_fast(a, b) == 0; } -MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR bool -operator<(const slice &a, const slice &b) noexcept { +MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR bool operator<(const slice &a, const slice &b) noexcept { return slice::compare_lexicographically(a, b) < 0; } -MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR bool -operator>(const slice &a, const slice &b) noexcept { +MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR bool operator>(const slice &a, const slice &b) noexcept { return slice::compare_lexicographically(a, b) > 0; } -MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR bool -operator<=(const slice &a, const slice &b) noexcept { +MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR bool operator<=(const slice &a, const slice &b) noexcept { return slice::compare_lexicographically(a, b) <= 0; } -MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR bool -operator>=(const slice &a, const slice &b) noexcept { +MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR bool operator>=(const slice &a, const slice &b) noexcept { return slice::compare_lexicographically(a, b) >= 0; } -MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR bool -operator!=(const slice &a, const slice &b) noexcept { +MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR bool operator!=(const slice &a, const slice &b) noexcept { return slice::compare_fast(a, b) != 0; } template -inline string -slice::as_hex_string(bool uppercase, unsigned wrap_width, - const ALLOCATOR &allocator) const { +inline string slice::as_hex_string(bool uppercase, unsigned wrap_width, const ALLOCATOR &allocator) const { return to_hex(*this, uppercase, wrap_width).as_string(allocator); } template -inline string -slice::as_base58_string(unsigned wrap_width, const ALLOCATOR &allocator) const { +inline string slice::as_base58_string(unsigned wrap_width, const ALLOCATOR &allocator) const { return to_base58(*this, wrap_width).as_string(allocator); } template -inline string -slice::as_base64_string(unsigned wrap_width, const ALLOCATOR &allocator) const { +inline string slice::as_base64_string(unsigned wrap_width, const ALLOCATOR &allocator) const { return to_base64(*this, wrap_width).as_string(allocator); } template -inline buffer -slice::encode_hex(bool uppercase, unsigned wrap_width, - const ALLOCATOR &allocator) const { - return to_hex(*this, uppercase, wrap_width) - .as_buffer(allocator); +inline buffer slice::encode_hex(bool uppercase, unsigned wrap_width, + const ALLOCATOR &allocator) const { + return to_hex(*this, uppercase, wrap_width).as_buffer(allocator); } template -inline buffer -slice::encode_base58(unsigned wrap_width, const ALLOCATOR &allocator) const { - return to_base58(*this, wrap_width) - .as_buffer(allocator); +inline buffer slice::encode_base58(unsigned wrap_width, const ALLOCATOR &allocator) const { + return to_base58(*this, wrap_width).as_buffer(allocator); } template -inline buffer -slice::encode_base64(unsigned wrap_width, const ALLOCATOR &allocator) const { - return to_base64(*this, wrap_width) - .as_buffer(allocator); +inline buffer slice::encode_base64(unsigned wrap_width, const ALLOCATOR &allocator) const { + return to_base64(*this, wrap_width).as_buffer(allocator); } template -inline buffer -slice::hex_decode(bool ignore_spaces, const ALLOCATOR &allocator) const { - return from_hex(*this, ignore_spaces) - .as_buffer(allocator); +inline buffer slice::hex_decode(bool ignore_spaces, const ALLOCATOR &allocator) const { + return from_hex(*this, ignore_spaces).as_buffer(allocator); } template -inline buffer -slice::base58_decode(bool ignore_spaces, const ALLOCATOR &allocator) const { - return from_base58(*this, ignore_spaces) - .as_buffer(allocator); +inline buffer slice::base58_decode(bool ignore_spaces, const ALLOCATOR &allocator) const { + return from_base58(*this, ignore_spaces).as_buffer(allocator); } template -inline buffer -slice::base64_decode(bool ignore_spaces, const ALLOCATOR &allocator) const { - return from_base64(*this, ignore_spaces) - .as_buffer(allocator); +inline buffer slice::base64_decode(bool ignore_spaces, const ALLOCATOR &allocator) const { + return from_base64(*this, ignore_spaces).as_buffer(allocator); } -inline MDBX_NOTHROW_PURE_FUNCTION bool -slice::is_hex(bool ignore_spaces) const noexcept { +MDBX_NOTHROW_PURE_FUNCTION inline bool slice::is_hex(bool ignore_spaces) const noexcept { return !from_hex(*this, ignore_spaces).is_erroneous(); } -inline MDBX_NOTHROW_PURE_FUNCTION bool -slice::is_base58(bool ignore_spaces) const noexcept { +MDBX_NOTHROW_PURE_FUNCTION inline bool slice::is_base58(bool ignore_spaces) const noexcept { return !from_base58(*this, ignore_spaces).is_erroneous(); } -inline MDBX_NOTHROW_PURE_FUNCTION bool -slice::is_base64(bool ignore_spaces) const noexcept { +MDBX_NOTHROW_PURE_FUNCTION inline bool slice::is_base64(bool ignore_spaces) const noexcept { return !from_base64(*this, ignore_spaces).is_erroneous(); } //------------------------------------------------------------------------------ +MDBX_CXX14_CONSTEXPR intptr_t pair::compare_fast(const pair &a, const pair &b) noexcept { + const auto diff = slice::compare_fast(a.key, b.key); + return diff ? diff : slice::compare_fast(a.value, b.value); +} + +MDBX_CXX14_CONSTEXPR intptr_t pair::compare_lexicographically(const pair &a, const pair &b) noexcept { + const auto diff = slice::compare_lexicographically(a.key, b.key); + return diff ? diff : slice::compare_lexicographically(a.value, b.value); +} + +MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR bool operator==(const pair &a, const pair &b) noexcept { + return a.key.length() == b.key.length() && a.value.length() == b.value.length() && + memcmp(a.key.data(), b.key.data(), a.key.length()) == 0 && + memcmp(a.value.data(), b.value.data(), a.value.length()) == 0; +} + +MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR bool operator<(const pair &a, const pair &b) noexcept { + return pair::compare_lexicographically(a, b) < 0; +} + +MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR bool operator>(const pair &a, const pair &b) noexcept { + return pair::compare_lexicographically(a, b) > 0; +} + +MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR bool operator<=(const pair &a, const pair &b) noexcept { + return pair::compare_lexicographically(a, b) <= 0; +} + +MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR bool operator>=(const pair &a, const pair &b) noexcept { + return pair::compare_lexicographically(a, b) >= 0; +} + +MDBX_NOTHROW_PURE_FUNCTION MDBX_CXX14_CONSTEXPR bool operator!=(const pair &a, const pair &b) noexcept { + return a.key.length() != b.key.length() || a.value.length() != b.value.length() || + memcmp(a.key.data(), b.key.data(), a.key.length()) != 0 || + memcmp(a.value.data(), b.value.data(), a.value.length()) != 0; +} + +//------------------------------------------------------------------------------ + template -inline buffer::buffer( - const txn &txn, const struct slice &src, const allocator_type &allocator) +inline buffer::buffer(const txn &txn, const struct slice &src, + const allocator_type &allocator) : buffer(src, !txn.is_dirty(src.data()), allocator) {} //------------------------------------------------------------------------------ -MDBX_CXX11_CONSTEXPR map_handle::info::info(map_handle::flags flags, - map_handle::state state) noexcept +MDBX_CXX11_CONSTEXPR map_handle::info::info(map_handle::flags flags, map_handle::state state) noexcept : flags(flags), state(state) {} -#if CONSTEXPR_ENUM_FLAGS_OPERATIONS -MDBX_CXX11_CONSTEXPR -#endif -::mdbx::key_mode map_handle::info::key_mode() const noexcept { +MDBX_CXX11_CONSTEXPR_ENUM mdbx::key_mode map_handle::info::key_mode() const noexcept { return ::mdbx::key_mode(flags & (MDBX_REVERSEKEY | MDBX_INTEGERKEY)); } -#if CONSTEXPR_ENUM_FLAGS_OPERATIONS -MDBX_CXX11_CONSTEXPR -#endif -::mdbx::value_mode map_handle::info::value_mode() const noexcept { - return ::mdbx::value_mode(flags & (MDBX_DUPSORT | MDBX_REVERSEDUP | - MDBX_DUPFIXED | MDBX_INTEGERDUP)); +MDBX_CXX11_CONSTEXPR_ENUM mdbx::value_mode map_handle::info::value_mode() const noexcept { + return ::mdbx::value_mode(flags & (MDBX_DUPSORT | MDBX_REVERSEDUP | MDBX_DUPFIXED | MDBX_INTEGERDUP)); } //------------------------------------------------------------------------------ @@ -5234,9 +5140,7 @@ inline env &env::operator=(env &&other) noexcept { return *this; } -inline env::env(env &&other) noexcept : handle_(other.handle_) { - other.handle_ = nullptr; -} +inline env::env(env &&other) noexcept : handle_(other.handle_) { other.handle_ = nullptr; } inline env::~env() noexcept { #ifndef NDEBUG @@ -5244,21 +5148,15 @@ inline env::~env() noexcept { #endif } -MDBX_CXX14_CONSTEXPR env::operator bool() const noexcept { - return handle_ != nullptr; -} +MDBX_CXX14_CONSTEXPR env::operator bool() const noexcept { return handle_ != nullptr; } MDBX_CXX14_CONSTEXPR env::operator const MDBX_env *() const { return handle_; } MDBX_CXX14_CONSTEXPR env::operator MDBX_env *() { return handle_; } -MDBX_CXX11_CONSTEXPR bool operator==(const env &a, const env &b) noexcept { - return a.handle_ == b.handle_; -} +MDBX_CXX11_CONSTEXPR bool operator==(const env &a, const env &b) noexcept { return a.handle_ == b.handle_; } -MDBX_CXX11_CONSTEXPR bool operator!=(const env &a, const env &b) noexcept { - return a.handle_ != b.handle_; -} +MDBX_CXX11_CONSTEXPR bool operator!=(const env &a, const env &b) noexcept { return a.handle_ != b.handle_; } inline env::geometry &env::geometry::make_fixed(intptr_t size) noexcept { size_lower = size_now = size_upper = size; @@ -5266,21 +5164,18 @@ inline env::geometry &env::geometry::make_fixed(intptr_t size) noexcept { return *this; } -inline env::geometry &env::geometry::make_dynamic(intptr_t lower, - intptr_t upper) noexcept { +inline env::geometry &env::geometry::make_dynamic(intptr_t lower, intptr_t upper) noexcept { size_now = size_lower = lower; size_upper = upper; growth_step = shrink_threshold = default_value; return *this; } -inline env::reclaiming_options env::operate_parameters::reclaiming_from_flags( - MDBX_env_flags_t flags) noexcept { +inline env::reclaiming_options env::operate_parameters::reclaiming_from_flags(MDBX_env_flags_t flags) noexcept { return reclaiming_options(flags); } -inline env::operate_options -env::operate_parameters::options_from_flags(MDBX_env_flags_t flags) noexcept { +inline env::operate_options env::operate_parameters::options_from_flags(MDBX_env_flags_t flags) noexcept { return operate_options(flags); } @@ -5302,13 +5197,9 @@ inline size_t env::limits::dbsize_max(intptr_t pagesize) { return static_cast(result); } -inline size_t env::limits::key_min(MDBX_db_flags_t flags) noexcept { - return (flags & MDBX_INTEGERKEY) ? 4 : 0; -} +inline size_t env::limits::key_min(MDBX_db_flags_t flags) noexcept { return (flags & MDBX_INTEGERKEY) ? 4 : 0; } -inline size_t env::limits::key_min(key_mode mode) noexcept { - return key_min(MDBX_db_flags_t(mode)); -} +inline size_t env::limits::key_min(key_mode mode) noexcept { return key_min(MDBX_db_flags_t(mode)); } inline size_t env::limits::key_max(intptr_t pagesize, MDBX_db_flags_t flags) { const intptr_t result = mdbx_limits_keysize_max(pagesize, flags); @@ -5328,17 +5219,11 @@ inline size_t env::limits::key_max(const env &env, MDBX_db_flags_t flags) { return static_cast(result); } -inline size_t env::limits::key_max(const env &env, key_mode mode) { - return key_max(env, MDBX_db_flags_t(mode)); -} +inline size_t env::limits::key_max(const env &env, key_mode mode) { return key_max(env, MDBX_db_flags_t(mode)); } -inline size_t env::limits::value_min(MDBX_db_flags_t flags) noexcept { - return (flags & MDBX_INTEGERDUP) ? 4 : 0; -} +inline size_t env::limits::value_min(MDBX_db_flags_t flags) noexcept { return (flags & MDBX_INTEGERDUP) ? 4 : 0; } -inline size_t env::limits::value_min(value_mode mode) noexcept { - return value_min(MDBX_db_flags_t(mode)); -} +inline size_t env::limits::value_min(value_mode mode) noexcept { return value_min(MDBX_db_flags_t(mode)); } inline size_t env::limits::value_max(intptr_t pagesize, MDBX_db_flags_t flags) { const intptr_t result = mdbx_limits_valsize_max(pagesize, flags); @@ -5358,25 +5243,20 @@ inline size_t env::limits::value_max(const env &env, MDBX_db_flags_t flags) { return static_cast(result); } -inline size_t env::limits::value_max(const env &env, value_mode mode) { - return value_max(env, MDBX_db_flags_t(mode)); -} +inline size_t env::limits::value_max(const env &env, value_mode mode) { return value_max(env, MDBX_db_flags_t(mode)); } -inline size_t env::limits::pairsize4page_max(intptr_t pagesize, - MDBX_db_flags_t flags) { +inline size_t env::limits::pairsize4page_max(intptr_t pagesize, MDBX_db_flags_t flags) { const intptr_t result = mdbx_limits_pairsize4page_max(pagesize, flags); if (result < 0) MDBX_CXX20_UNLIKELY error::throw_exception(MDBX_EINVAL); return static_cast(result); } -inline size_t env::limits::pairsize4page_max(intptr_t pagesize, - value_mode mode) { +inline size_t env::limits::pairsize4page_max(intptr_t pagesize, value_mode mode) { return pairsize4page_max(pagesize, MDBX_db_flags_t(mode)); } -inline size_t env::limits::pairsize4page_max(const env &env, - MDBX_db_flags_t flags) { +inline size_t env::limits::pairsize4page_max(const env &env, MDBX_db_flags_t flags) { const intptr_t result = mdbx_env_get_pairsize4page_max(env, flags); if (result < 0) MDBX_CXX20_UNLIKELY error::throw_exception(MDBX_EINVAL); @@ -5387,21 +5267,18 @@ inline size_t env::limits::pairsize4page_max(const env &env, value_mode mode) { return pairsize4page_max(env, MDBX_db_flags_t(mode)); } -inline size_t env::limits::valsize4page_max(intptr_t pagesize, - MDBX_db_flags_t flags) { +inline size_t env::limits::valsize4page_max(intptr_t pagesize, MDBX_db_flags_t flags) { const intptr_t result = mdbx_limits_valsize4page_max(pagesize, flags); if (result < 0) MDBX_CXX20_UNLIKELY error::throw_exception(MDBX_EINVAL); return static_cast(result); } -inline size_t env::limits::valsize4page_max(intptr_t pagesize, - value_mode mode) { +inline size_t env::limits::valsize4page_max(intptr_t pagesize, value_mode mode) { return valsize4page_max(pagesize, MDBX_db_flags_t(mode)); } -inline size_t env::limits::valsize4page_max(const env &env, - MDBX_db_flags_t flags) { +inline size_t env::limits::valsize4page_max(const env &env, MDBX_db_flags_t flags) { const intptr_t result = mdbx_env_get_valsize4page_max(env, flags); if (result < 0) MDBX_CXX20_UNLIKELY error::throw_exception(MDBX_EINVAL); @@ -5419,18 +5296,17 @@ inline size_t env::limits::transaction_size_max(intptr_t pagesize) { return static_cast(result); } +inline size_t env::limits::max_map_handles(void) { return MDBX_MAX_DBI; } + inline env::operate_parameters env::get_operation_parameters() const { const auto flags = get_flags(); - return operate_parameters(max_maps(), max_readers(), - operate_parameters::mode_from_flags(flags), + return operate_parameters(max_maps(), max_readers(), operate_parameters::mode_from_flags(flags), operate_parameters::durability_from_flags(flags), operate_parameters::reclaiming_from_flags(flags), operate_parameters::options_from_flags(flags)); } -inline env::mode env::get_mode() const { - return operate_parameters::mode_from_flags(get_flags()); -} +inline env::mode env::get_mode() const { return operate_parameters::mode_from_flags(get_flags()); } inline env::durability env::get_durability() const { return env::operate_parameters::durability_from_flags(get_flags()); @@ -5492,9 +5368,7 @@ inline unsigned env::max_maps() const { return r; } -inline void *env::get_context() const noexcept { - return mdbx_env_get_userctx(handle_); -} +inline void *env::get_context() const noexcept { return mdbx_env_get_userctx(handle_); } inline env &env::set_context(void *ptr) { error::success_or_throw(::mdbx_env_set_userctx(handle_, ptr)); @@ -5527,31 +5401,22 @@ inline env &env::set_sync_period__seconds_double(double seconds) { return set_sync_period__seconds_16dot16(unsigned(seconds * 65536)); } -inline double env::sync_period__seconds_double() const { - return sync_period__seconds_16dot16() / 65536.0; -} +inline double env::sync_period__seconds_double() const { return sync_period__seconds_16dot16() / 65536.0; } #if __cplusplus >= 201103L -inline env &env::set_sync_period(const duration &period) { - return set_sync_period__seconds_16dot16(period.count()); -} +inline env &env::set_sync_period(const duration &period) { return set_sync_period__seconds_16dot16(period.count()); } -inline duration env::sync_period() const { - return duration(sync_period__seconds_16dot16()); -} +inline duration env::sync_period() const { return duration(sync_period__seconds_16dot16()); } #endif -inline env &env::set_extra_option(enum env::extra_runtime_option option, - uint64_t value) { - error::success_or_throw( - ::mdbx_env_set_option(handle_, ::MDBX_option_t(option), value)); +inline env &env::set_extra_option(enum env::extra_runtime_option option, uint64_t value) { + error::success_or_throw(::mdbx_env_set_option(handle_, ::MDBX_option_t(option), value)); return *this; } inline uint64_t env::extra_option(enum env::extra_runtime_option option) const { uint64_t value; - error::success_or_throw( - ::mdbx_env_get_option(handle_, ::MDBX_option_t(option), &value)); + error::success_or_throw(::mdbx_env_get_option(handle_, ::MDBX_option_t(option), &value)); return value; } @@ -5561,9 +5426,8 @@ inline env &env::alter_flags(MDBX_env_flags_t flags, bool on_off) { } inline env &env::set_geometry(const geometry &geo) { - error::success_or_throw(::mdbx_env_set_geometry( - handle_, geo.size_lower, geo.size_now, geo.size_upper, geo.growth_step, - geo.shrink_threshold, geo.pagesize)); + error::success_or_throw(::mdbx_env_set_geometry(handle_, geo.size_lower, geo.size_now, geo.size_upper, + geo.growth_step, geo.shrink_threshold, geo.pagesize)); return *this; } @@ -5580,24 +5444,19 @@ inline bool env::sync_to_disk(bool force, bool nonblock) { } } -inline void env::close_map(const map_handle &handle) { - error::success_or_throw(::mdbx_dbi_close(*this, handle.dbi)); -} +inline void env::close_map(const map_handle &handle) { error::success_or_throw(::mdbx_dbi_close(*this, handle.dbi)); } MDBX_CXX11_CONSTEXPR -env::reader_info::reader_info(int slot, mdbx_pid_t pid, mdbx_tid_t thread, - uint64_t txnid, uint64_t lag, size_t used, +env::reader_info::reader_info(int slot, mdbx_pid_t pid, mdbx_tid_t thread, uint64_t txnid, uint64_t lag, size_t used, size_t retained) noexcept - : slot(slot), pid(pid), thread(thread), transaction_id(txnid), - transaction_lag(lag), bytes_used(used), bytes_retained(retained) {} + : slot(slot), pid(pid), thread(thread), transaction_id(txnid), transaction_lag(lag), bytes_used(used), + bytes_retained(retained) {} -template -inline int env::enumerate_readers(VISITOR &visitor) { +template inline int env::enumerate_readers(VISITOR &visitor) { struct reader_visitor_thunk : public exception_thunk { VISITOR &visitor_; - static int cb(void *ctx, int number, int slot, mdbx_pid_t pid, - mdbx_tid_t thread, uint64_t txnid, uint64_t lag, size_t used, - size_t retained) noexcept { + static int cb(void *ctx, int number, int slot, mdbx_pid_t pid, mdbx_tid_t thread, uint64_t txnid, uint64_t lag, + size_t used, size_t retained) noexcept { reader_visitor_thunk *thunk = static_cast(ctx); assert(thunk->is_clean()); try { @@ -5608,8 +5467,7 @@ inline int env::enumerate_readers(VISITOR &visitor) { return loop_control::exit_loop; } } - MDBX_CXX11_CONSTEXPR reader_visitor_thunk(VISITOR &visitor) noexcept - : visitor_(visitor) {} + MDBX_CXX11_CONSTEXPR reader_visitor_thunk(VISITOR &visitor) noexcept : visitor_(visitor) {} }; reader_visitor_thunk thunk(visitor); const auto rc = ::mdbx_reader_list(*this, thunk.cb, &thunk); @@ -5629,30 +5487,32 @@ inline env &env::set_HandleSlowReaders(MDBX_hsr_func *cb) { return *this; } -inline MDBX_hsr_func *env::get_HandleSlowReaders() const noexcept { - return ::mdbx_env_get_hsr(handle_); -} +inline MDBX_hsr_func *env::get_HandleSlowReaders() const noexcept { return ::mdbx_env_get_hsr(handle_); } inline txn_managed env::start_read() const { ::MDBX_txn *ptr; - error::success_or_throw( - ::mdbx_txn_begin(handle_, nullptr, MDBX_TXN_RDONLY, &ptr)); + error::success_or_throw(::mdbx_txn_begin(handle_, nullptr, MDBX_TXN_RDONLY, &ptr)); assert(ptr != nullptr); return txn_managed(ptr); } inline txn_managed env::prepare_read() const { ::MDBX_txn *ptr; - error::success_or_throw( - ::mdbx_txn_begin(handle_, nullptr, MDBX_TXN_RDONLY_PREPARE, &ptr)); + error::success_or_throw(::mdbx_txn_begin(handle_, nullptr, MDBX_TXN_RDONLY_PREPARE, &ptr)); assert(ptr != nullptr); return txn_managed(ptr); } inline txn_managed env::start_write(bool dont_wait) { ::MDBX_txn *ptr; - error::success_or_throw(::mdbx_txn_begin( - handle_, nullptr, dont_wait ? MDBX_TXN_TRY : MDBX_TXN_READWRITE, &ptr)); + error::success_or_throw(::mdbx_txn_begin(handle_, nullptr, dont_wait ? MDBX_TXN_TRY : MDBX_TXN_READWRITE, &ptr)); + assert(ptr != nullptr); + return txn_managed(ptr); +} + +inline txn_managed env::start_write(txn &parent) { + ::MDBX_txn *ptr; + error::success_or_throw(::mdbx_txn_begin(handle_, parent, MDBX_TXN_READWRITE, &ptr)); assert(ptr != nullptr); return txn_managed(ptr); } @@ -5669,9 +5529,7 @@ inline txn &txn::operator=(txn &&other) noexcept { return *this; } -inline txn::txn(txn &&other) noexcept : handle_(other.handle_) { - other.handle_ = nullptr; -} +inline txn::txn(txn &&other) noexcept : handle_(other.handle_) { other.handle_ = nullptr; } inline txn::~txn() noexcept { #ifndef NDEBUG @@ -5679,25 +5537,17 @@ inline txn::~txn() noexcept { #endif } -MDBX_CXX14_CONSTEXPR txn::operator bool() const noexcept { - return handle_ != nullptr; -} +MDBX_CXX14_CONSTEXPR txn::operator bool() const noexcept { return handle_ != nullptr; } MDBX_CXX14_CONSTEXPR txn::operator const MDBX_txn *() const { return handle_; } MDBX_CXX14_CONSTEXPR txn::operator MDBX_txn *() { return handle_; } -MDBX_CXX11_CONSTEXPR bool operator==(const txn &a, const txn &b) noexcept { - return a.handle_ == b.handle_; -} +MDBX_CXX11_CONSTEXPR bool operator==(const txn &a, const txn &b) noexcept { return a.handle_ == b.handle_; } -MDBX_CXX11_CONSTEXPR bool operator!=(const txn &a, const txn &b) noexcept { - return a.handle_ != b.handle_; -} +MDBX_CXX11_CONSTEXPR bool operator!=(const txn &a, const txn &b) noexcept { return a.handle_ != b.handle_; } -inline void *txn::get_context() const noexcept { - return mdbx_txn_get_userctx(handle_); -} +inline void *txn::get_context() const noexcept { return mdbx_txn_get_userctx(handle_); } inline txn &txn::set_context(void *ptr) { error::success_or_throw(::mdbx_txn_set_userctx(handle_, ptr)); @@ -5730,12 +5580,16 @@ inline uint64_t txn::id() const { return txnid; } -inline void txn::reset_reading() { - error::success_or_throw(::mdbx_txn_reset(handle_)); -} +inline void txn::reset_reading() { error::success_or_throw(::mdbx_txn_reset(handle_)); } + +inline void txn::make_broken() { error::success_or_throw(::mdbx_txn_break(handle_)); } + +inline void txn::renew_reading() { error::success_or_throw(::mdbx_txn_renew(handle_)); } + +inline void txn::park_reading(bool autounpark) { error::success_or_throw(::mdbx_txn_park(handle_, autounpark)); } -inline void txn::renew_reading() { - error::success_or_throw(::mdbx_txn_renew(handle_)); +inline bool txn::unpark_reading(bool restart_if_ousted) { + return error::boolean_or_throw(::mdbx_txn_unpark(handle_, restart_if_ousted)); } inline txn::info txn::get_info(bool scan_reader_lock_table) const { @@ -5750,55 +5604,98 @@ inline cursor_managed txn::open_cursor(map_handle map) const { return cursor_managed(ptr); } -inline ::mdbx::map_handle -txn::open_map(const char *name, const ::mdbx::key_mode key_mode, - const ::mdbx::value_mode value_mode) const { +inline size_t txn::release_all_cursors(bool unbind) const { + size_t count; + error::success_or_throw(::mdbx_txn_release_all_cursors_ex(handle_, unbind, &count)); + return count; +} + +inline ::mdbx::map_handle txn::open_map(const ::mdbx::slice &name, const ::mdbx::key_mode key_mode, + const ::mdbx::value_mode value_mode) const { ::mdbx::map_handle map; - error::success_or_throw(::mdbx_dbi_open( - handle_, name, MDBX_db_flags_t(key_mode) | MDBX_db_flags_t(value_mode), - &map.dbi)); + error::success_or_throw( + ::mdbx_dbi_open2(handle_, name, MDBX_db_flags_t(key_mode) | MDBX_db_flags_t(value_mode), &map.dbi)); assert(map.dbi != 0); return map; } -inline ::mdbx::map_handle -txn::open_map(const ::std::string &name, const ::mdbx::key_mode key_mode, - const ::mdbx::value_mode value_mode) const { - return open_map(name.c_str(), key_mode, value_mode); +inline ::mdbx::map_handle txn::open_map(const char *name, const ::mdbx::key_mode key_mode, + const ::mdbx::value_mode value_mode) const { + ::mdbx::map_handle map; + error::success_or_throw( + ::mdbx_dbi_open(handle_, name, MDBX_db_flags_t(key_mode) | MDBX_db_flags_t(value_mode), &map.dbi)); + assert(map.dbi != 0); + return map; } -inline ::mdbx::map_handle txn::create_map(const char *name, - const ::mdbx::key_mode key_mode, +inline ::mdbx::map_handle txn::open_map_accede(const ::mdbx::slice &name) const { + ::mdbx::map_handle map; + error::success_or_throw(::mdbx_dbi_open2(handle_, name, MDBX_DB_ACCEDE, &map.dbi)); + assert(map.dbi != 0); + return map; +} + +inline ::mdbx::map_handle txn::open_map_accede(const char *name) const { + ::mdbx::map_handle map; + error::success_or_throw(::mdbx_dbi_open(handle_, name, MDBX_DB_ACCEDE, &map.dbi)); + assert(map.dbi != 0); + return map; +} + +inline ::mdbx::map_handle txn::create_map(const ::mdbx::slice &name, const ::mdbx::key_mode key_mode, const ::mdbx::value_mode value_mode) { ::mdbx::map_handle map; - error::success_or_throw(::mdbx_dbi_open( - handle_, name, - MDBX_CREATE | MDBX_db_flags_t(key_mode) | MDBX_db_flags_t(value_mode), - &map.dbi)); + error::success_or_throw( + ::mdbx_dbi_open2(handle_, name, MDBX_CREATE | MDBX_db_flags_t(key_mode) | MDBX_db_flags_t(value_mode), &map.dbi)); assert(map.dbi != 0); return map; } -inline ::mdbx::map_handle txn::create_map(const ::std::string &name, - const ::mdbx::key_mode key_mode, +inline ::mdbx::map_handle txn::create_map(const char *name, const ::mdbx::key_mode key_mode, const ::mdbx::value_mode value_mode) { - return create_map(name.c_str(), key_mode, value_mode); + ::mdbx::map_handle map; + error::success_or_throw( + ::mdbx_dbi_open(handle_, name, MDBX_CREATE | MDBX_db_flags_t(key_mode) | MDBX_db_flags_t(value_mode), &map.dbi)); + assert(map.dbi != 0); + return map; } -inline void txn::drop_map(map_handle map) { - error::success_or_throw(::mdbx_drop(handle_, map.dbi, true)); +inline void txn::drop_map(map_handle map) { error::success_or_throw(::mdbx_drop(handle_, map.dbi, true)); } + +inline void txn::clear_map(map_handle map) { error::success_or_throw(::mdbx_drop(handle_, map.dbi, false)); } + +inline void txn::rename_map(map_handle map, const char *new_name) { + error::success_or_throw(::mdbx_dbi_rename(handle_, map, new_name)); } -inline bool txn::drop_map(const ::std::string &name, bool throw_if_absent) { - return drop_map(name.c_str(), throw_if_absent); +inline void txn::rename_map(map_handle map, const ::mdbx::slice &new_name) { + error::success_or_throw(::mdbx_dbi_rename2(handle_, map, new_name)); +} + +inline ::mdbx::map_handle txn::open_map(const ::std::string &name, const ::mdbx::key_mode key_mode, + const ::mdbx::value_mode value_mode) const { + return open_map(::mdbx::slice(name), key_mode, value_mode); +} + +inline ::mdbx::map_handle txn::open_map_accede(const ::std::string &name) const { + return open_map_accede(::mdbx::slice(name)); +} + +inline ::mdbx::map_handle txn::create_map(const ::std::string &name, const ::mdbx::key_mode key_mode, + const ::mdbx::value_mode value_mode) { + return create_map(::mdbx::slice(name), key_mode, value_mode); } -inline void txn::clear_map(map_handle map) { - error::success_or_throw(::mdbx_drop(handle_, map.dbi, false)); +inline bool txn::drop_map(const ::std::string &name, bool throw_if_absent) { + return drop_map(::mdbx::slice(name), throw_if_absent); } inline bool txn::clear_map(const ::std::string &name, bool throw_if_absent) { - return clear_map(name.c_str(), throw_if_absent); + return clear_map(::mdbx::slice(name), throw_if_absent); +} + +inline void txn::rename_map(map_handle map, const ::std::string &new_name) { + return rename_map(map, ::mdbx::slice(new_name)); } inline txn::map_stat txn::get_map_stat(map_handle map) const { @@ -5815,8 +5712,7 @@ inline uint32_t txn::get_tree_deepmask(map_handle map) const { inline map_handle::info txn::get_handle_info(map_handle map) const { unsigned flags, state; - error::success_or_throw( - ::mdbx_dbi_flags_ex(handle_, map.dbi, &flags, &state)); + error::success_or_throw(::mdbx_dbi_flags_ex(handle_, map.dbi, &flags, &state)); return map_handle::info(MDBX_db_flags_t(flags), MDBX_dbi_state_t(state)); } @@ -5839,28 +5735,23 @@ inline uint64_t txn::sequence(map_handle map) const { inline uint64_t txn::sequence(map_handle map, uint64_t increment) { uint64_t result; - error::success_or_throw( - ::mdbx_dbi_sequence(handle_, map.dbi, &result, increment)); + error::success_or_throw(::mdbx_dbi_sequence(handle_, map.dbi, &result, increment)); return result; } -inline int txn::compare_keys(map_handle map, const slice &a, - const slice &b) const noexcept { +inline int txn::compare_keys(map_handle map, const slice &a, const slice &b) const noexcept { return ::mdbx_cmp(handle_, map.dbi, &a, &b); } -inline int txn::compare_values(map_handle map, const slice &a, - const slice &b) const noexcept { +inline int txn::compare_values(map_handle map, const slice &a, const slice &b) const noexcept { return ::mdbx_dcmp(handle_, map.dbi, &a, &b); } -inline int txn::compare_keys(map_handle map, const pair &a, - const pair &b) const noexcept { +inline int txn::compare_keys(map_handle map, const pair &a, const pair &b) const noexcept { return compare_keys(map, a.key, b.key); } -inline int txn::compare_values(map_handle map, const pair &a, - const pair &b) const noexcept { +inline int txn::compare_values(map_handle map, const pair &a, const pair &b) const noexcept { return compare_values(map, a.value, b.value); } @@ -5872,13 +5763,11 @@ inline slice txn::get(map_handle map, const slice &key) const { inline slice txn::get(map_handle map, slice key, size_t &values_count) const { slice result; - error::success_or_throw( - ::mdbx_get_ex(handle_, map.dbi, &key, &result, &values_count)); + error::success_or_throw(::mdbx_get_ex(handle_, map.dbi, &key, &result, &values_count)); return result; } -inline slice txn::get(map_handle map, const slice &key, - const slice &value_at_absence) const { +inline slice txn::get(map_handle map, const slice &key, const slice &value_at_absence) const { slice result; const int err = ::mdbx_get(handle_, map.dbi, &key, &result); switch (err) { @@ -5891,8 +5780,7 @@ inline slice txn::get(map_handle map, const slice &key, } } -inline slice txn::get(map_handle map, slice key, size_t &values_count, - const slice &value_at_absence) const { +inline slice txn::get(map_handle map, slice key, size_t &values_count, const slice &value_at_absence) const { slice result; const int err = ::mdbx_get_ex(handle_, map.dbi, &key, &result, &values_count); switch (err) { @@ -5905,20 +5793,15 @@ inline slice txn::get(map_handle map, slice key, size_t &values_count, } } -inline pair_result txn::get_equal_or_great(map_handle map, - const slice &key) const { +inline pair_result txn::get_equal_or_great(map_handle map, const slice &key) const { pair result(key, slice()); - bool exact = !error::boolean_or_throw( - ::mdbx_get_equal_or_great(handle_, map.dbi, &result.key, &result.value)); + bool exact = !error::boolean_or_throw(::mdbx_get_equal_or_great(handle_, map.dbi, &result.key, &result.value)); return pair_result(result.key, result.value, exact); } -inline pair_result -txn::get_equal_or_great(map_handle map, const slice &key, - const slice &value_at_absence) const { +inline pair_result txn::get_equal_or_great(map_handle map, const slice &key, const slice &value_at_absence) const { pair result{key, slice()}; - const int err = - ::mdbx_get_equal_or_great(handle_, map.dbi, &result.key, &result.value); + const int err = ::mdbx_get_equal_or_great(handle_, map.dbi, &result.key, &result.value); switch (err) { case MDBX_SUCCESS: return pair_result{result.key, result.value, true}; @@ -5931,27 +5814,22 @@ txn::get_equal_or_great(map_handle map, const slice &key, } } -inline MDBX_error_t txn::put(map_handle map, const slice &key, slice *value, - MDBX_put_flags_t flags) noexcept { +inline MDBX_error_t txn::put(map_handle map, const slice &key, slice *value, MDBX_put_flags_t flags) noexcept { return MDBX_error_t(::mdbx_put(handle_, map.dbi, &key, value, flags)); } -inline void txn::put(map_handle map, const slice &key, slice value, - put_mode mode) { +inline void txn::put(map_handle map, const slice &key, slice value, put_mode mode) { error::success_or_throw(put(map, key, &value, MDBX_put_flags_t(mode))); } inline void txn::insert(map_handle map, const slice &key, slice value) { - error::success_or_throw( - put(map, key, &value /* takes the present value in case MDBX_KEYEXIST */, - MDBX_put_flags_t(put_mode::insert_unique))); + error::success_or_throw(put(map, key, &value /* takes the present value in case MDBX_KEYEXIST */, + MDBX_put_flags_t(put_mode::insert_unique))); } -inline value_result txn::try_insert(map_handle map, const slice &key, - slice value) { - const int err = - put(map, key, &value /* takes the present value in case MDBX_KEYEXIST */, - MDBX_put_flags_t(put_mode::insert_unique)); +inline value_result txn::try_insert(map_handle map, const slice &key, slice value) { + const int err = put(map, key, &value /* takes the present value in case MDBX_KEYEXIST */, + MDBX_put_flags_t(put_mode::insert_unique)); switch (err) { case MDBX_SUCCESS: return value_result{slice(), true}; @@ -5962,21 +5840,17 @@ inline value_result txn::try_insert(map_handle map, const slice &key, } } -inline slice txn::insert_reserve(map_handle map, const slice &key, - size_t value_length) { +inline slice txn::insert_reserve(map_handle map, const slice &key, size_t value_length) { slice result(nullptr, value_length); - error::success_or_throw( - put(map, key, &result /* takes the present value in case MDBX_KEYEXIST */, - MDBX_put_flags_t(put_mode::insert_unique) | MDBX_RESERVE)); + error::success_or_throw(put(map, key, &result /* takes the present value in case MDBX_KEYEXIST */, + MDBX_put_flags_t(put_mode::insert_unique) | MDBX_RESERVE)); return result; } -inline value_result txn::try_insert_reserve(map_handle map, const slice &key, - size_t value_length) { +inline value_result txn::try_insert_reserve(map_handle map, const slice &key, size_t value_length) { slice result(nullptr, value_length); - const int err = - put(map, key, &result /* takes the present value in case MDBX_KEYEXIST */, - MDBX_put_flags_t(put_mode::insert_unique) | MDBX_RESERVE); + const int err = put(map, key, &result /* takes the present value in case MDBX_KEYEXIST */, + MDBX_put_flags_t(put_mode::insert_unique) | MDBX_RESERVE); switch (err) { case MDBX_SUCCESS: return value_result{result, true}; @@ -5988,27 +5862,21 @@ inline value_result txn::try_insert_reserve(map_handle map, const slice &key, } inline void txn::upsert(map_handle map, const slice &key, const slice &value) { - error::success_or_throw(put(map, key, const_cast(&value), - MDBX_put_flags_t(put_mode::upsert))); + error::success_or_throw(put(map, key, const_cast(&value), MDBX_put_flags_t(put_mode::upsert))); } -inline slice txn::upsert_reserve(map_handle map, const slice &key, - size_t value_length) { +inline slice txn::upsert_reserve(map_handle map, const slice &key, size_t value_length) { slice result(nullptr, value_length); - error::success_or_throw(put( - map, key, &result, MDBX_put_flags_t(put_mode::upsert) | MDBX_RESERVE)); + error::success_or_throw(put(map, key, &result, MDBX_put_flags_t(put_mode::upsert) | MDBX_RESERVE)); return result; } inline void txn::update(map_handle map, const slice &key, const slice &value) { - error::success_or_throw(put(map, key, const_cast(&value), - MDBX_put_flags_t(put_mode::update))); + error::success_or_throw(put(map, key, const_cast(&value), MDBX_put_flags_t(put_mode::update))); } -inline bool txn::try_update(map_handle map, const slice &key, - const slice &value) { - const int err = put(map, key, const_cast(&value), - MDBX_put_flags_t(put_mode::update)); +inline bool txn::try_update(map_handle map, const slice &key, const slice &value) { + const int err = put(map, key, const_cast(&value), MDBX_put_flags_t(put_mode::update)); switch (err) { case MDBX_SUCCESS: return true; @@ -6019,19 +5887,15 @@ inline bool txn::try_update(map_handle map, const slice &key, } } -inline slice txn::update_reserve(map_handle map, const slice &key, - size_t value_length) { +inline slice txn::update_reserve(map_handle map, const slice &key, size_t value_length) { slice result(nullptr, value_length); - error::success_or_throw(put( - map, key, &result, MDBX_put_flags_t(put_mode::update) | MDBX_RESERVE)); + error::success_or_throw(put(map, key, &result, MDBX_put_flags_t(put_mode::update) | MDBX_RESERVE)); return result; } -inline value_result txn::try_update_reserve(map_handle map, const slice &key, - size_t value_length) { +inline value_result txn::try_update_reserve(map_handle map, const slice &key, size_t value_length) { slice result(nullptr, value_length); - const int err = - put(map, key, &result, MDBX_put_flags_t(put_mode::update) | MDBX_RESERVE); + const int err = put(map, key, &result, MDBX_put_flags_t(put_mode::update) | MDBX_RESERVE); switch (err) { case MDBX_SUCCESS: return value_result{result, true}; @@ -6066,68 +5930,53 @@ inline bool txn::erase(map_handle map, const slice &key, const slice &value) { } } -inline void txn::replace(map_handle map, const slice &key, slice old_value, - const slice &new_value) { - error::success_or_throw(::mdbx_replace_ex( - handle_, map.dbi, &key, const_cast(&new_value), &old_value, - MDBX_CURRENT | MDBX_NOOVERWRITE, nullptr, nullptr)); +inline void txn::replace(map_handle map, const slice &key, slice old_value, const slice &new_value) { + error::success_or_throw(::mdbx_replace_ex(handle_, map.dbi, &key, const_cast(&new_value), &old_value, + MDBX_CURRENT | MDBX_NOOVERWRITE, nullptr, nullptr)); } template inline buffer txn::extract(map_handle map, const slice &key, - const typename buffer::allocator_type - &allocator) { + const typename buffer::allocator_type &allocator) { typename buffer::data_preserver result(allocator); - error::success_or_throw(::mdbx_replace_ex(handle_, map.dbi, &key, nullptr, - &result.slice_, MDBX_CURRENT, - result, &result), - result); + error::success_or_throw( + ::mdbx_replace_ex(handle_, map.dbi, &key, nullptr, &result.slice_, MDBX_CURRENT, result, &result), result); return result; } template inline buffer txn::replace(map_handle map, const slice &key, const slice &new_value, - const typename buffer::allocator_type - &allocator) { + const typename buffer::allocator_type &allocator) { typename buffer::data_preserver result(allocator); - error::success_or_throw( - ::mdbx_replace_ex(handle_, map.dbi, &key, const_cast(&new_value), - &result.slice_, MDBX_CURRENT, result, &result), - result); + error::success_or_throw(::mdbx_replace_ex(handle_, map.dbi, &key, const_cast(&new_value), &result.slice_, + MDBX_CURRENT, result, &result), + result); return result; } template -inline buffer txn::replace_reserve( - map_handle map, const slice &key, slice &new_value, - const typename buffer::allocator_type - &allocator) { +inline buffer +txn::replace_reserve(map_handle map, const slice &key, slice &new_value, + const typename buffer::allocator_type &allocator) { typename buffer::data_preserver result(allocator); - error::success_or_throw( - ::mdbx_replace_ex(handle_, map.dbi, &key, &new_value, &result.slice_, - MDBX_CURRENT | MDBX_RESERVE, result, &result), - result); + error::success_or_throw(::mdbx_replace_ex(handle_, map.dbi, &key, &new_value, &result.slice_, + MDBX_CURRENT | MDBX_RESERVE, result, &result), + result); return result; } -inline void txn::append(map_handle map, const slice &key, const slice &value, - bool multivalue_order_preserved) { - error::success_or_throw(::mdbx_put( - handle_, map.dbi, const_cast(&key), const_cast(&value), - multivalue_order_preserved ? (MDBX_APPEND | MDBX_APPENDDUP) - : MDBX_APPEND)); +inline void txn::append(map_handle map, const slice &key, const slice &value, bool multivalue_order_preserved) { + error::success_or_throw(::mdbx_put(handle_, map.dbi, const_cast(&key), const_cast(&value), + multivalue_order_preserved ? (MDBX_APPEND | MDBX_APPENDDUP) : MDBX_APPEND)); } -inline size_t txn::put_multiple(map_handle map, const slice &key, - const size_t value_length, - const void *values_array, size_t values_count, - put_mode mode, bool allow_partial) { - MDBX_val args[2] = {{const_cast(values_array), value_length}, - {nullptr, values_count}}; - const int err = ::mdbx_put(handle_, map.dbi, const_cast(&key), args, - MDBX_put_flags_t(mode) | MDBX_MULTIPLE); +inline size_t txn::put_multiple_samelength(map_handle map, const slice &key, const size_t value_length, + const void *values_array, size_t values_count, put_mode mode, + bool allow_partial) { + MDBX_val args[2] = {{const_cast(values_array), value_length}, {nullptr, values_count}}; + const int err = ::mdbx_put(handle_, map.dbi, const_cast(&key), args, MDBX_put_flags_t(mode) | MDBX_MULTIPLE); switch (err) { case MDBX_SUCCESS: MDBX_CXX20_LIKELY break; @@ -6142,35 +5991,27 @@ inline size_t txn::put_multiple(map_handle map, const slice &key, return args[1].iov_len /* done item count */; } -inline ptrdiff_t txn::estimate(map_handle map, const pair &from, - const pair &to) const { +inline ptrdiff_t txn::estimate(map_handle map, const pair &from, const pair &to) const { ptrdiff_t result; - error::success_or_throw(mdbx_estimate_range( - handle_, map.dbi, &from.key, &from.value, &to.key, &to.value, &result)); + error::success_or_throw(mdbx_estimate_range(handle_, map.dbi, &from.key, &from.value, &to.key, &to.value, &result)); return result; } -inline ptrdiff_t txn::estimate(map_handle map, const slice &from, - const slice &to) const { +inline ptrdiff_t txn::estimate(map_handle map, const slice &from, const slice &to) const { ptrdiff_t result; - error::success_or_throw(mdbx_estimate_range(handle_, map.dbi, &from, nullptr, - &to, nullptr, &result)); + error::success_or_throw(mdbx_estimate_range(handle_, map.dbi, &from, nullptr, &to, nullptr, &result)); return result; } -inline ptrdiff_t txn::estimate_from_first(map_handle map, - const slice &to) const { +inline ptrdiff_t txn::estimate_from_first(map_handle map, const slice &to) const { ptrdiff_t result; - error::success_or_throw(mdbx_estimate_range(handle_, map.dbi, nullptr, - nullptr, &to, nullptr, &result)); + error::success_or_throw(mdbx_estimate_range(handle_, map.dbi, nullptr, nullptr, &to, nullptr, &result)); return result; } -inline ptrdiff_t txn::estimate_to_last(map_handle map, - const slice &from) const { +inline ptrdiff_t txn::estimate_to_last(map_handle map, const slice &from) const { ptrdiff_t result; - error::success_or_throw(mdbx_estimate_range(handle_, map.dbi, &from, nullptr, - nullptr, nullptr, &result)); + error::success_or_throw(mdbx_estimate_range(handle_, map.dbi, &from, nullptr, nullptr, nullptr, &result)); return result; } @@ -6184,9 +6025,7 @@ inline cursor_managed cursor::clone(void *your_context) const { return clone; } -inline void *cursor::get_context() const noexcept { - return mdbx_cursor_get_userctx(handle_); -} +inline void *cursor::get_context() const noexcept { return mdbx_cursor_get_userctx(handle_); } inline cursor &cursor::set_context(void *ptr) { error::success_or_throw(::mdbx_cursor_set_userctx(handle_, ptr)); @@ -6199,9 +6038,7 @@ inline cursor &cursor::operator=(cursor &&other) noexcept { return *this; } -inline cursor::cursor(cursor &&other) noexcept : handle_(other.handle_) { - other.handle_ = nullptr; -} +inline cursor::cursor(cursor &&other) noexcept : handle_(other.handle_) { other.handle_ = nullptr; } inline cursor::~cursor() noexcept { #ifndef NDEBUG @@ -6209,47 +6046,46 @@ inline cursor::~cursor() noexcept { #endif } -MDBX_CXX14_CONSTEXPR cursor::operator bool() const noexcept { - return handle_ != nullptr; -} +MDBX_CXX14_CONSTEXPR cursor::operator bool() const noexcept { return handle_ != nullptr; } -MDBX_CXX14_CONSTEXPR cursor::operator const MDBX_cursor *() const { - return handle_; -} +MDBX_CXX14_CONSTEXPR cursor::operator const MDBX_cursor *() const { return handle_; } MDBX_CXX14_CONSTEXPR cursor::operator MDBX_cursor *() { return handle_; } -MDBX_CXX11_CONSTEXPR bool operator==(const cursor &a, - const cursor &b) noexcept { - return a.handle_ == b.handle_; +MDBX_CXX11_CONSTEXPR bool operator==(const cursor &a, const cursor &b) noexcept { return a.handle_ == b.handle_; } + +MDBX_CXX11_CONSTEXPR bool operator!=(const cursor &a, const cursor &b) noexcept { return a.handle_ != b.handle_; } + +inline int compare_position_nothrow(const cursor &left, const cursor &right, bool ignore_nested = false) noexcept { + return mdbx_cursor_compare(left.handle_, right.handle_, ignore_nested); } -MDBX_CXX11_CONSTEXPR bool operator!=(const cursor &a, - const cursor &b) noexcept { - return a.handle_ != b.handle_; +inline int compare_position(const cursor &left, const cursor &right, bool ignore_nested = false) { + const auto diff = compare_position_nothrow(left, right, ignore_nested); + assert(compare_position_nothrow(right, left, ignore_nested) == -diff); + if (MDBX_LIKELY(int16_t(diff) == diff)) + MDBX_CXX20_LIKELY return int(diff); + else + throw_incomparable_cursors(); } -inline cursor::move_result::move_result(const cursor &cursor, - bool throw_notfound) - : pair_result(slice(), slice(), false) { +inline cursor::move_result::move_result(const cursor &cursor, bool throw_notfound) : pair_result() { done = cursor.move(get_current, &this->key, &this->value, throw_notfound); } -inline cursor::move_result::move_result(cursor &cursor, - move_operation operation, - const slice &key, const slice &value, +inline cursor::move_result::move_result(cursor &cursor, move_operation operation, const slice &key, const slice &value, bool throw_notfound) : pair_result(key, value, false) { this->done = cursor.move(operation, &this->key, &this->value, throw_notfound); } -inline bool cursor::move(move_operation operation, MDBX_val *key, - MDBX_val *value, bool throw_notfound) const { - const int err = - ::mdbx_cursor_get(handle_, key, value, MDBX_cursor_op(operation)); +inline bool cursor::move(move_operation operation, MDBX_val *key, MDBX_val *value, bool throw_notfound) const { + const int err = ::mdbx_cursor_get(handle_, key, value, MDBX_cursor_op(operation)); switch (err) { case MDBX_SUCCESS: MDBX_CXX20_LIKELY return true; + case MDBX_RESULT_TRUE: + return false; case MDBX_NOTFOUND: if (!throw_notfound) return false; @@ -6259,19 +6095,15 @@ inline bool cursor::move(move_operation operation, MDBX_val *key, } } -inline cursor::estimate_result::estimate_result(const cursor &cursor, - move_operation operation, - const slice &key, +inline cursor::estimate_result::estimate_result(const cursor &cursor, move_operation operation, const slice &key, const slice &value) : pair(key, value), approximate_quantity(PTRDIFF_MIN) { approximate_quantity = cursor.estimate(operation, &this->key, &this->value); } -inline ptrdiff_t cursor::estimate(move_operation operation, MDBX_val *key, - MDBX_val *value) const { +inline ptrdiff_t cursor::estimate(move_operation operation, MDBX_val *key, MDBX_val *value) const { ptrdiff_t result; - error::success_or_throw(::mdbx_estimate_move( - *this, key, value, MDBX_cursor_op(operation), &result)); + error::success_or_throw(::mdbx_estimate_move(*this, key, value, MDBX_cursor_op(operation), &result)); return result; } @@ -6281,95 +6113,31 @@ inline ptrdiff_t estimate(const cursor &from, const cursor &to) { return result; } -inline cursor::move_result cursor::move(move_operation operation, - bool throw_notfound) { - return move_result(*this, operation, throw_notfound); -} - -inline cursor::move_result cursor::to_first(bool throw_notfound) { - return move(first, throw_notfound); -} - -inline cursor::move_result cursor::to_previous(bool throw_notfound) { - return move(previous, throw_notfound); -} - -inline cursor::move_result cursor::to_previous_last_multi(bool throw_notfound) { - return move(multi_prevkey_lastvalue, throw_notfound); -} - -inline cursor::move_result cursor::to_current_first_multi(bool throw_notfound) { - return move(multi_currentkey_firstvalue, throw_notfound); -} - -inline cursor::move_result cursor::to_current_prev_multi(bool throw_notfound) { - return move(multi_currentkey_prevvalue, throw_notfound); -} - -inline cursor::move_result cursor::current(bool throw_notfound) const { - return move_result(*this, throw_notfound); -} - -inline cursor::move_result cursor::to_current_next_multi(bool throw_notfound) { - return move(multi_currentkey_nextvalue, throw_notfound); -} - -inline cursor::move_result cursor::to_current_last_multi(bool throw_notfound) { - return move(multi_currentkey_lastvalue, throw_notfound); -} - -inline cursor::move_result cursor::to_next_first_multi(bool throw_notfound) { - return move(multi_nextkey_firstvalue, throw_notfound); -} - -inline cursor::move_result cursor::to_next(bool throw_notfound) { - return move(next, throw_notfound); -} - -inline cursor::move_result cursor::to_last(bool throw_notfound) { - return move(last, throw_notfound); -} - -inline cursor::move_result cursor::move(move_operation operation, - const slice &key, bool throw_notfound) { - return move_result(*this, operation, key, throw_notfound); -} - inline cursor::move_result cursor::find(const slice &key, bool throw_notfound) { return move(key_exact, key, throw_notfound); } -inline cursor::move_result cursor::lower_bound(const slice &key, - bool throw_notfound) { +inline cursor::move_result cursor::lower_bound(const slice &key, bool throw_notfound) { return move(key_lowerbound, key, throw_notfound); } -inline cursor::move_result cursor::move(move_operation operation, - const slice &key, const slice &value, - bool throw_notfound) { - return move_result(*this, operation, key, value, throw_notfound); +inline cursor::move_result cursor::upper_bound(const slice &key, bool throw_notfound) { + return move(key_greater_than, key, throw_notfound); } -inline cursor::move_result cursor::find_multivalue(const slice &key, - const slice &value, - bool throw_notfound) { +inline cursor::move_result cursor::find_multivalue(const slice &key, const slice &value, bool throw_notfound) { return move(multi_find_pair, key, value, throw_notfound); } -inline cursor::move_result cursor::lower_bound_multivalue(const slice &key, - const slice &value, - bool throw_notfound) { +inline cursor::move_result cursor::lower_bound_multivalue(const slice &key, const slice &value, bool throw_notfound) { return move(multi_exactkey_lowerboundvalue, key, value, throw_notfound); } -inline bool cursor::seek(const slice &key) { - return move(find_key, const_cast(&key), nullptr, false); +inline cursor::move_result cursor::upper_bound_multivalue(const slice &key, const slice &value, bool throw_notfound) { + return move(multi_exactkey_value_greater, key, value, throw_notfound); } -inline bool cursor::move(move_operation operation, slice &key, slice &value, - bool throw_notfound) { - return move(operation, &key, &value, throw_notfound); -} +inline bool cursor::seek(const slice &key) { return move(seek_key, const_cast(&key), nullptr, false); } inline size_t cursor::count_multivalue() const { size_t result; @@ -6377,20 +6145,17 @@ inline size_t cursor::count_multivalue() const { return result; } -inline bool cursor::eof() const { - return error::boolean_or_throw(::mdbx_cursor_eof(*this)); -} +inline bool cursor::eof() const { return error::boolean_or_throw(::mdbx_cursor_eof(*this)); } -inline bool cursor::on_first() const { - return error::boolean_or_throw(::mdbx_cursor_on_first(*this)); -} +inline bool cursor::on_first() const { return error::boolean_or_throw(::mdbx_cursor_on_first(*this)); } -inline bool cursor::on_last() const { - return error::boolean_or_throw(::mdbx_cursor_on_last(*this)); -} +inline bool cursor::on_last() const { return error::boolean_or_throw(::mdbx_cursor_on_last(*this)); } + +inline bool cursor::on_first_multival() const { return error::boolean_or_throw(::mdbx_cursor_on_first_dup(*this)); } -inline cursor::estimate_result cursor::estimate(const slice &key, - const slice &value) const { +inline bool cursor::on_last_multival() const { return error::boolean_or_throw(::mdbx_cursor_on_last_dup(*this)); } + +inline cursor::estimate_result cursor::estimate(const slice &key, const slice &value) const { return estimate_result(*this, multi_exactkey_lowerboundvalue, key, value); } @@ -6398,23 +6163,20 @@ inline cursor::estimate_result cursor::estimate(const slice &key) const { return estimate_result(*this, key_lowerbound, key); } -inline cursor::estimate_result -cursor::estimate(move_operation operation) const { +inline cursor::estimate_result cursor::estimate(move_operation operation) const { return estimate_result(*this, operation); } -inline void cursor::renew(const ::mdbx::txn &txn) { - error::success_or_throw(::mdbx_cursor_renew(txn, handle_)); -} +inline void cursor::renew(::mdbx::txn &txn) { error::success_or_throw(::mdbx_cursor_renew(txn, handle_)); } -inline void cursor::bind(const ::mdbx::txn &txn, - ::mdbx::map_handle map_handle) { +inline void cursor::bind(::mdbx::txn &txn, ::mdbx::map_handle map_handle) { error::success_or_throw(::mdbx_cursor_bind(txn, handle_, map_handle.dbi)); } +inline void cursor::unbind() { error::success_or_throw(::mdbx_cursor_unbind(handle_)); } + inline txn cursor::txn() const { MDBX_txn *txn = ::mdbx_cursor_txn(handle_); - error::throw_on_nullptr(txn, MDBX_EINVAL); return ::mdbx::txn(txn); } @@ -6425,21 +6187,22 @@ inline map_handle cursor::map() const { return map_handle(dbi); } -inline MDBX_error_t cursor::put(const slice &key, slice *value, - MDBX_put_flags_t flags) noexcept { +inline MDBX_error_t cursor::put(const slice &key, slice *value, MDBX_put_flags_t flags) noexcept { return MDBX_error_t(::mdbx_cursor_put(handle_, &key, value, flags)); } +inline void cursor::put(const slice &key, slice value, put_mode mode) { + error::success_or_throw(put(key, &value, MDBX_put_flags_t(mode))); +} + inline void cursor::insert(const slice &key, slice value) { error::success_or_throw( - put(key, &value /* takes the present value in case MDBX_KEYEXIST */, - MDBX_put_flags_t(put_mode::insert_unique))); + put(key, &value /* takes the present value in case MDBX_KEYEXIST */, MDBX_put_flags_t(put_mode::insert_unique))); } inline value_result cursor::try_insert(const slice &key, slice value) { const int err = - put(key, &value /* takes the present value in case MDBX_KEYEXIST */, - MDBX_put_flags_t(put_mode::insert_unique)); + put(key, &value /* takes the present value in case MDBX_KEYEXIST */, MDBX_put_flags_t(put_mode::insert_unique)); switch (err) { case MDBX_SUCCESS: return value_result{slice(), true}; @@ -6452,18 +6215,15 @@ inline value_result cursor::try_insert(const slice &key, slice value) { inline slice cursor::insert_reserve(const slice &key, size_t value_length) { slice result(nullptr, value_length); - error::success_or_throw( - put(key, &result /* takes the present value in case MDBX_KEYEXIST */, - MDBX_put_flags_t(put_mode::insert_unique) | MDBX_RESERVE)); + error::success_or_throw(put(key, &result /* takes the present value in case MDBX_KEYEXIST */, + MDBX_put_flags_t(put_mode::insert_unique) | MDBX_RESERVE)); return result; } -inline value_result cursor::try_insert_reserve(const slice &key, - size_t value_length) { +inline value_result cursor::try_insert_reserve(const slice &key, size_t value_length) { slice result(nullptr, value_length); - const int err = - put(key, &result /* takes the present value in case MDBX_KEYEXIST */, - MDBX_put_flags_t(put_mode::insert_unique) | MDBX_RESERVE); + const int err = put(key, &result /* takes the present value in case MDBX_KEYEXIST */, + MDBX_put_flags_t(put_mode::insert_unique) | MDBX_RESERVE); switch (err) { case MDBX_SUCCESS: return value_result{result, true}; @@ -6475,25 +6235,21 @@ inline value_result cursor::try_insert_reserve(const slice &key, } inline void cursor::upsert(const slice &key, const slice &value) { - error::success_or_throw(put(key, const_cast(&value), - MDBX_put_flags_t(put_mode::upsert))); + error::success_or_throw(put(key, const_cast(&value), MDBX_put_flags_t(put_mode::upsert))); } inline slice cursor::upsert_reserve(const slice &key, size_t value_length) { slice result(nullptr, value_length); - error::success_or_throw( - put(key, &result, MDBX_put_flags_t(put_mode::upsert) | MDBX_RESERVE)); + error::success_or_throw(put(key, &result, MDBX_put_flags_t(put_mode::upsert) | MDBX_RESERVE)); return result; } inline void cursor::update(const slice &key, const slice &value) { - error::success_or_throw(put(key, const_cast(&value), - MDBX_put_flags_t(put_mode::update))); + error::success_or_throw(put(key, const_cast(&value), MDBX_put_flags_t(put_mode::update))); } inline bool cursor::try_update(const slice &key, const slice &value) { - const int err = - put(key, const_cast(&value), MDBX_put_flags_t(put_mode::update)); + const int err = put(key, const_cast(&value), MDBX_put_flags_t(put_mode::update)); switch (err) { case MDBX_SUCCESS: return true; @@ -6506,16 +6262,13 @@ inline bool cursor::try_update(const slice &key, const slice &value) { inline slice cursor::update_reserve(const slice &key, size_t value_length) { slice result(nullptr, value_length); - error::success_or_throw( - put(key, &result, MDBX_put_flags_t(put_mode::update) | MDBX_RESERVE)); + error::success_or_throw(put(key, &result, MDBX_put_flags_t(put_mode::update) | MDBX_RESERVE)); return result; } -inline value_result cursor::try_update_reserve(const slice &key, - size_t value_length) { +inline value_result cursor::try_update_reserve(const slice &key, size_t value_length) { slice result(nullptr, value_length); - const int err = - put(key, &result, MDBX_put_flags_t(put_mode::update) | MDBX_RESERVE); + const int err = put(key, &result, MDBX_put_flags_t(put_mode::update) | MDBX_RESERVE); switch (err) { case MDBX_SUCCESS: return value_result{result, true}; @@ -6527,8 +6280,7 @@ inline value_result cursor::try_update_reserve(const slice &key, } inline bool cursor::erase(bool whole_multivalue) { - const int err = ::mdbx_cursor_del(handle_, whole_multivalue ? MDBX_ALLDUPS - : MDBX_CURRENT); + const int err = ::mdbx_cursor_del(handle_, whole_multivalue ? MDBX_ALLDUPS : MDBX_CURRENT); switch (err) { case MDBX_SUCCESS: MDBX_CXX20_LIKELY return true; @@ -6549,6 +6301,24 @@ inline bool cursor::erase(const slice &key, const slice &value) { return data.done && erase(); } +inline size_t cursor::put_multiple_samelength(const slice &key, const size_t value_length, const void *values_array, + size_t values_count, put_mode mode, bool allow_partial) { + MDBX_val args[2] = {{const_cast(values_array), value_length}, {nullptr, values_count}}; + const int err = ::mdbx_cursor_put(handle_, const_cast(&key), args, MDBX_put_flags_t(mode) | MDBX_MULTIPLE); + switch (err) { + case MDBX_SUCCESS: + MDBX_CXX20_LIKELY break; + case MDBX_KEYEXIST: + if (allow_partial) + break; + mdbx_txn_break(txn()); + MDBX_CXX17_FALLTHROUGH /* fallthrough */; + default: + MDBX_CXX20_UNLIKELY error::throw_exception(err); + } + return args[1].iov_len /* done item count */; +} + /// end cxx_api @} } // namespace mdbx @@ -6568,8 +6338,7 @@ inline string to_string(const ::mdbx::slice &value) { } template -inline string -to_string(const ::mdbx::buffer &buffer) { +inline string to_string(const ::mdbx::buffer &buffer) { ostringstream out; out << buffer; return out.str(); @@ -6641,15 +6410,10 @@ inline string to_string(const ::mdbx::error &value) { return out.str(); } -inline string to_string(const ::MDBX_error_t &errcode) { - return to_string(::mdbx::error(errcode)); -} +inline string to_string(const ::MDBX_error_t &errcode) { return to_string(::mdbx::error(errcode)); } template <> struct hash<::mdbx::slice> { - MDBX_CXX14_CONSTEXPR size_t - operator()(::mdbx::slice const &slice) const noexcept { - return slice.hash_value(); - } + MDBX_CXX14_CONSTEXPR size_t operator()(::mdbx::slice const &slice) const noexcept { return slice.hash_value(); } }; /// end cxx_api @} diff --git a/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx_chk.c b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx_chk.c index 7f436b8feab..fdf5f8b406c 100644 --- a/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx_chk.c +++ b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx_chk.c @@ -1,18 +1,12 @@ -/* mdbx_chk.c - memory-mapped database check tool */ +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 +/// -/* - * Copyright 2015-2024 Leonid Yuriev - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . */ +/// +/// mdbx_chk.c - memory-mapped database check tool +/// +/* clang-format off */ #ifdef _MSC_VER #if _MSC_VER > 1800 #pragma warning(disable : 4464) /* relative include path contains '..' */ @@ -21,38 +15,26 @@ #endif /* _MSC_VER (warnings) */ #define xMDBX_TOOLS /* Avoid using internal eASSERT() */ -/* - * Copyright 2015-2024 Leonid Yuriev - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . */ +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 -#define MDBX_BUILD_SOURCERY e156c1a97c017ce89d6541cd9464ae5a9761d76b3fd2f1696521f5f3792904fc_v0_12_13_0_g1fff1f67 -#ifdef MDBX_CONFIG_H -#include MDBX_CONFIG_H -#endif +#define MDBX_BUILD_SOURCERY 6b5df6869d2bf5419e3a8189d9cc849cc9911b9c8a951b9750ed0a261ce43724_v0_13_7_0_g566b0f93 #define LIBMDBX_INTERNALS -#ifdef xMDBX_TOOLS #define MDBX_DEPRECATED -#endif /* xMDBX_TOOLS */ -#ifdef xMDBX_ALLOY -/* Amalgamated build */ -#define MDBX_INTERNAL_FUNC static -#define MDBX_INTERNAL_VAR static -#else -/* Non-amalgamated build */ -#define MDBX_INTERNAL_FUNC -#define MDBX_INTERNAL_VAR extern -#endif /* xMDBX_ALLOY */ +#ifdef MDBX_CONFIG_H +#include MDBX_CONFIG_H +#endif + +/* Undefine the NDEBUG if debugging is enforced by MDBX_DEBUG */ +#if (defined(MDBX_DEBUG) && MDBX_DEBUG > 0) || (defined(MDBX_FORCE_ASSERTIONS) && MDBX_FORCE_ASSERTIONS) +#undef NDEBUG +#ifndef MDBX_DEBUG +/* Чтобы избежать включения отладки только из-за включения assert-проверок */ +#define MDBX_DEBUG 0 +#endif +#endif /*----------------------------------------------------------------------------*/ @@ -70,14 +52,59 @@ #endif /* MDBX_DISABLE_GNU_SOURCE */ /* Should be defined before any includes */ -#if !defined(_FILE_OFFSET_BITS) && !defined(__ANDROID_API__) && \ - !defined(ANDROID) +#if !defined(_FILE_OFFSET_BITS) && !defined(__ANDROID_API__) && !defined(ANDROID) #define _FILE_OFFSET_BITS 64 -#endif +#endif /* _FILE_OFFSET_BITS */ -#ifdef __APPLE__ +#if defined(__APPLE__) && !defined(_DARWIN_C_SOURCE) #define _DARWIN_C_SOURCE -#endif +#endif /* _DARWIN_C_SOURCE */ + +#if (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) && !defined(__USE_MINGW_ANSI_STDIO) +#define __USE_MINGW_ANSI_STDIO 1 +#endif /* MinGW */ + +#if defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS) + +#ifndef _WIN32_WINNT +#define _WIN32_WINNT 0x0601 /* Windows 7 */ +#endif /* _WIN32_WINNT */ + +#if !defined(_CRT_SECURE_NO_WARNINGS) +#define _CRT_SECURE_NO_WARNINGS +#endif /* _CRT_SECURE_NO_WARNINGS */ +#if !defined(UNICODE) +#define UNICODE +#endif /* UNICODE */ + +#if !defined(_NO_CRT_STDIO_INLINE) && MDBX_BUILD_SHARED_LIBRARY && !defined(xMDBX_TOOLS) && MDBX_WITHOUT_MSVC_CRT +#define _NO_CRT_STDIO_INLINE +#endif /* _NO_CRT_STDIO_INLINE */ + +#elif !defined(_POSIX_C_SOURCE) +#define _POSIX_C_SOURCE 200809L +#endif /* Windows */ + +#ifdef __cplusplus + +#ifndef NOMINMAX +#define NOMINMAX +#endif /* NOMINMAX */ + +/* Workaround for modern libstdc++ with CLANG < 4.x */ +#if defined(__SIZEOF_INT128__) && !defined(__GLIBCXX_TYPE_INT_N_0) && defined(__clang__) && __clang_major__ < 4 +#define __GLIBCXX_BITSIZE_INT_N_0 128 +#define __GLIBCXX_TYPE_INT_N_0 __int128 +#endif /* Workaround for modern libstdc++ with CLANG < 4.x */ + +#ifdef _MSC_VER +/* Workaround for MSVC' header `extern "C"` vs `std::` redefinition bug */ +#if defined(__SANITIZE_ADDRESS__) && !defined(_DISABLE_VECTOR_ANNOTATION) +#define _DISABLE_VECTOR_ANNOTATION +#endif /* _DISABLE_VECTOR_ANNOTATION */ +#endif /* _MSC_VER */ + +#endif /* __cplusplus */ #ifdef _MSC_VER #if _MSC_FULL_VER < 190024234 @@ -99,12 +126,8 @@ * and how to and where you can obtain the latest "Visual Studio 2015" build * with all fixes. */ -#error \ - "At least \"Microsoft C/C++ Compiler\" version 19.00.24234 (Visual Studio 2015 Update 3) is required." +#error "At least \"Microsoft C/C++ Compiler\" version 19.00.24234 (Visual Studio 2015 Update 3) is required." #endif -#ifndef _CRT_SECURE_NO_WARNINGS -#define _CRT_SECURE_NO_WARNINGS -#endif /* _CRT_SECURE_NO_WARNINGS */ #if _MSC_VER > 1800 #pragma warning(disable : 4464) /* relative include path contains '..' */ #endif @@ -112,124 +135,78 @@ #pragma warning(disable : 5045) /* will insert Spectre mitigation... */ #endif #if _MSC_VER > 1914 -#pragma warning( \ - disable : 5105) /* winbase.h(9531): warning C5105: macro expansion \ - producing 'defined' has undefined behavior */ +#pragma warning(disable : 5105) /* winbase.h(9531): warning C5105: macro expansion \ + producing 'defined' has undefined behavior */ +#endif +#if _MSC_VER < 1920 +/* avoid "error C2219: syntax error: type qualifier must be after '*'" */ +#define __restrict #endif #if _MSC_VER > 1930 #pragma warning(disable : 6235) /* is always a constant */ -#pragma warning(disable : 6237) /* is never evaluated and might \ +#pragma warning(disable : 6237) /* is never evaluated and might \ have side effects */ +#pragma warning(disable : 5286) /* implicit conversion from enum type 'type 1' to enum type 'type 2' */ +#pragma warning(disable : 5287) /* operands are different enum types 'type 1' and 'type 2' */ #endif #pragma warning(disable : 4710) /* 'xyz': function not inlined */ -#pragma warning(disable : 4711) /* function 'xyz' selected for automatic \ +#pragma warning(disable : 4711) /* function 'xyz' selected for automatic \ inline expansion */ -#pragma warning(disable : 4201) /* nonstandard extension used: nameless \ +#pragma warning(disable : 4201) /* nonstandard extension used: nameless \ struct/union */ #pragma warning(disable : 4702) /* unreachable code */ #pragma warning(disable : 4706) /* assignment within conditional expression */ #pragma warning(disable : 4127) /* conditional expression is constant */ -#pragma warning(disable : 4324) /* 'xyz': structure was padded due to \ +#pragma warning(disable : 4324) /* 'xyz': structure was padded due to \ alignment specifier */ #pragma warning(disable : 4310) /* cast truncates constant value */ -#pragma warning(disable : 4820) /* bytes padding added after data member for \ +#pragma warning(disable : 4820) /* bytes padding added after data member for \ alignment */ -#pragma warning(disable : 4548) /* expression before comma has no effect; \ +#pragma warning(disable : 4548) /* expression before comma has no effect; \ expected expression with side - effect */ -#pragma warning(disable : 4366) /* the result of the unary '&' operator may be \ +#pragma warning(disable : 4366) /* the result of the unary '&' operator may be \ unaligned */ -#pragma warning(disable : 4200) /* nonstandard extension used: zero-sized \ +#pragma warning(disable : 4200) /* nonstandard extension used: zero-sized \ array in struct/union */ -#pragma warning(disable : 4204) /* nonstandard extension used: non-constant \ +#pragma warning(disable : 4204) /* nonstandard extension used: non-constant \ aggregate initializer */ -#pragma warning( \ - disable : 4505) /* unreferenced local function has been removed */ -#endif /* _MSC_VER (warnings) */ +#pragma warning(disable : 4505) /* unreferenced local function has been removed */ +#endif /* _MSC_VER (warnings) */ #if defined(__GNUC__) && __GNUC__ < 9 #pragma GCC diagnostic ignored "-Wattributes" #endif /* GCC < 9 */ -#if (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) && \ - !defined(__USE_MINGW_ANSI_STDIO) -#define __USE_MINGW_ANSI_STDIO 1 -#endif /* MinGW */ - -#if (defined(_WIN32) || defined(_WIN64)) && !defined(UNICODE) -#define UNICODE -#endif /* UNICODE */ - -#include "mdbx.h" -/* - * Copyright 2015-2024 Leonid Yuriev - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - */ - - /*----------------------------------------------------------------------------*/ /* Microsoft compiler generates a lot of warning for self includes... */ #ifdef _MSC_VER #pragma warning(push, 1) -#pragma warning(disable : 4548) /* expression before comma has no effect; \ +#pragma warning(disable : 4548) /* expression before comma has no effect; \ expected expression with side - effect */ -#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ +#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ * semantics are not enabled. Specify /EHsc */ -#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ - * mode specified; termination on exception is \ +#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ + * mode specified; termination on exception is \ * not guaranteed. Specify /EHsc */ #endif /* _MSC_VER (warnings) */ -#if defined(_WIN32) || defined(_WIN64) -#if !defined(_CRT_SECURE_NO_WARNINGS) -#define _CRT_SECURE_NO_WARNINGS -#endif /* _CRT_SECURE_NO_WARNINGS */ -#if !defined(_NO_CRT_STDIO_INLINE) && MDBX_BUILD_SHARED_LIBRARY && \ - !defined(xMDBX_TOOLS) && MDBX_WITHOUT_MSVC_CRT -#define _NO_CRT_STDIO_INLINE -#endif -#elif !defined(_POSIX_C_SOURCE) -#define _POSIX_C_SOURCE 200809L -#endif /* Windows */ - /*----------------------------------------------------------------------------*/ /* basic C99 includes */ + #include #include #include #include #include +#include #include #include #include #include #include -#if (-6 & 5) || CHAR_BIT != 8 || UINT_MAX < 0xffffffff || ULONG_MAX % 0xFFFF -#error \ - "Sanity checking failed: Two's complement, reasonably sized integer types" -#endif - -#ifndef SSIZE_MAX -#define SSIZE_MAX INTPTR_MAX -#endif - -#if UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul || defined(_WIN64) -#define MDBX_WORDBITS 64 -#else -#define MDBX_WORDBITS 32 -#endif /* MDBX_WORDBITS */ - /*----------------------------------------------------------------------------*/ /* feature testing */ @@ -241,6 +218,14 @@ #define __has_include(x) (0) #endif +#ifndef __has_attribute +#define __has_attribute(x) (0) +#endif + +#ifndef __has_cpp_attribute +#define __has_cpp_attribute(x) 0 +#endif + #ifndef __has_feature #define __has_feature(x) (0) #endif @@ -263,8 +248,7 @@ #ifndef __GNUC_PREREQ #if defined(__GNUC__) && defined(__GNUC_MINOR__) -#define __GNUC_PREREQ(maj, min) \ - ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min)) +#define __GNUC_PREREQ(maj, min) ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min)) #else #define __GNUC_PREREQ(maj, min) (0) #endif @@ -272,8 +256,7 @@ #ifndef __CLANG_PREREQ #ifdef __clang__ -#define __CLANG_PREREQ(maj, min) \ - ((__clang_major__ << 16) + __clang_minor__ >= ((maj) << 16) + (min)) +#define __CLANG_PREREQ(maj, min) ((__clang_major__ << 16) + __clang_minor__ >= ((maj) << 16) + (min)) #else #define __CLANG_PREREQ(maj, min) (0) #endif @@ -281,13 +264,51 @@ #ifndef __GLIBC_PREREQ #if defined(__GLIBC__) && defined(__GLIBC_MINOR__) -#define __GLIBC_PREREQ(maj, min) \ - ((__GLIBC__ << 16) + __GLIBC_MINOR__ >= ((maj) << 16) + (min)) +#define __GLIBC_PREREQ(maj, min) ((__GLIBC__ << 16) + __GLIBC_MINOR__ >= ((maj) << 16) + (min)) #else #define __GLIBC_PREREQ(maj, min) (0) #endif #endif /* __GLIBC_PREREQ */ +/*----------------------------------------------------------------------------*/ +/* pre-requirements */ + +#if (-6 & 5) || CHAR_BIT != 8 || UINT_MAX < 0xffffffff || ULONG_MAX % 0xFFFF +#error "Sanity checking failed: Two's complement, reasonably sized integer types" +#endif + +#ifndef SSIZE_MAX +#define SSIZE_MAX INTPTR_MAX +#endif + +#if defined(__GNUC__) && !__GNUC_PREREQ(4, 2) +/* Actually libmdbx was not tested with compilers older than GCC 4.2. + * But you could ignore this warning at your own risk. + * In such case please don't rise up an issues related ONLY to old compilers. + */ +#warning "libmdbx required GCC >= 4.2" +#endif + +#if defined(__clang__) && !__CLANG_PREREQ(3, 8) +/* Actually libmdbx was not tested with CLANG older than 3.8. + * But you could ignore this warning at your own risk. + * In such case please don't rise up an issues related ONLY to old compilers. + */ +#warning "libmdbx required CLANG >= 3.8" +#endif + +#if defined(__GLIBC__) && !__GLIBC_PREREQ(2, 12) +/* Actually libmdbx was not tested with something older than glibc 2.12. + * But you could ignore this warning at your own risk. + * In such case please don't rise up an issues related ONLY to old systems. + */ +#warning "libmdbx was only tested with GLIBC >= 2.12." +#endif + +#ifdef __SANITIZE_THREAD__ +#warning "libmdbx don't compatible with ThreadSanitizer, you will get a lot of false-positive issues." +#endif /* __SANITIZE_THREAD__ */ + /*----------------------------------------------------------------------------*/ /* C11' alignas() */ @@ -317,8 +338,7 @@ #endif #endif /* __extern_C */ -#if !defined(nullptr) && !defined(__cplusplus) || \ - (__cplusplus < 201103L && !defined(_MSC_VER)) +#if !defined(nullptr) && !defined(__cplusplus) || (__cplusplus < 201103L && !defined(_MSC_VER)) #define nullptr NULL #endif @@ -330,9 +350,8 @@ #endif #endif /* Apple OSX & iOS */ -#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \ - defined(__BSD__) || defined(__bsdi__) || defined(__DragonFly__) || \ - defined(__APPLE__) || defined(__MACH__) +#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__BSD__) || defined(__bsdi__) || \ + defined(__DragonFly__) || defined(__APPLE__) || defined(__MACH__) #include #include #include @@ -349,8 +368,7 @@ #endif #else #include -#if !(defined(__sun) || defined(__SVR4) || defined(__svr4__) || \ - defined(_WIN32) || defined(_WIN64)) +#if !(defined(__sun) || defined(__SVR4) || defined(__svr4__) || defined(_WIN32) || defined(_WIN64)) #include #endif /* !Solaris */ #endif /* !xBSD */ @@ -404,12 +422,14 @@ __extern_C key_t ftok(const char *, int); #ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN #endif /* WIN32_LEAN_AND_MEAN */ -#include -#include #include #include #include +/* После подгрузки windows.h, чтобы избежать проблем со сборкой MINGW и т.п. */ +#include +#include + #else /*----------------------------------------------------------------------*/ #include @@ -437,11 +457,6 @@ __extern_C key_t ftok(const char *, int); #if __ANDROID_API__ >= 21 #include #endif -#if defined(_FILE_OFFSET_BITS) && _FILE_OFFSET_BITS != MDBX_WORDBITS -#error "_FILE_OFFSET_BITS != MDBX_WORDBITS" (_FILE_OFFSET_BITS != MDBX_WORDBITS) -#elif defined(__FILE_OFFSET_BITS) && __FILE_OFFSET_BITS != MDBX_WORDBITS -#error "__FILE_OFFSET_BITS != MDBX_WORDBITS" (__FILE_OFFSET_BITS != MDBX_WORDBITS) -#endif #endif /* Android */ #if defined(HAVE_SYS_STAT_H) || __has_include() @@ -457,43 +472,38 @@ __extern_C key_t ftok(const char *, int); /*----------------------------------------------------------------------------*/ /* Byteorder */ -#if defined(i386) || defined(__386) || defined(__i386) || defined(__i386__) || \ - defined(i486) || defined(__i486) || defined(__i486__) || defined(i586) || \ - defined(__i586) || defined(__i586__) || defined(i686) || \ - defined(__i686) || defined(__i686__) || defined(_M_IX86) || \ - defined(_X86_) || defined(__THW_INTEL__) || defined(__I86__) || \ - defined(__INTEL__) || defined(__x86_64) || defined(__x86_64__) || \ - defined(__amd64__) || defined(__amd64) || defined(_M_X64) || \ - defined(_M_AMD64) || defined(__IA32__) || defined(__INTEL__) +#if defined(i386) || defined(__386) || defined(__i386) || defined(__i386__) || defined(i486) || defined(__i486) || \ + defined(__i486__) || defined(i586) || defined(__i586) || defined(__i586__) || defined(i686) || defined(__i686) || \ + defined(__i686__) || defined(_M_IX86) || defined(_X86_) || defined(__THW_INTEL__) || defined(__I86__) || \ + defined(__INTEL__) || defined(__x86_64) || defined(__x86_64__) || defined(__amd64__) || defined(__amd64) || \ + defined(_M_X64) || defined(_M_AMD64) || defined(__IA32__) || defined(__INTEL__) #ifndef __ia32__ /* LY: define neutral __ia32__ for x86 and x86-64 */ #define __ia32__ 1 #endif /* __ia32__ */ -#if !defined(__amd64__) && \ - (defined(__x86_64) || defined(__x86_64__) || defined(__amd64) || \ - defined(_M_X64) || defined(_M_AMD64)) +#if !defined(__amd64__) && \ + (defined(__x86_64) || defined(__x86_64__) || defined(__amd64) || defined(_M_X64) || defined(_M_AMD64)) /* LY: define trusty __amd64__ for all AMD64/x86-64 arch */ #define __amd64__ 1 #endif /* __amd64__ */ #endif /* all x86 */ -#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || \ - !defined(__ORDER_BIG_ENDIAN__) +#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || !defined(__ORDER_BIG_ENDIAN__) -#if defined(__GLIBC__) || defined(__GNU_LIBRARY__) || \ - defined(__ANDROID_API__) || defined(HAVE_ENDIAN_H) || __has_include() +#if defined(__GLIBC__) || defined(__GNU_LIBRARY__) || defined(__ANDROID_API__) || defined(HAVE_ENDIAN_H) || \ + __has_include() #include -#elif defined(__APPLE__) || defined(__MACH__) || defined(__OpenBSD__) || \ - defined(HAVE_MACHINE_ENDIAN_H) || __has_include() +#elif defined(__APPLE__) || defined(__MACH__) || defined(__OpenBSD__) || defined(HAVE_MACHINE_ENDIAN_H) || \ + __has_include() #include #elif defined(HAVE_SYS_ISA_DEFS_H) || __has_include() #include -#elif (defined(HAVE_SYS_TYPES_H) && defined(HAVE_SYS_ENDIAN_H)) || \ +#elif (defined(HAVE_SYS_TYPES_H) && defined(HAVE_SYS_ENDIAN_H)) || \ (__has_include() && __has_include()) #include #include -#elif defined(__bsdi__) || defined(__DragonFly__) || defined(__FreeBSD__) || \ - defined(__NetBSD__) || defined(HAVE_SYS_PARAM_H) || __has_include() +#elif defined(__bsdi__) || defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || \ + defined(HAVE_SYS_PARAM_H) || __has_include() #include #endif /* OS */ @@ -509,27 +519,19 @@ __extern_C key_t ftok(const char *, int); #define __ORDER_LITTLE_ENDIAN__ 1234 #define __ORDER_BIG_ENDIAN__ 4321 -#if defined(__LITTLE_ENDIAN__) || \ - (defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)) || \ - defined(__ARMEL__) || defined(__THUMBEL__) || defined(__AARCH64EL__) || \ - defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \ - defined(_M_ARM) || defined(_M_ARM64) || defined(__e2k__) || \ - defined(__elbrus_4c__) || defined(__elbrus_8c__) || defined(__bfin__) || \ - defined(__BFIN__) || defined(__ia64__) || defined(_IA64) || \ - defined(__IA64__) || defined(__ia64) || defined(_M_IA64) || \ - defined(__itanium__) || defined(__ia32__) || defined(__CYGWIN__) || \ - defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) || \ - defined(__WINDOWS__) +#if defined(__LITTLE_ENDIAN__) || (defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)) || defined(__ARMEL__) || \ + defined(__THUMBEL__) || defined(__AARCH64EL__) || defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \ + defined(_M_ARM) || defined(_M_ARM64) || defined(__e2k__) || defined(__elbrus_4c__) || defined(__elbrus_8c__) || \ + defined(__bfin__) || defined(__BFIN__) || defined(__ia64__) || defined(_IA64) || defined(__IA64__) || \ + defined(__ia64) || defined(_M_IA64) || defined(__itanium__) || defined(__ia32__) || defined(__CYGWIN__) || \ + defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) || defined(__WINDOWS__) #define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__ -#elif defined(__BIG_ENDIAN__) || \ - (defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) || \ - defined(__ARMEB__) || defined(__THUMBEB__) || defined(__AARCH64EB__) || \ - defined(__MIPSEB__) || defined(_MIPSEB) || defined(__MIPSEB) || \ - defined(__m68k__) || defined(M68000) || defined(__hppa__) || \ - defined(__hppa) || defined(__HPPA__) || defined(__sparc__) || \ - defined(__sparc) || defined(__370__) || defined(__THW_370__) || \ - defined(__s390__) || defined(__s390x__) || defined(__SYSC_ZARCH__) +#elif defined(__BIG_ENDIAN__) || (defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) || defined(__ARMEB__) || \ + defined(__THUMBEB__) || defined(__AARCH64EB__) || defined(__MIPSEB__) || defined(_MIPSEB) || defined(__MIPSEB) || \ + defined(__m68k__) || defined(M68000) || defined(__hppa__) || defined(__hppa) || defined(__HPPA__) || \ + defined(__sparc__) || defined(__sparc) || defined(__370__) || defined(__THW_370__) || defined(__s390__) || \ + defined(__s390x__) || defined(__SYSC_ZARCH__) #define __BYTE_ORDER__ __ORDER_BIG_ENDIAN__ #else @@ -539,6 +541,12 @@ __extern_C key_t ftok(const char *, int); #endif #endif /* __BYTE_ORDER__ || __ORDER_LITTLE_ENDIAN__ || __ORDER_BIG_ENDIAN__ */ +#if UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul || defined(_WIN64) +#define MDBX_WORDBITS 64 +#else +#define MDBX_WORDBITS 32 +#endif /* MDBX_WORDBITS */ + /*----------------------------------------------------------------------------*/ /* Availability of CMOV or equivalent */ @@ -549,17 +557,14 @@ __extern_C key_t ftok(const char *, int); #define MDBX_HAVE_CMOV 1 #elif defined(__thumb__) || defined(__thumb) || defined(__TARGET_ARCH_THUMB) #define MDBX_HAVE_CMOV 0 -#elif defined(_M_ARM) || defined(_M_ARM64) || defined(__aarch64__) || \ - defined(__aarch64) || defined(__arm__) || defined(__arm) || \ - defined(__CC_ARM) +#elif defined(_M_ARM) || defined(_M_ARM64) || defined(__aarch64__) || defined(__aarch64) || defined(__arm__) || \ + defined(__arm) || defined(__CC_ARM) #define MDBX_HAVE_CMOV 1 -#elif (defined(__riscv__) || defined(__riscv64)) && \ - (defined(__riscv_b) || defined(__riscv_bitmanip)) +#elif (defined(__riscv__) || defined(__riscv64)) && (defined(__riscv_b) || defined(__riscv_bitmanip)) #define MDBX_HAVE_CMOV 1 -#elif defined(i686) || defined(__i686) || defined(__i686__) || \ - (defined(_M_IX86) && _M_IX86 > 600) || defined(__x86_64) || \ - defined(__x86_64__) || defined(__amd64__) || defined(__amd64) || \ - defined(_M_X64) || defined(_M_AMD64) +#elif defined(i686) || defined(__i686) || defined(__i686__) || (defined(_M_IX86) && _M_IX86 > 600) || \ + defined(__x86_64) || defined(__x86_64__) || defined(__amd64__) || defined(__amd64) || defined(_M_X64) || \ + defined(_M_AMD64) #define MDBX_HAVE_CMOV 1 #else #define MDBX_HAVE_CMOV 0 @@ -585,8 +590,7 @@ __extern_C key_t ftok(const char *, int); #endif #elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) #include -#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ - (defined(HP_IA64) || defined(__ia64)) +#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && (defined(HP_IA64) || defined(__ia64)) #include #elif defined(__IBMC__) && defined(__powerpc) #include @@ -608,29 +612,26 @@ __extern_C key_t ftok(const char *, int); #endif /* Compiler */ #if !defined(__noop) && !defined(_MSC_VER) -#define __noop \ - do { \ +#define __noop \ + do { \ } while (0) #endif /* __noop */ -#if defined(__fallthrough) && \ - (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) +#if defined(__fallthrough) && (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) #undef __fallthrough #endif /* __fallthrough workaround for MinGW */ #ifndef __fallthrough -#if defined(__cplusplus) && (__has_cpp_attribute(fallthrough) && \ - (!defined(__clang__) || __clang__ > 4)) || \ +#if defined(__cplusplus) && (__has_cpp_attribute(fallthrough) && (!defined(__clang__) || __clang__ > 4)) || \ __cplusplus >= 201703L #define __fallthrough [[fallthrough]] #elif __GNUC_PREREQ(8, 0) && defined(__cplusplus) && __cplusplus >= 201103L #define __fallthrough [[fallthrough]] -#elif __GNUC_PREREQ(7, 0) && \ - (!defined(__LCC__) || (__LCC__ == 124 && __LCC_MINOR__ >= 12) || \ - (__LCC__ == 125 && __LCC_MINOR__ >= 5) || (__LCC__ >= 126)) +#elif __GNUC_PREREQ(7, 0) && (!defined(__LCC__) || (__LCC__ == 124 && __LCC_MINOR__ >= 12) || \ + (__LCC__ == 125 && __LCC_MINOR__ >= 5) || (__LCC__ >= 126)) #define __fallthrough __attribute__((__fallthrough__)) -#elif defined(__clang__) && defined(__cplusplus) && __cplusplus >= 201103L && \ - __has_feature(cxx_attributes) && __has_warning("-Wimplicit-fallthrough") +#elif defined(__clang__) && defined(__cplusplus) && __cplusplus >= 201103L && __has_feature(cxx_attributes) && \ + __has_warning("-Wimplicit-fallthrough") #define __fallthrough [[clang::fallthrough]] #else #define __fallthrough @@ -643,8 +644,8 @@ __extern_C key_t ftok(const char *, int); #elif defined(_MSC_VER) #define __unreachable() __assume(0) #else -#define __unreachable() \ - do { \ +#define __unreachable() \ + do { \ } while (1) #endif #endif /* __unreachable */ @@ -653,9 +654,9 @@ __extern_C key_t ftok(const char *, int); #if defined(__GNUC__) || defined(__clang__) || __has_builtin(__builtin_prefetch) #define __prefetch(ptr) __builtin_prefetch(ptr) #else -#define __prefetch(ptr) \ - do { \ - (void)(ptr); \ +#define __prefetch(ptr) \ + do { \ + (void)(ptr); \ } while (0) #endif #endif /* __prefetch */ @@ -665,11 +666,11 @@ __extern_C key_t ftok(const char *, int); #endif /* offsetof */ #ifndef container_of -#define container_of(ptr, type, member) \ - ((type *)((char *)(ptr) - offsetof(type, member))) +#define container_of(ptr, type, member) ((type *)((char *)(ptr) - offsetof(type, member))) #endif /* container_of */ /*----------------------------------------------------------------------------*/ +/* useful attributes */ #ifndef __always_inline #if defined(__GNUC__) || __has_attribute(__always_inline__) @@ -737,8 +738,7 @@ __extern_C key_t ftok(const char *, int); #ifndef __hot #if defined(__OPTIMIZE__) -#if defined(__clang__) && !__has_attribute(__hot__) && \ - __has_attribute(__section__) && \ +#if defined(__clang__) && !__has_attribute(__hot__) && __has_attribute(__section__) && \ (defined(__linux__) || defined(__gnu_linux__)) /* just put frequently used functions in separate section */ #define __hot __attribute__((__section__("text.hot"))) __optimize("O3") @@ -754,8 +754,7 @@ __extern_C key_t ftok(const char *, int); #ifndef __cold #if defined(__OPTIMIZE__) -#if defined(__clang__) && !__has_attribute(__cold__) && \ - __has_attribute(__section__) && \ +#if defined(__clang__) && !__has_attribute(__cold__) && __has_attribute(__section__) && \ (defined(__linux__) || defined(__gnu_linux__)) /* just put infrequently used functions in separate section */ #define __cold __attribute__((__section__("text.unlikely"))) __optimize("Os") @@ -778,8 +777,7 @@ __extern_C key_t ftok(const char *, int); #endif /* __flatten */ #ifndef likely -#if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && \ - !defined(__COVERITY__) +#if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__) #define likely(cond) __builtin_expect(!!(cond), 1) #else #define likely(x) (!!(x)) @@ -787,8 +785,7 @@ __extern_C key_t ftok(const char *, int); #endif /* likely */ #ifndef unlikely -#if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && \ - !defined(__COVERITY__) +#if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__) #define unlikely(cond) __builtin_expect(!!(cond), 0) #else #define unlikely(x) (!!(x)) @@ -803,29 +800,41 @@ __extern_C key_t ftok(const char *, int); #endif #endif /* __anonymous_struct_extension__ */ -#ifndef expect_with_probability -#if defined(__builtin_expect_with_probability) || \ - __has_builtin(__builtin_expect_with_probability) || __GNUC_PREREQ(9, 0) -#define expect_with_probability(expr, value, prob) \ - __builtin_expect_with_probability(expr, value, prob) -#else -#define expect_with_probability(expr, value, prob) (expr) -#endif -#endif /* expect_with_probability */ - #ifndef MDBX_WEAK_IMPORT_ATTRIBUTE #ifdef WEAK_IMPORT_ATTRIBUTE #define MDBX_WEAK_IMPORT_ATTRIBUTE WEAK_IMPORT_ATTRIBUTE #elif __has_attribute(__weak__) && __has_attribute(__weak_import__) #define MDBX_WEAK_IMPORT_ATTRIBUTE __attribute__((__weak__, __weak_import__)) -#elif __has_attribute(__weak__) || \ - (defined(__GNUC__) && __GNUC__ >= 4 && defined(__ELF__)) +#elif __has_attribute(__weak__) || (defined(__GNUC__) && __GNUC__ >= 4 && defined(__ELF__)) #define MDBX_WEAK_IMPORT_ATTRIBUTE __attribute__((__weak__)) #else #define MDBX_WEAK_IMPORT_ATTRIBUTE #endif #endif /* MDBX_WEAK_IMPORT_ATTRIBUTE */ +#if !defined(__thread) && (defined(_MSC_VER) || defined(__DMC__)) +#define __thread __declspec(thread) +#endif /* __thread */ + +#ifndef MDBX_EXCLUDE_FOR_GPROF +#ifdef ENABLE_GPROF +#define MDBX_EXCLUDE_FOR_GPROF __attribute__((__no_instrument_function__, __no_profile_instrument_function__)) +#else +#define MDBX_EXCLUDE_FOR_GPROF +#endif /* ENABLE_GPROF */ +#endif /* MDBX_EXCLUDE_FOR_GPROF */ + +/*----------------------------------------------------------------------------*/ + +#ifndef expect_with_probability +#if defined(__builtin_expect_with_probability) || __has_builtin(__builtin_expect_with_probability) || \ + __GNUC_PREREQ(9, 0) +#define expect_with_probability(expr, value, prob) __builtin_expect_with_probability(expr, value, prob) +#else +#define expect_with_probability(expr, value, prob) (expr) +#endif +#endif /* expect_with_probability */ + #ifndef MDBX_GOOFY_MSVC_STATIC_ANALYZER #ifdef _PREFAST_ #define MDBX_GOOFY_MSVC_STATIC_ANALYZER 1 @@ -837,20 +846,27 @@ __extern_C key_t ftok(const char *, int); #if MDBX_GOOFY_MSVC_STATIC_ANALYZER || (defined(_MSC_VER) && _MSC_VER > 1919) #define MDBX_ANALYSIS_ASSUME(expr) __analysis_assume(expr) #ifdef _PREFAST_ -#define MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(warn_id) \ - __pragma(prefast(suppress : warn_id)) +#define MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(warn_id) __pragma(prefast(suppress : warn_id)) #else -#define MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(warn_id) \ - __pragma(warning(suppress : warn_id)) +#define MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(warn_id) __pragma(warning(suppress : warn_id)) #endif #else #define MDBX_ANALYSIS_ASSUME(expr) assert(expr) #define MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(warn_id) #endif /* MDBX_GOOFY_MSVC_STATIC_ANALYZER */ +#ifndef FLEXIBLE_ARRAY_MEMBERS +#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || (!defined(__cplusplus) && defined(_MSC_VER)) +#define FLEXIBLE_ARRAY_MEMBERS 1 +#else +#define FLEXIBLE_ARRAY_MEMBERS 0 +#endif +#endif /* FLEXIBLE_ARRAY_MEMBERS */ + /*----------------------------------------------------------------------------*/ +/* Valgrind and Address Sanitizer */ -#if defined(MDBX_USE_VALGRIND) +#if defined(ENABLE_MEMCHECK) #include #ifndef VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE /* LY: available since Valgrind 3.10 */ @@ -872,7 +888,7 @@ __extern_C key_t ftok(const char *, int); #define VALGRIND_CHECK_MEM_IS_ADDRESSABLE(a, s) (0) #define VALGRIND_CHECK_MEM_IS_DEFINED(a, s) (0) #define RUNNING_ON_VALGRIND (0) -#endif /* MDBX_USE_VALGRIND */ +#endif /* ENABLE_MEMCHECK */ #ifdef __SANITIZE_ADDRESS__ #include @@ -899,8 +915,7 @@ template char (&__ArraySizeHelper(T (&array)[N]))[N]; #define CONCAT(a, b) a##b #define XCONCAT(a, b) CONCAT(a, b) -#define MDBX_TETRAD(a, b, c, d) \ - ((uint32_t)(a) << 24 | (uint32_t)(b) << 16 | (uint32_t)(c) << 8 | (d)) +#define MDBX_TETRAD(a, b, c, d) ((uint32_t)(a) << 24 | (uint32_t)(b) << 16 | (uint32_t)(c) << 8 | (d)) #define MDBX_STRING_TETRAD(str) MDBX_TETRAD(str[0], str[1], str[2], str[3]) @@ -914,14 +929,13 @@ template char (&__ArraySizeHelper(T (&array)[N]))[N]; #elif defined(_MSC_VER) #include #define STATIC_ASSERT_MSG(expr, msg) _STATIC_ASSERT(expr) -#elif (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) || \ - __has_feature(c_static_assert) +#elif (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) || __has_feature(c_static_assert) #define STATIC_ASSERT_MSG(expr, msg) _Static_assert(expr, msg) #else -#define STATIC_ASSERT_MSG(expr, msg) \ - switch (0) { \ - case 0: \ - case (expr):; \ +#define STATIC_ASSERT_MSG(expr, msg) \ + switch (0) { \ + case 0: \ + case (expr):; \ } #endif #endif /* STATIC_ASSERT */ @@ -930,42 +944,37 @@ template char (&__ArraySizeHelper(T (&array)[N]))[N]; #define STATIC_ASSERT(expr) STATIC_ASSERT_MSG(expr, #expr) #endif -#ifndef __Wpedantic_format_voidptr -MDBX_MAYBE_UNUSED MDBX_PURE_FUNCTION static __inline const void * -__Wpedantic_format_voidptr(const void *ptr) { - return ptr; -} -#define __Wpedantic_format_voidptr(ARG) __Wpedantic_format_voidptr(ARG) -#endif /* __Wpedantic_format_voidptr */ +/*----------------------------------------------------------------------------*/ -#if defined(__GNUC__) && !__GNUC_PREREQ(4, 2) -/* Actually libmdbx was not tested with compilers older than GCC 4.2. - * But you could ignore this warning at your own risk. - * In such case please don't rise up an issues related ONLY to old compilers. - */ -#warning "libmdbx required GCC >= 4.2" -#endif +#if defined(_MSC_VER) && _MSC_VER >= 1900 +/* LY: MSVC 2015/2017/2019 has buggy/inconsistent PRIuPTR/PRIxPTR macros + * for internal format-args checker. */ +#undef PRIuPTR +#undef PRIiPTR +#undef PRIdPTR +#undef PRIxPTR +#define PRIuPTR "Iu" +#define PRIiPTR "Ii" +#define PRIdPTR "Id" +#define PRIxPTR "Ix" +#define PRIuSIZE "zu" +#define PRIiSIZE "zi" +#define PRIdSIZE "zd" +#define PRIxSIZE "zx" +#endif /* fix PRI*PTR for _MSC_VER */ -#if defined(__clang__) && !__CLANG_PREREQ(3, 8) -/* Actually libmdbx was not tested with CLANG older than 3.8. - * But you could ignore this warning at your own risk. - * In such case please don't rise up an issues related ONLY to old compilers. - */ -#warning "libmdbx required CLANG >= 3.8" -#endif +#ifndef PRIuSIZE +#define PRIuSIZE PRIuPTR +#define PRIiSIZE PRIiPTR +#define PRIdSIZE PRIdPTR +#define PRIxSIZE PRIxPTR +#endif /* PRI*SIZE macros for MSVC */ -#if defined(__GLIBC__) && !__GLIBC_PREREQ(2, 12) -/* Actually libmdbx was not tested with something older than glibc 2.12. - * But you could ignore this warning at your own risk. - * In such case please don't rise up an issues related ONLY to old systems. - */ -#warning "libmdbx was only tested with GLIBC >= 2.12." +#ifdef _MSC_VER +#pragma warning(pop) #endif -#ifdef __SANITIZE_THREAD__ -#warning \ - "libmdbx don't compatible with ThreadSanitizer, you will get a lot of false-positive issues." -#endif /* __SANITIZE_THREAD__ */ +/*----------------------------------------------------------------------------*/ #if __has_warning("-Wnested-anon-types") #if defined(__clang__) @@ -1002,80 +1011,34 @@ __Wpedantic_format_voidptr(const void *ptr) { #endif #endif /* -Walignment-reduction-ignored */ -#ifndef MDBX_EXCLUDE_FOR_GPROF -#ifdef ENABLE_GPROF -#define MDBX_EXCLUDE_FOR_GPROF \ - __attribute__((__no_instrument_function__, \ - __no_profile_instrument_function__)) +#ifdef xMDBX_ALLOY +/* Amalgamated build */ +#define MDBX_INTERNAL static #else -#define MDBX_EXCLUDE_FOR_GPROF -#endif /* ENABLE_GPROF */ -#endif /* MDBX_EXCLUDE_FOR_GPROF */ - -#ifdef __cplusplus -extern "C" { -#endif +/* Non-amalgamated build */ +#define MDBX_INTERNAL +#endif /* xMDBX_ALLOY */ -/* https://en.wikipedia.org/wiki/Operating_system_abstraction_layer */ +#include "mdbx.h" -/* - * Copyright 2015-2024 Leonid Yuriev - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - */ +/*----------------------------------------------------------------------------*/ +/* Basic constants and types */ +typedef struct iov_ctx iov_ctx_t; +/// /*----------------------------------------------------------------------------*/ -/* C11 Atomics */ - -#if defined(__cplusplus) && !defined(__STDC_NO_ATOMICS__) && __has_include() -#include -#define MDBX_HAVE_C11ATOMICS -#elif !defined(__cplusplus) && \ - (__STDC_VERSION__ >= 201112L || __has_extension(c_atomic)) && \ - !defined(__STDC_NO_ATOMICS__) && \ - (__GNUC_PREREQ(4, 9) || __CLANG_PREREQ(3, 8) || \ - !(defined(__GNUC__) || defined(__clang__))) -#include -#define MDBX_HAVE_C11ATOMICS -#elif defined(__GNUC__) || defined(__clang__) -#elif defined(_MSC_VER) -#pragma warning(disable : 4163) /* 'xyz': not available as an intrinsic */ -#pragma warning(disable : 4133) /* 'function': incompatible types - from \ - 'size_t' to 'LONGLONG' */ -#pragma warning(disable : 4244) /* 'return': conversion from 'LONGLONG' to \ - 'std::size_t', possible loss of data */ -#pragma warning(disable : 4267) /* 'function': conversion from 'size_t' to \ - 'long', possible loss of data */ -#pragma intrinsic(_InterlockedExchangeAdd, _InterlockedCompareExchange) -#pragma intrinsic(_InterlockedExchangeAdd64, _InterlockedCompareExchange64) -#elif defined(__APPLE__) -#include -#else -#error FIXME atomic-ops -#endif - -/*----------------------------------------------------------------------------*/ -/* Memory/Compiler barriers, cache coherence */ +/* Memory/Compiler barriers, cache coherence */ #if __has_include() #include -#elif defined(__mips) || defined(__mips__) || defined(__mips64) || \ - defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \ - defined(__MWERKS__) || defined(__sgi) +#elif defined(__mips) || defined(__mips__) || defined(__mips64) || defined(__mips64__) || defined(_M_MRX000) || \ + defined(_MIPS_) || defined(__MWERKS__) || defined(__sgi) /* MIPS should have explicit cache control */ #include #endif -MDBX_MAYBE_UNUSED static __inline void osal_compiler_barrier(void) { +MDBX_MAYBE_UNUSED static inline void osal_compiler_barrier(void) { #if defined(__clang__) || defined(__GNUC__) __asm__ __volatile__("" ::: "memory"); #elif defined(_MSC_VER) @@ -1084,18 +1047,16 @@ MDBX_MAYBE_UNUSED static __inline void osal_compiler_barrier(void) { __memory_barrier(); #elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) __compiler_barrier(); -#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ - (defined(HP_IA64) || defined(__ia64)) +#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && (defined(HP_IA64) || defined(__ia64)) _Asm_sched_fence(/* LY: no-arg meaning 'all expect ALU', e.g. 0x3D3D */); -#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \ - defined(__ppc64__) || defined(__powerpc64__) +#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || defined(__ppc64__) || defined(__powerpc64__) __fence(); #else #error "Could not guess the kind of compiler, please report to us." #endif } -MDBX_MAYBE_UNUSED static __inline void osal_memory_barrier(void) { +MDBX_MAYBE_UNUSED static inline void osal_memory_barrier(void) { #ifdef MDBX_HAVE_C11ATOMICS atomic_thread_fence(memory_order_seq_cst); #elif defined(__ATOMIC_SEQ_CST) @@ -1116,11 +1077,9 @@ MDBX_MAYBE_UNUSED static __inline void osal_memory_barrier(void) { #endif #elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) __machine_rw_barrier(); -#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ - (defined(HP_IA64) || defined(__ia64)) +#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && (defined(HP_IA64) || defined(__ia64)) _Asm_mf(); -#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \ - defined(__ppc64__) || defined(__powerpc64__) +#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || defined(__ppc64__) || defined(__powerpc64__) __lwsync(); #else #error "Could not guess the kind of compiler, please report to us." @@ -1135,7 +1094,7 @@ MDBX_MAYBE_UNUSED static __inline void osal_memory_barrier(void) { #define HAVE_SYS_TYPES_H typedef HANDLE osal_thread_t; typedef unsigned osal_thread_key_t; -#define MAP_FAILED NULL +#define MAP_FAILED nullptr #define HIGH_DWORD(v) ((DWORD)((sizeof(v) > 4) ? ((uint64_t)(v) >> 32) : 0)) #define THREAD_CALL WINAPI #define THREAD_RESULT DWORD @@ -1147,15 +1106,13 @@ typedef CRITICAL_SECTION osal_fastmutex_t; #if !defined(_MSC_VER) && !defined(__try) #define __try -#define __except(COND) if (false) +#define __except(COND) if (/* (void)(COND), */ false) #endif /* stub for MSVC's __try/__except */ #if MDBX_WITHOUT_MSVC_CRT #ifndef osal_malloc -static inline void *osal_malloc(size_t bytes) { - return HeapAlloc(GetProcessHeap(), 0, bytes); -} +static inline void *osal_malloc(size_t bytes) { return HeapAlloc(GetProcessHeap(), 0, bytes); } #endif /* osal_malloc */ #ifndef osal_calloc @@ -1166,8 +1123,7 @@ static inline void *osal_calloc(size_t nelem, size_t size) { #ifndef osal_realloc static inline void *osal_realloc(void *ptr, size_t bytes) { - return ptr ? HeapReAlloc(GetProcessHeap(), 0, ptr, bytes) - : HeapAlloc(GetProcessHeap(), 0, bytes); + return ptr ? HeapReAlloc(GetProcessHeap(), 0, ptr, bytes) : HeapAlloc(GetProcessHeap(), 0, bytes); } #endif /* osal_realloc */ @@ -1213,29 +1169,16 @@ typedef pthread_mutex_t osal_fastmutex_t; #endif /* Platform */ #if __GLIBC_PREREQ(2, 12) || defined(__FreeBSD__) || defined(malloc_usable_size) -/* malloc_usable_size() already provided */ +#define osal_malloc_usable_size(ptr) malloc_usable_size(ptr) #elif defined(__APPLE__) -#define malloc_usable_size(ptr) malloc_size(ptr) +#define osal_malloc_usable_size(ptr) malloc_size(ptr) #elif defined(_MSC_VER) && !MDBX_WITHOUT_MSVC_CRT -#define malloc_usable_size(ptr) _msize(ptr) -#endif /* malloc_usable_size */ +#define osal_malloc_usable_size(ptr) _msize(ptr) +#endif /* osal_malloc_usable_size */ /*----------------------------------------------------------------------------*/ /* OS abstraction layer stuff */ -MDBX_INTERNAL_VAR unsigned sys_pagesize; -MDBX_MAYBE_UNUSED MDBX_INTERNAL_VAR unsigned sys_pagesize_ln2, - sys_allocation_granularity; - -/* Get the size of a memory page for the system. - * This is the basic size that the platform's memory manager uses, and is - * fundamental to the use of memory-mapped files. */ -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline size_t -osal_syspagesize(void) { - assert(sys_pagesize > 0 && (sys_pagesize & (sys_pagesize - 1)) == 0); - return sys_pagesize; -} - #if defined(_WIN32) || defined(_WIN64) typedef wchar_t pathchar_t; #define MDBX_PRIsPATH "ls" @@ -1247,7 +1190,7 @@ typedef char pathchar_t; typedef struct osal_mmap { union { void *base; - struct MDBX_lockinfo *lck; + struct shared_lck *lck; }; mdbx_filehandle_t fd; size_t limit; /* mapping length, but NOT a size of file nor DB */ @@ -1258,25 +1201,6 @@ typedef struct osal_mmap { #endif } osal_mmap_t; -typedef union bin128 { - __anonymous_struct_extension__ struct { - uint64_t x, y; - }; - __anonymous_struct_extension__ struct { - uint32_t a, b, c, d; - }; -} bin128_t; - -#if defined(_WIN32) || defined(_WIN64) -typedef union osal_srwlock { - __anonymous_struct_extension__ struct { - long volatile readerCount; - long volatile writerCount; - }; - RTL_SRWLOCK native; -} osal_srwlock_t; -#endif /* Windows */ - #ifndef MDBX_HAVE_PWRITEV #if defined(_WIN32) || defined(_WIN64) @@ -1285,14 +1209,21 @@ typedef union osal_srwlock { #elif defined(__ANDROID_API__) #if __ANDROID_API__ < 24 +/* https://android-developers.googleblog.com/2017/09/introducing-android-native-development.html + * https://android.googlesource.com/platform/bionic/+/master/docs/32-bit-abi.md */ #define MDBX_HAVE_PWRITEV 0 +#if defined(_FILE_OFFSET_BITS) && _FILE_OFFSET_BITS != MDBX_WORDBITS +#error "_FILE_OFFSET_BITS != MDBX_WORDBITS and __ANDROID_API__ < 24" (_FILE_OFFSET_BITS != MDBX_WORDBITS) +#elif defined(__FILE_OFFSET_BITS) && __FILE_OFFSET_BITS != MDBX_WORDBITS +#error "__FILE_OFFSET_BITS != MDBX_WORDBITS and __ANDROID_API__ < 24" (__FILE_OFFSET_BITS != MDBX_WORDBITS) +#endif #else #define MDBX_HAVE_PWRITEV 1 #endif #elif defined(__APPLE__) || defined(__MACH__) || defined(_DARWIN_C_SOURCE) -#if defined(MAC_OS_X_VERSION_MIN_REQUIRED) && defined(MAC_OS_VERSION_11_0) && \ +#if defined(MAC_OS_X_VERSION_MIN_REQUIRED) && defined(MAC_OS_VERSION_11_0) && \ MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_VERSION_11_0 /* FIXME: add checks for IOS versions, etc */ #define MDBX_HAVE_PWRITEV 1 @@ -1310,20 +1241,20 @@ typedef union osal_srwlock { typedef struct ior_item { #if defined(_WIN32) || defined(_WIN64) OVERLAPPED ov; -#define ior_svg_gap4terminator 1 +#define ior_sgv_gap4terminator 1 #define ior_sgv_element FILE_SEGMENT_ELEMENT #else size_t offset; #if MDBX_HAVE_PWRITEV size_t sgvcnt; -#define ior_svg_gap4terminator 0 +#define ior_sgv_gap4terminator 0 #define ior_sgv_element struct iovec #endif /* MDBX_HAVE_PWRITEV */ #endif /* !Windows */ union { MDBX_val single; #if defined(ior_sgv_element) - ior_sgv_element sgv[1 + ior_svg_gap4terminator]; + ior_sgv_element sgv[1 + ior_sgv_gap4terminator]; #endif /* ior_sgv_element */ }; } ior_item_t; @@ -1359,45 +1290,33 @@ typedef struct osal_ioring { char *boundary; } osal_ioring_t; -#ifndef __cplusplus - /* Actually this is not ioring for now, but on the way. */ -MDBX_INTERNAL_FUNC int osal_ioring_create(osal_ioring_t * +MDBX_INTERNAL int osal_ioring_create(osal_ioring_t * #if defined(_WIN32) || defined(_WIN64) - , - bool enable_direct, - mdbx_filehandle_t overlapped_fd + , + bool enable_direct, mdbx_filehandle_t overlapped_fd #endif /* Windows */ ); -MDBX_INTERNAL_FUNC int osal_ioring_resize(osal_ioring_t *, size_t items); -MDBX_INTERNAL_FUNC void osal_ioring_destroy(osal_ioring_t *); -MDBX_INTERNAL_FUNC void osal_ioring_reset(osal_ioring_t *); -MDBX_INTERNAL_FUNC int osal_ioring_add(osal_ioring_t *ctx, const size_t offset, - void *data, const size_t bytes); +MDBX_INTERNAL int osal_ioring_resize(osal_ioring_t *, size_t items); +MDBX_INTERNAL void osal_ioring_destroy(osal_ioring_t *); +MDBX_INTERNAL void osal_ioring_reset(osal_ioring_t *); +MDBX_INTERNAL int osal_ioring_add(osal_ioring_t *ctx, const size_t offset, void *data, const size_t bytes); typedef struct osal_ioring_write_result { int err; unsigned wops; } osal_ioring_write_result_t; -MDBX_INTERNAL_FUNC osal_ioring_write_result_t -osal_ioring_write(osal_ioring_t *ior, mdbx_filehandle_t fd); +MDBX_INTERNAL osal_ioring_write_result_t osal_ioring_write(osal_ioring_t *ior, mdbx_filehandle_t fd); -typedef struct iov_ctx iov_ctx_t; -MDBX_INTERNAL_FUNC void osal_ioring_walk( - osal_ioring_t *ior, iov_ctx_t *ctx, - void (*callback)(iov_ctx_t *ctx, size_t offset, void *data, size_t bytes)); +MDBX_INTERNAL void osal_ioring_walk(osal_ioring_t *ior, iov_ctx_t *ctx, + void (*callback)(iov_ctx_t *ctx, size_t offset, void *data, size_t bytes)); -MDBX_MAYBE_UNUSED static inline unsigned -osal_ioring_left(const osal_ioring_t *ior) { - return ior->slots_left; -} +MDBX_MAYBE_UNUSED static inline unsigned osal_ioring_left(const osal_ioring_t *ior) { return ior->slots_left; } -MDBX_MAYBE_UNUSED static inline unsigned -osal_ioring_used(const osal_ioring_t *ior) { +MDBX_MAYBE_UNUSED static inline unsigned osal_ioring_used(const osal_ioring_t *ior) { return ior->allocated - ior->slots_left; } -MDBX_MAYBE_UNUSED static inline int -osal_ioring_prepare(osal_ioring_t *ior, size_t items, size_t bytes) { +MDBX_MAYBE_UNUSED static inline int osal_ioring_prepare(osal_ioring_t *ior, size_t items, size_t bytes) { items = (items > 32) ? items : 32; #if defined(_WIN32) || defined(_WIN64) if (ior->direct) { @@ -1416,14 +1335,12 @@ osal_ioring_prepare(osal_ioring_t *ior, size_t items, size_t bytes) { /*----------------------------------------------------------------------------*/ /* libc compatibility stuff */ -#if (!defined(__GLIBC__) && __GLIBC_PREREQ(2, 1)) && \ - (defined(_GNU_SOURCE) || defined(_BSD_SOURCE)) +#if (!defined(__GLIBC__) && __GLIBC_PREREQ(2, 1)) && (defined(_GNU_SOURCE) || defined(_BSD_SOURCE)) #define osal_asprintf asprintf #define osal_vasprintf vasprintf #else -MDBX_MAYBE_UNUSED MDBX_INTERNAL_FUNC - MDBX_PRINTF_ARGS(2, 3) int osal_asprintf(char **strp, const char *fmt, ...); -MDBX_INTERNAL_FUNC int osal_vasprintf(char **strp, const char *fmt, va_list ap); +MDBX_MAYBE_UNUSED MDBX_INTERNAL MDBX_PRINTF_ARGS(2, 3) int osal_asprintf(char **strp, const char *fmt, ...); +MDBX_INTERNAL int osal_vasprintf(char **strp, const char *fmt, va_list ap); #endif #if !defined(MADV_DODUMP) && defined(MADV_CORE) @@ -1434,8 +1351,7 @@ MDBX_INTERNAL_FUNC int osal_vasprintf(char **strp, const char *fmt, va_list ap); #define MADV_DONTDUMP MADV_NOCORE #endif /* MADV_NOCORE -> MADV_DONTDUMP */ -MDBX_MAYBE_UNUSED MDBX_INTERNAL_FUNC void osal_jitter(bool tiny); -MDBX_MAYBE_UNUSED static __inline void jitter4testing(bool tiny); +MDBX_MAYBE_UNUSED MDBX_INTERNAL void osal_jitter(bool tiny); /* max bytes to write in one call */ #if defined(_WIN64) @@ -1445,14 +1361,12 @@ MDBX_MAYBE_UNUSED static __inline void jitter4testing(bool tiny); #else #define MAX_WRITE UINT32_C(0x3f000000) -#if defined(F_GETLK64) && defined(F_SETLK64) && defined(F_SETLKW64) && \ - !defined(__ANDROID_API__) +#if defined(F_GETLK64) && defined(F_SETLK64) && defined(F_SETLKW64) && !defined(__ANDROID_API__) #define MDBX_F_SETLK F_SETLK64 #define MDBX_F_SETLKW F_SETLKW64 #define MDBX_F_GETLK F_GETLK64 -#if (__GLIBC_PREREQ(2, 28) && \ - (defined(__USE_LARGEFILE64) || defined(__LARGEFILE64_SOURCE) || \ - defined(_USE_LARGEFILE64) || defined(_LARGEFILE64_SOURCE))) || \ +#if (__GLIBC_PREREQ(2, 28) && (defined(__USE_LARGEFILE64) || defined(__LARGEFILE64_SOURCE) || \ + defined(_USE_LARGEFILE64) || defined(_LARGEFILE64_SOURCE))) || \ defined(fcntl64) #define MDBX_FCNTL fcntl64 #else @@ -1470,8 +1384,7 @@ MDBX_MAYBE_UNUSED static __inline void jitter4testing(bool tiny); #define MDBX_STRUCT_FLOCK struct flock #endif /* MDBX_F_SETLK, MDBX_F_SETLKW, MDBX_F_GETLK */ -#if defined(F_OFD_SETLK64) && defined(F_OFD_SETLKW64) && \ - defined(F_OFD_GETLK64) && !defined(__ANDROID_API__) +#if defined(F_OFD_SETLK64) && defined(F_OFD_SETLKW64) && defined(F_OFD_GETLK64) && !defined(__ANDROID_API__) #define MDBX_F_OFD_SETLK F_OFD_SETLK64 #define MDBX_F_OFD_SETLKW F_OFD_SETLKW64 #define MDBX_F_OFD_GETLK F_OFD_GETLK64 @@ -1480,23 +1393,17 @@ MDBX_MAYBE_UNUSED static __inline void jitter4testing(bool tiny); #define MDBX_F_OFD_SETLKW F_OFD_SETLKW #define MDBX_F_OFD_GETLK F_OFD_GETLK #ifndef OFF_T_MAX -#define OFF_T_MAX \ - (((sizeof(off_t) > 4) ? INT64_MAX : INT32_MAX) & ~(size_t)0xFffff) +#define OFF_T_MAX (((sizeof(off_t) > 4) ? INT64_MAX : INT32_MAX) & ~(size_t)0xFffff) #endif /* OFF_T_MAX */ #endif /* MDBX_F_OFD_SETLK64, MDBX_F_OFD_SETLKW64, MDBX_F_OFD_GETLK64 */ -#endif - -#if defined(__linux__) || defined(__gnu_linux__) -MDBX_INTERNAL_VAR uint32_t linux_kernel_version; -MDBX_INTERNAL_VAR bool mdbx_RunningOnWSL1 /* Windows Subsystem 1 for Linux */; -#endif /* Linux */ +#endif /* !Windows */ #ifndef osal_strdup LIBMDBX_API char *osal_strdup(const char *str); #endif -MDBX_MAYBE_UNUSED static __inline int osal_get_errno(void) { +MDBX_MAYBE_UNUSED static inline int osal_get_errno(void) { #if defined(_WIN32) || defined(_WIN64) DWORD rc = GetLastError(); #else @@ -1506,40 +1413,32 @@ MDBX_MAYBE_UNUSED static __inline int osal_get_errno(void) { } #ifndef osal_memalign_alloc -MDBX_INTERNAL_FUNC int osal_memalign_alloc(size_t alignment, size_t bytes, - void **result); +MDBX_INTERNAL int osal_memalign_alloc(size_t alignment, size_t bytes, void **result); #endif #ifndef osal_memalign_free -MDBX_INTERNAL_FUNC void osal_memalign_free(void *ptr); -#endif - -MDBX_INTERNAL_FUNC int osal_condpair_init(osal_condpair_t *condpair); -MDBX_INTERNAL_FUNC int osal_condpair_lock(osal_condpair_t *condpair); -MDBX_INTERNAL_FUNC int osal_condpair_unlock(osal_condpair_t *condpair); -MDBX_INTERNAL_FUNC int osal_condpair_signal(osal_condpair_t *condpair, - bool part); -MDBX_INTERNAL_FUNC int osal_condpair_wait(osal_condpair_t *condpair, bool part); -MDBX_INTERNAL_FUNC int osal_condpair_destroy(osal_condpair_t *condpair); - -MDBX_INTERNAL_FUNC int osal_fastmutex_init(osal_fastmutex_t *fastmutex); -MDBX_INTERNAL_FUNC int osal_fastmutex_acquire(osal_fastmutex_t *fastmutex); -MDBX_INTERNAL_FUNC int osal_fastmutex_release(osal_fastmutex_t *fastmutex); -MDBX_INTERNAL_FUNC int osal_fastmutex_destroy(osal_fastmutex_t *fastmutex); - -MDBX_INTERNAL_FUNC int osal_pwritev(mdbx_filehandle_t fd, struct iovec *iov, - size_t sgvcnt, uint64_t offset); -MDBX_INTERNAL_FUNC int osal_pread(mdbx_filehandle_t fd, void *buf, size_t count, - uint64_t offset); -MDBX_INTERNAL_FUNC int osal_pwrite(mdbx_filehandle_t fd, const void *buf, - size_t count, uint64_t offset); -MDBX_INTERNAL_FUNC int osal_write(mdbx_filehandle_t fd, const void *buf, - size_t count); - -MDBX_INTERNAL_FUNC int -osal_thread_create(osal_thread_t *thread, - THREAD_RESULT(THREAD_CALL *start_routine)(void *), - void *arg); -MDBX_INTERNAL_FUNC int osal_thread_join(osal_thread_t thread); +MDBX_INTERNAL void osal_memalign_free(void *ptr); +#endif + +MDBX_INTERNAL int osal_condpair_init(osal_condpair_t *condpair); +MDBX_INTERNAL int osal_condpair_lock(osal_condpair_t *condpair); +MDBX_INTERNAL int osal_condpair_unlock(osal_condpair_t *condpair); +MDBX_INTERNAL int osal_condpair_signal(osal_condpair_t *condpair, bool part); +MDBX_INTERNAL int osal_condpair_wait(osal_condpair_t *condpair, bool part); +MDBX_INTERNAL int osal_condpair_destroy(osal_condpair_t *condpair); + +MDBX_INTERNAL int osal_fastmutex_init(osal_fastmutex_t *fastmutex); +MDBX_INTERNAL int osal_fastmutex_acquire(osal_fastmutex_t *fastmutex); +MDBX_INTERNAL int osal_fastmutex_release(osal_fastmutex_t *fastmutex); +MDBX_INTERNAL int osal_fastmutex_destroy(osal_fastmutex_t *fastmutex); + +MDBX_INTERNAL int osal_pwritev(mdbx_filehandle_t fd, struct iovec *iov, size_t sgvcnt, uint64_t offset); +MDBX_INTERNAL int osal_pread(mdbx_filehandle_t fd, void *buf, size_t count, uint64_t offset); +MDBX_INTERNAL int osal_pwrite(mdbx_filehandle_t fd, const void *buf, size_t count, uint64_t offset); +MDBX_INTERNAL int osal_write(mdbx_filehandle_t fd, const void *buf, size_t count); + +MDBX_INTERNAL int osal_thread_create(osal_thread_t *thread, THREAD_RESULT(THREAD_CALL *start_routine)(void *), + void *arg); +MDBX_INTERNAL int osal_thread_join(osal_thread_t thread); enum osal_syncmode_bits { MDBX_SYNC_NONE = 0, @@ -1549,11 +1448,10 @@ enum osal_syncmode_bits { MDBX_SYNC_IODQ = 8 }; -MDBX_INTERNAL_FUNC int osal_fsync(mdbx_filehandle_t fd, - const enum osal_syncmode_bits mode_bits); -MDBX_INTERNAL_FUNC int osal_ftruncate(mdbx_filehandle_t fd, uint64_t length); -MDBX_INTERNAL_FUNC int osal_fseek(mdbx_filehandle_t fd, uint64_t pos); -MDBX_INTERNAL_FUNC int osal_filesize(mdbx_filehandle_t fd, uint64_t *length); +MDBX_INTERNAL int osal_fsync(mdbx_filehandle_t fd, const enum osal_syncmode_bits mode_bits); +MDBX_INTERNAL int osal_ftruncate(mdbx_filehandle_t fd, uint64_t length); +MDBX_INTERNAL int osal_fseek(mdbx_filehandle_t fd, uint64_t pos); +MDBX_INTERNAL int osal_filesize(mdbx_filehandle_t fd, uint64_t *length); enum osal_openfile_purpose { MDBX_OPEN_DXB_READ, @@ -1568,7 +1466,7 @@ enum osal_openfile_purpose { MDBX_OPEN_DELETE }; -MDBX_MAYBE_UNUSED static __inline bool osal_isdirsep(pathchar_t c) { +MDBX_MAYBE_UNUSED static inline bool osal_isdirsep(pathchar_t c) { return #if defined(_WIN32) || defined(_WIN64) c == '\\' || @@ -1576,50 +1474,39 @@ MDBX_MAYBE_UNUSED static __inline bool osal_isdirsep(pathchar_t c) { c == '/'; } -MDBX_INTERNAL_FUNC bool osal_pathequal(const pathchar_t *l, const pathchar_t *r, - size_t len); -MDBX_INTERNAL_FUNC pathchar_t *osal_fileext(const pathchar_t *pathname, - size_t len); -MDBX_INTERNAL_FUNC int osal_fileexists(const pathchar_t *pathname); -MDBX_INTERNAL_FUNC int osal_openfile(const enum osal_openfile_purpose purpose, - const MDBX_env *env, - const pathchar_t *pathname, - mdbx_filehandle_t *fd, - mdbx_mode_t unix_mode_bits); -MDBX_INTERNAL_FUNC int osal_closefile(mdbx_filehandle_t fd); -MDBX_INTERNAL_FUNC int osal_removefile(const pathchar_t *pathname); -MDBX_INTERNAL_FUNC int osal_removedirectory(const pathchar_t *pathname); -MDBX_INTERNAL_FUNC int osal_is_pipe(mdbx_filehandle_t fd); -MDBX_INTERNAL_FUNC int osal_lockfile(mdbx_filehandle_t fd, bool wait); +MDBX_INTERNAL bool osal_pathequal(const pathchar_t *l, const pathchar_t *r, size_t len); +MDBX_INTERNAL pathchar_t *osal_fileext(const pathchar_t *pathname, size_t len); +MDBX_INTERNAL int osal_fileexists(const pathchar_t *pathname); +MDBX_INTERNAL int osal_openfile(const enum osal_openfile_purpose purpose, const MDBX_env *env, + const pathchar_t *pathname, mdbx_filehandle_t *fd, mdbx_mode_t unix_mode_bits); +MDBX_INTERNAL int osal_closefile(mdbx_filehandle_t fd); +MDBX_INTERNAL int osal_removefile(const pathchar_t *pathname); +MDBX_INTERNAL int osal_removedirectory(const pathchar_t *pathname); +MDBX_INTERNAL int osal_is_pipe(mdbx_filehandle_t fd); +MDBX_INTERNAL int osal_lockfile(mdbx_filehandle_t fd, bool wait); #define MMAP_OPTION_TRUNCATE 1 #define MMAP_OPTION_SEMAPHORE 2 -MDBX_INTERNAL_FUNC int osal_mmap(const int flags, osal_mmap_t *map, size_t size, - const size_t limit, const unsigned options); -MDBX_INTERNAL_FUNC int osal_munmap(osal_mmap_t *map); +MDBX_INTERNAL int osal_mmap(const int flags, osal_mmap_t *map, size_t size, const size_t limit, const unsigned options, + const pathchar_t *pathname4logging); +MDBX_INTERNAL int osal_munmap(osal_mmap_t *map); #define MDBX_MRESIZE_MAY_MOVE 0x00000100 #define MDBX_MRESIZE_MAY_UNMAP 0x00000200 -MDBX_INTERNAL_FUNC int osal_mresize(const int flags, osal_mmap_t *map, - size_t size, size_t limit); +MDBX_INTERNAL int osal_mresize(const int flags, osal_mmap_t *map, size_t size, size_t limit); #if defined(_WIN32) || defined(_WIN64) typedef struct { unsigned limit, count; HANDLE handles[31]; } mdbx_handle_array_t; -MDBX_INTERNAL_FUNC int -osal_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array); -MDBX_INTERNAL_FUNC int -osal_resume_threads_after_remap(mdbx_handle_array_t *array); +MDBX_INTERNAL int osal_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array); +MDBX_INTERNAL int osal_resume_threads_after_remap(mdbx_handle_array_t *array); #endif /* Windows */ -MDBX_INTERNAL_FUNC int osal_msync(const osal_mmap_t *map, size_t offset, - size_t length, - enum osal_syncmode_bits mode_bits); -MDBX_INTERNAL_FUNC int osal_check_fs_rdonly(mdbx_filehandle_t handle, - const pathchar_t *pathname, - int err); -MDBX_INTERNAL_FUNC int osal_check_fs_incore(mdbx_filehandle_t handle); - -MDBX_MAYBE_UNUSED static __inline uint32_t osal_getpid(void) { +MDBX_INTERNAL int osal_msync(const osal_mmap_t *map, size_t offset, size_t length, enum osal_syncmode_bits mode_bits); +MDBX_INTERNAL int osal_check_fs_rdonly(mdbx_filehandle_t handle, const pathchar_t *pathname, int err); +MDBX_INTERNAL int osal_check_fs_incore(mdbx_filehandle_t handle); +MDBX_INTERNAL int osal_check_fs_local(mdbx_filehandle_t handle, int flags); + +MDBX_MAYBE_UNUSED static inline uint32_t osal_getpid(void) { STATIC_ASSERT(sizeof(mdbx_pid_t) <= sizeof(uint32_t)); #if defined(_WIN32) || defined(_WIN64) return GetCurrentProcessId(); @@ -1629,7 +1516,7 @@ MDBX_MAYBE_UNUSED static __inline uint32_t osal_getpid(void) { #endif } -MDBX_MAYBE_UNUSED static __inline uintptr_t osal_thread_self(void) { +MDBX_MAYBE_UNUSED static inline uintptr_t osal_thread_self(void) { mdbx_tid_t thunk; STATIC_ASSERT(sizeof(uintptr_t) >= sizeof(thunk)); #if defined(_WIN32) || defined(_WIN64) @@ -1642,274 +1529,51 @@ MDBX_MAYBE_UNUSED static __inline uintptr_t osal_thread_self(void) { #if !defined(_WIN32) && !defined(_WIN64) #if defined(__ANDROID_API__) || defined(ANDROID) || defined(BIONIC) -MDBX_INTERNAL_FUNC int osal_check_tid4bionic(void); +MDBX_INTERNAL int osal_check_tid4bionic(void); #else -static __inline int osal_check_tid4bionic(void) { return 0; } +static inline int osal_check_tid4bionic(void) { return 0; } #endif /* __ANDROID_API__ || ANDROID) || BIONIC */ -MDBX_MAYBE_UNUSED static __inline int -osal_pthread_mutex_lock(pthread_mutex_t *mutex) { +MDBX_MAYBE_UNUSED static inline int osal_pthread_mutex_lock(pthread_mutex_t *mutex) { int err = osal_check_tid4bionic(); return unlikely(err) ? err : pthread_mutex_lock(mutex); } #endif /* !Windows */ -MDBX_INTERNAL_FUNC uint64_t osal_monotime(void); -MDBX_INTERNAL_FUNC uint64_t osal_cputime(size_t *optional_page_faults); -MDBX_INTERNAL_FUNC uint64_t osal_16dot16_to_monotime(uint32_t seconds_16dot16); -MDBX_INTERNAL_FUNC uint32_t osal_monotime_to_16dot16(uint64_t monotime); +MDBX_INTERNAL uint64_t osal_monotime(void); +MDBX_INTERNAL uint64_t osal_cputime(size_t *optional_page_faults); +MDBX_INTERNAL uint64_t osal_16dot16_to_monotime(uint32_t seconds_16dot16); +MDBX_INTERNAL uint32_t osal_monotime_to_16dot16(uint64_t monotime); -MDBX_MAYBE_UNUSED static inline uint32_t -osal_monotime_to_16dot16_noUnderflow(uint64_t monotime) { +MDBX_MAYBE_UNUSED static inline uint32_t osal_monotime_to_16dot16_noUnderflow(uint64_t monotime) { uint32_t seconds_16dot16 = osal_monotime_to_16dot16(monotime); return seconds_16dot16 ? seconds_16dot16 : /* fix underflow */ (monotime > 0); } -MDBX_INTERNAL_FUNC bin128_t osal_bootid(void); /*----------------------------------------------------------------------------*/ -/* lck stuff */ - -/// \brief Initialization of synchronization primitives linked with MDBX_env -/// instance both in LCK-file and within the current process. -/// \param -/// global_uniqueness_flag = true - denotes that there are no other processes -/// working with DB and LCK-file. Thus the function MUST initialize -/// shared synchronization objects in memory-mapped LCK-file. -/// global_uniqueness_flag = false - denotes that at least one process is -/// already working with DB and LCK-file, including the case when DB -/// has already been opened in the current process. Thus the function -/// MUST NOT initialize shared synchronization objects in memory-mapped -/// LCK-file that are already in use. -/// \return Error code or zero on success. -MDBX_INTERNAL_FUNC int osal_lck_init(MDBX_env *env, - MDBX_env *inprocess_neighbor, - int global_uniqueness_flag); - -/// \brief Disconnects from shared interprocess objects and destructs -/// synchronization objects linked with MDBX_env instance -/// within the current process. -/// \param -/// inprocess_neighbor = NULL - if the current process does not have other -/// instances of MDBX_env linked with the DB being closed. -/// Thus the function MUST check for other processes working with DB or -/// LCK-file, and keep or destroy shared synchronization objects in -/// memory-mapped LCK-file depending on the result. -/// inprocess_neighbor = not-NULL - pointer to another instance of MDBX_env -/// (anyone of there is several) working with DB or LCK-file within the -/// current process. Thus the function MUST NOT try to acquire exclusive -/// lock and/or try to destruct shared synchronization objects linked with -/// DB or LCK-file. Moreover, the implementation MUST ensure correct work -/// of other instances of MDBX_env within the current process, e.g. -/// restore POSIX-fcntl locks after the closing of file descriptors. -/// \return Error code (MDBX_PANIC) or zero on success. -MDBX_INTERNAL_FUNC int osal_lck_destroy(MDBX_env *env, - MDBX_env *inprocess_neighbor); - -/// \brief Connects to shared interprocess locking objects and tries to acquire -/// the maximum lock level (shared if exclusive is not available) -/// Depending on implementation or/and platform (Windows) this function may -/// acquire the non-OS super-level lock (e.g. for shared synchronization -/// objects initialization), which will be downgraded to OS-exclusive or -/// shared via explicit calling of osal_lck_downgrade(). -/// \return -/// MDBX_RESULT_TRUE (-1) - if an exclusive lock was acquired and thus -/// the current process is the first and only after the last use of DB. -/// MDBX_RESULT_FALSE (0) - if a shared lock was acquired and thus -/// DB has already been opened and now is used by other processes. -/// Otherwise (not 0 and not -1) - error code. -MDBX_INTERNAL_FUNC int osal_lck_seize(MDBX_env *env); - -/// \brief Downgrades the level of initially acquired lock to -/// operational level specified by argument. The reason for such downgrade: -/// - unblocking of other processes that are waiting for access, i.e. -/// if (env->me_flags & MDBX_EXCLUSIVE) != 0, then other processes -/// should be made aware that access is unavailable rather than -/// wait for it. -/// - freeing locks that interfere file operation (especially for Windows) -/// (env->me_flags & MDBX_EXCLUSIVE) == 0 - downgrade to shared lock. -/// (env->me_flags & MDBX_EXCLUSIVE) != 0 - downgrade to exclusive -/// operational lock. -/// \return Error code or zero on success -MDBX_INTERNAL_FUNC int osal_lck_downgrade(MDBX_env *env); - -/// \brief Locks LCK-file or/and table of readers for (de)registering. -/// \return Error code or zero on success -MDBX_INTERNAL_FUNC int osal_rdt_lock(MDBX_env *env); - -/// \brief Unlocks LCK-file or/and table of readers after (de)registering. -MDBX_INTERNAL_FUNC void osal_rdt_unlock(MDBX_env *env); - -/// \brief Acquires lock for DB change (on writing transaction start) -/// Reading transactions will not be blocked. -/// Declared as LIBMDBX_API because it is used in mdbx_chk. -/// \return Error code or zero on success -LIBMDBX_API int mdbx_txn_lock(MDBX_env *env, bool dont_wait); - -/// \brief Releases lock once DB changes is made (after writing transaction -/// has finished). -/// Declared as LIBMDBX_API because it is used in mdbx_chk. -LIBMDBX_API void mdbx_txn_unlock(MDBX_env *env); - -/// \brief Sets alive-flag of reader presence (indicative lock) for PID of -/// the current process. The function does no more than needed for -/// the correct working of osal_rpid_check() in other processes. -/// \return Error code or zero on success -MDBX_INTERNAL_FUNC int osal_rpid_set(MDBX_env *env); - -/// \brief Resets alive-flag of reader presence (indicative lock) -/// for PID of the current process. The function does no more than needed -/// for the correct working of osal_rpid_check() in other processes. -/// \return Error code or zero on success -MDBX_INTERNAL_FUNC int osal_rpid_clear(MDBX_env *env); - -/// \brief Checks for reading process status with the given pid with help of -/// alive-flag of presence (indicative lock) or using another way. -/// \return -/// MDBX_RESULT_TRUE (-1) - if the reader process with the given PID is alive -/// and working with DB (indicative lock is present). -/// MDBX_RESULT_FALSE (0) - if the reader process with the given PID is absent -/// or not working with DB (indicative lock is not present). -/// Otherwise (not 0 and not -1) - error code. -MDBX_INTERNAL_FUNC int osal_rpid_check(MDBX_env *env, uint32_t pid); -#if defined(_WIN32) || defined(_WIN64) - -MDBX_INTERNAL_FUNC int osal_mb2w(const char *const src, wchar_t **const pdst); - -typedef void(WINAPI *osal_srwlock_t_function)(osal_srwlock_t *); -MDBX_INTERNAL_VAR osal_srwlock_t_function osal_srwlock_Init, - osal_srwlock_AcquireShared, osal_srwlock_ReleaseShared, - osal_srwlock_AcquireExclusive, osal_srwlock_ReleaseExclusive; - -#if _WIN32_WINNT < 0x0600 /* prior to Windows Vista */ -typedef enum _FILE_INFO_BY_HANDLE_CLASS { - FileBasicInfo, - FileStandardInfo, - FileNameInfo, - FileRenameInfo, - FileDispositionInfo, - FileAllocationInfo, - FileEndOfFileInfo, - FileStreamInfo, - FileCompressionInfo, - FileAttributeTagInfo, - FileIdBothDirectoryInfo, - FileIdBothDirectoryRestartInfo, - FileIoPriorityHintInfo, - FileRemoteProtocolInfo, - MaximumFileInfoByHandleClass -} FILE_INFO_BY_HANDLE_CLASS, - *PFILE_INFO_BY_HANDLE_CLASS; - -typedef struct _FILE_END_OF_FILE_INFO { - LARGE_INTEGER EndOfFile; -} FILE_END_OF_FILE_INFO, *PFILE_END_OF_FILE_INFO; - -#define REMOTE_PROTOCOL_INFO_FLAG_LOOPBACK 0x00000001 -#define REMOTE_PROTOCOL_INFO_FLAG_OFFLINE 0x00000002 - -typedef struct _FILE_REMOTE_PROTOCOL_INFO { - USHORT StructureVersion; - USHORT StructureSize; - DWORD Protocol; - USHORT ProtocolMajorVersion; - USHORT ProtocolMinorVersion; - USHORT ProtocolRevision; - USHORT Reserved; - DWORD Flags; - struct { - DWORD Reserved[8]; - } GenericReserved; - struct { - DWORD Reserved[16]; - } ProtocolSpecificReserved; -} FILE_REMOTE_PROTOCOL_INFO, *PFILE_REMOTE_PROTOCOL_INFO; - -#endif /* _WIN32_WINNT < 0x0600 (prior to Windows Vista) */ - -typedef BOOL(WINAPI *MDBX_GetFileInformationByHandleEx)( - _In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass, - _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize); -MDBX_INTERNAL_VAR MDBX_GetFileInformationByHandleEx - mdbx_GetFileInformationByHandleEx; - -typedef BOOL(WINAPI *MDBX_GetVolumeInformationByHandleW)( - _In_ HANDLE hFile, _Out_opt_ LPWSTR lpVolumeNameBuffer, - _In_ DWORD nVolumeNameSize, _Out_opt_ LPDWORD lpVolumeSerialNumber, - _Out_opt_ LPDWORD lpMaximumComponentLength, - _Out_opt_ LPDWORD lpFileSystemFlags, - _Out_opt_ LPWSTR lpFileSystemNameBuffer, _In_ DWORD nFileSystemNameSize); -MDBX_INTERNAL_VAR MDBX_GetVolumeInformationByHandleW - mdbx_GetVolumeInformationByHandleW; - -typedef DWORD(WINAPI *MDBX_GetFinalPathNameByHandleW)(_In_ HANDLE hFile, - _Out_ LPWSTR lpszFilePath, - _In_ DWORD cchFilePath, - _In_ DWORD dwFlags); -MDBX_INTERNAL_VAR MDBX_GetFinalPathNameByHandleW mdbx_GetFinalPathNameByHandleW; - -typedef BOOL(WINAPI *MDBX_SetFileInformationByHandle)( - _In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass, - _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize); -MDBX_INTERNAL_VAR MDBX_SetFileInformationByHandle - mdbx_SetFileInformationByHandle; - -typedef NTSTATUS(NTAPI *MDBX_NtFsControlFile)( - IN HANDLE FileHandle, IN OUT HANDLE Event, - IN OUT PVOID /* PIO_APC_ROUTINE */ ApcRoutine, IN OUT PVOID ApcContext, - OUT PIO_STATUS_BLOCK IoStatusBlock, IN ULONG FsControlCode, - IN OUT PVOID InputBuffer, IN ULONG InputBufferLength, - OUT OPTIONAL PVOID OutputBuffer, IN ULONG OutputBufferLength); -MDBX_INTERNAL_VAR MDBX_NtFsControlFile mdbx_NtFsControlFile; - -typedef uint64_t(WINAPI *MDBX_GetTickCount64)(void); -MDBX_INTERNAL_VAR MDBX_GetTickCount64 mdbx_GetTickCount64; - -#if !defined(_WIN32_WINNT_WIN8) || _WIN32_WINNT < _WIN32_WINNT_WIN8 -typedef struct _WIN32_MEMORY_RANGE_ENTRY { - PVOID VirtualAddress; - SIZE_T NumberOfBytes; -} WIN32_MEMORY_RANGE_ENTRY, *PWIN32_MEMORY_RANGE_ENTRY; -#endif /* Windows 8.x */ - -typedef BOOL(WINAPI *MDBX_PrefetchVirtualMemory)( - HANDLE hProcess, ULONG_PTR NumberOfEntries, - PWIN32_MEMORY_RANGE_ENTRY VirtualAddresses, ULONG Flags); -MDBX_INTERNAL_VAR MDBX_PrefetchVirtualMemory mdbx_PrefetchVirtualMemory; - -typedef enum _SECTION_INHERIT { ViewShare = 1, ViewUnmap = 2 } SECTION_INHERIT; - -typedef NTSTATUS(NTAPI *MDBX_NtExtendSection)(IN HANDLE SectionHandle, - IN PLARGE_INTEGER NewSectionSize); -MDBX_INTERNAL_VAR MDBX_NtExtendSection mdbx_NtExtendSection; - -static __inline bool mdbx_RunningUnderWine(void) { - return !mdbx_NtExtendSection; -} - -typedef LSTATUS(WINAPI *MDBX_RegGetValueA)(HKEY hkey, LPCSTR lpSubKey, - LPCSTR lpValue, DWORD dwFlags, - LPDWORD pdwType, PVOID pvData, - LPDWORD pcbData); -MDBX_INTERNAL_VAR MDBX_RegGetValueA mdbx_RegGetValueA; - -NTSYSAPI ULONG RtlRandomEx(PULONG Seed); - -typedef BOOL(WINAPI *MDBX_SetFileIoOverlappedRange)(HANDLE FileHandle, - PUCHAR OverlappedRangeStart, - ULONG Length); -MDBX_INTERNAL_VAR MDBX_SetFileIoOverlappedRange mdbx_SetFileIoOverlappedRange; +MDBX_INTERNAL void osal_ctor(void); +MDBX_INTERNAL void osal_dtor(void); +#if defined(_WIN32) || defined(_WIN64) +MDBX_INTERNAL int osal_mb2w(const char *const src, wchar_t **const pdst); #endif /* Windows */ -#endif /* !__cplusplus */ +typedef union bin128 { + __anonymous_struct_extension__ struct { + uint64_t x, y; + }; + __anonymous_struct_extension__ struct { + uint32_t a, b, c, d; + }; +} bin128_t; + +MDBX_INTERNAL bin128_t osal_guid(const MDBX_env *); /*----------------------------------------------------------------------------*/ -MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static __always_inline uint64_t -osal_bswap64(uint64_t v) { -#if __GNUC_PREREQ(4, 4) || __CLANG_PREREQ(4, 0) || \ - __has_builtin(__builtin_bswap64) +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint64_t osal_bswap64(uint64_t v) { +#if __GNUC_PREREQ(4, 4) || __CLANG_PREREQ(4, 0) || __has_builtin(__builtin_bswap64) return __builtin_bswap64(v); #elif defined(_MSC_VER) && !defined(__clang__) return _byteswap_uint64(v); @@ -1918,19 +1582,14 @@ osal_bswap64(uint64_t v) { #elif defined(bswap_64) return bswap_64(v); #else - return v << 56 | v >> 56 | ((v << 40) & UINT64_C(0x00ff000000000000)) | - ((v << 24) & UINT64_C(0x0000ff0000000000)) | - ((v << 8) & UINT64_C(0x000000ff00000000)) | - ((v >> 8) & UINT64_C(0x00000000ff000000)) | - ((v >> 24) & UINT64_C(0x0000000000ff0000)) | - ((v >> 40) & UINT64_C(0x000000000000ff00)); + return v << 56 | v >> 56 | ((v << 40) & UINT64_C(0x00ff000000000000)) | ((v << 24) & UINT64_C(0x0000ff0000000000)) | + ((v << 8) & UINT64_C(0x000000ff00000000)) | ((v >> 8) & UINT64_C(0x00000000ff000000)) | + ((v >> 24) & UINT64_C(0x0000000000ff0000)) | ((v >> 40) & UINT64_C(0x000000000000ff00)); #endif } -MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static __always_inline uint32_t -osal_bswap32(uint32_t v) { -#if __GNUC_PREREQ(4, 4) || __CLANG_PREREQ(4, 0) || \ - __has_builtin(__builtin_bswap32) +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint32_t osal_bswap32(uint32_t v) { +#if __GNUC_PREREQ(4, 4) || __CLANG_PREREQ(4, 0) || __has_builtin(__builtin_bswap32) return __builtin_bswap32(v); #elif defined(_MSC_VER) && !defined(__clang__) return _byteswap_ulong(v); @@ -1939,50 +1598,14 @@ osal_bswap32(uint32_t v) { #elif defined(bswap_32) return bswap_32(v); #else - return v << 24 | v >> 24 | ((v << 8) & UINT32_C(0x00ff0000)) | - ((v >> 8) & UINT32_C(0x0000ff00)); + return v << 24 | v >> 24 | ((v << 8) & UINT32_C(0x00ff0000)) | ((v >> 8) & UINT32_C(0x0000ff00)); #endif } -/*----------------------------------------------------------------------------*/ - -#if defined(_MSC_VER) && _MSC_VER >= 1900 -/* LY: MSVC 2015/2017/2019 has buggy/inconsistent PRIuPTR/PRIxPTR macros - * for internal format-args checker. */ -#undef PRIuPTR -#undef PRIiPTR -#undef PRIdPTR -#undef PRIxPTR -#define PRIuPTR "Iu" -#define PRIiPTR "Ii" -#define PRIdPTR "Id" -#define PRIxPTR "Ix" -#define PRIuSIZE "zu" -#define PRIiSIZE "zi" -#define PRIdSIZE "zd" -#define PRIxSIZE "zx" -#endif /* fix PRI*PTR for _MSC_VER */ - -#ifndef PRIuSIZE -#define PRIuSIZE PRIuPTR -#define PRIiSIZE PRIiPTR -#define PRIdSIZE PRIdPTR -#define PRIxSIZE PRIxPTR -#endif /* PRI*SIZE macros for MSVC */ - -#ifdef _MSC_VER -#pragma warning(pop) -#endif - -#define mdbx_sourcery_anchor XCONCAT(mdbx_sourcery_, MDBX_BUILD_SOURCERY) -#if defined(xMDBX_TOOLS) -extern LIBMDBX_API const char *const mdbx_sourcery_anchor; -#endif - /******************************************************************************* - ******************************************************************************* ******************************************************************************* * + * BUILD TIME * * #### ##### ##### # #### # # #### * # # # # # # # # ## # # @@ -2003,23 +1626,15 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Using fsync() with chance of data lost on power failure */ #define MDBX_OSX_WANNA_SPEED 1 -#ifndef MDBX_OSX_SPEED_INSTEADOF_DURABILITY +#ifndef MDBX_APPLE_SPEED_INSTEADOF_DURABILITY /** Choices \ref MDBX_OSX_WANNA_DURABILITY or \ref MDBX_OSX_WANNA_SPEED * for OSX & iOS */ -#define MDBX_OSX_SPEED_INSTEADOF_DURABILITY MDBX_OSX_WANNA_DURABILITY -#endif /* MDBX_OSX_SPEED_INSTEADOF_DURABILITY */ - -/** Controls using of POSIX' madvise() and/or similar hints. */ -#ifndef MDBX_ENABLE_MADVISE -#define MDBX_ENABLE_MADVISE 1 -#elif !(MDBX_ENABLE_MADVISE == 0 || MDBX_ENABLE_MADVISE == 1) -#error MDBX_ENABLE_MADVISE must be defined as 0 or 1 -#endif /* MDBX_ENABLE_MADVISE */ +#define MDBX_APPLE_SPEED_INSTEADOF_DURABILITY MDBX_OSX_WANNA_DURABILITY +#endif /* MDBX_APPLE_SPEED_INSTEADOF_DURABILITY */ /** Controls checking PID against reuse DB environment after the fork() */ #ifndef MDBX_ENV_CHECKPID -#if (defined(MADV_DONTFORK) && MDBX_ENABLE_MADVISE) || defined(_WIN32) || \ - defined(_WIN64) +#if defined(MADV_DONTFORK) || defined(_WIN32) || defined(_WIN64) /* PID check could be omitted: * - on Linux when madvise(MADV_DONTFORK) is available, i.e. after the fork() * mapped pages will not be available for child process. @@ -2048,8 +1663,7 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Does a system have battery-backed Real-Time Clock or just a fake. */ #ifndef MDBX_TRUST_RTC -#if defined(__linux__) || defined(__gnu_linux__) || defined(__NetBSD__) || \ - defined(__OpenBSD__) +#if defined(__linux__) || defined(__gnu_linux__) || defined(__NetBSD__) || defined(__OpenBSD__) #define MDBX_TRUST_RTC 0 /* a lot of embedded systems have a fake RTC */ #else #define MDBX_TRUST_RTC 1 @@ -2084,24 +1698,21 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Controls using Unix' mincore() to determine whether DB-pages * are resident in memory. */ -#ifndef MDBX_ENABLE_MINCORE +#ifndef MDBX_USE_MINCORE #if defined(MINCORE_INCORE) || !(defined(_WIN32) || defined(_WIN64)) -#define MDBX_ENABLE_MINCORE 1 +#define MDBX_USE_MINCORE 1 #else -#define MDBX_ENABLE_MINCORE 0 +#define MDBX_USE_MINCORE 0 #endif -#elif !(MDBX_ENABLE_MINCORE == 0 || MDBX_ENABLE_MINCORE == 1) -#error MDBX_ENABLE_MINCORE must be defined as 0 or 1 -#endif /* MDBX_ENABLE_MINCORE */ +#define MDBX_USE_MINCORE_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_USE_MINCORE) +#elif !(MDBX_USE_MINCORE == 0 || MDBX_USE_MINCORE == 1) +#error MDBX_USE_MINCORE must be defined as 0 or 1 +#endif /* MDBX_USE_MINCORE */ /** Enables chunking long list of retired pages during huge transactions commit * to avoid use sequences of pages. */ #ifndef MDBX_ENABLE_BIGFOOT -#if MDBX_WORDBITS >= 64 || defined(DOXYGEN) #define MDBX_ENABLE_BIGFOOT 1 -#else -#define MDBX_ENABLE_BIGFOOT 0 -#endif #elif !(MDBX_ENABLE_BIGFOOT == 0 || MDBX_ENABLE_BIGFOOT == 1) #error MDBX_ENABLE_BIGFOOT must be defined as 0 or 1 #endif /* MDBX_ENABLE_BIGFOOT */ @@ -2116,25 +1727,27 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #ifndef MDBX_PNL_PREALLOC_FOR_RADIXSORT #define MDBX_PNL_PREALLOC_FOR_RADIXSORT 1 -#elif !(MDBX_PNL_PREALLOC_FOR_RADIXSORT == 0 || \ - MDBX_PNL_PREALLOC_FOR_RADIXSORT == 1) +#elif !(MDBX_PNL_PREALLOC_FOR_RADIXSORT == 0 || MDBX_PNL_PREALLOC_FOR_RADIXSORT == 1) #error MDBX_PNL_PREALLOC_FOR_RADIXSORT must be defined as 0 or 1 #endif /* MDBX_PNL_PREALLOC_FOR_RADIXSORT */ #ifndef MDBX_DPL_PREALLOC_FOR_RADIXSORT #define MDBX_DPL_PREALLOC_FOR_RADIXSORT 1 -#elif !(MDBX_DPL_PREALLOC_FOR_RADIXSORT == 0 || \ - MDBX_DPL_PREALLOC_FOR_RADIXSORT == 1) +#elif !(MDBX_DPL_PREALLOC_FOR_RADIXSORT == 0 || MDBX_DPL_PREALLOC_FOR_RADIXSORT == 1) #error MDBX_DPL_PREALLOC_FOR_RADIXSORT must be defined as 0 or 1 #endif /* MDBX_DPL_PREALLOC_FOR_RADIXSORT */ -/** Controls dirty pages tracking, spilling and persisting in MDBX_WRITEMAP - * mode. 0/OFF = Don't track dirty pages at all, don't spill ones, and use - * msync() to persist data. This is by-default on Linux and other systems where - * kernel provides properly LRU tracking and effective flushing on-demand. 1/ON - * = Tracking of dirty pages but with LRU labels for spilling and explicit - * persist ones by write(). This may be reasonable for systems which low - * performance of msync() and/or LRU tracking. */ +/** Controls dirty pages tracking, spilling and persisting in `MDBX_WRITEMAP` + * mode, i.e. disables in-memory database updating with consequent + * flush-to-disk/msync syscall. + * + * 0/OFF = Don't track dirty pages at all, don't spill ones, and use msync() to + * persist data. This is by-default on Linux and other systems where kernel + * provides properly LRU tracking and effective flushing on-demand. + * + * 1/ON = Tracking of dirty pages but with LRU labels for spilling and explicit + * persist ones by write(). This may be reasonable for goofy systems (Windows) + * which low performance of msync() and/or zany LRU tracking. */ #ifndef MDBX_AVOID_MSYNC #if defined(_WIN32) || defined(_WIN64) #define MDBX_AVOID_MSYNC 1 @@ -2145,6 +1758,22 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #error MDBX_AVOID_MSYNC must be defined as 0 or 1 #endif /* MDBX_AVOID_MSYNC */ +/** Управляет механизмом поддержки разреженных наборов DBI-хендлов для снижения + * накладных расходов при запуске и обработке транзакций. */ +#ifndef MDBX_ENABLE_DBI_SPARSE +#define MDBX_ENABLE_DBI_SPARSE 1 +#elif !(MDBX_ENABLE_DBI_SPARSE == 0 || MDBX_ENABLE_DBI_SPARSE == 1) +#error MDBX_ENABLE_DBI_SPARSE must be defined as 0 or 1 +#endif /* MDBX_ENABLE_DBI_SPARSE */ + +/** Управляет механизмом отложенного освобождения и поддержки пути быстрого + * открытия DBI-хендлов без захвата блокировок. */ +#ifndef MDBX_ENABLE_DBI_LOCKFREE +#define MDBX_ENABLE_DBI_LOCKFREE 1 +#elif !(MDBX_ENABLE_DBI_LOCKFREE == 0 || MDBX_ENABLE_DBI_LOCKFREE == 1) +#error MDBX_ENABLE_DBI_LOCKFREE must be defined as 0 or 1 +#endif /* MDBX_ENABLE_DBI_LOCKFREE */ + /** Controls sort order of internal page number lists. * This mostly experimental/advanced option with not for regular MDBX users. * \warning The database format depend on this option and libmdbx built with @@ -2157,7 +1786,11 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Avoid dependence from MSVC CRT and use ntdll.dll instead. */ #ifndef MDBX_WITHOUT_MSVC_CRT +#if defined(MDBX_BUILD_CXX) && !MDBX_BUILD_CXX #define MDBX_WITHOUT_MSVC_CRT 1 +#else +#define MDBX_WITHOUT_MSVC_CRT 0 +#endif #elif !(MDBX_WITHOUT_MSVC_CRT == 0 || MDBX_WITHOUT_MSVC_CRT == 1) #error MDBX_WITHOUT_MSVC_CRT must be defined as 0 or 1 #endif /* MDBX_WITHOUT_MSVC_CRT */ @@ -2165,12 +1798,11 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Size of buffer used during copying a environment/database file. */ #ifndef MDBX_ENVCOPY_WRITEBUF #define MDBX_ENVCOPY_WRITEBUF 1048576u -#elif MDBX_ENVCOPY_WRITEBUF < 65536u || MDBX_ENVCOPY_WRITEBUF > 1073741824u || \ - MDBX_ENVCOPY_WRITEBUF % 65536u +#elif MDBX_ENVCOPY_WRITEBUF < 65536u || MDBX_ENVCOPY_WRITEBUF > 1073741824u || MDBX_ENVCOPY_WRITEBUF % 65536u #error MDBX_ENVCOPY_WRITEBUF must be defined in range 65536..1073741824 and be multiple of 65536 #endif /* MDBX_ENVCOPY_WRITEBUF */ -/** Forces assertion checking */ +/** Forces assertion checking. */ #ifndef MDBX_FORCE_ASSERTIONS #define MDBX_FORCE_ASSERTIONS 0 #elif !(MDBX_FORCE_ASSERTIONS == 0 || MDBX_FORCE_ASSERTIONS == 1) @@ -2185,15 +1817,14 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #else #define MDBX_ASSUME_MALLOC_OVERHEAD (sizeof(void *) * 2u) #endif -#elif MDBX_ASSUME_MALLOC_OVERHEAD < 0 || MDBX_ASSUME_MALLOC_OVERHEAD > 64 || \ - MDBX_ASSUME_MALLOC_OVERHEAD % 4 +#elif MDBX_ASSUME_MALLOC_OVERHEAD < 0 || MDBX_ASSUME_MALLOC_OVERHEAD > 64 || MDBX_ASSUME_MALLOC_OVERHEAD % 4 #error MDBX_ASSUME_MALLOC_OVERHEAD must be defined in range 0..64 and be multiple of 4 #endif /* MDBX_ASSUME_MALLOC_OVERHEAD */ /** If defined then enables integration with Valgrind, * a memory analyzing tool. */ -#ifndef MDBX_USE_VALGRIND -#endif /* MDBX_USE_VALGRIND */ +#ifndef ENABLE_MEMCHECK +#endif /* ENABLE_MEMCHECK */ /** If defined then enables use C11 atomics, * otherwise detects ones availability automatically. */ @@ -2213,18 +1844,24 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #define MDBX_HAVE_BUILTIN_CPU_SUPPORTS 0 #elif defined(__e2k__) #define MDBX_HAVE_BUILTIN_CPU_SUPPORTS 0 -#elif __has_builtin(__builtin_cpu_supports) || \ - defined(__BUILTIN_CPU_SUPPORTS__) || \ +#elif __has_builtin(__builtin_cpu_supports) || defined(__BUILTIN_CPU_SUPPORTS__) || \ (defined(__ia32__) && __GNUC_PREREQ(4, 8) && __GLIBC_PREREQ(2, 23)) #define MDBX_HAVE_BUILTIN_CPU_SUPPORTS 1 #else #define MDBX_HAVE_BUILTIN_CPU_SUPPORTS 0 #endif -#elif !(MDBX_HAVE_BUILTIN_CPU_SUPPORTS == 0 || \ - MDBX_HAVE_BUILTIN_CPU_SUPPORTS == 1) +#elif !(MDBX_HAVE_BUILTIN_CPU_SUPPORTS == 0 || MDBX_HAVE_BUILTIN_CPU_SUPPORTS == 1) #error MDBX_HAVE_BUILTIN_CPU_SUPPORTS must be defined as 0 or 1 #endif /* MDBX_HAVE_BUILTIN_CPU_SUPPORTS */ +/** if enabled then instead of the returned error `MDBX_REMOTE`, only a warning is issued, when + * the database being opened in non-read-only mode is located in a file system exported via NFS. */ +#ifndef MDBX_ENABLE_NON_READONLY_EXPORT +#define MDBX_ENABLE_NON_READONLY_EXPORT 0 +#elif !(MDBX_ENABLE_NON_READONLY_EXPORT == 0 || MDBX_ENABLE_NON_READONLY_EXPORT == 1) +#error MDBX_ENABLE_NON_READONLY_EXPORT must be defined as 0 or 1 +#endif /* MDBX_ENABLE_NON_READONLY_EXPORT */ + //------------------------------------------------------------------------------ /** Win32 File Locking API for \ref MDBX_LOCKING */ @@ -2242,27 +1879,20 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** POSIX-2008 Robust Mutexes for \ref MDBX_LOCKING */ #define MDBX_LOCKING_POSIX2008 2008 -/** BeOS Benaphores, aka Futexes for \ref MDBX_LOCKING */ -#define MDBX_LOCKING_BENAPHORE 1995 - /** Advanced: Choices the locking implementation (autodetection by default). */ #if defined(_WIN32) || defined(_WIN64) #define MDBX_LOCKING MDBX_LOCKING_WIN32FILES #else #ifndef MDBX_LOCKING -#if defined(_POSIX_THREAD_PROCESS_SHARED) && \ - _POSIX_THREAD_PROCESS_SHARED >= 200112L && !defined(__FreeBSD__) +#if defined(_POSIX_THREAD_PROCESS_SHARED) && _POSIX_THREAD_PROCESS_SHARED >= 200112L && !defined(__FreeBSD__) /* Some platforms define the EOWNERDEAD error code even though they * don't support Robust Mutexes. If doubt compile with -MDBX_LOCKING=2001. */ -#if defined(EOWNERDEAD) && _POSIX_THREAD_PROCESS_SHARED >= 200809L && \ - ((defined(_POSIX_THREAD_ROBUST_PRIO_INHERIT) && \ - _POSIX_THREAD_ROBUST_PRIO_INHERIT > 0) || \ - (defined(_POSIX_THREAD_ROBUST_PRIO_PROTECT) && \ - _POSIX_THREAD_ROBUST_PRIO_PROTECT > 0) || \ - defined(PTHREAD_MUTEX_ROBUST) || defined(PTHREAD_MUTEX_ROBUST_NP)) && \ - (!defined(__GLIBC__) || \ - __GLIBC_PREREQ(2, 10) /* troubles with Robust mutexes before 2.10 */) +#if defined(EOWNERDEAD) && _POSIX_THREAD_PROCESS_SHARED >= 200809L && \ + ((defined(_POSIX_THREAD_ROBUST_PRIO_INHERIT) && _POSIX_THREAD_ROBUST_PRIO_INHERIT > 0) || \ + (defined(_POSIX_THREAD_ROBUST_PRIO_PROTECT) && _POSIX_THREAD_ROBUST_PRIO_PROTECT > 0) || \ + defined(PTHREAD_MUTEX_ROBUST) || defined(PTHREAD_MUTEX_ROBUST_NP)) && \ + (!defined(__GLIBC__) || __GLIBC_PREREQ(2, 10) /* troubles with Robust mutexes before 2.10 */) #define MDBX_LOCKING MDBX_LOCKING_POSIX2008 #else #define MDBX_LOCKING MDBX_LOCKING_POSIX2001 @@ -2280,12 +1910,9 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Advanced: Using POSIX OFD-locks (autodetection by default). */ #ifndef MDBX_USE_OFDLOCKS -#if ((defined(F_OFD_SETLK) && defined(F_OFD_SETLKW) && \ - defined(F_OFD_GETLK)) || \ - (defined(F_OFD_SETLK64) && defined(F_OFD_SETLKW64) && \ - defined(F_OFD_GETLK64))) && \ - !defined(MDBX_SAFE4QEMU) && \ - !defined(__sun) /* OFD-lock are broken on Solaris */ +#if ((defined(F_OFD_SETLK) && defined(F_OFD_SETLKW) && defined(F_OFD_GETLK)) || \ + (defined(F_OFD_SETLK64) && defined(F_OFD_SETLKW64) && defined(F_OFD_GETLK64))) && \ + !defined(MDBX_SAFE4QEMU) && !defined(__sun) /* OFD-lock are broken on Solaris */ #define MDBX_USE_OFDLOCKS 1 #else #define MDBX_USE_OFDLOCKS 0 @@ -2299,8 +1926,7 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Advanced: Using sendfile() syscall (autodetection by default). */ #ifndef MDBX_USE_SENDFILE -#if ((defined(__linux__) || defined(__gnu_linux__)) && \ - !defined(__ANDROID_API__)) || \ +#if ((defined(__linux__) || defined(__gnu_linux__)) && !defined(__ANDROID_API__)) || \ (defined(__ANDROID_API__) && __ANDROID_API__ >= 21) #define MDBX_USE_SENDFILE 1 #else @@ -2321,30 +1947,15 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #error MDBX_USE_COPYFILERANGE must be defined as 0 or 1 #endif /* MDBX_USE_COPYFILERANGE */ -/** Advanced: Using sync_file_range() syscall (autodetection by default). */ -#ifndef MDBX_USE_SYNCFILERANGE -#if ((defined(__linux__) || defined(__gnu_linux__)) && \ - defined(SYNC_FILE_RANGE_WRITE) && !defined(__ANDROID_API__)) || \ - (defined(__ANDROID_API__) && __ANDROID_API__ >= 26) -#define MDBX_USE_SYNCFILERANGE 1 -#else -#define MDBX_USE_SYNCFILERANGE 0 -#endif -#elif !(MDBX_USE_SYNCFILERANGE == 0 || MDBX_USE_SYNCFILERANGE == 1) -#error MDBX_USE_SYNCFILERANGE must be defined as 0 or 1 -#endif /* MDBX_USE_SYNCFILERANGE */ - //------------------------------------------------------------------------------ #ifndef MDBX_CPU_WRITEBACK_INCOHERENT -#if defined(__ia32__) || defined(__e2k__) || defined(__hppa) || \ - defined(__hppa__) || defined(DOXYGEN) +#if defined(__ia32__) || defined(__e2k__) || defined(__hppa) || defined(__hppa__) || defined(DOXYGEN) #define MDBX_CPU_WRITEBACK_INCOHERENT 0 #else #define MDBX_CPU_WRITEBACK_INCOHERENT 1 #endif -#elif !(MDBX_CPU_WRITEBACK_INCOHERENT == 0 || \ - MDBX_CPU_WRITEBACK_INCOHERENT == 1) +#elif !(MDBX_CPU_WRITEBACK_INCOHERENT == 0 || MDBX_CPU_WRITEBACK_INCOHERENT == 1) #error MDBX_CPU_WRITEBACK_INCOHERENT must be defined as 0 or 1 #endif /* MDBX_CPU_WRITEBACK_INCOHERENT */ @@ -2354,35 +1965,35 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #else #define MDBX_MMAP_INCOHERENT_FILE_WRITE 0 #endif -#elif !(MDBX_MMAP_INCOHERENT_FILE_WRITE == 0 || \ - MDBX_MMAP_INCOHERENT_FILE_WRITE == 1) +#elif !(MDBX_MMAP_INCOHERENT_FILE_WRITE == 0 || MDBX_MMAP_INCOHERENT_FILE_WRITE == 1) #error MDBX_MMAP_INCOHERENT_FILE_WRITE must be defined as 0 or 1 #endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */ #ifndef MDBX_MMAP_INCOHERENT_CPU_CACHE -#if defined(__mips) || defined(__mips__) || defined(__mips64) || \ - defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \ - defined(__MWERKS__) || defined(__sgi) +#if defined(__mips) || defined(__mips__) || defined(__mips64) || defined(__mips64__) || defined(_M_MRX000) || \ + defined(_MIPS_) || defined(__MWERKS__) || defined(__sgi) /* MIPS has cache coherency issues. */ #define MDBX_MMAP_INCOHERENT_CPU_CACHE 1 #else /* LY: assume no relevant mmap/dcache issues. */ #define MDBX_MMAP_INCOHERENT_CPU_CACHE 0 #endif -#elif !(MDBX_MMAP_INCOHERENT_CPU_CACHE == 0 || \ - MDBX_MMAP_INCOHERENT_CPU_CACHE == 1) +#elif !(MDBX_MMAP_INCOHERENT_CPU_CACHE == 0 || MDBX_MMAP_INCOHERENT_CPU_CACHE == 1) #error MDBX_MMAP_INCOHERENT_CPU_CACHE must be defined as 0 or 1 #endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */ -#ifndef MDBX_MMAP_USE_MS_ASYNC -#if MDBX_MMAP_INCOHERENT_FILE_WRITE || MDBX_MMAP_INCOHERENT_CPU_CACHE -#define MDBX_MMAP_USE_MS_ASYNC 1 +/** Assume system needs explicit syscall to sync/flush/write modified mapped + * memory. */ +#ifndef MDBX_MMAP_NEEDS_JOLT +#if MDBX_MMAP_INCOHERENT_FILE_WRITE || MDBX_MMAP_INCOHERENT_CPU_CACHE || !(defined(__linux__) || defined(__gnu_linux__)) +#define MDBX_MMAP_NEEDS_JOLT 1 #else -#define MDBX_MMAP_USE_MS_ASYNC 0 +#define MDBX_MMAP_NEEDS_JOLT 0 #endif -#elif !(MDBX_MMAP_USE_MS_ASYNC == 0 || MDBX_MMAP_USE_MS_ASYNC == 1) -#error MDBX_MMAP_USE_MS_ASYNC must be defined as 0 or 1 -#endif /* MDBX_MMAP_USE_MS_ASYNC */ +#define MDBX_MMAP_NEEDS_JOLT_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_MMAP_NEEDS_JOLT) +#elif !(MDBX_MMAP_NEEDS_JOLT == 0 || MDBX_MMAP_NEEDS_JOLT == 1) +#error MDBX_MMAP_NEEDS_JOLT must be defined as 0 or 1 +#endif /* MDBX_MMAP_NEEDS_JOLT */ #ifndef MDBX_64BIT_ATOMIC #if MDBX_WORDBITS >= 64 || defined(DOXYGEN) @@ -2429,8 +2040,7 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #endif /* MDBX_64BIT_CAS */ #ifndef MDBX_UNALIGNED_OK -#if defined(__ALIGNED__) || defined(__SANITIZE_UNDEFINED__) || \ - defined(ENABLE_UBSAN) +#if defined(__ALIGNED__) || defined(__SANITIZE_UNDEFINED__) || defined(ENABLE_UBSAN) #define MDBX_UNALIGNED_OK 0 /* no unaligned access allowed */ #elif defined(__ARM_FEATURE_UNALIGNED) #define MDBX_UNALIGNED_OK 4 /* ok unaligned for 32-bit words */ @@ -2464,6 +2074,19 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #endif #endif /* MDBX_CACHELINE_SIZE */ +/* Max length of iov-vector passed to writev() call, used for auxilary writes */ +#ifndef MDBX_AUXILARY_IOV_MAX +#define MDBX_AUXILARY_IOV_MAX 64 +#endif +#if defined(IOV_MAX) && IOV_MAX < MDBX_AUXILARY_IOV_MAX +#undef MDBX_AUXILARY_IOV_MAX +#define MDBX_AUXILARY_IOV_MAX IOV_MAX +#endif /* MDBX_AUXILARY_IOV_MAX */ + +/* An extra/custom information provided during library build */ +#ifndef MDBX_BUILD_METADATA +#define MDBX_BUILD_METADATA "" +#endif /* MDBX_BUILD_METADATA */ /** @} end of build options */ /******************************************************************************* ******************************************************************************* @@ -2478,6 +2101,9 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #else #define MDBX_DEBUG 1 #endif +#endif +#if MDBX_DEBUG < 0 || MDBX_DEBUG > 2 +#error "The MDBX_DEBUG must be defined to 0, 1 or 2" #endif /* MDBX_DEBUG */ #else @@ -2497,169 +2123,58 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; * Also enables \ref MDBX_DBG_AUDIT if `MDBX_DEBUG >= 2`. * * \ingroup build_option */ -#define MDBX_DEBUG 0...7 +#define MDBX_DEBUG 0...2 /** Disables using of GNU libc extensions. */ #define MDBX_DISABLE_GNU_SOURCE 0 or 1 #endif /* DOXYGEN */ -/* Undefine the NDEBUG if debugging is enforced by MDBX_DEBUG */ -#if MDBX_DEBUG -#undef NDEBUG -#endif - -#ifndef __cplusplus -/*----------------------------------------------------------------------------*/ -/* Debug and Logging stuff */ - -#define MDBX_RUNTIME_FLAGS_INIT \ - ((MDBX_DEBUG) > 0) * MDBX_DBG_ASSERT + ((MDBX_DEBUG) > 1) * MDBX_DBG_AUDIT - -extern uint8_t runtime_flags; -extern uint8_t loglevel; -extern MDBX_debug_func *debug_logger; - -MDBX_MAYBE_UNUSED static __inline void jitter4testing(bool tiny) { -#if MDBX_DEBUG - if (MDBX_DBG_JITTER & runtime_flags) - osal_jitter(tiny); -#else - (void)tiny; -#endif -} - -MDBX_INTERNAL_FUNC void MDBX_PRINTF_ARGS(4, 5) - debug_log(int level, const char *function, int line, const char *fmt, ...) - MDBX_PRINTF_ARGS(4, 5); -MDBX_INTERNAL_FUNC void debug_log_va(int level, const char *function, int line, - const char *fmt, va_list args); +#ifndef MDBX_64BIT_ATOMIC +#error "The MDBX_64BIT_ATOMIC must be defined before" +#endif /* MDBX_64BIT_ATOMIC */ -#if MDBX_DEBUG -#define LOG_ENABLED(msg) unlikely(msg <= loglevel) -#define AUDIT_ENABLED() unlikely((runtime_flags & MDBX_DBG_AUDIT)) -#else /* MDBX_DEBUG */ -#define LOG_ENABLED(msg) (msg < MDBX_LOG_VERBOSE && msg <= loglevel) -#define AUDIT_ENABLED() (0) -#endif /* MDBX_DEBUG */ +#ifndef MDBX_64BIT_CAS +#error "The MDBX_64BIT_CAS must be defined before" +#endif /* MDBX_64BIT_CAS */ -#if MDBX_FORCE_ASSERTIONS -#define ASSERT_ENABLED() (1) -#elif MDBX_DEBUG -#define ASSERT_ENABLED() likely((runtime_flags & MDBX_DBG_ASSERT)) +#if defined(__cplusplus) && !defined(__STDC_NO_ATOMICS__) && __has_include() +#include +#define MDBX_HAVE_C11ATOMICS +#elif !defined(__cplusplus) && (__STDC_VERSION__ >= 201112L || __has_extension(c_atomic)) && \ + !defined(__STDC_NO_ATOMICS__) && \ + (__GNUC_PREREQ(4, 9) || __CLANG_PREREQ(3, 8) || !(defined(__GNUC__) || defined(__clang__))) +#include +#define MDBX_HAVE_C11ATOMICS +#elif defined(__GNUC__) || defined(__clang__) +#elif defined(_MSC_VER) +#pragma warning(disable : 4163) /* 'xyz': not available as an intrinsic */ +#pragma warning(disable : 4133) /* 'function': incompatible types - from \ + 'size_t' to 'LONGLONG' */ +#pragma warning(disable : 4244) /* 'return': conversion from 'LONGLONG' to \ + 'std::size_t', possible loss of data */ +#pragma warning(disable : 4267) /* 'function': conversion from 'size_t' to \ + 'long', possible loss of data */ +#pragma intrinsic(_InterlockedExchangeAdd, _InterlockedCompareExchange) +#pragma intrinsic(_InterlockedExchangeAdd64, _InterlockedCompareExchange64) +#elif defined(__APPLE__) +#include #else -#define ASSERT_ENABLED() (0) -#endif /* assertions */ - -#define DEBUG_EXTRA(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_EXTRA)) \ - debug_log(MDBX_LOG_EXTRA, __func__, __LINE__, fmt, __VA_ARGS__); \ - } while (0) - -#define DEBUG_EXTRA_PRINT(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_EXTRA)) \ - debug_log(MDBX_LOG_EXTRA, NULL, 0, fmt, __VA_ARGS__); \ - } while (0) - -#define TRACE(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_TRACE)) \ - debug_log(MDBX_LOG_TRACE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ - } while (0) - -#define DEBUG(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_DEBUG)) \ - debug_log(MDBX_LOG_DEBUG, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ - } while (0) - -#define VERBOSE(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_VERBOSE)) \ - debug_log(MDBX_LOG_VERBOSE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ - } while (0) - -#define NOTICE(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_NOTICE)) \ - debug_log(MDBX_LOG_NOTICE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ - } while (0) - -#define WARNING(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_WARN)) \ - debug_log(MDBX_LOG_WARN, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ - } while (0) - -#undef ERROR /* wingdi.h \ - Yeah, morons from M$ put such definition to the public header. */ - -#define ERROR(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_ERROR)) \ - debug_log(MDBX_LOG_ERROR, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ - } while (0) - -#define FATAL(fmt, ...) \ - debug_log(MDBX_LOG_FATAL, __func__, __LINE__, fmt "\n", __VA_ARGS__); - -#if MDBX_DEBUG -#define ASSERT_FAIL(env, msg, func, line) mdbx_assert_fail(env, msg, func, line) -#else /* MDBX_DEBUG */ -MDBX_NORETURN __cold void assert_fail(const char *msg, const char *func, - unsigned line); -#define ASSERT_FAIL(env, msg, func, line) \ - do { \ - (void)(env); \ - assert_fail(msg, func, line); \ - } while (0) -#endif /* MDBX_DEBUG */ - -#define ENSURE_MSG(env, expr, msg) \ - do { \ - if (unlikely(!(expr))) \ - ASSERT_FAIL(env, msg, __func__, __LINE__); \ - } while (0) - -#define ENSURE(env, expr) ENSURE_MSG(env, expr, #expr) - -/* assert(3) variant in environment context */ -#define eASSERT(env, expr) \ - do { \ - if (ASSERT_ENABLED()) \ - ENSURE(env, expr); \ - } while (0) - -/* assert(3) variant in cursor context */ -#define cASSERT(mc, expr) eASSERT((mc)->mc_txn->mt_env, expr) - -/* assert(3) variant in transaction context */ -#define tASSERT(txn, expr) eASSERT((txn)->mt_env, expr) - -#ifndef xMDBX_TOOLS /* Avoid using internal eASSERT() */ -#undef assert -#define assert(expr) eASSERT(NULL, expr) +#error FIXME atomic-ops #endif -#endif /* __cplusplus */ - -/*----------------------------------------------------------------------------*/ -/* Atomics */ - -enum MDBX_memory_order { +typedef enum mdbx_memory_order { mo_Relaxed, mo_AcquireRelease /* , mo_SequentialConsistency */ -}; +} mdbx_memory_order_t; typedef union { volatile uint32_t weak; #ifdef MDBX_HAVE_C11ATOMICS volatile _Atomic uint32_t c11a; #endif /* MDBX_HAVE_C11ATOMICS */ -} MDBX_atomic_uint32_t; +} mdbx_atomic_uint32_t; typedef union { volatile uint64_t weak; @@ -2669,15 +2184,15 @@ typedef union { #if !defined(MDBX_HAVE_C11ATOMICS) || !MDBX_64BIT_CAS || !MDBX_64BIT_ATOMIC __anonymous_struct_extension__ struct { #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ - MDBX_atomic_uint32_t low, high; + mdbx_atomic_uint32_t low, high; #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - MDBX_atomic_uint32_t high, low; + mdbx_atomic_uint32_t high, low; #else #error "FIXME: Unsupported byte order" #endif /* __BYTE_ORDER__ */ }; #endif -} MDBX_atomic_uint64_t; +} mdbx_atomic_uint64_t; #ifdef MDBX_HAVE_C11ATOMICS @@ -2693,92 +2208,20 @@ typedef union { #define MDBX_c11a_rw(type, ptr) (&(ptr)->c11a) #endif /* Crutches for C11 atomic compiler's bugs */ -#define mo_c11_store(fence) \ - (((fence) == mo_Relaxed) ? memory_order_relaxed \ - : ((fence) == mo_AcquireRelease) ? memory_order_release \ +#define mo_c11_store(fence) \ + (((fence) == mo_Relaxed) ? memory_order_relaxed \ + : ((fence) == mo_AcquireRelease) ? memory_order_release \ : memory_order_seq_cst) -#define mo_c11_load(fence) \ - (((fence) == mo_Relaxed) ? memory_order_relaxed \ - : ((fence) == mo_AcquireRelease) ? memory_order_acquire \ +#define mo_c11_load(fence) \ + (((fence) == mo_Relaxed) ? memory_order_relaxed \ + : ((fence) == mo_AcquireRelease) ? memory_order_acquire \ : memory_order_seq_cst) #endif /* MDBX_HAVE_C11ATOMICS */ -#ifndef __cplusplus - -#ifdef MDBX_HAVE_C11ATOMICS -#define osal_memory_fence(order, write) \ - atomic_thread_fence((write) ? mo_c11_store(order) : mo_c11_load(order)) -#else /* MDBX_HAVE_C11ATOMICS */ -#define osal_memory_fence(order, write) \ - do { \ - osal_compiler_barrier(); \ - if (write && order > (MDBX_CPU_WRITEBACK_INCOHERENT ? mo_Relaxed \ - : mo_AcquireRelease)) \ - osal_memory_barrier(); \ - } while (0) -#endif /* MDBX_HAVE_C11ATOMICS */ - -#if defined(MDBX_HAVE_C11ATOMICS) && defined(__LCC__) -#define atomic_store32(p, value, order) \ - ({ \ - const uint32_t value_to_store = (value); \ - atomic_store_explicit(MDBX_c11a_rw(uint32_t, p), value_to_store, \ - mo_c11_store(order)); \ - value_to_store; \ - }) -#define atomic_load32(p, order) \ - atomic_load_explicit(MDBX_c11a_ro(uint32_t, p), mo_c11_load(order)) -#define atomic_store64(p, value, order) \ - ({ \ - const uint64_t value_to_store = (value); \ - atomic_store_explicit(MDBX_c11a_rw(uint64_t, p), value_to_store, \ - mo_c11_store(order)); \ - value_to_store; \ - }) -#define atomic_load64(p, order) \ - atomic_load_explicit(MDBX_c11a_ro(uint64_t, p), mo_c11_load(order)) -#endif /* LCC && MDBX_HAVE_C11ATOMICS */ - -#ifndef atomic_store32 -MDBX_MAYBE_UNUSED static __always_inline uint32_t -atomic_store32(MDBX_atomic_uint32_t *p, const uint32_t value, - enum MDBX_memory_order order) { - STATIC_ASSERT(sizeof(MDBX_atomic_uint32_t) == 4); -#ifdef MDBX_HAVE_C11ATOMICS - assert(atomic_is_lock_free(MDBX_c11a_rw(uint32_t, p))); - atomic_store_explicit(MDBX_c11a_rw(uint32_t, p), value, mo_c11_store(order)); -#else /* MDBX_HAVE_C11ATOMICS */ - if (order != mo_Relaxed) - osal_compiler_barrier(); - p->weak = value; - osal_memory_fence(order, true); -#endif /* MDBX_HAVE_C11ATOMICS */ - return value; -} -#endif /* atomic_store32 */ - -#ifndef atomic_load32 -MDBX_MAYBE_UNUSED static __always_inline uint32_t atomic_load32( - const volatile MDBX_atomic_uint32_t *p, enum MDBX_memory_order order) { - STATIC_ASSERT(sizeof(MDBX_atomic_uint32_t) == 4); -#ifdef MDBX_HAVE_C11ATOMICS - assert(atomic_is_lock_free(MDBX_c11a_ro(uint32_t, p))); - return atomic_load_explicit(MDBX_c11a_ro(uint32_t, p), mo_c11_load(order)); -#else /* MDBX_HAVE_C11ATOMICS */ - osal_memory_fence(order, false); - const uint32_t value = p->weak; - if (order != mo_Relaxed) - osal_compiler_barrier(); - return value; -#endif /* MDBX_HAVE_C11ATOMICS */ -} -#endif /* atomic_load32 */ - -#endif /* !__cplusplus */ +#define SAFE64_INVALID_THRESHOLD UINT64_C(0xffffFFFF00000000) -/*----------------------------------------------------------------------------*/ -/* Basic constants and types */ +#pragma pack(push, 4) /* A stamp that identifies a file as an MDBX file. * There's nothing special about this value other than that it is easily @@ -2787,8 +2230,10 @@ MDBX_MAYBE_UNUSED static __always_inline uint32_t atomic_load32( /* FROZEN: The version number for a database's datafile format. */ #define MDBX_DATA_VERSION 3 -/* The version number for a database's lockfile format. */ -#define MDBX_LOCK_VERSION 5 + +#define MDBX_DATA_MAGIC ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + MDBX_DATA_VERSION) +#define MDBX_DATA_MAGIC_LEGACY_COMPAT ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + 2) +#define MDBX_DATA_MAGIC_LEGACY_DEVEL ((MDBX_MAGIC << 8) + 255) /* handle for the DB used to track free pages. */ #define FREE_DBI 0 @@ -2805,203 +2250,285 @@ MDBX_MAYBE_UNUSED static __always_inline uint32_t atomic_load32( * MDBX uses 32 bit for page numbers. This limits database * size up to 2^44 bytes, in case of 4K pages. */ typedef uint32_t pgno_t; -typedef MDBX_atomic_uint32_t atomic_pgno_t; +typedef mdbx_atomic_uint32_t atomic_pgno_t; #define PRIaPGNO PRIu32 #define MAX_PAGENO UINT32_C(0x7FFFffff) #define MIN_PAGENO NUM_METAS -#define SAFE64_INVALID_THRESHOLD UINT64_C(0xffffFFFF00000000) +/* An invalid page number. + * Mainly used to denote an empty tree. */ +#define P_INVALID (~(pgno_t)0) /* A transaction ID. */ typedef uint64_t txnid_t; -typedef MDBX_atomic_uint64_t atomic_txnid_t; +typedef mdbx_atomic_uint64_t atomic_txnid_t; #define PRIaTXN PRIi64 #define MIN_TXNID UINT64_C(1) #define MAX_TXNID (SAFE64_INVALID_THRESHOLD - 1) #define INITIAL_TXNID (MIN_TXNID + NUM_METAS - 1) #define INVALID_TXNID UINT64_MAX -/* LY: for testing non-atomic 64-bit txnid on 32-bit arches. - * #define xMDBX_TXNID_STEP (UINT32_MAX / 3) */ -#ifndef xMDBX_TXNID_STEP -#if MDBX_64BIT_CAS -#define xMDBX_TXNID_STEP 1u -#else -#define xMDBX_TXNID_STEP 2u -#endif -#endif /* xMDBX_TXNID_STEP */ -/* Used for offsets within a single page. - * Since memory pages are typically 4 or 8KB in size, 12-13 bits, - * this is plenty. */ +/* Used for offsets within a single page. */ typedef uint16_t indx_t; -#define MEGABYTE ((size_t)1 << 20) - -/*----------------------------------------------------------------------------*/ -/* Core structures for database and shared memory (i.e. format definition) */ -#pragma pack(push, 4) - -/* Information about a single database in the environment. */ -typedef struct MDBX_db { - uint16_t md_flags; /* see mdbx_dbi_open */ - uint16_t md_depth; /* depth of this tree */ - uint32_t md_xsize; /* key-size for MDBX_DUPFIXED (LEAF2 pages) */ - pgno_t md_root; /* the root page of this tree */ - pgno_t md_branch_pages; /* number of internal pages */ - pgno_t md_leaf_pages; /* number of leaf pages */ - pgno_t md_overflow_pages; /* number of overflow pages */ - uint64_t md_seq; /* table sequence counter */ - uint64_t md_entries; /* number of data items */ - uint64_t md_mod_txnid; /* txnid of last committed modification */ -} MDBX_db; +typedef struct tree { + uint16_t flags; /* see mdbx_dbi_open */ + uint16_t height; /* height of this tree */ + uint32_t dupfix_size; /* key-size for MDBX_DUPFIXED (DUPFIX pages) */ + pgno_t root; /* the root page of this tree */ + pgno_t branch_pages; /* number of branch pages */ + pgno_t leaf_pages; /* number of leaf pages */ + pgno_t large_pages; /* number of large pages */ + uint64_t sequence; /* table sequence counter */ + uint64_t items; /* number of data items */ + uint64_t mod_txnid; /* txnid of last committed modification */ +} tree_t; /* database size-related parameters */ -typedef struct MDBX_geo { +typedef struct geo { uint16_t grow_pv; /* datafile growth step as a 16-bit packed (exponential quantized) value */ uint16_t shrink_pv; /* datafile shrink threshold as a 16-bit packed (exponential quantized) value */ pgno_t lower; /* minimal size of datafile in pages */ pgno_t upper; /* maximal size of datafile in pages */ - pgno_t now; /* current size of datafile in pages */ - pgno_t next; /* first unused page in the datafile, + union { + pgno_t now; /* current size of datafile in pages */ + pgno_t end_pgno; + }; + union { + pgno_t first_unallocated; /* first unused page in the datafile, but actually the file may be shorter. */ -} MDBX_geo; + pgno_t next_pgno; + }; +} geo_t; /* Meta page content. * A meta page is the start point for accessing a database snapshot. - * Pages 0-1 are meta pages. Transaction N writes meta page (N % 2). */ -typedef struct MDBX_meta { + * Pages 0-2 are meta pages. */ +typedef struct meta { /* Stamp identifying this as an MDBX file. * It must be set to MDBX_MAGIC with MDBX_DATA_VERSION. */ - uint32_t mm_magic_and_version[2]; + uint32_t magic_and_version[2]; - /* txnid that committed this page, the first of a two-phase-update pair */ + /* txnid that committed this meta, the first of a two-phase-update pair */ union { - MDBX_atomic_uint32_t mm_txnid_a[2]; + mdbx_atomic_uint32_t txnid_a[2]; uint64_t unsafe_txnid; }; - uint16_t mm_extra_flags; /* extra DB flags, zero (nothing) for now */ - uint8_t mm_validator_id; /* ID of checksum and page validation method, - * zero (nothing) for now */ - uint8_t mm_extra_pagehdr; /* extra bytes in the page header, - * zero (nothing) for now */ + uint16_t reserve16; /* extra flags, zero (nothing) for now */ + uint8_t validator_id; /* ID of checksum and page validation method, + * zero (nothing) for now */ + int8_t extra_pagehdr; /* extra bytes in the page header, + * zero (nothing) for now */ - MDBX_geo mm_geo; /* database size-related parameters */ + geo_t geometry; /* database size-related parameters */ - MDBX_db mm_dbs[CORE_DBS]; /* first is free space, 2nd is main db */ - /* The size of pages used in this DB */ -#define mm_psize mm_dbs[FREE_DBI].md_xsize - MDBX_canary mm_canary; + union { + struct { + tree_t gc, main; + } trees; + __anonymous_struct_extension__ struct { + uint16_t gc_flags; + uint16_t gc_height; + uint32_t pagesize; + }; + }; + + MDBX_canary canary; -#define MDBX_DATASIGN_NONE 0u -#define MDBX_DATASIGN_WEAK 1u -#define SIGN_IS_STEADY(sign) ((sign) > MDBX_DATASIGN_WEAK) -#define META_IS_STEADY(meta) \ - SIGN_IS_STEADY(unaligned_peek_u64_volatile(4, (meta)->mm_sign)) +#define DATASIGN_NONE 0u +#define DATASIGN_WEAK 1u +#define SIGN_IS_STEADY(sign) ((sign) > DATASIGN_WEAK) union { - uint32_t mm_sign[2]; + uint32_t sign[2]; uint64_t unsafe_sign; }; - /* txnid that committed this page, the second of a two-phase-update pair */ - MDBX_atomic_uint32_t mm_txnid_b[2]; + /* txnid that committed this meta, the second of a two-phase-update pair */ + mdbx_atomic_uint32_t txnid_b[2]; /* Number of non-meta pages which were put in GC after COW. May be 0 in case * DB was previously handled by libmdbx without corresponding feature. - * This value in couple with mr_snapshot_pages_retired allows fast estimation - * of "how much reader is restraining GC recycling". */ - uint32_t mm_pages_retired[2]; + * This value in couple with reader.snapshot_pages_retired allows fast + * estimation of "how much reader is restraining GC recycling". */ + uint32_t pages_retired[2]; /* The analogue /proc/sys/kernel/random/boot_id or similar to determine * whether the system was rebooted after the last use of the database files. * If there was no reboot, but there is no need to rollback to the last * steady sync point. Zeros mean that no relevant information is available * from the system. */ - bin128_t mm_bootid; + bin128_t bootid; -} MDBX_meta; + /* GUID базы данных, начиная с v0.13.1 */ + bin128_t dxbid; +} meta_t; #pragma pack(1) -/* Common header for all page types. The page type depends on mp_flags. +typedef enum page_type { + P_BRANCH = 0x01u /* branch page */, + P_LEAF = 0x02u /* leaf page */, + P_LARGE = 0x04u /* large/overflow page */, + P_META = 0x08u /* meta page */, + P_LEGACY_DIRTY = 0x10u /* legacy P_DIRTY flag prior to v0.10 958fd5b9 */, + P_BAD = P_LEGACY_DIRTY /* explicit flag for invalid/bad page */, + P_DUPFIX = 0x20u /* for MDBX_DUPFIXED records */, + P_SUBP = 0x40u /* for MDBX_DUPSORT sub-pages */, + P_SPILLED = 0x2000u /* spilled in parent txn */, + P_LOOSE = 0x4000u /* page was dirtied then freed, can be reused */, + P_FROZEN = 0x8000u /* used for retire page with known status */, + P_ILL_BITS = (uint16_t)~(P_BRANCH | P_LEAF | P_DUPFIX | P_LARGE | P_SPILLED), + + page_broken = 0, + page_large = P_LARGE, + page_branch = P_BRANCH, + page_leaf = P_LEAF, + page_dupfix_leaf = P_DUPFIX, + page_sub_leaf = P_SUBP | P_LEAF, + page_sub_dupfix_leaf = P_SUBP | P_DUPFIX, + page_sub_broken = P_SUBP, +} page_type_t; + +/* Common header for all page types. The page type depends on flags. * - * P_BRANCH and P_LEAF pages have unsorted 'MDBX_node's at the end, with - * sorted mp_ptrs[] entries referring to them. Exception: P_LEAF2 pages - * omit mp_ptrs and pack sorted MDBX_DUPFIXED values after the page header. + * P_BRANCH and P_LEAF pages have unsorted 'node_t's at the end, with + * sorted entries[] entries referring to them. Exception: P_DUPFIX pages + * omit entries and pack sorted MDBX_DUPFIXED values after the page header. * - * P_OVERFLOW records occupy one or more contiguous pages where only the - * first has a page header. They hold the real data of F_BIGDATA nodes. + * P_LARGE records occupy one or more contiguous pages where only the + * first has a page header. They hold the real data of N_BIG nodes. * * P_SUBP sub-pages are small leaf "pages" with duplicate data. - * A node with flag F_DUPDATA but not F_SUBDATA contains a sub-page. - * (Duplicate data can also go in sub-databases, which use normal pages.) + * A node with flag N_DUP but not N_TREE contains a sub-page. + * (Duplicate data can also go in tables, which use normal pages.) * - * P_META pages contain MDBX_meta, the start point of an MDBX snapshot. + * P_META pages contain meta_t, the start point of an MDBX snapshot. * - * Each non-metapage up to MDBX_meta.mm_last_pg is reachable exactly once + * Each non-metapage up to meta_t.mm_last_pg is reachable exactly once * in the snapshot: Either used by a database or listed in a GC record. */ -typedef struct MDBX_page { -#define IS_FROZEN(txn, p) ((p)->mp_txnid < (txn)->mt_txnid) -#define IS_SPILLED(txn, p) ((p)->mp_txnid == (txn)->mt_txnid) -#define IS_SHADOWED(txn, p) ((p)->mp_txnid > (txn)->mt_txnid) -#define IS_VALID(txn, p) ((p)->mp_txnid <= (txn)->mt_front) -#define IS_MODIFIABLE(txn, p) ((p)->mp_txnid == (txn)->mt_front) - uint64_t mp_txnid; /* txnid which created page, maybe zero in legacy DB */ - uint16_t mp_leaf2_ksize; /* key size if this is a LEAF2 page */ -#define P_BRANCH 0x01u /* branch page */ -#define P_LEAF 0x02u /* leaf page */ -#define P_OVERFLOW 0x04u /* overflow page */ -#define P_META 0x08u /* meta page */ -#define P_LEGACY_DIRTY 0x10u /* legacy P_DIRTY flag prior to v0.10 958fd5b9 */ -#define P_BAD P_LEGACY_DIRTY /* explicit flag for invalid/bad page */ -#define P_LEAF2 0x20u /* for MDBX_DUPFIXED records */ -#define P_SUBP 0x40u /* for MDBX_DUPSORT sub-pages */ -#define P_SPILLED 0x2000u /* spilled in parent txn */ -#define P_LOOSE 0x4000u /* page was dirtied then freed, can be reused */ -#define P_FROZEN 0x8000u /* used for retire page with known status */ -#define P_ILL_BITS \ - ((uint16_t)~(P_BRANCH | P_LEAF | P_LEAF2 | P_OVERFLOW | P_SPILLED)) - uint16_t mp_flags; +typedef struct page { + uint64_t txnid; /* txnid which created page, maybe zero in legacy DB */ + uint16_t dupfix_ksize; /* key size if this is a DUPFIX page */ + uint16_t flags; union { - uint32_t mp_pages; /* number of overflow pages */ + uint32_t pages; /* number of overflow pages */ __anonymous_struct_extension__ struct { - indx_t mp_lower; /* lower bound of free space */ - indx_t mp_upper; /* upper bound of free space */ + indx_t lower; /* lower bound of free space */ + indx_t upper; /* upper bound of free space */ }; }; - pgno_t mp_pgno; /* page number */ - -#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ - (!defined(__cplusplus) && defined(_MSC_VER)) - indx_t mp_ptrs[] /* dynamic size */; -#endif /* C99 */ -} MDBX_page; - -#define PAGETYPE_WHOLE(p) ((uint8_t)(p)->mp_flags) + pgno_t pgno; /* page number */ -/* Drop legacy P_DIRTY flag for sub-pages for compatilibity */ -#define PAGETYPE_COMPAT(p) \ - (unlikely(PAGETYPE_WHOLE(p) & P_SUBP) \ - ? PAGETYPE_WHOLE(p) & ~(P_SUBP | P_LEGACY_DIRTY) \ - : PAGETYPE_WHOLE(p)) +#if FLEXIBLE_ARRAY_MEMBERS + indx_t entries[] /* dynamic size */; +#endif /* FLEXIBLE_ARRAY_MEMBERS */ +} page_t; /* Size of the page header, excluding dynamic data at the end */ -#define PAGEHDRSZ offsetof(MDBX_page, mp_ptrs) +#define PAGEHDRSZ 20u -/* Pointer displacement without casting to char* to avoid pointer-aliasing */ -#define ptr_disp(ptr, disp) ((void *)(((intptr_t)(ptr)) + ((intptr_t)(disp)))) +/* Header for a single key/data pair within a page. + * Used in pages of type P_BRANCH and P_LEAF without P_DUPFIX. + * We guarantee 2-byte alignment for 'node_t's. + * + * Leaf node flags describe node contents. N_BIG says the node's + * data part is the page number of an overflow page with actual data. + * N_DUP and N_TREE can be combined giving duplicate data in + * a sub-page/table, and named databases (just N_TREE). */ +typedef struct node { +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + union { + uint32_t dsize; + uint32_t child_pgno; + }; + uint8_t flags; /* see node_flags */ + uint8_t extra; + uint16_t ksize; /* key size */ +#else + uint16_t ksize; /* key size */ + uint8_t extra; + uint8_t flags; /* see node_flags */ + union { + uint32_t child_pgno; + uint32_t dsize; + }; +#endif /* __BYTE_ORDER__ */ -/* Pointer distance as signed number of bytes */ -#define ptr_dist(more, less) (((intptr_t)(more)) - ((intptr_t)(less))) +#if FLEXIBLE_ARRAY_MEMBERS + uint8_t payload[] /* key and data are appended here */; +#endif /* FLEXIBLE_ARRAY_MEMBERS */ +} node_t; -#define mp_next(mp) \ - (*(MDBX_page **)ptr_disp((mp)->mp_ptrs, sizeof(void *) - sizeof(uint32_t))) +/* Size of the node header, excluding dynamic data at the end */ +#define NODESIZE 8u + +typedef enum node_flags { + N_BIG = 0x01 /* data put on large page */, + N_TREE = 0x02 /* data is a b-tree */, + N_DUP = 0x04 /* data has duplicates */ +} node_flags_t; #pragma pack(pop) -typedef struct profgc_stat { +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint8_t page_type(const page_t *mp) { return mp->flags; } + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint8_t page_type_compat(const page_t *mp) { + /* Drop legacy P_DIRTY flag for sub-pages for compatilibity, + * for assertions only. */ + return unlikely(mp->flags & P_SUBP) ? mp->flags & ~(P_SUBP | P_LEGACY_DIRTY) : mp->flags; +} + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_leaf(const page_t *mp) { + return (mp->flags & P_LEAF) != 0; +} + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_dupfix_leaf(const page_t *mp) { + return (mp->flags & P_DUPFIX) != 0; +} + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_branch(const page_t *mp) { + return (mp->flags & P_BRANCH) != 0; +} + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_largepage(const page_t *mp) { + return (mp->flags & P_LARGE) != 0; +} + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_subpage(const page_t *mp) { + return (mp->flags & P_SUBP) != 0; +} + +/* The version number for a database's lockfile format. */ +#define MDBX_LOCK_VERSION 6 + +#if MDBX_LOCKING == MDBX_LOCKING_WIN32FILES + +#define MDBX_LCK_SIGN UINT32_C(0xF10C) +typedef void osal_ipclock_t; +#elif MDBX_LOCKING == MDBX_LOCKING_SYSV + +#define MDBX_LCK_SIGN UINT32_C(0xF18D) +typedef mdbx_pid_t osal_ipclock_t; + +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || MDBX_LOCKING == MDBX_LOCKING_POSIX2008 + +#define MDBX_LCK_SIGN UINT32_C(0x8017) +typedef pthread_mutex_t osal_ipclock_t; + +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988 + +#define MDBX_LCK_SIGN UINT32_C(0xFC29) +typedef sem_t osal_ipclock_t; + +#else +#error "FIXME" +#endif /* MDBX_LOCKING */ + +/* Статистика профилирования работы GC */ +typedef struct gc_prof_stat { /* Монотонное время по "настенным часам" * затраченное на чтение и поиск внутри GC */ uint64_t rtime_monotonic; @@ -3017,42 +2544,44 @@ typedef struct profgc_stat { uint32_t spe_counter; /* page faults (hard page faults) */ uint32_t majflt; -} profgc_stat_t; - -/* Statistics of page operations overall of all (running, completed and aborted) - * transactions */ -typedef struct pgop_stat { - MDBX_atomic_uint64_t newly; /* Quantity of a new pages added */ - MDBX_atomic_uint64_t cow; /* Quantity of pages copied for update */ - MDBX_atomic_uint64_t clone; /* Quantity of parent's dirty pages clones + /* Для разборок с pnl_merge() */ + struct { + uint64_t time; + uint64_t volume; + uint32_t calls; + } pnl_merge; +} gc_prof_stat_t; + +/* Statistics of pages operations for all transactions, + * including incomplete and aborted. */ +typedef struct pgops { + mdbx_atomic_uint64_t newly; /* Quantity of a new pages added */ + mdbx_atomic_uint64_t cow; /* Quantity of pages copied for update */ + mdbx_atomic_uint64_t clone; /* Quantity of parent's dirty pages clones for nested transactions */ - MDBX_atomic_uint64_t split; /* Page splits */ - MDBX_atomic_uint64_t merge; /* Page merges */ - MDBX_atomic_uint64_t spill; /* Quantity of spilled dirty pages */ - MDBX_atomic_uint64_t unspill; /* Quantity of unspilled/reloaded pages */ - MDBX_atomic_uint64_t - wops; /* Number of explicit write operations (not a pages) to a disk */ - MDBX_atomic_uint64_t - msync; /* Number of explicit msync/flush-to-disk operations */ - MDBX_atomic_uint64_t - fsync; /* Number of explicit fsync/flush-to-disk operations */ - - MDBX_atomic_uint64_t prefault; /* Number of prefault write operations */ - MDBX_atomic_uint64_t mincore; /* Number of mincore() calls */ - - MDBX_atomic_uint32_t - incoherence; /* number of https://libmdbx.dqdkfa.ru/dead-github/issues/269 - caught */ - MDBX_atomic_uint32_t reserved; + mdbx_atomic_uint64_t split; /* Page splits */ + mdbx_atomic_uint64_t merge; /* Page merges */ + mdbx_atomic_uint64_t spill; /* Quantity of spilled dirty pages */ + mdbx_atomic_uint64_t unspill; /* Quantity of unspilled/reloaded pages */ + mdbx_atomic_uint64_t wops; /* Number of explicit write operations (not a pages) to a disk */ + mdbx_atomic_uint64_t msync; /* Number of explicit msync/flush-to-disk operations */ + mdbx_atomic_uint64_t fsync; /* Number of explicit fsync/flush-to-disk operations */ + + mdbx_atomic_uint64_t prefault; /* Number of prefault write operations */ + mdbx_atomic_uint64_t mincore; /* Number of mincore() calls */ + + mdbx_atomic_uint32_t incoherence; /* number of https://libmdbx.dqdkfa.ru/dead-github/issues/269 + caught */ + mdbx_atomic_uint32_t reserved; /* Статистика для профилирования GC. - * Логически эти данные может быть стоит вынести в другую структуру, + * Логически эти данные, возможно, стоит вынести в другую структуру, * но разница будет сугубо косметическая. */ struct { /* Затраты на поддержку данных пользователя */ - profgc_stat_t work; + gc_prof_stat_t work; /* Затраты на поддержку и обновления самой GC */ - profgc_stat_t self; + gc_prof_stat_t self; /* Итераций обновления GC, * больше 1 если были повторы/перезапуски */ uint32_t wloops; @@ -3067,33 +2596,6 @@ typedef struct pgop_stat { } gc_prof; } pgop_stat_t; -#if MDBX_LOCKING == MDBX_LOCKING_WIN32FILES -#define MDBX_CLOCK_SIGN UINT32_C(0xF10C) -typedef void osal_ipclock_t; -#elif MDBX_LOCKING == MDBX_LOCKING_SYSV - -#define MDBX_CLOCK_SIGN UINT32_C(0xF18D) -typedef mdbx_pid_t osal_ipclock_t; -#ifndef EOWNERDEAD -#define EOWNERDEAD MDBX_RESULT_TRUE -#endif - -#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ - MDBX_LOCKING == MDBX_LOCKING_POSIX2008 -#define MDBX_CLOCK_SIGN UINT32_C(0x8017) -typedef pthread_mutex_t osal_ipclock_t; -#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988 -#define MDBX_CLOCK_SIGN UINT32_C(0xFC29) -typedef sem_t osal_ipclock_t; -#else -#error "FIXME" -#endif /* MDBX_LOCKING */ - -#if MDBX_LOCKING > MDBX_LOCKING_SYSV && !defined(__cplusplus) -MDBX_INTERNAL_FUNC int osal_ipclock_stub(osal_ipclock_t *ipc); -MDBX_INTERNAL_FUNC int osal_ipclock_destroy(osal_ipclock_t *ipc); -#endif /* MDBX_LOCKING */ - /* Reader Lock Table * * Readers don't acquire any locks for their data access. Instead, they @@ -3103,8 +2605,9 @@ MDBX_INTERNAL_FUNC int osal_ipclock_destroy(osal_ipclock_t *ipc); * read transactions started by the same thread need no further locking to * proceed. * - * If MDBX_NOTLS is set, the slot address is not saved in thread-specific data. - * No reader table is used if the database is on a read-only filesystem. + * If MDBX_NOSTICKYTHREADS is set, the slot address is not saved in + * thread-specific data. No reader table is used if the database is on a + * read-only filesystem. * * Since the database uses multi-version concurrency control, readers don't * actually need any locking. This table is used to keep track of which @@ -3133,14 +2636,14 @@ MDBX_INTERNAL_FUNC int osal_ipclock_destroy(osal_ipclock_t *ipc); * many old transactions together. */ /* The actual reader record, with cacheline padding. */ -typedef struct MDBX_reader { - /* Current Transaction ID when this transaction began, or (txnid_t)-1. +typedef struct reader_slot { + /* Current Transaction ID when this transaction began, or INVALID_TXNID. * Multiple readers that start at the same time will probably have the * same ID here. Again, it's not important to exclude them from * anything; all we need to know is which version of the DB they * started from so we can avoid overwriting any data used in that * particular version. */ - MDBX_atomic_uint64_t /* txnid_t */ mr_txnid; + atomic_txnid_t txnid; /* The information we store in a single slot of the reader table. * In addition to a transaction ID, we also record the process and @@ -3151,708 +2654,320 @@ typedef struct MDBX_reader { * We simply re-init the table when we know that we're the only process * opening the lock file. */ + /* Псевдо thread_id для пометки вытесненных читающих транзакций. */ +#define MDBX_TID_TXN_OUSTED (UINT64_MAX - 1) + + /* Псевдо thread_id для пометки припаркованных читающих транзакций. */ +#define MDBX_TID_TXN_PARKED UINT64_MAX + /* The thread ID of the thread owning this txn. */ - MDBX_atomic_uint64_t mr_tid; + mdbx_atomic_uint64_t tid; /* The process ID of the process owning this reader txn. */ - MDBX_atomic_uint32_t mr_pid; + mdbx_atomic_uint32_t pid; /* The number of pages used in the reader's MVCC snapshot, - * i.e. the value of meta->mm_geo.next and txn->mt_next_pgno */ - atomic_pgno_t mr_snapshot_pages_used; + * i.e. the value of meta->geometry.first_unallocated and + * txn->geo.first_unallocated */ + atomic_pgno_t snapshot_pages_used; /* Number of retired pages at the time this reader starts transaction. So, - * at any time the difference mm_pages_retired - mr_snapshot_pages_retired - * will give the number of pages which this reader restraining from reuse. */ - MDBX_atomic_uint64_t mr_snapshot_pages_retired; -} MDBX_reader; + * at any time the difference meta.pages_retired - + * reader.snapshot_pages_retired will give the number of pages which this + * reader restraining from reuse. */ + mdbx_atomic_uint64_t snapshot_pages_retired; +} reader_slot_t; /* The header for the reader table (a memory-mapped lock file). */ -typedef struct MDBX_lockinfo { +typedef struct shared_lck { /* Stamp identifying this as an MDBX file. * It must be set to MDBX_MAGIC with with MDBX_LOCK_VERSION. */ - uint64_t mti_magic_and_version; + uint64_t magic_and_version; /* Format of this lock file. Must be set to MDBX_LOCK_FORMAT. */ - uint32_t mti_os_and_format; + uint32_t os_and_format; /* Flags which environment was opened. */ - MDBX_atomic_uint32_t mti_envmode; + mdbx_atomic_uint32_t envmode; /* Threshold of un-synced-with-disk pages for auto-sync feature, * zero means no-threshold, i.e. auto-sync is disabled. */ - atomic_pgno_t mti_autosync_threshold; + atomic_pgno_t autosync_threshold; /* Low 32-bit of txnid with which meta-pages was synced, * i.e. for sync-polling in the MDBX_NOMETASYNC mode. */ #define MDBX_NOMETASYNC_LAZY_UNK (UINT32_MAX / 3) #define MDBX_NOMETASYNC_LAZY_FD (MDBX_NOMETASYNC_LAZY_UNK + UINT32_MAX / 8) -#define MDBX_NOMETASYNC_LAZY_WRITEMAP \ - (MDBX_NOMETASYNC_LAZY_UNK - UINT32_MAX / 8) - MDBX_atomic_uint32_t mti_meta_sync_txnid; +#define MDBX_NOMETASYNC_LAZY_WRITEMAP (MDBX_NOMETASYNC_LAZY_UNK - UINT32_MAX / 8) + mdbx_atomic_uint32_t meta_sync_txnid; /* Period for timed auto-sync feature, i.e. at the every steady checkpoint - * the mti_unsynced_timeout sets to the current_time + mti_autosync_period. + * the mti_unsynced_timeout sets to the current_time + autosync_period. * The time value is represented in a suitable system-dependent form, for * example clock_gettime(CLOCK_BOOTTIME) or clock_gettime(CLOCK_MONOTONIC). * Zero means timed auto-sync is disabled. */ - MDBX_atomic_uint64_t mti_autosync_period; + mdbx_atomic_uint64_t autosync_period; /* Marker to distinguish uniqueness of DB/CLK. */ - MDBX_atomic_uint64_t mti_bait_uniqueness; + mdbx_atomic_uint64_t bait_uniqueness; /* Paired counter of processes that have mlock()ed part of mmapped DB. - * The (mti_mlcnt[0] - mti_mlcnt[1]) > 0 means at least one process + * The (mlcnt[0] - mlcnt[1]) > 0 means at least one process * lock at least one page, so therefore madvise() could return EINVAL. */ - MDBX_atomic_uint32_t mti_mlcnt[2]; + mdbx_atomic_uint32_t mlcnt[2]; MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/ /* Statistics of costly ops of all (running, completed and aborted) * transactions */ - pgop_stat_t mti_pgop_stat; + pgop_stat_t pgops; MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/ - /* Write transaction lock. */ #if MDBX_LOCKING > 0 - osal_ipclock_t mti_wlock; + /* Write transaction lock. */ + osal_ipclock_t wrt_lock; #endif /* MDBX_LOCKING > 0 */ - atomic_txnid_t mti_oldest_reader; + atomic_txnid_t cached_oldest; /* Timestamp of entering an out-of-sync state. Value is represented in a * suitable system-dependent form, for example clock_gettime(CLOCK_BOOTTIME) * or clock_gettime(CLOCK_MONOTONIC). */ - MDBX_atomic_uint64_t mti_eoos_timestamp; + mdbx_atomic_uint64_t eoos_timestamp; /* Number un-synced-with-disk pages for auto-sync feature. */ - MDBX_atomic_uint64_t mti_unsynced_pages; + mdbx_atomic_uint64_t unsynced_pages; /* Timestamp of the last readers check. */ - MDBX_atomic_uint64_t mti_reader_check_timestamp; + mdbx_atomic_uint64_t readers_check_timestamp; /* Number of page which was discarded last time by madvise(DONTNEED). */ - atomic_pgno_t mti_discarded_tail; + atomic_pgno_t discarded_tail; /* Shared anchor for tracking readahead edge and enabled/disabled status. */ - pgno_t mti_readahead_anchor; + pgno_t readahead_anchor; /* Shared cache for mincore() results */ struct { pgno_t begin[4]; uint64_t mask[4]; - } mti_mincore_cache; + } mincore_cache; MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/ - /* Readeaders registration lock. */ #if MDBX_LOCKING > 0 - osal_ipclock_t mti_rlock; + /* Readeaders table lock. */ + osal_ipclock_t rdt_lock; #endif /* MDBX_LOCKING > 0 */ /* The number of slots that have been used in the reader table. * This always records the maximum count, it is not decremented * when readers release their slots. */ - MDBX_atomic_uint32_t mti_numreaders; - MDBX_atomic_uint32_t mti_readers_refresh_flag; + mdbx_atomic_uint32_t rdt_length; + mdbx_atomic_uint32_t rdt_refresh_flag; -#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ - (!defined(__cplusplus) && defined(_MSC_VER)) +#if FLEXIBLE_ARRAY_MEMBERS MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/ - MDBX_reader mti_readers[] /* dynamic size */; -#endif /* C99 */ -} MDBX_lockinfo; + reader_slot_t rdt[] /* dynamic size */; /* Lockfile format signature: version, features and field layout */ -#define MDBX_LOCK_FORMAT \ - (MDBX_CLOCK_SIGN * 27733 + (unsigned)sizeof(MDBX_reader) * 13 + \ - (unsigned)offsetof(MDBX_reader, mr_snapshot_pages_used) * 251 + \ - (unsigned)offsetof(MDBX_lockinfo, mti_oldest_reader) * 83 + \ - (unsigned)offsetof(MDBX_lockinfo, mti_numreaders) * 37 + \ - (unsigned)offsetof(MDBX_lockinfo, mti_readers) * 29) - -#define MDBX_DATA_MAGIC \ - ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + MDBX_DATA_VERSION) - -#define MDBX_DATA_MAGIC_LEGACY_COMPAT \ - ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + 2) - -#define MDBX_DATA_MAGIC_LEGACY_DEVEL ((MDBX_MAGIC << 8) + 255) +#define MDBX_LOCK_FORMAT \ + (MDBX_LCK_SIGN * 27733 + (unsigned)sizeof(reader_slot_t) * 13 + \ + (unsigned)offsetof(reader_slot_t, snapshot_pages_used) * 251 + (unsigned)offsetof(lck_t, cached_oldest) * 83 + \ + (unsigned)offsetof(lck_t, rdt_length) * 37 + (unsigned)offsetof(lck_t, rdt) * 29) +#endif /* FLEXIBLE_ARRAY_MEMBERS */ +} lck_t; #define MDBX_LOCK_MAGIC ((MDBX_MAGIC << 8) + MDBX_LOCK_VERSION) -/* The maximum size of a database page. - * - * It is 64K, but value-PAGEHDRSZ must fit in MDBX_page.mp_upper. - * - * MDBX will use database pages < OS pages if needed. - * That causes more I/O in write transactions: The OS must - * know (read) the whole page before writing a partial page. - * - * Note that we don't currently support Huge pages. On Linux, - * regular data files cannot use Huge pages, and in general - * Huge pages aren't actually pageable. We rely on the OS - * demand-pager to read our data and page it out when memory - * pressure from other processes is high. So until OSs have - * actual paging support for Huge pages, they're not viable. */ -#define MAX_PAGESIZE MDBX_MAX_PAGESIZE -#define MIN_PAGESIZE MDBX_MIN_PAGESIZE - -#define MIN_MAPSIZE (MIN_PAGESIZE * MIN_PAGENO) +#define MDBX_READERS_LIMIT 32767 + +#define MIN_MAPSIZE (MDBX_MIN_PAGESIZE * MIN_PAGENO) #if defined(_WIN32) || defined(_WIN64) #define MAX_MAPSIZE32 UINT32_C(0x38000000) #else #define MAX_MAPSIZE32 UINT32_C(0x7f000000) #endif -#define MAX_MAPSIZE64 ((MAX_PAGENO + 1) * (uint64_t)MAX_PAGESIZE) +#define MAX_MAPSIZE64 ((MAX_PAGENO + 1) * (uint64_t)MDBX_MAX_PAGESIZE) #if MDBX_WORDBITS >= 64 #define MAX_MAPSIZE MAX_MAPSIZE64 -#define MDBX_PGL_LIMIT ((size_t)MAX_PAGENO) +#define PAGELIST_LIMIT ((size_t)MAX_PAGENO) #else #define MAX_MAPSIZE MAX_MAPSIZE32 -#define MDBX_PGL_LIMIT (MAX_MAPSIZE32 / MIN_PAGESIZE) +#define PAGELIST_LIMIT (MAX_MAPSIZE32 / MDBX_MIN_PAGESIZE) #endif /* MDBX_WORDBITS */ -#define MDBX_READERS_LIMIT 32767 -#define MDBX_RADIXSORT_THRESHOLD 142 #define MDBX_GOLD_RATIO_DBL 1.6180339887498948482 +#define MEGABYTE ((size_t)1 << 20) /*----------------------------------------------------------------------------*/ -/* An PNL is an Page Number List, a sorted array of IDs. - * The first element of the array is a counter for how many actual page-numbers - * are in the list. By default PNLs are sorted in descending order, this allow - * cut off a page with lowest pgno (at the tail) just truncating the list. The - * sort order of PNLs is controlled by the MDBX_PNL_ASCENDING build option. */ -typedef pgno_t *MDBX_PNL; - -#if MDBX_PNL_ASCENDING -#define MDBX_PNL_ORDERED(first, last) ((first) < (last)) -#define MDBX_PNL_DISORDERED(first, last) ((first) >= (last)) -#else -#define MDBX_PNL_ORDERED(first, last) ((first) > (last)) -#define MDBX_PNL_DISORDERED(first, last) ((first) <= (last)) -#endif +union logger_union { + void *ptr; + MDBX_debug_func *fmt; + MDBX_debug_func_nofmt *nofmt; +}; -/* List of txnid, only for MDBX_txn.tw.lifo_reclaimed */ -typedef txnid_t *MDBX_TXL; +struct libmdbx_globals { + bin128_t bootid; + unsigned sys_pagesize, sys_allocation_granularity; + uint8_t sys_pagesize_ln2; + uint8_t runtime_flags; + uint8_t loglevel; +#if defined(_WIN32) || defined(_WIN64) + bool running_under_Wine; +#elif defined(__linux__) || defined(__gnu_linux__) + bool running_on_WSL1 /* Windows Subsystem 1 for Linux */; + uint32_t linux_kernel_version; +#endif /* Linux */ + union logger_union logger; + osal_fastmutex_t debug_lock; + size_t logger_buffer_size; + char *logger_buffer; +}; -/* An Dirty-Page list item is an pgno/pointer pair. */ -typedef struct MDBX_dp { - MDBX_page *ptr; - pgno_t pgno, npages; -} MDBX_dp; +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ -/* An DPL (dirty-page list) is a sorted array of MDBX_DPs. */ -typedef struct MDBX_dpl { - size_t sorted; - size_t length; - size_t pages_including_loose; /* number of pages, but not an entries. */ - size_t detent; /* allocated size excluding the MDBX_DPL_RESERVE_GAP */ -#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ - (!defined(__cplusplus) && defined(_MSC_VER)) - MDBX_dp items[] /* dynamic size with holes at zero and after the last */; -#endif -} MDBX_dpl; +extern struct libmdbx_globals globals; +#if defined(_WIN32) || defined(_WIN64) +extern struct libmdbx_imports imports; +#endif /* Windows */ -/* PNL sizes */ -#define MDBX_PNL_GRANULATE_LOG2 10 -#define MDBX_PNL_GRANULATE (1 << MDBX_PNL_GRANULATE_LOG2) -#define MDBX_PNL_INITIAL \ - (MDBX_PNL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t)) +#ifndef __Wpedantic_format_voidptr +MDBX_MAYBE_UNUSED static inline const void *__Wpedantic_format_voidptr(const void *ptr) { return ptr; } +#define __Wpedantic_format_voidptr(ARG) __Wpedantic_format_voidptr(ARG) +#endif /* __Wpedantic_format_voidptr */ -#define MDBX_TXL_GRANULATE 32 -#define MDBX_TXL_INITIAL \ - (MDBX_TXL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t)) -#define MDBX_TXL_MAX \ - ((1u << 26) - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t)) +MDBX_INTERNAL void MDBX_PRINTF_ARGS(4, 5) debug_log(int level, const char *function, int line, const char *fmt, ...) + MDBX_PRINTF_ARGS(4, 5); +MDBX_INTERNAL void debug_log_va(int level, const char *function, int line, const char *fmt, va_list args); -#define MDBX_PNL_ALLOCLEN(pl) ((pl)[-1]) -#define MDBX_PNL_GETSIZE(pl) ((size_t)((pl)[0])) -#define MDBX_PNL_SETSIZE(pl, size) \ - do { \ - const size_t __size = size; \ - assert(__size < INT_MAX); \ - (pl)[0] = (pgno_t)__size; \ - } while (0) -#define MDBX_PNL_FIRST(pl) ((pl)[1]) -#define MDBX_PNL_LAST(pl) ((pl)[MDBX_PNL_GETSIZE(pl)]) -#define MDBX_PNL_BEGIN(pl) (&(pl)[1]) -#define MDBX_PNL_END(pl) (&(pl)[MDBX_PNL_GETSIZE(pl) + 1]) +#if MDBX_DEBUG +#define LOG_ENABLED(LVL) unlikely(LVL <= globals.loglevel) +#define AUDIT_ENABLED() unlikely((globals.runtime_flags & (unsigned)MDBX_DBG_AUDIT)) +#else /* MDBX_DEBUG */ +#define LOG_ENABLED(LVL) (LVL < MDBX_LOG_VERBOSE && LVL <= globals.loglevel) +#define AUDIT_ENABLED() (0) +#endif /* LOG_ENABLED() & AUDIT_ENABLED() */ -#if MDBX_PNL_ASCENDING -#define MDBX_PNL_EDGE(pl) ((pl) + 1) -#define MDBX_PNL_LEAST(pl) MDBX_PNL_FIRST(pl) -#define MDBX_PNL_MOST(pl) MDBX_PNL_LAST(pl) +#if MDBX_FORCE_ASSERTIONS +#define ASSERT_ENABLED() (1) +#elif MDBX_DEBUG +#define ASSERT_ENABLED() likely((globals.runtime_flags & (unsigned)MDBX_DBG_ASSERT)) #else -#define MDBX_PNL_EDGE(pl) ((pl) + MDBX_PNL_GETSIZE(pl)) -#define MDBX_PNL_LEAST(pl) MDBX_PNL_LAST(pl) -#define MDBX_PNL_MOST(pl) MDBX_PNL_FIRST(pl) -#endif - -#define MDBX_PNL_SIZEOF(pl) ((MDBX_PNL_GETSIZE(pl) + 1) * sizeof(pgno_t)) -#define MDBX_PNL_IS_EMPTY(pl) (MDBX_PNL_GETSIZE(pl) == 0) - -/*----------------------------------------------------------------------------*/ -/* Internal structures */ - -/* Auxiliary DB info. - * The information here is mostly static/read-only. There is - * only a single copy of this record in the environment. */ -typedef struct MDBX_dbx { - MDBX_val md_name; /* name of the database */ - MDBX_cmp_func *md_cmp; /* function for comparing keys */ - MDBX_cmp_func *md_dcmp; /* function for comparing data items */ - size_t md_klen_min, md_klen_max; /* min/max key length for the database */ - size_t md_vlen_min, - md_vlen_max; /* min/max value/data length for the database */ -} MDBX_dbx; - -typedef struct troika { - uint8_t fsm, recent, prefer_steady, tail_and_flags; -#if MDBX_WORDBITS > 32 /* Workaround for false-positives from Valgrind */ - uint32_t unused_pad; -#endif -#define TROIKA_HAVE_STEADY(troika) ((troika)->fsm & 7) -#define TROIKA_STRICT_VALID(troika) ((troika)->tail_and_flags & 64) -#define TROIKA_VALID(troika) ((troika)->tail_and_flags & 128) -#define TROIKA_TAIL(troika) ((troika)->tail_and_flags & 3) - txnid_t txnid[NUM_METAS]; -} meta_troika_t; - -/* A database transaction. - * Every operation requires a transaction handle. */ -struct MDBX_txn { -#define MDBX_MT_SIGNATURE UINT32_C(0x93D53A31) - uint32_t mt_signature; - - /* Transaction Flags */ - /* mdbx_txn_begin() flags */ -#define MDBX_TXN_RO_BEGIN_FLAGS (MDBX_TXN_RDONLY | MDBX_TXN_RDONLY_PREPARE) -#define MDBX_TXN_RW_BEGIN_FLAGS \ - (MDBX_TXN_NOMETASYNC | MDBX_TXN_NOSYNC | MDBX_TXN_TRY) - /* Additional flag for sync_locked() */ -#define MDBX_SHRINK_ALLOWED UINT32_C(0x40000000) - -#define MDBX_TXN_DRAINED_GC 0x20 /* GC was depleted up to oldest reader */ - -#define TXN_FLAGS \ - (MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_DIRTY | MDBX_TXN_SPILLS | \ - MDBX_TXN_HAS_CHILD | MDBX_TXN_INVALID | MDBX_TXN_DRAINED_GC) - -#if (TXN_FLAGS & (MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_RO_BEGIN_FLAGS)) || \ - ((MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_RO_BEGIN_FLAGS | TXN_FLAGS) & \ - MDBX_SHRINK_ALLOWED) -#error "Oops, some txn flags overlapped or wrong" -#endif - uint32_t mt_flags; - - MDBX_txn *mt_parent; /* parent of a nested txn */ - /* Nested txn under this txn, set together with flag MDBX_TXN_HAS_CHILD */ - MDBX_txn *mt_child; - MDBX_geo mt_geo; - /* next unallocated page */ -#define mt_next_pgno mt_geo.next - /* corresponding to the current size of datafile */ -#define mt_end_pgno mt_geo.now - - /* The ID of this transaction. IDs are integers incrementing from - * INITIAL_TXNID. Only committed write transactions increment the ID. If a - * transaction aborts, the ID may be re-used by the next writer. */ - txnid_t mt_txnid; - txnid_t mt_front; - - MDBX_env *mt_env; /* the DB environment */ - /* Array of records for each DB known in the environment. */ - MDBX_dbx *mt_dbxs; - /* Array of MDBX_db records for each known DB */ - MDBX_db *mt_dbs; - /* Array of sequence numbers for each DB handle */ - MDBX_atomic_uint32_t *mt_dbiseqs; - - /* Transaction DBI Flags */ -#define DBI_DIRTY MDBX_DBI_DIRTY /* DB was written in this txn */ -#define DBI_STALE MDBX_DBI_STALE /* Named-DB record is older than txnID */ -#define DBI_FRESH MDBX_DBI_FRESH /* Named-DB handle opened in this txn */ -#define DBI_CREAT MDBX_DBI_CREAT /* Named-DB handle created in this txn */ -#define DBI_VALID 0x10 /* DB handle is valid, see also DB_VALID */ -#define DBI_USRVALID 0x20 /* As DB_VALID, but not set for FREE_DBI */ -#define DBI_AUDITED 0x40 /* Internal flag for accounting during audit */ - /* Array of flags for each DB */ - uint8_t *mt_dbistate; - /* Number of DB records in use, or 0 when the txn is finished. - * This number only ever increments until the txn finishes; we - * don't decrement it when individual DB handles are closed. */ - MDBX_dbi mt_numdbs; - size_t mt_owner; /* thread ID that owns this transaction */ - MDBX_canary mt_canary; - void *mt_userctx; /* User-settable context */ - MDBX_cursor **mt_cursors; +#define ASSERT_ENABLED() (0) +#endif /* ASSERT_ENABLED() */ - union { - struct { - /* For read txns: This thread/txn's reader table slot, or NULL. */ - MDBX_reader *reader; - } to; - struct { - meta_troika_t troika; - /* In write txns, array of cursors for each DB */ - MDBX_PNL relist; /* Reclaimed GC pages */ - txnid_t last_reclaimed; /* ID of last used record */ -#if MDBX_ENABLE_REFUND - pgno_t loose_refund_wl /* FIXME: describe */; -#endif /* MDBX_ENABLE_REFUND */ - /* a sequence to spilling dirty page with LRU policy */ - unsigned dirtylru; - /* dirtylist room: Dirty array size - dirty pages visible to this txn. - * Includes ancestor txns' dirty pages not hidden by other txns' - * dirty/spilled pages. Thus commit(nested txn) has room to merge - * dirtylist into mt_parent after freeing hidden mt_parent pages. */ - size_t dirtyroom; - /* For write txns: Modified pages. Sorted when not MDBX_WRITEMAP. */ - MDBX_dpl *dirtylist; - /* The list of reclaimed txns from GC */ - MDBX_TXL lifo_reclaimed; - /* The list of pages that became unused during this transaction. */ - MDBX_PNL retired_pages; - /* The list of loose pages that became unused and may be reused - * in this transaction, linked through `mp_next`. */ - MDBX_page *loose_pages; - /* Number of loose pages (tw.loose_pages) */ - size_t loose_count; - union { - struct { - size_t least_removed; - /* The sorted list of dirty pages we temporarily wrote to disk - * because the dirty list was full. page numbers in here are - * shifted left by 1, deleted slots have the LSB set. */ - MDBX_PNL list; - } spilled; - size_t writemap_dirty_npages; - size_t writemap_spilled_npages; - }; - } tw; - }; -}; +#define DEBUG_EXTRA(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_EXTRA)) \ + debug_log(MDBX_LOG_EXTRA, __func__, __LINE__, fmt, __VA_ARGS__); \ + } while (0) -#if MDBX_WORDBITS >= 64 -#define CURSOR_STACK 32 -#else -#define CURSOR_STACK 24 -#endif - -struct MDBX_xcursor; - -/* Cursors are used for all DB operations. - * A cursor holds a path of (page pointer, key index) from the DB - * root to a position in the DB, plus other state. MDBX_DUPSORT - * cursors include an xcursor to the current data item. Write txns - * track their cursors and keep them up to date when data moves. - * Exception: An xcursor's pointer to a P_SUBP page can be stale. - * (A node with F_DUPDATA but no F_SUBDATA contains a subpage). */ -struct MDBX_cursor { -#define MDBX_MC_LIVE UINT32_C(0xFE05D5B1) -#define MDBX_MC_READY4CLOSE UINT32_C(0x2817A047) -#define MDBX_MC_WAIT4EOT UINT32_C(0x90E297A7) - uint32_t mc_signature; - /* The database handle this cursor operates on */ - MDBX_dbi mc_dbi; - /* Next cursor on this DB in this txn */ - MDBX_cursor *mc_next; - /* Backup of the original cursor if this cursor is a shadow */ - MDBX_cursor *mc_backup; - /* Context used for databases with MDBX_DUPSORT, otherwise NULL */ - struct MDBX_xcursor *mc_xcursor; - /* The transaction that owns this cursor */ - MDBX_txn *mc_txn; - /* The database record for this cursor */ - MDBX_db *mc_db; - /* The database auxiliary record for this cursor */ - MDBX_dbx *mc_dbx; - /* The mt_dbistate for this database */ - uint8_t *mc_dbistate; - uint8_t mc_snum; /* number of pushed pages */ - uint8_t mc_top; /* index of top page, normally mc_snum-1 */ - - /* Cursor state flags. */ -#define C_INITIALIZED 0x01 /* cursor has been initialized and is valid */ -#define C_EOF 0x02 /* No more data */ -#define C_SUB 0x04 /* Cursor is a sub-cursor */ -#define C_DEL 0x08 /* last op was a cursor_del */ -#define C_UNTRACK 0x10 /* Un-track cursor when closing */ -#define C_GCU \ - 0x20 /* Происходит подготовка к обновлению GC, поэтому \ - * можно брать страницы из GC даже для FREE_DBI */ - uint8_t mc_flags; - - /* Cursor checking flags. */ -#define CC_BRANCH 0x01 /* same as P_BRANCH for CHECK_LEAF_TYPE() */ -#define CC_LEAF 0x02 /* same as P_LEAF for CHECK_LEAF_TYPE() */ -#define CC_OVERFLOW 0x04 /* same as P_OVERFLOW for CHECK_LEAF_TYPE() */ -#define CC_UPDATING 0x08 /* update/rebalance pending */ -#define CC_SKIPORD 0x10 /* don't check keys ordering */ -#define CC_LEAF2 0x20 /* same as P_LEAF2 for CHECK_LEAF_TYPE() */ -#define CC_RETIRING 0x40 /* refs to child pages may be invalid */ -#define CC_PAGECHECK 0x80 /* perform page checking, see MDBX_VALIDATION */ - uint8_t mc_checking; - - MDBX_page *mc_pg[CURSOR_STACK]; /* stack of pushed pages */ - indx_t mc_ki[CURSOR_STACK]; /* stack of page indices */ -}; +#define DEBUG_EXTRA_PRINT(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_EXTRA)) \ + debug_log(MDBX_LOG_EXTRA, nullptr, 0, fmt, __VA_ARGS__); \ + } while (0) -#define CHECK_LEAF_TYPE(mc, mp) \ - (((PAGETYPE_WHOLE(mp) ^ (mc)->mc_checking) & \ - (CC_BRANCH | CC_LEAF | CC_OVERFLOW | CC_LEAF2)) == 0) - -/* Context for sorted-dup records. - * We could have gone to a fully recursive design, with arbitrarily - * deep nesting of sub-databases. But for now we only handle these - * levels - main DB, optional sub-DB, sorted-duplicate DB. */ -typedef struct MDBX_xcursor { - /* A sub-cursor for traversing the Dup DB */ - MDBX_cursor mx_cursor; - /* The database record for this Dup DB */ - MDBX_db mx_db; - /* The auxiliary DB record for this Dup DB */ - MDBX_dbx mx_dbx; -} MDBX_xcursor; - -typedef struct MDBX_cursor_couple { - MDBX_cursor outer; - void *mc_userctx; /* User-settable context */ - MDBX_xcursor inner; -} MDBX_cursor_couple; - -/* The database environment. */ -struct MDBX_env { - /* ----------------------------------------------------- mostly static part */ -#define MDBX_ME_SIGNATURE UINT32_C(0x9A899641) - MDBX_atomic_uint32_t me_signature; - /* Failed to update the meta page. Probably an I/O error. */ -#define MDBX_FATAL_ERROR UINT32_C(0x80000000) - /* Some fields are initialized. */ -#define MDBX_ENV_ACTIVE UINT32_C(0x20000000) - /* me_txkey is set */ -#define MDBX_ENV_TXKEY UINT32_C(0x10000000) - /* Legacy MDBX_MAPASYNC (prior v0.9) */ -#define MDBX_DEPRECATED_MAPASYNC UINT32_C(0x100000) - /* Legacy MDBX_COALESCE (prior v0.12) */ -#define MDBX_DEPRECATED_COALESCE UINT32_C(0x2000000) -#define ENV_INTERNAL_FLAGS (MDBX_FATAL_ERROR | MDBX_ENV_ACTIVE | MDBX_ENV_TXKEY) - uint32_t me_flags; - osal_mmap_t me_dxb_mmap; /* The main data file */ -#define me_map me_dxb_mmap.base -#define me_lazy_fd me_dxb_mmap.fd - mdbx_filehandle_t me_dsync_fd, me_fd4meta; -#if defined(_WIN32) || defined(_WIN64) -#define me_overlapped_fd me_ioring.overlapped_fd - HANDLE me_data_lock_event; -#endif /* Windows */ - osal_mmap_t me_lck_mmap; /* The lock file */ -#define me_lfd me_lck_mmap.fd - struct MDBX_lockinfo *me_lck; - - unsigned me_psize; /* DB page size, initialized from me_os_psize */ - uint16_t me_leaf_nodemax; /* max size of a leaf-node */ - uint16_t me_branch_nodemax; /* max size of a branch-node */ - uint16_t me_subpage_limit; - uint16_t me_subpage_room_threshold; - uint16_t me_subpage_reserve_prereq; - uint16_t me_subpage_reserve_limit; - atomic_pgno_t me_mlocked_pgno; - uint8_t me_psize2log; /* log2 of DB page size */ - int8_t me_stuck_meta; /* recovery-only: target meta page or less that zero */ - uint16_t me_merge_threshold, - me_merge_threshold_gc; /* pages emptier than this are candidates for - merging */ - unsigned me_os_psize; /* OS page size, from osal_syspagesize() */ - unsigned me_maxreaders; /* size of the reader table */ - MDBX_dbi me_maxdbs; /* size of the DB table */ - uint32_t me_pid; /* process ID of this env */ - osal_thread_key_t me_txkey; /* thread-key for readers */ - pathchar_t *me_pathname; /* path to the DB files */ - void *me_pbuf; /* scratch area for DUPSORT put() */ - MDBX_txn *me_txn0; /* preallocated write transaction */ - - MDBX_dbx *me_dbxs; /* array of static DB info */ - uint16_t *me_dbflags; /* array of flags from MDBX_db.md_flags */ - MDBX_atomic_uint32_t *me_dbiseqs; /* array of dbi sequence numbers */ - unsigned - me_maxgc_ov1page; /* Number of pgno_t fit in a single overflow page */ - unsigned me_maxgc_per_branch; - uint32_t me_live_reader; /* have liveness lock in reader table */ - void *me_userctx; /* User-settable context */ - MDBX_hsr_func *me_hsr_callback; /* Callback for kicking laggard readers */ - size_t me_madv_threshold; +#define TRACE(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_TRACE)) \ + debug_log(MDBX_LOG_TRACE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ + } while (0) - struct { - unsigned dp_reserve_limit; - unsigned rp_augment_limit; - unsigned dp_limit; - unsigned dp_initial; - uint8_t dp_loose_limit; - uint8_t spill_max_denominator; - uint8_t spill_min_denominator; - uint8_t spill_parent4child_denominator; - unsigned merge_threshold_16dot16_percent; -#if !(defined(_WIN32) || defined(_WIN64)) - unsigned writethrough_threshold; -#endif /* Windows */ - bool prefault_write; - union { - unsigned all; - /* tracks options with non-auto values but tuned by user */ - struct { - unsigned dp_limit : 1; - unsigned rp_augment_limit : 1; - unsigned prefault_write : 1; - } non_auto; - } flags; - } me_options; - - /* struct me_dbgeo used for accepting db-geo params from user for the new - * database creation, i.e. when mdbx_env_set_geometry() was called before - * mdbx_env_open(). */ - struct { - size_t lower; /* minimal size of datafile */ - size_t upper; /* maximal size of datafile */ - size_t now; /* current size of datafile */ - size_t grow; /* step to grow datafile */ - size_t shrink; /* threshold to shrink datafile */ - } me_dbgeo; - -#if MDBX_LOCKING == MDBX_LOCKING_SYSV - union { - key_t key; - int semid; - } me_sysv_ipc; -#endif /* MDBX_LOCKING == MDBX_LOCKING_SYSV */ - bool me_incore; +#define DEBUG(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_DEBUG)) \ + debug_log(MDBX_LOG_DEBUG, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ + } while (0) - MDBX_env *me_lcklist_next; +#define VERBOSE(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_VERBOSE)) \ + debug_log(MDBX_LOG_VERBOSE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ + } while (0) - /* --------------------------------------------------- mostly volatile part */ +#define NOTICE(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_NOTICE)) \ + debug_log(MDBX_LOG_NOTICE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ + } while (0) - MDBX_txn *me_txn; /* current write transaction */ - osal_fastmutex_t me_dbi_lock; - MDBX_dbi me_numdbs; /* number of DBs opened */ - bool me_prefault_write; +#define WARNING(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_WARN)) \ + debug_log(MDBX_LOG_WARN, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ + } while (0) - MDBX_page *me_dp_reserve; /* list of malloc'ed blocks for re-use */ - unsigned me_dp_reserve_len; - /* PNL of pages that became unused in a write txn */ - MDBX_PNL me_retired_pages; - osal_ioring_t me_ioring; +#undef ERROR /* wingdi.h \ + Yeah, morons from M$ put such definition to the public header. */ -#if defined(_WIN32) || defined(_WIN64) - osal_srwlock_t me_remap_guard; - /* Workaround for LockFileEx and WriteFile multithread bug */ - CRITICAL_SECTION me_windowsbug_lock; - char *me_pathname_char; /* cache of multi-byte representation of pathname - to the DB files */ -#else - osal_fastmutex_t me_remap_guard; -#endif +#define ERROR(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_ERROR)) \ + debug_log(MDBX_LOG_ERROR, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ + } while (0) - /* -------------------------------------------------------------- debugging */ +#define FATAL(fmt, ...) debug_log(MDBX_LOG_FATAL, __func__, __LINE__, fmt "\n", __VA_ARGS__); #if MDBX_DEBUG - MDBX_assert_func *me_assert_func; /* Callback for assertion failures */ -#endif -#ifdef MDBX_USE_VALGRIND - int me_valgrind_handle; -#endif -#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__) - MDBX_atomic_uint32_t me_ignore_EDEADLK; - pgno_t me_poison_edge; -#endif /* MDBX_USE_VALGRIND || __SANITIZE_ADDRESS__ */ +#define ASSERT_FAIL(env, msg, func, line) mdbx_assert_fail(env, msg, func, line) +#else /* MDBX_DEBUG */ +MDBX_NORETURN __cold void assert_fail(const char *msg, const char *func, unsigned line); +#define ASSERT_FAIL(env, msg, func, line) \ + do { \ + (void)(env); \ + assert_fail(msg, func, line); \ + } while (0) +#endif /* MDBX_DEBUG */ -#ifndef xMDBX_DEBUG_SPILLING -#define xMDBX_DEBUG_SPILLING 0 -#endif -#if xMDBX_DEBUG_SPILLING == 2 - size_t debug_dirtied_est, debug_dirtied_act; -#endif /* xMDBX_DEBUG_SPILLING */ +#define ENSURE_MSG(env, expr, msg) \ + do { \ + if (unlikely(!(expr))) \ + ASSERT_FAIL(env, msg, __func__, __LINE__); \ + } while (0) - /* ------------------------------------------------- stub for lck-less mode */ - MDBX_atomic_uint64_t - x_lckless_stub[(sizeof(MDBX_lockinfo) + MDBX_CACHELINE_SIZE - 1) / - sizeof(MDBX_atomic_uint64_t)]; -}; +#define ENSURE(env, expr) ENSURE_MSG(env, expr, #expr) -#ifndef __cplusplus -/*----------------------------------------------------------------------------*/ -/* Cache coherence and mmap invalidation */ +/* assert(3) variant in environment context */ +#define eASSERT(env, expr) \ + do { \ + if (ASSERT_ENABLED()) \ + ENSURE(env, expr); \ + } while (0) -#if MDBX_CPU_WRITEBACK_INCOHERENT -#define osal_flush_incoherent_cpu_writeback() osal_memory_barrier() -#else -#define osal_flush_incoherent_cpu_writeback() osal_compiler_barrier() -#endif /* MDBX_CPU_WRITEBACK_INCOHERENT */ +/* assert(3) variant in cursor context */ +#define cASSERT(mc, expr) eASSERT((mc)->txn->env, expr) -MDBX_MAYBE_UNUSED static __inline void -osal_flush_incoherent_mmap(const void *addr, size_t nbytes, - const intptr_t pagesize) { -#if MDBX_MMAP_INCOHERENT_FILE_WRITE - char *const begin = (char *)(-pagesize & (intptr_t)addr); - char *const end = - (char *)(-pagesize & (intptr_t)((char *)addr + nbytes + pagesize - 1)); - int err = msync(begin, end - begin, MS_SYNC | MS_INVALIDATE) ? errno : 0; - eASSERT(nullptr, err == 0); - (void)err; -#else - (void)pagesize; -#endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */ +/* assert(3) variant in transaction context */ +#define tASSERT(txn, expr) eASSERT((txn)->env, expr) -#if MDBX_MMAP_INCOHERENT_CPU_CACHE -#ifdef DCACHE - /* MIPS has cache coherency issues. - * Note: for any nbytes >= on-chip cache size, entire is flushed. */ - cacheflush((void *)addr, nbytes, DCACHE); -#else -#error "Oops, cacheflush() not available" -#endif /* DCACHE */ -#endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */ +#ifndef xMDBX_TOOLS /* Avoid using internal eASSERT() */ +#undef assert +#define assert(expr) eASSERT(nullptr, expr) +#endif -#if !MDBX_MMAP_INCOHERENT_FILE_WRITE && !MDBX_MMAP_INCOHERENT_CPU_CACHE - (void)addr; - (void)nbytes; +MDBX_MAYBE_UNUSED static inline void jitter4testing(bool tiny) { +#if MDBX_DEBUG + if (globals.runtime_flags & (unsigned)MDBX_DBG_JITTER) + osal_jitter(tiny); +#else + (void)tiny; #endif } -/*----------------------------------------------------------------------------*/ -/* Internal prototypes */ - -MDBX_INTERNAL_FUNC int cleanup_dead_readers(MDBX_env *env, int rlocked, - int *dead); -MDBX_INTERNAL_FUNC int rthc_alloc(osal_thread_key_t *key, MDBX_reader *begin, - MDBX_reader *end); -MDBX_INTERNAL_FUNC void rthc_remove(const osal_thread_key_t key); - -MDBX_INTERNAL_FUNC void global_ctor(void); -MDBX_INTERNAL_FUNC void osal_ctor(void); -MDBX_INTERNAL_FUNC void global_dtor(void); -MDBX_INTERNAL_FUNC void osal_dtor(void); -MDBX_INTERNAL_FUNC void thread_dtor(void *ptr); - -#endif /* !__cplusplus */ - -#define MDBX_IS_ERROR(rc) \ - ((rc) != MDBX_RESULT_TRUE && (rc) != MDBX_RESULT_FALSE) - -/* Internal error codes, not exposed outside libmdbx */ -#define MDBX_NO_ROOT (MDBX_LAST_ADDED_ERRCODE + 10) - -/* Debugging output value of a cursor DBI: Negative in a sub-cursor. */ -#define DDBI(mc) \ - (((mc)->mc_flags & C_SUB) ? -(int)(mc)->mc_dbi : (int)(mc)->mc_dbi) +MDBX_MAYBE_UNUSED MDBX_INTERNAL void page_list(page_t *mp); +MDBX_INTERNAL const char *pagetype_caption(const uint8_t type, char buf4unknown[16]); /* Key size which fits in a DKBUF (debug key buffer). */ -#define DKBUF_MAX 511 -#define DKBUF char _kbuf[DKBUF_MAX * 4 + 2] -#define DKEY(x) mdbx_dump_val(x, _kbuf, DKBUF_MAX * 2 + 1) -#define DVAL(x) mdbx_dump_val(x, _kbuf + DKBUF_MAX * 2 + 1, DKBUF_MAX * 2 + 1) +#define DKBUF_MAX 127 +#define DKBUF char dbg_kbuf[DKBUF_MAX * 4 + 2] +#define DKEY(x) mdbx_dump_val(x, dbg_kbuf, DKBUF_MAX * 2 + 1) +#define DVAL(x) mdbx_dump_val(x, dbg_kbuf + DKBUF_MAX * 2 + 1, DKBUF_MAX * 2 + 1) #if MDBX_DEBUG #define DKBUF_DEBUG DKBUF @@ -3864,102 +2979,24 @@ MDBX_INTERNAL_FUNC void thread_dtor(void *ptr); #define DVAL_DEBUG(x) ("-") #endif -/* An invalid page number. - * Mainly used to denote an empty tree. */ -#define P_INVALID (~(pgno_t)0) +MDBX_INTERNAL void log_error(const int err, const char *func, unsigned line); + +MDBX_MAYBE_UNUSED static inline int log_if_error(const int err, const char *func, unsigned line) { + if (unlikely(err != MDBX_SUCCESS)) + log_error(err, func, line); + return err; +} + +#define LOG_IFERR(err) log_if_error((err), __func__, __LINE__) /* Test if the flags f are set in a flag word w. */ #define F_ISSET(w, f) (((w) & (f)) == (f)) /* Round n up to an even number. */ -#define EVEN(n) (((n) + 1UL) & -2L) /* sign-extending -2 to match n+1U */ - -/* Default size of memory map. - * This is certainly too small for any actual applications. Apps should - * always set the size explicitly using mdbx_env_set_geometry(). */ -#define DEFAULT_MAPSIZE MEGABYTE - -/* Number of slots in the reader table. - * This value was chosen somewhat arbitrarily. The 61 is a prime number, - * and such readers plus a couple mutexes fit into single 4KB page. - * Applications should set the table size using mdbx_env_set_maxreaders(). */ -#define DEFAULT_READERS 61 - -/* Test if a page is a leaf page */ -#define IS_LEAF(p) (((p)->mp_flags & P_LEAF) != 0) -/* Test if a page is a LEAF2 page */ -#define IS_LEAF2(p) unlikely(((p)->mp_flags & P_LEAF2) != 0) -/* Test if a page is a branch page */ -#define IS_BRANCH(p) (((p)->mp_flags & P_BRANCH) != 0) -/* Test if a page is an overflow page */ -#define IS_OVERFLOW(p) unlikely(((p)->mp_flags & P_OVERFLOW) != 0) -/* Test if a page is a sub page */ -#define IS_SUBP(p) (((p)->mp_flags & P_SUBP) != 0) - -/* Header for a single key/data pair within a page. - * Used in pages of type P_BRANCH and P_LEAF without P_LEAF2. - * We guarantee 2-byte alignment for 'MDBX_node's. - * - * Leaf node flags describe node contents. F_BIGDATA says the node's - * data part is the page number of an overflow page with actual data. - * F_DUPDATA and F_SUBDATA can be combined giving duplicate data in - * a sub-page/sub-database, and named databases (just F_SUBDATA). */ -typedef struct MDBX_node { -#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ - union { - uint32_t mn_dsize; - uint32_t mn_pgno32; - }; - uint8_t mn_flags; /* see mdbx_node flags */ - uint8_t mn_extra; - uint16_t mn_ksize; /* key size */ -#else - uint16_t mn_ksize; /* key size */ - uint8_t mn_extra; - uint8_t mn_flags; /* see mdbx_node flags */ - union { - uint32_t mn_pgno32; - uint32_t mn_dsize; - }; -#endif /* __BYTE_ORDER__ */ - - /* mdbx_node Flags */ -#define F_BIGDATA 0x01 /* data put on overflow page */ -#define F_SUBDATA 0x02 /* data is a sub-database */ -#define F_DUPDATA 0x04 /* data has duplicates */ - - /* valid flags for mdbx_node_add() */ -#define NODE_ADD_FLAGS (F_DUPDATA | F_SUBDATA | MDBX_RESERVE | MDBX_APPEND) - -#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ - (!defined(__cplusplus) && defined(_MSC_VER)) - uint8_t mn_data[] /* key and data are appended here */; -#endif /* C99 */ -} MDBX_node; - -#define DB_PERSISTENT_FLAGS \ - (MDBX_REVERSEKEY | MDBX_DUPSORT | MDBX_INTEGERKEY | MDBX_DUPFIXED | \ - MDBX_INTEGERDUP | MDBX_REVERSEDUP) - -/* mdbx_dbi_open() flags */ -#define DB_USABLE_FLAGS (DB_PERSISTENT_FLAGS | MDBX_CREATE | MDBX_DB_ACCEDE) - -#define DB_VALID 0x8000 /* DB handle is valid, for me_dbflags */ -#define DB_INTERNAL_FLAGS DB_VALID - -#if DB_INTERNAL_FLAGS & DB_USABLE_FLAGS -#error "Oops, some flags overlapped or wrong" -#endif -#if DB_PERSISTENT_FLAGS & ~DB_USABLE_FLAGS -#error "Oops, some flags overlapped or wrong" -#endif +#define EVEN_CEIL(n) (((n) + 1UL) & -2L) /* sign-extending -2 to match n+1U */ -/* Max length of iov-vector passed to writev() call, used for auxilary writes */ -#define MDBX_AUXILARY_IOV_MAX 64 -#if defined(IOV_MAX) && IOV_MAX < MDBX_AUXILARY_IOV_MAX -#undef MDBX_AUXILARY_IOV_MAX -#define MDBX_AUXILARY_IOV_MAX IOV_MAX -#endif /* MDBX_AUXILARY_IOV_MAX */ +/* Round n down to an even number. */ +#define EVEN_FLOOR(n) ((n) & ~(size_t)1) /* * / @@ -3970,121 +3007,228 @@ typedef struct MDBX_node { */ #define CMP2INT(a, b) (((a) != (b)) ? (((a) < (b)) ? -1 : 1) : 0) -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline pgno_t -int64pgno(int64_t i64) { - if (likely(i64 >= (int64_t)MIN_PAGENO && i64 <= (int64_t)MAX_PAGENO + 1)) - return (pgno_t)i64; - return (i64 < (int64_t)MIN_PAGENO) ? MIN_PAGENO : MAX_PAGENO; -} +/* Pointer displacement without casting to char* to avoid pointer-aliasing */ +#define ptr_disp(ptr, disp) ((void *)(((intptr_t)(ptr)) + ((intptr_t)(disp)))) -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline pgno_t -pgno_add(size_t base, size_t augend) { - assert(base <= MAX_PAGENO + 1 && augend < MAX_PAGENO); - return int64pgno((int64_t)base + (int64_t)augend); -} +/* Pointer distance as signed number of bytes */ +#define ptr_dist(more, less) (((intptr_t)(more)) - ((intptr_t)(less))) -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline pgno_t -pgno_sub(size_t base, size_t subtrahend) { - assert(base >= MIN_PAGENO && base <= MAX_PAGENO + 1 && - subtrahend < MAX_PAGENO); - return int64pgno((int64_t)base - (int64_t)subtrahend); -} +#define MDBX_ASAN_POISON_MEMORY_REGION(addr, size) \ + do { \ + TRACE("POISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), (size_t)(size), __LINE__); \ + ASAN_POISON_MEMORY_REGION(addr, size); \ + } while (0) + +#define MDBX_ASAN_UNPOISON_MEMORY_REGION(addr, size) \ + do { \ + TRACE("UNPOISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), (size_t)(size), __LINE__); \ + ASAN_UNPOISON_MEMORY_REGION(addr, size); \ + } while (0) -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __always_inline bool -is_powerof2(size_t x) { - return (x & (x - 1)) == 0; +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline size_t branchless_abs(intptr_t value) { + assert(value > INT_MIN); + const size_t expanded_sign = (size_t)(value >> (sizeof(value) * CHAR_BIT - 1)); + return ((size_t)value + expanded_sign) ^ expanded_sign; } -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __always_inline size_t -floor_powerof2(size_t value, size_t granularity) { +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline bool is_powerof2(size_t x) { return (x & (x - 1)) == 0; } + +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline size_t floor_powerof2(size_t value, size_t granularity) { assert(is_powerof2(granularity)); return value & ~(granularity - 1); } -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __always_inline size_t -ceil_powerof2(size_t value, size_t granularity) { +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline size_t ceil_powerof2(size_t value, size_t granularity) { return floor_powerof2(value + granularity - 1, granularity); } -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static unsigned -log2n_powerof2(size_t value_uintptr) { - assert(value_uintptr > 0 && value_uintptr < INT32_MAX && - is_powerof2(value_uintptr)); - assert((value_uintptr & -(intptr_t)value_uintptr) == value_uintptr); - const uint32_t value_uint32 = (uint32_t)value_uintptr; -#if __GNUC_PREREQ(4, 1) || __has_builtin(__builtin_ctz) - STATIC_ASSERT(sizeof(value_uint32) <= sizeof(unsigned)); - return __builtin_ctz(value_uint32); -#elif defined(_MSC_VER) - unsigned long index; - STATIC_ASSERT(sizeof(value_uint32) <= sizeof(long)); - _BitScanForward(&index, value_uint32); - return index; -#else - static const uint8_t debruijn_ctz32[32] = { - 0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8, - 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9}; - return debruijn_ctz32[(uint32_t)(value_uint32 * 0x077CB531ul) >> 27]; -#endif -} +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED MDBX_INTERNAL unsigned log2n_powerof2(size_t value_uintptr); -/* Only a subset of the mdbx_env flags can be changed - * at runtime. Changing other flags requires closing the - * environment and re-opening it with the new flags. */ -#define ENV_CHANGEABLE_FLAGS \ - (MDBX_SAFE_NOSYNC | MDBX_NOMETASYNC | MDBX_DEPRECATED_MAPASYNC | \ - MDBX_NOMEMINIT | MDBX_COALESCE | MDBX_PAGEPERTURB | MDBX_ACCEDE | \ - MDBX_VALIDATION) -#define ENV_CHANGELESS_FLAGS \ - (MDBX_NOSUBDIR | MDBX_RDONLY | MDBX_WRITEMAP | MDBX_NOTLS | MDBX_NORDAHEAD | \ - MDBX_LIFORECLAIM | MDBX_EXCLUSIVE) -#define ENV_USABLE_FLAGS (ENV_CHANGEABLE_FLAGS | ENV_CHANGELESS_FLAGS) - -#if !defined(__cplusplus) || CONSTEXPR_ENUM_FLAGS_OPERATIONS -MDBX_MAYBE_UNUSED static void static_checks(void) { - STATIC_ASSERT_MSG(INT16_MAX - CORE_DBS == MDBX_MAX_DBI, - "Oops, MDBX_MAX_DBI or CORE_DBS?"); - STATIC_ASSERT_MSG((unsigned)(MDBX_DB_ACCEDE | MDBX_CREATE) == - ((DB_USABLE_FLAGS | DB_INTERNAL_FLAGS) & - (ENV_USABLE_FLAGS | ENV_INTERNAL_FLAGS)), - "Oops, some flags overlapped or wrong"); - STATIC_ASSERT_MSG((ENV_INTERNAL_FLAGS & ENV_USABLE_FLAGS) == 0, - "Oops, some flags overlapped or wrong"); -} -#endif /* Disabled for MSVC 19.0 (VisualStudio 2015) */ +MDBX_NOTHROW_CONST_FUNCTION MDBX_INTERNAL uint64_t rrxmrrxmsx_0(uint64_t v); -#ifdef __cplusplus -} -#endif +struct monotime_cache { + uint64_t value; + int expire_countdown; +}; -#define MDBX_ASAN_POISON_MEMORY_REGION(addr, size) \ - do { \ - TRACE("POISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), \ - (size_t)(size), __LINE__); \ - ASAN_POISON_MEMORY_REGION(addr, size); \ - } while (0) +MDBX_MAYBE_UNUSED static inline uint64_t monotime_since_cached(uint64_t begin_timestamp, struct monotime_cache *cache) { + if (cache->expire_countdown) + cache->expire_countdown -= 1; + else { + cache->value = osal_monotime(); + cache->expire_countdown = 42 / 3; + } + return cache->value - begin_timestamp; +} -#define MDBX_ASAN_UNPOISON_MEMORY_REGION(addr, size) \ - do { \ - TRACE("UNPOISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), \ - (size_t)(size), __LINE__); \ - ASAN_UNPOISON_MEMORY_REGION(addr, size); \ +/* An PNL is an Page Number List, a sorted array of IDs. + * + * The first element of the array is a counter for how many actual page-numbers + * are in the list. By default PNLs are sorted in descending order, this allow + * cut off a page with lowest pgno (at the tail) just truncating the list. The + * sort order of PNLs is controlled by the MDBX_PNL_ASCENDING build option. */ +typedef pgno_t *pnl_t; +typedef const pgno_t *const_pnl_t; + +#if MDBX_PNL_ASCENDING +#define MDBX_PNL_ORDERED(first, last) ((first) < (last)) +#define MDBX_PNL_DISORDERED(first, last) ((first) >= (last)) +#else +#define MDBX_PNL_ORDERED(first, last) ((first) > (last)) +#define MDBX_PNL_DISORDERED(first, last) ((first) <= (last)) +#endif + +#define MDBX_PNL_GRANULATE_LOG2 10 +#define MDBX_PNL_GRANULATE (1 << MDBX_PNL_GRANULATE_LOG2) +#define MDBX_PNL_INITIAL (MDBX_PNL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t)) + +#define MDBX_PNL_ALLOCLEN(pl) ((pl)[-1]) +#define MDBX_PNL_GETSIZE(pl) ((size_t)((pl)[0])) +#define MDBX_PNL_SETSIZE(pl, size) \ + do { \ + const size_t __size = size; \ + assert(__size < INT_MAX); \ + (pl)[0] = (pgno_t)__size; \ } while (0) +#define MDBX_PNL_FIRST(pl) ((pl)[1]) +#define MDBX_PNL_LAST(pl) ((pl)[MDBX_PNL_GETSIZE(pl)]) +#define MDBX_PNL_BEGIN(pl) (&(pl)[1]) +#define MDBX_PNL_END(pl) (&(pl)[MDBX_PNL_GETSIZE(pl) + 1]) -#include +#if MDBX_PNL_ASCENDING +#define MDBX_PNL_EDGE(pl) ((pl) + 1) +#define MDBX_PNL_LEAST(pl) MDBX_PNL_FIRST(pl) +#define MDBX_PNL_MOST(pl) MDBX_PNL_LAST(pl) +#else +#define MDBX_PNL_EDGE(pl) ((pl) + MDBX_PNL_GETSIZE(pl)) +#define MDBX_PNL_LEAST(pl) MDBX_PNL_LAST(pl) +#define MDBX_PNL_MOST(pl) MDBX_PNL_FIRST(pl) +#endif + +#define MDBX_PNL_SIZEOF(pl) ((MDBX_PNL_GETSIZE(pl) + 1) * sizeof(pgno_t)) +#define MDBX_PNL_IS_EMPTY(pl) (MDBX_PNL_GETSIZE(pl) == 0) + +MDBX_MAYBE_UNUSED static inline size_t pnl_size2bytes(size_t size) { + assert(size > 0 && size <= PAGELIST_LIMIT); +#if MDBX_PNL_PREALLOC_FOR_RADIXSORT -typedef struct flagbit { - int bit; - const char *name; -} flagbit; + size += size; +#endif /* MDBX_PNL_PREALLOC_FOR_RADIXSORT */ + STATIC_ASSERT(MDBX_ASSUME_MALLOC_OVERHEAD + + (PAGELIST_LIMIT * (MDBX_PNL_PREALLOC_FOR_RADIXSORT + 1) + MDBX_PNL_GRANULATE + 3) * sizeof(pgno_t) < + SIZE_MAX / 4 * 3); + size_t bytes = + ceil_powerof2(MDBX_ASSUME_MALLOC_OVERHEAD + sizeof(pgno_t) * (size + 3), MDBX_PNL_GRANULATE * sizeof(pgno_t)) - + MDBX_ASSUME_MALLOC_OVERHEAD; + return bytes; +} + +MDBX_MAYBE_UNUSED static inline pgno_t pnl_bytes2size(const size_t bytes) { + size_t size = bytes / sizeof(pgno_t); + assert(size > 3 && size <= PAGELIST_LIMIT + /* alignment gap */ 65536); + size -= 3; +#if MDBX_PNL_PREALLOC_FOR_RADIXSORT + size >>= 1; +#endif /* MDBX_PNL_PREALLOC_FOR_RADIXSORT */ + return (pgno_t)size; +} -const flagbit dbflags[] = {{MDBX_DUPSORT, "dupsort"}, - {MDBX_INTEGERKEY, "integerkey"}, - {MDBX_REVERSEKEY, "reversekey"}, - {MDBX_DUPFIXED, "dupfixed"}, - {MDBX_REVERSEDUP, "reversedup"}, - {MDBX_INTEGERDUP, "integerdup"}, - {0, nullptr}}; +MDBX_INTERNAL pnl_t pnl_alloc(size_t size); + +MDBX_INTERNAL void pnl_free(pnl_t pnl); + +MDBX_INTERNAL int pnl_reserve(pnl_t __restrict *__restrict ppnl, const size_t wanna); + +MDBX_MAYBE_UNUSED static inline int __must_check_result pnl_need(pnl_t __restrict *__restrict ppnl, size_t num) { + assert(MDBX_PNL_GETSIZE(*ppnl) <= PAGELIST_LIMIT && MDBX_PNL_ALLOCLEN(*ppnl) >= MDBX_PNL_GETSIZE(*ppnl)); + assert(num <= PAGELIST_LIMIT); + const size_t wanna = MDBX_PNL_GETSIZE(*ppnl) + num; + return likely(MDBX_PNL_ALLOCLEN(*ppnl) >= wanna) ? MDBX_SUCCESS : pnl_reserve(ppnl, wanna); +} + +MDBX_MAYBE_UNUSED static inline void pnl_append_prereserved(__restrict pnl_t pnl, pgno_t pgno) { + assert(MDBX_PNL_GETSIZE(pnl) < MDBX_PNL_ALLOCLEN(pnl)); + if (AUDIT_ENABLED()) { + for (size_t i = MDBX_PNL_GETSIZE(pnl); i > 0; --i) + assert(pgno != pnl[i]); + } + *pnl += 1; + MDBX_PNL_LAST(pnl) = pgno; +} + +MDBX_INTERNAL void pnl_shrink(pnl_t __restrict *__restrict ppnl); + +MDBX_INTERNAL int __must_check_result spill_append_span(__restrict pnl_t *ppnl, pgno_t pgno, size_t n); + +MDBX_INTERNAL int __must_check_result pnl_append_span(__restrict pnl_t *ppnl, pgno_t pgno, size_t n); + +MDBX_INTERNAL int __must_check_result pnl_insert_span(__restrict pnl_t *ppnl, pgno_t pgno, size_t n); + +MDBX_INTERNAL size_t pnl_search_nochk(const pnl_t pnl, pgno_t pgno); + +MDBX_INTERNAL void pnl_sort_nochk(pnl_t pnl); + +MDBX_INTERNAL bool pnl_check(const const_pnl_t pnl, const size_t limit); + +MDBX_MAYBE_UNUSED static inline bool pnl_check_allocated(const const_pnl_t pnl, const size_t limit) { + return pnl == nullptr || (MDBX_PNL_ALLOCLEN(pnl) >= MDBX_PNL_GETSIZE(pnl) && pnl_check(pnl, limit)); +} + +MDBX_MAYBE_UNUSED static inline void pnl_sort(pnl_t pnl, size_t limit4check) { + pnl_sort_nochk(pnl); + assert(pnl_check(pnl, limit4check)); + (void)limit4check; +} + +MDBX_MAYBE_UNUSED static inline size_t pnl_search(const pnl_t pnl, pgno_t pgno, size_t limit) { + assert(pnl_check_allocated(pnl, limit)); + if (MDBX_HAVE_CMOV) { + /* cmov-ускоренный бинарный поиск может читать (но не использовать) один + * элемент за концом данных, этот элемент в пределах выделенного участка + * памяти, но не инициализирован. */ + VALGRIND_MAKE_MEM_DEFINED(MDBX_PNL_END(pnl), sizeof(pgno_t)); + } + assert(pgno < limit); + (void)limit; + size_t n = pnl_search_nochk(pnl, pgno); + if (MDBX_HAVE_CMOV) { + VALGRIND_MAKE_MEM_UNDEFINED(MDBX_PNL_END(pnl), sizeof(pgno_t)); + } + return n; +} + +MDBX_INTERNAL size_t pnl_merge(pnl_t dst, const pnl_t src); + +#ifdef __cplusplus +} +#endif /* __cplusplus */ + +#define mdbx_sourcery_anchor XCONCAT(mdbx_sourcery_, MDBX_BUILD_SOURCERY) +#if defined(xMDBX_TOOLS) +extern LIBMDBX_API const char *const mdbx_sourcery_anchor; +#endif + +#define MDBX_IS_ERROR(rc) ((rc) != MDBX_RESULT_TRUE && (rc) != MDBX_RESULT_FALSE) + +/*----------------------------------------------------------------------------*/ + +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline pgno_t int64pgno(int64_t i64) { + if (likely(i64 >= (int64_t)MIN_PAGENO && i64 <= (int64_t)MAX_PAGENO + 1)) + return (pgno_t)i64; + return (i64 < (int64_t)MIN_PAGENO) ? MIN_PAGENO : MAX_PAGENO; +} + +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline pgno_t pgno_add(size_t base, size_t augend) { + assert(base <= MAX_PAGENO + 1 && augend < MAX_PAGENO); + return int64pgno((int64_t)base + (int64_t)augend); +} + +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline pgno_t pgno_sub(size_t base, size_t subtrahend) { + assert(base >= MIN_PAGENO && base <= MAX_PAGENO + 1 && subtrahend < MAX_PAGENO); + return int64pgno((int64_t)base - (int64_t)subtrahend); +} + +#include #if defined(_WIN32) || defined(_WIN64) /* @@ -4100,12 +3244,12 @@ const flagbit dbflags[] = {{MDBX_DUPSORT, "dupsort"}, #ifdef _MSC_VER #pragma warning(push, 1) -#pragma warning(disable : 4548) /* expression before comma has no effect; \ +#pragma warning(disable : 4548) /* expression before comma has no effect; \ expected expression with side - effect */ -#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ +#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ * semantics are not enabled. Specify /EHsc */ -#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ - * mode specified; termination on exception is \ +#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ + * mode specified; termination on exception is \ * not guaranteed. Specify /EHsc */ #if !defined(_CRT_SECURE_NO_WARNINGS) #define _CRT_SECURE_NO_WARNINGS @@ -4158,8 +3302,7 @@ int getopt(int argc, char *const argv[], const char *opts) { if (argv[optind][sp + 1] != '\0') optarg = &argv[optind++][sp + 1]; else if (++optind >= argc) { - fprintf(stderr, "%s: %s -- %c\n", argv[0], "option requires an argument", - c); + fprintf(stderr, "%s: %s -- %c\n", argv[0], "option requires an argument", c); sp = 1; return '?'; } else @@ -4184,8 +3327,7 @@ static BOOL WINAPI ConsoleBreakHandlerRoutine(DWORD dwCtrlType) { static uint64_t GetMilliseconds(void) { LARGE_INTEGER Counter, Frequency; - return (QueryPerformanceFrequency(&Frequency) && - QueryPerformanceCounter(&Counter)) + return (QueryPerformanceFrequency(&Frequency) && QueryPerformanceCounter(&Counter)) ? Counter.QuadPart * 1000ul / Frequency.QuadPart : 0; } @@ -4206,181 +3348,162 @@ static void signal_handler(int sig) { #define EXIT_FAILURE_CHECK_MAJOR (EXIT_FAILURE + 1) #define EXIT_FAILURE_CHECK_MINOR EXIT_FAILURE -typedef struct { - MDBX_val name; - struct { - uint64_t branch, large_count, large_volume, leaf; - uint64_t subleaf_dupsort, leaf_dupfixed, subleaf_dupfixed; - uint64_t total, empty, other; - } pages; - uint64_t payload_bytes; - uint64_t lost_bytes; -} walk_dbi_t; - -struct { - short *pagemap; - uint64_t total_payload_bytes; - uint64_t pgcount; - walk_dbi_t - dbi[MDBX_MAX_DBI + CORE_DBS + /* account pseudo-entry for meta */ 1]; -} walk; - -#define dbi_free walk.dbi[FREE_DBI] -#define dbi_main walk.dbi[MAIN_DBI] -#define dbi_meta walk.dbi[CORE_DBS] - -int envflags = MDBX_RDONLY | MDBX_EXCLUSIVE | MDBX_VALIDATION; +MDBX_env_flags_t env_flags = MDBX_RDONLY | MDBX_EXCLUSIVE | MDBX_VALIDATION; MDBX_env *env; MDBX_txn *txn; -MDBX_envinfo envinfo; -size_t userdb_count, skipped_subdb; -uint64_t total_unused_bytes, reclaimable_pages, gc_pages, alloc_pages, - unused_pages, backed_pages; -unsigned verbose; -bool ignore_wrong_order, quiet, dont_traversal; -MDBX_val only_subdb; +unsigned verbose = 0; +bool quiet; +MDBX_val only_table; int stuck_meta = -1; +MDBX_chk_context_t chk; +bool turn_meta = false; +bool force_turn_meta = false; +MDBX_chk_flags_t chk_flags = MDBX_CHK_DEFAULTS; +MDBX_chk_stage_t chk_stage = MDBX_chk_none; + +static MDBX_chk_line_t line_struct; +static size_t anchor_lineno; +static size_t line_count; +static FILE *line_output; + +#define LINE_SEVERITY_NONE 255 +static bool lf(void) { + if (!line_struct.empty) { + line_count += 1; + line_struct.empty = true; + line_struct.severity = LINE_SEVERITY_NONE; + line_struct.scope_depth = 0; + if (line_output) { + fputc('\n', line_output); + return true; + } + } + return false; +} -struct problem { - struct problem *pr_next; - size_t count; - const char *caption; -}; - -struct problem *problems_list; -unsigned total_problems, data_tree_problems, gc_tree_problems; - -static void MDBX_PRINTF_ARGS(1, 2) print(const char *msg, ...) { - if (!quiet) { - va_list args; +static void flush(void) { fflush(nullptr); } - fflush(stderr); - va_start(args, msg); - vfprintf(stdout, msg, args); - va_end(args); - } +static void lf_flush(void) { + if (lf()) + flush(); } -static MDBX_val printable_buf; -static void free_printable_buf(void) { osal_free(printable_buf.iov_base); } - -static const char *sdb_name(const MDBX_val *val) { - if (val == MDBX_PGWALK_MAIN) - return "@MAIN"; - if (val == MDBX_PGWALK_GC) - return "@GC"; - if (val == MDBX_PGWALK_META) - return "@META"; - - const unsigned char *const data = val->iov_base; - const size_t len = val->iov_len; - if (data == MDBX_PGWALK_MAIN) - return "@MAIN"; - if (data == MDBX_PGWALK_GC) - return "@GC"; - if (data == MDBX_PGWALK_META) - return "@META"; - - if (!len) - return ""; - if (!data) - return ""; - if (len > 65536) { - static char buf[64]; - /* NOTE: There is MSYS2 MinGW bug if you here got - * the "unknown conversion type character ‘z’ in format [-Werror=format=]" - * https://stackoverflow.com/questions/74504432/whats-the-proper-way-to-tell-mingw-based-gcc-to-use-ansi-stdio-output-on-windo - */ - snprintf(buf, sizeof(buf), "", len); - return buf; - } +static bool silently(enum MDBX_chk_severity severity) { + int cutoff = chk.scope ? chk.scope->verbosity >> MDBX_chk_severity_prio_shift + : verbose + (MDBX_chk_result >> MDBX_chk_severity_prio_shift); + int prio = (severity >> MDBX_chk_severity_prio_shift); + if (chk.scope && chk.scope->stage == MDBX_chk_tables && verbose < 2) + prio += 1; + return quiet || cutoff < ((prio > 0) ? prio : 0); +} - bool printable = true; - bool quoting = false; - size_t xchars = 0; - for (size_t i = 0; i < val->iov_len && printable; ++i) { - quoting |= data[i] != '_' && isalnum(data[i]) == 0; - printable = isprint(data[i]) != 0 || - (data[i] < ' ' && ++xchars < 4 && len > xchars * 4); - } +static FILE *prefix(enum MDBX_chk_severity severity) { + if (silently(severity)) + return nullptr; - size_t need = len + 1; - if (quoting || !printable) - need += len + /* quotes */ 2 + 2 * /* max xchars */ 4; - if (need > printable_buf.iov_len) { - void *ptr = osal_realloc(printable_buf.iov_base, need); - if (!ptr) - return ""; - if (!printable_buf.iov_base) - atexit(free_printable_buf); - printable_buf.iov_base = ptr; - printable_buf.iov_len = need; - } + static const char *const prefixes[16] = { + "!!!fatal: ", // 0 fatal + " ! ", // 1 error + " ~ ", // 2 warning + " ", // 3 notice + "", // 4 result + " = ", // 5 resolution + " - ", // 6 processing + " ", // 7 info + " ", // 8 verbose + " ", // 9 details + " // ", // A lib-verbose + " //// ", // B lib-debug + " ////// ", // C lib-trace + " ////// ", // D lib-extra + " ////// ", // E +1 + " ////// " // F +2 + }; - char *out = printable_buf.iov_base; - if (!quoting) { - memcpy(out, data, len); - out += len; - } else if (printable) { - *out++ = '\''; - for (size_t i = 0; i < len; ++i) { - if (data[i] < ' ') { - assert((char *)printable_buf.iov_base + printable_buf.iov_len > - out + 4); - static const char hex[] = "0123456789abcdef"; - out[0] = '\\'; - out[1] = 'x'; - out[2] = hex[data[i] >> 4]; - out[3] = hex[data[i] & 15]; - out += 4; - } else if (strchr("\"'`\\", data[i])) { - assert((char *)printable_buf.iov_base + printable_buf.iov_len > - out + 2); - out[0] = '\\'; - out[1] = data[i]; - out += 2; - } else { - assert((char *)printable_buf.iov_base + printable_buf.iov_len > - out + 1); - *out++ = data[i]; - } + const bool nl = line_struct.scope_depth != chk.scope_nesting || + (line_struct.severity != severity && (line_struct.severity != MDBX_chk_processing || + severity < MDBX_chk_result || severity > MDBX_chk_resolution)); + if (nl) + lf(); + if (severity < MDBX_chk_warning) + flush(); + FILE *out = (severity > MDBX_chk_error) ? stdout : stderr; + if (nl || line_struct.empty) { + line_struct.severity = severity; + line_struct.scope_depth = chk.scope_nesting; + unsigned kind = line_struct.severity & MDBX_chk_severity_kind_mask; + if (line_struct.scope_depth || *prefixes[kind]) { + line_struct.empty = false; + for (size_t i = 0; i < line_struct.scope_depth; ++i) + fputs(" ", out); + fputs(prefixes[kind], out); } - *out++ = '\''; } - assert((char *)printable_buf.iov_base + printable_buf.iov_len > out); - *out = 0; - return printable_buf.iov_base; + return line_output = out; } -static void va_log(MDBX_log_level_t level, const char *function, int line, - const char *msg, va_list args) { - static const char *const prefixes[] = { - "!!!fatal: ", " ! " /* error */, " ~ " /* warning */, - " " /* notice */, " // " /* verbose */, " //// " /* debug */, - " ////// " /* trace */ - }; - - FILE *out = stdout; - if (level <= MDBX_LOG_ERROR) { - total_problems++; - out = stderr; +static void suffix(size_t cookie, const char *str) { + if (cookie == line_count && !line_struct.empty) { + fprintf(line_output, " %s", str); + line_struct.empty = false; + lf(); } +} - if (!quiet && verbose + 1 >= (unsigned)level && - (unsigned)level < ARRAY_LENGTH(prefixes)) { - fflush(nullptr); - fputs(prefixes[level], out); +static size_t MDBX_PRINTF_ARGS(2, 3) print(enum MDBX_chk_severity severity, const char *msg, ...) { + FILE *out = prefix(severity); + if (out) { + va_list args; + va_start(args, msg); vfprintf(out, msg, args); - - const bool have_lf = msg[strlen(msg) - 1] == '\n'; - if (level == MDBX_LOG_FATAL && function && line) - fprintf(out, have_lf ? " %s(), %u\n" : " (%s:%u)\n", - function + (strncmp(function, "mdbx_", 5) ? 5 : 0), line); - else if (!have_lf) - fputc('\n', out); - fflush(nullptr); + va_end(args); + line_struct.empty = false; + return line_count; } + return 0; +} +static FILE *MDBX_PRINTF_ARGS(2, 3) print_ln(enum MDBX_chk_severity severity, const char *msg, ...) { + FILE *out = prefix(severity); + if (out) { + va_list args; + va_start(args, msg); + vfprintf(out, msg, args); + va_end(args); + line_struct.empty = false; + lf(); + } + return out; +} + +static void logger(MDBX_log_level_t level, const char *function, int line, const char *fmt, va_list args) { + if (level <= MDBX_LOG_ERROR) + mdbx_env_chk_encount_problem(&chk); + + const unsigned kind = + (level > MDBX_LOG_NOTICE) ? level - MDBX_LOG_NOTICE + (MDBX_chk_extra & MDBX_chk_severity_kind_mask) : level; + const unsigned prio = kind << MDBX_chk_severity_prio_shift; + enum MDBX_chk_severity severity = prio + kind; + FILE *out = prefix(severity); + if (out) { + vfprintf(out, fmt, args); + const bool have_lf = fmt[strlen(fmt) - 1] == '\n'; + if (level == MDBX_LOG_FATAL && function && line) { + if (have_lf) + for (size_t i = 0; i < line_struct.scope_depth; ++i) + fputs(" ", out); + fprintf(out, have_lf ? " %s(), %u" : " (%s:%u)", function + (strncmp(function, "mdbx_", 5) ? 0 : 5), + line); + lf(); + } else if (have_lf) { + line_struct.empty = true; + line_struct.severity = LINE_SEVERITY_NONE; + line_count += 1; + } else + lf(); + } + if (level < MDBX_LOG_VERBOSE) + flush(); if (level == MDBX_LOG_FATAL) { #if !MDBX_DEBUG && !MDBX_FORCE_ASSERTIONS exit(EXIT_FAILURE_MDBX); @@ -4389,908 +3512,206 @@ static void va_log(MDBX_log_level_t level, const char *function, int line, } } -static void MDBX_PRINTF_ARGS(1, 2) error(const char *msg, ...) { +static void MDBX_PRINTF_ARGS(1, 2) error_fmt(const char *msg, ...) { va_list args; va_start(args, msg); - va_log(MDBX_LOG_ERROR, nullptr, 0, msg, args); + logger(MDBX_LOG_ERROR, nullptr, 0, msg, args); va_end(args); } -static void logger(MDBX_log_level_t level, const char *function, int line, - const char *msg, va_list args) { - (void)line; - (void)function; - if (level < MDBX_LOG_EXTRA) - va_log(level, function, line, msg, args); +static int error_fn(const char *fn, int err) { + if (err) + error_fmt("%s() failed, error %d, %s", fn, err, mdbx_strerror(err)); + return err; } -static int check_user_break(void) { - switch (user_break) { - case 0: - return MDBX_SUCCESS; - case 1: - print(" - interrupted by signal\n"); - fflush(nullptr); +static bool check_break(MDBX_chk_context_t *ctx) { + (void)ctx; + if (!user_break) + return false; + if (user_break == 1) { + print(MDBX_chk_resolution, "interrupted by signal"); + lf_flush(); user_break = 2; } - return MDBX_EINTR; -} - -static void pagemap_cleanup(void) { - osal_free(walk.pagemap); - walk.pagemap = nullptr; -} - -static bool eq(const MDBX_val a, const MDBX_val b) { - return a.iov_len == b.iov_len && - (a.iov_base == b.iov_base || a.iov_len == 0 || - !memcmp(a.iov_base, b.iov_base, a.iov_len)); -} - -static walk_dbi_t *pagemap_lookup_dbi(const MDBX_val *dbi_name, bool silent) { - static walk_dbi_t *last; - - if (dbi_name == MDBX_PGWALK_MAIN) - return &dbi_main; - if (dbi_name == MDBX_PGWALK_GC) - return &dbi_free; - if (dbi_name == MDBX_PGWALK_META) - return &dbi_meta; - - if (last && eq(last->name, *dbi_name)) - return last; - - walk_dbi_t *dbi = walk.dbi + CORE_DBS + /* account pseudo-entry for meta */ 1; - for (; dbi < ARRAY_END(walk.dbi) && dbi->name.iov_base; ++dbi) { - if (eq(dbi->name, *dbi_name)) - return last = dbi; - } - - if (verbose > 0 && !silent) { - print(" - found %s area\n", sdb_name(dbi_name)); - fflush(nullptr); - } - - if (dbi == ARRAY_END(walk.dbi)) - return nullptr; - - dbi->name = *dbi_name; - return last = dbi; + return true; } -static void MDBX_PRINTF_ARGS(4, 5) - problem_add(const char *object, uint64_t entry_number, const char *msg, - const char *extra, ...) { - total_problems++; - - if (!quiet) { - int need_fflush = 0; - struct problem *p; - - for (p = problems_list; p; p = p->pr_next) - if (p->caption == msg) - break; - - if (!p) { - p = osal_calloc(1, sizeof(*p)); - if (unlikely(!p)) - return; - p->caption = msg; - p->pr_next = problems_list; - problems_list = p; - need_fflush = 1; - } - - p->count++; - if (verbose > 1) { - print(" %s #%" PRIu64 ": %s", object, entry_number, msg); - if (extra) { - va_list args; - printf(" ("); - va_start(args, extra); - vfprintf(stdout, extra, args); - va_end(args); - printf(")"); - } - printf("\n"); - if (need_fflush) - fflush(nullptr); +static int scope_push(MDBX_chk_context_t *ctx, MDBX_chk_scope_t *scope, MDBX_chk_scope_t *inner, const char *fmt, + va_list args) { + (void)scope; + if (fmt && *fmt) { + FILE *out = prefix(MDBX_chk_processing); + if (out) { + vfprintf(out, fmt, args); + inner->usr_o.number = line_count; + line_struct.ctx = ctx; + flush(); } } + return MDBX_SUCCESS; } -static struct problem *problems_push(void) { - struct problem *p = problems_list; - problems_list = nullptr; - return p; +static void scope_pop(MDBX_chk_context_t *ctx, MDBX_chk_scope_t *scope, MDBX_chk_scope_t *inner) { + (void)ctx; + (void)scope; + suffix(inner->usr_o.number, inner->subtotal_issues ? "error(s)" : "done"); + flush(); } -static size_t problems_pop(struct problem *list) { - size_t count = 0; - - if (problems_list) { - int i; - - print(" - problems: "); - for (i = 0; problems_list; ++i) { - struct problem *p = problems_list->pr_next; - count += problems_list->count; - print("%s%s (%" PRIuPTR ")", i ? ", " : "", problems_list->caption, - problems_list->count); - osal_free(problems_list); - problems_list = p; - } - print("\n"); - fflush(nullptr); - } - - problems_list = list; - return count; +static MDBX_chk_user_table_cookie_t *table_filter(MDBX_chk_context_t *ctx, const MDBX_val *name, + MDBX_db_flags_t flags) { + (void)ctx; + (void)flags; + return (!only_table.iov_base || + (only_table.iov_len == name->iov_len && memcmp(only_table.iov_base, name->iov_base, name->iov_len) == 0)) + ? (void *)(intptr_t)-1 + : nullptr; } -static int pgvisitor(const uint64_t pgno, const unsigned pgnumber, - void *const ctx, const int deep, const MDBX_val *dbi_name, - const size_t page_size, const MDBX_page_type_t pagetype, - const MDBX_error_t err, const size_t nentries, - const size_t payload_bytes, const size_t header_bytes, - const size_t unused_bytes) { +static int stage_begin(MDBX_chk_context_t *ctx, enum MDBX_chk_stage stage) { (void)ctx; - const bool is_gc_tree = dbi_name == MDBX_PGWALK_GC; - if (deep > 42) { - problem_add("deep", deep, "too large", nullptr); - data_tree_problems += !is_gc_tree; - gc_tree_problems += is_gc_tree; - return MDBX_CORRUPTED /* avoid infinite loop/recursion */; - } - - walk_dbi_t *dbi = pagemap_lookup_dbi(dbi_name, false); - if (!dbi) { - data_tree_problems += !is_gc_tree; - gc_tree_problems += is_gc_tree; - return MDBX_ENOMEM; - } - - const size_t page_bytes = payload_bytes + header_bytes + unused_bytes; - walk.pgcount += pgnumber; - - const char *pagetype_caption; - bool branch = false; - switch (pagetype) { - default: - problem_add("page", pgno, "unknown page-type", "type %u, deep %i", - (unsigned)pagetype, deep); - pagetype_caption = "unknown"; - dbi->pages.other += pgnumber; - data_tree_problems += !is_gc_tree; - gc_tree_problems += is_gc_tree; - break; - case MDBX_page_broken: - pagetype_caption = "broken"; - dbi->pages.other += pgnumber; - data_tree_problems += !is_gc_tree; - gc_tree_problems += is_gc_tree; - break; - case MDBX_subpage_broken: - pagetype_caption = "broken-subpage"; - data_tree_problems += !is_gc_tree; - gc_tree_problems += is_gc_tree; - break; - case MDBX_page_meta: - pagetype_caption = "meta"; - dbi->pages.other += pgnumber; - break; - case MDBX_page_large: - pagetype_caption = "large"; - dbi->pages.large_volume += pgnumber; - dbi->pages.large_count += 1; - break; - case MDBX_page_branch: - pagetype_caption = "branch"; - dbi->pages.branch += pgnumber; - branch = true; - break; - case MDBX_page_leaf: - pagetype_caption = "leaf"; - dbi->pages.leaf += pgnumber; - break; - case MDBX_page_dupfixed_leaf: - pagetype_caption = "leaf-dupfixed"; - dbi->pages.leaf_dupfixed += pgnumber; - break; - case MDBX_subpage_leaf: - pagetype_caption = "subleaf-dupsort"; - dbi->pages.subleaf_dupsort += 1; - break; - case MDBX_subpage_dupfixed_leaf: - pagetype_caption = "subleaf-dupfixed"; - dbi->pages.subleaf_dupfixed += 1; - break; - } - - if (pgnumber) { - if (verbose > 3 && (!only_subdb.iov_base || eq(only_subdb, dbi->name))) { - if (pgnumber == 1) - print(" %s-page %" PRIu64, pagetype_caption, pgno); - else - print(" %s-span %" PRIu64 "[%u]", pagetype_caption, pgno, pgnumber); - print(" of %s: header %" PRIiPTR ", %s %" PRIiPTR ", payload %" PRIiPTR - ", unused %" PRIiPTR ", deep %i\n", - sdb_name(&dbi->name), header_bytes, - (pagetype == MDBX_page_branch) ? "keys" : "entries", nentries, - payload_bytes, unused_bytes, deep); - } - - bool already_used = false; - for (unsigned n = 0; n < pgnumber; ++n) { - uint64_t spanpgno = pgno + n; - if (spanpgno >= alloc_pages) { - problem_add("page", spanpgno, "wrong page-no", - "%s-page: %" PRIu64 " > %" PRIu64 ", deep %i", - pagetype_caption, spanpgno, alloc_pages, deep); - data_tree_problems += !is_gc_tree; - gc_tree_problems += is_gc_tree; - } else if (walk.pagemap[spanpgno]) { - walk_dbi_t *coll_dbi = &walk.dbi[walk.pagemap[spanpgno] - 1]; - problem_add("page", spanpgno, - (branch && coll_dbi == dbi) ? "loop" : "already used", - "%s-page: by %s, deep %i", pagetype_caption, - sdb_name(&coll_dbi->name), deep); - already_used = true; - data_tree_problems += !is_gc_tree; - gc_tree_problems += is_gc_tree; - } else { - walk.pagemap[spanpgno] = (short)(dbi - walk.dbi + 1); - dbi->pages.total += 1; - } - } - - if (already_used) - return branch ? MDBX_RESULT_TRUE /* avoid infinite loop/recursion */ - : MDBX_SUCCESS; - } - - if (MDBX_IS_ERROR(err)) { - problem_add("page", pgno, "invalid/corrupted", "%s-page", pagetype_caption); - data_tree_problems += !is_gc_tree; - gc_tree_problems += is_gc_tree; - } else { - if (unused_bytes > page_size) { - problem_add("page", pgno, "illegal unused-bytes", - "%s-page: %u < %" PRIuPTR " < %u", pagetype_caption, 0, - unused_bytes, envinfo.mi_dxb_pagesize); - data_tree_problems += !is_gc_tree; - gc_tree_problems += is_gc_tree; - } - - if (header_bytes < (int)sizeof(long) || - (size_t)header_bytes >= envinfo.mi_dxb_pagesize - sizeof(long)) { - problem_add("page", pgno, "illegal header-length", - "%s-page: %" PRIuPTR " < %" PRIuPTR " < %" PRIuPTR, - pagetype_caption, sizeof(long), header_bytes, - envinfo.mi_dxb_pagesize - sizeof(long)); - data_tree_problems += !is_gc_tree; - gc_tree_problems += is_gc_tree; - } - if (nentries < 1 || (pagetype == MDBX_page_branch && nentries < 2)) { - problem_add("page", pgno, nentries ? "half-empty" : "empty", - "%s-page: payload %" PRIuPTR " bytes, %" PRIuPTR - " entries, deep %i", - pagetype_caption, payload_bytes, nentries, deep); - dbi->pages.empty += 1; - data_tree_problems += !is_gc_tree; - gc_tree_problems += is_gc_tree; - } - - if (pgnumber) { - if (page_bytes != page_size) { - problem_add("page", pgno, "misused", - "%s-page: %" PRIuPTR " != %" PRIuPTR " (%" PRIuPTR - "h + %" PRIuPTR "p + %" PRIuPTR "u), deep %i", - pagetype_caption, page_size, page_bytes, header_bytes, - payload_bytes, unused_bytes, deep); - if (page_size > page_bytes) - dbi->lost_bytes += page_size - page_bytes; - data_tree_problems += !is_gc_tree; - gc_tree_problems += is_gc_tree; - } else { - dbi->payload_bytes += (uint64_t)payload_bytes + header_bytes; - walk.total_payload_bytes += (uint64_t)payload_bytes + header_bytes; - } - } - } - - return check_user_break(); + chk_stage = stage; + anchor_lineno = line_count; + flush(); + return MDBX_SUCCESS; } -typedef int(visitor)(const uint64_t record_number, const MDBX_val *key, - const MDBX_val *data); -static int process_db(MDBX_dbi dbi_handle, const MDBX_val *dbi_name, - visitor *handler); - -static int handle_userdb(const uint64_t record_number, const MDBX_val *key, - const MDBX_val *data) { - (void)record_number; - (void)key; - (void)data; - return check_user_break(); +static int conclude(MDBX_chk_context_t *ctx); +static int stage_end(MDBX_chk_context_t *ctx, enum MDBX_chk_stage stage, int err) { + if (stage == MDBX_chk_conclude && !err) + err = conclude(ctx); + suffix(anchor_lineno, err ? "error(s)" : "done"); + flush(); + chk_stage = MDBX_chk_none; + return err; } -static int handle_freedb(const uint64_t record_number, const MDBX_val *key, - const MDBX_val *data) { - char *bad = ""; - pgno_t *iptr = data->iov_base; - - if (key->iov_len != sizeof(txnid_t)) - problem_add("entry", record_number, "wrong txn-id size", - "key-size %" PRIiPTR, key->iov_len); - else { - txnid_t txnid; - memcpy(&txnid, key->iov_base, sizeof(txnid)); - if (txnid < 1 || txnid > envinfo.mi_recent_txnid) - problem_add("entry", record_number, "wrong txn-id", "%" PRIaTXN, txnid); - else { - if (data->iov_len < sizeof(pgno_t) || data->iov_len % sizeof(pgno_t)) - problem_add("entry", txnid, "wrong idl size", "%" PRIuPTR, - data->iov_len); - size_t number = (data->iov_len >= sizeof(pgno_t)) ? *iptr++ : 0; - if (number > MDBX_PGL_LIMIT) - problem_add("entry", txnid, "wrong idl length", "%" PRIuPTR, number); - else if ((number + 1) * sizeof(pgno_t) > data->iov_len) { - problem_add("entry", txnid, "trimmed idl", - "%" PRIuSIZE " > %" PRIuSIZE " (corruption)", - (number + 1) * sizeof(pgno_t), data->iov_len); - number = data->iov_len / sizeof(pgno_t) - 1; - } else if (data->iov_len - (number + 1) * sizeof(pgno_t) >= - /* LY: allow gap up to one page. it is ok - * and better than shink-and-retry inside update_gc() */ - envinfo.mi_dxb_pagesize) - problem_add("entry", txnid, "extra idl space", - "%" PRIuSIZE " < %" PRIuSIZE " (minor, not a trouble)", - (number + 1) * sizeof(pgno_t), data->iov_len); - - gc_pages += number; - if (envinfo.mi_latter_reader_txnid > txnid) - reclaimable_pages += number; - - pgno_t prev = MDBX_PNL_ASCENDING ? NUM_METAS - 1 : txn->mt_next_pgno; - pgno_t span = 1; - for (size_t i = 0; i < number; ++i) { - if (check_user_break()) - return MDBX_EINTR; - const pgno_t pgno = iptr[i]; - if (pgno < NUM_METAS) - problem_add("entry", txnid, "wrong idl entry", - "pgno %" PRIaPGNO " < meta-pages %u", pgno, NUM_METAS); - else if (pgno >= backed_pages) - problem_add("entry", txnid, "wrong idl entry", - "pgno %" PRIaPGNO " > backed-pages %" PRIu64, pgno, - backed_pages); - else if (pgno >= alloc_pages) - problem_add("entry", txnid, "wrong idl entry", - "pgno %" PRIaPGNO " > alloc-pages %" PRIu64, pgno, - alloc_pages - 1); - else { - if (MDBX_PNL_DISORDERED(prev, pgno)) { - bad = " [bad sequence]"; - problem_add("entry", txnid, "bad sequence", - "%" PRIaPGNO " %c [%zu].%" PRIaPGNO, prev, - (prev == pgno) ? '=' : (MDBX_PNL_ASCENDING ? '>' : '<'), - i, pgno); - } - if (walk.pagemap) { - int idx = walk.pagemap[pgno]; - if (idx == 0) - walk.pagemap[pgno] = -1; - else if (idx > 0) - problem_add("page", pgno, "already used", "by %s", - sdb_name(&walk.dbi[idx - 1].name)); - else - problem_add("page", pgno, "already listed in GC", nullptr); - } - } - prev = pgno; - while (i + span < number && - iptr[i + span] == (MDBX_PNL_ASCENDING ? pgno_add(pgno, span) - : pgno_sub(pgno, span))) - ++span; - } - if (verbose > 3 && !only_subdb.iov_base) { - print(" transaction %" PRIaTXN ", %" PRIuPTR - " pages, maxspan %" PRIaPGNO "%s\n", - txnid, number, span, bad); - if (verbose > 4) { - for (size_t i = 0; i < number; i += span) { - const pgno_t pgno = iptr[i]; - for (span = 1; - i + span < number && - iptr[i + span] == (MDBX_PNL_ASCENDING ? pgno_add(pgno, span) - : pgno_sub(pgno, span)); - ++span) - ; - if (span > 1) { - print(" %9" PRIaPGNO "[%" PRIaPGNO "]\n", pgno, span); - } else - print(" %9" PRIaPGNO "\n", pgno); - } - } - } - } +static MDBX_chk_line_t *print_begin(MDBX_chk_context_t *ctx, enum MDBX_chk_severity severity) { + (void)ctx; + if (silently(severity)) + return nullptr; + if (line_struct.ctx) { + if (line_struct.severity == MDBX_chk_processing && severity >= MDBX_chk_result && severity <= MDBX_chk_resolution && + line_output) + fputc(' ', line_output); + else + lf(); + line_struct.ctx = nullptr; } - - return check_user_break(); + line_struct.severity = severity; + return &line_struct; } -static int equal_or_greater(const MDBX_val *a, const MDBX_val *b) { - return eq(*a, *b) ? 0 : 1; +static void print_flush(MDBX_chk_line_t *line) { + (void)line; + flush(); } -static int handle_maindb(const uint64_t record_number, const MDBX_val *key, - const MDBX_val *data) { - if (data->iov_len == sizeof(MDBX_db)) { - int rc = process_db(~0u, key, handle_userdb); - if (rc != MDBX_INCOMPATIBLE) { - userdb_count++; - return rc; - } - } - return handle_userdb(record_number, key, data); +static void print_done(MDBX_chk_line_t *line) { + lf(); + line->ctx = nullptr; } -static const char *db_flags2keymode(unsigned flags) { - flags &= (MDBX_REVERSEKEY | MDBX_INTEGERKEY); - switch (flags) { - case 0: - return "usual"; - case MDBX_REVERSEKEY: - return "reserve"; - case MDBX_INTEGERKEY: - return "ordinal"; - case MDBX_REVERSEKEY | MDBX_INTEGERKEY: - return "msgpack"; - default: - assert(false); - __unreachable(); - } +static void print_chars(MDBX_chk_line_t *line, const char *str, size_t len) { + if (line->empty) + prefix(line->severity); + fwrite(str, 1, len, line_output); } -static const char *db_flags2valuemode(unsigned flags) { - flags &= (MDBX_DUPSORT | MDBX_REVERSEDUP | MDBX_DUPFIXED | MDBX_INTEGERDUP); - switch (flags) { - case 0: - return "single"; - case MDBX_DUPSORT: - return "multi"; - case MDBX_REVERSEDUP: - case MDBX_DUPSORT | MDBX_REVERSEDUP: - return "multi-reverse"; - case MDBX_DUPFIXED: - case MDBX_DUPSORT | MDBX_DUPFIXED: - return "multi-samelength"; - case MDBX_DUPFIXED | MDBX_REVERSEDUP: - case MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_REVERSEDUP: - return "multi-reverse-samelength"; - case MDBX_INTEGERDUP: - case MDBX_DUPSORT | MDBX_INTEGERDUP: - case MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERDUP: - case MDBX_DUPFIXED | MDBX_INTEGERDUP: - return "multi-ordinal"; - case MDBX_INTEGERDUP | MDBX_REVERSEDUP: - case MDBX_DUPSORT | MDBX_INTEGERDUP | MDBX_REVERSEDUP: - return "multi-msgpack"; - case MDBX_DUPFIXED | MDBX_INTEGERDUP | MDBX_REVERSEDUP: - case MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERDUP | MDBX_REVERSEDUP: - return "reserved"; - default: - assert(false); - __unreachable(); - } +static void print_format(MDBX_chk_line_t *line, const char *fmt, va_list args) { + if (line->empty) + prefix(line->severity); + vfprintf(line_output, fmt, args); } -static int process_db(MDBX_dbi dbi_handle, const MDBX_val *dbi_name, - visitor *handler) { - MDBX_cursor *mc; - MDBX_stat ms; - MDBX_val key, data; - MDBX_val prev_key, prev_data; - unsigned flags; - int rc, i; - struct problem *saved_list; - uint64_t problems_count; - const bool second_pass = dbi_handle == MAIN_DBI; - - uint64_t record_count = 0, dups = 0; - uint64_t key_bytes = 0, data_bytes = 0; - - if ((MDBX_TXN_FINISHED | MDBX_TXN_ERROR) & mdbx_txn_flags(txn)) { - print(" ! abort processing %s due to a previous error\n", - sdb_name(dbi_name)); - return MDBX_BAD_TXN; - } - - if (dbi_handle == ~0u) { - rc = mdbx_dbi_open_ex2( - txn, dbi_name, MDBX_DB_ACCEDE, &dbi_handle, - (dbi_name && ignore_wrong_order) ? equal_or_greater : nullptr, - (dbi_name && ignore_wrong_order) ? equal_or_greater : nullptr); - if (rc) { - if (!dbi_name || - rc != - MDBX_INCOMPATIBLE) /* LY: mainDB's record is not a user's DB. */ { - error("mdbx_dbi_open(%s) failed, error %d %s\n", sdb_name(dbi_name), rc, - mdbx_strerror(rc)); - } - return rc; - } - } - - if (dbi_handle >= CORE_DBS && dbi_name && only_subdb.iov_base && - !eq(only_subdb, *dbi_name)) { - if (verbose) { - print("Skip processing %s...\n", sdb_name(dbi_name)); - fflush(nullptr); - } - skipped_subdb++; - return MDBX_SUCCESS; - } - - if (!second_pass && verbose) - print("Processing %s...\n", sdb_name(dbi_name)); - fflush(nullptr); - - rc = mdbx_dbi_flags(txn, dbi_handle, &flags); - if (rc) { - error("mdbx_dbi_flags() failed, error %d %s\n", rc, mdbx_strerror(rc)); - return rc; - } - - rc = mdbx_dbi_stat(txn, dbi_handle, &ms, sizeof(ms)); - if (rc) { - error("mdbx_dbi_stat() failed, error %d %s\n", rc, mdbx_strerror(rc)); - return rc; - } - - if (!second_pass && verbose) { - print(" - key-value kind: %s-key => %s-value", db_flags2keymode(flags), - db_flags2valuemode(flags)); - if (verbose > 1) { - print(", flags:"); - if (!flags) - print(" none"); - else { - for (i = 0; dbflags[i].bit; i++) - if (flags & dbflags[i].bit) - print(" %s", dbflags[i].name); - } - if (verbose > 2) - print(" (0x%02X), dbi-id %d", flags, dbi_handle); - } - print("\n"); - if (ms.ms_mod_txnid) - print(" - last modification txn#%" PRIu64 "\n", ms.ms_mod_txnid); - if (verbose > 1) { - print(" - page size %u, entries %" PRIu64 "\n", ms.ms_psize, - ms.ms_entries); - print(" - b-tree depth %u, pages: branch %" PRIu64 ", leaf %" PRIu64 - ", overflow %" PRIu64 "\n", - ms.ms_depth, ms.ms_branch_pages, ms.ms_leaf_pages, - ms.ms_overflow_pages); - } - } - - walk_dbi_t *dbi = (dbi_handle < CORE_DBS) - ? &walk.dbi[dbi_handle] - : pagemap_lookup_dbi(dbi_name, true); - if (!dbi) { - error("too many DBIs or out of memory\n"); - return MDBX_ENOMEM; - } - if (!dont_traversal) { - const uint64_t subtotal_pages = - ms.ms_branch_pages + ms.ms_leaf_pages + ms.ms_overflow_pages; - if (subtotal_pages != dbi->pages.total) - error("%s pages mismatch (%" PRIu64 " != walked %" PRIu64 ")\n", - "subtotal", subtotal_pages, dbi->pages.total); - if (ms.ms_branch_pages != dbi->pages.branch) - error("%s pages mismatch (%" PRIu64 " != walked %" PRIu64 ")\n", "branch", - ms.ms_branch_pages, dbi->pages.branch); - const uint64_t allleaf_pages = dbi->pages.leaf + dbi->pages.leaf_dupfixed; - if (ms.ms_leaf_pages != allleaf_pages) - error("%s pages mismatch (%" PRIu64 " != walked %" PRIu64 ")\n", - "all-leaf", ms.ms_leaf_pages, allleaf_pages); - if (ms.ms_overflow_pages != dbi->pages.large_volume) - error("%s pages mismatch (%" PRIu64 " != walked %" PRIu64 ")\n", - "large/overlow", ms.ms_overflow_pages, dbi->pages.large_volume); - } - rc = mdbx_cursor_open(txn, dbi_handle, &mc); - if (rc) { - error("mdbx_cursor_open() failed, error %d %s\n", rc, mdbx_strerror(rc)); - return rc; - } - - if (ignore_wrong_order) { /* for debugging with enabled assertions */ - mc->mc_checking |= CC_SKIPORD; - if (mc->mc_xcursor) - mc->mc_xcursor->mx_cursor.mc_checking |= CC_SKIPORD; - } - - const size_t maxkeysize = mdbx_env_get_maxkeysize_ex(env, flags); - saved_list = problems_push(); - prev_key.iov_base = nullptr; - prev_key.iov_len = 0; - prev_data.iov_base = nullptr; - prev_data.iov_len = 0; - rc = mdbx_cursor_get(mc, &key, &data, MDBX_FIRST); - while (rc == MDBX_SUCCESS) { - rc = check_user_break(); - if (rc) - goto bailout; - - if (!second_pass) { - bool bad_key = false; - if (key.iov_len > maxkeysize) { - problem_add("entry", record_count, "key length exceeds max-key-size", - "%" PRIuPTR " > %" PRIuPTR, key.iov_len, maxkeysize); - bad_key = true; - } else if ((flags & MDBX_INTEGERKEY) && key.iov_len != sizeof(uint64_t) && - key.iov_len != sizeof(uint32_t)) { - problem_add("entry", record_count, "wrong key length", - "%" PRIuPTR " != 4or8", key.iov_len); - bad_key = true; - } - - bool bad_data = false; - if ((flags & MDBX_INTEGERDUP) && data.iov_len != sizeof(uint64_t) && - data.iov_len != sizeof(uint32_t)) { - problem_add("entry", record_count, "wrong data length", - "%" PRIuPTR " != 4or8", data.iov_len); - bad_data = true; - } - - if (prev_key.iov_base) { - if (prev_data.iov_base && !bad_data && (flags & MDBX_DUPFIXED) && - prev_data.iov_len != data.iov_len) { - problem_add("entry", record_count, "different data length", - "%" PRIuPTR " != %" PRIuPTR, prev_data.iov_len, - data.iov_len); - bad_data = true; - } - - if (!bad_key) { - int cmp = mdbx_cmp(txn, dbi_handle, &key, &prev_key); - if (cmp == 0) { - ++dups; - if ((flags & MDBX_DUPSORT) == 0) { - problem_add("entry", record_count, "duplicated entries", nullptr); - if (prev_data.iov_base && data.iov_len == prev_data.iov_len && - memcmp(data.iov_base, prev_data.iov_base, data.iov_len) == - 0) { - problem_add("entry", record_count, "complete duplicate", - nullptr); - } - } else if (!bad_data && prev_data.iov_base) { - cmp = mdbx_dcmp(txn, dbi_handle, &data, &prev_data); - if (cmp == 0) { - problem_add("entry", record_count, "complete duplicate", - nullptr); - } else if (cmp < 0 && !ignore_wrong_order) { - problem_add("entry", record_count, - "wrong order of multi-values", nullptr); - } - } - } else if (cmp < 0 && !ignore_wrong_order) { - problem_add("entry", record_count, "wrong order of entries", - nullptr); - } - } - } - - if (!bad_key) { - if (verbose && (flags & MDBX_INTEGERKEY) && !prev_key.iov_base) - print(" - fixed key-size %" PRIuPTR "\n", key.iov_len); - prev_key = key; - } - if (!bad_data) { - if (verbose && (flags & (MDBX_INTEGERDUP | MDBX_DUPFIXED)) && - !prev_data.iov_base) - print(" - fixed data-size %" PRIuPTR "\n", data.iov_len); - prev_data = data; - } - } - - if (handler) { - rc = handler(record_count, &key, &data); - if (MDBX_IS_ERROR(rc)) - goto bailout; - } - - record_count++; - key_bytes += key.iov_len; - data_bytes += data.iov_len; - - rc = mdbx_cursor_get(mc, &key, &data, MDBX_NEXT); - } - if (rc != MDBX_NOTFOUND) - error("mdbx_cursor_get() failed, error %d %s\n", rc, mdbx_strerror(rc)); - else - rc = 0; - - if (record_count != ms.ms_entries) - problem_add("entry", record_count, "different number of entries", - "%" PRIu64 " != %" PRIu64, record_count, ms.ms_entries); -bailout: - problems_count = problems_pop(saved_list); - if (!second_pass && verbose) { - print(" - summary: %" PRIu64 " records, %" PRIu64 " dups, %" PRIu64 - " key's bytes, %" PRIu64 " data's " - "bytes, %" PRIu64 " problems\n", - record_count, dups, key_bytes, data_bytes, problems_count); - fflush(nullptr); - } - - mdbx_cursor_close(mc); - return (rc || problems_count) ? MDBX_RESULT_TRUE : MDBX_SUCCESS; -} +static const MDBX_chk_callbacks_t cb = {.check_break = check_break, + .scope_push = scope_push, + .scope_pop = scope_pop, + .table_filter = table_filter, + .stage_begin = stage_begin, + .stage_end = stage_end, + .print_begin = print_begin, + .print_flush = print_flush, + .print_done = print_done, + .print_chars = print_chars, + .print_format = print_format}; static void usage(char *prog) { - fprintf( - stderr, - "usage: %s " - "[-V] [-v] [-q] [-c] [-0|1|2] [-w] [-d] [-i] [-s subdb] [-u|U] dbpath\n" - " -V\t\tprint version and exit\n" - " -v\t\tmore verbose, could be used multiple times\n" - " -q\t\tbe quiet\n" - " -c\t\tforce cooperative mode (don't try exclusive)\n" - " -w\t\twrite-mode checking\n" - " -d\t\tdisable page-by-page traversal of B-tree\n" - " -i\t\tignore wrong order errors (for custom comparators case)\n" - " -s subdb\tprocess a specific subdatabase only\n" - " -u\t\twarmup database before checking\n" - " -U\t\twarmup and try lock database pages in memory before checking\n" - " -0|1|2\tforce using specific meta-page 0, or 2 for checking\n" - " -t\t\tturn to a specified meta-page on successful check\n" - " -T\t\tturn to a specified meta-page EVEN ON UNSUCCESSFUL CHECK!\n", - prog); + fprintf(stderr, + "usage: %s " + "[-V] [-v] [-q] [-c] [-0|1|2] [-w] [-d] [-i] [-s table] [-u|U] dbpath\n" + " -V\t\tprint version and exit\n" + " -v\t\tmore verbose, could be repeated upto 9 times for extra details\n" + " -q\t\tbe quiet\n" + " -c\t\tforce cooperative mode (don't try exclusive)\n" + " -w\t\twrite-mode checking\n" + " -d\t\tdisable page-by-page traversal of B-tree\n" + " -i\t\tignore wrong order errors (for custom comparators case)\n" + " -s table\tprocess a specific subdatabase only\n" + " -u\t\twarmup database before checking\n" + " -U\t\twarmup and try lock database pages in memory before checking\n" + " -0|1|2\tforce using specific meta-page 0, or 2 for checking\n" + " -t\t\tturn to a specified meta-page on successful check\n" + " -T\t\tturn to a specified meta-page EVEN ON UNSUCCESSFUL CHECK!\n", + prog); exit(EXIT_INTERRUPTED); } -static bool meta_ot(txnid_t txn_a, uint64_t sign_a, txnid_t txn_b, - uint64_t sign_b, const bool wanna_steady) { - if (txn_a == txn_b) - return SIGN_IS_STEADY(sign_b); - - if (wanna_steady && SIGN_IS_STEADY(sign_a) != SIGN_IS_STEADY(sign_b)) - return SIGN_IS_STEADY(sign_b); - - return txn_a < txn_b; -} - -static bool meta_eq(txnid_t txn_a, uint64_t sign_a, txnid_t txn_b, - uint64_t sign_b) { - if (!txn_a || txn_a != txn_b) - return false; - - if (SIGN_IS_STEADY(sign_a) != SIGN_IS_STEADY(sign_b)) - return false; - - return true; -} - -static int meta_recent(const bool wanna_steady) { - if (meta_ot(envinfo.mi_meta0_txnid, envinfo.mi_meta0_sign, - envinfo.mi_meta1_txnid, envinfo.mi_meta1_sign, wanna_steady)) - return meta_ot(envinfo.mi_meta2_txnid, envinfo.mi_meta2_sign, - envinfo.mi_meta1_txnid, envinfo.mi_meta1_sign, wanna_steady) - ? 1 - : 2; - else - return meta_ot(envinfo.mi_meta0_txnid, envinfo.mi_meta0_sign, - envinfo.mi_meta2_txnid, envinfo.mi_meta2_sign, wanna_steady) - ? 2 - : 0; -} - -static int meta_tail(int head) { - switch (head) { - case 0: - return meta_ot(envinfo.mi_meta1_txnid, envinfo.mi_meta1_sign, - envinfo.mi_meta2_txnid, envinfo.mi_meta2_sign, true) - ? 1 - : 2; - case 1: - return meta_ot(envinfo.mi_meta0_txnid, envinfo.mi_meta0_sign, - envinfo.mi_meta2_txnid, envinfo.mi_meta2_sign, true) - ? 0 - : 2; - case 2: - return meta_ot(envinfo.mi_meta0_txnid, envinfo.mi_meta0_sign, - envinfo.mi_meta1_txnid, envinfo.mi_meta1_sign, true) - ? 0 - : 1; - default: - assert(false); - return -1; - } -} - -static int meta_head(void) { return meta_recent(false); } - -void verbose_meta(int num, txnid_t txnid, uint64_t sign, uint64_t bootid_x, - uint64_t bootid_y) { - const bool have_bootid = (bootid_x | bootid_y) != 0; - const bool bootid_match = bootid_x == envinfo.mi_bootid.current.x && - bootid_y == envinfo.mi_bootid.current.y; - - print(" - meta-%d: ", num); - switch (sign) { - case MDBX_DATASIGN_NONE: - print("no-sync/legacy"); - break; - case MDBX_DATASIGN_WEAK: - print("weak-%s", bootid_match ? (have_bootid ? "intact (same boot-id)" - : "unknown (no boot-id") - : "dead"); - break; - default: - print("steady"); - break; +static int conclude(MDBX_chk_context_t *ctx) { + int err = MDBX_SUCCESS; + if (ctx->result.total_problems == 1 && ctx->result.problems_meta == 1 && + (chk_flags & (MDBX_CHK_SKIP_BTREE_TRAVERSAL | MDBX_CHK_SKIP_KV_TRAVERSAL)) == 0 && + (env_flags & MDBX_RDONLY) == 0 && !only_table.iov_base && stuck_meta < 0 && + ctx->result.steady_txnid < ctx->result.recent_txnid) { + const size_t step_lineno = print(MDBX_chk_resolution, + "Perform sync-to-disk for make steady checkpoint" + " at txn-id #%" PRIi64 "...", + ctx->result.recent_txnid); + flush(); + err = error_fn("walk_pages", mdbx_env_sync_ex(ctx->env, true, false)); + if (err == MDBX_SUCCESS) { + ctx->result.problems_meta -= 1; + ctx->result.total_problems -= 1; + suffix(step_lineno, "done"); + } } - print(" txn#%" PRIu64, txnid); - - const int head = meta_head(); - if (num == head) - print(", head"); - else if (num == meta_tail(head)) - print(", tail"); - else - print(", stay"); - - if (stuck_meta >= 0) { - if (num == stuck_meta) - print(", forced for checking"); - } else if (txnid > envinfo.mi_recent_txnid && - (envflags & (MDBX_EXCLUSIVE | MDBX_RDONLY)) == MDBX_EXCLUSIVE) - print(", rolled-back %" PRIu64 " (%" PRIu64 " >>> %" PRIu64 ")", - txnid - envinfo.mi_recent_txnid, txnid, envinfo.mi_recent_txnid); - print("\n"); -} -static uint64_t get_meta_txnid(const unsigned meta_id) { - switch (meta_id) { - default: - assert(false); - error("unexpected meta_id %u\n", meta_id); - return 0; - case 0: - return envinfo.mi_meta0_txnid; - case 1: - return envinfo.mi_meta1_txnid; - case 2: - return envinfo.mi_meta2_txnid; + if (turn_meta && stuck_meta >= 0 && (chk_flags & (MDBX_CHK_SKIP_BTREE_TRAVERSAL | MDBX_CHK_SKIP_KV_TRAVERSAL)) == 0 && + !only_table.iov_base && (env_flags & (MDBX_RDONLY | MDBX_EXCLUSIVE)) == MDBX_EXCLUSIVE) { + const bool successful_check = (err | ctx->result.total_problems | ctx->result.problems_meta) == 0; + if (successful_check || force_turn_meta) { + const size_t step_lineno = + print(MDBX_chk_resolution, "Performing turn to the specified meta-page (%d) due to %s!", stuck_meta, + successful_check ? "successful check" : "the -T option was given"); + flush(); + err = mdbx_env_turn_for_recovery(ctx->env, stuck_meta); + if (err != MDBX_SUCCESS) + error_fn("mdbx_env_turn_for_recovery", err); + else + suffix(step_lineno, "done"); + } else { + print(MDBX_chk_resolution, + "Skipping turn to the specified meta-page (%d) due to " + "unsuccessful check!", + stuck_meta); + lf_flush(); + } } -} -static void print_size(const char *prefix, const uint64_t value, - const char *suffix) { - const char sf[] = - "KMGTPEZY"; /* LY: Kilo, Mega, Giga, Tera, Peta, Exa, Zetta, Yotta! */ - double k = 1024.0; - size_t i; - for (i = 0; sf[i + 1] && value / k > 1000.0; ++i) - k *= 1024; - print("%s%" PRIu64 " (%.2f %cb)%s", prefix, value, value / k, sf[i], suffix); + return err; } int main(int argc, char *argv[]) { int rc; char *prog = argv[0]; char *envname; - unsigned problems_maindb = 0, problems_freedb = 0, problems_meta = 0; - bool write_locked = false; - bool turn_meta = false; - bool force_turn_meta = false; bool warmup = false; MDBX_warmup_flags_t warmup_flags = MDBX_warmup_default; + if (argc < 2) + usage(prog); + double elapsed; #if defined(_WIN32) || defined(_WIN64) uint64_t timestamp_start, timestamp_finish; @@ -5298,20 +3719,11 @@ int main(int argc, char *argv[]) { #else struct timespec timestamp_start, timestamp_finish; if (clock_gettime(CLOCK_MONOTONIC, ×tamp_start)) { - rc = errno; - error("clock_gettime() failed, error %d %s\n", rc, mdbx_strerror(rc)); + error_fn("clock_gettime", errno); return EXIT_FAILURE_SYS; } #endif - dbi_meta.name.iov_base = MDBX_PGWALK_META; - dbi_free.name.iov_base = MDBX_PGWALK_GC; - dbi_main.name.iov_base = MDBX_PGWALK_MAIN; - atexit(pagemap_cleanup); - - if (argc < 2) - usage(prog); - for (int i; (i = getopt(argc, argv, "uU" "0" @@ -5336,15 +3748,21 @@ int main(int argc, char *argv[]) { " - build: %s for %s by %s\n" " - flags: %s\n" " - options: %s\n", - mdbx_version.major, mdbx_version.minor, mdbx_version.release, - mdbx_version.revision, mdbx_version.git.describe, - mdbx_version.git.datetime, mdbx_version.git.commit, - mdbx_version.git.tree, mdbx_sourcery_anchor, mdbx_build.datetime, - mdbx_build.target, mdbx_build.compiler, mdbx_build.flags, - mdbx_build.options); + mdbx_version.major, mdbx_version.minor, mdbx_version.patch, mdbx_version.tweak, mdbx_version.git.describe, + mdbx_version.git.datetime, mdbx_version.git.commit, mdbx_version.git.tree, mdbx_sourcery_anchor, + mdbx_build.datetime, mdbx_build.target, mdbx_build.compiler, mdbx_build.flags, mdbx_build.options); return EXIT_SUCCESS; case 'v': - verbose++; + if (verbose >= 9 && 0) + usage(prog); + else { + verbose += 1; + if (verbose == 0 && !MDBX_DEBUG) + printf("Verbosity level %u exposures only to" + " a debug/extra-logging-enabled builds (with NDEBUG undefined" + " or MDBX_DEBUG > 0)\n", + verbose); + } break; case '0': stuck_meta = 0; @@ -5361,8 +3779,6 @@ int main(int argc, char *argv[]) { case 'T': turn_meta = force_turn_meta = true; quiet = false; - if (verbose < 2) - verbose = 2; break; case 'q': quiet = true; @@ -5370,35 +3786,37 @@ int main(int argc, char *argv[]) { case 'n': break; case 'w': - envflags &= ~MDBX_RDONLY; + env_flags &= ~MDBX_RDONLY; + chk_flags |= MDBX_CHK_READWRITE; #if MDBX_MMAP_INCOHERENT_FILE_WRITE /* Temporary `workaround` for OpenBSD kernel's flaw. * See https://libmdbx.dqdkfa.ru/dead-github/issues/67 */ - envflags |= MDBX_WRITEMAP; + env_flags |= MDBX_WRITEMAP; #endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */ break; case 'c': - envflags = (envflags & ~MDBX_EXCLUSIVE) | MDBX_ACCEDE; + env_flags = (env_flags & ~MDBX_EXCLUSIVE) | MDBX_ACCEDE; break; case 'd': - dont_traversal = true; + chk_flags |= MDBX_CHK_SKIP_BTREE_TRAVERSAL; break; case 's': - if (only_subdb.iov_base && strcmp(only_subdb.iov_base, optarg)) + if (only_table.iov_base && strcmp(only_table.iov_base, optarg)) usage(prog); - only_subdb.iov_base = optarg; - only_subdb.iov_len = strlen(optarg); + else { + only_table.iov_base = optarg; + only_table.iov_len = strlen(optarg); + } break; case 'i': - ignore_wrong_order = true; + chk_flags |= MDBX_CHK_IGNORE_ORDER; break; case 'u': warmup = true; break; case 'U': warmup = true; - warmup_flags = - MDBX_warmup_force | MDBX_warmup_touchlimit | MDBX_warmup_lock; + warmup_flags = MDBX_warmup_force | MDBX_warmup_touchlimit | MDBX_warmup_lock; break; default: usage(prog); @@ -5409,26 +3827,25 @@ int main(int argc, char *argv[]) { usage(prog); rc = MDBX_SUCCESS; - if (stuck_meta >= 0 && (envflags & MDBX_EXCLUSIVE) == 0) { - error("exclusive mode is required to using specific meta-page(%d) for " - "checking.\n", - stuck_meta); + if (stuck_meta >= 0 && (env_flags & MDBX_EXCLUSIVE) == 0) { + error_fmt("exclusive mode is required to using specific meta-page(%d) for " + "checking.", + stuck_meta); rc = EXIT_INTERRUPTED; } if (turn_meta) { if (stuck_meta < 0) { - error("meta-page must be specified (by -0, -1 or -2 options) to turn to " - "it.\n"); + error_fmt("meta-page must be specified (by -0, -1 or -2 options) to turn to " + "it."); rc = EXIT_INTERRUPTED; } - if (envflags & MDBX_RDONLY) { - error("write-mode must be enabled to turn to the specified meta-page.\n"); + if (env_flags & MDBX_RDONLY) { + error_fmt("write-mode must be enabled to turn to the specified meta-page."); rc = EXIT_INTERRUPTED; } - if (only_subdb.iov_base || dont_traversal) { - error( - "whole database checking with b-tree traversal are required to turn " - "to the specified meta-page.\n"); + if (only_table.iov_base || (chk_flags & (MDBX_CHK_SKIP_BTREE_TRAVERSAL | MDBX_CHK_SKIP_KV_TRAVERSAL))) { + error_fmt("whole database checking with b-tree traversal are required to turn " + "to the specified meta-page."); rc = EXIT_INTERRUPTED; } } @@ -5449,526 +3866,82 @@ int main(int argc, char *argv[]) { #endif /* !WINDOWS */ envname = argv[optind]; - print("mdbx_chk %s (%s, T-%s)\nRunning for %s in 'read-%s' mode...\n", - mdbx_version.git.describe, mdbx_version.git.datetime, - mdbx_version.git.tree, envname, - (envflags & MDBX_RDONLY) ? "only" : "write"); - fflush(nullptr); - mdbx_setup_debug((verbose < MDBX_LOG_TRACE - 1) - ? (MDBX_log_level_t)(verbose + 1) - : MDBX_LOG_TRACE, - MDBX_DBG_DUMP | MDBX_DBG_ASSERT | MDBX_DBG_AUDIT | - MDBX_DBG_LEGACY_OVERLAP | MDBX_DBG_DONT_UPGRADE, - logger); + print(MDBX_chk_result, + "mdbx_chk %s (%s, T-%s)\nRunning for %s in 'read-%s' mode with " + "verbosity level %u (%s)...", + mdbx_version.git.describe, mdbx_version.git.datetime, mdbx_version.git.tree, envname, + (env_flags & MDBX_RDONLY) ? "only" : "write", verbose, + (verbose > 8) + ? (MDBX_DEBUG ? "extra details for debugging" : "same as 8 for non-debug builds with MDBX_DEBUG=0") + : "of 0..9"); + lf_flush(); + mdbx_setup_debug( + (verbose + MDBX_LOG_WARN < MDBX_LOG_TRACE) ? (MDBX_log_level_t)(verbose + MDBX_LOG_WARN) : MDBX_LOG_TRACE, + MDBX_DBG_DUMP | MDBX_DBG_ASSERT | MDBX_DBG_AUDIT | MDBX_DBG_LEGACY_OVERLAP | MDBX_DBG_DONT_UPGRADE, logger); rc = mdbx_env_create(&env); if (rc) { - error("mdbx_env_create() failed, error %d %s\n", rc, mdbx_strerror(rc)); + error_fn("mdbx_env_create", rc); return rc < 0 ? EXIT_FAILURE_MDBX : EXIT_FAILURE_SYS; } - rc = mdbx_env_set_maxdbs(env, MDBX_MAX_DBI); + rc = mdbx_env_set_maxdbs(env, CORE_DBS); if (rc) { - error("mdbx_env_set_maxdbs() failed, error %d %s\n", rc, mdbx_strerror(rc)); + error_fn("mdbx_env_set_maxdbs", rc); goto bailout; } if (stuck_meta >= 0) { - rc = mdbx_env_open_for_recovery(env, envname, stuck_meta, - (envflags & MDBX_RDONLY) ? false : true); + rc = mdbx_env_open_for_recovery(env, envname, stuck_meta, (env_flags & MDBX_RDONLY) ? false : true); } else { - rc = mdbx_env_open(env, envname, envflags, 0); - if ((envflags & MDBX_EXCLUSIVE) && - (rc == MDBX_BUSY || + rc = mdbx_env_open(env, envname, env_flags, 0); + if ((env_flags & MDBX_EXCLUSIVE) && (rc == MDBX_BUSY || #if defined(_WIN32) || defined(_WIN64) - rc == ERROR_LOCK_VIOLATION || rc == ERROR_SHARING_VIOLATION + rc == ERROR_LOCK_VIOLATION || rc == ERROR_SHARING_VIOLATION #else - rc == EBUSY || rc == EAGAIN + rc == EBUSY || rc == EAGAIN #endif - )) { - envflags &= ~MDBX_EXCLUSIVE; - rc = mdbx_env_open(env, envname, envflags | MDBX_ACCEDE, 0); + )) { + env_flags &= ~MDBX_EXCLUSIVE; + rc = mdbx_env_open(env, envname, env_flags | MDBX_ACCEDE, 0); } } if (rc) { - error("mdbx_env_open() failed, error %d %s\n", rc, mdbx_strerror(rc)); - if (rc == MDBX_WANNA_RECOVERY && (envflags & MDBX_RDONLY)) - print("Please run %s in the read-write mode (with '-w' option).\n", prog); + error_fn("mdbx_env_open", rc); + if (rc == MDBX_WANNA_RECOVERY && (env_flags & MDBX_RDONLY)) + print_ln(MDBX_chk_result, "Please run %s in the read-write mode (with '-w' option).", prog); goto bailout; } - if (verbose) - print(" - %s mode\n", - (envflags & MDBX_EXCLUSIVE) ? "monopolistic" : "cooperative"); - - if ((envflags & (MDBX_RDONLY | MDBX_EXCLUSIVE)) == 0) { - if (verbose) { - print(" - taking write lock..."); - fflush(nullptr); - } - rc = mdbx_txn_lock(env, false); - if (rc != MDBX_SUCCESS) { - error("mdbx_txn_lock() failed, error %d %s\n", rc, mdbx_strerror(rc)); - goto bailout; - } - if (verbose) - print(" done\n"); - write_locked = true; - } + print_ln(MDBX_chk_verbose, "%s mode", (env_flags & MDBX_EXCLUSIVE) ? "monopolistic" : "cooperative"); if (warmup) { - if (verbose) { - print(" - warming up..."); - fflush(nullptr); - } + anchor_lineno = print(MDBX_chk_verbose, "warming up..."); + flush(); rc = mdbx_env_warmup(env, nullptr, warmup_flags, 3600 * 65536); if (MDBX_IS_ERROR(rc)) { - error("mdbx_env_warmup(flags %u) failed, error %d %s\n", warmup_flags, rc, - mdbx_strerror(rc)); + error_fn("mdbx_env_warmup", rc); goto bailout; } - if (verbose) - print(" %s\n", rc ? "timeout" : "done"); - } - - rc = mdbx_txn_begin(env, nullptr, MDBX_TXN_RDONLY, &txn); - if (rc) { - error("mdbx_txn_begin() failed, error %d %s\n", rc, mdbx_strerror(rc)); - goto bailout; + suffix(anchor_lineno, rc ? "timeout" : "done"); } - rc = mdbx_env_info_ex(env, txn, &envinfo, sizeof(envinfo)); + rc = mdbx_env_chk(env, &cb, &chk, chk_flags, MDBX_chk_result + (verbose << MDBX_chk_severity_prio_shift), 0); if (rc) { - error("mdbx_env_info_ex() failed, error %d %s\n", rc, mdbx_strerror(rc)); - goto bailout; - } - if (verbose) { - print(" - current boot-id "); - if (envinfo.mi_bootid.current.x | envinfo.mi_bootid.current.y) - print("%016" PRIx64 "-%016" PRIx64 "\n", envinfo.mi_bootid.current.x, - envinfo.mi_bootid.current.y); - else - print("unavailable\n"); - } - - mdbx_filehandle_t dxb_fd; - rc = mdbx_env_get_fd(env, &dxb_fd); - if (rc) { - error("mdbx_env_get_fd() failed, error %d %s\n", rc, mdbx_strerror(rc)); - goto bailout; - } - - uint64_t dxb_filesize = 0; -#if defined(_WIN32) || defined(_WIN64) - { - BY_HANDLE_FILE_INFORMATION info; - if (!GetFileInformationByHandle(dxb_fd, &info)) - rc = GetLastError(); - else - dxb_filesize = info.nFileSizeLow | (uint64_t)info.nFileSizeHigh << 32; - } -#else - { - struct stat st; - STATIC_ASSERT_MSG(sizeof(off_t) <= sizeof(uint64_t), - "libmdbx requires 64-bit file I/O on 64-bit systems"); - if (fstat(dxb_fd, &st)) - rc = errno; - else - dxb_filesize = st.st_size; - } -#endif - if (rc) { - error("osal_filesize() failed, error %d %s\n", rc, mdbx_strerror(rc)); - goto bailout; - } - - errno = 0; - const uint64_t dxbfile_pages = dxb_filesize / envinfo.mi_dxb_pagesize; - alloc_pages = txn->mt_next_pgno; - backed_pages = envinfo.mi_geo.current / envinfo.mi_dxb_pagesize; - if (backed_pages > dxbfile_pages) { - print(" ! backed-pages %" PRIu64 " > file-pages %" PRIu64 "\n", - backed_pages, dxbfile_pages); - ++problems_meta; - } - if (dxbfile_pages < NUM_METAS) - print(" ! file-pages %" PRIu64 " < %u\n", dxbfile_pages, NUM_METAS); - if (backed_pages < NUM_METAS) - print(" ! backed-pages %" PRIu64 " < %u\n", backed_pages, NUM_METAS); - if (backed_pages < NUM_METAS || dxbfile_pages < NUM_METAS) - goto bailout; - if (backed_pages > MAX_PAGENO + 1) { - print(" ! backed-pages %" PRIu64 " > max-pages %" PRIaPGNO "\n", - backed_pages, MAX_PAGENO + 1); - ++problems_meta; - backed_pages = MAX_PAGENO + 1; - } - - if ((envflags & (MDBX_EXCLUSIVE | MDBX_RDONLY)) != MDBX_RDONLY) { - if (backed_pages > dxbfile_pages) { - print(" ! backed-pages %" PRIu64 " > file-pages %" PRIu64 "\n", - backed_pages, dxbfile_pages); - ++problems_meta; - backed_pages = dxbfile_pages; - } - if (alloc_pages > backed_pages) { - print(" ! alloc-pages %" PRIu64 " > backed-pages %" PRIu64 "\n", - alloc_pages, backed_pages); - ++problems_meta; - alloc_pages = backed_pages; - } - } else { - /* LY: DB may be shrunk by writer down to the allocated pages. */ - if (alloc_pages > backed_pages) { - print(" ! alloc-pages %" PRIu64 " > backed-pages %" PRIu64 "\n", - alloc_pages, backed_pages); - ++problems_meta; - alloc_pages = backed_pages; - } - if (alloc_pages > dxbfile_pages) { - print(" ! alloc-pages %" PRIu64 " > file-pages %" PRIu64 "\n", - alloc_pages, dxbfile_pages); - ++problems_meta; - alloc_pages = dxbfile_pages; - } - if (backed_pages > dxbfile_pages) - backed_pages = dxbfile_pages; - } - - if (verbose) { - print(" - pagesize %u (%u system), max keysize %d..%d" - ", max readers %u\n", - envinfo.mi_dxb_pagesize, envinfo.mi_sys_pagesize, - mdbx_env_get_maxkeysize_ex(env, MDBX_DUPSORT), - mdbx_env_get_maxkeysize_ex(env, 0), envinfo.mi_maxreaders); - print_size(" - mapsize ", envinfo.mi_mapsize, "\n"); - if (envinfo.mi_geo.lower == envinfo.mi_geo.upper) - print_size(" - fixed datafile: ", envinfo.mi_geo.current, ""); - else { - print_size(" - dynamic datafile: ", envinfo.mi_geo.lower, ""); - print_size(" .. ", envinfo.mi_geo.upper, ", "); - print_size("+", envinfo.mi_geo.grow, ", "); - print_size("-", envinfo.mi_geo.shrink, "\n"); - print_size(" - current datafile: ", envinfo.mi_geo.current, ""); - } - printf(", %" PRIu64 " pages\n", - envinfo.mi_geo.current / envinfo.mi_dxb_pagesize); -#if defined(_WIN32) || defined(_WIN64) - if (envinfo.mi_geo.shrink && envinfo.mi_geo.current != envinfo.mi_geo.upper) - print( - " WARNING: Due Windows system limitations a " - "file couldn't\n be truncated while the database " - "is opened. So, the size\n database file " - "of may by large than the database itself,\n " - "until it will be closed or reopened in read-write mode.\n"); -#endif - verbose_meta(0, envinfo.mi_meta0_txnid, envinfo.mi_meta0_sign, - envinfo.mi_bootid.meta0.x, envinfo.mi_bootid.meta0.y); - verbose_meta(1, envinfo.mi_meta1_txnid, envinfo.mi_meta1_sign, - envinfo.mi_bootid.meta1.x, envinfo.mi_bootid.meta1.y); - verbose_meta(2, envinfo.mi_meta2_txnid, envinfo.mi_meta2_sign, - envinfo.mi_bootid.meta2.x, envinfo.mi_bootid.meta2.y); - } - - if (stuck_meta >= 0) { - if (verbose) { - print(" - skip checking meta-pages since the %u" - " is selected for verification\n", - stuck_meta); - print(" - transactions: recent %" PRIu64 - ", selected for verification %" PRIu64 ", lag %" PRIi64 "\n", - envinfo.mi_recent_txnid, get_meta_txnid(stuck_meta), - envinfo.mi_recent_txnid - get_meta_txnid(stuck_meta)); - } - } else { - if (verbose > 1) - print(" - performs check for meta-pages clashes\n"); - if (meta_eq(envinfo.mi_meta0_txnid, envinfo.mi_meta0_sign, - envinfo.mi_meta1_txnid, envinfo.mi_meta1_sign)) { - print(" ! meta-%d and meta-%d are clashed\n", 0, 1); - ++problems_meta; - } - if (meta_eq(envinfo.mi_meta1_txnid, envinfo.mi_meta1_sign, - envinfo.mi_meta2_txnid, envinfo.mi_meta2_sign)) { - print(" ! meta-%d and meta-%d are clashed\n", 1, 2); - ++problems_meta; - } - if (meta_eq(envinfo.mi_meta2_txnid, envinfo.mi_meta2_sign, - envinfo.mi_meta0_txnid, envinfo.mi_meta0_sign)) { - print(" ! meta-%d and meta-%d are clashed\n", 2, 0); - ++problems_meta; - } - - const unsigned steady_meta_id = meta_recent(true); - const uint64_t steady_meta_txnid = get_meta_txnid(steady_meta_id); - const unsigned weak_meta_id = meta_recent(false); - const uint64_t weak_meta_txnid = get_meta_txnid(weak_meta_id); - if (envflags & MDBX_EXCLUSIVE) { - if (verbose > 1) - print(" - performs full check recent-txn-id with meta-pages\n"); - if (steady_meta_txnid != envinfo.mi_recent_txnid) { - print(" ! steady meta-%d txn-id mismatch recent-txn-id (%" PRIi64 - " != %" PRIi64 ")\n", - steady_meta_id, steady_meta_txnid, envinfo.mi_recent_txnid); - ++problems_meta; - } - } else if (write_locked) { - if (verbose > 1) - print(" - performs lite check recent-txn-id with meta-pages (not a " - "monopolistic mode)\n"); - if (weak_meta_txnid != envinfo.mi_recent_txnid) { - print(" ! weak meta-%d txn-id mismatch recent-txn-id (%" PRIi64 - " != %" PRIi64 ")\n", - weak_meta_id, weak_meta_txnid, envinfo.mi_recent_txnid); - ++problems_meta; - } - } else if (verbose) { - print(" - skip check recent-txn-id with meta-pages (monopolistic or " - "read-write mode only)\n"); - } - total_problems += problems_meta; - - if (verbose) - print(" - transactions: recent %" PRIu64 ", latter reader %" PRIu64 - ", lag %" PRIi64 "\n", - envinfo.mi_recent_txnid, envinfo.mi_latter_reader_txnid, - envinfo.mi_recent_txnid - envinfo.mi_latter_reader_txnid); - } - - if (!dont_traversal) { - struct problem *saved_list; - size_t traversal_problems; - uint64_t empty_pages, lost_bytes; - - print("Traversal b-tree by txn#%" PRIaTXN "...\n", txn->mt_txnid); - fflush(nullptr); - walk.pagemap = osal_calloc((size_t)backed_pages, sizeof(*walk.pagemap)); - if (!walk.pagemap) { - rc = errno ? errno : MDBX_ENOMEM; - error("calloc() failed, error %d %s\n", rc, mdbx_strerror(rc)); - goto bailout; - } - - saved_list = problems_push(); - rc = mdbx_env_pgwalk(txn, pgvisitor, nullptr, - true /* always skip key ordering checking to avoid - MDBX_CORRUPTED when using custom comparators */); - traversal_problems = problems_pop(saved_list); - - if (rc) { - if (rc != MDBX_EINTR || !check_user_break()) - error("mdbx_env_pgwalk() failed, error %d %s\n", rc, mdbx_strerror(rc)); - goto bailout; - } - - for (uint64_t n = 0; n < alloc_pages; ++n) - if (!walk.pagemap[n]) - unused_pages += 1; - - empty_pages = lost_bytes = 0; - for (walk_dbi_t *dbi = &dbi_main; - dbi < ARRAY_END(walk.dbi) && dbi->name.iov_base; ++dbi) { - empty_pages += dbi->pages.empty; - lost_bytes += dbi->lost_bytes; - } - - if (verbose) { - uint64_t total_page_bytes = walk.pgcount * envinfo.mi_dxb_pagesize; - print(" - pages: walked %" PRIu64 ", left/unused %" PRIu64 "\n", - walk.pgcount, unused_pages); - if (verbose > 1) { - for (walk_dbi_t *dbi = walk.dbi; - dbi < ARRAY_END(walk.dbi) && dbi->name.iov_base; ++dbi) { - print(" %s: subtotal %" PRIu64, sdb_name(&dbi->name), - dbi->pages.total); - if (dbi->pages.other && dbi->pages.other != dbi->pages.total) - print(", other %" PRIu64, dbi->pages.other); - if (dbi->pages.branch) - print(", branch %" PRIu64, dbi->pages.branch); - if (dbi->pages.large_count) - print(", large %" PRIu64, dbi->pages.large_count); - uint64_t all_leaf = dbi->pages.leaf + dbi->pages.leaf_dupfixed; - if (all_leaf) { - print(", leaf %" PRIu64, all_leaf); - if (verbose > 2 && - (dbi->pages.subleaf_dupsort | dbi->pages.leaf_dupfixed | - dbi->pages.subleaf_dupfixed)) - print(" (usual %" PRIu64 ", sub-dupsort %" PRIu64 - ", dupfixed %" PRIu64 ", sub-dupfixed %" PRIu64 ")", - dbi->pages.leaf, dbi->pages.subleaf_dupsort, - dbi->pages.leaf_dupfixed, dbi->pages.subleaf_dupfixed); - } - print("\n"); - } - } - - if (verbose > 1) - print(" - usage: total %" PRIu64 " bytes, payload %" PRIu64 - " (%.1f%%), unused " - "%" PRIu64 " (%.1f%%)\n", - total_page_bytes, walk.total_payload_bytes, - walk.total_payload_bytes * 100.0 / total_page_bytes, - total_page_bytes - walk.total_payload_bytes, - (total_page_bytes - walk.total_payload_bytes) * 100.0 / - total_page_bytes); - if (verbose > 2) { - for (walk_dbi_t *dbi = walk.dbi; - dbi < ARRAY_END(walk.dbi) && dbi->name.iov_base; ++dbi) - if (dbi->pages.total) { - uint64_t dbi_bytes = dbi->pages.total * envinfo.mi_dxb_pagesize; - print(" %s: subtotal %" PRIu64 " bytes (%.1f%%)," - " payload %" PRIu64 " (%.1f%%), unused %" PRIu64 " (%.1f%%)", - sdb_name(&dbi->name), dbi_bytes, - dbi_bytes * 100.0 / total_page_bytes, dbi->payload_bytes, - dbi->payload_bytes * 100.0 / dbi_bytes, - dbi_bytes - dbi->payload_bytes, - (dbi_bytes - dbi->payload_bytes) * 100.0 / dbi_bytes); - if (dbi->pages.empty) - print(", %" PRIu64 " empty pages", dbi->pages.empty); - if (dbi->lost_bytes) - print(", %" PRIu64 " bytes lost", dbi->lost_bytes); - print("\n"); - } else - print(" %s: empty\n", sdb_name(&dbi->name)); - } - print(" - summary: average fill %.1f%%", - walk.total_payload_bytes * 100.0 / total_page_bytes); - if (empty_pages) - print(", %" PRIu64 " empty pages", empty_pages); - if (lost_bytes) - print(", %" PRIu64 " bytes lost", lost_bytes); - print(", %" PRIuPTR " problems\n", traversal_problems); - } - } else if (verbose) { - print("Skipping b-tree walk...\n"); - fflush(nullptr); - } - - if (gc_tree_problems) { - print("Skip processing %s since %s is corrupted (%u problems)\n", "@GC", - "b-tree", gc_tree_problems); - problems_freedb = gc_tree_problems; - } else - problems_freedb = process_db(FREE_DBI, MDBX_PGWALK_GC, handle_freedb); - - if (verbose) { - uint64_t value = envinfo.mi_mapsize / envinfo.mi_dxb_pagesize; - double percent = value / 100.0; - print(" - space: %" PRIu64 " total pages", value); - print(", backed %" PRIu64 " (%.1f%%)", backed_pages, - backed_pages / percent); - print(", allocated %" PRIu64 " (%.1f%%)", alloc_pages, - alloc_pages / percent); - - if (verbose > 1) { - value = envinfo.mi_mapsize / envinfo.mi_dxb_pagesize - alloc_pages; - print(", remained %" PRIu64 " (%.1f%%)", value, value / percent); - - value = dont_traversal ? alloc_pages - gc_pages : walk.pgcount; - print(", used %" PRIu64 " (%.1f%%)", value, value / percent); - - print(", gc %" PRIu64 " (%.1f%%)", gc_pages, gc_pages / percent); - - value = gc_pages - reclaimable_pages; - print(", detained %" PRIu64 " (%.1f%%)", value, value / percent); - - print(", reclaimable %" PRIu64 " (%.1f%%)", reclaimable_pages, - reclaimable_pages / percent); - } - - value = envinfo.mi_mapsize / envinfo.mi_dxb_pagesize - alloc_pages + - reclaimable_pages; - print(", available %" PRIu64 " (%.1f%%)\n", value, value / percent); - } - - if ((problems_maindb = data_tree_problems) == 0 && problems_freedb == 0) { - if (!dont_traversal && - (envflags & (MDBX_EXCLUSIVE | MDBX_RDONLY)) != MDBX_RDONLY) { - if (walk.pgcount != alloc_pages - gc_pages) { - error("used pages mismatch (%" PRIu64 "(walked) != %" PRIu64 - "(allocated - GC))\n", - walk.pgcount, alloc_pages - gc_pages); - } - if (unused_pages != gc_pages) { - error("GC pages mismatch (%" PRIu64 "(expected) != %" PRIu64 "(GC))\n", - unused_pages, gc_pages); - } - } else if (verbose) { - print(" - skip check used and GC pages (btree-traversal with " - "monopolistic or read-write mode only)\n"); - } - - problems_maindb = process_db(~0u, /* MAIN_DBI */ nullptr, nullptr); - if (problems_maindb == 0) { - print("Scanning %s for %s...\n", "@MAIN", "sub-database(s)"); - if (!process_db(MAIN_DBI, nullptr, handle_maindb)) { - if (!userdb_count && verbose) - print(" - does not contain multiple databases\n"); - } - } else { - print("Skip processing %s since %s is corrupted (%u problems)\n", - "sub-database(s)", "@MAIN", problems_maindb); - } - } else { - print("Skip processing %s since %s is corrupted (%u problems)\n", "@MAIN", - "b-tree", data_tree_problems); - } - - if (rc == 0 && total_problems == 1 && problems_meta == 1 && !dont_traversal && - (envflags & MDBX_RDONLY) == 0 && !only_subdb.iov_base && stuck_meta < 0 && - get_meta_txnid(meta_recent(true)) < envinfo.mi_recent_txnid) { - print("Perform sync-to-disk for make steady checkpoint at txn-id #%" PRIi64 - "\n", - envinfo.mi_recent_txnid); - fflush(nullptr); - if (write_locked) { - mdbx_txn_unlock(env); - write_locked = false; - } - rc = mdbx_env_sync_ex(env, true, false); - if (rc != MDBX_SUCCESS) - error("mdbx_env_pgwalk() failed, error %d %s\n", rc, mdbx_strerror(rc)); - else { - total_problems -= 1; - problems_meta -= 1; - } - } - - if (turn_meta && stuck_meta >= 0 && !dont_traversal && !only_subdb.iov_base && - (envflags & (MDBX_RDONLY | MDBX_EXCLUSIVE)) == MDBX_EXCLUSIVE) { - const bool successful_check = (rc | total_problems | problems_meta) == 0; - if (successful_check || force_turn_meta) { - fflush(nullptr); - print(" = Performing turn to the specified meta-page (%d) due to %s!\n", - stuck_meta, - successful_check ? "successful check" : "the -T option was given"); - fflush(nullptr); - rc = mdbx_env_turn_for_recovery(env, stuck_meta); - if (rc != MDBX_SUCCESS) - error("mdbx_env_turn_for_recovery() failed, error %d %s\n", rc, - mdbx_strerror(rc)); - } else { - print(" = Skipping turn to the specified meta-page (%d) due to " - "unsuccessful check!\n", - stuck_meta); - } + if (chk.result.total_problems == 0) + error_fn("mdbx_env_chk", rc); + else if (rc != MDBX_EINTR && rc != MDBX_RESULT_TRUE && !user_break) + rc = 0; } bailout: - if (txn) - mdbx_txn_abort(txn); - if (write_locked) { - mdbx_txn_unlock(env); - write_locked = false; - } if (env) { - const bool dont_sync = rc != 0 || total_problems; + const bool dont_sync = rc != 0 || chk.result.total_problems || (chk_flags & MDBX_CHK_READWRITE) == 0; mdbx_env_close_ex(env, dont_sync); } - fflush(nullptr); + flush(); if (rc) { - if (rc < 0) + if (rc > 0) return user_break ? EXIT_INTERRUPTED : EXIT_FAILURE_SYS; return EXIT_FAILURE_MDBX; } @@ -5978,21 +3951,20 @@ int main(int argc, char *argv[]) { elapsed = (timestamp_finish - timestamp_start) * 1e-3; #else if (clock_gettime(CLOCK_MONOTONIC, ×tamp_finish)) { - rc = errno; - error("clock_gettime() failed, error %d %s\n", rc, mdbx_strerror(rc)); + error_fn("clock_gettime", errno); return EXIT_FAILURE_SYS; } - elapsed = timestamp_finish.tv_sec - timestamp_start.tv_sec + - (timestamp_finish.tv_nsec - timestamp_start.tv_nsec) * 1e-9; + elapsed = + timestamp_finish.tv_sec - timestamp_start.tv_sec + (timestamp_finish.tv_nsec - timestamp_start.tv_nsec) * 1e-9; #endif /* !WINDOWS */ - if (total_problems) { - print("Total %u error%s detected, elapsed %.3f seconds.\n", total_problems, - (total_problems > 1) ? "s are" : " is", elapsed); - if (problems_meta || problems_maindb || problems_freedb) + if (chk.result.total_problems) { + print_ln(MDBX_chk_result, "Total %" PRIuSIZE " error%s detected, elapsed %.3f seconds.", chk.result.total_problems, + (chk.result.total_problems > 1) ? "s are" : " is", elapsed); + if (chk.result.problems_meta || chk.result.problems_kv || chk.result.problems_gc) return EXIT_FAILURE_CHECK_MAJOR; return EXIT_FAILURE_CHECK_MINOR; } - print("No error is detected, elapsed %.3f seconds\n", elapsed); + print_ln(MDBX_chk_result, "No error is detected, elapsed %.3f seconds.", elapsed); return EXIT_SUCCESS; } diff --git a/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx_copy.c b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx_copy.c index a4fff98c1c5..96e8c485c71 100644 --- a/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx_copy.c +++ b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx_copy.c @@ -1,18 +1,12 @@ -/* mdbx_copy.c - memory-mapped database backup tool */ - -/* - * Copyright 2015-2024 Leonid Yuriev - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . */ - +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \note Please refer to the COPYRIGHT file for explanations license change, +/// credits and acknowledgments. +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 +/// +/// mdbx_copy.c - memory-mapped database backup tool +/// + +/* clang-format off */ #ifdef _MSC_VER #if _MSC_VER > 1800 #pragma warning(disable : 4464) /* relative include path contains '..' */ @@ -21,38 +15,26 @@ #endif /* _MSC_VER (warnings) */ #define xMDBX_TOOLS /* Avoid using internal eASSERT() */ -/* - * Copyright 2015-2024 Leonid Yuriev - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . */ +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 -#define MDBX_BUILD_SOURCERY e156c1a97c017ce89d6541cd9464ae5a9761d76b3fd2f1696521f5f3792904fc_v0_12_13_0_g1fff1f67 -#ifdef MDBX_CONFIG_H -#include MDBX_CONFIG_H -#endif +#define MDBX_BUILD_SOURCERY 6b5df6869d2bf5419e3a8189d9cc849cc9911b9c8a951b9750ed0a261ce43724_v0_13_7_0_g566b0f93 #define LIBMDBX_INTERNALS -#ifdef xMDBX_TOOLS #define MDBX_DEPRECATED -#endif /* xMDBX_TOOLS */ -#ifdef xMDBX_ALLOY -/* Amalgamated build */ -#define MDBX_INTERNAL_FUNC static -#define MDBX_INTERNAL_VAR static -#else -/* Non-amalgamated build */ -#define MDBX_INTERNAL_FUNC -#define MDBX_INTERNAL_VAR extern -#endif /* xMDBX_ALLOY */ +#ifdef MDBX_CONFIG_H +#include MDBX_CONFIG_H +#endif + +/* Undefine the NDEBUG if debugging is enforced by MDBX_DEBUG */ +#if (defined(MDBX_DEBUG) && MDBX_DEBUG > 0) || (defined(MDBX_FORCE_ASSERTIONS) && MDBX_FORCE_ASSERTIONS) +#undef NDEBUG +#ifndef MDBX_DEBUG +/* Чтобы избежать включения отладки только из-за включения assert-проверок */ +#define MDBX_DEBUG 0 +#endif +#endif /*----------------------------------------------------------------------------*/ @@ -70,14 +52,59 @@ #endif /* MDBX_DISABLE_GNU_SOURCE */ /* Should be defined before any includes */ -#if !defined(_FILE_OFFSET_BITS) && !defined(__ANDROID_API__) && \ - !defined(ANDROID) +#if !defined(_FILE_OFFSET_BITS) && !defined(__ANDROID_API__) && !defined(ANDROID) #define _FILE_OFFSET_BITS 64 -#endif +#endif /* _FILE_OFFSET_BITS */ -#ifdef __APPLE__ +#if defined(__APPLE__) && !defined(_DARWIN_C_SOURCE) #define _DARWIN_C_SOURCE -#endif +#endif /* _DARWIN_C_SOURCE */ + +#if (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) && !defined(__USE_MINGW_ANSI_STDIO) +#define __USE_MINGW_ANSI_STDIO 1 +#endif /* MinGW */ + +#if defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS) + +#ifndef _WIN32_WINNT +#define _WIN32_WINNT 0x0601 /* Windows 7 */ +#endif /* _WIN32_WINNT */ + +#if !defined(_CRT_SECURE_NO_WARNINGS) +#define _CRT_SECURE_NO_WARNINGS +#endif /* _CRT_SECURE_NO_WARNINGS */ +#if !defined(UNICODE) +#define UNICODE +#endif /* UNICODE */ + +#if !defined(_NO_CRT_STDIO_INLINE) && MDBX_BUILD_SHARED_LIBRARY && !defined(xMDBX_TOOLS) && MDBX_WITHOUT_MSVC_CRT +#define _NO_CRT_STDIO_INLINE +#endif /* _NO_CRT_STDIO_INLINE */ + +#elif !defined(_POSIX_C_SOURCE) +#define _POSIX_C_SOURCE 200809L +#endif /* Windows */ + +#ifdef __cplusplus + +#ifndef NOMINMAX +#define NOMINMAX +#endif /* NOMINMAX */ + +/* Workaround for modern libstdc++ with CLANG < 4.x */ +#if defined(__SIZEOF_INT128__) && !defined(__GLIBCXX_TYPE_INT_N_0) && defined(__clang__) && __clang_major__ < 4 +#define __GLIBCXX_BITSIZE_INT_N_0 128 +#define __GLIBCXX_TYPE_INT_N_0 __int128 +#endif /* Workaround for modern libstdc++ with CLANG < 4.x */ + +#ifdef _MSC_VER +/* Workaround for MSVC' header `extern "C"` vs `std::` redefinition bug */ +#if defined(__SANITIZE_ADDRESS__) && !defined(_DISABLE_VECTOR_ANNOTATION) +#define _DISABLE_VECTOR_ANNOTATION +#endif /* _DISABLE_VECTOR_ANNOTATION */ +#endif /* _MSC_VER */ + +#endif /* __cplusplus */ #ifdef _MSC_VER #if _MSC_FULL_VER < 190024234 @@ -99,12 +126,8 @@ * and how to and where you can obtain the latest "Visual Studio 2015" build * with all fixes. */ -#error \ - "At least \"Microsoft C/C++ Compiler\" version 19.00.24234 (Visual Studio 2015 Update 3) is required." +#error "At least \"Microsoft C/C++ Compiler\" version 19.00.24234 (Visual Studio 2015 Update 3) is required." #endif -#ifndef _CRT_SECURE_NO_WARNINGS -#define _CRT_SECURE_NO_WARNINGS -#endif /* _CRT_SECURE_NO_WARNINGS */ #if _MSC_VER > 1800 #pragma warning(disable : 4464) /* relative include path contains '..' */ #endif @@ -112,124 +135,78 @@ #pragma warning(disable : 5045) /* will insert Spectre mitigation... */ #endif #if _MSC_VER > 1914 -#pragma warning( \ - disable : 5105) /* winbase.h(9531): warning C5105: macro expansion \ - producing 'defined' has undefined behavior */ +#pragma warning(disable : 5105) /* winbase.h(9531): warning C5105: macro expansion \ + producing 'defined' has undefined behavior */ +#endif +#if _MSC_VER < 1920 +/* avoid "error C2219: syntax error: type qualifier must be after '*'" */ +#define __restrict #endif #if _MSC_VER > 1930 #pragma warning(disable : 6235) /* is always a constant */ -#pragma warning(disable : 6237) /* is never evaluated and might \ +#pragma warning(disable : 6237) /* is never evaluated and might \ have side effects */ +#pragma warning(disable : 5286) /* implicit conversion from enum type 'type 1' to enum type 'type 2' */ +#pragma warning(disable : 5287) /* operands are different enum types 'type 1' and 'type 2' */ #endif #pragma warning(disable : 4710) /* 'xyz': function not inlined */ -#pragma warning(disable : 4711) /* function 'xyz' selected for automatic \ +#pragma warning(disable : 4711) /* function 'xyz' selected for automatic \ inline expansion */ -#pragma warning(disable : 4201) /* nonstandard extension used: nameless \ +#pragma warning(disable : 4201) /* nonstandard extension used: nameless \ struct/union */ #pragma warning(disable : 4702) /* unreachable code */ #pragma warning(disable : 4706) /* assignment within conditional expression */ #pragma warning(disable : 4127) /* conditional expression is constant */ -#pragma warning(disable : 4324) /* 'xyz': structure was padded due to \ +#pragma warning(disable : 4324) /* 'xyz': structure was padded due to \ alignment specifier */ #pragma warning(disable : 4310) /* cast truncates constant value */ -#pragma warning(disable : 4820) /* bytes padding added after data member for \ +#pragma warning(disable : 4820) /* bytes padding added after data member for \ alignment */ -#pragma warning(disable : 4548) /* expression before comma has no effect; \ +#pragma warning(disable : 4548) /* expression before comma has no effect; \ expected expression with side - effect */ -#pragma warning(disable : 4366) /* the result of the unary '&' operator may be \ +#pragma warning(disable : 4366) /* the result of the unary '&' operator may be \ unaligned */ -#pragma warning(disable : 4200) /* nonstandard extension used: zero-sized \ +#pragma warning(disable : 4200) /* nonstandard extension used: zero-sized \ array in struct/union */ -#pragma warning(disable : 4204) /* nonstandard extension used: non-constant \ +#pragma warning(disable : 4204) /* nonstandard extension used: non-constant \ aggregate initializer */ -#pragma warning( \ - disable : 4505) /* unreferenced local function has been removed */ -#endif /* _MSC_VER (warnings) */ +#pragma warning(disable : 4505) /* unreferenced local function has been removed */ +#endif /* _MSC_VER (warnings) */ #if defined(__GNUC__) && __GNUC__ < 9 #pragma GCC diagnostic ignored "-Wattributes" #endif /* GCC < 9 */ -#if (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) && \ - !defined(__USE_MINGW_ANSI_STDIO) -#define __USE_MINGW_ANSI_STDIO 1 -#endif /* MinGW */ - -#if (defined(_WIN32) || defined(_WIN64)) && !defined(UNICODE) -#define UNICODE -#endif /* UNICODE */ - -#include "mdbx.h" -/* - * Copyright 2015-2024 Leonid Yuriev - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - */ - - /*----------------------------------------------------------------------------*/ /* Microsoft compiler generates a lot of warning for self includes... */ #ifdef _MSC_VER #pragma warning(push, 1) -#pragma warning(disable : 4548) /* expression before comma has no effect; \ +#pragma warning(disable : 4548) /* expression before comma has no effect; \ expected expression with side - effect */ -#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ +#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ * semantics are not enabled. Specify /EHsc */ -#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ - * mode specified; termination on exception is \ +#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ + * mode specified; termination on exception is \ * not guaranteed. Specify /EHsc */ #endif /* _MSC_VER (warnings) */ -#if defined(_WIN32) || defined(_WIN64) -#if !defined(_CRT_SECURE_NO_WARNINGS) -#define _CRT_SECURE_NO_WARNINGS -#endif /* _CRT_SECURE_NO_WARNINGS */ -#if !defined(_NO_CRT_STDIO_INLINE) && MDBX_BUILD_SHARED_LIBRARY && \ - !defined(xMDBX_TOOLS) && MDBX_WITHOUT_MSVC_CRT -#define _NO_CRT_STDIO_INLINE -#endif -#elif !defined(_POSIX_C_SOURCE) -#define _POSIX_C_SOURCE 200809L -#endif /* Windows */ - /*----------------------------------------------------------------------------*/ /* basic C99 includes */ + #include #include #include #include #include +#include #include #include #include #include #include -#if (-6 & 5) || CHAR_BIT != 8 || UINT_MAX < 0xffffffff || ULONG_MAX % 0xFFFF -#error \ - "Sanity checking failed: Two's complement, reasonably sized integer types" -#endif - -#ifndef SSIZE_MAX -#define SSIZE_MAX INTPTR_MAX -#endif - -#if UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul || defined(_WIN64) -#define MDBX_WORDBITS 64 -#else -#define MDBX_WORDBITS 32 -#endif /* MDBX_WORDBITS */ - /*----------------------------------------------------------------------------*/ /* feature testing */ @@ -241,6 +218,14 @@ #define __has_include(x) (0) #endif +#ifndef __has_attribute +#define __has_attribute(x) (0) +#endif + +#ifndef __has_cpp_attribute +#define __has_cpp_attribute(x) 0 +#endif + #ifndef __has_feature #define __has_feature(x) (0) #endif @@ -263,8 +248,7 @@ #ifndef __GNUC_PREREQ #if defined(__GNUC__) && defined(__GNUC_MINOR__) -#define __GNUC_PREREQ(maj, min) \ - ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min)) +#define __GNUC_PREREQ(maj, min) ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min)) #else #define __GNUC_PREREQ(maj, min) (0) #endif @@ -272,8 +256,7 @@ #ifndef __CLANG_PREREQ #ifdef __clang__ -#define __CLANG_PREREQ(maj, min) \ - ((__clang_major__ << 16) + __clang_minor__ >= ((maj) << 16) + (min)) +#define __CLANG_PREREQ(maj, min) ((__clang_major__ << 16) + __clang_minor__ >= ((maj) << 16) + (min)) #else #define __CLANG_PREREQ(maj, min) (0) #endif @@ -281,13 +264,51 @@ #ifndef __GLIBC_PREREQ #if defined(__GLIBC__) && defined(__GLIBC_MINOR__) -#define __GLIBC_PREREQ(maj, min) \ - ((__GLIBC__ << 16) + __GLIBC_MINOR__ >= ((maj) << 16) + (min)) +#define __GLIBC_PREREQ(maj, min) ((__GLIBC__ << 16) + __GLIBC_MINOR__ >= ((maj) << 16) + (min)) #else #define __GLIBC_PREREQ(maj, min) (0) #endif #endif /* __GLIBC_PREREQ */ +/*----------------------------------------------------------------------------*/ +/* pre-requirements */ + +#if (-6 & 5) || CHAR_BIT != 8 || UINT_MAX < 0xffffffff || ULONG_MAX % 0xFFFF +#error "Sanity checking failed: Two's complement, reasonably sized integer types" +#endif + +#ifndef SSIZE_MAX +#define SSIZE_MAX INTPTR_MAX +#endif + +#if defined(__GNUC__) && !__GNUC_PREREQ(4, 2) +/* Actually libmdbx was not tested with compilers older than GCC 4.2. + * But you could ignore this warning at your own risk. + * In such case please don't rise up an issues related ONLY to old compilers. + */ +#warning "libmdbx required GCC >= 4.2" +#endif + +#if defined(__clang__) && !__CLANG_PREREQ(3, 8) +/* Actually libmdbx was not tested with CLANG older than 3.8. + * But you could ignore this warning at your own risk. + * In such case please don't rise up an issues related ONLY to old compilers. + */ +#warning "libmdbx required CLANG >= 3.8" +#endif + +#if defined(__GLIBC__) && !__GLIBC_PREREQ(2, 12) +/* Actually libmdbx was not tested with something older than glibc 2.12. + * But you could ignore this warning at your own risk. + * In such case please don't rise up an issues related ONLY to old systems. + */ +#warning "libmdbx was only tested with GLIBC >= 2.12." +#endif + +#ifdef __SANITIZE_THREAD__ +#warning "libmdbx don't compatible with ThreadSanitizer, you will get a lot of false-positive issues." +#endif /* __SANITIZE_THREAD__ */ + /*----------------------------------------------------------------------------*/ /* C11' alignas() */ @@ -317,8 +338,7 @@ #endif #endif /* __extern_C */ -#if !defined(nullptr) && !defined(__cplusplus) || \ - (__cplusplus < 201103L && !defined(_MSC_VER)) +#if !defined(nullptr) && !defined(__cplusplus) || (__cplusplus < 201103L && !defined(_MSC_VER)) #define nullptr NULL #endif @@ -330,9 +350,8 @@ #endif #endif /* Apple OSX & iOS */ -#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \ - defined(__BSD__) || defined(__bsdi__) || defined(__DragonFly__) || \ - defined(__APPLE__) || defined(__MACH__) +#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__BSD__) || defined(__bsdi__) || \ + defined(__DragonFly__) || defined(__APPLE__) || defined(__MACH__) #include #include #include @@ -349,8 +368,7 @@ #endif #else #include -#if !(defined(__sun) || defined(__SVR4) || defined(__svr4__) || \ - defined(_WIN32) || defined(_WIN64)) +#if !(defined(__sun) || defined(__SVR4) || defined(__svr4__) || defined(_WIN32) || defined(_WIN64)) #include #endif /* !Solaris */ #endif /* !xBSD */ @@ -404,12 +422,14 @@ __extern_C key_t ftok(const char *, int); #ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN #endif /* WIN32_LEAN_AND_MEAN */ -#include -#include #include #include #include +/* После подгрузки windows.h, чтобы избежать проблем со сборкой MINGW и т.п. */ +#include +#include + #else /*----------------------------------------------------------------------*/ #include @@ -437,11 +457,6 @@ __extern_C key_t ftok(const char *, int); #if __ANDROID_API__ >= 21 #include #endif -#if defined(_FILE_OFFSET_BITS) && _FILE_OFFSET_BITS != MDBX_WORDBITS -#error "_FILE_OFFSET_BITS != MDBX_WORDBITS" (_FILE_OFFSET_BITS != MDBX_WORDBITS) -#elif defined(__FILE_OFFSET_BITS) && __FILE_OFFSET_BITS != MDBX_WORDBITS -#error "__FILE_OFFSET_BITS != MDBX_WORDBITS" (__FILE_OFFSET_BITS != MDBX_WORDBITS) -#endif #endif /* Android */ #if defined(HAVE_SYS_STAT_H) || __has_include() @@ -457,43 +472,38 @@ __extern_C key_t ftok(const char *, int); /*----------------------------------------------------------------------------*/ /* Byteorder */ -#if defined(i386) || defined(__386) || defined(__i386) || defined(__i386__) || \ - defined(i486) || defined(__i486) || defined(__i486__) || defined(i586) || \ - defined(__i586) || defined(__i586__) || defined(i686) || \ - defined(__i686) || defined(__i686__) || defined(_M_IX86) || \ - defined(_X86_) || defined(__THW_INTEL__) || defined(__I86__) || \ - defined(__INTEL__) || defined(__x86_64) || defined(__x86_64__) || \ - defined(__amd64__) || defined(__amd64) || defined(_M_X64) || \ - defined(_M_AMD64) || defined(__IA32__) || defined(__INTEL__) +#if defined(i386) || defined(__386) || defined(__i386) || defined(__i386__) || defined(i486) || defined(__i486) || \ + defined(__i486__) || defined(i586) || defined(__i586) || defined(__i586__) || defined(i686) || defined(__i686) || \ + defined(__i686__) || defined(_M_IX86) || defined(_X86_) || defined(__THW_INTEL__) || defined(__I86__) || \ + defined(__INTEL__) || defined(__x86_64) || defined(__x86_64__) || defined(__amd64__) || defined(__amd64) || \ + defined(_M_X64) || defined(_M_AMD64) || defined(__IA32__) || defined(__INTEL__) #ifndef __ia32__ /* LY: define neutral __ia32__ for x86 and x86-64 */ #define __ia32__ 1 #endif /* __ia32__ */ -#if !defined(__amd64__) && \ - (defined(__x86_64) || defined(__x86_64__) || defined(__amd64) || \ - defined(_M_X64) || defined(_M_AMD64)) +#if !defined(__amd64__) && \ + (defined(__x86_64) || defined(__x86_64__) || defined(__amd64) || defined(_M_X64) || defined(_M_AMD64)) /* LY: define trusty __amd64__ for all AMD64/x86-64 arch */ #define __amd64__ 1 #endif /* __amd64__ */ #endif /* all x86 */ -#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || \ - !defined(__ORDER_BIG_ENDIAN__) +#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || !defined(__ORDER_BIG_ENDIAN__) -#if defined(__GLIBC__) || defined(__GNU_LIBRARY__) || \ - defined(__ANDROID_API__) || defined(HAVE_ENDIAN_H) || __has_include() +#if defined(__GLIBC__) || defined(__GNU_LIBRARY__) || defined(__ANDROID_API__) || defined(HAVE_ENDIAN_H) || \ + __has_include() #include -#elif defined(__APPLE__) || defined(__MACH__) || defined(__OpenBSD__) || \ - defined(HAVE_MACHINE_ENDIAN_H) || __has_include() +#elif defined(__APPLE__) || defined(__MACH__) || defined(__OpenBSD__) || defined(HAVE_MACHINE_ENDIAN_H) || \ + __has_include() #include #elif defined(HAVE_SYS_ISA_DEFS_H) || __has_include() #include -#elif (defined(HAVE_SYS_TYPES_H) && defined(HAVE_SYS_ENDIAN_H)) || \ +#elif (defined(HAVE_SYS_TYPES_H) && defined(HAVE_SYS_ENDIAN_H)) || \ (__has_include() && __has_include()) #include #include -#elif defined(__bsdi__) || defined(__DragonFly__) || defined(__FreeBSD__) || \ - defined(__NetBSD__) || defined(HAVE_SYS_PARAM_H) || __has_include() +#elif defined(__bsdi__) || defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || \ + defined(HAVE_SYS_PARAM_H) || __has_include() #include #endif /* OS */ @@ -509,27 +519,19 @@ __extern_C key_t ftok(const char *, int); #define __ORDER_LITTLE_ENDIAN__ 1234 #define __ORDER_BIG_ENDIAN__ 4321 -#if defined(__LITTLE_ENDIAN__) || \ - (defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)) || \ - defined(__ARMEL__) || defined(__THUMBEL__) || defined(__AARCH64EL__) || \ - defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \ - defined(_M_ARM) || defined(_M_ARM64) || defined(__e2k__) || \ - defined(__elbrus_4c__) || defined(__elbrus_8c__) || defined(__bfin__) || \ - defined(__BFIN__) || defined(__ia64__) || defined(_IA64) || \ - defined(__IA64__) || defined(__ia64) || defined(_M_IA64) || \ - defined(__itanium__) || defined(__ia32__) || defined(__CYGWIN__) || \ - defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) || \ - defined(__WINDOWS__) +#if defined(__LITTLE_ENDIAN__) || (defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)) || defined(__ARMEL__) || \ + defined(__THUMBEL__) || defined(__AARCH64EL__) || defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \ + defined(_M_ARM) || defined(_M_ARM64) || defined(__e2k__) || defined(__elbrus_4c__) || defined(__elbrus_8c__) || \ + defined(__bfin__) || defined(__BFIN__) || defined(__ia64__) || defined(_IA64) || defined(__IA64__) || \ + defined(__ia64) || defined(_M_IA64) || defined(__itanium__) || defined(__ia32__) || defined(__CYGWIN__) || \ + defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) || defined(__WINDOWS__) #define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__ -#elif defined(__BIG_ENDIAN__) || \ - (defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) || \ - defined(__ARMEB__) || defined(__THUMBEB__) || defined(__AARCH64EB__) || \ - defined(__MIPSEB__) || defined(_MIPSEB) || defined(__MIPSEB) || \ - defined(__m68k__) || defined(M68000) || defined(__hppa__) || \ - defined(__hppa) || defined(__HPPA__) || defined(__sparc__) || \ - defined(__sparc) || defined(__370__) || defined(__THW_370__) || \ - defined(__s390__) || defined(__s390x__) || defined(__SYSC_ZARCH__) +#elif defined(__BIG_ENDIAN__) || (defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) || defined(__ARMEB__) || \ + defined(__THUMBEB__) || defined(__AARCH64EB__) || defined(__MIPSEB__) || defined(_MIPSEB) || defined(__MIPSEB) || \ + defined(__m68k__) || defined(M68000) || defined(__hppa__) || defined(__hppa) || defined(__HPPA__) || \ + defined(__sparc__) || defined(__sparc) || defined(__370__) || defined(__THW_370__) || defined(__s390__) || \ + defined(__s390x__) || defined(__SYSC_ZARCH__) #define __BYTE_ORDER__ __ORDER_BIG_ENDIAN__ #else @@ -539,6 +541,12 @@ __extern_C key_t ftok(const char *, int); #endif #endif /* __BYTE_ORDER__ || __ORDER_LITTLE_ENDIAN__ || __ORDER_BIG_ENDIAN__ */ +#if UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul || defined(_WIN64) +#define MDBX_WORDBITS 64 +#else +#define MDBX_WORDBITS 32 +#endif /* MDBX_WORDBITS */ + /*----------------------------------------------------------------------------*/ /* Availability of CMOV or equivalent */ @@ -549,17 +557,14 @@ __extern_C key_t ftok(const char *, int); #define MDBX_HAVE_CMOV 1 #elif defined(__thumb__) || defined(__thumb) || defined(__TARGET_ARCH_THUMB) #define MDBX_HAVE_CMOV 0 -#elif defined(_M_ARM) || defined(_M_ARM64) || defined(__aarch64__) || \ - defined(__aarch64) || defined(__arm__) || defined(__arm) || \ - defined(__CC_ARM) +#elif defined(_M_ARM) || defined(_M_ARM64) || defined(__aarch64__) || defined(__aarch64) || defined(__arm__) || \ + defined(__arm) || defined(__CC_ARM) #define MDBX_HAVE_CMOV 1 -#elif (defined(__riscv__) || defined(__riscv64)) && \ - (defined(__riscv_b) || defined(__riscv_bitmanip)) +#elif (defined(__riscv__) || defined(__riscv64)) && (defined(__riscv_b) || defined(__riscv_bitmanip)) #define MDBX_HAVE_CMOV 1 -#elif defined(i686) || defined(__i686) || defined(__i686__) || \ - (defined(_M_IX86) && _M_IX86 > 600) || defined(__x86_64) || \ - defined(__x86_64__) || defined(__amd64__) || defined(__amd64) || \ - defined(_M_X64) || defined(_M_AMD64) +#elif defined(i686) || defined(__i686) || defined(__i686__) || (defined(_M_IX86) && _M_IX86 > 600) || \ + defined(__x86_64) || defined(__x86_64__) || defined(__amd64__) || defined(__amd64) || defined(_M_X64) || \ + defined(_M_AMD64) #define MDBX_HAVE_CMOV 1 #else #define MDBX_HAVE_CMOV 0 @@ -585,8 +590,7 @@ __extern_C key_t ftok(const char *, int); #endif #elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) #include -#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ - (defined(HP_IA64) || defined(__ia64)) +#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && (defined(HP_IA64) || defined(__ia64)) #include #elif defined(__IBMC__) && defined(__powerpc) #include @@ -608,29 +612,26 @@ __extern_C key_t ftok(const char *, int); #endif /* Compiler */ #if !defined(__noop) && !defined(_MSC_VER) -#define __noop \ - do { \ +#define __noop \ + do { \ } while (0) #endif /* __noop */ -#if defined(__fallthrough) && \ - (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) +#if defined(__fallthrough) && (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) #undef __fallthrough #endif /* __fallthrough workaround for MinGW */ #ifndef __fallthrough -#if defined(__cplusplus) && (__has_cpp_attribute(fallthrough) && \ - (!defined(__clang__) || __clang__ > 4)) || \ +#if defined(__cplusplus) && (__has_cpp_attribute(fallthrough) && (!defined(__clang__) || __clang__ > 4)) || \ __cplusplus >= 201703L #define __fallthrough [[fallthrough]] #elif __GNUC_PREREQ(8, 0) && defined(__cplusplus) && __cplusplus >= 201103L #define __fallthrough [[fallthrough]] -#elif __GNUC_PREREQ(7, 0) && \ - (!defined(__LCC__) || (__LCC__ == 124 && __LCC_MINOR__ >= 12) || \ - (__LCC__ == 125 && __LCC_MINOR__ >= 5) || (__LCC__ >= 126)) +#elif __GNUC_PREREQ(7, 0) && (!defined(__LCC__) || (__LCC__ == 124 && __LCC_MINOR__ >= 12) || \ + (__LCC__ == 125 && __LCC_MINOR__ >= 5) || (__LCC__ >= 126)) #define __fallthrough __attribute__((__fallthrough__)) -#elif defined(__clang__) && defined(__cplusplus) && __cplusplus >= 201103L && \ - __has_feature(cxx_attributes) && __has_warning("-Wimplicit-fallthrough") +#elif defined(__clang__) && defined(__cplusplus) && __cplusplus >= 201103L && __has_feature(cxx_attributes) && \ + __has_warning("-Wimplicit-fallthrough") #define __fallthrough [[clang::fallthrough]] #else #define __fallthrough @@ -643,8 +644,8 @@ __extern_C key_t ftok(const char *, int); #elif defined(_MSC_VER) #define __unreachable() __assume(0) #else -#define __unreachable() \ - do { \ +#define __unreachable() \ + do { \ } while (1) #endif #endif /* __unreachable */ @@ -653,9 +654,9 @@ __extern_C key_t ftok(const char *, int); #if defined(__GNUC__) || defined(__clang__) || __has_builtin(__builtin_prefetch) #define __prefetch(ptr) __builtin_prefetch(ptr) #else -#define __prefetch(ptr) \ - do { \ - (void)(ptr); \ +#define __prefetch(ptr) \ + do { \ + (void)(ptr); \ } while (0) #endif #endif /* __prefetch */ @@ -665,11 +666,11 @@ __extern_C key_t ftok(const char *, int); #endif /* offsetof */ #ifndef container_of -#define container_of(ptr, type, member) \ - ((type *)((char *)(ptr) - offsetof(type, member))) +#define container_of(ptr, type, member) ((type *)((char *)(ptr) - offsetof(type, member))) #endif /* container_of */ /*----------------------------------------------------------------------------*/ +/* useful attributes */ #ifndef __always_inline #if defined(__GNUC__) || __has_attribute(__always_inline__) @@ -737,8 +738,7 @@ __extern_C key_t ftok(const char *, int); #ifndef __hot #if defined(__OPTIMIZE__) -#if defined(__clang__) && !__has_attribute(__hot__) && \ - __has_attribute(__section__) && \ +#if defined(__clang__) && !__has_attribute(__hot__) && __has_attribute(__section__) && \ (defined(__linux__) || defined(__gnu_linux__)) /* just put frequently used functions in separate section */ #define __hot __attribute__((__section__("text.hot"))) __optimize("O3") @@ -754,8 +754,7 @@ __extern_C key_t ftok(const char *, int); #ifndef __cold #if defined(__OPTIMIZE__) -#if defined(__clang__) && !__has_attribute(__cold__) && \ - __has_attribute(__section__) && \ +#if defined(__clang__) && !__has_attribute(__cold__) && __has_attribute(__section__) && \ (defined(__linux__) || defined(__gnu_linux__)) /* just put infrequently used functions in separate section */ #define __cold __attribute__((__section__("text.unlikely"))) __optimize("Os") @@ -778,8 +777,7 @@ __extern_C key_t ftok(const char *, int); #endif /* __flatten */ #ifndef likely -#if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && \ - !defined(__COVERITY__) +#if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__) #define likely(cond) __builtin_expect(!!(cond), 1) #else #define likely(x) (!!(x)) @@ -787,8 +785,7 @@ __extern_C key_t ftok(const char *, int); #endif /* likely */ #ifndef unlikely -#if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && \ - !defined(__COVERITY__) +#if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__) #define unlikely(cond) __builtin_expect(!!(cond), 0) #else #define unlikely(x) (!!(x)) @@ -803,29 +800,41 @@ __extern_C key_t ftok(const char *, int); #endif #endif /* __anonymous_struct_extension__ */ -#ifndef expect_with_probability -#if defined(__builtin_expect_with_probability) || \ - __has_builtin(__builtin_expect_with_probability) || __GNUC_PREREQ(9, 0) -#define expect_with_probability(expr, value, prob) \ - __builtin_expect_with_probability(expr, value, prob) -#else -#define expect_with_probability(expr, value, prob) (expr) -#endif -#endif /* expect_with_probability */ - #ifndef MDBX_WEAK_IMPORT_ATTRIBUTE #ifdef WEAK_IMPORT_ATTRIBUTE #define MDBX_WEAK_IMPORT_ATTRIBUTE WEAK_IMPORT_ATTRIBUTE #elif __has_attribute(__weak__) && __has_attribute(__weak_import__) #define MDBX_WEAK_IMPORT_ATTRIBUTE __attribute__((__weak__, __weak_import__)) -#elif __has_attribute(__weak__) || \ - (defined(__GNUC__) && __GNUC__ >= 4 && defined(__ELF__)) +#elif __has_attribute(__weak__) || (defined(__GNUC__) && __GNUC__ >= 4 && defined(__ELF__)) #define MDBX_WEAK_IMPORT_ATTRIBUTE __attribute__((__weak__)) #else #define MDBX_WEAK_IMPORT_ATTRIBUTE #endif #endif /* MDBX_WEAK_IMPORT_ATTRIBUTE */ +#if !defined(__thread) && (defined(_MSC_VER) || defined(__DMC__)) +#define __thread __declspec(thread) +#endif /* __thread */ + +#ifndef MDBX_EXCLUDE_FOR_GPROF +#ifdef ENABLE_GPROF +#define MDBX_EXCLUDE_FOR_GPROF __attribute__((__no_instrument_function__, __no_profile_instrument_function__)) +#else +#define MDBX_EXCLUDE_FOR_GPROF +#endif /* ENABLE_GPROF */ +#endif /* MDBX_EXCLUDE_FOR_GPROF */ + +/*----------------------------------------------------------------------------*/ + +#ifndef expect_with_probability +#if defined(__builtin_expect_with_probability) || __has_builtin(__builtin_expect_with_probability) || \ + __GNUC_PREREQ(9, 0) +#define expect_with_probability(expr, value, prob) __builtin_expect_with_probability(expr, value, prob) +#else +#define expect_with_probability(expr, value, prob) (expr) +#endif +#endif /* expect_with_probability */ + #ifndef MDBX_GOOFY_MSVC_STATIC_ANALYZER #ifdef _PREFAST_ #define MDBX_GOOFY_MSVC_STATIC_ANALYZER 1 @@ -837,20 +846,27 @@ __extern_C key_t ftok(const char *, int); #if MDBX_GOOFY_MSVC_STATIC_ANALYZER || (defined(_MSC_VER) && _MSC_VER > 1919) #define MDBX_ANALYSIS_ASSUME(expr) __analysis_assume(expr) #ifdef _PREFAST_ -#define MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(warn_id) \ - __pragma(prefast(suppress : warn_id)) +#define MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(warn_id) __pragma(prefast(suppress : warn_id)) #else -#define MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(warn_id) \ - __pragma(warning(suppress : warn_id)) +#define MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(warn_id) __pragma(warning(suppress : warn_id)) #endif #else #define MDBX_ANALYSIS_ASSUME(expr) assert(expr) #define MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(warn_id) #endif /* MDBX_GOOFY_MSVC_STATIC_ANALYZER */ +#ifndef FLEXIBLE_ARRAY_MEMBERS +#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || (!defined(__cplusplus) && defined(_MSC_VER)) +#define FLEXIBLE_ARRAY_MEMBERS 1 +#else +#define FLEXIBLE_ARRAY_MEMBERS 0 +#endif +#endif /* FLEXIBLE_ARRAY_MEMBERS */ + /*----------------------------------------------------------------------------*/ +/* Valgrind and Address Sanitizer */ -#if defined(MDBX_USE_VALGRIND) +#if defined(ENABLE_MEMCHECK) #include #ifndef VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE /* LY: available since Valgrind 3.10 */ @@ -872,7 +888,7 @@ __extern_C key_t ftok(const char *, int); #define VALGRIND_CHECK_MEM_IS_ADDRESSABLE(a, s) (0) #define VALGRIND_CHECK_MEM_IS_DEFINED(a, s) (0) #define RUNNING_ON_VALGRIND (0) -#endif /* MDBX_USE_VALGRIND */ +#endif /* ENABLE_MEMCHECK */ #ifdef __SANITIZE_ADDRESS__ #include @@ -899,8 +915,7 @@ template char (&__ArraySizeHelper(T (&array)[N]))[N]; #define CONCAT(a, b) a##b #define XCONCAT(a, b) CONCAT(a, b) -#define MDBX_TETRAD(a, b, c, d) \ - ((uint32_t)(a) << 24 | (uint32_t)(b) << 16 | (uint32_t)(c) << 8 | (d)) +#define MDBX_TETRAD(a, b, c, d) ((uint32_t)(a) << 24 | (uint32_t)(b) << 16 | (uint32_t)(c) << 8 | (d)) #define MDBX_STRING_TETRAD(str) MDBX_TETRAD(str[0], str[1], str[2], str[3]) @@ -914,14 +929,13 @@ template char (&__ArraySizeHelper(T (&array)[N]))[N]; #elif defined(_MSC_VER) #include #define STATIC_ASSERT_MSG(expr, msg) _STATIC_ASSERT(expr) -#elif (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) || \ - __has_feature(c_static_assert) +#elif (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) || __has_feature(c_static_assert) #define STATIC_ASSERT_MSG(expr, msg) _Static_assert(expr, msg) #else -#define STATIC_ASSERT_MSG(expr, msg) \ - switch (0) { \ - case 0: \ - case (expr):; \ +#define STATIC_ASSERT_MSG(expr, msg) \ + switch (0) { \ + case 0: \ + case (expr):; \ } #endif #endif /* STATIC_ASSERT */ @@ -930,42 +944,37 @@ template char (&__ArraySizeHelper(T (&array)[N]))[N]; #define STATIC_ASSERT(expr) STATIC_ASSERT_MSG(expr, #expr) #endif -#ifndef __Wpedantic_format_voidptr -MDBX_MAYBE_UNUSED MDBX_PURE_FUNCTION static __inline const void * -__Wpedantic_format_voidptr(const void *ptr) { - return ptr; -} -#define __Wpedantic_format_voidptr(ARG) __Wpedantic_format_voidptr(ARG) -#endif /* __Wpedantic_format_voidptr */ +/*----------------------------------------------------------------------------*/ -#if defined(__GNUC__) && !__GNUC_PREREQ(4, 2) -/* Actually libmdbx was not tested with compilers older than GCC 4.2. - * But you could ignore this warning at your own risk. - * In such case please don't rise up an issues related ONLY to old compilers. - */ -#warning "libmdbx required GCC >= 4.2" -#endif +#if defined(_MSC_VER) && _MSC_VER >= 1900 +/* LY: MSVC 2015/2017/2019 has buggy/inconsistent PRIuPTR/PRIxPTR macros + * for internal format-args checker. */ +#undef PRIuPTR +#undef PRIiPTR +#undef PRIdPTR +#undef PRIxPTR +#define PRIuPTR "Iu" +#define PRIiPTR "Ii" +#define PRIdPTR "Id" +#define PRIxPTR "Ix" +#define PRIuSIZE "zu" +#define PRIiSIZE "zi" +#define PRIdSIZE "zd" +#define PRIxSIZE "zx" +#endif /* fix PRI*PTR for _MSC_VER */ -#if defined(__clang__) && !__CLANG_PREREQ(3, 8) -/* Actually libmdbx was not tested with CLANG older than 3.8. - * But you could ignore this warning at your own risk. - * In such case please don't rise up an issues related ONLY to old compilers. - */ -#warning "libmdbx required CLANG >= 3.8" -#endif +#ifndef PRIuSIZE +#define PRIuSIZE PRIuPTR +#define PRIiSIZE PRIiPTR +#define PRIdSIZE PRIdPTR +#define PRIxSIZE PRIxPTR +#endif /* PRI*SIZE macros for MSVC */ -#if defined(__GLIBC__) && !__GLIBC_PREREQ(2, 12) -/* Actually libmdbx was not tested with something older than glibc 2.12. - * But you could ignore this warning at your own risk. - * In such case please don't rise up an issues related ONLY to old systems. - */ -#warning "libmdbx was only tested with GLIBC >= 2.12." +#ifdef _MSC_VER +#pragma warning(pop) #endif -#ifdef __SANITIZE_THREAD__ -#warning \ - "libmdbx don't compatible with ThreadSanitizer, you will get a lot of false-positive issues." -#endif /* __SANITIZE_THREAD__ */ +/*----------------------------------------------------------------------------*/ #if __has_warning("-Wnested-anon-types") #if defined(__clang__) @@ -1002,80 +1011,34 @@ __Wpedantic_format_voidptr(const void *ptr) { #endif #endif /* -Walignment-reduction-ignored */ -#ifndef MDBX_EXCLUDE_FOR_GPROF -#ifdef ENABLE_GPROF -#define MDBX_EXCLUDE_FOR_GPROF \ - __attribute__((__no_instrument_function__, \ - __no_profile_instrument_function__)) +#ifdef xMDBX_ALLOY +/* Amalgamated build */ +#define MDBX_INTERNAL static #else -#define MDBX_EXCLUDE_FOR_GPROF -#endif /* ENABLE_GPROF */ -#endif /* MDBX_EXCLUDE_FOR_GPROF */ - -#ifdef __cplusplus -extern "C" { -#endif +/* Non-amalgamated build */ +#define MDBX_INTERNAL +#endif /* xMDBX_ALLOY */ -/* https://en.wikipedia.org/wiki/Operating_system_abstraction_layer */ +#include "mdbx.h" -/* - * Copyright 2015-2024 Leonid Yuriev - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - */ +/*----------------------------------------------------------------------------*/ +/* Basic constants and types */ +typedef struct iov_ctx iov_ctx_t; +/// /*----------------------------------------------------------------------------*/ -/* C11 Atomics */ - -#if defined(__cplusplus) && !defined(__STDC_NO_ATOMICS__) && __has_include() -#include -#define MDBX_HAVE_C11ATOMICS -#elif !defined(__cplusplus) && \ - (__STDC_VERSION__ >= 201112L || __has_extension(c_atomic)) && \ - !defined(__STDC_NO_ATOMICS__) && \ - (__GNUC_PREREQ(4, 9) || __CLANG_PREREQ(3, 8) || \ - !(defined(__GNUC__) || defined(__clang__))) -#include -#define MDBX_HAVE_C11ATOMICS -#elif defined(__GNUC__) || defined(__clang__) -#elif defined(_MSC_VER) -#pragma warning(disable : 4163) /* 'xyz': not available as an intrinsic */ -#pragma warning(disable : 4133) /* 'function': incompatible types - from \ - 'size_t' to 'LONGLONG' */ -#pragma warning(disable : 4244) /* 'return': conversion from 'LONGLONG' to \ - 'std::size_t', possible loss of data */ -#pragma warning(disable : 4267) /* 'function': conversion from 'size_t' to \ - 'long', possible loss of data */ -#pragma intrinsic(_InterlockedExchangeAdd, _InterlockedCompareExchange) -#pragma intrinsic(_InterlockedExchangeAdd64, _InterlockedCompareExchange64) -#elif defined(__APPLE__) -#include -#else -#error FIXME atomic-ops -#endif - -/*----------------------------------------------------------------------------*/ -/* Memory/Compiler barriers, cache coherence */ +/* Memory/Compiler barriers, cache coherence */ #if __has_include() #include -#elif defined(__mips) || defined(__mips__) || defined(__mips64) || \ - defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \ - defined(__MWERKS__) || defined(__sgi) +#elif defined(__mips) || defined(__mips__) || defined(__mips64) || defined(__mips64__) || defined(_M_MRX000) || \ + defined(_MIPS_) || defined(__MWERKS__) || defined(__sgi) /* MIPS should have explicit cache control */ #include #endif -MDBX_MAYBE_UNUSED static __inline void osal_compiler_barrier(void) { +MDBX_MAYBE_UNUSED static inline void osal_compiler_barrier(void) { #if defined(__clang__) || defined(__GNUC__) __asm__ __volatile__("" ::: "memory"); #elif defined(_MSC_VER) @@ -1084,18 +1047,16 @@ MDBX_MAYBE_UNUSED static __inline void osal_compiler_barrier(void) { __memory_barrier(); #elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) __compiler_barrier(); -#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ - (defined(HP_IA64) || defined(__ia64)) +#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && (defined(HP_IA64) || defined(__ia64)) _Asm_sched_fence(/* LY: no-arg meaning 'all expect ALU', e.g. 0x3D3D */); -#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \ - defined(__ppc64__) || defined(__powerpc64__) +#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || defined(__ppc64__) || defined(__powerpc64__) __fence(); #else #error "Could not guess the kind of compiler, please report to us." #endif } -MDBX_MAYBE_UNUSED static __inline void osal_memory_barrier(void) { +MDBX_MAYBE_UNUSED static inline void osal_memory_barrier(void) { #ifdef MDBX_HAVE_C11ATOMICS atomic_thread_fence(memory_order_seq_cst); #elif defined(__ATOMIC_SEQ_CST) @@ -1116,11 +1077,9 @@ MDBX_MAYBE_UNUSED static __inline void osal_memory_barrier(void) { #endif #elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) __machine_rw_barrier(); -#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ - (defined(HP_IA64) || defined(__ia64)) +#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && (defined(HP_IA64) || defined(__ia64)) _Asm_mf(); -#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \ - defined(__ppc64__) || defined(__powerpc64__) +#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || defined(__ppc64__) || defined(__powerpc64__) __lwsync(); #else #error "Could not guess the kind of compiler, please report to us." @@ -1135,7 +1094,7 @@ MDBX_MAYBE_UNUSED static __inline void osal_memory_barrier(void) { #define HAVE_SYS_TYPES_H typedef HANDLE osal_thread_t; typedef unsigned osal_thread_key_t; -#define MAP_FAILED NULL +#define MAP_FAILED nullptr #define HIGH_DWORD(v) ((DWORD)((sizeof(v) > 4) ? ((uint64_t)(v) >> 32) : 0)) #define THREAD_CALL WINAPI #define THREAD_RESULT DWORD @@ -1147,15 +1106,13 @@ typedef CRITICAL_SECTION osal_fastmutex_t; #if !defined(_MSC_VER) && !defined(__try) #define __try -#define __except(COND) if (false) +#define __except(COND) if (/* (void)(COND), */ false) #endif /* stub for MSVC's __try/__except */ #if MDBX_WITHOUT_MSVC_CRT #ifndef osal_malloc -static inline void *osal_malloc(size_t bytes) { - return HeapAlloc(GetProcessHeap(), 0, bytes); -} +static inline void *osal_malloc(size_t bytes) { return HeapAlloc(GetProcessHeap(), 0, bytes); } #endif /* osal_malloc */ #ifndef osal_calloc @@ -1166,8 +1123,7 @@ static inline void *osal_calloc(size_t nelem, size_t size) { #ifndef osal_realloc static inline void *osal_realloc(void *ptr, size_t bytes) { - return ptr ? HeapReAlloc(GetProcessHeap(), 0, ptr, bytes) - : HeapAlloc(GetProcessHeap(), 0, bytes); + return ptr ? HeapReAlloc(GetProcessHeap(), 0, ptr, bytes) : HeapAlloc(GetProcessHeap(), 0, bytes); } #endif /* osal_realloc */ @@ -1213,29 +1169,16 @@ typedef pthread_mutex_t osal_fastmutex_t; #endif /* Platform */ #if __GLIBC_PREREQ(2, 12) || defined(__FreeBSD__) || defined(malloc_usable_size) -/* malloc_usable_size() already provided */ +#define osal_malloc_usable_size(ptr) malloc_usable_size(ptr) #elif defined(__APPLE__) -#define malloc_usable_size(ptr) malloc_size(ptr) +#define osal_malloc_usable_size(ptr) malloc_size(ptr) #elif defined(_MSC_VER) && !MDBX_WITHOUT_MSVC_CRT -#define malloc_usable_size(ptr) _msize(ptr) -#endif /* malloc_usable_size */ +#define osal_malloc_usable_size(ptr) _msize(ptr) +#endif /* osal_malloc_usable_size */ /*----------------------------------------------------------------------------*/ /* OS abstraction layer stuff */ -MDBX_INTERNAL_VAR unsigned sys_pagesize; -MDBX_MAYBE_UNUSED MDBX_INTERNAL_VAR unsigned sys_pagesize_ln2, - sys_allocation_granularity; - -/* Get the size of a memory page for the system. - * This is the basic size that the platform's memory manager uses, and is - * fundamental to the use of memory-mapped files. */ -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline size_t -osal_syspagesize(void) { - assert(sys_pagesize > 0 && (sys_pagesize & (sys_pagesize - 1)) == 0); - return sys_pagesize; -} - #if defined(_WIN32) || defined(_WIN64) typedef wchar_t pathchar_t; #define MDBX_PRIsPATH "ls" @@ -1247,7 +1190,7 @@ typedef char pathchar_t; typedef struct osal_mmap { union { void *base; - struct MDBX_lockinfo *lck; + struct shared_lck *lck; }; mdbx_filehandle_t fd; size_t limit; /* mapping length, but NOT a size of file nor DB */ @@ -1258,25 +1201,6 @@ typedef struct osal_mmap { #endif } osal_mmap_t; -typedef union bin128 { - __anonymous_struct_extension__ struct { - uint64_t x, y; - }; - __anonymous_struct_extension__ struct { - uint32_t a, b, c, d; - }; -} bin128_t; - -#if defined(_WIN32) || defined(_WIN64) -typedef union osal_srwlock { - __anonymous_struct_extension__ struct { - long volatile readerCount; - long volatile writerCount; - }; - RTL_SRWLOCK native; -} osal_srwlock_t; -#endif /* Windows */ - #ifndef MDBX_HAVE_PWRITEV #if defined(_WIN32) || defined(_WIN64) @@ -1285,14 +1209,21 @@ typedef union osal_srwlock { #elif defined(__ANDROID_API__) #if __ANDROID_API__ < 24 +/* https://android-developers.googleblog.com/2017/09/introducing-android-native-development.html + * https://android.googlesource.com/platform/bionic/+/master/docs/32-bit-abi.md */ #define MDBX_HAVE_PWRITEV 0 +#if defined(_FILE_OFFSET_BITS) && _FILE_OFFSET_BITS != MDBX_WORDBITS +#error "_FILE_OFFSET_BITS != MDBX_WORDBITS and __ANDROID_API__ < 24" (_FILE_OFFSET_BITS != MDBX_WORDBITS) +#elif defined(__FILE_OFFSET_BITS) && __FILE_OFFSET_BITS != MDBX_WORDBITS +#error "__FILE_OFFSET_BITS != MDBX_WORDBITS and __ANDROID_API__ < 24" (__FILE_OFFSET_BITS != MDBX_WORDBITS) +#endif #else #define MDBX_HAVE_PWRITEV 1 #endif #elif defined(__APPLE__) || defined(__MACH__) || defined(_DARWIN_C_SOURCE) -#if defined(MAC_OS_X_VERSION_MIN_REQUIRED) && defined(MAC_OS_VERSION_11_0) && \ +#if defined(MAC_OS_X_VERSION_MIN_REQUIRED) && defined(MAC_OS_VERSION_11_0) && \ MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_VERSION_11_0 /* FIXME: add checks for IOS versions, etc */ #define MDBX_HAVE_PWRITEV 1 @@ -1310,20 +1241,20 @@ typedef union osal_srwlock { typedef struct ior_item { #if defined(_WIN32) || defined(_WIN64) OVERLAPPED ov; -#define ior_svg_gap4terminator 1 +#define ior_sgv_gap4terminator 1 #define ior_sgv_element FILE_SEGMENT_ELEMENT #else size_t offset; #if MDBX_HAVE_PWRITEV size_t sgvcnt; -#define ior_svg_gap4terminator 0 +#define ior_sgv_gap4terminator 0 #define ior_sgv_element struct iovec #endif /* MDBX_HAVE_PWRITEV */ #endif /* !Windows */ union { MDBX_val single; #if defined(ior_sgv_element) - ior_sgv_element sgv[1 + ior_svg_gap4terminator]; + ior_sgv_element sgv[1 + ior_sgv_gap4terminator]; #endif /* ior_sgv_element */ }; } ior_item_t; @@ -1359,45 +1290,33 @@ typedef struct osal_ioring { char *boundary; } osal_ioring_t; -#ifndef __cplusplus - /* Actually this is not ioring for now, but on the way. */ -MDBX_INTERNAL_FUNC int osal_ioring_create(osal_ioring_t * +MDBX_INTERNAL int osal_ioring_create(osal_ioring_t * #if defined(_WIN32) || defined(_WIN64) - , - bool enable_direct, - mdbx_filehandle_t overlapped_fd + , + bool enable_direct, mdbx_filehandle_t overlapped_fd #endif /* Windows */ ); -MDBX_INTERNAL_FUNC int osal_ioring_resize(osal_ioring_t *, size_t items); -MDBX_INTERNAL_FUNC void osal_ioring_destroy(osal_ioring_t *); -MDBX_INTERNAL_FUNC void osal_ioring_reset(osal_ioring_t *); -MDBX_INTERNAL_FUNC int osal_ioring_add(osal_ioring_t *ctx, const size_t offset, - void *data, const size_t bytes); +MDBX_INTERNAL int osal_ioring_resize(osal_ioring_t *, size_t items); +MDBX_INTERNAL void osal_ioring_destroy(osal_ioring_t *); +MDBX_INTERNAL void osal_ioring_reset(osal_ioring_t *); +MDBX_INTERNAL int osal_ioring_add(osal_ioring_t *ctx, const size_t offset, void *data, const size_t bytes); typedef struct osal_ioring_write_result { int err; unsigned wops; } osal_ioring_write_result_t; -MDBX_INTERNAL_FUNC osal_ioring_write_result_t -osal_ioring_write(osal_ioring_t *ior, mdbx_filehandle_t fd); +MDBX_INTERNAL osal_ioring_write_result_t osal_ioring_write(osal_ioring_t *ior, mdbx_filehandle_t fd); -typedef struct iov_ctx iov_ctx_t; -MDBX_INTERNAL_FUNC void osal_ioring_walk( - osal_ioring_t *ior, iov_ctx_t *ctx, - void (*callback)(iov_ctx_t *ctx, size_t offset, void *data, size_t bytes)); +MDBX_INTERNAL void osal_ioring_walk(osal_ioring_t *ior, iov_ctx_t *ctx, + void (*callback)(iov_ctx_t *ctx, size_t offset, void *data, size_t bytes)); -MDBX_MAYBE_UNUSED static inline unsigned -osal_ioring_left(const osal_ioring_t *ior) { - return ior->slots_left; -} +MDBX_MAYBE_UNUSED static inline unsigned osal_ioring_left(const osal_ioring_t *ior) { return ior->slots_left; } -MDBX_MAYBE_UNUSED static inline unsigned -osal_ioring_used(const osal_ioring_t *ior) { +MDBX_MAYBE_UNUSED static inline unsigned osal_ioring_used(const osal_ioring_t *ior) { return ior->allocated - ior->slots_left; } -MDBX_MAYBE_UNUSED static inline int -osal_ioring_prepare(osal_ioring_t *ior, size_t items, size_t bytes) { +MDBX_MAYBE_UNUSED static inline int osal_ioring_prepare(osal_ioring_t *ior, size_t items, size_t bytes) { items = (items > 32) ? items : 32; #if defined(_WIN32) || defined(_WIN64) if (ior->direct) { @@ -1416,14 +1335,12 @@ osal_ioring_prepare(osal_ioring_t *ior, size_t items, size_t bytes) { /*----------------------------------------------------------------------------*/ /* libc compatibility stuff */ -#if (!defined(__GLIBC__) && __GLIBC_PREREQ(2, 1)) && \ - (defined(_GNU_SOURCE) || defined(_BSD_SOURCE)) +#if (!defined(__GLIBC__) && __GLIBC_PREREQ(2, 1)) && (defined(_GNU_SOURCE) || defined(_BSD_SOURCE)) #define osal_asprintf asprintf #define osal_vasprintf vasprintf #else -MDBX_MAYBE_UNUSED MDBX_INTERNAL_FUNC - MDBX_PRINTF_ARGS(2, 3) int osal_asprintf(char **strp, const char *fmt, ...); -MDBX_INTERNAL_FUNC int osal_vasprintf(char **strp, const char *fmt, va_list ap); +MDBX_MAYBE_UNUSED MDBX_INTERNAL MDBX_PRINTF_ARGS(2, 3) int osal_asprintf(char **strp, const char *fmt, ...); +MDBX_INTERNAL int osal_vasprintf(char **strp, const char *fmt, va_list ap); #endif #if !defined(MADV_DODUMP) && defined(MADV_CORE) @@ -1434,8 +1351,7 @@ MDBX_INTERNAL_FUNC int osal_vasprintf(char **strp, const char *fmt, va_list ap); #define MADV_DONTDUMP MADV_NOCORE #endif /* MADV_NOCORE -> MADV_DONTDUMP */ -MDBX_MAYBE_UNUSED MDBX_INTERNAL_FUNC void osal_jitter(bool tiny); -MDBX_MAYBE_UNUSED static __inline void jitter4testing(bool tiny); +MDBX_MAYBE_UNUSED MDBX_INTERNAL void osal_jitter(bool tiny); /* max bytes to write in one call */ #if defined(_WIN64) @@ -1445,14 +1361,12 @@ MDBX_MAYBE_UNUSED static __inline void jitter4testing(bool tiny); #else #define MAX_WRITE UINT32_C(0x3f000000) -#if defined(F_GETLK64) && defined(F_SETLK64) && defined(F_SETLKW64) && \ - !defined(__ANDROID_API__) +#if defined(F_GETLK64) && defined(F_SETLK64) && defined(F_SETLKW64) && !defined(__ANDROID_API__) #define MDBX_F_SETLK F_SETLK64 #define MDBX_F_SETLKW F_SETLKW64 #define MDBX_F_GETLK F_GETLK64 -#if (__GLIBC_PREREQ(2, 28) && \ - (defined(__USE_LARGEFILE64) || defined(__LARGEFILE64_SOURCE) || \ - defined(_USE_LARGEFILE64) || defined(_LARGEFILE64_SOURCE))) || \ +#if (__GLIBC_PREREQ(2, 28) && (defined(__USE_LARGEFILE64) || defined(__LARGEFILE64_SOURCE) || \ + defined(_USE_LARGEFILE64) || defined(_LARGEFILE64_SOURCE))) || \ defined(fcntl64) #define MDBX_FCNTL fcntl64 #else @@ -1470,8 +1384,7 @@ MDBX_MAYBE_UNUSED static __inline void jitter4testing(bool tiny); #define MDBX_STRUCT_FLOCK struct flock #endif /* MDBX_F_SETLK, MDBX_F_SETLKW, MDBX_F_GETLK */ -#if defined(F_OFD_SETLK64) && defined(F_OFD_SETLKW64) && \ - defined(F_OFD_GETLK64) && !defined(__ANDROID_API__) +#if defined(F_OFD_SETLK64) && defined(F_OFD_SETLKW64) && defined(F_OFD_GETLK64) && !defined(__ANDROID_API__) #define MDBX_F_OFD_SETLK F_OFD_SETLK64 #define MDBX_F_OFD_SETLKW F_OFD_SETLKW64 #define MDBX_F_OFD_GETLK F_OFD_GETLK64 @@ -1480,23 +1393,17 @@ MDBX_MAYBE_UNUSED static __inline void jitter4testing(bool tiny); #define MDBX_F_OFD_SETLKW F_OFD_SETLKW #define MDBX_F_OFD_GETLK F_OFD_GETLK #ifndef OFF_T_MAX -#define OFF_T_MAX \ - (((sizeof(off_t) > 4) ? INT64_MAX : INT32_MAX) & ~(size_t)0xFffff) +#define OFF_T_MAX (((sizeof(off_t) > 4) ? INT64_MAX : INT32_MAX) & ~(size_t)0xFffff) #endif /* OFF_T_MAX */ #endif /* MDBX_F_OFD_SETLK64, MDBX_F_OFD_SETLKW64, MDBX_F_OFD_GETLK64 */ -#endif - -#if defined(__linux__) || defined(__gnu_linux__) -MDBX_INTERNAL_VAR uint32_t linux_kernel_version; -MDBX_INTERNAL_VAR bool mdbx_RunningOnWSL1 /* Windows Subsystem 1 for Linux */; -#endif /* Linux */ +#endif /* !Windows */ #ifndef osal_strdup LIBMDBX_API char *osal_strdup(const char *str); #endif -MDBX_MAYBE_UNUSED static __inline int osal_get_errno(void) { +MDBX_MAYBE_UNUSED static inline int osal_get_errno(void) { #if defined(_WIN32) || defined(_WIN64) DWORD rc = GetLastError(); #else @@ -1506,40 +1413,32 @@ MDBX_MAYBE_UNUSED static __inline int osal_get_errno(void) { } #ifndef osal_memalign_alloc -MDBX_INTERNAL_FUNC int osal_memalign_alloc(size_t alignment, size_t bytes, - void **result); +MDBX_INTERNAL int osal_memalign_alloc(size_t alignment, size_t bytes, void **result); #endif #ifndef osal_memalign_free -MDBX_INTERNAL_FUNC void osal_memalign_free(void *ptr); -#endif - -MDBX_INTERNAL_FUNC int osal_condpair_init(osal_condpair_t *condpair); -MDBX_INTERNAL_FUNC int osal_condpair_lock(osal_condpair_t *condpair); -MDBX_INTERNAL_FUNC int osal_condpair_unlock(osal_condpair_t *condpair); -MDBX_INTERNAL_FUNC int osal_condpair_signal(osal_condpair_t *condpair, - bool part); -MDBX_INTERNAL_FUNC int osal_condpair_wait(osal_condpair_t *condpair, bool part); -MDBX_INTERNAL_FUNC int osal_condpair_destroy(osal_condpair_t *condpair); - -MDBX_INTERNAL_FUNC int osal_fastmutex_init(osal_fastmutex_t *fastmutex); -MDBX_INTERNAL_FUNC int osal_fastmutex_acquire(osal_fastmutex_t *fastmutex); -MDBX_INTERNAL_FUNC int osal_fastmutex_release(osal_fastmutex_t *fastmutex); -MDBX_INTERNAL_FUNC int osal_fastmutex_destroy(osal_fastmutex_t *fastmutex); - -MDBX_INTERNAL_FUNC int osal_pwritev(mdbx_filehandle_t fd, struct iovec *iov, - size_t sgvcnt, uint64_t offset); -MDBX_INTERNAL_FUNC int osal_pread(mdbx_filehandle_t fd, void *buf, size_t count, - uint64_t offset); -MDBX_INTERNAL_FUNC int osal_pwrite(mdbx_filehandle_t fd, const void *buf, - size_t count, uint64_t offset); -MDBX_INTERNAL_FUNC int osal_write(mdbx_filehandle_t fd, const void *buf, - size_t count); - -MDBX_INTERNAL_FUNC int -osal_thread_create(osal_thread_t *thread, - THREAD_RESULT(THREAD_CALL *start_routine)(void *), - void *arg); -MDBX_INTERNAL_FUNC int osal_thread_join(osal_thread_t thread); +MDBX_INTERNAL void osal_memalign_free(void *ptr); +#endif + +MDBX_INTERNAL int osal_condpair_init(osal_condpair_t *condpair); +MDBX_INTERNAL int osal_condpair_lock(osal_condpair_t *condpair); +MDBX_INTERNAL int osal_condpair_unlock(osal_condpair_t *condpair); +MDBX_INTERNAL int osal_condpair_signal(osal_condpair_t *condpair, bool part); +MDBX_INTERNAL int osal_condpair_wait(osal_condpair_t *condpair, bool part); +MDBX_INTERNAL int osal_condpair_destroy(osal_condpair_t *condpair); + +MDBX_INTERNAL int osal_fastmutex_init(osal_fastmutex_t *fastmutex); +MDBX_INTERNAL int osal_fastmutex_acquire(osal_fastmutex_t *fastmutex); +MDBX_INTERNAL int osal_fastmutex_release(osal_fastmutex_t *fastmutex); +MDBX_INTERNAL int osal_fastmutex_destroy(osal_fastmutex_t *fastmutex); + +MDBX_INTERNAL int osal_pwritev(mdbx_filehandle_t fd, struct iovec *iov, size_t sgvcnt, uint64_t offset); +MDBX_INTERNAL int osal_pread(mdbx_filehandle_t fd, void *buf, size_t count, uint64_t offset); +MDBX_INTERNAL int osal_pwrite(mdbx_filehandle_t fd, const void *buf, size_t count, uint64_t offset); +MDBX_INTERNAL int osal_write(mdbx_filehandle_t fd, const void *buf, size_t count); + +MDBX_INTERNAL int osal_thread_create(osal_thread_t *thread, THREAD_RESULT(THREAD_CALL *start_routine)(void *), + void *arg); +MDBX_INTERNAL int osal_thread_join(osal_thread_t thread); enum osal_syncmode_bits { MDBX_SYNC_NONE = 0, @@ -1549,11 +1448,10 @@ enum osal_syncmode_bits { MDBX_SYNC_IODQ = 8 }; -MDBX_INTERNAL_FUNC int osal_fsync(mdbx_filehandle_t fd, - const enum osal_syncmode_bits mode_bits); -MDBX_INTERNAL_FUNC int osal_ftruncate(mdbx_filehandle_t fd, uint64_t length); -MDBX_INTERNAL_FUNC int osal_fseek(mdbx_filehandle_t fd, uint64_t pos); -MDBX_INTERNAL_FUNC int osal_filesize(mdbx_filehandle_t fd, uint64_t *length); +MDBX_INTERNAL int osal_fsync(mdbx_filehandle_t fd, const enum osal_syncmode_bits mode_bits); +MDBX_INTERNAL int osal_ftruncate(mdbx_filehandle_t fd, uint64_t length); +MDBX_INTERNAL int osal_fseek(mdbx_filehandle_t fd, uint64_t pos); +MDBX_INTERNAL int osal_filesize(mdbx_filehandle_t fd, uint64_t *length); enum osal_openfile_purpose { MDBX_OPEN_DXB_READ, @@ -1568,7 +1466,7 @@ enum osal_openfile_purpose { MDBX_OPEN_DELETE }; -MDBX_MAYBE_UNUSED static __inline bool osal_isdirsep(pathchar_t c) { +MDBX_MAYBE_UNUSED static inline bool osal_isdirsep(pathchar_t c) { return #if defined(_WIN32) || defined(_WIN64) c == '\\' || @@ -1576,50 +1474,39 @@ MDBX_MAYBE_UNUSED static __inline bool osal_isdirsep(pathchar_t c) { c == '/'; } -MDBX_INTERNAL_FUNC bool osal_pathequal(const pathchar_t *l, const pathchar_t *r, - size_t len); -MDBX_INTERNAL_FUNC pathchar_t *osal_fileext(const pathchar_t *pathname, - size_t len); -MDBX_INTERNAL_FUNC int osal_fileexists(const pathchar_t *pathname); -MDBX_INTERNAL_FUNC int osal_openfile(const enum osal_openfile_purpose purpose, - const MDBX_env *env, - const pathchar_t *pathname, - mdbx_filehandle_t *fd, - mdbx_mode_t unix_mode_bits); -MDBX_INTERNAL_FUNC int osal_closefile(mdbx_filehandle_t fd); -MDBX_INTERNAL_FUNC int osal_removefile(const pathchar_t *pathname); -MDBX_INTERNAL_FUNC int osal_removedirectory(const pathchar_t *pathname); -MDBX_INTERNAL_FUNC int osal_is_pipe(mdbx_filehandle_t fd); -MDBX_INTERNAL_FUNC int osal_lockfile(mdbx_filehandle_t fd, bool wait); +MDBX_INTERNAL bool osal_pathequal(const pathchar_t *l, const pathchar_t *r, size_t len); +MDBX_INTERNAL pathchar_t *osal_fileext(const pathchar_t *pathname, size_t len); +MDBX_INTERNAL int osal_fileexists(const pathchar_t *pathname); +MDBX_INTERNAL int osal_openfile(const enum osal_openfile_purpose purpose, const MDBX_env *env, + const pathchar_t *pathname, mdbx_filehandle_t *fd, mdbx_mode_t unix_mode_bits); +MDBX_INTERNAL int osal_closefile(mdbx_filehandle_t fd); +MDBX_INTERNAL int osal_removefile(const pathchar_t *pathname); +MDBX_INTERNAL int osal_removedirectory(const pathchar_t *pathname); +MDBX_INTERNAL int osal_is_pipe(mdbx_filehandle_t fd); +MDBX_INTERNAL int osal_lockfile(mdbx_filehandle_t fd, bool wait); #define MMAP_OPTION_TRUNCATE 1 #define MMAP_OPTION_SEMAPHORE 2 -MDBX_INTERNAL_FUNC int osal_mmap(const int flags, osal_mmap_t *map, size_t size, - const size_t limit, const unsigned options); -MDBX_INTERNAL_FUNC int osal_munmap(osal_mmap_t *map); +MDBX_INTERNAL int osal_mmap(const int flags, osal_mmap_t *map, size_t size, const size_t limit, const unsigned options, + const pathchar_t *pathname4logging); +MDBX_INTERNAL int osal_munmap(osal_mmap_t *map); #define MDBX_MRESIZE_MAY_MOVE 0x00000100 #define MDBX_MRESIZE_MAY_UNMAP 0x00000200 -MDBX_INTERNAL_FUNC int osal_mresize(const int flags, osal_mmap_t *map, - size_t size, size_t limit); +MDBX_INTERNAL int osal_mresize(const int flags, osal_mmap_t *map, size_t size, size_t limit); #if defined(_WIN32) || defined(_WIN64) typedef struct { unsigned limit, count; HANDLE handles[31]; } mdbx_handle_array_t; -MDBX_INTERNAL_FUNC int -osal_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array); -MDBX_INTERNAL_FUNC int -osal_resume_threads_after_remap(mdbx_handle_array_t *array); +MDBX_INTERNAL int osal_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array); +MDBX_INTERNAL int osal_resume_threads_after_remap(mdbx_handle_array_t *array); #endif /* Windows */ -MDBX_INTERNAL_FUNC int osal_msync(const osal_mmap_t *map, size_t offset, - size_t length, - enum osal_syncmode_bits mode_bits); -MDBX_INTERNAL_FUNC int osal_check_fs_rdonly(mdbx_filehandle_t handle, - const pathchar_t *pathname, - int err); -MDBX_INTERNAL_FUNC int osal_check_fs_incore(mdbx_filehandle_t handle); - -MDBX_MAYBE_UNUSED static __inline uint32_t osal_getpid(void) { +MDBX_INTERNAL int osal_msync(const osal_mmap_t *map, size_t offset, size_t length, enum osal_syncmode_bits mode_bits); +MDBX_INTERNAL int osal_check_fs_rdonly(mdbx_filehandle_t handle, const pathchar_t *pathname, int err); +MDBX_INTERNAL int osal_check_fs_incore(mdbx_filehandle_t handle); +MDBX_INTERNAL int osal_check_fs_local(mdbx_filehandle_t handle, int flags); + +MDBX_MAYBE_UNUSED static inline uint32_t osal_getpid(void) { STATIC_ASSERT(sizeof(mdbx_pid_t) <= sizeof(uint32_t)); #if defined(_WIN32) || defined(_WIN64) return GetCurrentProcessId(); @@ -1629,7 +1516,7 @@ MDBX_MAYBE_UNUSED static __inline uint32_t osal_getpid(void) { #endif } -MDBX_MAYBE_UNUSED static __inline uintptr_t osal_thread_self(void) { +MDBX_MAYBE_UNUSED static inline uintptr_t osal_thread_self(void) { mdbx_tid_t thunk; STATIC_ASSERT(sizeof(uintptr_t) >= sizeof(thunk)); #if defined(_WIN32) || defined(_WIN64) @@ -1642,274 +1529,51 @@ MDBX_MAYBE_UNUSED static __inline uintptr_t osal_thread_self(void) { #if !defined(_WIN32) && !defined(_WIN64) #if defined(__ANDROID_API__) || defined(ANDROID) || defined(BIONIC) -MDBX_INTERNAL_FUNC int osal_check_tid4bionic(void); +MDBX_INTERNAL int osal_check_tid4bionic(void); #else -static __inline int osal_check_tid4bionic(void) { return 0; } +static inline int osal_check_tid4bionic(void) { return 0; } #endif /* __ANDROID_API__ || ANDROID) || BIONIC */ -MDBX_MAYBE_UNUSED static __inline int -osal_pthread_mutex_lock(pthread_mutex_t *mutex) { +MDBX_MAYBE_UNUSED static inline int osal_pthread_mutex_lock(pthread_mutex_t *mutex) { int err = osal_check_tid4bionic(); return unlikely(err) ? err : pthread_mutex_lock(mutex); } #endif /* !Windows */ -MDBX_INTERNAL_FUNC uint64_t osal_monotime(void); -MDBX_INTERNAL_FUNC uint64_t osal_cputime(size_t *optional_page_faults); -MDBX_INTERNAL_FUNC uint64_t osal_16dot16_to_monotime(uint32_t seconds_16dot16); -MDBX_INTERNAL_FUNC uint32_t osal_monotime_to_16dot16(uint64_t monotime); +MDBX_INTERNAL uint64_t osal_monotime(void); +MDBX_INTERNAL uint64_t osal_cputime(size_t *optional_page_faults); +MDBX_INTERNAL uint64_t osal_16dot16_to_monotime(uint32_t seconds_16dot16); +MDBX_INTERNAL uint32_t osal_monotime_to_16dot16(uint64_t monotime); -MDBX_MAYBE_UNUSED static inline uint32_t -osal_monotime_to_16dot16_noUnderflow(uint64_t monotime) { +MDBX_MAYBE_UNUSED static inline uint32_t osal_monotime_to_16dot16_noUnderflow(uint64_t monotime) { uint32_t seconds_16dot16 = osal_monotime_to_16dot16(monotime); return seconds_16dot16 ? seconds_16dot16 : /* fix underflow */ (monotime > 0); } -MDBX_INTERNAL_FUNC bin128_t osal_bootid(void); /*----------------------------------------------------------------------------*/ -/* lck stuff */ - -/// \brief Initialization of synchronization primitives linked with MDBX_env -/// instance both in LCK-file and within the current process. -/// \param -/// global_uniqueness_flag = true - denotes that there are no other processes -/// working with DB and LCK-file. Thus the function MUST initialize -/// shared synchronization objects in memory-mapped LCK-file. -/// global_uniqueness_flag = false - denotes that at least one process is -/// already working with DB and LCK-file, including the case when DB -/// has already been opened in the current process. Thus the function -/// MUST NOT initialize shared synchronization objects in memory-mapped -/// LCK-file that are already in use. -/// \return Error code or zero on success. -MDBX_INTERNAL_FUNC int osal_lck_init(MDBX_env *env, - MDBX_env *inprocess_neighbor, - int global_uniqueness_flag); - -/// \brief Disconnects from shared interprocess objects and destructs -/// synchronization objects linked with MDBX_env instance -/// within the current process. -/// \param -/// inprocess_neighbor = NULL - if the current process does not have other -/// instances of MDBX_env linked with the DB being closed. -/// Thus the function MUST check for other processes working with DB or -/// LCK-file, and keep or destroy shared synchronization objects in -/// memory-mapped LCK-file depending on the result. -/// inprocess_neighbor = not-NULL - pointer to another instance of MDBX_env -/// (anyone of there is several) working with DB or LCK-file within the -/// current process. Thus the function MUST NOT try to acquire exclusive -/// lock and/or try to destruct shared synchronization objects linked with -/// DB or LCK-file. Moreover, the implementation MUST ensure correct work -/// of other instances of MDBX_env within the current process, e.g. -/// restore POSIX-fcntl locks after the closing of file descriptors. -/// \return Error code (MDBX_PANIC) or zero on success. -MDBX_INTERNAL_FUNC int osal_lck_destroy(MDBX_env *env, - MDBX_env *inprocess_neighbor); - -/// \brief Connects to shared interprocess locking objects and tries to acquire -/// the maximum lock level (shared if exclusive is not available) -/// Depending on implementation or/and platform (Windows) this function may -/// acquire the non-OS super-level lock (e.g. for shared synchronization -/// objects initialization), which will be downgraded to OS-exclusive or -/// shared via explicit calling of osal_lck_downgrade(). -/// \return -/// MDBX_RESULT_TRUE (-1) - if an exclusive lock was acquired and thus -/// the current process is the first and only after the last use of DB. -/// MDBX_RESULT_FALSE (0) - if a shared lock was acquired and thus -/// DB has already been opened and now is used by other processes. -/// Otherwise (not 0 and not -1) - error code. -MDBX_INTERNAL_FUNC int osal_lck_seize(MDBX_env *env); - -/// \brief Downgrades the level of initially acquired lock to -/// operational level specified by argument. The reason for such downgrade: -/// - unblocking of other processes that are waiting for access, i.e. -/// if (env->me_flags & MDBX_EXCLUSIVE) != 0, then other processes -/// should be made aware that access is unavailable rather than -/// wait for it. -/// - freeing locks that interfere file operation (especially for Windows) -/// (env->me_flags & MDBX_EXCLUSIVE) == 0 - downgrade to shared lock. -/// (env->me_flags & MDBX_EXCLUSIVE) != 0 - downgrade to exclusive -/// operational lock. -/// \return Error code or zero on success -MDBX_INTERNAL_FUNC int osal_lck_downgrade(MDBX_env *env); - -/// \brief Locks LCK-file or/and table of readers for (de)registering. -/// \return Error code or zero on success -MDBX_INTERNAL_FUNC int osal_rdt_lock(MDBX_env *env); - -/// \brief Unlocks LCK-file or/and table of readers after (de)registering. -MDBX_INTERNAL_FUNC void osal_rdt_unlock(MDBX_env *env); - -/// \brief Acquires lock for DB change (on writing transaction start) -/// Reading transactions will not be blocked. -/// Declared as LIBMDBX_API because it is used in mdbx_chk. -/// \return Error code or zero on success -LIBMDBX_API int mdbx_txn_lock(MDBX_env *env, bool dont_wait); - -/// \brief Releases lock once DB changes is made (after writing transaction -/// has finished). -/// Declared as LIBMDBX_API because it is used in mdbx_chk. -LIBMDBX_API void mdbx_txn_unlock(MDBX_env *env); - -/// \brief Sets alive-flag of reader presence (indicative lock) for PID of -/// the current process. The function does no more than needed for -/// the correct working of osal_rpid_check() in other processes. -/// \return Error code or zero on success -MDBX_INTERNAL_FUNC int osal_rpid_set(MDBX_env *env); - -/// \brief Resets alive-flag of reader presence (indicative lock) -/// for PID of the current process. The function does no more than needed -/// for the correct working of osal_rpid_check() in other processes. -/// \return Error code or zero on success -MDBX_INTERNAL_FUNC int osal_rpid_clear(MDBX_env *env); - -/// \brief Checks for reading process status with the given pid with help of -/// alive-flag of presence (indicative lock) or using another way. -/// \return -/// MDBX_RESULT_TRUE (-1) - if the reader process with the given PID is alive -/// and working with DB (indicative lock is present). -/// MDBX_RESULT_FALSE (0) - if the reader process with the given PID is absent -/// or not working with DB (indicative lock is not present). -/// Otherwise (not 0 and not -1) - error code. -MDBX_INTERNAL_FUNC int osal_rpid_check(MDBX_env *env, uint32_t pid); - -#if defined(_WIN32) || defined(_WIN64) - -MDBX_INTERNAL_FUNC int osal_mb2w(const char *const src, wchar_t **const pdst); - -typedef void(WINAPI *osal_srwlock_t_function)(osal_srwlock_t *); -MDBX_INTERNAL_VAR osal_srwlock_t_function osal_srwlock_Init, - osal_srwlock_AcquireShared, osal_srwlock_ReleaseShared, - osal_srwlock_AcquireExclusive, osal_srwlock_ReleaseExclusive; - -#if _WIN32_WINNT < 0x0600 /* prior to Windows Vista */ -typedef enum _FILE_INFO_BY_HANDLE_CLASS { - FileBasicInfo, - FileStandardInfo, - FileNameInfo, - FileRenameInfo, - FileDispositionInfo, - FileAllocationInfo, - FileEndOfFileInfo, - FileStreamInfo, - FileCompressionInfo, - FileAttributeTagInfo, - FileIdBothDirectoryInfo, - FileIdBothDirectoryRestartInfo, - FileIoPriorityHintInfo, - FileRemoteProtocolInfo, - MaximumFileInfoByHandleClass -} FILE_INFO_BY_HANDLE_CLASS, - *PFILE_INFO_BY_HANDLE_CLASS; - -typedef struct _FILE_END_OF_FILE_INFO { - LARGE_INTEGER EndOfFile; -} FILE_END_OF_FILE_INFO, *PFILE_END_OF_FILE_INFO; - -#define REMOTE_PROTOCOL_INFO_FLAG_LOOPBACK 0x00000001 -#define REMOTE_PROTOCOL_INFO_FLAG_OFFLINE 0x00000002 - -typedef struct _FILE_REMOTE_PROTOCOL_INFO { - USHORT StructureVersion; - USHORT StructureSize; - DWORD Protocol; - USHORT ProtocolMajorVersion; - USHORT ProtocolMinorVersion; - USHORT ProtocolRevision; - USHORT Reserved; - DWORD Flags; - struct { - DWORD Reserved[8]; - } GenericReserved; - struct { - DWORD Reserved[16]; - } ProtocolSpecificReserved; -} FILE_REMOTE_PROTOCOL_INFO, *PFILE_REMOTE_PROTOCOL_INFO; - -#endif /* _WIN32_WINNT < 0x0600 (prior to Windows Vista) */ - -typedef BOOL(WINAPI *MDBX_GetFileInformationByHandleEx)( - _In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass, - _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize); -MDBX_INTERNAL_VAR MDBX_GetFileInformationByHandleEx - mdbx_GetFileInformationByHandleEx; - -typedef BOOL(WINAPI *MDBX_GetVolumeInformationByHandleW)( - _In_ HANDLE hFile, _Out_opt_ LPWSTR lpVolumeNameBuffer, - _In_ DWORD nVolumeNameSize, _Out_opt_ LPDWORD lpVolumeSerialNumber, - _Out_opt_ LPDWORD lpMaximumComponentLength, - _Out_opt_ LPDWORD lpFileSystemFlags, - _Out_opt_ LPWSTR lpFileSystemNameBuffer, _In_ DWORD nFileSystemNameSize); -MDBX_INTERNAL_VAR MDBX_GetVolumeInformationByHandleW - mdbx_GetVolumeInformationByHandleW; - -typedef DWORD(WINAPI *MDBX_GetFinalPathNameByHandleW)(_In_ HANDLE hFile, - _Out_ LPWSTR lpszFilePath, - _In_ DWORD cchFilePath, - _In_ DWORD dwFlags); -MDBX_INTERNAL_VAR MDBX_GetFinalPathNameByHandleW mdbx_GetFinalPathNameByHandleW; - -typedef BOOL(WINAPI *MDBX_SetFileInformationByHandle)( - _In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass, - _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize); -MDBX_INTERNAL_VAR MDBX_SetFileInformationByHandle - mdbx_SetFileInformationByHandle; - -typedef NTSTATUS(NTAPI *MDBX_NtFsControlFile)( - IN HANDLE FileHandle, IN OUT HANDLE Event, - IN OUT PVOID /* PIO_APC_ROUTINE */ ApcRoutine, IN OUT PVOID ApcContext, - OUT PIO_STATUS_BLOCK IoStatusBlock, IN ULONG FsControlCode, - IN OUT PVOID InputBuffer, IN ULONG InputBufferLength, - OUT OPTIONAL PVOID OutputBuffer, IN ULONG OutputBufferLength); -MDBX_INTERNAL_VAR MDBX_NtFsControlFile mdbx_NtFsControlFile; - -typedef uint64_t(WINAPI *MDBX_GetTickCount64)(void); -MDBX_INTERNAL_VAR MDBX_GetTickCount64 mdbx_GetTickCount64; - -#if !defined(_WIN32_WINNT_WIN8) || _WIN32_WINNT < _WIN32_WINNT_WIN8 -typedef struct _WIN32_MEMORY_RANGE_ENTRY { - PVOID VirtualAddress; - SIZE_T NumberOfBytes; -} WIN32_MEMORY_RANGE_ENTRY, *PWIN32_MEMORY_RANGE_ENTRY; -#endif /* Windows 8.x */ - -typedef BOOL(WINAPI *MDBX_PrefetchVirtualMemory)( - HANDLE hProcess, ULONG_PTR NumberOfEntries, - PWIN32_MEMORY_RANGE_ENTRY VirtualAddresses, ULONG Flags); -MDBX_INTERNAL_VAR MDBX_PrefetchVirtualMemory mdbx_PrefetchVirtualMemory; - -typedef enum _SECTION_INHERIT { ViewShare = 1, ViewUnmap = 2 } SECTION_INHERIT; - -typedef NTSTATUS(NTAPI *MDBX_NtExtendSection)(IN HANDLE SectionHandle, - IN PLARGE_INTEGER NewSectionSize); -MDBX_INTERNAL_VAR MDBX_NtExtendSection mdbx_NtExtendSection; - -static __inline bool mdbx_RunningUnderWine(void) { - return !mdbx_NtExtendSection; -} - -typedef LSTATUS(WINAPI *MDBX_RegGetValueA)(HKEY hkey, LPCSTR lpSubKey, - LPCSTR lpValue, DWORD dwFlags, - LPDWORD pdwType, PVOID pvData, - LPDWORD pcbData); -MDBX_INTERNAL_VAR MDBX_RegGetValueA mdbx_RegGetValueA; -NTSYSAPI ULONG RtlRandomEx(PULONG Seed); - -typedef BOOL(WINAPI *MDBX_SetFileIoOverlappedRange)(HANDLE FileHandle, - PUCHAR OverlappedRangeStart, - ULONG Length); -MDBX_INTERNAL_VAR MDBX_SetFileIoOverlappedRange mdbx_SetFileIoOverlappedRange; +MDBX_INTERNAL void osal_ctor(void); +MDBX_INTERNAL void osal_dtor(void); +#if defined(_WIN32) || defined(_WIN64) +MDBX_INTERNAL int osal_mb2w(const char *const src, wchar_t **const pdst); #endif /* Windows */ -#endif /* !__cplusplus */ +typedef union bin128 { + __anonymous_struct_extension__ struct { + uint64_t x, y; + }; + __anonymous_struct_extension__ struct { + uint32_t a, b, c, d; + }; +} bin128_t; + +MDBX_INTERNAL bin128_t osal_guid(const MDBX_env *); /*----------------------------------------------------------------------------*/ -MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static __always_inline uint64_t -osal_bswap64(uint64_t v) { -#if __GNUC_PREREQ(4, 4) || __CLANG_PREREQ(4, 0) || \ - __has_builtin(__builtin_bswap64) +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint64_t osal_bswap64(uint64_t v) { +#if __GNUC_PREREQ(4, 4) || __CLANG_PREREQ(4, 0) || __has_builtin(__builtin_bswap64) return __builtin_bswap64(v); #elif defined(_MSC_VER) && !defined(__clang__) return _byteswap_uint64(v); @@ -1918,19 +1582,14 @@ osal_bswap64(uint64_t v) { #elif defined(bswap_64) return bswap_64(v); #else - return v << 56 | v >> 56 | ((v << 40) & UINT64_C(0x00ff000000000000)) | - ((v << 24) & UINT64_C(0x0000ff0000000000)) | - ((v << 8) & UINT64_C(0x000000ff00000000)) | - ((v >> 8) & UINT64_C(0x00000000ff000000)) | - ((v >> 24) & UINT64_C(0x0000000000ff0000)) | - ((v >> 40) & UINT64_C(0x000000000000ff00)); + return v << 56 | v >> 56 | ((v << 40) & UINT64_C(0x00ff000000000000)) | ((v << 24) & UINT64_C(0x0000ff0000000000)) | + ((v << 8) & UINT64_C(0x000000ff00000000)) | ((v >> 8) & UINT64_C(0x00000000ff000000)) | + ((v >> 24) & UINT64_C(0x0000000000ff0000)) | ((v >> 40) & UINT64_C(0x000000000000ff00)); #endif } -MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static __always_inline uint32_t -osal_bswap32(uint32_t v) { -#if __GNUC_PREREQ(4, 4) || __CLANG_PREREQ(4, 0) || \ - __has_builtin(__builtin_bswap32) +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint32_t osal_bswap32(uint32_t v) { +#if __GNUC_PREREQ(4, 4) || __CLANG_PREREQ(4, 0) || __has_builtin(__builtin_bswap32) return __builtin_bswap32(v); #elif defined(_MSC_VER) && !defined(__clang__) return _byteswap_ulong(v); @@ -1939,50 +1598,14 @@ osal_bswap32(uint32_t v) { #elif defined(bswap_32) return bswap_32(v); #else - return v << 24 | v >> 24 | ((v << 8) & UINT32_C(0x00ff0000)) | - ((v >> 8) & UINT32_C(0x0000ff00)); + return v << 24 | v >> 24 | ((v << 8) & UINT32_C(0x00ff0000)) | ((v >> 8) & UINT32_C(0x0000ff00)); #endif } -/*----------------------------------------------------------------------------*/ - -#if defined(_MSC_VER) && _MSC_VER >= 1900 -/* LY: MSVC 2015/2017/2019 has buggy/inconsistent PRIuPTR/PRIxPTR macros - * for internal format-args checker. */ -#undef PRIuPTR -#undef PRIiPTR -#undef PRIdPTR -#undef PRIxPTR -#define PRIuPTR "Iu" -#define PRIiPTR "Ii" -#define PRIdPTR "Id" -#define PRIxPTR "Ix" -#define PRIuSIZE "zu" -#define PRIiSIZE "zi" -#define PRIdSIZE "zd" -#define PRIxSIZE "zx" -#endif /* fix PRI*PTR for _MSC_VER */ - -#ifndef PRIuSIZE -#define PRIuSIZE PRIuPTR -#define PRIiSIZE PRIiPTR -#define PRIdSIZE PRIdPTR -#define PRIxSIZE PRIxPTR -#endif /* PRI*SIZE macros for MSVC */ - -#ifdef _MSC_VER -#pragma warning(pop) -#endif - -#define mdbx_sourcery_anchor XCONCAT(mdbx_sourcery_, MDBX_BUILD_SOURCERY) -#if defined(xMDBX_TOOLS) -extern LIBMDBX_API const char *const mdbx_sourcery_anchor; -#endif - /******************************************************************************* - ******************************************************************************* ******************************************************************************* * + * BUILD TIME * * #### ##### ##### # #### # # #### * # # # # # # # # ## # # @@ -2003,23 +1626,15 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Using fsync() with chance of data lost on power failure */ #define MDBX_OSX_WANNA_SPEED 1 -#ifndef MDBX_OSX_SPEED_INSTEADOF_DURABILITY +#ifndef MDBX_APPLE_SPEED_INSTEADOF_DURABILITY /** Choices \ref MDBX_OSX_WANNA_DURABILITY or \ref MDBX_OSX_WANNA_SPEED * for OSX & iOS */ -#define MDBX_OSX_SPEED_INSTEADOF_DURABILITY MDBX_OSX_WANNA_DURABILITY -#endif /* MDBX_OSX_SPEED_INSTEADOF_DURABILITY */ - -/** Controls using of POSIX' madvise() and/or similar hints. */ -#ifndef MDBX_ENABLE_MADVISE -#define MDBX_ENABLE_MADVISE 1 -#elif !(MDBX_ENABLE_MADVISE == 0 || MDBX_ENABLE_MADVISE == 1) -#error MDBX_ENABLE_MADVISE must be defined as 0 or 1 -#endif /* MDBX_ENABLE_MADVISE */ +#define MDBX_APPLE_SPEED_INSTEADOF_DURABILITY MDBX_OSX_WANNA_DURABILITY +#endif /* MDBX_APPLE_SPEED_INSTEADOF_DURABILITY */ /** Controls checking PID against reuse DB environment after the fork() */ #ifndef MDBX_ENV_CHECKPID -#if (defined(MADV_DONTFORK) && MDBX_ENABLE_MADVISE) || defined(_WIN32) || \ - defined(_WIN64) +#if defined(MADV_DONTFORK) || defined(_WIN32) || defined(_WIN64) /* PID check could be omitted: * - on Linux when madvise(MADV_DONTFORK) is available, i.e. after the fork() * mapped pages will not be available for child process. @@ -2048,8 +1663,7 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Does a system have battery-backed Real-Time Clock or just a fake. */ #ifndef MDBX_TRUST_RTC -#if defined(__linux__) || defined(__gnu_linux__) || defined(__NetBSD__) || \ - defined(__OpenBSD__) +#if defined(__linux__) || defined(__gnu_linux__) || defined(__NetBSD__) || defined(__OpenBSD__) #define MDBX_TRUST_RTC 0 /* a lot of embedded systems have a fake RTC */ #else #define MDBX_TRUST_RTC 1 @@ -2084,24 +1698,21 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Controls using Unix' mincore() to determine whether DB-pages * are resident in memory. */ -#ifndef MDBX_ENABLE_MINCORE +#ifndef MDBX_USE_MINCORE #if defined(MINCORE_INCORE) || !(defined(_WIN32) || defined(_WIN64)) -#define MDBX_ENABLE_MINCORE 1 +#define MDBX_USE_MINCORE 1 #else -#define MDBX_ENABLE_MINCORE 0 +#define MDBX_USE_MINCORE 0 #endif -#elif !(MDBX_ENABLE_MINCORE == 0 || MDBX_ENABLE_MINCORE == 1) -#error MDBX_ENABLE_MINCORE must be defined as 0 or 1 -#endif /* MDBX_ENABLE_MINCORE */ +#define MDBX_USE_MINCORE_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_USE_MINCORE) +#elif !(MDBX_USE_MINCORE == 0 || MDBX_USE_MINCORE == 1) +#error MDBX_USE_MINCORE must be defined as 0 or 1 +#endif /* MDBX_USE_MINCORE */ /** Enables chunking long list of retired pages during huge transactions commit * to avoid use sequences of pages. */ #ifndef MDBX_ENABLE_BIGFOOT -#if MDBX_WORDBITS >= 64 || defined(DOXYGEN) #define MDBX_ENABLE_BIGFOOT 1 -#else -#define MDBX_ENABLE_BIGFOOT 0 -#endif #elif !(MDBX_ENABLE_BIGFOOT == 0 || MDBX_ENABLE_BIGFOOT == 1) #error MDBX_ENABLE_BIGFOOT must be defined as 0 or 1 #endif /* MDBX_ENABLE_BIGFOOT */ @@ -2116,25 +1727,27 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #ifndef MDBX_PNL_PREALLOC_FOR_RADIXSORT #define MDBX_PNL_PREALLOC_FOR_RADIXSORT 1 -#elif !(MDBX_PNL_PREALLOC_FOR_RADIXSORT == 0 || \ - MDBX_PNL_PREALLOC_FOR_RADIXSORT == 1) +#elif !(MDBX_PNL_PREALLOC_FOR_RADIXSORT == 0 || MDBX_PNL_PREALLOC_FOR_RADIXSORT == 1) #error MDBX_PNL_PREALLOC_FOR_RADIXSORT must be defined as 0 or 1 #endif /* MDBX_PNL_PREALLOC_FOR_RADIXSORT */ #ifndef MDBX_DPL_PREALLOC_FOR_RADIXSORT #define MDBX_DPL_PREALLOC_FOR_RADIXSORT 1 -#elif !(MDBX_DPL_PREALLOC_FOR_RADIXSORT == 0 || \ - MDBX_DPL_PREALLOC_FOR_RADIXSORT == 1) +#elif !(MDBX_DPL_PREALLOC_FOR_RADIXSORT == 0 || MDBX_DPL_PREALLOC_FOR_RADIXSORT == 1) #error MDBX_DPL_PREALLOC_FOR_RADIXSORT must be defined as 0 or 1 #endif /* MDBX_DPL_PREALLOC_FOR_RADIXSORT */ -/** Controls dirty pages tracking, spilling and persisting in MDBX_WRITEMAP - * mode. 0/OFF = Don't track dirty pages at all, don't spill ones, and use - * msync() to persist data. This is by-default on Linux and other systems where - * kernel provides properly LRU tracking and effective flushing on-demand. 1/ON - * = Tracking of dirty pages but with LRU labels for spilling and explicit - * persist ones by write(). This may be reasonable for systems which low - * performance of msync() and/or LRU tracking. */ +/** Controls dirty pages tracking, spilling and persisting in `MDBX_WRITEMAP` + * mode, i.e. disables in-memory database updating with consequent + * flush-to-disk/msync syscall. + * + * 0/OFF = Don't track dirty pages at all, don't spill ones, and use msync() to + * persist data. This is by-default on Linux and other systems where kernel + * provides properly LRU tracking and effective flushing on-demand. + * + * 1/ON = Tracking of dirty pages but with LRU labels for spilling and explicit + * persist ones by write(). This may be reasonable for goofy systems (Windows) + * which low performance of msync() and/or zany LRU tracking. */ #ifndef MDBX_AVOID_MSYNC #if defined(_WIN32) || defined(_WIN64) #define MDBX_AVOID_MSYNC 1 @@ -2145,6 +1758,22 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #error MDBX_AVOID_MSYNC must be defined as 0 or 1 #endif /* MDBX_AVOID_MSYNC */ +/** Управляет механизмом поддержки разреженных наборов DBI-хендлов для снижения + * накладных расходов при запуске и обработке транзакций. */ +#ifndef MDBX_ENABLE_DBI_SPARSE +#define MDBX_ENABLE_DBI_SPARSE 1 +#elif !(MDBX_ENABLE_DBI_SPARSE == 0 || MDBX_ENABLE_DBI_SPARSE == 1) +#error MDBX_ENABLE_DBI_SPARSE must be defined as 0 or 1 +#endif /* MDBX_ENABLE_DBI_SPARSE */ + +/** Управляет механизмом отложенного освобождения и поддержки пути быстрого + * открытия DBI-хендлов без захвата блокировок. */ +#ifndef MDBX_ENABLE_DBI_LOCKFREE +#define MDBX_ENABLE_DBI_LOCKFREE 1 +#elif !(MDBX_ENABLE_DBI_LOCKFREE == 0 || MDBX_ENABLE_DBI_LOCKFREE == 1) +#error MDBX_ENABLE_DBI_LOCKFREE must be defined as 0 or 1 +#endif /* MDBX_ENABLE_DBI_LOCKFREE */ + /** Controls sort order of internal page number lists. * This mostly experimental/advanced option with not for regular MDBX users. * \warning The database format depend on this option and libmdbx built with @@ -2157,7 +1786,11 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Avoid dependence from MSVC CRT and use ntdll.dll instead. */ #ifndef MDBX_WITHOUT_MSVC_CRT +#if defined(MDBX_BUILD_CXX) && !MDBX_BUILD_CXX #define MDBX_WITHOUT_MSVC_CRT 1 +#else +#define MDBX_WITHOUT_MSVC_CRT 0 +#endif #elif !(MDBX_WITHOUT_MSVC_CRT == 0 || MDBX_WITHOUT_MSVC_CRT == 1) #error MDBX_WITHOUT_MSVC_CRT must be defined as 0 or 1 #endif /* MDBX_WITHOUT_MSVC_CRT */ @@ -2165,12 +1798,11 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Size of buffer used during copying a environment/database file. */ #ifndef MDBX_ENVCOPY_WRITEBUF #define MDBX_ENVCOPY_WRITEBUF 1048576u -#elif MDBX_ENVCOPY_WRITEBUF < 65536u || MDBX_ENVCOPY_WRITEBUF > 1073741824u || \ - MDBX_ENVCOPY_WRITEBUF % 65536u +#elif MDBX_ENVCOPY_WRITEBUF < 65536u || MDBX_ENVCOPY_WRITEBUF > 1073741824u || MDBX_ENVCOPY_WRITEBUF % 65536u #error MDBX_ENVCOPY_WRITEBUF must be defined in range 65536..1073741824 and be multiple of 65536 #endif /* MDBX_ENVCOPY_WRITEBUF */ -/** Forces assertion checking */ +/** Forces assertion checking. */ #ifndef MDBX_FORCE_ASSERTIONS #define MDBX_FORCE_ASSERTIONS 0 #elif !(MDBX_FORCE_ASSERTIONS == 0 || MDBX_FORCE_ASSERTIONS == 1) @@ -2185,15 +1817,14 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #else #define MDBX_ASSUME_MALLOC_OVERHEAD (sizeof(void *) * 2u) #endif -#elif MDBX_ASSUME_MALLOC_OVERHEAD < 0 || MDBX_ASSUME_MALLOC_OVERHEAD > 64 || \ - MDBX_ASSUME_MALLOC_OVERHEAD % 4 +#elif MDBX_ASSUME_MALLOC_OVERHEAD < 0 || MDBX_ASSUME_MALLOC_OVERHEAD > 64 || MDBX_ASSUME_MALLOC_OVERHEAD % 4 #error MDBX_ASSUME_MALLOC_OVERHEAD must be defined in range 0..64 and be multiple of 4 #endif /* MDBX_ASSUME_MALLOC_OVERHEAD */ /** If defined then enables integration with Valgrind, * a memory analyzing tool. */ -#ifndef MDBX_USE_VALGRIND -#endif /* MDBX_USE_VALGRIND */ +#ifndef ENABLE_MEMCHECK +#endif /* ENABLE_MEMCHECK */ /** If defined then enables use C11 atomics, * otherwise detects ones availability automatically. */ @@ -2213,18 +1844,24 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #define MDBX_HAVE_BUILTIN_CPU_SUPPORTS 0 #elif defined(__e2k__) #define MDBX_HAVE_BUILTIN_CPU_SUPPORTS 0 -#elif __has_builtin(__builtin_cpu_supports) || \ - defined(__BUILTIN_CPU_SUPPORTS__) || \ +#elif __has_builtin(__builtin_cpu_supports) || defined(__BUILTIN_CPU_SUPPORTS__) || \ (defined(__ia32__) && __GNUC_PREREQ(4, 8) && __GLIBC_PREREQ(2, 23)) #define MDBX_HAVE_BUILTIN_CPU_SUPPORTS 1 #else #define MDBX_HAVE_BUILTIN_CPU_SUPPORTS 0 #endif -#elif !(MDBX_HAVE_BUILTIN_CPU_SUPPORTS == 0 || \ - MDBX_HAVE_BUILTIN_CPU_SUPPORTS == 1) +#elif !(MDBX_HAVE_BUILTIN_CPU_SUPPORTS == 0 || MDBX_HAVE_BUILTIN_CPU_SUPPORTS == 1) #error MDBX_HAVE_BUILTIN_CPU_SUPPORTS must be defined as 0 or 1 #endif /* MDBX_HAVE_BUILTIN_CPU_SUPPORTS */ +/** if enabled then instead of the returned error `MDBX_REMOTE`, only a warning is issued, when + * the database being opened in non-read-only mode is located in a file system exported via NFS. */ +#ifndef MDBX_ENABLE_NON_READONLY_EXPORT +#define MDBX_ENABLE_NON_READONLY_EXPORT 0 +#elif !(MDBX_ENABLE_NON_READONLY_EXPORT == 0 || MDBX_ENABLE_NON_READONLY_EXPORT == 1) +#error MDBX_ENABLE_NON_READONLY_EXPORT must be defined as 0 or 1 +#endif /* MDBX_ENABLE_NON_READONLY_EXPORT */ + //------------------------------------------------------------------------------ /** Win32 File Locking API for \ref MDBX_LOCKING */ @@ -2242,27 +1879,20 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** POSIX-2008 Robust Mutexes for \ref MDBX_LOCKING */ #define MDBX_LOCKING_POSIX2008 2008 -/** BeOS Benaphores, aka Futexes for \ref MDBX_LOCKING */ -#define MDBX_LOCKING_BENAPHORE 1995 - /** Advanced: Choices the locking implementation (autodetection by default). */ #if defined(_WIN32) || defined(_WIN64) #define MDBX_LOCKING MDBX_LOCKING_WIN32FILES #else #ifndef MDBX_LOCKING -#if defined(_POSIX_THREAD_PROCESS_SHARED) && \ - _POSIX_THREAD_PROCESS_SHARED >= 200112L && !defined(__FreeBSD__) +#if defined(_POSIX_THREAD_PROCESS_SHARED) && _POSIX_THREAD_PROCESS_SHARED >= 200112L && !defined(__FreeBSD__) /* Some platforms define the EOWNERDEAD error code even though they * don't support Robust Mutexes. If doubt compile with -MDBX_LOCKING=2001. */ -#if defined(EOWNERDEAD) && _POSIX_THREAD_PROCESS_SHARED >= 200809L && \ - ((defined(_POSIX_THREAD_ROBUST_PRIO_INHERIT) && \ - _POSIX_THREAD_ROBUST_PRIO_INHERIT > 0) || \ - (defined(_POSIX_THREAD_ROBUST_PRIO_PROTECT) && \ - _POSIX_THREAD_ROBUST_PRIO_PROTECT > 0) || \ - defined(PTHREAD_MUTEX_ROBUST) || defined(PTHREAD_MUTEX_ROBUST_NP)) && \ - (!defined(__GLIBC__) || \ - __GLIBC_PREREQ(2, 10) /* troubles with Robust mutexes before 2.10 */) +#if defined(EOWNERDEAD) && _POSIX_THREAD_PROCESS_SHARED >= 200809L && \ + ((defined(_POSIX_THREAD_ROBUST_PRIO_INHERIT) && _POSIX_THREAD_ROBUST_PRIO_INHERIT > 0) || \ + (defined(_POSIX_THREAD_ROBUST_PRIO_PROTECT) && _POSIX_THREAD_ROBUST_PRIO_PROTECT > 0) || \ + defined(PTHREAD_MUTEX_ROBUST) || defined(PTHREAD_MUTEX_ROBUST_NP)) && \ + (!defined(__GLIBC__) || __GLIBC_PREREQ(2, 10) /* troubles with Robust mutexes before 2.10 */) #define MDBX_LOCKING MDBX_LOCKING_POSIX2008 #else #define MDBX_LOCKING MDBX_LOCKING_POSIX2001 @@ -2280,12 +1910,9 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Advanced: Using POSIX OFD-locks (autodetection by default). */ #ifndef MDBX_USE_OFDLOCKS -#if ((defined(F_OFD_SETLK) && defined(F_OFD_SETLKW) && \ - defined(F_OFD_GETLK)) || \ - (defined(F_OFD_SETLK64) && defined(F_OFD_SETLKW64) && \ - defined(F_OFD_GETLK64))) && \ - !defined(MDBX_SAFE4QEMU) && \ - !defined(__sun) /* OFD-lock are broken on Solaris */ +#if ((defined(F_OFD_SETLK) && defined(F_OFD_SETLKW) && defined(F_OFD_GETLK)) || \ + (defined(F_OFD_SETLK64) && defined(F_OFD_SETLKW64) && defined(F_OFD_GETLK64))) && \ + !defined(MDBX_SAFE4QEMU) && !defined(__sun) /* OFD-lock are broken on Solaris */ #define MDBX_USE_OFDLOCKS 1 #else #define MDBX_USE_OFDLOCKS 0 @@ -2299,8 +1926,7 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Advanced: Using sendfile() syscall (autodetection by default). */ #ifndef MDBX_USE_SENDFILE -#if ((defined(__linux__) || defined(__gnu_linux__)) && \ - !defined(__ANDROID_API__)) || \ +#if ((defined(__linux__) || defined(__gnu_linux__)) && !defined(__ANDROID_API__)) || \ (defined(__ANDROID_API__) && __ANDROID_API__ >= 21) #define MDBX_USE_SENDFILE 1 #else @@ -2321,30 +1947,15 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #error MDBX_USE_COPYFILERANGE must be defined as 0 or 1 #endif /* MDBX_USE_COPYFILERANGE */ -/** Advanced: Using sync_file_range() syscall (autodetection by default). */ -#ifndef MDBX_USE_SYNCFILERANGE -#if ((defined(__linux__) || defined(__gnu_linux__)) && \ - defined(SYNC_FILE_RANGE_WRITE) && !defined(__ANDROID_API__)) || \ - (defined(__ANDROID_API__) && __ANDROID_API__ >= 26) -#define MDBX_USE_SYNCFILERANGE 1 -#else -#define MDBX_USE_SYNCFILERANGE 0 -#endif -#elif !(MDBX_USE_SYNCFILERANGE == 0 || MDBX_USE_SYNCFILERANGE == 1) -#error MDBX_USE_SYNCFILERANGE must be defined as 0 or 1 -#endif /* MDBX_USE_SYNCFILERANGE */ - //------------------------------------------------------------------------------ #ifndef MDBX_CPU_WRITEBACK_INCOHERENT -#if defined(__ia32__) || defined(__e2k__) || defined(__hppa) || \ - defined(__hppa__) || defined(DOXYGEN) +#if defined(__ia32__) || defined(__e2k__) || defined(__hppa) || defined(__hppa__) || defined(DOXYGEN) #define MDBX_CPU_WRITEBACK_INCOHERENT 0 #else #define MDBX_CPU_WRITEBACK_INCOHERENT 1 #endif -#elif !(MDBX_CPU_WRITEBACK_INCOHERENT == 0 || \ - MDBX_CPU_WRITEBACK_INCOHERENT == 1) +#elif !(MDBX_CPU_WRITEBACK_INCOHERENT == 0 || MDBX_CPU_WRITEBACK_INCOHERENT == 1) #error MDBX_CPU_WRITEBACK_INCOHERENT must be defined as 0 or 1 #endif /* MDBX_CPU_WRITEBACK_INCOHERENT */ @@ -2354,35 +1965,35 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #else #define MDBX_MMAP_INCOHERENT_FILE_WRITE 0 #endif -#elif !(MDBX_MMAP_INCOHERENT_FILE_WRITE == 0 || \ - MDBX_MMAP_INCOHERENT_FILE_WRITE == 1) +#elif !(MDBX_MMAP_INCOHERENT_FILE_WRITE == 0 || MDBX_MMAP_INCOHERENT_FILE_WRITE == 1) #error MDBX_MMAP_INCOHERENT_FILE_WRITE must be defined as 0 or 1 #endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */ #ifndef MDBX_MMAP_INCOHERENT_CPU_CACHE -#if defined(__mips) || defined(__mips__) || defined(__mips64) || \ - defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \ - defined(__MWERKS__) || defined(__sgi) +#if defined(__mips) || defined(__mips__) || defined(__mips64) || defined(__mips64__) || defined(_M_MRX000) || \ + defined(_MIPS_) || defined(__MWERKS__) || defined(__sgi) /* MIPS has cache coherency issues. */ #define MDBX_MMAP_INCOHERENT_CPU_CACHE 1 #else /* LY: assume no relevant mmap/dcache issues. */ #define MDBX_MMAP_INCOHERENT_CPU_CACHE 0 #endif -#elif !(MDBX_MMAP_INCOHERENT_CPU_CACHE == 0 || \ - MDBX_MMAP_INCOHERENT_CPU_CACHE == 1) +#elif !(MDBX_MMAP_INCOHERENT_CPU_CACHE == 0 || MDBX_MMAP_INCOHERENT_CPU_CACHE == 1) #error MDBX_MMAP_INCOHERENT_CPU_CACHE must be defined as 0 or 1 #endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */ -#ifndef MDBX_MMAP_USE_MS_ASYNC -#if MDBX_MMAP_INCOHERENT_FILE_WRITE || MDBX_MMAP_INCOHERENT_CPU_CACHE -#define MDBX_MMAP_USE_MS_ASYNC 1 +/** Assume system needs explicit syscall to sync/flush/write modified mapped + * memory. */ +#ifndef MDBX_MMAP_NEEDS_JOLT +#if MDBX_MMAP_INCOHERENT_FILE_WRITE || MDBX_MMAP_INCOHERENT_CPU_CACHE || !(defined(__linux__) || defined(__gnu_linux__)) +#define MDBX_MMAP_NEEDS_JOLT 1 #else -#define MDBX_MMAP_USE_MS_ASYNC 0 +#define MDBX_MMAP_NEEDS_JOLT 0 #endif -#elif !(MDBX_MMAP_USE_MS_ASYNC == 0 || MDBX_MMAP_USE_MS_ASYNC == 1) -#error MDBX_MMAP_USE_MS_ASYNC must be defined as 0 or 1 -#endif /* MDBX_MMAP_USE_MS_ASYNC */ +#define MDBX_MMAP_NEEDS_JOLT_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_MMAP_NEEDS_JOLT) +#elif !(MDBX_MMAP_NEEDS_JOLT == 0 || MDBX_MMAP_NEEDS_JOLT == 1) +#error MDBX_MMAP_NEEDS_JOLT must be defined as 0 or 1 +#endif /* MDBX_MMAP_NEEDS_JOLT */ #ifndef MDBX_64BIT_ATOMIC #if MDBX_WORDBITS >= 64 || defined(DOXYGEN) @@ -2429,8 +2040,7 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #endif /* MDBX_64BIT_CAS */ #ifndef MDBX_UNALIGNED_OK -#if defined(__ALIGNED__) || defined(__SANITIZE_UNDEFINED__) || \ - defined(ENABLE_UBSAN) +#if defined(__ALIGNED__) || defined(__SANITIZE_UNDEFINED__) || defined(ENABLE_UBSAN) #define MDBX_UNALIGNED_OK 0 /* no unaligned access allowed */ #elif defined(__ARM_FEATURE_UNALIGNED) #define MDBX_UNALIGNED_OK 4 /* ok unaligned for 32-bit words */ @@ -2464,6 +2074,19 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #endif #endif /* MDBX_CACHELINE_SIZE */ +/* Max length of iov-vector passed to writev() call, used for auxilary writes */ +#ifndef MDBX_AUXILARY_IOV_MAX +#define MDBX_AUXILARY_IOV_MAX 64 +#endif +#if defined(IOV_MAX) && IOV_MAX < MDBX_AUXILARY_IOV_MAX +#undef MDBX_AUXILARY_IOV_MAX +#define MDBX_AUXILARY_IOV_MAX IOV_MAX +#endif /* MDBX_AUXILARY_IOV_MAX */ + +/* An extra/custom information provided during library build */ +#ifndef MDBX_BUILD_METADATA +#define MDBX_BUILD_METADATA "" +#endif /* MDBX_BUILD_METADATA */ /** @} end of build options */ /******************************************************************************* ******************************************************************************* @@ -2478,6 +2101,9 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #else #define MDBX_DEBUG 1 #endif +#endif +#if MDBX_DEBUG < 0 || MDBX_DEBUG > 2 +#error "The MDBX_DEBUG must be defined to 0, 1 or 2" #endif /* MDBX_DEBUG */ #else @@ -2497,169 +2123,58 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; * Also enables \ref MDBX_DBG_AUDIT if `MDBX_DEBUG >= 2`. * * \ingroup build_option */ -#define MDBX_DEBUG 0...7 +#define MDBX_DEBUG 0...2 /** Disables using of GNU libc extensions. */ #define MDBX_DISABLE_GNU_SOURCE 0 or 1 #endif /* DOXYGEN */ -/* Undefine the NDEBUG if debugging is enforced by MDBX_DEBUG */ -#if MDBX_DEBUG -#undef NDEBUG -#endif - -#ifndef __cplusplus -/*----------------------------------------------------------------------------*/ -/* Debug and Logging stuff */ - -#define MDBX_RUNTIME_FLAGS_INIT \ - ((MDBX_DEBUG) > 0) * MDBX_DBG_ASSERT + ((MDBX_DEBUG) > 1) * MDBX_DBG_AUDIT - -extern uint8_t runtime_flags; -extern uint8_t loglevel; -extern MDBX_debug_func *debug_logger; - -MDBX_MAYBE_UNUSED static __inline void jitter4testing(bool tiny) { -#if MDBX_DEBUG - if (MDBX_DBG_JITTER & runtime_flags) - osal_jitter(tiny); -#else - (void)tiny; -#endif -} - -MDBX_INTERNAL_FUNC void MDBX_PRINTF_ARGS(4, 5) - debug_log(int level, const char *function, int line, const char *fmt, ...) - MDBX_PRINTF_ARGS(4, 5); -MDBX_INTERNAL_FUNC void debug_log_va(int level, const char *function, int line, - const char *fmt, va_list args); +#ifndef MDBX_64BIT_ATOMIC +#error "The MDBX_64BIT_ATOMIC must be defined before" +#endif /* MDBX_64BIT_ATOMIC */ -#if MDBX_DEBUG -#define LOG_ENABLED(msg) unlikely(msg <= loglevel) -#define AUDIT_ENABLED() unlikely((runtime_flags & MDBX_DBG_AUDIT)) -#else /* MDBX_DEBUG */ -#define LOG_ENABLED(msg) (msg < MDBX_LOG_VERBOSE && msg <= loglevel) -#define AUDIT_ENABLED() (0) -#endif /* MDBX_DEBUG */ +#ifndef MDBX_64BIT_CAS +#error "The MDBX_64BIT_CAS must be defined before" +#endif /* MDBX_64BIT_CAS */ -#if MDBX_FORCE_ASSERTIONS -#define ASSERT_ENABLED() (1) -#elif MDBX_DEBUG -#define ASSERT_ENABLED() likely((runtime_flags & MDBX_DBG_ASSERT)) +#if defined(__cplusplus) && !defined(__STDC_NO_ATOMICS__) && __has_include() +#include +#define MDBX_HAVE_C11ATOMICS +#elif !defined(__cplusplus) && (__STDC_VERSION__ >= 201112L || __has_extension(c_atomic)) && \ + !defined(__STDC_NO_ATOMICS__) && \ + (__GNUC_PREREQ(4, 9) || __CLANG_PREREQ(3, 8) || !(defined(__GNUC__) || defined(__clang__))) +#include +#define MDBX_HAVE_C11ATOMICS +#elif defined(__GNUC__) || defined(__clang__) +#elif defined(_MSC_VER) +#pragma warning(disable : 4163) /* 'xyz': not available as an intrinsic */ +#pragma warning(disable : 4133) /* 'function': incompatible types - from \ + 'size_t' to 'LONGLONG' */ +#pragma warning(disable : 4244) /* 'return': conversion from 'LONGLONG' to \ + 'std::size_t', possible loss of data */ +#pragma warning(disable : 4267) /* 'function': conversion from 'size_t' to \ + 'long', possible loss of data */ +#pragma intrinsic(_InterlockedExchangeAdd, _InterlockedCompareExchange) +#pragma intrinsic(_InterlockedExchangeAdd64, _InterlockedCompareExchange64) +#elif defined(__APPLE__) +#include #else -#define ASSERT_ENABLED() (0) -#endif /* assertions */ - -#define DEBUG_EXTRA(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_EXTRA)) \ - debug_log(MDBX_LOG_EXTRA, __func__, __LINE__, fmt, __VA_ARGS__); \ - } while (0) - -#define DEBUG_EXTRA_PRINT(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_EXTRA)) \ - debug_log(MDBX_LOG_EXTRA, NULL, 0, fmt, __VA_ARGS__); \ - } while (0) - -#define TRACE(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_TRACE)) \ - debug_log(MDBX_LOG_TRACE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ - } while (0) - -#define DEBUG(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_DEBUG)) \ - debug_log(MDBX_LOG_DEBUG, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ - } while (0) - -#define VERBOSE(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_VERBOSE)) \ - debug_log(MDBX_LOG_VERBOSE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ - } while (0) - -#define NOTICE(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_NOTICE)) \ - debug_log(MDBX_LOG_NOTICE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ - } while (0) - -#define WARNING(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_WARN)) \ - debug_log(MDBX_LOG_WARN, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ - } while (0) - -#undef ERROR /* wingdi.h \ - Yeah, morons from M$ put such definition to the public header. */ - -#define ERROR(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_ERROR)) \ - debug_log(MDBX_LOG_ERROR, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ - } while (0) - -#define FATAL(fmt, ...) \ - debug_log(MDBX_LOG_FATAL, __func__, __LINE__, fmt "\n", __VA_ARGS__); - -#if MDBX_DEBUG -#define ASSERT_FAIL(env, msg, func, line) mdbx_assert_fail(env, msg, func, line) -#else /* MDBX_DEBUG */ -MDBX_NORETURN __cold void assert_fail(const char *msg, const char *func, - unsigned line); -#define ASSERT_FAIL(env, msg, func, line) \ - do { \ - (void)(env); \ - assert_fail(msg, func, line); \ - } while (0) -#endif /* MDBX_DEBUG */ - -#define ENSURE_MSG(env, expr, msg) \ - do { \ - if (unlikely(!(expr))) \ - ASSERT_FAIL(env, msg, __func__, __LINE__); \ - } while (0) - -#define ENSURE(env, expr) ENSURE_MSG(env, expr, #expr) - -/* assert(3) variant in environment context */ -#define eASSERT(env, expr) \ - do { \ - if (ASSERT_ENABLED()) \ - ENSURE(env, expr); \ - } while (0) - -/* assert(3) variant in cursor context */ -#define cASSERT(mc, expr) eASSERT((mc)->mc_txn->mt_env, expr) - -/* assert(3) variant in transaction context */ -#define tASSERT(txn, expr) eASSERT((txn)->mt_env, expr) - -#ifndef xMDBX_TOOLS /* Avoid using internal eASSERT() */ -#undef assert -#define assert(expr) eASSERT(NULL, expr) +#error FIXME atomic-ops #endif -#endif /* __cplusplus */ - -/*----------------------------------------------------------------------------*/ -/* Atomics */ - -enum MDBX_memory_order { +typedef enum mdbx_memory_order { mo_Relaxed, mo_AcquireRelease /* , mo_SequentialConsistency */ -}; +} mdbx_memory_order_t; typedef union { volatile uint32_t weak; #ifdef MDBX_HAVE_C11ATOMICS volatile _Atomic uint32_t c11a; #endif /* MDBX_HAVE_C11ATOMICS */ -} MDBX_atomic_uint32_t; +} mdbx_atomic_uint32_t; typedef union { volatile uint64_t weak; @@ -2669,15 +2184,15 @@ typedef union { #if !defined(MDBX_HAVE_C11ATOMICS) || !MDBX_64BIT_CAS || !MDBX_64BIT_ATOMIC __anonymous_struct_extension__ struct { #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ - MDBX_atomic_uint32_t low, high; + mdbx_atomic_uint32_t low, high; #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - MDBX_atomic_uint32_t high, low; + mdbx_atomic_uint32_t high, low; #else #error "FIXME: Unsupported byte order" #endif /* __BYTE_ORDER__ */ }; #endif -} MDBX_atomic_uint64_t; +} mdbx_atomic_uint64_t; #ifdef MDBX_HAVE_C11ATOMICS @@ -2693,92 +2208,20 @@ typedef union { #define MDBX_c11a_rw(type, ptr) (&(ptr)->c11a) #endif /* Crutches for C11 atomic compiler's bugs */ -#define mo_c11_store(fence) \ - (((fence) == mo_Relaxed) ? memory_order_relaxed \ - : ((fence) == mo_AcquireRelease) ? memory_order_release \ +#define mo_c11_store(fence) \ + (((fence) == mo_Relaxed) ? memory_order_relaxed \ + : ((fence) == mo_AcquireRelease) ? memory_order_release \ : memory_order_seq_cst) -#define mo_c11_load(fence) \ - (((fence) == mo_Relaxed) ? memory_order_relaxed \ - : ((fence) == mo_AcquireRelease) ? memory_order_acquire \ +#define mo_c11_load(fence) \ + (((fence) == mo_Relaxed) ? memory_order_relaxed \ + : ((fence) == mo_AcquireRelease) ? memory_order_acquire \ : memory_order_seq_cst) #endif /* MDBX_HAVE_C11ATOMICS */ -#ifndef __cplusplus - -#ifdef MDBX_HAVE_C11ATOMICS -#define osal_memory_fence(order, write) \ - atomic_thread_fence((write) ? mo_c11_store(order) : mo_c11_load(order)) -#else /* MDBX_HAVE_C11ATOMICS */ -#define osal_memory_fence(order, write) \ - do { \ - osal_compiler_barrier(); \ - if (write && order > (MDBX_CPU_WRITEBACK_INCOHERENT ? mo_Relaxed \ - : mo_AcquireRelease)) \ - osal_memory_barrier(); \ - } while (0) -#endif /* MDBX_HAVE_C11ATOMICS */ - -#if defined(MDBX_HAVE_C11ATOMICS) && defined(__LCC__) -#define atomic_store32(p, value, order) \ - ({ \ - const uint32_t value_to_store = (value); \ - atomic_store_explicit(MDBX_c11a_rw(uint32_t, p), value_to_store, \ - mo_c11_store(order)); \ - value_to_store; \ - }) -#define atomic_load32(p, order) \ - atomic_load_explicit(MDBX_c11a_ro(uint32_t, p), mo_c11_load(order)) -#define atomic_store64(p, value, order) \ - ({ \ - const uint64_t value_to_store = (value); \ - atomic_store_explicit(MDBX_c11a_rw(uint64_t, p), value_to_store, \ - mo_c11_store(order)); \ - value_to_store; \ - }) -#define atomic_load64(p, order) \ - atomic_load_explicit(MDBX_c11a_ro(uint64_t, p), mo_c11_load(order)) -#endif /* LCC && MDBX_HAVE_C11ATOMICS */ - -#ifndef atomic_store32 -MDBX_MAYBE_UNUSED static __always_inline uint32_t -atomic_store32(MDBX_atomic_uint32_t *p, const uint32_t value, - enum MDBX_memory_order order) { - STATIC_ASSERT(sizeof(MDBX_atomic_uint32_t) == 4); -#ifdef MDBX_HAVE_C11ATOMICS - assert(atomic_is_lock_free(MDBX_c11a_rw(uint32_t, p))); - atomic_store_explicit(MDBX_c11a_rw(uint32_t, p), value, mo_c11_store(order)); -#else /* MDBX_HAVE_C11ATOMICS */ - if (order != mo_Relaxed) - osal_compiler_barrier(); - p->weak = value; - osal_memory_fence(order, true); -#endif /* MDBX_HAVE_C11ATOMICS */ - return value; -} -#endif /* atomic_store32 */ - -#ifndef atomic_load32 -MDBX_MAYBE_UNUSED static __always_inline uint32_t atomic_load32( - const volatile MDBX_atomic_uint32_t *p, enum MDBX_memory_order order) { - STATIC_ASSERT(sizeof(MDBX_atomic_uint32_t) == 4); -#ifdef MDBX_HAVE_C11ATOMICS - assert(atomic_is_lock_free(MDBX_c11a_ro(uint32_t, p))); - return atomic_load_explicit(MDBX_c11a_ro(uint32_t, p), mo_c11_load(order)); -#else /* MDBX_HAVE_C11ATOMICS */ - osal_memory_fence(order, false); - const uint32_t value = p->weak; - if (order != mo_Relaxed) - osal_compiler_barrier(); - return value; -#endif /* MDBX_HAVE_C11ATOMICS */ -} -#endif /* atomic_load32 */ - -#endif /* !__cplusplus */ +#define SAFE64_INVALID_THRESHOLD UINT64_C(0xffffFFFF00000000) -/*----------------------------------------------------------------------------*/ -/* Basic constants and types */ +#pragma pack(push, 4) /* A stamp that identifies a file as an MDBX file. * There's nothing special about this value other than that it is easily @@ -2787,8 +2230,10 @@ MDBX_MAYBE_UNUSED static __always_inline uint32_t atomic_load32( /* FROZEN: The version number for a database's datafile format. */ #define MDBX_DATA_VERSION 3 -/* The version number for a database's lockfile format. */ -#define MDBX_LOCK_VERSION 5 + +#define MDBX_DATA_MAGIC ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + MDBX_DATA_VERSION) +#define MDBX_DATA_MAGIC_LEGACY_COMPAT ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + 2) +#define MDBX_DATA_MAGIC_LEGACY_DEVEL ((MDBX_MAGIC << 8) + 255) /* handle for the DB used to track free pages. */ #define FREE_DBI 0 @@ -2805,203 +2250,285 @@ MDBX_MAYBE_UNUSED static __always_inline uint32_t atomic_load32( * MDBX uses 32 bit for page numbers. This limits database * size up to 2^44 bytes, in case of 4K pages. */ typedef uint32_t pgno_t; -typedef MDBX_atomic_uint32_t atomic_pgno_t; +typedef mdbx_atomic_uint32_t atomic_pgno_t; #define PRIaPGNO PRIu32 #define MAX_PAGENO UINT32_C(0x7FFFffff) #define MIN_PAGENO NUM_METAS -#define SAFE64_INVALID_THRESHOLD UINT64_C(0xffffFFFF00000000) +/* An invalid page number. + * Mainly used to denote an empty tree. */ +#define P_INVALID (~(pgno_t)0) /* A transaction ID. */ typedef uint64_t txnid_t; -typedef MDBX_atomic_uint64_t atomic_txnid_t; +typedef mdbx_atomic_uint64_t atomic_txnid_t; #define PRIaTXN PRIi64 #define MIN_TXNID UINT64_C(1) #define MAX_TXNID (SAFE64_INVALID_THRESHOLD - 1) #define INITIAL_TXNID (MIN_TXNID + NUM_METAS - 1) #define INVALID_TXNID UINT64_MAX -/* LY: for testing non-atomic 64-bit txnid on 32-bit arches. - * #define xMDBX_TXNID_STEP (UINT32_MAX / 3) */ -#ifndef xMDBX_TXNID_STEP -#if MDBX_64BIT_CAS -#define xMDBX_TXNID_STEP 1u -#else -#define xMDBX_TXNID_STEP 2u -#endif -#endif /* xMDBX_TXNID_STEP */ -/* Used for offsets within a single page. - * Since memory pages are typically 4 or 8KB in size, 12-13 bits, - * this is plenty. */ +/* Used for offsets within a single page. */ typedef uint16_t indx_t; -#define MEGABYTE ((size_t)1 << 20) - -/*----------------------------------------------------------------------------*/ -/* Core structures for database and shared memory (i.e. format definition) */ -#pragma pack(push, 4) - -/* Information about a single database in the environment. */ -typedef struct MDBX_db { - uint16_t md_flags; /* see mdbx_dbi_open */ - uint16_t md_depth; /* depth of this tree */ - uint32_t md_xsize; /* key-size for MDBX_DUPFIXED (LEAF2 pages) */ - pgno_t md_root; /* the root page of this tree */ - pgno_t md_branch_pages; /* number of internal pages */ - pgno_t md_leaf_pages; /* number of leaf pages */ - pgno_t md_overflow_pages; /* number of overflow pages */ - uint64_t md_seq; /* table sequence counter */ - uint64_t md_entries; /* number of data items */ - uint64_t md_mod_txnid; /* txnid of last committed modification */ -} MDBX_db; +typedef struct tree { + uint16_t flags; /* see mdbx_dbi_open */ + uint16_t height; /* height of this tree */ + uint32_t dupfix_size; /* key-size for MDBX_DUPFIXED (DUPFIX pages) */ + pgno_t root; /* the root page of this tree */ + pgno_t branch_pages; /* number of branch pages */ + pgno_t leaf_pages; /* number of leaf pages */ + pgno_t large_pages; /* number of large pages */ + uint64_t sequence; /* table sequence counter */ + uint64_t items; /* number of data items */ + uint64_t mod_txnid; /* txnid of last committed modification */ +} tree_t; /* database size-related parameters */ -typedef struct MDBX_geo { +typedef struct geo { uint16_t grow_pv; /* datafile growth step as a 16-bit packed (exponential quantized) value */ uint16_t shrink_pv; /* datafile shrink threshold as a 16-bit packed (exponential quantized) value */ pgno_t lower; /* minimal size of datafile in pages */ pgno_t upper; /* maximal size of datafile in pages */ - pgno_t now; /* current size of datafile in pages */ - pgno_t next; /* first unused page in the datafile, + union { + pgno_t now; /* current size of datafile in pages */ + pgno_t end_pgno; + }; + union { + pgno_t first_unallocated; /* first unused page in the datafile, but actually the file may be shorter. */ -} MDBX_geo; + pgno_t next_pgno; + }; +} geo_t; /* Meta page content. * A meta page is the start point for accessing a database snapshot. - * Pages 0-1 are meta pages. Transaction N writes meta page (N % 2). */ -typedef struct MDBX_meta { + * Pages 0-2 are meta pages. */ +typedef struct meta { /* Stamp identifying this as an MDBX file. * It must be set to MDBX_MAGIC with MDBX_DATA_VERSION. */ - uint32_t mm_magic_and_version[2]; + uint32_t magic_and_version[2]; - /* txnid that committed this page, the first of a two-phase-update pair */ + /* txnid that committed this meta, the first of a two-phase-update pair */ union { - MDBX_atomic_uint32_t mm_txnid_a[2]; + mdbx_atomic_uint32_t txnid_a[2]; uint64_t unsafe_txnid; }; - uint16_t mm_extra_flags; /* extra DB flags, zero (nothing) for now */ - uint8_t mm_validator_id; /* ID of checksum and page validation method, - * zero (nothing) for now */ - uint8_t mm_extra_pagehdr; /* extra bytes in the page header, - * zero (nothing) for now */ + uint16_t reserve16; /* extra flags, zero (nothing) for now */ + uint8_t validator_id; /* ID of checksum and page validation method, + * zero (nothing) for now */ + int8_t extra_pagehdr; /* extra bytes in the page header, + * zero (nothing) for now */ + + geo_t geometry; /* database size-related parameters */ - MDBX_geo mm_geo; /* database size-related parameters */ + union { + struct { + tree_t gc, main; + } trees; + __anonymous_struct_extension__ struct { + uint16_t gc_flags; + uint16_t gc_height; + uint32_t pagesize; + }; + }; - MDBX_db mm_dbs[CORE_DBS]; /* first is free space, 2nd is main db */ - /* The size of pages used in this DB */ -#define mm_psize mm_dbs[FREE_DBI].md_xsize - MDBX_canary mm_canary; + MDBX_canary canary; -#define MDBX_DATASIGN_NONE 0u -#define MDBX_DATASIGN_WEAK 1u -#define SIGN_IS_STEADY(sign) ((sign) > MDBX_DATASIGN_WEAK) -#define META_IS_STEADY(meta) \ - SIGN_IS_STEADY(unaligned_peek_u64_volatile(4, (meta)->mm_sign)) +#define DATASIGN_NONE 0u +#define DATASIGN_WEAK 1u +#define SIGN_IS_STEADY(sign) ((sign) > DATASIGN_WEAK) union { - uint32_t mm_sign[2]; + uint32_t sign[2]; uint64_t unsafe_sign; }; - /* txnid that committed this page, the second of a two-phase-update pair */ - MDBX_atomic_uint32_t mm_txnid_b[2]; + /* txnid that committed this meta, the second of a two-phase-update pair */ + mdbx_atomic_uint32_t txnid_b[2]; /* Number of non-meta pages which were put in GC after COW. May be 0 in case * DB was previously handled by libmdbx without corresponding feature. - * This value in couple with mr_snapshot_pages_retired allows fast estimation - * of "how much reader is restraining GC recycling". */ - uint32_t mm_pages_retired[2]; + * This value in couple with reader.snapshot_pages_retired allows fast + * estimation of "how much reader is restraining GC recycling". */ + uint32_t pages_retired[2]; /* The analogue /proc/sys/kernel/random/boot_id or similar to determine * whether the system was rebooted after the last use of the database files. * If there was no reboot, but there is no need to rollback to the last * steady sync point. Zeros mean that no relevant information is available * from the system. */ - bin128_t mm_bootid; + bin128_t bootid; -} MDBX_meta; + /* GUID базы данных, начиная с v0.13.1 */ + bin128_t dxbid; +} meta_t; #pragma pack(1) -/* Common header for all page types. The page type depends on mp_flags. +typedef enum page_type { + P_BRANCH = 0x01u /* branch page */, + P_LEAF = 0x02u /* leaf page */, + P_LARGE = 0x04u /* large/overflow page */, + P_META = 0x08u /* meta page */, + P_LEGACY_DIRTY = 0x10u /* legacy P_DIRTY flag prior to v0.10 958fd5b9 */, + P_BAD = P_LEGACY_DIRTY /* explicit flag for invalid/bad page */, + P_DUPFIX = 0x20u /* for MDBX_DUPFIXED records */, + P_SUBP = 0x40u /* for MDBX_DUPSORT sub-pages */, + P_SPILLED = 0x2000u /* spilled in parent txn */, + P_LOOSE = 0x4000u /* page was dirtied then freed, can be reused */, + P_FROZEN = 0x8000u /* used for retire page with known status */, + P_ILL_BITS = (uint16_t)~(P_BRANCH | P_LEAF | P_DUPFIX | P_LARGE | P_SPILLED), + + page_broken = 0, + page_large = P_LARGE, + page_branch = P_BRANCH, + page_leaf = P_LEAF, + page_dupfix_leaf = P_DUPFIX, + page_sub_leaf = P_SUBP | P_LEAF, + page_sub_dupfix_leaf = P_SUBP | P_DUPFIX, + page_sub_broken = P_SUBP, +} page_type_t; + +/* Common header for all page types. The page type depends on flags. * - * P_BRANCH and P_LEAF pages have unsorted 'MDBX_node's at the end, with - * sorted mp_ptrs[] entries referring to them. Exception: P_LEAF2 pages - * omit mp_ptrs and pack sorted MDBX_DUPFIXED values after the page header. + * P_BRANCH and P_LEAF pages have unsorted 'node_t's at the end, with + * sorted entries[] entries referring to them. Exception: P_DUPFIX pages + * omit entries and pack sorted MDBX_DUPFIXED values after the page header. * - * P_OVERFLOW records occupy one or more contiguous pages where only the - * first has a page header. They hold the real data of F_BIGDATA nodes. + * P_LARGE records occupy one or more contiguous pages where only the + * first has a page header. They hold the real data of N_BIG nodes. * * P_SUBP sub-pages are small leaf "pages" with duplicate data. - * A node with flag F_DUPDATA but not F_SUBDATA contains a sub-page. - * (Duplicate data can also go in sub-databases, which use normal pages.) + * A node with flag N_DUP but not N_TREE contains a sub-page. + * (Duplicate data can also go in tables, which use normal pages.) * - * P_META pages contain MDBX_meta, the start point of an MDBX snapshot. + * P_META pages contain meta_t, the start point of an MDBX snapshot. * - * Each non-metapage up to MDBX_meta.mm_last_pg is reachable exactly once + * Each non-metapage up to meta_t.mm_last_pg is reachable exactly once * in the snapshot: Either used by a database or listed in a GC record. */ -typedef struct MDBX_page { -#define IS_FROZEN(txn, p) ((p)->mp_txnid < (txn)->mt_txnid) -#define IS_SPILLED(txn, p) ((p)->mp_txnid == (txn)->mt_txnid) -#define IS_SHADOWED(txn, p) ((p)->mp_txnid > (txn)->mt_txnid) -#define IS_VALID(txn, p) ((p)->mp_txnid <= (txn)->mt_front) -#define IS_MODIFIABLE(txn, p) ((p)->mp_txnid == (txn)->mt_front) - uint64_t mp_txnid; /* txnid which created page, maybe zero in legacy DB */ - uint16_t mp_leaf2_ksize; /* key size if this is a LEAF2 page */ -#define P_BRANCH 0x01u /* branch page */ -#define P_LEAF 0x02u /* leaf page */ -#define P_OVERFLOW 0x04u /* overflow page */ -#define P_META 0x08u /* meta page */ -#define P_LEGACY_DIRTY 0x10u /* legacy P_DIRTY flag prior to v0.10 958fd5b9 */ -#define P_BAD P_LEGACY_DIRTY /* explicit flag for invalid/bad page */ -#define P_LEAF2 0x20u /* for MDBX_DUPFIXED records */ -#define P_SUBP 0x40u /* for MDBX_DUPSORT sub-pages */ -#define P_SPILLED 0x2000u /* spilled in parent txn */ -#define P_LOOSE 0x4000u /* page was dirtied then freed, can be reused */ -#define P_FROZEN 0x8000u /* used for retire page with known status */ -#define P_ILL_BITS \ - ((uint16_t)~(P_BRANCH | P_LEAF | P_LEAF2 | P_OVERFLOW | P_SPILLED)) - uint16_t mp_flags; +typedef struct page { + uint64_t txnid; /* txnid which created page, maybe zero in legacy DB */ + uint16_t dupfix_ksize; /* key size if this is a DUPFIX page */ + uint16_t flags; union { - uint32_t mp_pages; /* number of overflow pages */ + uint32_t pages; /* number of overflow pages */ __anonymous_struct_extension__ struct { - indx_t mp_lower; /* lower bound of free space */ - indx_t mp_upper; /* upper bound of free space */ + indx_t lower; /* lower bound of free space */ + indx_t upper; /* upper bound of free space */ }; }; - pgno_t mp_pgno; /* page number */ + pgno_t pgno; /* page number */ -#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ - (!defined(__cplusplus) && defined(_MSC_VER)) - indx_t mp_ptrs[] /* dynamic size */; -#endif /* C99 */ -} MDBX_page; +#if FLEXIBLE_ARRAY_MEMBERS + indx_t entries[] /* dynamic size */; +#endif /* FLEXIBLE_ARRAY_MEMBERS */ +} page_t; -#define PAGETYPE_WHOLE(p) ((uint8_t)(p)->mp_flags) +/* Size of the page header, excluding dynamic data at the end */ +#define PAGEHDRSZ 20u -/* Drop legacy P_DIRTY flag for sub-pages for compatilibity */ -#define PAGETYPE_COMPAT(p) \ - (unlikely(PAGETYPE_WHOLE(p) & P_SUBP) \ - ? PAGETYPE_WHOLE(p) & ~(P_SUBP | P_LEGACY_DIRTY) \ - : PAGETYPE_WHOLE(p)) - -/* Size of the page header, excluding dynamic data at the end */ -#define PAGEHDRSZ offsetof(MDBX_page, mp_ptrs) +/* Header for a single key/data pair within a page. + * Used in pages of type P_BRANCH and P_LEAF without P_DUPFIX. + * We guarantee 2-byte alignment for 'node_t's. + * + * Leaf node flags describe node contents. N_BIG says the node's + * data part is the page number of an overflow page with actual data. + * N_DUP and N_TREE can be combined giving duplicate data in + * a sub-page/table, and named databases (just N_TREE). */ +typedef struct node { +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + union { + uint32_t dsize; + uint32_t child_pgno; + }; + uint8_t flags; /* see node_flags */ + uint8_t extra; + uint16_t ksize; /* key size */ +#else + uint16_t ksize; /* key size */ + uint8_t extra; + uint8_t flags; /* see node_flags */ + union { + uint32_t child_pgno; + uint32_t dsize; + }; +#endif /* __BYTE_ORDER__ */ -/* Pointer displacement without casting to char* to avoid pointer-aliasing */ -#define ptr_disp(ptr, disp) ((void *)(((intptr_t)(ptr)) + ((intptr_t)(disp)))) +#if FLEXIBLE_ARRAY_MEMBERS + uint8_t payload[] /* key and data are appended here */; +#endif /* FLEXIBLE_ARRAY_MEMBERS */ +} node_t; -/* Pointer distance as signed number of bytes */ -#define ptr_dist(more, less) (((intptr_t)(more)) - ((intptr_t)(less))) +/* Size of the node header, excluding dynamic data at the end */ +#define NODESIZE 8u -#define mp_next(mp) \ - (*(MDBX_page **)ptr_disp((mp)->mp_ptrs, sizeof(void *) - sizeof(uint32_t))) +typedef enum node_flags { + N_BIG = 0x01 /* data put on large page */, + N_TREE = 0x02 /* data is a b-tree */, + N_DUP = 0x04 /* data has duplicates */ +} node_flags_t; #pragma pack(pop) -typedef struct profgc_stat { +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint8_t page_type(const page_t *mp) { return mp->flags; } + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint8_t page_type_compat(const page_t *mp) { + /* Drop legacy P_DIRTY flag for sub-pages for compatilibity, + * for assertions only. */ + return unlikely(mp->flags & P_SUBP) ? mp->flags & ~(P_SUBP | P_LEGACY_DIRTY) : mp->flags; +} + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_leaf(const page_t *mp) { + return (mp->flags & P_LEAF) != 0; +} + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_dupfix_leaf(const page_t *mp) { + return (mp->flags & P_DUPFIX) != 0; +} + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_branch(const page_t *mp) { + return (mp->flags & P_BRANCH) != 0; +} + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_largepage(const page_t *mp) { + return (mp->flags & P_LARGE) != 0; +} + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_subpage(const page_t *mp) { + return (mp->flags & P_SUBP) != 0; +} + +/* The version number for a database's lockfile format. */ +#define MDBX_LOCK_VERSION 6 + +#if MDBX_LOCKING == MDBX_LOCKING_WIN32FILES + +#define MDBX_LCK_SIGN UINT32_C(0xF10C) +typedef void osal_ipclock_t; +#elif MDBX_LOCKING == MDBX_LOCKING_SYSV + +#define MDBX_LCK_SIGN UINT32_C(0xF18D) +typedef mdbx_pid_t osal_ipclock_t; + +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || MDBX_LOCKING == MDBX_LOCKING_POSIX2008 + +#define MDBX_LCK_SIGN UINT32_C(0x8017) +typedef pthread_mutex_t osal_ipclock_t; + +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988 + +#define MDBX_LCK_SIGN UINT32_C(0xFC29) +typedef sem_t osal_ipclock_t; + +#else +#error "FIXME" +#endif /* MDBX_LOCKING */ + +/* Статистика профилирования работы GC */ +typedef struct gc_prof_stat { /* Монотонное время по "настенным часам" * затраченное на чтение и поиск внутри GC */ uint64_t rtime_monotonic; @@ -3017,42 +2544,44 @@ typedef struct profgc_stat { uint32_t spe_counter; /* page faults (hard page faults) */ uint32_t majflt; -} profgc_stat_t; - -/* Statistics of page operations overall of all (running, completed and aborted) - * transactions */ -typedef struct pgop_stat { - MDBX_atomic_uint64_t newly; /* Quantity of a new pages added */ - MDBX_atomic_uint64_t cow; /* Quantity of pages copied for update */ - MDBX_atomic_uint64_t clone; /* Quantity of parent's dirty pages clones + /* Для разборок с pnl_merge() */ + struct { + uint64_t time; + uint64_t volume; + uint32_t calls; + } pnl_merge; +} gc_prof_stat_t; + +/* Statistics of pages operations for all transactions, + * including incomplete and aborted. */ +typedef struct pgops { + mdbx_atomic_uint64_t newly; /* Quantity of a new pages added */ + mdbx_atomic_uint64_t cow; /* Quantity of pages copied for update */ + mdbx_atomic_uint64_t clone; /* Quantity of parent's dirty pages clones for nested transactions */ - MDBX_atomic_uint64_t split; /* Page splits */ - MDBX_atomic_uint64_t merge; /* Page merges */ - MDBX_atomic_uint64_t spill; /* Quantity of spilled dirty pages */ - MDBX_atomic_uint64_t unspill; /* Quantity of unspilled/reloaded pages */ - MDBX_atomic_uint64_t - wops; /* Number of explicit write operations (not a pages) to a disk */ - MDBX_atomic_uint64_t - msync; /* Number of explicit msync/flush-to-disk operations */ - MDBX_atomic_uint64_t - fsync; /* Number of explicit fsync/flush-to-disk operations */ - - MDBX_atomic_uint64_t prefault; /* Number of prefault write operations */ - MDBX_atomic_uint64_t mincore; /* Number of mincore() calls */ - - MDBX_atomic_uint32_t - incoherence; /* number of https://libmdbx.dqdkfa.ru/dead-github/issues/269 - caught */ - MDBX_atomic_uint32_t reserved; + mdbx_atomic_uint64_t split; /* Page splits */ + mdbx_atomic_uint64_t merge; /* Page merges */ + mdbx_atomic_uint64_t spill; /* Quantity of spilled dirty pages */ + mdbx_atomic_uint64_t unspill; /* Quantity of unspilled/reloaded pages */ + mdbx_atomic_uint64_t wops; /* Number of explicit write operations (not a pages) to a disk */ + mdbx_atomic_uint64_t msync; /* Number of explicit msync/flush-to-disk operations */ + mdbx_atomic_uint64_t fsync; /* Number of explicit fsync/flush-to-disk operations */ + + mdbx_atomic_uint64_t prefault; /* Number of prefault write operations */ + mdbx_atomic_uint64_t mincore; /* Number of mincore() calls */ + + mdbx_atomic_uint32_t incoherence; /* number of https://libmdbx.dqdkfa.ru/dead-github/issues/269 + caught */ + mdbx_atomic_uint32_t reserved; /* Статистика для профилирования GC. - * Логически эти данные может быть стоит вынести в другую структуру, + * Логически эти данные, возможно, стоит вынести в другую структуру, * но разница будет сугубо косметическая. */ struct { /* Затраты на поддержку данных пользователя */ - profgc_stat_t work; + gc_prof_stat_t work; /* Затраты на поддержку и обновления самой GC */ - profgc_stat_t self; + gc_prof_stat_t self; /* Итераций обновления GC, * больше 1 если были повторы/перезапуски */ uint32_t wloops; @@ -3067,33 +2596,6 @@ typedef struct pgop_stat { } gc_prof; } pgop_stat_t; -#if MDBX_LOCKING == MDBX_LOCKING_WIN32FILES -#define MDBX_CLOCK_SIGN UINT32_C(0xF10C) -typedef void osal_ipclock_t; -#elif MDBX_LOCKING == MDBX_LOCKING_SYSV - -#define MDBX_CLOCK_SIGN UINT32_C(0xF18D) -typedef mdbx_pid_t osal_ipclock_t; -#ifndef EOWNERDEAD -#define EOWNERDEAD MDBX_RESULT_TRUE -#endif - -#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ - MDBX_LOCKING == MDBX_LOCKING_POSIX2008 -#define MDBX_CLOCK_SIGN UINT32_C(0x8017) -typedef pthread_mutex_t osal_ipclock_t; -#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988 -#define MDBX_CLOCK_SIGN UINT32_C(0xFC29) -typedef sem_t osal_ipclock_t; -#else -#error "FIXME" -#endif /* MDBX_LOCKING */ - -#if MDBX_LOCKING > MDBX_LOCKING_SYSV && !defined(__cplusplus) -MDBX_INTERNAL_FUNC int osal_ipclock_stub(osal_ipclock_t *ipc); -MDBX_INTERNAL_FUNC int osal_ipclock_destroy(osal_ipclock_t *ipc); -#endif /* MDBX_LOCKING */ - /* Reader Lock Table * * Readers don't acquire any locks for their data access. Instead, they @@ -3103,8 +2605,9 @@ MDBX_INTERNAL_FUNC int osal_ipclock_destroy(osal_ipclock_t *ipc); * read transactions started by the same thread need no further locking to * proceed. * - * If MDBX_NOTLS is set, the slot address is not saved in thread-specific data. - * No reader table is used if the database is on a read-only filesystem. + * If MDBX_NOSTICKYTHREADS is set, the slot address is not saved in + * thread-specific data. No reader table is used if the database is on a + * read-only filesystem. * * Since the database uses multi-version concurrency control, readers don't * actually need any locking. This table is used to keep track of which @@ -3133,14 +2636,14 @@ MDBX_INTERNAL_FUNC int osal_ipclock_destroy(osal_ipclock_t *ipc); * many old transactions together. */ /* The actual reader record, with cacheline padding. */ -typedef struct MDBX_reader { - /* Current Transaction ID when this transaction began, or (txnid_t)-1. +typedef struct reader_slot { + /* Current Transaction ID when this transaction began, or INVALID_TXNID. * Multiple readers that start at the same time will probably have the * same ID here. Again, it's not important to exclude them from * anything; all we need to know is which version of the DB they * started from so we can avoid overwriting any data used in that * particular version. */ - MDBX_atomic_uint64_t /* txnid_t */ mr_txnid; + atomic_txnid_t txnid; /* The information we store in a single slot of the reader table. * In addition to a transaction ID, we also record the process and @@ -3151,708 +2654,320 @@ typedef struct MDBX_reader { * We simply re-init the table when we know that we're the only process * opening the lock file. */ + /* Псевдо thread_id для пометки вытесненных читающих транзакций. */ +#define MDBX_TID_TXN_OUSTED (UINT64_MAX - 1) + + /* Псевдо thread_id для пометки припаркованных читающих транзакций. */ +#define MDBX_TID_TXN_PARKED UINT64_MAX + /* The thread ID of the thread owning this txn. */ - MDBX_atomic_uint64_t mr_tid; + mdbx_atomic_uint64_t tid; /* The process ID of the process owning this reader txn. */ - MDBX_atomic_uint32_t mr_pid; + mdbx_atomic_uint32_t pid; /* The number of pages used in the reader's MVCC snapshot, - * i.e. the value of meta->mm_geo.next and txn->mt_next_pgno */ - atomic_pgno_t mr_snapshot_pages_used; + * i.e. the value of meta->geometry.first_unallocated and + * txn->geo.first_unallocated */ + atomic_pgno_t snapshot_pages_used; /* Number of retired pages at the time this reader starts transaction. So, - * at any time the difference mm_pages_retired - mr_snapshot_pages_retired - * will give the number of pages which this reader restraining from reuse. */ - MDBX_atomic_uint64_t mr_snapshot_pages_retired; -} MDBX_reader; + * at any time the difference meta.pages_retired - + * reader.snapshot_pages_retired will give the number of pages which this + * reader restraining from reuse. */ + mdbx_atomic_uint64_t snapshot_pages_retired; +} reader_slot_t; /* The header for the reader table (a memory-mapped lock file). */ -typedef struct MDBX_lockinfo { +typedef struct shared_lck { /* Stamp identifying this as an MDBX file. * It must be set to MDBX_MAGIC with with MDBX_LOCK_VERSION. */ - uint64_t mti_magic_and_version; + uint64_t magic_and_version; /* Format of this lock file. Must be set to MDBX_LOCK_FORMAT. */ - uint32_t mti_os_and_format; + uint32_t os_and_format; /* Flags which environment was opened. */ - MDBX_atomic_uint32_t mti_envmode; + mdbx_atomic_uint32_t envmode; /* Threshold of un-synced-with-disk pages for auto-sync feature, * zero means no-threshold, i.e. auto-sync is disabled. */ - atomic_pgno_t mti_autosync_threshold; + atomic_pgno_t autosync_threshold; /* Low 32-bit of txnid with which meta-pages was synced, * i.e. for sync-polling in the MDBX_NOMETASYNC mode. */ #define MDBX_NOMETASYNC_LAZY_UNK (UINT32_MAX / 3) #define MDBX_NOMETASYNC_LAZY_FD (MDBX_NOMETASYNC_LAZY_UNK + UINT32_MAX / 8) -#define MDBX_NOMETASYNC_LAZY_WRITEMAP \ - (MDBX_NOMETASYNC_LAZY_UNK - UINT32_MAX / 8) - MDBX_atomic_uint32_t mti_meta_sync_txnid; +#define MDBX_NOMETASYNC_LAZY_WRITEMAP (MDBX_NOMETASYNC_LAZY_UNK - UINT32_MAX / 8) + mdbx_atomic_uint32_t meta_sync_txnid; /* Period for timed auto-sync feature, i.e. at the every steady checkpoint - * the mti_unsynced_timeout sets to the current_time + mti_autosync_period. + * the mti_unsynced_timeout sets to the current_time + autosync_period. * The time value is represented in a suitable system-dependent form, for * example clock_gettime(CLOCK_BOOTTIME) or clock_gettime(CLOCK_MONOTONIC). * Zero means timed auto-sync is disabled. */ - MDBX_atomic_uint64_t mti_autosync_period; + mdbx_atomic_uint64_t autosync_period; /* Marker to distinguish uniqueness of DB/CLK. */ - MDBX_atomic_uint64_t mti_bait_uniqueness; + mdbx_atomic_uint64_t bait_uniqueness; /* Paired counter of processes that have mlock()ed part of mmapped DB. - * The (mti_mlcnt[0] - mti_mlcnt[1]) > 0 means at least one process + * The (mlcnt[0] - mlcnt[1]) > 0 means at least one process * lock at least one page, so therefore madvise() could return EINVAL. */ - MDBX_atomic_uint32_t mti_mlcnt[2]; + mdbx_atomic_uint32_t mlcnt[2]; MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/ /* Statistics of costly ops of all (running, completed and aborted) * transactions */ - pgop_stat_t mti_pgop_stat; + pgop_stat_t pgops; MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/ - /* Write transaction lock. */ #if MDBX_LOCKING > 0 - osal_ipclock_t mti_wlock; + /* Write transaction lock. */ + osal_ipclock_t wrt_lock; #endif /* MDBX_LOCKING > 0 */ - atomic_txnid_t mti_oldest_reader; + atomic_txnid_t cached_oldest; /* Timestamp of entering an out-of-sync state. Value is represented in a * suitable system-dependent form, for example clock_gettime(CLOCK_BOOTTIME) * or clock_gettime(CLOCK_MONOTONIC). */ - MDBX_atomic_uint64_t mti_eoos_timestamp; + mdbx_atomic_uint64_t eoos_timestamp; /* Number un-synced-with-disk pages for auto-sync feature. */ - MDBX_atomic_uint64_t mti_unsynced_pages; + mdbx_atomic_uint64_t unsynced_pages; /* Timestamp of the last readers check. */ - MDBX_atomic_uint64_t mti_reader_check_timestamp; + mdbx_atomic_uint64_t readers_check_timestamp; /* Number of page which was discarded last time by madvise(DONTNEED). */ - atomic_pgno_t mti_discarded_tail; + atomic_pgno_t discarded_tail; /* Shared anchor for tracking readahead edge and enabled/disabled status. */ - pgno_t mti_readahead_anchor; + pgno_t readahead_anchor; /* Shared cache for mincore() results */ struct { pgno_t begin[4]; uint64_t mask[4]; - } mti_mincore_cache; + } mincore_cache; MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/ - /* Readeaders registration lock. */ #if MDBX_LOCKING > 0 - osal_ipclock_t mti_rlock; + /* Readeaders table lock. */ + osal_ipclock_t rdt_lock; #endif /* MDBX_LOCKING > 0 */ /* The number of slots that have been used in the reader table. * This always records the maximum count, it is not decremented * when readers release their slots. */ - MDBX_atomic_uint32_t mti_numreaders; - MDBX_atomic_uint32_t mti_readers_refresh_flag; + mdbx_atomic_uint32_t rdt_length; + mdbx_atomic_uint32_t rdt_refresh_flag; -#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ - (!defined(__cplusplus) && defined(_MSC_VER)) +#if FLEXIBLE_ARRAY_MEMBERS MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/ - MDBX_reader mti_readers[] /* dynamic size */; -#endif /* C99 */ -} MDBX_lockinfo; + reader_slot_t rdt[] /* dynamic size */; /* Lockfile format signature: version, features and field layout */ -#define MDBX_LOCK_FORMAT \ - (MDBX_CLOCK_SIGN * 27733 + (unsigned)sizeof(MDBX_reader) * 13 + \ - (unsigned)offsetof(MDBX_reader, mr_snapshot_pages_used) * 251 + \ - (unsigned)offsetof(MDBX_lockinfo, mti_oldest_reader) * 83 + \ - (unsigned)offsetof(MDBX_lockinfo, mti_numreaders) * 37 + \ - (unsigned)offsetof(MDBX_lockinfo, mti_readers) * 29) - -#define MDBX_DATA_MAGIC \ - ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + MDBX_DATA_VERSION) - -#define MDBX_DATA_MAGIC_LEGACY_COMPAT \ - ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + 2) - -#define MDBX_DATA_MAGIC_LEGACY_DEVEL ((MDBX_MAGIC << 8) + 255) +#define MDBX_LOCK_FORMAT \ + (MDBX_LCK_SIGN * 27733 + (unsigned)sizeof(reader_slot_t) * 13 + \ + (unsigned)offsetof(reader_slot_t, snapshot_pages_used) * 251 + (unsigned)offsetof(lck_t, cached_oldest) * 83 + \ + (unsigned)offsetof(lck_t, rdt_length) * 37 + (unsigned)offsetof(lck_t, rdt) * 29) +#endif /* FLEXIBLE_ARRAY_MEMBERS */ +} lck_t; #define MDBX_LOCK_MAGIC ((MDBX_MAGIC << 8) + MDBX_LOCK_VERSION) -/* The maximum size of a database page. - * - * It is 64K, but value-PAGEHDRSZ must fit in MDBX_page.mp_upper. - * - * MDBX will use database pages < OS pages if needed. - * That causes more I/O in write transactions: The OS must - * know (read) the whole page before writing a partial page. - * - * Note that we don't currently support Huge pages. On Linux, - * regular data files cannot use Huge pages, and in general - * Huge pages aren't actually pageable. We rely on the OS - * demand-pager to read our data and page it out when memory - * pressure from other processes is high. So until OSs have - * actual paging support for Huge pages, they're not viable. */ -#define MAX_PAGESIZE MDBX_MAX_PAGESIZE -#define MIN_PAGESIZE MDBX_MIN_PAGESIZE - -#define MIN_MAPSIZE (MIN_PAGESIZE * MIN_PAGENO) +#define MDBX_READERS_LIMIT 32767 + +#define MIN_MAPSIZE (MDBX_MIN_PAGESIZE * MIN_PAGENO) #if defined(_WIN32) || defined(_WIN64) #define MAX_MAPSIZE32 UINT32_C(0x38000000) #else #define MAX_MAPSIZE32 UINT32_C(0x7f000000) #endif -#define MAX_MAPSIZE64 ((MAX_PAGENO + 1) * (uint64_t)MAX_PAGESIZE) +#define MAX_MAPSIZE64 ((MAX_PAGENO + 1) * (uint64_t)MDBX_MAX_PAGESIZE) #if MDBX_WORDBITS >= 64 #define MAX_MAPSIZE MAX_MAPSIZE64 -#define MDBX_PGL_LIMIT ((size_t)MAX_PAGENO) +#define PAGELIST_LIMIT ((size_t)MAX_PAGENO) #else #define MAX_MAPSIZE MAX_MAPSIZE32 -#define MDBX_PGL_LIMIT (MAX_MAPSIZE32 / MIN_PAGESIZE) +#define PAGELIST_LIMIT (MAX_MAPSIZE32 / MDBX_MIN_PAGESIZE) #endif /* MDBX_WORDBITS */ -#define MDBX_READERS_LIMIT 32767 -#define MDBX_RADIXSORT_THRESHOLD 142 #define MDBX_GOLD_RATIO_DBL 1.6180339887498948482 +#define MEGABYTE ((size_t)1 << 20) /*----------------------------------------------------------------------------*/ -/* An PNL is an Page Number List, a sorted array of IDs. - * The first element of the array is a counter for how many actual page-numbers - * are in the list. By default PNLs are sorted in descending order, this allow - * cut off a page with lowest pgno (at the tail) just truncating the list. The - * sort order of PNLs is controlled by the MDBX_PNL_ASCENDING build option. */ -typedef pgno_t *MDBX_PNL; - -#if MDBX_PNL_ASCENDING -#define MDBX_PNL_ORDERED(first, last) ((first) < (last)) -#define MDBX_PNL_DISORDERED(first, last) ((first) >= (last)) -#else -#define MDBX_PNL_ORDERED(first, last) ((first) > (last)) -#define MDBX_PNL_DISORDERED(first, last) ((first) <= (last)) -#endif +union logger_union { + void *ptr; + MDBX_debug_func *fmt; + MDBX_debug_func_nofmt *nofmt; +}; -/* List of txnid, only for MDBX_txn.tw.lifo_reclaimed */ -typedef txnid_t *MDBX_TXL; +struct libmdbx_globals { + bin128_t bootid; + unsigned sys_pagesize, sys_allocation_granularity; + uint8_t sys_pagesize_ln2; + uint8_t runtime_flags; + uint8_t loglevel; +#if defined(_WIN32) || defined(_WIN64) + bool running_under_Wine; +#elif defined(__linux__) || defined(__gnu_linux__) + bool running_on_WSL1 /* Windows Subsystem 1 for Linux */; + uint32_t linux_kernel_version; +#endif /* Linux */ + union logger_union logger; + osal_fastmutex_t debug_lock; + size_t logger_buffer_size; + char *logger_buffer; +}; -/* An Dirty-Page list item is an pgno/pointer pair. */ -typedef struct MDBX_dp { - MDBX_page *ptr; - pgno_t pgno, npages; -} MDBX_dp; +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ -/* An DPL (dirty-page list) is a sorted array of MDBX_DPs. */ -typedef struct MDBX_dpl { - size_t sorted; - size_t length; - size_t pages_including_loose; /* number of pages, but not an entries. */ - size_t detent; /* allocated size excluding the MDBX_DPL_RESERVE_GAP */ -#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ - (!defined(__cplusplus) && defined(_MSC_VER)) - MDBX_dp items[] /* dynamic size with holes at zero and after the last */; -#endif -} MDBX_dpl; +extern struct libmdbx_globals globals; +#if defined(_WIN32) || defined(_WIN64) +extern struct libmdbx_imports imports; +#endif /* Windows */ -/* PNL sizes */ -#define MDBX_PNL_GRANULATE_LOG2 10 -#define MDBX_PNL_GRANULATE (1 << MDBX_PNL_GRANULATE_LOG2) -#define MDBX_PNL_INITIAL \ - (MDBX_PNL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t)) +#ifndef __Wpedantic_format_voidptr +MDBX_MAYBE_UNUSED static inline const void *__Wpedantic_format_voidptr(const void *ptr) { return ptr; } +#define __Wpedantic_format_voidptr(ARG) __Wpedantic_format_voidptr(ARG) +#endif /* __Wpedantic_format_voidptr */ -#define MDBX_TXL_GRANULATE 32 -#define MDBX_TXL_INITIAL \ - (MDBX_TXL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t)) -#define MDBX_TXL_MAX \ - ((1u << 26) - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t)) +MDBX_INTERNAL void MDBX_PRINTF_ARGS(4, 5) debug_log(int level, const char *function, int line, const char *fmt, ...) + MDBX_PRINTF_ARGS(4, 5); +MDBX_INTERNAL void debug_log_va(int level, const char *function, int line, const char *fmt, va_list args); -#define MDBX_PNL_ALLOCLEN(pl) ((pl)[-1]) -#define MDBX_PNL_GETSIZE(pl) ((size_t)((pl)[0])) -#define MDBX_PNL_SETSIZE(pl, size) \ - do { \ - const size_t __size = size; \ - assert(__size < INT_MAX); \ - (pl)[0] = (pgno_t)__size; \ - } while (0) -#define MDBX_PNL_FIRST(pl) ((pl)[1]) -#define MDBX_PNL_LAST(pl) ((pl)[MDBX_PNL_GETSIZE(pl)]) -#define MDBX_PNL_BEGIN(pl) (&(pl)[1]) -#define MDBX_PNL_END(pl) (&(pl)[MDBX_PNL_GETSIZE(pl) + 1]) +#if MDBX_DEBUG +#define LOG_ENABLED(LVL) unlikely(LVL <= globals.loglevel) +#define AUDIT_ENABLED() unlikely((globals.runtime_flags & (unsigned)MDBX_DBG_AUDIT)) +#else /* MDBX_DEBUG */ +#define LOG_ENABLED(LVL) (LVL < MDBX_LOG_VERBOSE && LVL <= globals.loglevel) +#define AUDIT_ENABLED() (0) +#endif /* LOG_ENABLED() & AUDIT_ENABLED() */ -#if MDBX_PNL_ASCENDING -#define MDBX_PNL_EDGE(pl) ((pl) + 1) -#define MDBX_PNL_LEAST(pl) MDBX_PNL_FIRST(pl) -#define MDBX_PNL_MOST(pl) MDBX_PNL_LAST(pl) +#if MDBX_FORCE_ASSERTIONS +#define ASSERT_ENABLED() (1) +#elif MDBX_DEBUG +#define ASSERT_ENABLED() likely((globals.runtime_flags & (unsigned)MDBX_DBG_ASSERT)) #else -#define MDBX_PNL_EDGE(pl) ((pl) + MDBX_PNL_GETSIZE(pl)) -#define MDBX_PNL_LEAST(pl) MDBX_PNL_LAST(pl) -#define MDBX_PNL_MOST(pl) MDBX_PNL_FIRST(pl) -#endif - -#define MDBX_PNL_SIZEOF(pl) ((MDBX_PNL_GETSIZE(pl) + 1) * sizeof(pgno_t)) -#define MDBX_PNL_IS_EMPTY(pl) (MDBX_PNL_GETSIZE(pl) == 0) - -/*----------------------------------------------------------------------------*/ -/* Internal structures */ - -/* Auxiliary DB info. - * The information here is mostly static/read-only. There is - * only a single copy of this record in the environment. */ -typedef struct MDBX_dbx { - MDBX_val md_name; /* name of the database */ - MDBX_cmp_func *md_cmp; /* function for comparing keys */ - MDBX_cmp_func *md_dcmp; /* function for comparing data items */ - size_t md_klen_min, md_klen_max; /* min/max key length for the database */ - size_t md_vlen_min, - md_vlen_max; /* min/max value/data length for the database */ -} MDBX_dbx; - -typedef struct troika { - uint8_t fsm, recent, prefer_steady, tail_and_flags; -#if MDBX_WORDBITS > 32 /* Workaround for false-positives from Valgrind */ - uint32_t unused_pad; -#endif -#define TROIKA_HAVE_STEADY(troika) ((troika)->fsm & 7) -#define TROIKA_STRICT_VALID(troika) ((troika)->tail_and_flags & 64) -#define TROIKA_VALID(troika) ((troika)->tail_and_flags & 128) -#define TROIKA_TAIL(troika) ((troika)->tail_and_flags & 3) - txnid_t txnid[NUM_METAS]; -} meta_troika_t; - -/* A database transaction. - * Every operation requires a transaction handle. */ -struct MDBX_txn { -#define MDBX_MT_SIGNATURE UINT32_C(0x93D53A31) - uint32_t mt_signature; - - /* Transaction Flags */ - /* mdbx_txn_begin() flags */ -#define MDBX_TXN_RO_BEGIN_FLAGS (MDBX_TXN_RDONLY | MDBX_TXN_RDONLY_PREPARE) -#define MDBX_TXN_RW_BEGIN_FLAGS \ - (MDBX_TXN_NOMETASYNC | MDBX_TXN_NOSYNC | MDBX_TXN_TRY) - /* Additional flag for sync_locked() */ -#define MDBX_SHRINK_ALLOWED UINT32_C(0x40000000) - -#define MDBX_TXN_DRAINED_GC 0x20 /* GC was depleted up to oldest reader */ - -#define TXN_FLAGS \ - (MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_DIRTY | MDBX_TXN_SPILLS | \ - MDBX_TXN_HAS_CHILD | MDBX_TXN_INVALID | MDBX_TXN_DRAINED_GC) - -#if (TXN_FLAGS & (MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_RO_BEGIN_FLAGS)) || \ - ((MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_RO_BEGIN_FLAGS | TXN_FLAGS) & \ - MDBX_SHRINK_ALLOWED) -#error "Oops, some txn flags overlapped or wrong" -#endif - uint32_t mt_flags; - - MDBX_txn *mt_parent; /* parent of a nested txn */ - /* Nested txn under this txn, set together with flag MDBX_TXN_HAS_CHILD */ - MDBX_txn *mt_child; - MDBX_geo mt_geo; - /* next unallocated page */ -#define mt_next_pgno mt_geo.next - /* corresponding to the current size of datafile */ -#define mt_end_pgno mt_geo.now - - /* The ID of this transaction. IDs are integers incrementing from - * INITIAL_TXNID. Only committed write transactions increment the ID. If a - * transaction aborts, the ID may be re-used by the next writer. */ - txnid_t mt_txnid; - txnid_t mt_front; - - MDBX_env *mt_env; /* the DB environment */ - /* Array of records for each DB known in the environment. */ - MDBX_dbx *mt_dbxs; - /* Array of MDBX_db records for each known DB */ - MDBX_db *mt_dbs; - /* Array of sequence numbers for each DB handle */ - MDBX_atomic_uint32_t *mt_dbiseqs; - - /* Transaction DBI Flags */ -#define DBI_DIRTY MDBX_DBI_DIRTY /* DB was written in this txn */ -#define DBI_STALE MDBX_DBI_STALE /* Named-DB record is older than txnID */ -#define DBI_FRESH MDBX_DBI_FRESH /* Named-DB handle opened in this txn */ -#define DBI_CREAT MDBX_DBI_CREAT /* Named-DB handle created in this txn */ -#define DBI_VALID 0x10 /* DB handle is valid, see also DB_VALID */ -#define DBI_USRVALID 0x20 /* As DB_VALID, but not set for FREE_DBI */ -#define DBI_AUDITED 0x40 /* Internal flag for accounting during audit */ - /* Array of flags for each DB */ - uint8_t *mt_dbistate; - /* Number of DB records in use, or 0 when the txn is finished. - * This number only ever increments until the txn finishes; we - * don't decrement it when individual DB handles are closed. */ - MDBX_dbi mt_numdbs; - size_t mt_owner; /* thread ID that owns this transaction */ - MDBX_canary mt_canary; - void *mt_userctx; /* User-settable context */ - MDBX_cursor **mt_cursors; +#define ASSERT_ENABLED() (0) +#endif /* ASSERT_ENABLED() */ - union { - struct { - /* For read txns: This thread/txn's reader table slot, or NULL. */ - MDBX_reader *reader; - } to; - struct { - meta_troika_t troika; - /* In write txns, array of cursors for each DB */ - MDBX_PNL relist; /* Reclaimed GC pages */ - txnid_t last_reclaimed; /* ID of last used record */ -#if MDBX_ENABLE_REFUND - pgno_t loose_refund_wl /* FIXME: describe */; -#endif /* MDBX_ENABLE_REFUND */ - /* a sequence to spilling dirty page with LRU policy */ - unsigned dirtylru; - /* dirtylist room: Dirty array size - dirty pages visible to this txn. - * Includes ancestor txns' dirty pages not hidden by other txns' - * dirty/spilled pages. Thus commit(nested txn) has room to merge - * dirtylist into mt_parent after freeing hidden mt_parent pages. */ - size_t dirtyroom; - /* For write txns: Modified pages. Sorted when not MDBX_WRITEMAP. */ - MDBX_dpl *dirtylist; - /* The list of reclaimed txns from GC */ - MDBX_TXL lifo_reclaimed; - /* The list of pages that became unused during this transaction. */ - MDBX_PNL retired_pages; - /* The list of loose pages that became unused and may be reused - * in this transaction, linked through `mp_next`. */ - MDBX_page *loose_pages; - /* Number of loose pages (tw.loose_pages) */ - size_t loose_count; - union { - struct { - size_t least_removed; - /* The sorted list of dirty pages we temporarily wrote to disk - * because the dirty list was full. page numbers in here are - * shifted left by 1, deleted slots have the LSB set. */ - MDBX_PNL list; - } spilled; - size_t writemap_dirty_npages; - size_t writemap_spilled_npages; - }; - } tw; - }; -}; +#define DEBUG_EXTRA(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_EXTRA)) \ + debug_log(MDBX_LOG_EXTRA, __func__, __LINE__, fmt, __VA_ARGS__); \ + } while (0) -#if MDBX_WORDBITS >= 64 -#define CURSOR_STACK 32 -#else -#define CURSOR_STACK 24 -#endif - -struct MDBX_xcursor; - -/* Cursors are used for all DB operations. - * A cursor holds a path of (page pointer, key index) from the DB - * root to a position in the DB, plus other state. MDBX_DUPSORT - * cursors include an xcursor to the current data item. Write txns - * track their cursors and keep them up to date when data moves. - * Exception: An xcursor's pointer to a P_SUBP page can be stale. - * (A node with F_DUPDATA but no F_SUBDATA contains a subpage). */ -struct MDBX_cursor { -#define MDBX_MC_LIVE UINT32_C(0xFE05D5B1) -#define MDBX_MC_READY4CLOSE UINT32_C(0x2817A047) -#define MDBX_MC_WAIT4EOT UINT32_C(0x90E297A7) - uint32_t mc_signature; - /* The database handle this cursor operates on */ - MDBX_dbi mc_dbi; - /* Next cursor on this DB in this txn */ - MDBX_cursor *mc_next; - /* Backup of the original cursor if this cursor is a shadow */ - MDBX_cursor *mc_backup; - /* Context used for databases with MDBX_DUPSORT, otherwise NULL */ - struct MDBX_xcursor *mc_xcursor; - /* The transaction that owns this cursor */ - MDBX_txn *mc_txn; - /* The database record for this cursor */ - MDBX_db *mc_db; - /* The database auxiliary record for this cursor */ - MDBX_dbx *mc_dbx; - /* The mt_dbistate for this database */ - uint8_t *mc_dbistate; - uint8_t mc_snum; /* number of pushed pages */ - uint8_t mc_top; /* index of top page, normally mc_snum-1 */ - - /* Cursor state flags. */ -#define C_INITIALIZED 0x01 /* cursor has been initialized and is valid */ -#define C_EOF 0x02 /* No more data */ -#define C_SUB 0x04 /* Cursor is a sub-cursor */ -#define C_DEL 0x08 /* last op was a cursor_del */ -#define C_UNTRACK 0x10 /* Un-track cursor when closing */ -#define C_GCU \ - 0x20 /* Происходит подготовка к обновлению GC, поэтому \ - * можно брать страницы из GC даже для FREE_DBI */ - uint8_t mc_flags; - - /* Cursor checking flags. */ -#define CC_BRANCH 0x01 /* same as P_BRANCH for CHECK_LEAF_TYPE() */ -#define CC_LEAF 0x02 /* same as P_LEAF for CHECK_LEAF_TYPE() */ -#define CC_OVERFLOW 0x04 /* same as P_OVERFLOW for CHECK_LEAF_TYPE() */ -#define CC_UPDATING 0x08 /* update/rebalance pending */ -#define CC_SKIPORD 0x10 /* don't check keys ordering */ -#define CC_LEAF2 0x20 /* same as P_LEAF2 for CHECK_LEAF_TYPE() */ -#define CC_RETIRING 0x40 /* refs to child pages may be invalid */ -#define CC_PAGECHECK 0x80 /* perform page checking, see MDBX_VALIDATION */ - uint8_t mc_checking; - - MDBX_page *mc_pg[CURSOR_STACK]; /* stack of pushed pages */ - indx_t mc_ki[CURSOR_STACK]; /* stack of page indices */ -}; +#define DEBUG_EXTRA_PRINT(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_EXTRA)) \ + debug_log(MDBX_LOG_EXTRA, nullptr, 0, fmt, __VA_ARGS__); \ + } while (0) -#define CHECK_LEAF_TYPE(mc, mp) \ - (((PAGETYPE_WHOLE(mp) ^ (mc)->mc_checking) & \ - (CC_BRANCH | CC_LEAF | CC_OVERFLOW | CC_LEAF2)) == 0) - -/* Context for sorted-dup records. - * We could have gone to a fully recursive design, with arbitrarily - * deep nesting of sub-databases. But for now we only handle these - * levels - main DB, optional sub-DB, sorted-duplicate DB. */ -typedef struct MDBX_xcursor { - /* A sub-cursor for traversing the Dup DB */ - MDBX_cursor mx_cursor; - /* The database record for this Dup DB */ - MDBX_db mx_db; - /* The auxiliary DB record for this Dup DB */ - MDBX_dbx mx_dbx; -} MDBX_xcursor; - -typedef struct MDBX_cursor_couple { - MDBX_cursor outer; - void *mc_userctx; /* User-settable context */ - MDBX_xcursor inner; -} MDBX_cursor_couple; - -/* The database environment. */ -struct MDBX_env { - /* ----------------------------------------------------- mostly static part */ -#define MDBX_ME_SIGNATURE UINT32_C(0x9A899641) - MDBX_atomic_uint32_t me_signature; - /* Failed to update the meta page. Probably an I/O error. */ -#define MDBX_FATAL_ERROR UINT32_C(0x80000000) - /* Some fields are initialized. */ -#define MDBX_ENV_ACTIVE UINT32_C(0x20000000) - /* me_txkey is set */ -#define MDBX_ENV_TXKEY UINT32_C(0x10000000) - /* Legacy MDBX_MAPASYNC (prior v0.9) */ -#define MDBX_DEPRECATED_MAPASYNC UINT32_C(0x100000) - /* Legacy MDBX_COALESCE (prior v0.12) */ -#define MDBX_DEPRECATED_COALESCE UINT32_C(0x2000000) -#define ENV_INTERNAL_FLAGS (MDBX_FATAL_ERROR | MDBX_ENV_ACTIVE | MDBX_ENV_TXKEY) - uint32_t me_flags; - osal_mmap_t me_dxb_mmap; /* The main data file */ -#define me_map me_dxb_mmap.base -#define me_lazy_fd me_dxb_mmap.fd - mdbx_filehandle_t me_dsync_fd, me_fd4meta; -#if defined(_WIN32) || defined(_WIN64) -#define me_overlapped_fd me_ioring.overlapped_fd - HANDLE me_data_lock_event; -#endif /* Windows */ - osal_mmap_t me_lck_mmap; /* The lock file */ -#define me_lfd me_lck_mmap.fd - struct MDBX_lockinfo *me_lck; - - unsigned me_psize; /* DB page size, initialized from me_os_psize */ - uint16_t me_leaf_nodemax; /* max size of a leaf-node */ - uint16_t me_branch_nodemax; /* max size of a branch-node */ - uint16_t me_subpage_limit; - uint16_t me_subpage_room_threshold; - uint16_t me_subpage_reserve_prereq; - uint16_t me_subpage_reserve_limit; - atomic_pgno_t me_mlocked_pgno; - uint8_t me_psize2log; /* log2 of DB page size */ - int8_t me_stuck_meta; /* recovery-only: target meta page or less that zero */ - uint16_t me_merge_threshold, - me_merge_threshold_gc; /* pages emptier than this are candidates for - merging */ - unsigned me_os_psize; /* OS page size, from osal_syspagesize() */ - unsigned me_maxreaders; /* size of the reader table */ - MDBX_dbi me_maxdbs; /* size of the DB table */ - uint32_t me_pid; /* process ID of this env */ - osal_thread_key_t me_txkey; /* thread-key for readers */ - pathchar_t *me_pathname; /* path to the DB files */ - void *me_pbuf; /* scratch area for DUPSORT put() */ - MDBX_txn *me_txn0; /* preallocated write transaction */ - - MDBX_dbx *me_dbxs; /* array of static DB info */ - uint16_t *me_dbflags; /* array of flags from MDBX_db.md_flags */ - MDBX_atomic_uint32_t *me_dbiseqs; /* array of dbi sequence numbers */ - unsigned - me_maxgc_ov1page; /* Number of pgno_t fit in a single overflow page */ - unsigned me_maxgc_per_branch; - uint32_t me_live_reader; /* have liveness lock in reader table */ - void *me_userctx; /* User-settable context */ - MDBX_hsr_func *me_hsr_callback; /* Callback for kicking laggard readers */ - size_t me_madv_threshold; +#define TRACE(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_TRACE)) \ + debug_log(MDBX_LOG_TRACE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ + } while (0) - struct { - unsigned dp_reserve_limit; - unsigned rp_augment_limit; - unsigned dp_limit; - unsigned dp_initial; - uint8_t dp_loose_limit; - uint8_t spill_max_denominator; - uint8_t spill_min_denominator; - uint8_t spill_parent4child_denominator; - unsigned merge_threshold_16dot16_percent; -#if !(defined(_WIN32) || defined(_WIN64)) - unsigned writethrough_threshold; -#endif /* Windows */ - bool prefault_write; - union { - unsigned all; - /* tracks options with non-auto values but tuned by user */ - struct { - unsigned dp_limit : 1; - unsigned rp_augment_limit : 1; - unsigned prefault_write : 1; - } non_auto; - } flags; - } me_options; - - /* struct me_dbgeo used for accepting db-geo params from user for the new - * database creation, i.e. when mdbx_env_set_geometry() was called before - * mdbx_env_open(). */ - struct { - size_t lower; /* minimal size of datafile */ - size_t upper; /* maximal size of datafile */ - size_t now; /* current size of datafile */ - size_t grow; /* step to grow datafile */ - size_t shrink; /* threshold to shrink datafile */ - } me_dbgeo; - -#if MDBX_LOCKING == MDBX_LOCKING_SYSV - union { - key_t key; - int semid; - } me_sysv_ipc; -#endif /* MDBX_LOCKING == MDBX_LOCKING_SYSV */ - bool me_incore; +#define DEBUG(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_DEBUG)) \ + debug_log(MDBX_LOG_DEBUG, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ + } while (0) - MDBX_env *me_lcklist_next; +#define VERBOSE(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_VERBOSE)) \ + debug_log(MDBX_LOG_VERBOSE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ + } while (0) - /* --------------------------------------------------- mostly volatile part */ +#define NOTICE(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_NOTICE)) \ + debug_log(MDBX_LOG_NOTICE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ + } while (0) - MDBX_txn *me_txn; /* current write transaction */ - osal_fastmutex_t me_dbi_lock; - MDBX_dbi me_numdbs; /* number of DBs opened */ - bool me_prefault_write; +#define WARNING(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_WARN)) \ + debug_log(MDBX_LOG_WARN, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ + } while (0) - MDBX_page *me_dp_reserve; /* list of malloc'ed blocks for re-use */ - unsigned me_dp_reserve_len; - /* PNL of pages that became unused in a write txn */ - MDBX_PNL me_retired_pages; - osal_ioring_t me_ioring; +#undef ERROR /* wingdi.h \ + Yeah, morons from M$ put such definition to the public header. */ -#if defined(_WIN32) || defined(_WIN64) - osal_srwlock_t me_remap_guard; - /* Workaround for LockFileEx and WriteFile multithread bug */ - CRITICAL_SECTION me_windowsbug_lock; - char *me_pathname_char; /* cache of multi-byte representation of pathname - to the DB files */ -#else - osal_fastmutex_t me_remap_guard; -#endif +#define ERROR(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_ERROR)) \ + debug_log(MDBX_LOG_ERROR, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ + } while (0) - /* -------------------------------------------------------------- debugging */ +#define FATAL(fmt, ...) debug_log(MDBX_LOG_FATAL, __func__, __LINE__, fmt "\n", __VA_ARGS__); #if MDBX_DEBUG - MDBX_assert_func *me_assert_func; /* Callback for assertion failures */ -#endif -#ifdef MDBX_USE_VALGRIND - int me_valgrind_handle; -#endif -#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__) - MDBX_atomic_uint32_t me_ignore_EDEADLK; - pgno_t me_poison_edge; -#endif /* MDBX_USE_VALGRIND || __SANITIZE_ADDRESS__ */ +#define ASSERT_FAIL(env, msg, func, line) mdbx_assert_fail(env, msg, func, line) +#else /* MDBX_DEBUG */ +MDBX_NORETURN __cold void assert_fail(const char *msg, const char *func, unsigned line); +#define ASSERT_FAIL(env, msg, func, line) \ + do { \ + (void)(env); \ + assert_fail(msg, func, line); \ + } while (0) +#endif /* MDBX_DEBUG */ -#ifndef xMDBX_DEBUG_SPILLING -#define xMDBX_DEBUG_SPILLING 0 -#endif -#if xMDBX_DEBUG_SPILLING == 2 - size_t debug_dirtied_est, debug_dirtied_act; -#endif /* xMDBX_DEBUG_SPILLING */ +#define ENSURE_MSG(env, expr, msg) \ + do { \ + if (unlikely(!(expr))) \ + ASSERT_FAIL(env, msg, __func__, __LINE__); \ + } while (0) - /* ------------------------------------------------- stub for lck-less mode */ - MDBX_atomic_uint64_t - x_lckless_stub[(sizeof(MDBX_lockinfo) + MDBX_CACHELINE_SIZE - 1) / - sizeof(MDBX_atomic_uint64_t)]; -}; +#define ENSURE(env, expr) ENSURE_MSG(env, expr, #expr) -#ifndef __cplusplus -/*----------------------------------------------------------------------------*/ -/* Cache coherence and mmap invalidation */ +/* assert(3) variant in environment context */ +#define eASSERT(env, expr) \ + do { \ + if (ASSERT_ENABLED()) \ + ENSURE(env, expr); \ + } while (0) -#if MDBX_CPU_WRITEBACK_INCOHERENT -#define osal_flush_incoherent_cpu_writeback() osal_memory_barrier() -#else -#define osal_flush_incoherent_cpu_writeback() osal_compiler_barrier() -#endif /* MDBX_CPU_WRITEBACK_INCOHERENT */ +/* assert(3) variant in cursor context */ +#define cASSERT(mc, expr) eASSERT((mc)->txn->env, expr) -MDBX_MAYBE_UNUSED static __inline void -osal_flush_incoherent_mmap(const void *addr, size_t nbytes, - const intptr_t pagesize) { -#if MDBX_MMAP_INCOHERENT_FILE_WRITE - char *const begin = (char *)(-pagesize & (intptr_t)addr); - char *const end = - (char *)(-pagesize & (intptr_t)((char *)addr + nbytes + pagesize - 1)); - int err = msync(begin, end - begin, MS_SYNC | MS_INVALIDATE) ? errno : 0; - eASSERT(nullptr, err == 0); - (void)err; -#else - (void)pagesize; -#endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */ +/* assert(3) variant in transaction context */ +#define tASSERT(txn, expr) eASSERT((txn)->env, expr) -#if MDBX_MMAP_INCOHERENT_CPU_CACHE -#ifdef DCACHE - /* MIPS has cache coherency issues. - * Note: for any nbytes >= on-chip cache size, entire is flushed. */ - cacheflush((void *)addr, nbytes, DCACHE); -#else -#error "Oops, cacheflush() not available" -#endif /* DCACHE */ -#endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */ +#ifndef xMDBX_TOOLS /* Avoid using internal eASSERT() */ +#undef assert +#define assert(expr) eASSERT(nullptr, expr) +#endif -#if !MDBX_MMAP_INCOHERENT_FILE_WRITE && !MDBX_MMAP_INCOHERENT_CPU_CACHE - (void)addr; - (void)nbytes; +MDBX_MAYBE_UNUSED static inline void jitter4testing(bool tiny) { +#if MDBX_DEBUG + if (globals.runtime_flags & (unsigned)MDBX_DBG_JITTER) + osal_jitter(tiny); +#else + (void)tiny; #endif } -/*----------------------------------------------------------------------------*/ -/* Internal prototypes */ - -MDBX_INTERNAL_FUNC int cleanup_dead_readers(MDBX_env *env, int rlocked, - int *dead); -MDBX_INTERNAL_FUNC int rthc_alloc(osal_thread_key_t *key, MDBX_reader *begin, - MDBX_reader *end); -MDBX_INTERNAL_FUNC void rthc_remove(const osal_thread_key_t key); - -MDBX_INTERNAL_FUNC void global_ctor(void); -MDBX_INTERNAL_FUNC void osal_ctor(void); -MDBX_INTERNAL_FUNC void global_dtor(void); -MDBX_INTERNAL_FUNC void osal_dtor(void); -MDBX_INTERNAL_FUNC void thread_dtor(void *ptr); - -#endif /* !__cplusplus */ - -#define MDBX_IS_ERROR(rc) \ - ((rc) != MDBX_RESULT_TRUE && (rc) != MDBX_RESULT_FALSE) - -/* Internal error codes, not exposed outside libmdbx */ -#define MDBX_NO_ROOT (MDBX_LAST_ADDED_ERRCODE + 10) - -/* Debugging output value of a cursor DBI: Negative in a sub-cursor. */ -#define DDBI(mc) \ - (((mc)->mc_flags & C_SUB) ? -(int)(mc)->mc_dbi : (int)(mc)->mc_dbi) +MDBX_MAYBE_UNUSED MDBX_INTERNAL void page_list(page_t *mp); +MDBX_INTERNAL const char *pagetype_caption(const uint8_t type, char buf4unknown[16]); /* Key size which fits in a DKBUF (debug key buffer). */ -#define DKBUF_MAX 511 -#define DKBUF char _kbuf[DKBUF_MAX * 4 + 2] -#define DKEY(x) mdbx_dump_val(x, _kbuf, DKBUF_MAX * 2 + 1) -#define DVAL(x) mdbx_dump_val(x, _kbuf + DKBUF_MAX * 2 + 1, DKBUF_MAX * 2 + 1) +#define DKBUF_MAX 127 +#define DKBUF char dbg_kbuf[DKBUF_MAX * 4 + 2] +#define DKEY(x) mdbx_dump_val(x, dbg_kbuf, DKBUF_MAX * 2 + 1) +#define DVAL(x) mdbx_dump_val(x, dbg_kbuf + DKBUF_MAX * 2 + 1, DKBUF_MAX * 2 + 1) #if MDBX_DEBUG #define DKBUF_DEBUG DKBUF @@ -3864,102 +2979,24 @@ MDBX_INTERNAL_FUNC void thread_dtor(void *ptr); #define DVAL_DEBUG(x) ("-") #endif -/* An invalid page number. - * Mainly used to denote an empty tree. */ -#define P_INVALID (~(pgno_t)0) +MDBX_INTERNAL void log_error(const int err, const char *func, unsigned line); + +MDBX_MAYBE_UNUSED static inline int log_if_error(const int err, const char *func, unsigned line) { + if (unlikely(err != MDBX_SUCCESS)) + log_error(err, func, line); + return err; +} + +#define LOG_IFERR(err) log_if_error((err), __func__, __LINE__) /* Test if the flags f are set in a flag word w. */ #define F_ISSET(w, f) (((w) & (f)) == (f)) /* Round n up to an even number. */ -#define EVEN(n) (((n) + 1UL) & -2L) /* sign-extending -2 to match n+1U */ - -/* Default size of memory map. - * This is certainly too small for any actual applications. Apps should - * always set the size explicitly using mdbx_env_set_geometry(). */ -#define DEFAULT_MAPSIZE MEGABYTE - -/* Number of slots in the reader table. - * This value was chosen somewhat arbitrarily. The 61 is a prime number, - * and such readers plus a couple mutexes fit into single 4KB page. - * Applications should set the table size using mdbx_env_set_maxreaders(). */ -#define DEFAULT_READERS 61 - -/* Test if a page is a leaf page */ -#define IS_LEAF(p) (((p)->mp_flags & P_LEAF) != 0) -/* Test if a page is a LEAF2 page */ -#define IS_LEAF2(p) unlikely(((p)->mp_flags & P_LEAF2) != 0) -/* Test if a page is a branch page */ -#define IS_BRANCH(p) (((p)->mp_flags & P_BRANCH) != 0) -/* Test if a page is an overflow page */ -#define IS_OVERFLOW(p) unlikely(((p)->mp_flags & P_OVERFLOW) != 0) -/* Test if a page is a sub page */ -#define IS_SUBP(p) (((p)->mp_flags & P_SUBP) != 0) - -/* Header for a single key/data pair within a page. - * Used in pages of type P_BRANCH and P_LEAF without P_LEAF2. - * We guarantee 2-byte alignment for 'MDBX_node's. - * - * Leaf node flags describe node contents. F_BIGDATA says the node's - * data part is the page number of an overflow page with actual data. - * F_DUPDATA and F_SUBDATA can be combined giving duplicate data in - * a sub-page/sub-database, and named databases (just F_SUBDATA). */ -typedef struct MDBX_node { -#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ - union { - uint32_t mn_dsize; - uint32_t mn_pgno32; - }; - uint8_t mn_flags; /* see mdbx_node flags */ - uint8_t mn_extra; - uint16_t mn_ksize; /* key size */ -#else - uint16_t mn_ksize; /* key size */ - uint8_t mn_extra; - uint8_t mn_flags; /* see mdbx_node flags */ - union { - uint32_t mn_pgno32; - uint32_t mn_dsize; - }; -#endif /* __BYTE_ORDER__ */ - - /* mdbx_node Flags */ -#define F_BIGDATA 0x01 /* data put on overflow page */ -#define F_SUBDATA 0x02 /* data is a sub-database */ -#define F_DUPDATA 0x04 /* data has duplicates */ - - /* valid flags for mdbx_node_add() */ -#define NODE_ADD_FLAGS (F_DUPDATA | F_SUBDATA | MDBX_RESERVE | MDBX_APPEND) - -#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ - (!defined(__cplusplus) && defined(_MSC_VER)) - uint8_t mn_data[] /* key and data are appended here */; -#endif /* C99 */ -} MDBX_node; - -#define DB_PERSISTENT_FLAGS \ - (MDBX_REVERSEKEY | MDBX_DUPSORT | MDBX_INTEGERKEY | MDBX_DUPFIXED | \ - MDBX_INTEGERDUP | MDBX_REVERSEDUP) - -/* mdbx_dbi_open() flags */ -#define DB_USABLE_FLAGS (DB_PERSISTENT_FLAGS | MDBX_CREATE | MDBX_DB_ACCEDE) +#define EVEN_CEIL(n) (((n) + 1UL) & -2L) /* sign-extending -2 to match n+1U */ -#define DB_VALID 0x8000 /* DB handle is valid, for me_dbflags */ -#define DB_INTERNAL_FLAGS DB_VALID - -#if DB_INTERNAL_FLAGS & DB_USABLE_FLAGS -#error "Oops, some flags overlapped or wrong" -#endif -#if DB_PERSISTENT_FLAGS & ~DB_USABLE_FLAGS -#error "Oops, some flags overlapped or wrong" -#endif - -/* Max length of iov-vector passed to writev() call, used for auxilary writes */ -#define MDBX_AUXILARY_IOV_MAX 64 -#if defined(IOV_MAX) && IOV_MAX < MDBX_AUXILARY_IOV_MAX -#undef MDBX_AUXILARY_IOV_MAX -#define MDBX_AUXILARY_IOV_MAX IOV_MAX -#endif /* MDBX_AUXILARY_IOV_MAX */ +/* Round n down to an even number. */ +#define EVEN_FLOOR(n) ((n) & ~(size_t)1) /* * / @@ -3970,106 +3007,226 @@ typedef struct MDBX_node { */ #define CMP2INT(a, b) (((a) != (b)) ? (((a) < (b)) ? -1 : 1) : 0) -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline pgno_t -int64pgno(int64_t i64) { - if (likely(i64 >= (int64_t)MIN_PAGENO && i64 <= (int64_t)MAX_PAGENO + 1)) - return (pgno_t)i64; - return (i64 < (int64_t)MIN_PAGENO) ? MIN_PAGENO : MAX_PAGENO; -} +/* Pointer displacement without casting to char* to avoid pointer-aliasing */ +#define ptr_disp(ptr, disp) ((void *)(((intptr_t)(ptr)) + ((intptr_t)(disp)))) -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline pgno_t -pgno_add(size_t base, size_t augend) { - assert(base <= MAX_PAGENO + 1 && augend < MAX_PAGENO); - return int64pgno((int64_t)base + (int64_t)augend); -} +/* Pointer distance as signed number of bytes */ +#define ptr_dist(more, less) (((intptr_t)(more)) - ((intptr_t)(less))) -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline pgno_t -pgno_sub(size_t base, size_t subtrahend) { - assert(base >= MIN_PAGENO && base <= MAX_PAGENO + 1 && - subtrahend < MAX_PAGENO); - return int64pgno((int64_t)base - (int64_t)subtrahend); -} +#define MDBX_ASAN_POISON_MEMORY_REGION(addr, size) \ + do { \ + TRACE("POISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), (size_t)(size), __LINE__); \ + ASAN_POISON_MEMORY_REGION(addr, size); \ + } while (0) -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __always_inline bool -is_powerof2(size_t x) { - return (x & (x - 1)) == 0; +#define MDBX_ASAN_UNPOISON_MEMORY_REGION(addr, size) \ + do { \ + TRACE("UNPOISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), (size_t)(size), __LINE__); \ + ASAN_UNPOISON_MEMORY_REGION(addr, size); \ + } while (0) + +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline size_t branchless_abs(intptr_t value) { + assert(value > INT_MIN); + const size_t expanded_sign = (size_t)(value >> (sizeof(value) * CHAR_BIT - 1)); + return ((size_t)value + expanded_sign) ^ expanded_sign; } -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __always_inline size_t -floor_powerof2(size_t value, size_t granularity) { +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline bool is_powerof2(size_t x) { return (x & (x - 1)) == 0; } + +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline size_t floor_powerof2(size_t value, size_t granularity) { assert(is_powerof2(granularity)); return value & ~(granularity - 1); } -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __always_inline size_t -ceil_powerof2(size_t value, size_t granularity) { +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline size_t ceil_powerof2(size_t value, size_t granularity) { return floor_powerof2(value + granularity - 1, granularity); } -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static unsigned -log2n_powerof2(size_t value_uintptr) { - assert(value_uintptr > 0 && value_uintptr < INT32_MAX && - is_powerof2(value_uintptr)); - assert((value_uintptr & -(intptr_t)value_uintptr) == value_uintptr); - const uint32_t value_uint32 = (uint32_t)value_uintptr; -#if __GNUC_PREREQ(4, 1) || __has_builtin(__builtin_ctz) - STATIC_ASSERT(sizeof(value_uint32) <= sizeof(unsigned)); - return __builtin_ctz(value_uint32); -#elif defined(_MSC_VER) - unsigned long index; - STATIC_ASSERT(sizeof(value_uint32) <= sizeof(long)); - _BitScanForward(&index, value_uint32); - return index; +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED MDBX_INTERNAL unsigned log2n_powerof2(size_t value_uintptr); + +MDBX_NOTHROW_CONST_FUNCTION MDBX_INTERNAL uint64_t rrxmrrxmsx_0(uint64_t v); + +struct monotime_cache { + uint64_t value; + int expire_countdown; +}; + +MDBX_MAYBE_UNUSED static inline uint64_t monotime_since_cached(uint64_t begin_timestamp, struct monotime_cache *cache) { + if (cache->expire_countdown) + cache->expire_countdown -= 1; + else { + cache->value = osal_monotime(); + cache->expire_countdown = 42 / 3; + } + return cache->value - begin_timestamp; +} + +/* An PNL is an Page Number List, a sorted array of IDs. + * + * The first element of the array is a counter for how many actual page-numbers + * are in the list. By default PNLs are sorted in descending order, this allow + * cut off a page with lowest pgno (at the tail) just truncating the list. The + * sort order of PNLs is controlled by the MDBX_PNL_ASCENDING build option. */ +typedef pgno_t *pnl_t; +typedef const pgno_t *const_pnl_t; + +#if MDBX_PNL_ASCENDING +#define MDBX_PNL_ORDERED(first, last) ((first) < (last)) +#define MDBX_PNL_DISORDERED(first, last) ((first) >= (last)) #else - static const uint8_t debruijn_ctz32[32] = { - 0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8, - 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9}; - return debruijn_ctz32[(uint32_t)(value_uint32 * 0x077CB531ul) >> 27]; +#define MDBX_PNL_ORDERED(first, last) ((first) > (last)) +#define MDBX_PNL_DISORDERED(first, last) ((first) <= (last)) #endif + +#define MDBX_PNL_GRANULATE_LOG2 10 +#define MDBX_PNL_GRANULATE (1 << MDBX_PNL_GRANULATE_LOG2) +#define MDBX_PNL_INITIAL (MDBX_PNL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t)) + +#define MDBX_PNL_ALLOCLEN(pl) ((pl)[-1]) +#define MDBX_PNL_GETSIZE(pl) ((size_t)((pl)[0])) +#define MDBX_PNL_SETSIZE(pl, size) \ + do { \ + const size_t __size = size; \ + assert(__size < INT_MAX); \ + (pl)[0] = (pgno_t)__size; \ + } while (0) +#define MDBX_PNL_FIRST(pl) ((pl)[1]) +#define MDBX_PNL_LAST(pl) ((pl)[MDBX_PNL_GETSIZE(pl)]) +#define MDBX_PNL_BEGIN(pl) (&(pl)[1]) +#define MDBX_PNL_END(pl) (&(pl)[MDBX_PNL_GETSIZE(pl) + 1]) + +#if MDBX_PNL_ASCENDING +#define MDBX_PNL_EDGE(pl) ((pl) + 1) +#define MDBX_PNL_LEAST(pl) MDBX_PNL_FIRST(pl) +#define MDBX_PNL_MOST(pl) MDBX_PNL_LAST(pl) +#else +#define MDBX_PNL_EDGE(pl) ((pl) + MDBX_PNL_GETSIZE(pl)) +#define MDBX_PNL_LEAST(pl) MDBX_PNL_LAST(pl) +#define MDBX_PNL_MOST(pl) MDBX_PNL_FIRST(pl) +#endif + +#define MDBX_PNL_SIZEOF(pl) ((MDBX_PNL_GETSIZE(pl) + 1) * sizeof(pgno_t)) +#define MDBX_PNL_IS_EMPTY(pl) (MDBX_PNL_GETSIZE(pl) == 0) + +MDBX_MAYBE_UNUSED static inline size_t pnl_size2bytes(size_t size) { + assert(size > 0 && size <= PAGELIST_LIMIT); +#if MDBX_PNL_PREALLOC_FOR_RADIXSORT + + size += size; +#endif /* MDBX_PNL_PREALLOC_FOR_RADIXSORT */ + STATIC_ASSERT(MDBX_ASSUME_MALLOC_OVERHEAD + + (PAGELIST_LIMIT * (MDBX_PNL_PREALLOC_FOR_RADIXSORT + 1) + MDBX_PNL_GRANULATE + 3) * sizeof(pgno_t) < + SIZE_MAX / 4 * 3); + size_t bytes = + ceil_powerof2(MDBX_ASSUME_MALLOC_OVERHEAD + sizeof(pgno_t) * (size + 3), MDBX_PNL_GRANULATE * sizeof(pgno_t)) - + MDBX_ASSUME_MALLOC_OVERHEAD; + return bytes; +} + +MDBX_MAYBE_UNUSED static inline pgno_t pnl_bytes2size(const size_t bytes) { + size_t size = bytes / sizeof(pgno_t); + assert(size > 3 && size <= PAGELIST_LIMIT + /* alignment gap */ 65536); + size -= 3; +#if MDBX_PNL_PREALLOC_FOR_RADIXSORT + size >>= 1; +#endif /* MDBX_PNL_PREALLOC_FOR_RADIXSORT */ + return (pgno_t)size; +} + +MDBX_INTERNAL pnl_t pnl_alloc(size_t size); + +MDBX_INTERNAL void pnl_free(pnl_t pnl); + +MDBX_INTERNAL int pnl_reserve(pnl_t __restrict *__restrict ppnl, const size_t wanna); + +MDBX_MAYBE_UNUSED static inline int __must_check_result pnl_need(pnl_t __restrict *__restrict ppnl, size_t num) { + assert(MDBX_PNL_GETSIZE(*ppnl) <= PAGELIST_LIMIT && MDBX_PNL_ALLOCLEN(*ppnl) >= MDBX_PNL_GETSIZE(*ppnl)); + assert(num <= PAGELIST_LIMIT); + const size_t wanna = MDBX_PNL_GETSIZE(*ppnl) + num; + return likely(MDBX_PNL_ALLOCLEN(*ppnl) >= wanna) ? MDBX_SUCCESS : pnl_reserve(ppnl, wanna); +} + +MDBX_MAYBE_UNUSED static inline void pnl_append_prereserved(__restrict pnl_t pnl, pgno_t pgno) { + assert(MDBX_PNL_GETSIZE(pnl) < MDBX_PNL_ALLOCLEN(pnl)); + if (AUDIT_ENABLED()) { + for (size_t i = MDBX_PNL_GETSIZE(pnl); i > 0; --i) + assert(pgno != pnl[i]); + } + *pnl += 1; + MDBX_PNL_LAST(pnl) = pgno; +} + +MDBX_INTERNAL void pnl_shrink(pnl_t __restrict *__restrict ppnl); + +MDBX_INTERNAL int __must_check_result spill_append_span(__restrict pnl_t *ppnl, pgno_t pgno, size_t n); + +MDBX_INTERNAL int __must_check_result pnl_append_span(__restrict pnl_t *ppnl, pgno_t pgno, size_t n); + +MDBX_INTERNAL int __must_check_result pnl_insert_span(__restrict pnl_t *ppnl, pgno_t pgno, size_t n); + +MDBX_INTERNAL size_t pnl_search_nochk(const pnl_t pnl, pgno_t pgno); + +MDBX_INTERNAL void pnl_sort_nochk(pnl_t pnl); + +MDBX_INTERNAL bool pnl_check(const const_pnl_t pnl, const size_t limit); + +MDBX_MAYBE_UNUSED static inline bool pnl_check_allocated(const const_pnl_t pnl, const size_t limit) { + return pnl == nullptr || (MDBX_PNL_ALLOCLEN(pnl) >= MDBX_PNL_GETSIZE(pnl) && pnl_check(pnl, limit)); } -/* Only a subset of the mdbx_env flags can be changed - * at runtime. Changing other flags requires closing the - * environment and re-opening it with the new flags. */ -#define ENV_CHANGEABLE_FLAGS \ - (MDBX_SAFE_NOSYNC | MDBX_NOMETASYNC | MDBX_DEPRECATED_MAPASYNC | \ - MDBX_NOMEMINIT | MDBX_COALESCE | MDBX_PAGEPERTURB | MDBX_ACCEDE | \ - MDBX_VALIDATION) -#define ENV_CHANGELESS_FLAGS \ - (MDBX_NOSUBDIR | MDBX_RDONLY | MDBX_WRITEMAP | MDBX_NOTLS | MDBX_NORDAHEAD | \ - MDBX_LIFORECLAIM | MDBX_EXCLUSIVE) -#define ENV_USABLE_FLAGS (ENV_CHANGEABLE_FLAGS | ENV_CHANGELESS_FLAGS) - -#if !defined(__cplusplus) || CONSTEXPR_ENUM_FLAGS_OPERATIONS -MDBX_MAYBE_UNUSED static void static_checks(void) { - STATIC_ASSERT_MSG(INT16_MAX - CORE_DBS == MDBX_MAX_DBI, - "Oops, MDBX_MAX_DBI or CORE_DBS?"); - STATIC_ASSERT_MSG((unsigned)(MDBX_DB_ACCEDE | MDBX_CREATE) == - ((DB_USABLE_FLAGS | DB_INTERNAL_FLAGS) & - (ENV_USABLE_FLAGS | ENV_INTERNAL_FLAGS)), - "Oops, some flags overlapped or wrong"); - STATIC_ASSERT_MSG((ENV_INTERNAL_FLAGS & ENV_USABLE_FLAGS) == 0, - "Oops, some flags overlapped or wrong"); +MDBX_MAYBE_UNUSED static inline void pnl_sort(pnl_t pnl, size_t limit4check) { + pnl_sort_nochk(pnl); + assert(pnl_check(pnl, limit4check)); + (void)limit4check; } -#endif /* Disabled for MSVC 19.0 (VisualStudio 2015) */ + +MDBX_MAYBE_UNUSED static inline size_t pnl_search(const pnl_t pnl, pgno_t pgno, size_t limit) { + assert(pnl_check_allocated(pnl, limit)); + if (MDBX_HAVE_CMOV) { + /* cmov-ускоренный бинарный поиск может читать (но не использовать) один + * элемент за концом данных, этот элемент в пределах выделенного участка + * памяти, но не инициализирован. */ + VALGRIND_MAKE_MEM_DEFINED(MDBX_PNL_END(pnl), sizeof(pgno_t)); + } + assert(pgno < limit); + (void)limit; + size_t n = pnl_search_nochk(pnl, pgno); + if (MDBX_HAVE_CMOV) { + VALGRIND_MAKE_MEM_UNDEFINED(MDBX_PNL_END(pnl), sizeof(pgno_t)); + } + return n; +} + +MDBX_INTERNAL size_t pnl_merge(pnl_t dst, const pnl_t src); #ifdef __cplusplus } +#endif /* __cplusplus */ + +#define mdbx_sourcery_anchor XCONCAT(mdbx_sourcery_, MDBX_BUILD_SOURCERY) +#if defined(xMDBX_TOOLS) +extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #endif -#define MDBX_ASAN_POISON_MEMORY_REGION(addr, size) \ - do { \ - TRACE("POISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), \ - (size_t)(size), __LINE__); \ - ASAN_POISON_MEMORY_REGION(addr, size); \ - } while (0) +#define MDBX_IS_ERROR(rc) ((rc) != MDBX_RESULT_TRUE && (rc) != MDBX_RESULT_FALSE) -#define MDBX_ASAN_UNPOISON_MEMORY_REGION(addr, size) \ - do { \ - TRACE("UNPOISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), \ - (size_t)(size), __LINE__); \ - ASAN_UNPOISON_MEMORY_REGION(addr, size); \ - } while (0) +/*----------------------------------------------------------------------------*/ + +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline pgno_t int64pgno(int64_t i64) { + if (likely(i64 >= (int64_t)MIN_PAGENO && i64 <= (int64_t)MAX_PAGENO + 1)) + return (pgno_t)i64; + return (i64 < (int64_t)MIN_PAGENO) ? MIN_PAGENO : MAX_PAGENO; +} + +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline pgno_t pgno_add(size_t base, size_t augend) { + assert(base <= MAX_PAGENO + 1 && augend < MAX_PAGENO); + return int64pgno((int64_t)base + (int64_t)augend); +} + +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline pgno_t pgno_sub(size_t base, size_t subtrahend) { + assert(base >= MIN_PAGENO && base <= MAX_PAGENO + 1 && subtrahend < MAX_PAGENO); + return int64pgno((int64_t)base - (int64_t)subtrahend); +} #if defined(_WIN32) || defined(_WIN64) /* @@ -4085,12 +3242,12 @@ MDBX_MAYBE_UNUSED static void static_checks(void) { #ifdef _MSC_VER #pragma warning(push, 1) -#pragma warning(disable : 4548) /* expression before comma has no effect; \ +#pragma warning(disable : 4548) /* expression before comma has no effect; \ expected expression with side - effect */ -#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ +#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ * semantics are not enabled. Specify /EHsc */ -#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ - * mode specified; termination on exception is \ +#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ + * mode specified; termination on exception is \ * not guaranteed. Specify /EHsc */ #if !defined(_CRT_SECURE_NO_WARNINGS) #define _CRT_SECURE_NO_WARNINGS @@ -4143,8 +3300,7 @@ int getopt(int argc, char *const argv[], const char *opts) { if (argv[optind][sp + 1] != '\0') optarg = &argv[optind++][sp + 1]; else if (++optind >= argc) { - fprintf(stderr, "%s: %s -- %c\n", argv[0], "option requires an argument", - c); + fprintf(stderr, "%s: %s -- %c\n", argv[0], "option requires an argument", c); sp = 1; return '?'; } else @@ -4178,23 +3334,40 @@ static void signal_handler(int sig) { #endif /* !WINDOWS */ static void usage(const char *prog) { - fprintf( - stderr, - "usage: %s [-V] [-q] [-c] [-u|U] src_path [dest_path]\n" - " -V\t\tprint version and exit\n" - " -q\t\tbe quiet\n" - " -c\t\tenable compactification (skip unused pages)\n" - " -u\t\twarmup database before copying\n" - " -U\t\twarmup and try lock database pages in memory before copying\n" - " src_path\tsource database\n" - " dest_path\tdestination (stdout if not specified)\n", - prog); + fprintf(stderr, + "usage: %s [-V] [-q] [-c] [-d] [-p] [-u|U] src_path [dest_path]\n" + " -V\t\tprint version and exit\n" + " -q\t\tbe quiet\n" + " -c\t\tenable compactification (skip unused pages)\n" + " -d\t\tenforce copy to be a dynamic size DB\n" + " -p\t\tusing transaction parking/ousting during copying MVCC-snapshot\n" + " \t\tto avoid stopping recycling and overflowing the DB\n" + " -u\t\twarmup database before copying\n" + " -U\t\twarmup and try lock database pages in memory before copying\n" + " src_path\tsource database\n" + " dest_path\tdestination (stdout if not specified)\n", + prog); exit(EXIT_FAILURE); } +static void logger(MDBX_log_level_t level, const char *function, int line, const char *fmt, va_list args) { + static const char *const prefixes[] = { + "!!!fatal: ", // 0 fatal + " ! ", // 1 error + " ~ ", // 2 warning + " ", // 3 notice + " //", // 4 verbose + }; + if (level < MDBX_LOG_DEBUG) { + if (function && line) + fprintf(stderr, "%s", prefixes[level]); + vfprintf(stderr, fmt, args); + } +} + int main(int argc, char *argv[]) { int rc; - MDBX_env *env = NULL; + MDBX_env *env = nullptr; const char *progname = argv[0], *act; unsigned flags = MDBX_RDONLY; unsigned cpflags = 0; @@ -4207,16 +3380,18 @@ int main(int argc, char *argv[]) { flags |= MDBX_NOSUBDIR; else if (argv[1][1] == 'c' && argv[1][2] == '\0') cpflags |= MDBX_CP_COMPACT; + else if (argv[1][1] == 'd' && argv[1][2] == '\0') + cpflags |= MDBX_CP_FORCE_DYNAMIC_SIZE; + else if (argv[1][1] == 'p' && argv[1][2] == '\0') + cpflags |= MDBX_CP_THROTTLE_MVCC; else if (argv[1][1] == 'q' && argv[1][2] == '\0') quiet = true; else if (argv[1][1] == 'u' && argv[1][2] == '\0') warmup = true; else if (argv[1][1] == 'U' && argv[1][2] == '\0') { warmup = true; - warmup_flags = - MDBX_warmup_force | MDBX_warmup_touchlimit | MDBX_warmup_lock; - } else if ((argv[1][1] == 'h' && argv[1][2] == '\0') || - strcmp(argv[1], "--help") == 0) + warmup_flags = MDBX_warmup_force | MDBX_warmup_touchlimit | MDBX_warmup_lock; + } else if ((argv[1][1] == 'h' && argv[1][2] == '\0') || strcmp(argv[1], "--help") == 0) usage(progname); else if (argv[1][1] == 'V' && argv[1][2] == '\0') { printf("mdbx_copy version %d.%d.%d.%d\n" @@ -4225,12 +3400,9 @@ int main(int argc, char *argv[]) { " - build: %s for %s by %s\n" " - flags: %s\n" " - options: %s\n", - mdbx_version.major, mdbx_version.minor, mdbx_version.release, - mdbx_version.revision, mdbx_version.git.describe, - mdbx_version.git.datetime, mdbx_version.git.commit, - mdbx_version.git.tree, mdbx_sourcery_anchor, mdbx_build.datetime, - mdbx_build.target, mdbx_build.compiler, mdbx_build.flags, - mdbx_build.options); + mdbx_version.major, mdbx_version.minor, mdbx_version.patch, mdbx_version.tweak, mdbx_version.git.describe, + mdbx_version.git.datetime, mdbx_version.git.commit, mdbx_version.git.tree, mdbx_sourcery_anchor, + mdbx_build.datetime, mdbx_build.target, mdbx_build.compiler, mdbx_build.flags, mdbx_build.options); return EXIT_SUCCESS; } else argc = 0; @@ -4253,11 +3425,11 @@ int main(int argc, char *argv[]) { #endif /* !WINDOWS */ if (!quiet) { - fprintf((argc == 2) ? stderr : stdout, - "mdbx_copy %s (%s, T-%s)\nRunning for copy %s to %s...\n", - mdbx_version.git.describe, mdbx_version.git.datetime, - mdbx_version.git.tree, argv[1], (argc == 2) ? "stdout" : argv[2]); - fflush(NULL); + fprintf((argc == 2) ? stderr : stdout, "mdbx_copy %s (%s, T-%s)\nRunning for copy %s to %s...\n", + mdbx_version.git.describe, mdbx_version.git.datetime, mdbx_version.git.tree, argv[1], + (argc == 2) ? "stdout" : argv[2]); + fflush(nullptr); + mdbx_setup_debug(MDBX_LOG_NOTICE, MDBX_DBG_DONTCHANGE, logger); } act = "opening environment"; @@ -4284,8 +3456,7 @@ int main(int argc, char *argv[]) { rc = mdbx_env_copy(env, argv[2], cpflags); } if (rc) - fprintf(stderr, "%s: %s failed, error %d (%s)\n", progname, act, rc, - mdbx_strerror(rc)); + fprintf(stderr, "%s: %s failed, error %d (%s)\n", progname, act, rc, mdbx_strerror(rc)); mdbx_env_close(env); return rc ? EXIT_FAILURE : EXIT_SUCCESS; diff --git a/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx_drop.c b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx_drop.c index 847fd31015c..319bcc13744 100644 --- a/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx_drop.c +++ b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx_drop.c @@ -1,20 +1,12 @@ -/* mdbx_drop.c - memory-mapped database delete tool */ - -/* - * Copyright 2021-2024 Leonid Yuriev - * and other libmdbx authors: please see AUTHORS file. - * - * Copyright 2016-2021 Howard Chu, Symas Corp. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . */ - +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \note Please refer to the COPYRIGHT file for explanations license change, +/// credits and acknowledgments. +/// \author Леонид Юрьев aka Leonid Yuriev \date 2021-2025 +/// +/// mdbx_drop.c - memory-mapped database delete tool +/// + +/* clang-format off */ #ifdef _MSC_VER #if _MSC_VER > 1800 #pragma warning(disable : 4464) /* relative include path contains '..' */ @@ -23,38 +15,26 @@ #endif /* _MSC_VER (warnings) */ #define xMDBX_TOOLS /* Avoid using internal eASSERT() */ -/* - * Copyright 2015-2024 Leonid Yuriev - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . */ +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 -#define MDBX_BUILD_SOURCERY e156c1a97c017ce89d6541cd9464ae5a9761d76b3fd2f1696521f5f3792904fc_v0_12_13_0_g1fff1f67 -#ifdef MDBX_CONFIG_H -#include MDBX_CONFIG_H -#endif +#define MDBX_BUILD_SOURCERY 6b5df6869d2bf5419e3a8189d9cc849cc9911b9c8a951b9750ed0a261ce43724_v0_13_7_0_g566b0f93 #define LIBMDBX_INTERNALS -#ifdef xMDBX_TOOLS #define MDBX_DEPRECATED -#endif /* xMDBX_TOOLS */ -#ifdef xMDBX_ALLOY -/* Amalgamated build */ -#define MDBX_INTERNAL_FUNC static -#define MDBX_INTERNAL_VAR static -#else -/* Non-amalgamated build */ -#define MDBX_INTERNAL_FUNC -#define MDBX_INTERNAL_VAR extern -#endif /* xMDBX_ALLOY */ +#ifdef MDBX_CONFIG_H +#include MDBX_CONFIG_H +#endif + +/* Undefine the NDEBUG if debugging is enforced by MDBX_DEBUG */ +#if (defined(MDBX_DEBUG) && MDBX_DEBUG > 0) || (defined(MDBX_FORCE_ASSERTIONS) && MDBX_FORCE_ASSERTIONS) +#undef NDEBUG +#ifndef MDBX_DEBUG +/* Чтобы избежать включения отладки только из-за включения assert-проверок */ +#define MDBX_DEBUG 0 +#endif +#endif /*----------------------------------------------------------------------------*/ @@ -72,14 +52,59 @@ #endif /* MDBX_DISABLE_GNU_SOURCE */ /* Should be defined before any includes */ -#if !defined(_FILE_OFFSET_BITS) && !defined(__ANDROID_API__) && \ - !defined(ANDROID) +#if !defined(_FILE_OFFSET_BITS) && !defined(__ANDROID_API__) && !defined(ANDROID) #define _FILE_OFFSET_BITS 64 -#endif +#endif /* _FILE_OFFSET_BITS */ -#ifdef __APPLE__ +#if defined(__APPLE__) && !defined(_DARWIN_C_SOURCE) #define _DARWIN_C_SOURCE -#endif +#endif /* _DARWIN_C_SOURCE */ + +#if (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) && !defined(__USE_MINGW_ANSI_STDIO) +#define __USE_MINGW_ANSI_STDIO 1 +#endif /* MinGW */ + +#if defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS) + +#ifndef _WIN32_WINNT +#define _WIN32_WINNT 0x0601 /* Windows 7 */ +#endif /* _WIN32_WINNT */ + +#if !defined(_CRT_SECURE_NO_WARNINGS) +#define _CRT_SECURE_NO_WARNINGS +#endif /* _CRT_SECURE_NO_WARNINGS */ +#if !defined(UNICODE) +#define UNICODE +#endif /* UNICODE */ + +#if !defined(_NO_CRT_STDIO_INLINE) && MDBX_BUILD_SHARED_LIBRARY && !defined(xMDBX_TOOLS) && MDBX_WITHOUT_MSVC_CRT +#define _NO_CRT_STDIO_INLINE +#endif /* _NO_CRT_STDIO_INLINE */ + +#elif !defined(_POSIX_C_SOURCE) +#define _POSIX_C_SOURCE 200809L +#endif /* Windows */ + +#ifdef __cplusplus + +#ifndef NOMINMAX +#define NOMINMAX +#endif /* NOMINMAX */ + +/* Workaround for modern libstdc++ with CLANG < 4.x */ +#if defined(__SIZEOF_INT128__) && !defined(__GLIBCXX_TYPE_INT_N_0) && defined(__clang__) && __clang_major__ < 4 +#define __GLIBCXX_BITSIZE_INT_N_0 128 +#define __GLIBCXX_TYPE_INT_N_0 __int128 +#endif /* Workaround for modern libstdc++ with CLANG < 4.x */ + +#ifdef _MSC_VER +/* Workaround for MSVC' header `extern "C"` vs `std::` redefinition bug */ +#if defined(__SANITIZE_ADDRESS__) && !defined(_DISABLE_VECTOR_ANNOTATION) +#define _DISABLE_VECTOR_ANNOTATION +#endif /* _DISABLE_VECTOR_ANNOTATION */ +#endif /* _MSC_VER */ + +#endif /* __cplusplus */ #ifdef _MSC_VER #if _MSC_FULL_VER < 190024234 @@ -101,12 +126,8 @@ * and how to and where you can obtain the latest "Visual Studio 2015" build * with all fixes. */ -#error \ - "At least \"Microsoft C/C++ Compiler\" version 19.00.24234 (Visual Studio 2015 Update 3) is required." +#error "At least \"Microsoft C/C++ Compiler\" version 19.00.24234 (Visual Studio 2015 Update 3) is required." #endif -#ifndef _CRT_SECURE_NO_WARNINGS -#define _CRT_SECURE_NO_WARNINGS -#endif /* _CRT_SECURE_NO_WARNINGS */ #if _MSC_VER > 1800 #pragma warning(disable : 4464) /* relative include path contains '..' */ #endif @@ -114,124 +135,78 @@ #pragma warning(disable : 5045) /* will insert Spectre mitigation... */ #endif #if _MSC_VER > 1914 -#pragma warning( \ - disable : 5105) /* winbase.h(9531): warning C5105: macro expansion \ - producing 'defined' has undefined behavior */ +#pragma warning(disable : 5105) /* winbase.h(9531): warning C5105: macro expansion \ + producing 'defined' has undefined behavior */ +#endif +#if _MSC_VER < 1920 +/* avoid "error C2219: syntax error: type qualifier must be after '*'" */ +#define __restrict #endif #if _MSC_VER > 1930 #pragma warning(disable : 6235) /* is always a constant */ -#pragma warning(disable : 6237) /* is never evaluated and might \ +#pragma warning(disable : 6237) /* is never evaluated and might \ have side effects */ +#pragma warning(disable : 5286) /* implicit conversion from enum type 'type 1' to enum type 'type 2' */ +#pragma warning(disable : 5287) /* operands are different enum types 'type 1' and 'type 2' */ #endif #pragma warning(disable : 4710) /* 'xyz': function not inlined */ -#pragma warning(disable : 4711) /* function 'xyz' selected for automatic \ +#pragma warning(disable : 4711) /* function 'xyz' selected for automatic \ inline expansion */ -#pragma warning(disable : 4201) /* nonstandard extension used: nameless \ +#pragma warning(disable : 4201) /* nonstandard extension used: nameless \ struct/union */ #pragma warning(disable : 4702) /* unreachable code */ #pragma warning(disable : 4706) /* assignment within conditional expression */ #pragma warning(disable : 4127) /* conditional expression is constant */ -#pragma warning(disable : 4324) /* 'xyz': structure was padded due to \ +#pragma warning(disable : 4324) /* 'xyz': structure was padded due to \ alignment specifier */ #pragma warning(disable : 4310) /* cast truncates constant value */ -#pragma warning(disable : 4820) /* bytes padding added after data member for \ +#pragma warning(disable : 4820) /* bytes padding added after data member for \ alignment */ -#pragma warning(disable : 4548) /* expression before comma has no effect; \ +#pragma warning(disable : 4548) /* expression before comma has no effect; \ expected expression with side - effect */ -#pragma warning(disable : 4366) /* the result of the unary '&' operator may be \ +#pragma warning(disable : 4366) /* the result of the unary '&' operator may be \ unaligned */ -#pragma warning(disable : 4200) /* nonstandard extension used: zero-sized \ +#pragma warning(disable : 4200) /* nonstandard extension used: zero-sized \ array in struct/union */ -#pragma warning(disable : 4204) /* nonstandard extension used: non-constant \ +#pragma warning(disable : 4204) /* nonstandard extension used: non-constant \ aggregate initializer */ -#pragma warning( \ - disable : 4505) /* unreferenced local function has been removed */ -#endif /* _MSC_VER (warnings) */ +#pragma warning(disable : 4505) /* unreferenced local function has been removed */ +#endif /* _MSC_VER (warnings) */ #if defined(__GNUC__) && __GNUC__ < 9 #pragma GCC diagnostic ignored "-Wattributes" #endif /* GCC < 9 */ -#if (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) && \ - !defined(__USE_MINGW_ANSI_STDIO) -#define __USE_MINGW_ANSI_STDIO 1 -#endif /* MinGW */ - -#if (defined(_WIN32) || defined(_WIN64)) && !defined(UNICODE) -#define UNICODE -#endif /* UNICODE */ - -#include "mdbx.h" -/* - * Copyright 2015-2024 Leonid Yuriev - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - */ - - /*----------------------------------------------------------------------------*/ /* Microsoft compiler generates a lot of warning for self includes... */ #ifdef _MSC_VER #pragma warning(push, 1) -#pragma warning(disable : 4548) /* expression before comma has no effect; \ +#pragma warning(disable : 4548) /* expression before comma has no effect; \ expected expression with side - effect */ -#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ +#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ * semantics are not enabled. Specify /EHsc */ -#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ - * mode specified; termination on exception is \ +#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ + * mode specified; termination on exception is \ * not guaranteed. Specify /EHsc */ #endif /* _MSC_VER (warnings) */ -#if defined(_WIN32) || defined(_WIN64) -#if !defined(_CRT_SECURE_NO_WARNINGS) -#define _CRT_SECURE_NO_WARNINGS -#endif /* _CRT_SECURE_NO_WARNINGS */ -#if !defined(_NO_CRT_STDIO_INLINE) && MDBX_BUILD_SHARED_LIBRARY && \ - !defined(xMDBX_TOOLS) && MDBX_WITHOUT_MSVC_CRT -#define _NO_CRT_STDIO_INLINE -#endif -#elif !defined(_POSIX_C_SOURCE) -#define _POSIX_C_SOURCE 200809L -#endif /* Windows */ - /*----------------------------------------------------------------------------*/ /* basic C99 includes */ + #include #include #include #include #include +#include #include #include #include #include #include -#if (-6 & 5) || CHAR_BIT != 8 || UINT_MAX < 0xffffffff || ULONG_MAX % 0xFFFF -#error \ - "Sanity checking failed: Two's complement, reasonably sized integer types" -#endif - -#ifndef SSIZE_MAX -#define SSIZE_MAX INTPTR_MAX -#endif - -#if UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul || defined(_WIN64) -#define MDBX_WORDBITS 64 -#else -#define MDBX_WORDBITS 32 -#endif /* MDBX_WORDBITS */ - /*----------------------------------------------------------------------------*/ /* feature testing */ @@ -243,6 +218,14 @@ #define __has_include(x) (0) #endif +#ifndef __has_attribute +#define __has_attribute(x) (0) +#endif + +#ifndef __has_cpp_attribute +#define __has_cpp_attribute(x) 0 +#endif + #ifndef __has_feature #define __has_feature(x) (0) #endif @@ -265,8 +248,7 @@ #ifndef __GNUC_PREREQ #if defined(__GNUC__) && defined(__GNUC_MINOR__) -#define __GNUC_PREREQ(maj, min) \ - ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min)) +#define __GNUC_PREREQ(maj, min) ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min)) #else #define __GNUC_PREREQ(maj, min) (0) #endif @@ -274,8 +256,7 @@ #ifndef __CLANG_PREREQ #ifdef __clang__ -#define __CLANG_PREREQ(maj, min) \ - ((__clang_major__ << 16) + __clang_minor__ >= ((maj) << 16) + (min)) +#define __CLANG_PREREQ(maj, min) ((__clang_major__ << 16) + __clang_minor__ >= ((maj) << 16) + (min)) #else #define __CLANG_PREREQ(maj, min) (0) #endif @@ -283,13 +264,51 @@ #ifndef __GLIBC_PREREQ #if defined(__GLIBC__) && defined(__GLIBC_MINOR__) -#define __GLIBC_PREREQ(maj, min) \ - ((__GLIBC__ << 16) + __GLIBC_MINOR__ >= ((maj) << 16) + (min)) +#define __GLIBC_PREREQ(maj, min) ((__GLIBC__ << 16) + __GLIBC_MINOR__ >= ((maj) << 16) + (min)) #else #define __GLIBC_PREREQ(maj, min) (0) #endif #endif /* __GLIBC_PREREQ */ +/*----------------------------------------------------------------------------*/ +/* pre-requirements */ + +#if (-6 & 5) || CHAR_BIT != 8 || UINT_MAX < 0xffffffff || ULONG_MAX % 0xFFFF +#error "Sanity checking failed: Two's complement, reasonably sized integer types" +#endif + +#ifndef SSIZE_MAX +#define SSIZE_MAX INTPTR_MAX +#endif + +#if defined(__GNUC__) && !__GNUC_PREREQ(4, 2) +/* Actually libmdbx was not tested with compilers older than GCC 4.2. + * But you could ignore this warning at your own risk. + * In such case please don't rise up an issues related ONLY to old compilers. + */ +#warning "libmdbx required GCC >= 4.2" +#endif + +#if defined(__clang__) && !__CLANG_PREREQ(3, 8) +/* Actually libmdbx was not tested with CLANG older than 3.8. + * But you could ignore this warning at your own risk. + * In such case please don't rise up an issues related ONLY to old compilers. + */ +#warning "libmdbx required CLANG >= 3.8" +#endif + +#if defined(__GLIBC__) && !__GLIBC_PREREQ(2, 12) +/* Actually libmdbx was not tested with something older than glibc 2.12. + * But you could ignore this warning at your own risk. + * In such case please don't rise up an issues related ONLY to old systems. + */ +#warning "libmdbx was only tested with GLIBC >= 2.12." +#endif + +#ifdef __SANITIZE_THREAD__ +#warning "libmdbx don't compatible with ThreadSanitizer, you will get a lot of false-positive issues." +#endif /* __SANITIZE_THREAD__ */ + /*----------------------------------------------------------------------------*/ /* C11' alignas() */ @@ -319,8 +338,7 @@ #endif #endif /* __extern_C */ -#if !defined(nullptr) && !defined(__cplusplus) || \ - (__cplusplus < 201103L && !defined(_MSC_VER)) +#if !defined(nullptr) && !defined(__cplusplus) || (__cplusplus < 201103L && !defined(_MSC_VER)) #define nullptr NULL #endif @@ -332,9 +350,8 @@ #endif #endif /* Apple OSX & iOS */ -#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \ - defined(__BSD__) || defined(__bsdi__) || defined(__DragonFly__) || \ - defined(__APPLE__) || defined(__MACH__) +#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__BSD__) || defined(__bsdi__) || \ + defined(__DragonFly__) || defined(__APPLE__) || defined(__MACH__) #include #include #include @@ -351,8 +368,7 @@ #endif #else #include -#if !(defined(__sun) || defined(__SVR4) || defined(__svr4__) || \ - defined(_WIN32) || defined(_WIN64)) +#if !(defined(__sun) || defined(__SVR4) || defined(__svr4__) || defined(_WIN32) || defined(_WIN64)) #include #endif /* !Solaris */ #endif /* !xBSD */ @@ -406,12 +422,14 @@ __extern_C key_t ftok(const char *, int); #ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN #endif /* WIN32_LEAN_AND_MEAN */ -#include -#include #include #include #include +/* После подгрузки windows.h, чтобы избежать проблем со сборкой MINGW и т.п. */ +#include +#include + #else /*----------------------------------------------------------------------*/ #include @@ -439,11 +457,6 @@ __extern_C key_t ftok(const char *, int); #if __ANDROID_API__ >= 21 #include #endif -#if defined(_FILE_OFFSET_BITS) && _FILE_OFFSET_BITS != MDBX_WORDBITS -#error "_FILE_OFFSET_BITS != MDBX_WORDBITS" (_FILE_OFFSET_BITS != MDBX_WORDBITS) -#elif defined(__FILE_OFFSET_BITS) && __FILE_OFFSET_BITS != MDBX_WORDBITS -#error "__FILE_OFFSET_BITS != MDBX_WORDBITS" (__FILE_OFFSET_BITS != MDBX_WORDBITS) -#endif #endif /* Android */ #if defined(HAVE_SYS_STAT_H) || __has_include() @@ -459,43 +472,38 @@ __extern_C key_t ftok(const char *, int); /*----------------------------------------------------------------------------*/ /* Byteorder */ -#if defined(i386) || defined(__386) || defined(__i386) || defined(__i386__) || \ - defined(i486) || defined(__i486) || defined(__i486__) || defined(i586) || \ - defined(__i586) || defined(__i586__) || defined(i686) || \ - defined(__i686) || defined(__i686__) || defined(_M_IX86) || \ - defined(_X86_) || defined(__THW_INTEL__) || defined(__I86__) || \ - defined(__INTEL__) || defined(__x86_64) || defined(__x86_64__) || \ - defined(__amd64__) || defined(__amd64) || defined(_M_X64) || \ - defined(_M_AMD64) || defined(__IA32__) || defined(__INTEL__) +#if defined(i386) || defined(__386) || defined(__i386) || defined(__i386__) || defined(i486) || defined(__i486) || \ + defined(__i486__) || defined(i586) || defined(__i586) || defined(__i586__) || defined(i686) || defined(__i686) || \ + defined(__i686__) || defined(_M_IX86) || defined(_X86_) || defined(__THW_INTEL__) || defined(__I86__) || \ + defined(__INTEL__) || defined(__x86_64) || defined(__x86_64__) || defined(__amd64__) || defined(__amd64) || \ + defined(_M_X64) || defined(_M_AMD64) || defined(__IA32__) || defined(__INTEL__) #ifndef __ia32__ /* LY: define neutral __ia32__ for x86 and x86-64 */ #define __ia32__ 1 #endif /* __ia32__ */ -#if !defined(__amd64__) && \ - (defined(__x86_64) || defined(__x86_64__) || defined(__amd64) || \ - defined(_M_X64) || defined(_M_AMD64)) +#if !defined(__amd64__) && \ + (defined(__x86_64) || defined(__x86_64__) || defined(__amd64) || defined(_M_X64) || defined(_M_AMD64)) /* LY: define trusty __amd64__ for all AMD64/x86-64 arch */ #define __amd64__ 1 #endif /* __amd64__ */ #endif /* all x86 */ -#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || \ - !defined(__ORDER_BIG_ENDIAN__) +#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || !defined(__ORDER_BIG_ENDIAN__) -#if defined(__GLIBC__) || defined(__GNU_LIBRARY__) || \ - defined(__ANDROID_API__) || defined(HAVE_ENDIAN_H) || __has_include() +#if defined(__GLIBC__) || defined(__GNU_LIBRARY__) || defined(__ANDROID_API__) || defined(HAVE_ENDIAN_H) || \ + __has_include() #include -#elif defined(__APPLE__) || defined(__MACH__) || defined(__OpenBSD__) || \ - defined(HAVE_MACHINE_ENDIAN_H) || __has_include() +#elif defined(__APPLE__) || defined(__MACH__) || defined(__OpenBSD__) || defined(HAVE_MACHINE_ENDIAN_H) || \ + __has_include() #include #elif defined(HAVE_SYS_ISA_DEFS_H) || __has_include() #include -#elif (defined(HAVE_SYS_TYPES_H) && defined(HAVE_SYS_ENDIAN_H)) || \ +#elif (defined(HAVE_SYS_TYPES_H) && defined(HAVE_SYS_ENDIAN_H)) || \ (__has_include() && __has_include()) #include #include -#elif defined(__bsdi__) || defined(__DragonFly__) || defined(__FreeBSD__) || \ - defined(__NetBSD__) || defined(HAVE_SYS_PARAM_H) || __has_include() +#elif defined(__bsdi__) || defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || \ + defined(HAVE_SYS_PARAM_H) || __has_include() #include #endif /* OS */ @@ -511,27 +519,19 @@ __extern_C key_t ftok(const char *, int); #define __ORDER_LITTLE_ENDIAN__ 1234 #define __ORDER_BIG_ENDIAN__ 4321 -#if defined(__LITTLE_ENDIAN__) || \ - (defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)) || \ - defined(__ARMEL__) || defined(__THUMBEL__) || defined(__AARCH64EL__) || \ - defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \ - defined(_M_ARM) || defined(_M_ARM64) || defined(__e2k__) || \ - defined(__elbrus_4c__) || defined(__elbrus_8c__) || defined(__bfin__) || \ - defined(__BFIN__) || defined(__ia64__) || defined(_IA64) || \ - defined(__IA64__) || defined(__ia64) || defined(_M_IA64) || \ - defined(__itanium__) || defined(__ia32__) || defined(__CYGWIN__) || \ - defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) || \ - defined(__WINDOWS__) +#if defined(__LITTLE_ENDIAN__) || (defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)) || defined(__ARMEL__) || \ + defined(__THUMBEL__) || defined(__AARCH64EL__) || defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \ + defined(_M_ARM) || defined(_M_ARM64) || defined(__e2k__) || defined(__elbrus_4c__) || defined(__elbrus_8c__) || \ + defined(__bfin__) || defined(__BFIN__) || defined(__ia64__) || defined(_IA64) || defined(__IA64__) || \ + defined(__ia64) || defined(_M_IA64) || defined(__itanium__) || defined(__ia32__) || defined(__CYGWIN__) || \ + defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) || defined(__WINDOWS__) #define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__ -#elif defined(__BIG_ENDIAN__) || \ - (defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) || \ - defined(__ARMEB__) || defined(__THUMBEB__) || defined(__AARCH64EB__) || \ - defined(__MIPSEB__) || defined(_MIPSEB) || defined(__MIPSEB) || \ - defined(__m68k__) || defined(M68000) || defined(__hppa__) || \ - defined(__hppa) || defined(__HPPA__) || defined(__sparc__) || \ - defined(__sparc) || defined(__370__) || defined(__THW_370__) || \ - defined(__s390__) || defined(__s390x__) || defined(__SYSC_ZARCH__) +#elif defined(__BIG_ENDIAN__) || (defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) || defined(__ARMEB__) || \ + defined(__THUMBEB__) || defined(__AARCH64EB__) || defined(__MIPSEB__) || defined(_MIPSEB) || defined(__MIPSEB) || \ + defined(__m68k__) || defined(M68000) || defined(__hppa__) || defined(__hppa) || defined(__HPPA__) || \ + defined(__sparc__) || defined(__sparc) || defined(__370__) || defined(__THW_370__) || defined(__s390__) || \ + defined(__s390x__) || defined(__SYSC_ZARCH__) #define __BYTE_ORDER__ __ORDER_BIG_ENDIAN__ #else @@ -541,6 +541,12 @@ __extern_C key_t ftok(const char *, int); #endif #endif /* __BYTE_ORDER__ || __ORDER_LITTLE_ENDIAN__ || __ORDER_BIG_ENDIAN__ */ +#if UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul || defined(_WIN64) +#define MDBX_WORDBITS 64 +#else +#define MDBX_WORDBITS 32 +#endif /* MDBX_WORDBITS */ + /*----------------------------------------------------------------------------*/ /* Availability of CMOV or equivalent */ @@ -551,17 +557,14 @@ __extern_C key_t ftok(const char *, int); #define MDBX_HAVE_CMOV 1 #elif defined(__thumb__) || defined(__thumb) || defined(__TARGET_ARCH_THUMB) #define MDBX_HAVE_CMOV 0 -#elif defined(_M_ARM) || defined(_M_ARM64) || defined(__aarch64__) || \ - defined(__aarch64) || defined(__arm__) || defined(__arm) || \ - defined(__CC_ARM) +#elif defined(_M_ARM) || defined(_M_ARM64) || defined(__aarch64__) || defined(__aarch64) || defined(__arm__) || \ + defined(__arm) || defined(__CC_ARM) #define MDBX_HAVE_CMOV 1 -#elif (defined(__riscv__) || defined(__riscv64)) && \ - (defined(__riscv_b) || defined(__riscv_bitmanip)) +#elif (defined(__riscv__) || defined(__riscv64)) && (defined(__riscv_b) || defined(__riscv_bitmanip)) #define MDBX_HAVE_CMOV 1 -#elif defined(i686) || defined(__i686) || defined(__i686__) || \ - (defined(_M_IX86) && _M_IX86 > 600) || defined(__x86_64) || \ - defined(__x86_64__) || defined(__amd64__) || defined(__amd64) || \ - defined(_M_X64) || defined(_M_AMD64) +#elif defined(i686) || defined(__i686) || defined(__i686__) || (defined(_M_IX86) && _M_IX86 > 600) || \ + defined(__x86_64) || defined(__x86_64__) || defined(__amd64__) || defined(__amd64) || defined(_M_X64) || \ + defined(_M_AMD64) #define MDBX_HAVE_CMOV 1 #else #define MDBX_HAVE_CMOV 0 @@ -587,8 +590,7 @@ __extern_C key_t ftok(const char *, int); #endif #elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) #include -#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ - (defined(HP_IA64) || defined(__ia64)) +#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && (defined(HP_IA64) || defined(__ia64)) #include #elif defined(__IBMC__) && defined(__powerpc) #include @@ -610,29 +612,26 @@ __extern_C key_t ftok(const char *, int); #endif /* Compiler */ #if !defined(__noop) && !defined(_MSC_VER) -#define __noop \ - do { \ +#define __noop \ + do { \ } while (0) #endif /* __noop */ -#if defined(__fallthrough) && \ - (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) +#if defined(__fallthrough) && (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) #undef __fallthrough #endif /* __fallthrough workaround for MinGW */ #ifndef __fallthrough -#if defined(__cplusplus) && (__has_cpp_attribute(fallthrough) && \ - (!defined(__clang__) || __clang__ > 4)) || \ +#if defined(__cplusplus) && (__has_cpp_attribute(fallthrough) && (!defined(__clang__) || __clang__ > 4)) || \ __cplusplus >= 201703L #define __fallthrough [[fallthrough]] #elif __GNUC_PREREQ(8, 0) && defined(__cplusplus) && __cplusplus >= 201103L #define __fallthrough [[fallthrough]] -#elif __GNUC_PREREQ(7, 0) && \ - (!defined(__LCC__) || (__LCC__ == 124 && __LCC_MINOR__ >= 12) || \ - (__LCC__ == 125 && __LCC_MINOR__ >= 5) || (__LCC__ >= 126)) +#elif __GNUC_PREREQ(7, 0) && (!defined(__LCC__) || (__LCC__ == 124 && __LCC_MINOR__ >= 12) || \ + (__LCC__ == 125 && __LCC_MINOR__ >= 5) || (__LCC__ >= 126)) #define __fallthrough __attribute__((__fallthrough__)) -#elif defined(__clang__) && defined(__cplusplus) && __cplusplus >= 201103L && \ - __has_feature(cxx_attributes) && __has_warning("-Wimplicit-fallthrough") +#elif defined(__clang__) && defined(__cplusplus) && __cplusplus >= 201103L && __has_feature(cxx_attributes) && \ + __has_warning("-Wimplicit-fallthrough") #define __fallthrough [[clang::fallthrough]] #else #define __fallthrough @@ -645,8 +644,8 @@ __extern_C key_t ftok(const char *, int); #elif defined(_MSC_VER) #define __unreachable() __assume(0) #else -#define __unreachable() \ - do { \ +#define __unreachable() \ + do { \ } while (1) #endif #endif /* __unreachable */ @@ -655,9 +654,9 @@ __extern_C key_t ftok(const char *, int); #if defined(__GNUC__) || defined(__clang__) || __has_builtin(__builtin_prefetch) #define __prefetch(ptr) __builtin_prefetch(ptr) #else -#define __prefetch(ptr) \ - do { \ - (void)(ptr); \ +#define __prefetch(ptr) \ + do { \ + (void)(ptr); \ } while (0) #endif #endif /* __prefetch */ @@ -667,11 +666,11 @@ __extern_C key_t ftok(const char *, int); #endif /* offsetof */ #ifndef container_of -#define container_of(ptr, type, member) \ - ((type *)((char *)(ptr) - offsetof(type, member))) +#define container_of(ptr, type, member) ((type *)((char *)(ptr) - offsetof(type, member))) #endif /* container_of */ /*----------------------------------------------------------------------------*/ +/* useful attributes */ #ifndef __always_inline #if defined(__GNUC__) || __has_attribute(__always_inline__) @@ -739,8 +738,7 @@ __extern_C key_t ftok(const char *, int); #ifndef __hot #if defined(__OPTIMIZE__) -#if defined(__clang__) && !__has_attribute(__hot__) && \ - __has_attribute(__section__) && \ +#if defined(__clang__) && !__has_attribute(__hot__) && __has_attribute(__section__) && \ (defined(__linux__) || defined(__gnu_linux__)) /* just put frequently used functions in separate section */ #define __hot __attribute__((__section__("text.hot"))) __optimize("O3") @@ -756,8 +754,7 @@ __extern_C key_t ftok(const char *, int); #ifndef __cold #if defined(__OPTIMIZE__) -#if defined(__clang__) && !__has_attribute(__cold__) && \ - __has_attribute(__section__) && \ +#if defined(__clang__) && !__has_attribute(__cold__) && __has_attribute(__section__) && \ (defined(__linux__) || defined(__gnu_linux__)) /* just put infrequently used functions in separate section */ #define __cold __attribute__((__section__("text.unlikely"))) __optimize("Os") @@ -780,8 +777,7 @@ __extern_C key_t ftok(const char *, int); #endif /* __flatten */ #ifndef likely -#if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && \ - !defined(__COVERITY__) +#if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__) #define likely(cond) __builtin_expect(!!(cond), 1) #else #define likely(x) (!!(x)) @@ -789,8 +785,7 @@ __extern_C key_t ftok(const char *, int); #endif /* likely */ #ifndef unlikely -#if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && \ - !defined(__COVERITY__) +#if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__) #define unlikely(cond) __builtin_expect(!!(cond), 0) #else #define unlikely(x) (!!(x)) @@ -805,29 +800,41 @@ __extern_C key_t ftok(const char *, int); #endif #endif /* __anonymous_struct_extension__ */ -#ifndef expect_with_probability -#if defined(__builtin_expect_with_probability) || \ - __has_builtin(__builtin_expect_with_probability) || __GNUC_PREREQ(9, 0) -#define expect_with_probability(expr, value, prob) \ - __builtin_expect_with_probability(expr, value, prob) -#else -#define expect_with_probability(expr, value, prob) (expr) -#endif -#endif /* expect_with_probability */ - #ifndef MDBX_WEAK_IMPORT_ATTRIBUTE #ifdef WEAK_IMPORT_ATTRIBUTE #define MDBX_WEAK_IMPORT_ATTRIBUTE WEAK_IMPORT_ATTRIBUTE #elif __has_attribute(__weak__) && __has_attribute(__weak_import__) #define MDBX_WEAK_IMPORT_ATTRIBUTE __attribute__((__weak__, __weak_import__)) -#elif __has_attribute(__weak__) || \ - (defined(__GNUC__) && __GNUC__ >= 4 && defined(__ELF__)) +#elif __has_attribute(__weak__) || (defined(__GNUC__) && __GNUC__ >= 4 && defined(__ELF__)) #define MDBX_WEAK_IMPORT_ATTRIBUTE __attribute__((__weak__)) #else #define MDBX_WEAK_IMPORT_ATTRIBUTE #endif #endif /* MDBX_WEAK_IMPORT_ATTRIBUTE */ +#if !defined(__thread) && (defined(_MSC_VER) || defined(__DMC__)) +#define __thread __declspec(thread) +#endif /* __thread */ + +#ifndef MDBX_EXCLUDE_FOR_GPROF +#ifdef ENABLE_GPROF +#define MDBX_EXCLUDE_FOR_GPROF __attribute__((__no_instrument_function__, __no_profile_instrument_function__)) +#else +#define MDBX_EXCLUDE_FOR_GPROF +#endif /* ENABLE_GPROF */ +#endif /* MDBX_EXCLUDE_FOR_GPROF */ + +/*----------------------------------------------------------------------------*/ + +#ifndef expect_with_probability +#if defined(__builtin_expect_with_probability) || __has_builtin(__builtin_expect_with_probability) || \ + __GNUC_PREREQ(9, 0) +#define expect_with_probability(expr, value, prob) __builtin_expect_with_probability(expr, value, prob) +#else +#define expect_with_probability(expr, value, prob) (expr) +#endif +#endif /* expect_with_probability */ + #ifndef MDBX_GOOFY_MSVC_STATIC_ANALYZER #ifdef _PREFAST_ #define MDBX_GOOFY_MSVC_STATIC_ANALYZER 1 @@ -839,20 +846,27 @@ __extern_C key_t ftok(const char *, int); #if MDBX_GOOFY_MSVC_STATIC_ANALYZER || (defined(_MSC_VER) && _MSC_VER > 1919) #define MDBX_ANALYSIS_ASSUME(expr) __analysis_assume(expr) #ifdef _PREFAST_ -#define MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(warn_id) \ - __pragma(prefast(suppress : warn_id)) +#define MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(warn_id) __pragma(prefast(suppress : warn_id)) #else -#define MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(warn_id) \ - __pragma(warning(suppress : warn_id)) +#define MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(warn_id) __pragma(warning(suppress : warn_id)) #endif #else #define MDBX_ANALYSIS_ASSUME(expr) assert(expr) #define MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(warn_id) #endif /* MDBX_GOOFY_MSVC_STATIC_ANALYZER */ +#ifndef FLEXIBLE_ARRAY_MEMBERS +#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || (!defined(__cplusplus) && defined(_MSC_VER)) +#define FLEXIBLE_ARRAY_MEMBERS 1 +#else +#define FLEXIBLE_ARRAY_MEMBERS 0 +#endif +#endif /* FLEXIBLE_ARRAY_MEMBERS */ + /*----------------------------------------------------------------------------*/ +/* Valgrind and Address Sanitizer */ -#if defined(MDBX_USE_VALGRIND) +#if defined(ENABLE_MEMCHECK) #include #ifndef VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE /* LY: available since Valgrind 3.10 */ @@ -874,7 +888,7 @@ __extern_C key_t ftok(const char *, int); #define VALGRIND_CHECK_MEM_IS_ADDRESSABLE(a, s) (0) #define VALGRIND_CHECK_MEM_IS_DEFINED(a, s) (0) #define RUNNING_ON_VALGRIND (0) -#endif /* MDBX_USE_VALGRIND */ +#endif /* ENABLE_MEMCHECK */ #ifdef __SANITIZE_ADDRESS__ #include @@ -901,8 +915,7 @@ template char (&__ArraySizeHelper(T (&array)[N]))[N]; #define CONCAT(a, b) a##b #define XCONCAT(a, b) CONCAT(a, b) -#define MDBX_TETRAD(a, b, c, d) \ - ((uint32_t)(a) << 24 | (uint32_t)(b) << 16 | (uint32_t)(c) << 8 | (d)) +#define MDBX_TETRAD(a, b, c, d) ((uint32_t)(a) << 24 | (uint32_t)(b) << 16 | (uint32_t)(c) << 8 | (d)) #define MDBX_STRING_TETRAD(str) MDBX_TETRAD(str[0], str[1], str[2], str[3]) @@ -916,14 +929,13 @@ template char (&__ArraySizeHelper(T (&array)[N]))[N]; #elif defined(_MSC_VER) #include #define STATIC_ASSERT_MSG(expr, msg) _STATIC_ASSERT(expr) -#elif (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) || \ - __has_feature(c_static_assert) +#elif (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) || __has_feature(c_static_assert) #define STATIC_ASSERT_MSG(expr, msg) _Static_assert(expr, msg) #else -#define STATIC_ASSERT_MSG(expr, msg) \ - switch (0) { \ - case 0: \ - case (expr):; \ +#define STATIC_ASSERT_MSG(expr, msg) \ + switch (0) { \ + case 0: \ + case (expr):; \ } #endif #endif /* STATIC_ASSERT */ @@ -932,42 +944,37 @@ template char (&__ArraySizeHelper(T (&array)[N]))[N]; #define STATIC_ASSERT(expr) STATIC_ASSERT_MSG(expr, #expr) #endif -#ifndef __Wpedantic_format_voidptr -MDBX_MAYBE_UNUSED MDBX_PURE_FUNCTION static __inline const void * -__Wpedantic_format_voidptr(const void *ptr) { - return ptr; -} -#define __Wpedantic_format_voidptr(ARG) __Wpedantic_format_voidptr(ARG) -#endif /* __Wpedantic_format_voidptr */ +/*----------------------------------------------------------------------------*/ -#if defined(__GNUC__) && !__GNUC_PREREQ(4, 2) -/* Actually libmdbx was not tested with compilers older than GCC 4.2. - * But you could ignore this warning at your own risk. - * In such case please don't rise up an issues related ONLY to old compilers. - */ -#warning "libmdbx required GCC >= 4.2" -#endif +#if defined(_MSC_VER) && _MSC_VER >= 1900 +/* LY: MSVC 2015/2017/2019 has buggy/inconsistent PRIuPTR/PRIxPTR macros + * for internal format-args checker. */ +#undef PRIuPTR +#undef PRIiPTR +#undef PRIdPTR +#undef PRIxPTR +#define PRIuPTR "Iu" +#define PRIiPTR "Ii" +#define PRIdPTR "Id" +#define PRIxPTR "Ix" +#define PRIuSIZE "zu" +#define PRIiSIZE "zi" +#define PRIdSIZE "zd" +#define PRIxSIZE "zx" +#endif /* fix PRI*PTR for _MSC_VER */ -#if defined(__clang__) && !__CLANG_PREREQ(3, 8) -/* Actually libmdbx was not tested with CLANG older than 3.8. - * But you could ignore this warning at your own risk. - * In such case please don't rise up an issues related ONLY to old compilers. - */ -#warning "libmdbx required CLANG >= 3.8" -#endif +#ifndef PRIuSIZE +#define PRIuSIZE PRIuPTR +#define PRIiSIZE PRIiPTR +#define PRIdSIZE PRIdPTR +#define PRIxSIZE PRIxPTR +#endif /* PRI*SIZE macros for MSVC */ -#if defined(__GLIBC__) && !__GLIBC_PREREQ(2, 12) -/* Actually libmdbx was not tested with something older than glibc 2.12. - * But you could ignore this warning at your own risk. - * In such case please don't rise up an issues related ONLY to old systems. - */ -#warning "libmdbx was only tested with GLIBC >= 2.12." +#ifdef _MSC_VER +#pragma warning(pop) #endif -#ifdef __SANITIZE_THREAD__ -#warning \ - "libmdbx don't compatible with ThreadSanitizer, you will get a lot of false-positive issues." -#endif /* __SANITIZE_THREAD__ */ +/*----------------------------------------------------------------------------*/ #if __has_warning("-Wnested-anon-types") #if defined(__clang__) @@ -1004,80 +1011,34 @@ __Wpedantic_format_voidptr(const void *ptr) { #endif #endif /* -Walignment-reduction-ignored */ -#ifndef MDBX_EXCLUDE_FOR_GPROF -#ifdef ENABLE_GPROF -#define MDBX_EXCLUDE_FOR_GPROF \ - __attribute__((__no_instrument_function__, \ - __no_profile_instrument_function__)) +#ifdef xMDBX_ALLOY +/* Amalgamated build */ +#define MDBX_INTERNAL static #else -#define MDBX_EXCLUDE_FOR_GPROF -#endif /* ENABLE_GPROF */ -#endif /* MDBX_EXCLUDE_FOR_GPROF */ - -#ifdef __cplusplus -extern "C" { -#endif +/* Non-amalgamated build */ +#define MDBX_INTERNAL +#endif /* xMDBX_ALLOY */ -/* https://en.wikipedia.org/wiki/Operating_system_abstraction_layer */ +#include "mdbx.h" -/* - * Copyright 2015-2024 Leonid Yuriev - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - */ +/*----------------------------------------------------------------------------*/ +/* Basic constants and types */ +typedef struct iov_ctx iov_ctx_t; +/// /*----------------------------------------------------------------------------*/ -/* C11 Atomics */ - -#if defined(__cplusplus) && !defined(__STDC_NO_ATOMICS__) && __has_include() -#include -#define MDBX_HAVE_C11ATOMICS -#elif !defined(__cplusplus) && \ - (__STDC_VERSION__ >= 201112L || __has_extension(c_atomic)) && \ - !defined(__STDC_NO_ATOMICS__) && \ - (__GNUC_PREREQ(4, 9) || __CLANG_PREREQ(3, 8) || \ - !(defined(__GNUC__) || defined(__clang__))) -#include -#define MDBX_HAVE_C11ATOMICS -#elif defined(__GNUC__) || defined(__clang__) -#elif defined(_MSC_VER) -#pragma warning(disable : 4163) /* 'xyz': not available as an intrinsic */ -#pragma warning(disable : 4133) /* 'function': incompatible types - from \ - 'size_t' to 'LONGLONG' */ -#pragma warning(disable : 4244) /* 'return': conversion from 'LONGLONG' to \ - 'std::size_t', possible loss of data */ -#pragma warning(disable : 4267) /* 'function': conversion from 'size_t' to \ - 'long', possible loss of data */ -#pragma intrinsic(_InterlockedExchangeAdd, _InterlockedCompareExchange) -#pragma intrinsic(_InterlockedExchangeAdd64, _InterlockedCompareExchange64) -#elif defined(__APPLE__) -#include -#else -#error FIXME atomic-ops -#endif - -/*----------------------------------------------------------------------------*/ -/* Memory/Compiler barriers, cache coherence */ +/* Memory/Compiler barriers, cache coherence */ #if __has_include() #include -#elif defined(__mips) || defined(__mips__) || defined(__mips64) || \ - defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \ - defined(__MWERKS__) || defined(__sgi) +#elif defined(__mips) || defined(__mips__) || defined(__mips64) || defined(__mips64__) || defined(_M_MRX000) || \ + defined(_MIPS_) || defined(__MWERKS__) || defined(__sgi) /* MIPS should have explicit cache control */ #include #endif -MDBX_MAYBE_UNUSED static __inline void osal_compiler_barrier(void) { +MDBX_MAYBE_UNUSED static inline void osal_compiler_barrier(void) { #if defined(__clang__) || defined(__GNUC__) __asm__ __volatile__("" ::: "memory"); #elif defined(_MSC_VER) @@ -1086,18 +1047,16 @@ MDBX_MAYBE_UNUSED static __inline void osal_compiler_barrier(void) { __memory_barrier(); #elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) __compiler_barrier(); -#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ - (defined(HP_IA64) || defined(__ia64)) +#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && (defined(HP_IA64) || defined(__ia64)) _Asm_sched_fence(/* LY: no-arg meaning 'all expect ALU', e.g. 0x3D3D */); -#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \ - defined(__ppc64__) || defined(__powerpc64__) +#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || defined(__ppc64__) || defined(__powerpc64__) __fence(); #else #error "Could not guess the kind of compiler, please report to us." #endif } -MDBX_MAYBE_UNUSED static __inline void osal_memory_barrier(void) { +MDBX_MAYBE_UNUSED static inline void osal_memory_barrier(void) { #ifdef MDBX_HAVE_C11ATOMICS atomic_thread_fence(memory_order_seq_cst); #elif defined(__ATOMIC_SEQ_CST) @@ -1118,11 +1077,9 @@ MDBX_MAYBE_UNUSED static __inline void osal_memory_barrier(void) { #endif #elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) __machine_rw_barrier(); -#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ - (defined(HP_IA64) || defined(__ia64)) +#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && (defined(HP_IA64) || defined(__ia64)) _Asm_mf(); -#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \ - defined(__ppc64__) || defined(__powerpc64__) +#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || defined(__ppc64__) || defined(__powerpc64__) __lwsync(); #else #error "Could not guess the kind of compiler, please report to us." @@ -1137,7 +1094,7 @@ MDBX_MAYBE_UNUSED static __inline void osal_memory_barrier(void) { #define HAVE_SYS_TYPES_H typedef HANDLE osal_thread_t; typedef unsigned osal_thread_key_t; -#define MAP_FAILED NULL +#define MAP_FAILED nullptr #define HIGH_DWORD(v) ((DWORD)((sizeof(v) > 4) ? ((uint64_t)(v) >> 32) : 0)) #define THREAD_CALL WINAPI #define THREAD_RESULT DWORD @@ -1149,15 +1106,13 @@ typedef CRITICAL_SECTION osal_fastmutex_t; #if !defined(_MSC_VER) && !defined(__try) #define __try -#define __except(COND) if (false) +#define __except(COND) if (/* (void)(COND), */ false) #endif /* stub for MSVC's __try/__except */ #if MDBX_WITHOUT_MSVC_CRT #ifndef osal_malloc -static inline void *osal_malloc(size_t bytes) { - return HeapAlloc(GetProcessHeap(), 0, bytes); -} +static inline void *osal_malloc(size_t bytes) { return HeapAlloc(GetProcessHeap(), 0, bytes); } #endif /* osal_malloc */ #ifndef osal_calloc @@ -1168,8 +1123,7 @@ static inline void *osal_calloc(size_t nelem, size_t size) { #ifndef osal_realloc static inline void *osal_realloc(void *ptr, size_t bytes) { - return ptr ? HeapReAlloc(GetProcessHeap(), 0, ptr, bytes) - : HeapAlloc(GetProcessHeap(), 0, bytes); + return ptr ? HeapReAlloc(GetProcessHeap(), 0, ptr, bytes) : HeapAlloc(GetProcessHeap(), 0, bytes); } #endif /* osal_realloc */ @@ -1215,29 +1169,16 @@ typedef pthread_mutex_t osal_fastmutex_t; #endif /* Platform */ #if __GLIBC_PREREQ(2, 12) || defined(__FreeBSD__) || defined(malloc_usable_size) -/* malloc_usable_size() already provided */ +#define osal_malloc_usable_size(ptr) malloc_usable_size(ptr) #elif defined(__APPLE__) -#define malloc_usable_size(ptr) malloc_size(ptr) +#define osal_malloc_usable_size(ptr) malloc_size(ptr) #elif defined(_MSC_VER) && !MDBX_WITHOUT_MSVC_CRT -#define malloc_usable_size(ptr) _msize(ptr) -#endif /* malloc_usable_size */ +#define osal_malloc_usable_size(ptr) _msize(ptr) +#endif /* osal_malloc_usable_size */ /*----------------------------------------------------------------------------*/ /* OS abstraction layer stuff */ -MDBX_INTERNAL_VAR unsigned sys_pagesize; -MDBX_MAYBE_UNUSED MDBX_INTERNAL_VAR unsigned sys_pagesize_ln2, - sys_allocation_granularity; - -/* Get the size of a memory page for the system. - * This is the basic size that the platform's memory manager uses, and is - * fundamental to the use of memory-mapped files. */ -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline size_t -osal_syspagesize(void) { - assert(sys_pagesize > 0 && (sys_pagesize & (sys_pagesize - 1)) == 0); - return sys_pagesize; -} - #if defined(_WIN32) || defined(_WIN64) typedef wchar_t pathchar_t; #define MDBX_PRIsPATH "ls" @@ -1249,7 +1190,7 @@ typedef char pathchar_t; typedef struct osal_mmap { union { void *base; - struct MDBX_lockinfo *lck; + struct shared_lck *lck; }; mdbx_filehandle_t fd; size_t limit; /* mapping length, but NOT a size of file nor DB */ @@ -1260,25 +1201,6 @@ typedef struct osal_mmap { #endif } osal_mmap_t; -typedef union bin128 { - __anonymous_struct_extension__ struct { - uint64_t x, y; - }; - __anonymous_struct_extension__ struct { - uint32_t a, b, c, d; - }; -} bin128_t; - -#if defined(_WIN32) || defined(_WIN64) -typedef union osal_srwlock { - __anonymous_struct_extension__ struct { - long volatile readerCount; - long volatile writerCount; - }; - RTL_SRWLOCK native; -} osal_srwlock_t; -#endif /* Windows */ - #ifndef MDBX_HAVE_PWRITEV #if defined(_WIN32) || defined(_WIN64) @@ -1287,14 +1209,21 @@ typedef union osal_srwlock { #elif defined(__ANDROID_API__) #if __ANDROID_API__ < 24 +/* https://android-developers.googleblog.com/2017/09/introducing-android-native-development.html + * https://android.googlesource.com/platform/bionic/+/master/docs/32-bit-abi.md */ #define MDBX_HAVE_PWRITEV 0 +#if defined(_FILE_OFFSET_BITS) && _FILE_OFFSET_BITS != MDBX_WORDBITS +#error "_FILE_OFFSET_BITS != MDBX_WORDBITS and __ANDROID_API__ < 24" (_FILE_OFFSET_BITS != MDBX_WORDBITS) +#elif defined(__FILE_OFFSET_BITS) && __FILE_OFFSET_BITS != MDBX_WORDBITS +#error "__FILE_OFFSET_BITS != MDBX_WORDBITS and __ANDROID_API__ < 24" (__FILE_OFFSET_BITS != MDBX_WORDBITS) +#endif #else #define MDBX_HAVE_PWRITEV 1 #endif #elif defined(__APPLE__) || defined(__MACH__) || defined(_DARWIN_C_SOURCE) -#if defined(MAC_OS_X_VERSION_MIN_REQUIRED) && defined(MAC_OS_VERSION_11_0) && \ +#if defined(MAC_OS_X_VERSION_MIN_REQUIRED) && defined(MAC_OS_VERSION_11_0) && \ MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_VERSION_11_0 /* FIXME: add checks for IOS versions, etc */ #define MDBX_HAVE_PWRITEV 1 @@ -1312,20 +1241,20 @@ typedef union osal_srwlock { typedef struct ior_item { #if defined(_WIN32) || defined(_WIN64) OVERLAPPED ov; -#define ior_svg_gap4terminator 1 +#define ior_sgv_gap4terminator 1 #define ior_sgv_element FILE_SEGMENT_ELEMENT #else size_t offset; #if MDBX_HAVE_PWRITEV size_t sgvcnt; -#define ior_svg_gap4terminator 0 +#define ior_sgv_gap4terminator 0 #define ior_sgv_element struct iovec #endif /* MDBX_HAVE_PWRITEV */ #endif /* !Windows */ union { MDBX_val single; #if defined(ior_sgv_element) - ior_sgv_element sgv[1 + ior_svg_gap4terminator]; + ior_sgv_element sgv[1 + ior_sgv_gap4terminator]; #endif /* ior_sgv_element */ }; } ior_item_t; @@ -1361,45 +1290,33 @@ typedef struct osal_ioring { char *boundary; } osal_ioring_t; -#ifndef __cplusplus - /* Actually this is not ioring for now, but on the way. */ -MDBX_INTERNAL_FUNC int osal_ioring_create(osal_ioring_t * +MDBX_INTERNAL int osal_ioring_create(osal_ioring_t * #if defined(_WIN32) || defined(_WIN64) - , - bool enable_direct, - mdbx_filehandle_t overlapped_fd + , + bool enable_direct, mdbx_filehandle_t overlapped_fd #endif /* Windows */ ); -MDBX_INTERNAL_FUNC int osal_ioring_resize(osal_ioring_t *, size_t items); -MDBX_INTERNAL_FUNC void osal_ioring_destroy(osal_ioring_t *); -MDBX_INTERNAL_FUNC void osal_ioring_reset(osal_ioring_t *); -MDBX_INTERNAL_FUNC int osal_ioring_add(osal_ioring_t *ctx, const size_t offset, - void *data, const size_t bytes); +MDBX_INTERNAL int osal_ioring_resize(osal_ioring_t *, size_t items); +MDBX_INTERNAL void osal_ioring_destroy(osal_ioring_t *); +MDBX_INTERNAL void osal_ioring_reset(osal_ioring_t *); +MDBX_INTERNAL int osal_ioring_add(osal_ioring_t *ctx, const size_t offset, void *data, const size_t bytes); typedef struct osal_ioring_write_result { int err; unsigned wops; } osal_ioring_write_result_t; -MDBX_INTERNAL_FUNC osal_ioring_write_result_t -osal_ioring_write(osal_ioring_t *ior, mdbx_filehandle_t fd); +MDBX_INTERNAL osal_ioring_write_result_t osal_ioring_write(osal_ioring_t *ior, mdbx_filehandle_t fd); -typedef struct iov_ctx iov_ctx_t; -MDBX_INTERNAL_FUNC void osal_ioring_walk( - osal_ioring_t *ior, iov_ctx_t *ctx, - void (*callback)(iov_ctx_t *ctx, size_t offset, void *data, size_t bytes)); +MDBX_INTERNAL void osal_ioring_walk(osal_ioring_t *ior, iov_ctx_t *ctx, + void (*callback)(iov_ctx_t *ctx, size_t offset, void *data, size_t bytes)); -MDBX_MAYBE_UNUSED static inline unsigned -osal_ioring_left(const osal_ioring_t *ior) { - return ior->slots_left; -} +MDBX_MAYBE_UNUSED static inline unsigned osal_ioring_left(const osal_ioring_t *ior) { return ior->slots_left; } -MDBX_MAYBE_UNUSED static inline unsigned -osal_ioring_used(const osal_ioring_t *ior) { +MDBX_MAYBE_UNUSED static inline unsigned osal_ioring_used(const osal_ioring_t *ior) { return ior->allocated - ior->slots_left; } -MDBX_MAYBE_UNUSED static inline int -osal_ioring_prepare(osal_ioring_t *ior, size_t items, size_t bytes) { +MDBX_MAYBE_UNUSED static inline int osal_ioring_prepare(osal_ioring_t *ior, size_t items, size_t bytes) { items = (items > 32) ? items : 32; #if defined(_WIN32) || defined(_WIN64) if (ior->direct) { @@ -1418,14 +1335,12 @@ osal_ioring_prepare(osal_ioring_t *ior, size_t items, size_t bytes) { /*----------------------------------------------------------------------------*/ /* libc compatibility stuff */ -#if (!defined(__GLIBC__) && __GLIBC_PREREQ(2, 1)) && \ - (defined(_GNU_SOURCE) || defined(_BSD_SOURCE)) +#if (!defined(__GLIBC__) && __GLIBC_PREREQ(2, 1)) && (defined(_GNU_SOURCE) || defined(_BSD_SOURCE)) #define osal_asprintf asprintf #define osal_vasprintf vasprintf #else -MDBX_MAYBE_UNUSED MDBX_INTERNAL_FUNC - MDBX_PRINTF_ARGS(2, 3) int osal_asprintf(char **strp, const char *fmt, ...); -MDBX_INTERNAL_FUNC int osal_vasprintf(char **strp, const char *fmt, va_list ap); +MDBX_MAYBE_UNUSED MDBX_INTERNAL MDBX_PRINTF_ARGS(2, 3) int osal_asprintf(char **strp, const char *fmt, ...); +MDBX_INTERNAL int osal_vasprintf(char **strp, const char *fmt, va_list ap); #endif #if !defined(MADV_DODUMP) && defined(MADV_CORE) @@ -1436,8 +1351,7 @@ MDBX_INTERNAL_FUNC int osal_vasprintf(char **strp, const char *fmt, va_list ap); #define MADV_DONTDUMP MADV_NOCORE #endif /* MADV_NOCORE -> MADV_DONTDUMP */ -MDBX_MAYBE_UNUSED MDBX_INTERNAL_FUNC void osal_jitter(bool tiny); -MDBX_MAYBE_UNUSED static __inline void jitter4testing(bool tiny); +MDBX_MAYBE_UNUSED MDBX_INTERNAL void osal_jitter(bool tiny); /* max bytes to write in one call */ #if defined(_WIN64) @@ -1447,14 +1361,12 @@ MDBX_MAYBE_UNUSED static __inline void jitter4testing(bool tiny); #else #define MAX_WRITE UINT32_C(0x3f000000) -#if defined(F_GETLK64) && defined(F_SETLK64) && defined(F_SETLKW64) && \ - !defined(__ANDROID_API__) +#if defined(F_GETLK64) && defined(F_SETLK64) && defined(F_SETLKW64) && !defined(__ANDROID_API__) #define MDBX_F_SETLK F_SETLK64 #define MDBX_F_SETLKW F_SETLKW64 #define MDBX_F_GETLK F_GETLK64 -#if (__GLIBC_PREREQ(2, 28) && \ - (defined(__USE_LARGEFILE64) || defined(__LARGEFILE64_SOURCE) || \ - defined(_USE_LARGEFILE64) || defined(_LARGEFILE64_SOURCE))) || \ +#if (__GLIBC_PREREQ(2, 28) && (defined(__USE_LARGEFILE64) || defined(__LARGEFILE64_SOURCE) || \ + defined(_USE_LARGEFILE64) || defined(_LARGEFILE64_SOURCE))) || \ defined(fcntl64) #define MDBX_FCNTL fcntl64 #else @@ -1472,8 +1384,7 @@ MDBX_MAYBE_UNUSED static __inline void jitter4testing(bool tiny); #define MDBX_STRUCT_FLOCK struct flock #endif /* MDBX_F_SETLK, MDBX_F_SETLKW, MDBX_F_GETLK */ -#if defined(F_OFD_SETLK64) && defined(F_OFD_SETLKW64) && \ - defined(F_OFD_GETLK64) && !defined(__ANDROID_API__) +#if defined(F_OFD_SETLK64) && defined(F_OFD_SETLKW64) && defined(F_OFD_GETLK64) && !defined(__ANDROID_API__) #define MDBX_F_OFD_SETLK F_OFD_SETLK64 #define MDBX_F_OFD_SETLKW F_OFD_SETLKW64 #define MDBX_F_OFD_GETLK F_OFD_GETLK64 @@ -1482,23 +1393,17 @@ MDBX_MAYBE_UNUSED static __inline void jitter4testing(bool tiny); #define MDBX_F_OFD_SETLKW F_OFD_SETLKW #define MDBX_F_OFD_GETLK F_OFD_GETLK #ifndef OFF_T_MAX -#define OFF_T_MAX \ - (((sizeof(off_t) > 4) ? INT64_MAX : INT32_MAX) & ~(size_t)0xFffff) +#define OFF_T_MAX (((sizeof(off_t) > 4) ? INT64_MAX : INT32_MAX) & ~(size_t)0xFffff) #endif /* OFF_T_MAX */ #endif /* MDBX_F_OFD_SETLK64, MDBX_F_OFD_SETLKW64, MDBX_F_OFD_GETLK64 */ -#endif - -#if defined(__linux__) || defined(__gnu_linux__) -MDBX_INTERNAL_VAR uint32_t linux_kernel_version; -MDBX_INTERNAL_VAR bool mdbx_RunningOnWSL1 /* Windows Subsystem 1 for Linux */; -#endif /* Linux */ +#endif /* !Windows */ #ifndef osal_strdup LIBMDBX_API char *osal_strdup(const char *str); #endif -MDBX_MAYBE_UNUSED static __inline int osal_get_errno(void) { +MDBX_MAYBE_UNUSED static inline int osal_get_errno(void) { #if defined(_WIN32) || defined(_WIN64) DWORD rc = GetLastError(); #else @@ -1508,40 +1413,32 @@ MDBX_MAYBE_UNUSED static __inline int osal_get_errno(void) { } #ifndef osal_memalign_alloc -MDBX_INTERNAL_FUNC int osal_memalign_alloc(size_t alignment, size_t bytes, - void **result); +MDBX_INTERNAL int osal_memalign_alloc(size_t alignment, size_t bytes, void **result); #endif #ifndef osal_memalign_free -MDBX_INTERNAL_FUNC void osal_memalign_free(void *ptr); -#endif - -MDBX_INTERNAL_FUNC int osal_condpair_init(osal_condpair_t *condpair); -MDBX_INTERNAL_FUNC int osal_condpair_lock(osal_condpair_t *condpair); -MDBX_INTERNAL_FUNC int osal_condpair_unlock(osal_condpair_t *condpair); -MDBX_INTERNAL_FUNC int osal_condpair_signal(osal_condpair_t *condpair, - bool part); -MDBX_INTERNAL_FUNC int osal_condpair_wait(osal_condpair_t *condpair, bool part); -MDBX_INTERNAL_FUNC int osal_condpair_destroy(osal_condpair_t *condpair); - -MDBX_INTERNAL_FUNC int osal_fastmutex_init(osal_fastmutex_t *fastmutex); -MDBX_INTERNAL_FUNC int osal_fastmutex_acquire(osal_fastmutex_t *fastmutex); -MDBX_INTERNAL_FUNC int osal_fastmutex_release(osal_fastmutex_t *fastmutex); -MDBX_INTERNAL_FUNC int osal_fastmutex_destroy(osal_fastmutex_t *fastmutex); - -MDBX_INTERNAL_FUNC int osal_pwritev(mdbx_filehandle_t fd, struct iovec *iov, - size_t sgvcnt, uint64_t offset); -MDBX_INTERNAL_FUNC int osal_pread(mdbx_filehandle_t fd, void *buf, size_t count, - uint64_t offset); -MDBX_INTERNAL_FUNC int osal_pwrite(mdbx_filehandle_t fd, const void *buf, - size_t count, uint64_t offset); -MDBX_INTERNAL_FUNC int osal_write(mdbx_filehandle_t fd, const void *buf, - size_t count); - -MDBX_INTERNAL_FUNC int -osal_thread_create(osal_thread_t *thread, - THREAD_RESULT(THREAD_CALL *start_routine)(void *), - void *arg); -MDBX_INTERNAL_FUNC int osal_thread_join(osal_thread_t thread); +MDBX_INTERNAL void osal_memalign_free(void *ptr); +#endif + +MDBX_INTERNAL int osal_condpair_init(osal_condpair_t *condpair); +MDBX_INTERNAL int osal_condpair_lock(osal_condpair_t *condpair); +MDBX_INTERNAL int osal_condpair_unlock(osal_condpair_t *condpair); +MDBX_INTERNAL int osal_condpair_signal(osal_condpair_t *condpair, bool part); +MDBX_INTERNAL int osal_condpair_wait(osal_condpair_t *condpair, bool part); +MDBX_INTERNAL int osal_condpair_destroy(osal_condpair_t *condpair); + +MDBX_INTERNAL int osal_fastmutex_init(osal_fastmutex_t *fastmutex); +MDBX_INTERNAL int osal_fastmutex_acquire(osal_fastmutex_t *fastmutex); +MDBX_INTERNAL int osal_fastmutex_release(osal_fastmutex_t *fastmutex); +MDBX_INTERNAL int osal_fastmutex_destroy(osal_fastmutex_t *fastmutex); + +MDBX_INTERNAL int osal_pwritev(mdbx_filehandle_t fd, struct iovec *iov, size_t sgvcnt, uint64_t offset); +MDBX_INTERNAL int osal_pread(mdbx_filehandle_t fd, void *buf, size_t count, uint64_t offset); +MDBX_INTERNAL int osal_pwrite(mdbx_filehandle_t fd, const void *buf, size_t count, uint64_t offset); +MDBX_INTERNAL int osal_write(mdbx_filehandle_t fd, const void *buf, size_t count); + +MDBX_INTERNAL int osal_thread_create(osal_thread_t *thread, THREAD_RESULT(THREAD_CALL *start_routine)(void *), + void *arg); +MDBX_INTERNAL int osal_thread_join(osal_thread_t thread); enum osal_syncmode_bits { MDBX_SYNC_NONE = 0, @@ -1551,11 +1448,10 @@ enum osal_syncmode_bits { MDBX_SYNC_IODQ = 8 }; -MDBX_INTERNAL_FUNC int osal_fsync(mdbx_filehandle_t fd, - const enum osal_syncmode_bits mode_bits); -MDBX_INTERNAL_FUNC int osal_ftruncate(mdbx_filehandle_t fd, uint64_t length); -MDBX_INTERNAL_FUNC int osal_fseek(mdbx_filehandle_t fd, uint64_t pos); -MDBX_INTERNAL_FUNC int osal_filesize(mdbx_filehandle_t fd, uint64_t *length); +MDBX_INTERNAL int osal_fsync(mdbx_filehandle_t fd, const enum osal_syncmode_bits mode_bits); +MDBX_INTERNAL int osal_ftruncate(mdbx_filehandle_t fd, uint64_t length); +MDBX_INTERNAL int osal_fseek(mdbx_filehandle_t fd, uint64_t pos); +MDBX_INTERNAL int osal_filesize(mdbx_filehandle_t fd, uint64_t *length); enum osal_openfile_purpose { MDBX_OPEN_DXB_READ, @@ -1570,7 +1466,7 @@ enum osal_openfile_purpose { MDBX_OPEN_DELETE }; -MDBX_MAYBE_UNUSED static __inline bool osal_isdirsep(pathchar_t c) { +MDBX_MAYBE_UNUSED static inline bool osal_isdirsep(pathchar_t c) { return #if defined(_WIN32) || defined(_WIN64) c == '\\' || @@ -1578,50 +1474,39 @@ MDBX_MAYBE_UNUSED static __inline bool osal_isdirsep(pathchar_t c) { c == '/'; } -MDBX_INTERNAL_FUNC bool osal_pathequal(const pathchar_t *l, const pathchar_t *r, - size_t len); -MDBX_INTERNAL_FUNC pathchar_t *osal_fileext(const pathchar_t *pathname, - size_t len); -MDBX_INTERNAL_FUNC int osal_fileexists(const pathchar_t *pathname); -MDBX_INTERNAL_FUNC int osal_openfile(const enum osal_openfile_purpose purpose, - const MDBX_env *env, - const pathchar_t *pathname, - mdbx_filehandle_t *fd, - mdbx_mode_t unix_mode_bits); -MDBX_INTERNAL_FUNC int osal_closefile(mdbx_filehandle_t fd); -MDBX_INTERNAL_FUNC int osal_removefile(const pathchar_t *pathname); -MDBX_INTERNAL_FUNC int osal_removedirectory(const pathchar_t *pathname); -MDBX_INTERNAL_FUNC int osal_is_pipe(mdbx_filehandle_t fd); -MDBX_INTERNAL_FUNC int osal_lockfile(mdbx_filehandle_t fd, bool wait); +MDBX_INTERNAL bool osal_pathequal(const pathchar_t *l, const pathchar_t *r, size_t len); +MDBX_INTERNAL pathchar_t *osal_fileext(const pathchar_t *pathname, size_t len); +MDBX_INTERNAL int osal_fileexists(const pathchar_t *pathname); +MDBX_INTERNAL int osal_openfile(const enum osal_openfile_purpose purpose, const MDBX_env *env, + const pathchar_t *pathname, mdbx_filehandle_t *fd, mdbx_mode_t unix_mode_bits); +MDBX_INTERNAL int osal_closefile(mdbx_filehandle_t fd); +MDBX_INTERNAL int osal_removefile(const pathchar_t *pathname); +MDBX_INTERNAL int osal_removedirectory(const pathchar_t *pathname); +MDBX_INTERNAL int osal_is_pipe(mdbx_filehandle_t fd); +MDBX_INTERNAL int osal_lockfile(mdbx_filehandle_t fd, bool wait); #define MMAP_OPTION_TRUNCATE 1 #define MMAP_OPTION_SEMAPHORE 2 -MDBX_INTERNAL_FUNC int osal_mmap(const int flags, osal_mmap_t *map, size_t size, - const size_t limit, const unsigned options); -MDBX_INTERNAL_FUNC int osal_munmap(osal_mmap_t *map); +MDBX_INTERNAL int osal_mmap(const int flags, osal_mmap_t *map, size_t size, const size_t limit, const unsigned options, + const pathchar_t *pathname4logging); +MDBX_INTERNAL int osal_munmap(osal_mmap_t *map); #define MDBX_MRESIZE_MAY_MOVE 0x00000100 #define MDBX_MRESIZE_MAY_UNMAP 0x00000200 -MDBX_INTERNAL_FUNC int osal_mresize(const int flags, osal_mmap_t *map, - size_t size, size_t limit); +MDBX_INTERNAL int osal_mresize(const int flags, osal_mmap_t *map, size_t size, size_t limit); #if defined(_WIN32) || defined(_WIN64) typedef struct { unsigned limit, count; HANDLE handles[31]; } mdbx_handle_array_t; -MDBX_INTERNAL_FUNC int -osal_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array); -MDBX_INTERNAL_FUNC int -osal_resume_threads_after_remap(mdbx_handle_array_t *array); +MDBX_INTERNAL int osal_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array); +MDBX_INTERNAL int osal_resume_threads_after_remap(mdbx_handle_array_t *array); #endif /* Windows */ -MDBX_INTERNAL_FUNC int osal_msync(const osal_mmap_t *map, size_t offset, - size_t length, - enum osal_syncmode_bits mode_bits); -MDBX_INTERNAL_FUNC int osal_check_fs_rdonly(mdbx_filehandle_t handle, - const pathchar_t *pathname, - int err); -MDBX_INTERNAL_FUNC int osal_check_fs_incore(mdbx_filehandle_t handle); - -MDBX_MAYBE_UNUSED static __inline uint32_t osal_getpid(void) { +MDBX_INTERNAL int osal_msync(const osal_mmap_t *map, size_t offset, size_t length, enum osal_syncmode_bits mode_bits); +MDBX_INTERNAL int osal_check_fs_rdonly(mdbx_filehandle_t handle, const pathchar_t *pathname, int err); +MDBX_INTERNAL int osal_check_fs_incore(mdbx_filehandle_t handle); +MDBX_INTERNAL int osal_check_fs_local(mdbx_filehandle_t handle, int flags); + +MDBX_MAYBE_UNUSED static inline uint32_t osal_getpid(void) { STATIC_ASSERT(sizeof(mdbx_pid_t) <= sizeof(uint32_t)); #if defined(_WIN32) || defined(_WIN64) return GetCurrentProcessId(); @@ -1631,7 +1516,7 @@ MDBX_MAYBE_UNUSED static __inline uint32_t osal_getpid(void) { #endif } -MDBX_MAYBE_UNUSED static __inline uintptr_t osal_thread_self(void) { +MDBX_MAYBE_UNUSED static inline uintptr_t osal_thread_self(void) { mdbx_tid_t thunk; STATIC_ASSERT(sizeof(uintptr_t) >= sizeof(thunk)); #if defined(_WIN32) || defined(_WIN64) @@ -1644,274 +1529,51 @@ MDBX_MAYBE_UNUSED static __inline uintptr_t osal_thread_self(void) { #if !defined(_WIN32) && !defined(_WIN64) #if defined(__ANDROID_API__) || defined(ANDROID) || defined(BIONIC) -MDBX_INTERNAL_FUNC int osal_check_tid4bionic(void); +MDBX_INTERNAL int osal_check_tid4bionic(void); #else -static __inline int osal_check_tid4bionic(void) { return 0; } +static inline int osal_check_tid4bionic(void) { return 0; } #endif /* __ANDROID_API__ || ANDROID) || BIONIC */ -MDBX_MAYBE_UNUSED static __inline int -osal_pthread_mutex_lock(pthread_mutex_t *mutex) { +MDBX_MAYBE_UNUSED static inline int osal_pthread_mutex_lock(pthread_mutex_t *mutex) { int err = osal_check_tid4bionic(); return unlikely(err) ? err : pthread_mutex_lock(mutex); } #endif /* !Windows */ -MDBX_INTERNAL_FUNC uint64_t osal_monotime(void); -MDBX_INTERNAL_FUNC uint64_t osal_cputime(size_t *optional_page_faults); -MDBX_INTERNAL_FUNC uint64_t osal_16dot16_to_monotime(uint32_t seconds_16dot16); -MDBX_INTERNAL_FUNC uint32_t osal_monotime_to_16dot16(uint64_t monotime); +MDBX_INTERNAL uint64_t osal_monotime(void); +MDBX_INTERNAL uint64_t osal_cputime(size_t *optional_page_faults); +MDBX_INTERNAL uint64_t osal_16dot16_to_monotime(uint32_t seconds_16dot16); +MDBX_INTERNAL uint32_t osal_monotime_to_16dot16(uint64_t monotime); -MDBX_MAYBE_UNUSED static inline uint32_t -osal_monotime_to_16dot16_noUnderflow(uint64_t monotime) { +MDBX_MAYBE_UNUSED static inline uint32_t osal_monotime_to_16dot16_noUnderflow(uint64_t monotime) { uint32_t seconds_16dot16 = osal_monotime_to_16dot16(monotime); return seconds_16dot16 ? seconds_16dot16 : /* fix underflow */ (monotime > 0); } -MDBX_INTERNAL_FUNC bin128_t osal_bootid(void); /*----------------------------------------------------------------------------*/ -/* lck stuff */ - -/// \brief Initialization of synchronization primitives linked with MDBX_env -/// instance both in LCK-file and within the current process. -/// \param -/// global_uniqueness_flag = true - denotes that there are no other processes -/// working with DB and LCK-file. Thus the function MUST initialize -/// shared synchronization objects in memory-mapped LCK-file. -/// global_uniqueness_flag = false - denotes that at least one process is -/// already working with DB and LCK-file, including the case when DB -/// has already been opened in the current process. Thus the function -/// MUST NOT initialize shared synchronization objects in memory-mapped -/// LCK-file that are already in use. -/// \return Error code or zero on success. -MDBX_INTERNAL_FUNC int osal_lck_init(MDBX_env *env, - MDBX_env *inprocess_neighbor, - int global_uniqueness_flag); - -/// \brief Disconnects from shared interprocess objects and destructs -/// synchronization objects linked with MDBX_env instance -/// within the current process. -/// \param -/// inprocess_neighbor = NULL - if the current process does not have other -/// instances of MDBX_env linked with the DB being closed. -/// Thus the function MUST check for other processes working with DB or -/// LCK-file, and keep or destroy shared synchronization objects in -/// memory-mapped LCK-file depending on the result. -/// inprocess_neighbor = not-NULL - pointer to another instance of MDBX_env -/// (anyone of there is several) working with DB or LCK-file within the -/// current process. Thus the function MUST NOT try to acquire exclusive -/// lock and/or try to destruct shared synchronization objects linked with -/// DB or LCK-file. Moreover, the implementation MUST ensure correct work -/// of other instances of MDBX_env within the current process, e.g. -/// restore POSIX-fcntl locks after the closing of file descriptors. -/// \return Error code (MDBX_PANIC) or zero on success. -MDBX_INTERNAL_FUNC int osal_lck_destroy(MDBX_env *env, - MDBX_env *inprocess_neighbor); - -/// \brief Connects to shared interprocess locking objects and tries to acquire -/// the maximum lock level (shared if exclusive is not available) -/// Depending on implementation or/and platform (Windows) this function may -/// acquire the non-OS super-level lock (e.g. for shared synchronization -/// objects initialization), which will be downgraded to OS-exclusive or -/// shared via explicit calling of osal_lck_downgrade(). -/// \return -/// MDBX_RESULT_TRUE (-1) - if an exclusive lock was acquired and thus -/// the current process is the first and only after the last use of DB. -/// MDBX_RESULT_FALSE (0) - if a shared lock was acquired and thus -/// DB has already been opened and now is used by other processes. -/// Otherwise (not 0 and not -1) - error code. -MDBX_INTERNAL_FUNC int osal_lck_seize(MDBX_env *env); - -/// \brief Downgrades the level of initially acquired lock to -/// operational level specified by argument. The reason for such downgrade: -/// - unblocking of other processes that are waiting for access, i.e. -/// if (env->me_flags & MDBX_EXCLUSIVE) != 0, then other processes -/// should be made aware that access is unavailable rather than -/// wait for it. -/// - freeing locks that interfere file operation (especially for Windows) -/// (env->me_flags & MDBX_EXCLUSIVE) == 0 - downgrade to shared lock. -/// (env->me_flags & MDBX_EXCLUSIVE) != 0 - downgrade to exclusive -/// operational lock. -/// \return Error code or zero on success -MDBX_INTERNAL_FUNC int osal_lck_downgrade(MDBX_env *env); - -/// \brief Locks LCK-file or/and table of readers for (de)registering. -/// \return Error code or zero on success -MDBX_INTERNAL_FUNC int osal_rdt_lock(MDBX_env *env); - -/// \brief Unlocks LCK-file or/and table of readers after (de)registering. -MDBX_INTERNAL_FUNC void osal_rdt_unlock(MDBX_env *env); - -/// \brief Acquires lock for DB change (on writing transaction start) -/// Reading transactions will not be blocked. -/// Declared as LIBMDBX_API because it is used in mdbx_chk. -/// \return Error code or zero on success -LIBMDBX_API int mdbx_txn_lock(MDBX_env *env, bool dont_wait); - -/// \brief Releases lock once DB changes is made (after writing transaction -/// has finished). -/// Declared as LIBMDBX_API because it is used in mdbx_chk. -LIBMDBX_API void mdbx_txn_unlock(MDBX_env *env); - -/// \brief Sets alive-flag of reader presence (indicative lock) for PID of -/// the current process. The function does no more than needed for -/// the correct working of osal_rpid_check() in other processes. -/// \return Error code or zero on success -MDBX_INTERNAL_FUNC int osal_rpid_set(MDBX_env *env); - -/// \brief Resets alive-flag of reader presence (indicative lock) -/// for PID of the current process. The function does no more than needed -/// for the correct working of osal_rpid_check() in other processes. -/// \return Error code or zero on success -MDBX_INTERNAL_FUNC int osal_rpid_clear(MDBX_env *env); - -/// \brief Checks for reading process status with the given pid with help of -/// alive-flag of presence (indicative lock) or using another way. -/// \return -/// MDBX_RESULT_TRUE (-1) - if the reader process with the given PID is alive -/// and working with DB (indicative lock is present). -/// MDBX_RESULT_FALSE (0) - if the reader process with the given PID is absent -/// or not working with DB (indicative lock is not present). -/// Otherwise (not 0 and not -1) - error code. -MDBX_INTERNAL_FUNC int osal_rpid_check(MDBX_env *env, uint32_t pid); - -#if defined(_WIN32) || defined(_WIN64) - -MDBX_INTERNAL_FUNC int osal_mb2w(const char *const src, wchar_t **const pdst); - -typedef void(WINAPI *osal_srwlock_t_function)(osal_srwlock_t *); -MDBX_INTERNAL_VAR osal_srwlock_t_function osal_srwlock_Init, - osal_srwlock_AcquireShared, osal_srwlock_ReleaseShared, - osal_srwlock_AcquireExclusive, osal_srwlock_ReleaseExclusive; - -#if _WIN32_WINNT < 0x0600 /* prior to Windows Vista */ -typedef enum _FILE_INFO_BY_HANDLE_CLASS { - FileBasicInfo, - FileStandardInfo, - FileNameInfo, - FileRenameInfo, - FileDispositionInfo, - FileAllocationInfo, - FileEndOfFileInfo, - FileStreamInfo, - FileCompressionInfo, - FileAttributeTagInfo, - FileIdBothDirectoryInfo, - FileIdBothDirectoryRestartInfo, - FileIoPriorityHintInfo, - FileRemoteProtocolInfo, - MaximumFileInfoByHandleClass -} FILE_INFO_BY_HANDLE_CLASS, - *PFILE_INFO_BY_HANDLE_CLASS; - -typedef struct _FILE_END_OF_FILE_INFO { - LARGE_INTEGER EndOfFile; -} FILE_END_OF_FILE_INFO, *PFILE_END_OF_FILE_INFO; - -#define REMOTE_PROTOCOL_INFO_FLAG_LOOPBACK 0x00000001 -#define REMOTE_PROTOCOL_INFO_FLAG_OFFLINE 0x00000002 - -typedef struct _FILE_REMOTE_PROTOCOL_INFO { - USHORT StructureVersion; - USHORT StructureSize; - DWORD Protocol; - USHORT ProtocolMajorVersion; - USHORT ProtocolMinorVersion; - USHORT ProtocolRevision; - USHORT Reserved; - DWORD Flags; - struct { - DWORD Reserved[8]; - } GenericReserved; - struct { - DWORD Reserved[16]; - } ProtocolSpecificReserved; -} FILE_REMOTE_PROTOCOL_INFO, *PFILE_REMOTE_PROTOCOL_INFO; - -#endif /* _WIN32_WINNT < 0x0600 (prior to Windows Vista) */ - -typedef BOOL(WINAPI *MDBX_GetFileInformationByHandleEx)( - _In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass, - _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize); -MDBX_INTERNAL_VAR MDBX_GetFileInformationByHandleEx - mdbx_GetFileInformationByHandleEx; - -typedef BOOL(WINAPI *MDBX_GetVolumeInformationByHandleW)( - _In_ HANDLE hFile, _Out_opt_ LPWSTR lpVolumeNameBuffer, - _In_ DWORD nVolumeNameSize, _Out_opt_ LPDWORD lpVolumeSerialNumber, - _Out_opt_ LPDWORD lpMaximumComponentLength, - _Out_opt_ LPDWORD lpFileSystemFlags, - _Out_opt_ LPWSTR lpFileSystemNameBuffer, _In_ DWORD nFileSystemNameSize); -MDBX_INTERNAL_VAR MDBX_GetVolumeInformationByHandleW - mdbx_GetVolumeInformationByHandleW; - -typedef DWORD(WINAPI *MDBX_GetFinalPathNameByHandleW)(_In_ HANDLE hFile, - _Out_ LPWSTR lpszFilePath, - _In_ DWORD cchFilePath, - _In_ DWORD dwFlags); -MDBX_INTERNAL_VAR MDBX_GetFinalPathNameByHandleW mdbx_GetFinalPathNameByHandleW; - -typedef BOOL(WINAPI *MDBX_SetFileInformationByHandle)( - _In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass, - _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize); -MDBX_INTERNAL_VAR MDBX_SetFileInformationByHandle - mdbx_SetFileInformationByHandle; - -typedef NTSTATUS(NTAPI *MDBX_NtFsControlFile)( - IN HANDLE FileHandle, IN OUT HANDLE Event, - IN OUT PVOID /* PIO_APC_ROUTINE */ ApcRoutine, IN OUT PVOID ApcContext, - OUT PIO_STATUS_BLOCK IoStatusBlock, IN ULONG FsControlCode, - IN OUT PVOID InputBuffer, IN ULONG InputBufferLength, - OUT OPTIONAL PVOID OutputBuffer, IN ULONG OutputBufferLength); -MDBX_INTERNAL_VAR MDBX_NtFsControlFile mdbx_NtFsControlFile; - -typedef uint64_t(WINAPI *MDBX_GetTickCount64)(void); -MDBX_INTERNAL_VAR MDBX_GetTickCount64 mdbx_GetTickCount64; - -#if !defined(_WIN32_WINNT_WIN8) || _WIN32_WINNT < _WIN32_WINNT_WIN8 -typedef struct _WIN32_MEMORY_RANGE_ENTRY { - PVOID VirtualAddress; - SIZE_T NumberOfBytes; -} WIN32_MEMORY_RANGE_ENTRY, *PWIN32_MEMORY_RANGE_ENTRY; -#endif /* Windows 8.x */ - -typedef BOOL(WINAPI *MDBX_PrefetchVirtualMemory)( - HANDLE hProcess, ULONG_PTR NumberOfEntries, - PWIN32_MEMORY_RANGE_ENTRY VirtualAddresses, ULONG Flags); -MDBX_INTERNAL_VAR MDBX_PrefetchVirtualMemory mdbx_PrefetchVirtualMemory; - -typedef enum _SECTION_INHERIT { ViewShare = 1, ViewUnmap = 2 } SECTION_INHERIT; - -typedef NTSTATUS(NTAPI *MDBX_NtExtendSection)(IN HANDLE SectionHandle, - IN PLARGE_INTEGER NewSectionSize); -MDBX_INTERNAL_VAR MDBX_NtExtendSection mdbx_NtExtendSection; - -static __inline bool mdbx_RunningUnderWine(void) { - return !mdbx_NtExtendSection; -} - -typedef LSTATUS(WINAPI *MDBX_RegGetValueA)(HKEY hkey, LPCSTR lpSubKey, - LPCSTR lpValue, DWORD dwFlags, - LPDWORD pdwType, PVOID pvData, - LPDWORD pcbData); -MDBX_INTERNAL_VAR MDBX_RegGetValueA mdbx_RegGetValueA; -NTSYSAPI ULONG RtlRandomEx(PULONG Seed); - -typedef BOOL(WINAPI *MDBX_SetFileIoOverlappedRange)(HANDLE FileHandle, - PUCHAR OverlappedRangeStart, - ULONG Length); -MDBX_INTERNAL_VAR MDBX_SetFileIoOverlappedRange mdbx_SetFileIoOverlappedRange; +MDBX_INTERNAL void osal_ctor(void); +MDBX_INTERNAL void osal_dtor(void); +#if defined(_WIN32) || defined(_WIN64) +MDBX_INTERNAL int osal_mb2w(const char *const src, wchar_t **const pdst); #endif /* Windows */ -#endif /* !__cplusplus */ +typedef union bin128 { + __anonymous_struct_extension__ struct { + uint64_t x, y; + }; + __anonymous_struct_extension__ struct { + uint32_t a, b, c, d; + }; +} bin128_t; + +MDBX_INTERNAL bin128_t osal_guid(const MDBX_env *); /*----------------------------------------------------------------------------*/ -MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static __always_inline uint64_t -osal_bswap64(uint64_t v) { -#if __GNUC_PREREQ(4, 4) || __CLANG_PREREQ(4, 0) || \ - __has_builtin(__builtin_bswap64) +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint64_t osal_bswap64(uint64_t v) { +#if __GNUC_PREREQ(4, 4) || __CLANG_PREREQ(4, 0) || __has_builtin(__builtin_bswap64) return __builtin_bswap64(v); #elif defined(_MSC_VER) && !defined(__clang__) return _byteswap_uint64(v); @@ -1920,19 +1582,14 @@ osal_bswap64(uint64_t v) { #elif defined(bswap_64) return bswap_64(v); #else - return v << 56 | v >> 56 | ((v << 40) & UINT64_C(0x00ff000000000000)) | - ((v << 24) & UINT64_C(0x0000ff0000000000)) | - ((v << 8) & UINT64_C(0x000000ff00000000)) | - ((v >> 8) & UINT64_C(0x00000000ff000000)) | - ((v >> 24) & UINT64_C(0x0000000000ff0000)) | - ((v >> 40) & UINT64_C(0x000000000000ff00)); + return v << 56 | v >> 56 | ((v << 40) & UINT64_C(0x00ff000000000000)) | ((v << 24) & UINT64_C(0x0000ff0000000000)) | + ((v << 8) & UINT64_C(0x000000ff00000000)) | ((v >> 8) & UINT64_C(0x00000000ff000000)) | + ((v >> 24) & UINT64_C(0x0000000000ff0000)) | ((v >> 40) & UINT64_C(0x000000000000ff00)); #endif } -MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static __always_inline uint32_t -osal_bswap32(uint32_t v) { -#if __GNUC_PREREQ(4, 4) || __CLANG_PREREQ(4, 0) || \ - __has_builtin(__builtin_bswap32) +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint32_t osal_bswap32(uint32_t v) { +#if __GNUC_PREREQ(4, 4) || __CLANG_PREREQ(4, 0) || __has_builtin(__builtin_bswap32) return __builtin_bswap32(v); #elif defined(_MSC_VER) && !defined(__clang__) return _byteswap_ulong(v); @@ -1941,50 +1598,14 @@ osal_bswap32(uint32_t v) { #elif defined(bswap_32) return bswap_32(v); #else - return v << 24 | v >> 24 | ((v << 8) & UINT32_C(0x00ff0000)) | - ((v >> 8) & UINT32_C(0x0000ff00)); + return v << 24 | v >> 24 | ((v << 8) & UINT32_C(0x00ff0000)) | ((v >> 8) & UINT32_C(0x0000ff00)); #endif } -/*----------------------------------------------------------------------------*/ - -#if defined(_MSC_VER) && _MSC_VER >= 1900 -/* LY: MSVC 2015/2017/2019 has buggy/inconsistent PRIuPTR/PRIxPTR macros - * for internal format-args checker. */ -#undef PRIuPTR -#undef PRIiPTR -#undef PRIdPTR -#undef PRIxPTR -#define PRIuPTR "Iu" -#define PRIiPTR "Ii" -#define PRIdPTR "Id" -#define PRIxPTR "Ix" -#define PRIuSIZE "zu" -#define PRIiSIZE "zi" -#define PRIdSIZE "zd" -#define PRIxSIZE "zx" -#endif /* fix PRI*PTR for _MSC_VER */ - -#ifndef PRIuSIZE -#define PRIuSIZE PRIuPTR -#define PRIiSIZE PRIiPTR -#define PRIdSIZE PRIdPTR -#define PRIxSIZE PRIxPTR -#endif /* PRI*SIZE macros for MSVC */ - -#ifdef _MSC_VER -#pragma warning(pop) -#endif - -#define mdbx_sourcery_anchor XCONCAT(mdbx_sourcery_, MDBX_BUILD_SOURCERY) -#if defined(xMDBX_TOOLS) -extern LIBMDBX_API const char *const mdbx_sourcery_anchor; -#endif - /******************************************************************************* - ******************************************************************************* ******************************************************************************* * + * BUILD TIME * * #### ##### ##### # #### # # #### * # # # # # # # # ## # # @@ -2005,23 +1626,15 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Using fsync() with chance of data lost on power failure */ #define MDBX_OSX_WANNA_SPEED 1 -#ifndef MDBX_OSX_SPEED_INSTEADOF_DURABILITY +#ifndef MDBX_APPLE_SPEED_INSTEADOF_DURABILITY /** Choices \ref MDBX_OSX_WANNA_DURABILITY or \ref MDBX_OSX_WANNA_SPEED * for OSX & iOS */ -#define MDBX_OSX_SPEED_INSTEADOF_DURABILITY MDBX_OSX_WANNA_DURABILITY -#endif /* MDBX_OSX_SPEED_INSTEADOF_DURABILITY */ - -/** Controls using of POSIX' madvise() and/or similar hints. */ -#ifndef MDBX_ENABLE_MADVISE -#define MDBX_ENABLE_MADVISE 1 -#elif !(MDBX_ENABLE_MADVISE == 0 || MDBX_ENABLE_MADVISE == 1) -#error MDBX_ENABLE_MADVISE must be defined as 0 or 1 -#endif /* MDBX_ENABLE_MADVISE */ +#define MDBX_APPLE_SPEED_INSTEADOF_DURABILITY MDBX_OSX_WANNA_DURABILITY +#endif /* MDBX_APPLE_SPEED_INSTEADOF_DURABILITY */ /** Controls checking PID against reuse DB environment after the fork() */ #ifndef MDBX_ENV_CHECKPID -#if (defined(MADV_DONTFORK) && MDBX_ENABLE_MADVISE) || defined(_WIN32) || \ - defined(_WIN64) +#if defined(MADV_DONTFORK) || defined(_WIN32) || defined(_WIN64) /* PID check could be omitted: * - on Linux when madvise(MADV_DONTFORK) is available, i.e. after the fork() * mapped pages will not be available for child process. @@ -2050,8 +1663,7 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Does a system have battery-backed Real-Time Clock or just a fake. */ #ifndef MDBX_TRUST_RTC -#if defined(__linux__) || defined(__gnu_linux__) || defined(__NetBSD__) || \ - defined(__OpenBSD__) +#if defined(__linux__) || defined(__gnu_linux__) || defined(__NetBSD__) || defined(__OpenBSD__) #define MDBX_TRUST_RTC 0 /* a lot of embedded systems have a fake RTC */ #else #define MDBX_TRUST_RTC 1 @@ -2086,24 +1698,21 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Controls using Unix' mincore() to determine whether DB-pages * are resident in memory. */ -#ifndef MDBX_ENABLE_MINCORE +#ifndef MDBX_USE_MINCORE #if defined(MINCORE_INCORE) || !(defined(_WIN32) || defined(_WIN64)) -#define MDBX_ENABLE_MINCORE 1 +#define MDBX_USE_MINCORE 1 #else -#define MDBX_ENABLE_MINCORE 0 +#define MDBX_USE_MINCORE 0 #endif -#elif !(MDBX_ENABLE_MINCORE == 0 || MDBX_ENABLE_MINCORE == 1) -#error MDBX_ENABLE_MINCORE must be defined as 0 or 1 -#endif /* MDBX_ENABLE_MINCORE */ +#define MDBX_USE_MINCORE_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_USE_MINCORE) +#elif !(MDBX_USE_MINCORE == 0 || MDBX_USE_MINCORE == 1) +#error MDBX_USE_MINCORE must be defined as 0 or 1 +#endif /* MDBX_USE_MINCORE */ /** Enables chunking long list of retired pages during huge transactions commit * to avoid use sequences of pages. */ #ifndef MDBX_ENABLE_BIGFOOT -#if MDBX_WORDBITS >= 64 || defined(DOXYGEN) #define MDBX_ENABLE_BIGFOOT 1 -#else -#define MDBX_ENABLE_BIGFOOT 0 -#endif #elif !(MDBX_ENABLE_BIGFOOT == 0 || MDBX_ENABLE_BIGFOOT == 1) #error MDBX_ENABLE_BIGFOOT must be defined as 0 or 1 #endif /* MDBX_ENABLE_BIGFOOT */ @@ -2118,25 +1727,27 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #ifndef MDBX_PNL_PREALLOC_FOR_RADIXSORT #define MDBX_PNL_PREALLOC_FOR_RADIXSORT 1 -#elif !(MDBX_PNL_PREALLOC_FOR_RADIXSORT == 0 || \ - MDBX_PNL_PREALLOC_FOR_RADIXSORT == 1) +#elif !(MDBX_PNL_PREALLOC_FOR_RADIXSORT == 0 || MDBX_PNL_PREALLOC_FOR_RADIXSORT == 1) #error MDBX_PNL_PREALLOC_FOR_RADIXSORT must be defined as 0 or 1 #endif /* MDBX_PNL_PREALLOC_FOR_RADIXSORT */ #ifndef MDBX_DPL_PREALLOC_FOR_RADIXSORT #define MDBX_DPL_PREALLOC_FOR_RADIXSORT 1 -#elif !(MDBX_DPL_PREALLOC_FOR_RADIXSORT == 0 || \ - MDBX_DPL_PREALLOC_FOR_RADIXSORT == 1) +#elif !(MDBX_DPL_PREALLOC_FOR_RADIXSORT == 0 || MDBX_DPL_PREALLOC_FOR_RADIXSORT == 1) #error MDBX_DPL_PREALLOC_FOR_RADIXSORT must be defined as 0 or 1 #endif /* MDBX_DPL_PREALLOC_FOR_RADIXSORT */ -/** Controls dirty pages tracking, spilling and persisting in MDBX_WRITEMAP - * mode. 0/OFF = Don't track dirty pages at all, don't spill ones, and use - * msync() to persist data. This is by-default on Linux and other systems where - * kernel provides properly LRU tracking and effective flushing on-demand. 1/ON - * = Tracking of dirty pages but with LRU labels for spilling and explicit - * persist ones by write(). This may be reasonable for systems which low - * performance of msync() and/or LRU tracking. */ +/** Controls dirty pages tracking, spilling and persisting in `MDBX_WRITEMAP` + * mode, i.e. disables in-memory database updating with consequent + * flush-to-disk/msync syscall. + * + * 0/OFF = Don't track dirty pages at all, don't spill ones, and use msync() to + * persist data. This is by-default on Linux and other systems where kernel + * provides properly LRU tracking and effective flushing on-demand. + * + * 1/ON = Tracking of dirty pages but with LRU labels for spilling and explicit + * persist ones by write(). This may be reasonable for goofy systems (Windows) + * which low performance of msync() and/or zany LRU tracking. */ #ifndef MDBX_AVOID_MSYNC #if defined(_WIN32) || defined(_WIN64) #define MDBX_AVOID_MSYNC 1 @@ -2147,6 +1758,22 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #error MDBX_AVOID_MSYNC must be defined as 0 or 1 #endif /* MDBX_AVOID_MSYNC */ +/** Управляет механизмом поддержки разреженных наборов DBI-хендлов для снижения + * накладных расходов при запуске и обработке транзакций. */ +#ifndef MDBX_ENABLE_DBI_SPARSE +#define MDBX_ENABLE_DBI_SPARSE 1 +#elif !(MDBX_ENABLE_DBI_SPARSE == 0 || MDBX_ENABLE_DBI_SPARSE == 1) +#error MDBX_ENABLE_DBI_SPARSE must be defined as 0 or 1 +#endif /* MDBX_ENABLE_DBI_SPARSE */ + +/** Управляет механизмом отложенного освобождения и поддержки пути быстрого + * открытия DBI-хендлов без захвата блокировок. */ +#ifndef MDBX_ENABLE_DBI_LOCKFREE +#define MDBX_ENABLE_DBI_LOCKFREE 1 +#elif !(MDBX_ENABLE_DBI_LOCKFREE == 0 || MDBX_ENABLE_DBI_LOCKFREE == 1) +#error MDBX_ENABLE_DBI_LOCKFREE must be defined as 0 or 1 +#endif /* MDBX_ENABLE_DBI_LOCKFREE */ + /** Controls sort order of internal page number lists. * This mostly experimental/advanced option with not for regular MDBX users. * \warning The database format depend on this option and libmdbx built with @@ -2159,7 +1786,11 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Avoid dependence from MSVC CRT and use ntdll.dll instead. */ #ifndef MDBX_WITHOUT_MSVC_CRT +#if defined(MDBX_BUILD_CXX) && !MDBX_BUILD_CXX #define MDBX_WITHOUT_MSVC_CRT 1 +#else +#define MDBX_WITHOUT_MSVC_CRT 0 +#endif #elif !(MDBX_WITHOUT_MSVC_CRT == 0 || MDBX_WITHOUT_MSVC_CRT == 1) #error MDBX_WITHOUT_MSVC_CRT must be defined as 0 or 1 #endif /* MDBX_WITHOUT_MSVC_CRT */ @@ -2167,12 +1798,11 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Size of buffer used during copying a environment/database file. */ #ifndef MDBX_ENVCOPY_WRITEBUF #define MDBX_ENVCOPY_WRITEBUF 1048576u -#elif MDBX_ENVCOPY_WRITEBUF < 65536u || MDBX_ENVCOPY_WRITEBUF > 1073741824u || \ - MDBX_ENVCOPY_WRITEBUF % 65536u +#elif MDBX_ENVCOPY_WRITEBUF < 65536u || MDBX_ENVCOPY_WRITEBUF > 1073741824u || MDBX_ENVCOPY_WRITEBUF % 65536u #error MDBX_ENVCOPY_WRITEBUF must be defined in range 65536..1073741824 and be multiple of 65536 #endif /* MDBX_ENVCOPY_WRITEBUF */ -/** Forces assertion checking */ +/** Forces assertion checking. */ #ifndef MDBX_FORCE_ASSERTIONS #define MDBX_FORCE_ASSERTIONS 0 #elif !(MDBX_FORCE_ASSERTIONS == 0 || MDBX_FORCE_ASSERTIONS == 1) @@ -2187,15 +1817,14 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #else #define MDBX_ASSUME_MALLOC_OVERHEAD (sizeof(void *) * 2u) #endif -#elif MDBX_ASSUME_MALLOC_OVERHEAD < 0 || MDBX_ASSUME_MALLOC_OVERHEAD > 64 || \ - MDBX_ASSUME_MALLOC_OVERHEAD % 4 +#elif MDBX_ASSUME_MALLOC_OVERHEAD < 0 || MDBX_ASSUME_MALLOC_OVERHEAD > 64 || MDBX_ASSUME_MALLOC_OVERHEAD % 4 #error MDBX_ASSUME_MALLOC_OVERHEAD must be defined in range 0..64 and be multiple of 4 #endif /* MDBX_ASSUME_MALLOC_OVERHEAD */ /** If defined then enables integration with Valgrind, * a memory analyzing tool. */ -#ifndef MDBX_USE_VALGRIND -#endif /* MDBX_USE_VALGRIND */ +#ifndef ENABLE_MEMCHECK +#endif /* ENABLE_MEMCHECK */ /** If defined then enables use C11 atomics, * otherwise detects ones availability automatically. */ @@ -2215,18 +1844,24 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #define MDBX_HAVE_BUILTIN_CPU_SUPPORTS 0 #elif defined(__e2k__) #define MDBX_HAVE_BUILTIN_CPU_SUPPORTS 0 -#elif __has_builtin(__builtin_cpu_supports) || \ - defined(__BUILTIN_CPU_SUPPORTS__) || \ +#elif __has_builtin(__builtin_cpu_supports) || defined(__BUILTIN_CPU_SUPPORTS__) || \ (defined(__ia32__) && __GNUC_PREREQ(4, 8) && __GLIBC_PREREQ(2, 23)) #define MDBX_HAVE_BUILTIN_CPU_SUPPORTS 1 #else #define MDBX_HAVE_BUILTIN_CPU_SUPPORTS 0 #endif -#elif !(MDBX_HAVE_BUILTIN_CPU_SUPPORTS == 0 || \ - MDBX_HAVE_BUILTIN_CPU_SUPPORTS == 1) +#elif !(MDBX_HAVE_BUILTIN_CPU_SUPPORTS == 0 || MDBX_HAVE_BUILTIN_CPU_SUPPORTS == 1) #error MDBX_HAVE_BUILTIN_CPU_SUPPORTS must be defined as 0 or 1 #endif /* MDBX_HAVE_BUILTIN_CPU_SUPPORTS */ +/** if enabled then instead of the returned error `MDBX_REMOTE`, only a warning is issued, when + * the database being opened in non-read-only mode is located in a file system exported via NFS. */ +#ifndef MDBX_ENABLE_NON_READONLY_EXPORT +#define MDBX_ENABLE_NON_READONLY_EXPORT 0 +#elif !(MDBX_ENABLE_NON_READONLY_EXPORT == 0 || MDBX_ENABLE_NON_READONLY_EXPORT == 1) +#error MDBX_ENABLE_NON_READONLY_EXPORT must be defined as 0 or 1 +#endif /* MDBX_ENABLE_NON_READONLY_EXPORT */ + //------------------------------------------------------------------------------ /** Win32 File Locking API for \ref MDBX_LOCKING */ @@ -2244,27 +1879,20 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** POSIX-2008 Robust Mutexes for \ref MDBX_LOCKING */ #define MDBX_LOCKING_POSIX2008 2008 -/** BeOS Benaphores, aka Futexes for \ref MDBX_LOCKING */ -#define MDBX_LOCKING_BENAPHORE 1995 - /** Advanced: Choices the locking implementation (autodetection by default). */ #if defined(_WIN32) || defined(_WIN64) #define MDBX_LOCKING MDBX_LOCKING_WIN32FILES #else #ifndef MDBX_LOCKING -#if defined(_POSIX_THREAD_PROCESS_SHARED) && \ - _POSIX_THREAD_PROCESS_SHARED >= 200112L && !defined(__FreeBSD__) +#if defined(_POSIX_THREAD_PROCESS_SHARED) && _POSIX_THREAD_PROCESS_SHARED >= 200112L && !defined(__FreeBSD__) /* Some platforms define the EOWNERDEAD error code even though they * don't support Robust Mutexes. If doubt compile with -MDBX_LOCKING=2001. */ -#if defined(EOWNERDEAD) && _POSIX_THREAD_PROCESS_SHARED >= 200809L && \ - ((defined(_POSIX_THREAD_ROBUST_PRIO_INHERIT) && \ - _POSIX_THREAD_ROBUST_PRIO_INHERIT > 0) || \ - (defined(_POSIX_THREAD_ROBUST_PRIO_PROTECT) && \ - _POSIX_THREAD_ROBUST_PRIO_PROTECT > 0) || \ - defined(PTHREAD_MUTEX_ROBUST) || defined(PTHREAD_MUTEX_ROBUST_NP)) && \ - (!defined(__GLIBC__) || \ - __GLIBC_PREREQ(2, 10) /* troubles with Robust mutexes before 2.10 */) +#if defined(EOWNERDEAD) && _POSIX_THREAD_PROCESS_SHARED >= 200809L && \ + ((defined(_POSIX_THREAD_ROBUST_PRIO_INHERIT) && _POSIX_THREAD_ROBUST_PRIO_INHERIT > 0) || \ + (defined(_POSIX_THREAD_ROBUST_PRIO_PROTECT) && _POSIX_THREAD_ROBUST_PRIO_PROTECT > 0) || \ + defined(PTHREAD_MUTEX_ROBUST) || defined(PTHREAD_MUTEX_ROBUST_NP)) && \ + (!defined(__GLIBC__) || __GLIBC_PREREQ(2, 10) /* troubles with Robust mutexes before 2.10 */) #define MDBX_LOCKING MDBX_LOCKING_POSIX2008 #else #define MDBX_LOCKING MDBX_LOCKING_POSIX2001 @@ -2282,12 +1910,9 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Advanced: Using POSIX OFD-locks (autodetection by default). */ #ifndef MDBX_USE_OFDLOCKS -#if ((defined(F_OFD_SETLK) && defined(F_OFD_SETLKW) && \ - defined(F_OFD_GETLK)) || \ - (defined(F_OFD_SETLK64) && defined(F_OFD_SETLKW64) && \ - defined(F_OFD_GETLK64))) && \ - !defined(MDBX_SAFE4QEMU) && \ - !defined(__sun) /* OFD-lock are broken on Solaris */ +#if ((defined(F_OFD_SETLK) && defined(F_OFD_SETLKW) && defined(F_OFD_GETLK)) || \ + (defined(F_OFD_SETLK64) && defined(F_OFD_SETLKW64) && defined(F_OFD_GETLK64))) && \ + !defined(MDBX_SAFE4QEMU) && !defined(__sun) /* OFD-lock are broken on Solaris */ #define MDBX_USE_OFDLOCKS 1 #else #define MDBX_USE_OFDLOCKS 0 @@ -2301,8 +1926,7 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Advanced: Using sendfile() syscall (autodetection by default). */ #ifndef MDBX_USE_SENDFILE -#if ((defined(__linux__) || defined(__gnu_linux__)) && \ - !defined(__ANDROID_API__)) || \ +#if ((defined(__linux__) || defined(__gnu_linux__)) && !defined(__ANDROID_API__)) || \ (defined(__ANDROID_API__) && __ANDROID_API__ >= 21) #define MDBX_USE_SENDFILE 1 #else @@ -2323,30 +1947,15 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #error MDBX_USE_COPYFILERANGE must be defined as 0 or 1 #endif /* MDBX_USE_COPYFILERANGE */ -/** Advanced: Using sync_file_range() syscall (autodetection by default). */ -#ifndef MDBX_USE_SYNCFILERANGE -#if ((defined(__linux__) || defined(__gnu_linux__)) && \ - defined(SYNC_FILE_RANGE_WRITE) && !defined(__ANDROID_API__)) || \ - (defined(__ANDROID_API__) && __ANDROID_API__ >= 26) -#define MDBX_USE_SYNCFILERANGE 1 -#else -#define MDBX_USE_SYNCFILERANGE 0 -#endif -#elif !(MDBX_USE_SYNCFILERANGE == 0 || MDBX_USE_SYNCFILERANGE == 1) -#error MDBX_USE_SYNCFILERANGE must be defined as 0 or 1 -#endif /* MDBX_USE_SYNCFILERANGE */ - //------------------------------------------------------------------------------ #ifndef MDBX_CPU_WRITEBACK_INCOHERENT -#if defined(__ia32__) || defined(__e2k__) || defined(__hppa) || \ - defined(__hppa__) || defined(DOXYGEN) +#if defined(__ia32__) || defined(__e2k__) || defined(__hppa) || defined(__hppa__) || defined(DOXYGEN) #define MDBX_CPU_WRITEBACK_INCOHERENT 0 #else #define MDBX_CPU_WRITEBACK_INCOHERENT 1 #endif -#elif !(MDBX_CPU_WRITEBACK_INCOHERENT == 0 || \ - MDBX_CPU_WRITEBACK_INCOHERENT == 1) +#elif !(MDBX_CPU_WRITEBACK_INCOHERENT == 0 || MDBX_CPU_WRITEBACK_INCOHERENT == 1) #error MDBX_CPU_WRITEBACK_INCOHERENT must be defined as 0 or 1 #endif /* MDBX_CPU_WRITEBACK_INCOHERENT */ @@ -2356,35 +1965,35 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #else #define MDBX_MMAP_INCOHERENT_FILE_WRITE 0 #endif -#elif !(MDBX_MMAP_INCOHERENT_FILE_WRITE == 0 || \ - MDBX_MMAP_INCOHERENT_FILE_WRITE == 1) +#elif !(MDBX_MMAP_INCOHERENT_FILE_WRITE == 0 || MDBX_MMAP_INCOHERENT_FILE_WRITE == 1) #error MDBX_MMAP_INCOHERENT_FILE_WRITE must be defined as 0 or 1 #endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */ #ifndef MDBX_MMAP_INCOHERENT_CPU_CACHE -#if defined(__mips) || defined(__mips__) || defined(__mips64) || \ - defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \ - defined(__MWERKS__) || defined(__sgi) +#if defined(__mips) || defined(__mips__) || defined(__mips64) || defined(__mips64__) || defined(_M_MRX000) || \ + defined(_MIPS_) || defined(__MWERKS__) || defined(__sgi) /* MIPS has cache coherency issues. */ #define MDBX_MMAP_INCOHERENT_CPU_CACHE 1 #else /* LY: assume no relevant mmap/dcache issues. */ #define MDBX_MMAP_INCOHERENT_CPU_CACHE 0 #endif -#elif !(MDBX_MMAP_INCOHERENT_CPU_CACHE == 0 || \ - MDBX_MMAP_INCOHERENT_CPU_CACHE == 1) +#elif !(MDBX_MMAP_INCOHERENT_CPU_CACHE == 0 || MDBX_MMAP_INCOHERENT_CPU_CACHE == 1) #error MDBX_MMAP_INCOHERENT_CPU_CACHE must be defined as 0 or 1 #endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */ -#ifndef MDBX_MMAP_USE_MS_ASYNC -#if MDBX_MMAP_INCOHERENT_FILE_WRITE || MDBX_MMAP_INCOHERENT_CPU_CACHE -#define MDBX_MMAP_USE_MS_ASYNC 1 +/** Assume system needs explicit syscall to sync/flush/write modified mapped + * memory. */ +#ifndef MDBX_MMAP_NEEDS_JOLT +#if MDBX_MMAP_INCOHERENT_FILE_WRITE || MDBX_MMAP_INCOHERENT_CPU_CACHE || !(defined(__linux__) || defined(__gnu_linux__)) +#define MDBX_MMAP_NEEDS_JOLT 1 #else -#define MDBX_MMAP_USE_MS_ASYNC 0 +#define MDBX_MMAP_NEEDS_JOLT 0 #endif -#elif !(MDBX_MMAP_USE_MS_ASYNC == 0 || MDBX_MMAP_USE_MS_ASYNC == 1) -#error MDBX_MMAP_USE_MS_ASYNC must be defined as 0 or 1 -#endif /* MDBX_MMAP_USE_MS_ASYNC */ +#define MDBX_MMAP_NEEDS_JOLT_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_MMAP_NEEDS_JOLT) +#elif !(MDBX_MMAP_NEEDS_JOLT == 0 || MDBX_MMAP_NEEDS_JOLT == 1) +#error MDBX_MMAP_NEEDS_JOLT must be defined as 0 or 1 +#endif /* MDBX_MMAP_NEEDS_JOLT */ #ifndef MDBX_64BIT_ATOMIC #if MDBX_WORDBITS >= 64 || defined(DOXYGEN) @@ -2431,8 +2040,7 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #endif /* MDBX_64BIT_CAS */ #ifndef MDBX_UNALIGNED_OK -#if defined(__ALIGNED__) || defined(__SANITIZE_UNDEFINED__) || \ - defined(ENABLE_UBSAN) +#if defined(__ALIGNED__) || defined(__SANITIZE_UNDEFINED__) || defined(ENABLE_UBSAN) #define MDBX_UNALIGNED_OK 0 /* no unaligned access allowed */ #elif defined(__ARM_FEATURE_UNALIGNED) #define MDBX_UNALIGNED_OK 4 /* ok unaligned for 32-bit words */ @@ -2466,6 +2074,19 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #endif #endif /* MDBX_CACHELINE_SIZE */ +/* Max length of iov-vector passed to writev() call, used for auxilary writes */ +#ifndef MDBX_AUXILARY_IOV_MAX +#define MDBX_AUXILARY_IOV_MAX 64 +#endif +#if defined(IOV_MAX) && IOV_MAX < MDBX_AUXILARY_IOV_MAX +#undef MDBX_AUXILARY_IOV_MAX +#define MDBX_AUXILARY_IOV_MAX IOV_MAX +#endif /* MDBX_AUXILARY_IOV_MAX */ + +/* An extra/custom information provided during library build */ +#ifndef MDBX_BUILD_METADATA +#define MDBX_BUILD_METADATA "" +#endif /* MDBX_BUILD_METADATA */ /** @} end of build options */ /******************************************************************************* ******************************************************************************* @@ -2480,6 +2101,9 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #else #define MDBX_DEBUG 1 #endif +#endif +#if MDBX_DEBUG < 0 || MDBX_DEBUG > 2 +#error "The MDBX_DEBUG must be defined to 0, 1 or 2" #endif /* MDBX_DEBUG */ #else @@ -2499,169 +2123,58 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; * Also enables \ref MDBX_DBG_AUDIT if `MDBX_DEBUG >= 2`. * * \ingroup build_option */ -#define MDBX_DEBUG 0...7 +#define MDBX_DEBUG 0...2 /** Disables using of GNU libc extensions. */ #define MDBX_DISABLE_GNU_SOURCE 0 or 1 #endif /* DOXYGEN */ -/* Undefine the NDEBUG if debugging is enforced by MDBX_DEBUG */ -#if MDBX_DEBUG -#undef NDEBUG -#endif - -#ifndef __cplusplus -/*----------------------------------------------------------------------------*/ -/* Debug and Logging stuff */ - -#define MDBX_RUNTIME_FLAGS_INIT \ - ((MDBX_DEBUG) > 0) * MDBX_DBG_ASSERT + ((MDBX_DEBUG) > 1) * MDBX_DBG_AUDIT - -extern uint8_t runtime_flags; -extern uint8_t loglevel; -extern MDBX_debug_func *debug_logger; - -MDBX_MAYBE_UNUSED static __inline void jitter4testing(bool tiny) { -#if MDBX_DEBUG - if (MDBX_DBG_JITTER & runtime_flags) - osal_jitter(tiny); -#else - (void)tiny; -#endif -} - -MDBX_INTERNAL_FUNC void MDBX_PRINTF_ARGS(4, 5) - debug_log(int level, const char *function, int line, const char *fmt, ...) - MDBX_PRINTF_ARGS(4, 5); -MDBX_INTERNAL_FUNC void debug_log_va(int level, const char *function, int line, - const char *fmt, va_list args); +#ifndef MDBX_64BIT_ATOMIC +#error "The MDBX_64BIT_ATOMIC must be defined before" +#endif /* MDBX_64BIT_ATOMIC */ -#if MDBX_DEBUG -#define LOG_ENABLED(msg) unlikely(msg <= loglevel) -#define AUDIT_ENABLED() unlikely((runtime_flags & MDBX_DBG_AUDIT)) -#else /* MDBX_DEBUG */ -#define LOG_ENABLED(msg) (msg < MDBX_LOG_VERBOSE && msg <= loglevel) -#define AUDIT_ENABLED() (0) -#endif /* MDBX_DEBUG */ +#ifndef MDBX_64BIT_CAS +#error "The MDBX_64BIT_CAS must be defined before" +#endif /* MDBX_64BIT_CAS */ -#if MDBX_FORCE_ASSERTIONS -#define ASSERT_ENABLED() (1) -#elif MDBX_DEBUG -#define ASSERT_ENABLED() likely((runtime_flags & MDBX_DBG_ASSERT)) +#if defined(__cplusplus) && !defined(__STDC_NO_ATOMICS__) && __has_include() +#include +#define MDBX_HAVE_C11ATOMICS +#elif !defined(__cplusplus) && (__STDC_VERSION__ >= 201112L || __has_extension(c_atomic)) && \ + !defined(__STDC_NO_ATOMICS__) && \ + (__GNUC_PREREQ(4, 9) || __CLANG_PREREQ(3, 8) || !(defined(__GNUC__) || defined(__clang__))) +#include +#define MDBX_HAVE_C11ATOMICS +#elif defined(__GNUC__) || defined(__clang__) +#elif defined(_MSC_VER) +#pragma warning(disable : 4163) /* 'xyz': not available as an intrinsic */ +#pragma warning(disable : 4133) /* 'function': incompatible types - from \ + 'size_t' to 'LONGLONG' */ +#pragma warning(disable : 4244) /* 'return': conversion from 'LONGLONG' to \ + 'std::size_t', possible loss of data */ +#pragma warning(disable : 4267) /* 'function': conversion from 'size_t' to \ + 'long', possible loss of data */ +#pragma intrinsic(_InterlockedExchangeAdd, _InterlockedCompareExchange) +#pragma intrinsic(_InterlockedExchangeAdd64, _InterlockedCompareExchange64) +#elif defined(__APPLE__) +#include #else -#define ASSERT_ENABLED() (0) -#endif /* assertions */ - -#define DEBUG_EXTRA(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_EXTRA)) \ - debug_log(MDBX_LOG_EXTRA, __func__, __LINE__, fmt, __VA_ARGS__); \ - } while (0) - -#define DEBUG_EXTRA_PRINT(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_EXTRA)) \ - debug_log(MDBX_LOG_EXTRA, NULL, 0, fmt, __VA_ARGS__); \ - } while (0) - -#define TRACE(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_TRACE)) \ - debug_log(MDBX_LOG_TRACE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ - } while (0) - -#define DEBUG(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_DEBUG)) \ - debug_log(MDBX_LOG_DEBUG, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ - } while (0) - -#define VERBOSE(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_VERBOSE)) \ - debug_log(MDBX_LOG_VERBOSE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ - } while (0) - -#define NOTICE(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_NOTICE)) \ - debug_log(MDBX_LOG_NOTICE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ - } while (0) - -#define WARNING(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_WARN)) \ - debug_log(MDBX_LOG_WARN, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ - } while (0) - -#undef ERROR /* wingdi.h \ - Yeah, morons from M$ put such definition to the public header. */ - -#define ERROR(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_ERROR)) \ - debug_log(MDBX_LOG_ERROR, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ - } while (0) - -#define FATAL(fmt, ...) \ - debug_log(MDBX_LOG_FATAL, __func__, __LINE__, fmt "\n", __VA_ARGS__); - -#if MDBX_DEBUG -#define ASSERT_FAIL(env, msg, func, line) mdbx_assert_fail(env, msg, func, line) -#else /* MDBX_DEBUG */ -MDBX_NORETURN __cold void assert_fail(const char *msg, const char *func, - unsigned line); -#define ASSERT_FAIL(env, msg, func, line) \ - do { \ - (void)(env); \ - assert_fail(msg, func, line); \ - } while (0) -#endif /* MDBX_DEBUG */ - -#define ENSURE_MSG(env, expr, msg) \ - do { \ - if (unlikely(!(expr))) \ - ASSERT_FAIL(env, msg, __func__, __LINE__); \ - } while (0) - -#define ENSURE(env, expr) ENSURE_MSG(env, expr, #expr) - -/* assert(3) variant in environment context */ -#define eASSERT(env, expr) \ - do { \ - if (ASSERT_ENABLED()) \ - ENSURE(env, expr); \ - } while (0) - -/* assert(3) variant in cursor context */ -#define cASSERT(mc, expr) eASSERT((mc)->mc_txn->mt_env, expr) - -/* assert(3) variant in transaction context */ -#define tASSERT(txn, expr) eASSERT((txn)->mt_env, expr) - -#ifndef xMDBX_TOOLS /* Avoid using internal eASSERT() */ -#undef assert -#define assert(expr) eASSERT(NULL, expr) +#error FIXME atomic-ops #endif -#endif /* __cplusplus */ - -/*----------------------------------------------------------------------------*/ -/* Atomics */ - -enum MDBX_memory_order { +typedef enum mdbx_memory_order { mo_Relaxed, mo_AcquireRelease /* , mo_SequentialConsistency */ -}; +} mdbx_memory_order_t; typedef union { volatile uint32_t weak; #ifdef MDBX_HAVE_C11ATOMICS volatile _Atomic uint32_t c11a; #endif /* MDBX_HAVE_C11ATOMICS */ -} MDBX_atomic_uint32_t; +} mdbx_atomic_uint32_t; typedef union { volatile uint64_t weak; @@ -2671,15 +2184,15 @@ typedef union { #if !defined(MDBX_HAVE_C11ATOMICS) || !MDBX_64BIT_CAS || !MDBX_64BIT_ATOMIC __anonymous_struct_extension__ struct { #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ - MDBX_atomic_uint32_t low, high; + mdbx_atomic_uint32_t low, high; #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - MDBX_atomic_uint32_t high, low; + mdbx_atomic_uint32_t high, low; #else #error "FIXME: Unsupported byte order" #endif /* __BYTE_ORDER__ */ }; #endif -} MDBX_atomic_uint64_t; +} mdbx_atomic_uint64_t; #ifdef MDBX_HAVE_C11ATOMICS @@ -2695,92 +2208,20 @@ typedef union { #define MDBX_c11a_rw(type, ptr) (&(ptr)->c11a) #endif /* Crutches for C11 atomic compiler's bugs */ -#define mo_c11_store(fence) \ - (((fence) == mo_Relaxed) ? memory_order_relaxed \ - : ((fence) == mo_AcquireRelease) ? memory_order_release \ +#define mo_c11_store(fence) \ + (((fence) == mo_Relaxed) ? memory_order_relaxed \ + : ((fence) == mo_AcquireRelease) ? memory_order_release \ : memory_order_seq_cst) -#define mo_c11_load(fence) \ - (((fence) == mo_Relaxed) ? memory_order_relaxed \ - : ((fence) == mo_AcquireRelease) ? memory_order_acquire \ +#define mo_c11_load(fence) \ + (((fence) == mo_Relaxed) ? memory_order_relaxed \ + : ((fence) == mo_AcquireRelease) ? memory_order_acquire \ : memory_order_seq_cst) #endif /* MDBX_HAVE_C11ATOMICS */ -#ifndef __cplusplus - -#ifdef MDBX_HAVE_C11ATOMICS -#define osal_memory_fence(order, write) \ - atomic_thread_fence((write) ? mo_c11_store(order) : mo_c11_load(order)) -#else /* MDBX_HAVE_C11ATOMICS */ -#define osal_memory_fence(order, write) \ - do { \ - osal_compiler_barrier(); \ - if (write && order > (MDBX_CPU_WRITEBACK_INCOHERENT ? mo_Relaxed \ - : mo_AcquireRelease)) \ - osal_memory_barrier(); \ - } while (0) -#endif /* MDBX_HAVE_C11ATOMICS */ - -#if defined(MDBX_HAVE_C11ATOMICS) && defined(__LCC__) -#define atomic_store32(p, value, order) \ - ({ \ - const uint32_t value_to_store = (value); \ - atomic_store_explicit(MDBX_c11a_rw(uint32_t, p), value_to_store, \ - mo_c11_store(order)); \ - value_to_store; \ - }) -#define atomic_load32(p, order) \ - atomic_load_explicit(MDBX_c11a_ro(uint32_t, p), mo_c11_load(order)) -#define atomic_store64(p, value, order) \ - ({ \ - const uint64_t value_to_store = (value); \ - atomic_store_explicit(MDBX_c11a_rw(uint64_t, p), value_to_store, \ - mo_c11_store(order)); \ - value_to_store; \ - }) -#define atomic_load64(p, order) \ - atomic_load_explicit(MDBX_c11a_ro(uint64_t, p), mo_c11_load(order)) -#endif /* LCC && MDBX_HAVE_C11ATOMICS */ - -#ifndef atomic_store32 -MDBX_MAYBE_UNUSED static __always_inline uint32_t -atomic_store32(MDBX_atomic_uint32_t *p, const uint32_t value, - enum MDBX_memory_order order) { - STATIC_ASSERT(sizeof(MDBX_atomic_uint32_t) == 4); -#ifdef MDBX_HAVE_C11ATOMICS - assert(atomic_is_lock_free(MDBX_c11a_rw(uint32_t, p))); - atomic_store_explicit(MDBX_c11a_rw(uint32_t, p), value, mo_c11_store(order)); -#else /* MDBX_HAVE_C11ATOMICS */ - if (order != mo_Relaxed) - osal_compiler_barrier(); - p->weak = value; - osal_memory_fence(order, true); -#endif /* MDBX_HAVE_C11ATOMICS */ - return value; -} -#endif /* atomic_store32 */ - -#ifndef atomic_load32 -MDBX_MAYBE_UNUSED static __always_inline uint32_t atomic_load32( - const volatile MDBX_atomic_uint32_t *p, enum MDBX_memory_order order) { - STATIC_ASSERT(sizeof(MDBX_atomic_uint32_t) == 4); -#ifdef MDBX_HAVE_C11ATOMICS - assert(atomic_is_lock_free(MDBX_c11a_ro(uint32_t, p))); - return atomic_load_explicit(MDBX_c11a_ro(uint32_t, p), mo_c11_load(order)); -#else /* MDBX_HAVE_C11ATOMICS */ - osal_memory_fence(order, false); - const uint32_t value = p->weak; - if (order != mo_Relaxed) - osal_compiler_barrier(); - return value; -#endif /* MDBX_HAVE_C11ATOMICS */ -} -#endif /* atomic_load32 */ - -#endif /* !__cplusplus */ +#define SAFE64_INVALID_THRESHOLD UINT64_C(0xffffFFFF00000000) -/*----------------------------------------------------------------------------*/ -/* Basic constants and types */ +#pragma pack(push, 4) /* A stamp that identifies a file as an MDBX file. * There's nothing special about this value other than that it is easily @@ -2789,8 +2230,10 @@ MDBX_MAYBE_UNUSED static __always_inline uint32_t atomic_load32( /* FROZEN: The version number for a database's datafile format. */ #define MDBX_DATA_VERSION 3 -/* The version number for a database's lockfile format. */ -#define MDBX_LOCK_VERSION 5 + +#define MDBX_DATA_MAGIC ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + MDBX_DATA_VERSION) +#define MDBX_DATA_MAGIC_LEGACY_COMPAT ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + 2) +#define MDBX_DATA_MAGIC_LEGACY_DEVEL ((MDBX_MAGIC << 8) + 255) /* handle for the DB used to track free pages. */ #define FREE_DBI 0 @@ -2807,203 +2250,285 @@ MDBX_MAYBE_UNUSED static __always_inline uint32_t atomic_load32( * MDBX uses 32 bit for page numbers. This limits database * size up to 2^44 bytes, in case of 4K pages. */ typedef uint32_t pgno_t; -typedef MDBX_atomic_uint32_t atomic_pgno_t; +typedef mdbx_atomic_uint32_t atomic_pgno_t; #define PRIaPGNO PRIu32 #define MAX_PAGENO UINT32_C(0x7FFFffff) #define MIN_PAGENO NUM_METAS -#define SAFE64_INVALID_THRESHOLD UINT64_C(0xffffFFFF00000000) +/* An invalid page number. + * Mainly used to denote an empty tree. */ +#define P_INVALID (~(pgno_t)0) /* A transaction ID. */ typedef uint64_t txnid_t; -typedef MDBX_atomic_uint64_t atomic_txnid_t; +typedef mdbx_atomic_uint64_t atomic_txnid_t; #define PRIaTXN PRIi64 #define MIN_TXNID UINT64_C(1) #define MAX_TXNID (SAFE64_INVALID_THRESHOLD - 1) #define INITIAL_TXNID (MIN_TXNID + NUM_METAS - 1) #define INVALID_TXNID UINT64_MAX -/* LY: for testing non-atomic 64-bit txnid on 32-bit arches. - * #define xMDBX_TXNID_STEP (UINT32_MAX / 3) */ -#ifndef xMDBX_TXNID_STEP -#if MDBX_64BIT_CAS -#define xMDBX_TXNID_STEP 1u -#else -#define xMDBX_TXNID_STEP 2u -#endif -#endif /* xMDBX_TXNID_STEP */ -/* Used for offsets within a single page. - * Since memory pages are typically 4 or 8KB in size, 12-13 bits, - * this is plenty. */ +/* Used for offsets within a single page. */ typedef uint16_t indx_t; -#define MEGABYTE ((size_t)1 << 20) - -/*----------------------------------------------------------------------------*/ -/* Core structures for database and shared memory (i.e. format definition) */ -#pragma pack(push, 4) - -/* Information about a single database in the environment. */ -typedef struct MDBX_db { - uint16_t md_flags; /* see mdbx_dbi_open */ - uint16_t md_depth; /* depth of this tree */ - uint32_t md_xsize; /* key-size for MDBX_DUPFIXED (LEAF2 pages) */ - pgno_t md_root; /* the root page of this tree */ - pgno_t md_branch_pages; /* number of internal pages */ - pgno_t md_leaf_pages; /* number of leaf pages */ - pgno_t md_overflow_pages; /* number of overflow pages */ - uint64_t md_seq; /* table sequence counter */ - uint64_t md_entries; /* number of data items */ - uint64_t md_mod_txnid; /* txnid of last committed modification */ -} MDBX_db; +typedef struct tree { + uint16_t flags; /* see mdbx_dbi_open */ + uint16_t height; /* height of this tree */ + uint32_t dupfix_size; /* key-size for MDBX_DUPFIXED (DUPFIX pages) */ + pgno_t root; /* the root page of this tree */ + pgno_t branch_pages; /* number of branch pages */ + pgno_t leaf_pages; /* number of leaf pages */ + pgno_t large_pages; /* number of large pages */ + uint64_t sequence; /* table sequence counter */ + uint64_t items; /* number of data items */ + uint64_t mod_txnid; /* txnid of last committed modification */ +} tree_t; /* database size-related parameters */ -typedef struct MDBX_geo { +typedef struct geo { uint16_t grow_pv; /* datafile growth step as a 16-bit packed (exponential quantized) value */ uint16_t shrink_pv; /* datafile shrink threshold as a 16-bit packed (exponential quantized) value */ pgno_t lower; /* minimal size of datafile in pages */ pgno_t upper; /* maximal size of datafile in pages */ - pgno_t now; /* current size of datafile in pages */ - pgno_t next; /* first unused page in the datafile, + union { + pgno_t now; /* current size of datafile in pages */ + pgno_t end_pgno; + }; + union { + pgno_t first_unallocated; /* first unused page in the datafile, but actually the file may be shorter. */ -} MDBX_geo; + pgno_t next_pgno; + }; +} geo_t; /* Meta page content. * A meta page is the start point for accessing a database snapshot. - * Pages 0-1 are meta pages. Transaction N writes meta page (N % 2). */ -typedef struct MDBX_meta { + * Pages 0-2 are meta pages. */ +typedef struct meta { /* Stamp identifying this as an MDBX file. * It must be set to MDBX_MAGIC with MDBX_DATA_VERSION. */ - uint32_t mm_magic_and_version[2]; + uint32_t magic_and_version[2]; - /* txnid that committed this page, the first of a two-phase-update pair */ + /* txnid that committed this meta, the first of a two-phase-update pair */ union { - MDBX_atomic_uint32_t mm_txnid_a[2]; + mdbx_atomic_uint32_t txnid_a[2]; uint64_t unsafe_txnid; }; - uint16_t mm_extra_flags; /* extra DB flags, zero (nothing) for now */ - uint8_t mm_validator_id; /* ID of checksum and page validation method, - * zero (nothing) for now */ - uint8_t mm_extra_pagehdr; /* extra bytes in the page header, - * zero (nothing) for now */ - - MDBX_geo mm_geo; /* database size-related parameters */ + uint16_t reserve16; /* extra flags, zero (nothing) for now */ + uint8_t validator_id; /* ID of checksum and page validation method, + * zero (nothing) for now */ + int8_t extra_pagehdr; /* extra bytes in the page header, + * zero (nothing) for now */ - MDBX_db mm_dbs[CORE_DBS]; /* first is free space, 2nd is main db */ - /* The size of pages used in this DB */ -#define mm_psize mm_dbs[FREE_DBI].md_xsize - MDBX_canary mm_canary; + geo_t geometry; /* database size-related parameters */ -#define MDBX_DATASIGN_NONE 0u -#define MDBX_DATASIGN_WEAK 1u -#define SIGN_IS_STEADY(sign) ((sign) > MDBX_DATASIGN_WEAK) -#define META_IS_STEADY(meta) \ - SIGN_IS_STEADY(unaligned_peek_u64_volatile(4, (meta)->mm_sign)) union { - uint32_t mm_sign[2]; - uint64_t unsafe_sign; + struct { + tree_t gc, main; + } trees; + __anonymous_struct_extension__ struct { + uint16_t gc_flags; + uint16_t gc_height; + uint32_t pagesize; + }; }; - /* txnid that committed this page, the second of a two-phase-update pair */ - MDBX_atomic_uint32_t mm_txnid_b[2]; + MDBX_canary canary; + +#define DATASIGN_NONE 0u +#define DATASIGN_WEAK 1u +#define SIGN_IS_STEADY(sign) ((sign) > DATASIGN_WEAK) + union { + uint32_t sign[2]; + uint64_t unsafe_sign; + }; + + /* txnid that committed this meta, the second of a two-phase-update pair */ + mdbx_atomic_uint32_t txnid_b[2]; /* Number of non-meta pages which were put in GC after COW. May be 0 in case * DB was previously handled by libmdbx without corresponding feature. - * This value in couple with mr_snapshot_pages_retired allows fast estimation - * of "how much reader is restraining GC recycling". */ - uint32_t mm_pages_retired[2]; + * This value in couple with reader.snapshot_pages_retired allows fast + * estimation of "how much reader is restraining GC recycling". */ + uint32_t pages_retired[2]; /* The analogue /proc/sys/kernel/random/boot_id or similar to determine * whether the system was rebooted after the last use of the database files. * If there was no reboot, but there is no need to rollback to the last * steady sync point. Zeros mean that no relevant information is available * from the system. */ - bin128_t mm_bootid; + bin128_t bootid; -} MDBX_meta; + /* GUID базы данных, начиная с v0.13.1 */ + bin128_t dxbid; +} meta_t; #pragma pack(1) -/* Common header for all page types. The page type depends on mp_flags. +typedef enum page_type { + P_BRANCH = 0x01u /* branch page */, + P_LEAF = 0x02u /* leaf page */, + P_LARGE = 0x04u /* large/overflow page */, + P_META = 0x08u /* meta page */, + P_LEGACY_DIRTY = 0x10u /* legacy P_DIRTY flag prior to v0.10 958fd5b9 */, + P_BAD = P_LEGACY_DIRTY /* explicit flag for invalid/bad page */, + P_DUPFIX = 0x20u /* for MDBX_DUPFIXED records */, + P_SUBP = 0x40u /* for MDBX_DUPSORT sub-pages */, + P_SPILLED = 0x2000u /* spilled in parent txn */, + P_LOOSE = 0x4000u /* page was dirtied then freed, can be reused */, + P_FROZEN = 0x8000u /* used for retire page with known status */, + P_ILL_BITS = (uint16_t)~(P_BRANCH | P_LEAF | P_DUPFIX | P_LARGE | P_SPILLED), + + page_broken = 0, + page_large = P_LARGE, + page_branch = P_BRANCH, + page_leaf = P_LEAF, + page_dupfix_leaf = P_DUPFIX, + page_sub_leaf = P_SUBP | P_LEAF, + page_sub_dupfix_leaf = P_SUBP | P_DUPFIX, + page_sub_broken = P_SUBP, +} page_type_t; + +/* Common header for all page types. The page type depends on flags. * - * P_BRANCH and P_LEAF pages have unsorted 'MDBX_node's at the end, with - * sorted mp_ptrs[] entries referring to them. Exception: P_LEAF2 pages - * omit mp_ptrs and pack sorted MDBX_DUPFIXED values after the page header. + * P_BRANCH and P_LEAF pages have unsorted 'node_t's at the end, with + * sorted entries[] entries referring to them. Exception: P_DUPFIX pages + * omit entries and pack sorted MDBX_DUPFIXED values after the page header. * - * P_OVERFLOW records occupy one or more contiguous pages where only the - * first has a page header. They hold the real data of F_BIGDATA nodes. + * P_LARGE records occupy one or more contiguous pages where only the + * first has a page header. They hold the real data of N_BIG nodes. * * P_SUBP sub-pages are small leaf "pages" with duplicate data. - * A node with flag F_DUPDATA but not F_SUBDATA contains a sub-page. - * (Duplicate data can also go in sub-databases, which use normal pages.) + * A node with flag N_DUP but not N_TREE contains a sub-page. + * (Duplicate data can also go in tables, which use normal pages.) * - * P_META pages contain MDBX_meta, the start point of an MDBX snapshot. + * P_META pages contain meta_t, the start point of an MDBX snapshot. * - * Each non-metapage up to MDBX_meta.mm_last_pg is reachable exactly once + * Each non-metapage up to meta_t.mm_last_pg is reachable exactly once * in the snapshot: Either used by a database or listed in a GC record. */ -typedef struct MDBX_page { -#define IS_FROZEN(txn, p) ((p)->mp_txnid < (txn)->mt_txnid) -#define IS_SPILLED(txn, p) ((p)->mp_txnid == (txn)->mt_txnid) -#define IS_SHADOWED(txn, p) ((p)->mp_txnid > (txn)->mt_txnid) -#define IS_VALID(txn, p) ((p)->mp_txnid <= (txn)->mt_front) -#define IS_MODIFIABLE(txn, p) ((p)->mp_txnid == (txn)->mt_front) - uint64_t mp_txnid; /* txnid which created page, maybe zero in legacy DB */ - uint16_t mp_leaf2_ksize; /* key size if this is a LEAF2 page */ -#define P_BRANCH 0x01u /* branch page */ -#define P_LEAF 0x02u /* leaf page */ -#define P_OVERFLOW 0x04u /* overflow page */ -#define P_META 0x08u /* meta page */ -#define P_LEGACY_DIRTY 0x10u /* legacy P_DIRTY flag prior to v0.10 958fd5b9 */ -#define P_BAD P_LEGACY_DIRTY /* explicit flag for invalid/bad page */ -#define P_LEAF2 0x20u /* for MDBX_DUPFIXED records */ -#define P_SUBP 0x40u /* for MDBX_DUPSORT sub-pages */ -#define P_SPILLED 0x2000u /* spilled in parent txn */ -#define P_LOOSE 0x4000u /* page was dirtied then freed, can be reused */ -#define P_FROZEN 0x8000u /* used for retire page with known status */ -#define P_ILL_BITS \ - ((uint16_t)~(P_BRANCH | P_LEAF | P_LEAF2 | P_OVERFLOW | P_SPILLED)) - uint16_t mp_flags; +typedef struct page { + uint64_t txnid; /* txnid which created page, maybe zero in legacy DB */ + uint16_t dupfix_ksize; /* key size if this is a DUPFIX page */ + uint16_t flags; union { - uint32_t mp_pages; /* number of overflow pages */ + uint32_t pages; /* number of overflow pages */ __anonymous_struct_extension__ struct { - indx_t mp_lower; /* lower bound of free space */ - indx_t mp_upper; /* upper bound of free space */ + indx_t lower; /* lower bound of free space */ + indx_t upper; /* upper bound of free space */ }; }; - pgno_t mp_pgno; /* page number */ - -#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ - (!defined(__cplusplus) && defined(_MSC_VER)) - indx_t mp_ptrs[] /* dynamic size */; -#endif /* C99 */ -} MDBX_page; + pgno_t pgno; /* page number */ -#define PAGETYPE_WHOLE(p) ((uint8_t)(p)->mp_flags) - -/* Drop legacy P_DIRTY flag for sub-pages for compatilibity */ -#define PAGETYPE_COMPAT(p) \ - (unlikely(PAGETYPE_WHOLE(p) & P_SUBP) \ - ? PAGETYPE_WHOLE(p) & ~(P_SUBP | P_LEGACY_DIRTY) \ - : PAGETYPE_WHOLE(p)) +#if FLEXIBLE_ARRAY_MEMBERS + indx_t entries[] /* dynamic size */; +#endif /* FLEXIBLE_ARRAY_MEMBERS */ +} page_t; /* Size of the page header, excluding dynamic data at the end */ -#define PAGEHDRSZ offsetof(MDBX_page, mp_ptrs) +#define PAGEHDRSZ 20u -/* Pointer displacement without casting to char* to avoid pointer-aliasing */ -#define ptr_disp(ptr, disp) ((void *)(((intptr_t)(ptr)) + ((intptr_t)(disp)))) +/* Header for a single key/data pair within a page. + * Used in pages of type P_BRANCH and P_LEAF without P_DUPFIX. + * We guarantee 2-byte alignment for 'node_t's. + * + * Leaf node flags describe node contents. N_BIG says the node's + * data part is the page number of an overflow page with actual data. + * N_DUP and N_TREE can be combined giving duplicate data in + * a sub-page/table, and named databases (just N_TREE). */ +typedef struct node { +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + union { + uint32_t dsize; + uint32_t child_pgno; + }; + uint8_t flags; /* see node_flags */ + uint8_t extra; + uint16_t ksize; /* key size */ +#else + uint16_t ksize; /* key size */ + uint8_t extra; + uint8_t flags; /* see node_flags */ + union { + uint32_t child_pgno; + uint32_t dsize; + }; +#endif /* __BYTE_ORDER__ */ -/* Pointer distance as signed number of bytes */ -#define ptr_dist(more, less) (((intptr_t)(more)) - ((intptr_t)(less))) +#if FLEXIBLE_ARRAY_MEMBERS + uint8_t payload[] /* key and data are appended here */; +#endif /* FLEXIBLE_ARRAY_MEMBERS */ +} node_t; + +/* Size of the node header, excluding dynamic data at the end */ +#define NODESIZE 8u -#define mp_next(mp) \ - (*(MDBX_page **)ptr_disp((mp)->mp_ptrs, sizeof(void *) - sizeof(uint32_t))) +typedef enum node_flags { + N_BIG = 0x01 /* data put on large page */, + N_TREE = 0x02 /* data is a b-tree */, + N_DUP = 0x04 /* data has duplicates */ +} node_flags_t; #pragma pack(pop) -typedef struct profgc_stat { +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint8_t page_type(const page_t *mp) { return mp->flags; } + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint8_t page_type_compat(const page_t *mp) { + /* Drop legacy P_DIRTY flag for sub-pages for compatilibity, + * for assertions only. */ + return unlikely(mp->flags & P_SUBP) ? mp->flags & ~(P_SUBP | P_LEGACY_DIRTY) : mp->flags; +} + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_leaf(const page_t *mp) { + return (mp->flags & P_LEAF) != 0; +} + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_dupfix_leaf(const page_t *mp) { + return (mp->flags & P_DUPFIX) != 0; +} + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_branch(const page_t *mp) { + return (mp->flags & P_BRANCH) != 0; +} + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_largepage(const page_t *mp) { + return (mp->flags & P_LARGE) != 0; +} + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_subpage(const page_t *mp) { + return (mp->flags & P_SUBP) != 0; +} + +/* The version number for a database's lockfile format. */ +#define MDBX_LOCK_VERSION 6 + +#if MDBX_LOCKING == MDBX_LOCKING_WIN32FILES + +#define MDBX_LCK_SIGN UINT32_C(0xF10C) +typedef void osal_ipclock_t; +#elif MDBX_LOCKING == MDBX_LOCKING_SYSV + +#define MDBX_LCK_SIGN UINT32_C(0xF18D) +typedef mdbx_pid_t osal_ipclock_t; + +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || MDBX_LOCKING == MDBX_LOCKING_POSIX2008 + +#define MDBX_LCK_SIGN UINT32_C(0x8017) +typedef pthread_mutex_t osal_ipclock_t; + +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988 + +#define MDBX_LCK_SIGN UINT32_C(0xFC29) +typedef sem_t osal_ipclock_t; + +#else +#error "FIXME" +#endif /* MDBX_LOCKING */ + +/* Статистика профилирования работы GC */ +typedef struct gc_prof_stat { /* Монотонное время по "настенным часам" * затраченное на чтение и поиск внутри GC */ uint64_t rtime_monotonic; @@ -3019,42 +2544,44 @@ typedef struct profgc_stat { uint32_t spe_counter; /* page faults (hard page faults) */ uint32_t majflt; -} profgc_stat_t; - -/* Statistics of page operations overall of all (running, completed and aborted) - * transactions */ -typedef struct pgop_stat { - MDBX_atomic_uint64_t newly; /* Quantity of a new pages added */ - MDBX_atomic_uint64_t cow; /* Quantity of pages copied for update */ - MDBX_atomic_uint64_t clone; /* Quantity of parent's dirty pages clones + /* Для разборок с pnl_merge() */ + struct { + uint64_t time; + uint64_t volume; + uint32_t calls; + } pnl_merge; +} gc_prof_stat_t; + +/* Statistics of pages operations for all transactions, + * including incomplete and aborted. */ +typedef struct pgops { + mdbx_atomic_uint64_t newly; /* Quantity of a new pages added */ + mdbx_atomic_uint64_t cow; /* Quantity of pages copied for update */ + mdbx_atomic_uint64_t clone; /* Quantity of parent's dirty pages clones for nested transactions */ - MDBX_atomic_uint64_t split; /* Page splits */ - MDBX_atomic_uint64_t merge; /* Page merges */ - MDBX_atomic_uint64_t spill; /* Quantity of spilled dirty pages */ - MDBX_atomic_uint64_t unspill; /* Quantity of unspilled/reloaded pages */ - MDBX_atomic_uint64_t - wops; /* Number of explicit write operations (not a pages) to a disk */ - MDBX_atomic_uint64_t - msync; /* Number of explicit msync/flush-to-disk operations */ - MDBX_atomic_uint64_t - fsync; /* Number of explicit fsync/flush-to-disk operations */ - - MDBX_atomic_uint64_t prefault; /* Number of prefault write operations */ - MDBX_atomic_uint64_t mincore; /* Number of mincore() calls */ - - MDBX_atomic_uint32_t - incoherence; /* number of https://libmdbx.dqdkfa.ru/dead-github/issues/269 - caught */ - MDBX_atomic_uint32_t reserved; + mdbx_atomic_uint64_t split; /* Page splits */ + mdbx_atomic_uint64_t merge; /* Page merges */ + mdbx_atomic_uint64_t spill; /* Quantity of spilled dirty pages */ + mdbx_atomic_uint64_t unspill; /* Quantity of unspilled/reloaded pages */ + mdbx_atomic_uint64_t wops; /* Number of explicit write operations (not a pages) to a disk */ + mdbx_atomic_uint64_t msync; /* Number of explicit msync/flush-to-disk operations */ + mdbx_atomic_uint64_t fsync; /* Number of explicit fsync/flush-to-disk operations */ + + mdbx_atomic_uint64_t prefault; /* Number of prefault write operations */ + mdbx_atomic_uint64_t mincore; /* Number of mincore() calls */ + + mdbx_atomic_uint32_t incoherence; /* number of https://libmdbx.dqdkfa.ru/dead-github/issues/269 + caught */ + mdbx_atomic_uint32_t reserved; /* Статистика для профилирования GC. - * Логически эти данные может быть стоит вынести в другую структуру, + * Логически эти данные, возможно, стоит вынести в другую структуру, * но разница будет сугубо косметическая. */ struct { /* Затраты на поддержку данных пользователя */ - profgc_stat_t work; + gc_prof_stat_t work; /* Затраты на поддержку и обновления самой GC */ - profgc_stat_t self; + gc_prof_stat_t self; /* Итераций обновления GC, * больше 1 если были повторы/перезапуски */ uint32_t wloops; @@ -3069,33 +2596,6 @@ typedef struct pgop_stat { } gc_prof; } pgop_stat_t; -#if MDBX_LOCKING == MDBX_LOCKING_WIN32FILES -#define MDBX_CLOCK_SIGN UINT32_C(0xF10C) -typedef void osal_ipclock_t; -#elif MDBX_LOCKING == MDBX_LOCKING_SYSV - -#define MDBX_CLOCK_SIGN UINT32_C(0xF18D) -typedef mdbx_pid_t osal_ipclock_t; -#ifndef EOWNERDEAD -#define EOWNERDEAD MDBX_RESULT_TRUE -#endif - -#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ - MDBX_LOCKING == MDBX_LOCKING_POSIX2008 -#define MDBX_CLOCK_SIGN UINT32_C(0x8017) -typedef pthread_mutex_t osal_ipclock_t; -#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988 -#define MDBX_CLOCK_SIGN UINT32_C(0xFC29) -typedef sem_t osal_ipclock_t; -#else -#error "FIXME" -#endif /* MDBX_LOCKING */ - -#if MDBX_LOCKING > MDBX_LOCKING_SYSV && !defined(__cplusplus) -MDBX_INTERNAL_FUNC int osal_ipclock_stub(osal_ipclock_t *ipc); -MDBX_INTERNAL_FUNC int osal_ipclock_destroy(osal_ipclock_t *ipc); -#endif /* MDBX_LOCKING */ - /* Reader Lock Table * * Readers don't acquire any locks for their data access. Instead, they @@ -3105,8 +2605,9 @@ MDBX_INTERNAL_FUNC int osal_ipclock_destroy(osal_ipclock_t *ipc); * read transactions started by the same thread need no further locking to * proceed. * - * If MDBX_NOTLS is set, the slot address is not saved in thread-specific data. - * No reader table is used if the database is on a read-only filesystem. + * If MDBX_NOSTICKYTHREADS is set, the slot address is not saved in + * thread-specific data. No reader table is used if the database is on a + * read-only filesystem. * * Since the database uses multi-version concurrency control, readers don't * actually need any locking. This table is used to keep track of which @@ -3135,14 +2636,14 @@ MDBX_INTERNAL_FUNC int osal_ipclock_destroy(osal_ipclock_t *ipc); * many old transactions together. */ /* The actual reader record, with cacheline padding. */ -typedef struct MDBX_reader { - /* Current Transaction ID when this transaction began, or (txnid_t)-1. +typedef struct reader_slot { + /* Current Transaction ID when this transaction began, or INVALID_TXNID. * Multiple readers that start at the same time will probably have the * same ID here. Again, it's not important to exclude them from * anything; all we need to know is which version of the DB they * started from so we can avoid overwriting any data used in that * particular version. */ - MDBX_atomic_uint64_t /* txnid_t */ mr_txnid; + atomic_txnid_t txnid; /* The information we store in a single slot of the reader table. * In addition to a transaction ID, we also record the process and @@ -3153,708 +2654,320 @@ typedef struct MDBX_reader { * We simply re-init the table when we know that we're the only process * opening the lock file. */ + /* Псевдо thread_id для пометки вытесненных читающих транзакций. */ +#define MDBX_TID_TXN_OUSTED (UINT64_MAX - 1) + + /* Псевдо thread_id для пометки припаркованных читающих транзакций. */ +#define MDBX_TID_TXN_PARKED UINT64_MAX + /* The thread ID of the thread owning this txn. */ - MDBX_atomic_uint64_t mr_tid; + mdbx_atomic_uint64_t tid; /* The process ID of the process owning this reader txn. */ - MDBX_atomic_uint32_t mr_pid; + mdbx_atomic_uint32_t pid; /* The number of pages used in the reader's MVCC snapshot, - * i.e. the value of meta->mm_geo.next and txn->mt_next_pgno */ - atomic_pgno_t mr_snapshot_pages_used; + * i.e. the value of meta->geometry.first_unallocated and + * txn->geo.first_unallocated */ + atomic_pgno_t snapshot_pages_used; /* Number of retired pages at the time this reader starts transaction. So, - * at any time the difference mm_pages_retired - mr_snapshot_pages_retired - * will give the number of pages which this reader restraining from reuse. */ - MDBX_atomic_uint64_t mr_snapshot_pages_retired; -} MDBX_reader; + * at any time the difference meta.pages_retired - + * reader.snapshot_pages_retired will give the number of pages which this + * reader restraining from reuse. */ + mdbx_atomic_uint64_t snapshot_pages_retired; +} reader_slot_t; /* The header for the reader table (a memory-mapped lock file). */ -typedef struct MDBX_lockinfo { +typedef struct shared_lck { /* Stamp identifying this as an MDBX file. * It must be set to MDBX_MAGIC with with MDBX_LOCK_VERSION. */ - uint64_t mti_magic_and_version; + uint64_t magic_and_version; /* Format of this lock file. Must be set to MDBX_LOCK_FORMAT. */ - uint32_t mti_os_and_format; + uint32_t os_and_format; /* Flags which environment was opened. */ - MDBX_atomic_uint32_t mti_envmode; + mdbx_atomic_uint32_t envmode; /* Threshold of un-synced-with-disk pages for auto-sync feature, * zero means no-threshold, i.e. auto-sync is disabled. */ - atomic_pgno_t mti_autosync_threshold; + atomic_pgno_t autosync_threshold; /* Low 32-bit of txnid with which meta-pages was synced, * i.e. for sync-polling in the MDBX_NOMETASYNC mode. */ #define MDBX_NOMETASYNC_LAZY_UNK (UINT32_MAX / 3) #define MDBX_NOMETASYNC_LAZY_FD (MDBX_NOMETASYNC_LAZY_UNK + UINT32_MAX / 8) -#define MDBX_NOMETASYNC_LAZY_WRITEMAP \ - (MDBX_NOMETASYNC_LAZY_UNK - UINT32_MAX / 8) - MDBX_atomic_uint32_t mti_meta_sync_txnid; +#define MDBX_NOMETASYNC_LAZY_WRITEMAP (MDBX_NOMETASYNC_LAZY_UNK - UINT32_MAX / 8) + mdbx_atomic_uint32_t meta_sync_txnid; /* Period for timed auto-sync feature, i.e. at the every steady checkpoint - * the mti_unsynced_timeout sets to the current_time + mti_autosync_period. + * the mti_unsynced_timeout sets to the current_time + autosync_period. * The time value is represented in a suitable system-dependent form, for * example clock_gettime(CLOCK_BOOTTIME) or clock_gettime(CLOCK_MONOTONIC). * Zero means timed auto-sync is disabled. */ - MDBX_atomic_uint64_t mti_autosync_period; + mdbx_atomic_uint64_t autosync_period; /* Marker to distinguish uniqueness of DB/CLK. */ - MDBX_atomic_uint64_t mti_bait_uniqueness; + mdbx_atomic_uint64_t bait_uniqueness; /* Paired counter of processes that have mlock()ed part of mmapped DB. - * The (mti_mlcnt[0] - mti_mlcnt[1]) > 0 means at least one process + * The (mlcnt[0] - mlcnt[1]) > 0 means at least one process * lock at least one page, so therefore madvise() could return EINVAL. */ - MDBX_atomic_uint32_t mti_mlcnt[2]; + mdbx_atomic_uint32_t mlcnt[2]; MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/ /* Statistics of costly ops of all (running, completed and aborted) * transactions */ - pgop_stat_t mti_pgop_stat; + pgop_stat_t pgops; MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/ - /* Write transaction lock. */ #if MDBX_LOCKING > 0 - osal_ipclock_t mti_wlock; + /* Write transaction lock. */ + osal_ipclock_t wrt_lock; #endif /* MDBX_LOCKING > 0 */ - atomic_txnid_t mti_oldest_reader; + atomic_txnid_t cached_oldest; /* Timestamp of entering an out-of-sync state. Value is represented in a * suitable system-dependent form, for example clock_gettime(CLOCK_BOOTTIME) * or clock_gettime(CLOCK_MONOTONIC). */ - MDBX_atomic_uint64_t mti_eoos_timestamp; + mdbx_atomic_uint64_t eoos_timestamp; /* Number un-synced-with-disk pages for auto-sync feature. */ - MDBX_atomic_uint64_t mti_unsynced_pages; + mdbx_atomic_uint64_t unsynced_pages; /* Timestamp of the last readers check. */ - MDBX_atomic_uint64_t mti_reader_check_timestamp; + mdbx_atomic_uint64_t readers_check_timestamp; /* Number of page which was discarded last time by madvise(DONTNEED). */ - atomic_pgno_t mti_discarded_tail; + atomic_pgno_t discarded_tail; /* Shared anchor for tracking readahead edge and enabled/disabled status. */ - pgno_t mti_readahead_anchor; + pgno_t readahead_anchor; /* Shared cache for mincore() results */ struct { pgno_t begin[4]; uint64_t mask[4]; - } mti_mincore_cache; + } mincore_cache; MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/ - /* Readeaders registration lock. */ #if MDBX_LOCKING > 0 - osal_ipclock_t mti_rlock; + /* Readeaders table lock. */ + osal_ipclock_t rdt_lock; #endif /* MDBX_LOCKING > 0 */ /* The number of slots that have been used in the reader table. * This always records the maximum count, it is not decremented * when readers release their slots. */ - MDBX_atomic_uint32_t mti_numreaders; - MDBX_atomic_uint32_t mti_readers_refresh_flag; + mdbx_atomic_uint32_t rdt_length; + mdbx_atomic_uint32_t rdt_refresh_flag; -#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ - (!defined(__cplusplus) && defined(_MSC_VER)) +#if FLEXIBLE_ARRAY_MEMBERS MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/ - MDBX_reader mti_readers[] /* dynamic size */; -#endif /* C99 */ -} MDBX_lockinfo; + reader_slot_t rdt[] /* dynamic size */; /* Lockfile format signature: version, features and field layout */ -#define MDBX_LOCK_FORMAT \ - (MDBX_CLOCK_SIGN * 27733 + (unsigned)sizeof(MDBX_reader) * 13 + \ - (unsigned)offsetof(MDBX_reader, mr_snapshot_pages_used) * 251 + \ - (unsigned)offsetof(MDBX_lockinfo, mti_oldest_reader) * 83 + \ - (unsigned)offsetof(MDBX_lockinfo, mti_numreaders) * 37 + \ - (unsigned)offsetof(MDBX_lockinfo, mti_readers) * 29) - -#define MDBX_DATA_MAGIC \ - ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + MDBX_DATA_VERSION) - -#define MDBX_DATA_MAGIC_LEGACY_COMPAT \ - ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + 2) - -#define MDBX_DATA_MAGIC_LEGACY_DEVEL ((MDBX_MAGIC << 8) + 255) +#define MDBX_LOCK_FORMAT \ + (MDBX_LCK_SIGN * 27733 + (unsigned)sizeof(reader_slot_t) * 13 + \ + (unsigned)offsetof(reader_slot_t, snapshot_pages_used) * 251 + (unsigned)offsetof(lck_t, cached_oldest) * 83 + \ + (unsigned)offsetof(lck_t, rdt_length) * 37 + (unsigned)offsetof(lck_t, rdt) * 29) +#endif /* FLEXIBLE_ARRAY_MEMBERS */ +} lck_t; #define MDBX_LOCK_MAGIC ((MDBX_MAGIC << 8) + MDBX_LOCK_VERSION) -/* The maximum size of a database page. - * - * It is 64K, but value-PAGEHDRSZ must fit in MDBX_page.mp_upper. - * - * MDBX will use database pages < OS pages if needed. - * That causes more I/O in write transactions: The OS must - * know (read) the whole page before writing a partial page. - * - * Note that we don't currently support Huge pages. On Linux, - * regular data files cannot use Huge pages, and in general - * Huge pages aren't actually pageable. We rely on the OS - * demand-pager to read our data and page it out when memory - * pressure from other processes is high. So until OSs have - * actual paging support for Huge pages, they're not viable. */ -#define MAX_PAGESIZE MDBX_MAX_PAGESIZE -#define MIN_PAGESIZE MDBX_MIN_PAGESIZE - -#define MIN_MAPSIZE (MIN_PAGESIZE * MIN_PAGENO) +#define MDBX_READERS_LIMIT 32767 + +#define MIN_MAPSIZE (MDBX_MIN_PAGESIZE * MIN_PAGENO) #if defined(_WIN32) || defined(_WIN64) #define MAX_MAPSIZE32 UINT32_C(0x38000000) #else #define MAX_MAPSIZE32 UINT32_C(0x7f000000) #endif -#define MAX_MAPSIZE64 ((MAX_PAGENO + 1) * (uint64_t)MAX_PAGESIZE) +#define MAX_MAPSIZE64 ((MAX_PAGENO + 1) * (uint64_t)MDBX_MAX_PAGESIZE) #if MDBX_WORDBITS >= 64 #define MAX_MAPSIZE MAX_MAPSIZE64 -#define MDBX_PGL_LIMIT ((size_t)MAX_PAGENO) +#define PAGELIST_LIMIT ((size_t)MAX_PAGENO) #else #define MAX_MAPSIZE MAX_MAPSIZE32 -#define MDBX_PGL_LIMIT (MAX_MAPSIZE32 / MIN_PAGESIZE) +#define PAGELIST_LIMIT (MAX_MAPSIZE32 / MDBX_MIN_PAGESIZE) #endif /* MDBX_WORDBITS */ -#define MDBX_READERS_LIMIT 32767 -#define MDBX_RADIXSORT_THRESHOLD 142 #define MDBX_GOLD_RATIO_DBL 1.6180339887498948482 +#define MEGABYTE ((size_t)1 << 20) /*----------------------------------------------------------------------------*/ -/* An PNL is an Page Number List, a sorted array of IDs. - * The first element of the array is a counter for how many actual page-numbers - * are in the list. By default PNLs are sorted in descending order, this allow - * cut off a page with lowest pgno (at the tail) just truncating the list. The - * sort order of PNLs is controlled by the MDBX_PNL_ASCENDING build option. */ -typedef pgno_t *MDBX_PNL; - -#if MDBX_PNL_ASCENDING -#define MDBX_PNL_ORDERED(first, last) ((first) < (last)) -#define MDBX_PNL_DISORDERED(first, last) ((first) >= (last)) -#else -#define MDBX_PNL_ORDERED(first, last) ((first) > (last)) -#define MDBX_PNL_DISORDERED(first, last) ((first) <= (last)) -#endif +union logger_union { + void *ptr; + MDBX_debug_func *fmt; + MDBX_debug_func_nofmt *nofmt; +}; -/* List of txnid, only for MDBX_txn.tw.lifo_reclaimed */ -typedef txnid_t *MDBX_TXL; +struct libmdbx_globals { + bin128_t bootid; + unsigned sys_pagesize, sys_allocation_granularity; + uint8_t sys_pagesize_ln2; + uint8_t runtime_flags; + uint8_t loglevel; +#if defined(_WIN32) || defined(_WIN64) + bool running_under_Wine; +#elif defined(__linux__) || defined(__gnu_linux__) + bool running_on_WSL1 /* Windows Subsystem 1 for Linux */; + uint32_t linux_kernel_version; +#endif /* Linux */ + union logger_union logger; + osal_fastmutex_t debug_lock; + size_t logger_buffer_size; + char *logger_buffer; +}; -/* An Dirty-Page list item is an pgno/pointer pair. */ -typedef struct MDBX_dp { - MDBX_page *ptr; - pgno_t pgno, npages; -} MDBX_dp; +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ -/* An DPL (dirty-page list) is a sorted array of MDBX_DPs. */ -typedef struct MDBX_dpl { - size_t sorted; - size_t length; - size_t pages_including_loose; /* number of pages, but not an entries. */ - size_t detent; /* allocated size excluding the MDBX_DPL_RESERVE_GAP */ -#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ - (!defined(__cplusplus) && defined(_MSC_VER)) - MDBX_dp items[] /* dynamic size with holes at zero and after the last */; -#endif -} MDBX_dpl; +extern struct libmdbx_globals globals; +#if defined(_WIN32) || defined(_WIN64) +extern struct libmdbx_imports imports; +#endif /* Windows */ -/* PNL sizes */ -#define MDBX_PNL_GRANULATE_LOG2 10 -#define MDBX_PNL_GRANULATE (1 << MDBX_PNL_GRANULATE_LOG2) -#define MDBX_PNL_INITIAL \ - (MDBX_PNL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t)) +#ifndef __Wpedantic_format_voidptr +MDBX_MAYBE_UNUSED static inline const void *__Wpedantic_format_voidptr(const void *ptr) { return ptr; } +#define __Wpedantic_format_voidptr(ARG) __Wpedantic_format_voidptr(ARG) +#endif /* __Wpedantic_format_voidptr */ -#define MDBX_TXL_GRANULATE 32 -#define MDBX_TXL_INITIAL \ - (MDBX_TXL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t)) -#define MDBX_TXL_MAX \ - ((1u << 26) - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t)) +MDBX_INTERNAL void MDBX_PRINTF_ARGS(4, 5) debug_log(int level, const char *function, int line, const char *fmt, ...) + MDBX_PRINTF_ARGS(4, 5); +MDBX_INTERNAL void debug_log_va(int level, const char *function, int line, const char *fmt, va_list args); -#define MDBX_PNL_ALLOCLEN(pl) ((pl)[-1]) -#define MDBX_PNL_GETSIZE(pl) ((size_t)((pl)[0])) -#define MDBX_PNL_SETSIZE(pl, size) \ - do { \ - const size_t __size = size; \ - assert(__size < INT_MAX); \ - (pl)[0] = (pgno_t)__size; \ - } while (0) -#define MDBX_PNL_FIRST(pl) ((pl)[1]) -#define MDBX_PNL_LAST(pl) ((pl)[MDBX_PNL_GETSIZE(pl)]) -#define MDBX_PNL_BEGIN(pl) (&(pl)[1]) -#define MDBX_PNL_END(pl) (&(pl)[MDBX_PNL_GETSIZE(pl) + 1]) +#if MDBX_DEBUG +#define LOG_ENABLED(LVL) unlikely(LVL <= globals.loglevel) +#define AUDIT_ENABLED() unlikely((globals.runtime_flags & (unsigned)MDBX_DBG_AUDIT)) +#else /* MDBX_DEBUG */ +#define LOG_ENABLED(LVL) (LVL < MDBX_LOG_VERBOSE && LVL <= globals.loglevel) +#define AUDIT_ENABLED() (0) +#endif /* LOG_ENABLED() & AUDIT_ENABLED() */ -#if MDBX_PNL_ASCENDING -#define MDBX_PNL_EDGE(pl) ((pl) + 1) -#define MDBX_PNL_LEAST(pl) MDBX_PNL_FIRST(pl) -#define MDBX_PNL_MOST(pl) MDBX_PNL_LAST(pl) +#if MDBX_FORCE_ASSERTIONS +#define ASSERT_ENABLED() (1) +#elif MDBX_DEBUG +#define ASSERT_ENABLED() likely((globals.runtime_flags & (unsigned)MDBX_DBG_ASSERT)) #else -#define MDBX_PNL_EDGE(pl) ((pl) + MDBX_PNL_GETSIZE(pl)) -#define MDBX_PNL_LEAST(pl) MDBX_PNL_LAST(pl) -#define MDBX_PNL_MOST(pl) MDBX_PNL_FIRST(pl) -#endif - -#define MDBX_PNL_SIZEOF(pl) ((MDBX_PNL_GETSIZE(pl) + 1) * sizeof(pgno_t)) -#define MDBX_PNL_IS_EMPTY(pl) (MDBX_PNL_GETSIZE(pl) == 0) - -/*----------------------------------------------------------------------------*/ -/* Internal structures */ - -/* Auxiliary DB info. - * The information here is mostly static/read-only. There is - * only a single copy of this record in the environment. */ -typedef struct MDBX_dbx { - MDBX_val md_name; /* name of the database */ - MDBX_cmp_func *md_cmp; /* function for comparing keys */ - MDBX_cmp_func *md_dcmp; /* function for comparing data items */ - size_t md_klen_min, md_klen_max; /* min/max key length for the database */ - size_t md_vlen_min, - md_vlen_max; /* min/max value/data length for the database */ -} MDBX_dbx; - -typedef struct troika { - uint8_t fsm, recent, prefer_steady, tail_and_flags; -#if MDBX_WORDBITS > 32 /* Workaround for false-positives from Valgrind */ - uint32_t unused_pad; -#endif -#define TROIKA_HAVE_STEADY(troika) ((troika)->fsm & 7) -#define TROIKA_STRICT_VALID(troika) ((troika)->tail_and_flags & 64) -#define TROIKA_VALID(troika) ((troika)->tail_and_flags & 128) -#define TROIKA_TAIL(troika) ((troika)->tail_and_flags & 3) - txnid_t txnid[NUM_METAS]; -} meta_troika_t; - -/* A database transaction. - * Every operation requires a transaction handle. */ -struct MDBX_txn { -#define MDBX_MT_SIGNATURE UINT32_C(0x93D53A31) - uint32_t mt_signature; - - /* Transaction Flags */ - /* mdbx_txn_begin() flags */ -#define MDBX_TXN_RO_BEGIN_FLAGS (MDBX_TXN_RDONLY | MDBX_TXN_RDONLY_PREPARE) -#define MDBX_TXN_RW_BEGIN_FLAGS \ - (MDBX_TXN_NOMETASYNC | MDBX_TXN_NOSYNC | MDBX_TXN_TRY) - /* Additional flag for sync_locked() */ -#define MDBX_SHRINK_ALLOWED UINT32_C(0x40000000) - -#define MDBX_TXN_DRAINED_GC 0x20 /* GC was depleted up to oldest reader */ - -#define TXN_FLAGS \ - (MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_DIRTY | MDBX_TXN_SPILLS | \ - MDBX_TXN_HAS_CHILD | MDBX_TXN_INVALID | MDBX_TXN_DRAINED_GC) - -#if (TXN_FLAGS & (MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_RO_BEGIN_FLAGS)) || \ - ((MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_RO_BEGIN_FLAGS | TXN_FLAGS) & \ - MDBX_SHRINK_ALLOWED) -#error "Oops, some txn flags overlapped or wrong" -#endif - uint32_t mt_flags; - - MDBX_txn *mt_parent; /* parent of a nested txn */ - /* Nested txn under this txn, set together with flag MDBX_TXN_HAS_CHILD */ - MDBX_txn *mt_child; - MDBX_geo mt_geo; - /* next unallocated page */ -#define mt_next_pgno mt_geo.next - /* corresponding to the current size of datafile */ -#define mt_end_pgno mt_geo.now - - /* The ID of this transaction. IDs are integers incrementing from - * INITIAL_TXNID. Only committed write transactions increment the ID. If a - * transaction aborts, the ID may be re-used by the next writer. */ - txnid_t mt_txnid; - txnid_t mt_front; - - MDBX_env *mt_env; /* the DB environment */ - /* Array of records for each DB known in the environment. */ - MDBX_dbx *mt_dbxs; - /* Array of MDBX_db records for each known DB */ - MDBX_db *mt_dbs; - /* Array of sequence numbers for each DB handle */ - MDBX_atomic_uint32_t *mt_dbiseqs; - - /* Transaction DBI Flags */ -#define DBI_DIRTY MDBX_DBI_DIRTY /* DB was written in this txn */ -#define DBI_STALE MDBX_DBI_STALE /* Named-DB record is older than txnID */ -#define DBI_FRESH MDBX_DBI_FRESH /* Named-DB handle opened in this txn */ -#define DBI_CREAT MDBX_DBI_CREAT /* Named-DB handle created in this txn */ -#define DBI_VALID 0x10 /* DB handle is valid, see also DB_VALID */ -#define DBI_USRVALID 0x20 /* As DB_VALID, but not set for FREE_DBI */ -#define DBI_AUDITED 0x40 /* Internal flag for accounting during audit */ - /* Array of flags for each DB */ - uint8_t *mt_dbistate; - /* Number of DB records in use, or 0 when the txn is finished. - * This number only ever increments until the txn finishes; we - * don't decrement it when individual DB handles are closed. */ - MDBX_dbi mt_numdbs; - size_t mt_owner; /* thread ID that owns this transaction */ - MDBX_canary mt_canary; - void *mt_userctx; /* User-settable context */ - MDBX_cursor **mt_cursors; +#define ASSERT_ENABLED() (0) +#endif /* ASSERT_ENABLED() */ - union { - struct { - /* For read txns: This thread/txn's reader table slot, or NULL. */ - MDBX_reader *reader; - } to; - struct { - meta_troika_t troika; - /* In write txns, array of cursors for each DB */ - MDBX_PNL relist; /* Reclaimed GC pages */ - txnid_t last_reclaimed; /* ID of last used record */ -#if MDBX_ENABLE_REFUND - pgno_t loose_refund_wl /* FIXME: describe */; -#endif /* MDBX_ENABLE_REFUND */ - /* a sequence to spilling dirty page with LRU policy */ - unsigned dirtylru; - /* dirtylist room: Dirty array size - dirty pages visible to this txn. - * Includes ancestor txns' dirty pages not hidden by other txns' - * dirty/spilled pages. Thus commit(nested txn) has room to merge - * dirtylist into mt_parent after freeing hidden mt_parent pages. */ - size_t dirtyroom; - /* For write txns: Modified pages. Sorted when not MDBX_WRITEMAP. */ - MDBX_dpl *dirtylist; - /* The list of reclaimed txns from GC */ - MDBX_TXL lifo_reclaimed; - /* The list of pages that became unused during this transaction. */ - MDBX_PNL retired_pages; - /* The list of loose pages that became unused and may be reused - * in this transaction, linked through `mp_next`. */ - MDBX_page *loose_pages; - /* Number of loose pages (tw.loose_pages) */ - size_t loose_count; - union { - struct { - size_t least_removed; - /* The sorted list of dirty pages we temporarily wrote to disk - * because the dirty list was full. page numbers in here are - * shifted left by 1, deleted slots have the LSB set. */ - MDBX_PNL list; - } spilled; - size_t writemap_dirty_npages; - size_t writemap_spilled_npages; - }; - } tw; - }; -}; +#define DEBUG_EXTRA(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_EXTRA)) \ + debug_log(MDBX_LOG_EXTRA, __func__, __LINE__, fmt, __VA_ARGS__); \ + } while (0) -#if MDBX_WORDBITS >= 64 -#define CURSOR_STACK 32 -#else -#define CURSOR_STACK 24 -#endif - -struct MDBX_xcursor; - -/* Cursors are used for all DB operations. - * A cursor holds a path of (page pointer, key index) from the DB - * root to a position in the DB, plus other state. MDBX_DUPSORT - * cursors include an xcursor to the current data item. Write txns - * track their cursors and keep them up to date when data moves. - * Exception: An xcursor's pointer to a P_SUBP page can be stale. - * (A node with F_DUPDATA but no F_SUBDATA contains a subpage). */ -struct MDBX_cursor { -#define MDBX_MC_LIVE UINT32_C(0xFE05D5B1) -#define MDBX_MC_READY4CLOSE UINT32_C(0x2817A047) -#define MDBX_MC_WAIT4EOT UINT32_C(0x90E297A7) - uint32_t mc_signature; - /* The database handle this cursor operates on */ - MDBX_dbi mc_dbi; - /* Next cursor on this DB in this txn */ - MDBX_cursor *mc_next; - /* Backup of the original cursor if this cursor is a shadow */ - MDBX_cursor *mc_backup; - /* Context used for databases with MDBX_DUPSORT, otherwise NULL */ - struct MDBX_xcursor *mc_xcursor; - /* The transaction that owns this cursor */ - MDBX_txn *mc_txn; - /* The database record for this cursor */ - MDBX_db *mc_db; - /* The database auxiliary record for this cursor */ - MDBX_dbx *mc_dbx; - /* The mt_dbistate for this database */ - uint8_t *mc_dbistate; - uint8_t mc_snum; /* number of pushed pages */ - uint8_t mc_top; /* index of top page, normally mc_snum-1 */ - - /* Cursor state flags. */ -#define C_INITIALIZED 0x01 /* cursor has been initialized and is valid */ -#define C_EOF 0x02 /* No more data */ -#define C_SUB 0x04 /* Cursor is a sub-cursor */ -#define C_DEL 0x08 /* last op was a cursor_del */ -#define C_UNTRACK 0x10 /* Un-track cursor when closing */ -#define C_GCU \ - 0x20 /* Происходит подготовка к обновлению GC, поэтому \ - * можно брать страницы из GC даже для FREE_DBI */ - uint8_t mc_flags; - - /* Cursor checking flags. */ -#define CC_BRANCH 0x01 /* same as P_BRANCH for CHECK_LEAF_TYPE() */ -#define CC_LEAF 0x02 /* same as P_LEAF for CHECK_LEAF_TYPE() */ -#define CC_OVERFLOW 0x04 /* same as P_OVERFLOW for CHECK_LEAF_TYPE() */ -#define CC_UPDATING 0x08 /* update/rebalance pending */ -#define CC_SKIPORD 0x10 /* don't check keys ordering */ -#define CC_LEAF2 0x20 /* same as P_LEAF2 for CHECK_LEAF_TYPE() */ -#define CC_RETIRING 0x40 /* refs to child pages may be invalid */ -#define CC_PAGECHECK 0x80 /* perform page checking, see MDBX_VALIDATION */ - uint8_t mc_checking; - - MDBX_page *mc_pg[CURSOR_STACK]; /* stack of pushed pages */ - indx_t mc_ki[CURSOR_STACK]; /* stack of page indices */ -}; +#define DEBUG_EXTRA_PRINT(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_EXTRA)) \ + debug_log(MDBX_LOG_EXTRA, nullptr, 0, fmt, __VA_ARGS__); \ + } while (0) -#define CHECK_LEAF_TYPE(mc, mp) \ - (((PAGETYPE_WHOLE(mp) ^ (mc)->mc_checking) & \ - (CC_BRANCH | CC_LEAF | CC_OVERFLOW | CC_LEAF2)) == 0) - -/* Context for sorted-dup records. - * We could have gone to a fully recursive design, with arbitrarily - * deep nesting of sub-databases. But for now we only handle these - * levels - main DB, optional sub-DB, sorted-duplicate DB. */ -typedef struct MDBX_xcursor { - /* A sub-cursor for traversing the Dup DB */ - MDBX_cursor mx_cursor; - /* The database record for this Dup DB */ - MDBX_db mx_db; - /* The auxiliary DB record for this Dup DB */ - MDBX_dbx mx_dbx; -} MDBX_xcursor; - -typedef struct MDBX_cursor_couple { - MDBX_cursor outer; - void *mc_userctx; /* User-settable context */ - MDBX_xcursor inner; -} MDBX_cursor_couple; - -/* The database environment. */ -struct MDBX_env { - /* ----------------------------------------------------- mostly static part */ -#define MDBX_ME_SIGNATURE UINT32_C(0x9A899641) - MDBX_atomic_uint32_t me_signature; - /* Failed to update the meta page. Probably an I/O error. */ -#define MDBX_FATAL_ERROR UINT32_C(0x80000000) - /* Some fields are initialized. */ -#define MDBX_ENV_ACTIVE UINT32_C(0x20000000) - /* me_txkey is set */ -#define MDBX_ENV_TXKEY UINT32_C(0x10000000) - /* Legacy MDBX_MAPASYNC (prior v0.9) */ -#define MDBX_DEPRECATED_MAPASYNC UINT32_C(0x100000) - /* Legacy MDBX_COALESCE (prior v0.12) */ -#define MDBX_DEPRECATED_COALESCE UINT32_C(0x2000000) -#define ENV_INTERNAL_FLAGS (MDBX_FATAL_ERROR | MDBX_ENV_ACTIVE | MDBX_ENV_TXKEY) - uint32_t me_flags; - osal_mmap_t me_dxb_mmap; /* The main data file */ -#define me_map me_dxb_mmap.base -#define me_lazy_fd me_dxb_mmap.fd - mdbx_filehandle_t me_dsync_fd, me_fd4meta; -#if defined(_WIN32) || defined(_WIN64) -#define me_overlapped_fd me_ioring.overlapped_fd - HANDLE me_data_lock_event; -#endif /* Windows */ - osal_mmap_t me_lck_mmap; /* The lock file */ -#define me_lfd me_lck_mmap.fd - struct MDBX_lockinfo *me_lck; - - unsigned me_psize; /* DB page size, initialized from me_os_psize */ - uint16_t me_leaf_nodemax; /* max size of a leaf-node */ - uint16_t me_branch_nodemax; /* max size of a branch-node */ - uint16_t me_subpage_limit; - uint16_t me_subpage_room_threshold; - uint16_t me_subpage_reserve_prereq; - uint16_t me_subpage_reserve_limit; - atomic_pgno_t me_mlocked_pgno; - uint8_t me_psize2log; /* log2 of DB page size */ - int8_t me_stuck_meta; /* recovery-only: target meta page or less that zero */ - uint16_t me_merge_threshold, - me_merge_threshold_gc; /* pages emptier than this are candidates for - merging */ - unsigned me_os_psize; /* OS page size, from osal_syspagesize() */ - unsigned me_maxreaders; /* size of the reader table */ - MDBX_dbi me_maxdbs; /* size of the DB table */ - uint32_t me_pid; /* process ID of this env */ - osal_thread_key_t me_txkey; /* thread-key for readers */ - pathchar_t *me_pathname; /* path to the DB files */ - void *me_pbuf; /* scratch area for DUPSORT put() */ - MDBX_txn *me_txn0; /* preallocated write transaction */ - - MDBX_dbx *me_dbxs; /* array of static DB info */ - uint16_t *me_dbflags; /* array of flags from MDBX_db.md_flags */ - MDBX_atomic_uint32_t *me_dbiseqs; /* array of dbi sequence numbers */ - unsigned - me_maxgc_ov1page; /* Number of pgno_t fit in a single overflow page */ - unsigned me_maxgc_per_branch; - uint32_t me_live_reader; /* have liveness lock in reader table */ - void *me_userctx; /* User-settable context */ - MDBX_hsr_func *me_hsr_callback; /* Callback for kicking laggard readers */ - size_t me_madv_threshold; +#define TRACE(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_TRACE)) \ + debug_log(MDBX_LOG_TRACE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ + } while (0) - struct { - unsigned dp_reserve_limit; - unsigned rp_augment_limit; - unsigned dp_limit; - unsigned dp_initial; - uint8_t dp_loose_limit; - uint8_t spill_max_denominator; - uint8_t spill_min_denominator; - uint8_t spill_parent4child_denominator; - unsigned merge_threshold_16dot16_percent; -#if !(defined(_WIN32) || defined(_WIN64)) - unsigned writethrough_threshold; -#endif /* Windows */ - bool prefault_write; - union { - unsigned all; - /* tracks options with non-auto values but tuned by user */ - struct { - unsigned dp_limit : 1; - unsigned rp_augment_limit : 1; - unsigned prefault_write : 1; - } non_auto; - } flags; - } me_options; - - /* struct me_dbgeo used for accepting db-geo params from user for the new - * database creation, i.e. when mdbx_env_set_geometry() was called before - * mdbx_env_open(). */ - struct { - size_t lower; /* minimal size of datafile */ - size_t upper; /* maximal size of datafile */ - size_t now; /* current size of datafile */ - size_t grow; /* step to grow datafile */ - size_t shrink; /* threshold to shrink datafile */ - } me_dbgeo; - -#if MDBX_LOCKING == MDBX_LOCKING_SYSV - union { - key_t key; - int semid; - } me_sysv_ipc; -#endif /* MDBX_LOCKING == MDBX_LOCKING_SYSV */ - bool me_incore; +#define DEBUG(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_DEBUG)) \ + debug_log(MDBX_LOG_DEBUG, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ + } while (0) - MDBX_env *me_lcklist_next; +#define VERBOSE(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_VERBOSE)) \ + debug_log(MDBX_LOG_VERBOSE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ + } while (0) - /* --------------------------------------------------- mostly volatile part */ +#define NOTICE(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_NOTICE)) \ + debug_log(MDBX_LOG_NOTICE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ + } while (0) - MDBX_txn *me_txn; /* current write transaction */ - osal_fastmutex_t me_dbi_lock; - MDBX_dbi me_numdbs; /* number of DBs opened */ - bool me_prefault_write; +#define WARNING(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_WARN)) \ + debug_log(MDBX_LOG_WARN, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ + } while (0) - MDBX_page *me_dp_reserve; /* list of malloc'ed blocks for re-use */ - unsigned me_dp_reserve_len; - /* PNL of pages that became unused in a write txn */ - MDBX_PNL me_retired_pages; - osal_ioring_t me_ioring; +#undef ERROR /* wingdi.h \ + Yeah, morons from M$ put such definition to the public header. */ -#if defined(_WIN32) || defined(_WIN64) - osal_srwlock_t me_remap_guard; - /* Workaround for LockFileEx and WriteFile multithread bug */ - CRITICAL_SECTION me_windowsbug_lock; - char *me_pathname_char; /* cache of multi-byte representation of pathname - to the DB files */ -#else - osal_fastmutex_t me_remap_guard; -#endif +#define ERROR(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_ERROR)) \ + debug_log(MDBX_LOG_ERROR, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ + } while (0) - /* -------------------------------------------------------------- debugging */ +#define FATAL(fmt, ...) debug_log(MDBX_LOG_FATAL, __func__, __LINE__, fmt "\n", __VA_ARGS__); #if MDBX_DEBUG - MDBX_assert_func *me_assert_func; /* Callback for assertion failures */ -#endif -#ifdef MDBX_USE_VALGRIND - int me_valgrind_handle; -#endif -#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__) - MDBX_atomic_uint32_t me_ignore_EDEADLK; - pgno_t me_poison_edge; -#endif /* MDBX_USE_VALGRIND || __SANITIZE_ADDRESS__ */ +#define ASSERT_FAIL(env, msg, func, line) mdbx_assert_fail(env, msg, func, line) +#else /* MDBX_DEBUG */ +MDBX_NORETURN __cold void assert_fail(const char *msg, const char *func, unsigned line); +#define ASSERT_FAIL(env, msg, func, line) \ + do { \ + (void)(env); \ + assert_fail(msg, func, line); \ + } while (0) +#endif /* MDBX_DEBUG */ -#ifndef xMDBX_DEBUG_SPILLING -#define xMDBX_DEBUG_SPILLING 0 -#endif -#if xMDBX_DEBUG_SPILLING == 2 - size_t debug_dirtied_est, debug_dirtied_act; -#endif /* xMDBX_DEBUG_SPILLING */ +#define ENSURE_MSG(env, expr, msg) \ + do { \ + if (unlikely(!(expr))) \ + ASSERT_FAIL(env, msg, __func__, __LINE__); \ + } while (0) - /* ------------------------------------------------- stub for lck-less mode */ - MDBX_atomic_uint64_t - x_lckless_stub[(sizeof(MDBX_lockinfo) + MDBX_CACHELINE_SIZE - 1) / - sizeof(MDBX_atomic_uint64_t)]; -}; +#define ENSURE(env, expr) ENSURE_MSG(env, expr, #expr) -#ifndef __cplusplus -/*----------------------------------------------------------------------------*/ -/* Cache coherence and mmap invalidation */ +/* assert(3) variant in environment context */ +#define eASSERT(env, expr) \ + do { \ + if (ASSERT_ENABLED()) \ + ENSURE(env, expr); \ + } while (0) -#if MDBX_CPU_WRITEBACK_INCOHERENT -#define osal_flush_incoherent_cpu_writeback() osal_memory_barrier() -#else -#define osal_flush_incoherent_cpu_writeback() osal_compiler_barrier() -#endif /* MDBX_CPU_WRITEBACK_INCOHERENT */ +/* assert(3) variant in cursor context */ +#define cASSERT(mc, expr) eASSERT((mc)->txn->env, expr) -MDBX_MAYBE_UNUSED static __inline void -osal_flush_incoherent_mmap(const void *addr, size_t nbytes, - const intptr_t pagesize) { -#if MDBX_MMAP_INCOHERENT_FILE_WRITE - char *const begin = (char *)(-pagesize & (intptr_t)addr); - char *const end = - (char *)(-pagesize & (intptr_t)((char *)addr + nbytes + pagesize - 1)); - int err = msync(begin, end - begin, MS_SYNC | MS_INVALIDATE) ? errno : 0; - eASSERT(nullptr, err == 0); - (void)err; -#else - (void)pagesize; -#endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */ +/* assert(3) variant in transaction context */ +#define tASSERT(txn, expr) eASSERT((txn)->env, expr) -#if MDBX_MMAP_INCOHERENT_CPU_CACHE -#ifdef DCACHE - /* MIPS has cache coherency issues. - * Note: for any nbytes >= on-chip cache size, entire is flushed. */ - cacheflush((void *)addr, nbytes, DCACHE); -#else -#error "Oops, cacheflush() not available" -#endif /* DCACHE */ -#endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */ +#ifndef xMDBX_TOOLS /* Avoid using internal eASSERT() */ +#undef assert +#define assert(expr) eASSERT(nullptr, expr) +#endif -#if !MDBX_MMAP_INCOHERENT_FILE_WRITE && !MDBX_MMAP_INCOHERENT_CPU_CACHE - (void)addr; - (void)nbytes; +MDBX_MAYBE_UNUSED static inline void jitter4testing(bool tiny) { +#if MDBX_DEBUG + if (globals.runtime_flags & (unsigned)MDBX_DBG_JITTER) + osal_jitter(tiny); +#else + (void)tiny; #endif } -/*----------------------------------------------------------------------------*/ -/* Internal prototypes */ - -MDBX_INTERNAL_FUNC int cleanup_dead_readers(MDBX_env *env, int rlocked, - int *dead); -MDBX_INTERNAL_FUNC int rthc_alloc(osal_thread_key_t *key, MDBX_reader *begin, - MDBX_reader *end); -MDBX_INTERNAL_FUNC void rthc_remove(const osal_thread_key_t key); - -MDBX_INTERNAL_FUNC void global_ctor(void); -MDBX_INTERNAL_FUNC void osal_ctor(void); -MDBX_INTERNAL_FUNC void global_dtor(void); -MDBX_INTERNAL_FUNC void osal_dtor(void); -MDBX_INTERNAL_FUNC void thread_dtor(void *ptr); - -#endif /* !__cplusplus */ - -#define MDBX_IS_ERROR(rc) \ - ((rc) != MDBX_RESULT_TRUE && (rc) != MDBX_RESULT_FALSE) - -/* Internal error codes, not exposed outside libmdbx */ -#define MDBX_NO_ROOT (MDBX_LAST_ADDED_ERRCODE + 10) - -/* Debugging output value of a cursor DBI: Negative in a sub-cursor. */ -#define DDBI(mc) \ - (((mc)->mc_flags & C_SUB) ? -(int)(mc)->mc_dbi : (int)(mc)->mc_dbi) +MDBX_MAYBE_UNUSED MDBX_INTERNAL void page_list(page_t *mp); +MDBX_INTERNAL const char *pagetype_caption(const uint8_t type, char buf4unknown[16]); /* Key size which fits in a DKBUF (debug key buffer). */ -#define DKBUF_MAX 511 -#define DKBUF char _kbuf[DKBUF_MAX * 4 + 2] -#define DKEY(x) mdbx_dump_val(x, _kbuf, DKBUF_MAX * 2 + 1) -#define DVAL(x) mdbx_dump_val(x, _kbuf + DKBUF_MAX * 2 + 1, DKBUF_MAX * 2 + 1) +#define DKBUF_MAX 127 +#define DKBUF char dbg_kbuf[DKBUF_MAX * 4 + 2] +#define DKEY(x) mdbx_dump_val(x, dbg_kbuf, DKBUF_MAX * 2 + 1) +#define DVAL(x) mdbx_dump_val(x, dbg_kbuf + DKBUF_MAX * 2 + 1, DKBUF_MAX * 2 + 1) #if MDBX_DEBUG #define DKBUF_DEBUG DKBUF @@ -3866,102 +2979,24 @@ MDBX_INTERNAL_FUNC void thread_dtor(void *ptr); #define DVAL_DEBUG(x) ("-") #endif -/* An invalid page number. - * Mainly used to denote an empty tree. */ -#define P_INVALID (~(pgno_t)0) +MDBX_INTERNAL void log_error(const int err, const char *func, unsigned line); + +MDBX_MAYBE_UNUSED static inline int log_if_error(const int err, const char *func, unsigned line) { + if (unlikely(err != MDBX_SUCCESS)) + log_error(err, func, line); + return err; +} + +#define LOG_IFERR(err) log_if_error((err), __func__, __LINE__) /* Test if the flags f are set in a flag word w. */ #define F_ISSET(w, f) (((w) & (f)) == (f)) /* Round n up to an even number. */ -#define EVEN(n) (((n) + 1UL) & -2L) /* sign-extending -2 to match n+1U */ - -/* Default size of memory map. - * This is certainly too small for any actual applications. Apps should - * always set the size explicitly using mdbx_env_set_geometry(). */ -#define DEFAULT_MAPSIZE MEGABYTE - -/* Number of slots in the reader table. - * This value was chosen somewhat arbitrarily. The 61 is a prime number, - * and such readers plus a couple mutexes fit into single 4KB page. - * Applications should set the table size using mdbx_env_set_maxreaders(). */ -#define DEFAULT_READERS 61 - -/* Test if a page is a leaf page */ -#define IS_LEAF(p) (((p)->mp_flags & P_LEAF) != 0) -/* Test if a page is a LEAF2 page */ -#define IS_LEAF2(p) unlikely(((p)->mp_flags & P_LEAF2) != 0) -/* Test if a page is a branch page */ -#define IS_BRANCH(p) (((p)->mp_flags & P_BRANCH) != 0) -/* Test if a page is an overflow page */ -#define IS_OVERFLOW(p) unlikely(((p)->mp_flags & P_OVERFLOW) != 0) -/* Test if a page is a sub page */ -#define IS_SUBP(p) (((p)->mp_flags & P_SUBP) != 0) - -/* Header for a single key/data pair within a page. - * Used in pages of type P_BRANCH and P_LEAF without P_LEAF2. - * We guarantee 2-byte alignment for 'MDBX_node's. - * - * Leaf node flags describe node contents. F_BIGDATA says the node's - * data part is the page number of an overflow page with actual data. - * F_DUPDATA and F_SUBDATA can be combined giving duplicate data in - * a sub-page/sub-database, and named databases (just F_SUBDATA). */ -typedef struct MDBX_node { -#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ - union { - uint32_t mn_dsize; - uint32_t mn_pgno32; - }; - uint8_t mn_flags; /* see mdbx_node flags */ - uint8_t mn_extra; - uint16_t mn_ksize; /* key size */ -#else - uint16_t mn_ksize; /* key size */ - uint8_t mn_extra; - uint8_t mn_flags; /* see mdbx_node flags */ - union { - uint32_t mn_pgno32; - uint32_t mn_dsize; - }; -#endif /* __BYTE_ORDER__ */ - - /* mdbx_node Flags */ -#define F_BIGDATA 0x01 /* data put on overflow page */ -#define F_SUBDATA 0x02 /* data is a sub-database */ -#define F_DUPDATA 0x04 /* data has duplicates */ - - /* valid flags for mdbx_node_add() */ -#define NODE_ADD_FLAGS (F_DUPDATA | F_SUBDATA | MDBX_RESERVE | MDBX_APPEND) - -#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ - (!defined(__cplusplus) && defined(_MSC_VER)) - uint8_t mn_data[] /* key and data are appended here */; -#endif /* C99 */ -} MDBX_node; - -#define DB_PERSISTENT_FLAGS \ - (MDBX_REVERSEKEY | MDBX_DUPSORT | MDBX_INTEGERKEY | MDBX_DUPFIXED | \ - MDBX_INTEGERDUP | MDBX_REVERSEDUP) - -/* mdbx_dbi_open() flags */ -#define DB_USABLE_FLAGS (DB_PERSISTENT_FLAGS | MDBX_CREATE | MDBX_DB_ACCEDE) - -#define DB_VALID 0x8000 /* DB handle is valid, for me_dbflags */ -#define DB_INTERNAL_FLAGS DB_VALID - -#if DB_INTERNAL_FLAGS & DB_USABLE_FLAGS -#error "Oops, some flags overlapped or wrong" -#endif -#if DB_PERSISTENT_FLAGS & ~DB_USABLE_FLAGS -#error "Oops, some flags overlapped or wrong" -#endif +#define EVEN_CEIL(n) (((n) + 1UL) & -2L) /* sign-extending -2 to match n+1U */ -/* Max length of iov-vector passed to writev() call, used for auxilary writes */ -#define MDBX_AUXILARY_IOV_MAX 64 -#if defined(IOV_MAX) && IOV_MAX < MDBX_AUXILARY_IOV_MAX -#undef MDBX_AUXILARY_IOV_MAX -#define MDBX_AUXILARY_IOV_MAX IOV_MAX -#endif /* MDBX_AUXILARY_IOV_MAX */ +/* Round n down to an even number. */ +#define EVEN_FLOOR(n) ((n) & ~(size_t)1) /* * / @@ -3972,106 +3007,226 @@ typedef struct MDBX_node { */ #define CMP2INT(a, b) (((a) != (b)) ? (((a) < (b)) ? -1 : 1) : 0) -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline pgno_t -int64pgno(int64_t i64) { - if (likely(i64 >= (int64_t)MIN_PAGENO && i64 <= (int64_t)MAX_PAGENO + 1)) - return (pgno_t)i64; - return (i64 < (int64_t)MIN_PAGENO) ? MIN_PAGENO : MAX_PAGENO; -} +/* Pointer displacement without casting to char* to avoid pointer-aliasing */ +#define ptr_disp(ptr, disp) ((void *)(((intptr_t)(ptr)) + ((intptr_t)(disp)))) -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline pgno_t -pgno_add(size_t base, size_t augend) { - assert(base <= MAX_PAGENO + 1 && augend < MAX_PAGENO); - return int64pgno((int64_t)base + (int64_t)augend); -} +/* Pointer distance as signed number of bytes */ +#define ptr_dist(more, less) (((intptr_t)(more)) - ((intptr_t)(less))) -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline pgno_t -pgno_sub(size_t base, size_t subtrahend) { - assert(base >= MIN_PAGENO && base <= MAX_PAGENO + 1 && - subtrahend < MAX_PAGENO); - return int64pgno((int64_t)base - (int64_t)subtrahend); -} +#define MDBX_ASAN_POISON_MEMORY_REGION(addr, size) \ + do { \ + TRACE("POISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), (size_t)(size), __LINE__); \ + ASAN_POISON_MEMORY_REGION(addr, size); \ + } while (0) + +#define MDBX_ASAN_UNPOISON_MEMORY_REGION(addr, size) \ + do { \ + TRACE("UNPOISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), (size_t)(size), __LINE__); \ + ASAN_UNPOISON_MEMORY_REGION(addr, size); \ + } while (0) -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __always_inline bool -is_powerof2(size_t x) { - return (x & (x - 1)) == 0; +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline size_t branchless_abs(intptr_t value) { + assert(value > INT_MIN); + const size_t expanded_sign = (size_t)(value >> (sizeof(value) * CHAR_BIT - 1)); + return ((size_t)value + expanded_sign) ^ expanded_sign; } -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __always_inline size_t -floor_powerof2(size_t value, size_t granularity) { +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline bool is_powerof2(size_t x) { return (x & (x - 1)) == 0; } + +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline size_t floor_powerof2(size_t value, size_t granularity) { assert(is_powerof2(granularity)); return value & ~(granularity - 1); } -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __always_inline size_t -ceil_powerof2(size_t value, size_t granularity) { +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline size_t ceil_powerof2(size_t value, size_t granularity) { return floor_powerof2(value + granularity - 1, granularity); } -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static unsigned -log2n_powerof2(size_t value_uintptr) { - assert(value_uintptr > 0 && value_uintptr < INT32_MAX && - is_powerof2(value_uintptr)); - assert((value_uintptr & -(intptr_t)value_uintptr) == value_uintptr); - const uint32_t value_uint32 = (uint32_t)value_uintptr; -#if __GNUC_PREREQ(4, 1) || __has_builtin(__builtin_ctz) - STATIC_ASSERT(sizeof(value_uint32) <= sizeof(unsigned)); - return __builtin_ctz(value_uint32); -#elif defined(_MSC_VER) - unsigned long index; - STATIC_ASSERT(sizeof(value_uint32) <= sizeof(long)); - _BitScanForward(&index, value_uint32); - return index; +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED MDBX_INTERNAL unsigned log2n_powerof2(size_t value_uintptr); + +MDBX_NOTHROW_CONST_FUNCTION MDBX_INTERNAL uint64_t rrxmrrxmsx_0(uint64_t v); + +struct monotime_cache { + uint64_t value; + int expire_countdown; +}; + +MDBX_MAYBE_UNUSED static inline uint64_t monotime_since_cached(uint64_t begin_timestamp, struct monotime_cache *cache) { + if (cache->expire_countdown) + cache->expire_countdown -= 1; + else { + cache->value = osal_monotime(); + cache->expire_countdown = 42 / 3; + } + return cache->value - begin_timestamp; +} + +/* An PNL is an Page Number List, a sorted array of IDs. + * + * The first element of the array is a counter for how many actual page-numbers + * are in the list. By default PNLs are sorted in descending order, this allow + * cut off a page with lowest pgno (at the tail) just truncating the list. The + * sort order of PNLs is controlled by the MDBX_PNL_ASCENDING build option. */ +typedef pgno_t *pnl_t; +typedef const pgno_t *const_pnl_t; + +#if MDBX_PNL_ASCENDING +#define MDBX_PNL_ORDERED(first, last) ((first) < (last)) +#define MDBX_PNL_DISORDERED(first, last) ((first) >= (last)) +#else +#define MDBX_PNL_ORDERED(first, last) ((first) > (last)) +#define MDBX_PNL_DISORDERED(first, last) ((first) <= (last)) +#endif + +#define MDBX_PNL_GRANULATE_LOG2 10 +#define MDBX_PNL_GRANULATE (1 << MDBX_PNL_GRANULATE_LOG2) +#define MDBX_PNL_INITIAL (MDBX_PNL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t)) + +#define MDBX_PNL_ALLOCLEN(pl) ((pl)[-1]) +#define MDBX_PNL_GETSIZE(pl) ((size_t)((pl)[0])) +#define MDBX_PNL_SETSIZE(pl, size) \ + do { \ + const size_t __size = size; \ + assert(__size < INT_MAX); \ + (pl)[0] = (pgno_t)__size; \ + } while (0) +#define MDBX_PNL_FIRST(pl) ((pl)[1]) +#define MDBX_PNL_LAST(pl) ((pl)[MDBX_PNL_GETSIZE(pl)]) +#define MDBX_PNL_BEGIN(pl) (&(pl)[1]) +#define MDBX_PNL_END(pl) (&(pl)[MDBX_PNL_GETSIZE(pl) + 1]) + +#if MDBX_PNL_ASCENDING +#define MDBX_PNL_EDGE(pl) ((pl) + 1) +#define MDBX_PNL_LEAST(pl) MDBX_PNL_FIRST(pl) +#define MDBX_PNL_MOST(pl) MDBX_PNL_LAST(pl) #else - static const uint8_t debruijn_ctz32[32] = { - 0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8, - 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9}; - return debruijn_ctz32[(uint32_t)(value_uint32 * 0x077CB531ul) >> 27]; +#define MDBX_PNL_EDGE(pl) ((pl) + MDBX_PNL_GETSIZE(pl)) +#define MDBX_PNL_LEAST(pl) MDBX_PNL_LAST(pl) +#define MDBX_PNL_MOST(pl) MDBX_PNL_FIRST(pl) #endif + +#define MDBX_PNL_SIZEOF(pl) ((MDBX_PNL_GETSIZE(pl) + 1) * sizeof(pgno_t)) +#define MDBX_PNL_IS_EMPTY(pl) (MDBX_PNL_GETSIZE(pl) == 0) + +MDBX_MAYBE_UNUSED static inline size_t pnl_size2bytes(size_t size) { + assert(size > 0 && size <= PAGELIST_LIMIT); +#if MDBX_PNL_PREALLOC_FOR_RADIXSORT + + size += size; +#endif /* MDBX_PNL_PREALLOC_FOR_RADIXSORT */ + STATIC_ASSERT(MDBX_ASSUME_MALLOC_OVERHEAD + + (PAGELIST_LIMIT * (MDBX_PNL_PREALLOC_FOR_RADIXSORT + 1) + MDBX_PNL_GRANULATE + 3) * sizeof(pgno_t) < + SIZE_MAX / 4 * 3); + size_t bytes = + ceil_powerof2(MDBX_ASSUME_MALLOC_OVERHEAD + sizeof(pgno_t) * (size + 3), MDBX_PNL_GRANULATE * sizeof(pgno_t)) - + MDBX_ASSUME_MALLOC_OVERHEAD; + return bytes; } -/* Only a subset of the mdbx_env flags can be changed - * at runtime. Changing other flags requires closing the - * environment and re-opening it with the new flags. */ -#define ENV_CHANGEABLE_FLAGS \ - (MDBX_SAFE_NOSYNC | MDBX_NOMETASYNC | MDBX_DEPRECATED_MAPASYNC | \ - MDBX_NOMEMINIT | MDBX_COALESCE | MDBX_PAGEPERTURB | MDBX_ACCEDE | \ - MDBX_VALIDATION) -#define ENV_CHANGELESS_FLAGS \ - (MDBX_NOSUBDIR | MDBX_RDONLY | MDBX_WRITEMAP | MDBX_NOTLS | MDBX_NORDAHEAD | \ - MDBX_LIFORECLAIM | MDBX_EXCLUSIVE) -#define ENV_USABLE_FLAGS (ENV_CHANGEABLE_FLAGS | ENV_CHANGELESS_FLAGS) - -#if !defined(__cplusplus) || CONSTEXPR_ENUM_FLAGS_OPERATIONS -MDBX_MAYBE_UNUSED static void static_checks(void) { - STATIC_ASSERT_MSG(INT16_MAX - CORE_DBS == MDBX_MAX_DBI, - "Oops, MDBX_MAX_DBI or CORE_DBS?"); - STATIC_ASSERT_MSG((unsigned)(MDBX_DB_ACCEDE | MDBX_CREATE) == - ((DB_USABLE_FLAGS | DB_INTERNAL_FLAGS) & - (ENV_USABLE_FLAGS | ENV_INTERNAL_FLAGS)), - "Oops, some flags overlapped or wrong"); - STATIC_ASSERT_MSG((ENV_INTERNAL_FLAGS & ENV_USABLE_FLAGS) == 0, - "Oops, some flags overlapped or wrong"); +MDBX_MAYBE_UNUSED static inline pgno_t pnl_bytes2size(const size_t bytes) { + size_t size = bytes / sizeof(pgno_t); + assert(size > 3 && size <= PAGELIST_LIMIT + /* alignment gap */ 65536); + size -= 3; +#if MDBX_PNL_PREALLOC_FOR_RADIXSORT + size >>= 1; +#endif /* MDBX_PNL_PREALLOC_FOR_RADIXSORT */ + return (pgno_t)size; +} + +MDBX_INTERNAL pnl_t pnl_alloc(size_t size); + +MDBX_INTERNAL void pnl_free(pnl_t pnl); + +MDBX_INTERNAL int pnl_reserve(pnl_t __restrict *__restrict ppnl, const size_t wanna); + +MDBX_MAYBE_UNUSED static inline int __must_check_result pnl_need(pnl_t __restrict *__restrict ppnl, size_t num) { + assert(MDBX_PNL_GETSIZE(*ppnl) <= PAGELIST_LIMIT && MDBX_PNL_ALLOCLEN(*ppnl) >= MDBX_PNL_GETSIZE(*ppnl)); + assert(num <= PAGELIST_LIMIT); + const size_t wanna = MDBX_PNL_GETSIZE(*ppnl) + num; + return likely(MDBX_PNL_ALLOCLEN(*ppnl) >= wanna) ? MDBX_SUCCESS : pnl_reserve(ppnl, wanna); } -#endif /* Disabled for MSVC 19.0 (VisualStudio 2015) */ + +MDBX_MAYBE_UNUSED static inline void pnl_append_prereserved(__restrict pnl_t pnl, pgno_t pgno) { + assert(MDBX_PNL_GETSIZE(pnl) < MDBX_PNL_ALLOCLEN(pnl)); + if (AUDIT_ENABLED()) { + for (size_t i = MDBX_PNL_GETSIZE(pnl); i > 0; --i) + assert(pgno != pnl[i]); + } + *pnl += 1; + MDBX_PNL_LAST(pnl) = pgno; +} + +MDBX_INTERNAL void pnl_shrink(pnl_t __restrict *__restrict ppnl); + +MDBX_INTERNAL int __must_check_result spill_append_span(__restrict pnl_t *ppnl, pgno_t pgno, size_t n); + +MDBX_INTERNAL int __must_check_result pnl_append_span(__restrict pnl_t *ppnl, pgno_t pgno, size_t n); + +MDBX_INTERNAL int __must_check_result pnl_insert_span(__restrict pnl_t *ppnl, pgno_t pgno, size_t n); + +MDBX_INTERNAL size_t pnl_search_nochk(const pnl_t pnl, pgno_t pgno); + +MDBX_INTERNAL void pnl_sort_nochk(pnl_t pnl); + +MDBX_INTERNAL bool pnl_check(const const_pnl_t pnl, const size_t limit); + +MDBX_MAYBE_UNUSED static inline bool pnl_check_allocated(const const_pnl_t pnl, const size_t limit) { + return pnl == nullptr || (MDBX_PNL_ALLOCLEN(pnl) >= MDBX_PNL_GETSIZE(pnl) && pnl_check(pnl, limit)); +} + +MDBX_MAYBE_UNUSED static inline void pnl_sort(pnl_t pnl, size_t limit4check) { + pnl_sort_nochk(pnl); + assert(pnl_check(pnl, limit4check)); + (void)limit4check; +} + +MDBX_MAYBE_UNUSED static inline size_t pnl_search(const pnl_t pnl, pgno_t pgno, size_t limit) { + assert(pnl_check_allocated(pnl, limit)); + if (MDBX_HAVE_CMOV) { + /* cmov-ускоренный бинарный поиск может читать (но не использовать) один + * элемент за концом данных, этот элемент в пределах выделенного участка + * памяти, но не инициализирован. */ + VALGRIND_MAKE_MEM_DEFINED(MDBX_PNL_END(pnl), sizeof(pgno_t)); + } + assert(pgno < limit); + (void)limit; + size_t n = pnl_search_nochk(pnl, pgno); + if (MDBX_HAVE_CMOV) { + VALGRIND_MAKE_MEM_UNDEFINED(MDBX_PNL_END(pnl), sizeof(pgno_t)); + } + return n; +} + +MDBX_INTERNAL size_t pnl_merge(pnl_t dst, const pnl_t src); #ifdef __cplusplus } +#endif /* __cplusplus */ + +#define mdbx_sourcery_anchor XCONCAT(mdbx_sourcery_, MDBX_BUILD_SOURCERY) +#if defined(xMDBX_TOOLS) +extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #endif -#define MDBX_ASAN_POISON_MEMORY_REGION(addr, size) \ - do { \ - TRACE("POISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), \ - (size_t)(size), __LINE__); \ - ASAN_POISON_MEMORY_REGION(addr, size); \ - } while (0) +#define MDBX_IS_ERROR(rc) ((rc) != MDBX_RESULT_TRUE && (rc) != MDBX_RESULT_FALSE) -#define MDBX_ASAN_UNPOISON_MEMORY_REGION(addr, size) \ - do { \ - TRACE("UNPOISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), \ - (size_t)(size), __LINE__); \ - ASAN_UNPOISON_MEMORY_REGION(addr, size); \ - } while (0) +/*----------------------------------------------------------------------------*/ + +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline pgno_t int64pgno(int64_t i64) { + if (likely(i64 >= (int64_t)MIN_PAGENO && i64 <= (int64_t)MAX_PAGENO + 1)) + return (pgno_t)i64; + return (i64 < (int64_t)MIN_PAGENO) ? MIN_PAGENO : MAX_PAGENO; +} + +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline pgno_t pgno_add(size_t base, size_t augend) { + assert(base <= MAX_PAGENO + 1 && augend < MAX_PAGENO); + return int64pgno((int64_t)base + (int64_t)augend); +} + +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline pgno_t pgno_sub(size_t base, size_t subtrahend) { + assert(base >= MIN_PAGENO && base <= MAX_PAGENO + 1 && subtrahend < MAX_PAGENO); + return int64pgno((int64_t)base - (int64_t)subtrahend); +} #include @@ -4089,12 +3244,12 @@ MDBX_MAYBE_UNUSED static void static_checks(void) { #ifdef _MSC_VER #pragma warning(push, 1) -#pragma warning(disable : 4548) /* expression before comma has no effect; \ +#pragma warning(disable : 4548) /* expression before comma has no effect; \ expected expression with side - effect */ -#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ +#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ * semantics are not enabled. Specify /EHsc */ -#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ - * mode specified; termination on exception is \ +#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ + * mode specified; termination on exception is \ * not guaranteed. Specify /EHsc */ #if !defined(_CRT_SECURE_NO_WARNINGS) #define _CRT_SECURE_NO_WARNINGS @@ -4147,8 +3302,7 @@ int getopt(int argc, char *const argv[], const char *opts) { if (argv[optind][sp + 1] != '\0') optarg = &argv[optind++][sp + 1]; else if (++optind >= argc) { - fprintf(stderr, "%s: %s -- %c\n", argv[0], "option requires an argument", - c); + fprintf(stderr, "%s: %s -- %c\n", argv[0], "option requires an argument", c); sp = 1; return '?'; } else @@ -4189,7 +3343,7 @@ static void usage(void) { " -V\t\tprint version and exit\n" " -q\t\tbe quiet\n" " -d\t\tdelete the specified database, don't just empty it\n" - " -s name\tdrop the specified named subDB\n" + " -s name\tdrop the specified named table\n" " \t\tby default empty the main DB\n", prog); exit(EXIT_FAILURE); @@ -4197,8 +3351,22 @@ static void usage(void) { static void error(const char *func, int rc) { if (!quiet) - fprintf(stderr, "%s: %s() error %d %s\n", prog, func, rc, - mdbx_strerror(rc)); + fprintf(stderr, "%s: %s() error %d %s\n", prog, func, rc, mdbx_strerror(rc)); +} + +static void logger(MDBX_log_level_t level, const char *function, int line, const char *fmt, va_list args) { + static const char *const prefixes[] = { + "!!!fatal: ", // 0 fatal + " ! ", // 1 error + " ~ ", // 2 warning + " ", // 3 notice + " //", // 4 verbose + }; + if (level < MDBX_LOG_DEBUG) { + if (function && line) + fprintf(stderr, "%s", prefixes[level]); + vfprintf(stderr, fmt, args); + } } int main(int argc, char *argv[]) { @@ -4229,12 +3397,9 @@ int main(int argc, char *argv[]) { " - build: %s for %s by %s\n" " - flags: %s\n" " - options: %s\n", - mdbx_version.major, mdbx_version.minor, mdbx_version.release, - mdbx_version.revision, mdbx_version.git.describe, - mdbx_version.git.datetime, mdbx_version.git.commit, - mdbx_version.git.tree, mdbx_sourcery_anchor, mdbx_build.datetime, - mdbx_build.target, mdbx_build.compiler, mdbx_build.flags, - mdbx_build.options); + mdbx_version.major, mdbx_version.minor, mdbx_version.patch, mdbx_version.tweak, mdbx_version.git.describe, + mdbx_version.git.datetime, mdbx_version.git.commit, mdbx_version.git.tree, mdbx_sourcery_anchor, + mdbx_build.datetime, mdbx_build.target, mdbx_build.compiler, mdbx_build.flags, mdbx_build.options); return EXIT_SUCCESS; case 'q': quiet = true; @@ -4270,10 +3435,10 @@ int main(int argc, char *argv[]) { envname = argv[optind]; if (!quiet) { - printf("mdbx_drop %s (%s, T-%s)\nRunning for %s/%s...\n", - mdbx_version.git.describe, mdbx_version.git.datetime, + printf("mdbx_drop %s (%s, T-%s)\nRunning for %s/%s...\n", mdbx_version.git.describe, mdbx_version.git.datetime, mdbx_version.git.tree, envname, subname ? subname : "@MAIN"); fflush(nullptr); + mdbx_setup_debug(MDBX_LOG_NOTICE, MDBX_DBG_DONTCHANGE, logger); } rc = mdbx_env_create(&env); @@ -4296,7 +3461,7 @@ int main(int argc, char *argv[]) { goto env_close; } - rc = mdbx_txn_begin(env, NULL, 0, &txn); + rc = mdbx_txn_begin(env, nullptr, 0, &txn); if (unlikely(rc != MDBX_SUCCESS)) { error("mdbx_txn_begin", rc); goto env_close; diff --git a/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx_dump.c b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx_dump.c index aa595c1589f..3193b1d34ce 100644 --- a/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx_dump.c +++ b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx_dump.c @@ -1,18 +1,12 @@ -/* mdbx_dump.c - memory-mapped database dump tool */ - -/* - * Copyright 2015-2024 Leonid Yuriev - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . */ - +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \note Please refer to the COPYRIGHT file for explanations license change, +/// credits and acknowledgments. +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 +/// +/// mdbx_dump.c - memory-mapped database dump tool +/// + +/* clang-format off */ #ifdef _MSC_VER #if _MSC_VER > 1800 #pragma warning(disable : 4464) /* relative include path contains '..' */ @@ -21,38 +15,26 @@ #endif /* _MSC_VER (warnings) */ #define xMDBX_TOOLS /* Avoid using internal eASSERT() */ -/* - * Copyright 2015-2024 Leonid Yuriev - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . */ +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 -#define MDBX_BUILD_SOURCERY e156c1a97c017ce89d6541cd9464ae5a9761d76b3fd2f1696521f5f3792904fc_v0_12_13_0_g1fff1f67 -#ifdef MDBX_CONFIG_H -#include MDBX_CONFIG_H -#endif +#define MDBX_BUILD_SOURCERY 6b5df6869d2bf5419e3a8189d9cc849cc9911b9c8a951b9750ed0a261ce43724_v0_13_7_0_g566b0f93 #define LIBMDBX_INTERNALS -#ifdef xMDBX_TOOLS #define MDBX_DEPRECATED -#endif /* xMDBX_TOOLS */ -#ifdef xMDBX_ALLOY -/* Amalgamated build */ -#define MDBX_INTERNAL_FUNC static -#define MDBX_INTERNAL_VAR static -#else -/* Non-amalgamated build */ -#define MDBX_INTERNAL_FUNC -#define MDBX_INTERNAL_VAR extern -#endif /* xMDBX_ALLOY */ +#ifdef MDBX_CONFIG_H +#include MDBX_CONFIG_H +#endif + +/* Undefine the NDEBUG if debugging is enforced by MDBX_DEBUG */ +#if (defined(MDBX_DEBUG) && MDBX_DEBUG > 0) || (defined(MDBX_FORCE_ASSERTIONS) && MDBX_FORCE_ASSERTIONS) +#undef NDEBUG +#ifndef MDBX_DEBUG +/* Чтобы избежать включения отладки только из-за включения assert-проверок */ +#define MDBX_DEBUG 0 +#endif +#endif /*----------------------------------------------------------------------------*/ @@ -70,14 +52,59 @@ #endif /* MDBX_DISABLE_GNU_SOURCE */ /* Should be defined before any includes */ -#if !defined(_FILE_OFFSET_BITS) && !defined(__ANDROID_API__) && \ - !defined(ANDROID) +#if !defined(_FILE_OFFSET_BITS) && !defined(__ANDROID_API__) && !defined(ANDROID) #define _FILE_OFFSET_BITS 64 -#endif +#endif /* _FILE_OFFSET_BITS */ -#ifdef __APPLE__ +#if defined(__APPLE__) && !defined(_DARWIN_C_SOURCE) #define _DARWIN_C_SOURCE -#endif +#endif /* _DARWIN_C_SOURCE */ + +#if (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) && !defined(__USE_MINGW_ANSI_STDIO) +#define __USE_MINGW_ANSI_STDIO 1 +#endif /* MinGW */ + +#if defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS) + +#ifndef _WIN32_WINNT +#define _WIN32_WINNT 0x0601 /* Windows 7 */ +#endif /* _WIN32_WINNT */ + +#if !defined(_CRT_SECURE_NO_WARNINGS) +#define _CRT_SECURE_NO_WARNINGS +#endif /* _CRT_SECURE_NO_WARNINGS */ +#if !defined(UNICODE) +#define UNICODE +#endif /* UNICODE */ + +#if !defined(_NO_CRT_STDIO_INLINE) && MDBX_BUILD_SHARED_LIBRARY && !defined(xMDBX_TOOLS) && MDBX_WITHOUT_MSVC_CRT +#define _NO_CRT_STDIO_INLINE +#endif /* _NO_CRT_STDIO_INLINE */ + +#elif !defined(_POSIX_C_SOURCE) +#define _POSIX_C_SOURCE 200809L +#endif /* Windows */ + +#ifdef __cplusplus + +#ifndef NOMINMAX +#define NOMINMAX +#endif /* NOMINMAX */ + +/* Workaround for modern libstdc++ with CLANG < 4.x */ +#if defined(__SIZEOF_INT128__) && !defined(__GLIBCXX_TYPE_INT_N_0) && defined(__clang__) && __clang_major__ < 4 +#define __GLIBCXX_BITSIZE_INT_N_0 128 +#define __GLIBCXX_TYPE_INT_N_0 __int128 +#endif /* Workaround for modern libstdc++ with CLANG < 4.x */ + +#ifdef _MSC_VER +/* Workaround for MSVC' header `extern "C"` vs `std::` redefinition bug */ +#if defined(__SANITIZE_ADDRESS__) && !defined(_DISABLE_VECTOR_ANNOTATION) +#define _DISABLE_VECTOR_ANNOTATION +#endif /* _DISABLE_VECTOR_ANNOTATION */ +#endif /* _MSC_VER */ + +#endif /* __cplusplus */ #ifdef _MSC_VER #if _MSC_FULL_VER < 190024234 @@ -99,12 +126,8 @@ * and how to and where you can obtain the latest "Visual Studio 2015" build * with all fixes. */ -#error \ - "At least \"Microsoft C/C++ Compiler\" version 19.00.24234 (Visual Studio 2015 Update 3) is required." +#error "At least \"Microsoft C/C++ Compiler\" version 19.00.24234 (Visual Studio 2015 Update 3) is required." #endif -#ifndef _CRT_SECURE_NO_WARNINGS -#define _CRT_SECURE_NO_WARNINGS -#endif /* _CRT_SECURE_NO_WARNINGS */ #if _MSC_VER > 1800 #pragma warning(disable : 4464) /* relative include path contains '..' */ #endif @@ -112,124 +135,78 @@ #pragma warning(disable : 5045) /* will insert Spectre mitigation... */ #endif #if _MSC_VER > 1914 -#pragma warning( \ - disable : 5105) /* winbase.h(9531): warning C5105: macro expansion \ - producing 'defined' has undefined behavior */ +#pragma warning(disable : 5105) /* winbase.h(9531): warning C5105: macro expansion \ + producing 'defined' has undefined behavior */ +#endif +#if _MSC_VER < 1920 +/* avoid "error C2219: syntax error: type qualifier must be after '*'" */ +#define __restrict #endif #if _MSC_VER > 1930 #pragma warning(disable : 6235) /* is always a constant */ -#pragma warning(disable : 6237) /* is never evaluated and might \ +#pragma warning(disable : 6237) /* is never evaluated and might \ have side effects */ +#pragma warning(disable : 5286) /* implicit conversion from enum type 'type 1' to enum type 'type 2' */ +#pragma warning(disable : 5287) /* operands are different enum types 'type 1' and 'type 2' */ #endif #pragma warning(disable : 4710) /* 'xyz': function not inlined */ -#pragma warning(disable : 4711) /* function 'xyz' selected for automatic \ +#pragma warning(disable : 4711) /* function 'xyz' selected for automatic \ inline expansion */ -#pragma warning(disable : 4201) /* nonstandard extension used: nameless \ +#pragma warning(disable : 4201) /* nonstandard extension used: nameless \ struct/union */ #pragma warning(disable : 4702) /* unreachable code */ #pragma warning(disable : 4706) /* assignment within conditional expression */ #pragma warning(disable : 4127) /* conditional expression is constant */ -#pragma warning(disable : 4324) /* 'xyz': structure was padded due to \ +#pragma warning(disable : 4324) /* 'xyz': structure was padded due to \ alignment specifier */ #pragma warning(disable : 4310) /* cast truncates constant value */ -#pragma warning(disable : 4820) /* bytes padding added after data member for \ +#pragma warning(disable : 4820) /* bytes padding added after data member for \ alignment */ -#pragma warning(disable : 4548) /* expression before comma has no effect; \ +#pragma warning(disable : 4548) /* expression before comma has no effect; \ expected expression with side - effect */ -#pragma warning(disable : 4366) /* the result of the unary '&' operator may be \ +#pragma warning(disable : 4366) /* the result of the unary '&' operator may be \ unaligned */ -#pragma warning(disable : 4200) /* nonstandard extension used: zero-sized \ +#pragma warning(disable : 4200) /* nonstandard extension used: zero-sized \ array in struct/union */ -#pragma warning(disable : 4204) /* nonstandard extension used: non-constant \ +#pragma warning(disable : 4204) /* nonstandard extension used: non-constant \ aggregate initializer */ -#pragma warning( \ - disable : 4505) /* unreferenced local function has been removed */ -#endif /* _MSC_VER (warnings) */ +#pragma warning(disable : 4505) /* unreferenced local function has been removed */ +#endif /* _MSC_VER (warnings) */ #if defined(__GNUC__) && __GNUC__ < 9 #pragma GCC diagnostic ignored "-Wattributes" #endif /* GCC < 9 */ -#if (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) && \ - !defined(__USE_MINGW_ANSI_STDIO) -#define __USE_MINGW_ANSI_STDIO 1 -#endif /* MinGW */ - -#if (defined(_WIN32) || defined(_WIN64)) && !defined(UNICODE) -#define UNICODE -#endif /* UNICODE */ - -#include "mdbx.h" -/* - * Copyright 2015-2024 Leonid Yuriev - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - */ - - /*----------------------------------------------------------------------------*/ /* Microsoft compiler generates a lot of warning for self includes... */ #ifdef _MSC_VER #pragma warning(push, 1) -#pragma warning(disable : 4548) /* expression before comma has no effect; \ +#pragma warning(disable : 4548) /* expression before comma has no effect; \ expected expression with side - effect */ -#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ +#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ * semantics are not enabled. Specify /EHsc */ -#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ - * mode specified; termination on exception is \ +#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ + * mode specified; termination on exception is \ * not guaranteed. Specify /EHsc */ #endif /* _MSC_VER (warnings) */ -#if defined(_WIN32) || defined(_WIN64) -#if !defined(_CRT_SECURE_NO_WARNINGS) -#define _CRT_SECURE_NO_WARNINGS -#endif /* _CRT_SECURE_NO_WARNINGS */ -#if !defined(_NO_CRT_STDIO_INLINE) && MDBX_BUILD_SHARED_LIBRARY && \ - !defined(xMDBX_TOOLS) && MDBX_WITHOUT_MSVC_CRT -#define _NO_CRT_STDIO_INLINE -#endif -#elif !defined(_POSIX_C_SOURCE) -#define _POSIX_C_SOURCE 200809L -#endif /* Windows */ - /*----------------------------------------------------------------------------*/ /* basic C99 includes */ + #include #include #include #include #include +#include #include #include #include #include #include -#if (-6 & 5) || CHAR_BIT != 8 || UINT_MAX < 0xffffffff || ULONG_MAX % 0xFFFF -#error \ - "Sanity checking failed: Two's complement, reasonably sized integer types" -#endif - -#ifndef SSIZE_MAX -#define SSIZE_MAX INTPTR_MAX -#endif - -#if UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul || defined(_WIN64) -#define MDBX_WORDBITS 64 -#else -#define MDBX_WORDBITS 32 -#endif /* MDBX_WORDBITS */ - /*----------------------------------------------------------------------------*/ /* feature testing */ @@ -241,6 +218,14 @@ #define __has_include(x) (0) #endif +#ifndef __has_attribute +#define __has_attribute(x) (0) +#endif + +#ifndef __has_cpp_attribute +#define __has_cpp_attribute(x) 0 +#endif + #ifndef __has_feature #define __has_feature(x) (0) #endif @@ -263,8 +248,7 @@ #ifndef __GNUC_PREREQ #if defined(__GNUC__) && defined(__GNUC_MINOR__) -#define __GNUC_PREREQ(maj, min) \ - ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min)) +#define __GNUC_PREREQ(maj, min) ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min)) #else #define __GNUC_PREREQ(maj, min) (0) #endif @@ -272,8 +256,7 @@ #ifndef __CLANG_PREREQ #ifdef __clang__ -#define __CLANG_PREREQ(maj, min) \ - ((__clang_major__ << 16) + __clang_minor__ >= ((maj) << 16) + (min)) +#define __CLANG_PREREQ(maj, min) ((__clang_major__ << 16) + __clang_minor__ >= ((maj) << 16) + (min)) #else #define __CLANG_PREREQ(maj, min) (0) #endif @@ -281,13 +264,51 @@ #ifndef __GLIBC_PREREQ #if defined(__GLIBC__) && defined(__GLIBC_MINOR__) -#define __GLIBC_PREREQ(maj, min) \ - ((__GLIBC__ << 16) + __GLIBC_MINOR__ >= ((maj) << 16) + (min)) +#define __GLIBC_PREREQ(maj, min) ((__GLIBC__ << 16) + __GLIBC_MINOR__ >= ((maj) << 16) + (min)) #else #define __GLIBC_PREREQ(maj, min) (0) #endif #endif /* __GLIBC_PREREQ */ +/*----------------------------------------------------------------------------*/ +/* pre-requirements */ + +#if (-6 & 5) || CHAR_BIT != 8 || UINT_MAX < 0xffffffff || ULONG_MAX % 0xFFFF +#error "Sanity checking failed: Two's complement, reasonably sized integer types" +#endif + +#ifndef SSIZE_MAX +#define SSIZE_MAX INTPTR_MAX +#endif + +#if defined(__GNUC__) && !__GNUC_PREREQ(4, 2) +/* Actually libmdbx was not tested with compilers older than GCC 4.2. + * But you could ignore this warning at your own risk. + * In such case please don't rise up an issues related ONLY to old compilers. + */ +#warning "libmdbx required GCC >= 4.2" +#endif + +#if defined(__clang__) && !__CLANG_PREREQ(3, 8) +/* Actually libmdbx was not tested with CLANG older than 3.8. + * But you could ignore this warning at your own risk. + * In such case please don't rise up an issues related ONLY to old compilers. + */ +#warning "libmdbx required CLANG >= 3.8" +#endif + +#if defined(__GLIBC__) && !__GLIBC_PREREQ(2, 12) +/* Actually libmdbx was not tested with something older than glibc 2.12. + * But you could ignore this warning at your own risk. + * In such case please don't rise up an issues related ONLY to old systems. + */ +#warning "libmdbx was only tested with GLIBC >= 2.12." +#endif + +#ifdef __SANITIZE_THREAD__ +#warning "libmdbx don't compatible with ThreadSanitizer, you will get a lot of false-positive issues." +#endif /* __SANITIZE_THREAD__ */ + /*----------------------------------------------------------------------------*/ /* C11' alignas() */ @@ -317,8 +338,7 @@ #endif #endif /* __extern_C */ -#if !defined(nullptr) && !defined(__cplusplus) || \ - (__cplusplus < 201103L && !defined(_MSC_VER)) +#if !defined(nullptr) && !defined(__cplusplus) || (__cplusplus < 201103L && !defined(_MSC_VER)) #define nullptr NULL #endif @@ -330,9 +350,8 @@ #endif #endif /* Apple OSX & iOS */ -#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \ - defined(__BSD__) || defined(__bsdi__) || defined(__DragonFly__) || \ - defined(__APPLE__) || defined(__MACH__) +#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__BSD__) || defined(__bsdi__) || \ + defined(__DragonFly__) || defined(__APPLE__) || defined(__MACH__) #include #include #include @@ -349,8 +368,7 @@ #endif #else #include -#if !(defined(__sun) || defined(__SVR4) || defined(__svr4__) || \ - defined(_WIN32) || defined(_WIN64)) +#if !(defined(__sun) || defined(__SVR4) || defined(__svr4__) || defined(_WIN32) || defined(_WIN64)) #include #endif /* !Solaris */ #endif /* !xBSD */ @@ -404,12 +422,14 @@ __extern_C key_t ftok(const char *, int); #ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN #endif /* WIN32_LEAN_AND_MEAN */ -#include -#include #include #include #include +/* После подгрузки windows.h, чтобы избежать проблем со сборкой MINGW и т.п. */ +#include +#include + #else /*----------------------------------------------------------------------*/ #include @@ -437,11 +457,6 @@ __extern_C key_t ftok(const char *, int); #if __ANDROID_API__ >= 21 #include #endif -#if defined(_FILE_OFFSET_BITS) && _FILE_OFFSET_BITS != MDBX_WORDBITS -#error "_FILE_OFFSET_BITS != MDBX_WORDBITS" (_FILE_OFFSET_BITS != MDBX_WORDBITS) -#elif defined(__FILE_OFFSET_BITS) && __FILE_OFFSET_BITS != MDBX_WORDBITS -#error "__FILE_OFFSET_BITS != MDBX_WORDBITS" (__FILE_OFFSET_BITS != MDBX_WORDBITS) -#endif #endif /* Android */ #if defined(HAVE_SYS_STAT_H) || __has_include() @@ -457,43 +472,38 @@ __extern_C key_t ftok(const char *, int); /*----------------------------------------------------------------------------*/ /* Byteorder */ -#if defined(i386) || defined(__386) || defined(__i386) || defined(__i386__) || \ - defined(i486) || defined(__i486) || defined(__i486__) || defined(i586) || \ - defined(__i586) || defined(__i586__) || defined(i686) || \ - defined(__i686) || defined(__i686__) || defined(_M_IX86) || \ - defined(_X86_) || defined(__THW_INTEL__) || defined(__I86__) || \ - defined(__INTEL__) || defined(__x86_64) || defined(__x86_64__) || \ - defined(__amd64__) || defined(__amd64) || defined(_M_X64) || \ - defined(_M_AMD64) || defined(__IA32__) || defined(__INTEL__) +#if defined(i386) || defined(__386) || defined(__i386) || defined(__i386__) || defined(i486) || defined(__i486) || \ + defined(__i486__) || defined(i586) || defined(__i586) || defined(__i586__) || defined(i686) || defined(__i686) || \ + defined(__i686__) || defined(_M_IX86) || defined(_X86_) || defined(__THW_INTEL__) || defined(__I86__) || \ + defined(__INTEL__) || defined(__x86_64) || defined(__x86_64__) || defined(__amd64__) || defined(__amd64) || \ + defined(_M_X64) || defined(_M_AMD64) || defined(__IA32__) || defined(__INTEL__) #ifndef __ia32__ /* LY: define neutral __ia32__ for x86 and x86-64 */ #define __ia32__ 1 #endif /* __ia32__ */ -#if !defined(__amd64__) && \ - (defined(__x86_64) || defined(__x86_64__) || defined(__amd64) || \ - defined(_M_X64) || defined(_M_AMD64)) +#if !defined(__amd64__) && \ + (defined(__x86_64) || defined(__x86_64__) || defined(__amd64) || defined(_M_X64) || defined(_M_AMD64)) /* LY: define trusty __amd64__ for all AMD64/x86-64 arch */ #define __amd64__ 1 #endif /* __amd64__ */ #endif /* all x86 */ -#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || \ - !defined(__ORDER_BIG_ENDIAN__) +#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || !defined(__ORDER_BIG_ENDIAN__) -#if defined(__GLIBC__) || defined(__GNU_LIBRARY__) || \ - defined(__ANDROID_API__) || defined(HAVE_ENDIAN_H) || __has_include() +#if defined(__GLIBC__) || defined(__GNU_LIBRARY__) || defined(__ANDROID_API__) || defined(HAVE_ENDIAN_H) || \ + __has_include() #include -#elif defined(__APPLE__) || defined(__MACH__) || defined(__OpenBSD__) || \ - defined(HAVE_MACHINE_ENDIAN_H) || __has_include() +#elif defined(__APPLE__) || defined(__MACH__) || defined(__OpenBSD__) || defined(HAVE_MACHINE_ENDIAN_H) || \ + __has_include() #include #elif defined(HAVE_SYS_ISA_DEFS_H) || __has_include() #include -#elif (defined(HAVE_SYS_TYPES_H) && defined(HAVE_SYS_ENDIAN_H)) || \ +#elif (defined(HAVE_SYS_TYPES_H) && defined(HAVE_SYS_ENDIAN_H)) || \ (__has_include() && __has_include()) #include #include -#elif defined(__bsdi__) || defined(__DragonFly__) || defined(__FreeBSD__) || \ - defined(__NetBSD__) || defined(HAVE_SYS_PARAM_H) || __has_include() +#elif defined(__bsdi__) || defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || \ + defined(HAVE_SYS_PARAM_H) || __has_include() #include #endif /* OS */ @@ -509,27 +519,19 @@ __extern_C key_t ftok(const char *, int); #define __ORDER_LITTLE_ENDIAN__ 1234 #define __ORDER_BIG_ENDIAN__ 4321 -#if defined(__LITTLE_ENDIAN__) || \ - (defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)) || \ - defined(__ARMEL__) || defined(__THUMBEL__) || defined(__AARCH64EL__) || \ - defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \ - defined(_M_ARM) || defined(_M_ARM64) || defined(__e2k__) || \ - defined(__elbrus_4c__) || defined(__elbrus_8c__) || defined(__bfin__) || \ - defined(__BFIN__) || defined(__ia64__) || defined(_IA64) || \ - defined(__IA64__) || defined(__ia64) || defined(_M_IA64) || \ - defined(__itanium__) || defined(__ia32__) || defined(__CYGWIN__) || \ - defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) || \ - defined(__WINDOWS__) +#if defined(__LITTLE_ENDIAN__) || (defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)) || defined(__ARMEL__) || \ + defined(__THUMBEL__) || defined(__AARCH64EL__) || defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \ + defined(_M_ARM) || defined(_M_ARM64) || defined(__e2k__) || defined(__elbrus_4c__) || defined(__elbrus_8c__) || \ + defined(__bfin__) || defined(__BFIN__) || defined(__ia64__) || defined(_IA64) || defined(__IA64__) || \ + defined(__ia64) || defined(_M_IA64) || defined(__itanium__) || defined(__ia32__) || defined(__CYGWIN__) || \ + defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) || defined(__WINDOWS__) #define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__ -#elif defined(__BIG_ENDIAN__) || \ - (defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) || \ - defined(__ARMEB__) || defined(__THUMBEB__) || defined(__AARCH64EB__) || \ - defined(__MIPSEB__) || defined(_MIPSEB) || defined(__MIPSEB) || \ - defined(__m68k__) || defined(M68000) || defined(__hppa__) || \ - defined(__hppa) || defined(__HPPA__) || defined(__sparc__) || \ - defined(__sparc) || defined(__370__) || defined(__THW_370__) || \ - defined(__s390__) || defined(__s390x__) || defined(__SYSC_ZARCH__) +#elif defined(__BIG_ENDIAN__) || (defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) || defined(__ARMEB__) || \ + defined(__THUMBEB__) || defined(__AARCH64EB__) || defined(__MIPSEB__) || defined(_MIPSEB) || defined(__MIPSEB) || \ + defined(__m68k__) || defined(M68000) || defined(__hppa__) || defined(__hppa) || defined(__HPPA__) || \ + defined(__sparc__) || defined(__sparc) || defined(__370__) || defined(__THW_370__) || defined(__s390__) || \ + defined(__s390x__) || defined(__SYSC_ZARCH__) #define __BYTE_ORDER__ __ORDER_BIG_ENDIAN__ #else @@ -539,6 +541,12 @@ __extern_C key_t ftok(const char *, int); #endif #endif /* __BYTE_ORDER__ || __ORDER_LITTLE_ENDIAN__ || __ORDER_BIG_ENDIAN__ */ +#if UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul || defined(_WIN64) +#define MDBX_WORDBITS 64 +#else +#define MDBX_WORDBITS 32 +#endif /* MDBX_WORDBITS */ + /*----------------------------------------------------------------------------*/ /* Availability of CMOV or equivalent */ @@ -549,17 +557,14 @@ __extern_C key_t ftok(const char *, int); #define MDBX_HAVE_CMOV 1 #elif defined(__thumb__) || defined(__thumb) || defined(__TARGET_ARCH_THUMB) #define MDBX_HAVE_CMOV 0 -#elif defined(_M_ARM) || defined(_M_ARM64) || defined(__aarch64__) || \ - defined(__aarch64) || defined(__arm__) || defined(__arm) || \ - defined(__CC_ARM) +#elif defined(_M_ARM) || defined(_M_ARM64) || defined(__aarch64__) || defined(__aarch64) || defined(__arm__) || \ + defined(__arm) || defined(__CC_ARM) #define MDBX_HAVE_CMOV 1 -#elif (defined(__riscv__) || defined(__riscv64)) && \ - (defined(__riscv_b) || defined(__riscv_bitmanip)) +#elif (defined(__riscv__) || defined(__riscv64)) && (defined(__riscv_b) || defined(__riscv_bitmanip)) #define MDBX_HAVE_CMOV 1 -#elif defined(i686) || defined(__i686) || defined(__i686__) || \ - (defined(_M_IX86) && _M_IX86 > 600) || defined(__x86_64) || \ - defined(__x86_64__) || defined(__amd64__) || defined(__amd64) || \ - defined(_M_X64) || defined(_M_AMD64) +#elif defined(i686) || defined(__i686) || defined(__i686__) || (defined(_M_IX86) && _M_IX86 > 600) || \ + defined(__x86_64) || defined(__x86_64__) || defined(__amd64__) || defined(__amd64) || defined(_M_X64) || \ + defined(_M_AMD64) #define MDBX_HAVE_CMOV 1 #else #define MDBX_HAVE_CMOV 0 @@ -585,8 +590,7 @@ __extern_C key_t ftok(const char *, int); #endif #elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) #include -#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ - (defined(HP_IA64) || defined(__ia64)) +#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && (defined(HP_IA64) || defined(__ia64)) #include #elif defined(__IBMC__) && defined(__powerpc) #include @@ -608,29 +612,26 @@ __extern_C key_t ftok(const char *, int); #endif /* Compiler */ #if !defined(__noop) && !defined(_MSC_VER) -#define __noop \ - do { \ +#define __noop \ + do { \ } while (0) #endif /* __noop */ -#if defined(__fallthrough) && \ - (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) +#if defined(__fallthrough) && (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) #undef __fallthrough #endif /* __fallthrough workaround for MinGW */ #ifndef __fallthrough -#if defined(__cplusplus) && (__has_cpp_attribute(fallthrough) && \ - (!defined(__clang__) || __clang__ > 4)) || \ +#if defined(__cplusplus) && (__has_cpp_attribute(fallthrough) && (!defined(__clang__) || __clang__ > 4)) || \ __cplusplus >= 201703L #define __fallthrough [[fallthrough]] #elif __GNUC_PREREQ(8, 0) && defined(__cplusplus) && __cplusplus >= 201103L #define __fallthrough [[fallthrough]] -#elif __GNUC_PREREQ(7, 0) && \ - (!defined(__LCC__) || (__LCC__ == 124 && __LCC_MINOR__ >= 12) || \ - (__LCC__ == 125 && __LCC_MINOR__ >= 5) || (__LCC__ >= 126)) +#elif __GNUC_PREREQ(7, 0) && (!defined(__LCC__) || (__LCC__ == 124 && __LCC_MINOR__ >= 12) || \ + (__LCC__ == 125 && __LCC_MINOR__ >= 5) || (__LCC__ >= 126)) #define __fallthrough __attribute__((__fallthrough__)) -#elif defined(__clang__) && defined(__cplusplus) && __cplusplus >= 201103L && \ - __has_feature(cxx_attributes) && __has_warning("-Wimplicit-fallthrough") +#elif defined(__clang__) && defined(__cplusplus) && __cplusplus >= 201103L && __has_feature(cxx_attributes) && \ + __has_warning("-Wimplicit-fallthrough") #define __fallthrough [[clang::fallthrough]] #else #define __fallthrough @@ -643,8 +644,8 @@ __extern_C key_t ftok(const char *, int); #elif defined(_MSC_VER) #define __unreachable() __assume(0) #else -#define __unreachable() \ - do { \ +#define __unreachable() \ + do { \ } while (1) #endif #endif /* __unreachable */ @@ -653,9 +654,9 @@ __extern_C key_t ftok(const char *, int); #if defined(__GNUC__) || defined(__clang__) || __has_builtin(__builtin_prefetch) #define __prefetch(ptr) __builtin_prefetch(ptr) #else -#define __prefetch(ptr) \ - do { \ - (void)(ptr); \ +#define __prefetch(ptr) \ + do { \ + (void)(ptr); \ } while (0) #endif #endif /* __prefetch */ @@ -665,11 +666,11 @@ __extern_C key_t ftok(const char *, int); #endif /* offsetof */ #ifndef container_of -#define container_of(ptr, type, member) \ - ((type *)((char *)(ptr) - offsetof(type, member))) +#define container_of(ptr, type, member) ((type *)((char *)(ptr) - offsetof(type, member))) #endif /* container_of */ /*----------------------------------------------------------------------------*/ +/* useful attributes */ #ifndef __always_inline #if defined(__GNUC__) || __has_attribute(__always_inline__) @@ -737,8 +738,7 @@ __extern_C key_t ftok(const char *, int); #ifndef __hot #if defined(__OPTIMIZE__) -#if defined(__clang__) && !__has_attribute(__hot__) && \ - __has_attribute(__section__) && \ +#if defined(__clang__) && !__has_attribute(__hot__) && __has_attribute(__section__) && \ (defined(__linux__) || defined(__gnu_linux__)) /* just put frequently used functions in separate section */ #define __hot __attribute__((__section__("text.hot"))) __optimize("O3") @@ -754,8 +754,7 @@ __extern_C key_t ftok(const char *, int); #ifndef __cold #if defined(__OPTIMIZE__) -#if defined(__clang__) && !__has_attribute(__cold__) && \ - __has_attribute(__section__) && \ +#if defined(__clang__) && !__has_attribute(__cold__) && __has_attribute(__section__) && \ (defined(__linux__) || defined(__gnu_linux__)) /* just put infrequently used functions in separate section */ #define __cold __attribute__((__section__("text.unlikely"))) __optimize("Os") @@ -778,8 +777,7 @@ __extern_C key_t ftok(const char *, int); #endif /* __flatten */ #ifndef likely -#if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && \ - !defined(__COVERITY__) +#if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__) #define likely(cond) __builtin_expect(!!(cond), 1) #else #define likely(x) (!!(x)) @@ -787,8 +785,7 @@ __extern_C key_t ftok(const char *, int); #endif /* likely */ #ifndef unlikely -#if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && \ - !defined(__COVERITY__) +#if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__) #define unlikely(cond) __builtin_expect(!!(cond), 0) #else #define unlikely(x) (!!(x)) @@ -803,29 +800,41 @@ __extern_C key_t ftok(const char *, int); #endif #endif /* __anonymous_struct_extension__ */ -#ifndef expect_with_probability -#if defined(__builtin_expect_with_probability) || \ - __has_builtin(__builtin_expect_with_probability) || __GNUC_PREREQ(9, 0) -#define expect_with_probability(expr, value, prob) \ - __builtin_expect_with_probability(expr, value, prob) -#else -#define expect_with_probability(expr, value, prob) (expr) -#endif -#endif /* expect_with_probability */ - #ifndef MDBX_WEAK_IMPORT_ATTRIBUTE #ifdef WEAK_IMPORT_ATTRIBUTE #define MDBX_WEAK_IMPORT_ATTRIBUTE WEAK_IMPORT_ATTRIBUTE #elif __has_attribute(__weak__) && __has_attribute(__weak_import__) #define MDBX_WEAK_IMPORT_ATTRIBUTE __attribute__((__weak__, __weak_import__)) -#elif __has_attribute(__weak__) || \ - (defined(__GNUC__) && __GNUC__ >= 4 && defined(__ELF__)) +#elif __has_attribute(__weak__) || (defined(__GNUC__) && __GNUC__ >= 4 && defined(__ELF__)) #define MDBX_WEAK_IMPORT_ATTRIBUTE __attribute__((__weak__)) #else #define MDBX_WEAK_IMPORT_ATTRIBUTE #endif #endif /* MDBX_WEAK_IMPORT_ATTRIBUTE */ +#if !defined(__thread) && (defined(_MSC_VER) || defined(__DMC__)) +#define __thread __declspec(thread) +#endif /* __thread */ + +#ifndef MDBX_EXCLUDE_FOR_GPROF +#ifdef ENABLE_GPROF +#define MDBX_EXCLUDE_FOR_GPROF __attribute__((__no_instrument_function__, __no_profile_instrument_function__)) +#else +#define MDBX_EXCLUDE_FOR_GPROF +#endif /* ENABLE_GPROF */ +#endif /* MDBX_EXCLUDE_FOR_GPROF */ + +/*----------------------------------------------------------------------------*/ + +#ifndef expect_with_probability +#if defined(__builtin_expect_with_probability) || __has_builtin(__builtin_expect_with_probability) || \ + __GNUC_PREREQ(9, 0) +#define expect_with_probability(expr, value, prob) __builtin_expect_with_probability(expr, value, prob) +#else +#define expect_with_probability(expr, value, prob) (expr) +#endif +#endif /* expect_with_probability */ + #ifndef MDBX_GOOFY_MSVC_STATIC_ANALYZER #ifdef _PREFAST_ #define MDBX_GOOFY_MSVC_STATIC_ANALYZER 1 @@ -837,20 +846,27 @@ __extern_C key_t ftok(const char *, int); #if MDBX_GOOFY_MSVC_STATIC_ANALYZER || (defined(_MSC_VER) && _MSC_VER > 1919) #define MDBX_ANALYSIS_ASSUME(expr) __analysis_assume(expr) #ifdef _PREFAST_ -#define MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(warn_id) \ - __pragma(prefast(suppress : warn_id)) +#define MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(warn_id) __pragma(prefast(suppress : warn_id)) #else -#define MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(warn_id) \ - __pragma(warning(suppress : warn_id)) +#define MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(warn_id) __pragma(warning(suppress : warn_id)) #endif #else #define MDBX_ANALYSIS_ASSUME(expr) assert(expr) #define MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(warn_id) #endif /* MDBX_GOOFY_MSVC_STATIC_ANALYZER */ +#ifndef FLEXIBLE_ARRAY_MEMBERS +#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || (!defined(__cplusplus) && defined(_MSC_VER)) +#define FLEXIBLE_ARRAY_MEMBERS 1 +#else +#define FLEXIBLE_ARRAY_MEMBERS 0 +#endif +#endif /* FLEXIBLE_ARRAY_MEMBERS */ + /*----------------------------------------------------------------------------*/ +/* Valgrind and Address Sanitizer */ -#if defined(MDBX_USE_VALGRIND) +#if defined(ENABLE_MEMCHECK) #include #ifndef VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE /* LY: available since Valgrind 3.10 */ @@ -872,7 +888,7 @@ __extern_C key_t ftok(const char *, int); #define VALGRIND_CHECK_MEM_IS_ADDRESSABLE(a, s) (0) #define VALGRIND_CHECK_MEM_IS_DEFINED(a, s) (0) #define RUNNING_ON_VALGRIND (0) -#endif /* MDBX_USE_VALGRIND */ +#endif /* ENABLE_MEMCHECK */ #ifdef __SANITIZE_ADDRESS__ #include @@ -899,8 +915,7 @@ template char (&__ArraySizeHelper(T (&array)[N]))[N]; #define CONCAT(a, b) a##b #define XCONCAT(a, b) CONCAT(a, b) -#define MDBX_TETRAD(a, b, c, d) \ - ((uint32_t)(a) << 24 | (uint32_t)(b) << 16 | (uint32_t)(c) << 8 | (d)) +#define MDBX_TETRAD(a, b, c, d) ((uint32_t)(a) << 24 | (uint32_t)(b) << 16 | (uint32_t)(c) << 8 | (d)) #define MDBX_STRING_TETRAD(str) MDBX_TETRAD(str[0], str[1], str[2], str[3]) @@ -914,14 +929,13 @@ template char (&__ArraySizeHelper(T (&array)[N]))[N]; #elif defined(_MSC_VER) #include #define STATIC_ASSERT_MSG(expr, msg) _STATIC_ASSERT(expr) -#elif (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) || \ - __has_feature(c_static_assert) +#elif (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) || __has_feature(c_static_assert) #define STATIC_ASSERT_MSG(expr, msg) _Static_assert(expr, msg) #else -#define STATIC_ASSERT_MSG(expr, msg) \ - switch (0) { \ - case 0: \ - case (expr):; \ +#define STATIC_ASSERT_MSG(expr, msg) \ + switch (0) { \ + case 0: \ + case (expr):; \ } #endif #endif /* STATIC_ASSERT */ @@ -930,42 +944,37 @@ template char (&__ArraySizeHelper(T (&array)[N]))[N]; #define STATIC_ASSERT(expr) STATIC_ASSERT_MSG(expr, #expr) #endif -#ifndef __Wpedantic_format_voidptr -MDBX_MAYBE_UNUSED MDBX_PURE_FUNCTION static __inline const void * -__Wpedantic_format_voidptr(const void *ptr) { - return ptr; -} -#define __Wpedantic_format_voidptr(ARG) __Wpedantic_format_voidptr(ARG) -#endif /* __Wpedantic_format_voidptr */ +/*----------------------------------------------------------------------------*/ -#if defined(__GNUC__) && !__GNUC_PREREQ(4, 2) -/* Actually libmdbx was not tested with compilers older than GCC 4.2. - * But you could ignore this warning at your own risk. - * In such case please don't rise up an issues related ONLY to old compilers. - */ -#warning "libmdbx required GCC >= 4.2" -#endif +#if defined(_MSC_VER) && _MSC_VER >= 1900 +/* LY: MSVC 2015/2017/2019 has buggy/inconsistent PRIuPTR/PRIxPTR macros + * for internal format-args checker. */ +#undef PRIuPTR +#undef PRIiPTR +#undef PRIdPTR +#undef PRIxPTR +#define PRIuPTR "Iu" +#define PRIiPTR "Ii" +#define PRIdPTR "Id" +#define PRIxPTR "Ix" +#define PRIuSIZE "zu" +#define PRIiSIZE "zi" +#define PRIdSIZE "zd" +#define PRIxSIZE "zx" +#endif /* fix PRI*PTR for _MSC_VER */ -#if defined(__clang__) && !__CLANG_PREREQ(3, 8) -/* Actually libmdbx was not tested with CLANG older than 3.8. - * But you could ignore this warning at your own risk. - * In such case please don't rise up an issues related ONLY to old compilers. - */ -#warning "libmdbx required CLANG >= 3.8" -#endif +#ifndef PRIuSIZE +#define PRIuSIZE PRIuPTR +#define PRIiSIZE PRIiPTR +#define PRIdSIZE PRIdPTR +#define PRIxSIZE PRIxPTR +#endif /* PRI*SIZE macros for MSVC */ -#if defined(__GLIBC__) && !__GLIBC_PREREQ(2, 12) -/* Actually libmdbx was not tested with something older than glibc 2.12. - * But you could ignore this warning at your own risk. - * In such case please don't rise up an issues related ONLY to old systems. - */ -#warning "libmdbx was only tested with GLIBC >= 2.12." +#ifdef _MSC_VER +#pragma warning(pop) #endif -#ifdef __SANITIZE_THREAD__ -#warning \ - "libmdbx don't compatible with ThreadSanitizer, you will get a lot of false-positive issues." -#endif /* __SANITIZE_THREAD__ */ +/*----------------------------------------------------------------------------*/ #if __has_warning("-Wnested-anon-types") #if defined(__clang__) @@ -1002,80 +1011,34 @@ __Wpedantic_format_voidptr(const void *ptr) { #endif #endif /* -Walignment-reduction-ignored */ -#ifndef MDBX_EXCLUDE_FOR_GPROF -#ifdef ENABLE_GPROF -#define MDBX_EXCLUDE_FOR_GPROF \ - __attribute__((__no_instrument_function__, \ - __no_profile_instrument_function__)) +#ifdef xMDBX_ALLOY +/* Amalgamated build */ +#define MDBX_INTERNAL static #else -#define MDBX_EXCLUDE_FOR_GPROF -#endif /* ENABLE_GPROF */ -#endif /* MDBX_EXCLUDE_FOR_GPROF */ - -#ifdef __cplusplus -extern "C" { -#endif +/* Non-amalgamated build */ +#define MDBX_INTERNAL +#endif /* xMDBX_ALLOY */ -/* https://en.wikipedia.org/wiki/Operating_system_abstraction_layer */ +#include "mdbx.h" -/* - * Copyright 2015-2024 Leonid Yuriev - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - */ +/*----------------------------------------------------------------------------*/ +/* Basic constants and types */ +typedef struct iov_ctx iov_ctx_t; +/// /*----------------------------------------------------------------------------*/ -/* C11 Atomics */ - -#if defined(__cplusplus) && !defined(__STDC_NO_ATOMICS__) && __has_include() -#include -#define MDBX_HAVE_C11ATOMICS -#elif !defined(__cplusplus) && \ - (__STDC_VERSION__ >= 201112L || __has_extension(c_atomic)) && \ - !defined(__STDC_NO_ATOMICS__) && \ - (__GNUC_PREREQ(4, 9) || __CLANG_PREREQ(3, 8) || \ - !(defined(__GNUC__) || defined(__clang__))) -#include -#define MDBX_HAVE_C11ATOMICS -#elif defined(__GNUC__) || defined(__clang__) -#elif defined(_MSC_VER) -#pragma warning(disable : 4163) /* 'xyz': not available as an intrinsic */ -#pragma warning(disable : 4133) /* 'function': incompatible types - from \ - 'size_t' to 'LONGLONG' */ -#pragma warning(disable : 4244) /* 'return': conversion from 'LONGLONG' to \ - 'std::size_t', possible loss of data */ -#pragma warning(disable : 4267) /* 'function': conversion from 'size_t' to \ - 'long', possible loss of data */ -#pragma intrinsic(_InterlockedExchangeAdd, _InterlockedCompareExchange) -#pragma intrinsic(_InterlockedExchangeAdd64, _InterlockedCompareExchange64) -#elif defined(__APPLE__) -#include -#else -#error FIXME atomic-ops -#endif - -/*----------------------------------------------------------------------------*/ -/* Memory/Compiler barriers, cache coherence */ +/* Memory/Compiler barriers, cache coherence */ #if __has_include() #include -#elif defined(__mips) || defined(__mips__) || defined(__mips64) || \ - defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \ - defined(__MWERKS__) || defined(__sgi) +#elif defined(__mips) || defined(__mips__) || defined(__mips64) || defined(__mips64__) || defined(_M_MRX000) || \ + defined(_MIPS_) || defined(__MWERKS__) || defined(__sgi) /* MIPS should have explicit cache control */ #include #endif -MDBX_MAYBE_UNUSED static __inline void osal_compiler_barrier(void) { +MDBX_MAYBE_UNUSED static inline void osal_compiler_barrier(void) { #if defined(__clang__) || defined(__GNUC__) __asm__ __volatile__("" ::: "memory"); #elif defined(_MSC_VER) @@ -1084,18 +1047,16 @@ MDBX_MAYBE_UNUSED static __inline void osal_compiler_barrier(void) { __memory_barrier(); #elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) __compiler_barrier(); -#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ - (defined(HP_IA64) || defined(__ia64)) +#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && (defined(HP_IA64) || defined(__ia64)) _Asm_sched_fence(/* LY: no-arg meaning 'all expect ALU', e.g. 0x3D3D */); -#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \ - defined(__ppc64__) || defined(__powerpc64__) +#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || defined(__ppc64__) || defined(__powerpc64__) __fence(); #else #error "Could not guess the kind of compiler, please report to us." #endif } -MDBX_MAYBE_UNUSED static __inline void osal_memory_barrier(void) { +MDBX_MAYBE_UNUSED static inline void osal_memory_barrier(void) { #ifdef MDBX_HAVE_C11ATOMICS atomic_thread_fence(memory_order_seq_cst); #elif defined(__ATOMIC_SEQ_CST) @@ -1116,11 +1077,9 @@ MDBX_MAYBE_UNUSED static __inline void osal_memory_barrier(void) { #endif #elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) __machine_rw_barrier(); -#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ - (defined(HP_IA64) || defined(__ia64)) +#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && (defined(HP_IA64) || defined(__ia64)) _Asm_mf(); -#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \ - defined(__ppc64__) || defined(__powerpc64__) +#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || defined(__ppc64__) || defined(__powerpc64__) __lwsync(); #else #error "Could not guess the kind of compiler, please report to us." @@ -1135,7 +1094,7 @@ MDBX_MAYBE_UNUSED static __inline void osal_memory_barrier(void) { #define HAVE_SYS_TYPES_H typedef HANDLE osal_thread_t; typedef unsigned osal_thread_key_t; -#define MAP_FAILED NULL +#define MAP_FAILED nullptr #define HIGH_DWORD(v) ((DWORD)((sizeof(v) > 4) ? ((uint64_t)(v) >> 32) : 0)) #define THREAD_CALL WINAPI #define THREAD_RESULT DWORD @@ -1147,15 +1106,13 @@ typedef CRITICAL_SECTION osal_fastmutex_t; #if !defined(_MSC_VER) && !defined(__try) #define __try -#define __except(COND) if (false) +#define __except(COND) if (/* (void)(COND), */ false) #endif /* stub for MSVC's __try/__except */ #if MDBX_WITHOUT_MSVC_CRT #ifndef osal_malloc -static inline void *osal_malloc(size_t bytes) { - return HeapAlloc(GetProcessHeap(), 0, bytes); -} +static inline void *osal_malloc(size_t bytes) { return HeapAlloc(GetProcessHeap(), 0, bytes); } #endif /* osal_malloc */ #ifndef osal_calloc @@ -1166,8 +1123,7 @@ static inline void *osal_calloc(size_t nelem, size_t size) { #ifndef osal_realloc static inline void *osal_realloc(void *ptr, size_t bytes) { - return ptr ? HeapReAlloc(GetProcessHeap(), 0, ptr, bytes) - : HeapAlloc(GetProcessHeap(), 0, bytes); + return ptr ? HeapReAlloc(GetProcessHeap(), 0, ptr, bytes) : HeapAlloc(GetProcessHeap(), 0, bytes); } #endif /* osal_realloc */ @@ -1213,29 +1169,16 @@ typedef pthread_mutex_t osal_fastmutex_t; #endif /* Platform */ #if __GLIBC_PREREQ(2, 12) || defined(__FreeBSD__) || defined(malloc_usable_size) -/* malloc_usable_size() already provided */ +#define osal_malloc_usable_size(ptr) malloc_usable_size(ptr) #elif defined(__APPLE__) -#define malloc_usable_size(ptr) malloc_size(ptr) +#define osal_malloc_usable_size(ptr) malloc_size(ptr) #elif defined(_MSC_VER) && !MDBX_WITHOUT_MSVC_CRT -#define malloc_usable_size(ptr) _msize(ptr) -#endif /* malloc_usable_size */ +#define osal_malloc_usable_size(ptr) _msize(ptr) +#endif /* osal_malloc_usable_size */ /*----------------------------------------------------------------------------*/ /* OS abstraction layer stuff */ -MDBX_INTERNAL_VAR unsigned sys_pagesize; -MDBX_MAYBE_UNUSED MDBX_INTERNAL_VAR unsigned sys_pagesize_ln2, - sys_allocation_granularity; - -/* Get the size of a memory page for the system. - * This is the basic size that the platform's memory manager uses, and is - * fundamental to the use of memory-mapped files. */ -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline size_t -osal_syspagesize(void) { - assert(sys_pagesize > 0 && (sys_pagesize & (sys_pagesize - 1)) == 0); - return sys_pagesize; -} - #if defined(_WIN32) || defined(_WIN64) typedef wchar_t pathchar_t; #define MDBX_PRIsPATH "ls" @@ -1247,7 +1190,7 @@ typedef char pathchar_t; typedef struct osal_mmap { union { void *base; - struct MDBX_lockinfo *lck; + struct shared_lck *lck; }; mdbx_filehandle_t fd; size_t limit; /* mapping length, but NOT a size of file nor DB */ @@ -1258,25 +1201,6 @@ typedef struct osal_mmap { #endif } osal_mmap_t; -typedef union bin128 { - __anonymous_struct_extension__ struct { - uint64_t x, y; - }; - __anonymous_struct_extension__ struct { - uint32_t a, b, c, d; - }; -} bin128_t; - -#if defined(_WIN32) || defined(_WIN64) -typedef union osal_srwlock { - __anonymous_struct_extension__ struct { - long volatile readerCount; - long volatile writerCount; - }; - RTL_SRWLOCK native; -} osal_srwlock_t; -#endif /* Windows */ - #ifndef MDBX_HAVE_PWRITEV #if defined(_WIN32) || defined(_WIN64) @@ -1285,14 +1209,21 @@ typedef union osal_srwlock { #elif defined(__ANDROID_API__) #if __ANDROID_API__ < 24 +/* https://android-developers.googleblog.com/2017/09/introducing-android-native-development.html + * https://android.googlesource.com/platform/bionic/+/master/docs/32-bit-abi.md */ #define MDBX_HAVE_PWRITEV 0 +#if defined(_FILE_OFFSET_BITS) && _FILE_OFFSET_BITS != MDBX_WORDBITS +#error "_FILE_OFFSET_BITS != MDBX_WORDBITS and __ANDROID_API__ < 24" (_FILE_OFFSET_BITS != MDBX_WORDBITS) +#elif defined(__FILE_OFFSET_BITS) && __FILE_OFFSET_BITS != MDBX_WORDBITS +#error "__FILE_OFFSET_BITS != MDBX_WORDBITS and __ANDROID_API__ < 24" (__FILE_OFFSET_BITS != MDBX_WORDBITS) +#endif #else #define MDBX_HAVE_PWRITEV 1 #endif #elif defined(__APPLE__) || defined(__MACH__) || defined(_DARWIN_C_SOURCE) -#if defined(MAC_OS_X_VERSION_MIN_REQUIRED) && defined(MAC_OS_VERSION_11_0) && \ +#if defined(MAC_OS_X_VERSION_MIN_REQUIRED) && defined(MAC_OS_VERSION_11_0) && \ MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_VERSION_11_0 /* FIXME: add checks for IOS versions, etc */ #define MDBX_HAVE_PWRITEV 1 @@ -1310,20 +1241,20 @@ typedef union osal_srwlock { typedef struct ior_item { #if defined(_WIN32) || defined(_WIN64) OVERLAPPED ov; -#define ior_svg_gap4terminator 1 +#define ior_sgv_gap4terminator 1 #define ior_sgv_element FILE_SEGMENT_ELEMENT #else size_t offset; #if MDBX_HAVE_PWRITEV size_t sgvcnt; -#define ior_svg_gap4terminator 0 +#define ior_sgv_gap4terminator 0 #define ior_sgv_element struct iovec #endif /* MDBX_HAVE_PWRITEV */ #endif /* !Windows */ union { MDBX_val single; #if defined(ior_sgv_element) - ior_sgv_element sgv[1 + ior_svg_gap4terminator]; + ior_sgv_element sgv[1 + ior_sgv_gap4terminator]; #endif /* ior_sgv_element */ }; } ior_item_t; @@ -1359,45 +1290,33 @@ typedef struct osal_ioring { char *boundary; } osal_ioring_t; -#ifndef __cplusplus - /* Actually this is not ioring for now, but on the way. */ -MDBX_INTERNAL_FUNC int osal_ioring_create(osal_ioring_t * +MDBX_INTERNAL int osal_ioring_create(osal_ioring_t * #if defined(_WIN32) || defined(_WIN64) - , - bool enable_direct, - mdbx_filehandle_t overlapped_fd + , + bool enable_direct, mdbx_filehandle_t overlapped_fd #endif /* Windows */ ); -MDBX_INTERNAL_FUNC int osal_ioring_resize(osal_ioring_t *, size_t items); -MDBX_INTERNAL_FUNC void osal_ioring_destroy(osal_ioring_t *); -MDBX_INTERNAL_FUNC void osal_ioring_reset(osal_ioring_t *); -MDBX_INTERNAL_FUNC int osal_ioring_add(osal_ioring_t *ctx, const size_t offset, - void *data, const size_t bytes); +MDBX_INTERNAL int osal_ioring_resize(osal_ioring_t *, size_t items); +MDBX_INTERNAL void osal_ioring_destroy(osal_ioring_t *); +MDBX_INTERNAL void osal_ioring_reset(osal_ioring_t *); +MDBX_INTERNAL int osal_ioring_add(osal_ioring_t *ctx, const size_t offset, void *data, const size_t bytes); typedef struct osal_ioring_write_result { int err; unsigned wops; } osal_ioring_write_result_t; -MDBX_INTERNAL_FUNC osal_ioring_write_result_t -osal_ioring_write(osal_ioring_t *ior, mdbx_filehandle_t fd); +MDBX_INTERNAL osal_ioring_write_result_t osal_ioring_write(osal_ioring_t *ior, mdbx_filehandle_t fd); -typedef struct iov_ctx iov_ctx_t; -MDBX_INTERNAL_FUNC void osal_ioring_walk( - osal_ioring_t *ior, iov_ctx_t *ctx, - void (*callback)(iov_ctx_t *ctx, size_t offset, void *data, size_t bytes)); +MDBX_INTERNAL void osal_ioring_walk(osal_ioring_t *ior, iov_ctx_t *ctx, + void (*callback)(iov_ctx_t *ctx, size_t offset, void *data, size_t bytes)); -MDBX_MAYBE_UNUSED static inline unsigned -osal_ioring_left(const osal_ioring_t *ior) { - return ior->slots_left; -} +MDBX_MAYBE_UNUSED static inline unsigned osal_ioring_left(const osal_ioring_t *ior) { return ior->slots_left; } -MDBX_MAYBE_UNUSED static inline unsigned -osal_ioring_used(const osal_ioring_t *ior) { +MDBX_MAYBE_UNUSED static inline unsigned osal_ioring_used(const osal_ioring_t *ior) { return ior->allocated - ior->slots_left; } -MDBX_MAYBE_UNUSED static inline int -osal_ioring_prepare(osal_ioring_t *ior, size_t items, size_t bytes) { +MDBX_MAYBE_UNUSED static inline int osal_ioring_prepare(osal_ioring_t *ior, size_t items, size_t bytes) { items = (items > 32) ? items : 32; #if defined(_WIN32) || defined(_WIN64) if (ior->direct) { @@ -1416,14 +1335,12 @@ osal_ioring_prepare(osal_ioring_t *ior, size_t items, size_t bytes) { /*----------------------------------------------------------------------------*/ /* libc compatibility stuff */ -#if (!defined(__GLIBC__) && __GLIBC_PREREQ(2, 1)) && \ - (defined(_GNU_SOURCE) || defined(_BSD_SOURCE)) +#if (!defined(__GLIBC__) && __GLIBC_PREREQ(2, 1)) && (defined(_GNU_SOURCE) || defined(_BSD_SOURCE)) #define osal_asprintf asprintf #define osal_vasprintf vasprintf #else -MDBX_MAYBE_UNUSED MDBX_INTERNAL_FUNC - MDBX_PRINTF_ARGS(2, 3) int osal_asprintf(char **strp, const char *fmt, ...); -MDBX_INTERNAL_FUNC int osal_vasprintf(char **strp, const char *fmt, va_list ap); +MDBX_MAYBE_UNUSED MDBX_INTERNAL MDBX_PRINTF_ARGS(2, 3) int osal_asprintf(char **strp, const char *fmt, ...); +MDBX_INTERNAL int osal_vasprintf(char **strp, const char *fmt, va_list ap); #endif #if !defined(MADV_DODUMP) && defined(MADV_CORE) @@ -1434,8 +1351,7 @@ MDBX_INTERNAL_FUNC int osal_vasprintf(char **strp, const char *fmt, va_list ap); #define MADV_DONTDUMP MADV_NOCORE #endif /* MADV_NOCORE -> MADV_DONTDUMP */ -MDBX_MAYBE_UNUSED MDBX_INTERNAL_FUNC void osal_jitter(bool tiny); -MDBX_MAYBE_UNUSED static __inline void jitter4testing(bool tiny); +MDBX_MAYBE_UNUSED MDBX_INTERNAL void osal_jitter(bool tiny); /* max bytes to write in one call */ #if defined(_WIN64) @@ -1445,14 +1361,12 @@ MDBX_MAYBE_UNUSED static __inline void jitter4testing(bool tiny); #else #define MAX_WRITE UINT32_C(0x3f000000) -#if defined(F_GETLK64) && defined(F_SETLK64) && defined(F_SETLKW64) && \ - !defined(__ANDROID_API__) +#if defined(F_GETLK64) && defined(F_SETLK64) && defined(F_SETLKW64) && !defined(__ANDROID_API__) #define MDBX_F_SETLK F_SETLK64 #define MDBX_F_SETLKW F_SETLKW64 #define MDBX_F_GETLK F_GETLK64 -#if (__GLIBC_PREREQ(2, 28) && \ - (defined(__USE_LARGEFILE64) || defined(__LARGEFILE64_SOURCE) || \ - defined(_USE_LARGEFILE64) || defined(_LARGEFILE64_SOURCE))) || \ +#if (__GLIBC_PREREQ(2, 28) && (defined(__USE_LARGEFILE64) || defined(__LARGEFILE64_SOURCE) || \ + defined(_USE_LARGEFILE64) || defined(_LARGEFILE64_SOURCE))) || \ defined(fcntl64) #define MDBX_FCNTL fcntl64 #else @@ -1470,8 +1384,7 @@ MDBX_MAYBE_UNUSED static __inline void jitter4testing(bool tiny); #define MDBX_STRUCT_FLOCK struct flock #endif /* MDBX_F_SETLK, MDBX_F_SETLKW, MDBX_F_GETLK */ -#if defined(F_OFD_SETLK64) && defined(F_OFD_SETLKW64) && \ - defined(F_OFD_GETLK64) && !defined(__ANDROID_API__) +#if defined(F_OFD_SETLK64) && defined(F_OFD_SETLKW64) && defined(F_OFD_GETLK64) && !defined(__ANDROID_API__) #define MDBX_F_OFD_SETLK F_OFD_SETLK64 #define MDBX_F_OFD_SETLKW F_OFD_SETLKW64 #define MDBX_F_OFD_GETLK F_OFD_GETLK64 @@ -1480,23 +1393,17 @@ MDBX_MAYBE_UNUSED static __inline void jitter4testing(bool tiny); #define MDBX_F_OFD_SETLKW F_OFD_SETLKW #define MDBX_F_OFD_GETLK F_OFD_GETLK #ifndef OFF_T_MAX -#define OFF_T_MAX \ - (((sizeof(off_t) > 4) ? INT64_MAX : INT32_MAX) & ~(size_t)0xFffff) +#define OFF_T_MAX (((sizeof(off_t) > 4) ? INT64_MAX : INT32_MAX) & ~(size_t)0xFffff) #endif /* OFF_T_MAX */ #endif /* MDBX_F_OFD_SETLK64, MDBX_F_OFD_SETLKW64, MDBX_F_OFD_GETLK64 */ -#endif - -#if defined(__linux__) || defined(__gnu_linux__) -MDBX_INTERNAL_VAR uint32_t linux_kernel_version; -MDBX_INTERNAL_VAR bool mdbx_RunningOnWSL1 /* Windows Subsystem 1 for Linux */; -#endif /* Linux */ +#endif /* !Windows */ #ifndef osal_strdup LIBMDBX_API char *osal_strdup(const char *str); #endif -MDBX_MAYBE_UNUSED static __inline int osal_get_errno(void) { +MDBX_MAYBE_UNUSED static inline int osal_get_errno(void) { #if defined(_WIN32) || defined(_WIN64) DWORD rc = GetLastError(); #else @@ -1506,40 +1413,32 @@ MDBX_MAYBE_UNUSED static __inline int osal_get_errno(void) { } #ifndef osal_memalign_alloc -MDBX_INTERNAL_FUNC int osal_memalign_alloc(size_t alignment, size_t bytes, - void **result); +MDBX_INTERNAL int osal_memalign_alloc(size_t alignment, size_t bytes, void **result); #endif #ifndef osal_memalign_free -MDBX_INTERNAL_FUNC void osal_memalign_free(void *ptr); -#endif - -MDBX_INTERNAL_FUNC int osal_condpair_init(osal_condpair_t *condpair); -MDBX_INTERNAL_FUNC int osal_condpair_lock(osal_condpair_t *condpair); -MDBX_INTERNAL_FUNC int osal_condpair_unlock(osal_condpair_t *condpair); -MDBX_INTERNAL_FUNC int osal_condpair_signal(osal_condpair_t *condpair, - bool part); -MDBX_INTERNAL_FUNC int osal_condpair_wait(osal_condpair_t *condpair, bool part); -MDBX_INTERNAL_FUNC int osal_condpair_destroy(osal_condpair_t *condpair); - -MDBX_INTERNAL_FUNC int osal_fastmutex_init(osal_fastmutex_t *fastmutex); -MDBX_INTERNAL_FUNC int osal_fastmutex_acquire(osal_fastmutex_t *fastmutex); -MDBX_INTERNAL_FUNC int osal_fastmutex_release(osal_fastmutex_t *fastmutex); -MDBX_INTERNAL_FUNC int osal_fastmutex_destroy(osal_fastmutex_t *fastmutex); - -MDBX_INTERNAL_FUNC int osal_pwritev(mdbx_filehandle_t fd, struct iovec *iov, - size_t sgvcnt, uint64_t offset); -MDBX_INTERNAL_FUNC int osal_pread(mdbx_filehandle_t fd, void *buf, size_t count, - uint64_t offset); -MDBX_INTERNAL_FUNC int osal_pwrite(mdbx_filehandle_t fd, const void *buf, - size_t count, uint64_t offset); -MDBX_INTERNAL_FUNC int osal_write(mdbx_filehandle_t fd, const void *buf, - size_t count); - -MDBX_INTERNAL_FUNC int -osal_thread_create(osal_thread_t *thread, - THREAD_RESULT(THREAD_CALL *start_routine)(void *), - void *arg); -MDBX_INTERNAL_FUNC int osal_thread_join(osal_thread_t thread); +MDBX_INTERNAL void osal_memalign_free(void *ptr); +#endif + +MDBX_INTERNAL int osal_condpair_init(osal_condpair_t *condpair); +MDBX_INTERNAL int osal_condpair_lock(osal_condpair_t *condpair); +MDBX_INTERNAL int osal_condpair_unlock(osal_condpair_t *condpair); +MDBX_INTERNAL int osal_condpair_signal(osal_condpair_t *condpair, bool part); +MDBX_INTERNAL int osal_condpair_wait(osal_condpair_t *condpair, bool part); +MDBX_INTERNAL int osal_condpair_destroy(osal_condpair_t *condpair); + +MDBX_INTERNAL int osal_fastmutex_init(osal_fastmutex_t *fastmutex); +MDBX_INTERNAL int osal_fastmutex_acquire(osal_fastmutex_t *fastmutex); +MDBX_INTERNAL int osal_fastmutex_release(osal_fastmutex_t *fastmutex); +MDBX_INTERNAL int osal_fastmutex_destroy(osal_fastmutex_t *fastmutex); + +MDBX_INTERNAL int osal_pwritev(mdbx_filehandle_t fd, struct iovec *iov, size_t sgvcnt, uint64_t offset); +MDBX_INTERNAL int osal_pread(mdbx_filehandle_t fd, void *buf, size_t count, uint64_t offset); +MDBX_INTERNAL int osal_pwrite(mdbx_filehandle_t fd, const void *buf, size_t count, uint64_t offset); +MDBX_INTERNAL int osal_write(mdbx_filehandle_t fd, const void *buf, size_t count); + +MDBX_INTERNAL int osal_thread_create(osal_thread_t *thread, THREAD_RESULT(THREAD_CALL *start_routine)(void *), + void *arg); +MDBX_INTERNAL int osal_thread_join(osal_thread_t thread); enum osal_syncmode_bits { MDBX_SYNC_NONE = 0, @@ -1549,11 +1448,10 @@ enum osal_syncmode_bits { MDBX_SYNC_IODQ = 8 }; -MDBX_INTERNAL_FUNC int osal_fsync(mdbx_filehandle_t fd, - const enum osal_syncmode_bits mode_bits); -MDBX_INTERNAL_FUNC int osal_ftruncate(mdbx_filehandle_t fd, uint64_t length); -MDBX_INTERNAL_FUNC int osal_fseek(mdbx_filehandle_t fd, uint64_t pos); -MDBX_INTERNAL_FUNC int osal_filesize(mdbx_filehandle_t fd, uint64_t *length); +MDBX_INTERNAL int osal_fsync(mdbx_filehandle_t fd, const enum osal_syncmode_bits mode_bits); +MDBX_INTERNAL int osal_ftruncate(mdbx_filehandle_t fd, uint64_t length); +MDBX_INTERNAL int osal_fseek(mdbx_filehandle_t fd, uint64_t pos); +MDBX_INTERNAL int osal_filesize(mdbx_filehandle_t fd, uint64_t *length); enum osal_openfile_purpose { MDBX_OPEN_DXB_READ, @@ -1568,7 +1466,7 @@ enum osal_openfile_purpose { MDBX_OPEN_DELETE }; -MDBX_MAYBE_UNUSED static __inline bool osal_isdirsep(pathchar_t c) { +MDBX_MAYBE_UNUSED static inline bool osal_isdirsep(pathchar_t c) { return #if defined(_WIN32) || defined(_WIN64) c == '\\' || @@ -1576,50 +1474,39 @@ MDBX_MAYBE_UNUSED static __inline bool osal_isdirsep(pathchar_t c) { c == '/'; } -MDBX_INTERNAL_FUNC bool osal_pathequal(const pathchar_t *l, const pathchar_t *r, - size_t len); -MDBX_INTERNAL_FUNC pathchar_t *osal_fileext(const pathchar_t *pathname, - size_t len); -MDBX_INTERNAL_FUNC int osal_fileexists(const pathchar_t *pathname); -MDBX_INTERNAL_FUNC int osal_openfile(const enum osal_openfile_purpose purpose, - const MDBX_env *env, - const pathchar_t *pathname, - mdbx_filehandle_t *fd, - mdbx_mode_t unix_mode_bits); -MDBX_INTERNAL_FUNC int osal_closefile(mdbx_filehandle_t fd); -MDBX_INTERNAL_FUNC int osal_removefile(const pathchar_t *pathname); -MDBX_INTERNAL_FUNC int osal_removedirectory(const pathchar_t *pathname); -MDBX_INTERNAL_FUNC int osal_is_pipe(mdbx_filehandle_t fd); -MDBX_INTERNAL_FUNC int osal_lockfile(mdbx_filehandle_t fd, bool wait); +MDBX_INTERNAL bool osal_pathequal(const pathchar_t *l, const pathchar_t *r, size_t len); +MDBX_INTERNAL pathchar_t *osal_fileext(const pathchar_t *pathname, size_t len); +MDBX_INTERNAL int osal_fileexists(const pathchar_t *pathname); +MDBX_INTERNAL int osal_openfile(const enum osal_openfile_purpose purpose, const MDBX_env *env, + const pathchar_t *pathname, mdbx_filehandle_t *fd, mdbx_mode_t unix_mode_bits); +MDBX_INTERNAL int osal_closefile(mdbx_filehandle_t fd); +MDBX_INTERNAL int osal_removefile(const pathchar_t *pathname); +MDBX_INTERNAL int osal_removedirectory(const pathchar_t *pathname); +MDBX_INTERNAL int osal_is_pipe(mdbx_filehandle_t fd); +MDBX_INTERNAL int osal_lockfile(mdbx_filehandle_t fd, bool wait); #define MMAP_OPTION_TRUNCATE 1 #define MMAP_OPTION_SEMAPHORE 2 -MDBX_INTERNAL_FUNC int osal_mmap(const int flags, osal_mmap_t *map, size_t size, - const size_t limit, const unsigned options); -MDBX_INTERNAL_FUNC int osal_munmap(osal_mmap_t *map); +MDBX_INTERNAL int osal_mmap(const int flags, osal_mmap_t *map, size_t size, const size_t limit, const unsigned options, + const pathchar_t *pathname4logging); +MDBX_INTERNAL int osal_munmap(osal_mmap_t *map); #define MDBX_MRESIZE_MAY_MOVE 0x00000100 #define MDBX_MRESIZE_MAY_UNMAP 0x00000200 -MDBX_INTERNAL_FUNC int osal_mresize(const int flags, osal_mmap_t *map, - size_t size, size_t limit); +MDBX_INTERNAL int osal_mresize(const int flags, osal_mmap_t *map, size_t size, size_t limit); #if defined(_WIN32) || defined(_WIN64) typedef struct { unsigned limit, count; HANDLE handles[31]; } mdbx_handle_array_t; -MDBX_INTERNAL_FUNC int -osal_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array); -MDBX_INTERNAL_FUNC int -osal_resume_threads_after_remap(mdbx_handle_array_t *array); +MDBX_INTERNAL int osal_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array); +MDBX_INTERNAL int osal_resume_threads_after_remap(mdbx_handle_array_t *array); #endif /* Windows */ -MDBX_INTERNAL_FUNC int osal_msync(const osal_mmap_t *map, size_t offset, - size_t length, - enum osal_syncmode_bits mode_bits); -MDBX_INTERNAL_FUNC int osal_check_fs_rdonly(mdbx_filehandle_t handle, - const pathchar_t *pathname, - int err); -MDBX_INTERNAL_FUNC int osal_check_fs_incore(mdbx_filehandle_t handle); - -MDBX_MAYBE_UNUSED static __inline uint32_t osal_getpid(void) { +MDBX_INTERNAL int osal_msync(const osal_mmap_t *map, size_t offset, size_t length, enum osal_syncmode_bits mode_bits); +MDBX_INTERNAL int osal_check_fs_rdonly(mdbx_filehandle_t handle, const pathchar_t *pathname, int err); +MDBX_INTERNAL int osal_check_fs_incore(mdbx_filehandle_t handle); +MDBX_INTERNAL int osal_check_fs_local(mdbx_filehandle_t handle, int flags); + +MDBX_MAYBE_UNUSED static inline uint32_t osal_getpid(void) { STATIC_ASSERT(sizeof(mdbx_pid_t) <= sizeof(uint32_t)); #if defined(_WIN32) || defined(_WIN64) return GetCurrentProcessId(); @@ -1629,7 +1516,7 @@ MDBX_MAYBE_UNUSED static __inline uint32_t osal_getpid(void) { #endif } -MDBX_MAYBE_UNUSED static __inline uintptr_t osal_thread_self(void) { +MDBX_MAYBE_UNUSED static inline uintptr_t osal_thread_self(void) { mdbx_tid_t thunk; STATIC_ASSERT(sizeof(uintptr_t) >= sizeof(thunk)); #if defined(_WIN32) || defined(_WIN64) @@ -1642,274 +1529,51 @@ MDBX_MAYBE_UNUSED static __inline uintptr_t osal_thread_self(void) { #if !defined(_WIN32) && !defined(_WIN64) #if defined(__ANDROID_API__) || defined(ANDROID) || defined(BIONIC) -MDBX_INTERNAL_FUNC int osal_check_tid4bionic(void); +MDBX_INTERNAL int osal_check_tid4bionic(void); #else -static __inline int osal_check_tid4bionic(void) { return 0; } +static inline int osal_check_tid4bionic(void) { return 0; } #endif /* __ANDROID_API__ || ANDROID) || BIONIC */ -MDBX_MAYBE_UNUSED static __inline int -osal_pthread_mutex_lock(pthread_mutex_t *mutex) { +MDBX_MAYBE_UNUSED static inline int osal_pthread_mutex_lock(pthread_mutex_t *mutex) { int err = osal_check_tid4bionic(); return unlikely(err) ? err : pthread_mutex_lock(mutex); } #endif /* !Windows */ -MDBX_INTERNAL_FUNC uint64_t osal_monotime(void); -MDBX_INTERNAL_FUNC uint64_t osal_cputime(size_t *optional_page_faults); -MDBX_INTERNAL_FUNC uint64_t osal_16dot16_to_monotime(uint32_t seconds_16dot16); -MDBX_INTERNAL_FUNC uint32_t osal_monotime_to_16dot16(uint64_t monotime); +MDBX_INTERNAL uint64_t osal_monotime(void); +MDBX_INTERNAL uint64_t osal_cputime(size_t *optional_page_faults); +MDBX_INTERNAL uint64_t osal_16dot16_to_monotime(uint32_t seconds_16dot16); +MDBX_INTERNAL uint32_t osal_monotime_to_16dot16(uint64_t monotime); -MDBX_MAYBE_UNUSED static inline uint32_t -osal_monotime_to_16dot16_noUnderflow(uint64_t monotime) { +MDBX_MAYBE_UNUSED static inline uint32_t osal_monotime_to_16dot16_noUnderflow(uint64_t monotime) { uint32_t seconds_16dot16 = osal_monotime_to_16dot16(monotime); return seconds_16dot16 ? seconds_16dot16 : /* fix underflow */ (monotime > 0); } -MDBX_INTERNAL_FUNC bin128_t osal_bootid(void); /*----------------------------------------------------------------------------*/ -/* lck stuff */ - -/// \brief Initialization of synchronization primitives linked with MDBX_env -/// instance both in LCK-file and within the current process. -/// \param -/// global_uniqueness_flag = true - denotes that there are no other processes -/// working with DB and LCK-file. Thus the function MUST initialize -/// shared synchronization objects in memory-mapped LCK-file. -/// global_uniqueness_flag = false - denotes that at least one process is -/// already working with DB and LCK-file, including the case when DB -/// has already been opened in the current process. Thus the function -/// MUST NOT initialize shared synchronization objects in memory-mapped -/// LCK-file that are already in use. -/// \return Error code or zero on success. -MDBX_INTERNAL_FUNC int osal_lck_init(MDBX_env *env, - MDBX_env *inprocess_neighbor, - int global_uniqueness_flag); - -/// \brief Disconnects from shared interprocess objects and destructs -/// synchronization objects linked with MDBX_env instance -/// within the current process. -/// \param -/// inprocess_neighbor = NULL - if the current process does not have other -/// instances of MDBX_env linked with the DB being closed. -/// Thus the function MUST check for other processes working with DB or -/// LCK-file, and keep or destroy shared synchronization objects in -/// memory-mapped LCK-file depending on the result. -/// inprocess_neighbor = not-NULL - pointer to another instance of MDBX_env -/// (anyone of there is several) working with DB or LCK-file within the -/// current process. Thus the function MUST NOT try to acquire exclusive -/// lock and/or try to destruct shared synchronization objects linked with -/// DB or LCK-file. Moreover, the implementation MUST ensure correct work -/// of other instances of MDBX_env within the current process, e.g. -/// restore POSIX-fcntl locks after the closing of file descriptors. -/// \return Error code (MDBX_PANIC) or zero on success. -MDBX_INTERNAL_FUNC int osal_lck_destroy(MDBX_env *env, - MDBX_env *inprocess_neighbor); - -/// \brief Connects to shared interprocess locking objects and tries to acquire -/// the maximum lock level (shared if exclusive is not available) -/// Depending on implementation or/and platform (Windows) this function may -/// acquire the non-OS super-level lock (e.g. for shared synchronization -/// objects initialization), which will be downgraded to OS-exclusive or -/// shared via explicit calling of osal_lck_downgrade(). -/// \return -/// MDBX_RESULT_TRUE (-1) - if an exclusive lock was acquired and thus -/// the current process is the first and only after the last use of DB. -/// MDBX_RESULT_FALSE (0) - if a shared lock was acquired and thus -/// DB has already been opened and now is used by other processes. -/// Otherwise (not 0 and not -1) - error code. -MDBX_INTERNAL_FUNC int osal_lck_seize(MDBX_env *env); - -/// \brief Downgrades the level of initially acquired lock to -/// operational level specified by argument. The reason for such downgrade: -/// - unblocking of other processes that are waiting for access, i.e. -/// if (env->me_flags & MDBX_EXCLUSIVE) != 0, then other processes -/// should be made aware that access is unavailable rather than -/// wait for it. -/// - freeing locks that interfere file operation (especially for Windows) -/// (env->me_flags & MDBX_EXCLUSIVE) == 0 - downgrade to shared lock. -/// (env->me_flags & MDBX_EXCLUSIVE) != 0 - downgrade to exclusive -/// operational lock. -/// \return Error code or zero on success -MDBX_INTERNAL_FUNC int osal_lck_downgrade(MDBX_env *env); - -/// \brief Locks LCK-file or/and table of readers for (de)registering. -/// \return Error code or zero on success -MDBX_INTERNAL_FUNC int osal_rdt_lock(MDBX_env *env); - -/// \brief Unlocks LCK-file or/and table of readers after (de)registering. -MDBX_INTERNAL_FUNC void osal_rdt_unlock(MDBX_env *env); - -/// \brief Acquires lock for DB change (on writing transaction start) -/// Reading transactions will not be blocked. -/// Declared as LIBMDBX_API because it is used in mdbx_chk. -/// \return Error code or zero on success -LIBMDBX_API int mdbx_txn_lock(MDBX_env *env, bool dont_wait); - -/// \brief Releases lock once DB changes is made (after writing transaction -/// has finished). -/// Declared as LIBMDBX_API because it is used in mdbx_chk. -LIBMDBX_API void mdbx_txn_unlock(MDBX_env *env); - -/// \brief Sets alive-flag of reader presence (indicative lock) for PID of -/// the current process. The function does no more than needed for -/// the correct working of osal_rpid_check() in other processes. -/// \return Error code or zero on success -MDBX_INTERNAL_FUNC int osal_rpid_set(MDBX_env *env); - -/// \brief Resets alive-flag of reader presence (indicative lock) -/// for PID of the current process. The function does no more than needed -/// for the correct working of osal_rpid_check() in other processes. -/// \return Error code or zero on success -MDBX_INTERNAL_FUNC int osal_rpid_clear(MDBX_env *env); - -/// \brief Checks for reading process status with the given pid with help of -/// alive-flag of presence (indicative lock) or using another way. -/// \return -/// MDBX_RESULT_TRUE (-1) - if the reader process with the given PID is alive -/// and working with DB (indicative lock is present). -/// MDBX_RESULT_FALSE (0) - if the reader process with the given PID is absent -/// or not working with DB (indicative lock is not present). -/// Otherwise (not 0 and not -1) - error code. -MDBX_INTERNAL_FUNC int osal_rpid_check(MDBX_env *env, uint32_t pid); - -#if defined(_WIN32) || defined(_WIN64) - -MDBX_INTERNAL_FUNC int osal_mb2w(const char *const src, wchar_t **const pdst); - -typedef void(WINAPI *osal_srwlock_t_function)(osal_srwlock_t *); -MDBX_INTERNAL_VAR osal_srwlock_t_function osal_srwlock_Init, - osal_srwlock_AcquireShared, osal_srwlock_ReleaseShared, - osal_srwlock_AcquireExclusive, osal_srwlock_ReleaseExclusive; - -#if _WIN32_WINNT < 0x0600 /* prior to Windows Vista */ -typedef enum _FILE_INFO_BY_HANDLE_CLASS { - FileBasicInfo, - FileStandardInfo, - FileNameInfo, - FileRenameInfo, - FileDispositionInfo, - FileAllocationInfo, - FileEndOfFileInfo, - FileStreamInfo, - FileCompressionInfo, - FileAttributeTagInfo, - FileIdBothDirectoryInfo, - FileIdBothDirectoryRestartInfo, - FileIoPriorityHintInfo, - FileRemoteProtocolInfo, - MaximumFileInfoByHandleClass -} FILE_INFO_BY_HANDLE_CLASS, - *PFILE_INFO_BY_HANDLE_CLASS; - -typedef struct _FILE_END_OF_FILE_INFO { - LARGE_INTEGER EndOfFile; -} FILE_END_OF_FILE_INFO, *PFILE_END_OF_FILE_INFO; - -#define REMOTE_PROTOCOL_INFO_FLAG_LOOPBACK 0x00000001 -#define REMOTE_PROTOCOL_INFO_FLAG_OFFLINE 0x00000002 - -typedef struct _FILE_REMOTE_PROTOCOL_INFO { - USHORT StructureVersion; - USHORT StructureSize; - DWORD Protocol; - USHORT ProtocolMajorVersion; - USHORT ProtocolMinorVersion; - USHORT ProtocolRevision; - USHORT Reserved; - DWORD Flags; - struct { - DWORD Reserved[8]; - } GenericReserved; - struct { - DWORD Reserved[16]; - } ProtocolSpecificReserved; -} FILE_REMOTE_PROTOCOL_INFO, *PFILE_REMOTE_PROTOCOL_INFO; - -#endif /* _WIN32_WINNT < 0x0600 (prior to Windows Vista) */ - -typedef BOOL(WINAPI *MDBX_GetFileInformationByHandleEx)( - _In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass, - _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize); -MDBX_INTERNAL_VAR MDBX_GetFileInformationByHandleEx - mdbx_GetFileInformationByHandleEx; - -typedef BOOL(WINAPI *MDBX_GetVolumeInformationByHandleW)( - _In_ HANDLE hFile, _Out_opt_ LPWSTR lpVolumeNameBuffer, - _In_ DWORD nVolumeNameSize, _Out_opt_ LPDWORD lpVolumeSerialNumber, - _Out_opt_ LPDWORD lpMaximumComponentLength, - _Out_opt_ LPDWORD lpFileSystemFlags, - _Out_opt_ LPWSTR lpFileSystemNameBuffer, _In_ DWORD nFileSystemNameSize); -MDBX_INTERNAL_VAR MDBX_GetVolumeInformationByHandleW - mdbx_GetVolumeInformationByHandleW; - -typedef DWORD(WINAPI *MDBX_GetFinalPathNameByHandleW)(_In_ HANDLE hFile, - _Out_ LPWSTR lpszFilePath, - _In_ DWORD cchFilePath, - _In_ DWORD dwFlags); -MDBX_INTERNAL_VAR MDBX_GetFinalPathNameByHandleW mdbx_GetFinalPathNameByHandleW; - -typedef BOOL(WINAPI *MDBX_SetFileInformationByHandle)( - _In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass, - _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize); -MDBX_INTERNAL_VAR MDBX_SetFileInformationByHandle - mdbx_SetFileInformationByHandle; - -typedef NTSTATUS(NTAPI *MDBX_NtFsControlFile)( - IN HANDLE FileHandle, IN OUT HANDLE Event, - IN OUT PVOID /* PIO_APC_ROUTINE */ ApcRoutine, IN OUT PVOID ApcContext, - OUT PIO_STATUS_BLOCK IoStatusBlock, IN ULONG FsControlCode, - IN OUT PVOID InputBuffer, IN ULONG InputBufferLength, - OUT OPTIONAL PVOID OutputBuffer, IN ULONG OutputBufferLength); -MDBX_INTERNAL_VAR MDBX_NtFsControlFile mdbx_NtFsControlFile; - -typedef uint64_t(WINAPI *MDBX_GetTickCount64)(void); -MDBX_INTERNAL_VAR MDBX_GetTickCount64 mdbx_GetTickCount64; - -#if !defined(_WIN32_WINNT_WIN8) || _WIN32_WINNT < _WIN32_WINNT_WIN8 -typedef struct _WIN32_MEMORY_RANGE_ENTRY { - PVOID VirtualAddress; - SIZE_T NumberOfBytes; -} WIN32_MEMORY_RANGE_ENTRY, *PWIN32_MEMORY_RANGE_ENTRY; -#endif /* Windows 8.x */ - -typedef BOOL(WINAPI *MDBX_PrefetchVirtualMemory)( - HANDLE hProcess, ULONG_PTR NumberOfEntries, - PWIN32_MEMORY_RANGE_ENTRY VirtualAddresses, ULONG Flags); -MDBX_INTERNAL_VAR MDBX_PrefetchVirtualMemory mdbx_PrefetchVirtualMemory; - -typedef enum _SECTION_INHERIT { ViewShare = 1, ViewUnmap = 2 } SECTION_INHERIT; - -typedef NTSTATUS(NTAPI *MDBX_NtExtendSection)(IN HANDLE SectionHandle, - IN PLARGE_INTEGER NewSectionSize); -MDBX_INTERNAL_VAR MDBX_NtExtendSection mdbx_NtExtendSection; - -static __inline bool mdbx_RunningUnderWine(void) { - return !mdbx_NtExtendSection; -} -typedef LSTATUS(WINAPI *MDBX_RegGetValueA)(HKEY hkey, LPCSTR lpSubKey, - LPCSTR lpValue, DWORD dwFlags, - LPDWORD pdwType, PVOID pvData, - LPDWORD pcbData); -MDBX_INTERNAL_VAR MDBX_RegGetValueA mdbx_RegGetValueA; - -NTSYSAPI ULONG RtlRandomEx(PULONG Seed); - -typedef BOOL(WINAPI *MDBX_SetFileIoOverlappedRange)(HANDLE FileHandle, - PUCHAR OverlappedRangeStart, - ULONG Length); -MDBX_INTERNAL_VAR MDBX_SetFileIoOverlappedRange mdbx_SetFileIoOverlappedRange; +MDBX_INTERNAL void osal_ctor(void); +MDBX_INTERNAL void osal_dtor(void); +#if defined(_WIN32) || defined(_WIN64) +MDBX_INTERNAL int osal_mb2w(const char *const src, wchar_t **const pdst); #endif /* Windows */ -#endif /* !__cplusplus */ +typedef union bin128 { + __anonymous_struct_extension__ struct { + uint64_t x, y; + }; + __anonymous_struct_extension__ struct { + uint32_t a, b, c, d; + }; +} bin128_t; + +MDBX_INTERNAL bin128_t osal_guid(const MDBX_env *); /*----------------------------------------------------------------------------*/ -MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static __always_inline uint64_t -osal_bswap64(uint64_t v) { -#if __GNUC_PREREQ(4, 4) || __CLANG_PREREQ(4, 0) || \ - __has_builtin(__builtin_bswap64) +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint64_t osal_bswap64(uint64_t v) { +#if __GNUC_PREREQ(4, 4) || __CLANG_PREREQ(4, 0) || __has_builtin(__builtin_bswap64) return __builtin_bswap64(v); #elif defined(_MSC_VER) && !defined(__clang__) return _byteswap_uint64(v); @@ -1918,19 +1582,14 @@ osal_bswap64(uint64_t v) { #elif defined(bswap_64) return bswap_64(v); #else - return v << 56 | v >> 56 | ((v << 40) & UINT64_C(0x00ff000000000000)) | - ((v << 24) & UINT64_C(0x0000ff0000000000)) | - ((v << 8) & UINT64_C(0x000000ff00000000)) | - ((v >> 8) & UINT64_C(0x00000000ff000000)) | - ((v >> 24) & UINT64_C(0x0000000000ff0000)) | - ((v >> 40) & UINT64_C(0x000000000000ff00)); + return v << 56 | v >> 56 | ((v << 40) & UINT64_C(0x00ff000000000000)) | ((v << 24) & UINT64_C(0x0000ff0000000000)) | + ((v << 8) & UINT64_C(0x000000ff00000000)) | ((v >> 8) & UINT64_C(0x00000000ff000000)) | + ((v >> 24) & UINT64_C(0x0000000000ff0000)) | ((v >> 40) & UINT64_C(0x000000000000ff00)); #endif } -MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static __always_inline uint32_t -osal_bswap32(uint32_t v) { -#if __GNUC_PREREQ(4, 4) || __CLANG_PREREQ(4, 0) || \ - __has_builtin(__builtin_bswap32) +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint32_t osal_bswap32(uint32_t v) { +#if __GNUC_PREREQ(4, 4) || __CLANG_PREREQ(4, 0) || __has_builtin(__builtin_bswap32) return __builtin_bswap32(v); #elif defined(_MSC_VER) && !defined(__clang__) return _byteswap_ulong(v); @@ -1939,50 +1598,14 @@ osal_bswap32(uint32_t v) { #elif defined(bswap_32) return bswap_32(v); #else - return v << 24 | v >> 24 | ((v << 8) & UINT32_C(0x00ff0000)) | - ((v >> 8) & UINT32_C(0x0000ff00)); + return v << 24 | v >> 24 | ((v << 8) & UINT32_C(0x00ff0000)) | ((v >> 8) & UINT32_C(0x0000ff00)); #endif } -/*----------------------------------------------------------------------------*/ - -#if defined(_MSC_VER) && _MSC_VER >= 1900 -/* LY: MSVC 2015/2017/2019 has buggy/inconsistent PRIuPTR/PRIxPTR macros - * for internal format-args checker. */ -#undef PRIuPTR -#undef PRIiPTR -#undef PRIdPTR -#undef PRIxPTR -#define PRIuPTR "Iu" -#define PRIiPTR "Ii" -#define PRIdPTR "Id" -#define PRIxPTR "Ix" -#define PRIuSIZE "zu" -#define PRIiSIZE "zi" -#define PRIdSIZE "zd" -#define PRIxSIZE "zx" -#endif /* fix PRI*PTR for _MSC_VER */ - -#ifndef PRIuSIZE -#define PRIuSIZE PRIuPTR -#define PRIiSIZE PRIiPTR -#define PRIdSIZE PRIdPTR -#define PRIxSIZE PRIxPTR -#endif /* PRI*SIZE macros for MSVC */ - -#ifdef _MSC_VER -#pragma warning(pop) -#endif - -#define mdbx_sourcery_anchor XCONCAT(mdbx_sourcery_, MDBX_BUILD_SOURCERY) -#if defined(xMDBX_TOOLS) -extern LIBMDBX_API const char *const mdbx_sourcery_anchor; -#endif - /******************************************************************************* - ******************************************************************************* ******************************************************************************* * + * BUILD TIME * * #### ##### ##### # #### # # #### * # # # # # # # # ## # # @@ -2003,23 +1626,15 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Using fsync() with chance of data lost on power failure */ #define MDBX_OSX_WANNA_SPEED 1 -#ifndef MDBX_OSX_SPEED_INSTEADOF_DURABILITY +#ifndef MDBX_APPLE_SPEED_INSTEADOF_DURABILITY /** Choices \ref MDBX_OSX_WANNA_DURABILITY or \ref MDBX_OSX_WANNA_SPEED * for OSX & iOS */ -#define MDBX_OSX_SPEED_INSTEADOF_DURABILITY MDBX_OSX_WANNA_DURABILITY -#endif /* MDBX_OSX_SPEED_INSTEADOF_DURABILITY */ - -/** Controls using of POSIX' madvise() and/or similar hints. */ -#ifndef MDBX_ENABLE_MADVISE -#define MDBX_ENABLE_MADVISE 1 -#elif !(MDBX_ENABLE_MADVISE == 0 || MDBX_ENABLE_MADVISE == 1) -#error MDBX_ENABLE_MADVISE must be defined as 0 or 1 -#endif /* MDBX_ENABLE_MADVISE */ +#define MDBX_APPLE_SPEED_INSTEADOF_DURABILITY MDBX_OSX_WANNA_DURABILITY +#endif /* MDBX_APPLE_SPEED_INSTEADOF_DURABILITY */ /** Controls checking PID against reuse DB environment after the fork() */ #ifndef MDBX_ENV_CHECKPID -#if (defined(MADV_DONTFORK) && MDBX_ENABLE_MADVISE) || defined(_WIN32) || \ - defined(_WIN64) +#if defined(MADV_DONTFORK) || defined(_WIN32) || defined(_WIN64) /* PID check could be omitted: * - on Linux when madvise(MADV_DONTFORK) is available, i.e. after the fork() * mapped pages will not be available for child process. @@ -2048,8 +1663,7 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Does a system have battery-backed Real-Time Clock or just a fake. */ #ifndef MDBX_TRUST_RTC -#if defined(__linux__) || defined(__gnu_linux__) || defined(__NetBSD__) || \ - defined(__OpenBSD__) +#if defined(__linux__) || defined(__gnu_linux__) || defined(__NetBSD__) || defined(__OpenBSD__) #define MDBX_TRUST_RTC 0 /* a lot of embedded systems have a fake RTC */ #else #define MDBX_TRUST_RTC 1 @@ -2084,24 +1698,21 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Controls using Unix' mincore() to determine whether DB-pages * are resident in memory. */ -#ifndef MDBX_ENABLE_MINCORE +#ifndef MDBX_USE_MINCORE #if defined(MINCORE_INCORE) || !(defined(_WIN32) || defined(_WIN64)) -#define MDBX_ENABLE_MINCORE 1 +#define MDBX_USE_MINCORE 1 #else -#define MDBX_ENABLE_MINCORE 0 +#define MDBX_USE_MINCORE 0 #endif -#elif !(MDBX_ENABLE_MINCORE == 0 || MDBX_ENABLE_MINCORE == 1) -#error MDBX_ENABLE_MINCORE must be defined as 0 or 1 -#endif /* MDBX_ENABLE_MINCORE */ +#define MDBX_USE_MINCORE_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_USE_MINCORE) +#elif !(MDBX_USE_MINCORE == 0 || MDBX_USE_MINCORE == 1) +#error MDBX_USE_MINCORE must be defined as 0 or 1 +#endif /* MDBX_USE_MINCORE */ /** Enables chunking long list of retired pages during huge transactions commit * to avoid use sequences of pages. */ #ifndef MDBX_ENABLE_BIGFOOT -#if MDBX_WORDBITS >= 64 || defined(DOXYGEN) #define MDBX_ENABLE_BIGFOOT 1 -#else -#define MDBX_ENABLE_BIGFOOT 0 -#endif #elif !(MDBX_ENABLE_BIGFOOT == 0 || MDBX_ENABLE_BIGFOOT == 1) #error MDBX_ENABLE_BIGFOOT must be defined as 0 or 1 #endif /* MDBX_ENABLE_BIGFOOT */ @@ -2116,25 +1727,27 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #ifndef MDBX_PNL_PREALLOC_FOR_RADIXSORT #define MDBX_PNL_PREALLOC_FOR_RADIXSORT 1 -#elif !(MDBX_PNL_PREALLOC_FOR_RADIXSORT == 0 || \ - MDBX_PNL_PREALLOC_FOR_RADIXSORT == 1) +#elif !(MDBX_PNL_PREALLOC_FOR_RADIXSORT == 0 || MDBX_PNL_PREALLOC_FOR_RADIXSORT == 1) #error MDBX_PNL_PREALLOC_FOR_RADIXSORT must be defined as 0 or 1 #endif /* MDBX_PNL_PREALLOC_FOR_RADIXSORT */ #ifndef MDBX_DPL_PREALLOC_FOR_RADIXSORT #define MDBX_DPL_PREALLOC_FOR_RADIXSORT 1 -#elif !(MDBX_DPL_PREALLOC_FOR_RADIXSORT == 0 || \ - MDBX_DPL_PREALLOC_FOR_RADIXSORT == 1) +#elif !(MDBX_DPL_PREALLOC_FOR_RADIXSORT == 0 || MDBX_DPL_PREALLOC_FOR_RADIXSORT == 1) #error MDBX_DPL_PREALLOC_FOR_RADIXSORT must be defined as 0 or 1 #endif /* MDBX_DPL_PREALLOC_FOR_RADIXSORT */ -/** Controls dirty pages tracking, spilling and persisting in MDBX_WRITEMAP - * mode. 0/OFF = Don't track dirty pages at all, don't spill ones, and use - * msync() to persist data. This is by-default on Linux and other systems where - * kernel provides properly LRU tracking and effective flushing on-demand. 1/ON - * = Tracking of dirty pages but with LRU labels for spilling and explicit - * persist ones by write(). This may be reasonable for systems which low - * performance of msync() and/or LRU tracking. */ +/** Controls dirty pages tracking, spilling and persisting in `MDBX_WRITEMAP` + * mode, i.e. disables in-memory database updating with consequent + * flush-to-disk/msync syscall. + * + * 0/OFF = Don't track dirty pages at all, don't spill ones, and use msync() to + * persist data. This is by-default on Linux and other systems where kernel + * provides properly LRU tracking and effective flushing on-demand. + * + * 1/ON = Tracking of dirty pages but with LRU labels for spilling and explicit + * persist ones by write(). This may be reasonable for goofy systems (Windows) + * which low performance of msync() and/or zany LRU tracking. */ #ifndef MDBX_AVOID_MSYNC #if defined(_WIN32) || defined(_WIN64) #define MDBX_AVOID_MSYNC 1 @@ -2145,6 +1758,22 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #error MDBX_AVOID_MSYNC must be defined as 0 or 1 #endif /* MDBX_AVOID_MSYNC */ +/** Управляет механизмом поддержки разреженных наборов DBI-хендлов для снижения + * накладных расходов при запуске и обработке транзакций. */ +#ifndef MDBX_ENABLE_DBI_SPARSE +#define MDBX_ENABLE_DBI_SPARSE 1 +#elif !(MDBX_ENABLE_DBI_SPARSE == 0 || MDBX_ENABLE_DBI_SPARSE == 1) +#error MDBX_ENABLE_DBI_SPARSE must be defined as 0 or 1 +#endif /* MDBX_ENABLE_DBI_SPARSE */ + +/** Управляет механизмом отложенного освобождения и поддержки пути быстрого + * открытия DBI-хендлов без захвата блокировок. */ +#ifndef MDBX_ENABLE_DBI_LOCKFREE +#define MDBX_ENABLE_DBI_LOCKFREE 1 +#elif !(MDBX_ENABLE_DBI_LOCKFREE == 0 || MDBX_ENABLE_DBI_LOCKFREE == 1) +#error MDBX_ENABLE_DBI_LOCKFREE must be defined as 0 or 1 +#endif /* MDBX_ENABLE_DBI_LOCKFREE */ + /** Controls sort order of internal page number lists. * This mostly experimental/advanced option with not for regular MDBX users. * \warning The database format depend on this option and libmdbx built with @@ -2157,7 +1786,11 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Avoid dependence from MSVC CRT and use ntdll.dll instead. */ #ifndef MDBX_WITHOUT_MSVC_CRT +#if defined(MDBX_BUILD_CXX) && !MDBX_BUILD_CXX #define MDBX_WITHOUT_MSVC_CRT 1 +#else +#define MDBX_WITHOUT_MSVC_CRT 0 +#endif #elif !(MDBX_WITHOUT_MSVC_CRT == 0 || MDBX_WITHOUT_MSVC_CRT == 1) #error MDBX_WITHOUT_MSVC_CRT must be defined as 0 or 1 #endif /* MDBX_WITHOUT_MSVC_CRT */ @@ -2165,12 +1798,11 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Size of buffer used during copying a environment/database file. */ #ifndef MDBX_ENVCOPY_WRITEBUF #define MDBX_ENVCOPY_WRITEBUF 1048576u -#elif MDBX_ENVCOPY_WRITEBUF < 65536u || MDBX_ENVCOPY_WRITEBUF > 1073741824u || \ - MDBX_ENVCOPY_WRITEBUF % 65536u +#elif MDBX_ENVCOPY_WRITEBUF < 65536u || MDBX_ENVCOPY_WRITEBUF > 1073741824u || MDBX_ENVCOPY_WRITEBUF % 65536u #error MDBX_ENVCOPY_WRITEBUF must be defined in range 65536..1073741824 and be multiple of 65536 #endif /* MDBX_ENVCOPY_WRITEBUF */ -/** Forces assertion checking */ +/** Forces assertion checking. */ #ifndef MDBX_FORCE_ASSERTIONS #define MDBX_FORCE_ASSERTIONS 0 #elif !(MDBX_FORCE_ASSERTIONS == 0 || MDBX_FORCE_ASSERTIONS == 1) @@ -2185,15 +1817,14 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #else #define MDBX_ASSUME_MALLOC_OVERHEAD (sizeof(void *) * 2u) #endif -#elif MDBX_ASSUME_MALLOC_OVERHEAD < 0 || MDBX_ASSUME_MALLOC_OVERHEAD > 64 || \ - MDBX_ASSUME_MALLOC_OVERHEAD % 4 +#elif MDBX_ASSUME_MALLOC_OVERHEAD < 0 || MDBX_ASSUME_MALLOC_OVERHEAD > 64 || MDBX_ASSUME_MALLOC_OVERHEAD % 4 #error MDBX_ASSUME_MALLOC_OVERHEAD must be defined in range 0..64 and be multiple of 4 #endif /* MDBX_ASSUME_MALLOC_OVERHEAD */ /** If defined then enables integration with Valgrind, * a memory analyzing tool. */ -#ifndef MDBX_USE_VALGRIND -#endif /* MDBX_USE_VALGRIND */ +#ifndef ENABLE_MEMCHECK +#endif /* ENABLE_MEMCHECK */ /** If defined then enables use C11 atomics, * otherwise detects ones availability automatically. */ @@ -2213,18 +1844,24 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #define MDBX_HAVE_BUILTIN_CPU_SUPPORTS 0 #elif defined(__e2k__) #define MDBX_HAVE_BUILTIN_CPU_SUPPORTS 0 -#elif __has_builtin(__builtin_cpu_supports) || \ - defined(__BUILTIN_CPU_SUPPORTS__) || \ +#elif __has_builtin(__builtin_cpu_supports) || defined(__BUILTIN_CPU_SUPPORTS__) || \ (defined(__ia32__) && __GNUC_PREREQ(4, 8) && __GLIBC_PREREQ(2, 23)) #define MDBX_HAVE_BUILTIN_CPU_SUPPORTS 1 #else #define MDBX_HAVE_BUILTIN_CPU_SUPPORTS 0 #endif -#elif !(MDBX_HAVE_BUILTIN_CPU_SUPPORTS == 0 || \ - MDBX_HAVE_BUILTIN_CPU_SUPPORTS == 1) +#elif !(MDBX_HAVE_BUILTIN_CPU_SUPPORTS == 0 || MDBX_HAVE_BUILTIN_CPU_SUPPORTS == 1) #error MDBX_HAVE_BUILTIN_CPU_SUPPORTS must be defined as 0 or 1 #endif /* MDBX_HAVE_BUILTIN_CPU_SUPPORTS */ +/** if enabled then instead of the returned error `MDBX_REMOTE`, only a warning is issued, when + * the database being opened in non-read-only mode is located in a file system exported via NFS. */ +#ifndef MDBX_ENABLE_NON_READONLY_EXPORT +#define MDBX_ENABLE_NON_READONLY_EXPORT 0 +#elif !(MDBX_ENABLE_NON_READONLY_EXPORT == 0 || MDBX_ENABLE_NON_READONLY_EXPORT == 1) +#error MDBX_ENABLE_NON_READONLY_EXPORT must be defined as 0 or 1 +#endif /* MDBX_ENABLE_NON_READONLY_EXPORT */ + //------------------------------------------------------------------------------ /** Win32 File Locking API for \ref MDBX_LOCKING */ @@ -2242,27 +1879,20 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** POSIX-2008 Robust Mutexes for \ref MDBX_LOCKING */ #define MDBX_LOCKING_POSIX2008 2008 -/** BeOS Benaphores, aka Futexes for \ref MDBX_LOCKING */ -#define MDBX_LOCKING_BENAPHORE 1995 - /** Advanced: Choices the locking implementation (autodetection by default). */ #if defined(_WIN32) || defined(_WIN64) #define MDBX_LOCKING MDBX_LOCKING_WIN32FILES #else #ifndef MDBX_LOCKING -#if defined(_POSIX_THREAD_PROCESS_SHARED) && \ - _POSIX_THREAD_PROCESS_SHARED >= 200112L && !defined(__FreeBSD__) +#if defined(_POSIX_THREAD_PROCESS_SHARED) && _POSIX_THREAD_PROCESS_SHARED >= 200112L && !defined(__FreeBSD__) /* Some platforms define the EOWNERDEAD error code even though they * don't support Robust Mutexes. If doubt compile with -MDBX_LOCKING=2001. */ -#if defined(EOWNERDEAD) && _POSIX_THREAD_PROCESS_SHARED >= 200809L && \ - ((defined(_POSIX_THREAD_ROBUST_PRIO_INHERIT) && \ - _POSIX_THREAD_ROBUST_PRIO_INHERIT > 0) || \ - (defined(_POSIX_THREAD_ROBUST_PRIO_PROTECT) && \ - _POSIX_THREAD_ROBUST_PRIO_PROTECT > 0) || \ - defined(PTHREAD_MUTEX_ROBUST) || defined(PTHREAD_MUTEX_ROBUST_NP)) && \ - (!defined(__GLIBC__) || \ - __GLIBC_PREREQ(2, 10) /* troubles with Robust mutexes before 2.10 */) +#if defined(EOWNERDEAD) && _POSIX_THREAD_PROCESS_SHARED >= 200809L && \ + ((defined(_POSIX_THREAD_ROBUST_PRIO_INHERIT) && _POSIX_THREAD_ROBUST_PRIO_INHERIT > 0) || \ + (defined(_POSIX_THREAD_ROBUST_PRIO_PROTECT) && _POSIX_THREAD_ROBUST_PRIO_PROTECT > 0) || \ + defined(PTHREAD_MUTEX_ROBUST) || defined(PTHREAD_MUTEX_ROBUST_NP)) && \ + (!defined(__GLIBC__) || __GLIBC_PREREQ(2, 10) /* troubles with Robust mutexes before 2.10 */) #define MDBX_LOCKING MDBX_LOCKING_POSIX2008 #else #define MDBX_LOCKING MDBX_LOCKING_POSIX2001 @@ -2280,12 +1910,9 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Advanced: Using POSIX OFD-locks (autodetection by default). */ #ifndef MDBX_USE_OFDLOCKS -#if ((defined(F_OFD_SETLK) && defined(F_OFD_SETLKW) && \ - defined(F_OFD_GETLK)) || \ - (defined(F_OFD_SETLK64) && defined(F_OFD_SETLKW64) && \ - defined(F_OFD_GETLK64))) && \ - !defined(MDBX_SAFE4QEMU) && \ - !defined(__sun) /* OFD-lock are broken on Solaris */ +#if ((defined(F_OFD_SETLK) && defined(F_OFD_SETLKW) && defined(F_OFD_GETLK)) || \ + (defined(F_OFD_SETLK64) && defined(F_OFD_SETLKW64) && defined(F_OFD_GETLK64))) && \ + !defined(MDBX_SAFE4QEMU) && !defined(__sun) /* OFD-lock are broken on Solaris */ #define MDBX_USE_OFDLOCKS 1 #else #define MDBX_USE_OFDLOCKS 0 @@ -2299,8 +1926,7 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Advanced: Using sendfile() syscall (autodetection by default). */ #ifndef MDBX_USE_SENDFILE -#if ((defined(__linux__) || defined(__gnu_linux__)) && \ - !defined(__ANDROID_API__)) || \ +#if ((defined(__linux__) || defined(__gnu_linux__)) && !defined(__ANDROID_API__)) || \ (defined(__ANDROID_API__) && __ANDROID_API__ >= 21) #define MDBX_USE_SENDFILE 1 #else @@ -2321,30 +1947,15 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #error MDBX_USE_COPYFILERANGE must be defined as 0 or 1 #endif /* MDBX_USE_COPYFILERANGE */ -/** Advanced: Using sync_file_range() syscall (autodetection by default). */ -#ifndef MDBX_USE_SYNCFILERANGE -#if ((defined(__linux__) || defined(__gnu_linux__)) && \ - defined(SYNC_FILE_RANGE_WRITE) && !defined(__ANDROID_API__)) || \ - (defined(__ANDROID_API__) && __ANDROID_API__ >= 26) -#define MDBX_USE_SYNCFILERANGE 1 -#else -#define MDBX_USE_SYNCFILERANGE 0 -#endif -#elif !(MDBX_USE_SYNCFILERANGE == 0 || MDBX_USE_SYNCFILERANGE == 1) -#error MDBX_USE_SYNCFILERANGE must be defined as 0 or 1 -#endif /* MDBX_USE_SYNCFILERANGE */ - //------------------------------------------------------------------------------ #ifndef MDBX_CPU_WRITEBACK_INCOHERENT -#if defined(__ia32__) || defined(__e2k__) || defined(__hppa) || \ - defined(__hppa__) || defined(DOXYGEN) +#if defined(__ia32__) || defined(__e2k__) || defined(__hppa) || defined(__hppa__) || defined(DOXYGEN) #define MDBX_CPU_WRITEBACK_INCOHERENT 0 #else #define MDBX_CPU_WRITEBACK_INCOHERENT 1 #endif -#elif !(MDBX_CPU_WRITEBACK_INCOHERENT == 0 || \ - MDBX_CPU_WRITEBACK_INCOHERENT == 1) +#elif !(MDBX_CPU_WRITEBACK_INCOHERENT == 0 || MDBX_CPU_WRITEBACK_INCOHERENT == 1) #error MDBX_CPU_WRITEBACK_INCOHERENT must be defined as 0 or 1 #endif /* MDBX_CPU_WRITEBACK_INCOHERENT */ @@ -2354,35 +1965,35 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #else #define MDBX_MMAP_INCOHERENT_FILE_WRITE 0 #endif -#elif !(MDBX_MMAP_INCOHERENT_FILE_WRITE == 0 || \ - MDBX_MMAP_INCOHERENT_FILE_WRITE == 1) +#elif !(MDBX_MMAP_INCOHERENT_FILE_WRITE == 0 || MDBX_MMAP_INCOHERENT_FILE_WRITE == 1) #error MDBX_MMAP_INCOHERENT_FILE_WRITE must be defined as 0 or 1 #endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */ #ifndef MDBX_MMAP_INCOHERENT_CPU_CACHE -#if defined(__mips) || defined(__mips__) || defined(__mips64) || \ - defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \ - defined(__MWERKS__) || defined(__sgi) +#if defined(__mips) || defined(__mips__) || defined(__mips64) || defined(__mips64__) || defined(_M_MRX000) || \ + defined(_MIPS_) || defined(__MWERKS__) || defined(__sgi) /* MIPS has cache coherency issues. */ #define MDBX_MMAP_INCOHERENT_CPU_CACHE 1 #else /* LY: assume no relevant mmap/dcache issues. */ #define MDBX_MMAP_INCOHERENT_CPU_CACHE 0 #endif -#elif !(MDBX_MMAP_INCOHERENT_CPU_CACHE == 0 || \ - MDBX_MMAP_INCOHERENT_CPU_CACHE == 1) +#elif !(MDBX_MMAP_INCOHERENT_CPU_CACHE == 0 || MDBX_MMAP_INCOHERENT_CPU_CACHE == 1) #error MDBX_MMAP_INCOHERENT_CPU_CACHE must be defined as 0 or 1 #endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */ -#ifndef MDBX_MMAP_USE_MS_ASYNC -#if MDBX_MMAP_INCOHERENT_FILE_WRITE || MDBX_MMAP_INCOHERENT_CPU_CACHE -#define MDBX_MMAP_USE_MS_ASYNC 1 +/** Assume system needs explicit syscall to sync/flush/write modified mapped + * memory. */ +#ifndef MDBX_MMAP_NEEDS_JOLT +#if MDBX_MMAP_INCOHERENT_FILE_WRITE || MDBX_MMAP_INCOHERENT_CPU_CACHE || !(defined(__linux__) || defined(__gnu_linux__)) +#define MDBX_MMAP_NEEDS_JOLT 1 #else -#define MDBX_MMAP_USE_MS_ASYNC 0 +#define MDBX_MMAP_NEEDS_JOLT 0 #endif -#elif !(MDBX_MMAP_USE_MS_ASYNC == 0 || MDBX_MMAP_USE_MS_ASYNC == 1) -#error MDBX_MMAP_USE_MS_ASYNC must be defined as 0 or 1 -#endif /* MDBX_MMAP_USE_MS_ASYNC */ +#define MDBX_MMAP_NEEDS_JOLT_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_MMAP_NEEDS_JOLT) +#elif !(MDBX_MMAP_NEEDS_JOLT == 0 || MDBX_MMAP_NEEDS_JOLT == 1) +#error MDBX_MMAP_NEEDS_JOLT must be defined as 0 or 1 +#endif /* MDBX_MMAP_NEEDS_JOLT */ #ifndef MDBX_64BIT_ATOMIC #if MDBX_WORDBITS >= 64 || defined(DOXYGEN) @@ -2429,8 +2040,7 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #endif /* MDBX_64BIT_CAS */ #ifndef MDBX_UNALIGNED_OK -#if defined(__ALIGNED__) || defined(__SANITIZE_UNDEFINED__) || \ - defined(ENABLE_UBSAN) +#if defined(__ALIGNED__) || defined(__SANITIZE_UNDEFINED__) || defined(ENABLE_UBSAN) #define MDBX_UNALIGNED_OK 0 /* no unaligned access allowed */ #elif defined(__ARM_FEATURE_UNALIGNED) #define MDBX_UNALIGNED_OK 4 /* ok unaligned for 32-bit words */ @@ -2464,6 +2074,19 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #endif #endif /* MDBX_CACHELINE_SIZE */ +/* Max length of iov-vector passed to writev() call, used for auxilary writes */ +#ifndef MDBX_AUXILARY_IOV_MAX +#define MDBX_AUXILARY_IOV_MAX 64 +#endif +#if defined(IOV_MAX) && IOV_MAX < MDBX_AUXILARY_IOV_MAX +#undef MDBX_AUXILARY_IOV_MAX +#define MDBX_AUXILARY_IOV_MAX IOV_MAX +#endif /* MDBX_AUXILARY_IOV_MAX */ + +/* An extra/custom information provided during library build */ +#ifndef MDBX_BUILD_METADATA +#define MDBX_BUILD_METADATA "" +#endif /* MDBX_BUILD_METADATA */ /** @} end of build options */ /******************************************************************************* ******************************************************************************* @@ -2478,6 +2101,9 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #else #define MDBX_DEBUG 1 #endif +#endif +#if MDBX_DEBUG < 0 || MDBX_DEBUG > 2 +#error "The MDBX_DEBUG must be defined to 0, 1 or 2" #endif /* MDBX_DEBUG */ #else @@ -2497,169 +2123,58 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; * Also enables \ref MDBX_DBG_AUDIT if `MDBX_DEBUG >= 2`. * * \ingroup build_option */ -#define MDBX_DEBUG 0...7 +#define MDBX_DEBUG 0...2 /** Disables using of GNU libc extensions. */ #define MDBX_DISABLE_GNU_SOURCE 0 or 1 #endif /* DOXYGEN */ -/* Undefine the NDEBUG if debugging is enforced by MDBX_DEBUG */ -#if MDBX_DEBUG -#undef NDEBUG -#endif - -#ifndef __cplusplus -/*----------------------------------------------------------------------------*/ -/* Debug and Logging stuff */ - -#define MDBX_RUNTIME_FLAGS_INIT \ - ((MDBX_DEBUG) > 0) * MDBX_DBG_ASSERT + ((MDBX_DEBUG) > 1) * MDBX_DBG_AUDIT - -extern uint8_t runtime_flags; -extern uint8_t loglevel; -extern MDBX_debug_func *debug_logger; - -MDBX_MAYBE_UNUSED static __inline void jitter4testing(bool tiny) { -#if MDBX_DEBUG - if (MDBX_DBG_JITTER & runtime_flags) - osal_jitter(tiny); -#else - (void)tiny; -#endif -} - -MDBX_INTERNAL_FUNC void MDBX_PRINTF_ARGS(4, 5) - debug_log(int level, const char *function, int line, const char *fmt, ...) - MDBX_PRINTF_ARGS(4, 5); -MDBX_INTERNAL_FUNC void debug_log_va(int level, const char *function, int line, - const char *fmt, va_list args); +#ifndef MDBX_64BIT_ATOMIC +#error "The MDBX_64BIT_ATOMIC must be defined before" +#endif /* MDBX_64BIT_ATOMIC */ -#if MDBX_DEBUG -#define LOG_ENABLED(msg) unlikely(msg <= loglevel) -#define AUDIT_ENABLED() unlikely((runtime_flags & MDBX_DBG_AUDIT)) -#else /* MDBX_DEBUG */ -#define LOG_ENABLED(msg) (msg < MDBX_LOG_VERBOSE && msg <= loglevel) -#define AUDIT_ENABLED() (0) -#endif /* MDBX_DEBUG */ +#ifndef MDBX_64BIT_CAS +#error "The MDBX_64BIT_CAS must be defined before" +#endif /* MDBX_64BIT_CAS */ -#if MDBX_FORCE_ASSERTIONS -#define ASSERT_ENABLED() (1) -#elif MDBX_DEBUG -#define ASSERT_ENABLED() likely((runtime_flags & MDBX_DBG_ASSERT)) +#if defined(__cplusplus) && !defined(__STDC_NO_ATOMICS__) && __has_include() +#include +#define MDBX_HAVE_C11ATOMICS +#elif !defined(__cplusplus) && (__STDC_VERSION__ >= 201112L || __has_extension(c_atomic)) && \ + !defined(__STDC_NO_ATOMICS__) && \ + (__GNUC_PREREQ(4, 9) || __CLANG_PREREQ(3, 8) || !(defined(__GNUC__) || defined(__clang__))) +#include +#define MDBX_HAVE_C11ATOMICS +#elif defined(__GNUC__) || defined(__clang__) +#elif defined(_MSC_VER) +#pragma warning(disable : 4163) /* 'xyz': not available as an intrinsic */ +#pragma warning(disable : 4133) /* 'function': incompatible types - from \ + 'size_t' to 'LONGLONG' */ +#pragma warning(disable : 4244) /* 'return': conversion from 'LONGLONG' to \ + 'std::size_t', possible loss of data */ +#pragma warning(disable : 4267) /* 'function': conversion from 'size_t' to \ + 'long', possible loss of data */ +#pragma intrinsic(_InterlockedExchangeAdd, _InterlockedCompareExchange) +#pragma intrinsic(_InterlockedExchangeAdd64, _InterlockedCompareExchange64) +#elif defined(__APPLE__) +#include #else -#define ASSERT_ENABLED() (0) -#endif /* assertions */ - -#define DEBUG_EXTRA(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_EXTRA)) \ - debug_log(MDBX_LOG_EXTRA, __func__, __LINE__, fmt, __VA_ARGS__); \ - } while (0) - -#define DEBUG_EXTRA_PRINT(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_EXTRA)) \ - debug_log(MDBX_LOG_EXTRA, NULL, 0, fmt, __VA_ARGS__); \ - } while (0) - -#define TRACE(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_TRACE)) \ - debug_log(MDBX_LOG_TRACE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ - } while (0) - -#define DEBUG(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_DEBUG)) \ - debug_log(MDBX_LOG_DEBUG, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ - } while (0) - -#define VERBOSE(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_VERBOSE)) \ - debug_log(MDBX_LOG_VERBOSE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ - } while (0) - -#define NOTICE(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_NOTICE)) \ - debug_log(MDBX_LOG_NOTICE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ - } while (0) - -#define WARNING(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_WARN)) \ - debug_log(MDBX_LOG_WARN, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ - } while (0) - -#undef ERROR /* wingdi.h \ - Yeah, morons from M$ put such definition to the public header. */ - -#define ERROR(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_ERROR)) \ - debug_log(MDBX_LOG_ERROR, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ - } while (0) - -#define FATAL(fmt, ...) \ - debug_log(MDBX_LOG_FATAL, __func__, __LINE__, fmt "\n", __VA_ARGS__); - -#if MDBX_DEBUG -#define ASSERT_FAIL(env, msg, func, line) mdbx_assert_fail(env, msg, func, line) -#else /* MDBX_DEBUG */ -MDBX_NORETURN __cold void assert_fail(const char *msg, const char *func, - unsigned line); -#define ASSERT_FAIL(env, msg, func, line) \ - do { \ - (void)(env); \ - assert_fail(msg, func, line); \ - } while (0) -#endif /* MDBX_DEBUG */ - -#define ENSURE_MSG(env, expr, msg) \ - do { \ - if (unlikely(!(expr))) \ - ASSERT_FAIL(env, msg, __func__, __LINE__); \ - } while (0) - -#define ENSURE(env, expr) ENSURE_MSG(env, expr, #expr) - -/* assert(3) variant in environment context */ -#define eASSERT(env, expr) \ - do { \ - if (ASSERT_ENABLED()) \ - ENSURE(env, expr); \ - } while (0) - -/* assert(3) variant in cursor context */ -#define cASSERT(mc, expr) eASSERT((mc)->mc_txn->mt_env, expr) - -/* assert(3) variant in transaction context */ -#define tASSERT(txn, expr) eASSERT((txn)->mt_env, expr) - -#ifndef xMDBX_TOOLS /* Avoid using internal eASSERT() */ -#undef assert -#define assert(expr) eASSERT(NULL, expr) +#error FIXME atomic-ops #endif -#endif /* __cplusplus */ - -/*----------------------------------------------------------------------------*/ -/* Atomics */ - -enum MDBX_memory_order { +typedef enum mdbx_memory_order { mo_Relaxed, mo_AcquireRelease /* , mo_SequentialConsistency */ -}; +} mdbx_memory_order_t; typedef union { volatile uint32_t weak; #ifdef MDBX_HAVE_C11ATOMICS volatile _Atomic uint32_t c11a; #endif /* MDBX_HAVE_C11ATOMICS */ -} MDBX_atomic_uint32_t; +} mdbx_atomic_uint32_t; typedef union { volatile uint64_t weak; @@ -2669,15 +2184,15 @@ typedef union { #if !defined(MDBX_HAVE_C11ATOMICS) || !MDBX_64BIT_CAS || !MDBX_64BIT_ATOMIC __anonymous_struct_extension__ struct { #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ - MDBX_atomic_uint32_t low, high; + mdbx_atomic_uint32_t low, high; #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - MDBX_atomic_uint32_t high, low; + mdbx_atomic_uint32_t high, low; #else #error "FIXME: Unsupported byte order" #endif /* __BYTE_ORDER__ */ }; #endif -} MDBX_atomic_uint64_t; +} mdbx_atomic_uint64_t; #ifdef MDBX_HAVE_C11ATOMICS @@ -2693,92 +2208,20 @@ typedef union { #define MDBX_c11a_rw(type, ptr) (&(ptr)->c11a) #endif /* Crutches for C11 atomic compiler's bugs */ -#define mo_c11_store(fence) \ - (((fence) == mo_Relaxed) ? memory_order_relaxed \ - : ((fence) == mo_AcquireRelease) ? memory_order_release \ +#define mo_c11_store(fence) \ + (((fence) == mo_Relaxed) ? memory_order_relaxed \ + : ((fence) == mo_AcquireRelease) ? memory_order_release \ : memory_order_seq_cst) -#define mo_c11_load(fence) \ - (((fence) == mo_Relaxed) ? memory_order_relaxed \ - : ((fence) == mo_AcquireRelease) ? memory_order_acquire \ +#define mo_c11_load(fence) \ + (((fence) == mo_Relaxed) ? memory_order_relaxed \ + : ((fence) == mo_AcquireRelease) ? memory_order_acquire \ : memory_order_seq_cst) #endif /* MDBX_HAVE_C11ATOMICS */ -#ifndef __cplusplus - -#ifdef MDBX_HAVE_C11ATOMICS -#define osal_memory_fence(order, write) \ - atomic_thread_fence((write) ? mo_c11_store(order) : mo_c11_load(order)) -#else /* MDBX_HAVE_C11ATOMICS */ -#define osal_memory_fence(order, write) \ - do { \ - osal_compiler_barrier(); \ - if (write && order > (MDBX_CPU_WRITEBACK_INCOHERENT ? mo_Relaxed \ - : mo_AcquireRelease)) \ - osal_memory_barrier(); \ - } while (0) -#endif /* MDBX_HAVE_C11ATOMICS */ - -#if defined(MDBX_HAVE_C11ATOMICS) && defined(__LCC__) -#define atomic_store32(p, value, order) \ - ({ \ - const uint32_t value_to_store = (value); \ - atomic_store_explicit(MDBX_c11a_rw(uint32_t, p), value_to_store, \ - mo_c11_store(order)); \ - value_to_store; \ - }) -#define atomic_load32(p, order) \ - atomic_load_explicit(MDBX_c11a_ro(uint32_t, p), mo_c11_load(order)) -#define atomic_store64(p, value, order) \ - ({ \ - const uint64_t value_to_store = (value); \ - atomic_store_explicit(MDBX_c11a_rw(uint64_t, p), value_to_store, \ - mo_c11_store(order)); \ - value_to_store; \ - }) -#define atomic_load64(p, order) \ - atomic_load_explicit(MDBX_c11a_ro(uint64_t, p), mo_c11_load(order)) -#endif /* LCC && MDBX_HAVE_C11ATOMICS */ - -#ifndef atomic_store32 -MDBX_MAYBE_UNUSED static __always_inline uint32_t -atomic_store32(MDBX_atomic_uint32_t *p, const uint32_t value, - enum MDBX_memory_order order) { - STATIC_ASSERT(sizeof(MDBX_atomic_uint32_t) == 4); -#ifdef MDBX_HAVE_C11ATOMICS - assert(atomic_is_lock_free(MDBX_c11a_rw(uint32_t, p))); - atomic_store_explicit(MDBX_c11a_rw(uint32_t, p), value, mo_c11_store(order)); -#else /* MDBX_HAVE_C11ATOMICS */ - if (order != mo_Relaxed) - osal_compiler_barrier(); - p->weak = value; - osal_memory_fence(order, true); -#endif /* MDBX_HAVE_C11ATOMICS */ - return value; -} -#endif /* atomic_store32 */ - -#ifndef atomic_load32 -MDBX_MAYBE_UNUSED static __always_inline uint32_t atomic_load32( - const volatile MDBX_atomic_uint32_t *p, enum MDBX_memory_order order) { - STATIC_ASSERT(sizeof(MDBX_atomic_uint32_t) == 4); -#ifdef MDBX_HAVE_C11ATOMICS - assert(atomic_is_lock_free(MDBX_c11a_ro(uint32_t, p))); - return atomic_load_explicit(MDBX_c11a_ro(uint32_t, p), mo_c11_load(order)); -#else /* MDBX_HAVE_C11ATOMICS */ - osal_memory_fence(order, false); - const uint32_t value = p->weak; - if (order != mo_Relaxed) - osal_compiler_barrier(); - return value; -#endif /* MDBX_HAVE_C11ATOMICS */ -} -#endif /* atomic_load32 */ - -#endif /* !__cplusplus */ +#define SAFE64_INVALID_THRESHOLD UINT64_C(0xffffFFFF00000000) -/*----------------------------------------------------------------------------*/ -/* Basic constants and types */ +#pragma pack(push, 4) /* A stamp that identifies a file as an MDBX file. * There's nothing special about this value other than that it is easily @@ -2787,8 +2230,10 @@ MDBX_MAYBE_UNUSED static __always_inline uint32_t atomic_load32( /* FROZEN: The version number for a database's datafile format. */ #define MDBX_DATA_VERSION 3 -/* The version number for a database's lockfile format. */ -#define MDBX_LOCK_VERSION 5 + +#define MDBX_DATA_MAGIC ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + MDBX_DATA_VERSION) +#define MDBX_DATA_MAGIC_LEGACY_COMPAT ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + 2) +#define MDBX_DATA_MAGIC_LEGACY_DEVEL ((MDBX_MAGIC << 8) + 255) /* handle for the DB used to track free pages. */ #define FREE_DBI 0 @@ -2805,203 +2250,285 @@ MDBX_MAYBE_UNUSED static __always_inline uint32_t atomic_load32( * MDBX uses 32 bit for page numbers. This limits database * size up to 2^44 bytes, in case of 4K pages. */ typedef uint32_t pgno_t; -typedef MDBX_atomic_uint32_t atomic_pgno_t; +typedef mdbx_atomic_uint32_t atomic_pgno_t; #define PRIaPGNO PRIu32 #define MAX_PAGENO UINT32_C(0x7FFFffff) #define MIN_PAGENO NUM_METAS -#define SAFE64_INVALID_THRESHOLD UINT64_C(0xffffFFFF00000000) +/* An invalid page number. + * Mainly used to denote an empty tree. */ +#define P_INVALID (~(pgno_t)0) /* A transaction ID. */ typedef uint64_t txnid_t; -typedef MDBX_atomic_uint64_t atomic_txnid_t; +typedef mdbx_atomic_uint64_t atomic_txnid_t; #define PRIaTXN PRIi64 #define MIN_TXNID UINT64_C(1) #define MAX_TXNID (SAFE64_INVALID_THRESHOLD - 1) #define INITIAL_TXNID (MIN_TXNID + NUM_METAS - 1) #define INVALID_TXNID UINT64_MAX -/* LY: for testing non-atomic 64-bit txnid on 32-bit arches. - * #define xMDBX_TXNID_STEP (UINT32_MAX / 3) */ -#ifndef xMDBX_TXNID_STEP -#if MDBX_64BIT_CAS -#define xMDBX_TXNID_STEP 1u -#else -#define xMDBX_TXNID_STEP 2u -#endif -#endif /* xMDBX_TXNID_STEP */ -/* Used for offsets within a single page. - * Since memory pages are typically 4 or 8KB in size, 12-13 bits, - * this is plenty. */ +/* Used for offsets within a single page. */ typedef uint16_t indx_t; -#define MEGABYTE ((size_t)1 << 20) - -/*----------------------------------------------------------------------------*/ -/* Core structures for database and shared memory (i.e. format definition) */ -#pragma pack(push, 4) - -/* Information about a single database in the environment. */ -typedef struct MDBX_db { - uint16_t md_flags; /* see mdbx_dbi_open */ - uint16_t md_depth; /* depth of this tree */ - uint32_t md_xsize; /* key-size for MDBX_DUPFIXED (LEAF2 pages) */ - pgno_t md_root; /* the root page of this tree */ - pgno_t md_branch_pages; /* number of internal pages */ - pgno_t md_leaf_pages; /* number of leaf pages */ - pgno_t md_overflow_pages; /* number of overflow pages */ - uint64_t md_seq; /* table sequence counter */ - uint64_t md_entries; /* number of data items */ - uint64_t md_mod_txnid; /* txnid of last committed modification */ -} MDBX_db; +typedef struct tree { + uint16_t flags; /* see mdbx_dbi_open */ + uint16_t height; /* height of this tree */ + uint32_t dupfix_size; /* key-size for MDBX_DUPFIXED (DUPFIX pages) */ + pgno_t root; /* the root page of this tree */ + pgno_t branch_pages; /* number of branch pages */ + pgno_t leaf_pages; /* number of leaf pages */ + pgno_t large_pages; /* number of large pages */ + uint64_t sequence; /* table sequence counter */ + uint64_t items; /* number of data items */ + uint64_t mod_txnid; /* txnid of last committed modification */ +} tree_t; /* database size-related parameters */ -typedef struct MDBX_geo { +typedef struct geo { uint16_t grow_pv; /* datafile growth step as a 16-bit packed (exponential quantized) value */ uint16_t shrink_pv; /* datafile shrink threshold as a 16-bit packed (exponential quantized) value */ pgno_t lower; /* minimal size of datafile in pages */ pgno_t upper; /* maximal size of datafile in pages */ - pgno_t now; /* current size of datafile in pages */ - pgno_t next; /* first unused page in the datafile, + union { + pgno_t now; /* current size of datafile in pages */ + pgno_t end_pgno; + }; + union { + pgno_t first_unallocated; /* first unused page in the datafile, but actually the file may be shorter. */ -} MDBX_geo; + pgno_t next_pgno; + }; +} geo_t; /* Meta page content. * A meta page is the start point for accessing a database snapshot. - * Pages 0-1 are meta pages. Transaction N writes meta page (N % 2). */ -typedef struct MDBX_meta { + * Pages 0-2 are meta pages. */ +typedef struct meta { /* Stamp identifying this as an MDBX file. * It must be set to MDBX_MAGIC with MDBX_DATA_VERSION. */ - uint32_t mm_magic_and_version[2]; + uint32_t magic_and_version[2]; - /* txnid that committed this page, the first of a two-phase-update pair */ + /* txnid that committed this meta, the first of a two-phase-update pair */ union { - MDBX_atomic_uint32_t mm_txnid_a[2]; + mdbx_atomic_uint32_t txnid_a[2]; uint64_t unsafe_txnid; }; - uint16_t mm_extra_flags; /* extra DB flags, zero (nothing) for now */ - uint8_t mm_validator_id; /* ID of checksum and page validation method, - * zero (nothing) for now */ - uint8_t mm_extra_pagehdr; /* extra bytes in the page header, - * zero (nothing) for now */ + uint16_t reserve16; /* extra flags, zero (nothing) for now */ + uint8_t validator_id; /* ID of checksum and page validation method, + * zero (nothing) for now */ + int8_t extra_pagehdr; /* extra bytes in the page header, + * zero (nothing) for now */ - MDBX_geo mm_geo; /* database size-related parameters */ + geo_t geometry; /* database size-related parameters */ + + union { + struct { + tree_t gc, main; + } trees; + __anonymous_struct_extension__ struct { + uint16_t gc_flags; + uint16_t gc_height; + uint32_t pagesize; + }; + }; - MDBX_db mm_dbs[CORE_DBS]; /* first is free space, 2nd is main db */ - /* The size of pages used in this DB */ -#define mm_psize mm_dbs[FREE_DBI].md_xsize - MDBX_canary mm_canary; + MDBX_canary canary; -#define MDBX_DATASIGN_NONE 0u -#define MDBX_DATASIGN_WEAK 1u -#define SIGN_IS_STEADY(sign) ((sign) > MDBX_DATASIGN_WEAK) -#define META_IS_STEADY(meta) \ - SIGN_IS_STEADY(unaligned_peek_u64_volatile(4, (meta)->mm_sign)) +#define DATASIGN_NONE 0u +#define DATASIGN_WEAK 1u +#define SIGN_IS_STEADY(sign) ((sign) > DATASIGN_WEAK) union { - uint32_t mm_sign[2]; + uint32_t sign[2]; uint64_t unsafe_sign; }; - /* txnid that committed this page, the second of a two-phase-update pair */ - MDBX_atomic_uint32_t mm_txnid_b[2]; + /* txnid that committed this meta, the second of a two-phase-update pair */ + mdbx_atomic_uint32_t txnid_b[2]; /* Number of non-meta pages which were put in GC after COW. May be 0 in case * DB was previously handled by libmdbx without corresponding feature. - * This value in couple with mr_snapshot_pages_retired allows fast estimation - * of "how much reader is restraining GC recycling". */ - uint32_t mm_pages_retired[2]; + * This value in couple with reader.snapshot_pages_retired allows fast + * estimation of "how much reader is restraining GC recycling". */ + uint32_t pages_retired[2]; /* The analogue /proc/sys/kernel/random/boot_id or similar to determine * whether the system was rebooted after the last use of the database files. * If there was no reboot, but there is no need to rollback to the last * steady sync point. Zeros mean that no relevant information is available * from the system. */ - bin128_t mm_bootid; + bin128_t bootid; -} MDBX_meta; + /* GUID базы данных, начиная с v0.13.1 */ + bin128_t dxbid; +} meta_t; #pragma pack(1) -/* Common header for all page types. The page type depends on mp_flags. +typedef enum page_type { + P_BRANCH = 0x01u /* branch page */, + P_LEAF = 0x02u /* leaf page */, + P_LARGE = 0x04u /* large/overflow page */, + P_META = 0x08u /* meta page */, + P_LEGACY_DIRTY = 0x10u /* legacy P_DIRTY flag prior to v0.10 958fd5b9 */, + P_BAD = P_LEGACY_DIRTY /* explicit flag for invalid/bad page */, + P_DUPFIX = 0x20u /* for MDBX_DUPFIXED records */, + P_SUBP = 0x40u /* for MDBX_DUPSORT sub-pages */, + P_SPILLED = 0x2000u /* spilled in parent txn */, + P_LOOSE = 0x4000u /* page was dirtied then freed, can be reused */, + P_FROZEN = 0x8000u /* used for retire page with known status */, + P_ILL_BITS = (uint16_t)~(P_BRANCH | P_LEAF | P_DUPFIX | P_LARGE | P_SPILLED), + + page_broken = 0, + page_large = P_LARGE, + page_branch = P_BRANCH, + page_leaf = P_LEAF, + page_dupfix_leaf = P_DUPFIX, + page_sub_leaf = P_SUBP | P_LEAF, + page_sub_dupfix_leaf = P_SUBP | P_DUPFIX, + page_sub_broken = P_SUBP, +} page_type_t; + +/* Common header for all page types. The page type depends on flags. * - * P_BRANCH and P_LEAF pages have unsorted 'MDBX_node's at the end, with - * sorted mp_ptrs[] entries referring to them. Exception: P_LEAF2 pages - * omit mp_ptrs and pack sorted MDBX_DUPFIXED values after the page header. + * P_BRANCH and P_LEAF pages have unsorted 'node_t's at the end, with + * sorted entries[] entries referring to them. Exception: P_DUPFIX pages + * omit entries and pack sorted MDBX_DUPFIXED values after the page header. * - * P_OVERFLOW records occupy one or more contiguous pages where only the - * first has a page header. They hold the real data of F_BIGDATA nodes. + * P_LARGE records occupy one or more contiguous pages where only the + * first has a page header. They hold the real data of N_BIG nodes. * * P_SUBP sub-pages are small leaf "pages" with duplicate data. - * A node with flag F_DUPDATA but not F_SUBDATA contains a sub-page. - * (Duplicate data can also go in sub-databases, which use normal pages.) + * A node with flag N_DUP but not N_TREE contains a sub-page. + * (Duplicate data can also go in tables, which use normal pages.) * - * P_META pages contain MDBX_meta, the start point of an MDBX snapshot. + * P_META pages contain meta_t, the start point of an MDBX snapshot. * - * Each non-metapage up to MDBX_meta.mm_last_pg is reachable exactly once + * Each non-metapage up to meta_t.mm_last_pg is reachable exactly once * in the snapshot: Either used by a database or listed in a GC record. */ -typedef struct MDBX_page { -#define IS_FROZEN(txn, p) ((p)->mp_txnid < (txn)->mt_txnid) -#define IS_SPILLED(txn, p) ((p)->mp_txnid == (txn)->mt_txnid) -#define IS_SHADOWED(txn, p) ((p)->mp_txnid > (txn)->mt_txnid) -#define IS_VALID(txn, p) ((p)->mp_txnid <= (txn)->mt_front) -#define IS_MODIFIABLE(txn, p) ((p)->mp_txnid == (txn)->mt_front) - uint64_t mp_txnid; /* txnid which created page, maybe zero in legacy DB */ - uint16_t mp_leaf2_ksize; /* key size if this is a LEAF2 page */ -#define P_BRANCH 0x01u /* branch page */ -#define P_LEAF 0x02u /* leaf page */ -#define P_OVERFLOW 0x04u /* overflow page */ -#define P_META 0x08u /* meta page */ -#define P_LEGACY_DIRTY 0x10u /* legacy P_DIRTY flag prior to v0.10 958fd5b9 */ -#define P_BAD P_LEGACY_DIRTY /* explicit flag for invalid/bad page */ -#define P_LEAF2 0x20u /* for MDBX_DUPFIXED records */ -#define P_SUBP 0x40u /* for MDBX_DUPSORT sub-pages */ -#define P_SPILLED 0x2000u /* spilled in parent txn */ -#define P_LOOSE 0x4000u /* page was dirtied then freed, can be reused */ -#define P_FROZEN 0x8000u /* used for retire page with known status */ -#define P_ILL_BITS \ - ((uint16_t)~(P_BRANCH | P_LEAF | P_LEAF2 | P_OVERFLOW | P_SPILLED)) - uint16_t mp_flags; +typedef struct page { + uint64_t txnid; /* txnid which created page, maybe zero in legacy DB */ + uint16_t dupfix_ksize; /* key size if this is a DUPFIX page */ + uint16_t flags; union { - uint32_t mp_pages; /* number of overflow pages */ + uint32_t pages; /* number of overflow pages */ __anonymous_struct_extension__ struct { - indx_t mp_lower; /* lower bound of free space */ - indx_t mp_upper; /* upper bound of free space */ + indx_t lower; /* lower bound of free space */ + indx_t upper; /* upper bound of free space */ }; }; - pgno_t mp_pgno; /* page number */ + pgno_t pgno; /* page number */ -#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ - (!defined(__cplusplus) && defined(_MSC_VER)) - indx_t mp_ptrs[] /* dynamic size */; -#endif /* C99 */ -} MDBX_page; - -#define PAGETYPE_WHOLE(p) ((uint8_t)(p)->mp_flags) - -/* Drop legacy P_DIRTY flag for sub-pages for compatilibity */ -#define PAGETYPE_COMPAT(p) \ - (unlikely(PAGETYPE_WHOLE(p) & P_SUBP) \ - ? PAGETYPE_WHOLE(p) & ~(P_SUBP | P_LEGACY_DIRTY) \ - : PAGETYPE_WHOLE(p)) +#if FLEXIBLE_ARRAY_MEMBERS + indx_t entries[] /* dynamic size */; +#endif /* FLEXIBLE_ARRAY_MEMBERS */ +} page_t; /* Size of the page header, excluding dynamic data at the end */ -#define PAGEHDRSZ offsetof(MDBX_page, mp_ptrs) +#define PAGEHDRSZ 20u -/* Pointer displacement without casting to char* to avoid pointer-aliasing */ -#define ptr_disp(ptr, disp) ((void *)(((intptr_t)(ptr)) + ((intptr_t)(disp)))) +/* Header for a single key/data pair within a page. + * Used in pages of type P_BRANCH and P_LEAF without P_DUPFIX. + * We guarantee 2-byte alignment for 'node_t's. + * + * Leaf node flags describe node contents. N_BIG says the node's + * data part is the page number of an overflow page with actual data. + * N_DUP and N_TREE can be combined giving duplicate data in + * a sub-page/table, and named databases (just N_TREE). */ +typedef struct node { +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + union { + uint32_t dsize; + uint32_t child_pgno; + }; + uint8_t flags; /* see node_flags */ + uint8_t extra; + uint16_t ksize; /* key size */ +#else + uint16_t ksize; /* key size */ + uint8_t extra; + uint8_t flags; /* see node_flags */ + union { + uint32_t child_pgno; + uint32_t dsize; + }; +#endif /* __BYTE_ORDER__ */ -/* Pointer distance as signed number of bytes */ -#define ptr_dist(more, less) (((intptr_t)(more)) - ((intptr_t)(less))) +#if FLEXIBLE_ARRAY_MEMBERS + uint8_t payload[] /* key and data are appended here */; +#endif /* FLEXIBLE_ARRAY_MEMBERS */ +} node_t; + +/* Size of the node header, excluding dynamic data at the end */ +#define NODESIZE 8u -#define mp_next(mp) \ - (*(MDBX_page **)ptr_disp((mp)->mp_ptrs, sizeof(void *) - sizeof(uint32_t))) +typedef enum node_flags { + N_BIG = 0x01 /* data put on large page */, + N_TREE = 0x02 /* data is a b-tree */, + N_DUP = 0x04 /* data has duplicates */ +} node_flags_t; #pragma pack(pop) -typedef struct profgc_stat { +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint8_t page_type(const page_t *mp) { return mp->flags; } + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint8_t page_type_compat(const page_t *mp) { + /* Drop legacy P_DIRTY flag for sub-pages for compatilibity, + * for assertions only. */ + return unlikely(mp->flags & P_SUBP) ? mp->flags & ~(P_SUBP | P_LEGACY_DIRTY) : mp->flags; +} + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_leaf(const page_t *mp) { + return (mp->flags & P_LEAF) != 0; +} + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_dupfix_leaf(const page_t *mp) { + return (mp->flags & P_DUPFIX) != 0; +} + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_branch(const page_t *mp) { + return (mp->flags & P_BRANCH) != 0; +} + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_largepage(const page_t *mp) { + return (mp->flags & P_LARGE) != 0; +} + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_subpage(const page_t *mp) { + return (mp->flags & P_SUBP) != 0; +} + +/* The version number for a database's lockfile format. */ +#define MDBX_LOCK_VERSION 6 + +#if MDBX_LOCKING == MDBX_LOCKING_WIN32FILES + +#define MDBX_LCK_SIGN UINT32_C(0xF10C) +typedef void osal_ipclock_t; +#elif MDBX_LOCKING == MDBX_LOCKING_SYSV + +#define MDBX_LCK_SIGN UINT32_C(0xF18D) +typedef mdbx_pid_t osal_ipclock_t; + +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || MDBX_LOCKING == MDBX_LOCKING_POSIX2008 + +#define MDBX_LCK_SIGN UINT32_C(0x8017) +typedef pthread_mutex_t osal_ipclock_t; + +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988 + +#define MDBX_LCK_SIGN UINT32_C(0xFC29) +typedef sem_t osal_ipclock_t; + +#else +#error "FIXME" +#endif /* MDBX_LOCKING */ + +/* Статистика профилирования работы GC */ +typedef struct gc_prof_stat { /* Монотонное время по "настенным часам" * затраченное на чтение и поиск внутри GC */ uint64_t rtime_monotonic; @@ -3017,42 +2544,44 @@ typedef struct profgc_stat { uint32_t spe_counter; /* page faults (hard page faults) */ uint32_t majflt; -} profgc_stat_t; - -/* Statistics of page operations overall of all (running, completed and aborted) - * transactions */ -typedef struct pgop_stat { - MDBX_atomic_uint64_t newly; /* Quantity of a new pages added */ - MDBX_atomic_uint64_t cow; /* Quantity of pages copied for update */ - MDBX_atomic_uint64_t clone; /* Quantity of parent's dirty pages clones + /* Для разборок с pnl_merge() */ + struct { + uint64_t time; + uint64_t volume; + uint32_t calls; + } pnl_merge; +} gc_prof_stat_t; + +/* Statistics of pages operations for all transactions, + * including incomplete and aborted. */ +typedef struct pgops { + mdbx_atomic_uint64_t newly; /* Quantity of a new pages added */ + mdbx_atomic_uint64_t cow; /* Quantity of pages copied for update */ + mdbx_atomic_uint64_t clone; /* Quantity of parent's dirty pages clones for nested transactions */ - MDBX_atomic_uint64_t split; /* Page splits */ - MDBX_atomic_uint64_t merge; /* Page merges */ - MDBX_atomic_uint64_t spill; /* Quantity of spilled dirty pages */ - MDBX_atomic_uint64_t unspill; /* Quantity of unspilled/reloaded pages */ - MDBX_atomic_uint64_t - wops; /* Number of explicit write operations (not a pages) to a disk */ - MDBX_atomic_uint64_t - msync; /* Number of explicit msync/flush-to-disk operations */ - MDBX_atomic_uint64_t - fsync; /* Number of explicit fsync/flush-to-disk operations */ - - MDBX_atomic_uint64_t prefault; /* Number of prefault write operations */ - MDBX_atomic_uint64_t mincore; /* Number of mincore() calls */ - - MDBX_atomic_uint32_t - incoherence; /* number of https://libmdbx.dqdkfa.ru/dead-github/issues/269 - caught */ - MDBX_atomic_uint32_t reserved; + mdbx_atomic_uint64_t split; /* Page splits */ + mdbx_atomic_uint64_t merge; /* Page merges */ + mdbx_atomic_uint64_t spill; /* Quantity of spilled dirty pages */ + mdbx_atomic_uint64_t unspill; /* Quantity of unspilled/reloaded pages */ + mdbx_atomic_uint64_t wops; /* Number of explicit write operations (not a pages) to a disk */ + mdbx_atomic_uint64_t msync; /* Number of explicit msync/flush-to-disk operations */ + mdbx_atomic_uint64_t fsync; /* Number of explicit fsync/flush-to-disk operations */ + + mdbx_atomic_uint64_t prefault; /* Number of prefault write operations */ + mdbx_atomic_uint64_t mincore; /* Number of mincore() calls */ + + mdbx_atomic_uint32_t incoherence; /* number of https://libmdbx.dqdkfa.ru/dead-github/issues/269 + caught */ + mdbx_atomic_uint32_t reserved; /* Статистика для профилирования GC. - * Логически эти данные может быть стоит вынести в другую структуру, + * Логически эти данные, возможно, стоит вынести в другую структуру, * но разница будет сугубо косметическая. */ struct { /* Затраты на поддержку данных пользователя */ - profgc_stat_t work; + gc_prof_stat_t work; /* Затраты на поддержку и обновления самой GC */ - profgc_stat_t self; + gc_prof_stat_t self; /* Итераций обновления GC, * больше 1 если были повторы/перезапуски */ uint32_t wloops; @@ -3067,33 +2596,6 @@ typedef struct pgop_stat { } gc_prof; } pgop_stat_t; -#if MDBX_LOCKING == MDBX_LOCKING_WIN32FILES -#define MDBX_CLOCK_SIGN UINT32_C(0xF10C) -typedef void osal_ipclock_t; -#elif MDBX_LOCKING == MDBX_LOCKING_SYSV - -#define MDBX_CLOCK_SIGN UINT32_C(0xF18D) -typedef mdbx_pid_t osal_ipclock_t; -#ifndef EOWNERDEAD -#define EOWNERDEAD MDBX_RESULT_TRUE -#endif - -#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ - MDBX_LOCKING == MDBX_LOCKING_POSIX2008 -#define MDBX_CLOCK_SIGN UINT32_C(0x8017) -typedef pthread_mutex_t osal_ipclock_t; -#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988 -#define MDBX_CLOCK_SIGN UINT32_C(0xFC29) -typedef sem_t osal_ipclock_t; -#else -#error "FIXME" -#endif /* MDBX_LOCKING */ - -#if MDBX_LOCKING > MDBX_LOCKING_SYSV && !defined(__cplusplus) -MDBX_INTERNAL_FUNC int osal_ipclock_stub(osal_ipclock_t *ipc); -MDBX_INTERNAL_FUNC int osal_ipclock_destroy(osal_ipclock_t *ipc); -#endif /* MDBX_LOCKING */ - /* Reader Lock Table * * Readers don't acquire any locks for their data access. Instead, they @@ -3103,8 +2605,9 @@ MDBX_INTERNAL_FUNC int osal_ipclock_destroy(osal_ipclock_t *ipc); * read transactions started by the same thread need no further locking to * proceed. * - * If MDBX_NOTLS is set, the slot address is not saved in thread-specific data. - * No reader table is used if the database is on a read-only filesystem. + * If MDBX_NOSTICKYTHREADS is set, the slot address is not saved in + * thread-specific data. No reader table is used if the database is on a + * read-only filesystem. * * Since the database uses multi-version concurrency control, readers don't * actually need any locking. This table is used to keep track of which @@ -3133,14 +2636,14 @@ MDBX_INTERNAL_FUNC int osal_ipclock_destroy(osal_ipclock_t *ipc); * many old transactions together. */ /* The actual reader record, with cacheline padding. */ -typedef struct MDBX_reader { - /* Current Transaction ID when this transaction began, or (txnid_t)-1. +typedef struct reader_slot { + /* Current Transaction ID when this transaction began, or INVALID_TXNID. * Multiple readers that start at the same time will probably have the * same ID here. Again, it's not important to exclude them from * anything; all we need to know is which version of the DB they * started from so we can avoid overwriting any data used in that * particular version. */ - MDBX_atomic_uint64_t /* txnid_t */ mr_txnid; + atomic_txnid_t txnid; /* The information we store in a single slot of the reader table. * In addition to a transaction ID, we also record the process and @@ -3151,708 +2654,320 @@ typedef struct MDBX_reader { * We simply re-init the table when we know that we're the only process * opening the lock file. */ + /* Псевдо thread_id для пометки вытесненных читающих транзакций. */ +#define MDBX_TID_TXN_OUSTED (UINT64_MAX - 1) + + /* Псевдо thread_id для пометки припаркованных читающих транзакций. */ +#define MDBX_TID_TXN_PARKED UINT64_MAX + /* The thread ID of the thread owning this txn. */ - MDBX_atomic_uint64_t mr_tid; + mdbx_atomic_uint64_t tid; /* The process ID of the process owning this reader txn. */ - MDBX_atomic_uint32_t mr_pid; + mdbx_atomic_uint32_t pid; /* The number of pages used in the reader's MVCC snapshot, - * i.e. the value of meta->mm_geo.next and txn->mt_next_pgno */ - atomic_pgno_t mr_snapshot_pages_used; + * i.e. the value of meta->geometry.first_unallocated and + * txn->geo.first_unallocated */ + atomic_pgno_t snapshot_pages_used; /* Number of retired pages at the time this reader starts transaction. So, - * at any time the difference mm_pages_retired - mr_snapshot_pages_retired - * will give the number of pages which this reader restraining from reuse. */ - MDBX_atomic_uint64_t mr_snapshot_pages_retired; -} MDBX_reader; + * at any time the difference meta.pages_retired - + * reader.snapshot_pages_retired will give the number of pages which this + * reader restraining from reuse. */ + mdbx_atomic_uint64_t snapshot_pages_retired; +} reader_slot_t; /* The header for the reader table (a memory-mapped lock file). */ -typedef struct MDBX_lockinfo { +typedef struct shared_lck { /* Stamp identifying this as an MDBX file. * It must be set to MDBX_MAGIC with with MDBX_LOCK_VERSION. */ - uint64_t mti_magic_and_version; + uint64_t magic_and_version; /* Format of this lock file. Must be set to MDBX_LOCK_FORMAT. */ - uint32_t mti_os_and_format; + uint32_t os_and_format; /* Flags which environment was opened. */ - MDBX_atomic_uint32_t mti_envmode; + mdbx_atomic_uint32_t envmode; /* Threshold of un-synced-with-disk pages for auto-sync feature, * zero means no-threshold, i.e. auto-sync is disabled. */ - atomic_pgno_t mti_autosync_threshold; + atomic_pgno_t autosync_threshold; /* Low 32-bit of txnid with which meta-pages was synced, * i.e. for sync-polling in the MDBX_NOMETASYNC mode. */ #define MDBX_NOMETASYNC_LAZY_UNK (UINT32_MAX / 3) #define MDBX_NOMETASYNC_LAZY_FD (MDBX_NOMETASYNC_LAZY_UNK + UINT32_MAX / 8) -#define MDBX_NOMETASYNC_LAZY_WRITEMAP \ - (MDBX_NOMETASYNC_LAZY_UNK - UINT32_MAX / 8) - MDBX_atomic_uint32_t mti_meta_sync_txnid; +#define MDBX_NOMETASYNC_LAZY_WRITEMAP (MDBX_NOMETASYNC_LAZY_UNK - UINT32_MAX / 8) + mdbx_atomic_uint32_t meta_sync_txnid; /* Period for timed auto-sync feature, i.e. at the every steady checkpoint - * the mti_unsynced_timeout sets to the current_time + mti_autosync_period. + * the mti_unsynced_timeout sets to the current_time + autosync_period. * The time value is represented in a suitable system-dependent form, for * example clock_gettime(CLOCK_BOOTTIME) or clock_gettime(CLOCK_MONOTONIC). * Zero means timed auto-sync is disabled. */ - MDBX_atomic_uint64_t mti_autosync_period; + mdbx_atomic_uint64_t autosync_period; /* Marker to distinguish uniqueness of DB/CLK. */ - MDBX_atomic_uint64_t mti_bait_uniqueness; + mdbx_atomic_uint64_t bait_uniqueness; /* Paired counter of processes that have mlock()ed part of mmapped DB. - * The (mti_mlcnt[0] - mti_mlcnt[1]) > 0 means at least one process + * The (mlcnt[0] - mlcnt[1]) > 0 means at least one process * lock at least one page, so therefore madvise() could return EINVAL. */ - MDBX_atomic_uint32_t mti_mlcnt[2]; + mdbx_atomic_uint32_t mlcnt[2]; MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/ /* Statistics of costly ops of all (running, completed and aborted) * transactions */ - pgop_stat_t mti_pgop_stat; + pgop_stat_t pgops; MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/ - /* Write transaction lock. */ #if MDBX_LOCKING > 0 - osal_ipclock_t mti_wlock; + /* Write transaction lock. */ + osal_ipclock_t wrt_lock; #endif /* MDBX_LOCKING > 0 */ - atomic_txnid_t mti_oldest_reader; + atomic_txnid_t cached_oldest; /* Timestamp of entering an out-of-sync state. Value is represented in a * suitable system-dependent form, for example clock_gettime(CLOCK_BOOTTIME) * or clock_gettime(CLOCK_MONOTONIC). */ - MDBX_atomic_uint64_t mti_eoos_timestamp; + mdbx_atomic_uint64_t eoos_timestamp; /* Number un-synced-with-disk pages for auto-sync feature. */ - MDBX_atomic_uint64_t mti_unsynced_pages; + mdbx_atomic_uint64_t unsynced_pages; /* Timestamp of the last readers check. */ - MDBX_atomic_uint64_t mti_reader_check_timestamp; + mdbx_atomic_uint64_t readers_check_timestamp; /* Number of page which was discarded last time by madvise(DONTNEED). */ - atomic_pgno_t mti_discarded_tail; + atomic_pgno_t discarded_tail; /* Shared anchor for tracking readahead edge and enabled/disabled status. */ - pgno_t mti_readahead_anchor; + pgno_t readahead_anchor; /* Shared cache for mincore() results */ struct { pgno_t begin[4]; uint64_t mask[4]; - } mti_mincore_cache; + } mincore_cache; MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/ - /* Readeaders registration lock. */ #if MDBX_LOCKING > 0 - osal_ipclock_t mti_rlock; + /* Readeaders table lock. */ + osal_ipclock_t rdt_lock; #endif /* MDBX_LOCKING > 0 */ /* The number of slots that have been used in the reader table. * This always records the maximum count, it is not decremented * when readers release their slots. */ - MDBX_atomic_uint32_t mti_numreaders; - MDBX_atomic_uint32_t mti_readers_refresh_flag; + mdbx_atomic_uint32_t rdt_length; + mdbx_atomic_uint32_t rdt_refresh_flag; -#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ - (!defined(__cplusplus) && defined(_MSC_VER)) +#if FLEXIBLE_ARRAY_MEMBERS MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/ - MDBX_reader mti_readers[] /* dynamic size */; -#endif /* C99 */ -} MDBX_lockinfo; + reader_slot_t rdt[] /* dynamic size */; /* Lockfile format signature: version, features and field layout */ -#define MDBX_LOCK_FORMAT \ - (MDBX_CLOCK_SIGN * 27733 + (unsigned)sizeof(MDBX_reader) * 13 + \ - (unsigned)offsetof(MDBX_reader, mr_snapshot_pages_used) * 251 + \ - (unsigned)offsetof(MDBX_lockinfo, mti_oldest_reader) * 83 + \ - (unsigned)offsetof(MDBX_lockinfo, mti_numreaders) * 37 + \ - (unsigned)offsetof(MDBX_lockinfo, mti_readers) * 29) - -#define MDBX_DATA_MAGIC \ - ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + MDBX_DATA_VERSION) - -#define MDBX_DATA_MAGIC_LEGACY_COMPAT \ - ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + 2) - -#define MDBX_DATA_MAGIC_LEGACY_DEVEL ((MDBX_MAGIC << 8) + 255) +#define MDBX_LOCK_FORMAT \ + (MDBX_LCK_SIGN * 27733 + (unsigned)sizeof(reader_slot_t) * 13 + \ + (unsigned)offsetof(reader_slot_t, snapshot_pages_used) * 251 + (unsigned)offsetof(lck_t, cached_oldest) * 83 + \ + (unsigned)offsetof(lck_t, rdt_length) * 37 + (unsigned)offsetof(lck_t, rdt) * 29) +#endif /* FLEXIBLE_ARRAY_MEMBERS */ +} lck_t; #define MDBX_LOCK_MAGIC ((MDBX_MAGIC << 8) + MDBX_LOCK_VERSION) -/* The maximum size of a database page. - * - * It is 64K, but value-PAGEHDRSZ must fit in MDBX_page.mp_upper. - * - * MDBX will use database pages < OS pages if needed. - * That causes more I/O in write transactions: The OS must - * know (read) the whole page before writing a partial page. - * - * Note that we don't currently support Huge pages. On Linux, - * regular data files cannot use Huge pages, and in general - * Huge pages aren't actually pageable. We rely on the OS - * demand-pager to read our data and page it out when memory - * pressure from other processes is high. So until OSs have - * actual paging support for Huge pages, they're not viable. */ -#define MAX_PAGESIZE MDBX_MAX_PAGESIZE -#define MIN_PAGESIZE MDBX_MIN_PAGESIZE - -#define MIN_MAPSIZE (MIN_PAGESIZE * MIN_PAGENO) +#define MDBX_READERS_LIMIT 32767 + +#define MIN_MAPSIZE (MDBX_MIN_PAGESIZE * MIN_PAGENO) #if defined(_WIN32) || defined(_WIN64) #define MAX_MAPSIZE32 UINT32_C(0x38000000) #else #define MAX_MAPSIZE32 UINT32_C(0x7f000000) #endif -#define MAX_MAPSIZE64 ((MAX_PAGENO + 1) * (uint64_t)MAX_PAGESIZE) +#define MAX_MAPSIZE64 ((MAX_PAGENO + 1) * (uint64_t)MDBX_MAX_PAGESIZE) #if MDBX_WORDBITS >= 64 #define MAX_MAPSIZE MAX_MAPSIZE64 -#define MDBX_PGL_LIMIT ((size_t)MAX_PAGENO) +#define PAGELIST_LIMIT ((size_t)MAX_PAGENO) #else #define MAX_MAPSIZE MAX_MAPSIZE32 -#define MDBX_PGL_LIMIT (MAX_MAPSIZE32 / MIN_PAGESIZE) +#define PAGELIST_LIMIT (MAX_MAPSIZE32 / MDBX_MIN_PAGESIZE) #endif /* MDBX_WORDBITS */ -#define MDBX_READERS_LIMIT 32767 -#define MDBX_RADIXSORT_THRESHOLD 142 #define MDBX_GOLD_RATIO_DBL 1.6180339887498948482 +#define MEGABYTE ((size_t)1 << 20) /*----------------------------------------------------------------------------*/ -/* An PNL is an Page Number List, a sorted array of IDs. - * The first element of the array is a counter for how many actual page-numbers - * are in the list. By default PNLs are sorted in descending order, this allow - * cut off a page with lowest pgno (at the tail) just truncating the list. The - * sort order of PNLs is controlled by the MDBX_PNL_ASCENDING build option. */ -typedef pgno_t *MDBX_PNL; - -#if MDBX_PNL_ASCENDING -#define MDBX_PNL_ORDERED(first, last) ((first) < (last)) -#define MDBX_PNL_DISORDERED(first, last) ((first) >= (last)) -#else -#define MDBX_PNL_ORDERED(first, last) ((first) > (last)) -#define MDBX_PNL_DISORDERED(first, last) ((first) <= (last)) -#endif +union logger_union { + void *ptr; + MDBX_debug_func *fmt; + MDBX_debug_func_nofmt *nofmt; +}; -/* List of txnid, only for MDBX_txn.tw.lifo_reclaimed */ -typedef txnid_t *MDBX_TXL; +struct libmdbx_globals { + bin128_t bootid; + unsigned sys_pagesize, sys_allocation_granularity; + uint8_t sys_pagesize_ln2; + uint8_t runtime_flags; + uint8_t loglevel; +#if defined(_WIN32) || defined(_WIN64) + bool running_under_Wine; +#elif defined(__linux__) || defined(__gnu_linux__) + bool running_on_WSL1 /* Windows Subsystem 1 for Linux */; + uint32_t linux_kernel_version; +#endif /* Linux */ + union logger_union logger; + osal_fastmutex_t debug_lock; + size_t logger_buffer_size; + char *logger_buffer; +}; -/* An Dirty-Page list item is an pgno/pointer pair. */ -typedef struct MDBX_dp { - MDBX_page *ptr; - pgno_t pgno, npages; -} MDBX_dp; +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ -/* An DPL (dirty-page list) is a sorted array of MDBX_DPs. */ -typedef struct MDBX_dpl { - size_t sorted; - size_t length; - size_t pages_including_loose; /* number of pages, but not an entries. */ - size_t detent; /* allocated size excluding the MDBX_DPL_RESERVE_GAP */ -#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ - (!defined(__cplusplus) && defined(_MSC_VER)) - MDBX_dp items[] /* dynamic size with holes at zero and after the last */; -#endif -} MDBX_dpl; +extern struct libmdbx_globals globals; +#if defined(_WIN32) || defined(_WIN64) +extern struct libmdbx_imports imports; +#endif /* Windows */ -/* PNL sizes */ -#define MDBX_PNL_GRANULATE_LOG2 10 -#define MDBX_PNL_GRANULATE (1 << MDBX_PNL_GRANULATE_LOG2) -#define MDBX_PNL_INITIAL \ - (MDBX_PNL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t)) +#ifndef __Wpedantic_format_voidptr +MDBX_MAYBE_UNUSED static inline const void *__Wpedantic_format_voidptr(const void *ptr) { return ptr; } +#define __Wpedantic_format_voidptr(ARG) __Wpedantic_format_voidptr(ARG) +#endif /* __Wpedantic_format_voidptr */ -#define MDBX_TXL_GRANULATE 32 -#define MDBX_TXL_INITIAL \ - (MDBX_TXL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t)) -#define MDBX_TXL_MAX \ - ((1u << 26) - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t)) +MDBX_INTERNAL void MDBX_PRINTF_ARGS(4, 5) debug_log(int level, const char *function, int line, const char *fmt, ...) + MDBX_PRINTF_ARGS(4, 5); +MDBX_INTERNAL void debug_log_va(int level, const char *function, int line, const char *fmt, va_list args); -#define MDBX_PNL_ALLOCLEN(pl) ((pl)[-1]) -#define MDBX_PNL_GETSIZE(pl) ((size_t)((pl)[0])) -#define MDBX_PNL_SETSIZE(pl, size) \ - do { \ - const size_t __size = size; \ - assert(__size < INT_MAX); \ - (pl)[0] = (pgno_t)__size; \ - } while (0) -#define MDBX_PNL_FIRST(pl) ((pl)[1]) -#define MDBX_PNL_LAST(pl) ((pl)[MDBX_PNL_GETSIZE(pl)]) -#define MDBX_PNL_BEGIN(pl) (&(pl)[1]) -#define MDBX_PNL_END(pl) (&(pl)[MDBX_PNL_GETSIZE(pl) + 1]) +#if MDBX_DEBUG +#define LOG_ENABLED(LVL) unlikely(LVL <= globals.loglevel) +#define AUDIT_ENABLED() unlikely((globals.runtime_flags & (unsigned)MDBX_DBG_AUDIT)) +#else /* MDBX_DEBUG */ +#define LOG_ENABLED(LVL) (LVL < MDBX_LOG_VERBOSE && LVL <= globals.loglevel) +#define AUDIT_ENABLED() (0) +#endif /* LOG_ENABLED() & AUDIT_ENABLED() */ -#if MDBX_PNL_ASCENDING -#define MDBX_PNL_EDGE(pl) ((pl) + 1) -#define MDBX_PNL_LEAST(pl) MDBX_PNL_FIRST(pl) -#define MDBX_PNL_MOST(pl) MDBX_PNL_LAST(pl) +#if MDBX_FORCE_ASSERTIONS +#define ASSERT_ENABLED() (1) +#elif MDBX_DEBUG +#define ASSERT_ENABLED() likely((globals.runtime_flags & (unsigned)MDBX_DBG_ASSERT)) #else -#define MDBX_PNL_EDGE(pl) ((pl) + MDBX_PNL_GETSIZE(pl)) -#define MDBX_PNL_LEAST(pl) MDBX_PNL_LAST(pl) -#define MDBX_PNL_MOST(pl) MDBX_PNL_FIRST(pl) -#endif - -#define MDBX_PNL_SIZEOF(pl) ((MDBX_PNL_GETSIZE(pl) + 1) * sizeof(pgno_t)) -#define MDBX_PNL_IS_EMPTY(pl) (MDBX_PNL_GETSIZE(pl) == 0) - -/*----------------------------------------------------------------------------*/ -/* Internal structures */ - -/* Auxiliary DB info. - * The information here is mostly static/read-only. There is - * only a single copy of this record in the environment. */ -typedef struct MDBX_dbx { - MDBX_val md_name; /* name of the database */ - MDBX_cmp_func *md_cmp; /* function for comparing keys */ - MDBX_cmp_func *md_dcmp; /* function for comparing data items */ - size_t md_klen_min, md_klen_max; /* min/max key length for the database */ - size_t md_vlen_min, - md_vlen_max; /* min/max value/data length for the database */ -} MDBX_dbx; - -typedef struct troika { - uint8_t fsm, recent, prefer_steady, tail_and_flags; -#if MDBX_WORDBITS > 32 /* Workaround for false-positives from Valgrind */ - uint32_t unused_pad; -#endif -#define TROIKA_HAVE_STEADY(troika) ((troika)->fsm & 7) -#define TROIKA_STRICT_VALID(troika) ((troika)->tail_and_flags & 64) -#define TROIKA_VALID(troika) ((troika)->tail_and_flags & 128) -#define TROIKA_TAIL(troika) ((troika)->tail_and_flags & 3) - txnid_t txnid[NUM_METAS]; -} meta_troika_t; - -/* A database transaction. - * Every operation requires a transaction handle. */ -struct MDBX_txn { -#define MDBX_MT_SIGNATURE UINT32_C(0x93D53A31) - uint32_t mt_signature; - - /* Transaction Flags */ - /* mdbx_txn_begin() flags */ -#define MDBX_TXN_RO_BEGIN_FLAGS (MDBX_TXN_RDONLY | MDBX_TXN_RDONLY_PREPARE) -#define MDBX_TXN_RW_BEGIN_FLAGS \ - (MDBX_TXN_NOMETASYNC | MDBX_TXN_NOSYNC | MDBX_TXN_TRY) - /* Additional flag for sync_locked() */ -#define MDBX_SHRINK_ALLOWED UINT32_C(0x40000000) - -#define MDBX_TXN_DRAINED_GC 0x20 /* GC was depleted up to oldest reader */ - -#define TXN_FLAGS \ - (MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_DIRTY | MDBX_TXN_SPILLS | \ - MDBX_TXN_HAS_CHILD | MDBX_TXN_INVALID | MDBX_TXN_DRAINED_GC) - -#if (TXN_FLAGS & (MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_RO_BEGIN_FLAGS)) || \ - ((MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_RO_BEGIN_FLAGS | TXN_FLAGS) & \ - MDBX_SHRINK_ALLOWED) -#error "Oops, some txn flags overlapped or wrong" -#endif - uint32_t mt_flags; - - MDBX_txn *mt_parent; /* parent of a nested txn */ - /* Nested txn under this txn, set together with flag MDBX_TXN_HAS_CHILD */ - MDBX_txn *mt_child; - MDBX_geo mt_geo; - /* next unallocated page */ -#define mt_next_pgno mt_geo.next - /* corresponding to the current size of datafile */ -#define mt_end_pgno mt_geo.now - - /* The ID of this transaction. IDs are integers incrementing from - * INITIAL_TXNID. Only committed write transactions increment the ID. If a - * transaction aborts, the ID may be re-used by the next writer. */ - txnid_t mt_txnid; - txnid_t mt_front; - - MDBX_env *mt_env; /* the DB environment */ - /* Array of records for each DB known in the environment. */ - MDBX_dbx *mt_dbxs; - /* Array of MDBX_db records for each known DB */ - MDBX_db *mt_dbs; - /* Array of sequence numbers for each DB handle */ - MDBX_atomic_uint32_t *mt_dbiseqs; - - /* Transaction DBI Flags */ -#define DBI_DIRTY MDBX_DBI_DIRTY /* DB was written in this txn */ -#define DBI_STALE MDBX_DBI_STALE /* Named-DB record is older than txnID */ -#define DBI_FRESH MDBX_DBI_FRESH /* Named-DB handle opened in this txn */ -#define DBI_CREAT MDBX_DBI_CREAT /* Named-DB handle created in this txn */ -#define DBI_VALID 0x10 /* DB handle is valid, see also DB_VALID */ -#define DBI_USRVALID 0x20 /* As DB_VALID, but not set for FREE_DBI */ -#define DBI_AUDITED 0x40 /* Internal flag for accounting during audit */ - /* Array of flags for each DB */ - uint8_t *mt_dbistate; - /* Number of DB records in use, or 0 when the txn is finished. - * This number only ever increments until the txn finishes; we - * don't decrement it when individual DB handles are closed. */ - MDBX_dbi mt_numdbs; - size_t mt_owner; /* thread ID that owns this transaction */ - MDBX_canary mt_canary; - void *mt_userctx; /* User-settable context */ - MDBX_cursor **mt_cursors; +#define ASSERT_ENABLED() (0) +#endif /* ASSERT_ENABLED() */ - union { - struct { - /* For read txns: This thread/txn's reader table slot, or NULL. */ - MDBX_reader *reader; - } to; - struct { - meta_troika_t troika; - /* In write txns, array of cursors for each DB */ - MDBX_PNL relist; /* Reclaimed GC pages */ - txnid_t last_reclaimed; /* ID of last used record */ -#if MDBX_ENABLE_REFUND - pgno_t loose_refund_wl /* FIXME: describe */; -#endif /* MDBX_ENABLE_REFUND */ - /* a sequence to spilling dirty page with LRU policy */ - unsigned dirtylru; - /* dirtylist room: Dirty array size - dirty pages visible to this txn. - * Includes ancestor txns' dirty pages not hidden by other txns' - * dirty/spilled pages. Thus commit(nested txn) has room to merge - * dirtylist into mt_parent after freeing hidden mt_parent pages. */ - size_t dirtyroom; - /* For write txns: Modified pages. Sorted when not MDBX_WRITEMAP. */ - MDBX_dpl *dirtylist; - /* The list of reclaimed txns from GC */ - MDBX_TXL lifo_reclaimed; - /* The list of pages that became unused during this transaction. */ - MDBX_PNL retired_pages; - /* The list of loose pages that became unused and may be reused - * in this transaction, linked through `mp_next`. */ - MDBX_page *loose_pages; - /* Number of loose pages (tw.loose_pages) */ - size_t loose_count; - union { - struct { - size_t least_removed; - /* The sorted list of dirty pages we temporarily wrote to disk - * because the dirty list was full. page numbers in here are - * shifted left by 1, deleted slots have the LSB set. */ - MDBX_PNL list; - } spilled; - size_t writemap_dirty_npages; - size_t writemap_spilled_npages; - }; - } tw; - }; -}; +#define DEBUG_EXTRA(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_EXTRA)) \ + debug_log(MDBX_LOG_EXTRA, __func__, __LINE__, fmt, __VA_ARGS__); \ + } while (0) -#if MDBX_WORDBITS >= 64 -#define CURSOR_STACK 32 -#else -#define CURSOR_STACK 24 -#endif - -struct MDBX_xcursor; - -/* Cursors are used for all DB operations. - * A cursor holds a path of (page pointer, key index) from the DB - * root to a position in the DB, plus other state. MDBX_DUPSORT - * cursors include an xcursor to the current data item. Write txns - * track their cursors and keep them up to date when data moves. - * Exception: An xcursor's pointer to a P_SUBP page can be stale. - * (A node with F_DUPDATA but no F_SUBDATA contains a subpage). */ -struct MDBX_cursor { -#define MDBX_MC_LIVE UINT32_C(0xFE05D5B1) -#define MDBX_MC_READY4CLOSE UINT32_C(0x2817A047) -#define MDBX_MC_WAIT4EOT UINT32_C(0x90E297A7) - uint32_t mc_signature; - /* The database handle this cursor operates on */ - MDBX_dbi mc_dbi; - /* Next cursor on this DB in this txn */ - MDBX_cursor *mc_next; - /* Backup of the original cursor if this cursor is a shadow */ - MDBX_cursor *mc_backup; - /* Context used for databases with MDBX_DUPSORT, otherwise NULL */ - struct MDBX_xcursor *mc_xcursor; - /* The transaction that owns this cursor */ - MDBX_txn *mc_txn; - /* The database record for this cursor */ - MDBX_db *mc_db; - /* The database auxiliary record for this cursor */ - MDBX_dbx *mc_dbx; - /* The mt_dbistate for this database */ - uint8_t *mc_dbistate; - uint8_t mc_snum; /* number of pushed pages */ - uint8_t mc_top; /* index of top page, normally mc_snum-1 */ - - /* Cursor state flags. */ -#define C_INITIALIZED 0x01 /* cursor has been initialized and is valid */ -#define C_EOF 0x02 /* No more data */ -#define C_SUB 0x04 /* Cursor is a sub-cursor */ -#define C_DEL 0x08 /* last op was a cursor_del */ -#define C_UNTRACK 0x10 /* Un-track cursor when closing */ -#define C_GCU \ - 0x20 /* Происходит подготовка к обновлению GC, поэтому \ - * можно брать страницы из GC даже для FREE_DBI */ - uint8_t mc_flags; - - /* Cursor checking flags. */ -#define CC_BRANCH 0x01 /* same as P_BRANCH for CHECK_LEAF_TYPE() */ -#define CC_LEAF 0x02 /* same as P_LEAF for CHECK_LEAF_TYPE() */ -#define CC_OVERFLOW 0x04 /* same as P_OVERFLOW for CHECK_LEAF_TYPE() */ -#define CC_UPDATING 0x08 /* update/rebalance pending */ -#define CC_SKIPORD 0x10 /* don't check keys ordering */ -#define CC_LEAF2 0x20 /* same as P_LEAF2 for CHECK_LEAF_TYPE() */ -#define CC_RETIRING 0x40 /* refs to child pages may be invalid */ -#define CC_PAGECHECK 0x80 /* perform page checking, see MDBX_VALIDATION */ - uint8_t mc_checking; - - MDBX_page *mc_pg[CURSOR_STACK]; /* stack of pushed pages */ - indx_t mc_ki[CURSOR_STACK]; /* stack of page indices */ -}; +#define DEBUG_EXTRA_PRINT(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_EXTRA)) \ + debug_log(MDBX_LOG_EXTRA, nullptr, 0, fmt, __VA_ARGS__); \ + } while (0) -#define CHECK_LEAF_TYPE(mc, mp) \ - (((PAGETYPE_WHOLE(mp) ^ (mc)->mc_checking) & \ - (CC_BRANCH | CC_LEAF | CC_OVERFLOW | CC_LEAF2)) == 0) - -/* Context for sorted-dup records. - * We could have gone to a fully recursive design, with arbitrarily - * deep nesting of sub-databases. But for now we only handle these - * levels - main DB, optional sub-DB, sorted-duplicate DB. */ -typedef struct MDBX_xcursor { - /* A sub-cursor for traversing the Dup DB */ - MDBX_cursor mx_cursor; - /* The database record for this Dup DB */ - MDBX_db mx_db; - /* The auxiliary DB record for this Dup DB */ - MDBX_dbx mx_dbx; -} MDBX_xcursor; - -typedef struct MDBX_cursor_couple { - MDBX_cursor outer; - void *mc_userctx; /* User-settable context */ - MDBX_xcursor inner; -} MDBX_cursor_couple; - -/* The database environment. */ -struct MDBX_env { - /* ----------------------------------------------------- mostly static part */ -#define MDBX_ME_SIGNATURE UINT32_C(0x9A899641) - MDBX_atomic_uint32_t me_signature; - /* Failed to update the meta page. Probably an I/O error. */ -#define MDBX_FATAL_ERROR UINT32_C(0x80000000) - /* Some fields are initialized. */ -#define MDBX_ENV_ACTIVE UINT32_C(0x20000000) - /* me_txkey is set */ -#define MDBX_ENV_TXKEY UINT32_C(0x10000000) - /* Legacy MDBX_MAPASYNC (prior v0.9) */ -#define MDBX_DEPRECATED_MAPASYNC UINT32_C(0x100000) - /* Legacy MDBX_COALESCE (prior v0.12) */ -#define MDBX_DEPRECATED_COALESCE UINT32_C(0x2000000) -#define ENV_INTERNAL_FLAGS (MDBX_FATAL_ERROR | MDBX_ENV_ACTIVE | MDBX_ENV_TXKEY) - uint32_t me_flags; - osal_mmap_t me_dxb_mmap; /* The main data file */ -#define me_map me_dxb_mmap.base -#define me_lazy_fd me_dxb_mmap.fd - mdbx_filehandle_t me_dsync_fd, me_fd4meta; -#if defined(_WIN32) || defined(_WIN64) -#define me_overlapped_fd me_ioring.overlapped_fd - HANDLE me_data_lock_event; -#endif /* Windows */ - osal_mmap_t me_lck_mmap; /* The lock file */ -#define me_lfd me_lck_mmap.fd - struct MDBX_lockinfo *me_lck; - - unsigned me_psize; /* DB page size, initialized from me_os_psize */ - uint16_t me_leaf_nodemax; /* max size of a leaf-node */ - uint16_t me_branch_nodemax; /* max size of a branch-node */ - uint16_t me_subpage_limit; - uint16_t me_subpage_room_threshold; - uint16_t me_subpage_reserve_prereq; - uint16_t me_subpage_reserve_limit; - atomic_pgno_t me_mlocked_pgno; - uint8_t me_psize2log; /* log2 of DB page size */ - int8_t me_stuck_meta; /* recovery-only: target meta page or less that zero */ - uint16_t me_merge_threshold, - me_merge_threshold_gc; /* pages emptier than this are candidates for - merging */ - unsigned me_os_psize; /* OS page size, from osal_syspagesize() */ - unsigned me_maxreaders; /* size of the reader table */ - MDBX_dbi me_maxdbs; /* size of the DB table */ - uint32_t me_pid; /* process ID of this env */ - osal_thread_key_t me_txkey; /* thread-key for readers */ - pathchar_t *me_pathname; /* path to the DB files */ - void *me_pbuf; /* scratch area for DUPSORT put() */ - MDBX_txn *me_txn0; /* preallocated write transaction */ - - MDBX_dbx *me_dbxs; /* array of static DB info */ - uint16_t *me_dbflags; /* array of flags from MDBX_db.md_flags */ - MDBX_atomic_uint32_t *me_dbiseqs; /* array of dbi sequence numbers */ - unsigned - me_maxgc_ov1page; /* Number of pgno_t fit in a single overflow page */ - unsigned me_maxgc_per_branch; - uint32_t me_live_reader; /* have liveness lock in reader table */ - void *me_userctx; /* User-settable context */ - MDBX_hsr_func *me_hsr_callback; /* Callback for kicking laggard readers */ - size_t me_madv_threshold; +#define TRACE(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_TRACE)) \ + debug_log(MDBX_LOG_TRACE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ + } while (0) - struct { - unsigned dp_reserve_limit; - unsigned rp_augment_limit; - unsigned dp_limit; - unsigned dp_initial; - uint8_t dp_loose_limit; - uint8_t spill_max_denominator; - uint8_t spill_min_denominator; - uint8_t spill_parent4child_denominator; - unsigned merge_threshold_16dot16_percent; -#if !(defined(_WIN32) || defined(_WIN64)) - unsigned writethrough_threshold; -#endif /* Windows */ - bool prefault_write; - union { - unsigned all; - /* tracks options with non-auto values but tuned by user */ - struct { - unsigned dp_limit : 1; - unsigned rp_augment_limit : 1; - unsigned prefault_write : 1; - } non_auto; - } flags; - } me_options; - - /* struct me_dbgeo used for accepting db-geo params from user for the new - * database creation, i.e. when mdbx_env_set_geometry() was called before - * mdbx_env_open(). */ - struct { - size_t lower; /* minimal size of datafile */ - size_t upper; /* maximal size of datafile */ - size_t now; /* current size of datafile */ - size_t grow; /* step to grow datafile */ - size_t shrink; /* threshold to shrink datafile */ - } me_dbgeo; - -#if MDBX_LOCKING == MDBX_LOCKING_SYSV - union { - key_t key; - int semid; - } me_sysv_ipc; -#endif /* MDBX_LOCKING == MDBX_LOCKING_SYSV */ - bool me_incore; +#define DEBUG(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_DEBUG)) \ + debug_log(MDBX_LOG_DEBUG, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ + } while (0) - MDBX_env *me_lcklist_next; +#define VERBOSE(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_VERBOSE)) \ + debug_log(MDBX_LOG_VERBOSE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ + } while (0) - /* --------------------------------------------------- mostly volatile part */ +#define NOTICE(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_NOTICE)) \ + debug_log(MDBX_LOG_NOTICE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ + } while (0) - MDBX_txn *me_txn; /* current write transaction */ - osal_fastmutex_t me_dbi_lock; - MDBX_dbi me_numdbs; /* number of DBs opened */ - bool me_prefault_write; +#define WARNING(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_WARN)) \ + debug_log(MDBX_LOG_WARN, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ + } while (0) - MDBX_page *me_dp_reserve; /* list of malloc'ed blocks for re-use */ - unsigned me_dp_reserve_len; - /* PNL of pages that became unused in a write txn */ - MDBX_PNL me_retired_pages; - osal_ioring_t me_ioring; +#undef ERROR /* wingdi.h \ + Yeah, morons from M$ put such definition to the public header. */ -#if defined(_WIN32) || defined(_WIN64) - osal_srwlock_t me_remap_guard; - /* Workaround for LockFileEx and WriteFile multithread bug */ - CRITICAL_SECTION me_windowsbug_lock; - char *me_pathname_char; /* cache of multi-byte representation of pathname - to the DB files */ -#else - osal_fastmutex_t me_remap_guard; -#endif +#define ERROR(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_ERROR)) \ + debug_log(MDBX_LOG_ERROR, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ + } while (0) - /* -------------------------------------------------------------- debugging */ +#define FATAL(fmt, ...) debug_log(MDBX_LOG_FATAL, __func__, __LINE__, fmt "\n", __VA_ARGS__); #if MDBX_DEBUG - MDBX_assert_func *me_assert_func; /* Callback for assertion failures */ -#endif -#ifdef MDBX_USE_VALGRIND - int me_valgrind_handle; -#endif -#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__) - MDBX_atomic_uint32_t me_ignore_EDEADLK; - pgno_t me_poison_edge; -#endif /* MDBX_USE_VALGRIND || __SANITIZE_ADDRESS__ */ +#define ASSERT_FAIL(env, msg, func, line) mdbx_assert_fail(env, msg, func, line) +#else /* MDBX_DEBUG */ +MDBX_NORETURN __cold void assert_fail(const char *msg, const char *func, unsigned line); +#define ASSERT_FAIL(env, msg, func, line) \ + do { \ + (void)(env); \ + assert_fail(msg, func, line); \ + } while (0) +#endif /* MDBX_DEBUG */ -#ifndef xMDBX_DEBUG_SPILLING -#define xMDBX_DEBUG_SPILLING 0 -#endif -#if xMDBX_DEBUG_SPILLING == 2 - size_t debug_dirtied_est, debug_dirtied_act; -#endif /* xMDBX_DEBUG_SPILLING */ +#define ENSURE_MSG(env, expr, msg) \ + do { \ + if (unlikely(!(expr))) \ + ASSERT_FAIL(env, msg, __func__, __LINE__); \ + } while (0) - /* ------------------------------------------------- stub for lck-less mode */ - MDBX_atomic_uint64_t - x_lckless_stub[(sizeof(MDBX_lockinfo) + MDBX_CACHELINE_SIZE - 1) / - sizeof(MDBX_atomic_uint64_t)]; -}; +#define ENSURE(env, expr) ENSURE_MSG(env, expr, #expr) -#ifndef __cplusplus -/*----------------------------------------------------------------------------*/ -/* Cache coherence and mmap invalidation */ +/* assert(3) variant in environment context */ +#define eASSERT(env, expr) \ + do { \ + if (ASSERT_ENABLED()) \ + ENSURE(env, expr); \ + } while (0) -#if MDBX_CPU_WRITEBACK_INCOHERENT -#define osal_flush_incoherent_cpu_writeback() osal_memory_barrier() -#else -#define osal_flush_incoherent_cpu_writeback() osal_compiler_barrier() -#endif /* MDBX_CPU_WRITEBACK_INCOHERENT */ +/* assert(3) variant in cursor context */ +#define cASSERT(mc, expr) eASSERT((mc)->txn->env, expr) -MDBX_MAYBE_UNUSED static __inline void -osal_flush_incoherent_mmap(const void *addr, size_t nbytes, - const intptr_t pagesize) { -#if MDBX_MMAP_INCOHERENT_FILE_WRITE - char *const begin = (char *)(-pagesize & (intptr_t)addr); - char *const end = - (char *)(-pagesize & (intptr_t)((char *)addr + nbytes + pagesize - 1)); - int err = msync(begin, end - begin, MS_SYNC | MS_INVALIDATE) ? errno : 0; - eASSERT(nullptr, err == 0); - (void)err; -#else - (void)pagesize; -#endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */ +/* assert(3) variant in transaction context */ +#define tASSERT(txn, expr) eASSERT((txn)->env, expr) -#if MDBX_MMAP_INCOHERENT_CPU_CACHE -#ifdef DCACHE - /* MIPS has cache coherency issues. - * Note: for any nbytes >= on-chip cache size, entire is flushed. */ - cacheflush((void *)addr, nbytes, DCACHE); -#else -#error "Oops, cacheflush() not available" -#endif /* DCACHE */ -#endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */ +#ifndef xMDBX_TOOLS /* Avoid using internal eASSERT() */ +#undef assert +#define assert(expr) eASSERT(nullptr, expr) +#endif -#if !MDBX_MMAP_INCOHERENT_FILE_WRITE && !MDBX_MMAP_INCOHERENT_CPU_CACHE - (void)addr; - (void)nbytes; +MDBX_MAYBE_UNUSED static inline void jitter4testing(bool tiny) { +#if MDBX_DEBUG + if (globals.runtime_flags & (unsigned)MDBX_DBG_JITTER) + osal_jitter(tiny); +#else + (void)tiny; #endif } -/*----------------------------------------------------------------------------*/ -/* Internal prototypes */ - -MDBX_INTERNAL_FUNC int cleanup_dead_readers(MDBX_env *env, int rlocked, - int *dead); -MDBX_INTERNAL_FUNC int rthc_alloc(osal_thread_key_t *key, MDBX_reader *begin, - MDBX_reader *end); -MDBX_INTERNAL_FUNC void rthc_remove(const osal_thread_key_t key); - -MDBX_INTERNAL_FUNC void global_ctor(void); -MDBX_INTERNAL_FUNC void osal_ctor(void); -MDBX_INTERNAL_FUNC void global_dtor(void); -MDBX_INTERNAL_FUNC void osal_dtor(void); -MDBX_INTERNAL_FUNC void thread_dtor(void *ptr); - -#endif /* !__cplusplus */ - -#define MDBX_IS_ERROR(rc) \ - ((rc) != MDBX_RESULT_TRUE && (rc) != MDBX_RESULT_FALSE) - -/* Internal error codes, not exposed outside libmdbx */ -#define MDBX_NO_ROOT (MDBX_LAST_ADDED_ERRCODE + 10) - -/* Debugging output value of a cursor DBI: Negative in a sub-cursor. */ -#define DDBI(mc) \ - (((mc)->mc_flags & C_SUB) ? -(int)(mc)->mc_dbi : (int)(mc)->mc_dbi) +MDBX_MAYBE_UNUSED MDBX_INTERNAL void page_list(page_t *mp); +MDBX_INTERNAL const char *pagetype_caption(const uint8_t type, char buf4unknown[16]); /* Key size which fits in a DKBUF (debug key buffer). */ -#define DKBUF_MAX 511 -#define DKBUF char _kbuf[DKBUF_MAX * 4 + 2] -#define DKEY(x) mdbx_dump_val(x, _kbuf, DKBUF_MAX * 2 + 1) -#define DVAL(x) mdbx_dump_val(x, _kbuf + DKBUF_MAX * 2 + 1, DKBUF_MAX * 2 + 1) +#define DKBUF_MAX 127 +#define DKBUF char dbg_kbuf[DKBUF_MAX * 4 + 2] +#define DKEY(x) mdbx_dump_val(x, dbg_kbuf, DKBUF_MAX * 2 + 1) +#define DVAL(x) mdbx_dump_val(x, dbg_kbuf + DKBUF_MAX * 2 + 1, DKBUF_MAX * 2 + 1) #if MDBX_DEBUG #define DKBUF_DEBUG DKBUF @@ -3864,102 +2979,24 @@ MDBX_INTERNAL_FUNC void thread_dtor(void *ptr); #define DVAL_DEBUG(x) ("-") #endif -/* An invalid page number. - * Mainly used to denote an empty tree. */ -#define P_INVALID (~(pgno_t)0) +MDBX_INTERNAL void log_error(const int err, const char *func, unsigned line); + +MDBX_MAYBE_UNUSED static inline int log_if_error(const int err, const char *func, unsigned line) { + if (unlikely(err != MDBX_SUCCESS)) + log_error(err, func, line); + return err; +} + +#define LOG_IFERR(err) log_if_error((err), __func__, __LINE__) /* Test if the flags f are set in a flag word w. */ #define F_ISSET(w, f) (((w) & (f)) == (f)) /* Round n up to an even number. */ -#define EVEN(n) (((n) + 1UL) & -2L) /* sign-extending -2 to match n+1U */ - -/* Default size of memory map. - * This is certainly too small for any actual applications. Apps should - * always set the size explicitly using mdbx_env_set_geometry(). */ -#define DEFAULT_MAPSIZE MEGABYTE - -/* Number of slots in the reader table. - * This value was chosen somewhat arbitrarily. The 61 is a prime number, - * and such readers plus a couple mutexes fit into single 4KB page. - * Applications should set the table size using mdbx_env_set_maxreaders(). */ -#define DEFAULT_READERS 61 - -/* Test if a page is a leaf page */ -#define IS_LEAF(p) (((p)->mp_flags & P_LEAF) != 0) -/* Test if a page is a LEAF2 page */ -#define IS_LEAF2(p) unlikely(((p)->mp_flags & P_LEAF2) != 0) -/* Test if a page is a branch page */ -#define IS_BRANCH(p) (((p)->mp_flags & P_BRANCH) != 0) -/* Test if a page is an overflow page */ -#define IS_OVERFLOW(p) unlikely(((p)->mp_flags & P_OVERFLOW) != 0) -/* Test if a page is a sub page */ -#define IS_SUBP(p) (((p)->mp_flags & P_SUBP) != 0) - -/* Header for a single key/data pair within a page. - * Used in pages of type P_BRANCH and P_LEAF without P_LEAF2. - * We guarantee 2-byte alignment for 'MDBX_node's. - * - * Leaf node flags describe node contents. F_BIGDATA says the node's - * data part is the page number of an overflow page with actual data. - * F_DUPDATA and F_SUBDATA can be combined giving duplicate data in - * a sub-page/sub-database, and named databases (just F_SUBDATA). */ -typedef struct MDBX_node { -#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ - union { - uint32_t mn_dsize; - uint32_t mn_pgno32; - }; - uint8_t mn_flags; /* see mdbx_node flags */ - uint8_t mn_extra; - uint16_t mn_ksize; /* key size */ -#else - uint16_t mn_ksize; /* key size */ - uint8_t mn_extra; - uint8_t mn_flags; /* see mdbx_node flags */ - union { - uint32_t mn_pgno32; - uint32_t mn_dsize; - }; -#endif /* __BYTE_ORDER__ */ - - /* mdbx_node Flags */ -#define F_BIGDATA 0x01 /* data put on overflow page */ -#define F_SUBDATA 0x02 /* data is a sub-database */ -#define F_DUPDATA 0x04 /* data has duplicates */ +#define EVEN_CEIL(n) (((n) + 1UL) & -2L) /* sign-extending -2 to match n+1U */ - /* valid flags for mdbx_node_add() */ -#define NODE_ADD_FLAGS (F_DUPDATA | F_SUBDATA | MDBX_RESERVE | MDBX_APPEND) - -#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ - (!defined(__cplusplus) && defined(_MSC_VER)) - uint8_t mn_data[] /* key and data are appended here */; -#endif /* C99 */ -} MDBX_node; - -#define DB_PERSISTENT_FLAGS \ - (MDBX_REVERSEKEY | MDBX_DUPSORT | MDBX_INTEGERKEY | MDBX_DUPFIXED | \ - MDBX_INTEGERDUP | MDBX_REVERSEDUP) - -/* mdbx_dbi_open() flags */ -#define DB_USABLE_FLAGS (DB_PERSISTENT_FLAGS | MDBX_CREATE | MDBX_DB_ACCEDE) - -#define DB_VALID 0x8000 /* DB handle is valid, for me_dbflags */ -#define DB_INTERNAL_FLAGS DB_VALID - -#if DB_INTERNAL_FLAGS & DB_USABLE_FLAGS -#error "Oops, some flags overlapped or wrong" -#endif -#if DB_PERSISTENT_FLAGS & ~DB_USABLE_FLAGS -#error "Oops, some flags overlapped or wrong" -#endif - -/* Max length of iov-vector passed to writev() call, used for auxilary writes */ -#define MDBX_AUXILARY_IOV_MAX 64 -#if defined(IOV_MAX) && IOV_MAX < MDBX_AUXILARY_IOV_MAX -#undef MDBX_AUXILARY_IOV_MAX -#define MDBX_AUXILARY_IOV_MAX IOV_MAX -#endif /* MDBX_AUXILARY_IOV_MAX */ +/* Round n down to an even number. */ +#define EVEN_FLOOR(n) ((n) & ~(size_t)1) /* * / @@ -3970,106 +3007,226 @@ typedef struct MDBX_node { */ #define CMP2INT(a, b) (((a) != (b)) ? (((a) < (b)) ? -1 : 1) : 0) -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline pgno_t -int64pgno(int64_t i64) { - if (likely(i64 >= (int64_t)MIN_PAGENO && i64 <= (int64_t)MAX_PAGENO + 1)) - return (pgno_t)i64; - return (i64 < (int64_t)MIN_PAGENO) ? MIN_PAGENO : MAX_PAGENO; -} +/* Pointer displacement without casting to char* to avoid pointer-aliasing */ +#define ptr_disp(ptr, disp) ((void *)(((intptr_t)(ptr)) + ((intptr_t)(disp)))) -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline pgno_t -pgno_add(size_t base, size_t augend) { - assert(base <= MAX_PAGENO + 1 && augend < MAX_PAGENO); - return int64pgno((int64_t)base + (int64_t)augend); -} +/* Pointer distance as signed number of bytes */ +#define ptr_dist(more, less) (((intptr_t)(more)) - ((intptr_t)(less))) -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline pgno_t -pgno_sub(size_t base, size_t subtrahend) { - assert(base >= MIN_PAGENO && base <= MAX_PAGENO + 1 && - subtrahend < MAX_PAGENO); - return int64pgno((int64_t)base - (int64_t)subtrahend); -} +#define MDBX_ASAN_POISON_MEMORY_REGION(addr, size) \ + do { \ + TRACE("POISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), (size_t)(size), __LINE__); \ + ASAN_POISON_MEMORY_REGION(addr, size); \ + } while (0) + +#define MDBX_ASAN_UNPOISON_MEMORY_REGION(addr, size) \ + do { \ + TRACE("UNPOISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), (size_t)(size), __LINE__); \ + ASAN_UNPOISON_MEMORY_REGION(addr, size); \ + } while (0) -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __always_inline bool -is_powerof2(size_t x) { - return (x & (x - 1)) == 0; +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline size_t branchless_abs(intptr_t value) { + assert(value > INT_MIN); + const size_t expanded_sign = (size_t)(value >> (sizeof(value) * CHAR_BIT - 1)); + return ((size_t)value + expanded_sign) ^ expanded_sign; } -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __always_inline size_t -floor_powerof2(size_t value, size_t granularity) { +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline bool is_powerof2(size_t x) { return (x & (x - 1)) == 0; } + +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline size_t floor_powerof2(size_t value, size_t granularity) { assert(is_powerof2(granularity)); return value & ~(granularity - 1); } -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __always_inline size_t -ceil_powerof2(size_t value, size_t granularity) { +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline size_t ceil_powerof2(size_t value, size_t granularity) { return floor_powerof2(value + granularity - 1, granularity); } -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static unsigned -log2n_powerof2(size_t value_uintptr) { - assert(value_uintptr > 0 && value_uintptr < INT32_MAX && - is_powerof2(value_uintptr)); - assert((value_uintptr & -(intptr_t)value_uintptr) == value_uintptr); - const uint32_t value_uint32 = (uint32_t)value_uintptr; -#if __GNUC_PREREQ(4, 1) || __has_builtin(__builtin_ctz) - STATIC_ASSERT(sizeof(value_uint32) <= sizeof(unsigned)); - return __builtin_ctz(value_uint32); -#elif defined(_MSC_VER) - unsigned long index; - STATIC_ASSERT(sizeof(value_uint32) <= sizeof(long)); - _BitScanForward(&index, value_uint32); - return index; +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED MDBX_INTERNAL unsigned log2n_powerof2(size_t value_uintptr); + +MDBX_NOTHROW_CONST_FUNCTION MDBX_INTERNAL uint64_t rrxmrrxmsx_0(uint64_t v); + +struct monotime_cache { + uint64_t value; + int expire_countdown; +}; + +MDBX_MAYBE_UNUSED static inline uint64_t monotime_since_cached(uint64_t begin_timestamp, struct monotime_cache *cache) { + if (cache->expire_countdown) + cache->expire_countdown -= 1; + else { + cache->value = osal_monotime(); + cache->expire_countdown = 42 / 3; + } + return cache->value - begin_timestamp; +} + +/* An PNL is an Page Number List, a sorted array of IDs. + * + * The first element of the array is a counter for how many actual page-numbers + * are in the list. By default PNLs are sorted in descending order, this allow + * cut off a page with lowest pgno (at the tail) just truncating the list. The + * sort order of PNLs is controlled by the MDBX_PNL_ASCENDING build option. */ +typedef pgno_t *pnl_t; +typedef const pgno_t *const_pnl_t; + +#if MDBX_PNL_ASCENDING +#define MDBX_PNL_ORDERED(first, last) ((first) < (last)) +#define MDBX_PNL_DISORDERED(first, last) ((first) >= (last)) #else - static const uint8_t debruijn_ctz32[32] = { - 0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8, - 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9}; - return debruijn_ctz32[(uint32_t)(value_uint32 * 0x077CB531ul) >> 27]; +#define MDBX_PNL_ORDERED(first, last) ((first) > (last)) +#define MDBX_PNL_DISORDERED(first, last) ((first) <= (last)) +#endif + +#define MDBX_PNL_GRANULATE_LOG2 10 +#define MDBX_PNL_GRANULATE (1 << MDBX_PNL_GRANULATE_LOG2) +#define MDBX_PNL_INITIAL (MDBX_PNL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t)) + +#define MDBX_PNL_ALLOCLEN(pl) ((pl)[-1]) +#define MDBX_PNL_GETSIZE(pl) ((size_t)((pl)[0])) +#define MDBX_PNL_SETSIZE(pl, size) \ + do { \ + const size_t __size = size; \ + assert(__size < INT_MAX); \ + (pl)[0] = (pgno_t)__size; \ + } while (0) +#define MDBX_PNL_FIRST(pl) ((pl)[1]) +#define MDBX_PNL_LAST(pl) ((pl)[MDBX_PNL_GETSIZE(pl)]) +#define MDBX_PNL_BEGIN(pl) (&(pl)[1]) +#define MDBX_PNL_END(pl) (&(pl)[MDBX_PNL_GETSIZE(pl) + 1]) + +#if MDBX_PNL_ASCENDING +#define MDBX_PNL_EDGE(pl) ((pl) + 1) +#define MDBX_PNL_LEAST(pl) MDBX_PNL_FIRST(pl) +#define MDBX_PNL_MOST(pl) MDBX_PNL_LAST(pl) +#else +#define MDBX_PNL_EDGE(pl) ((pl) + MDBX_PNL_GETSIZE(pl)) +#define MDBX_PNL_LEAST(pl) MDBX_PNL_LAST(pl) +#define MDBX_PNL_MOST(pl) MDBX_PNL_FIRST(pl) #endif + +#define MDBX_PNL_SIZEOF(pl) ((MDBX_PNL_GETSIZE(pl) + 1) * sizeof(pgno_t)) +#define MDBX_PNL_IS_EMPTY(pl) (MDBX_PNL_GETSIZE(pl) == 0) + +MDBX_MAYBE_UNUSED static inline size_t pnl_size2bytes(size_t size) { + assert(size > 0 && size <= PAGELIST_LIMIT); +#if MDBX_PNL_PREALLOC_FOR_RADIXSORT + + size += size; +#endif /* MDBX_PNL_PREALLOC_FOR_RADIXSORT */ + STATIC_ASSERT(MDBX_ASSUME_MALLOC_OVERHEAD + + (PAGELIST_LIMIT * (MDBX_PNL_PREALLOC_FOR_RADIXSORT + 1) + MDBX_PNL_GRANULATE + 3) * sizeof(pgno_t) < + SIZE_MAX / 4 * 3); + size_t bytes = + ceil_powerof2(MDBX_ASSUME_MALLOC_OVERHEAD + sizeof(pgno_t) * (size + 3), MDBX_PNL_GRANULATE * sizeof(pgno_t)) - + MDBX_ASSUME_MALLOC_OVERHEAD; + return bytes; +} + +MDBX_MAYBE_UNUSED static inline pgno_t pnl_bytes2size(const size_t bytes) { + size_t size = bytes / sizeof(pgno_t); + assert(size > 3 && size <= PAGELIST_LIMIT + /* alignment gap */ 65536); + size -= 3; +#if MDBX_PNL_PREALLOC_FOR_RADIXSORT + size >>= 1; +#endif /* MDBX_PNL_PREALLOC_FOR_RADIXSORT */ + return (pgno_t)size; +} + +MDBX_INTERNAL pnl_t pnl_alloc(size_t size); + +MDBX_INTERNAL void pnl_free(pnl_t pnl); + +MDBX_INTERNAL int pnl_reserve(pnl_t __restrict *__restrict ppnl, const size_t wanna); + +MDBX_MAYBE_UNUSED static inline int __must_check_result pnl_need(pnl_t __restrict *__restrict ppnl, size_t num) { + assert(MDBX_PNL_GETSIZE(*ppnl) <= PAGELIST_LIMIT && MDBX_PNL_ALLOCLEN(*ppnl) >= MDBX_PNL_GETSIZE(*ppnl)); + assert(num <= PAGELIST_LIMIT); + const size_t wanna = MDBX_PNL_GETSIZE(*ppnl) + num; + return likely(MDBX_PNL_ALLOCLEN(*ppnl) >= wanna) ? MDBX_SUCCESS : pnl_reserve(ppnl, wanna); } -/* Only a subset of the mdbx_env flags can be changed - * at runtime. Changing other flags requires closing the - * environment and re-opening it with the new flags. */ -#define ENV_CHANGEABLE_FLAGS \ - (MDBX_SAFE_NOSYNC | MDBX_NOMETASYNC | MDBX_DEPRECATED_MAPASYNC | \ - MDBX_NOMEMINIT | MDBX_COALESCE | MDBX_PAGEPERTURB | MDBX_ACCEDE | \ - MDBX_VALIDATION) -#define ENV_CHANGELESS_FLAGS \ - (MDBX_NOSUBDIR | MDBX_RDONLY | MDBX_WRITEMAP | MDBX_NOTLS | MDBX_NORDAHEAD | \ - MDBX_LIFORECLAIM | MDBX_EXCLUSIVE) -#define ENV_USABLE_FLAGS (ENV_CHANGEABLE_FLAGS | ENV_CHANGELESS_FLAGS) - -#if !defined(__cplusplus) || CONSTEXPR_ENUM_FLAGS_OPERATIONS -MDBX_MAYBE_UNUSED static void static_checks(void) { - STATIC_ASSERT_MSG(INT16_MAX - CORE_DBS == MDBX_MAX_DBI, - "Oops, MDBX_MAX_DBI or CORE_DBS?"); - STATIC_ASSERT_MSG((unsigned)(MDBX_DB_ACCEDE | MDBX_CREATE) == - ((DB_USABLE_FLAGS | DB_INTERNAL_FLAGS) & - (ENV_USABLE_FLAGS | ENV_INTERNAL_FLAGS)), - "Oops, some flags overlapped or wrong"); - STATIC_ASSERT_MSG((ENV_INTERNAL_FLAGS & ENV_USABLE_FLAGS) == 0, - "Oops, some flags overlapped or wrong"); +MDBX_MAYBE_UNUSED static inline void pnl_append_prereserved(__restrict pnl_t pnl, pgno_t pgno) { + assert(MDBX_PNL_GETSIZE(pnl) < MDBX_PNL_ALLOCLEN(pnl)); + if (AUDIT_ENABLED()) { + for (size_t i = MDBX_PNL_GETSIZE(pnl); i > 0; --i) + assert(pgno != pnl[i]); + } + *pnl += 1; + MDBX_PNL_LAST(pnl) = pgno; } -#endif /* Disabled for MSVC 19.0 (VisualStudio 2015) */ + +MDBX_INTERNAL void pnl_shrink(pnl_t __restrict *__restrict ppnl); + +MDBX_INTERNAL int __must_check_result spill_append_span(__restrict pnl_t *ppnl, pgno_t pgno, size_t n); + +MDBX_INTERNAL int __must_check_result pnl_append_span(__restrict pnl_t *ppnl, pgno_t pgno, size_t n); + +MDBX_INTERNAL int __must_check_result pnl_insert_span(__restrict pnl_t *ppnl, pgno_t pgno, size_t n); + +MDBX_INTERNAL size_t pnl_search_nochk(const pnl_t pnl, pgno_t pgno); + +MDBX_INTERNAL void pnl_sort_nochk(pnl_t pnl); + +MDBX_INTERNAL bool pnl_check(const const_pnl_t pnl, const size_t limit); + +MDBX_MAYBE_UNUSED static inline bool pnl_check_allocated(const const_pnl_t pnl, const size_t limit) { + return pnl == nullptr || (MDBX_PNL_ALLOCLEN(pnl) >= MDBX_PNL_GETSIZE(pnl) && pnl_check(pnl, limit)); +} + +MDBX_MAYBE_UNUSED static inline void pnl_sort(pnl_t pnl, size_t limit4check) { + pnl_sort_nochk(pnl); + assert(pnl_check(pnl, limit4check)); + (void)limit4check; +} + +MDBX_MAYBE_UNUSED static inline size_t pnl_search(const pnl_t pnl, pgno_t pgno, size_t limit) { + assert(pnl_check_allocated(pnl, limit)); + if (MDBX_HAVE_CMOV) { + /* cmov-ускоренный бинарный поиск может читать (но не использовать) один + * элемент за концом данных, этот элемент в пределах выделенного участка + * памяти, но не инициализирован. */ + VALGRIND_MAKE_MEM_DEFINED(MDBX_PNL_END(pnl), sizeof(pgno_t)); + } + assert(pgno < limit); + (void)limit; + size_t n = pnl_search_nochk(pnl, pgno); + if (MDBX_HAVE_CMOV) { + VALGRIND_MAKE_MEM_UNDEFINED(MDBX_PNL_END(pnl), sizeof(pgno_t)); + } + return n; +} + +MDBX_INTERNAL size_t pnl_merge(pnl_t dst, const pnl_t src); #ifdef __cplusplus } +#endif /* __cplusplus */ + +#define mdbx_sourcery_anchor XCONCAT(mdbx_sourcery_, MDBX_BUILD_SOURCERY) +#if defined(xMDBX_TOOLS) +extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #endif -#define MDBX_ASAN_POISON_MEMORY_REGION(addr, size) \ - do { \ - TRACE("POISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), \ - (size_t)(size), __LINE__); \ - ASAN_POISON_MEMORY_REGION(addr, size); \ - } while (0) +#define MDBX_IS_ERROR(rc) ((rc) != MDBX_RESULT_TRUE && (rc) != MDBX_RESULT_FALSE) -#define MDBX_ASAN_UNPOISON_MEMORY_REGION(addr, size) \ - do { \ - TRACE("UNPOISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), \ - (size_t)(size), __LINE__); \ - ASAN_UNPOISON_MEMORY_REGION(addr, size); \ - } while (0) +/*----------------------------------------------------------------------------*/ + +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline pgno_t int64pgno(int64_t i64) { + if (likely(i64 >= (int64_t)MIN_PAGENO && i64 <= (int64_t)MAX_PAGENO + 1)) + return (pgno_t)i64; + return (i64 < (int64_t)MIN_PAGENO) ? MIN_PAGENO : MAX_PAGENO; +} + +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline pgno_t pgno_add(size_t base, size_t augend) { + assert(base <= MAX_PAGENO + 1 && augend < MAX_PAGENO); + return int64pgno((int64_t)base + (int64_t)augend); +} + +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline pgno_t pgno_sub(size_t base, size_t subtrahend) { + assert(base >= MIN_PAGENO && base <= MAX_PAGENO + 1 && subtrahend < MAX_PAGENO); + return int64pgno((int64_t)base - (int64_t)subtrahend); +} #include @@ -4085,7 +3242,7 @@ typedef struct flagbit { flagbit dbflags[] = {{MDBX_REVERSEKEY, "reversekey"}, {MDBX_DUPSORT, "dupsort"}, {MDBX_INTEGERKEY, "integerkey"}, - {MDBX_DUPFIXED, "dupfixed"}, + {MDBX_DUPFIXED, "dupfix"}, {MDBX_INTEGERDUP, "integerdup"}, {MDBX_REVERSEDUP, "reversedup"}, {0, nullptr}}; @@ -4104,12 +3261,12 @@ flagbit dbflags[] = {{MDBX_REVERSEKEY, "reversekey"}, #ifdef _MSC_VER #pragma warning(push, 1) -#pragma warning(disable : 4548) /* expression before comma has no effect; \ +#pragma warning(disable : 4548) /* expression before comma has no effect; \ expected expression with side - effect */ -#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ +#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ * semantics are not enabled. Specify /EHsc */ -#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ - * mode specified; termination on exception is \ +#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ + * mode specified; termination on exception is \ * not guaranteed. Specify /EHsc */ #if !defined(_CRT_SECURE_NO_WARNINGS) #define _CRT_SECURE_NO_WARNINGS @@ -4162,8 +3319,7 @@ int getopt(int argc, char *const argv[], const char *opts) { if (argv[optind][sp + 1] != '\0') optarg = &argv[optind++][sp + 1]; else if (++optind >= argc) { - fprintf(stderr, "%s: %s -- %c\n", argv[0], "option requires an argument", - c); + fprintf(stderr, "%s: %s -- %c\n", argv[0], "option requires an argument", c); sp = 1; return '?'; } else @@ -4236,13 +3392,12 @@ bool quiet = false, rescue = false; const char *prog; static void error(const char *func, int rc) { if (!quiet) - fprintf(stderr, "%s: %s() error %d %s\n", prog, func, rc, - mdbx_strerror(rc)); + fprintf(stderr, "%s: %s() error %d %s\n", prog, func, rc, mdbx_strerror(rc)); } /* Dump in BDB-compatible format */ -static int dump_sdb(MDBX_txn *txn, MDBX_dbi dbi, char *name) { - unsigned int flags; +static int dump_tbl(MDBX_txn *txn, MDBX_dbi dbi, char *name) { + unsigned flags; int rc = mdbx_dbi_flags(txn, dbi, &flags); if (unlikely(rc != MDBX_SUCCESS)) { error("mdbx_dbi_flags", rc); @@ -4267,10 +3422,8 @@ static int dump_sdb(MDBX_txn *txn, MDBX_dbi dbi, char *name) { if (mode & GLOBAL) { mode -= GLOBAL; if (info.mi_geo.upper != info.mi_geo.lower) - printf("geometry=l%" PRIu64 ",c%" PRIu64 ",u%" PRIu64 ",s%" PRIu64 - ",g%" PRIu64 "\n", - info.mi_geo.lower, info.mi_geo.current, info.mi_geo.upper, - info.mi_geo.shrink, info.mi_geo.grow); + printf("geometry=l%" PRIu64 ",c%" PRIu64 ",u%" PRIu64 ",s%" PRIu64 ",g%" PRIu64 "\n", info.mi_geo.lower, + info.mi_geo.current, info.mi_geo.upper, info.mi_geo.shrink, info.mi_geo.grow); printf("mapsize=%" PRIu64 "\n", info.mi_geo.upper); printf("maxreaders=%u\n", info.mi_maxreaders); @@ -4281,8 +3434,7 @@ static int dump_sdb(MDBX_txn *txn, MDBX_dbi dbi, char *name) { return rc; } if (canary.v) - printf("canary=v%" PRIu64 ",x%" PRIu64 ",y%" PRIu64 ",z%" PRIu64 "\n", - canary.v, canary.x, canary.y, canary.z); + printf("canary=v%" PRIu64 ",x%" PRIu64 ",y%" PRIu64 ",z%" PRIu64 "\n", canary.v, canary.x, canary.y, canary.z); } printf("format=%s\n", mode & PRINT ? "print" : "bytevalue"); if (name) @@ -4294,10 +3446,7 @@ static int dump_sdb(MDBX_txn *txn, MDBX_dbi dbi, char *name) { else if (!name) printf("txnid=%" PRIaTXN "\n", mdbx_txn_id(txn)); */ - printf("duplicates=%d\n", (flags & (MDBX_DUPSORT | MDBX_DUPFIXED | - MDBX_INTEGERDUP | MDBX_REVERSEDUP)) - ? 1 - : 0); + printf("duplicates=%d\n", (flags & (MDBX_DUPSORT | MDBX_DUPFIXED | MDBX_INTEGERDUP | MDBX_REVERSEDUP)) ? 1 : 0); for (int i = 0; dbflags[i].bit; i++) if (flags & dbflags[i].bit) printf("%s=1\n", dbflags[i].name); @@ -4321,13 +3470,14 @@ static int dump_sdb(MDBX_txn *txn, MDBX_dbi dbi, char *name) { return rc; } if (rescue) { - cursor->mc_checking |= CC_SKIPORD; - if (cursor->mc_xcursor) - cursor->mc_xcursor->mx_cursor.mc_checking |= CC_SKIPORD; + rc = mdbx_cursor_ignord(cursor); + if (unlikely(rc != MDBX_SUCCESS)) { + error("mdbx_cursor_ignord", rc); + return rc; + } } - while ((rc = mdbx_cursor_get(cursor, &key, &data, MDBX_NEXT)) == - MDBX_SUCCESS) { + while ((rc = mdbx_cursor_get(cursor, &key, &data, MDBX_NEXT)) == MDBX_SUCCESS) { if (user_break) { rc = MDBX_EINTR; break; @@ -4351,35 +3501,46 @@ static int dump_sdb(MDBX_txn *txn, MDBX_dbi dbi, char *name) { } static void usage(void) { - fprintf( - stderr, - "usage: %s " - "[-V] [-q] [-f file] [-l] [-p] [-r] [-a|-s subdb] [-u|U] " - "dbpath\n" - " -V\t\tprint version and exit\n" - " -q\t\tbe quiet\n" - " -f\t\twrite to file instead of stdout\n" - " -l\t\tlist subDBs and exit\n" - " -p\t\tuse printable characters\n" - " -r\t\trescue mode (ignore errors to dump corrupted DB)\n" - " -a\t\tdump main DB and all subDBs\n" - " -s name\tdump only the specified named subDB\n" - " -u\t\twarmup database before dumping\n" - " -U\t\twarmup and try lock database pages in memory before dumping\n" - " \t\tby default dump only the main DB\n", - prog); + fprintf(stderr, + "usage: %s " + "[-V] [-q] [-f file] [-l] [-p] [-r] [-a|-s table] [-u|U] " + "dbpath\n" + " -V\t\tprint version and exit\n" + " -q\t\tbe quiet\n" + " -f\t\twrite to file instead of stdout\n" + " -l\t\tlist tables and exit\n" + " -p\t\tuse printable characters\n" + " -r\t\trescue mode (ignore errors to dump corrupted DB)\n" + " -a\t\tdump main DB and all tables\n" + " -s name\tdump only the specified named table\n" + " -u\t\twarmup database before dumping\n" + " -U\t\twarmup and try lock database pages in memory before dumping\n" + " \t\tby default dump only the main DB\n", + prog); exit(EXIT_FAILURE); } +static void logger(MDBX_log_level_t level, const char *function, int line, const char *fmt, va_list args) { + static const char *const prefixes[] = { + "!!!fatal: ", // 0 fatal + " ! ", // 1 error + " ~ ", // 2 warning + " ", // 3 notice + " //", // 4 verbose + }; + if (level < MDBX_LOG_DEBUG) { + if (function && line) + fprintf(stderr, "%s", prefixes[level]); + vfprintf(stderr, fmt, args); + } +} + static int equal_or_greater(const MDBX_val *a, const MDBX_val *b) { - return (a->iov_len == b->iov_len && - memcmp(a->iov_base, b->iov_base, a->iov_len) == 0) - ? 0 - : 1; + return (a->iov_len == b->iov_len && memcmp(a->iov_base, b->iov_base, a->iov_len) == 0) ? 0 : 1; } int main(int argc, char *argv[]) { - int i, rc; + int i, err; MDBX_env *env; MDBX_txn *txn; MDBX_dbi dbi; @@ -4413,12 +3574,9 @@ int main(int argc, char *argv[]) { " - build: %s for %s by %s\n" " - flags: %s\n" " - options: %s\n", - mdbx_version.major, mdbx_version.minor, mdbx_version.release, - mdbx_version.revision, mdbx_version.git.describe, - mdbx_version.git.datetime, mdbx_version.git.commit, - mdbx_version.git.tree, mdbx_sourcery_anchor, mdbx_build.datetime, - mdbx_build.target, mdbx_build.compiler, mdbx_build.flags, - mdbx_build.options); + mdbx_version.major, mdbx_version.minor, mdbx_version.patch, mdbx_version.tweak, mdbx_version.git.describe, + mdbx_version.git.datetime, mdbx_version.git.commit, mdbx_version.git.tree, mdbx_sourcery_anchor, + mdbx_build.datetime, mdbx_build.target, mdbx_build.compiler, mdbx_build.flags, mdbx_build.options); return EXIT_SUCCESS; case 'l': list = true; @@ -4431,8 +3589,7 @@ int main(int argc, char *argv[]) { break; case 'f': if (freopen(optarg, "w", stdout) == nullptr) { - fprintf(stderr, "%s: %s: reopen: %s\n", prog, optarg, - mdbx_strerror(errno)); + fprintf(stderr, "%s: %s: reopen: %s\n", prog, optarg, mdbx_strerror(errno)); exit(EXIT_FAILURE); } break; @@ -4457,8 +3614,7 @@ int main(int argc, char *argv[]) { break; case 'U': warmup = true; - warmup_flags = - MDBX_warmup_force | MDBX_warmup_touchlimit | MDBX_warmup_lock; + warmup_flags = MDBX_warmup_force | MDBX_warmup_touchlimit | MDBX_warmup_lock; break; default: usage(); @@ -4483,53 +3639,50 @@ int main(int argc, char *argv[]) { envname = argv[optind]; if (!quiet) { - fprintf(stderr, "mdbx_dump %s (%s, T-%s)\nRunning for %s...\n", - mdbx_version.git.describe, mdbx_version.git.datetime, - mdbx_version.git.tree, envname); + fprintf(stderr, "mdbx_dump %s (%s, T-%s)\nRunning for %s...\n", mdbx_version.git.describe, + mdbx_version.git.datetime, mdbx_version.git.tree, envname); fflush(nullptr); + mdbx_setup_debug(MDBX_LOG_NOTICE, MDBX_DBG_DONTCHANGE, logger); } - rc = mdbx_env_create(&env); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_env_create", rc); + err = mdbx_env_create(&env); + if (unlikely(err != MDBX_SUCCESS)) { + error("mdbx_env_create", err); return EXIT_FAILURE; } if (alldbs || subname) { - rc = mdbx_env_set_maxdbs(env, 2); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_env_set_maxdbs", rc); + err = mdbx_env_set_maxdbs(env, 2); + if (unlikely(err != MDBX_SUCCESS)) { + error("mdbx_env_set_maxdbs", err); goto env_close; } } - rc = mdbx_env_open( - env, envname, - envflags | (rescue ? MDBX_RDONLY | MDBX_EXCLUSIVE | MDBX_VALIDATION - : MDBX_RDONLY), - 0); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_env_open", rc); + err = mdbx_env_open(env, envname, envflags | (rescue ? MDBX_RDONLY | MDBX_EXCLUSIVE | MDBX_VALIDATION : MDBX_RDONLY), + 0); + if (unlikely(err != MDBX_SUCCESS)) { + error("mdbx_env_open", err); goto env_close; } if (warmup) { - rc = mdbx_env_warmup(env, nullptr, warmup_flags, 3600 * 65536); - if (MDBX_IS_ERROR(rc)) { - error("mdbx_env_warmup", rc); + err = mdbx_env_warmup(env, nullptr, warmup_flags, 3600 * 65536); + if (MDBX_IS_ERROR(err)) { + error("mdbx_env_warmup", err); goto env_close; } } - rc = mdbx_txn_begin(env, nullptr, MDBX_TXN_RDONLY, &txn); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_txn_begin", rc); + err = mdbx_txn_begin(env, nullptr, MDBX_TXN_RDONLY, &txn); + if (unlikely(err != MDBX_SUCCESS)) { + error("mdbx_txn_begin", err); goto env_close; } - rc = mdbx_dbi_open(txn, subname, MDBX_DB_ACCEDE, &dbi); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_dbi_open", rc); + err = mdbx_dbi_open(txn, subname, MDBX_DB_ACCEDE, &dbi); + if (unlikely(err != MDBX_SUCCESS)) { + error("mdbx_dbi_open", err); goto txn_abort; } @@ -4537,24 +3690,25 @@ int main(int argc, char *argv[]) { assert(dbi == MAIN_DBI); MDBX_cursor *cursor; - rc = mdbx_cursor_open(txn, MAIN_DBI, &cursor); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_cursor_open", rc); + err = mdbx_cursor_open(txn, MAIN_DBI, &cursor); + if (unlikely(err != MDBX_SUCCESS)) { + error("mdbx_cursor_open", err); goto txn_abort; } if (rescue) { - cursor->mc_checking |= CC_SKIPORD; - if (cursor->mc_xcursor) - cursor->mc_xcursor->mx_cursor.mc_checking |= CC_SKIPORD; + err = mdbx_cursor_ignord(cursor); + if (unlikely(err != MDBX_SUCCESS)) { + error("mdbx_cursor_ignord", err); + return err; + } } bool have_raw = false; int count = 0; MDBX_val key; - while (MDBX_SUCCESS == - (rc = mdbx_cursor_get(cursor, &key, nullptr, MDBX_NEXT_NODUP))) { + while (MDBX_SUCCESS == (err = mdbx_cursor_get(cursor, &key, nullptr, MDBX_NEXT_NODUP))) { if (user_break) { - rc = MDBX_EINTR; + err = MDBX_EINTR; break; } @@ -4562,7 +3716,7 @@ int main(int argc, char *argv[]) { continue; subname = osal_realloc(buf4free, key.iov_len + 1); if (!subname) { - rc = MDBX_ENOMEM; + err = MDBX_ENOMEM; break; } @@ -4571,15 +3725,14 @@ int main(int argc, char *argv[]) { subname[key.iov_len] = '\0'; MDBX_dbi sub_dbi; - rc = mdbx_dbi_open_ex(txn, subname, MDBX_DB_ACCEDE, &sub_dbi, - rescue ? equal_or_greater : nullptr, - rescue ? equal_or_greater : nullptr); - if (unlikely(rc != MDBX_SUCCESS)) { - if (rc == MDBX_INCOMPATIBLE) { + err = mdbx_dbi_open_ex(txn, subname, MDBX_DB_ACCEDE, &sub_dbi, rescue ? equal_or_greater : nullptr, + rescue ? equal_or_greater : nullptr); + if (unlikely(err != MDBX_SUCCESS)) { + if (err == MDBX_INCOMPATIBLE) { have_raw = true; continue; } - error("mdbx_dbi_open", rc); + error("mdbx_dbi_open", err); if (!rescue) break; } else { @@ -4587,13 +3740,12 @@ int main(int argc, char *argv[]) { if (list) { printf("%s\n", subname); } else { - rc = dump_sdb(txn, sub_dbi, subname); - if (unlikely(rc != MDBX_SUCCESS)) { + err = dump_tbl(txn, sub_dbi, subname); + if (unlikely(err != MDBX_SUCCESS)) { if (!rescue) break; if (!quiet) - fprintf(stderr, "%s: %s: ignore %s for `%s` and continue\n", prog, - envname, mdbx_strerror(rc), subname); + fprintf(stderr, "%s: %s: ignore %s for `%s` and continue\n", prog, envname, mdbx_strerror(err), subname); /* Here is a hack for rescue mode, don't do that: * - we should restart transaction in case error due * database corruption; @@ -4602,21 +3754,21 @@ int main(int argc, char *argv[]) { * - this is possible since DB is opened in read-only exclusive * mode and transaction is the same, i.e. has the same address * and so on. */ - rc = mdbx_txn_reset(txn); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_txn_reset", rc); + err = mdbx_txn_reset(txn); + if (unlikely(err != MDBX_SUCCESS)) { + error("mdbx_txn_reset", err); goto env_close; } - rc = mdbx_txn_renew(txn); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_txn_renew", rc); + err = mdbx_txn_renew(txn); + if (unlikely(err != MDBX_SUCCESS)) { + error("mdbx_txn_renew", err); goto env_close; } } } - rc = mdbx_dbi_close(env, sub_dbi); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_dbi_close", rc); + err = mdbx_dbi_close(env, sub_dbi); + if (unlikely(err != MDBX_SUCCESS)) { + error("mdbx_dbi_close", err); break; } } @@ -4625,20 +3777,19 @@ int main(int argc, char *argv[]) { cursor = nullptr; if (have_raw && (!count /* || rescue */)) - rc = dump_sdb(txn, MAIN_DBI, nullptr); + err = dump_tbl(txn, MAIN_DBI, nullptr); else if (!count) { if (!quiet) - fprintf(stderr, "%s: %s does not contain multiple databases\n", prog, - envname); - rc = MDBX_NOTFOUND; + fprintf(stderr, "%s: %s does not contain multiple databases\n", prog, envname); + err = MDBX_NOTFOUND; } } else { - rc = dump_sdb(txn, dbi, subname); + err = dump_tbl(txn, dbi, subname); } - switch (rc) { + switch (err) { case MDBX_NOTFOUND: - rc = MDBX_SUCCESS; + err = MDBX_SUCCESS; case MDBX_SUCCESS: break; case MDBX_EINTR: @@ -4646,8 +3797,8 @@ int main(int argc, char *argv[]) { fprintf(stderr, "Interrupted by signal/user\n"); break; default: - if (unlikely(rc != MDBX_SUCCESS)) - error("mdbx_cursor_get", rc); + if (unlikely(err != MDBX_SUCCESS)) + error("mdbx_cursor_get", err); } mdbx_dbi_close(env, dbi); @@ -4657,5 +3808,5 @@ int main(int argc, char *argv[]) { mdbx_env_close(env); free(buf4free); - return rc ? EXIT_FAILURE : EXIT_SUCCESS; + return err ? EXIT_FAILURE : EXIT_SUCCESS; } diff --git a/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx_load.c b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx_load.c index 45a7b348455..0c1ceba53c2 100644 --- a/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx_load.c +++ b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx_load.c @@ -1,18 +1,12 @@ -/* mdbx_load.c - memory-mapped database load tool */ - -/* - * Copyright 2015-2024 Leonid Yuriev - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . */ - +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \note Please refer to the COPYRIGHT file for explanations license change, +/// credits and acknowledgments. +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 +/// +/// mdbx_load.c - memory-mapped database load tool +/// + +/* clang-format off */ #ifdef _MSC_VER #if _MSC_VER > 1800 #pragma warning(disable : 4464) /* relative include path contains '..' */ @@ -21,38 +15,26 @@ #endif /* _MSC_VER (warnings) */ #define xMDBX_TOOLS /* Avoid using internal eASSERT() */ -/* - * Copyright 2015-2024 Leonid Yuriev - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . */ +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 -#define MDBX_BUILD_SOURCERY e156c1a97c017ce89d6541cd9464ae5a9761d76b3fd2f1696521f5f3792904fc_v0_12_13_0_g1fff1f67 -#ifdef MDBX_CONFIG_H -#include MDBX_CONFIG_H -#endif +#define MDBX_BUILD_SOURCERY 6b5df6869d2bf5419e3a8189d9cc849cc9911b9c8a951b9750ed0a261ce43724_v0_13_7_0_g566b0f93 #define LIBMDBX_INTERNALS -#ifdef xMDBX_TOOLS #define MDBX_DEPRECATED -#endif /* xMDBX_TOOLS */ -#ifdef xMDBX_ALLOY -/* Amalgamated build */ -#define MDBX_INTERNAL_FUNC static -#define MDBX_INTERNAL_VAR static -#else -/* Non-amalgamated build */ -#define MDBX_INTERNAL_FUNC -#define MDBX_INTERNAL_VAR extern -#endif /* xMDBX_ALLOY */ +#ifdef MDBX_CONFIG_H +#include MDBX_CONFIG_H +#endif + +/* Undefine the NDEBUG if debugging is enforced by MDBX_DEBUG */ +#if (defined(MDBX_DEBUG) && MDBX_DEBUG > 0) || (defined(MDBX_FORCE_ASSERTIONS) && MDBX_FORCE_ASSERTIONS) +#undef NDEBUG +#ifndef MDBX_DEBUG +/* Чтобы избежать включения отладки только из-за включения assert-проверок */ +#define MDBX_DEBUG 0 +#endif +#endif /*----------------------------------------------------------------------------*/ @@ -70,14 +52,59 @@ #endif /* MDBX_DISABLE_GNU_SOURCE */ /* Should be defined before any includes */ -#if !defined(_FILE_OFFSET_BITS) && !defined(__ANDROID_API__) && \ - !defined(ANDROID) +#if !defined(_FILE_OFFSET_BITS) && !defined(__ANDROID_API__) && !defined(ANDROID) #define _FILE_OFFSET_BITS 64 -#endif +#endif /* _FILE_OFFSET_BITS */ -#ifdef __APPLE__ +#if defined(__APPLE__) && !defined(_DARWIN_C_SOURCE) #define _DARWIN_C_SOURCE -#endif +#endif /* _DARWIN_C_SOURCE */ + +#if (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) && !defined(__USE_MINGW_ANSI_STDIO) +#define __USE_MINGW_ANSI_STDIO 1 +#endif /* MinGW */ + +#if defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS) + +#ifndef _WIN32_WINNT +#define _WIN32_WINNT 0x0601 /* Windows 7 */ +#endif /* _WIN32_WINNT */ + +#if !defined(_CRT_SECURE_NO_WARNINGS) +#define _CRT_SECURE_NO_WARNINGS +#endif /* _CRT_SECURE_NO_WARNINGS */ +#if !defined(UNICODE) +#define UNICODE +#endif /* UNICODE */ + +#if !defined(_NO_CRT_STDIO_INLINE) && MDBX_BUILD_SHARED_LIBRARY && !defined(xMDBX_TOOLS) && MDBX_WITHOUT_MSVC_CRT +#define _NO_CRT_STDIO_INLINE +#endif /* _NO_CRT_STDIO_INLINE */ + +#elif !defined(_POSIX_C_SOURCE) +#define _POSIX_C_SOURCE 200809L +#endif /* Windows */ + +#ifdef __cplusplus + +#ifndef NOMINMAX +#define NOMINMAX +#endif /* NOMINMAX */ + +/* Workaround for modern libstdc++ with CLANG < 4.x */ +#if defined(__SIZEOF_INT128__) && !defined(__GLIBCXX_TYPE_INT_N_0) && defined(__clang__) && __clang_major__ < 4 +#define __GLIBCXX_BITSIZE_INT_N_0 128 +#define __GLIBCXX_TYPE_INT_N_0 __int128 +#endif /* Workaround for modern libstdc++ with CLANG < 4.x */ + +#ifdef _MSC_VER +/* Workaround for MSVC' header `extern "C"` vs `std::` redefinition bug */ +#if defined(__SANITIZE_ADDRESS__) && !defined(_DISABLE_VECTOR_ANNOTATION) +#define _DISABLE_VECTOR_ANNOTATION +#endif /* _DISABLE_VECTOR_ANNOTATION */ +#endif /* _MSC_VER */ + +#endif /* __cplusplus */ #ifdef _MSC_VER #if _MSC_FULL_VER < 190024234 @@ -99,12 +126,8 @@ * and how to and where you can obtain the latest "Visual Studio 2015" build * with all fixes. */ -#error \ - "At least \"Microsoft C/C++ Compiler\" version 19.00.24234 (Visual Studio 2015 Update 3) is required." +#error "At least \"Microsoft C/C++ Compiler\" version 19.00.24234 (Visual Studio 2015 Update 3) is required." #endif -#ifndef _CRT_SECURE_NO_WARNINGS -#define _CRT_SECURE_NO_WARNINGS -#endif /* _CRT_SECURE_NO_WARNINGS */ #if _MSC_VER > 1800 #pragma warning(disable : 4464) /* relative include path contains '..' */ #endif @@ -112,124 +135,78 @@ #pragma warning(disable : 5045) /* will insert Spectre mitigation... */ #endif #if _MSC_VER > 1914 -#pragma warning( \ - disable : 5105) /* winbase.h(9531): warning C5105: macro expansion \ - producing 'defined' has undefined behavior */ +#pragma warning(disable : 5105) /* winbase.h(9531): warning C5105: macro expansion \ + producing 'defined' has undefined behavior */ +#endif +#if _MSC_VER < 1920 +/* avoid "error C2219: syntax error: type qualifier must be after '*'" */ +#define __restrict #endif #if _MSC_VER > 1930 #pragma warning(disable : 6235) /* is always a constant */ -#pragma warning(disable : 6237) /* is never evaluated and might \ +#pragma warning(disable : 6237) /* is never evaluated and might \ have side effects */ +#pragma warning(disable : 5286) /* implicit conversion from enum type 'type 1' to enum type 'type 2' */ +#pragma warning(disable : 5287) /* operands are different enum types 'type 1' and 'type 2' */ #endif #pragma warning(disable : 4710) /* 'xyz': function not inlined */ -#pragma warning(disable : 4711) /* function 'xyz' selected for automatic \ +#pragma warning(disable : 4711) /* function 'xyz' selected for automatic \ inline expansion */ -#pragma warning(disable : 4201) /* nonstandard extension used: nameless \ +#pragma warning(disable : 4201) /* nonstandard extension used: nameless \ struct/union */ #pragma warning(disable : 4702) /* unreachable code */ #pragma warning(disable : 4706) /* assignment within conditional expression */ #pragma warning(disable : 4127) /* conditional expression is constant */ -#pragma warning(disable : 4324) /* 'xyz': structure was padded due to \ +#pragma warning(disable : 4324) /* 'xyz': structure was padded due to \ alignment specifier */ #pragma warning(disable : 4310) /* cast truncates constant value */ -#pragma warning(disable : 4820) /* bytes padding added after data member for \ +#pragma warning(disable : 4820) /* bytes padding added after data member for \ alignment */ -#pragma warning(disable : 4548) /* expression before comma has no effect; \ +#pragma warning(disable : 4548) /* expression before comma has no effect; \ expected expression with side - effect */ -#pragma warning(disable : 4366) /* the result of the unary '&' operator may be \ +#pragma warning(disable : 4366) /* the result of the unary '&' operator may be \ unaligned */ -#pragma warning(disable : 4200) /* nonstandard extension used: zero-sized \ +#pragma warning(disable : 4200) /* nonstandard extension used: zero-sized \ array in struct/union */ -#pragma warning(disable : 4204) /* nonstandard extension used: non-constant \ +#pragma warning(disable : 4204) /* nonstandard extension used: non-constant \ aggregate initializer */ -#pragma warning( \ - disable : 4505) /* unreferenced local function has been removed */ -#endif /* _MSC_VER (warnings) */ +#pragma warning(disable : 4505) /* unreferenced local function has been removed */ +#endif /* _MSC_VER (warnings) */ #if defined(__GNUC__) && __GNUC__ < 9 #pragma GCC diagnostic ignored "-Wattributes" #endif /* GCC < 9 */ -#if (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) && \ - !defined(__USE_MINGW_ANSI_STDIO) -#define __USE_MINGW_ANSI_STDIO 1 -#endif /* MinGW */ - -#if (defined(_WIN32) || defined(_WIN64)) && !defined(UNICODE) -#define UNICODE -#endif /* UNICODE */ - -#include "mdbx.h" -/* - * Copyright 2015-2024 Leonid Yuriev - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - */ - - /*----------------------------------------------------------------------------*/ /* Microsoft compiler generates a lot of warning for self includes... */ #ifdef _MSC_VER #pragma warning(push, 1) -#pragma warning(disable : 4548) /* expression before comma has no effect; \ +#pragma warning(disable : 4548) /* expression before comma has no effect; \ expected expression with side - effect */ -#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ +#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ * semantics are not enabled. Specify /EHsc */ -#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ - * mode specified; termination on exception is \ +#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ + * mode specified; termination on exception is \ * not guaranteed. Specify /EHsc */ #endif /* _MSC_VER (warnings) */ -#if defined(_WIN32) || defined(_WIN64) -#if !defined(_CRT_SECURE_NO_WARNINGS) -#define _CRT_SECURE_NO_WARNINGS -#endif /* _CRT_SECURE_NO_WARNINGS */ -#if !defined(_NO_CRT_STDIO_INLINE) && MDBX_BUILD_SHARED_LIBRARY && \ - !defined(xMDBX_TOOLS) && MDBX_WITHOUT_MSVC_CRT -#define _NO_CRT_STDIO_INLINE -#endif -#elif !defined(_POSIX_C_SOURCE) -#define _POSIX_C_SOURCE 200809L -#endif /* Windows */ - /*----------------------------------------------------------------------------*/ /* basic C99 includes */ + #include #include #include #include #include +#include #include #include #include #include #include -#if (-6 & 5) || CHAR_BIT != 8 || UINT_MAX < 0xffffffff || ULONG_MAX % 0xFFFF -#error \ - "Sanity checking failed: Two's complement, reasonably sized integer types" -#endif - -#ifndef SSIZE_MAX -#define SSIZE_MAX INTPTR_MAX -#endif - -#if UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul || defined(_WIN64) -#define MDBX_WORDBITS 64 -#else -#define MDBX_WORDBITS 32 -#endif /* MDBX_WORDBITS */ - /*----------------------------------------------------------------------------*/ /* feature testing */ @@ -241,6 +218,14 @@ #define __has_include(x) (0) #endif +#ifndef __has_attribute +#define __has_attribute(x) (0) +#endif + +#ifndef __has_cpp_attribute +#define __has_cpp_attribute(x) 0 +#endif + #ifndef __has_feature #define __has_feature(x) (0) #endif @@ -263,8 +248,7 @@ #ifndef __GNUC_PREREQ #if defined(__GNUC__) && defined(__GNUC_MINOR__) -#define __GNUC_PREREQ(maj, min) \ - ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min)) +#define __GNUC_PREREQ(maj, min) ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min)) #else #define __GNUC_PREREQ(maj, min) (0) #endif @@ -272,8 +256,7 @@ #ifndef __CLANG_PREREQ #ifdef __clang__ -#define __CLANG_PREREQ(maj, min) \ - ((__clang_major__ << 16) + __clang_minor__ >= ((maj) << 16) + (min)) +#define __CLANG_PREREQ(maj, min) ((__clang_major__ << 16) + __clang_minor__ >= ((maj) << 16) + (min)) #else #define __CLANG_PREREQ(maj, min) (0) #endif @@ -281,13 +264,51 @@ #ifndef __GLIBC_PREREQ #if defined(__GLIBC__) && defined(__GLIBC_MINOR__) -#define __GLIBC_PREREQ(maj, min) \ - ((__GLIBC__ << 16) + __GLIBC_MINOR__ >= ((maj) << 16) + (min)) +#define __GLIBC_PREREQ(maj, min) ((__GLIBC__ << 16) + __GLIBC_MINOR__ >= ((maj) << 16) + (min)) #else #define __GLIBC_PREREQ(maj, min) (0) #endif #endif /* __GLIBC_PREREQ */ +/*----------------------------------------------------------------------------*/ +/* pre-requirements */ + +#if (-6 & 5) || CHAR_BIT != 8 || UINT_MAX < 0xffffffff || ULONG_MAX % 0xFFFF +#error "Sanity checking failed: Two's complement, reasonably sized integer types" +#endif + +#ifndef SSIZE_MAX +#define SSIZE_MAX INTPTR_MAX +#endif + +#if defined(__GNUC__) && !__GNUC_PREREQ(4, 2) +/* Actually libmdbx was not tested with compilers older than GCC 4.2. + * But you could ignore this warning at your own risk. + * In such case please don't rise up an issues related ONLY to old compilers. + */ +#warning "libmdbx required GCC >= 4.2" +#endif + +#if defined(__clang__) && !__CLANG_PREREQ(3, 8) +/* Actually libmdbx was not tested with CLANG older than 3.8. + * But you could ignore this warning at your own risk. + * In such case please don't rise up an issues related ONLY to old compilers. + */ +#warning "libmdbx required CLANG >= 3.8" +#endif + +#if defined(__GLIBC__) && !__GLIBC_PREREQ(2, 12) +/* Actually libmdbx was not tested with something older than glibc 2.12. + * But you could ignore this warning at your own risk. + * In such case please don't rise up an issues related ONLY to old systems. + */ +#warning "libmdbx was only tested with GLIBC >= 2.12." +#endif + +#ifdef __SANITIZE_THREAD__ +#warning "libmdbx don't compatible with ThreadSanitizer, you will get a lot of false-positive issues." +#endif /* __SANITIZE_THREAD__ */ + /*----------------------------------------------------------------------------*/ /* C11' alignas() */ @@ -317,8 +338,7 @@ #endif #endif /* __extern_C */ -#if !defined(nullptr) && !defined(__cplusplus) || \ - (__cplusplus < 201103L && !defined(_MSC_VER)) +#if !defined(nullptr) && !defined(__cplusplus) || (__cplusplus < 201103L && !defined(_MSC_VER)) #define nullptr NULL #endif @@ -330,9 +350,8 @@ #endif #endif /* Apple OSX & iOS */ -#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \ - defined(__BSD__) || defined(__bsdi__) || defined(__DragonFly__) || \ - defined(__APPLE__) || defined(__MACH__) +#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__BSD__) || defined(__bsdi__) || \ + defined(__DragonFly__) || defined(__APPLE__) || defined(__MACH__) #include #include #include @@ -349,8 +368,7 @@ #endif #else #include -#if !(defined(__sun) || defined(__SVR4) || defined(__svr4__) || \ - defined(_WIN32) || defined(_WIN64)) +#if !(defined(__sun) || defined(__SVR4) || defined(__svr4__) || defined(_WIN32) || defined(_WIN64)) #include #endif /* !Solaris */ #endif /* !xBSD */ @@ -404,12 +422,14 @@ __extern_C key_t ftok(const char *, int); #ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN #endif /* WIN32_LEAN_AND_MEAN */ -#include -#include #include #include #include +/* После подгрузки windows.h, чтобы избежать проблем со сборкой MINGW и т.п. */ +#include +#include + #else /*----------------------------------------------------------------------*/ #include @@ -437,11 +457,6 @@ __extern_C key_t ftok(const char *, int); #if __ANDROID_API__ >= 21 #include #endif -#if defined(_FILE_OFFSET_BITS) && _FILE_OFFSET_BITS != MDBX_WORDBITS -#error "_FILE_OFFSET_BITS != MDBX_WORDBITS" (_FILE_OFFSET_BITS != MDBX_WORDBITS) -#elif defined(__FILE_OFFSET_BITS) && __FILE_OFFSET_BITS != MDBX_WORDBITS -#error "__FILE_OFFSET_BITS != MDBX_WORDBITS" (__FILE_OFFSET_BITS != MDBX_WORDBITS) -#endif #endif /* Android */ #if defined(HAVE_SYS_STAT_H) || __has_include() @@ -457,43 +472,38 @@ __extern_C key_t ftok(const char *, int); /*----------------------------------------------------------------------------*/ /* Byteorder */ -#if defined(i386) || defined(__386) || defined(__i386) || defined(__i386__) || \ - defined(i486) || defined(__i486) || defined(__i486__) || defined(i586) || \ - defined(__i586) || defined(__i586__) || defined(i686) || \ - defined(__i686) || defined(__i686__) || defined(_M_IX86) || \ - defined(_X86_) || defined(__THW_INTEL__) || defined(__I86__) || \ - defined(__INTEL__) || defined(__x86_64) || defined(__x86_64__) || \ - defined(__amd64__) || defined(__amd64) || defined(_M_X64) || \ - defined(_M_AMD64) || defined(__IA32__) || defined(__INTEL__) +#if defined(i386) || defined(__386) || defined(__i386) || defined(__i386__) || defined(i486) || defined(__i486) || \ + defined(__i486__) || defined(i586) || defined(__i586) || defined(__i586__) || defined(i686) || defined(__i686) || \ + defined(__i686__) || defined(_M_IX86) || defined(_X86_) || defined(__THW_INTEL__) || defined(__I86__) || \ + defined(__INTEL__) || defined(__x86_64) || defined(__x86_64__) || defined(__amd64__) || defined(__amd64) || \ + defined(_M_X64) || defined(_M_AMD64) || defined(__IA32__) || defined(__INTEL__) #ifndef __ia32__ /* LY: define neutral __ia32__ for x86 and x86-64 */ #define __ia32__ 1 #endif /* __ia32__ */ -#if !defined(__amd64__) && \ - (defined(__x86_64) || defined(__x86_64__) || defined(__amd64) || \ - defined(_M_X64) || defined(_M_AMD64)) +#if !defined(__amd64__) && \ + (defined(__x86_64) || defined(__x86_64__) || defined(__amd64) || defined(_M_X64) || defined(_M_AMD64)) /* LY: define trusty __amd64__ for all AMD64/x86-64 arch */ #define __amd64__ 1 #endif /* __amd64__ */ #endif /* all x86 */ -#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || \ - !defined(__ORDER_BIG_ENDIAN__) +#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || !defined(__ORDER_BIG_ENDIAN__) -#if defined(__GLIBC__) || defined(__GNU_LIBRARY__) || \ - defined(__ANDROID_API__) || defined(HAVE_ENDIAN_H) || __has_include() +#if defined(__GLIBC__) || defined(__GNU_LIBRARY__) || defined(__ANDROID_API__) || defined(HAVE_ENDIAN_H) || \ + __has_include() #include -#elif defined(__APPLE__) || defined(__MACH__) || defined(__OpenBSD__) || \ - defined(HAVE_MACHINE_ENDIAN_H) || __has_include() +#elif defined(__APPLE__) || defined(__MACH__) || defined(__OpenBSD__) || defined(HAVE_MACHINE_ENDIAN_H) || \ + __has_include() #include #elif defined(HAVE_SYS_ISA_DEFS_H) || __has_include() #include -#elif (defined(HAVE_SYS_TYPES_H) && defined(HAVE_SYS_ENDIAN_H)) || \ +#elif (defined(HAVE_SYS_TYPES_H) && defined(HAVE_SYS_ENDIAN_H)) || \ (__has_include() && __has_include()) #include #include -#elif defined(__bsdi__) || defined(__DragonFly__) || defined(__FreeBSD__) || \ - defined(__NetBSD__) || defined(HAVE_SYS_PARAM_H) || __has_include() +#elif defined(__bsdi__) || defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || \ + defined(HAVE_SYS_PARAM_H) || __has_include() #include #endif /* OS */ @@ -509,27 +519,19 @@ __extern_C key_t ftok(const char *, int); #define __ORDER_LITTLE_ENDIAN__ 1234 #define __ORDER_BIG_ENDIAN__ 4321 -#if defined(__LITTLE_ENDIAN__) || \ - (defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)) || \ - defined(__ARMEL__) || defined(__THUMBEL__) || defined(__AARCH64EL__) || \ - defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \ - defined(_M_ARM) || defined(_M_ARM64) || defined(__e2k__) || \ - defined(__elbrus_4c__) || defined(__elbrus_8c__) || defined(__bfin__) || \ - defined(__BFIN__) || defined(__ia64__) || defined(_IA64) || \ - defined(__IA64__) || defined(__ia64) || defined(_M_IA64) || \ - defined(__itanium__) || defined(__ia32__) || defined(__CYGWIN__) || \ - defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) || \ - defined(__WINDOWS__) +#if defined(__LITTLE_ENDIAN__) || (defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)) || defined(__ARMEL__) || \ + defined(__THUMBEL__) || defined(__AARCH64EL__) || defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \ + defined(_M_ARM) || defined(_M_ARM64) || defined(__e2k__) || defined(__elbrus_4c__) || defined(__elbrus_8c__) || \ + defined(__bfin__) || defined(__BFIN__) || defined(__ia64__) || defined(_IA64) || defined(__IA64__) || \ + defined(__ia64) || defined(_M_IA64) || defined(__itanium__) || defined(__ia32__) || defined(__CYGWIN__) || \ + defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) || defined(__WINDOWS__) #define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__ -#elif defined(__BIG_ENDIAN__) || \ - (defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) || \ - defined(__ARMEB__) || defined(__THUMBEB__) || defined(__AARCH64EB__) || \ - defined(__MIPSEB__) || defined(_MIPSEB) || defined(__MIPSEB) || \ - defined(__m68k__) || defined(M68000) || defined(__hppa__) || \ - defined(__hppa) || defined(__HPPA__) || defined(__sparc__) || \ - defined(__sparc) || defined(__370__) || defined(__THW_370__) || \ - defined(__s390__) || defined(__s390x__) || defined(__SYSC_ZARCH__) +#elif defined(__BIG_ENDIAN__) || (defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) || defined(__ARMEB__) || \ + defined(__THUMBEB__) || defined(__AARCH64EB__) || defined(__MIPSEB__) || defined(_MIPSEB) || defined(__MIPSEB) || \ + defined(__m68k__) || defined(M68000) || defined(__hppa__) || defined(__hppa) || defined(__HPPA__) || \ + defined(__sparc__) || defined(__sparc) || defined(__370__) || defined(__THW_370__) || defined(__s390__) || \ + defined(__s390x__) || defined(__SYSC_ZARCH__) #define __BYTE_ORDER__ __ORDER_BIG_ENDIAN__ #else @@ -539,6 +541,12 @@ __extern_C key_t ftok(const char *, int); #endif #endif /* __BYTE_ORDER__ || __ORDER_LITTLE_ENDIAN__ || __ORDER_BIG_ENDIAN__ */ +#if UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul || defined(_WIN64) +#define MDBX_WORDBITS 64 +#else +#define MDBX_WORDBITS 32 +#endif /* MDBX_WORDBITS */ + /*----------------------------------------------------------------------------*/ /* Availability of CMOV or equivalent */ @@ -549,17 +557,14 @@ __extern_C key_t ftok(const char *, int); #define MDBX_HAVE_CMOV 1 #elif defined(__thumb__) || defined(__thumb) || defined(__TARGET_ARCH_THUMB) #define MDBX_HAVE_CMOV 0 -#elif defined(_M_ARM) || defined(_M_ARM64) || defined(__aarch64__) || \ - defined(__aarch64) || defined(__arm__) || defined(__arm) || \ - defined(__CC_ARM) +#elif defined(_M_ARM) || defined(_M_ARM64) || defined(__aarch64__) || defined(__aarch64) || defined(__arm__) || \ + defined(__arm) || defined(__CC_ARM) #define MDBX_HAVE_CMOV 1 -#elif (defined(__riscv__) || defined(__riscv64)) && \ - (defined(__riscv_b) || defined(__riscv_bitmanip)) +#elif (defined(__riscv__) || defined(__riscv64)) && (defined(__riscv_b) || defined(__riscv_bitmanip)) #define MDBX_HAVE_CMOV 1 -#elif defined(i686) || defined(__i686) || defined(__i686__) || \ - (defined(_M_IX86) && _M_IX86 > 600) || defined(__x86_64) || \ - defined(__x86_64__) || defined(__amd64__) || defined(__amd64) || \ - defined(_M_X64) || defined(_M_AMD64) +#elif defined(i686) || defined(__i686) || defined(__i686__) || (defined(_M_IX86) && _M_IX86 > 600) || \ + defined(__x86_64) || defined(__x86_64__) || defined(__amd64__) || defined(__amd64) || defined(_M_X64) || \ + defined(_M_AMD64) #define MDBX_HAVE_CMOV 1 #else #define MDBX_HAVE_CMOV 0 @@ -585,8 +590,7 @@ __extern_C key_t ftok(const char *, int); #endif #elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) #include -#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ - (defined(HP_IA64) || defined(__ia64)) +#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && (defined(HP_IA64) || defined(__ia64)) #include #elif defined(__IBMC__) && defined(__powerpc) #include @@ -608,29 +612,26 @@ __extern_C key_t ftok(const char *, int); #endif /* Compiler */ #if !defined(__noop) && !defined(_MSC_VER) -#define __noop \ - do { \ +#define __noop \ + do { \ } while (0) #endif /* __noop */ -#if defined(__fallthrough) && \ - (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) +#if defined(__fallthrough) && (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) #undef __fallthrough #endif /* __fallthrough workaround for MinGW */ #ifndef __fallthrough -#if defined(__cplusplus) && (__has_cpp_attribute(fallthrough) && \ - (!defined(__clang__) || __clang__ > 4)) || \ +#if defined(__cplusplus) && (__has_cpp_attribute(fallthrough) && (!defined(__clang__) || __clang__ > 4)) || \ __cplusplus >= 201703L #define __fallthrough [[fallthrough]] #elif __GNUC_PREREQ(8, 0) && defined(__cplusplus) && __cplusplus >= 201103L #define __fallthrough [[fallthrough]] -#elif __GNUC_PREREQ(7, 0) && \ - (!defined(__LCC__) || (__LCC__ == 124 && __LCC_MINOR__ >= 12) || \ - (__LCC__ == 125 && __LCC_MINOR__ >= 5) || (__LCC__ >= 126)) +#elif __GNUC_PREREQ(7, 0) && (!defined(__LCC__) || (__LCC__ == 124 && __LCC_MINOR__ >= 12) || \ + (__LCC__ == 125 && __LCC_MINOR__ >= 5) || (__LCC__ >= 126)) #define __fallthrough __attribute__((__fallthrough__)) -#elif defined(__clang__) && defined(__cplusplus) && __cplusplus >= 201103L && \ - __has_feature(cxx_attributes) && __has_warning("-Wimplicit-fallthrough") +#elif defined(__clang__) && defined(__cplusplus) && __cplusplus >= 201103L && __has_feature(cxx_attributes) && \ + __has_warning("-Wimplicit-fallthrough") #define __fallthrough [[clang::fallthrough]] #else #define __fallthrough @@ -643,8 +644,8 @@ __extern_C key_t ftok(const char *, int); #elif defined(_MSC_VER) #define __unreachable() __assume(0) #else -#define __unreachable() \ - do { \ +#define __unreachable() \ + do { \ } while (1) #endif #endif /* __unreachable */ @@ -653,9 +654,9 @@ __extern_C key_t ftok(const char *, int); #if defined(__GNUC__) || defined(__clang__) || __has_builtin(__builtin_prefetch) #define __prefetch(ptr) __builtin_prefetch(ptr) #else -#define __prefetch(ptr) \ - do { \ - (void)(ptr); \ +#define __prefetch(ptr) \ + do { \ + (void)(ptr); \ } while (0) #endif #endif /* __prefetch */ @@ -665,11 +666,11 @@ __extern_C key_t ftok(const char *, int); #endif /* offsetof */ #ifndef container_of -#define container_of(ptr, type, member) \ - ((type *)((char *)(ptr) - offsetof(type, member))) +#define container_of(ptr, type, member) ((type *)((char *)(ptr) - offsetof(type, member))) #endif /* container_of */ /*----------------------------------------------------------------------------*/ +/* useful attributes */ #ifndef __always_inline #if defined(__GNUC__) || __has_attribute(__always_inline__) @@ -737,8 +738,7 @@ __extern_C key_t ftok(const char *, int); #ifndef __hot #if defined(__OPTIMIZE__) -#if defined(__clang__) && !__has_attribute(__hot__) && \ - __has_attribute(__section__) && \ +#if defined(__clang__) && !__has_attribute(__hot__) && __has_attribute(__section__) && \ (defined(__linux__) || defined(__gnu_linux__)) /* just put frequently used functions in separate section */ #define __hot __attribute__((__section__("text.hot"))) __optimize("O3") @@ -754,8 +754,7 @@ __extern_C key_t ftok(const char *, int); #ifndef __cold #if defined(__OPTIMIZE__) -#if defined(__clang__) && !__has_attribute(__cold__) && \ - __has_attribute(__section__) && \ +#if defined(__clang__) && !__has_attribute(__cold__) && __has_attribute(__section__) && \ (defined(__linux__) || defined(__gnu_linux__)) /* just put infrequently used functions in separate section */ #define __cold __attribute__((__section__("text.unlikely"))) __optimize("Os") @@ -778,8 +777,7 @@ __extern_C key_t ftok(const char *, int); #endif /* __flatten */ #ifndef likely -#if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && \ - !defined(__COVERITY__) +#if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__) #define likely(cond) __builtin_expect(!!(cond), 1) #else #define likely(x) (!!(x)) @@ -787,8 +785,7 @@ __extern_C key_t ftok(const char *, int); #endif /* likely */ #ifndef unlikely -#if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && \ - !defined(__COVERITY__) +#if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__) #define unlikely(cond) __builtin_expect(!!(cond), 0) #else #define unlikely(x) (!!(x)) @@ -803,29 +800,41 @@ __extern_C key_t ftok(const char *, int); #endif #endif /* __anonymous_struct_extension__ */ -#ifndef expect_with_probability -#if defined(__builtin_expect_with_probability) || \ - __has_builtin(__builtin_expect_with_probability) || __GNUC_PREREQ(9, 0) -#define expect_with_probability(expr, value, prob) \ - __builtin_expect_with_probability(expr, value, prob) -#else -#define expect_with_probability(expr, value, prob) (expr) -#endif -#endif /* expect_with_probability */ - #ifndef MDBX_WEAK_IMPORT_ATTRIBUTE #ifdef WEAK_IMPORT_ATTRIBUTE #define MDBX_WEAK_IMPORT_ATTRIBUTE WEAK_IMPORT_ATTRIBUTE #elif __has_attribute(__weak__) && __has_attribute(__weak_import__) #define MDBX_WEAK_IMPORT_ATTRIBUTE __attribute__((__weak__, __weak_import__)) -#elif __has_attribute(__weak__) || \ - (defined(__GNUC__) && __GNUC__ >= 4 && defined(__ELF__)) +#elif __has_attribute(__weak__) || (defined(__GNUC__) && __GNUC__ >= 4 && defined(__ELF__)) #define MDBX_WEAK_IMPORT_ATTRIBUTE __attribute__((__weak__)) #else #define MDBX_WEAK_IMPORT_ATTRIBUTE #endif #endif /* MDBX_WEAK_IMPORT_ATTRIBUTE */ +#if !defined(__thread) && (defined(_MSC_VER) || defined(__DMC__)) +#define __thread __declspec(thread) +#endif /* __thread */ + +#ifndef MDBX_EXCLUDE_FOR_GPROF +#ifdef ENABLE_GPROF +#define MDBX_EXCLUDE_FOR_GPROF __attribute__((__no_instrument_function__, __no_profile_instrument_function__)) +#else +#define MDBX_EXCLUDE_FOR_GPROF +#endif /* ENABLE_GPROF */ +#endif /* MDBX_EXCLUDE_FOR_GPROF */ + +/*----------------------------------------------------------------------------*/ + +#ifndef expect_with_probability +#if defined(__builtin_expect_with_probability) || __has_builtin(__builtin_expect_with_probability) || \ + __GNUC_PREREQ(9, 0) +#define expect_with_probability(expr, value, prob) __builtin_expect_with_probability(expr, value, prob) +#else +#define expect_with_probability(expr, value, prob) (expr) +#endif +#endif /* expect_with_probability */ + #ifndef MDBX_GOOFY_MSVC_STATIC_ANALYZER #ifdef _PREFAST_ #define MDBX_GOOFY_MSVC_STATIC_ANALYZER 1 @@ -837,20 +846,27 @@ __extern_C key_t ftok(const char *, int); #if MDBX_GOOFY_MSVC_STATIC_ANALYZER || (defined(_MSC_VER) && _MSC_VER > 1919) #define MDBX_ANALYSIS_ASSUME(expr) __analysis_assume(expr) #ifdef _PREFAST_ -#define MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(warn_id) \ - __pragma(prefast(suppress : warn_id)) +#define MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(warn_id) __pragma(prefast(suppress : warn_id)) #else -#define MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(warn_id) \ - __pragma(warning(suppress : warn_id)) +#define MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(warn_id) __pragma(warning(suppress : warn_id)) #endif #else #define MDBX_ANALYSIS_ASSUME(expr) assert(expr) #define MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(warn_id) #endif /* MDBX_GOOFY_MSVC_STATIC_ANALYZER */ +#ifndef FLEXIBLE_ARRAY_MEMBERS +#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || (!defined(__cplusplus) && defined(_MSC_VER)) +#define FLEXIBLE_ARRAY_MEMBERS 1 +#else +#define FLEXIBLE_ARRAY_MEMBERS 0 +#endif +#endif /* FLEXIBLE_ARRAY_MEMBERS */ + /*----------------------------------------------------------------------------*/ +/* Valgrind and Address Sanitizer */ -#if defined(MDBX_USE_VALGRIND) +#if defined(ENABLE_MEMCHECK) #include #ifndef VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE /* LY: available since Valgrind 3.10 */ @@ -872,7 +888,7 @@ __extern_C key_t ftok(const char *, int); #define VALGRIND_CHECK_MEM_IS_ADDRESSABLE(a, s) (0) #define VALGRIND_CHECK_MEM_IS_DEFINED(a, s) (0) #define RUNNING_ON_VALGRIND (0) -#endif /* MDBX_USE_VALGRIND */ +#endif /* ENABLE_MEMCHECK */ #ifdef __SANITIZE_ADDRESS__ #include @@ -899,8 +915,7 @@ template char (&__ArraySizeHelper(T (&array)[N]))[N]; #define CONCAT(a, b) a##b #define XCONCAT(a, b) CONCAT(a, b) -#define MDBX_TETRAD(a, b, c, d) \ - ((uint32_t)(a) << 24 | (uint32_t)(b) << 16 | (uint32_t)(c) << 8 | (d)) +#define MDBX_TETRAD(a, b, c, d) ((uint32_t)(a) << 24 | (uint32_t)(b) << 16 | (uint32_t)(c) << 8 | (d)) #define MDBX_STRING_TETRAD(str) MDBX_TETRAD(str[0], str[1], str[2], str[3]) @@ -914,14 +929,13 @@ template char (&__ArraySizeHelper(T (&array)[N]))[N]; #elif defined(_MSC_VER) #include #define STATIC_ASSERT_MSG(expr, msg) _STATIC_ASSERT(expr) -#elif (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) || \ - __has_feature(c_static_assert) +#elif (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) || __has_feature(c_static_assert) #define STATIC_ASSERT_MSG(expr, msg) _Static_assert(expr, msg) #else -#define STATIC_ASSERT_MSG(expr, msg) \ - switch (0) { \ - case 0: \ - case (expr):; \ +#define STATIC_ASSERT_MSG(expr, msg) \ + switch (0) { \ + case 0: \ + case (expr):; \ } #endif #endif /* STATIC_ASSERT */ @@ -930,42 +944,37 @@ template char (&__ArraySizeHelper(T (&array)[N]))[N]; #define STATIC_ASSERT(expr) STATIC_ASSERT_MSG(expr, #expr) #endif -#ifndef __Wpedantic_format_voidptr -MDBX_MAYBE_UNUSED MDBX_PURE_FUNCTION static __inline const void * -__Wpedantic_format_voidptr(const void *ptr) { - return ptr; -} -#define __Wpedantic_format_voidptr(ARG) __Wpedantic_format_voidptr(ARG) -#endif /* __Wpedantic_format_voidptr */ +/*----------------------------------------------------------------------------*/ -#if defined(__GNUC__) && !__GNUC_PREREQ(4, 2) -/* Actually libmdbx was not tested with compilers older than GCC 4.2. - * But you could ignore this warning at your own risk. - * In such case please don't rise up an issues related ONLY to old compilers. - */ -#warning "libmdbx required GCC >= 4.2" -#endif +#if defined(_MSC_VER) && _MSC_VER >= 1900 +/* LY: MSVC 2015/2017/2019 has buggy/inconsistent PRIuPTR/PRIxPTR macros + * for internal format-args checker. */ +#undef PRIuPTR +#undef PRIiPTR +#undef PRIdPTR +#undef PRIxPTR +#define PRIuPTR "Iu" +#define PRIiPTR "Ii" +#define PRIdPTR "Id" +#define PRIxPTR "Ix" +#define PRIuSIZE "zu" +#define PRIiSIZE "zi" +#define PRIdSIZE "zd" +#define PRIxSIZE "zx" +#endif /* fix PRI*PTR for _MSC_VER */ -#if defined(__clang__) && !__CLANG_PREREQ(3, 8) -/* Actually libmdbx was not tested with CLANG older than 3.8. - * But you could ignore this warning at your own risk. - * In such case please don't rise up an issues related ONLY to old compilers. - */ -#warning "libmdbx required CLANG >= 3.8" -#endif +#ifndef PRIuSIZE +#define PRIuSIZE PRIuPTR +#define PRIiSIZE PRIiPTR +#define PRIdSIZE PRIdPTR +#define PRIxSIZE PRIxPTR +#endif /* PRI*SIZE macros for MSVC */ -#if defined(__GLIBC__) && !__GLIBC_PREREQ(2, 12) -/* Actually libmdbx was not tested with something older than glibc 2.12. - * But you could ignore this warning at your own risk. - * In such case please don't rise up an issues related ONLY to old systems. - */ -#warning "libmdbx was only tested with GLIBC >= 2.12." +#ifdef _MSC_VER +#pragma warning(pop) #endif -#ifdef __SANITIZE_THREAD__ -#warning \ - "libmdbx don't compatible with ThreadSanitizer, you will get a lot of false-positive issues." -#endif /* __SANITIZE_THREAD__ */ +/*----------------------------------------------------------------------------*/ #if __has_warning("-Wnested-anon-types") #if defined(__clang__) @@ -1002,80 +1011,34 @@ __Wpedantic_format_voidptr(const void *ptr) { #endif #endif /* -Walignment-reduction-ignored */ -#ifndef MDBX_EXCLUDE_FOR_GPROF -#ifdef ENABLE_GPROF -#define MDBX_EXCLUDE_FOR_GPROF \ - __attribute__((__no_instrument_function__, \ - __no_profile_instrument_function__)) +#ifdef xMDBX_ALLOY +/* Amalgamated build */ +#define MDBX_INTERNAL static #else -#define MDBX_EXCLUDE_FOR_GPROF -#endif /* ENABLE_GPROF */ -#endif /* MDBX_EXCLUDE_FOR_GPROF */ - -#ifdef __cplusplus -extern "C" { -#endif +/* Non-amalgamated build */ +#define MDBX_INTERNAL +#endif /* xMDBX_ALLOY */ -/* https://en.wikipedia.org/wiki/Operating_system_abstraction_layer */ +#include "mdbx.h" -/* - * Copyright 2015-2024 Leonid Yuriev - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - */ +/*----------------------------------------------------------------------------*/ +/* Basic constants and types */ +typedef struct iov_ctx iov_ctx_t; +/// /*----------------------------------------------------------------------------*/ -/* C11 Atomics */ - -#if defined(__cplusplus) && !defined(__STDC_NO_ATOMICS__) && __has_include() -#include -#define MDBX_HAVE_C11ATOMICS -#elif !defined(__cplusplus) && \ - (__STDC_VERSION__ >= 201112L || __has_extension(c_atomic)) && \ - !defined(__STDC_NO_ATOMICS__) && \ - (__GNUC_PREREQ(4, 9) || __CLANG_PREREQ(3, 8) || \ - !(defined(__GNUC__) || defined(__clang__))) -#include -#define MDBX_HAVE_C11ATOMICS -#elif defined(__GNUC__) || defined(__clang__) -#elif defined(_MSC_VER) -#pragma warning(disable : 4163) /* 'xyz': not available as an intrinsic */ -#pragma warning(disable : 4133) /* 'function': incompatible types - from \ - 'size_t' to 'LONGLONG' */ -#pragma warning(disable : 4244) /* 'return': conversion from 'LONGLONG' to \ - 'std::size_t', possible loss of data */ -#pragma warning(disable : 4267) /* 'function': conversion from 'size_t' to \ - 'long', possible loss of data */ -#pragma intrinsic(_InterlockedExchangeAdd, _InterlockedCompareExchange) -#pragma intrinsic(_InterlockedExchangeAdd64, _InterlockedCompareExchange64) -#elif defined(__APPLE__) -#include -#else -#error FIXME atomic-ops -#endif - -/*----------------------------------------------------------------------------*/ -/* Memory/Compiler barriers, cache coherence */ +/* Memory/Compiler barriers, cache coherence */ #if __has_include() #include -#elif defined(__mips) || defined(__mips__) || defined(__mips64) || \ - defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \ - defined(__MWERKS__) || defined(__sgi) +#elif defined(__mips) || defined(__mips__) || defined(__mips64) || defined(__mips64__) || defined(_M_MRX000) || \ + defined(_MIPS_) || defined(__MWERKS__) || defined(__sgi) /* MIPS should have explicit cache control */ #include #endif -MDBX_MAYBE_UNUSED static __inline void osal_compiler_barrier(void) { +MDBX_MAYBE_UNUSED static inline void osal_compiler_barrier(void) { #if defined(__clang__) || defined(__GNUC__) __asm__ __volatile__("" ::: "memory"); #elif defined(_MSC_VER) @@ -1084,18 +1047,16 @@ MDBX_MAYBE_UNUSED static __inline void osal_compiler_barrier(void) { __memory_barrier(); #elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) __compiler_barrier(); -#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ - (defined(HP_IA64) || defined(__ia64)) +#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && (defined(HP_IA64) || defined(__ia64)) _Asm_sched_fence(/* LY: no-arg meaning 'all expect ALU', e.g. 0x3D3D */); -#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \ - defined(__ppc64__) || defined(__powerpc64__) +#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || defined(__ppc64__) || defined(__powerpc64__) __fence(); #else #error "Could not guess the kind of compiler, please report to us." #endif } -MDBX_MAYBE_UNUSED static __inline void osal_memory_barrier(void) { +MDBX_MAYBE_UNUSED static inline void osal_memory_barrier(void) { #ifdef MDBX_HAVE_C11ATOMICS atomic_thread_fence(memory_order_seq_cst); #elif defined(__ATOMIC_SEQ_CST) @@ -1116,11 +1077,9 @@ MDBX_MAYBE_UNUSED static __inline void osal_memory_barrier(void) { #endif #elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) __machine_rw_barrier(); -#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ - (defined(HP_IA64) || defined(__ia64)) +#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && (defined(HP_IA64) || defined(__ia64)) _Asm_mf(); -#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \ - defined(__ppc64__) || defined(__powerpc64__) +#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || defined(__ppc64__) || defined(__powerpc64__) __lwsync(); #else #error "Could not guess the kind of compiler, please report to us." @@ -1135,7 +1094,7 @@ MDBX_MAYBE_UNUSED static __inline void osal_memory_barrier(void) { #define HAVE_SYS_TYPES_H typedef HANDLE osal_thread_t; typedef unsigned osal_thread_key_t; -#define MAP_FAILED NULL +#define MAP_FAILED nullptr #define HIGH_DWORD(v) ((DWORD)((sizeof(v) > 4) ? ((uint64_t)(v) >> 32) : 0)) #define THREAD_CALL WINAPI #define THREAD_RESULT DWORD @@ -1147,15 +1106,13 @@ typedef CRITICAL_SECTION osal_fastmutex_t; #if !defined(_MSC_VER) && !defined(__try) #define __try -#define __except(COND) if (false) +#define __except(COND) if (/* (void)(COND), */ false) #endif /* stub for MSVC's __try/__except */ #if MDBX_WITHOUT_MSVC_CRT #ifndef osal_malloc -static inline void *osal_malloc(size_t bytes) { - return HeapAlloc(GetProcessHeap(), 0, bytes); -} +static inline void *osal_malloc(size_t bytes) { return HeapAlloc(GetProcessHeap(), 0, bytes); } #endif /* osal_malloc */ #ifndef osal_calloc @@ -1166,8 +1123,7 @@ static inline void *osal_calloc(size_t nelem, size_t size) { #ifndef osal_realloc static inline void *osal_realloc(void *ptr, size_t bytes) { - return ptr ? HeapReAlloc(GetProcessHeap(), 0, ptr, bytes) - : HeapAlloc(GetProcessHeap(), 0, bytes); + return ptr ? HeapReAlloc(GetProcessHeap(), 0, ptr, bytes) : HeapAlloc(GetProcessHeap(), 0, bytes); } #endif /* osal_realloc */ @@ -1213,29 +1169,16 @@ typedef pthread_mutex_t osal_fastmutex_t; #endif /* Platform */ #if __GLIBC_PREREQ(2, 12) || defined(__FreeBSD__) || defined(malloc_usable_size) -/* malloc_usable_size() already provided */ +#define osal_malloc_usable_size(ptr) malloc_usable_size(ptr) #elif defined(__APPLE__) -#define malloc_usable_size(ptr) malloc_size(ptr) +#define osal_malloc_usable_size(ptr) malloc_size(ptr) #elif defined(_MSC_VER) && !MDBX_WITHOUT_MSVC_CRT -#define malloc_usable_size(ptr) _msize(ptr) -#endif /* malloc_usable_size */ +#define osal_malloc_usable_size(ptr) _msize(ptr) +#endif /* osal_malloc_usable_size */ /*----------------------------------------------------------------------------*/ /* OS abstraction layer stuff */ -MDBX_INTERNAL_VAR unsigned sys_pagesize; -MDBX_MAYBE_UNUSED MDBX_INTERNAL_VAR unsigned sys_pagesize_ln2, - sys_allocation_granularity; - -/* Get the size of a memory page for the system. - * This is the basic size that the platform's memory manager uses, and is - * fundamental to the use of memory-mapped files. */ -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline size_t -osal_syspagesize(void) { - assert(sys_pagesize > 0 && (sys_pagesize & (sys_pagesize - 1)) == 0); - return sys_pagesize; -} - #if defined(_WIN32) || defined(_WIN64) typedef wchar_t pathchar_t; #define MDBX_PRIsPATH "ls" @@ -1247,7 +1190,7 @@ typedef char pathchar_t; typedef struct osal_mmap { union { void *base; - struct MDBX_lockinfo *lck; + struct shared_lck *lck; }; mdbx_filehandle_t fd; size_t limit; /* mapping length, but NOT a size of file nor DB */ @@ -1258,25 +1201,6 @@ typedef struct osal_mmap { #endif } osal_mmap_t; -typedef union bin128 { - __anonymous_struct_extension__ struct { - uint64_t x, y; - }; - __anonymous_struct_extension__ struct { - uint32_t a, b, c, d; - }; -} bin128_t; - -#if defined(_WIN32) || defined(_WIN64) -typedef union osal_srwlock { - __anonymous_struct_extension__ struct { - long volatile readerCount; - long volatile writerCount; - }; - RTL_SRWLOCK native; -} osal_srwlock_t; -#endif /* Windows */ - #ifndef MDBX_HAVE_PWRITEV #if defined(_WIN32) || defined(_WIN64) @@ -1285,14 +1209,21 @@ typedef union osal_srwlock { #elif defined(__ANDROID_API__) #if __ANDROID_API__ < 24 +/* https://android-developers.googleblog.com/2017/09/introducing-android-native-development.html + * https://android.googlesource.com/platform/bionic/+/master/docs/32-bit-abi.md */ #define MDBX_HAVE_PWRITEV 0 +#if defined(_FILE_OFFSET_BITS) && _FILE_OFFSET_BITS != MDBX_WORDBITS +#error "_FILE_OFFSET_BITS != MDBX_WORDBITS and __ANDROID_API__ < 24" (_FILE_OFFSET_BITS != MDBX_WORDBITS) +#elif defined(__FILE_OFFSET_BITS) && __FILE_OFFSET_BITS != MDBX_WORDBITS +#error "__FILE_OFFSET_BITS != MDBX_WORDBITS and __ANDROID_API__ < 24" (__FILE_OFFSET_BITS != MDBX_WORDBITS) +#endif #else #define MDBX_HAVE_PWRITEV 1 #endif #elif defined(__APPLE__) || defined(__MACH__) || defined(_DARWIN_C_SOURCE) -#if defined(MAC_OS_X_VERSION_MIN_REQUIRED) && defined(MAC_OS_VERSION_11_0) && \ +#if defined(MAC_OS_X_VERSION_MIN_REQUIRED) && defined(MAC_OS_VERSION_11_0) && \ MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_VERSION_11_0 /* FIXME: add checks for IOS versions, etc */ #define MDBX_HAVE_PWRITEV 1 @@ -1310,20 +1241,20 @@ typedef union osal_srwlock { typedef struct ior_item { #if defined(_WIN32) || defined(_WIN64) OVERLAPPED ov; -#define ior_svg_gap4terminator 1 +#define ior_sgv_gap4terminator 1 #define ior_sgv_element FILE_SEGMENT_ELEMENT #else size_t offset; #if MDBX_HAVE_PWRITEV size_t sgvcnt; -#define ior_svg_gap4terminator 0 +#define ior_sgv_gap4terminator 0 #define ior_sgv_element struct iovec #endif /* MDBX_HAVE_PWRITEV */ #endif /* !Windows */ union { MDBX_val single; #if defined(ior_sgv_element) - ior_sgv_element sgv[1 + ior_svg_gap4terminator]; + ior_sgv_element sgv[1 + ior_sgv_gap4terminator]; #endif /* ior_sgv_element */ }; } ior_item_t; @@ -1359,45 +1290,33 @@ typedef struct osal_ioring { char *boundary; } osal_ioring_t; -#ifndef __cplusplus - /* Actually this is not ioring for now, but on the way. */ -MDBX_INTERNAL_FUNC int osal_ioring_create(osal_ioring_t * +MDBX_INTERNAL int osal_ioring_create(osal_ioring_t * #if defined(_WIN32) || defined(_WIN64) - , - bool enable_direct, - mdbx_filehandle_t overlapped_fd + , + bool enable_direct, mdbx_filehandle_t overlapped_fd #endif /* Windows */ ); -MDBX_INTERNAL_FUNC int osal_ioring_resize(osal_ioring_t *, size_t items); -MDBX_INTERNAL_FUNC void osal_ioring_destroy(osal_ioring_t *); -MDBX_INTERNAL_FUNC void osal_ioring_reset(osal_ioring_t *); -MDBX_INTERNAL_FUNC int osal_ioring_add(osal_ioring_t *ctx, const size_t offset, - void *data, const size_t bytes); +MDBX_INTERNAL int osal_ioring_resize(osal_ioring_t *, size_t items); +MDBX_INTERNAL void osal_ioring_destroy(osal_ioring_t *); +MDBX_INTERNAL void osal_ioring_reset(osal_ioring_t *); +MDBX_INTERNAL int osal_ioring_add(osal_ioring_t *ctx, const size_t offset, void *data, const size_t bytes); typedef struct osal_ioring_write_result { int err; unsigned wops; } osal_ioring_write_result_t; -MDBX_INTERNAL_FUNC osal_ioring_write_result_t -osal_ioring_write(osal_ioring_t *ior, mdbx_filehandle_t fd); +MDBX_INTERNAL osal_ioring_write_result_t osal_ioring_write(osal_ioring_t *ior, mdbx_filehandle_t fd); -typedef struct iov_ctx iov_ctx_t; -MDBX_INTERNAL_FUNC void osal_ioring_walk( - osal_ioring_t *ior, iov_ctx_t *ctx, - void (*callback)(iov_ctx_t *ctx, size_t offset, void *data, size_t bytes)); +MDBX_INTERNAL void osal_ioring_walk(osal_ioring_t *ior, iov_ctx_t *ctx, + void (*callback)(iov_ctx_t *ctx, size_t offset, void *data, size_t bytes)); -MDBX_MAYBE_UNUSED static inline unsigned -osal_ioring_left(const osal_ioring_t *ior) { - return ior->slots_left; -} +MDBX_MAYBE_UNUSED static inline unsigned osal_ioring_left(const osal_ioring_t *ior) { return ior->slots_left; } -MDBX_MAYBE_UNUSED static inline unsigned -osal_ioring_used(const osal_ioring_t *ior) { +MDBX_MAYBE_UNUSED static inline unsigned osal_ioring_used(const osal_ioring_t *ior) { return ior->allocated - ior->slots_left; } -MDBX_MAYBE_UNUSED static inline int -osal_ioring_prepare(osal_ioring_t *ior, size_t items, size_t bytes) { +MDBX_MAYBE_UNUSED static inline int osal_ioring_prepare(osal_ioring_t *ior, size_t items, size_t bytes) { items = (items > 32) ? items : 32; #if defined(_WIN32) || defined(_WIN64) if (ior->direct) { @@ -1416,14 +1335,12 @@ osal_ioring_prepare(osal_ioring_t *ior, size_t items, size_t bytes) { /*----------------------------------------------------------------------------*/ /* libc compatibility stuff */ -#if (!defined(__GLIBC__) && __GLIBC_PREREQ(2, 1)) && \ - (defined(_GNU_SOURCE) || defined(_BSD_SOURCE)) +#if (!defined(__GLIBC__) && __GLIBC_PREREQ(2, 1)) && (defined(_GNU_SOURCE) || defined(_BSD_SOURCE)) #define osal_asprintf asprintf #define osal_vasprintf vasprintf #else -MDBX_MAYBE_UNUSED MDBX_INTERNAL_FUNC - MDBX_PRINTF_ARGS(2, 3) int osal_asprintf(char **strp, const char *fmt, ...); -MDBX_INTERNAL_FUNC int osal_vasprintf(char **strp, const char *fmt, va_list ap); +MDBX_MAYBE_UNUSED MDBX_INTERNAL MDBX_PRINTF_ARGS(2, 3) int osal_asprintf(char **strp, const char *fmt, ...); +MDBX_INTERNAL int osal_vasprintf(char **strp, const char *fmt, va_list ap); #endif #if !defined(MADV_DODUMP) && defined(MADV_CORE) @@ -1434,8 +1351,7 @@ MDBX_INTERNAL_FUNC int osal_vasprintf(char **strp, const char *fmt, va_list ap); #define MADV_DONTDUMP MADV_NOCORE #endif /* MADV_NOCORE -> MADV_DONTDUMP */ -MDBX_MAYBE_UNUSED MDBX_INTERNAL_FUNC void osal_jitter(bool tiny); -MDBX_MAYBE_UNUSED static __inline void jitter4testing(bool tiny); +MDBX_MAYBE_UNUSED MDBX_INTERNAL void osal_jitter(bool tiny); /* max bytes to write in one call */ #if defined(_WIN64) @@ -1445,14 +1361,12 @@ MDBX_MAYBE_UNUSED static __inline void jitter4testing(bool tiny); #else #define MAX_WRITE UINT32_C(0x3f000000) -#if defined(F_GETLK64) && defined(F_SETLK64) && defined(F_SETLKW64) && \ - !defined(__ANDROID_API__) +#if defined(F_GETLK64) && defined(F_SETLK64) && defined(F_SETLKW64) && !defined(__ANDROID_API__) #define MDBX_F_SETLK F_SETLK64 #define MDBX_F_SETLKW F_SETLKW64 #define MDBX_F_GETLK F_GETLK64 -#if (__GLIBC_PREREQ(2, 28) && \ - (defined(__USE_LARGEFILE64) || defined(__LARGEFILE64_SOURCE) || \ - defined(_USE_LARGEFILE64) || defined(_LARGEFILE64_SOURCE))) || \ +#if (__GLIBC_PREREQ(2, 28) && (defined(__USE_LARGEFILE64) || defined(__LARGEFILE64_SOURCE) || \ + defined(_USE_LARGEFILE64) || defined(_LARGEFILE64_SOURCE))) || \ defined(fcntl64) #define MDBX_FCNTL fcntl64 #else @@ -1470,8 +1384,7 @@ MDBX_MAYBE_UNUSED static __inline void jitter4testing(bool tiny); #define MDBX_STRUCT_FLOCK struct flock #endif /* MDBX_F_SETLK, MDBX_F_SETLKW, MDBX_F_GETLK */ -#if defined(F_OFD_SETLK64) && defined(F_OFD_SETLKW64) && \ - defined(F_OFD_GETLK64) && !defined(__ANDROID_API__) +#if defined(F_OFD_SETLK64) && defined(F_OFD_SETLKW64) && defined(F_OFD_GETLK64) && !defined(__ANDROID_API__) #define MDBX_F_OFD_SETLK F_OFD_SETLK64 #define MDBX_F_OFD_SETLKW F_OFD_SETLKW64 #define MDBX_F_OFD_GETLK F_OFD_GETLK64 @@ -1480,23 +1393,17 @@ MDBX_MAYBE_UNUSED static __inline void jitter4testing(bool tiny); #define MDBX_F_OFD_SETLKW F_OFD_SETLKW #define MDBX_F_OFD_GETLK F_OFD_GETLK #ifndef OFF_T_MAX -#define OFF_T_MAX \ - (((sizeof(off_t) > 4) ? INT64_MAX : INT32_MAX) & ~(size_t)0xFffff) +#define OFF_T_MAX (((sizeof(off_t) > 4) ? INT64_MAX : INT32_MAX) & ~(size_t)0xFffff) #endif /* OFF_T_MAX */ #endif /* MDBX_F_OFD_SETLK64, MDBX_F_OFD_SETLKW64, MDBX_F_OFD_GETLK64 */ -#endif - -#if defined(__linux__) || defined(__gnu_linux__) -MDBX_INTERNAL_VAR uint32_t linux_kernel_version; -MDBX_INTERNAL_VAR bool mdbx_RunningOnWSL1 /* Windows Subsystem 1 for Linux */; -#endif /* Linux */ +#endif /* !Windows */ #ifndef osal_strdup LIBMDBX_API char *osal_strdup(const char *str); #endif -MDBX_MAYBE_UNUSED static __inline int osal_get_errno(void) { +MDBX_MAYBE_UNUSED static inline int osal_get_errno(void) { #if defined(_WIN32) || defined(_WIN64) DWORD rc = GetLastError(); #else @@ -1506,40 +1413,32 @@ MDBX_MAYBE_UNUSED static __inline int osal_get_errno(void) { } #ifndef osal_memalign_alloc -MDBX_INTERNAL_FUNC int osal_memalign_alloc(size_t alignment, size_t bytes, - void **result); +MDBX_INTERNAL int osal_memalign_alloc(size_t alignment, size_t bytes, void **result); #endif #ifndef osal_memalign_free -MDBX_INTERNAL_FUNC void osal_memalign_free(void *ptr); -#endif - -MDBX_INTERNAL_FUNC int osal_condpair_init(osal_condpair_t *condpair); -MDBX_INTERNAL_FUNC int osal_condpair_lock(osal_condpair_t *condpair); -MDBX_INTERNAL_FUNC int osal_condpair_unlock(osal_condpair_t *condpair); -MDBX_INTERNAL_FUNC int osal_condpair_signal(osal_condpair_t *condpair, - bool part); -MDBX_INTERNAL_FUNC int osal_condpair_wait(osal_condpair_t *condpair, bool part); -MDBX_INTERNAL_FUNC int osal_condpair_destroy(osal_condpair_t *condpair); - -MDBX_INTERNAL_FUNC int osal_fastmutex_init(osal_fastmutex_t *fastmutex); -MDBX_INTERNAL_FUNC int osal_fastmutex_acquire(osal_fastmutex_t *fastmutex); -MDBX_INTERNAL_FUNC int osal_fastmutex_release(osal_fastmutex_t *fastmutex); -MDBX_INTERNAL_FUNC int osal_fastmutex_destroy(osal_fastmutex_t *fastmutex); - -MDBX_INTERNAL_FUNC int osal_pwritev(mdbx_filehandle_t fd, struct iovec *iov, - size_t sgvcnt, uint64_t offset); -MDBX_INTERNAL_FUNC int osal_pread(mdbx_filehandle_t fd, void *buf, size_t count, - uint64_t offset); -MDBX_INTERNAL_FUNC int osal_pwrite(mdbx_filehandle_t fd, const void *buf, - size_t count, uint64_t offset); -MDBX_INTERNAL_FUNC int osal_write(mdbx_filehandle_t fd, const void *buf, - size_t count); - -MDBX_INTERNAL_FUNC int -osal_thread_create(osal_thread_t *thread, - THREAD_RESULT(THREAD_CALL *start_routine)(void *), - void *arg); -MDBX_INTERNAL_FUNC int osal_thread_join(osal_thread_t thread); +MDBX_INTERNAL void osal_memalign_free(void *ptr); +#endif + +MDBX_INTERNAL int osal_condpair_init(osal_condpair_t *condpair); +MDBX_INTERNAL int osal_condpair_lock(osal_condpair_t *condpair); +MDBX_INTERNAL int osal_condpair_unlock(osal_condpair_t *condpair); +MDBX_INTERNAL int osal_condpair_signal(osal_condpair_t *condpair, bool part); +MDBX_INTERNAL int osal_condpair_wait(osal_condpair_t *condpair, bool part); +MDBX_INTERNAL int osal_condpair_destroy(osal_condpair_t *condpair); + +MDBX_INTERNAL int osal_fastmutex_init(osal_fastmutex_t *fastmutex); +MDBX_INTERNAL int osal_fastmutex_acquire(osal_fastmutex_t *fastmutex); +MDBX_INTERNAL int osal_fastmutex_release(osal_fastmutex_t *fastmutex); +MDBX_INTERNAL int osal_fastmutex_destroy(osal_fastmutex_t *fastmutex); + +MDBX_INTERNAL int osal_pwritev(mdbx_filehandle_t fd, struct iovec *iov, size_t sgvcnt, uint64_t offset); +MDBX_INTERNAL int osal_pread(mdbx_filehandle_t fd, void *buf, size_t count, uint64_t offset); +MDBX_INTERNAL int osal_pwrite(mdbx_filehandle_t fd, const void *buf, size_t count, uint64_t offset); +MDBX_INTERNAL int osal_write(mdbx_filehandle_t fd, const void *buf, size_t count); + +MDBX_INTERNAL int osal_thread_create(osal_thread_t *thread, THREAD_RESULT(THREAD_CALL *start_routine)(void *), + void *arg); +MDBX_INTERNAL int osal_thread_join(osal_thread_t thread); enum osal_syncmode_bits { MDBX_SYNC_NONE = 0, @@ -1549,11 +1448,10 @@ enum osal_syncmode_bits { MDBX_SYNC_IODQ = 8 }; -MDBX_INTERNAL_FUNC int osal_fsync(mdbx_filehandle_t fd, - const enum osal_syncmode_bits mode_bits); -MDBX_INTERNAL_FUNC int osal_ftruncate(mdbx_filehandle_t fd, uint64_t length); -MDBX_INTERNAL_FUNC int osal_fseek(mdbx_filehandle_t fd, uint64_t pos); -MDBX_INTERNAL_FUNC int osal_filesize(mdbx_filehandle_t fd, uint64_t *length); +MDBX_INTERNAL int osal_fsync(mdbx_filehandle_t fd, const enum osal_syncmode_bits mode_bits); +MDBX_INTERNAL int osal_ftruncate(mdbx_filehandle_t fd, uint64_t length); +MDBX_INTERNAL int osal_fseek(mdbx_filehandle_t fd, uint64_t pos); +MDBX_INTERNAL int osal_filesize(mdbx_filehandle_t fd, uint64_t *length); enum osal_openfile_purpose { MDBX_OPEN_DXB_READ, @@ -1568,7 +1466,7 @@ enum osal_openfile_purpose { MDBX_OPEN_DELETE }; -MDBX_MAYBE_UNUSED static __inline bool osal_isdirsep(pathchar_t c) { +MDBX_MAYBE_UNUSED static inline bool osal_isdirsep(pathchar_t c) { return #if defined(_WIN32) || defined(_WIN64) c == '\\' || @@ -1576,50 +1474,39 @@ MDBX_MAYBE_UNUSED static __inline bool osal_isdirsep(pathchar_t c) { c == '/'; } -MDBX_INTERNAL_FUNC bool osal_pathequal(const pathchar_t *l, const pathchar_t *r, - size_t len); -MDBX_INTERNAL_FUNC pathchar_t *osal_fileext(const pathchar_t *pathname, - size_t len); -MDBX_INTERNAL_FUNC int osal_fileexists(const pathchar_t *pathname); -MDBX_INTERNAL_FUNC int osal_openfile(const enum osal_openfile_purpose purpose, - const MDBX_env *env, - const pathchar_t *pathname, - mdbx_filehandle_t *fd, - mdbx_mode_t unix_mode_bits); -MDBX_INTERNAL_FUNC int osal_closefile(mdbx_filehandle_t fd); -MDBX_INTERNAL_FUNC int osal_removefile(const pathchar_t *pathname); -MDBX_INTERNAL_FUNC int osal_removedirectory(const pathchar_t *pathname); -MDBX_INTERNAL_FUNC int osal_is_pipe(mdbx_filehandle_t fd); -MDBX_INTERNAL_FUNC int osal_lockfile(mdbx_filehandle_t fd, bool wait); +MDBX_INTERNAL bool osal_pathequal(const pathchar_t *l, const pathchar_t *r, size_t len); +MDBX_INTERNAL pathchar_t *osal_fileext(const pathchar_t *pathname, size_t len); +MDBX_INTERNAL int osal_fileexists(const pathchar_t *pathname); +MDBX_INTERNAL int osal_openfile(const enum osal_openfile_purpose purpose, const MDBX_env *env, + const pathchar_t *pathname, mdbx_filehandle_t *fd, mdbx_mode_t unix_mode_bits); +MDBX_INTERNAL int osal_closefile(mdbx_filehandle_t fd); +MDBX_INTERNAL int osal_removefile(const pathchar_t *pathname); +MDBX_INTERNAL int osal_removedirectory(const pathchar_t *pathname); +MDBX_INTERNAL int osal_is_pipe(mdbx_filehandle_t fd); +MDBX_INTERNAL int osal_lockfile(mdbx_filehandle_t fd, bool wait); #define MMAP_OPTION_TRUNCATE 1 #define MMAP_OPTION_SEMAPHORE 2 -MDBX_INTERNAL_FUNC int osal_mmap(const int flags, osal_mmap_t *map, size_t size, - const size_t limit, const unsigned options); -MDBX_INTERNAL_FUNC int osal_munmap(osal_mmap_t *map); +MDBX_INTERNAL int osal_mmap(const int flags, osal_mmap_t *map, size_t size, const size_t limit, const unsigned options, + const pathchar_t *pathname4logging); +MDBX_INTERNAL int osal_munmap(osal_mmap_t *map); #define MDBX_MRESIZE_MAY_MOVE 0x00000100 #define MDBX_MRESIZE_MAY_UNMAP 0x00000200 -MDBX_INTERNAL_FUNC int osal_mresize(const int flags, osal_mmap_t *map, - size_t size, size_t limit); +MDBX_INTERNAL int osal_mresize(const int flags, osal_mmap_t *map, size_t size, size_t limit); #if defined(_WIN32) || defined(_WIN64) typedef struct { unsigned limit, count; HANDLE handles[31]; } mdbx_handle_array_t; -MDBX_INTERNAL_FUNC int -osal_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array); -MDBX_INTERNAL_FUNC int -osal_resume_threads_after_remap(mdbx_handle_array_t *array); +MDBX_INTERNAL int osal_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array); +MDBX_INTERNAL int osal_resume_threads_after_remap(mdbx_handle_array_t *array); #endif /* Windows */ -MDBX_INTERNAL_FUNC int osal_msync(const osal_mmap_t *map, size_t offset, - size_t length, - enum osal_syncmode_bits mode_bits); -MDBX_INTERNAL_FUNC int osal_check_fs_rdonly(mdbx_filehandle_t handle, - const pathchar_t *pathname, - int err); -MDBX_INTERNAL_FUNC int osal_check_fs_incore(mdbx_filehandle_t handle); - -MDBX_MAYBE_UNUSED static __inline uint32_t osal_getpid(void) { +MDBX_INTERNAL int osal_msync(const osal_mmap_t *map, size_t offset, size_t length, enum osal_syncmode_bits mode_bits); +MDBX_INTERNAL int osal_check_fs_rdonly(mdbx_filehandle_t handle, const pathchar_t *pathname, int err); +MDBX_INTERNAL int osal_check_fs_incore(mdbx_filehandle_t handle); +MDBX_INTERNAL int osal_check_fs_local(mdbx_filehandle_t handle, int flags); + +MDBX_MAYBE_UNUSED static inline uint32_t osal_getpid(void) { STATIC_ASSERT(sizeof(mdbx_pid_t) <= sizeof(uint32_t)); #if defined(_WIN32) || defined(_WIN64) return GetCurrentProcessId(); @@ -1629,7 +1516,7 @@ MDBX_MAYBE_UNUSED static __inline uint32_t osal_getpid(void) { #endif } -MDBX_MAYBE_UNUSED static __inline uintptr_t osal_thread_self(void) { +MDBX_MAYBE_UNUSED static inline uintptr_t osal_thread_self(void) { mdbx_tid_t thunk; STATIC_ASSERT(sizeof(uintptr_t) >= sizeof(thunk)); #if defined(_WIN32) || defined(_WIN64) @@ -1642,274 +1529,51 @@ MDBX_MAYBE_UNUSED static __inline uintptr_t osal_thread_self(void) { #if !defined(_WIN32) && !defined(_WIN64) #if defined(__ANDROID_API__) || defined(ANDROID) || defined(BIONIC) -MDBX_INTERNAL_FUNC int osal_check_tid4bionic(void); +MDBX_INTERNAL int osal_check_tid4bionic(void); #else -static __inline int osal_check_tid4bionic(void) { return 0; } +static inline int osal_check_tid4bionic(void) { return 0; } #endif /* __ANDROID_API__ || ANDROID) || BIONIC */ -MDBX_MAYBE_UNUSED static __inline int -osal_pthread_mutex_lock(pthread_mutex_t *mutex) { +MDBX_MAYBE_UNUSED static inline int osal_pthread_mutex_lock(pthread_mutex_t *mutex) { int err = osal_check_tid4bionic(); return unlikely(err) ? err : pthread_mutex_lock(mutex); } #endif /* !Windows */ -MDBX_INTERNAL_FUNC uint64_t osal_monotime(void); -MDBX_INTERNAL_FUNC uint64_t osal_cputime(size_t *optional_page_faults); -MDBX_INTERNAL_FUNC uint64_t osal_16dot16_to_monotime(uint32_t seconds_16dot16); -MDBX_INTERNAL_FUNC uint32_t osal_monotime_to_16dot16(uint64_t monotime); +MDBX_INTERNAL uint64_t osal_monotime(void); +MDBX_INTERNAL uint64_t osal_cputime(size_t *optional_page_faults); +MDBX_INTERNAL uint64_t osal_16dot16_to_monotime(uint32_t seconds_16dot16); +MDBX_INTERNAL uint32_t osal_monotime_to_16dot16(uint64_t monotime); -MDBX_MAYBE_UNUSED static inline uint32_t -osal_monotime_to_16dot16_noUnderflow(uint64_t monotime) { +MDBX_MAYBE_UNUSED static inline uint32_t osal_monotime_to_16dot16_noUnderflow(uint64_t monotime) { uint32_t seconds_16dot16 = osal_monotime_to_16dot16(monotime); return seconds_16dot16 ? seconds_16dot16 : /* fix underflow */ (monotime > 0); } -MDBX_INTERNAL_FUNC bin128_t osal_bootid(void); /*----------------------------------------------------------------------------*/ -/* lck stuff */ - -/// \brief Initialization of synchronization primitives linked with MDBX_env -/// instance both in LCK-file and within the current process. -/// \param -/// global_uniqueness_flag = true - denotes that there are no other processes -/// working with DB and LCK-file. Thus the function MUST initialize -/// shared synchronization objects in memory-mapped LCK-file. -/// global_uniqueness_flag = false - denotes that at least one process is -/// already working with DB and LCK-file, including the case when DB -/// has already been opened in the current process. Thus the function -/// MUST NOT initialize shared synchronization objects in memory-mapped -/// LCK-file that are already in use. -/// \return Error code or zero on success. -MDBX_INTERNAL_FUNC int osal_lck_init(MDBX_env *env, - MDBX_env *inprocess_neighbor, - int global_uniqueness_flag); - -/// \brief Disconnects from shared interprocess objects and destructs -/// synchronization objects linked with MDBX_env instance -/// within the current process. -/// \param -/// inprocess_neighbor = NULL - if the current process does not have other -/// instances of MDBX_env linked with the DB being closed. -/// Thus the function MUST check for other processes working with DB or -/// LCK-file, and keep or destroy shared synchronization objects in -/// memory-mapped LCK-file depending on the result. -/// inprocess_neighbor = not-NULL - pointer to another instance of MDBX_env -/// (anyone of there is several) working with DB or LCK-file within the -/// current process. Thus the function MUST NOT try to acquire exclusive -/// lock and/or try to destruct shared synchronization objects linked with -/// DB or LCK-file. Moreover, the implementation MUST ensure correct work -/// of other instances of MDBX_env within the current process, e.g. -/// restore POSIX-fcntl locks after the closing of file descriptors. -/// \return Error code (MDBX_PANIC) or zero on success. -MDBX_INTERNAL_FUNC int osal_lck_destroy(MDBX_env *env, - MDBX_env *inprocess_neighbor); - -/// \brief Connects to shared interprocess locking objects and tries to acquire -/// the maximum lock level (shared if exclusive is not available) -/// Depending on implementation or/and platform (Windows) this function may -/// acquire the non-OS super-level lock (e.g. for shared synchronization -/// objects initialization), which will be downgraded to OS-exclusive or -/// shared via explicit calling of osal_lck_downgrade(). -/// \return -/// MDBX_RESULT_TRUE (-1) - if an exclusive lock was acquired and thus -/// the current process is the first and only after the last use of DB. -/// MDBX_RESULT_FALSE (0) - if a shared lock was acquired and thus -/// DB has already been opened and now is used by other processes. -/// Otherwise (not 0 and not -1) - error code. -MDBX_INTERNAL_FUNC int osal_lck_seize(MDBX_env *env); - -/// \brief Downgrades the level of initially acquired lock to -/// operational level specified by argument. The reason for such downgrade: -/// - unblocking of other processes that are waiting for access, i.e. -/// if (env->me_flags & MDBX_EXCLUSIVE) != 0, then other processes -/// should be made aware that access is unavailable rather than -/// wait for it. -/// - freeing locks that interfere file operation (especially for Windows) -/// (env->me_flags & MDBX_EXCLUSIVE) == 0 - downgrade to shared lock. -/// (env->me_flags & MDBX_EXCLUSIVE) != 0 - downgrade to exclusive -/// operational lock. -/// \return Error code or zero on success -MDBX_INTERNAL_FUNC int osal_lck_downgrade(MDBX_env *env); - -/// \brief Locks LCK-file or/and table of readers for (de)registering. -/// \return Error code or zero on success -MDBX_INTERNAL_FUNC int osal_rdt_lock(MDBX_env *env); - -/// \brief Unlocks LCK-file or/and table of readers after (de)registering. -MDBX_INTERNAL_FUNC void osal_rdt_unlock(MDBX_env *env); - -/// \brief Acquires lock for DB change (on writing transaction start) -/// Reading transactions will not be blocked. -/// Declared as LIBMDBX_API because it is used in mdbx_chk. -/// \return Error code or zero on success -LIBMDBX_API int mdbx_txn_lock(MDBX_env *env, bool dont_wait); - -/// \brief Releases lock once DB changes is made (after writing transaction -/// has finished). -/// Declared as LIBMDBX_API because it is used in mdbx_chk. -LIBMDBX_API void mdbx_txn_unlock(MDBX_env *env); - -/// \brief Sets alive-flag of reader presence (indicative lock) for PID of -/// the current process. The function does no more than needed for -/// the correct working of osal_rpid_check() in other processes. -/// \return Error code or zero on success -MDBX_INTERNAL_FUNC int osal_rpid_set(MDBX_env *env); - -/// \brief Resets alive-flag of reader presence (indicative lock) -/// for PID of the current process. The function does no more than needed -/// for the correct working of osal_rpid_check() in other processes. -/// \return Error code or zero on success -MDBX_INTERNAL_FUNC int osal_rpid_clear(MDBX_env *env); - -/// \brief Checks for reading process status with the given pid with help of -/// alive-flag of presence (indicative lock) or using another way. -/// \return -/// MDBX_RESULT_TRUE (-1) - if the reader process with the given PID is alive -/// and working with DB (indicative lock is present). -/// MDBX_RESULT_FALSE (0) - if the reader process with the given PID is absent -/// or not working with DB (indicative lock is not present). -/// Otherwise (not 0 and not -1) - error code. -MDBX_INTERNAL_FUNC int osal_rpid_check(MDBX_env *env, uint32_t pid); - -#if defined(_WIN32) || defined(_WIN64) - -MDBX_INTERNAL_FUNC int osal_mb2w(const char *const src, wchar_t **const pdst); - -typedef void(WINAPI *osal_srwlock_t_function)(osal_srwlock_t *); -MDBX_INTERNAL_VAR osal_srwlock_t_function osal_srwlock_Init, - osal_srwlock_AcquireShared, osal_srwlock_ReleaseShared, - osal_srwlock_AcquireExclusive, osal_srwlock_ReleaseExclusive; - -#if _WIN32_WINNT < 0x0600 /* prior to Windows Vista */ -typedef enum _FILE_INFO_BY_HANDLE_CLASS { - FileBasicInfo, - FileStandardInfo, - FileNameInfo, - FileRenameInfo, - FileDispositionInfo, - FileAllocationInfo, - FileEndOfFileInfo, - FileStreamInfo, - FileCompressionInfo, - FileAttributeTagInfo, - FileIdBothDirectoryInfo, - FileIdBothDirectoryRestartInfo, - FileIoPriorityHintInfo, - FileRemoteProtocolInfo, - MaximumFileInfoByHandleClass -} FILE_INFO_BY_HANDLE_CLASS, - *PFILE_INFO_BY_HANDLE_CLASS; - -typedef struct _FILE_END_OF_FILE_INFO { - LARGE_INTEGER EndOfFile; -} FILE_END_OF_FILE_INFO, *PFILE_END_OF_FILE_INFO; - -#define REMOTE_PROTOCOL_INFO_FLAG_LOOPBACK 0x00000001 -#define REMOTE_PROTOCOL_INFO_FLAG_OFFLINE 0x00000002 - -typedef struct _FILE_REMOTE_PROTOCOL_INFO { - USHORT StructureVersion; - USHORT StructureSize; - DWORD Protocol; - USHORT ProtocolMajorVersion; - USHORT ProtocolMinorVersion; - USHORT ProtocolRevision; - USHORT Reserved; - DWORD Flags; - struct { - DWORD Reserved[8]; - } GenericReserved; - struct { - DWORD Reserved[16]; - } ProtocolSpecificReserved; -} FILE_REMOTE_PROTOCOL_INFO, *PFILE_REMOTE_PROTOCOL_INFO; - -#endif /* _WIN32_WINNT < 0x0600 (prior to Windows Vista) */ - -typedef BOOL(WINAPI *MDBX_GetFileInformationByHandleEx)( - _In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass, - _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize); -MDBX_INTERNAL_VAR MDBX_GetFileInformationByHandleEx - mdbx_GetFileInformationByHandleEx; - -typedef BOOL(WINAPI *MDBX_GetVolumeInformationByHandleW)( - _In_ HANDLE hFile, _Out_opt_ LPWSTR lpVolumeNameBuffer, - _In_ DWORD nVolumeNameSize, _Out_opt_ LPDWORD lpVolumeSerialNumber, - _Out_opt_ LPDWORD lpMaximumComponentLength, - _Out_opt_ LPDWORD lpFileSystemFlags, - _Out_opt_ LPWSTR lpFileSystemNameBuffer, _In_ DWORD nFileSystemNameSize); -MDBX_INTERNAL_VAR MDBX_GetVolumeInformationByHandleW - mdbx_GetVolumeInformationByHandleW; - -typedef DWORD(WINAPI *MDBX_GetFinalPathNameByHandleW)(_In_ HANDLE hFile, - _Out_ LPWSTR lpszFilePath, - _In_ DWORD cchFilePath, - _In_ DWORD dwFlags); -MDBX_INTERNAL_VAR MDBX_GetFinalPathNameByHandleW mdbx_GetFinalPathNameByHandleW; - -typedef BOOL(WINAPI *MDBX_SetFileInformationByHandle)( - _In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass, - _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize); -MDBX_INTERNAL_VAR MDBX_SetFileInformationByHandle - mdbx_SetFileInformationByHandle; - -typedef NTSTATUS(NTAPI *MDBX_NtFsControlFile)( - IN HANDLE FileHandle, IN OUT HANDLE Event, - IN OUT PVOID /* PIO_APC_ROUTINE */ ApcRoutine, IN OUT PVOID ApcContext, - OUT PIO_STATUS_BLOCK IoStatusBlock, IN ULONG FsControlCode, - IN OUT PVOID InputBuffer, IN ULONG InputBufferLength, - OUT OPTIONAL PVOID OutputBuffer, IN ULONG OutputBufferLength); -MDBX_INTERNAL_VAR MDBX_NtFsControlFile mdbx_NtFsControlFile; - -typedef uint64_t(WINAPI *MDBX_GetTickCount64)(void); -MDBX_INTERNAL_VAR MDBX_GetTickCount64 mdbx_GetTickCount64; - -#if !defined(_WIN32_WINNT_WIN8) || _WIN32_WINNT < _WIN32_WINNT_WIN8 -typedef struct _WIN32_MEMORY_RANGE_ENTRY { - PVOID VirtualAddress; - SIZE_T NumberOfBytes; -} WIN32_MEMORY_RANGE_ENTRY, *PWIN32_MEMORY_RANGE_ENTRY; -#endif /* Windows 8.x */ - -typedef BOOL(WINAPI *MDBX_PrefetchVirtualMemory)( - HANDLE hProcess, ULONG_PTR NumberOfEntries, - PWIN32_MEMORY_RANGE_ENTRY VirtualAddresses, ULONG Flags); -MDBX_INTERNAL_VAR MDBX_PrefetchVirtualMemory mdbx_PrefetchVirtualMemory; - -typedef enum _SECTION_INHERIT { ViewShare = 1, ViewUnmap = 2 } SECTION_INHERIT; - -typedef NTSTATUS(NTAPI *MDBX_NtExtendSection)(IN HANDLE SectionHandle, - IN PLARGE_INTEGER NewSectionSize); -MDBX_INTERNAL_VAR MDBX_NtExtendSection mdbx_NtExtendSection; - -static __inline bool mdbx_RunningUnderWine(void) { - return !mdbx_NtExtendSection; -} - -typedef LSTATUS(WINAPI *MDBX_RegGetValueA)(HKEY hkey, LPCSTR lpSubKey, - LPCSTR lpValue, DWORD dwFlags, - LPDWORD pdwType, PVOID pvData, - LPDWORD pcbData); -MDBX_INTERNAL_VAR MDBX_RegGetValueA mdbx_RegGetValueA; -NTSYSAPI ULONG RtlRandomEx(PULONG Seed); - -typedef BOOL(WINAPI *MDBX_SetFileIoOverlappedRange)(HANDLE FileHandle, - PUCHAR OverlappedRangeStart, - ULONG Length); -MDBX_INTERNAL_VAR MDBX_SetFileIoOverlappedRange mdbx_SetFileIoOverlappedRange; +MDBX_INTERNAL void osal_ctor(void); +MDBX_INTERNAL void osal_dtor(void); +#if defined(_WIN32) || defined(_WIN64) +MDBX_INTERNAL int osal_mb2w(const char *const src, wchar_t **const pdst); #endif /* Windows */ -#endif /* !__cplusplus */ +typedef union bin128 { + __anonymous_struct_extension__ struct { + uint64_t x, y; + }; + __anonymous_struct_extension__ struct { + uint32_t a, b, c, d; + }; +} bin128_t; + +MDBX_INTERNAL bin128_t osal_guid(const MDBX_env *); /*----------------------------------------------------------------------------*/ -MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static __always_inline uint64_t -osal_bswap64(uint64_t v) { -#if __GNUC_PREREQ(4, 4) || __CLANG_PREREQ(4, 0) || \ - __has_builtin(__builtin_bswap64) +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint64_t osal_bswap64(uint64_t v) { +#if __GNUC_PREREQ(4, 4) || __CLANG_PREREQ(4, 0) || __has_builtin(__builtin_bswap64) return __builtin_bswap64(v); #elif defined(_MSC_VER) && !defined(__clang__) return _byteswap_uint64(v); @@ -1918,19 +1582,14 @@ osal_bswap64(uint64_t v) { #elif defined(bswap_64) return bswap_64(v); #else - return v << 56 | v >> 56 | ((v << 40) & UINT64_C(0x00ff000000000000)) | - ((v << 24) & UINT64_C(0x0000ff0000000000)) | - ((v << 8) & UINT64_C(0x000000ff00000000)) | - ((v >> 8) & UINT64_C(0x00000000ff000000)) | - ((v >> 24) & UINT64_C(0x0000000000ff0000)) | - ((v >> 40) & UINT64_C(0x000000000000ff00)); + return v << 56 | v >> 56 | ((v << 40) & UINT64_C(0x00ff000000000000)) | ((v << 24) & UINT64_C(0x0000ff0000000000)) | + ((v << 8) & UINT64_C(0x000000ff00000000)) | ((v >> 8) & UINT64_C(0x00000000ff000000)) | + ((v >> 24) & UINT64_C(0x0000000000ff0000)) | ((v >> 40) & UINT64_C(0x000000000000ff00)); #endif } -MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static __always_inline uint32_t -osal_bswap32(uint32_t v) { -#if __GNUC_PREREQ(4, 4) || __CLANG_PREREQ(4, 0) || \ - __has_builtin(__builtin_bswap32) +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint32_t osal_bswap32(uint32_t v) { +#if __GNUC_PREREQ(4, 4) || __CLANG_PREREQ(4, 0) || __has_builtin(__builtin_bswap32) return __builtin_bswap32(v); #elif defined(_MSC_VER) && !defined(__clang__) return _byteswap_ulong(v); @@ -1939,50 +1598,14 @@ osal_bswap32(uint32_t v) { #elif defined(bswap_32) return bswap_32(v); #else - return v << 24 | v >> 24 | ((v << 8) & UINT32_C(0x00ff0000)) | - ((v >> 8) & UINT32_C(0x0000ff00)); + return v << 24 | v >> 24 | ((v << 8) & UINT32_C(0x00ff0000)) | ((v >> 8) & UINT32_C(0x0000ff00)); #endif } -/*----------------------------------------------------------------------------*/ - -#if defined(_MSC_VER) && _MSC_VER >= 1900 -/* LY: MSVC 2015/2017/2019 has buggy/inconsistent PRIuPTR/PRIxPTR macros - * for internal format-args checker. */ -#undef PRIuPTR -#undef PRIiPTR -#undef PRIdPTR -#undef PRIxPTR -#define PRIuPTR "Iu" -#define PRIiPTR "Ii" -#define PRIdPTR "Id" -#define PRIxPTR "Ix" -#define PRIuSIZE "zu" -#define PRIiSIZE "zi" -#define PRIdSIZE "zd" -#define PRIxSIZE "zx" -#endif /* fix PRI*PTR for _MSC_VER */ - -#ifndef PRIuSIZE -#define PRIuSIZE PRIuPTR -#define PRIiSIZE PRIiPTR -#define PRIdSIZE PRIdPTR -#define PRIxSIZE PRIxPTR -#endif /* PRI*SIZE macros for MSVC */ - -#ifdef _MSC_VER -#pragma warning(pop) -#endif - -#define mdbx_sourcery_anchor XCONCAT(mdbx_sourcery_, MDBX_BUILD_SOURCERY) -#if defined(xMDBX_TOOLS) -extern LIBMDBX_API const char *const mdbx_sourcery_anchor; -#endif - /******************************************************************************* - ******************************************************************************* ******************************************************************************* * + * BUILD TIME * * #### ##### ##### # #### # # #### * # # # # # # # # ## # # @@ -2003,23 +1626,15 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Using fsync() with chance of data lost on power failure */ #define MDBX_OSX_WANNA_SPEED 1 -#ifndef MDBX_OSX_SPEED_INSTEADOF_DURABILITY +#ifndef MDBX_APPLE_SPEED_INSTEADOF_DURABILITY /** Choices \ref MDBX_OSX_WANNA_DURABILITY or \ref MDBX_OSX_WANNA_SPEED * for OSX & iOS */ -#define MDBX_OSX_SPEED_INSTEADOF_DURABILITY MDBX_OSX_WANNA_DURABILITY -#endif /* MDBX_OSX_SPEED_INSTEADOF_DURABILITY */ - -/** Controls using of POSIX' madvise() and/or similar hints. */ -#ifndef MDBX_ENABLE_MADVISE -#define MDBX_ENABLE_MADVISE 1 -#elif !(MDBX_ENABLE_MADVISE == 0 || MDBX_ENABLE_MADVISE == 1) -#error MDBX_ENABLE_MADVISE must be defined as 0 or 1 -#endif /* MDBX_ENABLE_MADVISE */ +#define MDBX_APPLE_SPEED_INSTEADOF_DURABILITY MDBX_OSX_WANNA_DURABILITY +#endif /* MDBX_APPLE_SPEED_INSTEADOF_DURABILITY */ /** Controls checking PID against reuse DB environment after the fork() */ #ifndef MDBX_ENV_CHECKPID -#if (defined(MADV_DONTFORK) && MDBX_ENABLE_MADVISE) || defined(_WIN32) || \ - defined(_WIN64) +#if defined(MADV_DONTFORK) || defined(_WIN32) || defined(_WIN64) /* PID check could be omitted: * - on Linux when madvise(MADV_DONTFORK) is available, i.e. after the fork() * mapped pages will not be available for child process. @@ -2048,8 +1663,7 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Does a system have battery-backed Real-Time Clock or just a fake. */ #ifndef MDBX_TRUST_RTC -#if defined(__linux__) || defined(__gnu_linux__) || defined(__NetBSD__) || \ - defined(__OpenBSD__) +#if defined(__linux__) || defined(__gnu_linux__) || defined(__NetBSD__) || defined(__OpenBSD__) #define MDBX_TRUST_RTC 0 /* a lot of embedded systems have a fake RTC */ #else #define MDBX_TRUST_RTC 1 @@ -2084,24 +1698,21 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Controls using Unix' mincore() to determine whether DB-pages * are resident in memory. */ -#ifndef MDBX_ENABLE_MINCORE +#ifndef MDBX_USE_MINCORE #if defined(MINCORE_INCORE) || !(defined(_WIN32) || defined(_WIN64)) -#define MDBX_ENABLE_MINCORE 1 +#define MDBX_USE_MINCORE 1 #else -#define MDBX_ENABLE_MINCORE 0 +#define MDBX_USE_MINCORE 0 #endif -#elif !(MDBX_ENABLE_MINCORE == 0 || MDBX_ENABLE_MINCORE == 1) -#error MDBX_ENABLE_MINCORE must be defined as 0 or 1 -#endif /* MDBX_ENABLE_MINCORE */ +#define MDBX_USE_MINCORE_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_USE_MINCORE) +#elif !(MDBX_USE_MINCORE == 0 || MDBX_USE_MINCORE == 1) +#error MDBX_USE_MINCORE must be defined as 0 or 1 +#endif /* MDBX_USE_MINCORE */ /** Enables chunking long list of retired pages during huge transactions commit * to avoid use sequences of pages. */ #ifndef MDBX_ENABLE_BIGFOOT -#if MDBX_WORDBITS >= 64 || defined(DOXYGEN) #define MDBX_ENABLE_BIGFOOT 1 -#else -#define MDBX_ENABLE_BIGFOOT 0 -#endif #elif !(MDBX_ENABLE_BIGFOOT == 0 || MDBX_ENABLE_BIGFOOT == 1) #error MDBX_ENABLE_BIGFOOT must be defined as 0 or 1 #endif /* MDBX_ENABLE_BIGFOOT */ @@ -2116,25 +1727,27 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #ifndef MDBX_PNL_PREALLOC_FOR_RADIXSORT #define MDBX_PNL_PREALLOC_FOR_RADIXSORT 1 -#elif !(MDBX_PNL_PREALLOC_FOR_RADIXSORT == 0 || \ - MDBX_PNL_PREALLOC_FOR_RADIXSORT == 1) +#elif !(MDBX_PNL_PREALLOC_FOR_RADIXSORT == 0 || MDBX_PNL_PREALLOC_FOR_RADIXSORT == 1) #error MDBX_PNL_PREALLOC_FOR_RADIXSORT must be defined as 0 or 1 #endif /* MDBX_PNL_PREALLOC_FOR_RADIXSORT */ #ifndef MDBX_DPL_PREALLOC_FOR_RADIXSORT #define MDBX_DPL_PREALLOC_FOR_RADIXSORT 1 -#elif !(MDBX_DPL_PREALLOC_FOR_RADIXSORT == 0 || \ - MDBX_DPL_PREALLOC_FOR_RADIXSORT == 1) +#elif !(MDBX_DPL_PREALLOC_FOR_RADIXSORT == 0 || MDBX_DPL_PREALLOC_FOR_RADIXSORT == 1) #error MDBX_DPL_PREALLOC_FOR_RADIXSORT must be defined as 0 or 1 #endif /* MDBX_DPL_PREALLOC_FOR_RADIXSORT */ -/** Controls dirty pages tracking, spilling and persisting in MDBX_WRITEMAP - * mode. 0/OFF = Don't track dirty pages at all, don't spill ones, and use - * msync() to persist data. This is by-default on Linux and other systems where - * kernel provides properly LRU tracking and effective flushing on-demand. 1/ON - * = Tracking of dirty pages but with LRU labels for spilling and explicit - * persist ones by write(). This may be reasonable for systems which low - * performance of msync() and/or LRU tracking. */ +/** Controls dirty pages tracking, spilling and persisting in `MDBX_WRITEMAP` + * mode, i.e. disables in-memory database updating with consequent + * flush-to-disk/msync syscall. + * + * 0/OFF = Don't track dirty pages at all, don't spill ones, and use msync() to + * persist data. This is by-default on Linux and other systems where kernel + * provides properly LRU tracking and effective flushing on-demand. + * + * 1/ON = Tracking of dirty pages but with LRU labels for spilling and explicit + * persist ones by write(). This may be reasonable for goofy systems (Windows) + * which low performance of msync() and/or zany LRU tracking. */ #ifndef MDBX_AVOID_MSYNC #if defined(_WIN32) || defined(_WIN64) #define MDBX_AVOID_MSYNC 1 @@ -2145,6 +1758,22 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #error MDBX_AVOID_MSYNC must be defined as 0 or 1 #endif /* MDBX_AVOID_MSYNC */ +/** Управляет механизмом поддержки разреженных наборов DBI-хендлов для снижения + * накладных расходов при запуске и обработке транзакций. */ +#ifndef MDBX_ENABLE_DBI_SPARSE +#define MDBX_ENABLE_DBI_SPARSE 1 +#elif !(MDBX_ENABLE_DBI_SPARSE == 0 || MDBX_ENABLE_DBI_SPARSE == 1) +#error MDBX_ENABLE_DBI_SPARSE must be defined as 0 or 1 +#endif /* MDBX_ENABLE_DBI_SPARSE */ + +/** Управляет механизмом отложенного освобождения и поддержки пути быстрого + * открытия DBI-хендлов без захвата блокировок. */ +#ifndef MDBX_ENABLE_DBI_LOCKFREE +#define MDBX_ENABLE_DBI_LOCKFREE 1 +#elif !(MDBX_ENABLE_DBI_LOCKFREE == 0 || MDBX_ENABLE_DBI_LOCKFREE == 1) +#error MDBX_ENABLE_DBI_LOCKFREE must be defined as 0 or 1 +#endif /* MDBX_ENABLE_DBI_LOCKFREE */ + /** Controls sort order of internal page number lists. * This mostly experimental/advanced option with not for regular MDBX users. * \warning The database format depend on this option and libmdbx built with @@ -2157,7 +1786,11 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Avoid dependence from MSVC CRT and use ntdll.dll instead. */ #ifndef MDBX_WITHOUT_MSVC_CRT +#if defined(MDBX_BUILD_CXX) && !MDBX_BUILD_CXX #define MDBX_WITHOUT_MSVC_CRT 1 +#else +#define MDBX_WITHOUT_MSVC_CRT 0 +#endif #elif !(MDBX_WITHOUT_MSVC_CRT == 0 || MDBX_WITHOUT_MSVC_CRT == 1) #error MDBX_WITHOUT_MSVC_CRT must be defined as 0 or 1 #endif /* MDBX_WITHOUT_MSVC_CRT */ @@ -2165,12 +1798,11 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Size of buffer used during copying a environment/database file. */ #ifndef MDBX_ENVCOPY_WRITEBUF #define MDBX_ENVCOPY_WRITEBUF 1048576u -#elif MDBX_ENVCOPY_WRITEBUF < 65536u || MDBX_ENVCOPY_WRITEBUF > 1073741824u || \ - MDBX_ENVCOPY_WRITEBUF % 65536u +#elif MDBX_ENVCOPY_WRITEBUF < 65536u || MDBX_ENVCOPY_WRITEBUF > 1073741824u || MDBX_ENVCOPY_WRITEBUF % 65536u #error MDBX_ENVCOPY_WRITEBUF must be defined in range 65536..1073741824 and be multiple of 65536 #endif /* MDBX_ENVCOPY_WRITEBUF */ -/** Forces assertion checking */ +/** Forces assertion checking. */ #ifndef MDBX_FORCE_ASSERTIONS #define MDBX_FORCE_ASSERTIONS 0 #elif !(MDBX_FORCE_ASSERTIONS == 0 || MDBX_FORCE_ASSERTIONS == 1) @@ -2185,15 +1817,14 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #else #define MDBX_ASSUME_MALLOC_OVERHEAD (sizeof(void *) * 2u) #endif -#elif MDBX_ASSUME_MALLOC_OVERHEAD < 0 || MDBX_ASSUME_MALLOC_OVERHEAD > 64 || \ - MDBX_ASSUME_MALLOC_OVERHEAD % 4 +#elif MDBX_ASSUME_MALLOC_OVERHEAD < 0 || MDBX_ASSUME_MALLOC_OVERHEAD > 64 || MDBX_ASSUME_MALLOC_OVERHEAD % 4 #error MDBX_ASSUME_MALLOC_OVERHEAD must be defined in range 0..64 and be multiple of 4 #endif /* MDBX_ASSUME_MALLOC_OVERHEAD */ /** If defined then enables integration with Valgrind, * a memory analyzing tool. */ -#ifndef MDBX_USE_VALGRIND -#endif /* MDBX_USE_VALGRIND */ +#ifndef ENABLE_MEMCHECK +#endif /* ENABLE_MEMCHECK */ /** If defined then enables use C11 atomics, * otherwise detects ones availability automatically. */ @@ -2213,18 +1844,24 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #define MDBX_HAVE_BUILTIN_CPU_SUPPORTS 0 #elif defined(__e2k__) #define MDBX_HAVE_BUILTIN_CPU_SUPPORTS 0 -#elif __has_builtin(__builtin_cpu_supports) || \ - defined(__BUILTIN_CPU_SUPPORTS__) || \ +#elif __has_builtin(__builtin_cpu_supports) || defined(__BUILTIN_CPU_SUPPORTS__) || \ (defined(__ia32__) && __GNUC_PREREQ(4, 8) && __GLIBC_PREREQ(2, 23)) #define MDBX_HAVE_BUILTIN_CPU_SUPPORTS 1 #else #define MDBX_HAVE_BUILTIN_CPU_SUPPORTS 0 #endif -#elif !(MDBX_HAVE_BUILTIN_CPU_SUPPORTS == 0 || \ - MDBX_HAVE_BUILTIN_CPU_SUPPORTS == 1) +#elif !(MDBX_HAVE_BUILTIN_CPU_SUPPORTS == 0 || MDBX_HAVE_BUILTIN_CPU_SUPPORTS == 1) #error MDBX_HAVE_BUILTIN_CPU_SUPPORTS must be defined as 0 or 1 #endif /* MDBX_HAVE_BUILTIN_CPU_SUPPORTS */ +/** if enabled then instead of the returned error `MDBX_REMOTE`, only a warning is issued, when + * the database being opened in non-read-only mode is located in a file system exported via NFS. */ +#ifndef MDBX_ENABLE_NON_READONLY_EXPORT +#define MDBX_ENABLE_NON_READONLY_EXPORT 0 +#elif !(MDBX_ENABLE_NON_READONLY_EXPORT == 0 || MDBX_ENABLE_NON_READONLY_EXPORT == 1) +#error MDBX_ENABLE_NON_READONLY_EXPORT must be defined as 0 or 1 +#endif /* MDBX_ENABLE_NON_READONLY_EXPORT */ + //------------------------------------------------------------------------------ /** Win32 File Locking API for \ref MDBX_LOCKING */ @@ -2242,27 +1879,20 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** POSIX-2008 Robust Mutexes for \ref MDBX_LOCKING */ #define MDBX_LOCKING_POSIX2008 2008 -/** BeOS Benaphores, aka Futexes for \ref MDBX_LOCKING */ -#define MDBX_LOCKING_BENAPHORE 1995 - /** Advanced: Choices the locking implementation (autodetection by default). */ #if defined(_WIN32) || defined(_WIN64) #define MDBX_LOCKING MDBX_LOCKING_WIN32FILES #else #ifndef MDBX_LOCKING -#if defined(_POSIX_THREAD_PROCESS_SHARED) && \ - _POSIX_THREAD_PROCESS_SHARED >= 200112L && !defined(__FreeBSD__) +#if defined(_POSIX_THREAD_PROCESS_SHARED) && _POSIX_THREAD_PROCESS_SHARED >= 200112L && !defined(__FreeBSD__) /* Some platforms define the EOWNERDEAD error code even though they * don't support Robust Mutexes. If doubt compile with -MDBX_LOCKING=2001. */ -#if defined(EOWNERDEAD) && _POSIX_THREAD_PROCESS_SHARED >= 200809L && \ - ((defined(_POSIX_THREAD_ROBUST_PRIO_INHERIT) && \ - _POSIX_THREAD_ROBUST_PRIO_INHERIT > 0) || \ - (defined(_POSIX_THREAD_ROBUST_PRIO_PROTECT) && \ - _POSIX_THREAD_ROBUST_PRIO_PROTECT > 0) || \ - defined(PTHREAD_MUTEX_ROBUST) || defined(PTHREAD_MUTEX_ROBUST_NP)) && \ - (!defined(__GLIBC__) || \ - __GLIBC_PREREQ(2, 10) /* troubles with Robust mutexes before 2.10 */) +#if defined(EOWNERDEAD) && _POSIX_THREAD_PROCESS_SHARED >= 200809L && \ + ((defined(_POSIX_THREAD_ROBUST_PRIO_INHERIT) && _POSIX_THREAD_ROBUST_PRIO_INHERIT > 0) || \ + (defined(_POSIX_THREAD_ROBUST_PRIO_PROTECT) && _POSIX_THREAD_ROBUST_PRIO_PROTECT > 0) || \ + defined(PTHREAD_MUTEX_ROBUST) || defined(PTHREAD_MUTEX_ROBUST_NP)) && \ + (!defined(__GLIBC__) || __GLIBC_PREREQ(2, 10) /* troubles with Robust mutexes before 2.10 */) #define MDBX_LOCKING MDBX_LOCKING_POSIX2008 #else #define MDBX_LOCKING MDBX_LOCKING_POSIX2001 @@ -2280,12 +1910,9 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Advanced: Using POSIX OFD-locks (autodetection by default). */ #ifndef MDBX_USE_OFDLOCKS -#if ((defined(F_OFD_SETLK) && defined(F_OFD_SETLKW) && \ - defined(F_OFD_GETLK)) || \ - (defined(F_OFD_SETLK64) && defined(F_OFD_SETLKW64) && \ - defined(F_OFD_GETLK64))) && \ - !defined(MDBX_SAFE4QEMU) && \ - !defined(__sun) /* OFD-lock are broken on Solaris */ +#if ((defined(F_OFD_SETLK) && defined(F_OFD_SETLKW) && defined(F_OFD_GETLK)) || \ + (defined(F_OFD_SETLK64) && defined(F_OFD_SETLKW64) && defined(F_OFD_GETLK64))) && \ + !defined(MDBX_SAFE4QEMU) && !defined(__sun) /* OFD-lock are broken on Solaris */ #define MDBX_USE_OFDLOCKS 1 #else #define MDBX_USE_OFDLOCKS 0 @@ -2299,8 +1926,7 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Advanced: Using sendfile() syscall (autodetection by default). */ #ifndef MDBX_USE_SENDFILE -#if ((defined(__linux__) || defined(__gnu_linux__)) && \ - !defined(__ANDROID_API__)) || \ +#if ((defined(__linux__) || defined(__gnu_linux__)) && !defined(__ANDROID_API__)) || \ (defined(__ANDROID_API__) && __ANDROID_API__ >= 21) #define MDBX_USE_SENDFILE 1 #else @@ -2321,30 +1947,15 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #error MDBX_USE_COPYFILERANGE must be defined as 0 or 1 #endif /* MDBX_USE_COPYFILERANGE */ -/** Advanced: Using sync_file_range() syscall (autodetection by default). */ -#ifndef MDBX_USE_SYNCFILERANGE -#if ((defined(__linux__) || defined(__gnu_linux__)) && \ - defined(SYNC_FILE_RANGE_WRITE) && !defined(__ANDROID_API__)) || \ - (defined(__ANDROID_API__) && __ANDROID_API__ >= 26) -#define MDBX_USE_SYNCFILERANGE 1 -#else -#define MDBX_USE_SYNCFILERANGE 0 -#endif -#elif !(MDBX_USE_SYNCFILERANGE == 0 || MDBX_USE_SYNCFILERANGE == 1) -#error MDBX_USE_SYNCFILERANGE must be defined as 0 or 1 -#endif /* MDBX_USE_SYNCFILERANGE */ - //------------------------------------------------------------------------------ #ifndef MDBX_CPU_WRITEBACK_INCOHERENT -#if defined(__ia32__) || defined(__e2k__) || defined(__hppa) || \ - defined(__hppa__) || defined(DOXYGEN) +#if defined(__ia32__) || defined(__e2k__) || defined(__hppa) || defined(__hppa__) || defined(DOXYGEN) #define MDBX_CPU_WRITEBACK_INCOHERENT 0 #else #define MDBX_CPU_WRITEBACK_INCOHERENT 1 #endif -#elif !(MDBX_CPU_WRITEBACK_INCOHERENT == 0 || \ - MDBX_CPU_WRITEBACK_INCOHERENT == 1) +#elif !(MDBX_CPU_WRITEBACK_INCOHERENT == 0 || MDBX_CPU_WRITEBACK_INCOHERENT == 1) #error MDBX_CPU_WRITEBACK_INCOHERENT must be defined as 0 or 1 #endif /* MDBX_CPU_WRITEBACK_INCOHERENT */ @@ -2354,35 +1965,35 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #else #define MDBX_MMAP_INCOHERENT_FILE_WRITE 0 #endif -#elif !(MDBX_MMAP_INCOHERENT_FILE_WRITE == 0 || \ - MDBX_MMAP_INCOHERENT_FILE_WRITE == 1) +#elif !(MDBX_MMAP_INCOHERENT_FILE_WRITE == 0 || MDBX_MMAP_INCOHERENT_FILE_WRITE == 1) #error MDBX_MMAP_INCOHERENT_FILE_WRITE must be defined as 0 or 1 #endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */ #ifndef MDBX_MMAP_INCOHERENT_CPU_CACHE -#if defined(__mips) || defined(__mips__) || defined(__mips64) || \ - defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \ - defined(__MWERKS__) || defined(__sgi) +#if defined(__mips) || defined(__mips__) || defined(__mips64) || defined(__mips64__) || defined(_M_MRX000) || \ + defined(_MIPS_) || defined(__MWERKS__) || defined(__sgi) /* MIPS has cache coherency issues. */ #define MDBX_MMAP_INCOHERENT_CPU_CACHE 1 #else /* LY: assume no relevant mmap/dcache issues. */ #define MDBX_MMAP_INCOHERENT_CPU_CACHE 0 #endif -#elif !(MDBX_MMAP_INCOHERENT_CPU_CACHE == 0 || \ - MDBX_MMAP_INCOHERENT_CPU_CACHE == 1) +#elif !(MDBX_MMAP_INCOHERENT_CPU_CACHE == 0 || MDBX_MMAP_INCOHERENT_CPU_CACHE == 1) #error MDBX_MMAP_INCOHERENT_CPU_CACHE must be defined as 0 or 1 #endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */ -#ifndef MDBX_MMAP_USE_MS_ASYNC -#if MDBX_MMAP_INCOHERENT_FILE_WRITE || MDBX_MMAP_INCOHERENT_CPU_CACHE -#define MDBX_MMAP_USE_MS_ASYNC 1 +/** Assume system needs explicit syscall to sync/flush/write modified mapped + * memory. */ +#ifndef MDBX_MMAP_NEEDS_JOLT +#if MDBX_MMAP_INCOHERENT_FILE_WRITE || MDBX_MMAP_INCOHERENT_CPU_CACHE || !(defined(__linux__) || defined(__gnu_linux__)) +#define MDBX_MMAP_NEEDS_JOLT 1 #else -#define MDBX_MMAP_USE_MS_ASYNC 0 +#define MDBX_MMAP_NEEDS_JOLT 0 #endif -#elif !(MDBX_MMAP_USE_MS_ASYNC == 0 || MDBX_MMAP_USE_MS_ASYNC == 1) -#error MDBX_MMAP_USE_MS_ASYNC must be defined as 0 or 1 -#endif /* MDBX_MMAP_USE_MS_ASYNC */ +#define MDBX_MMAP_NEEDS_JOLT_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_MMAP_NEEDS_JOLT) +#elif !(MDBX_MMAP_NEEDS_JOLT == 0 || MDBX_MMAP_NEEDS_JOLT == 1) +#error MDBX_MMAP_NEEDS_JOLT must be defined as 0 or 1 +#endif /* MDBX_MMAP_NEEDS_JOLT */ #ifndef MDBX_64BIT_ATOMIC #if MDBX_WORDBITS >= 64 || defined(DOXYGEN) @@ -2429,8 +2040,7 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #endif /* MDBX_64BIT_CAS */ #ifndef MDBX_UNALIGNED_OK -#if defined(__ALIGNED__) || defined(__SANITIZE_UNDEFINED__) || \ - defined(ENABLE_UBSAN) +#if defined(__ALIGNED__) || defined(__SANITIZE_UNDEFINED__) || defined(ENABLE_UBSAN) #define MDBX_UNALIGNED_OK 0 /* no unaligned access allowed */ #elif defined(__ARM_FEATURE_UNALIGNED) #define MDBX_UNALIGNED_OK 4 /* ok unaligned for 32-bit words */ @@ -2464,6 +2074,19 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #endif #endif /* MDBX_CACHELINE_SIZE */ +/* Max length of iov-vector passed to writev() call, used for auxilary writes */ +#ifndef MDBX_AUXILARY_IOV_MAX +#define MDBX_AUXILARY_IOV_MAX 64 +#endif +#if defined(IOV_MAX) && IOV_MAX < MDBX_AUXILARY_IOV_MAX +#undef MDBX_AUXILARY_IOV_MAX +#define MDBX_AUXILARY_IOV_MAX IOV_MAX +#endif /* MDBX_AUXILARY_IOV_MAX */ + +/* An extra/custom information provided during library build */ +#ifndef MDBX_BUILD_METADATA +#define MDBX_BUILD_METADATA "" +#endif /* MDBX_BUILD_METADATA */ /** @} end of build options */ /******************************************************************************* ******************************************************************************* @@ -2478,6 +2101,9 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #else #define MDBX_DEBUG 1 #endif +#endif +#if MDBX_DEBUG < 0 || MDBX_DEBUG > 2 +#error "The MDBX_DEBUG must be defined to 0, 1 or 2" #endif /* MDBX_DEBUG */ #else @@ -2497,169 +2123,58 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; * Also enables \ref MDBX_DBG_AUDIT if `MDBX_DEBUG >= 2`. * * \ingroup build_option */ -#define MDBX_DEBUG 0...7 +#define MDBX_DEBUG 0...2 /** Disables using of GNU libc extensions. */ #define MDBX_DISABLE_GNU_SOURCE 0 or 1 #endif /* DOXYGEN */ -/* Undefine the NDEBUG if debugging is enforced by MDBX_DEBUG */ -#if MDBX_DEBUG -#undef NDEBUG -#endif - -#ifndef __cplusplus -/*----------------------------------------------------------------------------*/ -/* Debug and Logging stuff */ - -#define MDBX_RUNTIME_FLAGS_INIT \ - ((MDBX_DEBUG) > 0) * MDBX_DBG_ASSERT + ((MDBX_DEBUG) > 1) * MDBX_DBG_AUDIT - -extern uint8_t runtime_flags; -extern uint8_t loglevel; -extern MDBX_debug_func *debug_logger; - -MDBX_MAYBE_UNUSED static __inline void jitter4testing(bool tiny) { -#if MDBX_DEBUG - if (MDBX_DBG_JITTER & runtime_flags) - osal_jitter(tiny); -#else - (void)tiny; -#endif -} - -MDBX_INTERNAL_FUNC void MDBX_PRINTF_ARGS(4, 5) - debug_log(int level, const char *function, int line, const char *fmt, ...) - MDBX_PRINTF_ARGS(4, 5); -MDBX_INTERNAL_FUNC void debug_log_va(int level, const char *function, int line, - const char *fmt, va_list args); +#ifndef MDBX_64BIT_ATOMIC +#error "The MDBX_64BIT_ATOMIC must be defined before" +#endif /* MDBX_64BIT_ATOMIC */ -#if MDBX_DEBUG -#define LOG_ENABLED(msg) unlikely(msg <= loglevel) -#define AUDIT_ENABLED() unlikely((runtime_flags & MDBX_DBG_AUDIT)) -#else /* MDBX_DEBUG */ -#define LOG_ENABLED(msg) (msg < MDBX_LOG_VERBOSE && msg <= loglevel) -#define AUDIT_ENABLED() (0) -#endif /* MDBX_DEBUG */ +#ifndef MDBX_64BIT_CAS +#error "The MDBX_64BIT_CAS must be defined before" +#endif /* MDBX_64BIT_CAS */ -#if MDBX_FORCE_ASSERTIONS -#define ASSERT_ENABLED() (1) -#elif MDBX_DEBUG -#define ASSERT_ENABLED() likely((runtime_flags & MDBX_DBG_ASSERT)) +#if defined(__cplusplus) && !defined(__STDC_NO_ATOMICS__) && __has_include() +#include +#define MDBX_HAVE_C11ATOMICS +#elif !defined(__cplusplus) && (__STDC_VERSION__ >= 201112L || __has_extension(c_atomic)) && \ + !defined(__STDC_NO_ATOMICS__) && \ + (__GNUC_PREREQ(4, 9) || __CLANG_PREREQ(3, 8) || !(defined(__GNUC__) || defined(__clang__))) +#include +#define MDBX_HAVE_C11ATOMICS +#elif defined(__GNUC__) || defined(__clang__) +#elif defined(_MSC_VER) +#pragma warning(disable : 4163) /* 'xyz': not available as an intrinsic */ +#pragma warning(disable : 4133) /* 'function': incompatible types - from \ + 'size_t' to 'LONGLONG' */ +#pragma warning(disable : 4244) /* 'return': conversion from 'LONGLONG' to \ + 'std::size_t', possible loss of data */ +#pragma warning(disable : 4267) /* 'function': conversion from 'size_t' to \ + 'long', possible loss of data */ +#pragma intrinsic(_InterlockedExchangeAdd, _InterlockedCompareExchange) +#pragma intrinsic(_InterlockedExchangeAdd64, _InterlockedCompareExchange64) +#elif defined(__APPLE__) +#include #else -#define ASSERT_ENABLED() (0) -#endif /* assertions */ - -#define DEBUG_EXTRA(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_EXTRA)) \ - debug_log(MDBX_LOG_EXTRA, __func__, __LINE__, fmt, __VA_ARGS__); \ - } while (0) - -#define DEBUG_EXTRA_PRINT(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_EXTRA)) \ - debug_log(MDBX_LOG_EXTRA, NULL, 0, fmt, __VA_ARGS__); \ - } while (0) - -#define TRACE(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_TRACE)) \ - debug_log(MDBX_LOG_TRACE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ - } while (0) - -#define DEBUG(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_DEBUG)) \ - debug_log(MDBX_LOG_DEBUG, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ - } while (0) - -#define VERBOSE(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_VERBOSE)) \ - debug_log(MDBX_LOG_VERBOSE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ - } while (0) - -#define NOTICE(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_NOTICE)) \ - debug_log(MDBX_LOG_NOTICE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ - } while (0) - -#define WARNING(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_WARN)) \ - debug_log(MDBX_LOG_WARN, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ - } while (0) - -#undef ERROR /* wingdi.h \ - Yeah, morons from M$ put such definition to the public header. */ - -#define ERROR(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_ERROR)) \ - debug_log(MDBX_LOG_ERROR, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ - } while (0) - -#define FATAL(fmt, ...) \ - debug_log(MDBX_LOG_FATAL, __func__, __LINE__, fmt "\n", __VA_ARGS__); - -#if MDBX_DEBUG -#define ASSERT_FAIL(env, msg, func, line) mdbx_assert_fail(env, msg, func, line) -#else /* MDBX_DEBUG */ -MDBX_NORETURN __cold void assert_fail(const char *msg, const char *func, - unsigned line); -#define ASSERT_FAIL(env, msg, func, line) \ - do { \ - (void)(env); \ - assert_fail(msg, func, line); \ - } while (0) -#endif /* MDBX_DEBUG */ - -#define ENSURE_MSG(env, expr, msg) \ - do { \ - if (unlikely(!(expr))) \ - ASSERT_FAIL(env, msg, __func__, __LINE__); \ - } while (0) - -#define ENSURE(env, expr) ENSURE_MSG(env, expr, #expr) - -/* assert(3) variant in environment context */ -#define eASSERT(env, expr) \ - do { \ - if (ASSERT_ENABLED()) \ - ENSURE(env, expr); \ - } while (0) - -/* assert(3) variant in cursor context */ -#define cASSERT(mc, expr) eASSERT((mc)->mc_txn->mt_env, expr) - -/* assert(3) variant in transaction context */ -#define tASSERT(txn, expr) eASSERT((txn)->mt_env, expr) - -#ifndef xMDBX_TOOLS /* Avoid using internal eASSERT() */ -#undef assert -#define assert(expr) eASSERT(NULL, expr) +#error FIXME atomic-ops #endif -#endif /* __cplusplus */ - -/*----------------------------------------------------------------------------*/ -/* Atomics */ - -enum MDBX_memory_order { +typedef enum mdbx_memory_order { mo_Relaxed, mo_AcquireRelease /* , mo_SequentialConsistency */ -}; +} mdbx_memory_order_t; typedef union { volatile uint32_t weak; #ifdef MDBX_HAVE_C11ATOMICS volatile _Atomic uint32_t c11a; #endif /* MDBX_HAVE_C11ATOMICS */ -} MDBX_atomic_uint32_t; +} mdbx_atomic_uint32_t; typedef union { volatile uint64_t weak; @@ -2669,15 +2184,15 @@ typedef union { #if !defined(MDBX_HAVE_C11ATOMICS) || !MDBX_64BIT_CAS || !MDBX_64BIT_ATOMIC __anonymous_struct_extension__ struct { #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ - MDBX_atomic_uint32_t low, high; + mdbx_atomic_uint32_t low, high; #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - MDBX_atomic_uint32_t high, low; + mdbx_atomic_uint32_t high, low; #else #error "FIXME: Unsupported byte order" #endif /* __BYTE_ORDER__ */ }; #endif -} MDBX_atomic_uint64_t; +} mdbx_atomic_uint64_t; #ifdef MDBX_HAVE_C11ATOMICS @@ -2693,92 +2208,20 @@ typedef union { #define MDBX_c11a_rw(type, ptr) (&(ptr)->c11a) #endif /* Crutches for C11 atomic compiler's bugs */ -#define mo_c11_store(fence) \ - (((fence) == mo_Relaxed) ? memory_order_relaxed \ - : ((fence) == mo_AcquireRelease) ? memory_order_release \ +#define mo_c11_store(fence) \ + (((fence) == mo_Relaxed) ? memory_order_relaxed \ + : ((fence) == mo_AcquireRelease) ? memory_order_release \ : memory_order_seq_cst) -#define mo_c11_load(fence) \ - (((fence) == mo_Relaxed) ? memory_order_relaxed \ - : ((fence) == mo_AcquireRelease) ? memory_order_acquire \ +#define mo_c11_load(fence) \ + (((fence) == mo_Relaxed) ? memory_order_relaxed \ + : ((fence) == mo_AcquireRelease) ? memory_order_acquire \ : memory_order_seq_cst) #endif /* MDBX_HAVE_C11ATOMICS */ -#ifndef __cplusplus - -#ifdef MDBX_HAVE_C11ATOMICS -#define osal_memory_fence(order, write) \ - atomic_thread_fence((write) ? mo_c11_store(order) : mo_c11_load(order)) -#else /* MDBX_HAVE_C11ATOMICS */ -#define osal_memory_fence(order, write) \ - do { \ - osal_compiler_barrier(); \ - if (write && order > (MDBX_CPU_WRITEBACK_INCOHERENT ? mo_Relaxed \ - : mo_AcquireRelease)) \ - osal_memory_barrier(); \ - } while (0) -#endif /* MDBX_HAVE_C11ATOMICS */ - -#if defined(MDBX_HAVE_C11ATOMICS) && defined(__LCC__) -#define atomic_store32(p, value, order) \ - ({ \ - const uint32_t value_to_store = (value); \ - atomic_store_explicit(MDBX_c11a_rw(uint32_t, p), value_to_store, \ - mo_c11_store(order)); \ - value_to_store; \ - }) -#define atomic_load32(p, order) \ - atomic_load_explicit(MDBX_c11a_ro(uint32_t, p), mo_c11_load(order)) -#define atomic_store64(p, value, order) \ - ({ \ - const uint64_t value_to_store = (value); \ - atomic_store_explicit(MDBX_c11a_rw(uint64_t, p), value_to_store, \ - mo_c11_store(order)); \ - value_to_store; \ - }) -#define atomic_load64(p, order) \ - atomic_load_explicit(MDBX_c11a_ro(uint64_t, p), mo_c11_load(order)) -#endif /* LCC && MDBX_HAVE_C11ATOMICS */ - -#ifndef atomic_store32 -MDBX_MAYBE_UNUSED static __always_inline uint32_t -atomic_store32(MDBX_atomic_uint32_t *p, const uint32_t value, - enum MDBX_memory_order order) { - STATIC_ASSERT(sizeof(MDBX_atomic_uint32_t) == 4); -#ifdef MDBX_HAVE_C11ATOMICS - assert(atomic_is_lock_free(MDBX_c11a_rw(uint32_t, p))); - atomic_store_explicit(MDBX_c11a_rw(uint32_t, p), value, mo_c11_store(order)); -#else /* MDBX_HAVE_C11ATOMICS */ - if (order != mo_Relaxed) - osal_compiler_barrier(); - p->weak = value; - osal_memory_fence(order, true); -#endif /* MDBX_HAVE_C11ATOMICS */ - return value; -} -#endif /* atomic_store32 */ - -#ifndef atomic_load32 -MDBX_MAYBE_UNUSED static __always_inline uint32_t atomic_load32( - const volatile MDBX_atomic_uint32_t *p, enum MDBX_memory_order order) { - STATIC_ASSERT(sizeof(MDBX_atomic_uint32_t) == 4); -#ifdef MDBX_HAVE_C11ATOMICS - assert(atomic_is_lock_free(MDBX_c11a_ro(uint32_t, p))); - return atomic_load_explicit(MDBX_c11a_ro(uint32_t, p), mo_c11_load(order)); -#else /* MDBX_HAVE_C11ATOMICS */ - osal_memory_fence(order, false); - const uint32_t value = p->weak; - if (order != mo_Relaxed) - osal_compiler_barrier(); - return value; -#endif /* MDBX_HAVE_C11ATOMICS */ -} -#endif /* atomic_load32 */ - -#endif /* !__cplusplus */ +#define SAFE64_INVALID_THRESHOLD UINT64_C(0xffffFFFF00000000) -/*----------------------------------------------------------------------------*/ -/* Basic constants and types */ +#pragma pack(push, 4) /* A stamp that identifies a file as an MDBX file. * There's nothing special about this value other than that it is easily @@ -2787,8 +2230,10 @@ MDBX_MAYBE_UNUSED static __always_inline uint32_t atomic_load32( /* FROZEN: The version number for a database's datafile format. */ #define MDBX_DATA_VERSION 3 -/* The version number for a database's lockfile format. */ -#define MDBX_LOCK_VERSION 5 + +#define MDBX_DATA_MAGIC ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + MDBX_DATA_VERSION) +#define MDBX_DATA_MAGIC_LEGACY_COMPAT ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + 2) +#define MDBX_DATA_MAGIC_LEGACY_DEVEL ((MDBX_MAGIC << 8) + 255) /* handle for the DB used to track free pages. */ #define FREE_DBI 0 @@ -2805,203 +2250,285 @@ MDBX_MAYBE_UNUSED static __always_inline uint32_t atomic_load32( * MDBX uses 32 bit for page numbers. This limits database * size up to 2^44 bytes, in case of 4K pages. */ typedef uint32_t pgno_t; -typedef MDBX_atomic_uint32_t atomic_pgno_t; +typedef mdbx_atomic_uint32_t atomic_pgno_t; #define PRIaPGNO PRIu32 #define MAX_PAGENO UINT32_C(0x7FFFffff) #define MIN_PAGENO NUM_METAS -#define SAFE64_INVALID_THRESHOLD UINT64_C(0xffffFFFF00000000) +/* An invalid page number. + * Mainly used to denote an empty tree. */ +#define P_INVALID (~(pgno_t)0) /* A transaction ID. */ typedef uint64_t txnid_t; -typedef MDBX_atomic_uint64_t atomic_txnid_t; +typedef mdbx_atomic_uint64_t atomic_txnid_t; #define PRIaTXN PRIi64 #define MIN_TXNID UINT64_C(1) #define MAX_TXNID (SAFE64_INVALID_THRESHOLD - 1) #define INITIAL_TXNID (MIN_TXNID + NUM_METAS - 1) #define INVALID_TXNID UINT64_MAX -/* LY: for testing non-atomic 64-bit txnid on 32-bit arches. - * #define xMDBX_TXNID_STEP (UINT32_MAX / 3) */ -#ifndef xMDBX_TXNID_STEP -#if MDBX_64BIT_CAS -#define xMDBX_TXNID_STEP 1u -#else -#define xMDBX_TXNID_STEP 2u -#endif -#endif /* xMDBX_TXNID_STEP */ -/* Used for offsets within a single page. - * Since memory pages are typically 4 or 8KB in size, 12-13 bits, - * this is plenty. */ +/* Used for offsets within a single page. */ typedef uint16_t indx_t; -#define MEGABYTE ((size_t)1 << 20) - -/*----------------------------------------------------------------------------*/ -/* Core structures for database and shared memory (i.e. format definition) */ -#pragma pack(push, 4) - -/* Information about a single database in the environment. */ -typedef struct MDBX_db { - uint16_t md_flags; /* see mdbx_dbi_open */ - uint16_t md_depth; /* depth of this tree */ - uint32_t md_xsize; /* key-size for MDBX_DUPFIXED (LEAF2 pages) */ - pgno_t md_root; /* the root page of this tree */ - pgno_t md_branch_pages; /* number of internal pages */ - pgno_t md_leaf_pages; /* number of leaf pages */ - pgno_t md_overflow_pages; /* number of overflow pages */ - uint64_t md_seq; /* table sequence counter */ - uint64_t md_entries; /* number of data items */ - uint64_t md_mod_txnid; /* txnid of last committed modification */ -} MDBX_db; +typedef struct tree { + uint16_t flags; /* see mdbx_dbi_open */ + uint16_t height; /* height of this tree */ + uint32_t dupfix_size; /* key-size for MDBX_DUPFIXED (DUPFIX pages) */ + pgno_t root; /* the root page of this tree */ + pgno_t branch_pages; /* number of branch pages */ + pgno_t leaf_pages; /* number of leaf pages */ + pgno_t large_pages; /* number of large pages */ + uint64_t sequence; /* table sequence counter */ + uint64_t items; /* number of data items */ + uint64_t mod_txnid; /* txnid of last committed modification */ +} tree_t; /* database size-related parameters */ -typedef struct MDBX_geo { +typedef struct geo { uint16_t grow_pv; /* datafile growth step as a 16-bit packed (exponential quantized) value */ uint16_t shrink_pv; /* datafile shrink threshold as a 16-bit packed (exponential quantized) value */ pgno_t lower; /* minimal size of datafile in pages */ pgno_t upper; /* maximal size of datafile in pages */ - pgno_t now; /* current size of datafile in pages */ - pgno_t next; /* first unused page in the datafile, + union { + pgno_t now; /* current size of datafile in pages */ + pgno_t end_pgno; + }; + union { + pgno_t first_unallocated; /* first unused page in the datafile, but actually the file may be shorter. */ -} MDBX_geo; + pgno_t next_pgno; + }; +} geo_t; /* Meta page content. * A meta page is the start point for accessing a database snapshot. - * Pages 0-1 are meta pages. Transaction N writes meta page (N % 2). */ -typedef struct MDBX_meta { + * Pages 0-2 are meta pages. */ +typedef struct meta { /* Stamp identifying this as an MDBX file. * It must be set to MDBX_MAGIC with MDBX_DATA_VERSION. */ - uint32_t mm_magic_and_version[2]; + uint32_t magic_and_version[2]; - /* txnid that committed this page, the first of a two-phase-update pair */ + /* txnid that committed this meta, the first of a two-phase-update pair */ union { - MDBX_atomic_uint32_t mm_txnid_a[2]; + mdbx_atomic_uint32_t txnid_a[2]; uint64_t unsafe_txnid; }; - uint16_t mm_extra_flags; /* extra DB flags, zero (nothing) for now */ - uint8_t mm_validator_id; /* ID of checksum and page validation method, - * zero (nothing) for now */ - uint8_t mm_extra_pagehdr; /* extra bytes in the page header, - * zero (nothing) for now */ + uint16_t reserve16; /* extra flags, zero (nothing) for now */ + uint8_t validator_id; /* ID of checksum and page validation method, + * zero (nothing) for now */ + int8_t extra_pagehdr; /* extra bytes in the page header, + * zero (nothing) for now */ - MDBX_geo mm_geo; /* database size-related parameters */ + geo_t geometry; /* database size-related parameters */ - MDBX_db mm_dbs[CORE_DBS]; /* first is free space, 2nd is main db */ - /* The size of pages used in this DB */ -#define mm_psize mm_dbs[FREE_DBI].md_xsize - MDBX_canary mm_canary; + union { + struct { + tree_t gc, main; + } trees; + __anonymous_struct_extension__ struct { + uint16_t gc_flags; + uint16_t gc_height; + uint32_t pagesize; + }; + }; + + MDBX_canary canary; -#define MDBX_DATASIGN_NONE 0u -#define MDBX_DATASIGN_WEAK 1u -#define SIGN_IS_STEADY(sign) ((sign) > MDBX_DATASIGN_WEAK) -#define META_IS_STEADY(meta) \ - SIGN_IS_STEADY(unaligned_peek_u64_volatile(4, (meta)->mm_sign)) +#define DATASIGN_NONE 0u +#define DATASIGN_WEAK 1u +#define SIGN_IS_STEADY(sign) ((sign) > DATASIGN_WEAK) union { - uint32_t mm_sign[2]; + uint32_t sign[2]; uint64_t unsafe_sign; }; - /* txnid that committed this page, the second of a two-phase-update pair */ - MDBX_atomic_uint32_t mm_txnid_b[2]; + /* txnid that committed this meta, the second of a two-phase-update pair */ + mdbx_atomic_uint32_t txnid_b[2]; /* Number of non-meta pages which were put in GC after COW. May be 0 in case * DB was previously handled by libmdbx without corresponding feature. - * This value in couple with mr_snapshot_pages_retired allows fast estimation - * of "how much reader is restraining GC recycling". */ - uint32_t mm_pages_retired[2]; + * This value in couple with reader.snapshot_pages_retired allows fast + * estimation of "how much reader is restraining GC recycling". */ + uint32_t pages_retired[2]; /* The analogue /proc/sys/kernel/random/boot_id or similar to determine * whether the system was rebooted after the last use of the database files. * If there was no reboot, but there is no need to rollback to the last * steady sync point. Zeros mean that no relevant information is available * from the system. */ - bin128_t mm_bootid; + bin128_t bootid; -} MDBX_meta; + /* GUID базы данных, начиная с v0.13.1 */ + bin128_t dxbid; +} meta_t; #pragma pack(1) -/* Common header for all page types. The page type depends on mp_flags. +typedef enum page_type { + P_BRANCH = 0x01u /* branch page */, + P_LEAF = 0x02u /* leaf page */, + P_LARGE = 0x04u /* large/overflow page */, + P_META = 0x08u /* meta page */, + P_LEGACY_DIRTY = 0x10u /* legacy P_DIRTY flag prior to v0.10 958fd5b9 */, + P_BAD = P_LEGACY_DIRTY /* explicit flag for invalid/bad page */, + P_DUPFIX = 0x20u /* for MDBX_DUPFIXED records */, + P_SUBP = 0x40u /* for MDBX_DUPSORT sub-pages */, + P_SPILLED = 0x2000u /* spilled in parent txn */, + P_LOOSE = 0x4000u /* page was dirtied then freed, can be reused */, + P_FROZEN = 0x8000u /* used for retire page with known status */, + P_ILL_BITS = (uint16_t)~(P_BRANCH | P_LEAF | P_DUPFIX | P_LARGE | P_SPILLED), + + page_broken = 0, + page_large = P_LARGE, + page_branch = P_BRANCH, + page_leaf = P_LEAF, + page_dupfix_leaf = P_DUPFIX, + page_sub_leaf = P_SUBP | P_LEAF, + page_sub_dupfix_leaf = P_SUBP | P_DUPFIX, + page_sub_broken = P_SUBP, +} page_type_t; + +/* Common header for all page types. The page type depends on flags. * - * P_BRANCH and P_LEAF pages have unsorted 'MDBX_node's at the end, with - * sorted mp_ptrs[] entries referring to them. Exception: P_LEAF2 pages - * omit mp_ptrs and pack sorted MDBX_DUPFIXED values after the page header. + * P_BRANCH and P_LEAF pages have unsorted 'node_t's at the end, with + * sorted entries[] entries referring to them. Exception: P_DUPFIX pages + * omit entries and pack sorted MDBX_DUPFIXED values after the page header. * - * P_OVERFLOW records occupy one or more contiguous pages where only the - * first has a page header. They hold the real data of F_BIGDATA nodes. + * P_LARGE records occupy one or more contiguous pages where only the + * first has a page header. They hold the real data of N_BIG nodes. * * P_SUBP sub-pages are small leaf "pages" with duplicate data. - * A node with flag F_DUPDATA but not F_SUBDATA contains a sub-page. - * (Duplicate data can also go in sub-databases, which use normal pages.) + * A node with flag N_DUP but not N_TREE contains a sub-page. + * (Duplicate data can also go in tables, which use normal pages.) * - * P_META pages contain MDBX_meta, the start point of an MDBX snapshot. + * P_META pages contain meta_t, the start point of an MDBX snapshot. * - * Each non-metapage up to MDBX_meta.mm_last_pg is reachable exactly once + * Each non-metapage up to meta_t.mm_last_pg is reachable exactly once * in the snapshot: Either used by a database or listed in a GC record. */ -typedef struct MDBX_page { -#define IS_FROZEN(txn, p) ((p)->mp_txnid < (txn)->mt_txnid) -#define IS_SPILLED(txn, p) ((p)->mp_txnid == (txn)->mt_txnid) -#define IS_SHADOWED(txn, p) ((p)->mp_txnid > (txn)->mt_txnid) -#define IS_VALID(txn, p) ((p)->mp_txnid <= (txn)->mt_front) -#define IS_MODIFIABLE(txn, p) ((p)->mp_txnid == (txn)->mt_front) - uint64_t mp_txnid; /* txnid which created page, maybe zero in legacy DB */ - uint16_t mp_leaf2_ksize; /* key size if this is a LEAF2 page */ -#define P_BRANCH 0x01u /* branch page */ -#define P_LEAF 0x02u /* leaf page */ -#define P_OVERFLOW 0x04u /* overflow page */ -#define P_META 0x08u /* meta page */ -#define P_LEGACY_DIRTY 0x10u /* legacy P_DIRTY flag prior to v0.10 958fd5b9 */ -#define P_BAD P_LEGACY_DIRTY /* explicit flag for invalid/bad page */ -#define P_LEAF2 0x20u /* for MDBX_DUPFIXED records */ -#define P_SUBP 0x40u /* for MDBX_DUPSORT sub-pages */ -#define P_SPILLED 0x2000u /* spilled in parent txn */ -#define P_LOOSE 0x4000u /* page was dirtied then freed, can be reused */ -#define P_FROZEN 0x8000u /* used for retire page with known status */ -#define P_ILL_BITS \ - ((uint16_t)~(P_BRANCH | P_LEAF | P_LEAF2 | P_OVERFLOW | P_SPILLED)) - uint16_t mp_flags; +typedef struct page { + uint64_t txnid; /* txnid which created page, maybe zero in legacy DB */ + uint16_t dupfix_ksize; /* key size if this is a DUPFIX page */ + uint16_t flags; union { - uint32_t mp_pages; /* number of overflow pages */ + uint32_t pages; /* number of overflow pages */ __anonymous_struct_extension__ struct { - indx_t mp_lower; /* lower bound of free space */ - indx_t mp_upper; /* upper bound of free space */ + indx_t lower; /* lower bound of free space */ + indx_t upper; /* upper bound of free space */ }; }; - pgno_t mp_pgno; /* page number */ - -#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ - (!defined(__cplusplus) && defined(_MSC_VER)) - indx_t mp_ptrs[] /* dynamic size */; -#endif /* C99 */ -} MDBX_page; + pgno_t pgno; /* page number */ -#define PAGETYPE_WHOLE(p) ((uint8_t)(p)->mp_flags) - -/* Drop legacy P_DIRTY flag for sub-pages for compatilibity */ -#define PAGETYPE_COMPAT(p) \ - (unlikely(PAGETYPE_WHOLE(p) & P_SUBP) \ - ? PAGETYPE_WHOLE(p) & ~(P_SUBP | P_LEGACY_DIRTY) \ - : PAGETYPE_WHOLE(p)) +#if FLEXIBLE_ARRAY_MEMBERS + indx_t entries[] /* dynamic size */; +#endif /* FLEXIBLE_ARRAY_MEMBERS */ +} page_t; /* Size of the page header, excluding dynamic data at the end */ -#define PAGEHDRSZ offsetof(MDBX_page, mp_ptrs) +#define PAGEHDRSZ 20u -/* Pointer displacement without casting to char* to avoid pointer-aliasing */ -#define ptr_disp(ptr, disp) ((void *)(((intptr_t)(ptr)) + ((intptr_t)(disp)))) +/* Header for a single key/data pair within a page. + * Used in pages of type P_BRANCH and P_LEAF without P_DUPFIX. + * We guarantee 2-byte alignment for 'node_t's. + * + * Leaf node flags describe node contents. N_BIG says the node's + * data part is the page number of an overflow page with actual data. + * N_DUP and N_TREE can be combined giving duplicate data in + * a sub-page/table, and named databases (just N_TREE). */ +typedef struct node { +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + union { + uint32_t dsize; + uint32_t child_pgno; + }; + uint8_t flags; /* see node_flags */ + uint8_t extra; + uint16_t ksize; /* key size */ +#else + uint16_t ksize; /* key size */ + uint8_t extra; + uint8_t flags; /* see node_flags */ + union { + uint32_t child_pgno; + uint32_t dsize; + }; +#endif /* __BYTE_ORDER__ */ -/* Pointer distance as signed number of bytes */ -#define ptr_dist(more, less) (((intptr_t)(more)) - ((intptr_t)(less))) +#if FLEXIBLE_ARRAY_MEMBERS + uint8_t payload[] /* key and data are appended here */; +#endif /* FLEXIBLE_ARRAY_MEMBERS */ +} node_t; + +/* Size of the node header, excluding dynamic data at the end */ +#define NODESIZE 8u -#define mp_next(mp) \ - (*(MDBX_page **)ptr_disp((mp)->mp_ptrs, sizeof(void *) - sizeof(uint32_t))) +typedef enum node_flags { + N_BIG = 0x01 /* data put on large page */, + N_TREE = 0x02 /* data is a b-tree */, + N_DUP = 0x04 /* data has duplicates */ +} node_flags_t; #pragma pack(pop) -typedef struct profgc_stat { +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint8_t page_type(const page_t *mp) { return mp->flags; } + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint8_t page_type_compat(const page_t *mp) { + /* Drop legacy P_DIRTY flag for sub-pages for compatilibity, + * for assertions only. */ + return unlikely(mp->flags & P_SUBP) ? mp->flags & ~(P_SUBP | P_LEGACY_DIRTY) : mp->flags; +} + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_leaf(const page_t *mp) { + return (mp->flags & P_LEAF) != 0; +} + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_dupfix_leaf(const page_t *mp) { + return (mp->flags & P_DUPFIX) != 0; +} + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_branch(const page_t *mp) { + return (mp->flags & P_BRANCH) != 0; +} + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_largepage(const page_t *mp) { + return (mp->flags & P_LARGE) != 0; +} + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_subpage(const page_t *mp) { + return (mp->flags & P_SUBP) != 0; +} + +/* The version number for a database's lockfile format. */ +#define MDBX_LOCK_VERSION 6 + +#if MDBX_LOCKING == MDBX_LOCKING_WIN32FILES + +#define MDBX_LCK_SIGN UINT32_C(0xF10C) +typedef void osal_ipclock_t; +#elif MDBX_LOCKING == MDBX_LOCKING_SYSV + +#define MDBX_LCK_SIGN UINT32_C(0xF18D) +typedef mdbx_pid_t osal_ipclock_t; + +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || MDBX_LOCKING == MDBX_LOCKING_POSIX2008 + +#define MDBX_LCK_SIGN UINT32_C(0x8017) +typedef pthread_mutex_t osal_ipclock_t; + +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988 + +#define MDBX_LCK_SIGN UINT32_C(0xFC29) +typedef sem_t osal_ipclock_t; + +#else +#error "FIXME" +#endif /* MDBX_LOCKING */ + +/* Статистика профилирования работы GC */ +typedef struct gc_prof_stat { /* Монотонное время по "настенным часам" * затраченное на чтение и поиск внутри GC */ uint64_t rtime_monotonic; @@ -3017,42 +2544,44 @@ typedef struct profgc_stat { uint32_t spe_counter; /* page faults (hard page faults) */ uint32_t majflt; -} profgc_stat_t; - -/* Statistics of page operations overall of all (running, completed and aborted) - * transactions */ -typedef struct pgop_stat { - MDBX_atomic_uint64_t newly; /* Quantity of a new pages added */ - MDBX_atomic_uint64_t cow; /* Quantity of pages copied for update */ - MDBX_atomic_uint64_t clone; /* Quantity of parent's dirty pages clones + /* Для разборок с pnl_merge() */ + struct { + uint64_t time; + uint64_t volume; + uint32_t calls; + } pnl_merge; +} gc_prof_stat_t; + +/* Statistics of pages operations for all transactions, + * including incomplete and aborted. */ +typedef struct pgops { + mdbx_atomic_uint64_t newly; /* Quantity of a new pages added */ + mdbx_atomic_uint64_t cow; /* Quantity of pages copied for update */ + mdbx_atomic_uint64_t clone; /* Quantity of parent's dirty pages clones for nested transactions */ - MDBX_atomic_uint64_t split; /* Page splits */ - MDBX_atomic_uint64_t merge; /* Page merges */ - MDBX_atomic_uint64_t spill; /* Quantity of spilled dirty pages */ - MDBX_atomic_uint64_t unspill; /* Quantity of unspilled/reloaded pages */ - MDBX_atomic_uint64_t - wops; /* Number of explicit write operations (not a pages) to a disk */ - MDBX_atomic_uint64_t - msync; /* Number of explicit msync/flush-to-disk operations */ - MDBX_atomic_uint64_t - fsync; /* Number of explicit fsync/flush-to-disk operations */ - - MDBX_atomic_uint64_t prefault; /* Number of prefault write operations */ - MDBX_atomic_uint64_t mincore; /* Number of mincore() calls */ - - MDBX_atomic_uint32_t - incoherence; /* number of https://libmdbx.dqdkfa.ru/dead-github/issues/269 - caught */ - MDBX_atomic_uint32_t reserved; + mdbx_atomic_uint64_t split; /* Page splits */ + mdbx_atomic_uint64_t merge; /* Page merges */ + mdbx_atomic_uint64_t spill; /* Quantity of spilled dirty pages */ + mdbx_atomic_uint64_t unspill; /* Quantity of unspilled/reloaded pages */ + mdbx_atomic_uint64_t wops; /* Number of explicit write operations (not a pages) to a disk */ + mdbx_atomic_uint64_t msync; /* Number of explicit msync/flush-to-disk operations */ + mdbx_atomic_uint64_t fsync; /* Number of explicit fsync/flush-to-disk operations */ + + mdbx_atomic_uint64_t prefault; /* Number of prefault write operations */ + mdbx_atomic_uint64_t mincore; /* Number of mincore() calls */ + + mdbx_atomic_uint32_t incoherence; /* number of https://libmdbx.dqdkfa.ru/dead-github/issues/269 + caught */ + mdbx_atomic_uint32_t reserved; /* Статистика для профилирования GC. - * Логически эти данные может быть стоит вынести в другую структуру, + * Логически эти данные, возможно, стоит вынести в другую структуру, * но разница будет сугубо косметическая. */ struct { /* Затраты на поддержку данных пользователя */ - profgc_stat_t work; + gc_prof_stat_t work; /* Затраты на поддержку и обновления самой GC */ - profgc_stat_t self; + gc_prof_stat_t self; /* Итераций обновления GC, * больше 1 если были повторы/перезапуски */ uint32_t wloops; @@ -3067,33 +2596,6 @@ typedef struct pgop_stat { } gc_prof; } pgop_stat_t; -#if MDBX_LOCKING == MDBX_LOCKING_WIN32FILES -#define MDBX_CLOCK_SIGN UINT32_C(0xF10C) -typedef void osal_ipclock_t; -#elif MDBX_LOCKING == MDBX_LOCKING_SYSV - -#define MDBX_CLOCK_SIGN UINT32_C(0xF18D) -typedef mdbx_pid_t osal_ipclock_t; -#ifndef EOWNERDEAD -#define EOWNERDEAD MDBX_RESULT_TRUE -#endif - -#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ - MDBX_LOCKING == MDBX_LOCKING_POSIX2008 -#define MDBX_CLOCK_SIGN UINT32_C(0x8017) -typedef pthread_mutex_t osal_ipclock_t; -#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988 -#define MDBX_CLOCK_SIGN UINT32_C(0xFC29) -typedef sem_t osal_ipclock_t; -#else -#error "FIXME" -#endif /* MDBX_LOCKING */ - -#if MDBX_LOCKING > MDBX_LOCKING_SYSV && !defined(__cplusplus) -MDBX_INTERNAL_FUNC int osal_ipclock_stub(osal_ipclock_t *ipc); -MDBX_INTERNAL_FUNC int osal_ipclock_destroy(osal_ipclock_t *ipc); -#endif /* MDBX_LOCKING */ - /* Reader Lock Table * * Readers don't acquire any locks for their data access. Instead, they @@ -3103,8 +2605,9 @@ MDBX_INTERNAL_FUNC int osal_ipclock_destroy(osal_ipclock_t *ipc); * read transactions started by the same thread need no further locking to * proceed. * - * If MDBX_NOTLS is set, the slot address is not saved in thread-specific data. - * No reader table is used if the database is on a read-only filesystem. + * If MDBX_NOSTICKYTHREADS is set, the slot address is not saved in + * thread-specific data. No reader table is used if the database is on a + * read-only filesystem. * * Since the database uses multi-version concurrency control, readers don't * actually need any locking. This table is used to keep track of which @@ -3133,14 +2636,14 @@ MDBX_INTERNAL_FUNC int osal_ipclock_destroy(osal_ipclock_t *ipc); * many old transactions together. */ /* The actual reader record, with cacheline padding. */ -typedef struct MDBX_reader { - /* Current Transaction ID when this transaction began, or (txnid_t)-1. +typedef struct reader_slot { + /* Current Transaction ID when this transaction began, or INVALID_TXNID. * Multiple readers that start at the same time will probably have the * same ID here. Again, it's not important to exclude them from * anything; all we need to know is which version of the DB they * started from so we can avoid overwriting any data used in that * particular version. */ - MDBX_atomic_uint64_t /* txnid_t */ mr_txnid; + atomic_txnid_t txnid; /* The information we store in a single slot of the reader table. * In addition to a transaction ID, we also record the process and @@ -3151,181 +2654,421 @@ typedef struct MDBX_reader { * We simply re-init the table when we know that we're the only process * opening the lock file. */ + /* Псевдо thread_id для пометки вытесненных читающих транзакций. */ +#define MDBX_TID_TXN_OUSTED (UINT64_MAX - 1) + + /* Псевдо thread_id для пометки припаркованных читающих транзакций. */ +#define MDBX_TID_TXN_PARKED UINT64_MAX + /* The thread ID of the thread owning this txn. */ - MDBX_atomic_uint64_t mr_tid; + mdbx_atomic_uint64_t tid; /* The process ID of the process owning this reader txn. */ - MDBX_atomic_uint32_t mr_pid; + mdbx_atomic_uint32_t pid; /* The number of pages used in the reader's MVCC snapshot, - * i.e. the value of meta->mm_geo.next and txn->mt_next_pgno */ - atomic_pgno_t mr_snapshot_pages_used; + * i.e. the value of meta->geometry.first_unallocated and + * txn->geo.first_unallocated */ + atomic_pgno_t snapshot_pages_used; /* Number of retired pages at the time this reader starts transaction. So, - * at any time the difference mm_pages_retired - mr_snapshot_pages_retired - * will give the number of pages which this reader restraining from reuse. */ - MDBX_atomic_uint64_t mr_snapshot_pages_retired; -} MDBX_reader; + * at any time the difference meta.pages_retired - + * reader.snapshot_pages_retired will give the number of pages which this + * reader restraining from reuse. */ + mdbx_atomic_uint64_t snapshot_pages_retired; +} reader_slot_t; /* The header for the reader table (a memory-mapped lock file). */ -typedef struct MDBX_lockinfo { +typedef struct shared_lck { /* Stamp identifying this as an MDBX file. * It must be set to MDBX_MAGIC with with MDBX_LOCK_VERSION. */ - uint64_t mti_magic_and_version; + uint64_t magic_and_version; /* Format of this lock file. Must be set to MDBX_LOCK_FORMAT. */ - uint32_t mti_os_and_format; + uint32_t os_and_format; /* Flags which environment was opened. */ - MDBX_atomic_uint32_t mti_envmode; + mdbx_atomic_uint32_t envmode; /* Threshold of un-synced-with-disk pages for auto-sync feature, * zero means no-threshold, i.e. auto-sync is disabled. */ - atomic_pgno_t mti_autosync_threshold; + atomic_pgno_t autosync_threshold; /* Low 32-bit of txnid with which meta-pages was synced, * i.e. for sync-polling in the MDBX_NOMETASYNC mode. */ #define MDBX_NOMETASYNC_LAZY_UNK (UINT32_MAX / 3) #define MDBX_NOMETASYNC_LAZY_FD (MDBX_NOMETASYNC_LAZY_UNK + UINT32_MAX / 8) -#define MDBX_NOMETASYNC_LAZY_WRITEMAP \ - (MDBX_NOMETASYNC_LAZY_UNK - UINT32_MAX / 8) - MDBX_atomic_uint32_t mti_meta_sync_txnid; +#define MDBX_NOMETASYNC_LAZY_WRITEMAP (MDBX_NOMETASYNC_LAZY_UNK - UINT32_MAX / 8) + mdbx_atomic_uint32_t meta_sync_txnid; /* Period for timed auto-sync feature, i.e. at the every steady checkpoint - * the mti_unsynced_timeout sets to the current_time + mti_autosync_period. + * the mti_unsynced_timeout sets to the current_time + autosync_period. * The time value is represented in a suitable system-dependent form, for * example clock_gettime(CLOCK_BOOTTIME) or clock_gettime(CLOCK_MONOTONIC). * Zero means timed auto-sync is disabled. */ - MDBX_atomic_uint64_t mti_autosync_period; + mdbx_atomic_uint64_t autosync_period; /* Marker to distinguish uniqueness of DB/CLK. */ - MDBX_atomic_uint64_t mti_bait_uniqueness; + mdbx_atomic_uint64_t bait_uniqueness; /* Paired counter of processes that have mlock()ed part of mmapped DB. - * The (mti_mlcnt[0] - mti_mlcnt[1]) > 0 means at least one process + * The (mlcnt[0] - mlcnt[1]) > 0 means at least one process * lock at least one page, so therefore madvise() could return EINVAL. */ - MDBX_atomic_uint32_t mti_mlcnt[2]; + mdbx_atomic_uint32_t mlcnt[2]; + + MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/ + + /* Statistics of costly ops of all (running, completed and aborted) + * transactions */ + pgop_stat_t pgops; + + MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/ + +#if MDBX_LOCKING > 0 + /* Write transaction lock. */ + osal_ipclock_t wrt_lock; +#endif /* MDBX_LOCKING > 0 */ + + atomic_txnid_t cached_oldest; + + /* Timestamp of entering an out-of-sync state. Value is represented in a + * suitable system-dependent form, for example clock_gettime(CLOCK_BOOTTIME) + * or clock_gettime(CLOCK_MONOTONIC). */ + mdbx_atomic_uint64_t eoos_timestamp; + + /* Number un-synced-with-disk pages for auto-sync feature. */ + mdbx_atomic_uint64_t unsynced_pages; + + /* Timestamp of the last readers check. */ + mdbx_atomic_uint64_t readers_check_timestamp; + + /* Number of page which was discarded last time by madvise(DONTNEED). */ + atomic_pgno_t discarded_tail; + + /* Shared anchor for tracking readahead edge and enabled/disabled status. */ + pgno_t readahead_anchor; + + /* Shared cache for mincore() results */ + struct { + pgno_t begin[4]; + uint64_t mask[4]; + } mincore_cache; MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/ - /* Statistics of costly ops of all (running, completed and aborted) - * transactions */ - pgop_stat_t mti_pgop_stat; +#if MDBX_LOCKING > 0 + /* Readeaders table lock. */ + osal_ipclock_t rdt_lock; +#endif /* MDBX_LOCKING > 0 */ + + /* The number of slots that have been used in the reader table. + * This always records the maximum count, it is not decremented + * when readers release their slots. */ + mdbx_atomic_uint32_t rdt_length; + mdbx_atomic_uint32_t rdt_refresh_flag; + +#if FLEXIBLE_ARRAY_MEMBERS + MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/ + reader_slot_t rdt[] /* dynamic size */; + +/* Lockfile format signature: version, features and field layout */ +#define MDBX_LOCK_FORMAT \ + (MDBX_LCK_SIGN * 27733 + (unsigned)sizeof(reader_slot_t) * 13 + \ + (unsigned)offsetof(reader_slot_t, snapshot_pages_used) * 251 + (unsigned)offsetof(lck_t, cached_oldest) * 83 + \ + (unsigned)offsetof(lck_t, rdt_length) * 37 + (unsigned)offsetof(lck_t, rdt) * 29) +#endif /* FLEXIBLE_ARRAY_MEMBERS */ +} lck_t; + +#define MDBX_LOCK_MAGIC ((MDBX_MAGIC << 8) + MDBX_LOCK_VERSION) + +#define MDBX_READERS_LIMIT 32767 + +#define MIN_MAPSIZE (MDBX_MIN_PAGESIZE * MIN_PAGENO) +#if defined(_WIN32) || defined(_WIN64) +#define MAX_MAPSIZE32 UINT32_C(0x38000000) +#else +#define MAX_MAPSIZE32 UINT32_C(0x7f000000) +#endif +#define MAX_MAPSIZE64 ((MAX_PAGENO + 1) * (uint64_t)MDBX_MAX_PAGESIZE) + +#if MDBX_WORDBITS >= 64 +#define MAX_MAPSIZE MAX_MAPSIZE64 +#define PAGELIST_LIMIT ((size_t)MAX_PAGENO) +#else +#define MAX_MAPSIZE MAX_MAPSIZE32 +#define PAGELIST_LIMIT (MAX_MAPSIZE32 / MDBX_MIN_PAGESIZE) +#endif /* MDBX_WORDBITS */ + +#define MDBX_GOLD_RATIO_DBL 1.6180339887498948482 +#define MEGABYTE ((size_t)1 << 20) + +/*----------------------------------------------------------------------------*/ + +union logger_union { + void *ptr; + MDBX_debug_func *fmt; + MDBX_debug_func_nofmt *nofmt; +}; + +struct libmdbx_globals { + bin128_t bootid; + unsigned sys_pagesize, sys_allocation_granularity; + uint8_t sys_pagesize_ln2; + uint8_t runtime_flags; + uint8_t loglevel; +#if defined(_WIN32) || defined(_WIN64) + bool running_under_Wine; +#elif defined(__linux__) || defined(__gnu_linux__) + bool running_on_WSL1 /* Windows Subsystem 1 for Linux */; + uint32_t linux_kernel_version; +#endif /* Linux */ + union logger_union logger; + osal_fastmutex_t debug_lock; + size_t logger_buffer_size; + char *logger_buffer; +}; + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +extern struct libmdbx_globals globals; +#if defined(_WIN32) || defined(_WIN64) +extern struct libmdbx_imports imports; +#endif /* Windows */ + +#ifndef __Wpedantic_format_voidptr +MDBX_MAYBE_UNUSED static inline const void *__Wpedantic_format_voidptr(const void *ptr) { return ptr; } +#define __Wpedantic_format_voidptr(ARG) __Wpedantic_format_voidptr(ARG) +#endif /* __Wpedantic_format_voidptr */ + +MDBX_INTERNAL void MDBX_PRINTF_ARGS(4, 5) debug_log(int level, const char *function, int line, const char *fmt, ...) + MDBX_PRINTF_ARGS(4, 5); +MDBX_INTERNAL void debug_log_va(int level, const char *function, int line, const char *fmt, va_list args); + +#if MDBX_DEBUG +#define LOG_ENABLED(LVL) unlikely(LVL <= globals.loglevel) +#define AUDIT_ENABLED() unlikely((globals.runtime_flags & (unsigned)MDBX_DBG_AUDIT)) +#else /* MDBX_DEBUG */ +#define LOG_ENABLED(LVL) (LVL < MDBX_LOG_VERBOSE && LVL <= globals.loglevel) +#define AUDIT_ENABLED() (0) +#endif /* LOG_ENABLED() & AUDIT_ENABLED() */ + +#if MDBX_FORCE_ASSERTIONS +#define ASSERT_ENABLED() (1) +#elif MDBX_DEBUG +#define ASSERT_ENABLED() likely((globals.runtime_flags & (unsigned)MDBX_DBG_ASSERT)) +#else +#define ASSERT_ENABLED() (0) +#endif /* ASSERT_ENABLED() */ + +#define DEBUG_EXTRA(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_EXTRA)) \ + debug_log(MDBX_LOG_EXTRA, __func__, __LINE__, fmt, __VA_ARGS__); \ + } while (0) + +#define DEBUG_EXTRA_PRINT(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_EXTRA)) \ + debug_log(MDBX_LOG_EXTRA, nullptr, 0, fmt, __VA_ARGS__); \ + } while (0) + +#define TRACE(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_TRACE)) \ + debug_log(MDBX_LOG_TRACE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ + } while (0) + +#define DEBUG(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_DEBUG)) \ + debug_log(MDBX_LOG_DEBUG, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ + } while (0) + +#define VERBOSE(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_VERBOSE)) \ + debug_log(MDBX_LOG_VERBOSE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ + } while (0) + +#define NOTICE(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_NOTICE)) \ + debug_log(MDBX_LOG_NOTICE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ + } while (0) + +#define WARNING(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_WARN)) \ + debug_log(MDBX_LOG_WARN, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ + } while (0) + +#undef ERROR /* wingdi.h \ + Yeah, morons from M$ put such definition to the public header. */ + +#define ERROR(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_ERROR)) \ + debug_log(MDBX_LOG_ERROR, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ + } while (0) + +#define FATAL(fmt, ...) debug_log(MDBX_LOG_FATAL, __func__, __LINE__, fmt "\n", __VA_ARGS__); + +#if MDBX_DEBUG +#define ASSERT_FAIL(env, msg, func, line) mdbx_assert_fail(env, msg, func, line) +#else /* MDBX_DEBUG */ +MDBX_NORETURN __cold void assert_fail(const char *msg, const char *func, unsigned line); +#define ASSERT_FAIL(env, msg, func, line) \ + do { \ + (void)(env); \ + assert_fail(msg, func, line); \ + } while (0) +#endif /* MDBX_DEBUG */ + +#define ENSURE_MSG(env, expr, msg) \ + do { \ + if (unlikely(!(expr))) \ + ASSERT_FAIL(env, msg, __func__, __LINE__); \ + } while (0) + +#define ENSURE(env, expr) ENSURE_MSG(env, expr, #expr) + +/* assert(3) variant in environment context */ +#define eASSERT(env, expr) \ + do { \ + if (ASSERT_ENABLED()) \ + ENSURE(env, expr); \ + } while (0) + +/* assert(3) variant in cursor context */ +#define cASSERT(mc, expr) eASSERT((mc)->txn->env, expr) + +/* assert(3) variant in transaction context */ +#define tASSERT(txn, expr) eASSERT((txn)->env, expr) + +#ifndef xMDBX_TOOLS /* Avoid using internal eASSERT() */ +#undef assert +#define assert(expr) eASSERT(nullptr, expr) +#endif + +MDBX_MAYBE_UNUSED static inline void jitter4testing(bool tiny) { +#if MDBX_DEBUG + if (globals.runtime_flags & (unsigned)MDBX_DBG_JITTER) + osal_jitter(tiny); +#else + (void)tiny; +#endif +} - MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/ +MDBX_MAYBE_UNUSED MDBX_INTERNAL void page_list(page_t *mp); - /* Write transaction lock. */ -#if MDBX_LOCKING > 0 - osal_ipclock_t mti_wlock; -#endif /* MDBX_LOCKING > 0 */ +MDBX_INTERNAL const char *pagetype_caption(const uint8_t type, char buf4unknown[16]); +/* Key size which fits in a DKBUF (debug key buffer). */ +#define DKBUF_MAX 127 +#define DKBUF char dbg_kbuf[DKBUF_MAX * 4 + 2] +#define DKEY(x) mdbx_dump_val(x, dbg_kbuf, DKBUF_MAX * 2 + 1) +#define DVAL(x) mdbx_dump_val(x, dbg_kbuf + DKBUF_MAX * 2 + 1, DKBUF_MAX * 2 + 1) - atomic_txnid_t mti_oldest_reader; +#if MDBX_DEBUG +#define DKBUF_DEBUG DKBUF +#define DKEY_DEBUG(x) DKEY(x) +#define DVAL_DEBUG(x) DVAL(x) +#else +#define DKBUF_DEBUG ((void)(0)) +#define DKEY_DEBUG(x) ("-") +#define DVAL_DEBUG(x) ("-") +#endif - /* Timestamp of entering an out-of-sync state. Value is represented in a - * suitable system-dependent form, for example clock_gettime(CLOCK_BOOTTIME) - * or clock_gettime(CLOCK_MONOTONIC). */ - MDBX_atomic_uint64_t mti_eoos_timestamp; +MDBX_INTERNAL void log_error(const int err, const char *func, unsigned line); - /* Number un-synced-with-disk pages for auto-sync feature. */ - MDBX_atomic_uint64_t mti_unsynced_pages; +MDBX_MAYBE_UNUSED static inline int log_if_error(const int err, const char *func, unsigned line) { + if (unlikely(err != MDBX_SUCCESS)) + log_error(err, func, line); + return err; +} - /* Timestamp of the last readers check. */ - MDBX_atomic_uint64_t mti_reader_check_timestamp; +#define LOG_IFERR(err) log_if_error((err), __func__, __LINE__) - /* Number of page which was discarded last time by madvise(DONTNEED). */ - atomic_pgno_t mti_discarded_tail; +/* Test if the flags f are set in a flag word w. */ +#define F_ISSET(w, f) (((w) & (f)) == (f)) - /* Shared anchor for tracking readahead edge and enabled/disabled status. */ - pgno_t mti_readahead_anchor; +/* Round n up to an even number. */ +#define EVEN_CEIL(n) (((n) + 1UL) & -2L) /* sign-extending -2 to match n+1U */ - /* Shared cache for mincore() results */ - struct { - pgno_t begin[4]; - uint64_t mask[4]; - } mti_mincore_cache; +/* Round n down to an even number. */ +#define EVEN_FLOOR(n) ((n) & ~(size_t)1) - MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/ +/* + * / + * | -1, a < b + * CMP2INT(a,b) = < 0, a == b + * | 1, a > b + * \ + */ +#define CMP2INT(a, b) (((a) != (b)) ? (((a) < (b)) ? -1 : 1) : 0) - /* Readeaders registration lock. */ -#if MDBX_LOCKING > 0 - osal_ipclock_t mti_rlock; -#endif /* MDBX_LOCKING > 0 */ +/* Pointer displacement without casting to char* to avoid pointer-aliasing */ +#define ptr_disp(ptr, disp) ((void *)(((intptr_t)(ptr)) + ((intptr_t)(disp)))) - /* The number of slots that have been used in the reader table. - * This always records the maximum count, it is not decremented - * when readers release their slots. */ - MDBX_atomic_uint32_t mti_numreaders; - MDBX_atomic_uint32_t mti_readers_refresh_flag; +/* Pointer distance as signed number of bytes */ +#define ptr_dist(more, less) (((intptr_t)(more)) - ((intptr_t)(less))) -#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ - (!defined(__cplusplus) && defined(_MSC_VER)) - MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/ - MDBX_reader mti_readers[] /* dynamic size */; -#endif /* C99 */ -} MDBX_lockinfo; +#define MDBX_ASAN_POISON_MEMORY_REGION(addr, size) \ + do { \ + TRACE("POISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), (size_t)(size), __LINE__); \ + ASAN_POISON_MEMORY_REGION(addr, size); \ + } while (0) -/* Lockfile format signature: version, features and field layout */ -#define MDBX_LOCK_FORMAT \ - (MDBX_CLOCK_SIGN * 27733 + (unsigned)sizeof(MDBX_reader) * 13 + \ - (unsigned)offsetof(MDBX_reader, mr_snapshot_pages_used) * 251 + \ - (unsigned)offsetof(MDBX_lockinfo, mti_oldest_reader) * 83 + \ - (unsigned)offsetof(MDBX_lockinfo, mti_numreaders) * 37 + \ - (unsigned)offsetof(MDBX_lockinfo, mti_readers) * 29) +#define MDBX_ASAN_UNPOISON_MEMORY_REGION(addr, size) \ + do { \ + TRACE("UNPOISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), (size_t)(size), __LINE__); \ + ASAN_UNPOISON_MEMORY_REGION(addr, size); \ + } while (0) -#define MDBX_DATA_MAGIC \ - ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + MDBX_DATA_VERSION) +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline size_t branchless_abs(intptr_t value) { + assert(value > INT_MIN); + const size_t expanded_sign = (size_t)(value >> (sizeof(value) * CHAR_BIT - 1)); + return ((size_t)value + expanded_sign) ^ expanded_sign; +} -#define MDBX_DATA_MAGIC_LEGACY_COMPAT \ - ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + 2) +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline bool is_powerof2(size_t x) { return (x & (x - 1)) == 0; } -#define MDBX_DATA_MAGIC_LEGACY_DEVEL ((MDBX_MAGIC << 8) + 255) +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline size_t floor_powerof2(size_t value, size_t granularity) { + assert(is_powerof2(granularity)); + return value & ~(granularity - 1); +} -#define MDBX_LOCK_MAGIC ((MDBX_MAGIC << 8) + MDBX_LOCK_VERSION) +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline size_t ceil_powerof2(size_t value, size_t granularity) { + return floor_powerof2(value + granularity - 1, granularity); +} -/* The maximum size of a database page. - * - * It is 64K, but value-PAGEHDRSZ must fit in MDBX_page.mp_upper. - * - * MDBX will use database pages < OS pages if needed. - * That causes more I/O in write transactions: The OS must - * know (read) the whole page before writing a partial page. - * - * Note that we don't currently support Huge pages. On Linux, - * regular data files cannot use Huge pages, and in general - * Huge pages aren't actually pageable. We rely on the OS - * demand-pager to read our data and page it out when memory - * pressure from other processes is high. So until OSs have - * actual paging support for Huge pages, they're not viable. */ -#define MAX_PAGESIZE MDBX_MAX_PAGESIZE -#define MIN_PAGESIZE MDBX_MIN_PAGESIZE - -#define MIN_MAPSIZE (MIN_PAGESIZE * MIN_PAGENO) -#if defined(_WIN32) || defined(_WIN64) -#define MAX_MAPSIZE32 UINT32_C(0x38000000) -#else -#define MAX_MAPSIZE32 UINT32_C(0x7f000000) -#endif -#define MAX_MAPSIZE64 ((MAX_PAGENO + 1) * (uint64_t)MAX_PAGESIZE) +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED MDBX_INTERNAL unsigned log2n_powerof2(size_t value_uintptr); -#if MDBX_WORDBITS >= 64 -#define MAX_MAPSIZE MAX_MAPSIZE64 -#define MDBX_PGL_LIMIT ((size_t)MAX_PAGENO) -#else -#define MAX_MAPSIZE MAX_MAPSIZE32 -#define MDBX_PGL_LIMIT (MAX_MAPSIZE32 / MIN_PAGESIZE) -#endif /* MDBX_WORDBITS */ +MDBX_NOTHROW_CONST_FUNCTION MDBX_INTERNAL uint64_t rrxmrrxmsx_0(uint64_t v); -#define MDBX_READERS_LIMIT 32767 -#define MDBX_RADIXSORT_THRESHOLD 142 -#define MDBX_GOLD_RATIO_DBL 1.6180339887498948482 +struct monotime_cache { + uint64_t value; + int expire_countdown; +}; -/*----------------------------------------------------------------------------*/ +MDBX_MAYBE_UNUSED static inline uint64_t monotime_since_cached(uint64_t begin_timestamp, struct monotime_cache *cache) { + if (cache->expire_countdown) + cache->expire_countdown -= 1; + else { + cache->value = osal_monotime(); + cache->expire_countdown = 42 / 3; + } + return cache->value - begin_timestamp; +} /* An PNL is an Page Number List, a sorted array of IDs. + * * The first element of the array is a counter for how many actual page-numbers * are in the list. By default PNLs are sorted in descending order, this allow * cut off a page with lowest pgno (at the tail) just truncating the list. The * sort order of PNLs is controlled by the MDBX_PNL_ASCENDING build option. */ -typedef pgno_t *MDBX_PNL; +typedef pgno_t *pnl_t; +typedef const pgno_t *const_pnl_t; #if MDBX_PNL_ASCENDING #define MDBX_PNL_ORDERED(first, last) ((first) < (last)) @@ -3335,46 +3078,17 @@ typedef pgno_t *MDBX_PNL; #define MDBX_PNL_DISORDERED(first, last) ((first) <= (last)) #endif -/* List of txnid, only for MDBX_txn.tw.lifo_reclaimed */ -typedef txnid_t *MDBX_TXL; - -/* An Dirty-Page list item is an pgno/pointer pair. */ -typedef struct MDBX_dp { - MDBX_page *ptr; - pgno_t pgno, npages; -} MDBX_dp; - -/* An DPL (dirty-page list) is a sorted array of MDBX_DPs. */ -typedef struct MDBX_dpl { - size_t sorted; - size_t length; - size_t pages_including_loose; /* number of pages, but not an entries. */ - size_t detent; /* allocated size excluding the MDBX_DPL_RESERVE_GAP */ -#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ - (!defined(__cplusplus) && defined(_MSC_VER)) - MDBX_dp items[] /* dynamic size with holes at zero and after the last */; -#endif -} MDBX_dpl; - -/* PNL sizes */ #define MDBX_PNL_GRANULATE_LOG2 10 #define MDBX_PNL_GRANULATE (1 << MDBX_PNL_GRANULATE_LOG2) -#define MDBX_PNL_INITIAL \ - (MDBX_PNL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t)) - -#define MDBX_TXL_GRANULATE 32 -#define MDBX_TXL_INITIAL \ - (MDBX_TXL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t)) -#define MDBX_TXL_MAX \ - ((1u << 26) - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t)) +#define MDBX_PNL_INITIAL (MDBX_PNL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t)) #define MDBX_PNL_ALLOCLEN(pl) ((pl)[-1]) #define MDBX_PNL_GETSIZE(pl) ((size_t)((pl)[0])) -#define MDBX_PNL_SETSIZE(pl, size) \ - do { \ - const size_t __size = size; \ - assert(__size < INT_MAX); \ - (pl)[0] = (pgno_t)__size; \ +#define MDBX_PNL_SETSIZE(pl, size) \ + do { \ + const size_t __size = size; \ + assert(__size < INT_MAX); \ + (pl)[0] = (pgno_t)__size; \ } while (0) #define MDBX_PNL_FIRST(pl) ((pl)[1]) #define MDBX_PNL_LAST(pl) ((pl)[MDBX_PNL_GETSIZE(pl)]) @@ -3394,683 +3108,126 @@ typedef struct MDBX_dpl { #define MDBX_PNL_SIZEOF(pl) ((MDBX_PNL_GETSIZE(pl) + 1) * sizeof(pgno_t)) #define MDBX_PNL_IS_EMPTY(pl) (MDBX_PNL_GETSIZE(pl) == 0) -/*----------------------------------------------------------------------------*/ -/* Internal structures */ - -/* Auxiliary DB info. - * The information here is mostly static/read-only. There is - * only a single copy of this record in the environment. */ -typedef struct MDBX_dbx { - MDBX_val md_name; /* name of the database */ - MDBX_cmp_func *md_cmp; /* function for comparing keys */ - MDBX_cmp_func *md_dcmp; /* function for comparing data items */ - size_t md_klen_min, md_klen_max; /* min/max key length for the database */ - size_t md_vlen_min, - md_vlen_max; /* min/max value/data length for the database */ -} MDBX_dbx; - -typedef struct troika { - uint8_t fsm, recent, prefer_steady, tail_and_flags; -#if MDBX_WORDBITS > 32 /* Workaround for false-positives from Valgrind */ - uint32_t unused_pad; -#endif -#define TROIKA_HAVE_STEADY(troika) ((troika)->fsm & 7) -#define TROIKA_STRICT_VALID(troika) ((troika)->tail_and_flags & 64) -#define TROIKA_VALID(troika) ((troika)->tail_and_flags & 128) -#define TROIKA_TAIL(troika) ((troika)->tail_and_flags & 3) - txnid_t txnid[NUM_METAS]; -} meta_troika_t; - -/* A database transaction. - * Every operation requires a transaction handle. */ -struct MDBX_txn { -#define MDBX_MT_SIGNATURE UINT32_C(0x93D53A31) - uint32_t mt_signature; - - /* Transaction Flags */ - /* mdbx_txn_begin() flags */ -#define MDBX_TXN_RO_BEGIN_FLAGS (MDBX_TXN_RDONLY | MDBX_TXN_RDONLY_PREPARE) -#define MDBX_TXN_RW_BEGIN_FLAGS \ - (MDBX_TXN_NOMETASYNC | MDBX_TXN_NOSYNC | MDBX_TXN_TRY) - /* Additional flag for sync_locked() */ -#define MDBX_SHRINK_ALLOWED UINT32_C(0x40000000) - -#define MDBX_TXN_DRAINED_GC 0x20 /* GC was depleted up to oldest reader */ - -#define TXN_FLAGS \ - (MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_DIRTY | MDBX_TXN_SPILLS | \ - MDBX_TXN_HAS_CHILD | MDBX_TXN_INVALID | MDBX_TXN_DRAINED_GC) - -#if (TXN_FLAGS & (MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_RO_BEGIN_FLAGS)) || \ - ((MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_RO_BEGIN_FLAGS | TXN_FLAGS) & \ - MDBX_SHRINK_ALLOWED) -#error "Oops, some txn flags overlapped or wrong" -#endif - uint32_t mt_flags; - - MDBX_txn *mt_parent; /* parent of a nested txn */ - /* Nested txn under this txn, set together with flag MDBX_TXN_HAS_CHILD */ - MDBX_txn *mt_child; - MDBX_geo mt_geo; - /* next unallocated page */ -#define mt_next_pgno mt_geo.next - /* corresponding to the current size of datafile */ -#define mt_end_pgno mt_geo.now - - /* The ID of this transaction. IDs are integers incrementing from - * INITIAL_TXNID. Only committed write transactions increment the ID. If a - * transaction aborts, the ID may be re-used by the next writer. */ - txnid_t mt_txnid; - txnid_t mt_front; - - MDBX_env *mt_env; /* the DB environment */ - /* Array of records for each DB known in the environment. */ - MDBX_dbx *mt_dbxs; - /* Array of MDBX_db records for each known DB */ - MDBX_db *mt_dbs; - /* Array of sequence numbers for each DB handle */ - MDBX_atomic_uint32_t *mt_dbiseqs; - - /* Transaction DBI Flags */ -#define DBI_DIRTY MDBX_DBI_DIRTY /* DB was written in this txn */ -#define DBI_STALE MDBX_DBI_STALE /* Named-DB record is older than txnID */ -#define DBI_FRESH MDBX_DBI_FRESH /* Named-DB handle opened in this txn */ -#define DBI_CREAT MDBX_DBI_CREAT /* Named-DB handle created in this txn */ -#define DBI_VALID 0x10 /* DB handle is valid, see also DB_VALID */ -#define DBI_USRVALID 0x20 /* As DB_VALID, but not set for FREE_DBI */ -#define DBI_AUDITED 0x40 /* Internal flag for accounting during audit */ - /* Array of flags for each DB */ - uint8_t *mt_dbistate; - /* Number of DB records in use, or 0 when the txn is finished. - * This number only ever increments until the txn finishes; we - * don't decrement it when individual DB handles are closed. */ - MDBX_dbi mt_numdbs; - size_t mt_owner; /* thread ID that owns this transaction */ - MDBX_canary mt_canary; - void *mt_userctx; /* User-settable context */ - MDBX_cursor **mt_cursors; - - union { - struct { - /* For read txns: This thread/txn's reader table slot, or NULL. */ - MDBX_reader *reader; - } to; - struct { - meta_troika_t troika; - /* In write txns, array of cursors for each DB */ - MDBX_PNL relist; /* Reclaimed GC pages */ - txnid_t last_reclaimed; /* ID of last used record */ -#if MDBX_ENABLE_REFUND - pgno_t loose_refund_wl /* FIXME: describe */; -#endif /* MDBX_ENABLE_REFUND */ - /* a sequence to spilling dirty page with LRU policy */ - unsigned dirtylru; - /* dirtylist room: Dirty array size - dirty pages visible to this txn. - * Includes ancestor txns' dirty pages not hidden by other txns' - * dirty/spilled pages. Thus commit(nested txn) has room to merge - * dirtylist into mt_parent after freeing hidden mt_parent pages. */ - size_t dirtyroom; - /* For write txns: Modified pages. Sorted when not MDBX_WRITEMAP. */ - MDBX_dpl *dirtylist; - /* The list of reclaimed txns from GC */ - MDBX_TXL lifo_reclaimed; - /* The list of pages that became unused during this transaction. */ - MDBX_PNL retired_pages; - /* The list of loose pages that became unused and may be reused - * in this transaction, linked through `mp_next`. */ - MDBX_page *loose_pages; - /* Number of loose pages (tw.loose_pages) */ - size_t loose_count; - union { - struct { - size_t least_removed; - /* The sorted list of dirty pages we temporarily wrote to disk - * because the dirty list was full. page numbers in here are - * shifted left by 1, deleted slots have the LSB set. */ - MDBX_PNL list; - } spilled; - size_t writemap_dirty_npages; - size_t writemap_spilled_npages; - }; - } tw; - }; -}; - -#if MDBX_WORDBITS >= 64 -#define CURSOR_STACK 32 -#else -#define CURSOR_STACK 24 -#endif - -struct MDBX_xcursor; - -/* Cursors are used for all DB operations. - * A cursor holds a path of (page pointer, key index) from the DB - * root to a position in the DB, plus other state. MDBX_DUPSORT - * cursors include an xcursor to the current data item. Write txns - * track their cursors and keep them up to date when data moves. - * Exception: An xcursor's pointer to a P_SUBP page can be stale. - * (A node with F_DUPDATA but no F_SUBDATA contains a subpage). */ -struct MDBX_cursor { -#define MDBX_MC_LIVE UINT32_C(0xFE05D5B1) -#define MDBX_MC_READY4CLOSE UINT32_C(0x2817A047) -#define MDBX_MC_WAIT4EOT UINT32_C(0x90E297A7) - uint32_t mc_signature; - /* The database handle this cursor operates on */ - MDBX_dbi mc_dbi; - /* Next cursor on this DB in this txn */ - MDBX_cursor *mc_next; - /* Backup of the original cursor if this cursor is a shadow */ - MDBX_cursor *mc_backup; - /* Context used for databases with MDBX_DUPSORT, otherwise NULL */ - struct MDBX_xcursor *mc_xcursor; - /* The transaction that owns this cursor */ - MDBX_txn *mc_txn; - /* The database record for this cursor */ - MDBX_db *mc_db; - /* The database auxiliary record for this cursor */ - MDBX_dbx *mc_dbx; - /* The mt_dbistate for this database */ - uint8_t *mc_dbistate; - uint8_t mc_snum; /* number of pushed pages */ - uint8_t mc_top; /* index of top page, normally mc_snum-1 */ - - /* Cursor state flags. */ -#define C_INITIALIZED 0x01 /* cursor has been initialized and is valid */ -#define C_EOF 0x02 /* No more data */ -#define C_SUB 0x04 /* Cursor is a sub-cursor */ -#define C_DEL 0x08 /* last op was a cursor_del */ -#define C_UNTRACK 0x10 /* Un-track cursor when closing */ -#define C_GCU \ - 0x20 /* Происходит подготовка к обновлению GC, поэтому \ - * можно брать страницы из GC даже для FREE_DBI */ - uint8_t mc_flags; - - /* Cursor checking flags. */ -#define CC_BRANCH 0x01 /* same as P_BRANCH for CHECK_LEAF_TYPE() */ -#define CC_LEAF 0x02 /* same as P_LEAF for CHECK_LEAF_TYPE() */ -#define CC_OVERFLOW 0x04 /* same as P_OVERFLOW for CHECK_LEAF_TYPE() */ -#define CC_UPDATING 0x08 /* update/rebalance pending */ -#define CC_SKIPORD 0x10 /* don't check keys ordering */ -#define CC_LEAF2 0x20 /* same as P_LEAF2 for CHECK_LEAF_TYPE() */ -#define CC_RETIRING 0x40 /* refs to child pages may be invalid */ -#define CC_PAGECHECK 0x80 /* perform page checking, see MDBX_VALIDATION */ - uint8_t mc_checking; - - MDBX_page *mc_pg[CURSOR_STACK]; /* stack of pushed pages */ - indx_t mc_ki[CURSOR_STACK]; /* stack of page indices */ -}; - -#define CHECK_LEAF_TYPE(mc, mp) \ - (((PAGETYPE_WHOLE(mp) ^ (mc)->mc_checking) & \ - (CC_BRANCH | CC_LEAF | CC_OVERFLOW | CC_LEAF2)) == 0) - -/* Context for sorted-dup records. - * We could have gone to a fully recursive design, with arbitrarily - * deep nesting of sub-databases. But for now we only handle these - * levels - main DB, optional sub-DB, sorted-duplicate DB. */ -typedef struct MDBX_xcursor { - /* A sub-cursor for traversing the Dup DB */ - MDBX_cursor mx_cursor; - /* The database record for this Dup DB */ - MDBX_db mx_db; - /* The auxiliary DB record for this Dup DB */ - MDBX_dbx mx_dbx; -} MDBX_xcursor; - -typedef struct MDBX_cursor_couple { - MDBX_cursor outer; - void *mc_userctx; /* User-settable context */ - MDBX_xcursor inner; -} MDBX_cursor_couple; - -/* The database environment. */ -struct MDBX_env { - /* ----------------------------------------------------- mostly static part */ -#define MDBX_ME_SIGNATURE UINT32_C(0x9A899641) - MDBX_atomic_uint32_t me_signature; - /* Failed to update the meta page. Probably an I/O error. */ -#define MDBX_FATAL_ERROR UINT32_C(0x80000000) - /* Some fields are initialized. */ -#define MDBX_ENV_ACTIVE UINT32_C(0x20000000) - /* me_txkey is set */ -#define MDBX_ENV_TXKEY UINT32_C(0x10000000) - /* Legacy MDBX_MAPASYNC (prior v0.9) */ -#define MDBX_DEPRECATED_MAPASYNC UINT32_C(0x100000) - /* Legacy MDBX_COALESCE (prior v0.12) */ -#define MDBX_DEPRECATED_COALESCE UINT32_C(0x2000000) -#define ENV_INTERNAL_FLAGS (MDBX_FATAL_ERROR | MDBX_ENV_ACTIVE | MDBX_ENV_TXKEY) - uint32_t me_flags; - osal_mmap_t me_dxb_mmap; /* The main data file */ -#define me_map me_dxb_mmap.base -#define me_lazy_fd me_dxb_mmap.fd - mdbx_filehandle_t me_dsync_fd, me_fd4meta; -#if defined(_WIN32) || defined(_WIN64) -#define me_overlapped_fd me_ioring.overlapped_fd - HANDLE me_data_lock_event; -#endif /* Windows */ - osal_mmap_t me_lck_mmap; /* The lock file */ -#define me_lfd me_lck_mmap.fd - struct MDBX_lockinfo *me_lck; - - unsigned me_psize; /* DB page size, initialized from me_os_psize */ - uint16_t me_leaf_nodemax; /* max size of a leaf-node */ - uint16_t me_branch_nodemax; /* max size of a branch-node */ - uint16_t me_subpage_limit; - uint16_t me_subpage_room_threshold; - uint16_t me_subpage_reserve_prereq; - uint16_t me_subpage_reserve_limit; - atomic_pgno_t me_mlocked_pgno; - uint8_t me_psize2log; /* log2 of DB page size */ - int8_t me_stuck_meta; /* recovery-only: target meta page or less that zero */ - uint16_t me_merge_threshold, - me_merge_threshold_gc; /* pages emptier than this are candidates for - merging */ - unsigned me_os_psize; /* OS page size, from osal_syspagesize() */ - unsigned me_maxreaders; /* size of the reader table */ - MDBX_dbi me_maxdbs; /* size of the DB table */ - uint32_t me_pid; /* process ID of this env */ - osal_thread_key_t me_txkey; /* thread-key for readers */ - pathchar_t *me_pathname; /* path to the DB files */ - void *me_pbuf; /* scratch area for DUPSORT put() */ - MDBX_txn *me_txn0; /* preallocated write transaction */ - - MDBX_dbx *me_dbxs; /* array of static DB info */ - uint16_t *me_dbflags; /* array of flags from MDBX_db.md_flags */ - MDBX_atomic_uint32_t *me_dbiseqs; /* array of dbi sequence numbers */ - unsigned - me_maxgc_ov1page; /* Number of pgno_t fit in a single overflow page */ - unsigned me_maxgc_per_branch; - uint32_t me_live_reader; /* have liveness lock in reader table */ - void *me_userctx; /* User-settable context */ - MDBX_hsr_func *me_hsr_callback; /* Callback for kicking laggard readers */ - size_t me_madv_threshold; - - struct { - unsigned dp_reserve_limit; - unsigned rp_augment_limit; - unsigned dp_limit; - unsigned dp_initial; - uint8_t dp_loose_limit; - uint8_t spill_max_denominator; - uint8_t spill_min_denominator; - uint8_t spill_parent4child_denominator; - unsigned merge_threshold_16dot16_percent; -#if !(defined(_WIN32) || defined(_WIN64)) - unsigned writethrough_threshold; -#endif /* Windows */ - bool prefault_write; - union { - unsigned all; - /* tracks options with non-auto values but tuned by user */ - struct { - unsigned dp_limit : 1; - unsigned rp_augment_limit : 1; - unsigned prefault_write : 1; - } non_auto; - } flags; - } me_options; - - /* struct me_dbgeo used for accepting db-geo params from user for the new - * database creation, i.e. when mdbx_env_set_geometry() was called before - * mdbx_env_open(). */ - struct { - size_t lower; /* minimal size of datafile */ - size_t upper; /* maximal size of datafile */ - size_t now; /* current size of datafile */ - size_t grow; /* step to grow datafile */ - size_t shrink; /* threshold to shrink datafile */ - } me_dbgeo; - -#if MDBX_LOCKING == MDBX_LOCKING_SYSV - union { - key_t key; - int semid; - } me_sysv_ipc; -#endif /* MDBX_LOCKING == MDBX_LOCKING_SYSV */ - bool me_incore; - - MDBX_env *me_lcklist_next; - - /* --------------------------------------------------- mostly volatile part */ - - MDBX_txn *me_txn; /* current write transaction */ - osal_fastmutex_t me_dbi_lock; - MDBX_dbi me_numdbs; /* number of DBs opened */ - bool me_prefault_write; - - MDBX_page *me_dp_reserve; /* list of malloc'ed blocks for re-use */ - unsigned me_dp_reserve_len; - /* PNL of pages that became unused in a write txn */ - MDBX_PNL me_retired_pages; - osal_ioring_t me_ioring; - -#if defined(_WIN32) || defined(_WIN64) - osal_srwlock_t me_remap_guard; - /* Workaround for LockFileEx and WriteFile multithread bug */ - CRITICAL_SECTION me_windowsbug_lock; - char *me_pathname_char; /* cache of multi-byte representation of pathname - to the DB files */ -#else - osal_fastmutex_t me_remap_guard; -#endif - - /* -------------------------------------------------------------- debugging */ - -#if MDBX_DEBUG - MDBX_assert_func *me_assert_func; /* Callback for assertion failures */ -#endif -#ifdef MDBX_USE_VALGRIND - int me_valgrind_handle; -#endif -#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__) - MDBX_atomic_uint32_t me_ignore_EDEADLK; - pgno_t me_poison_edge; -#endif /* MDBX_USE_VALGRIND || __SANITIZE_ADDRESS__ */ - -#ifndef xMDBX_DEBUG_SPILLING -#define xMDBX_DEBUG_SPILLING 0 -#endif -#if xMDBX_DEBUG_SPILLING == 2 - size_t debug_dirtied_est, debug_dirtied_act; -#endif /* xMDBX_DEBUG_SPILLING */ - - /* ------------------------------------------------- stub for lck-less mode */ - MDBX_atomic_uint64_t - x_lckless_stub[(sizeof(MDBX_lockinfo) + MDBX_CACHELINE_SIZE - 1) / - sizeof(MDBX_atomic_uint64_t)]; -}; - -#ifndef __cplusplus -/*----------------------------------------------------------------------------*/ -/* Cache coherence and mmap invalidation */ - -#if MDBX_CPU_WRITEBACK_INCOHERENT -#define osal_flush_incoherent_cpu_writeback() osal_memory_barrier() -#else -#define osal_flush_incoherent_cpu_writeback() osal_compiler_barrier() -#endif /* MDBX_CPU_WRITEBACK_INCOHERENT */ - -MDBX_MAYBE_UNUSED static __inline void -osal_flush_incoherent_mmap(const void *addr, size_t nbytes, - const intptr_t pagesize) { -#if MDBX_MMAP_INCOHERENT_FILE_WRITE - char *const begin = (char *)(-pagesize & (intptr_t)addr); - char *const end = - (char *)(-pagesize & (intptr_t)((char *)addr + nbytes + pagesize - 1)); - int err = msync(begin, end - begin, MS_SYNC | MS_INVALIDATE) ? errno : 0; - eASSERT(nullptr, err == 0); - (void)err; -#else - (void)pagesize; -#endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */ - -#if MDBX_MMAP_INCOHERENT_CPU_CACHE -#ifdef DCACHE - /* MIPS has cache coherency issues. - * Note: for any nbytes >= on-chip cache size, entire is flushed. */ - cacheflush((void *)addr, nbytes, DCACHE); -#else -#error "Oops, cacheflush() not available" -#endif /* DCACHE */ -#endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */ +MDBX_MAYBE_UNUSED static inline size_t pnl_size2bytes(size_t size) { + assert(size > 0 && size <= PAGELIST_LIMIT); +#if MDBX_PNL_PREALLOC_FOR_RADIXSORT -#if !MDBX_MMAP_INCOHERENT_FILE_WRITE && !MDBX_MMAP_INCOHERENT_CPU_CACHE - (void)addr; - (void)nbytes; -#endif + size += size; +#endif /* MDBX_PNL_PREALLOC_FOR_RADIXSORT */ + STATIC_ASSERT(MDBX_ASSUME_MALLOC_OVERHEAD + + (PAGELIST_LIMIT * (MDBX_PNL_PREALLOC_FOR_RADIXSORT + 1) + MDBX_PNL_GRANULATE + 3) * sizeof(pgno_t) < + SIZE_MAX / 4 * 3); + size_t bytes = + ceil_powerof2(MDBX_ASSUME_MALLOC_OVERHEAD + sizeof(pgno_t) * (size + 3), MDBX_PNL_GRANULATE * sizeof(pgno_t)) - + MDBX_ASSUME_MALLOC_OVERHEAD; + return bytes; } -/*----------------------------------------------------------------------------*/ -/* Internal prototypes */ - -MDBX_INTERNAL_FUNC int cleanup_dead_readers(MDBX_env *env, int rlocked, - int *dead); -MDBX_INTERNAL_FUNC int rthc_alloc(osal_thread_key_t *key, MDBX_reader *begin, - MDBX_reader *end); -MDBX_INTERNAL_FUNC void rthc_remove(const osal_thread_key_t key); +MDBX_MAYBE_UNUSED static inline pgno_t pnl_bytes2size(const size_t bytes) { + size_t size = bytes / sizeof(pgno_t); + assert(size > 3 && size <= PAGELIST_LIMIT + /* alignment gap */ 65536); + size -= 3; +#if MDBX_PNL_PREALLOC_FOR_RADIXSORT + size >>= 1; +#endif /* MDBX_PNL_PREALLOC_FOR_RADIXSORT */ + return (pgno_t)size; +} -MDBX_INTERNAL_FUNC void global_ctor(void); -MDBX_INTERNAL_FUNC void osal_ctor(void); -MDBX_INTERNAL_FUNC void global_dtor(void); -MDBX_INTERNAL_FUNC void osal_dtor(void); -MDBX_INTERNAL_FUNC void thread_dtor(void *ptr); +MDBX_INTERNAL pnl_t pnl_alloc(size_t size); -#endif /* !__cplusplus */ +MDBX_INTERNAL void pnl_free(pnl_t pnl); -#define MDBX_IS_ERROR(rc) \ - ((rc) != MDBX_RESULT_TRUE && (rc) != MDBX_RESULT_FALSE) +MDBX_INTERNAL int pnl_reserve(pnl_t __restrict *__restrict ppnl, const size_t wanna); -/* Internal error codes, not exposed outside libmdbx */ -#define MDBX_NO_ROOT (MDBX_LAST_ADDED_ERRCODE + 10) +MDBX_MAYBE_UNUSED static inline int __must_check_result pnl_need(pnl_t __restrict *__restrict ppnl, size_t num) { + assert(MDBX_PNL_GETSIZE(*ppnl) <= PAGELIST_LIMIT && MDBX_PNL_ALLOCLEN(*ppnl) >= MDBX_PNL_GETSIZE(*ppnl)); + assert(num <= PAGELIST_LIMIT); + const size_t wanna = MDBX_PNL_GETSIZE(*ppnl) + num; + return likely(MDBX_PNL_ALLOCLEN(*ppnl) >= wanna) ? MDBX_SUCCESS : pnl_reserve(ppnl, wanna); +} -/* Debugging output value of a cursor DBI: Negative in a sub-cursor. */ -#define DDBI(mc) \ - (((mc)->mc_flags & C_SUB) ? -(int)(mc)->mc_dbi : (int)(mc)->mc_dbi) +MDBX_MAYBE_UNUSED static inline void pnl_append_prereserved(__restrict pnl_t pnl, pgno_t pgno) { + assert(MDBX_PNL_GETSIZE(pnl) < MDBX_PNL_ALLOCLEN(pnl)); + if (AUDIT_ENABLED()) { + for (size_t i = MDBX_PNL_GETSIZE(pnl); i > 0; --i) + assert(pgno != pnl[i]); + } + *pnl += 1; + MDBX_PNL_LAST(pnl) = pgno; +} -/* Key size which fits in a DKBUF (debug key buffer). */ -#define DKBUF_MAX 511 -#define DKBUF char _kbuf[DKBUF_MAX * 4 + 2] -#define DKEY(x) mdbx_dump_val(x, _kbuf, DKBUF_MAX * 2 + 1) -#define DVAL(x) mdbx_dump_val(x, _kbuf + DKBUF_MAX * 2 + 1, DKBUF_MAX * 2 + 1) +MDBX_INTERNAL void pnl_shrink(pnl_t __restrict *__restrict ppnl); -#if MDBX_DEBUG -#define DKBUF_DEBUG DKBUF -#define DKEY_DEBUG(x) DKEY(x) -#define DVAL_DEBUG(x) DVAL(x) -#else -#define DKBUF_DEBUG ((void)(0)) -#define DKEY_DEBUG(x) ("-") -#define DVAL_DEBUG(x) ("-") -#endif +MDBX_INTERNAL int __must_check_result spill_append_span(__restrict pnl_t *ppnl, pgno_t pgno, size_t n); -/* An invalid page number. - * Mainly used to denote an empty tree. */ -#define P_INVALID (~(pgno_t)0) +MDBX_INTERNAL int __must_check_result pnl_append_span(__restrict pnl_t *ppnl, pgno_t pgno, size_t n); -/* Test if the flags f are set in a flag word w. */ -#define F_ISSET(w, f) (((w) & (f)) == (f)) +MDBX_INTERNAL int __must_check_result pnl_insert_span(__restrict pnl_t *ppnl, pgno_t pgno, size_t n); -/* Round n up to an even number. */ -#define EVEN(n) (((n) + 1UL) & -2L) /* sign-extending -2 to match n+1U */ - -/* Default size of memory map. - * This is certainly too small for any actual applications. Apps should - * always set the size explicitly using mdbx_env_set_geometry(). */ -#define DEFAULT_MAPSIZE MEGABYTE - -/* Number of slots in the reader table. - * This value was chosen somewhat arbitrarily. The 61 is a prime number, - * and such readers plus a couple mutexes fit into single 4KB page. - * Applications should set the table size using mdbx_env_set_maxreaders(). */ -#define DEFAULT_READERS 61 - -/* Test if a page is a leaf page */ -#define IS_LEAF(p) (((p)->mp_flags & P_LEAF) != 0) -/* Test if a page is a LEAF2 page */ -#define IS_LEAF2(p) unlikely(((p)->mp_flags & P_LEAF2) != 0) -/* Test if a page is a branch page */ -#define IS_BRANCH(p) (((p)->mp_flags & P_BRANCH) != 0) -/* Test if a page is an overflow page */ -#define IS_OVERFLOW(p) unlikely(((p)->mp_flags & P_OVERFLOW) != 0) -/* Test if a page is a sub page */ -#define IS_SUBP(p) (((p)->mp_flags & P_SUBP) != 0) +MDBX_INTERNAL size_t pnl_search_nochk(const pnl_t pnl, pgno_t pgno); -/* Header for a single key/data pair within a page. - * Used in pages of type P_BRANCH and P_LEAF without P_LEAF2. - * We guarantee 2-byte alignment for 'MDBX_node's. - * - * Leaf node flags describe node contents. F_BIGDATA says the node's - * data part is the page number of an overflow page with actual data. - * F_DUPDATA and F_SUBDATA can be combined giving duplicate data in - * a sub-page/sub-database, and named databases (just F_SUBDATA). */ -typedef struct MDBX_node { -#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ - union { - uint32_t mn_dsize; - uint32_t mn_pgno32; - }; - uint8_t mn_flags; /* see mdbx_node flags */ - uint8_t mn_extra; - uint16_t mn_ksize; /* key size */ -#else - uint16_t mn_ksize; /* key size */ - uint8_t mn_extra; - uint8_t mn_flags; /* see mdbx_node flags */ - union { - uint32_t mn_pgno32; - uint32_t mn_dsize; - }; -#endif /* __BYTE_ORDER__ */ +MDBX_INTERNAL void pnl_sort_nochk(pnl_t pnl); - /* mdbx_node Flags */ -#define F_BIGDATA 0x01 /* data put on overflow page */ -#define F_SUBDATA 0x02 /* data is a sub-database */ -#define F_DUPDATA 0x04 /* data has duplicates */ +MDBX_INTERNAL bool pnl_check(const const_pnl_t pnl, const size_t limit); - /* valid flags for mdbx_node_add() */ -#define NODE_ADD_FLAGS (F_DUPDATA | F_SUBDATA | MDBX_RESERVE | MDBX_APPEND) +MDBX_MAYBE_UNUSED static inline bool pnl_check_allocated(const const_pnl_t pnl, const size_t limit) { + return pnl == nullptr || (MDBX_PNL_ALLOCLEN(pnl) >= MDBX_PNL_GETSIZE(pnl) && pnl_check(pnl, limit)); +} -#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ - (!defined(__cplusplus) && defined(_MSC_VER)) - uint8_t mn_data[] /* key and data are appended here */; -#endif /* C99 */ -} MDBX_node; +MDBX_MAYBE_UNUSED static inline void pnl_sort(pnl_t pnl, size_t limit4check) { + pnl_sort_nochk(pnl); + assert(pnl_check(pnl, limit4check)); + (void)limit4check; +} -#define DB_PERSISTENT_FLAGS \ - (MDBX_REVERSEKEY | MDBX_DUPSORT | MDBX_INTEGERKEY | MDBX_DUPFIXED | \ - MDBX_INTEGERDUP | MDBX_REVERSEDUP) +MDBX_MAYBE_UNUSED static inline size_t pnl_search(const pnl_t pnl, pgno_t pgno, size_t limit) { + assert(pnl_check_allocated(pnl, limit)); + if (MDBX_HAVE_CMOV) { + /* cmov-ускоренный бинарный поиск может читать (но не использовать) один + * элемент за концом данных, этот элемент в пределах выделенного участка + * памяти, но не инициализирован. */ + VALGRIND_MAKE_MEM_DEFINED(MDBX_PNL_END(pnl), sizeof(pgno_t)); + } + assert(pgno < limit); + (void)limit; + size_t n = pnl_search_nochk(pnl, pgno); + if (MDBX_HAVE_CMOV) { + VALGRIND_MAKE_MEM_UNDEFINED(MDBX_PNL_END(pnl), sizeof(pgno_t)); + } + return n; +} -/* mdbx_dbi_open() flags */ -#define DB_USABLE_FLAGS (DB_PERSISTENT_FLAGS | MDBX_CREATE | MDBX_DB_ACCEDE) +MDBX_INTERNAL size_t pnl_merge(pnl_t dst, const pnl_t src); -#define DB_VALID 0x8000 /* DB handle is valid, for me_dbflags */ -#define DB_INTERNAL_FLAGS DB_VALID +#ifdef __cplusplus +} +#endif /* __cplusplus */ -#if DB_INTERNAL_FLAGS & DB_USABLE_FLAGS -#error "Oops, some flags overlapped or wrong" -#endif -#if DB_PERSISTENT_FLAGS & ~DB_USABLE_FLAGS -#error "Oops, some flags overlapped or wrong" +#define mdbx_sourcery_anchor XCONCAT(mdbx_sourcery_, MDBX_BUILD_SOURCERY) +#if defined(xMDBX_TOOLS) +extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #endif -/* Max length of iov-vector passed to writev() call, used for auxilary writes */ -#define MDBX_AUXILARY_IOV_MAX 64 -#if defined(IOV_MAX) && IOV_MAX < MDBX_AUXILARY_IOV_MAX -#undef MDBX_AUXILARY_IOV_MAX -#define MDBX_AUXILARY_IOV_MAX IOV_MAX -#endif /* MDBX_AUXILARY_IOV_MAX */ +#define MDBX_IS_ERROR(rc) ((rc) != MDBX_RESULT_TRUE && (rc) != MDBX_RESULT_FALSE) -/* - * / - * | -1, a < b - * CMP2INT(a,b) = < 0, a == b - * | 1, a > b - * \ - */ -#define CMP2INT(a, b) (((a) != (b)) ? (((a) < (b)) ? -1 : 1) : 0) +/*----------------------------------------------------------------------------*/ -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline pgno_t -int64pgno(int64_t i64) { +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline pgno_t int64pgno(int64_t i64) { if (likely(i64 >= (int64_t)MIN_PAGENO && i64 <= (int64_t)MAX_PAGENO + 1)) return (pgno_t)i64; return (i64 < (int64_t)MIN_PAGENO) ? MIN_PAGENO : MAX_PAGENO; } -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline pgno_t -pgno_add(size_t base, size_t augend) { +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline pgno_t pgno_add(size_t base, size_t augend) { assert(base <= MAX_PAGENO + 1 && augend < MAX_PAGENO); return int64pgno((int64_t)base + (int64_t)augend); } -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline pgno_t -pgno_sub(size_t base, size_t subtrahend) { - assert(base >= MIN_PAGENO && base <= MAX_PAGENO + 1 && - subtrahend < MAX_PAGENO); +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline pgno_t pgno_sub(size_t base, size_t subtrahend) { + assert(base >= MIN_PAGENO && base <= MAX_PAGENO + 1 && subtrahend < MAX_PAGENO); return int64pgno((int64_t)base - (int64_t)subtrahend); } -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __always_inline bool -is_powerof2(size_t x) { - return (x & (x - 1)) == 0; -} - -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __always_inline size_t -floor_powerof2(size_t value, size_t granularity) { - assert(is_powerof2(granularity)); - return value & ~(granularity - 1); -} - -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __always_inline size_t -ceil_powerof2(size_t value, size_t granularity) { - return floor_powerof2(value + granularity - 1, granularity); -} - -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static unsigned -log2n_powerof2(size_t value_uintptr) { - assert(value_uintptr > 0 && value_uintptr < INT32_MAX && - is_powerof2(value_uintptr)); - assert((value_uintptr & -(intptr_t)value_uintptr) == value_uintptr); - const uint32_t value_uint32 = (uint32_t)value_uintptr; -#if __GNUC_PREREQ(4, 1) || __has_builtin(__builtin_ctz) - STATIC_ASSERT(sizeof(value_uint32) <= sizeof(unsigned)); - return __builtin_ctz(value_uint32); -#elif defined(_MSC_VER) - unsigned long index; - STATIC_ASSERT(sizeof(value_uint32) <= sizeof(long)); - _BitScanForward(&index, value_uint32); - return index; -#else - static const uint8_t debruijn_ctz32[32] = { - 0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8, - 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9}; - return debruijn_ctz32[(uint32_t)(value_uint32 * 0x077CB531ul) >> 27]; -#endif -} - -/* Only a subset of the mdbx_env flags can be changed - * at runtime. Changing other flags requires closing the - * environment and re-opening it with the new flags. */ -#define ENV_CHANGEABLE_FLAGS \ - (MDBX_SAFE_NOSYNC | MDBX_NOMETASYNC | MDBX_DEPRECATED_MAPASYNC | \ - MDBX_NOMEMINIT | MDBX_COALESCE | MDBX_PAGEPERTURB | MDBX_ACCEDE | \ - MDBX_VALIDATION) -#define ENV_CHANGELESS_FLAGS \ - (MDBX_NOSUBDIR | MDBX_RDONLY | MDBX_WRITEMAP | MDBX_NOTLS | MDBX_NORDAHEAD | \ - MDBX_LIFORECLAIM | MDBX_EXCLUSIVE) -#define ENV_USABLE_FLAGS (ENV_CHANGEABLE_FLAGS | ENV_CHANGELESS_FLAGS) - -#if !defined(__cplusplus) || CONSTEXPR_ENUM_FLAGS_OPERATIONS -MDBX_MAYBE_UNUSED static void static_checks(void) { - STATIC_ASSERT_MSG(INT16_MAX - CORE_DBS == MDBX_MAX_DBI, - "Oops, MDBX_MAX_DBI or CORE_DBS?"); - STATIC_ASSERT_MSG((unsigned)(MDBX_DB_ACCEDE | MDBX_CREATE) == - ((DB_USABLE_FLAGS | DB_INTERNAL_FLAGS) & - (ENV_USABLE_FLAGS | ENV_INTERNAL_FLAGS)), - "Oops, some flags overlapped or wrong"); - STATIC_ASSERT_MSG((ENV_INTERNAL_FLAGS & ENV_USABLE_FLAGS) == 0, - "Oops, some flags overlapped or wrong"); -} -#endif /* Disabled for MSVC 19.0 (VisualStudio 2015) */ - -#ifdef __cplusplus -} -#endif - -#define MDBX_ASAN_POISON_MEMORY_REGION(addr, size) \ - do { \ - TRACE("POISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), \ - (size_t)(size), __LINE__); \ - ASAN_POISON_MEMORY_REGION(addr, size); \ - } while (0) - -#define MDBX_ASAN_UNPOISON_MEMORY_REGION(addr, size) \ - do { \ - TRACE("UNPOISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), \ - (size_t)(size), __LINE__); \ - ASAN_UNPOISON_MEMORY_REGION(addr, size); \ - } while (0) - #include #if defined(_WIN32) || defined(_WIN64) @@ -4087,12 +3244,12 @@ MDBX_MAYBE_UNUSED static void static_checks(void) { #ifdef _MSC_VER #pragma warning(push, 1) -#pragma warning(disable : 4548) /* expression before comma has no effect; \ +#pragma warning(disable : 4548) /* expression before comma has no effect; \ expected expression with side - effect */ -#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ +#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ * semantics are not enabled. Specify /EHsc */ -#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ - * mode specified; termination on exception is \ +#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ + * mode specified; termination on exception is \ * not guaranteed. Specify /EHsc */ #if !defined(_CRT_SECURE_NO_WARNINGS) #define _CRT_SECURE_NO_WARNINGS @@ -4145,8 +3302,7 @@ int getopt(int argc, char *const argv[], const char *opts) { if (argv[optind][sp + 1] != '\0') optarg = &argv[optind++][sp + 1]; else if (++optind >= argc) { - fprintf(stderr, "%s: %s -- %c\n", argv[0], "option requires an argument", - c); + fprintf(stderr, "%s: %s -- %c\n", argv[0], "option requires an argument", c); sp = 1; return '?'; } else @@ -4185,11 +3341,25 @@ static size_t lineno; static void error(const char *func, int rc) { if (!quiet) { if (lineno) - fprintf(stderr, "%s: at input line %" PRIiSIZE ": %s() error %d, %s\n", - prog, lineno, func, rc, mdbx_strerror(rc)); - else - fprintf(stderr, "%s: %s() error %d %s\n", prog, func, rc, + fprintf(stderr, "%s: at input line %" PRIiSIZE ": %s() error %d, %s\n", prog, lineno, func, rc, mdbx_strerror(rc)); + else + fprintf(stderr, "%s: %s() error %d %s\n", prog, func, rc, mdbx_strerror(rc)); + } +} + +static void logger(MDBX_log_level_t level, const char *function, int line, const char *fmt, va_list args) { + static const char *const prefixes[] = { + "!!!fatal: ", // 0 fatal + " ! ", // 1 error + " ~ ", // 2 warning + " ", // 3 notice + " //", // 4 verbose + }; + if (level < MDBX_LOG_DEBUG) { + if (function && line) + fprintf(stderr, "%s", prefixes[level]); + vfprintf(stderr, fmt, args); } } @@ -4201,9 +3371,7 @@ static char *valstr(char *line, const char *item) { if (line[len] > ' ') return nullptr; if (!quiet) - fprintf(stderr, - "%s: line %" PRIiSIZE ": unexpected line format for '%s'\n", prog, - lineno, item); + fprintf(stderr, "%s: line %" PRIiSIZE ": unexpected line format for '%s'\n", prog, lineno, item); exit(EXIT_FAILURE); } char *ptr = strchr(line, '\n'); @@ -4221,9 +3389,7 @@ static bool valnum(char *line, const char *item, uint64_t *value) { *value = strtoull(str, &end, 0); if (end && *end) { if (!quiet) - fprintf(stderr, - "%s: line %" PRIiSIZE ": unexpected number format for '%s'\n", - prog, lineno, item); + fprintf(stderr, "%s: line %" PRIiSIZE ": unexpected number format for '%s'\n", prog, lineno, item); exit(EXIT_FAILURE); } return true; @@ -4236,8 +3402,7 @@ static bool valbool(char *line, const char *item, bool *value) { if (u64 > 1) { if (!quiet) - fprintf(stderr, "%s: line %" PRIiSIZE ": unexpected value for '%s'\n", - prog, lineno, item); + fprintf(stderr, "%s: line %" PRIiSIZE ": unexpected value for '%s'\n", prog, lineno, item); exit(EXIT_FAILURE); } *value = u64 != 0; @@ -4270,11 +3435,10 @@ typedef struct flagbit { #define S(s) STRLENOF(s), s -flagbit dbflags[] = { - {MDBX_REVERSEKEY, S("reversekey")}, {MDBX_DUPSORT, S("duplicates")}, - {MDBX_DUPSORT, S("dupsort")}, {MDBX_INTEGERKEY, S("integerkey")}, - {MDBX_DUPFIXED, S("dupfixed")}, {MDBX_INTEGERDUP, S("integerdup")}, - {MDBX_REVERSEDUP, S("reversedup")}, {0, 0, nullptr}}; +flagbit dbflags[] = {{MDBX_REVERSEKEY, S("reversekey")}, {MDBX_DUPSORT, S("duplicates")}, + {MDBX_DUPSORT, S("dupsort")}, {MDBX_INTEGERKEY, S("integerkey")}, + {MDBX_DUPFIXED, S("dupfix")}, {MDBX_INTEGERDUP, S("integerdup")}, + {MDBX_REVERSEDUP, S("reversedup")}, {0, 0, nullptr}}; static int readhdr(void) { /* reset parameters */ @@ -4299,10 +3463,8 @@ static int readhdr(void) { if (valnum(dbuf.iov_base, "VERSION", &u64)) { if (u64 != 3) { if (!quiet) - fprintf(stderr, - "%s: line %" PRIiSIZE ": unsupported value %" PRIu64 - " for %s\n", - prog, lineno, u64, "VERSION"); + fprintf(stderr, "%s: line %" PRIiSIZE ": unsupported value %" PRIu64 " for %s\n", prog, lineno, u64, + "VERSION"); exit(EXIT_FAILURE); } continue; @@ -4311,16 +3473,12 @@ static int readhdr(void) { if (valnum(dbuf.iov_base, "db_pagesize", &u64)) { if (!(mode & GLOBAL) && envinfo.mi_dxb_pagesize != u64) { if (!quiet) - fprintf(stderr, - "%s: line %" PRIiSIZE ": ignore value %" PRIu64 - " for '%s' in non-global context\n", - prog, lineno, u64, "db_pagesize"); + fprintf(stderr, "%s: line %" PRIiSIZE ": ignore value %" PRIu64 " for '%s' in non-global context\n", prog, + lineno, u64, "db_pagesize"); } else if (u64 < MDBX_MIN_PAGESIZE || u64 > MDBX_MAX_PAGESIZE) { if (!quiet) - fprintf(stderr, - "%s: line %" PRIiSIZE ": ignore unsupported value %" PRIu64 - " for %s\n", - prog, lineno, u64, "db_pagesize"); + fprintf(stderr, "%s: line %" PRIiSIZE ": ignore unsupported value %" PRIu64 " for %s\n", prog, lineno, u64, + "db_pagesize"); } else envinfo.mi_dxb_pagesize = (uint32_t)u64; continue; @@ -4337,9 +3495,7 @@ static int readhdr(void) { continue; } if (!quiet) - fprintf(stderr, - "%s: line %" PRIiSIZE ": unsupported value '%s' for %s\n", prog, - lineno, str, "format"); + fprintf(stderr, "%s: line %" PRIiSIZE ": unsupported value '%s' for %s\n", prog, lineno, str, "format"); exit(EXIT_FAILURE); } @@ -4361,9 +3517,7 @@ static int readhdr(void) { if (str) { if (strcmp(str, "btree") != 0) { if (!quiet) - fprintf(stderr, - "%s: line %" PRIiSIZE ": unsupported value '%s' for %s\n", - prog, lineno, str, "type"); + fprintf(stderr, "%s: line %" PRIiSIZE ": unsupported value '%s' for %s\n", prog, lineno, str, "type"); free(subname); exit(EXIT_FAILURE); } @@ -4373,10 +3527,8 @@ static int readhdr(void) { if (valnum(dbuf.iov_base, "mapaddr", &u64)) { if (u64) { if (!quiet) - fprintf(stderr, - "%s: line %" PRIiSIZE ": ignore unsupported value 0x%" PRIx64 - " for %s\n", - prog, lineno, u64, "mapaddr"); + fprintf(stderr, "%s: line %" PRIiSIZE ": ignore unsupported value 0x%" PRIx64 " for %s\n", prog, lineno, u64, + "mapaddr"); } continue; } @@ -4384,16 +3536,12 @@ static int readhdr(void) { if (valnum(dbuf.iov_base, "mapsize", &u64)) { if (!(mode & GLOBAL)) { if (!quiet) - fprintf(stderr, - "%s: line %" PRIiSIZE ": ignore value %" PRIu64 - " for '%s' in non-global context\n", - prog, lineno, u64, "mapsize"); + fprintf(stderr, "%s: line %" PRIiSIZE ": ignore value %" PRIu64 " for '%s' in non-global context\n", prog, + lineno, u64, "mapsize"); } else if (u64 < MIN_MAPSIZE || u64 > MAX_MAPSIZE64) { if (!quiet) - fprintf(stderr, - "%s: line %" PRIiSIZE ": ignore unsupported value 0x%" PRIx64 - " for %s\n", - prog, lineno, u64, "mapsize"); + fprintf(stderr, "%s: line %" PRIiSIZE ": ignore unsupported value 0x%" PRIx64 " for %s\n", prog, lineno, u64, + "mapsize"); } else envinfo.mi_mapsize = (size_t)u64; continue; @@ -4402,16 +3550,12 @@ static int readhdr(void) { if (valnum(dbuf.iov_base, "maxreaders", &u64)) { if (!(mode & GLOBAL)) { if (!quiet) - fprintf(stderr, - "%s: line %" PRIiSIZE ": ignore value %" PRIu64 - " for '%s' in non-global context\n", - prog, lineno, u64, "maxreaders"); + fprintf(stderr, "%s: line %" PRIiSIZE ": ignore value %" PRIu64 " for '%s' in non-global context\n", prog, + lineno, u64, "maxreaders"); } else if (u64 < 1 || u64 > MDBX_READERS_LIMIT) { if (!quiet) - fprintf(stderr, - "%s: line %" PRIiSIZE ": ignore unsupported value 0x%" PRIx64 - " for %s\n", - prog, lineno, u64, "maxreaders"); + fprintf(stderr, "%s: line %" PRIiSIZE ": ignore unsupported value 0x%" PRIx64 " for %s\n", prog, lineno, u64, + "maxreaders"); } else envinfo.mi_maxreaders = (int)u64; continue; @@ -4420,10 +3564,8 @@ static int readhdr(void) { if (valnum(dbuf.iov_base, "txnid", &u64)) { if (u64 < MIN_TXNID || u64 > MAX_TXNID) { if (!quiet) - fprintf(stderr, - "%s: line %" PRIiSIZE ": ignore unsupported value 0x%" PRIx64 - " for %s\n", - prog, lineno, u64, "txnid"); + fprintf(stderr, "%s: line %" PRIiSIZE ": ignore unsupported value 0x%" PRIx64 " for %s\n", prog, lineno, u64, + "txnid"); } else txnid = u64; continue; @@ -4442,16 +3584,11 @@ static int readhdr(void) { "%s: line %" PRIiSIZE ": ignore values %s" " for '%s' in non-global context\n", prog, lineno, str, "geometry"); - } else if (sscanf(str, - "l%" PRIu64 ",c%" PRIu64 ",u%" PRIu64 ",s%" PRIu64 - ",g%" PRIu64, - &envinfo.mi_geo.lower, &envinfo.mi_geo.current, - &envinfo.mi_geo.upper, &envinfo.mi_geo.shrink, + } else if (sscanf(str, "l%" PRIu64 ",c%" PRIu64 ",u%" PRIu64 ",s%" PRIu64 ",g%" PRIu64, &envinfo.mi_geo.lower, + &envinfo.mi_geo.current, &envinfo.mi_geo.upper, &envinfo.mi_geo.shrink, &envinfo.mi_geo.grow) != 5) { if (!quiet) - fprintf(stderr, - "%s: line %" PRIiSIZE ": unexpected line format for '%s'\n", - prog, lineno, "geometry"); + fprintf(stderr, "%s: line %" PRIiSIZE ": unexpected line format for '%s'\n", prog, lineno, "geometry"); exit(EXIT_FAILURE); } continue; @@ -4465,12 +3602,10 @@ static int readhdr(void) { "%s: line %" PRIiSIZE ": ignore values %s" " for '%s' in non-global context\n", prog, lineno, str, "canary"); - } else if (sscanf(str, "v%" PRIu64 ",x%" PRIu64 ",y%" PRIu64 ",z%" PRIu64, - &canary.v, &canary.x, &canary.y, &canary.z) != 4) { + } else if (sscanf(str, "v%" PRIu64 ",x%" PRIu64 ",y%" PRIu64 ",z%" PRIu64, &canary.v, &canary.x, &canary.y, + &canary.z) != 4) { if (!quiet) - fprintf(stderr, - "%s: line %" PRIiSIZE ": unexpected line format for '%s'\n", - prog, lineno, "canary"); + fprintf(stderr, "%s: line %" PRIiSIZE ": unexpected line format for '%s'\n", prog, lineno, "canary"); exit(EXIT_FAILURE); } continue; @@ -4494,9 +3629,8 @@ static int readhdr(void) { } if (!quiet) - fprintf(stderr, - "%s: line %" PRIiSIZE ": unrecognized keyword ignored: %s\n", - prog, lineno, (char *)dbuf.iov_base); + fprintf(stderr, "%s: line %" PRIiSIZE ": unrecognized keyword ignored: %s\n", prog, lineno, + (char *)dbuf.iov_base); next:; } return EOF; @@ -4504,22 +3638,20 @@ static int readhdr(void) { static int badend(void) { if (!quiet) - fprintf(stderr, "%s: line %" PRIiSIZE ": unexpected end of input\n", prog, - lineno); + fprintf(stderr, "%s: line %" PRIiSIZE ": unexpected end of input\n", prog, lineno); return errno ? errno : MDBX_ENODATA; } -static __inline int unhex(unsigned char *c2) { - int x, c; - x = *c2++ & 0x4f; - if (x & 0x40) - x -= 55; - c = x << 4; - x = *c2 & 0x4f; - if (x & 0x40) - x -= 55; - c |= x; - return c; +static inline int unhex(unsigned char *c2) { + int8_t hi = c2[0]; + hi = (hi | 0x20) - 'a'; + hi += 10 + ((hi >> 7) & 39); + + int8_t lo = c2[1]; + lo = (lo | 0x20) - 'a'; + lo += 10 + ((lo >> 7) & 39); + + return hi << 4 | lo; } __hot static int readline(MDBX_val *out, MDBX_val *buf) { @@ -4558,9 +3690,7 @@ __hot static int readline(MDBX_val *out, MDBX_val *buf) { buf->iov_base = osal_realloc(buf->iov_base, buf->iov_len * 2); if (!buf->iov_base) { if (!quiet) - fprintf(stderr, - "%s: line %" PRIiSIZE ": out of memory, line too long\n", prog, - lineno); + fprintf(stderr, "%s: line %" PRIiSIZE ": out of memory, line too long\n", prog, lineno); return MDBX_ENOMEM; } c1 = buf->iov_base; @@ -4619,10 +3749,10 @@ static void usage(void) { " -a\t\tappend records in input order (required for custom " "comparators)\n" " -f file\tread from file instead of stdin\n" - " -s name\tload into specified named subDB\n" + " -s name\tload into specified named table\n" " -N\t\tdon't overwrite existing records when loading, just skip " "ones\n" - " -p\t\tpurge subDB before loading\n" + " -p\t\tpurge table before loading\n" " -T\t\tread plaintext\n" " -r\t\trescue mode (ignore errors to load corrupted DB dump)\n" " -n\t\tdon't use subdirectory for newly created database " @@ -4632,14 +3762,11 @@ static void usage(void) { } static int equal_or_greater(const MDBX_val *a, const MDBX_val *b) { - return (a->iov_len == b->iov_len && - memcmp(a->iov_base, b->iov_base, a->iov_len) == 0) - ? 0 - : 1; + return (a->iov_len == b->iov_len && memcmp(a->iov_base, b->iov_base, a->iov_len) == 0) ? 0 : 1; } int main(int argc, char *argv[]) { - int i, rc; + int i, err; MDBX_env *env = nullptr; MDBX_txn *txn = nullptr; MDBX_cursor *mc = nullptr; @@ -4672,12 +3799,9 @@ int main(int argc, char *argv[]) { " - build: %s for %s by %s\n" " - flags: %s\n" " - options: %s\n", - mdbx_version.major, mdbx_version.minor, mdbx_version.release, - mdbx_version.revision, mdbx_version.git.describe, - mdbx_version.git.datetime, mdbx_version.git.commit, - mdbx_version.git.tree, mdbx_sourcery_anchor, mdbx_build.datetime, - mdbx_build.target, mdbx_build.compiler, mdbx_build.flags, - mdbx_build.options); + mdbx_version.major, mdbx_version.minor, mdbx_version.patch, mdbx_version.tweak, mdbx_version.git.describe, + mdbx_version.git.datetime, mdbx_version.git.commit, mdbx_version.git.tree, mdbx_sourcery_anchor, + mdbx_build.datetime, mdbx_build.target, mdbx_build.compiler, mdbx_build.flags, mdbx_build.options); return EXIT_SUCCESS; case 'a': putflags |= MDBX_APPEND; @@ -4685,8 +3809,7 @@ int main(int argc, char *argv[]) { case 'f': if (freopen(optarg, "r", stdin) == nullptr) { if (!quiet) - fprintf(stderr, "%s: %s: open: %s\n", prog, optarg, - mdbx_strerror(errno)); + fprintf(stderr, "%s: %s: open: %s\n", prog, optarg, mdbx_strerror(errno)); exit(EXIT_FAILURE); } break; @@ -4733,242 +3856,238 @@ int main(int argc, char *argv[]) { #endif /* !WINDOWS */ envname = argv[optind]; - if (!quiet) - printf("mdbx_load %s (%s, T-%s)\nRunning for %s...\n", - mdbx_version.git.describe, mdbx_version.git.datetime, + if (!quiet) { + printf("mdbx_load %s (%s, T-%s)\nRunning for %s...\n", mdbx_version.git.describe, mdbx_version.git.datetime, mdbx_version.git.tree, envname); - fflush(nullptr); + fflush(nullptr); + mdbx_setup_debug(MDBX_LOG_NOTICE, MDBX_DBG_DONTCHANGE, logger); + } dbuf.iov_len = 4096; dbuf.iov_base = osal_malloc(dbuf.iov_len); if (!dbuf.iov_base) { - rc = MDBX_ENOMEM; - error("value-buffer", rc); - goto env_close; + err = MDBX_ENOMEM; + error("value-buffer", err); + goto bailout; } /* read first header for mapsize= */ if (!(mode & NOHDR)) { - rc = readhdr(); - if (unlikely(rc != MDBX_SUCCESS)) { - if (rc == EOF) - rc = MDBX_ENODATA; - error("readheader", rc); - goto env_close; + err = readhdr(); + if (unlikely(err != MDBX_SUCCESS)) { + if (err == EOF) + err = MDBX_ENODATA; + error("readheader", err); + goto bailout; } } - rc = mdbx_env_create(&env); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_env_create", rc); - return EXIT_FAILURE; + err = mdbx_env_create(&env); + if (unlikely(err != MDBX_SUCCESS)) { + error("mdbx_env_create", err); + goto bailout; + } + + err = mdbx_env_set_maxdbs(env, 2); + if (unlikely(err != MDBX_SUCCESS)) { + error("mdbx_env_set_maxdbs", err); + goto bailout; } - mdbx_env_set_maxdbs(env, 2); if (envinfo.mi_maxreaders) { - rc = mdbx_env_set_maxreaders(env, envinfo.mi_maxreaders); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_env_set_maxreaders", rc); - goto env_close; + err = mdbx_env_set_maxreaders(env, envinfo.mi_maxreaders); + if (unlikely(err != MDBX_SUCCESS)) { + error("mdbx_env_set_maxreaders", err); + goto bailout; } } if (envinfo.mi_geo.current | envinfo.mi_mapsize) { if (envinfo.mi_geo.current) { - rc = mdbx_env_set_geometry( - env, (intptr_t)envinfo.mi_geo.lower, (intptr_t)envinfo.mi_geo.current, - (intptr_t)envinfo.mi_geo.upper, (intptr_t)envinfo.mi_geo.shrink, - (intptr_t)envinfo.mi_geo.grow, - envinfo.mi_dxb_pagesize ? (intptr_t)envinfo.mi_dxb_pagesize : -1); + err = mdbx_env_set_geometry(env, (intptr_t)envinfo.mi_geo.lower, (intptr_t)envinfo.mi_geo.current, + (intptr_t)envinfo.mi_geo.upper, (intptr_t)envinfo.mi_geo.shrink, + (intptr_t)envinfo.mi_geo.grow, + envinfo.mi_dxb_pagesize ? (intptr_t)envinfo.mi_dxb_pagesize : -1); } else { if (envinfo.mi_mapsize > MAX_MAPSIZE) { if (!quiet) - fprintf( - stderr, - "Database size is too large for current system (mapsize=%" PRIu64 - " is great than system-limit %zu)\n", - envinfo.mi_mapsize, (size_t)MAX_MAPSIZE); - goto env_close; + fprintf(stderr, + "Database size is too large for current system (mapsize=%" PRIu64 + " is great than system-limit %zu)\n", + envinfo.mi_mapsize, (size_t)MAX_MAPSIZE); + goto bailout; } - rc = mdbx_env_set_geometry( - env, (intptr_t)envinfo.mi_mapsize, (intptr_t)envinfo.mi_mapsize, - (intptr_t)envinfo.mi_mapsize, 0, 0, - envinfo.mi_dxb_pagesize ? (intptr_t)envinfo.mi_dxb_pagesize : -1); + err = mdbx_env_set_geometry(env, (intptr_t)envinfo.mi_mapsize, (intptr_t)envinfo.mi_mapsize, + (intptr_t)envinfo.mi_mapsize, 0, 0, + envinfo.mi_dxb_pagesize ? (intptr_t)envinfo.mi_dxb_pagesize : -1); } - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_env_set_geometry", rc); - goto env_close; + if (unlikely(err != MDBX_SUCCESS)) { + error("mdbx_env_set_geometry", err); + goto bailout; } } - rc = mdbx_env_open(env, envname, envflags, 0664); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_env_open", rc); - goto env_close; + err = mdbx_env_open(env, envname, envflags, 0664); + if (unlikely(err != MDBX_SUCCESS)) { + error("mdbx_env_open", err); + goto bailout; } kbuf.iov_len = mdbx_env_get_maxvalsize_ex(env, 0) + (size_t)1; if (kbuf.iov_len >= INTPTR_MAX / 2) { if (!quiet) - fprintf(stderr, "mdbx_env_get_maxkeysize() failed, returns %zu\n", - kbuf.iov_len); - goto env_close; + fprintf(stderr, "mdbx_env_get_maxkeysize() failed, returns %zu\n", kbuf.iov_len); + goto bailout; } kbuf.iov_base = malloc(kbuf.iov_len); if (!kbuf.iov_base) { - rc = MDBX_ENOMEM; - error("key-buffer", rc); - goto env_close; + err = MDBX_ENOMEM; + error("key-buffer", err); + goto bailout; } - while (rc == MDBX_SUCCESS) { + while (err == MDBX_SUCCESS) { if (user_break) { - rc = MDBX_EINTR; + err = MDBX_EINTR; break; } - rc = mdbx_txn_begin(env, nullptr, 0, &txn); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_txn_begin", rc); - goto env_close; + err = mdbx_txn_begin(env, nullptr, 0, &txn); + if (unlikely(err != MDBX_SUCCESS)) { + error("mdbx_txn_begin", err); + goto bailout; } if (mode & GLOBAL) { mode -= GLOBAL; if (canary.v | canary.x | canary.y | canary.z) { - rc = mdbx_canary_put(txn, &canary); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_canary_put", rc); - goto txn_abort; + err = mdbx_canary_put(txn, &canary); + if (unlikely(err != MDBX_SUCCESS)) { + error("mdbx_canary_put", err); + goto bailout; } } } const char *const dbi_name = subname ? subname : "@MAIN"; - rc = - mdbx_dbi_open_ex(txn, subname, dbi_flags | MDBX_CREATE, &dbi, - (putflags & MDBX_APPEND) ? equal_or_greater : nullptr, - (putflags & MDBX_APPEND) ? equal_or_greater : nullptr); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_dbi_open_ex", rc); - goto txn_abort; + err = mdbx_dbi_open_ex(txn, subname, dbi_flags | MDBX_CREATE, &dbi, + (putflags & MDBX_APPEND) ? equal_or_greater : nullptr, + (putflags & MDBX_APPEND) ? equal_or_greater : nullptr); + if (unlikely(err != MDBX_SUCCESS)) { + error("mdbx_dbi_open_ex", err); + goto bailout; } uint64_t present_sequence; - rc = mdbx_dbi_sequence(txn, dbi, &present_sequence, 0); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_dbi_sequence", rc); - goto txn_abort; + err = mdbx_dbi_sequence(txn, dbi, &present_sequence, 0); + if (unlikely(err != MDBX_SUCCESS)) { + error("mdbx_dbi_sequence", err); + goto bailout; } if (present_sequence > sequence) { if (!quiet) - fprintf(stderr, - "present sequence for '%s' value (%" PRIu64 - ") is greater than loaded (%" PRIu64 ")\n", + fprintf(stderr, "present sequence for '%s' value (%" PRIu64 ") is greater than loaded (%" PRIu64 ")\n", dbi_name, present_sequence, sequence); - rc = MDBX_RESULT_TRUE; - goto txn_abort; + err = MDBX_RESULT_TRUE; + goto bailout; } if (present_sequence < sequence) { - rc = mdbx_dbi_sequence(txn, dbi, nullptr, sequence - present_sequence); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_dbi_sequence", rc); - goto txn_abort; + err = mdbx_dbi_sequence(txn, dbi, nullptr, sequence - present_sequence); + if (unlikely(err != MDBX_SUCCESS)) { + error("mdbx_dbi_sequence", err); + goto bailout; } } if (purge) { - rc = mdbx_drop(txn, dbi, false); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_drop", rc); - goto txn_abort; + err = mdbx_drop(txn, dbi, false); + if (unlikely(err != MDBX_SUCCESS)) { + error("mdbx_drop", err); + goto bailout; } } if (putflags & MDBX_APPEND) - putflags = (dbi_flags & MDBX_DUPSORT) ? putflags | MDBX_APPENDDUP - : putflags & ~MDBX_APPENDDUP; + putflags = (dbi_flags & MDBX_DUPSORT) ? putflags | MDBX_APPENDDUP : putflags & ~MDBX_APPENDDUP; - rc = mdbx_cursor_open(txn, dbi, &mc); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_cursor_open", rc); - goto txn_abort; + err = mdbx_cursor_open(txn, dbi, &mc); + if (unlikely(err != MDBX_SUCCESS)) { + error("mdbx_cursor_open", err); + goto bailout; } int batch = 0; - while (rc == MDBX_SUCCESS) { + while (err == MDBX_SUCCESS) { MDBX_val key, data; - rc = readline(&key, &kbuf); - if (rc == EOF) + err = readline(&key, &kbuf); + if (err == EOF) break; - if (rc == MDBX_SUCCESS) - rc = readline(&data, &dbuf); - if (rc) { + if (err == MDBX_SUCCESS) + err = readline(&data, &dbuf); + if (err) { if (!quiet) - fprintf(stderr, "%s: line %" PRIiSIZE ": failed to read key value\n", - prog, lineno); - goto txn_abort; + fprintf(stderr, "%s: line %" PRIiSIZE ": failed to read key value\n", prog, lineno); + goto bailout; } - rc = mdbx_cursor_put(mc, &key, &data, putflags); - if (rc == MDBX_KEYEXIST && putflags) + err = mdbx_cursor_put(mc, &key, &data, putflags); + if (err == MDBX_KEYEXIST && putflags) continue; - if (rc == MDBX_BAD_VALSIZE && rescue) { + if (err == MDBX_BAD_VALSIZE && rescue) { if (!quiet) - fprintf(stderr, "%s: skip line %" PRIiSIZE ": due %s\n", prog, lineno, - mdbx_strerror(rc)); + fprintf(stderr, "%s: skip line %" PRIiSIZE ": due %s\n", prog, lineno, mdbx_strerror(err)); continue; } - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_cursor_put", rc); - goto txn_abort; + if (unlikely(err != MDBX_SUCCESS)) { + error("mdbx_cursor_put", err); + goto bailout; } batch++; MDBX_txn_info txn_info; - rc = mdbx_txn_info(txn, &txn_info, false); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_txn_info", rc); - goto txn_abort; + err = mdbx_txn_info(txn, &txn_info, false); + if (unlikely(err != MDBX_SUCCESS)) { + error("mdbx_txn_info", err); + goto bailout; } if (batch == 10000 || txn_info.txn_space_dirty > MEGABYTE * 256) { - rc = mdbx_txn_commit(txn); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_txn_commit", rc); - goto env_close; + err = mdbx_txn_commit(txn); + if (unlikely(err != MDBX_SUCCESS)) { + error("mdbx_txn_commit", err); + goto bailout; } batch = 0; - rc = mdbx_txn_begin(env, nullptr, 0, &txn); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_txn_begin", rc); - goto env_close; + err = mdbx_txn_begin(env, nullptr, 0, &txn); + if (unlikely(err != MDBX_SUCCESS)) { + error("mdbx_txn_begin", err); + goto bailout; } - rc = mdbx_cursor_bind(txn, mc, dbi); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_cursor_bind", rc); - goto txn_abort; + err = mdbx_cursor_bind(txn, mc, dbi); + if (unlikely(err != MDBX_SUCCESS)) { + error("mdbx_cursor_bind", err); + goto bailout; } } } mdbx_cursor_close(mc); mc = nullptr; - rc = mdbx_txn_commit(txn); + err = mdbx_txn_commit(txn); txn = nullptr; - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_txn_commit", rc); - goto env_close; + if (unlikely(err != MDBX_SUCCESS)) { + error("mdbx_txn_commit", err); + goto bailout; } if (subname) { assert(dbi != MAIN_DBI); - rc = mdbx_dbi_close(env, dbi); - if (unlikely(rc != MDBX_SUCCESS)) { - error("mdbx_dbi_close", rc); - goto env_close; + err = mdbx_dbi_close(env, dbi); + if (unlikely(err != MDBX_SUCCESS)) { + error("mdbx_dbi_close", err); + goto bailout; } } else { assert(dbi == MAIN_DBI); @@ -4976,14 +4095,14 @@ int main(int argc, char *argv[]) { /* try read next header */ if (!(mode & NOHDR)) - rc = readhdr(); + err = readhdr(); else if (ferror(stdin) || feof(stdin)) break; } - switch (rc) { + switch (err) { case EOF: - rc = MDBX_SUCCESS; + err = MDBX_SUCCESS; case MDBX_SUCCESS: break; case MDBX_EINTR: @@ -4991,17 +4110,19 @@ int main(int argc, char *argv[]) { fprintf(stderr, "Interrupted by signal/user\n"); break; default: - if (unlikely(rc != MDBX_SUCCESS)) - error("readline", rc); + if (unlikely(err != MDBX_SUCCESS)) + error("readline", err); } -txn_abort: - mdbx_cursor_close(mc); - mdbx_txn_abort(txn); -env_close: - mdbx_env_close(env); +bailout: + if (mc) + mdbx_cursor_close(mc); + if (txn) + mdbx_txn_abort(txn); + if (env) + mdbx_env_close(env); free(kbuf.iov_base); free(dbuf.iov_base); - return rc ? EXIT_FAILURE : EXIT_SUCCESS; + return err ? EXIT_FAILURE : EXIT_SUCCESS; } diff --git a/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx_stat.c b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx_stat.c index af4bfcfe72d..bd052d70a3c 100644 --- a/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx_stat.c +++ b/crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx_stat.c @@ -1,18 +1,12 @@ -/* mdbx_stat.c - memory-mapped database status tool */ - -/* - * Copyright 2015-2024 Leonid Yuriev - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . */ - +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \note Please refer to the COPYRIGHT file for explanations license change, +/// credits and acknowledgments. +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 +/// +/// mdbx_stat.c - memory-mapped database status tool +/// + +/* clang-format off */ #ifdef _MSC_VER #if _MSC_VER > 1800 #pragma warning(disable : 4464) /* relative include path contains '..' */ @@ -21,38 +15,26 @@ #endif /* _MSC_VER (warnings) */ #define xMDBX_TOOLS /* Avoid using internal eASSERT() */ -/* - * Copyright 2015-2024 Leonid Yuriev - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . */ +/// \copyright SPDX-License-Identifier: Apache-2.0 +/// \author Леонид Юрьев aka Leonid Yuriev \date 2015-2025 -#define MDBX_BUILD_SOURCERY e156c1a97c017ce89d6541cd9464ae5a9761d76b3fd2f1696521f5f3792904fc_v0_12_13_0_g1fff1f67 -#ifdef MDBX_CONFIG_H -#include MDBX_CONFIG_H -#endif +#define MDBX_BUILD_SOURCERY 6b5df6869d2bf5419e3a8189d9cc849cc9911b9c8a951b9750ed0a261ce43724_v0_13_7_0_g566b0f93 #define LIBMDBX_INTERNALS -#ifdef xMDBX_TOOLS #define MDBX_DEPRECATED -#endif /* xMDBX_TOOLS */ -#ifdef xMDBX_ALLOY -/* Amalgamated build */ -#define MDBX_INTERNAL_FUNC static -#define MDBX_INTERNAL_VAR static -#else -/* Non-amalgamated build */ -#define MDBX_INTERNAL_FUNC -#define MDBX_INTERNAL_VAR extern -#endif /* xMDBX_ALLOY */ +#ifdef MDBX_CONFIG_H +#include MDBX_CONFIG_H +#endif + +/* Undefine the NDEBUG if debugging is enforced by MDBX_DEBUG */ +#if (defined(MDBX_DEBUG) && MDBX_DEBUG > 0) || (defined(MDBX_FORCE_ASSERTIONS) && MDBX_FORCE_ASSERTIONS) +#undef NDEBUG +#ifndef MDBX_DEBUG +/* Чтобы избежать включения отладки только из-за включения assert-проверок */ +#define MDBX_DEBUG 0 +#endif +#endif /*----------------------------------------------------------------------------*/ @@ -70,14 +52,59 @@ #endif /* MDBX_DISABLE_GNU_SOURCE */ /* Should be defined before any includes */ -#if !defined(_FILE_OFFSET_BITS) && !defined(__ANDROID_API__) && \ - !defined(ANDROID) +#if !defined(_FILE_OFFSET_BITS) && !defined(__ANDROID_API__) && !defined(ANDROID) #define _FILE_OFFSET_BITS 64 -#endif +#endif /* _FILE_OFFSET_BITS */ -#ifdef __APPLE__ +#if defined(__APPLE__) && !defined(_DARWIN_C_SOURCE) #define _DARWIN_C_SOURCE -#endif +#endif /* _DARWIN_C_SOURCE */ + +#if (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) && !defined(__USE_MINGW_ANSI_STDIO) +#define __USE_MINGW_ANSI_STDIO 1 +#endif /* MinGW */ + +#if defined(_WIN32) || defined(_WIN64) || defined(_WINDOWS) + +#ifndef _WIN32_WINNT +#define _WIN32_WINNT 0x0601 /* Windows 7 */ +#endif /* _WIN32_WINNT */ + +#if !defined(_CRT_SECURE_NO_WARNINGS) +#define _CRT_SECURE_NO_WARNINGS +#endif /* _CRT_SECURE_NO_WARNINGS */ +#if !defined(UNICODE) +#define UNICODE +#endif /* UNICODE */ + +#if !defined(_NO_CRT_STDIO_INLINE) && MDBX_BUILD_SHARED_LIBRARY && !defined(xMDBX_TOOLS) && MDBX_WITHOUT_MSVC_CRT +#define _NO_CRT_STDIO_INLINE +#endif /* _NO_CRT_STDIO_INLINE */ + +#elif !defined(_POSIX_C_SOURCE) +#define _POSIX_C_SOURCE 200809L +#endif /* Windows */ + +#ifdef __cplusplus + +#ifndef NOMINMAX +#define NOMINMAX +#endif /* NOMINMAX */ + +/* Workaround for modern libstdc++ with CLANG < 4.x */ +#if defined(__SIZEOF_INT128__) && !defined(__GLIBCXX_TYPE_INT_N_0) && defined(__clang__) && __clang_major__ < 4 +#define __GLIBCXX_BITSIZE_INT_N_0 128 +#define __GLIBCXX_TYPE_INT_N_0 __int128 +#endif /* Workaround for modern libstdc++ with CLANG < 4.x */ + +#ifdef _MSC_VER +/* Workaround for MSVC' header `extern "C"` vs `std::` redefinition bug */ +#if defined(__SANITIZE_ADDRESS__) && !defined(_DISABLE_VECTOR_ANNOTATION) +#define _DISABLE_VECTOR_ANNOTATION +#endif /* _DISABLE_VECTOR_ANNOTATION */ +#endif /* _MSC_VER */ + +#endif /* __cplusplus */ #ifdef _MSC_VER #if _MSC_FULL_VER < 190024234 @@ -99,12 +126,8 @@ * and how to and where you can obtain the latest "Visual Studio 2015" build * with all fixes. */ -#error \ - "At least \"Microsoft C/C++ Compiler\" version 19.00.24234 (Visual Studio 2015 Update 3) is required." +#error "At least \"Microsoft C/C++ Compiler\" version 19.00.24234 (Visual Studio 2015 Update 3) is required." #endif -#ifndef _CRT_SECURE_NO_WARNINGS -#define _CRT_SECURE_NO_WARNINGS -#endif /* _CRT_SECURE_NO_WARNINGS */ #if _MSC_VER > 1800 #pragma warning(disable : 4464) /* relative include path contains '..' */ #endif @@ -112,124 +135,78 @@ #pragma warning(disable : 5045) /* will insert Spectre mitigation... */ #endif #if _MSC_VER > 1914 -#pragma warning( \ - disable : 5105) /* winbase.h(9531): warning C5105: macro expansion \ - producing 'defined' has undefined behavior */ +#pragma warning(disable : 5105) /* winbase.h(9531): warning C5105: macro expansion \ + producing 'defined' has undefined behavior */ +#endif +#if _MSC_VER < 1920 +/* avoid "error C2219: syntax error: type qualifier must be after '*'" */ +#define __restrict #endif #if _MSC_VER > 1930 #pragma warning(disable : 6235) /* is always a constant */ -#pragma warning(disable : 6237) /* is never evaluated and might \ +#pragma warning(disable : 6237) /* is never evaluated and might \ have side effects */ +#pragma warning(disable : 5286) /* implicit conversion from enum type 'type 1' to enum type 'type 2' */ +#pragma warning(disable : 5287) /* operands are different enum types 'type 1' and 'type 2' */ #endif #pragma warning(disable : 4710) /* 'xyz': function not inlined */ -#pragma warning(disable : 4711) /* function 'xyz' selected for automatic \ +#pragma warning(disable : 4711) /* function 'xyz' selected for automatic \ inline expansion */ -#pragma warning(disable : 4201) /* nonstandard extension used: nameless \ +#pragma warning(disable : 4201) /* nonstandard extension used: nameless \ struct/union */ #pragma warning(disable : 4702) /* unreachable code */ #pragma warning(disable : 4706) /* assignment within conditional expression */ #pragma warning(disable : 4127) /* conditional expression is constant */ -#pragma warning(disable : 4324) /* 'xyz': structure was padded due to \ +#pragma warning(disable : 4324) /* 'xyz': structure was padded due to \ alignment specifier */ #pragma warning(disable : 4310) /* cast truncates constant value */ -#pragma warning(disable : 4820) /* bytes padding added after data member for \ +#pragma warning(disable : 4820) /* bytes padding added after data member for \ alignment */ -#pragma warning(disable : 4548) /* expression before comma has no effect; \ +#pragma warning(disable : 4548) /* expression before comma has no effect; \ expected expression with side - effect */ -#pragma warning(disable : 4366) /* the result of the unary '&' operator may be \ +#pragma warning(disable : 4366) /* the result of the unary '&' operator may be \ unaligned */ -#pragma warning(disable : 4200) /* nonstandard extension used: zero-sized \ +#pragma warning(disable : 4200) /* nonstandard extension used: zero-sized \ array in struct/union */ -#pragma warning(disable : 4204) /* nonstandard extension used: non-constant \ +#pragma warning(disable : 4204) /* nonstandard extension used: non-constant \ aggregate initializer */ -#pragma warning( \ - disable : 4505) /* unreferenced local function has been removed */ -#endif /* _MSC_VER (warnings) */ +#pragma warning(disable : 4505) /* unreferenced local function has been removed */ +#endif /* _MSC_VER (warnings) */ #if defined(__GNUC__) && __GNUC__ < 9 #pragma GCC diagnostic ignored "-Wattributes" #endif /* GCC < 9 */ -#if (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) && \ - !defined(__USE_MINGW_ANSI_STDIO) -#define __USE_MINGW_ANSI_STDIO 1 -#endif /* MinGW */ - -#if (defined(_WIN32) || defined(_WIN64)) && !defined(UNICODE) -#define UNICODE -#endif /* UNICODE */ - -#include "mdbx.h" -/* - * Copyright 2015-2024 Leonid Yuriev - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - */ - - /*----------------------------------------------------------------------------*/ /* Microsoft compiler generates a lot of warning for self includes... */ #ifdef _MSC_VER #pragma warning(push, 1) -#pragma warning(disable : 4548) /* expression before comma has no effect; \ +#pragma warning(disable : 4548) /* expression before comma has no effect; \ expected expression with side - effect */ -#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ +#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ * semantics are not enabled. Specify /EHsc */ -#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ - * mode specified; termination on exception is \ +#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ + * mode specified; termination on exception is \ * not guaranteed. Specify /EHsc */ #endif /* _MSC_VER (warnings) */ -#if defined(_WIN32) || defined(_WIN64) -#if !defined(_CRT_SECURE_NO_WARNINGS) -#define _CRT_SECURE_NO_WARNINGS -#endif /* _CRT_SECURE_NO_WARNINGS */ -#if !defined(_NO_CRT_STDIO_INLINE) && MDBX_BUILD_SHARED_LIBRARY && \ - !defined(xMDBX_TOOLS) && MDBX_WITHOUT_MSVC_CRT -#define _NO_CRT_STDIO_INLINE -#endif -#elif !defined(_POSIX_C_SOURCE) -#define _POSIX_C_SOURCE 200809L -#endif /* Windows */ - /*----------------------------------------------------------------------------*/ /* basic C99 includes */ + #include #include #include #include #include +#include #include #include #include #include #include -#if (-6 & 5) || CHAR_BIT != 8 || UINT_MAX < 0xffffffff || ULONG_MAX % 0xFFFF -#error \ - "Sanity checking failed: Two's complement, reasonably sized integer types" -#endif - -#ifndef SSIZE_MAX -#define SSIZE_MAX INTPTR_MAX -#endif - -#if UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul || defined(_WIN64) -#define MDBX_WORDBITS 64 -#else -#define MDBX_WORDBITS 32 -#endif /* MDBX_WORDBITS */ - /*----------------------------------------------------------------------------*/ /* feature testing */ @@ -241,6 +218,14 @@ #define __has_include(x) (0) #endif +#ifndef __has_attribute +#define __has_attribute(x) (0) +#endif + +#ifndef __has_cpp_attribute +#define __has_cpp_attribute(x) 0 +#endif + #ifndef __has_feature #define __has_feature(x) (0) #endif @@ -263,8 +248,7 @@ #ifndef __GNUC_PREREQ #if defined(__GNUC__) && defined(__GNUC_MINOR__) -#define __GNUC_PREREQ(maj, min) \ - ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min)) +#define __GNUC_PREREQ(maj, min) ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min)) #else #define __GNUC_PREREQ(maj, min) (0) #endif @@ -272,8 +256,7 @@ #ifndef __CLANG_PREREQ #ifdef __clang__ -#define __CLANG_PREREQ(maj, min) \ - ((__clang_major__ << 16) + __clang_minor__ >= ((maj) << 16) + (min)) +#define __CLANG_PREREQ(maj, min) ((__clang_major__ << 16) + __clang_minor__ >= ((maj) << 16) + (min)) #else #define __CLANG_PREREQ(maj, min) (0) #endif @@ -281,13 +264,51 @@ #ifndef __GLIBC_PREREQ #if defined(__GLIBC__) && defined(__GLIBC_MINOR__) -#define __GLIBC_PREREQ(maj, min) \ - ((__GLIBC__ << 16) + __GLIBC_MINOR__ >= ((maj) << 16) + (min)) +#define __GLIBC_PREREQ(maj, min) ((__GLIBC__ << 16) + __GLIBC_MINOR__ >= ((maj) << 16) + (min)) #else #define __GLIBC_PREREQ(maj, min) (0) #endif #endif /* __GLIBC_PREREQ */ +/*----------------------------------------------------------------------------*/ +/* pre-requirements */ + +#if (-6 & 5) || CHAR_BIT != 8 || UINT_MAX < 0xffffffff || ULONG_MAX % 0xFFFF +#error "Sanity checking failed: Two's complement, reasonably sized integer types" +#endif + +#ifndef SSIZE_MAX +#define SSIZE_MAX INTPTR_MAX +#endif + +#if defined(__GNUC__) && !__GNUC_PREREQ(4, 2) +/* Actually libmdbx was not tested with compilers older than GCC 4.2. + * But you could ignore this warning at your own risk. + * In such case please don't rise up an issues related ONLY to old compilers. + */ +#warning "libmdbx required GCC >= 4.2" +#endif + +#if defined(__clang__) && !__CLANG_PREREQ(3, 8) +/* Actually libmdbx was not tested with CLANG older than 3.8. + * But you could ignore this warning at your own risk. + * In such case please don't rise up an issues related ONLY to old compilers. + */ +#warning "libmdbx required CLANG >= 3.8" +#endif + +#if defined(__GLIBC__) && !__GLIBC_PREREQ(2, 12) +/* Actually libmdbx was not tested with something older than glibc 2.12. + * But you could ignore this warning at your own risk. + * In such case please don't rise up an issues related ONLY to old systems. + */ +#warning "libmdbx was only tested with GLIBC >= 2.12." +#endif + +#ifdef __SANITIZE_THREAD__ +#warning "libmdbx don't compatible with ThreadSanitizer, you will get a lot of false-positive issues." +#endif /* __SANITIZE_THREAD__ */ + /*----------------------------------------------------------------------------*/ /* C11' alignas() */ @@ -317,8 +338,7 @@ #endif #endif /* __extern_C */ -#if !defined(nullptr) && !defined(__cplusplus) || \ - (__cplusplus < 201103L && !defined(_MSC_VER)) +#if !defined(nullptr) && !defined(__cplusplus) || (__cplusplus < 201103L && !defined(_MSC_VER)) #define nullptr NULL #endif @@ -330,9 +350,8 @@ #endif #endif /* Apple OSX & iOS */ -#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \ - defined(__BSD__) || defined(__bsdi__) || defined(__DragonFly__) || \ - defined(__APPLE__) || defined(__MACH__) +#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__BSD__) || defined(__bsdi__) || \ + defined(__DragonFly__) || defined(__APPLE__) || defined(__MACH__) #include #include #include @@ -349,8 +368,7 @@ #endif #else #include -#if !(defined(__sun) || defined(__SVR4) || defined(__svr4__) || \ - defined(_WIN32) || defined(_WIN64)) +#if !(defined(__sun) || defined(__SVR4) || defined(__svr4__) || defined(_WIN32) || defined(_WIN64)) #include #endif /* !Solaris */ #endif /* !xBSD */ @@ -404,12 +422,14 @@ __extern_C key_t ftok(const char *, int); #ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN #endif /* WIN32_LEAN_AND_MEAN */ -#include -#include #include #include #include +/* После подгрузки windows.h, чтобы избежать проблем со сборкой MINGW и т.п. */ +#include +#include + #else /*----------------------------------------------------------------------*/ #include @@ -437,11 +457,6 @@ __extern_C key_t ftok(const char *, int); #if __ANDROID_API__ >= 21 #include #endif -#if defined(_FILE_OFFSET_BITS) && _FILE_OFFSET_BITS != MDBX_WORDBITS -#error "_FILE_OFFSET_BITS != MDBX_WORDBITS" (_FILE_OFFSET_BITS != MDBX_WORDBITS) -#elif defined(__FILE_OFFSET_BITS) && __FILE_OFFSET_BITS != MDBX_WORDBITS -#error "__FILE_OFFSET_BITS != MDBX_WORDBITS" (__FILE_OFFSET_BITS != MDBX_WORDBITS) -#endif #endif /* Android */ #if defined(HAVE_SYS_STAT_H) || __has_include() @@ -457,43 +472,38 @@ __extern_C key_t ftok(const char *, int); /*----------------------------------------------------------------------------*/ /* Byteorder */ -#if defined(i386) || defined(__386) || defined(__i386) || defined(__i386__) || \ - defined(i486) || defined(__i486) || defined(__i486__) || defined(i586) || \ - defined(__i586) || defined(__i586__) || defined(i686) || \ - defined(__i686) || defined(__i686__) || defined(_M_IX86) || \ - defined(_X86_) || defined(__THW_INTEL__) || defined(__I86__) || \ - defined(__INTEL__) || defined(__x86_64) || defined(__x86_64__) || \ - defined(__amd64__) || defined(__amd64) || defined(_M_X64) || \ - defined(_M_AMD64) || defined(__IA32__) || defined(__INTEL__) +#if defined(i386) || defined(__386) || defined(__i386) || defined(__i386__) || defined(i486) || defined(__i486) || \ + defined(__i486__) || defined(i586) || defined(__i586) || defined(__i586__) || defined(i686) || defined(__i686) || \ + defined(__i686__) || defined(_M_IX86) || defined(_X86_) || defined(__THW_INTEL__) || defined(__I86__) || \ + defined(__INTEL__) || defined(__x86_64) || defined(__x86_64__) || defined(__amd64__) || defined(__amd64) || \ + defined(_M_X64) || defined(_M_AMD64) || defined(__IA32__) || defined(__INTEL__) #ifndef __ia32__ /* LY: define neutral __ia32__ for x86 and x86-64 */ #define __ia32__ 1 #endif /* __ia32__ */ -#if !defined(__amd64__) && \ - (defined(__x86_64) || defined(__x86_64__) || defined(__amd64) || \ - defined(_M_X64) || defined(_M_AMD64)) +#if !defined(__amd64__) && \ + (defined(__x86_64) || defined(__x86_64__) || defined(__amd64) || defined(_M_X64) || defined(_M_AMD64)) /* LY: define trusty __amd64__ for all AMD64/x86-64 arch */ #define __amd64__ 1 #endif /* __amd64__ */ #endif /* all x86 */ -#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || \ - !defined(__ORDER_BIG_ENDIAN__) +#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || !defined(__ORDER_BIG_ENDIAN__) -#if defined(__GLIBC__) || defined(__GNU_LIBRARY__) || \ - defined(__ANDROID_API__) || defined(HAVE_ENDIAN_H) || __has_include() +#if defined(__GLIBC__) || defined(__GNU_LIBRARY__) || defined(__ANDROID_API__) || defined(HAVE_ENDIAN_H) || \ + __has_include() #include -#elif defined(__APPLE__) || defined(__MACH__) || defined(__OpenBSD__) || \ - defined(HAVE_MACHINE_ENDIAN_H) || __has_include() +#elif defined(__APPLE__) || defined(__MACH__) || defined(__OpenBSD__) || defined(HAVE_MACHINE_ENDIAN_H) || \ + __has_include() #include #elif defined(HAVE_SYS_ISA_DEFS_H) || __has_include() #include -#elif (defined(HAVE_SYS_TYPES_H) && defined(HAVE_SYS_ENDIAN_H)) || \ +#elif (defined(HAVE_SYS_TYPES_H) && defined(HAVE_SYS_ENDIAN_H)) || \ (__has_include() && __has_include()) #include #include -#elif defined(__bsdi__) || defined(__DragonFly__) || defined(__FreeBSD__) || \ - defined(__NetBSD__) || defined(HAVE_SYS_PARAM_H) || __has_include() +#elif defined(__bsdi__) || defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || \ + defined(HAVE_SYS_PARAM_H) || __has_include() #include #endif /* OS */ @@ -509,27 +519,19 @@ __extern_C key_t ftok(const char *, int); #define __ORDER_LITTLE_ENDIAN__ 1234 #define __ORDER_BIG_ENDIAN__ 4321 -#if defined(__LITTLE_ENDIAN__) || \ - (defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)) || \ - defined(__ARMEL__) || defined(__THUMBEL__) || defined(__AARCH64EL__) || \ - defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \ - defined(_M_ARM) || defined(_M_ARM64) || defined(__e2k__) || \ - defined(__elbrus_4c__) || defined(__elbrus_8c__) || defined(__bfin__) || \ - defined(__BFIN__) || defined(__ia64__) || defined(_IA64) || \ - defined(__IA64__) || defined(__ia64) || defined(_M_IA64) || \ - defined(__itanium__) || defined(__ia32__) || defined(__CYGWIN__) || \ - defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) || \ - defined(__WINDOWS__) +#if defined(__LITTLE_ENDIAN__) || (defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN)) || defined(__ARMEL__) || \ + defined(__THUMBEL__) || defined(__AARCH64EL__) || defined(__MIPSEL__) || defined(_MIPSEL) || defined(__MIPSEL) || \ + defined(_M_ARM) || defined(_M_ARM64) || defined(__e2k__) || defined(__elbrus_4c__) || defined(__elbrus_8c__) || \ + defined(__bfin__) || defined(__BFIN__) || defined(__ia64__) || defined(_IA64) || defined(__IA64__) || \ + defined(__ia64) || defined(_M_IA64) || defined(__itanium__) || defined(__ia32__) || defined(__CYGWIN__) || \ + defined(_WIN64) || defined(_WIN32) || defined(__TOS_WIN__) || defined(__WINDOWS__) #define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__ -#elif defined(__BIG_ENDIAN__) || \ - (defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) || \ - defined(__ARMEB__) || defined(__THUMBEB__) || defined(__AARCH64EB__) || \ - defined(__MIPSEB__) || defined(_MIPSEB) || defined(__MIPSEB) || \ - defined(__m68k__) || defined(M68000) || defined(__hppa__) || \ - defined(__hppa) || defined(__HPPA__) || defined(__sparc__) || \ - defined(__sparc) || defined(__370__) || defined(__THW_370__) || \ - defined(__s390__) || defined(__s390x__) || defined(__SYSC_ZARCH__) +#elif defined(__BIG_ENDIAN__) || (defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN)) || defined(__ARMEB__) || \ + defined(__THUMBEB__) || defined(__AARCH64EB__) || defined(__MIPSEB__) || defined(_MIPSEB) || defined(__MIPSEB) || \ + defined(__m68k__) || defined(M68000) || defined(__hppa__) || defined(__hppa) || defined(__HPPA__) || \ + defined(__sparc__) || defined(__sparc) || defined(__370__) || defined(__THW_370__) || defined(__s390__) || \ + defined(__s390x__) || defined(__SYSC_ZARCH__) #define __BYTE_ORDER__ __ORDER_BIG_ENDIAN__ #else @@ -539,6 +541,12 @@ __extern_C key_t ftok(const char *, int); #endif #endif /* __BYTE_ORDER__ || __ORDER_LITTLE_ENDIAN__ || __ORDER_BIG_ENDIAN__ */ +#if UINTPTR_MAX > 0xffffFFFFul || ULONG_MAX > 0xffffFFFFul || defined(_WIN64) +#define MDBX_WORDBITS 64 +#else +#define MDBX_WORDBITS 32 +#endif /* MDBX_WORDBITS */ + /*----------------------------------------------------------------------------*/ /* Availability of CMOV or equivalent */ @@ -549,17 +557,14 @@ __extern_C key_t ftok(const char *, int); #define MDBX_HAVE_CMOV 1 #elif defined(__thumb__) || defined(__thumb) || defined(__TARGET_ARCH_THUMB) #define MDBX_HAVE_CMOV 0 -#elif defined(_M_ARM) || defined(_M_ARM64) || defined(__aarch64__) || \ - defined(__aarch64) || defined(__arm__) || defined(__arm) || \ - defined(__CC_ARM) +#elif defined(_M_ARM) || defined(_M_ARM64) || defined(__aarch64__) || defined(__aarch64) || defined(__arm__) || \ + defined(__arm) || defined(__CC_ARM) #define MDBX_HAVE_CMOV 1 -#elif (defined(__riscv__) || defined(__riscv64)) && \ - (defined(__riscv_b) || defined(__riscv_bitmanip)) +#elif (defined(__riscv__) || defined(__riscv64)) && (defined(__riscv_b) || defined(__riscv_bitmanip)) #define MDBX_HAVE_CMOV 1 -#elif defined(i686) || defined(__i686) || defined(__i686__) || \ - (defined(_M_IX86) && _M_IX86 > 600) || defined(__x86_64) || \ - defined(__x86_64__) || defined(__amd64__) || defined(__amd64) || \ - defined(_M_X64) || defined(_M_AMD64) +#elif defined(i686) || defined(__i686) || defined(__i686__) || (defined(_M_IX86) && _M_IX86 > 600) || \ + defined(__x86_64) || defined(__x86_64__) || defined(__amd64__) || defined(__amd64) || defined(_M_X64) || \ + defined(_M_AMD64) #define MDBX_HAVE_CMOV 1 #else #define MDBX_HAVE_CMOV 0 @@ -585,8 +590,7 @@ __extern_C key_t ftok(const char *, int); #endif #elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) #include -#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ - (defined(HP_IA64) || defined(__ia64)) +#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && (defined(HP_IA64) || defined(__ia64)) #include #elif defined(__IBMC__) && defined(__powerpc) #include @@ -608,29 +612,26 @@ __extern_C key_t ftok(const char *, int); #endif /* Compiler */ #if !defined(__noop) && !defined(_MSC_VER) -#define __noop \ - do { \ +#define __noop \ + do { \ } while (0) #endif /* __noop */ -#if defined(__fallthrough) && \ - (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) +#if defined(__fallthrough) && (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) #undef __fallthrough #endif /* __fallthrough workaround for MinGW */ #ifndef __fallthrough -#if defined(__cplusplus) && (__has_cpp_attribute(fallthrough) && \ - (!defined(__clang__) || __clang__ > 4)) || \ +#if defined(__cplusplus) && (__has_cpp_attribute(fallthrough) && (!defined(__clang__) || __clang__ > 4)) || \ __cplusplus >= 201703L #define __fallthrough [[fallthrough]] #elif __GNUC_PREREQ(8, 0) && defined(__cplusplus) && __cplusplus >= 201103L #define __fallthrough [[fallthrough]] -#elif __GNUC_PREREQ(7, 0) && \ - (!defined(__LCC__) || (__LCC__ == 124 && __LCC_MINOR__ >= 12) || \ - (__LCC__ == 125 && __LCC_MINOR__ >= 5) || (__LCC__ >= 126)) +#elif __GNUC_PREREQ(7, 0) && (!defined(__LCC__) || (__LCC__ == 124 && __LCC_MINOR__ >= 12) || \ + (__LCC__ == 125 && __LCC_MINOR__ >= 5) || (__LCC__ >= 126)) #define __fallthrough __attribute__((__fallthrough__)) -#elif defined(__clang__) && defined(__cplusplus) && __cplusplus >= 201103L && \ - __has_feature(cxx_attributes) && __has_warning("-Wimplicit-fallthrough") +#elif defined(__clang__) && defined(__cplusplus) && __cplusplus >= 201103L && __has_feature(cxx_attributes) && \ + __has_warning("-Wimplicit-fallthrough") #define __fallthrough [[clang::fallthrough]] #else #define __fallthrough @@ -643,8 +644,8 @@ __extern_C key_t ftok(const char *, int); #elif defined(_MSC_VER) #define __unreachable() __assume(0) #else -#define __unreachable() \ - do { \ +#define __unreachable() \ + do { \ } while (1) #endif #endif /* __unreachable */ @@ -653,9 +654,9 @@ __extern_C key_t ftok(const char *, int); #if defined(__GNUC__) || defined(__clang__) || __has_builtin(__builtin_prefetch) #define __prefetch(ptr) __builtin_prefetch(ptr) #else -#define __prefetch(ptr) \ - do { \ - (void)(ptr); \ +#define __prefetch(ptr) \ + do { \ + (void)(ptr); \ } while (0) #endif #endif /* __prefetch */ @@ -665,11 +666,11 @@ __extern_C key_t ftok(const char *, int); #endif /* offsetof */ #ifndef container_of -#define container_of(ptr, type, member) \ - ((type *)((char *)(ptr) - offsetof(type, member))) +#define container_of(ptr, type, member) ((type *)((char *)(ptr) - offsetof(type, member))) #endif /* container_of */ /*----------------------------------------------------------------------------*/ +/* useful attributes */ #ifndef __always_inline #if defined(__GNUC__) || __has_attribute(__always_inline__) @@ -737,8 +738,7 @@ __extern_C key_t ftok(const char *, int); #ifndef __hot #if defined(__OPTIMIZE__) -#if defined(__clang__) && !__has_attribute(__hot__) && \ - __has_attribute(__section__) && \ +#if defined(__clang__) && !__has_attribute(__hot__) && __has_attribute(__section__) && \ (defined(__linux__) || defined(__gnu_linux__)) /* just put frequently used functions in separate section */ #define __hot __attribute__((__section__("text.hot"))) __optimize("O3") @@ -754,8 +754,7 @@ __extern_C key_t ftok(const char *, int); #ifndef __cold #if defined(__OPTIMIZE__) -#if defined(__clang__) && !__has_attribute(__cold__) && \ - __has_attribute(__section__) && \ +#if defined(__clang__) && !__has_attribute(__cold__) && __has_attribute(__section__) && \ (defined(__linux__) || defined(__gnu_linux__)) /* just put infrequently used functions in separate section */ #define __cold __attribute__((__section__("text.unlikely"))) __optimize("Os") @@ -778,8 +777,7 @@ __extern_C key_t ftok(const char *, int); #endif /* __flatten */ #ifndef likely -#if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && \ - !defined(__COVERITY__) +#if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__) #define likely(cond) __builtin_expect(!!(cond), 1) #else #define likely(x) (!!(x)) @@ -787,8 +785,7 @@ __extern_C key_t ftok(const char *, int); #endif /* likely */ #ifndef unlikely -#if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && \ - !defined(__COVERITY__) +#if (defined(__GNUC__) || __has_builtin(__builtin_expect)) && !defined(__COVERITY__) #define unlikely(cond) __builtin_expect(!!(cond), 0) #else #define unlikely(x) (!!(x)) @@ -803,29 +800,41 @@ __extern_C key_t ftok(const char *, int); #endif #endif /* __anonymous_struct_extension__ */ -#ifndef expect_with_probability -#if defined(__builtin_expect_with_probability) || \ - __has_builtin(__builtin_expect_with_probability) || __GNUC_PREREQ(9, 0) -#define expect_with_probability(expr, value, prob) \ - __builtin_expect_with_probability(expr, value, prob) -#else -#define expect_with_probability(expr, value, prob) (expr) -#endif -#endif /* expect_with_probability */ - #ifndef MDBX_WEAK_IMPORT_ATTRIBUTE #ifdef WEAK_IMPORT_ATTRIBUTE #define MDBX_WEAK_IMPORT_ATTRIBUTE WEAK_IMPORT_ATTRIBUTE #elif __has_attribute(__weak__) && __has_attribute(__weak_import__) #define MDBX_WEAK_IMPORT_ATTRIBUTE __attribute__((__weak__, __weak_import__)) -#elif __has_attribute(__weak__) || \ - (defined(__GNUC__) && __GNUC__ >= 4 && defined(__ELF__)) +#elif __has_attribute(__weak__) || (defined(__GNUC__) && __GNUC__ >= 4 && defined(__ELF__)) #define MDBX_WEAK_IMPORT_ATTRIBUTE __attribute__((__weak__)) #else #define MDBX_WEAK_IMPORT_ATTRIBUTE #endif #endif /* MDBX_WEAK_IMPORT_ATTRIBUTE */ +#if !defined(__thread) && (defined(_MSC_VER) || defined(__DMC__)) +#define __thread __declspec(thread) +#endif /* __thread */ + +#ifndef MDBX_EXCLUDE_FOR_GPROF +#ifdef ENABLE_GPROF +#define MDBX_EXCLUDE_FOR_GPROF __attribute__((__no_instrument_function__, __no_profile_instrument_function__)) +#else +#define MDBX_EXCLUDE_FOR_GPROF +#endif /* ENABLE_GPROF */ +#endif /* MDBX_EXCLUDE_FOR_GPROF */ + +/*----------------------------------------------------------------------------*/ + +#ifndef expect_with_probability +#if defined(__builtin_expect_with_probability) || __has_builtin(__builtin_expect_with_probability) || \ + __GNUC_PREREQ(9, 0) +#define expect_with_probability(expr, value, prob) __builtin_expect_with_probability(expr, value, prob) +#else +#define expect_with_probability(expr, value, prob) (expr) +#endif +#endif /* expect_with_probability */ + #ifndef MDBX_GOOFY_MSVC_STATIC_ANALYZER #ifdef _PREFAST_ #define MDBX_GOOFY_MSVC_STATIC_ANALYZER 1 @@ -837,20 +846,27 @@ __extern_C key_t ftok(const char *, int); #if MDBX_GOOFY_MSVC_STATIC_ANALYZER || (defined(_MSC_VER) && _MSC_VER > 1919) #define MDBX_ANALYSIS_ASSUME(expr) __analysis_assume(expr) #ifdef _PREFAST_ -#define MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(warn_id) \ - __pragma(prefast(suppress : warn_id)) +#define MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(warn_id) __pragma(prefast(suppress : warn_id)) #else -#define MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(warn_id) \ - __pragma(warning(suppress : warn_id)) +#define MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(warn_id) __pragma(warning(suppress : warn_id)) #endif #else #define MDBX_ANALYSIS_ASSUME(expr) assert(expr) #define MDBX_SUPPRESS_GOOFY_MSVC_ANALYZER(warn_id) #endif /* MDBX_GOOFY_MSVC_STATIC_ANALYZER */ +#ifndef FLEXIBLE_ARRAY_MEMBERS +#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || (!defined(__cplusplus) && defined(_MSC_VER)) +#define FLEXIBLE_ARRAY_MEMBERS 1 +#else +#define FLEXIBLE_ARRAY_MEMBERS 0 +#endif +#endif /* FLEXIBLE_ARRAY_MEMBERS */ + /*----------------------------------------------------------------------------*/ +/* Valgrind and Address Sanitizer */ -#if defined(MDBX_USE_VALGRIND) +#if defined(ENABLE_MEMCHECK) #include #ifndef VALGRIND_DISABLE_ADDR_ERROR_REPORTING_IN_RANGE /* LY: available since Valgrind 3.10 */ @@ -872,7 +888,7 @@ __extern_C key_t ftok(const char *, int); #define VALGRIND_CHECK_MEM_IS_ADDRESSABLE(a, s) (0) #define VALGRIND_CHECK_MEM_IS_DEFINED(a, s) (0) #define RUNNING_ON_VALGRIND (0) -#endif /* MDBX_USE_VALGRIND */ +#endif /* ENABLE_MEMCHECK */ #ifdef __SANITIZE_ADDRESS__ #include @@ -899,8 +915,7 @@ template char (&__ArraySizeHelper(T (&array)[N]))[N]; #define CONCAT(a, b) a##b #define XCONCAT(a, b) CONCAT(a, b) -#define MDBX_TETRAD(a, b, c, d) \ - ((uint32_t)(a) << 24 | (uint32_t)(b) << 16 | (uint32_t)(c) << 8 | (d)) +#define MDBX_TETRAD(a, b, c, d) ((uint32_t)(a) << 24 | (uint32_t)(b) << 16 | (uint32_t)(c) << 8 | (d)) #define MDBX_STRING_TETRAD(str) MDBX_TETRAD(str[0], str[1], str[2], str[3]) @@ -914,14 +929,13 @@ template char (&__ArraySizeHelper(T (&array)[N]))[N]; #elif defined(_MSC_VER) #include #define STATIC_ASSERT_MSG(expr, msg) _STATIC_ASSERT(expr) -#elif (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) || \ - __has_feature(c_static_assert) +#elif (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) || __has_feature(c_static_assert) #define STATIC_ASSERT_MSG(expr, msg) _Static_assert(expr, msg) #else -#define STATIC_ASSERT_MSG(expr, msg) \ - switch (0) { \ - case 0: \ - case (expr):; \ +#define STATIC_ASSERT_MSG(expr, msg) \ + switch (0) { \ + case 0: \ + case (expr):; \ } #endif #endif /* STATIC_ASSERT */ @@ -930,42 +944,37 @@ template char (&__ArraySizeHelper(T (&array)[N]))[N]; #define STATIC_ASSERT(expr) STATIC_ASSERT_MSG(expr, #expr) #endif -#ifndef __Wpedantic_format_voidptr -MDBX_MAYBE_UNUSED MDBX_PURE_FUNCTION static __inline const void * -__Wpedantic_format_voidptr(const void *ptr) { - return ptr; -} -#define __Wpedantic_format_voidptr(ARG) __Wpedantic_format_voidptr(ARG) -#endif /* __Wpedantic_format_voidptr */ +/*----------------------------------------------------------------------------*/ -#if defined(__GNUC__) && !__GNUC_PREREQ(4, 2) -/* Actually libmdbx was not tested with compilers older than GCC 4.2. - * But you could ignore this warning at your own risk. - * In such case please don't rise up an issues related ONLY to old compilers. - */ -#warning "libmdbx required GCC >= 4.2" -#endif +#if defined(_MSC_VER) && _MSC_VER >= 1900 +/* LY: MSVC 2015/2017/2019 has buggy/inconsistent PRIuPTR/PRIxPTR macros + * for internal format-args checker. */ +#undef PRIuPTR +#undef PRIiPTR +#undef PRIdPTR +#undef PRIxPTR +#define PRIuPTR "Iu" +#define PRIiPTR "Ii" +#define PRIdPTR "Id" +#define PRIxPTR "Ix" +#define PRIuSIZE "zu" +#define PRIiSIZE "zi" +#define PRIdSIZE "zd" +#define PRIxSIZE "zx" +#endif /* fix PRI*PTR for _MSC_VER */ -#if defined(__clang__) && !__CLANG_PREREQ(3, 8) -/* Actually libmdbx was not tested with CLANG older than 3.8. - * But you could ignore this warning at your own risk. - * In such case please don't rise up an issues related ONLY to old compilers. - */ -#warning "libmdbx required CLANG >= 3.8" -#endif +#ifndef PRIuSIZE +#define PRIuSIZE PRIuPTR +#define PRIiSIZE PRIiPTR +#define PRIdSIZE PRIdPTR +#define PRIxSIZE PRIxPTR +#endif /* PRI*SIZE macros for MSVC */ -#if defined(__GLIBC__) && !__GLIBC_PREREQ(2, 12) -/* Actually libmdbx was not tested with something older than glibc 2.12. - * But you could ignore this warning at your own risk. - * In such case please don't rise up an issues related ONLY to old systems. - */ -#warning "libmdbx was only tested with GLIBC >= 2.12." +#ifdef _MSC_VER +#pragma warning(pop) #endif -#ifdef __SANITIZE_THREAD__ -#warning \ - "libmdbx don't compatible with ThreadSanitizer, you will get a lot of false-positive issues." -#endif /* __SANITIZE_THREAD__ */ +/*----------------------------------------------------------------------------*/ #if __has_warning("-Wnested-anon-types") #if defined(__clang__) @@ -1002,80 +1011,34 @@ __Wpedantic_format_voidptr(const void *ptr) { #endif #endif /* -Walignment-reduction-ignored */ -#ifndef MDBX_EXCLUDE_FOR_GPROF -#ifdef ENABLE_GPROF -#define MDBX_EXCLUDE_FOR_GPROF \ - __attribute__((__no_instrument_function__, \ - __no_profile_instrument_function__)) +#ifdef xMDBX_ALLOY +/* Amalgamated build */ +#define MDBX_INTERNAL static #else -#define MDBX_EXCLUDE_FOR_GPROF -#endif /* ENABLE_GPROF */ -#endif /* MDBX_EXCLUDE_FOR_GPROF */ - -#ifdef __cplusplus -extern "C" { -#endif +/* Non-amalgamated build */ +#define MDBX_INTERNAL +#endif /* xMDBX_ALLOY */ -/* https://en.wikipedia.org/wiki/Operating_system_abstraction_layer */ +#include "mdbx.h" -/* - * Copyright 2015-2024 Leonid Yuriev - * and other libmdbx authors: please see AUTHORS file. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - */ +/*----------------------------------------------------------------------------*/ +/* Basic constants and types */ +typedef struct iov_ctx iov_ctx_t; +/// /*----------------------------------------------------------------------------*/ -/* C11 Atomics */ - -#if defined(__cplusplus) && !defined(__STDC_NO_ATOMICS__) && __has_include() -#include -#define MDBX_HAVE_C11ATOMICS -#elif !defined(__cplusplus) && \ - (__STDC_VERSION__ >= 201112L || __has_extension(c_atomic)) && \ - !defined(__STDC_NO_ATOMICS__) && \ - (__GNUC_PREREQ(4, 9) || __CLANG_PREREQ(3, 8) || \ - !(defined(__GNUC__) || defined(__clang__))) -#include -#define MDBX_HAVE_C11ATOMICS -#elif defined(__GNUC__) || defined(__clang__) -#elif defined(_MSC_VER) -#pragma warning(disable : 4163) /* 'xyz': not available as an intrinsic */ -#pragma warning(disable : 4133) /* 'function': incompatible types - from \ - 'size_t' to 'LONGLONG' */ -#pragma warning(disable : 4244) /* 'return': conversion from 'LONGLONG' to \ - 'std::size_t', possible loss of data */ -#pragma warning(disable : 4267) /* 'function': conversion from 'size_t' to \ - 'long', possible loss of data */ -#pragma intrinsic(_InterlockedExchangeAdd, _InterlockedCompareExchange) -#pragma intrinsic(_InterlockedExchangeAdd64, _InterlockedCompareExchange64) -#elif defined(__APPLE__) -#include -#else -#error FIXME atomic-ops -#endif - -/*----------------------------------------------------------------------------*/ -/* Memory/Compiler barriers, cache coherence */ +/* Memory/Compiler barriers, cache coherence */ #if __has_include() #include -#elif defined(__mips) || defined(__mips__) || defined(__mips64) || \ - defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \ - defined(__MWERKS__) || defined(__sgi) +#elif defined(__mips) || defined(__mips__) || defined(__mips64) || defined(__mips64__) || defined(_M_MRX000) || \ + defined(_MIPS_) || defined(__MWERKS__) || defined(__sgi) /* MIPS should have explicit cache control */ #include #endif -MDBX_MAYBE_UNUSED static __inline void osal_compiler_barrier(void) { +MDBX_MAYBE_UNUSED static inline void osal_compiler_barrier(void) { #if defined(__clang__) || defined(__GNUC__) __asm__ __volatile__("" ::: "memory"); #elif defined(_MSC_VER) @@ -1084,18 +1047,16 @@ MDBX_MAYBE_UNUSED static __inline void osal_compiler_barrier(void) { __memory_barrier(); #elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) __compiler_barrier(); -#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ - (defined(HP_IA64) || defined(__ia64)) +#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && (defined(HP_IA64) || defined(__ia64)) _Asm_sched_fence(/* LY: no-arg meaning 'all expect ALU', e.g. 0x3D3D */); -#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \ - defined(__ppc64__) || defined(__powerpc64__) +#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || defined(__ppc64__) || defined(__powerpc64__) __fence(); #else #error "Could not guess the kind of compiler, please report to us." #endif } -MDBX_MAYBE_UNUSED static __inline void osal_memory_barrier(void) { +MDBX_MAYBE_UNUSED static inline void osal_memory_barrier(void) { #ifdef MDBX_HAVE_C11ATOMICS atomic_thread_fence(memory_order_seq_cst); #elif defined(__ATOMIC_SEQ_CST) @@ -1116,11 +1077,9 @@ MDBX_MAYBE_UNUSED static __inline void osal_memory_barrier(void) { #endif #elif defined(__SUNPRO_C) || defined(__sun) || defined(sun) __machine_rw_barrier(); -#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && \ - (defined(HP_IA64) || defined(__ia64)) +#elif (defined(_HPUX_SOURCE) || defined(__hpux) || defined(__HP_aCC)) && (defined(HP_IA64) || defined(__ia64)) _Asm_mf(); -#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || \ - defined(__ppc64__) || defined(__powerpc64__) +#elif defined(_AIX) || defined(__ppc__) || defined(__powerpc__) || defined(__ppc64__) || defined(__powerpc64__) __lwsync(); #else #error "Could not guess the kind of compiler, please report to us." @@ -1135,7 +1094,7 @@ MDBX_MAYBE_UNUSED static __inline void osal_memory_barrier(void) { #define HAVE_SYS_TYPES_H typedef HANDLE osal_thread_t; typedef unsigned osal_thread_key_t; -#define MAP_FAILED NULL +#define MAP_FAILED nullptr #define HIGH_DWORD(v) ((DWORD)((sizeof(v) > 4) ? ((uint64_t)(v) >> 32) : 0)) #define THREAD_CALL WINAPI #define THREAD_RESULT DWORD @@ -1147,15 +1106,13 @@ typedef CRITICAL_SECTION osal_fastmutex_t; #if !defined(_MSC_VER) && !defined(__try) #define __try -#define __except(COND) if (false) +#define __except(COND) if (/* (void)(COND), */ false) #endif /* stub for MSVC's __try/__except */ #if MDBX_WITHOUT_MSVC_CRT #ifndef osal_malloc -static inline void *osal_malloc(size_t bytes) { - return HeapAlloc(GetProcessHeap(), 0, bytes); -} +static inline void *osal_malloc(size_t bytes) { return HeapAlloc(GetProcessHeap(), 0, bytes); } #endif /* osal_malloc */ #ifndef osal_calloc @@ -1166,8 +1123,7 @@ static inline void *osal_calloc(size_t nelem, size_t size) { #ifndef osal_realloc static inline void *osal_realloc(void *ptr, size_t bytes) { - return ptr ? HeapReAlloc(GetProcessHeap(), 0, ptr, bytes) - : HeapAlloc(GetProcessHeap(), 0, bytes); + return ptr ? HeapReAlloc(GetProcessHeap(), 0, ptr, bytes) : HeapAlloc(GetProcessHeap(), 0, bytes); } #endif /* osal_realloc */ @@ -1213,29 +1169,16 @@ typedef pthread_mutex_t osal_fastmutex_t; #endif /* Platform */ #if __GLIBC_PREREQ(2, 12) || defined(__FreeBSD__) || defined(malloc_usable_size) -/* malloc_usable_size() already provided */ +#define osal_malloc_usable_size(ptr) malloc_usable_size(ptr) #elif defined(__APPLE__) -#define malloc_usable_size(ptr) malloc_size(ptr) +#define osal_malloc_usable_size(ptr) malloc_size(ptr) #elif defined(_MSC_VER) && !MDBX_WITHOUT_MSVC_CRT -#define malloc_usable_size(ptr) _msize(ptr) -#endif /* malloc_usable_size */ +#define osal_malloc_usable_size(ptr) _msize(ptr) +#endif /* osal_malloc_usable_size */ /*----------------------------------------------------------------------------*/ /* OS abstraction layer stuff */ -MDBX_INTERNAL_VAR unsigned sys_pagesize; -MDBX_MAYBE_UNUSED MDBX_INTERNAL_VAR unsigned sys_pagesize_ln2, - sys_allocation_granularity; - -/* Get the size of a memory page for the system. - * This is the basic size that the platform's memory manager uses, and is - * fundamental to the use of memory-mapped files. */ -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline size_t -osal_syspagesize(void) { - assert(sys_pagesize > 0 && (sys_pagesize & (sys_pagesize - 1)) == 0); - return sys_pagesize; -} - #if defined(_WIN32) || defined(_WIN64) typedef wchar_t pathchar_t; #define MDBX_PRIsPATH "ls" @@ -1247,7 +1190,7 @@ typedef char pathchar_t; typedef struct osal_mmap { union { void *base; - struct MDBX_lockinfo *lck; + struct shared_lck *lck; }; mdbx_filehandle_t fd; size_t limit; /* mapping length, but NOT a size of file nor DB */ @@ -1258,25 +1201,6 @@ typedef struct osal_mmap { #endif } osal_mmap_t; -typedef union bin128 { - __anonymous_struct_extension__ struct { - uint64_t x, y; - }; - __anonymous_struct_extension__ struct { - uint32_t a, b, c, d; - }; -} bin128_t; - -#if defined(_WIN32) || defined(_WIN64) -typedef union osal_srwlock { - __anonymous_struct_extension__ struct { - long volatile readerCount; - long volatile writerCount; - }; - RTL_SRWLOCK native; -} osal_srwlock_t; -#endif /* Windows */ - #ifndef MDBX_HAVE_PWRITEV #if defined(_WIN32) || defined(_WIN64) @@ -1285,14 +1209,21 @@ typedef union osal_srwlock { #elif defined(__ANDROID_API__) #if __ANDROID_API__ < 24 +/* https://android-developers.googleblog.com/2017/09/introducing-android-native-development.html + * https://android.googlesource.com/platform/bionic/+/master/docs/32-bit-abi.md */ #define MDBX_HAVE_PWRITEV 0 +#if defined(_FILE_OFFSET_BITS) && _FILE_OFFSET_BITS != MDBX_WORDBITS +#error "_FILE_OFFSET_BITS != MDBX_WORDBITS and __ANDROID_API__ < 24" (_FILE_OFFSET_BITS != MDBX_WORDBITS) +#elif defined(__FILE_OFFSET_BITS) && __FILE_OFFSET_BITS != MDBX_WORDBITS +#error "__FILE_OFFSET_BITS != MDBX_WORDBITS and __ANDROID_API__ < 24" (__FILE_OFFSET_BITS != MDBX_WORDBITS) +#endif #else #define MDBX_HAVE_PWRITEV 1 #endif #elif defined(__APPLE__) || defined(__MACH__) || defined(_DARWIN_C_SOURCE) -#if defined(MAC_OS_X_VERSION_MIN_REQUIRED) && defined(MAC_OS_VERSION_11_0) && \ +#if defined(MAC_OS_X_VERSION_MIN_REQUIRED) && defined(MAC_OS_VERSION_11_0) && \ MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_VERSION_11_0 /* FIXME: add checks for IOS versions, etc */ #define MDBX_HAVE_PWRITEV 1 @@ -1310,20 +1241,20 @@ typedef union osal_srwlock { typedef struct ior_item { #if defined(_WIN32) || defined(_WIN64) OVERLAPPED ov; -#define ior_svg_gap4terminator 1 +#define ior_sgv_gap4terminator 1 #define ior_sgv_element FILE_SEGMENT_ELEMENT #else size_t offset; #if MDBX_HAVE_PWRITEV size_t sgvcnt; -#define ior_svg_gap4terminator 0 +#define ior_sgv_gap4terminator 0 #define ior_sgv_element struct iovec #endif /* MDBX_HAVE_PWRITEV */ #endif /* !Windows */ union { MDBX_val single; #if defined(ior_sgv_element) - ior_sgv_element sgv[1 + ior_svg_gap4terminator]; + ior_sgv_element sgv[1 + ior_sgv_gap4terminator]; #endif /* ior_sgv_element */ }; } ior_item_t; @@ -1359,45 +1290,33 @@ typedef struct osal_ioring { char *boundary; } osal_ioring_t; -#ifndef __cplusplus - /* Actually this is not ioring for now, but on the way. */ -MDBX_INTERNAL_FUNC int osal_ioring_create(osal_ioring_t * +MDBX_INTERNAL int osal_ioring_create(osal_ioring_t * #if defined(_WIN32) || defined(_WIN64) - , - bool enable_direct, - mdbx_filehandle_t overlapped_fd + , + bool enable_direct, mdbx_filehandle_t overlapped_fd #endif /* Windows */ ); -MDBX_INTERNAL_FUNC int osal_ioring_resize(osal_ioring_t *, size_t items); -MDBX_INTERNAL_FUNC void osal_ioring_destroy(osal_ioring_t *); -MDBX_INTERNAL_FUNC void osal_ioring_reset(osal_ioring_t *); -MDBX_INTERNAL_FUNC int osal_ioring_add(osal_ioring_t *ctx, const size_t offset, - void *data, const size_t bytes); +MDBX_INTERNAL int osal_ioring_resize(osal_ioring_t *, size_t items); +MDBX_INTERNAL void osal_ioring_destroy(osal_ioring_t *); +MDBX_INTERNAL void osal_ioring_reset(osal_ioring_t *); +MDBX_INTERNAL int osal_ioring_add(osal_ioring_t *ctx, const size_t offset, void *data, const size_t bytes); typedef struct osal_ioring_write_result { int err; unsigned wops; } osal_ioring_write_result_t; -MDBX_INTERNAL_FUNC osal_ioring_write_result_t -osal_ioring_write(osal_ioring_t *ior, mdbx_filehandle_t fd); +MDBX_INTERNAL osal_ioring_write_result_t osal_ioring_write(osal_ioring_t *ior, mdbx_filehandle_t fd); -typedef struct iov_ctx iov_ctx_t; -MDBX_INTERNAL_FUNC void osal_ioring_walk( - osal_ioring_t *ior, iov_ctx_t *ctx, - void (*callback)(iov_ctx_t *ctx, size_t offset, void *data, size_t bytes)); +MDBX_INTERNAL void osal_ioring_walk(osal_ioring_t *ior, iov_ctx_t *ctx, + void (*callback)(iov_ctx_t *ctx, size_t offset, void *data, size_t bytes)); -MDBX_MAYBE_UNUSED static inline unsigned -osal_ioring_left(const osal_ioring_t *ior) { - return ior->slots_left; -} +MDBX_MAYBE_UNUSED static inline unsigned osal_ioring_left(const osal_ioring_t *ior) { return ior->slots_left; } -MDBX_MAYBE_UNUSED static inline unsigned -osal_ioring_used(const osal_ioring_t *ior) { +MDBX_MAYBE_UNUSED static inline unsigned osal_ioring_used(const osal_ioring_t *ior) { return ior->allocated - ior->slots_left; } -MDBX_MAYBE_UNUSED static inline int -osal_ioring_prepare(osal_ioring_t *ior, size_t items, size_t bytes) { +MDBX_MAYBE_UNUSED static inline int osal_ioring_prepare(osal_ioring_t *ior, size_t items, size_t bytes) { items = (items > 32) ? items : 32; #if defined(_WIN32) || defined(_WIN64) if (ior->direct) { @@ -1416,14 +1335,12 @@ osal_ioring_prepare(osal_ioring_t *ior, size_t items, size_t bytes) { /*----------------------------------------------------------------------------*/ /* libc compatibility stuff */ -#if (!defined(__GLIBC__) && __GLIBC_PREREQ(2, 1)) && \ - (defined(_GNU_SOURCE) || defined(_BSD_SOURCE)) +#if (!defined(__GLIBC__) && __GLIBC_PREREQ(2, 1)) && (defined(_GNU_SOURCE) || defined(_BSD_SOURCE)) #define osal_asprintf asprintf #define osal_vasprintf vasprintf #else -MDBX_MAYBE_UNUSED MDBX_INTERNAL_FUNC - MDBX_PRINTF_ARGS(2, 3) int osal_asprintf(char **strp, const char *fmt, ...); -MDBX_INTERNAL_FUNC int osal_vasprintf(char **strp, const char *fmt, va_list ap); +MDBX_MAYBE_UNUSED MDBX_INTERNAL MDBX_PRINTF_ARGS(2, 3) int osal_asprintf(char **strp, const char *fmt, ...); +MDBX_INTERNAL int osal_vasprintf(char **strp, const char *fmt, va_list ap); #endif #if !defined(MADV_DODUMP) && defined(MADV_CORE) @@ -1434,8 +1351,7 @@ MDBX_INTERNAL_FUNC int osal_vasprintf(char **strp, const char *fmt, va_list ap); #define MADV_DONTDUMP MADV_NOCORE #endif /* MADV_NOCORE -> MADV_DONTDUMP */ -MDBX_MAYBE_UNUSED MDBX_INTERNAL_FUNC void osal_jitter(bool tiny); -MDBX_MAYBE_UNUSED static __inline void jitter4testing(bool tiny); +MDBX_MAYBE_UNUSED MDBX_INTERNAL void osal_jitter(bool tiny); /* max bytes to write in one call */ #if defined(_WIN64) @@ -1445,14 +1361,12 @@ MDBX_MAYBE_UNUSED static __inline void jitter4testing(bool tiny); #else #define MAX_WRITE UINT32_C(0x3f000000) -#if defined(F_GETLK64) && defined(F_SETLK64) && defined(F_SETLKW64) && \ - !defined(__ANDROID_API__) +#if defined(F_GETLK64) && defined(F_SETLK64) && defined(F_SETLKW64) && !defined(__ANDROID_API__) #define MDBX_F_SETLK F_SETLK64 #define MDBX_F_SETLKW F_SETLKW64 #define MDBX_F_GETLK F_GETLK64 -#if (__GLIBC_PREREQ(2, 28) && \ - (defined(__USE_LARGEFILE64) || defined(__LARGEFILE64_SOURCE) || \ - defined(_USE_LARGEFILE64) || defined(_LARGEFILE64_SOURCE))) || \ +#if (__GLIBC_PREREQ(2, 28) && (defined(__USE_LARGEFILE64) || defined(__LARGEFILE64_SOURCE) || \ + defined(_USE_LARGEFILE64) || defined(_LARGEFILE64_SOURCE))) || \ defined(fcntl64) #define MDBX_FCNTL fcntl64 #else @@ -1470,8 +1384,7 @@ MDBX_MAYBE_UNUSED static __inline void jitter4testing(bool tiny); #define MDBX_STRUCT_FLOCK struct flock #endif /* MDBX_F_SETLK, MDBX_F_SETLKW, MDBX_F_GETLK */ -#if defined(F_OFD_SETLK64) && defined(F_OFD_SETLKW64) && \ - defined(F_OFD_GETLK64) && !defined(__ANDROID_API__) +#if defined(F_OFD_SETLK64) && defined(F_OFD_SETLKW64) && defined(F_OFD_GETLK64) && !defined(__ANDROID_API__) #define MDBX_F_OFD_SETLK F_OFD_SETLK64 #define MDBX_F_OFD_SETLKW F_OFD_SETLKW64 #define MDBX_F_OFD_GETLK F_OFD_GETLK64 @@ -1480,23 +1393,17 @@ MDBX_MAYBE_UNUSED static __inline void jitter4testing(bool tiny); #define MDBX_F_OFD_SETLKW F_OFD_SETLKW #define MDBX_F_OFD_GETLK F_OFD_GETLK #ifndef OFF_T_MAX -#define OFF_T_MAX \ - (((sizeof(off_t) > 4) ? INT64_MAX : INT32_MAX) & ~(size_t)0xFffff) +#define OFF_T_MAX (((sizeof(off_t) > 4) ? INT64_MAX : INT32_MAX) & ~(size_t)0xFffff) #endif /* OFF_T_MAX */ #endif /* MDBX_F_OFD_SETLK64, MDBX_F_OFD_SETLKW64, MDBX_F_OFD_GETLK64 */ -#endif - -#if defined(__linux__) || defined(__gnu_linux__) -MDBX_INTERNAL_VAR uint32_t linux_kernel_version; -MDBX_INTERNAL_VAR bool mdbx_RunningOnWSL1 /* Windows Subsystem 1 for Linux */; -#endif /* Linux */ +#endif /* !Windows */ #ifndef osal_strdup LIBMDBX_API char *osal_strdup(const char *str); #endif -MDBX_MAYBE_UNUSED static __inline int osal_get_errno(void) { +MDBX_MAYBE_UNUSED static inline int osal_get_errno(void) { #if defined(_WIN32) || defined(_WIN64) DWORD rc = GetLastError(); #else @@ -1506,40 +1413,32 @@ MDBX_MAYBE_UNUSED static __inline int osal_get_errno(void) { } #ifndef osal_memalign_alloc -MDBX_INTERNAL_FUNC int osal_memalign_alloc(size_t alignment, size_t bytes, - void **result); +MDBX_INTERNAL int osal_memalign_alloc(size_t alignment, size_t bytes, void **result); #endif #ifndef osal_memalign_free -MDBX_INTERNAL_FUNC void osal_memalign_free(void *ptr); -#endif - -MDBX_INTERNAL_FUNC int osal_condpair_init(osal_condpair_t *condpair); -MDBX_INTERNAL_FUNC int osal_condpair_lock(osal_condpair_t *condpair); -MDBX_INTERNAL_FUNC int osal_condpair_unlock(osal_condpair_t *condpair); -MDBX_INTERNAL_FUNC int osal_condpair_signal(osal_condpair_t *condpair, - bool part); -MDBX_INTERNAL_FUNC int osal_condpair_wait(osal_condpair_t *condpair, bool part); -MDBX_INTERNAL_FUNC int osal_condpair_destroy(osal_condpair_t *condpair); - -MDBX_INTERNAL_FUNC int osal_fastmutex_init(osal_fastmutex_t *fastmutex); -MDBX_INTERNAL_FUNC int osal_fastmutex_acquire(osal_fastmutex_t *fastmutex); -MDBX_INTERNAL_FUNC int osal_fastmutex_release(osal_fastmutex_t *fastmutex); -MDBX_INTERNAL_FUNC int osal_fastmutex_destroy(osal_fastmutex_t *fastmutex); - -MDBX_INTERNAL_FUNC int osal_pwritev(mdbx_filehandle_t fd, struct iovec *iov, - size_t sgvcnt, uint64_t offset); -MDBX_INTERNAL_FUNC int osal_pread(mdbx_filehandle_t fd, void *buf, size_t count, - uint64_t offset); -MDBX_INTERNAL_FUNC int osal_pwrite(mdbx_filehandle_t fd, const void *buf, - size_t count, uint64_t offset); -MDBX_INTERNAL_FUNC int osal_write(mdbx_filehandle_t fd, const void *buf, - size_t count); - -MDBX_INTERNAL_FUNC int -osal_thread_create(osal_thread_t *thread, - THREAD_RESULT(THREAD_CALL *start_routine)(void *), - void *arg); -MDBX_INTERNAL_FUNC int osal_thread_join(osal_thread_t thread); +MDBX_INTERNAL void osal_memalign_free(void *ptr); +#endif + +MDBX_INTERNAL int osal_condpair_init(osal_condpair_t *condpair); +MDBX_INTERNAL int osal_condpair_lock(osal_condpair_t *condpair); +MDBX_INTERNAL int osal_condpair_unlock(osal_condpair_t *condpair); +MDBX_INTERNAL int osal_condpair_signal(osal_condpair_t *condpair, bool part); +MDBX_INTERNAL int osal_condpair_wait(osal_condpair_t *condpair, bool part); +MDBX_INTERNAL int osal_condpair_destroy(osal_condpair_t *condpair); + +MDBX_INTERNAL int osal_fastmutex_init(osal_fastmutex_t *fastmutex); +MDBX_INTERNAL int osal_fastmutex_acquire(osal_fastmutex_t *fastmutex); +MDBX_INTERNAL int osal_fastmutex_release(osal_fastmutex_t *fastmutex); +MDBX_INTERNAL int osal_fastmutex_destroy(osal_fastmutex_t *fastmutex); + +MDBX_INTERNAL int osal_pwritev(mdbx_filehandle_t fd, struct iovec *iov, size_t sgvcnt, uint64_t offset); +MDBX_INTERNAL int osal_pread(mdbx_filehandle_t fd, void *buf, size_t count, uint64_t offset); +MDBX_INTERNAL int osal_pwrite(mdbx_filehandle_t fd, const void *buf, size_t count, uint64_t offset); +MDBX_INTERNAL int osal_write(mdbx_filehandle_t fd, const void *buf, size_t count); + +MDBX_INTERNAL int osal_thread_create(osal_thread_t *thread, THREAD_RESULT(THREAD_CALL *start_routine)(void *), + void *arg); +MDBX_INTERNAL int osal_thread_join(osal_thread_t thread); enum osal_syncmode_bits { MDBX_SYNC_NONE = 0, @@ -1549,11 +1448,10 @@ enum osal_syncmode_bits { MDBX_SYNC_IODQ = 8 }; -MDBX_INTERNAL_FUNC int osal_fsync(mdbx_filehandle_t fd, - const enum osal_syncmode_bits mode_bits); -MDBX_INTERNAL_FUNC int osal_ftruncate(mdbx_filehandle_t fd, uint64_t length); -MDBX_INTERNAL_FUNC int osal_fseek(mdbx_filehandle_t fd, uint64_t pos); -MDBX_INTERNAL_FUNC int osal_filesize(mdbx_filehandle_t fd, uint64_t *length); +MDBX_INTERNAL int osal_fsync(mdbx_filehandle_t fd, const enum osal_syncmode_bits mode_bits); +MDBX_INTERNAL int osal_ftruncate(mdbx_filehandle_t fd, uint64_t length); +MDBX_INTERNAL int osal_fseek(mdbx_filehandle_t fd, uint64_t pos); +MDBX_INTERNAL int osal_filesize(mdbx_filehandle_t fd, uint64_t *length); enum osal_openfile_purpose { MDBX_OPEN_DXB_READ, @@ -1568,7 +1466,7 @@ enum osal_openfile_purpose { MDBX_OPEN_DELETE }; -MDBX_MAYBE_UNUSED static __inline bool osal_isdirsep(pathchar_t c) { +MDBX_MAYBE_UNUSED static inline bool osal_isdirsep(pathchar_t c) { return #if defined(_WIN32) || defined(_WIN64) c == '\\' || @@ -1576,50 +1474,39 @@ MDBX_MAYBE_UNUSED static __inline bool osal_isdirsep(pathchar_t c) { c == '/'; } -MDBX_INTERNAL_FUNC bool osal_pathequal(const pathchar_t *l, const pathchar_t *r, - size_t len); -MDBX_INTERNAL_FUNC pathchar_t *osal_fileext(const pathchar_t *pathname, - size_t len); -MDBX_INTERNAL_FUNC int osal_fileexists(const pathchar_t *pathname); -MDBX_INTERNAL_FUNC int osal_openfile(const enum osal_openfile_purpose purpose, - const MDBX_env *env, - const pathchar_t *pathname, - mdbx_filehandle_t *fd, - mdbx_mode_t unix_mode_bits); -MDBX_INTERNAL_FUNC int osal_closefile(mdbx_filehandle_t fd); -MDBX_INTERNAL_FUNC int osal_removefile(const pathchar_t *pathname); -MDBX_INTERNAL_FUNC int osal_removedirectory(const pathchar_t *pathname); -MDBX_INTERNAL_FUNC int osal_is_pipe(mdbx_filehandle_t fd); -MDBX_INTERNAL_FUNC int osal_lockfile(mdbx_filehandle_t fd, bool wait); +MDBX_INTERNAL bool osal_pathequal(const pathchar_t *l, const pathchar_t *r, size_t len); +MDBX_INTERNAL pathchar_t *osal_fileext(const pathchar_t *pathname, size_t len); +MDBX_INTERNAL int osal_fileexists(const pathchar_t *pathname); +MDBX_INTERNAL int osal_openfile(const enum osal_openfile_purpose purpose, const MDBX_env *env, + const pathchar_t *pathname, mdbx_filehandle_t *fd, mdbx_mode_t unix_mode_bits); +MDBX_INTERNAL int osal_closefile(mdbx_filehandle_t fd); +MDBX_INTERNAL int osal_removefile(const pathchar_t *pathname); +MDBX_INTERNAL int osal_removedirectory(const pathchar_t *pathname); +MDBX_INTERNAL int osal_is_pipe(mdbx_filehandle_t fd); +MDBX_INTERNAL int osal_lockfile(mdbx_filehandle_t fd, bool wait); #define MMAP_OPTION_TRUNCATE 1 #define MMAP_OPTION_SEMAPHORE 2 -MDBX_INTERNAL_FUNC int osal_mmap(const int flags, osal_mmap_t *map, size_t size, - const size_t limit, const unsigned options); -MDBX_INTERNAL_FUNC int osal_munmap(osal_mmap_t *map); +MDBX_INTERNAL int osal_mmap(const int flags, osal_mmap_t *map, size_t size, const size_t limit, const unsigned options, + const pathchar_t *pathname4logging); +MDBX_INTERNAL int osal_munmap(osal_mmap_t *map); #define MDBX_MRESIZE_MAY_MOVE 0x00000100 #define MDBX_MRESIZE_MAY_UNMAP 0x00000200 -MDBX_INTERNAL_FUNC int osal_mresize(const int flags, osal_mmap_t *map, - size_t size, size_t limit); +MDBX_INTERNAL int osal_mresize(const int flags, osal_mmap_t *map, size_t size, size_t limit); #if defined(_WIN32) || defined(_WIN64) typedef struct { unsigned limit, count; HANDLE handles[31]; } mdbx_handle_array_t; -MDBX_INTERNAL_FUNC int -osal_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array); -MDBX_INTERNAL_FUNC int -osal_resume_threads_after_remap(mdbx_handle_array_t *array); +MDBX_INTERNAL int osal_suspend_threads_before_remap(MDBX_env *env, mdbx_handle_array_t **array); +MDBX_INTERNAL int osal_resume_threads_after_remap(mdbx_handle_array_t *array); #endif /* Windows */ -MDBX_INTERNAL_FUNC int osal_msync(const osal_mmap_t *map, size_t offset, - size_t length, - enum osal_syncmode_bits mode_bits); -MDBX_INTERNAL_FUNC int osal_check_fs_rdonly(mdbx_filehandle_t handle, - const pathchar_t *pathname, - int err); -MDBX_INTERNAL_FUNC int osal_check_fs_incore(mdbx_filehandle_t handle); - -MDBX_MAYBE_UNUSED static __inline uint32_t osal_getpid(void) { +MDBX_INTERNAL int osal_msync(const osal_mmap_t *map, size_t offset, size_t length, enum osal_syncmode_bits mode_bits); +MDBX_INTERNAL int osal_check_fs_rdonly(mdbx_filehandle_t handle, const pathchar_t *pathname, int err); +MDBX_INTERNAL int osal_check_fs_incore(mdbx_filehandle_t handle); +MDBX_INTERNAL int osal_check_fs_local(mdbx_filehandle_t handle, int flags); + +MDBX_MAYBE_UNUSED static inline uint32_t osal_getpid(void) { STATIC_ASSERT(sizeof(mdbx_pid_t) <= sizeof(uint32_t)); #if defined(_WIN32) || defined(_WIN64) return GetCurrentProcessId(); @@ -1629,7 +1516,7 @@ MDBX_MAYBE_UNUSED static __inline uint32_t osal_getpid(void) { #endif } -MDBX_MAYBE_UNUSED static __inline uintptr_t osal_thread_self(void) { +MDBX_MAYBE_UNUSED static inline uintptr_t osal_thread_self(void) { mdbx_tid_t thunk; STATIC_ASSERT(sizeof(uintptr_t) >= sizeof(thunk)); #if defined(_WIN32) || defined(_WIN64) @@ -1642,274 +1529,51 @@ MDBX_MAYBE_UNUSED static __inline uintptr_t osal_thread_self(void) { #if !defined(_WIN32) && !defined(_WIN64) #if defined(__ANDROID_API__) || defined(ANDROID) || defined(BIONIC) -MDBX_INTERNAL_FUNC int osal_check_tid4bionic(void); +MDBX_INTERNAL int osal_check_tid4bionic(void); #else -static __inline int osal_check_tid4bionic(void) { return 0; } +static inline int osal_check_tid4bionic(void) { return 0; } #endif /* __ANDROID_API__ || ANDROID) || BIONIC */ -MDBX_MAYBE_UNUSED static __inline int -osal_pthread_mutex_lock(pthread_mutex_t *mutex) { +MDBX_MAYBE_UNUSED static inline int osal_pthread_mutex_lock(pthread_mutex_t *mutex) { int err = osal_check_tid4bionic(); return unlikely(err) ? err : pthread_mutex_lock(mutex); } #endif /* !Windows */ -MDBX_INTERNAL_FUNC uint64_t osal_monotime(void); -MDBX_INTERNAL_FUNC uint64_t osal_cputime(size_t *optional_page_faults); -MDBX_INTERNAL_FUNC uint64_t osal_16dot16_to_monotime(uint32_t seconds_16dot16); -MDBX_INTERNAL_FUNC uint32_t osal_monotime_to_16dot16(uint64_t monotime); +MDBX_INTERNAL uint64_t osal_monotime(void); +MDBX_INTERNAL uint64_t osal_cputime(size_t *optional_page_faults); +MDBX_INTERNAL uint64_t osal_16dot16_to_monotime(uint32_t seconds_16dot16); +MDBX_INTERNAL uint32_t osal_monotime_to_16dot16(uint64_t monotime); -MDBX_MAYBE_UNUSED static inline uint32_t -osal_monotime_to_16dot16_noUnderflow(uint64_t monotime) { +MDBX_MAYBE_UNUSED static inline uint32_t osal_monotime_to_16dot16_noUnderflow(uint64_t monotime) { uint32_t seconds_16dot16 = osal_monotime_to_16dot16(monotime); return seconds_16dot16 ? seconds_16dot16 : /* fix underflow */ (monotime > 0); } -MDBX_INTERNAL_FUNC bin128_t osal_bootid(void); /*----------------------------------------------------------------------------*/ -/* lck stuff */ - -/// \brief Initialization of synchronization primitives linked with MDBX_env -/// instance both in LCK-file and within the current process. -/// \param -/// global_uniqueness_flag = true - denotes that there are no other processes -/// working with DB and LCK-file. Thus the function MUST initialize -/// shared synchronization objects in memory-mapped LCK-file. -/// global_uniqueness_flag = false - denotes that at least one process is -/// already working with DB and LCK-file, including the case when DB -/// has already been opened in the current process. Thus the function -/// MUST NOT initialize shared synchronization objects in memory-mapped -/// LCK-file that are already in use. -/// \return Error code or zero on success. -MDBX_INTERNAL_FUNC int osal_lck_init(MDBX_env *env, - MDBX_env *inprocess_neighbor, - int global_uniqueness_flag); - -/// \brief Disconnects from shared interprocess objects and destructs -/// synchronization objects linked with MDBX_env instance -/// within the current process. -/// \param -/// inprocess_neighbor = NULL - if the current process does not have other -/// instances of MDBX_env linked with the DB being closed. -/// Thus the function MUST check for other processes working with DB or -/// LCK-file, and keep or destroy shared synchronization objects in -/// memory-mapped LCK-file depending on the result. -/// inprocess_neighbor = not-NULL - pointer to another instance of MDBX_env -/// (anyone of there is several) working with DB or LCK-file within the -/// current process. Thus the function MUST NOT try to acquire exclusive -/// lock and/or try to destruct shared synchronization objects linked with -/// DB or LCK-file. Moreover, the implementation MUST ensure correct work -/// of other instances of MDBX_env within the current process, e.g. -/// restore POSIX-fcntl locks after the closing of file descriptors. -/// \return Error code (MDBX_PANIC) or zero on success. -MDBX_INTERNAL_FUNC int osal_lck_destroy(MDBX_env *env, - MDBX_env *inprocess_neighbor); - -/// \brief Connects to shared interprocess locking objects and tries to acquire -/// the maximum lock level (shared if exclusive is not available) -/// Depending on implementation or/and platform (Windows) this function may -/// acquire the non-OS super-level lock (e.g. for shared synchronization -/// objects initialization), which will be downgraded to OS-exclusive or -/// shared via explicit calling of osal_lck_downgrade(). -/// \return -/// MDBX_RESULT_TRUE (-1) - if an exclusive lock was acquired and thus -/// the current process is the first and only after the last use of DB. -/// MDBX_RESULT_FALSE (0) - if a shared lock was acquired and thus -/// DB has already been opened and now is used by other processes. -/// Otherwise (not 0 and not -1) - error code. -MDBX_INTERNAL_FUNC int osal_lck_seize(MDBX_env *env); - -/// \brief Downgrades the level of initially acquired lock to -/// operational level specified by argument. The reason for such downgrade: -/// - unblocking of other processes that are waiting for access, i.e. -/// if (env->me_flags & MDBX_EXCLUSIVE) != 0, then other processes -/// should be made aware that access is unavailable rather than -/// wait for it. -/// - freeing locks that interfere file operation (especially for Windows) -/// (env->me_flags & MDBX_EXCLUSIVE) == 0 - downgrade to shared lock. -/// (env->me_flags & MDBX_EXCLUSIVE) != 0 - downgrade to exclusive -/// operational lock. -/// \return Error code or zero on success -MDBX_INTERNAL_FUNC int osal_lck_downgrade(MDBX_env *env); - -/// \brief Locks LCK-file or/and table of readers for (de)registering. -/// \return Error code or zero on success -MDBX_INTERNAL_FUNC int osal_rdt_lock(MDBX_env *env); - -/// \brief Unlocks LCK-file or/and table of readers after (de)registering. -MDBX_INTERNAL_FUNC void osal_rdt_unlock(MDBX_env *env); - -/// \brief Acquires lock for DB change (on writing transaction start) -/// Reading transactions will not be blocked. -/// Declared as LIBMDBX_API because it is used in mdbx_chk. -/// \return Error code or zero on success -LIBMDBX_API int mdbx_txn_lock(MDBX_env *env, bool dont_wait); - -/// \brief Releases lock once DB changes is made (after writing transaction -/// has finished). -/// Declared as LIBMDBX_API because it is used in mdbx_chk. -LIBMDBX_API void mdbx_txn_unlock(MDBX_env *env); - -/// \brief Sets alive-flag of reader presence (indicative lock) for PID of -/// the current process. The function does no more than needed for -/// the correct working of osal_rpid_check() in other processes. -/// \return Error code or zero on success -MDBX_INTERNAL_FUNC int osal_rpid_set(MDBX_env *env); - -/// \brief Resets alive-flag of reader presence (indicative lock) -/// for PID of the current process. The function does no more than needed -/// for the correct working of osal_rpid_check() in other processes. -/// \return Error code or zero on success -MDBX_INTERNAL_FUNC int osal_rpid_clear(MDBX_env *env); - -/// \brief Checks for reading process status with the given pid with help of -/// alive-flag of presence (indicative lock) or using another way. -/// \return -/// MDBX_RESULT_TRUE (-1) - if the reader process with the given PID is alive -/// and working with DB (indicative lock is present). -/// MDBX_RESULT_FALSE (0) - if the reader process with the given PID is absent -/// or not working with DB (indicative lock is not present). -/// Otherwise (not 0 and not -1) - error code. -MDBX_INTERNAL_FUNC int osal_rpid_check(MDBX_env *env, uint32_t pid); - -#if defined(_WIN32) || defined(_WIN64) - -MDBX_INTERNAL_FUNC int osal_mb2w(const char *const src, wchar_t **const pdst); - -typedef void(WINAPI *osal_srwlock_t_function)(osal_srwlock_t *); -MDBX_INTERNAL_VAR osal_srwlock_t_function osal_srwlock_Init, - osal_srwlock_AcquireShared, osal_srwlock_ReleaseShared, - osal_srwlock_AcquireExclusive, osal_srwlock_ReleaseExclusive; - -#if _WIN32_WINNT < 0x0600 /* prior to Windows Vista */ -typedef enum _FILE_INFO_BY_HANDLE_CLASS { - FileBasicInfo, - FileStandardInfo, - FileNameInfo, - FileRenameInfo, - FileDispositionInfo, - FileAllocationInfo, - FileEndOfFileInfo, - FileStreamInfo, - FileCompressionInfo, - FileAttributeTagInfo, - FileIdBothDirectoryInfo, - FileIdBothDirectoryRestartInfo, - FileIoPriorityHintInfo, - FileRemoteProtocolInfo, - MaximumFileInfoByHandleClass -} FILE_INFO_BY_HANDLE_CLASS, - *PFILE_INFO_BY_HANDLE_CLASS; - -typedef struct _FILE_END_OF_FILE_INFO { - LARGE_INTEGER EndOfFile; -} FILE_END_OF_FILE_INFO, *PFILE_END_OF_FILE_INFO; - -#define REMOTE_PROTOCOL_INFO_FLAG_LOOPBACK 0x00000001 -#define REMOTE_PROTOCOL_INFO_FLAG_OFFLINE 0x00000002 - -typedef struct _FILE_REMOTE_PROTOCOL_INFO { - USHORT StructureVersion; - USHORT StructureSize; - DWORD Protocol; - USHORT ProtocolMajorVersion; - USHORT ProtocolMinorVersion; - USHORT ProtocolRevision; - USHORT Reserved; - DWORD Flags; - struct { - DWORD Reserved[8]; - } GenericReserved; - struct { - DWORD Reserved[16]; - } ProtocolSpecificReserved; -} FILE_REMOTE_PROTOCOL_INFO, *PFILE_REMOTE_PROTOCOL_INFO; - -#endif /* _WIN32_WINNT < 0x0600 (prior to Windows Vista) */ - -typedef BOOL(WINAPI *MDBX_GetFileInformationByHandleEx)( - _In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass, - _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize); -MDBX_INTERNAL_VAR MDBX_GetFileInformationByHandleEx - mdbx_GetFileInformationByHandleEx; - -typedef BOOL(WINAPI *MDBX_GetVolumeInformationByHandleW)( - _In_ HANDLE hFile, _Out_opt_ LPWSTR lpVolumeNameBuffer, - _In_ DWORD nVolumeNameSize, _Out_opt_ LPDWORD lpVolumeSerialNumber, - _Out_opt_ LPDWORD lpMaximumComponentLength, - _Out_opt_ LPDWORD lpFileSystemFlags, - _Out_opt_ LPWSTR lpFileSystemNameBuffer, _In_ DWORD nFileSystemNameSize); -MDBX_INTERNAL_VAR MDBX_GetVolumeInformationByHandleW - mdbx_GetVolumeInformationByHandleW; - -typedef DWORD(WINAPI *MDBX_GetFinalPathNameByHandleW)(_In_ HANDLE hFile, - _Out_ LPWSTR lpszFilePath, - _In_ DWORD cchFilePath, - _In_ DWORD dwFlags); -MDBX_INTERNAL_VAR MDBX_GetFinalPathNameByHandleW mdbx_GetFinalPathNameByHandleW; - -typedef BOOL(WINAPI *MDBX_SetFileInformationByHandle)( - _In_ HANDLE hFile, _In_ FILE_INFO_BY_HANDLE_CLASS FileInformationClass, - _Out_ LPVOID lpFileInformation, _In_ DWORD dwBufferSize); -MDBX_INTERNAL_VAR MDBX_SetFileInformationByHandle - mdbx_SetFileInformationByHandle; - -typedef NTSTATUS(NTAPI *MDBX_NtFsControlFile)( - IN HANDLE FileHandle, IN OUT HANDLE Event, - IN OUT PVOID /* PIO_APC_ROUTINE */ ApcRoutine, IN OUT PVOID ApcContext, - OUT PIO_STATUS_BLOCK IoStatusBlock, IN ULONG FsControlCode, - IN OUT PVOID InputBuffer, IN ULONG InputBufferLength, - OUT OPTIONAL PVOID OutputBuffer, IN ULONG OutputBufferLength); -MDBX_INTERNAL_VAR MDBX_NtFsControlFile mdbx_NtFsControlFile; - -typedef uint64_t(WINAPI *MDBX_GetTickCount64)(void); -MDBX_INTERNAL_VAR MDBX_GetTickCount64 mdbx_GetTickCount64; - -#if !defined(_WIN32_WINNT_WIN8) || _WIN32_WINNT < _WIN32_WINNT_WIN8 -typedef struct _WIN32_MEMORY_RANGE_ENTRY { - PVOID VirtualAddress; - SIZE_T NumberOfBytes; -} WIN32_MEMORY_RANGE_ENTRY, *PWIN32_MEMORY_RANGE_ENTRY; -#endif /* Windows 8.x */ - -typedef BOOL(WINAPI *MDBX_PrefetchVirtualMemory)( - HANDLE hProcess, ULONG_PTR NumberOfEntries, - PWIN32_MEMORY_RANGE_ENTRY VirtualAddresses, ULONG Flags); -MDBX_INTERNAL_VAR MDBX_PrefetchVirtualMemory mdbx_PrefetchVirtualMemory; - -typedef enum _SECTION_INHERIT { ViewShare = 1, ViewUnmap = 2 } SECTION_INHERIT; - -typedef NTSTATUS(NTAPI *MDBX_NtExtendSection)(IN HANDLE SectionHandle, - IN PLARGE_INTEGER NewSectionSize); -MDBX_INTERNAL_VAR MDBX_NtExtendSection mdbx_NtExtendSection; - -static __inline bool mdbx_RunningUnderWine(void) { - return !mdbx_NtExtendSection; -} - -typedef LSTATUS(WINAPI *MDBX_RegGetValueA)(HKEY hkey, LPCSTR lpSubKey, - LPCSTR lpValue, DWORD dwFlags, - LPDWORD pdwType, PVOID pvData, - LPDWORD pcbData); -MDBX_INTERNAL_VAR MDBX_RegGetValueA mdbx_RegGetValueA; -NTSYSAPI ULONG RtlRandomEx(PULONG Seed); - -typedef BOOL(WINAPI *MDBX_SetFileIoOverlappedRange)(HANDLE FileHandle, - PUCHAR OverlappedRangeStart, - ULONG Length); -MDBX_INTERNAL_VAR MDBX_SetFileIoOverlappedRange mdbx_SetFileIoOverlappedRange; +MDBX_INTERNAL void osal_ctor(void); +MDBX_INTERNAL void osal_dtor(void); +#if defined(_WIN32) || defined(_WIN64) +MDBX_INTERNAL int osal_mb2w(const char *const src, wchar_t **const pdst); #endif /* Windows */ -#endif /* !__cplusplus */ +typedef union bin128 { + __anonymous_struct_extension__ struct { + uint64_t x, y; + }; + __anonymous_struct_extension__ struct { + uint32_t a, b, c, d; + }; +} bin128_t; + +MDBX_INTERNAL bin128_t osal_guid(const MDBX_env *); /*----------------------------------------------------------------------------*/ -MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static __always_inline uint64_t -osal_bswap64(uint64_t v) { -#if __GNUC_PREREQ(4, 4) || __CLANG_PREREQ(4, 0) || \ - __has_builtin(__builtin_bswap64) +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint64_t osal_bswap64(uint64_t v) { +#if __GNUC_PREREQ(4, 4) || __CLANG_PREREQ(4, 0) || __has_builtin(__builtin_bswap64) return __builtin_bswap64(v); #elif defined(_MSC_VER) && !defined(__clang__) return _byteswap_uint64(v); @@ -1918,19 +1582,14 @@ osal_bswap64(uint64_t v) { #elif defined(bswap_64) return bswap_64(v); #else - return v << 56 | v >> 56 | ((v << 40) & UINT64_C(0x00ff000000000000)) | - ((v << 24) & UINT64_C(0x0000ff0000000000)) | - ((v << 8) & UINT64_C(0x000000ff00000000)) | - ((v >> 8) & UINT64_C(0x00000000ff000000)) | - ((v >> 24) & UINT64_C(0x0000000000ff0000)) | - ((v >> 40) & UINT64_C(0x000000000000ff00)); + return v << 56 | v >> 56 | ((v << 40) & UINT64_C(0x00ff000000000000)) | ((v << 24) & UINT64_C(0x0000ff0000000000)) | + ((v << 8) & UINT64_C(0x000000ff00000000)) | ((v >> 8) & UINT64_C(0x00000000ff000000)) | + ((v >> 24) & UINT64_C(0x0000000000ff0000)) | ((v >> 40) & UINT64_C(0x000000000000ff00)); #endif } -MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static __always_inline uint32_t -osal_bswap32(uint32_t v) { -#if __GNUC_PREREQ(4, 4) || __CLANG_PREREQ(4, 0) || \ - __has_builtin(__builtin_bswap32) +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint32_t osal_bswap32(uint32_t v) { +#if __GNUC_PREREQ(4, 4) || __CLANG_PREREQ(4, 0) || __has_builtin(__builtin_bswap32) return __builtin_bswap32(v); #elif defined(_MSC_VER) && !defined(__clang__) return _byteswap_ulong(v); @@ -1939,50 +1598,14 @@ osal_bswap32(uint32_t v) { #elif defined(bswap_32) return bswap_32(v); #else - return v << 24 | v >> 24 | ((v << 8) & UINT32_C(0x00ff0000)) | - ((v >> 8) & UINT32_C(0x0000ff00)); + return v << 24 | v >> 24 | ((v << 8) & UINT32_C(0x00ff0000)) | ((v >> 8) & UINT32_C(0x0000ff00)); #endif } -/*----------------------------------------------------------------------------*/ - -#if defined(_MSC_VER) && _MSC_VER >= 1900 -/* LY: MSVC 2015/2017/2019 has buggy/inconsistent PRIuPTR/PRIxPTR macros - * for internal format-args checker. */ -#undef PRIuPTR -#undef PRIiPTR -#undef PRIdPTR -#undef PRIxPTR -#define PRIuPTR "Iu" -#define PRIiPTR "Ii" -#define PRIdPTR "Id" -#define PRIxPTR "Ix" -#define PRIuSIZE "zu" -#define PRIiSIZE "zi" -#define PRIdSIZE "zd" -#define PRIxSIZE "zx" -#endif /* fix PRI*PTR for _MSC_VER */ - -#ifndef PRIuSIZE -#define PRIuSIZE PRIuPTR -#define PRIiSIZE PRIiPTR -#define PRIdSIZE PRIdPTR -#define PRIxSIZE PRIxPTR -#endif /* PRI*SIZE macros for MSVC */ - -#ifdef _MSC_VER -#pragma warning(pop) -#endif - -#define mdbx_sourcery_anchor XCONCAT(mdbx_sourcery_, MDBX_BUILD_SOURCERY) -#if defined(xMDBX_TOOLS) -extern LIBMDBX_API const char *const mdbx_sourcery_anchor; -#endif - /******************************************************************************* - ******************************************************************************* ******************************************************************************* * + * BUILD TIME * * #### ##### ##### # #### # # #### * # # # # # # # # ## # # @@ -2003,23 +1626,15 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Using fsync() with chance of data lost on power failure */ #define MDBX_OSX_WANNA_SPEED 1 -#ifndef MDBX_OSX_SPEED_INSTEADOF_DURABILITY +#ifndef MDBX_APPLE_SPEED_INSTEADOF_DURABILITY /** Choices \ref MDBX_OSX_WANNA_DURABILITY or \ref MDBX_OSX_WANNA_SPEED * for OSX & iOS */ -#define MDBX_OSX_SPEED_INSTEADOF_DURABILITY MDBX_OSX_WANNA_DURABILITY -#endif /* MDBX_OSX_SPEED_INSTEADOF_DURABILITY */ - -/** Controls using of POSIX' madvise() and/or similar hints. */ -#ifndef MDBX_ENABLE_MADVISE -#define MDBX_ENABLE_MADVISE 1 -#elif !(MDBX_ENABLE_MADVISE == 0 || MDBX_ENABLE_MADVISE == 1) -#error MDBX_ENABLE_MADVISE must be defined as 0 or 1 -#endif /* MDBX_ENABLE_MADVISE */ +#define MDBX_APPLE_SPEED_INSTEADOF_DURABILITY MDBX_OSX_WANNA_DURABILITY +#endif /* MDBX_APPLE_SPEED_INSTEADOF_DURABILITY */ /** Controls checking PID against reuse DB environment after the fork() */ #ifndef MDBX_ENV_CHECKPID -#if (defined(MADV_DONTFORK) && MDBX_ENABLE_MADVISE) || defined(_WIN32) || \ - defined(_WIN64) +#if defined(MADV_DONTFORK) || defined(_WIN32) || defined(_WIN64) /* PID check could be omitted: * - on Linux when madvise(MADV_DONTFORK) is available, i.e. after the fork() * mapped pages will not be available for child process. @@ -2048,8 +1663,7 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Does a system have battery-backed Real-Time Clock or just a fake. */ #ifndef MDBX_TRUST_RTC -#if defined(__linux__) || defined(__gnu_linux__) || defined(__NetBSD__) || \ - defined(__OpenBSD__) +#if defined(__linux__) || defined(__gnu_linux__) || defined(__NetBSD__) || defined(__OpenBSD__) #define MDBX_TRUST_RTC 0 /* a lot of embedded systems have a fake RTC */ #else #define MDBX_TRUST_RTC 1 @@ -2084,24 +1698,21 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Controls using Unix' mincore() to determine whether DB-pages * are resident in memory. */ -#ifndef MDBX_ENABLE_MINCORE +#ifndef MDBX_USE_MINCORE #if defined(MINCORE_INCORE) || !(defined(_WIN32) || defined(_WIN64)) -#define MDBX_ENABLE_MINCORE 1 +#define MDBX_USE_MINCORE 1 #else -#define MDBX_ENABLE_MINCORE 0 +#define MDBX_USE_MINCORE 0 #endif -#elif !(MDBX_ENABLE_MINCORE == 0 || MDBX_ENABLE_MINCORE == 1) -#error MDBX_ENABLE_MINCORE must be defined as 0 or 1 -#endif /* MDBX_ENABLE_MINCORE */ +#define MDBX_USE_MINCORE_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_USE_MINCORE) +#elif !(MDBX_USE_MINCORE == 0 || MDBX_USE_MINCORE == 1) +#error MDBX_USE_MINCORE must be defined as 0 or 1 +#endif /* MDBX_USE_MINCORE */ /** Enables chunking long list of retired pages during huge transactions commit * to avoid use sequences of pages. */ #ifndef MDBX_ENABLE_BIGFOOT -#if MDBX_WORDBITS >= 64 || defined(DOXYGEN) #define MDBX_ENABLE_BIGFOOT 1 -#else -#define MDBX_ENABLE_BIGFOOT 0 -#endif #elif !(MDBX_ENABLE_BIGFOOT == 0 || MDBX_ENABLE_BIGFOOT == 1) #error MDBX_ENABLE_BIGFOOT must be defined as 0 or 1 #endif /* MDBX_ENABLE_BIGFOOT */ @@ -2116,25 +1727,27 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #ifndef MDBX_PNL_PREALLOC_FOR_RADIXSORT #define MDBX_PNL_PREALLOC_FOR_RADIXSORT 1 -#elif !(MDBX_PNL_PREALLOC_FOR_RADIXSORT == 0 || \ - MDBX_PNL_PREALLOC_FOR_RADIXSORT == 1) +#elif !(MDBX_PNL_PREALLOC_FOR_RADIXSORT == 0 || MDBX_PNL_PREALLOC_FOR_RADIXSORT == 1) #error MDBX_PNL_PREALLOC_FOR_RADIXSORT must be defined as 0 or 1 #endif /* MDBX_PNL_PREALLOC_FOR_RADIXSORT */ #ifndef MDBX_DPL_PREALLOC_FOR_RADIXSORT #define MDBX_DPL_PREALLOC_FOR_RADIXSORT 1 -#elif !(MDBX_DPL_PREALLOC_FOR_RADIXSORT == 0 || \ - MDBX_DPL_PREALLOC_FOR_RADIXSORT == 1) +#elif !(MDBX_DPL_PREALLOC_FOR_RADIXSORT == 0 || MDBX_DPL_PREALLOC_FOR_RADIXSORT == 1) #error MDBX_DPL_PREALLOC_FOR_RADIXSORT must be defined as 0 or 1 #endif /* MDBX_DPL_PREALLOC_FOR_RADIXSORT */ -/** Controls dirty pages tracking, spilling and persisting in MDBX_WRITEMAP - * mode. 0/OFF = Don't track dirty pages at all, don't spill ones, and use - * msync() to persist data. This is by-default on Linux and other systems where - * kernel provides properly LRU tracking and effective flushing on-demand. 1/ON - * = Tracking of dirty pages but with LRU labels for spilling and explicit - * persist ones by write(). This may be reasonable for systems which low - * performance of msync() and/or LRU tracking. */ +/** Controls dirty pages tracking, spilling and persisting in `MDBX_WRITEMAP` + * mode, i.e. disables in-memory database updating with consequent + * flush-to-disk/msync syscall. + * + * 0/OFF = Don't track dirty pages at all, don't spill ones, and use msync() to + * persist data. This is by-default on Linux and other systems where kernel + * provides properly LRU tracking and effective flushing on-demand. + * + * 1/ON = Tracking of dirty pages but with LRU labels for spilling and explicit + * persist ones by write(). This may be reasonable for goofy systems (Windows) + * which low performance of msync() and/or zany LRU tracking. */ #ifndef MDBX_AVOID_MSYNC #if defined(_WIN32) || defined(_WIN64) #define MDBX_AVOID_MSYNC 1 @@ -2145,6 +1758,22 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #error MDBX_AVOID_MSYNC must be defined as 0 or 1 #endif /* MDBX_AVOID_MSYNC */ +/** Управляет механизмом поддержки разреженных наборов DBI-хендлов для снижения + * накладных расходов при запуске и обработке транзакций. */ +#ifndef MDBX_ENABLE_DBI_SPARSE +#define MDBX_ENABLE_DBI_SPARSE 1 +#elif !(MDBX_ENABLE_DBI_SPARSE == 0 || MDBX_ENABLE_DBI_SPARSE == 1) +#error MDBX_ENABLE_DBI_SPARSE must be defined as 0 or 1 +#endif /* MDBX_ENABLE_DBI_SPARSE */ + +/** Управляет механизмом отложенного освобождения и поддержки пути быстрого + * открытия DBI-хендлов без захвата блокировок. */ +#ifndef MDBX_ENABLE_DBI_LOCKFREE +#define MDBX_ENABLE_DBI_LOCKFREE 1 +#elif !(MDBX_ENABLE_DBI_LOCKFREE == 0 || MDBX_ENABLE_DBI_LOCKFREE == 1) +#error MDBX_ENABLE_DBI_LOCKFREE must be defined as 0 or 1 +#endif /* MDBX_ENABLE_DBI_LOCKFREE */ + /** Controls sort order of internal page number lists. * This mostly experimental/advanced option with not for regular MDBX users. * \warning The database format depend on this option and libmdbx built with @@ -2157,7 +1786,11 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Avoid dependence from MSVC CRT and use ntdll.dll instead. */ #ifndef MDBX_WITHOUT_MSVC_CRT +#if defined(MDBX_BUILD_CXX) && !MDBX_BUILD_CXX #define MDBX_WITHOUT_MSVC_CRT 1 +#else +#define MDBX_WITHOUT_MSVC_CRT 0 +#endif #elif !(MDBX_WITHOUT_MSVC_CRT == 0 || MDBX_WITHOUT_MSVC_CRT == 1) #error MDBX_WITHOUT_MSVC_CRT must be defined as 0 or 1 #endif /* MDBX_WITHOUT_MSVC_CRT */ @@ -2165,12 +1798,11 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Size of buffer used during copying a environment/database file. */ #ifndef MDBX_ENVCOPY_WRITEBUF #define MDBX_ENVCOPY_WRITEBUF 1048576u -#elif MDBX_ENVCOPY_WRITEBUF < 65536u || MDBX_ENVCOPY_WRITEBUF > 1073741824u || \ - MDBX_ENVCOPY_WRITEBUF % 65536u +#elif MDBX_ENVCOPY_WRITEBUF < 65536u || MDBX_ENVCOPY_WRITEBUF > 1073741824u || MDBX_ENVCOPY_WRITEBUF % 65536u #error MDBX_ENVCOPY_WRITEBUF must be defined in range 65536..1073741824 and be multiple of 65536 #endif /* MDBX_ENVCOPY_WRITEBUF */ -/** Forces assertion checking */ +/** Forces assertion checking. */ #ifndef MDBX_FORCE_ASSERTIONS #define MDBX_FORCE_ASSERTIONS 0 #elif !(MDBX_FORCE_ASSERTIONS == 0 || MDBX_FORCE_ASSERTIONS == 1) @@ -2185,15 +1817,14 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #else #define MDBX_ASSUME_MALLOC_OVERHEAD (sizeof(void *) * 2u) #endif -#elif MDBX_ASSUME_MALLOC_OVERHEAD < 0 || MDBX_ASSUME_MALLOC_OVERHEAD > 64 || \ - MDBX_ASSUME_MALLOC_OVERHEAD % 4 +#elif MDBX_ASSUME_MALLOC_OVERHEAD < 0 || MDBX_ASSUME_MALLOC_OVERHEAD > 64 || MDBX_ASSUME_MALLOC_OVERHEAD % 4 #error MDBX_ASSUME_MALLOC_OVERHEAD must be defined in range 0..64 and be multiple of 4 #endif /* MDBX_ASSUME_MALLOC_OVERHEAD */ /** If defined then enables integration with Valgrind, * a memory analyzing tool. */ -#ifndef MDBX_USE_VALGRIND -#endif /* MDBX_USE_VALGRIND */ +#ifndef ENABLE_MEMCHECK +#endif /* ENABLE_MEMCHECK */ /** If defined then enables use C11 atomics, * otherwise detects ones availability automatically. */ @@ -2213,18 +1844,24 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #define MDBX_HAVE_BUILTIN_CPU_SUPPORTS 0 #elif defined(__e2k__) #define MDBX_HAVE_BUILTIN_CPU_SUPPORTS 0 -#elif __has_builtin(__builtin_cpu_supports) || \ - defined(__BUILTIN_CPU_SUPPORTS__) || \ +#elif __has_builtin(__builtin_cpu_supports) || defined(__BUILTIN_CPU_SUPPORTS__) || \ (defined(__ia32__) && __GNUC_PREREQ(4, 8) && __GLIBC_PREREQ(2, 23)) #define MDBX_HAVE_BUILTIN_CPU_SUPPORTS 1 #else #define MDBX_HAVE_BUILTIN_CPU_SUPPORTS 0 #endif -#elif !(MDBX_HAVE_BUILTIN_CPU_SUPPORTS == 0 || \ - MDBX_HAVE_BUILTIN_CPU_SUPPORTS == 1) +#elif !(MDBX_HAVE_BUILTIN_CPU_SUPPORTS == 0 || MDBX_HAVE_BUILTIN_CPU_SUPPORTS == 1) #error MDBX_HAVE_BUILTIN_CPU_SUPPORTS must be defined as 0 or 1 #endif /* MDBX_HAVE_BUILTIN_CPU_SUPPORTS */ +/** if enabled then instead of the returned error `MDBX_REMOTE`, only a warning is issued, when + * the database being opened in non-read-only mode is located in a file system exported via NFS. */ +#ifndef MDBX_ENABLE_NON_READONLY_EXPORT +#define MDBX_ENABLE_NON_READONLY_EXPORT 0 +#elif !(MDBX_ENABLE_NON_READONLY_EXPORT == 0 || MDBX_ENABLE_NON_READONLY_EXPORT == 1) +#error MDBX_ENABLE_NON_READONLY_EXPORT must be defined as 0 or 1 +#endif /* MDBX_ENABLE_NON_READONLY_EXPORT */ + //------------------------------------------------------------------------------ /** Win32 File Locking API for \ref MDBX_LOCKING */ @@ -2242,27 +1879,20 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** POSIX-2008 Robust Mutexes for \ref MDBX_LOCKING */ #define MDBX_LOCKING_POSIX2008 2008 -/** BeOS Benaphores, aka Futexes for \ref MDBX_LOCKING */ -#define MDBX_LOCKING_BENAPHORE 1995 - /** Advanced: Choices the locking implementation (autodetection by default). */ #if defined(_WIN32) || defined(_WIN64) #define MDBX_LOCKING MDBX_LOCKING_WIN32FILES #else #ifndef MDBX_LOCKING -#if defined(_POSIX_THREAD_PROCESS_SHARED) && \ - _POSIX_THREAD_PROCESS_SHARED >= 200112L && !defined(__FreeBSD__) +#if defined(_POSIX_THREAD_PROCESS_SHARED) && _POSIX_THREAD_PROCESS_SHARED >= 200112L && !defined(__FreeBSD__) /* Some platforms define the EOWNERDEAD error code even though they * don't support Robust Mutexes. If doubt compile with -MDBX_LOCKING=2001. */ -#if defined(EOWNERDEAD) && _POSIX_THREAD_PROCESS_SHARED >= 200809L && \ - ((defined(_POSIX_THREAD_ROBUST_PRIO_INHERIT) && \ - _POSIX_THREAD_ROBUST_PRIO_INHERIT > 0) || \ - (defined(_POSIX_THREAD_ROBUST_PRIO_PROTECT) && \ - _POSIX_THREAD_ROBUST_PRIO_PROTECT > 0) || \ - defined(PTHREAD_MUTEX_ROBUST) || defined(PTHREAD_MUTEX_ROBUST_NP)) && \ - (!defined(__GLIBC__) || \ - __GLIBC_PREREQ(2, 10) /* troubles with Robust mutexes before 2.10 */) +#if defined(EOWNERDEAD) && _POSIX_THREAD_PROCESS_SHARED >= 200809L && \ + ((defined(_POSIX_THREAD_ROBUST_PRIO_INHERIT) && _POSIX_THREAD_ROBUST_PRIO_INHERIT > 0) || \ + (defined(_POSIX_THREAD_ROBUST_PRIO_PROTECT) && _POSIX_THREAD_ROBUST_PRIO_PROTECT > 0) || \ + defined(PTHREAD_MUTEX_ROBUST) || defined(PTHREAD_MUTEX_ROBUST_NP)) && \ + (!defined(__GLIBC__) || __GLIBC_PREREQ(2, 10) /* troubles with Robust mutexes before 2.10 */) #define MDBX_LOCKING MDBX_LOCKING_POSIX2008 #else #define MDBX_LOCKING MDBX_LOCKING_POSIX2001 @@ -2280,12 +1910,9 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Advanced: Using POSIX OFD-locks (autodetection by default). */ #ifndef MDBX_USE_OFDLOCKS -#if ((defined(F_OFD_SETLK) && defined(F_OFD_SETLKW) && \ - defined(F_OFD_GETLK)) || \ - (defined(F_OFD_SETLK64) && defined(F_OFD_SETLKW64) && \ - defined(F_OFD_GETLK64))) && \ - !defined(MDBX_SAFE4QEMU) && \ - !defined(__sun) /* OFD-lock are broken on Solaris */ +#if ((defined(F_OFD_SETLK) && defined(F_OFD_SETLKW) && defined(F_OFD_GETLK)) || \ + (defined(F_OFD_SETLK64) && defined(F_OFD_SETLKW64) && defined(F_OFD_GETLK64))) && \ + !defined(MDBX_SAFE4QEMU) && !defined(__sun) /* OFD-lock are broken on Solaris */ #define MDBX_USE_OFDLOCKS 1 #else #define MDBX_USE_OFDLOCKS 0 @@ -2299,8 +1926,7 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; /** Advanced: Using sendfile() syscall (autodetection by default). */ #ifndef MDBX_USE_SENDFILE -#if ((defined(__linux__) || defined(__gnu_linux__)) && \ - !defined(__ANDROID_API__)) || \ +#if ((defined(__linux__) || defined(__gnu_linux__)) && !defined(__ANDROID_API__)) || \ (defined(__ANDROID_API__) && __ANDROID_API__ >= 21) #define MDBX_USE_SENDFILE 1 #else @@ -2321,30 +1947,15 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #error MDBX_USE_COPYFILERANGE must be defined as 0 or 1 #endif /* MDBX_USE_COPYFILERANGE */ -/** Advanced: Using sync_file_range() syscall (autodetection by default). */ -#ifndef MDBX_USE_SYNCFILERANGE -#if ((defined(__linux__) || defined(__gnu_linux__)) && \ - defined(SYNC_FILE_RANGE_WRITE) && !defined(__ANDROID_API__)) || \ - (defined(__ANDROID_API__) && __ANDROID_API__ >= 26) -#define MDBX_USE_SYNCFILERANGE 1 -#else -#define MDBX_USE_SYNCFILERANGE 0 -#endif -#elif !(MDBX_USE_SYNCFILERANGE == 0 || MDBX_USE_SYNCFILERANGE == 1) -#error MDBX_USE_SYNCFILERANGE must be defined as 0 or 1 -#endif /* MDBX_USE_SYNCFILERANGE */ - //------------------------------------------------------------------------------ #ifndef MDBX_CPU_WRITEBACK_INCOHERENT -#if defined(__ia32__) || defined(__e2k__) || defined(__hppa) || \ - defined(__hppa__) || defined(DOXYGEN) +#if defined(__ia32__) || defined(__e2k__) || defined(__hppa) || defined(__hppa__) || defined(DOXYGEN) #define MDBX_CPU_WRITEBACK_INCOHERENT 0 #else #define MDBX_CPU_WRITEBACK_INCOHERENT 1 #endif -#elif !(MDBX_CPU_WRITEBACK_INCOHERENT == 0 || \ - MDBX_CPU_WRITEBACK_INCOHERENT == 1) +#elif !(MDBX_CPU_WRITEBACK_INCOHERENT == 0 || MDBX_CPU_WRITEBACK_INCOHERENT == 1) #error MDBX_CPU_WRITEBACK_INCOHERENT must be defined as 0 or 1 #endif /* MDBX_CPU_WRITEBACK_INCOHERENT */ @@ -2354,35 +1965,35 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #else #define MDBX_MMAP_INCOHERENT_FILE_WRITE 0 #endif -#elif !(MDBX_MMAP_INCOHERENT_FILE_WRITE == 0 || \ - MDBX_MMAP_INCOHERENT_FILE_WRITE == 1) +#elif !(MDBX_MMAP_INCOHERENT_FILE_WRITE == 0 || MDBX_MMAP_INCOHERENT_FILE_WRITE == 1) #error MDBX_MMAP_INCOHERENT_FILE_WRITE must be defined as 0 or 1 #endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */ #ifndef MDBX_MMAP_INCOHERENT_CPU_CACHE -#if defined(__mips) || defined(__mips__) || defined(__mips64) || \ - defined(__mips64__) || defined(_M_MRX000) || defined(_MIPS_) || \ - defined(__MWERKS__) || defined(__sgi) +#if defined(__mips) || defined(__mips__) || defined(__mips64) || defined(__mips64__) || defined(_M_MRX000) || \ + defined(_MIPS_) || defined(__MWERKS__) || defined(__sgi) /* MIPS has cache coherency issues. */ #define MDBX_MMAP_INCOHERENT_CPU_CACHE 1 #else /* LY: assume no relevant mmap/dcache issues. */ #define MDBX_MMAP_INCOHERENT_CPU_CACHE 0 #endif -#elif !(MDBX_MMAP_INCOHERENT_CPU_CACHE == 0 || \ - MDBX_MMAP_INCOHERENT_CPU_CACHE == 1) +#elif !(MDBX_MMAP_INCOHERENT_CPU_CACHE == 0 || MDBX_MMAP_INCOHERENT_CPU_CACHE == 1) #error MDBX_MMAP_INCOHERENT_CPU_CACHE must be defined as 0 or 1 #endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */ -#ifndef MDBX_MMAP_USE_MS_ASYNC -#if MDBX_MMAP_INCOHERENT_FILE_WRITE || MDBX_MMAP_INCOHERENT_CPU_CACHE -#define MDBX_MMAP_USE_MS_ASYNC 1 +/** Assume system needs explicit syscall to sync/flush/write modified mapped + * memory. */ +#ifndef MDBX_MMAP_NEEDS_JOLT +#if MDBX_MMAP_INCOHERENT_FILE_WRITE || MDBX_MMAP_INCOHERENT_CPU_CACHE || !(defined(__linux__) || defined(__gnu_linux__)) +#define MDBX_MMAP_NEEDS_JOLT 1 #else -#define MDBX_MMAP_USE_MS_ASYNC 0 +#define MDBX_MMAP_NEEDS_JOLT 0 #endif -#elif !(MDBX_MMAP_USE_MS_ASYNC == 0 || MDBX_MMAP_USE_MS_ASYNC == 1) -#error MDBX_MMAP_USE_MS_ASYNC must be defined as 0 or 1 -#endif /* MDBX_MMAP_USE_MS_ASYNC */ +#define MDBX_MMAP_NEEDS_JOLT_CONFIG "AUTO=" MDBX_STRINGIFY(MDBX_MMAP_NEEDS_JOLT) +#elif !(MDBX_MMAP_NEEDS_JOLT == 0 || MDBX_MMAP_NEEDS_JOLT == 1) +#error MDBX_MMAP_NEEDS_JOLT must be defined as 0 or 1 +#endif /* MDBX_MMAP_NEEDS_JOLT */ #ifndef MDBX_64BIT_ATOMIC #if MDBX_WORDBITS >= 64 || defined(DOXYGEN) @@ -2429,8 +2040,7 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #endif /* MDBX_64BIT_CAS */ #ifndef MDBX_UNALIGNED_OK -#if defined(__ALIGNED__) || defined(__SANITIZE_UNDEFINED__) || \ - defined(ENABLE_UBSAN) +#if defined(__ALIGNED__) || defined(__SANITIZE_UNDEFINED__) || defined(ENABLE_UBSAN) #define MDBX_UNALIGNED_OK 0 /* no unaligned access allowed */ #elif defined(__ARM_FEATURE_UNALIGNED) #define MDBX_UNALIGNED_OK 4 /* ok unaligned for 32-bit words */ @@ -2464,6 +2074,19 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #endif #endif /* MDBX_CACHELINE_SIZE */ +/* Max length of iov-vector passed to writev() call, used for auxilary writes */ +#ifndef MDBX_AUXILARY_IOV_MAX +#define MDBX_AUXILARY_IOV_MAX 64 +#endif +#if defined(IOV_MAX) && IOV_MAX < MDBX_AUXILARY_IOV_MAX +#undef MDBX_AUXILARY_IOV_MAX +#define MDBX_AUXILARY_IOV_MAX IOV_MAX +#endif /* MDBX_AUXILARY_IOV_MAX */ + +/* An extra/custom information provided during library build */ +#ifndef MDBX_BUILD_METADATA +#define MDBX_BUILD_METADATA "" +#endif /* MDBX_BUILD_METADATA */ /** @} end of build options */ /******************************************************************************* ******************************************************************************* @@ -2478,6 +2101,9 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #else #define MDBX_DEBUG 1 #endif +#endif +#if MDBX_DEBUG < 0 || MDBX_DEBUG > 2 +#error "The MDBX_DEBUG must be defined to 0, 1 or 2" #endif /* MDBX_DEBUG */ #else @@ -2497,169 +2123,58 @@ extern LIBMDBX_API const char *const mdbx_sourcery_anchor; * Also enables \ref MDBX_DBG_AUDIT if `MDBX_DEBUG >= 2`. * * \ingroup build_option */ -#define MDBX_DEBUG 0...7 +#define MDBX_DEBUG 0...2 /** Disables using of GNU libc extensions. */ #define MDBX_DISABLE_GNU_SOURCE 0 or 1 #endif /* DOXYGEN */ -/* Undefine the NDEBUG if debugging is enforced by MDBX_DEBUG */ -#if MDBX_DEBUG -#undef NDEBUG -#endif - -#ifndef __cplusplus -/*----------------------------------------------------------------------------*/ -/* Debug and Logging stuff */ - -#define MDBX_RUNTIME_FLAGS_INIT \ - ((MDBX_DEBUG) > 0) * MDBX_DBG_ASSERT + ((MDBX_DEBUG) > 1) * MDBX_DBG_AUDIT - -extern uint8_t runtime_flags; -extern uint8_t loglevel; -extern MDBX_debug_func *debug_logger; - -MDBX_MAYBE_UNUSED static __inline void jitter4testing(bool tiny) { -#if MDBX_DEBUG - if (MDBX_DBG_JITTER & runtime_flags) - osal_jitter(tiny); -#else - (void)tiny; -#endif -} - -MDBX_INTERNAL_FUNC void MDBX_PRINTF_ARGS(4, 5) - debug_log(int level, const char *function, int line, const char *fmt, ...) - MDBX_PRINTF_ARGS(4, 5); -MDBX_INTERNAL_FUNC void debug_log_va(int level, const char *function, int line, - const char *fmt, va_list args); +#ifndef MDBX_64BIT_ATOMIC +#error "The MDBX_64BIT_ATOMIC must be defined before" +#endif /* MDBX_64BIT_ATOMIC */ -#if MDBX_DEBUG -#define LOG_ENABLED(msg) unlikely(msg <= loglevel) -#define AUDIT_ENABLED() unlikely((runtime_flags & MDBX_DBG_AUDIT)) -#else /* MDBX_DEBUG */ -#define LOG_ENABLED(msg) (msg < MDBX_LOG_VERBOSE && msg <= loglevel) -#define AUDIT_ENABLED() (0) -#endif /* MDBX_DEBUG */ +#ifndef MDBX_64BIT_CAS +#error "The MDBX_64BIT_CAS must be defined before" +#endif /* MDBX_64BIT_CAS */ -#if MDBX_FORCE_ASSERTIONS -#define ASSERT_ENABLED() (1) -#elif MDBX_DEBUG -#define ASSERT_ENABLED() likely((runtime_flags & MDBX_DBG_ASSERT)) +#if defined(__cplusplus) && !defined(__STDC_NO_ATOMICS__) && __has_include() +#include +#define MDBX_HAVE_C11ATOMICS +#elif !defined(__cplusplus) && (__STDC_VERSION__ >= 201112L || __has_extension(c_atomic)) && \ + !defined(__STDC_NO_ATOMICS__) && \ + (__GNUC_PREREQ(4, 9) || __CLANG_PREREQ(3, 8) || !(defined(__GNUC__) || defined(__clang__))) +#include +#define MDBX_HAVE_C11ATOMICS +#elif defined(__GNUC__) || defined(__clang__) +#elif defined(_MSC_VER) +#pragma warning(disable : 4163) /* 'xyz': not available as an intrinsic */ +#pragma warning(disable : 4133) /* 'function': incompatible types - from \ + 'size_t' to 'LONGLONG' */ +#pragma warning(disable : 4244) /* 'return': conversion from 'LONGLONG' to \ + 'std::size_t', possible loss of data */ +#pragma warning(disable : 4267) /* 'function': conversion from 'size_t' to \ + 'long', possible loss of data */ +#pragma intrinsic(_InterlockedExchangeAdd, _InterlockedCompareExchange) +#pragma intrinsic(_InterlockedExchangeAdd64, _InterlockedCompareExchange64) +#elif defined(__APPLE__) +#include #else -#define ASSERT_ENABLED() (0) -#endif /* assertions */ - -#define DEBUG_EXTRA(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_EXTRA)) \ - debug_log(MDBX_LOG_EXTRA, __func__, __LINE__, fmt, __VA_ARGS__); \ - } while (0) - -#define DEBUG_EXTRA_PRINT(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_EXTRA)) \ - debug_log(MDBX_LOG_EXTRA, NULL, 0, fmt, __VA_ARGS__); \ - } while (0) - -#define TRACE(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_TRACE)) \ - debug_log(MDBX_LOG_TRACE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ - } while (0) - -#define DEBUG(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_DEBUG)) \ - debug_log(MDBX_LOG_DEBUG, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ - } while (0) - -#define VERBOSE(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_VERBOSE)) \ - debug_log(MDBX_LOG_VERBOSE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ - } while (0) - -#define NOTICE(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_NOTICE)) \ - debug_log(MDBX_LOG_NOTICE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ - } while (0) - -#define WARNING(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_WARN)) \ - debug_log(MDBX_LOG_WARN, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ - } while (0) - -#undef ERROR /* wingdi.h \ - Yeah, morons from M$ put such definition to the public header. */ - -#define ERROR(fmt, ...) \ - do { \ - if (LOG_ENABLED(MDBX_LOG_ERROR)) \ - debug_log(MDBX_LOG_ERROR, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ - } while (0) - -#define FATAL(fmt, ...) \ - debug_log(MDBX_LOG_FATAL, __func__, __LINE__, fmt "\n", __VA_ARGS__); - -#if MDBX_DEBUG -#define ASSERT_FAIL(env, msg, func, line) mdbx_assert_fail(env, msg, func, line) -#else /* MDBX_DEBUG */ -MDBX_NORETURN __cold void assert_fail(const char *msg, const char *func, - unsigned line); -#define ASSERT_FAIL(env, msg, func, line) \ - do { \ - (void)(env); \ - assert_fail(msg, func, line); \ - } while (0) -#endif /* MDBX_DEBUG */ - -#define ENSURE_MSG(env, expr, msg) \ - do { \ - if (unlikely(!(expr))) \ - ASSERT_FAIL(env, msg, __func__, __LINE__); \ - } while (0) - -#define ENSURE(env, expr) ENSURE_MSG(env, expr, #expr) - -/* assert(3) variant in environment context */ -#define eASSERT(env, expr) \ - do { \ - if (ASSERT_ENABLED()) \ - ENSURE(env, expr); \ - } while (0) - -/* assert(3) variant in cursor context */ -#define cASSERT(mc, expr) eASSERT((mc)->mc_txn->mt_env, expr) - -/* assert(3) variant in transaction context */ -#define tASSERT(txn, expr) eASSERT((txn)->mt_env, expr) - -#ifndef xMDBX_TOOLS /* Avoid using internal eASSERT() */ -#undef assert -#define assert(expr) eASSERT(NULL, expr) +#error FIXME atomic-ops #endif -#endif /* __cplusplus */ - -/*----------------------------------------------------------------------------*/ -/* Atomics */ - -enum MDBX_memory_order { +typedef enum mdbx_memory_order { mo_Relaxed, mo_AcquireRelease /* , mo_SequentialConsistency */ -}; +} mdbx_memory_order_t; typedef union { volatile uint32_t weak; #ifdef MDBX_HAVE_C11ATOMICS volatile _Atomic uint32_t c11a; #endif /* MDBX_HAVE_C11ATOMICS */ -} MDBX_atomic_uint32_t; +} mdbx_atomic_uint32_t; typedef union { volatile uint64_t weak; @@ -2669,15 +2184,15 @@ typedef union { #if !defined(MDBX_HAVE_C11ATOMICS) || !MDBX_64BIT_CAS || !MDBX_64BIT_ATOMIC __anonymous_struct_extension__ struct { #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ - MDBX_atomic_uint32_t low, high; + mdbx_atomic_uint32_t low, high; #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - MDBX_atomic_uint32_t high, low; + mdbx_atomic_uint32_t high, low; #else #error "FIXME: Unsupported byte order" #endif /* __BYTE_ORDER__ */ }; #endif -} MDBX_atomic_uint64_t; +} mdbx_atomic_uint64_t; #ifdef MDBX_HAVE_C11ATOMICS @@ -2693,92 +2208,20 @@ typedef union { #define MDBX_c11a_rw(type, ptr) (&(ptr)->c11a) #endif /* Crutches for C11 atomic compiler's bugs */ -#define mo_c11_store(fence) \ - (((fence) == mo_Relaxed) ? memory_order_relaxed \ - : ((fence) == mo_AcquireRelease) ? memory_order_release \ +#define mo_c11_store(fence) \ + (((fence) == mo_Relaxed) ? memory_order_relaxed \ + : ((fence) == mo_AcquireRelease) ? memory_order_release \ : memory_order_seq_cst) -#define mo_c11_load(fence) \ - (((fence) == mo_Relaxed) ? memory_order_relaxed \ - : ((fence) == mo_AcquireRelease) ? memory_order_acquire \ +#define mo_c11_load(fence) \ + (((fence) == mo_Relaxed) ? memory_order_relaxed \ + : ((fence) == mo_AcquireRelease) ? memory_order_acquire \ : memory_order_seq_cst) #endif /* MDBX_HAVE_C11ATOMICS */ -#ifndef __cplusplus - -#ifdef MDBX_HAVE_C11ATOMICS -#define osal_memory_fence(order, write) \ - atomic_thread_fence((write) ? mo_c11_store(order) : mo_c11_load(order)) -#else /* MDBX_HAVE_C11ATOMICS */ -#define osal_memory_fence(order, write) \ - do { \ - osal_compiler_barrier(); \ - if (write && order > (MDBX_CPU_WRITEBACK_INCOHERENT ? mo_Relaxed \ - : mo_AcquireRelease)) \ - osal_memory_barrier(); \ - } while (0) -#endif /* MDBX_HAVE_C11ATOMICS */ - -#if defined(MDBX_HAVE_C11ATOMICS) && defined(__LCC__) -#define atomic_store32(p, value, order) \ - ({ \ - const uint32_t value_to_store = (value); \ - atomic_store_explicit(MDBX_c11a_rw(uint32_t, p), value_to_store, \ - mo_c11_store(order)); \ - value_to_store; \ - }) -#define atomic_load32(p, order) \ - atomic_load_explicit(MDBX_c11a_ro(uint32_t, p), mo_c11_load(order)) -#define atomic_store64(p, value, order) \ - ({ \ - const uint64_t value_to_store = (value); \ - atomic_store_explicit(MDBX_c11a_rw(uint64_t, p), value_to_store, \ - mo_c11_store(order)); \ - value_to_store; \ - }) -#define atomic_load64(p, order) \ - atomic_load_explicit(MDBX_c11a_ro(uint64_t, p), mo_c11_load(order)) -#endif /* LCC && MDBX_HAVE_C11ATOMICS */ - -#ifndef atomic_store32 -MDBX_MAYBE_UNUSED static __always_inline uint32_t -atomic_store32(MDBX_atomic_uint32_t *p, const uint32_t value, - enum MDBX_memory_order order) { - STATIC_ASSERT(sizeof(MDBX_atomic_uint32_t) == 4); -#ifdef MDBX_HAVE_C11ATOMICS - assert(atomic_is_lock_free(MDBX_c11a_rw(uint32_t, p))); - atomic_store_explicit(MDBX_c11a_rw(uint32_t, p), value, mo_c11_store(order)); -#else /* MDBX_HAVE_C11ATOMICS */ - if (order != mo_Relaxed) - osal_compiler_barrier(); - p->weak = value; - osal_memory_fence(order, true); -#endif /* MDBX_HAVE_C11ATOMICS */ - return value; -} -#endif /* atomic_store32 */ - -#ifndef atomic_load32 -MDBX_MAYBE_UNUSED static __always_inline uint32_t atomic_load32( - const volatile MDBX_atomic_uint32_t *p, enum MDBX_memory_order order) { - STATIC_ASSERT(sizeof(MDBX_atomic_uint32_t) == 4); -#ifdef MDBX_HAVE_C11ATOMICS - assert(atomic_is_lock_free(MDBX_c11a_ro(uint32_t, p))); - return atomic_load_explicit(MDBX_c11a_ro(uint32_t, p), mo_c11_load(order)); -#else /* MDBX_HAVE_C11ATOMICS */ - osal_memory_fence(order, false); - const uint32_t value = p->weak; - if (order != mo_Relaxed) - osal_compiler_barrier(); - return value; -#endif /* MDBX_HAVE_C11ATOMICS */ -} -#endif /* atomic_load32 */ - -#endif /* !__cplusplus */ +#define SAFE64_INVALID_THRESHOLD UINT64_C(0xffffFFFF00000000) -/*----------------------------------------------------------------------------*/ -/* Basic constants and types */ +#pragma pack(push, 4) /* A stamp that identifies a file as an MDBX file. * There's nothing special about this value other than that it is easily @@ -2787,8 +2230,10 @@ MDBX_MAYBE_UNUSED static __always_inline uint32_t atomic_load32( /* FROZEN: The version number for a database's datafile format. */ #define MDBX_DATA_VERSION 3 -/* The version number for a database's lockfile format. */ -#define MDBX_LOCK_VERSION 5 + +#define MDBX_DATA_MAGIC ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + MDBX_DATA_VERSION) +#define MDBX_DATA_MAGIC_LEGACY_COMPAT ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + 2) +#define MDBX_DATA_MAGIC_LEGACY_DEVEL ((MDBX_MAGIC << 8) + 255) /* handle for the DB used to track free pages. */ #define FREE_DBI 0 @@ -2805,203 +2250,285 @@ MDBX_MAYBE_UNUSED static __always_inline uint32_t atomic_load32( * MDBX uses 32 bit for page numbers. This limits database * size up to 2^44 bytes, in case of 4K pages. */ typedef uint32_t pgno_t; -typedef MDBX_atomic_uint32_t atomic_pgno_t; +typedef mdbx_atomic_uint32_t atomic_pgno_t; #define PRIaPGNO PRIu32 #define MAX_PAGENO UINT32_C(0x7FFFffff) #define MIN_PAGENO NUM_METAS -#define SAFE64_INVALID_THRESHOLD UINT64_C(0xffffFFFF00000000) +/* An invalid page number. + * Mainly used to denote an empty tree. */ +#define P_INVALID (~(pgno_t)0) /* A transaction ID. */ typedef uint64_t txnid_t; -typedef MDBX_atomic_uint64_t atomic_txnid_t; +typedef mdbx_atomic_uint64_t atomic_txnid_t; #define PRIaTXN PRIi64 #define MIN_TXNID UINT64_C(1) #define MAX_TXNID (SAFE64_INVALID_THRESHOLD - 1) #define INITIAL_TXNID (MIN_TXNID + NUM_METAS - 1) #define INVALID_TXNID UINT64_MAX -/* LY: for testing non-atomic 64-bit txnid on 32-bit arches. - * #define xMDBX_TXNID_STEP (UINT32_MAX / 3) */ -#ifndef xMDBX_TXNID_STEP -#if MDBX_64BIT_CAS -#define xMDBX_TXNID_STEP 1u -#else -#define xMDBX_TXNID_STEP 2u -#endif -#endif /* xMDBX_TXNID_STEP */ -/* Used for offsets within a single page. - * Since memory pages are typically 4 or 8KB in size, 12-13 bits, - * this is plenty. */ +/* Used for offsets within a single page. */ typedef uint16_t indx_t; -#define MEGABYTE ((size_t)1 << 20) - -/*----------------------------------------------------------------------------*/ -/* Core structures for database and shared memory (i.e. format definition) */ -#pragma pack(push, 4) - -/* Information about a single database in the environment. */ -typedef struct MDBX_db { - uint16_t md_flags; /* see mdbx_dbi_open */ - uint16_t md_depth; /* depth of this tree */ - uint32_t md_xsize; /* key-size for MDBX_DUPFIXED (LEAF2 pages) */ - pgno_t md_root; /* the root page of this tree */ - pgno_t md_branch_pages; /* number of internal pages */ - pgno_t md_leaf_pages; /* number of leaf pages */ - pgno_t md_overflow_pages; /* number of overflow pages */ - uint64_t md_seq; /* table sequence counter */ - uint64_t md_entries; /* number of data items */ - uint64_t md_mod_txnid; /* txnid of last committed modification */ -} MDBX_db; +typedef struct tree { + uint16_t flags; /* see mdbx_dbi_open */ + uint16_t height; /* height of this tree */ + uint32_t dupfix_size; /* key-size for MDBX_DUPFIXED (DUPFIX pages) */ + pgno_t root; /* the root page of this tree */ + pgno_t branch_pages; /* number of branch pages */ + pgno_t leaf_pages; /* number of leaf pages */ + pgno_t large_pages; /* number of large pages */ + uint64_t sequence; /* table sequence counter */ + uint64_t items; /* number of data items */ + uint64_t mod_txnid; /* txnid of last committed modification */ +} tree_t; /* database size-related parameters */ -typedef struct MDBX_geo { +typedef struct geo { uint16_t grow_pv; /* datafile growth step as a 16-bit packed (exponential quantized) value */ uint16_t shrink_pv; /* datafile shrink threshold as a 16-bit packed (exponential quantized) value */ pgno_t lower; /* minimal size of datafile in pages */ pgno_t upper; /* maximal size of datafile in pages */ - pgno_t now; /* current size of datafile in pages */ - pgno_t next; /* first unused page in the datafile, + union { + pgno_t now; /* current size of datafile in pages */ + pgno_t end_pgno; + }; + union { + pgno_t first_unallocated; /* first unused page in the datafile, but actually the file may be shorter. */ -} MDBX_geo; + pgno_t next_pgno; + }; +} geo_t; /* Meta page content. * A meta page is the start point for accessing a database snapshot. - * Pages 0-1 are meta pages. Transaction N writes meta page (N % 2). */ -typedef struct MDBX_meta { + * Pages 0-2 are meta pages. */ +typedef struct meta { /* Stamp identifying this as an MDBX file. * It must be set to MDBX_MAGIC with MDBX_DATA_VERSION. */ - uint32_t mm_magic_and_version[2]; + uint32_t magic_and_version[2]; - /* txnid that committed this page, the first of a two-phase-update pair */ + /* txnid that committed this meta, the first of a two-phase-update pair */ union { - MDBX_atomic_uint32_t mm_txnid_a[2]; + mdbx_atomic_uint32_t txnid_a[2]; uint64_t unsafe_txnid; }; - uint16_t mm_extra_flags; /* extra DB flags, zero (nothing) for now */ - uint8_t mm_validator_id; /* ID of checksum and page validation method, - * zero (nothing) for now */ - uint8_t mm_extra_pagehdr; /* extra bytes in the page header, - * zero (nothing) for now */ + uint16_t reserve16; /* extra flags, zero (nothing) for now */ + uint8_t validator_id; /* ID of checksum and page validation method, + * zero (nothing) for now */ + int8_t extra_pagehdr; /* extra bytes in the page header, + * zero (nothing) for now */ + + geo_t geometry; /* database size-related parameters */ - MDBX_geo mm_geo; /* database size-related parameters */ + union { + struct { + tree_t gc, main; + } trees; + __anonymous_struct_extension__ struct { + uint16_t gc_flags; + uint16_t gc_height; + uint32_t pagesize; + }; + }; - MDBX_db mm_dbs[CORE_DBS]; /* first is free space, 2nd is main db */ - /* The size of pages used in this DB */ -#define mm_psize mm_dbs[FREE_DBI].md_xsize - MDBX_canary mm_canary; + MDBX_canary canary; -#define MDBX_DATASIGN_NONE 0u -#define MDBX_DATASIGN_WEAK 1u -#define SIGN_IS_STEADY(sign) ((sign) > MDBX_DATASIGN_WEAK) -#define META_IS_STEADY(meta) \ - SIGN_IS_STEADY(unaligned_peek_u64_volatile(4, (meta)->mm_sign)) +#define DATASIGN_NONE 0u +#define DATASIGN_WEAK 1u +#define SIGN_IS_STEADY(sign) ((sign) > DATASIGN_WEAK) union { - uint32_t mm_sign[2]; + uint32_t sign[2]; uint64_t unsafe_sign; }; - /* txnid that committed this page, the second of a two-phase-update pair */ - MDBX_atomic_uint32_t mm_txnid_b[2]; + /* txnid that committed this meta, the second of a two-phase-update pair */ + mdbx_atomic_uint32_t txnid_b[2]; /* Number of non-meta pages which were put in GC after COW. May be 0 in case * DB was previously handled by libmdbx without corresponding feature. - * This value in couple with mr_snapshot_pages_retired allows fast estimation - * of "how much reader is restraining GC recycling". */ - uint32_t mm_pages_retired[2]; + * This value in couple with reader.snapshot_pages_retired allows fast + * estimation of "how much reader is restraining GC recycling". */ + uint32_t pages_retired[2]; /* The analogue /proc/sys/kernel/random/boot_id or similar to determine * whether the system was rebooted after the last use of the database files. * If there was no reboot, but there is no need to rollback to the last * steady sync point. Zeros mean that no relevant information is available * from the system. */ - bin128_t mm_bootid; + bin128_t bootid; -} MDBX_meta; + /* GUID базы данных, начиная с v0.13.1 */ + bin128_t dxbid; +} meta_t; #pragma pack(1) -/* Common header for all page types. The page type depends on mp_flags. +typedef enum page_type { + P_BRANCH = 0x01u /* branch page */, + P_LEAF = 0x02u /* leaf page */, + P_LARGE = 0x04u /* large/overflow page */, + P_META = 0x08u /* meta page */, + P_LEGACY_DIRTY = 0x10u /* legacy P_DIRTY flag prior to v0.10 958fd5b9 */, + P_BAD = P_LEGACY_DIRTY /* explicit flag for invalid/bad page */, + P_DUPFIX = 0x20u /* for MDBX_DUPFIXED records */, + P_SUBP = 0x40u /* for MDBX_DUPSORT sub-pages */, + P_SPILLED = 0x2000u /* spilled in parent txn */, + P_LOOSE = 0x4000u /* page was dirtied then freed, can be reused */, + P_FROZEN = 0x8000u /* used for retire page with known status */, + P_ILL_BITS = (uint16_t)~(P_BRANCH | P_LEAF | P_DUPFIX | P_LARGE | P_SPILLED), + + page_broken = 0, + page_large = P_LARGE, + page_branch = P_BRANCH, + page_leaf = P_LEAF, + page_dupfix_leaf = P_DUPFIX, + page_sub_leaf = P_SUBP | P_LEAF, + page_sub_dupfix_leaf = P_SUBP | P_DUPFIX, + page_sub_broken = P_SUBP, +} page_type_t; + +/* Common header for all page types. The page type depends on flags. * - * P_BRANCH and P_LEAF pages have unsorted 'MDBX_node's at the end, with - * sorted mp_ptrs[] entries referring to them. Exception: P_LEAF2 pages - * omit mp_ptrs and pack sorted MDBX_DUPFIXED values after the page header. + * P_BRANCH and P_LEAF pages have unsorted 'node_t's at the end, with + * sorted entries[] entries referring to them. Exception: P_DUPFIX pages + * omit entries and pack sorted MDBX_DUPFIXED values after the page header. * - * P_OVERFLOW records occupy one or more contiguous pages where only the - * first has a page header. They hold the real data of F_BIGDATA nodes. + * P_LARGE records occupy one or more contiguous pages where only the + * first has a page header. They hold the real data of N_BIG nodes. * * P_SUBP sub-pages are small leaf "pages" with duplicate data. - * A node with flag F_DUPDATA but not F_SUBDATA contains a sub-page. - * (Duplicate data can also go in sub-databases, which use normal pages.) + * A node with flag N_DUP but not N_TREE contains a sub-page. + * (Duplicate data can also go in tables, which use normal pages.) * - * P_META pages contain MDBX_meta, the start point of an MDBX snapshot. + * P_META pages contain meta_t, the start point of an MDBX snapshot. * - * Each non-metapage up to MDBX_meta.mm_last_pg is reachable exactly once + * Each non-metapage up to meta_t.mm_last_pg is reachable exactly once * in the snapshot: Either used by a database or listed in a GC record. */ -typedef struct MDBX_page { -#define IS_FROZEN(txn, p) ((p)->mp_txnid < (txn)->mt_txnid) -#define IS_SPILLED(txn, p) ((p)->mp_txnid == (txn)->mt_txnid) -#define IS_SHADOWED(txn, p) ((p)->mp_txnid > (txn)->mt_txnid) -#define IS_VALID(txn, p) ((p)->mp_txnid <= (txn)->mt_front) -#define IS_MODIFIABLE(txn, p) ((p)->mp_txnid == (txn)->mt_front) - uint64_t mp_txnid; /* txnid which created page, maybe zero in legacy DB */ - uint16_t mp_leaf2_ksize; /* key size if this is a LEAF2 page */ -#define P_BRANCH 0x01u /* branch page */ -#define P_LEAF 0x02u /* leaf page */ -#define P_OVERFLOW 0x04u /* overflow page */ -#define P_META 0x08u /* meta page */ -#define P_LEGACY_DIRTY 0x10u /* legacy P_DIRTY flag prior to v0.10 958fd5b9 */ -#define P_BAD P_LEGACY_DIRTY /* explicit flag for invalid/bad page */ -#define P_LEAF2 0x20u /* for MDBX_DUPFIXED records */ -#define P_SUBP 0x40u /* for MDBX_DUPSORT sub-pages */ -#define P_SPILLED 0x2000u /* spilled in parent txn */ -#define P_LOOSE 0x4000u /* page was dirtied then freed, can be reused */ -#define P_FROZEN 0x8000u /* used for retire page with known status */ -#define P_ILL_BITS \ - ((uint16_t)~(P_BRANCH | P_LEAF | P_LEAF2 | P_OVERFLOW | P_SPILLED)) - uint16_t mp_flags; +typedef struct page { + uint64_t txnid; /* txnid which created page, maybe zero in legacy DB */ + uint16_t dupfix_ksize; /* key size if this is a DUPFIX page */ + uint16_t flags; union { - uint32_t mp_pages; /* number of overflow pages */ + uint32_t pages; /* number of overflow pages */ __anonymous_struct_extension__ struct { - indx_t mp_lower; /* lower bound of free space */ - indx_t mp_upper; /* upper bound of free space */ + indx_t lower; /* lower bound of free space */ + indx_t upper; /* upper bound of free space */ }; }; - pgno_t mp_pgno; /* page number */ + pgno_t pgno; /* page number */ -#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ - (!defined(__cplusplus) && defined(_MSC_VER)) - indx_t mp_ptrs[] /* dynamic size */; -#endif /* C99 */ -} MDBX_page; +#if FLEXIBLE_ARRAY_MEMBERS + indx_t entries[] /* dynamic size */; +#endif /* FLEXIBLE_ARRAY_MEMBERS */ +} page_t; -#define PAGETYPE_WHOLE(p) ((uint8_t)(p)->mp_flags) +/* Size of the page header, excluding dynamic data at the end */ +#define PAGEHDRSZ 20u -/* Drop legacy P_DIRTY flag for sub-pages for compatilibity */ -#define PAGETYPE_COMPAT(p) \ - (unlikely(PAGETYPE_WHOLE(p) & P_SUBP) \ - ? PAGETYPE_WHOLE(p) & ~(P_SUBP | P_LEGACY_DIRTY) \ - : PAGETYPE_WHOLE(p)) - -/* Size of the page header, excluding dynamic data at the end */ -#define PAGEHDRSZ offsetof(MDBX_page, mp_ptrs) +/* Header for a single key/data pair within a page. + * Used in pages of type P_BRANCH and P_LEAF without P_DUPFIX. + * We guarantee 2-byte alignment for 'node_t's. + * + * Leaf node flags describe node contents. N_BIG says the node's + * data part is the page number of an overflow page with actual data. + * N_DUP and N_TREE can be combined giving duplicate data in + * a sub-page/table, and named databases (just N_TREE). */ +typedef struct node { +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + union { + uint32_t dsize; + uint32_t child_pgno; + }; + uint8_t flags; /* see node_flags */ + uint8_t extra; + uint16_t ksize; /* key size */ +#else + uint16_t ksize; /* key size */ + uint8_t extra; + uint8_t flags; /* see node_flags */ + union { + uint32_t child_pgno; + uint32_t dsize; + }; +#endif /* __BYTE_ORDER__ */ -/* Pointer displacement without casting to char* to avoid pointer-aliasing */ -#define ptr_disp(ptr, disp) ((void *)(((intptr_t)(ptr)) + ((intptr_t)(disp)))) +#if FLEXIBLE_ARRAY_MEMBERS + uint8_t payload[] /* key and data are appended here */; +#endif /* FLEXIBLE_ARRAY_MEMBERS */ +} node_t; -/* Pointer distance as signed number of bytes */ -#define ptr_dist(more, less) (((intptr_t)(more)) - ((intptr_t)(less))) +/* Size of the node header, excluding dynamic data at the end */ +#define NODESIZE 8u -#define mp_next(mp) \ - (*(MDBX_page **)ptr_disp((mp)->mp_ptrs, sizeof(void *) - sizeof(uint32_t))) +typedef enum node_flags { + N_BIG = 0x01 /* data put on large page */, + N_TREE = 0x02 /* data is a b-tree */, + N_DUP = 0x04 /* data has duplicates */ +} node_flags_t; #pragma pack(pop) -typedef struct profgc_stat { +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint8_t page_type(const page_t *mp) { return mp->flags; } + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint8_t page_type_compat(const page_t *mp) { + /* Drop legacy P_DIRTY flag for sub-pages for compatilibity, + * for assertions only. */ + return unlikely(mp->flags & P_SUBP) ? mp->flags & ~(P_SUBP | P_LEGACY_DIRTY) : mp->flags; +} + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_leaf(const page_t *mp) { + return (mp->flags & P_LEAF) != 0; +} + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_dupfix_leaf(const page_t *mp) { + return (mp->flags & P_DUPFIX) != 0; +} + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_branch(const page_t *mp) { + return (mp->flags & P_BRANCH) != 0; +} + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_largepage(const page_t *mp) { + return (mp->flags & P_LARGE) != 0; +} + +MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline bool is_subpage(const page_t *mp) { + return (mp->flags & P_SUBP) != 0; +} + +/* The version number for a database's lockfile format. */ +#define MDBX_LOCK_VERSION 6 + +#if MDBX_LOCKING == MDBX_LOCKING_WIN32FILES + +#define MDBX_LCK_SIGN UINT32_C(0xF10C) +typedef void osal_ipclock_t; +#elif MDBX_LOCKING == MDBX_LOCKING_SYSV + +#define MDBX_LCK_SIGN UINT32_C(0xF18D) +typedef mdbx_pid_t osal_ipclock_t; + +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || MDBX_LOCKING == MDBX_LOCKING_POSIX2008 + +#define MDBX_LCK_SIGN UINT32_C(0x8017) +typedef pthread_mutex_t osal_ipclock_t; + +#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988 + +#define MDBX_LCK_SIGN UINT32_C(0xFC29) +typedef sem_t osal_ipclock_t; + +#else +#error "FIXME" +#endif /* MDBX_LOCKING */ + +/* Статистика профилирования работы GC */ +typedef struct gc_prof_stat { /* Монотонное время по "настенным часам" * затраченное на чтение и поиск внутри GC */ uint64_t rtime_monotonic; @@ -3017,42 +2544,44 @@ typedef struct profgc_stat { uint32_t spe_counter; /* page faults (hard page faults) */ uint32_t majflt; -} profgc_stat_t; - -/* Statistics of page operations overall of all (running, completed and aborted) - * transactions */ -typedef struct pgop_stat { - MDBX_atomic_uint64_t newly; /* Quantity of a new pages added */ - MDBX_atomic_uint64_t cow; /* Quantity of pages copied for update */ - MDBX_atomic_uint64_t clone; /* Quantity of parent's dirty pages clones + /* Для разборок с pnl_merge() */ + struct { + uint64_t time; + uint64_t volume; + uint32_t calls; + } pnl_merge; +} gc_prof_stat_t; + +/* Statistics of pages operations for all transactions, + * including incomplete and aborted. */ +typedef struct pgops { + mdbx_atomic_uint64_t newly; /* Quantity of a new pages added */ + mdbx_atomic_uint64_t cow; /* Quantity of pages copied for update */ + mdbx_atomic_uint64_t clone; /* Quantity of parent's dirty pages clones for nested transactions */ - MDBX_atomic_uint64_t split; /* Page splits */ - MDBX_atomic_uint64_t merge; /* Page merges */ - MDBX_atomic_uint64_t spill; /* Quantity of spilled dirty pages */ - MDBX_atomic_uint64_t unspill; /* Quantity of unspilled/reloaded pages */ - MDBX_atomic_uint64_t - wops; /* Number of explicit write operations (not a pages) to a disk */ - MDBX_atomic_uint64_t - msync; /* Number of explicit msync/flush-to-disk operations */ - MDBX_atomic_uint64_t - fsync; /* Number of explicit fsync/flush-to-disk operations */ - - MDBX_atomic_uint64_t prefault; /* Number of prefault write operations */ - MDBX_atomic_uint64_t mincore; /* Number of mincore() calls */ - - MDBX_atomic_uint32_t - incoherence; /* number of https://libmdbx.dqdkfa.ru/dead-github/issues/269 - caught */ - MDBX_atomic_uint32_t reserved; + mdbx_atomic_uint64_t split; /* Page splits */ + mdbx_atomic_uint64_t merge; /* Page merges */ + mdbx_atomic_uint64_t spill; /* Quantity of spilled dirty pages */ + mdbx_atomic_uint64_t unspill; /* Quantity of unspilled/reloaded pages */ + mdbx_atomic_uint64_t wops; /* Number of explicit write operations (not a pages) to a disk */ + mdbx_atomic_uint64_t msync; /* Number of explicit msync/flush-to-disk operations */ + mdbx_atomic_uint64_t fsync; /* Number of explicit fsync/flush-to-disk operations */ + + mdbx_atomic_uint64_t prefault; /* Number of prefault write operations */ + mdbx_atomic_uint64_t mincore; /* Number of mincore() calls */ + + mdbx_atomic_uint32_t incoherence; /* number of https://libmdbx.dqdkfa.ru/dead-github/issues/269 + caught */ + mdbx_atomic_uint32_t reserved; /* Статистика для профилирования GC. - * Логически эти данные может быть стоит вынести в другую структуру, + * Логически эти данные, возможно, стоит вынести в другую структуру, * но разница будет сугубо косметическая. */ struct { /* Затраты на поддержку данных пользователя */ - profgc_stat_t work; + gc_prof_stat_t work; /* Затраты на поддержку и обновления самой GC */ - profgc_stat_t self; + gc_prof_stat_t self; /* Итераций обновления GC, * больше 1 если были повторы/перезапуски */ uint32_t wloops; @@ -3067,33 +2596,6 @@ typedef struct pgop_stat { } gc_prof; } pgop_stat_t; -#if MDBX_LOCKING == MDBX_LOCKING_WIN32FILES -#define MDBX_CLOCK_SIGN UINT32_C(0xF10C) -typedef void osal_ipclock_t; -#elif MDBX_LOCKING == MDBX_LOCKING_SYSV - -#define MDBX_CLOCK_SIGN UINT32_C(0xF18D) -typedef mdbx_pid_t osal_ipclock_t; -#ifndef EOWNERDEAD -#define EOWNERDEAD MDBX_RESULT_TRUE -#endif - -#elif MDBX_LOCKING == MDBX_LOCKING_POSIX2001 || \ - MDBX_LOCKING == MDBX_LOCKING_POSIX2008 -#define MDBX_CLOCK_SIGN UINT32_C(0x8017) -typedef pthread_mutex_t osal_ipclock_t; -#elif MDBX_LOCKING == MDBX_LOCKING_POSIX1988 -#define MDBX_CLOCK_SIGN UINT32_C(0xFC29) -typedef sem_t osal_ipclock_t; -#else -#error "FIXME" -#endif /* MDBX_LOCKING */ - -#if MDBX_LOCKING > MDBX_LOCKING_SYSV && !defined(__cplusplus) -MDBX_INTERNAL_FUNC int osal_ipclock_stub(osal_ipclock_t *ipc); -MDBX_INTERNAL_FUNC int osal_ipclock_destroy(osal_ipclock_t *ipc); -#endif /* MDBX_LOCKING */ - /* Reader Lock Table * * Readers don't acquire any locks for their data access. Instead, they @@ -3103,8 +2605,9 @@ MDBX_INTERNAL_FUNC int osal_ipclock_destroy(osal_ipclock_t *ipc); * read transactions started by the same thread need no further locking to * proceed. * - * If MDBX_NOTLS is set, the slot address is not saved in thread-specific data. - * No reader table is used if the database is on a read-only filesystem. + * If MDBX_NOSTICKYTHREADS is set, the slot address is not saved in + * thread-specific data. No reader table is used if the database is on a + * read-only filesystem. * * Since the database uses multi-version concurrency control, readers don't * actually need any locking. This table is used to keep track of which @@ -3133,14 +2636,14 @@ MDBX_INTERNAL_FUNC int osal_ipclock_destroy(osal_ipclock_t *ipc); * many old transactions together. */ /* The actual reader record, with cacheline padding. */ -typedef struct MDBX_reader { - /* Current Transaction ID when this transaction began, or (txnid_t)-1. +typedef struct reader_slot { + /* Current Transaction ID when this transaction began, or INVALID_TXNID. * Multiple readers that start at the same time will probably have the * same ID here. Again, it's not important to exclude them from * anything; all we need to know is which version of the DB they * started from so we can avoid overwriting any data used in that * particular version. */ - MDBX_atomic_uint64_t /* txnid_t */ mr_txnid; + atomic_txnid_t txnid; /* The information we store in a single slot of the reader table. * In addition to a transaction ID, we also record the process and @@ -3151,708 +2654,320 @@ typedef struct MDBX_reader { * We simply re-init the table when we know that we're the only process * opening the lock file. */ + /* Псевдо thread_id для пометки вытесненных читающих транзакций. */ +#define MDBX_TID_TXN_OUSTED (UINT64_MAX - 1) + + /* Псевдо thread_id для пометки припаркованных читающих транзакций. */ +#define MDBX_TID_TXN_PARKED UINT64_MAX + /* The thread ID of the thread owning this txn. */ - MDBX_atomic_uint64_t mr_tid; + mdbx_atomic_uint64_t tid; /* The process ID of the process owning this reader txn. */ - MDBX_atomic_uint32_t mr_pid; + mdbx_atomic_uint32_t pid; /* The number of pages used in the reader's MVCC snapshot, - * i.e. the value of meta->mm_geo.next and txn->mt_next_pgno */ - atomic_pgno_t mr_snapshot_pages_used; + * i.e. the value of meta->geometry.first_unallocated and + * txn->geo.first_unallocated */ + atomic_pgno_t snapshot_pages_used; /* Number of retired pages at the time this reader starts transaction. So, - * at any time the difference mm_pages_retired - mr_snapshot_pages_retired - * will give the number of pages which this reader restraining from reuse. */ - MDBX_atomic_uint64_t mr_snapshot_pages_retired; -} MDBX_reader; + * at any time the difference meta.pages_retired - + * reader.snapshot_pages_retired will give the number of pages which this + * reader restraining from reuse. */ + mdbx_atomic_uint64_t snapshot_pages_retired; +} reader_slot_t; /* The header for the reader table (a memory-mapped lock file). */ -typedef struct MDBX_lockinfo { +typedef struct shared_lck { /* Stamp identifying this as an MDBX file. * It must be set to MDBX_MAGIC with with MDBX_LOCK_VERSION. */ - uint64_t mti_magic_and_version; + uint64_t magic_and_version; /* Format of this lock file. Must be set to MDBX_LOCK_FORMAT. */ - uint32_t mti_os_and_format; + uint32_t os_and_format; /* Flags which environment was opened. */ - MDBX_atomic_uint32_t mti_envmode; + mdbx_atomic_uint32_t envmode; /* Threshold of un-synced-with-disk pages for auto-sync feature, * zero means no-threshold, i.e. auto-sync is disabled. */ - atomic_pgno_t mti_autosync_threshold; + atomic_pgno_t autosync_threshold; /* Low 32-bit of txnid with which meta-pages was synced, * i.e. for sync-polling in the MDBX_NOMETASYNC mode. */ #define MDBX_NOMETASYNC_LAZY_UNK (UINT32_MAX / 3) #define MDBX_NOMETASYNC_LAZY_FD (MDBX_NOMETASYNC_LAZY_UNK + UINT32_MAX / 8) -#define MDBX_NOMETASYNC_LAZY_WRITEMAP \ - (MDBX_NOMETASYNC_LAZY_UNK - UINT32_MAX / 8) - MDBX_atomic_uint32_t mti_meta_sync_txnid; +#define MDBX_NOMETASYNC_LAZY_WRITEMAP (MDBX_NOMETASYNC_LAZY_UNK - UINT32_MAX / 8) + mdbx_atomic_uint32_t meta_sync_txnid; /* Period for timed auto-sync feature, i.e. at the every steady checkpoint - * the mti_unsynced_timeout sets to the current_time + mti_autosync_period. + * the mti_unsynced_timeout sets to the current_time + autosync_period. * The time value is represented in a suitable system-dependent form, for * example clock_gettime(CLOCK_BOOTTIME) or clock_gettime(CLOCK_MONOTONIC). * Zero means timed auto-sync is disabled. */ - MDBX_atomic_uint64_t mti_autosync_period; + mdbx_atomic_uint64_t autosync_period; /* Marker to distinguish uniqueness of DB/CLK. */ - MDBX_atomic_uint64_t mti_bait_uniqueness; + mdbx_atomic_uint64_t bait_uniqueness; /* Paired counter of processes that have mlock()ed part of mmapped DB. - * The (mti_mlcnt[0] - mti_mlcnt[1]) > 0 means at least one process + * The (mlcnt[0] - mlcnt[1]) > 0 means at least one process * lock at least one page, so therefore madvise() could return EINVAL. */ - MDBX_atomic_uint32_t mti_mlcnt[2]; + mdbx_atomic_uint32_t mlcnt[2]; MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/ /* Statistics of costly ops of all (running, completed and aborted) * transactions */ - pgop_stat_t mti_pgop_stat; + pgop_stat_t pgops; MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/ - /* Write transaction lock. */ #if MDBX_LOCKING > 0 - osal_ipclock_t mti_wlock; + /* Write transaction lock. */ + osal_ipclock_t wrt_lock; #endif /* MDBX_LOCKING > 0 */ - atomic_txnid_t mti_oldest_reader; + atomic_txnid_t cached_oldest; /* Timestamp of entering an out-of-sync state. Value is represented in a * suitable system-dependent form, for example clock_gettime(CLOCK_BOOTTIME) * or clock_gettime(CLOCK_MONOTONIC). */ - MDBX_atomic_uint64_t mti_eoos_timestamp; + mdbx_atomic_uint64_t eoos_timestamp; /* Number un-synced-with-disk pages for auto-sync feature. */ - MDBX_atomic_uint64_t mti_unsynced_pages; + mdbx_atomic_uint64_t unsynced_pages; /* Timestamp of the last readers check. */ - MDBX_atomic_uint64_t mti_reader_check_timestamp; + mdbx_atomic_uint64_t readers_check_timestamp; /* Number of page which was discarded last time by madvise(DONTNEED). */ - atomic_pgno_t mti_discarded_tail; + atomic_pgno_t discarded_tail; /* Shared anchor for tracking readahead edge and enabled/disabled status. */ - pgno_t mti_readahead_anchor; + pgno_t readahead_anchor; /* Shared cache for mincore() results */ struct { pgno_t begin[4]; uint64_t mask[4]; - } mti_mincore_cache; + } mincore_cache; MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/ - /* Readeaders registration lock. */ #if MDBX_LOCKING > 0 - osal_ipclock_t mti_rlock; + /* Readeaders table lock. */ + osal_ipclock_t rdt_lock; #endif /* MDBX_LOCKING > 0 */ /* The number of slots that have been used in the reader table. * This always records the maximum count, it is not decremented * when readers release their slots. */ - MDBX_atomic_uint32_t mti_numreaders; - MDBX_atomic_uint32_t mti_readers_refresh_flag; + mdbx_atomic_uint32_t rdt_length; + mdbx_atomic_uint32_t rdt_refresh_flag; -#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ - (!defined(__cplusplus) && defined(_MSC_VER)) +#if FLEXIBLE_ARRAY_MEMBERS MDBX_ALIGNAS(MDBX_CACHELINE_SIZE) /* cacheline ----------------------------*/ - MDBX_reader mti_readers[] /* dynamic size */; -#endif /* C99 */ -} MDBX_lockinfo; + reader_slot_t rdt[] /* dynamic size */; /* Lockfile format signature: version, features and field layout */ -#define MDBX_LOCK_FORMAT \ - (MDBX_CLOCK_SIGN * 27733 + (unsigned)sizeof(MDBX_reader) * 13 + \ - (unsigned)offsetof(MDBX_reader, mr_snapshot_pages_used) * 251 + \ - (unsigned)offsetof(MDBX_lockinfo, mti_oldest_reader) * 83 + \ - (unsigned)offsetof(MDBX_lockinfo, mti_numreaders) * 37 + \ - (unsigned)offsetof(MDBX_lockinfo, mti_readers) * 29) - -#define MDBX_DATA_MAGIC \ - ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + MDBX_DATA_VERSION) - -#define MDBX_DATA_MAGIC_LEGACY_COMPAT \ - ((MDBX_MAGIC << 8) + MDBX_PNL_ASCENDING * 64 + 2) - -#define MDBX_DATA_MAGIC_LEGACY_DEVEL ((MDBX_MAGIC << 8) + 255) +#define MDBX_LOCK_FORMAT \ + (MDBX_LCK_SIGN * 27733 + (unsigned)sizeof(reader_slot_t) * 13 + \ + (unsigned)offsetof(reader_slot_t, snapshot_pages_used) * 251 + (unsigned)offsetof(lck_t, cached_oldest) * 83 + \ + (unsigned)offsetof(lck_t, rdt_length) * 37 + (unsigned)offsetof(lck_t, rdt) * 29) +#endif /* FLEXIBLE_ARRAY_MEMBERS */ +} lck_t; #define MDBX_LOCK_MAGIC ((MDBX_MAGIC << 8) + MDBX_LOCK_VERSION) -/* The maximum size of a database page. - * - * It is 64K, but value-PAGEHDRSZ must fit in MDBX_page.mp_upper. - * - * MDBX will use database pages < OS pages if needed. - * That causes more I/O in write transactions: The OS must - * know (read) the whole page before writing a partial page. - * - * Note that we don't currently support Huge pages. On Linux, - * regular data files cannot use Huge pages, and in general - * Huge pages aren't actually pageable. We rely on the OS - * demand-pager to read our data and page it out when memory - * pressure from other processes is high. So until OSs have - * actual paging support for Huge pages, they're not viable. */ -#define MAX_PAGESIZE MDBX_MAX_PAGESIZE -#define MIN_PAGESIZE MDBX_MIN_PAGESIZE - -#define MIN_MAPSIZE (MIN_PAGESIZE * MIN_PAGENO) +#define MDBX_READERS_LIMIT 32767 + +#define MIN_MAPSIZE (MDBX_MIN_PAGESIZE * MIN_PAGENO) #if defined(_WIN32) || defined(_WIN64) #define MAX_MAPSIZE32 UINT32_C(0x38000000) #else #define MAX_MAPSIZE32 UINT32_C(0x7f000000) #endif -#define MAX_MAPSIZE64 ((MAX_PAGENO + 1) * (uint64_t)MAX_PAGESIZE) +#define MAX_MAPSIZE64 ((MAX_PAGENO + 1) * (uint64_t)MDBX_MAX_PAGESIZE) #if MDBX_WORDBITS >= 64 #define MAX_MAPSIZE MAX_MAPSIZE64 -#define MDBX_PGL_LIMIT ((size_t)MAX_PAGENO) +#define PAGELIST_LIMIT ((size_t)MAX_PAGENO) #else #define MAX_MAPSIZE MAX_MAPSIZE32 -#define MDBX_PGL_LIMIT (MAX_MAPSIZE32 / MIN_PAGESIZE) +#define PAGELIST_LIMIT (MAX_MAPSIZE32 / MDBX_MIN_PAGESIZE) #endif /* MDBX_WORDBITS */ -#define MDBX_READERS_LIMIT 32767 -#define MDBX_RADIXSORT_THRESHOLD 142 #define MDBX_GOLD_RATIO_DBL 1.6180339887498948482 +#define MEGABYTE ((size_t)1 << 20) /*----------------------------------------------------------------------------*/ -/* An PNL is an Page Number List, a sorted array of IDs. - * The first element of the array is a counter for how many actual page-numbers - * are in the list. By default PNLs are sorted in descending order, this allow - * cut off a page with lowest pgno (at the tail) just truncating the list. The - * sort order of PNLs is controlled by the MDBX_PNL_ASCENDING build option. */ -typedef pgno_t *MDBX_PNL; - -#if MDBX_PNL_ASCENDING -#define MDBX_PNL_ORDERED(first, last) ((first) < (last)) -#define MDBX_PNL_DISORDERED(first, last) ((first) >= (last)) -#else -#define MDBX_PNL_ORDERED(first, last) ((first) > (last)) -#define MDBX_PNL_DISORDERED(first, last) ((first) <= (last)) -#endif +union logger_union { + void *ptr; + MDBX_debug_func *fmt; + MDBX_debug_func_nofmt *nofmt; +}; -/* List of txnid, only for MDBX_txn.tw.lifo_reclaimed */ -typedef txnid_t *MDBX_TXL; +struct libmdbx_globals { + bin128_t bootid; + unsigned sys_pagesize, sys_allocation_granularity; + uint8_t sys_pagesize_ln2; + uint8_t runtime_flags; + uint8_t loglevel; +#if defined(_WIN32) || defined(_WIN64) + bool running_under_Wine; +#elif defined(__linux__) || defined(__gnu_linux__) + bool running_on_WSL1 /* Windows Subsystem 1 for Linux */; + uint32_t linux_kernel_version; +#endif /* Linux */ + union logger_union logger; + osal_fastmutex_t debug_lock; + size_t logger_buffer_size; + char *logger_buffer; +}; -/* An Dirty-Page list item is an pgno/pointer pair. */ -typedef struct MDBX_dp { - MDBX_page *ptr; - pgno_t pgno, npages; -} MDBX_dp; +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ -/* An DPL (dirty-page list) is a sorted array of MDBX_DPs. */ -typedef struct MDBX_dpl { - size_t sorted; - size_t length; - size_t pages_including_loose; /* number of pages, but not an entries. */ - size_t detent; /* allocated size excluding the MDBX_DPL_RESERVE_GAP */ -#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ - (!defined(__cplusplus) && defined(_MSC_VER)) - MDBX_dp items[] /* dynamic size with holes at zero and after the last */; -#endif -} MDBX_dpl; +extern struct libmdbx_globals globals; +#if defined(_WIN32) || defined(_WIN64) +extern struct libmdbx_imports imports; +#endif /* Windows */ -/* PNL sizes */ -#define MDBX_PNL_GRANULATE_LOG2 10 -#define MDBX_PNL_GRANULATE (1 << MDBX_PNL_GRANULATE_LOG2) -#define MDBX_PNL_INITIAL \ - (MDBX_PNL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t)) +#ifndef __Wpedantic_format_voidptr +MDBX_MAYBE_UNUSED static inline const void *__Wpedantic_format_voidptr(const void *ptr) { return ptr; } +#define __Wpedantic_format_voidptr(ARG) __Wpedantic_format_voidptr(ARG) +#endif /* __Wpedantic_format_voidptr */ -#define MDBX_TXL_GRANULATE 32 -#define MDBX_TXL_INITIAL \ - (MDBX_TXL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t)) -#define MDBX_TXL_MAX \ - ((1u << 26) - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(txnid_t)) +MDBX_INTERNAL void MDBX_PRINTF_ARGS(4, 5) debug_log(int level, const char *function, int line, const char *fmt, ...) + MDBX_PRINTF_ARGS(4, 5); +MDBX_INTERNAL void debug_log_va(int level, const char *function, int line, const char *fmt, va_list args); -#define MDBX_PNL_ALLOCLEN(pl) ((pl)[-1]) -#define MDBX_PNL_GETSIZE(pl) ((size_t)((pl)[0])) -#define MDBX_PNL_SETSIZE(pl, size) \ - do { \ - const size_t __size = size; \ - assert(__size < INT_MAX); \ - (pl)[0] = (pgno_t)__size; \ - } while (0) -#define MDBX_PNL_FIRST(pl) ((pl)[1]) -#define MDBX_PNL_LAST(pl) ((pl)[MDBX_PNL_GETSIZE(pl)]) -#define MDBX_PNL_BEGIN(pl) (&(pl)[1]) -#define MDBX_PNL_END(pl) (&(pl)[MDBX_PNL_GETSIZE(pl) + 1]) +#if MDBX_DEBUG +#define LOG_ENABLED(LVL) unlikely(LVL <= globals.loglevel) +#define AUDIT_ENABLED() unlikely((globals.runtime_flags & (unsigned)MDBX_DBG_AUDIT)) +#else /* MDBX_DEBUG */ +#define LOG_ENABLED(LVL) (LVL < MDBX_LOG_VERBOSE && LVL <= globals.loglevel) +#define AUDIT_ENABLED() (0) +#endif /* LOG_ENABLED() & AUDIT_ENABLED() */ -#if MDBX_PNL_ASCENDING -#define MDBX_PNL_EDGE(pl) ((pl) + 1) -#define MDBX_PNL_LEAST(pl) MDBX_PNL_FIRST(pl) -#define MDBX_PNL_MOST(pl) MDBX_PNL_LAST(pl) +#if MDBX_FORCE_ASSERTIONS +#define ASSERT_ENABLED() (1) +#elif MDBX_DEBUG +#define ASSERT_ENABLED() likely((globals.runtime_flags & (unsigned)MDBX_DBG_ASSERT)) #else -#define MDBX_PNL_EDGE(pl) ((pl) + MDBX_PNL_GETSIZE(pl)) -#define MDBX_PNL_LEAST(pl) MDBX_PNL_LAST(pl) -#define MDBX_PNL_MOST(pl) MDBX_PNL_FIRST(pl) -#endif - -#define MDBX_PNL_SIZEOF(pl) ((MDBX_PNL_GETSIZE(pl) + 1) * sizeof(pgno_t)) -#define MDBX_PNL_IS_EMPTY(pl) (MDBX_PNL_GETSIZE(pl) == 0) - -/*----------------------------------------------------------------------------*/ -/* Internal structures */ - -/* Auxiliary DB info. - * The information here is mostly static/read-only. There is - * only a single copy of this record in the environment. */ -typedef struct MDBX_dbx { - MDBX_val md_name; /* name of the database */ - MDBX_cmp_func *md_cmp; /* function for comparing keys */ - MDBX_cmp_func *md_dcmp; /* function for comparing data items */ - size_t md_klen_min, md_klen_max; /* min/max key length for the database */ - size_t md_vlen_min, - md_vlen_max; /* min/max value/data length for the database */ -} MDBX_dbx; - -typedef struct troika { - uint8_t fsm, recent, prefer_steady, tail_and_flags; -#if MDBX_WORDBITS > 32 /* Workaround for false-positives from Valgrind */ - uint32_t unused_pad; -#endif -#define TROIKA_HAVE_STEADY(troika) ((troika)->fsm & 7) -#define TROIKA_STRICT_VALID(troika) ((troika)->tail_and_flags & 64) -#define TROIKA_VALID(troika) ((troika)->tail_and_flags & 128) -#define TROIKA_TAIL(troika) ((troika)->tail_and_flags & 3) - txnid_t txnid[NUM_METAS]; -} meta_troika_t; - -/* A database transaction. - * Every operation requires a transaction handle. */ -struct MDBX_txn { -#define MDBX_MT_SIGNATURE UINT32_C(0x93D53A31) - uint32_t mt_signature; - - /* Transaction Flags */ - /* mdbx_txn_begin() flags */ -#define MDBX_TXN_RO_BEGIN_FLAGS (MDBX_TXN_RDONLY | MDBX_TXN_RDONLY_PREPARE) -#define MDBX_TXN_RW_BEGIN_FLAGS \ - (MDBX_TXN_NOMETASYNC | MDBX_TXN_NOSYNC | MDBX_TXN_TRY) - /* Additional flag for sync_locked() */ -#define MDBX_SHRINK_ALLOWED UINT32_C(0x40000000) - -#define MDBX_TXN_DRAINED_GC 0x20 /* GC was depleted up to oldest reader */ - -#define TXN_FLAGS \ - (MDBX_TXN_FINISHED | MDBX_TXN_ERROR | MDBX_TXN_DIRTY | MDBX_TXN_SPILLS | \ - MDBX_TXN_HAS_CHILD | MDBX_TXN_INVALID | MDBX_TXN_DRAINED_GC) - -#if (TXN_FLAGS & (MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_RO_BEGIN_FLAGS)) || \ - ((MDBX_TXN_RW_BEGIN_FLAGS | MDBX_TXN_RO_BEGIN_FLAGS | TXN_FLAGS) & \ - MDBX_SHRINK_ALLOWED) -#error "Oops, some txn flags overlapped or wrong" -#endif - uint32_t mt_flags; - - MDBX_txn *mt_parent; /* parent of a nested txn */ - /* Nested txn under this txn, set together with flag MDBX_TXN_HAS_CHILD */ - MDBX_txn *mt_child; - MDBX_geo mt_geo; - /* next unallocated page */ -#define mt_next_pgno mt_geo.next - /* corresponding to the current size of datafile */ -#define mt_end_pgno mt_geo.now - - /* The ID of this transaction. IDs are integers incrementing from - * INITIAL_TXNID. Only committed write transactions increment the ID. If a - * transaction aborts, the ID may be re-used by the next writer. */ - txnid_t mt_txnid; - txnid_t mt_front; - - MDBX_env *mt_env; /* the DB environment */ - /* Array of records for each DB known in the environment. */ - MDBX_dbx *mt_dbxs; - /* Array of MDBX_db records for each known DB */ - MDBX_db *mt_dbs; - /* Array of sequence numbers for each DB handle */ - MDBX_atomic_uint32_t *mt_dbiseqs; - - /* Transaction DBI Flags */ -#define DBI_DIRTY MDBX_DBI_DIRTY /* DB was written in this txn */ -#define DBI_STALE MDBX_DBI_STALE /* Named-DB record is older than txnID */ -#define DBI_FRESH MDBX_DBI_FRESH /* Named-DB handle opened in this txn */ -#define DBI_CREAT MDBX_DBI_CREAT /* Named-DB handle created in this txn */ -#define DBI_VALID 0x10 /* DB handle is valid, see also DB_VALID */ -#define DBI_USRVALID 0x20 /* As DB_VALID, but not set for FREE_DBI */ -#define DBI_AUDITED 0x40 /* Internal flag for accounting during audit */ - /* Array of flags for each DB */ - uint8_t *mt_dbistate; - /* Number of DB records in use, or 0 when the txn is finished. - * This number only ever increments until the txn finishes; we - * don't decrement it when individual DB handles are closed. */ - MDBX_dbi mt_numdbs; - size_t mt_owner; /* thread ID that owns this transaction */ - MDBX_canary mt_canary; - void *mt_userctx; /* User-settable context */ - MDBX_cursor **mt_cursors; +#define ASSERT_ENABLED() (0) +#endif /* ASSERT_ENABLED() */ - union { - struct { - /* For read txns: This thread/txn's reader table slot, or NULL. */ - MDBX_reader *reader; - } to; - struct { - meta_troika_t troika; - /* In write txns, array of cursors for each DB */ - MDBX_PNL relist; /* Reclaimed GC pages */ - txnid_t last_reclaimed; /* ID of last used record */ -#if MDBX_ENABLE_REFUND - pgno_t loose_refund_wl /* FIXME: describe */; -#endif /* MDBX_ENABLE_REFUND */ - /* a sequence to spilling dirty page with LRU policy */ - unsigned dirtylru; - /* dirtylist room: Dirty array size - dirty pages visible to this txn. - * Includes ancestor txns' dirty pages not hidden by other txns' - * dirty/spilled pages. Thus commit(nested txn) has room to merge - * dirtylist into mt_parent after freeing hidden mt_parent pages. */ - size_t dirtyroom; - /* For write txns: Modified pages. Sorted when not MDBX_WRITEMAP. */ - MDBX_dpl *dirtylist; - /* The list of reclaimed txns from GC */ - MDBX_TXL lifo_reclaimed; - /* The list of pages that became unused during this transaction. */ - MDBX_PNL retired_pages; - /* The list of loose pages that became unused and may be reused - * in this transaction, linked through `mp_next`. */ - MDBX_page *loose_pages; - /* Number of loose pages (tw.loose_pages) */ - size_t loose_count; - union { - struct { - size_t least_removed; - /* The sorted list of dirty pages we temporarily wrote to disk - * because the dirty list was full. page numbers in here are - * shifted left by 1, deleted slots have the LSB set. */ - MDBX_PNL list; - } spilled; - size_t writemap_dirty_npages; - size_t writemap_spilled_npages; - }; - } tw; - }; -}; +#define DEBUG_EXTRA(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_EXTRA)) \ + debug_log(MDBX_LOG_EXTRA, __func__, __LINE__, fmt, __VA_ARGS__); \ + } while (0) -#if MDBX_WORDBITS >= 64 -#define CURSOR_STACK 32 -#else -#define CURSOR_STACK 24 -#endif - -struct MDBX_xcursor; - -/* Cursors are used for all DB operations. - * A cursor holds a path of (page pointer, key index) from the DB - * root to a position in the DB, plus other state. MDBX_DUPSORT - * cursors include an xcursor to the current data item. Write txns - * track their cursors and keep them up to date when data moves. - * Exception: An xcursor's pointer to a P_SUBP page can be stale. - * (A node with F_DUPDATA but no F_SUBDATA contains a subpage). */ -struct MDBX_cursor { -#define MDBX_MC_LIVE UINT32_C(0xFE05D5B1) -#define MDBX_MC_READY4CLOSE UINT32_C(0x2817A047) -#define MDBX_MC_WAIT4EOT UINT32_C(0x90E297A7) - uint32_t mc_signature; - /* The database handle this cursor operates on */ - MDBX_dbi mc_dbi; - /* Next cursor on this DB in this txn */ - MDBX_cursor *mc_next; - /* Backup of the original cursor if this cursor is a shadow */ - MDBX_cursor *mc_backup; - /* Context used for databases with MDBX_DUPSORT, otherwise NULL */ - struct MDBX_xcursor *mc_xcursor; - /* The transaction that owns this cursor */ - MDBX_txn *mc_txn; - /* The database record for this cursor */ - MDBX_db *mc_db; - /* The database auxiliary record for this cursor */ - MDBX_dbx *mc_dbx; - /* The mt_dbistate for this database */ - uint8_t *mc_dbistate; - uint8_t mc_snum; /* number of pushed pages */ - uint8_t mc_top; /* index of top page, normally mc_snum-1 */ - - /* Cursor state flags. */ -#define C_INITIALIZED 0x01 /* cursor has been initialized and is valid */ -#define C_EOF 0x02 /* No more data */ -#define C_SUB 0x04 /* Cursor is a sub-cursor */ -#define C_DEL 0x08 /* last op was a cursor_del */ -#define C_UNTRACK 0x10 /* Un-track cursor when closing */ -#define C_GCU \ - 0x20 /* Происходит подготовка к обновлению GC, поэтому \ - * можно брать страницы из GC даже для FREE_DBI */ - uint8_t mc_flags; - - /* Cursor checking flags. */ -#define CC_BRANCH 0x01 /* same as P_BRANCH for CHECK_LEAF_TYPE() */ -#define CC_LEAF 0x02 /* same as P_LEAF for CHECK_LEAF_TYPE() */ -#define CC_OVERFLOW 0x04 /* same as P_OVERFLOW for CHECK_LEAF_TYPE() */ -#define CC_UPDATING 0x08 /* update/rebalance pending */ -#define CC_SKIPORD 0x10 /* don't check keys ordering */ -#define CC_LEAF2 0x20 /* same as P_LEAF2 for CHECK_LEAF_TYPE() */ -#define CC_RETIRING 0x40 /* refs to child pages may be invalid */ -#define CC_PAGECHECK 0x80 /* perform page checking, see MDBX_VALIDATION */ - uint8_t mc_checking; - - MDBX_page *mc_pg[CURSOR_STACK]; /* stack of pushed pages */ - indx_t mc_ki[CURSOR_STACK]; /* stack of page indices */ -}; +#define DEBUG_EXTRA_PRINT(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_EXTRA)) \ + debug_log(MDBX_LOG_EXTRA, nullptr, 0, fmt, __VA_ARGS__); \ + } while (0) -#define CHECK_LEAF_TYPE(mc, mp) \ - (((PAGETYPE_WHOLE(mp) ^ (mc)->mc_checking) & \ - (CC_BRANCH | CC_LEAF | CC_OVERFLOW | CC_LEAF2)) == 0) - -/* Context for sorted-dup records. - * We could have gone to a fully recursive design, with arbitrarily - * deep nesting of sub-databases. But for now we only handle these - * levels - main DB, optional sub-DB, sorted-duplicate DB. */ -typedef struct MDBX_xcursor { - /* A sub-cursor for traversing the Dup DB */ - MDBX_cursor mx_cursor; - /* The database record for this Dup DB */ - MDBX_db mx_db; - /* The auxiliary DB record for this Dup DB */ - MDBX_dbx mx_dbx; -} MDBX_xcursor; - -typedef struct MDBX_cursor_couple { - MDBX_cursor outer; - void *mc_userctx; /* User-settable context */ - MDBX_xcursor inner; -} MDBX_cursor_couple; - -/* The database environment. */ -struct MDBX_env { - /* ----------------------------------------------------- mostly static part */ -#define MDBX_ME_SIGNATURE UINT32_C(0x9A899641) - MDBX_atomic_uint32_t me_signature; - /* Failed to update the meta page. Probably an I/O error. */ -#define MDBX_FATAL_ERROR UINT32_C(0x80000000) - /* Some fields are initialized. */ -#define MDBX_ENV_ACTIVE UINT32_C(0x20000000) - /* me_txkey is set */ -#define MDBX_ENV_TXKEY UINT32_C(0x10000000) - /* Legacy MDBX_MAPASYNC (prior v0.9) */ -#define MDBX_DEPRECATED_MAPASYNC UINT32_C(0x100000) - /* Legacy MDBX_COALESCE (prior v0.12) */ -#define MDBX_DEPRECATED_COALESCE UINT32_C(0x2000000) -#define ENV_INTERNAL_FLAGS (MDBX_FATAL_ERROR | MDBX_ENV_ACTIVE | MDBX_ENV_TXKEY) - uint32_t me_flags; - osal_mmap_t me_dxb_mmap; /* The main data file */ -#define me_map me_dxb_mmap.base -#define me_lazy_fd me_dxb_mmap.fd - mdbx_filehandle_t me_dsync_fd, me_fd4meta; -#if defined(_WIN32) || defined(_WIN64) -#define me_overlapped_fd me_ioring.overlapped_fd - HANDLE me_data_lock_event; -#endif /* Windows */ - osal_mmap_t me_lck_mmap; /* The lock file */ -#define me_lfd me_lck_mmap.fd - struct MDBX_lockinfo *me_lck; - - unsigned me_psize; /* DB page size, initialized from me_os_psize */ - uint16_t me_leaf_nodemax; /* max size of a leaf-node */ - uint16_t me_branch_nodemax; /* max size of a branch-node */ - uint16_t me_subpage_limit; - uint16_t me_subpage_room_threshold; - uint16_t me_subpage_reserve_prereq; - uint16_t me_subpage_reserve_limit; - atomic_pgno_t me_mlocked_pgno; - uint8_t me_psize2log; /* log2 of DB page size */ - int8_t me_stuck_meta; /* recovery-only: target meta page or less that zero */ - uint16_t me_merge_threshold, - me_merge_threshold_gc; /* pages emptier than this are candidates for - merging */ - unsigned me_os_psize; /* OS page size, from osal_syspagesize() */ - unsigned me_maxreaders; /* size of the reader table */ - MDBX_dbi me_maxdbs; /* size of the DB table */ - uint32_t me_pid; /* process ID of this env */ - osal_thread_key_t me_txkey; /* thread-key for readers */ - pathchar_t *me_pathname; /* path to the DB files */ - void *me_pbuf; /* scratch area for DUPSORT put() */ - MDBX_txn *me_txn0; /* preallocated write transaction */ - - MDBX_dbx *me_dbxs; /* array of static DB info */ - uint16_t *me_dbflags; /* array of flags from MDBX_db.md_flags */ - MDBX_atomic_uint32_t *me_dbiseqs; /* array of dbi sequence numbers */ - unsigned - me_maxgc_ov1page; /* Number of pgno_t fit in a single overflow page */ - unsigned me_maxgc_per_branch; - uint32_t me_live_reader; /* have liveness lock in reader table */ - void *me_userctx; /* User-settable context */ - MDBX_hsr_func *me_hsr_callback; /* Callback for kicking laggard readers */ - size_t me_madv_threshold; +#define TRACE(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_TRACE)) \ + debug_log(MDBX_LOG_TRACE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ + } while (0) - struct { - unsigned dp_reserve_limit; - unsigned rp_augment_limit; - unsigned dp_limit; - unsigned dp_initial; - uint8_t dp_loose_limit; - uint8_t spill_max_denominator; - uint8_t spill_min_denominator; - uint8_t spill_parent4child_denominator; - unsigned merge_threshold_16dot16_percent; -#if !(defined(_WIN32) || defined(_WIN64)) - unsigned writethrough_threshold; -#endif /* Windows */ - bool prefault_write; - union { - unsigned all; - /* tracks options with non-auto values but tuned by user */ - struct { - unsigned dp_limit : 1; - unsigned rp_augment_limit : 1; - unsigned prefault_write : 1; - } non_auto; - } flags; - } me_options; - - /* struct me_dbgeo used for accepting db-geo params from user for the new - * database creation, i.e. when mdbx_env_set_geometry() was called before - * mdbx_env_open(). */ - struct { - size_t lower; /* minimal size of datafile */ - size_t upper; /* maximal size of datafile */ - size_t now; /* current size of datafile */ - size_t grow; /* step to grow datafile */ - size_t shrink; /* threshold to shrink datafile */ - } me_dbgeo; - -#if MDBX_LOCKING == MDBX_LOCKING_SYSV - union { - key_t key; - int semid; - } me_sysv_ipc; -#endif /* MDBX_LOCKING == MDBX_LOCKING_SYSV */ - bool me_incore; +#define DEBUG(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_DEBUG)) \ + debug_log(MDBX_LOG_DEBUG, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ + } while (0) - MDBX_env *me_lcklist_next; +#define VERBOSE(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_VERBOSE)) \ + debug_log(MDBX_LOG_VERBOSE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ + } while (0) - /* --------------------------------------------------- mostly volatile part */ +#define NOTICE(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_NOTICE)) \ + debug_log(MDBX_LOG_NOTICE, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ + } while (0) - MDBX_txn *me_txn; /* current write transaction */ - osal_fastmutex_t me_dbi_lock; - MDBX_dbi me_numdbs; /* number of DBs opened */ - bool me_prefault_write; +#define WARNING(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_WARN)) \ + debug_log(MDBX_LOG_WARN, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ + } while (0) - MDBX_page *me_dp_reserve; /* list of malloc'ed blocks for re-use */ - unsigned me_dp_reserve_len; - /* PNL of pages that became unused in a write txn */ - MDBX_PNL me_retired_pages; - osal_ioring_t me_ioring; +#undef ERROR /* wingdi.h \ + Yeah, morons from M$ put such definition to the public header. */ -#if defined(_WIN32) || defined(_WIN64) - osal_srwlock_t me_remap_guard; - /* Workaround for LockFileEx and WriteFile multithread bug */ - CRITICAL_SECTION me_windowsbug_lock; - char *me_pathname_char; /* cache of multi-byte representation of pathname - to the DB files */ -#else - osal_fastmutex_t me_remap_guard; -#endif +#define ERROR(fmt, ...) \ + do { \ + if (LOG_ENABLED(MDBX_LOG_ERROR)) \ + debug_log(MDBX_LOG_ERROR, __func__, __LINE__, fmt "\n", __VA_ARGS__); \ + } while (0) - /* -------------------------------------------------------------- debugging */ +#define FATAL(fmt, ...) debug_log(MDBX_LOG_FATAL, __func__, __LINE__, fmt "\n", __VA_ARGS__); #if MDBX_DEBUG - MDBX_assert_func *me_assert_func; /* Callback for assertion failures */ -#endif -#ifdef MDBX_USE_VALGRIND - int me_valgrind_handle; -#endif -#if defined(MDBX_USE_VALGRIND) || defined(__SANITIZE_ADDRESS__) - MDBX_atomic_uint32_t me_ignore_EDEADLK; - pgno_t me_poison_edge; -#endif /* MDBX_USE_VALGRIND || __SANITIZE_ADDRESS__ */ +#define ASSERT_FAIL(env, msg, func, line) mdbx_assert_fail(env, msg, func, line) +#else /* MDBX_DEBUG */ +MDBX_NORETURN __cold void assert_fail(const char *msg, const char *func, unsigned line); +#define ASSERT_FAIL(env, msg, func, line) \ + do { \ + (void)(env); \ + assert_fail(msg, func, line); \ + } while (0) +#endif /* MDBX_DEBUG */ -#ifndef xMDBX_DEBUG_SPILLING -#define xMDBX_DEBUG_SPILLING 0 -#endif -#if xMDBX_DEBUG_SPILLING == 2 - size_t debug_dirtied_est, debug_dirtied_act; -#endif /* xMDBX_DEBUG_SPILLING */ +#define ENSURE_MSG(env, expr, msg) \ + do { \ + if (unlikely(!(expr))) \ + ASSERT_FAIL(env, msg, __func__, __LINE__); \ + } while (0) - /* ------------------------------------------------- stub for lck-less mode */ - MDBX_atomic_uint64_t - x_lckless_stub[(sizeof(MDBX_lockinfo) + MDBX_CACHELINE_SIZE - 1) / - sizeof(MDBX_atomic_uint64_t)]; -}; +#define ENSURE(env, expr) ENSURE_MSG(env, expr, #expr) -#ifndef __cplusplus -/*----------------------------------------------------------------------------*/ -/* Cache coherence and mmap invalidation */ +/* assert(3) variant in environment context */ +#define eASSERT(env, expr) \ + do { \ + if (ASSERT_ENABLED()) \ + ENSURE(env, expr); \ + } while (0) -#if MDBX_CPU_WRITEBACK_INCOHERENT -#define osal_flush_incoherent_cpu_writeback() osal_memory_barrier() -#else -#define osal_flush_incoherent_cpu_writeback() osal_compiler_barrier() -#endif /* MDBX_CPU_WRITEBACK_INCOHERENT */ +/* assert(3) variant in cursor context */ +#define cASSERT(mc, expr) eASSERT((mc)->txn->env, expr) -MDBX_MAYBE_UNUSED static __inline void -osal_flush_incoherent_mmap(const void *addr, size_t nbytes, - const intptr_t pagesize) { -#if MDBX_MMAP_INCOHERENT_FILE_WRITE - char *const begin = (char *)(-pagesize & (intptr_t)addr); - char *const end = - (char *)(-pagesize & (intptr_t)((char *)addr + nbytes + pagesize - 1)); - int err = msync(begin, end - begin, MS_SYNC | MS_INVALIDATE) ? errno : 0; - eASSERT(nullptr, err == 0); - (void)err; -#else - (void)pagesize; -#endif /* MDBX_MMAP_INCOHERENT_FILE_WRITE */ +/* assert(3) variant in transaction context */ +#define tASSERT(txn, expr) eASSERT((txn)->env, expr) -#if MDBX_MMAP_INCOHERENT_CPU_CACHE -#ifdef DCACHE - /* MIPS has cache coherency issues. - * Note: for any nbytes >= on-chip cache size, entire is flushed. */ - cacheflush((void *)addr, nbytes, DCACHE); -#else -#error "Oops, cacheflush() not available" -#endif /* DCACHE */ -#endif /* MDBX_MMAP_INCOHERENT_CPU_CACHE */ +#ifndef xMDBX_TOOLS /* Avoid using internal eASSERT() */ +#undef assert +#define assert(expr) eASSERT(nullptr, expr) +#endif -#if !MDBX_MMAP_INCOHERENT_FILE_WRITE && !MDBX_MMAP_INCOHERENT_CPU_CACHE - (void)addr; - (void)nbytes; +MDBX_MAYBE_UNUSED static inline void jitter4testing(bool tiny) { +#if MDBX_DEBUG + if (globals.runtime_flags & (unsigned)MDBX_DBG_JITTER) + osal_jitter(tiny); +#else + (void)tiny; #endif } -/*----------------------------------------------------------------------------*/ -/* Internal prototypes */ - -MDBX_INTERNAL_FUNC int cleanup_dead_readers(MDBX_env *env, int rlocked, - int *dead); -MDBX_INTERNAL_FUNC int rthc_alloc(osal_thread_key_t *key, MDBX_reader *begin, - MDBX_reader *end); -MDBX_INTERNAL_FUNC void rthc_remove(const osal_thread_key_t key); - -MDBX_INTERNAL_FUNC void global_ctor(void); -MDBX_INTERNAL_FUNC void osal_ctor(void); -MDBX_INTERNAL_FUNC void global_dtor(void); -MDBX_INTERNAL_FUNC void osal_dtor(void); -MDBX_INTERNAL_FUNC void thread_dtor(void *ptr); - -#endif /* !__cplusplus */ - -#define MDBX_IS_ERROR(rc) \ - ((rc) != MDBX_RESULT_TRUE && (rc) != MDBX_RESULT_FALSE) - -/* Internal error codes, not exposed outside libmdbx */ -#define MDBX_NO_ROOT (MDBX_LAST_ADDED_ERRCODE + 10) - -/* Debugging output value of a cursor DBI: Negative in a sub-cursor. */ -#define DDBI(mc) \ - (((mc)->mc_flags & C_SUB) ? -(int)(mc)->mc_dbi : (int)(mc)->mc_dbi) +MDBX_MAYBE_UNUSED MDBX_INTERNAL void page_list(page_t *mp); +MDBX_INTERNAL const char *pagetype_caption(const uint8_t type, char buf4unknown[16]); /* Key size which fits in a DKBUF (debug key buffer). */ -#define DKBUF_MAX 511 -#define DKBUF char _kbuf[DKBUF_MAX * 4 + 2] -#define DKEY(x) mdbx_dump_val(x, _kbuf, DKBUF_MAX * 2 + 1) -#define DVAL(x) mdbx_dump_val(x, _kbuf + DKBUF_MAX * 2 + 1, DKBUF_MAX * 2 + 1) +#define DKBUF_MAX 127 +#define DKBUF char dbg_kbuf[DKBUF_MAX * 4 + 2] +#define DKEY(x) mdbx_dump_val(x, dbg_kbuf, DKBUF_MAX * 2 + 1) +#define DVAL(x) mdbx_dump_val(x, dbg_kbuf + DKBUF_MAX * 2 + 1, DKBUF_MAX * 2 + 1) #if MDBX_DEBUG #define DKBUF_DEBUG DKBUF @@ -3864,102 +2979,24 @@ MDBX_INTERNAL_FUNC void thread_dtor(void *ptr); #define DVAL_DEBUG(x) ("-") #endif -/* An invalid page number. - * Mainly used to denote an empty tree. */ -#define P_INVALID (~(pgno_t)0) +MDBX_INTERNAL void log_error(const int err, const char *func, unsigned line); + +MDBX_MAYBE_UNUSED static inline int log_if_error(const int err, const char *func, unsigned line) { + if (unlikely(err != MDBX_SUCCESS)) + log_error(err, func, line); + return err; +} + +#define LOG_IFERR(err) log_if_error((err), __func__, __LINE__) /* Test if the flags f are set in a flag word w. */ #define F_ISSET(w, f) (((w) & (f)) == (f)) /* Round n up to an even number. */ -#define EVEN(n) (((n) + 1UL) & -2L) /* sign-extending -2 to match n+1U */ - -/* Default size of memory map. - * This is certainly too small for any actual applications. Apps should - * always set the size explicitly using mdbx_env_set_geometry(). */ -#define DEFAULT_MAPSIZE MEGABYTE - -/* Number of slots in the reader table. - * This value was chosen somewhat arbitrarily. The 61 is a prime number, - * and such readers plus a couple mutexes fit into single 4KB page. - * Applications should set the table size using mdbx_env_set_maxreaders(). */ -#define DEFAULT_READERS 61 - -/* Test if a page is a leaf page */ -#define IS_LEAF(p) (((p)->mp_flags & P_LEAF) != 0) -/* Test if a page is a LEAF2 page */ -#define IS_LEAF2(p) unlikely(((p)->mp_flags & P_LEAF2) != 0) -/* Test if a page is a branch page */ -#define IS_BRANCH(p) (((p)->mp_flags & P_BRANCH) != 0) -/* Test if a page is an overflow page */ -#define IS_OVERFLOW(p) unlikely(((p)->mp_flags & P_OVERFLOW) != 0) -/* Test if a page is a sub page */ -#define IS_SUBP(p) (((p)->mp_flags & P_SUBP) != 0) - -/* Header for a single key/data pair within a page. - * Used in pages of type P_BRANCH and P_LEAF without P_LEAF2. - * We guarantee 2-byte alignment for 'MDBX_node's. - * - * Leaf node flags describe node contents. F_BIGDATA says the node's - * data part is the page number of an overflow page with actual data. - * F_DUPDATA and F_SUBDATA can be combined giving duplicate data in - * a sub-page/sub-database, and named databases (just F_SUBDATA). */ -typedef struct MDBX_node { -#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ - union { - uint32_t mn_dsize; - uint32_t mn_pgno32; - }; - uint8_t mn_flags; /* see mdbx_node flags */ - uint8_t mn_extra; - uint16_t mn_ksize; /* key size */ -#else - uint16_t mn_ksize; /* key size */ - uint8_t mn_extra; - uint8_t mn_flags; /* see mdbx_node flags */ - union { - uint32_t mn_pgno32; - uint32_t mn_dsize; - }; -#endif /* __BYTE_ORDER__ */ - - /* mdbx_node Flags */ -#define F_BIGDATA 0x01 /* data put on overflow page */ -#define F_SUBDATA 0x02 /* data is a sub-database */ -#define F_DUPDATA 0x04 /* data has duplicates */ +#define EVEN_CEIL(n) (((n) + 1UL) & -2L) /* sign-extending -2 to match n+1U */ - /* valid flags for mdbx_node_add() */ -#define NODE_ADD_FLAGS (F_DUPDATA | F_SUBDATA | MDBX_RESERVE | MDBX_APPEND) - -#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || \ - (!defined(__cplusplus) && defined(_MSC_VER)) - uint8_t mn_data[] /* key and data are appended here */; -#endif /* C99 */ -} MDBX_node; - -#define DB_PERSISTENT_FLAGS \ - (MDBX_REVERSEKEY | MDBX_DUPSORT | MDBX_INTEGERKEY | MDBX_DUPFIXED | \ - MDBX_INTEGERDUP | MDBX_REVERSEDUP) - -/* mdbx_dbi_open() flags */ -#define DB_USABLE_FLAGS (DB_PERSISTENT_FLAGS | MDBX_CREATE | MDBX_DB_ACCEDE) - -#define DB_VALID 0x8000 /* DB handle is valid, for me_dbflags */ -#define DB_INTERNAL_FLAGS DB_VALID - -#if DB_INTERNAL_FLAGS & DB_USABLE_FLAGS -#error "Oops, some flags overlapped or wrong" -#endif -#if DB_PERSISTENT_FLAGS & ~DB_USABLE_FLAGS -#error "Oops, some flags overlapped or wrong" -#endif - -/* Max length of iov-vector passed to writev() call, used for auxilary writes */ -#define MDBX_AUXILARY_IOV_MAX 64 -#if defined(IOV_MAX) && IOV_MAX < MDBX_AUXILARY_IOV_MAX -#undef MDBX_AUXILARY_IOV_MAX -#define MDBX_AUXILARY_IOV_MAX IOV_MAX -#endif /* MDBX_AUXILARY_IOV_MAX */ +/* Round n down to an even number. */ +#define EVEN_FLOOR(n) ((n) & ~(size_t)1) /* * / @@ -3970,106 +3007,226 @@ typedef struct MDBX_node { */ #define CMP2INT(a, b) (((a) != (b)) ? (((a) < (b)) ? -1 : 1) : 0) -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline pgno_t -int64pgno(int64_t i64) { - if (likely(i64 >= (int64_t)MIN_PAGENO && i64 <= (int64_t)MAX_PAGENO + 1)) - return (pgno_t)i64; - return (i64 < (int64_t)MIN_PAGENO) ? MIN_PAGENO : MAX_PAGENO; -} +/* Pointer displacement without casting to char* to avoid pointer-aliasing */ +#define ptr_disp(ptr, disp) ((void *)(((intptr_t)(ptr)) + ((intptr_t)(disp)))) -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline pgno_t -pgno_add(size_t base, size_t augend) { - assert(base <= MAX_PAGENO + 1 && augend < MAX_PAGENO); - return int64pgno((int64_t)base + (int64_t)augend); -} +/* Pointer distance as signed number of bytes */ +#define ptr_dist(more, less) (((intptr_t)(more)) - ((intptr_t)(less))) -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __inline pgno_t -pgno_sub(size_t base, size_t subtrahend) { - assert(base >= MIN_PAGENO && base <= MAX_PAGENO + 1 && - subtrahend < MAX_PAGENO); - return int64pgno((int64_t)base - (int64_t)subtrahend); -} +#define MDBX_ASAN_POISON_MEMORY_REGION(addr, size) \ + do { \ + TRACE("POISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), (size_t)(size), __LINE__); \ + ASAN_POISON_MEMORY_REGION(addr, size); \ + } while (0) -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __always_inline bool -is_powerof2(size_t x) { - return (x & (x - 1)) == 0; +#define MDBX_ASAN_UNPOISON_MEMORY_REGION(addr, size) \ + do { \ + TRACE("UNPOISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), (size_t)(size), __LINE__); \ + ASAN_UNPOISON_MEMORY_REGION(addr, size); \ + } while (0) + +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline size_t branchless_abs(intptr_t value) { + assert(value > INT_MIN); + const size_t expanded_sign = (size_t)(value >> (sizeof(value) * CHAR_BIT - 1)); + return ((size_t)value + expanded_sign) ^ expanded_sign; } -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __always_inline size_t -floor_powerof2(size_t value, size_t granularity) { +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline bool is_powerof2(size_t x) { return (x & (x - 1)) == 0; } + +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline size_t floor_powerof2(size_t value, size_t granularity) { assert(is_powerof2(granularity)); return value & ~(granularity - 1); } -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static __always_inline size_t -ceil_powerof2(size_t value, size_t granularity) { +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline size_t ceil_powerof2(size_t value, size_t granularity) { return floor_powerof2(value + granularity - 1, granularity); } -MDBX_MAYBE_UNUSED MDBX_NOTHROW_CONST_FUNCTION static unsigned -log2n_powerof2(size_t value_uintptr) { - assert(value_uintptr > 0 && value_uintptr < INT32_MAX && - is_powerof2(value_uintptr)); - assert((value_uintptr & -(intptr_t)value_uintptr) == value_uintptr); - const uint32_t value_uint32 = (uint32_t)value_uintptr; -#if __GNUC_PREREQ(4, 1) || __has_builtin(__builtin_ctz) - STATIC_ASSERT(sizeof(value_uint32) <= sizeof(unsigned)); - return __builtin_ctz(value_uint32); -#elif defined(_MSC_VER) - unsigned long index; - STATIC_ASSERT(sizeof(value_uint32) <= sizeof(long)); - _BitScanForward(&index, value_uint32); - return index; +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED MDBX_INTERNAL unsigned log2n_powerof2(size_t value_uintptr); + +MDBX_NOTHROW_CONST_FUNCTION MDBX_INTERNAL uint64_t rrxmrrxmsx_0(uint64_t v); + +struct monotime_cache { + uint64_t value; + int expire_countdown; +}; + +MDBX_MAYBE_UNUSED static inline uint64_t monotime_since_cached(uint64_t begin_timestamp, struct monotime_cache *cache) { + if (cache->expire_countdown) + cache->expire_countdown -= 1; + else { + cache->value = osal_monotime(); + cache->expire_countdown = 42 / 3; + } + return cache->value - begin_timestamp; +} + +/* An PNL is an Page Number List, a sorted array of IDs. + * + * The first element of the array is a counter for how many actual page-numbers + * are in the list. By default PNLs are sorted in descending order, this allow + * cut off a page with lowest pgno (at the tail) just truncating the list. The + * sort order of PNLs is controlled by the MDBX_PNL_ASCENDING build option. */ +typedef pgno_t *pnl_t; +typedef const pgno_t *const_pnl_t; + +#if MDBX_PNL_ASCENDING +#define MDBX_PNL_ORDERED(first, last) ((first) < (last)) +#define MDBX_PNL_DISORDERED(first, last) ((first) >= (last)) #else - static const uint8_t debruijn_ctz32[32] = { - 0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8, - 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9}; - return debruijn_ctz32[(uint32_t)(value_uint32 * 0x077CB531ul) >> 27]; +#define MDBX_PNL_ORDERED(first, last) ((first) > (last)) +#define MDBX_PNL_DISORDERED(first, last) ((first) <= (last)) #endif + +#define MDBX_PNL_GRANULATE_LOG2 10 +#define MDBX_PNL_GRANULATE (1 << MDBX_PNL_GRANULATE_LOG2) +#define MDBX_PNL_INITIAL (MDBX_PNL_GRANULATE - 2 - MDBX_ASSUME_MALLOC_OVERHEAD / sizeof(pgno_t)) + +#define MDBX_PNL_ALLOCLEN(pl) ((pl)[-1]) +#define MDBX_PNL_GETSIZE(pl) ((size_t)((pl)[0])) +#define MDBX_PNL_SETSIZE(pl, size) \ + do { \ + const size_t __size = size; \ + assert(__size < INT_MAX); \ + (pl)[0] = (pgno_t)__size; \ + } while (0) +#define MDBX_PNL_FIRST(pl) ((pl)[1]) +#define MDBX_PNL_LAST(pl) ((pl)[MDBX_PNL_GETSIZE(pl)]) +#define MDBX_PNL_BEGIN(pl) (&(pl)[1]) +#define MDBX_PNL_END(pl) (&(pl)[MDBX_PNL_GETSIZE(pl) + 1]) + +#if MDBX_PNL_ASCENDING +#define MDBX_PNL_EDGE(pl) ((pl) + 1) +#define MDBX_PNL_LEAST(pl) MDBX_PNL_FIRST(pl) +#define MDBX_PNL_MOST(pl) MDBX_PNL_LAST(pl) +#else +#define MDBX_PNL_EDGE(pl) ((pl) + MDBX_PNL_GETSIZE(pl)) +#define MDBX_PNL_LEAST(pl) MDBX_PNL_LAST(pl) +#define MDBX_PNL_MOST(pl) MDBX_PNL_FIRST(pl) +#endif + +#define MDBX_PNL_SIZEOF(pl) ((MDBX_PNL_GETSIZE(pl) + 1) * sizeof(pgno_t)) +#define MDBX_PNL_IS_EMPTY(pl) (MDBX_PNL_GETSIZE(pl) == 0) + +MDBX_MAYBE_UNUSED static inline size_t pnl_size2bytes(size_t size) { + assert(size > 0 && size <= PAGELIST_LIMIT); +#if MDBX_PNL_PREALLOC_FOR_RADIXSORT + + size += size; +#endif /* MDBX_PNL_PREALLOC_FOR_RADIXSORT */ + STATIC_ASSERT(MDBX_ASSUME_MALLOC_OVERHEAD + + (PAGELIST_LIMIT * (MDBX_PNL_PREALLOC_FOR_RADIXSORT + 1) + MDBX_PNL_GRANULATE + 3) * sizeof(pgno_t) < + SIZE_MAX / 4 * 3); + size_t bytes = + ceil_powerof2(MDBX_ASSUME_MALLOC_OVERHEAD + sizeof(pgno_t) * (size + 3), MDBX_PNL_GRANULATE * sizeof(pgno_t)) - + MDBX_ASSUME_MALLOC_OVERHEAD; + return bytes; +} + +MDBX_MAYBE_UNUSED static inline pgno_t pnl_bytes2size(const size_t bytes) { + size_t size = bytes / sizeof(pgno_t); + assert(size > 3 && size <= PAGELIST_LIMIT + /* alignment gap */ 65536); + size -= 3; +#if MDBX_PNL_PREALLOC_FOR_RADIXSORT + size >>= 1; +#endif /* MDBX_PNL_PREALLOC_FOR_RADIXSORT */ + return (pgno_t)size; +} + +MDBX_INTERNAL pnl_t pnl_alloc(size_t size); + +MDBX_INTERNAL void pnl_free(pnl_t pnl); + +MDBX_INTERNAL int pnl_reserve(pnl_t __restrict *__restrict ppnl, const size_t wanna); + +MDBX_MAYBE_UNUSED static inline int __must_check_result pnl_need(pnl_t __restrict *__restrict ppnl, size_t num) { + assert(MDBX_PNL_GETSIZE(*ppnl) <= PAGELIST_LIMIT && MDBX_PNL_ALLOCLEN(*ppnl) >= MDBX_PNL_GETSIZE(*ppnl)); + assert(num <= PAGELIST_LIMIT); + const size_t wanna = MDBX_PNL_GETSIZE(*ppnl) + num; + return likely(MDBX_PNL_ALLOCLEN(*ppnl) >= wanna) ? MDBX_SUCCESS : pnl_reserve(ppnl, wanna); } -/* Only a subset of the mdbx_env flags can be changed - * at runtime. Changing other flags requires closing the - * environment and re-opening it with the new flags. */ -#define ENV_CHANGEABLE_FLAGS \ - (MDBX_SAFE_NOSYNC | MDBX_NOMETASYNC | MDBX_DEPRECATED_MAPASYNC | \ - MDBX_NOMEMINIT | MDBX_COALESCE | MDBX_PAGEPERTURB | MDBX_ACCEDE | \ - MDBX_VALIDATION) -#define ENV_CHANGELESS_FLAGS \ - (MDBX_NOSUBDIR | MDBX_RDONLY | MDBX_WRITEMAP | MDBX_NOTLS | MDBX_NORDAHEAD | \ - MDBX_LIFORECLAIM | MDBX_EXCLUSIVE) -#define ENV_USABLE_FLAGS (ENV_CHANGEABLE_FLAGS | ENV_CHANGELESS_FLAGS) - -#if !defined(__cplusplus) || CONSTEXPR_ENUM_FLAGS_OPERATIONS -MDBX_MAYBE_UNUSED static void static_checks(void) { - STATIC_ASSERT_MSG(INT16_MAX - CORE_DBS == MDBX_MAX_DBI, - "Oops, MDBX_MAX_DBI or CORE_DBS?"); - STATIC_ASSERT_MSG((unsigned)(MDBX_DB_ACCEDE | MDBX_CREATE) == - ((DB_USABLE_FLAGS | DB_INTERNAL_FLAGS) & - (ENV_USABLE_FLAGS | ENV_INTERNAL_FLAGS)), - "Oops, some flags overlapped or wrong"); - STATIC_ASSERT_MSG((ENV_INTERNAL_FLAGS & ENV_USABLE_FLAGS) == 0, - "Oops, some flags overlapped or wrong"); +MDBX_MAYBE_UNUSED static inline void pnl_append_prereserved(__restrict pnl_t pnl, pgno_t pgno) { + assert(MDBX_PNL_GETSIZE(pnl) < MDBX_PNL_ALLOCLEN(pnl)); + if (AUDIT_ENABLED()) { + for (size_t i = MDBX_PNL_GETSIZE(pnl); i > 0; --i) + assert(pgno != pnl[i]); + } + *pnl += 1; + MDBX_PNL_LAST(pnl) = pgno; +} + +MDBX_INTERNAL void pnl_shrink(pnl_t __restrict *__restrict ppnl); + +MDBX_INTERNAL int __must_check_result spill_append_span(__restrict pnl_t *ppnl, pgno_t pgno, size_t n); + +MDBX_INTERNAL int __must_check_result pnl_append_span(__restrict pnl_t *ppnl, pgno_t pgno, size_t n); + +MDBX_INTERNAL int __must_check_result pnl_insert_span(__restrict pnl_t *ppnl, pgno_t pgno, size_t n); + +MDBX_INTERNAL size_t pnl_search_nochk(const pnl_t pnl, pgno_t pgno); + +MDBX_INTERNAL void pnl_sort_nochk(pnl_t pnl); + +MDBX_INTERNAL bool pnl_check(const const_pnl_t pnl, const size_t limit); + +MDBX_MAYBE_UNUSED static inline bool pnl_check_allocated(const const_pnl_t pnl, const size_t limit) { + return pnl == nullptr || (MDBX_PNL_ALLOCLEN(pnl) >= MDBX_PNL_GETSIZE(pnl) && pnl_check(pnl, limit)); +} + +MDBX_MAYBE_UNUSED static inline void pnl_sort(pnl_t pnl, size_t limit4check) { + pnl_sort_nochk(pnl); + assert(pnl_check(pnl, limit4check)); + (void)limit4check; +} + +MDBX_MAYBE_UNUSED static inline size_t pnl_search(const pnl_t pnl, pgno_t pgno, size_t limit) { + assert(pnl_check_allocated(pnl, limit)); + if (MDBX_HAVE_CMOV) { + /* cmov-ускоренный бинарный поиск может читать (но не использовать) один + * элемент за концом данных, этот элемент в пределах выделенного участка + * памяти, но не инициализирован. */ + VALGRIND_MAKE_MEM_DEFINED(MDBX_PNL_END(pnl), sizeof(pgno_t)); + } + assert(pgno < limit); + (void)limit; + size_t n = pnl_search_nochk(pnl, pgno); + if (MDBX_HAVE_CMOV) { + VALGRIND_MAKE_MEM_UNDEFINED(MDBX_PNL_END(pnl), sizeof(pgno_t)); + } + return n; } -#endif /* Disabled for MSVC 19.0 (VisualStudio 2015) */ + +MDBX_INTERNAL size_t pnl_merge(pnl_t dst, const pnl_t src); #ifdef __cplusplus } +#endif /* __cplusplus */ + +#define mdbx_sourcery_anchor XCONCAT(mdbx_sourcery_, MDBX_BUILD_SOURCERY) +#if defined(xMDBX_TOOLS) +extern LIBMDBX_API const char *const mdbx_sourcery_anchor; #endif -#define MDBX_ASAN_POISON_MEMORY_REGION(addr, size) \ - do { \ - TRACE("POISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), \ - (size_t)(size), __LINE__); \ - ASAN_POISON_MEMORY_REGION(addr, size); \ - } while (0) +#define MDBX_IS_ERROR(rc) ((rc) != MDBX_RESULT_TRUE && (rc) != MDBX_RESULT_FALSE) -#define MDBX_ASAN_UNPOISON_MEMORY_REGION(addr, size) \ - do { \ - TRACE("UNPOISON_MEMORY_REGION(%p, %zu) at %u", (void *)(addr), \ - (size_t)(size), __LINE__); \ - ASAN_UNPOISON_MEMORY_REGION(addr, size); \ - } while (0) +/*----------------------------------------------------------------------------*/ + +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline pgno_t int64pgno(int64_t i64) { + if (likely(i64 >= (int64_t)MIN_PAGENO && i64 <= (int64_t)MAX_PAGENO + 1)) + return (pgno_t)i64; + return (i64 < (int64_t)MIN_PAGENO) ? MIN_PAGENO : MAX_PAGENO; +} + +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline pgno_t pgno_add(size_t base, size_t augend) { + assert(base <= MAX_PAGENO + 1 && augend < MAX_PAGENO); + return int64pgno((int64_t)base + (int64_t)augend); +} + +MDBX_NOTHROW_CONST_FUNCTION MDBX_MAYBE_UNUSED static inline pgno_t pgno_sub(size_t base, size_t subtrahend) { + assert(base >= MIN_PAGENO && base <= MAX_PAGENO + 1 && subtrahend < MAX_PAGENO); + return int64pgno((int64_t)base - (int64_t)subtrahend); +} #if defined(_WIN32) || defined(_WIN64) /* @@ -4085,12 +3242,12 @@ MDBX_MAYBE_UNUSED static void static_checks(void) { #ifdef _MSC_VER #pragma warning(push, 1) -#pragma warning(disable : 4548) /* expression before comma has no effect; \ +#pragma warning(disable : 4548) /* expression before comma has no effect; \ expected expression with side - effect */ -#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ +#pragma warning(disable : 4530) /* C++ exception handler used, but unwind \ * semantics are not enabled. Specify /EHsc */ -#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ - * mode specified; termination on exception is \ +#pragma warning(disable : 4577) /* 'noexcept' used with no exception handling \ + * mode specified; termination on exception is \ * not guaranteed. Specify /EHsc */ #if !defined(_CRT_SECURE_NO_WARNINGS) #define _CRT_SECURE_NO_WARNINGS @@ -4143,8 +3300,7 @@ int getopt(int argc, char *const argv[], const char *opts) { if (argv[optind][sp + 1] != '\0') optarg = &argv[optind++][sp + 1]; else if (++optind >= argc) { - fprintf(stderr, "%s: %s -- %c\n", argv[0], "option requires an argument", - c); + fprintf(stderr, "%s: %s -- %c\n", argv[0], "option requires an argument", c); sp = 1; return '?'; } else @@ -4188,35 +3344,38 @@ static void print_stat(MDBX_stat *ms) { static void usage(const char *prog) { fprintf(stderr, - "usage: %s [-V] [-q] [-e] [-f[f[f]]] [-r[r]] [-a|-s name] dbpath\n" + "usage: %s [-V] [-q] [-e] [-f[f[f]]] [-r[r]] [-a|-s table] dbpath\n" " -V\t\tprint version and exit\n" " -q\t\tbe quiet\n" " -p\t\tshow statistics of page operations for current session\n" " -e\t\tshow whole DB info\n" " -f\t\tshow GC info\n" " -r\t\tshow readers\n" - " -a\t\tprint stat of main DB and all subDBs\n" - " -s name\tprint stat of only the specified named subDB\n" + " -a\t\tprint stat of main DB and all tables\n" + " -s table\tprint stat of only the specified named table\n" " \t\tby default print stat of only the main DB\n", prog); exit(EXIT_FAILURE); } -static int reader_list_func(void *ctx, int num, int slot, mdbx_pid_t pid, - mdbx_tid_t thread, uint64_t txnid, uint64_t lag, - size_t bytes_used, size_t bytes_retained) { +static int reader_list_func(void *ctx, int num, int slot, mdbx_pid_t pid, mdbx_tid_t thread, uint64_t txnid, + uint64_t lag, size_t bytes_used, size_t bytes_retained) { (void)ctx; if (num == 1) printf("Reader Table\n" " #\tslot\t%6s %*s %20s %10s %13s %13s\n", - "pid", (int)sizeof(size_t) * 2, "thread", "txnid", "lag", "used", - "retained"); + "pid", (int)sizeof(size_t) * 2, "thread", "txnid", "lag", "used", "retained"); + + if (thread < (mdbx_tid_t)((intptr_t)MDBX_TID_TXN_OUSTED)) + printf(" %3d)\t[%d]\t%6" PRIdSIZE " %*" PRIxPTR, num, slot, (size_t)pid, (int)sizeof(size_t) * 2, + (uintptr_t)thread); + else + printf(" %3d)\t[%d]\t%6" PRIdSIZE " %sed", num, slot, (size_t)pid, + (thread == (mdbx_tid_t)((uintptr_t)MDBX_TID_TXN_PARKED)) ? "park" : "oust"); - printf(" %3d)\t[%d]\t%6" PRIdSIZE " %*" PRIxPTR, num, slot, (size_t)pid, - (int)sizeof(size_t) * 2, (uintptr_t)thread); if (txnid) - printf(" %20" PRIu64 " %10" PRIu64 " %12.1fM %12.1fM\n", txnid, lag, - bytes_used / 1048576.0, bytes_retained / 1048576.0); + printf(" %20" PRIu64 " %10" PRIu64 " %12.1fM %12.1fM\n", txnid, lag, bytes_used / 1048576.0, + bytes_retained / 1048576.0); else printf(" %20s %10s %13s %13s\n", "-", "0", "0", "0"); @@ -4227,8 +3386,22 @@ const char *prog; bool quiet = false; static void error(const char *func, int rc) { if (!quiet) - fprintf(stderr, "%s: %s() error %d %s\n", prog, func, rc, - mdbx_strerror(rc)); + fprintf(stderr, "%s: %s() error %d %s\n", prog, func, rc, mdbx_strerror(rc)); +} + +static void logger(MDBX_log_level_t level, const char *function, int line, const char *fmt, va_list args) { + static const char *const prefixes[] = { + "!!!fatal: ", // 0 fatal + " ! ", // 1 error + " ~ ", // 2 warning + " ", // 3 notice + " //", // 4 verbose + }; + if (level < MDBX_LOG_DEBUG) { + if (function && line) + fprintf(stderr, "%s", prefixes[level]); + vfprintf(stderr, fmt, args); + } } int main(int argc, char *argv[]) { @@ -4239,7 +3412,7 @@ int main(int argc, char *argv[]) { MDBX_envinfo mei; prog = argv[0]; char *envname; - char *subname = nullptr; + char *table = nullptr; bool alldbs = false, envinfo = false, pgop = false; int freinfo = 0, rdrinfo = 0; @@ -4264,12 +3437,9 @@ int main(int argc, char *argv[]) { " - build: %s for %s by %s\n" " - flags: %s\n" " - options: %s\n", - mdbx_version.major, mdbx_version.minor, mdbx_version.release, - mdbx_version.revision, mdbx_version.git.describe, - mdbx_version.git.datetime, mdbx_version.git.commit, - mdbx_version.git.tree, mdbx_sourcery_anchor, mdbx_build.datetime, - mdbx_build.target, mdbx_build.compiler, mdbx_build.flags, - mdbx_build.options); + mdbx_version.major, mdbx_version.minor, mdbx_version.patch, mdbx_version.tweak, mdbx_version.git.describe, + mdbx_version.git.datetime, mdbx_version.git.commit, mdbx_version.git.tree, mdbx_sourcery_anchor, + mdbx_build.datetime, mdbx_build.target, mdbx_build.compiler, mdbx_build.flags, mdbx_build.options); return EXIT_SUCCESS; case 'q': quiet = true; @@ -4278,7 +3448,7 @@ int main(int argc, char *argv[]) { pgop = true; break; case 'a': - if (subname) + if (table) usage(prog); alldbs = true; break; @@ -4296,7 +3466,7 @@ int main(int argc, char *argv[]) { case 's': if (alldbs) usage(prog); - subname = optarg; + table = optarg; break; default: usage(prog); @@ -4322,10 +3492,10 @@ int main(int argc, char *argv[]) { envname = argv[optind]; envname = argv[optind]; if (!quiet) { - printf("mdbx_stat %s (%s, T-%s)\nRunning for %s...\n", - mdbx_version.git.describe, mdbx_version.git.datetime, + printf("mdbx_stat %s (%s, T-%s)\nRunning for %s...\n", mdbx_version.git.describe, mdbx_version.git.datetime, mdbx_version.git.tree, envname); fflush(nullptr); + mdbx_setup_debug(MDBX_LOG_NOTICE, MDBX_DBG_DONTCHANGE, logger); } rc = mdbx_env_create(&env); @@ -4334,7 +3504,7 @@ int main(int argc, char *argv[]) { return EXIT_FAILURE; } - if (alldbs || subname) { + if (alldbs || table) { rc = mdbx_env_set_maxdbs(env, 2); if (unlikely(rc != MDBX_SUCCESS)) { error("mdbx_env_set_maxdbs", rc); @@ -4367,39 +3537,27 @@ int main(int argc, char *argv[]) { if (pgop) { printf("Page Operations (for current session):\n"); - printf(" New: %8" PRIu64 "\t// quantity of a new pages added\n", - mei.mi_pgop_stat.newly); - printf(" CoW: %8" PRIu64 - "\t// quantity of pages copied for altering\n", - mei.mi_pgop_stat.cow); + printf(" New: %8" PRIu64 "\t// quantity of a new pages added\n", mei.mi_pgop_stat.newly); + printf(" CoW: %8" PRIu64 "\t// quantity of pages copied for altering\n", mei.mi_pgop_stat.cow); printf(" Clone: %8" PRIu64 "\t// quantity of parent's dirty pages " "clones for nested transactions\n", mei.mi_pgop_stat.clone); - printf(" Split: %8" PRIu64 - "\t// page splits during insertions or updates\n", - mei.mi_pgop_stat.split); - printf(" Merge: %8" PRIu64 - "\t// page merges during deletions or updates\n", - mei.mi_pgop_stat.merge); + printf(" Split: %8" PRIu64 "\t// page splits during insertions or updates\n", mei.mi_pgop_stat.split); + printf(" Merge: %8" PRIu64 "\t// page merges during deletions or updates\n", mei.mi_pgop_stat.merge); printf(" Spill: %8" PRIu64 "\t// quantity of spilled/ousted `dirty` " "pages during large transactions\n", mei.mi_pgop_stat.spill); printf(" Unspill: %8" PRIu64 "\t// quantity of unspilled/redone `dirty` " "pages during large transactions\n", mei.mi_pgop_stat.unspill); - printf(" WOP: %8" PRIu64 - "\t// number of explicit write operations (not a pages) to a disk\n", + printf(" WOP: %8" PRIu64 "\t// number of explicit write operations (not a pages) to a disk\n", mei.mi_pgop_stat.wops); - printf(" PreFault: %8" PRIu64 - "\t// number of prefault write operations (not a pages)\n", + printf(" PreFault: %8" PRIu64 "\t// number of prefault write operations (not a pages)\n", mei.mi_pgop_stat.prefault); - printf(" mInCore: %8" PRIu64 "\t// number of mincore() calls\n", - mei.mi_pgop_stat.mincore); - printf(" mSync: %8" PRIu64 - "\t// number of explicit msync-to-disk operations (not a pages)\n", + printf(" mInCore: %8" PRIu64 "\t// number of mincore() calls\n", mei.mi_pgop_stat.mincore); + printf(" mSync: %8" PRIu64 "\t// number of explicit msync-to-disk operations (not a pages)\n", mei.mi_pgop_stat.msync); - printf(" fSync: %8" PRIu64 - "\t// number of explicit fsync-to-disk operations (not a pages)\n", + printf(" fSync: %8" PRIu64 "\t// number of explicit fsync-to-disk operations (not a pages)\n", mei.mi_pgop_stat.fsync); } @@ -4407,18 +3565,15 @@ int main(int argc, char *argv[]) { printf("Environment Info\n"); printf(" Pagesize: %u\n", mei.mi_dxb_pagesize); if (mei.mi_geo.lower != mei.mi_geo.upper) { - printf(" Dynamic datafile: %" PRIu64 "..%" PRIu64 " bytes (+%" PRIu64 - "/-%" PRIu64 "), %" PRIu64 "..%" PRIu64 " pages (+%" PRIu64 - "/-%" PRIu64 ")\n", - mei.mi_geo.lower, mei.mi_geo.upper, mei.mi_geo.grow, - mei.mi_geo.shrink, mei.mi_geo.lower / mei.mi_dxb_pagesize, - mei.mi_geo.upper / mei.mi_dxb_pagesize, - mei.mi_geo.grow / mei.mi_dxb_pagesize, - mei.mi_geo.shrink / mei.mi_dxb_pagesize); - printf(" Current mapsize: %" PRIu64 " bytes, %" PRIu64 " pages \n", - mei.mi_mapsize, mei.mi_mapsize / mei.mi_dxb_pagesize); - printf(" Current datafile: %" PRIu64 " bytes, %" PRIu64 " pages\n", - mei.mi_geo.current, mei.mi_geo.current / mei.mi_dxb_pagesize); + printf(" Dynamic datafile: %" PRIu64 "..%" PRIu64 " bytes (+%" PRIu64 "/-%" PRIu64 "), %" PRIu64 "..%" PRIu64 + " pages (+%" PRIu64 "/-%" PRIu64 ")\n", + mei.mi_geo.lower, mei.mi_geo.upper, mei.mi_geo.grow, mei.mi_geo.shrink, + mei.mi_geo.lower / mei.mi_dxb_pagesize, mei.mi_geo.upper / mei.mi_dxb_pagesize, + mei.mi_geo.grow / mei.mi_dxb_pagesize, mei.mi_geo.shrink / mei.mi_dxb_pagesize); + printf(" Current mapsize: %" PRIu64 " bytes, %" PRIu64 " pages \n", mei.mi_mapsize, + mei.mi_mapsize / mei.mi_dxb_pagesize); + printf(" Current datafile: %" PRIu64 " bytes, %" PRIu64 " pages\n", mei.mi_geo.current, + mei.mi_geo.current / mei.mi_dxb_pagesize); #if defined(_WIN32) || defined(_WIN64) if (mei.mi_geo.shrink && mei.mi_geo.current != mei.mi_geo.upper) printf(" WARNING: Due Windows system limitations a " @@ -4428,12 +3583,11 @@ int main(int argc, char *argv[]) { "until it will be closed or reopened in read-write mode.\n"); #endif } else { - printf(" Fixed datafile: %" PRIu64 " bytes, %" PRIu64 " pages\n", - mei.mi_geo.current, mei.mi_geo.current / mei.mi_dxb_pagesize); + printf(" Fixed datafile: %" PRIu64 " bytes, %" PRIu64 " pages\n", mei.mi_geo.current, + mei.mi_geo.current / mei.mi_dxb_pagesize); } printf(" Last transaction ID: %" PRIu64 "\n", mei.mi_recent_txnid); - printf(" Latter reader transaction ID: %" PRIu64 " (%" PRIi64 ")\n", - mei.mi_latter_reader_txnid, + printf(" Latter reader transaction ID: %" PRIu64 " (%" PRIi64 ")\n", mei.mi_latter_reader_txnid, mei.mi_latter_reader_txnid - mei.mi_recent_txnid); printf(" Max readers: %u\n", mei.mi_maxreaders); printf(" Number of reader slots uses: %u\n", mei.mi_numreaders); @@ -4446,7 +3600,7 @@ int main(int argc, char *argv[]) { goto txn_abort; } if (rc == MDBX_RESULT_TRUE) - printf("Reader Table is empty\n"); + printf("Reader Table is absent\n"); else if (rc == MDBX_SUCCESS && rdrinfo > 1) { int dead; rc = mdbx_reader_check(env, &dead); @@ -4462,7 +3616,7 @@ int main(int argc, char *argv[]) { } else printf(" No stale readers.\n"); } - if (!(subname || alldbs || freinfo)) + if (!(table || alldbs || freinfo)) goto txn_abort; } @@ -4487,8 +3641,7 @@ int main(int argc, char *argv[]) { pgno_t pages = 0, *iptr; pgno_t reclaimable = 0; MDBX_val key, data; - while (MDBX_SUCCESS == - (rc = mdbx_cursor_get(cursor, &key, &data, MDBX_NEXT))) { + while (MDBX_SUCCESS == (rc = mdbx_cursor_get(cursor, &key, &data, MDBX_NEXT))) { if (user_break) { rc = MDBX_EINTR; break; @@ -4502,29 +3655,23 @@ int main(int argc, char *argv[]) { if (freinfo > 1) { char *bad = ""; - pgno_t prev = - MDBX_PNL_ASCENDING ? NUM_METAS - 1 : (pgno_t)mei.mi_last_pgno + 1; + pgno_t prev = MDBX_PNL_ASCENDING ? NUM_METAS - 1 : (pgno_t)mei.mi_last_pgno + 1; pgno_t span = 1; for (unsigned i = 0; i < number; ++i) { pgno_t pg = iptr[i]; if (MDBX_PNL_DISORDERED(prev, pg)) bad = " [bad sequence]"; prev = pg; - while (i + span < number && - iptr[i + span] == (MDBX_PNL_ASCENDING ? pgno_add(pg, span) - : pgno_sub(pg, span))) + while (i + span < number && iptr[i + span] == (MDBX_PNL_ASCENDING ? pgno_add(pg, span) : pgno_sub(pg, span))) ++span; } - printf(" Transaction %" PRIaTXN ", %" PRIaPGNO - " pages, maxspan %" PRIaPGNO "%s\n", - *(txnid_t *)key.iov_base, number, span, bad); + printf(" Transaction %" PRIaTXN ", %" PRIaPGNO " pages, maxspan %" PRIaPGNO "%s\n", *(txnid_t *)key.iov_base, + number, span, bad); if (freinfo > 2) { for (unsigned i = 0; i < number; i += span) { const pgno_t pg = iptr[i]; for (span = 1; - i + span < number && - iptr[i + span] == (MDBX_PNL_ASCENDING ? pgno_add(pg, span) - : pgno_sub(pg, span)); + i + span < number && iptr[i + span] == (MDBX_PNL_ASCENDING ? pgno_add(pg, span) : pgno_sub(pg, span)); ++span) ; if (span > 1) @@ -4578,14 +3725,13 @@ int main(int argc, char *argv[]) { value = reclaimable; printf(" Reclaimable: %" PRIu64 " %.1f%%\n", value, value / percent); - value = mei.mi_mapsize / mei.mi_dxb_pagesize - (mei.mi_last_pgno + 1) + - reclaimable; + value = mei.mi_mapsize / mei.mi_dxb_pagesize - (mei.mi_last_pgno + 1) + reclaimable; printf(" Available: %" PRIu64 " %.1f%%\n", value, value / percent); } else printf(" GC: %" PRIaPGNO " pages\n", pages); } - rc = mdbx_dbi_open(txn, subname, MDBX_DB_ACCEDE, &dbi); + rc = mdbx_dbi_open(txn, table, MDBX_DB_ACCEDE, &dbi); if (unlikely(rc != MDBX_SUCCESS)) { error("mdbx_dbi_open", rc); goto txn_abort; @@ -4597,7 +3743,7 @@ int main(int argc, char *argv[]) { error("mdbx_dbi_stat", rc); goto txn_abort; } - printf("Status of %s\n", subname ? subname : "Main DB"); + printf("Status of %s\n", table ? table : "Main DB"); print_stat(&mst); if (alldbs) { @@ -4609,18 +3755,17 @@ int main(int argc, char *argv[]) { } MDBX_val key; - while (MDBX_SUCCESS == - (rc = mdbx_cursor_get(cursor, &key, nullptr, MDBX_NEXT_NODUP))) { - MDBX_dbi subdbi; + while (MDBX_SUCCESS == (rc = mdbx_cursor_get(cursor, &key, nullptr, MDBX_NEXT_NODUP))) { + MDBX_dbi xdbi; if (memchr(key.iov_base, '\0', key.iov_len)) continue; - subname = osal_malloc(key.iov_len + 1); - memcpy(subname, key.iov_base, key.iov_len); - subname[key.iov_len] = '\0'; - rc = mdbx_dbi_open(txn, subname, MDBX_DB_ACCEDE, &subdbi); + table = osal_malloc(key.iov_len + 1); + memcpy(table, key.iov_base, key.iov_len); + table[key.iov_len] = '\0'; + rc = mdbx_dbi_open(txn, table, MDBX_DB_ACCEDE, &xdbi); if (rc == MDBX_SUCCESS) - printf("Status of %s\n", subname); - osal_free(subname); + printf("Status of %s\n", table); + osal_free(table); if (unlikely(rc != MDBX_SUCCESS)) { if (rc == MDBX_INCOMPATIBLE) continue; @@ -4628,14 +3773,14 @@ int main(int argc, char *argv[]) { goto txn_abort; } - rc = mdbx_dbi_stat(txn, subdbi, &mst, sizeof(mst)); + rc = mdbx_dbi_stat(txn, xdbi, &mst, sizeof(mst)); if (unlikely(rc != MDBX_SUCCESS)) { error("mdbx_dbi_stat", rc); goto txn_abort; } print_stat(&mst); - rc = mdbx_dbi_close(env, subdbi); + rc = mdbx_dbi_close(env, xdbi); if (unlikely(rc != MDBX_SUCCESS)) { error("mdbx_dbi_close", rc); goto txn_abort; diff --git a/crates/storage/libmdbx-rs/mdbx-sys/src/lib.rs b/crates/storage/libmdbx-rs/mdbx-sys/src/lib.rs index 6f86951ca5e..4e166732e74 100644 --- a/crates/storage/libmdbx-rs/mdbx-sys/src/lib.rs +++ b/crates/storage/libmdbx-rs/mdbx-sys/src/lib.rs @@ -7,6 +7,6 @@ )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] #![allow(non_upper_case_globals, non_camel_case_types, non_snake_case, clippy::all)] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] include!(concat!(env!("OUT_DIR"), "/bindings.rs")); diff --git a/crates/storage/libmdbx-rs/src/codec.rs b/crates/storage/libmdbx-rs/src/codec.rs index c78f79db9f9..c0b2f0f1cf7 100644 --- a/crates/storage/libmdbx-rs/src/codec.rs +++ b/crates/storage/libmdbx-rs/src/codec.rs @@ -17,7 +17,7 @@ pub trait TableObject: Sized { _: *const ffi::MDBX_txn, data_val: ffi::MDBX_val, ) -> Result { - let s = slice::from_raw_parts(data_val.iov_base as *const u8, data_val.iov_len); + let s = unsafe { slice::from_raw_parts(data_val.iov_base as *const u8, data_val.iov_len) }; Self::decode(s) } } @@ -32,7 +32,7 @@ impl TableObject for Cow<'_, [u8]> { _txn: *const ffi::MDBX_txn, data_val: ffi::MDBX_val, ) -> Result { - let s = slice::from_raw_parts(data_val.iov_base as *const u8, data_val.iov_len); + let s = unsafe { slice::from_raw_parts(data_val.iov_base as *const u8, data_val.iov_len) }; #[cfg(feature = "return-borrowed")] { diff --git a/crates/storage/libmdbx-rs/src/environment.rs b/crates/storage/libmdbx-rs/src/environment.rs index fac3cd5084c..648526a7fc5 100644 --- a/crates/storage/libmdbx-rs/src/environment.rs +++ b/crates/storage/libmdbx-rs/src/environment.rs @@ -258,7 +258,7 @@ unsafe impl Sync for EnvironmentInner {} /// Determines how data is mapped into memory /// -/// It only takes affect when the environment is opened. +/// It only takes effect when the environment is opened. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub enum EnvironmentKind { /// Open the environment in default mode, without WRITEMAP. @@ -511,7 +511,7 @@ impl Default for Geometry { /// Read transactions prevent reuse of pages freed by newer write transactions, thus the database /// can grow quickly. This callback will be called when there is not enough space in the database /// (i.e. before increasing the database size or before `MDBX_MAP_FULL` error) and thus can be -/// used to resolve issues with a "long-lived" read transacttions. +/// used to resolve issues with a "long-lived" read transactions. /// /// Depending on the arguments and needs, your implementation may wait, /// terminate a process or thread that is performing a long read, or perform @@ -778,7 +778,7 @@ impl EnvironmentBuilder { /// Sets the maximum number of threads or reader slots for the environment. /// /// This defines the number of slots in the lock table that is used to track readers in the - /// the environment. The default is 126. Starting a read-only transaction normally ties a lock + /// environment. The default is 126. Starting a read-only transaction normally ties a lock /// table slot to the [Transaction] object until it or the [Environment] object is destroyed. pub const fn set_max_readers(&mut self, max_readers: u64) -> &mut Self { self.max_readers = Some(max_readers); diff --git a/crates/storage/libmdbx-rs/src/flags.rs b/crates/storage/libmdbx-rs/src/flags.rs index 1457195be78..6aefab57b19 100644 --- a/crates/storage/libmdbx-rs/src/flags.rs +++ b/crates/storage/libmdbx-rs/src/flags.rs @@ -1,12 +1,15 @@ +use std::str::FromStr; + use bitflags::bitflags; use ffi::*; /// MDBX sync mode -#[derive(Clone, Copy, Debug)] +#[derive(PartialEq, Eq, Clone, Copy, Debug, Default)] pub enum SyncMode { /// Default robust and durable sync mode. /// Metadata is written and flushed to disk after a data is written and flushed, which /// guarantees the integrity of the database in the event of a crash at any time. + #[default] Durable, /// Don't sync the meta-page after commit. @@ -100,12 +103,6 @@ pub enum SyncMode { UtterlyNoSync, } -impl Default for SyncMode { - fn default() -> Self { - Self::Durable - } -} - #[derive(Clone, Copy, Debug)] pub enum Mode { ReadOnly, @@ -124,6 +121,21 @@ impl From for EnvironmentFlags { } } +impl FromStr for SyncMode { + type Err = String; + + fn from_str(s: &str) -> Result { + let val = s.trim().to_ascii_lowercase(); + match val.as_str() { + "durable" => Ok(Self::Durable), + "safe-no-sync" | "safenosync" | "safe_no_sync" => Ok(Self::SafeNoSync), + _ => Err(format!( + "invalid value '{s}' for sync mode. valid values: durable, safe-no-sync" + )), + } + } +} + #[derive(Clone, Copy, Debug, Default)] pub struct EnvironmentFlags { pub no_sub_dir: bool, diff --git a/crates/storage/libmdbx-rs/src/lib.rs b/crates/storage/libmdbx-rs/src/lib.rs index 4f2cc7c5448..88fe8e3e7cc 100644 --- a/crates/storage/libmdbx-rs/src/lib.rs +++ b/crates/storage/libmdbx-rs/src/lib.rs @@ -6,7 +6,7 @@ )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] #![allow(missing_docs, clippy::needless_pass_by_ref_mut)] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![allow(clippy::borrow_as_ptr)] pub extern crate reth_mdbx_sys as ffi; @@ -43,7 +43,7 @@ mod test_utils { use tempfile::tempdir; /// Regression test for . - /// This test reliably segfaults when run against lmbdb compiled with opt level -O3 and newer + /// This test reliably segfaults when run against lmdb compiled with opt level -O3 and newer /// GCC compilers. #[test] fn issue_21_regression() { diff --git a/crates/storage/libmdbx-rs/src/transaction.rs b/crates/storage/libmdbx-rs/src/transaction.rs index a19e7095660..e47e71ac261 100644 --- a/crates/storage/libmdbx-rs/src/transaction.rs +++ b/crates/storage/libmdbx-rs/src/transaction.rs @@ -7,7 +7,6 @@ use crate::{ Cursor, Error, Stat, TableObject, }; use ffi::{MDBX_txn_flags_t, MDBX_TXN_RDONLY, MDBX_TXN_READWRITE}; -use indexmap::IndexSet; use parking_lot::{Mutex, MutexGuard}; use std::{ ffi::{c_uint, c_void}, @@ -94,7 +93,6 @@ where let inner = TransactionInner { txn, - primed_dbis: Mutex::new(IndexSet::new()), committed: AtomicBool::new(false), env, _marker: Default::default(), @@ -173,50 +171,25 @@ where /// /// Any pending operations will be saved. pub fn commit(self) -> Result<(bool, CommitLatency)> { - self.commit_and_rebind_open_dbs().map(|v| (v.0, v.1)) - } - - pub fn prime_for_permaopen(&self, db: Database) { - self.inner.primed_dbis.lock().insert(db.dbi()); - } + let result = self.txn_execute(|txn| { + if K::IS_READ_ONLY { + #[cfg(feature = "read-tx-timeouts")] + self.env().txn_manager().remove_active_read_transaction(txn); - /// Commits the transaction and returns table handles permanently open until dropped. - pub fn commit_and_rebind_open_dbs(self) -> Result<(bool, CommitLatency, Vec)> { - let result = { - let result = self.txn_execute(|txn| { - if K::IS_READ_ONLY { - #[cfg(feature = "read-tx-timeouts")] - self.env().txn_manager().remove_active_read_transaction(txn); - - let mut latency = CommitLatency::new(); - mdbx_result(unsafe { - ffi::mdbx_txn_commit_ex(txn, latency.mdb_commit_latency()) - }) + let mut latency = CommitLatency::new(); + mdbx_result(unsafe { ffi::mdbx_txn_commit_ex(txn, latency.mdb_commit_latency()) }) .map(|v| (v, latency)) - } else { - let (sender, rx) = sync_channel(0); - self.env() - .txn_manager() - .send_message(TxnManagerMessage::Commit { tx: TxnPtr(txn), sender }); - rx.recv().unwrap() - } - })?; + } else { + let (sender, rx) = sync_channel(0); + self.env() + .txn_manager() + .send_message(TxnManagerMessage::Commit { tx: TxnPtr(txn), sender }); + rx.recv().unwrap() + } + })?; - self.inner.set_committed(); - result - }; - result.map(|(v, latency)| { - ( - v, - latency, - self.inner - .primed_dbis - .lock() - .iter() - .map(|&dbi| Database::new_from_ptr(dbi, self.env().clone())) - .collect(), - ) - }) + self.inner.set_committed(); + result } /// Opens a handle to an MDBX database. @@ -308,8 +281,6 @@ where { /// The transaction pointer itself. txn: TransactionPtr, - /// A set of database handles that are primed for permaopen. - primed_dbis: Mutex>, /// Whether the transaction has committed. committed: AtomicBool, env: Environment, @@ -505,7 +476,7 @@ impl Transaction { /// Caller must close ALL other [Database] and [Cursor] instances pointing to the same dbi /// BEFORE calling this function. pub unsafe fn drop_db(&self, db: Database) -> Result<()> { - mdbx_result(self.txn_execute(|txn| ffi::mdbx_drop(txn, db.dbi(), true))?)?; + mdbx_result(self.txn_execute(|txn| unsafe { ffi::mdbx_drop(txn, db.dbi(), true) })?)?; Ok(()) } @@ -518,7 +489,7 @@ impl Transaction { /// Caller must close ALL other [Database] and [Cursor] instances pointing to the same dbi /// BEFORE calling this function. pub unsafe fn close_db(&self, db: Database) -> Result<()> { - mdbx_result(ffi::mdbx_dbi_close(self.env().env_ptr(), db.dbi()))?; + mdbx_result(unsafe { ffi::mdbx_dbi_close(self.env().env_ptr(), db.dbi()) })?; Ok(()) } diff --git a/crates/storage/libmdbx-rs/tests/cursor.rs b/crates/storage/libmdbx-rs/tests/cursor.rs index 0e02eafd9ab..aba11f480c0 100644 --- a/crates/storage/libmdbx-rs/tests/cursor.rs +++ b/crates/storage/libmdbx-rs/tests/cursor.rs @@ -324,14 +324,21 @@ fn test_put_del() { cursor.put(b"key3", b"val3", WriteFlags::empty()).unwrap(); assert_eq!( - cursor.get_current().unwrap().unwrap(), - (Cow::Borrowed(b"key3" as &[u8]), Cow::Borrowed(b"val3" as &[u8])) + cursor.set_key(b"key2").unwrap(), + Some((Cow::Borrowed(b"key2" as &[u8]), Cow::Borrowed(b"val2" as &[u8]))) + ); + assert_eq!( + cursor.get_current().unwrap(), + Some((Cow::Borrowed(b"key2" as &[u8]), Cow::Borrowed(b"val2" as &[u8]))) ); cursor.del(WriteFlags::empty()).unwrap(); - assert_eq!(cursor.get_current::, Vec>().unwrap(), None); assert_eq!( - cursor.last().unwrap().unwrap(), - (Cow::Borrowed(b"key2" as &[u8]), Cow::Borrowed(b"val2" as &[u8])) + cursor.get_current().unwrap(), + Some((Cow::Borrowed(b"key3" as &[u8]), Cow::Borrowed(b"val3" as &[u8]))) + ); + assert_eq!( + cursor.last().unwrap(), + Some((Cow::Borrowed(b"key3" as &[u8]), Cow::Borrowed(b"val3" as &[u8]))) ); } diff --git a/crates/storage/nippy-jar/src/compression/mod.rs b/crates/storage/nippy-jar/src/compression/mod.rs index f9bf8110eeb..ca98e72fe58 100644 --- a/crates/storage/nippy-jar/src/compression/mod.rs +++ b/crates/storage/nippy-jar/src/compression/mod.rs @@ -14,7 +14,7 @@ pub trait Compression: Serialize + for<'a> Deserialize<'a> { /// Returns decompressed data. fn decompress(&self, value: &[u8]) -> Result, NippyJarError>; - /// Appends compressed data from `src` to `dest`. `dest`. Requires `dest` to have sufficient + /// Appends compressed data from `src` to `dest`. Requires `dest` to have sufficient /// capacity. /// /// Returns number of bytes written to `dest`. diff --git a/crates/storage/nippy-jar/src/lib.rs b/crates/storage/nippy-jar/src/lib.rs index b4e39d709d8..1731dc87d04 100644 --- a/crates/storage/nippy-jar/src/lib.rs +++ b/crates/storage/nippy-jar/src/lib.rs @@ -10,7 +10,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] use memmap2::Mmap; use serde::{Deserialize, Serialize}; @@ -200,6 +200,9 @@ impl NippyJar { // Read [`Self`] located at the data file. let config_path = path.with_extension(CONFIG_FILE_EXTENSION); let config_file = File::open(&config_path) + .inspect_err(|e| { + warn!( ?path, %e, "Failed to load static file jar"); + }) .map_err(|err| reth_fs_util::FsPathError::open(err, config_path))?; let mut obj = Self::load_from_reader(config_file)?; @@ -240,6 +243,7 @@ impl NippyJar { [self.data_path().into(), self.index_path(), self.offsets_path(), self.config_path()] { if path.exists() { + debug!(target: "nippy-jar", ?path, "Removing file."); reth_fs_util::remove_file(path)?; } } @@ -308,10 +312,10 @@ impl NippyJar { return Err(NippyJarError::ColumnLenMismatch(self.columns, columns.len())) } - if let Some(compression) = &self.compressor { - if !compression.is_ready() { - return Err(NippyJarError::CompressorNotReady) - } + if let Some(compression) = &self.compressor && + !compression.is_ready() + { + return Err(NippyJarError::CompressorNotReady) } Ok(()) diff --git a/crates/storage/nippy-jar/src/writer.rs b/crates/storage/nippy-jar/src/writer.rs index d32d9b51408..cf899791eed 100644 --- a/crates/storage/nippy-jar/src/writer.rs +++ b/crates/storage/nippy-jar/src/writer.rs @@ -48,7 +48,7 @@ pub struct NippyJarWriter { impl NippyJarWriter { /// Creates a [`NippyJarWriter`] from [`NippyJar`]. /// - /// If will **always** attempt to heal any inconsistent state when called. + /// If will **always** attempt to heal any inconsistent state when called. pub fn new(jar: NippyJar) -> Result { let (data_file, offsets_file, is_created) = Self::create_or_open_files(jar.data_path(), &jar.offsets_path())?; @@ -404,10 +404,10 @@ impl NippyJarWriter { // Appends new offsets to disk for offset in self.offsets.drain(..) { - if let Some(last_offset_ondisk) = last_offset_ondisk.take() { - if last_offset_ondisk == offset { - continue - } + if let Some(last_offset_ondisk) = last_offset_ondisk.take() && + last_offset_ondisk == offset + { + continue } self.offsets_file.write_all(&offset.to_le_bytes())?; } diff --git a/crates/storage/provider/Cargo.toml b/crates/storage/provider/Cargo.toml index c45fde7729c..e8599a89706 100644 --- a/crates/storage/provider/Cargo.toml +++ b/crates/storage/provider/Cargo.toml @@ -29,7 +29,6 @@ reth-trie = { workspace = true, features = ["metrics"] } reth-trie-db = { workspace = true, features = ["metrics"] } reth-nippy-jar.workspace = true reth-codecs.workspace = true -reth-evm.workspace = true reth-chain-state.workspace = true reth-node-types.workspace = true reth-static-file-types.workspace = true @@ -74,14 +73,12 @@ reth-ethereum-primitives.workspace = true revm-database-interface.workspace = true revm-state.workspace = true -parking_lot.workspace = true + tempfile.workspace = true assert_matches.workspace = true rand.workspace = true -eyre.workspace = true tokio = { workspace = true, features = ["sync", "macros", "rt-multi-thread"] } -alloy-consensus.workspace = true [features] test-utils = [ @@ -92,7 +89,6 @@ test-utils = [ "reth-ethereum-engine-primitives", "reth-ethereum-primitives/test-utils", "reth-chainspec/test-utils", - "reth-evm/test-utils", "reth-primitives-traits/test-utils", "reth-codecs/test-utils", "reth-db-api/test-utils", diff --git a/crates/storage/provider/src/bundle_state/mod.rs b/crates/storage/provider/src/bundle_state/mod.rs deleted file mode 100644 index 58b76f1eacf..00000000000 --- a/crates/storage/provider/src/bundle_state/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -//! Bundle state module. -//! This module contains all the logic related to bundle state. - -mod state_reverts; -pub use state_reverts::StorageRevertsIter; diff --git a/crates/storage/provider/src/changesets_utils/mod.rs b/crates/storage/provider/src/changesets_utils/mod.rs new file mode 100644 index 00000000000..3b65825264b --- /dev/null +++ b/crates/storage/provider/src/changesets_utils/mod.rs @@ -0,0 +1,7 @@ +//! This module contains helpful utilities related to populating changesets tables. + +mod state_reverts; +pub use state_reverts::StorageRevertsIter; + +mod trie; +pub use trie::*; diff --git a/crates/storage/provider/src/bundle_state/state_reverts.rs b/crates/storage/provider/src/changesets_utils/state_reverts.rs similarity index 99% rename from crates/storage/provider/src/bundle_state/state_reverts.rs rename to crates/storage/provider/src/changesets_utils/state_reverts.rs index a44e038d49b..7ffdc153b22 100644 --- a/crates/storage/provider/src/bundle_state/state_reverts.rs +++ b/crates/storage/provider/src/changesets_utils/state_reverts.rs @@ -173,7 +173,7 @@ mod tests { (B256::from_slice(&[8; 32]), U256::from(70)), // Revert takes priority. (B256::from_slice(&[9; 32]), U256::from(80)), // Only revert present. (B256::from_slice(&[10; 32]), U256::from(85)), // Wiped entry. - (B256::from_slice(&[15; 32]), U256::from(90)), // WGreater revert entry + (B256::from_slice(&[15; 32]), U256::from(90)), // Greater revert entry ] ); } diff --git a/crates/storage/provider/src/changesets_utils/trie.rs b/crates/storage/provider/src/changesets_utils/trie.rs new file mode 100644 index 00000000000..f4365aab103 --- /dev/null +++ b/crates/storage/provider/src/changesets_utils/trie.rs @@ -0,0 +1,147 @@ +use itertools::{merge_join_by, EitherOrBoth}; +use reth_db_api::DatabaseError; +use reth_trie::{trie_cursor::TrieCursor, BranchNodeCompact, Nibbles}; +use std::cmp::{Ord, Ordering}; + +/// Combines a sorted iterator of trie node paths and a storage trie cursor into a new +/// iterator which produces the current values of all given paths in the same order. +#[derive(Debug)] +pub struct StorageTrieCurrentValuesIter<'cursor, P, C> { + /// Sorted iterator of node paths which we want the values of. + paths: P, + /// Storage trie cursor. + cursor: &'cursor mut C, + /// Current value at the cursor, allows us to treat the cursor as a peekable iterator. + cursor_current: Option<(Nibbles, BranchNodeCompact)>, +} + +impl<'cursor, P, C> StorageTrieCurrentValuesIter<'cursor, P, C> +where + P: Iterator, + C: TrieCursor, +{ + /// Instantiate a [`StorageTrieCurrentValuesIter`] from a sorted paths iterator and a cursor. + pub fn new(paths: P, cursor: &'cursor mut C) -> Result { + let mut new_self = Self { paths, cursor, cursor_current: None }; + new_self.seek_cursor(Nibbles::default())?; + Ok(new_self) + } + + fn seek_cursor(&mut self, path: Nibbles) -> Result<(), DatabaseError> { + self.cursor_current = self.cursor.seek(path)?; + Ok(()) + } +} + +impl<'cursor, P, C> Iterator for StorageTrieCurrentValuesIter<'cursor, P, C> +where + P: Iterator, + C: TrieCursor, +{ + type Item = Result<(Nibbles, Option), DatabaseError>; + + fn next(&mut self) -> Option { + let Some(curr_path) = self.paths.next() else { + // If there are no more paths then there is no further possible output. + return None + }; + + // If the path is ahead of the cursor then seek the cursor forward to catch up. The cursor + // will seek either to `curr_path` or beyond it. + if self.cursor_current.as_ref().is_some_and(|(cursor_path, _)| curr_path > *cursor_path) && + let Err(err) = self.seek_cursor(curr_path) + { + return Some(Err(err)) + } + + // If there is a path but the cursor is empty then that path has no node. + if self.cursor_current.is_none() { + return Some(Ok((curr_path, None))) + } + + let (cursor_path, cursor_node) = + self.cursor_current.as_mut().expect("already checked for None"); + + // There is both a path and a cursor value, compare their paths. + match curr_path.cmp(cursor_path) { + Ordering::Less => { + // If the path is behind the cursor then there is no value for that + // path, produce None. + Some(Ok((curr_path, None))) + } + Ordering::Equal => { + // If the target path and cursor's path match then there is a value for that path, + // return the value. We don't seek the cursor here, that will be handled on the + // next call to `next` after checking that `paths` isn't None. + let cursor_node = core::mem::take(cursor_node); + Some(Ok((*cursor_path, Some(cursor_node)))) + } + Ordering::Greater => { + panic!("cursor was seeked to {curr_path:?}, but produced a node at a lower path {cursor_path:?}") + } + } + } +} + +/// Returns an iterator which produces the values to be inserted into the `StoragesTrieChangeSets` +/// table for an account whose storage was wiped during a block. It is expected that this is called +/// prior to inserting the block's trie updates. +/// +/// ## Arguments +/// +/// - `curr_values_of_changed` is an iterator over the current values of all trie nodes modified by +/// the block, ordered by path. +/// - `all_nodes` is an iterator over all existing trie nodes for the account, ordered by path. +/// +/// ## Returns +/// +/// An iterator of trie node paths and a `Some(node)` (indicating the node was wiped) or a `None` +/// (indicating the node was modified in the block but didn't previously exist. The iterator's +/// results will be ordered by path. +pub fn storage_trie_wiped_changeset_iter( + curr_values_of_changed: impl Iterator< + Item = Result<(Nibbles, Option), DatabaseError>, + >, + all_nodes: impl Iterator>, +) -> Result< + impl Iterator), DatabaseError>>, + DatabaseError, +> { + let all_nodes = all_nodes.map(|e| e.map(|(nibbles, node)| (nibbles, Some(node)))); + + let merged = merge_join_by(curr_values_of_changed, all_nodes, |a, b| match (a, b) { + (Err(_), _) => Ordering::Less, + (_, Err(_)) => Ordering::Greater, + (Ok(a), Ok(b)) => a.0.cmp(&b.0), + }); + + Ok(merged.map(|either_or| match either_or { + EitherOrBoth::Left(changed) => { + // A path of a changed node (given in `paths`) which was not found in the database (or + // there's an error). The current value of this path must be None, otherwise it would + // have also been returned by the `all_nodes` iter. + debug_assert!( + changed.as_ref().is_err() || changed.as_ref().is_ok_and(|(_, node)| node.is_none()), + "changed node is Some but wasn't returned by `all_nodes` iterator: {changed:?}", + ); + changed + } + EitherOrBoth::Right(wiped) => { + // A node was found in the db (indicating it was wiped) but was not given in `paths`. + // Return it as-is. + wiped + } + EitherOrBoth::Both(changed, _wiped) => { + // A path of a changed node (given in `paths`) was found with a previous value in the + // database. The changed node must have a value which is equal to the one found by the + // `all_nodes` iterator. If the changed node had no previous value (None) it wouldn't + // be returned by `all_nodes` and so would be in the Left branch. + // + // Due to the ordering closure passed to `merge_join_by` it's not possible for either + // value to be an error here. + debug_assert!(changed.is_ok(), "unreachable error condition: {changed:?}"); + debug_assert_eq!(changed, _wiped); + changed + } + })) +} diff --git a/crates/storage/provider/src/lib.rs b/crates/storage/provider/src/lib.rs index 6c7826c82d7..70822c604bb 100644 --- a/crates/storage/provider/src/lib.rs +++ b/crates/storage/provider/src/lib.rs @@ -10,7 +10,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] /// Various provider traits. mod traits; @@ -35,7 +35,7 @@ pub use static_file::StaticFileSegment; pub use reth_execution_types::*; -pub mod bundle_state; +pub mod changesets_utils; /// Re-export `OriginalValuesKnown` pub use revm_database::states::OriginalValuesKnown; diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index ea7a9452ba9..9dbbed9e88c 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -1,50 +1,34 @@ -#![allow(unused)] use crate::{ providers::{ConsistentProvider, ProviderNodeTypes, StaticFileProvider}, AccountReader, BlockHashReader, BlockIdReader, BlockNumReader, BlockReader, BlockReaderIdExt, BlockSource, CanonChainTracker, CanonStateNotifications, CanonStateSubscriptions, - ChainSpecProvider, ChainStateBlockReader, ChangeSetReader, DatabaseProvider, - DatabaseProviderFactory, FullProvider, HashedPostStateProvider, HeaderProvider, ProviderError, - ProviderFactory, PruneCheckpointReader, ReceiptProvider, ReceiptProviderIdExt, - StageCheckpointReader, StateProviderBox, StateProviderFactory, StateReader, - StaticFileProviderFactory, TransactionVariant, TransactionsProvider, WithdrawalsProvider, + ChainSpecProvider, ChainStateBlockReader, ChangeSetReader, DatabaseProviderFactory, + HashedPostStateProvider, HeaderProvider, ProviderError, ProviderFactory, PruneCheckpointReader, + ReceiptProvider, ReceiptProviderIdExt, StageCheckpointReader, StateProviderBox, + StateProviderFactory, StateReader, StaticFileProviderFactory, TransactionVariant, + TransactionsProvider, TrieReader, }; -use alloy_consensus::{transaction::TransactionMeta, Header}; -use alloy_eips::{ - eip4895::{Withdrawal, Withdrawals}, - BlockHashOrNumber, BlockId, BlockNumHash, BlockNumberOrTag, -}; -use alloy_primitives::{Address, BlockHash, BlockNumber, Sealable, TxHash, TxNumber, B256, U256}; +use alloy_consensus::transaction::TransactionMeta; +use alloy_eips::{BlockHashOrNumber, BlockId, BlockNumHash, BlockNumberOrTag}; +use alloy_primitives::{Address, BlockHash, BlockNumber, TxHash, TxNumber, B256}; use alloy_rpc_types_engine::ForkchoiceState; use reth_chain_state::{ BlockState, CanonicalInMemoryState, ForkChoiceNotifications, ForkChoiceSubscriptions, MemoryOverlayStateProvider, }; -use reth_chainspec::{ChainInfo, EthereumHardforks}; -use reth_db_api::{ - models::{AccountBeforeTx, BlockNumberAddress, StoredBlockBodyIndices}, - transaction::DbTx, - Database, -}; -use reth_ethereum_primitives::{Block, EthPrimitives, Receipt, TransactionSigned}; -use reth_evm::{ConfigureEvm, EvmEnv}; +use reth_chainspec::ChainInfo; +use reth_db_api::models::{AccountBeforeTx, BlockNumberAddress, StoredBlockBodyIndices}; use reth_execution_types::ExecutionOutcome; use reth_node_types::{BlockTy, HeaderTy, NodeTypesWithDB, ReceiptTy, TxTy}; -use reth_primitives_traits::{ - Account, BlockBody, NodePrimitives, RecoveredBlock, SealedBlock, SealedHeader, StorageEntry, -}; +use reth_primitives_traits::{Account, RecoveredBlock, SealedHeader, StorageEntry}; use reth_prune_types::{PruneCheckpoint, PruneSegment}; use reth_stages_types::{StageCheckpoint, StageId}; -use reth_storage_api::{ - BlockBodyIndicesProvider, DBProvider, NodePrimitivesProvider, OmmersProvider, - StateCommitmentProvider, StorageChangeSetReader, -}; +use reth_storage_api::{BlockBodyIndicesProvider, NodePrimitivesProvider, StorageChangeSetReader}; use reth_storage_errors::provider::ProviderResult; -use reth_trie::HashedPostState; -use reth_trie_db::StateCommitment; +use reth_trie::{updates::TrieUpdatesSorted, HashedPostState, KeccakKeyHasher}; use revm_database::BundleState; use std::{ - ops::{Add, RangeBounds, RangeInclusive, Sub}, + ops::{RangeBounds, RangeInclusive}, sync::Arc, time::Instant, }; @@ -175,10 +159,6 @@ impl DatabaseProviderFactory for BlockchainProvider { } } -impl StateCommitmentProvider for BlockchainProvider { - type StateCommitment = N::StateCommitment; -} - impl StaticFileProviderFactory for BlockchainProvider { fn static_file_provider(&self) -> StaticFileProvider { self.database.static_file_provider() @@ -188,7 +168,7 @@ impl StaticFileProviderFactory for BlockchainProvider { impl HeaderProvider for BlockchainProvider { type Header = HeaderTy; - fn header(&self, block_hash: &BlockHash) -> ProviderResult> { + fn header(&self, block_hash: BlockHash) -> ProviderResult> { self.consistent_provider()?.header(block_hash) } @@ -196,14 +176,6 @@ impl HeaderProvider for BlockchainProvider { self.consistent_provider()?.header_by_number(num) } - fn header_td(&self, hash: &BlockHash) -> ProviderResult> { - self.consistent_provider()?.header_td(hash) - } - - fn header_td_by_number(&self, number: BlockNumber) -> ProviderResult> { - self.consistent_provider()?.header_td_by_number(number) - } - fn headers_range( &self, range: impl RangeBounds, @@ -261,6 +233,10 @@ impl BlockNumReader for BlockchainProvider { self.database.last_block_number() } + fn earliest_block_number(&self) -> ProviderResult { + self.database.earliest_block_number() + } + fn block_number(&self, hash: B256) -> ProviderResult> { self.consistent_provider()?.block_number(hash) } @@ -295,17 +271,13 @@ impl BlockReader for BlockchainProvider { self.consistent_provider()?.block(id) } - fn pending_block(&self) -> ProviderResult>> { - Ok(self.canonical_in_memory_state.pending_block()) - } - - fn pending_block_with_senders(&self) -> ProviderResult>> { + fn pending_block(&self) -> ProviderResult>> { Ok(self.canonical_in_memory_state.pending_recovered_block()) } fn pending_block_and_receipts( &self, - ) -> ProviderResult, Vec)>> { + ) -> ProviderResult, Vec)>> { Ok(self.canonical_in_memory_state.pending_block_and_receipts()) } @@ -348,6 +320,10 @@ impl BlockReader for BlockchainProvider { ) -> ProviderResult>> { self.consistent_provider()?.recovered_block_range(range) } + + fn block_by_transaction_id(&self, id: TxNumber) -> ProviderResult> { + self.consistent_provider()?.block_by_transaction_id(id) + } } impl TransactionsProvider for BlockchainProvider { @@ -440,27 +416,18 @@ impl ReceiptProvider for BlockchainProvider { ) -> ProviderResult> { self.consistent_provider()?.receipts_by_tx_range(range) } -} - -impl ReceiptProviderIdExt for BlockchainProvider { - fn receipts_by_block_id(&self, block: BlockId) -> ProviderResult>> { - self.consistent_provider()?.receipts_by_block_id(block) - } -} -impl WithdrawalsProvider for BlockchainProvider { - fn withdrawals_by_block( + fn receipts_by_block_range( &self, - id: BlockHashOrNumber, - timestamp: u64, - ) -> ProviderResult> { - self.consistent_provider()?.withdrawals_by_block(id, timestamp) + block_range: RangeInclusive, + ) -> ProviderResult>> { + self.consistent_provider()?.receipts_by_block_range(block_range) } } -impl OmmersProvider for BlockchainProvider { - fn ommers(&self, id: BlockHashOrNumber) -> ProviderResult>> { - self.consistent_provider()?.ommers(id) +impl ReceiptProviderIdExt for BlockchainProvider { + fn receipts_by_block_id(&self, block: BlockId) -> ProviderResult>> { + self.consistent_provider()?.receipts_by_block_id(block) } } @@ -529,6 +496,37 @@ impl StateProviderFactory for BlockchainProvider { } } + /// Returns a [`StateProviderBox`] indexed by the given block number or tag. + fn state_by_block_number_or_tag( + &self, + number_or_tag: BlockNumberOrTag, + ) -> ProviderResult { + match number_or_tag { + BlockNumberOrTag::Latest => self.latest(), + BlockNumberOrTag::Finalized => { + // we can only get the finalized state by hash, not by num + let hash = + self.finalized_block_hash()?.ok_or(ProviderError::FinalizedBlockNotFound)?; + self.state_by_block_hash(hash) + } + BlockNumberOrTag::Safe => { + // we can only get the safe state by hash, not by num + let hash = self.safe_block_hash()?.ok_or(ProviderError::SafeBlockNotFound)?; + self.state_by_block_hash(hash) + } + BlockNumberOrTag::Earliest => { + self.history_by_block_number(self.earliest_block_number()?) + } + BlockNumberOrTag::Pending => self.pending(), + BlockNumberOrTag::Number(num) => { + let hash = self + .block_hash(num)? + .ok_or_else(|| ProviderError::HeaderNotFound(num.into()))?; + self.state_by_block_hash(hash) + } + } + } + fn history_by_block_number( &self, block_number: BlockNumber, @@ -539,20 +537,12 @@ impl StateProviderFactory for BlockchainProvider { let hash = provider .block_hash(block_number)? .ok_or_else(|| ProviderError::HeaderNotFound(block_number.into()))?; - self.history_by_block_hash(hash) + provider.into_state_provider_at_block_hash(hash) } fn history_by_block_hash(&self, block_hash: BlockHash) -> ProviderResult { trace!(target: "providers::blockchain", ?block_hash, "Getting history by block hash"); - - self.consistent_provider()?.get_in_memory_or_storage_by_block( - block_hash.into(), - |_| self.database.history_by_block_hash(block_hash), - |block_state| { - let state_provider = self.block_state_provider(block_state)?; - Ok(Box::new(state_provider)) - }, - ) + self.consistent_provider()?.into_state_provider_at_block_hash(block_hash) } fn state_by_block_hash(&self, hash: BlockHash) -> ProviderResult { @@ -586,49 +576,26 @@ impl StateProviderFactory for BlockchainProvider { } fn pending_state_by_hash(&self, block_hash: B256) -> ProviderResult> { - if let Some(pending) = self.canonical_in_memory_state.pending_state() { - if pending.hash() == block_hash { - return Ok(Some(Box::new(self.block_state_provider(&pending)?))); - } + if let Some(pending) = self.canonical_in_memory_state.pending_state() && + pending.hash() == block_hash + { + return Ok(Some(Box::new(self.block_state_provider(&pending)?))); } Ok(None) } - /// Returns a [`StateProviderBox`] indexed by the given block number or tag. - fn state_by_block_number_or_tag( - &self, - number_or_tag: BlockNumberOrTag, - ) -> ProviderResult { - match number_or_tag { - BlockNumberOrTag::Latest => self.latest(), - BlockNumberOrTag::Finalized => { - // we can only get the finalized state by hash, not by num - let hash = - self.finalized_block_hash()?.ok_or(ProviderError::FinalizedBlockNotFound)?; - self.state_by_block_hash(hash) - } - BlockNumberOrTag::Safe => { - // we can only get the safe state by hash, not by num - let hash = self.safe_block_hash()?.ok_or(ProviderError::SafeBlockNotFound)?; - self.state_by_block_hash(hash) - } - BlockNumberOrTag::Earliest => self.history_by_block_number(0), - BlockNumberOrTag::Pending => self.pending(), - BlockNumberOrTag::Number(num) => { - let hash = self - .block_hash(num)? - .ok_or_else(|| ProviderError::HeaderNotFound(num.into()))?; - self.state_by_block_hash(hash) - } + fn maybe_pending(&self) -> ProviderResult> { + if let Some(pending) = self.canonical_in_memory_state.pending_state() { + return Ok(Some(Box::new(self.block_state_provider(&pending)?))) } + + Ok(None) } } impl HashedPostStateProvider for BlockchainProvider { fn hashed_post_state(&self, bundle_state: &BundleState) -> HashedPostState { - HashedPostState::from_bundle_state::<::KeyHasher>( - bundle_state.state(), - ) + HashedPostState::from_bundle_state::(bundle_state.state()) } } @@ -689,10 +656,6 @@ where fn header_by_id(&self, id: BlockId) -> ProviderResult> { self.consistent_provider()?.header_by_id(id) } - - fn ommers_by_id(&self, id: BlockId) -> ProviderResult>> { - self.consistent_provider()?.ommers_by_id(id) - } } impl CanonStateSubscriptions for BlockchainProvider { @@ -731,6 +694,14 @@ impl ChangeSetReader for BlockchainProvider { ) -> ProviderResult> { self.consistent_provider()?.account_block_changeset(block_number) } + + fn get_account_before_block( + &self, + block_number: BlockNumber, + address: Address, + ) -> ProviderResult> { + self.consistent_provider()?.get_account_before_block(block_number, address) + } } impl AccountReader for BlockchainProvider { @@ -760,6 +731,19 @@ impl StateReader for BlockchainProvider { } } +impl TrieReader for BlockchainProvider { + fn trie_reverts(&self, from: BlockNumber) -> ProviderResult { + self.consistent_provider()?.trie_reverts(from) + } + + fn get_block_trie_updates( + &self, + block_number: BlockNumber, + ) -> ProviderResult { + self.consistent_provider()?.get_block_trie_updates(block_number) + } +} + #[cfg(test)] mod tests { use crate::{ @@ -768,47 +752,36 @@ mod tests { create_test_provider_factory, create_test_provider_factory_with_chain_spec, MockNodeTypesWithDB, }, - writer::UnifiedStorageWriter, - BlockWriter, CanonChainTracker, ProviderFactory, StaticFileProviderFactory, - StaticFileWriter, + BlockWriter, CanonChainTracker, ProviderFactory, }; - use alloy_eips::{eip4895::Withdrawals, BlockHashOrNumber, BlockNumHash, BlockNumberOrTag}; + use alloy_eips::{BlockHashOrNumber, BlockNumHash, BlockNumberOrTag}; use alloy_primitives::{BlockNumber, TxNumber, B256}; use itertools::Itertools; use rand::Rng; use reth_chain_state::{ test_utils::TestBlockBuilder, CanonStateNotification, CanonStateSubscriptions, - CanonicalInMemoryState, ExecutedBlock, ExecutedBlockWithTrieUpdates, NewCanonicalChain, - }; - use reth_chainspec::{ - ChainSpec, ChainSpecBuilder, ChainSpecProvider, EthereumHardfork, MAINNET, - }; - use reth_db_api::{ - cursor::DbCursorRO, - models::{AccountBeforeTx, StoredBlockBodyIndices}, - tables, - transaction::DbTx, + CanonicalInMemoryState, ExecutedBlock, NewCanonicalChain, }; + use reth_chainspec::{ChainSpec, MAINNET}; + use reth_db_api::models::{AccountBeforeTx, StoredBlockBodyIndices}; use reth_errors::ProviderError; - use reth_ethereum_primitives::{Block, EthPrimitives, Receipt}; + use reth_ethereum_primitives::{Block, Receipt}; use reth_execution_types::{Chain, ExecutionOutcome}; - use reth_primitives_traits::{BlockBody, RecoveredBlock, SealedBlock, SignedTransaction}; - use reth_static_file_types::StaticFileSegment; + use reth_primitives_traits::{RecoveredBlock, SealedBlock, SignerRecoverable}; use reth_storage_api::{ BlockBodyIndicesProvider, BlockHashReader, BlockIdReader, BlockNumReader, BlockReader, - BlockReaderIdExt, BlockSource, ChangeSetReader, DatabaseProviderFactory, HeaderProvider, - OmmersProvider, ReceiptProvider, ReceiptProviderIdExt, StateProviderFactory, - TransactionVariant, TransactionsProvider, WithdrawalsProvider, + BlockReaderIdExt, BlockSource, ChangeSetReader, DBProvider, DatabaseProviderFactory, + HeaderProvider, ReceiptProvider, ReceiptProviderIdExt, StateProviderFactory, StateWriter, + TransactionVariant, TransactionsProvider, }; use reth_testing_utils::generators::{ self, random_block, random_block_range, random_changeset_range, random_eoa_accounts, random_receipt, BlockParams, BlockRangeParams, }; - use revm_database::BundleState; + use revm_database::{BundleState, OriginalValuesKnown}; use std::{ - ops::{Bound, Deref, Range, RangeBounds}, + ops::{Bound, Range, RangeBounds}, sync::Arc, - time::Instant, }; const TEST_BLOCKS_COUNT: usize = 5; @@ -874,42 +847,32 @@ mod tests { .iter() .chain(in_memory_blocks.iter()) .map(|block| block.body().transactions.iter()) - .map(|tx| tx.map(|tx| random_receipt(rng, tx, Some(2))).collect()) + .map(|tx| tx.map(|tx| random_receipt(rng, tx, Some(2), None)).collect()) .collect(); let factory = create_test_provider_factory_with_chain_spec(chain_spec); let provider_rw = factory.database_provider_rw()?; - let static_file_provider = factory.static_file_provider(); - - // Write transactions to static files with the right `tx_num`` - let mut tx_num = provider_rw - .block_body_indices(database_blocks.first().as_ref().unwrap().number.saturating_sub(1))? - .map(|indices| indices.next_tx_num()) - .unwrap_or_default(); // Insert blocks into the database - for (block, receipts) in database_blocks.iter().zip(&receipts) { - // TODO: this should be moved inside `insert_historical_block`: - let mut transactions_writer = - static_file_provider.latest_writer(StaticFileSegment::Transactions)?; - let mut receipts_writer = - static_file_provider.latest_writer(StaticFileSegment::Receipts)?; - transactions_writer.increment_block(block.number)?; - receipts_writer.increment_block(block.number)?; - - for (tx, receipt) in block.body().transactions().zip(receipts) { - transactions_writer.append_transaction(tx_num, tx)?; - receipts_writer.append_receipt(tx_num, receipt)?; - tx_num += 1; - } - - provider_rw.insert_historical_block( + for block in &database_blocks { + provider_rw.insert_block( block.clone().try_recover().expect("failed to seal block with senders"), )?; } - // Commit to both storages: database and static files - UnifiedStorageWriter::commit(provider_rw)?; + // Insert receipts into the database + if let Some(first_block) = database_blocks.first() { + provider_rw.write_state( + &ExecutionOutcome { + first_block: first_block.number, + receipts: receipts.iter().take(database_blocks.len()).cloned().collect(), + ..Default::default() + }, + OriginalValuesKnown::No, + )?; + } + + provider_rw.commit()?; let provider = BlockchainProvider::new(factory)?; @@ -923,12 +886,14 @@ mod tests { let execution_outcome = ExecutionOutcome { receipts: vec![block_receipts], ..Default::default() }; - ExecutedBlockWithTrieUpdates::new( - Arc::new(RecoveredBlock::new_sealed(block.clone(), senders)), - execution_outcome.into(), - Default::default(), - Default::default(), - ) + ExecutedBlock { + recovered_block: Arc::new(RecoveredBlock::new_sealed( + block.clone(), + senders, + )), + execution_output: execution_outcome.into(), + ..Default::default() + } }) .collect(), }; @@ -981,26 +946,24 @@ mod tests { ) { let hook_provider = provider.clone(); provider.database.db_ref().set_post_transaction_hook(Box::new(move || { - if let Some(state) = hook_provider.canonical_in_memory_state.head_state() { - if state.anchor().number + 1 == block_number { - let mut lowest_memory_block = - state.parent_state_chain().last().expect("qed").block(); - let num_hash = lowest_memory_block.recovered_block().num_hash(); - - let mut execution_output = (*lowest_memory_block.execution_output).clone(); - execution_output.first_block = lowest_memory_block.recovered_block().number; - lowest_memory_block.execution_output = Arc::new(execution_output); - - // Push to disk - let provider_rw = hook_provider.database_provider_rw().unwrap(); - UnifiedStorageWriter::from(&provider_rw, &hook_provider.static_file_provider()) - .save_blocks(vec![lowest_memory_block]) - .unwrap(); - UnifiedStorageWriter::commit(provider_rw).unwrap(); - - // Remove from memory - hook_provider.canonical_in_memory_state.remove_persisted_blocks(num_hash); - } + if let Some(state) = hook_provider.canonical_in_memory_state.head_state() && + state.anchor().number + 1 == block_number + { + let mut lowest_memory_block = + state.parent_state_chain().last().expect("qed").block(); + let num_hash = lowest_memory_block.recovered_block().num_hash(); + + let mut execution_output = (*lowest_memory_block.execution_output).clone(); + execution_output.first_block = lowest_memory_block.recovered_block().number; + lowest_memory_block.execution_output = Arc::new(execution_output); + + // Push to disk + let provider_rw = hook_provider.database_provider_rw().unwrap(); + provider_rw.save_blocks(vec![lowest_memory_block]).unwrap(); + provider_rw.commit().unwrap(); + + // Remove from memory + hook_provider.canonical_in_memory_state.remove_persisted_blocks(num_hash); } })); } @@ -1022,7 +985,7 @@ mod tests { // Insert first 5 blocks into the database let provider_rw = factory.provider_rw()?; for block in database_blocks { - provider_rw.insert_historical_block( + provider_rw.insert_block( block.clone().try_recover().expect("failed to seal block with senders"), )?; } @@ -1052,15 +1015,13 @@ mod tests { let in_memory_block_senders = first_in_mem_block.senders().expect("failed to recover senders"); let chain = NewCanonicalChain::Commit { - new: vec![ExecutedBlockWithTrieUpdates::new( - Arc::new(RecoveredBlock::new_sealed( + new: vec![ExecutedBlock { + recovered_block: Arc::new(RecoveredBlock::new_sealed( first_in_mem_block.clone(), in_memory_block_senders, )), - Default::default(), - Default::default(), - Default::default(), - )], + ..Default::default() + }], }; provider.canonical_in_memory_state.update_chain(chain); @@ -1088,16 +1049,12 @@ mod tests { assert_eq!(provider.find_block_by_hash(first_db_block.hash(), BlockSource::Pending)?, None); // Insert the last block into the pending state - provider.canonical_in_memory_state.set_pending_block(ExecutedBlockWithTrieUpdates { - block: ExecutedBlock { - recovered_block: Arc::new(RecoveredBlock::new_sealed( - last_in_mem_block.clone(), - Default::default(), - )), - execution_output: Default::default(), - hashed_state: Default::default(), - }, - trie: Default::default(), + provider.canonical_in_memory_state.set_pending_block(ExecutedBlock { + recovered_block: Arc::new(RecoveredBlock::new_sealed( + last_in_mem_block.clone(), + Default::default(), + )), + ..Default::default() }); // Now the last block should be found in memory @@ -1126,7 +1083,7 @@ mod tests { // Insert first 5 blocks into the database let provider_rw = factory.provider_rw()?; for block in database_blocks { - provider_rw.insert_historical_block( + provider_rw.insert_block( block.clone().try_recover().expect("failed to seal block with senders"), )?; } @@ -1148,15 +1105,13 @@ mod tests { let in_memory_block_senders = first_in_mem_block.senders().expect("failed to recover senders"); let chain = NewCanonicalChain::Commit { - new: vec![ExecutedBlockWithTrieUpdates::new( - Arc::new(RecoveredBlock::new_sealed( + new: vec![ExecutedBlock { + recovered_block: Arc::new(RecoveredBlock::new_sealed( first_in_mem_block.clone(), in_memory_block_senders, )), - Default::default(), - Default::default(), - Default::default(), - )], + ..Default::default() + }], }; provider.canonical_in_memory_state.update_chain(chain); @@ -1202,65 +1157,26 @@ mod tests { ); // Set the block as pending - provider.canonical_in_memory_state.set_pending_block(ExecutedBlockWithTrieUpdates { - block: ExecutedBlock { - recovered_block: Arc::new(RecoveredBlock::new_sealed( - block.clone(), - block.senders().unwrap(), - )), - execution_output: Default::default(), - hashed_state: Default::default(), - }, - trie: Default::default(), + provider.canonical_in_memory_state.set_pending_block(ExecutedBlock { + recovered_block: Arc::new(RecoveredBlock::new_sealed( + block.clone(), + block.senders().unwrap(), + )), + ..Default::default() }); // Assertions related to the pending block - assert_eq!(provider.pending_block()?, Some(block.clone())); assert_eq!( - provider.pending_block_with_senders()?, + provider.pending_block()?, Some(RecoveredBlock::new_sealed(block.clone(), block.senders().unwrap())) ); - assert_eq!(provider.pending_block_and_receipts()?, Some((block, vec![]))); - - Ok(()) - } - - #[test] - fn test_block_reader_ommers() -> eyre::Result<()> { - // Create a new provider - let mut rng = generators::rng(); - let (provider, _, in_memory_blocks, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams::default(), - )?; - - let first_in_mem_block = in_memory_blocks.first().unwrap(); - - // If the block is after the Merge, we should have an empty ommers list - assert_eq!( - provider.ommers( - (provider.chain_spec().paris_block_and_final_difficulty.unwrap().0 + 2).into() - )?, - Some(vec![]) - ); - - // First in memory block ommers should be found - assert_eq!( - provider.ommers(first_in_mem_block.number.into())?, - Some(first_in_mem_block.body().ommers.clone()) - ); assert_eq!( - provider.ommers(first_in_mem_block.hash().into())?, - Some(first_in_mem_block.body().ommers.clone()) + provider.pending_block_and_receipts()?, + Some((RecoveredBlock::new_sealed(block.clone(), block.senders().unwrap()), vec![])) ); - // A random hash should return None as the block number is not found - assert_eq!(provider.ommers(B256::random().into())?, None); - Ok(()) } @@ -1284,15 +1200,13 @@ mod tests { let in_memory_block_senders = first_in_mem_block.senders().expect("failed to recover senders"); let chain = NewCanonicalChain::Commit { - new: vec![ExecutedBlockWithTrieUpdates::new( - Arc::new(RecoveredBlock::new_sealed( + new: vec![ExecutedBlock { + recovered_block: Arc::new(RecoveredBlock::new_sealed( first_in_mem_block.clone(), in_memory_block_senders, )), - Default::default(), - Default::default(), - Default::default(), - )], + ..Default::default() + }], }; provider.canonical_in_memory_state.update_chain(chain); @@ -1358,24 +1272,12 @@ mod tests { BlockRangeParams::default(), )?; - let database_block = database_blocks.first().unwrap().clone(); - let in_memory_block = in_memory_blocks.last().unwrap().clone(); // make sure that the finalized block is on db let finalized_block = database_blocks.get(database_blocks.len() - 3).unwrap(); provider.set_finalized(finalized_block.clone_sealed_header()); let blocks = [database_blocks, in_memory_blocks].concat(); - assert_eq!( - provider.header_td_by_number(database_block.number)?, - Some(database_block.difficulty) - ); - - assert_eq!( - provider.header_td_by_number(in_memory_block.number)?, - Some(in_memory_block.difficulty) - ); - assert_eq!( provider.sealed_headers_while(0..=10, |header| header.number <= 8)?, blocks @@ -1392,14 +1294,14 @@ mod tests { async fn test_canon_state_subscriptions() -> eyre::Result<()> { let factory = create_test_provider_factory(); - // Generate a random block to initialise the blockchain provider. + // Generate a random block to initialize the blockchain provider. let mut test_block_builder = TestBlockBuilder::eth(); let block_1 = test_block_builder.generate_random_block(0, B256::ZERO); let block_hash_1 = block_1.hash(); // Insert and commit the block. let provider_rw = factory.provider_rw()?; - provider_rw.insert_historical_block(block_1)?; + provider_rw.insert_block(block_1)?; provider_rw.commit()?; let provider = BlockchainProvider::new(factory)?; @@ -1432,50 +1334,6 @@ mod tests { Ok(()) } - #[test] - fn test_withdrawals_provider() -> eyre::Result<()> { - let mut rng = generators::rng(); - let chain_spec = Arc::new(ChainSpecBuilder::mainnet().shanghai_activated().build()); - let (provider, database_blocks, in_memory_blocks, _) = - provider_with_chain_spec_and_random_blocks( - &mut rng, - chain_spec.clone(), - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams { withdrawals_count: Some(1..3), ..Default::default() }, - )?; - let blocks = [database_blocks, in_memory_blocks].concat(); - - let shainghai_timestamp = - chain_spec.hardforks.fork(EthereumHardfork::Shanghai).as_timestamp().unwrap(); - - assert_eq!( - provider - .withdrawals_by_block( - alloy_eips::BlockHashOrNumber::Number(15), - shainghai_timestamp - ) - .expect("could not call withdrawals by block"), - Some(Withdrawals::new(vec![])), - "Expected withdrawals_by_block to return empty list if block does not exist" - ); - - for block in blocks { - assert_eq!( - provider - .withdrawals_by_block( - alloy_eips::BlockHashOrNumber::Number(block.number), - shainghai_timestamp - )? - .unwrap(), - block.body().withdrawals.clone().unwrap(), - "Expected withdrawals_by_block to return correct withdrawals" - ); - } - - Ok(()) - } - #[test] fn test_block_num_reader() -> eyre::Result<()> { let mut rng = generators::rng(); @@ -1651,46 +1509,6 @@ mod tests { Ok(()) } - #[test] - fn test_block_reader_id_ext_ommers_by_id() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, database_blocks, in_memory_blocks, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams::default(), - )?; - - let database_block = database_blocks.first().unwrap().clone(); - let in_memory_block = in_memory_blocks.last().unwrap().clone(); - - let block_number = database_block.number; - let block_hash = database_block.hash(); - - assert_eq!( - provider.ommers_by_id(block_number.into()).unwrap().unwrap_or_default(), - database_block.body().ommers - ); - assert_eq!( - provider.ommers_by_id(block_hash.into()).unwrap().unwrap_or_default(), - database_block.body().ommers - ); - - let block_number = in_memory_block.number; - let block_hash = in_memory_block.hash(); - - assert_eq!( - provider.ommers_by_id(block_number.into()).unwrap().unwrap_or_default(), - in_memory_block.body().ommers - ); - assert_eq!( - provider.ommers_by_id(block_hash.into()).unwrap().unwrap_or_default(), - in_memory_block.body().ommers - ); - - Ok(()) - } - #[test] fn test_receipt_provider_id_ext_receipts_by_block_id() -> eyre::Result<()> { let mut rng = generators::rng(); @@ -1837,7 +1655,6 @@ mod tests { ..Default::default() }, Default::default(), - Default::default(), )?; provider_rw.commit()?; @@ -1849,9 +1666,12 @@ mod tests { .first() .map(|block| { let senders = block.senders().expect("failed to recover senders"); - ExecutedBlockWithTrieUpdates::new( - Arc::new(RecoveredBlock::new_sealed(block.clone(), senders)), - Arc::new(ExecutionOutcome { + ExecutedBlock { + recovered_block: Arc::new(RecoveredBlock::new_sealed( + block.clone(), + senders, + )), + execution_output: Arc::new(ExecutionOutcome { bundle: BundleState::new( in_memory_state.into_iter().map(|(address, (account, _))| { (address, None, Some(account.into()), Default::default()) @@ -1864,9 +1684,8 @@ mod tests { first_block: first_in_memory_block, ..Default::default() }), - Default::default(), - Default::default(), - ) + ..Default::default() + } }) .unwrap()], }; @@ -1984,19 +1803,13 @@ mod tests { // adding a pending block to state can test pending() and pending_state_by_hash() function let pending_block = database_blocks[database_blocks.len() - 1].clone(); - only_database_provider.canonical_in_memory_state.set_pending_block( - ExecutedBlockWithTrieUpdates { - block: ExecutedBlock { - recovered_block: Arc::new(RecoveredBlock::new_sealed( - pending_block.clone(), - Default::default(), - )), - execution_output: Default::default(), - hashed_state: Default::default(), - }, - trie: Default::default(), - }, - ); + only_database_provider.canonical_in_memory_state.set_pending_block(ExecutedBlock { + recovered_block: Arc::new(RecoveredBlock::new_sealed( + pending_block.clone(), + Default::default(), + )), + ..Default::default() + }); assert_eq!( pending_block.hash(), @@ -2082,16 +1895,12 @@ mod tests { // Set the pending block in memory let pending_block = in_memory_blocks.last().unwrap(); - provider.canonical_in_memory_state.set_pending_block(ExecutedBlockWithTrieUpdates { - block: ExecutedBlock { - recovered_block: Arc::new(RecoveredBlock::new_sealed( - pending_block.clone(), - Default::default(), - )), - execution_output: Default::default(), - hashed_state: Default::default(), - }, - trie: Default::default(), + provider.canonical_in_memory_state.set_pending_block(ExecutedBlock { + recovered_block: Arc::new(RecoveredBlock::new_sealed( + pending_block.clone(), + Default::default(), + )), + ..Default::default() }); // Set the safe block in memory @@ -2168,7 +1977,7 @@ mod tests { "partial mem data" ); - // Test range in in-memory to unbounded end + // Test range in memory to unbounded end assert_eq!(provider.$method(in_mem_range.start() + 1..)?, &in_memory_data[1..], "unbounded mem data"); // Test last element in-memory @@ -2187,7 +1996,7 @@ mod tests { // Test range that spans database and in-memory { - // This block will be persisted to disk and removed from memory AFTER the firsk database query. This ensures that we query the in-memory state before the database avoiding any race condition. + // This block will be persisted to disk and removed from memory AFTER the first database query. This ensures that we query the in-memory state before the database avoiding any race condition. persist_block_after_db_tx_creation(provider.clone(), in_memory_blocks[0].number); assert_eq!( @@ -2279,7 +2088,7 @@ mod tests { // Test range that spans database and in-memory { - // This block will be persisted to disk and removed from memory AFTER the firsk database query. This ensures that we query the in-memory state before the database avoiding any race condition. + // This block will be persisted to disk and removed from memory AFTER the first database query. This ensures that we query the in-memory state before the database avoiding any race condition. persist_block_after_db_tx_creation(provider.clone(), in_memory_blocks[0].number); assert_eq!( @@ -2396,7 +2205,7 @@ mod tests { // Ensure that the first generated in-memory block exists { - // This block will be persisted to disk and removed from memory AFTER the firsk database query. This ensures that we query the in-memory state before the database avoiding any race condition. + // This block will be persisted to disk and removed from memory AFTER the first database query. This ensures that we query the in-memory state before the database avoiding any race condition. persist_block_after_db_tx_creation(provider.clone(), in_memory_blocks[0].number); call_method!($arg_count, provider, $method, $item_extractor, tx_num, tx_hash, &in_memory_blocks[0], &receipts); @@ -2417,7 +2226,7 @@ mod tests { // Invalid/Non-existent argument should return `None` { - call_method!($arg_count, provider, $method, |_,_,_,_| ( ($invalid_args, None)), tx_num, tx_hash, &in_memory_blocks[0], &receipts); + call_method!($arg_count, provider, $method, |_,_,_,_| ($invalid_args, None), tx_num, tx_hash, &in_memory_blocks[0], &receipts); } // Check that the item is only in memory and not in database @@ -2428,7 +2237,7 @@ mod tests { call_method!($arg_count, provider, $method, |_,_,_,_| (args.clone(), expected_item), tx_num, tx_hash, last_mem_block, &receipts); // Ensure the item is not in storage - call_method!($arg_count, provider.database, $method, |_,_,_,_| ( (args, None)), tx_num, tx_hash, last_mem_block, &receipts); + call_method!($arg_count, provider.database, $method, |_,_,_,_| (args, None), tx_num, tx_hash, last_mem_block, &receipts); } )* }}; @@ -2439,13 +2248,15 @@ mod tests { let test_tx_index = 0; test_non_range!([ - // TODO: header should use B256 like others instead of &B256 - // ( - // ONE, - // header, - // |block: &SealedBlock, tx_num: TxNumber, tx_hash: B256, receipts: &Vec>| (&block.hash(), Some(block.header.header().clone())), - // (&B256::random()) - // ), + ( + ONE, + header, + |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( + block.hash(), + Some(block.header().clone()) + ), + B256::random() + ), ( ONE, header_by_number, @@ -2729,14 +2540,15 @@ mod tests { persist_block_after_db_tx_creation(provider.clone(), in_memory_blocks[1].number); let to_be_persisted_tx = in_memory_blocks[1].body().transactions[0].clone(); - assert!(matches!( + assert_eq!( correct_transaction_hash_fn( *to_be_persisted_tx.tx_hash(), provider.canonical_in_memory_state(), provider.database - ), - Ok(Some(to_be_persisted_tx)) - )); + ) + .unwrap(), + Some(to_be_persisted_tx) + ); } Ok(()) diff --git a/crates/storage/provider/src/providers/consistent.rs b/crates/storage/provider/src/providers/consistent.rs index c92cf303c1d..67113fc5c0c 100644 --- a/crates/storage/provider/src/providers/consistent.rs +++ b/crates/storage/provider/src/providers/consistent.rs @@ -4,32 +4,31 @@ use crate::{ BlockReader, BlockReaderIdExt, BlockSource, ChainSpecProvider, ChangeSetReader, HeaderProvider, ProviderError, PruneCheckpointReader, ReceiptProvider, ReceiptProviderIdExt, StageCheckpointReader, StateReader, StaticFileProviderFactory, TransactionVariant, - TransactionsProvider, WithdrawalsProvider, + TransactionsProvider, TrieReader, }; use alloy_consensus::{transaction::TransactionMeta, BlockHeader}; use alloy_eips::{ - eip2718::Encodable2718, eip4895::Withdrawals, BlockHashOrNumber, BlockId, BlockNumHash, - BlockNumberOrTag, HashOrNumber, + eip2718::Encodable2718, BlockHashOrNumber, BlockId, BlockNumHash, BlockNumberOrTag, + HashOrNumber, }; use alloy_primitives::{ map::{hash_map, HashMap}, - Address, BlockHash, BlockNumber, TxHash, TxNumber, B256, U256, + Address, BlockHash, BlockNumber, TxHash, TxNumber, B256, }; use reth_chain_state::{BlockState, CanonicalInMemoryState, MemoryOverlayStateProviderRef}; -use reth_chainspec::{ChainInfo, EthereumHardforks}; +use reth_chainspec::ChainInfo; use reth_db_api::models::{AccountBeforeTx, BlockNumberAddress, StoredBlockBodyIndices}; use reth_execution_types::{BundleStateInit, ExecutionOutcome, RevertsInit}; use reth_node_types::{BlockTy, HeaderTy, ReceiptTy, TxTy}; -use reth_primitives_traits::{ - Account, BlockBody, RecoveredBlock, SealedBlock, SealedHeader, StorageEntry, -}; +use reth_primitives_traits::{Account, BlockBody, RecoveredBlock, SealedHeader, StorageEntry}; use reth_prune_types::{PruneCheckpoint, PruneSegment}; use reth_stages_types::{StageCheckpoint, StageId}; use reth_storage_api::{ - BlockBodyIndicesProvider, DatabaseProviderFactory, NodePrimitivesProvider, OmmersProvider, - StateProvider, StorageChangeSetReader, + BlockBodyIndicesProvider, DatabaseProviderFactory, NodePrimitivesProvider, StateProvider, + StorageChangeSetReader, TryIntoHistoricalStateProvider, }; use reth_storage_errors::provider::ProviderResult; +use reth_trie::updates::TrieUpdatesSorted; use revm_database::states::PlainStorageRevert; use std::{ ops::{Add, Bound, RangeBounds, RangeInclusive, Sub}, @@ -538,10 +537,10 @@ impl ConsistentProvider { // If the transaction number is less than the first in-memory transaction number, make a // database lookup - if let HashOrNumber::Number(id) = id { - if id < in_memory_tx_num { - return fetch_from_db(provider) - } + if let HashOrNumber::Number(id) = id && + id < in_memory_tx_num + { + return fetch_from_db(provider) } // Iterate from the lowest block to the highest @@ -591,6 +590,28 @@ impl ConsistentProvider { } fetch_from_db(&self.storage_provider) } + + /// Consumes the provider and returns a state provider for the specific block hash. + pub(crate) fn into_state_provider_at_block_hash( + self, + block_hash: BlockHash, + ) -> ProviderResult> { + let Self { storage_provider, head_block, .. } = self; + let into_history_at_block_hash = |block_hash| -> ProviderResult> { + let block_number = storage_provider + .block_number(block_hash)? + .ok_or(ProviderError::BlockHashNotFound(block_hash))?; + storage_provider.try_into_history_at_block(block_number) + }; + if let Some(Some(block_state)) = + head_block.as_ref().map(|b| b.block_on_chain(block_hash.into())) + { + let anchor_hash = block_state.anchor().hash; + let latest_historical = into_history_at_block_hash(anchor_hash)?; + return Ok(Box::new(block_state.state_provider(latest_historical))); + } + into_history_at_block_hash(block_hash) + } } impl ConsistentProvider { @@ -626,9 +647,9 @@ impl StaticFileProviderFactory for ConsistentProvider { impl HeaderProvider for ConsistentProvider { type Header = HeaderTy; - fn header(&self, block_hash: &BlockHash) -> ProviderResult> { + fn header(&self, block_hash: BlockHash) -> ProviderResult> { self.get_in_memory_or_storage_by_block( - (*block_hash).into(), + block_hash.into(), |db_provider| db_provider.header(block_hash), |block_state| Ok(Some(block_state.block_ref().recovered_block().clone_header())), ) @@ -642,37 +663,6 @@ impl HeaderProvider for ConsistentProvider { ) } - fn header_td(&self, hash: &BlockHash) -> ProviderResult> { - if let Some(num) = self.block_number(*hash)? { - self.header_td_by_number(num) - } else { - Ok(None) - } - } - - fn header_td_by_number(&self, number: BlockNumber) -> ProviderResult> { - let number = if self.head_block.as_ref().map(|b| b.block_on_chain(number.into())).is_some() - { - // If the block exists in memory, we should return a TD for it. - // - // The canonical in memory state should only store post-merge blocks. Post-merge blocks - // have zero difficulty. This means we can use the total difficulty for the last - // finalized block number if present (so that we are not affected by reorgs), if not the - // last number in the database will be used. - if let Some(last_finalized_num_hash) = - self.canonical_in_memory_state.get_finalized_num_hash() - { - last_finalized_num_hash.number - } else { - self.last_block_number()? - } - } else { - // Otherwise, return what we have on disk for the input block - number - }; - self.storage_provider.header_td_by_number(number) - } - fn headers_range( &self, range: impl RangeBounds, @@ -796,14 +786,14 @@ impl BlockReader for ConsistentProvider { hash: B256, source: BlockSource, ) -> ProviderResult> { - if matches!(source, BlockSource::Canonical | BlockSource::Any) { - if let Some(block) = self.get_in_memory_or_storage_by_block( + if matches!(source, BlockSource::Canonical | BlockSource::Any) && + let Some(block) = self.get_in_memory_or_storage_by_block( hash.into(), |db_provider| db_provider.find_block_by_hash(hash, BlockSource::Canonical), |block_state| Ok(Some(block_state.block_ref().recovered_block().clone_block())), - )? { - return Ok(Some(block)) - } + )? + { + return Ok(Some(block)) } if matches!(source, BlockSource::Pending | BlockSource::Any) { @@ -825,17 +815,13 @@ impl BlockReader for ConsistentProvider { ) } - fn pending_block(&self) -> ProviderResult>> { - Ok(self.canonical_in_memory_state.pending_block()) - } - - fn pending_block_with_senders(&self) -> ProviderResult>> { + fn pending_block(&self) -> ProviderResult>> { Ok(self.canonical_in_memory_state.pending_recovered_block()) } fn pending_block_and_receipts( &self, - ) -> ProviderResult, Vec)>> { + ) -> ProviderResult, Vec)>> { Ok(self.canonical_in_memory_state.pending_block_and_receipts()) } @@ -901,6 +887,14 @@ impl BlockReader for ConsistentProvider { |_| true, ) } + + fn block_by_transaction_id(&self, id: TxNumber) -> ProviderResult> { + self.get_in_memory_or_storage_by_tx( + id.into(), + |db_provider| db_provider.block_by_transaction_id(id), + |_, _, block_state| Ok(Some(block_state.number())), + ) + } } impl TransactionsProvider for ConsistentProvider { @@ -1103,6 +1097,13 @@ impl ReceiptProvider for ConsistentProvider { }, ) } + + fn receipts_by_block_range( + &self, + block_range: RangeInclusive, + ) -> ProviderResult>> { + self.storage_provider.receipts_by_block_range(block_range) + } } impl ReceiptProviderIdExt for ConsistentProvider { @@ -1110,14 +1111,14 @@ impl ReceiptProviderIdExt for ConsistentProvider { match block { BlockId::Hash(rpc_block_hash) => { let mut receipts = self.receipts_by_block(rpc_block_hash.block_hash.into())?; - if receipts.is_none() && !rpc_block_hash.require_canonical.unwrap_or(false) { - if let Some(state) = self + if receipts.is_none() && + !rpc_block_hash.require_canonical.unwrap_or(false) && + let Some(state) = self .head_block .as_ref() .and_then(|b| b.block_on_chain(rpc_block_hash.block_hash.into())) - { - receipts = Some(state.executed_block_receipts()); - } + { + receipts = Some(state.executed_block_receipts()); } Ok(receipts) } @@ -1138,42 +1139,6 @@ impl ReceiptProviderIdExt for ConsistentProvider { } } -impl WithdrawalsProvider for ConsistentProvider { - fn withdrawals_by_block( - &self, - id: BlockHashOrNumber, - timestamp: u64, - ) -> ProviderResult> { - if !self.chain_spec().is_shanghai_active_at_timestamp(timestamp) { - return Ok(None) - } - - self.get_in_memory_or_storage_by_block( - id, - |db_provider| db_provider.withdrawals_by_block(id, timestamp), - |block_state| { - Ok(block_state.block_ref().recovered_block().body().withdrawals().cloned()) - }, - ) - } -} - -impl OmmersProvider for ConsistentProvider { - fn ommers(&self, id: BlockHashOrNumber) -> ProviderResult>>> { - self.get_in_memory_or_storage_by_block( - id, - |db_provider| db_provider.ommers(id), - |block_state| { - if self.chain_spec().is_paris_active_at_block(block_state.number()) { - return Ok(Some(Vec::new())) - } - - Ok(block_state.block_ref().recovered_block().body().ommers().map(|o| o.to_vec())) - }, - ) - } -} - impl BlockBodyIndicesProvider for ConsistentProvider { fn block_body_indices( &self, @@ -1283,7 +1248,7 @@ impl BlockReaderIdExt for ConsistentProvider { BlockNumberOrTag::Safe => { self.canonical_in_memory_state.get_safe_header().map(|h| h.unseal()) } - BlockNumberOrTag::Earliest => self.header_by_number(0)?, + BlockNumberOrTag::Earliest => self.header_by_number(self.earliest_block_number()?)?, BlockNumberOrTag::Pending => self.canonical_in_memory_state.pending_header(), BlockNumberOrTag::Number(num) => self.header_by_number(num)?, @@ -1303,7 +1268,7 @@ impl BlockReaderIdExt for ConsistentProvider { } BlockNumberOrTag::Safe => Ok(self.canonical_in_memory_state.get_safe_header()), BlockNumberOrTag::Earliest => self - .header_by_number(0)? + .header_by_number(self.earliest_block_number()?)? .map_or_else(|| Ok(None), |h| Ok(Some(SealedHeader::seal_slow(h)))), BlockNumberOrTag::Pending => Ok(self.canonical_in_memory_state.pending_sealed_header()), BlockNumberOrTag::Number(num) => self @@ -1318,27 +1283,16 @@ impl BlockReaderIdExt for ConsistentProvider { ) -> ProviderResult>>> { Ok(match id { BlockId::Number(num) => self.sealed_header_by_number_or_tag(num)?, - BlockId::Hash(hash) => self.header(&hash.block_hash)?.map(SealedHeader::seal_slow), + BlockId::Hash(hash) => self.header(hash.block_hash)?.map(SealedHeader::seal_slow), }) } fn header_by_id(&self, id: BlockId) -> ProviderResult>> { Ok(match id { BlockId::Number(num) => self.header_by_number_or_tag(num)?, - BlockId::Hash(hash) => self.header(&hash.block_hash)?, + BlockId::Hash(hash) => self.header(hash.block_hash)?, }) } - - fn ommers_by_id(&self, id: BlockId) -> ProviderResult>>> { - match id { - BlockId::Number(num) => self.ommers_by_number_or_tag(num), - BlockId::Hash(hash) => { - // TODO: EIP-1898 question, see above - // here it is not handled - self.ommers(BlockHashOrNumber::Hash(hash.block_hash)) - } - } - } } impl StorageChangeSetReader for ConsistentProvider { @@ -1438,6 +1392,52 @@ impl ChangeSetReader for ConsistentProvider { self.storage_provider.account_block_changeset(block_number) } } + + fn get_account_before_block( + &self, + block_number: BlockNumber, + address: Address, + ) -> ProviderResult> { + if let Some(state) = + self.head_block.as_ref().and_then(|b| b.block_on_chain(block_number.into())) + { + // Search in-memory state for the account changeset + let changeset = state + .block_ref() + .execution_output + .bundle + .reverts + .clone() + .to_plain_state_reverts() + .accounts + .into_iter() + .flatten() + .find(|(addr, _)| addr == &address) + .map(|(address, info)| AccountBeforeTx { address, info: info.map(Into::into) }); + Ok(changeset) + } else { + // Perform checks on whether or not changesets exist for the block. + // No prune checkpoint means history should exist and we should `unwrap_or(true)` + let account_history_exists = self + .storage_provider + .get_prune_checkpoint(PruneSegment::AccountHistory)? + .and_then(|checkpoint| { + // return true if the block number is ahead of the prune checkpoint. + // + // The checkpoint stores the highest pruned block number, so we should make + // sure the block_number is strictly greater. + checkpoint.block_number.map(|checkpoint| block_number > checkpoint) + }) + .unwrap_or(true); + + if !account_history_exists { + return Err(ProviderError::StateAtBlockPruned(block_number)) + } + + // Delegate to the storage provider for database lookups + self.storage_provider.get_account_before_block(block_number, address) + } + } } impl AccountReader for ConsistentProvider { @@ -1474,6 +1474,19 @@ impl StateReader for ConsistentProvider { } } +impl TrieReader for ConsistentProvider { + fn trie_reverts(&self, from: BlockNumber) -> ProviderResult { + self.storage_provider.trie_reverts(from) + } + + fn get_block_trie_updates( + &self, + block_number: BlockNumber, + ) -> ProviderResult { + self.storage_provider.get_block_trie_updates(block_number) + } +} + #[cfg(test)] mod tests { use crate::{ @@ -1484,7 +1497,7 @@ mod tests { use alloy_primitives::B256; use itertools::Itertools; use rand::Rng; - use reth_chain_state::{ExecutedBlock, ExecutedBlockWithTrieUpdates, NewCanonicalChain}; + use reth_chain_state::{ExecutedBlock, NewCanonicalChain}; use reth_db_api::models::AccountBeforeTx; use reth_ethereum_primitives::Block; use reth_execution_types::ExecutionOutcome; @@ -1551,7 +1564,7 @@ mod tests { // Insert first 5 blocks into the database let provider_rw = factory.provider_rw()?; for block in database_blocks { - provider_rw.insert_historical_block( + provider_rw.insert_block( block.clone().try_recover().expect("failed to seal block with senders"), )?; } @@ -1587,15 +1600,13 @@ mod tests { let in_memory_block_senders = first_in_mem_block.senders().expect("failed to recover senders"); let chain = NewCanonicalChain::Commit { - new: vec![ExecutedBlockWithTrieUpdates::new( - Arc::new(RecoveredBlock::new_sealed( + new: vec![ExecutedBlock { + recovered_block: Arc::new(RecoveredBlock::new_sealed( first_in_mem_block.clone(), in_memory_block_senders, )), - Default::default(), - Default::default(), - Default::default(), - )], + ..Default::default() + }], }; consistent_provider.canonical_in_memory_state.update_chain(chain); let consistent_provider = provider.consistent_provider()?; @@ -1629,16 +1640,12 @@ mod tests { ); // Insert the last block into the pending state - provider.canonical_in_memory_state.set_pending_block(ExecutedBlockWithTrieUpdates { - block: ExecutedBlock { - recovered_block: Arc::new(RecoveredBlock::new_sealed( - last_in_mem_block.clone(), - Default::default(), - )), - execution_output: Default::default(), - hashed_state: Default::default(), - }, - trie: Default::default(), + provider.canonical_in_memory_state.set_pending_block(ExecutedBlock { + recovered_block: Arc::new(RecoveredBlock::new_sealed( + last_in_mem_block.clone(), + Default::default(), + )), + ..Default::default() }); // Now the last block should be found in memory @@ -1668,7 +1675,7 @@ mod tests { // Insert first 5 blocks into the database let provider_rw = factory.provider_rw()?; for block in database_blocks { - provider_rw.insert_historical_block( + provider_rw.insert_block( block.clone().try_recover().expect("failed to seal block with senders"), )?; } @@ -1697,15 +1704,13 @@ mod tests { let in_memory_block_senders = first_in_mem_block.senders().expect("failed to recover senders"); let chain = NewCanonicalChain::Commit { - new: vec![ExecutedBlockWithTrieUpdates::new( - Arc::new(RecoveredBlock::new_sealed( + new: vec![ExecutedBlock { + recovered_block: Arc::new(RecoveredBlock::new_sealed( first_in_mem_block.clone(), in_memory_block_senders, )), - Default::default(), - Default::default(), - Default::default(), - )], + ..Default::default() + }], }; consistent_provider.canonical_in_memory_state.update_chain(chain); @@ -1791,7 +1796,6 @@ mod tests { ..Default::default() }, Default::default(), - Default::default(), )?; provider_rw.commit()?; @@ -1803,9 +1807,12 @@ mod tests { .first() .map(|block| { let senders = block.senders().expect("failed to recover senders"); - ExecutedBlockWithTrieUpdates::new( - Arc::new(RecoveredBlock::new_sealed(block.clone(), senders)), - Arc::new(ExecutionOutcome { + ExecutedBlock { + recovered_block: Arc::new(RecoveredBlock::new_sealed( + block.clone(), + senders, + )), + execution_output: Arc::new(ExecutionOutcome { bundle: BundleState::new( in_memory_state.into_iter().map(|(address, (account, _))| { (address, None, Some(account.into()), Default::default()) @@ -1818,9 +1825,8 @@ mod tests { first_block: first_in_memory_block, ..Default::default() }), - Default::default(), - Default::default(), - ) + ..Default::default() + } }) .unwrap()], }; diff --git a/crates/storage/provider/src/providers/consistent_view.rs b/crates/storage/provider/src/providers/consistent_view.rs index 4957def6e28..d8404af5416 100644 --- a/crates/storage/provider/src/providers/consistent_view.rs +++ b/crates/storage/provider/src/providers/consistent_view.rs @@ -1,6 +1,5 @@ use crate::{BlockNumReader, DatabaseProviderFactory, HeaderProvider}; use alloy_primitives::B256; -use reth_storage_api::StateCommitmentProvider; pub use reth_storage_errors::provider::ConsistentViewError; use reth_storage_errors::provider::ProviderResult; @@ -27,8 +26,7 @@ pub struct ConsistentDbView { impl ConsistentDbView where - Factory: DatabaseProviderFactory - + StateCommitmentProvider, + Factory: DatabaseProviderFactory, { /// Creates new consistent database view. pub const fn new(factory: Factory, tip: Option<(B256, u64)>) -> Self { @@ -69,10 +67,10 @@ where // // To ensure this doesn't happen, we just have to make sure that we fetch from the same // data source that we used during initialization. In this case, that is static files - if let Some((hash, number)) = self.tip { - if provider_ro.sealed_header(number)?.is_none_or(|header| header.hash() != hash) { - return Err(ConsistentViewError::Reorged { block: hash }.into()) - } + if let Some((hash, number)) = self.tip && + provider_ro.sealed_header(number)?.is_none_or(|header| header.hash() != hash) + { + return Err(ConsistentViewError::Reorged { block: hash }.into()) } Ok(provider_ro) @@ -85,31 +83,27 @@ mod tests { use std::str::FromStr; use super::*; - use crate::{ - test_utils::create_test_provider_factory_with_chain_spec, BlockWriter, - StaticFileProviderFactory, StaticFileWriter, - }; + use crate::{test_utils::create_test_provider_factory, BlockWriter}; use alloy_primitives::Bytes; use assert_matches::assert_matches; - use reth_chainspec::{EthChainSpec, MAINNET}; + use reth_chainspec::{ChainSpecProvider, EthChainSpec}; use reth_ethereum_primitives::{Block, BlockBody}; use reth_primitives_traits::{block::TestBlock, RecoveredBlock, SealedBlock}; - use reth_static_file_types::StaticFileSegment; - use reth_storage_api::StorageLocation; #[test] fn test_consistent_view_extend() { - let provider_factory = create_test_provider_factory_with_chain_spec(MAINNET.clone()); + let provider_factory = create_test_provider_factory(); - let genesis_header = MAINNET.genesis_header(); - let genesis_block = - SealedBlock::::seal_parts(genesis_header.clone(), BlockBody::default()); + let genesis_block = SealedBlock::::seal_parts( + provider_factory.chain_spec().genesis_header().clone(), + BlockBody::default(), + ); let genesis_hash: B256 = genesis_block.hash(); let genesis_block = RecoveredBlock::new_sealed(genesis_block, vec![]); // insert the block let provider_rw = provider_factory.provider_rw().unwrap(); - provider_rw.insert_block(genesis_block, StorageLocation::StaticFiles).unwrap(); + provider_rw.insert_block(genesis_block).unwrap(); provider_rw.commit().unwrap(); // create a consistent view provider and check that a ro provider can be made @@ -127,7 +121,7 @@ mod tests { // insert the block let provider_rw = provider_factory.provider_rw().unwrap(); - provider_rw.insert_block(recovered_block, StorageLocation::StaticFiles).unwrap(); + provider_rw.insert_block(recovered_block).unwrap(); provider_rw.commit().unwrap(); // ensure successful creation of a read-only provider, based on this new db state. @@ -142,7 +136,7 @@ mod tests { // insert the block let provider_rw = provider_factory.provider_rw().unwrap(); - provider_rw.insert_block(recovered_block, StorageLocation::StaticFiles).unwrap(); + provider_rw.insert_block(recovered_block).unwrap(); provider_rw.commit().unwrap(); // check that creation of a read-only provider still works @@ -151,18 +145,18 @@ mod tests { #[test] fn test_consistent_view_remove() { - let provider_factory = create_test_provider_factory_with_chain_spec(MAINNET.clone()); + let provider_factory = create_test_provider_factory(); - let genesis_header = MAINNET.genesis_header(); - let genesis_block = - SealedBlock::::seal_parts(genesis_header.clone(), BlockBody::default()); + let genesis_block = SealedBlock::::seal_parts( + provider_factory.chain_spec().genesis_header().clone(), + BlockBody::default(), + ); let genesis_hash: B256 = genesis_block.hash(); let genesis_block = RecoveredBlock::new_sealed(genesis_block, vec![]); // insert the block let provider_rw = provider_factory.provider_rw().unwrap(); - provider_rw.insert_block(genesis_block, StorageLocation::Both).unwrap(); - provider_rw.0.static_file_provider().commit().unwrap(); + provider_rw.insert_block(genesis_block).unwrap(); provider_rw.commit().unwrap(); // create a consistent view provider and check that a ro provider can be made @@ -180,8 +174,7 @@ mod tests { // insert the block let provider_rw = provider_factory.provider_rw().unwrap(); - provider_rw.insert_block(recovered_block, StorageLocation::Both).unwrap(); - provider_rw.0.static_file_provider().commit().unwrap(); + provider_rw.insert_block(recovered_block).unwrap(); provider_rw.commit().unwrap(); // create a second consistent view provider and check that a ro provider can be made @@ -193,10 +186,7 @@ mod tests { // remove the block above the genesis block let provider_rw = provider_factory.provider_rw().unwrap(); - provider_rw.remove_blocks_above(0, StorageLocation::Both).unwrap(); - let sf_provider = provider_rw.0.static_file_provider(); - sf_provider.get_writer(1, StaticFileSegment::Headers).unwrap().prune_headers(1).unwrap(); - sf_provider.commit().unwrap(); + provider_rw.remove_blocks_above(0).unwrap(); provider_rw.commit().unwrap(); // ensure unsuccessful creation of a read-only provider, based on this new db state. @@ -218,8 +208,7 @@ mod tests { // reinsert the block at the same height, but with a different hash let provider_rw = provider_factory.provider_rw().unwrap(); - provider_rw.insert_block(recovered_block, StorageLocation::Both).unwrap(); - provider_rw.0.static_file_provider().commit().unwrap(); + provider_rw.insert_block(recovered_block).unwrap(); provider_rw.commit().unwrap(); // ensure unsuccessful creation of a read-only provider, based on this new db state. diff --git a/crates/storage/provider/src/providers/database/builder.rs b/crates/storage/provider/src/providers/database/builder.rs index 2f25c806945..4bc8569432e 100644 --- a/crates/storage/provider/src/providers/database/builder.rs +++ b/crates/storage/provider/src/providers/database/builder.rs @@ -4,7 +4,10 @@ //! up to the intended build target. use crate::{providers::StaticFileProvider, ProviderFactory}; -use reth_db::{mdbx::DatabaseArguments, open_db_read_only, DatabaseEnv}; +use reth_db::{ + mdbx::{DatabaseArguments, MaxReadTransactionDuration}, + open_db_read_only, DatabaseEnv, +}; use reth_db_api::{database_metrics::DatabaseMetrics, Database}; use reth_node_types::{NodeTypes, NodeTypesWithDBAdapter}; use std::{ @@ -62,7 +65,7 @@ impl ProviderFactoryBuilder { /// ```no_run /// use reth_chainspec::MAINNET; /// use reth_node_types::NodeTypes; - /// /// + /// /// use reth_provider::providers::{ProviderFactoryBuilder, ReadOnlyConfig}; /// /// fn demo>() { @@ -71,6 +74,29 @@ impl ProviderFactoryBuilder { /// .unwrap(); /// } /// ``` + /// + /// # Open an instance with disabled read-transaction timeout + /// + /// By default, read transactions are automatically terminated after a timeout to prevent + /// database free list growth. However, if the database is static (no writes occurring), this + /// safety mechanism can be disabled using + /// [`ReadOnlyConfig::disable_long_read_transaction_safety`]. + /// + /// ```no_run + /// use reth_chainspec::MAINNET; + /// use reth_node_types::NodeTypes; + /// + /// use reth_provider::providers::{ProviderFactoryBuilder, ReadOnlyConfig}; + /// + /// fn demo>() { + /// let provider_factory = ProviderFactoryBuilder::::default() + /// .open_read_only( + /// MAINNET.clone(), + /// ReadOnlyConfig::from_datadir("datadir").disable_long_read_transaction_safety(), + /// ) + /// .unwrap(); + /// } + /// ``` pub fn open_read_only( self, chainspec: Arc, @@ -129,6 +155,15 @@ impl ReadOnlyConfig { Self::from_dirs(datadir.join("db"), datadir.join("static_files")) } + /// Disables long-lived read transaction safety guarantees. + /// + /// Caution: Keeping database transaction open indefinitely can cause the free list to grow if + /// changes to the database are made. + pub const fn disable_long_read_transaction_safety(mut self) -> Self { + self.db_args.max_read_transaction_duration(Some(MaxReadTransactionDuration::Unbounded)); + self + } + /// Derives the [`ReadOnlyConfig`] from the database dir. /// /// By default this assumes the following datadir layout: diff --git a/crates/storage/provider/src/providers/database/chain.rs b/crates/storage/provider/src/providers/database/chain.rs index 9d0e0158a58..9ce3861eb3c 100644 --- a/crates/storage/provider/src/providers/database/chain.rs +++ b/crates/storage/provider/src/providers/database/chain.rs @@ -1,12 +1,12 @@ use crate::{providers::NodeTypesForProvider, DatabaseProvider}; use reth_db_api::transaction::{DbTx, DbTxMut}; -use reth_node_types::FullNodePrimitives; +use reth_node_types::NodePrimitives; use reth_primitives_traits::{FullBlockHeader, FullSignedTx}; -use reth_storage_api::{ChainStorageReader, ChainStorageWriter, EthStorage}; +use reth_storage_api::{ChainStorageReader, ChainStorageWriter, EmptyBodyStorage, EthStorage}; /// Trait that provides access to implementations of [`ChainStorage`] -pub trait ChainStorage: Send + Sync { +pub trait ChainStorage: Send + Sync { /// Provides access to the chain reader. fn reader(&self) -> impl ChainStorageReader, Primitives> where @@ -24,7 +24,35 @@ impl ChainStorage for EthStorage where T: FullSignedTx, H: FullBlockHeader, - N: FullNodePrimitives< + N: NodePrimitives< + Block = alloy_consensus::Block, + BlockHeader = H, + BlockBody = alloy_consensus::BlockBody, + SignedTx = T, + >, +{ + fn reader(&self) -> impl ChainStorageReader, N> + where + TX: DbTx + 'static, + Types: NodeTypesForProvider, + { + self + } + + fn writer(&self) -> impl ChainStorageWriter, N> + where + TX: DbTxMut + DbTx + 'static, + Types: NodeTypesForProvider, + { + self + } +} + +impl ChainStorage for EmptyBodyStorage +where + T: FullSignedTx, + H: FullBlockHeader, + N: NodePrimitives< Block = alloy_consensus::Block, BlockHeader = H, BlockBody = alloy_consensus::BlockBody, diff --git a/crates/storage/provider/src/providers/database/metrics.rs b/crates/storage/provider/src/providers/database/metrics.rs index 4ee8f1ce5b1..4daac3dfddb 100644 --- a/crates/storage/provider/src/providers/database/metrics.rs +++ b/crates/storage/provider/src/providers/database/metrics.rs @@ -36,34 +36,21 @@ impl DurationsRecorder { #[derive(Debug, Copy, Clone)] pub(crate) enum Action { - InsertStorageHashing, - InsertAccountHashing, - InsertMerkleTree, InsertBlock, InsertState, InsertHashes, InsertHistoryIndices, UpdatePipelineStages, - InsertCanonicalHeaders, - InsertHeaders, InsertHeaderNumbers, - InsertHeaderTerminalDifficulties, InsertBlockBodyIndices, InsertTransactionBlocks, GetNextTxNum, - GetParentTD, } /// Database provider metrics #[derive(Metrics)] #[metrics(scope = "storage.providers.database")] struct DatabaseProviderMetrics { - /// Duration of insert storage hashing - insert_storage_hashing: Histogram, - /// Duration of insert account hashing - insert_account_hashing: Histogram, - /// Duration of insert merkle tree - insert_merkle_tree: Histogram, /// Duration of insert block insert_block: Histogram, /// Duration of insert state @@ -75,43 +62,29 @@ struct DatabaseProviderMetrics { /// Duration of update pipeline stages update_pipeline_stages: Histogram, /// Duration of insert canonical headers - insert_canonical_headers: Histogram, - /// Duration of insert headers - insert_headers: Histogram, /// Duration of insert header numbers insert_header_numbers: Histogram, - /// Duration of insert header TD - insert_header_td: Histogram, /// Duration of insert block body indices insert_block_body_indices: Histogram, /// Duration of insert transaction blocks insert_tx_blocks: Histogram, /// Duration of get next tx num get_next_tx_num: Histogram, - /// Duration of get parent TD - get_parent_td: Histogram, } impl DatabaseProviderMetrics { /// Records the duration for the given action. pub(crate) fn record_duration(&self, action: Action, duration: Duration) { match action { - Action::InsertStorageHashing => self.insert_storage_hashing.record(duration), - Action::InsertAccountHashing => self.insert_account_hashing.record(duration), - Action::InsertMerkleTree => self.insert_merkle_tree.record(duration), Action::InsertBlock => self.insert_block.record(duration), Action::InsertState => self.insert_state.record(duration), Action::InsertHashes => self.insert_hashes.record(duration), Action::InsertHistoryIndices => self.insert_history_indices.record(duration), Action::UpdatePipelineStages => self.update_pipeline_stages.record(duration), - Action::InsertCanonicalHeaders => self.insert_canonical_headers.record(duration), - Action::InsertHeaders => self.insert_headers.record(duration), Action::InsertHeaderNumbers => self.insert_header_numbers.record(duration), - Action::InsertHeaderTerminalDifficulties => self.insert_header_td.record(duration), Action::InsertBlockBodyIndices => self.insert_block_body_indices.record(duration), Action::InsertTransactionBlocks => self.insert_tx_blocks.record(duration), Action::GetNextTxNum => self.get_next_tx_num.record(duration), - Action::GetParentTD => self.get_parent_td.record(duration), } } } diff --git a/crates/storage/provider/src/providers/database/mod.rs b/crates/storage/provider/src/providers/database/mod.rs index 1a698e46f14..873b10b0cfc 100644 --- a/crates/storage/provider/src/providers/database/mod.rs +++ b/crates/storage/provider/src/providers/database/mod.rs @@ -5,11 +5,11 @@ use crate::{ BlockHashReader, BlockNumReader, BlockReader, ChainSpecProvider, DatabaseProviderFactory, HashedPostStateProvider, HeaderProvider, HeaderSyncGapProvider, ProviderError, PruneCheckpointReader, StageCheckpointReader, StateProviderBox, StaticFileProviderFactory, - TransactionVariant, TransactionsProvider, WithdrawalsProvider, + TransactionVariant, TransactionsProvider, }; use alloy_consensus::transaction::TransactionMeta; -use alloy_eips::{eip4895::Withdrawals, BlockHashOrNumber}; -use alloy_primitives::{Address, BlockHash, BlockNumber, TxHash, TxNumber, B256, U256}; +use alloy_eips::BlockHashOrNumber; +use alloy_primitives::{Address, BlockHash, BlockNumber, TxHash, TxNumber, B256}; use core::fmt; use reth_chainspec::ChainInfo; use reth_db::{init_db, mdbx::DatabaseArguments, DatabaseEnv}; @@ -18,17 +18,15 @@ use reth_errors::{RethError, RethResult}; use reth_node_types::{ BlockTy, HeaderTy, NodeTypes, NodeTypesWithDB, NodeTypesWithDBAdapter, ReceiptTy, TxTy, }; -use reth_primitives_traits::{RecoveredBlock, SealedBlock, SealedHeader}; +use reth_primitives_traits::{RecoveredBlock, SealedHeader}; use reth_prune_types::{PruneCheckpoint, PruneModes, PruneSegment}; use reth_stages_types::{StageCheckpoint, StageId}; use reth_static_file_types::StaticFileSegment; use reth_storage_api::{ - BlockBodyIndicesProvider, NodePrimitivesProvider, OmmersProvider, StateCommitmentProvider, - TryIntoHistoricalStateProvider, + BlockBodyIndicesProvider, NodePrimitivesProvider, TryIntoHistoricalStateProvider, }; use reth_storage_errors::provider::ProviderResult; use reth_trie::HashedPostState; -use reth_trie_db::StateCommitment; use revm_database::BundleState; use std::{ ops::{RangeBounds, RangeInclusive}, @@ -42,6 +40,7 @@ mod provider; pub use provider::{DatabaseProvider, DatabaseProviderRO, DatabaseProviderRW}; use super::ProviderNodeTypes; +use reth_trie::KeccakKeyHasher; mod builder; pub use builder::{ProviderFactoryBuilder, ReadOnlyConfig}; @@ -85,7 +84,7 @@ impl ProviderFactory { db, chain_spec, static_file_provider, - prune_modes: PruneModes::none(), + prune_modes: PruneModes::default(), storage: Default::default(), } } @@ -127,7 +126,7 @@ impl>> ProviderFactory { db: Arc::new(init_db(path, args).map_err(RethError::msg)?), chain_spec, static_file_provider, - prune_modes: PruneModes::none(), + prune_modes: PruneModes::default(), storage: Default::default(), }) } @@ -215,10 +214,6 @@ impl DatabaseProviderFactory for ProviderFactory { } } -impl StateCommitmentProvider for ProviderFactory { - type StateCommitment = N::StateCommitment; -} - impl StaticFileProviderFactory for ProviderFactory { /// Returns static file provider fn static_file_provider(&self) -> StaticFileProvider { @@ -239,57 +234,33 @@ impl HeaderSyncGapProvider for ProviderFactory { impl HeaderProvider for ProviderFactory { type Header = HeaderTy; - fn header(&self, block_hash: &BlockHash) -> ProviderResult> { + fn header(&self, block_hash: BlockHash) -> ProviderResult> { self.provider()?.header(block_hash) } fn header_by_number(&self, num: BlockNumber) -> ProviderResult> { - self.static_file_provider.get_with_static_file_or_database( - StaticFileSegment::Headers, - num, - |static_file| static_file.header_by_number(num), - || self.provider()?.header_by_number(num), - ) - } - - fn header_td(&self, hash: &BlockHash) -> ProviderResult> { - self.provider()?.header_td(hash) - } - - fn header_td_by_number(&self, number: BlockNumber) -> ProviderResult> { - self.provider()?.header_td_by_number(number) + self.static_file_provider.header_by_number(num) } fn headers_range( &self, range: impl RangeBounds, ) -> ProviderResult> { - self.static_file_provider.get_range_with_static_file_or_database( - StaticFileSegment::Headers, - to_range(range), - |static_file, range, _| static_file.headers_range(range), - |range, _| self.provider()?.headers_range(range), - |_| true, - ) + self.static_file_provider.headers_range(range) } fn sealed_header( &self, number: BlockNumber, ) -> ProviderResult>> { - self.static_file_provider.get_with_static_file_or_database( - StaticFileSegment::Headers, - number, - |static_file| static_file.sealed_header(number), - || self.provider()?.sealed_header(number), - ) + self.static_file_provider.sealed_header(number) } fn sealed_headers_range( &self, range: impl RangeBounds, ) -> ProviderResult>> { - self.sealed_headers_while(range, |_| true) + self.static_file_provider.sealed_headers_range(range) } fn sealed_headers_while( @@ -297,24 +268,13 @@ impl HeaderProvider for ProviderFactory { range: impl RangeBounds, predicate: impl FnMut(&SealedHeader) -> bool, ) -> ProviderResult>> { - self.static_file_provider.get_range_with_static_file_or_database( - StaticFileSegment::Headers, - to_range(range), - |static_file, range, predicate| static_file.sealed_headers_while(range, predicate), - |range, predicate| self.provider()?.sealed_headers_while(range, predicate), - predicate, - ) + self.static_file_provider.sealed_headers_while(range, predicate) } } impl BlockHashReader for ProviderFactory { fn block_hash(&self, number: u64) -> ProviderResult> { - self.static_file_provider.get_with_static_file_or_database( - StaticFileSegment::Headers, - number, - |static_file| static_file.block_hash(number), - || self.provider()?.block_hash(number), - ) + self.static_file_provider.block_hash(number) } fn canonical_hashes_range( @@ -322,13 +282,7 @@ impl BlockHashReader for ProviderFactory { start: BlockNumber, end: BlockNumber, ) -> ProviderResult> { - self.static_file_provider.get_range_with_static_file_or_database( - StaticFileSegment::Headers, - start..end, - |static_file, range, _| static_file.canonical_hashes_range(range.start, range.end), - |range, _| self.provider()?.canonical_hashes_range(range.start, range.end), - |_| true, - ) + self.static_file_provider.canonical_hashes_range(start, end) } } @@ -342,7 +296,13 @@ impl BlockNumReader for ProviderFactory { } fn last_block_number(&self) -> ProviderResult { - self.provider()?.last_block_number() + self.static_file_provider.last_block_number() + } + + fn earliest_block_number(&self) -> ProviderResult { + // earliest history height tracks the lowest block number that has __not__ been expired, in + // other words, the first/earliest available block. + Ok(self.static_file_provider.earliest_history_height()) } fn block_number(&self, hash: B256) -> ProviderResult> { @@ -365,17 +325,13 @@ impl BlockReader for ProviderFactory { self.provider()?.block(id) } - fn pending_block(&self) -> ProviderResult>> { + fn pending_block(&self) -> ProviderResult>> { self.provider()?.pending_block() } - fn pending_block_with_senders(&self) -> ProviderResult>> { - self.provider()?.pending_block_with_senders() - } - fn pending_block_and_receipts( &self, - ) -> ProviderResult, Vec)>> { + ) -> ProviderResult, Vec)>> { self.provider()?.pending_block_and_receipts() } @@ -412,6 +368,10 @@ impl BlockReader for ProviderFactory { ) -> ProviderResult>> { self.provider()?.recovered_block_range(range) } + + fn block_by_transaction_id(&self, id: TxNumber) -> ProviderResult> { + self.provider()?.block_by_transaction_id(id) + } } impl TransactionsProvider for ProviderFactory { @@ -422,24 +382,14 @@ impl TransactionsProvider for ProviderFactory { } fn transaction_by_id(&self, id: TxNumber) -> ProviderResult> { - self.static_file_provider.get_with_static_file_or_database( - StaticFileSegment::Transactions, - id, - |static_file| static_file.transaction_by_id(id), - || self.provider()?.transaction_by_id(id), - ) + self.static_file_provider.transaction_by_id(id) } fn transaction_by_id_unhashed( &self, id: TxNumber, ) -> ProviderResult> { - self.static_file_provider.get_with_static_file_or_database( - StaticFileSegment::Transactions, - id, - |static_file| static_file.transaction_by_id_unhashed(id), - || self.provider()?.transaction_by_id_unhashed(id), - ) + self.static_file_provider.transaction_by_id_unhashed(id) } fn transaction_by_hash(&self, hash: TxHash) -> ProviderResult> { @@ -475,7 +425,7 @@ impl TransactionsProvider for ProviderFactory { &self, range: impl RangeBounds, ) -> ProviderResult> { - self.provider()?.transactions_by_tx_range(range) + self.static_file_provider.transactions_by_tx_range(range) } fn senders_by_tx_range( @@ -492,6 +442,7 @@ impl TransactionsProvider for ProviderFactory { impl ReceiptProvider for ProviderFactory { type Receipt = ReceiptTy; + fn receipt(&self, id: TxNumber) -> ProviderResult> { self.static_file_provider.get_with_static_file_or_database( StaticFileSegment::Receipts, @@ -524,21 +475,12 @@ impl ReceiptProvider for ProviderFactory { |_| true, ) } -} -impl WithdrawalsProvider for ProviderFactory { - fn withdrawals_by_block( + fn receipts_by_block_range( &self, - id: BlockHashOrNumber, - timestamp: u64, - ) -> ProviderResult> { - self.provider()?.withdrawals_by_block(id, timestamp) - } -} - -impl OmmersProvider for ProviderFactory { - fn ommers(&self, id: BlockHashOrNumber) -> ProviderResult>> { - self.provider()?.ommers(id) + block_range: RangeInclusive, + ) -> ProviderResult>> { + self.provider()?.receipts_by_block_range(block_range) } } @@ -547,29 +489,14 @@ impl BlockBodyIndicesProvider for ProviderFactory { &self, number: BlockNumber, ) -> ProviderResult> { - self.static_file_provider.get_with_static_file_or_database( - StaticFileSegment::BlockMeta, - number, - |static_file| static_file.block_body_indices(number), - || self.provider()?.block_body_indices(number), - ) + self.provider()?.block_body_indices(number) } fn block_body_indices_range( &self, range: RangeInclusive, ) -> ProviderResult> { - self.static_file_provider.get_range_with_static_file_or_database( - StaticFileSegment::BlockMeta, - *range.start()..*range.end() + 1, - |static_file, range, _| { - static_file.block_body_indices_range(range.start..=range.end.saturating_sub(1)) - }, - |range, _| { - self.provider()?.block_body_indices_range(range.start..=range.end.saturating_sub(1)) - }, - |_| true, - ) + self.provider()?.block_body_indices_range(range) } } @@ -609,9 +536,7 @@ impl PruneCheckpointReader for ProviderFactory { impl HashedPostStateProvider for ProviderFactory { fn hashed_post_state(&self, bundle_state: &BundleState) -> HashedPostState { - HashedPostState::from_bundle_state::<::KeyHasher>( - bundle_state.state(), - ) + HashedPostState::from_bundle_state::(bundle_state.state()) } } @@ -650,9 +575,9 @@ mod tests { providers::{StaticFileProvider, StaticFileWriter}, test_utils::{blocks::TEST_BLOCK, create_test_provider_factory, MockNodeTypesWithDB}, BlockHashReader, BlockNumReader, BlockWriter, DBProvider, HeaderSyncGapProvider, - StorageLocation, TransactionsProvider, + TransactionsProvider, }; - use alloy_primitives::{TxNumber, B256, U256}; + use alloy_primitives::{TxNumber, B256}; use assert_matches::assert_matches; use reth_chainspec::ChainSpecBuilder; use reth_db::{ @@ -660,6 +585,7 @@ mod tests { test_utils::{create_test_static_files_dir, ERROR_TEMPDIR}, }; use reth_db_api::tables; + use reth_primitives_traits::SignerRecoverable; use reth_prune_types::{PruneMode, PruneModes}; use reth_storage_errors::provider::ProviderError; use reth_testing_utils::generators::{self, random_block, random_header, BlockParams}; @@ -696,13 +622,12 @@ mod tests { let chain_spec = ChainSpecBuilder::mainnet().build(); let (_static_dir, static_dir_path) = create_test_static_files_dir(); let factory = ProviderFactory::>::new_with_database_path( - tempfile::TempDir::new().expect(ERROR_TEMPDIR).into_path(), + tempfile::TempDir::new().expect(ERROR_TEMPDIR).keep(), Arc::new(chain_spec), DatabaseArguments::new(Default::default()), StaticFileProvider::read_write(static_dir_path).unwrap(), ) .unwrap(); - let provider = factory.provider().unwrap(); provider.block_hash(0).unwrap(); let provider_rw = factory.provider_rw().unwrap(); @@ -712,16 +637,12 @@ mod tests { #[test] fn insert_block_with_prune_modes() { - let factory = create_test_provider_factory(); - let block = TEST_BLOCK.clone(); + { + let factory = create_test_provider_factory(); let provider = factory.provider_rw().unwrap(); - assert_matches!( - provider - .insert_block(block.clone().try_recover().unwrap(), StorageLocation::Database), - Ok(_) - ); + assert_matches!(provider.insert_block(block.clone().try_recover().unwrap()), Ok(_)); assert_matches!( provider.transaction_sender(0), Ok(Some(sender)) if sender == block.body().transactions[0].recover_signer().unwrap() @@ -736,14 +657,11 @@ mod tests { let prune_modes = PruneModes { sender_recovery: Some(PruneMode::Full), transaction_lookup: Some(PruneMode::Full), - ..PruneModes::none() + ..PruneModes::default() }; + let factory = create_test_provider_factory(); let provider = factory.with_prune_modes(prune_modes).provider_rw().unwrap(); - assert_matches!( - provider - .insert_block(block.clone().try_recover().unwrap(), StorageLocation::Database), - Ok(_) - ); + assert_matches!(provider.insert_block(block.clone().try_recover().unwrap()), Ok(_)); assert_matches!(provider.transaction_sender(0), Ok(None)); assert_matches!( provider.transaction_id(*block.body().transactions[0].tx_hash()), @@ -754,21 +672,16 @@ mod tests { #[test] fn take_block_transaction_range_recover_senders() { - let factory = create_test_provider_factory(); - let mut rng = generators::rng(); let block = random_block(&mut rng, 0, BlockParams { tx_count: Some(3), ..Default::default() }); let tx_ranges: Vec> = vec![0..=0, 1..=1, 2..=2, 0..=1, 1..=2]; for range in tx_ranges { + let factory = create_test_provider_factory(); let provider = factory.provider_rw().unwrap(); - assert_matches!( - provider - .insert_block(block.clone().try_recover().unwrap(), StorageLocation::Database), - Ok(_) - ); + assert_matches!(provider.insert_block(block.clone().try_recover().unwrap()), Ok(_)); let senders = provider.take::(range.clone()); assert_eq!( @@ -809,7 +722,7 @@ mod tests { let static_file_provider = provider.static_file_provider(); let mut static_file_writer = static_file_provider.latest_writer(StaticFileSegment::Headers).unwrap(); - static_file_writer.append_header(head.header(), U256::ZERO, &head.hash()).unwrap(); + static_file_writer.append_header(head.header(), &head.hash()).unwrap(); static_file_writer.commit().unwrap(); drop(static_file_writer); diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index dff8ececc95..1f0a0aa391a 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -1,5 +1,7 @@ use crate::{ - bundle_state::StorageRevertsIter, + changesets_utils::{ + storage_trie_wiped_changeset_iter, StorageRevertsIter, StorageTrieCurrentValuesIter, + }, providers::{ database::{chain::ChainStorage, metrics}, static_file::StaticFileWriter, @@ -14,38 +16,40 @@ use crate::{ DBProvider, HashingWriter, HeaderProvider, HeaderSyncGapProvider, HistoricalStateProvider, HistoricalStateProviderRef, HistoryWriter, LatestStateProvider, LatestStateProviderRef, OriginalValuesKnown, ProviderError, PruneCheckpointReader, PruneCheckpointWriter, RevertsInit, - StageCheckpointReader, StateCommitmentProvider, StateProviderBox, StateWriter, - StaticFileProviderFactory, StatsReader, StorageLocation, StorageReader, StorageTrieWriter, - TransactionVariant, TransactionsProvider, TransactionsProviderExt, TrieWriter, - WithdrawalsProvider, + StageCheckpointReader, StateProviderBox, StateWriter, StaticFileProviderFactory, StatsReader, + StorageReader, StorageTrieWriter, TransactionVariant, TransactionsProvider, + TransactionsProviderExt, TrieReader, TrieWriter, +}; +use alloy_consensus::{ + transaction::{SignerRecoverable, TransactionMeta, TxHashRef}, + BlockHeader, TxReceipt, }; -use alloy_consensus::{transaction::TransactionMeta, BlockHeader, Header, TxReceipt}; -use alloy_eips::{eip2718::Encodable2718, eip4895::Withdrawals, BlockHashOrNumber}; +use alloy_eips::BlockHashOrNumber; use alloy_primitives::{ keccak256, map::{hash_map, B256Map, HashMap, HashSet}, - Address, BlockHash, BlockNumber, TxHash, TxNumber, B256, U256, + Address, BlockHash, BlockNumber, TxHash, TxNumber, B256, }; use itertools::Itertools; use rayon::slice::ParallelSliceMut; -use reth_chainspec::{ChainInfo, ChainSpecProvider, EthChainSpec, EthereumHardforks}; +use reth_chain_state::ExecutedBlock; +use reth_chainspec::{ChainInfo, ChainSpecProvider, EthChainSpec}; use reth_db_api::{ cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO, DbDupCursorRW}, database::Database, models::{ sharded_key, storage_sharded_key::StorageShardedKey, AccountBeforeTx, BlockNumberAddress, - ShardedKey, StoredBlockBodyIndices, + BlockNumberHashedAddress, ShardedKey, StoredBlockBodyIndices, }, table::Table, tables, transaction::{DbTx, DbTxMut}, - BlockNumberList, DatabaseError, PlainAccountState, PlainStorageState, + BlockNumberList, PlainAccountState, PlainStorageState, }; use reth_execution_types::{Chain, ExecutionOutcome}; use reth_node_types::{BlockTy, BodyTy, HeaderTy, NodeTypes, ReceiptTy, TxTy}; use reth_primitives_traits::{ - Account, Block as _, BlockBody as _, Bytecode, GotExpected, NodePrimitives, RecoveredBlock, - SealedBlock, SealedHeader, SignedTransaction, StorageEntry, + Account, Block as _, BlockBody as _, Bytecode, RecoveredBlock, SealedHeader, StorageEntry, }; use reth_prune_types::{ PruneCheckpoint, PruneMode, PruneModes, PruneSegment, MINIMUM_PRUNING_DISTANCE, @@ -53,16 +57,22 @@ use reth_prune_types::{ use reth_stages_types::{StageCheckpoint, StageId}; use reth_static_file_types::StaticFileSegment; use reth_storage_api::{ - BlockBodyIndicesProvider, BlockBodyReader, NodePrimitivesProvider, OmmersProvider, - StateProvider, StorageChangeSetReader, TryIntoHistoricalStateProvider, + BlockBodyIndicesProvider, BlockBodyReader, NodePrimitivesProvider, StateProvider, + StorageChangeSetReader, TryIntoHistoricalStateProvider, }; -use reth_storage_errors::provider::{ProviderResult, RootMismatch}; +use reth_storage_errors::provider::ProviderResult; use reth_trie::{ - prefix_set::{PrefixSet, PrefixSetMut, TriePrefixSets}, - updates::{StorageTrieUpdates, TrieUpdates}, - HashedPostStateSorted, Nibbles, StateRoot, StoredNibbles, + trie_cursor::{ + InMemoryTrieCursor, InMemoryTrieCursorFactory, TrieCursor, TrieCursorFactory, + TrieCursorIter, + }, + updates::{StorageTrieUpdatesSorted, TrieUpdatesSorted}, + BranchNodeCompact, HashedPostStateSorted, Nibbles, StoredNibbles, StoredNibblesSubKey, + TrieChangeSetsEntry, +}; +use reth_trie_db::{ + DatabaseAccountTrieCursor, DatabaseStorageTrieCursor, DatabaseTrieCursorFactory, }; -use reth_trie_db::{DatabaseStateRoot, DatabaseStorageTrieCursor}; use revm_database::states::{ PlainStateReverts, PlainStorageChangeset, PlainStorageRevert, StateChangeset, }; @@ -70,8 +80,8 @@ use std::{ cmp::Ordering, collections::{BTreeMap, BTreeSet}, fmt::Debug, - ops::{Deref, DerefMut, Range, RangeBounds, RangeInclusive}, - sync::{mpsc, Arc}, + ops::{Deref, DerefMut, Not, Range, RangeBounds, RangeFrom, RangeInclusive}, + sync::Arc, }; use tracing::{debug, trace}; @@ -250,89 +260,97 @@ impl AsRef for DatabaseProvider { } impl DatabaseProvider { - /// Unwinds trie state for the given range. + /// Writes executed blocks and state to storage. + pub fn save_blocks(&self, blocks: Vec>) -> ProviderResult<()> { + if blocks.is_empty() { + debug!(target: "providers::db", "Attempted to write empty block range"); + return Ok(()) + } + + // NOTE: checked non-empty above + let first_block = blocks.first().unwrap().recovered_block(); + + let last_block = blocks.last().unwrap().recovered_block(); + let first_number = first_block.number(); + let last_block_number = last_block.number(); + + debug!(target: "providers::db", block_count = %blocks.len(), "Writing blocks and execution data to storage"); + + // TODO: Do performant / batched writes for each type of object + // instead of a loop over all blocks, + // meaning: + // * blocks + // * state + // * hashed state + // * trie updates (cannot naively extend, need helper) + // * indices (already done basically) + // Insert the blocks + for ExecutedBlock { recovered_block, execution_output, hashed_state, trie_updates } in + blocks + { + let block_number = recovered_block.number(); + self.insert_block(Arc::unwrap_or_clone(recovered_block))?; + + // Write state and changesets to the database. + // Must be written after blocks because of the receipt lookup. + self.write_state(&execution_output, OriginalValuesKnown::No)?; + + // insert hashes and intermediate merkle nodes + self.write_hashed_state(&Arc::unwrap_or_clone(hashed_state).into_sorted())?; + + // sort trie updates and insert changesets + let trie_updates_sorted = (*trie_updates).clone().into_sorted(); + self.write_trie_changesets(block_number, &trie_updates_sorted, None)?; + self.write_trie_updates_sorted(&trie_updates_sorted)?; + } + + // update history indices + self.update_history_indices(first_number..=last_block_number)?; + + // Update pipeline progress + self.update_pipeline_stages(last_block_number, false)?; + + debug!(target: "providers::db", range = ?first_number..=last_block_number, "Appended block data"); + + Ok(()) + } + + /// Unwinds trie state starting at and including the given block. /// /// This includes calculating the resulted state root and comparing it with the parent block /// state root. - pub fn unwind_trie_state_range( - &self, - range: RangeInclusive, - ) -> ProviderResult<()> { + pub fn unwind_trie_state_from(&self, from: BlockNumber) -> ProviderResult<()> { let changed_accounts = self .tx .cursor_read::()? - .walk_range(range.clone())? + .walk_range(from..)? .collect::, _>>()?; - // Unwind account hashes. Add changed accounts to account prefix set. - let hashed_addresses = self.unwind_account_hashing(changed_accounts.iter())?; - let mut account_prefix_set = PrefixSetMut::with_capacity(hashed_addresses.len()); - let mut destroyed_accounts = HashSet::default(); - for (hashed_address, account) in hashed_addresses { - account_prefix_set.insert(Nibbles::unpack(hashed_address)); - if account.is_none() { - destroyed_accounts.insert(hashed_address); - } - } + // Unwind account hashes. + self.unwind_account_hashing(changed_accounts.iter())?; // Unwind account history indices. self.unwind_account_history_indices(changed_accounts.iter())?; - let storage_range = BlockNumberAddress::range(range.clone()); + let storage_start = BlockNumberAddress((from, Address::ZERO)); let changed_storages = self .tx .cursor_read::()? - .walk_range(storage_range)? + .walk_range(storage_start..)? .collect::, _>>()?; - // Unwind storage hashes. Add changed account and storage keys to corresponding prefix - // sets. - let mut storage_prefix_sets = B256Map::::default(); - let storage_entries = self.unwind_storage_hashing(changed_storages.iter().copied())?; - for (hashed_address, hashed_slots) in storage_entries { - account_prefix_set.insert(Nibbles::unpack(hashed_address)); - let mut storage_prefix_set = PrefixSetMut::with_capacity(hashed_slots.len()); - for slot in hashed_slots { - storage_prefix_set.insert(Nibbles::unpack(slot)); - } - storage_prefix_sets.insert(hashed_address, storage_prefix_set.freeze()); - } + // Unwind storage hashes. + self.unwind_storage_hashing(changed_storages.iter().copied())?; // Unwind storage history indices. self.unwind_storage_history_indices(changed_storages.iter().copied())?; - // Calculate the reverted merkle root. - // This is the same as `StateRoot::incremental_root_with_updates`, only the prefix sets - // are pre-loaded. - let prefix_sets = TriePrefixSets { - account_prefix_set: account_prefix_set.freeze(), - storage_prefix_sets, - destroyed_accounts, - }; - let (new_state_root, trie_updates) = StateRoot::from_tx(&self.tx) - .with_prefix_sets(prefix_sets) - .root_with_updates() - .map_err(reth_db_api::DatabaseError::from)?; - - let parent_number = range.start().saturating_sub(1); - let parent_state_root = self - .header_by_number(parent_number)? - .ok_or_else(|| ProviderError::HeaderNotFound(parent_number.into()))? - .state_root(); - - // state root should be always correct as we are reverting state. - // but for sake of double verification we will check it again. - if new_state_root != parent_state_root { - let parent_hash = self - .block_hash(parent_number)? - .ok_or_else(|| ProviderError::HeaderNotFound(parent_number.into()))?; - return Err(ProviderError::UnwindStateRootMismatch(Box::new(RootMismatch { - root: GotExpected { got: new_state_root, expected: parent_state_root }, - block_number: parent_number, - block_hash: parent_hash, - }))) - } - self.write_trie_updates(&trie_updates)?; + // Unwind accounts/storages trie tables using the revert. + let trie_revert = self.trie_reverts(from)?; + self.write_trie_updates_sorted(&trie_revert)?; + + // Clear trie changesets which have been unwound. + self.clear_trie_changesets_from(from)?; Ok(()) } @@ -342,14 +360,11 @@ impl DatabaseProvider ProviderResult<()> { - if remove_from.database() { - // iterate over block body and remove receipts - self.remove::>>(from_tx..)?; - } + // iterate over block body and remove receipts + self.remove::>>(from_tx..)?; - if remove_from.static_files() && !self.prune_modes.has_receipts_pruning() { + if !self.prune_modes.has_receipts_pruning() { let static_file_receipt_num = self.static_file_provider.get_highest_static_file_tx(StaticFileSegment::Receipts); @@ -408,60 +423,20 @@ impl TryIntoHistoricalStateProvider for Databa } } -impl StateCommitmentProvider for DatabaseProvider { - type StateCommitment = N::StateCommitment; -} - -impl< - Tx: DbTx + DbTxMut + 'static, - N: NodeTypesForProvider>, - > DatabaseProvider -{ - // TODO: uncomment below, once `reth debug_cmd` has been feature gated with dev. - // #[cfg(any(test, feature = "test-utils"))] - /// Inserts an historical block. **Used for setting up test environments** - pub fn insert_historical_block( - &self, - block: RecoveredBlock<::Block>, - ) -> ProviderResult { - let ttd = if block.number() == 0 { - block.header().difficulty() - } else { - let parent_block_number = block.number() - 1; - let parent_ttd = self.header_td_by_number(parent_block_number)?.unwrap_or_default(); - parent_ttd + block.header().difficulty() - }; - - let mut writer = self.static_file_provider.latest_writer(StaticFileSegment::Headers)?; - - // Backfill: some tests start at a forward block number, but static files require no gaps. - let segment_header = writer.user_header(); - if segment_header.block_end().is_none() && segment_header.expected_block_start() == 0 { - for block_number in 0..block.number() { - let mut prev = block.clone_header(); - prev.number = block_number; - writer.append_header(&prev, U256::ZERO, &B256::ZERO)?; - } - } - - writer.append_header(block.header(), ttd, &block.hash())?; - - self.insert_block(block, StorageLocation::Database) - } -} - -/// For a given key, unwind all history shards that are below the given block number. +/// For a given key, unwind all history shards that contain block numbers at or above the given +/// block number. /// /// S - Sharded key subtype. /// T - Table to walk over. /// C - Cursor implementation. /// /// This function walks the entries from the given start key and deletes all shards that belong to -/// the key and are below the given block number. +/// the key and contain block numbers at or above the given block number. Shards entirely below +/// the block number are preserved. /// -/// The boundary shard (the shard is split by the block number) is removed from the database. Any -/// indices that are above the block number are filtered out. The boundary shard is returned for -/// reinsertion (if it's not empty). +/// The boundary shard (the shard that spans across the block number) is removed from the database. +/// Any indices that are below the block number are filtered out and returned for reinsertion. +/// The boundary shard is returned for reinsertion (if it's not empty). fn unwind_history_shards( cursor: &mut C, start_key: T::Key, @@ -473,27 +448,41 @@ where T::Key: AsRef>, C: DbCursorRO + DbCursorRW, { + // Start from the given key and iterate through shards let mut item = cursor.seek_exact(start_key)?; while let Some((sharded_key, list)) = item { // If the shard does not belong to the key, break. if !shard_belongs_to_key(&sharded_key) { break } + + // Always delete the current shard from the database first + // We'll decide later what (if anything) to reinsert cursor.delete_current()?; - // Check the first item. - // If it is greater or eq to the block number, delete it. + // Get the first (lowest) block number in this shard + // All block numbers in a shard are sorted in ascending order let first = list.iter().next().expect("List can't be empty"); + + // Case 1: Entire shard is at or above the unwinding point + // Keep it deleted (don't return anything for reinsertion) if first >= block_number { item = cursor.prev()?; continue - } else if block_number <= sharded_key.as_ref().highest_block_number { - // Filter out all elements greater than block number. + } + // Case 2: This is a boundary shard (spans across the unwinding point) + // The shard contains some blocks below and some at/above the unwinding point + else if block_number <= sharded_key.as_ref().highest_block_number { + // Return only the block numbers that are below the unwinding point + // These will be reinserted to preserve the historical data return Ok(list.iter().take_while(|i| *i < block_number).collect::>()) } + // Case 3: Entire shard is below the unwinding point + // Return all block numbers for reinsertion (preserve entire shard) return Ok(list.iter().collect::>()) } + // No shards found or all processed Ok(Vec::new()) } @@ -531,23 +520,6 @@ impl DatabaseProvider { } impl DatabaseProvider { - fn transactions_by_tx_range_with_cursor( - &self, - range: impl RangeBounds, - cursor: &mut C, - ) -> ProviderResult>> - where - C: DbCursorRO>>, - { - self.static_file_provider.get_range_with_static_file_or_database( - StaticFileSegment::Transactions, - to_range(range), - |static_file, range, _| static_file.transactions_by_tx_range(range), - |range, _| self.cursor_collect(cursor, range), - |_| true, - ) - } - fn recovered_block( &self, id: BlockHashOrNumber, @@ -617,7 +589,6 @@ impl DatabaseProvider { let mut blocks = Vec::with_capacity(len); let headers = headers_range(range.clone())?; - let mut tx_cursor = self.tx.cursor_read::>>()?; // If the body indices are not found, this means that the transactions either do // not exist in the database yet, or they do exit but are @@ -636,7 +607,7 @@ impl DatabaseProvider { let transactions = if tx_range.is_empty() { Vec::new() } else { - self.transactions_by_tx_range_with_cursor(tx_range.clone(), &mut tx_cursor)? + self.transactions_by_tx_range(tx_range.clone())? }; inputs.push((header.as_ref(), transactions)); @@ -786,11 +757,6 @@ impl DatabaseProvider { } impl DatabaseProvider { - /// Commit database transaction. - pub fn commit(self) -> ProviderResult { - Ok(self.tx.commit()?) - } - /// Load shard and remove it. If list is empty, last shard was full or /// there are no shards at all. fn take_shard( @@ -930,6 +896,19 @@ impl ChangeSetReader for DatabaseProvider { }) .collect() } + + fn get_account_before_block( + &self, + block_number: BlockNumber, + address: Address, + ) -> ProviderResult> { + self.tx + .cursor_dup_read::()? + .seek_by_key_subkey(block_number, address)? + .filter(|acc| acc.address == address) + .map(Ok) + .transpose() + } } impl HeaderSyncGapProvider @@ -980,8 +959,8 @@ impl HeaderSyncGapProvider impl HeaderProvider for DatabaseProvider { type Header = HeaderTy; - fn header(&self, block_hash: &BlockHash) -> ProviderResult> { - if let Some(num) = self.block_number(*block_hash)? { + fn header(&self, block_hash: BlockHash) -> ProviderResult> { + if let Some(num) = self.block_number(block_hash)? { Ok(self.header_by_number(num)?) } else { Ok(None) @@ -989,71 +968,21 @@ impl HeaderProvider for DatabasePro } fn header_by_number(&self, num: BlockNumber) -> ProviderResult> { - self.static_file_provider.get_with_static_file_or_database( - StaticFileSegment::Headers, - num, - |static_file| static_file.header_by_number(num), - || Ok(self.tx.get::>(num)?), - ) - } - - fn header_td(&self, block_hash: &BlockHash) -> ProviderResult> { - if let Some(num) = self.block_number(*block_hash)? { - self.header_td_by_number(num) - } else { - Ok(None) - } - } - - fn header_td_by_number(&self, number: BlockNumber) -> ProviderResult> { - if self.chain_spec.is_paris_active_at_block(number) { - if let Some(td) = self.chain_spec.final_paris_total_difficulty() { - // if this block is higher than the final paris(merge) block, return the final paris - // difficulty - return Ok(Some(td)) - } - } - - self.static_file_provider.get_with_static_file_or_database( - StaticFileSegment::Headers, - number, - |static_file| static_file.header_td_by_number(number), - || Ok(self.tx.get::(number)?.map(|td| td.0)), - ) + self.static_file_provider.header_by_number(num) } fn headers_range( &self, range: impl RangeBounds, ) -> ProviderResult> { - self.static_file_provider.get_range_with_static_file_or_database( - StaticFileSegment::Headers, - to_range(range), - |static_file, range, _| static_file.headers_range(range), - |range, _| self.cursor_read_collect::>(range), - |_| true, - ) + self.static_file_provider.headers_range(range) } fn sealed_header( &self, number: BlockNumber, ) -> ProviderResult>> { - self.static_file_provider.get_with_static_file_or_database( - StaticFileSegment::Headers, - number, - |static_file| static_file.sealed_header(number), - || { - if let Some(header) = self.header_by_number(number)? { - let hash = self - .block_hash(number)? - .ok_or_else(|| ProviderError::HeaderNotFound(number.into()))?; - Ok(Some(SealedHeader::new(header, hash))) - } else { - Ok(None) - } - }, - ) + self.static_file_provider.sealed_header(number) } fn sealed_headers_while( @@ -1061,40 +990,13 @@ impl HeaderProvider for DatabasePro range: impl RangeBounds, predicate: impl FnMut(&SealedHeader) -> bool, ) -> ProviderResult>> { - self.static_file_provider.get_range_with_static_file_or_database( - StaticFileSegment::Headers, - to_range(range), - |static_file, range, predicate| static_file.sealed_headers_while(range, predicate), - |range, mut predicate| { - let mut headers = vec![]; - for entry in - self.tx.cursor_read::>()?.walk_range(range)? - { - let (number, header) = entry?; - let hash = self - .block_hash(number)? - .ok_or_else(|| ProviderError::HeaderNotFound(number.into()))?; - let sealed = SealedHeader::new(header, hash); - if !predicate(&sealed) { - break - } - headers.push(sealed); - } - Ok(headers) - }, - predicate, - ) + self.static_file_provider.sealed_headers_while(range, predicate) } } impl BlockHashReader for DatabaseProvider { fn block_hash(&self, number: u64) -> ProviderResult> { - self.static_file_provider.get_with_static_file_or_database( - StaticFileSegment::Headers, - number, - |static_file| static_file.block_hash(number), - || Ok(self.tx.get::(number)?), - ) + self.static_file_provider.block_hash(number) } fn canonical_hashes_range( @@ -1102,13 +1004,7 @@ impl BlockHashReader for DatabaseProvider ProviderResult> { - self.static_file_provider.get_range_with_static_file_or_database( - StaticFileSegment::Headers, - start..end, - |static_file, range, _| static_file.canonical_hashes_range(range.start, range.end), - |range, _| self.cursor_read_collect::(range), - |_| true, - ) + self.static_file_provider.canonical_hashes_range(start, end) } } @@ -1129,15 +1025,7 @@ impl BlockNumReader for DatabaseProvider ProviderResult { - Ok(self - .tx - .cursor_read::()? - .last()? - .map(|(num, _)| num) - .max( - self.static_file_provider.get_highest_static_file_block(StaticFileSegment::Headers), - ) - .unwrap_or_default()) + self.static_file_provider.last_block_number() } fn block_number(&self, hash: B256) -> ProviderResult> { @@ -1166,41 +1054,37 @@ impl BlockReader for DatabaseProvid /// If the header is found, but the transactions either do not exist, or are not indexed, this /// will return None. fn block(&self, id: BlockHashOrNumber) -> ProviderResult> { - if let Some(number) = self.convert_hash_or_number(id)? { - if let Some(header) = self.header_by_number(number)? { - // If the body indices are not found, this means that the transactions either do not - // exist in the database yet, or they do exit but are not indexed. - // If they exist but are not indexed, we don't have enough - // information to return the block anyways, so we return `None`. - let Some(transactions) = self.transactions_by_block(number.into())? else { - return Ok(None) - }; + if let Some(number) = self.convert_hash_or_number(id)? && + let Some(header) = self.header_by_number(number)? + { + // If the body indices are not found, this means that the transactions either do not + // exist in the database yet, or they do exit but are not indexed. + // If they exist but are not indexed, we don't have enough + // information to return the block anyways, so we return `None`. + let Some(transactions) = self.transactions_by_block(number.into())? else { + return Ok(None) + }; - let body = self - .storage - .reader() - .read_block_bodies(self, vec![(&header, transactions)])? - .pop() - .ok_or(ProviderError::InvalidStorageOutput)?; + let body = self + .storage + .reader() + .read_block_bodies(self, vec![(&header, transactions)])? + .pop() + .ok_or(ProviderError::InvalidStorageOutput)?; - return Ok(Some(Self::Block::new(header, body))) - } + return Ok(Some(Self::Block::new(header, body))) } Ok(None) } - fn pending_block(&self) -> ProviderResult>> { - Ok(None) - } - - fn pending_block_with_senders(&self) -> ProviderResult>> { + fn pending_block(&self) -> ProviderResult>> { Ok(None) } fn pending_block_and_receipts( &self, - ) -> ProviderResult, Vec)>> { + ) -> ProviderResult, Vec)>> { Ok(None) } @@ -1291,6 +1175,14 @@ impl BlockReader for DatabaseProvid }, ) } + + fn block_by_transaction_id(&self, id: TxNumber) -> ProviderResult> { + Ok(self + .tx + .cursor_read::()? + .seek(id) + .map(|b| b.map(|(_, bn)| bn))?) + } } impl TransactionsProviderExt @@ -1302,66 +1194,7 @@ impl TransactionsProviderExt &self, tx_range: Range, ) -> ProviderResult> { - self.static_file_provider.get_range_with_static_file_or_database( - StaticFileSegment::Transactions, - tx_range, - |static_file, range, _| static_file.transaction_hashes_by_range(range), - |tx_range, _| { - let mut tx_cursor = self.tx.cursor_read::>>()?; - let tx_range_size = tx_range.clone().count(); - let tx_walker = tx_cursor.walk_range(tx_range)?; - - let chunk_size = (tx_range_size / rayon::current_num_threads()).max(1); - let mut channels = Vec::with_capacity(chunk_size); - let mut transaction_count = 0; - - #[inline] - fn calculate_hash( - entry: Result<(TxNumber, T), DatabaseError>, - rlp_buf: &mut Vec, - ) -> Result<(B256, TxNumber), Box> - where - T: Encodable2718, - { - let (tx_id, tx) = entry.map_err(|e| Box::new(e.into()))?; - tx.encode_2718(rlp_buf); - Ok((keccak256(rlp_buf), tx_id)) - } - - for chunk in &tx_walker.chunks(chunk_size) { - let (tx, rx) = mpsc::channel(); - channels.push(rx); - - // Note: Unfortunate side-effect of how chunk is designed in itertools (it is - // not Send) - let chunk: Vec<_> = chunk.collect(); - transaction_count += chunk.len(); - - // Spawn the task onto the global rayon pool - // This task will send the results through the channel after it has calculated - // the hash. - rayon::spawn(move || { - let mut rlp_buf = Vec::with_capacity(128); - for entry in chunk { - rlp_buf.clear(); - let _ = tx.send(calculate_hash(entry, &mut rlp_buf)); - } - }); - } - let mut tx_list = Vec::with_capacity(transaction_count); - - // Iterate over channels and append the tx hashes unsorted - for channel in channels { - while let Ok(tx) = channel.recv() { - let (tx_hash, tx_id) = tx.map_err(|boxed| *boxed)?; - tx_list.push((tx_hash, tx_id)); - } - } - - Ok(tx_list) - }, - |_| true, - ) + self.static_file_provider.transaction_hashes_by_range(tx_range) } } @@ -1374,24 +1207,14 @@ impl TransactionsProvider for Datab } fn transaction_by_id(&self, id: TxNumber) -> ProviderResult> { - self.static_file_provider.get_with_static_file_or_database( - StaticFileSegment::Transactions, - id, - |static_file| static_file.transaction_by_id(id), - || Ok(self.tx.get::>(id)?), - ) + self.static_file_provider.transaction_by_id(id) } fn transaction_by_id_unhashed( &self, id: TxNumber, ) -> ProviderResult> { - self.static_file_provider.get_with_static_file_or_database( - StaticFileSegment::Transactions, - id, - |static_file| static_file.transaction_by_id_unhashed(id), - || Ok(self.tx.get::>(id)?), - ) + self.static_file_provider.transaction_by_id_unhashed(id) } fn transaction_by_hash(&self, hash: TxHash) -> ProviderResult> { @@ -1406,35 +1229,30 @@ impl TransactionsProvider for Datab &self, tx_hash: TxHash, ) -> ProviderResult> { - let mut transaction_cursor = self.tx.cursor_read::()?; - if let Some(transaction_id) = self.transaction_id(tx_hash)? { - if let Some(transaction) = self.transaction_by_id_unhashed(transaction_id)? { - if let Some(block_number) = - transaction_cursor.seek(transaction_id).map(|b| b.map(|(_, bn)| bn))? - { - if let Some(sealed_header) = self.sealed_header(block_number)? { - let (header, block_hash) = sealed_header.split(); - if let Some(block_body) = self.block_body_indices(block_number)? { - // the index of the tx in the block is the offset: - // len([start..tx_id]) - // NOTE: `transaction_id` is always `>=` the block's first - // index - let index = transaction_id - block_body.first_tx_num(); - - let meta = TransactionMeta { - tx_hash, - index, - block_hash, - block_number, - base_fee: header.base_fee_per_gas(), - excess_blob_gas: header.excess_blob_gas(), - timestamp: header.timestamp(), - }; - - return Ok(Some((transaction, meta))) - } - } - } + if let Some(transaction_id) = self.transaction_id(tx_hash)? && + let Some(transaction) = self.transaction_by_id_unhashed(transaction_id)? && + let Some(block_number) = self.block_by_transaction_id(transaction_id)? && + let Some(sealed_header) = self.sealed_header(block_number)? + { + let (header, block_hash) = sealed_header.split(); + if let Some(block_body) = self.block_body_indices(block_number)? { + // the index of the tx in the block is the offset: + // len([start..tx_id]) + // NOTE: `transaction_id` is always `>=` the block's first + // index + let index = transaction_id - block_body.first_tx_num(); + + let meta = TransactionMeta { + tx_hash, + index, + block_hash, + block_number, + base_fee: header.base_fee_per_gas(), + excess_blob_gas: header.excess_blob_gas(), + timestamp: header.timestamp(), + }; + + return Ok(Some((transaction, meta))) } } @@ -1450,16 +1268,14 @@ impl TransactionsProvider for Datab &self, id: BlockHashOrNumber, ) -> ProviderResult>> { - let mut tx_cursor = self.tx.cursor_read::>()?; - - if let Some(block_number) = self.convert_hash_or_number(id)? { - if let Some(body) = self.block_body_indices(block_number)? { - let tx_range = body.tx_num_range(); - return if tx_range.is_empty() { - Ok(Some(Vec::new())) - } else { - Ok(Some(self.transactions_by_tx_range_with_cursor(tx_range, &mut tx_cursor)?)) - } + if let Some(block_number) = self.convert_hash_or_number(id)? && + let Some(body) = self.block_body_indices(block_number)? + { + let tx_range = body.tx_num_range(); + return if tx_range.is_empty() { + Ok(Some(Vec::new())) + } else { + self.transactions_by_tx_range(tx_range).map(Some) } } Ok(None) @@ -1470,7 +1286,6 @@ impl TransactionsProvider for Datab range: impl RangeBounds, ) -> ProviderResult>> { let range = to_range(range); - let mut tx_cursor = self.tx.cursor_read::>()?; self.block_body_indices_range(range.start..=range.end.saturating_sub(1))? .into_iter() @@ -1479,10 +1294,7 @@ impl TransactionsProvider for Datab if tx_num_range.is_empty() { Ok(Vec::new()) } else { - Ok(self - .transactions_by_tx_range_with_cursor(tx_num_range, &mut tx_cursor)? - .into_iter() - .collect()) + self.transactions_by_tx_range(tx_num_range) } }) .collect() @@ -1492,10 +1304,7 @@ impl TransactionsProvider for Datab &self, range: impl RangeBounds, ) -> ProviderResult> { - self.transactions_by_tx_range_with_cursor( - range, - &mut self.tx.cursor_read::>()?, - ) + self.static_file_provider.transactions_by_tx_range(range) } fn senders_by_tx_range( @@ -1534,14 +1343,14 @@ impl ReceiptProvider for DatabasePr &self, block: BlockHashOrNumber, ) -> ProviderResult>> { - if let Some(number) = self.convert_hash_or_number(block)? { - if let Some(body) = self.block_body_indices(number)? { - let tx_range = body.tx_num_range(); - return if tx_range.is_empty() { - Ok(Some(Vec::new())) - } else { - self.receipts_by_tx_range(tx_range).map(Some) - } + if let Some(number) = self.convert_hash_or_number(block)? && + let Some(body) = self.block_body_indices(number)? + { + let tx_range = body.tx_num_range(); + return if tx_range.is_empty() { + Ok(Some(Vec::new())) + } else { + self.receipts_by_tx_range(tx_range).map(Some) } } Ok(None) @@ -1559,61 +1368,60 @@ impl ReceiptProvider for DatabasePr |_| true, ) } -} -impl> WithdrawalsProvider - for DatabaseProvider -{ - fn withdrawals_by_block( + fn receipts_by_block_range( &self, - id: BlockHashOrNumber, - timestamp: u64, - ) -> ProviderResult> { - if self.chain_spec.is_shanghai_active_at_timestamp(timestamp) { - if let Some(number) = self.convert_hash_or_number(id)? { - return self.static_file_provider.get_with_static_file_or_database( - StaticFileSegment::BlockMeta, - number, - |static_file| static_file.withdrawals_by_block(number.into(), timestamp), - || { - // If we are past shanghai, then all blocks should have a withdrawal list, - // even if empty - let withdrawals = self - .tx - .get::(number) - .map(|w| w.map(|w| w.withdrawals))? - .unwrap_or_default(); - Ok(Some(withdrawals)) - }, - ) - } + block_range: RangeInclusive, + ) -> ProviderResult>> { + if block_range.is_empty() { + return Ok(Vec::new()); } - Ok(None) - } -} -impl OmmersProvider for DatabaseProvider { - /// Returns the ommers for the block with matching id from the database. - /// - /// If the block is not found, this returns `None`. - /// If the block exists, but doesn't contain ommers, this returns `None`. - fn ommers(&self, id: BlockHashOrNumber) -> ProviderResult>> { - if let Some(number) = self.convert_hash_or_number(id)? { - // If the Paris (Merge) hardfork block is known and block is after it, return empty - // ommers. - if self.chain_spec.is_paris_active_at_block(number) { - return Ok(Some(Vec::new())) + // collect block body indices for each block in the range + let mut block_body_indices = Vec::new(); + for block_num in block_range { + if let Some(indices) = self.block_body_indices(block_num)? { + block_body_indices.push(indices); + } else { + // use default indices for missing blocks (empty block) + block_body_indices.push(StoredBlockBodyIndices::default()); } + } - return self.static_file_provider.get_with_static_file_or_database( - StaticFileSegment::BlockMeta, - number, - |static_file| static_file.ommers(id), - || Ok(self.tx.get::>(number)?.map(|o| o.ommers)), - ) + if block_body_indices.is_empty() { + return Ok(Vec::new()); } - Ok(None) + // find blocks with transactions to determine transaction range + let non_empty_blocks: Vec<_> = + block_body_indices.iter().filter(|indices| indices.tx_count > 0).collect(); + + if non_empty_blocks.is_empty() { + // all blocks are empty + return Ok(vec![Vec::new(); block_body_indices.len()]); + } + + // calculate the overall transaction range + let first_tx = non_empty_blocks[0].first_tx_num(); + let last_tx = non_empty_blocks[non_empty_blocks.len() - 1].last_tx_num(); + + // fetch all receipts in the transaction range + let all_receipts = self.receipts_by_tx_range(first_tx..=last_tx)?; + let mut receipts_iter = all_receipts.into_iter(); + + // distribute receipts to their respective blocks + let mut result = Vec::with_capacity(block_body_indices.len()); + for indices in &block_body_indices { + if indices.tx_count == 0 { + result.push(Vec::new()); + } else { + let block_receipts = + receipts_iter.by_ref().take(indices.tx_count as usize).collect(); + result.push(block_receipts); + } + } + + Ok(result) } } @@ -1621,33 +1429,24 @@ impl BlockBodyIndicesProvider for DatabaseProvider { fn block_body_indices(&self, num: u64) -> ProviderResult> { - self.static_file_provider.get_with_static_file_or_database( - StaticFileSegment::BlockMeta, - num, - |static_file| static_file.block_body_indices(num), - || Ok(self.tx.get::(num)?), - ) + Ok(self.tx.get::(num)?) } fn block_body_indices_range( &self, range: RangeInclusive, ) -> ProviderResult> { - self.static_file_provider.get_range_with_static_file_or_database( - StaticFileSegment::BlockMeta, - *range.start()..*range.end() + 1, - |static_file, range, _| { - static_file.block_body_indices_range(range.start..=range.end.saturating_sub(1)) - }, - |range, _| self.cursor_read_collect::(range), - |_| true, - ) + self.cursor_read_collect::(range) } } impl StageCheckpointReader for DatabaseProvider { fn get_stage_checkpoint(&self, id: StageId) -> ProviderResult> { - Ok(self.tx.get::(id.to_string())?) + Ok(if let Some(encoded) = id.get_pre_encoded() { + self.tx.get_by_encoded_key::(encoded)? + } else { + self.tx.get::(id.to_string())? + }) } /// Get stage checkpoint progress. @@ -1777,7 +1576,6 @@ impl StateWriter &self, execution_outcome: &ExecutionOutcome, is_value_known: OriginalValuesKnown, - write_receipts_to: StorageLocation, ) -> ProviderResult<()> { let first_block = execution_outcome.first_block(); let block_count = execution_outcome.len() as u64; @@ -1813,15 +1611,13 @@ impl StateWriter // // We are writing to database if requested or if there's any kind of receipt pruning // configured - let mut receipts_cursor = (write_receipts_to.database() || has_receipts_pruning) - .then(|| self.tx.cursor_write::>()) - .transpose()?; + let mut receipts_cursor = self.tx.cursor_write::>()?; // Prepare receipts static writer if we are going to write receipts to static files // // We are writing to static files if requested and if there's no receipt pruning configured - let mut receipts_static_writer = (write_receipts_to.static_files() && - !has_receipts_pruning) + let mut receipts_static_writer = has_receipts_pruning + .not() .then(|| self.static_file_provider.get_writer(first_block, StaticFileSegment::Receipts)) .transpose()?; @@ -1876,10 +1672,8 @@ impl StateWriter if let Some(writer) = &mut receipts_static_writer { writer.append_receipt(receipt_idx, receipt)?; - } - - if let Some(cursor) = &mut receipts_cursor { - cursor.append(receipt_idx, receipt)?; + } else { + receipts_cursor.append(receipt_idx, receipt)?; } } } @@ -1916,6 +1710,10 @@ impl StateWriter // If we are writing the primary storage wipe transition, the pre-existing plain // storage state has to be taken from the database and written to storage history. // See [StorageWipe::Primary] for more details. + // + // TODO(mediocregopher): This could be rewritten in a way which doesn't require + // collecting wiped entries into a Vec like this, see + // `write_storage_trie_changesets`. let mut wiped_storage = Vec::new(); if wiped { tracing::trace!(?address, "Wiping storage"); @@ -2001,10 +1799,10 @@ impl StateWriter for entry in storage { tracing::trace!(?address, ?entry.key, "Updating plain state storage"); - if let Some(db_entry) = storages_cursor.seek_by_key_subkey(address, entry.key)? { - if db_entry.key == entry.key { - storages_cursor.delete_current()?; - } + if let Some(db_entry) = storages_cursor.seek_by_key_subkey(address, entry.key)? && + db_entry.key == entry.key + { + storages_cursor.delete_current()?; } if !entry.value.is_zero() { @@ -2039,11 +1837,10 @@ impl StateWriter for (hashed_slot, value) in storage.storage_slots_sorted() { let entry = StorageEntry { key: hashed_slot, value }; if let Some(db_entry) = - hashed_storage_cursor.seek_by_key_subkey(*hashed_address, entry.key)? + hashed_storage_cursor.seek_by_key_subkey(*hashed_address, entry.key)? && + db_entry.key == entry.key { - if db_entry.key == entry.key { - hashed_storage_cursor.delete_current()?; - } + hashed_storage_cursor.delete_current()?; } if !entry.value.is_zero() { @@ -2076,11 +1873,7 @@ impl StateWriter /// 1. Take the old value from the changeset /// 2. Take the new value from the local state /// 3. Set the local state to the value in the changeset - fn remove_state_above( - &self, - block: BlockNumber, - remove_receipts_from: StorageLocation, - ) -> ProviderResult<()> { + fn remove_state_above(&self, block: BlockNumber) -> ProviderResult<()> { let range = block + 1..=self.last_block_number()?; if range.is_empty() { @@ -2129,7 +1922,6 @@ impl StateWriter for (storage_key, (old_storage_value, _new_storage_value)) in storage { let storage_entry = StorageEntry { key: *storage_key, value: *old_storage_value }; // delete previous value - // TODO: This does not use dupsort features if plain_storage_cursor .seek_by_key_subkey(*address, *storage_key)? .filter(|s| s.key == *storage_key) @@ -2145,7 +1937,7 @@ impl StateWriter } } - self.remove_receipts_from(from_transaction_num, block, remove_receipts_from)?; + self.remove_receipts_from(from_transaction_num, block)?; Ok(()) } @@ -2174,7 +1966,6 @@ impl StateWriter fn take_state_above( &self, block: BlockNumber, - remove_receipts_from: StorageLocation, ) -> ProviderResult> { let range = block + 1..=self.last_block_number()?; @@ -2229,7 +2020,6 @@ impl StateWriter for (storage_key, (old_storage_value, _new_storage_value)) in storage { let storage_entry = StorageEntry { key: *storage_key, value: *old_storage_value }; // delete previous value - // TODO: This does not use dupsort features if plain_storage_cursor .seek_by_key_subkey(*address, *storage_key)? .filter(|s| s.key == *storage_key) @@ -2280,7 +2070,7 @@ impl StateWriter receipts.push(block_receipts); } - self.remove_receipts_from(from_transaction_num, block, remove_receipts_from)?; + self.remove_receipts_from(from_transaction_num, block)?; Ok(ExecutionOutcome::new_init( state, @@ -2294,8 +2084,10 @@ impl StateWriter } impl TrieWriter for DatabaseProvider { - /// Writes trie updates. Returns the number of entries modified. - fn write_trie_updates(&self, trie_updates: &TrieUpdates) -> ProviderResult { + /// Writes trie updates to the database with already sorted updates. + /// + /// Returns the number of entries modified. + fn write_trie_updates_sorted(&self, trie_updates: &TrieUpdatesSorted) -> ProviderResult { if trie_updates.is_empty() { return Ok(0) } @@ -2303,24 +2095,12 @@ impl TrieWriter for DatabaseProvider // Track the number of inserted entries. let mut num_entries = 0; - // Merge updated and removed nodes. Updated nodes must take precedence. - let mut account_updates = trie_updates - .removed_nodes_ref() - .iter() - .filter_map(|n| { - (!trie_updates.account_nodes_ref().contains_key(n)).then_some((n, None)) - }) - .collect::>(); - account_updates.extend( - trie_updates.account_nodes_ref().iter().map(|(nibbles, node)| (nibbles, Some(node))), - ); - // Sort trie node updates. - account_updates.sort_unstable_by(|a, b| a.0.cmp(b.0)); - let tx = self.tx_ref(); let mut account_trie_cursor = tx.cursor_write::()?; - for (key, updated_node) in account_updates { - let nibbles = StoredNibbles(key.clone()); + + // Process sorted account nodes + for (key, updated_node) in trie_updates.account_nodes_ref() { + let nibbles = StoredNibbles(*key); match updated_node { Some(node) => { if !nibbles.0.is_empty() { @@ -2337,50 +2117,346 @@ impl TrieWriter for DatabaseProvider } } - num_entries += self.write_storage_trie_updates(trie_updates.storage_tries_ref())?; + num_entries += + self.write_storage_trie_updates_sorted(trie_updates.storage_tries_ref().iter())?; Ok(num_entries) } -} -impl StorageTrieWriter for DatabaseProvider { - /// Writes storage trie updates from the given storage trie map. First sorts the storage trie - /// updates by the hashed address, writing in sorted order. - fn write_storage_trie_updates( + /// Records the current values of all trie nodes which will be updated using the `TrieUpdates` + /// into the trie changesets tables. + /// + /// The intended usage of this method is to call it _prior_ to calling `write_trie_updates` with + /// the same `TrieUpdates`. + /// + /// Returns the number of keys written. + fn write_trie_changesets( &self, - storage_tries: &B256Map, + block_number: BlockNumber, + trie_updates: &TrieUpdatesSorted, + updates_overlay: Option<&TrieUpdatesSorted>, ) -> ProviderResult { let mut num_entries = 0; - let mut storage_tries = Vec::from_iter(storage_tries); - storage_tries.sort_unstable_by(|a, b| a.0.cmp(b.0)); - let mut cursor = self.tx_ref().cursor_dup_write::()?; - for (hashed_address, storage_trie_updates) in storage_tries { - let mut db_storage_trie_cursor = - DatabaseStorageTrieCursor::new(cursor, *hashed_address); - num_entries += - db_storage_trie_cursor.write_storage_trie_updates(storage_trie_updates)?; - cursor = db_storage_trie_cursor.cursor; + + let mut changeset_cursor = + self.tx_ref().cursor_dup_write::()?; + let curr_values_cursor = self.tx_ref().cursor_read::()?; + + // Wrap the cursor in DatabaseAccountTrieCursor + let mut db_account_cursor = DatabaseAccountTrieCursor::new(curr_values_cursor); + + // Static empty array for when updates_overlay is None + static EMPTY_ACCOUNT_UPDATES: Vec<(Nibbles, Option)> = Vec::new(); + + // Get the overlay updates for account trie, or use an empty array + let account_overlay_updates = updates_overlay + .map(|overlay| overlay.account_nodes_ref()) + .unwrap_or(&EMPTY_ACCOUNT_UPDATES); + + // Wrap the cursor in InMemoryTrieCursor with the overlay + let mut in_memory_account_cursor = + InMemoryTrieCursor::new(Some(&mut db_account_cursor), account_overlay_updates); + + for (path, _) in trie_updates.account_nodes_ref() { + num_entries += 1; + let node = in_memory_account_cursor.seek_exact(*path)?.map(|(_, node)| node); + changeset_cursor.append_dup( + block_number, + TrieChangeSetsEntry { nibbles: StoredNibblesSubKey(*path), node }, + )?; } + let mut storage_updates = trie_updates.storage_tries_ref().iter().collect::>(); + storage_updates.sort_unstable_by(|a, b| a.0.cmp(b.0)); + + num_entries += self.write_storage_trie_changesets( + block_number, + storage_updates.into_iter(), + updates_overlay, + )?; + Ok(num_entries) } - fn write_individual_storage_trie_updates( - &self, - hashed_address: B256, - updates: &StorageTrieUpdates, - ) -> ProviderResult { - if updates.is_empty() { - return Ok(0) + fn clear_trie_changesets(&self) -> ProviderResult<()> { + let tx = self.tx_ref(); + tx.clear::()?; + tx.clear::()?; + Ok(()) + } + + fn clear_trie_changesets_from(&self, from: BlockNumber) -> ProviderResult<()> { + let tx = self.tx_ref(); + { + let range = from..; + let mut cursor = tx.cursor_dup_write::()?; + let mut walker = cursor.walk_range(range)?; + + while walker.next().transpose()?.is_some() { + walker.delete_current()?; + } + } + + { + let range: RangeFrom = (from, B256::ZERO).into()..; + let mut cursor = tx.cursor_dup_write::()?; + let mut walker = cursor.walk_range(range)?; + + while walker.next().transpose()?.is_some() { + walker.delete_current()?; + } } - let cursor = self.tx_ref().cursor_dup_write::()?; - let mut trie_db_cursor = DatabaseStorageTrieCursor::new(cursor, hashed_address); - Ok(trie_db_cursor.write_storage_trie_updates(updates)?) + Ok(()) } } -impl HashingWriter for DatabaseProvider { +impl TrieReader for DatabaseProvider { + fn trie_reverts(&self, from: BlockNumber) -> ProviderResult { + let tx = self.tx_ref(); + + // Read account trie changes directly into a Vec - data is already sorted by nibbles + // within each block, and we want the oldest (first) version of each node sorted by path. + let mut account_nodes = Vec::new(); + let mut seen_account_keys = HashSet::new(); + let mut accounts_cursor = tx.cursor_dup_read::()?; + + for entry in accounts_cursor.walk_range(from..)? { + let (_, TrieChangeSetsEntry { nibbles, node }) = entry?; + // Only keep the first (oldest) version of each node + if seen_account_keys.insert(nibbles.0) { + account_nodes.push((nibbles.0, node)); + } + } + + account_nodes.sort_by_key(|(path, _)| *path); + + // Read storage trie changes - data is sorted by (block, hashed_address, nibbles) + // Keep track of seen (address, nibbles) pairs to only keep the oldest version per address, + // sorted by path. + let mut storage_tries = B256Map::>::default(); + let mut seen_storage_keys = HashSet::new(); + let mut storages_cursor = tx.cursor_dup_read::()?; + + // Create storage range starting from `from` block + let storage_range_start = BlockNumberHashedAddress((from, B256::ZERO)); + + for entry in storages_cursor.walk_range(storage_range_start..)? { + let ( + BlockNumberHashedAddress((_, hashed_address)), + TrieChangeSetsEntry { nibbles, node }, + ) = entry?; + + // Only keep the first (oldest) version of each node for this address + if seen_storage_keys.insert((hashed_address, nibbles.0)) { + storage_tries.entry(hashed_address).or_default().push((nibbles.0, node)); + } + } + + // Convert to StorageTrieUpdatesSorted + let storage_tries = storage_tries + .into_iter() + .map(|(address, mut nodes)| { + nodes.sort_by_key(|(path, _)| *path); + (address, StorageTrieUpdatesSorted { storage_nodes: nodes, is_deleted: false }) + }) + .collect(); + + Ok(TrieUpdatesSorted::new(account_nodes, storage_tries)) + } + + fn get_block_trie_updates( + &self, + block_number: BlockNumber, + ) -> ProviderResult { + let tx = self.tx_ref(); + + // Step 1: Get the trie reverts for the state after the target block + let reverts = self.trie_reverts(block_number + 1)?; + + // Step 2: Create an InMemoryTrieCursorFactory with the reverts + // This gives us the trie state as it was after the target block was processed + let db_cursor_factory = DatabaseTrieCursorFactory::new(tx); + let cursor_factory = InMemoryTrieCursorFactory::new(db_cursor_factory, &reverts); + + // Step 3: Collect all account trie nodes that changed in the target block + let mut account_nodes = Vec::new(); + + // Walk through all account trie changes for this block + let mut accounts_trie_cursor = tx.cursor_dup_read::()?; + let mut account_cursor = cursor_factory.account_trie_cursor()?; + + for entry in accounts_trie_cursor.walk_dup(Some(block_number), None)? { + let (_, TrieChangeSetsEntry { nibbles, .. }) = entry?; + // Look up the current value of this trie node using the overlay cursor + let node_value = account_cursor.seek_exact(nibbles.0)?.map(|(_, node)| node); + account_nodes.push((nibbles.0, node_value)); + } + + // Step 4: Collect all storage trie nodes that changed in the target block + let mut storage_tries = B256Map::default(); + let mut storages_trie_cursor = tx.cursor_dup_read::()?; + let storage_range_start = BlockNumberHashedAddress((block_number, B256::ZERO)); + let storage_range_end = BlockNumberHashedAddress((block_number + 1, B256::ZERO)); + + let mut current_hashed_address = None; + let mut storage_cursor = None; + + for entry in storages_trie_cursor.walk_range(storage_range_start..storage_range_end)? { + let ( + BlockNumberHashedAddress((_, hashed_address)), + TrieChangeSetsEntry { nibbles, .. }, + ) = entry?; + + // Check if we need to create a new storage cursor for a different account + if current_hashed_address != Some(hashed_address) { + storage_cursor = Some(cursor_factory.storage_trie_cursor(hashed_address)?); + current_hashed_address = Some(hashed_address); + } + + // Look up the current value of this storage trie node + let cursor = + storage_cursor.as_mut().expect("storage_cursor was just initialized above"); + let node_value = cursor.seek_exact(nibbles.0)?.map(|(_, node)| node); + storage_tries + .entry(hashed_address) + .or_insert_with(|| StorageTrieUpdatesSorted { + storage_nodes: Vec::new(), + is_deleted: false, + }) + .storage_nodes + .push((nibbles.0, node_value)); + } + + Ok(TrieUpdatesSorted::new(account_nodes, storage_tries)) + } +} + +impl StorageTrieWriter for DatabaseProvider { + /// Writes storage trie updates from the given storage trie map with already sorted updates. + /// + /// Expects the storage trie updates to already be sorted by the hashed address key. + /// + /// Returns the number of entries modified. + fn write_storage_trie_updates_sorted<'a>( + &self, + storage_tries: impl Iterator, + ) -> ProviderResult { + let mut num_entries = 0; + let mut storage_tries = storage_tries.collect::>(); + storage_tries.sort_unstable_by(|a, b| a.0.cmp(b.0)); + let mut cursor = self.tx_ref().cursor_dup_write::()?; + for (hashed_address, storage_trie_updates) in storage_tries { + let mut db_storage_trie_cursor = + DatabaseStorageTrieCursor::new(cursor, *hashed_address); + num_entries += + db_storage_trie_cursor.write_storage_trie_updates_sorted(storage_trie_updates)?; + cursor = db_storage_trie_cursor.cursor; + } + + Ok(num_entries) + } + + /// Records the current values of all trie nodes which will be updated using the + /// `StorageTrieUpdates` into the storage trie changesets table. + /// + /// The intended usage of this method is to call it _prior_ to calling + /// `write_storage_trie_updates` with the same set of `StorageTrieUpdates`. + /// + /// Returns the number of keys written. + fn write_storage_trie_changesets<'a>( + &self, + block_number: BlockNumber, + storage_tries: impl Iterator, + updates_overlay: Option<&TrieUpdatesSorted>, + ) -> ProviderResult { + let mut num_written = 0; + + let mut changeset_cursor = + self.tx_ref().cursor_dup_write::()?; + + // We hold two cursors to the same table because we use them simultaneously when an + // account's storage is wiped. We keep them outside the for-loop so they can be re-used + // between accounts. + let changed_curr_values_cursor = self.tx_ref().cursor_dup_read::()?; + let wiped_nodes_cursor = self.tx_ref().cursor_dup_read::()?; + + // DatabaseStorageTrieCursor requires ownership of the cursor. The easiest way to deal with + // this is to create this outer variable with an initial dummy account, and overwrite it on + // every loop for every real account. + let mut changed_curr_values_cursor = DatabaseStorageTrieCursor::new( + changed_curr_values_cursor, + B256::default(), // Will be set per iteration + ); + let mut wiped_nodes_cursor = DatabaseStorageTrieCursor::new( + wiped_nodes_cursor, + B256::default(), // Will be set per iteration + ); + + // Static empty array for when updates_overlay is None + static EMPTY_UPDATES: Vec<(Nibbles, Option)> = Vec::new(); + + for (hashed_address, storage_trie_updates) in storage_tries { + let changeset_key = BlockNumberHashedAddress((block_number, *hashed_address)); + + // Update the hashed address for the cursors + changed_curr_values_cursor = + DatabaseStorageTrieCursor::new(changed_curr_values_cursor.cursor, *hashed_address); + + // Get the overlay updates for this storage trie, or use an empty array + let overlay_updates = updates_overlay + .and_then(|overlay| overlay.storage_tries_ref().get(hashed_address)) + .map(|updates| updates.storage_nodes_ref()) + .unwrap_or(&EMPTY_UPDATES); + + // Wrap the cursor in InMemoryTrieCursor with the overlay + let mut in_memory_changed_cursor = + InMemoryTrieCursor::new(Some(&mut changed_curr_values_cursor), overlay_updates); + + // Create an iterator which produces the current values of all updated paths, or None if + // they are currently unset. + let curr_values_of_changed = StorageTrieCurrentValuesIter::new( + storage_trie_updates.storage_nodes.iter().map(|e| e.0), + &mut in_memory_changed_cursor, + )?; + + if storage_trie_updates.is_deleted() { + // Create an iterator that starts from the beginning of the storage trie for this + // account + wiped_nodes_cursor = + DatabaseStorageTrieCursor::new(wiped_nodes_cursor.cursor, *hashed_address); + + // Wrap the wiped nodes cursor in InMemoryTrieCursor with the overlay + let mut in_memory_wiped_cursor = + InMemoryTrieCursor::new(Some(&mut wiped_nodes_cursor), overlay_updates); + + let all_nodes = TrieCursorIter::new(&mut in_memory_wiped_cursor); + + for wiped in storage_trie_wiped_changeset_iter(curr_values_of_changed, all_nodes)? { + let (path, node) = wiped?; + num_written += 1; + changeset_cursor.append_dup( + changeset_key, + TrieChangeSetsEntry { nibbles: StoredNibblesSubKey(path), node }, + )?; + } + } else { + for curr_value in curr_values_of_changed { + let (path, node) = curr_value?; + num_written += 1; + changeset_cursor.append_dup( + changeset_key, + TrieChangeSetsEntry { nibbles: StoredNibblesSubKey(path), node }, + )?; + } + } + } + + Ok(num_written) + } +} + +impl HashingWriter for DatabaseProvider { fn unwind_account_hashing<'a>( &self, changesets: impl Iterator, @@ -2527,82 +2603,6 @@ impl HashingWriter for DatabaseProvi Ok(hashed_storage_keys) } - - fn insert_hashes( - &self, - range: RangeInclusive, - end_block_hash: B256, - expected_state_root: B256, - ) -> ProviderResult<()> { - // Initialize prefix sets. - let mut account_prefix_set = PrefixSetMut::default(); - let mut storage_prefix_sets: HashMap = HashMap::default(); - let mut destroyed_accounts = HashSet::default(); - - let mut durations_recorder = metrics::DurationsRecorder::default(); - - // storage hashing stage - { - let lists = self.changed_storages_with_range(range.clone())?; - let storages = self.plain_state_storages(lists)?; - let storage_entries = self.insert_storage_for_hashing(storages)?; - for (hashed_address, hashed_slots) in storage_entries { - account_prefix_set.insert(Nibbles::unpack(hashed_address)); - for slot in hashed_slots { - storage_prefix_sets - .entry(hashed_address) - .or_default() - .insert(Nibbles::unpack(slot)); - } - } - } - durations_recorder.record_relative(metrics::Action::InsertStorageHashing); - - // account hashing stage - { - let lists = self.changed_accounts_with_range(range.clone())?; - let accounts = self.basic_accounts(lists)?; - let hashed_addresses = self.insert_account_for_hashing(accounts)?; - for (hashed_address, account) in hashed_addresses { - account_prefix_set.insert(Nibbles::unpack(hashed_address)); - if account.is_none() { - destroyed_accounts.insert(hashed_address); - } - } - } - durations_recorder.record_relative(metrics::Action::InsertAccountHashing); - - // merkle tree - { - // This is the same as `StateRoot::incremental_root_with_updates`, only the prefix sets - // are pre-loaded. - let prefix_sets = TriePrefixSets { - account_prefix_set: account_prefix_set.freeze(), - storage_prefix_sets: storage_prefix_sets - .into_iter() - .map(|(k, v)| (k, v.freeze())) - .collect(), - destroyed_accounts, - }; - let (state_root, trie_updates) = StateRoot::from_tx(&self.tx) - .with_prefix_sets(prefix_sets) - .root_with_updates() - .map_err(reth_db_api::DatabaseError::from)?; - if state_root != expected_state_root { - return Err(ProviderError::StateRootMismatch(Box::new(RootMismatch { - root: GotExpected { got: state_root, expected: expected_state_root }, - block_number: *range.end(), - block_hash: end_block_hash, - }))) - } - self.write_trie_updates(&trie_updates)?; - } - durations_recorder.record_relative(metrics::Action::InsertMerkleTree); - - debug!(target: "providers::db", ?range, actions = ?durations_recorder.actions, "Inserted hashes"); - - Ok(()) - } } impl HistoryWriter for DatabaseProvider { @@ -2745,20 +2745,19 @@ impl BlockExecu fn take_block_and_execution_above( &self, block: BlockNumber, - remove_from: StorageLocation, ) -> ProviderResult> { let range = block + 1..=self.last_block_number()?; - self.unwind_trie_state_range(range.clone())?; + self.unwind_trie_state_from(block + 1)?; // get execution res - let execution_state = self.take_state_above(block, remove_from)?; + let execution_state = self.take_state_above(block)?; let blocks = self.recovered_block_range(range)?; // remove block bodies it is needed for both get block range and get block execution results // that is why it is deleted afterwards. - self.remove_blocks_above(block, remove_from)?; + self.remove_blocks_above(block)?; // Update pipeline progress self.update_pipeline_stages(block, true)?; @@ -2766,21 +2765,15 @@ impl BlockExecu Ok(Chain::new(blocks, execution_state, None)) } - fn remove_block_and_execution_above( - &self, - block: BlockNumber, - remove_from: StorageLocation, - ) -> ProviderResult<()> { - let range = block + 1..=self.last_block_number()?; - - self.unwind_trie_state_range(range)?; + fn remove_block_and_execution_above(&self, block: BlockNumber) -> ProviderResult<()> { + self.unwind_trie_state_from(block + 1)?; // remove execution res - self.remove_state_above(block, remove_from)?; + self.remove_state_above(block)?; // remove block bodies it is needed for both get block range and get block execution results // that is why it is deleted afterwards. - self.remove_blocks_above(block, remove_from)?; + self.remove_blocks_above(block)?; // Update pipeline progress self.update_pipeline_stages(block, true)?; @@ -2795,16 +2788,16 @@ impl BlockWrite type Block = BlockTy; type Receipt = ReceiptTy; - /// Inserts the block into the database, always modifying the following tables: - /// * [`CanonicalHeaders`](tables::CanonicalHeaders) - /// * [`Headers`](tables::Headers) - /// * [`HeaderNumbers`](tables::HeaderNumbers) - /// * [`HeaderTerminalDifficulties`](tables::HeaderTerminalDifficulties) - /// * [`BlockBodyIndices`](tables::BlockBodyIndices) + /// Inserts the block into the database, always modifying the following static file segments and + /// tables: + /// * [`StaticFileSegment::Headers`] + /// * [`tables::HeaderNumbers`] + /// * [`tables::BlockBodyIndices`] /// - /// If there are transactions in the block, the following tables will be modified: - /// * [`Transactions`](tables::Transactions) - /// * [`TransactionBlocks`](tables::TransactionBlocks) + /// If there are transactions in the block, the following static file segments and tables will + /// be modified: + /// * [`StaticFileSegment::Transactions`] + /// * [`tables::TransactionBlocks`] /// /// If ommers are not empty, this will modify [`BlockOmmers`](tables::BlockOmmers). /// If withdrawals are not empty, this will modify @@ -2818,39 +2811,14 @@ impl BlockWrite fn insert_block( &self, block: RecoveredBlock, - write_to: StorageLocation, ) -> ProviderResult { let block_number = block.number(); let mut durations_recorder = metrics::DurationsRecorder::default(); - // total difficulty - let ttd = if block_number == 0 { - block.header().difficulty() - } else { - let parent_block_number = block_number - 1; - let parent_ttd = self.header_td_by_number(parent_block_number)?.unwrap_or_default(); - durations_recorder.record_relative(metrics::Action::GetParentTD); - parent_ttd + block.header().difficulty() - }; - - if write_to.database() { - self.tx.put::(block_number, block.hash())?; - durations_recorder.record_relative(metrics::Action::InsertCanonicalHeaders); - - // Put header with canonical hashes. - self.tx.put::>>(block_number, block.header().clone())?; - durations_recorder.record_relative(metrics::Action::InsertHeaders); - - self.tx.put::(block_number, ttd.into())?; - durations_recorder.record_relative(metrics::Action::InsertHeaderTerminalDifficulties); - } - - if write_to.static_files() { - let mut writer = - self.static_file_provider.get_writer(block_number, StaticFileSegment::Headers)?; - writer.append_header(block.header(), ttd, &block.hash())?; - } + self.static_file_provider + .get_writer(block_number, StaticFileSegment::Headers)? + .append_header(block.header(), &block.hash())?; self.tx.put::(block.hash(), block_number)?; durations_recorder.record_relative(metrics::Action::InsertHeaderNumbers); @@ -2880,7 +2848,7 @@ impl BlockWrite next_tx_num += 1; } - self.append_block_bodies(vec![(block_number, Some(block.into_body()))], write_to)?; + self.append_block_bodies(vec![(block_number, Some(block.into_body()))])?; debug!( target: "providers::db", @@ -2895,35 +2863,22 @@ impl BlockWrite fn append_block_bodies( &self, bodies: Vec<(BlockNumber, Option>)>, - write_to: StorageLocation, ) -> ProviderResult<()> { let Some(from_block) = bodies.first().map(|(block, _)| *block) else { return Ok(()) }; // Initialize writer if we will be writing transactions to staticfiles - let mut tx_static_writer = write_to - .static_files() - .then(|| { - self.static_file_provider.get_writer(from_block, StaticFileSegment::Transactions) - }) - .transpose()?; + let mut tx_writer = + self.static_file_provider.get_writer(from_block, StaticFileSegment::Transactions)?; let mut block_indices_cursor = self.tx.cursor_write::()?; let mut tx_block_cursor = self.tx.cursor_write::()?; - // Initialize cursor if we will be writing transactions to database - let mut tx_cursor = write_to - .database() - .then(|| self.tx.cursor_write::>>()) - .transpose()?; - // Get id for the next tx_num or zero if there are no transactions. let mut next_tx_num = tx_block_cursor.last()?.map(|(id, _)| id + 1).unwrap_or_default(); for (block_number, body) in &bodies { // Increment block on static file header. - if let Some(writer) = tx_static_writer.as_mut() { - writer.increment_block(*block_number)?; - } + tx_writer.increment_block(*block_number)?; let tx_count = body.as_ref().map(|b| b.transactions().len() as u64).unwrap_or_default(); let block_indices = StoredBlockBodyIndices { first_tx_num: next_tx_num, tx_count }; @@ -2945,44 +2900,39 @@ impl BlockWrite // write transactions for transaction in body.transactions() { - if let Some(writer) = tx_static_writer.as_mut() { - writer.append_transaction(next_tx_num, transaction)?; - } - if let Some(cursor) = tx_cursor.as_mut() { - cursor.append(next_tx_num, transaction)?; - } + tx_writer.append_transaction(next_tx_num, transaction)?; // Increment transaction id for each transaction. next_tx_num += 1; } - - debug!( - target: "providers::db", - ?block_number, - actions = ?durations_recorder.actions, - "Inserted block body" - ); } - self.storage.writer().write_block_bodies(self, bodies, write_to)?; + self.storage.writer().write_block_bodies(self, bodies)?; Ok(()) } - fn remove_blocks_above( - &self, - block: BlockNumber, - remove_from: StorageLocation, - ) -> ProviderResult<()> { + fn remove_blocks_above(&self, block: BlockNumber) -> ProviderResult<()> { + // Clean up HeaderNumbers for blocks being removed, we must clear all indexes from MDBX. for hash in self.canonical_hashes_range(block + 1, self.last_block_number()? + 1)? { self.tx.delete::(hash, None)?; } - // Only prune canonical headers after we've removed the block hashes as we rely on data from - // this table in `canonical_hashes_range`. - self.remove::(block + 1..)?; - self.remove::>>(block + 1..)?; - self.remove::(block + 1..)?; + // Get highest static file block for the total block range + let highest_static_file_block = self + .static_file_provider() + .get_highest_static_file_block(StaticFileSegment::Headers) + .expect("todo: error handling, headers should exist"); + + // IMPORTANT: we use `highest_static_file_block.saturating_sub(block_number)` to make sure + // we remove only what is ABOVE the block. + // + // i.e., if the highest static file block is 8, we want to remove above block 5 only, we + // will have three blocks to remove, which will be block 8, 7, and 6. + debug!(target: "providers::db", ?block, "Removing static file blocks above block_number"); + self.static_file_provider() + .get_writer(block, StaticFileSegment::Headers)? + .prune_headers(highest_static_file_block.saturating_sub(block))?; // First transaction to be removed let unwind_tx_from = self @@ -3008,17 +2958,13 @@ impl BlockWrite self.remove::(unwind_tx_from..)?; - self.remove_bodies_above(block, remove_from)?; + self.remove_bodies_above(block)?; Ok(()) } - fn remove_bodies_above( - &self, - block: BlockNumber, - remove_from: StorageLocation, - ) -> ProviderResult<()> { - self.storage.writer().remove_block_bodies_above(self, block, remove_from)?; + fn remove_bodies_above(&self, block: BlockNumber) -> ProviderResult<()> { + self.storage.writer().remove_block_bodies_above(self, block)?; // First transaction to be removed let unwind_tx_from = self @@ -3029,23 +2975,16 @@ impl BlockWrite self.remove::(block + 1..)?; self.remove::(unwind_tx_from..)?; - if remove_from.database() { - self.remove::>>(unwind_tx_from..)?; - } - - if remove_from.static_files() { - let static_file_tx_num = self - .static_file_provider - .get_highest_static_file_tx(StaticFileSegment::Transactions); + let static_file_tx_num = + self.static_file_provider.get_highest_static_file_tx(StaticFileSegment::Transactions); - let to_delete = static_file_tx_num - .map(|static_tx| (static_tx + 1).saturating_sub(unwind_tx_from)) - .unwrap_or_default(); + let to_delete = static_file_tx_num + .map(|static_tx| (static_tx + 1).saturating_sub(unwind_tx_from)) + .unwrap_or_default(); - self.static_file_provider - .latest_writer(StaticFileSegment::Transactions)? - .prune_transactions(to_delete, block)?; - } + self.static_file_provider + .latest_writer(StaticFileSegment::Transactions)? + .prune_transactions(to_delete, block)?; Ok(()) } @@ -3056,32 +2995,33 @@ impl BlockWrite blocks: Vec>, execution_outcome: &ExecutionOutcome, hashed_state: HashedPostStateSorted, - trie_updates: TrieUpdates, ) -> ProviderResult<()> { if blocks.is_empty() { debug!(target: "providers::db", "Attempted to append empty block range"); return Ok(()) } - let first_number = blocks.first().unwrap().number(); + // Blocks are not empty, so no need to handle the case of `blocks.first()` being + // `None`. + let first_number = blocks[0].number(); - let last = blocks.last().unwrap(); - let last_block_number = last.number(); + // Blocks are not empty, so no need to handle the case of `blocks.last()` being + // `None`. + let last_block_number = blocks[blocks.len() - 1].number(); let mut durations_recorder = metrics::DurationsRecorder::default(); // Insert the blocks for block in blocks { - self.insert_block(block, StorageLocation::Database)?; + self.insert_block(block)?; durations_recorder.record_relative(metrics::Action::InsertBlock); } - self.write_state(execution_outcome, OriginalValuesKnown::No, StorageLocation::Database)?; + self.write_state(execution_outcome, OriginalValuesKnown::No)?; durations_recorder.record_relative(metrics::Action::InsertState); // insert hashes and intermediate merkle nodes self.write_hashed_state(&hashed_state)?; - self.write_trie_updates(&trie_updates)?; durations_recorder.record_relative(metrics::Action::InsertHashes); self.update_history_indices(first_number..=last_block_number)?; @@ -3106,10 +3046,13 @@ impl PruneCheckpointReader for DatabaseProvide } fn get_prune_checkpoints(&self) -> ProviderResult> { - Ok(self - .tx - .cursor_read::()? - .walk(None)? + Ok(PruneSegment::variants() + .filter_map(|segment| { + self.tx + .get::(segment) + .transpose() + .map(|chk| chk.map(|chk| (segment, chk))) + }) .collect::>()?) } } @@ -3154,7 +3097,7 @@ impl ChainStateBlockReader for DatabaseProvide let mut finalized_blocks = self .tx .cursor_read::()? - .walk(Some(tables::ChainStateKey::LastSafeBlockBlock))? + .walk(Some(tables::ChainStateKey::LastSafeBlock))? .take(1) .collect::, _>>()?; @@ -3171,9 +3114,7 @@ impl ChainStateBlockWriter for DatabaseProvider ProviderResult<()> { - Ok(self - .tx - .put::(tables::ChainStateKey::LastSafeBlockBlock, block_number)?) + Ok(self.tx.put::(tables::ChainStateKey::LastSafeBlock, block_number)?) } } @@ -3195,4 +3136,1506 @@ impl DBProvider for DatabaseProvider fn prune_modes_ref(&self) -> &PruneModes { self.prune_modes_ref() } + + /// Commit database transaction and static files. + fn commit(self) -> ProviderResult { + // For unwinding it makes more sense to commit the database first, since if + // it is interrupted before the static files commit, we can just + // truncate the static files according to the + // checkpoints on the next start-up. + if self.static_file_provider.has_unwind_queued() { + self.tx.commit()?; + self.static_file_provider.commit()?; + } else { + self.static_file_provider.commit()?; + self.tx.commit()?; + } + + Ok(true) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + test_utils::{blocks::BlockchainTestData, create_test_provider_factory}, + BlockWriter, + }; + use reth_testing_utils::generators::{self, random_block, BlockParams}; + + #[test] + fn test_receipts_by_block_range_empty_range() { + let factory = create_test_provider_factory(); + let provider = factory.provider().unwrap(); + + // empty range should return empty vec + let start = 10u64; + let end = 9u64; + let result = provider.receipts_by_block_range(start..=end).unwrap(); + assert_eq!(result, Vec::>::new()); + } + + #[test] + fn test_receipts_by_block_range_nonexistent_blocks() { + let factory = create_test_provider_factory(); + let provider = factory.provider().unwrap(); + + // non-existent blocks should return empty vecs for each block + let result = provider.receipts_by_block_range(10..=12).unwrap(); + assert_eq!(result, vec![vec![], vec![], vec![]]); + } + + #[test] + fn test_receipts_by_block_range_single_block() { + let factory = create_test_provider_factory(); + let data = BlockchainTestData::default(); + + let provider_rw = factory.provider_rw().unwrap(); + provider_rw.insert_block(data.genesis.clone().try_recover().unwrap()).unwrap(); + provider_rw + .write_state( + &ExecutionOutcome { first_block: 0, receipts: vec![vec![]], ..Default::default() }, + crate::OriginalValuesKnown::No, + ) + .unwrap(); + provider_rw.insert_block(data.blocks[0].0.clone()).unwrap(); + provider_rw.write_state(&data.blocks[0].1, crate::OriginalValuesKnown::No).unwrap(); + provider_rw.commit().unwrap(); + + let provider = factory.provider().unwrap(); + let result = provider.receipts_by_block_range(1..=1).unwrap(); + + // should have one vec with one receipt + assert_eq!(result.len(), 1); + assert_eq!(result[0].len(), 1); + assert_eq!(result[0][0], data.blocks[0].1.receipts()[0][0]); + } + + #[test] + fn test_receipts_by_block_range_multiple_blocks() { + let factory = create_test_provider_factory(); + let data = BlockchainTestData::default(); + + let provider_rw = factory.provider_rw().unwrap(); + provider_rw.insert_block(data.genesis.clone().try_recover().unwrap()).unwrap(); + provider_rw + .write_state( + &ExecutionOutcome { first_block: 0, receipts: vec![vec![]], ..Default::default() }, + crate::OriginalValuesKnown::No, + ) + .unwrap(); + for i in 0..3 { + provider_rw.insert_block(data.blocks[i].0.clone()).unwrap(); + provider_rw.write_state(&data.blocks[i].1, crate::OriginalValuesKnown::No).unwrap(); + } + provider_rw.commit().unwrap(); + + let provider = factory.provider().unwrap(); + let result = provider.receipts_by_block_range(1..=3).unwrap(); + + // should have 3 vecs, each with one receipt + assert_eq!(result.len(), 3); + for (i, block_receipts) in result.iter().enumerate() { + assert_eq!(block_receipts.len(), 1); + assert_eq!(block_receipts[0], data.blocks[i].1.receipts()[0][0]); + } + } + + #[test] + fn test_receipts_by_block_range_blocks_with_varying_tx_counts() { + let factory = create_test_provider_factory(); + let data = BlockchainTestData::default(); + + let provider_rw = factory.provider_rw().unwrap(); + provider_rw.insert_block(data.genesis.clone().try_recover().unwrap()).unwrap(); + provider_rw + .write_state( + &ExecutionOutcome { first_block: 0, receipts: vec![vec![]], ..Default::default() }, + crate::OriginalValuesKnown::No, + ) + .unwrap(); + + // insert blocks 1-3 with receipts + for i in 0..3 { + provider_rw.insert_block(data.blocks[i].0.clone()).unwrap(); + provider_rw.write_state(&data.blocks[i].1, crate::OriginalValuesKnown::No).unwrap(); + } + provider_rw.commit().unwrap(); + + let provider = factory.provider().unwrap(); + let result = provider.receipts_by_block_range(1..=3).unwrap(); + + // verify each block has one receipt + assert_eq!(result.len(), 3); + for block_receipts in &result { + assert_eq!(block_receipts.len(), 1); + } + } + + #[test] + fn test_receipts_by_block_range_partial_range() { + let factory = create_test_provider_factory(); + let data = BlockchainTestData::default(); + + let provider_rw = factory.provider_rw().unwrap(); + provider_rw.insert_block(data.genesis.clone().try_recover().unwrap()).unwrap(); + provider_rw + .write_state( + &ExecutionOutcome { first_block: 0, receipts: vec![vec![]], ..Default::default() }, + crate::OriginalValuesKnown::No, + ) + .unwrap(); + for i in 0..3 { + provider_rw.insert_block(data.blocks[i].0.clone()).unwrap(); + provider_rw.write_state(&data.blocks[i].1, crate::OriginalValuesKnown::No).unwrap(); + } + provider_rw.commit().unwrap(); + + let provider = factory.provider().unwrap(); + + // request range that includes both existing and non-existing blocks + let result = provider.receipts_by_block_range(2..=5).unwrap(); + assert_eq!(result.len(), 4); + + // blocks 2-3 should have receipts, blocks 4-5 should be empty + assert_eq!(result[0].len(), 1); // block 2 + assert_eq!(result[1].len(), 1); // block 3 + assert_eq!(result[2].len(), 0); // block 4 (doesn't exist) + assert_eq!(result[3].len(), 0); // block 5 (doesn't exist) + + assert_eq!(result[0][0], data.blocks[1].1.receipts()[0][0]); + assert_eq!(result[1][0], data.blocks[2].1.receipts()[0][0]); + } + + #[test] + fn test_receipts_by_block_range_all_empty_blocks() { + let factory = create_test_provider_factory(); + let mut rng = generators::rng(); + + // create blocks with no transactions + let mut blocks = Vec::new(); + for i in 0..3 { + let block = + random_block(&mut rng, i, BlockParams { tx_count: Some(0), ..Default::default() }); + blocks.push(block); + } + + let provider_rw = factory.provider_rw().unwrap(); + for block in blocks { + provider_rw.insert_block(block.try_recover().unwrap()).unwrap(); + } + provider_rw.commit().unwrap(); + + let provider = factory.provider().unwrap(); + let result = provider.receipts_by_block_range(1..=3).unwrap(); + + assert_eq!(result.len(), 3); + for block_receipts in result { + assert_eq!(block_receipts.len(), 0); + } + } + + #[test] + fn test_receipts_by_block_range_consistency_with_individual_calls() { + let factory = create_test_provider_factory(); + let data = BlockchainTestData::default(); + + let provider_rw = factory.provider_rw().unwrap(); + provider_rw.insert_block(data.genesis.clone().try_recover().unwrap()).unwrap(); + provider_rw + .write_state( + &ExecutionOutcome { first_block: 0, receipts: vec![vec![]], ..Default::default() }, + crate::OriginalValuesKnown::No, + ) + .unwrap(); + for i in 0..3 { + provider_rw.insert_block(data.blocks[i].0.clone()).unwrap(); + provider_rw.write_state(&data.blocks[i].1, crate::OriginalValuesKnown::No).unwrap(); + } + provider_rw.commit().unwrap(); + + let provider = factory.provider().unwrap(); + + // get receipts using block range method + let range_result = provider.receipts_by_block_range(1..=3).unwrap(); + + // get receipts using individual block calls + let mut individual_results = Vec::new(); + for block_num in 1..=3 { + let receipts = + provider.receipts_by_block(block_num.into()).unwrap().unwrap_or_default(); + individual_results.push(receipts); + } + + assert_eq!(range_result, individual_results); + } + + #[test] + fn test_write_trie_changesets() { + use reth_db_api::models::BlockNumberHashedAddress; + use reth_trie::{BranchNodeCompact, StorageTrieEntry}; + + let factory = create_test_provider_factory(); + let provider_rw = factory.provider_rw().unwrap(); + + let block_number = 1u64; + + // Create some test nibbles and nodes + let account_nibbles1 = Nibbles::from_nibbles([0x1, 0x2, 0x3, 0x4]); + let account_nibbles2 = Nibbles::from_nibbles([0x5, 0x6, 0x7, 0x8]); + + let node1 = BranchNodeCompact::new( + 0b1111_1111_1111_1111, // state_mask + 0b0000_0000_0000_0000, // tree_mask + 0b0000_0000_0000_0000, // hash_mask + vec![], // hashes + None, // root hash + ); + + // Pre-populate AccountsTrie with a node that will be updated (for account_nibbles1) + { + let mut cursor = provider_rw.tx_ref().cursor_write::().unwrap(); + cursor.insert(StoredNibbles(account_nibbles1), &node1).unwrap(); + } + + // Create account trie updates: one Some (update) and one None (removal) + let account_nodes = vec![ + (account_nibbles1, Some(node1.clone())), // This will update existing node + (account_nibbles2, None), // This will be a removal (no existing node) + ]; + + // Create storage trie updates + let storage_address1 = B256::from([1u8; 32]); // Normal storage trie + let storage_address2 = B256::from([2u8; 32]); // Wiped storage trie + + let storage_nibbles1 = Nibbles::from_nibbles([0xa, 0xb]); + let storage_nibbles2 = Nibbles::from_nibbles([0xc, 0xd]); + let storage_nibbles3 = Nibbles::from_nibbles([0xe, 0xf]); + + let storage_node1 = BranchNodeCompact::new( + 0b1111_0000_0000_0000, + 0b0000_0000_0000_0000, + 0b0000_0000_0000_0000, + vec![], + None, + ); + + let storage_node2 = BranchNodeCompact::new( + 0b0000_1111_0000_0000, + 0b0000_0000_0000_0000, + 0b0000_0000_0000_0000, + vec![], + None, + ); + + // Create an old version of storage_node1 to prepopulate + let storage_node1_old = BranchNodeCompact::new( + 0b1010_0000_0000_0000, // Different mask to show it's an old value + 0b0000_0000_0000_0000, + 0b0000_0000_0000_0000, + vec![], + None, + ); + + // Pre-populate StoragesTrie for normal storage (storage_address1) + { + let mut cursor = + provider_rw.tx_ref().cursor_dup_write::().unwrap(); + // Add node that will be updated (storage_nibbles1) with old value + let entry = StorageTrieEntry { + nibbles: StoredNibblesSubKey(storage_nibbles1), + node: storage_node1_old.clone(), + }; + cursor.upsert(storage_address1, &entry).unwrap(); + } + + // Pre-populate StoragesTrie for wiped storage (storage_address2) + { + let mut cursor = + provider_rw.tx_ref().cursor_dup_write::().unwrap(); + // Add node that will be updated (storage_nibbles1) + let entry1 = StorageTrieEntry { + nibbles: StoredNibblesSubKey(storage_nibbles1), + node: storage_node1.clone(), + }; + cursor.upsert(storage_address2, &entry1).unwrap(); + // Add node that won't be updated but exists (storage_nibbles3) + let entry3 = StorageTrieEntry { + nibbles: StoredNibblesSubKey(storage_nibbles3), + node: storage_node2.clone(), + }; + cursor.upsert(storage_address2, &entry3).unwrap(); + } + + // Normal storage trie: one Some (update) and one None (new) + let storage_trie1 = StorageTrieUpdatesSorted { + is_deleted: false, + storage_nodes: vec![ + (storage_nibbles1, Some(storage_node1.clone())), // This will update existing node + (storage_nibbles2, None), // This is a new node + ], + }; + + // Wiped storage trie + let storage_trie2 = StorageTrieUpdatesSorted { + is_deleted: true, + storage_nodes: vec![ + (storage_nibbles1, Some(storage_node1.clone())), // Updated node already in db + (storage_nibbles2, Some(storage_node2.clone())), /* Updated node not in db + * storage_nibbles3 is in db + * but not updated */ + ], + }; + + let mut storage_tries = B256Map::default(); + storage_tries.insert(storage_address1, storage_trie1); + storage_tries.insert(storage_address2, storage_trie2); + + let trie_updates = TrieUpdatesSorted::new(account_nodes, storage_tries); + + // Write the changesets + let num_written = + provider_rw.write_trie_changesets(block_number, &trie_updates, None).unwrap(); + + // Verify number of entries written + // Account changesets: 2 (one update, one removal) + // Storage changesets: + // - Normal storage: 2 (one update, one removal) + // - Wiped storage: 3 (two updated, one existing not updated) + // Total: 2 + 2 + 3 = 7 + assert_eq!(num_written, 7); + + // Verify account changesets were written correctly + { + let mut cursor = + provider_rw.tx_ref().cursor_dup_read::().unwrap(); + + // Get all entries for this block to see what was written + let all_entries = cursor + .walk_dup(Some(block_number), None) + .unwrap() + .collect::, _>>() + .unwrap(); + + // Assert the full value of all_entries in a single assert_eq + assert_eq!( + all_entries, + vec![ + ( + block_number, + TrieChangeSetsEntry { + nibbles: StoredNibblesSubKey(account_nibbles1), + node: Some(node1), + } + ), + ( + block_number, + TrieChangeSetsEntry { + nibbles: StoredNibblesSubKey(account_nibbles2), + node: None, + } + ), + ] + ); + } + + // Verify storage changesets were written correctly + { + let mut cursor = + provider_rw.tx_ref().cursor_dup_read::().unwrap(); + + // Check normal storage trie changesets + let key1 = BlockNumberHashedAddress((block_number, storage_address1)); + let entries1 = + cursor.walk_dup(Some(key1), None).unwrap().collect::, _>>().unwrap(); + + assert_eq!( + entries1, + vec![ + ( + key1, + TrieChangeSetsEntry { + nibbles: StoredNibblesSubKey(storage_nibbles1), + node: Some(storage_node1_old), // Old value that was prepopulated + } + ), + ( + key1, + TrieChangeSetsEntry { + nibbles: StoredNibblesSubKey(storage_nibbles2), + node: None, // New node, no previous value + } + ), + ] + ); + + // Check wiped storage trie changesets + let key2 = BlockNumberHashedAddress((block_number, storage_address2)); + let entries2 = + cursor.walk_dup(Some(key2), None).unwrap().collect::, _>>().unwrap(); + + assert_eq!( + entries2, + vec![ + ( + key2, + TrieChangeSetsEntry { + nibbles: StoredNibblesSubKey(storage_nibbles1), + node: Some(storage_node1), // Was in db, so has old value + } + ), + ( + key2, + TrieChangeSetsEntry { + nibbles: StoredNibblesSubKey(storage_nibbles2), + node: None, // Was not in db + } + ), + ( + key2, + TrieChangeSetsEntry { + nibbles: StoredNibblesSubKey(storage_nibbles3), + node: Some(storage_node2), // Existing node in wiped storage + } + ), + ] + ); + } + + provider_rw.commit().unwrap(); + } + + #[test] + fn test_write_trie_changesets_with_overlay() { + use reth_db_api::models::BlockNumberHashedAddress; + use reth_trie::BranchNodeCompact; + + let factory = create_test_provider_factory(); + let provider_rw = factory.provider_rw().unwrap(); + + let block_number = 1u64; + + // Create some test nibbles and nodes + let account_nibbles1 = Nibbles::from_nibbles([0x1, 0x2, 0x3, 0x4]); + let account_nibbles2 = Nibbles::from_nibbles([0x5, 0x6, 0x7, 0x8]); + + let node1 = BranchNodeCompact::new( + 0b1111_1111_1111_1111, // state_mask + 0b0000_0000_0000_0000, // tree_mask + 0b0000_0000_0000_0000, // hash_mask + vec![], // hashes + None, // root hash + ); + + // NOTE: Unlike the previous test, we're NOT pre-populating the database + // All node values will come from the overlay + + // Create the overlay with existing values that would normally be in the DB + let node1_old = BranchNodeCompact::new( + 0b1010_1010_1010_1010, // Different mask to show it's the overlay "existing" value + 0b0000_0000_0000_0000, + 0b0000_0000_0000_0000, + vec![], + None, + ); + + // Create overlay account nodes + let overlay_account_nodes = vec![ + (account_nibbles1, Some(node1_old.clone())), // This simulates existing node in overlay + ]; + + // Create account trie updates: one Some (update) and one None (removal) + let account_nodes = vec![ + (account_nibbles1, Some(node1)), // This will update overlay node + (account_nibbles2, None), // This will be a removal (no existing node) + ]; + + // Create storage trie updates + let storage_address1 = B256::from([1u8; 32]); // Normal storage trie + let storage_address2 = B256::from([2u8; 32]); // Wiped storage trie + + let storage_nibbles1 = Nibbles::from_nibbles([0xa, 0xb]); + let storage_nibbles2 = Nibbles::from_nibbles([0xc, 0xd]); + let storage_nibbles3 = Nibbles::from_nibbles([0xe, 0xf]); + + let storage_node1 = BranchNodeCompact::new( + 0b1111_0000_0000_0000, + 0b0000_0000_0000_0000, + 0b0000_0000_0000_0000, + vec![], + None, + ); + + let storage_node2 = BranchNodeCompact::new( + 0b0000_1111_0000_0000, + 0b0000_0000_0000_0000, + 0b0000_0000_0000_0000, + vec![], + None, + ); + + // Create old versions for overlay + let storage_node1_old = BranchNodeCompact::new( + 0b1010_0000_0000_0000, // Different mask to show it's an old value + 0b0000_0000_0000_0000, + 0b0000_0000_0000_0000, + vec![], + None, + ); + + // Create overlay storage nodes + let mut overlay_storage_tries = B256Map::default(); + + // Overlay for normal storage (storage_address1) + let overlay_storage_trie1 = StorageTrieUpdatesSorted { + is_deleted: false, + storage_nodes: vec![ + (storage_nibbles1, Some(storage_node1_old.clone())), /* Simulates existing in + * overlay */ + ], + }; + + // Overlay for wiped storage (storage_address2) + let overlay_storage_trie2 = StorageTrieUpdatesSorted { + is_deleted: false, + storage_nodes: vec![ + (storage_nibbles1, Some(storage_node1.clone())), // Existing in overlay + (storage_nibbles3, Some(storage_node2.clone())), // Also existing in overlay + ], + }; + + overlay_storage_tries.insert(storage_address1, overlay_storage_trie1); + overlay_storage_tries.insert(storage_address2, overlay_storage_trie2); + + let overlay = TrieUpdatesSorted::new(overlay_account_nodes, overlay_storage_tries); + + // Normal storage trie: one Some (update) and one None (new) + let storage_trie1 = StorageTrieUpdatesSorted { + is_deleted: false, + storage_nodes: vec![ + (storage_nibbles1, Some(storage_node1.clone())), // This will update overlay node + (storage_nibbles2, None), // This is a new node + ], + }; + + // Wiped storage trie + let storage_trie2 = StorageTrieUpdatesSorted { + is_deleted: true, + storage_nodes: vec![ + (storage_nibbles1, Some(storage_node1.clone())), // Updated node from overlay + (storage_nibbles2, Some(storage_node2.clone())), /* Updated node not in overlay + * storage_nibbles3 is in + * overlay + * but not updated */ + ], + }; + + let mut storage_tries = B256Map::default(); + storage_tries.insert(storage_address1, storage_trie1); + storage_tries.insert(storage_address2, storage_trie2); + + let trie_updates = TrieUpdatesSorted::new(account_nodes, storage_tries); + + // Write the changesets WITH OVERLAY + let num_written = + provider_rw.write_trie_changesets(block_number, &trie_updates, Some(&overlay)).unwrap(); + + // Verify number of entries written + // Account changesets: 2 (one update from overlay, one removal) + // Storage changesets: + // - Normal storage: 2 (one update from overlay, one new) + // - Wiped storage: 3 (two updated, one existing from overlay not updated) + // Total: 2 + 2 + 3 = 7 + assert_eq!(num_written, 7); + + // Verify account changesets were written correctly + { + let mut cursor = + provider_rw.tx_ref().cursor_dup_read::().unwrap(); + + // Get all entries for this block to see what was written + let all_entries = cursor + .walk_dup(Some(block_number), None) + .unwrap() + .collect::, _>>() + .unwrap(); + + // Assert the full value of all_entries in a single assert_eq + assert_eq!( + all_entries, + vec![ + ( + block_number, + TrieChangeSetsEntry { + nibbles: StoredNibblesSubKey(account_nibbles1), + node: Some(node1_old), // Value from overlay, not DB + } + ), + ( + block_number, + TrieChangeSetsEntry { + nibbles: StoredNibblesSubKey(account_nibbles2), + node: None, + } + ), + ] + ); + } + + // Verify storage changesets were written correctly + { + let mut cursor = + provider_rw.tx_ref().cursor_dup_read::().unwrap(); + + // Check normal storage trie changesets + let key1 = BlockNumberHashedAddress((block_number, storage_address1)); + let entries1 = + cursor.walk_dup(Some(key1), None).unwrap().collect::, _>>().unwrap(); + + assert_eq!( + entries1, + vec![ + ( + key1, + TrieChangeSetsEntry { + nibbles: StoredNibblesSubKey(storage_nibbles1), + node: Some(storage_node1_old), // Old value from overlay + } + ), + ( + key1, + TrieChangeSetsEntry { + nibbles: StoredNibblesSubKey(storage_nibbles2), + node: None, // New node, no previous value + } + ), + ] + ); + + // Check wiped storage trie changesets + let key2 = BlockNumberHashedAddress((block_number, storage_address2)); + let entries2 = + cursor.walk_dup(Some(key2), None).unwrap().collect::, _>>().unwrap(); + + assert_eq!( + entries2, + vec![ + ( + key2, + TrieChangeSetsEntry { + nibbles: StoredNibblesSubKey(storage_nibbles1), + node: Some(storage_node1), // Value from overlay + } + ), + ( + key2, + TrieChangeSetsEntry { + nibbles: StoredNibblesSubKey(storage_nibbles2), + node: None, // Was not in overlay + } + ), + ( + key2, + TrieChangeSetsEntry { + nibbles: StoredNibblesSubKey(storage_nibbles3), + node: Some(storage_node2), /* Existing node from overlay in wiped + * storage */ + } + ), + ] + ); + } + + provider_rw.commit().unwrap(); + } + + #[test] + fn test_clear_trie_changesets_from() { + use alloy_primitives::hex_literal::hex; + use reth_db_api::models::BlockNumberHashedAddress; + use reth_trie::{BranchNodeCompact, StoredNibblesSubKey, TrieChangeSetsEntry}; + + let factory = create_test_provider_factory(); + + // Create some test data for different block numbers + let block1 = 100u64; + let block2 = 101u64; + let block3 = 102u64; + let block4 = 103u64; + let block5 = 104u64; + + // Create test addresses for storage changesets + let storage_address1 = + B256::from(hex!("1111111111111111111111111111111111111111111111111111111111111111")); + let storage_address2 = + B256::from(hex!("2222222222222222222222222222222222222222222222222222222222222222")); + + // Create test nibbles + let nibbles1 = StoredNibblesSubKey(Nibbles::from_nibbles([0x1, 0x2, 0x3])); + let nibbles2 = StoredNibblesSubKey(Nibbles::from_nibbles([0x4, 0x5, 0x6])); + let nibbles3 = StoredNibblesSubKey(Nibbles::from_nibbles([0x7, 0x8, 0x9])); + + // Create test nodes + let node1 = BranchNodeCompact::new( + 0b1111_1111_1111_1111, + 0b1111_1111_1111_1111, + 0b0000_0000_0000_0001, + vec![B256::from(hex!( + "1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef" + ))], + None, + ); + let node2 = BranchNodeCompact::new( + 0b1111_1111_1111_1110, + 0b1111_1111_1111_1110, + 0b0000_0000_0000_0010, + vec![B256::from(hex!( + "abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890" + ))], + Some(B256::from(hex!( + "deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef" + ))), + ); + + // Populate AccountsTrieChangeSets with data across multiple blocks + { + let provider_rw = factory.provider_rw().unwrap(); + let mut cursor = + provider_rw.tx_ref().cursor_dup_write::().unwrap(); + + // Block 100: 2 entries (will be kept - before start block) + cursor + .upsert( + block1, + &TrieChangeSetsEntry { nibbles: nibbles1.clone(), node: Some(node1.clone()) }, + ) + .unwrap(); + cursor + .upsert(block1, &TrieChangeSetsEntry { nibbles: nibbles2.clone(), node: None }) + .unwrap(); + + // Block 101: 3 entries with duplicates (will be deleted - from this block onwards) + cursor + .upsert( + block2, + &TrieChangeSetsEntry { nibbles: nibbles1.clone(), node: Some(node2.clone()) }, + ) + .unwrap(); + cursor + .upsert( + block2, + &TrieChangeSetsEntry { nibbles: nibbles1.clone(), node: Some(node1.clone()) }, + ) + .unwrap(); // duplicate key + cursor + .upsert(block2, &TrieChangeSetsEntry { nibbles: nibbles3.clone(), node: None }) + .unwrap(); + + // Block 102: 2 entries (will be deleted - after start block) + cursor + .upsert( + block3, + &TrieChangeSetsEntry { nibbles: nibbles2.clone(), node: Some(node1.clone()) }, + ) + .unwrap(); + cursor + .upsert( + block3, + &TrieChangeSetsEntry { nibbles: nibbles3.clone(), node: Some(node2.clone()) }, + ) + .unwrap(); + + // Block 103: 1 entry (will be deleted - after start block) + cursor + .upsert(block4, &TrieChangeSetsEntry { nibbles: nibbles1.clone(), node: None }) + .unwrap(); + + // Block 104: 2 entries (will be deleted - after start block) + cursor + .upsert( + block5, + &TrieChangeSetsEntry { nibbles: nibbles2.clone(), node: Some(node2.clone()) }, + ) + .unwrap(); + cursor + .upsert(block5, &TrieChangeSetsEntry { nibbles: nibbles3.clone(), node: None }) + .unwrap(); + + provider_rw.commit().unwrap(); + } + + // Populate StoragesTrieChangeSets with data across multiple blocks + { + let provider_rw = factory.provider_rw().unwrap(); + let mut cursor = + provider_rw.tx_ref().cursor_dup_write::().unwrap(); + + // Block 100, address1: 2 entries (will be kept - before start block) + let key1_block1 = BlockNumberHashedAddress((block1, storage_address1)); + cursor + .upsert( + key1_block1, + &TrieChangeSetsEntry { nibbles: nibbles1.clone(), node: Some(node1.clone()) }, + ) + .unwrap(); + cursor + .upsert(key1_block1, &TrieChangeSetsEntry { nibbles: nibbles2.clone(), node: None }) + .unwrap(); + + // Block 101, address1: 3 entries with duplicates (will be deleted - from this block + // onwards) + let key1_block2 = BlockNumberHashedAddress((block2, storage_address1)); + cursor + .upsert( + key1_block2, + &TrieChangeSetsEntry { nibbles: nibbles1.clone(), node: Some(node2.clone()) }, + ) + .unwrap(); + cursor + .upsert(key1_block2, &TrieChangeSetsEntry { nibbles: nibbles1.clone(), node: None }) + .unwrap(); // duplicate key + cursor + .upsert( + key1_block2, + &TrieChangeSetsEntry { nibbles: nibbles2.clone(), node: Some(node1.clone()) }, + ) + .unwrap(); + + // Block 102, address2: 2 entries (will be deleted - after start block) + let key2_block3 = BlockNumberHashedAddress((block3, storage_address2)); + cursor + .upsert( + key2_block3, + &TrieChangeSetsEntry { nibbles: nibbles2.clone(), node: Some(node2.clone()) }, + ) + .unwrap(); + cursor + .upsert(key2_block3, &TrieChangeSetsEntry { nibbles: nibbles3.clone(), node: None }) + .unwrap(); + + // Block 103, address1: 2 entries with duplicate (will be deleted - after start block) + let key1_block4 = BlockNumberHashedAddress((block4, storage_address1)); + cursor + .upsert( + key1_block4, + &TrieChangeSetsEntry { nibbles: nibbles3.clone(), node: Some(node1) }, + ) + .unwrap(); + cursor + .upsert( + key1_block4, + &TrieChangeSetsEntry { nibbles: nibbles3, node: Some(node2.clone()) }, + ) + .unwrap(); // duplicate key + + // Block 104, address2: 2 entries (will be deleted - after start block) + let key2_block5 = BlockNumberHashedAddress((block5, storage_address2)); + cursor + .upsert(key2_block5, &TrieChangeSetsEntry { nibbles: nibbles1, node: None }) + .unwrap(); + cursor + .upsert(key2_block5, &TrieChangeSetsEntry { nibbles: nibbles2, node: Some(node2) }) + .unwrap(); + + provider_rw.commit().unwrap(); + } + + // Clear all changesets from block 101 onwards + { + let provider_rw = factory.provider_rw().unwrap(); + provider_rw.clear_trie_changesets_from(block2).unwrap(); + provider_rw.commit().unwrap(); + } + + // Verify AccountsTrieChangeSets after clearing + { + let provider = factory.provider().unwrap(); + let mut cursor = + provider.tx_ref().cursor_dup_read::().unwrap(); + + // Block 100 should still exist (before range) + let block1_entries = cursor + .walk_dup(Some(block1), None) + .unwrap() + .collect::, _>>() + .unwrap(); + assert_eq!(block1_entries.len(), 2, "Block 100 entries should be preserved"); + assert_eq!(block1_entries[0].0, block1); + assert_eq!(block1_entries[1].0, block1); + + // Blocks 101-104 should be deleted + let block2_entries = cursor + .walk_dup(Some(block2), None) + .unwrap() + .collect::, _>>() + .unwrap(); + assert!(block2_entries.is_empty(), "Block 101 entries should be deleted"); + + let block3_entries = cursor + .walk_dup(Some(block3), None) + .unwrap() + .collect::, _>>() + .unwrap(); + assert!(block3_entries.is_empty(), "Block 102 entries should be deleted"); + + let block4_entries = cursor + .walk_dup(Some(block4), None) + .unwrap() + .collect::, _>>() + .unwrap(); + assert!(block4_entries.is_empty(), "Block 103 entries should be deleted"); + + // Block 104 should also be deleted + let block5_entries = cursor + .walk_dup(Some(block5), None) + .unwrap() + .collect::, _>>() + .unwrap(); + assert!(block5_entries.is_empty(), "Block 104 entries should be deleted"); + } + + // Verify StoragesTrieChangeSets after clearing + { + let provider = factory.provider().unwrap(); + let mut cursor = + provider.tx_ref().cursor_dup_read::().unwrap(); + + // Block 100 entries should still exist (before range) + let key1_block1 = BlockNumberHashedAddress((block1, storage_address1)); + let block1_entries = cursor + .walk_dup(Some(key1_block1), None) + .unwrap() + .collect::, _>>() + .unwrap(); + assert_eq!(block1_entries.len(), 2, "Block 100 storage entries should be preserved"); + + // Blocks 101-104 entries should be deleted + let key1_block2 = BlockNumberHashedAddress((block2, storage_address1)); + let block2_entries = cursor + .walk_dup(Some(key1_block2), None) + .unwrap() + .collect::, _>>() + .unwrap(); + assert!(block2_entries.is_empty(), "Block 101 storage entries should be deleted"); + + let key2_block3 = BlockNumberHashedAddress((block3, storage_address2)); + let block3_entries = cursor + .walk_dup(Some(key2_block3), None) + .unwrap() + .collect::, _>>() + .unwrap(); + assert!(block3_entries.is_empty(), "Block 102 storage entries should be deleted"); + + let key1_block4 = BlockNumberHashedAddress((block4, storage_address1)); + let block4_entries = cursor + .walk_dup(Some(key1_block4), None) + .unwrap() + .collect::, _>>() + .unwrap(); + assert!(block4_entries.is_empty(), "Block 103 storage entries should be deleted"); + + // Block 104 entries should also be deleted + let key2_block5 = BlockNumberHashedAddress((block5, storage_address2)); + let block5_entries = cursor + .walk_dup(Some(key2_block5), None) + .unwrap() + .collect::, _>>() + .unwrap(); + assert!(block5_entries.is_empty(), "Block 104 storage entries should be deleted"); + } + } + + #[test] + fn test_write_trie_updates_sorted() { + use reth_trie::{ + updates::{StorageTrieUpdatesSorted, TrieUpdatesSorted}, + BranchNodeCompact, StorageTrieEntry, + }; + + let factory = create_test_provider_factory(); + let provider_rw = factory.provider_rw().unwrap(); + + // Pre-populate account trie with data that will be deleted + { + let tx = provider_rw.tx_ref(); + let mut cursor = tx.cursor_write::().unwrap(); + + // Add account node that will be deleted + let to_delete = StoredNibbles(Nibbles::from_nibbles([0x3, 0x4])); + cursor + .upsert( + to_delete, + &BranchNodeCompact::new( + 0b1010_1010_1010_1010, // state_mask + 0b0000_0000_0000_0000, // tree_mask + 0b0000_0000_0000_0000, // hash_mask + vec![], + None, + ), + ) + .unwrap(); + + // Add account node that will be updated + let to_update = StoredNibbles(Nibbles::from_nibbles([0x1, 0x2])); + cursor + .upsert( + to_update, + &BranchNodeCompact::new( + 0b0101_0101_0101_0101, // old state_mask (will be updated) + 0b0000_0000_0000_0000, // tree_mask + 0b0000_0000_0000_0000, // hash_mask + vec![], + None, + ), + ) + .unwrap(); + } + + // Pre-populate storage tries with data + let storage_address1 = B256::from([1u8; 32]); + let storage_address2 = B256::from([2u8; 32]); + { + let tx = provider_rw.tx_ref(); + let mut storage_cursor = tx.cursor_dup_write::().unwrap(); + + // Add storage nodes for address1 (one will be deleted) + storage_cursor + .upsert( + storage_address1, + &StorageTrieEntry { + nibbles: StoredNibblesSubKey(Nibbles::from_nibbles([0x2, 0x0])), + node: BranchNodeCompact::new( + 0b0011_0011_0011_0011, // will be deleted + 0b0000_0000_0000_0000, + 0b0000_0000_0000_0000, + vec![], + None, + ), + }, + ) + .unwrap(); + + // Add storage nodes for address2 (will be wiped) + storage_cursor + .upsert( + storage_address2, + &StorageTrieEntry { + nibbles: StoredNibblesSubKey(Nibbles::from_nibbles([0xa, 0xb])), + node: BranchNodeCompact::new( + 0b1100_1100_1100_1100, // will be wiped + 0b0000_0000_0000_0000, + 0b0000_0000_0000_0000, + vec![], + None, + ), + }, + ) + .unwrap(); + storage_cursor + .upsert( + storage_address2, + &StorageTrieEntry { + nibbles: StoredNibblesSubKey(Nibbles::from_nibbles([0xc, 0xd])), + node: BranchNodeCompact::new( + 0b0011_1100_0011_1100, // will be wiped + 0b0000_0000_0000_0000, + 0b0000_0000_0000_0000, + vec![], + None, + ), + }, + ) + .unwrap(); + } + + // Create sorted account trie updates + let account_nodes = vec![ + ( + Nibbles::from_nibbles([0x1, 0x2]), + Some(BranchNodeCompact::new( + 0b1111_1111_1111_1111, // state_mask (updated) + 0b0000_0000_0000_0000, // tree_mask + 0b0000_0000_0000_0000, // hash_mask (no hashes) + vec![], + None, + )), + ), + (Nibbles::from_nibbles([0x3, 0x4]), None), // Deletion + ( + Nibbles::from_nibbles([0x5, 0x6]), + Some(BranchNodeCompact::new( + 0b1111_1111_1111_1111, // state_mask + 0b0000_0000_0000_0000, // tree_mask + 0b0000_0000_0000_0000, // hash_mask (no hashes) + vec![], + None, + )), + ), + ]; + + // Create sorted storage trie updates + let storage_trie1 = StorageTrieUpdatesSorted { + is_deleted: false, + storage_nodes: vec![ + ( + Nibbles::from_nibbles([0x1, 0x0]), + Some(BranchNodeCompact::new( + 0b1111_0000_0000_0000, // state_mask + 0b0000_0000_0000_0000, // tree_mask + 0b0000_0000_0000_0000, // hash_mask (no hashes) + vec![], + None, + )), + ), + (Nibbles::from_nibbles([0x2, 0x0]), None), // Deletion of existing node + ], + }; + + let storage_trie2 = StorageTrieUpdatesSorted { + is_deleted: true, // Wipe all storage for this address + storage_nodes: vec![], + }; + + let mut storage_tries = B256Map::default(); + storage_tries.insert(storage_address1, storage_trie1); + storage_tries.insert(storage_address2, storage_trie2); + + let trie_updates = TrieUpdatesSorted::new(account_nodes, storage_tries); + + // Write the sorted trie updates + let num_entries = provider_rw.write_trie_updates_sorted(&trie_updates).unwrap(); + + // We should have 2 account insertions + 1 account deletion + 1 storage insertion + 1 + // storage deletion = 5 + assert_eq!(num_entries, 5); + + // Verify account trie updates were written correctly + let tx = provider_rw.tx_ref(); + let mut cursor = tx.cursor_read::().unwrap(); + + // Check first account node was updated + let nibbles1 = StoredNibbles(Nibbles::from_nibbles([0x1, 0x2])); + let entry1 = cursor.seek_exact(nibbles1).unwrap(); + assert!(entry1.is_some(), "Updated account node should exist"); + let expected_mask = reth_trie::TrieMask::new(0b1111_1111_1111_1111); + assert_eq!( + entry1.unwrap().1.state_mask, + expected_mask, + "Account node should have updated state_mask" + ); + + // Check deleted account node no longer exists + let nibbles2 = StoredNibbles(Nibbles::from_nibbles([0x3, 0x4])); + let entry2 = cursor.seek_exact(nibbles2).unwrap(); + assert!(entry2.is_none(), "Deleted account node should not exist"); + + // Check new account node exists + let nibbles3 = StoredNibbles(Nibbles::from_nibbles([0x5, 0x6])); + let entry3 = cursor.seek_exact(nibbles3).unwrap(); + assert!(entry3.is_some(), "New account node should exist"); + + // Verify storage trie updates were written correctly + let mut storage_cursor = tx.cursor_dup_read::().unwrap(); + + // Check storage for address1 + let storage_entries1: Vec<_> = storage_cursor + .walk_dup(Some(storage_address1), None) + .unwrap() + .collect::, _>>() + .unwrap(); + assert_eq!( + storage_entries1.len(), + 1, + "Storage address1 should have 1 entry after deletion" + ); + assert_eq!( + storage_entries1[0].1.nibbles.0, + Nibbles::from_nibbles([0x1, 0x0]), + "Remaining entry should be [0x1, 0x0]" + ); + + // Check storage for address2 was wiped + let storage_entries2: Vec<_> = storage_cursor + .walk_dup(Some(storage_address2), None) + .unwrap() + .collect::, _>>() + .unwrap(); + assert_eq!(storage_entries2.len(), 0, "Storage address2 should be empty after wipe"); + + provider_rw.commit().unwrap(); + } + + #[test] + fn test_get_block_trie_updates() { + use reth_db_api::models::BlockNumberHashedAddress; + use reth_trie::{BranchNodeCompact, StorageTrieEntry}; + + let factory = create_test_provider_factory(); + let provider_rw = factory.provider_rw().unwrap(); + + let target_block = 2u64; + let next_block = 3u64; + + // Create test nibbles and nodes for accounts + let account_nibbles1 = Nibbles::from_nibbles([0x1, 0x2, 0x3, 0x4]); + let account_nibbles2 = Nibbles::from_nibbles([0x5, 0x6, 0x7, 0x8]); + let account_nibbles3 = Nibbles::from_nibbles([0x9, 0xa, 0xb, 0xc]); + + let node1 = BranchNodeCompact::new( + 0b1111_1111_0000_0000, + 0b0000_0000_0000_0000, + 0b0000_0000_0000_0000, + vec![], + None, + ); + + let node2 = BranchNodeCompact::new( + 0b0000_0000_1111_1111, + 0b0000_0000_0000_0000, + 0b0000_0000_0000_0000, + vec![], + None, + ); + + let node3 = BranchNodeCompact::new( + 0b1010_1010_1010_1010, + 0b0000_0000_0000_0000, + 0b0000_0000_0000_0000, + vec![], + None, + ); + + // Pre-populate AccountsTrie with nodes that will be the final state + { + let mut cursor = provider_rw.tx_ref().cursor_write::().unwrap(); + cursor.insert(StoredNibbles(account_nibbles1), &node1).unwrap(); + cursor.insert(StoredNibbles(account_nibbles2), &node2).unwrap(); + // account_nibbles3 will be deleted (not in final state) + } + + // Insert trie changesets for target_block + { + let mut cursor = + provider_rw.tx_ref().cursor_dup_write::().unwrap(); + // nibbles1 was updated in target_block (old value stored) + cursor + .append_dup( + target_block, + TrieChangeSetsEntry { + nibbles: StoredNibblesSubKey(account_nibbles1), + node: Some(BranchNodeCompact::new( + 0b1111_0000_0000_0000, // old value + 0b0000_0000_0000_0000, + 0b0000_0000_0000_0000, + vec![], + None, + )), + }, + ) + .unwrap(); + // nibbles2 was created in target_block (no old value) + cursor + .append_dup( + target_block, + TrieChangeSetsEntry { + nibbles: StoredNibblesSubKey(account_nibbles2), + node: None, + }, + ) + .unwrap(); + } + + // Insert trie changesets for next_block (to test overlay) + { + let mut cursor = + provider_rw.tx_ref().cursor_dup_write::().unwrap(); + // nibbles3 was deleted in next_block (old value stored) + cursor + .append_dup( + next_block, + TrieChangeSetsEntry { + nibbles: StoredNibblesSubKey(account_nibbles3), + node: Some(node3), + }, + ) + .unwrap(); + } + + // Storage trie updates + let storage_address1 = B256::from([1u8; 32]); + let storage_nibbles1 = Nibbles::from_nibbles([0xa, 0xb]); + let storage_nibbles2 = Nibbles::from_nibbles([0xc, 0xd]); + + let storage_node1 = BranchNodeCompact::new( + 0b1111_1111_1111_0000, + 0b0000_0000_0000_0000, + 0b0000_0000_0000_0000, + vec![], + None, + ); + + let storage_node2 = BranchNodeCompact::new( + 0b0101_0101_0101_0101, + 0b0000_0000_0000_0000, + 0b0000_0000_0000_0000, + vec![], + None, + ); + + // Pre-populate StoragesTrie with final state + { + let mut cursor = + provider_rw.tx_ref().cursor_dup_write::().unwrap(); + cursor + .upsert( + storage_address1, + &StorageTrieEntry { + nibbles: StoredNibblesSubKey(storage_nibbles1), + node: storage_node1.clone(), + }, + ) + .unwrap(); + // storage_nibbles2 was deleted in next_block, so it's not in final state + } + + // Insert storage trie changesets for target_block + { + let mut cursor = + provider_rw.tx_ref().cursor_dup_write::().unwrap(); + let key = BlockNumberHashedAddress((target_block, storage_address1)); + + // storage_nibbles1 was updated + cursor + .append_dup( + key, + TrieChangeSetsEntry { + nibbles: StoredNibblesSubKey(storage_nibbles1), + node: Some(BranchNodeCompact::new( + 0b0000_0000_1111_1111, // old value + 0b0000_0000_0000_0000, + 0b0000_0000_0000_0000, + vec![], + None, + )), + }, + ) + .unwrap(); + + // storage_nibbles2 was created + cursor + .append_dup( + key, + TrieChangeSetsEntry { + nibbles: StoredNibblesSubKey(storage_nibbles2), + node: None, + }, + ) + .unwrap(); + } + + // Insert storage trie changesets for next_block (to test overlay) + { + let mut cursor = + provider_rw.tx_ref().cursor_dup_write::().unwrap(); + let key = BlockNumberHashedAddress((next_block, storage_address1)); + + // storage_nibbles2 was deleted in next_block + cursor + .append_dup( + key, + TrieChangeSetsEntry { + nibbles: StoredNibblesSubKey(storage_nibbles2), + node: Some(BranchNodeCompact::new( + 0b0101_0101_0101_0101, // value that was deleted + 0b0000_0000_0000_0000, + 0b0000_0000_0000_0000, + vec![], + None, + )), + }, + ) + .unwrap(); + } + + provider_rw.commit().unwrap(); + + // Now test get_block_trie_updates + let provider = factory.provider().unwrap(); + let result = provider.get_block_trie_updates(target_block).unwrap(); + + // Verify account trie updates + assert_eq!(result.account_nodes_ref().len(), 2, "Should have 2 account trie updates"); + + // Check nibbles1 - should have the current value (node1) + let nibbles1_update = result + .account_nodes_ref() + .iter() + .find(|(n, _)| n == &account_nibbles1) + .expect("Should find nibbles1"); + assert!(nibbles1_update.1.is_some(), "nibbles1 should have a value"); + assert_eq!( + nibbles1_update.1.as_ref().unwrap().state_mask, + node1.state_mask, + "nibbles1 should have current value" + ); + + // Check nibbles2 - should have the current value (node2) + let nibbles2_update = result + .account_nodes_ref() + .iter() + .find(|(n, _)| n == &account_nibbles2) + .expect("Should find nibbles2"); + assert!(nibbles2_update.1.is_some(), "nibbles2 should have a value"); + assert_eq!( + nibbles2_update.1.as_ref().unwrap().state_mask, + node2.state_mask, + "nibbles2 should have current value" + ); + + // nibbles3 should NOT be in the result (it was changed in next_block, not target_block) + assert!( + !result.account_nodes_ref().iter().any(|(n, _)| n == &account_nibbles3), + "nibbles3 should not be in target_block updates" + ); + + // Verify storage trie updates + assert_eq!(result.storage_tries_ref().len(), 1, "Should have 1 storage trie"); + let storage_updates = result + .storage_tries_ref() + .get(&storage_address1) + .expect("Should have storage updates for address1"); + + assert_eq!(storage_updates.storage_nodes.len(), 2, "Should have 2 storage node updates"); + + // Check storage_nibbles1 - should have current value + let storage1_update = storage_updates + .storage_nodes + .iter() + .find(|(n, _)| n == &storage_nibbles1) + .expect("Should find storage_nibbles1"); + assert!(storage1_update.1.is_some(), "storage_nibbles1 should have a value"); + assert_eq!( + storage1_update.1.as_ref().unwrap().state_mask, + storage_node1.state_mask, + "storage_nibbles1 should have current value" + ); + + // Check storage_nibbles2 - was created in target_block, will be deleted in next_block + // So it should have a value (the value that will be deleted) + let storage2_update = storage_updates + .storage_nodes + .iter() + .find(|(n, _)| n == &storage_nibbles2) + .expect("Should find storage_nibbles2"); + assert!( + storage2_update.1.is_some(), + "storage_nibbles2 should have a value (the node that will be deleted in next block)" + ); + assert_eq!( + storage2_update.1.as_ref().unwrap().state_mask, + storage_node2.state_mask, + "storage_nibbles2 should have the value that was created and will be deleted" + ); + } } diff --git a/crates/storage/provider/src/providers/mod.rs b/crates/storage/provider/src/providers/mod.rs index 36843a22fba..41e8121991b 100644 --- a/crates/storage/provider/src/providers/mod.rs +++ b/crates/storage/provider/src/providers/mod.rs @@ -2,7 +2,7 @@ use reth_chainspec::EthereumHardforks; use reth_db_api::table::Value; -use reth_node_types::{FullNodePrimitives, NodeTypes, NodeTypesWithDB}; +use reth_node_types::{NodePrimitives, NodeTypes, NodeTypesWithDB}; mod database; pub use database::*; @@ -17,6 +17,7 @@ mod state; pub use state::{ historical::{HistoricalStateProvider, HistoricalStateProviderRef, LowestAvailableBlocks}, latest::{LatestStateProvider, LatestStateProviderRef}, + overlay::{OverlayStateProvider, OverlayStateProviderFactory}, }; mod consistent_view; @@ -35,7 +36,7 @@ where Self: NodeTypes< ChainSpec: EthereumHardforks, Storage: ChainStorage, - Primitives: FullNodePrimitives, + Primitives: NodePrimitives, >, { } @@ -44,7 +45,7 @@ impl NodeTypesForProvider for T where T: NodeTypes< ChainSpec: EthereumHardforks, Storage: ChainStorage, - Primitives: FullNodePrimitives, + Primitives: NodePrimitives, > { } diff --git a/crates/storage/provider/src/providers/state/historical.rs b/crates/storage/provider/src/providers/state/historical.rs index b39b5e20a68..666138fae7b 100644 --- a/crates/storage/provider/src/providers/state/historical.rs +++ b/crates/storage/provider/src/providers/state/historical.rs @@ -1,6 +1,6 @@ use crate::{ providers::state::macros::delegate_provider_impls, AccountReader, BlockHashReader, - HashedPostStateProvider, ProviderError, StateProvider, StateRootProvider, + ChangeSetReader, HashedPostStateProvider, ProviderError, StateProvider, StateRootProvider, }; use alloy_eips::merge::EPOCH_SLOTS; use alloy_primitives::{Address, BlockNumber, Bytes, StorageKey, StorageValue, B256}; @@ -14,20 +14,21 @@ use reth_db_api::{ }; use reth_primitives_traits::{Account, Bytecode}; use reth_storage_api::{ - BlockNumReader, DBProvider, StateCommitmentProvider, StateProofProvider, StorageRootProvider, + BlockNumReader, BytecodeReader, DBProvider, StateProofProvider, StorageRootProvider, }; use reth_storage_errors::provider::ProviderResult; use reth_trie::{ proof::{Proof, StorageProof}, updates::TrieUpdates, witness::TrieWitness, - AccountProof, HashedPostState, HashedStorage, MultiProof, MultiProofTargets, StateRoot, - StorageMultiProof, StorageRoot, TrieInput, + AccountProof, HashedPostState, HashedStorage, KeccakKeyHasher, MultiProof, MultiProofTargets, + StateRoot, StorageMultiProof, StorageRoot, TrieInput, }; use reth_trie_db::{ DatabaseHashedPostState, DatabaseHashedStorage, DatabaseProof, DatabaseStateRoot, - DatabaseStorageProof, DatabaseStorageRoot, DatabaseTrieWitness, StateCommitment, + DatabaseStorageProof, DatabaseStorageRoot, DatabaseTrieWitness, }; + use std::fmt::Debug; /// State provider for a given block number which takes a tx reference. @@ -59,9 +60,7 @@ pub enum HistoryInfo { MaybeInPlainState, } -impl<'b, Provider: DBProvider + BlockNumReader + StateCommitmentProvider> - HistoricalStateProviderRef<'b, Provider> -{ +impl<'b, Provider: DBProvider + BlockNumReader> HistoricalStateProviderRef<'b, Provider> { /// Create new `StateProvider` for historical block number pub fn new(provider: &'b Provider, block_number: BlockNumber) -> Self { Self { provider, block_number, lowest_available_blocks: Default::default() } @@ -134,9 +133,7 @@ impl<'b, Provider: DBProvider + BlockNumReader + StateCommitmentProvider> ); } - Ok(HashedPostState::from_reverts::< - ::KeyHasher, - >(self.tx(), self.block_number)?) + Ok(HashedPostState::from_reverts::(self.tx(), self.block_number..)?) } /// Retrieve revert hashed storage for this history provider and target address. @@ -244,23 +241,23 @@ impl HistoricalStateProviderRef<'_, Provi } } -impl AccountReader +impl AccountReader for HistoricalStateProviderRef<'_, Provider> { /// Get basic account information. fn basic_account(&self, address: &Address) -> ProviderResult> { match self.account_history_lookup(*address)? { HistoryInfo::NotYetWritten => Ok(None), - HistoryInfo::InChangeset(changeset_block_number) => Ok(self - .tx() - .cursor_dup_read::()? - .seek_by_key_subkey(changeset_block_number, *address)? - .filter(|acc| &acc.address == address) - .ok_or(ProviderError::AccountChangesetNotFound { - block_number: changeset_block_number, - address: *address, - })? - .info), + HistoryInfo::InChangeset(changeset_block_number) => { + // Use ChangeSetReader trait method to get the account from changesets + self.provider + .get_account_before_block(changeset_block_number, *address)? + .ok_or(ProviderError::AccountChangesetNotFound { + block_number: changeset_block_number, + address: *address, + }) + .map(|account_before| account_before.info) + } HistoryInfo::InPlainState | HistoryInfo::MaybeInPlainState => { Ok(self.tx().get_by_encoded_key::(address)?) } @@ -285,7 +282,7 @@ impl BlockHashReader } } -impl StateRootProvider +impl StateRootProvider for HistoricalStateProviderRef<'_, Provider> { fn state_root(&self, hashed_state: HashedPostState) -> ProviderResult { @@ -321,7 +318,7 @@ impl StateRootP } } -impl StorageRootProvider +impl StorageRootProvider for HistoricalStateProviderRef<'_, Provider> { fn storage_root( @@ -360,7 +357,7 @@ impl StorageRoo } } -impl StateProofProvider +impl StateProofProvider for HistoricalStateProviderRef<'_, Provider> { /// Get account and storage proofs. @@ -371,7 +368,8 @@ impl StateProof slots: &[B256], ) -> ProviderResult { input.prepend(self.revert_state()?); - Proof::overlay_account_proof(self.tx(), input, address, slots).map_err(ProviderError::from) + let proof = as DatabaseProof>::from_tx(self.tx()); + proof.overlay_account_proof(input, address, slots).map_err(ProviderError::from) } fn multiproof( @@ -380,7 +378,8 @@ impl StateProof targets: MultiProofTargets, ) -> ProviderResult { input.prepend(self.revert_state()?); - Proof::overlay_multiproof(self.tx(), input, targets).map_err(ProviderError::from) + let proof = as DatabaseProof>::from_tx(self.tx()); + proof.overlay_multiproof(input, targets).map_err(ProviderError::from) } fn witness(&self, mut input: TrieInput, target: HashedPostState) -> ProviderResult> { @@ -391,18 +390,14 @@ impl StateProof } } -impl HashedPostStateProvider - for HistoricalStateProviderRef<'_, Provider> -{ +impl HashedPostStateProvider for HistoricalStateProviderRef<'_, Provider> { fn hashed_post_state(&self, bundle_state: &revm_database::BundleState) -> HashedPostState { - HashedPostState::from_bundle_state::< - ::KeyHasher, - >(bundle_state.state()) + HashedPostState::from_bundle_state::(bundle_state.state()) } } -impl - StateProvider for HistoricalStateProviderRef<'_, Provider> +impl StateProvider + for HistoricalStateProviderRef<'_, Provider> { /// Get storage. fn storage( @@ -433,19 +428,17 @@ impl BytecodeReader + for HistoricalStateProviderRef<'_, Provider> +{ /// Get account code by its hash fn bytecode_by_hash(&self, code_hash: &B256) -> ProviderResult> { self.tx().get_by_encoded_key::(code_hash).map_err(Into::into) } } -impl StateCommitmentProvider - for HistoricalStateProviderRef<'_, Provider> -{ - type StateCommitment = Provider::StateCommitment; -} - /// State provider for a given block number. /// For more detailed description, see [`HistoricalStateProviderRef`]. #[derive(Debug)] @@ -458,9 +451,7 @@ pub struct HistoricalStateProvider { lowest_available_blocks: LowestAvailableBlocks, } -impl - HistoricalStateProvider -{ +impl HistoricalStateProvider { /// Create new `StateProvider` for historical block number pub fn new(provider: Provider, block_number: BlockNumber) -> Self { Self { provider, block_number, lowest_available_blocks: Default::default() } @@ -495,14 +486,8 @@ impl } } -impl StateCommitmentProvider - for HistoricalStateProvider -{ - type StateCommitment = Provider::StateCommitment; -} - // Delegates all provider impls to [HistoricalStateProviderRef] -delegate_provider_impls!(HistoricalStateProvider where [Provider: DBProvider + BlockNumReader + BlockHashReader + StateCommitmentProvider]); +delegate_provider_impls!(HistoricalStateProvider where [Provider: DBProvider + BlockNumReader + BlockHashReader + ChangeSetReader]); /// Lowest blocks at which different parts of the state are available. /// They may be [Some] if pruning is enabled. @@ -548,8 +533,7 @@ mod tests { }; use reth_primitives_traits::{Account, StorageEntry}; use reth_storage_api::{ - BlockHashReader, BlockNumReader, DBProvider, DatabaseProviderFactory, - StateCommitmentProvider, + BlockHashReader, BlockNumReader, ChangeSetReader, DBProvider, DatabaseProviderFactory, }; use reth_storage_errors::provider::ProviderError; @@ -561,7 +545,7 @@ mod tests { const fn assert_state_provider() {} #[expect(dead_code)] const fn assert_historical_state_provider< - T: DBProvider + BlockNumReader + BlockHashReader + StateCommitmentProvider, + T: DBProvider + BlockNumReader + BlockHashReader + ChangeSetReader, >() { assert_state_provider::>(); } diff --git a/crates/storage/provider/src/providers/state/latest.rs b/crates/storage/provider/src/providers/state/latest.rs index 8443e6b4c58..092feb37c43 100644 --- a/crates/storage/provider/src/providers/state/latest.rs +++ b/crates/storage/provider/src/providers/state/latest.rs @@ -5,20 +5,18 @@ use crate::{ use alloy_primitives::{Address, BlockNumber, Bytes, StorageKey, StorageValue, B256}; use reth_db_api::{cursor::DbDupCursorRO, tables, transaction::DbTx}; use reth_primitives_traits::{Account, Bytecode}; -use reth_storage_api::{ - DBProvider, StateCommitmentProvider, StateProofProvider, StorageRootProvider, -}; +use reth_storage_api::{BytecodeReader, DBProvider, StateProofProvider, StorageRootProvider}; use reth_storage_errors::provider::{ProviderError, ProviderResult}; use reth_trie::{ proof::{Proof, StorageProof}, updates::TrieUpdates, witness::TrieWitness, - AccountProof, HashedPostState, HashedStorage, MultiProof, MultiProofTargets, StateRoot, - StorageMultiProof, StorageRoot, TrieInput, + AccountProof, HashedPostState, HashedStorage, KeccakKeyHasher, MultiProof, MultiProofTargets, + StateRoot, StorageMultiProof, StorageRoot, TrieInput, }; use reth_trie_db::{ DatabaseProof, DatabaseStateRoot, DatabaseStorageProof, DatabaseStorageRoot, - DatabaseTrieWitness, StateCommitment, + DatabaseTrieWitness, }; /// State provider over latest state that takes tx reference. @@ -60,9 +58,7 @@ impl BlockHashReader for LatestStateProviderRef<'_, P } } -impl StateRootProvider - for LatestStateProviderRef<'_, Provider> -{ +impl StateRootProvider for LatestStateProviderRef<'_, Provider> { fn state_root(&self, hashed_state: HashedPostState) -> ProviderResult { StateRoot::overlay_root(self.tx(), hashed_state) .map_err(|err| ProviderError::Database(err.into())) @@ -90,9 +86,7 @@ impl StateRootProvider } } -impl StorageRootProvider - for LatestStateProviderRef<'_, Provider> -{ +impl StorageRootProvider for LatestStateProviderRef<'_, Provider> { fn storage_root( &self, address: Address, @@ -123,16 +117,15 @@ impl StorageRootProvider } } -impl StateProofProvider - for LatestStateProviderRef<'_, Provider> -{ +impl StateProofProvider for LatestStateProviderRef<'_, Provider> { fn proof( &self, input: TrieInput, address: Address, slots: &[B256], ) -> ProviderResult { - Proof::overlay_account_proof(self.tx(), input, address, slots).map_err(ProviderError::from) + let proof = as DatabaseProof>::from_tx(self.tx()); + proof.overlay_account_proof(input, address, slots).map_err(ProviderError::from) } fn multiproof( @@ -140,7 +133,8 @@ impl StateProofProvider input: TrieInput, targets: MultiProofTargets, ) -> ProviderResult { - Proof::overlay_multiproof(self.tx(), input, targets).map_err(ProviderError::from) + let proof = as DatabaseProof>::from_tx(self.tx()); + proof.overlay_multiproof(input, targets).map_err(ProviderError::from) } fn witness(&self, input: TrieInput, target: HashedPostState) -> ProviderResult> { @@ -150,17 +144,13 @@ impl StateProofProvider } } -impl HashedPostStateProvider - for LatestStateProviderRef<'_, Provider> -{ +impl HashedPostStateProvider for LatestStateProviderRef<'_, Provider> { fn hashed_post_state(&self, bundle_state: &revm_database::BundleState) -> HashedPostState { - HashedPostState::from_bundle_state::< - ::KeyHasher, - >(bundle_state.state()) + HashedPostState::from_bundle_state::(bundle_state.state()) } } -impl StateProvider +impl StateProvider for LatestStateProviderRef<'_, Provider> { /// Get storage. @@ -170,31 +160,29 @@ impl StateProv storage_key: StorageKey, ) -> ProviderResult> { let mut cursor = self.tx().cursor_dup_read::()?; - if let Some(entry) = cursor.seek_by_key_subkey(account, storage_key)? { - if entry.key == storage_key { - return Ok(Some(entry.value)) - } + if let Some(entry) = cursor.seek_by_key_subkey(account, storage_key)? && + entry.key == storage_key + { + return Ok(Some(entry.value)) } Ok(None) } +} +impl BytecodeReader + for LatestStateProviderRef<'_, Provider> +{ /// Get account code by its hash fn bytecode_by_hash(&self, code_hash: &B256) -> ProviderResult> { self.tx().get_by_encoded_key::(code_hash).map_err(Into::into) } } -impl StateCommitmentProvider - for LatestStateProviderRef<'_, Provider> -{ - type StateCommitment = Provider::StateCommitment; -} - /// State provider for the latest state. #[derive(Debug)] pub struct LatestStateProvider(Provider); -impl LatestStateProvider { +impl LatestStateProvider { /// Create new state provider pub const fn new(db: Provider) -> Self { Self(db) @@ -207,12 +195,8 @@ impl LatestStateProvider StateCommitmentProvider for LatestStateProvider { - type StateCommitment = Provider::StateCommitment; -} - // Delegates all provider impls to [LatestStateProviderRef] -delegate_provider_impls!(LatestStateProvider where [Provider: DBProvider + BlockHashReader + StateCommitmentProvider]); +delegate_provider_impls!(LatestStateProvider where [Provider: DBProvider + BlockHashReader ]); #[cfg(test)] mod tests { @@ -220,9 +204,7 @@ mod tests { const fn assert_state_provider() {} #[expect(dead_code)] - const fn assert_latest_state_provider< - T: DBProvider + BlockHashReader + StateCommitmentProvider, - >() { + const fn assert_latest_state_provider() { assert_state_provider::>(); } } diff --git a/crates/storage/provider/src/providers/state/macros.rs b/crates/storage/provider/src/providers/state/macros.rs index 36216755ec8..74bb371819f 100644 --- a/crates/storage/provider/src/providers/state/macros.rs +++ b/crates/storage/provider/src/providers/state/macros.rs @@ -39,6 +39,8 @@ macro_rules! delegate_provider_impls { } StateProvider $(where [$($generics)*])? { fn storage(&self, account: alloy_primitives::Address, storage_key: alloy_primitives::StorageKey) -> reth_storage_errors::provider::ProviderResult>; + } + BytecodeReader $(where [$($generics)*])? { fn bytecode_by_hash(&self, code_hash: &alloy_primitives::B256) -> reth_storage_errors::provider::ProviderResult>; } StateRootProvider $(where [$($generics)*])? { diff --git a/crates/storage/provider/src/providers/state/mod.rs b/crates/storage/provider/src/providers/state/mod.rs index 06a5fefb417..f26302531eb 100644 --- a/crates/storage/provider/src/providers/state/mod.rs +++ b/crates/storage/provider/src/providers/state/mod.rs @@ -2,3 +2,4 @@ pub(crate) mod historical; pub(crate) mod latest; pub(crate) mod macros; +pub(crate) mod overlay; diff --git a/crates/storage/provider/src/providers/state/overlay.rs b/crates/storage/provider/src/providers/state/overlay.rs new file mode 100644 index 00000000000..d3ef87e6c49 --- /dev/null +++ b/crates/storage/provider/src/providers/state/overlay.rs @@ -0,0 +1,325 @@ +use alloy_primitives::{BlockNumber, B256}; +use reth_db_api::DatabaseError; +use reth_errors::{ProviderError, ProviderResult}; +use reth_prune_types::PruneSegment; +use reth_stages_types::StageId; +use reth_storage_api::{ + BlockNumReader, DBProvider, DatabaseProviderFactory, DatabaseProviderROFactory, + PruneCheckpointReader, StageCheckpointReader, TrieReader, +}; +use reth_trie::{ + hashed_cursor::{HashedCursorFactory, HashedPostStateCursorFactory}, + trie_cursor::{InMemoryTrieCursorFactory, TrieCursorFactory}, + updates::TrieUpdatesSorted, + HashedPostState, HashedPostStateSorted, KeccakKeyHasher, +}; +use reth_trie_db::{ + DatabaseHashedCursorFactory, DatabaseHashedPostState, DatabaseTrieCursorFactory, +}; +use std::sync::Arc; +use tracing::debug; + +/// Factory for creating overlay state providers with optional reverts and overlays. +/// +/// This factory allows building an `OverlayStateProvider` whose DB state has been reverted to a +/// particular block, and/or with additional overlay information added on top. +#[derive(Debug, Clone)] +pub struct OverlayStateProviderFactory { + /// The underlying database provider factory + factory: F, + /// Optional block hash for collecting reverts + block_hash: Option, + /// Optional trie overlay + trie_overlay: Option>, + /// Optional hashed state overlay + hashed_state_overlay: Option>, +} + +impl OverlayStateProviderFactory { + /// Create a new overlay state provider factory + pub const fn new(factory: F) -> Self { + Self { factory, block_hash: None, trie_overlay: None, hashed_state_overlay: None } + } + + /// Set the block hash for collecting reverts. All state will be reverted to the point + /// _after_ this block has been processed. + pub const fn with_block_hash(mut self, block_hash: Option) -> Self { + self.block_hash = block_hash; + self + } + + /// Set the trie overlay. + /// + /// This overlay will be applied on top of any reverts applied via `with_block_hash`. + pub fn with_trie_overlay(mut self, trie_overlay: Option>) -> Self { + self.trie_overlay = trie_overlay; + self + } + + /// Set the hashed state overlay + /// + /// This overlay will be applied on top of any reverts applied via `with_block_hash`. + pub fn with_hashed_state_overlay( + mut self, + hashed_state_overlay: Option>, + ) -> Self { + self.hashed_state_overlay = hashed_state_overlay; + self + } +} + +impl OverlayStateProviderFactory +where + F: DatabaseProviderFactory, + F::Provider: TrieReader + StageCheckpointReader + PruneCheckpointReader + BlockNumReader, +{ + /// Returns the block number for [`Self`]'s `block_hash` field, if any. + fn get_block_number(&self, provider: &F::Provider) -> ProviderResult> { + if let Some(block_hash) = self.block_hash { + Ok(Some( + provider + .convert_hash_or_number(block_hash.into())? + .ok_or_else(|| ProviderError::BlockHashNotFound(block_hash))?, + )) + } else { + Ok(None) + } + } + + /// Returns whether or not it is required to collect reverts, and validates that there are + /// sufficient changesets to revert to the requested block number if so. + /// + /// Returns an error if the `MerkleChangeSets` checkpoint doesn't cover the requested block. + /// Takes into account both the stage checkpoint and the prune checkpoint to determine the + /// available data range. + fn reverts_required( + &self, + provider: &F::Provider, + requested_block: BlockNumber, + ) -> ProviderResult { + // Get the MerkleChangeSets stage and prune checkpoints. + let stage_checkpoint = provider.get_stage_checkpoint(StageId::MerkleChangeSets)?; + let prune_checkpoint = provider.get_prune_checkpoint(PruneSegment::MerkleChangeSets)?; + + // Get the upper bound from stage checkpoint + let upper_bound = + stage_checkpoint.as_ref().map(|chk| chk.block_number).ok_or_else(|| { + ProviderError::InsufficientChangesets { + requested: requested_block, + available: 0..=0, + } + })?; + + // If the requested block is the DB tip (determined by the MerkleChangeSets stage + // checkpoint) then there won't be any reverts necessary, and we can simply return Ok. + if upper_bound == requested_block { + return Ok(false) + } + + // Extract the lower bound from prune checkpoint if available. + // + // If not available we assume pruning has never ran and so there is no lower bound. This + // should not generally happen, since MerkleChangeSets always have pruning enabled, but when + // starting a new node from scratch (e.g. in a test case or benchmark) it can surface. + // + // The prune checkpoint's block_number is the highest pruned block, so data is available + // starting from the next block + let lower_bound = prune_checkpoint + .and_then(|chk| chk.block_number) + .map(|block_number| block_number + 1) + .unwrap_or_default(); + + let available_range = lower_bound..=upper_bound; + + // Check if the requested block is within the available range + if !available_range.contains(&requested_block) { + return Err(ProviderError::InsufficientChangesets { + requested: requested_block, + available: available_range, + }); + } + + Ok(true) + } +} + +impl DatabaseProviderROFactory for OverlayStateProviderFactory +where + F: DatabaseProviderFactory, + F::Provider: TrieReader + StageCheckpointReader + PruneCheckpointReader + BlockNumReader, +{ + type Provider = OverlayStateProvider; + + /// Create a read-only [`OverlayStateProvider`]. + fn database_provider_ro(&self) -> ProviderResult> { + // Get a read-only provider + let provider = self.factory.database_provider_ro()?; + + // If block_hash is provided, collect reverts + let (trie_updates, hashed_state) = if let Some(from_block) = + self.get_block_number(&provider)? && + self.reverts_required(&provider, from_block)? + { + // Collect trie reverts + let mut trie_reverts = provider.trie_reverts(from_block + 1)?; + + // Collect state reverts + // + // TODO(mediocregopher) make from_reverts return sorted + // https://github.com/paradigmxyz/reth/issues/19382 + let mut hashed_state_reverts = HashedPostState::from_reverts::( + provider.tx_ref(), + from_block + 1.., + )? + .into_sorted(); + + // Extend with overlays if provided. If the reverts are empty we should just use the + // overlays directly, because `extend_ref` will actually clone the overlay. + let trie_updates = match self.trie_overlay.as_ref() { + Some(trie_overlay) if trie_reverts.is_empty() => Arc::clone(trie_overlay), + Some(trie_overlay) => { + trie_reverts.extend_ref(trie_overlay); + Arc::new(trie_reverts) + } + None => Arc::new(trie_reverts), + }; + + let hashed_state_updates = match self.hashed_state_overlay.as_ref() { + Some(hashed_state_overlay) if hashed_state_reverts.is_empty() => { + Arc::clone(hashed_state_overlay) + } + Some(hashed_state_overlay) => { + hashed_state_reverts.extend_ref(hashed_state_overlay); + Arc::new(hashed_state_reverts) + } + None => Arc::new(hashed_state_reverts), + }; + + debug!( + target: "providers::state::overlay", + block_hash = ?self.block_hash, + ?from_block, + num_trie_updates = ?trie_updates.total_len(), + num_state_updates = ?hashed_state_updates.total_len(), + "Reverted to target block", + ); + + (trie_updates, hashed_state_updates) + } else { + // If no block_hash, use overlays directly or defaults + let trie_updates = + self.trie_overlay.clone().unwrap_or_else(|| Arc::new(TrieUpdatesSorted::default())); + let hashed_state = self + .hashed_state_overlay + .clone() + .unwrap_or_else(|| Arc::new(HashedPostStateSorted::default())); + + (trie_updates, hashed_state) + }; + + Ok(OverlayStateProvider::new(provider, trie_updates, hashed_state)) + } +} + +/// State provider with in-memory overlay from trie updates and hashed post state. +/// +/// This provider uses in-memory trie updates and hashed post state as an overlay +/// on top of a database provider, implementing [`TrieCursorFactory`] and [`HashedCursorFactory`] +/// using the in-memory overlay factories. +#[derive(Debug)] +pub struct OverlayStateProvider { + provider: Provider, + trie_updates: Arc, + hashed_post_state: Arc, +} + +impl OverlayStateProvider +where + Provider: DBProvider, +{ + /// Create new overlay state provider. The `Provider` must be cloneable, which generally means + /// it should be wrapped in an `Arc`. + pub const fn new( + provider: Provider, + trie_updates: Arc, + hashed_post_state: Arc, + ) -> Self { + Self { provider, trie_updates, hashed_post_state } + } +} + +impl TrieCursorFactory for OverlayStateProvider +where + Provider: DBProvider, +{ + type AccountTrieCursor<'a> + = , + &'a TrieUpdatesSorted, + > as TrieCursorFactory>::AccountTrieCursor<'a> + where + Self: 'a; + + type StorageTrieCursor<'a> + = , + &'a TrieUpdatesSorted, + > as TrieCursorFactory>::StorageTrieCursor<'a> + where + Self: 'a; + + fn account_trie_cursor(&self) -> Result, DatabaseError> { + let db_trie_cursor_factory = DatabaseTrieCursorFactory::new(self.provider.tx_ref()); + let trie_cursor_factory = + InMemoryTrieCursorFactory::new(db_trie_cursor_factory, self.trie_updates.as_ref()); + trie_cursor_factory.account_trie_cursor() + } + + fn storage_trie_cursor( + &self, + hashed_address: B256, + ) -> Result, DatabaseError> { + let db_trie_cursor_factory = DatabaseTrieCursorFactory::new(self.provider.tx_ref()); + let trie_cursor_factory = + InMemoryTrieCursorFactory::new(db_trie_cursor_factory, self.trie_updates.as_ref()); + trie_cursor_factory.storage_trie_cursor(hashed_address) + } +} + +impl HashedCursorFactory for OverlayStateProvider +where + Provider: DBProvider, +{ + type AccountCursor<'a> + = , + &'a Arc, + > as HashedCursorFactory>::AccountCursor<'a> + where + Self: 'a; + + type StorageCursor<'a> + = , + &'a Arc, + > as HashedCursorFactory>::StorageCursor<'a> + where + Self: 'a; + + fn hashed_account_cursor(&self) -> Result, DatabaseError> { + let db_hashed_cursor_factory = DatabaseHashedCursorFactory::new(self.provider.tx_ref()); + let hashed_cursor_factory = + HashedPostStateCursorFactory::new(db_hashed_cursor_factory, &self.hashed_post_state); + hashed_cursor_factory.hashed_account_cursor() + } + + fn hashed_storage_cursor( + &self, + hashed_address: B256, + ) -> Result, DatabaseError> { + let db_hashed_cursor_factory = DatabaseHashedCursorFactory::new(self.provider.tx_ref()); + let hashed_cursor_factory = + HashedPostStateCursorFactory::new(db_hashed_cursor_factory, &self.hashed_post_state); + hashed_cursor_factory.hashed_storage_cursor(hashed_address) + } +} diff --git a/crates/storage/provider/src/providers/static_file/jar.rs b/crates/storage/provider/src/providers/static_file/jar.rs index 64b0d6e284b..2cd7ec98ae9 100644 --- a/crates/storage/provider/src/providers/static_file/jar.rs +++ b/crates/storage/provider/src/providers/static_file/jar.rs @@ -6,21 +6,16 @@ use crate::{ to_range, BlockHashReader, BlockNumReader, HeaderProvider, ReceiptProvider, TransactionsProvider, }; -use alloy_consensus::transaction::TransactionMeta; -use alloy_eips::{eip2718::Encodable2718, eip4895::Withdrawals, BlockHashOrNumber}; -use alloy_primitives::{Address, BlockHash, BlockNumber, TxHash, TxNumber, B256, U256}; +use alloy_consensus::transaction::{SignerRecoverable, TransactionMeta}; +use alloy_eips::{eip2718::Encodable2718, BlockHashOrNumber}; +use alloy_primitives::{Address, BlockHash, BlockNumber, TxHash, TxNumber, B256}; use reth_chainspec::ChainInfo; use reth_db::static_file::{ - BlockHashMask, BodyIndicesMask, HeaderMask, HeaderWithHashMask, OmmersMask, ReceiptMask, - StaticFileCursor, TDWithHashMask, TotalDifficultyMask, TransactionMask, WithdrawalsMask, + BlockHashMask, HeaderMask, HeaderWithHashMask, ReceiptMask, StaticFileCursor, TransactionMask, }; -use reth_db_api::{ - models::StoredBlockBodyIndices, - table::{Decompress, Value}, -}; -use reth_node_types::{FullNodePrimitives, NodePrimitives}; +use reth_db_api::table::{Decompress, Value}; +use reth_node_types::NodePrimitives; use reth_primitives_traits::{SealedHeader, SignedTransaction}; -use reth_storage_api::{BlockBodyIndicesProvider, OmmersProvider, WithdrawalsProvider}; use reth_storage_errors::provider::{ProviderError, ProviderResult}; use std::{ fmt::Debug, @@ -93,11 +88,11 @@ impl<'a, N: NodePrimitives> StaticFileJarProvider<'a, N> { impl> HeaderProvider for StaticFileJarProvider<'_, N> { type Header = N::BlockHeader; - fn header(&self, block_hash: &BlockHash) -> ProviderResult> { + fn header(&self, block_hash: BlockHash) -> ProviderResult> { Ok(self .cursor()? - .get_two::>(block_hash.into())? - .filter(|(_, hash)| hash == block_hash) + .get_two::>((&block_hash).into())? + .filter(|(_, hash)| hash == &block_hash) .map(|(header, _)| header)) } @@ -105,18 +100,6 @@ impl> HeaderProvider for StaticFileJarProv self.cursor()?.get_one::>(num.into()) } - fn header_td(&self, block_hash: &BlockHash) -> ProviderResult> { - Ok(self - .cursor()? - .get_two::(block_hash.into())? - .filter(|(_, hash)| hash == block_hash) - .map(|(td, _)| td.into())) - } - - fn header_td_by_number(&self, num: BlockNumber) -> ProviderResult> { - Ok(self.cursor()?.get_one::(num.into())?.map(Into::into)) - } - fn headers_range( &self, range: impl RangeBounds, @@ -318,10 +301,10 @@ impl ProviderResult> { - if let Some(tx_static_file) = &self.auxiliary_jar { - if let Some(num) = tx_static_file.transaction_id(hash)? { - return self.receipt(num) - } + if let Some(tx_static_file) = &self.auxiliary_jar && + let Some(num) = tx_static_file.transaction_id(hash)? + { + return self.receipt(num) } Ok(None) } @@ -350,55 +333,13 @@ impl WithdrawalsProvider for StaticFileJarProvider<'_, N> { - fn withdrawals_by_block( + fn receipts_by_block_range( &self, - id: BlockHashOrNumber, - _: u64, - ) -> ProviderResult> { - if let Some(num) = id.as_number() { - return Ok(self - .cursor()? - .get_one::(num.into())? - .and_then(|s| s.withdrawals)) - } - // Only accepts block number queries - Err(ProviderError::UnsupportedProvider) - } -} - -impl> OmmersProvider for StaticFileJarProvider<'_, N> { - fn ommers(&self, id: BlockHashOrNumber) -> ProviderResult>> { - if let Some(num) = id.as_number() { - return Ok(self - .cursor()? - .get_one::>(num.into())? - .map(|s| s.ommers)) - } - // Only accepts block number queries + _block_range: RangeInclusive, + ) -> ProviderResult>> { + // Related to indexing tables. StaticFile should get the tx_range and call static file + // provider with `receipt()` instead for each Err(ProviderError::UnsupportedProvider) } } - -impl BlockBodyIndicesProvider for StaticFileJarProvider<'_, N> { - fn block_body_indices(&self, num: u64) -> ProviderResult> { - self.cursor()?.get_one::(num.into()) - } - - fn block_body_indices_range( - &self, - range: RangeInclusive, - ) -> ProviderResult> { - let mut cursor = self.cursor()?; - let mut indices = Vec::with_capacity((range.end() - range.start() + 1) as usize); - - for num in range { - if let Some(block) = cursor.get_one::(num.into())? { - indices.push(block) - } - } - Ok(indices) - } -} diff --git a/crates/storage/provider/src/providers/static_file/manager.rs b/crates/storage/provider/src/providers/static_file/manager.rs index 9804697db8f..f9f0e688687 100644 --- a/crates/storage/provider/src/providers/static_file/manager.rs +++ b/crates/storage/provider/src/providers/static_file/manager.rs @@ -5,22 +5,23 @@ use super::{ use crate::{ to_range, BlockHashReader, BlockNumReader, BlockReader, BlockSource, HeaderProvider, ReceiptProvider, StageCheckpointReader, StatsReader, TransactionVariant, TransactionsProvider, - TransactionsProviderExt, WithdrawalsProvider, + TransactionsProviderExt, }; -use alloy_consensus::{transaction::TransactionMeta, Header}; -use alloy_eips::{eip2718::Encodable2718, eip4895::Withdrawals, BlockHashOrNumber}; -use alloy_primitives::{ - b256, keccak256, Address, BlockHash, BlockNumber, TxHash, TxNumber, B256, U256, +use alloy_consensus::{ + transaction::{SignerRecoverable, TransactionMeta}, + Header, }; +use alloy_eips::{eip2718::Encodable2718, BlockHashOrNumber}; +use alloy_primitives::{b256, keccak256, Address, BlockHash, BlockNumber, TxHash, TxNumber, B256}; use dashmap::DashMap; use notify::{RecommendedWatcher, RecursiveMode, Watcher}; use parking_lot::RwLock; -use reth_chainspec::{ChainInfo, ChainSpecProvider, EthChainSpec}; +use reth_chainspec::{ChainInfo, ChainSpecProvider, EthChainSpec, NamedChain}; use reth_db::{ lockfile::StorageLock, static_file::{ - iter_static_files, BlockHashMask, BodyIndicesMask, HeaderMask, HeaderWithHashMask, - ReceiptMask, StaticFileCursor, TDWithHashMask, TransactionMask, + iter_static_files, BlockHashMask, HeaderMask, HeaderWithHashMask, ReceiptMask, + StaticFileCursor, TransactionMask, }, }; use reth_db_api::{ @@ -32,14 +33,14 @@ use reth_db_api::{ }; use reth_ethereum_primitives::{Receipt, TransactionSigned}; use reth_nippy_jar::{NippyJar, NippyJarChecker, CONFIG_FILE_EXTENSION}; -use reth_node_types::{FullNodePrimitives, NodePrimitives}; -use reth_primitives_traits::{RecoveredBlock, SealedBlock, SealedHeader, SignedTransaction}; +use reth_node_types::NodePrimitives; +use reth_primitives_traits::{RecoveredBlock, SealedHeader, SignedTransaction}; use reth_stages_types::{PipelineTarget, StageId}; use reth_static_file_types::{ find_fixed_range, HighestStaticFiles, SegmentHeader, SegmentRangeInclusive, StaticFileSegment, DEFAULT_BLOCKS_PER_STATIC_FILE, }; -use reth_storage_api::{BlockBodyIndicesProvider, DBProvider, OmmersProvider}; +use reth_storage_api::{BlockBodyIndicesProvider, DBProvider}; use reth_storage_errors::provider::{ProviderError, ProviderResult}; use std::{ collections::{hash_map::Entry, BTreeMap, HashMap}, @@ -47,9 +48,9 @@ use std::{ marker::PhantomData, ops::{Deref, Range, RangeBounds, RangeInclusive}, path::{Path, PathBuf}, - sync::{mpsc, Arc}, + sync::{atomic::AtomicU64, mpsc, Arc}, }; -use tracing::{info, trace, warn}; +use tracing::{debug, info, trace, warn}; /// Alias type for a map that can be queried for block ranges from a transaction /// segment respectively. It uses `TxNumber` to represent the transaction end of a static file @@ -226,6 +227,27 @@ pub struct StaticFileProviderInner { /// Maintains a map which allows for concurrent access to different `NippyJars`, over different /// segments and ranges. map: DashMap<(BlockNumber, StaticFileSegment), LoadedJar>, + /// Min static file range for each segment. + /// This index is initialized on launch to keep track of the lowest, non-expired static file + /// per segment and gets updated on `Self::update_index()`. + /// + /// This tracks the lowest static file per segment together with the block range in that + /// file. E.g. static file is batched in 500k block intervals then the lowest static file + /// is [0..499K], and the block range is start = 0, end = 499K. + /// This index is mainly used to History expiry, which targets transactions, e.g. pre-merge + /// history expiry would lead to removing all static files below the merge height. + static_files_min_block: RwLock>, + /// This is an additional index that tracks the expired height, this will track the highest + /// block number that has been expired (missing). The first, non expired block is + /// `expired_history_height + 1`. + /// + /// This is effectively the transaction range that has been expired: + /// [`StaticFileProvider::delete_segment_below_block`] and mirrors + /// `static_files_min_block[transactions] - blocks_per_file`. + /// + /// This additional tracker exists for more efficient lookups because the node must be aware of + /// the expired height. + earliest_history_height: AtomicU64, /// Max static file block for each segment static_files_max_block: RwLock>, /// Available static file block ranges on disk indexed by max transactions. @@ -258,6 +280,8 @@ impl StaticFileProviderInner { let provider = Self { map: Default::default(), writers: Default::default(), + static_files_min_block: Default::default(), + earliest_history_height: Default::default(), static_files_max_block: Default::default(), static_files_tx_index: Default::default(), path: path.as_ref().to_path_buf(), @@ -419,28 +443,98 @@ impl StaticFileProvider { self.map.remove(&(fixed_block_range_end, segment)); } + /// This handles history expiry by deleting all static files for the given segment below the + /// given block. + /// + /// For example if block is 1M and the blocks per file are 500K this will delete all individual + /// files below 1M, so 0-499K and 500K-999K. + /// + /// This will not delete the file that contains the block itself, because files can only be + /// removed entirely. + /// + /// # Safety + /// + /// This method will never delete the highest static file for the segment, even if the + /// requested block is higher than the highest block in static files. This ensures we always + /// maintain at least one static file if any exist. + /// + /// Returns a list of `SegmentHeader`s from the deleted jars. + pub fn delete_segment_below_block( + &self, + segment: StaticFileSegment, + block: BlockNumber, + ) -> ProviderResult> { + // Nothing to delete if block is 0. + if block == 0 { + return Ok(Vec::new()) + } + + let highest_block = self.get_highest_static_file_block(segment); + let mut deleted_headers = Vec::new(); + + loop { + let Some(block_height) = self.get_lowest_static_file_block(segment) else { + return Ok(deleted_headers) + }; + + // Stop if we've reached the target block or the highest static file + if block_height >= block || Some(block_height) == highest_block { + return Ok(deleted_headers) + } + + debug!( + target: "provider::static_file", + ?segment, + ?block_height, + "Deleting static file below block" + ); + + // now we need to wipe the static file, this will take care of updating the index and + // advance the lowest tracked block height for the segment. + let header = self.delete_jar(segment, block_height).inspect_err(|err| { + warn!( target: "provider::static_file", ?segment, %block_height, ?err, "Failed to delete static file below block") + })?; + + deleted_headers.push(header); + } + } + /// Given a segment and block, it deletes the jar and all files from the respective block range. /// /// CAUTION: destructive. Deletes files on disk. - pub fn delete_jar(&self, segment: StaticFileSegment, block: BlockNumber) -> ProviderResult<()> { + /// + /// This will re-initialize the index after deletion, so all files are tracked. + /// + /// Returns the `SegmentHeader` of the deleted jar. + pub fn delete_jar( + &self, + segment: StaticFileSegment, + block: BlockNumber, + ) -> ProviderResult { let fixed_block_range = self.find_fixed_range(block); let key = (fixed_block_range.end(), segment); let jar = if let Some((_, jar)) = self.map.remove(&key) { jar.jar } else { - NippyJar::::load(&self.path.join(segment.filename(&fixed_block_range))) - .map_err(ProviderError::other)? + let file = self.path.join(segment.filename(&fixed_block_range)); + debug!( + target: "provider::static_file", + ?file, + ?fixed_block_range, + ?block, + "Loading static file jar for deletion" + ); + NippyJar::::load(&file).map_err(ProviderError::other)? }; + let header = jar.user_header().clone(); jar.delete().map_err(ProviderError::other)?; - let mut segment_max_block = None; - if fixed_block_range.start() > 0 { - segment_max_block = Some(fixed_block_range.start() - 1) - }; - self.update_index(segment, segment_max_block)?; + // SAFETY: this is currently necessary to ensure that certain indexes like + // `static_files_min_block` have the correct values after pruning. + self.initialize_index()?; - Ok(()) + Ok(header) } /// Given a segment and block range it returns a cached @@ -523,6 +617,7 @@ impl StaticFileProvider { segment: StaticFileSegment, segment_max_block: Option, ) -> ProviderResult<()> { + let mut min_block = self.static_files_min_block.write(); let mut max_block = self.static_files_max_block.write(); let mut tx_index = self.static_files_tx_index.write(); @@ -537,6 +632,34 @@ impl StaticFileProvider { ) .map_err(ProviderError::other)?; + // Update min_block to track the lowest block range of the segment. + // This is initially set by initialize_index() on node startup, but must be updated + // as the file grows to prevent stale values. + // + // Without this update, min_block can remain at genesis (e.g. Some([0..=0]) or None) + // even after syncing to higher blocks (e.g. [0..=100]). A stale + // min_block causes get_lowest_static_file_block() to return the + // wrong end value, which breaks pruning logic that relies on it for + // safety checks. + // + // Example progression: + // 1. Node starts, initialize_index() sets min_block = [0..=0] + // 2. Sync to block 100, this update sets min_block = [0..=100] + // 3. Pruner calls get_lowest_static_file_block() -> returns 100 (correct). Without + // this update, it would incorrectly return 0 (stale) + if let Some(current_block_range) = jar.user_header().block_range().copied() { + min_block + .entry(segment) + .and_modify(|current_min| { + // delete_jar WILL ALWAYS re-initialize all indexes, so we are always + // sure that current_min is always the lowest. + if current_block_range.start() == current_min.start() { + *current_min = current_block_range; + } + }) + .or_insert(current_block_range); + } + // Updates the tx index by first removing all entries which have a higher // block_start than our current static file. if let Some(tx_range) = jar.user_header().tx_range() { @@ -586,6 +709,7 @@ impl StaticFileProvider { None => { tx_index.remove(&segment); max_block.remove(&segment); + min_block.remove(&segment); } }; @@ -594,16 +718,21 @@ impl StaticFileProvider { /// Initializes the inner transaction and block index pub fn initialize_index(&self) -> ProviderResult<()> { + let mut min_block = self.static_files_min_block.write(); let mut max_block = self.static_files_max_block.write(); let mut tx_index = self.static_files_tx_index.write(); + min_block.clear(); max_block.clear(); tx_index.clear(); for (segment, ranges) in iter_static_files(&self.path).map_err(ProviderError::other)? { - // Update last block for each segment - if let Some((block_range, _)) = ranges.last() { - max_block.insert(segment, block_range.end()); + // Update first and last block for each segment + if let Some((first_block_range, _)) = ranges.first() { + min_block.insert(segment, *first_block_range); + } + if let Some((last_block_range, _)) = ranges.last() { + max_block.insert(segment, last_block_range.end()); } // Update tx -> block_range index @@ -626,6 +755,13 @@ impl StaticFileProvider { // If this is a re-initialization, we need to clear this as well self.map.clear(); + // initialize the expired history height to the lowest static file block + if let Some(lowest_range) = min_block.get(&StaticFileSegment::Transactions) { + // the earliest height is the lowest available block number + self.earliest_history_height + .store(lowest_range.start(), std::sync::atomic::Ordering::Relaxed); + } + Ok(()) } @@ -693,16 +829,21 @@ impl StaticFileProvider { }; for segment in StaticFileSegment::iter() { - // Not integrated yet - if segment.is_block_meta() { - continue - } - if has_receipt_pruning && segment.is_receipts() { // Pruned nodes (including full node) do not store receipts as static files. continue } + if segment.is_receipts() && + (NamedChain::Gnosis == provider.chain_spec().chain_id() || + NamedChain::Chiado == provider.chain_spec().chain_id()) + { + // Gnosis and Chiado's historical import is broken and does not work with this + // check. They are importing receipts along with importing + // headers/bodies. + continue; + } + let initial_highest_block = self.get_highest_static_file_block(segment); // File consistency is broken if: @@ -794,13 +935,6 @@ impl StaticFileProvider { highest_tx, highest_block, )?, - StaticFileSegment::BlockMeta => self - .ensure_invariants::<_, tables::BlockBodyIndices>( - provider, - segment, - highest_block, - highest_block, - )?, } { update_unwind_target(unwind); } @@ -869,12 +1003,11 @@ impl StaticFileProvider { } } - if let Some((db_last_entry, _)) = db_cursor.last()? { - if highest_static_file_entry + if let Some((db_last_entry, _)) = db_cursor.last()? && + highest_static_file_entry .is_none_or(|highest_entry| db_last_entry > highest_entry) - { - return Ok(None) - } + { + return Ok(None) } } @@ -886,7 +1019,7 @@ impl StaticFileProvider { let checkpoint_block_number = provider .get_stage_checkpoint(match segment { StaticFileSegment::Headers => StageId::Headers, - StaticFileSegment::Transactions | StaticFileSegment::BlockMeta => StageId::Bodies, + StaticFileSegment::Transactions => StageId::Bodies, StaticFileSegment::Receipts => StageId::Execution, })? .unwrap_or_default() @@ -935,7 +1068,43 @@ impl StaticFileProvider { Ok(None) } - /// Gets the highest static file block if it exists for a static file segment. + /// Returns the earliest available block number that has not been expired and is still + /// available. + /// + /// This means that the highest expired block (or expired block height) is + /// `earliest_history_height.saturating_sub(1)`. + /// + /// Returns `0` if no history has been expired. + pub fn earliest_history_height(&self) -> BlockNumber { + self.earliest_history_height.load(std::sync::atomic::Ordering::Relaxed) + } + + /// Gets the lowest transaction static file block if it exists. + /// + /// For example if the transactions static file has blocks 0-499, this will return 499.. + /// + /// If there is nothing on disk for the given segment, this will return [`None`]. + pub fn get_lowest_transaction_static_file_block(&self) -> Option { + self.get_lowest_static_file_block(StaticFileSegment::Transactions) + } + + /// Gets the lowest static file's block height if it exists for a static file segment. + /// + /// For example if the static file has blocks 0-499, this will return 499.. + /// + /// If there is nothing on disk for the given segment, this will return [`None`]. + pub fn get_lowest_static_file_block(&self, segment: StaticFileSegment) -> Option { + self.static_files_min_block.read().get(&segment).map(|range| range.end()) + } + + /// Gets the lowest static file's block range if it exists for a static file segment. + /// + /// If there is nothing on disk for the given segment, this will return [`None`]. + pub fn get_lowest_range(&self, segment: StaticFileSegment) -> Option { + self.static_files_min_block.read().get(&segment).copied() + } + + /// Gets the highest static file's block height if it exists for a static file segment. /// /// If there is nothing on disk for the given segment, this will return [`None`]. pub fn get_highest_static_file_block(&self, segment: StaticFileSegment) -> Option { @@ -955,10 +1124,7 @@ impl StaticFileProvider { /// Gets the highest static file block for all segments. pub fn get_highest_static_files(&self) -> HighestStaticFiles { HighestStaticFiles { - headers: self.get_highest_static_file_block(StaticFileSegment::Headers), receipts: self.get_highest_static_file_block(StaticFileSegment::Receipts), - transactions: self.get_highest_static_file_block(StaticFileSegment::Transactions), - block_meta: self.get_highest_static_file_block(StaticFileSegment::BlockMeta), } } @@ -1001,16 +1167,31 @@ impl StaticFileProvider { F: FnMut(&mut StaticFileCursor<'_>, u64) -> ProviderResult>, P: FnMut(&T) -> bool, { - let get_provider = |start: u64| { - if segment.is_block_based() { - self.get_segment_provider_from_block(segment, start, None) - } else { - self.get_segment_provider_from_transaction(segment, start, None) - } - }; - let mut result = Vec::with_capacity((range.end - range.start).min(100) as usize); - let mut provider = get_provider(range.start)?; + + /// Resolves to the provider for the given block or transaction number. + /// + /// If the static file is missing, the `result` is returned. + macro_rules! get_provider { + ($number:expr) => {{ + let provider = if segment.is_block_based() { + self.get_segment_provider_from_block(segment, $number, None) + } else { + self.get_segment_provider_from_transaction(segment, $number, None) + }; + + match provider { + Ok(provider) => provider, + Err( + ProviderError::MissingStaticFileBlock(_, _) | + ProviderError::MissingStaticFileTx(_, _), + ) => return Ok(result), + Err(err) => return Err(err), + } + }}; + } + + let mut provider = get_provider!(range.start); let mut cursor = provider.cursor()?; // advances number in range @@ -1032,19 +1213,7 @@ impl StaticFileProvider { } None => { if retrying { - warn!( - target: "provider::static_file", - ?segment, - ?number, - "Could not find block or tx number on a range request" - ); - - let err = if segment.is_block_based() { - ProviderError::MissingStaticFileBlock(segment, number) - } else { - ProviderError::MissingStaticFileTx(segment, number) - }; - return Err(err) + return Ok(result) } // There is a very small chance of hitting a deadlock if two consecutive // static files share the same bucket in the @@ -1052,7 +1221,7 @@ impl StaticFileProvider { // before requesting the next one. drop(cursor); drop(provider); - provider = get_provider(number)?; + provider = get_provider!(number); cursor = provider.cursor()?; retrying = true; } @@ -1172,16 +1341,15 @@ impl StaticFileProvider { self.get_highest_static_file_block(segment) } else { self.get_highest_static_file_tx(segment) - } { - if block_or_tx_range.start <= static_file_upper_bound { - let end = block_or_tx_range.end.min(static_file_upper_bound + 1); - data.extend(fetch_from_static_file( - self, - block_or_tx_range.start..end, - &mut predicate, - )?); - block_or_tx_range.start = end; - } + } && block_or_tx_range.start <= static_file_upper_bound + { + let end = block_or_tx_range.end.min(static_file_upper_bound + 1); + data.extend(fetch_from_static_file( + self, + block_or_tx_range.start..end, + &mut predicate, + )?); + block_or_tx_range.start = end; } if block_or_tx_range.end > block_or_tx_range.start { @@ -1225,6 +1393,9 @@ pub trait StaticFileWriter { /// Commits all changes of all [`StaticFileProviderRW`] of all [`StaticFileSegment`]. fn commit(&self) -> ProviderResult<()>; + + /// Returns `true` if the static file provider has unwind queued. + fn has_unwind_queued(&self) -> bool; } impl StaticFileWriter for StaticFileProvider { @@ -1255,18 +1426,22 @@ impl StaticFileWriter for StaticFileProvider { fn commit(&self) -> ProviderResult<()> { self.writers.commit() } + + fn has_unwind_queued(&self) -> bool { + self.writers.has_unwind_queued() + } } impl> HeaderProvider for StaticFileProvider { type Header = N::BlockHeader; - fn header(&self, block_hash: &BlockHash) -> ProviderResult> { + fn header(&self, block_hash: BlockHash) -> ProviderResult> { self.find_static_file(StaticFileSegment::Headers, |jar_provider| { Ok(jar_provider .cursor()? - .get_two::>(block_hash.into())? + .get_two::>((&block_hash).into())? .and_then(|(header, hash)| { - if &hash == block_hash { + if hash == block_hash { return Some(header) } None @@ -1286,27 +1461,6 @@ impl> HeaderProvider for StaticFileProvide }) } - fn header_td(&self, block_hash: &BlockHash) -> ProviderResult> { - self.find_static_file(StaticFileSegment::Headers, |jar_provider| { - Ok(jar_provider - .cursor()? - .get_two::(block_hash.into())? - .and_then(|(td, hash)| (&hash == block_hash).then_some(td.0))) - }) - } - - fn header_td_by_number(&self, num: BlockNumber) -> ProviderResult> { - self.get_segment_provider_from_block(StaticFileSegment::Headers, num, None) - .and_then(|provider| provider.header_td_by_number(num)) - .or_else(|err| { - if let ProviderError::MissingStaticFileBlock(_, _) = err { - Ok(None) - } else { - Err(err) - } - }) - } - fn headers_range( &self, range: impl RangeBounds, @@ -1354,7 +1508,15 @@ impl> HeaderProvider for StaticFileProvide impl BlockHashReader for StaticFileProvider { fn block_hash(&self, num: u64) -> ProviderResult> { - self.get_segment_provider_from_block(StaticFileSegment::Headers, num, None)?.block_hash(num) + self.get_segment_provider_from_block(StaticFileSegment::Headers, num, None) + .and_then(|provider| provider.block_hash(num)) + .or_else(|err| { + if let ProviderError::MissingStaticFileBlock(_, _) = err { + Ok(None) + } else { + Err(err) + } + }) } fn canonical_hashes_range( @@ -1413,10 +1575,17 @@ impl> Rec |_| true, ) } + + fn receipts_by_block_range( + &self, + _block_range: RangeInclusive, + ) -> ProviderResult>> { + Err(ProviderError::UnsupportedProvider) + } } -impl> - TransactionsProviderExt for StaticFileProvider +impl> TransactionsProviderExt + for StaticFileProvider { fn transaction_hashes_by_range( &self, @@ -1591,8 +1760,6 @@ impl> TransactionsPr } } -/* Cannot be successfully implemented but must exist for trait requirements */ - impl BlockNumReader for StaticFileProvider { fn chain_info(&self) -> ProviderResult { // Required data not present in static_files @@ -1605,8 +1772,7 @@ impl BlockNumReader for StaticFileProvider { } fn last_block_number(&self) -> ProviderResult { - // Required data not present in static_files - Err(ProviderError::UnsupportedProvider) + Ok(self.get_highest_static_file_block(StaticFileSegment::Headers).unwrap_or_default()) } fn block_number(&self, _hash: B256) -> ProviderResult> { @@ -1615,7 +1781,9 @@ impl BlockNumReader for StaticFileProvider { } } -impl> BlockReader +/* Cannot be successfully implemented but must exist for trait requirements */ + +impl> BlockReader for StaticFileProvider { type Block = N::Block; @@ -1634,19 +1802,14 @@ impl> Err(ProviderError::UnsupportedProvider) } - fn pending_block(&self) -> ProviderResult>> { - // Required data not present in static_files - Err(ProviderError::UnsupportedProvider) - } - - fn pending_block_with_senders(&self) -> ProviderResult>> { + fn pending_block(&self) -> ProviderResult>> { // Required data not present in static_files Err(ProviderError::UnsupportedProvider) } fn pending_block_and_receipts( &self, - ) -> ProviderResult, Vec)>> { + ) -> ProviderResult, Vec)>> { // Required data not present in static_files Err(ProviderError::UnsupportedProvider) } @@ -1687,73 +1850,22 @@ impl> ) -> ProviderResult>> { Err(ProviderError::UnsupportedProvider) } -} -impl WithdrawalsProvider for StaticFileProvider { - fn withdrawals_by_block( - &self, - id: BlockHashOrNumber, - timestamp: u64, - ) -> ProviderResult> { - if let Some(num) = id.as_number() { - return self - .get_segment_provider_from_block(StaticFileSegment::BlockMeta, num, None) - .and_then(|provider| provider.withdrawals_by_block(id, timestamp)) - .or_else(|err| { - if let ProviderError::MissingStaticFileBlock(_, _) = err { - Ok(None) - } else { - Err(err) - } - }) - } - // Only accepts block number queries - Err(ProviderError::UnsupportedProvider) - } -} - -impl> OmmersProvider for StaticFileProvider { - fn ommers(&self, id: BlockHashOrNumber) -> ProviderResult>> { - if let Some(num) = id.as_number() { - return self - .get_segment_provider_from_block(StaticFileSegment::BlockMeta, num, None) - .and_then(|provider| provider.ommers(id)) - .or_else(|err| { - if let ProviderError::MissingStaticFileBlock(_, _) = err { - Ok(None) - } else { - Err(err) - } - }) - } - // Only accepts block number queries + fn block_by_transaction_id(&self, _id: TxNumber) -> ProviderResult> { Err(ProviderError::UnsupportedProvider) } } impl BlockBodyIndicesProvider for StaticFileProvider { - fn block_body_indices(&self, num: u64) -> ProviderResult> { - self.get_segment_provider_from_block(StaticFileSegment::BlockMeta, num, None) - .and_then(|provider| provider.block_body_indices(num)) - .or_else(|err| { - if let ProviderError::MissingStaticFileBlock(_, _) = err { - Ok(None) - } else { - Err(err) - } - }) + fn block_body_indices(&self, _num: u64) -> ProviderResult> { + Err(ProviderError::UnsupportedProvider) } fn block_body_indices_range( &self, - range: RangeInclusive, + _range: RangeInclusive, ) -> ProviderResult> { - self.fetch_range_with_predicate( - StaticFileSegment::BlockMeta, - *range.start()..*range.end() + 1, - |cursor, number| cursor.get_one::(number.into()), - |_| true, - ) + Err(ProviderError::UnsupportedProvider) } } diff --git a/crates/storage/provider/src/providers/static_file/metrics.rs b/crates/storage/provider/src/providers/static_file/metrics.rs index ad738334837..8d7269e3d7e 100644 --- a/crates/storage/provider/src/providers/static_file/metrics.rs +++ b/crates/storage/provider/src/providers/static_file/metrics.rs @@ -66,18 +66,15 @@ impl StaticFileProviderMetrics { operation: StaticFileProviderOperation, duration: Option, ) { - self.segment_operations + let segment_operation = self + .segment_operations .get(&(segment, operation)) - .expect("segment operation metrics should exist") - .calls_total - .increment(1); + .expect("segment operation metrics should exist"); + + segment_operation.calls_total.increment(1); if let Some(duration) = duration { - self.segment_operations - .get(&(segment, operation)) - .expect("segment operation metrics should exist") - .write_duration_seconds - .record(duration.as_secs_f64()); + segment_operation.write_duration_seconds.record(duration.as_secs_f64()); } } diff --git a/crates/storage/provider/src/providers/static_file/mod.rs b/crates/storage/provider/src/providers/static_file/mod.rs index 2bf9cf66f9c..3c25f157bb3 100644 --- a/crates/storage/provider/src/providers/static_file/mod.rs +++ b/crates/storage/provider/src/providers/static_file/mod.rs @@ -58,12 +58,10 @@ mod tests { test_utils::create_test_provider_factory, HeaderProvider, StaticFileProviderFactory, }; use alloy_consensus::{Header, SignableTransaction, Transaction, TxLegacy}; - use alloy_primitives::{BlockHash, Signature, TxNumber, B256, U256}; + use alloy_primitives::{BlockHash, Signature, TxNumber, B256}; use rand::seq::SliceRandom; use reth_db::test_utils::create_test_static_files_dir; - use reth_db_api::{ - transaction::DbTxMut, CanonicalHeaders, HeaderNumbers, HeaderTerminalDifficulties, Headers, - }; + use reth_db_api::{transaction::DbTxMut, CanonicalHeaders, HeaderNumbers, Headers}; use reth_ethereum_primitives::{EthPrimitives, Receipt, TransactionSigned}; use reth_static_file_types::{ find_fixed_range, SegmentRangeInclusive, DEFAULT_BLOCKS_PER_STATIC_FILE, @@ -74,7 +72,7 @@ mod tests { fn assert_eyre(got: T, expected: T, msg: &str) -> eyre::Result<()> { if got != expected { - eyre::bail!("{msg} | got: {got:?} expected: {expected:?})"); + eyre::bail!("{msg} | got: {got:?} expected: {expected:?}"); } Ok(()) } @@ -102,14 +100,11 @@ mod tests { let mut provider_rw = factory.provider_rw().unwrap(); let tx = provider_rw.tx_mut(); - let mut td = U256::ZERO; for header in headers.clone() { - td += header.header().difficulty; let hash = header.hash(); tx.put::(header.number, hash).unwrap(); tx.put::(header.number, header.clone_header()).unwrap(); - tx.put::(header.number, td.into()).unwrap(); tx.put::(hash, header.number).unwrap(); } provider_rw.commit().unwrap(); @@ -118,12 +113,10 @@ mod tests { { let manager = factory.static_file_provider(); let mut writer = manager.latest_writer(StaticFileSegment::Headers).unwrap(); - let mut td = U256::ZERO; for header in headers.clone() { - td += header.header().difficulty; let hash = header.hash(); - writer.append_header(&header.unseal(), td, &hash).unwrap(); + writer.append_header(&header.unseal(), &hash).unwrap(); } writer.commit().unwrap(); } @@ -146,14 +139,8 @@ mod tests { let header = header.unseal(); // Compare Header - assert_eq!(header, db_provider.header(&header_hash).unwrap().unwrap()); + assert_eq!(header, db_provider.header(header_hash).unwrap().unwrap()); assert_eq!(header, jar_provider.header_by_number(header.number).unwrap().unwrap()); - - // Compare HeaderTerminalDifficulties - assert_eq!( - db_provider.header_td(&header_hash).unwrap().unwrap(), - jar_provider.header_td_by_number(header.number).unwrap().unwrap() - ); } } } @@ -180,9 +167,7 @@ mod tests { let mut header = Header::default(); for num in 0..=tip { header.number = num; - header_writer - .append_header(&header, U256::default(), &BlockHash::default()) - .unwrap(); + header_writer.append_header(&header, &BlockHash::default()).unwrap(); } header_writer.commit().unwrap(); } diff --git a/crates/storage/provider/src/providers/static_file/writer.rs b/crates/storage/provider/src/providers/static_file/writer.rs index 3781eff6621..7b0ae9ce11c 100644 --- a/crates/storage/provider/src/providers/static_file/writer.rs +++ b/crates/storage/provider/src/providers/static_file/writer.rs @@ -6,9 +6,7 @@ use alloy_consensus::BlockHeader; use alloy_primitives::{BlockHash, BlockNumber, TxNumber, U256}; use parking_lot::{lock_api::RwLockWriteGuard, RawRwLock, RwLock}; use reth_codecs::Compact; -use reth_db_api::models::{ - CompactU256, StoredBlockBodyIndices, StoredBlockOmmers, StoredBlockWithdrawals, -}; +use reth_db_api::models::CompactU256; use reth_nippy_jar::{NippyJar, NippyJarError, NippyJarWriter}; use reth_node_types::NodePrimitives; use reth_static_file_types::{SegmentHeader, SegmentRangeInclusive, StaticFileSegment}; @@ -31,7 +29,6 @@ pub(crate) struct StaticFileWriters { headers: RwLock>>, transactions: RwLock>>, receipts: RwLock>>, - block_meta: RwLock>>, } impl Default for StaticFileWriters { @@ -40,7 +37,6 @@ impl Default for StaticFileWriters { headers: Default::default(), transactions: Default::default(), receipts: Default::default(), - block_meta: Default::default(), } } } @@ -55,7 +51,6 @@ impl StaticFileWriters { StaticFileSegment::Headers => self.headers.write(), StaticFileSegment::Transactions => self.transactions.write(), StaticFileSegment::Receipts => self.receipts.write(), - StaticFileSegment::BlockMeta => self.block_meta.write(), }; if write_guard.is_none() { @@ -74,6 +69,18 @@ impl StaticFileWriters { } Ok(()) } + + pub(crate) fn has_unwind_queued(&self) -> bool { + for writer_lock in [&self.headers, &self.transactions, &self.receipts] { + let writer = writer_lock.read(); + if let Some(writer) = writer.as_ref() && + writer.will_prune_on_commit() + { + return true + } + } + false + } } /// Mutable reference to a [`StaticFileProviderRW`] behind a [`RwLockWriteGuard`]. @@ -206,7 +213,8 @@ impl StaticFileProviderRW { } else { self.user_header().tx_len().unwrap_or_default() }; - let pruned_rows = expected_rows - self.writer.rows() as u64; + let actual_rows = self.writer.rows() as u64; + let pruned_rows = expected_rows.saturating_sub(actual_rows); if pruned_rows > 0 { self.user_header_mut().prune(pruned_rows); } @@ -218,6 +226,11 @@ impl StaticFileProviderRW { Ok(()) } + /// Returns `true` if the writer will prune on commit. + pub const fn will_prune_on_commit(&self) -> bool { + self.prune_on_commit.is_some() + } + /// Commits configuration changes to disk and updates the reader index with the new changes. pub fn commit(&mut self) -> ProviderResult<()> { let start = Instant::now(); @@ -231,7 +244,6 @@ impl StaticFileProviderRW { StaticFileSegment::Receipts => { self.prune_receipt_data(to_delete, last_block_number.expect("should exist"))? } - StaticFileSegment::BlockMeta => todo!(), } } @@ -359,18 +371,22 @@ impl StaticFileProviderRW { Ok(()) } - /// Verifies if the incoming block number matches the next expected block number - /// for a static file. This ensures data continuity when adding new blocks. - fn check_next_block_number(&self, expected_block_number: u64) -> ProviderResult<()> { + /// Returns a block number that is one next to the current tip of static files. + pub fn next_block_number(&self) -> u64 { // The next static file block number can be found by checking the one after block_end. - // However if it's a new file that hasn't been added any data, its block range will actually - // be None. In that case, the next block will be found on `expected_block_start`. - let next_static_file_block = self - .writer + // However, if it's a new file that hasn't been added any data, its block range will + // actually be None. In that case, the next block will be found on `expected_block_start`. + self.writer .user_header() .block_end() .map(|b| b + 1) - .unwrap_or_else(|| self.writer.user_header().expected_block_start()); + .unwrap_or_else(|| self.writer.user_header().expected_block_start()) + } + + /// Verifies if the incoming block number matches the next expected block number + /// for a static file. This ensures data continuity when adding new blocks. + fn check_next_block_number(&self, expected_block_number: u64) -> ProviderResult<()> { + let next_static_file_block = self.next_block_number(); if expected_block_number != next_static_file_block { return Err(ProviderError::UnexpectedStaticFileBlockNumber( @@ -515,7 +531,20 @@ impl StaticFileProviderRW { /// blocks. /// /// Returns the current [`BlockNumber`] as seen in the static file. - pub fn append_header( + pub fn append_header(&mut self, header: &N::BlockHeader, hash: &BlockHash) -> ProviderResult<()> + where + N::BlockHeader: Compact, + { + self.append_header_with_td(header, U256::ZERO, hash) + } + + /// Appends header to static file with a specified total difficulty. + /// + /// It **CALLS** `increment_block()` since the number of headers is equal to the number of + /// blocks. + /// + /// Returns the current [`BlockNumber`] as seen in the static file. + pub fn append_header_with_td( &mut self, header: &N::BlockHeader, total_difficulty: U256, @@ -546,61 +575,6 @@ impl StaticFileProviderRW { Ok(()) } - /// Appends [`StoredBlockBodyIndices`], [`StoredBlockOmmers`] and [`StoredBlockWithdrawals`] to - /// static file. - /// - /// It **CALLS** `increment_block()` since it's a block based segment. - pub fn append_eth_block_meta( - &mut self, - body_indices: &StoredBlockBodyIndices, - ommers: &StoredBlockOmmers, - withdrawals: &StoredBlockWithdrawals, - expected_block_number: BlockNumber, - ) -> ProviderResult<()> - where - N::BlockHeader: Compact, - { - self.append_block_meta(body_indices, ommers, withdrawals, expected_block_number) - } - - /// Appends [`StoredBlockBodyIndices`] and any other two arbitrary types belonging to the block - /// body to static file. - /// - /// It **CALLS** `increment_block()` since it's a block based segment. - pub fn append_block_meta( - &mut self, - body_indices: &StoredBlockBodyIndices, - field1: &F1, - field2: &F2, - expected_block_number: BlockNumber, - ) -> ProviderResult<()> - where - N::BlockHeader: Compact, - F1: Compact, - F2: Compact, - { - let start = Instant::now(); - self.ensure_no_queued_prune()?; - - debug_assert!(self.writer.user_header().segment() == StaticFileSegment::BlockMeta); - - self.increment_block(expected_block_number)?; - - self.append_column(body_indices)?; - self.append_column(field1)?; - self.append_column(field2)?; - - if let Some(metrics) = &self.metrics { - metrics.record_segment_operation( - StaticFileSegment::BlockMeta, - StaticFileProviderOperation::Append, - Some(start.elapsed()), - ); - } - - Ok(()) - } - /// Appends transaction to static file. /// /// It **DOES NOT CALL** `increment_block()`, it should be handled elsewhere. There might be @@ -698,7 +672,7 @@ impl StaticFileProviderRW { Ok(Some(tx_number)) } - /// Adds an instruction to prune `to_delete`transactions during commit. + /// Adds an instruction to prune `to_delete` transactions during commit. /// /// Note: `last_block` refers to the block the unwinds ends at. pub fn prune_transactions( @@ -728,12 +702,6 @@ impl StaticFileProviderRW { self.queue_prune(to_delete, None) } - /// Adds an instruction to prune `to_delete` bloc_ meta rows during commit. - pub fn prune_block_meta(&mut self, to_delete: u64) -> ProviderResult<()> { - debug_assert_eq!(self.writer.user_header().segment(), StaticFileSegment::BlockMeta); - self.queue_prune(to_delete, None) - } - /// Adds an instruction to prune `to_delete` elements during commit. /// /// Note: `last_block` refers to the block the unwinds ends at if dealing with transaction-based diff --git a/crates/storage/provider/src/test_utils/blocks.rs b/crates/storage/provider/src/test_utils/blocks.rs index 2acb77b8b42..0b27c5dc992 100644 --- a/crates/storage/provider/src/test_utils/blocks.rs +++ b/crates/storage/provider/src/test_utils/blocks.rs @@ -26,15 +26,11 @@ pub fn assert_genesis_block( let h = B256::ZERO; let tx = provider; - // check if all tables are empty + // check if tables contain only the genesis block data assert_eq!(tx.table::().unwrap(), vec![(g.number, g.header().clone())]); assert_eq!(tx.table::().unwrap(), vec![(h, n)]); assert_eq!(tx.table::().unwrap(), vec![(n, h)]); - assert_eq!( - tx.table::().unwrap(), - vec![(n, g.difficulty.into())] - ); assert_eq!( tx.table::().unwrap(), vec![(0, StoredBlockBodyIndices::default())] @@ -85,7 +81,7 @@ pub(crate) static TEST_BLOCK: LazyLock { +pub struct MockEthProvider +{ ///local block store pub blocks: Arc>>, /// Local header store - pub headers: Arc>>, + pub headers: Arc::Header>>>, + /// Local receipt store indexed by block number + pub receipts: Arc>>>, /// Local account store pub accounts: Arc>>, /// Local chain spec pub chain_spec: Arc, /// Local state roots pub state_roots: Arc>>, + /// Local block body indices store + pub block_body_indices: Arc>>, tx: TxMock, prune_modes: Arc, } @@ -74,9 +80,11 @@ where Self { blocks: self.blocks.clone(), headers: self.headers.clone(), + receipts: self.receipts.clone(), accounts: self.accounts.clone(), chain_spec: self.chain_spec.clone(), state_roots: self.state_roots.clone(), + block_body_indices: self.block_body_indices.clone(), tx: self.tx.clone(), prune_modes: self.prune_modes.clone(), } @@ -89,40 +97,42 @@ impl MockEthProvider { Self { blocks: Default::default(), headers: Default::default(), + receipts: Default::default(), accounts: Default::default(), chain_spec: Arc::new(reth_chainspec::ChainSpecBuilder::mainnet().build()), state_roots: Default::default(), + block_body_indices: Default::default(), tx: Default::default(), prune_modes: Default::default(), } } } -impl MockEthProvider { +impl MockEthProvider { /// Add block to local block store - pub fn add_block(&self, hash: B256, block: reth_ethereum_primitives::Block) { - self.add_header(hash, block.header.clone()); + pub fn add_block(&self, hash: B256, block: T::Block) { + self.add_header(hash, block.header().clone()); self.blocks.lock().insert(hash, block); } /// Add multiple blocks to local block store - pub fn extend_blocks( - &self, - iter: impl IntoIterator, - ) { + pub fn extend_blocks(&self, iter: impl IntoIterator) { for (hash, block) in iter { - self.add_header(hash, block.header.clone()); + self.add_header(hash, block.header().clone()); self.add_block(hash, block) } } /// Add header to local header store - pub fn add_header(&self, hash: B256, header: Header) { + pub fn add_header(&self, hash: B256, header: ::Header) { self.headers.lock().insert(hash, header); } /// Add multiple headers to local header store - pub fn extend_headers(&self, iter: impl IntoIterator) { + pub fn extend_headers( + &self, + iter: impl IntoIterator::Header)>, + ) { for (hash, header) in iter { self.add_header(hash, header) } @@ -140,22 +150,42 @@ impl MockEthProvider) { + self.receipts.lock().insert(block_number, receipts); + } + + /// Add multiple receipts to local receipt store + pub fn extend_receipts(&self, iter: impl IntoIterator)>) { + for (block_number, receipts) in iter { + self.add_receipts(block_number, receipts); + } + } + + /// Add block body indices to local store + pub fn add_block_body_indices( + &self, + block_number: BlockNumber, + indices: StoredBlockBodyIndices, + ) { + self.block_body_indices.lock().insert(block_number, indices); + } + /// Add state root to local state root store pub fn add_state_root(&self, state_root: B256) { self.state_roots.lock().push(state_root); } /// Set chain spec. - pub fn with_chain_spec( - self, - chain_spec: C, - ) -> MockEthProvider { + pub fn with_chain_spec(self, chain_spec: C) -> MockEthProvider { MockEthProvider { blocks: self.blocks, headers: self.headers, + receipts: self.receipts, accounts: self.accounts, chain_spec: Arc::new(chain_spec), state_roots: self.state_roots, + block_body_indices: self.block_body_indices, tx: self.tx, prune_modes: self.prune_modes, } @@ -205,26 +235,6 @@ impl ExtendedAccount { } } -/// Mock node. -#[derive(Clone, Debug)] -pub struct MockNode; - -impl NodeTypes for MockNode { - type Primitives = EthPrimitives; - type ChainSpec = reth_chainspec::ChainSpec; - type StateCommitment = MerklePatriciaTrie; - type Storage = EthStorage; - type Payload = EthEngineTypes; -} - -impl StateCommitmentProvider for MockEthProvider -where - T: NodePrimitives, - ChainSpec: EthChainSpec + Send + Sync + 'static, -{ - type StateCommitment = ::StateCommitment; -} - impl DatabaseProviderFactory for MockEthProvider { @@ -233,16 +243,10 @@ impl DatabaseProvi type ProviderRW = Self; fn database_provider_ro(&self) -> ProviderResult { - // TODO: return Ok(self.clone()) when engine tests stops relying on an - // Error returned here https://github.com/paradigmxyz/reth/pull/14482 - //Ok(self.clone()) Err(ConsistentViewError::Syncing { best_block: GotExpected::new(0, 0) }.into()) } fn database_provider_rw(&self) -> ProviderResult { - // TODO: return Ok(self.clone()) when engine tests stops relying on an - // Error returned here https://github.com/paradigmxyz/reth/pull/14482 - //Ok(self.clone()) Err(ConsistentViewError::Syncing { best_block: GotExpected::new(0, 0) }.into()) } } @@ -264,63 +268,55 @@ impl DBProvider self.tx } + fn commit(self) -> ProviderResult { + Ok(self.tx.commit()?) + } + fn prune_modes_ref(&self) -> &PruneModes { &self.prune_modes } } -impl HeaderProvider - for MockEthProvider +impl HeaderProvider + for MockEthProvider { - type Header = Header; - - fn header(&self, block_hash: &BlockHash) -> ProviderResult> { - let lock = self.headers.lock(); - Ok(lock.get(block_hash).cloned()) - } + type Header = ::Header; - fn header_by_number(&self, num: u64) -> ProviderResult> { + fn header(&self, block_hash: BlockHash) -> ProviderResult> { let lock = self.headers.lock(); - Ok(lock.values().find(|h| h.number == num).cloned()) + Ok(lock.get(&block_hash).cloned()) } - fn header_td(&self, hash: &BlockHash) -> ProviderResult> { + fn header_by_number(&self, num: u64) -> ProviderResult> { let lock = self.headers.lock(); - Ok(lock.get(hash).map(|target| { - lock.values() - .filter(|h| h.number < target.number) - .fold(target.difficulty, |td, h| td + h.difficulty) - })) + Ok(lock.values().find(|h| h.number() == num).cloned()) } - fn header_td_by_number(&self, number: BlockNumber) -> ProviderResult> { - let lock = self.headers.lock(); - let sum = lock - .values() - .filter(|h| h.number <= number) - .fold(U256::ZERO, |td, h| td + h.difficulty); - Ok(Some(sum)) - } - - fn headers_range(&self, range: impl RangeBounds) -> ProviderResult> { + fn headers_range( + &self, + range: impl RangeBounds, + ) -> ProviderResult> { let lock = self.headers.lock(); let mut headers: Vec<_> = - lock.values().filter(|header| range.contains(&header.number)).cloned().collect(); - headers.sort_by_key(|header| header.number); + lock.values().filter(|header| range.contains(&header.number())).cloned().collect(); + headers.sort_by_key(|header| header.number()); Ok(headers) } - fn sealed_header(&self, number: BlockNumber) -> ProviderResult> { + fn sealed_header( + &self, + number: BlockNumber, + ) -> ProviderResult>> { Ok(self.header_by_number(number)?.map(SealedHeader::seal_slow)) } fn sealed_headers_while( &self, range: impl RangeBounds, - mut predicate: impl FnMut(&SealedHeader) -> bool, - ) -> ProviderResult> { + mut predicate: impl FnMut(&SealedHeader) -> bool, + ) -> ProviderResult>> { Ok(self .headers_range(range)? .into_iter() @@ -342,16 +338,16 @@ where } } -impl TransactionsProvider - for MockEthProvider +impl TransactionsProvider + for MockEthProvider { - type Transaction = reth_ethereum_primitives::TransactionSigned; + type Transaction = T::SignedTx; fn transaction_id(&self, tx_hash: TxHash) -> ProviderResult> { let lock = self.blocks.lock(); let tx_number = lock .values() - .flat_map(|block| &block.body.transactions) + .flat_map(|block| block.body().transactions()) .position(|tx| *tx.tx_hash() == tx_hash) .map(|pos| pos as TxNumber); @@ -361,7 +357,7 @@ impl TransactionsProvider fn transaction_by_id(&self, id: TxNumber) -> ProviderResult> { let lock = self.blocks.lock(); let transaction = - lock.values().flat_map(|block| &block.body.transactions).nth(id as usize).cloned(); + lock.values().flat_map(|block| block.body().transactions()).nth(id as usize).cloned(); Ok(transaction) } @@ -372,14 +368,14 @@ impl TransactionsProvider ) -> ProviderResult> { let lock = self.blocks.lock(); let transaction = - lock.values().flat_map(|block| &block.body.transactions).nth(id as usize).cloned(); + lock.values().flat_map(|block| block.body().transactions()).nth(id as usize).cloned(); Ok(transaction) } fn transaction_by_hash(&self, hash: TxHash) -> ProviderResult> { Ok(self.blocks.lock().iter().find_map(|(_, block)| { - block.body.transactions.iter().find(|tx| *tx.tx_hash() == hash).cloned() + block.body().transactions_iter().find(|tx| *tx.tx_hash() == hash).cloned() })) } @@ -389,16 +385,16 @@ impl TransactionsProvider ) -> ProviderResult> { let lock = self.blocks.lock(); for (block_hash, block) in lock.iter() { - for (index, tx) in block.body.transactions.iter().enumerate() { + for (index, tx) in block.body().transactions_iter().enumerate() { if *tx.tx_hash() == hash { let meta = TransactionMeta { tx_hash: hash, index: index as u64, block_hash: *block_hash, - block_number: block.header.number, - base_fee: block.header.base_fee_per_gas, - excess_blob_gas: block.header.excess_blob_gas, - timestamp: block.header.timestamp, + block_number: block.header().number(), + base_fee: block.header().base_fee_per_gas(), + excess_blob_gas: block.header().excess_blob_gas(), + timestamp: block.header().timestamp(), }; return Ok(Some((tx.clone(), meta))) } @@ -411,10 +407,10 @@ impl TransactionsProvider let lock = self.blocks.lock(); let mut current_tx_number: TxNumber = 0; for block in lock.values() { - if current_tx_number + (block.body.transactions.len() as TxNumber) > id { - return Ok(Some(block.header.number)) + if current_tx_number + (block.body().transaction_count() as TxNumber) > id { + return Ok(Some(block.header().number())) } - current_tx_number += block.body.transactions.len() as TxNumber; + current_tx_number += block.body().transaction_count() as TxNumber; } Ok(None) } @@ -423,7 +419,7 @@ impl TransactionsProvider &self, id: BlockHashOrNumber, ) -> ProviderResult>> { - Ok(self.block(id)?.map(|b| b.body.transactions)) + Ok(self.block(id)?.map(|b| b.body().clone_transactions())) } fn transactions_by_block_range( @@ -433,8 +429,8 @@ impl TransactionsProvider // init btreemap so we can return in order let mut map = BTreeMap::new(); for (_, block) in self.blocks.lock().iter() { - if range.contains(&block.number) { - map.insert(block.number, block.body.transactions.clone()); + if range.contains(&block.header().number()) { + map.insert(block.header().number(), block.body().clone_transactions()); } } @@ -448,7 +444,7 @@ impl TransactionsProvider let lock = self.blocks.lock(); let transactions = lock .values() - .flat_map(|block| &block.body.transactions) + .flat_map(|block| block.body().transactions()) .enumerate() .filter(|&(tx_number, _)| range.contains(&(tx_number as TxNumber))) .map(|(_, tx)| tx.clone()) @@ -464,7 +460,7 @@ impl TransactionsProvider let lock = self.blocks.lock(); let transactions = lock .values() - .flat_map(|block| &block.body.transactions) + .flat_map(|block| block.body().transactions()) .enumerate() .filter_map(|(tx_number, tx)| { if range.contains(&(tx_number as TxNumber)) { @@ -488,7 +484,7 @@ where T: NodePrimitives, ChainSpec: Send + Sync + 'static, { - type Receipt = Receipt; + type Receipt = T::Receipt; fn receipt(&self, _id: TxNumber) -> ProviderResult> { Ok(None) @@ -500,9 +496,22 @@ where fn receipts_by_block( &self, - _block: BlockHashOrNumber, + block: BlockHashOrNumber, ) -> ProviderResult>> { - Ok(None) + let receipts_lock = self.receipts.lock(); + + match block { + BlockHashOrNumber::Hash(hash) => { + // Find block number by hash first + let headers_lock = self.headers.lock(); + if let Some(header) = headers_lock.get(&hash) { + Ok(receipts_lock.get(&header.number()).cloned()) + } else { + Ok(None) + } + } + BlockHashOrNumber::Number(number) => Ok(receipts_lock.get(&number).cloned()), + } } fn receipts_by_tx_range( @@ -511,6 +520,29 @@ where ) -> ProviderResult> { Ok(vec![]) } + + fn receipts_by_block_range( + &self, + block_range: RangeInclusive, + ) -> ProviderResult>> { + let receipts_lock = self.receipts.lock(); + let headers_lock = self.headers.lock(); + + let mut result = Vec::new(); + for block_number in block_range { + // Only include blocks that exist in headers (i.e., have been added to the provider) + if headers_lock.values().any(|header| header.number() == block_number) { + if let Some(block_receipts) = receipts_lock.get(&block_number) { + result.push(block_receipts.clone()); + } else { + // If block exists but no receipts found, add empty vec + result.push(vec![]); + } + } + } + + Ok(result) + } } impl ReceiptProviderIdExt for MockEthProvider @@ -526,7 +558,7 @@ impl BlockHashReader fn block_hash(&self, number: u64) -> ProviderResult> { let lock = self.headers.lock(); let hash = - lock.iter().find_map(|(hash, header)| (header.number == number).then_some(*hash)); + lock.iter().find_map(|(hash, header)| (header.number() == number).then_some(*hash)); Ok(hash) } @@ -537,9 +569,9 @@ impl BlockHashReader ) -> ProviderResult> { let lock = self.headers.lock(); let mut hashes: Vec<_> = - lock.iter().filter(|(_, header)| (start..end).contains(&header.number)).collect(); + lock.iter().filter(|(_, header)| (start..end).contains(&header.number())).collect(); - hashes.sort_by_key(|(_, header)| header.number); + hashes.sort_by_key(|(_, header)| header.number()); Ok(hashes.into_iter().map(|(hash, _)| *hash).collect()) } @@ -554,16 +586,16 @@ impl BlockNumReader Ok(lock .iter() - .find(|(_, header)| header.number == best_block_number) - .map(|(hash, header)| ChainInfo { best_hash: *hash, best_number: header.number }) + .find(|(_, header)| header.number() == best_block_number) + .map(|(hash, header)| ChainInfo { best_hash: *hash, best_number: header.number() }) .unwrap_or_default()) } fn best_block_number(&self) -> ProviderResult { let lock = self.headers.lock(); lock.iter() - .max_by_key(|h| h.1.number) - .map(|(_, header)| header.number) + .max_by_key(|h| h.1.number()) + .map(|(_, header)| header.number()) .ok_or(ProviderError::BestBlockNotFound) } @@ -573,7 +605,7 @@ impl BlockNumReader fn block_number(&self, hash: B256) -> ProviderResult> { let lock = self.headers.lock(); - Ok(lock.get(&hash).map(|header| header.number)) + Ok(lock.get(&hash).map(|header| header.number())) } } @@ -594,10 +626,10 @@ impl BlockId } //look -impl BlockReader - for MockEthProvider +impl BlockReader + for MockEthProvider { - type Block = reth_ethereum_primitives::Block; + type Block = T::Block; fn find_block_by_hash( &self, @@ -611,21 +643,19 @@ impl BlockReader let lock = self.blocks.lock(); match id { BlockHashOrNumber::Hash(hash) => Ok(lock.get(&hash).cloned()), - BlockHashOrNumber::Number(num) => Ok(lock.values().find(|b| b.number == num).cloned()), + BlockHashOrNumber::Number(num) => { + Ok(lock.values().find(|b| b.header().number() == num).cloned()) + } } } - fn pending_block(&self) -> ProviderResult>> { - Ok(None) - } - - fn pending_block_with_senders(&self) -> ProviderResult>> { + fn pending_block(&self) -> ProviderResult>> { Ok(None) } fn pending_block_and_receipts( &self, - ) -> ProviderResult, Vec)>> { + ) -> ProviderResult, Vec)>> { Ok(None) } @@ -648,9 +678,12 @@ impl BlockReader fn block_range(&self, range: RangeInclusive) -> ProviderResult> { let lock = self.blocks.lock(); - let mut blocks: Vec<_> = - lock.values().filter(|block| range.contains(&block.number)).cloned().collect(); - blocks.sort_by_key(|block| block.number); + let mut blocks: Vec<_> = lock + .values() + .filter(|block| range.contains(&block.header().number())) + .cloned() + .collect(); + blocks.sort_by_key(|block| block.header().number()); Ok(blocks) } @@ -668,35 +701,35 @@ impl BlockReader ) -> ProviderResult>> { Ok(vec![]) } + + fn block_by_transaction_id(&self, _id: TxNumber) -> ProviderResult> { + Ok(None) + } } -impl BlockReaderIdExt - for MockEthProvider +impl BlockReaderIdExt for MockEthProvider where ChainSpec: EthChainSpec + Send + Sync + 'static, + T: NodePrimitives, { - fn block_by_id(&self, id: BlockId) -> ProviderResult> { + fn block_by_id(&self, id: BlockId) -> ProviderResult> { match id { BlockId::Number(num) => self.block_by_number_or_tag(num), BlockId::Hash(hash) => self.block_by_hash(hash.block_hash), } } - fn sealed_header_by_id(&self, id: BlockId) -> ProviderResult> { + fn sealed_header_by_id( + &self, + id: BlockId, + ) -> ProviderResult::Header>>> { self.header_by_id(id)?.map_or_else(|| Ok(None), |h| Ok(Some(SealedHeader::seal_slow(h)))) } - fn header_by_id(&self, id: BlockId) -> ProviderResult> { + fn header_by_id(&self, id: BlockId) -> ProviderResult::Header>> { match self.block_by_id(id)? { None => Ok(None), - Some(block) => Ok(Some(block.header)), - } - } - - fn ommers_by_id(&self, id: BlockId) -> ProviderResult>> { - match id { - BlockId::Number(num) => self.ommers_by_number_or_tag(num), - BlockId::Hash(hash) => self.ommers(BlockHashOrNumber::Hash(hash.block_hash)), + Some(block) => Ok(Some(block.into_header())), } } } @@ -723,6 +756,21 @@ impl StageCheckpointReader } } +impl PruneCheckpointReader + for MockEthProvider +{ + fn get_prune_checkpoint( + &self, + _segment: PruneSegment, + ) -> ProviderResult> { + Ok(None) + } + + fn get_prune_checkpoints(&self) -> ProviderResult> { + Ok(vec![]) + } +} + impl StateRootProvider for MockEthProvider where T: NodePrimitives, @@ -833,7 +881,13 @@ where let lock = self.accounts.lock(); Ok(lock.get(&account).and_then(|account| account.storage.get(&storage_key)).copied()) } +} +impl BytecodeReader for MockEthProvider +where + T: NodePrimitives, + ChainSpec: Send + Sync, +{ fn bytecode_by_hash(&self, code_hash: &B256) -> ProviderResult> { let lock = self.accounts.lock(); Ok(lock.values().find_map(|account| { @@ -874,7 +928,9 @@ impl StatePr self.history_by_block_hash(hash) } - BlockNumberOrTag::Earliest => self.history_by_block_number(0), + BlockNumberOrTag::Earliest => { + self.history_by_block_number(self.earliest_block_number()?) + } BlockNumberOrTag::Pending => self.pending(), BlockNumberOrTag::Number(num) => self.history_by_block_number(num), } @@ -899,36 +955,17 @@ impl StatePr fn pending_state_by_hash(&self, _block_hash: B256) -> ProviderResult> { Ok(Some(Box::new(self.clone()))) } -} - -impl WithdrawalsProvider - for MockEthProvider -{ - fn withdrawals_by_block( - &self, - _id: BlockHashOrNumber, - _timestamp: u64, - ) -> ProviderResult> { - Ok(None) - } -} -impl OmmersProvider for MockEthProvider -where - T: NodePrimitives, - ChainSpec: Send + Sync, - Self: HeaderProvider, -{ - fn ommers(&self, _id: BlockHashOrNumber) -> ProviderResult>> { - Ok(None) + fn maybe_pending(&self) -> ProviderResult> { + Ok(Some(Box::new(self.clone()))) } } impl BlockBodyIndicesProvider for MockEthProvider { - fn block_body_indices(&self, _num: u64) -> ProviderResult> { - Ok(None) + fn block_body_indices(&self, num: u64) -> ProviderResult> { + Ok(self.block_body_indices.lock().get(&num).copied()) } fn block_body_indices_range( &self, @@ -945,16 +982,40 @@ impl ChangeSetReader for MockEthProvi ) -> ProviderResult> { Ok(Vec::default()) } + + fn get_account_before_block( + &self, + _block_number: BlockNumber, + _address: Address, + ) -> ProviderResult> { + Ok(None) + } } impl StateReader for MockEthProvider { - type Receipt = Receipt; + type Receipt = T::Receipt; - fn get_state(&self, _block: BlockNumber) -> ProviderResult> { + fn get_state( + &self, + _block: BlockNumber, + ) -> ProviderResult>> { Ok(None) } } +impl TrieReader for MockEthProvider { + fn trie_reverts(&self, _from: BlockNumber) -> ProviderResult { + Ok(TrieUpdatesSorted::default()) + } + + fn get_block_trie_updates( + &self, + _block_number: BlockNumber, + ) -> ProviderResult { + Ok(TrieUpdatesSorted::default()) + } +} + impl CanonStateSubscriptions for MockEthProvider { @@ -968,3 +1029,74 @@ impl NodePrimitivesProvider { type Primitives = T; } + +#[cfg(test)] +mod tests { + use super::*; + use alloy_consensus::Header; + use alloy_primitives::BlockHash; + use reth_ethereum_primitives::Receipt; + + #[test] + fn test_mock_provider_receipts() { + let provider = MockEthProvider::::new(); + + let block_hash = BlockHash::random(); + let block_number = 1u64; + let header = Header { number: block_number, ..Default::default() }; + + let receipt1 = Receipt { cumulative_gas_used: 21000, success: true, ..Default::default() }; + let receipt2 = Receipt { cumulative_gas_used: 42000, success: true, ..Default::default() }; + let receipts = vec![receipt1, receipt2]; + + provider.add_header(block_hash, header); + provider.add_receipts(block_number, receipts.clone()); + + let result = provider.receipts_by_block(block_hash.into()).unwrap(); + assert_eq!(result, Some(receipts.clone())); + + let result = provider.receipts_by_block(block_number.into()).unwrap(); + assert_eq!(result, Some(receipts.clone())); + + let range_result = provider.receipts_by_block_range(1..=1).unwrap(); + assert_eq!(range_result, vec![receipts]); + + let non_existent = provider.receipts_by_block(BlockHash::random().into()).unwrap(); + assert_eq!(non_existent, None); + + let empty_range = provider.receipts_by_block_range(10..=20).unwrap(); + assert_eq!(empty_range, Vec::>::new()); + } + + #[test] + fn test_mock_provider_receipts_multiple_blocks() { + let provider = MockEthProvider::::new(); + + let block1_hash = BlockHash::random(); + let block2_hash = BlockHash::random(); + let block1_number = 1u64; + let block2_number = 2u64; + + let header1 = Header { number: block1_number, ..Default::default() }; + let header2 = Header { number: block2_number, ..Default::default() }; + + let receipts1 = + vec![Receipt { cumulative_gas_used: 21000, success: true, ..Default::default() }]; + let receipts2 = + vec![Receipt { cumulative_gas_used: 42000, success: true, ..Default::default() }]; + + provider.add_header(block1_hash, header1); + provider.add_header(block2_hash, header2); + provider.add_receipts(block1_number, receipts1.clone()); + provider.add_receipts(block2_number, receipts2.clone()); + + let range_result = provider.receipts_by_block_range(1..=2).unwrap(); + assert_eq!(range_result.len(), 2); + assert_eq!(range_result[0], receipts1); + assert_eq!(range_result[1], receipts2); + + let partial_range = provider.receipts_by_block_range(1..=1).unwrap(); + assert_eq!(partial_range.len(), 1); + assert_eq!(partial_range[0], receipts1); + } +} diff --git a/crates/storage/provider/src/test_utils/mod.rs b/crates/storage/provider/src/test_utils/mod.rs index a24ad9f46f5..ccda2d60e85 100644 --- a/crates/storage/provider/src/test_utils/mod.rs +++ b/crates/storage/provider/src/test_utils/mod.rs @@ -29,7 +29,6 @@ pub type MockNodeTypes = reth_node_types::AnyNodeTypesWithEngine< reth_ethereum_primitives::EthPrimitives, reth_ethereum_engine_primitives::EthEngineTypes, reth_chainspec::ChainSpec, - reth_trie_db::MerklePatriciaTrie, crate::EthStorage, EthEngineTypes, >; @@ -59,7 +58,7 @@ pub fn create_test_provider_factory_with_node_types( ProviderFactory::new( db, chain_spec, - StaticFileProvider::read_write(static_dir.into_path()).expect("static file provider"), + StaticFileProvider::read_write(static_dir.keep()).expect("static file provider"), ) } @@ -90,7 +89,7 @@ pub fn insert_genesis>( let (root, updates) = StateRoot::from_tx(provider.tx_ref()) .root_with_updates() .map_err(reth_db::DatabaseError::from)?; - provider.write_trie_updates(&updates).unwrap(); + provider.write_trie_updates(updates).unwrap(); provider.commit()?; diff --git a/crates/storage/provider/src/traits/full.rs b/crates/storage/provider/src/traits/full.rs index fe5e167bf0b..6fe88a6640a 100644 --- a/crates/storage/provider/src/traits/full.rs +++ b/crates/storage/provider/src/traits/full.rs @@ -1,8 +1,9 @@ //! Helper provider traits to encapsulate all provider traits for simplicity. use crate::{ - AccountReader, BlockReaderIdExt, ChainSpecProvider, ChangeSetReader, DatabaseProviderFactory, - StageCheckpointReader, StateProviderFactory, StaticFileProviderFactory, + AccountReader, BlockReader, BlockReaderIdExt, ChainSpecProvider, ChangeSetReader, + DatabaseProviderFactory, HashedPostStateProvider, PruneCheckpointReader, StageCheckpointReader, + StateProviderFactory, StateReader, StaticFileProviderFactory, TrieReader, }; use reth_chain_state::{CanonStateSubscriptions, ForkChoiceSubscriptions}; use reth_node_types::{BlockTy, HeaderTy, NodeTypesWithDB, ReceiptTy, TxTy}; @@ -11,8 +12,10 @@ use std::fmt::Debug; /// Helper trait to unify all provider traits for simplicity. pub trait FullProvider: - DatabaseProviderFactory - + NodePrimitivesProvider + DatabaseProviderFactory< + DB = N::DB, + Provider: BlockReader + TrieReader + StageCheckpointReader + PruneCheckpointReader, + > + NodePrimitivesProvider + StaticFileProviderFactory + BlockReaderIdExt< Transaction = TxTy, @@ -21,6 +24,8 @@ pub trait FullProvider: Header = HeaderTy, > + AccountReader + StateProviderFactory + + StateReader + + HashedPostStateProvider + ChainSpecProvider + ChangeSetReader + CanonStateSubscriptions @@ -34,8 +39,10 @@ pub trait FullProvider: } impl FullProvider for T where - T: DatabaseProviderFactory - + NodePrimitivesProvider + T: DatabaseProviderFactory< + DB = N::DB, + Provider: BlockReader + TrieReader + StageCheckpointReader + PruneCheckpointReader, + > + NodePrimitivesProvider + StaticFileProviderFactory + BlockReaderIdExt< Transaction = TxTy, @@ -44,6 +51,8 @@ impl FullProvider for T where Header = HeaderTy, > + AccountReader + StateProviderFactory + + StateReader + + HashedPostStateProvider + ChainSpecProvider + ChangeSetReader + CanonStateSubscriptions diff --git a/crates/storage/provider/src/writer/mod.rs b/crates/storage/provider/src/writer/mod.rs index 9ae2d7ad27d..6d990e17a49 100644 --- a/crates/storage/provider/src/writer/mod.rs +++ b/crates/storage/provider/src/writer/mod.rs @@ -1,229 +1,5 @@ -use crate::{ - providers::{StaticFileProvider, StaticFileWriter as SfWriter}, - BlockExecutionWriter, BlockWriter, HistoryWriter, StateWriter, StaticFileProviderFactory, - StorageLocation, TrieWriter, -}; -use alloy_consensus::BlockHeader; -use reth_chain_state::{ExecutedBlock, ExecutedBlockWithTrieUpdates}; -use reth_db_api::transaction::{DbTx, DbTxMut}; -use reth_errors::ProviderResult; -use reth_primitives_traits::{NodePrimitives, SignedTransaction}; -use reth_static_file_types::StaticFileSegment; -use reth_storage_api::{DBProvider, StageCheckpointWriter, TransactionsProviderExt}; -use reth_storage_errors::writer::UnifiedStorageWriterError; -use revm_database::OriginalValuesKnown; -use std::sync::Arc; -use tracing::debug; - -/// [`UnifiedStorageWriter`] is responsible for managing the writing to storage with both database -/// and static file providers. -#[derive(Debug)] -pub struct UnifiedStorageWriter<'a, ProviderDB, ProviderSF> { - database: &'a ProviderDB, - static_file: Option, -} - -impl<'a, ProviderDB, ProviderSF> UnifiedStorageWriter<'a, ProviderDB, ProviderSF> { - /// Creates a new instance of [`UnifiedStorageWriter`]. - /// - /// # Parameters - /// - `database`: An optional reference to a database provider. - /// - `static_file`: An optional mutable reference to a static file instance. - pub const fn new(database: &'a ProviderDB, static_file: Option) -> Self { - Self { database, static_file } - } - - /// Creates a new instance of [`UnifiedStorageWriter`] from a database provider and a static - /// file instance. - pub fn from

(database: &'a P, static_file: ProviderSF) -> Self - where - P: AsRef, - { - Self::new(database.as_ref(), Some(static_file)) - } - - /// Creates a new instance of [`UnifiedStorageWriter`] from a database provider. - pub fn from_database

(database: &'a P) -> Self - where - P: AsRef, - { - Self::new(database.as_ref(), None) - } - - /// Returns a reference to the database writer. - /// - /// # Panics - /// If the database provider is not set. - const fn database(&self) -> &ProviderDB { - self.database - } - - /// Returns a reference to the static file instance. - /// - /// # Panics - /// If the static file instance is not set. - const fn static_file(&self) -> &ProviderSF { - self.static_file.as_ref().expect("should exist") - } - - /// Ensures that the static file instance is set. - /// - /// # Returns - /// - `Ok(())` if the static file instance is set. - /// - `Err(StorageWriterError::MissingStaticFileWriter)` if the static file instance is not set. - #[expect(unused)] - const fn ensure_static_file(&self) -> Result<(), UnifiedStorageWriterError> { - if self.static_file.is_none() { - return Err(UnifiedStorageWriterError::MissingStaticFileWriter) - } - Ok(()) - } -} - -impl UnifiedStorageWriter<'_, (), ()> { - /// Commits both storage types in the right order. - /// - /// For non-unwinding operations it makes more sense to commit the static files first, since if - /// it is interrupted before the database commit, we can just truncate - /// the static files according to the checkpoints on the next - /// start-up. - /// - /// NOTE: If unwinding data from storage, use `commit_unwind` instead! - pub fn commit

(provider: P) -> ProviderResult<()> - where - P: DBProvider + StaticFileProviderFactory, - { - let static_file = provider.static_file_provider(); - static_file.commit()?; - provider.commit()?; - Ok(()) - } - - /// Commits both storage types in the right order for an unwind operation. - /// - /// For unwinding it makes more sense to commit the database first, since if - /// it is interrupted before the static files commit, we can just - /// truncate the static files according to the - /// checkpoints on the next start-up. - /// - /// NOTE: Should only be used after unwinding data from storage! - pub fn commit_unwind

(provider: P) -> ProviderResult<()> - where - P: DBProvider + StaticFileProviderFactory, - { - let static_file = provider.static_file_provider(); - provider.commit()?; - static_file.commit()?; - Ok(()) - } -} - -impl UnifiedStorageWriter<'_, ProviderDB, &StaticFileProvider> -where - ProviderDB: DBProvider - + BlockWriter - + TransactionsProviderExt - + TrieWriter - + StateWriter - + HistoryWriter - + StageCheckpointWriter - + BlockExecutionWriter - + AsRef - + StaticFileProviderFactory, -{ - /// Writes executed blocks and receipts to storage. - pub fn save_blocks(&self, blocks: Vec>) -> ProviderResult<()> - where - N: NodePrimitives, - ProviderDB: BlockWriter + StateWriter, - { - if blocks.is_empty() { - debug!(target: "provider::storage_writer", "Attempted to write empty block range"); - return Ok(()) - } - - // NOTE: checked non-empty above - let first_block = blocks.first().unwrap().recovered_block(); - - let last_block = blocks.last().unwrap().recovered_block(); - let first_number = first_block.number(); - let last_block_number = last_block.number(); - - debug!(target: "provider::storage_writer", block_count = %blocks.len(), "Writing blocks and execution data to storage"); - - // TODO: Do performant / batched writes for each type of object - // instead of a loop over all blocks, - // meaning: - // * blocks - // * state - // * hashed state - // * trie updates (cannot naively extend, need helper) - // * indices (already done basically) - // Insert the blocks - for ExecutedBlockWithTrieUpdates { - block: ExecutedBlock { recovered_block, execution_output, hashed_state }, - trie, - } in blocks - { - self.database() - .insert_block(Arc::unwrap_or_clone(recovered_block), StorageLocation::Both)?; - - // Write state and changesets to the database. - // Must be written after blocks because of the receipt lookup. - self.database().write_state( - &execution_output, - OriginalValuesKnown::No, - StorageLocation::StaticFiles, - )?; - - // insert hashes and intermediate merkle nodes - self.database() - .write_hashed_state(&Arc::unwrap_or_clone(hashed_state).into_sorted())?; - self.database().write_trie_updates(&trie)?; - } - - // update history indices - self.database().update_history_indices(first_number..=last_block_number)?; - - // Update pipeline progress - self.database().update_pipeline_stages(last_block_number, false)?; - - debug!(target: "provider::storage_writer", range = ?first_number..=last_block_number, "Appended block data"); - - Ok(()) - } - - /// Removes all block, transaction and receipt data above the given block number from the - /// database and static files. This is exclusive, i.e., it only removes blocks above - /// `block_number`, and does not remove `block_number`. - pub fn remove_blocks_above(&self, block_number: u64) -> ProviderResult<()> { - // IMPORTANT: we use `block_number+1` to make sure we remove only what is ABOVE the block - debug!(target: "provider::storage_writer", ?block_number, "Removing blocks from database above block_number"); - self.database().remove_block_and_execution_above(block_number, StorageLocation::Both)?; - - // Get highest static file block for the total block range - let highest_static_file_block = self - .static_file() - .get_highest_static_file_block(StaticFileSegment::Headers) - .expect("todo: error handling, headers should exist"); - - // IMPORTANT: we use `highest_static_file_block.saturating_sub(block_number)` to make sure - // we remove only what is ABOVE the block. - // - // i.e., if the highest static file block is 8, we want to remove above block 5 only, we - // will have three blocks to remove, which will be block 8, 7, and 6. - debug!(target: "provider::storage_writer", ?block_number, "Removing static file blocks above block_number"); - self.static_file() - .get_writer(block_number, StaticFileSegment::Headers)? - .prune_headers(highest_static_file_block.saturating_sub(block_number))?; - - Ok(()) - } -} - #[cfg(test)] mod tests { - use super::*; use crate::{ test_utils::create_test_provider_factory, AccountReader, StorageTrieWriter, TrieWriter, }; @@ -237,17 +13,17 @@ mod tests { use reth_ethereum_primitives::Receipt; use reth_execution_types::ExecutionOutcome; use reth_primitives_traits::{Account, StorageEntry}; - use reth_storage_api::{DatabaseProviderFactory, HashedPostStateProvider}; + use reth_storage_api::{DatabaseProviderFactory, HashedPostStateProvider, StateWriter}; use reth_trie::{ test_utils::{state_root, storage_root_prehashed}, - HashedPostState, HashedStorage, StateRoot, StorageRoot, + HashedPostState, HashedStorage, StateRoot, StorageRoot, StorageRootProgress, }; use reth_trie_db::{DatabaseStateRoot, DatabaseStorageRoot}; use revm_database::{ states::{ bundle_state::BundleRetention, changes::PlainStorageRevert, PlainStorageChangeset, }, - BundleState, State, + BundleState, OriginalValuesKnown, State, }; use revm_database_interface::{DatabaseCommit, EmptyDB}; use revm_state::{ @@ -333,6 +109,7 @@ mod tests { info: account_a.clone(), status: AccountStatus::Touched | AccountStatus::Created, storage: HashMap::default(), + transaction_id: 0, }, )])); @@ -343,6 +120,7 @@ mod tests { info: account_b_changed.clone(), status: AccountStatus::Touched, storage: HashMap::default(), + transaction_id: 0, }, )])); @@ -401,6 +179,7 @@ mod tests { status: AccountStatus::Touched | AccountStatus::SelfDestructed, info: account_b_changed, storage: HashMap::default(), + transaction_id: 0, }, )])); @@ -475,6 +254,7 @@ mod tests { EvmStorageSlot { present_value: U256::from(2), ..Default::default() }, ), ]), + transaction_id: 0, }, ), ( @@ -491,6 +271,7 @@ mod tests { ..Default::default() }, )]), + transaction_id: 0, }, ), ])); @@ -499,7 +280,7 @@ mod tests { let outcome = ExecutionOutcome::new(state.take_bundle(), Default::default(), 1, Vec::new()); provider - .write_state(&outcome, OriginalValuesKnown::Yes, StorageLocation::Database) + .write_state(&outcome, OriginalValuesKnown::Yes) .expect("Could not write bundle state to DB"); // Check plain storage state @@ -592,13 +373,14 @@ mod tests { status: AccountStatus::Touched | AccountStatus::SelfDestructed, info: RevmAccountInfo::default(), storage: HashMap::default(), + transaction_id: 0, }, )])); state.merge_transitions(BundleRetention::Reverts); let outcome = ExecutionOutcome::new(state.take_bundle(), Default::default(), 2, Vec::new()); provider - .write_state(&outcome, OriginalValuesKnown::Yes, StorageLocation::Database) + .write_state(&outcome, OriginalValuesKnown::Yes) .expect("Could not write bundle state to DB"); assert_eq!( @@ -658,6 +440,7 @@ mod tests { EvmStorageSlot { present_value: U256::from(2), ..Default::default() }, ), ]), + transaction_id: 0, }, )])); init_state.merge_transitions(BundleRetention::Reverts); @@ -665,7 +448,7 @@ mod tests { let outcome = ExecutionOutcome::new(init_state.take_bundle(), Default::default(), 0, Vec::new()); provider - .write_state(&outcome, OriginalValuesKnown::Yes, StorageLocation::Database) + .write_state(&outcome, OriginalValuesKnown::Yes) .expect("Could not write bundle state to DB"); let mut state = State::builder().with_bundle_update().build(); @@ -690,6 +473,7 @@ mod tests { ..Default::default() }, )]), + transaction_id: 0, }, )])); state.merge_transitions(BundleRetention::Reverts); @@ -701,6 +485,7 @@ mod tests { status: AccountStatus::Touched | AccountStatus::SelfDestructed, info: account_info.clone(), storage: HashMap::default(), + transaction_id: 0, }, )])); state.merge_transitions(BundleRetention::Reverts); @@ -712,6 +497,7 @@ mod tests { status: AccountStatus::Touched | AccountStatus::Created, info: account_info.clone(), storage: HashMap::default(), + transaction_id: 0, }, )])); state.merge_transitions(BundleRetention::Reverts); @@ -739,6 +525,7 @@ mod tests { EvmStorageSlot { present_value: U256::from(6), ..Default::default() }, ), ]), + transaction_id: 0, }, )])); state.merge_transitions(BundleRetention::Reverts); @@ -750,6 +537,7 @@ mod tests { status: AccountStatus::Touched | AccountStatus::SelfDestructed, info: account_info.clone(), storage: HashMap::default(), + transaction_id: 0, }, )])); state.merge_transitions(BundleRetention::Reverts); @@ -761,6 +549,7 @@ mod tests { status: AccountStatus::Touched | AccountStatus::Created, info: account_info.clone(), storage: HashMap::default(), + transaction_id: 0, }, )])); state.commit(HashMap::from_iter([( @@ -773,6 +562,7 @@ mod tests { U256::ZERO, EvmStorageSlot { present_value: U256::from(2), ..Default::default() }, )]), + transaction_id: 0, }, )])); state.commit(HashMap::from_iter([( @@ -781,6 +571,7 @@ mod tests { status: AccountStatus::Touched | AccountStatus::SelfDestructed, info: account_info.clone(), storage: HashMap::default(), + transaction_id: 0, }, )])); state.commit(HashMap::from_iter([( @@ -789,6 +580,7 @@ mod tests { status: AccountStatus::Touched | AccountStatus::Created, info: account_info.clone(), storage: HashMap::default(), + transaction_id: 0, }, )])); state.merge_transitions(BundleRetention::Reverts); @@ -804,8 +596,10 @@ mod tests { U256::ZERO, EvmStorageSlot { present_value: U256::from(9), ..Default::default() }, )]), + transaction_id: 0, }, )])); + state.merge_transitions(BundleRetention::Reverts); let bundle = state.take_bundle(); @@ -813,7 +607,7 @@ mod tests { let outcome: ExecutionOutcome = ExecutionOutcome::new(bundle, Default::default(), 1, Vec::new()); provider - .write_state(&outcome, OriginalValuesKnown::Yes, StorageLocation::Database) + .write_state(&outcome, OriginalValuesKnown::Yes) .expect("Could not write bundle state to DB"); let mut storage_changeset_cursor = provider @@ -972,13 +766,14 @@ mod tests { EvmStorageSlot { present_value: U256::from(2), ..Default::default() }, ), ]), + transaction_id: 0, }, )])); init_state.merge_transitions(BundleRetention::Reverts); let outcome = ExecutionOutcome::new(init_state.take_bundle(), Default::default(), 0, Vec::new()); provider - .write_state(&outcome, OriginalValuesKnown::Yes, StorageLocation::Database) + .write_state(&outcome, OriginalValuesKnown::Yes) .expect("Could not write bundle state to DB"); let mut state = State::builder().with_bundle_update().build(); @@ -995,6 +790,7 @@ mod tests { status: AccountStatus::Touched | AccountStatus::SelfDestructed, info: account1.clone(), storage: HashMap::default(), + transaction_id: 0, }, )])); @@ -1004,6 +800,7 @@ mod tests { status: AccountStatus::Touched | AccountStatus::Created, info: account1.clone(), storage: HashMap::default(), + transaction_id: 0, }, )])); @@ -1017,6 +814,7 @@ mod tests { U256::from(1), EvmStorageSlot { present_value: U256::from(5), ..Default::default() }, )]), + transaction_id: 0, }, )])); @@ -1024,7 +822,7 @@ mod tests { state.merge_transitions(BundleRetention::Reverts); let outcome = ExecutionOutcome::new(state.take_bundle(), Default::default(), 1, Vec::new()); provider - .write_state(&outcome, OriginalValuesKnown::Yes, StorageLocation::Database) + .write_state(&outcome, OriginalValuesKnown::Yes) .expect("Could not write bundle state to DB"); let mut storage_changeset_cursor = provider @@ -1111,7 +909,7 @@ mod tests { } let (_, updates) = StateRoot::from_tx(tx).root_with_updates().unwrap(); - provider_rw.write_trie_updates(&updates).unwrap(); + provider_rw.write_trie_updates(updates).unwrap(); let mut state = State::builder().with_bundle_update().build(); @@ -1143,6 +941,7 @@ mod tests { status: AccountStatus::Touched | AccountStatus::SelfDestructed, info: RevmAccountInfo::default(), storage: HashMap::default(), + transaction_id: 0, }, )])); state.merge_transitions(BundleRetention::PlainState); @@ -1169,8 +968,13 @@ mod tests { info: account2.0.into(), storage: HashMap::from_iter([( slot2, - EvmStorageSlot::new_changed(account2_slot2_old_value, account2_slot2_new_value), + EvmStorageSlot::new_changed( + account2_slot2_old_value, + account2_slot2_new_value, + 0, + ), )]), + transaction_id: 0, }, )])); state.merge_transitions(BundleRetention::PlainState); @@ -1188,6 +992,7 @@ mod tests { status: AccountStatus::Touched, info: account3.0.into(), storage: HashMap::default(), + transaction_id: 0, }, )])); state.merge_transitions(BundleRetention::PlainState); @@ -1205,6 +1010,7 @@ mod tests { status: AccountStatus::Touched, info: account4.0.into(), storage: HashMap::default(), + transaction_id: 0, }, )])); state.merge_transitions(BundleRetention::PlainState); @@ -1220,6 +1026,7 @@ mod tests { status: AccountStatus::Touched | AccountStatus::Created, info: account1_new.into(), storage: HashMap::default(), + transaction_id: 0, }, )])); state.merge_transitions(BundleRetention::PlainState); @@ -1237,8 +1044,9 @@ mod tests { info: account1_new.into(), storage: HashMap::from_iter([( slot20, - EvmStorageSlot::new_changed(U256::ZERO, account1_slot20_value), + EvmStorageSlot::new_changed(U256::ZERO, account1_slot20_value, 0), )]), + transaction_id: 0, }, )])); state.merge_transitions(BundleRetention::PlainState); @@ -1308,12 +1116,21 @@ mod tests { provider_rw.write_hashed_state(&state.clone().into_sorted()).unwrap(); // calculate database storage root and write intermediate storage nodes. - let (storage_root, _, storage_updates) = - StorageRoot::from_tx_hashed(tx, hashed_address).calculate(true).unwrap(); + let StorageRootProgress::Complete(storage_root, _, storage_updates) = + StorageRoot::from_tx_hashed(tx, hashed_address) + .with_no_threshold() + .calculate(true) + .unwrap() + else { + panic!("no threshold for root"); + }; assert_eq!(storage_root, storage_root_prehashed(init_storage.storage)); assert!(!storage_updates.is_empty()); provider_rw - .write_individual_storage_trie_updates(hashed_address, &storage_updates) + .write_storage_trie_updates_sorted(core::iter::once(( + &hashed_address, + &storage_updates.into_sorted(), + ))) .unwrap(); // destroy the storage and re-create with new slots diff --git a/crates/storage/rpc-provider/Cargo.toml b/crates/storage/rpc-provider/Cargo.toml new file mode 100644 index 00000000000..a47bf7ea218 --- /dev/null +++ b/crates/storage/rpc-provider/Cargo.toml @@ -0,0 +1,49 @@ +[package] +name = "reth-storage-rpc-provider" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true +description = "RPC-based blockchain provider for reth that fetches data via RPC calls" + +[lints] +workspace = true + +[dependencies] +# reth +reth-storage-api.workspace = true +reth-chainspec.workspace = true +reth-primitives.workspace = true +reth-provider.workspace = true +reth-errors.workspace = true +reth-execution-types.workspace = true +reth-prune-types.workspace = true +reth-node-types.workspace = true +reth-trie.workspace = true +reth-stages-types.workspace = true +reth-db-api.workspace = true +reth-rpc-convert.workspace = true + +# alloy +alloy-provider = { workspace = true, features = ["debug-api"] } +alloy-network.workspace = true +alloy-primitives.workspace = true +alloy-consensus.workspace = true +alloy-rpc-types.workspace = true +alloy-rpc-types-engine.workspace = true +alloy-eips.workspace = true + +# async +tokio = { workspace = true, features = ["sync", "macros", "rt-multi-thread"] } + +# other +tracing.workspace = true +parking_lot.workspace = true + +# revm +revm.workspace = true + +[dev-dependencies] +tokio = { workspace = true, features = ["rt", "macros"] } diff --git a/crates/storage/rpc-provider/README.md b/crates/storage/rpc-provider/README.md new file mode 100644 index 00000000000..7180d41840d --- /dev/null +++ b/crates/storage/rpc-provider/README.md @@ -0,0 +1,71 @@ +# RPC Blockchain Provider for Reth + +This crate provides an RPC-based implementation of reth's [`BlockchainProvider`](../provider/src/providers/blockchain_provider.rs) which provides access to local blockchain data, this crate offers the same functionality but for remote blockchain access via RPC. + +Originally created by [cakevm](https://github.com/cakevm/alloy-reth-provider). + +## Features + +- Provides the same interface as `BlockchainProvider` but for remote nodes +- Implements `StateProviderFactory` for remote RPC state access +- Supports Ethereum networks +- Useful for testing without requiring a full database +- Can be used with reth ExEx (Execution Extensions) for testing + +## Usage + +```rust +use alloy_provider::ProviderBuilder; +use reth_storage_rpc_provider::RpcBlockchainProvider; + +// Initialize provider +let provider = ProviderBuilder::new() + .builtin("https://eth.merkle.io") + .await + .unwrap(); + +// Create RPC blockchain provider with NodeTypes +let rpc_provider = RpcBlockchainProvider::new(provider); + +// Get state at specific block - same interface as BlockchainProvider +let state = rpc_provider.state_by_block_id(BlockId::number(16148323)).unwrap(); +``` + +## Configuration + +The provider can be configured with custom settings: + +```rust +use reth_storage_rpc_provider::{RpcBlockchainProvider, RpcBlockchainProviderConfig}; + +let config = RpcBlockchainProviderConfig { + compute_state_root: true, // Enable state root computation + reth_rpc_support: true, // Use Reth-specific RPC methods (default: true) +}; + +let rpc_provider = RpcBlockchainProvider::new_with_config(provider, config); +``` + +## Configuration Options + +- `compute_state_root`: When enabled, computes state root and trie updates (requires Reth-specific RPC methods) +- `reth_rpc_support`: When enabled (default), uses Reth-specific RPC methods for better performance: + - `eth_getAccountInfo`: Fetches account balance, nonce, and code in a single call + - `debug_codeByHash`: Retrieves bytecode by hash without needing the address + + When disabled, falls back to standard RPC methods and caches bytecode locally for compatibility with non-Reth nodes. + +## Technical Details + +The `RpcBlockchainProvider` uses `alloy_network::AnyNetwork` for network operations, providing compatibility with various Ethereum-based networks while maintaining the expected block structure with headers. + +This provider implements the same traits as the local `BlockchainProvider`, making it a drop-in replacement for scenarios where remote RPC access is preferred over local database access. + +## License + +Licensed under either of: + +- Apache License, Version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) +- MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT) + +at your option. diff --git a/crates/storage/rpc-provider/src/lib.rs b/crates/storage/rpc-provider/src/lib.rs new file mode 100644 index 00000000000..6e5bd17218b --- /dev/null +++ b/crates/storage/rpc-provider/src/lib.rs @@ -0,0 +1,1918 @@ +//! # RPC Blockchain Provider for Reth +//! +//! This crate provides an RPC-based implementation of reth's `StateProviderFactory` and related +//! traits that fetches blockchain data via RPC instead of from a local database. +//! +//! Similar to the [`BlockchainProvider`](../../provider/src/providers/blockchain_provider.rs) +//! which provides access to local blockchain data, this crate offers the same functionality but for +//! remote blockchain access via RPC. +//! +//! Originally created by [cakevm](https://github.com/cakevm/alloy-reth-provider). +//! +//! ## Features +//! +//! - Implements `StateProviderFactory` for remote RPC state access +//! - Supports Ethereum and Optimism network +//! - Useful for testing without requiring a full database +//! - Can be used with reth ExEx (Execution Extensions) for testing + +#![doc( + html_logo_url = "https://raw.githubusercontent.com/paradigmxyz/reth/main/assets/reth-docs.png", + html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256", + issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" +)] +#![cfg_attr(not(test), warn(unused_crate_dependencies))] +#![cfg_attr(docsrs, feature(doc_cfg))] + +use alloy_consensus::{constants::KECCAK_EMPTY, BlockHeader}; +use alloy_eips::{BlockHashOrNumber, BlockNumberOrTag}; +use alloy_network::{primitives::HeaderResponse, BlockResponse}; +use alloy_primitives::{ + map::HashMap, Address, BlockHash, BlockNumber, StorageKey, TxHash, TxNumber, B256, U256, +}; +use alloy_provider::{ext::DebugApi, network::Network, Provider}; +use alloy_rpc_types::{AccountInfo, BlockId}; +use alloy_rpc_types_engine::ForkchoiceState; +use parking_lot::RwLock; +use reth_chainspec::{ChainInfo, ChainSpecProvider}; +use reth_db_api::{ + mock::{DatabaseMock, TxMock}, + models::StoredBlockBodyIndices, +}; +use reth_errors::{ProviderError, ProviderResult}; +use reth_node_types::{ + Block, BlockBody, BlockTy, HeaderTy, NodeTypes, PrimitivesTy, ReceiptTy, TxTy, +}; +use reth_primitives::{Account, Bytecode, RecoveredBlock, SealedHeader, TransactionMeta}; +use reth_provider::{ + AccountReader, BlockHashReader, BlockIdReader, BlockNumReader, BlockReader, BytecodeReader, + CanonChainTracker, CanonStateNotification, CanonStateNotifications, CanonStateSubscriptions, + ChainStateBlockReader, ChainStateBlockWriter, ChangeSetReader, DatabaseProviderFactory, + HeaderProvider, PruneCheckpointReader, ReceiptProvider, StageCheckpointReader, StateProvider, + StateProviderBox, StateProviderFactory, StateReader, StateRootProvider, StorageReader, + TransactionVariant, TransactionsProvider, +}; +use reth_prune_types::{PruneCheckpoint, PruneSegment}; +use reth_rpc_convert::{TryFromBlockResponse, TryFromReceiptResponse, TryFromTransactionResponse}; +use reth_stages_types::{StageCheckpoint, StageId}; +use reth_storage_api::{ + BlockBodyIndicesProvider, BlockReaderIdExt, BlockSource, DBProvider, NodePrimitivesProvider, + ReceiptProviderIdExt, StatsReader, +}; +use reth_trie::{updates::TrieUpdates, AccountProof, HashedPostState, MultiProof, TrieInput}; +use std::{ + collections::BTreeMap, + future::{Future, IntoFuture}, + ops::{RangeBounds, RangeInclusive}, + sync::Arc, +}; +use tokio::{runtime::Handle, sync::broadcast}; +use tracing::{trace, warn}; + +/// Configuration for `RpcBlockchainProvider` +#[derive(Debug, Clone)] +pub struct RpcBlockchainProviderConfig { + /// Whether to compute state root when creating execution outcomes + pub compute_state_root: bool, + /// Whether to use Reth-specific RPC methods for better performance + /// + /// If enabled, the node will use Reth's RPC methods (`debug_codeByHash` and + /// `eth_getAccountInfo`) to speed up account information retrieval. When disabled, it will + /// use multiple standard RPC calls to get account information. + pub reth_rpc_support: bool, +} + +impl Default for RpcBlockchainProviderConfig { + fn default() -> Self { + Self { compute_state_root: false, reth_rpc_support: true } + } +} + +impl RpcBlockchainProviderConfig { + /// Sets whether to compute state root when creating execution outcomes + pub const fn with_compute_state_root(mut self, compute: bool) -> Self { + self.compute_state_root = compute; + self + } + + /// Sets whether to use Reth-specific RPC methods for better performance + pub const fn with_reth_rpc_support(mut self, support: bool) -> Self { + self.reth_rpc_support = support; + self + } +} + +/// An RPC-based blockchain provider that fetches blockchain data via remote RPC calls. +/// +/// This is the RPC equivalent of +/// [`BlockchainProvider`](../../provider/src/providers/blockchain_provider.rs), implementing +/// the same `StateProviderFactory` and related traits but fetching data from a remote node instead +/// of local storage. +/// +/// This provider is useful for: +/// - Testing without requiring a full local database +/// - Accessing blockchain state from remote nodes +/// - Building light clients or tools that don't need full node storage +/// +/// The provider type is generic over the network type N (defaulting to `AnyNetwork`), +/// but the current implementation is specialized for `alloy_network::AnyNetwork` +/// as it needs to access block header fields directly. +#[derive(Clone)] +pub struct RpcBlockchainProvider +where + Node: NodeTypes, +{ + /// The underlying Alloy provider + provider: P, + /// Node types marker + node_types: std::marker::PhantomData, + /// Network marker + network: std::marker::PhantomData, + /// Broadcast channel for canon state notifications + canon_state_notification: broadcast::Sender>>, + /// Configuration for the provider + config: RpcBlockchainProviderConfig, + /// Cached chain spec + chain_spec: Arc, +} + +impl std::fmt::Debug for RpcBlockchainProvider { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RpcBlockchainProvider").field("config", &self.config).finish() + } +} + +impl RpcBlockchainProvider { + /// Creates a new `RpcBlockchainProvider` with default configuration + pub fn new(provider: P) -> Self + where + Node::ChainSpec: Default, + { + Self::new_with_config(provider, RpcBlockchainProviderConfig::default()) + } + + /// Creates a new `RpcBlockchainProvider` with custom configuration + pub fn new_with_config(provider: P, config: RpcBlockchainProviderConfig) -> Self + where + Node::ChainSpec: Default, + { + let (canon_state_notification, _) = broadcast::channel(1); + Self { + provider, + node_types: std::marker::PhantomData, + network: std::marker::PhantomData, + canon_state_notification, + config, + chain_spec: Arc::new(Node::ChainSpec::default()), + } + } + + /// Use a custom chain spec for the provider + pub fn with_chain_spec(self, chain_spec: Arc) -> Self { + Self { + provider: self.provider, + node_types: std::marker::PhantomData, + network: std::marker::PhantomData, + canon_state_notification: self.canon_state_notification, + config: self.config, + chain_spec, + } + } + + /// Helper function to execute async operations in a blocking context + fn block_on_async(&self, fut: F) -> T + where + F: Future, + { + tokio::task::block_in_place(move || Handle::current().block_on(fut)) + } + + /// Get a reference to the canon state notification sender + pub const fn canon_state_notification( + &self, + ) -> &broadcast::Sender>> { + &self.canon_state_notification + } +} + +impl RpcBlockchainProvider +where + P: Provider + Clone + 'static, + N: Network, + Node: NodeTypes, +{ + /// Helper function to create a state provider for a given block ID + fn create_state_provider(&self, block_id: BlockId) -> RpcBlockchainStateProvider { + RpcBlockchainStateProvider::with_chain_spec( + self.provider.clone(), + block_id, + self.chain_spec.clone(), + ) + .with_compute_state_root(self.config.compute_state_root) + .with_reth_rpc_support(self.config.reth_rpc_support) + } + + /// Helper function to get state provider by block number + fn state_by_block_number( + &self, + block_number: BlockNumber, + ) -> Result { + Ok(Box::new(self.create_state_provider(BlockId::number(block_number)))) + } +} + +// Implementation note: While the types are generic over Network N, the trait implementations +// are specialized for AnyNetwork because they need to access block header fields. +// This allows the types to be instantiated with any network while the actual functionality +// requires AnyNetwork. Future improvements could add trait bounds for networks with +// compatible block structures. +impl BlockHashReader for RpcBlockchainProvider +where + P: Provider + Clone + 'static, + N: Network, + Node: NodeTypes, +{ + fn block_hash(&self, number: BlockNumber) -> Result, ProviderError> { + let block = self.block_on_async(async { + self.provider.get_block_by_number(number.into()).await.map_err(ProviderError::other) + })?; + Ok(block.map(|b| b.header().hash())) + } + + fn canonical_hashes_range( + &self, + _start: BlockNumber, + _end: BlockNumber, + ) -> Result, ProviderError> { + // Would need to make multiple RPC calls + Err(ProviderError::UnsupportedProvider) + } +} + +impl BlockNumReader for RpcBlockchainProvider +where + P: Provider + Clone + 'static, + N: Network, + Node: NodeTypes, +{ + fn chain_info(&self) -> Result { + self.block_on_async(async { + let block = self + .provider + .get_block(BlockId::Number(BlockNumberOrTag::Latest)) + .await + .map_err(ProviderError::other)? + .ok_or(ProviderError::HeaderNotFound(0.into()))?; + + Ok(ChainInfo { best_hash: block.header().hash(), best_number: block.header().number() }) + }) + } + + fn best_block_number(&self) -> Result { + self.block_on_async(async { + self.provider.get_block_number().await.map_err(ProviderError::other) + }) + } + + fn last_block_number(&self) -> Result { + self.best_block_number() + } + + fn block_number(&self, hash: B256) -> Result, ProviderError> { + let block = self.block_on_async(async { + self.provider.get_block_by_hash(hash).await.map_err(ProviderError::other) + })?; + Ok(block.map(|b| b.header().number())) + } +} + +impl BlockIdReader for RpcBlockchainProvider +where + P: Provider + Clone + 'static, + N: Network, + Node: NodeTypes, +{ + fn block_number_for_id(&self, block_id: BlockId) -> Result, ProviderError> { + match block_id { + BlockId::Hash(hash) => { + let block = self.block_on_async(async { + self.provider + .get_block_by_hash(hash.block_hash) + .await + .map_err(ProviderError::other) + })?; + Ok(block.map(|b| b.header().number())) + } + BlockId::Number(number_or_tag) => match number_or_tag { + alloy_rpc_types::BlockNumberOrTag::Number(num) => Ok(Some(num)), + alloy_rpc_types::BlockNumberOrTag::Latest => self.block_on_async(async { + self.provider.get_block_number().await.map(Some).map_err(ProviderError::other) + }), + _ => Ok(None), + }, + } + } + + fn pending_block_num_hash(&self) -> Result, ProviderError> { + // RPC doesn't provide pending block number and hash together + Err(ProviderError::UnsupportedProvider) + } + + fn safe_block_num_hash(&self) -> Result, ProviderError> { + // RPC doesn't provide safe block number and hash + Err(ProviderError::UnsupportedProvider) + } + + fn finalized_block_num_hash(&self) -> Result, ProviderError> { + // RPC doesn't provide finalized block number and hash + Err(ProviderError::UnsupportedProvider) + } +} + +impl HeaderProvider for RpcBlockchainProvider +where + P: Provider + Clone + 'static, + N: Network, + Node: NodeTypes, + BlockTy: TryFromBlockResponse, +{ + type Header = HeaderTy; + + fn header(&self, block_hash: BlockHash) -> ProviderResult> { + let block_response = self.block_on_async(async { + self.provider.get_block_by_hash(block_hash).await.map_err(ProviderError::other) + })?; + + let Some(block_response) = block_response else { + // If the block was not found, return None + return Ok(None); + }; + + // Convert the network block response to primitive block + let block = as TryFromBlockResponse>::from_block_response(block_response) + .map_err(ProviderError::other)?; + + Ok(Some(block.into_header())) + } + + fn header_by_number(&self, num: u64) -> ProviderResult> { + let Some(sealed_header) = self.sealed_header(num)? else { + // If the block was not found, return None + return Ok(None); + }; + + Ok(Some(sealed_header.into_header())) + } + + fn headers_range( + &self, + _range: impl RangeBounds, + ) -> ProviderResult> { + Err(ProviderError::UnsupportedProvider) + } + + fn sealed_header( + &self, + number: BlockNumber, + ) -> ProviderResult>> { + let block_response = self.block_on_async(async { + self.provider.get_block_by_number(number.into()).await.map_err(ProviderError::other) + })?; + + let Some(block_response) = block_response else { + // If the block was not found, return None + return Ok(None); + }; + let block_hash = block_response.header().hash(); + + // Convert the network block response to primitive block + let block = as TryFromBlockResponse>::from_block_response(block_response) + .map_err(ProviderError::other)?; + + Ok(Some(SealedHeader::new(block.into_header(), block_hash))) + } + + fn sealed_headers_while( + &self, + _range: impl RangeBounds, + _predicate: impl FnMut(&SealedHeader) -> bool, + ) -> ProviderResult>> { + Err(ProviderError::UnsupportedProvider) + } +} + +impl BlockBodyIndicesProvider for RpcBlockchainProvider +where + P: Provider + Clone + 'static, + N: Network, + Node: NodeTypes, +{ + fn block_body_indices(&self, _num: u64) -> ProviderResult> { + Err(ProviderError::UnsupportedProvider) + } + + fn block_body_indices_range( + &self, + _range: RangeInclusive, + ) -> ProviderResult> { + Err(ProviderError::UnsupportedProvider) + } +} + +impl BlockReader for RpcBlockchainProvider +where + P: Provider + Clone + 'static, + N: Network, + Node: NodeTypes, + BlockTy: TryFromBlockResponse, + TxTy: TryFromTransactionResponse, + ReceiptTy: TryFromReceiptResponse, +{ + type Block = BlockTy; + + fn find_block_by_hash( + &self, + _hash: B256, + _source: BlockSource, + ) -> ProviderResult> { + Err(ProviderError::UnsupportedProvider) + } + + fn block(&self, id: BlockHashOrNumber) -> ProviderResult> { + let block_response = self.block_on_async(async { + self.provider.get_block(id.into()).full().await.map_err(ProviderError::other) + })?; + + let Some(block_response) = block_response else { + // If the block was not found, return None + return Ok(None); + }; + + // Convert the network block response to primitive block + let block = as TryFromBlockResponse>::from_block_response(block_response) + .map_err(ProviderError::other)?; + + Ok(Some(block)) + } + + fn pending_block(&self) -> ProviderResult>> { + Err(ProviderError::UnsupportedProvider) + } + + fn pending_block_and_receipts( + &self, + ) -> ProviderResult, Vec)>> { + Err(ProviderError::UnsupportedProvider) + } + + fn recovered_block( + &self, + _id: BlockHashOrNumber, + _transaction_kind: TransactionVariant, + ) -> ProviderResult>> { + Err(ProviderError::UnsupportedProvider) + } + + fn sealed_block_with_senders( + &self, + _id: BlockHashOrNumber, + _transaction_kind: TransactionVariant, + ) -> ProviderResult>> { + Err(ProviderError::UnsupportedProvider) + } + + fn block_range(&self, _range: RangeInclusive) -> ProviderResult> { + Err(ProviderError::UnsupportedProvider) + } + + fn block_with_senders_range( + &self, + _range: RangeInclusive, + ) -> ProviderResult>> { + Err(ProviderError::UnsupportedProvider) + } + + fn recovered_block_range( + &self, + _range: RangeInclusive, + ) -> ProviderResult>> { + Err(ProviderError::UnsupportedProvider) + } + + fn block_by_transaction_id(&self, _id: TxNumber) -> ProviderResult> { + Err(ProviderError::UnsupportedProvider) + } +} + +impl BlockReaderIdExt for RpcBlockchainProvider +where + P: Provider + Clone + 'static, + N: Network, + Node: NodeTypes, + BlockTy: TryFromBlockResponse, + TxTy: TryFromTransactionResponse, + ReceiptTy: TryFromReceiptResponse, +{ + fn block_by_id(&self, id: BlockId) -> ProviderResult> { + match id { + BlockId::Hash(hash) => self.block_by_hash(hash.block_hash), + BlockId::Number(number_or_tag) => self.block_by_number_or_tag(number_or_tag), + } + } + + fn sealed_header_by_id( + &self, + id: BlockId, + ) -> ProviderResult>> { + match id { + BlockId::Hash(hash) => self.sealed_header_by_hash(hash.block_hash), + BlockId::Number(number_or_tag) => self.sealed_header_by_number_or_tag(number_or_tag), + } + } + + fn header_by_id(&self, id: BlockId) -> ProviderResult> { + match id { + BlockId::Hash(hash) => self.header_by_hash_or_number(hash.block_hash.into()), + BlockId::Number(number_or_tag) => self.header_by_number_or_tag(number_or_tag), + } + } +} + +impl ReceiptProvider for RpcBlockchainProvider +where + P: Provider + Clone + 'static, + N: Network, + Node: NodeTypes, + ReceiptTy: TryFromReceiptResponse, +{ + type Receipt = ReceiptTy; + + fn receipt(&self, _id: TxNumber) -> ProviderResult> { + Err(ProviderError::UnsupportedProvider) + } + + fn receipt_by_hash(&self, hash: TxHash) -> ProviderResult> { + let receipt_response = self.block_on_async(async { + self.provider.get_transaction_receipt(hash).await.map_err(ProviderError::other) + })?; + + let Some(receipt_response) = receipt_response else { + // If the receipt was not found, return None + return Ok(None); + }; + + // Convert the network receipt response to primitive receipt + let receipt = + as TryFromReceiptResponse>::from_receipt_response(receipt_response) + .map_err(ProviderError::other)?; + + Ok(Some(receipt)) + } + + fn receipts_by_block( + &self, + block: BlockHashOrNumber, + ) -> ProviderResult>> { + self.block_on_async(async { + let receipts_response = self + .provider + .get_block_receipts(block.into()) + .await + .map_err(ProviderError::other)?; + + let Some(receipts) = receipts_response else { + // If the receipts were not found, return None + return Ok(None); + }; + + // Convert the network receipts response to primitive receipts + let receipts = receipts + .into_iter() + .map(|receipt_response| { + as TryFromReceiptResponse>::from_receipt_response( + receipt_response, + ) + .map_err(ProviderError::other) + }) + .collect::, _>>()?; + + Ok(Some(receipts)) + }) + } + + fn receipts_by_tx_range( + &self, + _range: impl RangeBounds, + ) -> ProviderResult> { + Err(ProviderError::UnsupportedProvider) + } + + fn receipts_by_block_range( + &self, + _block_range: RangeInclusive, + ) -> ProviderResult>> { + Err(ProviderError::UnsupportedProvider) + } +} + +impl ReceiptProviderIdExt for RpcBlockchainProvider +where + P: Provider + Clone + 'static, + N: Network, + Node: NodeTypes, + ReceiptTy: TryFromReceiptResponse, +{ +} + +impl TransactionsProvider for RpcBlockchainProvider +where + P: Provider + Clone + 'static, + N: Network, + Node: NodeTypes, + BlockTy: TryFromBlockResponse, + TxTy: TryFromTransactionResponse, +{ + type Transaction = TxTy; + + fn transaction_id(&self, _tx_hash: TxHash) -> ProviderResult> { + Err(ProviderError::UnsupportedProvider) + } + + fn transaction_by_id(&self, _id: TxNumber) -> ProviderResult> { + Err(ProviderError::UnsupportedProvider) + } + + fn transaction_by_id_unhashed( + &self, + _id: TxNumber, + ) -> ProviderResult> { + Err(ProviderError::UnsupportedProvider) + } + + fn transaction_by_hash(&self, hash: TxHash) -> ProviderResult> { + let transaction_response = self.block_on_async(async { + self.provider.get_transaction_by_hash(hash).await.map_err(ProviderError::other) + })?; + + let Some(transaction_response) = transaction_response else { + // If the transaction was not found, return None + return Ok(None); + }; + + // Convert the network transaction response to primitive transaction + let transaction = as TryFromTransactionResponse>::from_transaction_response( + transaction_response, + ) + .map_err(ProviderError::other)?; + + Ok(Some(transaction)) + } + + fn transaction_by_hash_with_meta( + &self, + _hash: TxHash, + ) -> ProviderResult> { + Err(ProviderError::UnsupportedProvider) + } + + fn transaction_block(&self, _id: TxNumber) -> ProviderResult> { + Err(ProviderError::UnsupportedProvider) + } + + fn transactions_by_block( + &self, + block: BlockHashOrNumber, + ) -> ProviderResult>> { + let block_response = self.block_on_async(async { + self.provider.get_block(block.into()).full().await.map_err(ProviderError::other) + })?; + + let Some(block_response) = block_response else { + // If the block was not found, return None + return Ok(None); + }; + + // Convert the network block response to primitive block + let block = as TryFromBlockResponse>::from_block_response(block_response) + .map_err(ProviderError::other)?; + + Ok(Some(block.into_body().into_transactions())) + } + + fn transactions_by_block_range( + &self, + _range: impl RangeBounds, + ) -> ProviderResult>> { + Err(ProviderError::UnsupportedProvider) + } + + fn transactions_by_tx_range( + &self, + _range: impl RangeBounds, + ) -> ProviderResult> { + Err(ProviderError::UnsupportedProvider) + } + + fn senders_by_tx_range( + &self, + _range: impl RangeBounds, + ) -> ProviderResult> { + Err(ProviderError::UnsupportedProvider) + } + + fn transaction_sender(&self, _id: TxNumber) -> ProviderResult> { + Err(ProviderError::UnsupportedProvider) + } +} + +impl StateProviderFactory for RpcBlockchainProvider +where + P: Provider + Clone + 'static, + N: Network, + Node: NodeTypes, +{ + fn latest(&self) -> Result { + Ok(Box::new(self.create_state_provider(self.best_block_number()?.into()))) + } + + fn state_by_block_id(&self, block_id: BlockId) -> Result { + Ok(Box::new(self.create_state_provider(block_id))) + } + + fn state_by_block_number_or_tag( + &self, + number_or_tag: alloy_rpc_types::BlockNumberOrTag, + ) -> Result { + match number_or_tag { + alloy_rpc_types::BlockNumberOrTag::Latest => self.latest(), + alloy_rpc_types::BlockNumberOrTag::Pending => self.pending(), + alloy_rpc_types::BlockNumberOrTag::Number(num) => self.state_by_block_number(num), + _ => Err(ProviderError::UnsupportedProvider), + } + } + + fn history_by_block_number( + &self, + block_number: BlockNumber, + ) -> Result { + self.state_by_block_number(block_number) + } + + fn history_by_block_hash( + &self, + block_hash: BlockHash, + ) -> Result { + self.state_by_block_hash(block_hash) + } + + fn state_by_block_hash( + &self, + block_hash: BlockHash, + ) -> Result { + trace!(target: "alloy-provider", ?block_hash, "Getting state provider by block hash"); + + let block = self.block_on_async(async { + self.provider + .get_block_by_hash(block_hash) + .await + .map_err(ProviderError::other)? + .ok_or(ProviderError::BlockHashNotFound(block_hash)) + })?; + + let block_number = block.header().number(); + Ok(Box::new(self.create_state_provider(BlockId::number(block_number)))) + } + + fn pending(&self) -> Result { + trace!(target: "alloy-provider", "Getting pending state provider"); + self.latest() + } + + fn pending_state_by_hash( + &self, + _block_hash: B256, + ) -> Result, ProviderError> { + // RPC provider doesn't support pending state by hash + Err(ProviderError::UnsupportedProvider) + } + + fn maybe_pending(&self) -> Result, ProviderError> { + Ok(None) + } +} + +impl DatabaseProviderFactory for RpcBlockchainProvider +where + P: Provider + Clone + 'static, + N: Network, + Node: NodeTypes, +{ + type DB = DatabaseMock; + type Provider = RpcBlockchainStateProvider; + type ProviderRW = RpcBlockchainStateProvider; + + fn database_provider_ro(&self) -> Result { + // RPC provider returns a new state provider + let block_number = self.block_on_async(async { + self.provider.get_block_number().await.map_err(ProviderError::other) + })?; + + Ok(self.create_state_provider(BlockId::number(block_number))) + } + + fn database_provider_rw(&self) -> Result { + // RPC provider returns a new state provider + let block_number = self.block_on_async(async { + self.provider.get_block_number().await.map_err(ProviderError::other) + })?; + + Ok(self.create_state_provider(BlockId::number(block_number))) + } +} + +impl CanonChainTracker for RpcBlockchainProvider +where + P: Provider + Clone + 'static, + N: Network, + Node: NodeTypes, +{ + type Header = alloy_consensus::Header; + fn on_forkchoice_update_received(&self, _update: &ForkchoiceState) { + // No-op for RPC provider + } + + fn last_received_update_timestamp(&self) -> Option { + None + } + + fn set_canonical_head(&self, _header: SealedHeader) { + // No-op for RPC provider + } + + fn set_safe(&self, _header: SealedHeader) { + // No-op for RPC provider + } + + fn set_finalized(&self, _header: SealedHeader) { + // No-op for RPC provider + } +} + +impl NodePrimitivesProvider for RpcBlockchainProvider +where + P: Send + Sync, + N: Send + Sync, + Node: NodeTypes, +{ + type Primitives = PrimitivesTy; +} + +impl CanonStateSubscriptions for RpcBlockchainProvider +where + P: Provider + Clone + 'static, + N: Network, + Node: NodeTypes, +{ + fn subscribe_to_canonical_state(&self) -> CanonStateNotifications> { + trace!(target: "alloy-provider", "Subscribing to canonical state notifications"); + self.canon_state_notification.subscribe() + } +} + +impl ChainSpecProvider for RpcBlockchainProvider +where + P: Send + Sync, + N: Send + Sync, + Node: NodeTypes, + Node::ChainSpec: Default, +{ + type ChainSpec = Node::ChainSpec; + + fn chain_spec(&self) -> Arc { + self.chain_spec.clone() + } +} + +/// RPC-based state provider implementation that fetches blockchain state via remote RPC calls. +/// +/// This is the state provider counterpart to `RpcBlockchainProvider`, handling state queries +/// at specific block heights via RPC instead of local database access. +pub struct RpcBlockchainStateProvider +where + Node: NodeTypes, +{ + /// The underlying Alloy provider + provider: P, + /// The block ID to fetch state at + block_id: BlockId, + /// Node types marker + node_types: std::marker::PhantomData, + /// Network marker + network: std::marker::PhantomData, + /// Cached chain spec (shared with parent provider) + chain_spec: Option>, + /// Whether to enable state root calculation + compute_state_root: bool, + /// Cached bytecode for accounts + /// + /// Since the state provider is short-lived, we don't worry about memory leaks. + code_store: RwLock>, + /// Whether to use Reth-specific RPC methods for better performance + reth_rpc_support: bool, +} + +impl std::fmt::Debug + for RpcBlockchainStateProvider +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RpcBlockchainStateProvider") + .field("provider", &self.provider) + .field("block_id", &self.block_id) + .finish() + } +} + +impl RpcBlockchainStateProvider { + /// Creates a new state provider for the given block + pub fn new( + provider: P, + block_id: BlockId, + _primitives: std::marker::PhantomData, + ) -> Self { + Self { + provider, + block_id, + node_types: std::marker::PhantomData, + network: std::marker::PhantomData, + chain_spec: None, + compute_state_root: false, + code_store: RwLock::new(HashMap::default()), + reth_rpc_support: true, + } + } + + /// Creates a new state provider with a cached chain spec + pub fn with_chain_spec( + provider: P, + block_id: BlockId, + chain_spec: Arc, + ) -> Self { + Self { + provider, + block_id, + node_types: std::marker::PhantomData, + network: std::marker::PhantomData, + chain_spec: Some(chain_spec), + compute_state_root: false, + code_store: RwLock::new(HashMap::default()), + reth_rpc_support: true, + } + } + + /// Helper function to execute async operations in a blocking context + fn block_on_async(&self, fut: F) -> T + where + F: Future, + { + tokio::task::block_in_place(move || Handle::current().block_on(fut)) + } + + /// Helper function to create a new state provider with a different block ID + fn with_block_id(&self, block_id: BlockId) -> Self { + Self { + provider: self.provider.clone(), + block_id, + node_types: self.node_types, + network: self.network, + chain_spec: self.chain_spec.clone(), + compute_state_root: self.compute_state_root, + code_store: RwLock::new(HashMap::default()), + reth_rpc_support: self.reth_rpc_support, + } + } + + /// Helper function to enable state root calculation + /// + /// If enabled, the node will compute the state root and updates. + /// When disabled, it will return zero for state root and no updates. + pub const fn with_compute_state_root(mut self, is_enable: bool) -> Self { + self.compute_state_root = is_enable; + self + } + + /// Sets whether to use Reth-specific RPC methods for better performance + /// + /// If enabled, the node will use Reth's RPC methods (`debug_codeByHash` and + /// `eth_getAccountInfo`) to speed up account information retrieval. When disabled, it will + /// use multiple standard RPC calls to get account information. + pub const fn with_reth_rpc_support(mut self, is_enable: bool) -> Self { + self.reth_rpc_support = is_enable; + self + } + + /// Get account information from RPC + fn get_account(&self, address: Address) -> Result, ProviderError> + where + P: Provider + Clone + 'static, + N: Network, + { + let account_info = self.block_on_async(async { + // Get account info in a single RPC call using `eth_getAccountInfo` + if self.reth_rpc_support { + return self + .provider + .get_account_info(address) + .block_id(self.block_id) + .await + .map_err(ProviderError::other); + } + // Get account info in multiple RPC calls + let nonce = self.provider.get_transaction_count(address).block_id(self.block_id); + let balance = self.provider.get_balance(address).block_id(self.block_id); + let code = self.provider.get_code_at(address).block_id(self.block_id); + + let (nonce, balance, code) = tokio::join!(nonce, balance, code,); + + let account_info = AccountInfo { + balance: balance.map_err(ProviderError::other)?, + nonce: nonce.map_err(ProviderError::other)?, + code: code.map_err(ProviderError::other)?, + }; + + let code_hash = account_info.code_hash(); + if code_hash != KECCAK_EMPTY { + // Insert code into the cache + self.code_store + .write() + .insert(code_hash, Bytecode::new_raw(account_info.code.clone())); + } + + Ok(account_info) + })?; + + // Only return account if it exists (has balance, nonce, or code) + if account_info.balance.is_zero() && account_info.nonce == 0 && account_info.code.is_empty() + { + Ok(None) + } else { + let bytecode_hash = + if account_info.code.is_empty() { None } else { Some(account_info.code_hash()) }; + + Ok(Some(Account { + balance: account_info.balance, + nonce: account_info.nonce, + bytecode_hash, + })) + } + } +} + +impl StateProvider for RpcBlockchainStateProvider +where + P: Provider + Clone + 'static, + N: Network, + Node: NodeTypes, +{ + fn storage( + &self, + address: Address, + storage_key: StorageKey, + ) -> Result, ProviderError> { + self.block_on_async(async { + Ok(Some( + self.provider + .get_storage_at(address, storage_key.into()) + .block_id(self.block_id) + .await + .map_err(ProviderError::other)?, + )) + }) + } + + fn account_code(&self, addr: &Address) -> Result, ProviderError> { + self.block_on_async(async { + let code = self + .provider + .get_code_at(*addr) + .block_id(self.block_id) + .await + .map_err(ProviderError::other)?; + + if code.is_empty() { + Ok(None) + } else { + Ok(Some(Bytecode::new_raw(code))) + } + }) + } + + fn account_balance(&self, addr: &Address) -> Result, ProviderError> { + self.get_account(*addr).map(|acc| acc.map(|a| a.balance)) + } + + fn account_nonce(&self, addr: &Address) -> Result, ProviderError> { + self.get_account(*addr).map(|acc| acc.map(|a| a.nonce)) + } +} + +impl BytecodeReader for RpcBlockchainStateProvider +where + P: Provider + Clone + 'static, + N: Network, + Node: NodeTypes, +{ + fn bytecode_by_hash(&self, code_hash: &B256) -> Result, ProviderError> { + if !self.reth_rpc_support { + return Ok(self.code_store.read().get(code_hash).cloned()); + } + + self.block_on_async(async { + // The method `debug_codeByHash` is currently only available on a Reth node + let code = self + .provider + .debug_code_by_hash(*code_hash, None) + .await + .map_err(ProviderError::other)?; + + let Some(code) = code else { + // If the code was not found, return None + return Ok(None); + }; + + Ok(Some(Bytecode::new_raw(code))) + }) + } +} + +impl AccountReader for RpcBlockchainStateProvider +where + P: Provider + Clone + 'static, + N: Network, + Node: NodeTypes, +{ + fn basic_account(&self, address: &Address) -> Result, ProviderError> { + self.get_account(*address) + } +} + +impl StateRootProvider for RpcBlockchainStateProvider +where + P: Provider + Clone + 'static, + N: Network, + Node: NodeTypes, +{ + fn state_root(&self, hashed_state: HashedPostState) -> Result { + self.state_root_from_nodes(TrieInput::from_state(hashed_state)) + } + + fn state_root_from_nodes(&self, _input: TrieInput) -> Result { + warn!("state_root_from_nodes is not implemented and will return zero"); + Ok(B256::ZERO) + } + + fn state_root_with_updates( + &self, + hashed_state: HashedPostState, + ) -> Result<(B256, TrieUpdates), ProviderError> { + if !self.compute_state_root { + return Ok((B256::ZERO, TrieUpdates::default())); + } + + self.block_on_async(async { + self.provider + .raw_request::<(HashedPostState, BlockId), (B256, TrieUpdates)>( + "debug_stateRootWithUpdates".into(), + (hashed_state, self.block_id), + ) + .into_future() + .await + .map_err(ProviderError::other) + }) + } + + fn state_root_from_nodes_with_updates( + &self, + _input: TrieInput, + ) -> Result<(B256, TrieUpdates), ProviderError> { + warn!("state_root_from_nodes_with_updates is not implemented and will return zero"); + Ok((B256::ZERO, TrieUpdates::default())) + } +} + +impl StorageReader for RpcBlockchainStateProvider +where + P: Provider + Clone + 'static, + N: Network, + Node: NodeTypes, +{ + fn plain_state_storages( + &self, + addresses_with_keys: impl IntoIterator)>, + ) -> Result)>, ProviderError> { + let mut results = Vec::new(); + + for (address, keys) in addresses_with_keys { + let mut values = Vec::new(); + for key in keys { + let value = self.storage(address, key)?.unwrap_or_default(); + values.push(reth_primitives::StorageEntry::new(key, value)); + } + results.push((address, values)); + } + + Ok(results) + } + + fn changed_storages_with_range( + &self, + _range: RangeInclusive, + ) -> Result>, ProviderError> { + Ok(BTreeMap::new()) + } + + fn changed_storages_and_blocks_with_range( + &self, + _range: RangeInclusive, + ) -> Result>, ProviderError> { + Ok(BTreeMap::new()) + } +} + +impl reth_storage_api::StorageRootProvider for RpcBlockchainStateProvider +where + P: Provider + Clone + 'static, + N: Network, + Node: NodeTypes, +{ + fn storage_root( + &self, + _address: Address, + _hashed_storage: reth_trie::HashedStorage, + ) -> Result { + // RPC doesn't provide storage root computation + Err(ProviderError::UnsupportedProvider) + } + + fn storage_proof( + &self, + _address: Address, + _slot: B256, + _hashed_storage: reth_trie::HashedStorage, + ) -> Result { + Err(ProviderError::UnsupportedProvider) + } + + fn storage_multiproof( + &self, + _address: Address, + _slots: &[B256], + _hashed_storage: reth_trie::HashedStorage, + ) -> Result { + Err(ProviderError::UnsupportedProvider) + } +} + +impl reth_storage_api::StateProofProvider for RpcBlockchainStateProvider +where + P: Provider + Clone + 'static, + N: Network, + Node: NodeTypes, +{ + fn proof( + &self, + _input: TrieInput, + _address: Address, + _slots: &[B256], + ) -> Result { + Err(ProviderError::UnsupportedProvider) + } + + fn multiproof( + &self, + _input: TrieInput, + _targets: reth_trie::MultiProofTargets, + ) -> Result { + Err(ProviderError::UnsupportedProvider) + } + + fn witness( + &self, + _input: TrieInput, + _target: HashedPostState, + ) -> Result, ProviderError> { + Err(ProviderError::UnsupportedProvider) + } +} + +impl reth_storage_api::HashedPostStateProvider + for RpcBlockchainStateProvider +where + P: Provider + Clone + 'static, + N: Network, + Node: NodeTypes, +{ + fn hashed_post_state(&self, _bundle_state: &revm::database::BundleState) -> HashedPostState { + // Return empty hashed post state for RPC provider + HashedPostState::default() + } +} + +impl StateReader for RpcBlockchainStateProvider +where + P: Provider + Clone + 'static, + N: Network, + Node: NodeTypes, +{ + type Receipt = ReceiptTy; + + fn get_state( + &self, + _block: BlockNumber, + ) -> Result>, ProviderError> { + // RPC doesn't provide execution outcomes + Err(ProviderError::UnsupportedProvider) + } +} + +impl DBProvider for RpcBlockchainStateProvider +where + P: Provider + Clone + 'static, + N: Network, + Node: NodeTypes, +{ + type Tx = TxMock; + + fn tx_ref(&self) -> &Self::Tx { + // We can't use a static here since TxMock doesn't allow direct construction + // This is fine since we're just returning a mock transaction + unimplemented!("tx_ref not supported for RPC provider") + } + + fn tx_mut(&mut self) -> &mut Self::Tx { + unimplemented!("tx_mut not supported for RPC provider") + } + + fn into_tx(self) -> Self::Tx { + TxMock::default() + } + + fn disable_long_read_transaction_safety(self) -> Self { + // No-op for RPC provider + self + } + + fn commit(self) -> ProviderResult { + unimplemented!("commit not supported for RPC provider") + } + + fn prune_modes_ref(&self) -> &reth_prune_types::PruneModes { + unimplemented!("prune modes not supported for RPC provider") + } +} + +impl BlockNumReader for RpcBlockchainStateProvider +where + P: Provider + Clone + 'static, + N: Network, + Node: NodeTypes, +{ + fn chain_info(&self) -> Result { + self.block_on_async(async { + let block = self + .provider + .get_block(self.block_id) + .await + .map_err(ProviderError::other)? + .ok_or(ProviderError::HeaderNotFound(0.into()))?; + + Ok(ChainInfo { best_hash: block.header().hash(), best_number: block.header().number() }) + }) + } + + fn best_block_number(&self) -> Result { + self.block_on_async(async { + self.provider.get_block_number().await.map_err(ProviderError::other) + }) + } + + fn last_block_number(&self) -> Result { + self.best_block_number() + } + + fn block_number(&self, hash: B256) -> Result, ProviderError> { + self.block_on_async(async { + let block = + self.provider.get_block_by_hash(hash).await.map_err(ProviderError::other)?; + + Ok(block.map(|b| b.header().number())) + }) + } +} + +impl BlockHashReader for RpcBlockchainStateProvider +where + P: Provider + Clone + 'static, + N: Network, + Node: NodeTypes, +{ + fn block_hash(&self, number: u64) -> Result, ProviderError> { + self.block_on_async(async { + let block = self + .provider + .get_block_by_number(number.into()) + .await + .map_err(ProviderError::other)?; + + Ok(block.map(|b| b.header().hash())) + }) + } + + fn canonical_hashes_range( + &self, + _start: BlockNumber, + _end: BlockNumber, + ) -> Result, ProviderError> { + Err(ProviderError::UnsupportedProvider) + } +} + +impl BlockIdReader for RpcBlockchainStateProvider +where + P: Provider + Clone + 'static, + N: Network, + Node: NodeTypes, +{ + fn block_number_for_id( + &self, + _block_id: BlockId, + ) -> Result, ProviderError> { + Err(ProviderError::UnsupportedProvider) + } + + fn pending_block_num_hash(&self) -> Result, ProviderError> { + Err(ProviderError::UnsupportedProvider) + } + + fn safe_block_num_hash(&self) -> Result, ProviderError> { + Err(ProviderError::UnsupportedProvider) + } + + fn finalized_block_num_hash(&self) -> Result, ProviderError> { + Err(ProviderError::UnsupportedProvider) + } +} + +impl BlockReader for RpcBlockchainStateProvider +where + P: Provider + Clone + 'static, + N: Network, + Node: NodeTypes, +{ + type Block = BlockTy; + + fn find_block_by_hash( + &self, + _hash: B256, + _source: reth_provider::BlockSource, + ) -> Result, ProviderError> { + Err(ProviderError::UnsupportedProvider) + } + + fn block( + &self, + _id: alloy_rpc_types::BlockHashOrNumber, + ) -> Result, ProviderError> { + Err(ProviderError::UnsupportedProvider) + } + + fn pending_block(&self) -> Result>, ProviderError> { + Err(ProviderError::UnsupportedProvider) + } + + fn pending_block_and_receipts( + &self, + ) -> Result, Vec)>, ProviderError> { + Err(ProviderError::UnsupportedProvider) + } + + fn recovered_block( + &self, + _id: alloy_rpc_types::BlockHashOrNumber, + _transaction_kind: TransactionVariant, + ) -> Result>, ProviderError> { + Err(ProviderError::UnsupportedProvider) + } + + fn sealed_block_with_senders( + &self, + _id: alloy_rpc_types::BlockHashOrNumber, + _transaction_kind: TransactionVariant, + ) -> Result>>, ProviderError> { + Err(ProviderError::UnsupportedProvider) + } + + fn block_range( + &self, + _range: RangeInclusive, + ) -> Result, ProviderError> { + Err(ProviderError::UnsupportedProvider) + } + + fn block_with_senders_range( + &self, + _range: RangeInclusive, + ) -> Result>>, ProviderError> { + Err(ProviderError::UnsupportedProvider) + } + + fn recovered_block_range( + &self, + _range: RangeInclusive, + ) -> Result>, ProviderError> { + Err(ProviderError::UnsupportedProvider) + } + + fn block_by_transaction_id(&self, _id: TxNumber) -> ProviderResult> { + Err(ProviderError::UnsupportedProvider) + } +} + +impl TransactionsProvider for RpcBlockchainStateProvider +where + P: Provider + Clone + 'static, + N: Network, + Node: NodeTypes, +{ + type Transaction = TxTy; + + fn transaction_id(&self, _tx_hash: B256) -> Result, ProviderError> { + Err(ProviderError::UnsupportedProvider) + } + + fn transaction_by_id(&self, _id: TxNumber) -> Result, ProviderError> { + Err(ProviderError::UnsupportedProvider) + } + + fn transaction_by_id_unhashed( + &self, + _id: TxNumber, + ) -> Result, ProviderError> { + Err(ProviderError::UnsupportedProvider) + } + + fn transaction_by_hash(&self, _hash: B256) -> Result, ProviderError> { + Err(ProviderError::UnsupportedProvider) + } + + fn transaction_by_hash_with_meta( + &self, + _hash: B256, + ) -> Result, ProviderError> { + Err(ProviderError::UnsupportedProvider) + } + + fn transaction_block(&self, _id: TxNumber) -> Result, ProviderError> { + Err(ProviderError::UnsupportedProvider) + } + + fn transactions_by_block( + &self, + _block: alloy_rpc_types::BlockHashOrNumber, + ) -> Result>, ProviderError> { + Err(ProviderError::UnsupportedProvider) + } + + fn transactions_by_block_range( + &self, + _range: impl RangeBounds, + ) -> Result>, ProviderError> { + Err(ProviderError::UnsupportedProvider) + } + + fn transactions_by_tx_range( + &self, + _range: impl RangeBounds, + ) -> Result, ProviderError> { + Err(ProviderError::UnsupportedProvider) + } + + fn senders_by_tx_range( + &self, + _range: impl RangeBounds, + ) -> Result, ProviderError> { + Err(ProviderError::UnsupportedProvider) + } + + fn transaction_sender(&self, _id: TxNumber) -> Result, ProviderError> { + Err(ProviderError::UnsupportedProvider) + } +} + +impl ReceiptProvider for RpcBlockchainStateProvider +where + P: Provider + Clone + 'static, + N: Network, + Node: NodeTypes, +{ + type Receipt = ReceiptTy; + + fn receipt(&self, _id: TxNumber) -> Result, ProviderError> { + Err(ProviderError::UnsupportedProvider) + } + + fn receipt_by_hash(&self, _hash: B256) -> Result, ProviderError> { + Err(ProviderError::UnsupportedProvider) + } + + fn receipts_by_block( + &self, + _block: alloy_rpc_types::BlockHashOrNumber, + ) -> Result>, ProviderError> { + Err(ProviderError::UnsupportedProvider) + } + + fn receipts_by_tx_range( + &self, + _range: impl RangeBounds, + ) -> Result, ProviderError> { + Err(ProviderError::UnsupportedProvider) + } + + fn receipts_by_block_range( + &self, + _range: RangeInclusive, + ) -> Result>, ProviderError> { + Err(ProviderError::UnsupportedProvider) + } +} + +impl HeaderProvider for RpcBlockchainStateProvider +where + P: Provider + Clone + 'static, + N: Network, + Node: NodeTypes, +{ + type Header = HeaderTy; + + fn header(&self, _block_hash: BlockHash) -> Result, ProviderError> { + Err(ProviderError::UnsupportedProvider) + } + + fn header_by_number(&self, _num: BlockNumber) -> Result, ProviderError> { + Err(ProviderError::UnsupportedProvider) + } + + fn headers_range( + &self, + _range: impl RangeBounds, + ) -> Result, ProviderError> { + Err(ProviderError::UnsupportedProvider) + } + + fn sealed_header( + &self, + _number: BlockNumber, + ) -> Result>>, ProviderError> { + Err(ProviderError::UnsupportedProvider) + } + + fn sealed_headers_range( + &self, + _range: impl RangeBounds, + ) -> Result>>, ProviderError> { + Err(ProviderError::UnsupportedProvider) + } + + fn sealed_headers_while( + &self, + _range: impl RangeBounds, + _predicate: impl FnMut(&SealedHeader>) -> bool, + ) -> Result>>, ProviderError> { + Err(ProviderError::UnsupportedProvider) + } +} + +impl PruneCheckpointReader for RpcBlockchainStateProvider +where + P: Provider + Clone + 'static, + N: Network, + Node: NodeTypes, +{ + fn get_prune_checkpoint( + &self, + _segment: PruneSegment, + ) -> Result, ProviderError> { + Err(ProviderError::UnsupportedProvider) + } + + fn get_prune_checkpoints(&self) -> Result, ProviderError> { + Err(ProviderError::UnsupportedProvider) + } +} + +impl StageCheckpointReader for RpcBlockchainStateProvider +where + P: Provider + Clone + 'static, + N: Network, + Node: NodeTypes, +{ + fn get_stage_checkpoint(&self, _id: StageId) -> Result, ProviderError> { + Err(ProviderError::UnsupportedProvider) + } + + fn get_stage_checkpoint_progress( + &self, + _id: StageId, + ) -> Result>, ProviderError> { + Err(ProviderError::UnsupportedProvider) + } + + fn get_all_checkpoints(&self) -> Result, ProviderError> { + Err(ProviderError::UnsupportedProvider) + } +} + +impl ChangeSetReader for RpcBlockchainStateProvider +where + P: Provider + Clone + 'static, + N: Network, + Node: NodeTypes, +{ + fn account_block_changeset( + &self, + _block_number: BlockNumber, + ) -> Result, ProviderError> { + Err(ProviderError::UnsupportedProvider) + } + + fn get_account_before_block( + &self, + _block_number: BlockNumber, + _address: Address, + ) -> ProviderResult> { + Err(ProviderError::UnsupportedProvider) + } +} + +impl StateProviderFactory for RpcBlockchainStateProvider +where + P: Provider + Clone + 'static + Send + Sync, + Node: NodeTypes + 'static, + Node::ChainSpec: Send + Sync, + N: Network, + Self: Clone + 'static, +{ + fn latest(&self) -> Result { + Ok(Box::new(self.with_block_id(self.best_block_number()?.into()))) + } + + fn state_by_block_id(&self, block_id: BlockId) -> Result { + Ok(Box::new(self.with_block_id(block_id))) + } + + fn state_by_block_number_or_tag( + &self, + number_or_tag: alloy_rpc_types::BlockNumberOrTag, + ) -> Result { + match number_or_tag { + alloy_rpc_types::BlockNumberOrTag::Latest => self.latest(), + alloy_rpc_types::BlockNumberOrTag::Pending => self.pending(), + alloy_rpc_types::BlockNumberOrTag::Number(num) => self.history_by_block_number(num), + _ => Err(ProviderError::UnsupportedProvider), + } + } + + fn history_by_block_number( + &self, + block_number: BlockNumber, + ) -> Result { + Ok(Box::new(Self::new( + self.provider.clone(), + BlockId::number(block_number), + self.node_types, + ))) + } + + fn history_by_block_hash( + &self, + block_hash: BlockHash, + ) -> Result { + Ok(Box::new(self.with_block_id(BlockId::hash(block_hash)))) + } + + fn state_by_block_hash( + &self, + block_hash: BlockHash, + ) -> Result { + self.history_by_block_hash(block_hash) + } + + fn pending(&self) -> Result { + Ok(Box::new(self.clone())) + } + + fn pending_state_by_hash( + &self, + _block_hash: B256, + ) -> Result, ProviderError> { + // RPC provider doesn't support pending state by hash + Err(ProviderError::UnsupportedProvider) + } + + fn maybe_pending(&self) -> ProviderResult> { + Ok(None) + } +} + +impl ChainSpecProvider for RpcBlockchainStateProvider +where + P: Send + Sync + std::fmt::Debug, + N: Send + Sync, + Node: NodeTypes, + Node::ChainSpec: Default, +{ + type ChainSpec = Node::ChainSpec; + + fn chain_spec(&self) -> Arc { + if let Some(chain_spec) = &self.chain_spec { + chain_spec.clone() + } else { + // Fallback for when chain_spec is not provided + Arc::new(Node::ChainSpec::default()) + } + } +} + +// Note: FullExecutionDataProvider is already implemented via the blanket implementation +// for types that implement both ExecutionDataProvider and BlockExecutionForkProvider + +impl StatsReader for RpcBlockchainStateProvider +where + P: Provider + Clone + 'static, + N: Network, + Node: NodeTypes, +{ + fn count_entries(&self) -> Result { + Ok(0) + } +} + +impl BlockBodyIndicesProvider for RpcBlockchainStateProvider +where + P: Provider + Clone + 'static, + N: Network, + Node: NodeTypes, +{ + fn block_body_indices( + &self, + _num: u64, + ) -> Result, ProviderError> { + Err(ProviderError::UnsupportedProvider) + } + + fn block_body_indices_range( + &self, + _range: RangeInclusive, + ) -> Result, ProviderError> { + Err(ProviderError::UnsupportedProvider) + } +} + +impl NodePrimitivesProvider for RpcBlockchainStateProvider +where + P: Send + Sync + std::fmt::Debug, + N: Send + Sync, + Node: NodeTypes, +{ + type Primitives = PrimitivesTy; +} + +impl ChainStateBlockReader for RpcBlockchainStateProvider +where + P: Provider + Clone + 'static, + N: Network, + Node: NodeTypes, +{ + fn last_finalized_block_number(&self) -> Result, ProviderError> { + Err(ProviderError::UnsupportedProvider) + } + + fn last_safe_block_number(&self) -> Result, ProviderError> { + Err(ProviderError::UnsupportedProvider) + } +} + +impl ChainStateBlockWriter for RpcBlockchainStateProvider +where + P: Provider + Clone + 'static, + N: Network, + Node: NodeTypes, +{ + fn save_finalized_block_number(&self, _block_number: BlockNumber) -> Result<(), ProviderError> { + Err(ProviderError::UnsupportedProvider) + } + + fn save_safe_block_number(&self, _block_number: BlockNumber) -> Result<(), ProviderError> { + Err(ProviderError::UnsupportedProvider) + } +} diff --git a/crates/storage/storage-api/Cargo.toml b/crates/storage/storage-api/Cargo.toml index 1c43004216a..a62193a5dd8 100644 --- a/crates/storage/storage-api/Cargo.toml +++ b/crates/storage/storage-api/Cargo.toml @@ -22,7 +22,6 @@ reth-prune-types.workspace = true reth-stages-types.workspace = true reth-storage-errors.workspace = true reth-trie-common.workspace = true -reth-trie-db = { workspace = true, optional = true } revm-database.workspace = true reth-ethereum-primitives.workspace = true @@ -55,5 +54,28 @@ std = [ db-api = [ "dep:reth-db-api", - "dep:reth-trie-db", +] + +serde = [ + "reth-ethereum-primitives/serde", + "reth-db-models/serde", + "reth-execution-types/serde", + "reth-primitives-traits/serde", + "reth-prune-types/serde", + "reth-stages-types/serde", + "reth-trie-common/serde", + "revm-database/serde", + "alloy-eips/serde", + "alloy-primitives/serde", + "alloy-consensus/serde", + "alloy-rpc-types-engine/serde", +] + +serde-bincode-compat = [ + "reth-execution-types/serde-bincode-compat", + "reth-primitives-traits/serde-bincode-compat", + "reth-trie-common/serde-bincode-compat", + "reth-ethereum-primitives/serde-bincode-compat", + "alloy-eips/serde-bincode-compat", + "alloy-consensus/serde-bincode-compat", ] diff --git a/crates/storage/storage-api/src/account.rs b/crates/storage/storage-api/src/account.rs index 1bf4b783c2e..270bfd1226c 100644 --- a/crates/storage/storage-api/src/account.rs +++ b/crates/storage/storage-api/src/account.rs @@ -46,7 +46,7 @@ pub trait AccountExtReader { ) -> ProviderResult>>; } -/// AccountChange reader +/// `AccountChange` reader #[auto_impl(&, Arc, Box)] pub trait ChangeSetReader { /// Iterate over account changesets and return the account state from before this block. @@ -54,4 +54,13 @@ pub trait ChangeSetReader { &self, block_number: BlockNumber, ) -> ProviderResult>; + + /// Search the block's changesets for the given address, and return the result. + /// + /// Returns `None` if the account was not changed in this block. + fn get_account_before_block( + &self, + block_number: BlockNumber, + address: Address, + ) -> ProviderResult>; } diff --git a/crates/storage/storage-api/src/block.rs b/crates/storage/storage-api/src/block.rs index ce488aba887..b9ab206a6b8 100644 --- a/crates/storage/storage-api/src/block.rs +++ b/crates/storage/storage-api/src/block.rs @@ -1,12 +1,12 @@ use crate::{ - BlockBodyIndicesProvider, BlockNumReader, HeaderProvider, OmmersProvider, ReceiptProvider, - ReceiptProviderIdExt, TransactionVariant, TransactionsProvider, WithdrawalsProvider, + BlockBodyIndicesProvider, BlockNumReader, HeaderProvider, ReceiptProvider, + ReceiptProviderIdExt, TransactionVariant, TransactionsProvider, }; use alloc::{sync::Arc, vec::Vec}; use alloy_eips::{BlockHashOrNumber, BlockId, BlockNumberOrTag}; -use alloy_primitives::{BlockNumber, B256}; +use alloy_primitives::{BlockNumber, TxNumber, B256}; use core::ops::RangeInclusive; -use reth_primitives_traits::{RecoveredBlock, SealedBlock, SealedHeader}; +use reth_primitives_traits::{RecoveredBlock, SealedHeader}; use reth_storage_errors::provider::ProviderResult; /// A helper enum that represents the origin of the requested block. @@ -53,8 +53,6 @@ pub trait BlockReader: + BlockBodyIndicesProvider + TransactionsProvider + ReceiptProvider - + WithdrawalsProvider - + OmmersProvider + Send + Sync { @@ -80,23 +78,17 @@ pub trait BlockReader: /// Returns `None` if block is not found. fn block(&self, id: BlockHashOrNumber) -> ProviderResult>; - /// Returns the pending block if available - /// - /// Note: This returns a [`SealedBlock`] because it's expected that this is sealed by the - /// provider and the caller does not know the hash. - fn pending_block(&self) -> ProviderResult>>; - /// Returns the pending block if available /// /// Note: This returns a [`RecoveredBlock`] because it's expected that this is sealed by /// the provider and the caller does not know the hash. - fn pending_block_with_senders(&self) -> ProviderResult>>; + fn pending_block(&self) -> ProviderResult>>; /// Returns the pending block and receipts if available. #[expect(clippy::type_complexity)] fn pending_block_and_receipts( &self, - ) -> ProviderResult, Vec)>>; + ) -> ProviderResult, Vec)>>; /// Returns the block with matching hash from the database. /// @@ -152,6 +144,9 @@ pub trait BlockReader: &self, range: RangeInclusive, ) -> ProviderResult>>; + + /// Returns the block number that contains the given transaction. + fn block_by_transaction_id(&self, id: TxNumber) -> ProviderResult>; } impl BlockReader for Arc { @@ -167,15 +162,12 @@ impl BlockReader for Arc { fn block(&self, id: BlockHashOrNumber) -> ProviderResult> { T::block(self, id) } - fn pending_block(&self) -> ProviderResult>> { + fn pending_block(&self) -> ProviderResult>> { T::pending_block(self) } - fn pending_block_with_senders(&self) -> ProviderResult>> { - T::pending_block_with_senders(self) - } fn pending_block_and_receipts( &self, - ) -> ProviderResult, Vec)>> { + ) -> ProviderResult, Vec)>> { T::pending_block_and_receipts(self) } fn block_by_hash(&self, hash: B256) -> ProviderResult> { @@ -213,6 +205,9 @@ impl BlockReader for Arc { ) -> ProviderResult>> { T::recovered_block_range(self, range) } + fn block_by_transaction_id(&self, id: TxNumber) -> ProviderResult> { + T::block_by_transaction_id(self, id) + } } impl BlockReader for &T { @@ -228,15 +223,12 @@ impl BlockReader for &T { fn block(&self, id: BlockHashOrNumber) -> ProviderResult> { T::block(self, id) } - fn pending_block(&self) -> ProviderResult>> { + fn pending_block(&self) -> ProviderResult>> { T::pending_block(self) } - fn pending_block_with_senders(&self) -> ProviderResult>> { - T::pending_block_with_senders(self) - } fn pending_block_and_receipts( &self, - ) -> ProviderResult, Vec)>> { + ) -> ProviderResult, Vec)>> { T::pending_block_and_receipts(self) } fn block_by_hash(&self, hash: B256) -> ProviderResult> { @@ -274,6 +266,9 @@ impl BlockReader for &T { ) -> ProviderResult>> { T::recovered_block_range(self, range) } + fn block_by_transaction_id(&self, id: TxNumber) -> ProviderResult> { + T::block_by_transaction_id(self, id) + } } /// Trait extension for `BlockReader`, for types that implement `BlockId` conversion. @@ -384,19 +379,6 @@ pub trait BlockReaderIdExt: BlockReader + ReceiptProviderIdExt { /// /// Returns `None` if header is not found. fn header_by_id(&self, id: BlockId) -> ProviderResult>; - - /// Returns the ommers with the matching tag from the database. - fn ommers_by_number_or_tag( - &self, - id: BlockNumberOrTag, - ) -> ProviderResult>> { - self.convert_block_number(id)?.map_or_else(|| Ok(None), |num| self.ommers(num.into())) - } - - /// Returns the ommers with the matching `BlockId` from the database. - /// - /// Returns `None` if block is not found. - fn ommers_by_id(&self, id: BlockId) -> ProviderResult>>; } /// Functionality to read the last known chain blocks from the database. diff --git a/crates/storage/storage-api/src/block_id.rs b/crates/storage/storage-api/src/block_id.rs index 00856d348a5..e00ad950e2d 100644 --- a/crates/storage/storage-api/src/block_id.rs +++ b/crates/storage/storage-api/src/block_id.rs @@ -7,7 +7,7 @@ use reth_storage_errors::provider::{ProviderError, ProviderResult}; /// Client trait for getting important block numbers (such as the latest block number), converting /// block hashes to numbers, and fetching a block hash from its block number. /// -/// This trait also supports fetching block hashes and block numbers from a [BlockHashOrNumber]. +/// This trait also supports fetching block hashes and block numbers from a [`BlockHashOrNumber`]. #[auto_impl::auto_impl(&, Arc)] pub trait BlockNumReader: BlockHashReader + Send + Sync { /// Returns the current info for the chain. @@ -19,6 +19,11 @@ pub trait BlockNumReader: BlockHashReader + Send + Sync { /// Returns the last block number associated with the last canonical header in the database. fn last_block_number(&self) -> ProviderResult; + /// Returns earliest block number to keep track of the expired block range. + fn earliest_block_number(&self) -> ProviderResult { + Ok(0) + } + /// Gets the `BlockNumber` for the given hash. Returns `None` if no block with this hash exists. fn block_number(&self, hash: B256) -> ProviderResult>; @@ -48,7 +53,7 @@ pub trait BlockNumReader: BlockHashReader + Send + Sync { /// are provided if the type implements the `pending_block_num_hash`, `finalized_block_num`, and /// `safe_block_num` methods. /// -/// The resulting block numbers can be converted to hashes using the underlying [BlockNumReader] +/// The resulting block numbers can be converted to hashes using the underlying [`BlockNumReader`] /// methods, and vice versa. #[auto_impl::auto_impl(&, Arc)] pub trait BlockIdReader: BlockNumReader + Send + Sync { @@ -56,7 +61,7 @@ pub trait BlockIdReader: BlockNumReader + Send + Sync { fn convert_block_number(&self, num: BlockNumberOrTag) -> ProviderResult> { let num = match num { BlockNumberOrTag::Latest => self.best_block_number()?, - BlockNumberOrTag::Earliest => 0, + BlockNumberOrTag::Earliest => self.earliest_block_number()?, BlockNumberOrTag::Pending => { return self .pending_block_num_hash() @@ -84,7 +89,7 @@ pub trait BlockIdReader: BlockNumReader + Send + Sync { .map(|res_opt| res_opt.map(|num_hash| num_hash.hash)), BlockNumberOrTag::Finalized => self.finalized_block_hash(), BlockNumberOrTag::Safe => self.safe_block_hash(), - BlockNumberOrTag::Earliest => self.block_hash(0), + BlockNumberOrTag::Earliest => self.block_hash(self.earliest_block_number()?), BlockNumberOrTag::Number(num) => self.block_hash(num), }, } diff --git a/crates/storage/storage-api/src/block_writer.rs b/crates/storage/storage-api/src/block_writer.rs index 3ec5e5fb57f..3bbde88d3ed 100644 --- a/crates/storage/storage-api/src/block_writer.rs +++ b/crates/storage/storage-api/src/block_writer.rs @@ -1,11 +1,11 @@ -use crate::{NodePrimitivesProvider, StorageLocation}; +use crate::NodePrimitivesProvider; use alloc::vec::Vec; use alloy_primitives::BlockNumber; use reth_db_models::StoredBlockBodyIndices; use reth_execution_types::{Chain, ExecutionOutcome}; use reth_primitives_traits::{Block, NodePrimitives, RecoveredBlock}; use reth_storage_errors::provider::ProviderResult; -use reth_trie_common::{updates::TrieUpdates, HashedPostStateSorted}; +use reth_trie_common::HashedPostStateSorted; /// `BlockExecution` Writer pub trait BlockExecutionWriter: @@ -14,43 +14,27 @@ pub trait BlockExecutionWriter: /// Take all of the blocks above the provided number and their execution result /// /// The passed block number will stay in the database. - /// - /// Accepts [`StorageLocation`] specifying from where should transactions and receipts be - /// removed. fn take_block_and_execution_above( &self, block: BlockNumber, - remove_from: StorageLocation, ) -> ProviderResult>; /// Remove all of the blocks above the provided number and their execution result /// /// The passed block number will stay in the database. - /// - /// Accepts [`StorageLocation`] specifying from where should transactions and receipts be - /// removed. - fn remove_block_and_execution_above( - &self, - block: BlockNumber, - remove_from: StorageLocation, - ) -> ProviderResult<()>; + fn remove_block_and_execution_above(&self, block: BlockNumber) -> ProviderResult<()>; } impl BlockExecutionWriter for &T { fn take_block_and_execution_above( &self, block: BlockNumber, - remove_from: StorageLocation, ) -> ProviderResult> { - (*self).take_block_and_execution_above(block, remove_from) + (*self).take_block_and_execution_above(block) } - fn remove_block_and_execution_above( - &self, - block: BlockNumber, - remove_from: StorageLocation, - ) -> ProviderResult<()> { - (*self).remove_block_and_execution_above(block, remove_from) + fn remove_block_and_execution_above(&self, block: BlockNumber) -> ProviderResult<()> { + (*self).remove_block_and_execution_above(block) } } @@ -65,15 +49,11 @@ pub trait BlockWriter: Send + Sync { /// Insert full block and make it canonical. Parent tx num and transition id is taken from /// parent block in database. /// - /// Return [StoredBlockBodyIndices] that contains indices of the first and last transactions and - /// transition in the block. - /// - /// Accepts [`StorageLocation`] value which specifies where transactions and headers should be - /// written. + /// Return [`StoredBlockBodyIndices`] that contains indices of the first and last transactions + /// and transition in the block. fn insert_block( &self, block: RecoveredBlock, - write_to: StorageLocation, ) -> ProviderResult; /// Appends a batch of block bodies extending the canonical chain. This is invoked during @@ -84,30 +64,21 @@ pub trait BlockWriter: Send + Sync { fn append_block_bodies( &self, bodies: Vec<(BlockNumber, Option<::Body>)>, - write_to: StorageLocation, ) -> ProviderResult<()>; /// Removes all blocks above the given block number from the database. /// /// Note: This does not remove state or execution data. - fn remove_blocks_above( - &self, - block: BlockNumber, - remove_from: StorageLocation, - ) -> ProviderResult<()>; + fn remove_blocks_above(&self, block: BlockNumber) -> ProviderResult<()>; /// Removes all block bodies above the given block number from the database. - fn remove_bodies_above( - &self, - block: BlockNumber, - remove_from: StorageLocation, - ) -> ProviderResult<()>; + fn remove_bodies_above(&self, block: BlockNumber) -> ProviderResult<()>; /// Appends a batch of sealed blocks to the blockchain, including sender information, and /// updates the post-state. /// /// Inserts the blocks into the database and updates the state with - /// provided `BundleState`. + /// provided `BundleState`. The database's trie state is _not_ updated. /// /// # Parameters /// @@ -122,6 +93,5 @@ pub trait BlockWriter: Send + Sync { blocks: Vec>, execution_outcome: &ExecutionOutcome, hashed_state: HashedPostStateSorted, - trie_updates: TrieUpdates, ) -> ProviderResult<()>; } diff --git a/crates/storage/storage-api/src/chain.rs b/crates/storage/storage-api/src/chain.rs index c5f199fed7f..5b159715ad2 100644 --- a/crates/storage/storage-api/src/chain.rs +++ b/crates/storage/storage-api/src/chain.rs @@ -1,4 +1,4 @@ -use crate::{DBProvider, OmmersProvider, StorageLocation}; +use crate::DBProvider; use alloc::vec::Vec; use alloy_consensus::Header; use alloy_primitives::BlockNumber; @@ -14,7 +14,7 @@ use reth_db_api::{ use reth_db_models::StoredBlockWithdrawals; use reth_ethereum_primitives::TransactionSigned; use reth_primitives_traits::{ - Block, BlockBody, FullBlockHeader, FullNodePrimitives, SignedTransaction, + Block, BlockBody, FullBlockHeader, NodePrimitives, SignedTransaction, }; use reth_storage_errors::provider::ProviderResult; @@ -29,7 +29,6 @@ pub trait BlockBodyWriter { &self, provider: &Provider, bodies: Vec<(BlockNumber, Option)>, - write_to: StorageLocation, ) -> ProviderResult<()>; /// Removes all block bodies above the given block number from the database. @@ -37,16 +36,15 @@ pub trait BlockBodyWriter { &self, provider: &Provider, block: BlockNumber, - remove_from: StorageLocation, ) -> ProviderResult<()>; } /// Trait that implements how chain-specific types are written to the storage. -pub trait ChainStorageWriter: +pub trait ChainStorageWriter: BlockBodyWriter::Body> { } -impl ChainStorageWriter for T where +impl ChainStorageWriter for T where T: BlockBodyWriter::Body> { } @@ -75,11 +73,11 @@ pub trait BlockBodyReader { } /// Trait that implements how chain-specific types are read from storage. -pub trait ChainStorageReader: +pub trait ChainStorageReader: BlockBodyReader { } -impl ChainStorageReader for T where +impl ChainStorageReader for T where T: BlockBodyReader { } @@ -105,7 +103,6 @@ where &self, provider: &Provider, bodies: Vec<(u64, Option>)>, - _write_to: StorageLocation, ) -> ProviderResult<()> { let mut ommers_cursor = provider.tx_ref().cursor_write::>()?; let mut withdrawals_cursor = @@ -120,11 +117,10 @@ where } // Write withdrawals if any - if let Some(withdrawals) = body.withdrawals { - if !withdrawals.is_empty() { - withdrawals_cursor - .append(block_number, &StoredBlockWithdrawals { withdrawals })?; - } + if let Some(withdrawals) = body.withdrawals && + !withdrawals.is_empty() + { + withdrawals_cursor.append(block_number, &StoredBlockWithdrawals { withdrawals })?; } } @@ -135,10 +131,9 @@ where &self, provider: &Provider, block: BlockNumber, - _remove_from: StorageLocation, ) -> ProviderResult<()> { provider.tx_ref().unwind_table_by_num::(block)?; - provider.tx_ref().unwind_table_by_num::(block)?; + provider.tx_ref().unwind_table_by_num::>(block)?; Ok(()) } @@ -146,8 +141,7 @@ where impl BlockBodyReader for EthStorage where - Provider: - DBProvider + ChainSpecProvider + OmmersProvider

, + Provider: DBProvider + ChainSpecProvider, T: SignedTransaction, H: FullBlockHeader, { @@ -180,12 +174,86 @@ where let ommers = if chain_spec.is_paris_active_at_block(header.number()) { Vec::new() } else { - provider.ommers(header.number().into())?.unwrap_or_default() + // Pre-merge: fetch ommers from database using direct database access + provider + .tx_ref() + .cursor_read::>()? + .seek_exact(header.number())? + .map(|(_, stored_ommers)| stored_ommers.ommers) + .unwrap_or_default() }; - bodies.push(alloy_consensus::BlockBody { transactions, ommers, withdrawals }); } Ok(bodies) } } + +/// A noop storage for chains that don’t have custom body storage. +/// +/// This will never read nor write additional body content such as withdrawals or ommers. +/// But will respect the optionality of withdrawals if activated and fill them if the corresponding +/// hardfork is activated. +#[derive(Debug, Clone, Copy)] +pub struct EmptyBodyStorage(PhantomData<(T, H)>); + +impl Default for EmptyBodyStorage { + fn default() -> Self { + Self(PhantomData) + } +} + +impl BlockBodyWriter> + for EmptyBodyStorage +where + T: SignedTransaction, + H: FullBlockHeader, +{ + fn write_block_bodies( + &self, + _provider: &Provider, + _bodies: Vec<(u64, Option>)>, + ) -> ProviderResult<()> { + // noop + Ok(()) + } + + fn remove_block_bodies_above( + &self, + _provider: &Provider, + _block: BlockNumber, + ) -> ProviderResult<()> { + // noop + Ok(()) + } +} + +impl BlockBodyReader for EmptyBodyStorage +where + Provider: ChainSpecProvider, + T: SignedTransaction, + H: FullBlockHeader, +{ + type Block = alloy_consensus::Block; + + fn read_block_bodies( + &self, + provider: &Provider, + inputs: Vec>, + ) -> ProviderResult::Body>> { + let chain_spec = provider.chain_spec(); + + Ok(inputs + .into_iter() + .map(|(header, transactions)| { + alloy_consensus::BlockBody { + transactions, + ommers: vec![], // Empty storage never has ommers + withdrawals: chain_spec + .is_shanghai_active_at_timestamp(header.timestamp()) + .then(Default::default), + } + }) + .collect()) + } +} diff --git a/crates/storage/storage-api/src/database_provider.rs b/crates/storage/storage-api/src/database_provider.rs index 0d736c00e15..8b5d8281f42 100644 --- a/crates/storage/storage-api/src/database_provider.rs +++ b/crates/storage/storage-api/src/database_provider.rs @@ -37,9 +37,7 @@ pub trait DBProvider: Sized { } /// Commit database transaction - fn commit(self) -> ProviderResult { - Ok(self.into_tx().commit()?) - } + fn commit(self) -> ProviderResult; /// Returns a reference to prune modes. fn prune_modes_ref(&self) -> &PruneModes; @@ -162,6 +160,29 @@ pub trait DatabaseProviderFactory: Send + Sync { /// Helper type alias to get the associated transaction type from a [`DatabaseProviderFactory`]. pub type FactoryTx = <::DB as Database>::TX; +/// A trait which can be used to describe any factory-like type which returns a read-only provider. +pub trait DatabaseProviderROFactory { + /// Provider type returned by this factory. + /// + /// This type is intentionally left unconstrained; constraints can be added as-needed when this + /// is used. + type Provider; + + /// Creates and returns a Provider. + fn database_provider_ro(&self) -> ProviderResult; +} + +impl DatabaseProviderROFactory for T +where + T: DatabaseProviderFactory, +{ + type Provider = T::Provider; + + fn database_provider_ro(&self) -> ProviderResult { + ::database_provider_ro(self) + } +} + fn range_size_hint(range: &impl RangeBounds) -> Option { let start = match range.start_bound().cloned() { Bound::Included(start) => start, diff --git a/crates/storage/storage-api/src/hashing.rs b/crates/storage/storage-api/src/hashing.rs index 38964a244cd..dfbb00ab8f9 100644 --- a/crates/storage/storage-api/src/hashing.rs +++ b/crates/storage/storage-api/src/hashing.rs @@ -1,7 +1,7 @@ use alloc::collections::{BTreeMap, BTreeSet}; use alloy_primitives::{map::HashMap, Address, BlockNumber, B256}; use auto_impl::auto_impl; -use core::ops::{RangeBounds, RangeInclusive}; +use core::ops::RangeBounds; use reth_db_api::models::BlockNumberAddress; use reth_db_models::AccountBeforeTx; use reth_primitives_traits::{Account, StorageEntry}; @@ -69,17 +69,4 @@ pub trait HashingWriter: Send + Sync { &self, storages: impl IntoIterator)>, ) -> ProviderResult>>; - - /// Calculate the hashes of all changed accounts and storages, and finally calculate the state - /// root. - /// - /// The hashes are calculated from `fork_block_number + 1` to `current_block_number`. - /// - /// The resulting state root is compared with `expected_state_root`. - fn insert_hashes( - &self, - range: RangeInclusive, - end_block_hash: B256, - expected_state_root: B256, - ) -> ProviderResult<()>; } diff --git a/crates/storage/storage-api/src/header.rs b/crates/storage/storage-api/src/header.rs index a4c9b215f82..39b2eef9031 100644 --- a/crates/storage/storage-api/src/header.rs +++ b/crates/storage/storage-api/src/header.rs @@ -1,6 +1,6 @@ use alloc::vec::Vec; use alloy_eips::BlockHashOrNumber; -use alloy_primitives::{BlockHash, BlockNumber, U256}; +use alloy_primitives::{BlockHash, BlockNumber}; use core::ops::RangeBounds; use reth_primitives_traits::{BlockHeader, SealedHeader}; use reth_storage_errors::provider::ProviderResult; @@ -15,19 +15,19 @@ pub trait HeaderProvider: Send + Sync { type Header: BlockHeader; /// Check if block is known - fn is_known(&self, block_hash: &BlockHash) -> ProviderResult { + fn is_known(&self, block_hash: BlockHash) -> ProviderResult { self.header(block_hash).map(|header| header.is_some()) } /// Get header by block hash - fn header(&self, block_hash: &BlockHash) -> ProviderResult>; + fn header(&self, block_hash: BlockHash) -> ProviderResult>; /// Retrieves the header sealed by the given block hash. fn sealed_header_by_hash( &self, block_hash: BlockHash, ) -> ProviderResult>> { - Ok(self.header(&block_hash)?.map(|header| SealedHeader::new(header, block_hash))) + Ok(self.header(block_hash)?.map(|header| SealedHeader::new(header, block_hash))) } /// Get header by block number @@ -39,17 +39,11 @@ pub trait HeaderProvider: Send + Sync { hash_or_num: BlockHashOrNumber, ) -> ProviderResult> { match hash_or_num { - BlockHashOrNumber::Hash(hash) => self.header(&hash), + BlockHashOrNumber::Hash(hash) => self.header(hash), BlockHashOrNumber::Number(num) => self.header_by_number(num), } } - /// Get total difficulty by block hash. - fn header_td(&self, hash: &BlockHash) -> ProviderResult>; - - /// Get total difficulty by block number. - fn header_td_by_number(&self, number: BlockNumber) -> ProviderResult>; - /// Get headers in range of block numbers fn headers_range( &self, diff --git a/crates/storage/storage-api/src/history.rs b/crates/storage/storage-api/src/history.rs index 0287f1e640d..e15b791f0fd 100644 --- a/crates/storage/storage-api/src/history.rs +++ b/crates/storage/storage-api/src/history.rs @@ -25,7 +25,7 @@ pub trait HistoryWriter: Send + Sync { range: impl RangeBounds, ) -> ProviderResult; - /// Insert account change index to database. Used inside AccountHistoryIndex stage + /// Insert account change index to database. Used inside `AccountHistoryIndex` stage fn insert_account_history_index( &self, index_updates: impl IntoIterator)>, @@ -47,7 +47,7 @@ pub trait HistoryWriter: Send + Sync { range: impl RangeBounds, ) -> ProviderResult; - /// Insert storage change index to database. Used inside StorageHistoryIndex stage + /// Insert storage change index to database. Used inside `StorageHistoryIndex` stage fn insert_storage_history_index( &self, storage_transitions: impl IntoIterator)>, diff --git a/crates/storage/storage-api/src/legacy.rs b/crates/storage/storage-api/src/legacy.rs deleted file mode 100644 index bb6a21e4e15..00000000000 --- a/crates/storage/storage-api/src/legacy.rs +++ /dev/null @@ -1,84 +0,0 @@ -//! Traits used by the legacy execution engine. -//! -//! This module is scheduled for removal in the future. - -use alloc::boxed::Box; -use alloy_eips::BlockNumHash; -use alloy_primitives::{BlockHash, BlockNumber}; -use auto_impl::auto_impl; -use reth_execution_types::ExecutionOutcome; -use reth_storage_errors::provider::{ProviderError, ProviderResult}; - -/// Blockchain trait provider that gives access to the blockchain state that is not yet committed -/// (pending). -pub trait BlockchainTreePendingStateProvider: Send + Sync { - /// Returns a state provider that includes all state changes of the given (pending) block hash. - /// - /// In other words, the state provider will return the state after all transactions of the given - /// hash have been executed. - fn pending_state_provider( - &self, - block_hash: BlockHash, - ) -> ProviderResult> { - self.find_pending_state_provider(block_hash) - .ok_or(ProviderError::StateForHashNotFound(block_hash)) - } - - /// Returns state provider if a matching block exists. - fn find_pending_state_provider( - &self, - block_hash: BlockHash, - ) -> Option>; -} - -/// Provides data required for post-block execution. -/// -/// This trait offers methods to access essential post-execution data, including the state changes -/// in accounts and storage, as well as block hashes for both the pending and canonical chains. -/// -/// The trait includes: -/// * [`ExecutionOutcome`] - Captures all account and storage changes in the pending chain. -/// * Block hashes - Provides access to the block hashes of both the pending chain and canonical -/// blocks. -#[auto_impl(&, Box)] -pub trait ExecutionDataProvider: Send + Sync { - /// Return the execution outcome. - fn execution_outcome(&self) -> &ExecutionOutcome; - /// Return block hash by block number of pending or canonical chain. - fn block_hash(&self, block_number: BlockNumber) -> Option; -} - -impl ExecutionDataProvider for ExecutionOutcome { - fn execution_outcome(&self) -> &ExecutionOutcome { - self - } - - /// Always returns [None] because we don't have any information about the block header. - fn block_hash(&self, _block_number: BlockNumber) -> Option { - None - } -} - -/// Fork data needed for execution on it. -/// -/// It contains a canonical fork, the block on what pending chain was forked from. -#[auto_impl(&, Box)] -pub trait BlockExecutionForkProvider { - /// Return canonical fork, the block on what post state was forked from. - /// - /// Needed to create state provider. - fn canonical_fork(&self) -> BlockNumHash; -} - -/// Provides comprehensive post-execution state data required for further execution. -/// -/// This trait is used to create a state provider over the pending state and is a combination of -/// [`ExecutionDataProvider`] and [`BlockExecutionForkProvider`]. -/// -/// The pending state includes: -/// * `ExecutionOutcome`: Contains all changes to accounts and storage within the pending chain. -/// * Block hashes: Represents hashes of both the pending chain and canonical blocks. -/// * Canonical fork: Denotes the block from which the pending chain forked. -pub trait FullExecutionDataProvider: ExecutionDataProvider + BlockExecutionForkProvider {} - -impl FullExecutionDataProvider for T where T: ExecutionDataProvider + BlockExecutionForkProvider {} diff --git a/crates/storage/storage-api/src/lib.rs b/crates/storage/storage-api/src/lib.rs index c6505c5ae1f..897802da980 100644 --- a/crates/storage/storage-api/src/lib.rs +++ b/crates/storage/storage-api/src/lib.rs @@ -6,7 +6,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(feature = "std"), no_std)] extern crate alloc; @@ -57,12 +57,6 @@ pub use trie::*; mod chain_info; pub use chain_info::*; -mod withdrawals; -pub use withdrawals::*; - -mod ommers; -pub use ommers::*; - #[cfg(feature = "db-api")] mod database_provider; #[cfg(feature = "db-api")] @@ -85,9 +79,6 @@ mod stats; #[cfg(feature = "db-api")] pub use stats::*; -mod legacy; -pub use legacy::*; - mod primitives; pub use primitives::*; diff --git a/crates/storage/storage-api/src/noop.rs b/crates/storage/storage-api/src/noop.rs index 70188605baf..e538e1216e8 100644 --- a/crates/storage/storage-api/src/noop.rs +++ b/crates/storage/storage-api/src/noop.rs @@ -2,17 +2,20 @@ use crate::{ AccountReader, BlockBodyIndicesProvider, BlockHashReader, BlockIdReader, BlockNumReader, - BlockReader, BlockReaderIdExt, BlockSource, ChangeSetReader, HashedPostStateProvider, - HeaderProvider, NodePrimitivesProvider, OmmersProvider, PruneCheckpointReader, ReceiptProvider, - ReceiptProviderIdExt, StageCheckpointReader, StateProofProvider, StateProvider, - StateProviderBox, StateProviderFactory, StateRootProvider, StorageRootProvider, - TransactionVariant, TransactionsProvider, WithdrawalsProvider, + BlockReader, BlockReaderIdExt, BlockSource, BytecodeReader, ChangeSetReader, + HashedPostStateProvider, HeaderProvider, NodePrimitivesProvider, PruneCheckpointReader, + ReceiptProvider, ReceiptProviderIdExt, StageCheckpointReader, StateProofProvider, + StateProvider, StateProviderBox, StateProviderFactory, StateReader, StateRootProvider, + StorageRootProvider, TransactionVariant, TransactionsProvider, TrieReader, }; + +#[cfg(feature = "db-api")] +use crate::{DBProvider, DatabaseProviderFactory}; use alloc::{boxed::Box, string::String, sync::Arc, vec::Vec}; use alloy_consensus::transaction::TransactionMeta; -use alloy_eips::{eip4895::Withdrawals, BlockHashOrNumber, BlockId, BlockNumberOrTag}; +use alloy_eips::{BlockHashOrNumber, BlockId, BlockNumberOrTag}; use alloy_primitives::{ - Address, BlockHash, BlockNumber, Bytes, StorageKey, StorageValue, TxHash, TxNumber, B256, U256, + Address, BlockHash, BlockNumber, Bytes, StorageKey, StorageValue, TxHash, TxNumber, B256, }; use core::{ fmt::Debug, @@ -20,17 +23,21 @@ use core::{ ops::{RangeBounds, RangeInclusive}, }; use reth_chainspec::{ChainInfo, ChainSpecProvider, EthChainSpec, MAINNET}; +#[cfg(feature = "db-api")] +use reth_db_api::mock::{DatabaseMock, TxMock}; use reth_db_models::{AccountBeforeTx, StoredBlockBodyIndices}; use reth_ethereum_primitives::EthPrimitives; -use reth_primitives_traits::{ - Account, Bytecode, NodePrimitives, RecoveredBlock, SealedBlock, SealedHeader, -}; +use reth_execution_types::ExecutionOutcome; +use reth_primitives_traits::{Account, Bytecode, NodePrimitives, RecoveredBlock, SealedHeader}; +#[cfg(feature = "db-api")] +use reth_prune_types::PruneModes; use reth_prune_types::{PruneCheckpoint, PruneSegment}; use reth_stages_types::{StageCheckpoint, StageId}; use reth_storage_errors::provider::{ProviderError, ProviderResult}; use reth_trie_common::{ - updates::TrieUpdates, AccountProof, HashedPostState, HashedStorage, MultiProof, - MultiProofTargets, StorageMultiProof, StorageProof, TrieInput, + updates::{TrieUpdates, TrieUpdatesSorted}, + AccountProof, HashedPostState, HashedStorage, MultiProof, MultiProofTargets, StorageMultiProof, + StorageProof, TrieInput, }; /// Supports various api interfaces for testing purposes. @@ -38,20 +45,38 @@ use reth_trie_common::{ #[non_exhaustive] pub struct NoopProvider { chain_spec: Arc, + #[cfg(feature = "db-api")] + tx: TxMock, + #[cfg(feature = "db-api")] + prune_modes: PruneModes, _phantom: PhantomData, } impl NoopProvider { /// Create a new instance for specific primitive types. pub fn new(chain_spec: Arc) -> Self { - Self { chain_spec, _phantom: Default::default() } + Self { + chain_spec, + #[cfg(feature = "db-api")] + tx: TxMock::default(), + #[cfg(feature = "db-api")] + prune_modes: PruneModes::default(), + _phantom: Default::default(), + } } } impl NoopProvider { /// Create a new instance of the `NoopBlockReader`. pub fn eth(chain_spec: Arc) -> Self { - Self { chain_spec, _phantom: Default::default() } + Self { + chain_spec, + #[cfg(feature = "db-api")] + tx: TxMock::default(), + #[cfg(feature = "db-api")] + prune_modes: PruneModes::default(), + _phantom: Default::default(), + } } } @@ -70,7 +95,14 @@ impl Default for NoopProvider { impl Clone for NoopProvider { fn clone(&self) -> Self { - Self { chain_spec: Arc::clone(&self.chain_spec), _phantom: Default::default() } + Self { + chain_spec: Arc::clone(&self.chain_spec), + #[cfg(feature = "db-api")] + tx: self.tx.clone(), + #[cfg(feature = "db-api")] + prune_modes: self.prune_modes.clone(), + _phantom: Default::default(), + } } } @@ -146,10 +178,6 @@ impl BlockReaderIdExt for NoopProvider fn header_by_id(&self, _id: BlockId) -> ProviderResult> { Ok(None) } - - fn ommers_by_id(&self, _id: BlockId) -> ProviderResult>> { - Ok(None) - } } impl BlockReader for NoopProvider { @@ -167,17 +195,13 @@ impl BlockReader for NoopProvider { Ok(None) } - fn pending_block(&self) -> ProviderResult>> { - Ok(None) - } - - fn pending_block_with_senders(&self) -> ProviderResult>> { + fn pending_block(&self) -> ProviderResult>> { Ok(None) } fn pending_block_and_receipts( &self, - ) -> ProviderResult, Vec)>> { + ) -> ProviderResult, Vec)>> { Ok(None) } @@ -214,6 +238,10 @@ impl BlockReader for NoopProvider { ) -> ProviderResult>> { Ok(Vec::new()) } + + fn block_by_transaction_id(&self, _id: TxNumber) -> ProviderResult> { + Ok(None) + } } impl TransactionsProvider for NoopProvider { @@ -246,7 +274,7 @@ impl TransactionsProvider for NoopProvider ProviderResult> { - todo!() + Ok(None) } fn transactions_by_block( @@ -306,6 +334,13 @@ impl ReceiptProvider for NoopProvider { ) -> ProviderResult> { Ok(Vec::new()) } + + fn receipts_by_block_range( + &self, + _block_range: RangeInclusive, + ) -> ProviderResult>> { + Ok(Vec::new()) + } } impl ReceiptProviderIdExt for NoopProvider {} @@ -313,7 +348,7 @@ impl ReceiptProviderIdExt for NoopProvider HeaderProvider for NoopProvider { type Header = N::BlockHeader; - fn header(&self, _block_hash: &BlockHash) -> ProviderResult> { + fn header(&self, _block_hash: BlockHash) -> ProviderResult> { Ok(None) } @@ -321,14 +356,6 @@ impl HeaderProvider for NoopProvider { Ok(None) } - fn header_td(&self, _hash: &BlockHash) -> ProviderResult> { - Ok(None) - } - - fn header_td_by_number(&self, _number: BlockNumber) -> ProviderResult> { - Ok(None) - } - fn headers_range( &self, _range: impl RangeBounds, @@ -365,6 +392,14 @@ impl ChangeSetReader for NoopProvider { ) -> ProviderResult> { Ok(Vec::default()) } + + fn get_account_before_block( + &self, + _block_number: BlockNumber, + _address: Address, + ) -> ProviderResult> { + Ok(None) + } } impl StateRootProvider for NoopProvider { @@ -448,6 +483,17 @@ impl HashedPostStateProvider for NoopProvider } } +impl StateReader for NoopProvider { + type Receipt = N::Receipt; + + fn get_state( + &self, + _block: BlockNumber, + ) -> ProviderResult>> { + Ok(None) + } +} + impl StateProvider for NoopProvider { fn storage( &self, @@ -456,7 +502,9 @@ impl StateProvider for NoopProvider { ) -> ProviderResult> { Ok(None) } +} +impl BytecodeReader for NoopProvider { fn bytecode_by_hash(&self, _code_hash: &B256) -> ProviderResult> { Ok(None) } @@ -487,7 +535,9 @@ impl StateProviderFactory for NoopP self.history_by_block_hash(hash) } - BlockNumberOrTag::Earliest => self.history_by_block_number(0), + BlockNumberOrTag::Earliest => { + self.history_by_block_number(self.earliest_block_number()?) + } BlockNumberOrTag::Pending => self.pending(), BlockNumberOrTag::Number(num) => self.history_by_block_number(num), } @@ -512,6 +562,10 @@ impl StateProviderFactory for NoopP fn pending_state_by_hash(&self, _block_hash: B256) -> ProviderResult> { Ok(Some(Box::new(self.clone()))) } + + fn maybe_pending(&self) -> ProviderResult> { + Ok(Some(Box::new(self.clone()))) + } } impl StageCheckpointReader for NoopProvider { @@ -528,22 +582,6 @@ impl StageCheckpointReader for NoopProvider WithdrawalsProvider for NoopProvider { - fn withdrawals_by_block( - &self, - _id: BlockHashOrNumber, - _timestamp: u64, - ) -> ProviderResult> { - Ok(None) - } -} - -impl OmmersProvider for NoopProvider { - fn ommers(&self, _id: BlockHashOrNumber) -> ProviderResult>> { - Ok(None) - } -} - impl PruneCheckpointReader for NoopProvider { fn get_prune_checkpoint( &self, @@ -573,3 +611,60 @@ impl BlockBodyIndicesProvider for NoopProvider DBProvider for NoopProvider { + type Tx = TxMock; + + fn tx_ref(&self) -> &Self::Tx { + &self.tx + } + + fn tx_mut(&mut self) -> &mut Self::Tx { + &mut self.tx + } + + fn into_tx(self) -> Self::Tx { + self.tx + } + + fn prune_modes_ref(&self) -> &PruneModes { + &self.prune_modes + } + + fn commit(self) -> ProviderResult { + use reth_db_api::transaction::DbTx; + + Ok(self.tx.commit()?) + } +} + +impl TrieReader for NoopProvider { + fn trie_reverts(&self, _from: BlockNumber) -> ProviderResult { + Ok(TrieUpdatesSorted::default()) + } + + fn get_block_trie_updates( + &self, + _block_number: BlockNumber, + ) -> ProviderResult { + Ok(TrieUpdatesSorted::default()) + } +} + +#[cfg(feature = "db-api")] +impl DatabaseProviderFactory + for NoopProvider +{ + type DB = DatabaseMock; + type Provider = Self; + type ProviderRW = Self; + + fn database_provider_ro(&self) -> ProviderResult { + Ok(self.clone()) + } + + fn database_provider_rw(&self) -> ProviderResult { + Ok(self.clone()) + } +} diff --git a/crates/storage/storage-api/src/ommers.rs b/crates/storage/storage-api/src/ommers.rs deleted file mode 100644 index c3f68b4f96e..00000000000 --- a/crates/storage/storage-api/src/ommers.rs +++ /dev/null @@ -1,24 +0,0 @@ -use crate::HeaderProvider; -use alloc::{sync::Arc, vec::Vec}; -use alloy_eips::BlockHashOrNumber; -use reth_storage_errors::provider::ProviderResult; - -/// Client trait for fetching ommers. -pub trait OmmersProvider: HeaderProvider + Send + Sync { - /// Returns the ommers/uncle headers of the given block from the database. - /// - /// Returns `None` if block is not found. - fn ommers(&self, id: BlockHashOrNumber) -> ProviderResult>>; -} - -impl OmmersProvider for Arc { - fn ommers(&self, id: BlockHashOrNumber) -> ProviderResult>> { - T::ommers(self, id) - } -} - -impl OmmersProvider for &T { - fn ommers(&self, id: BlockHashOrNumber) -> ProviderResult>> { - T::ommers(self, id) - } -} diff --git a/crates/storage/storage-api/src/receipts.rs b/crates/storage/storage-api/src/receipts.rs index 969e0627c9b..f8390ee5384 100644 --- a/crates/storage/storage-api/src/receipts.rs +++ b/crates/storage/storage-api/src/receipts.rs @@ -1,8 +1,8 @@ use crate::BlockIdReader; use alloc::vec::Vec; use alloy_eips::{BlockHashOrNumber, BlockId, BlockNumberOrTag}; -use alloy_primitives::{TxHash, TxNumber}; -use core::ops::RangeBounds; +use alloy_primitives::{BlockNumber, TxHash, TxNumber}; +use core::ops::{RangeBounds, RangeInclusive}; use reth_primitives_traits::Receipt; use reth_storage_errors::provider::ProviderResult; @@ -38,6 +38,20 @@ pub trait ReceiptProvider: Send + Sync { &self, range: impl RangeBounds, ) -> ProviderResult>; + + /// Get receipts by block range. + /// + /// Returns a vector where each element contains all receipts for a block in the range. + /// The outer vector index corresponds to blocks in the range (`block_range.start()` + index). + /// Empty blocks will have empty inner vectors. + /// + /// This is more efficient than calling `receipts_by_block` multiple times for contiguous ranges + /// because it can leverage the underlying `receipts_by_tx_range` for the entire transaction + /// span. + fn receipts_by_block_range( + &self, + block_range: RangeInclusive, + ) -> ProviderResult>>; } /// Trait extension for `ReceiptProvider`, for types that implement `BlockId` conversion. diff --git a/crates/storage/storage-api/src/state.rs b/crates/storage/storage-api/src/state.rs index 920010cb3e6..dc8241fb95f 100644 --- a/crates/storage/storage-api/src/state.rs +++ b/crates/storage/storage-api/src/state.rs @@ -34,6 +34,7 @@ pub type StateProviderBox = Box; pub trait StateProvider: BlockHashReader + AccountReader + + BytecodeReader + StateRootProvider + StorageRootProvider + StateProofProvider @@ -48,9 +49,6 @@ pub trait StateProvider: storage_key: StorageKey, ) -> ProviderResult>; - /// Get account code by its hash - fn bytecode_by_hash(&self, code_hash: &B256) -> ProviderResult>; - /// Get account code by its address. /// /// Returns `None` if the account doesn't exist or account is not a contract @@ -94,14 +92,9 @@ pub trait StateProvider: } } -/// Trait implemented for database providers that can provide the [`reth_trie_db::StateCommitment`] -/// type. -#[cfg(feature = "db-api")] -pub trait StateCommitmentProvider: Send + Sync { - /// The [`reth_trie_db::StateCommitment`] type that can be used to perform state commitment - /// operations. - type StateCommitment: reth_trie_db::StateCommitment; -} +/// Minimal requirements to read a full account, for example, to validate its new transactions +pub trait AccountInfoReader: AccountReader + BytecodeReader {} +impl AccountInfoReader for T {} /// Trait that provides the hashed state from various sources. #[auto_impl(&, Arc, Box)] @@ -110,6 +103,13 @@ pub trait HashedPostStateProvider: Send + Sync { fn hashed_post_state(&self, bundle_state: &BundleState) -> HashedPostState; } +/// Trait for reading bytecode associated with a given code hash. +#[auto_impl(&, Arc, Box)] +pub trait BytecodeReader: Send + Sync { + /// Get account code by its hash + fn bytecode_by_hash(&self, code_hash: &B256) -> ProviderResult>; +} + /// Trait implemented for database providers that can be converted into a historical state provider. pub trait TryIntoHistoricalStateProvider { /// Returns a historical [`StateProvider`] indexed by the given historic block number. @@ -134,7 +134,7 @@ pub trait TryIntoHistoricalStateProvider { /// has the `latest` block as its parent. /// /// All states are _inclusive_, meaning they include _all_ all changes made (executed transactions) -/// in their respective blocks. For example [StateProviderFactory::history_by_block_number] for +/// in their respective blocks. For example [`StateProviderFactory::history_by_block_number`] for /// block number `n` will return the state after block `n` was executed (transactions, withdrawals). /// In other words, all states point to the end of the state's respective block, which is equivalent /// to state at the beginning of the child block. @@ -158,7 +158,7 @@ pub trait StateProviderFactory: BlockIdReader + Send + Sync { } } - /// Returns a [StateProvider] indexed by the given block number or tag. + /// Returns a [`StateProvider`] indexed by the given block number or tag. /// /// Note: if a number is provided this will only look at historical(canonical) state. fn state_by_block_number_or_tag( @@ -166,13 +166,13 @@ pub trait StateProviderFactory: BlockIdReader + Send + Sync { number_or_tag: BlockNumberOrTag, ) -> ProviderResult; - /// Returns a historical [StateProvider] indexed by the given historic block number. + /// Returns a historical [`StateProvider`] indexed by the given historic block number. /// /// /// Note: this only looks at historical blocks, not pending blocks. fn history_by_block_number(&self, block: BlockNumber) -> ProviderResult; - /// Returns a historical [StateProvider] indexed by the given block hash. + /// Returns a historical [`StateProvider`] indexed by the given block hash. /// /// Note: this only looks at historical blocks, not pending blocks. fn history_by_block_hash(&self, block: BlockHash) -> ProviderResult; @@ -185,7 +185,7 @@ pub trait StateProviderFactory: BlockIdReader + Send + Sync { /// Storage provider for pending state. /// /// Represents the state at the block that extends the canonical chain by one. - /// If there's no `pending` block, then this is equal to [StateProviderFactory::latest] + /// If there's no `pending` block, then this is equal to [`StateProviderFactory::latest`] fn pending(&self) -> ProviderResult; /// Storage provider for pending state for the given block hash. @@ -194,4 +194,9 @@ pub trait StateProviderFactory: BlockIdReader + Send + Sync { /// /// If the block couldn't be found, returns `None`. fn pending_state_by_hash(&self, block_hash: B256) -> ProviderResult>; + + /// Returns a pending [`StateProvider`] if it exists. + /// + /// This will return `None` if there's no pending state. + fn maybe_pending(&self) -> ProviderResult>; } diff --git a/crates/storage/storage-api/src/state_writer.rs b/crates/storage/storage-api/src/state_writer.rs index 0710d849778..711b9e569f5 100644 --- a/crates/storage/storage-api/src/state_writer.rs +++ b/crates/storage/storage-api/src/state_writer.rs @@ -7,8 +7,6 @@ use revm_database::{ OriginalValuesKnown, }; -use super::StorageLocation; - /// A trait specifically for writing state changes or reverts pub trait StateWriter { /// Receipt type included into [`ExecutionOutcome`]. @@ -20,7 +18,6 @@ pub trait StateWriter { &self, execution_outcome: &ExecutionOutcome, is_value_known: OriginalValuesKnown, - write_receipts_to: StorageLocation, ) -> ProviderResult<()>; /// Write state reverts to the database. @@ -40,17 +37,12 @@ pub trait StateWriter { /// Remove the block range of state above the given block. The state of the passed block is not /// removed. - fn remove_state_above( - &self, - block: BlockNumber, - remove_receipts_from: StorageLocation, - ) -> ProviderResult<()>; + fn remove_state_above(&self, block: BlockNumber) -> ProviderResult<()>; /// Take the block range of state, recreating the [`ExecutionOutcome`]. The state of the passed /// block is not removed. fn take_state_above( &self, block: BlockNumber, - remove_receipts_from: StorageLocation, ) -> ProviderResult>; } diff --git a/crates/storage/storage-api/src/storage.rs b/crates/storage/storage-api/src/storage.rs index 5fa8b0d2483..8f560d8cfb7 100644 --- a/crates/storage/storage-api/src/storage.rs +++ b/crates/storage/storage-api/src/storage.rs @@ -32,7 +32,7 @@ pub trait StorageReader: Send + Sync { ) -> ProviderResult>>; } -/// Storage ChangeSet reader +/// Storage `ChangeSet` reader #[cfg(feature = "db-api")] #[auto_impl::auto_impl(&, Arc, Box)] pub trait StorageChangeSetReader: Send + Sync { @@ -42,26 +42,3 @@ pub trait StorageChangeSetReader: Send + Sync { block_number: BlockNumber, ) -> ProviderResult>; } - -/// An enum that represents the storage location for a piece of data. -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -pub enum StorageLocation { - /// Write only to static files. - StaticFiles, - /// Write only to the database. - Database, - /// Write to both the database and static files. - Both, -} - -impl StorageLocation { - /// Returns true if the storage location includes static files. - pub const fn static_files(&self) -> bool { - matches!(self, Self::StaticFiles | Self::Both) - } - - /// Returns true if the storage location includes the database. - pub const fn database(&self) -> bool { - matches!(self, Self::Database | Self::Both) - } -} diff --git a/crates/storage/storage-api/src/transactions.rs b/crates/storage/storage-api/src/transactions.rs index 8d9f20bf23b..732d0437592 100644 --- a/crates/storage/storage-api/src/transactions.rs +++ b/crates/storage/storage-api/src/transactions.rs @@ -9,7 +9,7 @@ use reth_storage_errors::provider::{ProviderError, ProviderResult}; /// Enum to control transaction hash inclusion. /// -/// This serves as a hint to the provider to include or omit exclude hashes because hashes are +/// This serves as a hint to the provider to include or omit hashes because hashes are /// stored separately and are not always needed. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] pub enum TransactionVariant { @@ -28,7 +28,7 @@ pub trait TransactionsProvider: BlockNumReader + Send + Sync { /// Get internal transaction identifier by transaction hash. /// - /// This is the inverse of [TransactionsProvider::transaction_by_id]. + /// This is the inverse of [`TransactionsProvider::transaction_by_id`]. /// Returns None if the transaction is not found. fn transaction_id(&self, tx_hash: TxHash) -> ProviderResult>; diff --git a/crates/storage/storage-api/src/trie.rs b/crates/storage/storage-api/src/trie.rs index 9ae8ebee9a0..9ff02c106e5 100644 --- a/crates/storage/storage-api/src/trie.rs +++ b/crates/storage/storage-api/src/trie.rs @@ -1,8 +1,8 @@ use alloc::vec::Vec; -use alloy_primitives::{map::B256Map, Address, Bytes, B256}; +use alloy_primitives::{Address, BlockNumber, Bytes, B256}; use reth_storage_errors::provider::ProviderResult; use reth_trie_common::{ - updates::{StorageTrieUpdates, TrieUpdates}, + updates::{StorageTrieUpdatesSorted, TrieUpdates, TrieUpdatesSorted}, AccountProof, HashedPostState, HashedStorage, MultiProof, MultiProofTargets, StorageMultiProof, StorageProof, TrieInput, }; @@ -89,32 +89,93 @@ pub trait StateProofProvider: Send + Sync { fn witness(&self, input: TrieInput, target: HashedPostState) -> ProviderResult>; } +/// Trie Reader +#[auto_impl::auto_impl(&, Arc, Box)] +pub trait TrieReader: Send + Sync { + /// Returns the [`TrieUpdatesSorted`] for reverting the trie database to its state prior to the + /// given block and onwards having been processed. + fn trie_reverts(&self, from: BlockNumber) -> ProviderResult; + + /// Returns the trie updates that were applied by the specified block. + fn get_block_trie_updates( + &self, + block_number: BlockNumber, + ) -> ProviderResult; +} + /// Trie Writer #[auto_impl::auto_impl(&, Arc, Box)] pub trait TrieWriter: Send + Sync { /// Writes trie updates to the database. /// /// Returns the number of entries modified. - fn write_trie_updates(&self, trie_updates: &TrieUpdates) -> ProviderResult; + fn write_trie_updates(&self, trie_updates: TrieUpdates) -> ProviderResult { + self.write_trie_updates_sorted(&trie_updates.into_sorted()) + } + + /// Writes trie updates to the database with already sorted updates. + /// + /// Returns the number of entries modified. + fn write_trie_updates_sorted(&self, trie_updates: &TrieUpdatesSorted) -> ProviderResult; + + /// Records the current values of all trie nodes which will be updated using the [`TrieUpdates`] + /// into the trie changesets tables. + /// + /// The intended usage of this method is to call it _prior_ to calling `write_trie_updates` with + /// the same [`TrieUpdates`]. + /// + /// The `updates_overlay` parameter allows providing additional in-memory trie updates that + /// should be considered when looking up current node values. When provided, these overlay + /// updates are applied on top of the database state, allowing the method to see a view that + /// includes both committed database values and pending in-memory changes. This is useful + /// when writing changesets for updates that depend on previous uncommitted trie changes. + /// + /// Returns the number of keys written. + fn write_trie_changesets( + &self, + block_number: BlockNumber, + trie_updates: &TrieUpdatesSorted, + updates_overlay: Option<&TrieUpdatesSorted>, + ) -> ProviderResult; + + /// Clears contents of trie changesets completely + fn clear_trie_changesets(&self) -> ProviderResult<()>; + + /// Clears contents of trie changesets starting from the given block number (inclusive) onwards. + fn clear_trie_changesets_from(&self, from: BlockNumber) -> ProviderResult<()>; } /// Storage Trie Writer #[auto_impl::auto_impl(&, Arc, Box)] pub trait StorageTrieWriter: Send + Sync { - /// Writes storage trie updates from the given storage trie map. + /// Writes storage trie updates from the given storage trie map with already sorted updates. /// - /// First sorts the storage trie updates by the hashed address key, writing in sorted order. + /// Expects the storage trie updates to already be sorted by the hashed address key. /// /// Returns the number of entries modified. - fn write_storage_trie_updates( + fn write_storage_trie_updates_sorted<'a>( &self, - storage_tries: &B256Map, + storage_tries: impl Iterator, ) -> ProviderResult; - /// Writes storage trie updates for the given hashed address. - fn write_individual_storage_trie_updates( + /// Records the current values of all trie nodes which will be updated using the + /// [`StorageTrieUpdatesSorted`] into the storage trie changesets table. + /// + /// The intended usage of this method is to call it _prior_ to calling + /// `write_storage_trie_updates` with the same set of [`StorageTrieUpdatesSorted`]. + /// + /// The `updates_overlay` parameter allows providing additional in-memory trie updates that + /// should be considered when looking up current node values. When provided, these overlay + /// updates are applied on top of the database state for each storage trie, allowing the + /// method to see a view that includes both committed database values and pending in-memory + /// changes. This is useful when writing changesets for storage updates that depend on + /// previous uncommitted trie changes. + /// + /// Returns the number of keys written. + fn write_storage_trie_changesets<'a>( &self, - hashed_address: B256, - updates: &StorageTrieUpdates, + block_number: BlockNumber, + storage_tries: impl Iterator, + updates_overlay: Option<&TrieUpdatesSorted>, ) -> ProviderResult; } diff --git a/crates/storage/storage-api/src/withdrawals.rs b/crates/storage/storage-api/src/withdrawals.rs deleted file mode 100644 index fdfb27aa707..00000000000 --- a/crates/storage/storage-api/src/withdrawals.rs +++ /dev/null @@ -1,13 +0,0 @@ -use alloy_eips::{eip4895::Withdrawals, BlockHashOrNumber}; -use reth_storage_errors::provider::ProviderResult; - -/// Client trait for fetching [`alloy_eips::eip4895::Withdrawal`] related data. -#[auto_impl::auto_impl(&, Arc)] -pub trait WithdrawalsProvider: Send + Sync { - /// Get withdrawals by block id. - fn withdrawals_by_block( - &self, - id: BlockHashOrNumber, - timestamp: u64, - ) -> ProviderResult>; -} diff --git a/crates/storage/zstd-compressors/src/lib.rs b/crates/storage/zstd-compressors/src/lib.rs index d7f2b65904d..28f6259c25f 100644 --- a/crates/storage/zstd-compressors/src/lib.rs +++ b/crates/storage/zstd-compressors/src/lib.rs @@ -6,7 +6,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(feature = "std"), no_std)] extern crate alloc; @@ -118,10 +118,10 @@ impl ReusableDecompressor { // source. if !reserved_upper_bound { reserved_upper_bound = true; - if let Some(upper_bound) = Decompressor::upper_bound(src) { - if let Some(additional) = upper_bound.checked_sub(self.buf.capacity()) { - break 'b additional - } + if let Some(upper_bound) = Decompressor::upper_bound(src) && + let Some(additional) = upper_bound.checked_sub(self.buf.capacity()) + { + break 'b additional } } diff --git a/crates/tasks/src/lib.rs b/crates/tasks/src/lib.rs index 53ed1d2919a..de45c41e24d 100644 --- a/crates/tasks/src/lib.rs +++ b/crates/tasks/src/lib.rs @@ -10,7 +10,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] use crate::{ metrics::{IncCounterOnDrop, TaskExecutorMetrics}, @@ -162,10 +162,10 @@ pub struct TaskManager { /// /// See [`Handle`] docs. handle: Handle, - /// Sender half for sending panic signals to this type - panicked_tasks_tx: UnboundedSender, - /// Listens for panicked tasks - panicked_tasks_rx: UnboundedReceiver, + /// Sender half for sending task events to this type + task_events_tx: UnboundedSender, + /// Receiver for task events + task_events_rx: UnboundedReceiver, /// The [Signal] to fire when all tasks should be shutdown. /// /// This is fired when dropped. @@ -179,7 +179,11 @@ pub struct TaskManager { // === impl TaskManager === impl TaskManager { - /// Returns a new [`TaskManager`] over the currently running Runtime. + /// Returns a __new__ [`TaskManager`] over the currently running Runtime. + /// + /// This must be polled for the duration of the program. + /// + /// To obtain the current [`TaskExecutor`] see [`TaskExecutor::current`]. /// /// # Panics /// @@ -193,12 +197,12 @@ impl TaskManager { /// /// This also sets the global [`TaskExecutor`]. pub fn new(handle: Handle) -> Self { - let (panicked_tasks_tx, panicked_tasks_rx) = unbounded_channel(); + let (task_events_tx, task_events_rx) = unbounded_channel(); let (signal, on_shutdown) = signal(); let manager = Self { handle, - panicked_tasks_tx, - panicked_tasks_rx, + task_events_tx, + task_events_rx, signal: Some(signal), on_shutdown, graceful_tasks: Arc::new(AtomicUsize::new(0)), @@ -217,7 +221,7 @@ impl TaskManager { TaskExecutor { handle: self.handle.clone(), on_shutdown: self.on_shutdown.clone(), - panicked_tasks_tx: self.panicked_tasks_tx.clone(), + task_events_tx: self.task_events_tx.clone(), metrics: Default::default(), graceful_tasks: Arc::clone(&self.graceful_tasks), } @@ -255,16 +259,23 @@ impl TaskManager { /// /// See [`TaskExecutor::spawn_critical`] impl Future for TaskManager { - type Output = PanickedTaskError; - - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - let err = ready!(self.get_mut().panicked_tasks_rx.poll_recv(cx)); - Poll::Ready(err.expect("stream can not end")) + type Output = Result<(), PanickedTaskError>; + + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + match ready!(self.as_mut().get_mut().task_events_rx.poll_recv(cx)) { + Some(TaskEvent::Panic(err)) => Poll::Ready(Err(err)), + Some(TaskEvent::GracefulShutdown) | None => { + if let Some(signal) = self.get_mut().signal.take() { + signal.fire(); + } + Poll::Ready(Ok(())) + } + } } } /// Error with the name of the task that panicked and an error downcasted to string, if possible. -#[derive(Debug, thiserror::Error)] +#[derive(Debug, thiserror::Error, PartialEq, Eq)] pub struct PanickedTaskError { task_name: &'static str, error: Option, @@ -295,6 +306,15 @@ impl PanickedTaskError { } } +/// Represents the events that the `TaskManager`'s main future can receive. +#[derive(Debug)] +enum TaskEvent { + /// Indicates that a critical task has panicked. + Panic(PanickedTaskError), + /// A signal requesting a graceful shutdown of the `TaskManager`. + GracefulShutdown, +} + /// A type that can spawn new tokio tasks #[derive(Debug, Clone)] pub struct TaskExecutor { @@ -304,8 +324,8 @@ pub struct TaskExecutor { handle: Handle, /// Receiver of the shutdown signal. on_shutdown: Shutdown, - /// Sender half for sending panic signals to this type - panicked_tasks_tx: UnboundedSender, + /// Sender half for sending task events to this type + task_events_tx: UnboundedSender, /// Task Executor Metrics metrics: TaskExecutorMetrics, /// How many [`GracefulShutdown`] tasks are currently active @@ -363,15 +383,17 @@ impl TaskExecutor { { let on_shutdown = self.on_shutdown.clone(); - // Clone only the specific counter that we need. - let finished_regular_tasks_total_metrics = - self.metrics.finished_regular_tasks_total.clone(); + // Choose the appropriate finished counter based on task kind + let finished_counter = match task_kind { + TaskKind::Default => self.metrics.finished_regular_tasks_total.clone(), + TaskKind::Blocking => self.metrics.finished_regular_blocking_tasks_total.clone(), + }; + // Wrap the original future to increment the finished tasks counter upon completion let task = { async move { // Create an instance of IncCounterOnDrop with the counter to increment - let _inc_counter_on_drop = - IncCounterOnDrop::new(finished_regular_tasks_total_metrics); + let _inc_counter_on_drop = IncCounterOnDrop::new(finished_counter); let fut = pin!(fut); let _ = select(on_shutdown, fut).await; } @@ -429,7 +451,7 @@ impl TaskExecutor { where F: Future + Send + 'static, { - let panicked_tasks_tx = self.panicked_tasks_tx.clone(); + let panicked_tasks_tx = self.task_events_tx.clone(); let on_shutdown = self.on_shutdown.clone(); // wrap the task in catch unwind @@ -438,7 +460,7 @@ impl TaskExecutor { .map_err(move |error| { let task_error = PanickedTaskError::new(name, error); error!("{task_error}"); - let _ = panicked_tasks_tx.send(task_error); + let _ = panicked_tasks_tx.send(TaskEvent::Panic(task_error)); }) .in_current_span(); @@ -488,7 +510,7 @@ impl TaskExecutor { where F: Future + Send + 'static, { - let panicked_tasks_tx = self.panicked_tasks_tx.clone(); + let panicked_tasks_tx = self.task_events_tx.clone(); let on_shutdown = self.on_shutdown.clone(); let fut = f(on_shutdown); @@ -498,7 +520,7 @@ impl TaskExecutor { .map_err(move |error| { let task_error = PanickedTaskError::new(name, error); error!("{task_error}"); - let _ = panicked_tasks_tx.send(task_error); + let _ = panicked_tasks_tx.send(TaskEvent::Panic(task_error)); }) .map(drop) .in_current_span(); @@ -534,7 +556,7 @@ impl TaskExecutor { where F: Future + Send + 'static, { - let panicked_tasks_tx = self.panicked_tasks_tx.clone(); + let panicked_tasks_tx = self.task_events_tx.clone(); let on_shutdown = GracefulShutdown::new( self.on_shutdown.clone(), GracefulShutdownGuard::new(Arc::clone(&self.graceful_tasks)), @@ -547,7 +569,7 @@ impl TaskExecutor { .map_err(move |error| { let task_error = PanickedTaskError::new(name, error); error!("{task_error}"); - let _ = panicked_tasks_tx.send(task_error); + let _ = panicked_tasks_tx.send(TaskEvent::Panic(task_error)); }) .map(drop) .in_current_span(); @@ -589,6 +611,25 @@ impl TaskExecutor { self.handle.spawn(fut) } + + /// Sends a request to the `TaskManager` to initiate a graceful shutdown. + /// + /// Caution: This will terminate the entire program. + /// + /// The [`TaskManager`] upon receiving this event, will terminate and initiate the shutdown that + /// can be handled via the returned [`GracefulShutdown`]. + pub fn initiate_graceful_shutdown( + &self, + ) -> Result> { + self.task_events_tx + .send(TaskEvent::GracefulShutdown) + .map_err(|_send_error_with_task_event| tokio::sync::mpsc::error::SendError(()))?; + + Ok(GracefulShutdown::new( + self.on_shutdown.clone(), + GracefulShutdownGuard::new(Arc::clone(&self.graceful_tasks)), + )) + } } impl TaskSpawner for TaskExecutor { @@ -603,6 +644,7 @@ impl TaskSpawner for TaskExecutor { } fn spawn_blocking(&self, fut: BoxFuture<'static, ()>) -> JoinHandle<()> { + self.metrics.inc_regular_blocking_tasks(); self.spawn_blocking(fut) } @@ -611,6 +653,7 @@ impl TaskSpawner for TaskExecutor { name: &'static str, fut: BoxFuture<'static, ()>, ) -> JoinHandle<()> { + self.metrics.inc_critical_tasks(); Self::spawn_critical_blocking(self, name, fut) } } @@ -707,9 +750,12 @@ mod tests { executor.spawn_critical("this is a critical task", async { panic!("intentionally panic") }); runtime.block_on(async move { - let err = manager.await; - assert_eq!(err.task_name, "this is a critical task"); - assert_eq!(err.error, Some("intentionally panic".to_string())); + let err_result = manager.await; + assert!(err_result.is_err(), "Expected TaskManager to return an error due to panic"); + let panicked_err = err_result.unwrap_err(); + + assert_eq!(panicked_err.task_name, "this is a critical task"); + assert_eq!(panicked_err.error, Some("intentionally panic".to_string())); }) } @@ -825,4 +871,41 @@ mod tests { let _manager = TaskManager::new(handle); let _executor = TaskExecutor::try_current().unwrap(); } + + #[test] + fn test_graceful_shutdown_triggered_by_executor() { + let runtime = tokio::runtime::Runtime::new().unwrap(); + let task_manager = TaskManager::new(runtime.handle().clone()); + let executor = task_manager.executor(); + + let task_did_shutdown_flag = Arc::new(AtomicBool::new(false)); + let flag_clone = task_did_shutdown_flag.clone(); + + let spawned_task_handle = executor.spawn_with_signal(|shutdown_signal| async move { + shutdown_signal.await; + flag_clone.store(true, Ordering::SeqCst); + }); + + let manager_future_handle = runtime.spawn(task_manager); + + let send_result = executor.initiate_graceful_shutdown(); + assert!(send_result.is_ok(), "Sending the graceful shutdown signal should succeed and return a GracefulShutdown future"); + + let manager_final_result = runtime.block_on(manager_future_handle); + + assert!(manager_final_result.is_ok(), "TaskManager task should not panic"); + assert_eq!( + manager_final_result.unwrap(), + Ok(()), + "TaskManager should resolve cleanly with Ok(()) after graceful shutdown request" + ); + + let task_join_result = runtime.block_on(spawned_task_handle); + assert!(task_join_result.is_ok(), "Spawned task should complete without panic"); + + assert!( + task_did_shutdown_flag.load(Ordering::Relaxed), + "Task should have received the shutdown signal and set the flag" + ); + } } diff --git a/crates/tasks/src/metrics.rs b/crates/tasks/src/metrics.rs index c486fa681cc..24d3065a529 100644 --- a/crates/tasks/src/metrics.rs +++ b/crates/tasks/src/metrics.rs @@ -16,6 +16,10 @@ pub struct TaskExecutorMetrics { pub(crate) regular_tasks_total: Counter, /// Number of finished spawned regular tasks pub(crate) finished_regular_tasks_total: Counter, + /// Number of spawned regular blocking tasks + pub(crate) regular_blocking_tasks_total: Counter, + /// Number of finished spawned regular blocking tasks + pub(crate) finished_regular_blocking_tasks_total: Counter, } impl TaskExecutorMetrics { @@ -28,6 +32,11 @@ impl TaskExecutorMetrics { pub(crate) fn inc_regular_tasks(&self) { self.regular_tasks_total.increment(1); } + + /// Increments the counter for spawned regular blocking tasks. + pub(crate) fn inc_regular_blocking_tasks(&self) { + self.regular_blocking_tasks_total.increment(1); + } } /// Helper type for increasing counters even if a task fails diff --git a/crates/tasks/src/pool.rs b/crates/tasks/src/pool.rs index 10fedccedd1..76087b71ef6 100644 --- a/crates/tasks/src/pool.rs +++ b/crates/tasks/src/pool.rs @@ -69,8 +69,9 @@ impl BlockingTaskPool { /// Convenience function to build a new threadpool with the default configuration. /// - /// Uses [`rayon::ThreadPoolBuilder::build`](rayon::ThreadPoolBuilder::build) defaults but - /// increases the stack size to 8MB. + /// Uses [`rayon::ThreadPoolBuilder::build`](rayon::ThreadPoolBuilder::build) defaults. + /// If a different stack size or other parameters are needed, they can be configured via + /// [`rayon::ThreadPoolBuilder`] returned by [`Self::builder`]. pub fn build() -> Result { Self::builder().build().map(Self::new) } diff --git a/crates/tokio-util/src/lib.rs b/crates/tokio-util/src/lib.rs index e476c4063d9..124807fc5cc 100644 --- a/crates/tokio-util/src/lib.rs +++ b/crates/tokio-util/src/lib.rs @@ -6,7 +6,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] mod event_sender; mod event_stream; diff --git a/crates/tracing-otlp/Cargo.toml b/crates/tracing-otlp/Cargo.toml new file mode 100644 index 00000000000..5b01095d4ff --- /dev/null +++ b/crates/tracing-otlp/Cargo.toml @@ -0,0 +1,38 @@ +[package] +name = "reth-tracing-otlp" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true +exclude.workspace = true + +[dependencies] +# obs +opentelemetry_sdk = { workspace = true, optional = true } +opentelemetry = { workspace = true, optional = true } +opentelemetry-otlp = { workspace = true, optional = true, features = ["grpc-tonic"] } +opentelemetry-semantic-conventions = { workspace = true, optional = true } +tracing-opentelemetry = { workspace = true, optional = true } +tracing-subscriber.workspace = true +tracing.workspace = true + +# misc +clap = { workspace = true, features = ["derive"] } +eyre.workspace = true +url.workspace = true + +[lints] +workspace = true + +[features] +default = ["otlp"] + +otlp = [ + "opentelemetry", + "opentelemetry_sdk", + "opentelemetry-otlp", + "opentelemetry-semantic-conventions", + "tracing-opentelemetry", +] diff --git a/crates/tracing-otlp/src/lib.rs b/crates/tracing-otlp/src/lib.rs new file mode 100644 index 00000000000..2cfd332a408 --- /dev/null +++ b/crates/tracing-otlp/src/lib.rs @@ -0,0 +1,103 @@ +#![cfg(feature = "otlp")] + +//! Provides a tracing layer for `OpenTelemetry` that exports spans to an OTLP endpoint. +//! +//! This module simplifies the integration of `OpenTelemetry` tracing with OTLP export in Rust +//! applications. It allows for easily capturing and exporting distributed traces to compatible +//! backends like Jaeger, Zipkin, or any other OpenTelemetry-compatible tracing system. + +use clap::ValueEnum; +use eyre::ensure; +use opentelemetry::{global, trace::TracerProvider, KeyValue, Value}; +use opentelemetry_otlp::{SpanExporter, WithExportConfig}; +use opentelemetry_sdk::{ + propagation::TraceContextPropagator, + trace::{SdkTracer, SdkTracerProvider}, + Resource, +}; +use opentelemetry_semantic_conventions::{attribute::SERVICE_VERSION, SCHEMA_URL}; +use tracing::Subscriber; +use tracing_opentelemetry::OpenTelemetryLayer; +use tracing_subscriber::registry::LookupSpan; +use url::Url; + +// Otlp http endpoint is expected to end with this path. +// See also . +const HTTP_TRACE_ENDPOINT: &str = "/v1/traces"; + +/// Creates a tracing [`OpenTelemetryLayer`] that exports spans to an OTLP endpoint. +/// +/// This layer can be added to a [`tracing_subscriber::Registry`] to enable `OpenTelemetry` tracing +/// with OTLP export to an url. +pub fn span_layer( + service_name: impl Into, + endpoint: &Url, + protocol: OtlpProtocol, +) -> eyre::Result> +where + for<'span> S: Subscriber + LookupSpan<'span>, +{ + global::set_text_map_propagator(TraceContextPropagator::new()); + + let resource = build_resource(service_name); + + let span_builder = SpanExporter::builder(); + + let span_exporter = match protocol { + OtlpProtocol::Http => span_builder.with_http().with_endpoint(endpoint.as_str()).build()?, + OtlpProtocol::Grpc => span_builder.with_tonic().with_endpoint(endpoint.as_str()).build()?, + }; + + let tracer_provider = SdkTracerProvider::builder() + .with_resource(resource) + .with_batch_exporter(span_exporter) + .build(); + + global::set_tracer_provider(tracer_provider.clone()); + + let tracer = tracer_provider.tracer("reth"); + Ok(tracing_opentelemetry::layer().with_tracer(tracer)) +} + +// Builds OTLP resource with service information. +fn build_resource(service_name: impl Into) -> Resource { + Resource::builder() + .with_service_name(service_name) + .with_schema_url([KeyValue::new(SERVICE_VERSION, env!("CARGO_PKG_VERSION"))], SCHEMA_URL) + .build() +} + +/// OTLP transport protocol type +#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)] +pub enum OtlpProtocol { + /// HTTP/Protobuf transport, port 4318, requires `/v1/traces` path + Http, + /// gRPC transport, port 4317 + Grpc, +} + +impl OtlpProtocol { + /// Validate and correct the URL to match protocol requirements. + /// + /// For HTTP: Ensures the path ends with `/v1/traces`, appending it if necessary. + /// For gRPC: Ensures the path does NOT include `/v1/traces`. + pub fn validate_endpoint(&self, url: &mut Url) -> eyre::Result<()> { + match self { + Self::Http => { + if !url.path().ends_with(HTTP_TRACE_ENDPOINT) { + let path = url.path().trim_end_matches('/'); + url.set_path(&format!("{}{}", path, HTTP_TRACE_ENDPOINT)); + } + } + Self::Grpc => { + ensure!( + !url.path().ends_with(HTTP_TRACE_ENDPOINT), + "OTLP gRPC endpoint should not include {} path, got: {}", + HTTP_TRACE_ENDPOINT, + url + ); + } + } + Ok(()) + } +} diff --git a/crates/tracing/Cargo.toml b/crates/tracing/Cargo.toml index a5c09c23a35..8cf83e138ca 100644 --- a/crates/tracing/Cargo.toml +++ b/crates/tracing/Cargo.toml @@ -12,11 +12,22 @@ description = "tracing helpers" workspace = true [dependencies] +# reth +reth-tracing-otlp = { workspace = true, optional = true } + +# obs tracing.workspace = true tracing-subscriber = { workspace = true, features = ["env-filter", "fmt", "ansi", "json"] } tracing-appender.workspace = true tracing-journald.workspace = true tracing-logfmt.workspace = true -rolling-file.workspace = true -eyre.workspace = true + +# misc clap = { workspace = true, features = ["derive"] } +eyre.workspace = true +rolling-file.workspace = true +url = { workspace = true, optional = true } + +[features] +default = ["otlp"] +otlp = ["reth-tracing-otlp", "dep:url"] diff --git a/crates/tracing/src/layers.rs b/crates/tracing/src/layers.rs index 18ff68e1d8f..33f8c90ada5 100644 --- a/crates/tracing/src/layers.rs +++ b/crates/tracing/src/layers.rs @@ -1,10 +1,16 @@ -use std::path::{Path, PathBuf}; - +use crate::formatter::LogFormat; use rolling_file::{RollingConditionBasic, RollingFileAppender}; +use std::{ + fmt, + path::{Path, PathBuf}, +}; use tracing_appender::non_blocking::WorkerGuard; use tracing_subscriber::{filter::Directive, EnvFilter, Layer, Registry}; - -use crate::formatter::LogFormat; +#[cfg(feature = "otlp")] +use { + reth_tracing_otlp::{span_layer, OtlpProtocol}, + url::Url, +}; /// A worker guard returned by the file layer. /// @@ -15,30 +21,49 @@ pub type FileWorkerGuard = tracing_appender::non_blocking::WorkerGuard; /// A boxed tracing [Layer]. pub(crate) type BoxedLayer = Box + Send + Sync>; -const RETH_LOG_FILE_NAME: &str = "reth.log"; - -/// Default [directives](Directive) for [`EnvFilter`] which disables high-frequency debug logs from -/// `hyper`, `hickory-resolver`, `jsonrpsee-server`, and `discv5`. -const DEFAULT_ENV_FILTER_DIRECTIVES: [&str; 5] = [ +/// Default [directives](Directive) for [`EnvFilter`] which: +/// 1. Disable high-frequency debug logs from dependencies such as `hyper`, `hickory-resolver`, +/// `hickory_proto`, `discv5`, `jsonrpsee-server`, and `hyper_util::client::legacy::pool`. +/// 2. Set `opentelemetry_*` crates log level to `WARN`, as `DEBUG` is too noisy. +const DEFAULT_ENV_FILTER_DIRECTIVES: [&str; 9] = [ "hyper::proto::h1=off", "hickory_resolver=off", "hickory_proto=off", "discv5=off", "jsonrpsee-server=off", + "opentelemetry-otlp=warn", + "opentelemetry_sdk=warn", + "opentelemetry-http=warn", + "hyper_util::client::legacy::pool=off", ]; /// Manages the collection of layers for a tracing subscriber. /// /// `Layers` acts as a container for different logging layers such as stdout, file, or journald. /// Each layer can be configured separately and then combined into a tracing subscriber. -pub(crate) struct Layers { +#[derive(Default)] +pub struct Layers { inner: Vec>, } +impl fmt::Debug for Layers { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Layers").field("layers_count", &self.inner.len()).finish() + } +} + impl Layers { /// Creates a new `Layers` instance. - pub(crate) fn new() -> Self { - Self { inner: vec![] } + pub fn new() -> Self { + Self::default() + } + + /// Adds a layer to the collection of layers. + pub fn add_layer(&mut self, layer: L) + where + L: Layer + Send + Sync, + { + self.inner.push(layer.boxed()); } /// Consumes the `Layers` instance, returning the inner vector of layers. @@ -55,8 +80,8 @@ impl Layers { /// An `eyre::Result<()>` indicating the success or failure of the operation. pub(crate) fn journald(&mut self, filter: &str) -> eyre::Result<()> { let journald_filter = build_env_filter(None, filter)?; - let layer = tracing_journald::layer()?.with_filter(journald_filter).boxed(); - self.inner.push(layer); + let layer = tracing_journald::layer()?.with_filter(journald_filter); + self.add_layer(layer); Ok(()) } @@ -82,7 +107,7 @@ impl Layers { ) -> eyre::Result<()> { let filter = build_env_filter(Some(default_directive), filters)?; let layer = format.apply(filter, color, None); - self.inner.push(layer.boxed()); + self.add_layer(layer); Ok(()) } @@ -104,9 +129,29 @@ impl Layers { let (writer, guard) = file_info.create_log_writer(); let file_filter = build_env_filter(None, filter)?; let layer = format.apply(file_filter, None, Some(writer)); - self.inner.push(layer); + self.add_layer(layer); Ok(guard) } + + /// Add OTLP spans layer to the layer collection + #[cfg(feature = "otlp")] + pub fn with_span_layer( + &mut self, + service_name: String, + endpoint_exporter: Url, + filter: EnvFilter, + otlp_protocol: OtlpProtocol, + ) -> eyre::Result<()> { + // Create the span provider + + let span_layer = span_layer(service_name, &endpoint_exporter, otlp_protocol) + .map_err(|e| eyre::eyre!("Failed to build OTLP span exporter {}", e))? + .with_filter(filter); + + self.add_layer(span_layer); + + Ok(()) + } } /// Holds configuration information for file logging. @@ -122,8 +167,13 @@ pub struct FileInfo { impl FileInfo { /// Creates a new `FileInfo` instance. - pub fn new(dir: PathBuf, max_size_bytes: u64, max_files: usize) -> Self { - Self { dir, file_name: RETH_LOG_FILE_NAME.to_string(), max_size_bytes, max_files } + pub const fn new( + dir: PathBuf, + file_name: String, + max_size_bytes: u64, + max_files: usize, + ) -> Self { + Self { dir, file_name, max_size_bytes, max_files } } /// Creates the log directory if it doesn't exist. diff --git a/crates/tracing/src/lib.rs b/crates/tracing/src/lib.rs index 6c855e50d27..7b06398e8c7 100644 --- a/crates/tracing/src/lib.rs +++ b/crates/tracing/src/lib.rs @@ -41,7 +41,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] // Re-export tracing crates pub use tracing; @@ -50,14 +50,13 @@ pub use tracing_subscriber; // Re-export our types pub use formatter::LogFormat; -pub use layers::{FileInfo, FileWorkerGuard}; +pub use layers::{FileInfo, FileWorkerGuard, Layers}; pub use test_tracer::TestTracer; mod formatter; mod layers; mod test_tracer; -use crate::layers::Layers; use tracing::level_filters::LevelFilter; use tracing_appender::non_blocking::WorkerGuard; use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; @@ -171,12 +170,29 @@ impl Default for LayerInfo { /// in an application. Implementations of this trait can specify different logging setups, /// such as standard output logging, file logging, journald logging, or custom logging /// configurations tailored for specific environments (like testing). -pub trait Tracer { +pub trait Tracer: Sized { /// Initialize the logging configuration. - /// # Returns - /// An `eyre::Result` which is `Ok` with an optional `WorkerGuard` if a file layer is used, - /// or an `Err` in case of an error during initialization. - fn init(self) -> eyre::Result>; + /// + /// By default, this method creates a new `Layers` instance and delegates to `init_with_layers`. + /// + /// # Returns + /// An `eyre::Result` which is `Ok` with an optional `WorkerGuard` if a file layer is used, + /// or an `Err` in case of an error during initialization. + fn init(self) -> eyre::Result> { + self.init_with_layers(Layers::new()) + } + /// Initialize the logging configuration with additional custom layers. + /// + /// This method allows for more customized setup by accepting pre-configured + /// `Layers` which can be further customized before initialization. + /// + /// # Arguments + /// * `layers` - Pre-configured `Layers` instance to use for initialization + /// + /// # Returns + /// An `eyre::Result` which is `Ok` with an optional `WorkerGuard` if a file layer is used, + /// or an `Err` in case of an error during initialization. + fn init_with_layers(self, layers: Layers) -> eyre::Result>; } impl Tracer for RethTracer { @@ -190,9 +206,7 @@ impl Tracer for RethTracer { /// # Returns /// An `eyre::Result` which is `Ok` with an optional `WorkerGuard` if a file layer is used, /// or an `Err` in case of an error during initialization. - fn init(self) -> eyre::Result> { - let mut layers = Layers::new(); - + fn init_with_layers(self, mut layers: Layers) -> eyre::Result> { layers.stdout( self.stdout.format, self.stdout.default_directive.parse()?, diff --git a/crates/tracing/src/test_tracer.rs b/crates/tracing/src/test_tracer.rs index 532ad5243de..2cdf007dc89 100644 --- a/crates/tracing/src/test_tracer.rs +++ b/crates/tracing/src/test_tracer.rs @@ -1,7 +1,7 @@ use tracing_appender::non_blocking::WorkerGuard; use tracing_subscriber::EnvFilter; -use crate::Tracer; +use crate::{Layers, Tracer}; /// Initializes a tracing subscriber for tests. /// @@ -15,7 +15,7 @@ use crate::Tracer; pub struct TestTracer; impl Tracer for TestTracer { - fn init(self) -> eyre::Result> { + fn init_with_layers(self, _layers: Layers) -> eyre::Result> { let _ = tracing_subscriber::fmt() .with_env_filter(EnvFilter::from_default_env()) .with_writer(std::io::stderr) diff --git a/crates/transaction-pool/Cargo.toml b/crates/transaction-pool/Cargo.toml index 4b24a347e5e..02030719840 100644 --- a/crates/transaction-pool/Cargo.toml +++ b/crates/transaction-pool/Cargo.toml @@ -34,6 +34,7 @@ alloy-consensus = { workspace = true, features = ["kzg"] } # async/futures futures-util.workspace = true parking_lot.workspace = true +pin-project.workspace = true tokio = { workspace = true, features = ["sync"] } tokio-stream.workspace = true @@ -47,7 +48,8 @@ thiserror.workspace = true tracing.workspace = true rustc-hash.workspace = true schnellru.workspace = true -serde = { workspace = true, features = ["derive", "rc"], optional = true } +serde = { workspace = true, features = ["derive", "rc"] } +serde_json.workspace = true bitflags.workspace = true auto_impl.workspace = true smallvec.workspace = true @@ -71,10 +73,10 @@ assert_matches.workspace = true tempfile.workspace = true serde_json.workspace = true tokio = { workspace = true, features = ["rt-multi-thread"] } +futures.workspace = true [features] serde = [ - "dep:serde", "reth-execution-types/serde", "reth-eth-wire-types/serde", "alloy-consensus/serde", @@ -88,6 +90,8 @@ serde = [ "revm-primitives/serde", "reth-primitives-traits/serde", "reth-ethereum-primitives/serde", + "reth-chain-state/serde", + "reth-storage-api/serde", ] test-utils = [ "rand", @@ -130,3 +134,13 @@ harness = false name = "priority" required-features = ["arbitrary"] harness = false + +[[bench]] +name = "insertion" +required-features = ["test-utils", "arbitrary"] +harness = false + +[[bench]] +name = "canonical_state_change" +required-features = ["test-utils", "arbitrary"] +harness = false diff --git a/crates/transaction-pool/benches/canonical_state_change.rs b/crates/transaction-pool/benches/canonical_state_change.rs new file mode 100644 index 00000000000..7f2d5b91f56 --- /dev/null +++ b/crates/transaction-pool/benches/canonical_state_change.rs @@ -0,0 +1,159 @@ +#![allow(missing_docs)] +use alloy_consensus::Transaction; +use alloy_primitives::{Address, B256, U256}; +use criterion::{criterion_group, criterion_main, BatchSize, Criterion}; +use proptest::{prelude::*, strategy::ValueTree, test_runner::TestRunner}; +use rand::prelude::SliceRandom; +use reth_ethereum_primitives::{Block, BlockBody}; +use reth_execution_types::ChangedAccount; +use reth_primitives_traits::{Header, SealedBlock}; +use reth_transaction_pool::{ + test_utils::{MockTransaction, TestPoolBuilder}, + BlockInfo, CanonicalStateUpdate, PoolConfig, PoolTransaction, PoolUpdateKind, SubPoolLimit, + TransactionOrigin, TransactionPool, TransactionPoolExt, +}; +use std::{collections::HashMap, time::Duration}; +/// Generates a set of transactions for multiple senders +fn generate_transactions(num_senders: usize, txs_per_sender: usize) -> Vec { + let mut runner = TestRunner::deterministic(); + let mut txs = Vec::new(); + + for sender_idx in 0..num_senders { + // Create a unique sender address + let sender_bytes = sender_idx.to_be_bytes(); + let addr_slice = [0u8; 12].into_iter().chain(sender_bytes.into_iter()).collect::>(); + let sender = Address::from_slice(&addr_slice); + + // Generate transactions for this sender + for nonce in 0..txs_per_sender { + let mut tx = any::().new_tree(&mut runner).unwrap().current(); + tx.set_sender(sender); + tx.set_nonce(nonce as u64); + + // Ensure it's not a legacy transaction + if tx.is_legacy() || tx.is_eip2930() { + tx = MockTransaction::eip1559(); + tx.set_priority_fee(any::().new_tree(&mut runner).unwrap().current()); + tx.set_max_fee(any::().new_tree(&mut runner).unwrap().current()); + tx.set_sender(sender); + tx.set_nonce(nonce as u64); + } + + txs.push(tx); + } + } + + txs +} + +/// Fill the pool with transactions +async fn fill_pool(pool: &TestPoolBuilder, txs: Vec) -> HashMap { + let mut sender_nonces = HashMap::new(); + + // Add transactions one by one + for tx in txs { + let sender = tx.sender(); + let nonce = tx.nonce(); + + // Track the highest nonce for each sender + sender_nonces.insert(sender, nonce.max(sender_nonces.get(&sender).copied().unwrap_or(0))); + + // Add transaction to the pool + let _ = pool.add_transaction(TransactionOrigin::External, tx).await; + } + + sender_nonces +} + +fn canonical_state_change_bench(c: &mut Criterion) { + let mut group = c.benchmark_group("Transaction Pool Canonical State Change"); + group.measurement_time(Duration::from_secs(10)); + let rt = tokio::runtime::Runtime::new().unwrap(); + // Test different pool sizes + for num_senders in [500, 1000, 2000] { + for txs_per_sender in [1, 5, 10] { + let total_txs = num_senders * txs_per_sender; + + let group_id = format!( + "txpool | canonical_state_change | senders: {num_senders} | txs_per_sender: {txs_per_sender} | total: {total_txs}", + ); + + // Create the update + // Create a mock block - using default Ethereum block + let header = Header::default(); + let body = BlockBody::default(); + let block = Block { header, body }; + let sealed_block = SealedBlock::seal_slow(block); + + let txs = generate_transactions(num_senders, txs_per_sender); + let pool = TestPoolBuilder::default().with_config(PoolConfig { + pending_limit: SubPoolLimit::max(), + basefee_limit: SubPoolLimit::max(), + queued_limit: SubPoolLimit::max(), + blob_limit: SubPoolLimit::max(), + max_account_slots: 50, + ..Default::default() + }); + struct Input { + sealed_block: SealedBlock, + pool: TestPoolBuilder, + } + group.bench_with_input(group_id, &Input { sealed_block, pool }, |b, input| { + b.iter_batched( + || { + // Setup phase - create pool and transactions + let sealed_block = &input.sealed_block; + let pool = &input.pool; + let senders = pool.unique_senders(); + for sender in senders { + pool.remove_transactions_by_sender(sender); + } + // Set initial block info + pool.set_block_info(BlockInfo { + last_seen_block_number: 0, + last_seen_block_hash: B256::ZERO, + pending_basefee: 1_000_000_000, + pending_blob_fee: Some(1_000_000), + block_gas_limit: 30_000_000, + }); + let sender_nonces = rt.block_on(fill_pool(pool, txs.clone())); + let mut changed_accounts: Vec = sender_nonces + .into_iter() + .map(|(address, nonce)| ChangedAccount { + address, + nonce: nonce + 1, // Increment nonce as if transactions were mined + balance: U256::from(9_000_000_000_000_000u64), // Decrease balance + }) + .collect(); + changed_accounts.shuffle(&mut rand::rng()); + let changed_accounts = changed_accounts.drain(..100).collect(); + let update = CanonicalStateUpdate { + new_tip: sealed_block, + pending_block_base_fee: 1_000_000_000, // 1 gwei + pending_block_blob_fee: Some(1_000_000), // 0.001 gwei + changed_accounts, + mined_transactions: vec![], // No transactions mined in this benchmark + update_kind: PoolUpdateKind::Commit, + }; + + (pool, update) + }, + |(pool, update)| { + // The actual operation being benchmarked + pool.on_canonical_state_change(update); + }, + BatchSize::LargeInput, + ); + }); + } + } + + group.finish(); +} + +criterion_group! { + name = canonical_state_change; + config = Criterion::default(); + targets = canonical_state_change_bench +} +criterion_main!(canonical_state_change); diff --git a/crates/transaction-pool/benches/insertion.rs b/crates/transaction-pool/benches/insertion.rs new file mode 100644 index 00000000000..dc90d47366f --- /dev/null +++ b/crates/transaction-pool/benches/insertion.rs @@ -0,0 +1,128 @@ +#![allow(missing_docs)] +use alloy_primitives::Address; +use criterion::{criterion_group, criterion_main, Criterion}; +use proptest::{prelude::*, strategy::ValueTree, test_runner::TestRunner}; +use reth_transaction_pool::{ + batcher::{BatchTxProcessor, BatchTxRequest}, + test_utils::{testing_pool, MockTransaction}, + TransactionOrigin, TransactionPool, +}; +use tokio::sync::oneshot; + +/// Generates a set of transactions for multiple senders +fn generate_transactions(num_senders: usize, txs_per_sender: usize) -> Vec { + let mut runner = TestRunner::deterministic(); + let mut txs = Vec::new(); + + for sender_idx in 0..num_senders { + // Create a unique sender address + let sender_bytes = sender_idx.to_be_bytes(); + let addr_slice = [0u8; 12].into_iter().chain(sender_bytes.into_iter()).collect::>(); + let sender = Address::from_slice(&addr_slice); + + // Generate transactions for this sender + for nonce in 0..txs_per_sender { + let mut tx = any::().new_tree(&mut runner).unwrap().current(); + tx.set_sender(sender); + tx.set_nonce(nonce as u64); + + // Ensure it's not a legacy transaction + if tx.is_legacy() || tx.is_eip2930() { + tx = MockTransaction::eip1559(); + tx.set_priority_fee(any::().new_tree(&mut runner).unwrap().current()); + tx.set_max_fee(any::().new_tree(&mut runner).unwrap().current()); + tx.set_sender(sender); + tx.set_nonce(nonce as u64); + } + + txs.push(tx); + } + } + + txs +} + +/// Benchmark individual transaction insertion +fn txpool_insertion(c: &mut Criterion) { + let mut group = c.benchmark_group("Txpool insertion"); + let scenarios = [(1000, 100), (5000, 500), (10000, 1000), (20000, 2000)]; + + for (tx_count, sender_count) in scenarios { + let group_id = format!("txs: {tx_count} | senders: {sender_count}"); + + group.bench_function(group_id, |b| { + b.iter_with_setup( + || { + let rt = tokio::runtime::Runtime::new().unwrap(); + let pool = testing_pool(); + let txs = generate_transactions(tx_count, sender_count); + (rt, pool, txs) + }, + |(rt, pool, txs)| { + rt.block_on(async { + for tx in &txs { + let _ = + pool.add_transaction(TransactionOrigin::Local, tx.clone()).await; + } + }); + }, + ); + }); + } + + group.finish(); +} + +/// Benchmark batch transaction insertion +fn txpool_batch_insertion(c: &mut Criterion) { + let mut group = c.benchmark_group("Txpool batch insertion"); + let scenarios = [(1000, 100), (5000, 500), (10000, 1000), (20000, 2000)]; + + for (tx_count, sender_count) in scenarios { + let group_id = format!("txs: {tx_count} | senders: {sender_count}"); + + group.bench_function(group_id, |b| { + b.iter_with_setup( + || { + let rt = tokio::runtime::Runtime::new().unwrap(); + let pool = testing_pool(); + let txs = generate_transactions(tx_count, sender_count); + let (processor, request_tx) = BatchTxProcessor::new(pool, tx_count); + let processor_handle = rt.spawn(processor); + + let mut batch_requests = Vec::with_capacity(tx_count); + let mut response_futures = Vec::with_capacity(tx_count); + for tx in txs { + let (response_tx, response_rx) = oneshot::channel(); + let request = BatchTxRequest::new(tx, response_tx); + batch_requests.push(request); + response_futures.push(response_rx); + } + + (rt, request_tx, processor_handle, batch_requests, response_futures) + }, + |(rt, request_tx, _processor_handle, batch_requests, response_futures)| { + rt.block_on(async { + // Send all transactions + for request in batch_requests { + request_tx.send(request).unwrap(); + } + + for response_rx in response_futures { + let _res = response_rx.await.unwrap(); + } + }); + }, + ); + }); + } + + group.finish(); +} + +criterion_group! { + name = insertion; + config = Criterion::default(); + targets = txpool_insertion, txpool_batch_insertion +} +criterion_main!(insertion); diff --git a/crates/transaction-pool/docs/mermaid/txpool.mmd b/crates/transaction-pool/docs/mermaid/txpool.mmd index 94f3abda3e6..e183d8f3c91 100644 --- a/crates/transaction-pool/docs/mermaid/txpool.mmd +++ b/crates/transaction-pool/docs/mermaid/txpool.mmd @@ -16,7 +16,7 @@ graph TB A[Incoming Tx] --> B[Validation] -->|insert| pool pool --> |if ready + blobfee too low| B4 pool --> |if ready| B1 - pool --> |if ready + basfee too low| B2 + pool --> |if ready + basefee too low| B2 pool --> |nonce gap or lack of funds| B3 pool --> |update| pool B1 --> |best| production diff --git a/crates/transaction-pool/src/batcher.rs b/crates/transaction-pool/src/batcher.rs new file mode 100644 index 00000000000..75280e68b3c --- /dev/null +++ b/crates/transaction-pool/src/batcher.rs @@ -0,0 +1,247 @@ +//! Transaction batching for `Pool` insertion for high-throughput scenarios +//! +//! This module provides transaction batching logic to reduce lock contention when processing +//! many concurrent transaction pool insertions. + +use crate::{ + error::PoolError, AddedTransactionOutcome, PoolTransaction, TransactionOrigin, TransactionPool, +}; +use pin_project::pin_project; +use std::{ + future::Future, + pin::Pin, + task::{ready, Context, Poll}, +}; +use tokio::sync::{mpsc, oneshot}; + +/// A single batch transaction request +/// All transactions processed through the batcher are considered local +/// transactions (`TransactionOrigin::Local`) when inserted into the pool. +#[derive(Debug)] +pub struct BatchTxRequest { + /// Tx to be inserted in to the pool + pool_tx: T, + /// Channel to send result back to caller + response_tx: oneshot::Sender>, +} + +impl BatchTxRequest +where + T: PoolTransaction, +{ + /// Create a new batch transaction request + pub const fn new( + pool_tx: T, + response_tx: oneshot::Sender>, + ) -> Self { + Self { pool_tx, response_tx } + } +} + +/// Transaction batch processor that handles batch processing +#[pin_project] +#[derive(Debug)] +pub struct BatchTxProcessor { + pool: Pool, + max_batch_size: usize, + buf: Vec>, + #[pin] + request_rx: mpsc::UnboundedReceiver>, +} + +impl BatchTxProcessor +where + Pool: TransactionPool + 'static, +{ + /// Create a new `BatchTxProcessor` + pub fn new( + pool: Pool, + max_batch_size: usize, + ) -> (Self, mpsc::UnboundedSender>) { + let (request_tx, request_rx) = mpsc::unbounded_channel(); + + let processor = Self { pool, max_batch_size, buf: Vec::with_capacity(1), request_rx }; + + (processor, request_tx) + } + + async fn process_request(pool: &Pool, req: BatchTxRequest) { + let BatchTxRequest { pool_tx, response_tx } = req; + let pool_result = pool.add_transaction(TransactionOrigin::Local, pool_tx).await; + let _ = response_tx.send(pool_result); + } + + /// Process a batch of transaction requests, grouped by origin + async fn process_batch(pool: &Pool, mut batch: Vec>) { + if batch.len() == 1 { + Self::process_request(pool, batch.remove(0)).await; + return + } + + let (pool_transactions, response_tx): (Vec<_>, Vec<_>) = + batch.into_iter().map(|req| (req.pool_tx, req.response_tx)).unzip(); + + let pool_results = pool.add_transactions(TransactionOrigin::Local, pool_transactions).await; + + for (response_tx, pool_result) in response_tx.into_iter().zip(pool_results) { + let _ = response_tx.send(pool_result); + } + } +} + +impl Future for BatchTxProcessor +where + Pool: TransactionPool + 'static, +{ + type Output = (); + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let mut this = self.project(); + + loop { + // Drain all available requests from the receiver + ready!(this.request_rx.poll_recv_many(cx, this.buf, *this.max_batch_size)); + + if !this.buf.is_empty() { + let batch = std::mem::take(this.buf); + let pool = this.pool.clone(); + tokio::spawn(async move { + Self::process_batch(&pool, batch).await; + }); + this.buf.reserve(1); + + continue; + } + + // No requests available, return Pending to wait for more + return Poll::Pending; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::{testing_pool, MockTransaction}; + use futures::stream::{FuturesUnordered, StreamExt}; + use std::time::Duration; + use tokio::time::timeout; + + #[tokio::test] + async fn test_process_batch() { + let pool = testing_pool(); + + let mut batch_requests = Vec::new(); + let mut responses = Vec::new(); + + for i in 0..100 { + let tx = MockTransaction::legacy().with_nonce(i).with_gas_price(100); + let (response_tx, response_rx) = tokio::sync::oneshot::channel(); + + batch_requests.push(BatchTxRequest::new(tx, response_tx)); + responses.push(response_rx); + } + + BatchTxProcessor::process_batch(&pool, batch_requests).await; + + for response_rx in responses { + let result = timeout(Duration::from_millis(5), response_rx) + .await + .expect("Timeout waiting for response") + .expect("Response channel was closed unexpectedly"); + assert!(result.is_ok()); + } + } + + #[tokio::test] + async fn test_batch_processor() { + let pool = testing_pool(); + let (processor, request_tx) = BatchTxProcessor::new(pool.clone(), 1000); + + // Spawn the processor + let handle = tokio::spawn(processor); + + let mut responses = Vec::new(); + + for i in 0..50 { + let tx = MockTransaction::legacy().with_nonce(i).with_gas_price(100); + let (response_tx, response_rx) = tokio::sync::oneshot::channel(); + + request_tx.send(BatchTxRequest::new(tx, response_tx)).expect("Could not send batch tx"); + responses.push(response_rx); + } + + tokio::time::sleep(Duration::from_millis(10)).await; + + for rx in responses { + let result = timeout(Duration::from_millis(10), rx) + .await + .expect("Timeout waiting for response") + .expect("Response channel was closed unexpectedly"); + assert!(result.is_ok()); + } + + drop(request_tx); + handle.abort(); + } + + #[tokio::test] + async fn test_add_transaction() { + let pool = testing_pool(); + let (processor, request_tx) = BatchTxProcessor::new(pool.clone(), 1000); + + // Spawn the processor + let handle = tokio::spawn(processor); + + let mut results = Vec::new(); + for i in 0..10 { + let tx = MockTransaction::legacy().with_nonce(i).with_gas_price(100); + let (response_tx, response_rx) = tokio::sync::oneshot::channel(); + let request = BatchTxRequest::new(tx, response_tx); + request_tx.send(request).expect("Could not send batch tx"); + results.push(response_rx); + } + + for res in results { + let result = timeout(Duration::from_millis(10), res) + .await + .expect("Timeout waiting for transaction result"); + assert!(result.is_ok()); + } + + handle.abort(); + } + + #[tokio::test] + async fn test_max_batch_size() { + let pool = testing_pool(); + let max_batch_size = 10; + let (processor, request_tx) = BatchTxProcessor::new(pool.clone(), max_batch_size); + + // Spawn batch processor with threshold + let handle = tokio::spawn(processor); + + let mut futures = FuturesUnordered::new(); + for i in 0..max_batch_size { + let tx = MockTransaction::legacy().with_nonce(i as u64).with_gas_price(100); + let (response_tx, response_rx) = tokio::sync::oneshot::channel(); + let request = BatchTxRequest::new(tx, response_tx); + let request_tx_clone = request_tx.clone(); + + let tx_fut = async move { + request_tx_clone.send(request).expect("Could not send batch tx"); + response_rx.await.expect("Could not receive batch response") + }; + futures.push(tx_fut); + } + + while let Some(result) = timeout(Duration::from_millis(5), futures.next()) + .await + .expect("Timeout waiting for transaction result") + { + assert!(result.is_ok()); + } + + handle.abort(); + } +} diff --git a/crates/transaction-pool/src/blobstore/converter.rs b/crates/transaction-pool/src/blobstore/converter.rs new file mode 100644 index 00000000000..3f6abc56bff --- /dev/null +++ b/crates/transaction-pool/src/blobstore/converter.rs @@ -0,0 +1,30 @@ +use alloy_consensus::{BlobTransactionSidecar, EnvKzgSettings}; +use alloy_eips::eip7594::BlobTransactionSidecarEip7594; +use tokio::sync::Semaphore; + +// We allow up to 5 concurrent conversions to avoid excessive memory usage. +static SEMAPHORE: Semaphore = Semaphore::const_new(5); + +/// A simple semaphore-based blob sidecar converter. +#[derive(Debug, Clone, Default)] +#[non_exhaustive] +pub struct BlobSidecarConverter; + +impl BlobSidecarConverter { + /// Creates a new blob sidecar converter. + pub const fn new() -> Self { + Self + } + + /// Converts the blob sidecar to the EIP-7594 format. + pub async fn convert( + &self, + sidecar: BlobTransactionSidecar, + ) -> Option { + let _permit = SEMAPHORE.acquire().await.ok()?; + tokio::task::spawn_blocking(move || sidecar.try_into_7594(EnvKzgSettings::Default.get())) + .await + .ok()? + .ok() + } +} diff --git a/crates/transaction-pool/src/blobstore/disk.rs b/crates/transaction-pool/src/blobstore/disk.rs index 293eb5da563..b883345aac6 100644 --- a/crates/transaction-pool/src/blobstore/disk.rs +++ b/crates/transaction-pool/src/blobstore/disk.rs @@ -1,16 +1,28 @@ //! A simple diskstore for blobs use crate::blobstore::{BlobStore, BlobStoreCleanupStat, BlobStoreError, BlobStoreSize}; -use alloy_eips::eip4844::{BlobAndProofV1, BlobTransactionSidecar}; +use alloy_eips::{ + eip4844::{BlobAndProofV1, BlobAndProofV2}, + eip7594::BlobTransactionSidecarVariant, + eip7840::BlobParams, + merge::EPOCH_SLOTS, +}; use alloy_primitives::{TxHash, B256}; use parking_lot::{Mutex, RwLock}; use schnellru::{ByLength, LruMap}; use std::{collections::HashSet, fmt, fs, io, path::PathBuf, sync::Arc}; use tracing::{debug, trace}; -/// How many [`BlobTransactionSidecar`] to cache in memory. +/// How many [`BlobTransactionSidecarVariant`] to cache in memory. pub const DEFAULT_MAX_CACHED_BLOBS: u32 = 100; +/// A cache size heuristic based on the highest blob params +/// +/// This uses the max blobs per tx and max blobs per block over 16 epochs: `21 * 6 * 512 = 64512` +/// This should be ~4MB +const VERSIONED_HASH_TO_TX_HASH_CACHE_SIZE: u64 = + BlobParams::bpo2().max_blobs_per_tx * BlobParams::bpo2().max_blob_count * EPOCH_SLOTS * 16; + /// A blob store that stores blob data on disk. /// /// The type uses deferred deletion, meaning that blobs are not immediately deleted from disk, but @@ -50,11 +62,14 @@ impl DiskFileBlobStore { } impl BlobStore for DiskFileBlobStore { - fn insert(&self, tx: B256, data: BlobTransactionSidecar) -> Result<(), BlobStoreError> { + fn insert(&self, tx: B256, data: BlobTransactionSidecarVariant) -> Result<(), BlobStoreError> { self.inner.insert_one(tx, data) } - fn insert_all(&self, txs: Vec<(B256, BlobTransactionSidecar)>) -> Result<(), BlobStoreError> { + fn insert_all( + &self, + txs: Vec<(B256, BlobTransactionSidecarVariant)>, + ) -> Result<(), BlobStoreError> { if txs.is_empty() { return Ok(()) } @@ -99,7 +114,7 @@ impl BlobStore for DiskFileBlobStore { stat } - fn get(&self, tx: B256) -> Result>, BlobStoreError> { + fn get(&self, tx: B256) -> Result>, BlobStoreError> { self.inner.get_one(tx) } @@ -110,7 +125,7 @@ impl BlobStore for DiskFileBlobStore { fn get_all( &self, txs: Vec, - ) -> Result)>, BlobStoreError> { + ) -> Result)>, BlobStoreError> { if txs.is_empty() { return Ok(Vec::new()) } @@ -120,14 +135,14 @@ impl BlobStore for DiskFileBlobStore { fn get_exact( &self, txs: Vec, - ) -> Result>, BlobStoreError> { + ) -> Result>, BlobStoreError> { if txs.is_empty() { return Ok(Vec::new()) } self.inner.get_exact(txs) } - fn get_by_versioned_hashes( + fn get_by_versioned_hashes_v1( &self, versioned_hashes: &[B256], ) -> Result>, BlobStoreError> { @@ -136,8 +151,12 @@ impl BlobStore for DiskFileBlobStore { // first scan all cached full sidecars for (_tx_hash, blob_sidecar) in self.inner.blob_cache.lock().iter() { - for (hash_idx, match_result) in blob_sidecar.match_versioned_hashes(versioned_hashes) { - result[hash_idx] = Some(match_result); + if let Some(blob_sidecar) = blob_sidecar.as_eip4844() { + for (hash_idx, match_result) in + blob_sidecar.match_versioned_hashes(versioned_hashes) + { + result[hash_idx] = Some(match_result); + } } // return early if all blobs are found. @@ -167,17 +186,84 @@ impl BlobStore for DiskFileBlobStore { if !missing_tx_hashes.is_empty() { let blobs_from_disk = self.inner.read_many_decoded(missing_tx_hashes); for (_, blob_sidecar) in blobs_from_disk { + if let Some(blob_sidecar) = blob_sidecar.as_eip4844() { + for (hash_idx, match_result) in + blob_sidecar.match_versioned_hashes(versioned_hashes) + { + if result[hash_idx].is_none() { + result[hash_idx] = Some(match_result); + } + } + } + } + } + + Ok(result) + } + + fn get_by_versioned_hashes_v2( + &self, + versioned_hashes: &[B256], + ) -> Result>, BlobStoreError> { + // we must return the blobs in order but we don't necessarily find them in the requested + // order + let mut result = vec![None; versioned_hashes.len()]; + + // first scan all cached full sidecars + for (_tx_hash, blob_sidecar) in self.inner.blob_cache.lock().iter() { + if let Some(blob_sidecar) = blob_sidecar.as_eip7594() { for (hash_idx, match_result) in blob_sidecar.match_versioned_hashes(versioned_hashes) { - if result[hash_idx].is_none() { - result[hash_idx] = Some(match_result); + result[hash_idx] = Some(match_result); + } + } + + // return early if all blobs are found. + if result.iter().all(|blob| blob.is_some()) { + // got all blobs, can return early + return Ok(Some(result.into_iter().map(Option::unwrap).collect())) + } + } + + // not all versioned hashes were found, try to look up a matching tx + let mut missing_tx_hashes = Vec::new(); + + { + let mut versioned_to_txhashes = self.inner.versioned_hashes_to_txhash.lock(); + for (idx, _) in + result.iter().enumerate().filter(|(_, blob_and_proof)| blob_and_proof.is_none()) + { + // this is safe because the result vec has the same len + let versioned_hash = versioned_hashes[idx]; + if let Some(tx_hash) = versioned_to_txhashes.get(&versioned_hash).copied() { + missing_tx_hashes.push(tx_hash); + } + } + } + + // if we have missing blobs, try to read them from disk and try again + if !missing_tx_hashes.is_empty() { + let blobs_from_disk = self.inner.read_many_decoded(missing_tx_hashes); + for (_, blob_sidecar) in blobs_from_disk { + if let Some(blob_sidecar) = blob_sidecar.as_eip7594() { + for (hash_idx, match_result) in + blob_sidecar.match_versioned_hashes(versioned_hashes) + { + if result[hash_idx].is_none() { + result[hash_idx] = Some(match_result); + } } } } } - Ok(result) + // only return the blobs if we found all requested versioned hashes + if result.iter().all(|blob| blob.is_some()) { + Ok(Some(result.into_iter().map(Option::unwrap).collect())) + } else { + Ok(None) + } } fn data_size_hint(&self) -> Option { @@ -191,7 +277,7 @@ impl BlobStore for DiskFileBlobStore { struct DiskFileBlobStoreInner { blob_dir: PathBuf, - blob_cache: Mutex, ByLength>>, + blob_cache: Mutex, ByLength>>, size_tracker: BlobStoreSize, file_lock: RwLock<()>, txs_to_delete: RwLock>, @@ -211,7 +297,9 @@ impl DiskFileBlobStoreInner { size_tracker: Default::default(), file_lock: Default::default(), txs_to_delete: Default::default(), - versioned_hashes_to_txhash: Mutex::new(LruMap::new(ByLength::new(max_length * 6))), + versioned_hashes_to_txhash: Mutex::new(LruMap::new(ByLength::new( + VERSIONED_HASH_TO_TX_HASH_CACHE_SIZE as u32, + ))), } } @@ -235,7 +323,11 @@ impl DiskFileBlobStoreInner { } /// Ensures blob is in the blob cache and written to the disk. - fn insert_one(&self, tx: B256, data: BlobTransactionSidecar) -> Result<(), BlobStoreError> { + fn insert_one( + &self, + tx: B256, + data: BlobTransactionSidecarVariant, + ) -> Result<(), BlobStoreError> { let mut buf = Vec::with_capacity(data.rlp_encoded_fields_length()); data.rlp_encode_fields(&mut buf); @@ -244,7 +336,7 @@ impl DiskFileBlobStoreInner { let mut map = self.versioned_hashes_to_txhash.lock(); data.versioned_hashes().for_each(|hash| { map.insert(hash, tx); - }) + }); } self.blob_cache.lock().insert(tx, Arc::new(data)); @@ -257,7 +349,10 @@ impl DiskFileBlobStoreInner { } /// Ensures blobs are in the blob cache and written to the disk. - fn insert_many(&self, txs: Vec<(B256, BlobTransactionSidecar)>) -> Result<(), BlobStoreError> { + fn insert_many( + &self, + txs: Vec<(B256, BlobTransactionSidecarVariant)>, + ) -> Result<(), BlobStoreError> { let raw = txs .iter() .map(|(tx, data)| { @@ -273,7 +368,7 @@ impl DiskFileBlobStoreInner { for (tx, data) in &txs { data.versioned_hashes().for_each(|hash| { map.insert(hash, *tx); - }) + }); } } @@ -333,14 +428,16 @@ impl DiskFileBlobStoreInner { } /// Retrieves the blob for the given transaction hash from the blob cache or disk. - fn get_one(&self, tx: B256) -> Result>, BlobStoreError> { + fn get_one( + &self, + tx: B256, + ) -> Result>, BlobStoreError> { if let Some(blob) = self.blob_cache.lock().get(&tx) { return Ok(Some(blob.clone())) } - let blob = self.read_one(tx)?; - if let Some(blob) = &blob { - let blob_arc = Arc::new(blob.clone()); + if let Some(blob) = self.read_one(tx)? { + let blob_arc = Arc::new(blob); self.blob_cache.lock().insert(tx, blob_arc.clone()); return Ok(Some(blob_arc)) } @@ -356,7 +453,7 @@ impl DiskFileBlobStoreInner { /// Retrieves the blob data for the given transaction hash. #[inline] - fn read_one(&self, tx: B256) -> Result, BlobStoreError> { + fn read_one(&self, tx: B256) -> Result, BlobStoreError> { let path = self.blob_disk_file(tx); let data = { let _lock = self.file_lock.read(); @@ -370,7 +467,7 @@ impl DiskFileBlobStoreInner { } } }; - BlobTransactionSidecar::rlp_decode_fields(&mut data.as_slice()) + BlobTransactionSidecarVariant::rlp_decode_fields(&mut data.as_slice()) .map(Some) .map_err(BlobStoreError::DecodeError) } @@ -378,11 +475,11 @@ impl DiskFileBlobStoreInner { /// Returns decoded blobs read from disk. /// /// Only returns sidecars that were found and successfully decoded. - fn read_many_decoded(&self, txs: Vec) -> Vec<(TxHash, BlobTransactionSidecar)> { + fn read_many_decoded(&self, txs: Vec) -> Vec<(TxHash, BlobTransactionSidecarVariant)> { self.read_many_raw(txs) .into_iter() .filter_map(|(tx, data)| { - BlobTransactionSidecar::rlp_decode_fields(&mut data.as_slice()) + BlobTransactionSidecarVariant::rlp_decode_fields(&mut data.as_slice()) .map(|sidecar| (tx, sidecar)) .ok() }) @@ -391,7 +488,7 @@ impl DiskFileBlobStoreInner { /// Retrieves the raw blob data for the given transaction hashes. /// - /// Only returns the blobs that were found on file. + /// Only returns the blobs that were found in file. #[inline] fn read_many_raw(&self, txs: Vec) -> Vec<(TxHash, Vec)> { let mut res = Vec::with_capacity(txs.len()); @@ -435,7 +532,7 @@ impl DiskFileBlobStoreInner { fn get_all( &self, txs: Vec, - ) -> Result)>, BlobStoreError> { + ) -> Result)>, BlobStoreError> { let mut res = Vec::with_capacity(txs.len()); let mut cache_miss = Vec::new(); { @@ -455,11 +552,18 @@ impl DiskFileBlobStoreInner { if from_disk.is_empty() { return Ok(res) } + let from_disk = from_disk + .into_iter() + .map(|(tx, data)| { + let data = Arc::new(data); + res.push((tx, data.clone())); + (tx, data) + }) + .collect::>(); + let mut cache = self.blob_cache.lock(); for (tx, data) in from_disk { - let arc = Arc::new(data.clone()); - cache.insert(tx, arc.clone()); - res.push((tx, arc.clone())); + cache.insert(tx, data); } Ok(res) @@ -472,7 +576,7 @@ impl DiskFileBlobStoreInner { fn get_exact( &self, txs: Vec, - ) -> Result>, BlobStoreError> { + ) -> Result>, BlobStoreError> { txs.into_iter() .map(|tx| self.get_one(tx)?.ok_or(BlobStoreError::MissingSidecar(tx))) .collect() @@ -551,6 +655,9 @@ pub enum OpenDiskFileBlobStore { #[cfg(test)] mod tests { + use alloy_consensus::BlobTransactionSidecar; + use alloy_eips::eip7594::BlobTransactionSidecarVariant; + use super::*; use std::sync::atomic::Ordering; @@ -560,13 +667,16 @@ mod tests { (store, dir) } - fn rng_blobs(num: usize) -> Vec<(TxHash, BlobTransactionSidecar)> { + fn rng_blobs(num: usize) -> Vec<(TxHash, BlobTransactionSidecarVariant)> { let mut rng = rand::rng(); (0..num) .map(|_| { let tx = TxHash::random_with(&mut rng); - let blob = - BlobTransactionSidecar { blobs: vec![], commitments: vec![], proofs: vec![] }; + let blob = BlobTransactionSidecarVariant::Eip4844(BlobTransactionSidecar { + blobs: vec![], + commitments: vec![], + proofs: vec![], + }); (tx, blob) }) .collect() @@ -637,11 +747,11 @@ mod tests { let result = store.get(tx).unwrap(); assert_eq!( result, - Some(Arc::new(BlobTransactionSidecar { + Some(Arc::new(BlobTransactionSidecarVariant::Eip4844(BlobTransactionSidecar { blobs: vec![], commitments: vec![], proofs: vec![] - })) + }))) ); } @@ -664,11 +774,11 @@ mod tests { let result = store.get(tx).unwrap(); assert_eq!( result, - Some(Arc::new(BlobTransactionSidecar { + Some(Arc::new(BlobTransactionSidecarVariant::Eip4844(BlobTransactionSidecar { blobs: vec![], commitments: vec![], proofs: vec![] - })) + }))) ); } } diff --git a/crates/transaction-pool/src/blobstore/mem.rs b/crates/transaction-pool/src/blobstore/mem.rs index 815b9684093..44dff1ccebb 100644 --- a/crates/transaction-pool/src/blobstore/mem.rs +++ b/crates/transaction-pool/src/blobstore/mem.rs @@ -1,5 +1,8 @@ use crate::blobstore::{BlobStore, BlobStoreCleanupStat, BlobStoreError, BlobStoreSize}; -use alloy_eips::eip4844::{BlobAndProofV1, BlobTransactionSidecar}; +use alloy_eips::{ + eip4844::{BlobAndProofV1, BlobAndProofV2}, + eip7594::BlobTransactionSidecarVariant, +}; use alloy_primitives::B256; use parking_lot::RwLock; use std::{collections::HashMap, sync::Arc}; @@ -13,7 +16,7 @@ pub struct InMemoryBlobStore { #[derive(Debug, Default)] struct InMemoryBlobStoreInner { /// Storage for all blob data. - store: RwLock>>, + store: RwLock>>, size_tracker: BlobStoreSize, } @@ -24,14 +27,17 @@ impl PartialEq for InMemoryBlobStoreInner { } impl BlobStore for InMemoryBlobStore { - fn insert(&self, tx: B256, data: BlobTransactionSidecar) -> Result<(), BlobStoreError> { + fn insert(&self, tx: B256, data: BlobTransactionSidecarVariant) -> Result<(), BlobStoreError> { let mut store = self.inner.store.write(); self.inner.size_tracker.add_size(insert_size(&mut store, tx, data)); self.inner.size_tracker.update_len(store.len()); Ok(()) } - fn insert_all(&self, txs: Vec<(B256, BlobTransactionSidecar)>) -> Result<(), BlobStoreError> { + fn insert_all( + &self, + txs: Vec<(B256, BlobTransactionSidecarVariant)>, + ) -> Result<(), BlobStoreError> { if txs.is_empty() { return Ok(()) } @@ -73,7 +79,7 @@ impl BlobStore for InMemoryBlobStore { } // Retrieves the decoded blob data for the given transaction hash. - fn get(&self, tx: B256) -> Result>, BlobStoreError> { + fn get(&self, tx: B256) -> Result>, BlobStoreError> { Ok(self.inner.store.read().get(&tx).cloned()) } @@ -84,7 +90,7 @@ impl BlobStore for InMemoryBlobStore { fn get_all( &self, txs: Vec, - ) -> Result)>, BlobStoreError> { + ) -> Result)>, BlobStoreError> { let store = self.inner.store.read(); Ok(txs.into_iter().filter_map(|tx| store.get(&tx).map(|item| (tx, item.clone()))).collect()) } @@ -92,19 +98,23 @@ impl BlobStore for InMemoryBlobStore { fn get_exact( &self, txs: Vec, - ) -> Result>, BlobStoreError> { + ) -> Result>, BlobStoreError> { let store = self.inner.store.read(); Ok(txs.into_iter().filter_map(|tx| store.get(&tx).cloned()).collect()) } - fn get_by_versioned_hashes( + fn get_by_versioned_hashes_v1( &self, versioned_hashes: &[B256], ) -> Result>, BlobStoreError> { let mut result = vec![None; versioned_hashes.len()]; for (_tx_hash, blob_sidecar) in self.inner.store.read().iter() { - for (hash_idx, match_result) in blob_sidecar.match_versioned_hashes(versioned_hashes) { - result[hash_idx] = Some(match_result); + if let Some(blob_sidecar) = blob_sidecar.as_eip4844() { + for (hash_idx, match_result) in + blob_sidecar.match_versioned_hashes(versioned_hashes) + { + result[hash_idx] = Some(match_result); + } } // Return early if all blobs are found. @@ -115,6 +125,31 @@ impl BlobStore for InMemoryBlobStore { Ok(result) } + fn get_by_versioned_hashes_v2( + &self, + versioned_hashes: &[B256], + ) -> Result>, BlobStoreError> { + let mut result = vec![None; versioned_hashes.len()]; + for (_tx_hash, blob_sidecar) in self.inner.store.read().iter() { + if let Some(blob_sidecar) = blob_sidecar.as_eip7594() { + for (hash_idx, match_result) in + blob_sidecar.match_versioned_hashes(versioned_hashes) + { + result[hash_idx] = Some(match_result); + } + } + + if result.iter().all(|blob| blob.is_some()) { + break; + } + } + if result.iter().all(|blob| blob.is_some()) { + Ok(Some(result.into_iter().map(Option::unwrap).collect())) + } else { + Ok(None) + } + } + fn data_size_hint(&self) -> Option { Some(self.inner.size_tracker.data_size()) } @@ -126,7 +161,7 @@ impl BlobStore for InMemoryBlobStore { /// Removes the given blob from the store and returns the size of the blob that was removed. #[inline] -fn remove_size(store: &mut HashMap>, tx: &B256) -> usize { +fn remove_size(store: &mut HashMap>, tx: &B256) -> usize { store.remove(tx).map(|rem| rem.size()).unwrap_or_default() } @@ -135,9 +170,9 @@ fn remove_size(store: &mut HashMap>, tx: &B256 /// We don't need to handle the size updates for replacements because transactions are unique. #[inline] fn insert_size( - store: &mut HashMap>, + store: &mut HashMap>, tx: B256, - blob: BlobTransactionSidecar, + blob: BlobTransactionSidecarVariant, ) -> usize { let add = blob.size(); store.insert(tx, Arc::new(blob)); diff --git a/crates/transaction-pool/src/blobstore/mod.rs b/crates/transaction-pool/src/blobstore/mod.rs index ae9d6adb5ba..ee7eb45af0f 100644 --- a/crates/transaction-pool/src/blobstore/mod.rs +++ b/crates/transaction-pool/src/blobstore/mod.rs @@ -1,7 +1,11 @@ //! Storage for blob data of EIP4844 transactions. -use alloy_eips::eip4844::{BlobAndProofV1, BlobTransactionSidecar}; +use alloy_eips::{ + eip4844::{BlobAndProofV1, BlobAndProofV2}, + eip7594::BlobTransactionSidecarVariant, +}; use alloy_primitives::B256; +pub use converter::BlobSidecarConverter; pub use disk::{DiskFileBlobStore, DiskFileBlobStoreConfig, OpenDiskFileBlobStore}; pub use mem::InMemoryBlobStore; pub use noop::NoopBlobStore; @@ -14,6 +18,7 @@ use std::{ }; pub use tracker::{BlobStoreCanonTracker, BlobStoreUpdates}; +mod converter; pub mod disk; mod mem; mod noop; @@ -27,10 +32,13 @@ mod tracker; /// Note: this is Clone because it is expected to be wrapped in an Arc. pub trait BlobStore: fmt::Debug + Send + Sync + 'static { /// Inserts the blob sidecar into the store - fn insert(&self, tx: B256, data: BlobTransactionSidecar) -> Result<(), BlobStoreError>; + fn insert(&self, tx: B256, data: BlobTransactionSidecarVariant) -> Result<(), BlobStoreError>; /// Inserts multiple blob sidecars into the store - fn insert_all(&self, txs: Vec<(B256, BlobTransactionSidecar)>) -> Result<(), BlobStoreError>; + fn insert_all( + &self, + txs: Vec<(B256, BlobTransactionSidecarVariant)>, + ) -> Result<(), BlobStoreError>; /// Deletes the blob sidecar from the store fn delete(&self, tx: B256) -> Result<(), BlobStoreError>; @@ -46,7 +54,7 @@ pub trait BlobStore: fmt::Debug + Send + Sync + 'static { fn cleanup(&self) -> BlobStoreCleanupStat; /// Retrieves the decoded blob data for the given transaction hash. - fn get(&self, tx: B256) -> Result>, BlobStoreError>; + fn get(&self, tx: B256) -> Result>, BlobStoreError>; /// Checks if the given transaction hash is in the blob store. fn contains(&self, tx: B256) -> Result; @@ -60,21 +68,38 @@ pub trait BlobStore: fmt::Debug + Send + Sync + 'static { fn get_all( &self, txs: Vec, - ) -> Result)>, BlobStoreError>; + ) -> Result)>, BlobStoreError>; - /// Returns the exact [`BlobTransactionSidecar`] for the given transaction hashes in the exact - /// order they were requested. + /// Returns the exact [`BlobTransactionSidecarVariant`] for the given transaction hashes in the + /// exact order they were requested. /// /// Returns an error if any of the blobs are not found in the blob store. - fn get_exact(&self, txs: Vec) - -> Result>, BlobStoreError>; + fn get_exact( + &self, + txs: Vec, + ) -> Result>, BlobStoreError>; - /// Return the [`BlobTransactionSidecar`]s for a list of blob versioned hashes. - fn get_by_versioned_hashes( + /// Return the [`BlobAndProofV1`]s for a list of blob versioned hashes. + fn get_by_versioned_hashes_v1( &self, versioned_hashes: &[B256], ) -> Result>, BlobStoreError>; + /// Return the [`BlobAndProofV2`]s for a list of blob versioned hashes. + /// Blobs and proofs are returned only if they are present for _all_ requested + /// versioned hashes. + /// + /// This differs from [`BlobStore::get_by_versioned_hashes_v1`] in that it also returns all the + /// cell proofs in [`BlobAndProofV2`] supported by the EIP-7594 blob sidecar variant. + /// + /// The response also differs from [`BlobStore::get_by_versioned_hashes_v1`] in that this + /// returns `None` if any of the requested versioned hashes are not present in the blob store: + /// e.g. where v1 would return `[A, None, C]` v2 would return `None`. See also + fn get_by_versioned_hashes_v2( + &self, + versioned_hashes: &[B256], + ) -> Result>, BlobStoreError>; + /// Data size of all transactions in the blob store. fn data_size_hint(&self) -> Option; diff --git a/crates/transaction-pool/src/blobstore/noop.rs b/crates/transaction-pool/src/blobstore/noop.rs index 943a6eeda95..bb03253ee61 100644 --- a/crates/transaction-pool/src/blobstore/noop.rs +++ b/crates/transaction-pool/src/blobstore/noop.rs @@ -1,5 +1,8 @@ use crate::blobstore::{BlobStore, BlobStoreCleanupStat, BlobStoreError}; -use alloy_eips::eip4844::{BlobAndProofV1, BlobTransactionSidecar}; +use alloy_eips::{ + eip4844::{BlobAndProofV1, BlobAndProofV2}, + eip7594::BlobTransactionSidecarVariant, +}; use alloy_primitives::B256; use std::sync::Arc; @@ -9,11 +12,18 @@ use std::sync::Arc; pub struct NoopBlobStore; impl BlobStore for NoopBlobStore { - fn insert(&self, _tx: B256, _data: BlobTransactionSidecar) -> Result<(), BlobStoreError> { + fn insert( + &self, + _tx: B256, + _data: BlobTransactionSidecarVariant, + ) -> Result<(), BlobStoreError> { Ok(()) } - fn insert_all(&self, _txs: Vec<(B256, BlobTransactionSidecar)>) -> Result<(), BlobStoreError> { + fn insert_all( + &self, + _txs: Vec<(B256, BlobTransactionSidecarVariant)>, + ) -> Result<(), BlobStoreError> { Ok(()) } @@ -29,7 +39,7 @@ impl BlobStore for NoopBlobStore { BlobStoreCleanupStat::default() } - fn get(&self, _tx: B256) -> Result>, BlobStoreError> { + fn get(&self, _tx: B256) -> Result>, BlobStoreError> { Ok(None) } @@ -40,27 +50,34 @@ impl BlobStore for NoopBlobStore { fn get_all( &self, _txs: Vec, - ) -> Result)>, BlobStoreError> { + ) -> Result)>, BlobStoreError> { Ok(vec![]) } fn get_exact( &self, txs: Vec, - ) -> Result>, BlobStoreError> { + ) -> Result>, BlobStoreError> { if txs.is_empty() { return Ok(vec![]) } Err(BlobStoreError::MissingSidecar(txs[0])) } - fn get_by_versioned_hashes( + fn get_by_versioned_hashes_v1( &self, versioned_hashes: &[B256], ) -> Result>, BlobStoreError> { Ok(vec![None; versioned_hashes.len()]) } + fn get_by_versioned_hashes_v2( + &self, + _versioned_hashes: &[B256], + ) -> Result>, BlobStoreError> { + Ok(None) + } + fn data_size_hint(&self) -> Option { Some(0) } diff --git a/crates/transaction-pool/src/config.rs b/crates/transaction-pool/src/config.rs index 5263cd18344..c6fb4ecc88b 100644 --- a/crates/transaction-pool/src/config.rs +++ b/crates/transaction-pool/src/config.rs @@ -31,6 +31,9 @@ pub const REPLACE_BLOB_PRICE_BUMP: u128 = 100; /// Default maximum new transactions for broadcasting. pub const MAX_NEW_PENDING_TXS_NOTIFICATIONS: usize = 200; +/// Default maximum allowed in flight delegated transactions per account. +pub const DEFAULT_MAX_INFLIGHT_DELEGATED_SLOTS: usize = 1; + /// Configuration options for the Transaction pool. #[derive(Debug, Clone)] pub struct PoolConfig { @@ -50,6 +53,8 @@ pub struct PoolConfig { pub price_bumps: PriceBumpConfig, /// Minimum base fee required by the protocol. pub minimal_protocol_basefee: u64, + /// Minimum priority fee required for transaction acceptance into the pool. + pub minimum_priority_fee: Option, /// The max gas limit for transactions in the pool pub gas_limit: u64, /// How to handle locally received transactions: @@ -63,9 +68,38 @@ pub struct PoolConfig { pub max_new_pending_txs_notifications: usize, /// Maximum lifetime for transactions in the pool pub max_queued_lifetime: Duration, + /// The maximum allowed inflight transactions a delegated sender can have. + /// + /// This restricts how many executable transaction a delegated sender can stack. + pub max_inflight_delegated_slot_limit: usize, } impl PoolConfig { + /// Sets the minimal protocol base fee to 0, effectively disabling checks that enforce that a + /// transaction's fee must be higher than the [`MIN_PROTOCOL_BASE_FEE`] which is the lowest + /// value the ethereum EIP-1559 base fee can reach. + pub const fn with_disabled_protocol_base_fee(self) -> Self { + self.with_protocol_base_fee(0) + } + + /// Configures the minimal protocol base fee that should be enforced. + /// + /// Ethereum's EIP-1559 base fee can't drop below [`MIN_PROTOCOL_BASE_FEE`] hence this is + /// enforced by default in the pool. + pub const fn with_protocol_base_fee(mut self, protocol_base_fee: u64) -> Self { + self.minimal_protocol_basefee = protocol_base_fee; + self + } + + /// Configures how many slots are available for a delegated sender. + pub const fn with_max_inflight_delegated_slots( + mut self, + max_inflight_delegation_limit: usize, + ) -> Self { + self.max_inflight_delegated_slot_limit = max_inflight_delegation_limit; + self + } + /// Returns whether the size and amount constraints in any sub-pools are exceeded. #[inline] pub const fn is_exceeded(&self, pool_size: PoolSize) -> bool { @@ -87,12 +121,14 @@ impl Default for PoolConfig { max_account_slots: TXPOOL_MAX_ACCOUNT_SLOTS_PER_SENDER, price_bumps: Default::default(), minimal_protocol_basefee: MIN_PROTOCOL_BASE_FEE, + minimum_priority_fee: None, gas_limit: ETHEREUM_BLOCK_GAS_LIMIT_30M, local_transactions_config: Default::default(), pending_tx_listener_buffer_size: PENDING_TX_LISTENER_BUFFER_SIZE, new_tx_listener_buffer_size: NEW_TX_LISTENER_BUFFER_SIZE, max_new_pending_txs_notifications: MAX_NEW_PENDING_TXS_NOTIFICATIONS, max_queued_lifetime: MAX_QUEUED_TRANSACTION_LIFETIME, + max_inflight_delegated_slot_limit: DEFAULT_MAX_INFLIGHT_DELEGATED_SLOTS, } } } @@ -112,6 +148,11 @@ impl SubPoolLimit { Self { max_txs, max_size } } + /// Creates an unlimited [`SubPoolLimit`] + pub const fn max() -> Self { + Self::new(usize::MAX, usize::MAX) + } + /// Returns whether the size or amount constraint is violated. #[inline] pub const fn is_exceeded(&self, txs: usize, size: usize) -> bool { diff --git a/crates/transaction-pool/src/error.rs b/crates/transaction-pool/src/error.rs index 5327aff7a47..3bcbb4cd0ab 100644 --- a/crates/transaction-pool/src/error.rs +++ b/crates/transaction-pool/src/error.rs @@ -93,7 +93,7 @@ impl PoolError { /// /// Not all error variants are caused by the incorrect composition of the transaction (See also /// [`InvalidPoolTransactionError`]) and can be caused by the current state of the transaction - /// pool. For example the transaction pool is already full or the error was caused my an + /// pool. For example the transaction pool is already full or the error was caused by an /// internal error, such as database errors. /// /// This function returns true only if the transaction will never make it into the pool because @@ -157,7 +157,7 @@ pub enum Eip4844PoolTransactionError { /// Thrown if an EIP-4844 transaction without any blobs arrives #[error("blobless blob transaction")] NoEip4844Blobs, - /// Thrown if an EIP-4844 transaction without any blobs arrives + /// Thrown if an EIP-4844 transaction arrives with too many blobs #[error("too many blobs in transaction: have {have}, permitted {permitted}")] TooManyEip4844Blobs { /// Number of blobs the transaction has @@ -176,6 +176,12 @@ pub enum Eip4844PoolTransactionError { /// would introduce gap in the nonce sequence. #[error("nonce too high")] Eip4844NonceGap, + /// Thrown if blob transaction has an EIP-7594 style sidecar before Osaka. + #[error("unexpected eip-7594 sidecar before osaka")] + UnexpectedEip7594SidecarBeforeOsaka, + /// Thrown if blob transaction has an EIP-4844 style sidecar after Osaka. + #[error("unexpected eip-4844 sidecar after osaka")] + UnexpectedEip4844SidecarAfterOsaka, } /// Represents all errors that can happen when validating transactions for the pool for EIP-7702 @@ -212,11 +218,14 @@ pub enum InvalidPoolTransactionError { /// respect the size limits of the pool. #[error("transaction's gas limit {0} exceeds block's gas limit {1}")] ExceedsGasLimit(u64, u64), + /// Thrown when a transaction's gas limit exceeds the configured maximum per-transaction limit. + #[error("transaction's gas limit {0} exceeds maximum per-transaction gas limit {1}")] + MaxTxGasLimitExceeded(u64, u64), /// Thrown when a new transaction is added to the pool, but then immediately discarded to /// respect the tx fee exceeds the configured cap #[error("tx fee ({max_tx_fee_wei} wei) exceeds the configured cap ({tx_fee_cap_wei} wei)")] ExceedsFeeCap { - /// max fee in wei of new tx submitted to the pull (e.g. 0.11534 ETH) + /// max fee in wei of new tx submitted to the pool (e.g. 0.11534 ETH) max_tx_fee_wei: u128, /// configured tx fee cap in wei (e.g. 1.0 ETH) tx_fee_cap_wei: u128, @@ -228,8 +237,13 @@ pub enum InvalidPoolTransactionError { /// Thrown if the input data of a transaction is greater /// than some meaningful limit a user might use. This is not a consensus error /// making the transaction invalid, rather a DOS protection. - #[error("input data too large")] - OversizedData(usize, usize), + #[error("oversized data: transaction size {size}, limit {limit}")] + OversizedData { + /// Size of the transaction/input data that exceeded the limit. + size: usize, + /// Configured limit that was exceeded. + limit: usize, + }, /// Thrown if the transaction's fee is below the minimum fee #[error("transaction underpriced")] Underpriced, @@ -241,6 +255,10 @@ pub enum InvalidPoolTransactionError { /// Balance of account. balance: U256, }, + /// EIP-2681 error thrown if the nonce is higher or equal than `U64::max` + /// `` + #[error("nonce exceeds u64 limit")] + Eip2681, /// EIP-4844 related errors #[error(transparent)] Eip4844(#[from] Eip4844PoolTransactionError), @@ -254,6 +272,12 @@ pub enum InvalidPoolTransactionError { /// invocation. #[error("intrinsic gas too low")] IntrinsicGasTooLow, + /// The transaction priority fee is below the minimum required priority fee. + #[error("transaction priority fee below minimum required priority fee {minimum_priority_fee}")] + PriorityFeeBelowMinimum { + /// Minimum required priority fee. + minimum_priority_fee: u128, + }, } // === impl InvalidPoolTransactionError === @@ -305,13 +329,18 @@ impl InvalidPoolTransactionError { InvalidTransactionError::ChainIdMismatch | InvalidTransactionError::GasUintOverflow | InvalidTransactionError::TxTypeNotSupported | - InvalidTransactionError::SignerAccountHasBytecode => true, + InvalidTransactionError::SignerAccountHasBytecode | + InvalidTransactionError::GasLimitTooHigh => true, } } Self::ExceedsGasLimit(_, _) => true, + Self::MaxTxGasLimitExceeded(_, _) => { + // local setting + false + } Self::ExceedsFeeCap { max_tx_fee_wei: _, tx_fee_cap_wei: _ } => true, Self::ExceedsMaxInitCodeSize(_, _) => true, - Self::OversizedData(_, _) => true, + Self::OversizedData { .. } => true, Self::Underpriced => { // local setting false @@ -319,6 +348,7 @@ impl InvalidPoolTransactionError { Self::IntrinsicGasTooLow => true, Self::Overdraft { .. } => false, Self::Other(err) => err.is_bad_transaction(), + Self::Eip2681 => true, Self::Eip4844(eip4844_err) => { match eip4844_err { Eip4844PoolTransactionError::MissingEip4844BlobSidecar => { @@ -343,6 +373,12 @@ impl InvalidPoolTransactionError { // this is a malformed transaction and should not be sent over the network true } + Eip4844PoolTransactionError::UnexpectedEip4844SidecarAfterOsaka | + Eip4844PoolTransactionError::UnexpectedEip7594SidecarBeforeOsaka => { + // for now we do not want to penalize peers for broadcasting different + // sidecars + false + } } } Self::Eip7702(eip7702_err) => match eip7702_err { @@ -356,9 +392,15 @@ impl InvalidPoolTransactionError { Eip7702PoolTransactionError::InflightTxLimitReached => false, Eip7702PoolTransactionError::AuthorityReserved => false, }, + Self::PriorityFeeBelowMinimum { .. } => false, } } + /// Returns `true` if an import failed due to an oversized transaction + pub const fn is_oversized(&self) -> bool { + matches!(self, Self::OversizedData { .. }) + } + /// Returns `true` if an import failed due to nonce gap. pub const fn is_nonce_gap(&self) -> bool { matches!(self, Self::Consensus(InvalidTransactionError::NonceNotConsistent { .. })) || diff --git a/crates/transaction-pool/src/identifier.rs b/crates/transaction-pool/src/identifier.rs index 9b28d3789cd..d2610ee9ba3 100644 --- a/crates/transaction-pool/src/identifier.rs +++ b/crates/transaction-pool/src/identifier.rs @@ -1,7 +1,6 @@ //! Identifier types for transactions and senders. -use alloy_primitives::Address; +use alloy_primitives::{map::HashMap, Address}; use rustc_hash::FxHashMap; -use std::collections::HashMap; /// An internal mapping of addresses. /// @@ -38,6 +37,14 @@ impl SenderIdentifiers { }) } + /// Returns the existing [`SenderId`] or assigns a new one if it's missing + pub fn sender_ids_or_create( + &mut self, + addrs: impl IntoIterator, + ) -> Vec { + addrs.into_iter().map(|addr| self.sender_id_or_create(addr)).collect() + } + /// Returns the current identifier and increments the counter. fn next_id(&mut self) -> SenderId { let id = self.id; @@ -94,12 +101,13 @@ impl TransactionId { /// This returns `transaction_nonce - 1` if `transaction_nonce` is higher than the /// `on_chain_nonce` pub fn ancestor(transaction_nonce: u64, on_chain_nonce: u64, sender: SenderId) -> Option { - (transaction_nonce > on_chain_nonce) - .then(|| Self::new(sender, transaction_nonce.saturating_sub(1))) + // SAFETY: transaction_nonce > on_chain_nonce ⇒ transaction_nonce >= 1 + (transaction_nonce > on_chain_nonce).then(|| Self::new(sender, transaction_nonce - 1)) } /// Returns the [`TransactionId`] that would come before this transaction. pub fn unchecked_ancestor(&self) -> Option { + // SAFETY: self.nonce != 0 ⇒ self.nonce >= 1 (self.nonce != 0).then(|| Self::new(self.sender, self.nonce - 1)) } diff --git a/crates/transaction-pool/src/lib.rs b/crates/transaction-pool/src/lib.rs index 0dab27f4677..7f3fa4a1177 100644 --- a/crates/transaction-pool/src/lib.rs +++ b/crates/transaction-pool/src/lib.rs @@ -12,6 +12,127 @@ //! - monitoring memory footprint and enforce pool size limits //! - storing blob data for transactions in a separate blobstore on insertion //! +//! ## Transaction Flow: From Network/RPC to Pool +//! +//! Transactions enter the pool through two main paths: +//! +//! ### 1. Network Path (P2P) +//! +//! ```text +//! Network Peer +//! ↓ +//! Transactions or NewPooledTransactionHashes message +//! ↓ +//! TransactionsManager (crates/net/network/src/transactions/mod.rs) +//! │ +//! ├─→ For Transactions message: +//! │ ├─→ Validates message format +//! │ ├─→ Checks if transaction already known +//! │ ├─→ Marks peer as having seen the transaction +//! │ └─→ Queues for import +//! │ +//! └─→ For NewPooledTransactionHashes message: +//! ├─→ Filters out already known transactions +//! ├─→ Queues unknown hashes for fetching +//! ├─→ Sends GetPooledTransactions request +//! ├─→ Receives PooledTransactions response +//! └─→ Queues fetched transactions for import +//! ↓ +//! pool.add_external_transactions() [Origin: External] +//! ↓ +//! Transaction Validation & Pool Addition +//! ``` +//! +//! ### 2. RPC Path (Local submission) +//! +//! ```text +//! eth_sendRawTransaction RPC call +//! ├─→ Decodes raw bytes +//! └─→ Recovers sender +//! ↓ +//! pool.add_transaction() [Origin: Local] +//! ↓ +//! Transaction Validation & Pool Addition +//! ``` +//! +//! ### Transaction Origins +//! +//! - **Local**: Transactions submitted via RPC (trusted, may have different fee requirements) +//! - **External**: Transactions from network peers (untrusted, subject to stricter validation) +//! - **Private**: Local transactions that should not be propagated to the network +//! +//! ## Validation Process +//! +//! ### Stateless Checks +//! +//! Ethereum transactions undergo several stateless checks: +//! +//! - **Transaction Type**: Fork-dependent support (Legacy always, EIP-2930/1559/4844/7702 need +//! activation) +//! - **Size**: Input data ≤ 128KB (default) +//! - **Gas**: Limit ≤ block gas limit +//! - **Fees**: Priority fee ≤ max fee; local tx fee cap; external minimum priority fee +//! - **Chain ID**: Must match current chain +//! - **Intrinsic Gas**: Sufficient for data and access lists +//! - **Blobs** (EIP-4844): Valid count, KZG proofs +//! +//! ### Stateful Checks +//! +//! 1. **Sender**: No bytecode (unless EIP-7702 delegated in Prague) +//! 2. **Nonce**: ≥ account nonce +//! 3. **Balance**: Covers value + (`gas_limit` × `max_fee_per_gas`) +//! +//! ### Common Errors +//! +//! - [`NonceNotConsistent`](reth_primitives_traits::transaction::error::InvalidTransactionError::NonceNotConsistent): Nonce too low +//! - [`InsufficientFunds`](reth_primitives_traits::transaction::error::InvalidTransactionError::InsufficientFunds): Insufficient balance +//! - [`ExceedsGasLimit`](crate::error::InvalidPoolTransactionError::ExceedsGasLimit): Gas limit too +//! high +//! - [`SignerAccountHasBytecode`](reth_primitives_traits::transaction::error::InvalidTransactionError::SignerAccountHasBytecode): EOA has code +//! - [`Underpriced`](crate::error::InvalidPoolTransactionError::Underpriced): Fee too low +//! - [`ReplacementUnderpriced`](crate::error::PoolErrorKind::ReplacementUnderpriced): Replacement +//! transaction fee too low +//! - Blob errors: +//! - [`MissingEip4844BlobSidecar`](crate::error::Eip4844PoolTransactionError::MissingEip4844BlobSidecar): Missing sidecar +//! - [`InvalidEip4844Blob`](crate::error::Eip4844PoolTransactionError::InvalidEip4844Blob): +//! Invalid blob proofs +//! - [`NoEip4844Blobs`](crate::error::Eip4844PoolTransactionError::NoEip4844Blobs): EIP-4844 +//! transaction without blobs +//! - [`TooManyEip4844Blobs`](crate::error::Eip4844PoolTransactionError::TooManyEip4844Blobs): Too +//! many blobs +//! +//! ## Subpool Design +//! +//! The pool maintains four distinct subpools, each serving a specific purpose +//! +//! ### Subpools +//! +//! 1. **Pending**: Ready for inclusion (no gaps, sufficient balance/fees) +//! 2. **Queued**: Future transactions (nonce gaps or insufficient balance) +//! 3. **`BaseFee`**: Valid but below current base fee +//! 4. **Blob**: EIP-4844 transactions not pending due to insufficient base fee or blob fee +//! +//! ### State Transitions +//! +//! Transactions move between subpools based on state changes: +//! +//! ```text +//! Queued ─────────→ BaseFee/Blob ────────→ Pending +//! ↑ ↑ │ +//! │ │ │ +//! └────────────────────┴─────────────────────┘ +//! (demotions due to state changes) +//! ``` +//! +//! **Promotions**: Nonce gaps filled, balance/fee improvements +//! **Demotions**: Nonce gaps created, balance/fee degradation +//! +//! ## Pool Maintenance +//! +//! 1. **Block Updates**: Removes mined txs, updates accounts/fees, triggers movements +//! 2. **Size Enforcement**: Discards worst transactions when limits exceeded +//! 3. **Propagation**: External (always), Local (configurable), Private (never) +//! //! ## Assumptions //! //! ### Transaction type @@ -41,11 +162,7 @@ //! //! ### State Changes //! -//! Once a new block is mined, the pool needs to be updated with a changeset in order to: -//! -//! - remove mined transactions -//! - update using account changes: balance changes -//! - base fee updates +//! New blocks trigger pool updates via changesets (see Pool Maintenance). //! //! ## Implementation details //! @@ -118,9 +235,10 @@ //! use reth_transaction_pool::{TransactionValidationTaskExecutor, Pool}; //! use reth_transaction_pool::blobstore::InMemoryBlobStore; //! use reth_transaction_pool::maintain::{maintain_transaction_pool_future}; +//! use alloy_consensus::Header; //! //! async fn t(client: C, stream: St) -//! where C: StateProviderFactory + BlockReaderIdExt + ChainSpecProvider + Clone + 'static, +//! where C: StateProviderFactory + BlockReaderIdExt
+ ChainSpecProvider + Clone + 'static, //! St: Stream + Send + Unpin + 'static, //! { //! let blob_store = InMemoryBlobStore::default(); @@ -149,13 +267,15 @@ html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256", issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(test), warn(unused_crate_dependencies))] pub use crate::{ + batcher::{BatchTxProcessor, BatchTxRequest}, blobstore::{BlobStore, BlobStoreError}, config::{ - LocalTransactionConfig, PoolConfig, PriceBumpConfig, SubPoolLimit, DEFAULT_PRICE_BUMP, + LocalTransactionConfig, PoolConfig, PriceBumpConfig, SubPoolLimit, + DEFAULT_MAX_INFLIGHT_DELEGATED_SLOTS, DEFAULT_PRICE_BUMP, DEFAULT_TXPOOL_ADDITIONAL_VALIDATION_TASKS, MAX_NEW_PENDING_TXS_NOTIFICATIONS, REPLACE_BLOB_PRICE_BUMP, TXPOOL_MAX_ACCOUNT_SLOTS_PER_SENDER, TXPOOL_SUBPOOL_MAX_SIZE_MB_DEFAULT, TXPOOL_SUBPOOL_MAX_TXS_DEFAULT, @@ -163,8 +283,9 @@ pub use crate::{ error::PoolResult, ordering::{CoinbaseTipOrdering, Priority, TransactionOrdering}, pool::{ - blob_tx_priority, fee_delta, state::SubPool, AllTransactionsEvents, FullTransactionEvent, - NewTransactionEvent, TransactionEvent, TransactionEvents, TransactionListenerKind, + blob_tx_priority, fee_delta, state::SubPool, AddedTransactionOutcome, + AllTransactionsEvents, FullTransactionEvent, NewTransactionEvent, TransactionEvent, + TransactionEvents, TransactionListenerKind, }, traits::*, validate::{ @@ -173,7 +294,10 @@ pub use crate::{ }, }; use crate::{identifier::TransactionId, pool::PoolInner}; -use alloy_eips::eip4844::{BlobAndProofV1, BlobTransactionSidecar}; +use alloy_eips::{ + eip4844::{BlobAndProofV1, BlobAndProofV2}, + eip7594::BlobTransactionSidecarVariant, +}; use alloy_primitives::{Address, TxHash, B256, U256}; use aquamarine as _; use reth_chainspec::{ChainSpecProvider, EthereumHardforks}; @@ -192,6 +316,7 @@ pub mod noop; pub mod pool; pub mod validate; +pub mod batcher; pub mod blobstore; mod config; pub mod identifier; @@ -203,9 +328,9 @@ mod traits; pub mod test_utils; /// Type alias for default ethereum transaction pool -pub type EthTransactionPool = Pool< - TransactionValidationTaskExecutor>, - CoinbaseTipOrdering, +pub type EthTransactionPool = Pool< + TransactionValidationTaskExecutor>, + CoinbaseTipOrdering, S, >; @@ -239,29 +364,41 @@ where self.inner().config() } + /// Validates the given transaction + async fn validate( + &self, + origin: TransactionOrigin, + transaction: V::Transaction, + ) -> TransactionValidationOutcome { + self.pool.validator().validate_transaction(origin, transaction).await + } + /// Returns future that validates all transactions in the given iterator. /// /// This returns the validated transactions in the iterator's order. async fn validate_all( &self, origin: TransactionOrigin, - transactions: impl IntoIterator, - ) -> Vec<(TxHash, TransactionValidationOutcome)> { - futures_util::future::join_all(transactions.into_iter().map(|tx| self.validate(origin, tx))) - .await + transactions: impl IntoIterator + Send, + ) -> Vec> { + self.pool.validator().validate_transactions_with_origin(origin, transactions).await } - /// Validates the given transaction - async fn validate( + /// Validates all transactions with their individual origins. + /// + /// This returns the validated transactions in the same order as input. + async fn validate_all_with_origins( &self, - origin: TransactionOrigin, - transaction: V::Transaction, - ) -> (TxHash, TransactionValidationOutcome) { - let hash = *transaction.hash(); - - let outcome = self.pool.validator().validate_transaction(origin, transaction).await; - - (hash, outcome) + transactions: Vec<(TransactionOrigin, V::Transaction)>, + ) -> Vec<(TransactionOrigin, TransactionValidationOutcome)> { + if transactions.len() == 1 { + let (origin, tx) = transactions.into_iter().next().unwrap(); + let res = self.pool.validator().validate_transaction(origin, tx).await; + return vec![(origin, res)] + } + let origins: Vec<_> = transactions.iter().map(|(origin, _)| *origin).collect(); + let tx_outcomes = self.pool.validator().validate_transactions(transactions).await; + origins.into_iter().zip(tx_outcomes).collect() } /// Number of transactions in the entire pool @@ -352,7 +489,7 @@ where origin: TransactionOrigin, transaction: Self::Transaction, ) -> PoolResult { - let (_, tx) = self.validate(origin, transaction).await; + let tx = self.validate(origin, transaction).await; self.pool.add_transaction_and_subscribe(origin, tx) } @@ -360,8 +497,8 @@ where &self, origin: TransactionOrigin, transaction: Self::Transaction, - ) -> PoolResult { - let (_, tx) = self.validate(origin, transaction).await; + ) -> PoolResult { + let tx = self.validate(origin, transaction).await; let mut results = self.pool.add_transactions(origin, std::iter::once(tx)); results.pop().expect("result length is the same as the input") } @@ -370,13 +507,25 @@ where &self, origin: TransactionOrigin, transactions: Vec, - ) -> Vec> { + ) -> Vec> { if transactions.is_empty() { return Vec::new() } let validated = self.validate_all(origin, transactions).await; - self.pool.add_transactions(origin, validated.into_iter().map(|(_, tx)| tx)) + self.pool.add_transactions(origin, validated.into_iter()) + } + + async fn add_transactions_with_origins( + &self, + transactions: Vec<(TransactionOrigin, Self::Transaction)>, + ) -> Vec> { + if transactions.is_empty() { + return Vec::new() + } + let validated = self.validate_all_with_origins(transactions).await; + + self.pool.add_transactions_with_origins(validated) } fn transaction_event_listener(&self, tx_hash: TxHash) -> Option { @@ -465,10 +614,21 @@ where self.pool.queued_transactions() } + fn pending_and_queued_txn_count(&self) -> (usize, usize) { + let data = self.pool.get_pool_data(); + let pending = data.pending_transactions_count(); + let queued = data.queued_transactions_count(); + (pending, queued) + } + fn all_transactions(&self) -> AllPoolTransactions { self.pool.all_transactions() } + fn all_transaction_hashes(&self) -> Vec { + self.pool.all_transaction_hashes() + } + fn remove_transactions( &self, hashes: Vec, @@ -584,29 +744,36 @@ where fn get_blob( &self, tx_hash: TxHash, - ) -> Result>, BlobStoreError> { + ) -> Result>, BlobStoreError> { self.pool.blob_store().get(tx_hash) } fn get_all_blobs( &self, tx_hashes: Vec, - ) -> Result)>, BlobStoreError> { + ) -> Result)>, BlobStoreError> { self.pool.blob_store().get_all(tx_hashes) } fn get_all_blobs_exact( &self, tx_hashes: Vec, - ) -> Result>, BlobStoreError> { + ) -> Result>, BlobStoreError> { self.pool.blob_store().get_exact(tx_hashes) } - fn get_blobs_for_versioned_hashes( + fn get_blobs_for_versioned_hashes_v1( &self, versioned_hashes: &[B256], ) -> Result>, BlobStoreError> { - self.pool.blob_store().get_by_versioned_hashes(versioned_hashes) + self.pool.blob_store().get_by_versioned_hashes_v1(versioned_hashes) + } + + fn get_blobs_for_versioned_hashes_v2( + &self, + versioned_hashes: &[B256], + ) -> Result>, BlobStoreError> { + self.pool.blob_store().get_by_versioned_hashes_v2(versioned_hashes) } } diff --git a/crates/transaction-pool/src/maintain.rs b/crates/transaction-pool/src/maintain.rs index 1e328853094..0e30a2473b2 100644 --- a/crates/transaction-pool/src/maintain.rs +++ b/crates/transaction-pool/src/maintain.rs @@ -1,22 +1,23 @@ //! Support for maintaining the state of the transaction pool use crate::{ - blobstore::{BlobStoreCanonTracker, BlobStoreUpdates}, + blobstore::{BlobSidecarConverter, BlobStoreCanonTracker, BlobStoreUpdates}, error::PoolError, metrics::MaintainPoolMetrics, traits::{CanonicalStateUpdate, EthPoolTransaction, TransactionPool, TransactionPoolExt}, - BlockInfo, PoolTransaction, PoolUpdateKind, + AllPoolTransactions, BlobTransactionSidecarVariant, BlockInfo, PoolTransaction, PoolUpdateKind, + TransactionOrigin, }; -use alloy_consensus::{BlockHeader, Typed2718}; -use alloy_eips::BlockNumberOrTag; -use alloy_primitives::{Address, BlockHash, BlockNumber}; +use alloy_consensus::{transaction::TxHashRef, BlockHeader, Typed2718}; +use alloy_eips::{BlockNumberOrTag, Decodable2718, Encodable2718}; +use alloy_primitives::{Address, BlockHash, BlockNumber, Bytes}; use alloy_rlp::Encodable; use futures_util::{ future::{BoxFuture, Fuse, FusedFuture}, FutureExt, Stream, StreamExt, }; use reth_chain_state::CanonStateNotification; -use reth_chainspec::{ChainSpecProvider, EthChainSpec}; +use reth_chainspec::{ChainSpecProvider, EthChainSpec, EthereumHardforks}; use reth_execution_types::ChangedAccount; use reth_fs_util::FsPathError; use reth_primitives_traits::{ @@ -24,6 +25,7 @@ use reth_primitives_traits::{ }; use reth_storage_api::{errors::provider::ProviderError, BlockReaderIdExt, StateProviderFactory}; use reth_tasks::TaskSpawner; +use serde::{Deserialize, Serialize}; use std::{ borrow::Borrow, collections::HashSet, @@ -100,10 +102,14 @@ pub fn maintain_transaction_pool_future( ) -> BoxFuture<'static, ()> where N: NodePrimitives, - Client: StateProviderFactory + BlockReaderIdExt + ChainSpecProvider + Clone + 'static, + Client: StateProviderFactory + + BlockReaderIdExt
+ + ChainSpecProvider + EthereumHardforks> + + Clone + + 'static, P: TransactionPoolExt> + 'static, St: Stream> + Send + Unpin + 'static, - Tasks: TaskSpawner + 'static, + Tasks: TaskSpawner + Clone + 'static, { async move { maintain_transaction_pool(client, pool, events, task_spawner, config).await; @@ -122,10 +128,14 @@ pub async fn maintain_transaction_pool( config: MaintainPoolConfig, ) where N: NodePrimitives, - Client: StateProviderFactory + BlockReaderIdExt + ChainSpecProvider + Clone + 'static, + Client: StateProviderFactory + + BlockReaderIdExt
+ + ChainSpecProvider + EthereumHardforks> + + Clone + + 'static, P: TransactionPoolExt> + 'static, St: Stream> + Send + Unpin + 'static, - Tasks: TaskSpawner + 'static, + Tasks: TaskSpawner + Clone + 'static, { let metrics = MaintainPoolMetrics::default(); let MaintainPoolConfig { max_update_depth, max_reload_accounts, .. } = config; @@ -137,8 +147,8 @@ pub async fn maintain_transaction_pool( block_gas_limit: latest.gas_limit(), last_seen_block_hash: latest.hash(), last_seen_block_number: latest.number(), - pending_basefee: latest - .next_block_base_fee(chain_spec.base_fee_params_at_timestamp(latest.timestamp())) + pending_basefee: chain_spec + .next_block_base_fee(latest.header(), latest.timestamp()) .unwrap_or_default(), pending_blob_fee: latest .maybe_next_block_blob_fee(chain_spec.blob_params_at_timestamp(latest.timestamp())), @@ -220,21 +230,19 @@ pub async fn maintain_transaction_pool( // check if we have a new finalized block if let Some(finalized) = - last_finalized_block.update(client.finalized_block_number().ok().flatten()) - { - if let BlobStoreUpdates::Finalized(blobs) = + last_finalized_block.update(client.finalized_block_number().ok().flatten()) && + let BlobStoreUpdates::Finalized(blobs) = blob_store_tracker.on_finalized_block(finalized) - { - metrics.inc_deleted_tracked_blobs(blobs.len()); - // remove all finalized blobs from the blob store - pool.delete_blobs(blobs); - // and also do periodic cleanup - let pool = pool.clone(); - task_spawner.spawn_blocking(Box::pin(async move { - debug!(target: "txpool", finalized_block = %finalized, "cleaning up blob store"); - pool.cleanup_blobs(); - })); - } + { + metrics.inc_deleted_tracked_blobs(blobs.len()); + // remove all finalized blobs from the blob store + pool.delete_blobs(blobs); + // and also do periodic cleanup + let pool = pool.clone(); + task_spawner.spawn_blocking(Box::pin(async move { + debug!(target: "txpool", finalized_block = %finalized, "cleaning up blob store"); + pool.cleanup_blobs(); + })); } // outcomes of the futures we are waiting on @@ -317,11 +325,8 @@ pub async fn maintain_transaction_pool( let chain_spec = client.chain_spec(); // fees for the next block: `new_tip+1` - let pending_block_base_fee = new_tip - .header() - .next_block_base_fee( - chain_spec.base_fee_params_at_timestamp(new_tip.timestamp()), - ) + let pending_block_base_fee = chain_spec + .next_block_base_fee(new_tip.header(), new_tip.timestamp()) .unwrap_or_default(); let pending_block_blob_fee = new_tip.header().maybe_next_block_blob_fee( chain_spec.blob_params_at_timestamp(new_tip.timestamp()), @@ -423,9 +428,8 @@ pub async fn maintain_transaction_pool( let chain_spec = client.chain_spec(); // fees for the next block: `tip+1` - let pending_block_base_fee = tip - .header() - .next_block_base_fee(chain_spec.base_fee_params_at_timestamp(tip.timestamp())) + let pending_block_base_fee = chain_spec + .next_block_base_fee(tip.header(), tip.timestamp()) .unwrap_or_default(); let pending_block_blob_fee = tip.header().maybe_next_block_blob_fee( chain_spec.blob_params_at_timestamp(tip.timestamp()), @@ -491,6 +495,89 @@ pub async fn maintain_transaction_pool( // keep track of mined blob transactions blob_store_tracker.add_new_chain_blocks(&blocks); + + // If Osaka activates in 2 slots we need to convert blobs to new format. + if !chain_spec.is_osaka_active_at_timestamp(tip.timestamp()) && + !chain_spec.is_osaka_active_at_timestamp(tip.timestamp().saturating_add(12)) && + chain_spec.is_osaka_active_at_timestamp(tip.timestamp().saturating_add(24)) + { + let pool = pool.clone(); + let spawner = task_spawner.clone(); + let client = client.clone(); + task_spawner.spawn(Box::pin(async move { + // Start converting not eaerlier than 4 seconds into current slot to ensure + // that our pool only contains valid transactions for the next block (as + // it's not Osaka yet). + tokio::time::sleep(Duration::from_secs(4)).await; + + let mut interval = tokio::time::interval(Duration::from_secs(1)); + loop { + // Loop and replace blob transactions until we reach Osaka transition + // block after which no legacy blobs are going to be accepted. + let last_iteration = + client.latest_header().ok().flatten().is_none_or(|header| { + client + .chain_spec() + .is_osaka_active_at_timestamp(header.timestamp()) + }); + + let AllPoolTransactions { pending, queued } = pool.all_transactions(); + for tx in pending + .into_iter() + .chain(queued) + .filter(|tx| tx.transaction.is_eip4844()) + { + let tx_hash = *tx.transaction.hash(); + + // Fetch sidecar from the pool + let Ok(Some(sidecar)) = pool.get_blob(tx_hash) else { + continue; + }; + // Ensure it is a legacy blob + if !sidecar.is_eip4844() { + continue; + } + // Remove transaction and sidecar from the pool, both are in memory + // now + let Some(tx) = pool.remove_transactions(vec![tx_hash]).pop() else { + continue; + }; + pool.delete_blob(tx_hash); + + let BlobTransactionSidecarVariant::Eip4844(sidecar) = + Arc::unwrap_or_clone(sidecar) + else { + continue; + }; + + let converter = BlobSidecarConverter::new(); + let pool = pool.clone(); + spawner.spawn(Box::pin(async move { + // Convert sidecar to EIP-7594 format + let Some(sidecar) = converter.convert(sidecar).await else { + return; + }; + + // Re-insert transaction with the new sidecar + let origin = tx.origin; + let Some(tx) = EthPoolTransaction::try_from_eip4844( + tx.transaction.clone_into_consensus(), + sidecar.into(), + ) else { + return; + }; + let _ = pool.add_transaction(origin, tx).await; + })); + } + + if last_iteration { + break; + } + + interval.tick().await; + } + })); + } } } } @@ -597,8 +684,8 @@ where Ok(res) } -/// Loads transactions from a file, decodes them from the RLP format, and inserts them -/// into the transaction pool on node boot up. +/// Loads transactions from a file, decodes them from the JSON or RLP format, and +/// inserts them into the transaction pool on node boot up. /// The file is removed after the transactions have been successfully processed. async fn load_and_reinsert_transactions

( pool: P, @@ -618,21 +705,44 @@ where return Ok(()) } - let txs_signed: Vec<::Consensus> = - alloy_rlp::Decodable::decode(&mut data.as_slice())?; - - let pool_transactions = txs_signed - .into_iter() - .filter_map(|tx| tx.try_clone_into_recovered().ok()) - .filter_map(|tx| { - // Filter out errors - ::try_from_consensus(tx).ok() - }) - .collect(); + let pool_transactions: Vec<(TransactionOrigin,

::Transaction)> = + if let Ok(tx_backups) = serde_json::from_slice::>(&data) { + tx_backups + .into_iter() + .filter_map(|backup| { + let tx_signed = + ::Consensus::decode_2718_exact( + backup.rlp.as_ref(), + ) + .ok()?; + let recovered = tx_signed.try_into_recovered().ok()?; + let pool_tx = + ::try_from_consensus(recovered).ok()?; + + Some((backup.origin, pool_tx)) + }) + .collect() + } else { + let txs_signed: Vec<::Consensus> = + alloy_rlp::Decodable::decode(&mut data.as_slice())?; + + txs_signed + .into_iter() + .filter_map(|tx| tx.try_into_recovered().ok()) + .filter_map(|tx| { + ::try_from_consensus(tx) + .ok() + .map(|pool_tx| (TransactionOrigin::Local, pool_tx)) + }) + .collect() + }; - let outcome = pool.add_transactions(crate::TransactionOrigin::Local, pool_transactions).await; + let inserted = futures_util::future::join_all( + pool_transactions.into_iter().map(|(origin, tx)| pool.add_transaction(origin, tx)), + ) + .await; - info!(target: "txpool", txs_file =?file_path, num_txs=%outcome.len(), "Successfully reinserted local transactions from file"); + info!(target: "txpool", txs_file =?file_path, num_txs=%inserted.len(), "Successfully reinserted local transactions from file"); reth_fs_util::remove_file(file_path)?; Ok(()) } @@ -649,16 +759,26 @@ where let local_transactions = local_transactions .into_iter() - .map(|tx| tx.transaction.clone_into_consensus().into_inner()) + .map(|tx| { + let consensus_tx = tx.transaction.clone_into_consensus().into_inner(); + let rlp_data = consensus_tx.encoded_2718(); + + TxBackup { rlp: rlp_data.into(), origin: tx.origin } + }) .collect::>(); - let num_txs = local_transactions.len(); - let mut buf = Vec::new(); - alloy_rlp::encode_list(&local_transactions, &mut buf); - info!(target: "txpool", txs_file =?file_path, num_txs=%num_txs, "Saving current local transactions"); + let json_data = match serde_json::to_string(&local_transactions) { + Ok(data) => data, + Err(err) => { + warn!(target: "txpool", %err, txs_file=?file_path, "failed to serialize local transactions to json"); + return + } + }; + + info!(target: "txpool", txs_file =?file_path, num_txs=%local_transactions.len(), "Saving current local transactions"); let parent_dir = file_path.parent().map(std::fs::create_dir_all).transpose(); - match parent_dir.map(|_| reth_fs_util::write(file_path, buf)) { + match parent_dir.map(|_| reth_fs_util::write(file_path, json_data)) { Ok(_) => { info!(target: "txpool", txs_file=?file_path, "Wrote local transactions to file"); } @@ -668,12 +788,25 @@ where } } +/// A transaction backup that is saved as json to a file for +/// reinsertion into the pool +#[derive(Debug, Deserialize, Serialize)] +pub struct TxBackup { + /// Encoded transaction + pub rlp: Bytes, + /// The origin of the transaction + pub origin: TransactionOrigin, +} + /// Errors possible during txs backup load and decode #[derive(thiserror::Error, Debug)] pub enum TransactionsBackupError { /// Error during RLP decoding of transactions #[error("failed to apply transactions backup. Encountered RLP decode error: {0}")] Decode(#[from] alloy_rlp::Error), + /// Error during json decoding of transactions + #[error("failed to apply transactions backup. Encountered JSON decode error: {0}")] + Json(#[from] serde_json::Error), /// Error during file upload #[error("failed to apply transactions backup. Encountered file error: {0}")] FsPath(#[from] FsPathError), @@ -715,10 +848,9 @@ mod tests { blobstore::InMemoryBlobStore, validate::EthTransactionValidatorBuilder, CoinbaseTipOrdering, EthPooledTransaction, Pool, TransactionOrigin, }; - use alloy_consensus::transaction::PooledTransaction; use alloy_eips::eip2718::Decodable2718; use alloy_primitives::{hex, U256}; - use reth_ethereum_primitives::TransactionSigned; + use reth_ethereum_primitives::PooledTransactionVariant; use reth_fs_util as fs; use reth_provider::test_utils::{ExtendedAccount, MockEthProvider}; use reth_tasks::TaskManager; @@ -731,7 +863,7 @@ mod tests { assert!(changed_acc.eq(&ChangedAccountEntry(copy))); } - const EXTENSION: &str = "rlp"; + const EXTENSION: &str = "json"; const FILENAME: &str = "test_transactions_backup"; #[tokio::test(flavor = "multi_thread")] @@ -741,7 +873,7 @@ mod tests { let tx_bytes = hex!( "02f87201830655c2808505ef61f08482565f94388c818ca8b9251b393131c08a736a67ccb192978801049e39c4b5b1f580c001a01764ace353514e8abdfb92446de356b260e3c1225b73fc4c8876a6258d12a129a04f02294aa61ca7676061cd99f29275491218b4754b46a0248e5e42bc5091f507" ); - let tx = PooledTransaction::decode_2718(&mut &tx_bytes[..]).unwrap(); + let tx = PooledTransactionVariant::decode_2718(&mut &tx_bytes[..]).unwrap(); let provider = MockEthProvider::default(); let transaction = EthPooledTransaction::from_pooled(tx.try_into_recovered().unwrap()); let tx_to_cmp = transaction.clone(); @@ -751,7 +883,7 @@ mod tests { let validator = EthTransactionValidatorBuilder::new(provider).build(blob_store.clone()); let txpool = Pool::new( - validator.clone(), + validator, CoinbaseTipOrdering::default(), blob_store.clone(), Default::default(), @@ -776,8 +908,7 @@ mod tests { let data = fs::read(transactions_path).unwrap(); - let txs: Vec = - alloy_rlp::Decodable::decode(&mut data.as_slice()).unwrap(); + let txs: Vec = serde_json::from_slice::>(&data).unwrap(); assert_eq!(txs.len(), 1); temp_dir.close().unwrap(); diff --git a/crates/transaction-pool/src/metrics.rs b/crates/transaction-pool/src/metrics.rs index 85a78663d24..d9926dafa02 100644 --- a/crates/transaction-pool/src/metrics.rs +++ b/crates/transaction-pool/src/metrics.rs @@ -140,3 +140,11 @@ pub struct TxPoolValidationMetrics { /// How long to successfully validate a blob pub(crate) blob_validation_duration: Histogram, } + +/// Transaction pool validator task metrics +#[derive(Metrics)] +#[metrics(scope = "transaction_pool")] +pub struct TxPoolValidatorMetrics { + /// Number of in-flight validation job sends waiting for channel capacity + pub(crate) inflight_validation_jobs: Gauge, +} diff --git a/crates/transaction-pool/src/noop.rs b/crates/transaction-pool/src/noop.rs index b254180f041..dc5bb9c307c 100644 --- a/crates/transaction-pool/src/noop.rs +++ b/crates/transaction-pool/src/noop.rs @@ -9,14 +9,15 @@ use crate::{ pool::TransactionListenerKind, traits::{BestTransactionsAttributes, GetPooledTransactionLimit, NewBlobSidecar}, validate::ValidTransaction, - AllPoolTransactions, AllTransactionsEvents, BestTransactions, BlockInfo, EthPoolTransaction, - EthPooledTransaction, NewTransactionEvent, PoolResult, PoolSize, PoolTransaction, - PropagatedTransactions, TransactionEvents, TransactionOrigin, TransactionPool, + AddedTransactionOutcome, AllPoolTransactions, AllTransactionsEvents, BestTransactions, + BlockInfo, EthPoolTransaction, EthPooledTransaction, NewTransactionEvent, PoolResult, PoolSize, + PoolTransaction, PropagatedTransactions, TransactionEvents, TransactionOrigin, TransactionPool, TransactionValidationOutcome, TransactionValidator, ValidPoolTransaction, }; use alloy_eips::{ eip1559::ETHEREUM_BLOCK_GAS_LIMIT_30M, - eip4844::{BlobAndProofV1, BlobTransactionSidecar}, + eip4844::{BlobAndProofV1, BlobAndProofV2}, + eip7594::BlobTransactionSidecarVariant, }; use alloy_primitives::{Address, TxHash, B256, U256}; use reth_eth_wire_types::HandleMempoolData; @@ -30,12 +31,12 @@ use tokio::sync::{mpsc, mpsc::Receiver}; /// This type will never hold any transactions and is only useful for wiring components together. #[derive(Debug, Clone)] #[non_exhaustive] -pub struct NoopTransactionPool { +pub struct NoopTransactionPool { /// Type marker _marker: PhantomData, } -impl NoopTransactionPool { +impl NoopTransactionPool { /// Creates a new [`NoopTransactionPool`]. pub fn new() -> Self { Self { _marker: Default::default() } @@ -78,7 +79,7 @@ impl TransactionPool for NoopTransactionPool { &self, _origin: TransactionOrigin, transaction: Self::Transaction, - ) -> PoolResult { + ) -> PoolResult { let hash = *transaction.hash(); Err(PoolError::other(hash, Box::new(NoopInsertError::new(transaction)))) } @@ -87,7 +88,7 @@ impl TransactionPool for NoopTransactionPool { &self, _origin: TransactionOrigin, transactions: Vec, - ) -> Vec> { + ) -> Vec> { transactions .into_iter() .map(|transaction| { @@ -97,6 +98,19 @@ impl TransactionPool for NoopTransactionPool { .collect() } + async fn add_transactions_with_origins( + &self, + transactions: Vec<(TransactionOrigin, Self::Transaction)>, + ) -> Vec> { + transactions + .into_iter() + .map(|(_, transaction)| { + let hash = *transaction.hash(); + Err(PoolError::other(hash, Box::new(NoopInsertError::new(transaction)))) + }) + .collect() + } + fn transaction_event_listener(&self, _tx_hash: TxHash) -> Option { None } @@ -189,10 +203,18 @@ impl TransactionPool for NoopTransactionPool { vec![] } + fn pending_and_queued_txn_count(&self) -> (usize, usize) { + (0, 0) + } + fn all_transactions(&self) -> AllPoolTransactions { AllPoolTransactions::default() } + fn all_transaction_hashes(&self) -> Vec { + vec![] + } + fn remove_transactions( &self, _hashes: Vec, @@ -302,33 +324,40 @@ impl TransactionPool for NoopTransactionPool { fn get_blob( &self, _tx_hash: TxHash, - ) -> Result>, BlobStoreError> { + ) -> Result>, BlobStoreError> { Ok(None) } fn get_all_blobs( &self, _tx_hashes: Vec, - ) -> Result)>, BlobStoreError> { + ) -> Result)>, BlobStoreError> { Ok(vec![]) } fn get_all_blobs_exact( &self, tx_hashes: Vec, - ) -> Result>, BlobStoreError> { + ) -> Result>, BlobStoreError> { if tx_hashes.is_empty() { return Ok(vec![]) } Err(BlobStoreError::MissingSidecar(tx_hashes[0])) } - fn get_blobs_for_versioned_hashes( + fn get_blobs_for_versioned_hashes_v1( &self, versioned_hashes: &[B256], ) -> Result>, BlobStoreError> { Ok(vec![None; versioned_hashes.len()]) } + + fn get_blobs_for_versioned_hashes_v2( + &self, + _versioned_hashes: &[B256], + ) -> Result>, BlobStoreError> { + Ok(None) + } } /// A [`TransactionValidator`] that does nothing. diff --git a/crates/transaction-pool/src/ordering.rs b/crates/transaction-pool/src/ordering.rs index c6554220336..be2a26f7cf2 100644 --- a/crates/transaction-pool/src/ordering.rs +++ b/crates/transaction-pool/src/ordering.rs @@ -1,5 +1,4 @@ use crate::traits::PoolTransaction; -use alloy_primitives::U256; use std::{cmp::Ordering, fmt::Debug, marker::PhantomData}; /// Priority of the transaction that can be missing. @@ -71,7 +70,7 @@ impl TransactionOrdering for CoinbaseTipOrdering where T: PoolTransaction + 'static, { - type PriorityValue = U256; + type PriorityValue = u128; type Transaction = T; /// Source: . @@ -82,7 +81,7 @@ where transaction: &Self::Transaction, base_fee: u64, ) -> Priority { - transaction.effective_tip_per_gas(base_fee).map(U256::from).into() + transaction.effective_tip_per_gas(base_fee).into() } } diff --git a/crates/transaction-pool/src/pool/best.rs b/crates/transaction-pool/src/pool/best.rs index eba3c2c35d0..90cd042df69 100644 --- a/crates/transaction-pool/src/pool/best.rs +++ b/crates/transaction-pool/src/pool/best.rs @@ -2,7 +2,7 @@ use crate::{ error::{Eip4844PoolTransactionError, InvalidPoolTransactionError}, identifier::{SenderId, TransactionId}, pool::pending::PendingTransaction, - PoolTransaction, TransactionOrdering, ValidPoolTransaction, + PoolTransaction, Priority, TransactionOrdering, ValidPoolTransaction, }; use alloy_consensus::Transaction; use alloy_eips::Typed2718; @@ -16,12 +16,15 @@ use std::{ use tokio::sync::broadcast::{error::TryRecvError, Receiver}; use tracing::debug; +const MAX_NEW_TRANSACTIONS_PER_BATCH: usize = 16; + /// An iterator that returns transactions that can be executed on the current state (*best* /// transactions). /// /// This is a wrapper around [`BestTransactions`] that also enforces a specific basefee. /// -/// This iterator guarantees that all transaction it returns satisfy both the base fee and blob fee! +/// This iterator guarantees that all transactions it returns satisfy both the base fee and blob +/// fee! pub(crate) struct BestTransactionsWithFees { pub(crate) best: BestTransactions, pub(crate) base_fee: u64, @@ -91,17 +94,21 @@ pub struct BestTransactions { /// There might be the case where a yielded transactions is invalid, this will track it. pub(crate) invalid: HashSet, /// Used to receive any new pending transactions that have been added to the pool after this - /// iterator was static fileted + /// iterator was static filtered /// /// These new pending transactions are inserted into this iterator's pool before yielding the /// next value pub(crate) new_transaction_receiver: Option>>, + /// The priority value of most recently yielded transaction. + /// + /// This is required if new pending transactions are fed in while it yields new values. + pub(crate) last_priority: Option>, /// Flag to control whether to skip blob transactions (EIP4844). pub(crate) skip_blobs: bool, } impl BestTransactions { - /// Mark the transaction and it's descendants as invalid. + /// Mark the transaction and its descendants as invalid. pub(crate) fn mark_invalid( &mut self, tx: &Arc>, @@ -113,7 +120,7 @@ impl BestTransactions { /// Returns the ancestor the given transaction, the transaction with `nonce - 1`. /// /// Note: for a transaction with nonce higher than the current on chain nonce this will always - /// return an ancestor since all transaction in this pool are gapless. + /// return an ancestor since all transactions in this pool are gapless. pub(crate) fn ancestor(&self, id: &TransactionId) -> Option<&PendingTransaction> { self.all.get(&id.unchecked_ancestor()?) } @@ -122,7 +129,16 @@ impl BestTransactions { fn try_recv(&mut self) -> Option> { loop { match self.new_transaction_receiver.as_mut()?.try_recv() { - Ok(tx) => return Some(tx), + Ok(tx) => { + if let Some(last_priority) = &self.last_priority && + &tx.priority > last_priority + { + // we skip transactions if we already yielded a transaction with lower + // priority + return None + } + return Some(tx) + } // note TryRecvError::Lagged can be returned here, which is an error that attempts // to correct itself on consecutive try_recv() attempts @@ -151,13 +167,17 @@ impl BestTransactions { /// Checks for new transactions that have come into the `PendingPool` after this iterator was /// created and inserts them fn add_new_transactions(&mut self) { - while let Some(pending_tx) = self.try_recv() { - // same logic as PendingPool::add_transaction/PendingPool::best_with_unlocked - let tx_id = *pending_tx.transaction.id(); - if self.ancestor(&tx_id).is_none() { - self.independent.insert(pending_tx.clone()); + for _ in 0..MAX_NEW_TRANSACTIONS_PER_BATCH { + if let Some(pending_tx) = self.try_recv() { + // same logic as PendingPool::add_transaction/PendingPool::best_with_unlocked + let tx_id = *pending_tx.transaction.id(); + if self.ancestor(&tx_id).is_none() { + self.independent.insert(pending_tx.clone()); + } + self.all.insert(tx_id, pending_tx); + } else { + break; } - self.all.insert(tx_id, pending_tx); } } } @@ -169,6 +189,7 @@ impl crate::traits::BestTransactions for BestTransaction fn no_updates(&mut self) { self.new_transaction_receiver.take(); + self.last_priority.take(); } fn skip_blobs(&mut self) { @@ -215,6 +236,9 @@ impl Iterator for BestTransactions { ), ) } else { + if self.new_transaction_receiver.is_some() { + self.last_priority = Some(best.priority.clone()) + } return Some(best.transaction) } } @@ -377,7 +401,6 @@ mod tests { test_utils::{MockOrdering, MockTransaction, MockTransactionFactory}, BestTransactions, Priority, }; - use alloy_primitives::U256; #[test] fn test_best_iter() { @@ -648,7 +671,7 @@ mod tests { let pending_tx = PendingTransaction { submission_id: 10, transaction: Arc::new(valid_new_tx.clone()), - priority: Priority::Value(U256::from(1000)), + priority: Priority::Value(1000), }; tx_sender.send(pending_tx.clone()).unwrap(); @@ -695,7 +718,7 @@ mod tests { let pending_tx1 = PendingTransaction { submission_id: 10, transaction: Arc::new(valid_new_tx1.clone()), - priority: Priority::Value(U256::from(1000)), + priority: Priority::Value(1000), }; tx_sender.send(pending_tx1.clone()).unwrap(); @@ -718,7 +741,7 @@ mod tests { let pending_tx2 = PendingTransaction { submission_id: 11, // Different submission ID transaction: Arc::new(valid_new_tx2.clone()), - priority: Priority::Value(U256::from(1000)), + priority: Priority::Value(1000), }; tx_sender.send(pending_tx2.clone()).unwrap(); @@ -755,7 +778,7 @@ mod tests { // Create a filter that only returns transactions with even nonces let filter = BestTransactionFilter::new(best, |tx: &Arc>| { - tx.nonce() % 2 == 0 + tx.nonce().is_multiple_of(2) }); // Verify that the filter only returns transactions with even nonces @@ -802,7 +825,7 @@ mod tests { assert_eq!(iter.next().unwrap().max_fee_per_gas(), (gas_price + 1) * 10); } - // Due to the gas limit, the transaction from second prioritized sender was not + // Due to the gas limit, the transaction from second-prioritized sender was not // prioritized. let top_of_block_tx2 = iter.next().unwrap(); assert_eq!(top_of_block_tx2.max_fee_per_gas(), 3); @@ -931,5 +954,47 @@ mod tests { assert!(best.new_transaction_receiver.is_none()); } - // TODO: Same nonce test + #[test] + fn test_best_update_transaction_priority() { + let mut pool = PendingPool::new(MockOrdering::default()); + let mut f = MockTransactionFactory::default(); + + // Add 5 transactions with increasing nonces to the pool + let num_tx = 5; + let tx = MockTransaction::eip1559(); + for nonce in 0..num_tx { + let tx = tx.clone().rng_hash().with_nonce(nonce); + let valid_tx = f.validated(tx); + pool.add_transaction(Arc::new(valid_tx), 0); + } + + // Create a BestTransactions iterator from the pool + let mut best = pool.best(); + + // Use a broadcast channel for transaction updates + let (tx_sender, tx_receiver) = + tokio::sync::broadcast::channel::>(1000); + best.new_transaction_receiver = Some(tx_receiver); + + // yield one tx, effectively locking in the highest prio + let first = best.next().unwrap(); + + // Create a new transaction with nonce 5 and validate it + let new_higher_fee_tx = MockTransaction::eip1559().with_nonce(0); + let valid_new_higher_fee_tx = f.validated(new_higher_fee_tx); + + // Send the new transaction through the broadcast channel + let pending_tx = PendingTransaction { + submission_id: 10, + transaction: Arc::new(valid_new_higher_fee_tx.clone()), + priority: Priority::Value(u128::MAX), + }; + tx_sender.send(pending_tx).unwrap(); + + // ensure that the higher prio tx is skipped since we yielded a lower one + for tx in best { + assert_eq!(tx.sender_id(), first.sender_id()); + assert_ne!(tx.sender_id(), valid_new_higher_fee_tx.sender_id()); + } + } } diff --git a/crates/transaction-pool/src/pool/blob.rs b/crates/transaction-pool/src/pool/blob.rs index b083c62816b..68fa3606a80 100644 --- a/crates/transaction-pool/src/pool/blob.rs +++ b/crates/transaction-pool/src/pool/blob.rs @@ -15,7 +15,7 @@ use std::{ /// worst blob transactions once the sub-pool is full. /// /// This expects that certain constraints are met: -/// - blob transactions are always gap less +/// - blob transactions are always gapless #[derive(Debug, Clone)] pub struct BlobTransactions { /// Keeps track of transactions inserted in the pool. @@ -83,7 +83,7 @@ impl BlobTransactions { /// Returns all transactions that satisfy the given basefee and blobfee. /// - /// Note: This does not remove any the transactions from the pool. + /// Note: This does not remove any of the transactions from the pool. pub(crate) fn satisfy_attributes( &self, best_transactions_attributes: BestTransactionsAttributes, @@ -584,7 +584,7 @@ mod tests { ], network_fees: PendingFees { base_fee: 0, blob_fee: 1999 }, }, - // If both basefee and blobfee is specified, sort by the larger distance + // If both basefee and blobfee are specified, sort by the larger distance // of the two from the current network conditions, splitting same (loglog) // ones via the tip. // diff --git a/crates/transaction-pool/src/pool/events.rs b/crates/transaction-pool/src/pool/events.rs index 0dc07e7ee96..f6bdd4a4d04 100644 --- a/crates/transaction-pool/src/pool/events.rs +++ b/crates/transaction-pool/src/pool/events.rs @@ -2,6 +2,7 @@ use crate::{traits::PropagateKind, PoolTransaction, SubPool, ValidPoolTransactio use alloy_primitives::{TxHash, B256}; use std::sync::Arc; +use crate::pool::QueuedReason; #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; @@ -11,7 +12,9 @@ pub enum FullTransactionEvent { /// Transaction has been added to the pending pool. Pending(TxHash), /// Transaction has been added to the queued pool. - Queued(TxHash), + /// + /// If applicable, attached the specific reason why this was queued. + Queued(TxHash, Option), /// Transaction has been included in the block belonging to this hash. Mined { /// The hash of the mined transaction. @@ -40,7 +43,7 @@ impl Clone for FullTransactionEvent { fn clone(&self) -> Self { match self { Self::Pending(hash) => Self::Pending(*hash), - Self::Queued(hash) => Self::Queued(*hash), + Self::Queued(hash, reason) => Self::Queued(*hash, reason.clone()), Self::Mined { tx_hash, block_hash } => { Self::Mined { tx_hash: *tx_hash, block_hash: *block_hash } } @@ -93,6 +96,13 @@ pub struct NewTransactionEvent { pub transaction: Arc>, } +impl NewTransactionEvent { + /// Creates a new event for a pending transaction. + pub const fn pending(transaction: Arc>) -> Self { + Self { subpool: SubPool::Pending, transaction } + } +} + impl Clone for NewTransactionEvent { fn clone(&self) -> Self { Self { subpool: self.subpool, transaction: self.transaction.clone() } diff --git a/crates/transaction-pool/src/pool/listener.rs b/crates/transaction-pool/src/pool/listener.rs index 2b5111b73be..123c6cf956a 100644 --- a/crates/transaction-pool/src/pool/listener.rs +++ b/crates/transaction-pool/src/pool/listener.rs @@ -1,7 +1,10 @@ //! Listeners for the transaction-pool use crate::{ - pool::events::{FullTransactionEvent, NewTransactionEvent, TransactionEvent}, + pool::{ + events::{FullTransactionEvent, NewTransactionEvent, TransactionEvent}, + QueuedReason, + }, traits::{NewBlobSidecar, PropagateKind}, PoolTransaction, ValidPoolTransaction, }; @@ -17,6 +20,7 @@ use tokio::sync::mpsc::{ self as mpsc, error::TrySendError, Receiver, Sender, UnboundedReceiver, UnboundedSender, }; use tracing::debug; + /// The size of the event channel used to propagate transaction events. const TX_POOL_EVENT_CHANNEL_SIZE: usize = 1024; @@ -29,6 +33,11 @@ pub struct TransactionEvents { } impl TransactionEvents { + /// Create a new instance of this stream. + pub const fn new(hash: TxHash, events: UnboundedReceiver) -> Self { + Self { hash, events } + } + /// The hash for this transaction pub const fn hash(&self) -> TxHash { self.hash @@ -110,6 +119,12 @@ impl PoolEventBroadcast { self.all_events_broadcaster.broadcast(pool_event); } + /// Returns true if no listeners are installed + #[inline] + pub(crate) fn is_empty(&self) -> bool { + self.all_events_broadcaster.is_empty() && self.broadcasters_by_hash.is_empty() + } + /// Create a new subscription for the given transaction hash. pub(crate) fn subscribe(&mut self, tx_hash: TxHash) -> TransactionEvents { let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); @@ -153,8 +168,12 @@ impl PoolEventBroadcast { } /// Notify listeners about a transaction that was added to the queued pool. - pub(crate) fn queued(&mut self, tx: &TxHash) { - self.broadcast_event(tx, TransactionEvent::Queued, FullTransactionEvent::Queued(*tx)); + pub(crate) fn queued(&mut self, tx: &TxHash, reason: Option) { + self.broadcast_event( + tx, + TransactionEvent::Queued, + FullTransactionEvent::Queued(*tx, reason), + ); } /// Notify listeners about a transaction that was propagated. @@ -167,6 +186,17 @@ impl PoolEventBroadcast { ); } + /// Notify listeners about all discarded transactions. + #[inline] + pub(crate) fn discarded_many(&mut self, discarded: &[Arc>]) { + if self.is_empty() { + return + } + for tx in discarded { + self.discarded(tx.hash()); + } + } + /// Notify listeners about a transaction that was discarded. pub(crate) fn discarded(&mut self, tx: &TxHash) { self.broadcast_event(tx, TransactionEvent::Discarded, FullTransactionEvent::Discarded(*tx)); @@ -210,6 +240,12 @@ impl AllPoolEventsBroadcaster { Err(TrySendError::Closed(_)) => false, }) } + + /// Returns true if there are no listeners installed. + #[inline] + const fn is_empty(&self) -> bool { + self.senders.is_empty() + } } /// All Sender half(s) of the event channels for a specific transaction. @@ -223,7 +259,7 @@ struct PoolEventBroadcaster { impl PoolEventBroadcaster { /// Returns `true` if there are no more listeners remaining. - fn is_empty(&self) -> bool { + const fn is_empty(&self) -> bool { self.senders.is_empty() } diff --git a/crates/transaction-pool/src/pool/mod.rs b/crates/transaction-pool/src/pool/mod.rs index 29a4ff8beb6..50d959a4757 100644 --- a/crates/transaction-pool/src/pool/mod.rs +++ b/crates/transaction-pool/src/pool/mod.rs @@ -94,7 +94,7 @@ use parking_lot::{Mutex, RwLock, RwLockReadGuard, RwLockWriteGuard}; use reth_eth_wire_types::HandleMempoolData; use reth_execution_types::ChangedAccount; -use alloy_eips::{eip4844::BlobTransactionSidecar, Typed2718}; +use alloy_eips::{eip7594::BlobTransactionSidecarVariant, Typed2718}; use reth_primitives_traits::Recovered; use rustc_hash::FxHashMap; use std::{collections::HashSet, fmt, sync::Arc, time::Instant}; @@ -113,7 +113,7 @@ mod best; mod blob; mod listener; mod parked; -pub(crate) mod pending; +pub mod pending; pub(crate) mod size; pub(crate) mod state; pub mod txpool; @@ -201,6 +201,11 @@ where self.identifiers.write().sender_id_or_create(addr) } + /// Returns the internal [`SenderId`]s for the given addresses. + pub fn get_sender_ids(&self, addrs: impl IntoIterator) -> Vec { + self.identifiers.write().sender_ids_or_create(addrs) + } + /// Returns all senders in the pool pub fn unique_senders(&self) -> HashSet

{ self.get_pool_data().unique_senders() @@ -278,7 +283,7 @@ where self.pool.read() } - /// Returns hashes of _all_ transactions in the pool. + /// Returns hashes of transactions in the pool that can be propagated. pub fn pooled_transactions_hashes(&self) -> Vec { self.get_pool_data() .all() @@ -288,12 +293,12 @@ where .collect() } - /// Returns _all_ transactions in the pool. + /// Returns transactions in the pool that can be propagated pub fn pooled_transactions(&self) -> Vec>> { self.get_pool_data().all().transactions_iter().filter(|tx| tx.propagate).cloned().collect() } - /// Returns only the first `max` transactions in the pool. + /// Returns only the first `max` transactions in the pool that can be propagated. pub fn pooled_transactions_max( &self, max: usize, @@ -336,7 +341,8 @@ where } } - /// Returns pooled transactions for the given transaction hashes. + /// Returns pooled transactions for the given transaction hashes that are allowed to be + /// propagated. pub fn get_pooled_transaction_elements( &self, tx_hashes: Vec, @@ -345,7 +351,7 @@ where where ::Transaction: EthPoolTransaction, { - let transactions = self.get_all(tx_hashes); + let transactions = self.get_all_propagatable(tx_hashes); let mut elements = Vec::with_capacity(transactions.len()); let mut size = 0; for transaction in transactions { @@ -409,14 +415,50 @@ where /// Performs account updates on the pool. /// /// This will either promote or discard transactions based on the new account state. + /// + /// This should be invoked when the pool drifted and accounts are updated manually pub fn update_accounts(&self, accounts: Vec) { let changed_senders = self.changed_senders(accounts.into_iter()); let UpdateOutcome { promoted, discarded } = self.pool.write().update_accounts(changed_senders); - let mut listener = self.event_listener.write(); - promoted.iter().for_each(|tx| listener.pending(tx.hash(), None)); - discarded.iter().for_each(|tx| listener.discarded(tx.hash())); + // Notify about promoted pending transactions (similar to notify_on_new_state) + if !promoted.is_empty() { + self.pending_transaction_listener.lock().retain_mut(|listener| { + let promoted_hashes = promoted.iter().filter_map(|tx| { + if listener.kind.is_propagate_only() && !tx.propagate { + None + } else { + Some(*tx.hash()) + } + }); + listener.send_all(promoted_hashes) + }); + + // in this case we should also emit promoted transactions in full + self.transaction_listener.lock().retain_mut(|listener| { + let promoted_txs = promoted.iter().filter_map(|tx| { + if listener.kind.is_propagate_only() && !tx.propagate { + None + } else { + Some(NewTransactionEvent::pending(tx.clone())) + } + }); + listener.send_all(promoted_txs) + }); + } + + { + let mut listener = self.event_listener.write(); + if !listener.is_empty() { + for tx in &promoted { + listener.pending(tx.hash(), None); + } + for tx in &discarded { + listener.discarded(tx.hash()); + } + } + } // This deletes outdated blob txs from the blob store, based on the account's nonce. This is // called during txpool maintenance when the pool drifted. @@ -432,7 +474,7 @@ where pool: &mut RwLockWriteGuard<'_, TxPool>, origin: TransactionOrigin, tx: TransactionValidationOutcome, - ) -> PoolResult { + ) -> PoolResult { match tx { TransactionValidationOutcome::Valid { balance, @@ -463,12 +505,12 @@ where propagate, timestamp: Instant::now(), origin, - authority_ids: authorities - .map(|auths| auths.iter().map(|auth| self.get_sender_id(*auth)).collect()), + authority_ids: authorities.map(|auths| self.get_sender_ids(auths)), }; let added = pool.add_transaction(tx, balance, state_nonce, bytecode_hash)?; let hash = *added.hash(); + let state = added.transaction_state(); // transaction was successfully inserted into the pool if let Some(sidecar) = maybe_sidecar { @@ -499,7 +541,7 @@ where // Notify listeners for _all_ transactions self.on_new_transaction(added.into_new_transaction_event()); - Ok(hash) + Ok(AddedTransactionOutcome { hash, state }) } TransactionValidationOutcome::Invalid(tx, err) => { let mut listener = self.event_listener.write(); @@ -529,22 +571,24 @@ where Ok(listener) } - /// Adds all transactions in the iterator to the pool, returning a list of results. + /// Adds all transactions in the iterator to the pool, each with its individual origin, + /// returning a list of results. /// /// Note: A large batch may lock the pool for a long time that blocks important operations /// like updating the pool on canonical state changes. The caller should consider having /// a max batch size to balance transaction insertions with other updates. - pub fn add_transactions( + pub fn add_transactions_with_origins( &self, - origin: TransactionOrigin, - transactions: impl IntoIterator>, - ) -> Vec> { - // Add the transactions and enforce the pool size limits in one write lock + transactions: impl IntoIterator< + Item = (TransactionOrigin, TransactionValidationOutcome), + >, + ) -> Vec> { + // Process all transactions in one write lock, maintaining individual origins let (mut added, discarded) = { let mut pool = self.pool.write(); let added = transactions .into_iter() - .map(|tx| self.add_transaction(&mut pool, origin, tx)) + .map(|(origin, tx)| self.add_transaction(&mut pool, origin, tx)) .collect::>(); // Enforce the pool size limits if at least one transaction was added successfully @@ -560,22 +604,18 @@ where if !discarded.is_empty() { // Delete any blobs associated with discarded blob transactions self.delete_discarded_blobs(discarded.iter()); + self.event_listener.write().discarded_many(&discarded); let discarded_hashes = discarded.into_iter().map(|tx| *tx.hash()).collect::>(); - { - let mut listener = self.event_listener.write(); - discarded_hashes.iter().for_each(|hash| listener.discarded(hash)); - } - // A newly added transaction may be immediately discarded, so we need to // adjust the result here for res in &mut added { - if let Ok(hash) = res { - if discarded_hashes.contains(hash) { - *res = Err(PoolError::new(*hash, PoolErrorKind::DiscardedOnInsert)) - } + if let Ok(AddedTransactionOutcome { hash, .. }) = res && + discarded_hashes.contains(hash) + { + *res = Err(PoolError::new(*hash, PoolErrorKind::DiscardedOnInsert)) } } } @@ -583,6 +623,19 @@ where added } + /// Adds all transactions in the iterator to the pool, returning a list of results. + /// + /// Note: A large batch may lock the pool for a long time that blocks important operations + /// like updating the pool on canonical state changes. The caller should consider having + /// a max batch size to balance transaction insertions with other updates. + pub fn add_transactions( + &self, + origin: TransactionOrigin, + transactions: impl IntoIterator>, + ) -> Vec> { + self.add_transactions_with_origins(transactions.into_iter().map(|tx| (origin, tx))) + } + /// Notify all listeners about a new pending transaction. fn on_new_pending_transaction(&self, pending: &AddedPendingTransaction) { let propagate_allowed = pending.is_propagate_allowed(); @@ -615,7 +668,7 @@ where } /// Notify all listeners about a blob sidecar for a newly inserted blob (eip4844) transaction. - fn on_new_blob_sidecar(&self, tx_hash: &TxHash, sidecar: &BlobTransactionSidecar) { + fn on_new_blob_sidecar(&self, tx_hash: &TxHash, sidecar: &BlobTransactionSidecarVariant) { let mut sidecar_listeners = self.blob_transaction_sidecar_listener.lock(); if sidecar_listeners.is_empty() { return @@ -661,25 +714,41 @@ where // broadcast specific transaction events let mut listener = self.event_listener.write(); - mined.iter().for_each(|tx| listener.mined(tx, block_hash)); - promoted.iter().for_each(|tx| listener.pending(tx.hash(), None)); - discarded.iter().for_each(|tx| listener.discarded(tx.hash())); + if !listener.is_empty() { + for tx in &mined { + listener.mined(tx, block_hash); + } + for tx in &promoted { + listener.pending(tx.hash(), None); + } + for tx in &discarded { + listener.discarded(tx.hash()); + } + } } /// Fire events for the newly added transaction if there are any. fn notify_event_listeners(&self, tx: &AddedTransaction) { let mut listener = self.event_listener.write(); + if listener.is_empty() { + // nothing to notify + return + } match tx { AddedTransaction::Pending(tx) => { let AddedPendingTransaction { transaction, promoted, discarded, replaced } = tx; listener.pending(transaction.hash(), replaced.clone()); - promoted.iter().for_each(|tx| listener.pending(tx.hash(), None)); - discarded.iter().for_each(|tx| listener.discarded(tx.hash())); + for tx in promoted { + listener.pending(tx.hash(), None); + } + for tx in discarded { + listener.discarded(tx.hash()); + } } - AddedTransaction::Parked { transaction, replaced, .. } => { - listener.queued(transaction.hash()); + AddedTransaction::Parked { transaction, replaced, queued_reason, .. } => { + listener.queued(transaction.hash(), queued_reason.clone()); if let Some(replaced) = replaced { listener.replaced(replaced.clone(), *transaction.hash()); } @@ -729,6 +798,11 @@ where } } + /// Returns _all_ transactions in the pool + pub fn all_transaction_hashes(&self) -> Vec { + self.get_pool_data().all().transactions_iter().map(|tx| *tx.hash()).collect() + } + /// Removes and returns all matching transactions from the pool. /// /// This behaves as if the transactions got discarded (_not_ mined), effectively introducing a @@ -742,9 +816,7 @@ where } let removed = self.pool.write().remove_transactions(hashes); - let mut listener = self.event_listener.write(); - - removed.iter().for_each(|tx| listener.discarded(tx.hash())); + self.event_listener.write().discarded_many(&removed); removed } @@ -762,7 +834,9 @@ where let mut listener = self.event_listener.write(); - removed.iter().for_each(|tx| listener.discarded(tx.hash())); + for tx in &removed { + listener.discarded(tx.hash()); + } removed } @@ -775,9 +849,7 @@ where let sender_id = self.get_sender_id(sender); let removed = self.pool.write().remove_transactions_by_sender(sender_id); - let mut listener = self.event_listener.write(); - - removed.iter().for_each(|tx| listener.discarded(tx.hash())); + self.event_listener.write().discarded_many(&removed); removed } @@ -876,7 +948,7 @@ where .collect() } - /// Returns all pending transactions filted by [`TransactionOrigin`] + /// Returns all pending transactions filtered by [`TransactionOrigin`] pub fn get_pending_transactions_by_origin( &self, origin: TransactionOrigin, @@ -894,6 +966,19 @@ where self.get_pool_data().get_all(txs).collect() } + /// Returns all the transactions belonging to the hashes that are propagatable. + /// + /// If no transaction exists, it is skipped. + fn get_all_propagatable( + &self, + txs: Vec, + ) -> Vec>> { + if txs.is_empty() { + return Vec::new() + } + self.get_pool_data().get_all(txs).filter(|tx| tx.propagate).collect() + } + /// Notify about propagated transactions. pub fn on_propagated(&self, txs: PropagatedTransactions) { if txs.0.is_empty() { @@ -901,7 +986,9 @@ where } let mut listener = self.event_listener.write(); - txs.0.into_iter().for_each(|(hash, peers)| listener.propagated(&hash, peers)) + if !listener.is_empty() { + txs.0.into_iter().for_each(|(hash, peers)| listener.propagated(&hash, peers)); + } } /// Number of transactions in the entire pool @@ -920,7 +1007,7 @@ where } /// Inserts a blob transaction into the blob store - fn insert_blob(&self, hash: TxHash, blob: BlobTransactionSidecar) { + fn insert_blob(&self, hash: TxHash, blob: BlobTransactionSidecarVariant) { debug!(target: "txpool", "[{:?}] storing blob sidecar", hash); if let Err(err) = self.blob_store.insert(hash, blob) { warn!(target: "txpool", %err, "[{:?}] failed to insert blob", hash); @@ -1070,6 +1157,8 @@ pub enum AddedTransaction { replaced: Option>>, /// The subpool it was moved to. subpool: SubPool, + /// The specific reason why the transaction is queued (if applicable). + queued_reason: Option, }, } @@ -1124,7 +1213,6 @@ impl AddedTransaction { } /// Returns the subpool this transaction was added to - #[cfg(test)] pub(crate) const fn subpool(&self) -> SubPool { match self { Self::Pending(_) => SubPool::Pending, @@ -1140,6 +1228,98 @@ impl AddedTransaction { Self::Parked { transaction, .. } => transaction.id(), } } + + /// Returns the queued reason if the transaction is parked with a queued reason. + pub(crate) const fn queued_reason(&self) -> Option<&QueuedReason> { + match self { + Self::Pending(_) => None, + Self::Parked { queued_reason, .. } => queued_reason.as_ref(), + } + } + + /// Returns the transaction state based on the subpool and queued reason. + pub(crate) fn transaction_state(&self) -> AddedTransactionState { + match self.subpool() { + SubPool::Pending => AddedTransactionState::Pending, + _ => { + // For non-pending transactions, use the queued reason directly from the + // AddedTransaction + if let Some(reason) = self.queued_reason() { + AddedTransactionState::Queued(reason.clone()) + } else { + // Fallback - this shouldn't happen with the new implementation + AddedTransactionState::Queued(QueuedReason::NonceGap) + } + } + } + } +} + +/// The specific reason why a transaction is queued (not ready for execution) +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum QueuedReason { + /// Transaction has a nonce gap - missing prior transactions + NonceGap, + /// Transaction has parked ancestors - waiting for other transactions to be mined + ParkedAncestors, + /// Sender has insufficient balance to cover the transaction cost + InsufficientBalance, + /// Transaction exceeds the block gas limit + TooMuchGas, + /// Transaction doesn't meet the base fee requirement + InsufficientBaseFee, + /// Transaction doesn't meet the blob fee requirement (EIP-4844) + InsufficientBlobFee, +} + +/// The state of a transaction when is was added to the pool +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum AddedTransactionState { + /// Ready for execution + Pending, + /// Not ready for execution due to a specific condition + Queued(QueuedReason), +} + +impl AddedTransactionState { + /// Returns whether the transaction was submitted as queued. + pub const fn is_queued(&self) -> bool { + matches!(self, Self::Queued(_)) + } + + /// Returns whether the transaction was submitted as pending. + pub const fn is_pending(&self) -> bool { + matches!(self, Self::Pending) + } + + /// Returns the specific queued reason if the transaction is queued. + pub const fn queued_reason(&self) -> Option<&QueuedReason> { + match self { + Self::Queued(reason) => Some(reason), + Self::Pending => None, + } + } +} + +/// The outcome of a successful transaction addition +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AddedTransactionOutcome { + /// The hash of the transaction + pub hash: TxHash, + /// The state of the transaction + pub state: AddedTransactionState, +} + +impl AddedTransactionOutcome { + /// Returns whether the transaction was submitted as queued. + pub const fn is_queued(&self) -> bool { + self.state.is_queued() + } + + /// Returns whether the transaction was submitted as pending. + pub const fn is_pending(&self) -> bool { + self.state.is_pending() + } } /// Contains all state changes after a [`CanonicalStateUpdate`] was processed @@ -1187,11 +1367,13 @@ impl OnNewCanonicalStateOutcome { mod tests { use crate::{ blobstore::{BlobStore, InMemoryBlobStore}, + identifier::SenderId, test_utils::{MockTransaction, TestPoolBuilder}, validate::ValidTransaction, BlockInfo, PoolConfig, SubPoolLimit, TransactionOrigin, TransactionValidationOutcome, U256, }; - use alloy_eips::eip4844::BlobTransactionSidecar; + use alloy_eips::{eip4844::BlobTransactionSidecar, eip7594::BlobTransactionSidecarVariant}; + use alloy_primitives::Address; use std::{fs, path::PathBuf}; #[test] @@ -1220,7 +1402,9 @@ mod tests { }; // Generate a BlobTransactionSidecar from the blobs. - let sidecar = BlobTransactionSidecar::try_from_blobs_hex(blobs).unwrap(); + let sidecar = BlobTransactionSidecarVariant::Eip4844( + BlobTransactionSidecar::try_from_blobs_hex(blobs).unwrap(), + ); // Define the maximum limit for blobs in the sub-pool. let blob_limit = SubPoolLimit::new(1000, usize::MAX); @@ -1276,4 +1460,28 @@ mod tests { // Assert that the pool's blob store matches the expected blob store. assert_eq!(*test_pool.blob_store(), blob_store); } + + #[test] + fn test_auths_stored_in_identifiers() { + // Create a test pool with default configuration. + let test_pool = &TestPoolBuilder::default().with_config(Default::default()).pool; + + let auth = Address::new([1; 20]); + let tx = MockTransaction::eip7702(); + + test_pool.add_transactions( + TransactionOrigin::Local, + [TransactionValidationOutcome::Valid { + balance: U256::from(1_000), + state_nonce: 0, + bytecode_hash: None, + transaction: ValidTransaction::Valid(tx), + propagate: true, + authorities: Some(vec![auth]), + }], + ); + + let identifiers = test_pool.identifiers.read(); + assert_eq!(identifiers.sender_id(&auth), Some(SenderId::from(1))); + } } diff --git a/crates/transaction-pool/src/pool/parked.rs b/crates/transaction-pool/src/pool/parked.rs index 33056dd6ec5..193442174ca 100644 --- a/crates/transaction-pool/src/pool/parked.rs +++ b/crates/transaction-pool/src/pool/parked.rs @@ -16,9 +16,6 @@ use std::{ /// basefee, ancestor transactions, balance) that eventually move the transaction into the pending /// pool. /// -/// This pool is a bijection: at all times each set (`best`, `by_id`) contains the same -/// transactions. -/// /// Note: This type is generic over [`ParkedPool`] which enforces that the underlying transaction /// type is [`ValidPoolTransaction`] wrapped in an [Arc]. #[derive(Debug, Clone)] @@ -29,10 +26,6 @@ pub struct ParkedPool { submission_id: u64, /// _All_ Transactions that are currently inside the pool grouped by their identifier. by_id: BTreeMap>, - /// All transactions sorted by their order function. - /// - /// The higher, the better. - best: BTreeSet>, /// Keeps track of last submission id for each sender. /// /// This are sorted in reverse order, so the last (highest) submission id is first, and the @@ -51,13 +44,9 @@ pub struct ParkedPool { impl ParkedPool { /// Adds a new transactions to the pending queue. - /// - /// # Panics - /// - /// If the transaction is already included. pub fn add_transaction(&mut self, tx: Arc>) { let id = *tx.id(); - assert!( + debug_assert!( !self.contains(&id), "transaction already included {:?}", self.get(&id).unwrap().transaction.transaction @@ -71,8 +60,7 @@ impl ParkedPool { self.add_sender_count(tx.sender_id(), submission_id); let transaction = ParkedPoolTransaction { submission_id, transaction: tx.into() }; - self.by_id.insert(id, transaction.clone()); - self.best.insert(transaction); + self.by_id.insert(id, transaction); } /// Increments the count of transactions for the given sender and updates the tracked submission @@ -131,7 +119,7 @@ impl ParkedPool { /// Returns an iterator over all transactions in the pool pub(crate) fn all( &self, - ) -> impl Iterator>> + '_ { + ) -> impl ExactSizeIterator>> + '_ { self.by_id.values().map(|tx| tx.transaction.clone().into()) } @@ -142,7 +130,6 @@ impl ParkedPool { ) -> Option>> { // remove from queues let tx = self.by_id.remove(id)?; - self.best.remove(&tx); self.remove_sender_count(tx.transaction.sender_id()); // keep track of size @@ -195,10 +182,10 @@ impl ParkedPool { let mut removed = Vec::new(); - while limit.is_exceeded(self.len(), self.size()) && !self.last_sender_submission.is_empty() + while !self.last_sender_submission.is_empty() && limit.is_exceeded(self.len(), self.size()) { // NOTE: This will not panic due to `!last_sender_transaction.is_empty()` - let sender_id = self.last_sender_submission.last().expect("not empty").sender_id; + let sender_id = self.last_sender_submission.last().unwrap().sender_id; let list = self.get_txs_by_sender(sender_id); // Drop transactions from this sender until the pool is under limits @@ -254,15 +241,13 @@ impl ParkedPool { self.by_id.get(id) } - /// Asserts that the bijection between `by_id` and `best` is valid. + /// Asserts that all subpool invariants #[cfg(any(test, feature = "test-utils"))] pub(crate) fn assert_invariants(&self) { - assert_eq!(self.by_id.len(), self.best.len(), "by_id.len() != best.len()"); - assert_eq!( self.last_sender_submission.len(), self.sender_transaction_count.len(), - "last_sender_transaction.len() != sender_to_last_transaction.len()" + "last_sender_submission.len() != sender_transaction_count.len()" ); } } @@ -275,49 +260,75 @@ impl ParkedPool> { &self, basefee: u64, ) -> Vec>> { - let ids = self.satisfy_base_fee_ids(basefee); - let mut txs = Vec::with_capacity(ids.len()); - for id in ids { - txs.push(self.get(&id).expect("transaction exists").transaction.clone().into()); - } + let mut txs = Vec::new(); + self.satisfy_base_fee_ids(basefee as u128, |tx| { + txs.push(tx.clone()); + }); txs } /// Returns all transactions that satisfy the given basefee. - fn satisfy_base_fee_ids(&self, basefee: u64) -> Vec { - let mut transactions = Vec::new(); - { - let mut iter = self.by_id.iter().peekable(); - - while let Some((id, tx)) = iter.next() { - if tx.transaction.transaction.max_fee_per_gas() < basefee as u128 { - // still parked -> skip descendant transactions - 'this: while let Some((peek, _)) = iter.peek() { - if peek.sender != id.sender { - break 'this - } - iter.next(); + fn satisfy_base_fee_ids(&self, basefee: u128, mut tx_handler: F) + where + F: FnMut(&Arc>), + { + let mut iter = self.by_id.iter().peekable(); + + while let Some((id, tx)) = iter.next() { + if tx.transaction.transaction.max_fee_per_gas() < basefee { + // still parked -> skip descendant transactions + 'this: while let Some((peek, _)) = iter.peek() { + if peek.sender != id.sender { + break 'this } - } else { - transactions.push(*id); + iter.next(); } + } else { + tx_handler(&tx.transaction); } } - transactions } - /// Removes all transactions and their dependent transaction from the subpool that no longer - /// satisfy the given basefee. + /// Removes all transactions from this subpool that can afford the given basefee, + /// invoking the provided handler for each transaction as it is removed. + /// + /// This method enforces the basefee constraint by identifying transactions that now + /// satisfy the basefee requirement (typically after a basefee decrease) and processing + /// them via the provided transaction handler closure. + /// + /// Respects per-sender nonce ordering: if the lowest-nonce transaction for a sender + /// still cannot afford the basefee, higher-nonce transactions from that sender are skipped. /// /// Note: the transactions are not returned in a particular order. - pub(crate) fn enforce_basefee(&mut self, basefee: u64) -> Vec>> { - let to_remove = self.satisfy_base_fee_ids(basefee); + pub(crate) fn enforce_basefee_with(&mut self, basefee: u64, mut tx_handler: F) + where + F: FnMut(Arc>), + { + let mut to_remove = Vec::new(); + self.satisfy_base_fee_ids(basefee as u128, |tx| { + to_remove.push(*tx.id()); + }); - let mut removed = Vec::with_capacity(to_remove.len()); for id in to_remove { - removed.push(self.remove_transaction(&id).expect("transaction exists")); + if let Some(tx) = self.remove_transaction(&id) { + tx_handler(tx); + } } + } + /// Removes all transactions and their dependent transaction from the subpool that no longer + /// satisfy the given basefee. + /// + /// Legacy method maintained for compatibility with read-only queries. + /// For basefee enforcement, prefer `enforce_basefee_with` for better performance. + /// + /// Note: the transactions are not returned in a particular order. + #[cfg(test)] + pub(crate) fn enforce_basefee(&mut self, basefee: u64) -> Vec>> { + let mut removed = Vec::new(); + self.enforce_basefee_with(basefee, |tx| { + removed.push(tx); + }); removed } } @@ -327,7 +338,6 @@ impl Default for ParkedPool { Self { submission_id: 0, by_id: Default::default(), - best: Default::default(), last_sender_submission: Default::default(), sender_transaction_count: Default::default(), size_of: Default::default(), @@ -1053,48 +1063,66 @@ mod tests { } #[test] - fn test_parkpool_ord() { + fn test_enforce_basefee_with_handler_zero_allocation() { let mut f = MockTransactionFactory::default(); - let mut pool = ParkedPool::>::default(); - - let tx1 = MockTransaction::eip1559().with_max_fee(100); - let tx1_v = f.validated_arc(tx1.clone()); - - let tx2 = MockTransaction::eip1559().with_max_fee(101); - let tx2_v = f.validated_arc(tx2.clone()); - - let tx3 = MockTransaction::eip1559().with_max_fee(101); - let tx3_v = f.validated_arc(tx3.clone()); - - let tx4 = MockTransaction::eip1559().with_max_fee(101); - let mut tx4_v = f.validated(tx4.clone()); - tx4_v.timestamp = tx3_v.timestamp; + let mut pool = ParkedPool::>::default(); - let ord_1 = QueuedOrd(tx1_v.clone()); - let ord_2 = QueuedOrd(tx2_v.clone()); - let ord_3 = QueuedOrd(tx3_v.clone()); - assert!(ord_1 < ord_2); - // lower timestamp is better - assert!(ord_2 > ord_3); - assert!(ord_1 < ord_3); + // Add multiple transactions across different fee ranges + let sender_a = address!("0x000000000000000000000000000000000000000a"); + let sender_b = address!("0x000000000000000000000000000000000000000b"); + + // Add transactions where nonce ordering allows proper processing: + // Sender A: both transactions can afford basefee (500 >= 400, 600 >= 400) + // Sender B: transaction cannot afford basefee (300 < 400) + let txs = vec![ + f.validated_arc( + MockTransaction::eip1559() + .set_sender(sender_a) + .set_nonce(0) + .set_max_fee(500) + .clone(), + ), + f.validated_arc( + MockTransaction::eip1559() + .set_sender(sender_a) + .set_nonce(1) + .set_max_fee(600) + .clone(), + ), + f.validated_arc( + MockTransaction::eip1559() + .set_sender(sender_b) + .set_nonce(0) + .set_max_fee(300) + .clone(), + ), + ]; + + let expected_affordable = vec![txs[0].clone(), txs[1].clone()]; // Both sender A txs + for tx in txs { + pool.add_transaction(tx); + } - pool.add_transaction(tx1_v); - pool.add_transaction(tx2_v); - pool.add_transaction(tx3_v); - pool.add_transaction(Arc::new(tx4_v)); + // Test the handler approach with zero allocations + let mut processed_txs = Vec::new(); + let mut handler_call_count = 0; - // from worst to best - let mut iter = pool.best.iter(); - let tx = iter.next().unwrap(); - assert_eq!(tx.transaction.transaction, tx1); + pool.enforce_basefee_with(400, |tx| { + processed_txs.push(tx); + handler_call_count += 1; + }); - let tx = iter.next().unwrap(); - assert_eq!(tx.transaction.transaction, tx4); + // Verify correct number of transactions processed + assert_eq!(handler_call_count, 2); + assert_eq!(processed_txs.len(), 2); - let tx = iter.next().unwrap(); - assert_eq!(tx.transaction.transaction, tx3); + // Verify the correct transactions were processed (those with fee >= 400) + let processed_ids: Vec<_> = processed_txs.iter().map(|tx| *tx.id()).collect(); + for expected_tx in expected_affordable { + assert!(processed_ids.contains(expected_tx.id())); + } - let tx = iter.next().unwrap(); - assert_eq!(tx.transaction.transaction, tx2); + // Verify transactions were removed from pool + assert_eq!(pool.len(), 1); // Only the 300 fee tx should remain } } diff --git a/crates/transaction-pool/src/pool/pending.rs b/crates/transaction-pool/src/pool/pending.rs index 34bbef1d477..dc675031ea6 100644 --- a/crates/transaction-pool/src/pool/pending.rs +++ b/crates/transaction-pool/src/pool/pending.rs @@ -1,3 +1,5 @@ +//! Pending transactions + use crate::{ identifier::{SenderId, TransactionId}, pool::{ @@ -99,7 +101,7 @@ impl PendingPool { /// time in pool (were added earlier) are returned first. /// /// NOTE: while this iterator returns transaction that pool considers valid at this point, they - /// could potentially be become invalid at point of execution. Therefore, this iterator + /// could potentially become invalid at point of execution. Therefore, this iterator /// provides a way to mark transactions that the consumer of this iterator considers invalid. In /// which case the transaction's subgraph is also automatically marked invalid, See (1.). /// Invalid transactions are skipped. @@ -109,6 +111,7 @@ impl PendingPool { independent: self.independent_transactions.values().cloned().collect(), invalid: Default::default(), new_transaction_receiver: Some(self.new_transaction_notifier.subscribe()), + last_priority: None, skip_blobs: false, } } @@ -158,7 +161,7 @@ impl PendingPool { /// Returns an iterator over all transactions in the pool pub(crate) fn all( &self, - ) -> impl Iterator>> + '_ { + ) -> impl ExactSizeIterator>> + '_ { self.by_id.values().map(|tx| tx.transaction.clone()) } @@ -181,7 +184,8 @@ impl PendingPool { // Drain and iterate over all transactions. let mut transactions_iter = self.clear_transactions().into_iter().peekable(); while let Some((id, tx)) = transactions_iter.next() { - if tx.transaction.max_fee_per_blob_gas() < Some(blob_fee) { + if tx.transaction.is_eip4844() && tx.transaction.max_fee_per_blob_gas() < Some(blob_fee) + { // Add this tx to the removed collection since it no longer satisfies the blob fee // condition. Decrease the total pool size. removed.push(Arc::clone(&tx.transaction)); @@ -274,14 +278,6 @@ impl PendingPool { } } - /// Returns the ancestor the given transaction, the transaction with `nonce - 1`. - /// - /// Note: for a transaction with nonce higher than the current on chain nonce this will always - /// return an ancestor since all transaction in this pool are gapless. - fn ancestor(&self, id: &TransactionId) -> Option<&PendingTransaction> { - self.get(&id.unchecked_ancestor()?) - } - /// Adds a new transactions to the pending queue. /// /// # Panics @@ -292,7 +288,7 @@ impl PendingPool { tx: Arc>, base_fee: u64, ) { - assert!( + debug_assert!( !self.contains(tx.id()), "transaction already included {:?}", self.get(tx.id()).unwrap().transaction @@ -325,27 +321,48 @@ impl PendingPool { &mut self, id: &TransactionId, ) -> Option>> { - if let Some(lowest) = self.independent_transactions.get(&id.sender) { - if lowest.transaction.nonce() == id.nonce { - self.independent_transactions.remove(&id.sender); - // mark the next as independent if it exists - if let Some(unlocked) = self.get(&id.descendant()) { - self.independent_transactions.insert(id.sender, unlocked.clone()); - } + if let Some(lowest) = self.independent_transactions.get(&id.sender) && + lowest.transaction.nonce() == id.nonce + { + self.independent_transactions.remove(&id.sender); + // mark the next as independent if it exists + if let Some(unlocked) = self.get(&id.descendant()) { + self.independent_transactions.insert(id.sender, unlocked.clone()); } } let tx = self.by_id.remove(id)?; self.size_of -= tx.transaction.size(); - if let Some(highest) = self.highest_nonces.get(&id.sender) { - if highest.transaction.nonce() == id.nonce { - self.highest_nonces.remove(&id.sender); + match self.highest_nonces.entry(id.sender) { + Entry::Occupied(mut entry) => { + if entry.get().transaction.nonce() == id.nonce { + // we just removed the tx with the highest nonce for this sender, find the + // highest remaining tx from that sender + if let Some((_, new_highest)) = self + .by_id + .range(( + id.sender.start_bound(), + std::ops::Bound::Included(TransactionId::new(id.sender, u64::MAX)), + )) + .last() + { + // insert the new highest nonce for this sender + entry.insert(new_highest.clone()); + } else { + entry.remove(); + } + } } - if let Some(ancestor) = self.ancestor(id) { - self.highest_nonces.insert(id.sender, ancestor.clone()); + Entry::Vacant(_) => { + debug_assert!( + false, + "removed transaction without a tracked highest nonce {:?}", + id + ); } } + Some(tx.transaction) } @@ -509,6 +526,21 @@ impl PendingPool { self.by_id.len() } + /// All transactions grouped by id + pub const fn by_id(&self) -> &BTreeMap> { + &self.by_id + } + + /// Independent transactions + pub const fn independent_transactions(&self) -> &FxHashMap> { + &self.independent_transactions + } + + /// Subscribes to new transactions + pub fn new_transaction_receiver(&self) -> broadcast::Receiver> { + self.new_transaction_notifier.subscribe() + } + /// Whether the pool is empty #[cfg(test)] pub(crate) fn is_empty(&self) -> bool { @@ -522,11 +554,18 @@ impl PendingPool { /// Get transactions by sender pub(crate) fn get_txs_by_sender(&self, sender: SenderId) -> Vec { + self.iter_txs_by_sender(sender).copied().collect() + } + + /// Returns an iterator over all transaction with the sender id + pub(crate) fn iter_txs_by_sender( + &self, + sender: SenderId, + ) -> impl Iterator + '_ { self.by_id .range((sender.start_bound(), Unbounded)) .take_while(move |(other, _)| sender == other.sender) - .map(|(tx_id, _)| *tx_id) - .collect() + .map(|(tx_id, _)| tx_id) } /// Retrieves a transaction with the given ID from the pool, if it exists. @@ -545,34 +584,34 @@ impl PendingPool { pub(crate) fn assert_invariants(&self) { assert!( self.independent_transactions.len() <= self.by_id.len(), - "independent.len() > all.len()" + "independent_transactions.len() > by_id.len()" ); assert!( self.highest_nonces.len() <= self.by_id.len(), - "independent_descendants.len() > all.len()" + "highest_nonces.len() > by_id.len()" ); assert_eq!( self.highest_nonces.len(), self.independent_transactions.len(), - "independent.len() = independent_descendants.len()" + "highest_nonces.len() != independent_transactions.len()" ); } } /// A transaction that is ready to be included in a block. #[derive(Debug)] -pub(crate) struct PendingTransaction { +pub struct PendingTransaction { /// Identifier that tags when transaction was submitted in the pool. - pub(crate) submission_id: u64, + pub submission_id: u64, /// Actual transaction. - pub(crate) transaction: Arc>, + pub transaction: Arc>, /// The priority value assigned by the used `Ordering` function. - pub(crate) priority: Priority, + pub priority: Priority, } impl PendingTransaction { /// The next transaction of the sender: `nonce + 1` - pub(crate) fn unlocks(&self) -> TransactionId { + pub fn unlocks(&self) -> TransactionId { self.transaction.transaction_id.descendant() } } @@ -739,7 +778,7 @@ mod tests { // the independent set is the roots of each of these tx chains, these are the highest // nonces for each sender - let expected_highest_nonces = vec![d[0].clone(), c[2].clone(), b[2].clone(), a[3].clone()] + let expected_highest_nonces = [d[0].clone(), c[2].clone(), b[2].clone(), a[3].clone()] .iter() .map(|tx| (tx.sender(), tx.nonce())) .collect::>(); @@ -895,8 +934,7 @@ mod tests { assert!(removed.is_empty()); // Verify that retrieving transactions from an empty pool yields nothing - let all_txs: Vec<_> = pool.all().collect(); - assert!(all_txs.is_empty()); + assert!(pool.all().next().is_none()); } #[test] @@ -1029,4 +1067,61 @@ mod tests { assert!(pool.get_txs_by_sender(sender_b).is_empty()); assert!(pool.get_txs_by_sender(sender_c).is_empty()); } + + #[test] + fn test_remove_non_highest_keeps_highest() { + let mut f = MockTransactionFactory::default(); + let mut pool = PendingPool::new(MockOrdering::default()); + let sender = address!("0x00000000000000000000000000000000000000aa"); + let txs = MockTransactionSet::dependent(sender, 0, 3, TxType::Eip1559).into_vec(); + for tx in txs { + pool.add_transaction(f.validated_arc(tx), 0); + } + pool.assert_invariants(); + let sender_id = f.ids.sender_id(&sender).unwrap(); + let mid_id = TransactionId::new(sender_id, 1); + let _ = pool.remove_transaction(&mid_id); + let highest = pool.highest_nonces.get(&sender_id).unwrap(); + assert_eq!(highest.transaction.nonce(), 2); + pool.assert_invariants(); + } + + #[test] + fn test_cascade_removal_recomputes_highest() { + let mut f = MockTransactionFactory::default(); + let mut pool = PendingPool::new(MockOrdering::default()); + let sender = address!("0x00000000000000000000000000000000000000bb"); + let txs = MockTransactionSet::dependent(sender, 0, 4, TxType::Eip1559).into_vec(); + for tx in txs { + pool.add_transaction(f.validated_arc(tx), 0); + } + pool.assert_invariants(); + let sender_id = f.ids.sender_id(&sender).unwrap(); + let id3 = TransactionId::new(sender_id, 3); + let _ = pool.remove_transaction(&id3); + let highest = pool.highest_nonces.get(&sender_id).unwrap(); + assert_eq!(highest.transaction.nonce(), 2); + let id2 = TransactionId::new(sender_id, 2); + let _ = pool.remove_transaction(&id2); + let highest = pool.highest_nonces.get(&sender_id).unwrap(); + assert_eq!(highest.transaction.nonce(), 1); + pool.assert_invariants(); + } + + #[test] + fn test_remove_only_tx_clears_highest() { + let mut f = MockTransactionFactory::default(); + let mut pool = PendingPool::new(MockOrdering::default()); + let sender = address!("0x00000000000000000000000000000000000000cc"); + let txs = MockTransactionSet::dependent(sender, 0, 1, TxType::Eip1559).into_vec(); + for tx in txs { + pool.add_transaction(f.validated_arc(tx), 0); + } + pool.assert_invariants(); + let sender_id = f.ids.sender_id(&sender).unwrap(); + let id0 = TransactionId::new(sender_id, 0); + let _ = pool.remove_transaction(&id0); + assert!(!pool.highest_nonces.contains_key(&sender_id)); + pool.assert_invariants(); + } } diff --git a/crates/transaction-pool/src/pool/state.rs b/crates/transaction-pool/src/pool/state.rs index d65fc05b03f..187d472f5ae 100644 --- a/crates/transaction-pool/src/pool/state.rs +++ b/crates/transaction-pool/src/pool/state.rs @@ -1,3 +1,5 @@ +use crate::pool::QueuedReason; + bitflags::bitflags! { /// Marker to represents the current state of a transaction in the pool and from which the corresponding sub-pool is derived, depending on what bits are set. /// @@ -14,7 +16,7 @@ bitflags::bitflags! { pub(crate) struct TxState: u8 { /// Set to `1` if all ancestor transactions are pending. const NO_PARKED_ANCESTORS = 0b10000000; - /// Set to `1` of the transaction is either the next transaction of the sender (on chain nonce == tx.nonce) or all prior transactions are also present in the pool. + /// Set to `1` if the transaction is either the next transaction of the sender (on chain nonce == tx.nonce) or all prior transactions are also present in the pool. const NO_NONCE_GAPS = 0b01000000; /// Bit derived from the sender's balance. /// @@ -68,6 +70,56 @@ impl TxState { pub(crate) const fn has_nonce_gap(&self) -> bool { !self.intersects(Self::NO_NONCE_GAPS) } + + /// Adds the transaction into the pool. + /// + /// This pool consists of four sub-pools: `Queued`, `Pending`, `BaseFee`, and `Blob`. + /// + /// The `Queued` pool contains transactions with gaps in its dependency tree: It requires + /// additional transactions that are note yet present in the pool. And transactions that the + /// sender can not afford with the current balance. + /// + /// The `Pending` pool contains all transactions that have no nonce gaps, and can be afforded by + /// the sender. It only contains transactions that are ready to be included in the pending + /// block. The pending pool contains all transactions that could be listed currently, but not + /// necessarily independently. However, this pool never contains transactions with nonce gaps. A + /// transaction is considered `ready` when it has the lowest nonce of all transactions from the + /// same sender. Which is equals to the chain nonce of the sender in the pending pool. + /// + /// The `BaseFee` pool contains transactions that currently can't satisfy the dynamic fee + /// requirement. With EIP-1559, transactions can become executable or not without any changes to + /// the sender's balance or nonce and instead their `feeCap` determines whether the + /// transaction is _currently_ (on the current state) ready or needs to be parked until the + /// `feeCap` satisfies the block's `baseFee`. + /// + /// The `Blob` pool contains _blob_ transactions that currently can't satisfy the dynamic fee + /// requirement, or blob fee requirement. Transactions become executable only if the + /// transaction `feeCap` is greater than the block's `baseFee` and the `maxBlobFee` is greater + /// than the block's `blobFee`. + /// + /// Determines the specific reason why a transaction is queued based on its subpool and state. + pub(crate) const fn determine_queued_reason(&self, subpool: SubPool) -> Option { + match subpool { + SubPool::Pending => None, // Not queued + SubPool::Queued => { + // Check state flags to determine specific reason + if !self.contains(Self::NO_NONCE_GAPS) { + Some(QueuedReason::NonceGap) + } else if !self.contains(Self::ENOUGH_BALANCE) { + Some(QueuedReason::InsufficientBalance) + } else if !self.contains(Self::NO_PARKED_ANCESTORS) { + Some(QueuedReason::ParkedAncestors) + } else if !self.contains(Self::NOT_TOO_MUCH_GAS) { + Some(QueuedReason::TooMuchGas) + } else { + // Fallback for unexpected queued state + Some(QueuedReason::NonceGap) + } + } + SubPool::BaseFee => Some(QueuedReason::InsufficientBaseFee), + SubPool::Blob => Some(QueuedReason::InsufficientBlobFee), + } + } } /// Identifier for the transaction Sub-pool diff --git a/crates/transaction-pool/src/pool/txpool.rs b/crates/transaction-pool/src/pool/txpool.rs index 2b587036aaa..49247dc8b8c 100644 --- a/crates/transaction-pool/src/pool/txpool.rs +++ b/crates/transaction-pool/src/pool/txpool.rs @@ -40,7 +40,7 @@ use std::{ ops::Bound::{Excluded, Unbounded}, sync::Arc, }; -use tracing::trace; +use tracing::{trace, warn}; #[cfg_attr(doc, aquamarine::aquamarine)] // TODO: Inlined diagram due to a bug in aquamarine library, should become an include when it's @@ -221,7 +221,14 @@ impl TxPool { } /// Updates the tracked blob fee - fn update_blob_fee(&mut self, mut pending_blob_fee: u128, base_fee_update: Ordering) { + fn update_blob_fee( + &mut self, + mut pending_blob_fee: u128, + base_fee_update: Ordering, + mut on_promoted: F, + ) where + F: FnMut(&Arc>), + { std::mem::swap(&mut self.all_transactions.pending_fees.blob_fee, &mut pending_blob_fee); match (self.all_transactions.pending_fees.blob_fee.cmp(&pending_blob_fee), base_fee_update) { @@ -250,15 +257,20 @@ impl TxPool { let removed = self.blob_pool.enforce_pending_fees(&self.all_transactions.pending_fees); for tx in removed { - let to = { - let tx = + let subpool = { + let tx_meta = self.all_transactions.txs.get_mut(tx.id()).expect("tx exists in set"); - tx.state.insert(TxState::ENOUGH_BLOB_FEE_CAP_BLOCK); - tx.state.insert(TxState::ENOUGH_FEE_CAP_BLOCK); - tx.subpool = tx.state.into(); - tx.subpool + tx_meta.state.insert(TxState::ENOUGH_BLOB_FEE_CAP_BLOCK); + tx_meta.state.insert(TxState::ENOUGH_FEE_CAP_BLOCK); + tx_meta.subpool = tx_meta.state.into(); + tx_meta.subpool }; - self.add_transaction_to_subpool(to, tx); + + if subpool == SubPool::Pending { + on_promoted(&tx); + } + + self.add_transaction_to_subpool(subpool, tx); } } } @@ -268,7 +280,10 @@ impl TxPool { /// /// Depending on the change in direction of the basefee, this will promote or demote /// transactions from the basefee pool. - fn update_basefee(&mut self, mut pending_basefee: u64) -> Ordering { + fn update_basefee(&mut self, mut pending_basefee: u64, mut on_promoted: F) -> Ordering + where + F: FnMut(&Arc>), + { std::mem::swap(&mut self.all_transactions.pending_fees.base_fee, &mut pending_basefee); match self.all_transactions.pending_fees.base_fee.cmp(&pending_basefee) { Ordering::Equal => { @@ -293,19 +308,45 @@ impl TxPool { Ordering::Greater } Ordering::Less => { - // decreased base fee: recheck basefee pool and promote all that are now valid - let removed = - self.basefee_pool.enforce_basefee(self.all_transactions.pending_fees.base_fee); - for tx in removed { - let to = { - let tx = + // Base fee decreased: recheck BaseFee and promote. + // Invariants: + // - BaseFee contains only non-blob txs (blob txs live in Blob) and they already + // have ENOUGH_BLOB_FEE_CAP_BLOCK. + // - PENDING_POOL_BITS = BASE_FEE_POOL_BITS | ENOUGH_FEE_CAP_BLOCK | + // ENOUGH_BLOB_FEE_CAP_BLOCK. + // With the lower base fee they gain ENOUGH_FEE_CAP_BLOCK, so we can set the bit and + // insert directly into Pending (skip generic routing). + let current_base_fee = self.all_transactions.pending_fees.base_fee; + self.basefee_pool.enforce_basefee_with(current_base_fee, |tx| { + // Update transaction state — guaranteed Pending by the invariants above + let subpool = { + let meta = self.all_transactions.txs.get_mut(tx.id()).expect("tx exists in set"); - tx.state.insert(TxState::ENOUGH_FEE_CAP_BLOCK); - tx.subpool = tx.state.into(); - tx.subpool + meta.state.insert(TxState::ENOUGH_FEE_CAP_BLOCK); + meta.subpool = meta.state.into(); + meta.subpool }; - self.add_transaction_to_subpool(to, tx); - } + + if subpool == SubPool::Pending { + on_promoted(&tx); + } + + trace!(target: "txpool", hash=%tx.transaction.hash(), pool=?subpool, "Adding transaction to a subpool"); + match subpool { + SubPool::Queued => self.queued_pool.add_transaction(tx), + SubPool::Pending => { + self.pending_pool.add_transaction(tx, current_base_fee); + } + SubPool::Blob => { + self.blob_pool.add_transaction(tx); + } + SubPool::BaseFee => { + // This should be unreachable as transactions from BaseFee pool with decreased + // basefee are guaranteed to become Pending + warn!(target: "txpool", "BaseFee transactions should become Pending after basefee decrease"); + } + } + }); Ordering::Less } @@ -314,24 +355,15 @@ impl TxPool { /// Sets the current block info for the pool. /// - /// This will also apply updates to the pool based on the new base fee + /// This will also apply updates to the pool based on the new base fee and blob fee pub fn set_block_info(&mut self, info: BlockInfo) { - let BlockInfo { - block_gas_limit, - last_seen_block_hash, - last_seen_block_number, - pending_basefee, - pending_blob_fee, - } = info; - self.all_transactions.last_seen_block_hash = last_seen_block_hash; - self.all_transactions.last_seen_block_number = last_seen_block_number; - let basefee_ordering = self.update_basefee(pending_basefee); - - self.all_transactions.block_gas_limit = block_gas_limit; - - if let Some(blob_fee) = pending_blob_fee { - self.update_blob_fee(blob_fee, basefee_ordering) + // first update the subpools based on the new values + let basefee_ordering = self.update_basefee(info.pending_basefee, |_| {}); + if let Some(blob_fee) = info.pending_blob_fee { + self.update_blob_fee(blob_fee, basefee_ordering, |_| {}) } + // then update tracked values + self.all_transactions.set_block_info(info); } /// Returns an iterator that yields transactions that are ready to be included in the block with @@ -434,6 +466,11 @@ impl TxPool { self.pending_pool.all() } + /// Returns the number of transactions from the pending sub-pool + pub(crate) fn pending_transactions_count(&self) -> usize { + self.pending_pool.len() + } + /// Returns all pending transactions filtered by predicate pub(crate) fn pending_transactions_with_predicate( &self, @@ -462,6 +499,11 @@ impl TxPool { self.basefee_pool.all().chain(self.queued_pool.all()) } + /// Returns the number of transactions in parked pools + pub(crate) fn queued_transactions_count(&self) -> usize { + self.basefee_pool.len() + self.queued_pool.len() + } + /// Returns queued and pending transactions for the specified sender pub fn queued_and_pending_txs_by_sender( &self, @@ -524,6 +566,59 @@ impl TxPool { self.all_transactions.txs_iter(sender).map(|(_, tx)| Arc::clone(&tx.transaction)).collect() } + /// Updates only the pending fees without triggering subpool updates. + /// Returns the previous base fee and blob fee values. + const fn update_pending_fees_only( + &mut self, + mut new_base_fee: u64, + new_blob_fee: Option, + ) -> (u64, u128) { + std::mem::swap(&mut self.all_transactions.pending_fees.base_fee, &mut new_base_fee); + + let prev_blob_fee = if let Some(mut blob_fee) = new_blob_fee { + std::mem::swap(&mut self.all_transactions.pending_fees.blob_fee, &mut blob_fee); + blob_fee + } else { + self.all_transactions.pending_fees.blob_fee + }; + + (new_base_fee, prev_blob_fee) + } + + /// Applies fee-based promotion updates based on the previous fees. + /// + /// Records promoted transactions based on fee swings. + /// + /// Caution: This expects that the fees were previously already updated via + /// [`Self::update_pending_fees_only`]. + fn apply_fee_updates( + &mut self, + prev_base_fee: u64, + prev_blob_fee: u128, + outcome: &mut UpdateOutcome, + ) { + let new_base_fee = self.all_transactions.pending_fees.base_fee; + let new_blob_fee = self.all_transactions.pending_fees.blob_fee; + + if new_base_fee == prev_base_fee && new_blob_fee == prev_blob_fee { + // nothing to update + return; + } + + // IMPORTANT: + // Restore previous fees so that the update fee functions correctly handle fee swings + self.all_transactions.pending_fees.base_fee = prev_base_fee; + self.all_transactions.pending_fees.blob_fee = prev_blob_fee; + + let base_fee_ordering = self.update_basefee(new_base_fee, |tx| { + outcome.promoted.push(tx.clone()); + }); + + self.update_blob_fee(new_blob_fee, base_fee_ordering, |tx| { + outcome.promoted.push(tx.clone()); + }); + } + /// Updates the transactions for the changed senders. pub(crate) fn update_accounts( &mut self, @@ -544,8 +639,8 @@ impl TxPool { /// Updates the entire pool after a new block was mined. /// - /// This removes all mined transactions, updates according to the new base fee and rechecks - /// sender allowance. + /// This removes all mined transactions, updates according to the new base fee and blob fee and + /// rechecks sender allowance based on the given changed sender infos. pub(crate) fn on_canonical_state_change( &mut self, block_info: BlockInfo, @@ -555,7 +650,6 @@ impl TxPool { ) -> OnNewCanonicalStateOutcome { // update block info let block_hash = block_info.last_seen_block_hash; - self.all_transactions.set_block_info(block_info); // Remove all transaction that were included in the block let mut removed_txs_count = 0; @@ -568,7 +662,22 @@ impl TxPool { // Update removed transactions metric self.metrics.removed_transactions.increment(removed_txs_count); - let UpdateOutcome { promoted, discarded } = self.update_accounts(changed_senders); + // Update fees internally first without triggering subpool updates based on fee movements + // This must happen before we update the changed so that all account updates use the new fee + // values, this way all changed accounts remain unaffected by the fee updates that are + // performed in next step and we don't collect promotions twice + let (prev_base_fee, prev_blob_fee) = + self.update_pending_fees_only(block_info.pending_basefee, block_info.pending_blob_fee); + + // Now update accounts with the new fees already set + let mut outcome = self.update_accounts(changed_senders); + + // Apply subpool updates based on fee changes + // This will record any additional promotions based on fee movements + self.apply_fee_updates(prev_base_fee, prev_blob_fee, &mut outcome); + + // Update the rest of block info (without triggering fee updates again) + self.all_transactions.set_block_info(block_info); self.update_transaction_type_metrics(); self.metrics.performed_state_updates.increment(1); @@ -576,7 +685,12 @@ impl TxPool { // Update the latest update kind self.latest_update_kind = Some(update_kind); - OnNewCanonicalStateOutcome { block_hash, mined: mined_transactions, promoted, discarded } + OnNewCanonicalStateOutcome { + block_hash, + mined: mined_transactions, + promoted: outcome.promoted, + discarded: outcome.discarded, + } } /// Update sub-pools size metrics. @@ -619,31 +733,6 @@ impl TxPool { self.metrics.total_eip7702_transactions.set(eip7702_count as f64); } - /// Adds the transaction into the pool. - /// - /// This pool consists of four sub-pools: `Queued`, `Pending`, `BaseFee`, and `Blob`. - /// - /// The `Queued` pool contains transactions with gaps in its dependency tree: It requires - /// additional transactions that are note yet present in the pool. And transactions that the - /// sender can not afford with the current balance. - /// - /// The `Pending` pool contains all transactions that have no nonce gaps, and can be afforded by - /// the sender. It only contains transactions that are ready to be included in the pending - /// block. The pending pool contains all transactions that could be listed currently, but not - /// necessarily independently. However, this pool never contains transactions with nonce gaps. A - /// transaction is considered `ready` when it has the lowest nonce of all transactions from the - /// same sender. Which is equals to the chain nonce of the sender in the pending pool. - /// - /// The `BaseFee` pool contains transactions that currently can't satisfy the dynamic fee - /// requirement. With EIP-1559, transactions can become executable or not without any changes to - /// the sender's balance or nonce and instead their `feeCap` determines whether the - /// transaction is _currently_ (on the current state) ready or needs to be parked until the - /// `feeCap` satisfies the block's `baseFee`. - /// - /// The `Blob` pool contains _blob_ transactions that currently can't satisfy the dynamic fee - /// requirement, or blob fee requirement. Transactions become executable only if the - /// transaction `feeCap` is greater than the block's `baseFee` and the `maxBlobFee` is greater - /// than the block's `blobFee`. pub(crate) fn add_transaction( &mut self, tx: ValidPoolTransaction, @@ -664,7 +753,7 @@ impl TxPool { .update(on_chain_nonce, on_chain_balance); match self.all_transactions.insert_tx(tx, on_chain_balance, on_chain_nonce) { - Ok(InsertOk { transaction, move_to, replaced_tx, updates, .. }) => { + Ok(InsertOk { transaction, move_to, replaced_tx, updates, state }) => { // replace the new tx and remove the replaced in the subpool(s) self.add_new_transaction(transaction.clone(), replaced_tx.clone(), move_to); // Update inserted transactions metric @@ -682,7 +771,14 @@ impl TxPool { replaced, }) } else { - AddedTransaction::Parked { transaction, subpool: move_to, replaced } + // Determine the specific queued reason based on the transaction state + let queued_reason = state.determine_queued_reason(move_to); + AddedTransaction::Parked { + transaction, + subpool: move_to, + replaced, + queued_reason, + } }; // Update size metrics after adding and potentially moving transactions. @@ -748,10 +844,9 @@ impl TxPool { } } - /// Determines if the tx sender is delegated or has a - /// pending delegation, and if so, ensures they have at most one in-flight - /// **executable** transaction, e.g. disallow stacked and nonce-gapped transactions - /// from the account. + /// Determines if the tx sender is delegated or has a pending delegation, and if so, ensures + /// they have at most one configured amount of in-flight **executable** transactions (default at + /// most one), e.g. disallow stacked and nonce-gapped transactions from the account. fn check_delegation_limit( &self, transaction: &ValidPoolTransaction, @@ -765,10 +860,16 @@ impl TxPool { return Ok(()) } - let pending_txs = self.pending_pool.get_txs_by_sender(transaction.sender_id()); - if pending_txs.is_empty() { + let mut txs_by_sender = + self.pending_pool.iter_txs_by_sender(transaction.sender_id()).peekable(); + + if txs_by_sender.peek().is_none() { // Transaction with gapped nonce is not supported for delegated accounts - if transaction.nonce() > on_chain_nonce { + // but transaction can arrive out of order if more slots are allowed + // by default with a slot limit of 1 this will fail if the transaction's nonce > + // on_chain + let nonce_gap_distance = transaction.nonce().saturating_sub(on_chain_nonce); + if nonce_gap_distance >= self.config.max_inflight_delegated_slot_limit as u64 { return Err(PoolError::new( *transaction.hash(), PoolErrorKind::InvalidTransaction(InvalidPoolTransactionError::Eip7702( @@ -779,8 +880,17 @@ impl TxPool { return Ok(()) } - // Transaction replacement is supported - if pending_txs.contains(&transaction.transaction_id) { + let mut count = 0; + for id in txs_by_sender { + if id == &transaction.transaction_id { + // Transaction replacement is supported + return Ok(()) + } + count += 1; + } + + if count < self.config.max_inflight_delegated_slot_limit { + // account still has an available slot return Ok(()) } @@ -795,24 +905,25 @@ impl TxPool { /// This verifies that the transaction complies with code authorization /// restrictions brought by EIP-7702 transaction type: /// 1. Any account with a deployed delegation or an in-flight authorization to deploy a - /// delegation will only be allowed a single transaction slot instead of the standard limit. - /// This is due to the possibility of the account being sweeped by an unrelated account. - /// 2. In case the pool is tracking a pending / queued transaction from a specific account, it - /// will reject new transactions with delegations from that account with standard in-flight - /// transactions. + /// delegation will only be allowed a certain amount of transaction slots (default 1) instead + /// of the standard limit. This is due to the possibility of the account being sweeped by an + /// unrelated account. + /// 2. In case the pool is tracking a pending / queued transaction from a specific account, at + /// most one in-flight transaction is allowed; any additional delegated transactions from + /// that account will be rejected. fn validate_auth( &self, transaction: &ValidPoolTransaction, on_chain_nonce: u64, on_chain_code_hash: Option, ) -> Result<(), PoolError> { - // Allow at most one in-flight tx for delegated accounts or those with a - // pending authorization. + // Ensure in-flight limit for delegated accounts or those with a pending authorization. self.check_delegation_limit(transaction, on_chain_nonce, on_chain_code_hash)?; if let Some(authority_list) = &transaction.authority_ids { for sender_id in authority_list { - if self.all_transactions.txs_iter(*sender_id).next().is_some() { + // Ensure authority has at most 1 inflight transaction. + if self.all_transactions.txs_iter(*sender_id).nth(1).is_some() { return Err(PoolError::new( *transaction.hash(), PoolErrorKind::InvalidTransaction(InvalidPoolTransactionError::Eip7702( @@ -831,11 +942,11 @@ impl TxPool { /// This will move/discard the given transaction according to the `PoolUpdate` fn process_updates(&mut self, updates: Vec) -> UpdateOutcome { let mut outcome = UpdateOutcome::default(); - for PoolUpdate { id, hash, current, destination } in updates { + for PoolUpdate { id, current, destination } in updates { match destination { Destination::Discard => { // remove the transaction from the pool and subpool - if let Some(tx) = self.prune_transaction_by_hash(&hash) { + if let Some(tx) = self.prune_transaction_by_id(&id) { outcome.discarded.push(tx); } self.metrics.removed_transactions.increment(1); @@ -843,15 +954,16 @@ impl TxPool { Destination::Pool(move_to) => { debug_assert_ne!(&move_to, ¤t, "destination must be different"); let moved = self.move_transaction(current, move_to, &id); - if matches!(move_to, SubPool::Pending) { - if let Some(tx) = moved { - trace!(target: "txpool", hash=%tx.transaction.hash(), "Promoted transaction to pending"); - outcome.promoted.push(tx); - } + if matches!(move_to, SubPool::Pending) && + let Some(tx) = moved + { + trace!(target: "txpool", hash=%tx.transaction.hash(), "Promoted transaction to pending"); + outcome.promoted.push(tx); } } } } + outcome } @@ -957,6 +1069,17 @@ impl TxPool { let (tx, pool) = self.all_transactions.remove_transaction_by_hash(tx_hash)?; self.remove_from_subpool(pool, tx.id()) } + /// This removes the transaction from the pool and advances any descendant state inside the + /// subpool. + /// + /// This is intended to be used when we call [`Self::process_updates`]. + fn prune_transaction_by_id( + &mut self, + tx_id: &TransactionId, + ) -> Option>> { + let (tx, pool) = self.all_transactions.remove_transaction_by_id(tx_id)?; + self.remove_from_subpool(pool, tx.id()) + } /// Removes the transaction from the given pool. /// @@ -1165,18 +1288,19 @@ impl Drop for TxPool { } } -// Additional test impls -#[cfg(any(test, feature = "test-utils"))] impl TxPool { - pub(crate) const fn pending(&self) -> &PendingPool { + /// Pending subpool + pub const fn pending(&self) -> &PendingPool { &self.pending_pool } - pub(crate) const fn base_fee(&self) -> &ParkedPool> { + /// Base fee subpool + pub const fn base_fee(&self) -> &ParkedPool> { &self.basefee_pool } - pub(crate) const fn queued(&self) -> &ParkedPool> { + /// Queued sub pool + pub const fn queued(&self) -> &ParkedPool> { &self.queued_pool } } @@ -1353,16 +1477,14 @@ impl AllTransactions { } }; } - // tracks the balance if the sender was changed in the block - let mut changed_balance = None; + // track the balance if the sender was changed in the block // check if this is a changed account - if let Some(info) = changed_accounts.get(&id.sender) { + let changed_balance = if let Some(info) = changed_accounts.get(&id.sender) { // discard all transactions with a nonce lower than the current state nonce if id.nonce < info.state_nonce { updates.push(PoolUpdate { id: *tx.transaction.id(), - hash: *tx.transaction.hash(), current: tx.subpool, destination: Destination::Discard, }); @@ -1383,8 +1505,10 @@ impl AllTransactions { } } - changed_balance = Some(&info.balance); - } + Some(&info.balance) + } else { + None + }; // If there's a nonce gap, we can shortcircuit, because there's nothing to update yet. if tx.state.has_nonce_gap() { @@ -1472,7 +1596,6 @@ impl AllTransactions { if current_pool != tx.subpool { updates.push(PoolUpdate { id: *tx.transaction.id(), - hash: *tx.transaction.hash(), current: current_pool, destination: tx.subpool.into(), }) @@ -1558,7 +1681,21 @@ impl AllTransactions { self.remove_auths(&internal); // decrement the counter for the sender. self.tx_decr(tx.sender_id()); - self.update_size_metrics(); + Some((tx, internal.subpool)) + } + + /// Removes a transaction from the set using its id. + /// + /// This is intended for processing updates after state changes. + pub(crate) fn remove_transaction_by_id( + &mut self, + tx_id: &TransactionId, + ) -> Option<(Arc>, SubPool)> { + let internal = self.txs.remove(tx_id)?; + let tx = self.by_hash.remove(internal.transaction.hash())?; + self.remove_auths(&internal); + // decrement the counter for the sender. + self.tx_decr(tx.sender_id()); Some((tx, internal.subpool)) } @@ -1581,7 +1718,6 @@ impl AllTransactions { if current_pool != tx.subpool { updates.push(PoolUpdate { id: *id, - hash: *tx.transaction.hash(), current: current_pool, destination: tx.subpool.into(), }) @@ -1610,8 +1746,6 @@ impl AllTransactions { self.remove_auths(&internal); - self.update_size_metrics(); - result } @@ -1722,18 +1856,18 @@ impl AllTransactions { // overdraft let id = new_blob_tx.transaction_id; let mut descendants = self.descendant_txs_inclusive(&id).peekable(); - if let Some((maybe_replacement, _)) = descendants.peek() { - if **maybe_replacement == new_blob_tx.transaction_id { - // replacement transaction - descendants.next(); - - // check if any of descendant blob transactions should be shifted into overdraft - for (_, tx) in descendants { - cumulative_cost += tx.transaction.cost(); - if tx.transaction.is_eip4844() && cumulative_cost > on_chain_balance { - // the transaction would shift - return Err(InsertErr::Overdraft { transaction: Arc::new(new_blob_tx) }) - } + if let Some((maybe_replacement, _)) = descendants.peek() && + **maybe_replacement == new_blob_tx.transaction_id + { + // replacement transaction + descendants.next(); + + // check if any of descendant blob transactions should be shifted into overdraft + for (_, tx) in descendants { + cumulative_cost += tx.transaction.cost(); + if tx.transaction.is_eip4844() && cumulative_cost > on_chain_balance { + // the transaction would shift + return Err(InsertErr::Overdraft { transaction: Arc::new(new_blob_tx) }) } } } @@ -1941,7 +2075,6 @@ impl AllTransactions { if current_pool != tx.subpool { updates.push(PoolUpdate { id: *id, - hash: *tx.transaction.hash(), current: current_pool, destination: tx.subpool.into(), }) @@ -1977,7 +2110,7 @@ impl AllTransactions { #[cfg(any(test, feature = "test-utils"))] pub(crate) fn assert_invariants(&self) { assert_eq!(self.by_hash.len(), self.txs.len(), "by_hash.len() != txs.len()"); - assert!(self.auths.len() <= self.txs.len(), "auths > txs.len()"); + assert!(self.auths.len() <= self.txs.len(), "auths.len() > txs.len()"); } } @@ -2069,7 +2202,6 @@ pub(crate) struct InsertOk { /// Where to move the transaction to. move_to: SubPool, /// Current state of the inserted tx. - #[cfg_attr(not(test), expect(dead_code))] state: TxState, /// The transaction that was replaced by this. replaced_tx: Option<(Arc>, SubPool)>, @@ -2553,6 +2685,239 @@ mod tests { assert!(inserted.state.intersects(expected_state)); } + #[test] + // Test that on_canonical_state_change doesn't double-process transactions + // when both fee and account updates would affect the same transaction + fn test_on_canonical_state_change_no_double_processing() { + let mut tx_factory = MockTransactionFactory::default(); + let mut pool = TxPool::new(MockOrdering::default(), Default::default()); + + // Setup: Create a sender with a transaction in basefee pool + let tx = MockTransaction::eip1559().with_gas_price(50).with_gas_limit(30_000); + let sender = tx.sender(); + + // Set high base fee initially + let mut block_info = pool.block_info(); + block_info.pending_basefee = 100; + pool.set_block_info(block_info); + + let validated = tx_factory.validated(tx); + pool.add_transaction(validated, U256::from(10_000_000), 0, None).unwrap(); + + // Get sender_id after the transaction has been added + let sender_id = tx_factory.ids.sender_id(&sender).unwrap(); + + assert_eq!(pool.basefee_pool.len(), 1); + assert_eq!(pool.pending_pool.len(), 0); + + // Now simulate a canonical state change with: + // 1. Lower base fee (would promote tx) + // 2. Account balance update (would also evaluate tx) + block_info.pending_basefee = 40; + + let mut changed_senders = FxHashMap::default(); + changed_senders.insert( + sender_id, + SenderInfo { + state_nonce: 0, + balance: U256::from(20_000_000), // Increased balance + }, + ); + + let outcome = pool.on_canonical_state_change( + block_info, + vec![], // no mined transactions + changed_senders, + PoolUpdateKind::Commit, + ); + + // Transaction should be promoted exactly once + assert_eq!(pool.pending_pool.len(), 1, "Transaction should be in pending pool"); + assert_eq!(pool.basefee_pool.len(), 0, "Transaction should not be in basefee pool"); + assert_eq!(outcome.promoted.len(), 1, "Should report exactly one promotion"); + } + + #[test] + // Regression test: ensure we don't double-count promotions when base fee + // decreases and account is updated. This test would fail before the fix. + fn test_canonical_state_change_with_basefee_update_regression() { + let mut tx_factory = MockTransactionFactory::default(); + let mut pool = TxPool::new(MockOrdering::default(), Default::default()); + + // Create transactions from different senders to test independently + let sender_balance = U256::from(100_000_000); + + // Sender 1: tx will be promoted (gas price 60 > new base fee 50) + let tx1 = + MockTransaction::eip1559().with_gas_price(60).with_gas_limit(21_000).with_nonce(0); + let sender1 = tx1.sender(); + + // Sender 2: tx will be promoted (gas price 55 > new base fee 50) + let tx2 = + MockTransaction::eip1559().with_gas_price(55).with_gas_limit(21_000).with_nonce(0); + let sender2 = tx2.sender(); + + // Sender 3: tx will NOT be promoted (gas price 45 < new base fee 50) + let tx3 = + MockTransaction::eip1559().with_gas_price(45).with_gas_limit(21_000).with_nonce(0); + let sender3 = tx3.sender(); + + // Set high initial base fee (all txs will go to basefee pool) + let mut block_info = pool.block_info(); + block_info.pending_basefee = 70; + pool.set_block_info(block_info); + + // Add all transactions + let validated1 = tx_factory.validated(tx1); + let validated2 = tx_factory.validated(tx2); + let validated3 = tx_factory.validated(tx3); + + pool.add_transaction(validated1, sender_balance, 0, None).unwrap(); + pool.add_transaction(validated2, sender_balance, 0, None).unwrap(); + pool.add_transaction(validated3, sender_balance, 0, None).unwrap(); + + let sender1_id = tx_factory.ids.sender_id(&sender1).unwrap(); + let sender2_id = tx_factory.ids.sender_id(&sender2).unwrap(); + let sender3_id = tx_factory.ids.sender_id(&sender3).unwrap(); + + // All should be in basefee pool initially + assert_eq!(pool.basefee_pool.len(), 3, "All txs should be in basefee pool"); + assert_eq!(pool.pending_pool.len(), 0, "No txs should be in pending pool"); + + // Now decrease base fee to 50 - this should promote tx1 and tx2 (prices 60 and 55) + // but not tx3 (price 45) + block_info.pending_basefee = 50; + + // Update all senders' balances (simulating account state changes) + let mut changed_senders = FxHashMap::default(); + changed_senders.insert( + sender1_id, + SenderInfo { state_nonce: 0, balance: sender_balance + U256::from(1000) }, + ); + changed_senders.insert( + sender2_id, + SenderInfo { state_nonce: 0, balance: sender_balance + U256::from(1000) }, + ); + changed_senders.insert( + sender3_id, + SenderInfo { state_nonce: 0, balance: sender_balance + U256::from(1000) }, + ); + + let outcome = pool.on_canonical_state_change( + block_info, + vec![], + changed_senders, + PoolUpdateKind::Commit, + ); + + // Check final state + assert_eq!(pool.pending_pool.len(), 2, "tx1 and tx2 should be promoted"); + assert_eq!(pool.basefee_pool.len(), 1, "tx3 should remain in basefee"); + + // CRITICAL: Should report exactly 2 promotions, not 4 (which would happen with + // double-processing) + assert_eq!( + outcome.promoted.len(), + 2, + "Should report exactly 2 promotions, not double-counted" + ); + + // Verify the correct transactions were promoted + let promoted_prices: Vec = + outcome.promoted.iter().map(|tx| tx.max_fee_per_gas()).collect(); + assert!(promoted_prices.contains(&60)); + assert!(promoted_prices.contains(&55)); + } + + #[test] + fn test_basefee_decrease_with_empty_senders() { + // Test that fee promotions still occur when basefee decreases + // even with no changed_senders + let mut tx_factory = MockTransactionFactory::default(); + let mut pool = TxPool::new(MockOrdering::default(), Default::default()); + + // Create transaction that will be promoted when fee drops + let tx = MockTransaction::eip1559().with_gas_price(60).with_gas_limit(21_000); + + // Set high initial base fee + let mut block_info = pool.block_info(); + block_info.pending_basefee = 100; + pool.set_block_info(block_info); + + // Add transaction - should go to basefee pool + let validated = tx_factory.validated(tx); + pool.add_transaction(validated, U256::from(10_000_000), 0, None).unwrap(); + + assert_eq!(pool.basefee_pool.len(), 1); + assert_eq!(pool.pending_pool.len(), 0); + + // Decrease base fee with NO changed senders + block_info.pending_basefee = 50; + let outcome = pool.on_canonical_state_change( + block_info, + vec![], + FxHashMap::default(), // Empty changed_senders! + PoolUpdateKind::Commit, + ); + + // Transaction should still be promoted by fee-driven logic + assert_eq!(pool.pending_pool.len(), 1, "Fee decrease should promote tx"); + assert_eq!(pool.basefee_pool.len(), 0); + assert_eq!(outcome.promoted.len(), 1, "Should report promotion from fee update"); + } + + #[test] + fn test_basefee_decrease_account_makes_unfundable() { + // Test that when basefee decreases but account update makes tx unfundable, + // we don't get transient promote-then-discard double counting + let mut tx_factory = MockTransactionFactory::default(); + let mut pool = TxPool::new(MockOrdering::default(), Default::default()); + + let tx = MockTransaction::eip1559().with_gas_price(60).with_gas_limit(21_000); + let sender = tx.sender(); + + // High initial base fee + let mut block_info = pool.block_info(); + block_info.pending_basefee = 100; + pool.set_block_info(block_info); + + let validated = tx_factory.validated(tx); + pool.add_transaction(validated, U256::from(10_000_000), 0, None).unwrap(); + let sender_id = tx_factory.ids.sender_id(&sender).unwrap(); + + assert_eq!(pool.basefee_pool.len(), 1); + + // Decrease base fee (would normally promote) but also drain account + block_info.pending_basefee = 50; + let mut changed_senders = FxHashMap::default(); + changed_senders.insert( + sender_id, + SenderInfo { + state_nonce: 0, + balance: U256::from(100), // Too low to pay for gas! + }, + ); + + let outcome = pool.on_canonical_state_change( + block_info, + vec![], + changed_senders, + PoolUpdateKind::Commit, + ); + + // With insufficient balance, transaction goes to queued pool + assert_eq!(pool.pending_pool.len(), 0, "Unfunded tx should not be in pending"); + assert_eq!(pool.basefee_pool.len(), 0, "Tx no longer in basefee pool"); + assert_eq!(pool.queued_pool.len(), 1, "Unfunded tx should be in queued pool"); + + // Transaction is not removed, just moved to queued + let tx_count = pool.all_transactions.txs.len(); + assert_eq!(tx_count, 1, "Transaction should still be in pool (in queued)"); + + assert_eq!(outcome.promoted.len(), 0, "Should not report promotion"); + assert_eq!(outcome.discarded.len(), 0, "Queued tx is not reported as discarded"); + } + #[test] fn insert_already_imported() { let on_chain_balance = U256::ZERO; @@ -2900,7 +3265,7 @@ mod tests { assert_eq!(pool.pending_pool.len(), 1); - pool.update_basefee((tx.max_fee_per_gas() + 1) as u64); + pool.update_basefee((tx.max_fee_per_gas() + 1) as u64, |_| {}); assert!(pool.pending_pool.is_empty()); assert_eq!(pool.basefee_pool.len(), 1); @@ -2931,6 +3296,261 @@ mod tests { assert_eq!(pool.all_transactions.txs.get(&id).unwrap().subpool, SubPool::BaseFee) } + #[test] + fn basefee_decrease_promotes_affordable_and_keeps_unaffordable() { + use alloy_primitives::address; + let mut f = MockTransactionFactory::default(); + let mut pool = TxPool::new(MockOrdering::default(), Default::default()); + + // Create transactions that will be in basefee pool (can't afford initial high fee) + // Use different senders to avoid nonce gap issues + let sender_a = address!("0x000000000000000000000000000000000000000a"); + let sender_b = address!("0x000000000000000000000000000000000000000b"); + let sender_c = address!("0x000000000000000000000000000000000000000c"); + + let tx1 = MockTransaction::eip1559() + .set_sender(sender_a) + .set_nonce(0) + .set_max_fee(500) + .inc_limit(); + let tx2 = MockTransaction::eip1559() + .set_sender(sender_b) + .set_nonce(0) + .set_max_fee(600) + .inc_limit(); + let tx3 = MockTransaction::eip1559() + .set_sender(sender_c) + .set_nonce(0) + .set_max_fee(400) + .inc_limit(); + + // Set high initial basefee so transactions go to basefee pool + let mut block_info = pool.block_info(); + block_info.pending_basefee = 700; + pool.set_block_info(block_info); + + let validated1 = f.validated(tx1); + let validated2 = f.validated(tx2); + let validated3 = f.validated(tx3); + let id1 = *validated1.id(); + let id2 = *validated2.id(); + let id3 = *validated3.id(); + + // Add transactions - they should go to basefee pool due to high basefee + // All transactions have nonce 0 from different senders, so on_chain_nonce should be 0 for + // all + pool.add_transaction(validated1, U256::from(10_000), 0, None).unwrap(); + pool.add_transaction(validated2, U256::from(10_000), 0, None).unwrap(); + pool.add_transaction(validated3, U256::from(10_000), 0, None).unwrap(); + + // Debug: Check where transactions ended up + println!("Basefee pool len: {}", pool.basefee_pool.len()); + println!("Pending pool len: {}", pool.pending_pool.len()); + println!("tx1 subpool: {:?}", pool.all_transactions.txs.get(&id1).unwrap().subpool); + println!("tx2 subpool: {:?}", pool.all_transactions.txs.get(&id2).unwrap().subpool); + println!("tx3 subpool: {:?}", pool.all_transactions.txs.get(&id3).unwrap().subpool); + + // Verify they're in basefee pool + assert_eq!(pool.basefee_pool.len(), 3); + assert_eq!(pool.pending_pool.len(), 0); + assert_eq!(pool.all_transactions.txs.get(&id1).unwrap().subpool, SubPool::BaseFee); + assert_eq!(pool.all_transactions.txs.get(&id2).unwrap().subpool, SubPool::BaseFee); + assert_eq!(pool.all_transactions.txs.get(&id3).unwrap().subpool, SubPool::BaseFee); + + // Now decrease basefee to trigger the zero-allocation optimization + let mut block_info = pool.block_info(); + block_info.pending_basefee = 450; // tx1 (500) and tx2 (600) can now afford it, tx3 (400) cannot + pool.set_block_info(block_info); + + // Verify the optimization worked correctly: + // - tx1 and tx2 should be promoted to pending (mathematical certainty) + // - tx3 should remain in basefee pool + // - All state transitions should be correct + assert_eq!(pool.basefee_pool.len(), 1); + assert_eq!(pool.pending_pool.len(), 2); + + // tx3 should still be in basefee pool (fee 400 < basefee 450) + assert_eq!(pool.all_transactions.txs.get(&id3).unwrap().subpool, SubPool::BaseFee); + + // tx1 and tx2 should be in pending pool with correct state bits + let tx1_meta = pool.all_transactions.txs.get(&id1).unwrap(); + let tx2_meta = pool.all_transactions.txs.get(&id2).unwrap(); + assert_eq!(tx1_meta.subpool, SubPool::Pending); + assert_eq!(tx2_meta.subpool, SubPool::Pending); + assert!(tx1_meta.state.contains(TxState::ENOUGH_FEE_CAP_BLOCK)); + assert!(tx2_meta.state.contains(TxState::ENOUGH_FEE_CAP_BLOCK)); + + // Verify that best_transactions returns the promoted transactions + let best: Vec<_> = pool.best_transactions().take(3).collect(); + assert_eq!(best.len(), 2); // Only tx1 and tx2 should be returned + assert!(best.iter().any(|tx| tx.id() == &id1)); + assert!(best.iter().any(|tx| tx.id() == &id2)); + } + + #[test] + fn apply_fee_updates_records_promotions_after_basefee_drop() { + let mut f = MockTransactionFactory::default(); + let mut pool = TxPool::new(MockOrdering::default(), Default::default()); + + let tx = MockTransaction::eip1559() + .with_gas_limit(21_000) + .with_max_fee(500) + .with_priority_fee(1); + let validated = f.validated(tx); + let id = *validated.id(); + pool.add_transaction(validated, U256::from(1_000_000), 0, None).unwrap(); + + assert_eq!(pool.pending_pool.len(), 1); + + // Raise base fee beyond the transaction's cap so it gets parked in BaseFee pool. + pool.update_basefee(600, |_| {}); + assert!(pool.pending_pool.is_empty()); + assert_eq!(pool.basefee_pool.len(), 1); + + let prev_base_fee = 600; + let prev_blob_fee = pool.all_transactions.pending_fees.blob_fee; + + // Simulate the canonical state path updating pending fees before applying promotions. + pool.all_transactions.pending_fees.base_fee = 400; + + let mut outcome = UpdateOutcome::default(); + pool.apply_fee_updates(prev_base_fee, prev_blob_fee, &mut outcome); + + assert_eq!(pool.pending_pool.len(), 1); + assert!(pool.basefee_pool.is_empty()); + assert_eq!(outcome.promoted.len(), 1); + assert_eq!(outcome.promoted[0].id(), &id); + assert_eq!(pool.all_transactions.pending_fees.base_fee, 400); + assert_eq!(pool.all_transactions.pending_fees.blob_fee, prev_blob_fee); + + let tx_meta = pool.all_transactions.txs.get(&id).unwrap(); + assert_eq!(tx_meta.subpool, SubPool::Pending); + assert!(tx_meta.state.contains(TxState::ENOUGH_FEE_CAP_BLOCK)); + } + + #[test] + fn apply_fee_updates_records_promotions_after_blob_fee_drop() { + let mut f = MockTransactionFactory::default(); + let mut pool = TxPool::new(MockOrdering::default(), Default::default()); + + let initial_blob_fee = pool.all_transactions.pending_fees.blob_fee; + + let tx = MockTransaction::eip4844().with_blob_fee(initial_blob_fee + 100); + let validated = f.validated(tx.clone()); + let id = *validated.id(); + pool.add_transaction(validated, U256::from(1_000_000), 0, None).unwrap(); + + assert_eq!(pool.pending_pool.len(), 1); + + // Raise blob fee beyond the transaction's cap so it gets parked in Blob pool. + let increased_blob_fee = tx.max_fee_per_blob_gas().unwrap() + 200; + pool.update_blob_fee(increased_blob_fee, Ordering::Equal, |_| {}); + assert!(pool.pending_pool.is_empty()); + assert_eq!(pool.blob_pool.len(), 1); + + let prev_base_fee = pool.all_transactions.pending_fees.base_fee; + let prev_blob_fee = pool.all_transactions.pending_fees.blob_fee; + + // Simulate the canonical state path updating pending fees before applying promotions. + pool.all_transactions.pending_fees.blob_fee = tx.max_fee_per_blob_gas().unwrap(); + + let mut outcome = UpdateOutcome::default(); + pool.apply_fee_updates(prev_base_fee, prev_blob_fee, &mut outcome); + + assert_eq!(pool.pending_pool.len(), 1); + assert!(pool.blob_pool.is_empty()); + assert_eq!(outcome.promoted.len(), 1); + assert_eq!(outcome.promoted[0].id(), &id); + assert_eq!(pool.all_transactions.pending_fees.base_fee, prev_base_fee); + assert_eq!(pool.all_transactions.pending_fees.blob_fee, tx.max_fee_per_blob_gas().unwrap()); + + let tx_meta = pool.all_transactions.txs.get(&id).unwrap(); + assert_eq!(tx_meta.subpool, SubPool::Pending); + assert!(tx_meta.state.contains(TxState::ENOUGH_BLOB_FEE_CAP_BLOCK)); + assert!(tx_meta.state.contains(TxState::ENOUGH_FEE_CAP_BLOCK)); + } + + #[test] + fn apply_fee_updates_promotes_blob_after_basefee_drop() { + let mut f = MockTransactionFactory::default(); + let mut pool = TxPool::new(MockOrdering::default(), Default::default()); + + let initial_blob_fee = pool.all_transactions.pending_fees.blob_fee; + + let tx = MockTransaction::eip4844() + .with_max_fee(500) + .with_priority_fee(1) + .with_blob_fee(initial_blob_fee + 100); + let validated = f.validated(tx); + let id = *validated.id(); + pool.add_transaction(validated, U256::from(1_000_000), 0, None).unwrap(); + + assert_eq!(pool.pending_pool.len(), 1); + + // Raise base fee beyond the transaction's cap so it gets parked in Blob pool. + let high_base_fee = 600; + pool.update_basefee(high_base_fee, |_| {}); + assert!(pool.pending_pool.is_empty()); + assert_eq!(pool.blob_pool.len(), 1); + + let prev_base_fee = high_base_fee; + let prev_blob_fee = pool.all_transactions.pending_fees.blob_fee; + + // Simulate applying a lower base fee while keeping blob fee unchanged. + pool.all_transactions.pending_fees.base_fee = 400; + + let mut outcome = UpdateOutcome::default(); + pool.apply_fee_updates(prev_base_fee, prev_blob_fee, &mut outcome); + + assert_eq!(pool.pending_pool.len(), 1); + assert!(pool.blob_pool.is_empty()); + assert_eq!(outcome.promoted.len(), 1); + assert_eq!(outcome.promoted[0].id(), &id); + assert_eq!(pool.all_transactions.pending_fees.base_fee, 400); + assert_eq!(pool.all_transactions.pending_fees.blob_fee, prev_blob_fee); + + let tx_meta = pool.all_transactions.txs.get(&id).unwrap(); + assert_eq!(tx_meta.subpool, SubPool::Pending); + assert!(tx_meta.state.contains(TxState::ENOUGH_BLOB_FEE_CAP_BLOCK)); + assert!(tx_meta.state.contains(TxState::ENOUGH_FEE_CAP_BLOCK)); + } + + #[test] + fn apply_fee_updates_demotes_after_basefee_rise() { + let mut f = MockTransactionFactory::default(); + let mut pool = TxPool::new(MockOrdering::default(), Default::default()); + + let tx = MockTransaction::eip1559() + .with_gas_limit(21_000) + .with_max_fee(400) + .with_priority_fee(1); + let validated = f.validated(tx); + let id = *validated.id(); + pool.add_transaction(validated, U256::from(1_000_000), 0, None).unwrap(); + + assert_eq!(pool.pending_pool.len(), 1); + + let prev_base_fee = pool.all_transactions.pending_fees.base_fee; + let prev_blob_fee = pool.all_transactions.pending_fees.blob_fee; + + // Simulate canonical path raising the base fee beyond the transaction's cap. + let new_base_fee = prev_base_fee + 1_000; + pool.all_transactions.pending_fees.base_fee = new_base_fee; + + let mut outcome = UpdateOutcome::default(); + pool.apply_fee_updates(prev_base_fee, prev_blob_fee, &mut outcome); + + assert!(pool.pending_pool.is_empty()); + assert_eq!(pool.basefee_pool.len(), 1); + assert!(outcome.promoted.is_empty()); + assert_eq!(pool.all_transactions.pending_fees.base_fee, new_base_fee); + assert_eq!(pool.all_transactions.pending_fees.blob_fee, prev_blob_fee); + + let tx_meta = pool.all_transactions.txs.get(&id).unwrap(); + assert_eq!(tx_meta.subpool, SubPool::BaseFee); + assert!(!tx_meta.state.contains(TxState::ENOUGH_FEE_CAP_BLOCK)); + } + #[test] fn get_highest_transaction_by_sender_and_nonce() { // Set up a mock transaction factory and a new transaction pool. @@ -3088,7 +3708,7 @@ mod tests { // set the base fee of the pool let pool_base_fee = 100; - pool.update_basefee(pool_base_fee); + pool.update_basefee(pool_base_fee, |_| {}); // 2 txs, that should put the pool over the size limit but not max txs let a_txs = MockTransactionSet::dependent(a_sender, 0, 3, TxType::Eip1559) @@ -3745,4 +4365,168 @@ mod tests { assert_eq!(pool.pending_pool.independent().len(), 1); } + + #[test] + fn test_insertion_disorder() { + let mut f = MockTransactionFactory::default(); + let mut pool = TxPool::new(MockOrdering::default(), Default::default()); + + let sender = address!("0x1234567890123456789012345678901234567890"); + let tx0 = f.validated_arc( + MockTransaction::legacy().with_sender(sender).with_nonce(0).with_gas_price(10), + ); + let tx1 = f.validated_arc( + MockTransaction::eip1559() + .with_sender(sender) + .with_nonce(1) + .with_gas_limit(1000) + .with_gas_price(10), + ); + let tx2 = f.validated_arc( + MockTransaction::legacy().with_sender(sender).with_nonce(2).with_gas_price(10), + ); + let tx3 = f.validated_arc( + MockTransaction::legacy().with_sender(sender).with_nonce(3).with_gas_price(10), + ); + + // tx0 should be put in the pending subpool + pool.add_transaction((*tx0).clone(), U256::from(1000), 0, None).unwrap(); + let mut best = pool.best_transactions(); + let t0 = best.next().expect("tx0 should be put in the pending subpool"); + assert_eq!(t0.id(), tx0.id()); + // tx1 should be put in the queued subpool due to insufficient sender balance + pool.add_transaction((*tx1).clone(), U256::from(1000), 0, None).unwrap(); + let mut best = pool.best_transactions(); + let t0 = best.next().expect("tx0 should be put in the pending subpool"); + assert_eq!(t0.id(), tx0.id()); + assert!(best.next().is_none()); + + // tx2 should be put in the pending subpool, and tx1 should be promoted to pending + pool.add_transaction((*tx2).clone(), U256::MAX, 0, None).unwrap(); + + let mut best = pool.best_transactions(); + + let t0 = best.next().expect("tx0 should be put in the pending subpool"); + let t1 = best.next().expect("tx1 should be put in the pending subpool"); + let t2 = best.next().expect("tx2 should be put in the pending subpool"); + assert_eq!(t0.id(), tx0.id()); + assert_eq!(t1.id(), tx1.id()); + assert_eq!(t2.id(), tx2.id()); + + // tx3 should be put in the pending subpool, + pool.add_transaction((*tx3).clone(), U256::MAX, 0, None).unwrap(); + let mut best = pool.best_transactions(); + let t0 = best.next().expect("tx0 should be put in the pending subpool"); + let t1 = best.next().expect("tx1 should be put in the pending subpool"); + let t2 = best.next().expect("tx2 should be put in the pending subpool"); + let t3 = best.next().expect("tx3 should be put in the pending subpool"); + assert_eq!(t0.id(), tx0.id()); + assert_eq!(t1.id(), tx1.id()); + assert_eq!(t2.id(), tx2.id()); + assert_eq!(t3.id(), tx3.id()); + } + + #[test] + fn test_non_4844_blob_fee_bit_invariant() { + let mut f = MockTransactionFactory::default(); + let mut pool = TxPool::new(MockOrdering::default(), Default::default()); + + let non_4844_tx = MockTransaction::eip1559().set_max_fee(200).inc_limit(); + let validated = f.validated(non_4844_tx.clone()); + + assert!(!non_4844_tx.is_eip4844()); + pool.add_transaction(validated.clone(), U256::from(10_000), 0, None).unwrap(); + + // Core invariant: Non-4844 transactions must ALWAYS have ENOUGH_BLOB_FEE_CAP_BLOCK bit + let tx_meta = pool.all_transactions.txs.get(validated.id()).unwrap(); + assert!(tx_meta.state.contains(TxState::ENOUGH_BLOB_FEE_CAP_BLOCK)); + assert_eq!(tx_meta.subpool, SubPool::Pending); + } + + #[test] + fn test_blob_fee_enforcement_only_applies_to_eip4844() { + let mut f = MockTransactionFactory::default(); + let mut pool = TxPool::new(MockOrdering::default(), Default::default()); + + // Set blob fee higher than EIP-4844 tx can afford + let mut block_info = pool.block_info(); + block_info.pending_blob_fee = Some(160); + block_info.pending_basefee = 100; + pool.set_block_info(block_info); + + let eip4844_tx = MockTransaction::eip4844() + .with_sender(address!("0x000000000000000000000000000000000000000a")) + .with_max_fee(200) + .with_blob_fee(150) // Less than block blob fee (160) + .inc_limit(); + + let non_4844_tx = MockTransaction::eip1559() + .with_sender(address!("0x000000000000000000000000000000000000000b")) + .set_max_fee(200) + .inc_limit(); + + let validated_4844 = f.validated(eip4844_tx); + let validated_non_4844 = f.validated(non_4844_tx); + + pool.add_transaction(validated_4844.clone(), U256::from(10_000), 0, None).unwrap(); + pool.add_transaction(validated_non_4844.clone(), U256::from(10_000), 0, None).unwrap(); + + let tx_4844_meta = pool.all_transactions.txs.get(validated_4844.id()).unwrap(); + let tx_non_4844_meta = pool.all_transactions.txs.get(validated_non_4844.id()).unwrap(); + + // EIP-4844: blob fee enforcement applies - insufficient blob fee removes bit + assert!(!tx_4844_meta.state.contains(TxState::ENOUGH_BLOB_FEE_CAP_BLOCK)); + assert_eq!(tx_4844_meta.subpool, SubPool::Blob); + + // Non-4844: blob fee enforcement does NOT apply - bit always remains true + assert!(tx_non_4844_meta.state.contains(TxState::ENOUGH_BLOB_FEE_CAP_BLOCK)); + assert_eq!(tx_non_4844_meta.subpool, SubPool::Pending); + } + + #[test] + fn test_basefee_decrease_preserves_non_4844_blob_fee_bit() { + let mut f = MockTransactionFactory::default(); + let mut pool = TxPool::new(MockOrdering::default(), Default::default()); + + // Create non-4844 transaction with fee that initially can't afford high basefee + let non_4844_tx = MockTransaction::eip1559() + .with_sender(address!("0x000000000000000000000000000000000000000a")) + .set_max_fee(500) // Can't afford basefee of 600 + .inc_limit(); + + // Set high basefee so transaction goes to BaseFee pool initially + pool.update_basefee(600, |_| {}); + + let validated = f.validated(non_4844_tx); + let tx_id = *validated.id(); + pool.add_transaction(validated, U256::from(10_000), 0, None).unwrap(); + + // Initially should be in BaseFee pool but STILL have blob fee bit (critical invariant) + let tx_meta = pool.all_transactions.txs.get(&tx_id).unwrap(); + assert_eq!(tx_meta.subpool, SubPool::BaseFee); + assert!( + tx_meta.state.contains(TxState::ENOUGH_BLOB_FEE_CAP_BLOCK), + "Non-4844 tx in BaseFee pool must retain ENOUGH_BLOB_FEE_CAP_BLOCK bit" + ); + + // Decrease basefee - transaction should be promoted to Pending + // This is where PR #18215 bug would manifest: blob fee bit incorrectly removed + pool.update_basefee(400, |_| {}); + + // After basefee decrease: should be promoted to Pending with blob fee bit preserved + let tx_meta = pool.all_transactions.txs.get(&tx_id).unwrap(); + assert_eq!( + tx_meta.subpool, + SubPool::Pending, + "Non-4844 tx should be promoted from BaseFee to Pending after basefee decrease" + ); + assert!( + tx_meta.state.contains(TxState::ENOUGH_BLOB_FEE_CAP_BLOCK), + "Non-4844 tx must NEVER lose ENOUGH_BLOB_FEE_CAP_BLOCK bit during basefee promotion" + ); + assert!( + tx_meta.state.contains(TxState::ENOUGH_FEE_CAP_BLOCK), + "Non-4844 tx should gain ENOUGH_FEE_CAP_BLOCK bit after basefee decrease" + ); + } } diff --git a/crates/transaction-pool/src/pool/update.rs b/crates/transaction-pool/src/pool/update.rs index ca2b3358201..2322ccf6e65 100644 --- a/crates/transaction-pool/src/pool/update.rs +++ b/crates/transaction-pool/src/pool/update.rs @@ -3,7 +3,6 @@ use crate::{ identifier::TransactionId, pool::state::SubPool, PoolTransaction, ValidPoolTransaction, }; -use alloy_primitives::TxHash; use std::sync::Arc; /// A change of the transaction's location @@ -13,8 +12,6 @@ use std::sync::Arc; pub(crate) struct PoolUpdate { /// Internal tx id. pub(crate) id: TransactionId, - /// Hash of the transaction. - pub(crate) hash: TxHash, /// Where the transaction is currently held. pub(crate) current: SubPool, /// Where to move the transaction to. diff --git a/crates/transaction-pool/src/test_utils/mock.rs b/crates/transaction-pool/src/test_utils/mock.rs index aa8d1054618..c4b661b7964 100644 --- a/crates/transaction-pool/src/test_utils/mock.rs +++ b/crates/transaction-pool/src/test_utils/mock.rs @@ -12,20 +12,20 @@ use alloy_consensus::{ EIP1559_TX_TYPE_ID, EIP2930_TX_TYPE_ID, EIP4844_TX_TYPE_ID, EIP7702_TX_TYPE_ID, LEGACY_TX_TYPE_ID, }, - transaction::PooledTransaction, EthereumTxEnvelope, Signed, TxEip1559, TxEip2930, TxEip4844, TxEip4844Variant, TxEip7702, - TxEnvelope, TxLegacy, TxType, Typed2718, + TxLegacy, TxType, Typed2718, }; use alloy_eips::{ eip1559::MIN_PROTOCOL_BASE_FEE, eip2930::AccessList, eip4844::{BlobTransactionSidecar, BlobTransactionValidationError, DATA_GAS_PER_BLOB}, + eip7594::BlobTransactionSidecarVariant, eip7702::SignedAuthorization, }; use alloy_primitives::{Address, Bytes, ChainId, Signature, TxHash, TxKind, B256, U256}; use paste::paste; use rand::{distr::Uniform, prelude::Distribution}; -use reth_ethereum_primitives::{Transaction, TransactionSigned}; +use reth_ethereum_primitives::{PooledTransactionVariant, Transaction, TransactionSigned}; use reth_primitives_traits::{ transaction::error::TryFromRecoveredTransactionError, InMemorySize, Recovered, SignedTransaction, @@ -54,7 +54,31 @@ pub fn mock_tx_pool() -> MockTxPool { /// Sets the value for the field macro_rules! set_value { - ($this:ident => $field:ident) => { + // For mutable references + (&mut $this:expr => $field:ident) => {{ + let new_value = $field; + match $this { + MockTransaction::Legacy { $field, .. } => { + *$field = new_value; + } + MockTransaction::Eip1559 { $field, .. } => { + *$field = new_value; + } + MockTransaction::Eip4844 { $field, .. } => { + *$field = new_value; + } + MockTransaction::Eip2930 { $field, .. } => { + *$field = new_value; + } + MockTransaction::Eip7702 { $field, .. } => { + *$field = new_value; + } + } + // Ensure the tx cost is always correct after each mutation. + $this.update_cost(); + }}; + // For owned values + ($this:expr => $field:ident) => {{ let new_value = $field; match $this { MockTransaction::Legacy { ref mut $field, .. } | @@ -67,7 +91,7 @@ macro_rules! set_value { } // Ensure the tx cost is always correct after each mutation. $this.update_cost(); - }; + }}; } /// Gets the value for the field @@ -89,7 +113,7 @@ macro_rules! make_setters_getters { paste! {$( /// Sets the value of the specified field. pub fn [](&mut self, $name: $t) -> &mut Self { - set_value!(self => $name); + set_value!(&mut self => $name); self } @@ -218,7 +242,7 @@ pub enum MockTransaction { /// The transaction input data. input: Bytes, /// The sidecar information for the transaction. - sidecar: BlobTransactionSidecar, + sidecar: BlobTransactionSidecarVariant, /// The blob versioned hashes for the transaction. blob_versioned_hashes: Vec, /// The size of the transaction, returned in the implementation of [`PoolTransaction`]. @@ -361,7 +385,7 @@ impl MockTransaction { value: Default::default(), input: Bytes::new(), access_list: Default::default(), - sidecar: Default::default(), + sidecar: BlobTransactionSidecarVariant::Eip4844(Default::default()), blob_versioned_hashes: Default::default(), size: Default::default(), cost: U256::ZERO, @@ -369,7 +393,7 @@ impl MockTransaction { } /// Returns a new EIP4844 transaction with a provided sidecar - pub fn eip4844_with_sidecar(sidecar: BlobTransactionSidecar) -> Self { + pub fn eip4844_with_sidecar(sidecar: BlobTransactionSidecarVariant) -> Self { let mut transaction = Self::eip4844(); if let Self::Eip4844 { sidecar: existing_sidecar, blob_versioned_hashes, .. } = &mut transaction @@ -389,15 +413,12 @@ impl MockTransaction { /// * [`MockTransaction::eip1559`] /// * [`MockTransaction::eip4844`] pub fn new_from_type(tx_type: TxType) -> Self { - #[expect(unreachable_patterns)] match tx_type { TxType::Legacy => Self::legacy(), TxType::Eip2930 => Self::eip2930(), TxType::Eip1559 => Self::eip1559(), TxType::Eip4844 => Self::eip4844(), TxType::Eip7702 => Self::eip7702(), - - _ => unreachable!("Invalid transaction type"), } } @@ -656,7 +677,7 @@ impl MockTransaction { matches!(self, Self::Eip2930 { .. }) } - /// Checks if the transaction is of the EIP-2930 type. + /// Checks if the transaction is of the EIP-7702 type. pub const fn is_eip7702(&self) -> bool { matches!(self, Self::Eip7702 { .. }) } @@ -681,7 +702,7 @@ impl PoolTransaction for MockTransaction { type Consensus = TransactionSigned; - type Pooled = PooledTransaction; + type Pooled = PooledTransactionVariant; fn into_consensus(self) -> Recovered { self.into() @@ -802,21 +823,24 @@ impl alloy_consensus::Transaction for MockTransaction { } fn effective_gas_price(&self, base_fee: Option) -> u128 { - base_fee.map_or(self.max_fee_per_gas(), |base_fee| { - // if the tip is greater than the max priority fee per gas, set it to the max - // priority fee per gas + base fee - let tip = self.max_fee_per_gas().saturating_sub(base_fee as u128); - if let Some(max_tip) = self.max_priority_fee_per_gas() { - if tip > max_tip { - max_tip + base_fee as u128 + base_fee.map_or_else( + || self.max_fee_per_gas(), + |base_fee| { + // if the tip is greater than the max priority fee per gas, set it to the max + // priority fee per gas + base fee + let tip = self.max_fee_per_gas().saturating_sub(base_fee as u128); + if let Some(max_tip) = self.max_priority_fee_per_gas() { + if tip > max_tip { + max_tip + base_fee as u128 + } else { + // otherwise return the max fee per gas + self.max_fee_per_gas() + } } else { - // otherwise return the max fee per gas self.max_fee_per_gas() } - } else { - self.max_fee_per_gas() - } - }) + }, + ) } fn is_dynamic_fee(&self) -> bool { @@ -888,7 +912,7 @@ impl EthPoolTransaction for MockTransaction { fn try_into_pooled_eip4844( self, - sidecar: Arc, + sidecar: Arc, ) -> Option> { let (tx, signer) = self.into_consensus().into_parts(); tx.try_into_pooled_eip4844(Arc::unwrap_or_clone(sidecar)) @@ -898,7 +922,7 @@ impl EthPoolTransaction for MockTransaction { fn try_from_eip4844( tx: Recovered, - sidecar: BlobTransactionSidecar, + sidecar: BlobTransactionSidecarVariant, ) -> Option { let (tx, signer) = tx.into_parts(); tx.try_into_pooled_eip4844(sidecar) @@ -909,7 +933,7 @@ impl EthPoolTransaction for MockTransaction { fn validate_blob( &self, - _blob: &BlobTransactionSidecar, + _blob: &BlobTransactionSidecarVariant, _settings: &KzgSettings, ) -> Result<(), alloy_eips::eip4844::BlobTransactionValidationError> { match &self { @@ -1023,7 +1047,7 @@ impl TryFrom> for MockTransaction { value, input, access_list, - sidecar: BlobTransactionSidecar::default(), + sidecar: BlobTransactionSidecarVariant::Eip4844(BlobTransactionSidecar::default()), blob_versioned_hashes: Default::default(), size, cost: U256::from(gas_limit) * U256::from(max_fee_per_gas) + value, @@ -1059,10 +1083,14 @@ impl TryFrom> for MockTransaction { } } -impl TryFrom> for MockTransaction { +impl TryFrom>>> + for MockTransaction +{ type Error = TryFromRecoveredTransactionError; - fn try_from(tx: Recovered) -> Result { + fn try_from( + tx: Recovered>>, + ) -> Result { let sender = tx.signer(); let transaction = tx.into_inner(); let hash = *transaction.tx_hash(); @@ -1134,7 +1162,9 @@ impl TryFrom> for MockTransaction { value: tx.value, input: tx.input.clone(), access_list: tx.access_list.clone(), - sidecar: BlobTransactionSidecar::default(), + sidecar: BlobTransactionSidecarVariant::Eip4844( + BlobTransactionSidecar::default(), + ), blob_versioned_hashes: tx.blob_versioned_hashes.clone(), size, cost: U256::from(tx.gas_limit) * U256::from(tx.max_fee_per_gas) + tx.value, @@ -1164,8 +1194,8 @@ impl TryFrom> for MockTransaction { } } -impl From> for MockTransaction { - fn from(tx: Recovered) -> Self { +impl From> for MockTransaction { + fn from(tx: Recovered) -> Self { let (tx, signer) = tx.into_parts(); Recovered::::new_unchecked(tx.into(), signer).try_into().expect( "Failed to convert from PooledTransactionsElementEcRecovered to MockTransaction", @@ -1446,8 +1476,8 @@ impl MockFeeRange { max_fee_blob: Range, ) -> Self { assert!( - max_fee.start <= priority_fee.end, - "max_fee_range should be strictly below the priority fee range" + max_fee.start >= priority_fee.end, + "max_fee_range should be strictly above the priority fee range" ); Self { gas_price: gas_price.try_into().unwrap(), @@ -1704,7 +1734,7 @@ impl MockTransactionSet { /// /// Let an example transaction set be `[(tx1, 1), (tx2, 2)]`, where the first element of the /// tuple is a transaction, and the second element is the nonce. If the `gap_pct` is 50, and - /// the `gap_range` is `1..=1`, then the resulting transaction set could would be either + /// the `gap_range` is `1..=1`, then the resulting transaction set could be either /// `[(tx1, 1), (tx2, 2)]` or `[(tx1, 1), (tx2, 3)]`, with a 50% chance of either. pub fn with_nonce_gaps( &mut self, diff --git a/crates/transaction-pool/src/test_utils/okvalidator.rs b/crates/transaction-pool/src/test_utils/okvalidator.rs index 369839760c3..fc15dce74ec 100644 --- a/crates/transaction-pool/src/test_utils/okvalidator.rs +++ b/crates/transaction-pool/src/test_utils/okvalidator.rs @@ -10,11 +10,21 @@ use crate::{ #[non_exhaustive] pub struct OkValidator { _phantom: PhantomData, + /// Whether to mark transactions as propagatable. + propagate: bool, +} + +impl OkValidator { + /// Determines whether transactions should be allowed to be propagated + pub const fn set_propagate_transactions(mut self, propagate: bool) -> Self { + self.propagate = propagate; + self + } } impl Default for OkValidator { fn default() -> Self { - Self { _phantom: Default::default() } + Self { _phantom: Default::default(), propagate: false } } } @@ -38,7 +48,7 @@ where state_nonce: transaction.nonce(), bytecode_hash: None, transaction: ValidTransaction::Valid(transaction), - propagate: false, + propagate: self.propagate, authorities, } } diff --git a/crates/transaction-pool/src/test_utils/pool.rs b/crates/transaction-pool/src/test_utils/pool.rs index 7e22f3b8863..ab7bebae2f5 100644 --- a/crates/transaction-pool/src/test_utils/pool.rs +++ b/crates/transaction-pool/src/test_utils/pool.rs @@ -66,7 +66,7 @@ pub(crate) struct MockTransactionSimulator { balances: HashMap, /// represents the on chain nonce of a sender. nonces: HashMap, - /// A set of addresses to as senders. + /// A set of addresses to use as senders. senders: Vec
, /// What scenarios to execute. scenarios: Vec, @@ -166,7 +166,7 @@ impl MockSimulatorConfig { } } -/// Represents +/// Represents the different types of test scenarios. #[derive(Debug, Clone)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] pub(crate) enum ScenarioType { @@ -188,7 +188,7 @@ pub(crate) enum Scenario { HigherNonce { onchain: u64, nonce: u64 }, Multi { // Execute multiple test scenarios - scenario: Vec, + scenario: Vec, }, } diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index 795bbd33696..2b9d8bae8ab 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -1,32 +1,81 @@ +//! Transaction Pool Traits and Types +//! +//! This module defines the core abstractions for transaction pool implementations, +//! handling the complexity of different transaction representations across the +//! network, mempool, and the chain itself. +//! +//! ## Key Concepts +//! +//! ### Transaction Representations +//! +//! Transactions exist in different formats throughout their lifecycle: +//! +//! 1. **Consensus Format** ([`PoolTransaction::Consensus`]) +//! - The canonical format stored in blocks +//! - Minimal size for efficient storage +//! - Example: EIP-4844 transactions store only blob hashes: ([`TransactionSigned::Eip4844`]) +//! +//! 2. **Pooled Format** ([`PoolTransaction::Pooled`]) +//! - Extended format for network propagation +//! - Includes additional validation data +//! - Example: EIP-4844 transactions include full blob sidecars: ([`PooledTransactionVariant`]) +//! +//! ### Type Relationships +//! +//! ```text +//! NodePrimitives::SignedTx ←── NetworkPrimitives::BroadcastedTransaction +//! │ │ +//! │ (consensus format) │ (announced to peers) +//! │ │ +//! └──────────┐ ┌────────────────┘ +//! ▼ ▼ +//! PoolTransaction::Consensus +//! │ ▲ +//! │ │ from pooled (always succeeds) +//! │ │ +//! ▼ │ try_from consensus (may fail) +//! PoolTransaction::Pooled ←──→ NetworkPrimitives::PooledTransaction +//! (sent on request) +//! ``` +//! +//! ### Special Cases +//! +//! #### EIP-4844 Blob Transactions +//! - Consensus format: Only blob hashes (32 bytes each) +//! - Pooled format: Full blobs + commitments + proofs (large data per blob) +//! - Network behavior: Not broadcast automatically, only sent on explicit request +//! +//! #### Optimism Deposit Transactions +//! - Only exist in consensus format +//! - Never enter the mempool (system transactions) +//! - Conversion from consensus to pooled always fails + use crate::{ blobstore::BlobStoreError, - error::{InvalidPoolTransactionError, PoolResult}, + error::{InvalidPoolTransactionError, PoolError, PoolResult}, pool::{ state::SubPool, BestTransactionFilter, NewTransactionEvent, TransactionEvents, TransactionListenerKind, }, validate::ValidPoolTransaction, - AllTransactionsEvents, -}; -use alloy_consensus::{ - error::ValueError, transaction::PooledTransaction, BlockHeader, Signed, Typed2718, + AddedTransactionOutcome, AllTransactionsEvents, }; +use alloy_consensus::{error::ValueError, transaction::TxHashRef, BlockHeader, Signed, Typed2718}; use alloy_eips::{ - eip2718::Encodable2718, + eip2718::{Encodable2718, WithEncoded}, eip2930::AccessList, eip4844::{ - env_settings::KzgSettings, BlobAndProofV1, BlobTransactionSidecar, - BlobTransactionValidationError, + env_settings::KzgSettings, BlobAndProofV1, BlobAndProofV2, BlobTransactionValidationError, }, + eip7594::BlobTransactionSidecarVariant, eip7702::SignedAuthorization, }; use alloy_primitives::{Address, Bytes, TxHash, TxKind, B256, U256}; use futures_util::{ready, Stream}; use reth_eth_wire_types::HandleMempoolData; -use reth_ethereum_primitives::TransactionSigned; +use reth_ethereum_primitives::{PooledTransactionVariant, TransactionSigned}; use reth_execution_types::ChangedAccount; use reth_primitives_traits::{Block, InMemorySize, Recovered, SealedBlock, SignedTransaction}; -#[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; use std::{ collections::{HashMap, HashSet}, @@ -80,7 +129,7 @@ pub trait TransactionPool: Clone + Debug + Send + Sync { fn add_external_transaction( &self, transaction: Self::Transaction, - ) -> impl Future> + Send { + ) -> impl Future> + Send { self.add_transaction(TransactionOrigin::External, transaction) } @@ -90,7 +139,7 @@ pub trait TransactionPool: Clone + Debug + Send + Sync { fn add_external_transactions( &self, transactions: Vec, - ) -> impl Future>> + Send { + ) -> impl Future>> + Send { self.add_transactions(TransactionOrigin::External, transactions) } @@ -113,9 +162,11 @@ pub trait TransactionPool: Clone + Debug + Send + Sync { &self, origin: TransactionOrigin, transaction: Self::Transaction, - ) -> impl Future> + Send; + ) -> impl Future> + Send; - /// Adds the given _unvalidated_ transaction into the pool. + /// Adds the given _unvalidated_ transactions into the pool. + /// + /// All transactions will use the same `origin`. /// /// Returns a list of results. /// @@ -124,7 +175,53 @@ pub trait TransactionPool: Clone + Debug + Send + Sync { &self, origin: TransactionOrigin, transactions: Vec, - ) -> impl Future>> + Send; + ) -> impl Future>> + Send; + + /// Adds multiple _unvalidated_ transactions with individual origins. + /// + /// Each transaction can have its own [`TransactionOrigin`]. + /// + /// Consumer: RPC + fn add_transactions_with_origins( + &self, + transactions: Vec<(TransactionOrigin, Self::Transaction)>, + ) -> impl Future>> + Send; + + /// Submit a consensus transaction directly to the pool + fn add_consensus_transaction( + &self, + tx: Recovered<::Consensus>, + origin: TransactionOrigin, + ) -> impl Future> + Send { + async move { + let tx_hash = *tx.tx_hash(); + + let pool_transaction = match Self::Transaction::try_from_consensus(tx) { + Ok(tx) => tx, + Err(e) => return Err(PoolError::other(tx_hash, e.to_string())), + }; + + self.add_transaction(origin, pool_transaction).await + } + } + + /// Submit a consensus transaction and subscribe to event stream + fn add_consensus_transaction_and_subscribe( + &self, + tx: Recovered<::Consensus>, + origin: TransactionOrigin, + ) -> impl Future> + Send { + async move { + let tx_hash = *tx.tx_hash(); + + let pool_transaction = match Self::Transaction::try_from_consensus(tx) { + Ok(tx) => tx, + Err(e) => return Err(PoolError::other(tx_hash, e.to_string())), + }; + + self.add_transaction_and_subscribe(origin, pool_transaction).await + } + } /// Returns a new transaction change event stream for the given transaction. /// @@ -138,7 +235,7 @@ pub trait TransactionPool: Clone + Debug + Send + Sync { /// inserted into the pool that are allowed to be propagated. /// /// Note: This is intended for networking and will __only__ yield transactions that are allowed - /// to be propagated over the network, see also [TransactionListenerKind]. + /// to be propagated over the network, see also [`TransactionListenerKind`]. /// /// Consumer: RPC/P2P fn pending_transactions_listener(&self) -> Receiver { @@ -146,7 +243,7 @@ pub trait TransactionPool: Clone + Debug + Send + Sync { } /// Returns a new [Receiver] that yields transactions hashes for new __pending__ transactions - /// inserted into the pending pool depending on the given [TransactionListenerKind] argument. + /// inserted into the pending pool depending on the given [`TransactionListenerKind`] argument. fn pending_transactions_listener_for(&self, kind: TransactionListenerKind) -> Receiver; /// Returns a new stream that yields new valid transactions added to the pool. @@ -159,7 +256,7 @@ pub trait TransactionPool: Clone + Debug + Send + Sync { fn blob_transaction_sidecars_listener(&self) -> Receiver; /// Returns a new stream that yields new valid transactions added to the pool - /// depending on the given [TransactionListenerKind] argument. + /// depending on the given [`TransactionListenerKind`] argument. fn new_transactions_listener_for( &self, kind: TransactionListenerKind, @@ -167,8 +264,8 @@ pub trait TransactionPool: Clone + Debug + Send + Sync { /// Returns a new Stream that yields new transactions added to the pending sub-pool. /// - /// This is a convenience wrapper around [Self::new_transactions_listener] that filters for - /// [SubPool::Pending](crate::SubPool). + /// This is a convenience wrapper around [`Self::new_transactions_listener`] that filters for + /// [`SubPool::Pending`](crate::SubPool). fn new_pending_pool_transactions_listener( &self, ) -> NewSubpoolTransactionStream { @@ -180,8 +277,8 @@ pub trait TransactionPool: Clone + Debug + Send + Sync { /// Returns a new Stream that yields new transactions added to the basefee sub-pool. /// - /// This is a convenience wrapper around [Self::new_transactions_listener] that filters for - /// [SubPool::BaseFee](crate::SubPool). + /// This is a convenience wrapper around [`Self::new_transactions_listener`] that filters for + /// [`SubPool::BaseFee`](crate::SubPool). fn new_basefee_pool_transactions_listener( &self, ) -> NewSubpoolTransactionStream { @@ -190,13 +287,15 @@ pub trait TransactionPool: Clone + Debug + Send + Sync { /// Returns a new Stream that yields new transactions added to the queued-pool. /// - /// This is a convenience wrapper around [Self::new_transactions_listener] that filters for - /// [SubPool::Queued](crate::SubPool). + /// This is a convenience wrapper around [`Self::new_transactions_listener`] that filters for + /// [`SubPool::Queued`](crate::SubPool). fn new_queued_transactions_listener(&self) -> NewSubpoolTransactionStream { NewSubpoolTransactionStream::new(self.new_transactions_listener(), SubPool::Queued) } - /// Returns the _hashes_ of all transactions in the pool. + /// Returns the _hashes_ of all transactions in the pool that are allowed to be propagated. + /// + /// This excludes hashes that aren't allowed to be propagated. /// /// Note: This returns a `Vec` but should guarantee that all hashes are unique. /// @@ -208,7 +307,8 @@ pub trait TransactionPool: Clone + Debug + Send + Sync { /// Consumer: P2P fn pooled_transaction_hashes_max(&self, max: usize) -> Vec; - /// Returns the _full_ transaction objects all transactions in the pool. + /// Returns the _full_ transaction objects all transactions in the pool that are allowed to be + /// propagated. /// /// This is intended to be used by the network for the initial exchange of pooled transaction /// _hashes_ @@ -228,7 +328,8 @@ pub trait TransactionPool: Clone + Debug + Send + Sync { max: usize, ) -> Vec>>; - /// Returns converted [PooledTransaction] for the given transaction hashes. + /// Returns converted [`PooledTransactionVariant`] for the given transaction hashes that are + /// allowed to be propagated. /// /// This adheres to the expected behavior of /// [`GetPooledTransactions`](https://github.com/ethereum/devp2p/blob/master/caps/eth.md#getpooledtransactions-0x09): @@ -300,11 +401,15 @@ pub trait TransactionPool: Clone + Debug + Send + Sync { /// Returns all transactions that can be included in _future_ blocks. /// - /// This and [Self::pending_transactions] are mutually exclusive. + /// This and [`Self::pending_transactions`] are mutually exclusive. /// /// Consumer: RPC fn queued_transactions(&self) -> Vec>>; + /// Returns the number of transactions that are ready for inclusion in the next block and the + /// number of transactions that are ready for inclusion in future blocks: `(pending, queued)`. + fn pending_and_queued_txn_count(&self) -> (usize, usize); + /// Returns all transactions that are currently in the pool grouped by whether they are ready /// for inclusion in the next block or not. /// @@ -313,6 +418,31 @@ pub trait TransactionPool: Clone + Debug + Send + Sync { /// Consumer: RPC fn all_transactions(&self) -> AllPoolTransactions; + /// Returns the _hashes_ of all transactions regardless of whether they can be propagated or + /// not. + /// + /// Unlike [`Self::pooled_transaction_hashes`] this doesn't consider whether the transaction can + /// be propagated or not. + /// + /// Note: This returns a `Vec` but should guarantee that all hashes are unique. + /// + /// Consumer: Utility + fn all_transaction_hashes(&self) -> Vec; + + /// Removes a single transaction corresponding to the given hash. + /// + /// Note: This removes the transaction as if it got discarded (_not_ mined). + /// + /// Returns the removed transaction if it was found in the pool. + /// + /// Consumer: Utility + fn remove_transaction( + &self, + hash: TxHash, + ) -> Option>> { + self.remove_transactions(vec![hash]).pop() + } + /// Removes all transactions corresponding to the given hashes. /// /// Note: This removes the transactions as if they got discarded (_not_ mined). @@ -420,7 +550,7 @@ pub trait TransactionPool: Clone + Debug + Send + Sync { nonce: u64, ) -> Option>>; - /// Returns all transactions that where submitted with the given [TransactionOrigin] + /// Returns all transactions that where submitted with the given [`TransactionOrigin`] fn get_transactions_by_origin( &self, origin: TransactionOrigin, @@ -432,34 +562,34 @@ pub trait TransactionPool: Clone + Debug + Send + Sync { origin: TransactionOrigin, ) -> Vec>>; - /// Returns all transactions that where submitted as [TransactionOrigin::Local] + /// Returns all transactions that where submitted as [`TransactionOrigin::Local`] fn get_local_transactions(&self) -> Vec>> { self.get_transactions_by_origin(TransactionOrigin::Local) } - /// Returns all transactions that where submitted as [TransactionOrigin::Private] + /// Returns all transactions that where submitted as [`TransactionOrigin::Private`] fn get_private_transactions(&self) -> Vec>> { self.get_transactions_by_origin(TransactionOrigin::Private) } - /// Returns all transactions that where submitted as [TransactionOrigin::External] + /// Returns all transactions that where submitted as [`TransactionOrigin::External`] fn get_external_transactions(&self) -> Vec>> { self.get_transactions_by_origin(TransactionOrigin::External) } - /// Returns all pending transactions that where submitted as [TransactionOrigin::Local] + /// Returns all pending transactions that where submitted as [`TransactionOrigin::Local`] fn get_local_pending_transactions(&self) -> Vec>> { self.get_pending_transactions_by_origin(TransactionOrigin::Local) } - /// Returns all pending transactions that where submitted as [TransactionOrigin::Private] + /// Returns all pending transactions that where submitted as [`TransactionOrigin::Private`] fn get_private_pending_transactions( &self, ) -> Vec>> { self.get_pending_transactions_by_origin(TransactionOrigin::Private) } - /// Returns all pending transactions that where submitted as [TransactionOrigin::External] + /// Returns all pending transactions that where submitted as [`TransactionOrigin::External`] fn get_external_pending_transactions( &self, ) -> Vec>> { @@ -469,40 +599,48 @@ pub trait TransactionPool: Clone + Debug + Send + Sync { /// Returns a set of all senders of transactions in the pool fn unique_senders(&self) -> HashSet
; - /// Returns the [BlobTransactionSidecar] for the given transaction hash if it exists in the blob - /// store. + /// Returns the [`BlobTransactionSidecarVariant`] for the given transaction hash if it exists in + /// the blob store. fn get_blob( &self, tx_hash: TxHash, - ) -> Result>, BlobStoreError>; + ) -> Result>, BlobStoreError>; - /// Returns all [BlobTransactionSidecar] for the given transaction hashes if they exists in the - /// blob store. + /// Returns all [`BlobTransactionSidecarVariant`] for the given transaction hashes if they + /// exists in the blob store. /// /// This only returns the blobs that were found in the store. /// If there's no blob it will not be returned. fn get_all_blobs( &self, tx_hashes: Vec, - ) -> Result)>, BlobStoreError>; + ) -> Result)>, BlobStoreError>; - /// Returns the exact [BlobTransactionSidecar] for the given transaction hashes in the order - /// they were requested. + /// Returns the exact [`BlobTransactionSidecarVariant`] for the given transaction hashes in the + /// order they were requested. /// /// Returns an error if any of the blobs are not found in the blob store. fn get_all_blobs_exact( &self, tx_hashes: Vec, - ) -> Result>, BlobStoreError>; + ) -> Result>, BlobStoreError>; - /// Return the [`BlobTransactionSidecar`]s for a list of blob versioned hashes. - fn get_blobs_for_versioned_hashes( + /// Return the [`BlobAndProofV1`]s for a list of blob versioned hashes. + fn get_blobs_for_versioned_hashes_v1( &self, versioned_hashes: &[B256], ) -> Result>, BlobStoreError>; + + /// Return the [`BlobAndProofV2`]s for a list of blob versioned hashes. + /// Blobs and proofs are returned only if they are present for _all_ of the requested versioned + /// hashes. + fn get_blobs_for_versioned_hashes_v2( + &self, + versioned_hashes: &[B256], + ) -> Result>, BlobStoreError>; } -/// Extension for [TransactionPool] trait that allows to set the current block info. +/// Extension for [`TransactionPool`] trait that allows to set the current block info. #[auto_impl::auto_impl(&, Arc)] pub trait TransactionPoolExt: TransactionPool { /// Sets the current block info for the pool. @@ -554,6 +692,11 @@ pub struct AllPoolTransactions { // === impl AllPoolTransactions === impl AllPoolTransactions { + /// Returns the combined number of all transactions. + pub const fn count(&self) -> usize { + self.pending.len() + self.queued.len() + } + /// Returns an iterator over all pending [`Recovered`] transactions. pub fn pending_recovered(&self) -> impl Iterator> + '_ { self.pending.iter().map(|tx| tx.transaction.clone().into_consensus()) @@ -632,14 +775,14 @@ pub struct NewBlobSidecar { /// hash of the EIP-4844 transaction. pub tx_hash: TxHash, /// the blob transaction sidecar. - pub sidecar: Arc, + pub sidecar: Arc, } /// Where the transaction originates from. /// /// Depending on where the transaction was picked up, it affects how the transaction is handled /// internally, e.g. limits for simultaneous transaction of one sender. -#[derive(Debug, Copy, Clone, PartialEq, Eq, Default)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, Default, Deserialize, Serialize)] pub enum TransactionOrigin { /// Transaction is coming from a local source. #[default] @@ -926,19 +1069,62 @@ impl BestTransactionsAttributes { } } -/// Trait for transaction types used inside the pool. +/// Trait for transaction types stored in the transaction pool. +/// +/// This trait represents the actual transaction object stored in the mempool, which includes not +/// only the transaction data itself but also additional metadata needed for efficient pool +/// operations. Implementations typically cache values that are frequently accessed during +/// transaction ordering, validation, and eviction. +/// +/// ## Key Responsibilities +/// +/// 1. **Metadata Caching**: Store computed values like address, cost and encoded size +/// 2. **Representation Conversion**: Handle conversions between consensus and pooled +/// representations +/// 3. **Validation Support**: Provide methods for pool-specific validation rules +/// +/// ## Cached Metadata +/// +/// Implementations should cache frequently accessed values to avoid recomputation: +/// - **Address**: Recovered sender address of the transaction +/// - **Cost**: Max amount spendable (gas × price + value + blob costs) +/// - **Size**: RLP encoded length for mempool size limits +/// +/// See [`EthPooledTransaction`] for a reference implementation. /// -/// This supports two transaction formats -/// - Consensus format: the form the transaction takes when it is included in a block. -/// - Pooled format: the form the transaction takes when it is gossiping around the network. +/// ## Transaction Representations /// -/// This distinction is necessary for the EIP-4844 blob transactions, which require an additional -/// sidecar when they are gossiped around the network. It is expected that the `Consensus` format is -/// a subset of the `Pooled` format. +/// This trait abstracts over the different representations a transaction can have: /// -/// The assumption is that fallible conversion from `Consensus` to `Pooled` will encapsulate -/// handling of all valid `Consensus` transactions that can't be pooled (e.g Deposit transactions or -/// blob-less EIP-4844 transactions). +/// 1. **Consensus representation** (`Consensus` associated type): The canonical form included in +/// blocks +/// - Compact representation without networking metadata +/// - For EIP-4844: includes only blob hashes, not the actual blobs +/// - Used for block execution and state transitions +/// +/// 2. **Pooled representation** (`Pooled` associated type): The form used for network propagation +/// - May include additional data for validation +/// - For EIP-4844: includes full blob sidecars (blobs, commitments, proofs) +/// - Used for mempool validation and p2p gossiping +/// +/// ## Why Two Representations? +/// +/// This distinction is necessary because: +/// +/// - **EIP-4844 blob transactions**: Require large blob sidecars for validation that would bloat +/// blocks if included. Only blob hashes are stored on-chain. +/// +/// - **Network efficiency**: Blob transactions are not broadcast to all peers automatically but +/// must be explicitly requested to reduce bandwidth usage. +/// +/// - **Special transactions**: Some transactions (like OP deposit transactions) exist only in +/// consensus format and are never in the mempool. +/// +/// ## Conversion Rules +/// +/// - `Consensus` → `Pooled`: May fail for transactions that cannot be pooled (e.g., OP deposit +/// transactions, blob transactions without sidecars) +/// - `Pooled` → `Consensus`: Always succeeds (pooled is a superset) pub trait PoolTransaction: alloy_consensus::Transaction + InMemorySize + Debug + Send + Sync + Clone { @@ -953,8 +1139,13 @@ pub trait PoolTransaction: /// Define a method to convert from the `Consensus` type to `Self` /// - /// Note: this _must_ fail on any transactions that cannot be pooled (e.g OP Deposit - /// transactions). + /// This conversion may fail for transactions that are valid for inclusion in blocks + /// but cannot exist in the transaction pool. Examples include: + /// + /// - **OP Deposit transactions**: These are special system transactions that are directly + /// included in blocks by the sequencer/validator and never enter the mempool + /// - **Blob transactions without sidecars**: After being included in a block, the sidecar data + /// is pruned, making the consensus transaction unpoolable fn try_from_consensus( tx: Recovered, ) -> Result { @@ -972,6 +1163,14 @@ pub trait PoolTransaction: /// Define a method to convert from the `Self` type to `Consensus` fn into_consensus(self) -> Recovered; + /// Converts the transaction into consensus format while preserving the EIP-2718 encoded bytes. + /// This is used to optimize transaction execution by reusing cached encoded bytes instead of + /// re-encoding the transaction. The cached bytes are particularly useful in payload building + /// where the same transaction may be executed multiple times. + fn into_consensus_with2718(self) -> WithEncoded> { + self.into_consensus().into_encoded() + } + /// Define a method to convert from the `Pooled` type to `Self` fn from_pooled(pooled: Recovered) -> Self; @@ -1044,7 +1243,7 @@ pub trait EthPoolTransaction: PoolTransaction { /// transaction: [`Typed2718::is_eip4844`]. fn try_into_pooled_eip4844( self, - sidecar: Arc, + sidecar: Arc, ) -> Option>; /// Tries to convert the `Consensus` type with a blob sidecar into the `Pooled` type. @@ -1052,21 +1251,27 @@ pub trait EthPoolTransaction: PoolTransaction { /// Returns `None` if passed transaction is not a blob transaction. fn try_from_eip4844( tx: Recovered, - sidecar: BlobTransactionSidecar, + sidecar: BlobTransactionSidecarVariant, ) -> Option; /// Validates the blob sidecar of the transaction with the given settings. fn validate_blob( &self, - blob: &BlobTransactionSidecar, + blob: &BlobTransactionSidecarVariant, settings: &KzgSettings, ) -> Result<(), BlobTransactionValidationError>; } /// The default [`PoolTransaction`] for the [Pool](crate::Pool) for Ethereum. /// -/// This type is essentially a wrapper around [`Recovered`] with additional -/// fields derived from the transaction that are frequently used by the pools for ordering. +/// This type wraps a consensus transaction with additional cached data that's +/// frequently accessed by the pool for transaction ordering and validation: +/// +/// - `cost`: Pre-calculated max cost (gas * price + value + blob costs) +/// - `encoded_length`: Cached RLP encoding length for size limits +/// - `blob_sidecar`: Blob data state (None/Missing/Present) +/// +/// This avoids recalculating these values repeatedly during pool operations. #[derive(Debug, Clone, PartialEq, Eq)] pub struct EthPooledTransaction { /// `EcRecovered` transaction, the consensus format. @@ -1126,7 +1331,7 @@ impl PoolTransaction for EthPooledTransaction { type Consensus = TransactionSigned; - type Pooled = PooledTransaction; + type Pooled = PooledTransactionVariant; fn clone_into_consensus(&self) -> Recovered { self.transaction().clone() @@ -1140,7 +1345,7 @@ impl PoolTransaction for EthPooledTransaction { let encoded_length = tx.encode_2718_len(); let (tx, signer) = tx.into_parts(); match tx { - PooledTransaction::Eip4844(tx) => { + PooledTransactionVariant::Eip4844(tx) => { // include the blob sidecar let (tx, sig, hash) = tx.into_parts(); let (tx, blob) = tx.into_parts(); @@ -1283,7 +1488,7 @@ impl EthPoolTransaction for EthPooledTransaction { fn try_into_pooled_eip4844( self, - sidecar: Arc, + sidecar: Arc, ) -> Option> { let (signed_transaction, signer) = self.into_consensus().into_parts(); let pooled_transaction = @@ -1294,7 +1499,7 @@ impl EthPoolTransaction for EthPooledTransaction { fn try_from_eip4844( tx: Recovered, - sidecar: BlobTransactionSidecar, + sidecar: BlobTransactionSidecarVariant, ) -> Option { let (tx, signer) = tx.into_parts(); tx.try_into_pooled_eip4844(sidecar) @@ -1305,7 +1510,7 @@ impl EthPoolTransaction for EthPooledTransaction { fn validate_blob( &self, - sidecar: &BlobTransactionSidecar, + sidecar: &BlobTransactionSidecarVariant, settings: &KzgSettings, ) -> Result<(), BlobTransactionValidationError> { match self.transaction.inner().as_eip4844() { @@ -1316,22 +1521,37 @@ impl EthPoolTransaction for EthPooledTransaction { } /// Represents the blob sidecar of the [`EthPooledTransaction`]. +/// +/// EIP-4844 blob transactions require additional data (blobs, commitments, proofs) +/// for validation that is not included in the consensus format. This enum tracks +/// the sidecar state throughout the transaction's lifecycle in the pool. #[derive(Debug, Clone, PartialEq, Eq)] pub enum EthBlobTransactionSidecar { /// This transaction does not have a blob sidecar + /// (applies to all non-EIP-4844 transaction types) None, - /// This transaction has a blob sidecar (EIP-4844) but it is missing + /// This transaction has a blob sidecar (EIP-4844) but it is missing. /// - /// It was either extracted after being inserted into the pool or re-injected after reorg - /// without the blob sidecar + /// This can happen when: + /// - The sidecar was extracted after the transaction was added to the pool + /// - The transaction was re-injected after a reorg without its sidecar + /// - The transaction was recovered from the consensus format (e.g., from a block) Missing, - /// The eip-4844 transaction was pulled from the network and still has its blob sidecar - Present(BlobTransactionSidecar), + /// The EIP-4844 transaction was received from the network with its complete sidecar. + /// + /// This sidecar contains: + /// - The actual blob data (large data per blob) + /// - KZG commitments for each blob + /// - KZG proofs for validation + /// + /// The sidecar is required for validating the transaction but is not included + /// in blocks (only the blob hashes are included in the consensus format). + Present(BlobTransactionSidecarVariant), } impl EthBlobTransactionSidecar { /// Returns the blob sidecar if it is present - pub const fn maybe_sidecar(&self) -> Option<&BlobTransactionSidecar> { + pub const fn maybe_sidecar(&self) -> Option<&BlobTransactionSidecarVariant> { match self { Self::Present(sidecar) => Some(sidecar), _ => None, diff --git a/crates/transaction-pool/src/validate/constants.rs b/crates/transaction-pool/src/validate/constants.rs index 9607937c67a..d4fca5a2aeb 100644 --- a/crates/transaction-pool/src/validate/constants.rs +++ b/crates/transaction-pool/src/validate/constants.rs @@ -15,4 +15,4 @@ pub const DEFAULT_MAX_TX_INPUT_BYTES: usize = 4 * TX_SLOT_BYTE_SIZE; // 128KB pub const MAX_CODE_BYTE_SIZE: usize = revm_primitives::eip170::MAX_CODE_SIZE; /// Maximum initcode to permit in a creation transaction and create instructions. -pub const MAX_INIT_CODE_BYTE_SIZE: usize = revm_primitives::MAX_INITCODE_SIZE; +pub const MAX_INIT_CODE_BYTE_SIZE: usize = revm_primitives::eip3860::MAX_INITCODE_SIZE; diff --git a/crates/transaction-pool/src/validate/eth.rs b/crates/transaction-pool/src/validate/eth.rs index 9886cf6c2ab..8f427e5d9b3 100644 --- a/crates/transaction-pool/src/validate/eth.rs +++ b/crates/transaction-pool/src/validate/eth.rs @@ -9,9 +9,11 @@ use crate::{ metrics::TxPoolValidationMetrics, traits::TransactionOrigin, validate::{ValidTransaction, ValidationTask, MAX_INIT_CODE_BYTE_SIZE}, - EthBlobTransactionSidecar, EthPoolTransaction, LocalTransactionConfig, - TransactionValidationOutcome, TransactionValidationTaskExecutor, TransactionValidator, + Address, BlobTransactionSidecarVariant, EthBlobTransactionSidecar, EthPoolTransaction, + LocalTransactionConfig, TransactionValidationOutcome, TransactionValidationTaskExecutor, + TransactionValidator, }; + use alloy_consensus::{ constants::{ EIP1559_TX_TYPE_ID, EIP2930_TX_TYPE_ID, EIP4844_TX_TYPE_ID, EIP7702_TX_TYPE_ID, @@ -25,117 +27,22 @@ use alloy_eips::{ }; use reth_chainspec::{ChainSpecProvider, EthChainSpec, EthereumHardforks}; use reth_primitives_traits::{ - transaction::error::InvalidTransactionError, Block, GotExpected, SealedBlock, + constants::MAX_TX_GAS_LIMIT_OSAKA, transaction::error::InvalidTransactionError, Account, Block, + GotExpected, SealedBlock, }; -use reth_storage_api::{StateProvider, StateProviderFactory}; +use reth_storage_api::{AccountInfoReader, BytecodeReader, StateProviderFactory}; use reth_tasks::TaskSpawner; +use revm_primitives::U256; use std::{ marker::PhantomData, sync::{ atomic::{AtomicBool, AtomicU64}, Arc, }, - time::Instant, + time::{Instant, SystemTime}, }; use tokio::sync::Mutex; -/// Validator for Ethereum transactions. -/// It is a [`TransactionValidator`] implementation that validates ethereum transaction. -#[derive(Debug, Clone)] -pub struct EthTransactionValidator { - /// The type that performs the actual validation. - inner: Arc>, -} - -impl EthTransactionValidator { - /// Returns the configured chain spec - pub fn chain_spec(&self) -> Arc - where - Client: ChainSpecProvider, - { - self.client().chain_spec() - } - - /// Returns the configured client - pub fn client(&self) -> &Client { - &self.inner.client - } -} - -impl EthTransactionValidator -where - Client: ChainSpecProvider + StateProviderFactory, - Tx: EthPoolTransaction, -{ - /// Validates a single transaction. - /// - /// See also [`TransactionValidator::validate_transaction`] - pub fn validate_one( - &self, - origin: TransactionOrigin, - transaction: Tx, - ) -> TransactionValidationOutcome { - self.inner.validate_one(origin, transaction) - } - - /// Validates a single transaction with the provided state provider. - /// - /// This allows reusing the same provider across multiple transaction validations, - /// which can improve performance when validating many transactions. - /// - /// If `state` is `None`, a new state provider will be created. - pub fn validate_one_with_state( - &self, - origin: TransactionOrigin, - transaction: Tx, - state: &mut Option>, - ) -> TransactionValidationOutcome { - self.inner.validate_one_with_provider(origin, transaction, state) - } - - /// Validates all given transactions. - /// - /// Returns all outcomes for the given transactions in the same order. - /// - /// See also [`Self::validate_one`] - pub fn validate_all( - &self, - transactions: Vec<(TransactionOrigin, Tx)>, - ) -> Vec> { - self.inner.validate_batch(transactions) - } -} - -impl TransactionValidator for EthTransactionValidator -where - Client: ChainSpecProvider + StateProviderFactory, - Tx: EthPoolTransaction, -{ - type Transaction = Tx; - - async fn validate_transaction( - &self, - origin: TransactionOrigin, - transaction: Self::Transaction, - ) -> TransactionValidationOutcome { - self.validate_one(origin, transaction) - } - - async fn validate_transactions( - &self, - transactions: Vec<(TransactionOrigin, Self::Transaction)>, - ) -> Vec> { - self.validate_all(transactions) - } - - fn on_new_head_block(&self, new_tip_block: &SealedBlock) - where - B: Block, - { - self.inner.on_new_head_block(new_tip_block.header()) - } -} - /// A [`TransactionValidator`] implementation that validates ethereum transaction. /// /// It supports all known ethereum transaction types: @@ -151,7 +58,7 @@ where /// /// And adheres to the configured [`LocalTransactionConfig`]. #[derive(Debug)] -pub(crate) struct EthTransactionValidatorInner { +pub struct EthTransactionValidator { /// This type fetches account info from the db client: Client, /// Blobstore used for fetching re-injected blob transactions. @@ -178,29 +85,131 @@ pub(crate) struct EthTransactionValidatorInner { local_transactions_config: LocalTransactionConfig, /// Maximum size in bytes a single transaction can have in order to be accepted into the pool. max_tx_input_bytes: usize, + /// Maximum gas limit for individual transactions + max_tx_gas_limit: Option, + /// Disable balance checks during transaction validation + disable_balance_check: bool, /// Marker for the transaction type _marker: PhantomData, /// Metrics for tsx pool validation validation_metrics: TxPoolValidationMetrics, + /// Bitmap of custom transaction types that are allowed. + other_tx_types: U256, } -// === impl EthTransactionValidatorInner === +impl EthTransactionValidator { + /// Returns the configured chain spec + pub fn chain_spec(&self) -> Arc + where + Client: ChainSpecProvider, + { + self.client().chain_spec() + } -impl EthTransactionValidatorInner { /// Returns the configured chain id - pub(crate) fn chain_id(&self) -> u64 { - self.client.chain_spec().chain().id() + pub fn chain_id(&self) -> u64 + where + Client: ChainSpecProvider, + { + self.client().chain_spec().chain().id() + } + + /// Returns the configured client + pub const fn client(&self) -> &Client { + &self.client + } + + /// Returns the tracks activated forks relevant for transaction validation + pub const fn fork_tracker(&self) -> &ForkTracker { + &self.fork_tracker + } + + /// Returns if there are EIP-2718 type transactions + pub const fn eip2718(&self) -> bool { + self.eip2718 + } + + /// Returns if there are EIP-1559 type transactions + pub const fn eip1559(&self) -> bool { + self.eip1559 + } + + /// Returns if there are EIP-4844 blob transactions + pub const fn eip4844(&self) -> bool { + self.eip4844 + } + + /// Returns if there are EIP-7702 type transactions + pub const fn eip7702(&self) -> bool { + self.eip7702 + } + + /// Returns the current tx fee cap limit in wei locally submitted into the pool + pub const fn tx_fee_cap(&self) -> &Option { + &self.tx_fee_cap + } + + /// Returns the minimum priority fee to enforce for acceptance into the pool + pub const fn minimum_priority_fee(&self) -> &Option { + &self.minimum_priority_fee + } + + /// Returns the setup and parameters needed for validating KZG proofs. + pub const fn kzg_settings(&self) -> &EnvKzgSettings { + &self.kzg_settings + } + + /// Returns the config to handle [`TransactionOrigin::Local`](TransactionOrigin) transactions.. + pub const fn local_transactions_config(&self) -> &LocalTransactionConfig { + &self.local_transactions_config + } + + /// Returns the maximum size in bytes a single transaction can have in order to be accepted into + /// the pool. + pub const fn max_tx_input_bytes(&self) -> usize { + self.max_tx_input_bytes + } + + /// Returns whether balance checks are disabled for this validator. + pub const fn disable_balance_check(&self) -> bool { + self.disable_balance_check } } -impl EthTransactionValidatorInner +impl EthTransactionValidator where Client: ChainSpecProvider + StateProviderFactory, Tx: EthPoolTransaction, { - /// Returns the configured chain spec - fn chain_spec(&self) -> Arc { - self.client.chain_spec() + /// Returns the current max gas limit + pub fn block_gas_limit(&self) -> u64 { + self.max_gas_limit() + } + + /// Validates a single transaction. + /// + /// See also [`TransactionValidator::validate_transaction`] + pub fn validate_one( + &self, + origin: TransactionOrigin, + transaction: Tx, + ) -> TransactionValidationOutcome { + self.validate_one_with_provider(origin, transaction, &mut None) + } + + /// Validates a single transaction with the provided state provider. + /// + /// This allows reusing the same provider across multiple transaction validations, + /// which can improve performance when validating many transactions. + /// + /// If `state` is `None`, a new state provider will be created. + pub fn validate_one_with_state( + &self, + origin: TransactionOrigin, + transaction: Tx, + state: &mut Option>, + ) -> TransactionValidationOutcome { + self.validate_one_with_provider(origin, transaction, state) } /// Validates a single transaction using an optional cached state provider. @@ -210,7 +219,7 @@ where &self, origin: TransactionOrigin, transaction: Tx, - maybe_state: &mut Option>, + maybe_state: &mut Option>, ) -> TransactionValidationOutcome { match self.validate_one_no_state(origin, transaction) { Ok(transaction) => { @@ -219,7 +228,7 @@ where if maybe_state.is_none() { match self.client.latest() { Ok(new_state) => { - *maybe_state = Some(new_state); + *maybe_state = Some(Box::new(new_state)); } Err(err) => { return TransactionValidationOutcome::Error( @@ -288,30 +297,62 @@ where } } - _ => { + ty if !self.other_tx_types.bit(ty as usize) => { return Err(TransactionValidationOutcome::Invalid( transaction, InvalidTransactionError::TxTypeNotSupported.into(), )) } + + _ => {} }; - // Reject transactions over defined size to prevent DOS attacks - let tx_input_len = transaction.input().len(); - if tx_input_len > self.max_tx_input_bytes { + // Reject transactions with a nonce equal to U64::max according to EIP-2681 + let tx_nonce = transaction.nonce(); + if tx_nonce == u64::MAX { return Err(TransactionValidationOutcome::Invalid( transaction, - InvalidPoolTransactionError::OversizedData(tx_input_len, self.max_tx_input_bytes), + InvalidPoolTransactionError::Eip2681, )) } - // Check whether the init code size has been exceeded. - if self.fork_tracker.is_shanghai_activated() { - if let Err(err) = transaction.ensure_max_init_code_size(MAX_INIT_CODE_BYTE_SIZE) { - return Err(TransactionValidationOutcome::Invalid(transaction, err)) + // Reject transactions over defined size to prevent DOS attacks + if transaction.is_eip4844() { + // Since blob transactions are pulled instead of pushed, and only the consensus data is + // kept in memory while the sidecar is cached on disk, there is no critical limit that + // should be enforced. Still, enforcing some cap on the input bytes. blob txs also must + // be executable right away when they enter the pool. + let tx_input_len = transaction.input().len(); + if tx_input_len > self.max_tx_input_bytes { + return Err(TransactionValidationOutcome::Invalid( + transaction, + InvalidPoolTransactionError::OversizedData { + size: tx_input_len, + limit: self.max_tx_input_bytes, + }, + )) + } + } else { + // ensure the size of the non-blob transaction + let tx_size = transaction.encoded_length(); + if tx_size > self.max_tx_input_bytes { + return Err(TransactionValidationOutcome::Invalid( + transaction, + InvalidPoolTransactionError::OversizedData { + size: tx_size, + limit: self.max_tx_input_bytes, + }, + )) } } + // Check whether the init code size has been exceeded. + if self.fork_tracker.is_shanghai_activated() && + let Err(err) = transaction.ensure_max_init_code_size(MAX_INIT_CODE_BYTE_SIZE) + { + return Err(TransactionValidationOutcome::Invalid(transaction, err)) + } + // Checks for gas limit let transaction_gas_limit = transaction.gas_limit(); let block_gas_limit = self.max_gas_limit(); @@ -325,6 +366,19 @@ where )) } + // Check individual transaction gas limit if configured + if let Some(max_tx_gas_limit) = self.max_tx_gas_limit && + transaction_gas_limit > max_tx_gas_limit + { + return Err(TransactionValidationOutcome::Invalid( + transaction, + InvalidPoolTransactionError::MaxTxGasLimitExceeded( + transaction_gas_limit, + max_tx_gas_limit, + ), + )) + } + // Ensure max_priority_fee_per_gas (if EIP1559) is less than max_fee_per_gas if any. if transaction.max_priority_fee_per_gas() > Some(transaction.max_fee_per_gas()) { return Err(TransactionValidationOutcome::Invalid( @@ -342,15 +396,12 @@ where match self.tx_fee_cap { Some(0) | None => {} // Skip if cap is 0 or None Some(tx_fee_cap_wei) => { - // max possible tx fee is (gas_price * gas_limit) - // (if EIP1559) max possible tx fee is (max_fee_per_gas * gas_limit) - let gas_price = transaction.max_fee_per_gas(); - let max_tx_fee_wei = gas_price.saturating_mul(transaction.gas_limit() as u128); + let max_tx_fee_wei = transaction.cost().saturating_sub(transaction.value()); if max_tx_fee_wei > tx_fee_cap_wei { return Err(TransactionValidationOutcome::Invalid( transaction, InvalidPoolTransactionError::ExceedsFeeCap { - max_tx_fee_wei, + max_tx_fee_wei: max_tx_fee_wei.saturating_to(), tx_fee_cap_wei, }, )) @@ -367,18 +418,22 @@ where { return Err(TransactionValidationOutcome::Invalid( transaction, - InvalidPoolTransactionError::Underpriced, + InvalidPoolTransactionError::PriorityFeeBelowMinimum { + minimum_priority_fee: self + .minimum_priority_fee + .expect("minimum priority fee is expected inside if statement"), + }, )) } // Checks for chainid - if let Some(chain_id) = transaction.chain_id() { - if chain_id != self.chain_id() { - return Err(TransactionValidationOutcome::Invalid( - transaction, - InvalidTransactionError::ChainIdMismatch.into(), - )) - } + if let Some(chain_id) = transaction.chain_id() && + chain_id != self.chain_id() + { + return Err(TransactionValidationOutcome::Invalid( + transaction, + InvalidTransactionError::ChainIdMismatch.into(), + )) } if transaction.is_eip7702() { @@ -412,8 +467,7 @@ where )) } - let blob_count = - transaction.blob_versioned_hashes().map(|b| b.len() as u64).unwrap_or(0); + let blob_count = transaction.blob_count().unwrap_or(0); if blob_count == 0 { // no blobs return Err(TransactionValidationOutcome::Invalid( @@ -438,6 +492,16 @@ where } } + // Osaka validation of max tx gas. + if self.fork_tracker.is_osaka_activated() && + transaction.gas_limit() > MAX_TX_GAS_LIMIT_OSAKA + { + return Err(TransactionValidationOutcome::Invalid( + transaction, + InvalidTransactionError::GasLimitTooHigh.into(), + )) + } + Ok(transaction) } @@ -449,7 +513,7 @@ where state: P, ) -> TransactionValidationOutcome where - P: StateProvider, + P: AccountInfoReader, { // Use provider to get account info let account = match state.basic_account(transaction.sender_ref()) { @@ -459,21 +523,70 @@ where } }; + // check for bytecode + match self.validate_sender_bytecode(&transaction, &account, &state) { + Err(outcome) => return outcome, + Ok(Err(err)) => return TransactionValidationOutcome::Invalid(transaction, err), + _ => {} + }; + + // Checks for nonce + if let Err(err) = self.validate_sender_nonce(&transaction, &account) { + return TransactionValidationOutcome::Invalid(transaction, err) + } + + // checks for max cost not exceedng account_balance + if let Err(err) = self.validate_sender_balance(&transaction, &account) { + return TransactionValidationOutcome::Invalid(transaction, err) + } + + // heavy blob tx validation + let maybe_blob_sidecar = match self.validate_eip4844(&mut transaction) { + Err(err) => return TransactionValidationOutcome::Invalid(transaction, err), + Ok(sidecar) => sidecar, + }; + + let authorities = self.recover_authorities(&transaction); + // Return the valid transaction + TransactionValidationOutcome::Valid { + balance: account.balance, + state_nonce: account.nonce, + bytecode_hash: account.bytecode_hash, + transaction: ValidTransaction::new(transaction, maybe_blob_sidecar), + // by this point assume all external transactions should be propagated + propagate: match origin { + TransactionOrigin::External => true, + TransactionOrigin::Local => { + self.local_transactions_config.propagate_local_transactions + } + TransactionOrigin::Private => false, + }, + authorities, + } + } + + /// Validates that the sender’s account has valid or no bytecode. + pub fn validate_sender_bytecode( + &self, + transaction: &Tx, + sender: &Account, + state: impl BytecodeReader, + ) -> Result, TransactionValidationOutcome> { // Unless Prague is active, the signer account shouldn't have bytecode. // // If Prague is active, only EIP-7702 bytecode is allowed for the sender. // // Any other case means that the account is not an EOA, and should not be able to send // transactions. - if let Some(code_hash) = &account.bytecode_hash { + if let Some(code_hash) = &sender.bytecode_hash { let is_eip7702 = if self.fork_tracker.is_prague_activated() { match state.bytecode_by_hash(code_hash) { Ok(bytecode) => bytecode.unwrap_or_default().is_eip7702(), Err(err) => { - return TransactionValidationOutcome::Error( + return Err(TransactionValidationOutcome::Error( *transaction.hash(), Box::new(err), - ) + )) } } } else { @@ -481,38 +594,53 @@ where }; if !is_eip7702 { - return TransactionValidationOutcome::Invalid( - transaction, - InvalidTransactionError::SignerAccountHasBytecode.into(), - ) + return Ok(Err(InvalidTransactionError::SignerAccountHasBytecode.into())) } } + Ok(Ok(())) + } + /// Checks if the transaction nonce is valid. + pub fn validate_sender_nonce( + &self, + transaction: &Tx, + sender: &Account, + ) -> Result<(), InvalidPoolTransactionError> { let tx_nonce = transaction.nonce(); - // Checks for nonce - if tx_nonce < account.nonce { - return TransactionValidationOutcome::Invalid( - transaction, - InvalidTransactionError::NonceNotConsistent { tx: tx_nonce, state: account.nonce } - .into(), - ) + if tx_nonce < sender.nonce { + return Err(InvalidTransactionError::NonceNotConsistent { + tx: tx_nonce, + state: sender.nonce, + } + .into()) } + Ok(()) + } + /// Ensures the sender has sufficient account balance. + pub fn validate_sender_balance( + &self, + transaction: &Tx, + sender: &Account, + ) -> Result<(), InvalidPoolTransactionError> { let cost = transaction.cost(); - // Checks for max cost - if cost > &account.balance { + if !self.disable_balance_check && cost > &sender.balance { let expected = *cost; - return TransactionValidationOutcome::Invalid( - transaction, - InvalidTransactionError::InsufficientFunds( - GotExpected { got: account.balance, expected }.into(), - ) - .into(), + return Err(InvalidTransactionError::InsufficientFunds( + GotExpected { got: sender.balance, expected }.into(), ) + .into()) } + Ok(()) + } + /// Validates EIP-4844 blob sidecar data and returns the extracted sidecar, if any. + pub fn validate_eip4844( + &self, + transaction: &mut Tx, + ) -> Result, InvalidPoolTransactionError> { let mut maybe_blob_sidecar = None; // heavy blob tx validation @@ -521,109 +649,112 @@ where match transaction.take_blob() { EthBlobTransactionSidecar::None => { // this should not happen - return TransactionValidationOutcome::Invalid( - transaction, - InvalidTransactionError::TxTypeNotSupported.into(), - ) + return Err(InvalidTransactionError::TxTypeNotSupported.into()) } EthBlobTransactionSidecar::Missing => { // This can happen for re-injected blob transactions (on re-org), since the blob // is stripped from the transaction and not included in a block. // check if the blob is in the store, if it's included we previously validated // it and inserted it - if matches!(self.blob_store.contains(*transaction.hash()), Ok(true)) { + if self.blob_store.contains(*transaction.hash()).is_ok_and(|c| c) { // validated transaction is already in the store } else { - return TransactionValidationOutcome::Invalid( - transaction, - InvalidPoolTransactionError::Eip4844( - Eip4844PoolTransactionError::MissingEip4844BlobSidecar, - ), - ) + return Err(InvalidPoolTransactionError::Eip4844( + Eip4844PoolTransactionError::MissingEip4844BlobSidecar, + )) } } - EthBlobTransactionSidecar::Present(blob) => { + EthBlobTransactionSidecar::Present(sidecar) => { let now = Instant::now(); + + if self.fork_tracker.is_osaka_activated() { + if sidecar.is_eip4844() { + return Err(InvalidPoolTransactionError::Eip4844( + Eip4844PoolTransactionError::UnexpectedEip4844SidecarAfterOsaka, + )) + } + } else if sidecar.is_eip7594() && !self.allow_7594_sidecars() { + return Err(InvalidPoolTransactionError::Eip4844( + Eip4844PoolTransactionError::UnexpectedEip7594SidecarBeforeOsaka, + )) + } + // validate the blob - if let Err(err) = transaction.validate_blob(&blob, self.kzg_settings.get()) { - return TransactionValidationOutcome::Invalid( - transaction, - InvalidPoolTransactionError::Eip4844( - Eip4844PoolTransactionError::InvalidEip4844Blob(err), - ), - ) + if let Err(err) = transaction.validate_blob(&sidecar, self.kzg_settings.get()) { + return Err(InvalidPoolTransactionError::Eip4844( + Eip4844PoolTransactionError::InvalidEip4844Blob(err), + )) } // Record the duration of successful blob validation as histogram self.validation_metrics.blob_validation_duration.record(now.elapsed()); // store the extracted blob - maybe_blob_sidecar = Some(blob); + maybe_blob_sidecar = Some(sidecar); } } } + Ok(maybe_blob_sidecar) + } - let authorities = transaction.authorization_list().map(|auths| { - auths.iter().flat_map(|auth| auth.recover_authority()).collect::>() - }); - // Return the valid transaction - TransactionValidationOutcome::Valid { - balance: account.balance, - state_nonce: account.nonce, - bytecode_hash: account.bytecode_hash, - transaction: ValidTransaction::new(transaction, maybe_blob_sidecar), - // by this point assume all external transactions should be propagated - propagate: match origin { - TransactionOrigin::External => true, - TransactionOrigin::Local => { - self.local_transactions_config.propagate_local_transactions - } - TransactionOrigin::Private => false, - }, - authorities, - } + /// Returns the recovered authorities for the given transaction + fn recover_authorities(&self, transaction: &Tx) -> std::option::Option> { + transaction + .authorization_list() + .map(|auths| auths.iter().flat_map(|auth| auth.recover_authority()).collect::>()) } - /// Validates a single transaction. - fn validate_one( + /// Validates all given transactions. + fn validate_batch( &self, - origin: TransactionOrigin, - transaction: Tx, - ) -> TransactionValidationOutcome { + transactions: Vec<(TransactionOrigin, Tx)>, + ) -> Vec> { let mut provider = None; - self.validate_one_with_provider(origin, transaction, &mut provider) + transactions + .into_iter() + .map(|(origin, tx)| self.validate_one_with_provider(origin, tx, &mut provider)) + .collect() } - /// Validates all given transactions. - fn validate_batch( + /// Validates all given transactions with origin. + fn validate_batch_with_origin( &self, - transactions: Vec<(TransactionOrigin, Tx)>, + origin: TransactionOrigin, + transactions: impl IntoIterator + Send, ) -> Vec> { let mut provider = None; transactions .into_iter() - .map(|(origin, tx)| self.validate_one_with_provider(origin, tx, &mut provider)) + .map(|tx| self.validate_one_with_provider(origin, tx, &mut provider)) .collect() } fn on_new_head_block(&self, new_tip_block: &T) { // update all forks - if self.chain_spec().is_cancun_active_at_timestamp(new_tip_block.timestamp()) { - self.fork_tracker.cancun.store(true, std::sync::atomic::Ordering::Relaxed); - } - if self.chain_spec().is_shanghai_active_at_timestamp(new_tip_block.timestamp()) { self.fork_tracker.shanghai.store(true, std::sync::atomic::Ordering::Relaxed); } + if self.chain_spec().is_cancun_active_at_timestamp(new_tip_block.timestamp()) { + self.fork_tracker.cancun.store(true, std::sync::atomic::Ordering::Relaxed); + } + if self.chain_spec().is_prague_active_at_timestamp(new_tip_block.timestamp()) { self.fork_tracker.prague.store(true, std::sync::atomic::Ordering::Relaxed); } + if self.chain_spec().is_osaka_active_at_timestamp(new_tip_block.timestamp()) { + self.fork_tracker.osaka.store(true, std::sync::atomic::Ordering::Relaxed); + } + + self.fork_tracker + .tip_timestamp + .store(new_tip_block.timestamp(), std::sync::atomic::Ordering::Relaxed); + if let Some(blob_params) = self.chain_spec().blob_params_at_timestamp(new_tip_block.timestamp()) { self.fork_tracker .max_blob_count - .store(blob_params.max_blob_count, std::sync::atomic::Ordering::Relaxed); + .store(blob_params.max_blobs_per_tx, std::sync::atomic::Ordering::Relaxed); } self.block_gas_limit.store(new_tip_block.gas_limit(), std::sync::atomic::Ordering::Relaxed); @@ -632,6 +763,62 @@ where fn max_gas_limit(&self) -> u64 { self.block_gas_limit.load(std::sync::atomic::Ordering::Relaxed) } + + /// Returns whether EIP-7594 sidecars are allowed + fn allow_7594_sidecars(&self) -> bool { + let tip_timestamp = self.fork_tracker.tip_timestamp(); + + // If next block is Osaka, allow 7594 sidecars + if self.chain_spec().is_osaka_active_at_timestamp(tip_timestamp.saturating_add(12)) { + true + } else if self.chain_spec().is_osaka_active_at_timestamp(tip_timestamp.saturating_add(24)) { + let current_timestamp = + SystemTime::now().duration_since(SystemTime::UNIX_EPOCH).unwrap().as_secs(); + + // Allow after 4 seconds into last non-Osaka slot + current_timestamp >= tip_timestamp.saturating_add(4) + } else { + false + } + } +} + +impl TransactionValidator for EthTransactionValidator +where + Client: ChainSpecProvider + StateProviderFactory, + Tx: EthPoolTransaction, +{ + type Transaction = Tx; + + async fn validate_transaction( + &self, + origin: TransactionOrigin, + transaction: Self::Transaction, + ) -> TransactionValidationOutcome { + self.validate_one(origin, transaction) + } + + async fn validate_transactions( + &self, + transactions: Vec<(TransactionOrigin, Self::Transaction)>, + ) -> Vec> { + self.validate_batch(transactions) + } + + async fn validate_transactions_with_origin( + &self, + origin: TransactionOrigin, + transactions: impl IntoIterator + Send, + ) -> Vec> { + self.validate_batch_with_origin(origin, transactions) + } + + fn on_new_head_block(&self, new_tip_block: &SealedBlock) + where + B: Block, + { + self.on_new_head_block(new_tip_block.header()) + } } /// A builder for [`EthTransactionValidator`] and [`TransactionValidationTaskExecutor`] @@ -642,8 +829,12 @@ pub struct EthTransactionValidatorBuilder { shanghai: bool, /// Fork indicator whether we are in the Cancun hardfork. cancun: bool, - /// Fork indicator whether we are in the Cancun hardfork. + /// Fork indicator whether we are in the Prague hardfork. prague: bool, + /// Fork indicator whether we are in the Osaka hardfork. + osaka: bool, + /// Timestamp of the tip block. + tip_timestamp: u64, /// Max blob count at the block's timestamp. max_blob_count: u64, /// Whether using EIP-2718 type transactions is allowed @@ -671,17 +862,24 @@ pub struct EthTransactionValidatorBuilder { local_transactions_config: LocalTransactionConfig, /// Max size in bytes of a single transaction allowed max_tx_input_bytes: usize, + /// Maximum gas limit for individual transactions + max_tx_gas_limit: Option, + /// Disable balance checks during transaction validation + disable_balance_check: bool, + /// Bitmap of custom transaction types that are allowed. + other_tx_types: U256, } impl EthTransactionValidatorBuilder { /// Creates a new builder for the given client /// - /// By default this assumes the network is on the `Cancun` hardfork and the following + /// By default this assumes the network is on the `Prague` hardfork and the following /// transactions are allowed: /// - Legacy /// - EIP-2718 /// - EIP-1559 /// - EIP-4844 + /// - EIP-7702 pub fn new(client: Client) -> Self { Self { block_gas_limit: ETHEREUM_BLOCK_GAS_LIMIT_30M.into(), @@ -692,6 +890,7 @@ impl EthTransactionValidatorBuilder { local_transactions_config: Default::default(), max_tx_input_bytes: DEFAULT_MAX_TX_INPUT_BYTES, tx_fee_cap: Some(1e18 as u128), + max_tx_gas_limit: None, // by default all transaction types are allowed eip2718: true, eip1559: true, @@ -704,11 +903,22 @@ impl EthTransactionValidatorBuilder { // cancun is activated by default cancun: true, - // prague not yet activated - prague: false, + // prague is activated by default + prague: true, + + // osaka not yet activated + osaka: false, + + tip_timestamp: 0, - // max blob count is cancun by default - max_blob_count: BlobParams::cancun().max_blob_count, + // max blob count is prague by default + max_blob_count: BlobParams::prague().max_blobs_per_tx, + + // balance checks are enabled by default + disable_balance_check: false, + + // no custom transaction types by default + other_tx_types: U256::ZERO, } } @@ -754,6 +964,17 @@ impl EthTransactionValidatorBuilder { self } + /// Disables the Osaka fork. + pub const fn no_osaka(self) -> Self { + self.set_osaka(false) + } + + /// Set the Osaka fork. + pub const fn set_osaka(mut self, osaka: bool) -> Self { + self.osaka = osaka; + self + } + /// Disables the support for EIP-2718 transactions. pub const fn no_eip2718(self) -> Self { self.set_eip2718(false) @@ -794,8 +1015,8 @@ impl EthTransactionValidatorBuilder { } /// Sets a minimum priority fee that's enforced for acceptance into the pool. - pub const fn with_minimum_priority_fee(mut self, minimum_priority_fee: u128) -> Self { - self.minimum_priority_fee = Some(minimum_priority_fee); + pub const fn with_minimum_priority_fee(mut self, minimum_priority_fee: Option) -> Self { + self.minimum_priority_fee = minimum_priority_fee; self } @@ -807,20 +1028,23 @@ impl EthTransactionValidatorBuilder { /// Configures validation rules based on the head block's timestamp. /// - /// For example, whether the Shanghai and Cancun hardfork is activated at launch. + /// For example, whether the Shanghai and Cancun hardfork is activated at launch, or max blob + /// counts. pub fn with_head_timestamp(mut self, timestamp: u64) -> Self where Client: ChainSpecProvider, { - self.cancun = self.client.chain_spec().is_cancun_active_at_timestamp(timestamp); self.shanghai = self.client.chain_spec().is_shanghai_active_at_timestamp(timestamp); + self.cancun = self.client.chain_spec().is_cancun_active_at_timestamp(timestamp); self.prague = self.client.chain_spec().is_prague_active_at_timestamp(timestamp); + self.osaka = self.client.chain_spec().is_osaka_active_at_timestamp(timestamp); + self.tip_timestamp = timestamp; self.max_blob_count = self .client .chain_spec() .blob_params_at_timestamp(timestamp) .unwrap_or_else(BlobParams::cancun) - .max_blob_count; + .max_blobs_per_tx; self } @@ -846,6 +1070,24 @@ impl EthTransactionValidatorBuilder { self } + /// Sets the maximum gas limit for individual transactions + pub const fn with_max_tx_gas_limit(mut self, max_tx_gas_limit: Option) -> Self { + self.max_tx_gas_limit = max_tx_gas_limit; + self + } + + /// Disables balance checks during transaction validation + pub const fn disable_balance_check(mut self) -> Self { + self.disable_balance_check = true; + self + } + + /// Adds a custom transaction type to the validator. + pub const fn with_custom_tx_type(mut self, tx_type: u8) -> Self { + self.other_tx_types.set_bit(tx_type as usize, true); + self + } + /// Builds a the [`EthTransactionValidator`] without spawning validator tasks. pub fn build(self, blob_store: S) -> EthTransactionValidator where @@ -856,6 +1098,8 @@ impl EthTransactionValidatorBuilder { shanghai, cancun, prague, + osaka, + tip_timestamp, eip2718, eip1559, eip4844, @@ -866,23 +1110,23 @@ impl EthTransactionValidatorBuilder { kzg_settings, local_transactions_config, max_tx_input_bytes, - .. + max_tx_gas_limit, + disable_balance_check, + max_blob_count, + additional_tasks: _, + other_tx_types, } = self; - let max_blob_count = if prague { - BlobParams::prague().max_blob_count - } else { - BlobParams::cancun().max_blob_count - }; - let fork_tracker = ForkTracker { shanghai: AtomicBool::new(shanghai), cancun: AtomicBool::new(cancun), prague: AtomicBool::new(prague), + osaka: AtomicBool::new(osaka), + tip_timestamp: AtomicU64::new(tip_timestamp), max_blob_count: AtomicU64::new(max_blob_count), }; - let inner = EthTransactionValidatorInner { + EthTransactionValidator { client, eip2718, eip1559, @@ -896,11 +1140,12 @@ impl EthTransactionValidatorBuilder { kzg_settings, local_transactions_config, max_tx_input_bytes, + max_tx_gas_limit, + disable_balance_check, _marker: Default::default(), validation_metrics: TxPoolValidationMetrics::default(), - }; - - EthTransactionValidator { inner: Arc::new(inner) } + other_tx_types, + } } /// Builds a [`EthTransactionValidator`] and spawns validation tasks via the @@ -942,7 +1187,7 @@ impl EthTransactionValidatorBuilder { let to_validation_task = Arc::new(Mutex::new(tx)); - TransactionValidationTaskExecutor { validator, to_validation_task } + TransactionValidationTaskExecutor { validator: Arc::new(validator), to_validation_task } } } @@ -955,8 +1200,12 @@ pub struct ForkTracker { pub cancun: AtomicBool, /// Tracks if prague is activated at the block's timestamp. pub prague: AtomicBool, - /// Tracks max blob count at the block's timestamp. + /// Tracks if osaka is activated at the block's timestamp. + pub osaka: AtomicBool, + /// Tracks max blob count per transaction at the block's timestamp. pub max_blob_count: AtomicU64, + /// Tracks the timestamp of the tip block. + pub tip_timestamp: AtomicU64, } impl ForkTracker { @@ -975,7 +1224,17 @@ impl ForkTracker { self.prague.load(std::sync::atomic::Ordering::Relaxed) } - /// Returns the max blob count. + /// Returns `true` if Osaka fork is activated. + pub fn is_osaka_activated(&self) -> bool { + self.osaka.load(std::sync::atomic::Ordering::Relaxed) + } + + /// Returns the timestamp of the tip block. + pub fn tip_timestamp(&self) -> u64 { + self.tip_timestamp.load(std::sync::atomic::Ordering::Relaxed) + } + + /// Returns the max allowed blob count per transaction. pub fn max_blob_count(&self) -> u64 { self.max_blob_count.load(std::sync::atomic::Ordering::Relaxed) } @@ -1027,14 +1286,15 @@ mod tests { use alloy_consensus::Transaction; use alloy_eips::eip2718::Decodable2718; use alloy_primitives::{hex, U256}; - use reth_ethereum_primitives::PooledTransaction; + use reth_ethereum_primitives::PooledTransactionVariant; + use reth_primitives_traits::SignedTransaction; use reth_provider::test_utils::{ExtendedAccount, MockEthProvider}; fn get_transaction() -> EthPooledTransaction { let raw = "0x02f914950181ad84b2d05e0085117553845b830f7df88080b9143a6040608081523462000414576200133a803803806200001e8162000419565b9283398101608082820312620004145781516001600160401b03908181116200041457826200004f9185016200043f565b92602092838201519083821162000414576200006d9183016200043f565b8186015190946001600160a01b03821692909183900362000414576060015190805193808511620003145760038054956001938488811c9816801562000409575b89891014620003f3578190601f988981116200039d575b50899089831160011462000336576000926200032a575b505060001982841b1c191690841b1781555b8751918211620003145760049788548481811c9116801562000309575b89821014620002f457878111620002a9575b5087908784116001146200023e5793839491849260009562000232575b50501b92600019911b1c19161785555b6005556007805460ff60a01b19169055600880546001600160a01b0319169190911790553015620001f3575060025469d3c21bcecceda100000092838201809211620001de57506000917fddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef9160025530835282815284832084815401905584519384523093a351610e889081620004b28239f35b601190634e487b7160e01b6000525260246000fd5b90606493519262461bcd60e51b845283015260248201527f45524332303a206d696e7420746f20746865207a65726f2061646472657373006044820152fd5b0151935038806200013a565b9190601f198416928a600052848a6000209460005b8c8983831062000291575050501062000276575b50505050811b0185556200014a565b01519060f884600019921b161c191690553880808062000267565b86860151895590970196948501948893500162000253565b89600052886000208880860160051c8201928b8710620002ea575b0160051c019085905b828110620002dd5750506200011d565b60008155018590620002cd565b92508192620002c4565b60228a634e487b7160e01b6000525260246000fd5b90607f16906200010b565b634e487b7160e01b600052604160045260246000fd5b015190503880620000dc565b90869350601f19831691856000528b6000209260005b8d8282106200038657505084116200036d575b505050811b018155620000ee565b015160001983861b60f8161c191690553880806200035f565b8385015186558a979095019493840193016200034c565b90915083600052896000208980850160051c8201928c8610620003e9575b918891869594930160051c01915b828110620003d9575050620000c5565b60008155859450889101620003c9565b92508192620003bb565b634e487b7160e01b600052602260045260246000fd5b97607f1697620000ae565b600080fd5b6040519190601f01601f191682016001600160401b038111838210176200031457604052565b919080601f84011215620004145782516001600160401b038111620003145760209062000475601f8201601f1916830162000419565b92818452828287010111620004145760005b8181106200049d57508260009394955001015290565b85810183015184820184015282016200048756fe608060408181526004918236101561001657600080fd5b600092833560e01c91826306fdde0314610a1c57508163095ea7b3146109f257816318160ddd146109d35781631b4c84d2146109ac57816323b872dd14610833578163313ce5671461081757816339509351146107c357816370a082311461078c578163715018a6146107685781638124f7ac146107495781638da5cb5b1461072057816395d89b411461061d578163a457c2d714610575578163a9059cbb146104e4578163c9567bf914610120575063dd62ed3e146100d557600080fd5b3461011c578060031936011261011c57806020926100f1610b5a565b6100f9610b75565b6001600160a01b0391821683526001865283832091168252845220549051908152f35b5080fd5b905082600319360112610338576008546001600160a01b039190821633036104975760079283549160ff8360a01c1661045557737a250d5630b4cf539739df2c5dacb4c659f2488d92836bffffffffffffffffffffffff60a01b8092161786553087526020938785528388205430156104065730895260018652848920828a52865280858a205584519081527f8c5be1e5ebec7d5bd14f71427d1e84f3dd0314c0f7b2291e5b200ac8c7c3b925863092a38554835163c45a015560e01b815290861685828581845afa9182156103dd57849187918b946103e7575b5086516315ab88c960e31b815292839182905afa9081156103dd576044879289928c916103c0575b508b83895196879586946364e329cb60e11b8652308c870152166024850152165af19081156103b6579086918991610389575b50169060065416176006558385541660604730895288865260c4858a20548860085416928751958694859363f305d71960e01b8552308a86015260248501528d60448501528d606485015260848401524260a48401525af1801561037f579084929161034c575b50604485600654169587541691888551978894859363095ea7b360e01b855284015260001960248401525af1908115610343575061030c575b5050805460ff60a01b1916600160a01b17905580f35b81813d831161033c575b6103208183610b8b565b8101031261033857518015150361011c5738806102f6565b8280fd5b503d610316565b513d86823e3d90fd5b6060809293503d8111610378575b6103648183610b8b565b81010312610374578290386102bd565b8580fd5b503d61035a565b83513d89823e3d90fd5b6103a99150863d88116103af575b6103a18183610b8b565b810190610e33565b38610256565b503d610397565b84513d8a823e3d90fd5b6103d79150843d86116103af576103a18183610b8b565b38610223565b85513d8b823e3d90fd5b6103ff919450823d84116103af576103a18183610b8b565b92386101fb565b845162461bcd60e51b81528085018790526024808201527f45524332303a20617070726f76652066726f6d20746865207a65726f206164646044820152637265737360e01b6064820152608490fd5b6020606492519162461bcd60e51b8352820152601760248201527f74726164696e6720697320616c7265616479206f70656e0000000000000000006044820152fd5b608490602084519162461bcd60e51b8352820152602160248201527f4f6e6c79206f776e65722063616e2063616c6c20746869732066756e6374696f6044820152603760f91b6064820152fd5b9050346103385781600319360112610338576104fe610b5a565b9060243593303303610520575b602084610519878633610bc3565b5160018152f35b600594919454808302908382041483151715610562576127109004820391821161054f5750925080602061050b565b634e487b7160e01b815260118552602490fd5b634e487b7160e01b825260118652602482fd5b9050823461061a578260031936011261061a57610590610b5a565b918360243592338152600160205281812060018060a01b03861682526020522054908282106105c9576020856105198585038733610d31565b608490602086519162461bcd60e51b8352820152602560248201527f45524332303a2064656372656173656420616c6c6f77616e63652062656c6f77604482015264207a65726f60d81b6064820152fd5b80fd5b83833461011c578160031936011261011c57805191809380549160019083821c92828516948515610716575b6020958686108114610703578589529081156106df5750600114610687575b6106838787610679828c0383610b8b565b5191829182610b11565b0390f35b81529295507f8a35acfbc15ff81a39ae7d344fd709f28e8600b4aa8c65c6b64bfe7fe36bd19b5b8284106106cc57505050826106839461067992820101948680610668565b80548685018801529286019281016106ae565b60ff19168887015250505050151560051b8301019250610679826106838680610668565b634e487b7160e01b845260228352602484fd5b93607f1693610649565b50503461011c578160031936011261011c5760085490516001600160a01b039091168152602090f35b50503461011c578160031936011261011c576020906005549051908152f35b833461061a578060031936011261061a57600880546001600160a01b031916905580f35b50503461011c57602036600319011261011c5760209181906001600160a01b036107b4610b5a565b16815280845220549051908152f35b82843461061a578160031936011261061a576107dd610b5a565b338252600160209081528383206001600160a01b038316845290528282205460243581019290831061054f57602084610519858533610d31565b50503461011c578160031936011261011c576020905160128152f35b83833461011c57606036600319011261011c5761084e610b5a565b610856610b75565b6044359160018060a01b0381169485815260209560018752858220338352875285822054976000198903610893575b505050906105199291610bc3565b85891061096957811561091a5733156108cc5750948481979861051997845260018a528284203385528a52039120558594938780610885565b865162461bcd60e51b8152908101889052602260248201527f45524332303a20617070726f766520746f20746865207a65726f206164647265604482015261737360f01b6064820152608490fd5b865162461bcd60e51b81529081018890526024808201527f45524332303a20617070726f76652066726f6d20746865207a65726f206164646044820152637265737360e01b6064820152608490fd5b865162461bcd60e51b8152908101889052601d60248201527f45524332303a20696e73756666696369656e7420616c6c6f77616e63650000006044820152606490fd5b50503461011c578160031936011261011c5760209060ff60075460a01c1690519015158152f35b50503461011c578160031936011261011c576020906002549051908152f35b50503461011c578060031936011261011c57602090610519610a12610b5a565b6024359033610d31565b92915034610b0d5783600319360112610b0d57600354600181811c9186908281168015610b03575b6020958686108214610af05750848852908115610ace5750600114610a75575b6106838686610679828b0383610b8b565b929550600383527fc2575a0e9e593c00f959f8c92f12db2869c3395a3b0502d05e2516446f71f85b5b828410610abb575050508261068394610679928201019438610a64565b8054868501880152928601928101610a9e565b60ff191687860152505050151560051b83010192506106798261068338610a64565b634e487b7160e01b845260229052602483fd5b93607f1693610a44565b8380fd5b6020808252825181830181905290939260005b828110610b4657505060409293506000838284010152601f8019910116010190565b818101860151848201604001528501610b24565b600435906001600160a01b0382168203610b7057565b600080fd5b602435906001600160a01b0382168203610b7057565b90601f8019910116810190811067ffffffffffffffff821117610bad57604052565b634e487b7160e01b600052604160045260246000fd5b6001600160a01b03908116918215610cde5716918215610c8d57600082815280602052604081205491808310610c3957604082827fddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef958760209652828652038282205586815220818154019055604051908152a3565b60405162461bcd60e51b815260206004820152602660248201527f45524332303a207472616e7366657220616d6f756e7420657863656564732062604482015265616c616e636560d01b6064820152608490fd5b60405162461bcd60e51b815260206004820152602360248201527f45524332303a207472616e7366657220746f20746865207a65726f206164647260448201526265737360e81b6064820152608490fd5b60405162461bcd60e51b815260206004820152602560248201527f45524332303a207472616e736665722066726f6d20746865207a65726f206164604482015264647265737360d81b6064820152608490fd5b6001600160a01b03908116918215610de25716918215610d925760207f8c5be1e5ebec7d5bd14f71427d1e84f3dd0314c0f7b2291e5b200ac8c7c3b925918360005260018252604060002085600052825280604060002055604051908152a3565b60405162461bcd60e51b815260206004820152602260248201527f45524332303a20617070726f766520746f20746865207a65726f206164647265604482015261737360f01b6064820152608490fd5b60405162461bcd60e51b8152602060048201526024808201527f45524332303a20617070726f76652066726f6d20746865207a65726f206164646044820152637265737360e01b6064820152608490fd5b90816020910312610b7057516001600160a01b0381168103610b70579056fea2646970667358221220285c200b3978b10818ff576bb83f2dc4a2a7c98dfb6a36ea01170de792aa652764736f6c63430008140033000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000000c0000000000000000000000000d3fd4f95820a9aa848ce716d6c200eaefb9a2e4900000000000000000000000000000000000000000000000000000000000000640000000000000000000000000000000000000000000000000000000000000003543131000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000035431310000000000000000000000000000000000000000000000000000000000c001a04e551c75810ffdfe6caff57da9f5a8732449f42f0f4c57f935b05250a76db3b6a046cd47e6d01914270c1ec0d9ac7fae7dfb240ec9a8b6ec7898c4d6aa174388f2"; let data = hex::decode(raw).unwrap(); - let tx = PooledTransaction::decode_2718(&mut data.as_ref()).unwrap(); + let tx = PooledTransactionVariant::decode_2718(&mut data.as_ref()).unwrap(); EthPooledTransaction::from_pooled(tx.try_into_recovered().unwrap()) } @@ -1047,6 +1307,8 @@ mod tests { shanghai: false.into(), cancun: false.into(), prague: false.into(), + osaka: false.into(), + tip_timestamp: 0.into(), max_blob_count: 0.into(), }; @@ -1185,4 +1447,291 @@ mod tests { let outcome = validator.validate_one(TransactionOrigin::Local, transaction); assert!(outcome.is_valid()); } + + #[tokio::test] + async fn invalid_on_max_tx_gas_limit_exceeded() { + let transaction = get_transaction(); + let provider = MockEthProvider::default(); + provider.add_account( + transaction.sender(), + ExtendedAccount::new(transaction.nonce(), U256::MAX), + ); + + let blob_store = InMemoryBlobStore::default(); + let validator = EthTransactionValidatorBuilder::new(provider) + .with_max_tx_gas_limit(Some(500_000)) // Set limit lower than transaction gas limit (1_015_288) + .build(blob_store.clone()); + + let outcome = validator.validate_one(TransactionOrigin::External, transaction.clone()); + assert!(outcome.is_invalid()); + + let pool = + Pool::new(validator, CoinbaseTipOrdering::default(), blob_store, Default::default()); + + let res = pool.add_external_transaction(transaction.clone()).await; + assert!(res.is_err()); + assert!(matches!( + res.unwrap_err().kind, + PoolErrorKind::InvalidTransaction(InvalidPoolTransactionError::MaxTxGasLimitExceeded( + 1_015_288, 500_000 + )) + )); + let tx = pool.get(transaction.hash()); + assert!(tx.is_none()); + } + + #[tokio::test] + async fn valid_on_max_tx_gas_limit_disabled() { + let transaction = get_transaction(); + let provider = MockEthProvider::default(); + provider.add_account( + transaction.sender(), + ExtendedAccount::new(transaction.nonce(), U256::MAX), + ); + + let blob_store = InMemoryBlobStore::default(); + let validator = EthTransactionValidatorBuilder::new(provider) + .with_max_tx_gas_limit(None) // disabled + .build(blob_store); + + let outcome = validator.validate_one(TransactionOrigin::External, transaction); + assert!(outcome.is_valid()); + } + + #[tokio::test] + async fn valid_on_max_tx_gas_limit_within_limit() { + let transaction = get_transaction(); + let provider = MockEthProvider::default(); + provider.add_account( + transaction.sender(), + ExtendedAccount::new(transaction.nonce(), U256::MAX), + ); + + let blob_store = InMemoryBlobStore::default(); + let validator = EthTransactionValidatorBuilder::new(provider) + .with_max_tx_gas_limit(Some(2_000_000)) // Set limit higher than transaction gas limit (1_015_288) + .build(blob_store); + + let outcome = validator.validate_one(TransactionOrigin::External, transaction); + assert!(outcome.is_valid()); + } + + // Helper function to set up common test infrastructure for priority fee tests + fn setup_priority_fee_test() -> (EthPooledTransaction, MockEthProvider) { + let transaction = get_transaction(); + let provider = MockEthProvider::default(); + provider.add_account( + transaction.sender(), + ExtendedAccount::new(transaction.nonce(), U256::MAX), + ); + (transaction, provider) + } + + // Helper function to create a validator with minimum priority fee + fn create_validator_with_minimum_fee( + provider: MockEthProvider, + minimum_priority_fee: Option, + local_config: Option, + ) -> EthTransactionValidator { + let blob_store = InMemoryBlobStore::default(); + let mut builder = EthTransactionValidatorBuilder::new(provider) + .with_minimum_priority_fee(minimum_priority_fee); + + if let Some(config) = local_config { + builder = builder.with_local_transactions_config(config); + } + + builder.build(blob_store) + } + + #[tokio::test] + async fn invalid_on_priority_fee_lower_than_configured_minimum() { + let (transaction, provider) = setup_priority_fee_test(); + + // Verify the test transaction is a dynamic fee transaction + assert!(transaction.is_dynamic_fee()); + + // Set minimum priority fee to be double the transaction's priority fee + let minimum_priority_fee = + transaction.max_priority_fee_per_gas().expect("priority fee is expected") * 2; + + let validator = + create_validator_with_minimum_fee(provider, Some(minimum_priority_fee), None); + + // External transaction should be rejected due to low priority fee + let outcome = validator.validate_one(TransactionOrigin::External, transaction.clone()); + assert!(outcome.is_invalid()); + + if let TransactionValidationOutcome::Invalid(_, err) = outcome { + assert!(matches!( + err, + InvalidPoolTransactionError::PriorityFeeBelowMinimum { minimum_priority_fee: min_fee } + if min_fee == minimum_priority_fee + )); + } + + // Test pool integration + let blob_store = InMemoryBlobStore::default(); + let pool = + Pool::new(validator, CoinbaseTipOrdering::default(), blob_store, Default::default()); + + let res = pool.add_external_transaction(transaction.clone()).await; + assert!(res.is_err()); + assert!(matches!( + res.unwrap_err().kind, + PoolErrorKind::InvalidTransaction( + InvalidPoolTransactionError::PriorityFeeBelowMinimum { .. } + ) + )); + let tx = pool.get(transaction.hash()); + assert!(tx.is_none()); + + // Local transactions should still be accepted regardless of minimum priority fee + let (_, local_provider) = setup_priority_fee_test(); + let validator_local = + create_validator_with_minimum_fee(local_provider, Some(minimum_priority_fee), None); + + let local_outcome = validator_local.validate_one(TransactionOrigin::Local, transaction); + assert!(local_outcome.is_valid()); + } + + #[tokio::test] + async fn valid_on_priority_fee_equal_to_minimum() { + let (transaction, provider) = setup_priority_fee_test(); + + // Set minimum priority fee equal to transaction's priority fee + let tx_priority_fee = + transaction.max_priority_fee_per_gas().expect("priority fee is expected"); + let validator = create_validator_with_minimum_fee(provider, Some(tx_priority_fee), None); + + let outcome = validator.validate_one(TransactionOrigin::External, transaction); + assert!(outcome.is_valid()); + } + + #[tokio::test] + async fn valid_on_priority_fee_above_minimum() { + let (transaction, provider) = setup_priority_fee_test(); + + // Set minimum priority fee below transaction's priority fee + let tx_priority_fee = + transaction.max_priority_fee_per_gas().expect("priority fee is expected"); + let minimum_priority_fee = tx_priority_fee / 2; // Half of transaction's priority fee + + let validator = + create_validator_with_minimum_fee(provider, Some(minimum_priority_fee), None); + + let outcome = validator.validate_one(TransactionOrigin::External, transaction); + assert!(outcome.is_valid()); + } + + #[tokio::test] + async fn valid_on_minimum_priority_fee_disabled() { + let (transaction, provider) = setup_priority_fee_test(); + + // No minimum priority fee set (default is None) + let validator = create_validator_with_minimum_fee(provider, None, None); + + let outcome = validator.validate_one(TransactionOrigin::External, transaction); + assert!(outcome.is_valid()); + } + + #[tokio::test] + async fn priority_fee_validation_applies_to_private_transactions() { + let (transaction, provider) = setup_priority_fee_test(); + + // Set minimum priority fee to be double the transaction's priority fee + let minimum_priority_fee = + transaction.max_priority_fee_per_gas().expect("priority fee is expected") * 2; + + let validator = + create_validator_with_minimum_fee(provider, Some(minimum_priority_fee), None); + + // Private transactions are also subject to minimum priority fee validation + // because they are not considered "local" by default unless specifically configured + let outcome = validator.validate_one(TransactionOrigin::Private, transaction); + assert!(outcome.is_invalid()); + + if let TransactionValidationOutcome::Invalid(_, err) = outcome { + assert!(matches!( + err, + InvalidPoolTransactionError::PriorityFeeBelowMinimum { minimum_priority_fee: min_fee } + if min_fee == minimum_priority_fee + )); + } + } + + #[tokio::test] + async fn valid_on_local_config_exempts_private_transactions() { + let (transaction, provider) = setup_priority_fee_test(); + + // Set minimum priority fee to be double the transaction's priority fee + let minimum_priority_fee = + transaction.max_priority_fee_per_gas().expect("priority fee is expected") * 2; + + // Configure local transactions to include all private transactions + let local_config = + LocalTransactionConfig { propagate_local_transactions: true, ..Default::default() }; + + let validator = create_validator_with_minimum_fee( + provider, + Some(minimum_priority_fee), + Some(local_config), + ); + + // With appropriate local config, the behavior depends on the local transaction logic + // This test documents the current behavior - private transactions are still validated + // unless the sender is specifically whitelisted in local_transactions_config + let outcome = validator.validate_one(TransactionOrigin::Private, transaction); + assert!(outcome.is_invalid()); // Still invalid because sender not in whitelist + } + + #[test] + fn reject_oversized_tx() { + let mut transaction = get_transaction(); + transaction.encoded_length = DEFAULT_MAX_TX_INPUT_BYTES + 1; + let provider = MockEthProvider::default(); + + // No minimum priority fee set (default is None) + let validator = create_validator_with_minimum_fee(provider, None, None); + + let outcome = validator.validate_one(TransactionOrigin::External, transaction); + let invalid = outcome.as_invalid().unwrap(); + assert!(invalid.is_oversized()); + } + + #[tokio::test] + async fn valid_with_disabled_balance_check() { + let transaction = get_transaction(); + let provider = MockEthProvider::default(); + + // Set account with 0 balance + provider.add_account( + transaction.sender(), + ExtendedAccount::new(transaction.nonce(), alloy_primitives::U256::ZERO), + ); + + // Valdiate with balance check enabled + let validator = EthTransactionValidatorBuilder::new(provider.clone()) + .build(InMemoryBlobStore::default()); + + let outcome = validator.validate_one(TransactionOrigin::External, transaction.clone()); + let expected_cost = *transaction.cost(); + if let TransactionValidationOutcome::Invalid(_, err) = outcome { + assert!(matches!( + err, + InvalidPoolTransactionError::Consensus(InvalidTransactionError::InsufficientFunds(ref funds_err)) + if funds_err.got == alloy_primitives::U256::ZERO && funds_err.expected == expected_cost + )); + } else { + panic!("Expected Invalid outcome with InsufficientFunds error"); + } + + // Valdiate with balance check disabled + let validator = EthTransactionValidatorBuilder::new(provider) + .disable_balance_check() // This should allow the transaction through despite zero balance + .build(InMemoryBlobStore::default()); + + let outcome = validator.validate_one(TransactionOrigin::External, transaction); + assert!(outcome.is_valid()); // Should be valid because balance check is disabled + } } diff --git a/crates/transaction-pool/src/validate/mod.rs b/crates/transaction-pool/src/validate/mod.rs index 577f7b82c68..bccd4d7b347 100644 --- a/crates/transaction-pool/src/validate/mod.rs +++ b/crates/transaction-pool/src/validate/mod.rs @@ -6,7 +6,7 @@ use crate::{ traits::{PoolTransaction, TransactionOrigin}, PriceBumpConfig, }; -use alloy_eips::{eip4844::BlobTransactionSidecar, eip7702::SignedAuthorization}; +use alloy_eips::{eip7594::BlobTransactionSidecarVariant, eip7702::SignedAuthorization}; use alloy_primitives::{Address, TxHash, B256, U256}; use futures_util::future::Either; use reth_primitives_traits::{Recovered, SealedBlock}; @@ -66,6 +66,14 @@ impl TransactionValidationOutcome { } } + /// Returns the [`InvalidPoolTransactionError`] if this is an invalid variant. + pub const fn as_invalid(&self) -> Option<&InvalidPoolTransactionError> { + match self { + Self::Invalid(_, err) => Some(err), + _ => None, + } + } + /// Returns true if the transaction is valid. pub const fn is_valid(&self) -> bool { matches!(self, Self::Valid { .. }) @@ -103,13 +111,13 @@ pub enum ValidTransaction { /// The valid EIP-4844 transaction. transaction: T, /// The extracted sidecar of that transaction - sidecar: BlobTransactionSidecar, + sidecar: BlobTransactionSidecarVariant, }, } impl ValidTransaction { /// Creates a new valid transaction with an optional sidecar. - pub fn new(transaction: T, sidecar: Option) -> Self { + pub fn new(transaction: T, sidecar: Option) -> Self { if let Some(sidecar) = sidecar { Self::ValidWithSidecar { transaction, sidecar } } else { @@ -198,12 +206,23 @@ pub trait TransactionValidator: Debug + Send + Sync { &self, transactions: Vec<(TransactionOrigin, Self::Transaction)>, ) -> impl Future>> + Send { - async { - futures_util::future::join_all( - transactions.into_iter().map(|(origin, tx)| self.validate_transaction(origin, tx)), - ) - .await - } + futures_util::future::join_all( + transactions.into_iter().map(|(origin, tx)| self.validate_transaction(origin, tx)), + ) + } + + /// Validates a batch of transactions with that given origin. + /// + /// Must return all outcomes for the given transactions in the same order. + /// + /// See also [`Self::validate_transaction`]. + fn validate_transactions_with_origin( + &self, + origin: TransactionOrigin, + transactions: impl IntoIterator + Send, + ) -> impl Future>> + Send { + let futures = transactions.into_iter().map(|tx| self.validate_transaction(origin, tx)); + futures_util::future::join_all(futures) } /// Invoked when the head block changes. @@ -244,6 +263,17 @@ where } } + async fn validate_transactions_with_origin( + &self, + origin: TransactionOrigin, + transactions: impl IntoIterator + Send, + ) -> Vec> { + match self { + Self::Left(v) => v.validate_transactions_with_origin(origin, transactions).await, + Self::Right(v) => v.validate_transactions_with_origin(origin, transactions).await, + } + } + fn on_new_head_block(&self, new_tip_block: &SealedBlock) where Bl: Block, @@ -305,12 +335,12 @@ impl ValidPoolTransaction { } /// Returns the internal identifier for the sender of this transaction - pub(crate) const fn sender_id(&self) -> SenderId { + pub const fn sender_id(&self) -> SenderId { self.transaction_id.sender } /// Returns the internal identifier for this transaction. - pub(crate) const fn id(&self) -> &TransactionId { + pub const fn id(&self) -> &TransactionId { &self.transaction_id } @@ -485,7 +515,7 @@ impl fmt::Debug for ValidPoolTransaction { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("ValidPoolTransaction") .field("id", &self.transaction_id) - .field("pragate", &self.propagate) + .field("propagate", &self.propagate) .field("origin", &self.origin) .field("hash", self.transaction.hash()) .field("tx", &self.transaction) diff --git a/crates/transaction-pool/src/validate/task.rs b/crates/transaction-pool/src/validate/task.rs index 558cd6dfa2b..fc22ce4ceb1 100644 --- a/crates/transaction-pool/src/validate/task.rs +++ b/crates/transaction-pool/src/validate/task.rs @@ -2,6 +2,7 @@ use crate::{ blobstore::BlobStore, + metrics::TxPoolValidatorMetrics, validate::{EthTransactionValidatorBuilder, TransactionValidatorError}, EthTransactionValidator, PoolTransaction, TransactionOrigin, TransactionValidationOutcome, TransactionValidator, @@ -33,10 +34,18 @@ pub struct ValidationTask { } impl ValidationTask { - /// Creates a new cloneable task pair + /// Creates a new cloneable task pair. + /// + /// The sender sends new (transaction) validation tasks to an available validation task. pub fn new() -> (ValidationJobSender, Self) { - let (tx, rx) = mpsc::channel(1); - (ValidationJobSender { tx }, Self::with_receiver(rx)) + Self::with_capacity(1) + } + + /// Creates a new cloneable task pair with the given channel capacity. + pub fn with_capacity(capacity: usize) -> (ValidationJobSender, Self) { + let (tx, rx) = mpsc::channel(capacity); + let metrics = TxPoolValidatorMetrics::default(); + (ValidationJobSender { tx, metrics }, Self::with_receiver(rx)) } /// Creates a new task with the given receiver. @@ -64,6 +73,7 @@ impl std::fmt::Debug for ValidationTask { #[derive(Debug)] pub struct ValidationJobSender { tx: mpsc::Sender + Send>>>, + metrics: TxPoolValidatorMetrics, } impl ValidationJobSender { @@ -72,20 +82,36 @@ impl ValidationJobSender { &self, job: Pin + Send>>, ) -> Result<(), TransactionValidatorError> { - self.tx.send(job).await.map_err(|_| TransactionValidatorError::ValidationServiceUnreachable) + self.metrics.inflight_validation_jobs.increment(1); + let res = self + .tx + .send(job) + .await + .map_err(|_| TransactionValidatorError::ValidationServiceUnreachable); + self.metrics.inflight_validation_jobs.decrement(1); + res } } /// A [`TransactionValidator`] implementation that validates ethereum transaction. /// This validator is non-blocking, all validation work is done in a separate task. -#[derive(Debug, Clone)] +#[derive(Debug)] pub struct TransactionValidationTaskExecutor { /// The validator that will validate transactions on a separate task. - pub validator: V, + pub validator: Arc, /// The sender half to validation tasks that perform the actual validation. pub to_validation_task: Arc>, } +impl Clone for TransactionValidationTaskExecutor { + fn clone(&self) -> Self { + Self { + validator: self.validator.clone(), + to_validation_task: self.to_validation_task.clone(), + } + } +} + // === impl TransactionValidationTaskExecutor === impl TransactionValidationTaskExecutor<()> { @@ -102,10 +128,15 @@ impl TransactionValidationTaskExecutor { F: FnMut(V) -> T, { TransactionValidationTaskExecutor { - validator: f(self.validator), + validator: Arc::new(f(Arc::into_inner(self.validator).unwrap())), to_validation_task: self.to_validation_task, } } + + /// Returns the validator. + pub fn validator(&self) -> &V { + &self.validator + } } impl TransactionValidationTaskExecutor> { @@ -151,13 +182,13 @@ impl TransactionValidationTaskExecutor { /// validation tasks. pub fn new(validator: V) -> Self { let (tx, _) = ValidationTask::new(); - Self { validator, to_validation_task: Arc::new(sync::Mutex::new(tx)) } + Self { validator: Arc::new(validator), to_validation_task: Arc::new(sync::Mutex::new(tx)) } } } impl TransactionValidator for TransactionValidationTaskExecutor where - V: TransactionValidator + Clone + 'static, + V: TransactionValidator + 'static, { type Transaction = ::Transaction; @@ -171,20 +202,19 @@ where { let res = { let to_validation_task = self.to_validation_task.clone(); - let to_validation_task = to_validation_task.lock().await; let validator = self.validator.clone(); - to_validation_task - .send(Box::pin(async move { - let res = validator.validate_transaction(origin, transaction).await; - let _ = tx.send(res); - })) - .await + let fut = Box::pin(async move { + let res = validator.validate_transaction(origin, transaction).await; + let _ = tx.send(res); + }); + let to_validation_task = to_validation_task.lock().await; + to_validation_task.send(fut).await }; if res.is_err() { return TransactionValidationOutcome::Error( hash, Box::new(TransactionValidatorError::ValidationServiceUnreachable), - ) + ); } } @@ -197,6 +227,57 @@ where } } + async fn validate_transactions( + &self, + transactions: Vec<(TransactionOrigin, Self::Transaction)>, + ) -> Vec> { + let hashes: Vec<_> = transactions.iter().map(|(_, tx)| *tx.hash()).collect(); + let (tx, rx) = oneshot::channel(); + { + let res = { + let to_validation_task = self.to_validation_task.clone(); + let validator = self.validator.clone(); + let fut = Box::pin(async move { + let res = validator.validate_transactions(transactions).await; + let _ = tx.send(res); + }); + let to_validation_task = to_validation_task.lock().await; + to_validation_task.send(fut).await + }; + if res.is_err() { + return hashes + .into_iter() + .map(|hash| { + TransactionValidationOutcome::Error( + hash, + Box::new(TransactionValidatorError::ValidationServiceUnreachable), + ) + }) + .collect(); + } + } + match rx.await { + Ok(res) => res, + Err(_) => hashes + .into_iter() + .map(|hash| { + TransactionValidationOutcome::Error( + hash, + Box::new(TransactionValidatorError::ValidationServiceUnreachable), + ) + }) + .collect(), + } + } + + async fn validate_transactions_with_origin( + &self, + origin: TransactionOrigin, + transactions: impl IntoIterator + Send, + ) -> Vec> { + self.validate_transactions(transactions.into_iter().map(|tx| (origin, tx)).collect()).await + } + fn on_new_head_block(&self, new_tip_block: &SealedBlock) where B: Block, diff --git a/crates/transaction-pool/tests/it/blobs.rs b/crates/transaction-pool/tests/it/blobs.rs index 9417c62278b..9f7e224a235 100644 --- a/crates/transaction-pool/tests/it/blobs.rs +++ b/crates/transaction-pool/tests/it/blobs.rs @@ -3,7 +3,7 @@ use reth_transaction_pool::{ error::PoolErrorKind, test_utils::{MockTransaction, MockTransactionFactory, TestPoolBuilder}, - PoolTransaction, TransactionOrigin, TransactionPool, + AddedTransactionOutcome, PoolTransaction, TransactionOrigin, TransactionPool, }; #[tokio::test(flavor = "multi_thread")] @@ -12,7 +12,7 @@ async fn blobs_exclusive() { let mut mock_tx_factory = MockTransactionFactory::default(); let blob_tx = mock_tx_factory.create_eip4844(); - let hash = txpool + let AddedTransactionOutcome { hash, .. } = txpool .add_transaction(TransactionOrigin::External, blob_tx.transaction.clone()) .await .unwrap(); diff --git a/crates/transaction-pool/tests/it/evict.rs b/crates/transaction-pool/tests/it/evict.rs index 721988888b3..5a869702457 100644 --- a/crates/transaction-pool/tests/it/evict.rs +++ b/crates/transaction-pool/tests/it/evict.rs @@ -9,7 +9,8 @@ use reth_transaction_pool::{ test_utils::{ MockFeeRange, MockTransactionDistribution, MockTransactionRatio, TestPool, TestPoolBuilder, }, - BlockInfo, PoolConfig, SubPoolLimit, TransactionOrigin, TransactionPool, TransactionPoolExt, + AddedTransactionOutcome, BlockInfo, PoolConfig, SubPoolLimit, TransactionOrigin, + TransactionPool, TransactionPoolExt, }; #[tokio::test(flavor = "multi_thread")] @@ -97,7 +98,7 @@ async fn only_blobs_eviction() { let results = pool.add_transactions(TransactionOrigin::External, set).await; for (i, result) in results.iter().enumerate() { match result { - Ok(hash) => { + Ok(AddedTransactionOutcome { hash, .. }) => { println!("✅ Inserted tx into pool with hash: {hash}"); } Err(e) => { diff --git a/crates/transaction-pool/tests/it/listeners.rs b/crates/transaction-pool/tests/it/listeners.rs index 5eb296e8ae7..105caae12b4 100644 --- a/crates/transaction-pool/tests/it/listeners.rs +++ b/crates/transaction-pool/tests/it/listeners.rs @@ -82,7 +82,7 @@ async fn txpool_listener_queued_event() { assert_matches!(events.next().await, Some(TransactionEvent::Queued)); // The listener of all should receive queued event as well. - assert_matches!(all_tx_events.next().await, Some(FullTransactionEvent::Queued(hash)) if hash == *transaction.get_hash()); + assert_matches!(all_tx_events.next().await, Some(FullTransactionEvent::Queued(hash,_ )) if hash == *transaction.get_hash()); } #[tokio::test(flavor = "multi_thread")] @@ -113,7 +113,7 @@ async fn txpool_listener_all() { let added_result = txpool.add_transaction(TransactionOrigin::External, transaction.transaction.clone()).await; - assert_matches!(added_result, Ok(hash) if hash == *transaction.transaction.get_hash()); + assert_matches!(added_result, Ok(outcome) if outcome.hash == *transaction.transaction.get_hash()); assert_matches!( all_tx_events.next().await, diff --git a/crates/transaction-pool/tests/it/pending.rs b/crates/transaction-pool/tests/it/pending.rs index be559c71eec..095dcfe5085 100644 --- a/crates/transaction-pool/tests/it/pending.rs +++ b/crates/transaction-pool/tests/it/pending.rs @@ -12,7 +12,7 @@ async fn txpool_new_pending_txs() { let added_result = txpool.add_transaction(TransactionOrigin::External, transaction.transaction.clone()).await; - assert_matches!(added_result, Ok(hash) if hash == *transaction.transaction.get_hash()); + assert_matches!(added_result, Ok(outcome) if outcome.hash == *transaction.transaction.get_hash()); let mut best_txns = txpool.best_transactions(); assert_matches!(best_txns.next(), Some(tx) if tx.transaction.get_hash() == transaction.transaction.get_hash()); @@ -20,6 +20,6 @@ async fn txpool_new_pending_txs() { let transaction = mock_tx_factory.create_eip1559(); let added_result = txpool.add_transaction(TransactionOrigin::External, transaction.transaction.clone()).await; - assert_matches!(added_result, Ok(hash) if hash == *transaction.transaction.get_hash()); + assert_matches!(added_result, Ok(outcome) if outcome.hash == *transaction.transaction.get_hash()); assert_matches!(best_txns.next(), Some(tx) if tx.transaction.get_hash() == transaction.transaction.get_hash()); } diff --git a/crates/trie/common/Cargo.toml b/crates/trie/common/Cargo.toml index 4c22c18247f..2fcc23ab53b 100644 --- a/crates/trie/common/Cargo.toml +++ b/crates/trie/common/Cargo.toml @@ -23,6 +23,7 @@ reth-codecs = { workspace = true, optional = true } alloy-rpc-types-eth = { workspace = true, optional = true } alloy-serde = { workspace = true, optional = true } +arrayvec = { workspace = true, optional = true } bytes = { workspace = true, optional = true } derive_more.workspace = true itertools = { workspace = true, features = ["use_alloc"] } @@ -52,6 +53,7 @@ alloy-genesis.workspace = true alloy-primitives = { workspace = true, features = ["getrandom"] } alloy-trie = { workspace = true, features = ["arbitrary", "serde"] } bytes.workspace = true +arrayvec.workspace = true hash-db.workspace = true plain_hasher.workspace = true arbitrary = { workspace = true, features = ["derive"] } @@ -74,6 +76,7 @@ std = [ "alloy-rpc-types-eth?/std", "alloy-serde?/std", "alloy-trie/std", + "arrayvec?/std", "bytes?/std", "derive_more/std", "nybbles/std", @@ -84,12 +87,10 @@ std = [ "revm-database/std", "revm-state/std", ] -eip1186 = [ - "alloy-rpc-types-eth/serde", - "dep:alloy-serde", -] +eip1186 = ["alloy-rpc-types-eth/serde", "dep:alloy-serde"] serde = [ "dep:serde", + "arrayvec?/serde", "bytes?/serde", "nybbles/serde", "alloy-primitives/serde", @@ -101,15 +102,14 @@ serde = [ "revm-database/serde", "revm-state/serde", ] -reth-codec = [ - "dep:reth-codecs", - "dep:bytes", -] +reth-codec = ["dep:reth-codecs", "dep:bytes", "dep:arrayvec"] serde-bincode-compat = [ "serde", "reth-primitives-traits/serde-bincode-compat", "alloy-consensus/serde-bincode-compat", "dep:serde_with", + "alloy-genesis/serde-bincode-compat", + "alloy-rpc-types-eth?/serde-bincode-compat", ] test-utils = [ "dep:plain_hasher", diff --git a/crates/trie/common/benches/prefix_set.rs b/crates/trie/common/benches/prefix_set.rs index 5883b2d17dd..bc2a8dc2592 100644 --- a/crates/trie/common/benches/prefix_set.rs +++ b/crates/trie/common/benches/prefix_set.rs @@ -97,7 +97,7 @@ fn prefix_set_bench( let setup = || { let mut prefix_set = T::default(); for key in &preload { - prefix_set.insert(key.clone()); + prefix_set.insert(*key); } (prefix_set.freeze(), input.clone(), expected.clone()) }; @@ -131,7 +131,7 @@ fn generate_test_data(size: usize) -> (Vec, Vec, Vec) { let expected = input .iter() - .map(|prefix| preload.iter().any(|key| key.has_prefix(prefix))) + .map(|prefix| preload.iter().any(|key| key.starts_with(prefix))) .collect::>(); (preload, input, expected) } @@ -162,7 +162,7 @@ mod implementations { impl PrefixSetAbstraction for BTreeAnyPrefixSet { fn contains(&mut self, key: Nibbles) -> bool { - self.keys.iter().any(|k| k.has_prefix(&key)) + self.keys.iter().any(|k| k.starts_with(&key)) } } @@ -193,7 +193,7 @@ mod implementations { None => (Bound::Unbounded, Bound::Unbounded), }; for key in self.keys.range::(range) { - if key.has_prefix(&prefix) { + if key.starts_with(&prefix) { self.last_checked = Some(prefix); return true } @@ -237,7 +237,7 @@ mod implementations { match self.keys.binary_search(&prefix) { Ok(_) => true, Err(idx) => match self.keys.get(idx) { - Some(key) => key.has_prefix(&prefix), + Some(key) => key.starts_with(&prefix), None => false, // prefix > last key }, } @@ -271,14 +271,12 @@ mod implementations { self.sorted = true; } - let prefix = prefix; - while self.index > 0 && self.keys[self.index] > prefix { self.index -= 1; } for (idx, key) in self.keys[self.index..].iter().enumerate() { - if key.has_prefix(&prefix) { + if key.starts_with(&prefix) { self.index += idx; return true } @@ -294,6 +292,7 @@ mod implementations { } #[derive(Default)] + #[allow(dead_code)] pub struct VecBinarySearchWithLastFoundPrefixSet { keys: Vec, last_found_idx: usize, @@ -329,7 +328,7 @@ mod implementations { Err(idx) => match self.keys.get(idx) { Some(key) => { self.last_found_idx = idx; - key.has_prefix(&prefix) + key.starts_with(&prefix) } None => false, // prefix > last key }, diff --git a/crates/trie/common/src/added_removed_keys.rs b/crates/trie/common/src/added_removed_keys.rs new file mode 100644 index 00000000000..8e61423718a --- /dev/null +++ b/crates/trie/common/src/added_removed_keys.rs @@ -0,0 +1,218 @@ +//! Tracking of keys having been added and removed from the tries. + +use crate::HashedPostState; +use alloy_primitives::{map::B256Map, B256}; +use alloy_trie::proof::AddedRemovedKeys; + +/// Tracks added and removed keys across account and storage tries. +#[derive(Debug, Clone)] +pub struct MultiAddedRemovedKeys { + account: AddedRemovedKeys, + storages: B256Map, +} + +/// Returns [`AddedRemovedKeys`] with default parameters. This is necessary while we are not yet +/// tracking added keys. +fn default_added_removed_keys() -> AddedRemovedKeys { + AddedRemovedKeys::default().with_assume_added(true) +} + +impl Default for MultiAddedRemovedKeys { + fn default() -> Self { + Self::new() + } +} + +impl MultiAddedRemovedKeys { + /// Returns a new instance. + pub fn new() -> Self { + Self { account: default_added_removed_keys(), storages: Default::default() } + } + + /// Updates the set of removed keys based on a [`HashedPostState`]. + /// + /// Storage keys set to [`alloy_primitives::U256::ZERO`] are added to the set for their + /// respective account. Keys set to any other value are removed from their respective + /// account. + pub fn update_with_state(&mut self, update: &HashedPostState) { + for (hashed_address, storage) in &update.storages { + let account = update + .accounts + .get(hashed_address) + .map(|entry| entry.unwrap_or_default()) + .unwrap_or_default(); + + if storage.wiped { + self.storages.remove(hashed_address); + if account.is_empty() { + self.account.insert_removed(*hashed_address); + } + continue + } + + let storage_removed_keys = + self.storages.entry(*hashed_address).or_insert_with(default_added_removed_keys); + + for (key, val) in &storage.storage { + if val.is_zero() { + storage_removed_keys.insert_removed(*key); + } else { + storage_removed_keys.remove_removed(key); + } + } + + if !account.is_empty() { + self.account.remove_removed(hashed_address); + } + } + } + + /// Returns a [`AddedRemovedKeys`] for the storage trie of a particular account, if any. + pub fn get_storage(&self, hashed_address: &B256) -> Option<&AddedRemovedKeys> { + self.storages.get(hashed_address) + } + + /// Returns an [`AddedRemovedKeys`] for tracking account-level changes. + pub const fn get_accounts(&self) -> &AddedRemovedKeys { + &self.account + } + + /// Marks an account as existing, and therefore having storage. + pub fn touch_accounts(&mut self, addresses: impl Iterator) { + for address in addresses { + self.storages.entry(address).or_insert_with(default_added_removed_keys); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::HashedStorage; + use alloy_primitives::U256; + use reth_primitives_traits::Account; + + #[test] + fn test_update_with_state_storage_keys_non_zero() { + let mut multi_keys = MultiAddedRemovedKeys::new(); + let mut update = HashedPostState::default(); + + let addr = B256::random(); + let slot1 = B256::random(); + let slot2 = B256::random(); + + // First mark slots as removed + let mut storage = HashedStorage::default(); + storage.storage.insert(slot1, U256::ZERO); + storage.storage.insert(slot2, U256::ZERO); + update.storages.insert(addr, storage); + multi_keys.update_with_state(&update); + + // Verify they are removed + assert!(multi_keys.get_storage(&addr).unwrap().is_removed(&slot1)); + assert!(multi_keys.get_storage(&addr).unwrap().is_removed(&slot2)); + + // Now update with non-zero values + let mut update2 = HashedPostState::default(); + let mut storage2 = HashedStorage::default(); + storage2.storage.insert(slot1, U256::from(100)); + storage2.storage.insert(slot2, U256::from(200)); + update2.storages.insert(addr, storage2); + multi_keys.update_with_state(&update2); + + // Slots should no longer be marked as removed + let storage_keys = multi_keys.get_storage(&addr).unwrap(); + assert!(!storage_keys.is_removed(&slot1)); + assert!(!storage_keys.is_removed(&slot2)); + } + + #[test] + fn test_update_with_state_wiped_storage() { + let mut multi_keys = MultiAddedRemovedKeys::new(); + let mut update = HashedPostState::default(); + + let addr = B256::random(); + let slot1 = B256::random(); + + // First add some removed keys + let mut storage = HashedStorage::default(); + storage.storage.insert(slot1, U256::ZERO); + update.storages.insert(addr, storage); + multi_keys.update_with_state(&update); + assert!(multi_keys.get_storage(&addr).is_some()); + + // Now wipe the storage + let mut update2 = HashedPostState::default(); + let wiped_storage = HashedStorage::new(true); + update2.storages.insert(addr, wiped_storage); + multi_keys.update_with_state(&update2); + + // Storage and account should be removed + assert!(multi_keys.get_storage(&addr).is_none()); + assert!(multi_keys.get_accounts().is_removed(&addr)); + } + + #[test] + fn test_update_with_state_account_tracking() { + let mut multi_keys = MultiAddedRemovedKeys::new(); + let mut update = HashedPostState::default(); + + let addr = B256::random(); + let slot = B256::random(); + + // Add storage with zero value and empty account + let mut storage = HashedStorage::default(); + storage.storage.insert(slot, U256::ZERO); + update.storages.insert(addr, storage); + // Account is implicitly empty (not in accounts map) + + multi_keys.update_with_state(&update); + + // Storage should have removed keys but account should not be removed + assert!(multi_keys.get_storage(&addr).unwrap().is_removed(&slot)); + assert!(!multi_keys.get_accounts().is_removed(&addr)); + + // Now clear all removed storage keys and keep account empty + let mut update2 = HashedPostState::default(); + let mut storage2 = HashedStorage::default(); + storage2.storage.insert(slot, U256::from(100)); // Non-zero removes from removed set + update2.storages.insert(addr, storage2); + + multi_keys.update_with_state(&update2); + + // Account should not be marked as removed still + assert!(!multi_keys.get_accounts().is_removed(&addr)); + } + + #[test] + fn test_update_with_state_account_with_balance() { + let mut multi_keys = MultiAddedRemovedKeys::new(); + let mut update = HashedPostState::default(); + + let addr = B256::random(); + + // Add account with non-empty state (has balance) + let account = Account { balance: U256::from(1000), nonce: 0, bytecode_hash: None }; + update.accounts.insert(addr, Some(account)); + + // Add empty storage + let storage = HashedStorage::default(); + update.storages.insert(addr, storage); + + multi_keys.update_with_state(&update); + + // Account should not be marked as removed because it has balance + assert!(!multi_keys.get_accounts().is_removed(&addr)); + + // Now wipe the storage + let mut update2 = HashedPostState::default(); + let wiped_storage = HashedStorage::new(true); + update2.storages.insert(addr, wiped_storage); + update2.accounts.insert(addr, Some(account)); + multi_keys.update_with_state(&update2); + + // Storage should be None, but account should not be removed. + assert!(multi_keys.get_storage(&addr).is_none()); + assert!(!multi_keys.get_accounts().is_removed(&addr)); + } +} diff --git a/crates/trie/common/src/hash_builder/state.rs b/crates/trie/common/src/hash_builder/state.rs index 76abbd42ac6..0df582f8f5c 100644 --- a/crates/trie/common/src/hash_builder/state.rs +++ b/crates/trie/common/src/hash_builder/state.rs @@ -51,7 +51,7 @@ impl From for HashBuilder { impl From for HashBuilderState { fn from(state: HashBuilder) -> Self { Self { - key: state.key.into(), + key: state.key.to_vec(), stack: state.stack, value: state.value, groups: state.state_masks, diff --git a/crates/trie/common/src/hashed_state.rs b/crates/trie/common/src/hashed_state.rs index 1875a132dca..8d99ee5ebbb 100644 --- a/crates/trie/common/src/hashed_state.rs +++ b/crates/trie/common/src/hashed_state.rs @@ -1,7 +1,9 @@ use core::ops::Not; use crate::{ + added_removed_keys::MultiAddedRemovedKeys, prefix_set::{PrefixSetMut, TriePrefixSetsMut}, + utils::extend_sorted_vec, KeyHasher, MultiProofTargets, Nibbles, }; use alloc::{borrow::Cow, vec::Vec}; @@ -22,6 +24,7 @@ use revm_database::{AccountStatus, BundleAccount}; /// Representation of in-memory hashed state. #[derive(PartialEq, Eq, Clone, Default, Debug)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] pub struct HashedPostState { /// Mapping of hashed address to account info, `None` if destroyed. pub accounts: B256Map>, @@ -206,15 +209,23 @@ impl HashedPostState { /// /// CAUTION: The state updates are expected to be applied in order, so that the storage wipes /// are done correctly. - pub fn partition_by_targets(mut self, targets: &MultiProofTargets) -> (Self, Self) { + pub fn partition_by_targets( + mut self, + targets: &MultiProofTargets, + added_removed_keys: &MultiAddedRemovedKeys, + ) -> (Self, Self) { let mut state_updates_not_in_targets = Self::default(); self.storages.retain(|&address, storage| { + let storage_added_removed_keys = added_removed_keys.get_storage(&address); + let (retain, storage_not_in_targets) = match targets.get(&address) { Some(storage_in_targets) => { let mut storage_not_in_targets = HashedStorage::default(); storage.storage.retain(|&slot, value| { - if storage_in_targets.contains(&slot) { + if storage_in_targets.contains(&slot) && + !storage_added_removed_keys.is_some_and(|k| k.is_removed(&slot)) + { return true } @@ -267,6 +278,15 @@ impl HashedPostState { ChunkedHashedPostState::new(self, size) } + /// Returns the number of items that will be considered during chunking in `[Self::chunks]`. + pub fn chunking_length(&self) -> usize { + self.accounts.len() + + self.storages + .values() + .map(|storage| if storage.wiped { 1 } else { 0 } + storage.storage.len()) + .sum::() + } + /// Extend this hashed post state with contents of another. /// Entries in the second hashed post state take precedence. pub fn extend(&mut self, other: Self) { @@ -333,10 +353,46 @@ impl HashedPostState { HashedPostStateSorted { accounts, storages } } + + /// Converts hashed post state into [`HashedPostStateSorted`], but keeping the maps allocated by + /// draining. + /// + /// This effectively clears all the fields in the [`HashedPostStateSorted`]. + /// + /// This allows us to reuse the allocated space. This allocates new space for the sorted hashed + /// post state, like `into_sorted`. + pub fn drain_into_sorted(&mut self) -> HashedPostStateSorted { + let mut updated_accounts = Vec::new(); + let mut destroyed_accounts = HashSet::default(); + for (hashed_address, info) in self.accounts.drain() { + if let Some(info) = info { + updated_accounts.push((hashed_address, info)); + } else { + destroyed_accounts.insert(hashed_address); + } + } + updated_accounts.sort_unstable_by_key(|(address, _)| *address); + let accounts = HashedAccountsSorted { accounts: updated_accounts, destroyed_accounts }; + + let storages = self + .storages + .drain() + .map(|(hashed_address, storage)| (hashed_address, storage.into_sorted())) + .collect(); + + HashedPostStateSorted { accounts, storages } + } + + /// Clears the account and storage maps of this `HashedPostState`. + pub fn clear(&mut self) { + self.accounts.clear(); + self.storages.clear(); + } } /// Representation of in-memory hashed storage. #[derive(PartialEq, Eq, Clone, Debug, Default)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] pub struct HashedStorage { /// Flag indicating whether the storage was wiped or not. pub wiped: bool, @@ -438,6 +494,41 @@ impl HashedPostStateSorted { pub const fn account_storages(&self) -> &B256Map { &self.storages } + + /// Returns `true` if there are no account or storage updates. + pub fn is_empty(&self) -> bool { + self.accounts.accounts.is_empty() && + self.accounts.destroyed_accounts.is_empty() && + self.storages.is_empty() + } + + /// Returns the total number of updates including all accounts and storage updates. + pub fn total_len(&self) -> usize { + self.accounts.accounts.len() + + self.accounts.destroyed_accounts.len() + + self.storages.values().map(|storage| storage.len()).sum::() + } + + /// Extends this state with contents of another sorted state. + /// Entries in `other` take precedence for duplicate keys. + pub fn extend_ref(&mut self, other: &Self) { + // Extend accounts + self.accounts.extend_ref(&other.accounts); + + // Extend storages + for (hashed_address, other_storage) in &other.storages { + self.storages + .entry(*hashed_address) + .and_modify(|existing| existing.extend_ref(other_storage)) + .or_insert_with(|| other_storage.clone()); + } + } +} + +impl AsRef for HashedPostStateSorted { + fn as_ref(&self) -> &Self { + self + } } /// Sorted account state optimized for iterating during state trie calculation. @@ -458,6 +549,20 @@ impl HashedAccountsSorted { .chain(self.destroyed_accounts.iter().map(|address| (*address, None))) .sorted_by_key(|entry| *entry.0) } + + /// Extends this collection with contents of another sorted collection. + /// Entries in `other` take precedence for duplicate keys. + pub fn extend_ref(&mut self, other: &Self) { + // Updates take precedence over removals, so we want removals from `other` to only apply to + // the previous accounts. + self.accounts.retain(|(addr, _)| !other.destroyed_accounts.contains(addr)); + + // Extend the sorted accounts vector + extend_sorted_vec(&mut self.accounts, &other.accounts); + + // Merge destroyed accounts sets + self.destroyed_accounts.extend(&other.destroyed_accounts); + } } /// Sorted hashed storage optimized for iterating during state trie calculation. @@ -485,6 +590,38 @@ impl HashedStorageSorted { .chain(self.zero_valued_slots.iter().map(|hashed_slot| (*hashed_slot, U256::ZERO))) .sorted_by_key(|entry| *entry.0) } + + /// Returns the total number of storage slot updates. + pub fn len(&self) -> usize { + self.non_zero_valued_slots.len() + self.zero_valued_slots.len() + } + + /// Returns `true` if there are no storage slot updates. + pub fn is_empty(&self) -> bool { + self.non_zero_valued_slots.is_empty() && self.zero_valued_slots.is_empty() + } + + /// Extends this storage with contents of another sorted storage. + /// Entries in `other` take precedence for duplicate keys. + pub fn extend_ref(&mut self, other: &Self) { + if other.wiped { + // If other is wiped, clear everything and copy from other + self.wiped = true; + self.non_zero_valued_slots.clear(); + self.zero_valued_slots.clear(); + self.non_zero_valued_slots.extend_from_slice(&other.non_zero_valued_slots); + self.zero_valued_slots.extend(&other.zero_valued_slots); + return; + } + + self.non_zero_valued_slots.retain(|(slot, _)| !other.zero_valued_slots.contains(slot)); + + // Extend the sorted non-zero valued slots + extend_sorted_vec(&mut self.non_zero_valued_slots, &other.non_zero_valued_slots); + + // Merge zero valued slots sets + self.zero_valued_slots.extend(&other.zero_valued_slots); + } } /// An iterator that yields chunks of the state updates of at most `size` account and storage @@ -938,7 +1075,8 @@ mod tests { }; let targets = MultiProofTargets::from_iter([(addr1, HashSet::from_iter([slot1]))]); - let (with_targets, without_targets) = state.partition_by_targets(&targets); + let (with_targets, without_targets) = + state.partition_by_targets(&targets, &MultiAddedRemovedKeys::new()); assert_eq!( with_targets, @@ -1019,4 +1157,163 @@ mod tests { ); assert_eq!(chunks.next(), None); } + + #[test] + fn test_hashed_post_state_sorted_extend_ref() { + // Test extending accounts + let mut state1 = HashedPostStateSorted { + accounts: HashedAccountsSorted { + accounts: vec![ + (B256::from([1; 32]), Account::default()), + (B256::from([3; 32]), Account::default()), + ], + destroyed_accounts: B256Set::from_iter([B256::from([5; 32])]), + }, + storages: B256Map::default(), + }; + + let state2 = HashedPostStateSorted { + accounts: HashedAccountsSorted { + accounts: vec![ + (B256::from([2; 32]), Account::default()), + (B256::from([3; 32]), Account { nonce: 1, ..Default::default() }), // Override + (B256::from([4; 32]), Account::default()), + ], + destroyed_accounts: B256Set::from_iter([B256::from([6; 32])]), + }, + storages: B256Map::default(), + }; + + state1.extend_ref(&state2); + + // Check accounts are merged and sorted + assert_eq!(state1.accounts.accounts.len(), 4); + assert_eq!(state1.accounts.accounts[0].0, B256::from([1; 32])); + assert_eq!(state1.accounts.accounts[1].0, B256::from([2; 32])); + assert_eq!(state1.accounts.accounts[2].0, B256::from([3; 32])); + assert_eq!(state1.accounts.accounts[2].1.nonce, 1); // Should have state2's value + assert_eq!(state1.accounts.accounts[3].0, B256::from([4; 32])); + + // Check destroyed accounts are merged + assert!(state1.accounts.destroyed_accounts.contains(&B256::from([5; 32]))); + assert!(state1.accounts.destroyed_accounts.contains(&B256::from([6; 32]))); + } + + #[test] + fn test_hashed_storage_sorted_extend_ref() { + // Test normal extension + let mut storage1 = HashedStorageSorted { + non_zero_valued_slots: vec![ + (B256::from([1; 32]), U256::from(10)), + (B256::from([3; 32]), U256::from(30)), + ], + zero_valued_slots: B256Set::from_iter([B256::from([5; 32])]), + wiped: false, + }; + + let storage2 = HashedStorageSorted { + non_zero_valued_slots: vec![ + (B256::from([2; 32]), U256::from(20)), + (B256::from([3; 32]), U256::from(300)), // Override + (B256::from([4; 32]), U256::from(40)), + ], + zero_valued_slots: B256Set::from_iter([B256::from([6; 32])]), + wiped: false, + }; + + storage1.extend_ref(&storage2); + + assert_eq!(storage1.non_zero_valued_slots.len(), 4); + assert_eq!(storage1.non_zero_valued_slots[0].0, B256::from([1; 32])); + assert_eq!(storage1.non_zero_valued_slots[1].0, B256::from([2; 32])); + assert_eq!(storage1.non_zero_valued_slots[2].0, B256::from([3; 32])); + assert_eq!(storage1.non_zero_valued_slots[2].1, U256::from(300)); // Should have storage2's value + assert_eq!(storage1.non_zero_valued_slots[3].0, B256::from([4; 32])); + assert!(storage1.zero_valued_slots.contains(&B256::from([5; 32]))); + assert!(storage1.zero_valued_slots.contains(&B256::from([6; 32]))); + assert!(!storage1.wiped); + + // Test wiped storage + let mut storage3 = HashedStorageSorted { + non_zero_valued_slots: vec![(B256::from([1; 32]), U256::from(10))], + zero_valued_slots: B256Set::from_iter([B256::from([2; 32])]), + wiped: false, + }; + + let storage4 = HashedStorageSorted { + non_zero_valued_slots: vec![(B256::from([3; 32]), U256::from(30))], + zero_valued_slots: B256Set::from_iter([B256::from([4; 32])]), + wiped: true, + }; + + storage3.extend_ref(&storage4); + + assert!(storage3.wiped); + // When wiped, should only have storage4's values + assert_eq!(storage3.non_zero_valued_slots.len(), 1); + assert_eq!(storage3.non_zero_valued_slots[0].0, B256::from([3; 32])); + assert_eq!(storage3.zero_valued_slots.len(), 1); + assert!(storage3.zero_valued_slots.contains(&B256::from([4; 32]))); + } + + #[test] + fn test_hashed_post_state_chunking_length() { + let addr1 = B256::from([1; 32]); + let addr2 = B256::from([2; 32]); + let addr3 = B256::from([3; 32]); + let addr4 = B256::from([4; 32]); + let slot1 = B256::from([1; 32]); + let slot2 = B256::from([2; 32]); + let slot3 = B256::from([3; 32]); + + let state = HashedPostState { + accounts: B256Map::from_iter([(addr1, None), (addr2, None), (addr4, None)]), + storages: B256Map::from_iter([ + ( + addr1, + HashedStorage { + wiped: false, + storage: B256Map::from_iter([ + (slot1, U256::ZERO), + (slot2, U256::ZERO), + (slot3, U256::ZERO), + ]), + }, + ), + ( + addr2, + HashedStorage { + wiped: true, + storage: B256Map::from_iter([ + (slot1, U256::ZERO), + (slot2, U256::ZERO), + (slot3, U256::ZERO), + ]), + }, + ), + ( + addr3, + HashedStorage { + wiped: false, + storage: B256Map::from_iter([ + (slot1, U256::ZERO), + (slot2, U256::ZERO), + (slot3, U256::ZERO), + ]), + }, + ), + ]), + }; + + let chunking_length = state.chunking_length(); + for size in 1..=state.clone().chunks(1).count() { + let chunk_count = state.clone().chunks(size).count(); + let expected_count = chunking_length.div_ceil(size); + assert_eq!( + chunk_count, expected_count, + "chunking_length: {}, size: {}", + chunking_length, size + ); + } + } } diff --git a/crates/trie/common/src/input.rs b/crates/trie/common/src/input.rs index db15a61458d..522cfa9ed41 100644 --- a/crates/trie/common/src/input.rs +++ b/crates/trie/common/src/input.rs @@ -31,6 +31,39 @@ impl TrieInput { Self { nodes: TrieUpdates::default(), state, prefix_sets } } + /// Create new trie input from the provided blocks, from oldest to newest. See the documentation + /// for [`Self::extend_with_blocks`] for details. + pub fn from_blocks<'a>( + blocks: impl IntoIterator, + ) -> Self { + let mut input = Self::default(); + input.extend_with_blocks(blocks); + input + } + + /// Extend the trie input with the provided blocks, from oldest to newest. + /// + /// For blocks with missing trie updates, the trie input will be extended with prefix sets + /// constructed from the state of this block and the state itself, **without** trie updates. + pub fn extend_with_blocks<'a>( + &mut self, + blocks: impl IntoIterator, + ) { + for (hashed_state, trie_updates) in blocks { + self.append_cached_ref(trie_updates, hashed_state); + } + } + + /// Prepend another trie input to the current one. + pub fn prepend_self(&mut self, mut other: Self) { + core::mem::swap(&mut self.nodes, &mut other.nodes); + self.nodes.extend(other.nodes); + core::mem::swap(&mut self.state, &mut other.state); + self.state.extend(other.state); + // No need to swap prefix sets, as they will be sorted and deduplicated. + self.prefix_sets.extend(other.prefix_sets); + } + /// Prepend state to the input and extend the prefix sets. pub fn prepend(&mut self, mut state: HashedPostState) { self.prefix_sets.extend(state.construct_prefix_sets()); @@ -72,4 +105,17 @@ impl TrieInput { self.nodes.extend_ref(nodes); self.state.extend_ref(state); } + + /// This method clears the trie input nodes, state, and prefix sets. + pub fn clear(&mut self) { + self.nodes.clear(); + self.state.clear(); + self.prefix_sets.clear(); + } + + /// This method returns a cleared version of this trie input. + pub fn cleared(mut self) -> Self { + self.clear(); + self + } } diff --git a/crates/trie/common/src/lib.rs b/crates/trie/common/src/lib.rs index a710d1f4983..e4292a52016 100644 --- a/crates/trie/common/src/lib.rs +++ b/crates/trie/common/src/lib.rs @@ -6,7 +6,7 @@ issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] #![cfg_attr(not(test), warn(unused_crate_dependencies))] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(feature = "std"), no_std)] extern crate alloc; @@ -36,7 +36,7 @@ mod nibbles; pub use nibbles::{Nibbles, StoredNibbles, StoredNibblesSubKey}; mod storage; -pub use storage::StorageTrieEntry; +pub use storage::{StorageTrieEntry, TrieChangeSetsEntry}; mod subnode; pub use subnode::StoredSubNode; @@ -55,6 +55,11 @@ pub mod root; /// Buffer for trie updates. pub mod updates; +pub mod added_removed_keys; + +/// Utilities used by other modules in this crate. +mod utils; + /// Bincode-compatible serde implementations for trie types. /// /// `bincode` crate allows for more efficient serialization of trie types, because it allows diff --git a/crates/trie/common/src/nibbles.rs b/crates/trie/common/src/nibbles.rs index a7db55b854b..82d710395f9 100644 --- a/crates/trie/common/src/nibbles.rs +++ b/crates/trie/common/src/nibbles.rs @@ -22,35 +22,15 @@ impl From> for StoredNibbles { } } -impl PartialEq<[u8]> for StoredNibbles { - #[inline] - fn eq(&self, other: &[u8]) -> bool { - self.0.as_slice() == other - } -} - -impl PartialOrd<[u8]> for StoredNibbles { - #[inline] - fn partial_cmp(&self, other: &[u8]) -> Option { - self.0.as_slice().partial_cmp(other) - } -} - -impl core::borrow::Borrow<[u8]> for StoredNibbles { - #[inline] - fn borrow(&self) -> &[u8] { - self.0.as_slice() - } -} - #[cfg(any(test, feature = "reth-codec"))] impl reth_codecs::Compact for StoredNibbles { fn to_compact(&self, buf: &mut B) -> usize where B: bytes::BufMut + AsMut<[u8]>, { - buf.put_slice(self.0.as_slice()); - self.0.len() + let bytes = self.0.iter().collect::>(); + buf.put_slice(&bytes); + bytes.len() } fn from_compact(mut buf: &[u8], len: usize) -> (Self, &[u8]) { @@ -97,12 +77,14 @@ impl reth_codecs::Compact for StoredNibblesSubKey { { assert!(self.0.len() <= 64); - // right-pad with zeros - buf.put_slice(&self.0[..]); + let bytes = self.0.iter().collect::>(); + buf.put_slice(&bytes); + + // Right-pad with zeros static ZERO: &[u8; 64] = &[0; 64]; - buf.put_slice(&ZERO[self.0.len()..]); + buf.put_slice(&ZERO[bytes.len()..]); - buf.put_u8(self.0.len() as u8); + buf.put_u8(bytes.len() as u8); 64 + 1 } @@ -120,79 +102,58 @@ mod tests { #[test] fn test_stored_nibbles_from_nibbles() { - let nibbles = Nibbles::from_nibbles_unchecked(vec![0x12, 0x34, 0x56]); - let stored = StoredNibbles::from(nibbles.clone()); + let nibbles = Nibbles::from_nibbles_unchecked(vec![0x02, 0x04, 0x06]); + let stored = StoredNibbles::from(nibbles); assert_eq!(stored.0, nibbles); } #[test] fn test_stored_nibbles_from_vec() { - let bytes = vec![0x12, 0x34, 0x56]; - let stored = StoredNibbles::from(bytes.clone()); - assert_eq!(stored.0.as_slice(), bytes.as_slice()); - } - - #[test] - fn test_stored_nibbles_equality() { - let bytes = vec![0x12, 0x34]; + let bytes = vec![0x02, 0x04, 0x06]; let stored = StoredNibbles::from(bytes.clone()); - assert_eq!(stored, *bytes.as_slice()); - } - - #[test] - fn test_stored_nibbles_partial_cmp() { - let stored = StoredNibbles::from(vec![0x12, 0x34]); - let other = vec![0x12, 0x35]; - assert!(stored < *other.as_slice()); + assert_eq!(stored.0.to_vec(), bytes); } #[test] fn test_stored_nibbles_to_compact() { - let stored = StoredNibbles::from(vec![0x12, 0x34]); + let stored = StoredNibbles::from(vec![0x02, 0x04]); let mut buf = BytesMut::with_capacity(10); let len = stored.to_compact(&mut buf); assert_eq!(len, 2); - assert_eq!(buf, &vec![0x12, 0x34][..]); + assert_eq!(buf, &vec![0x02, 0x04][..]); } #[test] fn test_stored_nibbles_from_compact() { - let buf = vec![0x12, 0x34, 0x56]; + let buf = vec![0x02, 0x04, 0x06]; let (stored, remaining) = StoredNibbles::from_compact(&buf, 2); - assert_eq!(stored.0.as_slice(), &[0x12, 0x34]); - assert_eq!(remaining, &[0x56]); - } - - #[test] - fn test_stored_nibbles_subkey_from_nibbles() { - let nibbles = Nibbles::from_nibbles_unchecked(vec![0x12, 0x34]); - let subkey = StoredNibblesSubKey::from(nibbles.clone()); - assert_eq!(subkey.0, nibbles); + assert_eq!(stored.0.to_vec(), vec![0x02, 0x04]); + assert_eq!(remaining, &[0x06]); } #[test] fn test_stored_nibbles_subkey_to_compact() { - let subkey = StoredNibblesSubKey::from(vec![0x12, 0x34]); + let subkey = StoredNibblesSubKey::from(vec![0x02, 0x04]); let mut buf = BytesMut::with_capacity(65); let len = subkey.to_compact(&mut buf); assert_eq!(len, 65); - assert_eq!(buf[..2], [0x12, 0x34]); + assert_eq!(buf[..2], [0x02, 0x04]); assert_eq!(buf[64], 2); // Length byte } #[test] fn test_stored_nibbles_subkey_from_compact() { - let mut buf = vec![0x12, 0x34]; + let mut buf = vec![0x02, 0x04]; buf.resize(65, 0); buf[64] = 2; let (subkey, remaining) = StoredNibblesSubKey::from_compact(&buf, 65); - assert_eq!(subkey.0.as_slice(), &[0x12, 0x34]); + assert_eq!(subkey.0.to_vec(), vec![0x02, 0x04]); assert_eq!(remaining, &[] as &[u8]); } #[test] fn test_serialization_stored_nibbles() { - let stored = StoredNibbles::from(vec![0x12, 0x34]); + let stored = StoredNibbles::from(vec![0x02, 0x04]); let serialized = serde_json::to_string(&stored).unwrap(); let deserialized: StoredNibbles = serde_json::from_str(&serialized).unwrap(); assert_eq!(stored, deserialized); @@ -200,7 +161,7 @@ mod tests { #[test] fn test_serialization_stored_nibbles_subkey() { - let subkey = StoredNibblesSubKey::from(vec![0x12, 0x34]); + let subkey = StoredNibblesSubKey::from(vec![0x02, 0x04]); let serialized = serde_json::to_string(&subkey).unwrap(); let deserialized: StoredNibblesSubKey = serde_json::from_str(&serialized).unwrap(); assert_eq!(subkey, deserialized); diff --git a/crates/trie/common/src/prefix_set.rs b/crates/trie/common/src/prefix_set.rs index e4f97dafdb1..74fdb789113 100644 --- a/crates/trie/common/src/prefix_set.rs +++ b/crates/trie/common/src/prefix_set.rs @@ -15,6 +15,13 @@ pub struct TriePrefixSetsMut { } impl TriePrefixSetsMut { + /// Returns `true` if all prefix sets are empty. + pub fn is_empty(&self) -> bool { + self.account_prefix_set.is_empty() && + self.storage_prefix_sets.is_empty() && + self.destroyed_accounts.is_empty() + } + /// Extends prefix sets with contents of another prefix set. pub fn extend(&mut self, other: Self) { self.account_prefix_set.extend(other.account_prefix_set); @@ -38,10 +45,17 @@ impl TriePrefixSetsMut { destroyed_accounts: self.destroyed_accounts, } } + + /// Clears the prefix sets and destroyed accounts map. + pub fn clear(&mut self) { + self.destroyed_accounts.clear(); + self.storage_prefix_sets.clear(); + self.account_prefix_set.clear(); + } } /// Collection of trie prefix sets. -#[derive(Default, Debug)] +#[derive(Default, Debug, Clone)] pub struct TriePrefixSets { /// A set of account prefixes that have changed. pub account_prefix_set: PrefixSet, @@ -57,16 +71,18 @@ pub struct TriePrefixSets { /// This data structure stores a set of `Nibbles` and provides methods to insert /// new elements and check whether any existing element has a given prefix. /// -/// Internally, this implementation uses a `Vec` and aims to act like a `BTreeSet` in being both -/// sorted and deduplicated. It does this by keeping a `sorted` flag. The `sorted` flag represents -/// whether or not the `Vec` is definitely sorted. When a new element is added, it is set to -/// `false.`. The `Vec` is sorted and deduplicated when `sorted` is `true` and: -/// * An element is being checked for inclusion (`contains`), or -/// * The set is being converted into an immutable `PrefixSet` (`freeze`) +/// Internally, this implementation stores keys in an unsorted `Vec` together with an +/// `all` flag. The `all` flag indicates that every entry should be considered changed and that +/// individual keys can be ignored. /// -/// This means that a `PrefixSet` will always be sorted and deduplicated when constructed from a -/// `PrefixSetMut`. +/// Sorting and deduplication do not happen during insertion or membership checks on this mutable +/// structure. Instead, keys are sorted and deduplicated when converting into the immutable +/// `PrefixSet` via `freeze()`. The immutable `PrefixSet` provides `contains` and relies on the +/// sorted and unique keys produced by `freeze()`; it does not perform additional sorting or +/// deduplication. /// +/// This guarantees that a `PrefixSet` constructed from a `PrefixSetMut` is always sorted and +/// deduplicated. /// # Examples /// /// ``` @@ -76,8 +92,8 @@ pub struct TriePrefixSets { /// prefix_set_mut.insert(Nibbles::from_nibbles_unchecked(&[0xa, 0xb])); /// prefix_set_mut.insert(Nibbles::from_nibbles_unchecked(&[0xa, 0xb, 0xc])); /// let mut prefix_set = prefix_set_mut.freeze(); -/// assert!(prefix_set.contains(&[0xa, 0xb])); -/// assert!(prefix_set.contains(&[0xa, 0xb, 0xc])); +/// assert!(prefix_set.contains(&Nibbles::from_nibbles_unchecked([0xa, 0xb]))); +/// assert!(prefix_set.contains(&Nibbles::from_nibbles_unchecked([0xa, 0xb, 0xc]))); /// ``` #[derive(PartialEq, Eq, Clone, Default, Debug)] pub struct PrefixSetMut { @@ -127,15 +143,21 @@ impl PrefixSetMut { } /// Returns the number of elements in the set. - pub fn len(&self) -> usize { + pub const fn len(&self) -> usize { self.keys.len() } /// Returns `true` if the set is empty. - pub fn is_empty(&self) -> bool { + pub const fn is_empty(&self) -> bool { self.keys.is_empty() } + /// Clears the inner vec for reuse, setting `all` to `false`. + pub fn clear(&mut self) { + self.all = false; + self.keys.clear(); + } + /// Returns a `PrefixSet` with the same elements as this set. /// /// If not yet sorted, the elements will be sorted and deduplicated. @@ -145,8 +167,7 @@ impl PrefixSetMut { } else { self.keys.sort_unstable(); self.keys.dedup(); - // We need to shrink in both the sorted and non-sorted cases because deduping may have - // occurred either on `freeze`, or during `contains`. + // Shrink after deduplication to release unused capacity. self.keys.shrink_to_fit(); PrefixSet { index: 0, all: false, keys: Arc::new(self.keys) } } @@ -166,8 +187,21 @@ pub struct PrefixSet { impl PrefixSet { /// Returns `true` if any of the keys in the set has the given prefix + /// + /// # Note on Mutability + /// + /// This method requires `&mut self` (unlike typical `contains` methods) because it maintains an + /// internal position tracker (`self.index`) between calls. This enables significant performance + /// optimization for sequential lookups in sorted order, which is common during trie traversal. + /// + /// The `index` field allows subsequent searches to start where previous ones left off, + /// avoiding repeated full scans of the prefix array when keys are accessed in nearby ranges. + /// + /// This optimization was inspired by Silkworm's implementation and significantly improves + /// incremental state root calculation performance + /// ([see PR #2417](https://github.com/paradigmxyz/reth/pull/2417)). #[inline] - pub fn contains(&mut self, prefix: &[u8]) -> bool { + pub fn contains(&mut self, prefix: &Nibbles) -> bool { if self.all { return true } @@ -177,7 +211,7 @@ impl PrefixSet { } for (idx, key) in self.keys[self.index..].iter().enumerate() { - if key.has_prefix(prefix) { + if key.starts_with(prefix) { self.index += idx; return true } @@ -196,6 +230,11 @@ impl PrefixSet { self.keys.iter() } + /// Returns true if every entry should be considered changed. + pub const fn all(&self) -> bool { + self.all + } + /// Returns the number of elements in the set. pub fn len(&self) -> usize { self.keys.len() @@ -228,9 +267,9 @@ mod tests { prefix_set_mut.insert(Nibbles::from_nibbles([1, 2, 3])); // Duplicate let mut prefix_set = prefix_set_mut.freeze(); - assert!(prefix_set.contains(&[1, 2])); - assert!(prefix_set.contains(&[4, 5])); - assert!(!prefix_set.contains(&[7, 8])); + assert!(prefix_set.contains(&Nibbles::from_nibbles_unchecked([1, 2]))); + assert!(prefix_set.contains(&Nibbles::from_nibbles_unchecked([4, 5]))); + assert!(!prefix_set.contains(&Nibbles::from_nibbles_unchecked([7, 8]))); assert_eq!(prefix_set.len(), 3); // Length should be 3 (excluding duplicate) } @@ -242,13 +281,13 @@ mod tests { prefix_set_mut.insert(Nibbles::from_nibbles([4, 5, 6])); prefix_set_mut.insert(Nibbles::from_nibbles([1, 2, 3])); // Duplicate - assert_eq!(prefix_set_mut.keys.len(), 4); // Length should be 3 (including duplicate) - assert_eq!(prefix_set_mut.keys.capacity(), 4); // Capacity should be 4 (including duplicate) + assert_eq!(prefix_set_mut.keys.len(), 4); // Length is 4 (before deduplication) + assert_eq!(prefix_set_mut.keys.capacity(), 4); // Capacity is 4 (before deduplication) let mut prefix_set = prefix_set_mut.freeze(); - assert!(prefix_set.contains(&[1, 2])); - assert!(prefix_set.contains(&[4, 5])); - assert!(!prefix_set.contains(&[7, 8])); + assert!(prefix_set.contains(&Nibbles::from_nibbles_unchecked([1, 2]))); + assert!(prefix_set.contains(&Nibbles::from_nibbles_unchecked([4, 5]))); + assert!(!prefix_set.contains(&Nibbles::from_nibbles_unchecked([7, 8]))); assert_eq!(prefix_set.keys.len(), 3); // Length should be 3 (excluding duplicate) assert_eq!(prefix_set.keys.capacity(), 3); // Capacity should be 3 after shrinking } @@ -262,13 +301,13 @@ mod tests { prefix_set_mut.insert(Nibbles::from_nibbles([4, 5, 6])); prefix_set_mut.insert(Nibbles::from_nibbles([1, 2, 3])); // Duplicate - assert_eq!(prefix_set_mut.keys.len(), 4); // Length should be 3 (including duplicate) - assert_eq!(prefix_set_mut.keys.capacity(), 101); // Capacity should be 101 (including duplicate) + assert_eq!(prefix_set_mut.keys.len(), 4); // Length is 4 (before deduplication) + assert_eq!(prefix_set_mut.keys.capacity(), 101); // Capacity is 101 (before deduplication) let mut prefix_set = prefix_set_mut.freeze(); - assert!(prefix_set.contains(&[1, 2])); - assert!(prefix_set.contains(&[4, 5])); - assert!(!prefix_set.contains(&[7, 8])); + assert!(prefix_set.contains(&Nibbles::from_nibbles_unchecked([1, 2]))); + assert!(prefix_set.contains(&Nibbles::from_nibbles_unchecked([4, 5]))); + assert!(!prefix_set.contains(&Nibbles::from_nibbles_unchecked([7, 8]))); assert_eq!(prefix_set.keys.len(), 3); // Length should be 3 (excluding duplicate) assert_eq!(prefix_set.keys.capacity(), 3); // Capacity should be 3 after shrinking } diff --git a/crates/trie/common/src/proofs.rs b/crates/trie/common/src/proofs.rs index 35c75b5d0f1..a8e0bb59b93 100644 --- a/crates/trie/common/src/proofs.rs +++ b/crates/trie/common/src/proofs.rs @@ -89,6 +89,11 @@ impl MultiProofTargets { pub fn chunks(self, size: usize) -> ChunkedMultiProofTargets { ChunkedMultiProofTargets::new(self, size) } + + /// Returns the number of items that will be considered during chunking in `[Self::chunks]`. + pub fn chunking_length(&self) -> usize { + self.values().map(|slots| 1 + slots.len().saturating_sub(1)).sum::() + } } /// An iterator that yields chunks of the proof targets of at most `size` account and storage @@ -229,18 +234,16 @@ impl MultiProof { // Inspect the last node in the proof. If it's a leaf node with matching suffix, // then the node contains the encoded trie account. let info = 'info: { - if let Some(last) = proof.last() { - if let TrieNode::Leaf(leaf) = TrieNode::decode(&mut &last[..])? { - if nibbles.ends_with(&leaf.key) { - let account = TrieAccount::decode(&mut &leaf.value[..])?; - break 'info Some(Account { - balance: account.balance, - nonce: account.nonce, - bytecode_hash: (account.code_hash != KECCAK_EMPTY) - .then_some(account.code_hash), - }) - } - } + if let Some(last) = proof.last() && + let TrieNode::Leaf(leaf) = TrieNode::decode(&mut &last[..])? && + nibbles.ends_with(&leaf.key) + { + let account = TrieAccount::decode(&mut &leaf.value[..])?; + break 'info Some(Account { + balance: account.balance, + nonce: account.nonce, + bytecode_hash: (account.code_hash != KECCAK_EMPTY).then_some(account.code_hash), + }) } None }; @@ -308,6 +311,14 @@ pub struct DecodedMultiProof { } impl DecodedMultiProof { + /// Returns true if the multiproof is empty. + pub fn is_empty(&self) -> bool { + self.account_subtree.is_empty() && + self.branch_node_hash_masks.is_empty() && + self.branch_node_tree_masks.is_empty() && + self.storages.is_empty() + } + /// Return the account proof nodes for the given account path. pub fn account_proof_nodes(&self, path: &Nibbles) -> Vec<(Nibbles, TrieNode)> { self.account_subtree.matching_nodes_sorted(path) @@ -352,16 +363,15 @@ impl DecodedMultiProof { // Inspect the last node in the proof. If it's a leaf node with matching suffix, // then the node contains the encoded trie account. let info = 'info: { - if let Some(TrieNode::Leaf(leaf)) = proof.last() { - if nibbles.ends_with(&leaf.key) { - let account = TrieAccount::decode(&mut &leaf.value[..])?; - break 'info Some(Account { - balance: account.balance, - nonce: account.nonce, - bytecode_hash: (account.code_hash != KECCAK_EMPTY) - .then_some(account.code_hash), - }) - } + if let Some(TrieNode::Leaf(leaf)) = proof.last() && + nibbles.ends_with(&leaf.key) + { + let account = TrieAccount::decode(&mut &leaf.value[..])?; + break 'info Some(Account { + balance: account.balance, + nonce: account.nonce, + bytecode_hash: (account.code_hash != KECCAK_EMPTY).then_some(account.code_hash), + }) } None }; @@ -404,6 +414,36 @@ impl DecodedMultiProof { } } } + + /// Create a [`DecodedMultiProof`] from a [`DecodedStorageMultiProof`]. + pub fn from_storage_proof( + hashed_address: B256, + storage_proof: DecodedStorageMultiProof, + ) -> Self { + Self { + storages: B256Map::from_iter([(hashed_address, storage_proof)]), + ..Default::default() + } + } +} + +impl TryFrom for DecodedMultiProof { + type Error = alloy_rlp::Error; + + fn try_from(multi_proof: MultiProof) -> Result { + let account_subtree = DecodedProofNodes::try_from(multi_proof.account_subtree)?; + let storages = multi_proof + .storages + .into_iter() + .map(|(address, storage)| Ok((address, storage.try_into()?))) + .collect::, alloy_rlp::Error>>()?; + Ok(Self { + account_subtree, + branch_node_hash_masks: multi_proof.branch_node_hash_masks, + branch_node_tree_masks: multi_proof.branch_node_tree_masks, + storages, + }) + } } /// The merkle multiproof of storage trie. @@ -448,12 +488,11 @@ impl StorageMultiProof { // Inspect the last node in the proof. If it's a leaf node with matching suffix, // then the node contains the encoded slot value. let value = 'value: { - if let Some(last) = proof.last() { - if let TrieNode::Leaf(leaf) = TrieNode::decode(&mut &last[..])? { - if nibbles.ends_with(&leaf.key) { - break 'value U256::decode(&mut &leaf.value[..])? - } - } + if let Some(last) = proof.last() && + let TrieNode::Leaf(leaf) = TrieNode::decode(&mut &last[..])? && + nibbles.ends_with(&leaf.key) + { + break 'value U256::decode(&mut &leaf.value[..])? } U256::ZERO }; @@ -501,10 +540,10 @@ impl DecodedStorageMultiProof { // Inspect the last node in the proof. If it's a leaf node with matching suffix, // then the node contains the encoded slot value. let value = 'value: { - if let Some(TrieNode::Leaf(leaf)) = proof.last() { - if nibbles.ends_with(&leaf.key) { - break 'value U256::decode(&mut &leaf.value[..])? - } + if let Some(TrieNode::Leaf(leaf)) = proof.last() && + nibbles.ends_with(&leaf.key) + { + break 'value U256::decode(&mut &leaf.value[..])? } U256::ZERO }; @@ -513,6 +552,20 @@ impl DecodedStorageMultiProof { } } +impl TryFrom for DecodedStorageMultiProof { + type Error = alloy_rlp::Error; + + fn try_from(multi_proof: StorageMultiProof) -> Result { + let subtree = DecodedProofNodes::try_from(multi_proof.subtree)?; + Ok(Self { + root: multi_proof.root, + subtree, + branch_node_hash_masks: multi_proof.branch_node_hash_masks, + branch_node_tree_masks: multi_proof.branch_node_tree_masks, + }) + } +} + /// The merkle proof with the relevant account info. #[derive(Clone, PartialEq, Eq, Debug)] #[cfg_attr(any(test, feature = "serde"), derive(serde::Serialize, serde::Deserialize))] @@ -617,7 +670,6 @@ impl AccountProof { } /// Verify the storage proofs and account proof against the provided state root. - #[expect(clippy::result_large_err)] pub fn verify(&self, root: B256) -> Result<(), ProofVerificationError> { // Verify storage proofs. for storage_proof in &self.storage_proofs { @@ -711,11 +763,10 @@ impl StorageProof { } /// Verify the proof against the provided storage root. - #[expect(clippy::result_large_err)] pub fn verify(&self, root: B256) -> Result<(), ProofVerificationError> { let expected = if self.value.is_zero() { None } else { Some(encode_fixed_size(&self.value).to_vec()) }; - verify_proof(root, self.nibbles.clone(), expected, &self.proof) + verify_proof(root, self.nibbles, expected, &self.proof) } } @@ -939,8 +990,8 @@ mod tests { // populate some targets let (addr1, addr2) = (B256::random(), B256::random()); let (slot1, slot2) = (B256::random(), B256::random()); - targets.insert(addr1, vec![slot1].into_iter().collect()); - targets.insert(addr2, vec![slot2].into_iter().collect()); + targets.insert(addr1, std::iter::once(slot1).collect()); + targets.insert(addr2, std::iter::once(slot2).collect()); let mut retained = targets.clone(); retained.retain_difference(&Default::default()); @@ -1021,4 +1072,33 @@ mod tests { acc.storage_root = EMPTY_ROOT_HASH; assert_eq!(acc, inverse); } + + #[test] + fn test_multiproof_targets_chunking_length() { + let mut targets = MultiProofTargets::default(); + targets.insert(B256::with_last_byte(1), B256Set::default()); + targets.insert( + B256::with_last_byte(2), + B256Set::from_iter([B256::with_last_byte(10), B256::with_last_byte(20)]), + ); + targets.insert( + B256::with_last_byte(3), + B256Set::from_iter([ + B256::with_last_byte(30), + B256::with_last_byte(31), + B256::with_last_byte(32), + ]), + ); + + let chunking_length = targets.chunking_length(); + for size in 1..=targets.clone().chunks(1).count() { + let chunk_count = targets.clone().chunks(size).count(); + let expected_count = chunking_length.div_ceil(size); + assert_eq!( + chunk_count, expected_count, + "chunking_length: {}, size: {}", + chunking_length, size + ); + } + } } diff --git a/crates/trie/common/src/storage.rs b/crates/trie/common/src/storage.rs index 3ebcc4e810e..1e567393864 100644 --- a/crates/trie/common/src/storage.rs +++ b/crates/trie/common/src/storage.rs @@ -1,6 +1,8 @@ use super::{BranchNodeCompact, StoredNibblesSubKey}; /// Account storage trie node. +/// +/// `nibbles` is the subkey when used as a value in the `StorageTrie` table. #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] #[cfg_attr(any(test, feature = "serde"), derive(serde::Serialize, serde::Deserialize))] pub struct StorageTrieEntry { @@ -25,9 +27,179 @@ impl reth_codecs::Compact for StorageTrieEntry { } fn from_compact(buf: &[u8], len: usize) -> (Self, &[u8]) { - let (nibbles, buf) = StoredNibblesSubKey::from_compact(buf, 33); - let (node, buf) = BranchNodeCompact::from_compact(buf, len - 33); + let (nibbles, buf) = StoredNibblesSubKey::from_compact(buf, 65); + let (node, buf) = BranchNodeCompact::from_compact(buf, len - 65); let this = Self { nibbles, node }; (this, buf) } } + +/// Trie changeset entry representing the state of a trie node before a block. +/// +/// `nibbles` is the subkey when used as a value in the changeset tables. +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr(any(test, feature = "serde"), derive(serde::Serialize, serde::Deserialize))] +pub struct TrieChangeSetsEntry { + /// The nibbles of the intermediate node + pub nibbles: StoredNibblesSubKey, + /// Node value prior to the block being processed, None indicating it didn't exist. + pub node: Option, +} + +#[cfg(any(test, feature = "reth-codec"))] +impl reth_codecs::Compact for TrieChangeSetsEntry { + fn to_compact(&self, buf: &mut B) -> usize + where + B: bytes::BufMut + AsMut<[u8]>, + { + let nibbles_len = self.nibbles.to_compact(buf); + let node_len = self.node.as_ref().map(|node| node.to_compact(buf)).unwrap_or(0); + nibbles_len + node_len + } + + fn from_compact(buf: &[u8], len: usize) -> (Self, &[u8]) { + if len == 0 { + // Return an empty entry without trying to parse anything + return ( + Self { nibbles: StoredNibblesSubKey::from(super::Nibbles::default()), node: None }, + buf, + ) + } + + let (nibbles, buf) = StoredNibblesSubKey::from_compact(buf, 65); + + if len <= 65 { + return (Self { nibbles, node: None }, buf) + } + + let (node, buf) = BranchNodeCompact::from_compact(buf, len - 65); + (Self { nibbles, node: Some(node) }, buf) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use bytes::BytesMut; + use reth_codecs::Compact; + + #[test] + fn test_trie_changesets_entry_full_empty() { + // Test a fully empty entry (empty nibbles, None node) + let entry = TrieChangeSetsEntry { nibbles: StoredNibblesSubKey::from(vec![]), node: None }; + + let mut buf = BytesMut::new(); + let len = entry.to_compact(&mut buf); + + // Empty nibbles takes 65 bytes (64 for padding + 1 for length) + // None node adds 0 bytes + assert_eq!(len, 65); + assert_eq!(buf.len(), 65); + + // Deserialize and verify + let (decoded, remaining) = TrieChangeSetsEntry::from_compact(&buf, len); + assert_eq!(decoded.nibbles.0.to_vec(), Vec::::new()); + assert_eq!(decoded.node, None); + assert_eq!(remaining.len(), 0); + } + + #[test] + fn test_trie_changesets_entry_none_node() { + // Test non-empty nibbles with None node + let nibbles_data = vec![0x01, 0x02, 0x03, 0x04]; + let entry = TrieChangeSetsEntry { + nibbles: StoredNibblesSubKey::from(nibbles_data.clone()), + node: None, + }; + + let mut buf = BytesMut::new(); + let len = entry.to_compact(&mut buf); + + // Nibbles takes 65 bytes regardless of content + assert_eq!(len, 65); + + // Deserialize and verify + let (decoded, remaining) = TrieChangeSetsEntry::from_compact(&buf, len); + assert_eq!(decoded.nibbles.0.to_vec(), nibbles_data); + assert_eq!(decoded.node, None); + assert_eq!(remaining.len(), 0); + } + + #[test] + fn test_trie_changesets_entry_empty_path_with_node() { + // Test empty path with Some node + // Using the same signature as in the codebase: (state_mask, hash_mask, tree_mask, hashes, + // value) + let test_node = BranchNodeCompact::new( + 0b1111_1111_1111_1111, // state_mask: all children present + 0b1111_1111_1111_1111, // hash_mask: all have hashes + 0b0000_0000_0000_0000, // tree_mask: no embedded trees + vec![], // hashes + None, // value + ); + + let entry = TrieChangeSetsEntry { + nibbles: StoredNibblesSubKey::from(vec![]), + node: Some(test_node.clone()), + }; + + let mut buf = BytesMut::new(); + let len = entry.to_compact(&mut buf); + + // Calculate expected length + let mut temp_buf = BytesMut::new(); + let node_len = test_node.to_compact(&mut temp_buf); + assert_eq!(len, 65 + node_len); + + // Deserialize and verify + let (decoded, remaining) = TrieChangeSetsEntry::from_compact(&buf, len); + assert_eq!(decoded.nibbles.0.to_vec(), Vec::::new()); + assert_eq!(decoded.node, Some(test_node)); + assert_eq!(remaining.len(), 0); + } + + #[test] + fn test_trie_changesets_entry_normal() { + // Test normal case: non-empty path with Some node + let nibbles_data = vec![0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f]; + // Using the same signature as in the codebase + let test_node = BranchNodeCompact::new( + 0b0000_0000_1111_0000, // state_mask: some children present + 0b0000_0000_0011_0000, // hash_mask: some have hashes + 0b0000_0000_0000_0000, // tree_mask: no embedded trees + vec![], // hashes (empty for this test) + None, // value + ); + + let entry = TrieChangeSetsEntry { + nibbles: StoredNibblesSubKey::from(nibbles_data.clone()), + node: Some(test_node.clone()), + }; + + let mut buf = BytesMut::new(); + let len = entry.to_compact(&mut buf); + + // Verify serialization length + let mut temp_buf = BytesMut::new(); + let node_len = test_node.to_compact(&mut temp_buf); + assert_eq!(len, 65 + node_len); + + // Deserialize and verify + let (decoded, remaining) = TrieChangeSetsEntry::from_compact(&buf, len); + assert_eq!(decoded.nibbles.0.to_vec(), nibbles_data); + assert_eq!(decoded.node, Some(test_node)); + assert_eq!(remaining.len(), 0); + } + + #[test] + fn test_trie_changesets_entry_from_compact_zero_len() { + // Test from_compact with zero length + let buf = vec![0x01, 0x02, 0x03]; + let (decoded, remaining) = TrieChangeSetsEntry::from_compact(&buf, 0); + + // Should return empty nibbles and None node + assert_eq!(decoded.nibbles.0.to_vec(), Vec::::new()); + assert_eq!(decoded.node, None); + assert_eq!(remaining, &buf[..]); // Buffer should be unchanged + } +} diff --git a/crates/trie/common/src/updates.rs b/crates/trie/common/src/updates.rs index d4362542f00..b0d178cd1d0 100644 --- a/crates/trie/common/src/updates.rs +++ b/crates/trie/common/src/updates.rs @@ -1,8 +1,11 @@ -use crate::{BranchNodeCompact, HashBuilder, Nibbles}; -use alloc::vec::Vec; +use crate::{utils::extend_sorted_vec, BranchNodeCompact, HashBuilder, Nibbles}; +use alloc::{ + collections::{btree_map::BTreeMap, btree_set::BTreeSet}, + vec::Vec, +}; use alloy_primitives::{ map::{B256Map, B256Set, HashMap, HashSet}, - B256, + FixedBytes, B256, }; /// The aggregation of trie updates. @@ -58,9 +61,9 @@ impl TrieUpdates { pub fn extend_ref(&mut self, other: &Self) { self.extend_common(other); self.account_nodes.extend(exclude_empty_from_pair( - other.account_nodes.iter().map(|(k, v)| (k.clone(), v.clone())), + other.account_nodes.iter().map(|(k, v)| (*k, v.clone())), )); - self.removed_nodes.extend(exclude_empty(other.removed_nodes.iter().cloned())); + self.removed_nodes.extend(exclude_empty(other.removed_nodes.iter().copied())); for (hashed_address, storage_trie) in &other.storage_tries { self.storage_tries.entry(*hashed_address).or_default().extend_ref(storage_trie); } @@ -104,15 +107,60 @@ impl TrieUpdates { } /// Converts trie updates into [`TrieUpdatesSorted`]. - pub fn into_sorted(self) -> TrieUpdatesSorted { - let mut account_nodes = Vec::from_iter(self.account_nodes); + pub fn into_sorted(mut self) -> TrieUpdatesSorted { + self.drain_into_sorted() + } + + /// Converts trie updates into [`TrieUpdatesSorted`], but keeping the maps allocated by + /// draining. + /// + /// This effectively clears all the fields in the [`TrieUpdatesSorted`]. + /// + /// This allows us to reuse the allocated space. This allocates new space for the sorted + /// updates, like `into_sorted`. + pub fn drain_into_sorted(&mut self) -> TrieUpdatesSorted { + let mut account_nodes = self + .account_nodes + .drain() + .map(|(path, node)| { + // Updated nodes take precedence over removed nodes. + self.removed_nodes.remove(&path); + (path, Some(node)) + }) + .collect::>(); + + account_nodes.extend(self.removed_nodes.drain().map(|path| (path, None))); account_nodes.sort_unstable_by(|a, b| a.0.cmp(&b.0)); + let storage_tries = self .storage_tries - .into_iter() + .drain() .map(|(hashed_address, updates)| (hashed_address, updates.into_sorted())) .collect(); - TrieUpdatesSorted { removed_nodes: self.removed_nodes, account_nodes, storage_tries } + TrieUpdatesSorted { account_nodes, storage_tries } + } + + /// Converts trie updates into [`TrieUpdatesSortedRef`]. + pub fn into_sorted_ref<'a>(&'a self) -> TrieUpdatesSortedRef<'a> { + let mut account_nodes = self.account_nodes.iter().collect::>(); + account_nodes.sort_unstable_by(|a, b| a.0.cmp(b.0)); + + TrieUpdatesSortedRef { + removed_nodes: self.removed_nodes.iter().collect::>(), + account_nodes, + storage_tries: self + .storage_tries + .iter() + .map(|m| (*m.0, m.1.into_sorted_ref().clone())) + .collect(), + } + } + + /// Clears the nodes and storage trie maps in this `TrieUpdates`. + pub fn clear(&mut self) { + self.account_nodes.clear(); + self.removed_nodes.clear(); + self.storage_tries.clear(); } } @@ -191,9 +239,9 @@ impl StorageTrieUpdates { pub fn extend_ref(&mut self, other: &Self) { self.extend_common(other); self.storage_nodes.extend(exclude_empty_from_pair( - other.storage_nodes.iter().map(|(k, v)| (k.clone(), v.clone())), + other.storage_nodes.iter().map(|(k, v)| (*k, v.clone())), )); - self.removed_nodes.extend(exclude_empty(other.removed_nodes.iter().cloned())); + self.removed_nodes.extend(exclude_empty(other.removed_nodes.iter().copied())); } fn extend_common(&mut self, other: &Self) { @@ -216,13 +264,29 @@ impl StorageTrieUpdates { } /// Convert storage trie updates into [`StorageTrieUpdatesSorted`]. - pub fn into_sorted(self) -> StorageTrieUpdatesSorted { - let mut storage_nodes = Vec::from_iter(self.storage_nodes); + pub fn into_sorted(mut self) -> StorageTrieUpdatesSorted { + let mut storage_nodes = self + .storage_nodes + .into_iter() + .map(|(path, node)| { + // Updated nodes take precedence over removed nodes. + self.removed_nodes.remove(&path); + (path, Some(node)) + }) + .collect::>(); + + storage_nodes.extend(self.removed_nodes.into_iter().map(|path| (path, None))); storage_nodes.sort_unstable_by(|a, b| a.0.cmp(&b.0)); - StorageTrieUpdatesSorted { + + StorageTrieUpdatesSorted { is_deleted: self.is_deleted, storage_nodes } + } + + /// Convert storage trie updates into [`StorageTrieUpdatesSortedRef`]. + pub fn into_sorted_ref(&self) -> StorageTrieUpdatesSortedRef<'_> { + StorageTrieUpdatesSortedRef { is_deleted: self.is_deleted, - removed_nodes: self.removed_nodes, - storage_nodes, + removed_nodes: self.removed_nodes.iter().collect::>(), + storage_nodes: self.storage_nodes.iter().collect::>(), } } } @@ -350,43 +414,143 @@ mod serde_nibbles_map { } } -/// Sorted trie updates used for lookups and insertions. +/// Sorted trie updates reference used for serializing trie to file. #[derive(PartialEq, Eq, Clone, Default, Debug)] -pub struct TrieUpdatesSorted { +#[cfg_attr(any(test, feature = "serde"), derive(serde::Serialize))] +pub struct TrieUpdatesSortedRef<'a> { /// Sorted collection of updated state nodes with corresponding paths. - pub account_nodes: Vec<(Nibbles, BranchNodeCompact)>, + pub account_nodes: Vec<(&'a Nibbles, &'a BranchNodeCompact)>, /// The set of removed state node keys. - pub removed_nodes: HashSet, + pub removed_nodes: BTreeSet<&'a Nibbles>, /// Storage tries stored by hashed address of the account the trie belongs to. - pub storage_tries: B256Map, + pub storage_tries: BTreeMap, StorageTrieUpdatesSortedRef<'a>>, +} + +/// Sorted trie updates used for lookups and insertions. +#[derive(PartialEq, Eq, Clone, Default, Debug)] +#[cfg_attr(any(test, feature = "serde"), derive(serde::Serialize, serde::Deserialize))] +pub struct TrieUpdatesSorted { + /// Sorted collection of updated state nodes with corresponding paths. None indicates that a + /// node was removed. + account_nodes: Vec<(Nibbles, Option)>, + /// Storage tries stored by hashed address of the account the trie belongs to. + storage_tries: B256Map, } impl TrieUpdatesSorted { - /// Returns reference to updated account nodes. - pub fn account_nodes_ref(&self) -> &[(Nibbles, BranchNodeCompact)] { - &self.account_nodes + /// Creates a new `TrieUpdatesSorted` with the given account nodes and storage tries. + /// + /// # Panics + /// + /// In debug mode, panics if `account_nodes` is not sorted by the `Nibbles` key, + /// or if any storage trie's `storage_nodes` is not sorted by its `Nibbles` key. + pub fn new( + account_nodes: Vec<(Nibbles, Option)>, + storage_tries: B256Map, + ) -> Self { + debug_assert!( + account_nodes.is_sorted_by_key(|item| &item.0), + "account_nodes must be sorted by Nibbles key" + ); + debug_assert!( + storage_tries.values().all(|storage_trie| { + storage_trie.storage_nodes.is_sorted_by_key(|item| &item.0) + }), + "all storage_nodes in storage_tries must be sorted by Nibbles key" + ); + Self { account_nodes, storage_tries } } - /// Returns reference to removed account nodes. - pub const fn removed_nodes_ref(&self) -> &HashSet { - &self.removed_nodes + /// Returns `true` if the updates are empty. + pub fn is_empty(&self) -> bool { + self.account_nodes.is_empty() && self.storage_tries.is_empty() + } + + /// Returns reference to updated account nodes. + pub fn account_nodes_ref(&self) -> &[(Nibbles, Option)] { + &self.account_nodes } /// Returns reference to updated storage tries. pub const fn storage_tries_ref(&self) -> &B256Map { &self.storage_tries } + + /// Returns the total number of updates including account nodes and all storage updates. + pub fn total_len(&self) -> usize { + self.account_nodes.len() + + self.storage_tries.values().map(|storage| storage.len()).sum::() + } + + /// Extends the trie updates with another set of sorted updates. + /// + /// This merges the account nodes and storage tries from `other` into `self`. + /// Account nodes are merged and re-sorted, with `other`'s values taking precedence + /// for duplicate keys. + pub fn extend_ref(&mut self, other: &Self) { + // Extend account nodes + extend_sorted_vec(&mut self.account_nodes, &other.account_nodes); + + // Merge storage tries + for (hashed_address, storage_trie) in &other.storage_tries { + self.storage_tries + .entry(*hashed_address) + .and_modify(|existing| existing.extend_ref(storage_trie)) + .or_insert_with(|| storage_trie.clone()); + } + } } -/// Sorted trie updates used for lookups and insertions. +impl AsRef for TrieUpdatesSorted { + fn as_ref(&self) -> &Self { + self + } +} + +impl From for TrieUpdates { + fn from(sorted: TrieUpdatesSorted) -> Self { + let mut account_nodes = HashMap::default(); + let mut removed_nodes = HashSet::default(); + + for (nibbles, node) in sorted.account_nodes { + if let Some(node) = node { + account_nodes.insert(nibbles, node); + } else { + removed_nodes.insert(nibbles); + } + } + + let storage_tries = sorted + .storage_tries + .into_iter() + .map(|(address, storage)| (address, storage.into())) + .collect(); + + Self { account_nodes, removed_nodes, storage_tries } + } +} + +/// Sorted storage trie updates reference used for serializing to file. #[derive(PartialEq, Eq, Clone, Default, Debug)] -pub struct StorageTrieUpdatesSorted { +#[cfg_attr(any(test, feature = "serde"), derive(serde::Serialize))] +pub struct StorageTrieUpdatesSortedRef<'a> { /// Flag indicating whether the trie has been deleted/wiped. pub is_deleted: bool, /// Sorted collection of updated storage nodes with corresponding paths. - pub storage_nodes: Vec<(Nibbles, BranchNodeCompact)>, + pub storage_nodes: BTreeMap<&'a Nibbles, &'a BranchNodeCompact>, /// The set of removed storage node keys. - pub removed_nodes: HashSet, + pub removed_nodes: BTreeSet<&'a Nibbles>, +} + +/// Sorted trie updates used for lookups and insertions. +#[derive(PartialEq, Eq, Clone, Default, Debug)] +#[cfg_attr(any(test, feature = "serde"), derive(serde::Serialize, serde::Deserialize))] +pub struct StorageTrieUpdatesSorted { + /// Flag indicating whether the trie has been deleted/wiped. + pub is_deleted: bool, + /// Sorted collection of updated storage nodes with corresponding paths. None indicates a node + /// is removed. + pub storage_nodes: Vec<(Nibbles, Option)>, } impl StorageTrieUpdatesSorted { @@ -396,13 +560,35 @@ impl StorageTrieUpdatesSorted { } /// Returns reference to updated storage nodes. - pub fn storage_nodes_ref(&self) -> &[(Nibbles, BranchNodeCompact)] { + pub fn storage_nodes_ref(&self) -> &[(Nibbles, Option)] { &self.storage_nodes } - /// Returns reference to removed storage nodes. - pub const fn removed_nodes_ref(&self) -> &HashSet { - &self.removed_nodes + /// Returns the total number of storage node updates. + pub const fn len(&self) -> usize { + self.storage_nodes.len() + } + + /// Returns `true` if there are no storage node updates. + pub const fn is_empty(&self) -> bool { + self.storage_nodes.is_empty() + } + + /// Extends the storage trie updates with another set of sorted updates. + /// + /// If `other` is marked as deleted, this will be marked as deleted and all nodes cleared. + /// Otherwise, nodes are merged with `other`'s values taking precedence for duplicates. + pub fn extend_ref(&mut self, other: &Self) { + if other.is_deleted { + self.is_deleted = true; + self.storage_nodes.clear(); + self.storage_nodes.extend(other.storage_nodes.iter().cloned()); + return; + } + + // Extend storage nodes + extend_sorted_vec(&mut self.storage_nodes, &other.storage_nodes); + self.is_deleted = self.is_deleted || other.is_deleted; } } @@ -418,6 +604,153 @@ fn exclude_empty_from_pair( iter.into_iter().filter(|(n, _)| !n.is_empty()) } +impl From for StorageTrieUpdates { + fn from(sorted: StorageTrieUpdatesSorted) -> Self { + let mut storage_nodes = HashMap::default(); + let mut removed_nodes = HashSet::default(); + + for (nibbles, node) in sorted.storage_nodes { + if let Some(node) = node { + storage_nodes.insert(nibbles, node); + } else { + removed_nodes.insert(nibbles); + } + } + + Self { is_deleted: sorted.is_deleted, storage_nodes, removed_nodes } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use alloy_primitives::B256; + + #[test] + fn test_trie_updates_sorted_extend_ref() { + // Test extending with empty updates + let mut updates1 = TrieUpdatesSorted::default(); + let updates2 = TrieUpdatesSorted::default(); + updates1.extend_ref(&updates2); + assert_eq!(updates1.account_nodes.len(), 0); + assert_eq!(updates1.storage_tries.len(), 0); + + // Test extending account nodes + let mut updates1 = TrieUpdatesSorted { + account_nodes: vec![ + (Nibbles::from_nibbles_unchecked([0x01]), Some(BranchNodeCompact::default())), + (Nibbles::from_nibbles_unchecked([0x03]), None), + ], + storage_tries: B256Map::default(), + }; + let updates2 = TrieUpdatesSorted { + account_nodes: vec![ + (Nibbles::from_nibbles_unchecked([0x02]), Some(BranchNodeCompact::default())), + (Nibbles::from_nibbles_unchecked([0x03]), Some(BranchNodeCompact::default())), /* Override */ + ], + storage_tries: B256Map::default(), + }; + updates1.extend_ref(&updates2); + assert_eq!(updates1.account_nodes.len(), 3); + // Should be sorted: 0x01, 0x02, 0x03 + assert_eq!(updates1.account_nodes[0].0, Nibbles::from_nibbles_unchecked([0x01])); + assert_eq!(updates1.account_nodes[1].0, Nibbles::from_nibbles_unchecked([0x02])); + assert_eq!(updates1.account_nodes[2].0, Nibbles::from_nibbles_unchecked([0x03])); + // 0x03 should have Some value from updates2 (override) + assert!(updates1.account_nodes[2].1.is_some()); + + // Test extending storage tries + let storage_trie1 = StorageTrieUpdatesSorted { + is_deleted: false, + storage_nodes: vec![( + Nibbles::from_nibbles_unchecked([0x0a]), + Some(BranchNodeCompact::default()), + )], + }; + let storage_trie2 = StorageTrieUpdatesSorted { + is_deleted: false, + storage_nodes: vec![(Nibbles::from_nibbles_unchecked([0x0b]), None)], + }; + + let hashed_address1 = B256::from([1; 32]); + let hashed_address2 = B256::from([2; 32]); + + let mut updates1 = TrieUpdatesSorted { + account_nodes: vec![], + storage_tries: B256Map::from_iter([(hashed_address1, storage_trie1.clone())]), + }; + let updates2 = TrieUpdatesSorted { + account_nodes: vec![], + storage_tries: B256Map::from_iter([ + (hashed_address1, storage_trie2), + (hashed_address2, storage_trie1), + ]), + }; + updates1.extend_ref(&updates2); + assert_eq!(updates1.storage_tries.len(), 2); + assert!(updates1.storage_tries.contains_key(&hashed_address1)); + assert!(updates1.storage_tries.contains_key(&hashed_address2)); + // Check that storage trie for hashed_address1 was extended + let merged_storage = &updates1.storage_tries[&hashed_address1]; + assert_eq!(merged_storage.storage_nodes.len(), 2); + } + + #[test] + fn test_storage_trie_updates_sorted_extend_ref_deleted() { + // Test case 1: Extending with a deleted storage trie that has nodes + let mut storage1 = StorageTrieUpdatesSorted { + is_deleted: false, + storage_nodes: vec![ + (Nibbles::from_nibbles_unchecked([0x01]), Some(BranchNodeCompact::default())), + (Nibbles::from_nibbles_unchecked([0x02]), None), + ], + }; + + let storage2 = StorageTrieUpdatesSorted { + is_deleted: true, + storage_nodes: vec![ + (Nibbles::from_nibbles_unchecked([0x03]), Some(BranchNodeCompact::default())), + (Nibbles::from_nibbles_unchecked([0x04]), None), + ], + }; + + storage1.extend_ref(&storage2); + + // Should be marked as deleted + assert!(storage1.is_deleted); + // Original nodes should be cleared, but other's nodes should be added + assert_eq!(storage1.storage_nodes.len(), 2); + assert_eq!(storage1.storage_nodes[0].0, Nibbles::from_nibbles_unchecked([0x03])); + assert_eq!(storage1.storage_nodes[1].0, Nibbles::from_nibbles_unchecked([0x04])); + + // Test case 2: Extending a deleted storage trie with more nodes + let mut storage3 = StorageTrieUpdatesSorted { + is_deleted: true, + storage_nodes: vec![( + Nibbles::from_nibbles_unchecked([0x05]), + Some(BranchNodeCompact::default()), + )], + }; + + let storage4 = StorageTrieUpdatesSorted { + is_deleted: true, + storage_nodes: vec![ + (Nibbles::from_nibbles_unchecked([0x06]), Some(BranchNodeCompact::default())), + (Nibbles::from_nibbles_unchecked([0x07]), None), + ], + }; + + storage3.extend_ref(&storage4); + + // Should remain deleted + assert!(storage3.is_deleted); + // Should have nodes from other (original cleared then extended) + assert_eq!(storage3.storage_nodes.len(), 2); + assert_eq!(storage3.storage_nodes[0].0, Nibbles::from_nibbles_unchecked([0x06])); + assert_eq!(storage3.storage_nodes[1].0, Nibbles::from_nibbles_unchecked([0x07])); + } +} + /// Bincode-compatible trie updates type serde implementations. #[cfg(feature = "serde-bincode-compat")] pub mod serde_bincode_compat { @@ -579,13 +912,15 @@ pub mod serde_bincode_compat { let decoded: Data = bincode::deserialize(&encoded).unwrap(); assert_eq!(decoded, data); - data.trie_updates.removed_nodes.insert(Nibbles::from_vec(vec![0x0b, 0x0e, 0x0e, 0x0f])); + data.trie_updates + .removed_nodes + .insert(Nibbles::from_nibbles_unchecked([0x0b, 0x0e, 0x0e, 0x0f])); let encoded = bincode::serialize(&data).unwrap(); let decoded: Data = bincode::deserialize(&encoded).unwrap(); assert_eq!(decoded, data); data.trie_updates.account_nodes.insert( - Nibbles::from_vec(vec![0x0d, 0x0e, 0x0a, 0x0d]), + Nibbles::from_nibbles_unchecked([0x0d, 0x0e, 0x0a, 0x0d]), BranchNodeCompact::default(), ); let encoded = bincode::serialize(&data).unwrap(); @@ -612,13 +947,15 @@ pub mod serde_bincode_compat { let decoded: Data = bincode::deserialize(&encoded).unwrap(); assert_eq!(decoded, data); - data.trie_updates.removed_nodes.insert(Nibbles::from_vec(vec![0x0b, 0x0e, 0x0e, 0x0f])); + data.trie_updates + .removed_nodes + .insert(Nibbles::from_nibbles_unchecked([0x0b, 0x0e, 0x0e, 0x0f])); let encoded = bincode::serialize(&data).unwrap(); let decoded: Data = bincode::deserialize(&encoded).unwrap(); assert_eq!(decoded, data); data.trie_updates.storage_nodes.insert( - Nibbles::from_vec(vec![0x0d, 0x0e, 0x0a, 0x0d]), + Nibbles::from_nibbles_unchecked([0x0d, 0x0e, 0x0a, 0x0d]), BranchNodeCompact::default(), ); let encoded = bincode::serialize(&data).unwrap(); @@ -629,7 +966,7 @@ pub mod serde_bincode_compat { } #[cfg(all(test, feature = "serde"))] -mod tests { +mod serde_tests { use super::*; #[test] @@ -639,14 +976,17 @@ mod tests { let updates_deserialized: TrieUpdates = serde_json::from_str(&updates_serialized).unwrap(); assert_eq!(updates_deserialized, default_updates); - default_updates.removed_nodes.insert(Nibbles::from_vec(vec![0x0b, 0x0e, 0x0e, 0x0f])); + default_updates + .removed_nodes + .insert(Nibbles::from_nibbles_unchecked([0x0b, 0x0e, 0x0e, 0x0f])); let updates_serialized = serde_json::to_string(&default_updates).unwrap(); let updates_deserialized: TrieUpdates = serde_json::from_str(&updates_serialized).unwrap(); assert_eq!(updates_deserialized, default_updates); - default_updates - .account_nodes - .insert(Nibbles::from_vec(vec![0x0d, 0x0e, 0x0a, 0x0d]), BranchNodeCompact::default()); + default_updates.account_nodes.insert( + Nibbles::from_nibbles_unchecked([0x0d, 0x0e, 0x0a, 0x0d]), + BranchNodeCompact::default(), + ); let updates_serialized = serde_json::to_string(&default_updates).unwrap(); let updates_deserialized: TrieUpdates = serde_json::from_str(&updates_serialized).unwrap(); assert_eq!(updates_deserialized, default_updates); @@ -665,15 +1005,18 @@ mod tests { serde_json::from_str(&updates_serialized).unwrap(); assert_eq!(updates_deserialized, default_updates); - default_updates.removed_nodes.insert(Nibbles::from_vec(vec![0x0b, 0x0e, 0x0e, 0x0f])); + default_updates + .removed_nodes + .insert(Nibbles::from_nibbles_unchecked([0x0b, 0x0e, 0x0e, 0x0f])); let updates_serialized = serde_json::to_string(&default_updates).unwrap(); let updates_deserialized: StorageTrieUpdates = serde_json::from_str(&updates_serialized).unwrap(); assert_eq!(updates_deserialized, default_updates); - default_updates - .storage_nodes - .insert(Nibbles::from_vec(vec![0x0d, 0x0e, 0x0a, 0x0d]), BranchNodeCompact::default()); + default_updates.storage_nodes.insert( + Nibbles::from_nibbles_unchecked([0x0d, 0x0e, 0x0a, 0x0d]), + BranchNodeCompact::default(), + ); let updates_serialized = serde_json::to_string(&default_updates).unwrap(); let updates_deserialized: StorageTrieUpdates = serde_json::from_str(&updates_serialized).unwrap(); diff --git a/crates/trie/common/src/utils.rs b/crates/trie/common/src/utils.rs new file mode 100644 index 00000000000..5a2234fe26b --- /dev/null +++ b/crates/trie/common/src/utils.rs @@ -0,0 +1,53 @@ +use alloc::vec::Vec; + +/// Helper function to extend a sorted vector with another sorted vector. +/// Values from `other` take precedence for duplicate keys. +/// +/// This function efficiently merges two sorted vectors by: +/// 1. Iterating through the target vector with mutable references +/// 2. Using a peekable iterator for the other vector +/// 3. For each target item, processing other items that come before or equal to it +/// 4. Collecting items from other that need to be inserted +/// 5. Appending and re-sorting only if new items were added +pub(crate) fn extend_sorted_vec(target: &mut Vec<(K, V)>, other: &[(K, V)]) +where + K: Clone + Ord, + V: Clone, +{ + if other.is_empty() { + return; + } + + let mut other_iter = other.iter().peekable(); + let mut to_insert = Vec::new(); + + // Iterate through target and update/collect items from other + for target_item in target.iter_mut() { + while let Some(other_item) = other_iter.peek() { + use core::cmp::Ordering; + match other_item.0.cmp(&target_item.0) { + Ordering::Less => { + // Other item comes before current target item, collect it + to_insert.push(other_iter.next().unwrap().clone()); + } + Ordering::Equal => { + // Same key, update target with other's value + target_item.1 = other_iter.next().unwrap().1.clone(); + break; + } + Ordering::Greater => { + // Other item comes after current target item, keep target unchanged + break; + } + } + } + } + + // Append collected new items, as well as any remaining from `other` which are necessarily also + // new, and sort if needed + if !to_insert.is_empty() || other_iter.peek().is_some() { + target.extend(to_insert); + target.extend(other_iter.cloned()); + target.sort_unstable_by(|a, b| a.0.cmp(&b.0)); + } +} diff --git a/crates/trie/db/Cargo.toml b/crates/trie/db/Cargo.toml index f13acf5ad7f..09ccd301192 100644 --- a/crates/trie/db/Cargo.toml +++ b/crates/trie/db/Cargo.toml @@ -30,7 +30,6 @@ reth-chainspec.workspace = true reth-primitives-traits = { workspace = true, features = ["test-utils", "arbitrary"] } reth-db = { workspace = true, features = ["test-utils"] } reth-provider = { workspace = true, features = ["test-utils"] } -reth-storage-errors.workspace = true reth-trie-common = { workspace = true, features = ["test-utils", "arbitrary"] } reth-trie = { workspace = true, features = ["test-utils"] } diff --git a/crates/trie/db/src/commitment.rs b/crates/trie/db/src/commitment.rs deleted file mode 100644 index fec018061e6..00000000000 --- a/crates/trie/db/src/commitment.rs +++ /dev/null @@ -1,39 +0,0 @@ -use crate::{ - DatabaseHashedCursorFactory, DatabaseProof, DatabaseStateRoot, DatabaseStorageRoot, - DatabaseTrieCursorFactory, DatabaseTrieWitness, -}; -use reth_db_api::transaction::DbTx; -use reth_trie::{ - proof::Proof, witness::TrieWitness, KeccakKeyHasher, KeyHasher, StateRoot, StorageRoot, -}; - -/// The `StateCommitment` trait provides associated types for state commitment operations. -pub trait StateCommitment: std::fmt::Debug + Clone + Send + Sync + Unpin + 'static { - /// The state root type. - type StateRoot<'a, TX: DbTx + 'a>: DatabaseStateRoot<'a, TX>; - /// The storage root type. - type StorageRoot<'a, TX: DbTx + 'a>: DatabaseStorageRoot<'a, TX>; - /// The state proof type. - type StateProof<'a, TX: DbTx + 'a>: DatabaseProof<'a, TX>; - /// The state witness type. - type StateWitness<'a, TX: DbTx + 'a>: DatabaseTrieWitness<'a, TX>; - /// The key hasher type. - type KeyHasher: KeyHasher; -} - -/// The state commitment type for Ethereum's Merkle Patricia Trie. -#[derive(Clone, Debug)] -#[non_exhaustive] -pub struct MerklePatriciaTrie; - -impl StateCommitment for MerklePatriciaTrie { - type StateRoot<'a, TX: DbTx + 'a> = - StateRoot, DatabaseHashedCursorFactory<'a, TX>>; - type StorageRoot<'a, TX: DbTx + 'a> = - StorageRoot, DatabaseHashedCursorFactory<'a, TX>>; - type StateProof<'a, TX: DbTx + 'a> = - Proof, DatabaseHashedCursorFactory<'a, TX>>; - type StateWitness<'a, TX: DbTx + 'a> = - TrieWitness, DatabaseHashedCursorFactory<'a, TX>>; - type KeyHasher = KeccakKeyHasher; -} diff --git a/crates/trie/db/src/hashed_cursor.rs b/crates/trie/db/src/hashed_cursor.rs index 04ee663d7c0..4fe3d57429f 100644 --- a/crates/trie/db/src/hashed_cursor.rs +++ b/crates/trie/db/src/hashed_cursor.rs @@ -9,35 +9,34 @@ use reth_primitives_traits::Account; use reth_trie::hashed_cursor::{HashedCursor, HashedCursorFactory, HashedStorageCursor}; /// A struct wrapping database transaction that implements [`HashedCursorFactory`]. -#[derive(Debug)] -pub struct DatabaseHashedCursorFactory<'a, TX>(&'a TX); +#[derive(Debug, Clone)] +pub struct DatabaseHashedCursorFactory(T); -impl Clone for DatabaseHashedCursorFactory<'_, TX> { - fn clone(&self) -> Self { - Self(self.0) - } -} - -impl<'a, TX> DatabaseHashedCursorFactory<'a, TX> { +impl DatabaseHashedCursorFactory { /// Create new database hashed cursor factory. - pub const fn new(tx: &'a TX) -> Self { + pub const fn new(tx: T) -> Self { Self(tx) } } -impl HashedCursorFactory for DatabaseHashedCursorFactory<'_, TX> { - type AccountCursor = DatabaseHashedAccountCursor<::Cursor>; - type StorageCursor = - DatabaseHashedStorageCursor<::DupCursor>; - - fn hashed_account_cursor(&self) -> Result { +impl HashedCursorFactory for DatabaseHashedCursorFactory<&TX> { + type AccountCursor<'a> + = DatabaseHashedAccountCursor<::Cursor> + where + Self: 'a; + type StorageCursor<'a> + = DatabaseHashedStorageCursor<::DupCursor> + where + Self: 'a; + + fn hashed_account_cursor(&self) -> Result, DatabaseError> { Ok(DatabaseHashedAccountCursor(self.0.cursor_read::()?)) } fn hashed_storage_cursor( &self, hashed_address: B256, - ) -> Result { + ) -> Result, DatabaseError> { Ok(DatabaseHashedStorageCursor::new( self.0.cursor_dup_read::()?, hashed_address, diff --git a/crates/trie/db/src/lib.rs b/crates/trie/db/src/lib.rs index a0a19d0bb2f..5417e5bd1e5 100644 --- a/crates/trie/db/src/lib.rs +++ b/crates/trie/db/src/lib.rs @@ -1,8 +1,7 @@ -//! An integration of [`reth-trie`] with [`reth-db`]. +//! An integration of `reth-trie` with `reth-db`. #![cfg_attr(not(test), warn(unused_crate_dependencies))] -mod commitment; mod hashed_cursor; mod prefix_set; mod proof; @@ -11,7 +10,6 @@ mod storage; mod trie_cursor; mod witness; -pub use commitment::{MerklePatriciaTrie, StateCommitment}; pub use hashed_cursor::{ DatabaseHashedAccountCursor, DatabaseHashedCursorFactory, DatabaseHashedStorageCursor, }; diff --git a/crates/trie/db/src/proof.rs b/crates/trie/db/src/proof.rs index 137e661b056..8f79c21c156 100644 --- a/crates/trie/db/src/proof.rs +++ b/crates/trie/db/src/proof.rs @@ -11,13 +11,16 @@ use reth_trie::{ }; /// Extends [`Proof`] with operations specific for working with a database transaction. -pub trait DatabaseProof<'a, TX> { - /// Create a new [Proof] from database transaction. - fn from_tx(tx: &'a TX) -> Self; +pub trait DatabaseProof<'a> { + /// Associated type for the database transaction. + type Tx; + + /// Create a new [`Proof`] instance from database transaction. + fn from_tx(tx: &'a Self::Tx) -> Self; /// Generates the state proof for target account based on [`TrieInput`]. fn overlay_account_proof( - tx: &'a TX, + &self, input: TrieInput, address: Address, slots: &[B256], @@ -25,59 +28,49 @@ pub trait DatabaseProof<'a, TX> { /// Generates the state [`MultiProof`] for target hashed account and storage keys. fn overlay_multiproof( - tx: &'a TX, + &self, input: TrieInput, targets: MultiProofTargets, ) -> Result; } -impl<'a, TX: DbTx> DatabaseProof<'a, TX> - for Proof, DatabaseHashedCursorFactory<'a, TX>> +impl<'a, TX: DbTx> DatabaseProof<'a> + for Proof, DatabaseHashedCursorFactory<&'a TX>> { - /// Create a new [Proof] instance from database transaction. - fn from_tx(tx: &'a TX) -> Self { + type Tx = TX; + + fn from_tx(tx: &'a Self::Tx) -> Self { Self::new(DatabaseTrieCursorFactory::new(tx), DatabaseHashedCursorFactory::new(tx)) } - fn overlay_account_proof( - tx: &'a TX, + &self, input: TrieInput, address: Address, slots: &[B256], ) -> Result { let nodes_sorted = input.nodes.into_sorted(); let state_sorted = input.state.into_sorted(); - Self::from_tx(tx) - .with_trie_cursor_factory(InMemoryTrieCursorFactory::new( - DatabaseTrieCursorFactory::new(tx), - &nodes_sorted, - )) - .with_hashed_cursor_factory(HashedPostStateCursorFactory::new( - DatabaseHashedCursorFactory::new(tx), - &state_sorted, - )) - .with_prefix_sets_mut(input.prefix_sets) - .account_proof(address, slots) + Proof::new( + InMemoryTrieCursorFactory::new(self.trie_cursor_factory().clone(), &nodes_sorted), + HashedPostStateCursorFactory::new(self.hashed_cursor_factory().clone(), &state_sorted), + ) + .with_prefix_sets_mut(input.prefix_sets) + .account_proof(address, slots) } fn overlay_multiproof( - tx: &'a TX, + &self, input: TrieInput, targets: MultiProofTargets, ) -> Result { let nodes_sorted = input.nodes.into_sorted(); let state_sorted = input.state.into_sorted(); - Self::from_tx(tx) - .with_trie_cursor_factory(InMemoryTrieCursorFactory::new( - DatabaseTrieCursorFactory::new(tx), - &nodes_sorted, - )) - .with_hashed_cursor_factory(HashedPostStateCursorFactory::new( - DatabaseHashedCursorFactory::new(tx), - &state_sorted, - )) - .with_prefix_sets_mut(input.prefix_sets) - .multiproof(targets) + Proof::new( + InMemoryTrieCursorFactory::new(self.trie_cursor_factory().clone(), &nodes_sorted), + HashedPostStateCursorFactory::new(self.hashed_cursor_factory().clone(), &state_sorted), + ) + .with_prefix_sets_mut(input.prefix_sets) + .multiproof(targets) } } @@ -104,7 +97,7 @@ pub trait DatabaseStorageProof<'a, TX> { } impl<'a, TX: DbTx> DatabaseStorageProof<'a, TX> - for StorageProof, DatabaseHashedCursorFactory<'a, TX>> + for StorageProof, DatabaseHashedCursorFactory<&'a TX>> { fn from_tx(tx: &'a TX, address: Address) -> Self { Self::new(DatabaseTrieCursorFactory::new(tx), DatabaseHashedCursorFactory::new(tx), address) diff --git a/crates/trie/db/src/state.rs b/crates/trie/db/src/state.rs index 757e0b98eb4..6d37c5f3413 100644 --- a/crates/trie/db/src/state.rs +++ b/crates/trie/db/src/state.rs @@ -1,11 +1,11 @@ use crate::{DatabaseHashedCursorFactory, DatabaseTrieCursorFactory, PrefixSetLoader}; use alloy_primitives::{ map::{AddressMap, B256Map}, - Address, BlockNumber, B256, U256, + BlockNumber, B256, U256, }; use reth_db_api::{ cursor::DbCursorRO, - models::{AccountBeforeTx, BlockNumberAddress}, + models::{AccountBeforeTx, BlockNumberAddress, BlockNumberAddressRange}, tables, transaction::DbTx, DatabaseError, @@ -16,8 +16,11 @@ use reth_trie::{ updates::TrieUpdates, HashedPostState, HashedStorage, KeccakKeyHasher, KeyHasher, StateRoot, StateRootProgress, TrieInput, }; -use std::{collections::HashMap, ops::RangeInclusive}; -use tracing::debug; +use std::{ + collections::HashMap, + ops::{RangeBounds, RangeInclusive}, +}; +use tracing::{debug, instrument}; /// Extends [`StateRoot`] with operations specific for working with a database transaction. pub trait DatabaseStateRoot<'a, TX>: Sized { @@ -124,13 +127,16 @@ pub trait DatabaseStateRoot<'a, TX>: Sized { /// Extends [`HashedPostState`] with operations specific for working with a database transaction. pub trait DatabaseHashedPostState: Sized { - /// Initializes [`HashedPostState`] from reverts. Iterates over state reverts from the specified - /// block up to the current tip and aggregates them into hashed state in reverse. - fn from_reverts(tx: &TX, from: BlockNumber) -> Result; + /// Initializes [`HashedPostState`] from reverts. Iterates over state reverts in the specified + /// range and aggregates them into hashed state in reverse. + fn from_reverts( + tx: &TX, + range: impl RangeBounds, + ) -> Result; } impl<'a, TX: DbTx> DatabaseStateRoot<'a, TX> - for StateRoot, DatabaseHashedCursorFactory<'a, TX>> + for StateRoot, DatabaseHashedCursorFactory<&'a TX>> { fn from_tx(tx: &'a TX) -> Self { Self::new(DatabaseTrieCursorFactory::new(tx), DatabaseHashedCursorFactory::new(tx)) @@ -220,21 +226,25 @@ impl<'a, TX: DbTx> DatabaseStateRoot<'a, TX> } impl DatabaseHashedPostState for HashedPostState { - fn from_reverts(tx: &TX, from: BlockNumber) -> Result { + #[instrument(target = "trie::db", skip(tx), fields(range))] + fn from_reverts( + tx: &TX, + range: impl RangeBounds, + ) -> Result { // Iterate over account changesets and record value before first occurring account change. + let account_range = (range.start_bound(), range.end_bound()); // to avoid cloning let mut accounts = HashMap::new(); let mut account_changesets_cursor = tx.cursor_read::()?; - for entry in account_changesets_cursor.walk_range(from..)? { + for entry in account_changesets_cursor.walk_range(account_range)? { let (_, AccountBeforeTx { address, info }) = entry?; accounts.entry(address).or_insert(info); } // Iterate over storage changesets and record value before first occurring storage change. + let storage_range: BlockNumberAddressRange = range.into(); let mut storages = AddressMap::>::default(); let mut storage_changesets_cursor = tx.cursor_read::()?; - for entry in - storage_changesets_cursor.walk_range(BlockNumberAddress((from, Address::ZERO))..)? - { + for entry in storage_changesets_cursor.walk_range(storage_range)? { let (BlockNumberAddress((_, address)), storage) = entry?; let account_storage = storages.entry(address).or_default(); account_storage.entry(storage.key).or_insert(storage.value); @@ -250,8 +260,8 @@ impl DatabaseHashedPostState for HashedPostState { KH::hash_key(address), HashedStorage::from_iter( // The `wiped` flag indicates only whether previous storage entries - // should be looked up in db or not. For reverts it's a noop since all - // wiped changes had been written as storage reverts. + // should be looked up in db or not. For reverts it's a noop since all + // wiped changes had been written as storage reverts. false, storage.into_iter().map(|(slot, value)| (KH::hash_key(slot), value)), ), diff --git a/crates/trie/db/src/storage.rs b/crates/trie/db/src/storage.rs index f7c9fdc3a98..42d0d464c77 100644 --- a/crates/trie/db/src/storage.rs +++ b/crates/trie/db/src/storage.rs @@ -35,7 +35,7 @@ pub trait DatabaseHashedStorage: Sized { } impl<'a, TX: DbTx> DatabaseStorageRoot<'a, TX> - for StorageRoot, DatabaseHashedCursorFactory<'a, TX>> + for StorageRoot, DatabaseHashedCursorFactory<&'a TX>> { fn from_tx(tx: &'a TX, address: Address) -> Self { Self::new( diff --git a/crates/trie/db/src/trie_cursor.rs b/crates/trie/db/src/trie_cursor.rs index ad6b8eac171..d05c3fd92da 100644 --- a/crates/trie/db/src/trie_cursor.rs +++ b/crates/trie/db/src/trie_cursor.rs @@ -7,41 +7,43 @@ use reth_db_api::{ }; use reth_trie::{ trie_cursor::{TrieCursor, TrieCursorFactory}, - updates::StorageTrieUpdates, + updates::StorageTrieUpdatesSorted, BranchNodeCompact, Nibbles, StorageTrieEntry, StoredNibbles, StoredNibblesSubKey, }; /// Wrapper struct for database transaction implementing trie cursor factory trait. -#[derive(Debug)] -pub struct DatabaseTrieCursorFactory<'a, TX>(&'a TX); - -impl Clone for DatabaseTrieCursorFactory<'_, TX> { - fn clone(&self) -> Self { - Self(self.0) - } -} +#[derive(Debug, Clone)] +pub struct DatabaseTrieCursorFactory(T); -impl<'a, TX> DatabaseTrieCursorFactory<'a, TX> { +impl DatabaseTrieCursorFactory { /// Create new [`DatabaseTrieCursorFactory`]. - pub const fn new(tx: &'a TX) -> Self { + pub const fn new(tx: T) -> Self { Self(tx) } } -/// Implementation of the trie cursor factory for a database transaction. -impl TrieCursorFactory for DatabaseTrieCursorFactory<'_, TX> { - type AccountTrieCursor = DatabaseAccountTrieCursor<::Cursor>; - type StorageTrieCursor = - DatabaseStorageTrieCursor<::DupCursor>; +impl TrieCursorFactory for DatabaseTrieCursorFactory<&TX> +where + TX: DbTx, +{ + type AccountTrieCursor<'a> + = DatabaseAccountTrieCursor<::Cursor> + where + Self: 'a; - fn account_trie_cursor(&self) -> Result { + type StorageTrieCursor<'a> + = DatabaseStorageTrieCursor<::DupCursor> + where + Self: 'a; + + fn account_trie_cursor(&self) -> Result, DatabaseError> { Ok(DatabaseAccountTrieCursor::new(self.0.cursor_read::()?)) } fn storage_trie_cursor( &self, hashed_address: B256, - ) -> Result { + ) -> Result, DatabaseError> { Ok(DatabaseStorageTrieCursor::new( self.0.cursor_dup_read::()?, hashed_address, @@ -114,33 +116,21 @@ where + DbDupCursorRO + DbDupCursorRW, { - /// Writes storage updates - pub fn write_storage_trie_updates( + /// Writes storage updates that are already sorted + pub fn write_storage_trie_updates_sorted( &mut self, - updates: &StorageTrieUpdates, + updates: &StorageTrieUpdatesSorted, ) -> Result { // The storage trie for this account has to be deleted. if updates.is_deleted() && self.cursor.seek_exact(self.hashed_address)?.is_some() { self.cursor.delete_current_duplicates()?; } - // Merge updated and removed nodes. Updated nodes must take precedence. - let mut storage_updates = updates - .removed_nodes_ref() - .iter() - .filter_map(|n| (!updates.storage_nodes_ref().contains_key(n)).then_some((n, None))) - .collect::>(); - storage_updates.extend( - updates.storage_nodes_ref().iter().map(|(nibbles, node)| (nibbles, Some(node))), - ); - - // Sort trie node updates. - storage_updates.sort_unstable_by(|a, b| a.0.cmp(b.0)); - let mut num_entries = 0; - for (nibbles, maybe_updated) in storage_updates.into_iter().filter(|(n, _)| !n.is_empty()) { + for (nibbles, maybe_updated) in updates.storage_nodes.iter().filter(|(n, _)| !n.is_empty()) + { num_entries += 1; - let nibbles = StoredNibblesSubKey(nibbles.clone()); + let nibbles = StoredNibblesSubKey(*nibbles); // Delete the old entry if it exists. if self .cursor @@ -175,7 +165,7 @@ where ) -> Result, DatabaseError> { Ok(self .cursor - .seek_by_key_subkey(self.hashed_address, StoredNibblesSubKey(key.clone()))? + .seek_by_key_subkey(self.hashed_address, StoredNibblesSubKey(key))? .filter(|e| e.nibbles == StoredNibblesSubKey(key)) .map(|value| (value.nibbles.0, value.node))) } diff --git a/crates/trie/db/src/witness.rs b/crates/trie/db/src/witness.rs index a240734f8ce..c5995e4d982 100644 --- a/crates/trie/db/src/witness.rs +++ b/crates/trie/db/src/witness.rs @@ -21,7 +21,7 @@ pub trait DatabaseTrieWitness<'a, TX> { } impl<'a, TX: DbTx> DatabaseTrieWitness<'a, TX> - for TrieWitness, DatabaseHashedCursorFactory<'a, TX>> + for TrieWitness, DatabaseHashedCursorFactory<&'a TX>> { fn from_tx(tx: &'a TX) -> Self { Self::new(DatabaseTrieCursorFactory::new(tx), DatabaseHashedCursorFactory::new(tx)) @@ -44,6 +44,7 @@ impl<'a, TX: DbTx> DatabaseTrieWitness<'a, TX> &state_sorted, )) .with_prefix_sets_mut(input.prefix_sets) + .always_include_root_node() .compute(target) } } diff --git a/crates/trie/db/tests/post_state.rs b/crates/trie/db/tests/post_state.rs index 209b94ec944..ae59bc871ec 100644 --- a/crates/trie/db/tests/post_state.rs +++ b/crates/trie/db/tests/post_state.rs @@ -100,14 +100,14 @@ fn account_cursor_correct_order() { let db = create_test_rw_db(); db.update(|tx| { - for (key, account) in accounts.iter().filter(|x| x.0[31] % 2 == 0) { + for (key, account) in accounts.iter().filter(|x| x.0[31].is_multiple_of(2)) { tx.put::(*key, *account).unwrap(); } }) .unwrap(); let mut hashed_post_state = HashedPostState::default(); - for (hashed_address, account) in accounts.iter().filter(|x| x.0[31] % 2 != 0) { + for (hashed_address, account) in accounts.iter().filter(|x| !x.0[31].is_multiple_of(2)) { hashed_post_state.accounts.insert(*hashed_address, Some(*account)); } @@ -127,14 +127,14 @@ fn removed_accounts_are_discarded() { let db = create_test_rw_db(); db.update(|tx| { - for (key, account) in accounts.iter().filter(|x| x.0[31] % 2 == 0) { + for (key, account) in accounts.iter().filter(|x| x.0[31].is_multiple_of(2)) { tx.put::(*key, *account).unwrap(); } }) .unwrap(); let mut hashed_post_state = HashedPostState::default(); - for (hashed_address, account) in accounts.iter().filter(|x| x.0[31] % 2 != 0) { + for (hashed_address, account) in accounts.iter().filter(|x| !x.0[31].is_multiple_of(2)) { hashed_post_state.accounts.insert( *hashed_address, if removed_keys.contains(hashed_address) { None } else { Some(*account) }, @@ -227,7 +227,7 @@ fn storage_is_empty() { (0..10).map(|key| (B256::with_last_byte(key), U256::from(key))).collect::>(); db.update(|tx| { for (slot, value) in &db_storage { - // insert zero value accounts to the database + // insert storage entries to the database tx.put::(address, StorageEntry { key: *slot, value: *value }) .unwrap(); } @@ -338,14 +338,17 @@ fn zero_value_storage_entries_are_discarded() { (0..10).map(|key| (B256::with_last_byte(key), U256::from(key))).collect::>(); // every even number is changed to zero value let post_state_storage = (0..10) .map(|key| { - (B256::with_last_byte(key), if key % 2 == 0 { U256::ZERO } else { U256::from(key) }) + ( + B256::with_last_byte(key), + if key.is_multiple_of(2) { U256::ZERO } else { U256::from(key) }, + ) }) .collect::>(); let db = create_test_rw_db(); db.update(|tx| { for (slot, value) in db_storage { - // insert zero value accounts to the database + // insert storage entries to the database tx.put::(address, StorageEntry { key: slot, value }).unwrap(); } }) diff --git a/crates/trie/db/tests/proof.rs b/crates/trie/db/tests/proof.rs index 401ba07b22d..402f0cabff3 100644 --- a/crates/trie/db/tests/proof.rs +++ b/crates/trie/db/tests/proof.rs @@ -86,7 +86,8 @@ fn testspec_proofs() { let provider = factory.provider().unwrap(); for (target, expected_proof) in data { let target = Address::from_str(target).unwrap(); - let account_proof = Proof::from_tx(provider.tx_ref()).account_proof(target, &[]).unwrap(); + let proof = as DatabaseProof>::from_tx(provider.tx_ref()); + let account_proof = proof.account_proof(target, &[]).unwrap(); similar_asserts::assert_eq!( account_proof.proof, expected_proof, @@ -106,7 +107,8 @@ fn testspec_empty_storage_proof() { let slots = Vec::from([B256::with_last_byte(1), B256::with_last_byte(3)]); let provider = factory.provider().unwrap(); - let account_proof = Proof::from_tx(provider.tx_ref()).account_proof(target, &slots).unwrap(); + let proof = as DatabaseProof>::from_tx(provider.tx_ref()); + let account_proof = proof.account_proof(target, &slots).unwrap(); assert_eq!(account_proof.storage_root, EMPTY_ROOT_HASH, "expected empty storage root"); assert_eq!(slots.len(), account_proof.storage_proofs.len()); @@ -141,7 +143,8 @@ fn mainnet_genesis_account_proof() { ]); let provider = factory.provider().unwrap(); - let account_proof = Proof::from_tx(provider.tx_ref()).account_proof(target, &[]).unwrap(); + let proof = as DatabaseProof>::from_tx(provider.tx_ref()); + let account_proof = proof.account_proof(target, &[]).unwrap(); similar_asserts::assert_eq!(account_proof.proof, expected_account_proof); assert_eq!(account_proof.verify(root), Ok(())); } @@ -164,7 +167,8 @@ fn mainnet_genesis_account_proof_nonexistent() { ]); let provider = factory.provider().unwrap(); - let account_proof = Proof::from_tx(provider.tx_ref()).account_proof(target, &[]).unwrap(); + let proof = as DatabaseProof>::from_tx(provider.tx_ref()); + let account_proof = proof.account_proof(target, &[]).unwrap(); similar_asserts::assert_eq!(account_proof.proof, expected_account_proof); assert_eq!(account_proof.verify(root), Ok(())); } @@ -259,7 +263,8 @@ fn holesky_deposit_contract_proof() { }; let provider = factory.provider().unwrap(); - let account_proof = Proof::from_tx(provider.tx_ref()).account_proof(target, &slots).unwrap(); + let proof = as DatabaseProof>::from_tx(provider.tx_ref()); + let account_proof = proof.account_proof(target, &slots).unwrap(); similar_asserts::assert_eq!(account_proof, expected); assert_eq!(account_proof.verify(root), Ok(())); } diff --git a/crates/trie/db/tests/trie.rs b/crates/trie/db/tests/trie.rs index 232d36e66e6..8f543a711d8 100644 --- a/crates/trie/db/tests/trie.rs +++ b/crates/trie/db/tests/trie.rs @@ -1,7 +1,9 @@ #![allow(missing_docs)] use alloy_consensus::EMPTY_ROOT_HASH; -use alloy_primitives::{hex_literal::hex, keccak256, map::HashMap, Address, B256, U256}; +use alloy_primitives::{ + address, b256, hex_literal::hex, keccak256, map::HashMap, Address, B256, U256, +}; use alloy_rlp::Encodable; use proptest::{prelude::ProptestConfig, proptest}; use proptest_arbitrary_interop::arb; @@ -79,7 +81,11 @@ fn incremental_vs_full_root(inputs: &[&str], modified: &str) { let modified_root = loader.root().unwrap(); // Update the intermediate roots table so that we can run the incremental verification - tx.write_individual_storage_trie_updates(hashed_address, &trie_updates).unwrap(); + tx.write_storage_trie_updates_sorted(core::iter::once(( + &hashed_address, + &trie_updates.into_sorted(), + ))) + .unwrap(); // 3. Calculate the incremental root let mut storage_changes = PrefixSetMut::default(); @@ -295,7 +301,7 @@ fn storage_root_regression() { let factory = create_test_provider_factory(); let tx = factory.provider_rw().unwrap(); // Some address whose hash starts with 0xB041 - let address3 = Address::from_str("16b07afd1c635f77172e842a000ead9a2a222459").unwrap(); + let address3 = address!("0x16b07afd1c635f77172e842a000ead9a2a222459"); let key3 = keccak256(address3); assert_eq!(key3[0], 0xB0); assert_eq!(key3[1], 0x41); @@ -346,14 +352,13 @@ fn account_and_storage_trie() { let mut hash_builder = HashBuilder::default(); // Insert first account - let key1 = - B256::from_str("b000000000000000000000000000000000000000000000000000000000000000").unwrap(); + let key1 = b256!("0xb000000000000000000000000000000000000000000000000000000000000000"); let account1 = Account { nonce: 0, balance: U256::from(3).mul(ether), bytecode_hash: None }; hashed_account_cursor.upsert(key1, &account1).unwrap(); hash_builder.add_leaf(Nibbles::unpack(key1), &encode_account(account1, None)); // Some address whose hash starts with 0xB040 - let address2 = Address::from_str("7db3e81b72d2695e19764583f6d219dbee0f35ca").unwrap(); + let address2 = address!("0x7db3e81b72d2695e19764583f6d219dbee0f35ca"); let key2 = keccak256(address2); assert_eq!(key2[0], 0xB0); assert_eq!(key2[1], 0x40); @@ -362,12 +367,11 @@ fn account_and_storage_trie() { hash_builder.add_leaf(Nibbles::unpack(key2), &encode_account(account2, None)); // Some address whose hash starts with 0xB041 - let address3 = Address::from_str("16b07afd1c635f77172e842a000ead9a2a222459").unwrap(); + let address3 = address!("0x16b07afd1c635f77172e842a000ead9a2a222459"); let key3 = keccak256(address3); assert_eq!(key3[0], 0xB0); assert_eq!(key3[1], 0x41); - let code_hash = - B256::from_str("5be74cad16203c4905c068b012a2e9fb6d19d036c410f16fd177f337541440dd").unwrap(); + let code_hash = b256!("0x5be74cad16203c4905c068b012a2e9fb6d19d036c410f16fd177f337541440dd"); let account3 = Account { nonce: 0, balance: U256::from(2).mul(ether), bytecode_hash: Some(code_hash) }; hashed_account_cursor.upsert(key3, &account3).unwrap(); @@ -386,27 +390,23 @@ fn account_and_storage_trie() { hash_builder .add_leaf(Nibbles::unpack(key3), &encode_account(account3, Some(account3_storage_root))); - let key4a = - B256::from_str("B1A0000000000000000000000000000000000000000000000000000000000000").unwrap(); + let key4a = b256!("0xB1A0000000000000000000000000000000000000000000000000000000000000"); let account4a = Account { nonce: 0, balance: U256::from(4).mul(ether), ..Default::default() }; hashed_account_cursor.upsert(key4a, &account4a).unwrap(); hash_builder.add_leaf(Nibbles::unpack(key4a), &encode_account(account4a, None)); - let key5 = - B256::from_str("B310000000000000000000000000000000000000000000000000000000000000").unwrap(); + let key5 = b256!("0xB310000000000000000000000000000000000000000000000000000000000000"); let account5 = Account { nonce: 0, balance: U256::from(8).mul(ether), ..Default::default() }; hashed_account_cursor.upsert(key5, &account5).unwrap(); hash_builder.add_leaf(Nibbles::unpack(key5), &encode_account(account5, None)); - let key6 = - B256::from_str("B340000000000000000000000000000000000000000000000000000000000000").unwrap(); + let key6 = b256!("0xB340000000000000000000000000000000000000000000000000000000000000"); let account6 = Account { nonce: 0, balance: U256::from(1).mul(ether), ..Default::default() }; hashed_account_cursor.upsert(key6, &account6).unwrap(); hash_builder.add_leaf(Nibbles::unpack(key6), &encode_account(account6, None)); // Populate account & storage trie DB tables - let expected_root = - B256::from_str("72861041bc90cd2f93777956f058a545412b56de79af5eb6b8075fe2eabbe015").unwrap(); + let expected_root = b256!("0x72861041bc90cd2f93777956f058a545412b56de79af5eb6b8075fe2eabbe015"); let computed_expected_root: B256 = triehash::trie_root::([ (key1, encode_account(account1, None)), (key2, encode_account(account2, None)), @@ -431,7 +431,8 @@ fn account_and_storage_trie() { assert_eq!(account_updates.len(), 2); let (nibbles1a, node1a) = account_updates.first().unwrap(); - assert_eq!(nibbles1a[..], [0xB]); + assert_eq!(nibbles1a.to_vec(), vec![0xB]); + let node1a = node1a.as_ref().unwrap(); assert_eq!(node1a.state_mask, TrieMask::new(0b1011)); assert_eq!(node1a.tree_mask, TrieMask::new(0b0001)); assert_eq!(node1a.hash_mask, TrieMask::new(0b1001)); @@ -439,7 +440,8 @@ fn account_and_storage_trie() { assert_eq!(node1a.hashes.len(), 2); let (nibbles2a, node2a) = account_updates.last().unwrap(); - assert_eq!(nibbles2a[..], [0xB, 0x0]); + assert_eq!(nibbles2a.to_vec(), vec![0xB, 0x0]); + let node2a = node2a.as_ref().unwrap(); assert_eq!(node2a.state_mask, TrieMask::new(0b10001)); assert_eq!(node2a.tree_mask, TrieMask::new(0b00000)); assert_eq!(node2a.hash_mask, TrieMask::new(0b10000)); @@ -448,7 +450,7 @@ fn account_and_storage_trie() { // Add an account // Some address whose hash starts with 0xB1 - let address4b = Address::from_str("4f61f2d5ebd991b85aa1677db97307caf5215c91").unwrap(); + let address4b = address!("0x4f61f2d5ebd991b85aa1677db97307caf5215c91"); let key4b = keccak256(address4b); assert_eq!(key4b.0[0], key4a.0[0]); let account4b = Account { nonce: 0, balance: U256::from(5).mul(ether), bytecode_hash: None }; @@ -458,7 +460,7 @@ fn account_and_storage_trie() { prefix_set.insert(Nibbles::unpack(key4b)); let expected_state_root = - B256::from_str("8e263cd4eefb0c3cbbb14e5541a66a755cad25bcfab1e10dd9d706263e811b28").unwrap(); + b256!("0x8e263cd4eefb0c3cbbb14e5541a66a755cad25bcfab1e10dd9d706263e811b28"); let (root, trie_updates) = StateRoot::from_tx(tx.tx_ref()) .with_prefix_sets(TriePrefixSets { @@ -474,7 +476,8 @@ fn account_and_storage_trie() { assert_eq!(account_updates.len(), 2); let (nibbles1b, node1b) = account_updates.first().unwrap(); - assert_eq!(nibbles1b[..], [0xB]); + assert_eq!(nibbles1b.to_vec(), vec![0xB]); + let node1b = node1b.as_ref().unwrap(); assert_eq!(node1b.state_mask, TrieMask::new(0b1011)); assert_eq!(node1b.tree_mask, TrieMask::new(0b0001)); assert_eq!(node1b.hash_mask, TrieMask::new(0b1011)); @@ -484,7 +487,8 @@ fn account_and_storage_trie() { assert_eq!(node1a.hashes[1], node1b.hashes[2]); let (nibbles2b, node2b) = account_updates.last().unwrap(); - assert_eq!(nibbles2b[..], [0xB, 0x0]); + assert_eq!(nibbles2b.to_vec(), vec![0xB, 0x0]); + let node2b = node2b.as_ref().unwrap(); assert_eq!(node2a, node2b); tx.commit().unwrap(); @@ -524,8 +528,9 @@ fn account_and_storage_trie() { assert_eq!(trie_updates.account_nodes_ref().len(), 1); - let (nibbles1c, node1c) = trie_updates.account_nodes_ref().iter().next().unwrap(); - assert_eq!(nibbles1c[..], [0xB]); + let entry = trie_updates.account_nodes_ref().iter().next().unwrap(); + assert_eq!(entry.0.to_vec(), vec![0xB]); + let node1c = entry.1; assert_eq!(node1c.state_mask, TrieMask::new(0b1011)); assert_eq!(node1c.tree_mask, TrieMask::new(0b0000)); @@ -582,8 +587,9 @@ fn account_and_storage_trie() { assert_eq!(trie_updates.account_nodes_ref().len(), 1); - let (nibbles1d, node1d) = trie_updates.account_nodes_ref().iter().next().unwrap(); - assert_eq!(nibbles1d[..], [0xB]); + let entry = trie_updates.account_nodes_ref().iter().next().unwrap(); + assert_eq!(entry.0.to_vec(), vec![0xB]); + let node1d = entry.1; assert_eq!(node1d.state_mask, TrieMask::new(0b1011)); assert_eq!(node1d.tree_mask, TrieMask::new(0b0000)); @@ -618,7 +624,7 @@ fn account_trie_around_extension_node_with_dbtrie() { let (got, updates) = StateRoot::from_tx(tx.tx_ref()).root_with_updates().unwrap(); assert_eq!(expected, got); - tx.write_trie_updates(&updates).unwrap(); + tx.write_trie_updates(updates).unwrap(); // read the account updates from the db let mut accounts_trie = tx.tx_ref().cursor_read::().unwrap(); @@ -665,7 +671,7 @@ proptest! { state.iter().map(|(&key, &balance)| (key, (Account { balance, ..Default::default() }, std::iter::empty()))) ); assert_eq!(expected_root, state_root); - tx.write_trie_updates(&trie_updates).unwrap(); + tx.write_trie_updates(trie_updates).unwrap(); } } } @@ -742,11 +748,11 @@ fn extension_node_trie( fn assert_trie_updates(account_updates: &HashMap) { assert_eq!(account_updates.len(), 2); - let node = account_updates.get(&[0x3][..]).unwrap(); + let node = account_updates.get(&Nibbles::from_nibbles_unchecked([0x3])).unwrap(); let expected = BranchNodeCompact::new(0b0011, 0b0001, 0b0000, vec![], None); assert_eq!(node, &expected); - let node = account_updates.get(&[0x3, 0x0, 0xA, 0xF][..]).unwrap(); + let node = account_updates.get(&Nibbles::from_nibbles_unchecked([0x3, 0x0, 0xA, 0xF])).unwrap(); assert_eq!(node.state_mask, TrieMask::new(0b101100000)); assert_eq!(node.tree_mask, TrieMask::new(0b000000000)); assert_eq!(node.hash_mask, TrieMask::new(0b001000000)); diff --git a/crates/trie/db/tests/walker.rs b/crates/trie/db/tests/walker.rs index 5fd7538cd47..edc69e330b7 100644 --- a/crates/trie/db/tests/walker.rs +++ b/crates/trie/db/tests/walker.rs @@ -60,13 +60,13 @@ fn test_cursor(mut trie: T, expected: &[Vec]) where T: TrieCursor, { - let mut walker = TrieWalker::state_trie(&mut trie, Default::default()); + let mut walker = TrieWalker::<_>::state_trie(&mut trie, Default::default()); assert!(walker.key().unwrap().is_empty()); // We're traversing the path in lexicographical order. for expected in expected { walker.advance().unwrap(); - let got = walker.key().cloned(); + let got = walker.key().copied(); assert_eq!(got.unwrap(), Nibbles::from_nibbles_unchecked(expected.clone())); } @@ -114,28 +114,28 @@ fn cursor_rootnode_with_changesets() { let mut trie = DatabaseStorageTrieCursor::new(cursor, hashed_address); // No changes - let mut cursor = TrieWalker::state_trie(&mut trie, Default::default()); - assert_eq!(cursor.key().cloned(), Some(Nibbles::new())); // root + let mut cursor = TrieWalker::<_>::state_trie(&mut trie, Default::default()); + assert_eq!(cursor.key().copied(), Some(Nibbles::new())); // root assert!(cursor.can_skip_current_node); // due to root_hash cursor.advance().unwrap(); // skips to the end of trie - assert_eq!(cursor.key().cloned(), None); + assert_eq!(cursor.key().copied(), None); // We insert something that's not part of the existing trie/prefix. let mut changed = PrefixSetMut::default(); changed.insert(Nibbles::from_nibbles([0xF, 0x1])); - let mut cursor = TrieWalker::state_trie(&mut trie, changed.freeze()); + let mut cursor = TrieWalker::<_>::state_trie(&mut trie, changed.freeze()); // Root node - assert_eq!(cursor.key().cloned(), Some(Nibbles::new())); + assert_eq!(cursor.key().copied(), Some(Nibbles::new())); // Should not be able to skip state due to the changed values assert!(!cursor.can_skip_current_node); cursor.advance().unwrap(); - assert_eq!(cursor.key().cloned(), Some(Nibbles::from_nibbles([0x2]))); + assert_eq!(cursor.key().copied(), Some(Nibbles::from_nibbles([0x2]))); cursor.advance().unwrap(); - assert_eq!(cursor.key().cloned(), Some(Nibbles::from_nibbles([0x2, 0x1]))); + assert_eq!(cursor.key().copied(), Some(Nibbles::from_nibbles([0x2, 0x1]))); cursor.advance().unwrap(); - assert_eq!(cursor.key().cloned(), Some(Nibbles::from_nibbles([0x4]))); + assert_eq!(cursor.key().copied(), Some(Nibbles::from_nibbles([0x4]))); cursor.advance().unwrap(); - assert_eq!(cursor.key().cloned(), None); // the end of trie + assert_eq!(cursor.key().copied(), None); // the end of trie } diff --git a/crates/trie/db/tests/witness.rs b/crates/trie/db/tests/witness.rs index 5dfa1c3e4ae..14457fccc6e 100644 --- a/crates/trie/db/tests/witness.rs +++ b/crates/trie/db/tests/witness.rs @@ -41,7 +41,8 @@ fn includes_empty_node_preimage() { provider.insert_account_for_hashing([(address, Some(Account::default()))]).unwrap(); let state_root = StateRoot::from_tx(provider.tx_ref()).root().unwrap(); - let multiproof = Proof::from_tx(provider.tx_ref()) + let proof = as DatabaseProof>::from_tx(provider.tx_ref()); + let multiproof = proof .multiproof(MultiProofTargets::from_iter([( hashed_address, HashSet::from_iter([hashed_slot]), @@ -82,7 +83,8 @@ fn includes_nodes_for_destroyed_storage_nodes() { .unwrap(); let state_root = StateRoot::from_tx(provider.tx_ref()).root().unwrap(); - let multiproof = Proof::from_tx(provider.tx_ref()) + let proof = as DatabaseProof>::from_tx(provider.tx_ref()); + let multiproof = proof .multiproof(MultiProofTargets::from_iter([( hashed_address, HashSet::from_iter([hashed_slot]), @@ -130,7 +132,8 @@ fn correctly_decodes_branch_node_values() { .unwrap(); let state_root = StateRoot::from_tx(provider.tx_ref()).root().unwrap(); - let multiproof = Proof::from_tx(provider.tx_ref()) + let proof = as DatabaseProof>::from_tx(provider.tx_ref()); + let multiproof = proof .multiproof(MultiProofTargets::from_iter([( hashed_address, HashSet::from_iter([hashed_slot1, hashed_slot2]), diff --git a/crates/trie/parallel/Cargo.toml b/crates/trie/parallel/Cargo.toml index 3ee2c8b653d..9fb882b44a5 100644 --- a/crates/trie/parallel/Cargo.toml +++ b/crates/trie/parallel/Cargo.toml @@ -13,12 +13,10 @@ workspace = true [dependencies] # reth -reth-db-api.workspace = true reth-execution-errors.workspace = true reth-provider.workspace = true reth-storage-errors.workspace = true reth-trie-common.workspace = true -reth-trie-db.workspace = true reth-trie-sparse = { workspace = true, features = ["std"] } reth-trie.workspace = true @@ -30,11 +28,13 @@ alloy-primitives.workspace = true tracing.workspace = true # misc +dashmap.workspace = true thiserror.workspace = true derive_more.workspace = true rayon.workspace = true itertools.workspace = true -tokio = { workspace = true, features = ["rt"] } +tokio = { workspace = true, features = ["rt-multi-thread"] } +crossbeam-channel.workspace = true # `metrics` feature reth-metrics = { workspace = true, optional = true } @@ -44,11 +44,11 @@ metrics = { workspace = true, optional = true } # reth reth-primitives-traits.workspace = true reth-provider = { workspace = true, features = ["test-utils"] } +reth-trie-db.workspace = true reth-trie = { workspace = true, features = ["test-utils"] } # misc rand.workspace = true -rayon.workspace = true criterion.workspace = true proptest.workspace = true proptest-arbitrary-interop.workspace = true @@ -58,7 +58,6 @@ tokio = { workspace = true, features = ["rt", "rt-multi-thread", "macros"] } default = ["metrics"] metrics = ["reth-metrics", "dep:metrics", "reth-trie/metrics", "reth-trie-sparse/metrics"] test-utils = [ - "reth-db-api/test-utils", "reth-primitives-traits/test-utils", "reth-provider/test-utils", "reth-trie-common/test-utils", diff --git a/crates/trie/parallel/benches/root.rs b/crates/trie/parallel/benches/root.rs index fe1953b9055..53719892748 100644 --- a/crates/trie/parallel/benches/root.rs +++ b/crates/trie/parallel/benches/root.rs @@ -5,7 +5,8 @@ use proptest::{prelude::*, strategy::ValueTree, test_runner::TestRunner}; use proptest_arbitrary_interop::arb; use reth_primitives_traits::Account; use reth_provider::{ - providers::ConsistentDbView, test_utils::create_test_provider_factory, StateWriter, TrieWriter, + providers::OverlayStateProviderFactory, test_utils::create_test_provider_factory, StateWriter, + TrieWriter, }; use reth_trie::{ hashed_cursor::HashedPostStateCursorFactory, HashedPostState, HashedStorage, StateRoot, @@ -33,11 +34,11 @@ pub fn calculate_state_root(c: &mut Criterion) { provider_rw.write_hashed_state(&db_state.into_sorted()).unwrap(); let (_, updates) = StateRoot::from_tx(provider_rw.tx_ref()).root_with_updates().unwrap(); - provider_rw.write_trie_updates(&updates).unwrap(); + provider_rw.write_trie_updates(updates).unwrap(); provider_rw.commit().unwrap(); } - let view = ConsistentDbView::new(provider_factory.clone(), None); + let factory = OverlayStateProviderFactory::new(provider_factory.clone()); // state root group.bench_function(BenchmarkId::new("sync root", size), |b| { @@ -65,10 +66,8 @@ pub fn calculate_state_root(c: &mut Criterion) { group.bench_function(BenchmarkId::new("parallel root", size), |b| { b.iter_with_setup( || { - ParallelStateRoot::new( - view.clone(), - TrieInput::from_state(updated_state.clone()), - ) + let trie_input = TrieInput::from_state(updated_state.clone()); + ParallelStateRoot::new(factory.clone(), trie_input.prefix_sets.freeze()) }, |calculator| calculator.incremental_root(), ); diff --git a/crates/trie/parallel/src/lib.rs b/crates/trie/parallel/src/lib.rs index e3138d19a30..d713ce1520e 100644 --- a/crates/trie/parallel/src/lib.rs +++ b/crates/trie/parallel/src/lib.rs @@ -5,7 +5,7 @@ html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256", issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(test), warn(unused_crate_dependencies))] mod storage_root_targets; @@ -25,3 +25,7 @@ pub mod proof_task; /// Parallel state root metrics. #[cfg(feature = "metrics")] pub mod metrics; + +/// Proof task manager metrics. +#[cfg(feature = "metrics")] +pub mod proof_task_metrics; diff --git a/crates/trie/parallel/src/proof.rs b/crates/trie/parallel/src/proof.rs index f345919e129..433c13fb08f 100644 --- a/crates/trie/parallel/src/proof.rs +++ b/crates/trie/parallel/src/proof.rs @@ -1,79 +1,59 @@ use crate::{ metrics::ParallelTrieMetrics, - proof_task::{ProofTaskKind, ProofTaskManagerHandle, StorageProofInput}, + proof_task::{ + AccountMultiproofInput, ProofResultContext, ProofResultMessage, ProofWorkerHandle, + StorageProofInput, + }, root::ParallelStateRootError, - stats::ParallelTrieTracker, StorageRootTargets, }; -use alloy_primitives::{ - map::{B256Map, B256Set, HashMap}, - B256, -}; -use alloy_rlp::{BufMut, Encodable}; -use itertools::Itertools; +use alloy_primitives::{map::B256Set, B256}; +use crossbeam_channel::{unbounded as crossbeam_unbounded, Receiver as CrossbeamReceiver}; +use dashmap::DashMap; use reth_execution_errors::StorageRootError; -use reth_provider::{ - providers::ConsistentDbView, BlockReader, DBProvider, DatabaseProviderFactory, FactoryTx, - ProviderError, StateCommitmentProvider, -}; use reth_storage_errors::db::DatabaseError; use reth_trie::{ - hashed_cursor::{HashedCursorFactory, HashedPostStateCursorFactory}, - node_iter::{TrieElement, TrieNodeIter}, - prefix_set::{PrefixSet, PrefixSetMut, TriePrefixSetsMut}, - proof::StorageProof, - trie_cursor::{InMemoryTrieCursorFactory, TrieCursorFactory}, - updates::TrieUpdatesSorted, - walker::TrieWalker, - HashBuilder, HashedPostStateSorted, MultiProof, MultiProofTargets, Nibbles, StorageMultiProof, - TRIE_ACCOUNT_RLP_MAX_SIZE, + prefix_set::{PrefixSet, PrefixSetMut, TriePrefixSets, TriePrefixSetsMut}, + DecodedMultiProof, DecodedStorageMultiProof, HashedPostState, MultiProofTargets, Nibbles, }; -use reth_trie_common::proof::ProofRetainer; -use reth_trie_db::{DatabaseHashedCursorFactory, DatabaseTrieCursorFactory}; -use std::sync::{mpsc::Receiver, Arc}; -use tracing::debug; +use reth_trie_common::added_removed_keys::MultiAddedRemovedKeys; +use std::{sync::Arc, time::Instant}; +use tracing::trace; /// Parallel proof calculator. /// /// This can collect proof for many targets in parallel, spawning a task for each hashed address /// that has proof targets. #[derive(Debug)] -pub struct ParallelProof { - /// Consistent view of the database. - view: ConsistentDbView, - /// The sorted collection of cached in-memory intermediate trie nodes that - /// can be reused for computation. - pub nodes_sorted: Arc, - /// The sorted in-memory overlay hashed state. - pub state_sorted: Arc, - /// The collection of prefix sets for the computation. Since the prefix sets _always_ - /// invalidate the in-memory nodes, not all keys from `state_sorted` might be present here, - /// if we have cached nodes for them. +pub struct ParallelProof { + /// The collection of prefix sets for the computation. pub prefix_sets: Arc, /// Flag indicating whether to include branch node masks in the proof. collect_branch_node_masks: bool, - /// Handle to the storage proof task. - storage_proof_task_handle: ProofTaskManagerHandle>, + /// Provided by the user to give the necessary context to retain extra proofs. + multi_added_removed_keys: Option>, + /// Handle to the proof worker pools. + proof_worker_handle: ProofWorkerHandle, + /// Cached storage proof roots for missed leaves; this maps + /// hashed (missed) addresses to their storage proof roots. + missed_leaves_storage_roots: Arc>, #[cfg(feature = "metrics")] metrics: ParallelTrieMetrics, } -impl ParallelProof { +impl ParallelProof { /// Create new state proof generator. pub fn new( - view: ConsistentDbView, - nodes_sorted: Arc, - state_sorted: Arc, prefix_sets: Arc, - storage_proof_task_handle: ProofTaskManagerHandle>, + missed_leaves_storage_roots: Arc>, + proof_worker_handle: ProofWorkerHandle, ) -> Self { Self { - view, - nodes_sorted, - state_sorted, prefix_sets, + missed_leaves_storage_roots, collect_branch_node_masks: false, - storage_proof_task_handle, + multi_added_removed_keys: None, + proof_worker_handle, #[cfg(feature = "metrics")] metrics: ParallelTrieMetrics::new_with_labels(&[("type", "proof")]), } @@ -84,31 +64,42 @@ impl ParallelProof { self.collect_branch_node_masks = branch_node_masks; self } -} -impl ParallelProof -where - Factory: - DatabaseProviderFactory + StateCommitmentProvider + Clone + 'static, -{ - /// Spawns a storage proof on the storage proof task and returns a receiver for the result. - fn spawn_storage_proof( + /// Configure the `ParallelProof` with a [`MultiAddedRemovedKeys`], allowing for retaining + /// extra proofs needed to add and remove leaf nodes from the tries. + pub fn with_multi_added_removed_keys( + mut self, + multi_added_removed_keys: Option>, + ) -> Self { + self.multi_added_removed_keys = multi_added_removed_keys; + self + } + /// Queues a storage proof task and returns a receiver for the result. + fn send_storage_proof( &self, hashed_address: B256, prefix_set: PrefixSet, target_slots: B256Set, - ) -> Receiver> { + ) -> Result, ParallelStateRootError> { + let (result_tx, result_rx) = crossbeam_channel::unbounded(); + let start = Instant::now(); + let input = StorageProofInput::new( hashed_address, prefix_set, target_slots, self.collect_branch_node_masks, + self.multi_added_removed_keys.clone(), ); - let (sender, receiver) = std::sync::mpsc::channel(); - let _ = - self.storage_proof_task_handle.queue_task(ProofTaskKind::StorageProof(input, sender)); - receiver + self.proof_worker_handle + .dispatch_storage_proof( + input, + ProofResultContext::new(result_tx, 0, HashedPostState::default(), start), + ) + .map_err(|e| ParallelStateRootError::Other(e.to_string()))?; + + Ok(result_rx) } /// Generate a storage multiproof according to the specified targets and hashed address. @@ -116,45 +107,60 @@ where self, hashed_address: B256, target_slots: B256Set, - ) -> Result { + ) -> Result { let total_targets = target_slots.len(); let prefix_set = PrefixSetMut::from(target_slots.iter().map(Nibbles::unpack)); let prefix_set = prefix_set.freeze(); - debug!( + trace!( target: "trie::parallel_proof", total_targets, ?hashed_address, "Starting storage proof generation" ); - let receiver = self.spawn_storage_proof(hashed_address, prefix_set, target_slots); - let proof_result = receiver.recv().map_err(|_| { + let receiver = self.send_storage_proof(hashed_address, prefix_set, target_slots)?; + let proof_msg = receiver.recv().map_err(|_| { ParallelStateRootError::StorageRoot(StorageRootError::Database(DatabaseError::Other( format!("channel closed for {hashed_address}"), ))) })?; - debug!( + // Extract storage proof directly from the result + let storage_proof = match proof_msg.result? { + crate::proof_task::ProofResult::StorageProof { hashed_address: addr, proof } => { + debug_assert_eq!( + addr, + hashed_address, + "storage worker must return same address: expected {hashed_address}, got {addr}" + ); + proof + } + crate::proof_task::ProofResult::AccountMultiproof { .. } => { + unreachable!("storage worker only sends StorageProof variant") + } + }; + + trace!( target: "trie::parallel_proof", total_targets, ?hashed_address, "Storage proof generation completed" ); - proof_result + Ok(storage_proof) } - /// Generate a state multiproof according to specified targets. - pub fn multiproof( - self, - targets: MultiProofTargets, - ) -> Result { - let mut tracker = ParallelTrieTracker::default(); - - // Extend prefix sets with targets - let mut prefix_sets = (*self.prefix_sets).clone(); - prefix_sets.extend(TriePrefixSetsMut { + /// Extends prefix sets with the given multiproof targets and returns the frozen result. + /// + /// This is a helper function used to prepare prefix sets before computing multiproofs. + /// Returns frozen (immutable) prefix sets ready for use in proof computation. + pub fn extend_prefix_sets_with_targets( + base_prefix_sets: &TriePrefixSetsMut, + targets: &MultiProofTargets, + ) -> TriePrefixSets { + let mut extended = base_prefix_sets.clone(); + extended.extend(TriePrefixSetsMut { account_prefix_set: PrefixSetMut::from(targets.keys().copied().map(Nibbles::unpack)), storage_prefix_sets: targets .iter() @@ -165,146 +171,69 @@ where .collect(), destroyed_accounts: Default::default(), }); - let prefix_sets = prefix_sets.freeze(); + extended.freeze() + } + + /// Generate a state multiproof according to specified targets. + pub fn decoded_multiproof( + self, + targets: MultiProofTargets, + ) -> Result { + // Extend prefix sets with targets + let prefix_sets = Self::extend_prefix_sets_with_targets(&self.prefix_sets, &targets); - let storage_root_targets = StorageRootTargets::new( - prefix_sets.account_prefix_set.iter().map(|nibbles| B256::from_slice(&nibbles.pack())), - prefix_sets.storage_prefix_sets.clone(), + let storage_root_targets_len = StorageRootTargets::count( + &prefix_sets.account_prefix_set, + &prefix_sets.storage_prefix_sets, ); - let storage_root_targets_len = storage_root_targets.len(); - debug!( + trace!( target: "trie::parallel_proof", total_targets = storage_root_targets_len, "Starting parallel proof generation" ); - // Pre-calculate storage roots for accounts which were changed. - tracker.set_precomputed_storage_roots(storage_root_targets_len as u64); - - // stores the receiver for the storage proof outcome for the hashed addresses - // this way we can lazily await the outcome when we iterate over the map - let mut storage_proofs = - B256Map::with_capacity_and_hasher(storage_root_targets.len(), Default::default()); + // Queue account multiproof request to account worker pool + // Create channel for receiving ProofResultMessage + let (result_tx, result_rx) = crossbeam_unbounded(); + let account_multiproof_start_time = Instant::now(); - for (hashed_address, prefix_set) in - storage_root_targets.into_iter().sorted_unstable_by_key(|(address, _)| *address) - { - let target_slots = targets.get(&hashed_address).cloned().unwrap_or_default(); - let receiver = self.spawn_storage_proof(hashed_address, prefix_set, target_slots); + let input = AccountMultiproofInput { + targets, + prefix_sets, + collect_branch_node_masks: self.collect_branch_node_masks, + multi_added_removed_keys: self.multi_added_removed_keys.clone(), + missed_leaves_storage_roots: self.missed_leaves_storage_roots.clone(), + proof_result_sender: ProofResultContext::new( + result_tx, + 0, + HashedPostState::default(), + account_multiproof_start_time, + ), + }; - // store the receiver for that result with the hashed address so we can await this in - // place when we iterate over the trie - storage_proofs.insert(hashed_address, receiver); - } + self.proof_worker_handle + .dispatch_account_multiproof(input) + .map_err(|e| ParallelStateRootError::Other(e.to_string()))?; - let provider_ro = self.view.provider_ro()?; - let trie_cursor_factory = InMemoryTrieCursorFactory::new( - DatabaseTrieCursorFactory::new(provider_ro.tx_ref()), - &self.nodes_sorted, - ); - let hashed_cursor_factory = HashedPostStateCursorFactory::new( - DatabaseHashedCursorFactory::new(provider_ro.tx_ref()), - &self.state_sorted, - ); + // Wait for account multiproof result from worker + let proof_result_msg = result_rx.recv().map_err(|_| { + ParallelStateRootError::Other( + "Account multiproof channel dropped: worker died or pool shutdown".to_string(), + ) + })?; - // Create the walker. - let walker = TrieWalker::state_trie( - trie_cursor_factory.account_trie_cursor().map_err(ProviderError::Database)?, - prefix_sets.account_prefix_set, - ) - .with_deletions_retained(true); - - // Create a hash builder to rebuild the root node since it is not available in the database. - let retainer: ProofRetainer = targets.keys().map(Nibbles::unpack).collect(); - let mut hash_builder = HashBuilder::default() - .with_proof_retainer(retainer) - .with_updates(self.collect_branch_node_masks); - - // Initialize all storage multiproofs as empty. - // Storage multiproofs for non empty tries will be overwritten if necessary. - let mut storages: B256Map<_> = - targets.keys().map(|key| (*key, StorageMultiProof::empty())).collect(); - let mut account_rlp = Vec::with_capacity(TRIE_ACCOUNT_RLP_MAX_SIZE); - let mut account_node_iter = TrieNodeIter::state_trie( - walker, - hashed_cursor_factory.hashed_account_cursor().map_err(ProviderError::Database)?, - ); - while let Some(account_node) = - account_node_iter.try_next().map_err(ProviderError::Database)? - { - match account_node { - TrieElement::Branch(node) => { - hash_builder.add_branch(node.key, node.value, node.children_are_in_trie); - } - TrieElement::Leaf(hashed_address, account) => { - let storage_multiproof = match storage_proofs.remove(&hashed_address) { - Some(rx) => rx.recv().map_err(|_| { - ParallelStateRootError::StorageRoot(StorageRootError::Database( - DatabaseError::Other(format!( - "channel closed for {hashed_address}" - )), - )) - })??, - // Since we do not store all intermediate nodes in the database, there might - // be a possibility of re-adding a non-modified leaf to the hash builder. - None => { - tracker.inc_missed_leaves(); - StorageProof::new_hashed( - trie_cursor_factory.clone(), - hashed_cursor_factory.clone(), - hashed_address, - ) - .with_prefix_set_mut(Default::default()) - .storage_multiproof( - targets.get(&hashed_address).cloned().unwrap_or_default(), - ) - .map_err(|e| { - ParallelStateRootError::StorageRoot(StorageRootError::Database( - DatabaseError::Other(e.to_string()), - )) - })? - } - }; - - // Encode account - account_rlp.clear(); - let account = account.into_trie_account(storage_multiproof.root); - account.encode(&mut account_rlp as &mut dyn BufMut); - - hash_builder.add_leaf(Nibbles::unpack(hashed_address), &account_rlp); - - // We might be adding leaves that are not necessarily our proof targets. - if targets.contains_key(&hashed_address) { - storages.insert(hashed_address, storage_multiproof); - } - } + let (multiproof, stats) = match proof_result_msg.result? { + crate::proof_task::ProofResult::AccountMultiproof { proof, stats } => (proof, stats), + crate::proof_task::ProofResult::StorageProof { .. } => { + unreachable!("account worker only sends AccountMultiproof variant") } - } - let _ = hash_builder.root(); + }; - let stats = tracker.finish(); #[cfg(feature = "metrics")] self.metrics.record(stats); - let account_subtree = hash_builder.take_proof_nodes(); - let (branch_node_hash_masks, branch_node_tree_masks) = if self.collect_branch_node_masks { - let updated_branch_nodes = hash_builder.updated_branch_nodes.unwrap_or_default(); - ( - updated_branch_nodes - .iter() - .map(|(path, node)| (path.clone(), node.hash_mask)) - .collect(), - updated_branch_nodes - .into_iter() - .map(|(path, node)| (path, node.tree_mask)) - .collect(), - ) - } else { - (HashMap::default(), HashMap::default()) - }; - - debug!( + trace!( target: "trie::parallel_proof", total_targets = storage_root_targets_len, duration = ?stats.duration(), @@ -312,32 +241,32 @@ where leaves_added = stats.leaves_added(), missed_leaves = stats.missed_leaves(), precomputed_storage_roots = stats.precomputed_storage_roots(), - "Calculated proof" + "Calculated decoded proof" ); - Ok(MultiProof { account_subtree, branch_node_hash_masks, branch_node_tree_masks, storages }) + Ok(multiproof) } } #[cfg(test)] mod tests { use super::*; - use crate::proof_task::{ProofTaskCtx, ProofTaskManager}; + use crate::proof_task::{ProofTaskCtx, ProofWorkerHandle}; use alloy_primitives::{ keccak256, - map::{B256Set, DefaultHashBuilder}, + map::{B256Set, DefaultHashBuilder, HashMap}, Address, U256, }; use rand::Rng; use reth_primitives_traits::{Account, StorageEntry}; use reth_provider::{test_utils::create_test_provider_factory, HashingWriter}; use reth_trie::proof::Proof; + use reth_trie_db::{DatabaseHashedCursorFactory, DatabaseTrieCursorFactory}; use tokio::runtime::Runtime; #[test] fn random_parallel_proof() { let factory = create_test_provider_factory(); - let consistent_view = ConsistentDbView::new(factory.clone(), None); let mut rng = rand::rng(); let state = (0..100) @@ -399,47 +328,39 @@ mod tests { let rt = Runtime::new().unwrap(); - let task_ctx = - ProofTaskCtx::new(Default::default(), Default::default(), Default::default()); - let proof_task = - ProofTaskManager::new(rt.handle().clone(), consistent_view.clone(), task_ctx, 1); - let proof_task_handle = proof_task.handle(); - - // keep the join handle around to make sure it does not return any errors - // after we compute the state root - let join_handle = rt.spawn_blocking(move || proof_task.run()); - - let parallel_result = ParallelProof::new( - consistent_view, - Default::default(), - Default::default(), - Default::default(), - proof_task_handle.clone(), - ) - .multiproof(targets.clone()) - .unwrap(); - - let sequential_result = - Proof::new(trie_cursor_factory, hashed_cursor_factory).multiproof(targets).unwrap(); + let factory = reth_provider::providers::OverlayStateProviderFactory::new(factory); + let task_ctx = ProofTaskCtx::new(factory); + let proof_worker_handle = ProofWorkerHandle::new(rt.handle().clone(), task_ctx, 1, 1); + + let parallel_result = + ParallelProof::new(Default::default(), Default::default(), proof_worker_handle.clone()) + .decoded_multiproof(targets.clone()) + .unwrap(); + + let sequential_result_raw = Proof::new(trie_cursor_factory, hashed_cursor_factory) + .multiproof(targets.clone()) + .unwrap(); // targets might be consumed by parallel_result + let sequential_result_decoded: DecodedMultiProof = sequential_result_raw + .try_into() + .expect("Failed to decode sequential_result for test comparison"); // to help narrow down what is wrong - first compare account subtries - assert_eq!(parallel_result.account_subtree, sequential_result.account_subtree); + assert_eq!(parallel_result.account_subtree, sequential_result_decoded.account_subtree); // then compare length of all storage subtries - assert_eq!(parallel_result.storages.len(), sequential_result.storages.len()); + assert_eq!(parallel_result.storages.len(), sequential_result_decoded.storages.len()); // then compare each storage subtrie for (hashed_address, storage_proof) in ¶llel_result.storages { - let sequential_storage_proof = sequential_result.storages.get(hashed_address).unwrap(); + let sequential_storage_proof = + sequential_result_decoded.storages.get(hashed_address).unwrap(); assert_eq!(storage_proof, sequential_storage_proof); } // then compare the entire thing for any mask differences - assert_eq!(parallel_result, sequential_result); + assert_eq!(parallel_result, sequential_result_decoded); - // drop the handle to terminate the task and then block on the proof task handle to make - // sure it does not return any errors - drop(proof_task_handle); - rt.block_on(join_handle).unwrap().expect("The proof task should not return an error"); + // Workers shut down automatically when handle is dropped + drop(proof_worker_handle); } } diff --git a/crates/trie/parallel/src/proof_task.rs b/crates/trie/parallel/src/proof_task.rs index 516d92c4daa..8da4c28d91a 100644 --- a/crates/trie/parallel/src/proof_task.rs +++ b/crates/trie/parallel/src/proof_task.rs @@ -1,375 +1,1392 @@ -//! A Task that manages sending proof requests to a number of tasks that have longer-running -//! database transactions. +//! Parallel proof computation using worker pools with dedicated database transactions. //! -//! The [`ProofTaskManager`] ensures that there are a max number of currently executing proof tasks, -//! and is responsible for managing the fixed number of database transactions created at the start -//! of the task. //! -//! Individual [`ProofTaskTx`] instances manage a dedicated [`InMemoryTrieCursorFactory`] and -//! [`HashedPostStateCursorFactory`], which are each backed by a database transaction. - -use crate::root::ParallelStateRootError; -use alloy_primitives::{map::B256Set, B256}; -use reth_db_api::transaction::DbTx; -use reth_execution_errors::SparseTrieError; -use reth_provider::{ - providers::ConsistentDbView, BlockReader, DBProvider, DatabaseProviderFactory, FactoryTx, - ProviderResult, StateCommitmentProvider, +//! # Architecture +//! +//! - **Worker Pools**: Pre-spawned workers with dedicated database transactions +//! - Storage pool: Handles storage proofs and blinded storage node requests +//! - Account pool: Handles account multiproofs and blinded account node requests +//! - **Direct Channel Access**: [`ProofWorkerHandle`] provides type-safe queue methods with direct +//! access to worker channels, eliminating routing overhead +//! - **Automatic Shutdown**: Workers terminate gracefully when all handles are dropped +//! +//! # Message Flow +//! +//! 1. `MultiProofTask` prepares a storage or account job and hands it to [`ProofWorkerHandle`]. The +//! job carries a [`ProofResultContext`] so the worker knows how to send the result back. +//! 2. A worker receives the job, runs the proof, and sends a [`ProofResultMessage`] through the +//! provided [`ProofResultSender`]. +//! 3. `MultiProofTask` receives the message, uses `sequence_number` to keep proofs in order, and +//! proceeds with its state-root logic. +//! +//! Each job gets its own direct channel so results go straight back to `MultiProofTask`. That keeps +//! ordering decisions in one place and lets workers run independently. +//! +//! ```text +//! MultiProofTask -> MultiproofManager -> ProofWorkerHandle -> Storage/Account Worker +//! ^ | +//! | v +//! ProofResultMessage <-------- ProofResultSender --- +//! ``` + +use crate::{ + root::ParallelStateRootError, + stats::{ParallelTrieStats, ParallelTrieTracker}, + StorageRootTargets, +}; +use alloy_primitives::{ + map::{B256Map, B256Set}, + B256, }; +use alloy_rlp::{BufMut, Encodable}; +use crossbeam_channel::{unbounded, Receiver as CrossbeamReceiver, Sender as CrossbeamSender}; +use dashmap::DashMap; +use reth_execution_errors::{SparseTrieError, SparseTrieErrorKind}; +use reth_provider::{DatabaseProviderROFactory, ProviderError, ProviderResult}; +use reth_storage_errors::db::DatabaseError; use reth_trie::{ - hashed_cursor::HashedPostStateCursorFactory, - prefix_set::TriePrefixSetsMut, - proof::{ProofBlindedProviderFactory, StorageProof}, - trie_cursor::InMemoryTrieCursorFactory, - updates::TrieUpdatesSorted, - HashedPostStateSorted, Nibbles, StorageMultiProof, + hashed_cursor::HashedCursorFactory, + node_iter::{TrieElement, TrieNodeIter}, + prefix_set::TriePrefixSets, + proof::{ProofBlindedAccountProvider, ProofBlindedStorageProvider, StorageProof}, + trie_cursor::TrieCursorFactory, + walker::TrieWalker, + DecodedMultiProof, DecodedStorageMultiProof, HashBuilder, HashedPostState, MultiProofTargets, + Nibbles, TRIE_ACCOUNT_RLP_MAX_SIZE, }; -use reth_trie_common::prefix_set::{PrefixSet, PrefixSetMut}; -use reth_trie_db::{DatabaseHashedCursorFactory, DatabaseTrieCursorFactory}; -use reth_trie_sparse::blinded::{BlindedProvider, BlindedProviderFactory, RevealedNode}; +use reth_trie_common::{ + added_removed_keys::MultiAddedRemovedKeys, + prefix_set::{PrefixSet, PrefixSetMut}, + proof::{DecodedProofNodes, ProofRetainer}, +}; +use reth_trie_sparse::provider::{RevealedNode, TrieNodeProvider, TrieNodeProviderFactory}; use std::{ - collections::VecDeque, sync::{ atomic::{AtomicUsize, Ordering}, - mpsc::{channel, Receiver, SendError, Sender}, + mpsc::{channel, Receiver, Sender}, Arc, }, - time::Instant, + time::{Duration, Instant}, }; use tokio::runtime::Handle; -use tracing::debug; +use tracing::{debug, debug_span, error, trace}; -type StorageProofResult = Result; -type BlindedNodeResult = Result, SparseTrieError>; +#[cfg(feature = "metrics")] +use crate::proof_task_metrics::ProofTaskTrieMetrics; -/// A task that manages sending multiproof requests to a number of tasks that have longer-running -/// database transactions -#[derive(Debug)] -pub struct ProofTaskManager { - /// Max number of database transactions to create - max_concurrency: usize, - /// Number of database transactions created - total_transactions: usize, - /// Consistent view provider used for creating transactions on-demand - view: ConsistentDbView, - /// Proof task context shared across all proof tasks - task_ctx: ProofTaskCtx, - /// Proof tasks pending execution - pending_tasks: VecDeque, - /// The underlying handle from which to spawn proof tasks - executor: Handle, - /// The proof task transactions, containing owned cursor factories that are reused for proof - /// calculation. - proof_task_txs: Vec>>, - /// A receiver for new proof tasks. - proof_task_rx: Receiver>>, - /// A sender for sending back transactions. - tx_sender: Sender>>, - /// The number of active handles. - /// - /// Incremented in [`ProofTaskManagerHandle::new`] and decremented in - /// [`ProofTaskManagerHandle::drop`]. - active_handles: Arc, +type StorageProofResult = Result; +type TrieNodeProviderResult = Result, SparseTrieError>; + +/// A handle that provides type-safe access to proof worker pools. +/// +/// The handle stores direct senders to both storage and account worker pools, +/// eliminating the need for a routing thread. All handles share reference-counted +/// channels, and workers shut down gracefully when all handles are dropped. +#[derive(Debug, Clone)] +pub struct ProofWorkerHandle { + /// Direct sender to storage worker pool + storage_work_tx: CrossbeamSender, + /// Direct sender to account worker pool + account_work_tx: CrossbeamSender, + /// Counter tracking available storage workers. Workers decrement when starting work, + /// increment when finishing. Used to determine whether to chunk multiproofs. + storage_available_workers: Arc, + /// Counter tracking available account workers. Workers decrement when starting work, + /// increment when finishing. Used to determine whether to chunk multiproofs. + account_available_workers: Arc, + /// Total number of storage workers spawned + storage_worker_count: usize, + /// Total number of account workers spawned + account_worker_count: usize, } -impl ProofTaskManager { - /// Creates a new [`ProofTaskManager`] with the given max concurrency, creating that number of - /// cursor factories. +impl ProofWorkerHandle { + /// Spawns storage and account worker pools with dedicated database transactions. /// - /// Returns an error if the consistent view provider fails to create a read-only transaction. - pub fn new( + /// Returns a handle for submitting proof tasks to the worker pools. + /// Workers run until the last handle is dropped. + /// + /// # Parameters + /// - `executor`: Tokio runtime handle for spawning blocking tasks + /// - `task_ctx`: Shared context with database view and prefix sets + /// - `storage_worker_count`: Number of storage workers to spawn + /// - `account_worker_count`: Number of account workers to spawn + pub fn new( executor: Handle, - view: ConsistentDbView, - task_ctx: ProofTaskCtx, - max_concurrency: usize, - ) -> Self { - let (tx_sender, proof_task_rx) = channel(); + task_ctx: ProofTaskCtx, + storage_worker_count: usize, + account_worker_count: usize, + ) -> Self + where + Factory: DatabaseProviderROFactory + + Clone + + Send + + 'static, + { + let (storage_work_tx, storage_work_rx) = unbounded::(); + let (account_work_tx, account_work_rx) = unbounded::(); + + // Initialize availability counters at zero. Each worker will increment when it + // successfully initializes, ensuring only healthy workers are counted. + let storage_available_workers = Arc::new(AtomicUsize::new(0)); + let account_available_workers = Arc::new(AtomicUsize::new(0)); + + debug!( + target: "trie::proof_task", + storage_worker_count, + account_worker_count, + "Spawning proof worker pools" + ); + + let parent_span = + debug_span!(target: "trie::proof_task", "storage proof workers", ?storage_worker_count) + .entered(); + // Spawn storage workers + for worker_id in 0..storage_worker_count { + let span = debug_span!(target: "trie::proof_task", "storage worker", ?worker_id); + let task_ctx_clone = task_ctx.clone(); + let work_rx_clone = storage_work_rx.clone(); + let storage_available_workers_clone = storage_available_workers.clone(); + + executor.spawn_blocking(move || { + #[cfg(feature = "metrics")] + let metrics = ProofTaskTrieMetrics::default(); + + let _guard = span.enter(); + let worker = StorageProofWorker::new( + task_ctx_clone, + work_rx_clone, + worker_id, + storage_available_workers_clone, + #[cfg(feature = "metrics")] + metrics, + ); + if let Err(error) = worker.run() { + error!( + target: "trie::proof_task", + worker_id, + ?error, + "Storage worker failed" + ); + } + }); + } + drop(parent_span); + + let parent_span = + debug_span!(target: "trie::proof_task", "account proof workers", ?storage_worker_count) + .entered(); + // Spawn account workers + for worker_id in 0..account_worker_count { + let span = debug_span!(target: "trie::proof_task", "account worker", ?worker_id); + let task_ctx_clone = task_ctx.clone(); + let work_rx_clone = account_work_rx.clone(); + let storage_work_tx_clone = storage_work_tx.clone(); + let account_available_workers_clone = account_available_workers.clone(); + + executor.spawn_blocking(move || { + #[cfg(feature = "metrics")] + let metrics = ProofTaskTrieMetrics::default(); + + let _guard = span.enter(); + let worker = AccountProofWorker::new( + task_ctx_clone, + work_rx_clone, + worker_id, + storage_work_tx_clone, + account_available_workers_clone, + #[cfg(feature = "metrics")] + metrics, + ); + if let Err(error) = worker.run() { + error!( + target: "trie::proof_task", + worker_id, + ?error, + "Account worker failed" + ); + } + }); + } + drop(parent_span); + Self { - max_concurrency, - total_transactions: 0, - view, - task_ctx, - pending_tasks: VecDeque::new(), - executor, - proof_task_txs: Vec::new(), - proof_task_rx, - tx_sender, - active_handles: Arc::new(AtomicUsize::new(0)), + storage_work_tx, + account_work_tx, + storage_available_workers, + account_available_workers, + storage_worker_count, + account_worker_count, } } - /// Returns a handle for sending new proof tasks to the [`ProofTaskManager`]. - pub fn handle(&self) -> ProofTaskManagerHandle> { - ProofTaskManagerHandle::new(self.tx_sender.clone(), self.active_handles.clone()) + /// Returns how many storage workers are currently available/idle. + pub fn available_storage_workers(&self) -> usize { + self.storage_available_workers.load(Ordering::Relaxed) + } + + /// Returns how many account workers are currently available/idle. + pub fn available_account_workers(&self) -> usize { + self.account_available_workers.load(Ordering::Relaxed) + } + + /// Returns the number of pending storage tasks in the queue. + pub fn pending_storage_tasks(&self) -> usize { + self.storage_work_tx.len() + } + + /// Returns the number of pending account tasks in the queue. + pub fn pending_account_tasks(&self) -> usize { + self.account_work_tx.len() + } + + /// Returns the total number of storage workers in the pool. + pub const fn total_storage_workers(&self) -> usize { + self.storage_worker_count + } + + /// Returns the total number of account workers in the pool. + pub const fn total_account_workers(&self) -> usize { + self.account_worker_count + } + + /// Returns the number of storage workers currently processing tasks. + /// + /// This is calculated as total workers minus available workers. + pub fn active_storage_workers(&self) -> usize { + self.storage_worker_count.saturating_sub(self.available_storage_workers()) + } + + /// Returns the number of account workers currently processing tasks. + /// + /// This is calculated as total workers minus available workers. + pub fn active_account_workers(&self) -> usize { + self.account_worker_count.saturating_sub(self.available_account_workers()) } + + /// Dispatch a storage proof computation to storage worker pool + /// + /// The result will be sent via the `proof_result_sender` channel. + pub fn dispatch_storage_proof( + &self, + input: StorageProofInput, + proof_result_sender: ProofResultContext, + ) -> Result<(), ProviderError> { + self.storage_work_tx + .send(StorageWorkerJob::StorageProof { input, proof_result_sender }) + .map_err(|err| { + let error = + ProviderError::other(std::io::Error::other("storage workers unavailable")); + + if let StorageWorkerJob::StorageProof { proof_result_sender, .. } = err.0 { + let ProofResultContext { + sender: result_tx, + sequence_number: seq, + state, + start_time: start, + } = proof_result_sender; + + let _ = result_tx.send(ProofResultMessage { + sequence_number: seq, + result: Err(ParallelStateRootError::Provider(error.clone())), + elapsed: start.elapsed(), + state, + }); + } + + error + }) + } + + /// Dispatch an account multiproof computation + /// + /// The result will be sent via the `result_sender` channel included in the input. + pub fn dispatch_account_multiproof( + &self, + input: AccountMultiproofInput, + ) -> Result<(), ProviderError> { + self.account_work_tx + .send(AccountWorkerJob::AccountMultiproof { input: Box::new(input) }) + .map_err(|err| { + let error = + ProviderError::other(std::io::Error::other("account workers unavailable")); + + if let AccountWorkerJob::AccountMultiproof { input } = err.0 { + let AccountMultiproofInput { + proof_result_sender: + ProofResultContext { + sender: result_tx, + sequence_number: seq, + state, + start_time: start, + }, + .. + } = *input; + + let _ = result_tx.send(ProofResultMessage { + sequence_number: seq, + result: Err(ParallelStateRootError::Provider(error.clone())), + elapsed: start.elapsed(), + state, + }); + } + + error + }) + } + + /// Dispatch blinded storage node request to storage worker pool + pub(crate) fn dispatch_blinded_storage_node( + &self, + account: B256, + path: Nibbles, + ) -> Result, ProviderError> { + let (tx, rx) = channel(); + self.storage_work_tx + .send(StorageWorkerJob::BlindedStorageNode { account, path, result_sender: tx }) + .map_err(|_| { + ProviderError::other(std::io::Error::other("storage workers unavailable")) + })?; + + Ok(rx) + } + + /// Dispatch blinded account node request to account worker pool + pub(crate) fn dispatch_blinded_account_node( + &self, + path: Nibbles, + ) -> Result, ProviderError> { + let (tx, rx) = channel(); + self.account_work_tx + .send(AccountWorkerJob::BlindedAccountNode { path, result_sender: tx }) + .map_err(|_| { + ProviderError::other(std::io::Error::other("account workers unavailable")) + })?; + + Ok(rx) + } +} + +/// Data used for initializing cursor factories that is shared across all storage proof instances. +#[derive(Clone, Debug)] +pub struct ProofTaskCtx { + /// The factory for creating state providers. + factory: Factory, +} + +impl ProofTaskCtx { + /// Creates a new [`ProofTaskCtx`] with the given factory. + pub const fn new(factory: Factory) -> Self { + Self { factory } + } +} + +/// This contains all information shared between all storage proof instances. +#[derive(Debug)] +pub struct ProofTaskTx { + /// The provider that implements `TrieCursorFactory` and `HashedCursorFactory`. + provider: Provider, + + /// Identifier for the worker within the worker pool, used only for tracing. + id: usize, } -impl ProofTaskManager +impl ProofTaskTx { + /// Initializes a [`ProofTaskTx`] with the given provider and ID. + const fn new(provider: Provider, id: usize) -> Self { + Self { provider, id } + } +} + +impl ProofTaskTx where - Factory: DatabaseProviderFactory + StateCommitmentProvider + 'static, + Provider: TrieCursorFactory + HashedCursorFactory, { - /// Inserts the task into the pending tasks queue. - pub fn queue_proof_task(&mut self, task: ProofTaskKind) { - self.pending_tasks.push_back(task); - } + /// Compute storage proof. + /// + /// Used by storage workers in the worker pool to compute storage proofs. + #[inline] + fn compute_storage_proof(&self, input: StorageProofInput) -> StorageProofResult { + // Consume the input so we can move large collections (e.g. target slots) without cloning. + let StorageProofInput { + hashed_address, + prefix_set, + target_slots, + with_branch_node_masks, + multi_added_removed_keys, + } = input; - /// Gets either the next available transaction, or creates a new one if all are in use and the - /// total number of transactions created is less than the max concurrency. - pub fn get_or_create_tx(&mut self) -> ProviderResult>>> { - if let Some(proof_task_tx) = self.proof_task_txs.pop() { - return Ok(Some(proof_task_tx)); - } + // Get or create added/removed keys context + let multi_added_removed_keys = + multi_added_removed_keys.unwrap_or_else(|| Arc::new(MultiAddedRemovedKeys::new())); + let added_removed_keys = multi_added_removed_keys.get_storage(&hashed_address); - // if we can create a new tx within our concurrency limits, create one on-demand - if self.total_transactions < self.max_concurrency { - let provider_ro = self.view.provider_ro()?; - let tx = provider_ro.into_tx(); - self.total_transactions += 1; - return Ok(Some(ProofTaskTx::new(tx, self.task_ctx.clone()))); - } + let span = debug_span!( + target: "trie::proof_task", + "Storage proof calculation", + hashed_address = ?hashed_address, + worker_id = self.id, + ); + let _span_guard = span.enter(); + + let proof_start = Instant::now(); + + // Compute raw storage multiproof + let raw_proof_result = + StorageProof::new_hashed(&self.provider, &self.provider, hashed_address) + .with_prefix_set_mut(PrefixSetMut::from(prefix_set.iter().copied())) + .with_branch_node_masks(with_branch_node_masks) + .with_added_removed_keys(added_removed_keys) + .storage_multiproof(target_slots) + .map_err(|e| ParallelStateRootError::Other(e.to_string())); - Ok(None) + // Decode proof into DecodedStorageMultiProof + let decoded_result = raw_proof_result.and_then(|raw_proof| { + raw_proof.try_into().map_err(|e: alloy_rlp::Error| { + ParallelStateRootError::Other(format!( + "Failed to decode storage proof for {}: {}", + hashed_address, e + )) + }) + }); + + trace!( + target: "trie::proof_task", + hashed_address = ?hashed_address, + proof_time_us = proof_start.elapsed().as_micros(), + worker_id = self.id, + "Completed storage proof calculation" + ); + + decoded_result } - /// Spawns the next queued proof task on the executor with the given input, if there are any - /// transactions available. + /// Process a blinded storage node request. /// - /// This will return an error if a transaction must be created on-demand and the consistent view - /// provider fails. - pub fn try_spawn_next(&mut self) -> ProviderResult<()> { - let Some(task) = self.pending_tasks.pop_front() else { return Ok(()) }; - - let Some(proof_task_tx) = self.get_or_create_tx()? else { - // if there are no txs available, requeue the proof task - self.pending_tasks.push_front(task); - return Ok(()) - }; + /// Used by storage workers to retrieve blinded storage trie nodes for proof construction. + fn process_blinded_storage_node( + &self, + account: B256, + path: &Nibbles, + ) -> TrieNodeProviderResult { + let storage_node_provider = + ProofBlindedStorageProvider::new(&self.provider, &self.provider, account); + storage_node_provider.trie_node(path) + } - let tx_sender = self.tx_sender.clone(); - self.executor.spawn_blocking(move || match task { - ProofTaskKind::StorageProof(input, sender) => { - proof_task_tx.storage_proof(input, sender, tx_sender); - } - ProofTaskKind::BlindedAccountNode(path, sender) => { - proof_task_tx.blinded_account_node(path, sender, tx_sender); + /// Process a blinded account node request. + /// + /// Used by account workers to retrieve blinded account trie nodes for proof construction. + fn process_blinded_account_node(&self, path: &Nibbles) -> TrieNodeProviderResult { + let account_node_provider = + ProofBlindedAccountProvider::new(&self.provider, &self.provider); + account_node_provider.trie_node(path) + } +} +impl TrieNodeProviderFactory for ProofWorkerHandle { + type AccountNodeProvider = ProofTaskTrieNodeProvider; + type StorageNodeProvider = ProofTaskTrieNodeProvider; + + fn account_node_provider(&self) -> Self::AccountNodeProvider { + ProofTaskTrieNodeProvider::AccountNode { handle: self.clone() } + } + + fn storage_node_provider(&self, account: B256) -> Self::StorageNodeProvider { + ProofTaskTrieNodeProvider::StorageNode { account, handle: self.clone() } + } +} + +/// Trie node provider for retrieving trie nodes by path. +#[derive(Debug)] +pub enum ProofTaskTrieNodeProvider { + /// Blinded account trie node provider. + AccountNode { + /// Handle to the proof worker pools. + handle: ProofWorkerHandle, + }, + /// Blinded storage trie node provider. + StorageNode { + /// Target account. + account: B256, + /// Handle to the proof worker pools. + handle: ProofWorkerHandle, + }, +} + +impl TrieNodeProvider for ProofTaskTrieNodeProvider { + fn trie_node(&self, path: &Nibbles) -> Result, SparseTrieError> { + match self { + Self::AccountNode { handle } => { + let rx = handle + .dispatch_blinded_account_node(*path) + .map_err(|error| SparseTrieErrorKind::Other(Box::new(error)))?; + rx.recv().map_err(|error| SparseTrieErrorKind::Other(Box::new(error)))? } - ProofTaskKind::BlindedStorageNode(account, path, sender) => { - proof_task_tx.blinded_storage_node(account, path, sender, tx_sender); + Self::StorageNode { handle, account } => { + let rx = handle + .dispatch_blinded_storage_node(*account, *path) + .map_err(|error| SparseTrieErrorKind::Other(Box::new(error)))?; + rx.recv().map_err(|error| SparseTrieErrorKind::Other(Box::new(error)))? } - }); - - Ok(()) + } } +} +/// Result of a proof calculation, which can be either an account multiproof or a storage proof. +#[derive(Debug)] +pub enum ProofResult { + /// Account multiproof with statistics + AccountMultiproof { + /// The account multiproof + proof: DecodedMultiProof, + /// Statistics collected during proof computation + stats: ParallelTrieStats, + }, + /// Storage proof for a specific account + StorageProof { + /// The hashed address this storage proof belongs to + hashed_address: B256, + /// The storage multiproof + proof: DecodedStorageMultiProof, + }, +} - /// Loops, managing the proof tasks, and sending new tasks to the executor. - pub fn run(mut self) -> ProviderResult<()> { - loop { - match self.proof_task_rx.recv() { - Ok(message) => match message { - ProofTaskMessage::QueueTask(task) => { - // queue the task - self.queue_proof_task(task) - } - ProofTaskMessage::Transaction(tx) => { - // return the transaction to the pool - self.proof_task_txs.push(tx); - } - ProofTaskMessage::Terminate => return Ok(()), - }, - // All senders are disconnected, so we can terminate - // However this should never happen, as this struct stores a sender - Err(_) => return Ok(()), - }; - - // try spawning the next task - self.try_spawn_next()?; +impl ProofResult { + /// Convert this proof result into a `DecodedMultiProof`. + /// + /// For account multiproofs, returns the multiproof directly (discarding stats). + /// For storage proofs, wraps the storage proof into a minimal multiproof. + pub fn into_multiproof(self) -> DecodedMultiProof { + match self { + Self::AccountMultiproof { proof, stats: _ } => proof, + Self::StorageProof { hashed_address, proof } => { + DecodedMultiProof::from_storage_proof(hashed_address, proof) + } } } } +/// Channel used by worker threads to deliver `ProofResultMessage` items back to +/// `MultiProofTask`. +/// +/// Workers use this sender to deliver proof results directly to `MultiProofTask`. +pub type ProofResultSender = CrossbeamSender; -/// This contains all information shared between all storage proof instances. +/// Message containing a completed proof result with metadata for direct delivery to +/// `MultiProofTask`. +/// +/// This type enables workers to send proof results directly to the `MultiProofTask` event loop. #[derive(Debug)] -pub struct ProofTaskTx { - /// The tx that is reused for proof calculations. - tx: Tx, +pub struct ProofResultMessage { + /// Sequence number for ordering proofs + pub sequence_number: u64, + /// The proof calculation result (either account multiproof or storage proof) + pub result: Result, + /// Time taken for the entire proof calculation (from dispatch to completion) + pub elapsed: Duration, + /// Original state update that triggered this proof + pub state: HashedPostState, +} - /// Trie updates, prefix sets, and state updates - task_ctx: ProofTaskCtx, +/// Context for sending proof calculation results back to `MultiProofTask`. +/// +/// This struct contains all context needed to send and track proof calculation results. +/// Workers use this to deliver completed proofs back to the main event loop. +#[derive(Debug, Clone)] +pub struct ProofResultContext { + /// Channel sender for result delivery + pub sender: ProofResultSender, + /// Sequence number for proof ordering + pub sequence_number: u64, + /// Original state update that triggered this proof + pub state: HashedPostState, + /// Calculation start time for measuring elapsed duration + pub start_time: Instant, } -impl ProofTaskTx { - /// Initializes a [`ProofTaskTx`] using the given transaction anda[`ProofTaskCtx`]. - const fn new(tx: Tx, task_ctx: ProofTaskCtx) -> Self { - Self { tx, task_ctx } +impl ProofResultContext { + /// Creates a new proof result context. + pub const fn new( + sender: ProofResultSender, + sequence_number: u64, + state: HashedPostState, + start_time: Instant, + ) -> Self { + Self { sender, sequence_number, state, start_time } } } +/// Internal message for storage workers. +#[derive(Debug)] +enum StorageWorkerJob { + /// Storage proof computation request + StorageProof { + /// Storage proof input parameters + input: StorageProofInput, + /// Context for sending the proof result. + proof_result_sender: ProofResultContext, + }, + /// Blinded storage node retrieval request + BlindedStorageNode { + /// Target account + account: B256, + /// Path to the storage node + path: Nibbles, + /// Channel to send result back to original caller + result_sender: Sender, + }, +} + +/// Worker for storage trie operations. +/// +/// Each worker maintains a dedicated database transaction and processes +/// storage proof requests and blinded node lookups. +struct StorageProofWorker { + /// Shared task context with database factory and prefix sets + task_ctx: ProofTaskCtx, + /// Channel for receiving work + work_rx: CrossbeamReceiver, + /// Unique identifier for this worker (used for tracing) + worker_id: usize, + /// Counter tracking worker availability + available_workers: Arc, + /// Metrics collector for this worker + #[cfg(feature = "metrics")] + metrics: ProofTaskTrieMetrics, +} -impl ProofTaskTx +impl StorageProofWorker where - Tx: DbTx, + Factory: DatabaseProviderROFactory, { - fn create_factories( - &self, - ) -> ( - InMemoryTrieCursorFactory<'_, DatabaseTrieCursorFactory<'_, Tx>>, - HashedPostStateCursorFactory<'_, DatabaseHashedCursorFactory<'_, Tx>>, - ) { - let trie_cursor_factory = InMemoryTrieCursorFactory::new( - DatabaseTrieCursorFactory::new(&self.tx), - &self.task_ctx.nodes_sorted, + /// Creates a new storage proof worker. + const fn new( + task_ctx: ProofTaskCtx, + work_rx: CrossbeamReceiver, + worker_id: usize, + available_workers: Arc, + #[cfg(feature = "metrics")] metrics: ProofTaskTrieMetrics, + ) -> Self { + Self { + task_ctx, + work_rx, + worker_id, + available_workers, + #[cfg(feature = "metrics")] + metrics, + } + } + + /// Runs the worker loop, processing jobs until the channel closes. + /// + /// # Lifecycle + /// + /// 1. Initializes database provider and transaction + /// 2. Advertises availability + /// 3. Processes jobs in a loop: + /// - Receives job from channel + /// - Marks worker as busy + /// - Processes the job + /// - Marks worker as available + /// 4. Shuts down when channel closes + /// + /// # Panic Safety + /// + /// If this function panics, the worker thread terminates but other workers + /// continue operating and the system degrades gracefully. + fn run(self) -> ProviderResult<()> { + let Self { + task_ctx, + work_rx, + worker_id, + available_workers, + #[cfg(feature = "metrics")] + metrics, + } = self; + + // Create provider from factory + let provider = task_ctx.factory.database_provider_ro()?; + let proof_tx = ProofTaskTx::new(provider, worker_id); + + trace!( + target: "trie::proof_task", + worker_id, + "Storage worker started" ); - let hashed_cursor_factory = HashedPostStateCursorFactory::new( - DatabaseHashedCursorFactory::new(&self.tx), - &self.task_ctx.state_sorted, + let mut storage_proofs_processed = 0u64; + let mut storage_nodes_processed = 0u64; + + // Initially mark this worker as available. + available_workers.fetch_add(1, Ordering::Relaxed); + + while let Ok(job) = work_rx.recv() { + // Mark worker as busy. + available_workers.fetch_sub(1, Ordering::Relaxed); + + match job { + StorageWorkerJob::StorageProof { input, proof_result_sender } => { + Self::process_storage_proof( + worker_id, + &proof_tx, + input, + proof_result_sender, + &mut storage_proofs_processed, + ); + } + + StorageWorkerJob::BlindedStorageNode { account, path, result_sender } => { + Self::process_blinded_node( + worker_id, + &proof_tx, + account, + path, + result_sender, + &mut storage_nodes_processed, + ); + } + } + + // Mark worker as available again. + available_workers.fetch_add(1, Ordering::Relaxed); + } + + trace!( + target: "trie::proof_task", + worker_id, + storage_proofs_processed, + storage_nodes_processed, + "Storage worker shutting down" ); - (trie_cursor_factory, hashed_cursor_factory) + #[cfg(feature = "metrics")] + metrics.record_storage_nodes(storage_nodes_processed as usize); + + Ok(()) } - /// Calculates a storage proof for the given hashed address, and desired prefix set. - fn storage_proof( - self, + /// Processes a storage proof request. + fn process_storage_proof( + worker_id: usize, + proof_tx: &ProofTaskTx, input: StorageProofInput, - result_sender: Sender, - tx_sender: Sender>, - ) { - debug!( + proof_result_sender: ProofResultContext, + storage_proofs_processed: &mut u64, + ) where + Provider: TrieCursorFactory + HashedCursorFactory, + { + let hashed_address = input.hashed_address; + let ProofResultContext { sender, sequence_number: seq, state, start_time } = + proof_result_sender; + + trace!( target: "trie::proof_task", - hashed_address=?input.hashed_address, - "Starting storage proof task calculation" + worker_id, + hashed_address = ?hashed_address, + prefix_set_len = input.prefix_set.len(), + target_slots_len = input.target_slots.len(), + "Processing storage proof" ); - let (trie_cursor_factory, hashed_cursor_factory) = self.create_factories(); - - let target_slots_len = input.target_slots.len(); let proof_start = Instant::now(); - let result = StorageProof::new_hashed( - trie_cursor_factory, - hashed_cursor_factory, - input.hashed_address, - ) - .with_prefix_set_mut(PrefixSetMut::from(input.prefix_set.iter().cloned())) - .with_branch_node_masks(input.with_branch_node_masks) - .storage_multiproof(input.target_slots) - .map_err(|e| ParallelStateRootError::Other(e.to_string())); + let result = proof_tx.compute_storage_proof(input); - debug!( - target: "trie::proof_task", - hashed_address=?input.hashed_address, - prefix_set = ?input.prefix_set.len(), - target_slots = ?target_slots_len, - proof_time = ?proof_start.elapsed(), - "Completed storage proof task calculation" - ); + let proof_elapsed = proof_start.elapsed(); + *storage_proofs_processed += 1; - // send the result back - if let Err(error) = result_sender.send(result) { - debug!( + let result_msg = result.map(|storage_proof| ProofResult::StorageProof { + hashed_address, + proof: storage_proof, + }); + + if sender + .send(ProofResultMessage { + sequence_number: seq, + result: result_msg, + elapsed: start_time.elapsed(), + state, + }) + .is_err() + { + trace!( target: "trie::proof_task", - hashed_address = ?input.hashed_address, - ?error, - task_time = ?proof_start.elapsed(), - "Failed to send proof result" + worker_id, + hashed_address = ?hashed_address, + storage_proofs_processed, + "Proof result receiver dropped, discarding result" ); } - // send the tx back - let _ = tx_sender.send(ProofTaskMessage::Transaction(self)); + trace!( + target: "trie::proof_task", + worker_id, + hashed_address = ?hashed_address, + proof_time_us = proof_elapsed.as_micros(), + total_processed = storage_proofs_processed, + "Storage proof completed" + ); } - /// Retrieves blinded account node by path. - fn blinded_account_node( - self, + /// Processes a blinded storage node lookup request. + fn process_blinded_node( + worker_id: usize, + proof_tx: &ProofTaskTx, + account: B256, path: Nibbles, - result_sender: Sender, - tx_sender: Sender>, - ) { - debug!( + result_sender: Sender, + storage_nodes_processed: &mut u64, + ) where + Provider: TrieCursorFactory + HashedCursorFactory, + { + trace!( target: "trie::proof_task", + worker_id, + ?account, ?path, - "Starting blinded account node retrieval" + "Processing blinded storage node" ); - let (trie_cursor_factory, hashed_cursor_factory) = self.create_factories(); + let start = Instant::now(); + let result = proof_tx.process_blinded_storage_node(account, &path); + let elapsed = start.elapsed(); - let blinded_provider_factory = ProofBlindedProviderFactory::new( - trie_cursor_factory, - hashed_cursor_factory, - self.task_ctx.prefix_sets.clone(), - ); + *storage_nodes_processed += 1; - let start = Instant::now(); - let result = blinded_provider_factory.account_node_provider().blinded_node(&path); - debug!( + if result_sender.send(result).is_err() { + trace!( + target: "trie::proof_task", + worker_id, + ?account, + ?path, + storage_nodes_processed, + "Blinded storage node receiver dropped, discarding result" + ); + } + + trace!( target: "trie::proof_task", + worker_id, + ?account, ?path, - elapsed = ?start.elapsed(), - "Completed blinded account node retrieval" + elapsed_us = elapsed.as_micros(), + total_processed = storage_nodes_processed, + "Blinded storage node completed" ); + } +} - if let Err(error) = result_sender.send(result) { - tracing::error!( +/// Worker for account trie operations. +/// +/// Each worker maintains a dedicated database transaction and processes +/// account multiproof requests and blinded node lookups. +struct AccountProofWorker { + /// Shared task context with database factory and prefix sets + task_ctx: ProofTaskCtx, + /// Channel for receiving work + work_rx: CrossbeamReceiver, + /// Unique identifier for this worker (used for tracing) + worker_id: usize, + /// Channel for dispatching storage proof work + storage_work_tx: CrossbeamSender, + /// Counter tracking worker availability + available_workers: Arc, + /// Metrics collector for this worker + #[cfg(feature = "metrics")] + metrics: ProofTaskTrieMetrics, +} + +impl AccountProofWorker +where + Factory: DatabaseProviderROFactory, +{ + /// Creates a new account proof worker. + const fn new( + task_ctx: ProofTaskCtx, + work_rx: CrossbeamReceiver, + worker_id: usize, + storage_work_tx: CrossbeamSender, + available_workers: Arc, + #[cfg(feature = "metrics")] metrics: ProofTaskTrieMetrics, + ) -> Self { + Self { + task_ctx, + work_rx, + worker_id, + storage_work_tx, + available_workers, + #[cfg(feature = "metrics")] + metrics, + } + } + + /// Runs the worker loop, processing jobs until the channel closes. + /// + /// # Lifecycle + /// + /// 1. Initializes database provider and transaction + /// 2. Advertises availability + /// 3. Processes jobs in a loop: + /// - Receives job from channel + /// - Marks worker as busy + /// - Processes the job + /// - Marks worker as available + /// 4. Shuts down when channel closes + /// + /// # Panic Safety + /// + /// If this function panics, the worker thread terminates but other workers + /// continue operating and the system degrades gracefully. + fn run(self) -> ProviderResult<()> { + let Self { + task_ctx, + work_rx, + worker_id, + storage_work_tx, + available_workers, + #[cfg(feature = "metrics")] + metrics, + } = self; + + // Create provider from factory + let provider = task_ctx.factory.database_provider_ro()?; + let proof_tx = ProofTaskTx::new(provider, worker_id); + + trace!( + target: "trie::proof_task", + worker_id, + "Account worker started" + ); + + let mut account_proofs_processed = 0u64; + let mut account_nodes_processed = 0u64; + + // Count this worker as available only after successful initialization. + available_workers.fetch_add(1, Ordering::Relaxed); + + while let Ok(job) = work_rx.recv() { + // Mark worker as busy. + available_workers.fetch_sub(1, Ordering::Relaxed); + + match job { + AccountWorkerJob::AccountMultiproof { input } => { + Self::process_account_multiproof( + worker_id, + &proof_tx, + storage_work_tx.clone(), + *input, + &mut account_proofs_processed, + ); + } + + AccountWorkerJob::BlindedAccountNode { path, result_sender } => { + Self::process_blinded_node( + worker_id, + &proof_tx, + path, + result_sender, + &mut account_nodes_processed, + ); + } + } + + // Mark worker as available again. + available_workers.fetch_add(1, Ordering::Relaxed); + } + + trace!( + target: "trie::proof_task", + worker_id, + account_proofs_processed, + account_nodes_processed, + "Account worker shutting down" + ); + + #[cfg(feature = "metrics")] + metrics.record_account_nodes(account_nodes_processed as usize); + + Ok(()) + } + + /// Processes an account multiproof request. + fn process_account_multiproof( + worker_id: usize, + proof_tx: &ProofTaskTx, + storage_work_tx: CrossbeamSender, + input: AccountMultiproofInput, + account_proofs_processed: &mut u64, + ) where + Provider: TrieCursorFactory + HashedCursorFactory, + { + let AccountMultiproofInput { + targets, + mut prefix_sets, + collect_branch_node_masks, + multi_added_removed_keys, + missed_leaves_storage_roots, + proof_result_sender: + ProofResultContext { sender: result_tx, sequence_number: seq, state, start_time: start }, + } = input; + + let span = debug_span!( + target: "trie::proof_task", + "Account multiproof calculation", + targets = targets.len(), + worker_id, + ); + let _span_guard = span.enter(); + + trace!( + target: "trie::proof_task", + "Processing account multiproof" + ); + + let proof_start = Instant::now(); + + let mut tracker = ParallelTrieTracker::default(); + + let mut storage_prefix_sets = std::mem::take(&mut prefix_sets.storage_prefix_sets); + + let storage_root_targets_len = + StorageRootTargets::count(&prefix_sets.account_prefix_set, &storage_prefix_sets); + + tracker.set_precomputed_storage_roots(storage_root_targets_len as u64); + + let storage_proof_receivers = match dispatch_storage_proofs( + &storage_work_tx, + &targets, + &mut storage_prefix_sets, + collect_branch_node_masks, + multi_added_removed_keys.as_ref(), + ) { + Ok(receivers) => receivers, + Err(error) => { + // Send error through result channel + error!(target: "trie::proof_task", "Failed to dispatch storage proofs: {error}"); + let _ = result_tx.send(ProofResultMessage { + sequence_number: seq, + result: Err(error), + elapsed: start.elapsed(), + state, + }); + return; + } + }; + + // Use the missed leaves cache passed from the multiproof manager + let account_prefix_set = std::mem::take(&mut prefix_sets.account_prefix_set); + + let ctx = AccountMultiproofParams { + targets: &targets, + prefix_set: account_prefix_set, + collect_branch_node_masks, + multi_added_removed_keys: multi_added_removed_keys.as_ref(), + storage_proof_receivers, + missed_leaves_storage_roots: missed_leaves_storage_roots.as_ref(), + }; + + let result = + build_account_multiproof_with_storage_roots(&proof_tx.provider, ctx, &mut tracker); + + let proof_elapsed = proof_start.elapsed(); + let total_elapsed = start.elapsed(); + let stats = tracker.finish(); + let result = result.map(|proof| ProofResult::AccountMultiproof { proof, stats }); + *account_proofs_processed += 1; + + // Send result to MultiProofTask + if result_tx + .send(ProofResultMessage { + sequence_number: seq, + result, + elapsed: total_elapsed, + state, + }) + .is_err() + { + trace!( target: "trie::proof_task", - ?path, - ?error, - "Failed to send blinded account node result" + worker_id, + account_proofs_processed, + "Account multiproof receiver dropped, discarding result" ); } - // send the tx back - let _ = tx_sender.send(ProofTaskMessage::Transaction(self)); + trace!( + target: "trie::proof_task", + proof_time_us = proof_elapsed.as_micros(), + total_elapsed_us = total_elapsed.as_micros(), + total_processed = account_proofs_processed, + "Account multiproof completed" + ); } - /// Retrieves blinded storage node of the given account by path. - fn blinded_storage_node( - self, - account: B256, + /// Processes a blinded account node lookup request. + fn process_blinded_node( + worker_id: usize, + proof_tx: &ProofTaskTx, path: Nibbles, - result_sender: Sender, - tx_sender: Sender>, - ) { - debug!( + result_sender: Sender, + account_nodes_processed: &mut u64, + ) where + Provider: TrieCursorFactory + HashedCursorFactory, + { + let span = debug_span!( target: "trie::proof_task", - ?account, + "Blinded account node calculation", ?path, - "Starting blinded storage node retrieval" + worker_id, ); + let _span_guard = span.enter(); - let (trie_cursor_factory, hashed_cursor_factory) = self.create_factories(); - - let blinded_provider_factory = ProofBlindedProviderFactory::new( - trie_cursor_factory, - hashed_cursor_factory, - self.task_ctx.prefix_sets.clone(), + trace!( + target: "trie::proof_task", + "Processing blinded account node" ); let start = Instant::now(); - let result = blinded_provider_factory.storage_node_provider(account).blinded_node(&path); - debug!( - target: "trie::proof_task", - ?account, - ?path, - elapsed = ?start.elapsed(), - "Completed blinded storage node retrieval" - ); + let result = proof_tx.process_blinded_account_node(&path); + let elapsed = start.elapsed(); + + *account_nodes_processed += 1; - if let Err(error) = result_sender.send(result) { - tracing::error!( + if result_sender.send(result).is_err() { + trace!( target: "trie::proof_task", - ?account, + worker_id, ?path, - ?error, - "Failed to send blinded storage node result" + account_nodes_processed, + "Blinded account node receiver dropped, discarding result" ); } - // send the tx back - let _ = tx_sender.send(ProofTaskMessage::Transaction(self)); + trace!( + target: "trie::proof_task", + node_time_us = elapsed.as_micros(), + total_processed = account_nodes_processed, + "Blinded account node completed" + ); } } -/// This represents an input for a storage proof. +/// Builds an account multiproof by consuming storage proof receivers lazily during trie walk. +/// +/// This is a helper function used by account workers to build the account subtree proof +/// while storage proofs are still being computed. Receivers are consumed only when needed, +/// enabling interleaved parallelism between account trie traversal and storage proof computation. +/// +/// Returns a `DecodedMultiProof` containing the account subtree and storage proofs. +fn build_account_multiproof_with_storage_roots

( + provider: &P, + ctx: AccountMultiproofParams<'_>, + tracker: &mut ParallelTrieTracker, +) -> Result +where + P: TrieCursorFactory + HashedCursorFactory, +{ + let accounts_added_removed_keys = + ctx.multi_added_removed_keys.as_ref().map(|keys| keys.get_accounts()); + + // Create the walker. + let walker = TrieWalker::<_>::state_trie( + provider.account_trie_cursor().map_err(ProviderError::Database)?, + ctx.prefix_set, + ) + .with_added_removed_keys(accounts_added_removed_keys) + .with_deletions_retained(true); + + // Create a hash builder to rebuild the root node since it is not available in the database. + let retainer = ctx + .targets + .keys() + .map(Nibbles::unpack) + .collect::() + .with_added_removed_keys(accounts_added_removed_keys); + let mut hash_builder = HashBuilder::default() + .with_proof_retainer(retainer) + .with_updates(ctx.collect_branch_node_masks); + + // Initialize storage multiproofs map with pre-allocated capacity. + // Proofs will be inserted as they're consumed from receivers during trie walk. + let mut collected_decoded_storages: B256Map = + B256Map::with_capacity_and_hasher(ctx.targets.len(), Default::default()); + let mut account_rlp = Vec::with_capacity(TRIE_ACCOUNT_RLP_MAX_SIZE); + let mut account_node_iter = TrieNodeIter::state_trie( + walker, + provider.hashed_account_cursor().map_err(ProviderError::Database)?, + ); + + let mut storage_proof_receivers = ctx.storage_proof_receivers; + + while let Some(account_node) = account_node_iter.try_next().map_err(ProviderError::Database)? { + match account_node { + TrieElement::Branch(node) => { + hash_builder.add_branch(node.key, node.value, node.children_are_in_trie); + } + TrieElement::Leaf(hashed_address, account) => { + let root = match storage_proof_receivers.remove(&hashed_address) { + Some(receiver) => { + // Block on this specific storage proof receiver - enables interleaved + // parallelism + let proof_msg = receiver.recv().map_err(|_| { + ParallelStateRootError::StorageRoot( + reth_execution_errors::StorageRootError::Database( + DatabaseError::Other(format!( + "Storage proof channel closed for {hashed_address}" + )), + ), + ) + })?; + + // Extract storage proof from the result + let proof = match proof_msg.result? { + ProofResult::StorageProof { hashed_address: addr, proof } => { + debug_assert_eq!( + addr, + hashed_address, + "storage worker must return same address: expected {hashed_address}, got {addr}" + ); + proof + } + ProofResult::AccountMultiproof { .. } => { + unreachable!("storage worker only sends StorageProof variant") + } + }; + + let root = proof.root; + collected_decoded_storages.insert(hashed_address, proof); + root + } + // Since we do not store all intermediate nodes in the database, there might + // be a possibility of re-adding a non-modified leaf to the hash builder. + None => { + tracker.inc_missed_leaves(); + + match ctx.missed_leaves_storage_roots.entry(hashed_address) { + dashmap::Entry::Occupied(occ) => *occ.get(), + dashmap::Entry::Vacant(vac) => { + let root = + StorageProof::new_hashed(provider, provider, hashed_address) + .with_prefix_set_mut(Default::default()) + .storage_multiproof( + ctx.targets + .get(&hashed_address) + .cloned() + .unwrap_or_default(), + ) + .map_err(|e| { + ParallelStateRootError::StorageRoot( + reth_execution_errors::StorageRootError::Database( + DatabaseError::Other(e.to_string()), + ), + ) + })? + .root; + + vac.insert(root); + root + } + } + } + }; + + // Encode account + account_rlp.clear(); + let account = account.into_trie_account(root); + account.encode(&mut account_rlp as &mut dyn BufMut); + + hash_builder.add_leaf(Nibbles::unpack(hashed_address), &account_rlp); + } + } + } + + // Consume remaining storage proof receivers for accounts not encountered during trie walk. + for (hashed_address, receiver) in storage_proof_receivers { + if let Ok(proof_msg) = receiver.recv() { + // Extract storage proof from the result + if let Ok(ProofResult::StorageProof { proof, .. }) = proof_msg.result { + collected_decoded_storages.insert(hashed_address, proof); + } + } + } + + let _ = hash_builder.root(); + + let account_subtree_raw_nodes = hash_builder.take_proof_nodes(); + let decoded_account_subtree = DecodedProofNodes::try_from(account_subtree_raw_nodes)?; + + let (branch_node_hash_masks, branch_node_tree_masks) = if ctx.collect_branch_node_masks { + let updated_branch_nodes = hash_builder.updated_branch_nodes.unwrap_or_default(); + ( + updated_branch_nodes.iter().map(|(path, node)| (*path, node.hash_mask)).collect(), + updated_branch_nodes.into_iter().map(|(path, node)| (path, node.tree_mask)).collect(), + ) + } else { + (Default::default(), Default::default()) + }; + + Ok(DecodedMultiProof { + account_subtree: decoded_account_subtree, + branch_node_hash_masks, + branch_node_tree_masks, + storages: collected_decoded_storages, + }) +} +/// Queues storage proofs for all accounts in the targets and returns receivers. +/// +/// This function queues all storage proof tasks to the worker pool but returns immediately +/// with receivers, allowing the account trie walk to proceed in parallel with storage proof +/// computation. This enables interleaved parallelism for better performance. +/// +/// Propagates errors up if queuing fails. Receivers must be consumed by the caller. +fn dispatch_storage_proofs( + storage_work_tx: &CrossbeamSender, + targets: &MultiProofTargets, + storage_prefix_sets: &mut B256Map, + with_branch_node_masks: bool, + multi_added_removed_keys: Option<&Arc>, +) -> Result>, ParallelStateRootError> { + let mut storage_proof_receivers = + B256Map::with_capacity_and_hasher(targets.len(), Default::default()); + + // Dispatch all storage proofs to worker pool + for (hashed_address, target_slots) in targets.iter() { + let prefix_set = storage_prefix_sets.remove(hashed_address).unwrap_or_default(); + + // Create channel for receiving ProofResultMessage + let (result_tx, result_rx) = crossbeam_channel::unbounded(); + let start = Instant::now(); + + // Create computation input (data only, no communication channel) + let input = StorageProofInput::new( + *hashed_address, + prefix_set, + target_slots.clone(), + with_branch_node_masks, + multi_added_removed_keys.cloned(), + ); + + // Always dispatch a storage proof so we obtain the storage root even when no slots are + // requested. + storage_work_tx + .send(StorageWorkerJob::StorageProof { + input, + proof_result_sender: ProofResultContext::new( + result_tx, + 0, + HashedPostState::default(), + start, + ), + }) + .map_err(|_| { + ParallelStateRootError::Other(format!( + "Failed to queue storage proof for {}: storage worker pool unavailable", + hashed_address + )) + })?; + + storage_proof_receivers.insert(*hashed_address, result_rx); + } + + Ok(storage_proof_receivers) +} +/// Input parameters for storage proof computation. #[derive(Debug)] pub struct StorageProofInput { /// The hashed address for which the proof is calculated. @@ -380,6 +1397,8 @@ pub struct StorageProofInput { target_slots: B256Set, /// Whether or not to collect branch node masks with_branch_node_masks: bool, + /// Provided by the user to give the necessary context to retain extra proofs. + multi_added_removed_keys: Option>, } impl StorageProofInput { @@ -390,151 +1409,96 @@ impl StorageProofInput { prefix_set: PrefixSet, target_slots: B256Set, with_branch_node_masks: bool, + multi_added_removed_keys: Option>, ) -> Self { - Self { hashed_address, prefix_set, target_slots, with_branch_node_masks } + Self { + hashed_address, + prefix_set, + target_slots, + with_branch_node_masks, + multi_added_removed_keys, + } } } - -/// Data used for initializing cursor factories that is shared across all storage proof instances. +/// Input parameters for account multiproof computation. #[derive(Debug, Clone)] -pub struct ProofTaskCtx { - /// The sorted collection of cached in-memory intermediate trie nodes that can be reused for - /// computation. - nodes_sorted: Arc, - /// The sorted in-memory overlay hashed state. - state_sorted: Arc, - /// The collection of prefix sets for the computation. Since the prefix sets _always_ - /// invalidate the in-memory nodes, not all keys from `state_sorted` might be present here, - /// if we have cached nodes for them. - prefix_sets: Arc, +pub struct AccountMultiproofInput { + /// The targets for which to compute the multiproof. + pub targets: MultiProofTargets, + /// The prefix sets for the proof calculation. + pub prefix_sets: TriePrefixSets, + /// Whether or not to collect branch node masks. + pub collect_branch_node_masks: bool, + /// Provided by the user to give the necessary context to retain extra proofs. + pub multi_added_removed_keys: Option>, + /// Cached storage proof roots for missed leaves encountered during account trie walk. + pub missed_leaves_storage_roots: Arc>, + /// Context for sending the proof result. + pub proof_result_sender: ProofResultContext, } -impl ProofTaskCtx { - /// Creates a new [`ProofTaskCtx`] with the given sorted nodes and state. - pub const fn new( - nodes_sorted: Arc, - state_sorted: Arc, - prefix_sets: Arc, - ) -> Self { - Self { nodes_sorted, state_sorted, prefix_sets } - } -} - -/// Message used to communicate with [`ProofTaskManager`]. -#[derive(Debug)] -pub enum ProofTaskMessage { - /// A request to queue a proof task. - QueueTask(ProofTaskKind), - /// A returned database transaction. - Transaction(ProofTaskTx), - /// A request to terminate the proof task manager. - Terminate, -} - -/// Proof task kind. -/// -/// When queueing a task using [`ProofTaskMessage::QueueTask`], this enum -/// specifies the type of proof task to be executed. -#[derive(Debug)] -pub enum ProofTaskKind { - /// A storage proof request. - StorageProof(StorageProofInput, Sender), - /// A blinded account node request. - BlindedAccountNode(Nibbles, Sender), - /// A blinded storage node request. - BlindedStorageNode(B256, Nibbles, Sender), +/// Parameters for building an account multiproof with pre-computed storage roots. +struct AccountMultiproofParams<'a> { + /// The targets for which to compute the multiproof. + targets: &'a MultiProofTargets, + /// The prefix set for the account trie walk. + prefix_set: PrefixSet, + /// Whether or not to collect branch node masks. + collect_branch_node_masks: bool, + /// Provided by the user to give the necessary context to retain extra proofs. + multi_added_removed_keys: Option<&'a Arc>, + /// Receivers for storage proofs being computed in parallel. + storage_proof_receivers: B256Map>, + /// Cached storage proof roots for missed leaves encountered during account trie walk. + missed_leaves_storage_roots: &'a DashMap, } -/// A handle that wraps a single proof task sender that sends a terminate message on `Drop` if the -/// number of active handles went to zero. +/// Internal message for account workers. #[derive(Debug)] -pub struct ProofTaskManagerHandle { - /// The sender for the proof task manager. - sender: Sender>, - /// The number of active handles. - active_handles: Arc, -} - -impl ProofTaskManagerHandle { - /// Creates a new [`ProofTaskManagerHandle`] with the given sender. - pub fn new(sender: Sender>, active_handles: Arc) -> Self { - active_handles.fetch_add(1, Ordering::SeqCst); - Self { sender, active_handles } - } - - /// Queues a task to the proof task manager. - pub fn queue_task(&self, task: ProofTaskKind) -> Result<(), SendError>> { - self.sender.send(ProofTaskMessage::QueueTask(task)) - } - - /// Terminates the proof task manager. - pub fn terminate(&self) { - let _ = self.sender.send(ProofTaskMessage::Terminate); - } -} - -impl Clone for ProofTaskManagerHandle { - fn clone(&self) -> Self { - Self::new(self.sender.clone(), self.active_handles.clone()) - } -} - -impl Drop for ProofTaskManagerHandle { - fn drop(&mut self) { - // Decrement the number of active handles and terminate the manager if it was the last - // handle. - if self.active_handles.fetch_sub(1, Ordering::SeqCst) == 1 { - self.terminate(); - } - } +enum AccountWorkerJob { + /// Account multiproof computation request + AccountMultiproof { + /// Account multiproof input parameters + input: Box, + }, + /// Blinded account node retrieval request + BlindedAccountNode { + /// Path to the account node + path: Nibbles, + /// Channel to send result back to original caller + result_sender: Sender, + }, } -impl BlindedProviderFactory for ProofTaskManagerHandle { - type AccountNodeProvider = ProofTaskBlindedNodeProvider; - type StorageNodeProvider = ProofTaskBlindedNodeProvider; +#[cfg(test)] +mod tests { + use super::*; + use reth_provider::test_utils::create_test_provider_factory; + use tokio::{runtime::Builder, task}; - fn account_node_provider(&self) -> Self::AccountNodeProvider { - ProofTaskBlindedNodeProvider::AccountNode { sender: self.sender.clone() } + fn test_ctx(factory: Factory) -> ProofTaskCtx { + ProofTaskCtx::new(factory) } - fn storage_node_provider(&self, account: B256) -> Self::StorageNodeProvider { - ProofTaskBlindedNodeProvider::StorageNode { account, sender: self.sender.clone() } - } -} + /// Ensures `ProofWorkerHandle::new` spawns workers correctly. + #[test] + fn spawn_proof_workers_creates_handle() { + let runtime = Builder::new_multi_thread().worker_threads(1).enable_all().build().unwrap(); + runtime.block_on(async { + let handle = tokio::runtime::Handle::current(); + let provider_factory = create_test_provider_factory(); + let factory = + reth_provider::providers::OverlayStateProviderFactory::new(provider_factory); + let ctx = test_ctx(factory); -/// Blinded node provider for retrieving trie nodes by path. -#[derive(Debug)] -pub enum ProofTaskBlindedNodeProvider { - /// Blinded account trie node provider. - AccountNode { - /// Sender to the proof task. - sender: Sender>, - }, - /// Blinded storage trie node provider. - StorageNode { - /// Target account. - account: B256, - /// Sender to the proof task. - sender: Sender>, - }, -} + let proof_handle = ProofWorkerHandle::new(handle.clone(), ctx, 5, 3); -impl BlindedProvider for ProofTaskBlindedNodeProvider { - fn blinded_node(&self, path: &Nibbles) -> Result, SparseTrieError> { - let (tx, rx) = channel(); - match self { - Self::AccountNode { sender } => { - let _ = sender.send(ProofTaskMessage::QueueTask( - ProofTaskKind::BlindedAccountNode(path.clone(), tx), - )); - } - Self::StorageNode { sender, account } => { - let _ = sender.send(ProofTaskMessage::QueueTask( - ProofTaskKind::BlindedStorageNode(*account, path.clone(), tx), - )); - } - } + // Verify handle can be cloned + let _cloned_handle = proof_handle.clone(); - rx.recv().unwrap() + // Workers shut down automatically when handle is dropped + drop(proof_handle); + task::yield_now().await; + }); } } diff --git a/crates/trie/parallel/src/proof_task_metrics.rs b/crates/trie/parallel/src/proof_task_metrics.rs new file mode 100644 index 00000000000..6492e28d12d --- /dev/null +++ b/crates/trie/parallel/src/proof_task_metrics.rs @@ -0,0 +1,23 @@ +use reth_metrics::{metrics::Histogram, Metrics}; + +/// Metrics for the proof task. +#[derive(Clone, Metrics)] +#[metrics(scope = "trie.proof_task")] +pub struct ProofTaskTrieMetrics { + /// A histogram for the number of blinded account nodes fetched. + blinded_account_nodes: Histogram, + /// A histogram for the number of blinded storage nodes fetched. + blinded_storage_nodes: Histogram, +} + +impl ProofTaskTrieMetrics { + /// Record account nodes fetched. + pub fn record_account_nodes(&self, count: usize) { + self.blinded_account_nodes.record(count as f64); + } + + /// Record storage nodes fetched. + pub fn record_storage_nodes(&self, count: usize) { + self.blinded_storage_nodes.record(count as f64); + } +} diff --git a/crates/trie/parallel/src/root.rs b/crates/trie/parallel/src/root.rs index 4b6964c70d5..5c9294e8f92 100644 --- a/crates/trie/parallel/src/root.rs +++ b/crates/trie/parallel/src/root.rs @@ -5,22 +5,24 @@ use alloy_primitives::B256; use alloy_rlp::{BufMut, Encodable}; use itertools::Itertools; use reth_execution_errors::StorageRootError; -use reth_provider::{ - providers::ConsistentDbView, BlockReader, DBProvider, DatabaseProviderFactory, ProviderError, - StateCommitmentProvider, -}; +use reth_provider::{DatabaseProviderROFactory, ProviderError}; use reth_storage_errors::db::DatabaseError; use reth_trie::{ - hashed_cursor::{HashedCursorFactory, HashedPostStateCursorFactory}, + hashed_cursor::HashedCursorFactory, node_iter::{TrieElement, TrieNodeIter}, - trie_cursor::{InMemoryTrieCursorFactory, TrieCursorFactory}, + prefix_set::TriePrefixSets, + trie_cursor::TrieCursorFactory, updates::TrieUpdates, walker::TrieWalker, - HashBuilder, Nibbles, StorageRoot, TrieInput, TRIE_ACCOUNT_RLP_MAX_SIZE, + HashBuilder, Nibbles, StorageRoot, TRIE_ACCOUNT_RLP_MAX_SIZE, +}; +use std::{ + collections::HashMap, + sync::{mpsc, OnceLock}, + time::Duration, }; -use reth_trie_db::{DatabaseHashedCursorFactory, DatabaseTrieCursorFactory}; -use std::{collections::HashMap, sync::Arc}; use thiserror::Error; +use tokio::runtime::{Builder, Handle, Runtime}; use tracing::*; /// Parallel incremental state root calculator. @@ -30,16 +32,15 @@ use tracing::*; /// nodes in the process. Upon encountering a leaf node, it will poll the storage root /// task for the corresponding hashed address. /// -/// Internally, the calculator uses [`ConsistentDbView`] since -/// it needs to rely on database state saying the same until -/// the last transaction is open. -/// See docs of using [`ConsistentDbView`] for caveats. +/// Note: This implementation only serves as a fallback for the sparse trie-based +/// state root calculation. The sparse trie approach is more efficient as it avoids traversing +/// the entire trie, only operating on the modified parts. #[derive(Debug)] pub struct ParallelStateRoot { - /// Consistent view of the database. - view: ConsistentDbView, - /// Trie input. - input: TrieInput, + /// Factory for creating state providers. + factory: Factory, + // Prefix sets indicating which portions of the trie need to be recomputed. + prefix_sets: TriePrefixSets, /// Parallel state root metrics. #[cfg(feature = "metrics")] metrics: ParallelStateRootMetrics, @@ -47,10 +48,10 @@ pub struct ParallelStateRoot { impl ParallelStateRoot { /// Create new parallel state root calculator. - pub fn new(view: ConsistentDbView, input: TrieInput) -> Self { + pub fn new(factory: Factory, prefix_sets: TriePrefixSets) -> Self { Self { - view, - input, + factory, + prefix_sets, #[cfg(feature = "metrics")] metrics: ParallelStateRootMetrics::default(), } @@ -59,11 +60,9 @@ impl ParallelStateRoot { impl ParallelStateRoot where - Factory: DatabaseProviderFactory - + StateCommitmentProvider + Factory: DatabaseProviderROFactory + Clone + Send - + Sync + 'static, { /// Calculate incremental state root in parallel. @@ -78,48 +77,45 @@ where self.calculate(true) } + /// Computes the state root by calculating storage roots in parallel for modified accounts, + /// then walking the state trie to build the final state root hash. fn calculate( self, retain_updates: bool, ) -> Result<(B256, TrieUpdates), ParallelStateRootError> { let mut tracker = ParallelTrieTracker::default(); - let trie_nodes_sorted = Arc::new(self.input.nodes.into_sorted()); - let hashed_state_sorted = Arc::new(self.input.state.into_sorted()); - let prefix_sets = self.input.prefix_sets.freeze(); let storage_root_targets = StorageRootTargets::new( - prefix_sets.account_prefix_set.iter().map(|nibbles| B256::from_slice(&nibbles.pack())), - prefix_sets.storage_prefix_sets, + self.prefix_sets + .account_prefix_set + .iter() + .map(|nibbles| B256::from_slice(&nibbles.pack())), + self.prefix_sets.storage_prefix_sets, ); // Pre-calculate storage roots in parallel for accounts which were changed. tracker.set_precomputed_storage_roots(storage_root_targets.len() as u64); debug!(target: "trie::parallel_state_root", len = storage_root_targets.len(), "pre-calculating storage roots"); let mut storage_roots = HashMap::with_capacity(storage_root_targets.len()); + + // Get runtime handle once outside the loop + let handle = get_runtime_handle(); + for (hashed_address, prefix_set) in storage_root_targets.into_iter().sorted_unstable_by_key(|(address, _)| *address) { - let view = self.view.clone(); - let hashed_state_sorted = hashed_state_sorted.clone(); - let trie_nodes_sorted = trie_nodes_sorted.clone(); + let factory = self.factory.clone(); #[cfg(feature = "metrics")] let metrics = self.metrics.storage_trie.clone(); - let (tx, rx) = std::sync::mpsc::sync_channel(1); + let (tx, rx) = mpsc::sync_channel(1); - rayon::spawn_fifo(move || { + // Spawn a blocking task to calculate account's storage root from database I/O + drop(handle.spawn_blocking(move || { let result = (|| -> Result<_, ParallelStateRootError> { - let provider_ro = view.provider_ro()?; - let trie_cursor_factory = InMemoryTrieCursorFactory::new( - DatabaseTrieCursorFactory::new(provider_ro.tx_ref()), - &trie_nodes_sorted, - ); - let hashed_state = HashedPostStateCursorFactory::new( - DatabaseHashedCursorFactory::new(provider_ro.tx_ref()), - &hashed_state_sorted, - ); + let provider = factory.database_provider_ro()?; Ok(StorageRoot::new_hashed( - trie_cursor_factory, - hashed_state, + &provider, + &provider, hashed_address, prefix_set, #[cfg(feature = "metrics")] @@ -128,31 +124,23 @@ where .calculate(retain_updates)?) })(); let _ = tx.send(result); - }); + })); storage_roots.insert(hashed_address, rx); } trace!(target: "trie::parallel_state_root", "calculating state root"); let mut trie_updates = TrieUpdates::default(); - let provider_ro = self.view.provider_ro()?; - let trie_cursor_factory = InMemoryTrieCursorFactory::new( - DatabaseTrieCursorFactory::new(provider_ro.tx_ref()), - &trie_nodes_sorted, - ); - let hashed_cursor_factory = HashedPostStateCursorFactory::new( - DatabaseHashedCursorFactory::new(provider_ro.tx_ref()), - &hashed_state_sorted, - ); + let provider = self.factory.database_provider_ro()?; - let walker = TrieWalker::state_trie( - trie_cursor_factory.account_trie_cursor().map_err(ProviderError::Database)?, - prefix_sets.account_prefix_set, + let walker = TrieWalker::<_>::state_trie( + provider.account_trie_cursor().map_err(ProviderError::Database)?, + self.prefix_sets.account_prefix_set, ) .with_deletions_retained(retain_updates); let mut account_node_iter = TrieNodeIter::state_trie( walker, - hashed_cursor_factory.hashed_account_cursor().map_err(ProviderError::Database)?, + provider.hashed_account_cursor().map_err(ProviderError::Database)?, ); let mut hash_builder = HashBuilder::default().with_updates(retain_updates); @@ -163,7 +151,7 @@ where hash_builder.add_branch(node.key, node.value, node.children_are_in_trie); } TrieElement::Leaf(hashed_address, account) => { - let (storage_root, _, updates) = match storage_roots.remove(&hashed_address) { + let storage_root_result = match storage_roots.remove(&hashed_address) { Some(rx) => rx.recv().map_err(|_| { ParallelStateRootError::StorageRoot(StorageRootError::Database( DatabaseError::Other(format!( @@ -176,8 +164,8 @@ where None => { tracker.inc_missed_leaves(); StorageRoot::new_hashed( - trie_cursor_factory.clone(), - hashed_cursor_factory.clone(), + &provider, + &provider, hashed_address, Default::default(), #[cfg(feature = "metrics")] @@ -187,6 +175,17 @@ where } }; + let (storage_root, _, updates) = match storage_root_result { + reth_trie::StorageRootProgress::Complete(root, _, updates) => (root, (), updates), + reth_trie::StorageRootProgress::Progress(..) => { + return Err(ParallelStateRootError::StorageRoot( + StorageRootError::Database(DatabaseError::Other( + "StorageRoot returned Progress variant in parallel trie calculation".to_string() + )) + )) + } + }; + if retain_updates { trie_updates.insert_storage_updates(hashed_address, updates); } @@ -202,7 +201,7 @@ where let root = hash_builder.root(); let removed_keys = account_node_iter.walker.take_removed_keys(); - trie_updates.finalize(hash_builder, removed_keys, prefix_sets.destroyed_accounts); + trie_updates.finalize(hash_builder, removed_keys, self.prefix_sets.destroyed_accounts); let stats = tracker.finish(); @@ -250,6 +249,33 @@ impl From for ProviderError { } } +impl From for ParallelStateRootError { + fn from(error: alloy_rlp::Error) -> Self { + Self::Provider(ProviderError::Rlp(error)) + } +} + +/// Gets or creates a tokio runtime handle for spawning blocking tasks. +/// This ensures we always have a runtime available for I/O operations. +fn get_runtime_handle() -> Handle { + Handle::try_current().unwrap_or_else(|_| { + // Create a new runtime if no runtime is available + static RT: OnceLock = OnceLock::new(); + + let rt = RT.get_or_init(|| { + Builder::new_multi_thread() + // Keep the threads alive for at least the block time (12 seconds) plus buffer. + // This prevents the costly process of spawning new threads on every + // new block, and instead reuses the existing threads. + .thread_keep_alive(Duration::from_secs(15)) + .build() + .expect("Failed to create tokio runtime") + }); + + rt.handle().clone() + }) +} + #[cfg(test)] mod tests { use super::*; @@ -258,11 +284,13 @@ mod tests { use reth_primitives_traits::{Account, StorageEntry}; use reth_provider::{test_utils::create_test_provider_factory, HashingWriter}; use reth_trie::{test_utils, HashedPostState, HashedStorage}; + use std::sync::Arc; - #[test] - fn random_parallel_root() { + #[tokio::test] + async fn random_parallel_root() { let factory = create_test_provider_factory(); - let consistent_view = ConsistentDbView::new(factory.clone(), None); + let mut overlay_factory = + reth_provider::providers::OverlayStateProviderFactory::new(factory.clone()); let mut rng = rand::rng(); let mut state = (0..100) @@ -305,7 +333,7 @@ mod tests { } assert_eq!( - ParallelStateRoot::new(consistent_view.clone(), Default::default()) + ParallelStateRoot::new(overlay_factory.clone(), Default::default()) .incremental_root() .unwrap(), test_utils::state_root(state.clone()) @@ -336,8 +364,12 @@ mod tests { } } + let prefix_sets = hashed_state.construct_prefix_sets(); + overlay_factory = + overlay_factory.with_hashed_state_overlay(Some(Arc::new(hashed_state.into_sorted()))); + assert_eq!( - ParallelStateRoot::new(consistent_view, TrieInput::from_state(hashed_state)) + ParallelStateRoot::new(overlay_factory, prefix_sets.freeze()) .incremental_root() .unwrap(), test_utils::state_root(state) diff --git a/crates/trie/parallel/src/storage_root_targets.rs b/crates/trie/parallel/src/storage_root_targets.rs index f844b70fca5..0c6d9f43498 100644 --- a/crates/trie/parallel/src/storage_root_targets.rs +++ b/crates/trie/parallel/src/storage_root_targets.rs @@ -24,6 +24,23 @@ impl StorageRootTargets { .collect(), ) } + + /// Returns the total number of unique storage root targets without allocating new maps. + pub fn count( + account_prefix_set: &PrefixSet, + storage_prefix_sets: &B256Map, + ) -> usize { + let mut count = storage_prefix_sets.len(); + + for nibbles in account_prefix_set { + let hashed_address = B256::from_slice(&nibbles.pack()); + if !storage_prefix_sets.contains_key(&hashed_address) { + count += 1; + } + } + + count + } } impl IntoIterator for StorageRootTargets { diff --git a/crates/trie/sparse-parallel/Cargo.toml b/crates/trie/sparse-parallel/Cargo.toml new file mode 100644 index 00000000000..9c62aabaddf --- /dev/null +++ b/crates/trie/sparse-parallel/Cargo.toml @@ -0,0 +1,70 @@ +[package] +name = "reth-trie-sparse-parallel" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true +description = "Parallel Sparse MPT implementation" + +[lints] +workspace = true + +[dependencies] +# reth +reth-execution-errors.workspace = true +reth-trie-common.workspace = true +reth-trie-sparse.workspace = true +tracing = { workspace = true, features = ["attributes"] } +alloy-trie.workspace = true + +# alloy +alloy-primitives.workspace = true +alloy-rlp.workspace = true + +# metrics +reth-metrics = { workspace = true, optional = true } +metrics = { workspace = true, optional = true } + +# misc +smallvec.workspace = true +rayon = { workspace = true, optional = true } + +[dev-dependencies] +# reth +reth-primitives-traits.workspace = true +reth-provider = { workspace = true, features = ["test-utils"] } +reth-trie-common = { workspace = true, features = ["test-utils", "arbitrary"] } +reth-trie-db.workspace = true +reth-trie-sparse = { workspace = true, features = ["test-utils"] } +reth-trie.workspace = true + +# misc +arbitrary.workspace = true +assert_matches.workspace = true +itertools.workspace = true +pretty_assertions.workspace = true +proptest-arbitrary-interop.workspace = true +proptest.workspace = true +rand.workspace = true +rand_08.workspace = true + +[features] +default = ["std", "metrics"] +std = [ + "dep:rayon", + "alloy-primitives/std", + "alloy-rlp/std", + "alloy-trie/std", + "reth-execution-errors/std", + "reth-primitives-traits/std", + "reth-trie-common/std", + "reth-trie-sparse/std", + "tracing/std", +] +metrics = [ + "dep:reth-metrics", + "dep:metrics", + "std", +] diff --git a/crates/trie/sparse-parallel/src/lib.rs b/crates/trie/sparse-parallel/src/lib.rs new file mode 100644 index 00000000000..f37b274e41c --- /dev/null +++ b/crates/trie/sparse-parallel/src/lib.rs @@ -0,0 +1,14 @@ +//! The implementation of parallel sparse MPT. + +#![cfg_attr(not(test), warn(unused_crate_dependencies))] + +extern crate alloc; + +mod trie; +pub use trie::*; + +mod lower; +use lower::*; + +#[cfg(feature = "metrics")] +mod metrics; diff --git a/crates/trie/sparse-parallel/src/lower.rs b/crates/trie/sparse-parallel/src/lower.rs new file mode 100644 index 00000000000..b7eceb133b8 --- /dev/null +++ b/crates/trie/sparse-parallel/src/lower.rs @@ -0,0 +1,131 @@ +use crate::SparseSubtrie; +use reth_trie_common::Nibbles; + +/// Tracks the state of the lower subtries. +/// +/// When a [`crate::ParallelSparseTrie`] is initialized/cleared then its `LowerSparseSubtrie`s are +/// all blinded, meaning they have no nodes. A blinded `LowerSparseSubtrie` may hold onto a cleared +/// [`SparseSubtrie`] in order to reuse allocations. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum LowerSparseSubtrie { + Blind(Option>), + Revealed(Box), +} + +impl Default for LowerSparseSubtrie { + /// Creates a new blinded subtrie with no allocated storage. + fn default() -> Self { + Self::Blind(None) + } +} + +impl LowerSparseSubtrie { + /// Returns a reference to the underlying [`SparseSubtrie`] if this subtrie is revealed. + /// + /// Returns `None` if the subtrie is blinded (has no nodes). + pub(crate) fn as_revealed_ref(&self) -> Option<&SparseSubtrie> { + match self { + Self::Blind(_) => None, + Self::Revealed(subtrie) => Some(subtrie.as_ref()), + } + } + + /// Returns a mutable reference to the underlying [`SparseSubtrie`] if this subtrie is revealed. + /// + /// Returns `None` if the subtrie is blinded (has no nodes). + pub(crate) fn as_revealed_mut(&mut self) -> Option<&mut SparseSubtrie> { + match self { + Self::Blind(_) => None, + Self::Revealed(subtrie) => Some(subtrie.as_mut()), + } + } + + /// Reveals the lower [`SparseSubtrie`], transitioning it from the Blinded to the Revealed + /// variant, preserving allocations if possible. + /// + /// The given path is the path of a node which will be set into the [`SparseSubtrie`]'s `nodes` + /// map immediately upon being revealed. If the subtrie is blinded, or if its current root path + /// is longer than this one, than this one becomes the new root path of the subtrie. + pub(crate) fn reveal(&mut self, path: &Nibbles) { + match self { + Self::Blind(allocated) => { + debug_assert!(allocated.as_ref().is_none_or(|subtrie| subtrie.is_empty())); + *self = if let Some(mut subtrie) = allocated.take() { + subtrie.path = *path; + Self::Revealed(subtrie) + } else { + Self::Revealed(Box::new(SparseSubtrie::new(*path))) + } + } + Self::Revealed(subtrie) => { + if path.len() < subtrie.path.len() { + subtrie.path = *path; + } + } + }; + } + + /// Clears the subtrie and transitions it to the blinded state, preserving a cleared + /// [`SparseSubtrie`] if possible. + pub(crate) fn clear(&mut self) { + *self = match core::mem::take(self) { + Self::Blind(allocated) => { + debug_assert!(allocated.as_ref().is_none_or(|subtrie| subtrie.is_empty())); + Self::Blind(allocated) + } + Self::Revealed(mut subtrie) => { + subtrie.clear(); + Self::Blind(Some(subtrie)) + } + } + } + + /// Takes ownership of the underlying [`SparseSubtrie`] if revealed, putting this + /// `LowerSparseSubtrie` will be put into the blinded state. + /// + /// Otherwise returns None. + pub(crate) fn take_revealed(&mut self) -> Option> { + self.take_revealed_if(|_| true) + } + + /// Takes ownership of the underlying [`SparseSubtrie`] if revealed and the predicate returns + /// true. + /// + /// If the subtrie is revealed, and the predicate function returns `true` when called with it, + /// then this method will take ownership of the subtrie and transition this `LowerSparseSubtrie` + /// to the blinded state. Otherwise, returns `None`. + pub(crate) fn take_revealed_if

{ - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("SparseStateTrie") - .field("state", &self.state) - .field("storages", &self.storages) - .field("revealed_account_paths", &self.revealed_account_paths) - .field("revealed_storage_paths", &self.revealed_storage_paths) - .field("retain_updates", &self.retain_updates) - .field("account_rlp_buf", &hex::encode(&self.account_rlp_buf)) - .finish_non_exhaustive() - } -} - #[cfg(test)] impl SparseStateTrie { /// Create state trie from state trie. @@ -79,28 +122,36 @@ impl SparseStateTrie { } } -impl SparseStateTrie { - /// Create new [`SparseStateTrie`] with blinded node provider factory. - pub fn new(provider_factory: F) -> Self { - Self { - provider_factory, - state: Default::default(), - storages: Default::default(), - revealed_account_paths: Default::default(), - revealed_storage_paths: Default::default(), - retain_updates: false, - account_rlp_buf: Vec::with_capacity(TRIE_ACCOUNT_RLP_MAX_SIZE), - #[cfg(feature = "metrics")] - metrics: Default::default(), - } - } - +impl SparseStateTrie { /// Set the retention of branch node updates and deletions. pub const fn with_updates(mut self, retain_updates: bool) -> Self { self.retain_updates = retain_updates; self } + /// Set the accounts trie to the given `SparseTrie`. + pub fn with_accounts_trie(mut self, trie: SparseTrie) -> Self { + self.state = trie; + self + } + + /// Set the default trie which will be cloned when creating new storage [`SparseTrie`]s. + pub fn with_default_storage_trie(mut self, trie: SparseTrie) -> Self { + self.storage.default_trie = trie; + self + } +} + +impl SparseStateTrie +where + A: SparseTrieInterface + Default, + S: SparseTrieInterface + Default + Clone, +{ + /// Create new [`SparseStateTrie`] + pub fn new() -> Self { + Self::default() + } + /// Returns `true` if account was already revealed. pub fn is_account_revealed(&self, account: B256) -> bool { self.revealed_account_paths.contains(&Nibbles::unpack(account)) @@ -114,10 +165,7 @@ impl SparseStateTrie { None => return false, }; - matches!( - trie.find_leaf(&path, None), - Ok(LeafLookup::Exists | LeafLookup::NonExistent { .. }) - ) + trie.find_leaf(&path, None).is_ok() } /// Was the storage-slot witness for (`address`,`slot`) complete? @@ -128,15 +176,13 @@ impl SparseStateTrie { None => return false, }; - matches!( - trie.find_leaf(&path, None), - Ok(LeafLookup::Exists | LeafLookup::NonExistent { .. }) - ) + trie.find_leaf(&path, None).is_ok() } /// Returns `true` if storage slot for account was already revealed. pub fn is_storage_slot_revealed(&self, account: B256, slot: B256) -> bool { - self.revealed_storage_paths + self.storage + .revealed_paths .get(&account) .is_some_and(|slots| slots.contains(&Nibbles::unpack(slot))) } @@ -148,141 +194,59 @@ impl SparseStateTrie { /// Returns reference to bytes representing leaf value for the target account and storage slot. pub fn get_storage_slot_value(&self, account: &B256, slot: &B256) -> Option<&Vec> { - self.storages.get(account)?.as_revealed_ref()?.get_leaf_value(&Nibbles::unpack(slot)) + self.storage.tries.get(account)?.as_revealed_ref()?.get_leaf_value(&Nibbles::unpack(slot)) } /// Returns reference to state trie if it was revealed. - pub const fn state_trie_ref(&self) -> Option<&RevealedSparseTrie> { + pub const fn state_trie_ref(&self) -> Option<&A> { self.state.as_revealed_ref() } /// Returns reference to storage trie if it was revealed. - pub fn storage_trie_ref( - &self, - address: &B256, - ) -> Option<&RevealedSparseTrie> { - self.storages.get(address).and_then(|e| e.as_revealed_ref()) + pub fn storage_trie_ref(&self, address: &B256) -> Option<&S> { + self.storage.tries.get(address).and_then(|e| e.as_revealed_ref()) } /// Returns mutable reference to storage sparse trie if it was revealed. - pub fn storage_trie_mut( - &mut self, - address: &B256, - ) -> Option<&mut RevealedSparseTrie> { - self.storages.get_mut(address).and_then(|e| e.as_revealed_mut()) + pub fn storage_trie_mut(&mut self, address: &B256) -> Option<&mut S> { + self.storage.tries.get_mut(address).and_then(|e| e.as_revealed_mut()) } /// Takes the storage trie for the provided address. - pub fn take_storage_trie( - &mut self, - address: &B256, - ) -> Option> { - self.storages.remove(address) + pub fn take_storage_trie(&mut self, address: &B256) -> Option> { + self.storage.tries.remove(address) } /// Inserts storage trie for the provided address. - pub fn insert_storage_trie( - &mut self, - address: B256, - storage_trie: SparseTrie, - ) { - self.storages.insert(address, storage_trie); + pub fn insert_storage_trie(&mut self, address: B256, storage_trie: SparseTrie) { + self.storage.tries.insert(address, storage_trie); } - /// Reveal unknown trie paths from provided leaf path and its proof for the account. - /// - /// Panics if trie updates retention is enabled. - /// + /// Reveal unknown trie paths from multiproof. /// NOTE: This method does not extensively validate the proof. - pub fn reveal_account( - &mut self, - account: B256, - proof: impl IntoIterator, - ) -> SparseStateTrieResult<()> { - assert!(!self.retain_updates); - - if self.is_account_revealed(account) { - return Ok(()); - } - - let mut proof = proof.into_iter().peekable(); - - let Some(root_node) = self.validate_root_node(&mut proof)? else { return Ok(()) }; - - // Reveal root node if it wasn't already. - let trie = self.state.reveal_root_with_provider( - self.provider_factory.account_node_provider(), - root_node, - TrieMasks::none(), - self.retain_updates, - )?; - - // Reveal the remaining proof nodes. - for (path, bytes) in proof { - if self.revealed_account_paths.contains(&path) { - continue - } - let node = TrieNode::decode(&mut &bytes[..])?; - trie.reveal_node(path.clone(), node, TrieMasks::none())?; - - // Track the revealed path. - self.revealed_account_paths.insert(path); - } + pub fn reveal_multiproof(&mut self, multiproof: MultiProof) -> SparseStateTrieResult<()> { + // first decode the multiproof + let decoded_multiproof = multiproof.try_into()?; - Ok(()) + // then reveal the decoded multiproof + self.reveal_decoded_multiproof(decoded_multiproof) } - /// Reveal unknown trie paths from provided leaf path and its proof for the storage slot. - /// - /// Panics if trie updates retention is enabled. - /// + /// Reveal unknown trie paths from decoded multiproof. /// NOTE: This method does not extensively validate the proof. - pub fn reveal_storage_slot( + #[instrument( + target = "trie::sparse", + skip_all, + fields( + account_nodes = multiproof.account_subtree.len(), + storages = multiproof.storages.len() + ) + )] + pub fn reveal_decoded_multiproof( &mut self, - account: B256, - slot: B256, - proof: impl IntoIterator, + multiproof: DecodedMultiProof, ) -> SparseStateTrieResult<()> { - assert!(!self.retain_updates); - - if self.is_storage_slot_revealed(account, slot) { - return Ok(()); - } - - let mut proof = proof.into_iter().peekable(); - - let Some(root_node) = self.validate_root_node(&mut proof)? else { return Ok(()) }; - - // Reveal root node if it wasn't already. - let trie = self.storages.entry(account).or_default().reveal_root_with_provider( - self.provider_factory.storage_node_provider(account), - root_node, - TrieMasks::none(), - self.retain_updates, - )?; - - let revealed_nodes = self.revealed_storage_paths.entry(account).or_default(); - - // Reveal the remaining proof nodes. - for (path, bytes) in proof { - // If the node is already revealed, skip it. - if revealed_nodes.contains(&path) { - continue - } - let node = TrieNode::decode(&mut &bytes[..])?; - trie.reveal_node(path.clone(), node, TrieMasks::none())?; - - // Track the revealed path. - revealed_nodes.insert(path); - } - - Ok(()) - } - - /// Reveal unknown trie paths from multiproof. - /// NOTE: This method does not extensively validate the proof. - pub fn reveal_multiproof(&mut self, multiproof: MultiProof) -> SparseStateTrieResult<()> { - let MultiProof { + let DecodedMultiProof { account_subtree, storages, branch_node_hash_masks, @@ -290,18 +254,77 @@ impl SparseStateTrie { } = multiproof; // first reveal the account proof nodes - self.reveal_account_multiproof( + self.reveal_decoded_account_multiproof( account_subtree, branch_node_hash_masks, branch_node_tree_masks, )?; - // then reveal storage proof nodes for each storage trie - for (account, storage_subtree) in storages { - self.reveal_storage_multiproof(account, storage_subtree)?; + #[cfg(not(feature = "std"))] + // If nostd then serially reveal storage proof nodes for each storage trie + { + for (account, storage_subtree) in storages { + self.reveal_decoded_storage_multiproof(account, storage_subtree)?; + } + + Ok(()) } - Ok(()) + #[cfg(feature = "std")] + // If std then reveal storage proofs in parallel + { + use rayon::iter::{ParallelBridge, ParallelIterator}; + + let (tx, rx) = std::sync::mpsc::channel(); + let retain_updates = self.retain_updates; + + // Process all storage trie revealings in parallel, having first removed the + // `reveal_nodes` tracking and `SparseTrie`s for each account from their HashMaps. + // These will be returned after processing. + storages + .into_iter() + .map(|(account, storage_subtree)| { + let revealed_nodes = self.storage.take_or_create_revealed_paths(&account); + let trie = self.storage.take_or_create_trie(&account); + (account, storage_subtree, revealed_nodes, trie) + }) + .par_bridge() + .map(|(account, storage_subtree, mut revealed_nodes, mut trie)| { + let result = Self::reveal_decoded_storage_multiproof_inner( + account, + storage_subtree, + &mut revealed_nodes, + &mut trie, + retain_updates, + ); + + (account, revealed_nodes, trie, result) + }) + .for_each_init(|| tx.clone(), |tx, result| tx.send(result).unwrap()); + + drop(tx); + + // Return `revealed_nodes` and `SparseTrie` for each account, incrementing metrics and + // returning the last error seen if any. + let mut any_err = Ok(()); + for (account, revealed_nodes, trie, result) in rx { + self.storage.revealed_paths.insert(account, revealed_nodes); + self.storage.tries.insert(account, trie); + if let Ok(_metric_values) = result { + #[cfg(feature = "metrics")] + { + self.metrics + .increment_total_storage_nodes(_metric_values.total_nodes as u64); + self.metrics + .increment_skipped_storage_nodes(_metric_values.skipped_nodes as u64); + } + } else { + any_err = result.map(|_| ()); + } + } + + any_err + } } /// Reveals an account multiproof. @@ -311,51 +334,47 @@ impl SparseStateTrie { branch_node_hash_masks: HashMap, branch_node_tree_masks: HashMap, ) -> SparseStateTrieResult<()> { - let DecodedProofNodes { - nodes, - new_nodes, - total_nodes: _total_nodes, - skipped_nodes: _skipped_nodes, - } = decode_proof_nodes(account_subtree, &self.revealed_account_paths)?; + // decode the multiproof first + let decoded_multiproof = account_subtree.try_into()?; + self.reveal_decoded_account_multiproof( + decoded_multiproof, + branch_node_hash_masks, + branch_node_tree_masks, + ) + } + + /// Reveals a decoded account multiproof. + pub fn reveal_decoded_account_multiproof( + &mut self, + account_subtree: DecodedProofNodes, + branch_node_hash_masks: HashMap, + branch_node_tree_masks: HashMap, + ) -> SparseStateTrieResult<()> { + let FilterMappedProofNodes { root_node, nodes, new_nodes, metric_values: _metric_values } = + filter_map_revealed_nodes( + account_subtree, + &mut self.revealed_account_paths, + &branch_node_hash_masks, + &branch_node_tree_masks, + )?; #[cfg(feature = "metrics")] { - self.metrics.increment_total_account_nodes(_total_nodes as u64); - self.metrics.increment_skipped_account_nodes(_skipped_nodes as u64); + self.metrics.increment_total_account_nodes(_metric_values.total_nodes as u64); + self.metrics.increment_skipped_account_nodes(_metric_values.skipped_nodes as u64); } - let mut account_nodes = nodes.into_iter().peekable(); - if let Some(root_node) = Self::validate_root_node_decoded(&mut account_nodes)? { + if let Some(root_node) = root_node { // Reveal root node if it wasn't already. - let trie = self.state.reveal_root_with_provider( - self.provider_factory.account_node_provider(), - root_node, - TrieMasks { - hash_mask: branch_node_hash_masks.get(&Nibbles::default()).copied(), - tree_mask: branch_node_tree_masks.get(&Nibbles::default()).copied(), - }, - self.retain_updates, - )?; + trace!(target: "trie::sparse", ?root_node, "Revealing root account node"); + let trie = + self.state.reveal_root(root_node.node, root_node.masks, self.retain_updates)?; - // Reserve the capacity for new nodes ahead of time. + // Reserve the capacity for new nodes ahead of time, if the trie implementation + // supports doing so. trie.reserve_nodes(new_nodes); - // Reveal the remaining proof nodes. - for (path, node) in account_nodes { - let (hash_mask, tree_mask) = if let TrieNode::Branch(_) = node { - ( - branch_node_hash_masks.get(&path).copied(), - branch_node_tree_masks.get(&path).copied(), - ) - } else { - (None, None) - }; - - trace!(target: "trie::sparse", ?path, ?node, ?hash_mask, ?tree_mask, "Revealing account node"); - trie.reveal_node(path.clone(), node, TrieMasks { hash_mask, tree_mask })?; - - // Track the revealed path. - self.revealed_account_paths.insert(path); - } + trace!(target: "trie::sparse", total_nodes = ?nodes.len(), "Revealing account nodes"); + trie.reveal_nodes(nodes)?; } Ok(()) @@ -367,62 +386,66 @@ impl SparseStateTrie { account: B256, storage_subtree: StorageMultiProof, ) -> SparseStateTrieResult<()> { - let revealed_nodes = self.revealed_storage_paths.entry(account).or_default(); - - let DecodedProofNodes { - nodes, - new_nodes, - total_nodes: _total_nodes, - skipped_nodes: _skipped_nodes, - } = decode_proof_nodes(storage_subtree.subtree, revealed_nodes)?; + // decode the multiproof first + let decoded_multiproof = storage_subtree.try_into()?; + self.reveal_decoded_storage_multiproof(account, decoded_multiproof) + } + + /// Reveals a decoded storage multiproof for the given address. + pub fn reveal_decoded_storage_multiproof( + &mut self, + account: B256, + storage_subtree: DecodedStorageMultiProof, + ) -> SparseStateTrieResult<()> { + let (trie, revealed_paths) = self.storage.get_trie_and_revealed_paths_mut(account); + let _metric_values = Self::reveal_decoded_storage_multiproof_inner( + account, + storage_subtree, + revealed_paths, + trie, + self.retain_updates, + )?; + #[cfg(feature = "metrics")] { - self.metrics.increment_total_storage_nodes(_total_nodes as u64); - self.metrics.increment_skipped_storage_nodes(_skipped_nodes as u64); + self.metrics.increment_total_storage_nodes(_metric_values.total_nodes as u64); + self.metrics.increment_skipped_storage_nodes(_metric_values.skipped_nodes as u64); } - let mut nodes = nodes.into_iter().peekable(); - if let Some(root_node) = Self::validate_root_node_decoded(&mut nodes)? { - // Reveal root node if it wasn't already. - let trie = self.storages.entry(account).or_default().reveal_root_with_provider( - self.provider_factory.storage_node_provider(account), - root_node, - TrieMasks { - hash_mask: storage_subtree - .branch_node_hash_masks - .get(&Nibbles::default()) - .copied(), - tree_mask: storage_subtree - .branch_node_tree_masks - .get(&Nibbles::default()) - .copied(), - }, - self.retain_updates, - )?; + Ok(()) + } - // Reserve the capacity for new nodes ahead of time. - trie.reserve_nodes(new_nodes); + /// Reveals a decoded storage multiproof for the given address. This is internal static function + /// is designed to handle a variety of associated public functions. + fn reveal_decoded_storage_multiproof_inner( + account: B256, + storage_subtree: DecodedStorageMultiProof, + revealed_nodes: &mut HashSet, + trie: &mut SparseTrie, + retain_updates: bool, + ) -> SparseStateTrieResult { + let FilterMappedProofNodes { root_node, nodes, new_nodes, metric_values } = + filter_map_revealed_nodes( + storage_subtree.subtree, + revealed_nodes, + &storage_subtree.branch_node_hash_masks, + &storage_subtree.branch_node_tree_masks, + )?; - // Reveal the remaining proof nodes. - for (path, node) in nodes { - let (hash_mask, tree_mask) = if let TrieNode::Branch(_) = node { - ( - storage_subtree.branch_node_hash_masks.get(&path).copied(), - storage_subtree.branch_node_tree_masks.get(&path).copied(), - ) - } else { - (None, None) - }; + if let Some(root_node) = root_node { + // Reveal root node if it wasn't already. + trace!(target: "trie::sparse", ?account, ?root_node, "Revealing root storage node"); + let trie = trie.reveal_root(root_node.node, root_node.masks, retain_updates)?; - trace!(target: "trie::sparse", ?account, ?path, ?node, ?hash_mask, ?tree_mask, "Revealing storage node"); - trie.reveal_node(path.clone(), node, TrieMasks { hash_mask, tree_mask })?; + // Reserve the capacity for new nodes ahead of time, if the trie implementation + // supports doing so. + trie.reserve_nodes(new_nodes); - // Track the revealed path. - revealed_nodes.insert(path); - } + trace!(target: "trie::sparse", ?account, total_nodes = ?nodes.len(), "Revealing storage nodes"); + trie.reveal_nodes(nodes)?; } - Ok(()) + Ok(metric_values) } /// Reveal state witness with the given state root. @@ -447,7 +470,7 @@ impl SparseStateTrie { TrieNode::Branch(branch) => { for (idx, maybe_child) in branch.as_ref().children() { if let Some(child_hash) = maybe_child.and_then(RlpNode::as_hash) { - let mut child_path = path.clone(); + let mut child_path = path; child_path.push_unchecked(idx); queue.push_back((child_hash, child_path, maybe_account)); } @@ -455,14 +478,14 @@ impl SparseStateTrie { } TrieNode::Extension(ext) => { if let Some(child_hash) = ext.child.as_hash() { - let mut child_path = path.clone(); - child_path.extend_from_slice_unchecked(&ext.key); + let mut child_path = path; + child_path.extend(&ext.key); queue.push_back((child_hash, child_path, maybe_account)); } } TrieNode::Leaf(leaf) => { - let mut full_path = path.clone(); - full_path.extend_from_slice_unchecked(&leaf.key); + let mut full_path = path; + full_path.extend(&leaf.key); if maybe_account.is_none() { let hashed_address = B256::from_slice(&full_path.pack()); let account = TrieAccount::decode(&mut &leaf.value[..])?; @@ -482,45 +505,43 @@ impl SparseStateTrie { if let Some(account) = maybe_account { // Check that the path was not already revealed. if self - .revealed_storage_paths + .storage + .revealed_paths .get(&account) .is_none_or(|paths| !paths.contains(&path)) { - let storage_trie_entry = self.storages.entry(account).or_default(); + let retain_updates = self.retain_updates; + let (storage_trie_entry, revealed_storage_paths) = + self.storage.get_trie_and_revealed_paths_mut(account); + if path.is_empty() { // Handle special storage state root node case. - storage_trie_entry.reveal_root_with_provider( - self.provider_factory.storage_node_provider(account), + storage_trie_entry.reveal_root( trie_node, TrieMasks::none(), - self.retain_updates, + retain_updates, )?; } else { // Reveal non-root storage trie node. storage_trie_entry .as_revealed_mut() .ok_or(SparseTrieErrorKind::Blind)? - .reveal_node(path.clone(), trie_node, TrieMasks::none())?; + .reveal_node(path, trie_node, TrieMasks::none())?; } // Track the revealed path. - self.revealed_storage_paths.entry(account).or_default().insert(path); + revealed_storage_paths.insert(path); } } // Check that the path was not already revealed. else if !self.revealed_account_paths.contains(&path) { if path.is_empty() { // Handle special state root node case. - self.state.reveal_root_with_provider( - self.provider_factory.account_node_provider(), - trie_node, - TrieMasks::none(), - self.retain_updates, - )?; + self.state.reveal_root(trie_node, TrieMasks::none(), self.retain_updates)?; } else { // Reveal non-root state trie node. self.state.as_revealed_mut().ok_or(SparseTrieErrorKind::Blind)?.reveal_node( - path.clone(), + path, trie_node, TrieMasks::none(), )?; @@ -534,86 +555,41 @@ impl SparseStateTrie { Ok(()) } - /// Validates the root node of the proof and returns it if it exists and is valid. - fn validate_root_node>( - &self, - proof: &mut Peekable, - ) -> SparseStateTrieResult> { - // Validate root node. - let Some((path, node)) = proof.next() else { return Ok(None) }; - if !path.is_empty() { - return Err(SparseStateTrieErrorKind::InvalidRootNode { path, node }.into()) - } - - // Decode root node and perform sanity check. - let root_node = TrieNode::decode(&mut &node[..])?; - if matches!(root_node, TrieNode::EmptyRoot) && proof.peek().is_some() { - return Err(SparseStateTrieErrorKind::InvalidRootNode { path, node }.into()) - } - - Ok(Some(root_node)) - } - - /// Validates the decoded root node of the proof and returns it if it exists and is valid. - fn validate_root_node_decoded>( - proof: &mut Peekable, - ) -> SparseStateTrieResult> { - // Validate root node. - let Some((path, root_node)) = proof.next() else { return Ok(None) }; - if !path.is_empty() { - return Err(SparseStateTrieErrorKind::InvalidRootNode { - path, - node: alloy_rlp::encode(&root_node).into(), - } - .into()) - } - - // Perform sanity check. - if matches!(root_node, TrieNode::EmptyRoot) && proof.peek().is_some() { - return Err(SparseStateTrieErrorKind::InvalidRootNode { - path, - node: alloy_rlp::encode(&root_node).into(), - } - .into()) - } - - Ok(Some(root_node)) - } - /// Wipe the storage trie at the provided address. pub fn wipe_storage(&mut self, address: B256) -> SparseStateTrieResult<()> { - if let Some(trie) = self.storages.get_mut(&address) { + if let Some(trie) = self.storage.tries.get_mut(&address) { trie.wipe()?; } Ok(()) } - /// Calculates the hashes of the nodes below the provided level. + /// Calculates the hashes of subtries. /// /// If the trie has not been revealed, this function does nothing. - pub fn calculate_below_level(&mut self, level: usize) { + #[instrument(target = "trie::sparse", skip_all)] + pub fn calculate_subtries(&mut self) { if let SparseTrie::Revealed(trie) = &mut self.state { - trie.update_rlp_node_level(level); + trie.update_subtrie_hashes(); } } /// Returns storage sparse trie root if the trie has been revealed. pub fn storage_root(&mut self, account: B256) -> Option { - self.storages.get_mut(&account).and_then(|trie| trie.root()) + self.storage.tries.get_mut(&account).and_then(|trie| trie.root()) } - /// Returns mutable reference to the revealed sparse trie. + /// Returns mutable reference to the revealed account sparse trie. /// - /// If the trie is not revealed yet, its root will be revealed using the blinded node provider. + /// If the trie is not revealed yet, its root will be revealed using the trie node provider. fn revealed_trie_mut( &mut self, - ) -> SparseStateTrieResult<&mut RevealedSparseTrie> { + provider_factory: impl TrieNodeProviderFactory, + ) -> SparseStateTrieResult<&mut A> { match self.state { - SparseTrie::Blind => { - let (root_node, hash_mask, tree_mask) = self - .provider_factory + SparseTrie::Blind(_) => { + let (root_node, hash_mask, tree_mask) = provider_factory .account_node_provider() - .blinded_node(&Nibbles::default())? + .trie_node(&Nibbles::default())? .map(|node| { TrieNode::decode(&mut &node.node[..]) .map(|decoded| (decoded, node.hash_mask, node.tree_mask)) @@ -621,12 +597,7 @@ impl SparseStateTrie { .transpose()? .unwrap_or((TrieNode::EmptyRoot, None, None)); self.state - .reveal_root_with_provider( - self.provider_factory.account_node_provider(), - root_node, - TrieMasks { hash_mask, tree_mask }, - self.retain_updates, - ) + .reveal_root(root_node, TrieMasks { hash_mask, tree_mask }, self.retain_updates) .map_err(Into::into) } SparseTrie::Revealed(ref mut trie) => Ok(trie), @@ -636,22 +607,29 @@ impl SparseStateTrie { /// Returns sparse trie root. /// /// If the trie has not been revealed, this function reveals the root node and returns its hash. - pub fn root(&mut self) -> SparseStateTrieResult { + pub fn root( + &mut self, + provider_factory: impl TrieNodeProviderFactory, + ) -> SparseStateTrieResult { // record revealed node metrics #[cfg(feature = "metrics")] self.metrics.record(); - Ok(self.revealed_trie_mut()?.root()) + Ok(self.revealed_trie_mut(provider_factory)?.root()) } /// Returns sparse trie root and trie updates if the trie has been revealed. - pub fn root_with_updates(&mut self) -> SparseStateTrieResult<(B256, TrieUpdates)> { + #[instrument(target = "trie::sparse", skip_all)] + pub fn root_with_updates( + &mut self, + provider_factory: impl TrieNodeProviderFactory, + ) -> SparseStateTrieResult<(B256, TrieUpdates)> { // record revealed node metrics #[cfg(feature = "metrics")] self.metrics.record(); let storage_tries = self.storage_trie_updates(); - let revealed = self.revealed_trie_mut()?; + let revealed = self.revealed_trie_mut(provider_factory)?; let (root, updates) = (revealed.root(), revealed.take_updates()); let updates = TrieUpdates { @@ -666,7 +644,8 @@ impl SparseStateTrie { /// /// Panics if any of the storage tries are not revealed. pub fn storage_trie_updates(&mut self) -> B256Map { - self.storages + self.storage + .tries .iter_mut() .map(|(address, trie)| { let trie = trie.as_revealed_mut().unwrap(); @@ -702,46 +681,55 @@ impl SparseStateTrie { &mut self, path: Nibbles, value: Vec, + provider_factory: impl TrieNodeProviderFactory, ) -> SparseStateTrieResult<()> { if !self.revealed_account_paths.contains(&path) { - self.revealed_account_paths.insert(path.clone()); + self.revealed_account_paths.insert(path); } - self.state.update_leaf(path, value)?; + let provider = provider_factory.account_node_provider(); + self.state.update_leaf(path, value, provider)?; Ok(()) } - /// Update the leaf node of a storage trie at the provided address. + /// Update the leaf node of a revealed storage trie at the provided address. pub fn update_storage_leaf( &mut self, address: B256, slot: Nibbles, value: Vec, + provider_factory: impl TrieNodeProviderFactory, ) -> SparseStateTrieResult<()> { - if !self.revealed_storage_paths.get(&address).is_some_and(|slots| slots.contains(&slot)) { - self.revealed_storage_paths.entry(address).or_default().insert(slot.clone()); - } - - let storage_trie = self.storages.get_mut(&address).ok_or(SparseTrieErrorKind::Blind)?; - storage_trie.update_leaf(slot, value)?; + let provider = provider_factory.storage_node_provider(address); + self.storage + .tries + .get_mut(&address) + .ok_or(SparseTrieErrorKind::Blind)? + .update_leaf(slot, value, provider)?; + self.storage.get_revealed_paths_mut(address).insert(slot); Ok(()) } /// Update or remove trie account based on new account info. This method will either recompute /// the storage root based on update storage trie or look it up from existing leaf value. /// - /// If the new account info and storage trie are empty, the account leaf will be removed. - pub fn update_account(&mut self, address: B256, account: Account) -> SparseStateTrieResult<()> { - let nibbles = Nibbles::unpack(address); - - let storage_root = if let Some(storage_trie) = self.storages.get_mut(&address) { + /// Returns false if the new account info and storage trie are empty, indicating the account + /// leaf should be removed. + #[instrument(level = "trace", target = "trie::sparse", skip_all)] + pub fn update_account( + &mut self, + address: B256, + account: Account, + provider_factory: impl TrieNodeProviderFactory, + ) -> SparseStateTrieResult { + let storage_root = if let Some(storage_trie) = self.storage.tries.get_mut(&address) { trace!(target: "trie::sparse", ?address, "Calculating storage root to update account"); storage_trie.root().ok_or(SparseTrieErrorKind::Blind)? } else if self.is_account_revealed(address) { trace!(target: "trie::sparse", ?address, "Retrieving storage root from account leaf to update account"); // The account was revealed, either... if let Some(value) = self.get_account_value(&address) { - // ..it exists and we should take it's current storage root or... + // ..it exists and we should take its current storage root or... TrieAccount::decode(&mut &value[..])?.storage_root } else { // ...the account is newly created and the storage trie is empty. @@ -752,23 +740,30 @@ impl SparseStateTrie { }; if account.is_empty() && storage_root == EMPTY_ROOT_HASH { - trace!(target: "trie::sparse", ?address, "Removing account"); - self.remove_account_leaf(&nibbles) - } else { - trace!(target: "trie::sparse", ?address, "Updating account"); - self.account_rlp_buf.clear(); - account.into_trie_account(storage_root).encode(&mut self.account_rlp_buf); - self.update_account_leaf(nibbles, self.account_rlp_buf.clone()) + return Ok(false); } + + trace!(target: "trie::sparse", ?address, "Updating account"); + let nibbles = Nibbles::unpack(address); + self.account_rlp_buf.clear(); + account.into_trie_account(storage_root).encode(&mut self.account_rlp_buf); + self.update_account_leaf(nibbles, self.account_rlp_buf.clone(), provider_factory)?; + + Ok(true) } /// Update the storage root of a revealed account. /// /// If the account doesn't exist in the trie, the function is a no-op. /// - /// If the new storage root is empty, and the account info was already empty, the account leaf - /// will be removed. - pub fn update_account_storage_root(&mut self, address: B256) -> SparseStateTrieResult<()> { + /// Returns false if the new storage root is empty, and the account info was already empty, + /// indicating the account leaf should be removed. + #[instrument(target = "trie::sparse", skip_all)] + pub fn update_account_storage_root( + &mut self, + address: B256, + provider_factory: impl TrieNodeProviderFactory, + ) -> SparseStateTrieResult { if !self.is_account_revealed(address) { return Err(SparseTrieErrorKind::Blind.into()) } @@ -780,12 +775,12 @@ impl SparseStateTrie { .transpose()? else { trace!(target: "trie::sparse", ?address, "Account not found in trie, skipping storage root update"); - return Ok(()) + return Ok(true) }; // Calculate the new storage root. If the storage trie doesn't exist, the storage root will // be empty. - let storage_root = if let Some(storage_trie) = self.storages.get_mut(&address) { + let storage_root = if let Some(storage_trie) = self.storage.tries.get_mut(&address) { trace!(target: "trie::sparse", ?address, "Calculating storage root to update account"); storage_trie.root().ok_or(SparseTrieErrorKind::Blind)? } else { @@ -795,25 +790,30 @@ impl SparseStateTrie { // Update the account with the new storage root. trie_account.storage_root = storage_root; - let nibbles = Nibbles::unpack(address); + // If the account is empty, indicate that it should be removed. if trie_account == TrieAccount::default() { - // If the account is empty, remove it. - trace!(target: "trie::sparse", ?address, "Removing account because the storage root is empty"); - self.remove_account_leaf(&nibbles)?; - } else { - // Otherwise, update the account leaf. - trace!(target: "trie::sparse", ?address, "Updating account with the new storage root"); - self.account_rlp_buf.clear(); - trie_account.encode(&mut self.account_rlp_buf); - self.update_account_leaf(nibbles, self.account_rlp_buf.clone())?; + return Ok(false) } - Ok(()) + // Otherwise, update the account leaf. + trace!(target: "trie::sparse", ?address, "Updating account with the new storage root"); + let nibbles = Nibbles::unpack(address); + self.account_rlp_buf.clear(); + trie_account.encode(&mut self.account_rlp_buf); + self.update_account_leaf(nibbles, self.account_rlp_buf.clone(), provider_factory)?; + + Ok(true) } /// Remove the account leaf node. - pub fn remove_account_leaf(&mut self, path: &Nibbles) -> SparseStateTrieResult<()> { - self.state.remove_leaf(path)?; + #[instrument(target = "trie::sparse", skip_all)] + pub fn remove_account_leaf( + &mut self, + path: &Nibbles, + provider_factory: impl TrieNodeProviderFactory, + ) -> SparseStateTrieResult<()> { + let provider = provider_factory.account_node_provider(); + self.state.remove_leaf(path, provider)?; Ok(()) } @@ -822,143 +822,235 @@ impl SparseStateTrie { &mut self, address: B256, slot: &Nibbles, + provider_factory: impl TrieNodeProviderFactory, ) -> SparseStateTrieResult<()> { - let storage_trie = self.storages.get_mut(&address).ok_or(SparseTrieErrorKind::Blind)?; - storage_trie.remove_leaf(slot)?; + let storage_trie = + self.storage.tries.get_mut(&address).ok_or(SparseTrieErrorKind::Blind)?; + + let provider = provider_factory.storage_node_provider(address); + storage_trie.remove_leaf(slot, provider)?; Ok(()) } } -/// Result of [`decode_proof_nodes`]. -#[derive(Debug, PartialEq, Eq)] -struct DecodedProofNodes { - /// Filtered, decoded and sorted proof nodes. - nodes: Vec<(Nibbles, TrieNode)>, +/// The fields of [`SparseStateTrie`] related to storage tries. This is kept separate from the rest +/// of [`SparseStateTrie`] both to help enforce allocation re-use and to allow us to implement +/// methods like `get_trie_and_revealed_paths` which return multiple mutable borrows. +#[derive(Debug, Default)] +struct StorageTries { + /// Sparse storage tries. + tries: B256Map>, + /// Cleared storage tries, kept for re-use. + cleared_tries: Vec>, + /// Collection of revealed storage trie paths, per account. + revealed_paths: B256Map>, + /// Cleared revealed storage trie path collections, kept for re-use. + cleared_revealed_paths: Vec>, + /// A default cleared trie instance, which will be cloned when creating new tries. + default_trie: SparseTrie, +} + +impl StorageTries { + /// Returns all fields to a cleared state, equivalent to the default state, keeping cleared + /// collections for re-use later when possible. + fn clear(&mut self) { + self.cleared_tries.extend(self.tries.drain().map(|(_, trie)| trie.clear())); + self.cleared_revealed_paths.extend(self.revealed_paths.drain().map(|(_, mut set)| { + set.clear(); + set + })); + } + + /// Shrinks the capacity of all storage tries (active, cleared, and default) to the given sizes. + /// The capacity is distributed equally among all tries that have allocations. + fn shrink_to(&mut self, node_size: usize, value_size: usize) { + // Count total number of tries with capacity (active + cleared + default) + let active_count = self.tries.len(); + let cleared_count = self.cleared_tries.len(); + let total_tries = 1 + active_count + cleared_count; + + // Distribute capacity equally among all tries + let node_size_per_trie = node_size / total_tries; + let value_size_per_trie = value_size / total_tries; + + // Shrink active storage tries + for trie in self.tries.values_mut() { + trie.shrink_nodes_to(node_size_per_trie); + trie.shrink_values_to(value_size_per_trie); + } + + // Shrink cleared storage tries + for trie in &mut self.cleared_tries { + trie.shrink_nodes_to(node_size_per_trie); + trie.shrink_values_to(value_size_per_trie); + } + } +} + +impl StorageTries { + /// Returns the set of already revealed trie node paths for an account's storage, creating the + /// set if it didn't previously exist. + fn get_revealed_paths_mut(&mut self, account: B256) -> &mut HashSet { + self.revealed_paths + .entry(account) + .or_insert_with(|| self.cleared_revealed_paths.pop().unwrap_or_default()) + } + + /// Returns the `SparseTrie` and the set of already revealed trie node paths for an account's + /// storage, creating them if they didn't previously exist. + fn get_trie_and_revealed_paths_mut( + &mut self, + account: B256, + ) -> (&mut SparseTrie, &mut HashSet) { + let trie = self.tries.entry(account).or_insert_with(|| { + self.cleared_tries.pop().unwrap_or_else(|| self.default_trie.clone()) + }); + + let revealed_paths = self + .revealed_paths + .entry(account) + .or_insert_with(|| self.cleared_revealed_paths.pop().unwrap_or_default()); + + (trie, revealed_paths) + } + + /// Takes the storage trie for the account from the internal `HashMap`, creating it if it + /// doesn't already exist. + #[cfg(feature = "std")] + fn take_or_create_trie(&mut self, account: &B256) -> SparseTrie { + self.tries.remove(account).unwrap_or_else(|| { + self.cleared_tries.pop().unwrap_or_else(|| self.default_trie.clone()) + }) + } + + /// Takes the revealed paths set from the account from the internal `HashMap`, creating one if + /// it doesn't exist. + #[cfg(feature = "std")] + fn take_or_create_revealed_paths(&mut self, account: &B256) -> HashSet { + self.revealed_paths + .remove(account) + .unwrap_or_else(|| self.cleared_revealed_paths.pop().unwrap_or_default()) + } +} + +#[derive(Debug, PartialEq, Eq, Default)] +struct ProofNodesMetricValues { /// Number of nodes in the proof. total_nodes: usize, /// Number of nodes that were skipped because they were already revealed. skipped_nodes: usize, +} + +/// Result of [`filter_map_revealed_nodes`]. +#[derive(Debug, PartialEq, Eq)] +struct FilterMappedProofNodes { + /// Root node which was pulled out of the original node set to be handled specially. + root_node: Option, + /// Filtered, decoded and unsorted proof nodes. Root node is removed. + nodes: Vec, /// Number of new nodes that will be revealed. This includes all children of branch nodes, even /// if they are not in the proof. new_nodes: usize, + /// Values which are being returned so they can be incremented into metrics. + metric_values: ProofNodesMetricValues, } -/// Decodes the proof nodes returning additional information about the number of total, skipped, and -/// new nodes. -fn decode_proof_nodes( - proof_nodes: ProofNodes, - revealed_nodes: &HashSet, -) -> alloy_rlp::Result { - let mut result = DecodedProofNodes { +/// Filters the decoded nodes that are already revealed, maps them to `RevealedSparseNodes`, +/// separates the root node if present, and returns additional information about the number of +/// total, skipped, and new nodes. +fn filter_map_revealed_nodes( + proof_nodes: DecodedProofNodes, + revealed_nodes: &mut HashSet, + branch_node_hash_masks: &HashMap, + branch_node_tree_masks: &HashMap, +) -> SparseStateTrieResult { + let mut result = FilterMappedProofNodes { + root_node: None, nodes: Vec::with_capacity(proof_nodes.len()), - total_nodes: 0, - skipped_nodes: 0, new_nodes: 0, + metric_values: Default::default(), }; - for (path, bytes) in proof_nodes.into_inner() { - result.total_nodes += 1; - // If the node is already revealed, skip it. - if revealed_nodes.contains(&path) { - result.skipped_nodes += 1; + let proof_nodes_len = proof_nodes.len(); + for (path, proof_node) in proof_nodes.into_inner() { + result.metric_values.total_nodes += 1; + + let is_root = path.is_empty(); + + // If the node is already revealed, skip it. We don't ever skip the root node, nor do we add + // it to `revealed_nodes`. + if !is_root && !revealed_nodes.insert(path) { + result.metric_values.skipped_nodes += 1; continue } - let node = TrieNode::decode(&mut &bytes[..])?; result.new_nodes += 1; - // If it's a branch node, increase the number of new nodes by the number of children - // according to the state mask. - if let TrieNode::Branch(branch) = &node { - result.new_nodes += branch.state_mask.count_ones() as usize; + + // Extract hash/tree masks based on the node type (only branch nodes have masks). At the + // same time increase the new_nodes counter if the node is a type which has children. + let masks = match &proof_node { + TrieNode::Branch(branch) => { + // If it's a branch node, increase the number of new nodes by the number of children + // according to the state mask. + result.new_nodes += branch.state_mask.count_ones() as usize; + TrieMasks { + hash_mask: branch_node_hash_masks.get(&path).copied(), + tree_mask: branch_node_tree_masks.get(&path).copied(), + } + } + TrieNode::Extension(_) => { + // There is always exactly one child of an extension node. + result.new_nodes += 1; + TrieMasks::none() + } + _ => TrieMasks::none(), + }; + + let node = RevealedSparseNode { path, node: proof_node, masks }; + + if is_root { + // Perform sanity check. + if matches!(node.node, TrieNode::EmptyRoot) && proof_nodes_len > 1 { + return Err(SparseStateTrieErrorKind::InvalidRootNode { + path, + node: alloy_rlp::encode(&node.node).into(), + } + .into()) + } + + result.root_node = Some(node); + + continue } - result.nodes.push((path, node)); + result.nodes.push(node); } - result.nodes.sort_unstable_by(|a, b| a.0.cmp(&b.0)); Ok(result) } #[cfg(test)] mod tests { use super::*; + use crate::provider::DefaultTrieNodeProviderFactory; use alloy_primitives::{ b256, map::{HashMap, HashSet}, - Bytes, U256, + U256, }; - use alloy_rlp::EMPTY_STRING_CODE; use arbitrary::Arbitrary; - use assert_matches::assert_matches; use rand::{rngs::StdRng, Rng, SeedableRng}; use reth_primitives_traits::Account; - use reth_trie::{updates::StorageTrieUpdates, HashBuilder, EMPTY_ROOT_HASH}; + use reth_trie::{updates::StorageTrieUpdates, HashBuilder, MultiProof, EMPTY_ROOT_HASH}; use reth_trie_common::{ proof::{ProofNodes, ProofRetainer}, BranchNode, LeafNode, StorageMultiProof, TrieMask, }; - #[test] - fn validate_root_node_first_node_not_root() { - let sparse = SparseStateTrie::default(); - let proof = [(Nibbles::from_nibbles([0x1]), Bytes::from([EMPTY_STRING_CODE]))]; - assert_matches!( - sparse.validate_root_node(&mut proof.into_iter().peekable()).map_err(|e| e.into_kind()), - Err(SparseStateTrieErrorKind::InvalidRootNode { .. }) - ); - } - - #[test] - fn validate_root_node_invalid_proof_with_empty_root() { - let sparse = SparseStateTrie::default(); - let proof = [ - (Nibbles::default(), Bytes::from([EMPTY_STRING_CODE])), - (Nibbles::from_nibbles([0x1]), Bytes::new()), - ]; - assert_matches!( - sparse.validate_root_node(&mut proof.into_iter().peekable()).map_err(|e| e.into_kind()), - Err(SparseStateTrieErrorKind::InvalidRootNode { .. }) - ); - } - - #[test] - fn reveal_account_empty() { - let retainer = ProofRetainer::from_iter([Nibbles::default()]); - let mut hash_builder = HashBuilder::default().with_proof_retainer(retainer); - hash_builder.root(); - let proofs = hash_builder.take_proof_nodes(); - assert_eq!(proofs.len(), 1); - - let mut sparse = SparseStateTrie::default(); - assert_eq!(sparse.state, SparseTrie::Blind); - - sparse.reveal_account(Default::default(), proofs.into_inner()).unwrap(); - assert_eq!(sparse.state, SparseTrie::revealed_empty()); - } - - #[test] - fn reveal_storage_slot_empty() { - let retainer = ProofRetainer::from_iter([Nibbles::default()]); - let mut hash_builder = HashBuilder::default().with_proof_retainer(retainer); - hash_builder.root(); - let proofs = hash_builder.take_proof_nodes(); - assert_eq!(proofs.len(), 1); - - let mut sparse = SparseStateTrie::default(); - assert!(sparse.storages.is_empty()); - - sparse - .reveal_storage_slot(Default::default(), Default::default(), proofs.into_inner()) - .unwrap(); - assert_eq!( - sparse.storages, - HashMap::from_iter([(Default::default(), SparseTrie::revealed_empty())]) - ); - } - #[test] fn reveal_account_path_twice() { - let mut sparse = SparseStateTrie::default(); + let provider_factory = DefaultTrieNodeProviderFactory; + let mut sparse = SparseStateTrie::::default(); let leaf_value = alloy_rlp::encode(TrieAccount::default()); let leaf_1 = alloy_rlp::encode(TrieNode::Leaf(LeafNode::new( @@ -987,7 +1079,7 @@ mod tests { }; // Reveal multiproof and check that the state trie contains the leaf node and value - sparse.reveal_multiproof(multiproof.clone()).unwrap(); + sparse.reveal_decoded_multiproof(multiproof.clone().try_into().unwrap()).unwrap(); assert!(sparse .state_trie_ref() .unwrap() @@ -1000,7 +1092,7 @@ mod tests { // Remove the leaf node and check that the state trie does not contain the leaf node and // value - sparse.remove_account_leaf(&Nibbles::from_nibbles([0x0])).unwrap(); + sparse.remove_account_leaf(&Nibbles::from_nibbles([0x0]), &provider_factory).unwrap(); assert!(!sparse .state_trie_ref() .unwrap() @@ -1014,7 +1106,7 @@ mod tests { // Reveal multiproof again and check that the state trie still does not contain the leaf // node and value, because they were already revealed before - sparse.reveal_multiproof(multiproof).unwrap(); + sparse.reveal_decoded_multiproof(multiproof.try_into().unwrap()).unwrap(); assert!(!sparse .state_trie_ref() .unwrap() @@ -1029,7 +1121,8 @@ mod tests { #[test] fn reveal_storage_path_twice() { - let mut sparse = SparseStateTrie::default(); + let provider_factory = DefaultTrieNodeProviderFactory; + let mut sparse = SparseStateTrie::::default(); let leaf_value = alloy_rlp::encode(TrieAccount::default()); let leaf_1 = alloy_rlp::encode(TrieNode::Leaf(LeafNode::new( @@ -1066,7 +1159,7 @@ mod tests { }; // Reveal multiproof and check that the storage trie contains the leaf node and value - sparse.reveal_multiproof(multiproof.clone()).unwrap(); + sparse.reveal_decoded_multiproof(multiproof.clone().try_into().unwrap()).unwrap(); assert!(sparse .storage_trie_ref(&B256::ZERO) .unwrap() @@ -1082,7 +1175,9 @@ mod tests { // Remove the leaf node and check that the storage trie does not contain the leaf node and // value - sparse.remove_storage_leaf(B256::ZERO, &Nibbles::from_nibbles([0x0])).unwrap(); + sparse + .remove_storage_leaf(B256::ZERO, &Nibbles::from_nibbles([0x0]), &provider_factory) + .unwrap(); assert!(!sparse .storage_trie_ref(&B256::ZERO) .unwrap() @@ -1096,7 +1191,7 @@ mod tests { // Reveal multiproof again and check that the storage trie still does not contain the leaf // node and value, because they were already revealed before - sparse.reveal_multiproof(multiproof).unwrap(); + sparse.reveal_decoded_multiproof(multiproof.try_into().unwrap()).unwrap(); assert!(!sparse .storage_trie_ref(&B256::ZERO) .unwrap() @@ -1129,11 +1224,8 @@ mod tests { let slot_path_3 = Nibbles::unpack(slot_3); let value_3 = U256::from(rng.random::()); - let mut storage_hash_builder = - HashBuilder::default().with_proof_retainer(ProofRetainer::from_iter([ - slot_path_1.clone(), - slot_path_2.clone(), - ])); + let mut storage_hash_builder = HashBuilder::default() + .with_proof_retainer(ProofRetainer::from_iter([slot_path_1, slot_path_2])); storage_hash_builder.add_leaf(slot_path_1, &alloy_rlp::encode_fixed_size(&value_1)); storage_hash_builder.add_leaf(slot_path_2, &alloy_rlp::encode_fixed_size(&value_2)); @@ -1153,67 +1245,94 @@ mod tests { let account_2 = Account::arbitrary(&mut arbitrary::Unstructured::new(&bytes)).unwrap(); let mut trie_account_2 = account_2.into_trie_account(EMPTY_ROOT_HASH); - let mut hash_builder = - HashBuilder::default().with_proof_retainer(ProofRetainer::from_iter([ - address_path_1.clone(), - address_path_2.clone(), - ])); - hash_builder.add_leaf(address_path_1.clone(), &alloy_rlp::encode(trie_account_1)); - hash_builder.add_leaf(address_path_2.clone(), &alloy_rlp::encode(trie_account_2)); + let mut hash_builder = HashBuilder::default() + .with_proof_retainer(ProofRetainer::from_iter([address_path_1, address_path_2])); + hash_builder.add_leaf(address_path_1, &alloy_rlp::encode(trie_account_1)); + hash_builder.add_leaf(address_path_2, &alloy_rlp::encode(trie_account_2)); let root = hash_builder.root(); let proof_nodes = hash_builder.take_proof_nodes(); - let mut sparse = SparseStateTrie::default().with_updates(true); + let provider_factory = DefaultTrieNodeProviderFactory; + let mut sparse = SparseStateTrie::::default().with_updates(true); sparse - .reveal_multiproof(MultiProof { - account_subtree: proof_nodes, - branch_node_hash_masks: HashMap::from_iter([( - Nibbles::from_nibbles([0x1]), - TrieMask::new(0b00), - )]), - branch_node_tree_masks: HashMap::default(), - storages: HashMap::from_iter([ - ( - address_1, - StorageMultiProof { - root, - subtree: storage_proof_nodes.clone(), - branch_node_hash_masks: storage_branch_node_hash_masks.clone(), - branch_node_tree_masks: HashMap::default(), - }, - ), - ( - address_2, - StorageMultiProof { - root, - subtree: storage_proof_nodes, - branch_node_hash_masks: storage_branch_node_hash_masks, - branch_node_tree_masks: HashMap::default(), - }, - ), - ]), - }) + .reveal_decoded_multiproof( + MultiProof { + account_subtree: proof_nodes, + branch_node_hash_masks: HashMap::from_iter([( + Nibbles::from_nibbles([0x1]), + TrieMask::new(0b00), + )]), + branch_node_tree_masks: HashMap::default(), + storages: HashMap::from_iter([ + ( + address_1, + StorageMultiProof { + root, + subtree: storage_proof_nodes.clone(), + branch_node_hash_masks: storage_branch_node_hash_masks.clone(), + branch_node_tree_masks: HashMap::default(), + }, + ), + ( + address_2, + StorageMultiProof { + root, + subtree: storage_proof_nodes, + branch_node_hash_masks: storage_branch_node_hash_masks, + branch_node_tree_masks: HashMap::default(), + }, + ), + ]), + } + .try_into() + .unwrap(), + ) .unwrap(); - assert_eq!(sparse.root().unwrap(), root); + assert_eq!(sparse.root(&provider_factory).unwrap(), root); let address_3 = b256!("0x2000000000000000000000000000000000000000000000000000000000000000"); let address_path_3 = Nibbles::unpack(address_3); let account_3 = Account { nonce: account_1.nonce + 1, ..account_1 }; let trie_account_3 = account_3.into_trie_account(EMPTY_ROOT_HASH); - sparse.update_account_leaf(address_path_3, alloy_rlp::encode(trie_account_3)).unwrap(); + sparse + .update_account_leaf( + address_path_3, + alloy_rlp::encode(trie_account_3), + &provider_factory, + ) + .unwrap(); - sparse.update_storage_leaf(address_1, slot_path_3, alloy_rlp::encode(value_3)).unwrap(); + sparse + .update_storage_leaf( + address_1, + slot_path_3, + alloy_rlp::encode(value_3), + &provider_factory, + ) + .unwrap(); trie_account_1.storage_root = sparse.storage_root(address_1).unwrap(); - sparse.update_account_leaf(address_path_1, alloy_rlp::encode(trie_account_1)).unwrap(); + sparse + .update_account_leaf( + address_path_1, + alloy_rlp::encode(trie_account_1), + &provider_factory, + ) + .unwrap(); sparse.wipe_storage(address_2).unwrap(); trie_account_2.storage_root = sparse.storage_root(address_2).unwrap(); - sparse.update_account_leaf(address_path_2, alloy_rlp::encode(trie_account_2)).unwrap(); + sparse + .update_account_leaf( + address_path_2, + alloy_rlp::encode(trie_account_2), + &provider_factory, + ) + .unwrap(); - sparse.root().unwrap(); + sparse.root(&provider_factory).unwrap(); let sparse_updates = sparse.take_trie_updates().unwrap(); // TODO(alexey): assert against real state root calculation updates @@ -1235,32 +1354,53 @@ mod tests { } #[test] - fn test_decode_proof_nodes() { - let revealed_nodes = HashSet::from_iter([Nibbles::from_nibbles([0x0])]); + fn test_filter_map_revealed_nodes() { + let mut revealed_nodes = HashSet::from_iter([Nibbles::from_nibbles([0x0])]); let leaf = TrieNode::Leaf(LeafNode::new(Nibbles::default(), alloy_rlp::encode([]))); let leaf_encoded = alloy_rlp::encode(&leaf); let branch = TrieNode::Branch(BranchNode::new( vec![RlpNode::from_rlp(&leaf_encoded), RlpNode::from_rlp(&leaf_encoded)], TrieMask::new(0b11), )); - let proof_nodes = ProofNodes::from_iter([ - (Nibbles::default(), alloy_rlp::encode(&branch).into()), - (Nibbles::from_nibbles([0x0]), leaf_encoded.clone().into()), - (Nibbles::from_nibbles([0x1]), leaf_encoded.into()), + let proof_nodes = alloy_trie::proof::DecodedProofNodes::from_iter([ + (Nibbles::default(), branch.clone()), + (Nibbles::from_nibbles([0x0]), leaf.clone()), + (Nibbles::from_nibbles([0x1]), leaf.clone()), ]); - let decoded = decode_proof_nodes(proof_nodes, &revealed_nodes).unwrap(); + let branch_node_hash_masks = HashMap::default(); + let branch_node_tree_masks = HashMap::default(); + + let decoded = filter_map_revealed_nodes( + proof_nodes, + &mut revealed_nodes, + &branch_node_hash_masks, + &branch_node_tree_masks, + ) + .unwrap(); assert_eq!( decoded, - DecodedProofNodes { - nodes: vec![(Nibbles::default(), branch), (Nibbles::from_nibbles([0x1]), leaf)], - // Branch, leaf, leaf - total_nodes: 3, - // Revealed leaf node with path 0x1 - skipped_nodes: 1, + FilterMappedProofNodes { + root_node: Some(RevealedSparseNode { + path: Nibbles::default(), + node: branch, + masks: TrieMasks::none(), + }), + nodes: vec![RevealedSparseNode { + path: Nibbles::from_nibbles([0x1]), + node: leaf, + masks: TrieMasks::none(), + }], // Branch, two of its children, one leaf - new_nodes: 4 + new_nodes: 4, + // Metric values + metric_values: ProofNodesMetricValues { + // Branch, leaf, leaf + total_nodes: 3, + // Revealed leaf node with path 0x1 + skipped_nodes: 1, + }, } ); } diff --git a/crates/trie/sparse/src/traits.rs b/crates/trie/sparse/src/traits.rs new file mode 100644 index 00000000000..308695ec0fd --- /dev/null +++ b/crates/trie/sparse/src/traits.rs @@ -0,0 +1,320 @@ +//! Traits for sparse trie implementations. + +use core::fmt::Debug; + +use alloc::{borrow::Cow, vec, vec::Vec}; +use alloy_primitives::{ + map::{HashMap, HashSet}, + B256, +}; +use alloy_trie::{BranchNodeCompact, TrieMask}; +use reth_execution_errors::SparseTrieResult; +use reth_trie_common::{Nibbles, TrieNode}; + +use crate::provider::TrieNodeProvider; + +/// Trait defining common operations for revealed sparse trie implementations. +/// +/// This trait abstracts over different sparse trie implementations (serial vs parallel) +/// while providing a unified interface for the core trie operations needed by the +/// [`crate::SparseTrie`] enum. +pub trait SparseTrieInterface: Sized + Debug + Send + Sync { + /// Configures the trie to have the given root node revealed. + /// + /// # Arguments + /// + /// * `root` - The root node to reveal + /// * `masks` - Trie masks for root branch node + /// * `retain_updates` - Whether to track updates + /// + /// # Returns + /// + /// Self if successful, or an error if revealing fails. + /// + /// # Panics + /// + /// May panic if the trie is not new/cleared, and has already revealed nodes. + fn with_root( + self, + root: TrieNode, + masks: TrieMasks, + retain_updates: bool, + ) -> SparseTrieResult; + + /// Configures the trie to retain information about updates. + /// + /// If `retain_updates` is true, the trie will record branch node updates + /// and deletions. This information can be used to efficiently update + /// an external database. + /// + /// # Arguments + /// + /// * `retain_updates` - Whether to track updates + /// + /// # Returns + /// + /// Self for method chaining. + fn with_updates(self, retain_updates: bool) -> Self; + + /// Reserves capacity for additional trie nodes. + /// + /// # Arguments + /// + /// * `additional` - The number of additional trie nodes to reserve capacity for. + fn reserve_nodes(&mut self, _additional: usize) {} + + /// The single-node version of `reveal_nodes`. + /// + /// # Returns + /// + /// `Ok(())` if successful, or an error if the node was not revealed. + fn reveal_node( + &mut self, + path: Nibbles, + node: TrieNode, + masks: TrieMasks, + ) -> SparseTrieResult<()> { + self.reveal_nodes(vec![RevealedSparseNode { path, node, masks }]) + } + + /// Reveals one or more trie nodes if they have not been revealed before. + /// + /// This function decodes trie nodes and inserts them into the trie structure. It handles + /// different node types (leaf, extension, branch) by appropriately adding them to the trie and + /// recursively revealing their children. + /// + /// # Arguments + /// + /// * `nodes` - The nodes to be revealed, each having a path and optional set of branch node + /// masks. The nodes will be unsorted. + /// + /// # Returns + /// + /// `Ok(())` if successful, or an error if any of the nodes was not revealed. + fn reveal_nodes(&mut self, nodes: Vec) -> SparseTrieResult<()>; + + /// Updates the value of a leaf node at the specified path. + /// + /// If the leaf doesn't exist, it will be created. + /// If it does exist, its value will be updated. + /// + /// # Arguments + /// + /// * `full_path` - The full path to the leaf + /// * `value` - The new value for the leaf + /// * `provider` - The trie provider for resolving missing nodes + /// + /// # Returns + /// + /// `Ok(())` if successful, or an error if the update failed. + fn update_leaf( + &mut self, + full_path: Nibbles, + value: Vec, + provider: P, + ) -> SparseTrieResult<()>; + + /// Removes a leaf node at the specified path. + /// + /// This will also handle collapsing the trie structure as needed + /// (e.g., removing branch nodes that become unnecessary). + /// + /// # Arguments + /// + /// * `full_path` - The full path to the leaf to remove + /// * `provider` - The trie node provider for resolving missing nodes + /// + /// # Returns + /// + /// `Ok(())` if successful, or an error if the removal failed. + fn remove_leaf( + &mut self, + full_path: &Nibbles, + provider: P, + ) -> SparseTrieResult<()>; + + /// Calculates and returns the root hash of the trie. + /// + /// This processes any dirty nodes by updating their RLP encodings + /// and returns the root hash. + /// + /// # Returns + /// + /// The root hash of the trie. + fn root(&mut self) -> B256; + + /// Recalculates and updates the RLP hashes of subtries deeper than a certain level. The level + /// is defined in the implementation. + /// + /// The root node is considered to be at level 0. This method is useful for optimizing + /// hash recalculations after localized changes to the trie structure. + fn update_subtrie_hashes(&mut self); + + /// Retrieves a reference to the leaf value at the specified path. + /// + /// # Arguments + /// + /// * `full_path` - The full path to the leaf value + /// + /// # Returns + /// + /// A reference to the leaf value stored at the given full path, if it is revealed. + /// + /// Note: a value can exist in the full trie and this function still returns `None` + /// because the value has not been revealed. + /// + /// Hence a `None` indicates two possibilities: + /// - The value does not exists in the trie, so it cannot be revealed + /// - The value has not yet been revealed. In order to determine which is true, one would need + /// an exclusion proof. + fn get_leaf_value(&self, full_path: &Nibbles) -> Option<&Vec>; + + /// Attempts to find a leaf node at the specified path. + /// + /// This method traverses the trie from the root down to the given path, checking + /// if a leaf exists at that path. It can be used to verify the existence of a leaf + /// or to generate an exclusion proof (proof that a leaf does not exist). + /// + /// # Parameters + /// + /// - `full_path`: The path to search for. + /// - `expected_value`: Optional expected value. If provided, will verify the leaf value + /// matches. + /// + /// # Returns + /// + /// - `Ok(LeafLookup::Exists)` if the leaf exists with the expected value. + /// - `Ok(LeafLookup::NonExistent)` if the leaf definitely does not exist (exclusion proof). + /// - `Err(LeafLookupError)` if the search encountered a blinded node or found a different + /// value. + fn find_leaf( + &self, + full_path: &Nibbles, + expected_value: Option<&Vec>, + ) -> Result; + + /// Returns a reference to the current sparse trie updates. + /// + /// If no updates have been made/recorded, returns an empty update set. + fn updates_ref(&self) -> Cow<'_, SparseTrieUpdates>; + + /// Consumes and returns the currently accumulated trie updates. + /// + /// This is useful when you want to apply the updates to an external database + /// and then start tracking a new set of updates. + /// + /// # Returns + /// + /// The accumulated updates, or an empty set if updates weren't being tracked. + fn take_updates(&mut self) -> SparseTrieUpdates; + + /// Removes all nodes and values from the trie, resetting it to a blank state + /// with only an empty root node. This is used when a storage root is deleted. + /// + /// This should not be used when intending to reuse the trie for a fresh account/storage root; + /// use `clear` for that. + /// + /// Note: All previously tracked changes to the trie are also removed. + fn wipe(&mut self); + + /// This clears all data structures in the sparse trie, keeping the backing data structures + /// allocated. A [`crate::SparseNode::Empty`] is inserted at the root. + /// + /// This is useful for reusing the trie without needing to reallocate memory. + fn clear(&mut self); + + /// Shrink the capacity of the sparse trie's node storage to the given size. + /// This will reduce memory usage if the current capacity is higher than the given size. + fn shrink_nodes_to(&mut self, size: usize); + + /// Shrink the capacity of the sparse trie's value storage to the given size. + /// This will reduce memory usage if the current capacity is higher than the given size. + fn shrink_values_to(&mut self, size: usize); +} + +/// Struct for passing around branch node mask information. +/// +/// Branch nodes can have up to 16 children (one for each nibble). +/// The masks represent which children are stored in different ways: +/// - `hash_mask`: Indicates which children are stored as hashes in the database +/// - `tree_mask`: Indicates which children are complete subtrees stored in the database +/// +/// These masks are essential for efficient trie traversal and serialization, as they +/// determine how nodes should be encoded and stored on disk. +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +pub struct TrieMasks { + /// Branch node hash mask, if any. + /// + /// When a bit is set, the corresponding child node's hash is stored in the trie. + /// + /// This mask enables selective hashing of child nodes. + pub hash_mask: Option, + /// Branch node tree mask, if any. + /// + /// When a bit is set, the corresponding child subtree is stored in the database. + pub tree_mask: Option, +} + +impl TrieMasks { + /// Helper function, returns both fields `hash_mask` and `tree_mask` as [`None`] + pub const fn none() -> Self { + Self { hash_mask: None, tree_mask: None } + } +} + +/// Tracks modifications to the sparse trie structure. +/// +/// Maintains references to both modified and pruned/removed branches, enabling +/// one to make batch updates to a persistent database. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct SparseTrieUpdates { + /// Collection of updated intermediate nodes indexed by full path. + pub updated_nodes: HashMap, + /// Collection of removed intermediate nodes indexed by full path. + pub removed_nodes: HashSet, + /// Flag indicating whether the trie was wiped. + pub wiped: bool, +} + +/// Error type for a leaf lookup operation +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum LeafLookupError { + /// The path leads to a blinded node, cannot determine if leaf exists. + /// This means the witness is not complete. + BlindedNode { + /// Path to the blinded node. + path: Nibbles, + /// Hash of the blinded node. + hash: B256, + }, + /// The path leads to a leaf with a different value than expected. + /// This means the witness is malformed. + ValueMismatch { + /// Path to the leaf. + path: Nibbles, + /// Expected value. + expected: Option>, + /// Actual value found. + actual: Vec, + }, +} + +/// Success value for a leaf lookup operation +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum LeafLookup { + /// Leaf exists with expected value. + Exists, + /// Leaf does not exist (exclusion proof found). + NonExistent, +} + +/// Carries all information needed by a sparse trie to reveal a particular node. +#[derive(Debug, PartialEq, Eq)] +pub struct RevealedSparseNode { + /// Path of the node. + pub path: Nibbles, + /// The node itself. + pub node: TrieNode, + /// Tree and hash masks for the node, if known. + pub masks: TrieMasks, +} diff --git a/crates/trie/sparse/src/trie.rs b/crates/trie/sparse/src/trie.rs index 35b741cfd29..500b642cd1e 100644 --- a/crates/trie/sparse/src/trie.rs +++ b/crates/trie/sparse/src/trie.rs @@ -1,4 +1,8 @@ -use crate::blinded::{BlindedProvider, DefaultBlindedProvider, RevealedNode}; +use crate::{ + provider::{RevealedNode, TrieNodeProvider}, + LeafLookup, LeafLookupError, RevealedSparseNode, SparseTrieInterface, SparseTrieUpdates, + TrieMasks, +}; use alloc::{ borrow::Cow, boxed::Box, @@ -20,37 +24,11 @@ use reth_trie_common::{ TrieNode, CHILD_INDEX_RANGE, EMPTY_ROOT_HASH, }; use smallvec::SmallVec; -use tracing::trace; - -/// Struct for passing around branch node mask information. -/// -/// Branch nodes can have up to 16 children (one for each nibble). -/// The masks represent which children are stored in different ways: -/// - `hash_mask`: Indicates which children are stored as hashes in the database -/// - `tree_mask`: Indicates which children are complete subtrees stored in the database -/// -/// These masks are essential for efficient trie traversal and serialization, as they -/// determine how nodes should be encoded and stored on disk. -#[derive(Debug)] -pub struct TrieMasks { - /// Branch node hash mask, if any. - /// - /// When a bit is set, the corresponding child node's hash is stored in the trie. - /// - /// This mask enables selective hashing of child nodes. - pub hash_mask: Option, - /// Branch node tree mask, if any. - /// - /// When a bit is set, the corresponding child subtree is stored in the database. - pub tree_mask: Option, -} +use tracing::{debug, instrument, trace}; -impl TrieMasks { - /// Helper function, returns both fields `hash_mask` and `tree_mask` as [`None`] - pub const fn none() -> Self { - Self { hash_mask: None, tree_mask: None } - } -} +/// The level below which the sparse trie hashes are calculated in +/// [`SerialSparseTrie::update_subtrie_hashes`]. +const SPARSE_TRIE_SUBTRIE_HASHES_LEVEL: usize = 2; /// A sparse trie that is either in a "blind" state (no nodes are revealed, root node hash is /// unknown) or in a "revealed" state (root node has been revealed and the trie can be updated). @@ -64,56 +42,39 @@ impl TrieMasks { /// 2. Update tracking - changes to the trie structure can be tracked and selectively persisted /// 3. Incremental operations - nodes can be revealed as needed without loading the entire trie. /// This is what gives rise to the notion of a "sparse" trie. -#[derive(PartialEq, Eq, Default)] -pub enum SparseTrie

{ +#[derive(PartialEq, Eq, Debug, Clone)] +pub enum SparseTrie { /// The trie is blind -- no nodes have been revealed /// - /// This is the default state. In this state, - /// the trie cannot be directly queried or modified until nodes are revealed. - #[default] - Blind, + /// This is the default state. In this state, the trie cannot be directly queried or modified + /// until nodes are revealed. + /// + /// In this state the `SparseTrie` can optionally carry with it a cleared `SerialSparseTrie`. + /// This allows for reusing the trie's allocations between payload executions. + Blind(Option>), /// Some nodes in the Trie have been revealed. /// /// In this state, the trie can be queried and modified for the parts /// that have been revealed. Other parts remain blind and require revealing /// before they can be accessed. - Revealed(Box>), + Revealed(Box), } -impl

fmt::Debug for SparseTrie

{ - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Blind => write!(f, "Blind"), - Self::Revealed(revealed) => write!(f, "Revealed({revealed:?})"), - } +impl Default for SparseTrie { + fn default() -> Self { + Self::Blind(None) } } -impl SparseTrie { - /// Creates a new blind sparse trie. - /// - /// # Examples - /// - /// ``` - /// use reth_trie_sparse::{blinded::DefaultBlindedProvider, SparseTrie}; - /// - /// let trie: SparseTrie = SparseTrie::blind(); - /// assert!(trie.is_blind()); - /// let trie: SparseTrie = SparseTrie::default(); - /// assert!(trie.is_blind()); - /// ``` - pub const fn blind() -> Self { - Self::Blind - } - +impl SparseTrie { /// Creates a new revealed but empty sparse trie with `SparseNode::Empty` as root node. /// /// # Examples /// /// ``` - /// use reth_trie_sparse::{blinded::DefaultBlindedProvider, SparseTrie}; + /// use reth_trie_sparse::{provider::DefaultTrieNodeProvider, SerialSparseTrie, SparseTrie}; /// - /// let trie: SparseTrie = SparseTrie::revealed_empty(); + /// let trie = SparseTrie::::revealed_empty(); /// assert!(!trie.is_blind()); /// ``` pub fn revealed_empty() -> Self { @@ -130,27 +91,68 @@ impl SparseTrie { /// /// # Returns /// - /// A mutable reference to the underlying [`RevealedSparseTrie`]. + /// A mutable reference to the underlying [`SparseTrieInterface`]. pub fn reveal_root( &mut self, root: TrieNode, masks: TrieMasks, retain_updates: bool, - ) -> SparseTrieResult<&mut RevealedSparseTrie> { - self.reveal_root_with_provider(Default::default(), root, masks, retain_updates) + ) -> SparseTrieResult<&mut T> { + // if `Blind`, we initialize the revealed trie with the given root node, using a + // pre-allocated trie if available. + if self.is_blind() { + let mut revealed_trie = if let Self::Blind(Some(cleared_trie)) = core::mem::take(self) { + cleared_trie + } else { + Box::default() + }; + + *revealed_trie = revealed_trie.with_root(root, masks, retain_updates)?; + *self = Self::Revealed(revealed_trie); + } + + Ok(self.as_revealed_mut().unwrap()) } } -impl

SparseTrie

{ +impl SparseTrie { + /// Creates a new blind sparse trie. + /// + /// # Examples + /// + /// ``` + /// use reth_trie_sparse::{provider::DefaultTrieNodeProvider, SerialSparseTrie, SparseTrie}; + /// + /// let trie = SparseTrie::::blind(); + /// assert!(trie.is_blind()); + /// let trie = SparseTrie::::default(); + /// assert!(trie.is_blind()); + /// ``` + pub const fn blind() -> Self { + Self::Blind(None) + } + + /// Creates a new blind sparse trie, clearing and later reusing the given + /// [`SparseTrieInterface`]. + pub fn blind_from(mut trie: T) -> Self { + trie.clear(); + Self::Blind(Some(Box::new(trie))) + } + /// Returns `true` if the sparse trie has no revealed nodes. pub const fn is_blind(&self) -> bool { - matches!(self, Self::Blind) + matches!(self, Self::Blind(_)) + } + + /// Returns `true` if the sparse trie is revealed. + pub const fn is_revealed(&self) -> bool { + matches!(self, Self::Revealed(_)) } /// Returns an immutable reference to the underlying revealed sparse trie. /// /// Returns `None` if the trie is blinded. - pub const fn as_revealed_ref(&self) -> Option<&RevealedSparseTrie

> { + pub const fn as_revealed_ref(&self) -> Option<&T> { if let Self::Revealed(revealed) = self { Some(revealed) } else { @@ -161,7 +163,7 @@ impl

SparseTrie

{ /// Returns a mutable reference to the underlying revealed sparse trie. /// /// Returns `None` if the trie is blinded. - pub fn as_revealed_mut(&mut self) -> Option<&mut RevealedSparseTrie

> { + pub fn as_revealed_mut(&mut self) -> Option<&mut T> { if let Self::Revealed(revealed) = self { Some(revealed) } else { @@ -169,32 +171,6 @@ impl

SparseTrie

{ } } - /// Reveals the root node using a specified provider. - /// - /// This function is similar to [`Self::reveal_root`] but allows the caller to provide - /// a custom provider for fetching blinded nodes. - /// - /// # Returns - /// - /// Mutable reference to [`RevealedSparseTrie`]. - pub fn reveal_root_with_provider( - &mut self, - provider: P, - root: TrieNode, - masks: TrieMasks, - retain_updates: bool, - ) -> SparseTrieResult<&mut RevealedSparseTrie

> { - if self.is_blind() { - *self = Self::Revealed(Box::new(RevealedSparseTrie::from_provider_and_root( - provider, - root, - masks, - retain_updates, - )?)) - } - Ok(self.as_revealed_mut().unwrap()) - } - /// Wipes the trie by removing all nodes and values, /// and resetting the trie to only contain an empty root node. /// @@ -235,17 +211,34 @@ impl

SparseTrie

{ let revealed = self.as_revealed_mut()?; Some((revealed.root(), revealed.take_updates())) } -} -impl SparseTrie

{ + /// Returns a [`SparseTrie::Blind`] based on this one. If this instance was revealed, or was + /// itself a `Blind` with a pre-allocated [`SparseTrieInterface`], this will return + /// a `Blind` carrying a cleared pre-allocated [`SparseTrieInterface`]. + pub fn clear(self) -> Self { + match self { + Self::Blind(_) => self, + Self::Revealed(mut trie) => { + trie.clear(); + Self::Blind(Some(trie)) + } + } + } + /// Updates (or inserts) a leaf at the given key path with the specified RLP-encoded value. /// /// # Errors /// /// Returns an error if the trie is still blind, or if the update fails. - pub fn update_leaf(&mut self, path: Nibbles, value: Vec) -> SparseTrieResult<()> { + #[instrument(level = "trace", target = "trie::sparse", skip_all)] + pub fn update_leaf( + &mut self, + path: Nibbles, + value: Vec, + provider: impl TrieNodeProvider, + ) -> SparseTrieResult<()> { let revealed = self.as_revealed_mut().ok_or(SparseTrieErrorKind::Blind)?; - revealed.update_leaf(path, value)?; + revealed.update_leaf(path, value, provider)?; Ok(()) } @@ -254,11 +247,38 @@ impl SparseTrie

{ /// # Errors /// /// Returns an error if the trie is still blind, or if the leaf cannot be removed - pub fn remove_leaf(&mut self, path: &Nibbles) -> SparseTrieResult<()> { + #[instrument(level = "trace", target = "trie::sparse", skip_all)] + pub fn remove_leaf( + &mut self, + path: &Nibbles, + provider: impl TrieNodeProvider, + ) -> SparseTrieResult<()> { let revealed = self.as_revealed_mut().ok_or(SparseTrieErrorKind::Blind)?; - revealed.remove_leaf(path)?; + revealed.remove_leaf(path, provider)?; Ok(()) } + + /// Shrinks the capacity of the sparse trie's node storage. + /// Works for both revealed and blind tries with allocated storage. + pub fn shrink_nodes_to(&mut self, size: usize) { + match self { + Self::Blind(Some(trie)) | Self::Revealed(trie) => { + trie.shrink_nodes_to(size); + } + _ => {} + } + } + + /// Shrinks the capacity of the sparse trie's value storage. + /// Works for both revealed and blind tries with allocated storage. + pub fn shrink_values_to(&mut self, size: usize) { + match self { + Self::Blind(Some(trie)) | Self::Revealed(trie) => { + trie.shrink_values_to(size); + } + _ => {} + } + } } /// The representation of revealed sparse trie. @@ -275,10 +295,7 @@ impl SparseTrie

{ /// The opposite is also true. /// - All keys in `values` collection are full leaf paths. #[derive(Clone, PartialEq, Eq)] -pub struct RevealedSparseTrie

{ - /// Provider used for retrieving blinded nodes. - /// This allows lazily loading parts of the trie from an external source. - provider: P, +pub struct SerialSparseTrie { /// Map from a path (nibbles) to its corresponding sparse trie node. /// This contains all of the revealed nodes in trie. nodes: HashMap, @@ -298,9 +315,9 @@ pub struct RevealedSparseTrie

{ rlp_buf: Vec, } -impl

fmt::Debug for RevealedSparseTrie

{ +impl fmt::Debug for SerialSparseTrie { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("RevealedSparseTrie") + f.debug_struct("SerialSparseTrie") .field("nodes", &self.nodes) .field("branch_tree_masks", &self.branch_node_tree_masks) .field("branch_hash_masks", &self.branch_node_hash_masks) @@ -318,7 +335,7 @@ fn encode_nibbles(nibbles: &Nibbles) -> String { encoded[..nibbles.len()].to_string() } -impl fmt::Display for RevealedSparseTrie

{ +impl fmt::Display for SerialSparseTrie { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { // This prints the trie in preorder traversal, using a stack let mut stack = Vec::new(); @@ -331,7 +348,7 @@ impl fmt::Display for RevealedSparseTrie

{ stack.push((Nibbles::default(), self.nodes_ref().get(&Nibbles::default()).unwrap(), 0)); while let Some((path, node, depth)) = stack.pop() { - if !visited.insert(path.clone()) { + if !visited.insert(path) { continue; } @@ -348,8 +365,8 @@ impl fmt::Display for RevealedSparseTrie

{ } SparseNode::Leaf { key, .. } => { // we want to append the key to the path - let mut full_path = path.clone(); - full_path.extend_from_slice_unchecked(key); + let mut full_path = path; + full_path.extend(key); let packed_path = encode_nibbles(&full_path); writeln!(f, "{packed_path} -> {node:?}")?; @@ -358,8 +375,8 @@ impl fmt::Display for RevealedSparseTrie

{ writeln!(f, "{packed_path} -> {node:?}")?; // push the child node onto the stack with increased depth - let mut child_path = path.clone(); - child_path.extend_from_slice_unchecked(key); + let mut child_path = path; + child_path.extend(key); if let Some(child_node) = self.nodes_ref().get(&child_path) { stack.push((child_path, child_node, depth + 1)); } @@ -369,7 +386,7 @@ impl fmt::Display for RevealedSparseTrie

{ for i in CHILD_INDEX_RANGE.rev() { if state_mask.is_bit_set(i) { - let mut child_path = path.clone(); + let mut child_path = path; child_path.push_unchecked(i); if let Some(child_node) = self.nodes_ref().get(&child_path) { stack.push((child_path, child_node, depth + 1)); @@ -384,10 +401,9 @@ impl fmt::Display for RevealedSparseTrie

{ } } -impl Default for RevealedSparseTrie { +impl Default for SerialSparseTrie { fn default() -> Self { Self { - provider: Default::default(), nodes: HashMap::from_iter([(Nibbles::default(), SparseNode::Empty)]), branch_node_tree_masks: HashMap::default(), branch_node_hash_masks: HashMap::default(), @@ -399,165 +415,53 @@ impl Default for RevealedSparseTrie { } } -impl RevealedSparseTrie { - /// Creates a new revealed sparse trie from the given root node. - /// - /// This function initializes the internal structures and then reveals the root. - /// It is a convenient method to create a [`RevealedSparseTrie`] when you already have - /// the root node available. - /// - /// # Returns - /// - /// A [`RevealedSparseTrie`] if successful, or an error if revealing fails. - pub fn from_root( +impl SparseTrieInterface for SerialSparseTrie { + fn with_root( + mut self, root: TrieNode, masks: TrieMasks, retain_updates: bool, ) -> SparseTrieResult { - let mut this = Self { - provider: Default::default(), - nodes: HashMap::default(), - branch_node_tree_masks: HashMap::default(), - branch_node_hash_masks: HashMap::default(), - values: HashMap::default(), - prefix_set: PrefixSetMut::default(), - rlp_buf: Vec::new(), - updates: None, - } - .with_updates(retain_updates); - this.reveal_node(Nibbles::default(), root, masks)?; - Ok(this) - } -} + self = self.with_updates(retain_updates); -impl

RevealedSparseTrie

{ - /// Creates a new revealed sparse trie from the given provider and root node. - /// - /// Similar to `from_root`, but allows specifying a custom provider for - /// retrieving blinded nodes. - /// - /// # Returns - /// - /// A [`RevealedSparseTrie`] if successful, or an error if revealing fails. - pub fn from_provider_and_root( - provider: P, - node: TrieNode, - masks: TrieMasks, - retain_updates: bool, - ) -> SparseTrieResult { - let mut this = Self { - provider, - nodes: HashMap::default(), - branch_node_tree_masks: HashMap::default(), - branch_node_hash_masks: HashMap::default(), - values: HashMap::default(), - prefix_set: PrefixSetMut::default(), - rlp_buf: Vec::new(), - updates: None, - } - .with_updates(retain_updates); - this.reveal_node(Nibbles::default(), node, masks)?; - Ok(this) - } + // A fresh/cleared `SerialSparseTrie` has a `SparseNode::Empty` at its root. Delete that + // so we can reveal the new root node. + let path = Nibbles::default(); + let _removed_root = self.nodes.remove(&path).expect("root node should exist"); + debug_assert_eq!(_removed_root, SparseNode::Empty); - /// Replaces the current provider with a new provider. - /// - /// This allows changing how blinded nodes are retrieved without - /// rebuilding the entire trie structure. - /// - /// # Returns - /// - /// A new [`RevealedSparseTrie`] with the updated provider. - pub fn with_provider(self, provider: BP) -> RevealedSparseTrie { - RevealedSparseTrie { - provider, - nodes: self.nodes, - branch_node_tree_masks: self.branch_node_tree_masks, - branch_node_hash_masks: self.branch_node_hash_masks, - values: self.values, - prefix_set: self.prefix_set, - updates: self.updates, - rlp_buf: self.rlp_buf, - } + self.reveal_node(path, root, masks)?; + Ok(self) } - /// Configures the trie to retain information about updates. - /// - /// If `retain_updates` is true, the trie will record branch node updates and deletions. - /// This information can then be used to efficiently update an external database. - pub fn with_updates(mut self, retain_updates: bool) -> Self { + fn with_updates(mut self, retain_updates: bool) -> Self { if retain_updates { self.updates = Some(SparseTrieUpdates::default()); } self } - /// Returns a reference to the current sparse trie updates. - /// - /// If no updates have been made/recorded, returns an empty update set. - pub fn updates_ref(&self) -> Cow<'_, SparseTrieUpdates> { - self.updates.as_ref().map_or(Cow::Owned(SparseTrieUpdates::default()), Cow::Borrowed) - } - - /// Returns an immutable reference to all nodes in the sparse trie. - pub const fn nodes_ref(&self) -> &HashMap { - &self.nodes - } - - /// Retrieves a reference to the leaf value stored at the given key path, if it is revealed. - /// - /// This method efficiently retrieves values from the trie without traversing - /// the entire node structure, as values are stored in a separate map. - /// - /// Note: a value can exist in the full trie and this function still returns `None` - /// because the value has not been revealed. - /// Hence a `None` indicates two possibilities: - /// - The value does not exists in the trie, so it cannot be revealed - /// - The value has not yet been revealed. In order to determine which is true, one would need - /// an exclusion proof. - pub fn get_leaf_value(&self, path: &Nibbles) -> Option<&Vec> { - self.values.get(path) - } - - /// Consumes and returns the currently accumulated trie updates. - /// - /// This is useful when you want to apply the updates to an external database, - /// and then start tracking a new set of updates. - pub fn take_updates(&mut self) -> SparseTrieUpdates { - self.updates.take().unwrap_or_default() - } - - /// Reserves capacity in the nodes map for at least `additional` more nodes. - pub fn reserve_nodes(&mut self, additional: usize) { + fn reserve_nodes(&mut self, additional: usize) { self.nodes.reserve(additional); } - - /// Reveals a trie node if it has not been revealed before. - /// - /// This internal function decodes a trie node and inserts it into the nodes map. - /// It handles different node types (leaf, extension, branch) by appropriately - /// adding them to the trie structure and recursively revealing their children. - /// - /// - /// # Returns - /// - /// `Ok(())` if successful, or an error if node was not revealed. - pub fn reveal_node( + fn reveal_node( &mut self, path: Nibbles, node: TrieNode, masks: TrieMasks, ) -> SparseTrieResult<()> { + trace!(target: "trie::sparse", ?path, ?node, ?masks, "reveal_node called"); + // If the node is already revealed and it's not a hash node, do nothing. if self.nodes.get(&path).is_some_and(|node| !node.is_hash()) { return Ok(()) } if let Some(tree_mask) = masks.tree_mask { - self.branch_node_tree_masks.insert(path.clone(), tree_mask); + self.branch_node_tree_masks.insert(path, tree_mask); } if let Some(hash_mask) = masks.hash_mask { - self.branch_node_hash_masks.insert(path.clone(), hash_mask); + self.branch_node_hash_masks.insert(path, hash_mask); } match node { @@ -571,7 +475,7 @@ impl

RevealedSparseTrie

{ let mut stack_ptr = branch.as_ref().first_child_index(); for idx in CHILD_INDEX_RANGE { if branch.state_mask.is_bit_set(idx) { - let mut child_path = path.clone(); + let mut child_path = path; child_path.push_unchecked(idx); // Reveal each child node or hash it has self.reveal_node_or_hash(child_path, &branch.stack[stack_ptr])?; @@ -601,7 +505,7 @@ impl

RevealedSparseTrie

{ // All other node types can't be handled. node @ (SparseNode::Empty | SparseNode::Leaf { .. }) => { return Err(SparseTrieErrorKind::Reveal { - path: entry.key().clone(), + path: *entry.key(), node: Box::new(node.clone()), } .into()) @@ -616,8 +520,8 @@ impl

RevealedSparseTrie

{ Entry::Occupied(mut entry) => match entry.get() { // Replace a hash node with a revealed extension node. SparseNode::Hash(hash) => { - let mut child_path = entry.key().clone(); - child_path.extend_from_slice_unchecked(&ext.key); + let mut child_path = *entry.key(); + child_path.extend(&ext.key); entry.insert(SparseNode::Extension { key: ext.key, // Memoize the hash of a previously blinded node in a new extension @@ -633,15 +537,15 @@ impl

RevealedSparseTrie

{ // All other node types can't be handled. node @ (SparseNode::Empty | SparseNode::Leaf { .. }) => { return Err(SparseTrieErrorKind::Reveal { - path: entry.key().clone(), + path: *entry.key(), node: Box::new(node.clone()), } .into()) } }, Entry::Vacant(entry) => { - let mut child_path = entry.key().clone(); - child_path.extend_from_slice_unchecked(&ext.key); + let mut child_path = *entry.key(); + child_path.extend(&ext.key); entry.insert(SparseNode::new_ext(ext.key)); self.reveal_node_or_hash(child_path, &ext.child)?; } @@ -650,9 +554,9 @@ impl

RevealedSparseTrie

{ Entry::Occupied(mut entry) => match entry.get() { // Replace a hash node with a revealed leaf node and store leaf node value. SparseNode::Hash(hash) => { - let mut full = entry.key().clone(); - full.extend_from_slice_unchecked(&leaf.key); - self.values.insert(full, leaf.value); + let mut full = *entry.key(); + full.extend(&leaf.key); + self.values.insert(full, leaf.value.clone()); entry.insert(SparseNode::Leaf { key: leaf.key, // Memoize the hash of a previously blinded node in a new leaf @@ -667,17 +571,17 @@ impl

RevealedSparseTrie

{ SparseNode::Extension { .. } | SparseNode::Branch { .. }) => { return Err(SparseTrieErrorKind::Reveal { - path: entry.key().clone(), + path: *entry.key(), node: Box::new(node.clone()), } .into()) } }, Entry::Vacant(entry) => { - let mut full = entry.key().clone(); - full.extend_from_slice_unchecked(&leaf.key); + let mut full = *entry.key(); + full.extend(&leaf.key); entry.insert(SparseNode::new_leaf(leaf.key)); - self.values.insert(full, leaf.value); + self.values.insert(full, leaf.value.clone()); } }, } @@ -685,675 +589,405 @@ impl

RevealedSparseTrie

{ Ok(()) } - /// Reveals either a node or its hash placeholder based on the provided child data. - /// - /// When traversing the trie, we often encounter references to child nodes that - /// are either directly embedded or represented by their hash. This method - /// handles both cases: - /// - /// 1. If the child data represents a hash (32+1=33 bytes), store it as a hash node - /// 2. Otherwise, decode the data as a [`TrieNode`] and recursively reveal it using - /// `reveal_node` - /// - /// # Returns - /// - /// Returns `Ok(())` if successful, or an error if the node cannot be revealed. - /// - /// # Error Handling - /// - /// Will error if there's a conflict between a new hash node and an existing one - /// at the same path - fn reveal_node_or_hash(&mut self, path: Nibbles, child: &[u8]) -> SparseTrieResult<()> { - if child.len() == B256::len_bytes() + 1 { - let hash = B256::from_slice(&child[1..]); - match self.nodes.entry(path) { - Entry::Occupied(entry) => match entry.get() { - // Hash node with a different hash can't be handled. - SparseNode::Hash(previous_hash) if previous_hash != &hash => { - return Err(SparseTrieErrorKind::Reveal { - path: entry.key().clone(), - node: Box::new(SparseNode::Hash(hash)), - } - .into()) - } - _ => {} - }, - Entry::Vacant(entry) => { - entry.insert(SparseNode::Hash(hash)); - } - } - return Ok(()) + fn reveal_nodes(&mut self, mut nodes: Vec) -> SparseTrieResult<()> { + nodes.sort_unstable_by_key(|node| node.path); + for node in nodes { + self.reveal_node(node.path, node.node, node.masks)?; } - - self.reveal_node(path, TrieNode::decode(&mut &child[..])?, TrieMasks::none()) + Ok(()) } - /// Traverse the trie from the root down to the leaf at the given path, - /// removing and collecting all nodes along that path. - /// - /// This helper function is used during leaf removal to extract the nodes of the trie - /// that will be affected by the deletion. These nodes are then re-inserted and modified - /// as needed (collapsing extension nodes etc) given that the leaf has now been removed. - /// - /// # Returns - /// - /// Returns a vector of [`RemovedSparseNode`] representing the nodes removed during the - /// traversal. - /// - /// # Errors - /// - /// Returns an error if a blinded node or an empty node is encountered unexpectedly, - /// as these prevent proper removal of the leaf. - fn take_nodes_for_path(&mut self, path: &Nibbles) -> SparseTrieResult> { - let mut current = Nibbles::default(); // Start traversal from the root - let mut nodes = Vec::new(); // Collect traversed nodes + #[instrument(level = "trace", target = "trie::sparse::serial", skip(self, provider))] + fn update_leaf( + &mut self, + full_path: Nibbles, + value: Vec, + provider: P, + ) -> SparseTrieResult<()> { + self.prefix_set.insert(full_path); + let existing = self.values.insert(full_path, value); + if existing.is_some() { + // trie structure unchanged, return immediately + return Ok(()) + } - while let Some(node) = self.nodes.remove(¤t) { - match &node { - SparseNode::Empty => return Err(SparseTrieErrorKind::Blind.into()), - &SparseNode::Hash(hash) => { + let mut current = Nibbles::default(); + while let Some(node) = self.nodes.get_mut(¤t) { + match node { + SparseNode::Empty => { + *node = SparseNode::new_leaf(full_path); + break + } + &mut SparseNode::Hash(hash) => { return Err(SparseTrieErrorKind::BlindedNode { path: current, hash }.into()) } - SparseNode::Leaf { key: _key, .. } => { - // Leaf node is always the one that we're deleting, and no other leaf nodes can - // be found during traversal. + SparseNode::Leaf { key: current_key, .. } => { + current.extend(current_key); - #[cfg(debug_assertions)] - { - let mut current = current.clone(); - current.extend_from_slice_unchecked(_key); - assert_eq!(¤t, path); + // this leaf is being updated + if current == full_path { + unreachable!("we already checked leaf presence in the beginning"); } - nodes.push(RemovedSparseNode { - path: current.clone(), - node, - unset_branch_nibble: None, - }); - break - } - SparseNode::Extension { key, .. } => { - #[cfg(debug_assertions)] - { - let mut current = current.clone(); - current.extend_from_slice_unchecked(key); - assert!( - path.starts_with(¤t), - "path: {path:?}, current: {current:?}, key: {key:?}", - ); - } + // find the common prefix + let common = current.common_prefix_length(&full_path); - let path = current.clone(); - current.extend_from_slice_unchecked(key); - nodes.push(RemovedSparseNode { path, node, unset_branch_nibble: None }); - } - SparseNode::Branch { state_mask, .. } => { - let nibble = path[current.len()]; - debug_assert!( - state_mask.is_bit_set(nibble), - "current: {current:?}, path: {path:?}, nibble: {nibble:?}, state_mask: {state_mask:?}", - ); - - // If the branch node has a child that is a leaf node that we're removing, - // we need to unset this nibble. - // Any other branch nodes will not require unsetting the nibble, because - // deleting one leaf node can not remove the whole path - // where the branch node is located. - let mut child_path = - Nibbles::from_nibbles([current.as_slice(), &[nibble]].concat()); - let unset_branch_nibble = self - .nodes - .get(&child_path) - .is_some_and(move |node| match node { - SparseNode::Leaf { key, .. } => { - // Get full path of the leaf node - child_path.extend_from_slice_unchecked(key); - &child_path == path - } - _ => false, - }) - .then_some(nibble); + // update existing node + let new_ext_key = current.slice(current.len() - current_key.len()..common); + *node = SparseNode::new_ext(new_ext_key); - nodes.push(RemovedSparseNode { - path: current.clone(), - node, - unset_branch_nibble, - }); + // create a branch node and corresponding leaves + self.nodes.reserve(3); + self.nodes.insert( + current.slice(..common), + SparseNode::new_split_branch( + current.get_unchecked(common), + full_path.get_unchecked(common), + ), + ); + self.nodes.insert( + full_path.slice(..=common), + SparseNode::new_leaf(full_path.slice(common + 1..)), + ); + self.nodes.insert( + current.slice(..=common), + SparseNode::new_leaf(current.slice(common + 1..)), + ); - current.push_unchecked(nibble); + break; } - } - } - - Ok(nodes) - } - - /// Removes all nodes and values from the trie, resetting it to a blank state - /// with only an empty root node. - /// - /// Note: All previously tracked changes to the trie are also removed. - pub fn wipe(&mut self) { - self.nodes = HashMap::from_iter([(Nibbles::default(), SparseNode::Empty)]); - self.values = HashMap::default(); - self.prefix_set = PrefixSetMut::all(); - self.updates = self.updates.is_some().then(SparseTrieUpdates::wiped); - } - - /// Calculates and returns the root hash of the trie. - /// - /// Before computing the hash, this function processes any remaining (dirty) nodes by - /// updating their RLP encodings. The root hash is either: - /// 1. The cached hash (if no dirty nodes were found) - /// 2. The keccak256 hash of the root node's RLP representation - pub fn root(&mut self) -> B256 { - // Take the current prefix set - let mut prefix_set = core::mem::take(&mut self.prefix_set).freeze(); - let rlp_node = self.rlp_node_allocate(&mut prefix_set); - if let Some(root_hash) = rlp_node.as_hash() { - root_hash - } else { - keccak256(rlp_node) - } - } - - /// Recalculates and updates the RLP hashes of nodes deeper than or equal to the specified - /// `depth`. - /// - /// The root node is considered to be at level 0. This method is useful for optimizing - /// hash recalculations after localized changes to the trie structure: - /// - /// This function identifies all nodes that have changed (based on the prefix set) at the given - /// depth and recalculates their RLP representation. - pub fn update_rlp_node_level(&mut self, depth: usize) { - // Take the current prefix set - let mut prefix_set = core::mem::take(&mut self.prefix_set).freeze(); - let mut buffers = RlpNodeBuffers::default(); - - // Get the nodes that have changed at the given depth. - let (targets, new_prefix_set) = self.get_changed_nodes_at_depth(&mut prefix_set, depth); - // Update the prefix set to the prefix set of the nodes that still need to be updated. - self.prefix_set = new_prefix_set; + SparseNode::Extension { key, .. } => { + current.extend(key); - trace!(target: "trie::sparse", ?depth, ?targets, "Updating nodes at depth"); - for (level, path) in targets { - buffers.path_stack.push(RlpNodePathStackItem { - level, - path, - is_in_prefix_set: Some(true), - }); - self.rlp_node(&mut prefix_set, &mut buffers); - } - } + if !full_path.starts_with(¤t) { + // find the common prefix + let common = current.common_prefix_length(&full_path); + *key = current.slice(current.len() - key.len()..common); - /// Returns a list of (level, path) tuples identifying the nodes that have changed at the - /// specified depth, along with a new prefix set for the paths above the provided depth that - /// remain unchanged. - /// - /// Leaf nodes with a depth less than `depth` are returned too. - /// - /// This method helps optimize hash recalculations by identifying which specific - /// nodes need to be updated at each level of the trie. - /// - /// # Parameters - /// - /// - `prefix_set`: The current prefix set tracking which paths need updates. - /// - `depth`: The minimum depth (relative to the root) to include nodes in the targets. - /// - /// # Returns - /// - /// A tuple containing: - /// - A vector of `(level, Nibbles)` pairs for nodes that require updates at or below the - /// specified depth. - /// - A `PrefixSetMut` containing paths shallower than the specified depth that still need to be - /// tracked for future updates. - fn get_changed_nodes_at_depth( - &self, - prefix_set: &mut PrefixSet, - depth: usize, - ) -> (Vec<(usize, Nibbles)>, PrefixSetMut) { - let mut unchanged_prefix_set = PrefixSetMut::default(); - let mut paths = Vec::from([(Nibbles::default(), 0)]); - let mut targets = Vec::new(); + // If branch node updates retention is enabled, we need to query the + // extension node child to later set the hash mask for a parent branch node + // correctly. + if self.updates.is_some() { + // Check if the extension node child is a hash that needs to be revealed + if self.nodes.get(¤t).unwrap().is_hash() { + debug!( + target: "trie::sparse", + leaf_full_path = ?full_path, + child_path = ?current, + "Extension node child not revealed in update_leaf, falling back to db", + ); + if let Some(RevealedNode { node, tree_mask, hash_mask }) = + provider.trie_node(¤t)? + { + let decoded = TrieNode::decode(&mut &node[..])?; + trace!( + target: "trie::sparse", + ?current, + ?decoded, + ?tree_mask, + ?hash_mask, + "Revealing extension node child", + ); + self.reveal_node( + current, + decoded, + TrieMasks { hash_mask, tree_mask }, + )?; + } + } + } - while let Some((mut path, level)) = paths.pop() { - match self.nodes.get(&path).unwrap() { - SparseNode::Empty | SparseNode::Hash(_) => {} - SparseNode::Leaf { key: _, hash } => { - if hash.is_some() && !prefix_set.contains(&path) { - continue - } + // create state mask for new branch node + // NOTE: this might overwrite the current extension node + self.nodes.reserve(3); + let branch = SparseNode::new_split_branch( + current.get_unchecked(common), + full_path.get_unchecked(common), + ); + self.nodes.insert(current.slice(..common), branch); - targets.push((level, path)); - } - SparseNode::Extension { key, hash, store_in_db_trie: _ } => { - if hash.is_some() && !prefix_set.contains(&path) { - continue - } + // create new leaf + let new_leaf = SparseNode::new_leaf(full_path.slice(common + 1..)); + self.nodes.insert(full_path.slice(..=common), new_leaf); - if level >= depth { - targets.push((level, path)); - } else { - unchanged_prefix_set.insert(path.clone()); + // recreate extension to previous child if needed + let key = current.slice(common + 1..); + if !key.is_empty() { + self.nodes.insert(current.slice(..=common), SparseNode::new_ext(key)); + } - path.extend_from_slice_unchecked(key); - paths.push((path, level + 1)); + break; } } - SparseNode::Branch { state_mask, hash, store_in_db_trie: _ } => { - if hash.is_some() && !prefix_set.contains(&path) { - continue - } - - if level >= depth { - targets.push((level, path)); - } else { - unchanged_prefix_set.insert(path.clone()); - - for bit in CHILD_INDEX_RANGE.rev() { - if state_mask.is_bit_set(bit) { - let mut child_path = path.clone(); - child_path.push_unchecked(bit); - paths.push((child_path, level + 1)); - } - } + SparseNode::Branch { state_mask, .. } => { + let nibble = full_path.get_unchecked(current.len()); + current.push_unchecked(nibble); + if !state_mask.is_bit_set(nibble) { + state_mask.set_bit(nibble); + let new_leaf = SparseNode::new_leaf(full_path.slice(current.len()..)); + self.nodes.insert(current, new_leaf); + break; } } - } + }; } - (targets, unchanged_prefix_set) - } - - /// Look up or calculate the RLP of the node at the root path. - /// - /// # Panics - /// - /// If the node at provided path does not exist. - pub fn rlp_node_allocate(&mut self, prefix_set: &mut PrefixSet) -> RlpNode { - let mut buffers = RlpNodeBuffers::new_with_root_path(); - self.rlp_node(prefix_set, &mut buffers) + Ok(()) } - /// Looks up or computes the RLP encoding of the node specified by the current - /// path in the provided buffers. - /// - /// The function uses a stack (`RlpNodeBuffers::path_stack`) to track the traversal and - /// accumulate RLP encodings. - /// - /// # Parameters - /// - /// - `prefix_set`: The set of trie paths that need their nodes updated. - /// - `buffers`: The reusable buffers for stack management and temporary RLP values. - /// - /// # Panics - /// - /// If the node at provided path does not exist. - pub fn rlp_node( + #[instrument(level = "trace", target = "trie::sparse::serial", skip(self, provider))] + fn remove_leaf( &mut self, - prefix_set: &mut PrefixSet, - buffers: &mut RlpNodeBuffers, - ) -> RlpNode { - let _starting_path = buffers.path_stack.last().map(|item| item.path.clone()); + full_path: &Nibbles, + provider: P, + ) -> SparseTrieResult<()> { + trace!(target: "trie::sparse", ?full_path, "remove_leaf called"); - 'main: while let Some(RlpNodePathStackItem { level, path, mut is_in_prefix_set }) = - buffers.path_stack.pop() - { - let node = self.nodes.get_mut(&path).unwrap(); - trace!( - target: "trie::sparse", - ?_starting_path, - ?level, - ?path, - ?is_in_prefix_set, - ?node, - "Popped node from path stack" - ); + if self.values.remove(full_path).is_none() { + if let Some(&SparseNode::Hash(hash)) = self.nodes.get(full_path) { + // Leaf is present in the trie, but it's blinded. + return Err(SparseTrieErrorKind::BlindedNode { path: *full_path, hash }.into()) + } - // Check if the path is in the prefix set. - // First, check the cached value. If it's `None`, then check the prefix set, and update - // the cached value. - let mut prefix_set_contains = - |path: &Nibbles| *is_in_prefix_set.get_or_insert_with(|| prefix_set.contains(path)); + trace!(target: "trie::sparse", ?full_path, "Leaf node is not present in the trie"); + // Leaf is not present in the trie. + return Ok(()) + } + self.prefix_set.insert(*full_path); - let (rlp_node, node_type) = match node { - SparseNode::Empty => (RlpNode::word_rlp(&EMPTY_ROOT_HASH), SparseNodeType::Empty), - SparseNode::Hash(hash) => (RlpNode::word_rlp(hash), SparseNodeType::Hash), - SparseNode::Leaf { key, hash } => { - let mut path = path.clone(); - path.extend_from_slice_unchecked(key); - if let Some(hash) = hash.filter(|_| !prefix_set_contains(&path)) { - (RlpNode::word_rlp(&hash), SparseNodeType::Leaf) - } else { - let value = self.values.get(&path).unwrap(); - self.rlp_buf.clear(); - let rlp_node = LeafNodeRef { key, value }.rlp(&mut self.rlp_buf); - *hash = rlp_node.as_hash(); - (rlp_node, SparseNodeType::Leaf) - } - } - SparseNode::Extension { key, hash, store_in_db_trie } => { - let mut child_path = path.clone(); - child_path.extend_from_slice_unchecked(key); - if let Some((hash, store_in_db_trie)) = - hash.zip(*store_in_db_trie).filter(|_| !prefix_set_contains(&path)) - { - ( - RlpNode::word_rlp(&hash), - SparseNodeType::Extension { store_in_db_trie: Some(store_in_db_trie) }, - ) - } else if buffers.rlp_node_stack.last().is_some_and(|e| e.path == child_path) { - let RlpNodeStackItem { - path: _, - rlp_node: child, - node_type: child_node_type, - } = buffers.rlp_node_stack.pop().unwrap(); - self.rlp_buf.clear(); - let rlp_node = ExtensionNodeRef::new(key, &child).rlp(&mut self.rlp_buf); - *hash = rlp_node.as_hash(); + // If the path wasn't present in `values`, we still need to walk the trie and ensure that + // there is no node at the path. When a leaf node is a blinded `Hash`, it will have an entry + // in `nodes`, but not in the `values`. - let store_in_db_trie_value = child_node_type.store_in_db_trie(); + let mut removed_nodes = self.take_nodes_for_path(full_path)?; + // Pop the first node from the stack which is the leaf node we want to remove. + let mut child = removed_nodes.pop().expect("leaf exists"); + #[cfg(debug_assertions)] + { + let mut child_path = child.path; + let SparseNode::Leaf { key, .. } = &child.node else { panic!("expected leaf node") }; + child_path.extend(key); + assert_eq!(&child_path, full_path); + } - trace!( - target: "trie::sparse", - ?path, - ?child_path, - ?child_node_type, - "Extension node" - ); + // If we don't have any other removed nodes, insert an empty node at the root. + if removed_nodes.is_empty() { + debug_assert!(self.nodes.is_empty()); + self.nodes.insert(Nibbles::default(), SparseNode::Empty); - *store_in_db_trie = store_in_db_trie_value; + return Ok(()) + } - ( - rlp_node, - SparseNodeType::Extension { - // Inherit the `store_in_db_trie` flag from the child node, which is - // always the branch node - store_in_db_trie: store_in_db_trie_value, - }, - ) - } else { - // need to get rlp node for child first - buffers.path_stack.extend([ - RlpNodePathStackItem { level, path, is_in_prefix_set }, - RlpNodePathStackItem { - level: level + 1, - path: child_path, - is_in_prefix_set: None, - }, - ]); - continue - } + // Walk the stack of removed nodes from the back and re-insert them back into the trie, + // adjusting the node type as needed. + while let Some(removed_node) = removed_nodes.pop() { + let removed_path = removed_node.path; + + let new_node = match &removed_node.node { + SparseNode::Empty => return Err(SparseTrieErrorKind::Blind.into()), + &SparseNode::Hash(hash) => { + return Err(SparseTrieErrorKind::BlindedNode { path: removed_path, hash }.into()) } - SparseNode::Branch { state_mask, hash, store_in_db_trie } => { - if let Some((hash, store_in_db_trie)) = - hash.zip(*store_in_db_trie).filter(|_| !prefix_set_contains(&path)) - { - buffers.rlp_node_stack.push(RlpNodeStackItem { - path, - rlp_node: RlpNode::word_rlp(&hash), - node_type: SparseNodeType::Branch { - store_in_db_trie: Some(store_in_db_trie), - }, - }); - continue - } - let retain_updates = self.updates.is_some() && prefix_set_contains(&path); + SparseNode::Leaf { .. } => { + unreachable!("we already popped the leaf node") + } + SparseNode::Extension { key, .. } => { + // If the node is an extension node, we need to look at its child to see if we + // need to merge them. + match &child.node { + SparseNode::Empty => return Err(SparseTrieErrorKind::Blind.into()), + &SparseNode::Hash(hash) => { + return Err( + SparseTrieErrorKind::BlindedNode { path: child.path, hash }.into() + ) + } + // For a leaf node, we collapse the extension node into a leaf node, + // extending the key. While it's impossible to encounter an extension node + // followed by a leaf node in a complete trie, it's possible here because we + // could have downgraded the extension node's child into a leaf node from + // another node type. + SparseNode::Leaf { key: leaf_key, .. } => { + self.nodes.remove(&child.path); - buffers.branch_child_buf.clear(); - // Walk children in a reverse order from `f` to `0`, so we pop the `0` first - // from the stack and keep walking in the sorted order. - for bit in CHILD_INDEX_RANGE.rev() { - if state_mask.is_bit_set(bit) { - let mut child = path.clone(); - child.push_unchecked(bit); - buffers.branch_child_buf.push(child); + let mut new_key = *key; + new_key.extend(leaf_key); + SparseNode::new_leaf(new_key) } - } + // For an extension node, we collapse them into one extension node, + // extending the key + SparseNode::Extension { key: extension_key, .. } => { + self.nodes.remove(&child.path); - buffers - .branch_value_stack_buf - .resize(buffers.branch_child_buf.len(), Default::default()); - let mut added_children = false; + let mut new_key = *key; + new_key.extend(extension_key); + SparseNode::new_ext(new_key) + } + // For a branch node, we just leave the extension node as-is. + SparseNode::Branch { .. } => removed_node.node, + } + } + &SparseNode::Branch { mut state_mask, hash: _, store_in_db_trie: _ } => { + // If the node is a branch node, we need to check the number of children left + // after deleting the child at the given nibble. - let mut tree_mask = TrieMask::default(); - let mut hash_mask = TrieMask::default(); - let mut hashes = Vec::new(); - for (i, child_path) in buffers.branch_child_buf.iter().enumerate() { - if buffers.rlp_node_stack.last().is_some_and(|e| &e.path == child_path) { - let RlpNodeStackItem { - path: _, - rlp_node: child, - node_type: child_node_type, - } = buffers.rlp_node_stack.pop().unwrap(); + if let Some(removed_nibble) = removed_node.unset_branch_nibble { + state_mask.unset_bit(removed_nibble); + } - // Update the masks only if we need to retain trie updates - if retain_updates { - // SAFETY: it's a child, so it's never empty - let last_child_nibble = child_path.last().unwrap(); + // If only one child is left set in the branch node, we need to collapse it. + if state_mask.count_bits() == 1 { + let child_nibble = + state_mask.first_set_bit_index().expect("state mask is not empty"); - // Determine whether we need to set trie mask bit. - let should_set_tree_mask_bit = if let Some(store_in_db_trie) = - child_node_type.store_in_db_trie() - { - // A branch or an extension node explicitly set the - // `store_in_db_trie` flag - store_in_db_trie - } else { - // A blinded node has the tree mask bit set - child_node_type.is_hash() && - self.branch_node_tree_masks.get(&path).is_some_and( - |mask| mask.is_bit_set(last_child_nibble), - ) - }; - if should_set_tree_mask_bit { - tree_mask.set_bit(last_child_nibble); - } + // Get full path of the only child node left. + let mut child_path = removed_path; + child_path.push_unchecked(child_nibble); - // Set the hash mask. If a child node is a revealed branch node OR - // is a blinded node that has its hash mask bit set according to the - // database, set the hash mask bit and save the hash. - let hash = child.as_hash().filter(|_| { - child_node_type.is_branch() || - (child_node_type.is_hash() && - self.branch_node_hash_masks - .get(&path) - .is_some_and(|mask| { - mask.is_bit_set(last_child_nibble) - })) - }); - if let Some(hash) = hash { - hash_mask.set_bit(last_child_nibble); - hashes.push(hash); - } - } + trace!(target: "trie::sparse", ?removed_path, ?child_path, "Branch node has only one child"); - // Insert children in the resulting buffer in a normal order, - // because initially we iterated in reverse. - // SAFETY: i < len and len is never 0 - let original_idx = buffers.branch_child_buf.len() - i - 1; - buffers.branch_value_stack_buf[original_idx] = child; - added_children = true; - } else { - debug_assert!(!added_children); - buffers.path_stack.push(RlpNodePathStackItem { - level, - path, - is_in_prefix_set, - }); - buffers.path_stack.extend(buffers.branch_child_buf.drain(..).map( - |path| RlpNodePathStackItem { - level: level + 1, - path, - is_in_prefix_set: None, - }, - )); - continue 'main - } - } + // If the remaining child node is not yet revealed then we have to reveal + // it here, otherwise it's not possible to know how to collapse the branch. + let child = self.reveal_remaining_child_on_leaf_removal( + &provider, + full_path, + &child_path, + true, // recurse_into_extension + )?; - trace!( - target: "trie::sparse", - ?path, - ?tree_mask, - ?hash_mask, - "Branch node masks" - ); + let mut delete_child = false; + let new_node = match &child { + SparseNode::Empty => return Err(SparseTrieErrorKind::Blind.into()), + &SparseNode::Hash(hash) => { + return Err(SparseTrieErrorKind::BlindedNode { + path: child_path, + hash, + } + .into()) + } + // If the only child is a leaf node, we downgrade the branch node into a + // leaf node, prepending the nibble to the key, and delete the old + // child. + SparseNode::Leaf { key, .. } => { + delete_child = true; - self.rlp_buf.clear(); - let branch_node_ref = - BranchNodeRef::new(&buffers.branch_value_stack_buf, *state_mask); - let rlp_node = branch_node_ref.rlp(&mut self.rlp_buf); - *hash = rlp_node.as_hash(); + let mut new_key = Nibbles::from_nibbles_unchecked([child_nibble]); + new_key.extend(key); + SparseNode::new_leaf(new_key) + } + // If the only child node is an extension node, we downgrade the branch + // node into an even longer extension node, prepending the nibble to the + // key, and delete the old child. + SparseNode::Extension { key, .. } => { + delete_child = true; - // Save a branch node update only if it's not a root node, and we need to - // persist updates. - let store_in_db_trie_value = if let Some(updates) = - self.updates.as_mut().filter(|_| retain_updates && !path.is_empty()) - { - let store_in_db_trie = !tree_mask.is_empty() || !hash_mask.is_empty(); - if store_in_db_trie { - // Store in DB trie if there are either any children that are stored in - // the DB trie, or any children represent hashed values - hashes.reverse(); - let branch_node = BranchNodeCompact::new( - *state_mask, - tree_mask, - hash_mask, - hashes, - hash.filter(|_| path.is_empty()), - ); - updates.updated_nodes.insert(path.clone(), branch_node); - } else if self - .branch_node_tree_masks - .get(&path) - .is_some_and(|mask| !mask.is_empty()) || - self.branch_node_hash_masks - .get(&path) - .is_some_and(|mask| !mask.is_empty()) - { - // If new tree and hash masks are empty, but previously they weren't, we - // need to remove the node update and add the node itself to the list of - // removed nodes. - updates.updated_nodes.remove(&path); - updates.removed_nodes.insert(path.clone()); - } else if self - .branch_node_hash_masks - .get(&path) - .is_none_or(|mask| mask.is_empty()) && - self.branch_node_hash_masks - .get(&path) - .is_none_or(|mask| mask.is_empty()) - { - // If new tree and hash masks are empty, and they were previously empty - // as well, we need to remove the node update. - updates.updated_nodes.remove(&path); + let mut new_key = Nibbles::from_nibbles_unchecked([child_nibble]); + new_key.extend(key); + SparseNode::new_ext(new_key) + } + // If the only child is a branch node, we downgrade the current branch + // node into a one-nibble extension node. + SparseNode::Branch { .. } => { + SparseNode::new_ext(Nibbles::from_nibbles_unchecked([child_nibble])) + } + }; + + if delete_child { + self.nodes.remove(&child_path); } - store_in_db_trie - } else { - false - }; - *store_in_db_trie = Some(store_in_db_trie_value); + if let Some(updates) = self.updates.as_mut() { + updates.updated_nodes.remove(&removed_path); + updates.removed_nodes.insert(removed_path); + } - ( - rlp_node, - SparseNodeType::Branch { store_in_db_trie: Some(store_in_db_trie_value) }, - ) + new_node + } + // If more than one child is left set in the branch, we just re-insert it as-is. + else { + SparseNode::new_branch(state_mask) + } } }; - trace!( - target: "trie::sparse", - ?_starting_path, - ?level, - ?path, - ?node, - ?node_type, - ?is_in_prefix_set, - "Added node to rlp node stack" - ); + child = RemovedSparseNode { + path: removed_path, + node: new_node.clone(), + unset_branch_nibble: None, + }; + trace!(target: "trie::sparse", ?removed_path, ?new_node, "Re-inserting the node"); + self.nodes.insert(removed_path, new_node); + } - buffers.rlp_node_stack.push(RlpNodeStackItem { path, rlp_node, node_type }); + Ok(()) + } + + #[instrument(target = "trie::sparse::serial", skip(self))] + fn root(&mut self) -> B256 { + // Take the current prefix set + let mut prefix_set = core::mem::take(&mut self.prefix_set).freeze(); + let rlp_node = self.rlp_node_allocate(&mut prefix_set); + if let Some(root_hash) = rlp_node.as_hash() { + root_hash + } else { + keccak256(rlp_node) } + } - debug_assert_eq!(buffers.rlp_node_stack.len(), 1); - buffers.rlp_node_stack.pop().unwrap().rlp_node + fn update_subtrie_hashes(&mut self) { + self.update_rlp_node_level(SPARSE_TRIE_SUBTRIE_HASHES_LEVEL); } -} -/// Error type for a leaf lookup operation -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum LeafLookupError { - /// The path leads to a blinded node, cannot determine if leaf exists. - /// This means the witness is not complete. - BlindedNode { - /// Path to the blinded node. - path: Nibbles, - /// Hash of the blinded node. - hash: B256, - }, - /// The path leads to a leaf with a different value than expected. - /// This means the witness is malformed. - ValueMismatch { - /// Path to the leaf. - path: Nibbles, - /// Expected value. - expected: Option>, - /// Actual value found. - actual: Vec, - }, -} + fn get_leaf_value(&self, full_path: &Nibbles) -> Option<&Vec> { + self.values.get(full_path) + } -/// Success value for a leaf lookup operation -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum LeafLookup { - /// Leaf exists with expected value. - Exists, - /// Leaf does not exist (exclusion proof found). - NonExistent { - /// Path where the search diverged from the target path. - diverged_at: Nibbles, - }, -} + fn updates_ref(&self) -> Cow<'_, SparseTrieUpdates> { + self.updates.as_ref().map_or(Cow::Owned(SparseTrieUpdates::default()), Cow::Borrowed) + } -impl RevealedSparseTrie

{ - /// Attempts to find a leaf node at the specified path. - /// - /// This method traverses the trie from the root down to the given path, checking - /// if a leaf exists at that path. It can be used to verify the existence of a leaf - /// or to generate an exclusion proof (proof that a leaf does not exist). - /// - /// # Parameters - /// - /// - `path`: The path to search for. - /// - `expected_value`: Optional expected value. If provided, will verify the leaf value - /// matches. - /// - /// # Returns - /// - /// - `Ok(LeafLookup::Exists)` if the leaf exists with the expected value. - /// - `Ok(LeafLookup::NonExistent)` if the leaf definitely does not exist (exclusion proof). - /// - `Err(LeafLookupError)` if the search encountered a blinded node or found a different - /// value. - pub fn find_leaf( + fn take_updates(&mut self) -> SparseTrieUpdates { + self.updates.take().unwrap_or_default() + } + + fn wipe(&mut self) { + self.nodes = HashMap::from_iter([(Nibbles::default(), SparseNode::Empty)]); + self.values = HashMap::default(); + self.prefix_set = PrefixSetMut::all(); + self.updates = self.updates.is_some().then(SparseTrieUpdates::wiped); + } + + fn clear(&mut self) { + self.nodes.clear(); + self.nodes.insert(Nibbles::default(), SparseNode::Empty); + + self.branch_node_tree_masks.clear(); + self.branch_node_hash_masks.clear(); + self.values.clear(); + self.prefix_set.clear(); + self.updates = None; + self.rlp_buf.clear(); + } + + fn find_leaf( &self, - path: &Nibbles, + full_path: &Nibbles, expected_value: Option<&Vec>, ) -> Result { // Helper function to check if a value matches the expected value + #[inline] fn check_value_match( actual_value: &Vec, expected_value: Option<&Vec>, path: &Nibbles, ) -> Result<(), LeafLookupError> { - if let Some(expected) = expected_value { - if actual_value != expected { - return Err(LeafLookupError::ValueMismatch { - path: path.clone(), - expected: Some(expected.clone()), - actual: actual_value.clone(), - }); - } + if let Some(expected) = expected_value && + actual_value != expected + { + return Err(LeafLookupError::ValueMismatch { + path: *path, + expected: Some(expected.clone()), + actual: actual_value.clone(), + }); } Ok(()) } @@ -1365,9 +999,9 @@ impl RevealedSparseTrie

{ // First, do a quick check if the value exists in our values map. // We assume that if there exists a leaf node, then its value will // be in the `values` map. - if let Some(actual_value) = self.values.get(path) { + if let Some(actual_value) = self.values.get(full_path) { // We found the leaf, check if the value matches (if expected value was provided) - check_value_match(actual_value, expected_value, path)?; + check_value_match(actual_value, expected_value, full_path)?; return Ok(LeafLookup::Exists); } @@ -1377,435 +1011,786 @@ impl RevealedSparseTrie

{ // We traverse the trie to find the location where this leaf would have been, showing // that it is not in the trie. Or we find a blinded node, showing that the witness is // not complete. - while current.len() < path.len() { + while current.len() < full_path.len() { match self.nodes.get(¤t) { Some(SparseNode::Empty) | None => { // None implies no node is at the current path (even in the full trie) // Empty node means there is a node at this path and it is "Empty" - return Ok(LeafLookup::NonExistent { diverged_at: current }); + return Ok(LeafLookup::NonExistent); } Some(&SparseNode::Hash(hash)) => { // We hit a blinded node - cannot determine if leaf exists - return Err(LeafLookupError::BlindedNode { path: current.clone(), hash }); + return Err(LeafLookupError::BlindedNode { path: current, hash }); } Some(SparseNode::Leaf { key, .. }) => { // We found a leaf node before reaching our target depth + current.extend(key); + if ¤t == full_path { + // This should have been handled by our initial values map check + if let Some(value) = self.values.get(full_path) { + check_value_match(value, expected_value, full_path)?; + return Ok(LeafLookup::Exists); + } + } - // Temporarily append the leaf key to `current` + // The leaf node's path doesn't match our target path, + // providing an exclusion proof + return Ok(LeafLookup::NonExistent); + } + Some(SparseNode::Extension { key, .. }) => { + // Temporarily append the extension key to `current` let saved_len = current.len(); - current.extend_from_slice_unchecked(key); + current.extend(key); - if ¤t == path { - // This should have been handled by our initial values map check - if let Some(value) = self.values.get(path) { - check_value_match(value, expected_value, path)?; - return Ok(LeafLookup::Exists); + if full_path.len() < current.len() || !full_path.starts_with(¤t) { + current.truncate(saved_len); // restore + return Ok(LeafLookup::NonExistent); + } + // Prefix matched, so we keep walking with the longer `current`. + } + Some(SparseNode::Branch { state_mask, .. }) => { + // Check if branch has a child at the next nibble in our path + let nibble = full_path.get_unchecked(current.len()); + if !state_mask.is_bit_set(nibble) { + // No child at this nibble - exclusion proof + return Ok(LeafLookup::NonExistent); + } + + // Continue down the branch + current.push_unchecked(nibble); + } + } + } + + // We've traversed to the end of the path and didn't find a leaf + // Check if there's a node exactly at our target path + match self.nodes.get(full_path) { + Some(SparseNode::Leaf { key, .. }) if key.is_empty() => { + // We found a leaf with an empty key (exact match) + // This should be handled by the values map check above + if let Some(value) = self.values.get(full_path) { + check_value_match(value, expected_value, full_path)?; + return Ok(LeafLookup::Exists); + } + } + Some(&SparseNode::Hash(hash)) => { + return Err(LeafLookupError::BlindedNode { path: *full_path, hash }); + } + _ => { + // No leaf at exactly the target path + return Ok(LeafLookup::NonExistent); + } + } + + // If we get here, there's no leaf at the target path + Ok(LeafLookup::NonExistent) + } + + fn shrink_nodes_to(&mut self, size: usize) { + self.nodes.shrink_to(size); + self.branch_node_tree_masks.shrink_to(size); + self.branch_node_hash_masks.shrink_to(size); + } + + fn shrink_values_to(&mut self, size: usize) { + self.values.shrink_to(size); + } +} + +impl SerialSparseTrie { + /// Creates a new revealed sparse trie from the given root node. + /// + /// This function initializes the internal structures and then reveals the root. + /// It is a convenient method to create a trie when you already have the root node available. + /// + /// # Arguments + /// + /// * `root` - The root node of the trie + /// * `masks` - Trie masks for root branch node + /// * `retain_updates` - Whether to track updates + /// + /// # Returns + /// + /// Self if successful, or an error if revealing fails. + pub fn from_root( + root: TrieNode, + masks: TrieMasks, + retain_updates: bool, + ) -> SparseTrieResult { + Self::default().with_root(root, masks, retain_updates) + } + + /// Returns a reference to the current sparse trie updates. + /// + /// If no updates have been made/recorded, returns an empty update set. + pub fn updates_ref(&self) -> Cow<'_, SparseTrieUpdates> { + self.updates.as_ref().map_or(Cow::Owned(SparseTrieUpdates::default()), Cow::Borrowed) + } + + /// Returns an immutable reference to all nodes in the sparse trie. + pub const fn nodes_ref(&self) -> &HashMap { + &self.nodes + } + + /// Reveals either a node or its hash placeholder based on the provided child data. + /// + /// When traversing the trie, we often encounter references to child nodes that + /// are either directly embedded or represented by their hash. This method + /// handles both cases: + /// + /// 1. If the child data represents a hash (32+1=33 bytes), store it as a hash node + /// 2. Otherwise, decode the data as a [`TrieNode`] and recursively reveal it using + /// `reveal_node` + /// + /// # Returns + /// + /// Returns `Ok(())` if successful, or an error if the node cannot be revealed. + /// + /// # Error Handling + /// + /// Will error if there's a conflict between a new hash node and an existing one + /// at the same path + fn reveal_node_or_hash(&mut self, path: Nibbles, child: &[u8]) -> SparseTrieResult<()> { + if child.len() == B256::len_bytes() + 1 { + let hash = B256::from_slice(&child[1..]); + match self.nodes.entry(path) { + Entry::Occupied(entry) => match entry.get() { + // Hash node with a different hash can't be handled. + SparseNode::Hash(previous_hash) if previous_hash != &hash => { + return Err(SparseTrieErrorKind::Reveal { + path: *entry.key(), + node: Box::new(SparseNode::Hash(hash)), } + .into()) } + _ => {} + }, + Entry::Vacant(entry) => { + entry.insert(SparseNode::Hash(hash)); + } + } + return Ok(()) + } - let diverged_at = current.slice(..saved_len); + self.reveal_node(path, TrieNode::decode(&mut &child[..])?, TrieMasks::none()) + } - // The leaf node's path doesn't match our target path, - // providing an exclusion proof - return Ok(LeafLookup::NonExistent { diverged_at }); + /// Traverse the trie from the root down to the leaf at the given path, + /// removing and collecting all nodes along that path. + /// + /// This helper function is used during leaf removal to extract the nodes of the trie + /// that will be affected by the deletion. These nodes are then re-inserted and modified + /// as needed (collapsing extension nodes etc) given that the leaf has now been removed. + /// + /// # Returns + /// + /// Returns a vector of [`RemovedSparseNode`] representing the nodes removed during the + /// traversal. + /// + /// # Errors + /// + /// Returns an error if a blinded node or an empty node is encountered unexpectedly, + /// as these prevent proper removal of the leaf. + fn take_nodes_for_path(&mut self, path: &Nibbles) -> SparseTrieResult> { + let mut current = Nibbles::default(); // Start traversal from the root + let mut nodes = Vec::new(); // Collect traversed nodes + + while let Some(node) = self.nodes.remove(¤t) { + match &node { + SparseNode::Empty => return Err(SparseTrieErrorKind::Blind.into()), + &SparseNode::Hash(hash) => { + return Err(SparseTrieErrorKind::BlindedNode { path: current, hash }.into()) } - Some(SparseNode::Extension { key, .. }) => { - // Temporarily append the extension key to `current` - let saved_len = current.len(); - current.extend_from_slice_unchecked(key); + SparseNode::Leaf { key: _key, .. } => { + // Leaf node is always the one that we're deleting, and no other leaf nodes can + // be found during traversal. - if path.len() < current.len() || !path.starts_with(¤t) { - let diverged_at = current.slice(..saved_len); - current.truncate(saved_len); // restore - return Ok(LeafLookup::NonExistent { diverged_at }); + #[cfg(debug_assertions)] + { + let mut current = current; + current.extend(_key); + assert_eq!(¤t, path); } - // Prefix matched, so we keep walking with the longer `current`. + + nodes.push(RemovedSparseNode { + path: current, + node, + unset_branch_nibble: None, + }); + break } - Some(SparseNode::Branch { state_mask, .. }) => { - // Check if branch has a child at the next nibble in our path - let nibble = path[current.len()]; - if !state_mask.is_bit_set(nibble) { - // No child at this nibble - exclusion proof - return Ok(LeafLookup::NonExistent { diverged_at: current }); + SparseNode::Extension { key, .. } => { + #[cfg(debug_assertions)] + { + let mut current = current; + current.extend(key); + assert!( + path.starts_with(¤t), + "path: {path:?}, current: {current:?}, key: {key:?}", + ); } - // Continue down the branch - current.push_unchecked(nibble); + let path = current; + current.extend(key); + nodes.push(RemovedSparseNode { path, node, unset_branch_nibble: None }); } - } - } + SparseNode::Branch { state_mask, .. } => { + let nibble = path.get_unchecked(current.len()); + debug_assert!( + state_mask.is_bit_set(nibble), + "current: {current:?}, path: {path:?}, nibble: {nibble:?}, state_mask: {state_mask:?}", + ); - // We've traversed to the end of the path and didn't find a leaf - // Check if there's a node exactly at our target path - match self.nodes.get(path) { - Some(SparseNode::Leaf { key, .. }) if key.is_empty() => { - // We found a leaf with an empty key (exact match) - // This should be handled by the values map check above - if let Some(value) = self.values.get(path) { - check_value_match(value, expected_value, path)?; - return Ok(LeafLookup::Exists); + // If the branch node has a child that is a leaf node that we're removing, + // we need to unset this nibble. + // Any other branch nodes will not require unsetting the nibble, because + // deleting one leaf node can not remove the whole path + // where the branch node is located. + let mut child_path = current; + child_path.push_unchecked(nibble); + let unset_branch_nibble = self + .nodes + .get(&child_path) + .is_some_and(move |node| match node { + SparseNode::Leaf { key, .. } => { + // Get full path of the leaf node + child_path.extend(key); + &child_path == path + } + _ => false, + }) + .then_some(nibble); + + nodes.push(RemovedSparseNode { path: current, node, unset_branch_nibble }); + + current.push_unchecked(nibble); } } - Some(&SparseNode::Hash(hash)) => { - return Err(LeafLookupError::BlindedNode { path: path.clone(), hash }); - } - _ => { - // No leaf at exactly the target path - let parent_path = if path.is_empty() { - Nibbles::default() - } else { - path.slice(0..path.len() - 1) - }; - return Ok(LeafLookup::NonExistent { diverged_at: parent_path }); - } } - // If we get here, there's no leaf at the target path - Ok(LeafLookup::NonExistent { diverged_at: current }) + Ok(nodes) } - /// Updates or inserts a leaf node at the specified key path with the provided RLP-encoded - /// value. - /// - /// This method updates the internal prefix set and, if the leaf did not previously exist, - /// adjusts the trie structure by inserting new leaf nodes, splitting branch nodes, or - /// collapsing extension nodes as needed. + /// Called when a leaf is removed on a branch which has only one other remaining child. That + /// child must be revealed in order to properly collapse the branch. /// - /// # Returns + /// If `recurse_into_extension` is true, and the remaining child is an extension node, then its + /// child will be ensured to be revealed as well. /// - /// Returns `Ok(())` if the update is successful. + /// ## Returns /// - /// Note: If an update requires revealing a blinded node, an error is returned if the blinded - /// provider returns an error. - pub fn update_leaf(&mut self, path: Nibbles, value: Vec) -> SparseTrieResult<()> { - self.prefix_set.insert(path.clone()); - let existing = self.values.insert(path.clone(), value); - if existing.is_some() { - // trie structure unchanged, return immediately - return Ok(()) - } - - let mut current = Nibbles::default(); - while let Some(node) = self.nodes.get_mut(¤t) { - match node { - SparseNode::Empty => { - *node = SparseNode::new_leaf(path); - break - } - &mut SparseNode::Hash(hash) => { - return Err(SparseTrieErrorKind::BlindedNode { path: current, hash }.into()) + /// The node of the remaining child, whether it was already revealed or not. + fn reveal_remaining_child_on_leaf_removal( + &mut self, + provider: P, + full_path: &Nibbles, // only needed for logs + remaining_child_path: &Nibbles, + recurse_into_extension: bool, + ) -> SparseTrieResult { + let remaining_child_node = match self.nodes.get(remaining_child_path).unwrap() { + SparseNode::Hash(_) => { + debug!( + target: "trie::parallel_sparse", + child_path = ?remaining_child_path, + leaf_full_path = ?full_path, + "Node child not revealed in remove_leaf, falling back to db", + ); + if let Some(RevealedNode { node, tree_mask, hash_mask }) = + provider.trie_node(remaining_child_path)? + { + let decoded = TrieNode::decode(&mut &node[..])?; + trace!( + target: "trie::parallel_sparse", + ?remaining_child_path, + ?decoded, + ?tree_mask, + ?hash_mask, + "Revealing remaining blinded branch child" + ); + self.reveal_node( + *remaining_child_path, + decoded, + TrieMasks { hash_mask, tree_mask }, + )?; + self.nodes.get(remaining_child_path).unwrap().clone() + } else { + return Err(SparseTrieErrorKind::NodeNotFoundInProvider { + path: *remaining_child_path, + } + .into()) } - SparseNode::Leaf { key: current_key, .. } => { - current.extend_from_slice_unchecked(current_key); + } + node => node.clone(), + }; - // this leaf is being updated - if current == path { - unreachable!("we already checked leaf presence in the beginning"); - } + // If `recurse_into_extension` is true, and the remaining child is an extension node, then + // its child will be ensured to be revealed as well. This is required for generation of + // trie updates; without revealing the grandchild branch it's not always possible to know + // if the tree mask bit should be set for the child extension on its parent branch. + if let SparseNode::Extension { key, .. } = &remaining_child_node && + recurse_into_extension + { + let mut remaining_grandchild_path = *remaining_child_path; + remaining_grandchild_path.extend(key); - // find the common prefix - let common = current.common_prefix_length(&path); + trace!( + target: "trie::parallel_sparse", + remaining_grandchild_path = ?remaining_grandchild_path, + child_path = ?remaining_child_path, + leaf_full_path = ?full_path, + "Revealing child of extension node, which is the last remaining child of the branch" + ); - // update existing node - let new_ext_key = current.slice(current.len() - current_key.len()..common); - *node = SparseNode::new_ext(new_ext_key); + self.reveal_remaining_child_on_leaf_removal( + provider, + full_path, + &remaining_grandchild_path, + false, // recurse_into_extension + )?; + } - // create a branch node and corresponding leaves - self.nodes.reserve(3); - self.nodes.insert( - current.slice(..common), - SparseNode::new_split_branch(current[common], path[common]), - ); - self.nodes.insert( - path.slice(..=common), - SparseNode::new_leaf(path.slice(common + 1..)), - ); - self.nodes.insert( - current.slice(..=common), - SparseNode::new_leaf(current.slice(common + 1..)), - ); + Ok(remaining_child_node) + } - break; - } - SparseNode::Extension { key, .. } => { - current.extend_from_slice(key); + /// Recalculates and updates the RLP hashes of nodes deeper than or equal to the specified + /// `depth`. + /// + /// The root node is considered to be at level 0. This method is useful for optimizing + /// hash recalculations after localized changes to the trie structure: + /// + /// This function identifies all nodes that have changed (based on the prefix set) at the given + /// depth and recalculates their RLP representation. + #[instrument(level = "trace", target = "trie::sparse::serial", skip(self))] + pub fn update_rlp_node_level(&mut self, depth: usize) { + // Take the current prefix set + let mut prefix_set = core::mem::take(&mut self.prefix_set).freeze(); + let mut buffers = RlpNodeBuffers::default(); - if !path.starts_with(¤t) { - // find the common prefix - let common = current.common_prefix_length(&path); - *key = current.slice(current.len() - key.len()..common); + // Get the nodes that have changed at the given depth. + let (targets, new_prefix_set) = self.get_changed_nodes_at_depth(&mut prefix_set, depth); + // Update the prefix set to the prefix set of the nodes that still need to be updated. + self.prefix_set = new_prefix_set; - // If branch node updates retention is enabled, we need to query the - // extension node child to later set the hash mask for a parent branch node - // correctly. - if self.updates.is_some() { - // Check if the extension node child is a hash that needs to be revealed - if self.nodes.get(¤t).unwrap().is_hash() { - if let Some(RevealedNode { node, tree_mask, hash_mask }) = - self.provider.blinded_node(¤t)? - { - let decoded = TrieNode::decode(&mut &node[..])?; - trace!( - target: "trie::sparse", - ?current, - ?decoded, - ?tree_mask, - ?hash_mask, - "Revealing extension node child", - ); - self.reveal_node( - current.clone(), - decoded, - TrieMasks { hash_mask, tree_mask }, - )?; - } - } - } + trace!(target: "trie::sparse", ?depth, ?targets, "Updating nodes at depth"); + + let mut temp_rlp_buf = core::mem::take(&mut self.rlp_buf); + for (level, path) in targets { + buffers.path_stack.push(RlpNodePathStackItem { + level, + path, + is_in_prefix_set: Some(true), + }); + self.rlp_node(&mut prefix_set, &mut buffers, &mut temp_rlp_buf); + } + self.rlp_buf = temp_rlp_buf; + } + + /// Returns a list of (level, path) tuples identifying the nodes that have changed at the + /// specified depth, along with a new prefix set for the paths above the provided depth that + /// remain unchanged. + /// + /// Leaf nodes with a depth less than `depth` are returned too. + /// + /// This method helps optimize hash recalculations by identifying which specific + /// nodes need to be updated at each level of the trie. + /// + /// # Parameters + /// + /// - `prefix_set`: The current prefix set tracking which paths need updates. + /// - `depth`: The minimum depth (relative to the root) to include nodes in the targets. + /// + /// # Returns + /// + /// A tuple containing: + /// - A vector of `(level, Nibbles)` pairs for nodes that require updates at or below the + /// specified depth. + /// - A `PrefixSetMut` containing paths shallower than the specified depth that still need to be + /// tracked for future updates. + #[instrument(level = "trace", target = "trie::sparse::serial", skip(self))] + fn get_changed_nodes_at_depth( + &self, + prefix_set: &mut PrefixSet, + depth: usize, + ) -> (Vec<(usize, Nibbles)>, PrefixSetMut) { + let mut unchanged_prefix_set = PrefixSetMut::default(); + let mut paths = Vec::from([(Nibbles::default(), 0)]); + let mut targets = Vec::new(); - // create state mask for new branch node - // NOTE: this might overwrite the current extension node - self.nodes.reserve(3); - let branch = SparseNode::new_split_branch(current[common], path[common]); - self.nodes.insert(current.slice(..common), branch); + while let Some((mut path, level)) = paths.pop() { + match self.nodes.get(&path).unwrap() { + SparseNode::Empty | SparseNode::Hash(_) => {} + SparseNode::Leaf { key: _, hash } => { + if hash.is_some() && !prefix_set.contains(&path) { + continue + } - // create new leaf - let new_leaf = SparseNode::new_leaf(path.slice(common + 1..)); - self.nodes.insert(path.slice(..=common), new_leaf); + targets.push((level, path)); + } + SparseNode::Extension { key, hash, store_in_db_trie: _ } => { + if hash.is_some() && !prefix_set.contains(&path) { + continue + } - // recreate extension to previous child if needed - let key = current.slice(common + 1..); - if !key.is_empty() { - self.nodes.insert(current.slice(..=common), SparseNode::new_ext(key)); - } + if level >= depth { + targets.push((level, path)); + } else { + unchanged_prefix_set.insert(path); - break; + path.extend(key); + paths.push((path, level + 1)); } } - SparseNode::Branch { state_mask, .. } => { - let nibble = path[current.len()]; - current.push_unchecked(nibble); - if !state_mask.is_bit_set(nibble) { - state_mask.set_bit(nibble); - let new_leaf = SparseNode::new_leaf(path.slice(current.len()..)); - self.nodes.insert(current, new_leaf); - break; + SparseNode::Branch { state_mask, hash, store_in_db_trie: _ } => { + if hash.is_some() && !prefix_set.contains(&path) { + continue + } + + if level >= depth { + targets.push((level, path)); + } else { + unchanged_prefix_set.insert(path); + + for bit in CHILD_INDEX_RANGE.rev() { + if state_mask.is_bit_set(bit) { + let mut child_path = path; + child_path.push_unchecked(bit); + paths.push((child_path, level + 1)); + } + } } } - }; + } } - Ok(()) + (targets, unchanged_prefix_set) } - /// Removes a leaf node from the trie at the specified key path. - /// - /// This function removes the leaf value from the internal values map and then traverses - /// the trie to remove or adjust intermediate nodes, merging or collapsing them as necessary. + /// Look up or calculate the RLP of the node at the root path. /// - /// # Returns + /// # Panics /// - /// Returns `Ok(())` if the leaf is successfully removed, otherwise returns an error - /// if the leaf is not present or if a blinded node prevents removal. - pub fn remove_leaf(&mut self, path: &Nibbles) -> SparseTrieResult<()> { - if self.values.remove(path).is_none() { - if let Some(&SparseNode::Hash(hash)) = self.nodes.get(path) { - // Leaf is present in the trie, but it's blinded. - return Err(SparseTrieErrorKind::BlindedNode { path: path.clone(), hash }.into()) - } + /// If the node at provided path does not exist. + pub fn rlp_node_allocate(&mut self, prefix_set: &mut PrefixSet) -> RlpNode { + let mut buffers = RlpNodeBuffers::new_with_root_path(); + let mut temp_rlp_buf = core::mem::take(&mut self.rlp_buf); + let result = self.rlp_node(prefix_set, &mut buffers, &mut temp_rlp_buf); + self.rlp_buf = temp_rlp_buf; - trace!(target: "trie::sparse", ?path, "Leaf node is not present in the trie"); - // Leaf is not present in the trie. - return Ok(()) - } - self.prefix_set.insert(path.clone()); + result + } - // If the path wasn't present in `values`, we still need to walk the trie and ensure that - // there is no node at the path. When a leaf node is a blinded `Hash`, it will have an entry - // in `nodes`, but not in the `values`. + /// Looks up or computes the RLP encoding of the node specified by the current + /// path in the provided buffers. + /// + /// The function uses a stack (`RlpNodeBuffers::path_stack`) to track the traversal and + /// accumulate RLP encodings. + /// + /// # Parameters + /// + /// - `prefix_set`: The set of trie paths that need their nodes updated. + /// - `buffers`: The reusable buffers for stack management and temporary RLP values. + /// + /// # Panics + /// + /// If the node at provided path does not exist. + #[instrument(level = "trace", target = "trie::sparse::serial", skip_all, ret(level = "trace"))] + pub fn rlp_node( + &mut self, + prefix_set: &mut PrefixSet, + buffers: &mut RlpNodeBuffers, + rlp_buf: &mut Vec, + ) -> RlpNode { + let _starting_path = buffers.path_stack.last().map(|item| item.path); - let mut removed_nodes = self.take_nodes_for_path(path)?; - trace!(target: "trie::sparse", ?path, ?removed_nodes, "Removed nodes for path"); - // Pop the first node from the stack which is the leaf node we want to remove. - let mut child = removed_nodes.pop().expect("leaf exists"); - #[cfg(debug_assertions)] + 'main: while let Some(RlpNodePathStackItem { level, path, mut is_in_prefix_set }) = + buffers.path_stack.pop() { - let mut child_path = child.path.clone(); - let SparseNode::Leaf { key, .. } = &child.node else { panic!("expected leaf node") }; - child_path.extend_from_slice_unchecked(key); - assert_eq!(&child_path, path); - } + let node = self.nodes.get_mut(&path).unwrap(); + trace!( + target: "trie::sparse", + ?_starting_path, + ?level, + ?path, + ?is_in_prefix_set, + ?node, + "Popped node from path stack" + ); - // If we don't have any other removed nodes, insert an empty node at the root. - if removed_nodes.is_empty() { - debug_assert!(self.nodes.is_empty()); - self.nodes.insert(Nibbles::default(), SparseNode::Empty); + // Check if the path is in the prefix set. + // First, check the cached value. If it's `None`, then check the prefix set, and update + // the cached value. + let mut prefix_set_contains = + |path: &Nibbles| *is_in_prefix_set.get_or_insert_with(|| prefix_set.contains(path)); - return Ok(()) - } + let (rlp_node, node_type) = match node { + SparseNode::Empty => (RlpNode::word_rlp(&EMPTY_ROOT_HASH), SparseNodeType::Empty), + SparseNode::Hash(hash) => (RlpNode::word_rlp(hash), SparseNodeType::Hash), + SparseNode::Leaf { key, hash } => { + let mut path = path; + path.extend(key); + if let Some(hash) = hash.filter(|_| !prefix_set_contains(&path)) { + (RlpNode::word_rlp(&hash), SparseNodeType::Leaf) + } else { + let value = self.values.get(&path).unwrap(); + rlp_buf.clear(); + let rlp_node = LeafNodeRef { key, value }.rlp(rlp_buf); + *hash = rlp_node.as_hash(); + (rlp_node, SparseNodeType::Leaf) + } + } + SparseNode::Extension { key, hash, store_in_db_trie } => { + let mut child_path = path; + child_path.extend(key); + if let Some((hash, store_in_db_trie)) = + hash.zip(*store_in_db_trie).filter(|_| !prefix_set_contains(&path)) + { + ( + RlpNode::word_rlp(&hash), + SparseNodeType::Extension { store_in_db_trie: Some(store_in_db_trie) }, + ) + } else if buffers.rlp_node_stack.last().is_some_and(|e| e.path == child_path) { + let RlpNodeStackItem { + path: _, + rlp_node: child, + node_type: child_node_type, + } = buffers.rlp_node_stack.pop().unwrap(); + rlp_buf.clear(); + let rlp_node = ExtensionNodeRef::new(key, &child).rlp(rlp_buf); + *hash = rlp_node.as_hash(); - // Walk the stack of removed nodes from the back and re-insert them back into the trie, - // adjusting the node type as needed. - while let Some(removed_node) = removed_nodes.pop() { - let removed_path = removed_node.path; + let store_in_db_trie_value = child_node_type.store_in_db_trie(); - let new_node = match &removed_node.node { - SparseNode::Empty => return Err(SparseTrieErrorKind::Blind.into()), - &SparseNode::Hash(hash) => { - return Err(SparseTrieErrorKind::BlindedNode { path: removed_path, hash }.into()) - } - SparseNode::Leaf { .. } => { - unreachable!("we already popped the leaf node") - } - SparseNode::Extension { key, .. } => { - // If the node is an extension node, we need to look at its child to see if we - // need to merge them. - match &child.node { - SparseNode::Empty => return Err(SparseTrieErrorKind::Blind.into()), - &SparseNode::Hash(hash) => { - return Err( - SparseTrieErrorKind::BlindedNode { path: child.path, hash }.into() - ) - } - // For a leaf node, we collapse the extension node into a leaf node, - // extending the key. While it's impossible to encounter an extension node - // followed by a leaf node in a complete trie, it's possible here because we - // could have downgraded the extension node's child into a leaf node from - // another node type. - SparseNode::Leaf { key: leaf_key, .. } => { - self.nodes.remove(&child.path); + trace!( + target: "trie::sparse", + ?path, + ?child_path, + ?child_node_type, + "Extension node" + ); - let mut new_key = key.clone(); - new_key.extend_from_slice_unchecked(leaf_key); - SparseNode::new_leaf(new_key) - } - // For an extension node, we collapse them into one extension node, - // extending the key - SparseNode::Extension { key: extension_key, .. } => { - self.nodes.remove(&child.path); + *store_in_db_trie = store_in_db_trie_value; - let mut new_key = key.clone(); - new_key.extend_from_slice_unchecked(extension_key); - SparseNode::new_ext(new_key) - } - // For a branch node, we just leave the extension node as-is. - SparseNode::Branch { .. } => removed_node.node, + ( + rlp_node, + SparseNodeType::Extension { + // Inherit the `store_in_db_trie` flag from the child node, which is + // always the branch node + store_in_db_trie: store_in_db_trie_value, + }, + ) + } else { + // need to get rlp node for child first + buffers.path_stack.extend([ + RlpNodePathStackItem { level, path, is_in_prefix_set }, + RlpNodePathStackItem { + level: level + 1, + path: child_path, + is_in_prefix_set: None, + }, + ]); + continue } } - &SparseNode::Branch { mut state_mask, hash: _, store_in_db_trie: _ } => { - // If the node is a branch node, we need to check the number of children left - // after deleting the child at the given nibble. - - if let Some(removed_nibble) = removed_node.unset_branch_nibble { - state_mask.unset_bit(removed_nibble); + SparseNode::Branch { state_mask, hash, store_in_db_trie } => { + if let Some((hash, store_in_db_trie)) = + hash.zip(*store_in_db_trie).filter(|_| !prefix_set_contains(&path)) + { + buffers.rlp_node_stack.push(RlpNodeStackItem { + path, + rlp_node: RlpNode::word_rlp(&hash), + node_type: SparseNodeType::Branch { + store_in_db_trie: Some(store_in_db_trie), + }, + }); + continue } + let retain_updates = self.updates.is_some() && prefix_set_contains(&path); - // If only one child is left set in the branch node, we need to collapse it. - if state_mask.count_bits() == 1 { - let child_nibble = - state_mask.first_set_bit_index().expect("state mask is not empty"); + buffers.branch_child_buf.clear(); + // Walk children in a reverse order from `f` to `0`, so we pop the `0` first + // from the stack and keep walking in the sorted order. + for bit in CHILD_INDEX_RANGE.rev() { + if state_mask.is_bit_set(bit) { + let mut child = path; + child.push_unchecked(bit); + buffers.branch_child_buf.push(child); + } + } - // Get full path of the only child node left. - let mut child_path = removed_path.clone(); - child_path.push_unchecked(child_nibble); + buffers + .branch_value_stack_buf + .resize(buffers.branch_child_buf.len(), Default::default()); + let mut added_children = false; - trace!(target: "trie::sparse", ?removed_path, ?child_path, "Branch node has only one child"); + let mut tree_mask = TrieMask::default(); + let mut hash_mask = TrieMask::default(); + let mut hashes = Vec::new(); + for (i, child_path) in buffers.branch_child_buf.iter().enumerate() { + if buffers.rlp_node_stack.last().is_some_and(|e| &e.path == child_path) { + let RlpNodeStackItem { + path: _, + rlp_node: child, + node_type: child_node_type, + } = buffers.rlp_node_stack.pop().unwrap(); - if self.nodes.get(&child_path).unwrap().is_hash() { - trace!(target: "trie::sparse", ?child_path, "Retrieving remaining blinded branch child"); - if let Some(RevealedNode { node, tree_mask, hash_mask }) = - self.provider.blinded_node(&child_path)? - { - let decoded = TrieNode::decode(&mut &node[..])?; - trace!( - target: "trie::sparse", - ?child_path, - ?decoded, - ?tree_mask, - ?hash_mask, - "Revealing remaining blinded branch child" - ); - self.reveal_node( - child_path.clone(), - decoded, - TrieMasks { hash_mask, tree_mask }, - )?; - } - } + // Update the masks only if we need to retain trie updates + if retain_updates { + // SAFETY: it's a child, so it's never empty + let last_child_nibble = child_path.last().unwrap(); - // Get the only child node. - let child = self.nodes.get(&child_path).unwrap(); + // Determine whether we need to set trie mask bit. + let should_set_tree_mask_bit = if let Some(store_in_db_trie) = + child_node_type.store_in_db_trie() + { + // A branch or an extension node explicitly set the + // `store_in_db_trie` flag + store_in_db_trie + } else { + // A blinded node has the tree mask bit set + child_node_type.is_hash() && + self.branch_node_tree_masks.get(&path).is_some_and( + |mask| mask.is_bit_set(last_child_nibble), + ) + }; + if should_set_tree_mask_bit { + tree_mask.set_bit(last_child_nibble); + } - let mut delete_child = false; - let new_node = match child { - SparseNode::Empty => return Err(SparseTrieErrorKind::Blind.into()), - &SparseNode::Hash(hash) => { - return Err(SparseTrieErrorKind::BlindedNode { - path: child_path, - hash, + // Set the hash mask. If a child node is a revealed branch node OR + // is a blinded node that has its hash mask bit set according to the + // database, set the hash mask bit and save the hash. + let hash = child.as_hash().filter(|_| { + child_node_type.is_branch() || + (child_node_type.is_hash() && + self.branch_node_hash_masks + .get(&path) + .is_some_and(|mask| { + mask.is_bit_set(last_child_nibble) + })) + }); + if let Some(hash) = hash { + hash_mask.set_bit(last_child_nibble); + hashes.push(hash); } - .into()) } - // If the only child is a leaf node, we downgrade the branch node into a - // leaf node, prepending the nibble to the key, and delete the old - // child. - SparseNode::Leaf { key, .. } => { - delete_child = true; - let mut new_key = Nibbles::from_nibbles_unchecked([child_nibble]); - new_key.extend_from_slice_unchecked(key); - SparseNode::new_leaf(new_key) - } - // If the only child node is an extension node, we downgrade the branch - // node into an even longer extension node, prepending the nibble to the - // key, and delete the old child. - SparseNode::Extension { key, .. } => { - delete_child = true; + // Insert children in the resulting buffer in a normal order, + // because initially we iterated in reverse. + // SAFETY: i < len and len is never 0 + let original_idx = buffers.branch_child_buf.len() - i - 1; + buffers.branch_value_stack_buf[original_idx] = child; + added_children = true; + } else { + debug_assert!(!added_children); + buffers.path_stack.push(RlpNodePathStackItem { + level, + path, + is_in_prefix_set, + }); + buffers.path_stack.extend(buffers.branch_child_buf.drain(..).map( + |path| RlpNodePathStackItem { + level: level + 1, + path, + is_in_prefix_set: None, + }, + )); + continue 'main + } + } - let mut new_key = Nibbles::from_nibbles_unchecked([child_nibble]); - new_key.extend_from_slice_unchecked(key); - SparseNode::new_ext(new_key) - } - // If the only child is a branch node, we downgrade the current branch - // node into a one-nibble extension node. - SparseNode::Branch { .. } => { - SparseNode::new_ext(Nibbles::from_nibbles_unchecked([child_nibble])) - } - }; + trace!( + target: "trie::sparse", + ?path, + ?tree_mask, + ?hash_mask, + "Branch node masks" + ); - if delete_child { - self.nodes.remove(&child_path); - } + rlp_buf.clear(); + let branch_node_ref = + BranchNodeRef::new(&buffers.branch_value_stack_buf, *state_mask); + let rlp_node = branch_node_ref.rlp(rlp_buf); + *hash = rlp_node.as_hash(); - if let Some(updates) = self.updates.as_mut() { - updates.updated_nodes.remove(&removed_path); - updates.removed_nodes.insert(removed_path.clone()); + // Save a branch node update only if it's not a root node, and we need to + // persist updates. + let store_in_db_trie_value = if let Some(updates) = + self.updates.as_mut().filter(|_| retain_updates && !path.is_empty()) + { + let store_in_db_trie = !tree_mask.is_empty() || !hash_mask.is_empty(); + if store_in_db_trie { + // Store in DB trie if there are either any children that are stored in + // the DB trie, or any children represent hashed values + hashes.reverse(); + let branch_node = BranchNodeCompact::new( + *state_mask, + tree_mask, + hash_mask, + hashes, + hash.filter(|_| path.is_empty()), + ); + updates.updated_nodes.insert(path, branch_node); + } else if self + .branch_node_tree_masks + .get(&path) + .is_some_and(|mask| !mask.is_empty()) || + self.branch_node_hash_masks + .get(&path) + .is_some_and(|mask| !mask.is_empty()) + { + // If new tree and hash masks are empty, but previously they weren't, we + // need to remove the node update and add the node itself to the list of + // removed nodes. + updates.updated_nodes.remove(&path); + updates.removed_nodes.insert(path); + } else if self + .branch_node_tree_masks + .get(&path) + .is_none_or(|mask| mask.is_empty()) && + self.branch_node_hash_masks + .get(&path) + .is_none_or(|mask| mask.is_empty()) + { + // If new tree and hash masks are empty, and they were previously empty + // as well, we need to remove the node update. + updates.updated_nodes.remove(&path); } - new_node - } - // If more than one child is left set in the branch, we just re-insert it as-is. - else { - SparseNode::new_branch(state_mask) - } + store_in_db_trie + } else { + false + }; + *store_in_db_trie = Some(store_in_db_trie_value); + + ( + rlp_node, + SparseNodeType::Branch { store_in_db_trie: Some(store_in_db_trie_value) }, + ) } }; - child = RemovedSparseNode { - path: removed_path.clone(), - node: new_node.clone(), - unset_branch_nibble: None, - }; - trace!(target: "trie::sparse", ?removed_path, ?new_node, "Re-inserting the node"); - self.nodes.insert(removed_path, new_node); + trace!( + target: "trie::sparse", + ?_starting_path, + ?level, + ?path, + ?node, + ?node_type, + ?is_in_prefix_set, + "Added node to rlp node stack" + ); + + buffers.rlp_node_stack.push(RlpNodeStackItem { path, rlp_node, node_type }); } - Ok(()) + debug_assert_eq!(buffers.rlp_node_stack.len(), 1); + buffers.rlp_node_stack.pop().unwrap().rlp_node } } /// Enum representing sparse trie node type. #[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum SparseNodeType { +pub enum SparseNodeType { /// Empty trie node. Empty, /// A placeholder that stores only the hash for a node that has not been fully revealed. @@ -1825,15 +1810,18 @@ enum SparseNodeType { } impl SparseNodeType { - const fn is_hash(&self) -> bool { + /// Returns true if the node is a hash node. + pub const fn is_hash(&self) -> bool { matches!(self, Self::Hash) } - const fn is_branch(&self) -> bool { + /// Returns true if the node is a branch node. + pub const fn is_branch(&self) -> bool { matches!(self, Self::Branch { .. }) } - const fn store_in_db_trie(&self) -> Option { + /// Returns true if the node should be stored in the database. + pub const fn store_in_db_trie(&self) -> Option { match *self { Self::Extension { store_in_db_trie } | Self::Branch { store_in_db_trie } => { store_in_db_trie @@ -1929,6 +1917,32 @@ impl SparseNode { pub const fn is_hash(&self) -> bool { matches!(self, Self::Hash(_)) } + + /// Returns the hash of the node if it exists. + pub const fn hash(&self) -> Option { + match self { + Self::Empty => None, + Self::Hash(hash) => Some(*hash), + Self::Leaf { hash, .. } | Self::Extension { hash, .. } | Self::Branch { hash, .. } => { + *hash + } + } + } + + /// Sets the hash of the node for testing purposes. + /// + /// For [`SparseNode::Empty`] and [`SparseNode::Hash`] nodes, this method does nothing. + #[cfg(any(test, feature = "test-utils"))] + pub const fn set_hash(&mut self, new_hash: Option) { + match self { + Self::Empty | Self::Hash(_) => { + // Cannot set hash for Empty or Hash nodes + } + Self::Leaf { hash, .. } | Self::Extension { hash, .. } | Self::Branch { hash, .. } => { + *hash = new_hash; + } + } + } } /// A helper struct used to store information about a node that has been removed @@ -1950,7 +1964,7 @@ struct RemovedSparseNode { unset_branch_nibble: Option, } -/// Collection of reusable buffers for [`RevealedSparseTrie::rlp_node`] calculations. +/// Collection of reusable buffers for [`SerialSparseTrie::rlp_node`] calculations. /// /// These buffers reduce allocations when computing RLP representations during trie updates. #[derive(Debug, Default)] @@ -1982,36 +1996,25 @@ impl RlpNodeBuffers { } /// RLP node path stack item. -#[derive(Debug)] -struct RlpNodePathStackItem { +#[derive(Clone, PartialEq, Eq, Debug)] +pub struct RlpNodePathStackItem { /// Level at which the node is located. Higher numbers correspond to lower levels in the trie. - level: usize, + pub level: usize, /// Path to the node. - path: Nibbles, + pub path: Nibbles, /// Whether the path is in the prefix set. If [`None`], then unknown yet. - is_in_prefix_set: Option, + pub is_in_prefix_set: Option, } /// RLP node stack item. -#[derive(Debug)] -struct RlpNodeStackItem { +#[derive(Clone, PartialEq, Eq, Debug)] +pub struct RlpNodeStackItem { /// Path to the node. - path: Nibbles, + pub path: Nibbles, /// RLP node. - rlp_node: RlpNode, + pub rlp_node: RlpNode, /// Type of the node. - node_type: SparseNodeType, -} - -/// Tracks modifications to the sparse trie structure. -/// -/// Maintains references to both modified and pruned/removed branches, enabling -/// one to make batch updates to a persistent database. -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct SparseTrieUpdates { - pub(crate) updated_nodes: HashMap, - pub(crate) removed_nodes: HashSet, - pub(crate) wiped: bool, + pub node_type: SparseNodeType, } impl SparseTrieUpdates { @@ -2019,14 +2022,28 @@ impl SparseTrieUpdates { pub fn wiped() -> Self { Self { wiped: true, ..Default::default() } } + + /// Clears the updates, but keeps the backing data structures allocated. + /// + /// Sets `wiped` to `false`. + pub fn clear(&mut self) { + self.updated_nodes.clear(); + self.removed_nodes.clear(); + self.wiped = false; + } + + /// Extends the updates with another set of updates. + pub fn extend(&mut self, other: Self) { + self.updated_nodes.extend(other.updated_nodes); + self.removed_nodes.extend(other.removed_nodes); + self.wiped |= other.wiped; + } } #[cfg(test)] mod find_leaf_tests { use super::*; - use crate::blinded::DefaultBlindedProvider; - use alloy_primitives::map::foldhash::fast::RandomState; - // Assuming this exists + use crate::provider::DefaultTrieNodeProvider; use alloy_rlp::Encodable; use assert_matches::assert_matches; use reth_primitives_traits::Account; @@ -2047,11 +2064,12 @@ mod find_leaf_tests { #[test] fn find_leaf_existing_leaf() { // Create a simple trie with one leaf - let mut sparse = RevealedSparseTrie::default(); + let provider = DefaultTrieNodeProvider; + let mut sparse = SerialSparseTrie::default(); let path = Nibbles::from_nibbles([0x1, 0x2, 0x3]); let value = b"test_value".to_vec(); - sparse.update_leaf(path.clone(), value.clone()).unwrap(); + sparse.update_leaf(path, value.clone(), &provider).unwrap(); // Check that the leaf exists let result = sparse.find_leaf(&path, None); @@ -2065,12 +2083,13 @@ mod find_leaf_tests { #[test] fn find_leaf_value_mismatch() { // Create a simple trie with one leaf - let mut sparse = RevealedSparseTrie::default(); + let provider = DefaultTrieNodeProvider; + let mut sparse = SerialSparseTrie::default(); let path = Nibbles::from_nibbles([0x1, 0x2, 0x3]); let value = b"test_value".to_vec(); let wrong_value = b"wrong_value".to_vec(); - sparse.update_leaf(path.clone(), value).unwrap(); + sparse.update_leaf(path, value, &provider).unwrap(); // Check with wrong expected value let result = sparse.find_leaf(&path, Some(&wrong_value)); @@ -2083,33 +2102,29 @@ mod find_leaf_tests { #[test] fn find_leaf_not_found_empty_trie() { // Empty trie - let sparse = RevealedSparseTrie::default(); + let sparse = SerialSparseTrie::default(); let path = Nibbles::from_nibbles([0x1, 0x2, 0x3]); // Leaf should not exist let result = sparse.find_leaf(&path, None); - assert_matches!( - result, - Ok(LeafLookup::NonExistent { diverged_at }) if diverged_at == Nibbles::default() - ); + assert_matches!(result, Ok(LeafLookup::NonExistent)); } #[test] fn find_leaf_empty_trie() { - let sparse = RevealedSparseTrie::::default(); + let sparse = SerialSparseTrie::default(); let path = Nibbles::from_nibbles_unchecked([0x1, 0x2, 0x3, 0x4]); let result = sparse.find_leaf(&path, None); - - // In an empty trie, the search diverges immediately at the root. - assert_matches!(result, Ok(LeafLookup::NonExistent { diverged_at }) if diverged_at == Nibbles::default()); + assert_matches!(result, Ok(LeafLookup::NonExistent)); } #[test] fn find_leaf_exists_no_value_check() { - let mut sparse = RevealedSparseTrie::::default(); + let provider = DefaultTrieNodeProvider; + let mut sparse = SerialSparseTrie::default(); let path = Nibbles::from_nibbles_unchecked([0x1, 0x2, 0x3, 0x4]); - sparse.update_leaf(path.clone(), VALUE_A()).unwrap(); + sparse.update_leaf(path, VALUE_A(), &provider).unwrap(); let result = sparse.find_leaf(&path, None); assert_matches!(result, Ok(LeafLookup::Exists)); @@ -2117,10 +2132,11 @@ mod find_leaf_tests { #[test] fn find_leaf_exists_with_value_check_ok() { - let mut sparse = RevealedSparseTrie::::default(); + let provider = DefaultTrieNodeProvider; + let mut sparse = SerialSparseTrie::default(); let path = Nibbles::from_nibbles_unchecked([0x1, 0x2, 0x3, 0x4]); let value = VALUE_A(); - sparse.update_leaf(path.clone(), value.clone()).unwrap(); + sparse.update_leaf(path, value.clone(), &provider).unwrap(); let result = sparse.find_leaf(&path, Some(&value)); assert_matches!(result, Ok(LeafLookup::Exists)); @@ -2128,80 +2144,69 @@ mod find_leaf_tests { #[test] fn find_leaf_exclusion_branch_divergence() { - let mut sparse = RevealedSparseTrie::::default(); + let provider = DefaultTrieNodeProvider; + let mut sparse = SerialSparseTrie::default(); let path1 = Nibbles::from_nibbles_unchecked([0x1, 0x2, 0x3, 0x4]); // Creates branch at 0x12 let path2 = Nibbles::from_nibbles_unchecked([0x1, 0x2, 0x5, 0x6]); // Belongs to same branch let search_path = Nibbles::from_nibbles_unchecked([0x1, 0x2, 0x7, 0x8]); // Diverges at nibble 7 - sparse.update_leaf(path1, VALUE_A()).unwrap(); - sparse.update_leaf(path2, VALUE_B()).unwrap(); + sparse.update_leaf(path1, VALUE_A(), &provider).unwrap(); + sparse.update_leaf(path2, VALUE_B(), &provider).unwrap(); let result = sparse.find_leaf(&search_path, None); - - // Diverged at the branch node because nibble '7' is not present. - let expected_divergence = Nibbles::from_nibbles_unchecked([0x1, 0x2]); - assert_matches!(result, Ok(LeafLookup::NonExistent { diverged_at }) if diverged_at == expected_divergence); + assert_matches!(result, Ok(LeafLookup::NonExistent)); } #[test] fn find_leaf_exclusion_extension_divergence() { - let mut sparse = RevealedSparseTrie::::default(); + let provider = DefaultTrieNodeProvider; + let mut sparse = SerialSparseTrie::default(); // This will create an extension node at root with key 0x12 let path1 = Nibbles::from_nibbles_unchecked([0x1, 0x2, 0x3, 0x4, 0x5, 0x6]); // This path diverges from the extension key let search_path = Nibbles::from_nibbles_unchecked([0x1, 0x2, 0x7, 0x8]); - sparse.update_leaf(path1, VALUE_A()).unwrap(); + sparse.update_leaf(path1, VALUE_A(), &provider).unwrap(); let result = sparse.find_leaf(&search_path, None); - - // Diverged where the extension node started because the path doesn't match its key prefix. - let expected_divergence = Nibbles::default(); - assert_matches!(result, Ok(LeafLookup::NonExistent { diverged_at }) if diverged_at == expected_divergence); + assert_matches!(result, Ok(LeafLookup::NonExistent)); } #[test] fn find_leaf_exclusion_leaf_divergence() { - let mut sparse = RevealedSparseTrie::::default(); + let provider = DefaultTrieNodeProvider; + let mut sparse = SerialSparseTrie::default(); let existing_leaf_path = Nibbles::from_nibbles_unchecked([0x1, 0x2, 0x3, 0x4]); let search_path = Nibbles::from_nibbles_unchecked([0x1, 0x2, 0x3, 0x4, 0x5, 0x6]); - sparse.update_leaf(existing_leaf_path, VALUE_A()).unwrap(); + sparse.update_leaf(existing_leaf_path, VALUE_A(), &provider).unwrap(); let result = sparse.find_leaf(&search_path, None); - - // Diverged when it hit the leaf node at the root, because the search path is longer - // than the leaf's key stored there. The code returns the path of the node (root) - // where the divergence occurred. - let expected_divergence = Nibbles::default(); - assert_matches!(result, Ok(LeafLookup::NonExistent { diverged_at }) if diverged_at == expected_divergence); + assert_matches!(result, Ok(LeafLookup::NonExistent)); } #[test] fn find_leaf_exclusion_path_ends_at_branch() { - let mut sparse = RevealedSparseTrie::::default(); + let provider = DefaultTrieNodeProvider; + let mut sparse = SerialSparseTrie::default(); let path1 = Nibbles::from_nibbles_unchecked([0x1, 0x2, 0x3, 0x4]); // Creates branch at 0x12 let path2 = Nibbles::from_nibbles_unchecked([0x1, 0x2, 0x5, 0x6]); let search_path = Nibbles::from_nibbles_unchecked([0x1, 0x2]); // Path of the branch itself - sparse.update_leaf(path1, VALUE_A()).unwrap(); - sparse.update_leaf(path2, VALUE_B()).unwrap(); + sparse.update_leaf(path1, VALUE_A(), &provider).unwrap(); + sparse.update_leaf(path2, VALUE_B(), &provider).unwrap(); let result = sparse.find_leaf(&search_path, None); - - // The path ends, but the node at the path is a branch, not a leaf. - // Diverged at the parent of the node found at the search path. - let expected_divergence = Nibbles::from_nibbles_unchecked([0x1]); - assert_matches!(result, Ok(LeafLookup::NonExistent { diverged_at }) if diverged_at == expected_divergence); + assert_matches!(result, Ok(LeafLookup::NonExistent)); } #[test] - fn find_leaf_error_blinded_node_at_leaf_path() { + fn find_leaf_error_trie_node_at_leaf_path() { // Scenario: The node *at* the leaf path is blinded. let blinded_hash = B256::repeat_byte(0xBB); let leaf_path = Nibbles::from_nibbles_unchecked([0x1, 0x2, 0x3, 0x4]); - let mut nodes = alloy_primitives::map::HashMap::with_hasher(RandomState::default()); + let mut nodes = alloy_primitives::map::HashMap::default(); // Create path to the blinded node nodes.insert( Nibbles::default(), @@ -2215,10 +2220,9 @@ mod find_leaf_tests { Nibbles::from_nibbles_unchecked([0x1, 0x2, 0x3]), SparseNode::new_branch(TrieMask::new(0b10000)), ); // Branch at 0x123, child 4 - nodes.insert(leaf_path.clone(), SparseNode::Hash(blinded_hash)); // Blinded node at 0x1234 + nodes.insert(leaf_path, SparseNode::Hash(blinded_hash)); // Blinded node at 0x1234 - let sparse = RevealedSparseTrie { - provider: DefaultBlindedProvider, + let sparse = SerialSparseTrie { nodes, branch_node_tree_masks: Default::default(), branch_node_hash_masks: Default::default(), @@ -2238,19 +2242,19 @@ mod find_leaf_tests { } #[test] - fn find_leaf_error_blinded_node() { + fn find_leaf_error_trie_node() { let blinded_hash = B256::repeat_byte(0xAA); let path_to_blind = Nibbles::from_nibbles_unchecked([0x1]); let search_path = Nibbles::from_nibbles_unchecked([0x1, 0x2, 0x3, 0x4]); - let mut nodes = HashMap::with_hasher(RandomState::default()); + let mut nodes = HashMap::default(); // Root is a branch with child 0x1 (blinded) and 0x5 (revealed leaf) // So we set Bit 1 and Bit 5 in the state_mask let state_mask = TrieMask::new(0b100010); nodes.insert(Nibbles::default(), SparseNode::new_branch(state_mask)); - nodes.insert(path_to_blind.clone(), SparseNode::Hash(blinded_hash)); + nodes.insert(path_to_blind, SparseNode::Hash(blinded_hash)); let path_revealed = Nibbles::from_nibbles_unchecked([0x5]); let path_revealed_leaf = Nibbles::from_nibbles_unchecked([0x5, 0x6, 0x7, 0x8]); nodes.insert( @@ -2258,11 +2262,10 @@ mod find_leaf_tests { SparseNode::new_leaf(Nibbles::from_nibbles_unchecked([0x6, 0x7, 0x8])), ); - let mut values = HashMap::with_hasher(RandomState::default()); + let mut values = HashMap::default(); values.insert(path_revealed_leaf, VALUE_A()); - let sparse = RevealedSparseTrie { - provider: DefaultBlindedProvider, + let sparse = SerialSparseTrie { nodes, branch_node_tree_masks: Default::default(), branch_node_hash_masks: Default::default(), @@ -2281,7 +2284,7 @@ mod find_leaf_tests { } #[test] - fn find_leaf_error_blinded_node_via_reveal() { + fn find_leaf_error_trie_node_via_reveal() { let blinded_hash = B256::repeat_byte(0xAA); let path_to_blind = Nibbles::from_nibbles_unchecked([0x1]); // Path of the blinded node itself let search_path = Nibbles::from_nibbles_unchecked([0x1, 0x2, 0x3, 0x4]); // Path we will search for @@ -2294,7 +2297,7 @@ mod find_leaf_tests { // 1. Construct the RLP representation of the children for the root branch let rlp_node_child1 = RlpNode::word_rlp(&blinded_hash); // Blinded node - let leaf_node_child5 = LeafNode::new(revealed_leaf_suffix.clone(), revealed_value.clone()); + let leaf_node_child5 = LeafNode::new(revealed_leaf_suffix, revealed_value.clone()); let leaf_node_child5_rlp_buf = alloy_rlp::encode(&leaf_node_child5); let hash_of_child5 = keccak256(&leaf_node_child5_rlp_buf); let rlp_node_child5 = RlpNode::word_rlp(&hash_of_child5); @@ -2309,7 +2312,7 @@ mod find_leaf_tests { // 3. Initialize the sparse trie using from_root // This will internally create Hash nodes for paths "1" and "5" initially. - let mut sparse = RevealedSparseTrie::from_root(root_trie_node, TrieMasks::none(), false) + let mut sparse = SerialSparseTrie::from_root(root_trie_node, TrieMasks::none(), false) .expect("Failed to create trie from root"); // Assertions before we reveal child5 @@ -2320,11 +2323,7 @@ mod find_leaf_tests { // 4. Explicitly reveal the leaf node for child 5 sparse - .reveal_node( - revealed_leaf_prefix.clone(), - TrieNode::Leaf(leaf_node_child5), - TrieMasks::none(), - ) + .reveal_node(revealed_leaf_prefix, TrieNode::Leaf(leaf_node_child5), TrieMasks::none()) .expect("Failed to reveal leaf node"); // Assertions after we reveal child 5 @@ -2346,6 +2345,7 @@ mod find_leaf_tests { #[cfg(test)] mod tests { use super::*; + use crate::provider::DefaultTrieNodeProvider; use alloy_primitives::{map::B256Set, U256}; use alloy_rlp::Encodable; use assert_matches::assert_matches; @@ -2374,13 +2374,16 @@ mod tests { fn pad_nibbles_left(nibbles: Nibbles) -> Nibbles { let mut base = Nibbles::from_nibbles_unchecked(vec![0; B256::len_bytes() * 2 - nibbles.len()]); - base.extend_from_slice_unchecked(&nibbles); + base.extend(&nibbles); base } /// Pad nibbles to the length of a B256 hash with zeros on the right. fn pad_nibbles_right(mut nibbles: Nibbles) -> Nibbles { - nibbles.extend_from_slice_unchecked(&vec![0; B256::len_bytes() * 2 - nibbles.len()]); + nibbles.extend(&Nibbles::from_nibbles_unchecked(vec![ + 0; + B256::len_bytes() * 2 - nibbles.len() + ])); nibbles } @@ -2404,8 +2407,8 @@ mod tests { let mut prefix_set = PrefixSetMut::default(); prefix_set.extend_keys(state.clone().into_iter().map(|(nibbles, _)| nibbles)); prefix_set.extend_keys(destroyed_accounts.iter().map(Nibbles::unpack)); - let walker = - TrieWalker::state_trie(trie_cursor, prefix_set.freeze()).with_deletions_retained(true); + let walker = TrieWalker::<_>::state_trie(trie_cursor, prefix_set.freeze()) + .with_deletions_retained(true); let hashed_post_state = HashedPostState::default() .with_accounts(state.into_iter().map(|(nibbles, account)| { (nibbles.pack().into_inner().unwrap().into(), Some(account)) @@ -2440,14 +2443,14 @@ mod tests { .clone() .unwrap_or_default() .iter() - .map(|(path, node)| (path.clone(), node.hash_mask)) + .map(|(path, node)| (*path, node.hash_mask)) .collect(); let branch_node_tree_masks = hash_builder .updated_branch_nodes .clone() .unwrap_or_default() .iter() - .map(|(path, node)| (path.clone(), node.tree_mask)) + .map(|(path, node)| (*path, node.tree_mask)) .collect(); let mut trie_updates = TrieUpdates::default(); @@ -2458,10 +2461,7 @@ mod tests { } /// Assert that the sparse trie nodes and the proof nodes from the hash builder are equal. - fn assert_eq_sparse_trie_proof_nodes( - sparse_trie: &RevealedSparseTrie, - proof_nodes: ProofNodes, - ) { + fn assert_eq_sparse_trie_proof_nodes(sparse_trie: &SerialSparseTrie, proof_nodes: ProofNodes) { let proof_nodes = proof_nodes .into_nodes_sorted() .into_iter() @@ -2505,8 +2505,8 @@ mod tests { #[test] fn sparse_trie_is_blind() { - assert!(SparseTrie::blind().is_blind()); - assert!(!SparseTrie::revealed_empty().is_blind()); + assert!(SparseTrie::::blind().is_blind()); + assert!(!SparseTrie::::revealed_empty().is_blind()); } #[test] @@ -2521,14 +2521,15 @@ mod tests { let (hash_builder_root, hash_builder_updates, hash_builder_proof_nodes, _, _) = run_hash_builder( - [(key.clone(), value())], + [(key, value())], NoopAccountTrieCursor::default(), Default::default(), - [key.clone()], + [key], ); - let mut sparse = RevealedSparseTrie::default().with_updates(true); - sparse.update_leaf(key, value_encoded()).unwrap(); + let provider = DefaultTrieNodeProvider; + let mut sparse = SerialSparseTrie::default().with_updates(true); + sparse.update_leaf(key, value_encoded(), &provider).unwrap(); let sparse_root = sparse.root(); let sparse_updates = sparse.take_updates(); @@ -2551,15 +2552,16 @@ mod tests { let (hash_builder_root, hash_builder_updates, hash_builder_proof_nodes, _, _) = run_hash_builder( - paths.iter().cloned().zip(std::iter::repeat_with(value)), + paths.iter().copied().zip(std::iter::repeat_with(value)), NoopAccountTrieCursor::default(), Default::default(), paths.clone(), ); - let mut sparse = RevealedSparseTrie::default().with_updates(true); + let provider = DefaultTrieNodeProvider; + let mut sparse = SerialSparseTrie::default().with_updates(true); for path in &paths { - sparse.update_leaf(path.clone(), value_encoded()).unwrap(); + sparse.update_leaf(*path, value_encoded(), &provider).unwrap(); } let sparse_root = sparse.root(); let sparse_updates = sparse.take_updates(); @@ -2581,15 +2583,16 @@ mod tests { let (hash_builder_root, hash_builder_updates, hash_builder_proof_nodes, _, _) = run_hash_builder( - paths.iter().cloned().zip(std::iter::repeat_with(value)), + paths.iter().copied().zip(std::iter::repeat_with(value)), NoopAccountTrieCursor::default(), Default::default(), paths.clone(), ); - let mut sparse = RevealedSparseTrie::default().with_updates(true); + let provider = DefaultTrieNodeProvider; + let mut sparse = SerialSparseTrie::default().with_updates(true); for path in &paths { - sparse.update_leaf(path.clone(), value_encoded()).unwrap(); + sparse.update_leaf(*path, value_encoded(), &provider).unwrap(); } let sparse_root = sparse.root(); let sparse_updates = sparse.take_updates(); @@ -2619,15 +2622,16 @@ mod tests { let (hash_builder_root, hash_builder_updates, hash_builder_proof_nodes, _, _) = run_hash_builder( - paths.iter().sorted_unstable().cloned().zip(std::iter::repeat_with(value)), + paths.iter().sorted_unstable().copied().zip(std::iter::repeat_with(value)), NoopAccountTrieCursor::default(), Default::default(), paths.clone(), ); - let mut sparse = RevealedSparseTrie::default().with_updates(true); + let provider = DefaultTrieNodeProvider; + let mut sparse = SerialSparseTrie::default().with_updates(true); for path in &paths { - sparse.update_leaf(path.clone(), value_encoded()).unwrap(); + sparse.update_leaf(*path, value_encoded(), &provider).unwrap(); } let sparse_root = sparse.root(); let sparse_updates = sparse.take_updates(); @@ -2658,15 +2662,16 @@ mod tests { let (hash_builder_root, hash_builder_updates, hash_builder_proof_nodes, _, _) = run_hash_builder( - paths.iter().cloned().zip(std::iter::repeat_with(|| old_value)), + paths.iter().copied().zip(std::iter::repeat_with(|| old_value)), NoopAccountTrieCursor::default(), Default::default(), paths.clone(), ); - let mut sparse = RevealedSparseTrie::default().with_updates(true); + let provider = DefaultTrieNodeProvider; + let mut sparse = SerialSparseTrie::default().with_updates(true); for path in &paths { - sparse.update_leaf(path.clone(), old_value_encoded.clone()).unwrap(); + sparse.update_leaf(*path, old_value_encoded.clone(), &provider).unwrap(); } let sparse_root = sparse.root(); let sparse_updates = sparse.updates_ref(); @@ -2677,14 +2682,14 @@ mod tests { let (hash_builder_root, hash_builder_updates, hash_builder_proof_nodes, _, _) = run_hash_builder( - paths.iter().cloned().zip(std::iter::repeat_with(|| new_value)), + paths.iter().copied().zip(std::iter::repeat_with(|| new_value)), NoopAccountTrieCursor::default(), Default::default(), paths.clone(), ); for path in &paths { - sparse.update_leaf(path.clone(), new_value_encoded.clone()).unwrap(); + sparse.update_leaf(*path, new_value_encoded.clone(), &provider).unwrap(); } let sparse_root = sparse.root(); let sparse_updates = sparse.take_updates(); @@ -2698,26 +2703,29 @@ mod tests { fn sparse_trie_remove_leaf() { reth_tracing::init_test_tracing(); - let mut sparse = RevealedSparseTrie::default(); + let provider = DefaultTrieNodeProvider; + let mut sparse = SerialSparseTrie::default(); let value = alloy_rlp::encode_fixed_size(&U256::ZERO).to_vec(); sparse - .update_leaf(Nibbles::from_nibbles([0x5, 0x0, 0x2, 0x3, 0x1]), value.clone()) + .update_leaf(Nibbles::from_nibbles([0x5, 0x0, 0x2, 0x3, 0x1]), value.clone(), &provider) + .unwrap(); + sparse + .update_leaf(Nibbles::from_nibbles([0x5, 0x0, 0x2, 0x3, 0x3]), value.clone(), &provider) .unwrap(); sparse - .update_leaf(Nibbles::from_nibbles([0x5, 0x0, 0x2, 0x3, 0x3]), value.clone()) + .update_leaf(Nibbles::from_nibbles([0x5, 0x2, 0x0, 0x1, 0x3]), value.clone(), &provider) .unwrap(); sparse - .update_leaf(Nibbles::from_nibbles([0x5, 0x2, 0x0, 0x1, 0x3]), value.clone()) + .update_leaf(Nibbles::from_nibbles([0x5, 0x3, 0x1, 0x0, 0x2]), value.clone(), &provider) .unwrap(); sparse - .update_leaf(Nibbles::from_nibbles([0x5, 0x3, 0x1, 0x0, 0x2]), value.clone()) + .update_leaf(Nibbles::from_nibbles([0x5, 0x3, 0x3, 0x0, 0x2]), value.clone(), &provider) .unwrap(); sparse - .update_leaf(Nibbles::from_nibbles([0x5, 0x3, 0x3, 0x0, 0x2]), value.clone()) + .update_leaf(Nibbles::from_nibbles([0x5, 0x3, 0x3, 0x2, 0x0]), value, &provider) .unwrap(); - sparse.update_leaf(Nibbles::from_nibbles([0x5, 0x3, 0x3, 0x2, 0x0]), value).unwrap(); // Extension (Key = 5) // └── Branch (Mask = 1011) @@ -2773,7 +2781,7 @@ mod tests { ]) ); - sparse.remove_leaf(&Nibbles::from_nibbles([0x5, 0x2, 0x0, 0x1, 0x3])).unwrap(); + sparse.remove_leaf(&Nibbles::from_nibbles([0x5, 0x2, 0x0, 0x1, 0x3]), &provider).unwrap(); // Extension (Key = 5) // └── Branch (Mask = 1001) @@ -2824,7 +2832,7 @@ mod tests { ]) ); - sparse.remove_leaf(&Nibbles::from_nibbles([0x5, 0x0, 0x2, 0x3, 0x1])).unwrap(); + sparse.remove_leaf(&Nibbles::from_nibbles([0x5, 0x0, 0x2, 0x3, 0x1]), &provider).unwrap(); // Extension (Key = 5) // └── Branch (Mask = 1001) @@ -2860,7 +2868,7 @@ mod tests { ]) ); - sparse.remove_leaf(&Nibbles::from_nibbles([0x5, 0x3, 0x1, 0x0, 0x2])).unwrap(); + sparse.remove_leaf(&Nibbles::from_nibbles([0x5, 0x3, 0x1, 0x0, 0x2]), &provider).unwrap(); // Extension (Key = 5) // └── Branch (Mask = 1001) @@ -2893,7 +2901,7 @@ mod tests { ]) ); - sparse.remove_leaf(&Nibbles::from_nibbles([0x5, 0x3, 0x3, 0x2, 0x0])).unwrap(); + sparse.remove_leaf(&Nibbles::from_nibbles([0x5, 0x3, 0x3, 0x2, 0x0]), &provider).unwrap(); // Extension (Key = 5) // └── Branch (Mask = 1001) @@ -2915,7 +2923,7 @@ mod tests { ]) ); - sparse.remove_leaf(&Nibbles::from_nibbles([0x5, 0x0, 0x2, 0x3, 0x3])).unwrap(); + sparse.remove_leaf(&Nibbles::from_nibbles([0x5, 0x0, 0x2, 0x3, 0x3]), &provider).unwrap(); // Leaf (Key = 53302) pretty_assertions::assert_eq!( @@ -2926,7 +2934,7 @@ mod tests { ),]) ); - sparse.remove_leaf(&Nibbles::from_nibbles([0x5, 0x3, 0x3, 0x0, 0x2])).unwrap(); + sparse.remove_leaf(&Nibbles::from_nibbles([0x5, 0x3, 0x3, 0x0, 0x2]), &provider).unwrap(); // Empty pretty_assertions::assert_eq!( @@ -2949,7 +2957,8 @@ mod tests { TrieMask::new(0b11), )); - let mut sparse = RevealedSparseTrie::from_root( + let provider = DefaultTrieNodeProvider; + let mut sparse = SerialSparseTrie::from_root( branch.clone(), TrieMasks { hash_mask: Some(TrieMask::new(0b01)), tree_mask: None }, false, @@ -2974,7 +2983,7 @@ mod tests { // Removing a blinded leaf should result in an error assert_matches!( - sparse.remove_leaf(&Nibbles::from_nibbles([0x0])).map_err(|e| e.into_kind()), + sparse.remove_leaf(&Nibbles::from_nibbles([0x0]), &provider).map_err(|e| e.into_kind()), Err(SparseTrieErrorKind::BlindedNode { path, hash }) if path == Nibbles::from_nibbles([0x0]) && hash == B256::repeat_byte(1) ); } @@ -2993,7 +3002,8 @@ mod tests { TrieMask::new(0b11), )); - let mut sparse = RevealedSparseTrie::from_root( + let provider = DefaultTrieNodeProvider; + let mut sparse = SerialSparseTrie::from_root( branch.clone(), TrieMasks { hash_mask: Some(TrieMask::new(0b01)), tree_mask: None }, false, @@ -3018,7 +3028,7 @@ mod tests { // Removing a non-existent leaf should be a noop let sparse_old = sparse.clone(); - assert_matches!(sparse.remove_leaf(&Nibbles::from_nibbles([0x2])), Ok(())); + assert_matches!(sparse.remove_leaf(&Nibbles::from_nibbles([0x2]), &provider), Ok(())); assert_eq!(sparse, sparse_old); } @@ -3032,8 +3042,9 @@ mod tests { fn test(updates: Vec<(BTreeMap, BTreeSet)>) { { let mut state = BTreeMap::default(); + let default_provider = DefaultTrieNodeProvider; let provider_factory = create_test_provider_factory(); - let mut sparse = RevealedSparseTrie::default().with_updates(true); + let mut sparse = SerialSparseTrie::default().with_updates(true); for (update, keys_to_delete) in updates { // Insert state updates into the sparse trie and calculate the root @@ -3041,7 +3052,7 @@ mod tests { let account = account.into_trie_account(EMPTY_ROOT_HASH); let mut account_rlp = Vec::new(); account.encode(&mut account_rlp); - sparse.update_leaf(key, account_rlp).unwrap(); + sparse.update_leaf(key, account_rlp, &default_provider).unwrap(); } // We need to clone the sparse trie, so that all updated branch nodes are // preserved, and not only those that were changed after the last call to @@ -3059,12 +3070,15 @@ mod tests { state.clone(), trie_cursor.account_trie_cursor().unwrap(), Default::default(), - state.keys().cloned().collect::>(), + state.keys().copied(), ); + // Extract account nodes before moving hash_builder_updates + let hash_builder_account_nodes = hash_builder_updates.account_nodes.clone(); + // Write trie updates to the database let provider_rw = provider_factory.provider_rw().unwrap(); - provider_rw.write_trie_updates(&hash_builder_updates).unwrap(); + provider_rw.write_trie_updates(hash_builder_updates).unwrap(); provider_rw.commit().unwrap(); // Assert that the sparse trie root matches the hash builder root @@ -3072,7 +3086,7 @@ mod tests { // Assert that the sparse trie updates match the hash builder updates pretty_assertions::assert_eq!( BTreeMap::from_iter(sparse_updates.updated_nodes), - BTreeMap::from_iter(hash_builder_updates.account_nodes) + BTreeMap::from_iter(hash_builder_account_nodes) ); // Assert that the sparse trie nodes match the hash builder proof nodes assert_eq_sparse_trie_proof_nodes(&updated_sparse, hash_builder_proof_nodes); @@ -3081,7 +3095,7 @@ mod tests { // that the sparse trie root still matches the hash builder root for key in &keys_to_delete { state.remove(key).unwrap(); - sparse.remove_leaf(key).unwrap(); + sparse.remove_leaf(key, &default_provider).unwrap(); } // We need to clone the sparse trie, so that all updated branch nodes are @@ -3101,12 +3115,15 @@ mod tests { .iter() .map(|nibbles| B256::from_slice(&nibbles.pack())) .collect(), - state.keys().cloned().collect::>(), + state.keys().copied(), ); + // Extract account nodes before moving hash_builder_updates + let hash_builder_account_nodes = hash_builder_updates.account_nodes.clone(); + // Write trie updates to the database let provider_rw = provider_factory.provider_rw().unwrap(); - provider_rw.write_trie_updates(&hash_builder_updates).unwrap(); + provider_rw.write_trie_updates(hash_builder_updates).unwrap(); provider_rw.commit().unwrap(); // Assert that the sparse trie root matches the hash builder root @@ -3114,7 +3131,7 @@ mod tests { // Assert that the sparse trie updates match the hash builder updates pretty_assertions::assert_eq!( BTreeMap::from_iter(sparse_updates.updated_nodes), - BTreeMap::from_iter(hash_builder_updates.account_nodes) + BTreeMap::from_iter(hash_builder_account_nodes) ); // Assert that the sparse trie nodes match the hash builder proof nodes assert_eq_sparse_trie_proof_nodes(&updated_sparse, hash_builder_proof_nodes); @@ -3124,20 +3141,19 @@ mod tests { fn transform_updates( updates: Vec>, - mut rng: impl rand_08::Rng, + mut rng: impl rand::Rng, ) -> Vec<(BTreeMap, BTreeSet)> { let mut keys = BTreeSet::new(); updates .into_iter() .map(|update| { - keys.extend(update.keys().cloned()); + keys.extend(update.keys().copied()); let keys_to_delete_len = update.len() / 2; let keys_to_delete = (0..keys_to_delete_len) .map(|_| { - let key = rand_08::seq::IteratorRandom::choose(keys.iter(), &mut rng) - .unwrap() - .clone(); + let key = + *rand::seq::IteratorRandom::choose(keys.iter(), &mut rng).unwrap(); keys.take(&key).unwrap() }) .collect(); @@ -3162,7 +3178,7 @@ mod tests { } /// We have three leaves that share the same prefix: 0x00, 0x01 and 0x02. Hash builder trie has - /// only nodes 0x00 and 0x01, and we have proofs for them. Node B is new and inserted in the + /// only nodes 0x00 and 0x02, and we have proofs for them. Node 0x01 is new and inserted in the /// sparse trie first. /// /// 1. Reveal the hash builder proof to leaf 0x00 in the sparse trie. @@ -3192,7 +3208,9 @@ mod tests { Default::default(), [Nibbles::default()], ); - let mut sparse = RevealedSparseTrie::from_root( + + let provider = DefaultTrieNodeProvider; + let mut sparse = SerialSparseTrie::from_root( TrieNode::decode(&mut &hash_builder_proof_nodes.nodes_sorted()[0].1[..]).unwrap(), TrieMasks { hash_mask: branch_node_hash_masks.get(&Nibbles::default()).copied(), @@ -3229,7 +3247,7 @@ mod tests { ); // Insert the leaf for the second key - sparse.update_leaf(key2(), value_encoded()).unwrap(); + sparse.update_leaf(key2(), value_encoded(), &provider).unwrap(); // Check that the branch node was updated and another nibble was set assert_eq!( @@ -3300,7 +3318,9 @@ mod tests { Default::default(), [Nibbles::default()], ); - let mut sparse = RevealedSparseTrie::from_root( + + let provider = DefaultTrieNodeProvider; + let mut sparse = SerialSparseTrie::from_root( TrieNode::decode(&mut &hash_builder_proof_nodes.nodes_sorted()[0].1[..]).unwrap(), TrieMasks { hash_mask: branch_node_hash_masks.get(&Nibbles::default()).copied(), @@ -3338,7 +3358,7 @@ mod tests { ); // Remove the leaf for the first key - sparse.remove_leaf(&key1()).unwrap(); + sparse.remove_leaf(&key1(), &provider).unwrap(); // Check that the branch node was turned into an extension node assert_eq!( @@ -3401,7 +3421,9 @@ mod tests { Default::default(), [Nibbles::default()], ); - let mut sparse = RevealedSparseTrie::from_root( + + let provider = DefaultTrieNodeProvider; + let mut sparse = SerialSparseTrie::from_root( TrieNode::decode(&mut &hash_builder_proof_nodes.nodes_sorted()[0].1[..]).unwrap(), TrieMasks { hash_mask: branch_node_hash_masks.get(&Nibbles::default()).copied(), @@ -3418,7 +3440,7 @@ mod tests { ); // Insert the leaf with a different prefix - sparse.update_leaf(key3(), value_encoded()).unwrap(); + sparse.update_leaf(key3(), value_encoded(), &provider).unwrap(); // Check that the extension node was turned into a branch node assert_matches!( @@ -3455,7 +3477,8 @@ mod tests { #[test] fn sparse_trie_get_changed_nodes_at_depth() { - let mut sparse = RevealedSparseTrie::default(); + let provider = DefaultTrieNodeProvider; + let mut sparse = SerialSparseTrie::default(); let value = alloy_rlp::encode_fixed_size(&U256::ZERO).to_vec(); @@ -3472,21 +3495,23 @@ mod tests { // ├── 0 -> Leaf (Key = 3302, Path = 53302) – Level 4 // └── 2 -> Leaf (Key = 3320, Path = 53320) – Level 4 sparse - .update_leaf(Nibbles::from_nibbles([0x5, 0x0, 0x2, 0x3, 0x1]), value.clone()) + .update_leaf(Nibbles::from_nibbles([0x5, 0x0, 0x2, 0x3, 0x1]), value.clone(), &provider) + .unwrap(); + sparse + .update_leaf(Nibbles::from_nibbles([0x5, 0x0, 0x2, 0x3, 0x3]), value.clone(), &provider) .unwrap(); sparse - .update_leaf(Nibbles::from_nibbles([0x5, 0x0, 0x2, 0x3, 0x3]), value.clone()) + .update_leaf(Nibbles::from_nibbles([0x5, 0x2, 0x0, 0x1, 0x3]), value.clone(), &provider) .unwrap(); sparse - .update_leaf(Nibbles::from_nibbles([0x5, 0x2, 0x0, 0x1, 0x3]), value.clone()) + .update_leaf(Nibbles::from_nibbles([0x5, 0x3, 0x1, 0x0, 0x2]), value.clone(), &provider) .unwrap(); sparse - .update_leaf(Nibbles::from_nibbles([0x5, 0x3, 0x1, 0x0, 0x2]), value.clone()) + .update_leaf(Nibbles::from_nibbles([0x5, 0x3, 0x3, 0x0, 0x2]), value.clone(), &provider) .unwrap(); sparse - .update_leaf(Nibbles::from_nibbles([0x5, 0x3, 0x3, 0x0, 0x2]), value.clone()) + .update_leaf(Nibbles::from_nibbles([0x5, 0x3, 0x3, 0x2, 0x0]), value, &provider) .unwrap(); - sparse.update_leaf(Nibbles::from_nibbles([0x5, 0x3, 0x3, 0x2, 0x0]), value).unwrap(); assert_eq!( sparse.get_changed_nodes_at_depth(&mut PrefixSet::default(), 0), @@ -3566,9 +3591,11 @@ mod tests { Default::default(), [Nibbles::default()], ); - let mut sparse = RevealedSparseTrie::default(); - sparse.update_leaf(key1(), value_encoded()).unwrap(); - sparse.update_leaf(key2(), value_encoded()).unwrap(); + + let provider = DefaultTrieNodeProvider; + let mut sparse = SerialSparseTrie::default(); + sparse.update_leaf(key1(), value_encoded(), &provider).unwrap(); + sparse.update_leaf(key2(), value_encoded(), &provider).unwrap(); let sparse_root = sparse.root(); let sparse_updates = sparse.take_updates(); @@ -3578,7 +3605,8 @@ mod tests { #[test] fn sparse_trie_wipe() { - let mut sparse = RevealedSparseTrie::default().with_updates(true); + let provider = DefaultTrieNodeProvider; + let mut sparse = SerialSparseTrie::default().with_updates(true); let value = alloy_rlp::encode_fixed_size(&U256::ZERO).to_vec(); @@ -3595,30 +3623,64 @@ mod tests { // ├── 0 -> Leaf (Key = 3302, Path = 53302) – Level 4 // └── 2 -> Leaf (Key = 3320, Path = 53320) – Level 4 sparse - .update_leaf(Nibbles::from_nibbles([0x5, 0x0, 0x2, 0x3, 0x1]), value.clone()) + .update_leaf(Nibbles::from_nibbles([0x5, 0x0, 0x2, 0x3, 0x1]), value.clone(), &provider) .unwrap(); sparse - .update_leaf(Nibbles::from_nibbles([0x5, 0x0, 0x2, 0x3, 0x3]), value.clone()) + .update_leaf(Nibbles::from_nibbles([0x5, 0x0, 0x2, 0x3, 0x3]), value.clone(), &provider) .unwrap(); sparse - .update_leaf(Nibbles::from_nibbles([0x5, 0x2, 0x0, 0x1, 0x3]), value.clone()) + .update_leaf(Nibbles::from_nibbles([0x5, 0x2, 0x0, 0x1, 0x3]), value.clone(), &provider) .unwrap(); sparse - .update_leaf(Nibbles::from_nibbles([0x5, 0x3, 0x1, 0x0, 0x2]), value.clone()) + .update_leaf(Nibbles::from_nibbles([0x5, 0x3, 0x1, 0x0, 0x2]), value.clone(), &provider) .unwrap(); sparse - .update_leaf(Nibbles::from_nibbles([0x5, 0x3, 0x3, 0x0, 0x2]), value.clone()) + .update_leaf(Nibbles::from_nibbles([0x5, 0x3, 0x3, 0x0, 0x2]), value.clone(), &provider) + .unwrap(); + sparse + .update_leaf(Nibbles::from_nibbles([0x5, 0x3, 0x3, 0x2, 0x0]), value, &provider) .unwrap(); - sparse.update_leaf(Nibbles::from_nibbles([0x5, 0x3, 0x3, 0x2, 0x0]), value).unwrap(); sparse.wipe(); + assert_matches!( + &sparse.updates, + Some(SparseTrieUpdates{ updated_nodes, removed_nodes, wiped }) + if updated_nodes.is_empty() && removed_nodes.is_empty() && *wiped + ); assert_eq!(sparse.root(), EMPTY_ROOT_HASH); } + #[test] + fn sparse_trie_clear() { + // tests that if we fill a sparse trie with some nodes and then clear it, it has the same + // contents as an empty sparse trie + let provider = DefaultTrieNodeProvider; + let mut sparse = SerialSparseTrie::default(); + let value = alloy_rlp::encode_fixed_size(&U256::ZERO).to_vec(); + sparse + .update_leaf(Nibbles::from_nibbles([0x5, 0x0, 0x2, 0x3, 0x1]), value.clone(), &provider) + .unwrap(); + sparse + .update_leaf(Nibbles::from_nibbles([0x5, 0x0, 0x2, 0x3, 0x3]), value.clone(), &provider) + .unwrap(); + sparse + .update_leaf(Nibbles::from_nibbles([0x5, 0x2, 0x0, 0x1, 0x3]), value.clone(), &provider) + .unwrap(); + sparse + .update_leaf(Nibbles::from_nibbles([0x5, 0x3, 0x1, 0x0, 0x2]), value, &provider) + .unwrap(); + + sparse.clear(); + + let empty_trie = SerialSparseTrie::default(); + assert_eq!(empty_trie, sparse); + } + #[test] fn sparse_trie_display() { - let mut sparse = RevealedSparseTrie::default(); + let provider = DefaultTrieNodeProvider; + let mut sparse = SerialSparseTrie::default(); let value = alloy_rlp::encode_fixed_size(&U256::ZERO).to_vec(); @@ -3635,53 +3697,55 @@ mod tests { // ├── 0 -> Leaf (Key = 3302, Path = 53302) – Level 4 // └── 2 -> Leaf (Key = 3320, Path = 53320) – Level 4 sparse - .update_leaf(Nibbles::from_nibbles([0x5, 0x0, 0x2, 0x3, 0x1]), value.clone()) + .update_leaf(Nibbles::from_nibbles([0x5, 0x0, 0x2, 0x3, 0x1]), value.clone(), &provider) + .unwrap(); + sparse + .update_leaf(Nibbles::from_nibbles([0x5, 0x0, 0x2, 0x3, 0x3]), value.clone(), &provider) .unwrap(); sparse - .update_leaf(Nibbles::from_nibbles([0x5, 0x0, 0x2, 0x3, 0x3]), value.clone()) + .update_leaf(Nibbles::from_nibbles([0x5, 0x2, 0x0, 0x1, 0x3]), value.clone(), &provider) .unwrap(); sparse - .update_leaf(Nibbles::from_nibbles([0x5, 0x2, 0x0, 0x1, 0x3]), value.clone()) + .update_leaf(Nibbles::from_nibbles([0x5, 0x3, 0x1, 0x0, 0x2]), value.clone(), &provider) .unwrap(); sparse - .update_leaf(Nibbles::from_nibbles([0x5, 0x3, 0x1, 0x0, 0x2]), value.clone()) + .update_leaf(Nibbles::from_nibbles([0x5, 0x3, 0x3, 0x0, 0x2]), value.clone(), &provider) .unwrap(); sparse - .update_leaf(Nibbles::from_nibbles([0x5, 0x3, 0x3, 0x0, 0x2]), value.clone()) + .update_leaf(Nibbles::from_nibbles([0x5, 0x3, 0x3, 0x2, 0x0]), value, &provider) .unwrap(); - sparse.update_leaf(Nibbles::from_nibbles([0x5, 0x3, 0x3, 0x2, 0x0]), value).unwrap(); let normal_printed = format!("{sparse}"); let expected = "\ -Root -> Extension { key: Nibbles(0x05), hash: None, store_in_db_trie: None } +Root -> Extension { key: Nibbles(0x5), hash: None, store_in_db_trie: None } 5 -> Branch { state_mask: TrieMask(0000000000001101), hash: None, store_in_db_trie: None } -50 -> Extension { key: Nibbles(0x0203), hash: None, store_in_db_trie: None } +50 -> Extension { key: Nibbles(0x23), hash: None, store_in_db_trie: None } 5023 -> Branch { state_mask: TrieMask(0000000000001010), hash: None, store_in_db_trie: None } 50231 -> Leaf { key: Nibbles(0x), hash: None } 50233 -> Leaf { key: Nibbles(0x), hash: None } -52013 -> Leaf { key: Nibbles(0x000103), hash: None } +52013 -> Leaf { key: Nibbles(0x013), hash: None } 53 -> Branch { state_mask: TrieMask(0000000000001010), hash: None, store_in_db_trie: None } -53102 -> Leaf { key: Nibbles(0x0002), hash: None } +53102 -> Leaf { key: Nibbles(0x02), hash: None } 533 -> Branch { state_mask: TrieMask(0000000000000101), hash: None, store_in_db_trie: None } -53302 -> Leaf { key: Nibbles(0x02), hash: None } -53320 -> Leaf { key: Nibbles(0x00), hash: None } +53302 -> Leaf { key: Nibbles(0x2), hash: None } +53320 -> Leaf { key: Nibbles(0x0), hash: None } "; assert_eq!(normal_printed, expected); let alternate_printed = format!("{sparse:#}"); let expected = "\ -Root -> Extension { key: Nibbles(0x05), hash: None, store_in_db_trie: None } +Root -> Extension { key: Nibbles(0x5), hash: None, store_in_db_trie: None } 5 -> Branch { state_mask: TrieMask(0000000000001101), hash: None, store_in_db_trie: None } - 50 -> Extension { key: Nibbles(0x0203), hash: None, store_in_db_trie: None } + 50 -> Extension { key: Nibbles(0x23), hash: None, store_in_db_trie: None } 5023 -> Branch { state_mask: TrieMask(0000000000001010), hash: None, store_in_db_trie: None } 50231 -> Leaf { key: Nibbles(0x), hash: None } 50233 -> Leaf { key: Nibbles(0x), hash: None } - 52013 -> Leaf { key: Nibbles(0x000103), hash: None } + 52013 -> Leaf { key: Nibbles(0x013), hash: None } 53 -> Branch { state_mask: TrieMask(0000000000001010), hash: None, store_in_db_trie: None } - 53102 -> Leaf { key: Nibbles(0x0002), hash: None } + 53102 -> Leaf { key: Nibbles(0x02), hash: None } 533 -> Branch { state_mask: TrieMask(0000000000000101), hash: None, store_in_db_trie: None } - 53302 -> Leaf { key: Nibbles(0x02), hash: None } - 53320 -> Leaf { key: Nibbles(0x00), hash: None } + 53302 -> Leaf { key: Nibbles(0x2), hash: None } + 53320 -> Leaf { key: Nibbles(0x0), hash: None } "; assert_eq!(alternate_printed, expected); diff --git a/crates/trie/trie/Cargo.toml b/crates/trie/trie/Cargo.toml index adee3291b80..403d187e46a 100644 --- a/crates/trie/trie/Cargo.toml +++ b/crates/trie/trie/Cargo.toml @@ -57,6 +57,7 @@ revm-state.workspace = true triehash.workspace = true # misc +assert_matches.workspace = true criterion.workspace = true parking_lot.workspace = true pretty_assertions.workspace = true diff --git a/crates/trie/trie/src/forward_cursor.rs b/crates/trie/trie/src/forward_cursor.rs index bad44fcf517..c99b0d049ee 100644 --- a/crates/trie/trie/src/forward_cursor.rs +++ b/crates/trie/trie/src/forward_cursor.rs @@ -11,7 +11,7 @@ pub struct ForwardInMemoryCursor<'a, K, V> { impl<'a, K, V> ForwardInMemoryCursor<'a, K, V> { /// Create new forward cursor positioned at the beginning of the collection. /// - /// The cursor expects all of the entries have been sorted in advance. + /// The cursor expects all of the entries to have been sorted in advance. #[inline] pub fn new(entries: &'a [(K, V)]) -> Self { Self { entries: entries.iter(), is_empty: entries.is_empty() } @@ -23,8 +23,9 @@ impl<'a, K, V> ForwardInMemoryCursor<'a, K, V> { self.is_empty } + /// Returns the current entry pointed to be the cursor, or `None` if no entries are left. #[inline] - fn peek(&self) -> Option<&(K, V)> { + pub fn current(&self) -> Option<&(K, V)> { self.entries.clone().next() } @@ -59,7 +60,7 @@ where fn advance_while(&mut self, predicate: impl Fn(&K) -> bool) -> Option<(K, V)> { let mut entry; loop { - entry = self.peek(); + entry = self.current(); if entry.is_some_and(|(k, _)| predicate(k)) { self.next(); } else { @@ -77,20 +78,21 @@ mod tests { #[test] fn test_cursor() { let mut cursor = ForwardInMemoryCursor::new(&[(1, ()), (2, ()), (3, ()), (4, ()), (5, ())]); + assert_eq!(cursor.current(), Some(&(1, ()))); assert_eq!(cursor.seek(&0), Some((1, ()))); - assert_eq!(cursor.peek(), Some(&(1, ()))); + assert_eq!(cursor.current(), Some(&(1, ()))); assert_eq!(cursor.seek(&3), Some((3, ()))); - assert_eq!(cursor.peek(), Some(&(3, ()))); + assert_eq!(cursor.current(), Some(&(3, ()))); assert_eq!(cursor.seek(&3), Some((3, ()))); - assert_eq!(cursor.peek(), Some(&(3, ()))); + assert_eq!(cursor.current(), Some(&(3, ()))); assert_eq!(cursor.seek(&4), Some((4, ()))); - assert_eq!(cursor.peek(), Some(&(4, ()))); + assert_eq!(cursor.current(), Some(&(4, ()))); assert_eq!(cursor.seek(&6), None); - assert_eq!(cursor.peek(), None); + assert_eq!(cursor.current(), None); } } diff --git a/crates/trie/trie/src/hashed_cursor/mock.rs b/crates/trie/trie/src/hashed_cursor/mock.rs index 895bf852a22..f091ae6ffe5 100644 --- a/crates/trie/trie/src/hashed_cursor/mock.rs +++ b/crates/trie/trie/src/hashed_cursor/mock.rs @@ -55,17 +55,23 @@ impl MockHashedCursorFactory { } impl HashedCursorFactory for MockHashedCursorFactory { - type AccountCursor = MockHashedCursor; - type StorageCursor = MockHashedCursor; - - fn hashed_account_cursor(&self) -> Result { + type AccountCursor<'a> + = MockHashedCursor + where + Self: 'a; + type StorageCursor<'a> + = MockHashedCursor + where + Self: 'a; + + fn hashed_account_cursor(&self) -> Result, DatabaseError> { Ok(MockHashedCursor::new(self.hashed_accounts.clone(), self.visited_account_keys.clone())) } fn hashed_storage_cursor( &self, hashed_address: B256, - ) -> Result { + ) -> Result, DatabaseError> { Ok(MockHashedCursor::new( self.hashed_storage_tries .get(&hashed_address) @@ -101,7 +107,7 @@ impl MockHashedCursor { impl HashedCursor for MockHashedCursor { type Value = T; - #[instrument(level = "trace", skip(self), ret)] + #[instrument(skip(self), ret(level = "trace"))] fn seek(&mut self, key: B256) -> Result, DatabaseError> { // Find the first key that is greater than or equal to the given key. let entry = self.values.iter().find_map(|(k, v)| (k >= &key).then(|| (*k, v.clone()))); @@ -115,7 +121,7 @@ impl HashedCursor for MockHashedCursor { Ok(entry) } - #[instrument(level = "trace", skip(self), ret)] + #[instrument(skip(self), ret(level = "trace"))] fn next(&mut self) -> Result, DatabaseError> { let mut iter = self.values.iter(); // Jump to the first key that has a prefix of the current key if it's set, or to the first diff --git a/crates/trie/trie/src/hashed_cursor/mod.rs b/crates/trie/trie/src/hashed_cursor/mod.rs index bc4bbd88c56..6c4788a3360 100644 --- a/crates/trie/trie/src/hashed_cursor/mod.rs +++ b/crates/trie/trie/src/hashed_cursor/mod.rs @@ -14,29 +14,35 @@ pub mod noop; pub mod mock; /// The factory trait for creating cursors over the hashed state. +#[auto_impl::auto_impl(&)] pub trait HashedCursorFactory { /// The hashed account cursor type. - type AccountCursor: HashedCursor; + type AccountCursor<'a>: HashedCursor + where + Self: 'a; /// The hashed storage cursor type. - type StorageCursor: HashedStorageCursor; + type StorageCursor<'a>: HashedStorageCursor + where + Self: 'a; /// Returns a cursor for iterating over all hashed accounts in the state. - fn hashed_account_cursor(&self) -> Result; + fn hashed_account_cursor(&self) -> Result, DatabaseError>; /// Returns a cursor for iterating over all hashed storage entries in the state. fn hashed_storage_cursor( &self, hashed_address: B256, - ) -> Result; + ) -> Result, DatabaseError>; } /// The cursor for iterating over hashed entries. +#[auto_impl::auto_impl(&mut)] pub trait HashedCursor { /// Value returned by the cursor. type Value: std::fmt::Debug; - /// Seek an entry greater or equal to the given key and position the cursor there. - /// Returns the first entry with the key greater or equal to the sought key. + /// Seek an entry greater than or equal to the given key and position the cursor there. + /// Returns the first entry with the key greater than or equal to the sought key. fn seek(&mut self, key: B256) -> Result, DatabaseError>; /// Move the cursor to the next entry and return it. @@ -44,6 +50,7 @@ pub trait HashedCursor { } /// The cursor for iterating over hashed storage entries. +#[auto_impl::auto_impl(&mut)] pub trait HashedStorageCursor: HashedCursor { /// Returns `true` if there are no entries for a given key. fn is_storage_empty(&mut self) -> Result; diff --git a/crates/trie/trie/src/hashed_cursor/noop.rs b/crates/trie/trie/src/hashed_cursor/noop.rs index 58b78dc245f..e5bc44f0f5c 100644 --- a/crates/trie/trie/src/hashed_cursor/noop.rs +++ b/crates/trie/trie/src/hashed_cursor/noop.rs @@ -9,17 +9,23 @@ use reth_storage_errors::db::DatabaseError; pub struct NoopHashedCursorFactory; impl HashedCursorFactory for NoopHashedCursorFactory { - type AccountCursor = NoopHashedAccountCursor; - type StorageCursor = NoopHashedStorageCursor; + type AccountCursor<'a> + = NoopHashedAccountCursor + where + Self: 'a; + type StorageCursor<'a> + = NoopHashedStorageCursor + where + Self: 'a; - fn hashed_account_cursor(&self) -> Result { + fn hashed_account_cursor(&self) -> Result, DatabaseError> { Ok(NoopHashedAccountCursor::default()) } fn hashed_storage_cursor( &self, _hashed_address: B256, - ) -> Result { + ) -> Result, DatabaseError> { Ok(NoopHashedStorageCursor::default()) } } diff --git a/crates/trie/trie/src/hashed_cursor/post_state.rs b/crates/trie/trie/src/hashed_cursor/post_state.rs index b6c8994e137..896251f3634 100644 --- a/crates/trie/trie/src/hashed_cursor/post_state.rs +++ b/crates/trie/trie/src/hashed_cursor/post_state.rs @@ -7,33 +7,46 @@ use reth_trie_common::{HashedAccountsSorted, HashedPostStateSorted, HashedStorag /// The hashed cursor factory for the post state. #[derive(Clone, Debug)] -pub struct HashedPostStateCursorFactory<'a, CF> { +pub struct HashedPostStateCursorFactory { cursor_factory: CF, - post_state: &'a HashedPostStateSorted, + post_state: T, } -impl<'a, CF> HashedPostStateCursorFactory<'a, CF> { +impl HashedPostStateCursorFactory { /// Create a new factory. - pub const fn new(cursor_factory: CF, post_state: &'a HashedPostStateSorted) -> Self { + pub const fn new(cursor_factory: CF, post_state: T) -> Self { Self { cursor_factory, post_state } } } -impl<'a, CF: HashedCursorFactory> HashedCursorFactory for HashedPostStateCursorFactory<'a, CF> { - type AccountCursor = HashedPostStateAccountCursor<'a, CF::AccountCursor>; - type StorageCursor = HashedPostStateStorageCursor<'a, CF::StorageCursor>; - - fn hashed_account_cursor(&self) -> Result { +impl<'overlay, CF, T> HashedCursorFactory for HashedPostStateCursorFactory +where + CF: HashedCursorFactory, + T: AsRef, +{ + type AccountCursor<'cursor> + = HashedPostStateAccountCursor<'overlay, CF::AccountCursor<'cursor>> + where + Self: 'cursor; + type StorageCursor<'cursor> + = HashedPostStateStorageCursor<'overlay, CF::StorageCursor<'cursor>> + where + Self: 'cursor; + + fn hashed_account_cursor(&self) -> Result, DatabaseError> { let cursor = self.cursor_factory.hashed_account_cursor()?; - Ok(HashedPostStateAccountCursor::new(cursor, &self.post_state.accounts)) + Ok(HashedPostStateAccountCursor::new(cursor, &self.post_state.as_ref().accounts)) } fn hashed_storage_cursor( &self, hashed_address: B256, - ) -> Result { + ) -> Result, DatabaseError> { let cursor = self.cursor_factory.hashed_storage_cursor(hashed_address)?; - Ok(HashedPostStateStorageCursor::new(cursor, self.post_state.storages.get(&hashed_address))) + Ok(HashedPostStateStorageCursor::new( + cursor, + self.post_state.as_ref().storages.get(&hashed_address), + )) } } diff --git a/crates/trie/trie/src/lib.rs b/crates/trie/trie/src/lib.rs index 356f3dac93f..e53049b5872 100644 --- a/crates/trie/trie/src/lib.rs +++ b/crates/trie/trie/src/lib.rs @@ -12,7 +12,7 @@ html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256", issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] /// The implementation of forward-only in-memory cursor. pub mod forward_cursor; @@ -41,7 +41,10 @@ pub use trie::{StateRoot, StorageRoot, TrieType}; /// Utilities for state root checkpoint progress. mod progress; -pub use progress::{IntermediateStateRootState, StateRootProgress}; +pub use progress::{ + IntermediateStateRootState, IntermediateStorageRootState, StateRootProgress, + StorageRootProgress, +}; /// Trie calculation stats. pub mod stats; @@ -60,3 +63,6 @@ pub mod test_utils; /// Collection of mock types for testing. #[cfg(test)] pub mod mock; + +/// Verification of existing stored trie nodes against state data. +pub mod verify; diff --git a/crates/trie/trie/src/metrics.rs b/crates/trie/trie/src/metrics.rs index 7f114219988..c88c1cde271 100644 --- a/crates/trie/trie/src/metrics.rs +++ b/crates/trie/trie/src/metrics.rs @@ -83,8 +83,6 @@ pub struct TrieNodeIterMetrics { /// iterator. It does not mean the database seek was actually done, as the trie node /// iterator caches the last hashed cursor seek. leaf_nodes_same_seeked_total: Counter, - /// The number of times the same leaf node as we just advanced to was seeked by the iterator. - leaf_nodes_same_seeked_as_advanced_total: Counter, /// The number of leaf nodes seeked by the iterator. leaf_nodes_seeked_total: Counter, /// The number of leaf nodes advanced by the iterator. @@ -109,11 +107,6 @@ impl TrieNodeIterMetrics { self.leaf_nodes_same_seeked_total.increment(1); } - /// Increment `leaf_nodes_same_seeked_as_advanced_total`. - pub fn inc_leaf_nodes_same_seeked_as_advanced(&self) { - self.leaf_nodes_same_seeked_as_advanced_total.increment(1); - } - /// Increment `leaf_nodes_seeked_total`. pub fn inc_leaf_nodes_seeked(&self) { self.leaf_nodes_seeked_total.increment(1); diff --git a/crates/trie/trie/src/node_iter.rs b/crates/trie/trie/src/node_iter.rs index 1f6a0f8b108..862176c803a 100644 --- a/crates/trie/trie/src/node_iter.rs +++ b/crates/trie/trie/src/node_iter.rs @@ -1,7 +1,10 @@ -use crate::{hashed_cursor::HashedCursor, trie_cursor::TrieCursor, walker::TrieWalker, Nibbles}; +use crate::{ + hashed_cursor::HashedCursor, trie_cursor::TrieCursor, walker::TrieWalker, Nibbles, TrieType, +}; use alloy_primitives::B256; +use alloy_trie::proof::AddedRemovedKeys; use reth_storage_errors::db::DatabaseError; -use tracing::trace; +use tracing::{instrument, trace}; /// Represents a branch node in the trie. #[derive(Debug)] @@ -41,13 +44,18 @@ struct SeekedHashedEntry { result: Option<(B256, V)>, } -/// An iterator over existing intermediate branch nodes and updated leaf nodes. +/// Iterates over trie nodes for hash building. +/// +/// This iterator depends on the ordering guarantees of [`TrieCursor`], +/// and additionally uses hashed cursor lookups when operating on storage tries. #[derive(Debug)] -pub struct TrieNodeIter { +pub struct TrieNodeIter { /// The walker over intermediate nodes. - pub walker: TrieWalker, + pub walker: TrieWalker, /// The cursor for the hashed entries. pub hashed_cursor: H, + /// The type of the trie. + trie_type: TrieType, /// The previous hashed key. If the iteration was previously interrupted, this value can be /// used to resume iterating from the last returned leaf node. previous_hashed_key: Option, @@ -64,52 +72,40 @@ pub struct TrieNodeIter { #[cfg(feature = "metrics")] metrics: crate::metrics::TrieNodeIterMetrics, - /// The key that the [`HashedCursor`] previously advanced to using [`HashedCursor::next`]. - #[cfg(feature = "metrics")] - previously_advanced_to_key: Option, + /// Stores the result of the last successful [`Self::next_hashed_entry`], used to avoid a + /// redundant [`Self::seek_hashed_entry`] call if the walker points to the same key that + /// was just returned by `next()`. + last_next_result: Option<(B256, H::Value)>, } -impl TrieNodeIter +impl TrieNodeIter where H::Value: Copy, + K: AsRef, { /// Creates a new [`TrieNodeIter`] for the state trie. - pub fn state_trie(walker: TrieWalker, hashed_cursor: H) -> Self { - Self::new( - walker, - hashed_cursor, - #[cfg(feature = "metrics")] - crate::TrieType::State, - ) + pub fn state_trie(walker: TrieWalker, hashed_cursor: H) -> Self { + Self::new(walker, hashed_cursor, TrieType::State) } /// Creates a new [`TrieNodeIter`] for the storage trie. - pub fn storage_trie(walker: TrieWalker, hashed_cursor: H) -> Self { - Self::new( - walker, - hashed_cursor, - #[cfg(feature = "metrics")] - crate::TrieType::Storage, - ) + pub fn storage_trie(walker: TrieWalker, hashed_cursor: H) -> Self { + Self::new(walker, hashed_cursor, TrieType::Storage) } /// Creates a new [`TrieNodeIter`]. - fn new( - walker: TrieWalker, - hashed_cursor: H, - #[cfg(feature = "metrics")] trie_type: crate::TrieType, - ) -> Self { + fn new(walker: TrieWalker, hashed_cursor: H, trie_type: TrieType) -> Self { Self { walker, hashed_cursor, + trie_type, previous_hashed_key: None, current_hashed_entry: None, should_check_walker_key: false, last_seeked_hashed_entry: None, #[cfg(feature = "metrics")] metrics: crate::metrics::TrieNodeIterMetrics::new(trie_type), - #[cfg(feature = "metrics")] - previously_advanced_to_key: None, + last_next_result: None, } } @@ -124,8 +120,20 @@ where /// /// If the key is the same as the last seeked key, the result of the last seek is returned. /// - /// If `metrics` feature is enabled, also updates the metrics. + /// If `metrics` feature is enabled, it also updates the metrics. fn seek_hashed_entry(&mut self, key: B256) -> Result, DatabaseError> { + if let Some((last_key, last_value)) = self.last_next_result && + last_key == key + { + trace!(target: "trie::node_iter", seek_key = ?key, "reusing result from last next() call instead of seeking"); + self.last_next_result = None; // Consume the cached value + + let result = Some((last_key, last_value)); + self.last_seeked_hashed_entry = Some(SeekedHashedEntry { seeked_key: key, result }); + + return Ok(result); + } + if let Some(entry) = self .last_seeked_hashed_entry .as_ref() @@ -137,41 +145,39 @@ where return Ok(entry); } + trace!(target: "trie::node_iter", ?key, "performing hashed cursor seek"); let result = self.hashed_cursor.seek(key)?; self.last_seeked_hashed_entry = Some(SeekedHashedEntry { seeked_key: key, result }); #[cfg(feature = "metrics")] { self.metrics.inc_leaf_nodes_seeked(); - - if Some(key) == self.previously_advanced_to_key { - self.metrics.inc_leaf_nodes_same_seeked_as_advanced(); - } } Ok(result) } /// Advances the hashed cursor to the next entry. /// - /// If `metrics` feature is enabled, also updates the metrics. + /// If `metrics` feature is enabled, it also updates the metrics. fn next_hashed_entry(&mut self) -> Result, DatabaseError> { let result = self.hashed_cursor.next(); + + self.last_next_result = result.clone()?; + #[cfg(feature = "metrics")] { self.metrics.inc_leaf_nodes_advanced(); - - self.previously_advanced_to_key = - result.as_ref().ok().and_then(|result| result.as_ref().map(|(k, _)| *k)); } result } } -impl TrieNodeIter +impl TrieNodeIter where C: TrieCursor, H: HashedCursor, H::Value: Copy, + K: AsRef, { /// Return the next trie node to be added to the hash builder. /// @@ -184,6 +190,13 @@ where /// 5. Repeat. /// /// NOTE: The iteration will start from the key of the previous hashed entry if it was supplied. + #[instrument( + level = "trace", + target = "trie::node_iter", + skip_all, + fields(trie_type = ?self.trie_type), + ret + )] pub fn try_next( &mut self, ) -> Result::Value>>, DatabaseError> { @@ -201,7 +214,7 @@ where #[cfg(feature = "metrics")] self.metrics.inc_branch_nodes_returned(); return Ok(Some(TrieElement::Branch(TrieBranchNode::new( - key.clone(), + *key, self.walker.hash().unwrap(), self.walker.children_are_in_trie(), )))) @@ -268,13 +281,13 @@ where // of this, we need to check that the current walker key has a prefix of the key // that we seeked to. if can_skip_node && - self.walker.key().is_some_and(|key| key.has_prefix(&seek_prefix)) && + self.walker.key().is_some_and(|key| key.starts_with(&seek_prefix)) && self.walker.children_are_in_trie() { trace!( target: "trie::node_iter", ?seek_key, - walker_hash = ?self.walker.hash(), + walker_hash = ?self.walker.maybe_hash(), "skipping hashed seek" ); @@ -330,7 +343,7 @@ mod tests { let mut prefix_set = PrefixSetMut::default(); prefix_set.extend_keys(state.clone().into_iter().map(|(nibbles, _)| nibbles)); - let walker = TrieWalker::state_trie(NoopAccountTrieCursor, prefix_set.freeze()); + let walker = TrieWalker::<_>::state_trie(NoopAccountTrieCursor, prefix_set.freeze()); let hashed_post_state = HashedPostState::default() .with_accounts(state.into_iter().map(|(nibbles, account)| { @@ -459,8 +472,10 @@ mod tests { prefix_set.insert(Nibbles::unpack(account_3)); let prefix_set = prefix_set.freeze(); - let walker = - TrieWalker::state_trie(trie_cursor_factory.account_trie_cursor().unwrap(), prefix_set); + let walker = TrieWalker::<_>::state_trie( + trie_cursor_factory.account_trie_cursor().unwrap(), + prefix_set, + ); let hashed_cursor_factory = MockHashedCursorFactory::new( BTreeMap::from([ @@ -493,7 +508,7 @@ mod tests { visited_key: Some(branch_node_0.0) }, KeyVisit { - visit_type: KeyVisitType::SeekNonExact(branch_node_2.0.clone()), + visit_type: KeyVisitType::SeekNonExact(branch_node_2.0), visited_key: Some(branch_node_2.0) }, KeyVisit { @@ -518,12 +533,6 @@ mod tests { // Collect the siblings of the modified account KeyVisit { visit_type: KeyVisitType::Next, visited_key: Some(account_4) }, KeyVisit { visit_type: KeyVisitType::Next, visited_key: Some(account_5) }, - // We seek the account 5 because its hash is not in the branch node, but we already - // walked it before, so there should be no need for it. - KeyVisit { - visit_type: KeyVisitType::SeekNonExact(account_5), - visited_key: Some(account_5) - }, KeyVisit { visit_type: KeyVisitType::Next, visited_key: None }, ], ); diff --git a/crates/trie/trie/src/progress.rs b/crates/trie/trie/src/progress.rs index 25195b48adb..1eab18318f2 100644 --- a/crates/trie/trie/src/progress.rs +++ b/crates/trie/trie/src/progress.rs @@ -1,34 +1,90 @@ -use crate::{hash_builder::HashBuilder, trie_cursor::CursorSubNode, updates::TrieUpdates}; +use crate::{ + hash_builder::HashBuilder, + trie_cursor::CursorSubNode, + updates::{StorageTrieUpdates, TrieUpdates}, +}; use alloy_primitives::B256; +use reth_primitives_traits::Account; use reth_stages_types::MerkleCheckpoint; /// The progress of the state root computation. #[derive(Debug)] pub enum StateRootProgress { - /// The complete state root computation with updates and computed root. + /// The complete state root computation with updates, the total number of entries walked, and + /// the computed root. Complete(B256, usize, TrieUpdates), /// The intermediate progress of state root computation. - /// Contains the walker stack, the hash builder and the trie updates. + /// Contains the walker stack, the hash builder, and the trie updates. + /// + /// Also contains any progress in an inner storage root computation. Progress(Box, usize, TrieUpdates), } /// The intermediate state of the state root computation. #[derive(Debug)] pub struct IntermediateStateRootState { - /// Previously constructed hash builder. - pub hash_builder: HashBuilder, - /// Previously recorded walker stack. - pub walker_stack: Vec, - /// The last hashed account key processed. - pub last_account_key: B256, + /// The intermediate account root state. + pub account_root_state: IntermediateRootState, + /// The intermediate storage root state with account data. + pub storage_root_state: Option, +} + +/// The intermediate state of a storage root computation along with the account. +#[derive(Debug)] +pub struct IntermediateStorageRootState { + /// The intermediate storage trie state. + pub state: IntermediateRootState, + /// The account for which the storage root is being computed. + pub account: Account, } impl From for IntermediateStateRootState { fn from(value: MerkleCheckpoint) -> Self { Self { - hash_builder: HashBuilder::from(value.state), - walker_stack: value.walker_stack.into_iter().map(CursorSubNode::from).collect(), - last_account_key: value.last_account_key, + account_root_state: IntermediateRootState { + hash_builder: HashBuilder::from(value.state), + walker_stack: value.walker_stack.into_iter().map(CursorSubNode::from).collect(), + last_hashed_key: value.last_account_key, + }, + storage_root_state: value.storage_root_checkpoint.map(|checkpoint| { + IntermediateStorageRootState { + state: IntermediateRootState { + hash_builder: HashBuilder::from(checkpoint.state), + walker_stack: checkpoint + .walker_stack + .into_iter() + .map(CursorSubNode::from) + .collect(), + last_hashed_key: checkpoint.last_storage_key, + }, + account: Account { + nonce: checkpoint.account_nonce, + balance: checkpoint.account_balance, + bytecode_hash: Some(checkpoint.account_bytecode_hash), + }, + } + }), } } } + +/// The intermediate state of a state root computation, whether account or storage root. +#[derive(Debug)] +pub struct IntermediateRootState { + /// Previously constructed hash builder. + pub hash_builder: HashBuilder, + /// Previously recorded walker stack. + pub walker_stack: Vec, + /// The last hashed key processed. + pub last_hashed_key: B256, +} + +/// The progress of a storage root calculation. +#[derive(Debug)] +pub enum StorageRootProgress { + /// The complete storage root computation with updates and computed root. + Complete(B256, usize, StorageTrieUpdates), + /// The intermediate progress of state root computation. + /// Contains the walker stack, the hash builder, and the trie updates. + Progress(Box, usize, StorageTrieUpdates), +} diff --git a/crates/trie/trie/src/proof/mod.rs b/crates/trie/trie/src/proof/mod.rs index 64a8f4d3b93..efd958e5743 100644 --- a/crates/trie/trie/src/proof/mod.rs +++ b/crates/trie/trie/src/proof/mod.rs @@ -12,13 +12,14 @@ use alloy_primitives::{ Address, B256, }; use alloy_rlp::{BufMut, Encodable}; +use alloy_trie::proof::AddedRemovedKeys; use reth_execution_errors::trie::StateProofError; use reth_trie_common::{ proof::ProofRetainer, AccountProof, MultiProof, MultiProofTargets, StorageMultiProof, }; -mod blinded; -pub use blinded::*; +mod trie_node; +pub use trie_node::*; /// A struct for generating merkle proofs. /// @@ -79,6 +80,16 @@ impl Proof { self.collect_branch_node_masks = branch_node_masks; self } + + /// Get a reference to the trie cursor factory. + pub const fn trie_cursor_factory(&self) -> &T { + &self.trie_cursor_factory + } + + /// Get a reference to the hashed cursor factory. + pub const fn hashed_cursor_factory(&self) -> &H { + &self.hashed_cursor_factory + } } impl Proof @@ -111,7 +122,7 @@ where // Create the walker. let mut prefix_set = self.prefix_sets.account_prefix_set.clone(); prefix_set.extend_keys(targets.keys().map(Nibbles::unpack)); - let walker = TrieWalker::state_trie(trie_cursor, prefix_set.freeze()); + let walker = TrieWalker::<_>::state_trie(trie_cursor, prefix_set.freeze()); // Create a hash builder to rebuild the root node since it is not available in the database. let retainer = targets.keys().map(Nibbles::unpack).collect(); @@ -120,7 +131,7 @@ where .with_updates(self.collect_branch_node_masks); // Initialize all storage multiproofs as empty. - // Storage multiproofs for non empty tries will be overwritten if necessary. + // Storage multiproofs for non-empty tries will be overwritten if necessary. let mut storages: B256Map<_> = targets.keys().map(|key| (*key, StorageMultiProof::empty())).collect(); let mut account_rlp = Vec::with_capacity(TRIE_ACCOUNT_RLP_MAX_SIZE); @@ -167,10 +178,7 @@ where let (branch_node_hash_masks, branch_node_tree_masks) = if self.collect_branch_node_masks { let updated_branch_nodes = hash_builder.updated_branch_nodes.unwrap_or_default(); ( - updated_branch_nodes - .iter() - .map(|(path, node)| (path.clone(), node.hash_mask)) - .collect(), + updated_branch_nodes.iter().map(|(path, node)| (*path, node.hash_mask)).collect(), updated_branch_nodes .into_iter() .map(|(path, node)| (path, node.tree_mask)) @@ -186,7 +194,7 @@ where /// Generates storage merkle proofs. #[derive(Debug)] -pub struct StorageProof { +pub struct StorageProof { /// The factory for traversing trie nodes. trie_cursor_factory: T, /// The factory for hashed cursors. @@ -197,6 +205,8 @@ pub struct StorageProof { prefix_set: PrefixSetMut, /// Flag indicating whether to include branch node masks in the proof. collect_branch_node_masks: bool, + /// Provided by the user to give the necessary context to retain extra proofs. + added_removed_keys: Option, } impl StorageProof { @@ -213,28 +223,36 @@ impl StorageProof { hashed_address, prefix_set: PrefixSetMut::default(), collect_branch_node_masks: false, + added_removed_keys: None, } } +} +impl StorageProof { /// Set the trie cursor factory. - pub fn with_trie_cursor_factory(self, trie_cursor_factory: TF) -> StorageProof { + pub fn with_trie_cursor_factory(self, trie_cursor_factory: TF) -> StorageProof { StorageProof { trie_cursor_factory, hashed_cursor_factory: self.hashed_cursor_factory, hashed_address: self.hashed_address, prefix_set: self.prefix_set, collect_branch_node_masks: self.collect_branch_node_masks, + added_removed_keys: self.added_removed_keys, } } /// Set the hashed cursor factory. - pub fn with_hashed_cursor_factory(self, hashed_cursor_factory: HF) -> StorageProof { + pub fn with_hashed_cursor_factory( + self, + hashed_cursor_factory: HF, + ) -> StorageProof { StorageProof { trie_cursor_factory: self.trie_cursor_factory, hashed_cursor_factory, hashed_address: self.hashed_address, prefix_set: self.prefix_set, collect_branch_node_masks: self.collect_branch_node_masks, + added_removed_keys: self.added_removed_keys, } } @@ -249,12 +267,32 @@ impl StorageProof { self.collect_branch_node_masks = branch_node_masks; self } + + /// Configures the retainer to retain proofs for certain nodes which would otherwise fall + /// outside the target set, when those nodes might be required to calculate the state root when + /// keys have been added or removed to the trie. + /// + /// If None is given then retention of extra proofs is disabled. + pub fn with_added_removed_keys( + self, + added_removed_keys: Option, + ) -> StorageProof { + StorageProof { + trie_cursor_factory: self.trie_cursor_factory, + hashed_cursor_factory: self.hashed_cursor_factory, + hashed_address: self.hashed_address, + prefix_set: self.prefix_set, + collect_branch_node_masks: self.collect_branch_node_masks, + added_removed_keys, + } + } } -impl StorageProof +impl StorageProof where T: TrieCursorFactory, H: HashedCursorFactory, + K: AsRef, { /// Generate an account proof from intermediate nodes. pub fn storage_proof( @@ -282,9 +320,11 @@ where self.prefix_set.extend_keys(target_nibbles.clone()); let trie_cursor = self.trie_cursor_factory.storage_trie_cursor(self.hashed_address)?; - let walker = TrieWalker::storage_trie(trie_cursor, self.prefix_set.freeze()); + let walker = TrieWalker::<_>::storage_trie(trie_cursor, self.prefix_set.freeze()) + .with_added_removed_keys(self.added_removed_keys.as_ref()); - let retainer = ProofRetainer::from_iter(target_nibbles); + let retainer = ProofRetainer::from_iter(target_nibbles) + .with_added_removed_keys(self.added_removed_keys.as_ref()); let mut hash_builder = HashBuilder::default() .with_proof_retainer(retainer) .with_updates(self.collect_branch_node_masks); @@ -308,10 +348,7 @@ where let (branch_node_hash_masks, branch_node_tree_masks) = if self.collect_branch_node_masks { let updated_branch_nodes = hash_builder.updated_branch_nodes.unwrap_or_default(); ( - updated_branch_nodes - .iter() - .map(|(path, node)| (path.clone(), node.hash_mask)) - .collect(), + updated_branch_nodes.iter().map(|(path, node)| (*path, node.hash_mask)).collect(), updated_branch_nodes .into_iter() .map(|(path, node)| (path, node.tree_mask)) diff --git a/crates/trie/trie/src/proof/blinded.rs b/crates/trie/trie/src/proof/trie_node.rs similarity index 60% rename from crates/trie/trie/src/proof/blinded.rs rename to crates/trie/trie/src/proof/trie_node.rs index 363add7116b..8625412f3ae 100644 --- a/crates/trie/trie/src/proof/blinded.rs +++ b/crates/trie/trie/src/proof/trie_node.rs @@ -2,36 +2,30 @@ use super::{Proof, StorageProof}; use crate::{hashed_cursor::HashedCursorFactory, trie_cursor::TrieCursorFactory}; use alloy_primitives::{map::HashSet, B256}; use reth_execution_errors::{SparseTrieError, SparseTrieErrorKind}; -use reth_trie_common::{prefix_set::TriePrefixSetsMut, MultiProofTargets, Nibbles}; -use reth_trie_sparse::blinded::{ - pad_path_to_key, BlindedProvider, BlindedProviderFactory, RevealedNode, +use reth_trie_common::{MultiProofTargets, Nibbles}; +use reth_trie_sparse::provider::{ + pad_path_to_key, RevealedNode, TrieNodeProvider, TrieNodeProviderFactory, }; -use std::{sync::Arc, time::Instant}; +use std::time::Instant; use tracing::{enabled, trace, Level}; /// Factory for instantiating providers capable of retrieving blinded trie nodes via proofs. #[derive(Debug, Clone)] -pub struct ProofBlindedProviderFactory { +pub struct ProofTrieNodeProviderFactory { /// The cursor factory for traversing trie nodes. trie_cursor_factory: T, /// The factory for hashed cursors. hashed_cursor_factory: H, - /// A set of prefix sets that have changes. - prefix_sets: Arc, } -impl ProofBlindedProviderFactory { +impl ProofTrieNodeProviderFactory { /// Create new proof-based blinded provider factory. - pub const fn new( - trie_cursor_factory: T, - hashed_cursor_factory: H, - prefix_sets: Arc, - ) -> Self { - Self { trie_cursor_factory, hashed_cursor_factory, prefix_sets } + pub const fn new(trie_cursor_factory: T, hashed_cursor_factory: H) -> Self { + Self { trie_cursor_factory, hashed_cursor_factory } } } -impl BlindedProviderFactory for ProofBlindedProviderFactory +impl TrieNodeProviderFactory for ProofTrieNodeProviderFactory where T: TrieCursorFactory + Clone + Send + Sync, H: HashedCursorFactory + Clone + Send + Sync, @@ -43,7 +37,6 @@ where ProofBlindedAccountProvider { trie_cursor_factory: self.trie_cursor_factory.clone(), hashed_cursor_factory: self.hashed_cursor_factory.clone(), - prefix_sets: self.prefix_sets.clone(), } } @@ -51,7 +44,6 @@ where ProofBlindedStorageProvider { trie_cursor_factory: self.trie_cursor_factory.clone(), hashed_cursor_factory: self.hashed_cursor_factory.clone(), - prefix_sets: self.prefix_sets.clone(), account, } } @@ -64,36 +56,28 @@ pub struct ProofBlindedAccountProvider { trie_cursor_factory: T, /// The factory for hashed cursors. hashed_cursor_factory: H, - /// A set of prefix sets that have changes. - prefix_sets: Arc, } impl ProofBlindedAccountProvider { /// Create new proof-based blinded account node provider. - pub const fn new( - trie_cursor_factory: T, - hashed_cursor_factory: H, - prefix_sets: Arc, - ) -> Self { - Self { trie_cursor_factory, hashed_cursor_factory, prefix_sets } + pub const fn new(trie_cursor_factory: T, hashed_cursor_factory: H) -> Self { + Self { trie_cursor_factory, hashed_cursor_factory } } } -impl BlindedProvider for ProofBlindedAccountProvider +impl TrieNodeProvider for ProofBlindedAccountProvider where - T: TrieCursorFactory + Clone + Send + Sync, - H: HashedCursorFactory + Clone + Send + Sync, + T: TrieCursorFactory, + H: HashedCursorFactory, { - fn blinded_node(&self, path: &Nibbles) -> Result, SparseTrieError> { + fn trie_node(&self, path: &Nibbles) -> Result, SparseTrieError> { let start = enabled!(target: "trie::proof::blinded", Level::TRACE).then(Instant::now); let targets = MultiProofTargets::from_iter([(pad_path_to_key(path), HashSet::default())]); - let mut proof = - Proof::new(self.trie_cursor_factory.clone(), self.hashed_cursor_factory.clone()) - .with_prefix_sets_mut(self.prefix_sets.as_ref().clone()) - .with_branch_node_masks(true) - .multiproof(targets) - .map_err(|error| SparseTrieErrorKind::Other(Box::new(error)))?; + let mut proof = Proof::new(&self.trie_cursor_factory, &self.hashed_cursor_factory) + .with_branch_node_masks(true) + .multiproof(targets) + .map_err(|error| SparseTrieErrorKind::Other(Box::new(error)))?; let node = proof.account_subtree.into_inner().remove(path); let tree_mask = proof.branch_node_tree_masks.remove(path); let hash_mask = proof.branch_node_hash_masks.remove(path); @@ -118,41 +102,31 @@ pub struct ProofBlindedStorageProvider { trie_cursor_factory: T, /// The factory for hashed cursors. hashed_cursor_factory: H, - /// A set of prefix sets that have changes. - prefix_sets: Arc, /// Target account. account: B256, } impl ProofBlindedStorageProvider { /// Create new proof-based blinded storage node provider. - pub const fn new( - trie_cursor_factory: T, - hashed_cursor_factory: H, - prefix_sets: Arc, - account: B256, - ) -> Self { - Self { trie_cursor_factory, hashed_cursor_factory, prefix_sets, account } + pub const fn new(trie_cursor_factory: T, hashed_cursor_factory: H, account: B256) -> Self { + Self { trie_cursor_factory, hashed_cursor_factory, account } } } -impl BlindedProvider for ProofBlindedStorageProvider +impl TrieNodeProvider for ProofBlindedStorageProvider where - T: TrieCursorFactory + Clone + Send + Sync, - H: HashedCursorFactory + Clone + Send + Sync, + T: TrieCursorFactory, + H: HashedCursorFactory, { - fn blinded_node(&self, path: &Nibbles) -> Result, SparseTrieError> { + fn trie_node(&self, path: &Nibbles) -> Result, SparseTrieError> { let start = enabled!(target: "trie::proof::blinded", Level::TRACE).then(Instant::now); let targets = HashSet::from_iter([pad_path_to_key(path)]); - let storage_prefix_set = - self.prefix_sets.storage_prefix_sets.get(&self.account).cloned().unwrap_or_default(); let mut proof = StorageProof::new_hashed( - self.trie_cursor_factory.clone(), - self.hashed_cursor_factory.clone(), + &self.trie_cursor_factory, + &self.hashed_cursor_factory, self.account, ) - .with_prefix_set_mut(storage_prefix_set) .with_branch_node_masks(true) .storage_multiproof(targets) .map_err(|error| SparseTrieErrorKind::Other(Box::new(error)))?; diff --git a/crates/trie/trie/src/trie.rs b/crates/trie/trie/src/trie.rs index fc38b653d8c..17cdd1f96c5 100644 --- a/crates/trie/trie/src/trie.rs +++ b/crates/trie/trie/src/trie.rs @@ -1,10 +1,13 @@ use crate::{ - hashed_cursor::{HashedCursorFactory, HashedStorageCursor}, + hashed_cursor::{HashedCursor, HashedCursorFactory, HashedStorageCursor}, node_iter::{TrieElement, TrieNodeIter}, prefix_set::{PrefixSet, TriePrefixSets}, - progress::{IntermediateStateRootState, StateRootProgress}, + progress::{ + IntermediateRootState, IntermediateStateRootState, IntermediateStorageRootState, + StateRootProgress, StorageRootProgress, + }, stats::TrieTracker, - trie_cursor::TrieCursorFactory, + trie_cursor::{TrieCursor, TrieCursorFactory}, updates::{StorageTrieUpdates, TrieUpdates}, walker::TrieWalker, HashBuilder, Nibbles, TRIE_ACCOUNT_RLP_MAX_SIZE, @@ -12,8 +15,14 @@ use crate::{ use alloy_consensus::EMPTY_ROOT_HASH; use alloy_primitives::{keccak256, Address, B256}; use alloy_rlp::{BufMut, Encodable}; +use alloy_trie::proof::AddedRemovedKeys; use reth_execution_errors::{StateRootError, StorageRootError}; -use tracing::trace; +use reth_primitives_traits::Account; +use tracing::{debug, instrument, trace}; + +/// The default updates after which root algorithms should return intermediate progress rather than +/// finishing the computation. +const DEFAULT_INTERMEDIATE_THRESHOLD: u64 = 100_000; #[cfg(feature = "metrics")] use crate::metrics::{StateRootMetrics, TrieRootMetrics}; @@ -48,7 +57,7 @@ impl StateRoot { hashed_cursor_factory, prefix_sets: TriePrefixSets::default(), previous_state: None, - threshold: 100_000, + threshold: DEFAULT_INTERMEDIATE_THRESHOLD, #[cfg(feature = "metrics")] metrics: StateRootMetrics::default(), } @@ -117,7 +126,7 @@ where /// /// # Returns /// - /// The intermediate progress of state root computation and the trie updates. + /// The state root and the trie updates. pub fn root_with_updates(self) -> Result<(B256, TrieUpdates), StateRootError> { match self.with_no_threshold().calculate(true)? { StateRootProgress::Complete(root, _, updates) => Ok((root, updates)), @@ -134,7 +143,7 @@ where pub fn root(self) -> Result { match self.calculate(false)? { StateRootProgress::Complete(root, _, _) => Ok(root), - StateRootProgress::Progress(..) => unreachable!(), // update retenion is disabled + StateRootProgress::Progress(..) => unreachable!(), // update retention is disabled } } @@ -151,37 +160,90 @@ where fn calculate(self, retain_updates: bool) -> Result { trace!(target: "trie::state_root", "calculating state root"); let mut tracker = TrieTracker::default(); - let mut trie_updates = TrieUpdates::default(); let trie_cursor = self.trie_cursor_factory.account_trie_cursor()?; - let hashed_account_cursor = self.hashed_cursor_factory.hashed_account_cursor()?; - let (mut hash_builder, mut account_node_iter) = match self.previous_state { - Some(state) => { - let hash_builder = state.hash_builder.with_updates(retain_updates); - let walker = TrieWalker::state_trie_from_stack( - trie_cursor, - state.walker_stack, - self.prefix_sets.account_prefix_set, + + // create state root context once for reuse + let mut storage_ctx = StateRootContext::new(); + + // first handle any in-progress storage root calculation + let (mut hash_builder, mut account_node_iter) = if let Some(state) = self.previous_state { + let IntermediateStateRootState { account_root_state, storage_root_state } = state; + + // resume account trie iteration + let mut hash_builder = account_root_state.hash_builder.with_updates(retain_updates); + let walker = TrieWalker::<_>::state_trie_from_stack( + trie_cursor, + account_root_state.walker_stack, + self.prefix_sets.account_prefix_set, + ) + .with_deletions_retained(retain_updates); + let account_node_iter = TrieNodeIter::state_trie(walker, hashed_account_cursor) + .with_last_hashed_key(account_root_state.last_hashed_key); + + // if we have an in-progress storage root, complete it first + if let Some(storage_state) = storage_root_state { + let hashed_address = account_root_state.last_hashed_key; + let account = storage_state.account; + + debug!( + target: "trie::state_root", + account_nonce = account.nonce, + account_balance = ?account.balance, + last_hashed_key = ?account_root_state.last_hashed_key, + "Resuming storage root calculation" + ); + + // resume the storage root calculation + let remaining_threshold = self.threshold.saturating_sub( + storage_ctx.total_updates_len(&account_node_iter, &hash_builder), + ); + + let storage_root_calculator = StorageRoot::new_hashed( + self.trie_cursor_factory.clone(), + self.hashed_cursor_factory.clone(), + hashed_address, + self.prefix_sets + .storage_prefix_sets + .get(&hashed_address) + .cloned() + .unwrap_or_default(), + #[cfg(feature = "metrics")] + self.metrics.storage_trie.clone(), ) - .with_deletions_retained(retain_updates); - let node_iter = TrieNodeIter::state_trie(walker, hashed_account_cursor) - .with_last_hashed_key(state.last_account_key); - (hash_builder, node_iter) - } - None => { - let hash_builder = HashBuilder::default().with_updates(retain_updates); - let walker = - TrieWalker::state_trie(trie_cursor, self.prefix_sets.account_prefix_set) - .with_deletions_retained(retain_updates); - let node_iter = TrieNodeIter::state_trie(walker, hashed_account_cursor); - (hash_builder, node_iter) + .with_intermediate_state(Some(storage_state.state)) + .with_threshold(remaining_threshold); + + let storage_result = storage_root_calculator.calculate(retain_updates)?; + if let Some(storage_state) = storage_ctx.process_storage_root_result( + storage_result, + hashed_address, + account, + &mut hash_builder, + retain_updates, + )? { + // still in progress, need to pause again + return Ok(storage_ctx.create_progress_state( + account_node_iter, + hash_builder, + account_root_state.last_hashed_key, + Some(storage_state), + )) + } } + + (hash_builder, account_node_iter) + } else { + // no intermediate state, create new hash builder and node iter for state root + // calculation + let hash_builder = HashBuilder::default().with_updates(retain_updates); + let walker = TrieWalker::state_trie(trie_cursor, self.prefix_sets.account_prefix_set) + .with_deletions_retained(retain_updates); + let node_iter = TrieNodeIter::state_trie(walker, hashed_account_cursor); + (hash_builder, node_iter) }; - let mut account_rlp = Vec::with_capacity(TRIE_ACCOUNT_RLP_MAX_SIZE); - let mut hashed_entries_walked = 0; - let mut updated_storage_nodes = 0; while let Some(node) = account_node_iter.try_next()? { match node { TrieElement::Branch(node) => { @@ -190,15 +252,14 @@ where } TrieElement::Leaf(hashed_address, account) => { tracker.inc_leaf(); - hashed_entries_walked += 1; + storage_ctx.hashed_entries_walked += 1; + + // calculate storage root, calculating the remaining threshold so we have + // bounded memory usage even while in the middle of storage root calculation + let remaining_threshold = self.threshold.saturating_sub( + storage_ctx.total_updates_len(&account_node_iter, &hash_builder), + ); - // We assume we can always calculate a storage root without - // OOMing. This opens us up to a potential DOS vector if - // a contract had too many storage entries and they were - // all buffered w/o us returning and committing our intermediate - // progress. - // TODO: We can consider introducing the TrieProgress::Progress/Complete - // abstraction inside StorageRoot, but let's give it a try as-is for now. let storage_root_calculator = StorageRoot::new_hashed( self.trie_cursor_factory.clone(), self.hashed_cursor_factory.clone(), @@ -210,45 +271,35 @@ where .unwrap_or_default(), #[cfg(feature = "metrics")] self.metrics.storage_trie.clone(), - ); + ) + .with_threshold(remaining_threshold); - let storage_root = if retain_updates { - let (root, storage_slots_walked, updates) = - storage_root_calculator.root_with_updates()?; - hashed_entries_walked += storage_slots_walked; - // We only walk over hashed address once, so it's safe to insert. - updated_storage_nodes += updates.len(); - trie_updates.insert_storage_updates(hashed_address, updates); - root - } else { - storage_root_calculator.root()? - }; - - account_rlp.clear(); - let account = account.into_trie_account(storage_root); - account.encode(&mut account_rlp as &mut dyn BufMut); - hash_builder.add_leaf(Nibbles::unpack(hashed_address), &account_rlp); - - // Decide if we need to return intermediate progress. - let total_updates_len = updated_storage_nodes + - account_node_iter.walker.removed_keys_len() + - hash_builder.updates_len(); - if retain_updates && total_updates_len as u64 >= self.threshold { - let (walker_stack, walker_deleted_keys) = account_node_iter.walker.split(); - trie_updates.removed_nodes.extend(walker_deleted_keys); - let (hash_builder, hash_builder_updates) = hash_builder.split(); - trie_updates.account_nodes.extend(hash_builder_updates); - - let state = IntermediateStateRootState { + let storage_result = storage_root_calculator.calculate(retain_updates)?; + if let Some(storage_state) = storage_ctx.process_storage_root_result( + storage_result, + hashed_address, + account, + &mut hash_builder, + retain_updates, + )? { + // storage root hit threshold, need to pause + return Ok(storage_ctx.create_progress_state( + account_node_iter, hash_builder, - walker_stack, - last_account_key: hashed_address, - }; + hashed_address, + Some(storage_state), + )) + } - return Ok(StateRootProgress::Progress( - Box::new(state), - hashed_entries_walked, - trie_updates, + // decide if we need to return intermediate progress + let total_updates_len = + storage_ctx.total_updates_len(&account_node_iter, &hash_builder); + if retain_updates && total_updates_len >= self.threshold { + return Ok(storage_ctx.create_progress_state( + account_node_iter, + hash_builder, + hashed_address, + None, )) } } @@ -258,6 +309,7 @@ where let root = hash_builder.root(); let removed_keys = account_node_iter.walker.take_removed_keys(); + let StateRootContext { mut trie_updates, hashed_entries_walked, .. } = storage_ctx; trie_updates.finalize(hash_builder, removed_keys, self.prefix_sets.destroyed_accounts); let stats = tracker.finish(); @@ -278,6 +330,130 @@ where } } +/// Contains state mutated during state root calculation and storage root result handling. +#[derive(Debug)] +pub(crate) struct StateRootContext { + /// Reusable buffer for encoding account data. + account_rlp: Vec, + /// Accumulates updates from account and storage root calculation. + trie_updates: TrieUpdates, + /// Tracks total hashed entries walked. + hashed_entries_walked: usize, + /// Counts storage trie nodes updated. + updated_storage_nodes: usize, +} + +impl StateRootContext { + /// Creates a new state root context. + fn new() -> Self { + Self { + account_rlp: Vec::with_capacity(TRIE_ACCOUNT_RLP_MAX_SIZE), + trie_updates: TrieUpdates::default(), + hashed_entries_walked: 0, + updated_storage_nodes: 0, + } + } + + /// Creates a [`StateRootProgress`] when the threshold is hit, from the state of the current + /// [`TrieNodeIter`], [`HashBuilder`], last hashed key and any storage root intermediate state. + fn create_progress_state( + mut self, + account_node_iter: TrieNodeIter, + hash_builder: HashBuilder, + last_hashed_key: B256, + storage_state: Option, + ) -> StateRootProgress + where + C: TrieCursor, + H: HashedCursor, + K: AsRef, + { + let (walker_stack, walker_deleted_keys) = account_node_iter.walker.split(); + self.trie_updates.removed_nodes.extend(walker_deleted_keys); + let (hash_builder, hash_builder_updates) = hash_builder.split(); + self.trie_updates.account_nodes.extend(hash_builder_updates); + + let account_state = IntermediateRootState { hash_builder, walker_stack, last_hashed_key }; + + let state = IntermediateStateRootState { + account_root_state: account_state, + storage_root_state: storage_state, + }; + + StateRootProgress::Progress(Box::new(state), self.hashed_entries_walked, self.trie_updates) + } + + /// Calculates the total number of updated nodes. + fn total_updates_len( + &self, + account_node_iter: &TrieNodeIter, + hash_builder: &HashBuilder, + ) -> u64 + where + C: TrieCursor, + H: HashedCursor, + K: AsRef, + { + (self.updated_storage_nodes + + account_node_iter.walker.removed_keys_len() + + hash_builder.updates_len()) as u64 + } + + /// Processes the result of a storage root calculation. + /// + /// Handles both completed and in-progress storage root calculations: + /// - For completed roots: encodes the account with the storage root, updates the hash builder + /// with the new account, and updates metrics. + /// - For in-progress roots: returns the intermediate state for later resumption + /// + /// Returns an [`IntermediateStorageRootState`] if the calculation needs to be resumed later, or + /// `None` if the storage root was successfully computed and added to the trie. + fn process_storage_root_result( + &mut self, + storage_result: StorageRootProgress, + hashed_address: B256, + account: Account, + hash_builder: &mut HashBuilder, + retain_updates: bool, + ) -> Result, StateRootError> { + match storage_result { + StorageRootProgress::Complete(storage_root, storage_slots_walked, updates) => { + // Storage root completed + self.hashed_entries_walked += storage_slots_walked; + if retain_updates { + self.updated_storage_nodes += updates.len(); + self.trie_updates.insert_storage_updates(hashed_address, updates); + } + + // Encode the account with the computed storage root + self.account_rlp.clear(); + let trie_account = account.into_trie_account(storage_root); + trie_account.encode(&mut self.account_rlp as &mut dyn BufMut); + hash_builder.add_leaf(Nibbles::unpack(hashed_address), &self.account_rlp); + Ok(None) + } + StorageRootProgress::Progress(state, storage_slots_walked, updates) => { + // Storage root hit threshold or resumed calculation hit threshold + debug!( + target: "trie::state_root", + ?hashed_address, + storage_slots_walked, + last_storage_key = ?state.last_hashed_key, + ?account, + "Pausing storage root calculation" + ); + + self.hashed_entries_walked += storage_slots_walked; + if retain_updates { + self.trie_updates.insert_storage_updates(hashed_address, updates); + } + + Ok(Some(IntermediateStorageRootState { state: *state, account })) + } + } + } +} + /// `StorageRoot` is used to compute the root node of an account storage trie. #[derive(Debug)] pub struct StorageRoot { @@ -289,6 +465,10 @@ pub struct StorageRoot { pub hashed_address: B256, /// The set of storage slot prefixes that have changed. pub prefix_set: PrefixSet, + /// Previous intermediate state. + previous_state: Option, + /// The number of updates after which the intermediate progress should be returned. + threshold: u64, /// Storage root metrics. #[cfg(feature = "metrics")] metrics: TrieRootMetrics, @@ -326,6 +506,8 @@ impl StorageRoot { hashed_cursor_factory, hashed_address, prefix_set, + previous_state: None, + threshold: DEFAULT_INTERMEDIATE_THRESHOLD, #[cfg(feature = "metrics")] metrics, } @@ -337,6 +519,24 @@ impl StorageRoot { self } + /// Set the threshold. + pub const fn with_threshold(mut self, threshold: u64) -> Self { + self.threshold = threshold; + self + } + + /// Set the threshold to maximum value so that intermediate progress is not returned. + pub const fn with_no_threshold(mut self) -> Self { + self.threshold = u64::MAX; + self + } + + /// Set the previously recorded intermediate state. + pub fn with_intermediate_state(mut self, state: Option) -> Self { + self.previous_state = state; + self + } + /// Set the hashed cursor factory. pub fn with_hashed_cursor_factory(self, hashed_cursor_factory: HF) -> StorageRoot { StorageRoot { @@ -344,6 +544,8 @@ impl StorageRoot { hashed_cursor_factory, hashed_address: self.hashed_address, prefix_set: self.prefix_set, + previous_state: self.previous_state, + threshold: self.threshold, #[cfg(feature = "metrics")] metrics: self.metrics, } @@ -356,6 +558,8 @@ impl StorageRoot { hashed_cursor_factory: self.hashed_cursor_factory, hashed_address: self.hashed_address, prefix_set: self.prefix_set, + previous_state: self.previous_state, + threshold: self.threshold, #[cfg(feature = "metrics")] metrics: self.metrics, } @@ -367,13 +571,26 @@ where T: TrieCursorFactory, H: HashedCursorFactory, { + /// Walks the intermediate nodes of existing storage trie (if any) and hashed entries. Feeds the + /// nodes into the hash builder. Collects the updates in the process. + /// + /// # Returns + /// + /// The intermediate progress of state root computation. + pub fn root_with_progress(self) -> Result { + self.calculate(true) + } + /// Walks the hashed storage table entries for a given address and calculates the storage root. /// /// # Returns /// /// The storage root and storage trie updates for a given address. pub fn root_with_updates(self) -> Result<(B256, usize, StorageTrieUpdates), StorageRootError> { - self.calculate(true) + match self.with_no_threshold().calculate(true)? { + StorageRootProgress::Complete(root, walked, updates) => Ok((root, walked, updates)), + StorageRootProgress::Progress(..) => unreachable!(), // unreachable threshold + } } /// Walks the hashed storage table entries for a given address and calculates the storage root. @@ -382,8 +599,10 @@ where /// /// The storage root. pub fn root(self) -> Result { - let (root, _, _) = self.calculate(false)?; - Ok(root) + match self.calculate(false)? { + StorageRootProgress::Complete(root, _, _) => Ok(root), + StorageRootProgress::Progress(..) => unreachable!(), // update retenion is disabled + } } /// Walks the hashed storage table entries for a given address and calculates the storage root. @@ -392,28 +611,50 @@ where /// /// The storage root, number of walked entries and trie updates /// for a given address if requested. - pub fn calculate( - self, - retain_updates: bool, - ) -> Result<(B256, usize, StorageTrieUpdates), StorageRootError> { - trace!(target: "trie::storage_root", hashed_address = ?self.hashed_address, "calculating storage root"); + #[instrument(skip_all, target = "trie::storage_root", name = "Storage trie", fields(hashed_address = ?self.hashed_address))] + pub fn calculate(self, retain_updates: bool) -> Result { + trace!(target: "trie::storage_root", "calculating storage root"); let mut hashed_storage_cursor = self.hashed_cursor_factory.hashed_storage_cursor(self.hashed_address)?; // short circuit on empty storage if hashed_storage_cursor.is_storage_empty()? { - return Ok((EMPTY_ROOT_HASH, 0, StorageTrieUpdates::deleted())) + return Ok(StorageRootProgress::Complete( + EMPTY_ROOT_HASH, + 0, + StorageTrieUpdates::deleted(), + )) } let mut tracker = TrieTracker::default(); + let mut trie_updates = StorageTrieUpdates::default(); + let trie_cursor = self.trie_cursor_factory.storage_trie_cursor(self.hashed_address)?; - let walker = TrieWalker::storage_trie(trie_cursor, self.prefix_set) - .with_deletions_retained(retain_updates); - let mut hash_builder = HashBuilder::default().with_updates(retain_updates); + let (mut hash_builder, mut storage_node_iter) = match self.previous_state { + Some(state) => { + let hash_builder = state.hash_builder.with_updates(retain_updates); + let walker = TrieWalker::<_>::storage_trie_from_stack( + trie_cursor, + state.walker_stack, + self.prefix_set, + ) + .with_deletions_retained(retain_updates); + let node_iter = TrieNodeIter::storage_trie(walker, hashed_storage_cursor) + .with_last_hashed_key(state.last_hashed_key); + (hash_builder, node_iter) + } + None => { + let hash_builder = HashBuilder::default().with_updates(retain_updates); + let walker = TrieWalker::storage_trie(trie_cursor, self.prefix_set) + .with_deletions_retained(retain_updates); + let node_iter = TrieNodeIter::storage_trie(walker, hashed_storage_cursor); + (hash_builder, node_iter) + } + }; - let mut storage_node_iter = TrieNodeIter::storage_trie(walker, hashed_storage_cursor); + let mut hashed_entries_walked = 0; while let Some(node) = storage_node_iter.try_next()? { match node { TrieElement::Branch(node) => { @@ -422,17 +663,39 @@ where } TrieElement::Leaf(hashed_slot, value) => { tracker.inc_leaf(); + hashed_entries_walked += 1; hash_builder.add_leaf( Nibbles::unpack(hashed_slot), alloy_rlp::encode_fixed_size(&value).as_ref(), ); + + // Check if we need to return intermediate progress + let total_updates_len = + storage_node_iter.walker.removed_keys_len() + hash_builder.updates_len(); + if retain_updates && total_updates_len as u64 >= self.threshold { + let (walker_stack, walker_deleted_keys) = storage_node_iter.walker.split(); + trie_updates.removed_nodes.extend(walker_deleted_keys); + let (hash_builder, hash_builder_updates) = hash_builder.split(); + trie_updates.storage_nodes.extend(hash_builder_updates); + + let state = IntermediateRootState { + hash_builder, + walker_stack, + last_hashed_key: hashed_slot, + }; + + return Ok(StorageRootProgress::Progress( + Box::new(state), + hashed_entries_walked, + trie_updates, + )) + } } } } let root = hash_builder.root(); - let mut trie_updates = StorageTrieUpdates::default(); let removed_keys = storage_node_iter.walker.take_removed_keys(); trie_updates.finalize(hash_builder, removed_keys); @@ -452,7 +715,7 @@ where ); let storage_slots_walked = stats.leaves_added() as usize; - Ok((root, storage_slots_walked, trie_updates)) + Ok(StorageRootProgress::Complete(root, storage_slots_walked, trie_updates)) } } diff --git a/crates/trie/trie/src/trie_cursor/depth_first.rs b/crates/trie/trie/src/trie_cursor/depth_first.rs new file mode 100644 index 00000000000..b9cef85e3ff --- /dev/null +++ b/crates/trie/trie/src/trie_cursor/depth_first.rs @@ -0,0 +1,401 @@ +use super::TrieCursor; +use crate::{BranchNodeCompact, Nibbles}; +use reth_storage_errors::db::DatabaseError; +use std::cmp::Ordering; +use tracing::trace; + +/// Compares two Nibbles in depth-first order. +/// +/// In depth-first ordering: +/// - Descendants come before their ancestors (children before parents) +/// - Siblings are ordered lexicographically +/// +/// # Example +/// +/// ```text +/// 0x11 comes before 0x1 (child before parent) +/// 0x12 comes before 0x1 (child before parent) +/// 0x11 comes before 0x12 (lexicographical among siblings) +/// 0x1 comes before 0x21 (lexicographical among siblings) +/// Result: 0x11, 0x12, 0x1, 0x21 +/// ``` +pub fn cmp(a: &Nibbles, b: &Nibbles) -> Ordering { + // If the two are of equal length, then compare them lexicographically + if a.len() == b.len() { + return a.cmp(b) + } + + // If one is a prefix of the other, then the other comes first + let common_prefix_len = a.common_prefix_length(b); + if a.len() == common_prefix_len { + return Ordering::Greater + } else if b.len() == common_prefix_len { + return Ordering::Less + } + + // Otherwise the nibble after the prefix determines the ordering. We know that neither is empty + // at this point, otherwise the previous if/else block would have caught it. + a.get_unchecked(common_prefix_len).cmp(&b.get_unchecked(common_prefix_len)) +} + +/// An iterator that traverses trie nodes in depth-first post-order. +/// +/// This iterator yields nodes in post-order traversal (children before parents), +/// which matches the `cmp` comparison function where descendants +/// come before their ancestors. +#[derive(Debug)] +pub struct DepthFirstTrieIterator { + /// The underlying trie cursor. + cursor: C, + /// Set to true once the trie cursor has done its initial seek to the root node. + initialized: bool, + /// Stack of nodes which have been fetched. Each node's path is a prefix of the next's. + stack: Vec<(Nibbles, BranchNodeCompact)>, + /// Nodes which are ready to be yielded from `next`. + next: Vec<(Nibbles, BranchNodeCompact)>, + /// Set to true once the cursor has been exhausted. + complete: bool, +} + +impl DepthFirstTrieIterator { + /// Create a new depth-first iterator from a trie cursor. + pub fn new(cursor: C) -> Self { + Self { + cursor, + initialized: false, + stack: Default::default(), + next: Default::default(), + complete: false, + } + } + + fn push(&mut self, path: Nibbles, node: BranchNodeCompact) { + loop { + match self.stack.last() { + None => { + // If the stack is empty then we push this node onto it, as it may have child + // nodes which need to be yielded first. + self.stack.push((path, node)); + break + } + Some((top_path, _)) if path.starts_with(top_path) => { + // If the top of the stack is a prefix of this node, it means this node is a + // child of the top of the stack (and all other nodes on the stack). Push this + // node onto the stack, as future nodes may be children of it. + self.stack.push((path, node)); + break + } + Some((_, _)) => { + // The top of the stack is not a prefix of this node, therefore it is not a + // parent of this node. Yield the top of the stack, and loop back to see if this + // node is a child of the new top-of-stack. + self.next.push(self.stack.pop().expect("stack is not empty")); + } + } + } + + // We will have popped off the top of the stack in the order we want to yield nodes, but + // `next` is itself popped off so it needs to be reversed. + self.next.reverse(); + } + + fn fill_next(&mut self) -> Result<(), DatabaseError> { + debug_assert!(self.next.is_empty()); + + loop { + let Some((path, node)) = (if self.initialized { + self.cursor.next()? + } else { + self.initialized = true; + self.cursor.seek(Nibbles::new())? + }) else { + // Record that the cursor is empty and yield the stack. The stack is in reverse + // order of what we want to yield, but `next` is popped from, so we don't have to + // reverse it. + self.complete = true; + self.next = core::mem::take(&mut self.stack); + return Ok(()) + }; + + trace!( + target: "trie::trie_cursor::depth_first", + ?path, + "Iterated from cursor", + ); + + self.push(path, node); + if !self.next.is_empty() { + return Ok(()) + } + } + } +} + +impl Iterator for DepthFirstTrieIterator { + type Item = Result<(Nibbles, BranchNodeCompact), DatabaseError>; + + fn next(&mut self) -> Option { + loop { + if let Some(next) = self.next.pop() { + return Some(Ok(next)) + } + + if self.complete { + return None + } + + if let Err(err) = self.fill_next() { + return Some(Err(err)) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::trie_cursor::{mock::MockTrieCursorFactory, TrieCursorFactory}; + use alloy_trie::TrieMask; + use std::{collections::BTreeMap, sync::Arc}; + + fn create_test_node(state_nibbles: &[u8], tree_nibbles: &[u8]) -> BranchNodeCompact { + let mut state_mask = TrieMask::default(); + for &nibble in state_nibbles { + state_mask.set_bit(nibble); + } + + let mut tree_mask = TrieMask::default(); + for &nibble in tree_nibbles { + tree_mask.set_bit(nibble); + } + + BranchNodeCompact { + state_mask, + tree_mask, + hash_mask: TrieMask::default(), + hashes: Arc::new(vec![]), + root_hash: None, + } + } + + #[test] + fn test_depth_first_cmp() { + // Test case 1: Child comes before parent + let child = Nibbles::from_nibbles([0x1, 0x1]); + let parent = Nibbles::from_nibbles([0x1]); + assert_eq!(cmp(&child, &parent), Ordering::Less); + assert_eq!(cmp(&parent, &child), Ordering::Greater); + + // Test case 2: Deeper descendant comes before ancestor + let deep = Nibbles::from_nibbles([0x1, 0x2, 0x3, 0x4]); + let ancestor = Nibbles::from_nibbles([0x1, 0x2]); + assert_eq!(cmp(&deep, &ancestor), Ordering::Less); + assert_eq!(cmp(&ancestor, &deep), Ordering::Greater); + + // Test case 3: Siblings use lexicographical ordering + let sibling1 = Nibbles::from_nibbles([0x1, 0x2]); + let sibling2 = Nibbles::from_nibbles([0x1, 0x3]); + assert_eq!(cmp(&sibling1, &sibling2), Ordering::Less); + assert_eq!(cmp(&sibling2, &sibling1), Ordering::Greater); + + // Test case 4: Different branches use lexicographical ordering + let branch1 = Nibbles::from_nibbles([0x1]); + let branch2 = Nibbles::from_nibbles([0x2]); + assert_eq!(cmp(&branch1, &branch2), Ordering::Less); + assert_eq!(cmp(&branch2, &branch1), Ordering::Greater); + + // Test case 5: Empty path comes after everything + let empty = Nibbles::new(); + let non_empty = Nibbles::from_nibbles([0x0]); + assert_eq!(cmp(&non_empty, &empty), Ordering::Less); + assert_eq!(cmp(&empty, &non_empty), Ordering::Greater); + + // Test case 6: Same paths are equal + let same1 = Nibbles::from_nibbles([0x1, 0x2, 0x3]); + let same2 = Nibbles::from_nibbles([0x1, 0x2, 0x3]); + assert_eq!(cmp(&same1, &same2), Ordering::Equal); + } + + #[test] + fn test_depth_first_ordering_complex() { + // Test the example from the conversation: 0x11, 0x12, 0x1, 0x2 + let mut paths = [ + Nibbles::from_nibbles([0x1]), // 0x1 + Nibbles::from_nibbles([0x2]), // 0x2 + Nibbles::from_nibbles([0x1, 0x1]), // 0x11 + Nibbles::from_nibbles([0x1, 0x2]), // 0x12 + ]; + + // Shuffle to ensure sorting works regardless of input order + paths.reverse(); + + // Sort using depth-first ordering + paths.sort_by(cmp); + + // Expected order: 0x11, 0x12, 0x1, 0x2 + assert_eq!(paths[0], Nibbles::from_nibbles([0x1, 0x1])); // 0x11 + assert_eq!(paths[1], Nibbles::from_nibbles([0x1, 0x2])); // 0x12 + assert_eq!(paths[2], Nibbles::from_nibbles([0x1])); // 0x1 + assert_eq!(paths[3], Nibbles::from_nibbles([0x2])); // 0x2 + } + + #[test] + fn test_depth_first_ordering_tree() { + // Test a more complex tree structure + let mut paths = vec![ + Nibbles::new(), // root (empty) + Nibbles::from_nibbles([0x1]), // 0x1 + Nibbles::from_nibbles([0x1, 0x1]), // 0x11 + Nibbles::from_nibbles([0x1, 0x1, 0x1]), // 0x111 + Nibbles::from_nibbles([0x1, 0x1, 0x2]), // 0x112 + Nibbles::from_nibbles([0x1, 0x2]), // 0x12 + Nibbles::from_nibbles([0x2]), // 0x2 + Nibbles::from_nibbles([0x2, 0x1]), // 0x21 + ]; + + // Shuffle + paths.reverse(); + + // Sort using depth-first ordering + paths.sort_by(cmp); + + // Expected depth-first order: + // All descendants come before ancestors + // Within the same level, lexicographical order + assert_eq!(paths[0], Nibbles::from_nibbles([0x1, 0x1, 0x1])); // 0x111 (deepest in 0x1 branch) + assert_eq!(paths[1], Nibbles::from_nibbles([0x1, 0x1, 0x2])); // 0x112 (sibling of 0x111) + assert_eq!(paths[2], Nibbles::from_nibbles([0x1, 0x1])); // 0x11 (parent of 0x111, 0x112) + assert_eq!(paths[3], Nibbles::from_nibbles([0x1, 0x2])); // 0x12 (sibling of 0x11) + assert_eq!(paths[4], Nibbles::from_nibbles([0x1])); // 0x1 (parent of 0x11, 0x12) + assert_eq!(paths[5], Nibbles::from_nibbles([0x2, 0x1])); // 0x21 (child of 0x2) + assert_eq!(paths[6], Nibbles::from_nibbles([0x2])); // 0x2 (parent of 0x21) + assert_eq!(paths[7], Nibbles::new()); // root (empty, parent of all) + } + + #[test] + fn test_empty_trie() { + let factory = MockTrieCursorFactory::new(BTreeMap::new(), Default::default()); + let cursor = factory.account_trie_cursor().unwrap(); + let mut iter = DepthFirstTrieIterator::new(cursor); + assert!(iter.next().is_none()); + } + + #[test] + fn test_single_node() { + let path = Nibbles::from_nibbles([0x1, 0x2, 0x3]); + let node = create_test_node(&[0x4], &[0x5]); + + let mut nodes = BTreeMap::new(); + nodes.insert(path, node.clone()); + let factory = MockTrieCursorFactory::new(nodes, Default::default()); + let cursor = factory.account_trie_cursor().unwrap(); + let mut iter = DepthFirstTrieIterator::new(cursor); + + let result = iter.next().unwrap().unwrap(); + assert_eq!(result.0, path); + assert_eq!(result.1, node); + assert!(iter.next().is_none()); + } + + #[test] + fn test_depth_first_order() { + // Create a simple trie structure: + // root + // ├── 0x1 (has children 0x2 and 0x3) + // │ ├── 0x12 + // │ └── 0x13 + // └── 0x2 (has child 0x4) + // └── 0x24 + + let nodes = vec![ + // Root node with children at nibbles 1 and 2 + (Nibbles::default(), create_test_node(&[], &[0x1, 0x2])), + // Node at path 0x1 with children at nibbles 2 and 3 + (Nibbles::from_nibbles([0x1]), create_test_node(&[], &[0x2, 0x3])), + // Leaf nodes + (Nibbles::from_nibbles([0x1, 0x2]), create_test_node(&[0xF], &[])), + (Nibbles::from_nibbles([0x1, 0x3]), create_test_node(&[0xF], &[])), + // Node at path 0x2 with child at nibble 4 + (Nibbles::from_nibbles([0x2]), create_test_node(&[], &[0x4])), + // Leaf node + (Nibbles::from_nibbles([0x2, 0x4]), create_test_node(&[0xF], &[])), + ]; + + let nodes_map: BTreeMap<_, _> = nodes.into_iter().collect(); + let factory = MockTrieCursorFactory::new(nodes_map, Default::default()); + let cursor = factory.account_trie_cursor().unwrap(); + let iter = DepthFirstTrieIterator::new(cursor); + + // Expected post-order (depth-first with children before parents): + // 1. 0x12 (leaf, child of 0x1) + // 2. 0x13 (leaf, child of 0x1) + // 3. 0x1 (parent of 0x12 and 0x13) + // 4. 0x24 (leaf, child of 0x2) + // 5. 0x2 (parent of 0x24) + // 6. Root (parent of 0x1 and 0x2) + + let expected_order = vec![ + Nibbles::from_nibbles([0x1, 0x2]), + Nibbles::from_nibbles([0x1, 0x3]), + Nibbles::from_nibbles([0x1]), + Nibbles::from_nibbles([0x2, 0x4]), + Nibbles::from_nibbles([0x2]), + Nibbles::default(), + ]; + + let mut actual_order = Vec::new(); + for result in iter { + let (path, _) = result.unwrap(); + actual_order.push(path); + } + + assert_eq!(actual_order, expected_order); + } + + #[test] + fn test_complex_tree() { + // Create a more complex tree structure with multiple levels + let nodes = vec![ + // Root with multiple children + (Nibbles::default(), create_test_node(&[], &[0x0, 0x5, 0xA, 0xF])), + // Branch at 0x0 with children + (Nibbles::from_nibbles([0x0]), create_test_node(&[], &[0x1, 0x2])), + (Nibbles::from_nibbles([0x0, 0x1]), create_test_node(&[0x3], &[])), + (Nibbles::from_nibbles([0x0, 0x2]), create_test_node(&[0x4], &[])), + // Branch at 0x5 with no children (leaf) + (Nibbles::from_nibbles([0x5]), create_test_node(&[0xB], &[])), + // Branch at 0xA with deep nesting + (Nibbles::from_nibbles([0xA]), create_test_node(&[], &[0xB])), + (Nibbles::from_nibbles([0xA, 0xB]), create_test_node(&[], &[0xC])), + (Nibbles::from_nibbles([0xA, 0xB, 0xC]), create_test_node(&[0xD], &[])), + // Branch at 0xF (leaf) + (Nibbles::from_nibbles([0xF]), create_test_node(&[0xE], &[])), + ]; + + let nodes_map: BTreeMap<_, _> = nodes.into_iter().collect(); + let factory = MockTrieCursorFactory::new(nodes_map, Default::default()); + let cursor = factory.account_trie_cursor().unwrap(); + let iter = DepthFirstTrieIterator::new(cursor); + + // Verify post-order traversal (children before parents) + let expected_order = vec![ + Nibbles::from_nibbles([0x0, 0x1]), // leaf child of 0x0 + Nibbles::from_nibbles([0x0, 0x2]), // leaf child of 0x0 + Nibbles::from_nibbles([0x0]), // parent of 0x01 and 0x02 + Nibbles::from_nibbles([0x5]), // leaf + Nibbles::from_nibbles([0xA, 0xB, 0xC]), // deepest leaf + Nibbles::from_nibbles([0xA, 0xB]), // parent of 0xABC + Nibbles::from_nibbles([0xA]), // parent of 0xAB + Nibbles::from_nibbles([0xF]), // leaf + Nibbles::default(), // root (last) + ]; + + let mut actual_order = Vec::new(); + for result in iter { + let (path, _node) = result.unwrap(); + actual_order.push(path); + } + + assert_eq!(actual_order, expected_order); + } +} diff --git a/crates/trie/trie/src/trie_cursor/in_memory.rs b/crates/trie/trie/src/trie_cursor/in_memory.rs index 40f4447daa6..d9658150f3a 100644 --- a/crates/trie/trie/src/trie_cursor/in_memory.rs +++ b/crates/trie/trie/src/trie_cursor/in_memory.rs @@ -1,126 +1,208 @@ use super::{TrieCursor, TrieCursorFactory}; -use crate::{ - forward_cursor::ForwardInMemoryCursor, - updates::{StorageTrieUpdatesSorted, TrieUpdatesSorted}, -}; -use alloy_primitives::{map::HashSet, B256}; +use crate::{forward_cursor::ForwardInMemoryCursor, updates::TrieUpdatesSorted}; +use alloy_primitives::B256; use reth_storage_errors::db::DatabaseError; use reth_trie_common::{BranchNodeCompact, Nibbles}; /// The trie cursor factory for the trie updates. #[derive(Debug, Clone)] -pub struct InMemoryTrieCursorFactory<'a, CF> { +pub struct InMemoryTrieCursorFactory { /// Underlying trie cursor factory. cursor_factory: CF, /// Reference to sorted trie updates. - trie_updates: &'a TrieUpdatesSorted, + trie_updates: T, } -impl<'a, CF> InMemoryTrieCursorFactory<'a, CF> { +impl InMemoryTrieCursorFactory { /// Create a new trie cursor factory. - pub const fn new(cursor_factory: CF, trie_updates: &'a TrieUpdatesSorted) -> Self { + pub const fn new(cursor_factory: CF, trie_updates: T) -> Self { Self { cursor_factory, trie_updates } } } -impl<'a, CF: TrieCursorFactory> TrieCursorFactory for InMemoryTrieCursorFactory<'a, CF> { - type AccountTrieCursor = InMemoryAccountTrieCursor<'a, CF::AccountTrieCursor>; - type StorageTrieCursor = InMemoryStorageTrieCursor<'a, CF::StorageTrieCursor>; +impl<'overlay, CF, T> TrieCursorFactory for InMemoryTrieCursorFactory +where + CF: TrieCursorFactory + 'overlay, + T: AsRef, +{ + type AccountTrieCursor<'cursor> + = InMemoryTrieCursor<'overlay, CF::AccountTrieCursor<'cursor>> + where + Self: 'cursor; - fn account_trie_cursor(&self) -> Result { + type StorageTrieCursor<'cursor> + = InMemoryTrieCursor<'overlay, CF::StorageTrieCursor<'cursor>> + where + Self: 'cursor; + + fn account_trie_cursor(&self) -> Result, DatabaseError> { let cursor = self.cursor_factory.account_trie_cursor()?; - Ok(InMemoryAccountTrieCursor::new(cursor, self.trie_updates)) + Ok(InMemoryTrieCursor::new(Some(cursor), self.trie_updates.as_ref().account_nodes_ref())) } fn storage_trie_cursor( &self, hashed_address: B256, - ) -> Result { - let cursor = self.cursor_factory.storage_trie_cursor(hashed_address)?; - Ok(InMemoryStorageTrieCursor::new( - hashed_address, - cursor, - self.trie_updates.storage_tries.get(&hashed_address), - )) + ) -> Result, DatabaseError> { + // if the storage trie has no updates then we use this as the in-memory overlay. + static EMPTY_UPDATES: Vec<(Nibbles, Option)> = Vec::new(); + + let storage_trie_updates = + self.trie_updates.as_ref().storage_tries_ref().get(&hashed_address); + let (storage_nodes, cleared) = storage_trie_updates + .map(|u| (u.storage_nodes_ref(), u.is_deleted())) + .unwrap_or((&EMPTY_UPDATES, false)); + + let cursor = if cleared { + None + } else { + Some(self.cursor_factory.storage_trie_cursor(hashed_address)?) + }; + + Ok(InMemoryTrieCursor::new(cursor, storage_nodes)) } } -/// The cursor to iterate over account trie updates and corresponding database entries. +/// A cursor to iterate over trie updates and corresponding database entries. /// It will always give precedence to the data from the trie updates. #[derive(Debug)] -pub struct InMemoryAccountTrieCursor<'a, C> { - /// The underlying cursor. - cursor: C, +pub struct InMemoryTrieCursor<'a, C> { + /// The underlying cursor. If None then it is assumed there is no DB data. + cursor: Option, + /// Entry that `cursor` is currently pointing to. + cursor_entry: Option<(Nibbles, BranchNodeCompact)>, /// Forward-only in-memory cursor over storage trie nodes. - in_memory_cursor: ForwardInMemoryCursor<'a, Nibbles, BranchNodeCompact>, - /// Collection of removed trie nodes. - removed_nodes: &'a HashSet, - /// Last key returned by the cursor. + in_memory_cursor: ForwardInMemoryCursor<'a, Nibbles, Option>, + /// The key most recently returned from the Cursor. last_key: Option, + #[cfg(debug_assertions)] + /// Whether an initial seek was called. + seeked: bool, } -impl<'a, C: TrieCursor> InMemoryAccountTrieCursor<'a, C> { - /// Create new account trie cursor from underlying cursor and reference to - /// [`TrieUpdatesSorted`]. - pub fn new(cursor: C, trie_updates: &'a TrieUpdatesSorted) -> Self { - let in_memory_cursor = ForwardInMemoryCursor::new(&trie_updates.account_nodes); +impl<'a, C: TrieCursor> InMemoryTrieCursor<'a, C> { + /// Create new trie cursor which combines a DB cursor (None to assume empty DB) and a set of + /// in-memory trie nodes. + pub fn new( + cursor: Option, + trie_updates: &'a [(Nibbles, Option)], + ) -> Self { + let in_memory_cursor = ForwardInMemoryCursor::new(trie_updates); Self { cursor, + cursor_entry: None, in_memory_cursor, - removed_nodes: &trie_updates.removed_nodes, last_key: None, + #[cfg(debug_assertions)] + seeked: false, } } - fn seek_inner( - &mut self, - key: Nibbles, - exact: bool, - ) -> Result, DatabaseError> { - let in_memory = self.in_memory_cursor.seek(&key); - if in_memory.as_ref().is_some_and(|entry| entry.0 == key) { - return Ok(in_memory) - } + /// Asserts that the next entry to be returned from the cursor is not previous to the last entry + /// returned. + fn set_last_key(&mut self, next_entry: &Option<(Nibbles, BranchNodeCompact)>) { + let next_key = next_entry.as_ref().map(|e| e.0); + debug_assert!( + self.last_key.is_none_or(|last| next_key.is_none_or(|next| next >= last)), + "Cannot return entry {:?} previous to the last returned entry at {:?}", + next_key, + self.last_key, + ); + self.last_key = next_key; + } - // Reposition the cursor to the first greater or equal node that wasn't removed. - let mut db_entry = self.cursor.seek(key.clone())?; - while db_entry.as_ref().is_some_and(|entry| self.removed_nodes.contains(&entry.0)) { - db_entry = self.cursor.next()?; + /// Seeks the `cursor_entry` field of the struct using the cursor. + fn cursor_seek(&mut self, key: Nibbles) -> Result<(), DatabaseError> { + if let Some(entry) = self.cursor_entry.as_ref() && + entry.0 >= key + { + // If already seeked to the given key then don't do anything. Also if we're seeked past + // the given key then don't anything, because `TrieCursor` is specifically a + // forward-only cursor. + } else { + self.cursor_entry = self.cursor.as_mut().map(|c| c.seek(key)).transpose()?.flatten(); } - // Compare two entries and return the lowest. - // If seek is exact, filter the entry for exact key match. - Ok(compare_trie_node_entries(in_memory, db_entry) - .filter(|(nibbles, _)| !exact || nibbles == &key)) + Ok(()) } - fn next_inner( - &mut self, - last: Nibbles, - ) -> Result, DatabaseError> { - let in_memory = self.in_memory_cursor.first_after(&last); - - // Reposition the cursor to the first greater or equal node that wasn't removed. - let mut db_entry = self.cursor.seek(last.clone())?; - while db_entry - .as_ref() - .is_some_and(|entry| entry.0 < last || self.removed_nodes.contains(&entry.0)) + /// Seeks the `cursor_entry` field of the struct to the subsequent entry using the cursor. + fn cursor_next(&mut self) -> Result<(), DatabaseError> { + #[cfg(debug_assertions)] { - db_entry = self.cursor.next()?; + debug_assert!(self.seeked); + } + + // If the previous entry is `None`, and we've done a seek previously, then the cursor is + // exhausted and we shouldn't call `next` again. + if self.cursor_entry.is_some() { + self.cursor_entry = self.cursor.as_mut().map(|c| c.next()).transpose()?.flatten(); } - // Compare two entries and return the lowest. - Ok(compare_trie_node_entries(in_memory, db_entry)) + Ok(()) + } + + /// Compares the current in-memory entry with the current entry of the cursor, and applies the + /// in-memory entry to the cursor entry as an overlay. + // + /// This may consume and move forward the current entries when the overlay indicates a removed + /// node. + fn choose_next_entry(&mut self) -> Result, DatabaseError> { + loop { + match (self.in_memory_cursor.current().cloned(), &self.cursor_entry) { + (Some((mem_key, None)), _) + if self.cursor_entry.as_ref().is_none_or(|(db_key, _)| &mem_key < db_key) => + { + // If overlay has a removed node but DB cursor is exhausted or ahead of the + // in-memory cursor then move ahead in-memory, as there might be further + // non-removed overlay nodes. + self.in_memory_cursor.first_after(&mem_key); + } + (Some((mem_key, None)), Some((db_key, _))) if &mem_key == db_key => { + // If overlay has a removed node which is returned from DB then move both + // cursors ahead to the next key. + self.in_memory_cursor.first_after(&mem_key); + self.cursor_next()?; + } + (Some((mem_key, Some(node))), _) + if self.cursor_entry.as_ref().is_none_or(|(db_key, _)| &mem_key <= db_key) => + { + // If overlay returns a node prior to the DB's node, or the DB is exhausted, + // then we return the overlay's node. + return Ok(Some((mem_key, node))) + } + // All other cases: + // - mem_key > db_key + // - overlay is exhausted + // Return the db_entry. If DB is also exhausted then this returns None. + _ => return Ok(self.cursor_entry.clone()), + } + } } } -impl TrieCursor for InMemoryAccountTrieCursor<'_, C> { +impl TrieCursor for InMemoryTrieCursor<'_, C> { fn seek_exact( &mut self, key: Nibbles, ) -> Result, DatabaseError> { - let entry = self.seek_inner(key, true)?; - self.last_key = entry.as_ref().map(|(nibbles, _)| nibbles.clone()); + self.cursor_seek(key)?; + let mem_entry = self.in_memory_cursor.seek(&key); + + #[cfg(debug_assertions)] + { + self.seeked = true; + } + + let entry = match (mem_entry, &self.cursor_entry) { + (Some((mem_key, entry_inner)), _) if mem_key == key => { + entry_inner.map(|node| (key, node)) + } + (_, Some((db_key, node))) if db_key == &key => Some((key, node.clone())), + _ => None, + }; + + self.set_last_key(&entry); Ok(entry) } @@ -128,179 +210,605 @@ impl TrieCursor for InMemoryAccountTrieCursor<'_, C> { &mut self, key: Nibbles, ) -> Result, DatabaseError> { - let entry = self.seek_inner(key, false)?; - self.last_key = entry.as_ref().map(|(nibbles, _)| nibbles.clone()); + self.cursor_seek(key)?; + self.in_memory_cursor.seek(&key); + + #[cfg(debug_assertions)] + { + self.seeked = true; + } + + let entry = self.choose_next_entry()?; + self.set_last_key(&entry); Ok(entry) } fn next(&mut self) -> Result, DatabaseError> { - let next = match &self.last_key { - Some(last) => { - let entry = self.next_inner(last.clone())?; - self.last_key = entry.as_ref().map(|entry| entry.0.clone()); - entry - } - // no previous entry was found - None => None, + #[cfg(debug_assertions)] + { + debug_assert!(self.seeked, "Cursor must be seek'd before next is called"); + } + + // A `last_key` of `None` indicates that the cursor is exhausted. + let Some(last_key) = self.last_key else { + return Ok(None); }; - Ok(next) + + // If either cursor is currently pointing to the last entry which was returned then consume + // that entry so that `choose_next_entry` is looking at the subsequent one. + if let Some((key, _)) = self.in_memory_cursor.current() && + key == &last_key + { + self.in_memory_cursor.first_after(&last_key); + } + + if let Some((key, _)) = &self.cursor_entry && + key == &last_key + { + self.cursor_next()?; + } + + let entry = self.choose_next_entry()?; + self.set_last_key(&entry); + Ok(entry) } fn current(&mut self) -> Result, DatabaseError> { match &self.last_key { - Some(key) => Ok(Some(key.clone())), - None => self.cursor.current(), + Some(key) => Ok(Some(*key)), + None => Ok(self.cursor.as_mut().map(|c| c.current()).transpose()?.flatten()), } } } -/// The cursor to iterate over storage trie updates and corresponding database entries. -/// It will always give precedence to the data from the trie updates. -#[derive(Debug)] -#[expect(dead_code)] -pub struct InMemoryStorageTrieCursor<'a, C> { - /// The hashed address of the account that trie belongs to. - hashed_address: B256, - /// The underlying cursor. - cursor: C, - /// Forward-only in-memory cursor over storage trie nodes. - in_memory_cursor: Option>, - /// Reference to the set of removed storage node keys. - removed_nodes: Option<&'a HashSet>, - /// The flag indicating whether the storage trie was cleared. - storage_trie_cleared: bool, - /// Last key returned by the cursor. - last_key: Option, -} +#[cfg(test)] +mod tests { + use super::*; + use crate::trie_cursor::mock::MockTrieCursor; + use parking_lot::Mutex; + use std::{collections::BTreeMap, sync::Arc}; -impl<'a, C> InMemoryStorageTrieCursor<'a, C> { - /// Create new storage trie cursor from underlying cursor and reference to - /// [`StorageTrieUpdatesSorted`]. - pub fn new( - hashed_address: B256, - cursor: C, - updates: Option<&'a StorageTrieUpdatesSorted>, - ) -> Self { - let in_memory_cursor = updates.map(|u| ForwardInMemoryCursor::new(&u.storage_nodes)); - let removed_nodes = updates.map(|u| &u.removed_nodes); - let storage_trie_cleared = updates.is_some_and(|u| u.is_deleted); - Self { - hashed_address, - cursor, - in_memory_cursor, - removed_nodes, - storage_trie_cleared, - last_key: None, - } + #[derive(Debug)] + struct InMemoryTrieCursorTestCase { + db_nodes: Vec<(Nibbles, BranchNodeCompact)>, + in_memory_nodes: Vec<(Nibbles, Option)>, + expected_results: Vec<(Nibbles, BranchNodeCompact)>, } -} -impl InMemoryStorageTrieCursor<'_, C> { - fn seek_inner( - &mut self, - key: Nibbles, - exact: bool, - ) -> Result, DatabaseError> { - let in_memory = self.in_memory_cursor.as_mut().and_then(|c| c.seek(&key)); - if self.storage_trie_cleared || in_memory.as_ref().is_some_and(|entry| entry.0 == key) { - return Ok(in_memory.filter(|(nibbles, _)| !exact || nibbles == &key)) - } + fn execute_test(test_case: InMemoryTrieCursorTestCase) { + let db_nodes_map: BTreeMap = + test_case.db_nodes.into_iter().collect(); + let db_nodes_arc = Arc::new(db_nodes_map); + let visited_keys = Arc::new(Mutex::new(Vec::new())); + let mock_cursor = MockTrieCursor::new(db_nodes_arc, visited_keys); - // Reposition the cursor to the first greater or equal node that wasn't removed. - let mut db_entry = self.cursor.seek(key.clone())?; - while db_entry - .as_ref() - .is_some_and(|entry| self.removed_nodes.as_ref().is_some_and(|r| r.contains(&entry.0))) + let mut cursor = InMemoryTrieCursor::new(Some(mock_cursor), &test_case.in_memory_nodes); + + let mut results = Vec::new(); + + if let Some(first_expected) = test_case.expected_results.first() && + let Ok(Some(entry)) = cursor.seek(first_expected.0) { - db_entry = self.cursor.next()?; + results.push(entry); } - // Compare two entries and return the lowest. - // If seek is exact, filter the entry for exact key match. - Ok(compare_trie_node_entries(in_memory, db_entry) - .filter(|(nibbles, _)| !exact || nibbles == &key)) + if !test_case.expected_results.is_empty() { + while let Ok(Some(entry)) = cursor.next() { + results.push(entry); + } + } + + assert_eq!( + results, test_case.expected_results, + "Results mismatch.\nGot: {:?}\nExpected: {:?}", + results, test_case.expected_results + ); } - fn next_inner( - &mut self, - last: Nibbles, - ) -> Result, DatabaseError> { - let in_memory = self.in_memory_cursor.as_mut().and_then(|c| c.first_after(&last)); - if self.storage_trie_cleared { - return Ok(in_memory) - } + #[test] + fn test_empty_db_and_memory() { + let test_case = InMemoryTrieCursorTestCase { + db_nodes: vec![], + in_memory_nodes: vec![], + expected_results: vec![], + }; + execute_test(test_case); + } - // Reposition the cursor to the first greater or equal node that wasn't removed. - let mut db_entry = self.cursor.seek(last.clone())?; - while db_entry.as_ref().is_some_and(|entry| { - entry.0 < last || self.removed_nodes.as_ref().is_some_and(|r| r.contains(&entry.0)) - }) { - db_entry = self.cursor.next()?; - } + #[test] + fn test_only_db_nodes() { + let db_nodes = vec![ + (Nibbles::from_nibbles([0x1]), BranchNodeCompact::new(0b0011, 0b0001, 0, vec![], None)), + (Nibbles::from_nibbles([0x2]), BranchNodeCompact::new(0b0011, 0b0010, 0, vec![], None)), + (Nibbles::from_nibbles([0x3]), BranchNodeCompact::new(0b0011, 0b0011, 0, vec![], None)), + ]; - // Compare two entries and return the lowest. - Ok(compare_trie_node_entries(in_memory, db_entry)) + let test_case = InMemoryTrieCursorTestCase { + db_nodes: db_nodes.clone(), + in_memory_nodes: vec![], + expected_results: db_nodes, + }; + execute_test(test_case); } -} -impl TrieCursor for InMemoryStorageTrieCursor<'_, C> { - fn seek_exact( - &mut self, - key: Nibbles, - ) -> Result, DatabaseError> { - let entry = self.seek_inner(key, true)?; - self.last_key = entry.as_ref().map(|(nibbles, _)| nibbles.clone()); - Ok(entry) + #[test] + fn test_only_in_memory_nodes() { + let in_memory_nodes = vec![ + ( + Nibbles::from_nibbles([0x1]), + Some(BranchNodeCompact::new(0b0011, 0b0001, 0, vec![], None)), + ), + ( + Nibbles::from_nibbles([0x2]), + Some(BranchNodeCompact::new(0b0011, 0b0010, 0, vec![], None)), + ), + ( + Nibbles::from_nibbles([0x3]), + Some(BranchNodeCompact::new(0b0011, 0b0011, 0, vec![], None)), + ), + ]; + + let expected_results: Vec<(Nibbles, BranchNodeCompact)> = in_memory_nodes + .iter() + .filter_map(|(k, v)| v.as_ref().map(|node| (*k, node.clone()))) + .collect(); + + let test_case = + InMemoryTrieCursorTestCase { db_nodes: vec![], in_memory_nodes, expected_results }; + execute_test(test_case); } - fn seek( - &mut self, - key: Nibbles, - ) -> Result, DatabaseError> { - let entry = self.seek_inner(key, false)?; - self.last_key = entry.as_ref().map(|(nibbles, _)| nibbles.clone()); - Ok(entry) + #[test] + fn test_in_memory_overwrites_db() { + let db_nodes = vec![ + (Nibbles::from_nibbles([0x1]), BranchNodeCompact::new(0b0011, 0b0001, 0, vec![], None)), + (Nibbles::from_nibbles([0x2]), BranchNodeCompact::new(0b0011, 0b0010, 0, vec![], None)), + ]; + + let in_memory_nodes = vec![ + ( + Nibbles::from_nibbles([0x1]), + Some(BranchNodeCompact::new(0b1111, 0b1111, 0, vec![], None)), + ), + ( + Nibbles::from_nibbles([0x3]), + Some(BranchNodeCompact::new(0b0011, 0b0011, 0, vec![], None)), + ), + ]; + + let expected_results = vec![ + (Nibbles::from_nibbles([0x1]), BranchNodeCompact::new(0b1111, 0b1111, 0, vec![], None)), + (Nibbles::from_nibbles([0x2]), BranchNodeCompact::new(0b0011, 0b0010, 0, vec![], None)), + (Nibbles::from_nibbles([0x3]), BranchNodeCompact::new(0b0011, 0b0011, 0, vec![], None)), + ]; + + let test_case = InMemoryTrieCursorTestCase { db_nodes, in_memory_nodes, expected_results }; + execute_test(test_case); } - fn next(&mut self) -> Result, DatabaseError> { - let next = match &self.last_key { - Some(last) => { - let entry = self.next_inner(last.clone())?; - self.last_key = entry.as_ref().map(|entry| entry.0.clone()); - entry - } - // no previous entry was found - None => None, - }; - Ok(next) + #[test] + fn test_in_memory_deletes_db_nodes() { + let db_nodes = vec![ + (Nibbles::from_nibbles([0x1]), BranchNodeCompact::new(0b0011, 0b0001, 0, vec![], None)), + (Nibbles::from_nibbles([0x2]), BranchNodeCompact::new(0b0011, 0b0010, 0, vec![], None)), + (Nibbles::from_nibbles([0x3]), BranchNodeCompact::new(0b0011, 0b0011, 0, vec![], None)), + ]; + + let in_memory_nodes = vec![(Nibbles::from_nibbles([0x2]), None)]; + + let expected_results = vec![ + (Nibbles::from_nibbles([0x1]), BranchNodeCompact::new(0b0011, 0b0001, 0, vec![], None)), + (Nibbles::from_nibbles([0x3]), BranchNodeCompact::new(0b0011, 0b0011, 0, vec![], None)), + ]; + + let test_case = InMemoryTrieCursorTestCase { db_nodes, in_memory_nodes, expected_results }; + execute_test(test_case); } - fn current(&mut self) -> Result, DatabaseError> { - match &self.last_key { - Some(key) => Ok(Some(key.clone())), - None => self.cursor.current(), - } + #[test] + fn test_complex_interleaving() { + let db_nodes = vec![ + (Nibbles::from_nibbles([0x1]), BranchNodeCompact::new(0b0001, 0b0001, 0, vec![], None)), + (Nibbles::from_nibbles([0x3]), BranchNodeCompact::new(0b0011, 0b0011, 0, vec![], None)), + (Nibbles::from_nibbles([0x5]), BranchNodeCompact::new(0b0101, 0b0101, 0, vec![], None)), + (Nibbles::from_nibbles([0x7]), BranchNodeCompact::new(0b0111, 0b0111, 0, vec![], None)), + ]; + + let in_memory_nodes = vec![ + ( + Nibbles::from_nibbles([0x2]), + Some(BranchNodeCompact::new(0b0010, 0b0010, 0, vec![], None)), + ), + (Nibbles::from_nibbles([0x3]), None), + ( + Nibbles::from_nibbles([0x4]), + Some(BranchNodeCompact::new(0b0100, 0b0100, 0, vec![], None)), + ), + ( + Nibbles::from_nibbles([0x6]), + Some(BranchNodeCompact::new(0b0110, 0b0110, 0, vec![], None)), + ), + (Nibbles::from_nibbles([0x7]), None), + ( + Nibbles::from_nibbles([0x8]), + Some(BranchNodeCompact::new(0b1000, 0b1000, 0, vec![], None)), + ), + ]; + + let expected_results = vec![ + (Nibbles::from_nibbles([0x1]), BranchNodeCompact::new(0b0001, 0b0001, 0, vec![], None)), + (Nibbles::from_nibbles([0x2]), BranchNodeCompact::new(0b0010, 0b0010, 0, vec![], None)), + (Nibbles::from_nibbles([0x4]), BranchNodeCompact::new(0b0100, 0b0100, 0, vec![], None)), + (Nibbles::from_nibbles([0x5]), BranchNodeCompact::new(0b0101, 0b0101, 0, vec![], None)), + (Nibbles::from_nibbles([0x6]), BranchNodeCompact::new(0b0110, 0b0110, 0, vec![], None)), + (Nibbles::from_nibbles([0x8]), BranchNodeCompact::new(0b1000, 0b1000, 0, vec![], None)), + ]; + + let test_case = InMemoryTrieCursorTestCase { db_nodes, in_memory_nodes, expected_results }; + execute_test(test_case); } -} -/// Return the node with the lowest nibbles. -/// -/// Given the next in-memory and database entries, return the smallest of the two. -/// If the node keys are the same, the in-memory entry is given precedence. -fn compare_trie_node_entries( - mut in_memory_item: Option<(Nibbles, BranchNodeCompact)>, - mut db_item: Option<(Nibbles, BranchNodeCompact)>, -) -> Option<(Nibbles, BranchNodeCompact)> { - if let Some((in_memory_entry, db_entry)) = in_memory_item.as_ref().zip(db_item.as_ref()) { - // If both are not empty, return the smallest of the two - // In-memory is given precedence if keys are equal - if in_memory_entry.0 <= db_entry.0 { - in_memory_item.take() - } else { - db_item.take() + #[test] + fn test_seek_exact() { + let db_nodes = vec![ + (Nibbles::from_nibbles([0x1]), BranchNodeCompact::new(0b0001, 0b0001, 0, vec![], None)), + (Nibbles::from_nibbles([0x3]), BranchNodeCompact::new(0b0011, 0b0011, 0, vec![], None)), + ]; + + let in_memory_nodes = vec![( + Nibbles::from_nibbles([0x2]), + Some(BranchNodeCompact::new(0b0010, 0b0010, 0, vec![], None)), + )]; + + let db_nodes_map: BTreeMap = db_nodes.into_iter().collect(); + let db_nodes_arc = Arc::new(db_nodes_map); + let visited_keys = Arc::new(Mutex::new(Vec::new())); + let mock_cursor = MockTrieCursor::new(db_nodes_arc, visited_keys); + + let mut cursor = InMemoryTrieCursor::new(Some(mock_cursor), &in_memory_nodes); + + let result = cursor.seek_exact(Nibbles::from_nibbles([0x2])).unwrap(); + assert_eq!( + result, + Some(( + Nibbles::from_nibbles([0x2]), + BranchNodeCompact::new(0b0010, 0b0010, 0, vec![], None) + )) + ); + + let result = cursor.seek_exact(Nibbles::from_nibbles([0x3])).unwrap(); + assert_eq!( + result, + Some(( + Nibbles::from_nibbles([0x3]), + BranchNodeCompact::new(0b0011, 0b0011, 0, vec![], None) + )) + ); + + let result = cursor.seek_exact(Nibbles::from_nibbles([0x4])).unwrap(); + assert_eq!(result, None); + } + + #[test] + fn test_multiple_consecutive_deletes() { + let db_nodes: Vec<(Nibbles, BranchNodeCompact)> = (1..=10) + .map(|i| { + ( + Nibbles::from_nibbles([i]), + BranchNodeCompact::new(i as u16, i as u16, 0, vec![], None), + ) + }) + .collect(); + + let in_memory_nodes = vec![ + (Nibbles::from_nibbles([0x3]), None), + (Nibbles::from_nibbles([0x4]), None), + (Nibbles::from_nibbles([0x5]), None), + (Nibbles::from_nibbles([0x6]), None), + ]; + + let expected_results = vec![ + (Nibbles::from_nibbles([0x1]), BranchNodeCompact::new(1, 1, 0, vec![], None)), + (Nibbles::from_nibbles([0x2]), BranchNodeCompact::new(2, 2, 0, vec![], None)), + (Nibbles::from_nibbles([0x7]), BranchNodeCompact::new(7, 7, 0, vec![], None)), + (Nibbles::from_nibbles([0x8]), BranchNodeCompact::new(8, 8, 0, vec![], None)), + (Nibbles::from_nibbles([0x9]), BranchNodeCompact::new(9, 9, 0, vec![], None)), + (Nibbles::from_nibbles([0xa]), BranchNodeCompact::new(10, 10, 0, vec![], None)), + ]; + + let test_case = InMemoryTrieCursorTestCase { db_nodes, in_memory_nodes, expected_results }; + execute_test(test_case); + } + + #[test] + fn test_empty_db_with_in_memory_deletes() { + let in_memory_nodes = vec![ + (Nibbles::from_nibbles([0x1]), None), + ( + Nibbles::from_nibbles([0x2]), + Some(BranchNodeCompact::new(0b0010, 0b0010, 0, vec![], None)), + ), + (Nibbles::from_nibbles([0x3]), None), + ]; + + let expected_results = vec![( + Nibbles::from_nibbles([0x2]), + BranchNodeCompact::new(0b0010, 0b0010, 0, vec![], None), + )]; + + let test_case = + InMemoryTrieCursorTestCase { db_nodes: vec![], in_memory_nodes, expected_results }; + execute_test(test_case); + } + + #[test] + fn test_current_key_tracking() { + let db_nodes = vec![( + Nibbles::from_nibbles([0x2]), + BranchNodeCompact::new(0b0010, 0b0010, 0, vec![], None), + )]; + + let in_memory_nodes = vec![ + ( + Nibbles::from_nibbles([0x1]), + Some(BranchNodeCompact::new(0b0001, 0b0001, 0, vec![], None)), + ), + ( + Nibbles::from_nibbles([0x3]), + Some(BranchNodeCompact::new(0b0011, 0b0011, 0, vec![], None)), + ), + ]; + + let db_nodes_map: BTreeMap = db_nodes.into_iter().collect(); + let db_nodes_arc = Arc::new(db_nodes_map); + let visited_keys = Arc::new(Mutex::new(Vec::new())); + let mock_cursor = MockTrieCursor::new(db_nodes_arc, visited_keys); + + let mut cursor = InMemoryTrieCursor::new(Some(mock_cursor), &in_memory_nodes); + + assert_eq!(cursor.current().unwrap(), None); + + cursor.seek(Nibbles::from_nibbles([0x1])).unwrap(); + assert_eq!(cursor.current().unwrap(), Some(Nibbles::from_nibbles([0x1]))); + + cursor.next().unwrap(); + assert_eq!(cursor.current().unwrap(), Some(Nibbles::from_nibbles([0x2]))); + + cursor.next().unwrap(); + assert_eq!(cursor.current().unwrap(), Some(Nibbles::from_nibbles([0x3]))); + } + + mod proptest_tests { + use super::*; + use itertools::Itertools; + use proptest::prelude::*; + + /// Merge `db_nodes` with `in_memory_nodes`, applying the in-memory overlay. + /// This properly handles deletions (None values in `in_memory_nodes`). + fn merge_with_overlay( + db_nodes: Vec<(Nibbles, BranchNodeCompact)>, + in_memory_nodes: Vec<(Nibbles, Option)>, + ) -> Vec<(Nibbles, BranchNodeCompact)> { + db_nodes + .into_iter() + .merge_join_by(in_memory_nodes, |db_entry, mem_entry| db_entry.0.cmp(&mem_entry.0)) + .filter_map(|entry| match entry { + // Only in db: keep it + itertools::EitherOrBoth::Left((key, node)) => Some((key, node)), + // Only in memory: keep if not a deletion + itertools::EitherOrBoth::Right((key, node_opt)) => { + node_opt.map(|node| (key, node)) + } + // In both: memory takes precedence (keep if not a deletion) + itertools::EitherOrBoth::Both(_, (key, node_opt)) => { + node_opt.map(|node| (key, node)) + } + }) + .collect() + } + + /// Generate a strategy for a `BranchNodeCompact` with simplified parameters. + /// The constraints are: + /// - `tree_mask` must be a subset of `state_mask` + /// - `hash_mask` must be a subset of `state_mask` + /// - `hash_mask.count_ones()` must equal `hashes.len()` + /// + /// To keep it simple, we use an empty hashes vec and `hash_mask` of 0. + fn branch_node_strategy() -> impl Strategy { + any::() + .prop_flat_map(|state_mask| { + let tree_mask_strategy = any::().prop_map(move |tree| tree & state_mask); + (Just(state_mask), tree_mask_strategy) + }) + .prop_map(|(state_mask, tree_mask)| { + BranchNodeCompact::new(state_mask, tree_mask, 0, vec![], None) + }) + } + + /// Generate a sorted vector of (Nibbles, `BranchNodeCompact`) entries + fn sorted_db_nodes_strategy() -> impl Strategy> { + prop::collection::vec( + (prop::collection::vec(any::(), 0..3), branch_node_strategy()), + 0..20, + ) + .prop_map(|entries| { + // Convert Vec to Nibbles and sort + let mut result: Vec<(Nibbles, BranchNodeCompact)> = entries + .into_iter() + .map(|(bytes, node)| (Nibbles::from_nibbles_unchecked(bytes), node)) + .collect(); + result.sort_by(|a, b| a.0.cmp(&b.0)); + result.dedup_by(|a, b| a.0 == b.0); + result + }) + } + + /// Generate a sorted vector of (Nibbles, Option) entries + fn sorted_in_memory_nodes_strategy( + ) -> impl Strategy)>> { + prop::collection::vec( + ( + prop::collection::vec(any::(), 0..3), + prop::option::of(branch_node_strategy()), + ), + 0..20, + ) + .prop_map(|entries| { + // Convert Vec to Nibbles and sort + let mut result: Vec<(Nibbles, Option)> = entries + .into_iter() + .map(|(bytes, node)| (Nibbles::from_nibbles_unchecked(bytes), node)) + .collect(); + result.sort_by(|a, b| a.0.cmp(&b.0)); + result.dedup_by(|a, b| a.0 == b.0); + result + }) + } + + proptest! { + #![proptest_config(ProptestConfig::with_cases(1000))] + + #[test] + fn proptest_in_memory_trie_cursor( + db_nodes in sorted_db_nodes_strategy(), + in_memory_nodes in sorted_in_memory_nodes_strategy(), + op_choices in prop::collection::vec(any::(), 10..500), + ) { + reth_tracing::init_test_tracing(); + use tracing::debug; + + debug!("Starting proptest!"); + + // Create the expected results by merging the two sorted vectors, + // properly handling deletions (None values in in_memory_nodes) + let expected_combined = merge_with_overlay(db_nodes.clone(), in_memory_nodes.clone()); + + // Collect all keys for operation generation + let all_keys: Vec = expected_combined.iter().map(|(k, _)| *k).collect(); + + // Create a control cursor using the combined result with a mock cursor + let control_db_map: BTreeMap = + expected_combined.into_iter().collect(); + let control_db_arc = Arc::new(control_db_map); + let control_visited_keys = Arc::new(Mutex::new(Vec::new())); + let mut control_cursor = MockTrieCursor::new(control_db_arc, control_visited_keys); + + // Create the InMemoryTrieCursor being tested + let db_nodes_map: BTreeMap = + db_nodes.into_iter().collect(); + let db_nodes_arc = Arc::new(db_nodes_map); + let visited_keys = Arc::new(Mutex::new(Vec::new())); + let mock_cursor = MockTrieCursor::new(db_nodes_arc, visited_keys); + let mut test_cursor = InMemoryTrieCursor::new(Some(mock_cursor), &in_memory_nodes); + + // Test: seek to the beginning first + let control_first = control_cursor.seek(Nibbles::default()).unwrap(); + let test_first = test_cursor.seek(Nibbles::default()).unwrap(); + debug!( + control=?control_first.as_ref().map(|(k, _)| k), + test=?test_first.as_ref().map(|(k, _)| k), + "Initial seek returned", + ); + assert_eq!(control_first, test_first, "Initial seek mismatch"); + + // If both cursors returned None, nothing to test + if control_first.is_none() && test_first.is_none() { + return Ok(()); + } + + // Track the last key returned from the cursor + let mut last_returned_key = control_first.as_ref().map(|(k, _)| *k); + + // Execute a sequence of random operations + for choice in op_choices { + let op_type = choice % 3; + + match op_type { + 0 => { + // Next operation + let control_result = control_cursor.next().unwrap(); + let test_result = test_cursor.next().unwrap(); + debug!( + control=?control_result.as_ref().map(|(k, _)| k), + test=?test_result.as_ref().map(|(k, _)| k), + "Next returned", + ); + assert_eq!(control_result, test_result, "Next operation mismatch"); + + last_returned_key = control_result.as_ref().map(|(k, _)| *k); + + // Stop if both cursors are exhausted + if control_result.is_none() && test_result.is_none() { + break; + } + } + 1 => { + // Seek operation - choose a key >= last_returned_key + if all_keys.is_empty() { + continue; + } + + let valid_keys: Vec<_> = all_keys + .iter() + .filter(|k| last_returned_key.is_none_or(|last| **k >= last)) + .collect(); + + if valid_keys.is_empty() { + continue; + } + + let key = *valid_keys[(choice as usize / 3) % valid_keys.len()]; + + let control_result = control_cursor.seek(key).unwrap(); + let test_result = test_cursor.seek(key).unwrap(); + debug!( + control=?control_result.as_ref().map(|(k, _)| k), + test=?test_result.as_ref().map(|(k, _)| k), + ?key, + "Seek returned", + ); + assert_eq!(control_result, test_result, "Seek operation mismatch for key {:?}", key); + + last_returned_key = control_result.as_ref().map(|(k, _)| *k); + + // Stop if both cursors are exhausted + if control_result.is_none() && test_result.is_none() { + break; + } + } + _ => { + // SeekExact operation - choose a key >= last_returned_key + if all_keys.is_empty() { + continue; + } + + let valid_keys: Vec<_> = all_keys + .iter() + .filter(|k| last_returned_key.is_none_or(|last| **k >= last)) + .collect(); + + if valid_keys.is_empty() { + continue; + } + + let key = *valid_keys[(choice as usize / 3) % valid_keys.len()]; + + let control_result = control_cursor.seek_exact(key).unwrap(); + let test_result = test_cursor.seek_exact(key).unwrap(); + debug!( + control=?control_result.as_ref().map(|(k, _)| k), + test=?test_result.as_ref().map(|(k, _)| k), + ?key, + "SeekExact returned", + ); + assert_eq!(control_result, test_result, "SeekExact operation mismatch for key {:?}", key); + + // seek_exact updates the last_key internally but only if it found something + last_returned_key = control_result.as_ref().map(|(k, _)| *k); + } + } + } + } } - } else { - // Return either non-empty entry - db_item.or(in_memory_item) } } diff --git a/crates/trie/trie/src/trie_cursor/mock.rs b/crates/trie/trie/src/trie_cursor/mock.rs index 4c7d20defb0..313df0443e3 100644 --- a/crates/trie/trie/src/trie_cursor/mock.rs +++ b/crates/trie/trie/src/trie_cursor/mock.rs @@ -52,11 +52,17 @@ impl MockTrieCursorFactory { } impl TrieCursorFactory for MockTrieCursorFactory { - type AccountTrieCursor = MockTrieCursor; - type StorageTrieCursor = MockTrieCursor; + type AccountTrieCursor<'a> + = MockTrieCursor + where + Self: 'a; + type StorageTrieCursor<'a> + = MockTrieCursor + where + Self: 'a; /// Generates a mock account trie cursor. - fn account_trie_cursor(&self) -> Result { + fn account_trie_cursor(&self) -> Result, DatabaseError> { Ok(MockTrieCursor::new(self.account_trie_nodes.clone(), self.visited_account_keys.clone())) } @@ -64,7 +70,7 @@ impl TrieCursorFactory for MockTrieCursorFactory { fn storage_trie_cursor( &self, hashed_address: B256, - ) -> Result { + ) -> Result, DatabaseError> { Ok(MockTrieCursor::new( self.storage_tries .get(&hashed_address) @@ -93,7 +99,8 @@ pub struct MockTrieCursor { } impl MockTrieCursor { - fn new( + /// Creates a new mock trie cursor with the given trie nodes and key tracking. + pub fn new( trie_nodes: Arc>, visited_keys: Arc>>>, ) -> Self { @@ -102,41 +109,40 @@ impl MockTrieCursor { } impl TrieCursor for MockTrieCursor { - #[instrument(level = "trace", skip(self), ret)] + #[instrument(skip(self), ret(level = "trace"))] fn seek_exact( &mut self, key: Nibbles, ) -> Result, DatabaseError> { - let entry = self.trie_nodes.get(&key).cloned().map(|value| (key.clone(), value)); + let entry = self.trie_nodes.get(&key).cloned().map(|value| (key, value)); if let Some((key, _)) = &entry { - self.current_key = Some(key.clone()); + self.current_key = Some(*key); } self.visited_keys.lock().push(KeyVisit { visit_type: KeyVisitType::SeekExact(key), - visited_key: entry.as_ref().map(|(k, _)| k.clone()), + visited_key: entry.as_ref().map(|(k, _)| *k), }); Ok(entry) } - #[instrument(level = "trace", skip(self), ret)] + #[instrument(skip(self), ret(level = "trace"))] fn seek( &mut self, key: Nibbles, ) -> Result, DatabaseError> { // Find the first key that is greater than or equal to the given key. - let entry = - self.trie_nodes.iter().find_map(|(k, v)| (k >= &key).then(|| (k.clone(), v.clone()))); + let entry = self.trie_nodes.iter().find_map(|(k, v)| (k >= &key).then(|| (*k, v.clone()))); if let Some((key, _)) = &entry { - self.current_key = Some(key.clone()); + self.current_key = Some(*key); } self.visited_keys.lock().push(KeyVisit { visit_type: KeyVisitType::SeekNonExact(key), - visited_key: entry.as_ref().map(|(k, _)| k.clone()), + visited_key: entry.as_ref().map(|(k, _)| *k), }); Ok(entry) } - #[instrument(level = "trace", skip(self), ret)] + #[instrument(skip(self), ret(level = "trace"))] fn next(&mut self) -> Result, DatabaseError> { let mut iter = self.trie_nodes.iter(); // Jump to the first key that has a prefix of the current key if it's set, or to the first @@ -144,19 +150,19 @@ impl TrieCursor for MockTrieCursor { iter.find(|(k, _)| self.current_key.as_ref().is_none_or(|current| k.starts_with(current))) .expect("current key should exist in trie nodes"); // Get the next key-value pair. - let entry = iter.next().map(|(k, v)| (k.clone(), v.clone())); + let entry = iter.next().map(|(k, v)| (*k, v.clone())); if let Some((key, _)) = &entry { - self.current_key = Some(key.clone()); + self.current_key = Some(*key); } self.visited_keys.lock().push(KeyVisit { visit_type: KeyVisitType::Next, - visited_key: entry.as_ref().map(|(k, _)| k.clone()), + visited_key: entry.as_ref().map(|(k, _)| *k), }); Ok(entry) } - #[instrument(level = "trace", skip(self), ret)] + #[instrument(skip(self), ret(level = "trace"))] fn current(&mut self) -> Result, DatabaseError> { - Ok(self.current_key.clone()) + Ok(self.current_key) } } diff --git a/crates/trie/trie/src/trie_cursor/mod.rs b/crates/trie/trie/src/trie_cursor/mod.rs index 69f7d1caec2..05a6c09e948 100644 --- a/crates/trie/trie/src/trie_cursor/mod.rs +++ b/crates/trie/trie/src/trie_cursor/mod.rs @@ -11,33 +11,42 @@ pub mod subnode; /// Noop trie cursor implementations. pub mod noop; +/// Depth-first trie iterator. +pub mod depth_first; + /// Mock trie cursor implementations. #[cfg(test)] pub mod mock; -pub use self::{in_memory::*, subnode::CursorSubNode}; +pub use self::{depth_first::DepthFirstTrieIterator, in_memory::*, subnode::CursorSubNode}; /// Factory for creating trie cursors. #[auto_impl::auto_impl(&)] pub trait TrieCursorFactory { /// The account trie cursor type. - type AccountTrieCursor: TrieCursor; + type AccountTrieCursor<'a>: TrieCursor + where + Self: 'a; + /// The storage trie cursor type. - type StorageTrieCursor: TrieCursor; + type StorageTrieCursor<'a>: TrieCursor + where + Self: 'a; /// Create an account trie cursor. - fn account_trie_cursor(&self) -> Result; + fn account_trie_cursor(&self) -> Result, DatabaseError>; /// Create a storage tries cursor. fn storage_trie_cursor( &self, hashed_address: B256, - ) -> Result; + ) -> Result, DatabaseError>; } -/// A cursor for navigating a trie that works with both Tables and DupSort tables. -#[auto_impl::auto_impl(&mut, Box)] -pub trait TrieCursor: Send + Sync { +/// A cursor for traversing stored trie nodes. The cursor must iterate over keys in +/// lexicographical order. +#[auto_impl::auto_impl(&mut)] +pub trait TrieCursor { /// Move the cursor to the key and return if it is an exact match. fn seek_exact( &mut self, @@ -54,3 +63,48 @@ pub trait TrieCursor: Send + Sync { /// Get the current entry. fn current(&mut self) -> Result, DatabaseError>; } + +/// Iterator wrapper for `TrieCursor` types +#[derive(Debug)] +pub struct TrieCursorIter<'a, C> { + cursor: &'a mut C, + /// The initial value from seek, if any + initial: Option>, +} + +impl<'a, C> TrieCursorIter<'a, C> { + /// Create a new iterator from a mutable reference to a cursor. The Iterator will start from the + /// empty path. + pub fn new(cursor: &'a mut C) -> Self + where + C: TrieCursor, + { + let initial = cursor.seek(Nibbles::default()).transpose(); + Self { cursor, initial } + } +} + +impl<'a, C> From<&'a mut C> for TrieCursorIter<'a, C> +where + C: TrieCursor, +{ + fn from(cursor: &'a mut C) -> Self { + Self::new(cursor) + } +} + +impl<'a, C> Iterator for TrieCursorIter<'a, C> +where + C: TrieCursor, +{ + type Item = Result<(Nibbles, BranchNodeCompact), DatabaseError>; + + fn next(&mut self) -> Option { + // If we have an initial value from seek, return it first + if let Some(initial) = self.initial.take() { + return Some(initial); + } + + self.cursor.next().transpose() + } +} diff --git a/crates/trie/trie/src/trie_cursor/noop.rs b/crates/trie/trie/src/trie_cursor/noop.rs index de409c59fe1..a00a18e4f00 100644 --- a/crates/trie/trie/src/trie_cursor/noop.rs +++ b/crates/trie/trie/src/trie_cursor/noop.rs @@ -9,11 +9,18 @@ use reth_storage_errors::db::DatabaseError; pub struct NoopTrieCursorFactory; impl TrieCursorFactory for NoopTrieCursorFactory { - type AccountTrieCursor = NoopAccountTrieCursor; - type StorageTrieCursor = NoopStorageTrieCursor; + type AccountTrieCursor<'a> + = NoopAccountTrieCursor + where + Self: 'a; + + type StorageTrieCursor<'a> + = NoopStorageTrieCursor + where + Self: 'a; /// Generates a noop account trie cursor. - fn account_trie_cursor(&self) -> Result { + fn account_trie_cursor(&self) -> Result, DatabaseError> { Ok(NoopAccountTrieCursor::default()) } @@ -21,7 +28,7 @@ impl TrieCursorFactory for NoopTrieCursorFactory { fn storage_trie_cursor( &self, _hashed_address: B256, - ) -> Result { + ) -> Result, DatabaseError> { Ok(NoopStorageTrieCursor::default()) } } diff --git a/crates/trie/trie/src/trie_cursor/subnode.rs b/crates/trie/trie/src/trie_cursor/subnode.rs index 85fe98f47ec..9c9b5e03d7d 100644 --- a/crates/trie/trie/src/trie_cursor/subnode.rs +++ b/crates/trie/trie/src/trie_cursor/subnode.rs @@ -1,5 +1,6 @@ use crate::{BranchNodeCompact, Nibbles, StoredSubNode, CHILD_INDEX_RANGE}; use alloy_primitives::B256; +use alloy_trie::proof::AddedRemovedKeys; /// Cursor for iterating over a subtrie. #[derive(Clone)] @@ -74,7 +75,7 @@ impl CursorSubNode { node: Option, position: SubNodePosition, ) -> Self { - let mut full_key = key.clone(); + let mut full_key = key; if let Some(nibble) = position.as_child() { full_key.push(nibble); } @@ -87,6 +88,26 @@ impl CursorSubNode { &self.full_key } + /// Returns true if all of: + /// - Position is a child + /// - There is a branch node + /// - All children except the current are removed according to the [`AddedRemovedKeys`]. + pub fn full_key_is_only_nonremoved_child(&self, added_removed_keys: &AddedRemovedKeys) -> bool { + self.position.as_child().zip(self.node.as_ref()).is_some_and(|(nibble, node)| { + let removed_mask = added_removed_keys.get_removed_mask(&self.key); + let nonremoved_mask = !removed_mask & node.state_mask; + tracing::trace!( + target: "trie::walker", + key = ?self.key, + ?removed_mask, + ?nonremoved_mask, + ?nibble, + "Checking full_key_is_only_nonremoved_node", + ); + nonremoved_mask.count_ones() == 1 && nonremoved_mask.is_bit_set(nibble) + }) + } + /// Updates the full key by replacing or appending a child nibble based on the old subnode /// position. #[inline] @@ -158,7 +179,7 @@ impl CursorSubNode { /// /// Differs from [`Self::hash`] in that it returns `None` if the subnode is positioned at the /// child without a hash mask bit set. [`Self::hash`] panics in that case. - fn maybe_hash(&self) -> Option { + pub fn maybe_hash(&self) -> Option { self.node.as_ref().and_then(|node| match self.position { // Get the root hash for the parent branch node SubNodePosition::ParentBranch => node.root_hash, diff --git a/crates/trie/trie/src/verify.rs b/crates/trie/trie/src/verify.rs new file mode 100644 index 00000000000..4299a669165 --- /dev/null +++ b/crates/trie/trie/src/verify.rs @@ -0,0 +1,1011 @@ +use crate::{ + hashed_cursor::{HashedCursor, HashedCursorFactory}, + progress::{IntermediateStateRootState, StateRootProgress}, + trie::StateRoot, + trie_cursor::{ + depth_first::{self, DepthFirstTrieIterator}, + noop::NoopTrieCursorFactory, + TrieCursor, TrieCursorFactory, + }, + Nibbles, +}; +use alloy_primitives::B256; +use alloy_trie::BranchNodeCompact; +use reth_execution_errors::StateRootError; +use reth_storage_errors::db::DatabaseError; +use std::cmp::Ordering; +use tracing::trace; + +/// Used by [`StateRootBranchNodesIter`] to iterate over branch nodes in a state root. +#[derive(Debug)] +enum BranchNode { + Account(Nibbles, BranchNodeCompact), + Storage(B256, Nibbles, BranchNodeCompact), +} + +/// Iterates over branch nodes produced by a [`StateRoot`]. The `StateRoot` will only used the +/// hashed accounts/storages tables, meaning it is recomputing the trie from scratch without the use +/// of the trie tables. +/// +/// [`BranchNode`]s are iterated over such that: +/// * Account nodes and storage nodes may be interleaved. +/// * Storage nodes for the same account will be ordered by ascending path relative to each other. +/// * Account nodes will be ordered by ascending path relative to each other. +/// * All storage nodes for one account will finish before storage nodes for another account are +/// started. In other words, if the current storage account is not equal to the previous, the +/// previous has no more nodes. +#[derive(Debug)] +struct StateRootBranchNodesIter { + hashed_cursor_factory: H, + account_nodes: Vec<(Nibbles, BranchNodeCompact)>, + storage_tries: Vec<(B256, Vec<(Nibbles, BranchNodeCompact)>)>, + curr_storage: Option<(B256, Vec<(Nibbles, BranchNodeCompact)>)>, + intermediate_state: Option>, + complete: bool, +} + +impl StateRootBranchNodesIter { + fn new(hashed_cursor_factory: H) -> Self { + Self { + hashed_cursor_factory, + account_nodes: Default::default(), + storage_tries: Default::default(), + curr_storage: None, + intermediate_state: None, + complete: false, + } + } + + /// Sorts a Vec of updates such that it is ready to be yielded from the `next` method. We yield + /// by popping off of the account/storage vecs, so we sort them in reverse order. + /// + /// Depth-first sorting is used because this is the order that the `HashBuilder` computes + /// branch nodes internally, even if it produces them as `B256Map`s. + fn sort_updates(updates: &mut [(Nibbles, BranchNodeCompact)]) { + updates.sort_unstable_by(|a, b| depth_first::cmp(&b.0, &a.0)); + } +} + +impl Iterator for StateRootBranchNodesIter { + type Item = Result; + + fn next(&mut self) -> Option { + loop { + // If we already started iterating through a storage trie's updates, continue doing + // so. + if let Some((account, storage_updates)) = self.curr_storage.as_mut() && + let Some((path, node)) = storage_updates.pop() + { + let node = BranchNode::Storage(*account, path, node); + return Some(Ok(node)) + } + + // If there's not a storage trie already being iterated over then check if there's a + // storage trie we could start iterating over. + if let Some((account, storage_updates)) = self.storage_tries.pop() { + debug_assert!(!storage_updates.is_empty()); + + self.curr_storage = Some((account, storage_updates)); + continue; + } + + // `storage_updates` is empty, check if there are account updates. + if let Some((path, node)) = self.account_nodes.pop() { + return Some(Ok(BranchNode::Account(path, node))) + } + + // All data from any previous runs of the `StateRoot` has been produced, run the next + // partial computation, unless `StateRootProgress::Complete` has been returned in which + // case iteration is over. + if self.complete { + return None + } + + let state_root = + StateRoot::new(NoopTrieCursorFactory, self.hashed_cursor_factory.clone()) + .with_intermediate_state(self.intermediate_state.take().map(|s| *s)); + + let updates = match state_root.root_with_progress() { + Err(err) => return Some(Err(err)), + Ok(StateRootProgress::Complete(_, _, updates)) => { + self.complete = true; + updates + } + Ok(StateRootProgress::Progress(intermediate_state, _, updates)) => { + self.intermediate_state = Some(intermediate_state); + updates + } + }; + + // collect account updates and sort them in descending order, so that when we pop them + // off the Vec they are popped in ascending order. + self.account_nodes.extend(updates.account_nodes); + Self::sort_updates(&mut self.account_nodes); + + self.storage_tries = updates + .storage_tries + .into_iter() + .filter_map(|(account, t)| { + (!t.storage_nodes.is_empty()).then(|| { + let mut storage_nodes = t.storage_nodes.into_iter().collect::>(); + Self::sort_updates(&mut storage_nodes); + (account, storage_nodes) + }) + }) + .collect::>(); + + // `root_with_progress` will output storage updates ordered by their account hash. If + // `root_with_progress` only returns a partial result then it will pick up where + // it left off in the storage trie on the next run. + // + // By sorting by the account we ensure that we continue with the partially processed + // trie (the last of the previous run) first. We sort in reverse order because we pop + // off of this Vec. + self.storage_tries.sort_unstable_by(|a, b| b.0.cmp(&a.0)); + + // loop back to the top. + } + } +} + +/// Output describes an inconsistency found when comparing the hashed state tables +/// ([`HashedCursorFactory`]) with that of the trie tables ([`TrieCursorFactory`]). The hashed +/// tables are considered the source of truth; outputs are on the part of the trie tables. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Output { + /// An extra account node was found. + AccountExtra(Nibbles, BranchNodeCompact), + /// An extra storage node was found. + StorageExtra(B256, Nibbles, BranchNodeCompact), + /// An account node had the wrong value. + AccountWrong { + /// Path of the node + path: Nibbles, + /// The node's expected value. + expected: BranchNodeCompact, + /// The node's found value. + found: BranchNodeCompact, + }, + /// A storage node had the wrong value. + StorageWrong { + /// The account the storage trie belongs to. + account: B256, + /// Path of the node + path: Nibbles, + /// The node's expected value. + expected: BranchNodeCompact, + /// The node's found value. + found: BranchNodeCompact, + }, + /// An account node was missing. + AccountMissing(Nibbles, BranchNodeCompact), + /// A storage node was missing. + StorageMissing(B256, Nibbles, BranchNodeCompact), + /// Progress indicator with the last seen account path. + Progress(Nibbles), +} + +/// Verifies the contents of a trie table against some other data source which is able to produce +/// stored trie nodes. +#[derive(Debug)] +struct SingleVerifier { + account: Option, // None for accounts trie + trie_iter: I, + curr: Option<(Nibbles, BranchNodeCompact)>, +} + +impl SingleVerifier> { + fn new(account: Option, trie_cursor: C) -> Result { + let mut trie_iter = DepthFirstTrieIterator::new(trie_cursor); + let curr = trie_iter.next().transpose()?; + Ok(Self { account, trie_iter, curr }) + } + + const fn output_extra(&self, path: Nibbles, node: BranchNodeCompact) -> Output { + if let Some(account) = self.account { + Output::StorageExtra(account, path, node) + } else { + Output::AccountExtra(path, node) + } + } + + const fn output_wrong( + &self, + path: Nibbles, + expected: BranchNodeCompact, + found: BranchNodeCompact, + ) -> Output { + if let Some(account) = self.account { + Output::StorageWrong { account, path, expected, found } + } else { + Output::AccountWrong { path, expected, found } + } + } + + const fn output_missing(&self, path: Nibbles, node: BranchNodeCompact) -> Output { + if let Some(account) = self.account { + Output::StorageMissing(account, path, node) + } else { + Output::AccountMissing(path, node) + } + } + + /// Called with the next path and node in the canonical sequence of stored trie nodes. Will + /// append to the given `outputs` Vec if walking the trie cursor produces data + /// inconsistent with that given. + /// + /// `next` must be called with paths in depth-first order. + fn next( + &mut self, + outputs: &mut Vec, + path: Nibbles, + node: BranchNodeCompact, + ) -> Result<(), DatabaseError> { + loop { + // `curr` is None only if the end of the iterator has been reached. Any further nodes + // found must be considered missing. + if self.curr.is_none() { + outputs.push(self.output_missing(path, node)); + return Ok(()) + } + + let (curr_path, curr_node) = self.curr.as_ref().expect("not None"); + trace!(target: "trie::verify", account=?self.account, ?curr_path, ?path, "Current cursor node"); + + // Use depth-first ordering for comparison + match depth_first::cmp(&path, curr_path) { + Ordering::Less => { + // If the given path comes before the cursor's current path in depth-first + // order, then the given path was not produced by the cursor. + outputs.push(self.output_missing(path, node)); + return Ok(()) + } + Ordering::Equal => { + // If the current path matches the given one (happy path) but the nodes + // aren't equal then we produce a wrong node. Either way we want to move the + // iterator forward. + if *curr_node != node { + outputs.push(self.output_wrong(path, node, curr_node.clone())) + } + self.curr = self.trie_iter.next().transpose()?; + return Ok(()) + } + Ordering::Greater => { + // If the given path comes after the current path in depth-first order, + // it means the cursor's path was not found by the caller (otherwise it would + // have hit the equal case) and so is extraneous. + outputs.push(self.output_extra(*curr_path, curr_node.clone())); + self.curr = self.trie_iter.next().transpose()?; + // back to the top of the loop to check the latest `self.curr` value against the + // given path/node. + } + } + } + } + + /// Must be called once there are no more calls to `next` to made. All further nodes produced + /// by the iterator will be considered extraneous. + fn finalize(&mut self, outputs: &mut Vec) -> Result<(), DatabaseError> { + loop { + if let Some((curr_path, curr_node)) = self.curr.take() { + outputs.push(self.output_extra(curr_path, curr_node)); + self.curr = self.trie_iter.next().transpose()?; + } else { + return Ok(()) + } + } + } +} + +/// Checks that data stored in the trie database is consistent, using hashed accounts/storages +/// database tables as the source of truth. This will iteratively recompute the entire trie based +/// on the hashed state, and produce any discovered [`Output`]s via the `next` method. +#[derive(Debug)] +pub struct Verifier<'a, T: TrieCursorFactory, H> { + trie_cursor_factory: &'a T, + hashed_cursor_factory: H, + branch_node_iter: StateRootBranchNodesIter, + outputs: Vec, + account: SingleVerifier>>, + storage: Option<(B256, SingleVerifier>>)>, + complete: bool, +} + +impl<'a, T: TrieCursorFactory, H: HashedCursorFactory + Clone> Verifier<'a, T, H> { + /// Creates a new verifier instance. + pub fn new( + trie_cursor_factory: &'a T, + hashed_cursor_factory: H, + ) -> Result { + Ok(Self { + trie_cursor_factory, + hashed_cursor_factory: hashed_cursor_factory.clone(), + branch_node_iter: StateRootBranchNodesIter::new(hashed_cursor_factory), + outputs: Default::default(), + account: SingleVerifier::new(None, trie_cursor_factory.account_trie_cursor()?)?, + storage: None, + complete: false, + }) + } +} + +impl<'a, T: TrieCursorFactory, H: HashedCursorFactory + Clone> Verifier<'a, T, H> { + fn new_storage( + &mut self, + account: B256, + path: Nibbles, + node: BranchNodeCompact, + ) -> Result<(), DatabaseError> { + let trie_cursor = self.trie_cursor_factory.storage_trie_cursor(account)?; + let mut storage = SingleVerifier::new(Some(account), trie_cursor)?; + storage.next(&mut self.outputs, path, node)?; + self.storage = Some((account, storage)); + Ok(()) + } + + /// This method is called using the account hashes at the boundary of [`BranchNode::Storage`] + /// sequences, ie once the [`StateRootBranchNodesIter`] has begun yielding storage nodes for a + /// different account than it was yielding previously. All accounts between the two should have + /// empty storages. + fn verify_empty_storages( + &mut self, + last_account: B256, + next_account: B256, + start_inclusive: bool, + end_inclusive: bool, + ) -> Result<(), DatabaseError> { + let mut account_cursor = self.hashed_cursor_factory.hashed_account_cursor()?; + let mut account_seeked = false; + + if !start_inclusive { + account_seeked = true; + account_cursor.seek(last_account)?; + } + + loop { + let Some((curr_account, _)) = (if account_seeked { + account_cursor.next()? + } else { + account_seeked = true; + account_cursor.seek(last_account)? + }) else { + return Ok(()) + }; + + if curr_account < next_account || (end_inclusive && curr_account == next_account) { + trace!(target: "trie::verify", account = ?curr_account, "Verying account has empty storage"); + + let mut storage_cursor = + self.trie_cursor_factory.storage_trie_cursor(curr_account)?; + let mut seeked = false; + while let Some((path, node)) = if seeked { + storage_cursor.next()? + } else { + seeked = true; + storage_cursor.seek(Nibbles::new())? + } { + self.outputs.push(Output::StorageExtra(curr_account, path, node)); + } + } else { + return Ok(()) + } + } + } + + fn try_next(&mut self) -> Result<(), StateRootError> { + match self.branch_node_iter.next().transpose()? { + None => { + self.account.finalize(&mut self.outputs)?; + if let Some((prev_account, storage)) = self.storage.as_mut() { + storage.finalize(&mut self.outputs)?; + + // If there was a previous storage account, and it is the final one, then we + // need to validate that all accounts coming after it have empty storages. + let prev_account = *prev_account; + + // Calculate the max possible account address (all bits set). + let max_account = B256::from([0xFFu8; 32]); + + self.verify_empty_storages(prev_account, max_account, false, true)?; + } + self.complete = true; + } + Some(BranchNode::Account(path, node)) => { + trace!(target: "trie::verify", ?path, "Account node from state root"); + self.account.next(&mut self.outputs, path, node)?; + // Push progress indicator + if !path.is_empty() { + self.outputs.push(Output::Progress(path)); + } + } + Some(BranchNode::Storage(account, path, node)) => { + trace!(target: "trie::verify", ?account, ?path, "Storage node from state root"); + match self.storage.as_mut() { + None => { + // First storage account - check for any empty storages before it + self.verify_empty_storages(B256::ZERO, account, true, false)?; + self.new_storage(account, path, node)?; + } + Some((prev_account, storage)) if *prev_account == account => { + storage.next(&mut self.outputs, path, node)?; + } + Some((prev_account, storage)) => { + storage.finalize(&mut self.outputs)?; + // Clear any storage entries between the previous account and the new one + let prev_account = *prev_account; + self.verify_empty_storages(prev_account, account, false, false)?; + self.new_storage(account, path, node)?; + } + } + } + } + + // If any outputs were appended we want to reverse them, so they are popped off + // in the same order they were appended. + self.outputs.reverse(); + Ok(()) + } +} + +impl<'a, T: TrieCursorFactory, H: HashedCursorFactory + Clone> Iterator for Verifier<'a, T, H> { + type Item = Result; + + fn next(&mut self) -> Option { + loop { + if let Some(output) = self.outputs.pop() { + return Some(Ok(output)) + } + + if self.complete { + return None + } + + if let Err(err) = self.try_next() { + return Some(Err(err)) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + hashed_cursor::mock::MockHashedCursorFactory, + trie_cursor::mock::{MockTrieCursor, MockTrieCursorFactory}, + }; + use alloy_primitives::{address, keccak256, map::B256Map, U256}; + use alloy_trie::TrieMask; + use assert_matches::assert_matches; + use reth_primitives_traits::Account; + use std::collections::BTreeMap; + + /// Helper function to create a simple test `BranchNodeCompact` + fn test_branch_node( + state_mask: u16, + tree_mask: u16, + hash_mask: u16, + hashes: Vec, + ) -> BranchNodeCompact { + // Ensure the number of hashes matches the number of bits set in hash_mask + let expected_hashes = hash_mask.count_ones() as usize; + let mut final_hashes = hashes; + let mut counter = 100u8; + while final_hashes.len() < expected_hashes { + final_hashes.push(B256::from([counter; 32])); + counter += 1; + } + final_hashes.truncate(expected_hashes); + + BranchNodeCompact::new( + TrieMask::new(state_mask), + TrieMask::new(tree_mask), + TrieMask::new(hash_mask), + final_hashes, + None, + ) + } + + /// Helper function to create a simple test `MockTrieCursor` + fn create_mock_cursor(trie_nodes: BTreeMap) -> MockTrieCursor { + let factory = MockTrieCursorFactory::new(trie_nodes, B256Map::default()); + factory.account_trie_cursor().unwrap() + } + + #[test] + fn test_state_root_branch_nodes_iter_empty() { + // Test with completely empty state + let factory = MockHashedCursorFactory::new(BTreeMap::new(), B256Map::default()); + let mut iter = StateRootBranchNodesIter::new(factory); + + // Collect all results - with empty state, should complete without producing nodes + let mut count = 0; + for result in iter.by_ref() { + assert!(result.is_ok(), "Unexpected error: {:?}", result.unwrap_err()); + count += 1; + // Prevent infinite loop in test + assert!(count <= 1000, "Too many iterations"); + } + + assert!(iter.complete); + } + + #[test] + fn test_state_root_branch_nodes_iter_basic() { + // Simple test with a few accounts and storage + let mut accounts = BTreeMap::new(); + let mut storage_tries = B256Map::default(); + + // Create test accounts + let addr1 = keccak256(address!("0000000000000000000000000000000000000001")); + accounts.insert( + addr1, + Account { + nonce: 1, + balance: U256::from(1000), + bytecode_hash: Some(keccak256(b"code1")), + }, + ); + + // Add storage for the account + let mut storage1 = BTreeMap::new(); + storage1.insert(keccak256(B256::from(U256::from(1))), U256::from(100)); + storage1.insert(keccak256(B256::from(U256::from(2))), U256::from(200)); + storage_tries.insert(addr1, storage1); + + let factory = MockHashedCursorFactory::new(accounts, storage_tries); + let mut iter = StateRootBranchNodesIter::new(factory); + + // Collect nodes and verify basic properties + let mut account_paths = Vec::new(); + let mut storage_paths_by_account: B256Map> = B256Map::default(); + let mut iterations = 0; + + for result in iter.by_ref() { + iterations += 1; + assert!(iterations <= 10000, "Too many iterations - possible infinite loop"); + + match result { + Ok(BranchNode::Account(path, _)) => { + account_paths.push(path); + } + Ok(BranchNode::Storage(account, path, _)) => { + storage_paths_by_account.entry(account).or_default().push(path); + } + Err(e) => panic!("Unexpected error: {:?}", e), + } + } + + // Verify account paths are in ascending order + for i in 1..account_paths.len() { + assert!( + account_paths[i - 1] < account_paths[i], + "Account paths should be in ascending order" + ); + } + + // Verify storage paths for each account are in ascending order + for (account, paths) in storage_paths_by_account { + for i in 1..paths.len() { + assert!( + paths[i - 1] < paths[i], + "Storage paths for account {:?} should be in ascending order", + account + ); + } + } + + assert!(iter.complete); + } + + #[test] + fn test_state_root_branch_nodes_iter_multiple_accounts() { + // Test with multiple accounts to verify ordering + let mut accounts = BTreeMap::new(); + let mut storage_tries = B256Map::default(); + + // Create multiple test addresses + for i in 1u8..=3 { + let addr = keccak256([i; 20]); + accounts.insert( + addr, + Account { + nonce: i as u64, + balance: U256::from(i as u64 * 1000), + bytecode_hash: (i == 2).then(|| keccak256([i])), + }, + ); + + // Add some storage for each account + let mut storage = BTreeMap::new(); + for j in 0..i { + storage.insert(keccak256(B256::from(U256::from(j))), U256::from(j as u64 * 10)); + } + if !storage.is_empty() { + storage_tries.insert(addr, storage); + } + } + + let factory = MockHashedCursorFactory::new(accounts, storage_tries); + let mut iter = StateRootBranchNodesIter::new(factory); + + // Track what we see + let mut seen_storage_accounts = Vec::new(); + let mut current_storage_account = None; + let mut iterations = 0; + + for result in iter.by_ref() { + iterations += 1; + assert!(iterations <= 10000, "Too many iterations"); + + match result { + Ok(BranchNode::Storage(account, _, _)) => { + if current_storage_account != Some(account) { + // Verify we don't revisit a storage account + assert!( + !seen_storage_accounts.contains(&account), + "Should not revisit storage account {:?}", + account + ); + seen_storage_accounts.push(account); + current_storage_account = Some(account); + } + } + Ok(BranchNode::Account(_, _)) => { + // Account nodes are fine + } + Err(e) => panic!("Unexpected error: {:?}", e), + } + } + + assert!(iter.complete); + } + + #[test] + fn test_single_verifier_new() { + // Test creating a new SingleVerifier for account trie + let trie_nodes = BTreeMap::from([( + Nibbles::from_nibbles([0x1]), + test_branch_node(0b1111, 0, 0, vec![]), + )]); + + let cursor = create_mock_cursor(trie_nodes); + let verifier = SingleVerifier::new(None, cursor).unwrap(); + + // Should have seeked to the beginning and found the first node + assert!(verifier.curr.is_some()); + } + + #[test] + fn test_single_verifier_next_exact_match() { + // Test when the expected node matches exactly + let node1 = test_branch_node(0b1111, 0, 0b1111, vec![B256::from([1u8; 32])]); + let node2 = test_branch_node(0b0101, 0b0001, 0b0100, vec![B256::from([2u8; 32])]); + + let trie_nodes = BTreeMap::from([ + (Nibbles::from_nibbles([0x1]), node1.clone()), + (Nibbles::from_nibbles([0x2]), node2), + ]); + + let cursor = create_mock_cursor(trie_nodes); + let mut verifier = SingleVerifier::new(None, cursor).unwrap(); + let mut outputs = Vec::new(); + + // Call next with the exact node that exists + verifier.next(&mut outputs, Nibbles::from_nibbles([0x1]), node1).unwrap(); + + // Should have no outputs + assert!(outputs.is_empty()); + } + + #[test] + fn test_single_verifier_next_wrong_value() { + // Test when the path matches but value is different + let node_in_trie = test_branch_node(0b1111, 0, 0b1111, vec![B256::from([1u8; 32])]); + let node_expected = test_branch_node(0b0101, 0b0001, 0b0100, vec![B256::from([2u8; 32])]); + + let trie_nodes = BTreeMap::from([(Nibbles::from_nibbles([0x1]), node_in_trie.clone())]); + + let cursor = create_mock_cursor(trie_nodes); + let mut verifier = SingleVerifier::new(None, cursor).unwrap(); + let mut outputs = Vec::new(); + + // Call next with different node value + verifier.next(&mut outputs, Nibbles::from_nibbles([0x1]), node_expected.clone()).unwrap(); + + // Should have one "wrong" output + assert_eq!(outputs.len(), 1); + assert_matches!( + &outputs[0], + Output::AccountWrong { path, expected, found } + if *path == Nibbles::from_nibbles([0x1]) && *expected == node_expected && *found == node_in_trie + ); + } + + #[test] + fn test_single_verifier_next_missing() { + // Test when expected node doesn't exist in trie + let node1 = test_branch_node(0b1111, 0, 0b1111, vec![B256::from([1u8; 32])]); + let node_missing = test_branch_node(0b0101, 0b0001, 0b0100, vec![B256::from([2u8; 32])]); + + let trie_nodes = BTreeMap::from([(Nibbles::from_nibbles([0x3]), node1)]); + + let cursor = create_mock_cursor(trie_nodes); + let mut verifier = SingleVerifier::new(None, cursor).unwrap(); + let mut outputs = Vec::new(); + + // Call next with a node that comes before any in the trie + verifier.next(&mut outputs, Nibbles::from_nibbles([0x1]), node_missing.clone()).unwrap(); + + // Should have one "missing" output + assert_eq!(outputs.len(), 1); + assert_matches!( + &outputs[0], + Output::AccountMissing(path, node) + if *path == Nibbles::from_nibbles([0x1]) && *node == node_missing + ); + } + + #[test] + fn test_single_verifier_next_extra() { + // Test when trie has extra nodes not in expected + // Create a proper trie structure with root + let node_root = test_branch_node(0b1110, 0, 0b1110, vec![]); // root has children at 1, 2, 3 + let node1 = test_branch_node(0b0001, 0, 0b0001, vec![]); + let node2 = test_branch_node(0b0010, 0, 0b0010, vec![]); + let node3 = test_branch_node(0b0100, 0, 0b0100, vec![]); + + let trie_nodes = BTreeMap::from([ + (Nibbles::new(), node_root.clone()), + (Nibbles::from_nibbles([0x1]), node1.clone()), + (Nibbles::from_nibbles([0x2]), node2.clone()), + (Nibbles::from_nibbles([0x3]), node3.clone()), + ]); + + let cursor = create_mock_cursor(trie_nodes); + let mut verifier = SingleVerifier::new(None, cursor).unwrap(); + let mut outputs = Vec::new(); + + // The depth-first iterator produces in post-order: 0x1, 0x2, 0x3, root + // We only provide 0x1 and 0x3, skipping 0x2 and root + verifier.next(&mut outputs, Nibbles::from_nibbles([0x1]), node1).unwrap(); + verifier.next(&mut outputs, Nibbles::from_nibbles([0x3]), node3).unwrap(); + verifier.finalize(&mut outputs).unwrap(); + + // Should have two "extra" outputs for nodes in the trie that we skipped + if outputs.len() != 2 { + eprintln!("Expected 2 outputs, got {}:", outputs.len()); + for inc in &outputs { + eprintln!(" {:?}", inc); + } + } + assert_eq!(outputs.len(), 2); + assert_matches!( + &outputs[0], + Output::AccountExtra(path, node) + if *path == Nibbles::from_nibbles([0x2]) && *node == node2 + ); + assert_matches!( + &outputs[1], + Output::AccountExtra(path, node) + if *path == Nibbles::new() && *node == node_root + ); + } + + #[test] + fn test_single_verifier_finalize() { + // Test finalize marks all remaining nodes as extra + let node_root = test_branch_node(0b1110, 0, 0b1110, vec![]); // root has children at 1, 2, 3 + let node1 = test_branch_node(0b0001, 0, 0b0001, vec![]); + let node2 = test_branch_node(0b0010, 0, 0b0010, vec![]); + let node3 = test_branch_node(0b0100, 0, 0b0100, vec![]); + + let trie_nodes = BTreeMap::from([ + (Nibbles::new(), node_root.clone()), + (Nibbles::from_nibbles([0x1]), node1.clone()), + (Nibbles::from_nibbles([0x2]), node2.clone()), + (Nibbles::from_nibbles([0x3]), node3.clone()), + ]); + + let cursor = create_mock_cursor(trie_nodes); + let mut verifier = SingleVerifier::new(None, cursor).unwrap(); + let mut outputs = Vec::new(); + + // The depth-first iterator produces in post-order: 0x1, 0x2, 0x3, root + // Process first two nodes correctly + verifier.next(&mut outputs, Nibbles::from_nibbles([0x1]), node1).unwrap(); + verifier.next(&mut outputs, Nibbles::from_nibbles([0x2]), node2).unwrap(); + assert!(outputs.is_empty()); + + // Finalize - should mark remaining nodes (0x3 and root) as extra + verifier.finalize(&mut outputs).unwrap(); + + // Should have two extra nodes + assert_eq!(outputs.len(), 2); + assert_matches!( + &outputs[0], + Output::AccountExtra(path, node) + if *path == Nibbles::from_nibbles([0x3]) && *node == node3 + ); + assert_matches!( + &outputs[1], + Output::AccountExtra(path, node) + if *path == Nibbles::new() && *node == node_root + ); + } + + #[test] + fn test_single_verifier_storage_trie() { + // Test SingleVerifier for storage trie (with account set) + let account = B256::from([42u8; 32]); + let node = test_branch_node(0b1111, 0, 0b1111, vec![B256::from([1u8; 32])]); + + let trie_nodes = BTreeMap::from([(Nibbles::from_nibbles([0x1]), node)]); + + let cursor = create_mock_cursor(trie_nodes); + let mut verifier = SingleVerifier::new(Some(account), cursor).unwrap(); + let mut outputs = Vec::new(); + + // Call next with missing node + let missing_node = test_branch_node(0b0101, 0b0001, 0b0100, vec![B256::from([2u8; 32])]); + verifier.next(&mut outputs, Nibbles::from_nibbles([0x0]), missing_node.clone()).unwrap(); + + // Should produce StorageMissing, not AccountMissing + assert_eq!(outputs.len(), 1); + assert_matches!( + &outputs[0], + Output::StorageMissing(acc, path, node) + if *acc == account && *path == Nibbles::from_nibbles([0x0]) && *node == missing_node + ); + } + + #[test] + fn test_single_verifier_empty_trie() { + // Test with empty trie cursor + let trie_nodes = BTreeMap::new(); + let cursor = create_mock_cursor(trie_nodes); + let mut verifier = SingleVerifier::new(None, cursor).unwrap(); + let mut outputs = Vec::new(); + + // Any node should be marked as missing + let node = test_branch_node(0b1111, 0, 0b1111, vec![B256::from([1u8; 32])]); + verifier.next(&mut outputs, Nibbles::from_nibbles([0x1]), node.clone()).unwrap(); + + assert_eq!(outputs.len(), 1); + assert_matches!( + &outputs[0], + Output::AccountMissing(path, n) + if *path == Nibbles::from_nibbles([0x1]) && *n == node + ); + } + + #[test] + fn test_single_verifier_depth_first_ordering() { + // Test that nodes must be provided in depth-first order + // Create nodes with proper parent-child relationships + let node_root = test_branch_node(0b0110, 0, 0b0110, vec![]); // root has children at 1 and 2 + let node1 = test_branch_node(0b0110, 0, 0b0110, vec![]); // 0x1 has children at 1 and 2 + let node11 = test_branch_node(0b0001, 0, 0b0001, vec![]); // 0x11 is a leaf + let node12 = test_branch_node(0b0010, 0, 0b0010, vec![]); // 0x12 is a leaf + let node2 = test_branch_node(0b0100, 0, 0b0100, vec![]); // 0x2 is a leaf + + // The depth-first iterator will iterate from the root in this order: + // root -> 0x1 -> 0x11, 0x12 (children of 0x1), then 0x2 + // But because of depth-first, we get: root, 0x1, 0x11, 0x12, 0x2 + let trie_nodes = BTreeMap::from([ + (Nibbles::new(), node_root.clone()), // root + (Nibbles::from_nibbles([0x1]), node1.clone()), // 0x1 + (Nibbles::from_nibbles([0x1, 0x1]), node11.clone()), // 0x11 + (Nibbles::from_nibbles([0x1, 0x2]), node12.clone()), // 0x12 + (Nibbles::from_nibbles([0x2]), node2.clone()), // 0x2 + ]); + + let cursor = create_mock_cursor(trie_nodes); + let mut verifier = SingleVerifier::new(None, cursor).unwrap(); + let mut outputs = Vec::new(); + + // The depth-first iterator produces nodes in post-order (children before parents) + // Order: 0x11, 0x12, 0x1, 0x2, root + verifier.next(&mut outputs, Nibbles::from_nibbles([0x1, 0x1]), node11).unwrap(); + verifier.next(&mut outputs, Nibbles::from_nibbles([0x1, 0x2]), node12).unwrap(); + verifier.next(&mut outputs, Nibbles::from_nibbles([0x1]), node1).unwrap(); + verifier.next(&mut outputs, Nibbles::from_nibbles([0x2]), node2).unwrap(); + verifier.next(&mut outputs, Nibbles::new(), node_root).unwrap(); + verifier.finalize(&mut outputs).unwrap(); + + // All should match, no outputs + if !outputs.is_empty() { + eprintln!( + "Test test_single_verifier_depth_first_ordering failed with {} outputs:", + outputs.len() + ); + for inc in &outputs { + eprintln!(" {:?}", inc); + } + } + assert!(outputs.is_empty()); + } + + #[test] + fn test_single_verifier_wrong_depth_first_order() { + // Test that providing nodes in wrong order produces outputs + // Create a trie with parent-child relationship + let node_root = test_branch_node(0b0010, 0, 0b0010, vec![]); // root has child at 1 + let node1 = test_branch_node(0b0010, 0, 0b0010, vec![]); // 0x1 has child at 1 + let node11 = test_branch_node(0b0001, 0, 0b0001, vec![]); // 0x11 is a leaf + + let trie_nodes = BTreeMap::from([ + (Nibbles::new(), node_root.clone()), + (Nibbles::from_nibbles([0x1]), node1.clone()), + (Nibbles::from_nibbles([0x1, 0x1]), node11.clone()), + ]); + + let cursor = create_mock_cursor(trie_nodes); + let mut verifier = SingleVerifier::new(None, cursor).unwrap(); + let mut outputs = Vec::new(); + + // Process in WRONG order (skip root, provide child before processing all nodes correctly) + // The iterator will produce: root, 0x1, 0x11 + // But we provide: 0x11, root, 0x1 (completely wrong order) + verifier.next(&mut outputs, Nibbles::from_nibbles([0x1, 0x1]), node11).unwrap(); + verifier.next(&mut outputs, Nibbles::new(), node_root).unwrap(); + verifier.next(&mut outputs, Nibbles::from_nibbles([0x1]), node1).unwrap(); + + // Should have outputs since we provided them in wrong order + assert!(!outputs.is_empty()); + } + + #[test] + fn test_single_verifier_complex_depth_first() { + // Test a complex tree structure with depth-first ordering + // Build a tree structure with proper parent-child relationships + let node_root = test_branch_node(0b0110, 0, 0b0110, vec![]); // root: children at nibbles 1 and 2 + let node1 = test_branch_node(0b0110, 0, 0b0110, vec![]); // 0x1: children at nibbles 1 and 2 + let node11 = test_branch_node(0b0110, 0, 0b0110, vec![]); // 0x11: children at nibbles 1 and 2 + let node111 = test_branch_node(0b0001, 0, 0b0001, vec![]); // 0x111: leaf + let node112 = test_branch_node(0b0010, 0, 0b0010, vec![]); // 0x112: leaf + let node12 = test_branch_node(0b0100, 0, 0b0100, vec![]); // 0x12: leaf + let node2 = test_branch_node(0b0010, 0, 0b0010, vec![]); // 0x2: child at nibble 1 + let node21 = test_branch_node(0b0001, 0, 0b0001, vec![]); // 0x21: leaf + + // Create the trie structure + let trie_nodes = BTreeMap::from([ + (Nibbles::new(), node_root.clone()), + (Nibbles::from_nibbles([0x1]), node1.clone()), + (Nibbles::from_nibbles([0x1, 0x1]), node11.clone()), + (Nibbles::from_nibbles([0x1, 0x1, 0x1]), node111.clone()), + (Nibbles::from_nibbles([0x1, 0x1, 0x2]), node112.clone()), + (Nibbles::from_nibbles([0x1, 0x2]), node12.clone()), + (Nibbles::from_nibbles([0x2]), node2.clone()), + (Nibbles::from_nibbles([0x2, 0x1]), node21.clone()), + ]); + + let cursor = create_mock_cursor(trie_nodes); + let mut verifier = SingleVerifier::new(None, cursor).unwrap(); + let mut outputs = Vec::new(); + + // The depth-first iterator produces nodes in post-order (children before parents) + // Order: 0x111, 0x112, 0x11, 0x12, 0x1, 0x21, 0x2, root + verifier.next(&mut outputs, Nibbles::from_nibbles([0x1, 0x1, 0x1]), node111).unwrap(); + verifier.next(&mut outputs, Nibbles::from_nibbles([0x1, 0x1, 0x2]), node112).unwrap(); + verifier.next(&mut outputs, Nibbles::from_nibbles([0x1, 0x1]), node11).unwrap(); + verifier.next(&mut outputs, Nibbles::from_nibbles([0x1, 0x2]), node12).unwrap(); + verifier.next(&mut outputs, Nibbles::from_nibbles([0x1]), node1).unwrap(); + verifier.next(&mut outputs, Nibbles::from_nibbles([0x2, 0x1]), node21).unwrap(); + verifier.next(&mut outputs, Nibbles::from_nibbles([0x2]), node2).unwrap(); + verifier.next(&mut outputs, Nibbles::new(), node_root).unwrap(); + verifier.finalize(&mut outputs).unwrap(); + + // All should match, no outputs + if !outputs.is_empty() { + eprintln!( + "Test test_single_verifier_complex_depth_first failed with {} outputs:", + outputs.len() + ); + for inc in &outputs { + eprintln!(" {:?}", inc); + } + } + assert!(outputs.is_empty()); + } +} diff --git a/crates/trie/trie/src/walker.rs b/crates/trie/trie/src/walker.rs index 5beea7597fd..f12bf46f748 100644 --- a/crates/trie/trie/src/walker.rs +++ b/crates/trie/trie/src/walker.rs @@ -4,17 +4,18 @@ use crate::{ BranchNodeCompact, Nibbles, }; use alloy_primitives::{map::HashSet, B256}; +use alloy_trie::proof::AddedRemovedKeys; use reth_storage_errors::db::DatabaseError; use tracing::{instrument, trace}; #[cfg(feature = "metrics")] use crate::metrics::WalkerMetrics; -/// `TrieWalker` is a structure that enables traversal of a Merkle trie. -/// It allows moving through the trie in a depth-first manner, skipping certain branches -/// if they have not changed. +/// Traverses the trie in lexicographic order. +/// +/// This iterator depends on the ordering guarantees of [`TrieCursor`]. #[derive(Debug)] -pub struct TrieWalker { +pub struct TrieWalker { /// A mutable reference to a trie cursor instance used for navigating the trie. pub cursor: C, /// A vector containing the trie nodes that have been visited. @@ -27,12 +28,16 @@ pub struct TrieWalker { pub changes: PrefixSet, /// The retained trie node keys that need to be removed. removed_keys: Option>, + /// Provided when it's necessary not to skip certain nodes during proof generation. + /// Specifically we don't skip certain branch nodes even when they are not in the `PrefixSet`, + /// when they might be required to support leaf removal. + added_removed_keys: Option, #[cfg(feature = "metrics")] /// Walker metrics. metrics: WalkerMetrics, } -impl TrieWalker { +impl> TrieWalker { /// Constructs a new `TrieWalker` for the state trie from existing stack and a cursor. pub fn state_trie_from_stack(cursor: C, stack: Vec, changes: PrefixSet) -> Self { Self::from_stack( @@ -72,6 +77,7 @@ impl TrieWalker { stack, can_skip_current_node: false, removed_keys: None, + added_removed_keys: None, #[cfg(feature = "metrics")] metrics: WalkerMetrics::new(trie_type), }; @@ -87,6 +93,21 @@ impl TrieWalker { self } + /// Configures the walker to not skip certain branch nodes, even when they are not in the + /// `PrefixSet`, when they might be needed to support leaf removal. + pub fn with_added_removed_keys(self, added_removed_keys: Option) -> TrieWalker { + TrieWalker { + cursor: self.cursor, + stack: self.stack, + can_skip_current_node: self.can_skip_current_node, + changes: self.changes, + removed_keys: self.removed_keys, + added_removed_keys, + #[cfg(feature = "metrics")] + metrics: self.metrics, + } + } + /// Split the walker into stack and trie updates. pub fn split(mut self) -> (Vec, HashSet) { let keys = self.take_removed_keys(); @@ -117,11 +138,19 @@ impl TrieWalker { self.stack.last().map(|n| n.full_key()) } - /// Returns the current hash in the trie if any. + /// Returns the current hash in the trie, if any. pub fn hash(&self) -> Option { self.stack.last().and_then(|n| n.hash()) } + /// Returns the current hash in the trie, if any. + /// + /// Differs from [`Self::hash`] in that it returns `None` if the subnode is positioned at the + /// child without a hash mask bit set. [`Self::hash`] panics in that case. + pub fn maybe_hash(&self) -> Option { + self.stack.last().and_then(|n| n.maybe_hash()) + } + /// Indicates whether the children of the current node are present in the trie. pub fn children_are_in_trie(&self) -> bool { self.stack.last().is_some_and(|n| n.tree_flag()) @@ -131,9 +160,7 @@ impl TrieWalker { #[instrument(level = "trace", skip(self), ret)] pub fn next_unprocessed_key(&self) -> Option<(B256, Nibbles)> { self.key() - .and_then( - |key| if self.can_skip_current_node { key.increment() } else { Some(key.clone()) }, - ) + .and_then(|key| if self.can_skip_current_node { key.increment() } else { Some(*key) }) .map(|key| { let mut packed = key.pack(); packed.resize(32, 0); @@ -144,10 +171,27 @@ impl TrieWalker { /// Updates the skip node flag based on the walker's current state. fn update_skip_node(&mut self) { let old = self.can_skip_current_node; - self.can_skip_current_node = self - .stack - .last() - .is_some_and(|node| !self.changes.contains(node.full_key()) && node.hash_flag()); + self.can_skip_current_node = self.stack.last().is_some_and(|node| { + // If the current key is not removed according to the [`AddedRemovedKeys`], and all of + // its siblings are removed, then we don't want to skip it. This allows the + // `ProofRetainer` to include this node in the returned proofs. Required to support + // leaf removal. + let key_is_only_nonremoved_child = + self.added_removed_keys.as_ref().is_some_and(|added_removed_keys| { + node.full_key_is_only_nonremoved_child(added_removed_keys.as_ref()) + }); + + trace!( + target: "trie::walker", + ?key_is_only_nonremoved_child, + full_key=?node.full_key(), + "Checked for only non-removed child", + ); + + !self.changes.contains(node.full_key()) && + node.hash_flag() && + !key_is_only_nonremoved_child + }); trace!( target: "trie::walker", old, @@ -156,9 +200,7 @@ impl TrieWalker { "updated skip node flag" ); } -} -impl TrieWalker { /// Constructs a new [`TrieWalker`] for the state trie. pub fn state_trie(cursor: C, changes: PrefixSet) -> Self { Self::new( @@ -192,6 +234,7 @@ impl TrieWalker { stack: vec![CursorSubNode::default()], can_skip_current_node: false, removed_keys: None, + added_removed_keys: Default::default(), #[cfg(feature = "metrics")] metrics: WalkerMetrics::new(trie_type), }; @@ -241,8 +284,8 @@ impl TrieWalker { /// Retrieves the current root node from the DB, seeking either the exact node or the next one. fn node(&mut self, exact: bool) -> Result, DatabaseError> { - let key = self.key().expect("key must exist").clone(); - let entry = if exact { self.cursor.seek_exact(key)? } else { self.cursor.seek(key)? }; + let key = self.key().expect("key must exist"); + let entry = if exact { self.cursor.seek_exact(*key)? } else { self.cursor.seek(*key)? }; #[cfg(feature = "metrics")] self.metrics.inc_branch_nodes_seeked(); @@ -266,20 +309,20 @@ impl TrieWalker { // We need to sync the stack with the trie structure when consuming a new node. This is // necessary for proper traversal and accurately representing the trie in the stack. if !key.is_empty() && !self.stack.is_empty() { - self.stack[0].set_nibble(key[0]); + self.stack[0].set_nibble(key.get_unchecked(0)); } // The current tree mask might have been set incorrectly. // Sanity check that the newly retrieved trie node key is the child of the last item // on the stack. If not, advance to the next sibling instead of adding the node to the // stack. - if let Some(subnode) = self.stack.last() { - if !key.starts_with(subnode.full_key()) { - #[cfg(feature = "metrics")] - self.metrics.inc_out_of_order_subnode(1); - self.move_to_next_sibling(false)?; - return Ok(()) - } + if let Some(subnode) = self.stack.last() && + !key.starts_with(subnode.full_key()) + { + #[cfg(feature = "metrics")] + self.metrics.inc_out_of_order_subnode(1); + self.move_to_next_sibling(false)?; + return Ok(()) } // Create a new CursorSubNode and push it to the stack. @@ -290,10 +333,10 @@ impl TrieWalker { // Delete the current node if it's included in the prefix set or it doesn't contain the root // hash. - if !self.can_skip_current_node || position.is_child() { - if let Some((keys, key)) = self.removed_keys.as_mut().zip(self.cursor.current()?) { - keys.insert(key); - } + if (!self.can_skip_current_node || position.is_child()) && + let Some((keys, key)) = self.removed_keys.as_mut().zip(self.cursor.current()?) + { + keys.insert(key); } Ok(()) diff --git a/crates/trie/trie/src/witness.rs b/crates/trie/trie/src/witness.rs index ce40a01e1c0..763908c242d 100644 --- a/crates/trie/trie/src/witness.rs +++ b/crates/trie/trie/src/witness.rs @@ -1,12 +1,13 @@ use crate::{ hashed_cursor::{HashedCursor, HashedCursorFactory}, prefix_set::TriePrefixSetsMut, - proof::{Proof, ProofBlindedProviderFactory}, + proof::{Proof, ProofTrieNodeProviderFactory}, trie_cursor::TrieCursorFactory, }; use alloy_rlp::EMPTY_STRING_CODE; use alloy_trie::EMPTY_ROOT_HASH; use reth_trie_common::HashedPostState; +use reth_trie_sparse::SparseTrieInterface; use alloy_primitives::{ keccak256, @@ -20,10 +21,10 @@ use reth_execution_errors::{ }; use reth_trie_common::{MultiProofTargets, Nibbles}; use reth_trie_sparse::{ - blinded::{BlindedProvider, BlindedProviderFactory, RevealedNode}, - SparseStateTrie, + provider::{RevealedNode, TrieNodeProvider, TrieNodeProviderFactory}, + SerialSparseTrie, SparseStateTrie, }; -use std::sync::{mpsc, Arc}; +use std::sync::mpsc; /// State transition witness for the trie. #[derive(Debug)] @@ -83,7 +84,7 @@ impl TrieWitness { self } - /// Set `always_include_root_node` to true. Root node will be included even on empty state. + /// Set `always_include_root_node` to true. Root node will be included even in empty state. /// This setting is useful if the caller wants to verify the witness against the /// parent state root. pub const fn always_include_root_node(mut self) -> Self { @@ -145,15 +146,11 @@ where } let (tx, rx) = mpsc::channel(); - let blinded_provider_factory = WitnessBlindedProviderFactory::new( - ProofBlindedProviderFactory::new( - self.trie_cursor_factory, - self.hashed_cursor_factory, - Arc::new(self.prefix_sets), - ), + let blinded_provider_factory = WitnessTrieNodeProviderFactory::new( + ProofTrieNodeProviderFactory::new(self.trie_cursor_factory, self.hashed_cursor_factory), tx, ); - let mut sparse_trie = SparseStateTrie::new(blinded_provider_factory); + let mut sparse_trie = SparseStateTrie::::new(); sparse_trie.reveal_multiproof(multiproof)?; // Attempt to update state trie to gather additional information for the witness. @@ -161,6 +158,7 @@ where proof_targets.into_iter().sorted_unstable_by_key(|(ha, _)| *ha) { // Update storage trie first. + let provider = blinded_provider_factory.storage_node_provider(hashed_address); let storage = state.storages.get(&hashed_address); let storage_trie = sparse_trie.storage_trie_mut(&hashed_address).ok_or( SparseStateTrieErrorKind::SparseStorageTrie( @@ -176,11 +174,11 @@ where .map(|v| alloy_rlp::encode_fixed_size(v).to_vec()); if let Some(value) = maybe_leaf_value { - storage_trie.update_leaf(storage_nibbles, value).map_err(|err| { + storage_trie.update_leaf(storage_nibbles, value, &provider).map_err(|err| { SparseStateTrieErrorKind::SparseStorageTrie(hashed_address, err.into_kind()) })?; } else { - storage_trie.remove_leaf(&storage_nibbles).map_err(|err| { + storage_trie.remove_leaf(&storage_nibbles, &provider).map_err(|err| { SparseStateTrieErrorKind::SparseStorageTrie(hashed_address, err.into_kind()) })?; } @@ -194,7 +192,11 @@ where .get(&hashed_address) .ok_or(TrieWitnessError::MissingAccount(hashed_address))? .unwrap_or_default(); - sparse_trie.update_account(hashed_address, account)?; + + if !sparse_trie.update_account(hashed_address, account, &blinded_provider_factory)? { + let nibbles = Nibbles::unpack(hashed_address); + sparse_trie.remove_account_leaf(&nibbles, &blinded_provider_factory)?; + } while let Ok(node) = rx.try_recv() { self.witness.insert(keccak256(&node), node); @@ -235,56 +237,56 @@ where } #[derive(Debug, Clone)] -struct WitnessBlindedProviderFactory { - /// Blinded node provider factory. +struct WitnessTrieNodeProviderFactory { + /// Trie node provider factory. provider_factory: F, - /// Sender for forwarding fetched blinded node. + /// Sender for forwarding fetched trie node. tx: mpsc::Sender, } -impl WitnessBlindedProviderFactory { +impl WitnessTrieNodeProviderFactory { const fn new(provider_factory: F, tx: mpsc::Sender) -> Self { Self { provider_factory, tx } } } -impl BlindedProviderFactory for WitnessBlindedProviderFactory +impl TrieNodeProviderFactory for WitnessTrieNodeProviderFactory where - F: BlindedProviderFactory, - F::AccountNodeProvider: BlindedProvider, - F::StorageNodeProvider: BlindedProvider, + F: TrieNodeProviderFactory, + F::AccountNodeProvider: TrieNodeProvider, + F::StorageNodeProvider: TrieNodeProvider, { - type AccountNodeProvider = WitnessBlindedProvider; - type StorageNodeProvider = WitnessBlindedProvider; + type AccountNodeProvider = WitnessTrieNodeProvider; + type StorageNodeProvider = WitnessTrieNodeProvider; fn account_node_provider(&self) -> Self::AccountNodeProvider { let provider = self.provider_factory.account_node_provider(); - WitnessBlindedProvider::new(provider, self.tx.clone()) + WitnessTrieNodeProvider::new(provider, self.tx.clone()) } fn storage_node_provider(&self, account: B256) -> Self::StorageNodeProvider { let provider = self.provider_factory.storage_node_provider(account); - WitnessBlindedProvider::new(provider, self.tx.clone()) + WitnessTrieNodeProvider::new(provider, self.tx.clone()) } } #[derive(Debug)] -struct WitnessBlindedProvider

{ +struct WitnessTrieNodeProvider

{ /// Proof-based blinded. provider: P, /// Sender for forwarding fetched blinded node. tx: mpsc::Sender, } -impl

WitnessBlindedProvider

{ +impl

WitnessTrieNodeProvider

{ const fn new(provider: P, tx: mpsc::Sender) -> Self { Self { provider, tx } } } -impl BlindedProvider for WitnessBlindedProvider

{ - fn blinded_node(&self, path: &Nibbles) -> Result, SparseTrieError> { - let maybe_node = self.provider.blinded_node(path)?; +impl TrieNodeProvider for WitnessTrieNodeProvider

{ + fn trie_node(&self, path: &Nibbles) -> Result, SparseTrieError> { + let maybe_node = self.provider.trie_node(path)?; if let Some(node) = &maybe_node { self.tx .send(node.node.clone()) diff --git a/deny.toml b/deny.toml index 7c588e50fdd..fd2eb5c11cd 100644 --- a/deny.toml +++ b/deny.toml @@ -53,6 +53,7 @@ allow = [ # https://github.com/rustls/webpki/blob/main/LICENSE ISC Style "LicenseRef-rustls-webpki", "CDLA-Permissive-2.0", + "MPL-2.0", ] # Allow 1 or more licenses on a per-crate basis, so that particular licenses @@ -87,4 +88,5 @@ allow-git = [ "https://github.com/paradigmxyz/revm-inspectors", "https://github.com/alloy-rs/evm", "https://github.com/alloy-rs/hardforks", + "https://github.com/paradigmxyz/jsonrpsee", ] diff --git a/book/cli/help.rs b/docs/cli/help.rs similarity index 66% rename from book/cli/help.rs rename to docs/cli/help.rs index 963f53deb0a..0474d00e723 100755 --- a/book/cli/help.rs +++ b/docs/cli/help.rs @@ -5,30 +5,27 @@ edition = "2021" [dependencies] clap = { version = "4", features = ["derive"] } -pathdiff = "0.2" regex = "1" --- use clap::Parser; use regex::Regex; -use std::borrow::Cow; -use std::fs::{self, File}; -use std::io::{self, Write}; -use std::iter::once; -use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; -use std::str; -use std::sync::LazyLock; -use std::{fmt, process}; - -const SECTION_START: &str = ""; -const SECTION_END: &str = ""; -const README: &str = r#"# CLI Reference - - +use std::{ + borrow::Cow, + fmt, fs, io, + iter::once, + path::{Path, PathBuf}, + process::{Command, Stdio}, + str, + sync::LazyLock, +}; + +const README: &str = r#"import Summary from './SUMMARY.mdx'; + +# CLI Reference Automatically-generated CLI reference from `--help` output. -{{#include ./SUMMARY.md}} +

"#; const TRIM_LINE_END_MARKDOWN: bool = true; @@ -41,7 +38,7 @@ macro_rules! regex { }}; } -/// Generate markdown files from help output of commands +/// Generate markdown files from the help output of commands #[derive(Parser, Debug)] #[command(about, long_about = None)] struct Args { @@ -49,7 +46,7 @@ struct Args { #[arg(long, default_value_t = String::from("."))] root_dir: String, - /// Indentation for the root SUMMARY.md file + /// Indentation for the root SUMMARY.mdx file #[arg(long, default_value_t = 2)] root_indentation: usize, @@ -61,7 +58,7 @@ struct Args { #[arg(long)] readme: bool, - /// Whether to update the root SUMMARY.md file + /// Whether to update the root SUMMARY.mdx file #[arg(long)] root_summary: bool, @@ -76,11 +73,7 @@ struct Args { fn write_file(file_path: &Path, content: &str) -> io::Result<()> { let content = if TRIM_LINE_END_MARKDOWN { - content - .lines() - .map(|line| line.trim_end()) - .collect::>() - .join("\n") + content.lines().map(|line| line.trim_end()).collect::>().join("\n") } else { content.to_string() }; @@ -106,25 +99,13 @@ fn main() -> io::Result<()> { while let Some(cmd) = todo_iter.pop() { let (new_subcmds, stdout) = get_entry(&cmd)?; if args.verbose && !new_subcmds.is_empty() { - println!( - "Found subcommands for \"{}\": {:?}", - cmd.command_name(), - new_subcmds - ); + println!("Found subcommands for \"{}\": {:?}", cmd.command_name(), new_subcmds); } // Add new subcommands to todo_iter (so that they are processed in the correct order). for subcmd in new_subcmds.into_iter().rev() { - let new_subcmds: Vec<_> = cmd - .subcommands - .iter() - .cloned() - .chain(once(subcmd)) - .collect(); - - todo_iter.push(Cmd { - cmd: cmd.cmd, - subcommands: new_subcmds, - }); + let new_subcmds: Vec<_> = cmd.subcommands.iter().cloned().chain(once(subcmd)).collect(); + + todo_iter.push(Cmd { cmd: cmd.cmd, subcommands: new_subcmds }); } output.push((cmd, stdout)); } @@ -134,38 +115,32 @@ fn main() -> io::Result<()> { cmd_markdown(&out_dir, cmd, stdout)?; } - // Generate SUMMARY.md. - let summary: String = output - .iter() - .map(|(cmd, _)| cmd_summary(None, cmd, 0)) - .chain(once("\n".to_string())) - .collect(); + // Generate SUMMARY.mdx. + let summary: String = + output.iter().map(|(cmd, _)| cmd_summary(cmd, 0)).chain(once("\n".to_string())).collect(); - write_file(&out_dir.clone().join("SUMMARY.md"), &summary)?; + println!("Writing SUMMARY.mdx to \"{}\"", out_dir.to_string_lossy()); + write_file(&out_dir.join("SUMMARY.mdx"), &summary)?; // Generate README.md. if args.readme { - let path = &out_dir.join("README.md"); + let path = &out_dir.join("README.mdx"); if args.verbose { - println!("Writing README.md to \"{}\"", path.to_string_lossy()); + println!("Writing README.mdx to \"{}\"", path.to_string_lossy()); } write_file(path, README)?; } - // Generate root SUMMARY.md. + // Generate root SUMMARY.mdx. if args.root_summary { - let root_summary: String = output - .iter() - .map(|(cmd, _)| { - let root_path = pathdiff::diff_paths(&out_dir, &args.root_dir); - cmd_summary(root_path, cmd, args.root_indentation) - }) - .collect(); + let root_summary: String = + output.iter().map(|(cmd, _)| cmd_summary(cmd, args.root_indentation)).collect(); let path = Path::new(args.root_dir.as_str()); if args.verbose { println!("Updating root summary in \"{}\"", path.to_string_lossy()); } + // TODO: This is where we update the cli reference sidebar.ts update_root_summary(path, &root_summary)?; } @@ -213,8 +188,7 @@ fn parse_sub_commands(s: &str) -> Vec { .lines() .take_while(|line| !line.starts_with("Options:") && !line.starts_with("Arguments:")) .filter_map(|line| { - re.captures(line) - .and_then(|cap| cap.get(1).map(|m| m.as_str().to_string())) + re.captures(line).and_then(|cap| cap.get(1).map(|m| m.as_str().to_string())) }) .filter(|cmd| cmd != "help") .map(String::from) @@ -229,7 +203,7 @@ fn cmd_markdown(out_dir: &Path, cmd: &Cmd, stdout: &str) -> io::Result<()> { let out_path = out_dir.join(cmd.to_string().replace(" ", "/")); fs::create_dir_all(out_path.parent().unwrap())?; - write_file(&out_path.with_extension("md"), &out)?; + write_file(&out_path.with_extension("mdx"), &out)?; Ok(()) } @@ -257,48 +231,20 @@ fn parse_description(s: &str) -> (&str, &str) { } /// Returns the summary for a command and its subcommands. -fn cmd_summary(md_root: Option, cmd: &Cmd, indent: usize) -> String { +fn cmd_summary(cmd: &Cmd, indent: usize) -> String { let cmd_s = cmd.to_string(); let cmd_path = cmd_s.replace(" ", "/"); - let full_cmd_path = match md_root { - None => cmd_path, - Some(md_root) => format!("{}/{}", md_root.to_string_lossy(), cmd_path), - }; let indent_string = " ".repeat(indent + (cmd.subcommands.len() * 2)); - format!("{}- [`{}`](./{}.md)\n", indent_string, cmd_s, full_cmd_path) + format!("{}- [`{}`](/cli/{})\n", indent_string, cmd_s, cmd_path) } -/// Replaces the CLI_REFERENCE section in the root SUMMARY.md file. +/// Overwrites the root SUMMARY.mdx file with the generated content. fn update_root_summary(root_dir: &Path, root_summary: &str) -> io::Result<()> { - let summary_file = root_dir.join("SUMMARY.md"); - let original_summary_content = fs::read_to_string(&summary_file)?; - - let section_re = regex!(&format!(r"(?s)\s*{SECTION_START}.*?{SECTION_END}")); - if !section_re.is_match(&original_summary_content) { - eprintln!( - "Could not find CLI_REFERENCE section in {}. Please add the following section to the file:\n{}\n... CLI Reference goes here ...\n\n{}", - summary_file.display(), - SECTION_START, - SECTION_END - ); - process::exit(1); - } + let summary_file = root_dir.join("vocs/docs/pages/cli/SUMMARY.mdx"); + println!("Overwriting {}", summary_file.display()); - let section_end_re = regex!(&format!(r".*{SECTION_END}")); - let last_line = section_end_re - .find(&original_summary_content) - .map(|m| m.as_str().to_string()) - .expect("Could not extract last line of CLI_REFERENCE section"); - - let root_summary_s = root_summary.trim_end().replace("\n\n", "\n"); - let replace_with = format!(" {}\n{}\n{}", SECTION_START, root_summary_s, last_line); - - let new_root_summary = section_re - .replace(&original_summary_content, replace_with.as_str()) - .to_string(); - - let mut root_summary_file = File::create(&summary_file)?; - root_summary_file.write_all(new_root_summary.as_bytes()) + // Simply write the root summary content to the file + write_file(&summary_file, root_summary) } /// Preprocesses the help output of a command. @@ -323,6 +269,11 @@ fn preprocess_help(s: &str) -> Cow<'_, str> { r"(rpc.max-tracing-requests \n.*\n.*\n.*\n.*\n.*)\[default: \d+\]", r"$1[default: ]", ), + // Handle engine.reserved-cpu-cores dynamic default + ( + r"(engine\.reserved-cpu-cores.*)\[default: \d+\]", + r"$1[default: ]", + ), ]; patterns .iter() @@ -349,17 +300,11 @@ struct Cmd<'a> { impl<'a> Cmd<'a> { fn command_name(&self) -> &str { - self.cmd - .file_name() - .and_then(|os_str| os_str.to_str()) - .expect("Expect valid command") + self.cmd.file_name().and_then(|os_str| os_str.to_str()).expect("Expect valid command") } fn new(cmd: &'a PathBuf) -> Self { - Self { - cmd, - subcommands: Vec::new(), - } + Self { cmd, subcommands: Vec::new() } } } diff --git a/docs/cli/update.sh b/docs/cli/update.sh new file mode 100755 index 00000000000..b75dbd789af --- /dev/null +++ b/docs/cli/update.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -eo pipefail + +DOCS_ROOT="$(dirname "$(dirname "$0")")" +RETH=${1:-"$(dirname "$DOCS_ROOT")/target/debug/reth"} +VOCS_PAGES_ROOT="$DOCS_ROOT/vocs/docs/pages" +echo "Generating CLI documentation for reth at $RETH" + +echo "Using docs root: $DOCS_ROOT" +echo "Using vocs pages root: $VOCS_PAGES_ROOT" +cmd=( + "$(dirname "$0")/help.rs" + --root-dir "$DOCS_ROOT/" + --root-indentation 2 + --root-summary + --verbose + --out-dir "$VOCS_PAGES_ROOT/cli/" + "$RETH" +) +echo "Running: $" "${cmd[*]}" +"${cmd[@]}" diff --git a/docs/crates/db.md b/docs/crates/db.md index 688f7ea76cc..abaa1c83bbb 100644 --- a/docs/crates/db.md +++ b/docs/crates/db.md @@ -30,7 +30,7 @@ pub trait Value: Compress + Decompress + Serialize {} ``` -The `Table` trait has two generic values, `Key` and `Value`, which need to implement the `Key` and `Value` traits, respectively. The `Encode` trait is responsible for transforming data into bytes so it can be stored in the database, while the `Decode` trait transforms the bytes back into its original form. Similarly, the `Compress` and `Decompress` traits transform the data to and from a compressed format when storing or reading data from the database. +The `Table` trait has two generic values, `Key` and `Value`, which need to implement the `Key` and `Value` traits, respectively. The `Encode` trait is responsible for transforming data into bytes so it can be stored in the database, while the `Decode` trait transforms the bytes back into their original form. Similarly, the `Compress` and `Decompress` traits transform the data to and from a compressed format when storing or reading data from the database. There are many tables within the node, all used to store different types of data from `Headers` to `Transactions` and more. Below is a list of all of the tables. You can follow [this link](https://github.com/paradigmxyz/reth/blob/bf9cac7571f018fec581fe3647862dab527aeafb/crates/storage/db/src/tables/mod.rs#L274-L414) if you would like to see the table definitions for any of the tables below. @@ -67,7 +67,7 @@ There are many tables within the node, all used to store different types of data ## Database -Reth's database design revolves around it's main [Database trait](https://github.com/paradigmxyz/reth/blob/bf9cac7571f018fec581fe3647862dab527aeafb/crates/storage/db-api/src/database.rs#L8-L52), which implements the database's functionality across many types. Let's take a quick look at the `Database` trait and how it works. +Reth's database design revolves around its main [Database trait](https://github.com/paradigmxyz/reth/blob/bf9cac7571f018fec581fe3647862dab527aeafb/crates/storage/db-api/src/database.rs#L8-L52), which implements the database's functionality across many types. Let's take a quick look at the `Database` trait and how it works. [File: crates/storage/db-api/src/database.rs](https://github.com/paradigmxyz/reth/blob/bf9cac7571f018fec581fe3647862dab527aeafb/crates/storage/db-api/src/database.rs#L8-L52) @@ -196,7 +196,7 @@ pub trait DbTxMut: Send + Sync { + Send + Sync; - /// Put value to database + /// Put value in database fn put(&self, key: T::Key, value: T::Value) -> Result<(), DatabaseError>; /// Delete value from database fn delete(&self, key: T::Key, value: Option) @@ -243,7 +243,7 @@ fn get(&self, key: T::Key) -> Result, DatabaseError>; This design pattern is very powerful and allows Reth to use the methods available to the `DbTx` and `DbTxMut` traits without having to define implementation blocks for each table within the database. -Let's take a look at a couple examples before moving on. In the snippet below, the `DbTxMut::put()` method is used to insert values into the `CanonicalHeaders`, `Headers` and `HeaderNumbers` tables. +Let's take a look at a couple of examples before moving on. In the snippet below, the `DbTxMut::put()` method is used to insert values into the `CanonicalHeaders`, `Headers` and `HeaderNumbers` tables. [File: crates/storage/provider/src/providers/database/provider.rs](https://github.com/paradigmxyz/reth/blob/bf9cac7571f018fec581fe3647862dab527aeafb/crates/storage/provider/src/providers/database/provider.rs#L2606-L2745) @@ -254,9 +254,9 @@ self.tx.put::(block.hash(), block_number)?; ``` Let's take a look at the `DatabaseProviderRW` struct, which is used to create a mutable transaction to interact with the database. -The `DatabaseProviderRW` struct implements the `Deref` and `DerefMut` trait, which returns a reference to its first field, which is a `TxMut`. Recall that `TxMut` is a generic type on the `Database` trait, which is defined as `type TXMut: DbTxMut + DbTx + Send + Sync;`, giving it access to all of the functions available to `DbTx`, including the `DbTx::get()` function. +The `DatabaseProviderRW` struct implements the `Deref` and `DerefMut` traits, which return a reference to its first field, which is a `TxMut`. Recall that `TxMut` is a generic type on the `Database` trait, which is defined as `type TXMut: DbTxMut + DbTx + Send + Sync;`, giving it access to all of the functions available to `DbTx`, including the `DbTx::get()` function. -This next example uses the `DbTx::cursor()` method to get a `Cursor`. The `Cursor` type provides a way to traverse through rows in a database table, one row at a time. A cursor enables the program to perform an operation (updating, deleting, etc) on each row in the table individually. The following code snippet gets a cursor for a few different tables in the database. +This next example uses the `DbTx::cursor_read()` method to get a `Cursor`. The `Cursor` type provides a way to traverse through rows in a database table, one row at a time. A cursor enables the program to perform an operation (updating, deleting, etc) on each row in the table individually. The following code snippet gets a cursor for a few different tables in the database. [File: crates/static-file/static-file/src/segments/headers.rs](https://github.com/paradigmxyz/reth/blob/bf9cac7571f018fec581fe3647862dab527aeafb/crates/static-file/static-file/src/segments/headers.rs#L22-L58) @@ -267,7 +267,7 @@ let mut headers_cursor = provider.tx_ref().cursor_read::()?; let headers_walker = headers_cursor.walk_range(block_range.clone())?; ``` -Let's look at an examples of how cursors are used. The code snippet below contains the `unwind` method from the `BodyStage` defined in the `stages` crate. This function is responsible for unwinding any changes to the database if there is an error when executing the body stage within the Reth pipeline. +Let's look at an example of how cursors are used. The code snippet below contains the `unwind` method from the `BodyStage` defined in the `stages` crate. This function is responsible for unwinding any changes to the database if there is an error when executing the body stage within the Reth pipeline. [File: crates/stages/stages/src/stages/bodies.rs](https://github.com/paradigmxyz/reth/blob/bf9cac7571f018fec581fe3647862dab527aeafb/crates/stages/stages/src/stages/bodies.rs#L267-L345) @@ -301,12 +301,7 @@ fn unwind(&mut self, provider: &DatabaseProviderRW, input: UnwindInput) { withdrawals_cursor.delete_current()?; } - // Delete the requests entry if any - if requests_cursor.seek_exact(number)?.is_some() { - requests_cursor.delete_current()?; - } - - // Delete all transaction to block values. + // Delete all transactions to block values. if !block_meta.is_empty() && tx_block_cursor.seek_exact(block_meta.last_tx_num())?.is_some() { diff --git a/docs/crates/eth-wire.md b/docs/crates/eth-wire.md index 7ab87e914b2..cf62ab143e8 100644 --- a/docs/crates/eth-wire.md +++ b/docs/crates/eth-wire.md @@ -1,6 +1,6 @@ # eth-wire -The `eth-wire` crate provides abstractions over the [``RLPx``](https://github.com/ethereum/devp2p/blob/master/rlpx.md) and +The `eth-wire` crate provides abstractions over the [`RLPx`](https://github.com/ethereum/devp2p/blob/master/rlpx.md) and [Eth wire](https://github.com/ethereum/devp2p/blob/master/caps/eth.md) protocols. This crate can be thought of as having 2 components: @@ -9,48 +9,70 @@ This crate can be thought of as having 2 components: 2. Abstractions over Tokio Streams that operate on these types. (Note that ECIES is implemented in a separate `reth-ecies` crate.) +Additionally, this crate focuses on stream implementations (P2P and Eth), handshakes, and multiplexing. The protocol +message types and RLP encoding/decoding live in the separate `eth-wire-types` crate and are re-exported by `eth-wire` +for convenience. ## Types -The most basic Eth-wire type is an `ProtocolMessage`. It describes all messages that reth can send/receive. +The most basic Eth-wire type is a `ProtocolMessage`. It describes all messages that reth can send/receive. -[File: crates/net/eth-wire/src/types/message.rs](https://github.com/paradigmxyz/reth/blob/1563506aea09049a85e5cc72c2894f3f7a371581/crates/net/eth-wire/src/types/message.rs) +[File: crates/net/eth-wire-types/src/message.rs](../../crates/net/eth-wire-types/src/message.rs) ```rust, ignore /// An `eth` protocol message, containing a message ID and payload. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct ProtocolMessage { +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ProtocolMessage { pub message_type: EthMessageID, - pub message: EthMessage, + pub message: EthMessage, } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub enum EthMessage { - Status(Status), +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum EthMessage { + Status(StatusMessage), NewBlockHashes(NewBlockHashes), - Transactions(Transactions), - NewPooledTransactionHashes(NewPooledTransactionHashes), + NewBlock(Box), + Transactions(Transactions), + NewPooledTransactionHashes66(NewPooledTransactionHashes66), + NewPooledTransactionHashes68(NewPooledTransactionHashes68), GetBlockHeaders(RequestPair), - // ... + BlockHeaders(RequestPair>), + GetBlockBodies(RequestPair), + BlockBodies(RequestPair>), + GetPooledTransactions(RequestPair), + PooledTransactions(RequestPair>), + GetNodeData(RequestPair), + NodeData(RequestPair), GetReceipts(RequestPair), - Receipts(RequestPair), + Receipts(RequestPair>), + Receipts69(RequestPair>), + BlockRangeUpdate(BlockRangeUpdate), } /// Represents message IDs for eth protocol messages. #[repr(u8)] -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum EthMessageID { Status = 0x00, NewBlockHashes = 0x01, Transactions = 0x02, - // ... + GetBlockHeaders = 0x03, + BlockHeaders = 0x04, + GetBlockBodies = 0x05, + BlockBodies = 0x06, + NewBlock = 0x07, + NewPooledTransactionHashes = 0x08, + GetPooledTransactions = 0x09, + PooledTransactions = 0x0a, + GetNodeData = 0x0d, NodeData = 0x0e, GetReceipts = 0x0f, Receipts = 0x10, + BlockRangeUpdate = 0x11, } ``` Messages can either be broadcast to the network, or can be a request/response message to a single peer. This 2nd type of message is described using a `RequestPair` struct, which is simply a concatenation of the underlying message with a request id. -[File: crates/net/eth-wire/src/types/message.rs](https://github.com/paradigmxyz/reth/blob/1563506aea09049a85e5cc72c2894f3f7a371581/crates/net/eth-wire/src/types/message.rs) +[File: crates/net/eth-wire-types/src/message.rs](../../crates/net/eth-wire-types/src/message.rs) ```rust, ignore #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct RequestPair { @@ -59,10 +81,8 @@ pub struct RequestPair { } ``` -Every `Ethmessage` has a corresponding rust struct that implements the `Encodable` and `Decodable` traits. -These traits are defined as follows: - -[Crate: crates/rlp](https://github.com/paradigmxyz/reth/tree/1563506aea09049a85e5cc72c2894f3f7a371581/crates/rlp) +Every `EthMessage` has a corresponding Rust struct that implements `alloy_rlp::Encodable` and `alloy_rlp::Decodable` +(often via derive macros like `RlpEncodable`/`RlpDecodable`). These traits are defined in `alloy_rlp`: ```rust, ignore pub trait Decodable: Sized { fn decode(buf: &mut &[u8]) -> alloy_rlp::Result; @@ -72,20 +92,21 @@ pub trait Encodable { fn length(&self) -> usize; } ``` -These traits describe how the `Ethmessage` should be serialized/deserialized into raw bytes using the RLP format. -In reth all [RLP](https://ethereum.org/en/developers/docs/data-structures-and-encoding/rlp/) encode/decode operations are handled by the `common/rlp` and `common/rlp-derive` crates. +These traits describe how the `EthMessage` should be serialized/deserialized into raw bytes using the RLP format. +In reth all [RLP](https://ethereum.org/en/developers/docs/data-structures-and-encoding/rlp/) encode/decode operations are handled by `alloy_rlp` and the derive macros used in `eth-wire-types`. -Note that the `ProtocolMessage` itself implements these traits, so any stream of bytes can be converted into it by calling `ProtocolMessage::decode()` and vice versa with `ProtocolMessage::encode()`. The message type is determined by the first byte of the byte stream. +Note: `ProtocolMessage` implements `Encodable`, while decoding is performed via +`ProtocolMessage::decode_message(version, &mut bytes)` because decoding must respect the negotiated `EthVersion`. ### Example: The Transactions message -Let's understand how an `EthMessage` is implemented by taking a look at the `Transactions` Message. The eth specification describes a Transaction message as a list of RLP encoded transactions: +Let's understand how an `EthMessage` is implemented by taking a look at the `Transactions` Message. The eth specification describes a Transaction message as a list of RLP-encoded transactions: [File: ethereum/devp2p/caps/eth.md](https://github.com/ethereum/devp2p/blob/master/caps/eth.md#transactions-0x02) ``` Transactions (0x02) [tx₁, tx₂, ...] -Specify transactions that the peer should make sure is included on its transaction queue. +Specify transactions that the peer should make sure are included in its transaction queue. The items in the list are transactions in the format described in the main Ethereum specification. ... @@ -93,17 +114,17 @@ The items in the list are transactions in the format described in the main Ether In reth, this is represented as: -[File: crates/net/eth-wire/src/types/broadcast.rs](https://github.com/paradigmxyz/reth/blob/1563506aea09049a85e5cc72c2894f3f7a371581/crates/net/eth-wire/src/types/broadcast.rs) +[File: crates/net/eth-wire-types/src/broadcast.rs](../../crates/net/eth-wire-types/src/broadcast.rs) ```rust,ignore -pub struct Transactions( +pub struct Transactions( /// New transactions for the peer to include in its mempool. - pub Vec, + pub Vec, ); ``` -And the corresponding trait implementations are present in the primitives crate. +And the corresponding transaction type is defined here: -[File: crates/primitives/src/transaction/mod.rs](https://github.com/paradigmxyz/reth/blob/1563506aea09049a85e5cc72c2894f3f7a371581/crates/primitives/src/transaction/mod.rs) +[File: crates/ethereum/primitives/src/transaction.rs](../../crates/ethereum/primitives/src/transaction.rs) ```rust, ignore #[reth_codec] #[derive(Debug, Clone, PartialEq, Eq, Hash, AsRef, Deref, Default, Serialize, Deserialize)] @@ -138,7 +159,7 @@ Now that we know how the types work, let's take a look at how these are utilized ## P2PStream The lowest level stream to communicate with other peers is the P2P stream. It takes an underlying Tokio stream and does the following: -- Tracks and Manages Ping and pong messages and sends them when needed. +- Tracks and Manages Ping and Pong messages and sends them when needed. - Keeps track of the SharedCapabilities between the reth node and its peers. - Receives bytes from peers, decompresses and forwards them to its parent stream. - Receives bytes from its parent stream, compresses them and sends it to peers. @@ -146,7 +167,7 @@ The lowest level stream to communicate with other peers is the P2P stream. It ta Decompression/Compression of bytes is done with snappy algorithm ([EIP 706](https://eips.ethereum.org/EIPS/eip-706)) using the external `snap` crate. -[File: crates/net/eth-wire/src/p2pstream.rs](https://github.com/paradigmxyz/reth/blob/1563506aea09049a85e5cc72c2894f3f7a371581/crates/net/eth-wire/src/p2pstream.rs) +[File: crates/net/eth-wire/src/p2pstream.rs](../../crates/net/eth-wire/src/p2pstream.rs) ```rust,ignore #[pin_project] pub struct P2PStream { @@ -155,23 +176,29 @@ pub struct P2PStream { encoder: snap::raw::Encoder, decoder: snap::raw::Decoder, pinger: Pinger, - shared_capability: SharedCapability, + /// Negotiated shared capabilities + shared_capabilities: SharedCapabilities, + /// Outgoing messages buffered for sending to the underlying stream. outgoing_messages: VecDeque, + /// Maximum number of messages that can be buffered before yielding backpressure. + outgoing_message_buffer_capacity: usize, + /// Whether this stream is currently in the process of gracefully disconnecting. disconnecting: bool, } ``` ### Pinger -To manage pinging, an instance of the `Pinger` struct is used. This is a state machine which keeps track of how many pings -we have sent/received and the timeouts associated with them. +To manage pinging, an instance of the `Pinger` struct is used. This is a state machine that keeps track of pings +we have sent/received and the timeout associated with them. -[File: crates/net/eth-wire/src/pinger.rs](https://github.com/paradigmxyz/reth/blob/1563506aea09049a85e5cc72c2894f3f7a371581/crates/net/eth-wire/src/pinger.rs) +[File: crates/net/eth-wire/src/pinger.rs](../../crates/net/eth-wire/src/pinger.rs) ```rust,ignore #[derive(Debug)] pub(crate) struct Pinger { /// The timer used for the next ping. ping_interval: Interval, - /// The timer used for the next ping. + /// The timer used to detect a ping timeout. timeout_timer: Pin>, + /// The timeout duration for each ping. timeout: Duration, state: PingState, } @@ -205,7 +232,7 @@ pub(crate) fn poll_ping( } } PingState::WaitingForPong => { - if self.timeout_timer.is_elapsed() { + if self.timeout_timer.as_mut().poll(cx).is_ready() { self.state = PingState::TimedOut; return Poll::Ready(Ok(PingerEvent::Timeout)) } @@ -218,12 +245,12 @@ pub(crate) fn poll_ping( ``` ### Sending and receiving data -To send and receive data, the P2PStream itself is a future which implements the `Stream` and `Sink` traits from the `futures` crate. +To send and receive data, the P2PStream itself is a future that implements the `Stream` and `Sink` traits from the `futures` crate. For the `Stream` trait, the `inner` stream is polled, decompressed and returned. Most of the code is just error handling and is omitted here for clarity. -[File: crates/net/eth-wire/src/p2pstream.rs](https://github.com/paradigmxyz/reth/blob/1563506aea09049a85e5cc72c2894f3f7a371581/crates/net/eth-wire/src/p2pstream.rs) +[File: crates/net/eth-wire/src/p2pstream.rs](../../crates/net/eth-wire/src/p2pstream.rs) ```rust,ignore impl Stream for P2PStream { @@ -240,7 +267,8 @@ impl Stream for P2PStream { let mut decompress_buf = BytesMut::zeroed(decompressed_len + 1); this.decoder.decompress(&bytes[1..], &mut decompress_buf[1..])?; // ... Omitted Error handling - decompress_buf[0] = bytes[0] - this.shared_capability.offset(); + // Normalize IDs: reserved p2p range is 0x00..=0x0f; subprotocols start at 0x10 + decompress_buf[0] = bytes[0] - MAX_RESERVED_MESSAGE_ID - 1; return Poll::Ready(Some(Ok(decompress_buf))) } } @@ -250,7 +278,7 @@ impl Stream for P2PStream { Similarly, for the `Sink` trait, we do the reverse, compressing and sending data out to the `inner` stream. The important functions in this trait are shown below. -[File: crates/net/eth-wire/src/p2pstream.rs](https://github.com/paradigmxyz/reth/blob/1563506aea09049a85e5cc72c2894f3f7a371581/crates/net/eth-wire/src/p2pstream.rs) +[File: crates/net/eth-wire/src/p2pstream.rs](../../crates/net/eth-wire/src/p2pstream.rs) ```rust, ignore impl Sink for P2PStream { fn start_send(self: Pin<&mut Self>, item: Bytes) -> Result<(), Self::Error> { @@ -258,7 +286,8 @@ impl Sink for P2PStream { let mut compressed = BytesMut::zeroed(1 + snap::raw::max_compress_len(item.len() - 1)); let compressed_size = this.encoder.compress(&item[1..], &mut compressed[1..])?; compressed.truncate(compressed_size + 1); - compressed[0] = item[0] + this.shared_capability.offset(); + // Mask subprotocol IDs into global space above reserved p2p IDs + compressed[0] = item[0] + MAX_RESERVED_MESSAGE_ID + 1; this.outgoing_messages.push_back(compressed.freeze()); Ok(()) } @@ -285,9 +314,9 @@ impl Sink for P2PStream { ## EthStream -The EthStream is very simple, it does not keep track of any state, it simply wraps the P2Pstream. +The EthStream wraps a stream and handles eth message (RLP) encoding/decoding with respect to the negotiated `EthVersion`. -[File: crates/net/eth-wire/src/ethstream.rs](https://github.com/paradigmxyz/reth/blob/1563506aea09049a85e5cc72c2894f3f7a371581/crates/net/eth-wire/src/ethstream.rs) +[File: crates/net/eth-wire/src/ethstream.rs](../../crates/net/eth-wire/src/ethstream.rs) ```rust,ignore #[pin_project] pub struct EthStream { @@ -295,10 +324,10 @@ pub struct EthStream { inner: S, } ``` -EthStream's only job is to perform the RLP decoding/encoding, using the `ProtocolMessage::decode()` and `ProtocolMessage::encode()` -functions we looked at earlier. +EthStream performs RLP decoding/encoding using `ProtocolMessage::decode_message(version, &mut bytes)` +and `ProtocolMessage::encode()`, and enforces protocol rules (e.g., prohibiting `Status` after handshake). -[File: crates/net/eth-wire/src/ethstream.rs](https://github.com/paradigmxyz/reth/blob/1563506aea09049a85e5cc72c2894f3f7a371581/crates/net/eth-wire/src/ethstream.rs) +[File: crates/net/eth-wire/src/ethstream.rs](../../crates/net/eth-wire/src/ethstream.rs) ```rust,ignore impl Stream for EthStream { // ... @@ -306,7 +335,7 @@ impl Stream for EthStream { let this = self.project(); let bytes = ready!(this.inner.poll_next(cx)).unwrap(); // ... - let msg = match ProtocolMessage::decode(&mut bytes.as_ref()) { + let msg = match ProtocolMessage::decode_message(self.version(), &mut bytes.as_ref()) { Ok(m) => m, Err(err) => { return Poll::Ready(Some(Err(err.into()))) @@ -319,10 +348,12 @@ impl Stream for EthStream { impl Sink for EthStream { // ... fn start_send(self: Pin<&mut Self>, item: EthMessage) -> Result<(), Self::Error> { - // ... + if matches!(item, EthMessage::Status(_)) { + let _ = self.project().inner.disconnect(DisconnectReason::ProtocolBreach); + return Err(EthStreamError::EthHandshakeError(EthHandshakeError::StatusNotInHandshake)) + } let mut bytes = BytesMut::new(); ProtocolMessage::from(item).encode(&mut bytes); - let bytes = bytes.freeze(); self.project().inner.start_send(bytes)?; Ok(()) @@ -334,14 +365,14 @@ impl Sink for EthStream { } ``` ## Unauthed streams -For a session to be established, peers in the Ethereum network must first exchange a `Hello` message in the ``RLPx`` layer and then a +For a session to be established, peers in the Ethereum network must first exchange a `Hello` message in the `RLPx` layer and then a `Status` message in the eth-wire layer. To perform these, reth has special `Unauthed` versions of streams described above. -The `UnauthedP2Pstream` does the `Hello` handshake and returns a `P2PStream`. +The `UnauthedP2PStream` does the `Hello` handshake and returns a `P2PStream`. -[File: crates/net/eth-wire/src/p2pstream.rs](https://github.com/paradigmxyz/reth/blob/1563506aea09049a85e5cc72c2894f3f7a371581/crates/net/eth-wire/src/p2pstream.rs) +[File: crates/net/eth-wire/src/p2pstream.rs](../../crates/net/eth-wire/src/p2pstream.rs) ```rust, ignore #[pin_project] pub struct UnauthedP2PStream { @@ -351,8 +382,8 @@ pub struct UnauthedP2PStream { impl UnauthedP2PStream { // ... - pub async fn handshake(mut self, hello: HelloMessage) -> Result<(P2PStream, HelloMessage), Error> { - self.inner.send(alloy_rlp::encode(P2PMessage::Hello(hello.clone())).into()).await?; + pub async fn handshake(mut self, hello: HelloMessageWithProtocols) -> Result<(P2PStream, HelloMessage), Error> { + self.inner.send(alloy_rlp::encode(P2PMessage::Hello(hello.message())).into()).await?; let first_message_bytes = tokio::time::timeout(HANDSHAKE_TIMEOUT, self.inner.next()).await; let their_hello = match P2PMessage::decode(&mut &first_message_bytes[..]) { @@ -360,11 +391,25 @@ impl UnauthedP2PStream { // ... } }?; - let stream = P2PStream::new(self.inner, capability); + let stream = P2PStream::new(self.inner, shared_capabilities); Ok((stream, their_hello)) } } ``` -Similarly, UnauthedEthStream does the `Status` handshake and returns an `EthStream`. The code is [here](https://github.com/paradigmxyz/reth/blob/1563506aea09049a85e5cc72c2894f3f7a371581/crates/net/eth-wire/src/ethstream.rs) +Similarly, `UnauthedEthStream` does the `Status` handshake and returns an `EthStream`. It accepts a `UnifiedStatus` +and a `ForkFilter`, and provides a timeout wrapper. The code is [here](../../crates/net/eth-wire/src/ethstream.rs) + +### Multiplexing and satellites + +`eth-wire` also provides `RlpxProtocolMultiplexer`/`RlpxSatelliteStream` to run the primary `eth` protocol alongside +additional "satellite" protocols (e.g. `snap`) using negotiated `SharedCapabilities`. + +## Message variants and versions + +- `NewPooledTransactionHashes` differs between ETH66 (`NewPooledTransactionHashes66`) and ETH68 (`NewPooledTransactionHashes68`). +- Starting with ETH67, `GetNodeData` and `NodeData` are removed (decoding them for >=67 yields an error). +- Starting with ETH69: + - `BlockRangeUpdate (0x11)` announces the historical block range served. + - Receipts omit bloom: encoded as `Receipts69` instead of `Receipts`. diff --git a/docs/crates/network.md b/docs/crates/network.md index a35b0c9de90..9aa112b17ef 100644 --- a/docs/crates/network.md +++ b/docs/crates/network.md @@ -215,7 +215,7 @@ pub struct NetworkManager { /// Sender half to send events to the /// [`EthRequestHandler`](crate::eth_requests::EthRequestHandler) task, if configured. to_eth_request_handler: Option>, - /// Tracks the number of active session (connected peers). + /// Tracks the number of active sessions (connected peers). /// /// This is updated via internal events and shared via `Arc` with the [`NetworkHandle`] /// Updated by the `NetworkWorker` and loaded by the `NetworkService`. @@ -400,7 +400,7 @@ pub struct BodiesDownloader { } ``` -Here, similarly, a `FetchClient` is passed in to the `client` field, and the `get_block_bodies` method it implements is used when constructing the stream created by the `BodiesDownloader` in the `execute` method of the `BodyStage`. +Here, similarly, a `FetchClient` is passed into the `client` field, and the `get_block_bodies` method it implements is used when constructing the stream created by the `BodiesDownloader` in the `execute` method of the `BodyStage`. [File: crates/net/downloaders/src/bodies/bodies.rs](https://github.com/paradigmxyz/reth/blob/1563506aea09049a85e5cc72c2894f3f7a371581/crates/net/downloaders/src/bodies/bodies.rs) ```rust,ignore @@ -494,6 +494,7 @@ fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { } IncomingEthRequest::GetNodeData { .. } => {} IncomingEthRequest::GetReceipts { .. } => {} + IncomingEthRequest::GetReceipts69 { .. } => {} }, } } @@ -656,9 +657,9 @@ pub struct TransactionsManager { pool: Pool, /// Network access. network: NetworkHandle, - /// Subscriptions to all network related events. + /// Subscriptions to all network-related events. /// - /// From which we get all new incoming transaction related messages. + /// From which we get all new incoming transaction-related messages. network_events: UnboundedReceiverStream, /// All currently active requests for pooled transactions. inflight_requests: Vec, @@ -695,7 +696,7 @@ pub struct TransactionsHandle { ### Input Streams to the Transactions Task We'll touch on most of the fields in the `TransactionsManager` as the chapter continues, but some worth noting now are the 4 streams from which inputs to the task are fed: -- `transaction_events`: A listener for `NetworkTransactionEvent`s sent from the `NetworkManager`, which consist solely of events related to transactions emitted by the network. +- `transaction_events`: A listener for `NetworkTransactionEvent`s sent from the `NetworkManager`, which consists solely of events related to transactions emitted by the network. - `network_events`: A listener for `NetworkEvent`s sent from the `NetworkManager`, which consist of other "meta" events such as sessions with peers being established or closed. - `command_rx`: A listener for `TransactionsCommand`s sent from the `TransactionsHandle` - `pending`: A listener for new pending transactions added to the `TransactionPool` @@ -1120,7 +1121,7 @@ It iterates over `TransactionsManager.pool_imports`, polling each one, and if it `on_good_import`, called when the transaction was successfully imported into the transaction pool, removes the entry for the given transaction hash from `TransactionsManager.transactions_by_peers`. -`on_bad_import` also removes the entry for the given transaction hash from `TransactionsManager.transactions_by_peers`, but also calls `report_bad_message` for each peer in the entry, decreasing all of their reputation scores as they were propagating a transaction that could not validated. +`on_bad_import` also removes the entry for the given transaction hash from `TransactionsManager.transactions_by_peers`, but also calls `report_bad_message` for each peer in the entry, decreasing all of their reputation scores as they were propagating a transaction that could not be validated. #### Checking on `pending_transactions` diff --git a/docs/crates/stages.md b/docs/crates/stages.md index cfa2d5012d5..a6f107c2c0b 100644 --- a/docs/crates/stages.md +++ b/docs/crates/stages.md @@ -1,6 +1,6 @@ # Stages -The `stages` lib plays a central role in syncing the node, maintaining state, updating the database and more. The stages involved in the Reth pipeline are the `HeaderStage`, `BodyStage`, `SenderRecoveryStage`, and `ExecutionStage` (note that this list is non-exhaustive, and more pipeline stages will be added in the near future). Each of these stages are queued up and stored within the Reth pipeline. +The `stages` lib plays a central role in syncing the node, maintaining state, updating the database and more. The stages involved in the Reth pipeline are the `HeaderStage`, `BodyStage`, `SenderRecoveryStage`, and `ExecutionStage` (note that this list is non-exhaustive, and more pipeline stages will be added in the near future). Each of these stages is queued up and stored within the Reth pipeline. When the node is first started, a new `Pipeline` is initialized and all of the stages are added into `Pipeline.stages`. Then, the `Pipeline::run` function is called, which starts the pipeline, executing all of the stages continuously in an infinite loop. This process syncs the chain, keeping everything up to date with the chain tip. @@ -36,7 +36,7 @@ The transactions root is a value that is calculated based on the transactions in When the `BodyStage` is looking at the headers to determine which block to download, it will skip the blocks where the `header.ommers_hash` and the `header.transaction_root` are empty, denoting that the block is empty as well. -Once the `BodyStage` determines which block bodies to fetch, a new `bodies_stream` is created which downloads all of the bodies from the `starting_block`, up until the `target_block` specified. Each time the `bodies_stream` yields a value, a `SealedBlock` is created using the block header, the ommers hash and the newly downloaded block body. +Once the `BodyStage` determines which block bodies to fetch, a new `bodies_stream` is created which downloads all of the bodies from the `starting_block`, up until the `target_block` is specified. Each time the `bodies_stream` yields a value, a `SealedBlock` is created using the block header, the ommers hash and the newly downloaded block body. The new block is then pre-validated, checking that the ommers hash and transactions root in the block header are the same in the block body. Following a successful pre-validation, the `BodyStage` loops through each transaction in the `block.body`, adding the transaction to the database. This process is repeated for every downloaded block body, with the `BodyStage` returning `Ok(ExecOutput { stage_progress, done: true })` signaling it successfully completed. @@ -108,7 +108,7 @@ The `IndexAccountHistoryStage` builds indices for account history, tracking how ## FinishStage -The `FinishStage` is the final stage in the pipeline that performs cleanup and verification tasks. It ensures that all previous stages have completed successfully and that the node's state is consistent. This stage may also update various metrics and status indicators to reflect the completion of a sync cycle. +The `FinishStage` is the final stage in the pipeline that performs cleanup and verification tasks. It ensures that all previous stages have been completed successfully and that the node's state is consistent. This stage may also update various metrics and status indicators to reflect the completion of a sync cycle.
diff --git a/docs/design/README.md b/docs/design/README.md index 7828a42500f..21f95055b0c 100644 --- a/docs/design/README.md +++ b/docs/design/README.md @@ -2,6 +2,7 @@ Docs under this page contain some context on how we've iterated on the Reth design (still WIP, please contribute!): +- [Reth Goals](./goals.md) - [Database](./database.md) - Networking - [P2P](./p2p.md) diff --git a/docs/design/codecs.md b/docs/design/codecs.md deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/docs/design/database.md b/docs/design/database.md index b45c783bc5f..e0874c21551 100644 --- a/docs/design/database.md +++ b/docs/design/database.md @@ -2,28 +2,28 @@ ## Abstractions -- We created a [Database trait abstraction](https://github.com/paradigmxyz/reth/blob/0d9b9a392d4196793736522f3fc2ac804991b45d/crates/interfaces/src/db/mod.rs) using Rust Stable GATs which frees us from being bound to a single database implementation. We currently use MDBX, but are exploring [redb](https://github.com/cberner/redb) as an alternative. -- We then iterated on [`Transaction`](https://github.com/paradigmxyz/reth/blob/0d9b9a392d4196793736522f3fc2ac804991b45d/crates/stages/src/db.rs#L14-L19) as a non-leaky abstraction with helpers for strictly-typed and unit-tested higher-level database abstractions. +- We created a [Database trait abstraction](https://github.com/paradigmxyz/reth/blob/main/crates/cli/commands/src/db/mod.rs) using Rust Stable GATs which frees us from being bound to a single database implementation. We currently use MDBX, but are exploring [redb](https://github.com/cberner/redb) as an alternative. +- We then iterated on [`Transaction`](https://github.com/paradigmxyz/reth/blob/main/crates/storage/errors/src/db.rs) as a non-leaky abstraction with helpers for strictly-typed and unit-tested higher-level database abstractions. ## Codecs - We want Reth's serialized format to be able to trade off read/write speed for size, depending on who the user is. -- To achieve that, we created the [Encode/Decode/Compress/Decompress traits](https://github.com/paradigmxyz/reth/blob/0d9b9a392d4196793736522f3fc2ac804991b45d/crates/interfaces/src/db/table.rs#L9-L36) to make the (de)serialization of database `Table::Key` and `Table::Values` generic. - - This allows for [out-of-the-box benchmarking](https://github.com/paradigmxyz/reth/blob/0d9b9a392d4196793736522f3fc2ac804991b45d/crates/db/benches/encoding_iai.rs#L5) (using [Criterion](https://github.com/bheisler/criterion.rs)) - - It also enables [out-of-the-box fuzzing](https://github.com/paradigmxyz/reth/blob/0d9b9a392d4196793736522f3fc2ac804991b45d/crates/interfaces/src/db/codecs/fuzz/mod.rs) using [trailofbits/test-fuzz](https://github.com/trailofbits/test-fuzz). +- To achieve that, we created the [Encode/Decode/Compress/Decompress traits](https://github.com/paradigmxyz/reth/blob/main/crates/storage/db-api/src/table.rs) to make the (de)serialization of database `Table::Key` and `Table::Values` generic. + - This allows for [out-of-the-box benchmarking](https://github.com/paradigmxyz/reth/blob/main/crates/storage/db/benches/criterion.rs) (using [Criterion](https://github.com/bheisler/criterion.rs)) + - It also enables [out-of-the-box fuzzing](https://github.com/paradigmxyz/reth/blob/main/crates/storage/db-api/src/tables/codecs/fuzz/mod.rs) using [trailofbits/test-fuzz](https://github.com/trailofbits/test-fuzz). - We implemented that trait for the following encoding formats: - - [Ethereum-specific Compact Encoding](https://github.com/paradigmxyz/reth/blob/0d9b9a392d4196793736522f3fc2ac804991b45d/crates/codecs/derive/src/compact/mod.rs): A lot of Ethereum datatypes have unnecessary zeros when serialized, or optional (e.g. on empty hashes) which would be nice not to pay in storage costs. + - [Ethereum-specific Compact Encoding](https://github.com/paradigmxyz/reth/blob/main/crates/storage/codecs/derive/src/compact/mod.rs): A lot of Ethereum datatypes have unnecessary zeros when serialized, or optional (e.g. on empty hashes) which would be nice not to pay in storage costs. - [Erigon](https://github.com/ledgerwatch/erigon/blob/12ee33a492f5d240458822d052820d9998653a63/docs/programmers_guide/db_walkthrough.MD) achieves that by having a `bitfield` set on Table "PlainState which adds a bitfield to Accounts. - [Akula](https://github.com/akula-bft/akula/) expanded it for other tables and datatypes manually. It also saved some more space by storing the length of certain types (U256, u64) using the [`modular_bitfield`](https://docs.rs/modular-bitfield/latest/modular_bitfield/) crate, which compacts this information. - We generalized it for all types, by writing a derive macro that autogenerates code for implementing the trait. It, also generates the interfaces required for fuzzing using ToB/test-fuzz: - [Scale Encoding](https://github.com/paritytech/parity-scale-codec) - [Postcard Encoding](https://github.com/jamesmunns/postcard) - Passthrough (called `no_codec` in the codebase) -- We made implementation of these traits easy via a derive macro called [`reth_codec`](https://github.com/paradigmxyz/reth/blob/0d9b9a392d4196793736522f3fc2ac804991b45d/crates/codecs/derive/src/lib.rs#L15) that delegates to one of Compact (default), Scale, Postcard or Passthrough encoding. This is [derived on every struct we need](https://github.com/search?q=repo%3Aparadigmxyz%2Freth%20%22%23%5Breth_codec%5D%22&type=code), and lets us experiment with different encoding formats without having to modify the entire codebase each time. +- We made implementation of these traits easy via a derive macro called [`reth_codec`](https://github.com/paradigmxyz/reth/blob/main/crates/storage/codecs/derive/src/lib.rs) that delegates to one of Compact (default), Scale, Postcard or Passthrough encoding. This is [derived on every struct we need](https://github.com/search?q=repo%3Aparadigmxyz%2Freth%20%22%23%5Breth_codec%5D%22&type=code), and lets us experiment with different encoding formats without having to modify the entire codebase each time. ### Table layout -Historical state changes are indexed by `BlockNumber`. This means that `reth` stores the state for every account after every block that touched it, and it provides indexes for accessing that data quickly. While this may make the database size bigger (needs benchmark once `reth` is closer to prod). +Historical state changes are indexed by `BlockNumber`. This means that `reth` stores the state for every account after every block that touched it, and it provides indexes for accessing that data quickly. While this may make the database size bigger (needs benchmark once `reth` is closer to prod), it provides fast access to the historical state. Below, you can see the table design that implements this scheme: diff --git a/docs/design/goals.md b/docs/design/goals.md index 819d6ca6fa9..6edfb1282c7 100644 --- a/docs/design/goals.md +++ b/docs/design/goals.md @@ -34,7 +34,7 @@ Why? This is a win for everyone. RPC providers meet more impressive SLAs, MEV se The biggest bottleneck in this pipeline is not the execution of the EVM interpreter itself, but rather in accessing state and managing I/O. As such, we think the largest optimizations to be made are closest to the DB layer. -Ideally, we can achieve such fast runtime operation that we can avoid storing certain things (e.g.?) on the disk, and are able to generate them on the fly, instead - minimizing disk footprint. +Ideally, we can achieve such fast runtime operation that we can avoid storing certain things (e.g., transaction receipts) on the disk, and are able to generate them on the fly, instead - minimizing disk footprint. --- @@ -44,7 +44,7 @@ Ideally, we can achieve such fast runtime operation that we can avoid storing ce **Control over tradeoffs** -Almost any given design choice or optimization to the client comes with its own tradeoffs. As such, our long-term goal is not to make opinionated decisions on behalf of everyone, as some users will be negatively impacted and turned away from what could be a great client. +Almost any given design choice or optimization for the client comes with its own tradeoffs. As such, our long-term goal is not to make opinionated decisions on behalf of everyone, as some users will be negatively impacted and turned away from what could be a great client. **Profiles** @@ -80,4 +80,4 @@ It goes without saying that verbose and thorough documentation is a must. The do **Issue tracking** -Everything that is (and is not) being worked on within the client should be tracked accordingly so that anyone in the community can stay on top of the state of development. This makes it clear what kind of help is needed, and where. \ No newline at end of file +Everything that is (and is not) being worked on within the client should be tracked accordingly so that anyone in the community can stay on top of the state of development. This makes it clear what kind of help is needed, and where. diff --git a/docs/design/headers-downloader.md b/docs/design/headers-downloader.md index 8b160265a2b..c31aeefc249 100644 --- a/docs/design/headers-downloader.md +++ b/docs/design/headers-downloader.md @@ -6,6 +6,6 @@ * First, we implemented the reverse linear download. It received the current chain tip and local head as arguments and requested blocks in batches starting from the tip, and retried on request failure. See [`reth#58`](https://github.com/paradigmxyz/reth/pull/58) and [`reth#119`](https://github.com/paradigmxyz/reth/pull/119). * The first complete implementation of the headers stage was introduced in [`reth#126`](https://github.com/paradigmxyz/reth/pull/126). The stage looked up the local head & queried the consensus for the chain tip and queried the downloader passing them as arguments. After the download finished, the stage would proceed to insert headers in the ascending order by appending the entries to the corresponding tables. * The original downloader was refactored in [`reth#249`](https://github.com/paradigmxyz/reth/pull/249) to return a `Future` which would resolve when either the download is completed or the error occurred during polling. This future kept a pointer to a current request at any time, allowing to retry the request in case of failure. The insert logic of the headers stage remained unchanged. - * NOTE: Up to this point the headers stage awaited full range of blocks (from local head to tip) to be downloaded before proceeding to insert. -* [`reth#296`](https://github.com/paradigmxyz/reth/pull/296) introduced the `Stream` implementation of the download as well as the commit threshold for the headers stage. The `Stream` implementation yields headers as soon as they are received and validated. It dispatches the request for the next header batch until the head is reached. The headers stage now has a configurable commit threshold which allows configuring the insert batch size. With this change, the headers stage no longer waits for the download to be complete, but rather collects the headers from the stream up to the commit threshold parameter. After collecting, the stage proceeds to insert the batch. The process is repeated until the stream is drained. At this point, we populated all tables except for HeadersTD since it has to be computed in a linear ascending order. The stage starts walking the populated headers table and computes & inserts new total difficulty values. -* This header implementation is unique because it is implemented as a Stream, it yields headers as soon as they become available (contrary to waiting for download to complete) and it keeps only one header in buffer (required to form the next header request) . + * NOTE: Up to this point the headers stage awaited the full range of blocks (from local head to tip) to be downloaded before proceeding to insert. +* [`reth#296`](https://github.com/paradigmxyz/reth/pull/296) introduced the `Stream` implementation of the download as well as the commit threshold for the headers stage. The `Stream` implementation yields headers as soon as they are received and validated. It dispatches the request for the next header batch until the head is reached. The headers stage now has a configurable commit threshold which allows configuring the insert batch size. With this change, the headers stage no longer waits for the download to be complete, but rather collects the headers from the stream up to the commit threshold parameter. After collecting, the stage proceeds to insert the batch. The process is repeated until the stream is drained. At this point, we populated all tables except for HeadersTD since it has to be computed in a linear ascending order. The stage starts walking through the populated headers table and computes & inserts new total difficulty values. +* This header implementation is unique because it is implemented as a Stream, it yields headers as soon as they become available (contrary to waiting for download to complete), and it keeps only one header in buffer (required to form the next header request) . diff --git a/docs/design/metrics.md b/docs/design/metrics.md index a769f9d625f..1aeb2f37c1e 100644 --- a/docs/design/metrics.md +++ b/docs/design/metrics.md @@ -13,7 +13,7 @@ The main difference between metrics and traces is therefore that metrics are sys **For most things, you likely want a metric**, except for two scenarios: - For contributors, traces are a good profiling tool -- For end-users that run complicated infrastructure, traces in the RPC component makes sense +- For end-users who run complicated infrastructure, traces in the RPC component make sense ### How to add a metric diff --git a/docs/design/review.md b/docs/design/review.md index 2a3c5c20867..304d3582f5e 100644 --- a/docs/design/review.md +++ b/docs/design/review.md @@ -1,10 +1,10 @@ # Review of other codebases -This document contains some of our research in how other codebases designed various parts of their stack. +This document contains some of our research on how other codebases designed various parts of their stack. ## P2P -* [`Sentry`](https://erigon.gitbook.io/erigon/advanced-usage/sentry), a pluggable p2p node following the [Erigon gRPC architecture](https://erigon.substack.com/p/current-status-of-silkworm-and-silkrpc): +* [`Sentry`](https://erigon.gitbook.io/docs/summary/fundamentals/modules/sentry), a pluggable p2p node following the [Erigon gRPC architecture](https://erigon.substack.com/p/current-status-of-silkworm-and-silkrpc): * [`vorot93`](https://github.com/vorot93/) first started by implementing a rust devp2p stack in [`devp2p`](https://github.com/vorot93/devp2p) * vorot93 then started work on sentry, using devp2p, to satisfy the erigon architecture of modular components connected with gRPC. * The code from rust-ethereum/devp2p was merged into sentry, and rust-ethereum/devp2p was archived @@ -18,15 +18,15 @@ This document contains some of our research in how other codebases designed vari ## Database -* [Erigon's DB walkthrough](https://github.com/ledgerwatch/erigon/blob/12ee33a492f5d240458822d052820d9998653a63/docs/programmers_guide/db_walkthrough.MD) contains an overview. They made the most noticeable improvements on storage reduction. +* [Erigon's DB walkthrough](https://github.com/ledgerwatch/erigon/blob/12ee33a492f5d240458822d052820d9998653a63/docs/programmers_guide/db_walkthrough.MD) contains an overview. They made the most noticeable improvements in storage reduction. * [Gio's erigon-db table macros](https://github.com/gio256/erigon-db) + [Akula's macros](https://github.com/akula-bft/akula/blob/74b172ee1d2d2a4f04ce057b5a76679c1b83df9c/src/kv/tables.rs#L61). ## Header Downloaders * Erigon Header Downloader: - * A header downloader algo was introduced in [`erigon#1016`](https://github.com/ledgerwatch/erigon/pull/1016) and finished in [`erigon#1145`](https://github.com/ledgerwatch/erigon/pull/1145). At a high level, the downloader concurrently requested headers by hash, then sorted, validated and fused the responses into chain segments. Smaller segments were fused into larger as the gaps between them were filled. The downloader is also used to maintain hardcoded hashes (later renamed to preverified) to bootstrap the sync. + * A header downloader algorithm was introduced in [`erigon#1016`](https://github.com/ledgerwatch/erigon/pull/1016) and finished in [`erigon#1145`](https://github.com/ledgerwatch/erigon/pull/1145). At a high level, the downloader concurrently requested headers by hash, then sorted, validated and fused the responses into chain segments. Smaller segments were fused into larger as the gaps between them were filled. The downloader is also used to maintain hardcoded hashes (later renamed to preverified) to bootstrap the sync. * The downloader was refactored multiple times: [`erigon#1471`](https://github.com/ledgerwatch/erigon/pull/1471), [`erigon#1559`](https://github.com/ledgerwatch/erigon/pull/1559) and [`erigon#2035`](https://github.com/ledgerwatch/erigon/pull/2035). - * With PoS transition in [`erigon#3075`](https://github.com/ledgerwatch/erigon/pull/3075) terminal td was introduced to the algo to stop forward syncing. For the downward sync (post merge), the download was now delegated to [`EthBackendServer`](https://github.com/ledgerwatch/erigon/blob/3c95db00788dc740849c2207d886fe4db5a8c473/ethdb/privateapi/ethbackend.go#L245) + * With PoS transition in [`erigon#3075`](https://github.com/ledgerwatch/erigon/pull/3075) terminal td was introduced to the algorithm to stop forward syncing. For the downward sync (post merge), the downloader was now delegated to [`EthBackendServer`](https://github.com/ledgerwatch/erigon/blob/3c95db00788dc740849c2207d886fe4db5a8c473/ethdb/privateapi/ethbackend.go#L245) * Proper reverse PoS downloader was introduced in [`erigon#3092`](https://github.com/ledgerwatch/erigon/pull/3092) which downloads the header batches from tip until local head is reached. Refactored later in [`erigon#3340`](https://github.com/ledgerwatch/erigon/pull/3340) and [`erigon#3717`](https://github.com/ledgerwatch/erigon/pull/3717). * Akula Headers & Stage Downloader: diff --git a/docs/release.md b/docs/release.md index 20ad4141e1e..0e92dc793ac 100644 --- a/docs/release.md +++ b/docs/release.md @@ -26,7 +26,7 @@ It is assumed that the commit that is being considered for release has been mark - [ ] Tag the new commit on main with `vx.y.z` (`git tag vx.y.z SHA`) - [ ] Push the tag (`git push origin vx.y.z`)[^1] - [ ] Update [Homebrew Tap](https://github.com/paradigmxyz/homebrew-brew) -- [ ] Run the release commit on testing infrastructure for 1-3 days to check for inconsistencies and bugs +- [ ] Run the release commit on the testing infrastructure for 1-3 days to check for inconsistencies and bugs - This testing infrastructure is going to sync and keep up with a live testnet, and includes monitoring of bandwidth, CPU, disk space etc. > **Note** @@ -43,6 +43,6 @@ The release artifacts are automatically added to the draft release. Once ready, #### Release summaries -The release summary should include general notes on what the release contains that is important to operators. These changes can be found using the https://github.com/paradigmxyz/reth/labels/M-changelog label. +The release summary should include general notes on what the release contains that are important to operators. These changes can be found using the https://github.com/paradigmxyz/reth/labels/M-changelog label. -[^1]: It is possible to use `git push --tags`, but this is discouraged since it can be very difficult to get rid of bad tags. \ No newline at end of file +[^1]: It is possible to use `git push --tags`, but this is discouraged since it can be very difficult to get rid of bad tags. diff --git a/docs/repo/labels.md b/docs/repo/labels.md index 6772b828ffc..2c830194415 100644 --- a/docs/repo/labels.md +++ b/docs/repo/labels.md @@ -4,7 +4,7 @@ Each label in the repository has a description attached that describes what the There are 7 label categories in the repository: -- **Area labels**: These labels denote the general area of the project an issue or PR affects. These start with [`A-`][area]. +- **Area labels**: These labels denote the general area of the project that an issue or PR affects. These start with [`A-`][area]. - **Category labels**: These labels denote the type of issue or change being made, for example https://github.com/paradigmxyz/reth/labels/C-bug or https://github.com/paradigmxyz/reth/labels/C-enhancement. These start with [`C-`][category]. - **Difficulty labels**: These are reserved for the very easy or very hard issues. Any issue without one of these labels can be considered to be of "average difficulty". They start with [`D-`][difficulty]. - **Meta labels**: These start with [`M-`][meta] and convey meaning to the core contributors, usually about the release process. diff --git a/docs/repo/layout.md b/docs/repo/layout.md index 525405216e1..22aae4c3512 100644 --- a/docs/repo/layout.md +++ b/docs/repo/layout.md @@ -2,7 +2,7 @@ This repository contains several Rust crates that implement the different building blocks of an Ethereum node. The high-level structure of the repository is as follows: -Generally reth is composed of a few components, with supporting crates. The main components can be defined as: +Generally, reth is composed of a few components, with supporting crates. The main components can be defined as: - [Project Layout](#project-layout) - [Documentation](#documentation) @@ -29,7 +29,7 @@ The supporting crates are split into two categories: [primitives](#primitives) a ### Documentation -Contributor documentation is in [`docs`](../../docs) and end-user documentation is in [`book`](../../book). +Contributor documentation is in [`docs`](../../docs). ### Binaries @@ -135,7 +135,7 @@ The IPC transport lives in [`rpc/ipc`](../../crates/rpc/ipc). #### Utilities Crates -- [`rpc/rpc-types-compat`](../../crates/rpc/rpc-types-compat): This crate various helper functions to convert between reth primitive types and rpc types. +- [`rpc/rpc-convert`](../../crates/rpc/rpc-convert): This crate provides various helper functions to convert between reth primitive types and rpc types. - [`rpc/layer`](../../crates/rpc/rpc-layer/): Some RPC middleware layers (e.g. `AuthValidator`, `JwtAuthValidator`) - [`rpc/rpc-testing-util`](../../crates/rpc/rpc-testing-util/): Reth RPC testing helpers diff --git a/docs/vocs/.claude/settings.local.json b/docs/vocs/.claude/settings.local.json new file mode 100644 index 00000000000..c2dc67502f5 --- /dev/null +++ b/docs/vocs/.claude/settings.local.json @@ -0,0 +1,8 @@ +{ + "permissions": { + "allow": [ + "Bash(git checkout:*)" + ], + "deny": [] + } +} \ No newline at end of file diff --git a/docs/vocs/CLAUDE.md b/docs/vocs/CLAUDE.md new file mode 100644 index 00000000000..98b57a5791f --- /dev/null +++ b/docs/vocs/CLAUDE.md @@ -0,0 +1,103 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +This is the **Reth documentation website** built with [Vocs](https://vocs.dev), a modern documentation framework. The site contains comprehensive documentation for Reth, the Ethereum execution client, including installation guides, CLI references, SDK documentation, and tutorials. + +## Repository Structure + +- **`docs/pages/`**: All documentation content in MDX format + - `cli/`: Command-line interface documentation and references + - `exex/`: Execution Extensions (ExEx) guides and examples + - `installation/`: Installation and setup guides + - `introduction/`: Introduction, benchmarks, and why-reth content + - `jsonrpc/`: JSON-RPC API documentation + - `run/`: Node running guides and configuration + - `sdk/`: SDK documentation and examples +- **`docs/snippets/`**: Code examples and snippets used in documentation +- **`sidebar.ts`**: Navigation configuration +- **`vocs.config.ts`**: Vocs configuration file + +## Essential Commands + +```bash +# Install dependencies +bun install + +# Start development server +bun run dev + +# Build for production +bun run build + +# Preview production build +bun run preview +``` + +## Development Workflow + +### Content Organization + +1. **MDX Files**: All content is written in MDX (Markdown + React components) +2. **Navigation**: Update `sidebar.ts` when adding new pages +3. **Code Examples**: Place reusable code snippets in `docs/snippets/` +4. **Assets**: Place images and static assets in `docs/public/` + +### Adding New Documentation + +1. Create new `.mdx` files in appropriate subdirectories under `docs/pages/` +2. Update `sidebar.ts` to include new pages in navigation +3. Use consistent heading structure and markdown formatting +4. Reference code examples from `docs/snippets/` when possible + +### Code Examples and Snippets + +- **Live Examples**: Use the snippets system to include actual runnable code +- **Rust Code**: Include cargo project examples in `docs/snippets/sources/` +- **CLI Examples**: Show actual command usage with expected outputs + +### Configuration + +- **Base Path**: Site deploys to `/reth` path (configured in `vocs.config.ts`) +- **Theme**: Custom accent colors for light/dark themes +- **Vite**: Uses Vite as the underlying build tool + +### Content Guidelines + +1. **Be Practical**: Focus on actionable guides and real-world examples +2. **Code First**: Show working code examples before explaining concepts +3. **Consistent Structure**: Follow existing page structures for consistency +4. **Cross-References**: Link between related pages and sections +5. **Keep Current**: Ensure documentation matches latest Reth features + +### File Naming Conventions + +- Use kebab-case for file and directory names +- Match URL structure to file structure +- Use descriptive names that reflect content purpose + +### Common Tasks + +**Adding a new CLI command documentation:** +1. Create `.mdx` file in `docs/pages/cli/reth/` +2. Add to sidebar navigation +3. Include usage examples and parameter descriptions + +**Adding a new guide:** +1. Create `.mdx` file in appropriate category +2. Update sidebar with new entry +3. Include practical examples and next steps + +**Updating code examples:** +1. Modify files in `docs/snippets/sources/` +2. Ensure examples compile and run correctly +3. Test that documentation references work properly + +## Development Notes + +- This is a TypeScript/React project using Vocs framework +- Content is primarily MDX with some TypeScript configuration +- Focus on clear, practical documentation that helps users succeed with Reth +- Maintain consistency with existing documentation style and structure \ No newline at end of file diff --git a/docs/vocs/README.md b/docs/vocs/README.md new file mode 100644 index 00000000000..3bb11a44a0a --- /dev/null +++ b/docs/vocs/README.md @@ -0,0 +1 @@ +This is a [Vocs](https://vocs.dev) project bootstrapped with the Vocs CLI. diff --git a/docs/vocs/bun.lock b/docs/vocs/bun.lock new file mode 100644 index 00000000000..4203e94aa62 --- /dev/null +++ b/docs/vocs/bun.lock @@ -0,0 +1,1542 @@ +{ + "lockfileVersion": 1, + "workspaces": { + "": { + "name": "vocs", + "dependencies": { + "react": "^19.1.0", + "react-dom": "^19.1.0", + "vocs": "^1.0.13", + }, + "devDependencies": { + "@types/node": "^24.0.14", + "@types/react": "^19.1.8", + "glob": "^11.0.3", + "typescript": "^5.8.3", + }, + }, + }, + "packages": { + "@ampproject/remapping": ["@ampproject/remapping@2.3.0", "", { "dependencies": { "@jridgewell/gen-mapping": "^0.3.5", "@jridgewell/trace-mapping": "^0.3.24" } }, "sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw=="], + + "@antfu/install-pkg": ["@antfu/install-pkg@1.1.0", "", { "dependencies": { "package-manager-detector": "^1.3.0", "tinyexec": "^1.0.1" } }, "sha512-MGQsmw10ZyI+EJo45CdSER4zEb+p31LpDAFp2Z3gkSd1yqVZGi0Ebx++YTEMonJy4oChEMLsxZ64j8FH6sSqtQ=="], + + "@antfu/utils": ["@antfu/utils@8.1.1", "", {}, "sha512-Mex9nXf9vR6AhcXmMrlz/HVgYYZpVGJ6YlPgwl7UnaFpnshXs6EK/oa5Gpf3CzENMjkvEx2tQtntGnb7UtSTOQ=="], + + "@babel/code-frame": ["@babel/code-frame@7.27.1", "", { "dependencies": { "@babel/helper-validator-identifier": "^7.27.1", "js-tokens": "^4.0.0", "picocolors": "^1.1.1" } }, "sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg=="], + + "@babel/compat-data": ["@babel/compat-data@7.28.0", "", {}, "sha512-60X7qkglvrap8mn1lh2ebxXdZYtUcpd7gsmy9kLaBJ4i/WdY8PqTSdxyA8qraikqKQK5C1KRBKXqznrVapyNaw=="], + + "@babel/core": ["@babel/core@7.28.0", "", { "dependencies": { "@ampproject/remapping": "^2.2.0", "@babel/code-frame": "^7.27.1", "@babel/generator": "^7.28.0", "@babel/helper-compilation-targets": "^7.27.2", "@babel/helper-module-transforms": "^7.27.3", "@babel/helpers": "^7.27.6", "@babel/parser": "^7.28.0", "@babel/template": "^7.27.2", "@babel/traverse": "^7.28.0", "@babel/types": "^7.28.0", "convert-source-map": "^2.0.0", "debug": "^4.1.0", "gensync": "^1.0.0-beta.2", "json5": "^2.2.3", "semver": "^6.3.1" } }, "sha512-UlLAnTPrFdNGoFtbSXwcGFQBtQZJCNjaN6hQNP3UPvuNXT1i82N26KL3dZeIpNalWywr9IuQuncaAfUaS1g6sQ=="], + + "@babel/generator": ["@babel/generator@7.28.0", "", { "dependencies": { "@babel/parser": "^7.28.0", "@babel/types": "^7.28.0", "@jridgewell/gen-mapping": "^0.3.12", "@jridgewell/trace-mapping": "^0.3.28", "jsesc": "^3.0.2" } }, "sha512-lJjzvrbEeWrhB4P3QBsH7tey117PjLZnDbLiQEKjQ/fNJTjuq4HSqgFA+UNSwZT8D7dxxbnuSBMsa1lrWzKlQg=="], + + "@babel/helper-compilation-targets": ["@babel/helper-compilation-targets@7.27.2", "", { "dependencies": { "@babel/compat-data": "^7.27.2", "@babel/helper-validator-option": "^7.27.1", "browserslist": "^4.24.0", "lru-cache": "^5.1.1", "semver": "^6.3.1" } }, "sha512-2+1thGUUWWjLTYTHZWK1n8Yga0ijBz1XAhUXcKy81rd5g6yh7hGqMp45v7cadSbEHc9G3OTv45SyneRN3ps4DQ=="], + + "@babel/helper-globals": ["@babel/helper-globals@7.28.0", "", {}, "sha512-+W6cISkXFa1jXsDEdYA8HeevQT/FULhxzR99pxphltZcVaugps53THCeiWA8SguxxpSp3gKPiuYfSWopkLQ4hw=="], + + "@babel/helper-module-imports": ["@babel/helper-module-imports@7.27.1", "", { "dependencies": { "@babel/traverse": "^7.27.1", "@babel/types": "^7.27.1" } }, "sha512-0gSFWUPNXNopqtIPQvlD5WgXYI5GY2kP2cCvoT8kczjbfcfuIljTbcWrulD1CIPIX2gt1wghbDy08yE1p+/r3w=="], + + "@babel/helper-module-transforms": ["@babel/helper-module-transforms@7.27.3", "", { "dependencies": { "@babel/helper-module-imports": "^7.27.1", "@babel/helper-validator-identifier": "^7.27.1", "@babel/traverse": "^7.27.3" }, "peerDependencies": { "@babel/core": "^7.0.0" } }, "sha512-dSOvYwvyLsWBeIRyOeHXp5vPj5l1I011r52FM1+r1jCERv+aFXYk4whgQccYEGYxK2H3ZAIA8nuPkQ0HaUo3qg=="], + + "@babel/helper-plugin-utils": ["@babel/helper-plugin-utils@7.27.1", "", {}, "sha512-1gn1Up5YXka3YYAHGKpbideQ5Yjf1tDa9qYcgysz+cNCXukyLl6DjPXhD3VRwSb8c0J9tA4b2+rHEZtc6R0tlw=="], + + "@babel/helper-string-parser": ["@babel/helper-string-parser@7.27.1", "", {}, "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA=="], + + "@babel/helper-validator-identifier": ["@babel/helper-validator-identifier@7.27.1", "", {}, "sha512-D2hP9eA+Sqx1kBZgzxZh0y1trbuU+JoDkiEwqhQ36nodYqJwyEIhPSdMNd7lOm/4io72luTPWH20Yda0xOuUow=="], + + "@babel/helper-validator-option": ["@babel/helper-validator-option@7.27.1", "", {}, "sha512-YvjJow9FxbhFFKDSuFnVCe2WxXk1zWc22fFePVNEaWJEu8IrZVlda6N0uHwzZrUM1il7NC9Mlp4MaJYbYd9JSg=="], + + "@babel/helpers": ["@babel/helpers@7.27.6", "", { "dependencies": { "@babel/template": "^7.27.2", "@babel/types": "^7.27.6" } }, "sha512-muE8Tt8M22638HU31A3CgfSUciwz1fhATfoVai05aPXGor//CdWDCbnlY1yvBPo07njuVOCNGCSp/GTt12lIug=="], + + "@babel/parser": ["@babel/parser@7.28.0", "", { "dependencies": { "@babel/types": "^7.28.0" }, "bin": "./bin/babel-parser.js" }, "sha512-jVZGvOxOuNSsuQuLRTh13nU0AogFlw32w/MT+LV6D3sP5WdbW61E77RnkbaO2dUvmPAYrBDJXGn5gGS6tH4j8g=="], + + "@babel/plugin-syntax-typescript": ["@babel/plugin-syntax-typescript@7.27.1", "", { "dependencies": { "@babel/helper-plugin-utils": "^7.27.1" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "sha512-xfYCBMxveHrRMnAWl1ZlPXOZjzkN82THFvLhQhFXFt81Z5HnN+EtUkZhv/zcKpmT3fzmWZB0ywiBrbC3vogbwQ=="], + + "@babel/plugin-transform-react-jsx-self": ["@babel/plugin-transform-react-jsx-self@7.27.1", "", { "dependencies": { "@babel/helper-plugin-utils": "^7.27.1" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "sha512-6UzkCs+ejGdZ5mFFC/OCUrv028ab2fp1znZmCZjAOBKiBK2jXD1O+BPSfX8X2qjJ75fZBMSnQn3Rq2mrBJK2mw=="], + + "@babel/plugin-transform-react-jsx-source": ["@babel/plugin-transform-react-jsx-source@7.27.1", "", { "dependencies": { "@babel/helper-plugin-utils": "^7.27.1" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "sha512-zbwoTsBruTeKB9hSq73ha66iFeJHuaFkUbwvqElnygoNbj/jHRsSeokowZFN3CZ64IvEqcmmkVe89OPXc7ldAw=="], + + "@babel/runtime": ["@babel/runtime@7.27.6", "", {}, "sha512-vbavdySgbTTrmFE+EsiqUTzlOr5bzlnJtUv9PynGCAKvfQqjIXbvFdumPM/GxMDfyuGMJaJAU6TO4zc1Jf1i8Q=="], + + "@babel/template": ["@babel/template@7.27.2", "", { "dependencies": { "@babel/code-frame": "^7.27.1", "@babel/parser": "^7.27.2", "@babel/types": "^7.27.1" } }, "sha512-LPDZ85aEJyYSd18/DkjNh4/y1ntkE5KwUHWTiqgRxruuZL2F1yuHligVHLvcHY2vMHXttKFpJn6LwfI7cw7ODw=="], + + "@babel/traverse": ["@babel/traverse@7.28.0", "", { "dependencies": { "@babel/code-frame": "^7.27.1", "@babel/generator": "^7.28.0", "@babel/helper-globals": "^7.28.0", "@babel/parser": "^7.28.0", "@babel/template": "^7.27.2", "@babel/types": "^7.28.0", "debug": "^4.3.1" } }, "sha512-mGe7UK5wWyh0bKRfupsUchrQGqvDbZDbKJw+kcRGSmdHVYrv+ltd0pnpDTVpiTqnaBru9iEvA8pz8W46v0Amwg=="], + + "@babel/types": ["@babel/types@7.28.1", "", { "dependencies": { "@babel/helper-string-parser": "^7.27.1", "@babel/helper-validator-identifier": "^7.27.1" } }, "sha512-x0LvFTekgSX+83TI28Y9wYPUfzrnl2aT5+5QLnO6v7mSJYtEEevuDRN0F0uSHRk1G1IWZC43o00Y0xDDrpBGPQ=="], + + "@braintree/sanitize-url": ["@braintree/sanitize-url@7.1.1", "", {}, "sha512-i1L7noDNxtFyL5DmZafWy1wRVhGehQmzZaz1HiN5e7iylJMSZR7ekOV7NsIqa5qBldlLrsKv4HbgFUVlQrz8Mw=="], + + "@chevrotain/cst-dts-gen": ["@chevrotain/cst-dts-gen@11.0.3", "", { "dependencies": { "@chevrotain/gast": "11.0.3", "@chevrotain/types": "11.0.3", "lodash-es": "4.17.21" } }, "sha512-BvIKpRLeS/8UbfxXxgC33xOumsacaeCKAjAeLyOn7Pcp95HiRbrpl14S+9vaZLolnbssPIUuiUd8IvgkRyt6NQ=="], + + "@chevrotain/gast": ["@chevrotain/gast@11.0.3", "", { "dependencies": { "@chevrotain/types": "11.0.3", "lodash-es": "4.17.21" } }, "sha512-+qNfcoNk70PyS/uxmj3li5NiECO+2YKZZQMbmjTqRI3Qchu8Hig/Q9vgkHpI3alNjr7M+a2St5pw5w5F6NL5/Q=="], + + "@chevrotain/regexp-to-ast": ["@chevrotain/regexp-to-ast@11.0.3", "", {}, "sha512-1fMHaBZxLFvWI067AVbGJav1eRY7N8DDvYCTwGBiE/ytKBgP8azTdgyrKyWZ9Mfh09eHWb5PgTSO8wi7U824RA=="], + + "@chevrotain/types": ["@chevrotain/types@11.0.3", "", {}, "sha512-gsiM3G8b58kZC2HaWR50gu6Y1440cHiJ+i3JUvcp/35JchYejb2+5MVeJK0iKThYpAa/P2PYFV4hoi44HD+aHQ=="], + + "@chevrotain/utils": ["@chevrotain/utils@11.0.3", "", {}, "sha512-YslZMgtJUyuMbZ+aKvfF3x1f5liK4mWNxghFRv7jqRR9C3R3fAOGTTKvxXDa2Y1s9zSbcpuO0cAxDYsc9SrXoQ=="], + + "@clack/core": ["@clack/core@0.3.5", "", { "dependencies": { "picocolors": "^1.0.0", "sisteransi": "^1.0.5" } }, "sha512-5cfhQNH+1VQ2xLQlmzXMqUoiaH0lRBq9/CLW9lTyMbuKLC3+xEK01tHVvyut++mLOn5urSHmkm6I0Lg9MaJSTQ=="], + + "@clack/prompts": ["@clack/prompts@0.7.0", "", { "dependencies": { "@clack/core": "^0.3.3", "is-unicode-supported": "*", "picocolors": "^1.0.0", "sisteransi": "^1.0.5" } }, "sha512-0MhX9/B4iL6Re04jPrttDm+BsP8y6mS7byuv0BvXgdXhbV5PdlsHt55dvNsuBCPZ7xq1oTAOOuotR9NFbQyMSA=="], + + "@emotion/hash": ["@emotion/hash@0.9.2", "", {}, "sha512-MyqliTZGuOm3+5ZRSaaBGP3USLw6+EGykkwZns2EPC5g8jJ4z9OrdZY9apkl3+UP9+sdz76YYkwCKP5gh8iY3g=="], + + "@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.25.6", "", { "os": "aix", "cpu": "ppc64" }, "sha512-ShbM/3XxwuxjFiuVBHA+d3j5dyac0aEVVq1oluIDf71hUw0aRF59dV/efUsIwFnR6m8JNM2FjZOzmaZ8yG61kw=="], + + "@esbuild/android-arm": ["@esbuild/android-arm@0.25.6", "", { "os": "android", "cpu": "arm" }, "sha512-S8ToEOVfg++AU/bHwdksHNnyLyVM+eMVAOf6yRKFitnwnbwwPNqKr3srzFRe7nzV69RQKb5DgchIX5pt3L53xg=="], + + "@esbuild/android-arm64": ["@esbuild/android-arm64@0.25.6", "", { "os": "android", "cpu": "arm64" }, "sha512-hd5zdUarsK6strW+3Wxi5qWws+rJhCCbMiC9QZyzoxfk5uHRIE8T287giQxzVpEvCwuJ9Qjg6bEjcRJcgfLqoA=="], + + "@esbuild/android-x64": ["@esbuild/android-x64@0.25.6", "", { "os": "android", "cpu": "x64" }, "sha512-0Z7KpHSr3VBIO9A/1wcT3NTy7EB4oNC4upJ5ye3R7taCc2GUdeynSLArnon5G8scPwaU866d3H4BCrE5xLW25A=="], + + "@esbuild/darwin-arm64": ["@esbuild/darwin-arm64@0.25.6", "", { "os": "darwin", "cpu": "arm64" }, "sha512-FFCssz3XBavjxcFxKsGy2DYK5VSvJqa6y5HXljKzhRZ87LvEi13brPrf/wdyl/BbpbMKJNOr1Sd0jtW4Ge1pAA=="], + + "@esbuild/darwin-x64": ["@esbuild/darwin-x64@0.25.6", "", { "os": "darwin", "cpu": "x64" }, "sha512-GfXs5kry/TkGM2vKqK2oyiLFygJRqKVhawu3+DOCk7OxLy/6jYkWXhlHwOoTb0WqGnWGAS7sooxbZowy+pK9Yg=="], + + "@esbuild/freebsd-arm64": ["@esbuild/freebsd-arm64@0.25.6", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-aoLF2c3OvDn2XDTRvn8hN6DRzVVpDlj2B/F66clWd/FHLiHaG3aVZjxQX2DYphA5y/evbdGvC6Us13tvyt4pWg=="], + + "@esbuild/freebsd-x64": ["@esbuild/freebsd-x64@0.25.6", "", { "os": "freebsd", "cpu": "x64" }, "sha512-2SkqTjTSo2dYi/jzFbU9Plt1vk0+nNg8YC8rOXXea+iA3hfNJWebKYPs3xnOUf9+ZWhKAaxnQNUf2X9LOpeiMQ=="], + + "@esbuild/linux-arm": ["@esbuild/linux-arm@0.25.6", "", { "os": "linux", "cpu": "arm" }, "sha512-SZHQlzvqv4Du5PrKE2faN0qlbsaW/3QQfUUc6yO2EjFcA83xnwm91UbEEVx4ApZ9Z5oG8Bxz4qPE+HFwtVcfyw=="], + + "@esbuild/linux-arm64": ["@esbuild/linux-arm64@0.25.6", "", { "os": "linux", "cpu": "arm64" }, "sha512-b967hU0gqKd9Drsh/UuAm21Khpoh6mPBSgz8mKRq4P5mVK8bpA+hQzmm/ZwGVULSNBzKdZPQBRT3+WuVavcWsQ=="], + + "@esbuild/linux-ia32": ["@esbuild/linux-ia32@0.25.6", "", { "os": "linux", "cpu": "ia32" }, "sha512-aHWdQ2AAltRkLPOsKdi3xv0mZ8fUGPdlKEjIEhxCPm5yKEThcUjHpWB1idN74lfXGnZ5SULQSgtr5Qos5B0bPw=="], + + "@esbuild/linux-loong64": ["@esbuild/linux-loong64@0.25.6", "", { "os": "linux", "cpu": "none" }, "sha512-VgKCsHdXRSQ7E1+QXGdRPlQ/e08bN6WMQb27/TMfV+vPjjTImuT9PmLXupRlC90S1JeNNW5lzkAEO/McKeJ2yg=="], + + "@esbuild/linux-mips64el": ["@esbuild/linux-mips64el@0.25.6", "", { "os": "linux", "cpu": "none" }, "sha512-WViNlpivRKT9/py3kCmkHnn44GkGXVdXfdc4drNmRl15zVQ2+D2uFwdlGh6IuK5AAnGTo2qPB1Djppj+t78rzw=="], + + "@esbuild/linux-ppc64": ["@esbuild/linux-ppc64@0.25.6", "", { "os": "linux", "cpu": "ppc64" }, "sha512-wyYKZ9NTdmAMb5730I38lBqVu6cKl4ZfYXIs31Baf8aoOtB4xSGi3THmDYt4BTFHk7/EcVixkOV2uZfwU3Q2Jw=="], + + "@esbuild/linux-riscv64": ["@esbuild/linux-riscv64@0.25.6", "", { "os": "linux", "cpu": "none" }, "sha512-KZh7bAGGcrinEj4qzilJ4hqTY3Dg2U82c8bv+e1xqNqZCrCyc+TL9AUEn5WGKDzm3CfC5RODE/qc96OcbIe33w=="], + + "@esbuild/linux-s390x": ["@esbuild/linux-s390x@0.25.6", "", { "os": "linux", "cpu": "s390x" }, "sha512-9N1LsTwAuE9oj6lHMyyAM+ucxGiVnEqUdp4v7IaMmrwb06ZTEVCIs3oPPplVsnjPfyjmxwHxHMF8b6vzUVAUGw=="], + + "@esbuild/linux-x64": ["@esbuild/linux-x64@0.25.6", "", { "os": "linux", "cpu": "x64" }, "sha512-A6bJB41b4lKFWRKNrWoP2LHsjVzNiaurf7wyj/XtFNTsnPuxwEBWHLty+ZE0dWBKuSK1fvKgrKaNjBS7qbFKig=="], + + "@esbuild/netbsd-arm64": ["@esbuild/netbsd-arm64@0.25.6", "", { "os": "none", "cpu": "arm64" }, "sha512-IjA+DcwoVpjEvyxZddDqBY+uJ2Snc6duLpjmkXm/v4xuS3H+3FkLZlDm9ZsAbF9rsfP3zeA0/ArNDORZgrxR/Q=="], + + "@esbuild/netbsd-x64": ["@esbuild/netbsd-x64@0.25.6", "", { "os": "none", "cpu": "x64" }, "sha512-dUXuZr5WenIDlMHdMkvDc1FAu4xdWixTCRgP7RQLBOkkGgwuuzaGSYcOpW4jFxzpzL1ejb8yF620UxAqnBrR9g=="], + + "@esbuild/openbsd-arm64": ["@esbuild/openbsd-arm64@0.25.6", "", { "os": "openbsd", "cpu": "arm64" }, "sha512-l8ZCvXP0tbTJ3iaqdNf3pjaOSd5ex/e6/omLIQCVBLmHTlfXW3zAxQ4fnDmPLOB1x9xrcSi/xtCWFwCZRIaEwg=="], + + "@esbuild/openbsd-x64": ["@esbuild/openbsd-x64@0.25.6", "", { "os": "openbsd", "cpu": "x64" }, "sha512-hKrmDa0aOFOr71KQ/19JC7az1P0GWtCN1t2ahYAf4O007DHZt/dW8ym5+CUdJhQ/qkZmI1HAF8KkJbEFtCL7gw=="], + + "@esbuild/openharmony-arm64": ["@esbuild/openharmony-arm64@0.25.6", "", { "os": "none", "cpu": "arm64" }, "sha512-+SqBcAWoB1fYKmpWoQP4pGtx+pUUC//RNYhFdbcSA16617cchuryuhOCRpPsjCblKukAckWsV+aQ3UKT/RMPcA=="], + + "@esbuild/sunos-x64": ["@esbuild/sunos-x64@0.25.6", "", { "os": "sunos", "cpu": "x64" }, "sha512-dyCGxv1/Br7MiSC42qinGL8KkG4kX0pEsdb0+TKhmJZgCUDBGmyo1/ArCjNGiOLiIAgdbWgmWgib4HoCi5t7kA=="], + + "@esbuild/win32-arm64": ["@esbuild/win32-arm64@0.25.6", "", { "os": "win32", "cpu": "arm64" }, "sha512-42QOgcZeZOvXfsCBJF5Afw73t4veOId//XD3i+/9gSkhSV6Gk3VPlWncctI+JcOyERv85FUo7RxuxGy+z8A43Q=="], + + "@esbuild/win32-ia32": ["@esbuild/win32-ia32@0.25.6", "", { "os": "win32", "cpu": "ia32" }, "sha512-4AWhgXmDuYN7rJI6ORB+uU9DHLq/erBbuMoAuB4VWJTu5KtCgcKYPynF0YI1VkBNuEfjNlLrFr9KZPJzrtLkrQ=="], + + "@esbuild/win32-x64": ["@esbuild/win32-x64@0.25.6", "", { "os": "win32", "cpu": "x64" }, "sha512-NgJPHHbEpLQgDH2MjQu90pzW/5vvXIZ7KOnPyNBm92A6WgZ/7b6fJyUBjoumLqeOQQGqY2QjQxRo97ah4Sj0cA=="], + + "@floating-ui/core": ["@floating-ui/core@1.7.2", "", { "dependencies": { "@floating-ui/utils": "^0.2.10" } }, "sha512-wNB5ooIKHQc+Kui96jE/n69rHFWAVoxn5CAzL1Xdd8FG03cgY3MLO+GF9U3W737fYDSgPWA6MReKhBQBop6Pcw=="], + + "@floating-ui/dom": ["@floating-ui/dom@1.7.2", "", { "dependencies": { "@floating-ui/core": "^1.7.2", "@floating-ui/utils": "^0.2.10" } }, "sha512-7cfaOQuCS27HD7DX+6ib2OrnW+b4ZBwDNnCcT0uTyidcmyWb03FnQqJybDBoCnpdxwBSfA94UAYlRCt7mV+TbA=="], + + "@floating-ui/react": ["@floating-ui/react@0.27.13", "", { "dependencies": { "@floating-ui/react-dom": "^2.1.4", "@floating-ui/utils": "^0.2.10", "tabbable": "^6.0.0" }, "peerDependencies": { "react": ">=17.0.0", "react-dom": ">=17.0.0" } }, "sha512-Qmj6t9TjgWAvbygNEu1hj4dbHI9CY0ziCMIJrmYoDIn9TUAH5lRmiIeZmRd4c6QEZkzdoH7jNnoNyoY1AIESiA=="], + + "@floating-ui/react-dom": ["@floating-ui/react-dom@2.1.4", "", { "dependencies": { "@floating-ui/dom": "^1.7.2" }, "peerDependencies": { "react": ">=16.8.0", "react-dom": ">=16.8.0" } }, "sha512-JbbpPhp38UmXDDAu60RJmbeme37Jbgsm7NrHGgzYYFKmblzRUh6Pa641dII6LsjwF4XlScDrde2UAzDo/b9KPw=="], + + "@floating-ui/utils": ["@floating-ui/utils@0.2.10", "", {}, "sha512-aGTxbpbg8/b5JfU1HXSrbH3wXZuLPJcNEcZQFMxLs3oSzgtVu6nFPkbbGGUvBcUjKV2YyB9Wxxabo+HEH9tcRQ=="], + + "@fortawesome/fontawesome-free": ["@fortawesome/fontawesome-free@6.7.2", "", {}, "sha512-JUOtgFW6k9u4Y+xeIaEiLr3+cjoUPiAuLXoyKOJSia6Duzb7pq+A76P9ZdPDoAoxHdHzq6gE9/jKBGXlZT8FbA=="], + + "@hono/node-server": ["@hono/node-server@1.16.0", "", { "peerDependencies": { "hono": "^4" } }, "sha512-9LwRb5XOrTFapOABiQjGC50wRVlzUvWZsDHINCnkBniP+Q+LQf4waN0nzk9t+2kqcTsnGnieSmqpHsr6kH2bdw=="], + + "@iconify/types": ["@iconify/types@2.0.0", "", {}, "sha512-+wluvCrRhXrhyOmRDJ3q8mux9JkKy5SJ/v8ol2tu4FVjyYvtEzkc/3pK15ET6RKg4b4w4BmTk1+gsCUhf21Ykg=="], + + "@iconify/utils": ["@iconify/utils@2.3.0", "", { "dependencies": { "@antfu/install-pkg": "^1.0.0", "@antfu/utils": "^8.1.0", "@iconify/types": "^2.0.0", "debug": "^4.4.0", "globals": "^15.14.0", "kolorist": "^1.8.0", "local-pkg": "^1.0.0", "mlly": "^1.7.4" } }, "sha512-GmQ78prtwYW6EtzXRU1rY+KwOKfz32PD7iJh6Iyqw68GiKuoZ2A6pRtzWONz5VQJbp50mEjXh/7NkumtrAgRKA=="], + + "@isaacs/balanced-match": ["@isaacs/balanced-match@4.0.1", "", {}, "sha512-yzMTt9lEb8Gv7zRioUilSglI0c0smZ9k5D65677DLWLtWJaXIS3CqcGyUFByYKlnUj6TkjLVs54fBl6+TiGQDQ=="], + + "@isaacs/brace-expansion": ["@isaacs/brace-expansion@5.0.0", "", { "dependencies": { "@isaacs/balanced-match": "^4.0.1" } }, "sha512-ZT55BDLV0yv0RBm2czMiZ+SqCGO7AvmOM3G/w2xhVPH+te0aKgFjmBvGlL1dH+ql2tgGO3MVrbb3jCKyvpgnxA=="], + + "@isaacs/cliui": ["@isaacs/cliui@8.0.2", "", { "dependencies": { "string-width": "^5.1.2", "string-width-cjs": "npm:string-width@^4.2.0", "strip-ansi": "^7.0.1", "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", "wrap-ansi": "^8.1.0", "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" } }, "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA=="], + + "@jridgewell/gen-mapping": ["@jridgewell/gen-mapping@0.3.12", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.0", "@jridgewell/trace-mapping": "^0.3.24" } }, "sha512-OuLGC46TjB5BbN1dH8JULVVZY4WTdkF7tV9Ys6wLL1rubZnCMstOhNHueU5bLCrnRuDhKPDM4g6sw4Bel5Gzqg=="], + + "@jridgewell/resolve-uri": ["@jridgewell/resolve-uri@3.1.2", "", {}, "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw=="], + + "@jridgewell/sourcemap-codec": ["@jridgewell/sourcemap-codec@1.5.4", "", {}, "sha512-VT2+G1VQs/9oz078bLrYbecdZKs912zQlkelYpuf+SXF+QvZDYJlbx/LSx+meSAwdDFnF8FVXW92AVjjkVmgFw=="], + + "@jridgewell/trace-mapping": ["@jridgewell/trace-mapping@0.3.29", "", { "dependencies": { "@jridgewell/resolve-uri": "^3.1.0", "@jridgewell/sourcemap-codec": "^1.4.14" } }, "sha512-uw6guiW/gcAGPDhLmd77/6lW8QLeiV5RUTsAX46Db6oLhGaVj4lhnPwb184s1bkc8kdVg/+h988dro8GRDpmYQ=="], + + "@mdx-js/mdx": ["@mdx-js/mdx@3.1.0", "", { "dependencies": { "@types/estree": "^1.0.0", "@types/estree-jsx": "^1.0.0", "@types/hast": "^3.0.0", "@types/mdx": "^2.0.0", "collapse-white-space": "^2.0.0", "devlop": "^1.0.0", "estree-util-is-identifier-name": "^3.0.0", "estree-util-scope": "^1.0.0", "estree-walker": "^3.0.0", "hast-util-to-jsx-runtime": "^2.0.0", "markdown-extensions": "^2.0.0", "recma-build-jsx": "^1.0.0", "recma-jsx": "^1.0.0", "recma-stringify": "^1.0.0", "rehype-recma": "^1.0.0", "remark-mdx": "^3.0.0", "remark-parse": "^11.0.0", "remark-rehype": "^11.0.0", "source-map": "^0.7.0", "unified": "^11.0.0", "unist-util-position-from-estree": "^2.0.0", "unist-util-stringify-position": "^4.0.0", "unist-util-visit": "^5.0.0", "vfile": "^6.0.0" } }, "sha512-/QxEhPAvGwbQmy1Px8F899L5Uc2KZ6JtXwlCgJmjSTBedwOZkByYcBG4GceIGPXRDsmfxhHazuS+hlOShRLeDw=="], + + "@mdx-js/react": ["@mdx-js/react@3.1.0", "", { "dependencies": { "@types/mdx": "^2.0.0" }, "peerDependencies": { "@types/react": ">=16", "react": ">=16" } }, "sha512-QjHtSaoameoalGnKDT3FoIl4+9RwyTmo9ZJGBdLOks/YOiWHoRDI3PUwEzOE7kEmGcV3AFcp9K6dYu9rEuKLAQ=="], + + "@mdx-js/rollup": ["@mdx-js/rollup@3.1.0", "", { "dependencies": { "@mdx-js/mdx": "^3.0.0", "@rollup/pluginutils": "^5.0.0", "source-map": "^0.7.0", "vfile": "^6.0.0" }, "peerDependencies": { "rollup": ">=2" } }, "sha512-q4xOtUXpCzeouE8GaJ8StT4rDxm/U5j6lkMHL2srb2Q3Y7cobE0aXyPzXVVlbeIMBi+5R5MpbiaVE5/vJUdnHg=="], + + "@mermaid-js/parser": ["@mermaid-js/parser@0.6.2", "", { "dependencies": { "langium": "3.3.1" } }, "sha512-+PO02uGF6L6Cs0Bw8RpGhikVvMWEysfAyl27qTlroUB8jSWr1lL0Sf6zi78ZxlSnmgSY2AMMKVgghnN9jTtwkQ=="], + + "@noble/hashes": ["@noble/hashes@1.8.0", "", {}, "sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A=="], + + "@nodelib/fs.scandir": ["@nodelib/fs.scandir@2.1.5", "", { "dependencies": { "@nodelib/fs.stat": "2.0.5", "run-parallel": "^1.1.9" } }, "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g=="], + + "@nodelib/fs.stat": ["@nodelib/fs.stat@2.0.5", "", {}, "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A=="], + + "@nodelib/fs.walk": ["@nodelib/fs.walk@1.2.8", "", { "dependencies": { "@nodelib/fs.scandir": "2.1.5", "fastq": "^1.6.0" } }, "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg=="], + + "@radix-ui/colors": ["@radix-ui/colors@3.0.0", "", {}, "sha512-FUOsGBkHrYJwCSEtWRCIfQbZG7q1e6DgxCIOe1SUQzDe/7rXXeA47s8yCn6fuTNQAj1Zq4oTFi9Yjp3wzElcxg=="], + + "@radix-ui/number": ["@radix-ui/number@1.1.1", "", {}, "sha512-MkKCwxlXTgz6CFoJx3pCwn07GKp36+aZyu/u2Ln2VrA5DcdyCZkASEDBTd8x5whTQQL5CiYf4prXKLcgQdv29g=="], + + "@radix-ui/primitive": ["@radix-ui/primitive@1.1.2", "", {}, "sha512-XnbHrrprsNqZKQhStrSwgRUQzoCI1glLzdw79xiZPoofhGICeZRSQ3dIxAKH1gb3OHfNf4d6f+vAv3kil2eggA=="], + + "@radix-ui/react-accessible-icon": ["@radix-ui/react-accessible-icon@1.1.7", "", { "dependencies": { "@radix-ui/react-visually-hidden": "1.2.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-XM+E4WXl0OqUJFovy6GjmxxFyx9opfCAIUku4dlKRd5YEPqt4kALOkQOp0Of6reHuUkJuiPBEc5k0o4z4lTC8A=="], + + "@radix-ui/react-accordion": ["@radix-ui/react-accordion@1.2.11", "", { "dependencies": { "@radix-ui/primitive": "1.1.2", "@radix-ui/react-collapsible": "1.1.11", "@radix-ui/react-collection": "1.1.7", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-direction": "1.1.1", "@radix-ui/react-id": "1.1.1", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-controllable-state": "1.2.2" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-l3W5D54emV2ues7jjeG1xcyN7S3jnK3zE2zHqgn0CmMsy9lNJwmgcrmaxS+7ipw15FAivzKNzH3d5EcGoFKw0A=="], + + "@radix-ui/react-alert-dialog": ["@radix-ui/react-alert-dialog@1.1.14", "", { "dependencies": { "@radix-ui/primitive": "1.1.2", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-dialog": "1.1.14", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-slot": "1.2.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-IOZfZ3nPvN6lXpJTBCunFQPRSvK8MDgSc1FB85xnIpUKOw9en0dJj8JmCAxV7BiZdtYlUpmrQjoTFkVYtdoWzQ=="], + + "@radix-ui/react-arrow": ["@radix-ui/react-arrow@1.1.7", "", { "dependencies": { "@radix-ui/react-primitive": "2.1.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-F+M1tLhO+mlQaOWspE8Wstg+z6PwxwRd8oQ8IXceWz92kfAmalTRf0EjrouQeo7QssEPfCn05B4Ihs1K9WQ/7w=="], + + "@radix-ui/react-aspect-ratio": ["@radix-ui/react-aspect-ratio@1.1.7", "", { "dependencies": { "@radix-ui/react-primitive": "2.1.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-Yq6lvO9HQyPwev1onK1daHCHqXVLzPhSVjmsNjCa2Zcxy2f7uJD2itDtxknv6FzAKCwD1qQkeVDmX/cev13n/g=="], + + "@radix-ui/react-avatar": ["@radix-ui/react-avatar@1.1.10", "", { "dependencies": { "@radix-ui/react-context": "1.1.2", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-callback-ref": "1.1.1", "@radix-ui/react-use-is-hydrated": "0.1.0", "@radix-ui/react-use-layout-effect": "1.1.1" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-V8piFfWapM5OmNCXTzVQY+E1rDa53zY+MQ4Y7356v4fFz6vqCyUtIz2rUD44ZEdwg78/jKmMJHj07+C/Z/rcog=="], + + "@radix-ui/react-checkbox": ["@radix-ui/react-checkbox@1.3.2", "", { "dependencies": { "@radix-ui/primitive": "1.1.2", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-presence": "1.1.4", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-controllable-state": "1.2.2", "@radix-ui/react-use-previous": "1.1.1", "@radix-ui/react-use-size": "1.1.1" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-yd+dI56KZqawxKZrJ31eENUwqc1QSqg4OZ15rybGjF2ZNwMO+wCyHzAVLRp9qoYJf7kYy0YpZ2b0JCzJ42HZpA=="], + + "@radix-ui/react-collapsible": ["@radix-ui/react-collapsible@1.1.11", "", { "dependencies": { "@radix-ui/primitive": "1.1.2", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-id": "1.1.1", "@radix-ui/react-presence": "1.1.4", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-controllable-state": "1.2.2", "@radix-ui/react-use-layout-effect": "1.1.1" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-2qrRsVGSCYasSz1RFOorXwl0H7g7J1frQtgpQgYrt+MOidtPAINHn9CPovQXb83r8ahapdx3Tu0fa/pdFFSdPg=="], + + "@radix-ui/react-collection": ["@radix-ui/react-collection@1.1.7", "", { "dependencies": { "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-slot": "1.2.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-Fh9rGN0MoI4ZFUNyfFVNU4y9LUz93u9/0K+yLgA2bwRojxM8JU1DyvvMBabnZPBgMWREAJvU2jjVzq+LrFUglw=="], + + "@radix-ui/react-compose-refs": ["@radix-ui/react-compose-refs@1.1.2", "", { "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-z4eqJvfiNnFMHIIvXP3CY57y2WJs5g2v3X0zm9mEJkrkNv4rDxu+sg9Jh8EkXyeqBkB7SOcboo9dMVqhyrACIg=="], + + "@radix-ui/react-context": ["@radix-ui/react-context@1.1.2", "", { "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-jCi/QKUM2r1Ju5a3J64TH2A5SpKAgh0LpknyqdQ4m6DCV0xJ2HG1xARRwNGPQfi1SLdLWZ1OJz6F4OMBBNiGJA=="], + + "@radix-ui/react-context-menu": ["@radix-ui/react-context-menu@2.2.15", "", { "dependencies": { "@radix-ui/primitive": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-menu": "2.1.15", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-callback-ref": "1.1.1", "@radix-ui/react-use-controllable-state": "1.2.2" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-UsQUMjcYTsBjTSXw0P3GO0werEQvUY2plgRQuKoCTtkNr45q1DiL51j4m7gxhABzZ0BadoXNsIbg7F3KwiUBbw=="], + + "@radix-ui/react-dialog": ["@radix-ui/react-dialog@1.1.14", "", { "dependencies": { "@radix-ui/primitive": "1.1.2", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-dismissable-layer": "1.1.10", "@radix-ui/react-focus-guards": "1.1.2", "@radix-ui/react-focus-scope": "1.1.7", "@radix-ui/react-id": "1.1.1", "@radix-ui/react-portal": "1.1.9", "@radix-ui/react-presence": "1.1.4", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-slot": "1.2.3", "@radix-ui/react-use-controllable-state": "1.2.2", "aria-hidden": "^1.2.4", "react-remove-scroll": "^2.6.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-+CpweKjqpzTmwRwcYECQcNYbI8V9VSQt0SNFKeEBLgfucbsLssU6Ppq7wUdNXEGb573bMjFhVjKVll8rmV6zMw=="], + + "@radix-ui/react-direction": ["@radix-ui/react-direction@1.1.1", "", { "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-1UEWRX6jnOA2y4H5WczZ44gOOjTEmlqv1uNW4GAJEO5+bauCBhv8snY65Iw5/VOS/ghKN9gr2KjnLKxrsvoMVw=="], + + "@radix-ui/react-dismissable-layer": ["@radix-ui/react-dismissable-layer@1.1.10", "", { "dependencies": { "@radix-ui/primitive": "1.1.2", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-callback-ref": "1.1.1", "@radix-ui/react-use-escape-keydown": "1.1.1" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-IM1zzRV4W3HtVgftdQiiOmA0AdJlCtMLe00FXaHwgt3rAnNsIyDqshvkIW3hj/iu5hu8ERP7KIYki6NkqDxAwQ=="], + + "@radix-ui/react-dropdown-menu": ["@radix-ui/react-dropdown-menu@2.1.15", "", { "dependencies": { "@radix-ui/primitive": "1.1.2", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-id": "1.1.1", "@radix-ui/react-menu": "2.1.15", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-controllable-state": "1.2.2" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-mIBnOjgwo9AH3FyKaSWoSu/dYj6VdhJ7frEPiGTeXCdUFHjl9h3mFh2wwhEtINOmYXWhdpf1rY2minFsmaNgVQ=="], + + "@radix-ui/react-focus-guards": ["@radix-ui/react-focus-guards@1.1.2", "", { "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-fyjAACV62oPV925xFCrH8DR5xWhg9KYtJT4s3u54jxp+L/hbpTY2kIeEFFbFe+a/HCE94zGQMZLIpVTPVZDhaA=="], + + "@radix-ui/react-focus-scope": ["@radix-ui/react-focus-scope@1.1.7", "", { "dependencies": { "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-callback-ref": "1.1.1" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-t2ODlkXBQyn7jkl6TNaw/MtVEVvIGelJDCG41Okq/KwUsJBwQ4XVZsHAVUkK4mBv3ewiAS3PGuUWuY2BoK4ZUw=="], + + "@radix-ui/react-form": ["@radix-ui/react-form@0.1.7", "", { "dependencies": { "@radix-ui/primitive": "1.1.2", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-id": "1.1.1", "@radix-ui/react-label": "2.1.7", "@radix-ui/react-primitive": "2.1.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-IXLKFnaYvFg/KkeV5QfOX7tRnwHXp127koOFUjLWMTrRv5Rny3DQcAtIFFeA/Cli4HHM8DuJCXAUsgnFVJndlw=="], + + "@radix-ui/react-hover-card": ["@radix-ui/react-hover-card@1.1.14", "", { "dependencies": { "@radix-ui/primitive": "1.1.2", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-dismissable-layer": "1.1.10", "@radix-ui/react-popper": "1.2.7", "@radix-ui/react-portal": "1.1.9", "@radix-ui/react-presence": "1.1.4", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-controllable-state": "1.2.2" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-CPYZ24Mhirm+g6D8jArmLzjYu4Eyg3TTUHswR26QgzXBHBe64BO/RHOJKzmF/Dxb4y4f9PKyJdwm/O/AhNkb+Q=="], + + "@radix-ui/react-icons": ["@radix-ui/react-icons@1.3.2", "", { "peerDependencies": { "react": "^16.x || ^17.x || ^18.x || ^19.0.0 || ^19.0.0-rc" } }, "sha512-fyQIhGDhzfc9pK2kH6Pl9c4BDJGfMkPqkyIgYDthyNYoNg3wVhoJMMh19WS4Up/1KMPFVpNsT2q3WmXn2N1m6g=="], + + "@radix-ui/react-id": ["@radix-ui/react-id@1.1.1", "", { "dependencies": { "@radix-ui/react-use-layout-effect": "1.1.1" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-kGkGegYIdQsOb4XjsfM97rXsiHaBwco+hFI66oO4s9LU+PLAC5oJ7khdOVFxkhsmlbpUqDAvXw11CluXP+jkHg=="], + + "@radix-ui/react-label": ["@radix-ui/react-label@2.1.7", "", { "dependencies": { "@radix-ui/react-primitive": "2.1.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-YT1GqPSL8kJn20djelMX7/cTRp/Y9w5IZHvfxQTVHrOqa2yMl7i/UfMqKRU5V7mEyKTrUVgJXhNQPVCG8PBLoQ=="], + + "@radix-ui/react-menu": ["@radix-ui/react-menu@2.1.15", "", { "dependencies": { "@radix-ui/primitive": "1.1.2", "@radix-ui/react-collection": "1.1.7", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-direction": "1.1.1", "@radix-ui/react-dismissable-layer": "1.1.10", "@radix-ui/react-focus-guards": "1.1.2", "@radix-ui/react-focus-scope": "1.1.7", "@radix-ui/react-id": "1.1.1", "@radix-ui/react-popper": "1.2.7", "@radix-ui/react-portal": "1.1.9", "@radix-ui/react-presence": "1.1.4", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-roving-focus": "1.1.10", "@radix-ui/react-slot": "1.2.3", "@radix-ui/react-use-callback-ref": "1.1.1", "aria-hidden": "^1.2.4", "react-remove-scroll": "^2.6.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-tVlmA3Vb9n8SZSd+YSbuFR66l87Wiy4du+YE+0hzKQEANA+7cWKH1WgqcEX4pXqxUFQKrWQGHdvEfw00TjFiew=="], + + "@radix-ui/react-menubar": ["@radix-ui/react-menubar@1.1.15", "", { "dependencies": { "@radix-ui/primitive": "1.1.2", "@radix-ui/react-collection": "1.1.7", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-direction": "1.1.1", "@radix-ui/react-id": "1.1.1", "@radix-ui/react-menu": "2.1.15", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-roving-focus": "1.1.10", "@radix-ui/react-use-controllable-state": "1.2.2" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-Z71C7LGD+YDYo3TV81paUs8f3Zbmkvg6VLRQpKYfzioOE6n7fOhA3ApK/V/2Odolxjoc4ENk8AYCjohCNayd5A=="], + + "@radix-ui/react-navigation-menu": ["@radix-ui/react-navigation-menu@1.2.13", "", { "dependencies": { "@radix-ui/primitive": "1.1.2", "@radix-ui/react-collection": "1.1.7", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-direction": "1.1.1", "@radix-ui/react-dismissable-layer": "1.1.10", "@radix-ui/react-id": "1.1.1", "@radix-ui/react-presence": "1.1.4", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-callback-ref": "1.1.1", "@radix-ui/react-use-controllable-state": "1.2.2", "@radix-ui/react-use-layout-effect": "1.1.1", "@radix-ui/react-use-previous": "1.1.1", "@radix-ui/react-visually-hidden": "1.2.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-WG8wWfDiJlSF5hELjwfjSGOXcBR/ZMhBFCGYe8vERpC39CQYZeq1PQ2kaYHdye3V95d06H89KGMsVCIE4LWo3g=="], + + "@radix-ui/react-one-time-password-field": ["@radix-ui/react-one-time-password-field@0.1.7", "", { "dependencies": { "@radix-ui/number": "1.1.1", "@radix-ui/primitive": "1.1.2", "@radix-ui/react-collection": "1.1.7", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-direction": "1.1.1", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-roving-focus": "1.1.10", "@radix-ui/react-use-controllable-state": "1.2.2", "@radix-ui/react-use-effect-event": "0.0.2", "@radix-ui/react-use-is-hydrated": "0.1.0", "@radix-ui/react-use-layout-effect": "1.1.1" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-w1vm7AGI8tNXVovOK7TYQHrAGpRF7qQL+ENpT1a743De5Zmay2RbWGKAiYDKIyIuqptns+znCKwNztE2xl1n0Q=="], + + "@radix-ui/react-password-toggle-field": ["@radix-ui/react-password-toggle-field@0.1.2", "", { "dependencies": { "@radix-ui/primitive": "1.1.2", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-id": "1.1.1", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-controllable-state": "1.2.2", "@radix-ui/react-use-effect-event": "0.0.2", "@radix-ui/react-use-is-hydrated": "0.1.0" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-F90uYnlBsLPU1UbSLciLsWQmk8+hdWa6SFw4GXaIdNWxFxI5ITKVdAG64f+Twaa9ic6xE7pqxPyUmodrGjT4pQ=="], + + "@radix-ui/react-popover": ["@radix-ui/react-popover@1.1.14", "", { "dependencies": { "@radix-ui/primitive": "1.1.2", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-dismissable-layer": "1.1.10", "@radix-ui/react-focus-guards": "1.1.2", "@radix-ui/react-focus-scope": "1.1.7", "@radix-ui/react-id": "1.1.1", "@radix-ui/react-popper": "1.2.7", "@radix-ui/react-portal": "1.1.9", "@radix-ui/react-presence": "1.1.4", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-slot": "1.2.3", "@radix-ui/react-use-controllable-state": "1.2.2", "aria-hidden": "^1.2.4", "react-remove-scroll": "^2.6.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-ODz16+1iIbGUfFEfKx2HTPKizg2MN39uIOV8MXeHnmdd3i/N9Wt7vU46wbHsqA0xoaQyXVcs0KIlBdOA2Y95bw=="], + + "@radix-ui/react-popper": ["@radix-ui/react-popper@1.2.7", "", { "dependencies": { "@floating-ui/react-dom": "^2.0.0", "@radix-ui/react-arrow": "1.1.7", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-callback-ref": "1.1.1", "@radix-ui/react-use-layout-effect": "1.1.1", "@radix-ui/react-use-rect": "1.1.1", "@radix-ui/react-use-size": "1.1.1", "@radix-ui/rect": "1.1.1" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-IUFAccz1JyKcf/RjB552PlWwxjeCJB8/4KxT7EhBHOJM+mN7LdW+B3kacJXILm32xawcMMjb2i0cIZpo+f9kiQ=="], + + "@radix-ui/react-portal": ["@radix-ui/react-portal@1.1.9", "", { "dependencies": { "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-layout-effect": "1.1.1" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-bpIxvq03if6UNwXZ+HTK71JLh4APvnXntDc6XOX8UVq4XQOVl7lwok0AvIl+b8zgCw3fSaVTZMpAPPagXbKmHQ=="], + + "@radix-ui/react-presence": ["@radix-ui/react-presence@1.1.4", "", { "dependencies": { "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-use-layout-effect": "1.1.1" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-ueDqRbdc4/bkaQT3GIpLQssRlFgWaL/U2z/S31qRwwLWoxHLgry3SIfCwhxeQNbirEUXFa+lq3RL3oBYXtcmIA=="], + + "@radix-ui/react-primitive": ["@radix-ui/react-primitive@2.1.3", "", { "dependencies": { "@radix-ui/react-slot": "1.2.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-m9gTwRkhy2lvCPe6QJp4d3G1TYEUHn/FzJUtq9MjH46an1wJU+GdoGC5VLof8RX8Ft/DlpshApkhswDLZzHIcQ=="], + + "@radix-ui/react-progress": ["@radix-ui/react-progress@1.1.7", "", { "dependencies": { "@radix-ui/react-context": "1.1.2", "@radix-ui/react-primitive": "2.1.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-vPdg/tF6YC/ynuBIJlk1mm7Le0VgW6ub6J2UWnTQ7/D23KXcPI1qy+0vBkgKgd38RCMJavBXpB83HPNFMTb0Fg=="], + + "@radix-ui/react-radio-group": ["@radix-ui/react-radio-group@1.3.7", "", { "dependencies": { "@radix-ui/primitive": "1.1.2", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-direction": "1.1.1", "@radix-ui/react-presence": "1.1.4", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-roving-focus": "1.1.10", "@radix-ui/react-use-controllable-state": "1.2.2", "@radix-ui/react-use-previous": "1.1.1", "@radix-ui/react-use-size": "1.1.1" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-9w5XhD0KPOrm92OTTE0SysH3sYzHsSTHNvZgUBo/VZ80VdYyB5RneDbc0dKpURS24IxkoFRu/hI0i4XyfFwY6g=="], + + "@radix-ui/react-roving-focus": ["@radix-ui/react-roving-focus@1.1.10", "", { "dependencies": { "@radix-ui/primitive": "1.1.2", "@radix-ui/react-collection": "1.1.7", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-direction": "1.1.1", "@radix-ui/react-id": "1.1.1", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-callback-ref": "1.1.1", "@radix-ui/react-use-controllable-state": "1.2.2" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-dT9aOXUen9JSsxnMPv/0VqySQf5eDQ6LCk5Sw28kamz8wSOW2bJdlX2Bg5VUIIcV+6XlHpWTIuTPCf/UNIyq8Q=="], + + "@radix-ui/react-scroll-area": ["@radix-ui/react-scroll-area@1.2.9", "", { "dependencies": { "@radix-ui/number": "1.1.1", "@radix-ui/primitive": "1.1.2", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-direction": "1.1.1", "@radix-ui/react-presence": "1.1.4", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-callback-ref": "1.1.1", "@radix-ui/react-use-layout-effect": "1.1.1" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-YSjEfBXnhUELsO2VzjdtYYD4CfQjvao+lhhrX5XsHD7/cyUNzljF1FHEbgTPN7LH2MClfwRMIsYlqTYpKTTe2A=="], + + "@radix-ui/react-select": ["@radix-ui/react-select@2.2.5", "", { "dependencies": { "@radix-ui/number": "1.1.1", "@radix-ui/primitive": "1.1.2", "@radix-ui/react-collection": "1.1.7", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-direction": "1.1.1", "@radix-ui/react-dismissable-layer": "1.1.10", "@radix-ui/react-focus-guards": "1.1.2", "@radix-ui/react-focus-scope": "1.1.7", "@radix-ui/react-id": "1.1.1", "@radix-ui/react-popper": "1.2.7", "@radix-ui/react-portal": "1.1.9", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-slot": "1.2.3", "@radix-ui/react-use-callback-ref": "1.1.1", "@radix-ui/react-use-controllable-state": "1.2.2", "@radix-ui/react-use-layout-effect": "1.1.1", "@radix-ui/react-use-previous": "1.1.1", "@radix-ui/react-visually-hidden": "1.2.3", "aria-hidden": "^1.2.4", "react-remove-scroll": "^2.6.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-HnMTdXEVuuyzx63ME0ut4+sEMYW6oouHWNGUZc7ddvUWIcfCva/AMoqEW/3wnEllriMWBa0RHspCYnfCWJQYmA=="], + + "@radix-ui/react-separator": ["@radix-ui/react-separator@1.1.7", "", { "dependencies": { "@radix-ui/react-primitive": "2.1.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-0HEb8R9E8A+jZjvmFCy/J4xhbXy3TV+9XSnGJ3KvTtjlIUy/YQ/p6UYZvi7YbeoeXdyU9+Y3scizK6hkY37baA=="], + + "@radix-ui/react-slider": ["@radix-ui/react-slider@1.3.5", "", { "dependencies": { "@radix-ui/number": "1.1.1", "@radix-ui/primitive": "1.1.2", "@radix-ui/react-collection": "1.1.7", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-direction": "1.1.1", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-controllable-state": "1.2.2", "@radix-ui/react-use-layout-effect": "1.1.1", "@radix-ui/react-use-previous": "1.1.1", "@radix-ui/react-use-size": "1.1.1" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-rkfe2pU2NBAYfGaxa3Mqosi7VZEWX5CxKaanRv0vZd4Zhl9fvQrg0VM93dv3xGLGfrHuoTRF3JXH8nb9g+B3fw=="], + + "@radix-ui/react-slot": ["@radix-ui/react-slot@1.2.3", "", { "dependencies": { "@radix-ui/react-compose-refs": "1.1.2" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-aeNmHnBxbi2St0au6VBVC7JXFlhLlOnvIIlePNniyUNAClzmtAUEY8/pBiK3iHjufOlwA+c20/8jngo7xcrg8A=="], + + "@radix-ui/react-switch": ["@radix-ui/react-switch@1.2.5", "", { "dependencies": { "@radix-ui/primitive": "1.1.2", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-controllable-state": "1.2.2", "@radix-ui/react-use-previous": "1.1.1", "@radix-ui/react-use-size": "1.1.1" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-5ijLkak6ZMylXsaImpZ8u4Rlf5grRmoc0p0QeX9VJtlrM4f5m3nCTX8tWga/zOA8PZYIR/t0p2Mnvd7InrJ6yQ=="], + + "@radix-ui/react-tabs": ["@radix-ui/react-tabs@1.1.12", "", { "dependencies": { "@radix-ui/primitive": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-direction": "1.1.1", "@radix-ui/react-id": "1.1.1", "@radix-ui/react-presence": "1.1.4", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-roving-focus": "1.1.10", "@radix-ui/react-use-controllable-state": "1.2.2" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-GTVAlRVrQrSw3cEARM0nAx73ixrWDPNZAruETn3oHCNP6SbZ/hNxdxp+u7VkIEv3/sFoLq1PfcHrl7Pnp0CDpw=="], + + "@radix-ui/react-toast": ["@radix-ui/react-toast@1.2.14", "", { "dependencies": { "@radix-ui/primitive": "1.1.2", "@radix-ui/react-collection": "1.1.7", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-dismissable-layer": "1.1.10", "@radix-ui/react-portal": "1.1.9", "@radix-ui/react-presence": "1.1.4", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-callback-ref": "1.1.1", "@radix-ui/react-use-controllable-state": "1.2.2", "@radix-ui/react-use-layout-effect": "1.1.1", "@radix-ui/react-visually-hidden": "1.2.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-nAP5FBxBJGQ/YfUB+r+O6USFVkWq3gAInkxyEnmvEV5jtSbfDhfa4hwX8CraCnbjMLsE7XSf/K75l9xXY7joWg=="], + + "@radix-ui/react-toggle": ["@radix-ui/react-toggle@1.1.9", "", { "dependencies": { "@radix-ui/primitive": "1.1.2", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-use-controllable-state": "1.2.2" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-ZoFkBBz9zv9GWer7wIjvdRxmh2wyc2oKWw6C6CseWd6/yq1DK/l5lJ+wnsmFwJZbBYqr02mrf8A2q/CVCuM3ZA=="], + + "@radix-ui/react-toggle-group": ["@radix-ui/react-toggle-group@1.1.10", "", { "dependencies": { "@radix-ui/primitive": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-direction": "1.1.1", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-roving-focus": "1.1.10", "@radix-ui/react-toggle": "1.1.9", "@radix-ui/react-use-controllable-state": "1.2.2" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-kiU694Km3WFLTC75DdqgM/3Jauf3rD9wxeS9XtyWFKsBUeZA337lC+6uUazT7I1DhanZ5gyD5Stf8uf2dbQxOQ=="], + + "@radix-ui/react-toolbar": ["@radix-ui/react-toolbar@1.1.10", "", { "dependencies": { "@radix-ui/primitive": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-direction": "1.1.1", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-roving-focus": "1.1.10", "@radix-ui/react-separator": "1.1.7", "@radix-ui/react-toggle-group": "1.1.10" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-jiwQsduEL++M4YBIurjSa+voD86OIytCod0/dbIxFZDLD8NfO1//keXYMfsW8BPcfqwoNjt+y06XcJqAb4KR7A=="], + + "@radix-ui/react-tooltip": ["@radix-ui/react-tooltip@1.2.7", "", { "dependencies": { "@radix-ui/primitive": "1.1.2", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-dismissable-layer": "1.1.10", "@radix-ui/react-id": "1.1.1", "@radix-ui/react-popper": "1.2.7", "@radix-ui/react-portal": "1.1.9", "@radix-ui/react-presence": "1.1.4", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-slot": "1.2.3", "@radix-ui/react-use-controllable-state": "1.2.2", "@radix-ui/react-visually-hidden": "1.2.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-Ap+fNYwKTYJ9pzqW+Xe2HtMRbQ/EeWkj2qykZ6SuEV4iS/o1bZI5ssJbk4D2r8XuDuOBVz/tIx2JObtuqU+5Zw=="], + + "@radix-ui/react-use-callback-ref": ["@radix-ui/react-use-callback-ref@1.1.1", "", { "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-FkBMwD+qbGQeMu1cOHnuGB6x4yzPjho8ap5WtbEJ26umhgqVXbhekKUQO+hZEL1vU92a3wHwdp0HAcqAUF5iDg=="], + + "@radix-ui/react-use-controllable-state": ["@radix-ui/react-use-controllable-state@1.2.2", "", { "dependencies": { "@radix-ui/react-use-effect-event": "0.0.2", "@radix-ui/react-use-layout-effect": "1.1.1" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-BjasUjixPFdS+NKkypcyyN5Pmg83Olst0+c6vGov0diwTEo6mgdqVR6hxcEgFuh4QrAs7Rc+9KuGJ9TVCj0Zzg=="], + + "@radix-ui/react-use-effect-event": ["@radix-ui/react-use-effect-event@0.0.2", "", { "dependencies": { "@radix-ui/react-use-layout-effect": "1.1.1" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-Qp8WbZOBe+blgpuUT+lw2xheLP8q0oatc9UpmiemEICxGvFLYmHm9QowVZGHtJlGbS6A6yJ3iViad/2cVjnOiA=="], + + "@radix-ui/react-use-escape-keydown": ["@radix-ui/react-use-escape-keydown@1.1.1", "", { "dependencies": { "@radix-ui/react-use-callback-ref": "1.1.1" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-Il0+boE7w/XebUHyBjroE+DbByORGR9KKmITzbR7MyQ4akpORYP/ZmbhAr0DG7RmmBqoOnZdy2QlvajJ2QA59g=="], + + "@radix-ui/react-use-is-hydrated": ["@radix-ui/react-use-is-hydrated@0.1.0", "", { "dependencies": { "use-sync-external-store": "^1.5.0" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-U+UORVEq+cTnRIaostJv9AGdV3G6Y+zbVd+12e18jQ5A3c0xL03IhnHuiU4UV69wolOQp5GfR58NW/EgdQhwOA=="], + + "@radix-ui/react-use-layout-effect": ["@radix-ui/react-use-layout-effect@1.1.1", "", { "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-RbJRS4UWQFkzHTTwVymMTUv8EqYhOp8dOOviLj2ugtTiXRaRQS7GLGxZTLL1jWhMeoSCf5zmcZkqTl9IiYfXcQ=="], + + "@radix-ui/react-use-previous": ["@radix-ui/react-use-previous@1.1.1", "", { "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-2dHfToCj/pzca2Ck724OZ5L0EVrr3eHRNsG/b3xQJLA2hZpVCS99bLAX+hm1IHXDEnzU6by5z/5MIY794/a8NQ=="], + + "@radix-ui/react-use-rect": ["@radix-ui/react-use-rect@1.1.1", "", { "dependencies": { "@radix-ui/rect": "1.1.1" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-QTYuDesS0VtuHNNvMh+CjlKJ4LJickCMUAqjlE3+j8w+RlRpwyX3apEQKGFzbZGdo7XNG1tXa+bQqIE7HIXT2w=="], + + "@radix-ui/react-use-size": ["@radix-ui/react-use-size@1.1.1", "", { "dependencies": { "@radix-ui/react-use-layout-effect": "1.1.1" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-ewrXRDTAqAXlkl6t/fkXWNAhFX9I+CkKlw6zjEwk86RSPKwZr3xpBRso655aqYafwtnbpHLj6toFzmd6xdVptQ=="], + + "@radix-ui/react-visually-hidden": ["@radix-ui/react-visually-hidden@1.2.3", "", { "dependencies": { "@radix-ui/react-primitive": "2.1.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-pzJq12tEaaIhqjbzpCuv/OypJY/BPavOofm+dbab+MHLajy277+1lLm6JFcGgF5eskJ6mquGirhXY2GD/8u8Ug=="], + + "@radix-ui/rect": ["@radix-ui/rect@1.1.1", "", {}, "sha512-HPwpGIzkl28mWyZqG52jiqDJ12waP11Pa1lGoiyUkIEuMLBP0oeK/C89esbXrxsky5we7dfd8U58nm0SgAWpVw=="], + + "@rolldown/pluginutils": ["@rolldown/pluginutils@1.0.0-beta.19", "", {}, "sha512-3FL3mnMbPu0muGOCaKAhhFEYmqv9eTfPSJRJmANrCwtgK8VuxpsZDGK+m0LYAGoyO8+0j5uRe4PeyPDK1yA/hA=="], + + "@rollup/pluginutils": ["@rollup/pluginutils@5.2.0", "", { "dependencies": { "@types/estree": "^1.0.0", "estree-walker": "^2.0.2", "picomatch": "^4.0.2" }, "peerDependencies": { "rollup": "^1.20.0||^2.0.0||^3.0.0||^4.0.0" }, "optionalPeers": ["rollup"] }, "sha512-qWJ2ZTbmumwiLFomfzTyt5Kng4hwPi9rwCYN4SHb6eaRU1KNO4ccxINHr/VhH4GgPlt1XfSTLX2LBTme8ne4Zw=="], + + "@rollup/rollup-android-arm-eabi": ["@rollup/rollup-android-arm-eabi@4.45.1", "", { "os": "android", "cpu": "arm" }, "sha512-NEySIFvMY0ZQO+utJkgoMiCAjMrGvnbDLHvcmlA33UXJpYBCvlBEbMMtV837uCkS+plG2umfhn0T5mMAxGrlRA=="], + + "@rollup/rollup-android-arm64": ["@rollup/rollup-android-arm64@4.45.1", "", { "os": "android", "cpu": "arm64" }, "sha512-ujQ+sMXJkg4LRJaYreaVx7Z/VMgBBd89wGS4qMrdtfUFZ+TSY5Rs9asgjitLwzeIbhwdEhyj29zhst3L1lKsRQ=="], + + "@rollup/rollup-darwin-arm64": ["@rollup/rollup-darwin-arm64@4.45.1", "", { "os": "darwin", "cpu": "arm64" }, "sha512-FSncqHvqTm3lC6Y13xncsdOYfxGSLnP+73k815EfNmpewPs+EyM49haPS105Rh4aF5mJKywk9X0ogzLXZzN9lA=="], + + "@rollup/rollup-darwin-x64": ["@rollup/rollup-darwin-x64@4.45.1", "", { "os": "darwin", "cpu": "x64" }, "sha512-2/vVn/husP5XI7Fsf/RlhDaQJ7x9zjvC81anIVbr4b/f0xtSmXQTFcGIQ/B1cXIYM6h2nAhJkdMHTnD7OtQ9Og=="], + + "@rollup/rollup-freebsd-arm64": ["@rollup/rollup-freebsd-arm64@4.45.1", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-4g1kaDxQItZsrkVTdYQ0bxu4ZIQ32cotoQbmsAnW1jAE4XCMbcBPDirX5fyUzdhVCKgPcrwWuucI8yrVRBw2+g=="], + + "@rollup/rollup-freebsd-x64": ["@rollup/rollup-freebsd-x64@4.45.1", "", { "os": "freebsd", "cpu": "x64" }, "sha512-L/6JsfiL74i3uK1Ti2ZFSNsp5NMiM4/kbbGEcOCps99aZx3g8SJMO1/9Y0n/qKlWZfn6sScf98lEOUe2mBvW9A=="], + + "@rollup/rollup-linux-arm-gnueabihf": ["@rollup/rollup-linux-arm-gnueabihf@4.45.1", "", { "os": "linux", "cpu": "arm" }, "sha512-RkdOTu2jK7brlu+ZwjMIZfdV2sSYHK2qR08FUWcIoqJC2eywHbXr0L8T/pONFwkGukQqERDheaGTeedG+rra6Q=="], + + "@rollup/rollup-linux-arm-musleabihf": ["@rollup/rollup-linux-arm-musleabihf@4.45.1", "", { "os": "linux", "cpu": "arm" }, "sha512-3kJ8pgfBt6CIIr1o+HQA7OZ9mp/zDk3ctekGl9qn/pRBgrRgfwiffaUmqioUGN9hv0OHv2gxmvdKOkARCtRb8Q=="], + + "@rollup/rollup-linux-arm64-gnu": ["@rollup/rollup-linux-arm64-gnu@4.45.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-k3dOKCfIVixWjG7OXTCOmDfJj3vbdhN0QYEqB+OuGArOChek22hn7Uy5A/gTDNAcCy5v2YcXRJ/Qcnm4/ma1xw=="], + + "@rollup/rollup-linux-arm64-musl": ["@rollup/rollup-linux-arm64-musl@4.45.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-PmI1vxQetnM58ZmDFl9/Uk2lpBBby6B6rF4muJc65uZbxCs0EA7hhKCk2PKlmZKuyVSHAyIw3+/SiuMLxKxWog=="], + + "@rollup/rollup-linux-loongarch64-gnu": ["@rollup/rollup-linux-loongarch64-gnu@4.45.1", "", { "os": "linux", "cpu": "none" }, "sha512-9UmI0VzGmNJ28ibHW2GpE2nF0PBQqsyiS4kcJ5vK+wuwGnV5RlqdczVocDSUfGX/Na7/XINRVoUgJyFIgipoRg=="], + + "@rollup/rollup-linux-powerpc64le-gnu": ["@rollup/rollup-linux-powerpc64le-gnu@4.45.1", "", { "os": "linux", "cpu": "ppc64" }, "sha512-7nR2KY8oEOUTD3pBAxIBBbZr0U7U+R9HDTPNy+5nVVHDXI4ikYniH1oxQz9VoB5PbBU1CZuDGHkLJkd3zLMWsg=="], + + "@rollup/rollup-linux-riscv64-gnu": ["@rollup/rollup-linux-riscv64-gnu@4.45.1", "", { "os": "linux", "cpu": "none" }, "sha512-nlcl3jgUultKROfZijKjRQLUu9Ma0PeNv/VFHkZiKbXTBQXhpytS8CIj5/NfBeECZtY2FJQubm6ltIxm/ftxpw=="], + + "@rollup/rollup-linux-riscv64-musl": ["@rollup/rollup-linux-riscv64-musl@4.45.1", "", { "os": "linux", "cpu": "none" }, "sha512-HJV65KLS51rW0VY6rvZkiieiBnurSzpzore1bMKAhunQiECPuxsROvyeaot/tcK3A3aGnI+qTHqisrpSgQrpgA=="], + + "@rollup/rollup-linux-s390x-gnu": ["@rollup/rollup-linux-s390x-gnu@4.45.1", "", { "os": "linux", "cpu": "s390x" }, "sha512-NITBOCv3Qqc6hhwFt7jLV78VEO/il4YcBzoMGGNxznLgRQf43VQDae0aAzKiBeEPIxnDrACiMgbqjuihx08OOw=="], + + "@rollup/rollup-linux-x64-gnu": ["@rollup/rollup-linux-x64-gnu@4.45.1", "", { "os": "linux", "cpu": "x64" }, "sha512-+E/lYl6qu1zqgPEnTrs4WysQtvc/Sh4fC2nByfFExqgYrqkKWp1tWIbe+ELhixnenSpBbLXNi6vbEEJ8M7fiHw=="], + + "@rollup/rollup-linux-x64-musl": ["@rollup/rollup-linux-x64-musl@4.45.1", "", { "os": "linux", "cpu": "x64" }, "sha512-a6WIAp89p3kpNoYStITT9RbTbTnqarU7D8N8F2CV+4Cl9fwCOZraLVuVFvlpsW0SbIiYtEnhCZBPLoNdRkjQFw=="], + + "@rollup/rollup-win32-arm64-msvc": ["@rollup/rollup-win32-arm64-msvc@4.45.1", "", { "os": "win32", "cpu": "arm64" }, "sha512-T5Bi/NS3fQiJeYdGvRpTAP5P02kqSOpqiopwhj0uaXB6nzs5JVi2XMJb18JUSKhCOX8+UE1UKQufyD6Or48dJg=="], + + "@rollup/rollup-win32-ia32-msvc": ["@rollup/rollup-win32-ia32-msvc@4.45.1", "", { "os": "win32", "cpu": "ia32" }, "sha512-lxV2Pako3ujjuUe9jiU3/s7KSrDfH6IgTSQOnDWr9aJ92YsFd7EurmClK0ly/t8dzMkDtd04g60WX6yl0sGfdw=="], + + "@rollup/rollup-win32-x64-msvc": ["@rollup/rollup-win32-x64-msvc@4.45.1", "", { "os": "win32", "cpu": "x64" }, "sha512-M/fKi4sasCdM8i0aWJjCSFm2qEnYRR8AMLG2kxp6wD13+tMGA4Z1tVAuHkNRjud5SW2EM3naLuK35w9twvf6aA=="], + + "@shikijs/core": ["@shikijs/core@1.29.2", "", { "dependencies": { "@shikijs/engine-javascript": "1.29.2", "@shikijs/engine-oniguruma": "1.29.2", "@shikijs/types": "1.29.2", "@shikijs/vscode-textmate": "^10.0.1", "@types/hast": "^3.0.4", "hast-util-to-html": "^9.0.4" } }, "sha512-vju0lY9r27jJfOY4Z7+Rt/nIOjzJpZ3y+nYpqtUZInVoXQ/TJZcfGnNOGnKjFdVZb8qexiCuSlZRKcGfhhTTZQ=="], + + "@shikijs/engine-javascript": ["@shikijs/engine-javascript@1.29.2", "", { "dependencies": { "@shikijs/types": "1.29.2", "@shikijs/vscode-textmate": "^10.0.1", "oniguruma-to-es": "^2.2.0" } }, "sha512-iNEZv4IrLYPv64Q6k7EPpOCE/nuvGiKl7zxdq0WFuRPF5PAE9PRo2JGq/d8crLusM59BRemJ4eOqrFrC4wiQ+A=="], + + "@shikijs/engine-oniguruma": ["@shikijs/engine-oniguruma@1.29.2", "", { "dependencies": { "@shikijs/types": "1.29.2", "@shikijs/vscode-textmate": "^10.0.1" } }, "sha512-7iiOx3SG8+g1MnlzZVDYiaeHe7Ez2Kf2HrJzdmGwkRisT7r4rak0e655AcM/tF9JG/kg5fMNYlLLKglbN7gBqA=="], + + "@shikijs/langs": ["@shikijs/langs@1.29.2", "", { "dependencies": { "@shikijs/types": "1.29.2" } }, "sha512-FIBA7N3LZ+223U7cJDUYd5shmciFQlYkFXlkKVaHsCPgfVLiO+e12FmQE6Tf9vuyEsFe3dIl8qGWKXgEHL9wmQ=="], + + "@shikijs/rehype": ["@shikijs/rehype@1.29.2", "", { "dependencies": { "@shikijs/types": "1.29.2", "@types/hast": "^3.0.4", "hast-util-to-string": "^3.0.1", "shiki": "1.29.2", "unified": "^11.0.5", "unist-util-visit": "^5.0.0" } }, "sha512-sxi53HZe5XDz0s2UqF+BVN/kgHPMS9l6dcacM4Ra3ZDzCJa5rDGJ+Ukpk4LxdD1+MITBM6hoLbPfGv9StV8a5Q=="], + + "@shikijs/themes": ["@shikijs/themes@1.29.2", "", { "dependencies": { "@shikijs/types": "1.29.2" } }, "sha512-i9TNZlsq4uoyqSbluIcZkmPL9Bfi3djVxRnofUHwvx/h6SRW3cwgBC5SML7vsDcWyukY0eCzVN980rqP6qNl9g=="], + + "@shikijs/transformers": ["@shikijs/transformers@1.29.2", "", { "dependencies": { "@shikijs/core": "1.29.2", "@shikijs/types": "1.29.2" } }, "sha512-NHQuA+gM7zGuxGWP9/Ub4vpbwrYCrho9nQCLcCPfOe3Yc7LOYwmSuhElI688oiqIXk9dlZwDiyAG9vPBTuPJMA=="], + + "@shikijs/twoslash": ["@shikijs/twoslash@1.29.2", "", { "dependencies": { "@shikijs/core": "1.29.2", "@shikijs/types": "1.29.2", "twoslash": "^0.2.12" } }, "sha512-2S04ppAEa477tiaLfGEn1QJWbZUmbk8UoPbAEw4PifsrxkBXtAtOflIZJNtuCwz8ptc/TPxy7CO7gW4Uoi6o/g=="], + + "@shikijs/types": ["@shikijs/types@1.29.2", "", { "dependencies": { "@shikijs/vscode-textmate": "^10.0.1", "@types/hast": "^3.0.4" } }, "sha512-VJjK0eIijTZf0QSTODEXCqinjBn0joAHQ+aPSBzrv4O2d/QSbsMw+ZeSRx03kV34Hy7NzUvV/7NqfYGRLrASmw=="], + + "@shikijs/vscode-textmate": ["@shikijs/vscode-textmate@10.0.2", "", {}, "sha512-83yeghZ2xxin3Nj8z1NMd/NCuca+gsYXswywDy5bHvwlWL8tpTQmzGeUuHd9FC3E/SBEMvzJRwWEOz5gGes9Qg=="], + + "@sindresorhus/merge-streams": ["@sindresorhus/merge-streams@2.3.0", "", {}, "sha512-LtoMMhxAlorcGhmFYI+LhPgbPZCkgP6ra1YL604EeF6U98pLlQ3iWIGMdWSC+vWmPBWBNgmDBAhnAobLROJmwg=="], + + "@tailwindcss/node": ["@tailwindcss/node@4.0.7", "", { "dependencies": { "enhanced-resolve": "^5.18.1", "jiti": "^2.4.2", "tailwindcss": "4.0.7" } }, "sha512-dkFXufkbRB2mu3FPsW5xLAUWJyexpJA+/VtQj18k3SUiJVLdpgzBd1v1gRRcIpEJj7K5KpxBKfOXlZxT3ZZRuA=="], + + "@tailwindcss/oxide": ["@tailwindcss/oxide@4.0.7", "", { "optionalDependencies": { "@tailwindcss/oxide-android-arm64": "4.0.7", "@tailwindcss/oxide-darwin-arm64": "4.0.7", "@tailwindcss/oxide-darwin-x64": "4.0.7", "@tailwindcss/oxide-freebsd-x64": "4.0.7", "@tailwindcss/oxide-linux-arm-gnueabihf": "4.0.7", "@tailwindcss/oxide-linux-arm64-gnu": "4.0.7", "@tailwindcss/oxide-linux-arm64-musl": "4.0.7", "@tailwindcss/oxide-linux-x64-gnu": "4.0.7", "@tailwindcss/oxide-linux-x64-musl": "4.0.7", "@tailwindcss/oxide-win32-arm64-msvc": "4.0.7", "@tailwindcss/oxide-win32-x64-msvc": "4.0.7" } }, "sha512-yr6w5YMgjy+B+zkJiJtIYGXW+HNYOPfRPtSs+aqLnKwdEzNrGv4ZuJh9hYJ3mcA+HMq/K1rtFV+KsEr65S558g=="], + + "@tailwindcss/oxide-android-arm64": ["@tailwindcss/oxide-android-arm64@4.0.7", "", { "os": "android", "cpu": "arm64" }, "sha512-5iQXXcAeOHBZy8ASfHFm1k0O/9wR2E3tKh6+P+ilZZbQiMgu+qrnfpBWYPc3FPuQdWiWb73069WT5D+CAfx/tg=="], + + "@tailwindcss/oxide-darwin-arm64": ["@tailwindcss/oxide-darwin-arm64@4.0.7", "", { "os": "darwin", "cpu": "arm64" }, "sha512-7yGZtEc5IgVYylqK/2B0yVqoofk4UAbkn1ygNpIJZyrOhbymsfr8uUFCueTu2fUxmAYIfMZ8waWo2dLg/NgLgg=="], + + "@tailwindcss/oxide-darwin-x64": ["@tailwindcss/oxide-darwin-x64@4.0.7", "", { "os": "darwin", "cpu": "x64" }, "sha512-tPQDV20fBjb26yWbPqT1ZSoDChomMCiXTKn4jupMSoMCFyU7+OJvIY1ryjqBuY622dEBJ8LnCDDWsnj1lX9nNQ=="], + + "@tailwindcss/oxide-freebsd-x64": ["@tailwindcss/oxide-freebsd-x64@4.0.7", "", { "os": "freebsd", "cpu": "x64" }, "sha512-sZqJpTyTZiknU9LLHuByg5GKTW+u3FqM7q7myequAXxKOpAFiOfXpY710FuMY+gjzSapyRbDXJlsTQtCyiTo5w=="], + + "@tailwindcss/oxide-linux-arm-gnueabihf": ["@tailwindcss/oxide-linux-arm-gnueabihf@4.0.7", "", { "os": "linux", "cpu": "arm" }, "sha512-PBgvULgeSswjd8cbZ91gdIcIDMdc3TUHV5XemEpxlqt9M8KoydJzkuB/Dt910jYdofOIaTWRL6adG9nJICvU4A=="], + + "@tailwindcss/oxide-linux-arm64-gnu": ["@tailwindcss/oxide-linux-arm64-gnu@4.0.7", "", { "os": "linux", "cpu": "arm64" }, "sha512-By/a2yeh+e9b+C67F88ndSwVJl2A3tcUDb29FbedDi+DZ4Mr07Oqw9Y1DrDrtHIDhIZ3bmmiL1dkH2YxrtV+zw=="], + + "@tailwindcss/oxide-linux-arm64-musl": ["@tailwindcss/oxide-linux-arm64-musl@4.0.7", "", { "os": "linux", "cpu": "arm64" }, "sha512-WHYs3cpPEJb/ccyT20NOzopYQkl7JKncNBUbb77YFlwlXMVJLLV3nrXQKhr7DmZxz2ZXqjyUwsj2rdzd9stYdw=="], + + "@tailwindcss/oxide-linux-x64-gnu": ["@tailwindcss/oxide-linux-x64-gnu@4.0.7", "", { "os": "linux", "cpu": "x64" }, "sha512-7bP1UyuX9kFxbOwkeIJhBZNevKYPXB6xZI37v09fqi6rqRJR8elybwjMUHm54GVP+UTtJ14ueB1K54Dy1tIO6w=="], + + "@tailwindcss/oxide-linux-x64-musl": ["@tailwindcss/oxide-linux-x64-musl@4.0.7", "", { "os": "linux", "cpu": "x64" }, "sha512-gBQIV8nL/LuhARNGeroqzXymMzzW5wQzqlteVqOVoqwEfpHOP3GMird5pGFbnpY+NP0fOlsZGrxxOPQ4W/84bQ=="], + + "@tailwindcss/oxide-win32-arm64-msvc": ["@tailwindcss/oxide-win32-arm64-msvc@4.0.7", "", { "os": "win32", "cpu": "arm64" }, "sha512-aH530NFfx0kpQpvYMfWoeG03zGnRCMVlQG8do/5XeahYydz+6SIBxA1tl/cyITSJyWZHyVt6GVNkXeAD30v0Xg=="], + + "@tailwindcss/oxide-win32-x64-msvc": ["@tailwindcss/oxide-win32-x64-msvc@4.0.7", "", { "os": "win32", "cpu": "x64" }, "sha512-8Cva6bbJN7ZJx320k7vxGGdU0ewmpfS5A4PudyzUuofdi8MgeINuiiWiPQ0VZCda/GX88K6qp+6UpDZNVr8HMQ=="], + + "@tailwindcss/vite": ["@tailwindcss/vite@4.0.7", "", { "dependencies": { "@tailwindcss/node": "4.0.7", "@tailwindcss/oxide": "4.0.7", "lightningcss": "^1.29.1", "tailwindcss": "4.0.7" }, "peerDependencies": { "vite": "^5.2.0 || ^6" } }, "sha512-GYx5sxArfIMtdZCsxfya3S/efMmf4RvfqdiLUozkhmSFBNUFnYVodatpoO/en4/BsOIGvq/RB6HwcTLn9prFnQ=="], + + "@types/babel__core": ["@types/babel__core@7.20.5", "", { "dependencies": { "@babel/parser": "^7.20.7", "@babel/types": "^7.20.7", "@types/babel__generator": "*", "@types/babel__template": "*", "@types/babel__traverse": "*" } }, "sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA=="], + + "@types/babel__generator": ["@types/babel__generator@7.27.0", "", { "dependencies": { "@babel/types": "^7.0.0" } }, "sha512-ufFd2Xi92OAVPYsy+P4n7/U7e68fex0+Ee8gSG9KX7eo084CWiQ4sdxktvdl0bOPupXtVJPY19zk6EwWqUQ8lg=="], + + "@types/babel__template": ["@types/babel__template@7.4.4", "", { "dependencies": { "@babel/parser": "^7.1.0", "@babel/types": "^7.0.0" } }, "sha512-h/NUaSyG5EyxBIp8YRxo4RMe2/qQgvyowRwVMzhYhBCONbW8PUsg4lkFMrhgZhUe5z3L3MiLDuvyJ/CaPa2A8A=="], + + "@types/babel__traverse": ["@types/babel__traverse@7.20.7", "", { "dependencies": { "@babel/types": "^7.20.7" } }, "sha512-dkO5fhS7+/oos4ciWxyEyjWe48zmG6wbCheo/G2ZnHx4fs3EU6YC6UM8rk56gAjNJ9P3MTH2jo5jb92/K6wbng=="], + + "@types/d3": ["@types/d3@7.4.3", "", { "dependencies": { "@types/d3-array": "*", "@types/d3-axis": "*", "@types/d3-brush": "*", "@types/d3-chord": "*", "@types/d3-color": "*", "@types/d3-contour": "*", "@types/d3-delaunay": "*", "@types/d3-dispatch": "*", "@types/d3-drag": "*", "@types/d3-dsv": "*", "@types/d3-ease": "*", "@types/d3-fetch": "*", "@types/d3-force": "*", "@types/d3-format": "*", "@types/d3-geo": "*", "@types/d3-hierarchy": "*", "@types/d3-interpolate": "*", "@types/d3-path": "*", "@types/d3-polygon": "*", "@types/d3-quadtree": "*", "@types/d3-random": "*", "@types/d3-scale": "*", "@types/d3-scale-chromatic": "*", "@types/d3-selection": "*", "@types/d3-shape": "*", "@types/d3-time": "*", "@types/d3-time-format": "*", "@types/d3-timer": "*", "@types/d3-transition": "*", "@types/d3-zoom": "*" } }, "sha512-lZXZ9ckh5R8uiFVt8ogUNf+pIrK4EsWrx2Np75WvF/eTpJ0FMHNhjXk8CKEx/+gpHbNQyJWehbFaTvqmHWB3ww=="], + + "@types/d3-array": ["@types/d3-array@3.2.1", "", {}, "sha512-Y2Jn2idRrLzUfAKV2LyRImR+y4oa2AntrgID95SHJxuMUrkNXmanDSed71sRNZysveJVt1hLLemQZIady0FpEg=="], + + "@types/d3-axis": ["@types/d3-axis@3.0.6", "", { "dependencies": { "@types/d3-selection": "*" } }, "sha512-pYeijfZuBd87T0hGn0FO1vQ/cgLk6E1ALJjfkC0oJ8cbwkZl3TpgS8bVBLZN+2jjGgg38epgxb2zmoGtSfvgMw=="], + + "@types/d3-brush": ["@types/d3-brush@3.0.6", "", { "dependencies": { "@types/d3-selection": "*" } }, "sha512-nH60IZNNxEcrh6L1ZSMNA28rj27ut/2ZmI3r96Zd+1jrZD++zD3LsMIjWlvg4AYrHn/Pqz4CF3veCxGjtbqt7A=="], + + "@types/d3-chord": ["@types/d3-chord@3.0.6", "", {}, "sha512-LFYWWd8nwfwEmTZG9PfQxd17HbNPksHBiJHaKuY1XeqscXacsS2tyoo6OdRsjf+NQYeB6XrNL3a25E3gH69lcg=="], + + "@types/d3-color": ["@types/d3-color@3.1.3", "", {}, "sha512-iO90scth9WAbmgv7ogoq57O9YpKmFBbmoEoCHDB2xMBY0+/KVrqAaCDyCE16dUspeOvIxFFRI+0sEtqDqy2b4A=="], + + "@types/d3-contour": ["@types/d3-contour@3.0.6", "", { "dependencies": { "@types/d3-array": "*", "@types/geojson": "*" } }, "sha512-BjzLgXGnCWjUSYGfH1cpdo41/hgdWETu4YxpezoztawmqsvCeep+8QGfiY6YbDvfgHz/DkjeIkkZVJavB4a3rg=="], + + "@types/d3-delaunay": ["@types/d3-delaunay@6.0.4", "", {}, "sha512-ZMaSKu4THYCU6sV64Lhg6qjf1orxBthaC161plr5KuPHo3CNm8DTHiLw/5Eq2b6TsNP0W0iJrUOFscY6Q450Hw=="], + + "@types/d3-dispatch": ["@types/d3-dispatch@3.0.6", "", {}, "sha512-4fvZhzMeeuBJYZXRXrRIQnvUYfyXwYmLsdiN7XXmVNQKKw1cM8a5WdID0g1hVFZDqT9ZqZEY5pD44p24VS7iZQ=="], + + "@types/d3-drag": ["@types/d3-drag@3.0.7", "", { "dependencies": { "@types/d3-selection": "*" } }, "sha512-HE3jVKlzU9AaMazNufooRJ5ZpWmLIoc90A37WU2JMmeq28w1FQqCZswHZ3xR+SuxYftzHq6WU6KJHvqxKzTxxQ=="], + + "@types/d3-dsv": ["@types/d3-dsv@3.0.7", "", {}, "sha512-n6QBF9/+XASqcKK6waudgL0pf/S5XHPPI8APyMLLUHd8NqouBGLsU8MgtO7NINGtPBtk9Kko/W4ea0oAspwh9g=="], + + "@types/d3-ease": ["@types/d3-ease@3.0.2", "", {}, "sha512-NcV1JjO5oDzoK26oMzbILE6HW7uVXOHLQvHshBUW4UMdZGfiY6v5BeQwh9a9tCzv+CeefZQHJt5SRgK154RtiA=="], + + "@types/d3-fetch": ["@types/d3-fetch@3.0.7", "", { "dependencies": { "@types/d3-dsv": "*" } }, "sha512-fTAfNmxSb9SOWNB9IoG5c8Hg6R+AzUHDRlsXsDZsNp6sxAEOP0tkP3gKkNSO/qmHPoBFTxNrjDprVHDQDvo5aA=="], + + "@types/d3-force": ["@types/d3-force@3.0.10", "", {}, "sha512-ZYeSaCF3p73RdOKcjj+swRlZfnYpK1EbaDiYICEEp5Q6sUiqFaFQ9qgoshp5CzIyyb/yD09kD9o2zEltCexlgw=="], + + "@types/d3-format": ["@types/d3-format@3.0.4", "", {}, "sha512-fALi2aI6shfg7vM5KiR1wNJnZ7r6UuggVqtDA+xiEdPZQwy/trcQaHnwShLuLdta2rTymCNpxYTiMZX/e09F4g=="], + + "@types/d3-geo": ["@types/d3-geo@3.1.0", "", { "dependencies": { "@types/geojson": "*" } }, "sha512-856sckF0oP/diXtS4jNsiQw/UuK5fQG8l/a9VVLeSouf1/PPbBE1i1W852zVwKwYCBkFJJB7nCFTbk6UMEXBOQ=="], + + "@types/d3-hierarchy": ["@types/d3-hierarchy@3.1.7", "", {}, "sha512-tJFtNoYBtRtkNysX1Xq4sxtjK8YgoWUNpIiUee0/jHGRwqvzYxkq0hGVbbOGSz+JgFxxRu4K8nb3YpG3CMARtg=="], + + "@types/d3-interpolate": ["@types/d3-interpolate@3.0.4", "", { "dependencies": { "@types/d3-color": "*" } }, "sha512-mgLPETlrpVV1YRJIglr4Ez47g7Yxjl1lj7YKsiMCb27VJH9W8NVM6Bb9d8kkpG/uAQS5AmbA48q2IAolKKo1MA=="], + + "@types/d3-path": ["@types/d3-path@3.1.1", "", {}, "sha512-VMZBYyQvbGmWyWVea0EHs/BwLgxc+MKi1zLDCONksozI4YJMcTt8ZEuIR4Sb1MMTE8MMW49v0IwI5+b7RmfWlg=="], + + "@types/d3-polygon": ["@types/d3-polygon@3.0.2", "", {}, "sha512-ZuWOtMaHCkN9xoeEMr1ubW2nGWsp4nIql+OPQRstu4ypeZ+zk3YKqQT0CXVe/PYqrKpZAi+J9mTs05TKwjXSRA=="], + + "@types/d3-quadtree": ["@types/d3-quadtree@3.0.6", "", {}, "sha512-oUzyO1/Zm6rsxKRHA1vH0NEDG58HrT5icx/azi9MF1TWdtttWl0UIUsjEQBBh+SIkrpd21ZjEv7ptxWys1ncsg=="], + + "@types/d3-random": ["@types/d3-random@3.0.3", "", {}, "sha512-Imagg1vJ3y76Y2ea0871wpabqp613+8/r0mCLEBfdtqC7xMSfj9idOnmBYyMoULfHePJyxMAw3nWhJxzc+LFwQ=="], + + "@types/d3-scale": ["@types/d3-scale@4.0.9", "", { "dependencies": { "@types/d3-time": "*" } }, "sha512-dLmtwB8zkAeO/juAMfnV+sItKjlsw2lKdZVVy6LRr0cBmegxSABiLEpGVmSJJ8O08i4+sGR6qQtb6WtuwJdvVw=="], + + "@types/d3-scale-chromatic": ["@types/d3-scale-chromatic@3.1.0", "", {}, "sha512-iWMJgwkK7yTRmWqRB5plb1kadXyQ5Sj8V/zYlFGMUBbIPKQScw+Dku9cAAMgJG+z5GYDoMjWGLVOvjghDEFnKQ=="], + + "@types/d3-selection": ["@types/d3-selection@3.0.11", "", {}, "sha512-bhAXu23DJWsrI45xafYpkQ4NtcKMwWnAC/vKrd2l+nxMFuvOT3XMYTIj2opv8vq8AO5Yh7Qac/nSeP/3zjTK0w=="], + + "@types/d3-shape": ["@types/d3-shape@3.1.7", "", { "dependencies": { "@types/d3-path": "*" } }, "sha512-VLvUQ33C+3J+8p+Daf+nYSOsjB4GXp19/S/aGo60m9h1v6XaxjiT82lKVWJCfzhtuZ3yD7i/TPeC/fuKLLOSmg=="], + + "@types/d3-time": ["@types/d3-time@3.0.4", "", {}, "sha512-yuzZug1nkAAaBlBBikKZTgzCeA+k1uy4ZFwWANOfKw5z5LRhV0gNA7gNkKm7HoK+HRN0wX3EkxGk0fpbWhmB7g=="], + + "@types/d3-time-format": ["@types/d3-time-format@4.0.3", "", {}, "sha512-5xg9rC+wWL8kdDj153qZcsJ0FWiFt0J5RB6LYUNZjwSnesfblqrI/bJ1wBdJ8OQfncgbJG5+2F+qfqnqyzYxyg=="], + + "@types/d3-timer": ["@types/d3-timer@3.0.2", "", {}, "sha512-Ps3T8E8dZDam6fUyNiMkekK3XUsaUEik+idO9/YjPtfj2qruF8tFBXS7XhtE4iIXBLxhmLjP3SXpLhVf21I9Lw=="], + + "@types/d3-transition": ["@types/d3-transition@3.0.9", "", { "dependencies": { "@types/d3-selection": "*" } }, "sha512-uZS5shfxzO3rGlu0cC3bjmMFKsXv+SmZZcgp0KD22ts4uGXp5EVYGzu/0YdwZeKmddhcAccYtREJKkPfXkZuCg=="], + + "@types/d3-zoom": ["@types/d3-zoom@3.0.8", "", { "dependencies": { "@types/d3-interpolate": "*", "@types/d3-selection": "*" } }, "sha512-iqMC4/YlFCSlO8+2Ii1GGGliCAY4XdeG748w5vQUbevlbDu0zSjH/+jojorQVBK/se0j6DUFNPBGSqD3YWYnDw=="], + + "@types/debug": ["@types/debug@4.1.12", "", { "dependencies": { "@types/ms": "*" } }, "sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ=="], + + "@types/estree": ["@types/estree@1.0.8", "", {}, "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w=="], + + "@types/estree-jsx": ["@types/estree-jsx@1.0.5", "", { "dependencies": { "@types/estree": "*" } }, "sha512-52CcUVNFyfb1A2ALocQw/Dd1BQFNmSdkuC3BkZ6iqhdMfQz7JWOFRuJFloOzjk+6WijU56m9oKXFAXc7o3Towg=="], + + "@types/geojson": ["@types/geojson@7946.0.16", "", {}, "sha512-6C8nqWur3j98U6+lXDfTUWIfgvZU+EumvpHKcYjujKH7woYyLj2sUmff0tRhrqM7BohUw7Pz3ZB1jj2gW9Fvmg=="], + + "@types/hast": ["@types/hast@3.0.4", "", { "dependencies": { "@types/unist": "*" } }, "sha512-WPs+bbQw5aCj+x6laNGWLH3wviHtoCv/P3+otBhbOhJgG8qtpdAMlTCxLtsTWA7LH1Oh/bFCHsBn0TPS5m30EQ=="], + + "@types/mdast": ["@types/mdast@4.0.4", "", { "dependencies": { "@types/unist": "*" } }, "sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA=="], + + "@types/mdx": ["@types/mdx@2.0.13", "", {}, "sha512-+OWZQfAYyio6YkJb3HLxDrvnx6SWWDbC0zVPfBRzUk0/nqoDyf6dNxQi3eArPe8rJ473nobTMQ/8Zk+LxJ+Yuw=="], + + "@types/ms": ["@types/ms@2.1.0", "", {}, "sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA=="], + + "@types/node": ["@types/node@24.0.14", "", { "dependencies": { "undici-types": "~7.8.0" } }, "sha512-4zXMWD91vBLGRtHK3YbIoFMia+1nqEz72coM42C5ETjnNCa/heoj7NT1G67iAfOqMmcfhuCZ4uNpyz8EjlAejw=="], + + "@types/react": ["@types/react@19.1.8", "", { "dependencies": { "csstype": "^3.0.2" } }, "sha512-AwAfQ2Wa5bCx9WP8nZL2uMZWod7J7/JSplxbTmBQ5ms6QpqNYm672H0Vu9ZVKVngQ+ii4R/byguVEUZQyeg44g=="], + + "@types/trusted-types": ["@types/trusted-types@2.0.7", "", {}, "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw=="], + + "@types/unist": ["@types/unist@3.0.3", "", {}, "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q=="], + + "@typescript/vfs": ["@typescript/vfs@1.6.1", "", { "dependencies": { "debug": "^4.1.1" }, "peerDependencies": { "typescript": "*" } }, "sha512-JwoxboBh7Oz1v38tPbkrZ62ZXNHAk9bJ7c9x0eI5zBfBnBYGhURdbnh7Z4smN/MV48Y5OCcZb58n972UtbazsA=="], + + "@ungap/structured-clone": ["@ungap/structured-clone@1.3.0", "", {}, "sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g=="], + + "@vanilla-extract/babel-plugin-debug-ids": ["@vanilla-extract/babel-plugin-debug-ids@1.2.2", "", { "dependencies": { "@babel/core": "^7.23.9" } }, "sha512-MeDWGICAF9zA/OZLOKwhoRlsUW+fiMwnfuOAqFVohL31Agj7Q/RBWAYweqjHLgFBCsdnr6XIfwjJnmb2znEWxw=="], + + "@vanilla-extract/compiler": ["@vanilla-extract/compiler@0.3.0", "", { "dependencies": { "@vanilla-extract/css": "^1.17.4", "@vanilla-extract/integration": "^8.0.4", "vite": "^5.0.0 || ^6.0.0", "vite-node": "^3.2.2" } }, "sha512-8EbPmDMXhY9NrN38Kh8xYDENgBk4i6s6ce4p7E9F3kHtCqxtEgfaKSNS08z/SVCTmaX3IB3N/kGSO0gr+APffg=="], + + "@vanilla-extract/css": ["@vanilla-extract/css@1.17.4", "", { "dependencies": { "@emotion/hash": "^0.9.0", "@vanilla-extract/private": "^1.0.9", "css-what": "^6.1.0", "cssesc": "^3.0.0", "csstype": "^3.0.7", "dedent": "^1.5.3", "deep-object-diff": "^1.1.9", "deepmerge": "^4.2.2", "lru-cache": "^10.4.3", "media-query-parser": "^2.0.2", "modern-ahocorasick": "^1.0.0", "picocolors": "^1.0.0" } }, "sha512-m3g9nQDWPtL+sTFdtCGRMI1Vrp86Ay4PBYq1Bo7Bnchj5ElNtAJpOqD+zg+apthVA4fB7oVpMWNjwpa6ElDWFQ=="], + + "@vanilla-extract/dynamic": ["@vanilla-extract/dynamic@2.1.5", "", { "dependencies": { "@vanilla-extract/private": "^1.0.9" } }, "sha512-QGIFGb1qyXQkbzx6X6i3+3LMc/iv/ZMBttMBL+Wm/DetQd36KsKsFg5CtH3qy+1hCA/5w93mEIIAiL4fkM8ycw=="], + + "@vanilla-extract/integration": ["@vanilla-extract/integration@8.0.4", "", { "dependencies": { "@babel/core": "^7.23.9", "@babel/plugin-syntax-typescript": "^7.23.3", "@vanilla-extract/babel-plugin-debug-ids": "^1.2.2", "@vanilla-extract/css": "^1.17.4", "dedent": "^1.5.3", "esbuild": "npm:esbuild@>=0.17.6 <0.26.0", "eval": "0.1.8", "find-up": "^5.0.0", "javascript-stringify": "^2.0.1", "mlly": "^1.4.2" } }, "sha512-cmOb7tR+g3ulKvFtSbmdw3YUyIS1d7MQqN+FcbwNhdieyno5xzUyfDCMjeWJhmCSMvZ6WlinkrOkgs6SHB+FRg=="], + + "@vanilla-extract/private": ["@vanilla-extract/private@1.0.9", "", {}, "sha512-gT2jbfZuaaCLrAxwXbRgIhGhcXbRZCG3v4TTUnjw0EJ7ArdBRxkq4msNJkbuRkCgfIK5ATmprB5t9ljvLeFDEA=="], + + "@vanilla-extract/vite-plugin": ["@vanilla-extract/vite-plugin@5.1.0", "", { "dependencies": { "@vanilla-extract/compiler": "^0.3.0", "@vanilla-extract/integration": "^8.0.4" }, "peerDependencies": { "vite": "^5.0.0 || ^6.0.0" } }, "sha512-BzVdmBD+FUyJnY6I29ZezwtDBc1B78l+VvHvIgoJYbgfPj0hvY0RmrGL8B4oNNGY/lOt7KgQflXY5kBMd3MGZg=="], + + "@vitejs/plugin-react": ["@vitejs/plugin-react@4.6.0", "", { "dependencies": { "@babel/core": "^7.27.4", "@babel/plugin-transform-react-jsx-self": "^7.27.1", "@babel/plugin-transform-react-jsx-source": "^7.27.1", "@rolldown/pluginutils": "1.0.0-beta.19", "@types/babel__core": "^7.20.5", "react-refresh": "^0.17.0" }, "peerDependencies": { "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0-beta.0" } }, "sha512-5Kgff+m8e2PB+9j51eGHEpn5kUzRKH2Ry0qGoe8ItJg7pqnkPrYPkDQZGgGmTa0EGarHrkjLvOdU3b1fzI8otQ=="], + + "acorn": ["acorn@8.15.0", "", { "bin": { "acorn": "bin/acorn" } }, "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg=="], + + "acorn-jsx": ["acorn-jsx@5.3.2", "", { "peerDependencies": { "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ=="], + + "ansi-regex": ["ansi-regex@6.1.0", "", {}, "sha512-7HSX4QQb4CspciLpVFwyRe79O3xsIZDDLER21kERQ71oaPodF8jL725AgJMFAYbooIqolJoRLuM81SpeUkpkvA=="], + + "ansi-styles": ["ansi-styles@6.2.1", "", {}, "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug=="], + + "aria-hidden": ["aria-hidden@1.2.6", "", { "dependencies": { "tslib": "^2.0.0" } }, "sha512-ik3ZgC9dY/lYVVM++OISsaYDeg1tb0VtP5uL3ouh1koGOaUMDPpbFIei4JkFimWUFPn90sbMNMXQAIVOlnYKJA=="], + + "astring": ["astring@1.9.0", "", { "bin": { "astring": "bin/astring" } }, "sha512-LElXdjswlqjWrPpJFg1Fx4wpkOCxj1TDHlSV4PlaRxHGWko024xICaa97ZkMfs6DRKlCguiAI+rbXv5GWwXIkg=="], + + "autoprefixer": ["autoprefixer@10.4.21", "", { "dependencies": { "browserslist": "^4.24.4", "caniuse-lite": "^1.0.30001702", "fraction.js": "^4.3.7", "normalize-range": "^0.1.2", "picocolors": "^1.1.1", "postcss-value-parser": "^4.2.0" }, "peerDependencies": { "postcss": "^8.1.0" }, "bin": { "autoprefixer": "bin/autoprefixer" } }, "sha512-O+A6LWV5LDHSJD3LjHYoNi4VLsj/Whi7k6zG12xTYaU4cQ8oxQGckXNX8cRHK5yOZ/ppVHe0ZBXGzSV9jXdVbQ=="], + + "bail": ["bail@2.0.2", "", {}, "sha512-0xO6mYd7JB2YesxDKplafRpsiOzPt9V02ddPCLbY1xYGPOX24NTyN50qnUxgCPcSoYMhKpAuBTjQoRZCAkUDRw=="], + + "balanced-match": ["balanced-match@1.0.2", "", {}, "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="], + + "base64-js": ["base64-js@1.5.1", "", {}, "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA=="], + + "bcp-47-match": ["bcp-47-match@2.0.3", "", {}, "sha512-JtTezzbAibu8G0R9op9zb3vcWZd9JF6M0xOYGPn0fNCd7wOpRB1mU2mH9T8gaBGbAAyIIVgB2G7xG0GP98zMAQ=="], + + "bl": ["bl@5.1.0", "", { "dependencies": { "buffer": "^6.0.3", "inherits": "^2.0.4", "readable-stream": "^3.4.0" } }, "sha512-tv1ZJHLfTDnXE6tMHv73YgSJaWR2AFuPwMntBe7XL/GBFHnT0CLnsHMogfk5+GzCDC5ZWarSCYaIGATZt9dNsQ=="], + + "boolbase": ["boolbase@1.0.0", "", {}, "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww=="], + + "brace-expansion": ["brace-expansion@2.0.2", "", { "dependencies": { "balanced-match": "^1.0.0" } }, "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ=="], + + "braces": ["braces@3.0.3", "", { "dependencies": { "fill-range": "^7.1.1" } }, "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA=="], + + "browserslist": ["browserslist@4.25.1", "", { "dependencies": { "caniuse-lite": "^1.0.30001726", "electron-to-chromium": "^1.5.173", "node-releases": "^2.0.19", "update-browserslist-db": "^1.1.3" }, "bin": { "browserslist": "cli.js" } }, "sha512-KGj0KoOMXLpSNkkEI6Z6mShmQy0bc1I+T7K9N81k4WWMrfz+6fQ6es80B/YLAeRoKvjYE1YSHHOW1qe9xIVzHw=="], + + "buffer": ["buffer@6.0.3", "", { "dependencies": { "base64-js": "^1.3.1", "ieee754": "^1.2.1" } }, "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA=="], + + "bytes": ["bytes@3.1.2", "", {}, "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg=="], + + "cac": ["cac@6.7.14", "", {}, "sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ=="], + + "caniuse-lite": ["caniuse-lite@1.0.30001727", "", {}, "sha512-pB68nIHmbN6L/4C6MH1DokyR3bYqFwjaSs/sWDHGj4CTcFtQUQMuJftVwWkXq7mNWOybD3KhUv3oWHoGxgP14Q=="], + + "ccount": ["ccount@2.0.1", "", {}, "sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg=="], + + "chalk": ["chalk@5.4.1", "", {}, "sha512-zgVZuo2WcZgfUEmsn6eO3kINexW8RAE4maiQ8QNs8CtpPCSyMiYsULR3HQYkm3w8FIA3SberyMJMSldGsW+U3w=="], + + "character-entities": ["character-entities@2.0.2", "", {}, "sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ=="], + + "character-entities-html4": ["character-entities-html4@2.1.0", "", {}, "sha512-1v7fgQRj6hnSwFpq1Eu0ynr/CDEw0rXo2B61qXrLNdHZmPKgb7fqS1a2JwF0rISo9q77jDI8VMEHoApn8qDoZA=="], + + "character-entities-legacy": ["character-entities-legacy@3.0.0", "", {}, "sha512-RpPp0asT/6ufRm//AJVwpViZbGM/MkjQFxJccQRHmISF/22NBtsHqAWmL+/pmkPWoIUJdWyeVleTl1wydHATVQ=="], + + "character-reference-invalid": ["character-reference-invalid@2.0.1", "", {}, "sha512-iBZ4F4wRbyORVsu0jPV7gXkOsGYjGHPmAyv+HiHG8gi5PtC9KI2j1+v8/tlibRvjoWX027ypmG/n0HtO5t7unw=="], + + "chevrotain": ["chevrotain@11.0.3", "", { "dependencies": { "@chevrotain/cst-dts-gen": "11.0.3", "@chevrotain/gast": "11.0.3", "@chevrotain/regexp-to-ast": "11.0.3", "@chevrotain/types": "11.0.3", "@chevrotain/utils": "11.0.3", "lodash-es": "4.17.21" } }, "sha512-ci2iJH6LeIkvP9eJW6gpueU8cnZhv85ELY8w8WiFtNjMHA5ad6pQLaJo9mEly/9qUyCpvqX8/POVUTf18/HFdw=="], + + "chevrotain-allstar": ["chevrotain-allstar@0.3.1", "", { "dependencies": { "lodash-es": "^4.17.21" }, "peerDependencies": { "chevrotain": "^11.0.0" } }, "sha512-b7g+y9A0v4mxCW1qUhf3BSVPg+/NvGErk/dOkrDaHA0nQIQGAtrOjlX//9OQtRlSCy+x9rfB5N8yC71lH1nvMw=="], + + "chroma-js": ["chroma-js@3.1.2", "", {}, "sha512-IJnETTalXbsLx1eKEgx19d5L6SRM7cH4vINw/99p/M11HCuXGRWL+6YmCm7FWFGIo6dtWuQoQi1dc5yQ7ESIHg=="], + + "cli-cursor": ["cli-cursor@4.0.0", "", { "dependencies": { "restore-cursor": "^4.0.0" } }, "sha512-VGtlMu3x/4DOtIUwEkRezxUZ2lBacNJCHash0N0WeZDBS+7Ux1dm3XWAgWYxLJFMMdOeXMHXorshEFhbMSGelg=="], + + "cli-spinners": ["cli-spinners@2.9.2", "", {}, "sha512-ywqV+5MmyL4E7ybXgKys4DugZbX0FC6LnwrhjuykIjnK9k8OQacQ7axGKnjDXWNhns0xot3bZI5h55H8yo9cJg=="], + + "clsx": ["clsx@2.1.1", "", {}, "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA=="], + + "collapse-white-space": ["collapse-white-space@2.1.0", "", {}, "sha512-loKTxY1zCOuG4j9f6EPnuyyYkf58RnhhWTvRoZEokgB+WbdXehfjFviyOVYkqzEWz1Q5kRiZdBYS5SwxbQYwzw=="], + + "color-convert": ["color-convert@2.0.1", "", { "dependencies": { "color-name": "~1.1.4" } }, "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ=="], + + "color-name": ["color-name@1.1.4", "", {}, "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="], + + "comma-separated-tokens": ["comma-separated-tokens@2.0.3", "", {}, "sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg=="], + + "commander": ["commander@8.3.0", "", {}, "sha512-OkTL9umf+He2DZkUq8f8J9of7yL6RJKI24dVITBmNfZBmri9zYZQrKkuXiKhyfPSu8tUhnVBB1iKXevvnlR4Ww=="], + + "compressible": ["compressible@2.0.18", "", { "dependencies": { "mime-db": ">= 1.43.0 < 2" } }, "sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg=="], + + "compression": ["compression@1.8.0", "", { "dependencies": { "bytes": "3.1.2", "compressible": "~2.0.18", "debug": "2.6.9", "negotiator": "~0.6.4", "on-headers": "~1.0.2", "safe-buffer": "5.2.1", "vary": "~1.1.2" } }, "sha512-k6WLKfunuqCYD3t6AsuPGvQWaKwuLLh2/xHNcX4qE+vIfDNXpSqnrhwA7O53R7WVQUnt8dVAIW+YHr7xTgOgGA=="], + + "confbox": ["confbox@0.1.8", "", {}, "sha512-RMtmw0iFkeR4YV+fUOSucriAQNb9g8zFR52MWCtl+cCZOFRNL6zeB395vPzFhEjjn4fMxXudmELnl/KF/WrK6w=="], + + "convert-source-map": ["convert-source-map@2.0.0", "", {}, "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg=="], + + "cookie": ["cookie@1.0.2", "", {}, "sha512-9Kr/j4O16ISv8zBBhJoi4bXOYNTkFLOqSL3UDB0njXxCXNezjeyVrJyGOWtgfs/q2km1gwBcfH8q1yEGoMYunA=="], + + "cose-base": ["cose-base@1.0.3", "", { "dependencies": { "layout-base": "^1.0.0" } }, "sha512-s9whTXInMSgAp/NVXVNuVxVKzGH2qck3aQlVHxDCdAEPgtMKwc4Wq6/QKhgdEdgbLSi9rBTAcPoRa6JpiG4ksg=="], + + "create-vocs": ["create-vocs@1.0.0", "", { "dependencies": { "@clack/prompts": "^0.7.0", "cac": "^6.7.14", "detect-package-manager": "^3.0.2", "fs-extra": "^11.3.0", "picocolors": "^1.1.1" }, "bin": { "create-vocs": "_lib/bin.js" } }, "sha512-Lv1Bd3WZEgwG4nrogkM54m8viW+TWPlGivLyEi7aNb3cuKPsEfMDZ/kTbo87fzOGtsZ2yh7scO54ZmVhhgBgTw=="], + + "cross-spawn": ["cross-spawn@7.0.6", "", { "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", "which": "^2.0.1" } }, "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA=="], + + "css-selector-parser": ["css-selector-parser@3.1.3", "", {}, "sha512-gJMigczVZqYAk0hPVzx/M4Hm1D9QOtqkdQk9005TNzDIUGzo5cnHEDiKUT7jGPximL/oYb+LIitcHFQ4aKupxg=="], + + "css-what": ["css-what@6.2.2", "", {}, "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA=="], + + "cssesc": ["cssesc@3.0.0", "", { "bin": { "cssesc": "bin/cssesc" } }, "sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg=="], + + "csstype": ["csstype@3.1.3", "", {}, "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw=="], + + "cytoscape": ["cytoscape@3.32.1", "", {}, "sha512-dbeqFTLYEwlFg7UGtcZhCCG/2WayX72zK3Sq323CEX29CY81tYfVhw1MIdduCtpstB0cTOhJswWlM/OEB3Xp+Q=="], + + "cytoscape-cose-bilkent": ["cytoscape-cose-bilkent@4.1.0", "", { "dependencies": { "cose-base": "^1.0.0" }, "peerDependencies": { "cytoscape": "^3.2.0" } }, "sha512-wgQlVIUJF13Quxiv5e1gstZ08rnZj2XaLHGoFMYXz7SkNfCDOOteKBE6SYRfA9WxxI/iBc3ajfDoc6hb/MRAHQ=="], + + "cytoscape-fcose": ["cytoscape-fcose@2.2.0", "", { "dependencies": { "cose-base": "^2.2.0" }, "peerDependencies": { "cytoscape": "^3.2.0" } }, "sha512-ki1/VuRIHFCzxWNrsshHYPs6L7TvLu3DL+TyIGEsRcvVERmxokbf5Gdk7mFxZnTdiGtnA4cfSmjZJMviqSuZrQ=="], + + "d3": ["d3@7.9.0", "", { "dependencies": { "d3-array": "3", "d3-axis": "3", "d3-brush": "3", "d3-chord": "3", "d3-color": "3", "d3-contour": "4", "d3-delaunay": "6", "d3-dispatch": "3", "d3-drag": "3", "d3-dsv": "3", "d3-ease": "3", "d3-fetch": "3", "d3-force": "3", "d3-format": "3", "d3-geo": "3", "d3-hierarchy": "3", "d3-interpolate": "3", "d3-path": "3", "d3-polygon": "3", "d3-quadtree": "3", "d3-random": "3", "d3-scale": "4", "d3-scale-chromatic": "3", "d3-selection": "3", "d3-shape": "3", "d3-time": "3", "d3-time-format": "4", "d3-timer": "3", "d3-transition": "3", "d3-zoom": "3" } }, "sha512-e1U46jVP+w7Iut8Jt8ri1YsPOvFpg46k+K8TpCb0P+zjCkjkPnV7WzfDJzMHy1LnA+wj5pLT1wjO901gLXeEhA=="], + + "d3-array": ["d3-array@3.2.4", "", { "dependencies": { "internmap": "1 - 2" } }, "sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg=="], + + "d3-axis": ["d3-axis@3.0.0", "", {}, "sha512-IH5tgjV4jE/GhHkRV0HiVYPDtvfjHQlQfJHs0usq7M30XcSBvOotpmH1IgkcXsO/5gEQZD43B//fc7SRT5S+xw=="], + + "d3-brush": ["d3-brush@3.0.0", "", { "dependencies": { "d3-dispatch": "1 - 3", "d3-drag": "2 - 3", "d3-interpolate": "1 - 3", "d3-selection": "3", "d3-transition": "3" } }, "sha512-ALnjWlVYkXsVIGlOsuWH1+3udkYFI48Ljihfnh8FZPF2QS9o+PzGLBslO0PjzVoHLZ2KCVgAM8NVkXPJB2aNnQ=="], + + "d3-chord": ["d3-chord@3.0.1", "", { "dependencies": { "d3-path": "1 - 3" } }, "sha512-VE5S6TNa+j8msksl7HwjxMHDM2yNK3XCkusIlpX5kwauBfXuyLAtNg9jCp/iHH61tgI4sb6R/EIMWCqEIdjT/g=="], + + "d3-color": ["d3-color@3.1.0", "", {}, "sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA=="], + + "d3-contour": ["d3-contour@4.0.2", "", { "dependencies": { "d3-array": "^3.2.0" } }, "sha512-4EzFTRIikzs47RGmdxbeUvLWtGedDUNkTcmzoeyg4sP/dvCexO47AaQL7VKy/gul85TOxw+IBgA8US2xwbToNA=="], + + "d3-delaunay": ["d3-delaunay@6.0.4", "", { "dependencies": { "delaunator": "5" } }, "sha512-mdjtIZ1XLAM8bm/hx3WwjfHt6Sggek7qH043O8KEjDXN40xi3vx/6pYSVTwLjEgiXQTbvaouWKynLBiUZ6SK6A=="], + + "d3-dispatch": ["d3-dispatch@3.0.1", "", {}, "sha512-rzUyPU/S7rwUflMyLc1ETDeBj0NRuHKKAcvukozwhshr6g6c5d8zh4c2gQjY2bZ0dXeGLWc1PF174P2tVvKhfg=="], + + "d3-drag": ["d3-drag@3.0.0", "", { "dependencies": { "d3-dispatch": "1 - 3", "d3-selection": "3" } }, "sha512-pWbUJLdETVA8lQNJecMxoXfH6x+mO2UQo8rSmZ+QqxcbyA3hfeprFgIT//HW2nlHChWeIIMwS2Fq+gEARkhTkg=="], + + "d3-dsv": ["d3-dsv@3.0.1", "", { "dependencies": { "commander": "7", "iconv-lite": "0.6", "rw": "1" }, "bin": { "csv2json": "bin/dsv2json.js", "csv2tsv": "bin/dsv2dsv.js", "dsv2dsv": "bin/dsv2dsv.js", "dsv2json": "bin/dsv2json.js", "json2csv": "bin/json2dsv.js", "json2dsv": "bin/json2dsv.js", "json2tsv": "bin/json2dsv.js", "tsv2csv": "bin/dsv2dsv.js", "tsv2json": "bin/dsv2json.js" } }, "sha512-UG6OvdI5afDIFP9w4G0mNq50dSOsXHJaRE8arAS5o9ApWnIElp8GZw1Dun8vP8OyHOZ/QJUKUJwxiiCCnUwm+Q=="], + + "d3-ease": ["d3-ease@3.0.1", "", {}, "sha512-wR/XK3D3XcLIZwpbvQwQ5fK+8Ykds1ip7A2Txe0yxncXSdq1L9skcG7blcedkOX+ZcgxGAmLX1FrRGbADwzi0w=="], + + "d3-fetch": ["d3-fetch@3.0.1", "", { "dependencies": { "d3-dsv": "1 - 3" } }, "sha512-kpkQIM20n3oLVBKGg6oHrUchHM3xODkTzjMoj7aWQFq5QEM+R6E4WkzT5+tojDY7yjez8KgCBRoj4aEr99Fdqw=="], + + "d3-force": ["d3-force@3.0.0", "", { "dependencies": { "d3-dispatch": "1 - 3", "d3-quadtree": "1 - 3", "d3-timer": "1 - 3" } }, "sha512-zxV/SsA+U4yte8051P4ECydjD/S+qeYtnaIyAs9tgHCqfguma/aAQDjo85A9Z6EKhBirHRJHXIgJUlffT4wdLg=="], + + "d3-format": ["d3-format@3.1.0", "", {}, "sha512-YyUI6AEuY/Wpt8KWLgZHsIU86atmikuoOmCfommt0LYHiQSPjvX2AcFc38PX0CBpr2RCyZhjex+NS/LPOv6YqA=="], + + "d3-geo": ["d3-geo@3.1.1", "", { "dependencies": { "d3-array": "2.5.0 - 3" } }, "sha512-637ln3gXKXOwhalDzinUgY83KzNWZRKbYubaG+fGVuc/dxO64RRljtCTnf5ecMyE1RIdtqpkVcq0IbtU2S8j2Q=="], + + "d3-hierarchy": ["d3-hierarchy@3.1.2", "", {}, "sha512-FX/9frcub54beBdugHjDCdikxThEqjnR93Qt7PvQTOHxyiNCAlvMrHhclk3cD5VeAaq9fxmfRp+CnWw9rEMBuA=="], + + "d3-interpolate": ["d3-interpolate@3.0.1", "", { "dependencies": { "d3-color": "1 - 3" } }, "sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g=="], + + "d3-path": ["d3-path@3.1.0", "", {}, "sha512-p3KP5HCf/bvjBSSKuXid6Zqijx7wIfNW+J/maPs+iwR35at5JCbLUT0LzF1cnjbCHWhqzQTIN2Jpe8pRebIEFQ=="], + + "d3-polygon": ["d3-polygon@3.0.1", "", {}, "sha512-3vbA7vXYwfe1SYhED++fPUQlWSYTTGmFmQiany/gdbiWgU/iEyQzyymwL9SkJjFFuCS4902BSzewVGsHHmHtXg=="], + + "d3-quadtree": ["d3-quadtree@3.0.1", "", {}, "sha512-04xDrxQTDTCFwP5H6hRhsRcb9xxv2RzkcsygFzmkSIOJy3PeRJP7sNk3VRIbKXcog561P9oU0/rVH6vDROAgUw=="], + + "d3-random": ["d3-random@3.0.1", "", {}, "sha512-FXMe9GfxTxqd5D6jFsQ+DJ8BJS4E/fT5mqqdjovykEB2oFbTMDVdg1MGFxfQW+FBOGoB++k8swBrgwSHT1cUXQ=="], + + "d3-sankey": ["d3-sankey@0.12.3", "", { "dependencies": { "d3-array": "1 - 2", "d3-shape": "^1.2.0" } }, "sha512-nQhsBRmM19Ax5xEIPLMY9ZmJ/cDvd1BG3UVvt5h3WRxKg5zGRbvnteTyWAbzeSvlh3tW7ZEmq4VwR5mB3tutmQ=="], + + "d3-scale": ["d3-scale@4.0.2", "", { "dependencies": { "d3-array": "2.10.0 - 3", "d3-format": "1 - 3", "d3-interpolate": "1.2.0 - 3", "d3-time": "2.1.1 - 3", "d3-time-format": "2 - 4" } }, "sha512-GZW464g1SH7ag3Y7hXjf8RoUuAFIqklOAq3MRl4OaWabTFJY9PN/E1YklhXLh+OQ3fM9yS2nOkCoS+WLZ6kvxQ=="], + + "d3-scale-chromatic": ["d3-scale-chromatic@3.1.0", "", { "dependencies": { "d3-color": "1 - 3", "d3-interpolate": "1 - 3" } }, "sha512-A3s5PWiZ9YCXFye1o246KoscMWqf8BsD9eRiJ3He7C9OBaxKhAd5TFCdEx/7VbKtxxTsu//1mMJFrEt572cEyQ=="], + + "d3-selection": ["d3-selection@3.0.0", "", {}, "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ=="], + + "d3-shape": ["d3-shape@3.2.0", "", { "dependencies": { "d3-path": "^3.1.0" } }, "sha512-SaLBuwGm3MOViRq2ABk3eLoxwZELpH6zhl3FbAoJ7Vm1gofKx6El1Ib5z23NUEhF9AsGl7y+dzLe5Cw2AArGTA=="], + + "d3-time": ["d3-time@3.1.0", "", { "dependencies": { "d3-array": "2 - 3" } }, "sha512-VqKjzBLejbSMT4IgbmVgDjpkYrNWUYJnbCGo874u7MMKIWsILRX+OpX/gTk8MqjpT1A/c6HY2dCA77ZN0lkQ2Q=="], + + "d3-time-format": ["d3-time-format@4.1.0", "", { "dependencies": { "d3-time": "1 - 3" } }, "sha512-dJxPBlzC7NugB2PDLwo9Q8JiTR3M3e4/XANkreKSUxF8vvXKqm1Yfq4Q5dl8budlunRVlUUaDUgFt7eA8D6NLg=="], + + "d3-timer": ["d3-timer@3.0.1", "", {}, "sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA=="], + + "d3-transition": ["d3-transition@3.0.1", "", { "dependencies": { "d3-color": "1 - 3", "d3-dispatch": "1 - 3", "d3-ease": "1 - 3", "d3-interpolate": "1 - 3", "d3-timer": "1 - 3" }, "peerDependencies": { "d3-selection": "2 - 3" } }, "sha512-ApKvfjsSR6tg06xrL434C0WydLr7JewBB3V+/39RMHsaXTOG0zmt/OAXeng5M5LBm0ojmxJrpomQVZ1aPvBL4w=="], + + "d3-zoom": ["d3-zoom@3.0.0", "", { "dependencies": { "d3-dispatch": "1 - 3", "d3-drag": "2 - 3", "d3-interpolate": "1 - 3", "d3-selection": "2 - 3", "d3-transition": "2 - 3" } }, "sha512-b8AmV3kfQaqWAuacbPuNbL6vahnOJflOhexLzMMNLga62+/nh0JzvJ0aO/5a5MVgUFGS7Hu1P9P03o3fJkDCyw=="], + + "dagre-d3-es": ["dagre-d3-es@7.0.11", "", { "dependencies": { "d3": "^7.9.0", "lodash-es": "^4.17.21" } }, "sha512-tvlJLyQf834SylNKax8Wkzco/1ias1OPw8DcUMDE7oUIoSEW25riQVuiu/0OWEFqT0cxHT3Pa9/D82Jr47IONw=="], + + "dayjs": ["dayjs@1.11.13", "", {}, "sha512-oaMBel6gjolK862uaPQOVTA7q3TZhuSvuMQAAglQDOWYO9A91IrAOUJEyKVlqJlHE0vq5p5UXxzdPfMH/x6xNg=="], + + "debug": ["debug@2.6.9", "", { "dependencies": { "ms": "2.0.0" } }, "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA=="], + + "decode-named-character-reference": ["decode-named-character-reference@1.2.0", "", { "dependencies": { "character-entities": "^2.0.0" } }, "sha512-c6fcElNV6ShtZXmsgNgFFV5tVX2PaV4g+MOAkb8eXHvn6sryJBrZa9r0zV6+dtTyoCKxtDy5tyQ5ZwQuidtd+Q=="], + + "dedent": ["dedent@1.6.0", "", { "peerDependencies": { "babel-plugin-macros": "^3.1.0" }, "optionalPeers": ["babel-plugin-macros"] }, "sha512-F1Z+5UCFpmQUzJa11agbyPVMbpgT/qA3/SKyJ1jyBgm7dUcUEa8v9JwDkerSQXfakBwFljIxhOJqGkjUwZ9FSA=="], + + "deep-object-diff": ["deep-object-diff@1.1.9", "", {}, "sha512-Rn+RuwkmkDwCi2/oXOFS9Gsr5lJZu/yTGpK7wAaAIE75CC+LCGEZHpY6VQJa/RoJcrmaA/docWJZvYohlNkWPA=="], + + "deepmerge": ["deepmerge@4.3.1", "", {}, "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A=="], + + "delaunator": ["delaunator@5.0.1", "", { "dependencies": { "robust-predicates": "^3.0.2" } }, "sha512-8nvh+XBe96aCESrGOqMp/84b13H9cdKbG5P2ejQCh4d4sK9RL4371qou9drQjMhvnPmhWl5hnmqbEE0fXr9Xnw=="], + + "depd": ["depd@2.0.0", "", {}, "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw=="], + + "dequal": ["dequal@2.0.3", "", {}, "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA=="], + + "destroy": ["destroy@1.2.0", "", {}, "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg=="], + + "detect-libc": ["detect-libc@2.0.4", "", {}, "sha512-3UDv+G9CsCKO1WKMGw9fwq/SWJYbI0c5Y7LU1AXYoDdbhE2AHQ6N6Nb34sG8Fj7T5APy8qXDCKuuIHd1BR0tVA=="], + + "detect-node-es": ["detect-node-es@1.1.0", "", {}, "sha512-ypdmJU/TbBby2Dxibuv7ZLW3Bs1QEmM7nHjEANfohJLvE0XVujisn1qPJcZxg+qDucsr+bP6fLD1rPS3AhJ7EQ=="], + + "detect-package-manager": ["detect-package-manager@3.0.2", "", { "dependencies": { "execa": "^5.1.1" } }, "sha512-8JFjJHutStYrfWwzfretQoyNGoZVW1Fsrp4JO9spa7h/fBfwgTMEIy4/LBzRDGsxwVPHU0q+T9YvwLDJoOApLQ=="], + + "devlop": ["devlop@1.1.0", "", { "dependencies": { "dequal": "^2.0.0" } }, "sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA=="], + + "direction": ["direction@2.0.1", "", { "bin": { "direction": "cli.js" } }, "sha512-9S6m9Sukh1cZNknO1CWAr2QAWsbKLafQiyM5gZ7VgXHeuaoUwffKN4q6NC4A/Mf9iiPlOXQEKW/Mv/mh9/3YFA=="], + + "dompurify": ["dompurify@3.2.6", "", { "optionalDependencies": { "@types/trusted-types": "^2.0.7" } }, "sha512-/2GogDQlohXPZe6D6NOgQvXLPSYBqIWMnZ8zzOhn09REE4eyAzb+Hed3jhoM9OkuaJ8P6ZGTTVWQKAi8ieIzfQ=="], + + "eastasianwidth": ["eastasianwidth@0.2.0", "", {}, "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA=="], + + "ee-first": ["ee-first@1.1.1", "", {}, "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow=="], + + "electron-to-chromium": ["electron-to-chromium@1.5.186", "", {}, "sha512-lur7L4BFklgepaJxj4DqPk7vKbTEl0pajNlg2QjE5shefmlmBLm2HvQ7PMf1R/GvlevT/581cop33/quQcfX3A=="], + + "emoji-regex": ["emoji-regex@10.4.0", "", {}, "sha512-EC+0oUMY1Rqm4O6LLrgjtYDvcVYTy7chDnM4Q7030tP4Kwj3u/pR6gP9ygnp2CJMK5Gq+9Q2oqmrFJAz01DXjw=="], + + "emoji-regex-xs": ["emoji-regex-xs@1.0.0", "", {}, "sha512-LRlerrMYoIDrT6jgpeZ2YYl/L8EulRTt5hQcYjy5AInh7HWXKimpqx68aknBFpGL2+/IcogTcaydJEgaTmOpDg=="], + + "encodeurl": ["encodeurl@2.0.0", "", {}, "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg=="], + + "enhanced-resolve": ["enhanced-resolve@5.18.2", "", { "dependencies": { "graceful-fs": "^4.2.4", "tapable": "^2.2.0" } }, "sha512-6Jw4sE1maoRJo3q8MsSIn2onJFbLTOjY9hlx4DZXmOKvLRd1Ok2kXmAGXaafL2+ijsJZ1ClYbl/pmqr9+k4iUQ=="], + + "entities": ["entities@6.0.1", "", {}, "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g=="], + + "es-module-lexer": ["es-module-lexer@1.7.0", "", {}, "sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA=="], + + "esast-util-from-estree": ["esast-util-from-estree@2.0.0", "", { "dependencies": { "@types/estree-jsx": "^1.0.0", "devlop": "^1.0.0", "estree-util-visit": "^2.0.0", "unist-util-position-from-estree": "^2.0.0" } }, "sha512-4CyanoAudUSBAn5K13H4JhsMH6L9ZP7XbLVe/dKybkxMO7eDyLsT8UHl9TRNrU2Gr9nz+FovfSIjuXWJ81uVwQ=="], + + "esast-util-from-js": ["esast-util-from-js@2.0.1", "", { "dependencies": { "@types/estree-jsx": "^1.0.0", "acorn": "^8.0.0", "esast-util-from-estree": "^2.0.0", "vfile-message": "^4.0.0" } }, "sha512-8Ja+rNJ0Lt56Pcf3TAmpBZjmx8ZcK5Ts4cAzIOjsjevg9oSXJnl6SUQ2EevU8tv3h6ZLWmoKL5H4fgWvdvfETw=="], + + "esbuild": ["esbuild@0.25.6", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.25.6", "@esbuild/android-arm": "0.25.6", "@esbuild/android-arm64": "0.25.6", "@esbuild/android-x64": "0.25.6", "@esbuild/darwin-arm64": "0.25.6", "@esbuild/darwin-x64": "0.25.6", "@esbuild/freebsd-arm64": "0.25.6", "@esbuild/freebsd-x64": "0.25.6", "@esbuild/linux-arm": "0.25.6", "@esbuild/linux-arm64": "0.25.6", "@esbuild/linux-ia32": "0.25.6", "@esbuild/linux-loong64": "0.25.6", "@esbuild/linux-mips64el": "0.25.6", "@esbuild/linux-ppc64": "0.25.6", "@esbuild/linux-riscv64": "0.25.6", "@esbuild/linux-s390x": "0.25.6", "@esbuild/linux-x64": "0.25.6", "@esbuild/netbsd-arm64": "0.25.6", "@esbuild/netbsd-x64": "0.25.6", "@esbuild/openbsd-arm64": "0.25.6", "@esbuild/openbsd-x64": "0.25.6", "@esbuild/openharmony-arm64": "0.25.6", "@esbuild/sunos-x64": "0.25.6", "@esbuild/win32-arm64": "0.25.6", "@esbuild/win32-ia32": "0.25.6", "@esbuild/win32-x64": "0.25.6" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-GVuzuUwtdsghE3ocJ9Bs8PNoF13HNQ5TXbEi2AhvVb8xU1Iwt9Fos9FEamfoee+u/TOsn7GUWc04lz46n2bbTg=="], + + "escalade": ["escalade@3.2.0", "", {}, "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA=="], + + "escape-html": ["escape-html@1.0.3", "", {}, "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow=="], + + "escape-string-regexp": ["escape-string-regexp@5.0.0", "", {}, "sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw=="], + + "estree-util-attach-comments": ["estree-util-attach-comments@3.0.0", "", { "dependencies": { "@types/estree": "^1.0.0" } }, "sha512-cKUwm/HUcTDsYh/9FgnuFqpfquUbwIqwKM26BVCGDPVgvaCl/nDCCjUfiLlx6lsEZ3Z4RFxNbOQ60pkaEwFxGw=="], + + "estree-util-build-jsx": ["estree-util-build-jsx@3.0.1", "", { "dependencies": { "@types/estree-jsx": "^1.0.0", "devlop": "^1.0.0", "estree-util-is-identifier-name": "^3.0.0", "estree-walker": "^3.0.0" } }, "sha512-8U5eiL6BTrPxp/CHbs2yMgP8ftMhR5ww1eIKoWRMlqvltHF8fZn5LRDvTKuxD3DUn+shRbLGqXemcP51oFCsGQ=="], + + "estree-util-is-identifier-name": ["estree-util-is-identifier-name@3.0.0", "", {}, "sha512-hFtqIDZTIUZ9BXLb8y4pYGyk6+wekIivNVTcmvk8NoOh+VeRn5y6cEHzbURrWbfp1fIqdVipilzj+lfaadNZmg=="], + + "estree-util-scope": ["estree-util-scope@1.0.0", "", { "dependencies": { "@types/estree": "^1.0.0", "devlop": "^1.0.0" } }, "sha512-2CAASclonf+JFWBNJPndcOpA8EMJwa0Q8LUFJEKqXLW6+qBvbFZuF5gItbQOs/umBUkjviCSDCbBwU2cXbmrhQ=="], + + "estree-util-to-js": ["estree-util-to-js@2.0.0", "", { "dependencies": { "@types/estree-jsx": "^1.0.0", "astring": "^1.8.0", "source-map": "^0.7.0" } }, "sha512-WDF+xj5rRWmD5tj6bIqRi6CkLIXbbNQUcxQHzGysQzvHmdYG2G7p/Tf0J0gpxGgkeMZNTIjT/AoSvC9Xehcgdg=="], + + "estree-util-value-to-estree": ["estree-util-value-to-estree@3.4.0", "", { "dependencies": { "@types/estree": "^1.0.0" } }, "sha512-Zlp+gxis+gCfK12d3Srl2PdX2ybsEA8ZYy6vQGVQTNNYLEGRQQ56XB64bjemN8kxIKXP1nC9ip4Z+ILy9LGzvQ=="], + + "estree-util-visit": ["estree-util-visit@2.0.0", "", { "dependencies": { "@types/estree-jsx": "^1.0.0", "@types/unist": "^3.0.0" } }, "sha512-m5KgiH85xAhhW8Wta0vShLcUvOsh3LLPI2YVwcbio1l7E09NTLL1EyMZFM1OyWowoH0skScNbhOPl4kcBgzTww=="], + + "estree-walker": ["estree-walker@3.0.3", "", { "dependencies": { "@types/estree": "^1.0.0" } }, "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g=="], + + "etag": ["etag@1.8.1", "", {}, "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg=="], + + "eval": ["eval@0.1.8", "", { "dependencies": { "@types/node": "*", "require-like": ">= 0.1.1" } }, "sha512-EzV94NYKoO09GLXGjXj9JIlXijVck4ONSr5wiCWDvhsvj5jxSrzTmRU/9C1DyB6uToszLs8aifA6NQ7lEQdvFw=="], + + "execa": ["execa@5.1.1", "", { "dependencies": { "cross-spawn": "^7.0.3", "get-stream": "^6.0.0", "human-signals": "^2.1.0", "is-stream": "^2.0.0", "merge-stream": "^2.0.0", "npm-run-path": "^4.0.1", "onetime": "^5.1.2", "signal-exit": "^3.0.3", "strip-final-newline": "^2.0.0" } }, "sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg=="], + + "exsolve": ["exsolve@1.0.7", "", {}, "sha512-VO5fQUzZtI6C+vx4w/4BWJpg3s/5l+6pRQEHzFRM8WFi4XffSP1Z+4qi7GbjWbvRQEbdIco5mIMq+zX4rPuLrw=="], + + "extend": ["extend@3.0.2", "", {}, "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g=="], + + "fast-glob": ["fast-glob@3.3.3", "", { "dependencies": { "@nodelib/fs.stat": "^2.0.2", "@nodelib/fs.walk": "^1.2.3", "glob-parent": "^5.1.2", "merge2": "^1.3.0", "micromatch": "^4.0.8" } }, "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg=="], + + "fastq": ["fastq@1.19.1", "", { "dependencies": { "reusify": "^1.0.4" } }, "sha512-GwLTyxkCXjXbxqIhTsMI2Nui8huMPtnxg7krajPJAjnEG/iiOS7i+zCtWGZR9G0NBKbXKh6X9m9UIsYX/N6vvQ=="], + + "fault": ["fault@2.0.1", "", { "dependencies": { "format": "^0.2.0" } }, "sha512-WtySTkS4OKev5JtpHXnib4Gxiurzh5NCGvWrFaZ34m6JehfTUhKZvn9njTfw48t6JumVQOmrKqpmGcdwxnhqBQ=="], + + "fdir": ["fdir@6.4.6", "", { "peerDependencies": { "picomatch": "^3 || ^4" }, "optionalPeers": ["picomatch"] }, "sha512-hiFoqpyZcfNm1yc4u8oWCf9A2c4D3QjCrks3zmoVKVxpQRzmPNar1hUJcBG2RQHvEVGDN+Jm81ZheVLAQMK6+w=="], + + "fill-range": ["fill-range@7.1.1", "", { "dependencies": { "to-regex-range": "^5.0.1" } }, "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg=="], + + "find-up": ["find-up@5.0.0", "", { "dependencies": { "locate-path": "^6.0.0", "path-exists": "^4.0.0" } }, "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng=="], + + "foreground-child": ["foreground-child@3.3.1", "", { "dependencies": { "cross-spawn": "^7.0.6", "signal-exit": "^4.0.1" } }, "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw=="], + + "format": ["format@0.2.2", "", {}, "sha512-wzsgA6WOq+09wrU1tsJ09udeR/YZRaeArL9e1wPbFg3GG2yDnC2ldKpxs4xunpFF9DgqCqOIra3bc1HWrJ37Ww=="], + + "fraction.js": ["fraction.js@4.3.7", "", {}, "sha512-ZsDfxO51wGAXREY55a7la9LScWpwv9RxIrYABrlvOFBlH/ShPnrtsXeuUIfXKKOVicNxQ+o8JTbJvjS4M89yew=="], + + "fresh": ["fresh@0.5.2", "", {}, "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q=="], + + "fs-extra": ["fs-extra@11.3.0", "", { "dependencies": { "graceful-fs": "^4.2.0", "jsonfile": "^6.0.1", "universalify": "^2.0.0" } }, "sha512-Z4XaCL6dUDHfP/jT25jJKMmtxvuwbkrD1vNSMFlo9lNLY2c5FHYSQgHPRZUjAB26TpDEoW9HCOgplrdbaPV/ew=="], + + "fsevents": ["fsevents@2.3.2", "", { "os": "darwin" }, "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA=="], + + "gensync": ["gensync@1.0.0-beta.2", "", {}, "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg=="], + + "get-nonce": ["get-nonce@1.0.1", "", {}, "sha512-FJhYRoDaiatfEkUK8HKlicmu/3SGFD51q3itKDGoSTysQJBnfOcxU5GxnhE1E6soB76MbT0MBtnKJuXyAx+96Q=="], + + "get-stream": ["get-stream@6.0.1", "", {}, "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg=="], + + "github-slugger": ["github-slugger@2.0.0", "", {}, "sha512-IaOQ9puYtjrkq7Y0Ygl9KDZnrf/aiUJYUpVf89y8kyaxbRG7Y1SrX/jaumrv81vc61+kiMempujsM3Yw7w5qcw=="], + + "glob": ["glob@11.0.3", "", { "dependencies": { "foreground-child": "^3.3.1", "jackspeak": "^4.1.1", "minimatch": "^10.0.3", "minipass": "^7.1.2", "package-json-from-dist": "^1.0.0", "path-scurry": "^2.0.0" }, "bin": { "glob": "dist/esm/bin.mjs" } }, "sha512-2Nim7dha1KVkaiF4q6Dj+ngPPMdfvLJEOpZk/jKiUAkqKebpGAWQXAq9z1xu9HKu5lWfqw/FASuccEjyznjPaA=="], + + "glob-parent": ["glob-parent@5.1.2", "", { "dependencies": { "is-glob": "^4.0.1" } }, "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow=="], + + "globals": ["globals@15.15.0", "", {}, "sha512-7ACyT3wmyp3I61S4fG682L0VA2RGD9otkqGJIwNUMF1SWUombIIk+af1unuDYgMm082aHYwD+mzJvv9Iu8dsgg=="], + + "globby": ["globby@14.1.0", "", { "dependencies": { "@sindresorhus/merge-streams": "^2.1.0", "fast-glob": "^3.3.3", "ignore": "^7.0.3", "path-type": "^6.0.0", "slash": "^5.1.0", "unicorn-magic": "^0.3.0" } }, "sha512-0Ia46fDOaT7k4og1PDW4YbodWWr3scS2vAr2lTbsplOt2WkKp0vQbkI9wKis/T5LV/dqPjO3bpS/z6GTJB82LA=="], + + "graceful-fs": ["graceful-fs@4.2.11", "", {}, "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ=="], + + "hachure-fill": ["hachure-fill@0.5.2", "", {}, "sha512-3GKBOn+m2LX9iq+JC1064cSFprJY4jL1jCXTcpnfER5HYE2l/4EfWSGzkPa/ZDBmYI0ZOEj5VHV/eKnPGkHuOg=="], + + "hast-util-classnames": ["hast-util-classnames@3.0.0", "", { "dependencies": { "@types/hast": "^3.0.0", "space-separated-tokens": "^2.0.0" } }, "sha512-tI3JjoGDEBVorMAWK4jNRsfLMYmih1BUOG3VV36pH36njs1IEl7xkNrVTD2mD2yYHmQCa5R/fj61a8IAF4bRaQ=="], + + "hast-util-from-dom": ["hast-util-from-dom@5.0.1", "", { "dependencies": { "@types/hast": "^3.0.0", "hastscript": "^9.0.0", "web-namespaces": "^2.0.0" } }, "sha512-N+LqofjR2zuzTjCPzyDUdSshy4Ma6li7p/c3pA78uTwzFgENbgbUrm2ugwsOdcjI1muO+o6Dgzp9p8WHtn/39Q=="], + + "hast-util-from-html": ["hast-util-from-html@2.0.3", "", { "dependencies": { "@types/hast": "^3.0.0", "devlop": "^1.1.0", "hast-util-from-parse5": "^8.0.0", "parse5": "^7.0.0", "vfile": "^6.0.0", "vfile-message": "^4.0.0" } }, "sha512-CUSRHXyKjzHov8yKsQjGOElXy/3EKpyX56ELnkHH34vDVw1N1XSQ1ZcAvTyAPtGqLTuKP/uxM+aLkSPqF/EtMw=="], + + "hast-util-from-html-isomorphic": ["hast-util-from-html-isomorphic@2.0.0", "", { "dependencies": { "@types/hast": "^3.0.0", "hast-util-from-dom": "^5.0.0", "hast-util-from-html": "^2.0.0", "unist-util-remove-position": "^5.0.0" } }, "sha512-zJfpXq44yff2hmE0XmwEOzdWin5xwH+QIhMLOScpX91e/NSGPsAzNCvLQDIEPyO2TXi+lBmU6hjLIhV8MwP2kw=="], + + "hast-util-from-parse5": ["hast-util-from-parse5@8.0.3", "", { "dependencies": { "@types/hast": "^3.0.0", "@types/unist": "^3.0.0", "devlop": "^1.0.0", "hastscript": "^9.0.0", "property-information": "^7.0.0", "vfile": "^6.0.0", "vfile-location": "^5.0.0", "web-namespaces": "^2.0.0" } }, "sha512-3kxEVkEKt0zvcZ3hCRYI8rqrgwtlIOFMWkbclACvjlDw8Li9S2hk/d51OI0nr/gIpdMHNepwgOKqZ/sy0Clpyg=="], + + "hast-util-has-property": ["hast-util-has-property@3.0.0", "", { "dependencies": { "@types/hast": "^3.0.0" } }, "sha512-MNilsvEKLFpV604hwfhVStK0usFY/QmM5zX16bo7EjnAEGofr5YyI37kzopBlZJkHD4t887i+q/C8/tr5Q94cA=="], + + "hast-util-heading-rank": ["hast-util-heading-rank@3.0.0", "", { "dependencies": { "@types/hast": "^3.0.0" } }, "sha512-EJKb8oMUXVHcWZTDepnr+WNbfnXKFNf9duMesmr4S8SXTJBJ9M4Yok08pu9vxdJwdlGRhVumk9mEhkEvKGifwA=="], + + "hast-util-is-element": ["hast-util-is-element@3.0.0", "", { "dependencies": { "@types/hast": "^3.0.0" } }, "sha512-Val9mnv2IWpLbNPqc/pUem+a7Ipj2aHacCwgNfTiK0vJKl0LF+4Ba4+v1oPHFpf3bLYmreq0/l3Gud9S5OH42g=="], + + "hast-util-parse-selector": ["hast-util-parse-selector@4.0.0", "", { "dependencies": { "@types/hast": "^3.0.0" } }, "sha512-wkQCkSYoOGCRKERFWcxMVMOcYE2K1AaNLU8DXS9arxnLOUEWbOXKXiJUNzEpqZ3JOKpnha3jkFrumEjVliDe7A=="], + + "hast-util-select": ["hast-util-select@6.0.4", "", { "dependencies": { "@types/hast": "^3.0.0", "@types/unist": "^3.0.0", "bcp-47-match": "^2.0.0", "comma-separated-tokens": "^2.0.0", "css-selector-parser": "^3.0.0", "devlop": "^1.0.0", "direction": "^2.0.0", "hast-util-has-property": "^3.0.0", "hast-util-to-string": "^3.0.0", "hast-util-whitespace": "^3.0.0", "nth-check": "^2.0.0", "property-information": "^7.0.0", "space-separated-tokens": "^2.0.0", "unist-util-visit": "^5.0.0", "zwitch": "^2.0.0" } }, "sha512-RqGS1ZgI0MwxLaKLDxjprynNzINEkRHY2i8ln4DDjgv9ZhcYVIHN9rlpiYsqtFwrgpYU361SyWDQcGNIBVu3lw=="], + + "hast-util-to-estree": ["hast-util-to-estree@3.1.3", "", { "dependencies": { "@types/estree": "^1.0.0", "@types/estree-jsx": "^1.0.0", "@types/hast": "^3.0.0", "comma-separated-tokens": "^2.0.0", "devlop": "^1.0.0", "estree-util-attach-comments": "^3.0.0", "estree-util-is-identifier-name": "^3.0.0", "hast-util-whitespace": "^3.0.0", "mdast-util-mdx-expression": "^2.0.0", "mdast-util-mdx-jsx": "^3.0.0", "mdast-util-mdxjs-esm": "^2.0.0", "property-information": "^7.0.0", "space-separated-tokens": "^2.0.0", "style-to-js": "^1.0.0", "unist-util-position": "^5.0.0", "zwitch": "^2.0.0" } }, "sha512-48+B/rJWAp0jamNbAAf9M7Uf//UVqAoMmgXhBdxTDJLGKY+LRnZ99qcG+Qjl5HfMpYNzS5v4EAwVEF34LeAj7w=="], + + "hast-util-to-html": ["hast-util-to-html@9.0.5", "", { "dependencies": { "@types/hast": "^3.0.0", "@types/unist": "^3.0.0", "ccount": "^2.0.0", "comma-separated-tokens": "^2.0.0", "hast-util-whitespace": "^3.0.0", "html-void-elements": "^3.0.0", "mdast-util-to-hast": "^13.0.0", "property-information": "^7.0.0", "space-separated-tokens": "^2.0.0", "stringify-entities": "^4.0.0", "zwitch": "^2.0.4" } }, "sha512-OguPdidb+fbHQSU4Q4ZiLKnzWo8Wwsf5bZfbvu7//a9oTYoqD/fWpe96NuHkoS9h0ccGOTe0C4NGXdtS0iObOw=="], + + "hast-util-to-jsx-runtime": ["hast-util-to-jsx-runtime@2.3.6", "", { "dependencies": { "@types/estree": "^1.0.0", "@types/hast": "^3.0.0", "@types/unist": "^3.0.0", "comma-separated-tokens": "^2.0.0", "devlop": "^1.0.0", "estree-util-is-identifier-name": "^3.0.0", "hast-util-whitespace": "^3.0.0", "mdast-util-mdx-expression": "^2.0.0", "mdast-util-mdx-jsx": "^3.0.0", "mdast-util-mdxjs-esm": "^2.0.0", "property-information": "^7.0.0", "space-separated-tokens": "^2.0.0", "style-to-js": "^1.0.0", "unist-util-position": "^5.0.0", "vfile-message": "^4.0.0" } }, "sha512-zl6s8LwNyo1P9uw+XJGvZtdFF1GdAkOg8ujOw+4Pyb76874fLps4ueHXDhXWdk6YHQ6OgUtinliG7RsYvCbbBg=="], + + "hast-util-to-string": ["hast-util-to-string@3.0.1", "", { "dependencies": { "@types/hast": "^3.0.0" } }, "sha512-XelQVTDWvqcl3axRfI0xSeoVKzyIFPwsAGSLIsKdJKQMXDYJS4WYrBNF/8J7RdhIcFI2BOHgAifggsvsxp/3+A=="], + + "hast-util-to-text": ["hast-util-to-text@4.0.2", "", { "dependencies": { "@types/hast": "^3.0.0", "@types/unist": "^3.0.0", "hast-util-is-element": "^3.0.0", "unist-util-find-after": "^5.0.0" } }, "sha512-KK6y/BN8lbaq654j7JgBydev7wuNMcID54lkRav1P0CaE1e47P72AWWPiGKXTJU271ooYzcvTAn/Zt0REnvc7A=="], + + "hast-util-whitespace": ["hast-util-whitespace@3.0.0", "", { "dependencies": { "@types/hast": "^3.0.0" } }, "sha512-88JUN06ipLwsnv+dVn+OIYOvAuvBMy/Qoi6O7mQHxdPXpjy+Cd6xRkWwux7DKO+4sYILtLBRIKgsdpS2gQc7qw=="], + + "hastscript": ["hastscript@8.0.0", "", { "dependencies": { "@types/hast": "^3.0.0", "comma-separated-tokens": "^2.0.0", "hast-util-parse-selector": "^4.0.0", "property-information": "^6.0.0", "space-separated-tokens": "^2.0.0" } }, "sha512-dMOtzCEd3ABUeSIISmrETiKuyydk1w0pa+gE/uormcTpSYuaNJPbX1NU3JLyscSLjwAQM8bWMhhIlnCqnRvDTw=="], + + "hono": ["hono@4.8.5", "", {}, "sha512-Up2cQbtNz1s111qpnnECdTGqSIUIhZJMLikdKkshebQSEBcoUKq6XJayLGqSZWidiH0zfHRCJqFu062Mz5UuRA=="], + + "html-void-elements": ["html-void-elements@3.0.0", "", {}, "sha512-bEqo66MRXsUGxWHV5IP0PUiAWwoEjba4VCzg0LjFJBpchPaTfyfCKTG6bc5F8ucKec3q5y6qOdGyYTSBEvhCrg=="], + + "http-errors": ["http-errors@2.0.0", "", { "dependencies": { "depd": "2.0.0", "inherits": "2.0.4", "setprototypeof": "1.2.0", "statuses": "2.0.1", "toidentifier": "1.0.1" } }, "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ=="], + + "human-signals": ["human-signals@2.1.0", "", {}, "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw=="], + + "iconv-lite": ["iconv-lite@0.6.3", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw=="], + + "ieee754": ["ieee754@1.2.1", "", {}, "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA=="], + + "ignore": ["ignore@7.0.5", "", {}, "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg=="], + + "inherits": ["inherits@2.0.4", "", {}, "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="], + + "inline-style-parser": ["inline-style-parser@0.2.4", "", {}, "sha512-0aO8FkhNZlj/ZIbNi7Lxxr12obT7cL1moPfE4tg1LkX7LlLfC6DeX4l2ZEud1ukP9jNQyNnfzQVqwbwmAATY4Q=="], + + "internmap": ["internmap@1.0.1", "", {}, "sha512-lDB5YccMydFBtasVtxnZ3MRBHuaoE8GKsppq+EchKL2U4nK/DmEpPHNH8MZe5HkMtpSiTSOZwfN0tzYjO/lJEw=="], + + "is-alphabetical": ["is-alphabetical@2.0.1", "", {}, "sha512-FWyyY60MeTNyeSRpkM2Iry0G9hpr7/9kD40mD/cGQEuilcZYS4okz8SN2Q6rLCJ8gbCt6fN+rC+6tMGS99LaxQ=="], + + "is-alphanumerical": ["is-alphanumerical@2.0.1", "", { "dependencies": { "is-alphabetical": "^2.0.0", "is-decimal": "^2.0.0" } }, "sha512-hmbYhX/9MUMF5uh7tOXyK/n0ZvWpad5caBA17GsC6vyuCqaWliRG5K1qS9inmUhEMaOBIW7/whAnSwveW/LtZw=="], + + "is-decimal": ["is-decimal@2.0.1", "", {}, "sha512-AAB9hiomQs5DXWcRB1rqsxGUstbRroFOPPVAomNk/3XHR5JyEZChOyTWe2oayKnsSsr/kcGqF+z6yuH6HHpN0A=="], + + "is-extglob": ["is-extglob@2.1.1", "", {}, "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ=="], + + "is-fullwidth-code-point": ["is-fullwidth-code-point@3.0.0", "", {}, "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg=="], + + "is-glob": ["is-glob@4.0.3", "", { "dependencies": { "is-extglob": "^2.1.1" } }, "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg=="], + + "is-hexadecimal": ["is-hexadecimal@2.0.1", "", {}, "sha512-DgZQp241c8oO6cA1SbTEWiXeoxV42vlcJxgH+B3hi1AiqqKruZR3ZGF8In3fj4+/y/7rHvlOZLZtgJ/4ttYGZg=="], + + "is-interactive": ["is-interactive@2.0.0", "", {}, "sha512-qP1vozQRI+BMOPcjFzrjXuQvdak2pHNUMZoeG2eRbiSqyvbEf/wQtEOTOX1guk6E3t36RkaqiSt8A/6YElNxLQ=="], + + "is-number": ["is-number@7.0.0", "", {}, "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng=="], + + "is-plain-obj": ["is-plain-obj@4.1.0", "", {}, "sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg=="], + + "is-stream": ["is-stream@2.0.1", "", {}, "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg=="], + + "is-unicode-supported": ["is-unicode-supported@1.3.0", "", {}, "sha512-43r2mRvz+8JRIKnWJ+3j8JtjRKZ6GmjzfaE/qiBJnikNnYv/6bagRJ1kUhNk8R5EX/GkobD+r+sfxCPJsiKBLQ=="], + + "isexe": ["isexe@2.0.0", "", {}, "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="], + + "jackspeak": ["jackspeak@4.1.1", "", { "dependencies": { "@isaacs/cliui": "^8.0.2" } }, "sha512-zptv57P3GpL+O0I7VdMJNBZCu+BPHVQUk55Ft8/QCJjTVxrnJHuVuX/0Bl2A6/+2oyR/ZMEuFKwmzqqZ/U5nPQ=="], + + "javascript-stringify": ["javascript-stringify@2.1.0", "", {}, "sha512-JVAfqNPTvNq3sB/VHQJAFxN/sPgKnsKrCwyRt15zwNCdrMMJDdcEOdubuy+DuJYYdm0ox1J4uzEuYKkN+9yhVg=="], + + "jiti": ["jiti@2.4.2", "", { "bin": { "jiti": "lib/jiti-cli.mjs" } }, "sha512-rg9zJN+G4n2nfJl5MW3BMygZX56zKPNVEYYqq7adpmMh4Jn2QNEwhvQlFy6jPVdcod7txZtKHWnyZiA3a0zP7A=="], + + "js-tokens": ["js-tokens@4.0.0", "", {}, "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ=="], + + "jsesc": ["jsesc@3.1.0", "", { "bin": { "jsesc": "bin/jsesc" } }, "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA=="], + + "json5": ["json5@2.2.3", "", { "bin": { "json5": "lib/cli.js" } }, "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg=="], + + "jsonfile": ["jsonfile@6.1.0", "", { "dependencies": { "universalify": "^2.0.0" }, "optionalDependencies": { "graceful-fs": "^4.1.6" } }, "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ=="], + + "katex": ["katex@0.16.22", "", { "dependencies": { "commander": "^8.3.0" }, "bin": { "katex": "cli.js" } }, "sha512-XCHRdUw4lf3SKBaJe4EvgqIuWwkPSo9XoeO8GjQW94Bp7TWv9hNhzZjZ+OH9yf1UmLygb7DIT5GSFQiyt16zYg=="], + + "khroma": ["khroma@2.1.0", "", {}, "sha512-Ls993zuzfayK269Svk9hzpeGUKob/sIgZzyHYdjQoAdQetRKpOLj+k/QQQ/6Qi0Yz65mlROrfd+Ev+1+7dz9Kw=="], + + "kolorist": ["kolorist@1.8.0", "", {}, "sha512-Y+60/zizpJ3HRH8DCss+q95yr6145JXZo46OTpFvDZWLfRCE4qChOyk1b26nMaNpfHHgxagk9dXT5OP0Tfe+dQ=="], + + "langium": ["langium@3.3.1", "", { "dependencies": { "chevrotain": "~11.0.3", "chevrotain-allstar": "~0.3.0", "vscode-languageserver": "~9.0.1", "vscode-languageserver-textdocument": "~1.0.11", "vscode-uri": "~3.0.8" } }, "sha512-QJv/h939gDpvT+9SiLVlY7tZC3xB2qK57v0J04Sh9wpMb6MP1q8gB21L3WIo8T5P1MSMg3Ep14L7KkDCFG3y4w=="], + + "layout-base": ["layout-base@1.0.2", "", {}, "sha512-8h2oVEZNktL4BH2JCOI90iD1yXwL6iNW7KcCKT2QZgQJR2vbqDsldCTPRU9NifTCqHZci57XvQQ15YTu+sTYPg=="], + + "lightningcss": ["lightningcss@1.30.1", "", { "dependencies": { "detect-libc": "^2.0.3" }, "optionalDependencies": { "lightningcss-darwin-arm64": "1.30.1", "lightningcss-darwin-x64": "1.30.1", "lightningcss-freebsd-x64": "1.30.1", "lightningcss-linux-arm-gnueabihf": "1.30.1", "lightningcss-linux-arm64-gnu": "1.30.1", "lightningcss-linux-arm64-musl": "1.30.1", "lightningcss-linux-x64-gnu": "1.30.1", "lightningcss-linux-x64-musl": "1.30.1", "lightningcss-win32-arm64-msvc": "1.30.1", "lightningcss-win32-x64-msvc": "1.30.1" } }, "sha512-xi6IyHML+c9+Q3W0S4fCQJOym42pyurFiJUHEcEyHS0CeKzia4yZDEsLlqOFykxOdHpNy0NmvVO31vcSqAxJCg=="], + + "lightningcss-darwin-arm64": ["lightningcss-darwin-arm64@1.30.1", "", { "os": "darwin", "cpu": "arm64" }, "sha512-c8JK7hyE65X1MHMN+Viq9n11RRC7hgin3HhYKhrMyaXflk5GVplZ60IxyoVtzILeKr+xAJwg6zK6sjTBJ0FKYQ=="], + + "lightningcss-darwin-x64": ["lightningcss-darwin-x64@1.30.1", "", { "os": "darwin", "cpu": "x64" }, "sha512-k1EvjakfumAQoTfcXUcHQZhSpLlkAuEkdMBsI/ivWw9hL+7FtilQc0Cy3hrx0AAQrVtQAbMI7YjCgYgvn37PzA=="], + + "lightningcss-freebsd-x64": ["lightningcss-freebsd-x64@1.30.1", "", { "os": "freebsd", "cpu": "x64" }, "sha512-kmW6UGCGg2PcyUE59K5r0kWfKPAVy4SltVeut+umLCFoJ53RdCUWxcRDzO1eTaxf/7Q2H7LTquFHPL5R+Gjyig=="], + + "lightningcss-linux-arm-gnueabihf": ["lightningcss-linux-arm-gnueabihf@1.30.1", "", { "os": "linux", "cpu": "arm" }, "sha512-MjxUShl1v8pit+6D/zSPq9S9dQ2NPFSQwGvxBCYaBYLPlCWuPh9/t1MRS8iUaR8i+a6w7aps+B4N0S1TYP/R+Q=="], + + "lightningcss-linux-arm64-gnu": ["lightningcss-linux-arm64-gnu@1.30.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-gB72maP8rmrKsnKYy8XUuXi/4OctJiuQjcuqWNlJQ6jZiWqtPvqFziskH3hnajfvKB27ynbVCucKSm2rkQp4Bw=="], + + "lightningcss-linux-arm64-musl": ["lightningcss-linux-arm64-musl@1.30.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-jmUQVx4331m6LIX+0wUhBbmMX7TCfjF5FoOH6SD1CttzuYlGNVpA7QnrmLxrsub43ClTINfGSYyHe2HWeLl5CQ=="], + + "lightningcss-linux-x64-gnu": ["lightningcss-linux-x64-gnu@1.30.1", "", { "os": "linux", "cpu": "x64" }, "sha512-piWx3z4wN8J8z3+O5kO74+yr6ze/dKmPnI7vLqfSqI8bccaTGY5xiSGVIJBDd5K5BHlvVLpUB3S2YCfelyJ1bw=="], + + "lightningcss-linux-x64-musl": ["lightningcss-linux-x64-musl@1.30.1", "", { "os": "linux", "cpu": "x64" }, "sha512-rRomAK7eIkL+tHY0YPxbc5Dra2gXlI63HL+v1Pdi1a3sC+tJTcFrHX+E86sulgAXeI7rSzDYhPSeHHjqFhqfeQ=="], + + "lightningcss-win32-arm64-msvc": ["lightningcss-win32-arm64-msvc@1.30.1", "", { "os": "win32", "cpu": "arm64" }, "sha512-mSL4rqPi4iXq5YVqzSsJgMVFENoa4nGTT/GjO2c0Yl9OuQfPsIfncvLrEW6RbbB24WtZ3xP/2CCmI3tNkNV4oA=="], + + "lightningcss-win32-x64-msvc": ["lightningcss-win32-x64-msvc@1.30.1", "", { "os": "win32", "cpu": "x64" }, "sha512-PVqXh48wh4T53F/1CCu8PIPCxLzWyCnn/9T5W1Jpmdy5h9Cwd+0YQS6/LwhHXSafuc61/xg9Lv5OrCby6a++jg=="], + + "local-pkg": ["local-pkg@1.1.1", "", { "dependencies": { "mlly": "^1.7.4", "pkg-types": "^2.0.1", "quansync": "^0.2.8" } }, "sha512-WunYko2W1NcdfAFpuLUoucsgULmgDBRkdxHxWQ7mK0cQqwPiy8E1enjuRBrhLtZkB5iScJ1XIPdhVEFK8aOLSg=="], + + "locate-path": ["locate-path@6.0.0", "", { "dependencies": { "p-locate": "^5.0.0" } }, "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw=="], + + "lodash-es": ["lodash-es@4.17.21", "", {}, "sha512-mKnC+QJ9pWVzv+C4/U3rRsHapFfHvQFoFB92e52xeyGMcX6/OlIl78je1u8vePzYZSkkogMPJ2yjxxsb89cxyw=="], + + "log-symbols": ["log-symbols@5.1.0", "", { "dependencies": { "chalk": "^5.0.0", "is-unicode-supported": "^1.1.0" } }, "sha512-l0x2DvrW294C9uDCoQe1VSU4gf529FkSZ6leBl4TiqZH/e+0R7hSfHQBNut2mNygDgHwvYHfFLn6Oxb3VWj2rA=="], + + "longest-streak": ["longest-streak@3.1.0", "", {}, "sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g=="], + + "lru-cache": ["lru-cache@11.1.0", "", {}, "sha512-QIXZUBJUx+2zHUdQujWejBkcD9+cs94tLn0+YL8UrCh+D5sCXZ4c7LaEH48pNwRY3MLDgqUFyhlCyjJPf1WP0A=="], + + "mark.js": ["mark.js@8.11.1", "", {}, "sha512-1I+1qpDt4idfgLQG+BNWmrqku+7/2bi5nLf4YwF8y8zXvmfiTBY3PV3ZibfrjBueCByROpuBjLLFCajqkgYoLQ=="], + + "markdown-extensions": ["markdown-extensions@2.0.0", "", {}, "sha512-o5vL7aDWatOTX8LzaS1WMoaoxIiLRQJuIKKe2wAw6IeULDHaqbiqiggmx+pKvZDb1Sj+pE46Sn1T7lCqfFtg1Q=="], + + "markdown-table": ["markdown-table@3.0.4", "", {}, "sha512-wiYz4+JrLyb/DqW2hkFJxP7Vd7JuTDm77fvbM8VfEQdmSMqcImWeeRbHwZjBjIFki/VaMK2BhFi7oUUZeM5bqw=="], + + "marked": ["marked@16.0.0", "", { "bin": { "marked": "bin/marked.js" } }, "sha512-MUKMXDjsD/eptB7GPzxo4xcnLS6oo7/RHimUMHEDRhUooPwmN9BEpMl7AEOJv3bmso169wHI2wUF9VQgL7zfmA=="], + + "mdast-util-directive": ["mdast-util-directive@3.1.0", "", { "dependencies": { "@types/mdast": "^4.0.0", "@types/unist": "^3.0.0", "ccount": "^2.0.0", "devlop": "^1.0.0", "mdast-util-from-markdown": "^2.0.0", "mdast-util-to-markdown": "^2.0.0", "parse-entities": "^4.0.0", "stringify-entities": "^4.0.0", "unist-util-visit-parents": "^6.0.0" } }, "sha512-I3fNFt+DHmpWCYAT7quoM6lHf9wuqtI+oCOfvILnoicNIqjh5E3dEJWiXuYME2gNe8vl1iMQwyUHa7bgFmak6Q=="], + + "mdast-util-find-and-replace": ["mdast-util-find-and-replace@3.0.2", "", { "dependencies": { "@types/mdast": "^4.0.0", "escape-string-regexp": "^5.0.0", "unist-util-is": "^6.0.0", "unist-util-visit-parents": "^6.0.0" } }, "sha512-Tmd1Vg/m3Xz43afeNxDIhWRtFZgM2VLyaf4vSTYwudTyeuTneoL3qtWMA5jeLyz/O1vDJmmV4QuScFCA2tBPwg=="], + + "mdast-util-from-markdown": ["mdast-util-from-markdown@2.0.2", "", { "dependencies": { "@types/mdast": "^4.0.0", "@types/unist": "^3.0.0", "decode-named-character-reference": "^1.0.0", "devlop": "^1.0.0", "mdast-util-to-string": "^4.0.0", "micromark": "^4.0.0", "micromark-util-decode-numeric-character-reference": "^2.0.0", "micromark-util-decode-string": "^2.0.0", "micromark-util-normalize-identifier": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0", "unist-util-stringify-position": "^4.0.0" } }, "sha512-uZhTV/8NBuw0WHkPTrCqDOl0zVe1BIng5ZtHoDk49ME1qqcjYmmLmOf0gELgcRMxN4w2iuIeVso5/6QymSrgmA=="], + + "mdast-util-frontmatter": ["mdast-util-frontmatter@2.0.1", "", { "dependencies": { "@types/mdast": "^4.0.0", "devlop": "^1.0.0", "escape-string-regexp": "^5.0.0", "mdast-util-from-markdown": "^2.0.0", "mdast-util-to-markdown": "^2.0.0", "micromark-extension-frontmatter": "^2.0.0" } }, "sha512-LRqI9+wdgC25P0URIJY9vwocIzCcksduHQ9OF2joxQoyTNVduwLAFUzjoopuRJbJAReaKrNQKAZKL3uCMugWJA=="], + + "mdast-util-gfm": ["mdast-util-gfm@3.1.0", "", { "dependencies": { "mdast-util-from-markdown": "^2.0.0", "mdast-util-gfm-autolink-literal": "^2.0.0", "mdast-util-gfm-footnote": "^2.0.0", "mdast-util-gfm-strikethrough": "^2.0.0", "mdast-util-gfm-table": "^2.0.0", "mdast-util-gfm-task-list-item": "^2.0.0", "mdast-util-to-markdown": "^2.0.0" } }, "sha512-0ulfdQOM3ysHhCJ1p06l0b0VKlhU0wuQs3thxZQagjcjPrlFRqY215uZGHHJan9GEAXd9MbfPjFJz+qMkVR6zQ=="], + + "mdast-util-gfm-autolink-literal": ["mdast-util-gfm-autolink-literal@2.0.1", "", { "dependencies": { "@types/mdast": "^4.0.0", "ccount": "^2.0.0", "devlop": "^1.0.0", "mdast-util-find-and-replace": "^3.0.0", "micromark-util-character": "^2.0.0" } }, "sha512-5HVP2MKaP6L+G6YaxPNjuL0BPrq9orG3TsrZ9YXbA3vDw/ACI4MEsnoDpn6ZNm7GnZgtAcONJyPhOP8tNJQavQ=="], + + "mdast-util-gfm-footnote": ["mdast-util-gfm-footnote@2.1.0", "", { "dependencies": { "@types/mdast": "^4.0.0", "devlop": "^1.1.0", "mdast-util-from-markdown": "^2.0.0", "mdast-util-to-markdown": "^2.0.0", "micromark-util-normalize-identifier": "^2.0.0" } }, "sha512-sqpDWlsHn7Ac9GNZQMeUzPQSMzR6Wv0WKRNvQRg0KqHh02fpTz69Qc1QSseNX29bhz1ROIyNyxExfawVKTm1GQ=="], + + "mdast-util-gfm-strikethrough": ["mdast-util-gfm-strikethrough@2.0.0", "", { "dependencies": { "@types/mdast": "^4.0.0", "mdast-util-from-markdown": "^2.0.0", "mdast-util-to-markdown": "^2.0.0" } }, "sha512-mKKb915TF+OC5ptj5bJ7WFRPdYtuHv0yTRxK2tJvi+BDqbkiG7h7u/9SI89nRAYcmap2xHQL9D+QG/6wSrTtXg=="], + + "mdast-util-gfm-table": ["mdast-util-gfm-table@2.0.0", "", { "dependencies": { "@types/mdast": "^4.0.0", "devlop": "^1.0.0", "markdown-table": "^3.0.0", "mdast-util-from-markdown": "^2.0.0", "mdast-util-to-markdown": "^2.0.0" } }, "sha512-78UEvebzz/rJIxLvE7ZtDd/vIQ0RHv+3Mh5DR96p7cS7HsBhYIICDBCu8csTNWNO6tBWfqXPWekRuj2FNOGOZg=="], + + "mdast-util-gfm-task-list-item": ["mdast-util-gfm-task-list-item@2.0.0", "", { "dependencies": { "@types/mdast": "^4.0.0", "devlop": "^1.0.0", "mdast-util-from-markdown": "^2.0.0", "mdast-util-to-markdown": "^2.0.0" } }, "sha512-IrtvNvjxC1o06taBAVJznEnkiHxLFTzgonUdy8hzFVeDun0uTjxxrRGVaNFqkU1wJR3RBPEfsxmU6jDWPofrTQ=="], + + "mdast-util-mdx": ["mdast-util-mdx@3.0.0", "", { "dependencies": { "mdast-util-from-markdown": "^2.0.0", "mdast-util-mdx-expression": "^2.0.0", "mdast-util-mdx-jsx": "^3.0.0", "mdast-util-mdxjs-esm": "^2.0.0", "mdast-util-to-markdown": "^2.0.0" } }, "sha512-JfbYLAW7XnYTTbUsmpu0kdBUVe+yKVJZBItEjwyYJiDJuZ9w4eeaqks4HQO+R7objWgS2ymV60GYpI14Ug554w=="], + + "mdast-util-mdx-expression": ["mdast-util-mdx-expression@2.0.1", "", { "dependencies": { "@types/estree-jsx": "^1.0.0", "@types/hast": "^3.0.0", "@types/mdast": "^4.0.0", "devlop": "^1.0.0", "mdast-util-from-markdown": "^2.0.0", "mdast-util-to-markdown": "^2.0.0" } }, "sha512-J6f+9hUp+ldTZqKRSg7Vw5V6MqjATc+3E4gf3CFNcuZNWD8XdyI6zQ8GqH7f8169MM6P7hMBRDVGnn7oHB9kXQ=="], + + "mdast-util-mdx-jsx": ["mdast-util-mdx-jsx@3.2.0", "", { "dependencies": { "@types/estree-jsx": "^1.0.0", "@types/hast": "^3.0.0", "@types/mdast": "^4.0.0", "@types/unist": "^3.0.0", "ccount": "^2.0.0", "devlop": "^1.1.0", "mdast-util-from-markdown": "^2.0.0", "mdast-util-to-markdown": "^2.0.0", "parse-entities": "^4.0.0", "stringify-entities": "^4.0.0", "unist-util-stringify-position": "^4.0.0", "vfile-message": "^4.0.0" } }, "sha512-lj/z8v0r6ZtsN/cGNNtemmmfoLAFZnjMbNyLzBafjzikOM+glrjNHPlf6lQDOTccj9n5b0PPihEBbhneMyGs1Q=="], + + "mdast-util-mdxjs-esm": ["mdast-util-mdxjs-esm@2.0.1", "", { "dependencies": { "@types/estree-jsx": "^1.0.0", "@types/hast": "^3.0.0", "@types/mdast": "^4.0.0", "devlop": "^1.0.0", "mdast-util-from-markdown": "^2.0.0", "mdast-util-to-markdown": "^2.0.0" } }, "sha512-EcmOpxsZ96CvlP03NghtH1EsLtr0n9Tm4lPUJUBccV9RwUOneqSycg19n5HGzCf+10LozMRSObtVr3ee1WoHtg=="], + + "mdast-util-phrasing": ["mdast-util-phrasing@4.1.0", "", { "dependencies": { "@types/mdast": "^4.0.0", "unist-util-is": "^6.0.0" } }, "sha512-TqICwyvJJpBwvGAMZjj4J2n0X8QWp21b9l0o7eXyVJ25YNWYbJDVIyD1bZXE6WtV6RmKJVYmQAKWa0zWOABz2w=="], + + "mdast-util-to-hast": ["mdast-util-to-hast@13.2.0", "", { "dependencies": { "@types/hast": "^3.0.0", "@types/mdast": "^4.0.0", "@ungap/structured-clone": "^1.0.0", "devlop": "^1.0.0", "micromark-util-sanitize-uri": "^2.0.0", "trim-lines": "^3.0.0", "unist-util-position": "^5.0.0", "unist-util-visit": "^5.0.0", "vfile": "^6.0.0" } }, "sha512-QGYKEuUsYT9ykKBCMOEDLsU5JRObWQusAolFMeko/tYPufNkRffBAQjIE+99jbA87xv6FgmjLtwjh9wBWajwAA=="], + + "mdast-util-to-markdown": ["mdast-util-to-markdown@2.1.2", "", { "dependencies": { "@types/mdast": "^4.0.0", "@types/unist": "^3.0.0", "longest-streak": "^3.0.0", "mdast-util-phrasing": "^4.0.0", "mdast-util-to-string": "^4.0.0", "micromark-util-classify-character": "^2.0.0", "micromark-util-decode-string": "^2.0.0", "unist-util-visit": "^5.0.0", "zwitch": "^2.0.0" } }, "sha512-xj68wMTvGXVOKonmog6LwyJKrYXZPvlwabaryTjLh9LuvovB/KAH+kvi8Gjj+7rJjsFi23nkUxRQv1KqSroMqA=="], + + "mdast-util-to-string": ["mdast-util-to-string@4.0.0", "", { "dependencies": { "@types/mdast": "^4.0.0" } }, "sha512-0H44vDimn51F0YwvxSJSm0eCDOJTRlmN0R1yBh4HLj9wiV1Dn0QoXGbvFAWj2hSItVTlCmBF1hqKlIyUBVFLPg=="], + + "media-query-parser": ["media-query-parser@2.0.2", "", { "dependencies": { "@babel/runtime": "^7.12.5" } }, "sha512-1N4qp+jE0pL5Xv4uEcwVUhIkwdUO3S/9gML90nqKA7v7FcOS5vUtatfzok9S9U1EJU8dHWlcv95WLnKmmxZI9w=="], + + "merge-stream": ["merge-stream@2.0.0", "", {}, "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w=="], + + "merge2": ["merge2@1.4.1", "", {}, "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg=="], + + "mermaid": ["mermaid@11.9.0", "", { "dependencies": { "@braintree/sanitize-url": "^7.0.4", "@iconify/utils": "^2.1.33", "@mermaid-js/parser": "^0.6.2", "@types/d3": "^7.4.3", "cytoscape": "^3.29.3", "cytoscape-cose-bilkent": "^4.1.0", "cytoscape-fcose": "^2.2.0", "d3": "^7.9.0", "d3-sankey": "^0.12.3", "dagre-d3-es": "7.0.11", "dayjs": "^1.11.13", "dompurify": "^3.2.5", "katex": "^0.16.22", "khroma": "^2.1.0", "lodash-es": "^4.17.21", "marked": "^16.0.0", "roughjs": "^4.6.6", "stylis": "^4.3.6", "ts-dedent": "^2.2.0", "uuid": "^11.1.0" } }, "sha512-YdPXn9slEwO0omQfQIsW6vS84weVQftIyyTGAZCwM//MGhPzL1+l6vO6bkf0wnP4tHigH1alZ5Ooy3HXI2gOag=="], + + "mermaid-isomorphic": ["mermaid-isomorphic@3.0.4", "", { "dependencies": { "@fortawesome/fontawesome-free": "^6.0.0", "mermaid": "^11.0.0" }, "peerDependencies": { "playwright": "1" }, "optionalPeers": ["playwright"] }, "sha512-XQTy7H1XwHK3DPEHf+ZNWiqUEd9BwX3Xws38R9Fj2gx718srmgjlZoUzHr+Tca+O+dqJOJsAJaKzCoP65QDfDg=="], + + "micromark": ["micromark@4.0.2", "", { "dependencies": { "@types/debug": "^4.0.0", "debug": "^4.0.0", "decode-named-character-reference": "^1.0.0", "devlop": "^1.0.0", "micromark-core-commonmark": "^2.0.0", "micromark-factory-space": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-chunked": "^2.0.0", "micromark-util-combine-extensions": "^2.0.0", "micromark-util-decode-numeric-character-reference": "^2.0.0", "micromark-util-encode": "^2.0.0", "micromark-util-normalize-identifier": "^2.0.0", "micromark-util-resolve-all": "^2.0.0", "micromark-util-sanitize-uri": "^2.0.0", "micromark-util-subtokenize": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-zpe98Q6kvavpCr1NPVSCMebCKfD7CA2NqZ+rykeNhONIJBpc1tFKt9hucLGwha3jNTNI8lHpctWJWoimVF4PfA=="], + + "micromark-core-commonmark": ["micromark-core-commonmark@2.0.3", "", { "dependencies": { "decode-named-character-reference": "^1.0.0", "devlop": "^1.0.0", "micromark-factory-destination": "^2.0.0", "micromark-factory-label": "^2.0.0", "micromark-factory-space": "^2.0.0", "micromark-factory-title": "^2.0.0", "micromark-factory-whitespace": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-chunked": "^2.0.0", "micromark-util-classify-character": "^2.0.0", "micromark-util-html-tag-name": "^2.0.0", "micromark-util-normalize-identifier": "^2.0.0", "micromark-util-resolve-all": "^2.0.0", "micromark-util-subtokenize": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-RDBrHEMSxVFLg6xvnXmb1Ayr2WzLAWjeSATAoxwKYJV94TeNavgoIdA0a9ytzDSVzBy2YKFK+emCPOEibLeCrg=="], + + "micromark-extension-directive": ["micromark-extension-directive@3.0.2", "", { "dependencies": { "devlop": "^1.0.0", "micromark-factory-space": "^2.0.0", "micromark-factory-whitespace": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0", "parse-entities": "^4.0.0" } }, "sha512-wjcXHgk+PPdmvR58Le9d7zQYWy+vKEU9Se44p2CrCDPiLr2FMyiT4Fyb5UFKFC66wGB3kPlgD7q3TnoqPS7SZA=="], + + "micromark-extension-frontmatter": ["micromark-extension-frontmatter@2.0.0", "", { "dependencies": { "fault": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-C4AkuM3dA58cgZha7zVnuVxBhDsbttIMiytjgsM2XbHAB2faRVaHRle40558FBN+DJcrLNCoqG5mlrpdU4cRtg=="], + + "micromark-extension-gfm": ["micromark-extension-gfm@3.0.0", "", { "dependencies": { "micromark-extension-gfm-autolink-literal": "^2.0.0", "micromark-extension-gfm-footnote": "^2.0.0", "micromark-extension-gfm-strikethrough": "^2.0.0", "micromark-extension-gfm-table": "^2.0.0", "micromark-extension-gfm-tagfilter": "^2.0.0", "micromark-extension-gfm-task-list-item": "^2.0.0", "micromark-util-combine-extensions": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-vsKArQsicm7t0z2GugkCKtZehqUm31oeGBV/KVSorWSy8ZlNAv7ytjFhvaryUiCUJYqs+NoE6AFhpQvBTM6Q4w=="], + + "micromark-extension-gfm-autolink-literal": ["micromark-extension-gfm-autolink-literal@2.1.0", "", { "dependencies": { "micromark-util-character": "^2.0.0", "micromark-util-sanitize-uri": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-oOg7knzhicgQ3t4QCjCWgTmfNhvQbDDnJeVu9v81r7NltNCVmhPy1fJRX27pISafdjL+SVc4d3l48Gb6pbRypw=="], + + "micromark-extension-gfm-footnote": ["micromark-extension-gfm-footnote@2.1.0", "", { "dependencies": { "devlop": "^1.0.0", "micromark-core-commonmark": "^2.0.0", "micromark-factory-space": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-normalize-identifier": "^2.0.0", "micromark-util-sanitize-uri": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-/yPhxI1ntnDNsiHtzLKYnE3vf9JZ6cAisqVDauhp4CEHxlb4uoOTxOCJ+9s51bIB8U1N1FJ1RXOKTIlD5B/gqw=="], + + "micromark-extension-gfm-strikethrough": ["micromark-extension-gfm-strikethrough@2.1.0", "", { "dependencies": { "devlop": "^1.0.0", "micromark-util-chunked": "^2.0.0", "micromark-util-classify-character": "^2.0.0", "micromark-util-resolve-all": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-ADVjpOOkjz1hhkZLlBiYA9cR2Anf8F4HqZUO6e5eDcPQd0Txw5fxLzzxnEkSkfnD0wziSGiv7sYhk/ktvbf1uw=="], + + "micromark-extension-gfm-table": ["micromark-extension-gfm-table@2.1.1", "", { "dependencies": { "devlop": "^1.0.0", "micromark-factory-space": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-t2OU/dXXioARrC6yWfJ4hqB7rct14e8f7m0cbI5hUmDyyIlwv5vEtooptH8INkbLzOatzKuVbQmAYcbWoyz6Dg=="], + + "micromark-extension-gfm-tagfilter": ["micromark-extension-gfm-tagfilter@2.0.0", "", { "dependencies": { "micromark-util-types": "^2.0.0" } }, "sha512-xHlTOmuCSotIA8TW1mDIM6X2O1SiX5P9IuDtqGonFhEK0qgRI4yeC6vMxEV2dgyr2TiD+2PQ10o+cOhdVAcwfg=="], + + "micromark-extension-gfm-task-list-item": ["micromark-extension-gfm-task-list-item@2.1.0", "", { "dependencies": { "devlop": "^1.0.0", "micromark-factory-space": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-qIBZhqxqI6fjLDYFTBIa4eivDMnP+OZqsNwmQ3xNLE4Cxwc+zfQEfbs6tzAo2Hjq+bh6q5F+Z8/cksrLFYWQQw=="], + + "micromark-extension-mdx-expression": ["micromark-extension-mdx-expression@3.0.1", "", { "dependencies": { "@types/estree": "^1.0.0", "devlop": "^1.0.0", "micromark-factory-mdx-expression": "^2.0.0", "micromark-factory-space": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-events-to-acorn": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-dD/ADLJ1AeMvSAKBwO22zG22N4ybhe7kFIZ3LsDI0GlsNr2A3KYxb0LdC1u5rj4Nw+CHKY0RVdnHX8vj8ejm4Q=="], + + "micromark-extension-mdx-jsx": ["micromark-extension-mdx-jsx@3.0.2", "", { "dependencies": { "@types/estree": "^1.0.0", "devlop": "^1.0.0", "estree-util-is-identifier-name": "^3.0.0", "micromark-factory-mdx-expression": "^2.0.0", "micromark-factory-space": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-events-to-acorn": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0", "vfile-message": "^4.0.0" } }, "sha512-e5+q1DjMh62LZAJOnDraSSbDMvGJ8x3cbjygy2qFEi7HCeUT4BDKCvMozPozcD6WmOt6sVvYDNBKhFSz3kjOVQ=="], + + "micromark-extension-mdx-md": ["micromark-extension-mdx-md@2.0.0", "", { "dependencies": { "micromark-util-types": "^2.0.0" } }, "sha512-EpAiszsB3blw4Rpba7xTOUptcFeBFi+6PY8VnJ2hhimH+vCQDirWgsMpz7w1XcZE7LVrSAUGb9VJpG9ghlYvYQ=="], + + "micromark-extension-mdxjs": ["micromark-extension-mdxjs@3.0.0", "", { "dependencies": { "acorn": "^8.0.0", "acorn-jsx": "^5.0.0", "micromark-extension-mdx-expression": "^3.0.0", "micromark-extension-mdx-jsx": "^3.0.0", "micromark-extension-mdx-md": "^2.0.0", "micromark-extension-mdxjs-esm": "^3.0.0", "micromark-util-combine-extensions": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-A873fJfhnJ2siZyUrJ31l34Uqwy4xIFmvPY1oj+Ean5PHcPBYzEsvqvWGaWcfEIr11O5Dlw3p2y0tZWpKHDejQ=="], + + "micromark-extension-mdxjs-esm": ["micromark-extension-mdxjs-esm@3.0.0", "", { "dependencies": { "@types/estree": "^1.0.0", "devlop": "^1.0.0", "micromark-core-commonmark": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-events-to-acorn": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0", "unist-util-position-from-estree": "^2.0.0", "vfile-message": "^4.0.0" } }, "sha512-DJFl4ZqkErRpq/dAPyeWp15tGrcrrJho1hKK5uBS70BCtfrIFg81sqcTVu3Ta+KD1Tk5vAtBNElWxtAa+m8K9A=="], + + "micromark-factory-destination": ["micromark-factory-destination@2.0.1", "", { "dependencies": { "micromark-util-character": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-Xe6rDdJlkmbFRExpTOmRj9N3MaWmbAgdpSrBQvCFqhezUn4AHqJHbaEnfbVYYiexVSs//tqOdY/DxhjdCiJnIA=="], + + "micromark-factory-label": ["micromark-factory-label@2.0.1", "", { "dependencies": { "devlop": "^1.0.0", "micromark-util-character": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-VFMekyQExqIW7xIChcXn4ok29YE3rnuyveW3wZQWWqF4Nv9Wk5rgJ99KzPvHjkmPXF93FXIbBp6YdW3t71/7Vg=="], + + "micromark-factory-mdx-expression": ["micromark-factory-mdx-expression@2.0.3", "", { "dependencies": { "@types/estree": "^1.0.0", "devlop": "^1.0.0", "micromark-factory-space": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-events-to-acorn": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0", "unist-util-position-from-estree": "^2.0.0", "vfile-message": "^4.0.0" } }, "sha512-kQnEtA3vzucU2BkrIa8/VaSAsP+EJ3CKOvhMuJgOEGg9KDC6OAY6nSnNDVRiVNRqj7Y4SlSzcStaH/5jge8JdQ=="], + + "micromark-factory-space": ["micromark-factory-space@2.0.1", "", { "dependencies": { "micromark-util-character": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-zRkxjtBxxLd2Sc0d+fbnEunsTj46SWXgXciZmHq0kDYGnck/ZSGj9/wULTV95uoeYiK5hRXP2mJ98Uo4cq/LQg=="], + + "micromark-factory-title": ["micromark-factory-title@2.0.1", "", { "dependencies": { "micromark-factory-space": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-5bZ+3CjhAd9eChYTHsjy6TGxpOFSKgKKJPJxr293jTbfry2KDoWkhBb6TcPVB4NmzaPhMs1Frm9AZH7OD4Cjzw=="], + + "micromark-factory-whitespace": ["micromark-factory-whitespace@2.0.1", "", { "dependencies": { "micromark-factory-space": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-Ob0nuZ3PKt/n0hORHyvoD9uZhr+Za8sFoP+OnMcnWK5lngSzALgQYKMr9RJVOWLqQYuyn6ulqGWSXdwf6F80lQ=="], + + "micromark-util-character": ["micromark-util-character@2.1.1", "", { "dependencies": { "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-wv8tdUTJ3thSFFFJKtpYKOYiGP2+v96Hvk4Tu8KpCAsTMs6yi+nVmGh1syvSCsaxz45J6Jbw+9DD6g97+NV67Q=="], + + "micromark-util-chunked": ["micromark-util-chunked@2.0.1", "", { "dependencies": { "micromark-util-symbol": "^2.0.0" } }, "sha512-QUNFEOPELfmvv+4xiNg2sRYeS/P84pTW0TCgP5zc9FpXetHY0ab7SxKyAQCNCc1eK0459uoLI1y5oO5Vc1dbhA=="], + + "micromark-util-classify-character": ["micromark-util-classify-character@2.0.1", "", { "dependencies": { "micromark-util-character": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-K0kHzM6afW/MbeWYWLjoHQv1sgg2Q9EccHEDzSkxiP/EaagNzCm7T/WMKZ3rjMbvIpvBiZgwR3dKMygtA4mG1Q=="], + + "micromark-util-combine-extensions": ["micromark-util-combine-extensions@2.0.1", "", { "dependencies": { "micromark-util-chunked": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-OnAnH8Ujmy59JcyZw8JSbK9cGpdVY44NKgSM7E9Eh7DiLS2E9RNQf0dONaGDzEG9yjEl5hcqeIsj4hfRkLH/Bg=="], + + "micromark-util-decode-numeric-character-reference": ["micromark-util-decode-numeric-character-reference@2.0.2", "", { "dependencies": { "micromark-util-symbol": "^2.0.0" } }, "sha512-ccUbYk6CwVdkmCQMyr64dXz42EfHGkPQlBj5p7YVGzq8I7CtjXZJrubAYezf7Rp+bjPseiROqe7G6foFd+lEuw=="], + + "micromark-util-decode-string": ["micromark-util-decode-string@2.0.1", "", { "dependencies": { "decode-named-character-reference": "^1.0.0", "micromark-util-character": "^2.0.0", "micromark-util-decode-numeric-character-reference": "^2.0.0", "micromark-util-symbol": "^2.0.0" } }, "sha512-nDV/77Fj6eH1ynwscYTOsbK7rR//Uj0bZXBwJZRfaLEJ1iGBR6kIfNmlNqaqJf649EP0F3NWNdeJi03elllNUQ=="], + + "micromark-util-encode": ["micromark-util-encode@2.0.1", "", {}, "sha512-c3cVx2y4KqUnwopcO9b/SCdo2O67LwJJ/UyqGfbigahfegL9myoEFoDYZgkT7f36T0bLrM9hZTAaAyH+PCAXjw=="], + + "micromark-util-events-to-acorn": ["micromark-util-events-to-acorn@2.0.3", "", { "dependencies": { "@types/estree": "^1.0.0", "@types/unist": "^3.0.0", "devlop": "^1.0.0", "estree-util-visit": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0", "vfile-message": "^4.0.0" } }, "sha512-jmsiEIiZ1n7X1Rr5k8wVExBQCg5jy4UXVADItHmNk1zkwEVhBuIUKRu3fqv+hs4nxLISi2DQGlqIOGiFxgbfHg=="], + + "micromark-util-html-tag-name": ["micromark-util-html-tag-name@2.0.1", "", {}, "sha512-2cNEiYDhCWKI+Gs9T0Tiysk136SnR13hhO8yW6BGNyhOC4qYFnwF1nKfD3HFAIXA5c45RrIG1ub11GiXeYd1xA=="], + + "micromark-util-normalize-identifier": ["micromark-util-normalize-identifier@2.0.1", "", { "dependencies": { "micromark-util-symbol": "^2.0.0" } }, "sha512-sxPqmo70LyARJs0w2UclACPUUEqltCkJ6PhKdMIDuJ3gSf/Q+/GIe3WKl0Ijb/GyH9lOpUkRAO2wp0GVkLvS9Q=="], + + "micromark-util-resolve-all": ["micromark-util-resolve-all@2.0.1", "", { "dependencies": { "micromark-util-types": "^2.0.0" } }, "sha512-VdQyxFWFT2/FGJgwQnJYbe1jjQoNTS4RjglmSjTUlpUMa95Htx9NHeYW4rGDJzbjvCsl9eLjMQwGeElsqmzcHg=="], + + "micromark-util-sanitize-uri": ["micromark-util-sanitize-uri@2.0.1", "", { "dependencies": { "micromark-util-character": "^2.0.0", "micromark-util-encode": "^2.0.0", "micromark-util-symbol": "^2.0.0" } }, "sha512-9N9IomZ/YuGGZZmQec1MbgxtlgougxTodVwDzzEouPKo3qFWvymFHWcnDi2vzV1ff6kas9ucW+o3yzJK9YB1AQ=="], + + "micromark-util-subtokenize": ["micromark-util-subtokenize@2.1.0", "", { "dependencies": { "devlop": "^1.0.0", "micromark-util-chunked": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-XQLu552iSctvnEcgXw6+Sx75GflAPNED1qx7eBJ+wydBb2KCbRZe+NwvIEEMM83uml1+2WSXpBAcp9IUCgCYWA=="], + + "micromark-util-symbol": ["micromark-util-symbol@2.0.1", "", {}, "sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q=="], + + "micromark-util-types": ["micromark-util-types@2.0.2", "", {}, "sha512-Yw0ECSpJoViF1qTU4DC6NwtC4aWGt1EkzaQB8KPPyCRR8z9TWeV0HbEFGTO+ZY1wB22zmxnJqhPyTpOVCpeHTA=="], + + "micromatch": ["micromatch@4.0.8", "", { "dependencies": { "braces": "^3.0.3", "picomatch": "^2.3.1" } }, "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA=="], + + "mime": ["mime@1.6.0", "", { "bin": { "mime": "cli.js" } }, "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg=="], + + "mime-db": ["mime-db@1.54.0", "", {}, "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ=="], + + "mimic-fn": ["mimic-fn@2.1.0", "", {}, "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg=="], + + "mini-svg-data-uri": ["mini-svg-data-uri@1.4.4", "", { "bin": { "mini-svg-data-uri": "cli.js" } }, "sha512-r9deDe9p5FJUPZAk3A59wGH7Ii9YrjjWw0jmw/liSbHl2CHiyXj6FcDXDu2K3TjVAXqiJdaw3xxwlZZr9E6nHg=="], + + "minimatch": ["minimatch@10.0.3", "", { "dependencies": { "@isaacs/brace-expansion": "^5.0.0" } }, "sha512-IPZ167aShDZZUMdRk66cyQAW3qr0WzbHkPdMYa8bzZhlHhO3jALbKdxcaak7W9FfT2rZNpQuUu4Od7ILEpXSaw=="], + + "minipass": ["minipass@7.1.2", "", {}, "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw=="], + + "minisearch": ["minisearch@6.3.0", "", {}, "sha512-ihFnidEeU8iXzcVHy74dhkxh/dn8Dc08ERl0xwoMMGqp4+LvRSCgicb+zGqWthVokQKvCSxITlh3P08OzdTYCQ=="], + + "mlly": ["mlly@1.7.4", "", { "dependencies": { "acorn": "^8.14.0", "pathe": "^2.0.1", "pkg-types": "^1.3.0", "ufo": "^1.5.4" } }, "sha512-qmdSIPC4bDJXgZTCR7XosJiNKySV7O215tsPtDN9iEO/7q/76b/ijtgRu/+epFXSJhijtTCCGp3DWS549P3xKw=="], + + "modern-ahocorasick": ["modern-ahocorasick@1.1.0", "", {}, "sha512-sEKPVl2rM+MNVkGQt3ChdmD8YsigmXdn5NifZn6jiwn9LRJpWm8F3guhaqrJT/JOat6pwpbXEk6kv+b9DMIjsQ=="], + + "ms": ["ms@2.0.0", "", {}, "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="], + + "nanoid": ["nanoid@3.3.11", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w=="], + + "negotiator": ["negotiator@0.6.4", "", {}, "sha512-myRT3DiWPHqho5PrJaIRyaMv2kgYf0mUVgBNOYMuCH5Ki1yEiQaf/ZJuQ62nvpc44wL5WDbTX7yGJi1Neevw8w=="], + + "node-releases": ["node-releases@2.0.19", "", {}, "sha512-xxOWJsBKtzAq7DY0J+DTzuz58K8e7sJbdgwkbMWQe8UYB6ekmsQ45q0M/tJDsGaZmbC+l7n57UV8Hl5tHxO9uw=="], + + "normalize-range": ["normalize-range@0.1.2", "", {}, "sha512-bdok/XvKII3nUpklnV6P2hxtMNrCboOjAcyBuQnWEhO665FwrSNRxU+AqpsyvO6LgGYPspN+lu5CLtw4jPRKNA=="], + + "npm-run-path": ["npm-run-path@4.0.1", "", { "dependencies": { "path-key": "^3.0.0" } }, "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw=="], + + "nth-check": ["nth-check@2.1.1", "", { "dependencies": { "boolbase": "^1.0.0" } }, "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w=="], + + "on-finished": ["on-finished@2.4.1", "", { "dependencies": { "ee-first": "1.1.1" } }, "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg=="], + + "on-headers": ["on-headers@1.0.2", "", {}, "sha512-pZAE+FJLoyITytdqK0U5s+FIpjN0JP3OzFi/u8Rx+EV5/W+JTWGXG8xFzevE7AjBfDqHv/8vL8qQsIhHnqRkrA=="], + + "onetime": ["onetime@5.1.2", "", { "dependencies": { "mimic-fn": "^2.1.0" } }, "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg=="], + + "oniguruma-to-es": ["oniguruma-to-es@2.3.0", "", { "dependencies": { "emoji-regex-xs": "^1.0.0", "regex": "^5.1.1", "regex-recursion": "^5.1.1" } }, "sha512-bwALDxriqfKGfUufKGGepCzu9x7nJQuoRoAFp4AnwehhC2crqrDIAP/uN2qdlsAvSMpeRC3+Yzhqc7hLmle5+g=="], + + "ora": ["ora@7.0.1", "", { "dependencies": { "chalk": "^5.3.0", "cli-cursor": "^4.0.0", "cli-spinners": "^2.9.0", "is-interactive": "^2.0.0", "is-unicode-supported": "^1.3.0", "log-symbols": "^5.1.0", "stdin-discarder": "^0.1.0", "string-width": "^6.1.0", "strip-ansi": "^7.1.0" } }, "sha512-0TUxTiFJWv+JnjWm4o9yvuskpEJLXTcng8MJuKd+SzAzp2o+OP3HWqNhB4OdJRt1Vsd9/mR0oyaEYlOnL7XIRw=="], + + "p-limit": ["p-limit@5.0.0", "", { "dependencies": { "yocto-queue": "^1.0.0" } }, "sha512-/Eaoq+QyLSiXQ4lyYV23f14mZRQcXnxfHrN0vCai+ak9G0pp9iEQukIIZq5NccEvwRB8PUnZT0KsOoDCINS1qQ=="], + + "p-locate": ["p-locate@5.0.0", "", { "dependencies": { "p-limit": "^3.0.2" } }, "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw=="], + + "package-json-from-dist": ["package-json-from-dist@1.0.1", "", {}, "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw=="], + + "package-manager-detector": ["package-manager-detector@1.3.0", "", {}, "sha512-ZsEbbZORsyHuO00lY1kV3/t72yp6Ysay6Pd17ZAlNGuGwmWDLCJxFpRs0IzfXfj1o4icJOkUEioexFHzyPurSQ=="], + + "parse-entities": ["parse-entities@4.0.2", "", { "dependencies": { "@types/unist": "^2.0.0", "character-entities-legacy": "^3.0.0", "character-reference-invalid": "^2.0.0", "decode-named-character-reference": "^1.0.0", "is-alphanumerical": "^2.0.0", "is-decimal": "^2.0.0", "is-hexadecimal": "^2.0.0" } }, "sha512-GG2AQYWoLgL877gQIKeRPGO1xF9+eG1ujIb5soS5gPvLQ1y2o8FL90w2QWNdf9I361Mpp7726c+lj3U0qK1uGw=="], + + "parse5": ["parse5@7.3.0", "", { "dependencies": { "entities": "^6.0.0" } }, "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw=="], + + "parseurl": ["parseurl@1.3.3", "", {}, "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ=="], + + "path-data-parser": ["path-data-parser@0.1.0", "", {}, "sha512-NOnmBpt5Y2RWbuv0LMzsayp3lVylAHLPUTut412ZA3l+C4uw4ZVkQbjShYCQ8TCpUMdPapr4YjUqLYD6v68j+w=="], + + "path-exists": ["path-exists@4.0.0", "", {}, "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w=="], + + "path-key": ["path-key@3.1.1", "", {}, "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q=="], + + "path-scurry": ["path-scurry@2.0.0", "", { "dependencies": { "lru-cache": "^11.0.0", "minipass": "^7.1.2" } }, "sha512-ypGJsmGtdXUOeM5u93TyeIEfEhM6s+ljAhrk5vAvSx8uyY/02OvrZnA0YNGUrPXfpJMgI1ODd3nwz8Npx4O4cg=="], + + "path-type": ["path-type@6.0.0", "", {}, "sha512-Vj7sf++t5pBD637NSfkxpHSMfWaeig5+DKWLhcqIYx6mWQz5hdJTGDVMQiJcw1ZYkhs7AazKDGpRVji1LJCZUQ=="], + + "pathe": ["pathe@2.0.3", "", {}, "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w=="], + + "picocolors": ["picocolors@1.1.1", "", {}, "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA=="], + + "picomatch": ["picomatch@4.0.3", "", {}, "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q=="], + + "pkg-types": ["pkg-types@1.3.1", "", { "dependencies": { "confbox": "^0.1.8", "mlly": "^1.7.4", "pathe": "^2.0.1" } }, "sha512-/Jm5M4RvtBFVkKWRu2BLUTNP8/M2a+UwuAX+ae4770q1qVGtfjG+WTCupoZixokjmHiry8uI+dlY8KXYV5HVVQ=="], + + "playwright": ["playwright@1.54.1", "", { "dependencies": { "playwright-core": "1.54.1" }, "optionalDependencies": { "fsevents": "2.3.2" }, "bin": { "playwright": "cli.js" } }, "sha512-peWpSwIBmSLi6aW2auvrUtf2DqY16YYcCMO8rTVx486jKmDTJg7UAhyrraP98GB8BoPURZP8+nxO7TSd4cPr5g=="], + + "playwright-core": ["playwright-core@1.54.1", "", { "bin": { "playwright-core": "cli.js" } }, "sha512-Nbjs2zjj0htNhzgiy5wu+3w09YetDx5pkrpI/kZotDlDUaYk0HVA5xrBVPdow4SAUIlhgKcJeJg4GRKW6xHusA=="], + + "points-on-curve": ["points-on-curve@0.2.0", "", {}, "sha512-0mYKnYYe9ZcqMCWhUjItv/oHjvgEsfKvnUTg8sAtnHr3GVy7rGkXCb6d5cSyqrWqL4k81b9CPg3urd+T7aop3A=="], + + "points-on-path": ["points-on-path@0.2.1", "", { "dependencies": { "path-data-parser": "0.1.0", "points-on-curve": "0.2.0" } }, "sha512-25ClnWWuw7JbWZcgqY/gJ4FQWadKxGWk+3kR/7kD0tCaDtPPMj7oHu2ToLaVhfpnHrZzYby2w6tUA0eOIuUg8g=="], + + "postcss": ["postcss@8.5.6", "", { "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" } }, "sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg=="], + + "postcss-value-parser": ["postcss-value-parser@4.2.0", "", {}, "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ=="], + + "property-information": ["property-information@6.5.0", "", {}, "sha512-PgTgs/BlvHxOu8QuEN7wi5A0OmXaBcHpmCSTehcs6Uuu9IkDIEo13Hy7n898RHfrQ49vKCoGeWZSaAK01nwVig=="], + + "quansync": ["quansync@0.2.10", "", {}, "sha512-t41VRkMYbkHyCYmOvx/6URnN80H7k4X0lLdBMGsz+maAwrJQYB1djpV6vHrQIBE0WBSGqhtEHrK9U3DWWH8v7A=="], + + "queue-microtask": ["queue-microtask@1.2.3", "", {}, "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A=="], + + "radix-ui": ["radix-ui@1.4.2", "", { "dependencies": { "@radix-ui/primitive": "1.1.2", "@radix-ui/react-accessible-icon": "1.1.7", "@radix-ui/react-accordion": "1.2.11", "@radix-ui/react-alert-dialog": "1.1.14", "@radix-ui/react-arrow": "1.1.7", "@radix-ui/react-aspect-ratio": "1.1.7", "@radix-ui/react-avatar": "1.1.10", "@radix-ui/react-checkbox": "1.3.2", "@radix-ui/react-collapsible": "1.1.11", "@radix-ui/react-collection": "1.1.7", "@radix-ui/react-compose-refs": "1.1.2", "@radix-ui/react-context": "1.1.2", "@radix-ui/react-context-menu": "2.2.15", "@radix-ui/react-dialog": "1.1.14", "@radix-ui/react-direction": "1.1.1", "@radix-ui/react-dismissable-layer": "1.1.10", "@radix-ui/react-dropdown-menu": "2.1.15", "@radix-ui/react-focus-guards": "1.1.2", "@radix-ui/react-focus-scope": "1.1.7", "@radix-ui/react-form": "0.1.7", "@radix-ui/react-hover-card": "1.1.14", "@radix-ui/react-label": "2.1.7", "@radix-ui/react-menu": "2.1.15", "@radix-ui/react-menubar": "1.1.15", "@radix-ui/react-navigation-menu": "1.2.13", "@radix-ui/react-one-time-password-field": "0.1.7", "@radix-ui/react-password-toggle-field": "0.1.2", "@radix-ui/react-popover": "1.1.14", "@radix-ui/react-popper": "1.2.7", "@radix-ui/react-portal": "1.1.9", "@radix-ui/react-presence": "1.1.4", "@radix-ui/react-primitive": "2.1.3", "@radix-ui/react-progress": "1.1.7", "@radix-ui/react-radio-group": "1.3.7", "@radix-ui/react-roving-focus": "1.1.10", "@radix-ui/react-scroll-area": "1.2.9", "@radix-ui/react-select": "2.2.5", "@radix-ui/react-separator": "1.1.7", "@radix-ui/react-slider": "1.3.5", "@radix-ui/react-slot": "1.2.3", "@radix-ui/react-switch": "1.2.5", "@radix-ui/react-tabs": "1.1.12", "@radix-ui/react-toast": "1.2.14", "@radix-ui/react-toggle": "1.1.9", "@radix-ui/react-toggle-group": "1.1.10", "@radix-ui/react-toolbar": "1.1.10", "@radix-ui/react-tooltip": "1.2.7", "@radix-ui/react-use-callback-ref": "1.1.1", "@radix-ui/react-use-controllable-state": "1.2.2", "@radix-ui/react-use-effect-event": "0.0.2", "@radix-ui/react-use-escape-keydown": "1.1.1", "@radix-ui/react-use-is-hydrated": "0.1.0", "@radix-ui/react-use-layout-effect": "1.1.1", "@radix-ui/react-use-size": "1.1.1", "@radix-ui/react-visually-hidden": "1.2.3" }, "peerDependencies": { "@types/react": "*", "@types/react-dom": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-fT/3YFPJzf2WUpqDoQi005GS8EpCi+53VhcLaHUj5fwkPYiZAjk1mSxFvbMA8Uq71L03n+WysuYC+mlKkXxt/Q=="], + + "range-parser": ["range-parser@1.2.1", "", {}, "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg=="], + + "react": ["react@19.1.0", "", {}, "sha512-FS+XFBNvn3GTAWq26joslQgWNoFu08F4kl0J4CgdNKADkdSGXQyTCnKteIAJy96Br6YbpEU1LSzV5dYtjMkMDg=="], + + "react-dom": ["react-dom@19.1.0", "", { "dependencies": { "scheduler": "^0.26.0" }, "peerDependencies": { "react": "^19.1.0" } }, "sha512-Xs1hdnE+DyKgeHJeJznQmYMIBG3TKIHJJT95Q58nHLSrElKlGQqDTR2HQ9fx5CN/Gk6Vh/kupBTDLU11/nDk/g=="], + + "react-intersection-observer": ["react-intersection-observer@9.16.0", "", { "peerDependencies": { "react": "^17.0.0 || ^18.0.0 || ^19.0.0", "react-dom": "^17.0.0 || ^18.0.0 || ^19.0.0" }, "optionalPeers": ["react-dom"] }, "sha512-w9nJSEp+DrW9KmQmeWHQyfaP6b03v+TdXynaoA964Wxt7mdR3An11z4NNCQgL4gKSK7y1ver2Fq+JKH6CWEzUA=="], + + "react-refresh": ["react-refresh@0.17.0", "", {}, "sha512-z6F7K9bV85EfseRCp2bzrpyQ0Gkw1uLoCel9XBVWPg/TjRj94SkJzUTGfOa4bs7iJvBWtQG0Wq7wnI0syw3EBQ=="], + + "react-remove-scroll": ["react-remove-scroll@2.7.1", "", { "dependencies": { "react-remove-scroll-bar": "^2.3.7", "react-style-singleton": "^2.2.3", "tslib": "^2.1.0", "use-callback-ref": "^1.3.3", "use-sidecar": "^1.1.3" }, "peerDependencies": { "@types/react": "*", "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-HpMh8+oahmIdOuS5aFKKY6Pyog+FNaZV/XyJOq7b4YFwsFHe5yYfdbIalI4k3vU2nSDql7YskmUseHsRrJqIPA=="], + + "react-remove-scroll-bar": ["react-remove-scroll-bar@2.3.8", "", { "dependencies": { "react-style-singleton": "^2.2.2", "tslib": "^2.0.0" }, "peerDependencies": { "@types/react": "*", "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" }, "optionalPeers": ["@types/react"] }, "sha512-9r+yi9+mgU33AKcj6IbT9oRCO78WriSj6t/cF8DWBZJ9aOGPOTEDvdUDz1FwKim7QXWwmHqtdHnRJfhAxEG46Q=="], + + "react-router": ["react-router@7.7.0", "", { "dependencies": { "cookie": "^1.0.1", "set-cookie-parser": "^2.6.0" }, "peerDependencies": { "react": ">=18", "react-dom": ">=18" }, "optionalPeers": ["react-dom"] }, "sha512-3FUYSwlvB/5wRJVTL/aavqHmfUKe0+Xm9MllkYgGo9eDwNdkvwlJGjpPxono1kCycLt6AnDTgjmXvK3/B4QGuw=="], + + "react-style-singleton": ["react-style-singleton@2.2.3", "", { "dependencies": { "get-nonce": "^1.0.0", "tslib": "^2.0.0" }, "peerDependencies": { "@types/react": "*", "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-b6jSvxvVnyptAiLjbkWLE/lOnR4lfTtDAl+eUC7RZy+QQWc6wRzIV2CE6xBuMmDxc2qIihtDCZD5NPOFl7fRBQ=="], + + "readable-stream": ["readable-stream@3.6.2", "", { "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", "util-deprecate": "^1.0.1" } }, "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA=="], + + "recma-build-jsx": ["recma-build-jsx@1.0.0", "", { "dependencies": { "@types/estree": "^1.0.0", "estree-util-build-jsx": "^3.0.0", "vfile": "^6.0.0" } }, "sha512-8GtdyqaBcDfva+GUKDr3nev3VpKAhup1+RvkMvUxURHpW7QyIvk9F5wz7Vzo06CEMSilw6uArgRqhpiUcWp8ew=="], + + "recma-jsx": ["recma-jsx@1.0.0", "", { "dependencies": { "acorn-jsx": "^5.0.0", "estree-util-to-js": "^2.0.0", "recma-parse": "^1.0.0", "recma-stringify": "^1.0.0", "unified": "^11.0.0" } }, "sha512-5vwkv65qWwYxg+Atz95acp8DMu1JDSqdGkA2Of1j6rCreyFUE/gp15fC8MnGEuG1W68UKjM6x6+YTWIh7hZM/Q=="], + + "recma-parse": ["recma-parse@1.0.0", "", { "dependencies": { "@types/estree": "^1.0.0", "esast-util-from-js": "^2.0.0", "unified": "^11.0.0", "vfile": "^6.0.0" } }, "sha512-OYLsIGBB5Y5wjnSnQW6t3Xg7q3fQ7FWbw/vcXtORTnyaSFscOtABg+7Pnz6YZ6c27fG1/aN8CjfwoUEUIdwqWQ=="], + + "recma-stringify": ["recma-stringify@1.0.0", "", { "dependencies": { "@types/estree": "^1.0.0", "estree-util-to-js": "^2.0.0", "unified": "^11.0.0", "vfile": "^6.0.0" } }, "sha512-cjwII1MdIIVloKvC9ErQ+OgAtwHBmcZ0Bg4ciz78FtbT8In39aAYbaA7zvxQ61xVMSPE8WxhLwLbhif4Js2C+g=="], + + "regex": ["regex@5.1.1", "", { "dependencies": { "regex-utilities": "^2.3.0" } }, "sha512-dN5I359AVGPnwzJm2jN1k0W9LPZ+ePvoOeVMMfqIMFz53sSwXkxaJoxr50ptnsC771lK95BnTrVSZxq0b9yCGw=="], + + "regex-recursion": ["regex-recursion@5.1.1", "", { "dependencies": { "regex": "^5.1.1", "regex-utilities": "^2.3.0" } }, "sha512-ae7SBCbzVNrIjgSbh7wMznPcQel1DNlDtzensnFxpiNpXt1U2ju/bHugH422r+4LAVS1FpW1YCwilmnNsjum9w=="], + + "regex-utilities": ["regex-utilities@2.3.0", "", {}, "sha512-8VhliFJAWRaUiVvREIiW2NXXTmHs4vMNnSzuJVhscgmGav3g9VDxLrQndI3dZZVVdp0ZO/5v0xmX516/7M9cng=="], + + "rehype-autolink-headings": ["rehype-autolink-headings@7.1.0", "", { "dependencies": { "@types/hast": "^3.0.0", "@ungap/structured-clone": "^1.0.0", "hast-util-heading-rank": "^3.0.0", "hast-util-is-element": "^3.0.0", "unified": "^11.0.0", "unist-util-visit": "^5.0.0" } }, "sha512-rItO/pSdvnvsP4QRB1pmPiNHUskikqtPojZKJPPPAVx9Hj8i8TwMBhofrrAYRhYOOBZH9tgmG5lPqDLuIWPWmw=="], + + "rehype-class-names": ["rehype-class-names@2.0.0", "", { "dependencies": { "@types/hast": "^3.0.0", "hast-util-classnames": "^3.0.0", "hast-util-select": "^6.0.0", "unified": "^11.0.4" } }, "sha512-jldCIiAEvXKdq8hqr5f5PzNdIDkvHC6zfKhwta9oRoMu7bn0W7qLES/JrrjBvr9rKz3nJ8x4vY1EWI+dhjHVZQ=="], + + "rehype-mermaid": ["rehype-mermaid@3.0.0", "", { "dependencies": { "@types/hast": "^3.0.0", "hast-util-from-html-isomorphic": "^2.0.0", "hast-util-to-text": "^4.0.0", "mermaid-isomorphic": "^3.0.0", "mini-svg-data-uri": "^1.0.0", "space-separated-tokens": "^2.0.0", "unified": "^11.0.0", "unist-util-visit-parents": "^6.0.0", "vfile": "^6.0.0" }, "peerDependencies": { "playwright": "1" }, "optionalPeers": ["playwright"] }, "sha512-fxrD5E4Fa1WXUjmjNDvLOMT4XB1WaxcfycFIWiYU0yEMQhcTDElc9aDFnbDFRLxG1Cfo1I3mfD5kg4sjlWaB+Q=="], + + "rehype-recma": ["rehype-recma@1.0.0", "", { "dependencies": { "@types/estree": "^1.0.0", "@types/hast": "^3.0.0", "hast-util-to-estree": "^3.0.0" } }, "sha512-lqA4rGUf1JmacCNWWZx0Wv1dHqMwxzsDWYMTowuplHF3xH0N/MmrZ/G3BDZnzAkRmxDadujCjaKM2hqYdCBOGw=="], + + "rehype-slug": ["rehype-slug@6.0.0", "", { "dependencies": { "@types/hast": "^3.0.0", "github-slugger": "^2.0.0", "hast-util-heading-rank": "^3.0.0", "hast-util-to-string": "^3.0.0", "unist-util-visit": "^5.0.0" } }, "sha512-lWyvf/jwu+oS5+hL5eClVd3hNdmwM1kAC0BUvEGD19pajQMIzcNUd/k9GsfQ+FfECvX+JE+e9/btsKH0EjJT6A=="], + + "remark-directive": ["remark-directive@3.0.1", "", { "dependencies": { "@types/mdast": "^4.0.0", "mdast-util-directive": "^3.0.0", "micromark-extension-directive": "^3.0.0", "unified": "^11.0.0" } }, "sha512-gwglrEQEZcZYgVyG1tQuA+h58EZfq5CSULw7J90AFuCTyib1thgHPoqQ+h9iFvU6R+vnZ5oNFQR5QKgGpk741A=="], + + "remark-frontmatter": ["remark-frontmatter@5.0.0", "", { "dependencies": { "@types/mdast": "^4.0.0", "mdast-util-frontmatter": "^2.0.0", "micromark-extension-frontmatter": "^2.0.0", "unified": "^11.0.0" } }, "sha512-XTFYvNASMe5iPN0719nPrdItC9aU0ssC4v14mH1BCi1u0n1gAocqcujWUrByftZTbLhRtiKRyjYTSIOcr69UVQ=="], + + "remark-gfm": ["remark-gfm@4.0.1", "", { "dependencies": { "@types/mdast": "^4.0.0", "mdast-util-gfm": "^3.0.0", "micromark-extension-gfm": "^3.0.0", "remark-parse": "^11.0.0", "remark-stringify": "^11.0.0", "unified": "^11.0.0" } }, "sha512-1quofZ2RQ9EWdeN34S79+KExV1764+wCUGop5CPL1WGdD0ocPpu91lzPGbwWMECpEpd42kJGQwzRfyov9j4yNg=="], + + "remark-mdx": ["remark-mdx@3.1.0", "", { "dependencies": { "mdast-util-mdx": "^3.0.0", "micromark-extension-mdxjs": "^3.0.0" } }, "sha512-Ngl/H3YXyBV9RcRNdlYsZujAmhsxwzxpDzpDEhFBVAGthS4GDgnctpDjgFl/ULx5UEDzqtW1cyBSNKqYYrqLBA=="], + + "remark-mdx-frontmatter": ["remark-mdx-frontmatter@5.2.0", "", { "dependencies": { "@types/mdast": "^4.0.0", "estree-util-value-to-estree": "^3.0.0", "toml": "^3.0.0", "unified": "^11.0.0", "unist-util-mdx-define": "^1.0.0", "yaml": "^2.0.0" } }, "sha512-U/hjUYTkQqNjjMRYyilJgLXSPF65qbLPdoESOkXyrwz2tVyhAnm4GUKhfXqOOS9W34M3545xEMq+aMpHgVjEeQ=="], + + "remark-parse": ["remark-parse@11.0.0", "", { "dependencies": { "@types/mdast": "^4.0.0", "mdast-util-from-markdown": "^2.0.0", "micromark-util-types": "^2.0.0", "unified": "^11.0.0" } }, "sha512-FCxlKLNGknS5ba/1lmpYijMUzX2esxW5xQqjWxw2eHFfS2MSdaHVINFmhjo+qN1WhZhNimq0dZATN9pH0IDrpA=="], + + "remark-rehype": ["remark-rehype@11.1.2", "", { "dependencies": { "@types/hast": "^3.0.0", "@types/mdast": "^4.0.0", "mdast-util-to-hast": "^13.0.0", "unified": "^11.0.0", "vfile": "^6.0.0" } }, "sha512-Dh7l57ianaEoIpzbp0PC9UKAdCSVklD8E5Rpw7ETfbTl3FqcOOgq5q2LVDhgGCkaBv7p24JXikPdvhhmHvKMsw=="], + + "remark-stringify": ["remark-stringify@11.0.0", "", { "dependencies": { "@types/mdast": "^4.0.0", "mdast-util-to-markdown": "^2.0.0", "unified": "^11.0.0" } }, "sha512-1OSmLd3awB/t8qdoEOMazZkNsfVTeY4fTsgzcQFdXNq8ToTN4ZGwrMnlda4K6smTFKD+GRV6O48i6Z4iKgPPpw=="], + + "require-like": ["require-like@0.1.2", "", {}, "sha512-oyrU88skkMtDdauHDuKVrgR+zuItqr6/c//FXzvmxRGMexSDc6hNvJInGW3LL46n+8b50RykrvwSUIIQH2LQ5A=="], + + "restore-cursor": ["restore-cursor@4.0.0", "", { "dependencies": { "onetime": "^5.1.0", "signal-exit": "^3.0.2" } }, "sha512-I9fPXU9geO9bHOt9pHHOhOkYerIMsmVaWB0rA2AI9ERh/+x/i7MV5HKBNrg+ljO5eoPVgCcnFuRjJ9uH6I/3eg=="], + + "reusify": ["reusify@1.1.0", "", {}, "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw=="], + + "robust-predicates": ["robust-predicates@3.0.2", "", {}, "sha512-IXgzBWvWQwE6PrDI05OvmXUIruQTcoMDzRsOd5CDvHCVLcLHMTSYvOK5Cm46kWqlV3yAbuSpBZdJ5oP5OUoStg=="], + + "rollup": ["rollup@4.45.1", "", { "dependencies": { "@types/estree": "1.0.8" }, "optionalDependencies": { "@rollup/rollup-android-arm-eabi": "4.45.1", "@rollup/rollup-android-arm64": "4.45.1", "@rollup/rollup-darwin-arm64": "4.45.1", "@rollup/rollup-darwin-x64": "4.45.1", "@rollup/rollup-freebsd-arm64": "4.45.1", "@rollup/rollup-freebsd-x64": "4.45.1", "@rollup/rollup-linux-arm-gnueabihf": "4.45.1", "@rollup/rollup-linux-arm-musleabihf": "4.45.1", "@rollup/rollup-linux-arm64-gnu": "4.45.1", "@rollup/rollup-linux-arm64-musl": "4.45.1", "@rollup/rollup-linux-loongarch64-gnu": "4.45.1", "@rollup/rollup-linux-powerpc64le-gnu": "4.45.1", "@rollup/rollup-linux-riscv64-gnu": "4.45.1", "@rollup/rollup-linux-riscv64-musl": "4.45.1", "@rollup/rollup-linux-s390x-gnu": "4.45.1", "@rollup/rollup-linux-x64-gnu": "4.45.1", "@rollup/rollup-linux-x64-musl": "4.45.1", "@rollup/rollup-win32-arm64-msvc": "4.45.1", "@rollup/rollup-win32-ia32-msvc": "4.45.1", "@rollup/rollup-win32-x64-msvc": "4.45.1", "fsevents": "~2.3.2" }, "bin": { "rollup": "dist/bin/rollup" } }, "sha512-4iya7Jb76fVpQyLoiVpzUrsjQ12r3dM7fIVz+4NwoYvZOShknRmiv+iu9CClZml5ZLGb0XMcYLutK6w9tgxHDw=="], + + "roughjs": ["roughjs@4.6.6", "", { "dependencies": { "hachure-fill": "^0.5.2", "path-data-parser": "^0.1.0", "points-on-curve": "^0.2.0", "points-on-path": "^0.2.1" } }, "sha512-ZUz/69+SYpFN/g/lUlo2FXcIjRkSu3nDarreVdGGndHEBJ6cXPdKguS8JGxwj5HA5xIbVKSmLgr5b3AWxtRfvQ=="], + + "run-parallel": ["run-parallel@1.2.0", "", { "dependencies": { "queue-microtask": "^1.2.2" } }, "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA=="], + + "rw": ["rw@1.3.3", "", {}, "sha512-PdhdWy89SiZogBLaw42zdeqtRJ//zFd2PgQavcICDUgJT5oW10QCRKbJ6bg4r0/UY2M6BWd5tkxuGFRvCkgfHQ=="], + + "safe-buffer": ["safe-buffer@5.2.1", "", {}, "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="], + + "safer-buffer": ["safer-buffer@2.1.2", "", {}, "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="], + + "scheduler": ["scheduler@0.26.0", "", {}, "sha512-NlHwttCI/l5gCPR3D1nNXtWABUmBwvZpEQiD4IXSbIDq8BzLIK/7Ir5gTFSGZDUu37K5cMNp0hFtzO38sC7gWA=="], + + "semver": ["semver@6.3.1", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA=="], + + "send": ["send@0.19.0", "", { "dependencies": { "debug": "2.6.9", "depd": "2.0.0", "destroy": "1.2.0", "encodeurl": "~1.0.2", "escape-html": "~1.0.3", "etag": "~1.8.1", "fresh": "0.5.2", "http-errors": "2.0.0", "mime": "1.6.0", "ms": "2.1.3", "on-finished": "2.4.1", "range-parser": "~1.2.1", "statuses": "2.0.1" } }, "sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw=="], + + "serve-static": ["serve-static@1.16.2", "", { "dependencies": { "encodeurl": "~2.0.0", "escape-html": "~1.0.3", "parseurl": "~1.3.3", "send": "0.19.0" } }, "sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw=="], + + "set-cookie-parser": ["set-cookie-parser@2.7.1", "", {}, "sha512-IOc8uWeOZgnb3ptbCURJWNjWUPcO3ZnTTdzsurqERrP6nPyv+paC55vJM0LpOlT2ne+Ix+9+CRG1MNLlyZ4GjQ=="], + + "setprototypeof": ["setprototypeof@1.2.0", "", {}, "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw=="], + + "shebang-command": ["shebang-command@2.0.0", "", { "dependencies": { "shebang-regex": "^3.0.0" } }, "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA=="], + + "shebang-regex": ["shebang-regex@3.0.0", "", {}, "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A=="], + + "shiki": ["shiki@1.29.2", "", { "dependencies": { "@shikijs/core": "1.29.2", "@shikijs/engine-javascript": "1.29.2", "@shikijs/engine-oniguruma": "1.29.2", "@shikijs/langs": "1.29.2", "@shikijs/themes": "1.29.2", "@shikijs/types": "1.29.2", "@shikijs/vscode-textmate": "^10.0.1", "@types/hast": "^3.0.4" } }, "sha512-njXuliz/cP+67jU2hukkxCNuH1yUi4QfdZZY+sMr5PPrIyXSu5iTb/qYC4BiWWB0vZ+7TbdvYUCeL23zpwCfbg=="], + + "signal-exit": ["signal-exit@4.1.0", "", {}, "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw=="], + + "sisteransi": ["sisteransi@1.0.5", "", {}, "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg=="], + + "slash": ["slash@5.1.0", "", {}, "sha512-ZA6oR3T/pEyuqwMgAKT0/hAv8oAXckzbkmR0UkUosQ+Mc4RxGoJkRmwHgHufaenlyAgE1Mxgpdcrf75y6XcnDg=="], + + "source-map": ["source-map@0.7.4", "", {}, "sha512-l3BikUxvPOcn5E74dZiq5BGsTb5yEwhaTSzccU6t4sDOH8NWJCstKO5QT2CvtFoK6F0saL7p9xHAqHOlCPJygA=="], + + "source-map-js": ["source-map-js@1.2.1", "", {}, "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA=="], + + "space-separated-tokens": ["space-separated-tokens@2.0.2", "", {}, "sha512-PEGlAwrG8yXGXRjW32fGbg66JAlOAwbObuqVoJpv/mRgoWDQfgH1wDPvtzWyUSNAXBGSk8h755YDbbcEy3SH2Q=="], + + "statuses": ["statuses@2.0.1", "", {}, "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ=="], + + "stdin-discarder": ["stdin-discarder@0.1.0", "", { "dependencies": { "bl": "^5.0.0" } }, "sha512-xhV7w8S+bUwlPTb4bAOUQhv8/cSS5offJuX8GQGq32ONF0ZtDWKfkdomM3HMRA+LhX6um/FZ0COqlwsjD53LeQ=="], + + "string-width": ["string-width@6.1.0", "", { "dependencies": { "eastasianwidth": "^0.2.0", "emoji-regex": "^10.2.1", "strip-ansi": "^7.0.1" } }, "sha512-k01swCJAgQmuADB0YIc+7TuatfNvTBVOoaUWJjTB9R4VJzR5vNWzf5t42ESVZFPS8xTySF7CAdV4t/aaIm3UnQ=="], + + "string-width-cjs": ["string-width@4.2.3", "", { "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", "strip-ansi": "^6.0.1" } }, "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g=="], + + "string_decoder": ["string_decoder@1.3.0", "", { "dependencies": { "safe-buffer": "~5.2.0" } }, "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA=="], + + "stringify-entities": ["stringify-entities@4.0.4", "", { "dependencies": { "character-entities-html4": "^2.0.0", "character-entities-legacy": "^3.0.0" } }, "sha512-IwfBptatlO+QCJUo19AqvrPNqlVMpW9YEL2LIVY+Rpv2qsjCGxaDLNRgeGsQWJhfItebuJhsGSLjaBbNSQ+ieg=="], + + "strip-ansi": ["strip-ansi@7.1.0", "", { "dependencies": { "ansi-regex": "^6.0.1" } }, "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ=="], + + "strip-ansi-cjs": ["strip-ansi@6.0.1", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="], + + "strip-final-newline": ["strip-final-newline@2.0.0", "", {}, "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA=="], + + "style-to-js": ["style-to-js@1.1.17", "", { "dependencies": { "style-to-object": "1.0.9" } }, "sha512-xQcBGDxJb6jjFCTzvQtfiPn6YvvP2O8U1MDIPNfJQlWMYfktPy+iGsHE7cssjs7y84d9fQaK4UF3RIJaAHSoYA=="], + + "style-to-object": ["style-to-object@1.0.9", "", { "dependencies": { "inline-style-parser": "0.2.4" } }, "sha512-G4qppLgKu/k6FwRpHiGiKPaPTFcG3g4wNVX/Qsfu+RqQM30E7Tyu/TEgxcL9PNLF5pdRLwQdE3YKKf+KF2Dzlw=="], + + "stylis": ["stylis@4.3.6", "", {}, "sha512-yQ3rwFWRfwNUY7H5vpU0wfdkNSnvnJinhF9830Swlaxl03zsOjCfmX0ugac+3LtK0lYSgwL/KXc8oYL3mG4YFQ=="], + + "tabbable": ["tabbable@6.2.0", "", {}, "sha512-Cat63mxsVJlzYvN51JmVXIgNoUokrIaT2zLclCXjRd8boZ0004U4KCs/sToJ75C6sdlByWxpYnb5Boif1VSFew=="], + + "tailwindcss": ["tailwindcss@4.0.7", "", {}, "sha512-yH5bPPyapavo7L+547h3c4jcBXcrKwybQRjwdEIVAd9iXRvy/3T1CC6XSQEgZtRySjKfqvo3Cc0ZF1DTheuIdA=="], + + "tapable": ["tapable@2.2.2", "", {}, "sha512-Re10+NauLTMCudc7T5WLFLAwDhQ0JWdrMK+9B2M8zR5hRExKmsRDCBA7/aV/pNJFltmBFO5BAMlQFi/vq3nKOg=="], + + "tinyexec": ["tinyexec@1.0.1", "", {}, "sha512-5uC6DDlmeqiOwCPmK9jMSdOuZTh8bU39Ys6yidB+UTt5hfZUPGAypSgFRiEp+jbi9qH40BLDvy85jIU88wKSqw=="], + + "tinyglobby": ["tinyglobby@0.2.14", "", { "dependencies": { "fdir": "^6.4.4", "picomatch": "^4.0.2" } }, "sha512-tX5e7OM1HnYr2+a2C/4V0htOcSQcoSTH9KgJnVvNm5zm/cyEWKJ7j7YutsH9CxMdtOkkLFy2AHrMci9IM8IPZQ=="], + + "to-regex-range": ["to-regex-range@5.0.1", "", { "dependencies": { "is-number": "^7.0.0" } }, "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ=="], + + "toidentifier": ["toidentifier@1.0.1", "", {}, "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA=="], + + "toml": ["toml@3.0.0", "", {}, "sha512-y/mWCZinnvxjTKYhJ+pYxwD0mRLVvOtdS2Awbgxln6iEnt4rk0yBxeSBHkGJcPucRiG0e55mwWp+g/05rsrd6w=="], + + "trim-lines": ["trim-lines@3.0.1", "", {}, "sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg=="], + + "trough": ["trough@2.2.0", "", {}, "sha512-tmMpK00BjZiUyVyvrBK7knerNgmgvcV/KLVyuma/SC+TQN167GrMRciANTz09+k3zW8L8t60jWO1GpfkZdjTaw=="], + + "ts-dedent": ["ts-dedent@2.2.0", "", {}, "sha512-q5W7tVM71e2xjHZTlgfTDoPF/SmqKG5hddq9SzR49CH2hayqRKJtQ4mtRlSxKaJlR/+9rEM+mnBHf7I2/BQcpQ=="], + + "tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="], + + "twoslash": ["twoslash@0.2.12", "", { "dependencies": { "@typescript/vfs": "^1.6.0", "twoslash-protocol": "0.2.12" }, "peerDependencies": { "typescript": "*" } }, "sha512-tEHPASMqi7kqwfJbkk7hc/4EhlrKCSLcur+TcvYki3vhIfaRMXnXjaYFgXpoZRbT6GdprD4tGuVBEmTpUgLBsw=="], + + "twoslash-protocol": ["twoslash-protocol@0.2.12", "", {}, "sha512-5qZLXVYfZ9ABdjqbvPc4RWMr7PrpPaaDSeaYY55vl/w1j6H6kzsWK/urAEIXlzYlyrFmyz1UbwIt+AA0ck+wbg=="], + + "typescript": ["typescript@5.8.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ=="], + + "ua-parser-js": ["ua-parser-js@1.0.40", "", { "bin": { "ua-parser-js": "script/cli.js" } }, "sha512-z6PJ8Lml+v3ichVojCiB8toQJBuwR42ySM4ezjXIqXK3M0HczmKQ3LF4rhU55PfD99KEEXQG6yb7iOMyvYuHew=="], + + "ufo": ["ufo@1.6.1", "", {}, "sha512-9a4/uxlTWJ4+a5i0ooc1rU7C7YOw3wT+UGqdeNNHWnOF9qcMBgLRS+4IYUqbczewFx4mLEig6gawh7X6mFlEkA=="], + + "undici-types": ["undici-types@7.8.0", "", {}, "sha512-9UJ2xGDvQ43tYyVMpuHlsgApydB8ZKfVYTsLDhXkFL/6gfkp+U8xTGdh8pMJv1SpZna0zxG1DwsKZsreLbXBxw=="], + + "unicorn-magic": ["unicorn-magic@0.3.0", "", {}, "sha512-+QBBXBCvifc56fsbuxZQ6Sic3wqqc3WWaqxs58gvJrcOuN83HGTCwz3oS5phzU9LthRNE9VrJCFCLUgHeeFnfA=="], + + "unified": ["unified@11.0.5", "", { "dependencies": { "@types/unist": "^3.0.0", "bail": "^2.0.0", "devlop": "^1.0.0", "extend": "^3.0.0", "is-plain-obj": "^4.0.0", "trough": "^2.0.0", "vfile": "^6.0.0" } }, "sha512-xKvGhPWw3k84Qjh8bI3ZeJjqnyadK+GEFtazSfZv/rKeTkTjOJho6mFqh2SM96iIcZokxiOpg78GazTSg8+KHA=="], + + "unist-util-find-after": ["unist-util-find-after@5.0.0", "", { "dependencies": { "@types/unist": "^3.0.0", "unist-util-is": "^6.0.0" } }, "sha512-amQa0Ep2m6hE2g72AugUItjbuM8X8cGQnFoHk0pGfrFeT9GZhzN5SW8nRsiGKK7Aif4CrACPENkA6P/Lw6fHGQ=="], + + "unist-util-is": ["unist-util-is@6.0.0", "", { "dependencies": { "@types/unist": "^3.0.0" } }, "sha512-2qCTHimwdxLfz+YzdGfkqNlH0tLi9xjTnHddPmJwtIG9MGsdbutfTc4P+haPD7l7Cjxf/WZj+we5qfVPvvxfYw=="], + + "unist-util-mdx-define": ["unist-util-mdx-define@1.1.2", "", { "dependencies": { "@types/estree": "^1.0.0", "@types/hast": "^3.0.0", "@types/mdast": "^4.0.0", "estree-util-is-identifier-name": "^3.0.0", "estree-util-scope": "^1.0.0", "estree-walker": "^3.0.0", "vfile": "^6.0.0" } }, "sha512-9ncH7i7TN5Xn7/tzX5bE3rXgz1X/u877gYVAUB3mLeTKYJmQHmqKTDBi6BTGXV7AeolBCI9ErcVsOt2qryoD0g=="], + + "unist-util-position": ["unist-util-position@5.0.0", "", { "dependencies": { "@types/unist": "^3.0.0" } }, "sha512-fucsC7HjXvkB5R3kTCO7kUjRdrS0BJt3M/FPxmHMBOm8JQi2BsHAHFsy27E0EolP8rp0NzXsJ+jNPyDWvOJZPA=="], + + "unist-util-position-from-estree": ["unist-util-position-from-estree@2.0.0", "", { "dependencies": { "@types/unist": "^3.0.0" } }, "sha512-KaFVRjoqLyF6YXCbVLNad/eS4+OfPQQn2yOd7zF/h5T/CSL2v8NpN6a5TPvtbXthAGw5nG+PuTtq+DdIZr+cRQ=="], + + "unist-util-remove-position": ["unist-util-remove-position@5.0.0", "", { "dependencies": { "@types/unist": "^3.0.0", "unist-util-visit": "^5.0.0" } }, "sha512-Hp5Kh3wLxv0PHj9m2yZhhLt58KzPtEYKQQ4yxfYFEO7EvHwzyDYnduhHnY1mDxoqr7VUwVuHXk9RXKIiYS1N8Q=="], + + "unist-util-stringify-position": ["unist-util-stringify-position@4.0.0", "", { "dependencies": { "@types/unist": "^3.0.0" } }, "sha512-0ASV06AAoKCDkS2+xw5RXJywruurpbC4JZSm7nr7MOt1ojAzvyyaO+UxZf18j8FCF6kmzCZKcAgN/yu2gm2XgQ=="], + + "unist-util-visit": ["unist-util-visit@5.0.0", "", { "dependencies": { "@types/unist": "^3.0.0", "unist-util-is": "^6.0.0", "unist-util-visit-parents": "^6.0.0" } }, "sha512-MR04uvD+07cwl/yhVuVWAtw+3GOR/knlL55Nd/wAdblk27GCVt3lqpTivy/tkJcZoNPzTwS1Y+KMojlLDhoTzg=="], + + "unist-util-visit-parents": ["unist-util-visit-parents@6.0.1", "", { "dependencies": { "@types/unist": "^3.0.0", "unist-util-is": "^6.0.0" } }, "sha512-L/PqWzfTP9lzzEa6CKs0k2nARxTdZduw3zyh8d2NVBnsyvHjSX4TWse388YrrQKbvI8w20fGjGlhgT96WwKykw=="], + + "universalify": ["universalify@2.0.1", "", {}, "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw=="], + + "update-browserslist-db": ["update-browserslist-db@1.1.3", "", { "dependencies": { "escalade": "^3.2.0", "picocolors": "^1.1.1" }, "peerDependencies": { "browserslist": ">= 4.21.0" }, "bin": { "update-browserslist-db": "cli.js" } }, "sha512-UxhIZQ+QInVdunkDAaiazvvT/+fXL5Osr0JZlJulepYu6Jd7qJtDZjlur0emRlT71EN3ScPoE7gvsuIKKNavKw=="], + + "use-callback-ref": ["use-callback-ref@1.3.3", "", { "dependencies": { "tslib": "^2.0.0" }, "peerDependencies": { "@types/react": "*", "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-jQL3lRnocaFtu3V00JToYz/4QkNWswxijDaCVNZRiRTO3HQDLsdu1ZtmIUvV4yPp+rvWm5j0y0TG/S61cuijTg=="], + + "use-sidecar": ["use-sidecar@1.1.3", "", { "dependencies": { "detect-node-es": "^1.1.0", "tslib": "^2.0.0" }, "peerDependencies": { "@types/react": "*", "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-Fedw0aZvkhynoPYlA5WXrMCAMm+nSWdZt6lzJQ7Ok8S6Q+VsHmHpRWndVRJ8Be0ZbkfPc5LRYH+5XrzXcEeLRQ=="], + + "use-sync-external-store": ["use-sync-external-store@1.5.0", "", { "peerDependencies": { "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-Rb46I4cGGVBmjamjphe8L/UnvJD+uPPtTkNvX5mZgqdbavhI4EbgIWJiIHXJ8bc/i9EQGPRh4DwEURJ552Do0A=="], + + "util-deprecate": ["util-deprecate@1.0.2", "", {}, "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw=="], + + "uuid": ["uuid@11.1.0", "", { "bin": { "uuid": "dist/esm/bin/uuid" } }, "sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A=="], + + "vary": ["vary@1.1.2", "", {}, "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg=="], + + "vfile": ["vfile@6.0.3", "", { "dependencies": { "@types/unist": "^3.0.0", "vfile-message": "^4.0.0" } }, "sha512-KzIbH/9tXat2u30jf+smMwFCsno4wHVdNmzFyL+T/L3UGqqk6JKfVqOFOZEpZSHADH1k40ab6NUIXZq422ov3Q=="], + + "vfile-location": ["vfile-location@5.0.3", "", { "dependencies": { "@types/unist": "^3.0.0", "vfile": "^6.0.0" } }, "sha512-5yXvWDEgqeiYiBe1lbxYF7UMAIm/IcopxMHrMQDq3nvKcjPKIhZklUKL+AE7J7uApI4kwe2snsK+eI6UTj9EHg=="], + + "vfile-message": ["vfile-message@4.0.2", "", { "dependencies": { "@types/unist": "^3.0.0", "unist-util-stringify-position": "^4.0.0" } }, "sha512-jRDZ1IMLttGj41KcZvlrYAaI3CfqpLpfpf+Mfig13viT6NKvRzWZ+lXz0Y5D60w6uJIBAOGq9mSHf0gktF0duw=="], + + "vite": ["vite@6.3.5", "", { "dependencies": { "esbuild": "^0.25.0", "fdir": "^6.4.4", "picomatch": "^4.0.2", "postcss": "^8.5.3", "rollup": "^4.34.9", "tinyglobby": "^0.2.13" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0", "jiti": ">=1.21.0", "less": "*", "lightningcss": "^1.21.0", "sass": "*", "sass-embedded": "*", "stylus": "*", "sugarss": "*", "terser": "^5.16.0", "tsx": "^4.8.1", "yaml": "^2.4.2" }, "optionalPeers": ["@types/node", "jiti", "less", "lightningcss", "sass", "sass-embedded", "stylus", "sugarss", "terser", "tsx", "yaml"], "bin": { "vite": "bin/vite.js" } }, "sha512-cZn6NDFE7wdTpINgs++ZJ4N49W2vRp8LCKrn3Ob1kYNtOo21vfDoaV5GzBfLU4MovSAB8uNRm4jgzVQZ+mBzPQ=="], + + "vite-node": ["vite-node@3.2.4", "", { "dependencies": { "cac": "^6.7.14", "debug": "^4.4.1", "es-module-lexer": "^1.7.0", "pathe": "^2.0.3", "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0" }, "bin": { "vite-node": "vite-node.mjs" } }, "sha512-EbKSKh+bh1E1IFxeO0pg1n4dvoOTt0UDiXMd/qn++r98+jPO1xtJilvXldeuQ8giIB5IkpjCgMleHMNEsGH6pg=="], + + "vocs": ["vocs@1.0.13", "", { "dependencies": { "@floating-ui/react": "^0.27.4", "@hono/node-server": "^1.13.8", "@mdx-js/react": "^3.1.0", "@mdx-js/rollup": "^3.1.0", "@noble/hashes": "^1.7.1", "@radix-ui/colors": "^3.0.0", "@radix-ui/react-accordion": "^1.2.3", "@radix-ui/react-dialog": "^1.1.6", "@radix-ui/react-icons": "^1.3.2", "@radix-ui/react-label": "^2.1.2", "@radix-ui/react-navigation-menu": "^1.2.5", "@radix-ui/react-popover": "^1.1.6", "@radix-ui/react-tabs": "^1.1.3", "@shikijs/rehype": "^1", "@shikijs/transformers": "^1", "@shikijs/twoslash": "^1", "@tailwindcss/vite": "4.0.7", "@vanilla-extract/css": "^1.17.1", "@vanilla-extract/dynamic": "^2.1.2", "@vanilla-extract/vite-plugin": "^5.0.1", "@vitejs/plugin-react": "^4.3.4", "autoprefixer": "^10.4.20", "cac": "^6.7.14", "chroma-js": "^3.1.2", "clsx": "^2.1.1", "compression": "^1.8.0", "create-vocs": "^1.0.0-alpha.5", "cross-spawn": "^7.0.6", "fs-extra": "^11.3.0", "globby": "^14.1.0", "hastscript": "^8.0.0", "hono": "^4.7.1", "mark.js": "^8.11.1", "mdast-util-directive": "^3.1.0", "mdast-util-from-markdown": "^2.0.2", "mdast-util-frontmatter": "^2.0.1", "mdast-util-gfm": "^3.1.0", "mdast-util-mdx": "^3.0.0", "mdast-util-mdx-jsx": "^3.2.0", "mdast-util-to-hast": "^13.2.0", "mdast-util-to-markdown": "^2.1.2", "minimatch": "^9.0.5", "minisearch": "^6.3.0", "ora": "^7.0.1", "p-limit": "^5.0.0", "playwright": "^1.52.0", "postcss": "^8.5.2", "radix-ui": "^1.1.3", "react-intersection-observer": "^9.15.1", "react-router": "^7.2.0", "rehype-autolink-headings": "^7.1.0", "rehype-class-names": "^2.0.0", "rehype-mermaid": "^3.0.0", "rehype-slug": "^6.0.0", "remark-directive": "^3.0.1", "remark-frontmatter": "^5.0.0", "remark-gfm": "^4.0.1", "remark-mdx": "^3.1.0", "remark-mdx-frontmatter": "^5.0.0", "remark-parse": "^11.0.0", "serve-static": "^1.16.2", "shiki": "^1", "toml": "^3.0.0", "twoslash": "~0.2.12", "ua-parser-js": "^1.0.40", "unified": "^11.0.5", "unist-util-visit": "^5.0.0", "vite": "^6.1.0" }, "peerDependencies": { "react": "^19", "react-dom": "^19" }, "bin": { "vocs": "_lib/cli/index.js" } }, "sha512-V/ogXG5xw7jMFXI2Wv0d0ZdCeeT5jzaX0PKdRKcqhnd21UtLZrqa5pKZkStNIZyVpvfsLW0WB7wjB4iBOpueiw=="], + + "vscode-jsonrpc": ["vscode-jsonrpc@8.2.0", "", {}, "sha512-C+r0eKJUIfiDIfwJhria30+TYWPtuHJXHtI7J0YlOmKAo7ogxP20T0zxB7HZQIFhIyvoBPwWskjxrvAtfjyZfA=="], + + "vscode-languageserver": ["vscode-languageserver@9.0.1", "", { "dependencies": { "vscode-languageserver-protocol": "3.17.5" }, "bin": { "installServerIntoExtension": "bin/installServerIntoExtension" } }, "sha512-woByF3PDpkHFUreUa7Hos7+pUWdeWMXRd26+ZX2A8cFx6v/JPTtd4/uN0/jB6XQHYaOlHbio03NTHCqrgG5n7g=="], + + "vscode-languageserver-protocol": ["vscode-languageserver-protocol@3.17.5", "", { "dependencies": { "vscode-jsonrpc": "8.2.0", "vscode-languageserver-types": "3.17.5" } }, "sha512-mb1bvRJN8SVznADSGWM9u/b07H7Ecg0I3OgXDuLdn307rl/J3A9YD6/eYOssqhecL27hK1IPZAsaqh00i/Jljg=="], + + "vscode-languageserver-textdocument": ["vscode-languageserver-textdocument@1.0.12", "", {}, "sha512-cxWNPesCnQCcMPeenjKKsOCKQZ/L6Tv19DTRIGuLWe32lyzWhihGVJ/rcckZXJxfdKCFvRLS3fpBIsV/ZGX4zA=="], + + "vscode-languageserver-types": ["vscode-languageserver-types@3.17.5", "", {}, "sha512-Ld1VelNuX9pdF39h2Hgaeb5hEZM2Z3jUrrMgWQAu82jMtZp7p3vJT3BzToKtZI7NgQssZje5o0zryOrhQvzQAg=="], + + "vscode-uri": ["vscode-uri@3.0.8", "", {}, "sha512-AyFQ0EVmsOZOlAnxoFOGOq1SQDWAB7C6aqMGS23svWAllfOaxbuFvcT8D1i8z3Gyn8fraVeZNNmN6e9bxxXkKw=="], + + "web-namespaces": ["web-namespaces@2.0.1", "", {}, "sha512-bKr1DkiNa2krS7qxNtdrtHAmzuYGFQLiQ13TsorsdT6ULTkPLKuu5+GsFpDlg6JFjUTwX2DyhMPG2be8uPrqsQ=="], + + "which": ["which@2.0.2", "", { "dependencies": { "isexe": "^2.0.0" }, "bin": { "node-which": "./bin/node-which" } }, "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA=="], + + "wrap-ansi": ["wrap-ansi@8.1.0", "", { "dependencies": { "ansi-styles": "^6.1.0", "string-width": "^5.0.1", "strip-ansi": "^7.0.1" } }, "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ=="], + + "wrap-ansi-cjs": ["wrap-ansi@7.0.0", "", { "dependencies": { "ansi-styles": "^4.0.0", "string-width": "^4.1.0", "strip-ansi": "^6.0.0" } }, "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q=="], + + "yallist": ["yallist@3.1.1", "", {}, "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g=="], + + "yaml": ["yaml@2.8.0", "", { "bin": { "yaml": "bin.mjs" } }, "sha512-4lLa/EcQCB0cJkyts+FpIRx5G/llPxfP6VQU5KByHEhLxY3IJCH0f0Hy1MHI8sClTvsIb8qwRJ6R/ZdlDJ/leQ=="], + + "yocto-queue": ["yocto-queue@1.2.1", "", {}, "sha512-AyeEbWOu/TAXdxlV9wmGcR0+yh2j3vYPGOECcIj2S7MkrLyC7ne+oye2BKTItt0ii2PHk4cDy+95+LshzbXnGg=="], + + "zwitch": ["zwitch@2.0.4", "", {}, "sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A=="], + + "@babel/core/debug": ["debug@4.4.1", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ=="], + + "@babel/helper-compilation-targets/lru-cache": ["lru-cache@5.1.1", "", { "dependencies": { "yallist": "^3.0.2" } }, "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w=="], + + "@babel/traverse/debug": ["debug@4.4.1", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ=="], + + "@clack/prompts/is-unicode-supported": ["is-unicode-supported@1.3.0", "", { "bundled": true }, "sha512-43r2mRvz+8JRIKnWJ+3j8JtjRKZ6GmjzfaE/qiBJnikNnYv/6bagRJ1kUhNk8R5EX/GkobD+r+sfxCPJsiKBLQ=="], + + "@iconify/utils/debug": ["debug@4.4.1", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ=="], + + "@isaacs/cliui/string-width": ["string-width@5.1.2", "", { "dependencies": { "eastasianwidth": "^0.2.0", "emoji-regex": "^9.2.2", "strip-ansi": "^7.0.1" } }, "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA=="], + + "@rollup/pluginutils/estree-walker": ["estree-walker@2.0.2", "", {}, "sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w=="], + + "@typescript/vfs/debug": ["debug@4.4.1", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ=="], + + "@vanilla-extract/css/lru-cache": ["lru-cache@10.4.3", "", {}, "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ=="], + + "cytoscape-fcose/cose-base": ["cose-base@2.2.0", "", { "dependencies": { "layout-base": "^2.0.0" } }, "sha512-AzlgcsCbUMymkADOJtQm3wO9S3ltPfYOFD5033keQn9NJzIbtnZj+UdBJe7DYml/8TdbtHJW3j58SOnKhWY/5g=="], + + "d3-dsv/commander": ["commander@7.2.0", "", {}, "sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw=="], + + "d3-sankey/d3-array": ["d3-array@2.12.1", "", { "dependencies": { "internmap": "^1.0.0" } }, "sha512-B0ErZK/66mHtEsR1TkPEEkwdy+WDesimkM5gpZr5Dsg54BiTA5RXtYW5qTLIAcekaS9xfZrzBLF/OAkB3Qn1YQ=="], + + "d3-sankey/d3-shape": ["d3-shape@1.3.7", "", { "dependencies": { "d3-path": "1" } }, "sha512-EUkvKjqPFUAZyOlhY5gzCxCeI0Aep04LwIRpsZ/mLFelJiUfnK56jo5JMDSE7yyP2kLSb6LtF+S5chMk7uqPqw=="], + + "execa/signal-exit": ["signal-exit@3.0.7", "", {}, "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ=="], + + "hast-util-from-dom/hastscript": ["hastscript@9.0.1", "", { "dependencies": { "@types/hast": "^3.0.0", "comma-separated-tokens": "^2.0.0", "hast-util-parse-selector": "^4.0.0", "property-information": "^7.0.0", "space-separated-tokens": "^2.0.0" } }, "sha512-g7df9rMFX/SPi34tyGCyUBREQoKkapwdY/T04Qn9TDWfHhAYt4/I0gMVirzK5wEzeUqIjEB+LXC/ypb7Aqno5w=="], + + "hast-util-from-parse5/hastscript": ["hastscript@9.0.1", "", { "dependencies": { "@types/hast": "^3.0.0", "comma-separated-tokens": "^2.0.0", "hast-util-parse-selector": "^4.0.0", "property-information": "^7.0.0", "space-separated-tokens": "^2.0.0" } }, "sha512-g7df9rMFX/SPi34tyGCyUBREQoKkapwdY/T04Qn9TDWfHhAYt4/I0gMVirzK5wEzeUqIjEB+LXC/ypb7Aqno5w=="], + + "hast-util-from-parse5/property-information": ["property-information@7.1.0", "", {}, "sha512-TwEZ+X+yCJmYfL7TPUOcvBZ4QfoT5YenQiJuX//0th53DE6w0xxLEtfK3iyryQFddXuvkIk51EEgrJQ0WJkOmQ=="], + + "hast-util-select/property-information": ["property-information@7.1.0", "", {}, "sha512-TwEZ+X+yCJmYfL7TPUOcvBZ4QfoT5YenQiJuX//0th53DE6w0xxLEtfK3iyryQFddXuvkIk51EEgrJQ0WJkOmQ=="], + + "hast-util-to-estree/property-information": ["property-information@7.1.0", "", {}, "sha512-TwEZ+X+yCJmYfL7TPUOcvBZ4QfoT5YenQiJuX//0th53DE6w0xxLEtfK3iyryQFddXuvkIk51EEgrJQ0WJkOmQ=="], + + "hast-util-to-html/property-information": ["property-information@7.1.0", "", {}, "sha512-TwEZ+X+yCJmYfL7TPUOcvBZ4QfoT5YenQiJuX//0th53DE6w0xxLEtfK3iyryQFddXuvkIk51EEgrJQ0WJkOmQ=="], + + "hast-util-to-jsx-runtime/property-information": ["property-information@7.1.0", "", {}, "sha512-TwEZ+X+yCJmYfL7TPUOcvBZ4QfoT5YenQiJuX//0th53DE6w0xxLEtfK3iyryQFddXuvkIk51EEgrJQ0WJkOmQ=="], + + "local-pkg/pkg-types": ["pkg-types@2.2.0", "", { "dependencies": { "confbox": "^0.2.2", "exsolve": "^1.0.7", "pathe": "^2.0.3" } }, "sha512-2SM/GZGAEkPp3KWORxQZns4M+WSeXbC2HEvmOIJe3Cmiv6ieAJvdVhDldtHqM5J1Y7MrR1XhkBT/rMlhh9FdqQ=="], + + "micromark/debug": ["debug@4.4.1", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ=="], + + "micromatch/picomatch": ["picomatch@2.3.1", "", {}, "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA=="], + + "p-locate/p-limit": ["p-limit@3.1.0", "", { "dependencies": { "yocto-queue": "^0.1.0" } }, "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ=="], + + "parse-entities/@types/unist": ["@types/unist@2.0.11", "", {}, "sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA=="], + + "restore-cursor/signal-exit": ["signal-exit@3.0.7", "", {}, "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ=="], + + "rollup/fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="], + + "send/encodeurl": ["encodeurl@1.0.2", "", {}, "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w=="], + + "send/ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], + + "string-width-cjs/emoji-regex": ["emoji-regex@8.0.0", "", {}, "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="], + + "string-width-cjs/strip-ansi": ["strip-ansi@6.0.1", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="], + + "strip-ansi-cjs/ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="], + + "vite/fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="], + + "vite-node/debug": ["debug@4.4.1", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ=="], + + "vocs/minimatch": ["minimatch@9.0.5", "", { "dependencies": { "brace-expansion": "^2.0.1" } }, "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow=="], + + "wrap-ansi/string-width": ["string-width@5.1.2", "", { "dependencies": { "eastasianwidth": "^0.2.0", "emoji-regex": "^9.2.2", "strip-ansi": "^7.0.1" } }, "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA=="], + + "wrap-ansi-cjs/ansi-styles": ["ansi-styles@4.3.0", "", { "dependencies": { "color-convert": "^2.0.1" } }, "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg=="], + + "wrap-ansi-cjs/string-width": ["string-width@4.2.3", "", { "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", "strip-ansi": "^6.0.1" } }, "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g=="], + + "wrap-ansi-cjs/strip-ansi": ["strip-ansi@6.0.1", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="], + + "@babel/core/debug/ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], + + "@babel/traverse/debug/ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], + + "@iconify/utils/debug/ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], + + "@isaacs/cliui/string-width/emoji-regex": ["emoji-regex@9.2.2", "", {}, "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg=="], + + "@typescript/vfs/debug/ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], + + "cytoscape-fcose/cose-base/layout-base": ["layout-base@2.0.1", "", {}, "sha512-dp3s92+uNI1hWIpPGH3jK2kxE2lMjdXdr+DH8ynZHpd6PUlH6x6cbuXnoMmiNumznqaNO31xu9e79F0uuZ0JFg=="], + + "d3-sankey/d3-shape/d3-path": ["d3-path@1.0.9", "", {}, "sha512-VLaYcn81dtHVTjEHd8B+pbe9yHWpXKZUC87PzoFmsFrJqgFwDe/qxfp5MlfsfM1V5E/iVt0MmEbWQ7FVIXh/bg=="], + + "hast-util-from-dom/hastscript/property-information": ["property-information@7.1.0", "", {}, "sha512-TwEZ+X+yCJmYfL7TPUOcvBZ4QfoT5YenQiJuX//0th53DE6w0xxLEtfK3iyryQFddXuvkIk51EEgrJQ0WJkOmQ=="], + + "local-pkg/pkg-types/confbox": ["confbox@0.2.2", "", {}, "sha512-1NB+BKqhtNipMsov4xI/NnhCKp9XG9NamYp5PVm9klAT0fsrNPjaFICsCFhNhwZJKNh7zB/3q8qXz0E9oaMNtQ=="], + + "micromark/debug/ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], + + "p-locate/p-limit/yocto-queue": ["yocto-queue@0.1.0", "", {}, "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q=="], + + "string-width-cjs/strip-ansi/ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="], + + "vite-node/debug/ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], + + "wrap-ansi-cjs/string-width/emoji-regex": ["emoji-regex@8.0.0", "", {}, "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="], + + "wrap-ansi-cjs/strip-ansi/ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="], + + "wrap-ansi/string-width/emoji-regex": ["emoji-regex@9.2.2", "", {}, "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg=="], + } +} diff --git a/docs/vocs/bunfig.toml b/docs/vocs/bunfig.toml new file mode 100644 index 00000000000..a38b9b61752 --- /dev/null +++ b/docs/vocs/bunfig.toml @@ -0,0 +1,4 @@ +telemetry = false + +# ensures runtime is always bun regardless of shebang +run.bun = true diff --git a/docs/vocs/docs/components/SdkShowcase.tsx b/docs/vocs/docs/components/SdkShowcase.tsx new file mode 100644 index 00000000000..442d6676f4f --- /dev/null +++ b/docs/vocs/docs/components/SdkShowcase.tsx @@ -0,0 +1,86 @@ +interface SdkProject { + name: string + description: string + loc: string + githubUrl: string + logoUrl?: string + company: string +} + +const projects: SdkProject[] = [ + { + name: 'Base Node', + description: "Coinbase's L2 scaling solution node implementation", + loc: '~3K', + githubUrl: 'https://github.com/base/node-reth', + company: 'Coinbase' + }, + { + name: 'Bera Reth', + description: "Berachain's high-performance EVM node with custom features", + loc: '~1K', + githubUrl: 'https://github.com/berachain/bera-reth', + company: 'Berachain' + }, + { + name: 'Reth Gnosis', + description: "Gnosis Chain's xDai-compatible execution client", + loc: '~5K', + githubUrl: 'https://github.com/gnosischain/reth_gnosis', + company: 'Gnosis' + }, + { + name: 'Reth BSC', + description: "BNB Smart Chain execution client implementation", + loc: '~6K', + githubUrl: 'https://github.com/loocapro/reth-bsc', + company: 'Binance Smart Chain' + } +] + +export function SdkShowcase() { + return ( + + ) +} diff --git a/docs/vocs/docs/components/TrustedBy.tsx b/docs/vocs/docs/components/TrustedBy.tsx new file mode 100644 index 00000000000..41b78e8787a --- /dev/null +++ b/docs/vocs/docs/components/TrustedBy.tsx @@ -0,0 +1,47 @@ +interface TrustedCompany { + name: string + logoUrl: string +} + +const companies: TrustedCompany[] = [ + { + name: 'Flashbots', + logoUrl: '/flashbots.png' + }, + { + name: 'Coinbase', + logoUrl: '/coinbase.png' + }, + { + name: 'Alchemy', + logoUrl: '/alchemy.png' + }, + { + name: 'Succinct Labs', + logoUrl: '/succinct.png' + } +] + +export function TrustedBy() { + return ( +
+ {companies.map((company) => ( +
+ {/* Company Logo */} +
+ {`${company.name} +
+
+ ))} +
+ ) +} diff --git a/docs/vocs/docs/pages/cli/SUMMARY.mdx b/docs/vocs/docs/pages/cli/SUMMARY.mdx new file mode 100644 index 00000000000..7f7012f4c1e --- /dev/null +++ b/docs/vocs/docs/pages/cli/SUMMARY.mdx @@ -0,0 +1,44 @@ + - [`reth`](/cli/reth) + - [`reth node`](/cli/reth/node) + - [`reth init`](/cli/reth/init) + - [`reth init-state`](/cli/reth/init-state) + - [`reth import`](/cli/reth/import) + - [`reth import-era`](/cli/reth/import-era) + - [`reth export-era`](/cli/reth/export-era) + - [`reth dump-genesis`](/cli/reth/dump-genesis) + - [`reth db`](/cli/reth/db) + - [`reth db stats`](/cli/reth/db/stats) + - [`reth db list`](/cli/reth/db/list) + - [`reth db checksum`](/cli/reth/db/checksum) + - [`reth db diff`](/cli/reth/db/diff) + - [`reth db get`](/cli/reth/db/get) + - [`reth db get mdbx`](/cli/reth/db/get/mdbx) + - [`reth db get static-file`](/cli/reth/db/get/static-file) + - [`reth db drop`](/cli/reth/db/drop) + - [`reth db clear`](/cli/reth/db/clear) + - [`reth db clear mdbx`](/cli/reth/db/clear/mdbx) + - [`reth db clear static-file`](/cli/reth/db/clear/static-file) + - [`reth db repair-trie`](/cli/reth/db/repair-trie) + - [`reth db version`](/cli/reth/db/version) + - [`reth db path`](/cli/reth/db/path) + - [`reth download`](/cli/reth/download) + - [`reth stage`](/cli/reth/stage) + - [`reth stage run`](/cli/reth/stage/run) + - [`reth stage drop`](/cli/reth/stage/drop) + - [`reth stage dump`](/cli/reth/stage/dump) + - [`reth stage dump execution`](/cli/reth/stage/dump/execution) + - [`reth stage dump storage-hashing`](/cli/reth/stage/dump/storage-hashing) + - [`reth stage dump account-hashing`](/cli/reth/stage/dump/account-hashing) + - [`reth stage dump merkle`](/cli/reth/stage/dump/merkle) + - [`reth stage unwind`](/cli/reth/stage/unwind) + - [`reth stage unwind to-block`](/cli/reth/stage/unwind/to-block) + - [`reth stage unwind num-blocks`](/cli/reth/stage/unwind/num-blocks) + - [`reth p2p`](/cli/reth/p2p) + - [`reth p2p header`](/cli/reth/p2p/header) + - [`reth p2p body`](/cli/reth/p2p/body) + - [`reth p2p rlpx`](/cli/reth/p2p/rlpx) + - [`reth p2p rlpx ping`](/cli/reth/p2p/rlpx/ping) + - [`reth p2p bootnode`](/cli/reth/p2p/bootnode) + - [`reth config`](/cli/reth/config) + - [`reth prune`](/cli/reth/prune) + - [`reth re-execute`](/cli/reth/re-execute) \ No newline at end of file diff --git a/book/cli/cli.md b/docs/vocs/docs/pages/cli/cli.mdx similarity index 58% rename from book/cli/cli.md rename to docs/vocs/docs/pages/cli/cli.mdx index ef1a98af525..d7a02e2b738 100644 --- a/book/cli/cli.md +++ b/docs/vocs/docs/pages/cli/cli.mdx @@ -1,7 +1,9 @@ +import Summary from './SUMMARY.mdx'; + # CLI Reference -The Reth node is operated via the CLI by running the `reth node` command. To stop it, press `ctrl-c`. You may need to wait a bit as Reth tears down existing p2p connections or other cleanup tasks. +The Reth node is operated via the CLI by running the `reth node` command. To stop it, press `ctrl-c`. You may need to wait a bit as Reth tears down existing p2p connections or performs other cleanup tasks. However, Reth has more commands: -{{#include ./SUMMARY.md}} + diff --git a/book/cli/op-reth.md b/docs/vocs/docs/pages/cli/op-reth.md similarity index 100% rename from book/cli/op-reth.md rename to docs/vocs/docs/pages/cli/op-reth.md diff --git a/book/cli/reth.md b/docs/vocs/docs/pages/cli/reth.mdx similarity index 67% rename from book/cli/reth.md rename to docs/vocs/docs/pages/cli/reth.mdx index 8225d71b3b7..c35216d6b5c 100644 --- a/book/cli/reth.md +++ b/docs/vocs/docs/pages/cli/reth.mdx @@ -12,17 +12,17 @@ Commands: node Start the node init Initialize the database from a genesis file init-state Initialize the database from a state dump file - import This syncs RLP encoded blocks from a file + import This syncs RLP encoded blocks from a file or files import-era This syncs ERA encoded blocks from a directory + export-era Exports block to era1 files in a specified directory dump-genesis Dumps genesis block JSON configuration to stdout db Database debugging utilities download Download public node snapshots stage Manipulate individual stages p2p P2P Debugging utilities config Write config to stdout - debug Various debug routines - recover Scripts for node recovery prune Prune according to the configuration without any limits + re-execute Re-execute blocks in parallel to verify historical sync correctness help Print this message or the help of the given subcommand(s) Options: @@ -36,13 +36,13 @@ Logging: --log.stdout.format The format to use for logs written to stdout - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.stdout.filter The filter to use for logs written to stdout @@ -51,13 +51,13 @@ Logging: --log.file.format The format to use for logs written to the log file - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.file.filter The filter to use for logs written to the log file @@ -68,6 +68,11 @@ Logging: [default: /logs] + --log.file.name + The prefix name of the log files + + [default: reth.log] + --log.file.max-size The maximum size (in MB) of one log file @@ -89,13 +94,13 @@ Logging: --color Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting - [default: always] - Possible values: - always: Colors on - - auto: Colors on + - auto: Auto-detect - never: Colors off + [default: always] + Display: -v, --verbosity... Set the minimum log level. @@ -108,4 +113,37 @@ Display: -q, --quiet Silence all log output + +Tracing: + --tracing-otlp[=] + Enable `Opentelemetry` tracing export to an OTLP endpoint. + + If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + + Example: --tracing-otlp=http://collector:4318/v1/traces + + [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] + + --tracing-otlp-protocol + OTLP transport protocol to use for exporting traces. + + - `http`: expects endpoint path to end with `/v1/traces` - `grpc`: expects endpoint without a path + + Defaults to HTTP if not specified. + + Possible values: + - http: HTTP/Protobuf transport, port 4318, requires `/v1/traces` path + - grpc: gRPC transport, port 4317 + + [env: OTEL_EXPORTER_OTLP_PROTOCOL=] + [default: http] + + --tracing-otlp.filter + Set a filter directive for the OTLP tracer. This controls the verbosity of spans and events sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable. + + Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off + + Defaults to TRACE if not specified. + + [default: debug] ``` \ No newline at end of file diff --git a/book/cli/reth/config.md b/docs/vocs/docs/pages/cli/reth/config.mdx similarity index 66% rename from book/cli/reth/config.md rename to docs/vocs/docs/pages/cli/reth/config.mdx index 86384a169a1..6b3c9e4b657 100644 --- a/book/cli/reth/config.md +++ b/docs/vocs/docs/pages/cli/reth/config.mdx @@ -22,13 +22,13 @@ Logging: --log.stdout.format The format to use for logs written to stdout - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.stdout.filter The filter to use for logs written to stdout @@ -37,13 +37,13 @@ Logging: --log.file.format The format to use for logs written to the log file - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.file.filter The filter to use for logs written to the log file @@ -54,6 +54,11 @@ Logging: [default: /logs] + --log.file.name + The prefix name of the log files + + [default: reth.log] + --log.file.max-size The maximum size (in MB) of one log file @@ -75,13 +80,13 @@ Logging: --color Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting - [default: always] - Possible values: - always: Colors on - - auto: Colors on + - auto: Auto-detect - never: Colors off + [default: always] + Display: -v, --verbosity... Set the minimum log level. @@ -94,4 +99,37 @@ Display: -q, --quiet Silence all log output + +Tracing: + --tracing-otlp[=] + Enable `Opentelemetry` tracing export to an OTLP endpoint. + + If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + + Example: --tracing-otlp=http://collector:4318/v1/traces + + [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] + + --tracing-otlp-protocol + OTLP transport protocol to use for exporting traces. + + - `http`: expects endpoint path to end with `/v1/traces` - `grpc`: expects endpoint without a path + + Defaults to HTTP if not specified. + + Possible values: + - http: HTTP/Protobuf transport, port 4318, requires `/v1/traces` path + - grpc: gRPC transport, port 4317 + + [env: OTEL_EXPORTER_OTLP_PROTOCOL=] + [default: http] + + --tracing-otlp.filter + Set a filter directive for the OTLP tracer. This controls the verbosity of spans and events sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable. + + Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off + + Defaults to TRACE if not specified. + + [default: debug] ``` \ No newline at end of file diff --git a/book/cli/reth/db.md b/docs/vocs/docs/pages/cli/reth/db.mdx similarity index 67% rename from book/cli/reth/db.md rename to docs/vocs/docs/pages/cli/reth/db.mdx index e0079bf2616..feb902d4938 100644 --- a/book/cli/reth/db.md +++ b/docs/vocs/docs/pages/cli/reth/db.mdx @@ -9,16 +9,17 @@ $ reth db --help Usage: reth db [OPTIONS] Commands: - stats Lists all the tables, their entry count and their size - list Lists the contents of a table - checksum Calculates the content checksum of a table - diff Create a diff between two database tables or two entire databases - get Gets the content of a table for the given key - drop Deletes all database entries - clear Deletes all table entries - version Lists current and local database versions - path Returns the full database path - help Print this message or the help of the given subcommand(s) + stats Lists all the tables, their entry count and their size + list Lists the contents of a table + checksum Calculates the content checksum of a table + diff Create a diff between two database tables or two entire databases + get Gets the content of a table for the given key + drop Deletes all database entries + clear Deletes all table entries + repair-trie Verifies trie consistency and outputs any inconsistencies + version Lists current and local database versions + path Returns the full database path + help Print this message or the help of the given subcommand(s) Options: -h, --help @@ -79,17 +80,23 @@ Database: --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout + --db.max-readers + Maximum number of readers allowed to access the database concurrently + + --db.sync-mode + Controls how aggressively the database synchronizes data to disk + Logging: --log.stdout.format The format to use for logs written to stdout - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.stdout.filter The filter to use for logs written to stdout @@ -98,13 +105,13 @@ Logging: --log.file.format The format to use for logs written to the log file - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.file.filter The filter to use for logs written to the log file @@ -115,6 +122,11 @@ Logging: [default: /logs] + --log.file.name + The prefix name of the log files + + [default: reth.log] + --log.file.max-size The maximum size (in MB) of one log file @@ -136,13 +148,13 @@ Logging: --color Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting - [default: always] - Possible values: - always: Colors on - - auto: Colors on + - auto: Auto-detect - never: Colors off + [default: always] + Display: -v, --verbosity... Set the minimum log level. @@ -155,4 +167,37 @@ Display: -q, --quiet Silence all log output + +Tracing: + --tracing-otlp[=] + Enable `Opentelemetry` tracing export to an OTLP endpoint. + + If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + + Example: --tracing-otlp=http://collector:4318/v1/traces + + [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] + + --tracing-otlp-protocol + OTLP transport protocol to use for exporting traces. + + - `http`: expects endpoint path to end with `/v1/traces` - `grpc`: expects endpoint without a path + + Defaults to HTTP if not specified. + + Possible values: + - http: HTTP/Protobuf transport, port 4318, requires `/v1/traces` path + - grpc: gRPC transport, port 4317 + + [env: OTEL_EXPORTER_OTLP_PROTOCOL=] + [default: http] + + --tracing-otlp.filter + Set a filter directive for the OTLP tracer. This controls the verbosity of spans and events sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable. + + Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off + + Defaults to TRACE if not specified. + + [default: debug] ``` \ No newline at end of file diff --git a/book/cli/reth/db/checksum.md b/docs/vocs/docs/pages/cli/reth/db/checksum.mdx similarity index 70% rename from book/cli/reth/db/checksum.md rename to docs/vocs/docs/pages/cli/reth/db/checksum.mdx index c914aaed98b..4b8b8ca2cce 100644 --- a/book/cli/reth/db/checksum.md +++ b/docs/vocs/docs/pages/cli/reth/db/checksum.mdx @@ -39,13 +39,13 @@ Logging: --log.stdout.format The format to use for logs written to stdout - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.stdout.filter The filter to use for logs written to stdout @@ -54,13 +54,13 @@ Logging: --log.file.format The format to use for logs written to the log file - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.file.filter The filter to use for logs written to the log file @@ -71,6 +71,11 @@ Logging: [default: /logs] + --log.file.name + The prefix name of the log files + + [default: reth.log] + --log.file.max-size The maximum size (in MB) of one log file @@ -92,13 +97,13 @@ Logging: --color Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting - [default: always] - Possible values: - always: Colors on - - auto: Colors on + - auto: Auto-detect - never: Colors off + [default: always] + Display: -v, --verbosity... Set the minimum log level. @@ -111,4 +116,37 @@ Display: -q, --quiet Silence all log output + +Tracing: + --tracing-otlp[=] + Enable `Opentelemetry` tracing export to an OTLP endpoint. + + If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + + Example: --tracing-otlp=http://collector:4318/v1/traces + + [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] + + --tracing-otlp-protocol + OTLP transport protocol to use for exporting traces. + + - `http`: expects endpoint path to end with `/v1/traces` - `grpc`: expects endpoint without a path + + Defaults to HTTP if not specified. + + Possible values: + - http: HTTP/Protobuf transport, port 4318, requires `/v1/traces` path + - grpc: gRPC transport, port 4317 + + [env: OTEL_EXPORTER_OTLP_PROTOCOL=] + [default: http] + + --tracing-otlp.filter + Set a filter directive for the OTLP tracer. This controls the verbosity of spans and events sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable. + + Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off + + Defaults to TRACE if not specified. + + [default: debug] ``` \ No newline at end of file diff --git a/book/cli/reth/db/clear.md b/docs/vocs/docs/pages/cli/reth/db/clear.mdx similarity index 69% rename from book/cli/reth/db/clear.md rename to docs/vocs/docs/pages/cli/reth/db/clear.mdx index 87dae39c51e..1548558fe39 100644 --- a/book/cli/reth/db/clear.md +++ b/docs/vocs/docs/pages/cli/reth/db/clear.mdx @@ -31,13 +31,13 @@ Logging: --log.stdout.format The format to use for logs written to stdout - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.stdout.filter The filter to use for logs written to stdout @@ -46,13 +46,13 @@ Logging: --log.file.format The format to use for logs written to the log file - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.file.filter The filter to use for logs written to the log file @@ -63,6 +63,11 @@ Logging: [default: /logs] + --log.file.name + The prefix name of the log files + + [default: reth.log] + --log.file.max-size The maximum size (in MB) of one log file @@ -84,13 +89,13 @@ Logging: --color Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting - [default: always] - Possible values: - always: Colors on - - auto: Colors on + - auto: Auto-detect - never: Colors off + [default: always] + Display: -v, --verbosity... Set the minimum log level. @@ -103,4 +108,37 @@ Display: -q, --quiet Silence all log output + +Tracing: + --tracing-otlp[=] + Enable `Opentelemetry` tracing export to an OTLP endpoint. + + If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + + Example: --tracing-otlp=http://collector:4318/v1/traces + + [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] + + --tracing-otlp-protocol + OTLP transport protocol to use for exporting traces. + + - `http`: expects endpoint path to end with `/v1/traces` - `grpc`: expects endpoint without a path + + Defaults to HTTP if not specified. + + Possible values: + - http: HTTP/Protobuf transport, port 4318, requires `/v1/traces` path + - grpc: gRPC transport, port 4317 + + [env: OTEL_EXPORTER_OTLP_PROTOCOL=] + [default: http] + + --tracing-otlp.filter + Set a filter directive for the OTLP tracer. This controls the verbosity of spans and events sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable. + + Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off + + Defaults to TRACE if not specified. + + [default: debug] ``` \ No newline at end of file diff --git a/book/cli/reth/db/clear/mdbx.md b/docs/vocs/docs/pages/cli/reth/db/clear/mdbx.mdx similarity index 68% rename from book/cli/reth/db/clear/mdbx.md rename to docs/vocs/docs/pages/cli/reth/db/clear/mdbx.mdx index 20bfd3d5b1c..b48ba180982 100644 --- a/book/cli/reth/db/clear/mdbx.md +++ b/docs/vocs/docs/pages/cli/reth/db/clear/mdbx.mdx @@ -30,13 +30,13 @@ Logging: --log.stdout.format The format to use for logs written to stdout - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.stdout.filter The filter to use for logs written to stdout @@ -45,13 +45,13 @@ Logging: --log.file.format The format to use for logs written to the log file - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.file.filter The filter to use for logs written to the log file @@ -62,6 +62,11 @@ Logging: [default: /logs] + --log.file.name + The prefix name of the log files + + [default: reth.log] + --log.file.max-size The maximum size (in MB) of one log file @@ -83,13 +88,13 @@ Logging: --color Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting - [default: always] - Possible values: - always: Colors on - - auto: Colors on + - auto: Auto-detect - never: Colors off + [default: always] + Display: -v, --verbosity... Set the minimum log level. @@ -102,4 +107,37 @@ Display: -q, --quiet Silence all log output + +Tracing: + --tracing-otlp[=] + Enable `Opentelemetry` tracing export to an OTLP endpoint. + + If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + + Example: --tracing-otlp=http://collector:4318/v1/traces + + [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] + + --tracing-otlp-protocol + OTLP transport protocol to use for exporting traces. + + - `http`: expects endpoint path to end with `/v1/traces` - `grpc`: expects endpoint without a path + + Defaults to HTTP if not specified. + + Possible values: + - http: HTTP/Protobuf transport, port 4318, requires `/v1/traces` path + - grpc: gRPC transport, port 4317 + + [env: OTEL_EXPORTER_OTLP_PROTOCOL=] + [default: http] + + --tracing-otlp.filter + Set a filter directive for the OTLP tracer. This controls the verbosity of spans and events sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable. + + Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off + + Defaults to TRACE if not specified. + + [default: debug] ``` \ No newline at end of file diff --git a/book/cli/reth/db/clear/static-file.md b/docs/vocs/docs/pages/cli/reth/db/clear/static-file.mdx similarity index 70% rename from book/cli/reth/db/clear/static-file.md rename to docs/vocs/docs/pages/cli/reth/db/clear/static-file.mdx index 739557bd071..9f22178ec4c 100644 --- a/book/cli/reth/db/clear/static-file.md +++ b/docs/vocs/docs/pages/cli/reth/db/clear/static-file.mdx @@ -14,7 +14,6 @@ Arguments: - headers: Static File segment responsible for the `CanonicalHeaders`, `Headers`, `HeaderTerminalDifficulties` tables - transactions: Static File segment responsible for the `Transactions` table - receipts: Static File segment responsible for the `Receipts` table - - block-meta: Static File segment responsible for the `BlockBodyIndices`, `BlockOmmers`, `BlockWithdrawals` tables Options: -h, --help @@ -34,13 +33,13 @@ Logging: --log.stdout.format The format to use for logs written to stdout - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.stdout.filter The filter to use for logs written to stdout @@ -49,13 +48,13 @@ Logging: --log.file.format The format to use for logs written to the log file - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.file.filter The filter to use for logs written to the log file @@ -66,6 +65,11 @@ Logging: [default: /logs] + --log.file.name + The prefix name of the log files + + [default: reth.log] + --log.file.max-size The maximum size (in MB) of one log file @@ -87,13 +91,13 @@ Logging: --color Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting - [default: always] - Possible values: - always: Colors on - - auto: Colors on + - auto: Auto-detect - never: Colors off + [default: always] + Display: -v, --verbosity... Set the minimum log level. @@ -106,4 +110,37 @@ Display: -q, --quiet Silence all log output + +Tracing: + --tracing-otlp[=] + Enable `Opentelemetry` tracing export to an OTLP endpoint. + + If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + + Example: --tracing-otlp=http://collector:4318/v1/traces + + [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] + + --tracing-otlp-protocol + OTLP transport protocol to use for exporting traces. + + - `http`: expects endpoint path to end with `/v1/traces` - `grpc`: expects endpoint without a path + + Defaults to HTTP if not specified. + + Possible values: + - http: HTTP/Protobuf transport, port 4318, requires `/v1/traces` path + - grpc: gRPC transport, port 4317 + + [env: OTEL_EXPORTER_OTLP_PROTOCOL=] + [default: http] + + --tracing-otlp.filter + Set a filter directive for the OTLP tracer. This controls the verbosity of spans and events sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable. + + Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off + + Defaults to TRACE if not specified. + + [default: debug] ``` \ No newline at end of file diff --git a/book/cli/reth/db/diff.md b/docs/vocs/docs/pages/cli/reth/db/diff.mdx similarity index 73% rename from book/cli/reth/db/diff.md rename to docs/vocs/docs/pages/cli/reth/db/diff.mdx index cf726a03b75..27cb2198aaf 100644 --- a/book/cli/reth/db/diff.md +++ b/docs/vocs/docs/pages/cli/reth/db/diff.mdx @@ -43,6 +43,12 @@ Database: --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout + --db.max-readers + Maximum number of readers allowed to access the database concurrently + + --db.sync-mode + Controls how aggressively the database synchronizes data to disk + --table The table name to diff. If not specified, all tables are diffed. @@ -63,13 +69,13 @@ Logging: --log.stdout.format The format to use for logs written to stdout - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.stdout.filter The filter to use for logs written to stdout @@ -78,13 +84,13 @@ Logging: --log.file.format The format to use for logs written to the log file - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.file.filter The filter to use for logs written to the log file @@ -95,6 +101,11 @@ Logging: [default: /logs] + --log.file.name + The prefix name of the log files + + [default: reth.log] + --log.file.max-size The maximum size (in MB) of one log file @@ -116,13 +127,13 @@ Logging: --color Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting - [default: always] - Possible values: - always: Colors on - - auto: Colors on + - auto: Auto-detect - never: Colors off + [default: always] + Display: -v, --verbosity... Set the minimum log level. @@ -135,4 +146,37 @@ Display: -q, --quiet Silence all log output + +Tracing: + --tracing-otlp[=] + Enable `Opentelemetry` tracing export to an OTLP endpoint. + + If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + + Example: --tracing-otlp=http://collector:4318/v1/traces + + [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] + + --tracing-otlp-protocol + OTLP transport protocol to use for exporting traces. + + - `http`: expects endpoint path to end with `/v1/traces` - `grpc`: expects endpoint without a path + + Defaults to HTTP if not specified. + + Possible values: + - http: HTTP/Protobuf transport, port 4318, requires `/v1/traces` path + - grpc: gRPC transport, port 4317 + + [env: OTEL_EXPORTER_OTLP_PROTOCOL=] + [default: http] + + --tracing-otlp.filter + Set a filter directive for the OTLP tracer. This controls the verbosity of spans and events sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable. + + Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off + + Defaults to TRACE if not specified. + + [default: debug] ``` \ No newline at end of file diff --git a/book/cli/reth/db/drop.md b/docs/vocs/docs/pages/cli/reth/db/drop.mdx similarity index 68% rename from book/cli/reth/db/drop.md rename to docs/vocs/docs/pages/cli/reth/db/drop.mdx index 2988f658d20..c778320f2d8 100644 --- a/book/cli/reth/db/drop.md +++ b/docs/vocs/docs/pages/cli/reth/db/drop.mdx @@ -29,13 +29,13 @@ Logging: --log.stdout.format The format to use for logs written to stdout - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.stdout.filter The filter to use for logs written to stdout @@ -44,13 +44,13 @@ Logging: --log.file.format The format to use for logs written to the log file - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.file.filter The filter to use for logs written to the log file @@ -61,6 +61,11 @@ Logging: [default: /logs] + --log.file.name + The prefix name of the log files + + [default: reth.log] + --log.file.max-size The maximum size (in MB) of one log file @@ -82,13 +87,13 @@ Logging: --color Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting - [default: always] - Possible values: - always: Colors on - - auto: Colors on + - auto: Auto-detect - never: Colors off + [default: always] + Display: -v, --verbosity... Set the minimum log level. @@ -101,4 +106,37 @@ Display: -q, --quiet Silence all log output + +Tracing: + --tracing-otlp[=] + Enable `Opentelemetry` tracing export to an OTLP endpoint. + + If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + + Example: --tracing-otlp=http://collector:4318/v1/traces + + [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] + + --tracing-otlp-protocol + OTLP transport protocol to use for exporting traces. + + - `http`: expects endpoint path to end with `/v1/traces` - `grpc`: expects endpoint without a path + + Defaults to HTTP if not specified. + + Possible values: + - http: HTTP/Protobuf transport, port 4318, requires `/v1/traces` path + - grpc: gRPC transport, port 4317 + + [env: OTEL_EXPORTER_OTLP_PROTOCOL=] + [default: http] + + --tracing-otlp.filter + Set a filter directive for the OTLP tracer. This controls the verbosity of spans and events sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable. + + Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off + + Defaults to TRACE if not specified. + + [default: debug] ``` \ No newline at end of file diff --git a/book/cli/reth/db/get.md b/docs/vocs/docs/pages/cli/reth/db/get.mdx similarity index 69% rename from book/cli/reth/db/get.md rename to docs/vocs/docs/pages/cli/reth/db/get.mdx index d1b9251ca06..dfcfcac1886 100644 --- a/book/cli/reth/db/get.md +++ b/docs/vocs/docs/pages/cli/reth/db/get.mdx @@ -31,13 +31,13 @@ Logging: --log.stdout.format The format to use for logs written to stdout - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.stdout.filter The filter to use for logs written to stdout @@ -46,13 +46,13 @@ Logging: --log.file.format The format to use for logs written to the log file - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.file.filter The filter to use for logs written to the log file @@ -63,6 +63,11 @@ Logging: [default: /logs] + --log.file.name + The prefix name of the log files + + [default: reth.log] + --log.file.max-size The maximum size (in MB) of one log file @@ -84,13 +89,13 @@ Logging: --color Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting - [default: always] - Possible values: - always: Colors on - - auto: Colors on + - auto: Auto-detect - never: Colors off + [default: always] + Display: -v, --verbosity... Set the minimum log level. @@ -103,4 +108,37 @@ Display: -q, --quiet Silence all log output + +Tracing: + --tracing-otlp[=] + Enable `Opentelemetry` tracing export to an OTLP endpoint. + + If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + + Example: --tracing-otlp=http://collector:4318/v1/traces + + [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] + + --tracing-otlp-protocol + OTLP transport protocol to use for exporting traces. + + - `http`: expects endpoint path to end with `/v1/traces` - `grpc`: expects endpoint without a path + + Defaults to HTTP if not specified. + + Possible values: + - http: HTTP/Protobuf transport, port 4318, requires `/v1/traces` path + - grpc: gRPC transport, port 4317 + + [env: OTEL_EXPORTER_OTLP_PROTOCOL=] + [default: http] + + --tracing-otlp.filter + Set a filter directive for the OTLP tracer. This controls the verbosity of spans and events sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable. + + Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off + + Defaults to TRACE if not specified. + + [default: debug] ``` \ No newline at end of file diff --git a/book/cli/reth/db/get/mdbx.md b/docs/vocs/docs/pages/cli/reth/db/get/mdbx.mdx similarity index 69% rename from book/cli/reth/db/get/mdbx.md rename to docs/vocs/docs/pages/cli/reth/db/get/mdbx.mdx index 0ac2e31e208..981d0c9f9a5 100644 --- a/book/cli/reth/db/get/mdbx.md +++ b/docs/vocs/docs/pages/cli/reth/db/get/mdbx.mdx @@ -39,13 +39,13 @@ Logging: --log.stdout.format The format to use for logs written to stdout - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.stdout.filter The filter to use for logs written to stdout @@ -54,13 +54,13 @@ Logging: --log.file.format The format to use for logs written to the log file - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.file.filter The filter to use for logs written to the log file @@ -71,6 +71,11 @@ Logging: [default: /logs] + --log.file.name + The prefix name of the log files + + [default: reth.log] + --log.file.max-size The maximum size (in MB) of one log file @@ -92,13 +97,13 @@ Logging: --color Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting - [default: always] - Possible values: - always: Colors on - - auto: Colors on + - auto: Auto-detect - never: Colors off + [default: always] + Display: -v, --verbosity... Set the minimum log level. @@ -111,4 +116,37 @@ Display: -q, --quiet Silence all log output + +Tracing: + --tracing-otlp[=] + Enable `Opentelemetry` tracing export to an OTLP endpoint. + + If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + + Example: --tracing-otlp=http://collector:4318/v1/traces + + [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] + + --tracing-otlp-protocol + OTLP transport protocol to use for exporting traces. + + - `http`: expects endpoint path to end with `/v1/traces` - `grpc`: expects endpoint without a path + + Defaults to HTTP if not specified. + + Possible values: + - http: HTTP/Protobuf transport, port 4318, requires `/v1/traces` path + - grpc: gRPC transport, port 4317 + + [env: OTEL_EXPORTER_OTLP_PROTOCOL=] + [default: http] + + --tracing-otlp.filter + Set a filter directive for the OTLP tracer. This controls the verbosity of spans and events sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable. + + Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off + + Defaults to TRACE if not specified. + + [default: debug] ``` \ No newline at end of file diff --git a/book/cli/reth/db/get/static-file.md b/docs/vocs/docs/pages/cli/reth/db/get/static-file.mdx similarity index 71% rename from book/cli/reth/db/get/static-file.md rename to docs/vocs/docs/pages/cli/reth/db/get/static-file.mdx index d274c5b4760..8e045a4cdf1 100644 --- a/book/cli/reth/db/get/static-file.md +++ b/docs/vocs/docs/pages/cli/reth/db/get/static-file.mdx @@ -14,7 +14,6 @@ Arguments: - headers: Static File segment responsible for the `CanonicalHeaders`, `Headers`, `HeaderTerminalDifficulties` tables - transactions: Static File segment responsible for the `Transactions` table - receipts: Static File segment responsible for the `Receipts` table - - block-meta: Static File segment responsible for the `BlockBodyIndices`, `BlockOmmers`, `BlockWithdrawals` tables The key to get content for @@ -40,13 +39,13 @@ Logging: --log.stdout.format The format to use for logs written to stdout - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.stdout.filter The filter to use for logs written to stdout @@ -55,13 +54,13 @@ Logging: --log.file.format The format to use for logs written to the log file - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.file.filter The filter to use for logs written to the log file @@ -72,6 +71,11 @@ Logging: [default: /logs] + --log.file.name + The prefix name of the log files + + [default: reth.log] + --log.file.max-size The maximum size (in MB) of one log file @@ -93,13 +97,13 @@ Logging: --color Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting - [default: always] - Possible values: - always: Colors on - - auto: Colors on + - auto: Auto-detect - never: Colors off + [default: always] + Display: -v, --verbosity... Set the minimum log level. @@ -112,4 +116,37 @@ Display: -q, --quiet Silence all log output + +Tracing: + --tracing-otlp[=] + Enable `Opentelemetry` tracing export to an OTLP endpoint. + + If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + + Example: --tracing-otlp=http://collector:4318/v1/traces + + [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] + + --tracing-otlp-protocol + OTLP transport protocol to use for exporting traces. + + - `http`: expects endpoint path to end with `/v1/traces` - `grpc`: expects endpoint without a path + + Defaults to HTTP if not specified. + + Possible values: + - http: HTTP/Protobuf transport, port 4318, requires `/v1/traces` path + - grpc: gRPC transport, port 4317 + + [env: OTEL_EXPORTER_OTLP_PROTOCOL=] + [default: http] + + --tracing-otlp.filter + Set a filter directive for the OTLP tracer. This controls the verbosity of spans and events sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable. + + Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off + + Defaults to TRACE if not specified. + + [default: debug] ``` \ No newline at end of file diff --git a/book/cli/reth/db/list.md b/docs/vocs/docs/pages/cli/reth/db/list.mdx similarity index 74% rename from book/cli/reth/db/list.md rename to docs/vocs/docs/pages/cli/reth/db/list.mdx index b9b667323b7..3be1cd183b2 100644 --- a/book/cli/reth/db/list.md +++ b/docs/vocs/docs/pages/cli/reth/db/list.mdx @@ -72,13 +72,13 @@ Logging: --log.stdout.format The format to use for logs written to stdout - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.stdout.filter The filter to use for logs written to stdout @@ -87,13 +87,13 @@ Logging: --log.file.format The format to use for logs written to the log file - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.file.filter The filter to use for logs written to the log file @@ -104,6 +104,11 @@ Logging: [default: /logs] + --log.file.name + The prefix name of the log files + + [default: reth.log] + --log.file.max-size The maximum size (in MB) of one log file @@ -125,13 +130,13 @@ Logging: --color Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting - [default: always] - Possible values: - always: Colors on - - auto: Colors on + - auto: Auto-detect - never: Colors off + [default: always] + Display: -v, --verbosity... Set the minimum log level. @@ -144,4 +149,37 @@ Display: -q, --quiet Silence all log output + +Tracing: + --tracing-otlp[=] + Enable `Opentelemetry` tracing export to an OTLP endpoint. + + If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + + Example: --tracing-otlp=http://collector:4318/v1/traces + + [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] + + --tracing-otlp-protocol + OTLP transport protocol to use for exporting traces. + + - `http`: expects endpoint path to end with `/v1/traces` - `grpc`: expects endpoint without a path + + Defaults to HTTP if not specified. + + Possible values: + - http: HTTP/Protobuf transport, port 4318, requires `/v1/traces` path + - grpc: gRPC transport, port 4317 + + [env: OTEL_EXPORTER_OTLP_PROTOCOL=] + [default: http] + + --tracing-otlp.filter + Set a filter directive for the OTLP tracer. This controls the verbosity of spans and events sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable. + + Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off + + Defaults to TRACE if not specified. + + [default: debug] ``` \ No newline at end of file diff --git a/book/cli/reth/db/path.md b/docs/vocs/docs/pages/cli/reth/db/path.mdx similarity index 67% rename from book/cli/reth/db/path.md rename to docs/vocs/docs/pages/cli/reth/db/path.mdx index 2929c47ed74..a954093dd5d 100644 --- a/book/cli/reth/db/path.md +++ b/docs/vocs/docs/pages/cli/reth/db/path.mdx @@ -26,13 +26,13 @@ Logging: --log.stdout.format The format to use for logs written to stdout - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.stdout.filter The filter to use for logs written to stdout @@ -41,13 +41,13 @@ Logging: --log.file.format The format to use for logs written to the log file - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.file.filter The filter to use for logs written to the log file @@ -58,6 +58,11 @@ Logging: [default: /logs] + --log.file.name + The prefix name of the log files + + [default: reth.log] + --log.file.max-size The maximum size (in MB) of one log file @@ -79,13 +84,13 @@ Logging: --color Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting - [default: always] - Possible values: - always: Colors on - - auto: Colors on + - auto: Auto-detect - never: Colors off + [default: always] + Display: -v, --verbosity... Set the minimum log level. @@ -98,4 +103,37 @@ Display: -q, --quiet Silence all log output + +Tracing: + --tracing-otlp[=] + Enable `Opentelemetry` tracing export to an OTLP endpoint. + + If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + + Example: --tracing-otlp=http://collector:4318/v1/traces + + [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] + + --tracing-otlp-protocol + OTLP transport protocol to use for exporting traces. + + - `http`: expects endpoint path to end with `/v1/traces` - `grpc`: expects endpoint without a path + + Defaults to HTTP if not specified. + + Possible values: + - http: HTTP/Protobuf transport, port 4318, requires `/v1/traces` path + - grpc: gRPC transport, port 4317 + + [env: OTEL_EXPORTER_OTLP_PROTOCOL=] + [default: http] + + --tracing-otlp.filter + Set a filter directive for the OTLP tracer. This controls the verbosity of spans and events sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable. + + Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off + + Defaults to TRACE if not specified. + + [default: debug] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/db/repair-trie.mdx b/docs/vocs/docs/pages/cli/reth/db/repair-trie.mdx new file mode 100644 index 00000000000..6436afc2133 --- /dev/null +++ b/docs/vocs/docs/pages/cli/reth/db/repair-trie.mdx @@ -0,0 +1,142 @@ +# reth db repair-trie + +Verifies trie consistency and outputs any inconsistencies + +```bash +$ reth db repair-trie --help +``` +```txt +Usage: reth db repair-trie [OPTIONS] + +Options: + --dry-run + Only show inconsistencies without making any repairs + + -h, --help + Print help (see a summary with '-h') + +Datadir: + --chain + The chain this node is running. + Possible values are either a built-in chain or the path to a chain specification file. + + Built-in chains: + mainnet, sepolia, holesky, hoodi, dev + + [default: mainnet] + +Logging: + --log.stdout.format + The format to use for logs written to stdout + + Possible values: + - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging + - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications + - terminal: Represents terminal-friendly formatting for logs + + [default: terminal] + + --log.stdout.filter + The filter to use for logs written to stdout + + [default: ] + + --log.file.format + The format to use for logs written to the log file + + Possible values: + - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging + - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications + - terminal: Represents terminal-friendly formatting for logs + + [default: terminal] + + --log.file.filter + The filter to use for logs written to the log file + + [default: debug] + + --log.file.directory + The path to put log files in + + [default: /logs] + + --log.file.name + The prefix name of the log files + + [default: reth.log] + + --log.file.max-size + The maximum size (in MB) of one log file + + [default: 200] + + --log.file.max-files + The maximum amount of log files that will be stored. If set to 0, background file logging is disabled + + [default: 5] + + --log.journald + Write logs to journald + + --log.journald.filter + The filter to use for logs written to journald + + [default: error] + + --color + Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting + + Possible values: + - always: Colors on + - auto: Auto-detect + - never: Colors off + + [default: always] + +Display: + -v, --verbosity... + Set the minimum log level. + + -v Errors + -vv Warnings + -vvv Info + -vvvv Debug + -vvvvv Traces (warning: very verbose!) + + -q, --quiet + Silence all log output + +Tracing: + --tracing-otlp[=] + Enable `Opentelemetry` tracing export to an OTLP endpoint. + + If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + + Example: --tracing-otlp=http://collector:4318/v1/traces + + [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] + + --tracing-otlp-protocol + OTLP transport protocol to use for exporting traces. + + - `http`: expects endpoint path to end with `/v1/traces` - `grpc`: expects endpoint without a path + + Defaults to HTTP if not specified. + + Possible values: + - http: HTTP/Protobuf transport, port 4318, requires `/v1/traces` path + - grpc: gRPC transport, port 4317 + + [env: OTEL_EXPORTER_OTLP_PROTOCOL=] + [default: http] + + --tracing-otlp.filter + Set a filter directive for the OTLP tracer. This controls the verbosity of spans and events sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable. + + Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off + + Defaults to TRACE if not specified. + + [default: debug] +``` \ No newline at end of file diff --git a/book/cli/reth/db/stats.md b/docs/vocs/docs/pages/cli/reth/db/stats.mdx similarity index 70% rename from book/cli/reth/db/stats.md rename to docs/vocs/docs/pages/cli/reth/db/stats.mdx index 2bc28cb490d..5bd316847c0 100644 --- a/book/cli/reth/db/stats.md +++ b/docs/vocs/docs/pages/cli/reth/db/stats.mdx @@ -39,13 +39,13 @@ Logging: --log.stdout.format The format to use for logs written to stdout - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.stdout.filter The filter to use for logs written to stdout @@ -54,13 +54,13 @@ Logging: --log.file.format The format to use for logs written to the log file - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.file.filter The filter to use for logs written to the log file @@ -71,6 +71,11 @@ Logging: [default: /logs] + --log.file.name + The prefix name of the log files + + [default: reth.log] + --log.file.max-size The maximum size (in MB) of one log file @@ -92,13 +97,13 @@ Logging: --color Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting - [default: always] - Possible values: - always: Colors on - - auto: Colors on + - auto: Auto-detect - never: Colors off + [default: always] + Display: -v, --verbosity... Set the minimum log level. @@ -111,4 +116,37 @@ Display: -q, --quiet Silence all log output + +Tracing: + --tracing-otlp[=] + Enable `Opentelemetry` tracing export to an OTLP endpoint. + + If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + + Example: --tracing-otlp=http://collector:4318/v1/traces + + [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] + + --tracing-otlp-protocol + OTLP transport protocol to use for exporting traces. + + - `http`: expects endpoint path to end with `/v1/traces` - `grpc`: expects endpoint without a path + + Defaults to HTTP if not specified. + + Possible values: + - http: HTTP/Protobuf transport, port 4318, requires `/v1/traces` path + - grpc: gRPC transport, port 4317 + + [env: OTEL_EXPORTER_OTLP_PROTOCOL=] + [default: http] + + --tracing-otlp.filter + Set a filter directive for the OTLP tracer. This controls the verbosity of spans and events sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable. + + Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off + + Defaults to TRACE if not specified. + + [default: debug] ``` \ No newline at end of file diff --git a/book/cli/reth/db/version.md b/docs/vocs/docs/pages/cli/reth/db/version.mdx similarity index 67% rename from book/cli/reth/db/version.md rename to docs/vocs/docs/pages/cli/reth/db/version.mdx index a59992da6f3..c87496d910d 100644 --- a/book/cli/reth/db/version.md +++ b/docs/vocs/docs/pages/cli/reth/db/version.mdx @@ -26,13 +26,13 @@ Logging: --log.stdout.format The format to use for logs written to stdout - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.stdout.filter The filter to use for logs written to stdout @@ -41,13 +41,13 @@ Logging: --log.file.format The format to use for logs written to the log file - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.file.filter The filter to use for logs written to the log file @@ -58,6 +58,11 @@ Logging: [default: /logs] + --log.file.name + The prefix name of the log files + + [default: reth.log] + --log.file.max-size The maximum size (in MB) of one log file @@ -79,13 +84,13 @@ Logging: --color Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting - [default: always] - Possible values: - always: Colors on - - auto: Colors on + - auto: Auto-detect - never: Colors off + [default: always] + Display: -v, --verbosity... Set the minimum log level. @@ -98,4 +103,37 @@ Display: -q, --quiet Silence all log output + +Tracing: + --tracing-otlp[=] + Enable `Opentelemetry` tracing export to an OTLP endpoint. + + If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + + Example: --tracing-otlp=http://collector:4318/v1/traces + + [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] + + --tracing-otlp-protocol + OTLP transport protocol to use for exporting traces. + + - `http`: expects endpoint path to end with `/v1/traces` - `grpc`: expects endpoint without a path + + Defaults to HTTP if not specified. + + Possible values: + - http: HTTP/Protobuf transport, port 4318, requires `/v1/traces` path + - grpc: gRPC transport, port 4317 + + [env: OTEL_EXPORTER_OTLP_PROTOCOL=] + [default: http] + + --tracing-otlp.filter + Set a filter directive for the OTLP tracer. This controls the verbosity of spans and events sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable. + + Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off + + Defaults to TRACE if not specified. + + [default: debug] ``` \ No newline at end of file diff --git a/book/cli/reth/debug.md b/docs/vocs/docs/pages/cli/reth/debug.mdx similarity index 95% rename from book/cli/reth/debug.md rename to docs/vocs/docs/pages/cli/reth/debug.mdx index 0f616236a67..f56a60aa941 100644 --- a/book/cli/reth/debug.md +++ b/docs/vocs/docs/pages/cli/reth/debug.mdx @@ -9,10 +9,8 @@ $ reth debug --help Usage: reth debug [OPTIONS] Commands: - execution Debug the roundtrip execution of blocks as well as the generated data merkle Debug the clean & incremental state root calculations in-memory-merkle Debug in-memory state root calculation - build-block Debug block building help Print this message or the help of the given subcommand(s) Options: diff --git a/book/cli/reth/download.md b/docs/vocs/docs/pages/cli/reth/download.mdx similarity index 73% rename from book/cli/reth/download.md rename to docs/vocs/docs/pages/cli/reth/download.mdx index 04a7228f212..6cdaa9ca2d3 100644 --- a/book/cli/reth/download.md +++ b/docs/vocs/docs/pages/cli/reth/download.mdx @@ -67,27 +67,33 @@ Database: --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout + --db.max-readers + Maximum number of readers allowed to access the database concurrently + + --db.sync-mode + Controls how aggressively the database synchronizes data to disk + -u, --url Specify a snapshot URL or let the command propose a default one. Available snapshot sources: - - https://downloads.merkle.io (default, mainnet archive) + - https://www.merkle.io/snapshots (default, mainnet archive) - https://publicnode.com/snapshots (full nodes & testnets) If no URL is provided, the latest mainnet archive snapshot - will be proposed for download from merkle.io + will be proposed for download from https://downloads.merkle.io Logging: --log.stdout.format The format to use for logs written to stdout - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.stdout.filter The filter to use for logs written to stdout @@ -96,13 +102,13 @@ Logging: --log.file.format The format to use for logs written to the log file - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.file.filter The filter to use for logs written to the log file @@ -113,6 +119,11 @@ Logging: [default: /logs] + --log.file.name + The prefix name of the log files + + [default: reth.log] + --log.file.max-size The maximum size (in MB) of one log file @@ -134,13 +145,13 @@ Logging: --color Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting - [default: always] - Possible values: - always: Colors on - - auto: Colors on + - auto: Auto-detect - never: Colors off + [default: always] + Display: -v, --verbosity... Set the minimum log level. @@ -153,4 +164,37 @@ Display: -q, --quiet Silence all log output + +Tracing: + --tracing-otlp[=] + Enable `Opentelemetry` tracing export to an OTLP endpoint. + + If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + + Example: --tracing-otlp=http://collector:4318/v1/traces + + [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] + + --tracing-otlp-protocol + OTLP transport protocol to use for exporting traces. + + - `http`: expects endpoint path to end with `/v1/traces` - `grpc`: expects endpoint without a path + + Defaults to HTTP if not specified. + + Possible values: + - http: HTTP/Protobuf transport, port 4318, requires `/v1/traces` path + - grpc: gRPC transport, port 4317 + + [env: OTEL_EXPORTER_OTLP_PROTOCOL=] + [default: http] + + --tracing-otlp.filter + Set a filter directive for the OTLP tracer. This controls the verbosity of spans and events sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable. + + Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off + + Defaults to TRACE if not specified. + + [default: debug] ``` \ No newline at end of file diff --git a/book/cli/reth/dump-genesis.md b/docs/vocs/docs/pages/cli/reth/dump-genesis.mdx similarity index 68% rename from book/cli/reth/dump-genesis.md rename to docs/vocs/docs/pages/cli/reth/dump-genesis.mdx index 30e7ea76afa..7aeaa8db49a 100644 --- a/book/cli/reth/dump-genesis.md +++ b/docs/vocs/docs/pages/cli/reth/dump-genesis.mdx @@ -25,13 +25,13 @@ Logging: --log.stdout.format The format to use for logs written to stdout - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.stdout.filter The filter to use for logs written to stdout @@ -40,13 +40,13 @@ Logging: --log.file.format The format to use for logs written to the log file - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.file.filter The filter to use for logs written to the log file @@ -57,6 +57,11 @@ Logging: [default: /logs] + --log.file.name + The prefix name of the log files + + [default: reth.log] + --log.file.max-size The maximum size (in MB) of one log file @@ -78,13 +83,13 @@ Logging: --color Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting - [default: always] - Possible values: - always: Colors on - - auto: Colors on + - auto: Auto-detect - never: Colors off + [default: always] + Display: -v, --verbosity... Set the minimum log level. @@ -97,4 +102,37 @@ Display: -q, --quiet Silence all log output + +Tracing: + --tracing-otlp[=] + Enable `Opentelemetry` tracing export to an OTLP endpoint. + + If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + + Example: --tracing-otlp=http://collector:4318/v1/traces + + [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] + + --tracing-otlp-protocol + OTLP transport protocol to use for exporting traces. + + - `http`: expects endpoint path to end with `/v1/traces` - `grpc`: expects endpoint without a path + + Defaults to HTTP if not specified. + + Possible values: + - http: HTTP/Protobuf transport, port 4318, requires `/v1/traces` path + - grpc: gRPC transport, port 4317 + + [env: OTEL_EXPORTER_OTLP_PROTOCOL=] + [default: http] + + --tracing-otlp.filter + Set a filter directive for the OTLP tracer. This controls the verbosity of spans and events sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable. + + Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off + + Defaults to TRACE if not specified. + + [default: debug] ``` \ No newline at end of file diff --git a/book/cli/reth/debug/build-block.md b/docs/vocs/docs/pages/cli/reth/export-era.mdx similarity index 65% rename from book/cli/reth/debug/build-block.md rename to docs/vocs/docs/pages/cli/reth/export-era.mdx index ac8ab6d3214..a873781d9c3 100644 --- a/book/cli/reth/debug/build-block.md +++ b/docs/vocs/docs/pages/cli/reth/export-era.mdx @@ -1,12 +1,12 @@ -# reth debug build-block +# reth export-era -Debug block building +Exports block to era1 files in a specified directory ```bash -$ reth debug build-block --help +$ reth export-era --help ``` ```txt -Usage: reth debug build-block [OPTIONS] --prev-randao --timestamp --suggested-fee-recipient +Usage: reth export-era [OPTIONS] Options: -h, --help @@ -67,35 +67,39 @@ Database: --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout - --parent-beacon-block-root + --db.max-readers + Maximum number of readers allowed to access the database concurrently + --db.sync-mode + Controls how aggressively the database synchronizes data to disk - --prev-randao + --first-block-number + Optional first block number to export from the db. + It is by default 0. + --last-block-number + Optional last block number to export from the db. + It is by default 8191. - --timestamp + --max-blocks-per-file + The maximum number of blocks per file, it can help you to decrease the size of the files. + Must be less than or equal to 8192. - - --suggested-fee-recipient - - - --transactions - Array of transactions. NOTE: 4844 transactions must be provided in the same order as they appear in the blobs bundle - - --blobs-bundle-path - Path to the file that contains a corresponding blobs bundle + --path + The directory path where to export era1 files. + The block data are read from the database. Logging: --log.stdout.format The format to use for logs written to stdout - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.stdout.filter The filter to use for logs written to stdout @@ -104,13 +108,13 @@ Logging: --log.file.format The format to use for logs written to the log file - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.file.filter The filter to use for logs written to the log file @@ -121,6 +125,11 @@ Logging: [default: /logs] + --log.file.name + The prefix name of the log files + + [default: reth.log] + --log.file.max-size The maximum size (in MB) of one log file @@ -142,13 +151,13 @@ Logging: --color Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting - [default: always] - Possible values: - always: Colors on - - auto: Colors on + - auto: Auto-detect - never: Colors off + [default: always] + Display: -v, --verbosity... Set the minimum log level. @@ -161,4 +170,37 @@ Display: -q, --quiet Silence all log output + +Tracing: + --tracing-otlp[=] + Enable `Opentelemetry` tracing export to an OTLP endpoint. + + If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + + Example: --tracing-otlp=http://collector:4318/v1/traces + + [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] + + --tracing-otlp-protocol + OTLP transport protocol to use for exporting traces. + + - `http`: expects endpoint path to end with `/v1/traces` - `grpc`: expects endpoint without a path + + Defaults to HTTP if not specified. + + Possible values: + - http: HTTP/Protobuf transport, port 4318, requires `/v1/traces` path + - grpc: gRPC transport, port 4317 + + [env: OTEL_EXPORTER_OTLP_PROTOCOL=] + [default: http] + + --tracing-otlp.filter + Set a filter directive for the OTLP tracer. This controls the verbosity of spans and events sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable. + + Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off + + Defaults to TRACE if not specified. + + [default: debug] ``` \ No newline at end of file diff --git a/book/cli/reth/import-era.md b/docs/vocs/docs/pages/cli/reth/import-era.mdx similarity index 75% rename from book/cli/reth/import-era.md rename to docs/vocs/docs/pages/cli/reth/import-era.mdx index 9dc2cb7ad46..77e7883e1bd 100644 --- a/book/cli/reth/import-era.md +++ b/docs/vocs/docs/pages/cli/reth/import-era.mdx @@ -67,6 +67,12 @@ Database: --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout + --db.max-readers + Maximum number of readers allowed to access the database concurrently + + --db.sync-mode + Controls how aggressively the database synchronizes data to disk + --path The path to a directory for import. @@ -82,13 +88,13 @@ Logging: --log.stdout.format The format to use for logs written to stdout - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.stdout.filter The filter to use for logs written to stdout @@ -97,13 +103,13 @@ Logging: --log.file.format The format to use for logs written to the log file - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.file.filter The filter to use for logs written to the log file @@ -114,6 +120,11 @@ Logging: [default: /logs] + --log.file.name + The prefix name of the log files + + [default: reth.log] + --log.file.max-size The maximum size (in MB) of one log file @@ -135,13 +146,13 @@ Logging: --color Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting - [default: always] - Possible values: - always: Colors on - - auto: Colors on + - auto: Auto-detect - never: Colors off + [default: always] + Display: -v, --verbosity... Set the minimum log level. @@ -154,4 +165,37 @@ Display: -q, --quiet Silence all log output + +Tracing: + --tracing-otlp[=] + Enable `Opentelemetry` tracing export to an OTLP endpoint. + + If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + + Example: --tracing-otlp=http://collector:4318/v1/traces + + [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] + + --tracing-otlp-protocol + OTLP transport protocol to use for exporting traces. + + - `http`: expects endpoint path to end with `/v1/traces` - `grpc`: expects endpoint without a path + + Defaults to HTTP if not specified. + + Possible values: + - http: HTTP/Protobuf transport, port 4318, requires `/v1/traces` path + - grpc: gRPC transport, port 4317 + + [env: OTEL_EXPORTER_OTLP_PROTOCOL=] + [default: http] + + --tracing-otlp.filter + Set a filter directive for the OTLP tracer. This controls the verbosity of spans and events sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable. + + Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off + + Defaults to TRACE if not specified. + + [default: debug] ``` \ No newline at end of file diff --git a/book/cli/reth/import.md b/docs/vocs/docs/pages/cli/reth/import.mdx similarity index 72% rename from book/cli/reth/import.md rename to docs/vocs/docs/pages/cli/reth/import.mdx index 958fedf38d3..39762051649 100644 --- a/book/cli/reth/import.md +++ b/docs/vocs/docs/pages/cli/reth/import.mdx @@ -1,12 +1,12 @@ # reth import -This syncs RLP encoded blocks from a file +This syncs RLP encoded blocks from a file or files ```bash $ reth import --help ``` ```txt -Usage: reth import [OPTIONS] +Usage: reth import [OPTIONS] ... Options: -h, --help @@ -67,29 +67,35 @@ Database: --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout + --db.max-readers + Maximum number of readers allowed to access the database concurrently + + --db.sync-mode + Controls how aggressively the database synchronizes data to disk + --no-state Disables stages that require state. --chunk-len Chunk byte length to read from file. - - The path to a block file for import. + ... + The path(s) to block file(s) for import. The online stages (headers and bodies) are replaced by a file import, after which the - remaining stages are executed. + remaining stages are executed. Multiple files will be imported sequentially. Logging: --log.stdout.format The format to use for logs written to stdout - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.stdout.filter The filter to use for logs written to stdout @@ -98,13 +104,13 @@ Logging: --log.file.format The format to use for logs written to the log file - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.file.filter The filter to use for logs written to the log file @@ -115,6 +121,11 @@ Logging: [default: /logs] + --log.file.name + The prefix name of the log files + + [default: reth.log] + --log.file.max-size The maximum size (in MB) of one log file @@ -136,13 +147,13 @@ Logging: --color Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting - [default: always] - Possible values: - always: Colors on - - auto: Colors on + - auto: Auto-detect - never: Colors off + [default: always] + Display: -v, --verbosity... Set the minimum log level. @@ -155,4 +166,37 @@ Display: -q, --quiet Silence all log output + +Tracing: + --tracing-otlp[=] + Enable `Opentelemetry` tracing export to an OTLP endpoint. + + If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + + Example: --tracing-otlp=http://collector:4318/v1/traces + + [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] + + --tracing-otlp-protocol + OTLP transport protocol to use for exporting traces. + + - `http`: expects endpoint path to end with `/v1/traces` - `grpc`: expects endpoint without a path + + Defaults to HTTP if not specified. + + Possible values: + - http: HTTP/Protobuf transport, port 4318, requires `/v1/traces` path + - grpc: gRPC transport, port 4317 + + [env: OTEL_EXPORTER_OTLP_PROTOCOL=] + [default: http] + + --tracing-otlp.filter + Set a filter directive for the OTLP tracer. This controls the verbosity of spans and events sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable. + + Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off + + Defaults to TRACE if not specified. + + [default: debug] ``` \ No newline at end of file diff --git a/book/cli/reth/init-state.md b/docs/vocs/docs/pages/cli/reth/init-state.mdx similarity index 78% rename from book/cli/reth/init-state.md rename to docs/vocs/docs/pages/cli/reth/init-state.mdx index 8154798828e..7e97d087165 100644 --- a/book/cli/reth/init-state.md +++ b/docs/vocs/docs/pages/cli/reth/init-state.mdx @@ -67,6 +67,12 @@ Database: --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout + --db.max-readers + Maximum number of readers allowed to access the database concurrently + + --db.sync-mode + Controls how aggressively the database synchronizes data to disk + --without-evm Specifies whether to initialize the state without relying on EVM historical data. @@ -77,9 +83,6 @@ Database: --header Header file containing the header in an RLP encoded format. - --total-difficulty - Total difficulty of the header. - --header-hash Hash of the header. @@ -106,13 +109,13 @@ Logging: --log.stdout.format The format to use for logs written to stdout - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.stdout.filter The filter to use for logs written to stdout @@ -121,13 +124,13 @@ Logging: --log.file.format The format to use for logs written to the log file - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.file.filter The filter to use for logs written to the log file @@ -138,6 +141,11 @@ Logging: [default: /logs] + --log.file.name + The prefix name of the log files + + [default: reth.log] + --log.file.max-size The maximum size (in MB) of one log file @@ -159,13 +167,13 @@ Logging: --color Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting - [default: always] - Possible values: - always: Colors on - - auto: Colors on + - auto: Auto-detect - never: Colors off + [default: always] + Display: -v, --verbosity... Set the minimum log level. @@ -178,4 +186,37 @@ Display: -q, --quiet Silence all log output + +Tracing: + --tracing-otlp[=] + Enable `Opentelemetry` tracing export to an OTLP endpoint. + + If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + + Example: --tracing-otlp=http://collector:4318/v1/traces + + [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] + + --tracing-otlp-protocol + OTLP transport protocol to use for exporting traces. + + - `http`: expects endpoint path to end with `/v1/traces` - `grpc`: expects endpoint without a path + + Defaults to HTTP if not specified. + + Possible values: + - http: HTTP/Protobuf transport, port 4318, requires `/v1/traces` path + - grpc: gRPC transport, port 4317 + + [env: OTEL_EXPORTER_OTLP_PROTOCOL=] + [default: http] + + --tracing-otlp.filter + Set a filter directive for the OTLP tracer. This controls the verbosity of spans and events sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable. + + Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off + + Defaults to TRACE if not specified. + + [default: debug] ``` \ No newline at end of file diff --git a/book/cli/reth/init.md b/docs/vocs/docs/pages/cli/reth/init.mdx similarity index 74% rename from book/cli/reth/init.md rename to docs/vocs/docs/pages/cli/reth/init.mdx index 80e1558ffa0..bf9dd671db6 100644 --- a/book/cli/reth/init.md +++ b/docs/vocs/docs/pages/cli/reth/init.mdx @@ -67,17 +67,23 @@ Database: --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout + --db.max-readers + Maximum number of readers allowed to access the database concurrently + + --db.sync-mode + Controls how aggressively the database synchronizes data to disk + Logging: --log.stdout.format The format to use for logs written to stdout - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.stdout.filter The filter to use for logs written to stdout @@ -86,13 +92,13 @@ Logging: --log.file.format The format to use for logs written to the log file - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.file.filter The filter to use for logs written to the log file @@ -103,6 +109,11 @@ Logging: [default: /logs] + --log.file.name + The prefix name of the log files + + [default: reth.log] + --log.file.max-size The maximum size (in MB) of one log file @@ -124,13 +135,13 @@ Logging: --color Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting - [default: always] - Possible values: - always: Colors on - - auto: Colors on + - auto: Auto-detect - never: Colors off + [default: always] + Display: -v, --verbosity... Set the minimum log level. @@ -143,4 +154,37 @@ Display: -q, --quiet Silence all log output + +Tracing: + --tracing-otlp[=] + Enable `Opentelemetry` tracing export to an OTLP endpoint. + + If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + + Example: --tracing-otlp=http://collector:4318/v1/traces + + [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] + + --tracing-otlp-protocol + OTLP transport protocol to use for exporting traces. + + - `http`: expects endpoint path to end with `/v1/traces` - `grpc`: expects endpoint without a path + + Defaults to HTTP if not specified. + + Possible values: + - http: HTTP/Protobuf transport, port 4318, requires `/v1/traces` path + - grpc: gRPC transport, port 4317 + + [env: OTEL_EXPORTER_OTLP_PROTOCOL=] + [default: http] + + --tracing-otlp.filter + Set a filter directive for the OTLP tracer. This controls the verbosity of spans and events sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable. + + Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off + + Defaults to TRACE if not specified. + + [default: debug] ``` \ No newline at end of file diff --git a/book/cli/reth/node.md b/docs/vocs/docs/pages/cli/reth/node.mdx similarity index 75% rename from book/cli/reth/node.md rename to docs/vocs/docs/pages/cli/reth/node.mdx index c696378e94f..d92bad0c53d 100644 --- a/book/cli/reth/node.md +++ b/docs/vocs/docs/pages/cli/reth/node.mdx @@ -39,11 +39,23 @@ Options: Print help (see a summary with '-h') Metrics: - --metrics + --metrics Enable Prometheus metrics. The metrics will be served at the given interface and port. + --metrics.prometheus.push.url + URL for pushing Prometheus metrics to a push gateway. + + If set, the node will periodically push metrics to the specified push gateway URL. + + --metrics.prometheus.push.interval + Interval in seconds for pushing metrics to push gateway. + + Default: 5 seconds + + [default: 5] + Datadir: --datadir The path to the data dir for all reth files and subdirectories. @@ -233,6 +245,24 @@ Networking: [default: All] + --disable-tx-gossip + Disable transaction pool gossip + + Disables gossiping of transactions in the mempool to peers. This can be omitted for personal nodes, though providers should always opt to enable this flag. + + --tx-propagation-mode + Sets the transaction propagation mode by determining how new pending transactions are propagated to other peers in full. + + Examples: sqrt, all, max:10 + + [default: sqrt] + + --required-block-hashes + Comma separated list of required block hashes. Peers that don't have these blocks will be filtered out + + --network-id + Optional network ID to override the chain specification's network ID for P2P connections + RPC: --http Enable the HTTP-RPC server @@ -247,6 +277,9 @@ RPC: [default: 8545] + --http.disable-compression + Disable compression for HTTP responses + --http.api Rpc Modules to be configured for the HTTP server @@ -284,6 +317,11 @@ RPC: [default: .ipc] + --ipc.permissions + Set the permissions for the IPC socket file, in octal format. + + If not specified, the permissions will be set by the system's umask. + --authrpc.addr Auth server address to listen on @@ -309,6 +347,11 @@ RPC: [default: _engine_api.ipc] + --disable-auth-server + Disable the auth/engine API server. + + This will prevent the authenticated engine-API server from starting. Use this if you're running a node that doesn't need to serve engine API requests. + --rpc.jwtsecret Hex encoded JWT secret to authenticate the regular RPC server(s), see `--http.api` and `--ws.api`. @@ -323,7 +366,7 @@ RPC: Set the maximum RPC response payload size for both HTTP and WS in megabytes [default: 160] - [aliases: rpc.returndata.limit] + [aliases: --rpc.returndata.limit] --rpc.max-subscriptions-per-connection Set the maximum concurrent subscriptions per connection @@ -362,8 +405,13 @@ RPC: [default: 50000000] + --rpc.evm-memory-limit + Maximum memory the EVM can allocate per RPC request + + [default: 4294967295] + --rpc.txfeecap - Maximum eth transaction fee that can be sent via the RPC APIs (0 = no cap) + Maximum eth transaction fee (in ether) that can be sent via the RPC APIs (0 = no cap) [default: 1.0] @@ -382,6 +430,16 @@ RPC: [default: 25] + --rpc.pending-block + Configures the pending block behavior for RPC responses. + + Options: full (include all transactions), empty (header only), none (disable pending blocks). + + [default: full] + + --rpc.forwarder + Endpoint to forward transactions to + --builder.disallow Path to file containing disallowed addresses, json-encoded list of strings. Block validation API will reject blocks containing transactions from these addresses @@ -427,6 +485,14 @@ Gas Price Oracle: [default: 60] + --gpo.default-suggested-fee + The default gas price to use if there are no blocks to use + + --rpc.send-raw-transaction-sync-timeout + Timeout for `send_raw_transaction_sync` RPC method + + [default: 30s] + TxPool: --txpool.pending-max-count Max number of transaction in the pending sub-pool @@ -486,11 +552,17 @@ TxPool: [default: 7] + --txpool.minimum-priority-fee + Minimum priority fee required for transaction acceptance into the pool. Transactions with priority fee below this value will be rejected + --txpool.gas-limit The default enforced gas limit for transactions entering the pool [default: 30000000] + --txpool.max-tx-gas + Maximum gas limit for individual transactions. Transactions exceeding this limit will be rejected by the transaction pool + --blobpool.pricebump Price bump percentage to replace an already existing blob transaction @@ -546,6 +618,11 @@ TxPool: --txpool.disable-transactions-backup Disables transaction backup to disk on node shutdown + --txpool.max-batch-size + Max batch size for transaction pool insertions + + [default: 1] + Builder: --builder.extradata Block extra data set by the payload builder @@ -555,8 +632,6 @@ Builder: --builder.gaslimit Target gas limit for built blocks - [default: 36000000] - --builder.interval The interval at which the job should build a new payload after the last. @@ -589,8 +664,8 @@ Debug: --debug.etherscan [] Runs a fake consensus client that advances the chain using recent block hashes on Etherscan. If specified, requires an `ETHERSCAN_API_KEY` environment variable - --debug.rpc-consensus-ws - Runs a fake consensus client using blocks fetched from an RPC `WebSocket` endpoint + --debug.rpc-consensus-url + Runs a fake consensus client using blocks fetched from an RPC endpoint. Supports both HTTP and `WebSocket` endpoints - `WebSocket` endpoints will use subscriptions, while HTTP endpoints will poll for new blocks --debug.skip-fcu If provided, the engine will skip `n` consecutive FCUs @@ -618,6 +693,19 @@ Debug: --debug.healthy-node-rpc-url The RPC URL of a healthy node to use for comparing invalid block hook results against. + Debug setting that enables execution witness comparison for troubleshooting bad blocks. + When enabled, the node will collect execution witnesses from the specified source and + compare them against local execution when a bad block is encountered, helping identify + discrepancies in state execution. + + --ethstats + The URL of the ethstats server to connect to. Example: `nodename:secret@host:port` + + --debug.startup-sync-state-idle + Set the node to idle state when the backfill is not running. + + This makes the `eth_syncing` RPC return "Idle" when the node has just started or finished the backfill, but did not yet receive any new blocks. + Database: --db.log-level Database logging level. Levels higher than "notice" require a debug build @@ -646,6 +734,12 @@ Database: --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout + --db.max-readers + Maximum number of readers allowed to access the database concurrently + + --db.sync-mode + Controls how aggressively the database synchronizes data to disk + Dev testnet: --dev Start the node in dev mode @@ -665,60 +759,77 @@ Dev testnet: Parses strings using [`humantime::parse_duration`] --dev.block-time 12s + --dev.mnemonic + Derive dev accounts from a fixed mnemonic instead of random ones. + + [default: "test test test test test test test test test test test junk"] + Pruning: --full Run full node. Only the most recent [`MINIMUM_PRUNING_DISTANCE`] block states are stored - --block-interval + --prune.block-interval Minimum pruning interval measured in blocks - --prune.senderrecovery.full + --prune.sender-recovery.full Prunes all sender recovery data - --prune.senderrecovery.distance + --prune.sender-recovery.distance Prune sender recovery data before the `head-N` block number. In other words, keep last N + 1 blocks - --prune.senderrecovery.before + --prune.sender-recovery.before Prune sender recovery data before the specified block number. The specified block number is not pruned - --prune.transactionlookup.full + --prune.transaction-lookup.full Prunes all transaction lookup data - --prune.transactionlookup.distance + --prune.transaction-lookup.distance Prune transaction lookup data before the `head-N` block number. In other words, keep last N + 1 blocks - --prune.transactionlookup.before + --prune.transaction-lookup.before Prune transaction lookup data before the specified block number. The specified block number is not pruned --prune.receipts.full Prunes all receipt data + --prune.receipts.pre-merge + Prune receipts before the merge block + --prune.receipts.distance Prune receipts before the `head-N` block number. In other words, keep last N + 1 blocks --prune.receipts.before Prune receipts before the specified block number. The specified block number is not pruned - --prune.accounthistory.full + --prune.receiptslogfilter + Configure receipts log filter. Format: <`address`>:<`prune_mode`>... where <`prune_mode`> can be 'full', 'distance:<`blocks`>', or 'before:<`block_number`>' + + --prune.account-history.full Prunes all account history - --prune.accounthistory.distance + --prune.account-history.distance Prune account before the `head-N` block number. In other words, keep last N + 1 blocks - --prune.accounthistory.before + --prune.account-history.before Prune account history before the specified block number. The specified block number is not pruned - --prune.storagehistory.full + --prune.storage-history.full Prunes all storage history data - --prune.storagehistory.distance + --prune.storage-history.distance Prune storage history before the `head-N` block number. In other words, keep last N + 1 blocks - --prune.storagehistory.before + --prune.storage-history.before Prune storage history before the specified block number. The specified block number is not pruned - --prune.receiptslogfilter - Configure receipts log filter. Format: <`address`>:<`prune_mode`>[,<`address`>:<`prune_mode`>...] Where <`prune_mode`> can be 'full', 'distance:<`blocks`>', or 'before:<`block_number`>' + --prune.bodies.pre-merge + Prune bodies before the merge block + + --prune.bodies.distance + Prune bodies before the `head-N` block number. In other words, keep last N + 1 blocks + + --prune.bodies.before + Prune storage history before the specified block number. The specified block number is not pruned Engine: --engine.persistence-threshold @@ -729,16 +840,16 @@ Engine: --engine.memory-block-buffer-target Configure the target number of blocks to keep in memory - [default: 2] + [default: 0] --engine.legacy-state-root Enable legacy state root - --engine.caching-and-prewarming - CAUTION: This CLI flag has no effect anymore, use --engine.disable-caching-and-prewarming if you want to disable caching and prewarming + --engine.disable-prewarming + Disable parallel prewarming - --engine.disable-caching-and-prewarming - Disable cross-block caching and parallel prewarming + --engine.disable-parallel-sparse-trie + Disable the parallel sparse trie in the engine --engine.state-provider-metrics Enable state provider latency metrics. This allows the engine to collect and report stats about how long state provider calls took during execution, but this does introduce slight overhead to state provider calls @@ -754,16 +865,54 @@ Engine: --engine.accept-execution-requests-hash Enables accepting requests hash instead of an array of requests in `engine_newPayloadV4` - --engine.max-proof-task-concurrency - Configure the maximum number of concurrent proof tasks + --engine.multiproof-chunking + Whether multiproof task should chunk proof targets - [default: 256] + --engine.multiproof-chunk-size + Multiproof task chunk size for proof targets + + [default: 10] --engine.reserved-cpu-cores Configure the number of reserved CPU cores for non-reth processes [default: 1] + --engine.disable-precompile-cache + Disable precompile cache + + --engine.state-root-fallback + Enable state root fallback, useful for testing + + --engine.always-process-payload-attributes-on-canonical-head + Always process payload attributes and begin a payload build process even if `forkchoiceState.headBlockHash` is already the canonical head or an ancestor. See `TreeConfig::always_process_payload_attributes_on_canonical_head` for more details. + + Note: This is a no-op on OP Stack. + + --engine.allow-unwind-canonical-header + Allow unwinding canonical header to ancestor during forkchoice updates. See `TreeConfig::unwind_canonical_header` for more details + + --engine.storage-worker-count + Configure the number of storage proof workers in the Tokio blocking pool. If not specified, defaults to 2x available parallelism, clamped between 2 and 64 + + --engine.account-worker-count + Configure the number of account proof workers in the Tokio blocking pool. If not specified, defaults to the same count as storage workers + +ERA: + --era.enable + Enable import from ERA1 files + + --era.path + The path to a directory for import. + + The ERA1 files are read from the local directory parsing headers and bodies. + + --era.url + The URL to a remote host where the ERA1 files are hosted. + + The ERA1 files are read from the remote host using HTTP GET requests parsing headers + and bodies. + Ress: --ress.enable Enable support for `ress` subprotocol @@ -792,13 +941,13 @@ Logging: --log.stdout.format The format to use for logs written to stdout - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.stdout.filter The filter to use for logs written to stdout @@ -807,13 +956,13 @@ Logging: --log.file.format The format to use for logs written to the log file - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.file.filter The filter to use for logs written to the log file @@ -824,6 +973,11 @@ Logging: [default: /logs] + --log.file.name + The prefix name of the log files + + [default: reth.log] + --log.file.max-size The maximum size (in MB) of one log file @@ -845,13 +999,13 @@ Logging: --color Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting - [default: always] - Possible values: - always: Colors on - - auto: Colors on + - auto: Auto-detect - never: Colors off + [default: always] + Display: -v, --verbosity... Set the minimum log level. @@ -864,4 +1018,37 @@ Display: -q, --quiet Silence all log output + +Tracing: + --tracing-otlp[=] + Enable `Opentelemetry` tracing export to an OTLP endpoint. + + If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + + Example: --tracing-otlp=http://collector:4318/v1/traces + + [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] + + --tracing-otlp-protocol + OTLP transport protocol to use for exporting traces. + + - `http`: expects endpoint path to end with `/v1/traces` - `grpc`: expects endpoint without a path + + Defaults to HTTP if not specified. + + Possible values: + - http: HTTP/Protobuf transport, port 4318, requires `/v1/traces` path + - grpc: gRPC transport, port 4317 + + [env: OTEL_EXPORTER_OTLP_PROTOCOL=] + [default: http] + + --tracing-otlp.filter + Set a filter directive for the OTLP tracer. This controls the verbosity of spans and events sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable. + + Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off + + Defaults to TRACE if not specified. + + [default: debug] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/p2p.mdx b/docs/vocs/docs/pages/cli/reth/p2p.mdx new file mode 100644 index 00000000000..7b37fdfdaa3 --- /dev/null +++ b/docs/vocs/docs/pages/cli/reth/p2p.mdx @@ -0,0 +1,136 @@ +# reth p2p + +P2P Debugging utilities + +```bash +$ reth p2p --help +``` +```txt +Usage: reth p2p [OPTIONS] + +Commands: + header Download block header + body Download block body + rlpx RLPx commands + bootnode Bootnode command + help Print this message or the help of the given subcommand(s) + +Options: + -h, --help + Print help (see a summary with '-h') + +Logging: + --log.stdout.format + The format to use for logs written to stdout + + Possible values: + - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging + - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications + - terminal: Represents terminal-friendly formatting for logs + + [default: terminal] + + --log.stdout.filter + The filter to use for logs written to stdout + + [default: ] + + --log.file.format + The format to use for logs written to the log file + + Possible values: + - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging + - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications + - terminal: Represents terminal-friendly formatting for logs + + [default: terminal] + + --log.file.filter + The filter to use for logs written to the log file + + [default: debug] + + --log.file.directory + The path to put log files in + + [default: /logs] + + --log.file.name + The prefix name of the log files + + [default: reth.log] + + --log.file.max-size + The maximum size (in MB) of one log file + + [default: 200] + + --log.file.max-files + The maximum amount of log files that will be stored. If set to 0, background file logging is disabled + + [default: 5] + + --log.journald + Write logs to journald + + --log.journald.filter + The filter to use for logs written to journald + + [default: error] + + --color + Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting + + Possible values: + - always: Colors on + - auto: Auto-detect + - never: Colors off + + [default: always] + +Display: + -v, --verbosity... + Set the minimum log level. + + -v Errors + -vv Warnings + -vvv Info + -vvvv Debug + -vvvvv Traces (warning: very verbose!) + + -q, --quiet + Silence all log output + +Tracing: + --tracing-otlp[=] + Enable `Opentelemetry` tracing export to an OTLP endpoint. + + If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + + Example: --tracing-otlp=http://collector:4318/v1/traces + + [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] + + --tracing-otlp-protocol + OTLP transport protocol to use for exporting traces. + + - `http`: expects endpoint path to end with `/v1/traces` - `grpc`: expects endpoint without a path + + Defaults to HTTP if not specified. + + Possible values: + - http: HTTP/Protobuf transport, port 4318, requires `/v1/traces` path + - grpc: gRPC transport, port 4317 + + [env: OTEL_EXPORTER_OTLP_PROTOCOL=] + [default: http] + + --tracing-otlp.filter + Set a filter directive for the OTLP tracer. This controls the verbosity of spans and events sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable. + + Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off + + Defaults to TRACE if not specified. + + [default: debug] +``` \ No newline at end of file diff --git a/book/cli/reth/p2p.md b/docs/vocs/docs/pages/cli/reth/p2p/body.mdx similarity index 80% rename from book/cli/reth/p2p.md rename to docs/vocs/docs/pages/cli/reth/p2p/body.mdx index a435c916169..bbe6b375e5b 100644 --- a/book/cli/reth/p2p.md +++ b/docs/vocs/docs/pages/cli/reth/p2p/body.mdx @@ -1,32 +1,14 @@ -# reth p2p +# reth p2p body -P2P Debugging utilities +Download block body ```bash -$ reth p2p --help +$ reth p2p body --help ``` ```txt -Usage: reth p2p [OPTIONS] - -Commands: - header Download block header - body Download block body - rlpx RLPx commands - help Print this message or the help of the given subcommand(s) +Usage: reth p2p body [OPTIONS] Options: - --config - The path to the configuration file to use. - - --chain - The chain this node is running. - Possible values are either a built-in chain or the path to a chain specification file. - - Built-in chains: - mainnet, sepolia, holesky, hoodi, dev - - [default: mainnet] - --retries The number of retries per request @@ -209,6 +191,24 @@ Networking: [default: All] + --disable-tx-gossip + Disable transaction pool gossip + + Disables gossiping of transactions in the mempool to peers. This can be omitted for personal nodes, though providers should always opt to enable this flag. + + --tx-propagation-mode + Sets the transaction propagation mode by determining how new pending transactions are propagated to other peers in full. + + Examples: sqrt, all, max:10 + + [default: sqrt] + + --required-block-hashes + Comma separated list of required block hashes. Peers that don't have these blocks will be filtered out + + --network-id + Optional network ID to override the chain specification's network ID for P2P connections + Datadir: --datadir The path to the data dir for all reth files and subdirectories. @@ -224,45 +224,32 @@ Datadir: --datadir.static-files The absolute path to store static files in. -Database: - --db.log-level - Database logging level. Levels higher than "notice" require a debug build - - Possible values: - - fatal: Enables logging for critical conditions, i.e. assertion failures - - error: Enables logging for error conditions - - warn: Enables logging for warning conditions - - notice: Enables logging for normal but significant condition - - verbose: Enables logging for verbose informational - - debug: Enables logging for debug-level messages - - trace: Enables logging for trace debug-level messages - - extra: Enables logging for extra debug-level messages - - --db.exclusive - Open environment in exclusive/monopolistic mode. Makes it possible to open a database on an NFS volume + --config + The path to the configuration file to use. - [possible values: true, false] + --chain + The chain this node is running. + Possible values are either a built-in chain or the path to a chain specification file. - --db.max-size - Maximum database size (e.g., 4TB, 8MB) + Built-in chains: + mainnet, sepolia, holesky, hoodi, dev - --db.growth-step - Database growth step (e.g., 4GB, 4KB) + [default: mainnet] - --db.read-transaction-timeout - Read transaction timeout in seconds, 0 means no timeout + + The block number or hash Logging: --log.stdout.format The format to use for logs written to stdout - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.stdout.filter The filter to use for logs written to stdout @@ -271,13 +258,13 @@ Logging: --log.file.format The format to use for logs written to the log file - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.file.filter The filter to use for logs written to the log file @@ -288,6 +275,11 @@ Logging: [default: /logs] + --log.file.name + The prefix name of the log files + + [default: reth.log] + --log.file.max-size The maximum size (in MB) of one log file @@ -309,13 +301,13 @@ Logging: --color Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting - [default: always] - Possible values: - always: Colors on - - auto: Colors on + - auto: Auto-detect - never: Colors off + [default: always] + Display: -v, --verbosity... Set the minimum log level. @@ -328,4 +320,37 @@ Display: -q, --quiet Silence all log output + +Tracing: + --tracing-otlp[=] + Enable `Opentelemetry` tracing export to an OTLP endpoint. + + If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + + Example: --tracing-otlp=http://collector:4318/v1/traces + + [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] + + --tracing-otlp-protocol + OTLP transport protocol to use for exporting traces. + + - `http`: expects endpoint path to end with `/v1/traces` - `grpc`: expects endpoint without a path + + Defaults to HTTP if not specified. + + Possible values: + - http: HTTP/Protobuf transport, port 4318, requires `/v1/traces` path + - grpc: gRPC transport, port 4317 + + [env: OTEL_EXPORTER_OTLP_PROTOCOL=] + [default: http] + + --tracing-otlp.filter + Set a filter directive for the OTLP tracer. This controls the verbosity of spans and events sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable. + + Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off + + Defaults to TRACE if not specified. + + [default: debug] ``` \ No newline at end of file diff --git a/book/cli/reth/import-op.md b/docs/vocs/docs/pages/cli/reth/p2p/bootnode.mdx similarity index 55% rename from book/cli/reth/import-op.md rename to docs/vocs/docs/pages/cli/reth/p2p/bootnode.mdx index d2d81980ce3..324b01daac5 100644 --- a/book/cli/reth/import-op.md +++ b/docs/vocs/docs/pages/cli/reth/p2p/bootnode.mdx @@ -1,71 +1,46 @@ -# op-reth import +# reth p2p bootnode -This syncs RLP encoded blocks from a file. Supports import of OVM blocks -from the Bedrock datadir. Requires blocks, up to same height as receipts -file, to already be imported. +Bootnode command ```bash -$ op-reth import-op --help -Usage: op-reth import-op [OPTIONS] +$ reth p2p bootnode --help +``` +```txt +Usage: reth p2p bootnode [OPTIONS] Options: - --config - The path to the configuration file to use. + --addr + Listen address for the bootnode (default: "0.0.0.0:30301") - --datadir - The path to the data dir for all reth files and subdirectories. + [default: 0.0.0.0:30301] - Defaults to the OS-specific data directory: + --p2p-secret-key + Secret key to use for the bootnode. - - Linux: `$XDG_DATA_HOME/reth/` or `$HOME/.local/share/reth/` - - Windows: `{FOLDERID_RoamingAppData}/reth/` - - macOS: `$HOME/Library/Application Support/reth/` + This will also deterministically set the peer ID. If a path is provided but no key exists at that path, a new random secret will be generated and stored there. If no path is specified, a new ephemeral random secret will be used. - [default: default] + --nat + NAT resolution method (any|none|upnp|publicip|extip:\) - --chunk-len - Chunk byte length to read from file. + [default: any] - [default: 1GB] + --v5 + Run a v5 topic discovery bootnode -h, --help Print help (see a summary with '-h') -Database: - --db.log-level - Database logging level. Levels higher than "notice" require a debug build - - Possible values: - - fatal: Enables logging for critical conditions, i.e. assertion failures - - error: Enables logging for error conditions - - warn: Enables logging for warning conditions - - notice: Enables logging for normal but significant condition - - verbose: Enables logging for verbose informational - - debug: Enables logging for debug-level messages - - trace: Enables logging for trace debug-level messages - - extra: Enables logging for extra debug-level messages - - --db.exclusive - Open environment in exclusive/monopolistic mode. Makes it possible to open a database on an NFS volume - - [possible values: true, false] - - - The path to a `.rlp` block file for import. - - The online sync pipeline stages (headers and bodies) are replaced by a file import. Skips block execution since blocks below Bedrock are built on OVM. - Logging: --log.stdout.format The format to use for logs written to stdout - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.stdout.filter The filter to use for logs written to stdout @@ -74,13 +49,13 @@ Logging: --log.file.format The format to use for logs written to the log file - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.file.filter The filter to use for logs written to the log file @@ -91,6 +66,11 @@ Logging: [default: /logs] + --log.file.name + The prefix name of the log files + + [default: reth.log] + --log.file.max-size The maximum size (in MB) of one log file @@ -112,13 +92,13 @@ Logging: --color Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting - [default: always] - Possible values: - always: Colors on - - auto: Colors on + - auto: Auto-detect - never: Colors off + [default: always] + Display: -v, --verbosity... Set the minimum log level. @@ -131,4 +111,37 @@ Display: -q, --quiet Silence all log output + +Tracing: + --tracing-otlp[=] + Enable `Opentelemetry` tracing export to an OTLP endpoint. + + If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + + Example: --tracing-otlp=http://collector:4318/v1/traces + + [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] + + --tracing-otlp-protocol + OTLP transport protocol to use for exporting traces. + + - `http`: expects endpoint path to end with `/v1/traces` - `grpc`: expects endpoint without a path + + Defaults to HTTP if not specified. + + Possible values: + - http: HTTP/Protobuf transport, port 4318, requires `/v1/traces` path + - grpc: gRPC transport, port 4317 + + [env: OTEL_EXPORTER_OTLP_PROTOCOL=] + [default: http] + + --tracing-otlp.filter + Set a filter directive for the OTLP tracer. This controls the verbosity of spans and events sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable. + + Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off + + Defaults to TRACE if not specified. + + [default: debug] ``` \ No newline at end of file diff --git a/book/cli/reth/debug/merkle.md b/docs/vocs/docs/pages/cli/reth/p2p/header.mdx similarity index 79% rename from book/cli/reth/debug/merkle.md rename to docs/vocs/docs/pages/cli/reth/p2p/header.mdx index 03b16a35e38..533bd71de2e 100644 --- a/book/cli/reth/debug/merkle.md +++ b/docs/vocs/docs/pages/cli/reth/p2p/header.mdx @@ -1,71 +1,21 @@ -# reth debug merkle +# reth p2p header -Debug the clean & incremental state root calculations +Download block header ```bash -$ reth debug merkle --help +$ reth p2p header --help ``` ```txt -Usage: reth debug merkle [OPTIONS] --to +Usage: reth p2p header [OPTIONS] Options: - -h, --help - Print help (see a summary with '-h') - -Datadir: - --datadir - The path to the data dir for all reth files and subdirectories. - - Defaults to the OS-specific data directory: - - - Linux: `$XDG_DATA_HOME/reth/` or `$HOME/.local/share/reth/` - - Windows: `{FOLDERID_RoamingAppData}/reth/` - - macOS: `$HOME/Library/Application Support/reth/` - - [default: default] - - --datadir.static-files - The absolute path to store static files in. - - --config - The path to the configuration file to use - - --chain - The chain this node is running. - Possible values are either a built-in chain or the path to a chain specification file. - - Built-in chains: - mainnet, sepolia, holesky, hoodi, dev - - [default: mainnet] - -Database: - --db.log-level - Database logging level. Levels higher than "notice" require a debug build - - Possible values: - - fatal: Enables logging for critical conditions, i.e. assertion failures - - error: Enables logging for error conditions - - warn: Enables logging for warning conditions - - notice: Enables logging for normal but significant condition - - verbose: Enables logging for verbose informational - - debug: Enables logging for debug-level messages - - trace: Enables logging for trace debug-level messages - - extra: Enables logging for extra debug-level messages - - --db.exclusive - Open environment in exclusive/monopolistic mode. Makes it possible to open a database on an NFS volume - - [possible values: true, false] - - --db.max-size - Maximum database size (e.g., 4TB, 8MB) + --retries + The number of retries per request - --db.growth-step - Database growth step (e.g., 4GB, 4KB) + [default: 5] - --db.read-transaction-timeout - Read transaction timeout in seconds, 0 means no timeout + -h, --help + Print help (see a summary with '-h') Networking: -d, --disable-discovery @@ -241,28 +191,65 @@ Networking: [default: All] - --retries - The number of retries per request + --disable-tx-gossip + Disable transaction pool gossip - [default: 5] + Disables gossiping of transactions in the mempool to peers. This can be omitted for personal nodes, though providers should always opt to enable this flag. + + --tx-propagation-mode + Sets the transaction propagation mode by determining how new pending transactions are propagated to other peers in full. + + Examples: sqrt, all, max:10 + + [default: sqrt] - --to - The height to finish at + --required-block-hashes + Comma separated list of required block hashes. Peers that don't have these blocks will be filtered out - --skip-node-depth - The depth after which we should start comparing branch nodes + --network-id + Optional network ID to override the chain specification's network ID for P2P connections + +Datadir: + --datadir + The path to the data dir for all reth files and subdirectories. + + Defaults to the OS-specific data directory: + + - Linux: `$XDG_DATA_HOME/reth/` or `$HOME/.local/share/reth/` + - Windows: `{FOLDERID_RoamingAppData}/reth/` + - macOS: `$HOME/Library/Application Support/reth/` + + [default: default] + + --datadir.static-files + The absolute path to store static files in. + + --config + The path to the configuration file to use. + + --chain + The chain this node is running. + Possible values are either a built-in chain or the path to a chain specification file. + + Built-in chains: + mainnet, sepolia, holesky, hoodi, dev + + [default: mainnet] + + + The header number or hash Logging: --log.stdout.format The format to use for logs written to stdout - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.stdout.filter The filter to use for logs written to stdout @@ -271,13 +258,13 @@ Logging: --log.file.format The format to use for logs written to the log file - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.file.filter The filter to use for logs written to the log file @@ -288,6 +275,11 @@ Logging: [default: /logs] + --log.file.name + The prefix name of the log files + + [default: reth.log] + --log.file.max-size The maximum size (in MB) of one log file @@ -309,13 +301,13 @@ Logging: --color Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting - [default: always] - Possible values: - always: Colors on - - auto: Colors on + - auto: Auto-detect - never: Colors off + [default: always] + Display: -v, --verbosity... Set the minimum log level. @@ -328,4 +320,37 @@ Display: -q, --quiet Silence all log output + +Tracing: + --tracing-otlp[=] + Enable `Opentelemetry` tracing export to an OTLP endpoint. + + If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + + Example: --tracing-otlp=http://collector:4318/v1/traces + + [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] + + --tracing-otlp-protocol + OTLP transport protocol to use for exporting traces. + + - `http`: expects endpoint path to end with `/v1/traces` - `grpc`: expects endpoint without a path + + Defaults to HTTP if not specified. + + Possible values: + - http: HTTP/Protobuf transport, port 4318, requires `/v1/traces` path + - grpc: gRPC transport, port 4317 + + [env: OTEL_EXPORTER_OTLP_PROTOCOL=] + [default: http] + + --tracing-otlp.filter + Set a filter directive for the OTLP tracer. This controls the verbosity of spans and events sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable. + + Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off + + Defaults to TRACE if not specified. + + [default: debug] ``` \ No newline at end of file diff --git a/book/cli/reth/p2p/rlpx.md b/docs/vocs/docs/pages/cli/reth/p2p/rlpx.mdx similarity index 66% rename from book/cli/reth/p2p/rlpx.md rename to docs/vocs/docs/pages/cli/reth/p2p/rlpx.mdx index 484a8005cbd..a8ac7fbd0df 100644 --- a/book/cli/reth/p2p/rlpx.md +++ b/docs/vocs/docs/pages/cli/reth/p2p/rlpx.mdx @@ -20,13 +20,13 @@ Logging: --log.stdout.format The format to use for logs written to stdout - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.stdout.filter The filter to use for logs written to stdout @@ -35,13 +35,13 @@ Logging: --log.file.format The format to use for logs written to the log file - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.file.filter The filter to use for logs written to the log file @@ -52,6 +52,11 @@ Logging: [default: /logs] + --log.file.name + The prefix name of the log files + + [default: reth.log] + --log.file.max-size The maximum size (in MB) of one log file @@ -73,13 +78,13 @@ Logging: --color Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting - [default: always] - Possible values: - always: Colors on - - auto: Colors on + - auto: Auto-detect - never: Colors off + [default: always] + Display: -v, --verbosity... Set the minimum log level. @@ -92,4 +97,37 @@ Display: -q, --quiet Silence all log output + +Tracing: + --tracing-otlp[=] + Enable `Opentelemetry` tracing export to an OTLP endpoint. + + If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + + Example: --tracing-otlp=http://collector:4318/v1/traces + + [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] + + --tracing-otlp-protocol + OTLP transport protocol to use for exporting traces. + + - `http`: expects endpoint path to end with `/v1/traces` - `grpc`: expects endpoint without a path + + Defaults to HTTP if not specified. + + Possible values: + - http: HTTP/Protobuf transport, port 4318, requires `/v1/traces` path + - grpc: gRPC transport, port 4317 + + [env: OTEL_EXPORTER_OTLP_PROTOCOL=] + [default: http] + + --tracing-otlp.filter + Set a filter directive for the OTLP tracer. This controls the verbosity of spans and events sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable. + + Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off + + Defaults to TRACE if not specified. + + [default: debug] ``` \ No newline at end of file diff --git a/book/cli/reth/p2p/rlpx/ping.md b/docs/vocs/docs/pages/cli/reth/p2p/rlpx/ping.mdx similarity index 65% rename from book/cli/reth/p2p/rlpx/ping.md rename to docs/vocs/docs/pages/cli/reth/p2p/rlpx/ping.mdx index 5bedf145f3a..2d136630298 100644 --- a/book/cli/reth/p2p/rlpx/ping.md +++ b/docs/vocs/docs/pages/cli/reth/p2p/rlpx/ping.mdx @@ -20,13 +20,13 @@ Logging: --log.stdout.format The format to use for logs written to stdout - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.stdout.filter The filter to use for logs written to stdout @@ -35,13 +35,13 @@ Logging: --log.file.format The format to use for logs written to the log file - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.file.filter The filter to use for logs written to the log file @@ -52,6 +52,11 @@ Logging: [default: /logs] + --log.file.name + The prefix name of the log files + + [default: reth.log] + --log.file.max-size The maximum size (in MB) of one log file @@ -73,13 +78,13 @@ Logging: --color Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting - [default: always] - Possible values: - always: Colors on - - auto: Colors on + - auto: Auto-detect - never: Colors off + [default: always] + Display: -v, --verbosity... Set the minimum log level. @@ -92,4 +97,37 @@ Display: -q, --quiet Silence all log output + +Tracing: + --tracing-otlp[=] + Enable `Opentelemetry` tracing export to an OTLP endpoint. + + If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + + Example: --tracing-otlp=http://collector:4318/v1/traces + + [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] + + --tracing-otlp-protocol + OTLP transport protocol to use for exporting traces. + + - `http`: expects endpoint path to end with `/v1/traces` - `grpc`: expects endpoint without a path + + Defaults to HTTP if not specified. + + Possible values: + - http: HTTP/Protobuf transport, port 4318, requires `/v1/traces` path + - grpc: gRPC transport, port 4317 + + [env: OTEL_EXPORTER_OTLP_PROTOCOL=] + [default: http] + + --tracing-otlp.filter + Set a filter directive for the OTLP tracer. This controls the verbosity of spans and events sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable. + + Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off + + Defaults to TRACE if not specified. + + [default: debug] ``` \ No newline at end of file diff --git a/book/cli/reth/prune.md b/docs/vocs/docs/pages/cli/reth/prune.mdx similarity index 74% rename from book/cli/reth/prune.md rename to docs/vocs/docs/pages/cli/reth/prune.mdx index 5b604fa6ce7..2d586edd5c3 100644 --- a/book/cli/reth/prune.md +++ b/docs/vocs/docs/pages/cli/reth/prune.mdx @@ -67,17 +67,23 @@ Database: --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout + --db.max-readers + Maximum number of readers allowed to access the database concurrently + + --db.sync-mode + Controls how aggressively the database synchronizes data to disk + Logging: --log.stdout.format The format to use for logs written to stdout - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.stdout.filter The filter to use for logs written to stdout @@ -86,13 +92,13 @@ Logging: --log.file.format The format to use for logs written to the log file - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.file.filter The filter to use for logs written to the log file @@ -103,6 +109,11 @@ Logging: [default: /logs] + --log.file.name + The prefix name of the log files + + [default: reth.log] + --log.file.max-size The maximum size (in MB) of one log file @@ -124,13 +135,13 @@ Logging: --color Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting - [default: always] - Possible values: - always: Colors on - - auto: Colors on + - auto: Auto-detect - never: Colors off + [default: always] + Display: -v, --verbosity... Set the minimum log level. @@ -143,4 +154,37 @@ Display: -q, --quiet Silence all log output + +Tracing: + --tracing-otlp[=] + Enable `Opentelemetry` tracing export to an OTLP endpoint. + + If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + + Example: --tracing-otlp=http://collector:4318/v1/traces + + [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] + + --tracing-otlp-protocol + OTLP transport protocol to use for exporting traces. + + - `http`: expects endpoint path to end with `/v1/traces` - `grpc`: expects endpoint without a path + + Defaults to HTTP if not specified. + + Possible values: + - http: HTTP/Protobuf transport, port 4318, requires `/v1/traces` path + - grpc: gRPC transport, port 4317 + + [env: OTEL_EXPORTER_OTLP_PROTOCOL=] + [default: http] + + --tracing-otlp.filter + Set a filter directive for the OTLP tracer. This controls the verbosity of spans and events sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable. + + Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off + + Defaults to TRACE if not specified. + + [default: debug] ``` \ No newline at end of file diff --git a/book/cli/reth/stage/drop.md b/docs/vocs/docs/pages/cli/reth/re-execute.mdx similarity index 69% rename from book/cli/reth/stage/drop.md rename to docs/vocs/docs/pages/cli/reth/re-execute.mdx index 5d3312b3ea0..e07b3f542c3 100644 --- a/book/cli/reth/stage/drop.md +++ b/docs/vocs/docs/pages/cli/reth/re-execute.mdx @@ -1,12 +1,12 @@ -# reth stage drop +# reth re-execute -Drop a stage's tables from the database +Re-execute blocks in parallel to verify historical sync correctness ```bash -$ reth stage drop --help +$ reth re-execute --help ``` ```txt -Usage: reth stage drop [OPTIONS] +Usage: reth re-execute [OPTIONS] Options: -h, --help @@ -67,31 +67,36 @@ Database: --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout - - Possible values: - - headers: The headers stage within the pipeline - - bodies: The bodies stage within the pipeline - - senders: The senders stage within the pipeline - - execution: The execution stage within the pipeline - - account-hashing: The account hashing stage within the pipeline - - storage-hashing: The storage hashing stage within the pipeline - - hashing: The account and storage hashing stages within the pipeline - - merkle: The merkle stage within the pipeline - - tx-lookup: The transaction lookup stage within the pipeline - - account-history: The account history stage within the pipeline - - storage-history: The storage history stage within the pipeline + --db.max-readers + Maximum number of readers allowed to access the database concurrently + + --db.sync-mode + Controls how aggressively the database synchronizes data to disk + + --from + The height to start at + + [default: 1] + + --to + The height to end at. Defaults to the latest block + + --num-tasks + Number of tasks to run in parallel + + [default: 10] Logging: --log.stdout.format The format to use for logs written to stdout - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.stdout.filter The filter to use for logs written to stdout @@ -100,13 +105,13 @@ Logging: --log.file.format The format to use for logs written to the log file - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.file.filter The filter to use for logs written to the log file @@ -117,6 +122,11 @@ Logging: [default: /logs] + --log.file.name + The prefix name of the log files + + [default: reth.log] + --log.file.max-size The maximum size (in MB) of one log file @@ -138,13 +148,13 @@ Logging: --color Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting - [default: always] - Possible values: - always: Colors on - - auto: Colors on + - auto: Auto-detect - never: Colors off + [default: always] + Display: -v, --verbosity... Set the minimum log level. @@ -157,4 +167,37 @@ Display: -q, --quiet Silence all log output + +Tracing: + --tracing-otlp[=] + Enable `Opentelemetry` tracing export to an OTLP endpoint. + + If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + + Example: --tracing-otlp=http://collector:4318/v1/traces + + [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] + + --tracing-otlp-protocol + OTLP transport protocol to use for exporting traces. + + - `http`: expects endpoint path to end with `/v1/traces` - `grpc`: expects endpoint without a path + + Defaults to HTTP if not specified. + + Possible values: + - http: HTTP/Protobuf transport, port 4318, requires `/v1/traces` path + - grpc: gRPC transport, port 4317 + + [env: OTEL_EXPORTER_OTLP_PROTOCOL=] + [default: http] + + --tracing-otlp.filter + Set a filter directive for the OTLP tracer. This controls the verbosity of spans and events sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable. + + Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off + + Defaults to TRACE if not specified. + + [default: debug] ``` \ No newline at end of file diff --git a/book/cli/reth/recover.md b/docs/vocs/docs/pages/cli/reth/recover.mdx similarity index 96% rename from book/cli/reth/recover.md rename to docs/vocs/docs/pages/cli/reth/recover.mdx index af5b685ab7c..880b8482d01 100644 --- a/book/cli/reth/recover.md +++ b/docs/vocs/docs/pages/cli/reth/recover.mdx @@ -20,13 +20,13 @@ Logging: --log.stdout.format The format to use for logs written to stdout - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.stdout.filter The filter to use for logs written to stdout @@ -35,13 +35,13 @@ Logging: --log.file.format The format to use for logs written to the log file - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.file.filter The filter to use for logs written to the log file @@ -52,6 +52,11 @@ Logging: [default: /logs] + --log.file.name + The prefix name of the log files + + [default: reth.log] + --log.file.max-size The maximum size (in MB) of one log file @@ -73,13 +78,13 @@ Logging: --color Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting - [default: always] - Possible values: - always: Colors on - auto: Colors on - never: Colors off + [default: always] + Display: -v, --verbosity... Set the minimum log level. diff --git a/book/cli/reth/recover/storage-tries.md b/docs/vocs/docs/pages/cli/reth/recover/storage-tries.mdx similarity index 95% rename from book/cli/reth/recover/storage-tries.md rename to docs/vocs/docs/pages/cli/reth/recover/storage-tries.mdx index aafce289076..701dd393686 100644 --- a/book/cli/reth/recover/storage-tries.md +++ b/docs/vocs/docs/pages/cli/reth/recover/storage-tries.mdx @@ -67,17 +67,20 @@ Database: --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout + --db.max-readers + Maximum number of readers allowed to access the database concurrently + Logging: --log.stdout.format The format to use for logs written to stdout - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.stdout.filter The filter to use for logs written to stdout @@ -86,13 +89,13 @@ Logging: --log.file.format The format to use for logs written to the log file - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.file.filter The filter to use for logs written to the log file @@ -103,6 +106,11 @@ Logging: [default: /logs] + --log.file.name + The prefix name of the log files + + [default: reth.log] + --log.file.max-size The maximum size (in MB) of one log file @@ -124,13 +132,13 @@ Logging: --color Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting - [default: always] - Possible values: - always: Colors on - auto: Colors on - never: Colors off + [default: always] + Display: -v, --verbosity... Set the minimum log level. diff --git a/book/cli/reth/stage.md b/docs/vocs/docs/pages/cli/reth/stage.mdx similarity index 67% rename from book/cli/reth/stage.md rename to docs/vocs/docs/pages/cli/reth/stage.mdx index f2fa612b097..006c6c74340 100644 --- a/book/cli/reth/stage.md +++ b/docs/vocs/docs/pages/cli/reth/stage.mdx @@ -23,13 +23,13 @@ Logging: --log.stdout.format The format to use for logs written to stdout - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.stdout.filter The filter to use for logs written to stdout @@ -38,13 +38,13 @@ Logging: --log.file.format The format to use for logs written to the log file - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.file.filter The filter to use for logs written to the log file @@ -55,6 +55,11 @@ Logging: [default: /logs] + --log.file.name + The prefix name of the log files + + [default: reth.log] + --log.file.max-size The maximum size (in MB) of one log file @@ -76,13 +81,13 @@ Logging: --color Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting - [default: always] - Possible values: - always: Colors on - - auto: Colors on + - auto: Auto-detect - never: Colors off + [default: always] + Display: -v, --verbosity... Set the minimum log level. @@ -95,4 +100,37 @@ Display: -q, --quiet Silence all log output + +Tracing: + --tracing-otlp[=] + Enable `Opentelemetry` tracing export to an OTLP endpoint. + + If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + + Example: --tracing-otlp=http://collector:4318/v1/traces + + [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] + + --tracing-otlp-protocol + OTLP transport protocol to use for exporting traces. + + - `http`: expects endpoint path to end with `/v1/traces` - `grpc`: expects endpoint without a path + + Defaults to HTTP if not specified. + + Possible values: + - http: HTTP/Protobuf transport, port 4318, requires `/v1/traces` path + - grpc: gRPC transport, port 4317 + + [env: OTEL_EXPORTER_OTLP_PROTOCOL=] + [default: http] + + --tracing-otlp.filter + Set a filter directive for the OTLP tracer. This controls the verbosity of spans and events sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable. + + Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off + + Defaults to TRACE if not specified. + + [default: debug] ``` \ No newline at end of file diff --git a/docs/vocs/docs/pages/cli/reth/stage/drop.mdx b/docs/vocs/docs/pages/cli/reth/stage/drop.mdx new file mode 100644 index 00000000000..c14db19c58c --- /dev/null +++ b/docs/vocs/docs/pages/cli/reth/stage/drop.mdx @@ -0,0 +1,205 @@ +# reth stage drop + +Drop a stage's tables from the database + +```bash +$ reth stage drop --help +``` +```txt +Usage: reth stage drop [OPTIONS] + +Options: + -h, --help + Print help (see a summary with '-h') + +Datadir: + --datadir + The path to the data dir for all reth files and subdirectories. + + Defaults to the OS-specific data directory: + + - Linux: `$XDG_DATA_HOME/reth/` or `$HOME/.local/share/reth/` + - Windows: `{FOLDERID_RoamingAppData}/reth/` + - macOS: `$HOME/Library/Application Support/reth/` + + [default: default] + + --datadir.static-files + The absolute path to store static files in. + + --config + The path to the configuration file to use + + --chain + The chain this node is running. + Possible values are either a built-in chain or the path to a chain specification file. + + Built-in chains: + mainnet, sepolia, holesky, hoodi, dev + + [default: mainnet] + +Database: + --db.log-level + Database logging level. Levels higher than "notice" require a debug build + + Possible values: + - fatal: Enables logging for critical conditions, i.e. assertion failures + - error: Enables logging for error conditions + - warn: Enables logging for warning conditions + - notice: Enables logging for normal but significant condition + - verbose: Enables logging for verbose informational + - debug: Enables logging for debug-level messages + - trace: Enables logging for trace debug-level messages + - extra: Enables logging for extra debug-level messages + + --db.exclusive + Open environment in exclusive/monopolistic mode. Makes it possible to open a database on an NFS volume + + [possible values: true, false] + + --db.max-size + Maximum database size (e.g., 4TB, 8MB) + + --db.growth-step + Database growth step (e.g., 4GB, 4KB) + + --db.read-transaction-timeout + Read transaction timeout in seconds, 0 means no timeout + + --db.max-readers + Maximum number of readers allowed to access the database concurrently + + --db.sync-mode + Controls how aggressively the database synchronizes data to disk + + + Possible values: + - headers: The headers stage within the pipeline + - bodies: The bodies stage within the pipeline + - senders: The senders stage within the pipeline + - execution: The execution stage within the pipeline + - account-hashing: The account hashing stage within the pipeline + - storage-hashing: The storage hashing stage within the pipeline + - hashing: The account and storage hashing stages within the pipeline + - merkle: The merkle stage within the pipeline + - merkle-changesets: The merkle changesets stage within the pipeline + - tx-lookup: The transaction lookup stage within the pipeline + - account-history: The account history stage within the pipeline + - storage-history: The storage history stage within the pipeline + +Logging: + --log.stdout.format + The format to use for logs written to stdout + + Possible values: + - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging + - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications + - terminal: Represents terminal-friendly formatting for logs + + [default: terminal] + + --log.stdout.filter + The filter to use for logs written to stdout + + [default: ] + + --log.file.format + The format to use for logs written to the log file + + Possible values: + - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging + - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications + - terminal: Represents terminal-friendly formatting for logs + + [default: terminal] + + --log.file.filter + The filter to use for logs written to the log file + + [default: debug] + + --log.file.directory + The path to put log files in + + [default: /logs] + + --log.file.name + The prefix name of the log files + + [default: reth.log] + + --log.file.max-size + The maximum size (in MB) of one log file + + [default: 200] + + --log.file.max-files + The maximum amount of log files that will be stored. If set to 0, background file logging is disabled + + [default: 5] + + --log.journald + Write logs to journald + + --log.journald.filter + The filter to use for logs written to journald + + [default: error] + + --color + Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting + + Possible values: + - always: Colors on + - auto: Auto-detect + - never: Colors off + + [default: always] + +Display: + -v, --verbosity... + Set the minimum log level. + + -v Errors + -vv Warnings + -vvv Info + -vvvv Debug + -vvvvv Traces (warning: very verbose!) + + -q, --quiet + Silence all log output + +Tracing: + --tracing-otlp[=] + Enable `Opentelemetry` tracing export to an OTLP endpoint. + + If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + + Example: --tracing-otlp=http://collector:4318/v1/traces + + [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] + + --tracing-otlp-protocol + OTLP transport protocol to use for exporting traces. + + - `http`: expects endpoint path to end with `/v1/traces` - `grpc`: expects endpoint without a path + + Defaults to HTTP if not specified. + + Possible values: + - http: HTTP/Protobuf transport, port 4318, requires `/v1/traces` path + - grpc: gRPC transport, port 4317 + + [env: OTEL_EXPORTER_OTLP_PROTOCOL=] + [default: http] + + --tracing-otlp.filter + Set a filter directive for the OTLP tracer. This controls the verbosity of spans and events sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable. + + Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off + + Defaults to TRACE if not specified. + + [default: debug] +``` \ No newline at end of file diff --git a/book/cli/reth/stage/dump.md b/docs/vocs/docs/pages/cli/reth/stage/dump.mdx similarity index 75% rename from book/cli/reth/stage/dump.md rename to docs/vocs/docs/pages/cli/reth/stage/dump.mdx index 8af42029fa0..c29547401be 100644 --- a/book/cli/reth/stage/dump.md +++ b/docs/vocs/docs/pages/cli/reth/stage/dump.mdx @@ -74,17 +74,23 @@ Database: --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout + --db.max-readers + Maximum number of readers allowed to access the database concurrently + + --db.sync-mode + Controls how aggressively the database synchronizes data to disk + Logging: --log.stdout.format The format to use for logs written to stdout - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.stdout.filter The filter to use for logs written to stdout @@ -93,13 +99,13 @@ Logging: --log.file.format The format to use for logs written to the log file - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.file.filter The filter to use for logs written to the log file @@ -110,6 +116,11 @@ Logging: [default: /logs] + --log.file.name + The prefix name of the log files + + [default: reth.log] + --log.file.max-size The maximum size (in MB) of one log file @@ -131,13 +142,13 @@ Logging: --color Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting - [default: always] - Possible values: - always: Colors on - - auto: Colors on + - auto: Auto-detect - never: Colors off + [default: always] + Display: -v, --verbosity... Set the minimum log level. @@ -150,4 +161,37 @@ Display: -q, --quiet Silence all log output + +Tracing: + --tracing-otlp[=] + Enable `Opentelemetry` tracing export to an OTLP endpoint. + + If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + + Example: --tracing-otlp=http://collector:4318/v1/traces + + [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] + + --tracing-otlp-protocol + OTLP transport protocol to use for exporting traces. + + - `http`: expects endpoint path to end with `/v1/traces` - `grpc`: expects endpoint without a path + + Defaults to HTTP if not specified. + + Possible values: + - http: HTTP/Protobuf transport, port 4318, requires `/v1/traces` path + - grpc: gRPC transport, port 4317 + + [env: OTEL_EXPORTER_OTLP_PROTOCOL=] + [default: http] + + --tracing-otlp.filter + Set a filter directive for the OTLP tracer. This controls the verbosity of spans and events sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable. + + Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off + + Defaults to TRACE if not specified. + + [default: debug] ``` \ No newline at end of file diff --git a/book/cli/reth/stage/dump/account-hashing.md b/docs/vocs/docs/pages/cli/reth/stage/dump/account-hashing.mdx similarity index 70% rename from book/cli/reth/stage/dump/account-hashing.md rename to docs/vocs/docs/pages/cli/reth/stage/dump/account-hashing.mdx index 6b5b97250ec..70fad94ea3a 100644 --- a/book/cli/reth/stage/dump/account-hashing.md +++ b/docs/vocs/docs/pages/cli/reth/stage/dump/account-hashing.mdx @@ -38,13 +38,13 @@ Logging: --log.stdout.format The format to use for logs written to stdout - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.stdout.filter The filter to use for logs written to stdout @@ -53,13 +53,13 @@ Logging: --log.file.format The format to use for logs written to the log file - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.file.filter The filter to use for logs written to the log file @@ -70,6 +70,11 @@ Logging: [default: /logs] + --log.file.name + The prefix name of the log files + + [default: reth.log] + --log.file.max-size The maximum size (in MB) of one log file @@ -91,13 +96,13 @@ Logging: --color Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting - [default: always] - Possible values: - always: Colors on - - auto: Colors on + - auto: Auto-detect - never: Colors off + [default: always] + Display: -v, --verbosity... Set the minimum log level. @@ -110,4 +115,37 @@ Display: -q, --quiet Silence all log output + +Tracing: + --tracing-otlp[=] + Enable `Opentelemetry` tracing export to an OTLP endpoint. + + If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + + Example: --tracing-otlp=http://collector:4318/v1/traces + + [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] + + --tracing-otlp-protocol + OTLP transport protocol to use for exporting traces. + + - `http`: expects endpoint path to end with `/v1/traces` - `grpc`: expects endpoint without a path + + Defaults to HTTP if not specified. + + Possible values: + - http: HTTP/Protobuf transport, port 4318, requires `/v1/traces` path + - grpc: gRPC transport, port 4317 + + [env: OTEL_EXPORTER_OTLP_PROTOCOL=] + [default: http] + + --tracing-otlp.filter + Set a filter directive for the OTLP tracer. This controls the verbosity of spans and events sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable. + + Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off + + Defaults to TRACE if not specified. + + [default: debug] ``` \ No newline at end of file diff --git a/book/cli/reth/stage/dump/execution.md b/docs/vocs/docs/pages/cli/reth/stage/dump/execution.mdx similarity index 70% rename from book/cli/reth/stage/dump/execution.md rename to docs/vocs/docs/pages/cli/reth/stage/dump/execution.mdx index 8842d393671..bed5d33329a 100644 --- a/book/cli/reth/stage/dump/execution.md +++ b/docs/vocs/docs/pages/cli/reth/stage/dump/execution.mdx @@ -38,13 +38,13 @@ Logging: --log.stdout.format The format to use for logs written to stdout - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.stdout.filter The filter to use for logs written to stdout @@ -53,13 +53,13 @@ Logging: --log.file.format The format to use for logs written to the log file - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.file.filter The filter to use for logs written to the log file @@ -70,6 +70,11 @@ Logging: [default: /logs] + --log.file.name + The prefix name of the log files + + [default: reth.log] + --log.file.max-size The maximum size (in MB) of one log file @@ -91,13 +96,13 @@ Logging: --color Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting - [default: always] - Possible values: - always: Colors on - - auto: Colors on + - auto: Auto-detect - never: Colors off + [default: always] + Display: -v, --verbosity... Set the minimum log level. @@ -110,4 +115,37 @@ Display: -q, --quiet Silence all log output + +Tracing: + --tracing-otlp[=] + Enable `Opentelemetry` tracing export to an OTLP endpoint. + + If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + + Example: --tracing-otlp=http://collector:4318/v1/traces + + [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] + + --tracing-otlp-protocol + OTLP transport protocol to use for exporting traces. + + - `http`: expects endpoint path to end with `/v1/traces` - `grpc`: expects endpoint without a path + + Defaults to HTTP if not specified. + + Possible values: + - http: HTTP/Protobuf transport, port 4318, requires `/v1/traces` path + - grpc: gRPC transport, port 4317 + + [env: OTEL_EXPORTER_OTLP_PROTOCOL=] + [default: http] + + --tracing-otlp.filter + Set a filter directive for the OTLP tracer. This controls the verbosity of spans and events sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable. + + Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off + + Defaults to TRACE if not specified. + + [default: debug] ``` \ No newline at end of file diff --git a/book/cli/reth/stage/dump/merkle.md b/docs/vocs/docs/pages/cli/reth/stage/dump/merkle.mdx similarity index 70% rename from book/cli/reth/stage/dump/merkle.md rename to docs/vocs/docs/pages/cli/reth/stage/dump/merkle.mdx index 1e781ec4f96..3bada103c87 100644 --- a/book/cli/reth/stage/dump/merkle.md +++ b/docs/vocs/docs/pages/cli/reth/stage/dump/merkle.mdx @@ -38,13 +38,13 @@ Logging: --log.stdout.format The format to use for logs written to stdout - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.stdout.filter The filter to use for logs written to stdout @@ -53,13 +53,13 @@ Logging: --log.file.format The format to use for logs written to the log file - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.file.filter The filter to use for logs written to the log file @@ -70,6 +70,11 @@ Logging: [default: /logs] + --log.file.name + The prefix name of the log files + + [default: reth.log] + --log.file.max-size The maximum size (in MB) of one log file @@ -91,13 +96,13 @@ Logging: --color Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting - [default: always] - Possible values: - always: Colors on - - auto: Colors on + - auto: Auto-detect - never: Colors off + [default: always] + Display: -v, --verbosity... Set the minimum log level. @@ -110,4 +115,37 @@ Display: -q, --quiet Silence all log output + +Tracing: + --tracing-otlp[=] + Enable `Opentelemetry` tracing export to an OTLP endpoint. + + If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + + Example: --tracing-otlp=http://collector:4318/v1/traces + + [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] + + --tracing-otlp-protocol + OTLP transport protocol to use for exporting traces. + + - `http`: expects endpoint path to end with `/v1/traces` - `grpc`: expects endpoint without a path + + Defaults to HTTP if not specified. + + Possible values: + - http: HTTP/Protobuf transport, port 4318, requires `/v1/traces` path + - grpc: gRPC transport, port 4317 + + [env: OTEL_EXPORTER_OTLP_PROTOCOL=] + [default: http] + + --tracing-otlp.filter + Set a filter directive for the OTLP tracer. This controls the verbosity of spans and events sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable. + + Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off + + Defaults to TRACE if not specified. + + [default: debug] ``` \ No newline at end of file diff --git a/book/cli/reth/stage/dump/storage-hashing.md b/docs/vocs/docs/pages/cli/reth/stage/dump/storage-hashing.mdx similarity index 70% rename from book/cli/reth/stage/dump/storage-hashing.md rename to docs/vocs/docs/pages/cli/reth/stage/dump/storage-hashing.mdx index 7bfb08b94f3..723a54e9272 100644 --- a/book/cli/reth/stage/dump/storage-hashing.md +++ b/docs/vocs/docs/pages/cli/reth/stage/dump/storage-hashing.mdx @@ -38,13 +38,13 @@ Logging: --log.stdout.format The format to use for logs written to stdout - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.stdout.filter The filter to use for logs written to stdout @@ -53,13 +53,13 @@ Logging: --log.file.format The format to use for logs written to the log file - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.file.filter The filter to use for logs written to the log file @@ -70,6 +70,11 @@ Logging: [default: /logs] + --log.file.name + The prefix name of the log files + + [default: reth.log] + --log.file.max-size The maximum size (in MB) of one log file @@ -91,13 +96,13 @@ Logging: --color Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting - [default: always] - Possible values: - always: Colors on - - auto: Colors on + - auto: Auto-detect - never: Colors off + [default: always] + Display: -v, --verbosity... Set the minimum log level. @@ -110,4 +115,37 @@ Display: -q, --quiet Silence all log output + +Tracing: + --tracing-otlp[=] + Enable `Opentelemetry` tracing export to an OTLP endpoint. + + If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + + Example: --tracing-otlp=http://collector:4318/v1/traces + + [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] + + --tracing-otlp-protocol + OTLP transport protocol to use for exporting traces. + + - `http`: expects endpoint path to end with `/v1/traces` - `grpc`: expects endpoint without a path + + Defaults to HTTP if not specified. + + Possible values: + - http: HTTP/Protobuf transport, port 4318, requires `/v1/traces` path + - grpc: gRPC transport, port 4317 + + [env: OTEL_EXPORTER_OTLP_PROTOCOL=] + [default: http] + + --tracing-otlp.filter + Set a filter directive for the OTLP tracer. This controls the verbosity of spans and events sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable. + + Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off + + Defaults to TRACE if not specified. + + [default: debug] ``` \ No newline at end of file diff --git a/book/cli/reth/stage/run.md b/docs/vocs/docs/pages/cli/reth/stage/run.mdx similarity index 77% rename from book/cli/reth/stage/run.md rename to docs/vocs/docs/pages/cli/reth/stage/run.mdx index 5a7a9ad10cf..f3e4ccc0e0c 100644 --- a/book/cli/reth/stage/run.md +++ b/docs/vocs/docs/pages/cli/reth/stage/run.mdx @@ -67,6 +67,12 @@ Database: --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout + --db.max-readers + Maximum number of readers allowed to access the database concurrently + + --db.sync-mode + Controls how aggressively the database synchronizes data to disk + --metrics Enable Prometheus metrics. @@ -98,17 +104,18 @@ Database: The name of the stage to run Possible values: - - headers: The headers stage within the pipeline - - bodies: The bodies stage within the pipeline - - senders: The senders stage within the pipeline - - execution: The execution stage within the pipeline - - account-hashing: The account hashing stage within the pipeline - - storage-hashing: The storage hashing stage within the pipeline - - hashing: The account and storage hashing stages within the pipeline - - merkle: The merkle stage within the pipeline - - tx-lookup: The transaction lookup stage within the pipeline - - account-history: The account history stage within the pipeline - - storage-history: The storage history stage within the pipeline + - headers: The headers stage within the pipeline + - bodies: The bodies stage within the pipeline + - senders: The senders stage within the pipeline + - execution: The execution stage within the pipeline + - account-hashing: The account hashing stage within the pipeline + - storage-hashing: The storage hashing stage within the pipeline + - hashing: The account and storage hashing stages within the pipeline + - merkle: The merkle stage within the pipeline + - merkle-changesets: The merkle changesets stage within the pipeline + - tx-lookup: The transaction lookup stage within the pipeline + - account-history: The account history stage within the pipeline + - storage-history: The storage history stage within the pipeline Networking: -d, --disable-discovery @@ -284,17 +291,35 @@ Networking: [default: All] + --disable-tx-gossip + Disable transaction pool gossip + + Disables gossiping of transactions in the mempool to peers. This can be omitted for personal nodes, though providers should always opt to enable this flag. + + --tx-propagation-mode + Sets the transaction propagation mode by determining how new pending transactions are propagated to other peers in full. + + Examples: sqrt, all, max:10 + + [default: sqrt] + + --required-block-hashes + Comma separated list of required block hashes. Peers that don't have these blocks will be filtered out + + --network-id + Optional network ID to override the chain specification's network ID for P2P connections + Logging: --log.stdout.format The format to use for logs written to stdout - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.stdout.filter The filter to use for logs written to stdout @@ -303,13 +328,13 @@ Logging: --log.file.format The format to use for logs written to the log file - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.file.filter The filter to use for logs written to the log file @@ -320,6 +345,11 @@ Logging: [default: /logs] + --log.file.name + The prefix name of the log files + + [default: reth.log] + --log.file.max-size The maximum size (in MB) of one log file @@ -341,13 +371,13 @@ Logging: --color Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting - [default: always] - Possible values: - always: Colors on - - auto: Colors on + - auto: Auto-detect - never: Colors off + [default: always] + Display: -v, --verbosity... Set the minimum log level. @@ -360,4 +390,37 @@ Display: -q, --quiet Silence all log output + +Tracing: + --tracing-otlp[=] + Enable `Opentelemetry` tracing export to an OTLP endpoint. + + If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + + Example: --tracing-otlp=http://collector:4318/v1/traces + + [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] + + --tracing-otlp-protocol + OTLP transport protocol to use for exporting traces. + + - `http`: expects endpoint path to end with `/v1/traces` - `grpc`: expects endpoint without a path + + Defaults to HTTP if not specified. + + Possible values: + - http: HTTP/Protobuf transport, port 4318, requires `/v1/traces` path + - grpc: gRPC transport, port 4317 + + [env: OTEL_EXPORTER_OTLP_PROTOCOL=] + [default: http] + + --tracing-otlp.filter + Set a filter directive for the OTLP tracer. This controls the verbosity of spans and events sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable. + + Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off + + Defaults to TRACE if not specified. + + [default: debug] ``` \ No newline at end of file diff --git a/book/cli/reth/stage/unwind.md b/docs/vocs/docs/pages/cli/reth/stage/unwind.mdx similarity index 76% rename from book/cli/reth/stage/unwind.md rename to docs/vocs/docs/pages/cli/reth/stage/unwind.mdx index d0671040dc4..8bb44279f8d 100644 --- a/book/cli/reth/stage/unwind.md +++ b/docs/vocs/docs/pages/cli/reth/stage/unwind.mdx @@ -72,6 +72,12 @@ Database: --db.read-transaction-timeout Read transaction timeout in seconds, 0 means no timeout + --db.max-readers + Maximum number of readers allowed to access the database concurrently + + --db.sync-mode + Controls how aggressively the database synchronizes data to disk + --offline If this is enabled, then all stages except headers, bodies, and sender recovery will be unwound @@ -79,13 +85,13 @@ Logging: --log.stdout.format The format to use for logs written to stdout - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.stdout.filter The filter to use for logs written to stdout @@ -94,13 +100,13 @@ Logging: --log.file.format The format to use for logs written to the log file - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.file.filter The filter to use for logs written to the log file @@ -111,6 +117,11 @@ Logging: [default: /logs] + --log.file.name + The prefix name of the log files + + [default: reth.log] + --log.file.max-size The maximum size (in MB) of one log file @@ -132,13 +143,13 @@ Logging: --color Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting - [default: always] - Possible values: - always: Colors on - - auto: Colors on + - auto: Auto-detect - never: Colors off + [default: always] + Display: -v, --verbosity... Set the minimum log level. @@ -151,4 +162,37 @@ Display: -q, --quiet Silence all log output + +Tracing: + --tracing-otlp[=] + Enable `Opentelemetry` tracing export to an OTLP endpoint. + + If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + + Example: --tracing-otlp=http://collector:4318/v1/traces + + [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] + + --tracing-otlp-protocol + OTLP transport protocol to use for exporting traces. + + - `http`: expects endpoint path to end with `/v1/traces` - `grpc`: expects endpoint without a path + + Defaults to HTTP if not specified. + + Possible values: + - http: HTTP/Protobuf transport, port 4318, requires `/v1/traces` path + - grpc: gRPC transport, port 4317 + + [env: OTEL_EXPORTER_OTLP_PROTOCOL=] + [default: http] + + --tracing-otlp.filter + Set a filter directive for the OTLP tracer. This controls the verbosity of spans and events sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable. + + Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off + + Defaults to TRACE if not specified. + + [default: debug] ``` \ No newline at end of file diff --git a/book/cli/reth/stage/unwind/num-blocks.md b/docs/vocs/docs/pages/cli/reth/stage/unwind/num-blocks.mdx similarity index 68% rename from book/cli/reth/stage/unwind/num-blocks.md rename to docs/vocs/docs/pages/cli/reth/stage/unwind/num-blocks.mdx index 04d6cfb0114..b04e1920b75 100644 --- a/book/cli/reth/stage/unwind/num-blocks.md +++ b/docs/vocs/docs/pages/cli/reth/stage/unwind/num-blocks.mdx @@ -30,13 +30,13 @@ Logging: --log.stdout.format The format to use for logs written to stdout - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.stdout.filter The filter to use for logs written to stdout @@ -45,13 +45,13 @@ Logging: --log.file.format The format to use for logs written to the log file - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.file.filter The filter to use for logs written to the log file @@ -62,6 +62,11 @@ Logging: [default: /logs] + --log.file.name + The prefix name of the log files + + [default: reth.log] + --log.file.max-size The maximum size (in MB) of one log file @@ -83,13 +88,13 @@ Logging: --color Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting - [default: always] - Possible values: - always: Colors on - - auto: Colors on + - auto: Auto-detect - never: Colors off + [default: always] + Display: -v, --verbosity... Set the minimum log level. @@ -102,4 +107,37 @@ Display: -q, --quiet Silence all log output + +Tracing: + --tracing-otlp[=] + Enable `Opentelemetry` tracing export to an OTLP endpoint. + + If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + + Example: --tracing-otlp=http://collector:4318/v1/traces + + [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] + + --tracing-otlp-protocol + OTLP transport protocol to use for exporting traces. + + - `http`: expects endpoint path to end with `/v1/traces` - `grpc`: expects endpoint without a path + + Defaults to HTTP if not specified. + + Possible values: + - http: HTTP/Protobuf transport, port 4318, requires `/v1/traces` path + - grpc: gRPC transport, port 4317 + + [env: OTEL_EXPORTER_OTLP_PROTOCOL=] + [default: http] + + --tracing-otlp.filter + Set a filter directive for the OTLP tracer. This controls the verbosity of spans and events sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable. + + Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off + + Defaults to TRACE if not specified. + + [default: debug] ``` \ No newline at end of file diff --git a/book/cli/reth/stage/unwind/to-block.md b/docs/vocs/docs/pages/cli/reth/stage/unwind/to-block.mdx similarity index 69% rename from book/cli/reth/stage/unwind/to-block.md rename to docs/vocs/docs/pages/cli/reth/stage/unwind/to-block.mdx index 591a47258df..2c22f8127c1 100644 --- a/book/cli/reth/stage/unwind/to-block.md +++ b/docs/vocs/docs/pages/cli/reth/stage/unwind/to-block.mdx @@ -30,13 +30,13 @@ Logging: --log.stdout.format The format to use for logs written to stdout - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.stdout.filter The filter to use for logs written to stdout @@ -45,13 +45,13 @@ Logging: --log.file.format The format to use for logs written to the log file - [default: terminal] - Possible values: - json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging - log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications - terminal: Represents terminal-friendly formatting for logs + [default: terminal] + --log.file.filter The filter to use for logs written to the log file @@ -62,6 +62,11 @@ Logging: [default: /logs] + --log.file.name + The prefix name of the log files + + [default: reth.log] + --log.file.max-size The maximum size (in MB) of one log file @@ -83,13 +88,13 @@ Logging: --color Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting - [default: always] - Possible values: - always: Colors on - - auto: Colors on + - auto: Auto-detect - never: Colors off + [default: always] + Display: -v, --verbosity... Set the minimum log level. @@ -102,4 +107,37 @@ Display: -q, --quiet Silence all log output + +Tracing: + --tracing-otlp[=] + Enable `Opentelemetry` tracing export to an OTLP endpoint. + + If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317` + + Example: --tracing-otlp=http://collector:4318/v1/traces + + [env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=] + + --tracing-otlp-protocol + OTLP transport protocol to use for exporting traces. + + - `http`: expects endpoint path to end with `/v1/traces` - `grpc`: expects endpoint without a path + + Defaults to HTTP if not specified. + + Possible values: + - http: HTTP/Protobuf transport, port 4318, requires `/v1/traces` path + - grpc: gRPC transport, port 4317 + + [env: OTEL_EXPORTER_OTLP_PROTOCOL=] + [default: http] + + --tracing-otlp.filter + Set a filter directive for the OTLP tracer. This controls the verbosity of spans and events sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable. + + Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off + + Defaults to TRACE if not specified. + + [default: debug] ``` \ No newline at end of file diff --git a/book/cli/reth/test-vectors/tables.md b/docs/vocs/docs/pages/cli/reth/test-vectors/tables.mdx similarity index 100% rename from book/cli/reth/test-vectors/tables.md rename to docs/vocs/docs/pages/cli/reth/test-vectors/tables.mdx diff --git a/book/developers/exex/hello-world.md b/docs/vocs/docs/pages/exex/hello-world.mdx similarity index 70% rename from book/developers/exex/hello-world.md rename to docs/vocs/docs/pages/exex/hello-world.mdx index c1f3e5af944..30eac91ee99 100644 --- a/book/developers/exex/hello-world.md +++ b/docs/vocs/docs/pages/exex/hello-world.mdx @@ -1,3 +1,7 @@ +--- +description: Example of a minimal Hello World ExEx in Reth. +--- + # Hello World Let's write a simple "Hello World" ExEx that emits a log every time a new chain of blocks is committed, reverted, or reorged. @@ -14,15 +18,15 @@ cd my-exex And add Reth as a dependency in `Cargo.toml` ```toml -{{#include ../../sources/exex/hello-world/Cargo.toml}} +// [!include ~/snippets/sources/exex/hello-world/Cargo.toml] ``` ### Default Reth node Now, let's jump to our `main.rs` and start by initializing and launching a default Reth node -```rust,norun,noplayground,ignore -{{#include ../../sources/exex/hello-world/src/bin/1.rs}} +```rust +// [!include ~/snippets/sources/exex/hello-world/src/bin/1.rs] ``` You can already test that it works by running the binary and initializing the Holesky node in a custom datadir @@ -42,8 +46,8 @@ $ cargo run -- init --chain holesky --datadir data The simplest ExEx is just an async function that never returns. We need to install it into our node -```rust,norun,noplayground,ignore -{{#include ../../sources/exex/hello-world/src/bin/2.rs}} +```rust +// [!include ~/snippets/sources/exex/hello-world/src/bin/2.rs] ``` See that unused `_ctx`? That's the context that we'll use to listen to new notifications coming from the main node, @@ -63,17 +67,17 @@ If you try running a node with an ExEx that exits, the node will exit as well. Now, let's extend our simplest ExEx and start actually listening to new notifications, log them, and send events back to the main node -```rust,norun,noplayground,ignore -{{#include ../../sources/exex/hello-world/src/bin/3.rs}} +```rust +// [!include ~/snippets/sources/exex/hello-world/src/bin/3.rs] ``` Woah, there's a lot of new stuff here! Let's go through it step by step: -- First, we've added a `while let Some(notification) = ctx.notifications.recv().await` loop that waits for new notifications to come in. - - The main node is responsible for sending notifications to the ExEx, so we're waiting for them to come in. -- Next, we've added a `match ¬ification { ... }` block that matches on the type of the notification. - - In each case, we're logging the notification and the corresponding block range, be it a chain commit, revert, or reorg. -- Finally, we're checking if the notification contains a committed chain, and if it does, we're sending a `ExExEvent::FinishedHeight` event back to the main node using the `ctx.events.send` method. +- First, we've added a `while let Some(notification) = ctx.notifications.recv().await` loop that waits for new notifications to come in. + - The main node is responsible for sending notifications to the ExEx, so we're waiting for them to come in. +- Next, we've added a `match ¬ification { ... }` block that matches on the type of the notification. + - In each case, we're logging the notification and the corresponding block range, be it a chain commit, revert, or reorg. +- Finally, we're checking if the notification contains a committed chain, and if it does, we're sending a `ExExEvent::FinishedHeight` event back to the main node using the `ctx.events.send` method.
@@ -88,4 +92,4 @@ What we've arrived at is the [minimal ExEx example](https://github.com/paradigmx ## What's next? -Let's do something a bit more interesting, and see how you can [keep track of some state](./tracking-state.md) inside your ExEx. +Let's do something a bit more interesting, and see how you can [keep track of some state](/exex/tracking-state) inside your ExEx. diff --git a/book/developers/exex/how-it-works.md b/docs/vocs/docs/pages/exex/how-it-works.mdx similarity index 67% rename from book/developers/exex/how-it-works.md rename to docs/vocs/docs/pages/exex/how-it-works.mdx index 7f80d71cbff..21162a75620 100644 --- a/book/developers/exex/how-it-works.md +++ b/docs/vocs/docs/pages/exex/how-it-works.mdx @@ -1,3 +1,7 @@ +--- +description: How Execution Extensions (ExExes) work in Reth. +--- + # How do ExExes work? ExExes are just [Futures](https://doc.rust-lang.org/std/future/trait.Future.html) that run indefinitely alongside Reth @@ -7,12 +11,13 @@ An ExEx is usually driven by and acts on new notifications about chain commits, They are installed into the node by using the [node builder](https://reth.rs/docs/reth/builder/struct.NodeBuilder.html). Reth manages the lifecycle of all ExExes, including: -- Polling ExEx futures -- Sending [notifications](https://reth.rs/docs/reth_exex/enum.ExExNotification.html) about new chain, reverts, - and reorgs from historical and live sync -- Processing [events](https://reth.rs/docs/reth_exex/enum.ExExEvent.html) emitted by ExExes -- Pruning (in case of a full or pruned node) only the data that has been processed by all ExExes -- Shutting ExExes down when the node is shut down + +- Polling ExEx futures +- Sending [notifications](https://reth.rs/docs/reth_exex/enum.ExExNotification.html) about new chain, reverts, + and reorgs from historical and live sync +- Processing [events](https://reth.rs/docs/reth_exex/enum.ExExEvent.html) emitted by ExExes +- Pruning (in case of a full or pruned node) only the data that has been processed by all ExExes +- Shutting ExExes down when the node is shut down ## Pruning diff --git a/book/developers/exex/exex.md b/docs/vocs/docs/pages/exex/overview.mdx similarity index 62% rename from book/developers/exex/exex.md rename to docs/vocs/docs/pages/exex/overview.mdx index 25372a7c922..abfcc8f3b82 100644 --- a/book/developers/exex/exex.md +++ b/docs/vocs/docs/pages/exex/overview.mdx @@ -1,9 +1,13 @@ +--- +description: Introduction to Execution Extensions (ExEx) in Reth. +--- + # Execution Extensions (ExEx) ## What are Execution Extensions? Execution Extensions (or ExExes, for short) allow developers to build their own infrastructure that relies on Reth -as a base for driving the chain (be it [Ethereum](../../run/mainnet.md) or [OP Stack](../../run/optimism.md)) forward. +as a base for driving the chain (be it [Ethereum](/run/ethereum) or [OP Stack](/run/opstack)) forward. An Execution Extension is a task that derives its state from changes in Reth's state. Some examples of such state derivations are rollups, bridges, and indexers. @@ -18,14 +22,18 @@ Read more about things you can build with Execution Extensions in the [Paradigm Execution Extensions are not separate processes that connect to the main Reth node process. Instead, ExExes are compiled into the same binary as Reth, and run alongside it, using shared memory for communication. -If you want to build an Execution Extension that sends data into a separate process, check out the [Remote](./remote.md) chapter. +If you want to build an Execution Extension that sends data into a separate process, check out the [Remote](/exex/remote) chapter. ## How do I build an Execution Extension? Let's dive into how to build our own ExEx from scratch, add tests for it, and run it on the Holesky testnet. -1. [How do ExExes work?](./how-it-works.md) -1. [Hello World](./hello-world.md) -1. [Tracking State](./tracking-state.md) -1. [Remote](./remote.md) +1. [How do ExExes work?](/exex/how-it-works) +1. [Hello World](/exex/hello-world) +1. [Tracking State](/exex/tracking-state) +1. [Remote](/exex/remote) + +:::tip +For more practical examples and ready-to-use ExEx implementations, check out the [reth-exex-examples](https://github.com/paradigmxyz/reth-exex-examples) repository which contains various ExEx examples including indexers, bridges, and other state derivation patterns. +::: diff --git a/book/developers/exex/remote.md b/docs/vocs/docs/pages/exex/remote.mdx similarity index 76% rename from book/developers/exex/remote.md rename to docs/vocs/docs/pages/exex/remote.mdx index 0ec704308ff..772b56d7fd7 100644 --- a/book/developers/exex/remote.md +++ b/docs/vocs/docs/pages/exex/remote.mdx @@ -1,10 +1,15 @@ +--- +description: Building a remote ExEx that communicates via gRPC. +--- + # Remote Execution Extensions In this chapter, we will learn how to create an ExEx that emits all notifications to an external process. We will use [Tonic](https://github.com/hyperium/tonic) to create a gRPC server and a client. -- The server binary will have the Reth client, our ExEx and the gRPC server. -- The client binary will have the gRPC client that connects to the server. + +- The server binary will have the Reth client, our ExEx and the gRPC server. +- The client binary will have the gRPC client that connects to the server. ## Prerequisites @@ -21,11 +26,11 @@ $ cargo new --lib exex-remote $ cd exex-remote ``` -We will also need a bunch of dependencies. Some of them you know from the [Hello World](./hello-world.md) chapter, +We will also need a bunch of dependencies. Some of them you know from the [Hello World](/exex/hello-world) chapter, but some of specific to what we need now. ```toml -{{#include ../../sources/exex/remote/Cargo.toml}} +// [!include ~/snippets/sources/exex/remote/Cargo.toml] ``` We also added a build dependency for Tonic. We will use it to generate the Rust code for our @@ -33,8 +38,9 @@ Protobuf definitions at compile time. Read more about using Tonic in the [introductory tutorial](https://github.com/hyperium/tonic/blob/6a213e9485965db0628591e30577ed81cdaeaf2b/examples/helloworld-tutorial.md). Also, we now have two separate binaries: -- `exex` is the server binary that will run the ExEx and the gRPC server. -- `consumer` is the client binary that will connect to the server and receive notifications. + +- `exex` is the server binary that will run the ExEx and the gRPC server. +- `consumer` is the client binary that will connect to the server and receive notifications. ### Create the Protobuf definitions @@ -53,12 +59,13 @@ For an example of a full schema, see the [Remote ExEx](https://github.com/paradi
```protobuf -{{#include ../../sources/exex/remote/proto/exex.proto}} +// [!include ~/snippets/sources/exex/remote/proto/exex.proto] ``` To instruct Tonic to generate the Rust code using this `.proto`, add the following lines to your `lib.rs` file: -```rust,norun,noplayground,ignore -{{#include ../../sources/exex/remote/src/lib.rs}} + +```rust +// [!include ~/snippets/sources/exex/remote/src/lib.rs] ``` ## ExEx and gRPC server @@ -70,8 +77,8 @@ We will now create the ExEx and the gRPC server in our `src/exex.rs` file. Let's create a minimal gRPC server that listens on the port `:10000`, and spawn it using the [NodeBuilder](https://reth.rs/docs/reth/builder/struct.NodeBuilder.html)'s [task executor](https://reth.rs/docs/reth/tasks/struct.TaskExecutor.html). -```rust,norun,noplayground,ignore -{{#include ../../sources/exex/remote/src/exex_1.rs}} +```rust +// [!include ~/snippets/sources/exex/remote/src/exex_1.rs] ``` Currently, it does not send anything on the stream. @@ -81,8 +88,8 @@ to send new `ExExNotification` on it. Let's create this channel in the `main` function where we will have both gRPC server and ExEx initiated, and save the sender part (that way we will be able to create new receivers) of this channel in our gRPC server. -```rust,norun,noplayground,ignore -{{#include ../../sources/exex/remote/src/exex_2.rs}} +```rust +// [!include ~/snippets/sources/exex/remote/src/exex_2.rs] ``` And with that, we're ready to handle incoming notifications, serialize them with [bincode](https://docs.rs/bincode/) @@ -91,8 +98,8 @@ and send back to the client. For each incoming request, we spawn a separate tokio task that will run in the background, and then return the stream receiver to the client. -```rust,norun,noplayground,ignore -{{#rustdoc_include ../../sources/exex/remote/src/exex_3.rs:snippet}} +```rust +// [!include ~/snippets/sources/exex/remote/src/exex_3.rs] ``` That's it for the gRPC server part! It doesn't receive anything on the `notifications` channel yet, @@ -110,25 +117,24 @@ Don't forget to emit `ExExEvent::FinishedHeight` -```rust,norun,noplayground,ignore -{{#rustdoc_include ../../sources/exex/remote/src/exex_4.rs:snippet}} +```rust +// [!include ~/snippets/sources/exex/remote/src/exex_4.rs] ``` All that's left is to connect all pieces together: install our ExEx in the node and pass the sender part of communication channel to it. -```rust,norun,noplayground,ignore -{{#rustdoc_include ../../sources/exex/remote/src/exex.rs:snippet}} +```rust +// [!include ~/snippets/sources/exex/remote/src/exex.rs] ``` ### Full `exex.rs` code
-Click to expand - -```rust,norun,noplayground,ignore -{{#include ../../sources/exex/remote/src/exex.rs}} -``` + Click to expand + ```rust + // [!include ~/snippets/sources/exex/remote/src/exex.rs] + ```
## Consumer @@ -143,8 +149,8 @@ because notifications can get very heavy -```rust,norun,noplayground,ignore -{{#include ../../sources/exex/remote/src/consumer.rs}} +```rust +// [!include ~/snippets/sources/exex/remote/src/consumer.rs] ``` ## Running @@ -162,4 +168,4 @@ And in the other, we will run our consumer: cargo run --bin consumer --release ``` - +![remote_exex](/remote_exex.png) diff --git a/book/developers/exex/tracking-state.md b/docs/vocs/docs/pages/exex/tracking-state.mdx similarity index 61% rename from book/developers/exex/tracking-state.md rename to docs/vocs/docs/pages/exex/tracking-state.mdx index d2a9fe6ca3e..fb3486e7fab 100644 --- a/book/developers/exex/tracking-state.md +++ b/docs/vocs/docs/pages/exex/tracking-state.mdx @@ -1,8 +1,12 @@ +--- +description: How to track state in a custom ExEx. +--- + # Tracking State In this chapter, we'll learn how to keep track of some state inside our ExEx. -Let's continue with our Hello World example from the [previous chapter](./hello-world.md). +Let's continue with our Hello World example from the [previous chapter](/exex/hello-world). ### Turning ExEx into a struct @@ -18,8 +22,8 @@ because you can't access variables inside the function to assert the state of yo -```rust,norun,noplayground,ignore -{{#include ../../sources/exex/tracking-state/src/bin/1.rs}} +```rust +// [!include ~/snippets/sources/exex/tracking-state/src/bin/1.rs] ``` For those who are not familiar with how async Rust works on a lower level, that may seem scary, @@ -27,7 +31,7 @@ but let's unpack what's going on here: 1. Our ExEx is now a `struct` that contains the context and implements the `Future` trait. It's now pollable (hence `await`-able). 1. We can't use `self` directly inside our `poll` method, and instead need to acquire a mutable reference to the data inside of the `Pin`. - Read more about pinning in [the book](https://rust-lang.github.io/async-book/04_pinning/01_chapter.html). + Read more about pinning in [the book](https://rust-lang.github.io/async-book/part-reference/pinning.html). 1. We also can't use `await` directly inside `poll`, and instead need to poll futures manually. We wrap the call to `poll_recv(cx)` into a [`ready!`](https://doc.rust-lang.org/std/task/macro.ready.html) macro, so that if the channel of notifications has no value ready, we will instantly return `Poll::Pending` from our Future. @@ -39,23 +43,25 @@ With all that done, we're now free to add more fields to our `MyExEx` struct, an Our ExEx will count the number of transactions in each block and log it to the console. -```rust,norun,noplayground,ignore -{{#include ../../sources/exex/tracking-state/src/bin/2.rs}} +```rust +// [!include ~/snippets/sources/exex/tracking-state/src/bin/2.rs] ``` As you can see, we added two fields to our ExEx struct: -- `first_block` to keep track of the first block that was committed since the start of the ExEx. -- `transactions` to keep track of the total number of transactions committed, accounting for reorgs and reverts. + +- `first_block` to keep track of the first block that was committed since the start of the ExEx. +- `transactions` to keep track of the total number of transactions committed, accounting for reorgs and reverts. We also changed our `match` block to two `if` clauses: -- First one checks if there's a reverted chain using `notification.reverted_chain()`. If there is: - - We subtract the number of transactions in the reverted chain from the total number of transactions. - - It's important to do the `saturating_sub` here, because if we just started our node and - instantly received a reorg, our `transactions` field will still be zero. -- Second one checks if there's a committed chain using `notification.committed_chain()`. If there is: - - We update the `first_block` field to the first block of the committed chain. - - We add the number of transactions in the committed chain to the total number of transactions. - - We send a `FinishedHeight` event back to the main node. + +- First one checks if there's a reverted chain using `notification.reverted_chain()`. If there is: + - We subtract the number of transactions in the reverted chain from the total number of transactions. + - It's important to do the `saturating_sub` here, because if we just started our node and + instantly received a reorg, our `transactions` field will still be zero. +- Second one checks if there's a committed chain using `notification.committed_chain()`. If there is: + - We update the `first_block` field to the first block of the committed chain. + - We add the number of transactions in the committed chain to the total number of transactions. + - We send a `FinishedHeight` event back to the main node. Finally, on every notification, we log the total number of transactions and the first block that was committed since the start of the ExEx. diff --git a/docs/vocs/docs/pages/guides/history-expiry.mdx b/docs/vocs/docs/pages/guides/history-expiry.mdx new file mode 100644 index 00000000000..e4b09c1a530 --- /dev/null +++ b/docs/vocs/docs/pages/guides/history-expiry.mdx @@ -0,0 +1,80 @@ +--- +description: Usage of tools for importing, exporting and pruning historical blocks +--- + +# History Expiry + +In this chapter, we will learn how to use tools for dealing with historical data, it's import, export and removal. + +We will use [reth cli](/cli/cli) to import and export historical data. + +## Enabling Pre-merge history expiry + +Opting in into pre-merge history expiry will remove all pre-merge transaction/receipt data (static files) for mainnet and sepolia. + +For new and existing nodes: + +Use the flags `--prune.bodies.pre-merge` `--prune.receipts.pre-merge` + +See also [Partial history expiry announcement](https://blog.ethereum.org/2025/07/08/partial-history-exp) + +## File format + +The historical data is packaged and distributed in files of special formats with different names, all of which are based on [e2store](https://github.com/status-im/nimbus-eth2/blob/613f4a9a50c9c4bd8568844eaffb3ac15d067e56/docs/e2store.md#introduction). The most important ones are the **ERA1**, which deals with block range from genesis until the last pre-merge block, and **ERA**, which deals with block range from the merge onwards. + +See the following specifications for more details : +- [E2store specification](https://github.com/eth-clients/e2store-format-specs) +- [ERA1 specification](https://github.com/eth-clients/e2store-format-specs/blob/main/formats/era1.md) +- [ERA specification](https://github.com/eth-clients/e2store-format-specs/blob/main/formats/era.md) + +The contents of these archives is an ordered sequence of blocks. We're mostly concerned with headers and transactions. For ERA1, there is 8192 blocks per file except for the last one, i.e. the one containing pre-merge block, which can be less than that. + +## Import + +In this section we discuss how to get blocks from ERA1 files. + +### Automatic sync + +If enabled, importing blocks from ERA1 files can be done automatically with no manual steps required. + +#### Enabling the ERA stage + +The import from ERA1 files within the pre-merge block range is included in the [reth node](/cli/reth/node) synchronization pipeline. It is disabled by default. To enable it, pass the `--era.enable` flag when running the [`node`](/cli/reth/node) command. + +The benefit of using this option is significant increase in the synchronization speed for the headers and mainly bodies stage of the pipeline within the ERA1 block range. We encourage you to use it! Eventually, it will become enabled by default. + +#### Using the ERA stage + +When enabled, the import from ERA1 files runs as its own separate stage before all others. It is an optional stage that is doing the work of headers and bodies stage at a significantly higher speed. The checkpoints of these stages are shifted by the ERA stage. + +### Manual import + +If you want to import block headers and transactions from ERA1 files without running the synchronization pipeline, you may use the [`import-era`](/cli/reth/import-era) command. + +### Options + +Both ways of importing the ERA1 files have the same options because they use the same underlying subsystems. No options are mandatory. + +#### Sources + +There are two kinds of data sources for the ERA1 import. +* Remote from an HTTP URL. Use the option `--era.url` with an ERA1 hosting provider URL. +* Local from a file-system directory. Use the option `--era.path` with a directory containing ERA1 files. + +Both options cannot be used at the same time. If no option is specified, the remote source is used with a URL derived from the chain ID. Only Mainnet and Sepolia have ERA1 files. If the node is running on a different chain, no source is provided and nothing is imported. + +## Export + +In this section we discuss how to export blocks data into ERA1 files. + +### Manual export +You can manually export block data from your database to ERA1 files using the [`export-era`](/cli/reth/export-era) command. + +The CLI reads block headers, bodies, and receipts from your local database and packages them into the standardized ERA1 format with up to 8,192 blocks per file. + +#### Set up +The export command allows you to specify: + +- Block ranges with `--first-block-number` and `--last-block-number` +- Output directory with `--path` for the export destination +- File size limits with `--max-blocks-per-file` with a maximum of 8,192 blocks per ERA1 file diff --git a/docs/vocs/docs/pages/index.mdx b/docs/vocs/docs/pages/index.mdx new file mode 100644 index 00000000000..8778914f4c8 --- /dev/null +++ b/docs/vocs/docs/pages/index.mdx @@ -0,0 +1,162 @@ +--- +content: + width: 100% +layout: landing +showLogo: false +title: Reth +description: Secure, performant and modular node implementation that supports both Ethereum and OP-Stack chains. +--- + +import { HomePage, Sponsors } from "vocs/components"; +import { SdkShowcase } from "../components/SdkShowcase"; +import { TrustedBy } from "../components/TrustedBy"; + +
+
+
+
+
+ Reth +
Secure, performant, and modular blockchain SDK and node.
+
+
+ Run a Node + Build a Node + Why Reth? +
+
+
+
+ :::code-group + + ```bash [Run a Node] + # Install the binary + brew install paradigmxyz/brew/reth + + # Run the node with JSON-RPC enabled + reth node --http --http.api eth,trace + ``` + + ```rust [Build a Node] + // .. snip .. + let handle = node_builder + .with_types::() + .with_components(EthereumNode::components()) + .with_add_ons(EthereumAddOns::default()) + .launch() + .await?; + ``` + + ::: +
+
+ +
+
+ stars +
+
+ 4.7K +
+
+
+
+ + +
+
+ contributors +
+
+ 580+ +
+
+
+ +
+
+
+ +
Institutional Security
+
Run reliable staking nodes trusted by Coinbase Staking
+ +
+
+
+
+
+
+ +
Performant
+
Sync faster with optimal transaction processing
+ +
+
+
+
+
+ +
+ +## Trusted by the Best + +Leading infra companies use Reth for MEV applications, staking, RPC services and generating zero-knowledge proofs. + +
+ +
+ +## Built with Reth SDK + +Production chains and networks are powered by Reth's modular architecture. These nodes are built using existing components without forking, saving several engineering hours while improving maintainability. + +
+ +
+ +## Supporters + + +
diff --git a/book/installation/binaries.md b/docs/vocs/docs/pages/installation/binaries.mdx similarity index 90% rename from book/installation/binaries.md rename to docs/vocs/docs/pages/installation/binaries.mdx index fc741805cd9..56c5cf2bacc 100644 --- a/book/installation/binaries.md +++ b/docs/vocs/docs/pages/installation/binaries.mdx @@ -1,3 +1,7 @@ +--- +description: Instructions for installing Reth using pre-built binaries for Windows, macOS, and Linux, including Homebrew and Arch Linux AUR options. Explains how to verify binary signatures and provides details about the release signing key. +--- + # Binaries [**Archives of precompiled binaries of reth are available for Windows, macOS and Linux.**](https://github.com/paradigmxyz/reth/releases) They are static executables. Users of platforms not explicitly listed below should download one of these archives. @@ -41,7 +45,7 @@ Replace the filenames by those corresponding to the downloaded Reth release. Releases are signed using the key with ID [`50FB7CC55B2E8AFA59FE03B7AA5ED56A7FBF253E`](https://keyserver.ubuntu.com/pks/lookup?search=50FB7CC55B2E8AFA59FE03B7AA5ED56A7FBF253E&fingerprint=on&op=index). -```none +```text -----BEGIN PGP PUBLIC KEY BLOCK----- mDMEZl4GjhYJKwYBBAHaRw8BAQdAU5gnINBAfIgF9S9GzZ1zHDwZtv/WcJRIQI+h diff --git a/book/installation/build-for-arm-devices.md b/docs/vocs/docs/pages/installation/build-for-arm-devices.mdx similarity index 81% rename from book/installation/build-for-arm-devices.md rename to docs/vocs/docs/pages/installation/build-for-arm-devices.mdx index 21d32c9e8bd..23b91e08770 100644 --- a/book/installation/build-for-arm-devices.md +++ b/docs/vocs/docs/pages/installation/build-for-arm-devices.mdx @@ -1,3 +1,7 @@ +--- +description: Building and troubleshooting Reth on ARM devices. +--- + # Building for ARM devices Reth can be built for and run on ARM devices, but there are a few things to take into consideration before. @@ -37,12 +41,12 @@ Some newer versions of ARM architecture offer support for Large Virtual Address ### Additional Resources -- [ARM developer documentation](https://developer.arm.com/documentation/ddi0406/cb/Appendixes/ARMv4-and-ARMv5-Differences/System-level-memory-model/Virtual-memory-support) -- [ARM Community Forums](https://community.arm.com) +- [ARM developer documentation](https://developer.arm.com/documentation/ddi0406/cb/Appendixes/ARMv4-and-ARMv5-Differences/System-level-memory-model/Virtual-memory-support) +- [ARM Community Forums](https://community.arm.com) ## Build Reth -If both your CPU architecture and the memory layout are valid, the instructions for building Reth will not differ from [the standard process](https://paradigmxyz.github.io/reth/installation/source.html). +If both your CPU architecture and the memory layout are valid, the instructions for building Reth will not differ from [the standard process](https://reth.rs/installation/source/). ## Troubleshooting @@ -57,16 +61,21 @@ This error is raised whenever MDBX can not open a database due to the limitation You will need to recompile the Linux Kernel to fix the issue. A simple and safe approach to achieve this is to use the Armbian build framework to create a new image of the OS that will be flashed to a storage device of your choice - an SD card for example - with the following kernel feature values: -- **Page Size**: 64 KB -- **Virtual Address Space Size**: 48 Bits + +- **Page Size**: 64 KB +- **Virtual Address Space Size**: 48 Bits To be able to build an Armbian image and set those values, you will need to: -- Clone the Armbian build framework repository + +- Clone the Armbian build framework repository + ```bash git clone https://github.com/armbian/build cd build ``` -- Run the compile script with the following parameters: + +- Run the compile script with the following parameters: + ```bash ./compile.sh \ BUILD_MINIMAL=yes \ @@ -74,5 +83,6 @@ BUILD_DESKTOP=no \ KERNEL_CONFIGURE=yes \ CARD_DEVICE="/dev/sdX" # Replace sdX with your own storage device ``` -- From there, you will be able to select the target board, the OS release and branch. Then, once you get in the **Kernel Configuration** screen, select the **Kernel Features options** and set the previous values accordingly. -- Wait for the process to finish, plug your storage device into your board and start it. You can now download or install Reth and it should work properly. + +- From there, you will be able to select the target board, the OS release and branch. Then, once you get in the **Kernel Configuration** screen, select the **Kernel Features options** and set the previous values accordingly. +- Wait for the process to finish, plug your storage device into your board and start it. You can now download or install Reth and it should work properly. diff --git a/book/installation/docker.md b/docs/vocs/docs/pages/installation/docker.mdx similarity index 80% rename from book/installation/docker.md rename to docs/vocs/docs/pages/installation/docker.mdx index 6ce2ae50a5b..ecf55f6b3da 100644 --- a/book/installation/docker.md +++ b/docs/vocs/docs/pages/installation/docker.mdx @@ -1,3 +1,7 @@ +--- +description: Guide to running Reth using Docker, including obtaining images from GitHub or building locally, using Docker Compose. +--- + # Docker There are two ways to obtain a Reth Docker image: @@ -8,9 +12,10 @@ There are two ways to obtain a Reth Docker image: Once you have obtained the Docker image, proceed to [Using the Docker image](#using-the-docker-image). -> **Note** -> -> Reth requires Docker Engine version 20.10.10 or higher due to [missing support](https://docs.docker.com/engine/release-notes/20.10/#201010) for the `clone3` syscall in previous versions. +:::note +Reth requires Docker Engine version 20.10.10 or higher due to [missing support](https://docs.docker.com/engine/release-notes/20.10/#201010) for the `clone3` syscall in previous versions. +::: + ## GitHub Reth docker images for both x86_64 and ARM64 machines are published with every release of reth on GitHub Container Registry. @@ -52,6 +57,7 @@ docker run reth:local --version ## Using the Docker image There are two ways to use the Docker image: + 1. [Using Docker](#using-plain-docker) 2. [Using Docker Compose](#using-docker-compose) @@ -86,12 +92,12 @@ To run Reth with Docker Compose, run the following command from a shell inside t docker compose -f etc/docker-compose.yml -f etc/lighthouse.yml up -d ``` -> **Note** -> -> If you want to run Reth with a CL that is not Lighthouse: -> -> - The JWT for the consensus client can be found at `etc/jwttoken/jwt.hex` in this repository, after the `etc/generate-jwt.sh` script is run -> - The Reth Engine API is accessible on `localhost:8551` +:::note +If you want to run Reth with a CL that is not Lighthouse: + +- The JWT for the consensus client can be found at `etc/jwttoken/jwt.hex` in this repository, after the `etc/generate-jwt.sh` script is run +- The Reth Engine API is accessible on `localhost:8551` + ::: To check if Reth is running correctly, run: @@ -101,18 +107,19 @@ docker compose -f etc/docker-compose.yml -f etc/lighthouse.yml logs -f reth The default `docker-compose.yml` file will create three containers: -- Reth -- Prometheus -- Grafana +- Reth +- Prometheus +- Grafana The optional `lighthouse.yml` file will create two containers: -- Lighthouse -- [`ethereum-metrics-exporter`](https://github.com/ethpandaops/ethereum-metrics-exporter) +- Lighthouse +- [`ethereum-metrics-exporter`](https://github.com/ethpandaops/ethereum-metrics-exporter) Grafana will be exposed on `localhost:3000` and accessible via default credentials (username and password is `admin`), with two available dashboards: -- reth -- Ethereum Metrics Exporter (works only if Lighthouse is also running) + +- reth +- Ethereum Metrics Exporter (works only if Lighthouse is also running) ## Interacting with Reth inside Docker @@ -124,7 +131,7 @@ docker exec -it reth bash **If Reth is running with Docker Compose, replace `reth` with `reth-reth-1` in the above command** -Refer to the [CLI docs](../cli/cli.md) to interact with Reth once inside the Reth container. +Refer to the [CLI docs](/cli/reth) to interact with Reth once inside the Reth container. ## Run only Grafana in Docker @@ -134,4 +141,4 @@ This allows importing existing Grafana dashboards, without running Reth in Docke docker compose -f etc/docker-compose.yml up -d --no-deps grafana ``` -After login with `admin:admin` credentials, Prometheus should be listed under [`Grafana datasources`](http://localhost:3000/connections/datasources). Replace its `Prometheus server URL` so it points to locally running one. On Mac or Windows, use `http://host.docker.internal:9090`. On Linux, try `http://172.17.0.1:9090`. \ No newline at end of file +After login with `admin:admin` credentials, Prometheus should be listed under [`Grafana datasources`](http://localhost:3000/connections/datasources). Replace its `Prometheus server URL` so it points to locally running one. On Mac or Windows, use `http://host.docker.internal:9090`. On Linux, try `http://172.17.0.1:9090`. diff --git a/docs/vocs/docs/pages/installation/overview.mdx b/docs/vocs/docs/pages/installation/overview.mdx new file mode 100644 index 00000000000..2a5c21522e2 --- /dev/null +++ b/docs/vocs/docs/pages/installation/overview.mdx @@ -0,0 +1,18 @@ +--- +description: Installation instructions for Reth and hardware recommendations. +--- + +# Installation + +Reth runs on Linux and macOS (Windows tracked). + +There are three core methods to obtain Reth: + +- [Pre-built binaries](/installation/binaries) +- [Docker images](/installation/docker) +- [Building from source.](/installation/source) + +:::note +If you have Docker installed, we recommend using the [Docker Compose](/installation/docker#using-docker-compose) configuration +that will get you Reth, Lighthouse (Consensus Client), Prometheus and Grafana running and syncing with just one command. +::: diff --git a/docs/vocs/docs/pages/installation/priorities.mdx b/docs/vocs/docs/pages/installation/priorities.mdx new file mode 100644 index 00000000000..4494083e399 --- /dev/null +++ b/docs/vocs/docs/pages/installation/priorities.mdx @@ -0,0 +1,22 @@ +--- +description: Explains Reth update priorities for user classes such as payload builders and non-payload builders. +--- + +# Update Priorities + +When publishing releases, reth will include an "Update Priority" section in the release notes, in the same manner Lighthouse does. + +The "Update Priority" section will include a table which may appear like so: + +| User Class | Priority | +| -------------------- | --------------- | +| Payload Builders | Medium Priority | +| Non-Payload Builders | Low Priority | + +To understand this table, the following terms are important: + +- _Payload builders_ are those who use reth to build and validate payloads. +- _Non-payload builders_ are those who run reth for other purposes (e.g., data analysis, RPC or applications). +- _High priority_ updates should be completed as soon as possible (e.g., hours or days). +- _Medium priority_ updates should be completed at the next convenience (e.g., days or a week). +- _Low priority_ updates should be completed in the next routine update cycle (e.g., two weeks). diff --git a/book/installation/source.md b/docs/vocs/docs/pages/installation/source.mdx similarity index 72% rename from book/installation/source.md rename to docs/vocs/docs/pages/installation/source.mdx index d9642c4bc48..a7e1a2c33cc 100644 --- a/book/installation/source.md +++ b/docs/vocs/docs/pages/installation/source.mdx @@ -1,14 +1,18 @@ +--- +description: How to build, update, and troubleshoot Reth from source. +--- + # Build from Source You can build Reth on Linux, macOS, Windows, and Windows WSL2. -> **Note** -> -> Reth does **not** work on Windows WSL1. +:::note +Reth does **not** work on Windows WSL1. +::: ## Dependencies -First, **install Rust** using [rustup](https://rustup.rs/)： +First, **install Rust** using [rustup](https://rustup.rs/)： ```bash curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh @@ -16,19 +20,20 @@ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh The rustup installer provides an easy way to update the Rust compiler, and works on all platforms. -> **Tips** -> -> - During installation, when prompted, enter `1` for the default installation. -> - After Rust installation completes, try running `cargo version` . If it cannot -> be found, run `source $HOME/.cargo/env`. After that, running `cargo version` should return the version, for example `cargo 1.68.2`. -> - It's generally advisable to append `source $HOME/.cargo/env` to `~/.bashrc`. +:::tip + +- During installation, when prompted, enter `1` for the default installation. +- After Rust installation completes, try running `cargo version` . If it cannot + be found, run `source $HOME/.cargo/env`. After that, running `cargo version` should return the version, for example `cargo 1.68.2`. +- It's generally advisable to append `source $HOME/.cargo/env` to `~/.bashrc`. + ::: With Rust installed, follow the instructions below to install dependencies relevant to your operating system: -- **Ubuntu**: `apt-get install libclang-dev pkg-config build-essential` -- **macOS**: `brew install llvm pkg-config` -- **Windows**: `choco install llvm` or `winget install LLVM.LLVM` +- **Ubuntu**: `apt-get install libclang-dev pkg-config build-essential` +- **macOS**: `brew install llvm pkg-config` +- **Windows**: `choco install llvm` or `winget install LLVM.LLVM` These are needed to build bindings for Reth's database. @@ -60,7 +65,7 @@ cargo build --release This will place the reth binary under `./target/release/reth`, and you can copy it to your directory of preference after that. -Compilation may take around 10 minutes. Installation was successful if `reth --help` displays the [command-line documentation](../cli/cli.md). +Compilation may take around 10 minutes. Installation was successful if `reth --help` displays the [command-line documentation](/cli/reth). If you run into any issues, please check the [Troubleshooting](#troubleshooting) section, or reach out to us on [Telegram](https://t.me/paradigm_reth). @@ -88,11 +93,11 @@ You can customise the compiler settings used to compile Reth via Reth includes several profiles which can be selected via the Cargo flag `--profile`. -* `release`: default for source builds, enables most optimisations while not taking too long to - compile. -* `maxperf`: default for binary releases, enables aggressive optimisations including full LTO. - Although compiling with this profile improves some benchmarks by around 20% compared to `release`, - it imposes a _significant_ cost at compile time and is only recommended if you have a fast CPU. +- `release`: default for source builds, enables most optimisations while not taking too long to + compile. +- `maxperf`: default for binary releases, enables aggressive optimisations including full LTO. + Although compiling with this profile improves some benchmarks by around 20% compared to `release`, + it imposes a _significant_ cost at compile time and is only recommended if you have a fast CPU. **Rust compiler flags** @@ -107,9 +112,10 @@ RUSTFLAGS="-C target-cpu=native" cargo build --profile maxperf Finally, some optional features are present that may improve performance, but may not very portable, and as such might not compile on your particular system. These are currently: -- `jemalloc`: replaces the default system memory allocator with [`jemalloc`](https://jemalloc.net/); this feature is unstable on Windows -- `asm-keccak`: replaces the default, pure-Rust implementation of Keccak256 with one implemented in assembly; see [the `keccak-asm` crate](https://github.com/DaniPopes/keccak-asm) for more details and supported targets -- `min-LEVEL-logs`, where `LEVEL` is one of `error`, `warn`, `info`, `debug`, `trace`: disables compilation of logs of lower level than the given one; this in general isn't that significant, and is not recommended due to the loss of debugging that the logs would provide + +- `jemalloc`: replaces the default system memory allocator with [`jemalloc`](https://jemalloc.net/); this feature is unstable on Windows +- `asm-keccak`: replaces the default, pure-Rust implementation of Keccak256 with one implemented in assembly; see [the `keccak-asm` crate](https://github.com/DaniPopes/keccak-asm) for more details and supported targets +- `min-LEVEL-logs`, where `LEVEL` is one of `error`, `warn`, `info`, `debug`, `trace`: disables compilation of logs of lower level than the given one; this in general isn't that significant, and is not recommended due to the loss of debugging that the logs would provide You can activate features by passing them to the `--features` or `-F` Cargo flag; multiple features can be activated with a space- or comma-separated list to the flag: @@ -136,7 +142,7 @@ Rust Version (MSRV) which is listed under the `rust-version` key in Reth's If compilation fails with `(signal: 9, SIGKILL: kill)`, this could mean your machine ran out of memory during compilation. If you are on Docker, consider increasing the memory of the container, or use a [pre-built -binary](../installation/binaries.md). +binary](/installation/binaries). If compilation fails in either the `keccak-asm` or `sha3-asm` crates, it is likely that your current system configuration is not supported. See the [`keccak-asm` target table](https://github.com/DaniPopes/keccak-asm?tab=readme-ov-file#support) for supported targets. @@ -147,7 +153,7 @@ _(Thanks to Sigma Prime for this section from [their Lighthouse book](https://li ### Bus error (WSL2) -In WSL 2 on Windows, the default virtual disk size is set to 1TB. +In WSL 2 on Windows, the default virtual disk size is set to 1TB. You must increase the allocated disk size for your WSL2 instance before syncing reth. diff --git a/docs/vocs/docs/pages/introduction/contributing.mdx b/docs/vocs/docs/pages/introduction/contributing.mdx new file mode 100644 index 00000000000..aa30ee5faf2 --- /dev/null +++ b/docs/vocs/docs/pages/introduction/contributing.mdx @@ -0,0 +1,258 @@ +# Contributing to Reth + +Reth has docs specifically geared for developers and contributors, including documentation on the structure and architecture of reth, the general workflow we employ, and other useful tips. + +## Getting Help + +Need support or have questions? Open a github issue and/or join the TG chat: + +- **GitHub Issues**: [Open an issue](https://github.com/paradigmxyz/reth/issues/new) for bugs or feature requests +- **Telegram Chat**: [Join our Telegram](https://t.me/paradigm_reth) for real-time support and discussions + +## Repository and Project Structure + +Reth is organized as a modular codebase with clear separation and a contributor friendly architecture, you can read about it in detail [here](https://github.com/paradigmxyz/reth/tree/main/docs). Here's the TL;DR: + +### Design + +Reth follows a modular architecture where each component can be used independently: + +- **Consensus**: Block validation and consensus rules +- **Storage**: Hybrid database with MDBX + static files +- **Networking**: P2P networking stack +- **RPC**: JSON-RPC server implementation +- **Engine**: Consensus layer integration +- **EVM**: Transaction execution +- **Node Builder**: High-level orchestration + +### Crates + +The repository is organized into focused crates under `/crates/`: + +``` +crates/ +├── consensus/ # Consensus and validation logic +├── storage/ # Database and storage implementations +├── net/ # Networking components +├── rpc/ # JSON-RPC server and APIs +├── engine/ # Engine API and consensus integration +├── evm/ # EVM execution +├── node/ # Node building and orchestration +├── ethereum/ # Ethereum-specific implementations +├── optimism/ # Optimism L2 support +└── ... +``` + +## Workflow: The Lifecycle of PRs + +### 1. Before You Start + +- Check existing issues to avoid duplicate work +- For large features, open an issue first to discuss the approach +- Fork the repository and create a feature branch + +### 2. Development Process + +#### Setting Up Your Environment + +```bash +# Clone your fork +git clone https://github.com/YOUR_USERNAME/reth.git +cd reth + +# Install dependencies and tools +# Use nightly Rust for formatting +rustup install nightly +rustup component add rustfmt --toolchain nightly + +# Run the validation suite +make pr +``` + +#### Code Style and Standards + +- **Formatting**: Use nightly rustfmt (`cargo +nightly fmt`) +- **Linting**: All clippy warnings must be addressed +- **Documentation**: Add doc comments for public APIs +- **Testing**: Include appropriate tests for your changes + +#### Recommended VS Code Settings + +Install the `rust-analyzer` extension and use these settings for the best development experience: + +```json +{ + "rust-analyzer.rustfmt.overrideCommand": ["rustfmt", "+nightly"], + "rust-analyzer.check.overrideCommand": [ + "cargo", + "clippy", + "--workspace", + "--message-format=json", + "--all-targets", + "--all-features" + ] +} +``` + +### 3. Testing Your Changes + +Reth uses comprehensive testing at multiple levels: + +#### Unit Tests + +Test specific functions and components: + +```bash +cargo test --package reth-ethereum-consensus +``` + +#### Integration Tests + +Test component interactions: + +```bash +cargo test --test integration_tests +``` + +#### Full Test Suite + +Run all tests including Ethereum Foundation tests: + +```bash +make test +``` + +#### Validation Suite + +Before submitting, always run: + +```bash +make pr +``` + +This runs: + +- Code formatting checks +- Clippy linting +- Documentation generation +- Full test suite + +### 4. Submitting Your PR + +#### Draft PRs for Large Features + +For substantial changes, open a draft PR early to get feedback on the approach. + +#### PR Requirements + +- [ ] Clear, descriptive title and description +- [ ] Tests for new functionality +- [ ] Documentation updates if needed +- [ ] All CI checks passing +- [ ] Commit messages follow conventional format + +#### Commit Message Format + +``` +type: brief description + +More detailed explanation if needed. + +- feat: new feature +- fix: bug fix +- docs: documentation changes +- refactor: code refactoring +- test: adding tests +- chore: maintenance tasks +``` + +### 5. Review Process + +#### Who Can Review + +Any community member can review PRs. We encourage participation from all skill levels. + +#### What Reviewers Look For + +- **Does the change improve Reth?** +- **Are there clear bugs or issues?** +- **Are commit messages clear and descriptive?** +- **Is the code well-tested?** +- **Is documentation updated appropriately?** + +#### Review Guidelines + +- Be constructive and respectful +- Provide specific, actionable feedback +- Focus on significant issues first +- Acknowledge good work and improvements + +## Releases: How Reth is Released + +### Release Schedule + +- **Regular releases**: Following semantic versioning +- **Security releases**: As needed for critical vulnerabilities +- **Pre-releases**: For testing major changes + +### Release Process + +1. **Version bump**: Update version numbers across crates +2. **Changelog**: Update `CHANGELOG.md` with notable changes +3. **Testing**: Final validation on testnet and mainnet +4. **Tagging**: Create release tags and GitHub releases +5. **Distribution**: Update package registries and Docker images + +### Release Criteria + +- All CI checks passing +- No known critical bugs +- Documentation up to date +- Backwards compatibility considerations addressed + +## Ways to Contribute + +### 💡 Feature Requests + +For feature requests, please include: + +- **Detailed explanation**: What should the feature do? +- **Context and motivation**: Why is this feature needed? +- **Examples**: How would it be used? +- **Similar tools**: References to similar functionality elsewhere + +### 📝 Documentation + +Documentation improvements are always welcome: + +- Add missing documentation +- Improve code examples +- Create tutorials or guides + +### 🔧 Code Contributions + +Contributing code changes: + +- Fix bugs identified in issues +- Implement requested features +- Improve performance +- Refactor for better maintainability + +## Code of Conduct + +Reth follows the [Rust Code of Conduct](https://www.rust-lang.org/conduct.html). We are committed to providing a welcoming and inclusive environment for all contributors. + +### Our Standards + +- Be respectful and constructive +- Focus on what's best for the community +- Show empathy towards other contributors +- Accept constructive criticism gracefully + +### Reporting Issues + +If you experience or witness behavior that violates our code of conduct, please report it to [georgios@paradigm.xyz](mailto:georgios@paradigm.xyz). + +:::note +Also read [CONTRIBUTING.md](https://github.com/paradigmxyz/reth/blob/main/CONTRIBUTING.md) for in-depth guidelines. +::: diff --git a/docs/vocs/docs/pages/introduction/why-reth.mdx b/docs/vocs/docs/pages/introduction/why-reth.mdx new file mode 100644 index 00000000000..df83681a38d --- /dev/null +++ b/docs/vocs/docs/pages/introduction/why-reth.mdx @@ -0,0 +1,50 @@ +--- +description: Why Reth is the future of Ethereum infrastructure - powering everything from production staking to cutting-edge L2s and ZK applications. +--- + +# Why Reth? + +Reth is more than just another Ethereum client—it's the foundation upon which the next generation of blockchain infrastructure is being built. From powering production staking environments at institutions like Coinbase to enabling cutting-edge L2 sequencers and ZK applications, Reth represents the convergence of security, performance, and extensibility that the ecosystem demands. + +Every piece of crypto infrastructure will be touching Reth one way or another. Here's why the world's leading developers and institutions are choosing Reth as their node of choice. + +## Institutional-Grade Security + +Reth secures real value on Ethereum mainnet today, trusted by institutions like [Coinbase](https://x.com/CoinbasePltfrm/status/1933546893742579890) for production staking infrastructure. It powers RPC providers such as Alchemy. + +## Future Proof Performance + +Reth pushes the performance frontier across every dimension, from L2 sequencers to MEV block building. + +- **L2 Sequencer Performance**: Used by [Base](https://www.base.org/), other production L2s and also rollup-as-a-service providers such as [Conduit](https://conduit.xyz) which require high throughput and fast block times. +- **MEV & Block Building**: [rbuilder](https://github.com/flashbots/rbuilder) is an open-source implementation of a block builder built on Reth due to developer friendliness and blazing fast performance. + +## Infinitely Customizable + +Reth's modular architecture means you are not locked into someone else's design decisions—build exactly the chain you need. + +- **Component-Based Design**: Swap out consensus, execution, mempool, or networking modules independently +- **Custom Transaction Types**: Build specialized DeFi chains, and unique economic models +- **Rapid Development**: Reth SDK accelerates custom blockchain development with pre-built components + +## ZK & Stateless Ready + +Reth is designed from the ground up to excel in the zero-knowledge future with stateless execution and modular architecture. + +[SP1](https://github.com/succinctlabs/sp1), a zkVM for proving arbitrary Rust programs, and [Ress](https://www.paradigm.xyz/2025/03/stateless-reth-nodes), an experimental stateless node, demonstrate how Reth enables scalable zero-knowledge applications for Ethereum. + +## Thriving Open Source Ecosystem + +The most important factor in Reth's success is our vibrant open source community building the future together. + +500+ geo-distributed developers from leading companies and academia have played a role to build Reth into what it is today. + +## Join the community + +Reth isn't just a tool—it's a movement toward better blockchain infrastructure. Whether you're running a validator, building the next generation of L2s, or creating cutting-edge ZK applications, Reth provides the foundation you need to succeed. + +**Ready to build the future?** + +- [Get Started](/run/ethereum) with running your first Reth node +- [Explore the SDK](/sdk) to build custom blockchain infrastructure +- [Join the Community](https://github.com/paradigmxyz/reth) and contribute to the future of Ethereum diff --git a/book/jsonrpc/admin.md b/docs/vocs/docs/pages/jsonrpc/admin.mdx similarity index 78% rename from book/jsonrpc/admin.md rename to docs/vocs/docs/pages/jsonrpc/admin.mdx index b85cd194b6d..481a4f76d76 100644 --- a/book/jsonrpc/admin.md +++ b/docs/vocs/docs/pages/jsonrpc/admin.mdx @@ -1,10 +1,13 @@ +--- +description: Admin API for node configuration and peer management. +--- # `admin` Namespace The `admin` API allows you to configure your node, including adding and removing peers. -> **Note** -> -> As this namespace can configure your node at runtime, it is generally **not advised** to expose it publicly. +:::note +As this namespace can configure your node at runtime, it is generally **not advised** to expose it publicly. +::: ## `admin_addPeer` @@ -13,7 +16,7 @@ Add the given peer to the current peer set of the node. The method accepts a single argument, the [`enode`][enode] URL of the remote peer to connect to, and returns a `bool` indicating whether the peer was accepted or not. | Client | Method invocation | -|--------|------------------------------------------------| +| ------ | ---------------------------------------------- | | RPC | `{"method": "admin_addPeer", "params": [url]}` | ### Example @@ -27,9 +30,9 @@ The method accepts a single argument, the [`enode`][enode] URL of the remote pee Disconnects from a peer if the connection exists. Returns a `bool` indicating whether the peer was successfully removed or not. -| Client | Method invocation | -|--------|----------------------------------------------------| -| RPC | `{"method": "admin_removePeer", "params": [url]}` | +| Client | Method invocation | +| ------ | ------------------------------------------------- | +| RPC | `{"method": "admin_removePeer", "params": [url]}` | ### Example @@ -40,12 +43,12 @@ Disconnects from a peer if the connection exists. Returns a `bool` indicating wh ## `admin_addTrustedPeer` -Adds the given peer to a list of trusted peers, which allows the peer to always connect, even if there would be no room for it otherwise. +Adds the given peer to a list of trusted peers, which allows the peer to always connect, even if there is no room for it otherwise. It returns a `bool` indicating whether the peer was added to the list or not. | Client | Method invocation | -|--------|-------------------------------------------------------| +| ------ | ----------------------------------------------------- | | RPC | `{"method": "admin_addTrustedPeer", "params": [url]}` | ### Example @@ -62,7 +65,7 @@ Removes a remote node from the trusted peer set, but it does not disconnect it a Returns true if the peer was successfully removed. | Client | Method invocation | -|--------|----------------------------------------------------------| +| ------ | -------------------------------------------------------- | | RPC | `{"method": "admin_removeTrustedPeer", "params": [url]}` | ### Example @@ -79,7 +82,7 @@ Returns all information known about the running node. These include general information about the node itself, as well as what protocols it participates in, its IP and ports. | Client | Method invocation | -|--------|--------------------------------| +| ------ | ------------------------------ | | RPC | `{"method": "admin_nodeInfo"}` | ### Example @@ -121,9 +124,9 @@ Like other subscription methods, this returns the ID of the subscription, which To unsubscribe from peer events, call `admin_peerEvents_unsubscribe` with the subscription ID. -| Client | Method invocation | -|--------|-------------------------------------------------------| -| RPC | `{"method": "admin_peerEvents", "params": []}` | +| Client | Method invocation | +| ------ | ------------------------------------------------------------ | +| RPC | `{"method": "admin_peerEvents", "params": []}` | | RPC | `{"method": "admin_peerEvents_unsubscribe", "params": [id]}` | ### Event Types @@ -132,20 +135,20 @@ The subscription emits events with the following structure: ```json { - "jsonrpc": "2.0", - "method": "admin_subscription", - "params": { - "subscription": "0xcd0c3e8af590364c09d0fa6a1210faf5", - "result": { - "type": "add", // or "drop", "error" - "peer": { - "id": "44826a5d6a55f88a18298bca4773fca5749cdc3a5c9f308aa7d810e9b31123f3e7c5fba0b1d70aac5308426f47df2a128a6747040a3815cc7dd7167d03be320d", - "enode": "enode://44826a5d6a55f88a18298bca4773fca5749cdc3a5c9f308aa7d810e9b31123f3e7c5fba0b1d70aac5308426f47df2a128a6747040a3815cc7dd7167d03be320d@192.168.1.1:30303", - "addr": "192.168.1.1:30303" - }, - "error": "reason for disconnect or error" // only present for "drop" and "error" events + "jsonrpc": "2.0", + "method": "admin_subscription", + "params": { + "subscription": "0xcd0c3e8af590364c09d0fa6a1210faf5", + "result": { + "type": "add", // or "drop", "error" + "peer": { + "id": "44826a5d6a55f88a18298bca4773fca5749cdc3a5c9f308aa7d810e9b31123f3e7c5fba0b1d70aac5308426f47df2a128a6747040a3815cc7dd7167d03be320d", + "enode": "enode://44826a5d6a55f88a18298bca4773fca5749cdc3a5c9f308aa7d810e9b31123f3e7c5fba0b1d70aac5308426f47df2a128a6747040a3815cc7dd7167d03be320d@192.168.1.1:30303", + "addr": "192.168.1.1:30303" + }, + "error": "reason for disconnect or error" // only present for "drop" and "error" events + } } - } } ``` diff --git a/book/jsonrpc/debug.md b/docs/vocs/docs/pages/jsonrpc/debug.mdx similarity index 77% rename from book/jsonrpc/debug.md rename to docs/vocs/docs/pages/jsonrpc/debug.mdx index 7965e2e0d50..5b435d7dca7 100644 --- a/book/jsonrpc/debug.md +++ b/docs/vocs/docs/pages/jsonrpc/debug.mdx @@ -1,3 +1,6 @@ +--- +description: Debug API for inspecting Ethereum state and traces. +--- # `debug` Namespace The `debug` API provides several methods to inspect the Ethereum state, including Geth-style traces. @@ -7,7 +10,7 @@ The `debug` API provides several methods to inspect the Ethereum state, includin Returns an RLP-encoded header. | Client | Method invocation | -|--------|-------------------------------------------------------| +| ------ | ----------------------------------------------------- | | RPC | `{"method": "debug_getRawHeader", "params": [block]}` | ## `debug_getRawBlock` @@ -15,7 +18,7 @@ Returns an RLP-encoded header. Retrieves and returns the RLP encoded block by number, hash or tag. | Client | Method invocation | -|--------|------------------------------------------------------| +| ------ | ---------------------------------------------------- | | RPC | `{"method": "debug_getRawBlock", "params": [block]}` | ## `debug_getRawTransaction` @@ -23,7 +26,7 @@ Retrieves and returns the RLP encoded block by number, hash or tag. Returns an EIP-2718 binary-encoded transaction. | Client | Method invocation | -|--------|--------------------------------------------------------------| +| ------ | ------------------------------------------------------------ | | RPC | `{"method": "debug_getRawTransaction", "params": [tx_hash]}` | ## `debug_getRawReceipts` @@ -31,7 +34,7 @@ Returns an EIP-2718 binary-encoded transaction. Returns an array of EIP-2718 binary-encoded receipts. | Client | Method invocation | -|--------|---------------------------------------------------------| +| ------ | ------------------------------------------------------- | | RPC | `{"method": "debug_getRawReceipts", "params": [block]}` | ## `debug_getBadBlocks` @@ -39,7 +42,7 @@ Returns an array of EIP-2718 binary-encoded receipts. Returns an array of recent bad blocks that the client has seen on the network. | Client | Method invocation | -|--------|--------------------------------------------------| +| ------ | ------------------------------------------------ | | RPC | `{"method": "debug_getBadBlocks", "params": []}` | ## `debug_traceChain` @@ -47,21 +50,21 @@ Returns an array of recent bad blocks that the client has seen on the network. Returns the structured logs created during the execution of EVM between two blocks (excluding start) as a JSON object. | Client | Method invocation | -|--------|----------------------------------------------------------------------| +| ------ | -------------------------------------------------------------------- | | RPC | `{"method": "debug_traceChain", "params": [start_block, end_block]}` | ## `debug_traceBlock` -The `debug_traceBlock` method will return a full stack trace of all invoked opcodes of all transaction that were included in this block. +The `debug_traceBlock` method will return a full stack trace of all invoked opcodes of all transactions that were included in this block. This expects an RLP-encoded block. > **Note** -> +> > The parent of this block must be present, or it will fail. | Client | Method invocation | -|--------|---------------------------------------------------------| +| ------ | ------------------------------------------------------- | | RPC | `{"method": "debug_traceBlock", "params": [rlp, opts]}` | ## `debug_traceBlockByHash` @@ -69,7 +72,7 @@ This expects an RLP-encoded block. Similar to [`debug_traceBlock`](#debug_traceblock), `debug_traceBlockByHash` accepts a block hash and will replay the block that is already present in the database. | Client | Method invocation | -|--------|----------------------------------------------------------------------| +| ------ | -------------------------------------------------------------------- | | RPC | `{"method": "debug_traceBlockByHash", "params": [block_hash, opts]}` | ## `debug_traceBlockByNumber` @@ -77,25 +80,25 @@ Similar to [`debug_traceBlock`](#debug_traceblock), `debug_traceBlockByHash` acc Similar to [`debug_traceBlockByHash`](#debug_traceblockbyhash), `debug_traceBlockByNumber` accepts a block number and will replay the block that is already present in the database. | Client | Method invocation | -|--------|--------------------------------------------------------------------------| +| ------ | ------------------------------------------------------------------------ | | RPC | `{"method": "debug_traceBlockByNumber", "params": [block_number, opts]}` | ## `debug_traceTransaction` The `debug_traceTransaction` debugging method will attempt to run the transaction in the exact same manner as it was executed on the network. It will replay any transaction that may have been executed prior to this one before it will finally attempt to execute the transaction that corresponds to the given hash. -| Client | Method invocation | -|--------|-------------------------------------------------------------| +| Client | Method invocation | +| ------ | ----------------------------------------------------------------- | | RPC | `{"method": "debug_traceTransaction", "params": [tx_hash, opts]}` | ## `debug_traceCall` -The `debug_traceCall` method lets you run an `eth_call` within the context of the given block execution using the final state of parent block as the base. +The `debug_traceCall` method lets you run an `eth_call` within the context of the given block execution using the final state of the parent block as the base. The first argument (just as in `eth_call`) is a transaction request. The block can optionally be specified either by hash or by number as the second argument. | Client | Method invocation | -|--------|-----------------------------------------------------------------------| +| ------ | --------------------------------------------------------------------- | | RPC | `{"method": "debug_traceCall", "params": [call, block_number, opts]}` | diff --git a/book/jsonrpc/eth.md b/docs/vocs/docs/pages/jsonrpc/eth.mdx similarity index 72% rename from book/jsonrpc/eth.md rename to docs/vocs/docs/pages/jsonrpc/eth.mdx index 0a3003c4052..052beb4c7b9 100644 --- a/book/jsonrpc/eth.md +++ b/docs/vocs/docs/pages/jsonrpc/eth.mdx @@ -1,3 +1,7 @@ +--- +description: Standard Ethereum JSON-RPC API methods. +--- + # `eth` Namespace Documentation for the API methods in the `eth` namespace can be found on [ethereum.org](https://ethereum.org/en/developers/docs/apis/json-rpc/). diff --git a/book/jsonrpc/intro.md b/docs/vocs/docs/pages/jsonrpc/intro.mdx similarity index 69% rename from book/jsonrpc/intro.md rename to docs/vocs/docs/pages/jsonrpc/intro.mdx index 6f9b894988d..93cccf46921 100644 --- a/book/jsonrpc/intro.md +++ b/docs/vocs/docs/pages/jsonrpc/intro.mdx @@ -1,3 +1,7 @@ +--- +description: Overview of Reth's JSON-RPC API and namespaces. +--- + # JSON-RPC You can interact with Reth over JSON-RPC. Reth supports all standard Ethereum JSON-RPC API methods. @@ -12,22 +16,21 @@ Each namespace must be explicitly enabled. The methods are grouped into namespaces, which are listed below: -| Namespace | Description | Sensitive | -|-------------------------|--------------------------------------------------------------------------------------------------------|-----------| -| [`eth`](./eth.md) | The `eth` API allows you to interact with Ethereum. | Maybe | -| [`web3`](./web3.md) | The `web3` API provides utility functions for the web3 client. | No | -| [`net`](./net.md) | The `net` API provides access to network information of the node. | No | -| [`txpool`](./txpool.md) | The `txpool` API allows you to inspect the transaction pool. | No | -| [`debug`](./debug.md) | The `debug` API provides several methods to inspect the Ethereum state, including Geth-style traces. | No | -| [`trace`](./trace.md) | The `trace` API provides several methods to inspect the Ethereum state, including Parity-style traces. | No | -| [`admin`](./admin.md) | The `admin` API allows you to configure your node. | **Yes** | -| [`rpc`](./rpc.md) | The `rpc` API provides information about the RPC server and its modules. | No | +| Namespace | Description | Sensitive | +| -------------------- | ------------------------------------------------------------------------------------------------------ | --------- | +| [`eth`](/jsonrpc/eth) | The `eth` API allows you to interact with Ethereum. | Maybe | +| [`web3`](/jsonrpc/web3) | The `web3` API provides utility functions for the web3 client. | No | +| [`net`](/jsonrpc/net) | The `net` API provides access to network information of the node. | No | +| [`txpool`](/jsonrpc/txpool) | The `txpool` API allows you to inspect the transaction pool. | No | +| [`debug`](/jsonrpc/debug) | The `debug` API provides several methods to inspect the Ethereum state, including Geth-style traces. | No | +| [`trace`](/jsonrpc/trace) | The `trace` API provides several methods to inspect the Ethereum state, including Parity-style traces. | No | +| [`admin`](/jsonrpc/admin) | The `admin` API allows you to configure your node. | **Yes** | +| [`rpc`](/jsonrpc/rpc) | The `rpc` API provides information about the RPC server and its modules. | No | Note that some APIs are sensitive, since they can be used to configure your node (`admin`), or access accounts stored on the node (`eth`). Generally, it is advisable to not expose any JSONRPC namespace publicly, unless you know what you are doing. - ## Transports Reth supports HTTP, WebSockets and IPC. @@ -90,10 +93,10 @@ Because WebSockets are bidirectional, nodes can push events to clients, which en The configuration of the WebSocket server follows the same pattern as the HTTP server: -- Enable it using `--ws` -- Configure the server address by passing `--ws.addr` and `--ws.port` (default `8546`) -- Configure cross-origin requests using `--ws.origins` -- Enable APIs using `--ws.api` +- Enable it using `--ws` +- Configure the server address by passing `--ws.addr` and `--ws.port` (default `8546`) +- Configure cross-origin requests using `--ws.origins` +- Enable APIs using `--ws.api` ### IPC diff --git a/book/jsonrpc/net.md b/docs/vocs/docs/pages/jsonrpc/net.mdx similarity index 82% rename from book/jsonrpc/net.md rename to docs/vocs/docs/pages/jsonrpc/net.mdx index ac40c75b2ab..145b9c27676 100644 --- a/book/jsonrpc/net.md +++ b/docs/vocs/docs/pages/jsonrpc/net.mdx @@ -1,3 +1,7 @@ +--- +description: net_ namespace for Ethereum nodes. +--- + # `net` Namespace The `net` API provides information about the networking component of the node. @@ -7,7 +11,7 @@ The `net` API provides information about the networking component of the node. Returns a `bool` indicating whether or not the node is listening for network connections. | Client | Method invocation | -|--------|---------------------------------------------| +| ------ | ------------------------------------------- | | RPC | `{"method": "net_listening", "params": []}` | ### Example @@ -22,7 +26,7 @@ Returns a `bool` indicating whether or not the node is listening for network con Returns the number of peers connected to the node. | Client | Method invocation | -|--------|---------------------------------------------| +| ------ | ------------------------------------------- | | RPC | `{"method": "net_peerCount", "params": []}` | ### Example @@ -37,7 +41,7 @@ Returns the number of peers connected to the node. Returns the network ID (e.g. 1 for mainnet) | Client | Method invocation | -|--------|-------------------------------------------| +| ------ | ----------------------------------------- | | RPC | `{"method": "net_version", "params": []}` | ### Example @@ -45,4 +49,4 @@ Returns the network ID (e.g. 1 for mainnet) ```js // > {"jsonrpc":"2.0","id":1,"method":"net_version","params":[]} {"jsonrpc":"2.0","id":1,"result":1} -``` \ No newline at end of file +``` diff --git a/book/jsonrpc/rpc.md b/docs/vocs/docs/pages/jsonrpc/rpc.mdx similarity index 91% rename from book/jsonrpc/rpc.md rename to docs/vocs/docs/pages/jsonrpc/rpc.mdx index 0a4739718be..c85babcfe3c 100644 --- a/book/jsonrpc/rpc.md +++ b/docs/vocs/docs/pages/jsonrpc/rpc.mdx @@ -1,3 +1,7 @@ +--- +description: rpc_ namespace for retrieving server information such as enabled namespaces +--- + # `rpc` Namespace The `rpc` API provides methods to get information about the RPC server itself, such as the enabled namespaces. @@ -7,7 +11,7 @@ The `rpc` API provides methods to get information about the RPC server itself, s Lists the enabled RPC namespaces and the versions of each. | Client | Method invocation | -|--------|-------------------------------------------| +| ------ | ----------------------------------------- | | RPC | `{"method": "rpc_modules", "params": []}` | ### Example diff --git a/book/jsonrpc/trace.md b/docs/vocs/docs/pages/jsonrpc/trace.mdx similarity index 57% rename from book/jsonrpc/trace.md rename to docs/vocs/docs/pages/jsonrpc/trace.mdx index ba0f2490b57..182b6c2f703 100644 --- a/book/jsonrpc/trace.md +++ b/docs/vocs/docs/pages/jsonrpc/trace.mdx @@ -1,33 +1,157 @@ -# `trace` Namespace +--- +description: Trace API for inspecting Ethereum state and transactions. +--- - +# `trace` Namespace The `trace` API provides several methods to inspect the Ethereum state, including Parity-style traces. -A similar module exists (with other debug functions) with Geth-style traces ([`debug`](./debug.md)). +A similar module exists (with other debug functions) with Geth-style traces ([`debug`](https://github.com/paradigmxyz/reth/blob/main/docs/vocs/docs/pages/jsonrpc/debug.mdx)). The `trace` API gives deeper insight into transaction processing. There are two types of methods in this API: -- **Ad-hoc tracing APIs** for performing diagnostics on calls or transactions (historical or hypothetical). -- **Transaction-trace filtering APIs** for getting full externality traces on any transaction executed by reth. +- **Ad-hoc tracing APIs** for performing diagnostics on calls or transactions (historical or hypothetical). +- **Transaction-trace filtering APIs** for getting full externality traces on any transaction executed by reth. + +## Trace Format Specification + +The trace API returns different types of trace data depending on the requested trace types. Understanding these formats is crucial for interpreting the results. + +### TraceResults + +The `TraceResults` object is returned by ad-hoc tracing methods (`trace_call`, `trace_callMany`, `trace_rawTransaction`, `trace_replayTransaction`, `trace_replayBlockTransactions`). It contains the following fields: + +| Field | Type | Description | +|-------|------|-------------| +| `output` | `string` | The return value of the traced call, encoded as hex | +| `stateDiff` | `object \| null` | State changes caused by the transaction (only if `stateDiff` trace type requested) | +| `trace` | `array \| null` | Array of transaction traces (only if `trace` trace type requested) | +| `vmTrace` | `object \| null` | Virtual machine execution trace (only if `vmTrace` trace type requested) | + +### LocalizedTransactionTrace + +Individual transaction traces in `trace_block`, `trace_filter`, `trace_get`, and `trace_transaction` methods return `LocalizedTransactionTrace` objects: + +| Field | Type | Description | +|-------|------|-------------| +| `action` | `object` | The action performed by this trace | +| `result` | `object \| null` | The result of the trace execution | +| `error` | `string \| null` | Error message if the trace failed | +| `blockHash` | `string \| null` | Hash of the block containing this trace | +| `blockNumber` | `number \| null` | Number of the block containing this trace | +| `transactionHash` | `string \| null` | Hash of the transaction containing this trace | +| `transactionPosition` | `number \| null` | Position of the transaction in the block | +| `subtraces` | `number` | Number of child traces | +| `traceAddress` | `array` | Position of this trace in the call tree | +| `type` | `string` | Type of action: `"call"`, `"create"`, `"suicide"`, or `"reward"` | + +### Action Types + +#### Call Action (`type: "call"`) + +| Field | Type | Description | +|-------|------|-------------| +| `callType` | `string` | Type of call: `"call"`, `"callcode"`, `"delegatecall"`, or `"staticcall"` | +| `from` | `string` | Address of the caller | +| `to` | `string` | Address of the callee | +| `gas` | `string` | Gas provided for the call | +| `input` | `string` | Input data for the call | +| `value` | `string` | Value transferred in the call | + +#### Create Action (`type: "create"`) + +| Field | Type | Description | +|-------|------|-------------| +| `from` | `string` | Address of the creator | +| `gas` | `string` | Gas provided for contract creation | +| `init` | `string` | Contract initialization code | +| `value` | `string` | Value sent to the new contract | + +#### Suicide Action (`type: "suicide"`) + +| Field | Type | Description | +|-------|------|-------------| +| `address` | `string` | Address of the contract being destroyed | +| `refundAddress` | `string` | Address receiving the remaining balance | +| `balance` | `string` | Balance transferred to refund address | + +#### Reward Action (`type: "reward"`) + +| Field | Type | Description | +|-------|------|-------------| +| `author` | `string` | Address receiving the reward | +| `value` | `string` | Amount of the reward | +| `rewardType` | `string` | Type of reward: `"block"` or `"uncle"` | + +### Result Format + +When a trace executes successfully, the `result` field contains: + +| Field | Type | Description | +|-------|------|-------------| +| `gasUsed` | `string` | Amount of gas consumed by this trace | +| `output` | `string` | Return data from the trace execution | +| `address` | `string` | Created contract address (for create actions only) | +| `code` | `string` | Deployed contract code (for create actions only) | + +### State Diff Format + +When `stateDiff` trace type is requested, the `stateDiff` field contains an object mapping addresses to their state changes: + +```json +{ + "0x123...": { + "balance": { + "*": { + "from": "0x0", + "to": "0x1000" + } + }, + "nonce": { + "*": { + "from": "0x0", + "to": "0x1" + } + }, + "code": { + "*": { + "from": "0x", + "to": "0x608060405234801561001057600080fd5b50..." + } + }, + "storage": { + "0x290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563": { + "*": { + "from": "0x0", + "to": "0x1" + } + } + } + } +} +``` + +### VM Trace Format + +When `vmTrace` trace type is requested, the `vmTrace` field contains detailed virtual machine execution information including opcodes, stack, memory, and storage changes at each step. The exact format depends on the specific VM tracer implementation. ## Ad-hoc tracing APIs Ad-hoc tracing APIs allow you to perform diagnostics on calls or transactions (historical or hypothetical), including: -- Transaction traces (`trace`) -- VM traces (`vmTrace`) -- State difference traces (`stateDiff`) +- Transaction traces (`trace`) +- VM traces (`vmTrace`) +- State difference traces (`stateDiff`) The ad-hoc tracing APIs are: -- [`trace_call`](#trace_call) -- [`trace_callMany`](#trace_callmany) -- [`trace_rawTransaction`](#trace_rawtransaction) -- [`trace_replayBlockTransactions`](#trace_replayblocktransactions) -- [`trace_replayTransaction`](#trace_replaytransaction) +- [`trace_call`](#trace_call) +- [`trace_callMany`](#trace_callmany) +- [`trace_rawTransaction`](#trace_rawtransaction) +- [`trace_replayBlockTransactions`](#trace_replayblocktransactions) +- [`trace_replayTransaction`](#trace_replaytransaction) ## Transaction-trace filtering APIs @@ -37,10 +161,10 @@ Information returned includes the execution of all contract creations, destructi The transaction trace filtering APIs are: -- [`trace_block`](#trace_block) -- [`trace_filter`](#trace_filter) -- [`trace_get`](#trace_get) -- [`trace_transaction`](#trace_transaction) +- [`trace_block`](#trace_block) +- [`trace_filter`](#trace_filter) +- [`trace_get`](#trace_get) +- [`trace_transaction`](#trace_transaction) ## `trace_call` @@ -52,9 +176,9 @@ The second parameter is an array of one or more trace types (`vmTrace`, `trace`, The third and optional parameter is a block number, block hash, or a block tag (`latest`, `finalized`, `safe`, `earliest`, `pending`). -| Client | Method invocation | -|--------|-----------------------------------------------------------| -| RPC | `{"method": "trace_call", "params": [tx, type[], block]}` | +| Client | Method invocation | +| ------ | -------------------------------------------------------------- | +| RPC | `{"method": "trace_callMany", "params": [trace[], block]}` | ### Example @@ -67,7 +191,14 @@ The third and optional parameter is a block number, block hash, or a block tag ( "output": "0x", "stateDiff": null, "trace": [{ - "action": { ... }, + "action": { + "callType": "call", + "from": "0x0000000000000000000000000000000000000000", + "to": "0x0000000000000000000000000000000000000000", + "gas": "0x76c0", + "input": "0x", + "value": "0x0" + }, "result": { "gasUsed": "0x0", "output": "0x" @@ -89,9 +220,9 @@ The first parameter is a list of call traces, where each call trace is of the fo The second and optional parameter is a block number, block hash, or a block tag (`latest`, `finalized`, `safe`, `earliest`, `pending`). -| Client | Method invocation | -|--------|--------------------------------------------------------| -| RPC | `{"method": "trace_call", "params": [trace[], block]}` | +| Client | Method invocation | +| ------ | ---------------------------------------------------------- | +| RPC | `{"method": "trace_callMany", "params": [trace[], block]}` | ### Example @@ -153,9 +284,9 @@ The second and optional parameter is a block number, block hash, or a block tag Traces a call to `eth_sendRawTransaction` without making the call, returning the traces. -| Client | Method invocation | -|--------|--------------------------------------------------------| -| RPC | `{"method": "trace_call", "params": [raw_tx, type[]]}` | +| Client | Method invocation | +| ------ | --------------------------------------------------------------------- | +| RPC | `{"method": "trace_rawTransaction", "params": [raw_tx, type[]]}` | ### Example @@ -166,9 +297,16 @@ Traces a call to `eth_sendRawTransaction` without making the call, returning the "jsonrpc": "2.0", "result": { "output": "0x", - "stateDiff": null, - "trace": [{ - "action": { ... }, + "stateDiff": null, + "trace": [{ + "action": { + "callType": "call", + "from": "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b", + "to": "0x6295ee1b4f6dd65047762f924ecd367c17eabf8f", + "gas": "0x186a0", + "input": "0x", + "value": "0x0" + }, "result": { "gasUsed": "0x0", "output": "0x" @@ -177,7 +315,7 @@ Traces a call to `eth_sendRawTransaction` without making the call, returning the "traceAddress": [], "type": "call" }], - "vmTrace": null + "vmTrace": null } } ``` @@ -187,7 +325,7 @@ Traces a call to `eth_sendRawTransaction` without making the call, returning the Replays all transactions in a block returning the requested traces for each transaction. | Client | Method invocation | -|--------|--------------------------------------------------------------------------| +| ------ | ------------------------------------------------------------------------ | | RPC | `{"method": "trace_replayBlockTransactions", "params": [block, type[]]}` | ### Example @@ -202,7 +340,14 @@ Replays all transactions in a block returning the requested traces for each tran "output": "0x", "stateDiff": null, "trace": [{ - "action": { ... }, + "action": { + "callType": "call", + "from": "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b", + "to": "0x6295ee1b4f6dd65047762f924ecd367c17eabf8f", + "gas": "0x186a0", + "input": "0x", + "value": "0x0" + }, "result": { "gasUsed": "0x0", "output": "0x" @@ -211,10 +356,9 @@ Replays all transactions in a block returning the requested traces for each tran "traceAddress": [], "type": "call" }], - "transactionHash": "0x...", + "transactionHash": "0x4e70b5d8d5dc43e0e61e4a8f1e6e4e6e4e6e4e6e4e6e4e6e4e6e4e6e4e6e4e6e4", "vmTrace": null - }, - { ... } + } ] } ``` @@ -224,7 +368,7 @@ Replays all transactions in a block returning the requested traces for each tran Replays a transaction, returning the traces. | Client | Method invocation | -|--------|----------------------------------------------------------------------| +| ------ | -------------------------------------------------------------------- | | RPC | `{"method": "trace_replayTransaction", "params": [tx_hash, type[]]}` | ### Example @@ -238,10 +382,17 @@ Replays a transaction, returning the traces. "output": "0x", "stateDiff": null, "trace": [{ - "action": { ... }, + "action": { + "callType": "call", + "from": "0x1c39ba39e4735cb65978d4db400ddd70a72dc750", + "to": "0x2bd2326c993dfaef84f696526064ff22eba5b362", + "gas": "0x13e99", + "input": "0x16c72721", + "value": "0x0" + }, "result": { - "gasUsed": "0x0", - "output": "0x" + "gasUsed": "0x183", + "output": "0x0000000000000000000000000000000000000000000000000000000000000001" }, "subtraces": 0, "traceAddress": [], @@ -257,7 +408,7 @@ Replays a transaction, returning the traces. Returns traces created at given block. | Client | Method invocation | -|--------|------------------------------------------------| +| ------ | ---------------------------------------------- | | RPC | `{"method": "trace_block", "params": [block]}` | ### Example @@ -288,8 +439,7 @@ Returns traces created at given block. "transactionHash": "0x07da28d752aba3b9dd7060005e554719c6205c8a3aea358599fc9b245c52f1f6", "transactionPosition": 0, "type": "call" - }, - ... + } ] } ``` @@ -300,17 +450,17 @@ Returns traces matching given filter. Filters are objects with the following properties: -- `fromBlock`: Returns traces from the given block (a number, hash, or a tag like `latest`). -- `toBlock`: Returns traces to the given block. -- `fromAddress`: Sent from these addresses -- `toAddress`: Sent to these addresses -- `after`: The offset trace number -- `count`: The number of traces to display in a batch +- `fromBlock`: Returns traces from the given block (a number, hash, or a tag like `latest`). +- `toBlock`: Returns traces to the given block. +- `fromAddress`: Sent from these addresses +- `toAddress`: Sent to these addresses +- `after`: The offset trace number +- `count`: The number of traces to display in a batch All properties are optional. | Client | Method invocation | -|--------|--------------------------------------------------| +| ------ | ------------------------------------------------ | | RPC | `{"method": "trace_filter", "params": [filter]}` | ### Example @@ -341,8 +491,7 @@ All properties are optional. "transactionHash": "0x3321a7708b1083130bd78da0d62ead9f6683033231617c9d268e2c7e3fa6c104", "transactionPosition": 3, "type": "call" - }, - ... + } ] } ``` @@ -352,7 +501,7 @@ All properties are optional. Returns trace at given position. | Client | Method invocation | -|--------|----------------------------------------------------------| +| ------ | -------------------------------------------------------- | | RPC | `{"method": "trace_get", "params": [tx_hash,indices[]]}` | ### Example @@ -393,7 +542,7 @@ Returns trace at given position. Returns all traces of given transaction | Client | Method invocation | -|--------|--------------------------------------------------------| +| ------ | ------------------------------------------------------ | | RPC | `{"method": "trace_transaction", "params": [tx_hash]}` | ### Example @@ -426,8 +575,7 @@ Returns all traces of given transaction "transactionHash": "0x17104ac9d3312d8c136b7f44d4b8b47852618065ebfa534bd2d3b5ef218ca1f3", "transactionPosition": 2, "type": "call" - }, - ... + } ] } -``` \ No newline at end of file +``` diff --git a/book/jsonrpc/txpool.md b/docs/vocs/docs/pages/jsonrpc/txpool.mdx similarity index 81% rename from book/jsonrpc/txpool.md rename to docs/vocs/docs/pages/jsonrpc/txpool.mdx index cb9e9c0e69d..57f89c643c6 100644 --- a/book/jsonrpc/txpool.md +++ b/docs/vocs/docs/pages/jsonrpc/txpool.mdx @@ -1,3 +1,7 @@ +--- +description: API for inspecting the transaction pool. +--- + # `txpool` Namespace The `txpool` API allows you to inspect the transaction pool. @@ -9,7 +13,7 @@ Returns the details of all transactions currently pending for inclusion in the n See [here](https://geth.ethereum.org/docs/rpc/ns-txpool#txpool-content) for more details | Client | Method invocation | -|--------|----------------------------------------------| +| ------ | -------------------------------------------- | | RPC | `{"method": "txpool_content", "params": []}` | ## `txpool_contentFrom` @@ -19,7 +23,7 @@ Retrieves the transactions contained within the txpool, returning pending as wel See [here](https://geth.ethereum.org/docs/rpc/ns-txpool#txpool-contentfrom) for more details | Client | Method invocation | -|--------|---------------------------------------------------------| +| ------ | ------------------------------------------------------- | | RPC | `{"method": "txpool_contentFrom", "params": [address]}` | ## `txpool_inspect` @@ -29,7 +33,7 @@ Returns a summary of all the transactions currently pending for inclusion in the See [here](https://geth.ethereum.org/docs/rpc/ns-txpool#txpool-inspect) for more details | Client | Method invocation | -|--------|----------------------------------------------| +| ------ | -------------------------------------------- | | RPC | `{"method": "txpool_inspect", "params": []}` | ## `txpool_status` @@ -39,5 +43,5 @@ Returns the number of transactions currently pending for inclusion in the next b See [here](https://geth.ethereum.org/docs/rpc/ns-txpool#txpool-status) for more details | Client | Method invocation | -|--------|---------------------------------------------| -| RPC | `{"method": "txpool_status", "params": []}` | \ No newline at end of file +| ------ | ------------------------------------------- | +| RPC | `{"method": "txpool_status", "params": []}` | diff --git a/book/jsonrpc/web3.md b/docs/vocs/docs/pages/jsonrpc/web3.mdx similarity index 83% rename from book/jsonrpc/web3.md rename to docs/vocs/docs/pages/jsonrpc/web3.mdx index 8221e5c2507..f1eb68bcafe 100644 --- a/book/jsonrpc/web3.md +++ b/docs/vocs/docs/pages/jsonrpc/web3.mdx @@ -1,3 +1,7 @@ +--- +description: Web3 API utility methods for Ethereum clients. +--- + # `web3` Namespace The `web3` API provides utility functions for the web3 client. @@ -6,9 +10,8 @@ The `web3` API provides utility functions for the web3 client. Get the web3 client version. - | Client | Method invocation | -|--------|------------------------------------| +| ------ | ---------------------------------- | | RPC | `{"method": "web3_clientVersion"}` | ### Example @@ -23,7 +26,7 @@ Get the web3 client version. Get the Keccak-256 hash of the given data. | Client | Method invocation | -|--------|----------------------------------------------| +| ------ | -------------------------------------------- | | RPC | `{"method": "web3_sha3", "params": [bytes]}` | ### Example @@ -36,4 +39,4 @@ Get the Keccak-256 hash of the given data. ```js // > {"jsonrpc":"2.0","id":1,"method":"web3_sha3","params":["0x7275737420697320617765736f6d65"]} {"jsonrpc":"2.0","id":1,"result":"0xe421b3428564a5c509ac118bad93a3b84485ec3f927e214b0c4c23076d4bc4e0"} -``` \ No newline at end of file +``` diff --git a/book/intro.md b/docs/vocs/docs/pages/overview.mdx similarity index 68% rename from book/intro.md rename to docs/vocs/docs/pages/overview.mdx index 077cfed3088..5c3a8f9381c 100644 --- a/book/intro.md +++ b/docs/vocs/docs/pages/overview.mdx @@ -1,15 +1,14 @@ -# Reth Book -_Documentation for Reth users and developers._ +--- +description: Reth - A secure, performant, and modular blockchain SDK and Ethereum node. +--- -[![Telegram Chat][tg-badge]][tg-url] +# Reth [Documentation for Reth users and developers] Reth (short for Rust Ethereum, [pronunciation](https://twitter.com/kelvinfichter/status/1597653609411268608)) is an **Ethereum full node implementation that is focused on being user-friendly, highly modular, as well as being fast and efficient.** Reth is production ready, and suitable for usage in mission-critical environments such as staking or high-uptime services. We also actively recommend professional node operators to switch to Reth in production for performance and cost reasons in use cases where high performance with great margins is required such as RPC, MEV, Indexing, Simulations, and P2P activities. - - - +![Reth](https://raw.githubusercontent.com/paradigmxyz/reth/main/assets/reth-prod.png) ## What is this about? @@ -41,7 +40,7 @@ We also use our Ethereum libraries (including [Alloy](https://github.com/alloy-r **3. Free for anyone to use any way they want** -Reth is free open source software, built for the community, by the community. +Reth is free open-source software, built for the community, by the community. By licensing the software under the Apache/MIT license, we want developers to use it without being bound by business licenses, or having to think about the implications of GPL-like licenses. @@ -60,8 +59,9 @@ We envision that Reth will be configurable enough for the tradeoffs that each te ## Who is this for? Reth is a new Ethereum full node that allows users to sync and interact with the entire blockchain, including its historical state if in archive mode. -- Full node: It can be used as a full node, which stores and processes the entire blockchain, validates blocks and transactions, and participates in the consensus process. -- Archive node: It can also be used as an archive node, which stores the entire history of the blockchain and is useful for applications that need access to historical data. + +- Full node: It can be used as a full node, which stores and processes the entire blockchain, validates blocks and transactions, and participates in the consensus process. +- Archive node: It can also be used as an archive node, which stores the entire history of the blockchain and is useful for applications that need access to historical data. As a data engineer/analyst, or as a data indexer, you'll want to use Archive mode. For all other use cases where historical access is not needed, you can use Full mode. @@ -71,30 +71,44 @@ Reth implements the specification of Ethereum as defined in the [ethereum/execut 1. EVM state tests are run on every [Revm Pull Request](https://github.com/bluealloy/revm/blob/main/.github/workflows/ethereum-tests.yml) 1. Hive tests are [run every 24 hours](https://github.com/paradigmxyz/reth/blob/main/.github/workflows/hive.yml) in the main Reth repository. -1. We regularly re-sync multiple nodes from scratch. -1. We operate multiple nodes at the tip of Ethereum mainnet and various testnets. +1. We regularly resync multiple nodes from scratch. +1. We operate multiple nodes at the tip of the Ethereum mainnet and various testnets. 1. We extensively unit test, fuzz test and document all our code, while also restricting PRs with aggressive lint rules. We have completed an audit of the [Reth v1.0.0-rc.2](https://github.com/paradigmxyz/reth/releases/tag/v1.0.0-rc.2) with [Sigma Prime](https://sigmaprime.io/), the developers of [Lighthouse](https://github.com/sigp/lighthouse), the Rust Consensus Layer implementation. Find it [here](https://github.com/paradigmxyz/reth/blob/main/audit/sigma_prime_audit_v2.pdf). [Revm](https://github.com/bluealloy/revm) (the EVM used in Reth) underwent an audit with [Guido Vranken](https://twitter.com/guidovranken) (#1 [Ethereum Bug Bounty](https://ethereum.org/en/bug-bounty)). We will publish the results soon. +## Reth Metrics + +We operate several public Reth nodes across different networks. You can monitor their performance metrics through our public Grafana dashboards: + +| Name | Chain ID | Type | Grafana | +| -------- | -------- | ------- | ---------------------------------------------------------------------------------- | +| Ethereum | 1 | Full | [View](https://reth.ithaca.xyz/public-dashboards/23ceb3bd26594e349aaaf2bcf336d0d4) | +| Ethereum | 1 | Archive | [View](https://reth.ithaca.xyz/public-dashboards/a49fa110dc9149298fa6763d5c89c8c0) | +| Base | 8453 | Archive | [View](https://reth.ithaca.xyz/public-dashboards/b3e9f2e668ee4b86960b7fac691b5e64) | +| OP | 10 | Full | [View](https://reth.ithaca.xyz/public-dashboards/aa32f6c39a664f9aa371399b59622527) | + +:::tip +Want to set up metrics for your own Reth node? Check out our [monitoring guide](/run/monitoring) to learn how to configure Prometheus metrics and build your own dashboards. +::: ## Sections Here are some useful sections to jump to: -- Install Reth by following the [guide](./installation/installation.md). -- Sync your node on any [official network](./run/run-a-node.md). -- View [statistics and metrics](./run/observability.md) about your node. -- Query the [JSON-RPC](./jsonrpc/intro.md) using Foundry's `cast` or `curl`. -- Set up your [development environment and contribute](./developers/contribute.md)! +- Install Reth by following the [guide](/installation/overview). +- Sync your node on any [official network](/run/overview). +- View [statistics and metrics](/run/monitoring) about your node. +- Query the [JSON-RPC](/jsonrpc/intro) using Foundry's `cast` or `curl`. +- Set up your [development environment and contribute](/introduction/contributing)! -> 📖 **About this book** -> -> The book is continuously rendered [here](https://paradigmxyz.github.io/reth/)! -> You can contribute to this book on [GitHub][gh-book]. +:::note +The documentation is continuously rendered [here](https://reth.rs)! +You can contribute to the docs on [GitHub][gh-docs]. +::: [tg-badge]: https://img.shields.io/endpoint?color=neon&logo=telegram&label=chat&url=https%3A%2F%2Ftg.sumanjay.workers.dev%2Fparadigm%5Freth [tg-url]: https://t.me/paradigm_reth -[gh-book]: https://github.com/paradigmxyz/reth/tree/main/book +[gh-docs]: https://github.com/paradigmxyz/reth/tree/main/docs diff --git a/book/run/config.md b/docs/vocs/docs/pages/run/configuration.mdx similarity index 90% rename from book/run/config.md rename to docs/vocs/docs/pages/run/configuration.mdx index bb28d855de8..8f34cfc691f 100644 --- a/book/run/config.md +++ b/docs/vocs/docs/pages/run/configuration.mdx @@ -1,32 +1,36 @@ +--- +description: How to configure Reth using reth.toml and its options. +--- + # Configuring Reth Reth places a configuration file named `reth.toml` in the data directory specified when starting the node. It is written in the [TOML] format. The default data directory is platform dependent: -- Linux: `$XDG_DATA_HOME/reth/` or `$HOME/.local/share/reth/` -- Windows: `{FOLDERID_RoamingAppData}/reth/` -- macOS: `$HOME/Library/Application Support/reth/` +- Linux: `$XDG_DATA_HOME/reth/` or `$HOME/.local/share/reth/` +- Windows: `{FOLDERID_RoamingAppData}/reth/` +- macOS: `$HOME/Library/Application Support/reth/` The configuration file contains the following sections: -- [`[stages]`](#the-stages-section) -- Configuration of the individual sync stages - - [`headers`](#headers) - - [`bodies`](#bodies) - - [`sender_recovery`](#sender_recovery) - - [`execution`](#execution) - - [`account_hashing`](#account_hashing) - - [`storage_hashing`](#storage_hashing) - - [`merkle`](#merkle) - - [`transaction_lookup`](#transaction_lookup) - - [`index_account_history`](#index_account_history) - - [`index_storage_history`](#index_storage_history) -- [`[peers]`](#the-peers-section) - - [`connection_info`](#connection_info) - - [`reputation_weights`](#reputation_weights) - - [`backoff_durations`](#backoff_durations) -- [`[sessions]`](#the-sessions-section) -- [`[prune]`](#the-prune-section) +- [`[stages]`](#the-stages-section) -- Configuration of the individual sync stages + - [`headers`](#headers) + - [`bodies`](#bodies) + - [`sender_recovery`](#sender_recovery) + - [`execution`](#execution) + - [`account_hashing`](#account_hashing) + - [`storage_hashing`](#storage_hashing) + - [`merkle`](#merkle) + - [`transaction_lookup`](#transaction_lookup) + - [`index_account_history`](#index_account_history) + - [`index_storage_history`](#index_storage_history) +- [`[peers]`](#the-peers-section) + - [`connection_info`](#connection_info) + - [`reputation_weights`](#reputation_weights) + - [`backoff_durations`](#backoff_durations) +- [`[sessions]`](#the-sessions-section) +- [`[prune]`](#the-prune-section) ## The `[stages]` section @@ -305,8 +309,8 @@ The sessions section configures the internal behavior of a single peer-to-peer c You can configure the session buffer sizes, which limits the amount of pending events (incoming messages) and commands (outgoing messages) each session can hold before it will start to ignore messages. > **Note** -> -> These buffers are allocated *per peer*, which means that increasing the buffer sizes can have large impact on memory consumption. +> +> These buffers are allocated _per peer_, which means that increasing the buffer sizes can have large impact on memory consumption. ```toml [sessions] @@ -342,10 +346,11 @@ No pruning, run as archive node. ### Example of the custom pruning configuration This configuration will: -- Run pruning every 5 blocks -- Continuously prune all transaction senders, account history and storage history before the block `head-100_000`, -i.e. keep the data for the last `100_000` blocks -- Prune all receipts before the block 1920000, i.e. keep receipts from the block 1920000 + +- Run pruning every 5 blocks +- Continuously prune all transaction senders, account history and storage history before the block `head-100_000`, + i.e. keep the data for the last `100_000` blocks +- Prune all receipts before the block 1920000, i.e. keep receipts from the block 1920000 ```toml [prune] @@ -370,6 +375,7 @@ storage_history = { distance = 100_000 } # Prune all historical storage states b ``` We can also prune receipts more granular, using the logs filtering: + ```toml # Receipts pruning configuration by retaining only those receipts that contain logs emitted # by the specified addresses, discarding all others. This setting is overridden by `receipts`. diff --git a/docs/vocs/docs/pages/run/ethereum.mdx b/docs/vocs/docs/pages/run/ethereum.mdx new file mode 100644 index 00000000000..ef6f558a978 --- /dev/null +++ b/docs/vocs/docs/pages/run/ethereum.mdx @@ -0,0 +1,148 @@ +--- +description: How to run Reth on Ethereum mainnet and testnets. +--- + +# Running Reth on Ethereum Mainnet or testnets + +Reth is an [_execution client_](https://ethereum.org/en/developers/docs/nodes-and-clients/#execution-clients). After Ethereum's transition to Proof of Stake (aka the Merge) it became required to run a [_consensus client_](https://ethereum.org/en/developers/docs/nodes-and-clients/#consensus-clients) along with your execution client in order to sync into any "post-Merge" network. This is because the Ethereum execution layer now outsources consensus to a separate component, known as the consensus client. + +Consensus clients decide what blocks are part of the chain, while execution clients only validate that transactions and blocks are valid in themselves and with respect to the world state. In other words, execution clients execute blocks and transactions and check their validity, while consensus clients determine which valid blocks should be part of the chain. Therefore, running a consensus client in parallel with the execution client is necessary to ensure synchronization and participation in the network. + +By running both an execution client like Reth and a consensus client, such as Lighthouse 🦀 (which we will assume for this guide), you can effectively contribute to the Ethereum network and participate in the consensus process, even if you don't intend to run validators. + +| Client | Role | +| --------- | --------------------------------------------- | +| Execution | Validates transactions and blocks | +| | (checks their validity and global state) | +| Consensus | Determines which blocks are part of the chain | +| | (makes consensus decisions) | + +## Running the Reth Node + +First, ensure that you have Reth installed by following the [installation instructions][installation]. + +Now, to start the archive node, run: + +```bash +reth node +``` + +And to start the full node, run: + +```bash +reth node --full +``` + +On differences between archive and full nodes, see [Pruning & Full Node](/run/faq/pruning#basic-concepts) section. + +:::note +These commands will not open any HTTP/WS ports by default. + +You can change this by adding the `--http`, `--ws` flags, respectively and using the `--http.api` and `--ws.api` flags to enable various [JSON-RPC APIs](/jsonrpc/intro). + +For more commands, see the [`reth node` CLI reference](/cli/cli). +::: + +The EL \<> CL communication happens over the [Engine API](https://github.com/ethereum/execution-apis/blob/main/src/engine/common.md), which is by default exposed at `http://localhost:8551`. The connection is authenticated over JWT using a JWT secret which is auto-generated by Reth and placed in a file called `jwt.hex` in the data directory, which on Linux by default is `$HOME/.local/share/reth/` (`/Users//Library/Application Support/reth/mainnet/jwt.hex` in Mac). + +You can override this path using the `--authrpc.jwtsecret` option. You MUST use the same JWT secret in BOTH Reth and the chosen Consensus Layer. If you want to override the address or port, you can use the `--authrpc.addr` and `--authrpc.port` options, respectively. + +So one might do: + +```bash +reth node \ + --authrpc.jwtsecret /path/to/secret \ + --authrpc.addr 127.0.0.1 \ + --authrpc.port 8551 +``` + +At this point, our Reth node has started discovery, and even discovered some new peers. But it will not start syncing until you spin up the consensus layer! + +## Running the Consensus Layer + +First, make sure you have Lighthouse installed. Sigma Prime provides excellent [installation](https://lighthouse-book.sigmaprime.io/installation.html) and [node operation](https://lighthouse-book.sigmaprime.io/run_a_node.html) instructions. + +Assuming you have done that, run: + +```bash +lighthouse bn \ + --checkpoint-sync-url https://mainnet.checkpoint.sigp.io \ + --execution-endpoint http://localhost:8551 \ + --execution-jwt /path/to/secret +``` + +If you don't intend on running validators on your node you can add: + +```bash + --disable-deposit-contract-sync +``` + +The `--checkpoint-sync-url` argument value can be replaced with any checkpoint sync endpoint from a [community-maintained list](https://eth-clients.github.io/checkpoint-sync-endpoints/#mainnet). + +Your Reth node should start receiving "fork choice updated" messages, and begin syncing the chain. + +## Verify the chain is growing + +You can easily verify that by inspecting the logs, and seeing that headers are arriving in Reth. Sit back now and wait for the stages to run! +In the meantime, consider setting up [observability](/run/monitoring) to monitor your node's health or [test the JSON RPC API](/jsonrpc/intro). + +{/* TODO: Add more logs to help node operators debug any weird CL to EL messages! */} + +[installation]: ./../../installation/overview +[docs]: https://github.com/paradigmxyz/reth/tree/main/docs +[metrics]: https://github.com/paradigmxyz/reth/blob/main/docs/design/metrics.md#metrics + +## Running without a Consensus Layer + +We provide several methods for running Reth without a Consensus Layer for testing and debugging purposes: + +### Manual Chain Tip Setting + +Use the `--debug.tip ` parameter to set the chain tip manually. If you provide this to your node, it will simulate sending an `engine_forkchoiceUpdated` message _once_ and will trigger syncing to the provided block hash. This is useful for testing and debugging purposes, but in order to have a node that can keep up with the tip you'll need to run a CL alongside with it. + +Example, sync up to block https://etherscan.io/block/23450000: + +```bash +reth node --debug.tip 0x9ba680d8479f936f84065ce94f58c5f0cc1adb128945167e0875ba41a36cd93b +``` + +Note: This is a temporary flag for testing purposes. At the moment we have no plans of including a Consensus Layer implementation in Reth, and we are open to including light clients and other methods of syncing like importing Lighthouse as a library. + +### Running with Etherscan as Block Source + +You can use `--debug.etherscan` to run Reth with a fake consensus client that advances the chain using recent blocks on Etherscan. This requires an Etherscan API key (set via `ETHERSCAN_API_KEY` environment variable). Optionally, specify a custom API URL with `--debug.etherscan `. + +Example: + +```bash +export ETHERSCAN_API_KEY=your_api_key_here +reth node --debug.etherscan +``` + +Or with a custom Etherscan API URL: + +```bash +export ETHERSCAN_API_KEY=your_api_key_here +reth node --debug.etherscan https://api.etherscan.io/api +``` + +### Running with RPC Consensus + +Use `--debug.rpc-consensus-url` to run Reth with a fake consensus client that fetches blocks from an existing RPC endpoint. This supports both HTTP and WebSocket endpoints: + +- **WebSocket endpoints**: Will use subscriptions for real-time block updates +- **HTTP endpoints**: Will poll for new blocks periodically + +Example with HTTP RPC: + +```bash +reth node --debug.rpc-consensus-url https://eth-mainnet.g.alchemy.com/v2/your-api-key +``` + +Example with WebSocket RPC: + +```bash +reth node --debug.rpc-consensus-url wss://eth-mainnet.g.alchemy.com/v2/your-api-key +``` + +Note: The `--debug.tip`, `--debug.etherscan`, and `--debug.rpc-consensus-url` flags are mutually exclusive and cannot be used together. diff --git a/docs/vocs/docs/pages/run/ethereum/snapshots.mdx b/docs/vocs/docs/pages/run/ethereum/snapshots.mdx new file mode 100644 index 00000000000..116d4359e53 --- /dev/null +++ b/docs/vocs/docs/pages/run/ethereum/snapshots.mdx @@ -0,0 +1 @@ +# Snapshots \ No newline at end of file diff --git a/docs/vocs/docs/pages/run/faq.mdx b/docs/vocs/docs/pages/run/faq.mdx new file mode 100644 index 00000000000..bdd0a9f68e7 --- /dev/null +++ b/docs/vocs/docs/pages/run/faq.mdx @@ -0,0 +1,11 @@ +# FAQ + +1. [Transaction Types](/run/faq/transactions) - Learn about the transaction types supported by Reth. + +2. [Pruning & Full Node](/run/faq/pruning) - Understand the differences between archive nodes, full nodes, and pruned nodes. Learn how to configure pruning options and what RPC methods are available for each node type. + +3. [Ports](/run/faq/ports) - Information about the network ports used by Reth for P2P communication, JSON-RPC APIs, and the Engine API for consensus layer communication. + +4. [Profiling](/run/faq/profiling) - Performance profiling techniques and tools for analyzing Reth node performance, including CPU profiling, memory analysis, and bottleneck identification. + +5. [Sync OP Mainnet](/run/faq/sync-op-mainnet) - Detailed guide for syncing a Reth node with OP Mainnet, including specific configuration requirements and considerations for the Optimism ecosystem. diff --git a/docs/vocs/docs/pages/run/faq/ports.mdx b/docs/vocs/docs/pages/run/faq/ports.mdx new file mode 100644 index 00000000000..f9a3ba9950d --- /dev/null +++ b/docs/vocs/docs/pages/run/faq/ports.mdx @@ -0,0 +1,42 @@ +--- +description: Ports used by Reth. +--- + +# Ports + +This section provides essential information about the ports used by the system, their primary purposes, and recommendations for exposure settings. + +## Peering Ports + +- **Port:** `30303` +- **Protocol:** TCP and UDP +- **Purpose:** Peering with other nodes for synchronization of blockchain data. Nodes communicate through this port to maintain network consensus and share updated information. +- **Exposure Recommendation:** This port should be exposed to enable seamless interaction and synchronization with other nodes in the network. + +## Metrics Port + +- **Port:** `9001` +- **Protocol:** TCP +- **Purpose:** This port is designated for serving metrics related to the system's performance and operation. It allows internal monitoring and data collection for analysis. +- **Exposure Recommendation:** By default, this port should not be exposed to the public. It is intended for internal monitoring and analysis purposes. + +## HTTP RPC Port + +- **Port:** `8545` +- **Protocol:** TCP +- **Purpose:** Port 8545 provides an HTTP-based Remote Procedure Call (RPC) interface. It enables external applications to interact with the blockchain by sending requests over HTTP. +- **Exposure Recommendation:** Similar to the metrics port, exposing this port to the public is not recommended by default due to security considerations. + +## WS RPC Port + +- **Port:** `8546` +- **Protocol:** TCP +- **Purpose:** Port 8546 offers a WebSocket-based Remote Procedure Call (RPC) interface. It allows real-time communication between external applications and the blockchain. +- **Exposure Recommendation:** As with the HTTP RPC port, the WS RPC port should not be exposed by default for security reasons. + +## Engine API Port + +- **Port:** `8551` +- **Protocol:** TCP +- **Purpose:** Port 8551 facilitates communication between specific components, such as "reth" and "CL" (assuming their definitions are understood within the context of the system). It enables essential internal processes. +- **Exposure Recommendation:** This port is not meant to be exposed to the public by default. It should be reserved for internal communication between vital components of the system. diff --git a/book/developers/profiling.md b/docs/vocs/docs/pages/run/faq/profiling.mdx similarity index 84% rename from book/developers/profiling.md rename to docs/vocs/docs/pages/run/faq/profiling.mdx index fdae94e2d4a..123808ad2d3 100644 --- a/book/developers/profiling.md +++ b/docs/vocs/docs/pages/run/faq/profiling.mdx @@ -1,11 +1,8 @@ -# Profiling reth +--- +description: Profiling and debugging memory usage in Reth. +--- -#### Table of Contents - - [Memory profiling](#memory-profiling) - - [Jemalloc](#jemalloc) - - [Monitoring memory usage](#monitoring-memory-usage) - - [Limiting process memory](#limiting-process-memory) - - [Understanding allocation with jeprof](#understanding-allocation-with-jeprof) +# Profiling Reth ## Memory profiling @@ -16,10 +13,11 @@ Reth is also a complex program, with many moving pieces, and it can be difficult Understanding how to profile memory usage is an extremely valuable skill when faced with this type of problem, and can quickly help shed light on the root cause of a memory leak. In this tutorial, we will be reviewing: - * How to monitor reth's memory usage, - * How to emulate a low-memory environment to lab-reproduce OOM crashes, - * How to enable `jemalloc` and its built-in memory profiling, and - * How to use `jeprof` to interpret heap profiles and identify potential root causes for a memory leak. + +- How to monitor reth's memory usage, +- How to emulate a low-memory environment to lab-reproduce OOM crashes, +- How to enable `jemalloc` and its built-in memory profiling, and +- How to use `jeprof` to interpret heap profiles and identify potential root causes for a memory leak. ### Jemalloc @@ -27,21 +25,24 @@ In this tutorial, we will be reviewing: We've seen significant performance benefits in reth when using jemalloc, but will be primarily focusing on its profiling capabilities. Jemalloc also provides tools for analyzing and visualizing its allocation profiles it generates, notably `jeprof`. - #### Enabling jemalloc in reth + Reth includes a `jemalloc` feature to explicitly use jemalloc instead of the system allocator: + ``` cargo build --features jemalloc ``` While the `jemalloc` feature does enable jemalloc, reth has an additional feature, `profiling`, that must be used to enable heap profiling. This feature implicitly enables the `jemalloc` feature as well: + ``` cargo build --features jemalloc-prof ``` When performing a longer-running or performance-sensitive task with reth, such as a sync test or load benchmark, it's usually recommended to use the `maxperf` profile. However, the `maxperf` profile does not enable debug symbols, which are required for tools like `perf` and `jemalloc` to produce results that a human can interpret. Reth includes a performance profile with debug symbols called `profiling`. To compile reth with debug symbols, jemalloc, profiling, and a performance profile: + ``` cargo build --features jemalloc-prof --profile profiling @@ -51,19 +52,39 @@ RUSTFLAGS="-C target-cpu=native" cargo build --features jemalloc-prof --profile ### Monitoring memory usage -Reth's dashboard has a few metrics that are important when monitoring memory usage. The **Jemalloc memory** graph shows reth's memory usage. The *allocated* label shows the memory used by the reth process which cannot be reclaimed unless reth frees that memory. This metric exceeding the available system memory would cause reth to be killed by the OOM killer. -Jemalloc memory +Reth's dashboard has a few metrics that are important when monitoring memory usage. The **Jemalloc memory** graph shows reth's memory usage. The _allocated_ label shows the memory used by the reth process which cannot be reclaimed unless reth frees that memory. This metric exceeding the available system memory would cause reth to be killed by the OOM killer. + +Jemalloc memory Some of reth's internal components also have metrics for the memory usage of certain data structures, usually data structures that are likely to contain many elements or may consume a lot of memory at peak load. **The bodies downloader buffer**: -The bodies downloader buffer graph + +The bodies downloader buffer graph **The blockchain tree block buffer**: -The blockchain tree block buffer graph + +The blockchain tree block buffer graph **The transaction pool subpools**: -The transaction pool subpool size graph + +The transaction pool subpool size graph One of these metrics growing beyond, 2GB for example, is likely a bug and could lead to an OOM on a low memory machine. It isn't likely for that to happen frequently, so in the best case these metrics can be used to rule out these components from having a leak, if an OOM is occurring. @@ -81,28 +102,37 @@ See the [canonical documentation for cgroups](https://git.kernel.org/pub/scm/lin In order to use cgroups to limit process memory, sometimes it must be explicitly enabled as a kernel parameter. For example, the following line is sometimes necessary to enable cgroup memory limits on Ubuntu machines that use GRUB: + ``` GRUB_CMDLINE_LINUX_DEFAULT="cgroup_enable=memory" ``` + Then, create a named cgroup: + ``` sudo cgcreate -t $USER:$USER -a $USER:$USER -g memory:rethMemory ``` + The memory limit for the named cgroup can be set in `sys/fs/cgroup/memory`. This for example sets an 8 gigabyte memory limit: + ``` echo 8G > /sys/fs/cgroup/memory/rethMemory/memory.limit_in_bytes ``` + If the intention of setting up the cgroup is to strictly limit memory and simulate OOMs, a high amount of swap may prevent those OOMs from happening. To check swap, use `free -m`: + ``` ubuntu@bench-box:~/reth$ free -m total used free shared buff/cache available Mem: 257668 10695 218760 12 28213 244761 Swap: 8191 159 8032 ``` + If this is a problem, it may be worth either adjusting the system swappiness or disabling swap overall. Finally, `cgexec` can be used to run reth under the cgroup: + ``` cgexec -g memory:rethMemory reth node ``` @@ -111,11 +141,13 @@ cgexec -g memory:rethMemory reth node When reth is built with the `jemalloc-prof` feature and debug symbols, the profiling still needs to be configured and enabled at runtime. This is done with the `_RJEM_MALLOC_CONF` environment variable. Take the following command to launch reth with jemalloc profiling enabled: + ``` _RJEM_MALLOC_CONF=prof:true,lg_prof_interval:32,lg_prof_sample:19 reth node ``` If reth is not built properly, you will see this when you try to run reth: + ``` ~/p/reth (dan/managing-memory)> _RJEM_MALLOC_CONF=prof:true,lg_prof_interval:32,lg_prof_sample:19 reth node : Invalid conf pair: prof:true diff --git a/book/run/pruning.md b/docs/vocs/docs/pages/run/faq/pruning.mdx similarity index 91% rename from book/run/pruning.md rename to docs/vocs/docs/pages/run/faq/pruning.mdx index 25d11b4e46e..6f646b2ee76 100644 --- a/book/run/pruning.md +++ b/docs/vocs/docs/pages/run/faq/pruning.mdx @@ -1,8 +1,14 @@ +--- +description: Pruning and full node options in Reth. +--- + # Pruning & Full Node -> Pruning and full node are new features of Reth, -> and we will be happy to hear about your experience using them either -> on [GitHub](https://github.com/paradigmxyz/reth/issues) or in the [Telegram group](https://t.me/paradigm_reth). +:::info +Pruning and full node are new features of Reth, +and we will be happy to hear about your experience using them either +on [GitHub](https://github.com/paradigmxyz/reth/issues) or in the [Telegram group](https://t.me/paradigm_reth). +::: By default, Reth runs as an archive node. Such nodes have all historical blocks and the state at each of these blocks available for querying and tracing. @@ -12,31 +18,31 @@ the steps for running Reth as a full node, what caveats to expect and how to con ## Basic concepts -- Archive node – Reth node that has all historical data from genesis. -- Pruned node – Reth node that has its historical data pruned partially or fully through - a [custom configuration](./config.md#the-prune-section). -- Full Node – Reth node that has the latest state and historical data for only the last 10064 blocks available - for querying in the same way as an archive node. +- Archive node – Reth node that has all historical data from genesis. +- Pruned node – Reth node that has its historical data pruned partially or fully through + a [custom configuration](/run/configuration#the-prune-section). +- Full Node – Reth node that has the latest state and historical data for only the last 10064 blocks available + for querying in the same way as an archive node. -The node type that was chosen when first [running a node](./run-a-node.md) **cannot** be changed after +The node type that was chosen when first [running a node](/run/overview) **cannot** be changed after the initial sync. Turning Archive into Pruned, or Pruned into Full is not supported. ## Modes ### Archive Node -Default mode, follow the steps from the previous chapter on [how to run on mainnet or official testnets](./mainnet.md). +Default mode, follow the steps from the previous chapter on [how to run on mainnet or official testnets](/run/ethereum). ### Pruned Node -To run Reth as a pruned node configured through a [custom configuration](./config.md#the-prune-section), +To run Reth as a pruned node configured through a [custom configuration](/run/configuration#the-prune-section), modify the `reth.toml` file and run Reth in the same way as archive node by following the steps from -the previous chapter on [how to run on mainnet or official testnets](./mainnet.md). +the previous chapter on [how to run on mainnet or official testnets](/run/ethereum). ### Full Node To run Reth as a full node, follow the steps from the previous chapter on -[how to run on mainnet or official testnets](./mainnet.md), and add a `--full` flag. For example: +[how to run on mainnet or official testnets](/run/ethereum), and add a `--full` flag. For example: ```bash reth node \ @@ -55,7 +61,8 @@ All numbers are as of April 2024 at block number 19.6M for mainnet. Archive node occupies at least 2.14TB. You can track the growth of Reth archive node size with our -[public Grafana dashboard](https://reth.paradigm.xyz/d/2k8BXz24x/reth?orgId=1&refresh=30s&viewPanel=52). +[public Ethereum Grafana dashboard](https://reth.ithaca.xyz/public-dashboards/a49fa110dc9149298fa6763d5c89c8c0). +[public Base Grafana dashboard](https://reth.ithaca.xyz/public-dashboards/b3e9f2e668ee4b86960b7fac691b5e64). ### Pruned Node @@ -95,21 +102,21 @@ storage_history = { distance = 10_064 } Meaning, it prunes: -- Account History and Storage History up to the last 10064 blocks -- All of Sender Recovery data. The caveat is that it's pruned gradually after the initial sync - is completed, so the disk space is reclaimed slowly. -- Receipts up to the last 10064 blocks, preserving all receipts with the logs from Beacon Deposit Contract +- Account History and Storage History up to the last 10064 blocks +- All of Sender Recovery data. The caveat is that it's pruned gradually after the initial sync + is completed, so the disk space is reclaimed slowly. +- Receipts up to the last 10064 blocks, preserving all receipts with the logs from Beacon Deposit Contract ## RPC support -As it was mentioned in the [pruning configuration chapter](./config.md#the-prune-section), there are several segments which can be pruned +As it was mentioned in the [pruning configuration chapter](/run/configuration#the-prune-section), there are several segments which can be pruned independently of each other: -- Sender Recovery -- Transaction Lookup -- Receipts -- Account History -- Storage History +- Sender Recovery +- Transaction Lookup +- Receipts +- Account History +- Storage History Pruning of each of these segments disables different RPC methods, because the historical data or lookup indexes become unavailable. @@ -215,8 +222,8 @@ The following tables describe RPC methods available in the full node. The following tables describe the requirements for prune segments, per RPC method: -- ✅ – if the segment is pruned, the RPC method still works -- ❌ - if the segment is pruned, the RPC method doesn't work anymore +- ✅ – if the segment is pruned, the RPC method still works +- ❌ - if the segment is pruned, the RPC method doesn't work anymore #### `debug` namespace diff --git a/book/run/sync-op-mainnet.md b/docs/vocs/docs/pages/run/faq/sync-op-mainnet.mdx similarity index 69% rename from book/run/sync-op-mainnet.md rename to docs/vocs/docs/pages/run/faq/sync-op-mainnet.mdx index 0e2090acbcb..ed857da7c41 100644 --- a/book/run/sync-op-mainnet.md +++ b/docs/vocs/docs/pages/run/faq/sync-op-mainnet.mdx @@ -1,32 +1,50 @@ +--- +description: Syncing Reth with OP Mainnet and Bedrock state. +--- + # Sync OP Mainnet To sync OP mainnet, Bedrock state needs to be imported as a starting point. There are currently two ways: -* Minimal bootstrap **(recommended)**: only state snapshot at Bedrock block is imported without any OVM historical data. -* Full bootstrap **(not recommended)**: state, blocks and receipts are imported. *Not recommended for now: [storage consistency issue](https://github.com/paradigmxyz/reth/pull/11099) tldr: sudden crash may break the node +- Minimal bootstrap **(recommended)**: only state snapshot at Bedrock block is imported without any OVM historical data. +- Full bootstrap **(not recommended)**: state, blocks and receipts are imported. ## Minimal bootstrap (recommended) **The state snapshot at Bedrock block is required.** It can be exported from [op-geth](https://github.com/testinprod-io/op-erigon/blob/pcw109550/bedrock-db-migration/bedrock-migration.md#export-state) (**.jsonl**) or downloaded directly from [here](https://mega.nz/file/GdZ1xbAT#a9cBv3AqzsTGXYgX7nZc_3fl--tcBmOAIwIA5ND6kwc). -Import the state snapshot +### 1. Download and decompress + +After you downloaded the state file, ensure the state file is decompressed into **.jsonl** format: + +```sh +$ unzstd /path/to/world_trie_state.jsonl.zstd +``` + +### 2. Import the state + +Import the state snapshot: ```sh $ op-reth init-state --without-ovm --chain optimism --datadir op-mainnet world_trie_state.jsonl ``` -Sync the node to a recent finalized block (e.g. 125200000) to catch up close to the tip, before pairing with op-node. +### 3. Sync from Bedrock to tip + +Running the node with `--debug.tip ` syncs the node without help from CL until a fixed tip. The +block hash can be taken from the latest block on [https://optimistic.etherscan.io](https://optimistic.etherscan.io). + +Eg, sync the node to a recent finalized block (e.g. 125200000) to catch up close to the tip, before pairing with op-node. ```sh $ op-reth node --chain optimism --datadir op-mainnet --debug.tip 0x098f87b75c8b861c775984f9d5dbe7b70cbbbc30fc15adb03a5044de0144f2d0 # block #125200000 ``` - ## Full bootstrap (not recommended) **Not recommended for now**: [storage consistency issue](https://github.com/paradigmxyz/reth/pull/11099) tldr: sudden crash may break the node. -### Import state +### Import state To sync OP mainnet, the Bedrock datadir needs to be imported to use as starting point. Blocks lower than the OP mainnet Bedrock fork, are built on the OVM and cannot be executed on the EVM. @@ -40,10 +58,10 @@ Importing OP mainnet Bedrock datadir requires exported data: ### Manual Export Steps -The `op-geth` Bedrock datadir can be downloaded from . +The `op-geth` Bedrock datadir can be downloaded from [https://datadirs.optimism.io](https://datadirs.optimism.io). To export the OVM chain from `op-geth`, clone the `testinprod-io/op-geth` repo and checkout -. Commands to export blocks, receipts and state dump can be +[testinprod-io/op-geth#1](https://github.com/testinprod-io/op-geth/pull/1). Commands to export blocks, receipts and state dump can be found in `op-geth/migrate.sh`. ### Manual Import Steps @@ -64,7 +82,7 @@ This step is optional. To run a full node, skip this step. If however receipts a corresponding transactions must already be imported (see [step 1](#1-import-blocks)). Imports a `.rlp` file of receipts, that has been exported with command specified in - (command for exporting receipts uses custom RLP-encoding). +[testinprod-io/op-geth#1](https://github.com/testinprod-io/op-geth/pull/1) (command for exporting receipts uses custom RLP-encoding). Import of >100 million OVM receipts, from genesis to Bedrock, completes in 30 minutes. @@ -83,10 +101,7 @@ Import of >4 million OP mainnet accounts at Bedrock, completes in 10 minutes. $ op-reth init-state --chain optimism ``` -## Sync from Bedrock to tip - -Running the node with `--debug.tip `syncs the node without help from CL until a fixed tip. The -block hash can be taken from the latest block on . +### Start with op-node Use `op-node` to track the tip. Start `op-node` with `--syncmode=execution-layer` and `--l2.enginekind=reth`. If `op-node`'s RPC connection to L1 is over localhost, `--l1.trustrpc` can be set to improve performance. diff --git a/book/run/transactions.md b/docs/vocs/docs/pages/run/faq/transactions.mdx similarity index 71% rename from book/run/transactions.md rename to docs/vocs/docs/pages/run/faq/transactions.mdx index edb3a24d76f..c760c3507c6 100644 --- a/book/run/transactions.md +++ b/docs/vocs/docs/pages/run/faq/transactions.mdx @@ -1,11 +1,16 @@ +--- +description: Overview of Ethereum transaction types in Reth. +--- + # Transaction types -Over time, the Ethereum network has undergone various upgrades and improvements to enhance transaction efficiency, security, and user experience. Four significant transaction types that have evolved are: +Over time, the Ethereum network has undergone various upgrades and improvements to enhance transaction efficiency, security, and user experience. Five significant transaction types that have evolved are: - Legacy Transactions, - EIP-2930 Transactions, - EIP-1559 Transactions, -- EIP-4844 Transactions +- EIP-4844 Transactions, +- EIP-7702 Transactions Each of these transaction types brings unique features and improvements to the Ethereum network. @@ -40,10 +45,27 @@ The base fee is burned, while the priority fee is paid to the miner who includes ## EIP-4844 Transactions -[EIP-4844](https://eips.ethereum.org/EIPS/eip-4844) transactions (type `0x3`) was introduced in Ethereum's Dencun fork. This provides a temporary but significant scaling relief for rollups by allowing them to initially scale to 0.375 MB per slot, with a separate fee market allowing fees to be very low while usage of this system is limited. +[EIP-4844](https://eips.ethereum.org/EIPS/eip-4844) transactions (type `0x3`) were introduced in Ethereum's Dencun fork. This provides a temporary but significant scaling relief for rollups by allowing them to initially scale to 0.375 MB per slot, with a separate fee market allowing fees to be very low while usage of this system is limited. Alongside the legacy parameters & parameters from EIP-1559, the EIP-4844 transactions include: - `max_fee_per_blob_gas`, The maximum total fee per gas the sender is willing to pay for blob gas in wei - `blob_versioned_hashes`, List of versioned blob hashes associated with the transaction's EIP-4844 data blobs. The actual blob fee is deducted from the sender balance before transaction execution and burned, and is not refunded in case of transaction failure. + +## EIP-7702 Transactions + +[EIP-7702](https://eips.ethereum.org/EIPS/eip-7702) transactions (type `0x4`) were introduced in Ethereum's Pectra fork. This provides the ability for a wallet to delegate its execution to an authorized smart contract. Alongside the fields present from the EIP-1559 transaction type, EIP-7702 transactions include: +- `authorization_list` +- `signature_y_parity` +- `signature_r` +- `signature_s` + +where `authorization_list` is a list of authorizations, which are tuples containing the following fields: +- `chain_id` +- `address` +- `nonce` +- `y_parity` +- `r` +- `s` + diff --git a/book/run/troubleshooting.md b/docs/vocs/docs/pages/run/faq/troubleshooting.mdx similarity index 51% rename from book/run/troubleshooting.md rename to docs/vocs/docs/pages/run/faq/troubleshooting.mdx index 7b8ec6ba19c..1f26cba9dae 100644 --- a/book/run/troubleshooting.md +++ b/docs/vocs/docs/pages/run/faq/troubleshooting.mdx @@ -1,102 +1,107 @@ +--- +description: Troubleshooting common Reth node and database issues. +--- + # Troubleshooting This page tries to answer how to deal with the most popular issues. -- [Troubleshooting](#troubleshooting) - - [Database](#database) - - [Docker](#docker) - - [Error code 13](#error-code-13) - - [Slow database inserts and updates](#slow-database-inserts-and-updates) - - [Compact the database](#compact-the-database) - - [Re-sync from scratch](#re-sync-from-scratch) - - [Database write error](#database-write-error) - - [Concurrent database access error (using containers/Docker)](#concurrent-database-access-error-using-containersdocker) - - [Hardware Performance Testing](#hardware-performance-testing) - - [Disk Speed Testing with IOzone](#disk-speed-testing-with-iozone) - +- [Troubleshooting](#troubleshooting) + - [Database](#database) + - [Docker](#docker) + - [Error code 13](#error-code-13) + - [Slow database inserts and updates](#slow-database-inserts-and-updates) + - [Compact the database](#compact-the-database) + - [Re-sync from scratch](#re-sync-from-scratch) + - [Database write error](#database-write-error) + - [Concurrent database access error (using containers/Docker)](#concurrent-database-access-error-using-containersdocker) + - [Hardware Performance Testing](#hardware-performance-testing) + - [Disk Speed Testing with IOzone](#disk-speed-testing-with-iozone) ## Database -### Docker +### Docker Externally accessing a `datadir` inside a named docker volume will usually come with folder/file ownership/permissions issues. **It is not recommended** to use the path to the named volume as it will trigger an error code 13. `RETH_DB_PATH: /var/lib/docker/volumes/named_volume/_data/eth/db cargo r --examples db-access --path ` is **DISCOURAGED** and a mounted volume with the right permissions should be used instead. -### Error code 13 +### Error code 13 `the environment opened in read-only code: 13` Externally accessing a database in a read-only folder is not supported, **UNLESS** there's no `mdbx.lck` present, and it's called with `exclusive` on calling `open_db_read_only`. Meaning that there's no node syncing concurrently. -If the error persists, ensure that you have the right `rx` permissions on the `datadir` **and its parent** folders. Eg. the following command should succeed: +If the error persists, ensure that you have the right `rx` permissions on the `datadir` **and its parent** folders. Eg. the following command should succeed: ```bash,ignore stat /full/path/datadir ``` - ### Slow database inserts and updates If you're: + 1. Running behind the tip -2. Have slow canonical commit time according to the `Canonical Commit Latency Time` chart on [Grafana dashboard](./observability.md#prometheus--grafana) (more than 2-3 seconds) -3. Seeing warnings in your logs such as - ```console - 2023-11-08T15:17:24.789731Z WARN providers::db: Transaction insertion took too long block_number=18528075 tx_num=2150227643 hash=0xb7de1d6620efbdd3aa8547c47a0ff09a7fd3e48ba3fd2c53ce94c6683ed66e7c elapsed=6.793759034s - ``` +2. Have slow canonical commit time according to the `Canonical Commit Latency Time` chart on [Grafana dashboard](/run/monitoring#prometheus--grafana) (more than 2-3 seconds) +3. Seeing warnings in your logs such as + ```console + 2023-11-08T15:17:24.789731Z WARN providers::db: Transaction insertion took too long block_number=18528075 tx_num=2150227643 hash=0xb7de1d6620efbdd3aa8547c47a0ff09a7fd3e48ba3fd2c53ce94c6683ed66e7c elapsed=6.793759034s + ``` then most likely you're experiencing issues with the [database freelist](https://github.com/paradigmxyz/reth/issues/5228). -To confirm it, check if the values on the `Freelist` chart on [Grafana dashboard](./observability.md#prometheus--grafana) +To confirm it, check if the values on the `Freelist` chart on [Grafana dashboard](/run/monitoring#prometheus--grafana) is greater than 10M. Currently, there are two main ways to fix this issue. - #### Compact the database + It will take around 5-6 hours and require **additional** disk space located on the same or different drive -equal to the [freshly synced node](../installation/installation.md#hardware-requirements). +equal to the [freshly synced node](/run/system-requirements). 1. Clone Reth - ```bash - git clone https://github.com/paradigmxyz/reth - cd reth - ``` + ```bash + git clone https://github.com/paradigmxyz/reth + cd reth + ``` 2. Build database debug tools - ```bash - make db-tools - ``` + ```bash + make db-tools + ``` 3. Run compaction (this step will take 5-6 hours, depending on the I/O speed) - ```bash - ./db-tools/mdbx_copy -c $(reth db path) reth_compact.dat - ``` + ```bash + ./db-tools/mdbx_copy -c $(reth db path) reth_compact.dat + ``` 4. Stop Reth 5. Backup original database - ```bash - mv $(reth db path)/mdbx.dat reth_old.dat - ``` + ```bash + mv $(reth db path)/mdbx.dat reth_old.dat + ``` 6. Move compacted database in place of the original database - ```bash - mv reth_compact.dat $(reth db path)/mdbx.dat - ``` + ```bash + mv reth_compact.dat $(reth db path)/mdbx.dat + ``` 7. Start Reth 8. Confirm that the values on the `Freelist` chart are near zero and the values on the `Canonical Commit Latency Time` chart -is less than 1 second. + is less than 1 second. 9. Delete original database - ```bash - rm reth_old.dat - ``` + ```bash + rm reth_old.dat + ``` #### Re-sync from scratch + It will take the same time as initial sync. 1. Stop Reth -2. Drop the database using [`reth db drop`](../cli/reth/db/drop.md) +2. Drop the database using [`reth db drop`](/cli/reth/db/drop) 3. Start reth ### Database write error -If you encounter an irrecoverable database-related errors, in most of the cases it's related to the RAM/NVMe/SSD you use. For example: +If you encounter irrecoverable database-related errors, in most cases it's related to the RAM/NVMe/SSD you use. For example: + ```console Error: A stage encountered an irrecoverable error. @@ -132,6 +137,7 @@ If you encounter an error while accessing the database from multiple processes a ```console mdbx:0: panic: Assertion `osal_rdt_unlock() failed: err 1' failed. ``` + or ```console @@ -151,61 +157,71 @@ If your hardware performance is significantly lower than these reference numbers ### Disk Speed Testing with [IOzone](https://linux.die.net/man/1/iozone) 1. Test disk speed: - ```bash - iozone -e -t1 -i0 -i2 -r1k -s1g /tmp - ``` - Reference numbers (on Latitude c3.large.x86): - - ```console - Children see throughput for 1 initial writers = 907733.81 kB/sec - Parent sees throughput for 1 initial writers = 907239.68 kB/sec - Children see throughput for 1 rewriters = 1765222.62 kB/sec - Parent sees throughput for 1 rewriters = 1763433.35 kB/sec - Children see throughput for 1 random readers = 1557497.38 kB/sec - Parent sees throughput for 1 random readers = 1554846.58 kB/sec - Children see throughput for 1 random writers = 984428.69 kB/sec - Parent sees throughput for 1 random writers = 983476.67 kB/sec - ``` + + ```bash + iozone -e -t1 -i0 -i2 -r1k -s1g /tmp + ``` + + Reference numbers (on Latitude c3.large.x86): + + ```console + Children see throughput for 1 initial writers = 907733.81 kB/sec + Parent sees throughput for 1 initial writers = 907239.68 kB/sec + Children see throughput for 1 rewriters = 1765222.62 kB/sec + Parent sees throughput for 1 rewriters = 1763433.35 kB/sec + Children see throughput for 1 random readers = 1557497.38 kB/sec + Parent sees throughput for 1 random readers = 1554846.58 kB/sec + Children see throughput for 1 random writers = 984428.69 kB/sec + Parent sees throughput for 1 random writers = 983476.67 kB/sec + ``` + 2. Test disk speed with memory-mapped files: - ```bash - iozone -B -G -e -t1 -i0 -i2 -r1k -s1g /tmp - ``` - Reference numbers (on Latitude c3.large.x86): - - ```console - Children see throughput for 1 initial writers = 56471.06 kB/sec - Parent sees throughput for 1 initial writers = 56365.14 kB/sec - Children see throughput for 1 rewriters = 241650.69 kB/sec - Parent sees throughput for 1 rewriters = 239067.96 kB/sec - Children see throughput for 1 random readers = 6833161.00 kB/sec - Parent sees throughput for 1 random readers = 5597659.65 kB/sec - Children see throughput for 1 random writers = 220248.53 kB/sec - Parent sees throughput for 1 random writers = 219112.26 kB/sec + + ```bash + iozone -B -G -e -t1 -i0 -i2 -r1k -s1g /tmp + ``` + + Reference numbers (on Latitude c3.large.x86): + + ```console + Children see throughput for 1 initial writers = 56471.06 kB/sec + Parent sees throughput for 1 initial writers = 56365.14 kB/sec + Children see throughput for 1 rewriters = 241650.69 kB/sec + Parent sees throughput for 1 rewriters = 239067.96 kB/sec + Children see throughput for 1 random readers = 6833161.00 kB/sec + Parent sees throughput for 1 random readers = 5597659.65 kB/sec + Children see throughput for 1 random writers = 220248.53 kB/sec + Parent sees throughput for 1 random writers = 219112.26 kB/sec ``` ### RAM Speed and Health Testing 1. Check RAM speed with [lshw](https://linux.die.net/man/1/lshw): - ```bash - sudo lshw -short -C memory - ``` - Look for the frequency in the output. Reference output: - - ```console - H/W path Device Class Description - ================================================================ - /0/24/0 memory 64GiB DIMM DDR4 Synchronous Registered (Buffered) 3200 MHz (0.3 ns) - /0/24/1 memory 64GiB DIMM DDR4 Synchronous Registered (Buffered) 3200 MHz (0.3 ns) - ... - ``` + + ```bash + sudo lshw -short -C memory + ``` + + Look for the frequency in the output. Reference output: + + ```console + H/W path Device Class Description + ================================================================ + /0/24/0 memory 64GiB DIMM DDR4 Synchronous Registered (Buffered) 3200 MHz (0.3 ns) + /0/24/1 memory 64GiB DIMM DDR4 Synchronous Registered (Buffered) 3200 MHz (0.3 ns) + ... + ``` 2. Test RAM health with [memtester](https://linux.die.net/man/8/memtester): - ```bash - sudo memtester 10G - ``` - This will take a while. You can test with a smaller amount first: - - ```bash - sudo memtester 1G 1 - ``` - All checks should report "ok". + + ```bash + sudo memtester 10G + ``` + + This will take a while. You can test with a smaller amount first: + + ```bash + sudo memtester 1G 1 + ``` + + All checks should report "ok". diff --git a/book/run/observability.md b/docs/vocs/docs/pages/run/monitoring.mdx similarity index 85% rename from book/run/observability.md rename to docs/vocs/docs/pages/run/monitoring.mdx index e654bd8fa16..1b463efdb7d 100644 --- a/book/run/observability.md +++ b/docs/vocs/docs/pages/run/monitoring.mdx @@ -1,3 +1,7 @@ +--- +description: Reth observability and metrics with Prometheus and Grafana. +--- + # Observability with Prometheus & Grafana Reth exposes a number of metrics which can be enabled by adding the `--metrics` flag: @@ -6,6 +10,12 @@ Reth exposes a number of metrics which can be enabled by adding the `--metrics` reth node --metrics 127.0.0.1:9001 ``` +Additionally, you can export spans to an OpenTelemetry collector using `--tracing-otlp`: + +```bash +reth node --tracing-otlp=http://localhost:4318/v1/traces +``` + Now, as the node is running, you can `curl` the endpoint you provided to the `--metrics` flag to get a text dump of the metrics at that time: ```bash @@ -41,6 +51,7 @@ brew install grafana ### Linux #### Debian/Ubuntu + ```bash # Install Prometheus # Visit https://prometheus.io/download/ for the latest version @@ -52,12 +63,13 @@ cd prometheus-* # Install Grafana sudo apt-get install -y apt-transport-https software-properties-common wget -q -O - https://packages.grafana.com/gpg.key | sudo apt-key add - -echo "deb https://packages.grafana.com/oss/deb stable main" | sudo tee -a /etc/apt/sources.list.d/grafana.list +echo "deb https://packages.grafana.com stable main" | sudo tee -a /etc/apt/sources.list.d/grafana.list sudo apt-get update sudo apt-get install grafana ``` #### Fedora/RHEL/CentOS + ```bash # Install Prometheus # Visit https://prometheus.io/download/ for the latest version @@ -74,16 +86,18 @@ sudo dnf install -y https://dl.grafana.com/oss/release/grafana-latest-1.x86_64.r ### Windows #### Using Chocolatey + ```powershell choco install prometheus choco install grafana ``` #### Manual installation + 1. Download the latest Prometheus from [prometheus.io/download](https://prometheus.io/download/) - - Select the Windows binary (.zip) for your architecture (typically windows-amd64) + - Select the Windows binary (.zip) for your architecture (typically windows-amd64) 2. Download the latest Grafana from [grafana.com/grafana/download](https://grafana.com/grafana/download) - - Choose the Windows installer (.msi) or standalone version + - Choose the Windows installer (.msi) or standalone version 3. Extract Prometheus to a location of your choice (e.g., `C:\prometheus`) 4. Install Grafana by running the installer or extracting the standalone version 5. Configure Prometheus and Grafana to run as services if needed @@ -95,7 +109,7 @@ Then, kick off the Prometheus and Grafana services: brew services start prometheus brew services start grafana -# For Linux (systemd-based distributions) +# For Linux (syst-based distributions) sudo systemctl start prometheus sudo systemctl start grafana-server @@ -104,15 +118,15 @@ Start-Service prometheus Start-Service grafana ``` -This will start a Prometheus service which [by default scrapes itself about the current instance](https://prometheus.io/docs/introduction/first_steps/#:~:text=The%20job%20contains%20a%20single,%3A%2F%2Flocalhost%3A9090%2Fmetrics.). So you'll need to change its config to hit your Reth nodes metrics endpoint at `localhost:9001` which you set using the `--metrics` flag. +This will start a Prometheus service which [by default scrapes itself about the current instance](https://prometheus.io/docs/introduction/first_steps/#:~:text=The%20job%20contains%20a%20single,%3A%2F%2Flocalhost%3A9090%2Fmetrics.). So you'll need to change its config to hit your Reth node’s metrics endpoint at `localhost:9001` which you set using the `--metrics` flag. You can find an example config for the Prometheus service in the repo here: [`etc/prometheus/prometheus.yml`](https://github.com/paradigmxyz/reth/blob/main/etc/prometheus/prometheus.yml) Depending on your installation you may find the config for your Prometheus service at: -- OSX: `/opt/homebrew/etc/prometheus.yml` -- Linuxbrew: `/home/linuxbrew/.linuxbrew/etc/prometheus.yml` -- Others: `/usr/local/etc/prometheus/prometheus.yml` +- OSX: `/opt/homebrew/etc/prometheus.yml` +- Linuxbrew: `/home/linuxbrew/.linuxbrew/etc/prometheus.yml` +- Others: `/usr/local/etc/prometheus/prometheus.yml` Next, open up "localhost:3000" in your browser, which is the default URL for Grafana. Here, "admin" is the default for both the username and password. @@ -122,7 +136,7 @@ As this might be a point of confusion, `localhost:9001`, which we supplied to `- To configure the dashboard in Grafana, click on the squares icon in the upper left, and click on "New", then "Import". From there, click on "Upload JSON file", and select the example file in [`reth/etc/grafana/dashboards/overview.json`](https://github.com/paradigmxyz/reth/blob/main/etc/grafana/dashboards/overview.json). Finally, select the Prometheus data source you just created, and click "Import". -And voilá, you should see your dashboard! If you're not yet connected to any peers, the dashboard will look like it's in an empty state, but once you are, you should see it start populating with data. +And voilà, you should see your dashboard! If you're not yet connected to any peers, the dashboard will look like it's in an empty state, but once you are, you should see it start populating with data. ## Conclusion @@ -130,7 +144,7 @@ In this runbook, we took you through starting the node, exposing different log l This will all be very useful to you, whether you're simply running a home node and want to keep an eye on its performance, or if you're a contributor and want to see the effect that your (or others') changes have on Reth's operations. -[installation]: ../installation/installation.md +[installation]: ../installation/installation [release-profile]: https://doc.rust-lang.org/cargo/reference/profiles.html#release [docs]: https://github.com/paradigmxyz/reth/tree/main/docs -[metrics]: https://github.com/paradigmxyz/reth/blob/main/docs/design/metrics.md#current-metrics +[metrics]: https://reth.rs/run/observability.html diff --git a/docs/vocs/docs/pages/run/networks.mdx b/docs/vocs/docs/pages/run/networks.mdx new file mode 100644 index 00000000000..1bb6593b2e4 --- /dev/null +++ b/docs/vocs/docs/pages/run/networks.mdx @@ -0,0 +1 @@ +# Networks diff --git a/book/run/optimism.md b/docs/vocs/docs/pages/run/opstack.mdx similarity index 91% rename from book/run/optimism.md rename to docs/vocs/docs/pages/run/opstack.mdx index 4d43b553bba..d472485be60 100644 --- a/book/run/optimism.md +++ b/docs/vocs/docs/pages/run/opstack.mdx @@ -1,7 +1,12 @@ +--- +description: Running Reth on Optimism and OP Stack chains. +--- + # Running Reth on OP Stack chains `reth` ships with the `optimism` feature flag in several crates, including the binary, enabling support for OP Stack chains out of the box. Optimism has a small diff from the [L1 EELS][l1-el-spec], comprising of the following key changes: + 1. A new transaction type, [`0x7E (Deposit)`][deposit-spec], which is used to deposit funds from L1 to L2. 1. Modifications to the `PayloadAttributes` that allow the [sequencer][sequencer] to submit transactions to the EL through the Engine API. Payloads will be built with deposit transactions at the top of the block, with the first deposit transaction always being the "L1 Info Transaction." @@ -12,9 +17,14 @@ comprising of the following key changes: For a more in-depth list of changes and their rationale, as well as specifics about the OP Stack specification such as transaction ordering and more, see the documented [`op-geth` diff][op-geth-forkdiff], the [L2 EL specification][l2-el-spec], and the [OP Stack specification][op-stack-spec]. +### Superchain Registry + +Since 1.4.0 op-reth has built in support for all chains in the [superchain registry][superchain-registry]. All superchains are supported by the `--chain` argument, e.g. `--chain unichain` or `--chain unichain-sepolia`. + ## Running on Optimism You will need three things to run `op-reth`: + 1. An archival L1 node, synced to the settlement layer of the OP Stack chain you want to sync (e.g. `reth`, `geth`, `besu`, `nethermind`, etc.) 1. A rollup node (e.g. `op-node`, `magi`, `hildr`, etc.) 1. An instance of `op-reth`. @@ -36,6 +46,7 @@ This will install the `op-reth` binary to `~/.cargo/bin/op-reth`. ### Installing a Rollup Node Next, you'll need to install a [Rollup Node][rollup-node-spec], which is the equivalent to the Consensus Client on the OP Stack. Available options include: + 1. [`op-node`][op-node] 1. [`magi`][magi] 1. [`hildr`][hildr] @@ -44,13 +55,14 @@ For the sake of this tutorial, we'll use the reference implementation of the Rol ### Running `op-reth` -The `optimism` feature flag in `op-reth` adds several new CLI flags to the `reth` binary: +op-reth supports additional OP Stack specific CLI arguments: + 1. `--rollup.sequencer-http ` - The sequencer endpoint to connect to. Transactions sent to the `op-reth` EL are also forwarded to this sequencer endpoint for inclusion, as the sequencer is the entity that builds blocks on OP Stack chains. 1. `--rollup.disable-tx-pool-gossip` - Disables gossiping of transactions in the mempool to peers. This can be omitted for personal nodes, though providers should always opt to enable this flag. -1. `--rollup.enable-genesis-walkback` - Disables setting the forkchoice status to tip on startup, making the `op-node` walk back to genesis and verify the integrity of the chain before starting to sync. This can be omitted unless a corruption of local chainstate is suspected. 1. `--rollup.discovery.v4` - Enables the discovery v4 protocol for peer discovery. By default, op-reth, similar to op-geth, has discovery v5 enabled and discovery v4 disabled, whereas regular reth has discovery v4 enabled and discovery v5 disabled. First, ensure that your L1 archival node is running and synced to tip. Also make sure that the beacon node / consensus layer client is running and has http APIs enabled. Then, start `op-reth` with the `--rollup.sequencer-http` flag set to the `Base Mainnet` sequencer endpoint: + ```sh op-reth node \ --chain base \ @@ -62,6 +74,7 @@ op-reth node \ ``` Then, once `op-reth` has been started, start up the `op-node`: + ```sh op-node \ --network="base-mainnet" \ @@ -85,9 +98,8 @@ Consider adding the `--l1.trustrpc` flag to improve performance, if the connecti [l2-el-spec]: https://github.com/ethereum-optimism/specs/blob/main/specs/protocol/exec-engine.md [deposit-spec]: https://github.com/ethereum-optimism/specs/blob/main/specs/protocol/deposits.md [derivation-spec]: https://github.com/ethereum-optimism/specs/blob/main/specs/protocol/derivation.md - +[superchain-registry]: https://github.com/ethereum-optimism/superchain-registry [op-node-docker]: https://console.cloud.google.com/artifacts/docker/oplabs-tools-artifacts/us/images/op-node - [reth]: https://github.com/paradigmxyz/reth [op-node]: https://github.com/ethereum-optimism/optimism/tree/develop/op-node [magi]: https://github.com/a16z/magi diff --git a/docs/vocs/docs/pages/run/opstack/op-mainnet-caveats.mdx b/docs/vocs/docs/pages/run/opstack/op-mainnet-caveats.mdx new file mode 100644 index 00000000000..94f1024dfca --- /dev/null +++ b/docs/vocs/docs/pages/run/opstack/op-mainnet-caveats.mdx @@ -0,0 +1 @@ +# Caveats OP-Mainnet \ No newline at end of file diff --git a/docs/vocs/docs/pages/run/overview.mdx b/docs/vocs/docs/pages/run/overview.mdx new file mode 100644 index 00000000000..d603a7be64b --- /dev/null +++ b/docs/vocs/docs/pages/run/overview.mdx @@ -0,0 +1,47 @@ +--- +description: Guide to running a Reth node. +--- + +# Run a Node + +Congratulations, now that you have installed Reth, it's time to run it! + +In this section, we'll guide you through running a Reth node on various networks and configurations. + +## Networks + +Choose the network you want to run your node on: + +- **[Ethereum](/run/ethereum)** - Run a node on Ethereum mainnet or testnets +- **[OP-stack](/run/opstack)** - Run a node on OP Stack chains like Base, Optimism, and others +- **[Private testnets](/run/private-testnets)** - Set up and run private test networks + +## Configuration & Monitoring + +Learn how to configure and monitor your node: + +- **[Configuration](/run/configuration)** - Configure your node using reth.toml +- **[Monitoring](/run/monitoring)** - Set up logs, metrics, and observability + +## Frequently Asked Questions + +Find answers to common questions and troubleshooting tips: + +- **[Transaction Types](/run/faq/transactions)** - Understanding different transaction types +- **[Pruning & Full Node](/run/faq/pruning)** - Storage management and node types +- **[Ports](/run/faq/ports)** - Network port configuration +- **[Profiling](/run/faq/profiling)** - Performance profiling and optimization +- **[Sync OP Mainnet](/run/faq/sync-op-mainnet)** - Tips for syncing OP Mainnet + +## List of Supported Networks + +| Network | Chain ID | RPC URL | +| --------------- | -------- | ------------------------------------ | +| Ethereum | 1 | https://reth-ethereum.ithaca.xyz/rpc | +| Sepolia Testnet | 11155111 | https://sepolia.drpc.org | +| Base | 8453 | https://base-mainnet.rpc.ithaca.xyz | +| Base Sepolia | 84532 | https://base-sepolia.drpc.org | + +:::tip +Want to add more networks to this table? Feel free to [contribute](https://github.com/paradigmxyz/reth/edit/main/book/vocs/docs/pages/run/overview.mdx) by submitting a PR with additional networks that Reth supports! +::: diff --git a/book/run/private-testnet.md b/docs/vocs/docs/pages/run/private-testnets.mdx similarity index 90% rename from book/run/private-testnet.md rename to docs/vocs/docs/pages/run/private-testnets.mdx index 28253ca9f01..af281fc5127 100644 --- a/book/run/private-testnet.md +++ b/docs/vocs/docs/pages/run/private-testnets.mdx @@ -1,10 +1,17 @@ +--- +description: Running Reth in a private testnet using Kurtosis. +--- + # Run Reth in a private testnet using Kurtosis + For those who need a private testnet to validate functionality or scale with Reth. ## Using Docker locally + This guide uses [Kurtosis' ethereum-package](https://github.com/ethpandaops/ethereum-package) and assumes you have Kurtosis and Docker installed and have Docker already running on your machine. -* Go [here](https://docs.kurtosis.com/install/) to install Kurtosis -* Go [here](https://docs.docker.com/get-docker/) to install Docker + +- Go [here](https://docs.kurtosis.com/install/) to install Kurtosis +- Go [here](https://docs.docker.com/get-docker/) to install Docker The [`ethereum-package`](https://github.com/ethpandaops/ethereum-package) is a [package](https://docs.kurtosis.com/advanced-concepts/packages) for a general purpose Ethereum testnet definition used for instantiating private testnets at any scale over Docker or Kubernetes, locally or in the cloud. This guide will go through how to spin up a local private testnet with Reth and various CL clients locally. Specifically, you will instantiate a 2-node network over Docker with Reth/Lighthouse and Reth/Teku client combinations. @@ -13,17 +20,19 @@ To see all possible configurations and flags you can use, including metrics and Genesis data will be generated using this [genesis-generator](https://github.com/ethpandaops/ethereum-genesis-generator) to be used to bootstrap the EL and CL clients for each node. The end result will be a private testnet with nodes deployed as Docker containers in an ephemeral, isolated environment on your machine called an [enclave](https://docs.kurtosis.com/advanced-concepts/enclaves/). Read more about how the `ethereum-package` works by going [here](https://github.com/ethpandaops/ethereum-package/). ### Step 1: Define the parameters and shape of your private network + First, in your home directory, create a file with the name `network_params.yaml` with the following contents: + ```yaml participants: - - el_type: reth - el_image: ghcr.io/paradigmxyz/reth - cl_type: lighthouse - cl_image: sigp/lighthouse:latest - - el_type: reth - el_image: ghcr.io/paradigmxyz/reth - cl_type: teku - cl_image: consensys/teku:latest + - el_type: reth + el_image: ghcr.io/paradigmxyz/reth + cl_type: lighthouse + cl_image: sigp/lighthouse:latest + - el_type: reth + el_image: ghcr.io/paradigmxyz/reth + cl_type: teku + cl_image: consensys/teku:latest ``` > [!TIP] @@ -32,10 +41,13 @@ participants: ### Step 2: Spin up your network Next, run the following command from your command line: + ```bash kurtosis run github.com/ethpandaops/ethereum-package --args-file ~/network_params.yaml --image-download always ``` + Kurtosis will spin up an [enclave](https://docs.kurtosis.com/advanced-concepts/enclaves/) (i.e an ephemeral, isolated environment) and begin to configure and instantiate the nodes in your network. In the end, Kurtosis will print the services running in your enclave that form your private testnet alongside all the container ports and files that were generated & used to start up the private testnet. Here is a sample output: + ```console INFO[2024-07-09T12:01:35+02:00] ======================================================== INFO[2024-07-09T12:01:35+02:00] || Created enclave: silent-mountain || @@ -88,14 +100,18 @@ f0a7d5343346 vc-1-reth-lighthouse metrics: 8080/tc Great! You now have a private network with 2 full Ethereum nodes on your local machine over Docker - one that is a Reth/Lighthouse pair and another that is Reth/Teku. Check out the [Kurtosis docs](https://docs.kurtosis.com/cli) to learn about the various ways you can interact with and inspect your network. ## Using Kurtosis on Kubernetes + Kurtosis packages are portable and reproducible, meaning they will work the same way over Docker or Kubernetes, locally or on remote infrastructure. For use cases that require a larger scale, Kurtosis can be deployed on Kubernetes by following these docs [here](https://docs.kurtosis.com/k8s/). ## Running the network with additional services + The [`ethereum-package`](https://github.com/ethpandaops/ethereum-package) comes with many optional flags and arguments you can enable for your private network. Some include: -- A Grafana + Prometheus instance -- A transaction spammer called [`tx-fuzz`](https://github.com/MariusVanDerWijden/tx-fuzz) -- [A network metrics collector](https://github.com/dapplion/beacon-metrics-gazer) -- Flashbot's `mev-boost` implementation of PBS (to test/simulate MEV workflows) + +- A Grafana + Prometheus instance +- A transaction spammer called [`tx-fuzz`](https://github.com/MariusVanDerWijden/tx-fuzz) +- [A network metrics collector](https://github.com/dapplion/beacon-metrics-gazer) +- Flashbot's `mev-boost` implementation of PBS (to test/simulate MEV workflows) ### Questions? + Please reach out to the [Kurtosis discord](https://discord.com/invite/6Jjp9c89z9) should you have any questions about how to use the `ethereum-package` for your private testnet needs. Thanks! diff --git a/book/installation/installation.md b/docs/vocs/docs/pages/run/system-requirements.mdx similarity index 52% rename from book/installation/installation.md rename to docs/vocs/docs/pages/run/system-requirements.mdx index 602601b9f30..cb014a01972 100644 --- a/book/installation/installation.md +++ b/docs/vocs/docs/pages/run/system-requirements.mdx @@ -1,72 +1,71 @@ -# Installation +# System Requirements -Reth runs on Linux and macOS (Windows tracked). - -There are three core methods to obtain Reth: - -* [Pre-built binaries](./binaries.md) -* [Docker images](./docker.md) -* [Building from source.](./source.md) +The hardware requirements for running Reth depend on the node configuration and can change over time as the network grows or new features are implemented. -> **Note** -> -> If you have Docker installed, we recommend using the [Docker Compose](./docker.md#using-docker-compose) configuration -> that will get you Reth, Lighthouse (Consensus Client), Prometheus and Grafana running and syncing with just one command. +The most important requirement is by far the disk, whereas CPU and RAM requirements are relatively flexible. -## Hardware Requirements +## Chain Specific Requirements -The hardware requirements for running Reth depend on the node configuration and can change over time as the network grows or new features are implemented. +### Ethereum Mainnet -The most important requirement is by far the disk, whereas CPU and RAM requirements are relatively flexible. +Below are the requirements for running an Ethereum Mainnet node as of 2025-06-23 block number `22700000`: | | Archive Node | Full Node | -|-----------|---------------------------------------|---------------------------------------| -| Disk | At least 2.8TB (TLC NVMe recommended) | At least 1.8TB (TLC NVMe recommended) | +| --------- | ------------------------------------- | ------------------------------------- | +| Disk | At least 2.8TB (TLC NVMe recommended) | At least 1.2TB (TLC NVMe recommended) | | Memory | 16GB+ | 8GB+ | | CPU | Higher clock speed over core count | Higher clock speeds over core count | | Bandwidth | Stable 24Mbps+ | Stable 24Mbps+ | -#### QLC and TLC +### Base Mainnet -It is crucial to understand the difference between QLC and TLC NVMe drives when considering the disk requirement. +Below are the minimum system requirements for running a Base Mainnet node as of 2025-06-23, block number `31900000`: -QLC (Quad-Level Cell) NVMe drives utilize four bits of data per cell, allowing for higher storage density and lower manufacturing costs. However, this increased density comes at the expense of performance. QLC drives have slower read and write speeds compared to TLC drives. They also have a lower endurance, meaning they may have a shorter lifespan and be less suitable for heavy workloads or constant data rewriting. +| | Archive Node | Full Node | +| --------- | -------------------------------------------- | -------------------------------------------- | +| Disk | At least 4.1TB (TLC NVMe recommended) | At least 2TB (TLC NVMe recommended) | +| Memory | 128GB+ | 128GB+ | +| CPU | 6 cores+, Higher clock speed over core count | 6 cores+, Higher clock speed over core count | +| Bandwidth | Stable 24Mbps+ | Stable 24Mbps+ | -TLC (Triple-Level Cell) NVMe drives, on the other hand, use three bits of data per cell. While they have a slightly lower storage density compared to QLC drives, TLC drives offer faster performance. They typically have higher read and write speeds, making them more suitable for demanding tasks such as data-intensive applications, gaming, and multimedia editing. TLC drives also tend to have a higher endurance, making them more durable and longer-lasting. +:::note +**On CPU clock speeds**: The AMD EPYC 4005/4004 series is a cost-effective high-clock speed option with support for up to 192GB memory. + +**On CPU cores for Base**: 5+ cores are needed because the state root task splits work into separate threads that run in parallel with each other. The state root task is generally more performant and can scale with the number of CPU cores, while regular state root always uses only one core. This is not a requirement for Mainnet, but for Base you may encounter block processing latencies of more than 2s, which can lead to lagging behind the head of the chain. +::: -Prior to purchasing an NVMe drive, it is advisable to research and determine whether the disk will be based on QLC or TLC technology. An overview of recommended and not-so-recommended NVMe boards can be found at [here]( https://gist.github.com/yorickdowne/f3a3e79a573bf35767cd002cc977b038). +## Disk -### Disk +Simplest approach: Use a [good TLC NVMe](https://gist.github.com/yorickdowne/f3a3e79a573bf35767cd002cc977b038) drive for everything. -There are multiple types of disks to sync Reth, with varying size requirements, depending on the syncing mode. -As of April 2025 at block number 22.1M: +Advanced Storage Optimization (Optional): -* Archive Node: At least 2.8TB is required -* Full Node: At least 1.8TB is required +- TLC NVMe: All application data except static files (`--datadir`) +- SATA SSD/HDD: Static files can be stored on slower & cheaper storage (`--datadir.static-files`) -NVMe based SSD drives are recommended for the best performance, with SATA SSDs being a cheaper alternative. HDDs are the cheapest option, but they will take the longest to sync, and are not recommended. +### QLC and TLC + +It is crucial to understand the difference between QLC and TLC NVMe drives when considering the disk requirement. -As of February 2024, syncing an Ethereum mainnet node to block 19.3M on NVMe drives takes about 50 hours, while on a GCP "Persistent SSD" it takes around 5 days. +QLC (Quad-Level Cell) NVMe drives utilize four bits of data per cell, allowing for higher storage density and lower manufacturing costs. However, this increased density comes at the expense of performance. QLC drives have slower read and write speeds compared to TLC drives. They also have a lower endurance, meaning they may have a shorter lifespan and be less suitable for heavy workloads or constant data rewriting. -> **Note** -> -> It is highly recommended to choose a TLC drive when using an NVMe drive, and not a QLC drive. See [the note](#qlc-and-tlc) above. A list of recommended drives can be found [here]( https://gist.github.com/yorickdowne/f3a3e79a573bf35767cd002cc977b038). +TLC (Triple-Level Cell) NVMe drives, on the other hand, use three bits of data per cell. While they have a slightly lower storage density compared to QLC drives, TLC drives offer faster performance. They typically have higher read and write speeds, making them more suitable for demanding tasks such as data-intensive applications, gaming, and multimedia editing. TLC drives also tend to have a higher endurance, making them more durable and longer-lasting. -### CPU +## CPU Most of the time during syncing is spent executing transactions, which is a single-threaded operation due to potential state dependencies of a transaction on previous ones. As a result, the number of cores matters less, but in general higher clock speeds are better. More cores are better for parallelizable [stages](https://github.com/paradigmxyz/reth/blob/main/docs/crates/stages.md) (like sender recovery or bodies downloading), but these stages are not the primary bottleneck for syncing. -### Memory +## Memory -It is recommended to use at least 8GB of RAM. +It is recommended to use at least 16GB of RAM. Most of Reth's components tend to consume a low amount of memory, unless you are under heavy RPC load, so this should matter less than the other requirements. Higher memory is generally better as it allows for better caching, resulting in less stress on the disk. -### Bandwidth +## Bandwidth A stable and dependable internet connection is crucial for both syncing a node from genesis and for keeping up with the chain's tip. @@ -76,6 +75,13 @@ Once you're synced to the tip you will need a reliable connection, especially if ## What hardware can I get? -If you are buying your own NVMe SSD, please consult [this hardware comparison](https://gist.github.com/yorickdowne/f3a3e79a573bf35767cd002cc977b038) which is being actively maintained. We recommend against buying DRAM-less or QLC devices as these are noticeably slower. +### Build your own + +- Storage: Consult the [Great and less great SSDs for Ethereum nodes](https://gist.github.com/yorickdowne/f3a3e79a573bf35767cd002cc977b038) gist. The Seagate Firecuda 530 and WD Black SN850(X) are popular TLC NVMe options. Ensure proper cooling via heatsinks or active fans. +- CPU: AMD Ryzen 5000/7000/9000 series, AMD EPYC 4004/4005 or Intel Core i5/i7 (11th gen or newer) with at least 6 cores. The AMD Ryzen 9000 series and the AMD EPYC 4005 series offer good value. +- Memory: 32GB DDR4 or DDR5 (ECC if your motherboard & CPU supports it). + +### Hosted -All our benchmarks have been produced on [Latitude.sh](https://www.latitude.sh/), a bare metal provider. We use `c3.large.x86` boxes, and also recommend trying the `c3.small.x86` box for pruned/full nodes. So far our experience has been smooth with some users reporting that the NVMEs there outperform AWS NVMEs by 3x or more. We're excited for more Reth nodes on Latitude.sh, so for a limited time you can use `RETH400` for a $250 discount. [Run a node now!](https://metal.new/reth) +- [Latitude.sh](https://www.latitude.sh): `f4.metal.small`, `c3.large.x86` or better +- [OVH](https://www.ovhcloud.com/en/bare-metal/advance/): `Advance-1` or better diff --git a/docs/vocs/docs/pages/sdk.mdx b/docs/vocs/docs/pages/sdk.mdx new file mode 100644 index 00000000000..b308dee77ae --- /dev/null +++ b/docs/vocs/docs/pages/sdk.mdx @@ -0,0 +1,125 @@ +# Reth for Developers + +Reth can be used as a library to build custom Ethereum nodes, interact with blockchain data, or create specialized tools for blockchain analysis and indexing. + +## What is the Reth SDK? + +The Reth SDK allows developers to: + +- Use components of the Reth node as libraries +- Build custom Ethereum execution nodes with modified behavior (e.g. payload building) +- Access blockchain data directly from the database +- Create high-performance indexing solutions +- Extend a new with new RPC endpoints and functionality +- Implement custom consensus mechanisms +- Build specialized tools for blockchain analysis + +## Quick Start + +Add Reth to your project + +### Ethereum + +```toml +[dependencies] +# Ethereum meta crate +reth-ethereum = { git = "https://github.com/paradigmxyz/reth" } +``` + +### Opstack + +```toml +[dependencies] +reth-op = { git = "https://github.com/paradigmxyz/reth" } +``` + +## Key Concepts + +### Node Architecture + +Reth is built with modularity in mind. The main components include: + +- **Primitives**: Core data type abstractions like `Block` +- **Node Builder**: Constructs and configures node instances +- **Database**: Efficient storage using MDBX and static files +- **Network**: P2P communication and block synchronization +- **Consensus**: Block validation and chain management +- **EVM**: Transaction execution and state transitions +- **RPC**: JSON-RPC server for external communication +- **Transaction Pool**: Pending transaction management + +### Dependency Management + +Reth is primarily built on top of the [alloy](https://github.com/alloy-rs/alloy) ecosystem, which provides the necessary abstractions and implementations for core ethereum blockchain data types, transaction handling, and EVM execution. + +### Type System + +Reth uses its own type system to handle different representations of blockchain data: + +- **Primitives**: Core types like `B256`, `Address`, `U256` +- **Transactions**: Multiple representations for different contexts (pooled, consensus, RPC) +- **Blocks**: Headers, bodies, and sealed blocks with proven properties +- **State**: Accounts, storage, and state transitions + +### Building Custom Nodes + +The node builder pattern allows you to customize every aspect of node behavior: + +```rust +use reth_ethereum::node::{EthereumNode, NodeBuilder}; + +// Build a custom node with modified components +let node = NodeBuilder::new(config) + // install the ethereum specific node primitives + .with_types::() + .with_components(|components| { + // Customize components here + components + }) + .build() + .await?; +``` + +## Architecture Overview + +```mermaid +graph TD + A[Node Builder] --> B[Database] + A --> C[Network] + A --> D[Consensus] + A --> E[EVM] + A --> F[RPC Server] + A --> G[Transaction Pool] + + B --> H[DB Storage] + B --> I[Static Files] + + C --> J[Discovery] + C --> K[ETH Protocol] + + E --> L[State Provider] + E --> M[Block Executor] +``` + +## Nodes Built with Reth + +Several production networks have been built using Reth's node builder pattern: + +| Node | Company | Description | Lines of Code | +|------|---------|-------------|---------------| +| [Base Node](https://github.com/base/node-reth) | Coinbase | Coinbase's L2 scaling solution node implementation | ~3K | +| [Bera Reth](https://github.com/berachain/bera-reth) | Berachain | Berachain's high-performance EVM node with custom features | ~1K | +| [Reth Gnosis](https://github.com/gnosischain/reth_gnosis) | Gnosis | Gnosis Chain's xDai-compatible execution client | ~5K | +| [Reth BSC](https://github.com/loocapro/reth-bsc) | Binance Smart Chain | BNB Smart Chain execution client implementation | ~6K | + +## Next Steps + +- **[Node Components](/sdk/node-components)**: Deep dive into each component +- **[Type System](/sdk/typesystem/block)**: Understanding Reth's type system +- **[Custom Nodes](/sdk/custom-node/prerequisites)**: Building production nodes +- **[Examples](/sdk/examples/modify-node)**: Real-world implementations + +## Resources + +- [API Documentation](https://docs.rs/reth/latest/reth/) +- [GitHub Repository](https://github.com/paradigmxyz/reth) diff --git a/docs/vocs/docs/pages/sdk/custom-node/modifications.mdx b/docs/vocs/docs/pages/sdk/custom-node/modifications.mdx new file mode 100644 index 00000000000..b375feb901b --- /dev/null +++ b/docs/vocs/docs/pages/sdk/custom-node/modifications.mdx @@ -0,0 +1 @@ +# Modifying Node Components diff --git a/docs/vocs/docs/pages/sdk/custom-node/prerequisites.mdx b/docs/vocs/docs/pages/sdk/custom-node/prerequisites.mdx new file mode 100644 index 00000000000..8dbf0a1bf48 --- /dev/null +++ b/docs/vocs/docs/pages/sdk/custom-node/prerequisites.mdx @@ -0,0 +1 @@ +# Prerequisites and Considerations diff --git a/docs/vocs/docs/pages/sdk/custom-node/transactions.mdx b/docs/vocs/docs/pages/sdk/custom-node/transactions.mdx new file mode 100644 index 00000000000..52881a368fb --- /dev/null +++ b/docs/vocs/docs/pages/sdk/custom-node/transactions.mdx @@ -0,0 +1,299 @@ +# Custom transactions + +In this chapter, we'll learn how to define custom crate-local transaction envelope types and configure our node to use it. +We'll extend it with a custom variant, implement custom processing logic and configure our custom node to use it. + +All the while trying to minimize boilerplate, trivial, unnecessary or copy-pasted code. + +# Motivation + +Historically, custom node operators were forced to fork the blockchain client repository they were working with if they +wanted to introduce complex custom changes beyond the scope of the vanilla node configuration. Forking represents a huge +maintenance burden due to the complexities of keeping the code up-to-date with the custom changes intact. + +We introduced Reth SDK to address this widespread issue, where we operate in a continuous feed-back loop, continuously +shaping it to fit the needs of node operators. + +Oftentimes we may want to preserve the full capabilities of an Ethereum blockchain but introduce a special transaction +that has different processing. For example, one may introduce a transaction type that does not invoke any EVM processing, +but still produces a new state, performing a computation at no gas cost. + +# Type definition using declarative macro + +We'll showcase the macro on the `custom-node` example in the `reth` repository. +Please refer to it to see the complete implementation: https://github.com/paradigmxyz/reth/tree/main/examples/custom-node + +## Introduction + +In this example, we assume that we are building our node on top of an Optimism stack. But all the things we're doing are +analogous to the way you would build on top off of an L1 node, for example. Just use Ethereum counterparts to the Optimism +ones. + +## Dependencies + +We recommend copying out the dependencies list from the [manifest of the custom node example](https://github.com/paradigmxyz/reth/blob/main/examples/custom-node/Cargo.toml). + +The transaction envelope macro resides in the `alloy_consensus` crate since version `1.0.10`. It's being consistently improved upon so it's recommended to use the latest version. +Since we're building on top of Optimism we will also need `op-alloy` that contains Optimism specific extensions. + +Our goal is Reth compatibility, hence we need to import relevant Reth crates. Sometimes items from REVM are referenced +by Reth as its transaction execution environment, so we also need to import it. + +There may be occasionally additional dependencies needed. Refer to the [custom node example](https://github.com/paradigmxyz/reth/blob/main/examples/custom-node/Cargo.toml) for a complete list. + +## Declaration + +### Consensus + +When one thinks of a transaction, usually they mean as it is defined in the consensus layer. There are however more +interpretations depending on context. We'll start with the consensus definition. + +This definition is how the blockchain stores it, hence why a lot of the declarative properties relate to RLP encoding. +In the context of reth, the consensus definition is also adapted into the RPC API representation. Therefore, it is also +being JSON encoded. And lastly, it is being stored in database, it uses `Compact` encoding, which is a custom reth +database encoding. This one needs to be implemented manually, but can reuse a lot of existing functionality. More on +that later on in this chapter. + +Here is our top level consensus transaction envelope declaration: + +```rust +use alloy_consensus::{Signed, TransactionEnvelope}; +use op_alloy_consensus::OpTxEnvelope; + +/// Either [`OpTxEnvelope`] or [`TxPayment`]. +#[derive(Debug, Clone, TransactionEnvelope)] +#[envelope(tx_type_name = TxTypeCustom)] +pub enum CustomTransaction { + /// A regular Optimism transaction as defined by [`OpTxEnvelope`]. + #[envelope(flatten)] + Op(OpTxEnvelope), + /// A [`TxPayment`] tagged with type 0x7E. + #[envelope(ty = 42)] + Payment(Signed), +} +``` + +Few things to note here, let's start from up top. We added: +* `derive(TransactionEnvelope)` which generates a lot of derivable trait implementations for transaction envelope types. +* `derive(Debug, Clone)` which are necessary for this macro to work. +* The `envelope` attribute with parameter `tx_type_name` generates an enum with a given name, in our case `TxTypeCustom`. This enum contains a *flattened* list of transaction variants. +* The enum `CustomTransaction` is declared hence it remains a crate-local type, allowing us to implement methods and foreign traits for it, which is very useful. +* The enum has two variants: + * `Op` the base regular Optimism transaction envelope. + * `Payment` the custom extension we added. +* We can add more custom extensions if we need to by extending this enum with another variant. +* Both variants have `envelope` attribute telling the macro how to process them +* The `flatten` parameter tells the macro to adapt all the variants of the wrapped type as the variants of this transaction. +This affects the serialization of the transaction, making so that on the outside all the `OpTxEnvelope` encoded types can be deserialized into this one. +It only works with already existing transaction envelope types. +* The `ty` parameter sets the transaction numerical identifier for the serialization. It identifies the transaction +variant during deserialization. It's important that this number fits into one byte and does not collide with +identifier of any other transaction variant. + +The `TxPayment` is our custom transaction representation. In our example, it is meant only for money transfers, not for +smart contract interaction. Therefore, it needs fewer parameters and is smaller to encode. + +```rust +#[derive( + Clone, + Debug, + Default, + PartialEq, + Eq, + Hash, + serde::Serialize, + serde::Deserialize, + reth_codecs::Compact, +)] +#[serde(rename_all = "camelCase")] +pub struct TxPayment { + /// EIP-155: Simple replay attack protection + #[serde(with = "alloy_serde::quantity")] + pub chain_id: ChainId, + /// A scalar value equal to the number of transactions sent by the sender; formally Tn. + #[serde(with = "alloy_serde::quantity")] + pub nonce: u64, + /// A scalar value equal to the maximum + /// amount of gas that should be used in executing + /// this transaction. This is paid up-front, before any + /// computation is done and may not be increased + /// later; formally Tg. + #[serde(with = "alloy_serde::quantity", rename = "gas", alias = "gasLimit")] + pub gas_limit: u64, + /// A scalar value equal to the maximum + /// amount of gas that should be used in executing + /// this transaction. This is paid up-front, before any + /// computation is done and may not be increased + /// later; formally Tg. + /// + /// As ethereum circulation is around 120mil eth as of 2022 that is around + /// 120000000000000000000000000 wei we are safe to use u128 as its max number is: + /// 340282366920938463463374607431768211455 + /// + /// This is also known as `GasFeeCap` + #[serde(with = "alloy_serde::quantity")] + pub max_fee_per_gas: u128, + /// Max Priority fee that transaction is paying + /// + /// As ethereum circulation is around 120mil eth as of 2022 that is around + /// 120000000000000000000000000 wei we are safe to use u128 as its max number is: + /// 340282366920938463463374607431768211455 + /// + /// This is also known as `GasTipCap` + #[serde(with = "alloy_serde::quantity")] + pub max_priority_fee_per_gas: u128, + /// The 160-bit address of the message call’s recipient. + pub to: Address, + /// A scalar value equal to the number of Wei to + /// be transferred to the message call’s recipient or, + /// in the case of contract creation, as an endowment + /// to the newly created account; formally Tv. + pub value: U256, +} +``` + +On top of our declaration, there are several traits derivations from the standard library. For our purposes, it is +enough to know that these are expected by Reth. + +Due to it being serialized in JSON, it needs to be `serde` compatible. The struct is annotated with +the`#[serde(rename_all = "camelCase")]` attribute that assumes JSON keys formatting in the serialized representation. + +A custom Reth derive macro is used here to generate a `Compact` implementation for us. As mentioned earlier, this +encoding is used for database storage. + +## Pooled + +Another important representation is the mempool one. This declaration should be made to contain any transaction that +users can submit into the node's mempool. + +Here is the declaration: + +```rust +use alloy_consensus::{Signed, TransactionEnvelope}; +use op_alloy_consensus::OpPooledTransaction; + +#[derive(Clone, Debug, TransactionEnvelope)] +#[envelope(tx_type_name = CustomPooledTxType)] +pub enum CustomPooledTransaction { + /// A regular Optimism transaction as defined by [`OpPooledTransaction`]. + #[envelope(flatten)] + Op(OpPooledTransaction), + /// A [`TxPayment`] tagged with type 0x7E. + #[envelope(ty = 42)] + Payment(Signed), +} +``` + +As you can see it is almost the same as the consensus one. The main difference is the use of `OpPooledTransaction` +as the base. This one does not contain the deposit transactions. In Optimism, deposits don't go into the mempool, +because they are not user-submitted, but rather received from the engine API that only the sequencer can use. + +## Manual trait implementations + +There are more traits to be implemented a lot of them are `reth` specific and due to the macro being defined in `alloy`, +it cannot provide these implementations automatically. + +We'll dissect the several kinds of trait implementations you may encounter. To see the complete list refer to these +source code sections: +* Consensus envelope: https://github.com/paradigmxyz/reth/blob/main/examples/custom-node/src/primitives/tx.rs#L29-L140 +* Pooled envelope: https://github.com/paradigmxyz/reth/blob/main/examples/custom-node/src/pool.rs#L23-L89 +* Transaction: https://github.com/paradigmxyz/reth/blob/main/examples/custom-node/src/primitives/tx_custom.rs#L71-L288 + +Most of these implementations simply match the envelope enum variant and then delegate the responsibility to each variant, for example: + +```rust +impl InMemorySize for CustomTransaction { + fn size(&self) -> usize { + match self { + CustomTransaction::Op(tx) => InMemorySize::size(tx), + CustomTransaction::Payment(tx) => InMemorySize::size(tx), + } + } +} +``` + +Some of these implementations are trivial, for example: + +```rust +impl OpTransaction for CustomTransaction { + fn is_deposit(&self) -> bool { + match self { + CustomTransaction::Op(op) => op.is_deposit(), + CustomTransaction::Payment(_) => false, + } + } + + fn as_deposit(&self) -> Option<&Sealed> { + match self { + CustomTransaction::Op(op) => op.as_deposit(), + CustomTransaction::Payment(_) => None, + } + } +} +``` + +This one is simply saying that the custom transaction variant is not an optimism deposit. + +A few of these trait implementations are a marker trait with no body like so: + +```rust +impl RlpBincode for CustomTransaction {} +``` + +Sometimes the fact that `CustomTransactionEnvelope` is a wrapper type means that it needs to reimplement some traits that +it's `inner` field already implements, like so: + +```rust +impl SignedTransaction for CustomTransaction { + fn tx_hash(&self) -> &B256 { + match self { + CustomTransaction::Op(tx) => SignedTransaction::tx_hash(tx), + CustomTransaction::Payment(tx) => tx.hash(), + } + } +} +``` + +The `Compact` support is largely derived and abstracted away with the help of a few minimal trivial implementations. One +slightly interesting case is `FromTxCompact`. The actual decoding is delegated to `TxPayment`, where it is macro generated. +Then it is put together with the `signature`, that given as one of the function arguments, via `Signed::new_unhashed` +which creates the `Signed` instance with no signature validation. + +Since the signature validation is done before encoding it, it would be redundant and infallible, but still need a +`Result` type or an `unwrap`. + +```rust +impl FromTxCompact for CustomTransaction { + type TxType = TxTypeCustom; + + fn from_tx_compact(buf: &[u8], tx_type: Self::TxType, signature: Signature) -> (Self, &[u8]) + where + Self: Sized, + { + match tx_type { + TxTypeCustom::Op(tx_type) => { + let (tx, buf) = OpTxEnvelope::from_tx_compact(buf, tx_type, signature); + (Self::Op(tx), buf) + } + TxTypeCustom::Payment => { + let (tx, buf) = TxPayment::from_compact(buf, buf.len()); + let tx = Signed::new_unhashed(tx, signature); + (Self::Payment(tx), buf) + } + } + } +} +``` + +# Conclusion + +We have declared our own transaction representation that is ready to be used with Reth! We have also declared our own +transaction envelope that contains either our custom representation or any other Optimism type. This means that it is +fully compatible with Optimism while also supporting the payment transaction, capable of being filling the role of an +execution client that belongs to an Op stack. + +# Where to go next + +Our work is not finished! What follows is to: +* Configure the node to use the custom type +* Implement components that work with transaction variants diff --git a/docs/vocs/docs/pages/sdk/examples/modify-node.mdx b/docs/vocs/docs/pages/sdk/examples/modify-node.mdx new file mode 100644 index 00000000000..b5297504f3a --- /dev/null +++ b/docs/vocs/docs/pages/sdk/examples/modify-node.mdx @@ -0,0 +1,85 @@ +# How to Modify an Existing Node + +This guide demonstrates how to extend a Reth node with custom functionality, including adding RPC endpoints, modifying transaction validation, and implementing custom services. + +## Adding Custom RPC Endpoints + +One of the most common modifications is adding custom RPC methods to expose additional functionality. This allows you to extend the standard Ethereum RPC API with your own methods while maintaining compatibility with existing tools and clients. + +### Basic Custom RPC Module + +The following example shows how to add a custom RPC namespace called `txpoolExt` that provides additional transaction pool functionality. This example is based on the `node-custom-rpc` example in the Reth repository. + +#### Project Structure + +First, create a new binary crate with the following dependencies in your `Cargo.toml`: + +```toml +[package] +name = "node-custom-rpc" +version = "0.1.0" +edition = "2021" + +[[bin]] +name = "node-custom-rpc" +path = "src/main.rs" + +[dependencies] +clap = { version = "4.0", features = ["derive"] } +jsonrpsee = { version = "0.22", features = ["macros", "server", "http-server", "ws-server"] } +reth-ethereum = { path = "../../crates/ethereum" } +tokio = { version = "1.0", features = ["full"] } +``` + +#### Implementation + +The complete implementation can be found in the [node-custom-rpc example](https://github.com/paradigmxyz/reth/tree/main/examples/node-custom-rpc). Here's a summary of the key components: + +1. **RPC Interface**: Define your custom RPC methods using `jsonrpsee` proc macros with a custom namespace +2. **RPC Handler**: Implement the trait with access to node components like the transaction pool +3. **CLI Extension**: Add custom CLI arguments to control your extensions +4. **Node Integration**: Use `extend_rpc_modules` to integrate your custom functionality + +#### Running the Custom Node + +Build and run your custom node with the extension enabled: + +```bash +cargo run -p node-custom-rpc -- node --http --ws --enable-ext +``` + +This will start a Reth node with your custom RPC methods available on both HTTP and WebSocket transports. + +#### Testing the Custom RPC Methods + +You can test your custom RPC methods using tools like `cast` from the Foundry suite: + +```bash +# Get transaction count +cast rpc txpoolExt_transactionCount + +# Clear the transaction pool +cast rpc txpoolExt_clearTxpool + +# Subscribe to transaction count updates (WebSocket only) +cast rpc txpoolExt_subscribeTransactionCount +``` + +### Key Concepts + +1. **RPC Namespaces**: Use the `namespace` parameter in the `rpc` macro to create a custom namespace for your methods. + +2. **Node Context**: Access node components like the transaction pool through the `ctx` parameter in `extend_rpc_modules`. + +3. **Transport Integration**: Your custom RPC methods are automatically available on all configured transports (HTTP, WebSocket, IPC). + +4. **CLI Integration**: Extend the default Reth CLI with your own arguments to control custom functionality. + +5. **Error Handling**: Use `RpcResult` for methods that can fail and handle errors appropriately. + +## Next Steps + +- Explore [Standalone Components](/sdk/examples/standalone-components) for direct blockchain interaction +- Learn about [Custom Node Building](/sdk/custom-node/prerequisites) for production deployments +- Review [Type System](/sdk/typesystem/block) for working with blockchain data +- Check out the [node-custom-rpc example](https://github.com/paradigmxyz/reth/tree/main/examples/node-custom-rpc) for the complete implementation diff --git a/docs/vocs/docs/pages/sdk/examples/standalone-components.mdx b/docs/vocs/docs/pages/sdk/examples/standalone-components.mdx new file mode 100644 index 00000000000..9093858c6c2 --- /dev/null +++ b/docs/vocs/docs/pages/sdk/examples/standalone-components.mdx @@ -0,0 +1,92 @@ +# Using Standalone Components + +This guide demonstrates how to use Reth components independently without running a full node. This is useful for building tools, analyzers, indexers, or any application that needs direct access to blockchain data. + +## Direct Database Access + +Reth uses MDBX as its primary database backend, storing blockchain data in a structured format. You can access this database directly from external processes for read-only operations, which is useful for analytics, indexing, or building custom tools. + +### Understanding the Database Architecture + +Reth's storage architecture consists of two main components: + +1. **MDBX Database**: Primary storage for blockchain state, headers, bodies, receipts, and indices +2. **Static Files**: Immutable historical data (headers, bodies, receipts, transactions) stored in compressed files for better performance + +Both components must be accessed together for complete data access. + +### Database Location + +The database is stored in the node's data directory: +- **Default location**: `$HOME/.local/share/reth/mainnet/db` (Linux/macOS) or `%APPDATA%\reth\mainnet\db` (Windows) +- **Custom location**: Set with `--datadir` flag when running reth +- **Static files**: Located in `/static_files` subdirectory + +### Opening the Database from External Processes + +When accessing the database while a node is running, you **must** open it in read-only mode to prevent corruption and conflicts. + +#### Using the High-Level API + +The safest way to access the database is through Reth's provider factory: + +```rust +use reth_ethereum::node::EthereumNode; +use reth_ethereum::chainspec::MAINNET; + +// Open with automatic configuration +let factory = EthereumNode::provider_factory_builder() + .open_read_only(MAINNET.clone(), "path/to/datadir")?; + +// Get a provider for queries +let provider = factory.provider()?; +let latest_block = provider.last_block_number()?; +``` + +### Performance Implications + +External reads while the node is syncing or processing blocks: + +- **I/O Competition**: May compete with the node for disk I/O +- **Cache Pollution**: Can evict hot data from OS page cache +- **CPU Impact**: Complex queries can impact node performance + +### Important Considerations + +1. **Read-Only Access Only**: Never open the database in write mode while the regular reth process is running. + +2. **Consistency**: When reading from an external process: + - Data may be slightly behind the latest processed block (if it hasn't been written to disk yet) + - Use transactions for consistent views across multiple reads + - Be aware of potential reorgs affecting recent blocks + +3. **Performance**: + - MDBX uses memory-mapped files for efficient access + - Multiple readers don't block each other + - Consider caching frequently accessed data + +### Disabling long-lived read transactions: + +By default long lived read transactions are terminated after a few minutes, this is because long read transaction can cause the free list to grow if changes to the database are made (reth node is running). +To opt out of this, this safety mechanism can be disabled: + +```rust +let factory = EthereumNode::provider_factory_builder() + .open_read_only(MAINNET.clone(), ReadOnlyConfig::from_datadir("datadir").disable_long_read_transaction_safety())?; +``` + +### Real-time Block Access Configuration + +Reth buffers new blocks in memory before persisting them to disk for performance optimization. If your external process needs immediate access to the latest blocks, configure the node to persist blocks immediately: + +- `--engine.persistence-threshold 0` - Persists new canonical blocks to disk immediately + +Using this flag ensures external processes can read new blocks without delay. + +As soon as the reth process has persisted the block data, the external reader can read it from the database. + +## Next Steps + +- Learn about [Modifying Nodes](/sdk/examples/modify-node) to add functionality +- Explore the [Type System](/sdk/typesystem/block) for working with data +- Check [Custom Node Building](/sdk/custom-node/prerequisites) for production use diff --git a/docs/vocs/docs/pages/sdk/node-components.mdx b/docs/vocs/docs/pages/sdk/node-components.mdx new file mode 100644 index 00000000000..d569d499dd9 --- /dev/null +++ b/docs/vocs/docs/pages/sdk/node-components.mdx @@ -0,0 +1,112 @@ +# Node Components + +Reth's modular architecture allows developers to customize and extend individual components of the node. Each component serves a specific purpose and can be replaced or modified to suit your needs. + +## Architecture Overview + +A Reth node consists of several key components that work together and can interact with each other: + +```mermaid +graph LR + Network[Network] --> Pool[Transaction Pool] + Network --> Consensus[Consensus] + Pool --> DB[(Database)] + Consensus --> EVM + EVM --> DB[(Database)] + RPC[RPC Server] --> Pool + RPC --> DB + RPC --> EVM +``` + +## Core Components + +### [Network](/sdk/node-components/network) +Handles P2P communication, peer discovery, and block/transaction propagation. The network component is responsible for: +- Peer discovery and management +- Transaction gossip +- State synchronization (downloading blocks) +- Protocol message handling + +### [Transaction Pool](/sdk/node-components/pool) +Manages pending transactions before they're included in blocks: +- Transaction validation +- Ordering and prioritization +- Transaction replacement logic +- Pool size management and eviction + +### [Consensus](/sdk/node-components/consensus) +Validates blocks according to protocol rules: +- Header validation (e.g. gas limit, base fee) +- Block body validation (e.g. transaction root) + +### [EVM](/sdk/node-components/evm) +Executes transactions and manages state transitions: +- Block execution +- Transaction execution +- Block building + +### [RPC](/sdk/node-components/rpc) +Provides external API access to the node: +- Standard Ethereum JSON-RPC methods +- Custom endpoints +- WebSocket subscriptions + +## Component Customization + +Each component can be customized through Reth's builder pattern: + +```rust +use reth_ethereum::node::{EthereumNode, NodeBuilder}; + +let node = NodeBuilder::new(config) + .with_types::() + .with_components(|ctx| { + // Use the ComponentBuilder to customize components + ctx.components_builder() + // Custom network configuration + .network(|network_builder| { + network_builder + .peer_manager(custom_peer_manager) + .build() + }) + // Custom transaction pool + .pool(|pool_builder| { + pool_builder + .validator(custom_validator) + .ordering(custom_ordering) + .build() + }) + // Custom consensus + .consensus(custom_consensus) + // Custom EVM configuration + .evm(|evm_builder| { + evm_builder + .with_precompiles(custom_precompiles) + .build() + }) + // Build all components + .build() + }) + .build() + .await?; +``` + +## Component Lifecycle + +Components follow a specific lifecycle starting from node builder initialization to shutdown: + +1. **Initialization**: Components are created with their dependencies +2. **Configuration**: Settings and parameters are applied +3. **Startup**: Components begin their main operations +4. **Runtime**: Components process requests and events +5. **Shutdown**: Graceful cleanup and resource release + + +## Next Steps + +Explore each component in detail: +- [Network Component](/sdk/node-components/network) - P2P and synchronization +- [Transaction Pool](/sdk/node-components/pool) - Mempool management +- [Consensus](/sdk/node-components/consensus) - Block validation +- [EVM](/sdk/node-components/evm) - Transaction execution +- [RPC](/sdk/node-components/rpc) - External APIs diff --git a/docs/vocs/docs/pages/sdk/node-components/consensus.mdx b/docs/vocs/docs/pages/sdk/node-components/consensus.mdx new file mode 100644 index 00000000000..1541d351d5f --- /dev/null +++ b/docs/vocs/docs/pages/sdk/node-components/consensus.mdx @@ -0,0 +1,45 @@ +# Consensus Component + +The consensus component validates blocks according to Ethereum protocol rules, handles chain reorganizations, and manages the canonical chain state. + +## Overview + +The consensus component is responsible for: +- Validating block headers and bodies +- Verifying state transitions +- Managing fork choice rules +- Handling chain reorganizations +- Tracking finalized and safe blocks +- Validating blob transactions (EIP-4844) + +## Key Concepts + +### Block Validation +The consensus component performs multiple validation steps: +1. **Pre-execution validation**: Header and body checks before running transactions +2. **Post-execution validation**: State root and receipts verification after execution + +### Header Validation +Headers must pass several checks: +- **Timestamp**: Must be greater than parent's timestamp +- **Gas limit**: Changes must be within protocol limits (1/1024 of parent) +- **Extra data**: Size restrictions based on network rules +- **Difficulty/PoS**: Appropriate validation for pre/post-merge + +### Body Validation +Block bodies are validated against headers: +- **Transaction root**: Merkle root must match header +- **Withdrawals root**: For post-Shanghai blocks +- **Blob validation**: For EIP-4844 transactions + +### Fork Choice +The consensus engine determines the canonical chain: +- Tracks multiple chain branches +- Applies fork choice rules (longest chain, most work, etc.) +- Handles reorganizations when better chains are found + +## Next Steps + +- Explore [EVM](/sdk/node-components/evm) execution +- Learn about [RPC](/sdk/node-components/rpc) server integration +- Understand [Transaction Pool](/sdk/node-components/pool) interaction \ No newline at end of file diff --git a/docs/vocs/docs/pages/sdk/node-components/evm.mdx b/docs/vocs/docs/pages/sdk/node-components/evm.mdx new file mode 100644 index 00000000000..1460f8938f4 --- /dev/null +++ b/docs/vocs/docs/pages/sdk/node-components/evm.mdx @@ -0,0 +1,45 @@ +# EVM Component + +The EVM (Ethereum Virtual Machine) component handles transaction execution and state transitions. It's responsible for processing transactions and updating the blockchain state. + +## Overview + +The EVM component manages: +- Transaction execution +- State transitions and updates +- Gas calculation and metering +- Custom precompiles and opcodes +- Block execution and validation +- State management and caching + +## Architecture + + +## Key Concepts + +### Transaction Execution +The EVM executes transactions in a deterministic way: +1. **Environment Setup**: Configure block and transaction context +2. **State Access**: Load accounts and storage from the database +3. **Execution**: Run EVM bytecode with gas metering +4. **State Updates**: Apply changes to accounts and storage +5. **Receipt Generation**: Create execution receipts with logs + +### Block Execution +Block executors process all transactions in a block: +- Validate pre-state conditions +- Execute transactions sequentially +- Apply block rewards +- Verify post-state (state root, receipts root) + +### Block Building +Block builders construct new blocks for proposal: +- Select transactions (e.g. mempool) +- Order and execute transactions +- Seal the block with a header (state root) + +## Next Steps + +- Learn about [RPC](/sdk/node-components/rpc) server integration +- Explore [Transaction Pool](/sdk/node-components/pool) interaction +- Review [Consensus](/sdk/node-components/consensus) validation diff --git a/docs/vocs/docs/pages/sdk/node-components/network.mdx b/docs/vocs/docs/pages/sdk/node-components/network.mdx new file mode 100644 index 00000000000..f9af6f5ddc0 --- /dev/null +++ b/docs/vocs/docs/pages/sdk/node-components/network.mdx @@ -0,0 +1,55 @@ +# Network Component + +The network component handles all peer-to-peer communication in Reth, including peer discovery, connection management, and protocol message handling. + +## Overview + +The network stack implements the Ethereum Wire Protocol (ETH) and provides: +- Peer discovery via discv4 and discv5 +- Connection management with configurable peer limits +- Transaction propagation +- State synchronization +- Request/response protocols (e.g. GetBlockHeaders, GetBodies) + +## Architecture + +```mermaid +graph TD + NetworkManager[Network Manager] --> Discovery[Discovery] + NetworkManager --> Sessions[Session Manager] + NetworkManager --> Swarm[Swarm] + + Discovery --> discv4[discv4] + Discovery --> discv5[discv5] + Discovery --> DNS[DNS Discovery] + + Sessions --> ETH[ETH Protocol] +``` + +## Key Concepts + +### Peer Discovery +The network uses multiple discovery mechanisms to find and connect to peers: +- **discv4**: UDP-based discovery protocol for finding peers +- **discv5**: Improved discovery protocol with better security +- **DNS Discovery**: Peer lists published via DNS for bootstrap + +### Connection Management +- Maintains separate limits for inbound and outbound connections +- Implements peer scoring and reputation tracking +- Handles connection lifecycle and graceful disconnections + +### Protocol Support +- **ETH Protocol**: Core Ethereum wire protocol for blocks and transactions + +### Message Broadcasting +The network efficiently propagates new blocks and transactions to peers using: +- Transaction pooling and deduplication +- Block announcement strategies +- Bandwidth management + +## Next Steps + +- Learn about the [Transaction Pool](/sdk/node-components/pool) +- Understand [Consensus](/sdk/node-components/consensus) integration +- Explore [RPC](/sdk/node-components/rpc) server setup \ No newline at end of file diff --git a/docs/vocs/docs/pages/sdk/node-components/pool.mdx b/docs/vocs/docs/pages/sdk/node-components/pool.mdx new file mode 100644 index 00000000000..301d794b3fd --- /dev/null +++ b/docs/vocs/docs/pages/sdk/node-components/pool.mdx @@ -0,0 +1,80 @@ +# Transaction Pool Component + +The transaction pool (mempool) manages pending transactions before they are included in blocks. It handles validation, ordering, replacement, and eviction of transactions. + +## Overview + +The transaction pool is responsible for: +- Validating incoming transactions +- Maintaining transaction ordering (e.g. by fees) +- Handling transaction replacement +- Managing pool size limits +- Broadcasting transactions to peers +- Providing transactions for block building + +## Architecture + +```mermaid +graph TD + API[Pool API] --> Validator[Transaction Validator] + API --> Pool[Transaction Pool] + + Pool --> SubPools[Sub-Pools] + SubPools --> Pending[Pending Pool] + SubPools --> Queued[Queued Pool] + SubPools --> Base[Base Fee Pool] + + Pool --> Ordering[Transaction Ordering] + Pool --> Listeners[Event Listeners] + + Validator --> Checks[Validation Checks] + Checks --> Nonce[Nonce Check] + Checks --> Balance[Balance Check] +``` + +## Key Concepts + +### Transaction Validation +The pool validates transactions before accepting them, checking: +- Sender has sufficient balance for gas and value +- Nonce is correct (either next expected or future) +- Gas price meets minimum requirements +- Transaction size is within limits +- Signature is valid + +### Transaction Ordering +Transactions are ordered by their effective tip per gas to maximize block rewards. Custom ordering strategies can prioritize certain addresses or implement MEV protection. + +### Sub-Pools +- **Pending**: Transactions ready for inclusion (correct nonce) +- **Queued**: Future transactions (nonce gap exists) +- **Base Fee**: Transactions priced below current base fee + +### Pool Maintenance +The pool requires periodic maintenance to: +- Remove stale transactions +- Revalidate after chain reorganizations +- Update base fee thresholds +- Enforce size limits + +## Advanced Features + +### Blob Transaction Support +EIP-4844 introduces blob transactions with separate blob storage and special validation rules. + +### Transaction Filters +Custom filters can block specific addresses, limit gas prices, or implement custom acceptance criteria. + +### Event System +The pool supports an event system that allows other components to listen for transaction lifecycle events such as: +- Transaction added +- Transaction removed +- Transaction replaced +- Transaction promoted to pending state + + +## Next Steps + +- Learn about [Consensus](/sdk/node-components/consensus) validation +- Explore [EVM](/sdk/node-components/evm) execution +- Understand [RPC](/sdk/node-components/rpc) server integration \ No newline at end of file diff --git a/docs/vocs/docs/pages/sdk/node-components/rpc.mdx b/docs/vocs/docs/pages/sdk/node-components/rpc.mdx new file mode 100644 index 00000000000..4f9fa1e3d7b --- /dev/null +++ b/docs/vocs/docs/pages/sdk/node-components/rpc.mdx @@ -0,0 +1,20 @@ +# RPC Component + +The RPC component provides external API access to the node, implementing the Ethereum JSON-RPC specification and allowing custom extensions. + +## Overview + +The RPC component provides: +- Standard Ethereum JSON-RPC methods +- WebSocket subscriptions +- Custom method extensions +- Rate limiting and access control +- Request batching support +- Multiple transport protocols (HTTP, WebSocket, IPC) + + +## Next Steps + +- Explore [Network](/sdk/node-components/network) component integration +- Learn about [Transaction Pool](/sdk/node-components/pool) APIs +- Understand [EVM](/sdk/node-components/evm) execution context \ No newline at end of file diff --git a/docs/vocs/docs/pages/sdk/typesystem/block.mdx b/docs/vocs/docs/pages/sdk/typesystem/block.mdx new file mode 100644 index 00000000000..450b4f93d1a --- /dev/null +++ b/docs/vocs/docs/pages/sdk/typesystem/block.mdx @@ -0,0 +1,26 @@ +# Block Types + +The Reth type system provides a flexible abstraction for blocks through traits, allowing different implementations while maintaining type safety and consistency. + +## Type Relationships + +```mermaid +graph TD + Block[Block Trait] --> Header[BlockHeader Trait] + Block --> Body[BlockBody Trait] + + SealedBlock -.-> Block + SealedBlock --> SealedHeader + RecoveredBlock --> SealedBlock + + SealedHeader --> Header + + Body --> Transaction[Transactions] + Body --> Withdrawals[Withdrawals] +``` + +## Next Steps + +- Learn about [Transaction Types](/sdk/typesystem/transaction-types) +- Understand [Consensus](/sdk/node-components/consensus) validation +- Explore [EVM](/sdk/node-components/evm) execution diff --git a/docs/vocs/docs/pages/sdk/typesystem/transaction-types.mdx b/docs/vocs/docs/pages/sdk/typesystem/transaction-types.mdx new file mode 100644 index 00000000000..e541727da87 --- /dev/null +++ b/docs/vocs/docs/pages/sdk/typesystem/transaction-types.mdx @@ -0,0 +1,92 @@ +# Transaction Types and Representations + +Reth provides multiple transaction representations optimized for different stages of the transaction lifecycle. Understanding these types is crucial for working with the node's transaction handling pipeline. + +## Transaction Lifecycle + +Transactions go through several stages, each with its own optimized representation: + +```mermaid +graph LR + RPC[RPC Transaction] --> Pool[Pooled Transaction] + Pool --> Consensus[Consensus Transaction] + Consensus --> Executed[Executed Transaction] + + Pool -.-> RPC + Consensus -.-> Pool +``` + +## Transaction Representations + +### RPC Transaction + +The RPC representation is designed for JSON-RPC communication with external clients. It uses JSON-compatible types and includes all information clients need to understand transaction status. + +Key characteristics: +- **JSON-compatible types**: Uses U256 for numbers, hex strings for binary data +- **Optional fields**: Supports both legacy and EIP-1559 transactions with appropriate fields +- **Block context**: Includes block hash, number, and index when transaction is mined +- **Human-readable**: Optimized for external consumption and debugging +- **Complete information**: Contains all transaction details including signature components + +Use cases: +- Sending transactions via `eth_sendTransaction` +- Querying transaction details via `eth_getTransactionByHash` +- Transaction receipts and history +- Block explorer displays + +### Pooled Transaction + +The pooled representation is optimized for mempool storage and validation. It pre-computes expensive values and includes additional data needed for pool management. + +Key characteristics: +- **Cached values**: Pre-computed sender address and transaction cost to avoid repeated calculations +- **Validation ready**: Includes all data needed for quick pool validation +- **Blob support**: Handles EIP-4844 blob sidecars separately from the core transaction +- **Memory efficient**: Optimized structure for storing thousands of pending transactions +- **Priority ordering**: Structured for efficient sorting by gas price/priority fee + +Use cases: +- Transaction pool storage and management +- Gas price ordering and replacement logic +- Validation against account state +- Broadcasting to peers + +### Consensus Transaction + +The consensus representation is the canonical format used in blocks and for network propagation. It's the most compact representation and follows Ethereum's wire protocol. + +Key characteristics: +- **Type safety**: Enum variants for different transaction types (Legacy, EIP-2930, EIP-1559, EIP-4844) +- **Compact encoding**: For storage on disk +- **No redundancy**: Minimal data, with values like sender recovered from signature when needed + +Use cases: +- Block construction and validation +- Network propagation between nodes +- Persistent storage in the database +- State transition execution + +## Representation Conversions + +### RPC → Pooled +When transactions arrive via RPC: +1. Validate JSON format and fields +2. Convert to consensus format +3. Recover sender from signature +4. Create pooled representation + +### Pooled → Consensus +When including in a block: +1. Extract core transaction consensus data +2. Remove cached values (sender, cost) + +### Consensus → RPC +When serving RPC requests: +1. Add block context (hash, number, index) + +## Next Steps + +- Learn about [Block Types](/sdk/typesystem/block) and how transactions fit in blocks +- Understand [Transaction Pool](/sdk/node-components/pool) management +- Explore [EVM](/sdk/node-components/evm) transaction execution \ No newline at end of file diff --git a/docs/vocs/docs/public/alchemy.png b/docs/vocs/docs/public/alchemy.png new file mode 100644 index 00000000000..422feb03277 Binary files /dev/null and b/docs/vocs/docs/public/alchemy.png differ diff --git a/docs/vocs/docs/public/coinbase.png b/docs/vocs/docs/public/coinbase.png new file mode 100644 index 00000000000..2e71f9ec3a1 Binary files /dev/null and b/docs/vocs/docs/public/coinbase.png differ diff --git a/docs/vocs/docs/public/flashbots.png b/docs/vocs/docs/public/flashbots.png new file mode 100644 index 00000000000..1a4622becd2 Binary files /dev/null and b/docs/vocs/docs/public/flashbots.png differ diff --git a/docs/vocs/docs/public/logo.png b/docs/vocs/docs/public/logo.png new file mode 100644 index 00000000000..fa113d2a674 Binary files /dev/null and b/docs/vocs/docs/public/logo.png differ diff --git a/book/developers/exex/assets/remote_exex.png b/docs/vocs/docs/public/remote_exex.png similarity index 100% rename from book/developers/exex/assets/remote_exex.png rename to docs/vocs/docs/public/remote_exex.png diff --git a/docs/vocs/docs/public/reth-prod.png b/docs/vocs/docs/public/reth-prod.png new file mode 100644 index 00000000000..5b31a569a36 Binary files /dev/null and b/docs/vocs/docs/public/reth-prod.png differ diff --git a/docs/vocs/docs/public/succinct.png b/docs/vocs/docs/public/succinct.png new file mode 100644 index 00000000000..1261974aa8a Binary files /dev/null and b/docs/vocs/docs/public/succinct.png differ diff --git a/docs/vocs/docs/snippets/sources/Cargo.toml b/docs/vocs/docs/snippets/sources/Cargo.toml new file mode 100644 index 00000000000..245734ce83a --- /dev/null +++ b/docs/vocs/docs/snippets/sources/Cargo.toml @@ -0,0 +1,42 @@ +[workspace] +members = ["exex/hello-world", "exex/remote", "exex/tracking-state"] + +# Explicitly set the resolver to version 2, which is the default for packages with edition >= 2021 +# https://doc.rust-lang.org/edition-guide/rust-2021/default-cargo-resolver.html +resolver = "2" + +[patch.'https://github.com/paradigmxyz/reth'] +reth = { path = "../../bin/reth" } +reth-exex = { path = "../../crates/exex/exex" } +reth-node-ethereum = { path = "../../crates/ethereum/node" } +reth-tracing = { path = "../../crates/tracing" } +reth-node-api = { path = "../../crates/node/api" } + +[patch.crates-io] +alloy-consensus = { git = "https://github.com/alloy-rs/alloy", rev = "08fa016ed950b6e65f810fc9cdef7cf38fbc63f6" } +alloy-contract = { git = "https://github.com/alloy-rs/alloy", rev = "08fa016ed950b6e65f810fc9cdef7cf38fbc63f6" } +alloy-eips = { git = "https://github.com/alloy-rs/alloy", rev = "08fa016ed950b6e65f810fc9cdef7cf38fbc63f6" } +alloy-genesis = { git = "https://github.com/alloy-rs/alloy", rev = "08fa016ed950b6e65f810fc9cdef7cf38fbc63f6" } +alloy-json-rpc = { git = "https://github.com/alloy-rs/alloy", rev = "08fa016ed950b6e65f810fc9cdef7cf38fbc63f6" } +alloy-network = { git = "https://github.com/alloy-rs/alloy", rev = "08fa016ed950b6e65f810fc9cdef7cf38fbc63f6" } +alloy-network-primitives = { git = "https://github.com/alloy-rs/alloy", rev = "08fa016ed950b6e65f810fc9cdef7cf38fbc63f6" } +alloy-provider = { git = "https://github.com/alloy-rs/alloy", rev = "08fa016ed950b6e65f810fc9cdef7cf38fbc63f6" } +alloy-pubsub = { git = "https://github.com/alloy-rs/alloy", rev = "08fa016ed950b6e65f810fc9cdef7cf38fbc63f6" } +alloy-rpc-client = { git = "https://github.com/alloy-rs/alloy", rev = "08fa016ed950b6e65f810fc9cdef7cf38fbc63f6" } +alloy-rpc-types = { git = "https://github.com/alloy-rs/alloy", rev = "08fa016ed950b6e65f810fc9cdef7cf38fbc63f6" } +alloy-rpc-types-admin = { git = "https://github.com/alloy-rs/alloy", rev = "08fa016ed950b6e65f810fc9cdef7cf38fbc63f6" } +alloy-rpc-types-anvil = { git = "https://github.com/alloy-rs/alloy", rev = "08fa016ed950b6e65f810fc9cdef7cf38fbc63f6" } +alloy-rpc-types-beacon = { git = "https://github.com/alloy-rs/alloy", rev = "08fa016ed950b6e65f810fc9cdef7cf38fbc63f6" } +alloy-rpc-types-debug = { git = "https://github.com/alloy-rs/alloy", rev = "08fa016ed950b6e65f810fc9cdef7cf38fbc63f6" } +alloy-rpc-types-engine = { git = "https://github.com/alloy-rs/alloy", rev = "08fa016ed950b6e65f810fc9cdef7cf38fbc63f6" } +alloy-rpc-types-eth = { git = "https://github.com/alloy-rs/alloy", rev = "08fa016ed950b6e65f810fc9cdef7cf38fbc63f6" } +alloy-rpc-types-mev = { git = "https://github.com/alloy-rs/alloy", rev = "08fa016ed950b6e65f810fc9cdef7cf38fbc63f6" } +alloy-rpc-types-trace = { git = "https://github.com/alloy-rs/alloy", rev = "08fa016ed950b6e65f810fc9cdef7cf38fbc63f6" } +alloy-rpc-types-txpool = { git = "https://github.com/alloy-rs/alloy", rev = "08fa016ed950b6e65f810fc9cdef7cf38fbc63f6" } +alloy-serde = { git = "https://github.com/alloy-rs/alloy", rev = "08fa016ed950b6e65f810fc9cdef7cf38fbc63f6" } +alloy-signer = { git = "https://github.com/alloy-rs/alloy", rev = "08fa016ed950b6e65f810fc9cdef7cf38fbc63f6" } +alloy-signer-local = { git = "https://github.com/alloy-rs/alloy", rev = "08fa016ed950b6e65f810fc9cdef7cf38fbc63f6" } +alloy-transport = { git = "https://github.com/alloy-rs/alloy", rev = "08fa016ed950b6e65f810fc9cdef7cf38fbc63f6" } +alloy-transport-http = { git = "https://github.com/alloy-rs/alloy", rev = "08fa016ed950b6e65f810fc9cdef7cf38fbc63f6" } +alloy-transport-ipc = { git = "https://github.com/alloy-rs/alloy", rev = "08fa016ed950b6e65f810fc9cdef7cf38fbc63f6" } +alloy-transport-ws = { git = "https://github.com/alloy-rs/alloy", rev = "08fa016ed950b6e65f810fc9cdef7cf38fbc63f6" } diff --git a/book/sources/exex/hello-world/Cargo.toml b/docs/vocs/docs/snippets/sources/exex/hello-world/Cargo.toml similarity index 96% rename from book/sources/exex/hello-world/Cargo.toml rename to docs/vocs/docs/snippets/sources/exex/hello-world/Cargo.toml index e5d32a14054..d3438032ec3 100644 --- a/book/sources/exex/hello-world/Cargo.toml +++ b/docs/vocs/docs/snippets/sources/exex/hello-world/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "my-exex" version = "0.1.0" -edition = "2021" +edition = "2024" [dependencies] reth = { git = "https://github.com/paradigmxyz/reth.git" } # Reth diff --git a/book/sources/exex/hello-world/src/bin/1.rs b/docs/vocs/docs/snippets/sources/exex/hello-world/src/bin/1.rs similarity index 87% rename from book/sources/exex/hello-world/src/bin/1.rs rename to docs/vocs/docs/snippets/sources/exex/hello-world/src/bin/1.rs index e1e46b42c31..58abe0ab1ea 100644 --- a/book/sources/exex/hello-world/src/bin/1.rs +++ b/docs/vocs/docs/snippets/sources/exex/hello-world/src/bin/1.rs @@ -2,7 +2,7 @@ use reth_node_ethereum::EthereumNode; fn main() -> eyre::Result<()> { reth::cli::Cli::parse_args().run(async move |builder, _| { - let handle = builder.node(EthereumNode::default()).launch().await?; + let handle = builder.node(EthereumNode::default()).launch_with_debug_capabilities().await?; handle.wait_for_node_exit().await }) diff --git a/book/sources/exex/hello-world/src/bin/2.rs b/docs/vocs/docs/snippets/sources/exex/hello-world/src/bin/2.rs similarity index 92% rename from book/sources/exex/hello-world/src/bin/2.rs rename to docs/vocs/docs/snippets/sources/exex/hello-world/src/bin/2.rs index cb4289469fa..80ec8484e4f 100644 --- a/book/sources/exex/hello-world/src/bin/2.rs +++ b/docs/vocs/docs/snippets/sources/exex/hello-world/src/bin/2.rs @@ -12,7 +12,7 @@ fn main() -> eyre::Result<()> { let handle = builder .node(EthereumNode::default()) .install_exex("my-exex", async move |ctx| Ok(my_exex(ctx))) - .launch() + .launch_with_debug_capabilities() .await?; handle.wait_for_node_exit().await diff --git a/book/sources/exex/hello-world/src/bin/3.rs b/docs/vocs/docs/snippets/sources/exex/hello-world/src/bin/3.rs similarity index 96% rename from book/sources/exex/hello-world/src/bin/3.rs rename to docs/vocs/docs/snippets/sources/exex/hello-world/src/bin/3.rs index 1a5a2a83884..f9a407b3109 100644 --- a/book/sources/exex/hello-world/src/bin/3.rs +++ b/docs/vocs/docs/snippets/sources/exex/hello-world/src/bin/3.rs @@ -33,7 +33,7 @@ fn main() -> eyre::Result<()> { let handle = builder .node(EthereumNode::default()) .install_exex("my-exex", async move |ctx| Ok(my_exex(ctx))) - .launch() + .launch_with_debug_capabilities() .await?; handle.wait_for_node_exit().await diff --git a/book/sources/exex/remote/Cargo.toml b/docs/vocs/docs/snippets/sources/exex/remote/Cargo.toml similarity index 98% rename from book/sources/exex/remote/Cargo.toml rename to docs/vocs/docs/snippets/sources/exex/remote/Cargo.toml index bbc4fe595cc..4d170be57cb 100644 --- a/book/sources/exex/remote/Cargo.toml +++ b/docs/vocs/docs/snippets/sources/exex/remote/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "remote-exex" version = "0.1.0" -edition = "2021" +edition = "2024" [dependencies] # reth diff --git a/book/sources/exex/remote/build.rs b/docs/vocs/docs/snippets/sources/exex/remote/build.rs similarity index 100% rename from book/sources/exex/remote/build.rs rename to docs/vocs/docs/snippets/sources/exex/remote/build.rs diff --git a/book/sources/exex/remote/proto/exex.proto b/docs/vocs/docs/snippets/sources/exex/remote/proto/exex.proto similarity index 100% rename from book/sources/exex/remote/proto/exex.proto rename to docs/vocs/docs/snippets/sources/exex/remote/proto/exex.proto diff --git a/book/sources/exex/remote/src/consumer.rs b/docs/vocs/docs/snippets/sources/exex/remote/src/consumer.rs similarity index 100% rename from book/sources/exex/remote/src/consumer.rs rename to docs/vocs/docs/snippets/sources/exex/remote/src/consumer.rs diff --git a/book/sources/exex/remote/src/exex.rs b/docs/vocs/docs/snippets/sources/exex/remote/src/exex.rs similarity index 98% rename from book/sources/exex/remote/src/exex.rs rename to docs/vocs/docs/snippets/sources/exex/remote/src/exex.rs index c823d98ded4..67dfac53b58 100644 --- a/book/sources/exex/remote/src/exex.rs +++ b/docs/vocs/docs/snippets/sources/exex/remote/src/exex.rs @@ -75,7 +75,7 @@ fn main() -> eyre::Result<()> { let handle = builder .node(EthereumNode::default()) .install_exex("remote-exex", |ctx| async move { Ok(remote_exex(ctx, notifications)) }) - .launch() + .launch_with_debug_capabilities() .await?; handle.node.task_executor.spawn_critical("gRPC server", async move { diff --git a/book/sources/exex/remote/src/exex_1.rs b/docs/vocs/docs/snippets/sources/exex/remote/src/exex_1.rs similarity index 100% rename from book/sources/exex/remote/src/exex_1.rs rename to docs/vocs/docs/snippets/sources/exex/remote/src/exex_1.rs diff --git a/book/sources/exex/remote/src/exex_2.rs b/docs/vocs/docs/snippets/sources/exex/remote/src/exex_2.rs similarity index 100% rename from book/sources/exex/remote/src/exex_2.rs rename to docs/vocs/docs/snippets/sources/exex/remote/src/exex_2.rs diff --git a/book/sources/exex/remote/src/exex_3.rs b/docs/vocs/docs/snippets/sources/exex/remote/src/exex_3.rs similarity index 100% rename from book/sources/exex/remote/src/exex_3.rs rename to docs/vocs/docs/snippets/sources/exex/remote/src/exex_3.rs diff --git a/book/sources/exex/remote/src/exex_4.rs b/docs/vocs/docs/snippets/sources/exex/remote/src/exex_4.rs similarity index 100% rename from book/sources/exex/remote/src/exex_4.rs rename to docs/vocs/docs/snippets/sources/exex/remote/src/exex_4.rs diff --git a/book/sources/exex/remote/src/lib.rs b/docs/vocs/docs/snippets/sources/exex/remote/src/lib.rs similarity index 100% rename from book/sources/exex/remote/src/lib.rs rename to docs/vocs/docs/snippets/sources/exex/remote/src/lib.rs diff --git a/book/sources/exex/tracking-state/Cargo.toml b/docs/vocs/docs/snippets/sources/exex/tracking-state/Cargo.toml similarity index 96% rename from book/sources/exex/tracking-state/Cargo.toml rename to docs/vocs/docs/snippets/sources/exex/tracking-state/Cargo.toml index 1fc940214c1..658608cac28 100644 --- a/book/sources/exex/tracking-state/Cargo.toml +++ b/docs/vocs/docs/snippets/sources/exex/tracking-state/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "tracking-state" version = "0.1.0" -edition = "2021" +edition = "2024" [dependencies] reth = { git = "https://github.com/paradigmxyz/reth.git" } diff --git a/book/sources/exex/tracking-state/src/bin/1.rs b/docs/vocs/docs/snippets/sources/exex/tracking-state/src/bin/1.rs similarity index 97% rename from book/sources/exex/tracking-state/src/bin/1.rs rename to docs/vocs/docs/snippets/sources/exex/tracking-state/src/bin/1.rs index bfae3ba9508..3ebe2d338e1 100644 --- a/book/sources/exex/tracking-state/src/bin/1.rs +++ b/docs/vocs/docs/snippets/sources/exex/tracking-state/src/bin/1.rs @@ -51,7 +51,7 @@ fn main() -> eyre::Result<()> { let handle = builder .node(EthereumNode::default()) .install_exex("my-exex", async move |ctx| Ok(MyExEx { ctx })) - .launch() + .launch_with_debug_capabilities() .await?; handle.wait_for_node_exit().await diff --git a/book/sources/exex/tracking-state/src/bin/2.rs b/docs/vocs/docs/snippets/sources/exex/tracking-state/src/bin/2.rs similarity index 98% rename from book/sources/exex/tracking-state/src/bin/2.rs rename to docs/vocs/docs/snippets/sources/exex/tracking-state/src/bin/2.rs index 630f2d5072d..8a1af03e6e9 100644 --- a/book/sources/exex/tracking-state/src/bin/2.rs +++ b/docs/vocs/docs/snippets/sources/exex/tracking-state/src/bin/2.rs @@ -71,7 +71,7 @@ fn main() -> eyre::Result<()> { let handle = builder .node(EthereumNode::default()) .install_exex("my-exex", async move |ctx| Ok(MyExEx::new(ctx))) - .launch() + .launch_with_debug_capabilities() .await?; handle.wait_for_node_exit().await diff --git a/docs/vocs/docs/styles.css b/docs/vocs/docs/styles.css new file mode 100644 index 00000000000..fcfc8cf2cd6 --- /dev/null +++ b/docs/vocs/docs/styles.css @@ -0,0 +1,31 @@ +@import "tailwindcss" important; + +@custom-variant dark (&:where(.dark, .dark *)); + +[data-layout="landing"] .vocs_Button_button { + border-radius: 4px !important; + height: 36px !important; + padding: 0 16px !important; +} + +[data-layout="landing"] .vocs_Content { + position: inherit; +} + +#home-install .vocs_CodeGroup { + display: flex; + height: 100%; + flex-direction: column; +} + +#home-install .vocs_Tabs_content { + flex: 1; +} + +#home-install .vocs_Code { + font-size: 18px; +} + +.border-accent { + border: 1px solid var(--vocs-color_borderAccent) !important; +} diff --git a/docs/vocs/package.json b/docs/vocs/package.json new file mode 100644 index 00000000000..b3278dd0be4 --- /dev/null +++ b/docs/vocs/package.json @@ -0,0 +1,26 @@ +{ + "name": "vocs", + "private": true, + "version": "0.0.0", + "type": "module", + "scripts": { + "dev": "vocs dev", + "build": "bash scripts/build-cargo-docs.sh && vocs build && bun scripts/generate-redirects.ts && bun scripts/inject-cargo-docs.ts && bun scripts/fix-search-index.ts", + "preview": "vocs preview", + "check-links": "bun scripts/check-links.ts", + "generate-redirects": "bun scripts/generate-redirects.ts", + "build-cargo-docs": "bash scripts/build-cargo-docs.sh", + "inject-cargo-docs": "bun scripts/inject-cargo-docs.ts" + }, + "dependencies": { + "react": "^19.1.0", + "react-dom": "^19.1.0", + "vocs": "^1.0.13" + }, + "devDependencies": { + "@types/node": "^24.0.14", + "@types/react": "^19.1.8", + "glob": "^11.0.3", + "typescript": "^5.8.3" + } +} \ No newline at end of file diff --git a/docs/vocs/redirects.config.ts b/docs/vocs/redirects.config.ts new file mode 100644 index 00000000000..82a911b6bfc --- /dev/null +++ b/docs/vocs/redirects.config.ts @@ -0,0 +1,32 @@ +export const redirects: Record = { + '/intro': '/overview', + // Installation redirects + '/installation/installation': '/installation/overview', + '/binaries': '/installation/binaries', + '/docker': '/installation/docker', + '/source': '/installation/source', + // Run a node redirects + '/run/run-a-node': '/run/overview', + '/run/mainnet': '/run/ethereum', + '/run/optimism': '/run/opstack', + '/run/sync-op-mainnet': '/run/faq/sync-op-mainnet', + '/run/private-testnet': '/run/private-testnets', + '/run/observability': '/run/monitoring', + '/run/config': '/run/configuration', + '/run/transactions': '/run/faq/transactions', + '/run/pruning': '/run/faq/pruning', + '/run/ports': '/run/faq/ports', + '/run/troubleshooting': '/run/faq/troubleshooting', + // SDK + '/sdk/overview': '/sdk', + // Exex + '/developers/exex': '/exex/overview', + '/developers/exex/how-it-works': '/exex/how-it-works', + '/developers/exex/hello-world': '/exex/hello-world', + '/developers/exex/tracking-state': '/exex/tracking-state', + '/developers/exex/remote': '/exex/remote', + // Contributing + '/developers/contribute': '/introduction/contributing', +} + +export const basePath = '/'; \ No newline at end of file diff --git a/docs/vocs/scripts/build-cargo-docs.sh b/docs/vocs/scripts/build-cargo-docs.sh new file mode 100755 index 00000000000..a1a8eeec0a7 --- /dev/null +++ b/docs/vocs/scripts/build-cargo-docs.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +# Script to build cargo docs with the same flags as used in CI + +# Navigate to the reth root directory (two levels up from book/vocs) +cd ../.. || exit 1 + +echo "Building cargo docs..." + +# Build the documentation +export RUSTDOCFLAGS="--cfg docsrs --show-type-layout --generate-link-to-definition --enable-index-page -Zunstable-options" +cargo docs --exclude "example-*" + +echo "Cargo docs built successfully at ./target/doc" \ No newline at end of file diff --git a/docs/vocs/scripts/check-links.ts b/docs/vocs/scripts/check-links.ts new file mode 100644 index 00000000000..e6bf42c8cb5 --- /dev/null +++ b/docs/vocs/scripts/check-links.ts @@ -0,0 +1,316 @@ +#!/usr/bin/env bun +import { Glob } from "bun"; +import { readFileSync } from "node:fs"; +import { join, dirname, resolve, relative } from "node:path"; + +const CONFIG = { + DOCS_DIR: "./docs/pages", + PUBLIC_DIR: "./docs/public", + REPORT_PATH: "links-report.json", + FILE_PATTERNS: "**/*.{md,mdx}", + MARKDOWN_EXTENSIONS: /\.(md|mdx)$/, +} as const; + +interface BrokenLink { + file: string; + link: string; + line: number; + reason: string; +} + +interface LinkCheckReport { + timestamp: string; + totalFiles: number; + totalLinks: number; + brokenLinks: Array; + summary: { + brokenCount: number; + validCount: number; + }; +} + +main(); + +async function main() { + try { + const report = await checkLinks(); + await saveReport(report); + displayResults(report); + + process.exit(report.summary.brokenCount > 0 ? 1 : 0); + } catch (error) { + console.error("\n❌ Fatal error during link checking:"); + + if (error instanceof Error) { + console.error(` ${error.message}`); + if (error.stack) { + [console.error("\nStack trace:"), console.error(error.stack)]; + } + } else console.error(error); + + process.exit(2); + } +} + +async function checkLinks(): Promise { + console.log("🔍 Finding markdown files..."); + const files = await getAllMarkdownFiles(); + console.log(`📄 Found ${files.length} markdown files`); + + console.log("🔍 Finding public assets..."); + const publicAssets = await getAllPublicAssets(); + console.log(`🖼️ Found ${publicAssets.length} public assets`); + + console.log("🗺️ Building file path map..."); + const pathMap = buildFilePathMap(files, publicAssets); + console.log(`📍 Mapped ${pathMap.size} possible paths`); + + const brokenLinks: BrokenLink[] = []; + let totalLinks = 0; + + console.log("🔗 Checking links in files..."); + + for (let index = 0; index < files.length; index++) { + const file = files[index]; + + try { + const content = readFileSync(file, "utf-8"); + const links = extractLinksFromMarkdown(content); + + for (const { link, line } of links) { + totalLinks++; + const error = validateLink(link, file, pathMap); + + if (error) { + brokenLinks.push({ + file: relative(process.cwd(), file), + link, + line, + reason: error, + }); + } + } + } catch (error) { + console.error(`\nError reading ${file}:`, error); + } + } + + console.log("\n✅ Link checking complete!"); + + return { + timestamp: new Date().toISOString(), + totalFiles: files.length, + totalLinks, + brokenLinks, + summary: { + brokenCount: brokenLinks.length, + validCount: totalLinks - brokenLinks.length, + }, + }; +} + +async function getAllMarkdownFiles(): Promise { + const glob = new Glob(CONFIG.FILE_PATTERNS); + const files = await Array.fromAsync(glob.scan({ cwd: CONFIG.DOCS_DIR })); + return files.map((file) => join(CONFIG.DOCS_DIR, file)); +} + +async function getAllPublicAssets(): Promise { + const glob = new Glob("**/*"); + const files = await Array.fromAsync(glob.scan({ cwd: CONFIG.PUBLIC_DIR })); + return files; +} + +function buildFilePathMap( + files: Array, + publicAssets: Array, +): Set { + const pathMap = new Set(); + + const addPath = (path: string) => { + if (path && typeof path === "string") pathMap.add(path); + }; + + for (const file of files) { + const relativePath = relative(CONFIG.DOCS_DIR, file); + + addPath(relativePath); + + const withoutExt = relativePath.replace(CONFIG.MARKDOWN_EXTENSIONS, ""); + addPath(withoutExt); + + if (withoutExt.endsWith("/index")) + addPath(withoutExt.replace("/index", "")); + + addPath(`/${withoutExt}`); + if (withoutExt.endsWith("/index")) + addPath(`/${withoutExt.replace("/index", "")}`); + } + + for (const asset of publicAssets) addPath(`/${asset}`); + + return pathMap; +} + +function extractLinksFromMarkdown( + content: string, +): Array<{ link: string; line: number }> { + const lines = content.split("\n"); + const links: Array<{ link: string; line: number }> = []; + let inCodeBlock = false; + + for (let lineIndex = 0; lineIndex < lines.length; lineIndex++) { + const line = lines[lineIndex]; + const lineNumber = lineIndex + 1; + + // Toggle code block state + if (line.trim().startsWith("```")) { + inCodeBlock = !inCodeBlock; + continue; + } + + if (inCodeBlock) continue; + + const processedLine = line + .split("`") + .filter((_, index) => index % 2 === 0) + .join(""); + + links.push(...extractMarkdownLinks(processedLine, lineNumber)); + links.push(...extractHtmlLinks(processedLine, lineNumber)); + } + + return links; +} + +function extractMarkdownLinks( + line: string, + lineNumber: number, +): Array<{ link: string; line: number }> { + const regex = /\[([^\]]*)\]\(([^)]+)\)/g; + return [...line.matchAll(regex)] + .map(([, , url]) => ({ link: url, line: lineNumber })) + .filter(({ link }) => isInternalLink(link)); +} + +function extractHtmlLinks( + line: string, + lineNumber: number, +): Array<{ link: string; line: number }> { + const regex = /]+href=["']([^"']+)["'][^>]*>/g; + return [...line.matchAll(regex)] + .map(([, url]) => ({ link: url, line: lineNumber })) + .filter(({ link }) => isInternalLink(link)); +} + +function isInternalLink(url: string): boolean { + return ( + !url.startsWith("http") && + !url.startsWith("mailto:") && + !url.startsWith("#") + ); +} + +function validateLink( + link: string, + sourceFile: string, + pathMap: Set, +): string | null { + const [linkPath] = link.split("#"); + if (!linkPath) return null; // Pure anchor link + + if (linkPath.startsWith("/")) return validateAbsolutePath(linkPath, pathMap); + return validateRelativePath(linkPath, sourceFile, pathMap); +} + +function validateAbsolutePath( + linkPath: string, + pathMap: Set, +): string | null { + const variations = [ + linkPath, + linkPath.slice(1), // Remove leading slash + linkPath.replace(/\/$/, ""), // Remove trailing slash + linkPath + .slice(1) + .replace(/\/$/, ""), // Remove both + ]; + + return variations.some((path) => pathMap.has(path)) + ? null + : `Absolute path not found: ${linkPath}`; +} + +function validateRelativePath( + linkPath: string, + sourceFile: string, + pathMap: Set, +): string | null { + const sourceDir = dirname(relative(CONFIG.DOCS_DIR, sourceFile)); + const resolvedPath = resolve(sourceDir, linkPath); + const normalizedPath = relative(".", resolvedPath); + + const variations = [ + linkPath, + normalizedPath, + `/${normalizedPath}`, + normalizedPath.replace(CONFIG.MARKDOWN_EXTENSIONS, ""), + `/${normalizedPath.replace(CONFIG.MARKDOWN_EXTENSIONS, "")}`, + ]; + + return variations.some((path) => pathMap.has(path)) + ? null + : `Relative path not found: ${linkPath} (resolved to: ${normalizedPath})`; +} + +async function saveReport(report: LinkCheckReport) { + try { + await Bun.write(CONFIG.REPORT_PATH, JSON.stringify(report, null, 2)); + console.log(`\n📝 Report saved to: ${CONFIG.REPORT_PATH}`); + } catch (error) { + console.error( + `\n⚠️ Warning: Failed to save report to ${CONFIG.REPORT_PATH}`, + ); + console.error(error); + } +} + +function displayResults(report: LinkCheckReport) { + LinkCheckReporter.printSummary(report); + + if (report.brokenLinks.length > 0) + LinkCheckReporter.printBrokenLinks(report.brokenLinks); + else console.log("\n✅ All links are valid!"); +} + +const LinkCheckReporter = { + printSummary: (report: LinkCheckReport) => { + console.log("\n📊 Link Check Summary:"); + console.log(` 📄 Files checked: ${report.totalFiles}`); + console.log(` 🔗 Total links: ${report.totalLinks}`); + console.log(` ✅ Valid links: ${report.summary.validCount}`); + console.log(` ❌ Broken links: ${report.summary.brokenCount}`); + }, + printBrokenLinks: (brokenLinks: Array) => { + if (brokenLinks.length === 0) return; + + console.log("\n❌ Broken Links Found:\n"); + + const byFile = brokenLinks.reduce( + (acc, broken) => { + if (!acc[broken.file]) acc[broken.file] = []; + acc[broken.file].push(broken); + return acc; + }, + {} as Record, + ); + + for (const [file, links] of Object.entries(byFile)) { + console.log(`📄 ${file}:`); + for (const broken of links) { + console.log(` Line ${broken.line}: ${broken.link}`); + console.log(` └─ ${broken.reason}\n`); + } + } + }, +}; \ No newline at end of file diff --git a/docs/vocs/scripts/fix-search-index.ts b/docs/vocs/scripts/fix-search-index.ts new file mode 100644 index 00000000000..99b53971f7b --- /dev/null +++ b/docs/vocs/scripts/fix-search-index.ts @@ -0,0 +1,79 @@ +#!/usr/bin/env bun +import { readdir, copyFile, readFile, writeFile } from 'fs/promises'; +import { join } from 'path'; + +async function fixSearchIndex() { + const distDir = 'docs/dist'; + const vocsDir = join(distDir, '.vocs'); + + try { + // 1. Find the search index file + const files = await readdir(vocsDir); + const searchIndexFile = files.find(f => f.startsWith('search-index-') && f.endsWith('.json')); + + if (!searchIndexFile) { + console.error('❌ No search index file found in .vocs directory'); + process.exit(1); + return; + } + + console.log(`📁 Found search index: ${searchIndexFile}`); + + // 2. Copy search index to root of dist + const sourcePath = join(vocsDir, searchIndexFile); + const destPath = join(distDir, searchIndexFile); + await copyFile(sourcePath, destPath); + console.log(`✅ Copied search index to root: ${destPath}`); + + // 3. Find and update all HTML and JS files that reference the search index + const htmlFiles = await findFiles(distDir, '.html'); + const jsFiles = await findFiles(distDir, '.js'); + console.log(`📝 Found ${htmlFiles.length} HTML files and ${jsFiles.length} JS files to update`); + + // 4. Replace references in all files + const allFiles = [...htmlFiles, ...jsFiles]; + for (const file of allFiles) { + const content = await readFile(file, 'utf-8'); + + // Replace /.vocs/search-index-*.json with /search-index-*.json + const updatedContent = content.replace( + /\/.vocs\/search-index-[a-f0-9]+\.json/g, + `/${searchIndexFile}` + ); + + if (content !== updatedContent) { + await writeFile(file, updatedContent); + console.log(` ✓ Updated ${file}`); + } + } + + console.log('✨ Search index fix complete!'); + + } catch (error) { + console.error('❌ Error fixing search index:', error); + process.exit(1); + } +} + +async function findFiles(dir: string, extension: string, files: string[] = []): Promise { + const { readdir } = await import('fs/promises'); + const entries = await readdir(dir, { withFileTypes: true }); + + for (const entry of entries) { + const fullPath = join(dir, entry.name); + + // Skip .vocs, docs, and _site directories + if (entry.name === '.vocs' || entry.name === 'docs' || entry.name === '_site') continue; + + if (entry.isDirectory()) { + files = await findFiles(fullPath, extension, files); + } else if (entry.name.endsWith(extension)) { + files.push(fullPath); + } + } + + return files; +} + +// Run the fix +fixSearchIndex().catch(console.error); diff --git a/docs/vocs/scripts/generate-redirects.ts b/docs/vocs/scripts/generate-redirects.ts new file mode 100644 index 00000000000..c56861a5a90 --- /dev/null +++ b/docs/vocs/scripts/generate-redirects.ts @@ -0,0 +1,54 @@ +#!/usr/bin/env bun +import { writeFileSync, mkdirSync } from 'fs' +import { join, dirname } from 'path' +import { redirects, basePath } from '../redirects.config' +// Base path for the site + +function generateRedirectHtml(targetPath: string): string { + return ` + + + + Redirecting... + + + + + +

Reth mdbook has been migrated to new docs. If you are not redirected please click here.

+ +` +} + +// Generate redirect files +Object.entries(redirects).forEach(([from, to]) => { + // Add base path to target if it doesn't already have it + const finalTarget = to.startsWith(basePath) ? to : `${basePath}${to}` + + // Remove base path if present in from path + const fromPath = from.replace(/^\/reth\//, '') + + // Generate both with and without .html + const paths = [fromPath] + if (!fromPath.endsWith('.html')) { + paths.push(`${fromPath}.html`) + } + + paths.forEach(path => { + const filePath = join('./docs/dist', path) + if (!path.includes('.')) { + // It's a directory path, create index.html + const indexPath = join('./docs/dist', path, 'index.html') + mkdirSync(dirname(indexPath), { recursive: true }) + writeFileSync(indexPath, generateRedirectHtml(finalTarget)) + } else { + // It's a file path + mkdirSync(dirname(filePath), { recursive: true }) + writeFileSync(filePath, generateRedirectHtml(finalTarget)) + } + }) +}) + +console.log('Redirects generated successfully!') \ No newline at end of file diff --git a/docs/vocs/scripts/inject-cargo-docs.ts b/docs/vocs/scripts/inject-cargo-docs.ts new file mode 100644 index 00000000000..74857cb03e9 --- /dev/null +++ b/docs/vocs/scripts/inject-cargo-docs.ts @@ -0,0 +1,213 @@ +import { promises as fs } from 'fs'; +import { glob } from 'glob'; + +const CARGO_DOCS_PATH = '../../target/doc'; +const VOCS_DIST_PATH = './docs/dist/docs'; +const BASE_PATH = '/docs'; + +async function injectCargoDocs() { + console.log('Injecting cargo docs into Vocs dist...'); + + // Check if cargo docs exist + try { + await fs.access(CARGO_DOCS_PATH); + } catch { + console.error(`Error: Cargo docs not found at ${CARGO_DOCS_PATH}`); + console.error("Please run: cargo doc --no-deps --workspace --exclude 'example-*'"); + process.exit(1); + } + + // Check if Vocs dist exists + try { + await fs.access('./docs/dist'); + } catch { + console.error('Error: Vocs dist not found. Please run: bun run build'); + process.exit(1); + } + + // Create docs directory in dist if it doesn't exist + await fs.mkdir(VOCS_DIST_PATH, { recursive: true }); + + // Copy all cargo docs to the dist/docs folder + console.log(`Copying cargo docs to ${VOCS_DIST_PATH}...`); + await fs.cp(CARGO_DOCS_PATH, VOCS_DIST_PATH, { recursive: true }); + + // Fix relative paths in HTML files to work from /reth/docs + console.log('Fixing relative paths in HTML files...'); + + const htmlFiles = await glob(`${VOCS_DIST_PATH}/**/*.html`); + + for (const file of htmlFiles) { + let content = await fs.readFile(file, 'utf-8'); + + // Extract the current crate name and module path from the file path + // Remove the base path to get the relative path within the docs + const relativePath = file.startsWith('./') ? file.slice(2) : file; + const docsRelativePath = relativePath.replace(/^docs\/dist\/docs\//, ''); + const pathParts = docsRelativePath.split('/'); + const fileName = pathParts[pathParts.length - 1]; + + // Determine if this is the root index + const isRootIndex = pathParts.length === 1 && fileName === 'index.html'; + + // Extract crate name - it's the first directory in the docs-relative path + const crateName = isRootIndex ? null : pathParts[0]; + + // Build the current module path (everything between crate and filename) + const modulePath = pathParts.slice(1, -1).join('/'); + + // Fix static file references + content = content + // CSS and JS in static.files + .replace(/href="\.\/static\.files\//g, `href="${BASE_PATH}/static.files/`) + .replace(/src="\.\/static\.files\//g, `src="${BASE_PATH}/static.files/`) + .replace(/href="\.\.\/static\.files\//g, `href="${BASE_PATH}/static.files/`) + .replace(/src="\.\.\/static\.files\//g, `src="${BASE_PATH}/static.files/`) + + // Fix the dynamic font loading in the script tag + .replace(/href="\$\{f\}"/g, `href="${BASE_PATH}/static.files/\${f}"`) + .replace(/href="\.\/static\.files\/\$\{f\}"/g, `href="${BASE_PATH}/static.files/\${f}"`) + + // Fix crate navigation links + .replace(/href="\.\/([^/]+)\/index\.html"/g, `href="${BASE_PATH}/$1/index.html"`) + .replace(/href="\.\.\/([^/]+)\/index\.html"/g, `href="${BASE_PATH}/$1/index.html"`) + // Fix module links within the same crate (relative paths without ./ or ../) + // These need to include the current crate name in the path + .replace(/href="([^/:"\.](?:[^/:"]*)?)\/index\.html"/g, (match, moduleName) => { + // Skip if it's already an absolute path or contains a protocol + if (moduleName.startsWith('/') || moduleName.includes('://')) { + return match; + } + // For the root index page, these are crate links, not module links + if (isRootIndex) { + return `href="${BASE_PATH}/${moduleName}/index.html"`; + } + // For module links within a crate, we need to build the full path + // If we're in a nested module, we need to go up to the crate root then down to the target + const fullPath = modulePath ? `${crateName}/${modulePath}/${moduleName}` : `${crateName}/${moduleName}`; + return `href="${BASE_PATH}/${fullPath}/index.html"`; + }) + + // Also fix other relative links (structs, enums, traits) that don't have index.html + .replace(/href="([^/:"\.#][^/:"#]*\.html)"/g, (match, pageName) => { + // Skip if it's already an absolute path or contains a protocol + if (pageName.startsWith('/') || pageName.includes('://')) { + return match; + } + // Skip for root index page as it shouldn't have such links + if (isRootIndex) { + return match; + } + // For other doc pages in nested modules, build the full path + const fullPath = modulePath ? `${crateName}/${modulePath}/${pageName}` : `${crateName}/${pageName}`; + return `href="${BASE_PATH}/${fullPath}"`; + }) + + // Fix root index.html links + .replace(/href="\.\/index\.html"/g, `href="${BASE_PATH}/index.html"`) + .replace(/href="\.\.\/index\.html"/g, `href="${BASE_PATH}/index.html"`) + + // Fix rustdoc data attributes + .replace(/data-root-path="\.\/"/g, `data-root-path="${BASE_PATH}/"`) + .replace(/data-root-path="\.\.\/"/g, `data-root-path="${BASE_PATH}/"`) + .replace(/data-static-root-path="\.\/static\.files\/"/g, `data-static-root-path="${BASE_PATH}/static.files/"`) + .replace(/data-static-root-path="\.\.\/static\.files\/"/g, `data-static-root-path="${BASE_PATH}/static.files/"`) + + // Fix search index paths + .replace(/data-search-index-js="[^"]+"/g, `data-search-index-js="${BASE_PATH}/search-index.js"`) + .replace(/data-search-js="([^"]+)"/g, `data-search-js="${BASE_PATH}/static.files/$1"`) + .replace(/data-settings-js="([^"]+)"/g, `data-settings-js="${BASE_PATH}/static.files/$1"`) + + // Fix logo paths + .replace(/src="\.\/static\.files\/rust-logo/g, `src="${BASE_PATH}/static.files/rust-logo`) + .replace(/src="\.\.\/static\.files\/rust-logo/g, `src="${BASE_PATH}/static.files/rust-logo`) + + // Fix search functionality by ensuring correct load order + // Add the rustdoc-vars initialization before other scripts + .replace(/`); + + await fs.writeFile(file, content, 'utf-8'); + } + + // Find the actual search JS filename from the HTML files + let actualSearchJsFile = ''; + for (const htmlFile of htmlFiles) { + const htmlContent = await fs.readFile(htmlFile, 'utf-8'); + const searchMatch = htmlContent.match(/data-search-js="[^"]*\/([^"]+)"/); + if (searchMatch && searchMatch[1]) { + actualSearchJsFile = searchMatch[1]; + console.log(`Found search JS file: ${actualSearchJsFile} in ${htmlFile}`); + break; + } + } + + if (!actualSearchJsFile) { + console.error('Could not detect search JS filename from HTML files'); + process.exit(1); + } + + // Also fix paths in JavaScript files + const jsFiles = await glob(`${VOCS_DIST_PATH}/**/*.js`); + + for (const file of jsFiles) { + let content = await fs.readFile(file, 'utf-8'); + + // Fix any hardcoded paths in JS files + content = content + .replace(/"\.\/static\.files\//g, `"${BASE_PATH}/static.files/`) + .replace(/"\.\.\/static\.files\//g, `"${BASE_PATH}/static.files/`) + .replace(/"\.\/([^/]+)\/index\.html"/g, `"${BASE_PATH}/$1/index.html"`) + .replace(/"\.\.\/([^/]+)\/index\.html"/g, `"${BASE_PATH}/$1/index.html"`); + + // Fix the search form submission issue that causes page reload + // Instead of submitting a form, just ensure the search functionality is loaded + if (file.includes('main-') && file.endsWith('.js')) { + content = content.replace( + /function sendSearchForm\(\)\{document\.getElementsByClassName\("search-form"\)\[0\]\.submit\(\)\}/g, + 'function sendSearchForm(){/* Fixed: No form submission needed - search loads via script */}' + ); + + // Also fix the root path references in the search functionality + content = content.replace( + /getVar\("root-path"\)/g, + `"${BASE_PATH}/"` + ); + + // Fix static-root-path to avoid double paths + content = content.replace( + /getVar\("static-root-path"\)/g, + `"${BASE_PATH}/static.files/"` + ); + + // Fix the search-js variable to return just the filename + // Use the detected search filename + content = content.replace( + /getVar\("search-js"\)/g, + `"${actualSearchJsFile}"` + ); + + // Fix the search index loading path + content = content.replace( + /resourcePath\("search-index",".js"\)/g, + `"${BASE_PATH}/search-index.js"` + ); + } + + // Fix paths in storage.js which contains the web components + if (file.includes('storage-') && file.endsWith('.js')) { + content = content.replace( + /getVar\("root-path"\)/g, + `"${BASE_PATH}/"` + ); + } + + await fs.writeFile(file, content, 'utf-8'); + } + + console.log('Cargo docs successfully injected!'); + console.log(`The crate documentation will be available at ${BASE_PATH}`); +} + +// Run the script +injectCargoDocs().catch(console.error); \ No newline at end of file diff --git a/docs/vocs/sidebar.ts b/docs/vocs/sidebar.ts new file mode 100644 index 00000000000..e51af1c260c --- /dev/null +++ b/docs/vocs/sidebar.ts @@ -0,0 +1,518 @@ +import { SidebarItem } from "vocs"; + +export const sidebar: SidebarItem[] = [ + { + text: "Introduction", + items: [ + { + text: "Overview", + link: "/overview" + }, + { + text: "Why Reth?", + link: "/introduction/why-reth" + }, + { + text: "Contributing", + link: "/introduction/contributing" + } + ] + }, + { + text: "Reth for Node Operators", + items: [ + { + text: "System Requirements", + link: "/run/system-requirements" + }, + { + text: "Installation", + collapsed: true, + items: [ + { + text: "Overview", + link: "/installation/overview" + }, + { + text: "Pre-Built Binaries", + link: "/installation/binaries" + }, + { + text: "Docker", + link: "/installation/docker" + }, + { + text: "Build from Source", + link: "/installation/source" + }, + { + text: "Build for ARM devices", + link: "/installation/build-for-arm-devices" + }, + { + text: "Update Priorities", + link: "/installation/priorities" + } + ] + }, + { + text: "Running a Node", + items: [ + { + text: "Overview", + link: "/run/overview", + }, + { + text: "Networks", + // link: "/run/networks", + items: [ + { + text: "Ethereum", + link: "/run/ethereum", + // items: [ + // { + // text: "Snapshots", + // link: "/run/ethereum/snapshots" + // } + // ] + }, + { + text: "OP-stack", + link: "/run/opstack", + // items: [ + // { + // text: "Caveats OP-Mainnet", + // link: "/run/opstack/op-mainnet-caveats" + // } + // ] + }, + { + text: "Private testnets", + link: "/run/private-testnets" + } + ] + }, + ] + }, + { + text: "Configuration", + link: "/run/configuration" + }, + { + text: "Monitoring", + link: "/run/monitoring" + }, + { + text: "FAQ", + link: "/run/faq", + collapsed: true, + items: [ + { + text: "Transaction Types", + link: "/run/faq/transactions" + }, + { + text: "Pruning & Full Node", + link: "/run/faq/pruning" + }, + { + text: "Ports", + link: "/run/faq/ports" + }, + { + text: "Profiling", + link: "/run/faq/profiling" + }, + { + text: "Sync OP Mainnet", + link: "/run/faq/sync-op-mainnet" + } + ] + } + ] + }, + { + text: "Reth as a library", + items: [ + { + text: "Overview", + link: "/sdk" + }, + { + text: "Typesystem", + items: [ + { + text: "Block", + link: "/sdk/typesystem/block" + }, + { + text: "Transaction types", + link: "/sdk/typesystem/transaction-types" + } + ] + }, + { + text: "What is in a node?", + collapsed: false, + items: [ + { + text: "Network", + link: "/sdk/node-components/network" + }, + { + text: "Pool", + link: "/sdk/node-components/pool" + }, + { + text: "Consensus", + link: "/sdk/node-components/consensus" + }, + { + text: "EVM", + link: "/sdk/node-components/evm" + }, + { + text: "RPC", + link: "/sdk/node-components/rpc" + } + ] + }, + // TODO + // { + // text: "Build a custom node", + // items: [ + // { + // text: "Prerequisites and Considerations", + // link: "/sdk/custom-node/prerequisites" + // }, + // { + // text: "What modifications and how", + // link: "/sdk/custom-node/modifications" + // } + // ] + // }, + // { + // text: "Examples", + // items: [ + // { + // text: "How to modify an existing node", + // items: [ + // { + // text: "Additional features: RPC endpoints, services", + // link: "/sdk/examples/modify-node" + // } + // ] + // }, + // { + // text: "How to use standalone components", + // items: [ + // { + // text: "Interact with the disk directly + caveats", + // link: "/sdk/examples/standalone-components" + // } + // ] + // } + // ] + // } + ] + }, + { + text: "Execution Extensions", + items: [ + { + text: "Overview", + link: "/exex/overview" + }, + { + text: "How do ExExes work?", + link: "/exex/how-it-works" + }, + { + text: "Hello World", + link: "/exex/hello-world" + }, + { + text: "Tracking State", + link: "/exex/tracking-state" + }, + { + text: "Remote", + link: "/exex/remote" + } + ] + }, + { + text: "Interacting with Reth over JSON-RPC", + + items: [ + { + text: "Overview", + link: "/jsonrpc/intro", + }, + { + text: "eth", + link: "/jsonrpc/eth" + }, + { + text: "web3", + link: "/jsonrpc/web3" + }, + { + text: "net", + link: "/jsonrpc/net" + }, + { + text: "txpool", + link: "/jsonrpc/txpool" + }, + { + text: "debug", + link: "/jsonrpc/debug" + }, + { + text: "trace", + link: "/jsonrpc/trace" + }, + { + text: "admin", + link: "/jsonrpc/admin" + }, + { + text: "rpc", + link: "/jsonrpc/rpc" + } + ] + }, + { + text: "CLI Reference", + link: "/cli/cli", + collapsed: false, + items: [ + { + text: "reth", + link: "/cli/reth", + collapsed: false, + items: [ + { + text: "reth node", + link: "/cli/reth/node" + }, + { + text: "reth init", + link: "/cli/reth/init" + }, + { + text: "reth init-state", + link: "/cli/reth/init-state" + }, + { + text: "reth import", + link: "/cli/reth/import" + }, + { + text: "reth import-era", + link: "/cli/reth/import-era" + }, + { + text: "reth export-era", + link: "/cli/reth/export-era" + }, + { + text: "reth dump-genesis", + link: "/cli/reth/dump-genesis" + }, + { + text: "reth db", + link: "/cli/reth/db", + collapsed: true, + items: [ + { + text: "reth db stats", + link: "/cli/reth/db/stats" + }, + { + text: "reth db list", + link: "/cli/reth/db/list" + }, + { + text: "reth db checksum", + link: "/cli/reth/db/checksum" + }, + { + text: "reth db diff", + link: "/cli/reth/db/diff" + }, + { + text: "reth db get", + link: "/cli/reth/db/get", + collapsed: true, + items: [ + { + text: "reth db get mdbx", + link: "/cli/reth/db/get/mdbx" + }, + { + text: "reth db get static-file", + link: "/cli/reth/db/get/static-file" + } + ] + }, + { + text: "reth db drop", + link: "/cli/reth/db/drop" + }, + { + text: "reth db clear", + link: "/cli/reth/db/clear", + collapsed: true, + items: [ + { + text: "reth db clear mdbx", + link: "/cli/reth/db/clear/mdbx" + }, + { + text: "reth db clear static-file", + link: "/cli/reth/db/clear/static-file" + } + ] + }, + { + text: "reth db version", + link: "/cli/reth/db/version" + }, + { + text: "reth db path", + link: "/cli/reth/db/path" + } + ] + }, + { + text: "reth download", + link: "/cli/reth/download" + }, + { + text: "reth stage", + link: "/cli/reth/stage", + collapsed: true, + items: [ + { + text: "reth stage run", + link: "/cli/reth/stage/run" + }, + { + text: "reth stage drop", + link: "/cli/reth/stage/drop" + }, + { + text: "reth stage dump", + link: "/cli/reth/stage/dump", + collapsed: true, + items: [ + { + text: "reth stage dump execution", + link: "/cli/reth/stage/dump/execution" + }, + { + text: "reth stage dump storage-hashing", + link: "/cli/reth/stage/dump/storage-hashing" + }, + { + text: "reth stage dump account-hashing", + link: "/cli/reth/stage/dump/account-hashing" + }, + { + text: "reth stage dump merkle", + link: "/cli/reth/stage/dump/merkle" + } + ] + }, + { + text: "reth stage unwind", + link: "/cli/reth/stage/unwind", + collapsed: true, + items: [ + { + text: "reth stage unwind to-block", + link: "/cli/reth/stage/unwind/to-block" + }, + { + text: "reth stage unwind num-blocks", + link: "/cli/reth/stage/unwind/num-blocks" + } + ] + } + ] + }, + { + text: "reth p2p", + link: "/cli/reth/p2p", + collapsed: true, + items: [ + { + text: "reth p2p header", + link: "/cli/reth/p2p/header" + }, + { + text: "reth p2p body", + link: "/cli/reth/p2p/body" + }, + { + text: "reth p2p rlpx", + link: "/cli/reth/p2p/rlpx", + collapsed: true, + items: [ + { + text: "reth p2p rlpx ping", + link: "/cli/reth/p2p/rlpx/ping" + } + ] + } + ] + }, + { + text: "reth config", + link: "/cli/reth/config" + }, + { + text: "reth debug", + link: "/cli/reth/debug", + collapsed: true, + items: [ + { + text: "reth debug execution", + link: "/cli/reth/debug/execution" + }, + { + text: "reth debug merkle", + link: "/cli/reth/debug/merkle" + }, + { + text: "reth debug in-memory-merkle", + link: "/cli/reth/debug/in-memory-merkle" + }, + { + text: "reth debug build-block", + link: "/cli/reth/debug/build-block" + } + ] + }, + { + text: "reth recover", + link: "/cli/reth/recover", + collapsed: true, + items: [ + { + text: "reth recover storage-tries", + link: "/cli/reth/recover/storage-tries" + } + ] + }, + { + text: "reth prune", + link: "/cli/reth/prune" + } + ] + } + ] + }, +] \ No newline at end of file diff --git a/docs/vocs/tsconfig.json b/docs/vocs/tsconfig.json new file mode 100644 index 00000000000..d2636aac47e --- /dev/null +++ b/docs/vocs/tsconfig.json @@ -0,0 +1,24 @@ +{ + "compilerOptions": { + "target": "ES2020", + "useDefineForClassFields": true, + "lib": ["ES2020", "DOM", "DOM.Iterable"], + "module": "ESNext", + "skipLibCheck": true, + + /* Bundler mode */ + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "resolveJsonModule": true, + "isolatedModules": true, + "noEmit": true, + "jsx": "react-jsx", + + /* Linting */ + "strict": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "noFallthroughCasesInSwitch": true + }, + "include": ["**/*.ts", "**/*.tsx"] +} diff --git a/docs/vocs/vocs.config.ts b/docs/vocs/vocs.config.ts new file mode 100644 index 00000000000..8664eedd3bf --- /dev/null +++ b/docs/vocs/vocs.config.ts @@ -0,0 +1,76 @@ +import React from 'react' +import { defineConfig } from 'vocs' +import { sidebar } from './sidebar' +import { basePath } from './redirects.config' + +export default defineConfig({ + title: 'Reth', + logoUrl: '/logo.png', + iconUrl: '/logo.png', + ogImageUrl: '/reth-prod.png', + sidebar, + basePath, + search: { + fuzzy: true + }, + topNav: [ + { text: 'Run', link: '/run/ethereum' }, + { text: 'SDK', link: '/sdk' }, + { + element: React.createElement('a', { href: '/docs', target: '_self' }, 'Rustdocs') + }, + { text: 'GitHub', link: 'https://github.com/paradigmxyz/reth' }, + { + text: 'v1.9.3', + items: [ + { + text: 'Releases', + link: 'https://github.com/paradigmxyz/reth/releases' + }, + { + text: 'Contributing', + link: 'https://github.com/paradigmxyz/reth/blob/main/CONTRIBUTING.md' + } + ] + } + ], + socials: [ + { + icon: 'github', + link: 'https://github.com/paradigmxyz/reth', + }, + { + icon: 'telegram', + link: 'https://t.me/paradigm_reth', + }, + ], + sponsors: [ + { + name: 'Collaborators', + height: 120, + items: [ + [ + { + name: 'Paradigm', + link: 'https://paradigm.xyz', + image: 'https://raw.githubusercontent.com/wevm/.github/main/content/sponsors/paradigm-light.svg', + }, + { + name: 'Ithaca', + link: 'https://ithaca.xyz', + image: 'https://raw.githubusercontent.com/wevm/.github/main/content/sponsors/ithaca-light.svg', + } + ] + ] + } + ], + theme: { + accentColor: { + light: '#1f1f1f', + dark: '#ffffff', + } + }, + editLink: { + pattern: "https://github.com/paradigmxyz/reth/edit/main/docs/vocs/docs/pages/:path", + } +}) diff --git a/docs/workflow.md b/docs/workflow.md index 1f4c9147a7a..544a41731ae 100644 --- a/docs/workflow.md +++ b/docs/workflow.md @@ -2,7 +2,7 @@ ### Assigning issues -Before working on an issue, it should be assigned to the person who wants to work on it. For core contributors, this means assigning yourself to the issue, and for external contributors this means asking to be assigned on the issue. This is to avoid double work. +Before working on an issue, it should be assigned to the person who wants to work on it. For core contributors, this means assigning yourself to the issue, and for external contributors, this means asking to be assigned to the issue. This is to avoid double work. ### Pull requests @@ -25,7 +25,7 @@ gitGraph - Features and bug fixes live on feature branches off of the main branch, and they are merged onto main as well. This means that the latest version of reth (which might be unstable) always lives on main. -- Pull requests should not be merged without the review of at least one core contributor. For larger pull requests, at least two is recommended. +- Pull requests should not be merged without the review of at least one core contributor. For larger pull requests, at least two reviewers are recommended. - Important pull requests that should be highlighted in the changelog should be marked with the https://github.com/paradigmxyz/reth/labels/M-changelog label. ### Releases @@ -59,4 +59,4 @@ gitGraph - Additionally, each PR is again tested before release by being run every night on a live testnet [clippy]: https://github.com/rust-lang/rust-clippy -[rustfmt]: https://github.com/rust-lang/rustfmt \ No newline at end of file +[rustfmt]: https://github.com/rust-lang/rustfmt diff --git a/etc/README.md b/etc/README.md index 4f4ce7f20e4..0c431e8f463 100644 --- a/etc/README.md +++ b/etc/README.md @@ -13,7 +13,7 @@ up to date. ### Docker Compose To run Reth, Grafana or Prometheus with Docker Compose, refer to -the [docker docs](/book/installation/docker.md#using-docker-compose). +the [docker docs](https://reth.rs/installation/docker#using-docker-compose). ### Grafana @@ -45,7 +45,7 @@ To set up a new metric in Reth and its Grafana dashboard (this assumes running R 1. Save and arrange: - Click `Apply` to save the panel - - Drag the panel to desired position on the dashboard + - Drag the panel to the desired position on the dashboard 1. Export the dashboard: @@ -61,7 +61,7 @@ Your new metric is now integrated into the Reth Grafana dashboard. #### Import Grafana dashboards -If you are running Reth and Grafana outside of docker, and wish to import new Grafana dashboards or update a dashboard: +If you are running Reth and Grafana outside of Docker, and wish to import new Grafana dashboards or update a dashboard: 1. Go to `Home` > `Dashboards` @@ -74,5 +74,5 @@ If you are running Reth and Grafana outside of docker, and wish to import new Gr 1. Delete the old dashboard -If you are running Reth and Grafana using docker, after having pulled the updated dashboards from `main`, restart the -Grafana service. This will update all dashboards. \ No newline at end of file +If you are running Reth and Grafana using Docker, after having pulled the updated dashboards from `main`, restart the +Grafana service. This will update all dashboards. diff --git a/etc/docker-compose.yml b/etc/docker-compose.yml index 5e10225018f..73311616fd1 100644 --- a/etc/docker-compose.yml +++ b/etc/docker-compose.yml @@ -1,24 +1,26 @@ -name: 'reth' +name: reth services: reth: restart: unless-stopped image: ghcr.io/paradigmxyz/reth ports: - - '9001:9001' # metrics - - '30303:30303' # eth/66 peering - - '8545:8545' # rpc - - '8551:8551' # engine + - "9001:9001" # metrics + - "30303:30303" # eth/66 peering + - "8545:8545" # rpc + - "8551:8551" # engine volumes: - mainnet_data:/root/.local/share/reth/mainnet - sepolia_data:/root/.local/share/reth/sepolia - holesky_data:/root/.local/share/reth/holesky + - hoodi_data:/root/.local/share/reth/hoodi - logs:/root/logs - ./jwttoken:/root/jwt/:ro # https://paradigmxyz.github.io/reth/run/troubleshooting.html#concurrent-database-access-error-using-containersdocker pid: host # For Sepolia, replace `--chain mainnet` with `--chain sepolia` # For Holesky, replace `--chain mainnet` with `--chain holesky` + # For Hoodi, replace `--chain mainnet` with `--chain hoodi` command: > node --chain mainnet @@ -39,7 +41,7 @@ services: - 9090:9090 volumes: - ./prometheus/:/etc/prometheus/ - - prometheusdata:/prometheus + - prometheus_data:/prometheus command: - --config.file=/etc/prometheus/prometheus.yml - --storage.tsdb.path=/prometheus @@ -55,7 +57,7 @@ services: environment: PROMETHEUS_URL: ${PROMETHEUS_URL:-http://prometheus:9090} volumes: - - grafanadata:/var/lib/grafana + - grafana_data:/var/lib/grafana - ./grafana/datasources:/etc/grafana/provisioning/datasources - ./grafana/dashboards:/etc/grafana/provisioning_temp/dashboards # 1. Copy dashboards from temp directory to prevent modifying original host files @@ -73,9 +75,11 @@ volumes: driver: local holesky_data: driver: local + hoodi_data: + driver: local logs: driver: local - prometheusdata: + prometheus_data: driver: local - grafanadata: + grafana_data: driver: local diff --git a/etc/generate-jwt.sh b/etc/generate-jwt.sh index 1e06db9651d..5549ed83f24 100755 --- a/etc/generate-jwt.sh +++ b/etc/generate-jwt.sh @@ -1,6 +1,6 @@ #!/bin/bash # Borrowed from EthStaker's prepare for the merge guide -# See https://github.com/remyroy/ethstaker/blob/main/prepare-for-the-merge.md#configuring-a-jwt-token-file +# See https://github.com/eth-educators/ethstaker-guides/blob/main/docs/prepare-for-the-merge.md#configuring-a-jwt-token-file SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) mkdir -p "${SCRIPT_DIR}/jwttoken" diff --git a/etc/grafana/dashboards/overview.json b/etc/grafana/dashboards/overview.json index 1f2ebe36943..6d9563ffd2d 100644 --- a/etc/grafana/dashboards/overview.json +++ b/etc/grafana/dashboards/overview.json @@ -14,6 +14,13 @@ "description": "", "type": "datasource", "pluginId": "__expr__" + }, + { + "name": "VAR_INSTANCE_LABEL", + "type": "constant", + "label": "Instance Label", + "value": "job", + "description": "" } ], "__elements": {}, @@ -39,7 +46,7 @@ "type": "grafana", "id": "grafana", "name": "Grafana", - "version": "11.5.3" + "version": "12.2.1" }, { "type": "panel", @@ -103,7 +110,6 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": null, "links": [], "panels": [ { @@ -136,7 +142,7 @@ "steps": [ { "color": "green", - "value": null + "value": 0 } ] } @@ -145,7 +151,7 @@ }, "gridPos": { "h": 3, - "w": 3, + "w": 4, "x": 0, "y": 1 }, @@ -157,9 +163,7 @@ "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "fields": "", "values": false }, @@ -170,16 +174,16 @@ "textMode": "name", "wideLayout": true }, - "pluginVersion": "11.5.3", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", "exemplar": false, - "expr": "reth_info{instance=~\"$instance\"}", + "expr": "reth_info{$instance_label=\"$instance\"}", "instant": true, "legendFormat": "{{version}}", "range": false, @@ -206,7 +210,7 @@ "steps": [ { "color": "green", - "value": null + "value": 0 } ] } @@ -215,8 +219,8 @@ }, "gridPos": { "h": 3, - "w": 6, - "x": 3, + "w": 4, + "x": 4, "y": 1 }, "id": 192, @@ -227,9 +231,7 @@ "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "fields": "", "values": false }, @@ -240,16 +242,16 @@ "textMode": "name", "wideLayout": true }, - "pluginVersion": "11.5.3", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", "exemplar": false, - "expr": "reth_info{instance=~\"$instance\"}", + "expr": "reth_info{$instance_label=\"$instance\"}", "instant": true, "legendFormat": "{{build_timestamp}}", "range": false, @@ -276,7 +278,7 @@ "steps": [ { "color": "green", - "value": null + "value": 0 } ] } @@ -285,8 +287,8 @@ }, "gridPos": { "h": 3, - "w": 3, - "x": 9, + "w": 4, + "x": 8, "y": 1 }, "id": 193, @@ -297,9 +299,7 @@ "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "fields": "", "values": false }, @@ -310,16 +310,16 @@ "textMode": "name", "wideLayout": true }, - "pluginVersion": "11.5.3", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", "exemplar": false, - "expr": "reth_info{instance=~\"$instance\"}", + "expr": "reth_info{$instance_label=\"$instance\"}", "instant": true, "legendFormat": "{{git_sha}}", "range": false, @@ -346,7 +346,7 @@ "steps": [ { "color": "green", - "value": null + "value": 0 } ] } @@ -355,7 +355,7 @@ }, "gridPos": { "h": 3, - "w": 2, + "w": 4, "x": 12, "y": 1 }, @@ -367,9 +367,7 @@ "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "fields": "", "values": false }, @@ -380,16 +378,16 @@ "textMode": "name", "wideLayout": true }, - "pluginVersion": "11.5.3", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", "exemplar": false, - "expr": "reth_info{instance=~\"$instance\"}", + "expr": "reth_info{$instance_label=\"$instance\"}", "instant": true, "legendFormat": "{{build_profile}}", "range": false, @@ -416,7 +414,7 @@ "steps": [ { "color": "green", - "value": null + "value": 0 } ] } @@ -425,8 +423,8 @@ }, "gridPos": { "h": 3, - "w": 5, - "x": 14, + "w": 4, + "x": 16, "y": 1 }, "id": 196, @@ -437,9 +435,7 @@ "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "fields": "", "values": false }, @@ -450,16 +446,16 @@ "textMode": "name", "wideLayout": true }, - "pluginVersion": "11.5.3", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", "exemplar": false, - "expr": "reth_info{instance=~\"$instance\"}", + "expr": "reth_info{$instance_label=\"$instance\"}", "instant": true, "legendFormat": "{{target_triple}}", "range": false, @@ -486,7 +482,7 @@ "steps": [ { "color": "green", - "value": null + "value": 0 } ] } @@ -495,8 +491,8 @@ }, "gridPos": { "h": 3, - "w": 5, - "x": 19, + "w": 4, + "x": 20, "y": 1 }, "id": 197, @@ -507,9 +503,7 @@ "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "fields": "", "values": false }, @@ -520,16 +514,16 @@ "textMode": "name", "wideLayout": true }, - "pluginVersion": "11.5.3", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", "exemplar": false, - "expr": "reth_info{instance=~\"$instance\"}", + "expr": "reth_info{$instance_label=\"$instance\"}", "instant": true, "legendFormat": "{{cargo_features}}", "range": false, @@ -558,7 +552,7 @@ "steps": [ { "color": "dark-red", - "value": null + "value": 0 }, { "color": "semi-dark-orange", @@ -578,7 +572,7 @@ "overrides": [] }, "gridPos": { - "h": 8, + "h": 9, "w": 8, "x": 0, "y": 4 @@ -589,9 +583,7 @@ "minVizWidth": 75, "orientation": "auto", "reduceOptions": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "fields": "", "values": false }, @@ -599,16 +591,16 @@ "showThresholdMarkers": true, "sizing": "auto" }, - "pluginVersion": "11.5.3", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", "exemplar": false, - "expr": "reth_network_connected_peers{instance=~\"$instance\"}", + "expr": "reth_network_connected_peers{$instance_label=\"$instance\"}", "instant": true, "legendFormat": "__auto", "range": false, @@ -637,7 +629,7 @@ "steps": [ { "color": "green", - "value": null + "value": 0 } ] } @@ -645,7 +637,7 @@ "overrides": [] }, "gridPos": { - "h": 8, + "h": 9, "w": 8, "x": 8, "y": 4 @@ -660,14 +652,12 @@ "showLegend": false }, "maxVizHeight": 300, - "minVizHeight": 10, - "minVizWidth": 0, + "minVizHeight": 16, + "minVizWidth": 8, "namePlacement": "auto", "orientation": "horizontal", "reduceOptions": { - "calcs": [ - "last" - ], + "calcs": ["lastNotNull"], "fields": "", "values": false }, @@ -675,16 +665,16 @@ "sizing": "auto", "valueMode": "color" }, - "pluginVersion": "11.5.3", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", "exemplar": false, - "expr": "reth_sync_checkpoint{instance=~\"$instance\"}", + "expr": "max by (stage) (reth_sync_checkpoint{$instance_label=\"$instance\"})", "instant": true, "legendFormat": "{{stage}}", "range": false, @@ -745,7 +735,7 @@ "steps": [ { "color": "green", - "value": null + "value": 0 } ] }, @@ -754,7 +744,7 @@ "overrides": [] }, "gridPos": { - "h": 8, + "h": 9, "w": 8, "x": 16, "y": 4 @@ -767,9 +757,7 @@ "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "fields": "", "values": false }, @@ -777,15 +765,15 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.5.3", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "sum(reth_db_table_size{instance=~\"$instance\"})", + "expr": "sum(reth_db_table_size{$instance_label=\"$instance\"})", "legendFormat": "Database", "range": true, "refId": "A" @@ -796,7 +784,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(reth_db_freelist{instance=~\"$instance\"} * reth_db_page_size{instance=~\"$instance\"})", + "expr": "sum(reth_db_freelist{$instance_label=\"$instance\"} * reth_db_page_size{$instance_label=\"$instance\"})", "hide": false, "instant": false, "legendFormat": "Freelist", @@ -806,10 +794,10 @@ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "sum(reth_static_files_segment_size{instance=~\"$instance\"})", + "expr": "sum(reth_static_files_segment_size{$instance_label=\"$instance\"})", "hide": false, "instant": false, "legendFormat": "Static Files", @@ -822,7 +810,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(reth_db_table_size{instance=~\"$instance\"}) + sum(reth_db_freelist{instance=~\"$instance\"} * reth_db_page_size{instance=~\"$instance\"}) + sum(reth_static_files_segment_size{instance=~\"$instance\"})", + "expr": "sum(reth_db_table_size{$instance_label=\"$instance\"}) + sum(reth_db_freelist{$instance_label=\"$instance\"} * reth_db_page_size{$instance_label=\"$instance\"}) + sum(reth_static_files_segment_size{$instance_label=\"$instance\"})", "hide": false, "instant": false, "legendFormat": "Total", @@ -837,7 +825,7 @@ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -868,6 +856,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": true, "stacking": { "group": "A", @@ -885,7 +874,7 @@ "steps": [ { "color": "green", - "value": null + "value": 0 } ] }, @@ -897,7 +886,7 @@ "h": 8, "w": 12, "x": 0, - "y": 12 + "y": 13 }, "id": 69, "options": { @@ -913,7 +902,7 @@ "sort": "none" } }, - "pluginVersion": "11.5.3", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { @@ -921,7 +910,7 @@ "uid": "${datasource}" }, "editorMode": "builder", - "expr": "reth_sync_entities_processed{instance=~\"$instance\"} / reth_sync_entities_total{instance=~\"$instance\"}", + "expr": "avg by (stage) (reth_sync_entities_processed{$instance_label=\"$instance\"} / reth_sync_entities_total{$instance_label=\"$instance\"})", "legendFormat": "{{stage}}", "range": true, "refId": "A" @@ -933,7 +922,7 @@ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -964,6 +953,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -979,7 +969,7 @@ "steps": [ { "color": "green", - "value": null + "value": 0 }, { "color": "red", @@ -994,7 +984,7 @@ "h": 8, "w": 12, "x": 12, - "y": 12 + "y": 13 }, "id": 12, "options": { @@ -1010,7 +1000,7 @@ "sort": "none" } }, - "pluginVersion": "11.5.3", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { @@ -1018,7 +1008,7 @@ "uid": "${datasource}" }, "editorMode": "builder", - "expr": "reth_sync_checkpoint{instance=~\"$instance\"}", + "expr": "max by (stage) (reth_sync_checkpoint{$instance_label=\"$instance\"})", "legendFormat": "{{stage}}", "range": true, "refId": "A" @@ -1030,9 +1020,9 @@ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "description": "Tracks the number of critical tasks currently ran by the executor.", + "description": "Latency histogram for the engine_forkchoiceUpdated RPC API", "fieldConfig": { "defaults": { "color": { @@ -1062,6 +1052,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -1077,127 +1068,34 @@ "steps": [ { "color": "green", - "value": null - }, - { - "color": "semi-dark-red", "value": 0 - } - ] - }, - "unit": "tasks" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 20 - }, - "id": 248, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "reth_executor_spawn_critical_tasks_total{instance=\"$instance\"}- reth_executor_spawn_finished_critical_tasks_total{instance=\"$instance\"}", - "hide": false, - "instant": false, - "legendFormat": "Tasks running", - "range": true, - "refId": "C" - } - ], - "title": "Task Executor critical tasks", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Tracks the number of regular tasks currently ran by the executor.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null }, { - "color": "semi-dark-red", + "color": "red", "value": 80 } ] }, - "unit": "tasks/s" + "unit": "s" }, "overrides": [ { "matcher": { - "id": "byFrameRefID", - "options": "C" + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } }, "properties": [ { - "id": "unit", - "value": "tasks" + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": true + } } ] } @@ -1206,10 +1104,10 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 20 + "x": 0, + "y": 21 }, - "id": 247, + "id": 211, "options": { "legend": { "calcs": [], @@ -1223,7 +1121,7 @@ "sort": "none" } }, - "pluginVersion": "11.5.3", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { @@ -1232,13 +1130,10 @@ }, "disableTextWrap": false, "editorMode": "builder", - "exemplar": false, - "expr": "rate(reth_executor_spawn_regular_tasks_total{instance=\"$instance\"}[$__rate_interval])", + "expr": "reth_engine_rpc_fork_choice_updated_v1{$instance_label=\"$instance\", quantile=\"0\"}", "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "Tasks started", + "includeNullMetadata": true, + "legendFormat": "engine_forkchoiceUpdatedV1 min", "range": true, "refId": "A", "useBackend": false @@ -1246,516 +1141,322 @@ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "editorMode": "code", - "expr": "reth_executor_spawn_regular_tasks_total{instance=\"$instance\"}- reth_executor_spawn_finished_regular_tasks_total{instance=\"$instance\"}", + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_engine_rpc_fork_choice_updated_v1{$instance_label=\"$instance\", quantile=\"0.5\"}", + "fullMetaSearch": false, "hide": false, - "instant": false, - "legendFormat": "Tasks running", + "includeNullMetadata": true, + "legendFormat": "engine_forkchoiceUpdatedV1 p50", "range": true, - "refId": "C" - } - ], - "title": "Task Executor regular tasks", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 28 - }, - "id": 38, - "panels": [], - "repeat": "instance", - "title": "Database", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "The average commit time for database transactions. Generally, this should not be a limiting factor in syncing.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic", - "seriesBy": "last" + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_engine_rpc_fork_choice_updated_v1{$instance_label=\"$instance\", quantile=\"0.9\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "engine_forkchoiceUpdatedV1 p90", + "range": true, + "refId": "C", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_engine_rpc_fork_choice_updated_v1{$instance_label=\"$instance\", quantile=\"0.95\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "engine_forkchoiceUpdatedV1 p95", + "range": true, + "refId": "D", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" }, - "unit": "s" + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_engine_rpc_fork_choice_updated_v1{$instance_label=\"$instance\", quantile=\"0.99\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "engine_forkchoiceUpdatedV1 p99", + "range": true, + "refId": "E", + "useBackend": false }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 29 - }, - "id": 40, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_engine_rpc_fork_choice_updated_v2{$instance_label=\"$instance\", quantile=\"0\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "engine_forkchoiceUpdatedV2 min", + "range": true, + "refId": "F", + "useBackend": false }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.3", - "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "editorMode": "code", - "exemplar": false, - "expr": "avg(rate(reth_database_transaction_close_duration_seconds_sum{instance=~\"$instance\", outcome=\"commit\"}[$__rate_interval]) / rate(reth_database_transaction_close_duration_seconds_count{instance=~\"$instance\", outcome=\"commit\"}[$__rate_interval]) >= 0)", - "format": "time_series", - "instant": false, - "legendFormat": "Commit time", + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_engine_rpc_fork_choice_updated_v2{$instance_label=\"$instance\", quantile=\"0.5\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "engine_forkchoiceUpdatedV2 p50", "range": true, - "refId": "A" + "refId": "G", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_engine_rpc_fork_choice_updated_v2{$instance_label=\"$instance\", quantile=\"0.9\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "engine_forkchoiceUpdatedV2 p90", + "range": true, + "refId": "H", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_engine_rpc_fork_choice_updated_v2{$instance_label=\"$instance\", quantile=\"0.95\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "engine_forkchoiceUpdatedV2 p95", + "range": true, + "refId": "I", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_engine_rpc_fork_choice_updated_v2{$instance_label=\"$instance\", quantile=\"0.99\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "engine_forkchoiceUpdatedV2 p99", + "range": true, + "refId": "J", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_engine_rpc_fork_choice_updated_v3{$instance_label=\"$instance\", quantile=\"0\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "engine_forkchoiceUpdatedV3 min", + "range": true, + "refId": "K", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_engine_rpc_fork_choice_updated_v3{$instance_label=\"$instance\", quantile=\"0.5\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "engine_forkchoiceUpdatedV3 p50", + "range": true, + "refId": "L", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_engine_rpc_fork_choice_updated_v3{$instance_label=\"$instance\", quantile=\"0.9\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "engine_forkchoiceUpdatedV3 p90", + "range": true, + "refId": "M", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_engine_rpc_fork_choice_updated_v3{$instance_label=\"$instance\", quantile=\"0.95\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "engine_forkchoiceUpdatedV3 p95", + "range": true, + "refId": "N", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_engine_rpc_fork_choice_updated_v3{$instance_label=\"$instance\", quantile=\"0.99\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "engine_forkchoiceUpdatedV3 p99", + "range": true, + "refId": "O", + "useBackend": false } ], - "title": "Average commit time", + "title": "Engine API forkchoiceUpdated Latency", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "description": "", + "description": "Latency histogram for the engine_newPayload RPC API", "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, "scaleDistribution": { "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } - } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": true + } + } + ] + } + ] }, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 29 - }, - "id": 42, - "maxDataPoints": 25, - "options": { - "calculate": false, - "cellGap": 1, - "cellValues": { - "unit": "s" - }, - "color": { - "exponent": 0.2, - "fill": "dark-orange", - "min": 0, - "mode": "opacity", - "reverse": false, - "scale": "exponential", - "scheme": "Oranges", - "steps": 128 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto", - "value": "Commit time" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": false - }, - "yAxis": { - "axisLabel": "Quantile", - "axisPlacement": "left", - "reverse": false, - "unit": "percentunit" - } - }, - "pluginVersion": "11.5.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "avg(max_over_time(reth_database_transaction_close_duration_seconds{instance=~\"$instance\", outcome=\"commit\"}[$__rate_interval])) by (quantile)", - "format": "time_series", - "instant": false, - "legendFormat": "{{quantile}}", - "range": true, - "refId": "A" - } - ], - "title": "Commit time heatmap", - "type": "heatmap" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "The average time a database transaction was open.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic", - "seriesBy": "last" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 37 - }, - "id": 117, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(rate(reth_database_transaction_open_duration_seconds_sum{instance=~\"$instance\", outcome!=\"\"}[$__rate_interval]) / rate(reth_database_transaction_open_duration_seconds_count{instance=~\"$instance\", outcome!=\"\"}[$__rate_interval])) by (outcome, mode)", - "format": "time_series", - "instant": false, - "legendFormat": "{{mode}}, {{outcome}}", - "range": true, - "refId": "A" - } - ], - "title": "Average transaction open time", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "The maximum time the database transaction was open.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 37 - }, - "id": 116, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max(max_over_time(reth_database_transaction_open_duration_seconds{instance=~\"$instance\", outcome!=\"\", quantile=\"1\"}[$__interval])) by (outcome, mode)", - "format": "time_series", - "instant": false, - "legendFormat": "{{mode}}, {{outcome}}", - "range": true, - "refId": "A" - } - ], - "title": "Max transaction open time", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "txs", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 3, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [ - { - "matcher": { - "id": "byFrameRefID", - "options": "Diff(opened-closed)" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "custom.lineStyle", - "value": { - "dash": [ - 0, - 10 - ], - "fill": "dot" - } - }, - { - "id": "custom.axisLabel", - "value": "diff" - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 45 + "y": 21 }, - "id": 119, + "id": 210, "options": { "legend": { "calcs": [], @@ -1769,7 +1470,7 @@ "sort": "none" } }, - "pluginVersion": "11.5.3", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { @@ -1777,159 +1478,159 @@ "uid": "${datasource}" }, "disableTextWrap": false, - "editorMode": "code", - "exemplar": false, - "expr": "sum(reth_database_transaction_opened_total{instance=~\"$instance\", mode=\"read-write\"})", - "format": "time_series", + "editorMode": "builder", + "expr": "reth_engine_rpc_new_payload_v1{$instance_label=\"$instance\", quantile=\"0\"}", "fullMetaSearch": false, "includeNullMetadata": true, - "instant": false, - "legendFormat": "Opened", + "legendFormat": "engine_newPayloadV1 min", "range": true, "refId": "A", "useBackend": false }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_engine_rpc_new_payload_v1{$instance_label=\"$instance\", quantile=\"0.5\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "engine_newPayloadV1 p50", + "range": true, + "refId": "B", + "useBackend": false + }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(reth_database_transaction_closed_total{instance=~\"$instance\", mode=\"read-write\"})", - "format": "time_series", - "instant": false, - "legendFormat": "Closed {{mode}}", + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_engine_rpc_new_payload_v1{$instance_label=\"$instance\", quantile=\"0.9\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "engine_newPayloadV1 p90", "range": true, - "refId": "B" + "refId": "C", + "useBackend": false }, { "datasource": { - "type": "__expr__", - "uid": "${DS_EXPRESSION}" + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - "expression": "${A} - ${B}", + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_engine_rpc_new_payload_v1{$instance_label=\"$instance\", quantile=\"0.95\"}", + "fullMetaSearch": false, "hide": false, - "refId": "Diff(opened-closed)", - "type": "math" - } - ], - "title": "Number of read-write transactions", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "includeNullMetadata": true, + "legendFormat": "engine_newPayloadV1 p95", + "range": true, + "refId": "D", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "txs", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 3, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_engine_rpc_new_payload_v1{$instance_label=\"$instance\", quantile=\"0.99\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "engine_newPayloadV1 p99", + "range": true, + "refId": "E", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_engine_rpc_new_payload_v2{$instance_label=\"$instance\", quantile=\"0\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "engine_newPayloadV2 min", + "range": true, + "refId": "F", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" }, - "unit": "none" + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_engine_rpc_new_payload_v2{$instance_label=\"$instance\", quantile=\"0.5\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "engine_newPayloadV2 p50", + "range": true, + "refId": "G", + "useBackend": false }, - "overrides": [ - { - "matcher": { - "id": "byFrameRefID", - "options": "Diff(opened, closed)" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "custom.lineStyle", - "value": { - "dash": [ - 0, - 10 - ], - "fill": "dot" - } - }, - { - "id": "custom.axisLabel", - "value": "diff" - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 45 - }, - "id": 250, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_engine_rpc_new_payload_v2{$instance_label=\"$instance\", quantile=\"0.9\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "engine_newPayloadV2 p90", + "range": true, + "refId": "H", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_engine_rpc_new_payload_v2{$instance_label=\"$instance\", quantile=\"0.95\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "engine_newPayloadV2 p95", + "range": true, + "refId": "I", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_engine_rpc_new_payload_v2{$instance_label=\"$instance\", quantile=\"0.99\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "engine_newPayloadV2 p99", + "range": true, + "refId": "J", + "useBackend": false }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.3", - "targets": [ { "datasource": { "type": "prometheus", @@ -1937,15 +1638,29 @@ }, "disableTextWrap": false, "editorMode": "builder", - "exemplar": false, - "expr": "reth_database_transaction_opened_total{instance=~\"$instance\", mode=\"read-only\"}", - "format": "time_series", + "expr": "reth_engine_rpc_new_payload_v3{$instance_label=\"$instance\", quantile=\"0\"}", "fullMetaSearch": false, + "hide": false, "includeNullMetadata": true, - "instant": false, - "legendFormat": "Opened", + "legendFormat": "engine_newPayloadV3 min", "range": true, - "refId": "A", + "refId": "K", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_engine_rpc_new_payload_v3{$instance_label=\"$instance\", quantile=\"0.5\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "engine_newPayloadV3 p50", + "range": true, + "refId": "L", "useBackend": false }, { @@ -1955,111 +1670,137 @@ }, "disableTextWrap": false, "editorMode": "builder", - "exemplar": false, - "expr": "sum(reth_database_transaction_closed_total{instance=~\"$instance\", mode=\"read-only\"})", - "format": "time_series", + "expr": "reth_engine_rpc_new_payload_v3{$instance_label=\"$instance\", quantile=\"0.9\"}", "fullMetaSearch": false, + "hide": false, "includeNullMetadata": true, - "instant": false, - "legendFormat": "Closed {{mode}}", + "legendFormat": "engine_newPayloadV3 p90", "range": true, - "refId": "B", + "refId": "M", "useBackend": false }, { "datasource": { - "type": "__expr__", - "uid": "${DS_EXPRESSION}" + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - "expression": "${A} - ${B}", + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_engine_rpc_new_payload_v3{$instance_label=\"$instance\", quantile=\"0.95\"}", + "fullMetaSearch": false, "hide": false, - "refId": "Diff(opened, closed)", - "type": "math" - } - ], - "title": "Number of read-only transactions", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "The size of tables in the database", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "includeNullMetadata": true, + "legendFormat": "engine_newPayloadV3 p95", + "range": true, + "refId": "N", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - } + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_engine_rpc_new_payload_v3{$instance_label=\"$instance\", quantile=\"0.99\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "engine_newPayloadV3 p99", + "range": true, + "refId": "O", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - "mappings": [], - "unit": "bytes" + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_engine_rpc_new_payload_v4{$instance_label=\"$instance\", quantile=\"0\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "engine_newPayloadV4 min", + "range": true, + "refId": "P", + "useBackend": false }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 53 - }, - "id": 48, - "options": { - "displayLabels": [ - "name" - ], - "legend": { - "displayMode": "table", - "placement": "right", - "showLegend": true, - "values": [ - "value" - ] + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_engine_rpc_new_payload_v4{$instance_label=\"$instance\", quantile=\"0.5\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "engine_newPayloadV4 p50", + "range": true, + "refId": "Q", + "useBackend": false }, - "pieType": "pie", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_engine_rpc_new_payload_v4{$instance_label=\"$instance\", quantile=\"0.9\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "engine_newPayloadV4 p90", + "range": true, + "refId": "R", + "useBackend": false }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.3", - "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "disableTextWrap": false, "editorMode": "builder", - "expr": "reth_db_table_size{instance=~\"$instance\"}", - "interval": "", - "legendFormat": "{{table}}", + "expr": "reth_engine_rpc_new_payload_v4{$instance_label=\"$instance\", quantile=\"0.95\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "engine_newPayloadV4 p95", "range": true, - "refId": "A" + "refId": "S", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_engine_rpc_new_payload_v4{$instance_label=\"$instance\", quantile=\"0.99\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "engine_newPayloadV4 p99", + "range": true, + "refId": "T", + "useBackend": false } ], - "title": "Database tables", - "type": "piechart" + "title": "Engine API newPayload Latency", + "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "description": "The maximum time the database transaction operation which inserts a large value took.", + "description": "The metric is the amount of gas processed in a block", "fieldConfig": { "defaults": { "color": { @@ -2073,7 +1814,7 @@ "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, - "drawStyle": "points", + "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { @@ -2089,6 +1830,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -2103,104 +1845,28 @@ "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 80 + "color": "green", + "value": 0 } ] }, - "unit": "s" + "unit": "sishort" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 53 + "x": 0, + "y": 29 }, - "id": 118, + "id": 1004, "options": { "legend": { "calcs": [], "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max(max_over_time(reth_database_operation_large_value_duration_seconds{instance=~\"$instance\", quantile=\"1\"}[$__interval]) > 0) by (table)", - "format": "time_series", - "instant": false, - "legendFormat": "{{table}}", - "range": true, - "refId": "A" - } - ], - "title": "Max insertion operation time", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "The type of the pages in the database:\n\n- **Leaf** pages contain KV pairs.\n- **Branch** pages contain information about keys in the leaf pages\n- **Overflow** pages store large values and should generally be avoided if possible", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - } - }, - "mappings": [], - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 61 - }, - "id": 50, - "options": { - "legend": { - "displayMode": "table", - "placement": "right", - "showLegend": true, - "values": [ - "value" - ] - }, - "pieType": "pie", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + "placement": "bottom", + "showLegend": true }, "tooltip": { "hideZeros": false, @@ -2208,29 +1874,65 @@ "sort": "none" } }, - "pluginVersion": "11.5.3", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "editorMode": "builder", - "expr": "sum by (type) ( reth_db_table_pages{instance=~\"$instance\"} )", - "legendFormat": "__auto", + "editorMode": "code", + "expr": "reth_engine_rpc_new_payload_total_gas{$instance_label=\"$instance\", quantile=\"0.5\"}", + "legendFormat": "p50", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "reth_engine_rpc_new_payload_total_gas{$instance_label=\"$instance\", quantile=\"0.9\"}", + "hide": false, + "legendFormat": "p90", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "reth_engine_rpc_new_payload_total_gas{$instance_label=\"$instance\", quantile=\"0.95\"}", + "hide": false, + "legendFormat": "p95", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "reth_engine_rpc_new_payload_total_gas{$instance_label=\"$instance\", quantile=\"0.99\"}", + "hide": false, + "legendFormat": "p99", + "range": true, + "refId": "D" } ], - "title": "Database pages", - "type": "piechart" + "title": "Engine API newPayload Total Gas", + "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "description": "The size of the database over time", + "description": "The throughput of the Engine API newPayload method. The metric is the amount of gas processed in a block, divided by the time it took to process the newPayload request.", "fieldConfig": { "defaults": { "color": { @@ -2260,6 +1962,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -2269,21 +1972,17 @@ "mode": "off" } }, - "decimals": 4, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 80 + "color": "green", + "value": 0 } ] }, - "unit": "bytes" + "unit": "si: gas/s" }, "overrides": [] }, @@ -2291,9 +1990,9 @@ "h": 8, "w": 12, "x": 12, - "y": 61 + "y": 29 }, - "id": 52, + "id": 1003, "options": { "legend": { "calcs": [], @@ -2307,21 +2006,57 @@ "sort": "none" } }, - "pluginVersion": "11.5.3", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "sum by (job) ( reth_db_table_size{instance=~\"$instance\"} )", - "legendFormat": "Size ({{job}})", + "expr": "reth_engine_rpc_new_payload_gas_per_second{$instance_label=\"$instance\", quantile=\"0.5\"}", + "legendFormat": "p50", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "reth_engine_rpc_new_payload_gas_per_second{$instance_label=\"$instance\", quantile=\"0.9\"}", + "hide": false, + "legendFormat": "p90", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "reth_engine_rpc_new_payload_gas_per_second{$instance_label=\"$instance\", quantile=\"0.95\"}", + "hide": false, + "legendFormat": "p95", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "reth_engine_rpc_new_payload_gas_per_second{$instance_label=\"$instance\", quantile=\"0.99\"}", + "hide": false, + "legendFormat": "p99", + "range": true, + "refId": "D" } ], - "title": "Database growth", + "title": "Engine API newPayload Throughput", "type": "timeseries" }, { @@ -2329,7 +2064,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "The number of pages on the MDBX freelist", + "description": "The throughput of the node's executor. The metric is the amount of gas processed in a block, divided by the time it took to process the block.\n\nNote: For mainnet, the block range 2,383,397-2,620,384 will be slow because of the 2016 DoS attack.", "fieldConfig": { "defaults": { "color": { @@ -2359,6 +2094,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -2373,15 +2109,12 @@ "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 80 + "color": "green", + "value": 0 } ] }, - "unit": "none" + "unit": "si: gas/s" }, "overrides": [] }, @@ -2389,9 +2122,9 @@ "h": 8, "w": 12, "x": 0, - "y": 69 + "y": 37 }, - "id": 113, + "id": 56, "options": { "legend": { "calcs": [], @@ -2405,21 +2138,117 @@ "sort": "none" } }, - "pluginVersion": "11.5.3", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "sum(reth_db_freelist{instance=~\"$instance\"}) by (job)", - "legendFormat": "Pages ({{job}})", + "expr": "reth_sync_execution_gas_per_second{$instance_label=\"$instance\"}", + "legendFormat": "Gas/s", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "avg_over_time(reth_sync_execution_gas_per_second{$instance_label=\"$instance\"}[1m])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "Avg Gas/s (1m)", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "avg_over_time(reth_sync_execution_gas_per_second{$instance_label=\"$instance\"}[5m])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "Avg Gas/s (5m)", + "range": true, + "refId": "C", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "avg_over_time(reth_sync_execution_gas_per_second{$instance_label=\"$instance\"}[10m])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "Avg Gas/s (10m)", + "range": true, + "refId": "D", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "avg_over_time(reth_sync_execution_gas_per_second{$instance_label=\"$instance\"}[30m])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "Avg Gas/s (30m)", + "range": true, + "refId": "E", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "avg_over_time(reth_sync_execution_gas_per_second{$instance_label=\"$instance\"}[1h])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "Avg Gas/s (1h)", + "range": true, + "refId": "F", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "avg_over_time(reth_sync_execution_gas_per_second{$instance_label=\"$instance\"}[24h])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "Avg Gas/s (24h)", + "range": true, + "refId": "G", + "useBackend": false } ], - "title": "Freelist", + "title": "Execution throughput", "type": "timeseries" }, { @@ -2430,158 +2259,118 @@ "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" + "mode": "palette-classic" }, "custom": { - "align": "left", - "cellOptions": { - "type": "auto" - }, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Time" - }, - "properties": [ - { - "id": "custom.hidden", - "value": true - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "__name__" - }, - "properties": [ - { - "id": "custom.hidden", - "value": true - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "instance" - }, - "properties": [ - { - "id": "custom.hidden", - "value": true - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "job" + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "properties": [ - { - "id": "custom.hidden", - "value": true - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "type" + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "properties": [ - { - "id": "custom.hidden", - "value": true - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value" + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, - "properties": [ + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "id": "unit", - "value": "locale" + "color": "green", + "value": 0 }, { - "id": "displayName", - "value": "Overflow pages" + "color": "red", + "value": 80 } ] }, - { - "matcher": { - "id": "byName", - "options": "table" - }, - "properties": [ - { - "id": "displayName", - "value": "Table" - } - ] - } - ] + "unit": "s" + }, + "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 69 + "y": 37 }, - "id": 58, + "id": 240, "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "showHeader": true + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } }, - "pluginVersion": "11.5.3", + "pluginVersion": "12.2.1", "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "avg(reth_sync_block_validation_state_root_duration{$instance_label=\"$instance\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "State Root Duration", + "range": true, + "refId": "A", + "useBackend": false + }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "editorMode": "code", - "exemplar": false, - "expr": "sort_desc(reth_db_table_pages{instance=~\"$instance\", type=\"overflow\"} != 0)", - "format": "table", - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "A" + "disableTextWrap": false, + "editorMode": "builder", + "expr": "avg(reth_sync_execution_execution_duration{$instance_label=\"$instance\"})", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Execution Duration", + "range": true, + "refId": "B", + "useBackend": false } ], - "title": "Overflow pages by table", - "type": "table" + "title": "Block Processing Latency", + "type": "timeseries" }, { "collapsed": false, @@ -2589,62 +2378,89 @@ "h": 1, "w": 24, "x": 0, - "y": 77 + "y": 45 }, - "id": 203, + "id": 87, "panels": [], - "title": "Static Files", + "repeat": "instance", + "title": "Engine API", "type": "row" }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "description": "The size of segments in the static files", + "description": "Engine API messages received by the CL, either engine_newPayload or engine_forkchoiceUpdated", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, "mappings": [], - "unit": "bytes" + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } }, "overrides": [] }, "gridPos": { "h": 8, - "w": 8, + "w": 12, "x": 0, - "y": 78 + "y": 46 }, - "id": 202, + "id": 84, "options": { - "displayLabels": [ - "name" - ], "legend": { - "displayMode": "table", - "placement": "right", - "showLegend": true, - "values": [ - "value" - ] - }, - "pieType": "pie", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, "tooltip": { "hideZeros": false, @@ -2652,7 +2468,7 @@ "sort": "none" } }, - "pluginVersion": "11.5.3", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { @@ -2660,320 +2476,244 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "reth_static_files_segment_size{instance=~\"$instance\"}", - "interval": "", - "legendFormat": "{{segment}}", + "expr": "rate(reth_consensus_engine_beacon_forkchoice_updated_messages{$instance_label=\"$instance\"}[$__rate_interval])", + "legendFormat": "forkchoiceUpdated", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(reth_consensus_engine_beacon_new_payload_messages{$instance_label=\"$instance\"}[$__rate_interval])", + "hide": false, + "legendFormat": "newPayload", + "range": true, + "refId": "B" } ], - "title": "Segments size", - "type": "piechart" + "title": "Engine API messages", + "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "description": "Counts the number of failed response deliveries due to client request termination.", "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" + "mode": "palette-classic" }, "custom": { - "align": "left", - "cellOptions": { - "type": "auto" - }, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Value" + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "properties": [ - { - "id": "unit", - "value": "locale" - }, - { - "id": "displayName", - "value": "Entries" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "segment" + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "properties": [ - { - "id": "displayName", - "value": "Segment" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Time" + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, - "properties": [ - { - "id": "custom.hidden", - "value": true - } - ] + "thresholdsStyle": { + "mode": "off" + } }, - { - "matcher": { - "id": "byName", - "options": "instance" - }, - "properties": [ + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "id": "custom.hidden", - "value": true - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "job" - }, - "properties": [ + "color": "green", + "value": 0 + }, { - "id": "custom.hidden", - "value": true + "color": "red", + "value": 80 } ] }, - { - "matcher": { - "id": "byName", - "options": "__name__" - }, - "properties": [ - { - "id": "custom.hidden", - "value": true - } - ] - } - ] + "unit": "none" + }, + "overrides": [] }, "gridPos": { "h": 8, - "w": 8, - "x": 8, - "y": 78 + "w": 12, + "x": 12, + "y": 46 }, - "id": 204, + "id": 249, "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "showHeader": true + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } }, - "pluginVersion": "11.5.3", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "exemplar": false, - "expr": "reth_static_files_segment_entries{instance=~\"$instance\"}", - "format": "table", - "instant": true, - "legendFormat": "__auto", - "range": false, + "expr": "rate(reth_consensus_engine_beacon_failed_new_payload_response_deliveries{$instance_label=\"$instance\"}[$__rate_interval])", + "legendFormat": "newPayload", + "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(reth_consensus_engine_beacon_failed_forkchoice_updated_response_deliveries{$instance_label=\"$instance\"}[$__rate_interval])", + "legendFormat": "forkchoiceUpdated", + "range": true, + "refId": "B" } ], - "title": "Entries per segment", - "type": "table" + "title": "Failed Engine API Response Deliveries", + "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, + "description": "Latency histogram for the engine_newPayload to engine_forkchoiceUpdated", "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" + "mode": "palette-classic" }, "custom": { - "align": "left", - "cellOptions": { - "type": "auto" + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "inspect": false + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", "value": 80 } ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Value" - }, - "properties": [ - { - "id": "unit", - "value": "locale" - }, - { - "id": "displayName", - "value": "Files" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "segment" - }, - "properties": [ - { - "id": "displayName", - "value": "Segment" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Time" - }, - "properties": [ - { - "id": "custom.hidden", - "value": true - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "instance" - }, - "properties": [ - { - "id": "custom.hidden", - "value": true - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "job" - }, - "properties": [ - { - "id": "custom.hidden", - "value": true - } - ] }, - { - "matcher": { - "id": "byName", - "options": "__name__" - }, - "properties": [ - { - "id": "custom.hidden", - "value": true - } - ] - } - ] + "unit": "s" + }, + "overrides": [] }, "gridPos": { "h": 8, - "w": 8, - "x": 16, - "y": 78 + "w": 12, + "x": 0, + "y": 54 }, - "id": 205, + "id": 213, "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "showHeader": true + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } }, - "pluginVersion": "11.5.3", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "reth_static_files_segment_files{instance=~\"$instance\"}", - "format": "table", - "instant": true, - "legendFormat": "__auto", - "range": false, + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_engine_rpc_new_payload_forkchoice_updated_time_diff{$instance_label=\"$instance\"}", + "legendFormat": "p{{quantile}}", + "range": true, "refId": "A" } ], - "title": "Files per segment", - "type": "table" + "title": "Engine API latency between forkchoiceUpdated and newPayload", + "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "description": "The size of the static files over time", + "description": "Latency histograms for the engine_getPayloadBodiesByHashV1 and engine_getPayloadBodiesByRangeV1 RPC APIs", "fieldConfig": { "defaults": { "color": { @@ -3003,6 +2743,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -3017,7 +2758,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -3025,17 +2767,17 @@ } ] }, - "unit": "bytes" + "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 86 + "x": 12, + "y": 54 }, - "id": 206, + "id": 212, "options": { "legend": { "calcs": [], @@ -3049,21 +2791,170 @@ "sort": "none" } }, - "pluginVersion": "11.5.3", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "editorMode": "code", - "expr": "sum by (job) ( reth_static_files_segment_size{instance=~\"$instance\"} )", - "legendFormat": "__auto", + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_engine_rpc_get_payload_bodies_by_hash_v1{$instance_label=\"$instance\", quantile=\"0\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "engine_getPayloadBodiesByHashV1 min", "range": true, - "refId": "A" + "refId": "O", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_engine_rpc_get_payload_bodies_by_hash_v1{$instance_label=\"$instance\", quantile=\"0.5\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "engine_getPayloadBodiesByHashV1 p50", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_engine_rpc_get_payload_bodies_by_hash_v1{$instance_label=\"$instance\", quantile=\"0.9\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "engine_getPayloadBodiesByHashV1 p90", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_engine_rpc_get_payload_bodies_by_hash_v1{$instance_label=\"$instance\", quantile=\"0.95\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "engine_getPayloadBodiesByHashV1 p95", + "range": true, + "refId": "C", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_engine_rpc_get_payload_bodies_by_hash_v1{$instance_label=\"$instance\", quantile=\"0.99\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "engine_getPayloadBodiesByHashV1 p99", + "range": true, + "refId": "D", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_engine_rpc_get_payload_bodies_by_range_v1{$instance_label=\"$instance\", quantile=\"0\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "engine_getPayloadBodiesByRangeV1 min", + "range": true, + "refId": "E", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_engine_rpc_get_payload_bodies_by_range_v1{$instance_label=\"$instance\", quantile=\"0.5\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "engine_getPayloadBodiesByRangeV1 p50", + "range": true, + "refId": "F", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_engine_rpc_get_payload_bodies_by_range_v1{$instance_label=\"$instance\", quantile=\"0.9\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "engine_getPayloadBodiesByRangeV1 p90", + "range": true, + "refId": "G", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_engine_rpc_get_payload_bodies_by_range_v1{$instance_label=\"$instance\", quantile=\"0.95\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "engine_getPayloadBodiesByRangeV1 p95", + "range": true, + "refId": "H", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_engine_rpc_get_payload_bodies_by_range_v1{$instance_label=\"$instance\", quantile=\"0.99\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "engine_getPayloadBodiesByRangeV1 p99", + "range": true, + "refId": "I", + "useBackend": false } ], - "title": "Static Files growth", + "title": "Engine API getPayloadBodies Latency", "type": "timeseries" }, { @@ -3071,7 +2962,6 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "The maximum time the static files operation which commits a writer took.", "fieldConfig": { "defaults": { "color": { @@ -3085,8 +2975,8 @@ "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, + "drawStyle": "line", + "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, @@ -3100,7 +2990,8 @@ "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", + "showPoints": "never", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -3115,25 +3006,22 @@ "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 80 + "color": "green", + "value": 0 } ] }, - "unit": "s" + "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 86 + "x": 0, + "y": 62 }, - "id": 207, + "id": 1000, "options": { "legend": { "calcs": [], @@ -3147,43 +3035,40 @@ "sort": "none" } }, - "pluginVersion": "11.5.3", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "max(max_over_time(reth_static_files_jar_provider_write_duration_seconds{instance=~\"$instance\", operation=\"commit-writer\", quantile=\"1\"}[$__interval]) > 0) by (segment)", - "legendFormat": "{{segment}}", + "expr": "rate(reth_engine_rpc_blobs_blob_count{$instance_label=\"$instance\"}[$__rate_interval])", + "legendFormat": "Found", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(reth_engine_rpc_blobs_blob_misses{$instance_label=\"$instance\"}[$__rate_interval])", + "hide": false, + "legendFormat": "Missed", + "range": true, + "refId": "B" } ], - "title": "Max writer commit time", + "title": "Blob Count and Misses", "type": "timeseries" }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 94 - }, - "id": 46, - "panels": [], - "repeat": "instance", - "title": "Execution", - "type": "row" - }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "description": "The throughput of the node's executor. The metric is the amount of gas processed in a block, divided by the time it took to process the block.\n\nNote: For mainnet, the block range 2,383,397-2,620,384 will be slow because of the 2016 DoS attack.", "fieldConfig": { "defaults": { "color": { @@ -3213,6 +3098,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -3227,21 +3113,26 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 } ] }, - "unit": "si: gas/s" + "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, - "w": 24, - "x": 0, - "y": 95 + "w": 12, + "x": 12, + "y": 62 }, - "id": 56, + "id": 258, "options": { "legend": { "calcs": [], @@ -3249,41 +3140,14 @@ "placement": "bottom", "showLegend": true }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "reth_sync_execution_gas_per_second{instance=~\"$instance\"}", - "legendFormat": "Gas/s", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "avg_over_time(reth_sync_execution_gas_per_second{instance=~\"$instance\"}[1m])", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "legendFormat": "Avg Gas/s (1m)", - "range": true, - "refId": "B", - "useBackend": false - }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ { "datasource": { "type": "prometheus", @@ -3291,29 +3155,29 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "avg_over_time(reth_sync_execution_gas_per_second{instance=~\"$instance\"}[5m])", + "expr": "reth_engine_rpc_get_blobs_v1{$instance_label=\"$instance\", quantile=\"0.5\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, - "legendFormat": "Avg Gas/s (5m)", + "legendFormat": "engine_getBlobsV1 p50", "range": true, - "refId": "C", + "refId": "A", "useBackend": false }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "disableTextWrap": false, "editorMode": "builder", - "expr": "avg_over_time(reth_sync_execution_gas_per_second{instance=~\"$instance\"}[10m])", + "expr": "reth_engine_rpc_get_blobs_v1{$instance_label=\"$instance\", quantile=\"0.95\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, - "legendFormat": "Avg Gas/s (10m)", + "legendFormat": "engine_getBlobsV1 p95", "range": true, - "refId": "D", + "refId": "B", "useBackend": false }, { @@ -3323,29 +3187,29 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "avg_over_time(reth_sync_execution_gas_per_second{instance=~\"$instance\"}[30m])", + "expr": "reth_engine_rpc_get_blobs_v1{$instance_label=\"$instance\", quantile=\"0.99\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, - "legendFormat": "Avg Gas/s (30m)", + "legendFormat": "engine_getBlobsV1 p99", "range": true, - "refId": "E", + "refId": "C", "useBackend": false }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "disableTextWrap": false, "editorMode": "builder", - "expr": "avg_over_time(reth_sync_execution_gas_per_second{instance=~\"$instance\"}[1h])", + "expr": "reth_engine_rpc_get_blobs_v1{$instance_label=\"$instance\", quantile=\"0\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, - "legendFormat": "Avg Gas/s (1h)", + "legendFormat": "engine_getBlobsV1 min", "range": true, - "refId": "F", + "refId": "D", "useBackend": false }, { @@ -3355,24 +3219,25 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "avg_over_time(reth_sync_execution_gas_per_second{instance=~\"$instance\"}[24h])", + "expr": "reth_engine_rpc_get_blobs_v1{$instance_label=\"$instance\", quantile=\"1\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, - "legendFormat": "Avg Gas/s (24h)", + "legendFormat": "engine_getBlobsV1 max", "range": true, - "refId": "G", + "refId": "E", "useBackend": false } ], - "title": "Execution throughput", + "title": "Engine API getBlobs Latency", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, + "description": "Total pipeline runs triggered by the sync controller", "fieldConfig": { "defaults": { "color": { @@ -3387,7 +3252,7 @@ "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 25, + "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, @@ -3402,10 +3267,11 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", - "mode": "percent" + "mode": "none" }, "thresholdsStyle": { "mode": "off" @@ -3416,31 +3282,31 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", "value": 80 } ] - }, - "unit": "s" + } }, "overrides": [] }, "gridPos": { - "h": 11, - "w": 24, + "h": 8, + "w": 12, "x": 0, - "y": 103 + "y": 70 }, - "id": 240, + "id": 85, "options": { "legend": { "calcs": [], - "displayMode": "hidden", - "placement": "right", - "showLegend": false + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, "tooltip": { "hideZeros": false, @@ -3448,50 +3314,29 @@ "sort": "none" } }, - "pluginVersion": "11.5.3", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_sync_block_validation_state_root_duration{instance=\"$instance\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "State Root Duration", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, "editorMode": "builder", - "expr": "reth_sync_execution_execution_duration{instance=\"$instance\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Execution Duration", + "expr": "reth_consensus_engine_beacon_pipeline_runs{$instance_label=\"$instance\"}", + "legendFormat": "Pipeline runs", "range": true, - "refId": "B", - "useBackend": false + "refId": "A" } ], - "title": "Block Processing Latency", + "title": "Pipeline runs", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, + "description": "", "fieldConfig": { "defaults": { "color": { @@ -3521,6 +3366,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -3535,31 +3381,31 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", "value": 80 } ] - }, - "unit": "percentunit" + } }, "overrides": [] }, "gridPos": { - "h": 11, - "w": 24, - "x": 0, - "y": 114 + "h": 8, + "w": 12, + "x": 12, + "y": 70 }, - "id": 251, + "id": 83, "options": { "legend": { "calcs": [], - "displayMode": "hidden", - "placement": "right", - "showLegend": false + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, "tooltip": { "hideZeros": false, @@ -3567,69 +3413,42 @@ "sort": "none" } }, - "pluginVersion": "11.5.3", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "reth_sync_caching_account_cache_hits{instance=\"$instance\"} / (reth_sync_caching_account_cache_hits{instance=\"$instance\"} + reth_sync_caching_account_cache_misses{instance=\"$instance\"})", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Account cache hits", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "reth_sync_caching_storage_cache_hits{instance=\"$instance\"} / (reth_sync_caching_storage_cache_hits{instance=\"$instance\"} + reth_sync_caching_storage_cache_misses{instance=\"$instance\"})", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Storage cache hits", - "range": true, - "refId": "B", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "reth_sync_caching_code_cache_hits{instance=\"$instance\"} / (reth_sync_caching_code_cache_hits{instance=\"$instance\"} + reth_sync_caching_code_cache_misses{instance=\"$instance\"})", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Code cache hits", + "editorMode": "builder", + "expr": "reth_consensus_engine_beacon_active_block_downloads{$instance_label=\"$instance\"}", + "legendFormat": "Active block downloads", "range": true, - "refId": "C", - "useBackend": false + "refId": "A" } ], - "title": "Execution cache hitrate", + "title": "Active block downloads", "type": "timeseries" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 78 + }, + "id": 46, + "panels": [], + "repeat": "instance", + "title": "Execution", + "type": "row" + }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "description": "The time it takes for operations that are part of block validation, but not execution or state root, to complete.", "fieldConfig": { "defaults": { "color": { @@ -3644,7 +3463,7 @@ "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 24, "gradientMode": "none", "hideFrom": { "legend": false, @@ -3659,10 +3478,11 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", - "mode": "none" + "mode": "percent" }, "thresholdsStyle": { "mode": "off" @@ -3673,7 +3493,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -3686,12 +3507,12 @@ "overrides": [] }, "gridPos": { - "h": 11, - "w": 24, + "h": 8, + "w": 12, "x": 0, - "y": 125 + "y": 79 }, - "id": 252, + "id": 1001, "options": { "legend": { "calcs": [], @@ -3705,7 +3526,7 @@ "sort": "none" } }, - "pluginVersion": "11.5.3", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { @@ -3713,35 +3534,37 @@ "uid": "${datasource}" }, "disableTextWrap": false, - "editorMode": "code", - "expr": "reth_sync_block_validation_trie_input_duration{instance=\"$instance\", quantile=~\"(0|0.5|0.9|0.95|1)\"}", + "editorMode": "builder", + "expr": "avg(reth_sync_block_validation_state_root_duration{$instance_label=\"$instance\"})", "fullMetaSearch": false, - "hide": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "Trie input creation duration p{{quantile}}", + "legendFormat": "State Root Duration", "range": true, "refId": "A", "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "avg(reth_sync_execution_execution_duration{$instance_label=\"$instance\"})", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Execution Duration", + "range": true, + "refId": "B", + "useBackend": false } ], - "title": "Block validation overhead", + "title": "Block Processing Latency", "type": "timeseries" }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 136 - }, - "id": 24, - "panels": [], - "repeat": "instance", - "title": "Downloader: Headers", - "type": "row" - }, { "datasource": { "type": "prometheus", @@ -3776,6 +3599,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -3790,49 +3614,26 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", "value": 80 } ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byFrameRefID", - "options": "C" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" - } - ] }, - { - "matcher": { - "id": "byFrameRefID", - "options": "D" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" - } - ] - } - ] + "unit": "percentunit" + }, + "overrides": [] }, "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 137 + "x": 12, + "y": 79 }, - "id": 26, + "id": 251, "options": { "legend": { "calcs": [], @@ -3842,62 +3643,65 @@ }, "tooltip": { "hideZeros": false, - "mode": "multi", + "mode": "single", "sort": "none" } }, - "pluginVersion": "11.5.3", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "builder", - "expr": "reth_downloaders_headers_total_downloaded{instance=~\"$instance\"}", - "legendFormat": "Downloaded", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "editorMode": "builder", - "expr": "reth_downloaders_headers_total_flushed{instance=~\"$instance\"}", + "disableTextWrap": false, + "editorMode": "code", + "expr": "reth_sync_caching_account_cache_hits{$instance_label=\"$instance\"} / (reth_sync_caching_account_cache_hits{$instance_label=\"$instance\"} + reth_sync_caching_account_cache_misses{$instance_label=\"$instance\"})", + "fullMetaSearch": false, "hide": false, - "legendFormat": "Flushed", + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Account cache hits", "range": true, - "refId": "B" + "refId": "A", + "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "editorMode": "builder", - "expr": "rate(reth_downloaders_headers_total_downloaded{instance=~\"$instance\"}[$__rate_interval])", + "disableTextWrap": false, + "editorMode": "code", + "expr": "reth_sync_caching_storage_cache_hits{$instance_label=\"$instance\"} / (reth_sync_caching_storage_cache_hits{$instance_label=\"$instance\"} + reth_sync_caching_storage_cache_misses{$instance_label=\"$instance\"})", + "fullMetaSearch": false, "hide": false, + "includeNullMetadata": true, "instant": false, - "legendFormat": "Downloaded/s", + "legendFormat": "Storage cache hits", "range": true, - "refId": "C" + "refId": "B", + "useBackend": false }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "editorMode": "builder", - "expr": "rate(reth_downloaders_headers_total_flushed{instance=~\"$instance\"}[$__rate_interval])", + "disableTextWrap": false, + "editorMode": "code", + "expr": "reth_sync_caching_code_cache_hits{$instance_label=\"$instance\"} / (reth_sync_caching_code_cache_hits{$instance_label=\"$instance\"} + reth_sync_caching_code_cache_misses{$instance_label=\"$instance\"})", + "fullMetaSearch": false, "hide": false, - "legendFormat": "Flushed/s", + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Code cache hits", "range": true, - "refId": "D" + "refId": "C", + "useBackend": false } ], - "title": "I/O", + "title": "Execution cache hitrate", "type": "timeseries" }, { @@ -3905,7 +3709,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "Internal errors in the header downloader. These are expected to happen from time to time.", + "description": "The time it takes for operations that are part of block validation, but not execution or state root, to complete.", "fieldConfig": { "defaults": { "color": { @@ -3935,6 +3739,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -3949,7 +3754,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -3957,17 +3763,17 @@ } ] }, - "unit": "cps" + "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 137 + "x": 0, + "y": 87 }, - "id": 33, + "id": 252, "options": { "legend": { "calcs": [], @@ -3981,45 +3787,27 @@ "sort": "none" } }, - "pluginVersion": "11.5.3", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "builder", - "expr": "rate(reth_downloaders_headers_timeout_errors{instance=~\"$instance\"}[$__rate_interval])", - "legendFormat": "Request timed out", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "builder", - "expr": "rate(reth_downloaders_headers_unexpected_errors{instance=~\"$instance\"}[$__rate_interval])", - "hide": false, - "legendFormat": "Unexpected error", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "editorMode": "builder", - "expr": "rate(reth_downloaders_headers_validation_errors{instance=~\"$instance\"}[$__rate_interval])", + "disableTextWrap": false, + "editorMode": "code", + "expr": "avg by(quantile) (reth_sync_block_validation_trie_input_duration{$instance_label=\"$instance\", quantile=~\"(0|0.5|0.9|0.95|1)\"})", + "fullMetaSearch": false, "hide": false, - "legendFormat": "Invalid response", + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Trie input creation duration p{{quantile}}", "range": true, - "refId": "C" + "refId": "A", + "useBackend": false } ], - "title": "Errors", + "title": "Block validation overhead", "type": "timeseries" }, { @@ -4027,7 +3815,6 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "The number of connected peers and in-progress requests for headers.", "fieldConfig": { "defaults": { "color": { @@ -4057,6 +3844,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -4067,28 +3855,54 @@ } }, "mappings": [], + "max": 1, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", "value": 80 } ] - } + }, + "unit": "percentunit" }, - "overrides": [] + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": ["Precompile cache hits"], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": true, + "viz": true + } + } + ] + } + ] }, "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 145 + "x": 12, + "y": 87 }, - "id": 36, + "id": 1005, "options": { "legend": { "calcs": [], @@ -4097,37 +3911,32 @@ "showLegend": true }, "tooltip": { - "mode": "multi", + "hideZeros": false, + "mode": "single", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "builder", - "expr": "reth_downloaders_headers_in_flight_requests{instance=~\"$instance\"}", - "legendFormat": "In flight requests", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "editorMode": "builder", - "expr": "reth_network_connected_peers{instance=~\"$instance\"}", + "disableTextWrap": false, + "editorMode": "code", + "expr": "reth_sync_caching_precompile_cache_hits{$instance_label=\"$instance\"} / (reth_sync_caching_precompile_cache_hits{$instance_label=\"$instance\"} + reth_sync_caching_precompile_cache_misses{$instance_label=\"$instance\"})", + "fullMetaSearch": false, "hide": false, - "legendFormat": "Connected peers", + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{address}}", "range": true, - "refId": "B" + "refId": "A", + "useBackend": false } ], - "title": "Requests", + "title": "Precompile cache hitrate", "type": "timeseries" }, { @@ -4136,12 +3945,11 @@ "h": 1, "w": 24, "x": 0, - "y": 153 + "y": 95 }, - "id": 32, + "id": 214, "panels": [], - "repeat": "instance", - "title": "Downloader: Bodies", + "title": "State Root Task", "type": "row" }, { @@ -4149,7 +3957,6 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "The internal state of the headers downloader: the number of downloaded headers, and the number of headers sent to the header stage.", "fieldConfig": { "defaults": { "color": { @@ -4179,6 +3986,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -4193,157 +4001,54 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", "value": 80 } ] - }, - "unit": "locale" - }, - "overrides": [ - { - "matcher": { - "id": "byFrameRefID", - "options": "C" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "unit", - "value": "ops" - } - ] - }, - { - "matcher": { - "id": "byFrameRefID", - "options": "D" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "unit", - "value": "ops" - } - ] } - ] + }, + "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 154 - }, - "id": 30, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } + "y": 96 }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "builder", - "expr": "reth_downloaders_bodies_total_downloaded{instance=~\"$instance\"}", - "legendFormat": "Downloaded", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "builder", - "expr": "reth_downloaders_bodies_total_flushed{instance=~\"$instance\"}", - "hide": false, - "legendFormat": "Flushed", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "builder", - "expr": "rate(reth_downloaders_bodies_total_flushed{instance=~\"$instance\"}[$__rate_interval])", - "hide": false, - "legendFormat": "Flushed/s", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "builder", - "expr": "rate(reth_downloaders_bodies_total_downloaded{instance=~\"$instance\"}[$__rate_interval])", - "hide": false, - "legendFormat": "Downloaded/s", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "builder", - "expr": "reth_downloaders_bodies_buffered_responses{instance=~\"$instance\"}", - "hide": false, - "legendFormat": "Buffered responses", - "range": true, - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "builder", - "expr": "reth_downloaders_bodies_buffered_blocks{instance=~\"$instance\"}", - "hide": false, - "legendFormat": "Buffered blocks", - "range": true, - "refId": "F" + "id": 255, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "editorMode": "builder", - "expr": "reth_downloaders_bodies_queued_blocks{instance=~\"$instance\"}", - "hide": false, - "legendFormat": "Queued blocks", + "editorMode": "code", + "expr": "reth_tree_root_proofs_processed_histogram{$instance_label=\"$instance\",quantile=~\"(0|0.5|0.9|0.95|1)\"}", + "instant": false, + "legendFormat": "{{quantile}} percentile", "range": true, - "refId": "G" + "refId": "Branch Nodes" } ], - "title": "I/O", + "title": "Proofs Processed", "type": "timeseries" }, { @@ -4351,7 +4056,6 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "Internal errors in the bodies downloader. These are expected to happen from time to time.", "fieldConfig": { "defaults": { "color": { @@ -4381,6 +4085,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -4391,16 +4096,20 @@ } }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 } ] }, - "unit": "cps" + "unit": "s" }, "overrides": [] }, @@ -4408,9 +4117,9 @@ "h": 8, "w": 12, "x": 12, - "y": 154 + "y": 96 }, - "id": 28, + "id": 254, "options": { "legend": { "calcs": [], @@ -4419,49 +4128,27 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "builder", - "expr": "rate(reth_downloaders_bodies_timeout_errors{instance=~\"$instance\"}[$__rate_interval])", - "legendFormat": "Request timed out", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "builder", - "expr": "rate(reth_downloaders_bodies_unexpected_errors{instance=~\"$instance\"}[$__rate_interval])", - "hide": false, - "legendFormat": "Unexpected error", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "editorMode": "builder", - "expr": "rate(reth_downloaders_bodies_validation_errors{instance=~\"$instance\"}[$__rate_interval])", - "hide": false, - "legendFormat": "Invalid response", + "editorMode": "code", + "expr": "avg by (quantile) (reth_tree_root_proof_calculation_duration_histogram{$instance_label=\"$instance\",quantile=~\"(0|0.5|0.9|0.95|1)\"})", + "instant": false, + "legendFormat": "{{quantile}} percentile", "range": true, - "refId": "C" + "refId": "Branch Nodes" } ], - "title": "Errors", + "title": "Proof calculation duration", "type": "timeseries" }, { @@ -4469,7 +4156,6 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "The number of connected peers and in-progress requests for bodies.", "fieldConfig": { "defaults": { "color": { @@ -4499,6 +4185,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -4513,14 +4200,16 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", "value": 80 } ] - } + }, + "unit": "none" }, "overrides": [] }, @@ -4528,9 +4217,9 @@ "h": 8, "w": 12, "x": 0, - "y": 162 + "y": 104 }, - "id": 35, + "id": 257, "options": { "legend": { "calcs": [], @@ -4539,37 +4228,39 @@ "showLegend": true }, "tooltip": { - "mode": "multi", + "hideZeros": false, + "mode": "single", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "editorMode": "builder", - "expr": "reth_downloaders_bodies_in_flight_requests{instance=~\"$instance\"}", - "legendFormat": "In flight requests", + "editorMode": "code", + "expr": "reth_tree_root_pending_storage_multiproofs_histogram{$instance_label=\"$instance\",quantile=~\"(0|0.5|0.9|0.95|1)\"}", + "instant": false, + "legendFormat": "storage {{quantile}} percentile", "range": true, - "refId": "A" + "refId": "Storage" }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "editorMode": "builder", - "expr": "reth_network_connected_peers{instance=~\"$instance\"}", - "hide": false, - "legendFormat": "Connected peers", + "editorMode": "code", + "expr": "reth_tree_root_pending_account_multiproofs_histogram{$instance_label=\"$instance\",quantile=~\"(0|0.5|0.9|0.95|1)\"}", + "instant": false, + "legendFormat": "account {{quantile}} percentile", "range": true, - "refId": "B" + "refId": "Account" } ], - "title": "Requests", + "title": "Pending MultiProof requests", "type": "timeseries" }, { @@ -4577,7 +4268,6 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "The number of blocks and size in bytes of those blocks", "fieldConfig": { "defaults": { "color": { @@ -4607,6 +4297,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -4621,7 +4312,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -4629,22 +4321,36 @@ } ] }, - "unit": "bytes" + "unit": "none" }, "overrides": [ { "matcher": { - "id": "byFrameRefID", - "options": "B" + "id": "byName", + "options": "Max storage workers" }, "properties": [ { - "id": "custom.axisPlacement", - "value": "right" - }, + "id": "custom.lineStyle", + "value": { + "dash": [10, 10], + "fill": "dash" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Max account workers" + }, + "properties": [ { - "id": "unit", - "value": "blocks" + "id": "custom.lineStyle", + "value": { + "dash": [10, 10], + "fill": "dash" + } } ] } @@ -4654,9 +4360,10 @@ "h": 8, "w": 12, "x": 12, - "y": 162 + "y": 104 }, - "id": 73, + "description": "The max metrics (Max storage workers and Max account workers) are displayed as dotted lines to highlight the configured upper limits.", + "id": 256, "options": { "legend": { "calcs": [], @@ -4665,38 +4372,63 @@ "showLegend": true }, "tooltip": { - "mode": "multi", + "hideZeros": false, + "mode": "single", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "editorMode": "builder", - "expr": "reth_downloaders_bodies_buffered_blocks_size_bytes{instance=~\"$instance\"}", - "hide": false, - "legendFormat": "Buffered blocks size ", + "editorMode": "code", + "expr": "reth_tree_root_active_storage_workers_histogram{$instance_label=\"$instance\",quantile=~\"(0|0.5|0.9|0.95|1)\"}", + "instant": false, + "legendFormat": "Storage workers {{quantile}} percentile", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "editorMode": "builder", - "expr": "reth_downloaders_bodies_buffered_blocks{instance=~\"$instance\"}", - "hide": false, - "legendFormat": "Buffered blocks", + "editorMode": "code", + "expr": "reth_tree_root_active_account_workers_histogram{$instance_label=\"$instance\",quantile=~\"(0|0.5|0.9|0.95|1)\"}", + "instant": false, + "legendFormat": "Account workers {{quantile}} percentile", "range": true, "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "reth_tree_root_max_storage_workers{$instance_label=\"$instance\"}", + "instant": false, + "legendFormat": "Max storage workers", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "reth_tree_root_max_account_workers{$instance_label=\"$instance\"}", + "instant": false, + "legendFormat": "Max account workers", + "range": true, + "refId": "D" } ], - "title": "Downloader buffer", + "title": "Active MultiProof Workers", "type": "timeseries" }, { @@ -4704,7 +4436,6 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "The number of blocks in a request and size in bytes of those block responses", "fieldConfig": { "defaults": { "color": { @@ -4734,6 +4465,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -4748,7 +4480,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -4756,34 +4489,17 @@ } ] }, - "unit": "bytes" + "unit": "none" }, - "overrides": [ - { - "matcher": { - "id": "byFrameRefID", - "options": "B" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "unit", - "value": "blocks" - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 170 + "y": 112 }, - "id": 102, + "id": 260, "options": { "legend": { "calcs": [], @@ -4792,73 +4508,34 @@ "showLegend": true }, "tooltip": { - "mode": "multi", + "hideZeros": false, + "mode": "single", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "builder", - "expr": "reth_downloaders_bodies_response_response_size_bytes{instance=~\"$instance\"}", - "hide": false, - "legendFormat": "Response size", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "builder", - "expr": "reth_downloaders_bodies_response_response_length{instance=~\"$instance\"}", - "hide": false, - "legendFormat": "Individual response length (number of bodies in response)", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "editorMode": "builder", - "expr": "reth_downloaders_bodies_response_response_size_bytes / reth_downloaders_bodies_response_response_length{instance=~\"$instance\"}", - "hide": false, + "editorMode": "code", + "expr": "reth_sparse_state_trie_multiproof_total_account_nodes{$instance_label=\"$instance\",quantile=~\"(0|0.5|0.9|0.95|1)\"}", "instant": false, - "legendFormat": "Mean body size in response", + "legendFormat": "Account {{quantile}} percentile", "range": true, - "refId": "C" + "refId": "Branch Nodes" } ], - "title": "Block body response sizes", + "title": "Total multiproof account nodes", "type": "timeseries" }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 178 - }, - "id": 79, - "panels": [], - "repeat": "instance", - "title": "Blockchain Tree", - "type": "row" - }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "description": "The block number of the tip of the canonical chain from the blockchain tree.", "fieldConfig": { "defaults": { "color": { @@ -4888,6 +4565,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -4902,24 +4580,26 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", "value": 80 } ] - } + }, + "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 179 + "x": 12, + "y": 112 }, - "id": 74, + "id": 259, "options": { "legend": { "calcs": [], @@ -4928,26 +4608,27 @@ "showLegend": true }, "tooltip": { - "mode": "multi", + "hideZeros": false, + "mode": "single", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "editorMode": "builder", - "expr": "reth_blockchain_tree_canonical_chain_height{instance=~\"$instance\"}", - "hide": false, - "legendFormat": "Canonical chain height", + "editorMode": "code", + "expr": "reth_sparse_state_trie_multiproof_total_storage_nodes{$instance_label=\"$instance\",quantile=~\"(0|0.5|0.9|0.95|1)\"}", + "instant": false, + "legendFormat": "Storage {{quantile}} percentile", "range": true, - "refId": "B" + "refId": "Branch Nodes" } ], - "title": "Canonical chain height", + "title": "Total multiproof storage nodes", "type": "timeseries" }, { @@ -4955,7 +4636,6 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "Total number of blocks in the tree's block buffer", "fieldConfig": { "defaults": { "color": { @@ -4985,6 +4665,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -4999,24 +4680,26 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", "value": 80 } ] - } + }, + "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 179 + "x": 0, + "y": 120 }, - "id": 80, + "id": 262, "options": { "legend": { "calcs": [], @@ -5025,26 +4708,28 @@ "showLegend": true }, "tooltip": { - "mode": "multi", + "hideZeros": false, + "mode": "single", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "editorMode": "builder", - "expr": "reth_blockchain_tree_block_buffer_blocks{instance=~\"$instance\"}", + "editorMode": "code", + "expr": "reth_sparse_state_trie_multiproof_skipped_account_nodes{$instance_label=\"$instance\",quantile=~\"(0|0.5|0.9|0.95|1)\"}", "hide": false, - "legendFormat": "Buffered blocks", + "instant": false, + "legendFormat": "Account {{quantile}} percentile", "range": true, - "refId": "B" + "refId": "A" } ], - "title": "Block buffer blocks", + "title": "Redundant multiproof account nodes", "type": "timeseries" }, { @@ -5052,7 +4737,6 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "Total number of sidechains in the blockchain tree", "fieldConfig": { "defaults": { "color": { @@ -5082,6 +4766,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -5096,24 +4781,26 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", "value": 80 } ] - } + }, + "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 187 + "x": 12, + "y": 120 }, - "id": 81, + "id": 261, "options": { "legend": { "calcs": [], @@ -5122,26 +4809,27 @@ "showLegend": true }, "tooltip": { - "mode": "multi", + "hideZeros": false, + "mode": "single", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "editorMode": "builder", - "expr": "reth_blockchain_tree_sidechains{instance=~\"$instance\"}", - "hide": false, - "legendFormat": "Total number of sidechains", + "editorMode": "code", + "expr": "reth_sparse_state_trie_multiproof_skipped_storage_nodes{$instance_label=\"$instance\",quantile=~\"(0|0.5|0.9|0.95|1)\"}", + "instant": false, + "legendFormat": "Storage {{quantile}} percentile", "range": true, - "refId": "B" + "refId": "Branch Nodes" } ], - "title": "Sidechains", + "title": "Redundant multiproof storage nodes", "type": "timeseries" }, { @@ -5149,6 +4837,7 @@ "type": "prometheus", "uid": "${datasource}" }, + "description": "How much time is spent in the multiproof task", "fieldConfig": { "defaults": { "color": { @@ -5178,6 +4867,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -5192,7 +4882,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -5207,38 +4898,40 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 187 + "x": 0, + "y": 128 }, - "id": 114, + "id": 263, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": false + "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(reth_consensus_engine_beacon_make_canonical_committed_latency_sum{instance=~\"$instance\"}[$__rate_interval]) / rate(reth_consensus_engine_beacon_make_canonical_committed_latency_count{instance=~\"$instance\"}[$__rate_interval])", + "expr": "avg by (quantile) (reth_tree_root_multiproof_task_total_duration_histogram{$instance_label=\"$instance\",quantile=~\"(0|0.5|0.9|0.95|1)\"})", + "hide": false, "instant": false, - "legendFormat": "__auto", + "legendFormat": "Task duration {{quantile}} percentile", "range": true, "refId": "A" } ], - "title": "Canonical Commit Latency time", + "title": "Proof fetching total duration", "type": "timeseries" }, { @@ -5246,6 +4939,7 @@ "type": "prometheus", "uid": "${datasource}" }, + "description": "Histogram for state root latency, the time spent blocked waiting for the state root.", "fieldConfig": { "defaults": { "color": { @@ -5275,6 +4969,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -5289,7 +4984,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -5297,7 +4993,7 @@ } ] }, - "unit": "none" + "unit": "s" }, "overrides": [] }, @@ -5305,37 +5001,42 @@ "h": 8, "w": 12, "x": 12, - "y": 195 + "y": 128 }, - "id": 190, + "id": 1006, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": false + "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "editorMode": "code", - "expr": "reth_blockchain_tree_latest_reorg_depth{instance=~\"$instance\"}", + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_sync_block_validation_state_root_histogram{$instance_label=\"$instance\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, "instant": false, - "legendFormat": "__auto", + "legendFormat": "State Root Duration p{{quantile}}", "range": true, - "refId": "A" + "refId": "A", + "useBackend": false } ], - "title": "Latest Reorg Depth", + "title": "State root latency", "type": "timeseries" }, { @@ -5344,12 +5045,12 @@ "h": 1, "w": 24, "x": 0, - "y": 203 + "y": 136 }, - "id": 87, + "id": 38, "panels": [], "repeat": "instance", - "title": "Engine API", + "title": "Database", "type": "row" }, { @@ -5357,11 +5058,12 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "", + "description": "The average commit time for database transactions. Generally, this should not be a limiting factor in syncing.", "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" + "mode": "palette-classic", + "seriesBy": "last" }, "custom": { "axisBorderShow": false, @@ -5371,7 +5073,7 @@ "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, - "drawStyle": "line", + "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { @@ -5387,6 +5089,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -5401,14 +5104,12 @@ "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 80 + "color": "green", + "value": 0 } ] - } + }, + "unit": "s" }, "overrides": [] }, @@ -5416,36 +5117,40 @@ "h": 8, "w": 12, "x": 0, - "y": 204 + "y": 137 }, - "id": 83, + "id": 40, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "editorMode": "builder", - "expr": "reth_consensus_engine_beacon_active_block_downloads{instance=~\"$instance\"}", - "legendFormat": "Active block downloads", + "editorMode": "code", + "exemplar": false, + "expr": "avg(rate(reth_database_transaction_close_duration_seconds_sum{$instance_label=\"$instance\", outcome=\"commit\"}[$__rate_interval]) / rate(reth_database_transaction_close_duration_seconds_count{$instance_label=\"$instance\", outcome=\"commit\"}[$__rate_interval]) >= 0)", + "format": "time_series", + "instant": false, + "legendFormat": "Commit time", "range": true, "refId": "A" } ], - "title": "Active block downloads", + "title": "Average commit time", "type": "timeseries" }, { @@ -5453,11 +5158,102 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "Engine API messages received by the CL, either engine_newPayload or engine_forkchoiceUpdated", + "description": "", + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 137 + }, + "id": 42, + "maxDataPoints": 25, + "options": { + "calculate": false, + "cellGap": 1, + "cellValues": { + "unit": "s" + }, + "color": { + "exponent": 0.2, + "fill": "dark-orange", + "min": 0, + "mode": "opacity", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto", + "value": "Commit time" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisLabel": "Quantile", + "axisPlacement": "left", + "reverse": false, + "unit": "percentunit" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "avg(max_over_time(reth_database_transaction_close_duration_seconds{$instance_label=\"$instance\", outcome=\"commit\"}[$__rate_interval])) by (quantile)", + "format": "time_series", + "instant": false, + "legendFormat": "{{quantile}}", + "range": true, + "refId": "A" + } + ], + "title": "Commit time heatmap", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The average time a database transaction was open.", "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" + "mode": "palette-classic", + "seriesBy": "last" }, "custom": { "axisBorderShow": false, @@ -5467,7 +5263,7 @@ "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, - "drawStyle": "line", + "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { @@ -5483,6 +5279,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -5497,24 +5294,22 @@ "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 80 + "color": "green", + "value": 0 } ] - } + }, + "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 204 + "x": 0, + "y": 145 }, - "id": 84, + "id": 117, "options": { "legend": { "calcs": [], @@ -5523,37 +5318,29 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "editorMode": "builder", - "expr": "reth_consensus_engine_beacon_forkchoice_updated_messages{instance=~\"$instance\"}", - "legendFormat": "Forkchoice updated messages", + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(reth_database_transaction_open_duration_seconds_sum{$instance_label=\"$instance\", outcome!=\"\"}[$__rate_interval]) / rate(reth_database_transaction_open_duration_seconds_count{$instance_label=\"$instance\", outcome!=\"\"}[$__rate_interval])) by (outcome, mode)", + "format": "time_series", + "instant": false, + "legendFormat": "{{mode}}, {{outcome}}", "range": true, "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "builder", - "expr": "reth_consensus_engine_beacon_new_payload_messages{instance=~\"$instance\"}", - "hide": false, - "legendFormat": "New payload messages", - "range": true, - "refId": "B" } ], - "title": "Engine API messages", + "title": "Average transaction open time", "type": "timeseries" }, { @@ -5561,7 +5348,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "Latency histogram for the engine_newPayload to Forkchoice Update", + "description": "The maximum time the database transaction was open.", "fieldConfig": { "defaults": { "color": { @@ -5575,7 +5362,7 @@ "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, - "drawStyle": "line", + "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { @@ -5590,7 +5377,8 @@ "scaleDistribution": { "type": "linear" }, - "showPoints": "never", + "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -5605,11 +5393,8 @@ "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 80 + "color": "green", + "value": 0 } ] }, @@ -5620,10 +5405,10 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 212 + "x": 12, + "y": 145 }, - "id": 213, + "id": 116, "options": { "legend": { "calcs": [], @@ -5632,25 +5417,29 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "editorMode": "builder", - "expr": "reth_engine_rpc_new_payload_forkchoice_updated_time_diff{instance=~\"$instance\"}", - "legendFormat": "new_payload_forkchoice_updated", + "editorMode": "code", + "exemplar": false, + "expr": "max(max_over_time(reth_database_transaction_open_duration_seconds{$instance_label=\"$instance\", outcome!=\"\", quantile=\"1\"}[$__interval])) by (outcome, mode)", + "format": "time_series", + "instant": false, + "legendFormat": "{{mode}}, {{outcome}}", "range": true, "refId": "A" } ], - "title": "Engine API newPayload Forkchoice Update Latency", + "title": "Max transaction open time", "type": "timeseries" }, { @@ -5658,7 +5447,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "Latency histogram for the engine_newPayload RPC API", + "description": "", "fieldConfig": { "defaults": { "color": { @@ -5668,7 +5457,7 @@ "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "", + "axisLabel": "txs", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, @@ -5682,12 +5471,16 @@ }, "insertNulls": false, "lineInterpolation": "linear", - "lineWidth": 1, + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 3, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -5702,7 +5495,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -5710,252 +5504,72 @@ } ] }, - "unit": "s" + "unit": "none" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "Diff(opened-closed)" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [0, 10], + "fill": "dot" + } + }, + { + "id": "custom.axisLabel", + "value": "diff" + } + ] + } + ] }, "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 212 + "x": 0, + "y": 153 }, - "id": 210, + "id": 119, "options": { "legend": { "calcs": [], "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_engine_rpc_new_payload_v1{instance=~\"$instance\", quantile=\"0\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "legendFormat": "engine_newPayloadV1 min", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_engine_rpc_new_payload_v1{instance=~\"$instance\", quantile=\"0.5\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "legendFormat": "engine_newPayloadV1 p50", - "range": true, - "refId": "B", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_engine_rpc_new_payload_v1{instance=~\"$instance\", quantile=\"0.9\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "legendFormat": "engine_newPayloadV1 p90", - "range": true, - "refId": "C", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_engine_rpc_new_payload_v1{instance=~\"$instance\", quantile=\"0.95\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "legendFormat": "engine_newPayloadV1 p95", - "range": true, - "refId": "D", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_engine_rpc_new_payload_v1{instance=~\"$instance\", quantile=\"0.99\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "legendFormat": "engine_newPayloadV1 p99", - "range": true, - "refId": "E", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_engine_rpc_new_payload_v2{instance=~\"$instance\", quantile=\"0\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "legendFormat": "engine_newPayloadV2 min", - "range": true, - "refId": "F", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_engine_rpc_new_payload_v2{instance=~\"$instance\", quantile=\"0.5\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "legendFormat": "engine_newPayloadV2 p50", - "range": true, - "refId": "G", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_engine_rpc_new_payload_v2{instance=~\"$instance\", quantile=\"0.9\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "legendFormat": "engine_newPayloadV2 p90", - "range": true, - "refId": "H", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_engine_rpc_new_payload_v2{instance=~\"$instance\", quantile=\"0.95\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "legendFormat": "engine_newPayloadV2 p95", - "range": true, - "refId": "I", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_engine_rpc_new_payload_v2{instance=~\"$instance\", quantile=\"0.99\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "legendFormat": "engine_newPayloadV2 p99", - "range": true, - "refId": "J", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_engine_rpc_new_payload_v3{instance=~\"$instance\", quantile=\"0\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "legendFormat": "engine_newPayloadV3 min", - "range": true, - "refId": "K", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_engine_rpc_new_payload_v3{instance=~\"$instance\", quantile=\"0.5\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "legendFormat": "engine_newPayloadV3 p50", - "range": true, - "refId": "L", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_engine_rpc_new_payload_v3{instance=~\"$instance\", quantile=\"0.9\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "legendFormat": "engine_newPayloadV3 p90", - "range": true, - "refId": "M", - "useBackend": false + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_engine_rpc_new_payload_v3{instance=~\"$instance\", quantile=\"0.95\"}", + "editorMode": "code", + "exemplar": false, + "expr": "sum(reth_database_transaction_opened_total{$instance_label=\"$instance\", mode=\"read-write\"})", + "format": "time_series", "fullMetaSearch": false, - "hide": false, "includeNullMetadata": true, - "legendFormat": "engine_newPayloadV3 p95", + "instant": false, + "legendFormat": "Opened", "range": true, - "refId": "N", + "refId": "A", "useBackend": false }, { @@ -5963,47 +5577,157 @@ "type": "prometheus", "uid": "${datasource}" }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_engine_rpc_new_payload_v3{instance=~\"$instance\", quantile=\"0.99\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "legendFormat": "engine_newPayloadV3 p99", + "editorMode": "code", + "exemplar": false, + "expr": "sum(reth_database_transaction_closed_total{$instance_label=\"$instance\", mode=\"read-write\"})", + "format": "time_series", + "instant": false, + "legendFormat": "Closed {{mode}}", "range": true, - "refId": "O", - "useBackend": false + "refId": "B" }, { "datasource": { - "type": "prometheus", - "uid": "${datasource}" + "type": "__expr__", + "uid": "${DS_EXPRESSION}" }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_engine_rpc_new_payload_v4{instance=~\"$instance\", quantile=\"0\"}", - "fullMetaSearch": false, + "expression": "${A} - ${B}", "hide": false, - "includeNullMetadata": true, - "legendFormat": "engine_newPayloadV4 min", - "range": true, - "refId": "P", - "useBackend": false + "refId": "Diff(opened-closed)", + "type": "math" + } + ], + "title": "Number of read-write transactions", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "txs", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 3, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "Diff(opened, closed)" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [0, 10], + "fill": "dot" + } + }, + { + "id": "custom.axisLabel", + "value": "diff" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 153 + }, + "id": 250, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "disableTextWrap": false, "editorMode": "builder", - "expr": "reth_engine_rpc_new_payload_v4{instance=~\"$instance\", quantile=\"0.5\"}", + "exemplar": false, + "expr": "reth_database_transaction_opened_total{$instance_label=\"$instance\", mode=\"read-only\"}", + "format": "time_series", "fullMetaSearch": false, - "hide": false, "includeNullMetadata": true, - "legendFormat": "engine_newPayloadV4 p50", + "instant": false, + "legendFormat": "Opened", "range": true, - "refId": "Q", + "refId": "A", "useBackend": false }, { @@ -6013,57 +5737,106 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "reth_engine_rpc_new_payload_v4{instance=~\"$instance\", quantile=\"0.9\"}", + "exemplar": false, + "expr": "sum(reth_database_transaction_closed_total{$instance_label=\"$instance\", mode=\"read-only\"})", + "format": "time_series", "fullMetaSearch": false, - "hide": false, "includeNullMetadata": true, - "legendFormat": "engine_newPayloadV4 p90", + "instant": false, + "legendFormat": "Closed {{mode}}", "range": true, - "refId": "R", + "refId": "B", "useBackend": false }, { "datasource": { - "type": "prometheus", - "uid": "${datasource}" + "type": "__expr__", + "uid": "${DS_EXPRESSION}" }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_engine_rpc_new_payload_v4{instance=~\"$instance\", quantile=\"0.95\"}", - "fullMetaSearch": false, + "expression": "${A} - ${B}", "hide": false, - "includeNullMetadata": true, - "legendFormat": "engine_newPayloadV4 p95", - "range": true, - "refId": "S", - "useBackend": false + "refId": "Diff(opened, closed)", + "type": "math" + } + ], + "title": "Number of read-only transactions", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The size of tables in the database", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 161 + }, + "id": 48, + "options": { + "displayLabels": ["name"], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": ["value"] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false }, + "sort": "desc", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "disableTextWrap": false, "editorMode": "builder", - "expr": "reth_engine_rpc_new_payload_v4{instance=~\"$instance\", quantile=\"0.99\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "legendFormat": "engine_newPayloadV4 p99", + "expr": "reth_db_table_size{$instance_label=\"$instance\"}", + "interval": "", + "legendFormat": "{{table}}", "range": true, - "refId": "T", - "useBackend": false - } + "refId": "A" + } ], - "title": "Engine API newPayload Latency", - "type": "timeseries" + "title": "Database tables", + "type": "piechart" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "description": "Total pipeline runs triggered by the sync controller", + "description": "The maximum time the database transaction operation which inserts a large value took.", "fieldConfig": { "defaults": { "color": { @@ -6077,7 +5850,7 @@ "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, - "drawStyle": "line", + "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { @@ -6093,6 +5866,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -6107,24 +5881,26 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", "value": 80 } ] - } + }, + "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 220 + "x": 12, + "y": 161 }, - "id": 85, + "id": 118, "options": { "legend": { "calcs": [], @@ -6133,25 +5909,29 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "editorMode": "builder", - "expr": "reth_consensus_engine_beacon_pipeline_runs{instance=~\"$instance\"}", - "legendFormat": "Pipeline runs", + "editorMode": "code", + "exemplar": false, + "expr": "max(max_over_time(reth_database_operation_large_value_duration_seconds{$instance_label=\"$instance\", quantile=\"1\"}[$__interval]) > 0) by (table)", + "format": "time_series", + "instant": false, + "legendFormat": "{{table}}", "range": true, "refId": "A" } ], - "title": "Pipeline runs", + "title": "Max insertion operation time", "type": "timeseries" }, { @@ -6159,7 +5939,74 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "Latency histograms for the engine_getPayloadBodiesByHashV1 and engine_getPayloadBodiesByRangeV1 RPC APIs", + "description": "The type of the pages in the database:\n\n- **Leaf** pages contain KV pairs.\n- **Branch** pages contain information about keys in the leaf pages\n- **Overflow** pages store large values and should generally be avoided if possible", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 169 + }, + "id": 50, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": ["value"] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "sort": "desc", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "sum by (type) ( reth_db_table_pages{$instance_label=\"$instance\"} )", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Database pages", + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The size of the database over time", "fieldConfig": { "defaults": { "color": { @@ -6189,6 +6036,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -6198,206 +6046,60 @@ "mode": "off" } }, + "decimals": 4, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 220 - }, - "id": 212, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_engine_rpc_get_payload_bodies_by_hash_v1{instance=~\"$instance\", quantile=\"0\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "legendFormat": "engine_getPayloadBodiesByHashV1 min", - "range": true, - "refId": "O", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_engine_rpc_get_payload_bodies_by_hash_v1{instance=~\"$instance\", quantile=\"0.5\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "legendFormat": "engine_getPayloadBodiesByHashV1 p50", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_engine_rpc_get_payload_bodies_by_hash_v1{instance=~\"$instance\", quantile=\"0.9\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "legendFormat": "engine_getPayloadBodiesByHashV1 p90", - "range": true, - "refId": "B", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_engine_rpc_get_payload_bodies_by_hash_v1{instance=~\"$instance\", quantile=\"0.95\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "legendFormat": "engine_getPayloadBodiesByHashV1 p95", - "range": true, - "refId": "C", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_engine_rpc_get_payload_bodies_by_hash_v1{instance=~\"$instance\", quantile=\"0.99\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "legendFormat": "engine_getPayloadBodiesByHashV1 p99", - "range": true, - "refId": "D", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_engine_rpc_get_payload_bodies_by_range_v1{instance=~\"$instance\", quantile=\"0\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "legendFormat": "engine_getPayloadBodiesByRangeV1 min", - "range": true, - "refId": "E", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_engine_rpc_get_payload_bodies_by_range_v1{instance=~\"$instance\", quantile=\"0.5\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "legendFormat": "engine_getPayloadBodiesByRangeV1 p50", - "range": true, - "refId": "F", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_engine_rpc_get_payload_bodies_by_range_v1{instance=~\"$instance\", quantile=\"0.9\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "legendFormat": "engine_getPayloadBodiesByRangeV1 p90", - "range": true, - "refId": "G", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_engine_rpc_get_payload_bodies_by_range_v1{instance=~\"$instance\", quantile=\"0.95\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "legendFormat": "engine_getPayloadBodiesByRangeV1 p95", - "range": true, - "refId": "H", - "useBackend": false + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 169 + }, + "id": 52, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_engine_rpc_get_payload_bodies_by_range_v1{instance=~\"$instance\", quantile=\"0.99\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "legendFormat": "engine_getPayloadBodiesByRangeV1 p99", + "editorMode": "code", + "expr": "sum by (job) ( reth_db_table_size{$instance_label=\"$instance\"} )", + "legendFormat": "Size ({{job}})", "range": true, - "refId": "I", - "useBackend": false + "refId": "A" } ], - "title": "Engine API getPayloadBodies Latency", + "title": "Database growth", "type": "timeseries" }, { @@ -6405,7 +6107,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "Latency histogram for the engine_forkchoiceUpdated RPC API", + "description": "The number of pages on the MDBX freelist", "fieldConfig": { "defaults": { "color": { @@ -6435,6 +6137,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -6449,7 +6152,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -6457,437 +6161,430 @@ } ] }, - "unit": "s" + "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 228 - }, - "id": 211, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_engine_rpc_fork_choice_updated_v1{instance=~\"$instance\", quantile=\"0\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "legendFormat": "engine_forkchoiceUpdatedV1 min", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_engine_rpc_fork_choice_updated_v1{instance=~\"$instance\", quantile=\"0.5\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "legendFormat": "engine_forkchoiceUpdatedV1 p50", - "range": true, - "refId": "B", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_engine_rpc_fork_choice_updated_v1{instance=~\"$instance\", quantile=\"0.9\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "legendFormat": "engine_forkchoiceUpdatedV1 p90", - "range": true, - "refId": "C", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_engine_rpc_fork_choice_updated_v1{instance=~\"$instance\", quantile=\"0.95\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "legendFormat": "engine_forkchoiceUpdatedV1 p95", - "range": true, - "refId": "D", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_engine_rpc_fork_choice_updated_v1{instance=~\"$instance\", quantile=\"0.99\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "legendFormat": "engine_forkchoiceUpdatedV1 p99", - "range": true, - "refId": "E", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_engine_rpc_fork_choice_updated_v2{instance=~\"$instance\", quantile=\"0\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "legendFormat": "engine_forkchoiceUpdatedV2 min", - "range": true, - "refId": "F", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_engine_rpc_fork_choice_updated_v2{instance=~\"$instance\", quantile=\"0.5\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "legendFormat": "engine_forkchoiceUpdatedV2 p50", - "range": true, - "refId": "G", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_engine_rpc_fork_choice_updated_v2{instance=~\"$instance\", quantile=\"0.9\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "legendFormat": "engine_forkchoiceUpdatedV2 p90", - "range": true, - "refId": "H", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_engine_rpc_fork_choice_updated_v2{instance=~\"$instance\", quantile=\"0.95\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "legendFormat": "engine_forkchoiceUpdatedV2 p95", - "range": true, - "refId": "I", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_engine_rpc_fork_choice_updated_v2{instance=~\"$instance\", quantile=\"0.99\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "legendFormat": "engine_forkchoiceUpdatedV2 p99", - "range": true, - "refId": "J", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_engine_rpc_fork_choice_updated_v3{instance=~\"$instance\", quantile=\"0\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "legendFormat": "engine_forkchoiceUpdatedV3 min", - "range": true, - "refId": "K", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_engine_rpc_fork_choice_updated_v3{instance=~\"$instance\", quantile=\"0.5\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "legendFormat": "engine_forkchoiceUpdatedV3 p50", - "range": true, - "refId": "L", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_engine_rpc_fork_choice_updated_v3{instance=~\"$instance\", quantile=\"0.9\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "legendFormat": "engine_forkchoiceUpdatedV3 p90", - "range": true, - "refId": "M", - "useBackend": false + "x": 0, + "y": 177 + }, + "id": 113, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_engine_rpc_fork_choice_updated_v3{instance=~\"$instance\", quantile=\"0.95\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "legendFormat": "engine_forkchoiceUpdatedV3 p95", + "editorMode": "code", + "expr": "sum(reth_db_freelist{$instance_label=\"$instance\"}) by (job)", + "legendFormat": "Pages ({{job}})", "range": true, - "refId": "N", - "useBackend": false + "refId": "A" + } + ], + "title": "Freelist", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "left", + "cellOptions": { + "type": "auto" + }, + "footer": { + "reducers": [] + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.hideFrom.viz", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "__name__" + }, + "properties": [ + { + "id": "custom.hideFrom.viz", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "instance" + }, + "properties": [ + { + "id": "custom.hideFrom.viz", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "job" + }, + "properties": [ + { + "id": "custom.hideFrom.viz", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "type" + }, + "properties": [ + { + "id": "custom.hideFrom.viz", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "unit", + "value": "locale" + }, + { + "id": "displayName", + "value": "Overflow pages" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "table" + }, + "properties": [ + { + "id": "displayName", + "value": "Table" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 177 + }, + "id": 58, + "options": { + "cellHeight": "sm", + "showHeader": true + }, + "pluginVersion": "12.2.1", + "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_engine_rpc_fork_choice_updated_v3{instance=~\"$instance\", quantile=\"0.99\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "legendFormat": "engine_forkchoiceUpdatedV3 p99", - "range": true, - "refId": "O", - "useBackend": false + "editorMode": "code", + "exemplar": false, + "expr": "sort_desc(reth_db_table_pages{$instance_label=\"$instance\", type=\"overflow\"} != 0)", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" } ], - "title": "Engine API forkchoiceUpdated Latency", - "type": "timeseries" + "title": "Overflow pages by table", + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 185 + }, + "id": 203, + "panels": [], + "title": "Static Files", + "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "description": "The size of segments in the static files", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false + } + }, + "mappings": [], + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 186 + }, + "id": 202, + "options": { + "displayLabels": ["name"], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": ["value"] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "sort": "desc", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "reth_static_files_segment_size{$instance_label=\"$instance\"}", + "interval": "", + "legendFormat": "{{segment}}", + "range": true, + "refId": "A" + } + ], + "title": "Segments size", + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "left", + "cellOptions": { + "type": "auto" }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "footer": { + "reducers": [] }, - "thresholdsStyle": { - "mode": "off" - } + "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", "value": 80 } ] - }, - "unit": "s" + } }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "unit", + "value": "locale" + }, + { + "id": "displayName", + "value": "Entries" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "segment" + }, + "properties": [ + { + "id": "displayName", + "value": "Segment" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.hideFrom.viz", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "instance" + }, + "properties": [ + { + "id": "custom.hideFrom.viz", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "job" + }, + "properties": [ + { + "id": "custom.hideFrom.viz", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "__name__" + }, + "properties": [ + { + "id": "custom.hideFrom.viz", + "value": true + } + ] + } + ] }, "gridPos": { "h": 8, - "w": 12, - "x": 12, - "y": 228 + "w": 8, + "x": 8, + "y": 186 }, - "id": 258, + "id": 204, "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "cellHeight": "sm", + "showHeader": true }, + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_engine_rpc_get_blobs_v1{instance=~\"$instance\", quantile=\"0.5\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "legendFormat": "engine_getBlobsV1 p50", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_engine_rpc_get_blobs_v1{instance=~\"$instance\", quantile=\"0.95\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "legendFormat": "engine_getBlobsV1 p95", - "range": true, - "refId": "B", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_engine_rpc_get_blobs_v1{instance=~\"$instance\", quantile=\"0.99\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "legendFormat": "engine_getBlobsV1 p99", - "range": true, - "refId": "C", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_engine_rpc_get_blobs_v1{instance=~\"$instance\", quantile=\"0\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "legendFormat": "engine_getBlobsV1 min", - "range": true, - "refId": "D", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_engine_rpc_get_blobs_v1{instance=~\"$instance\", quantile=\"1\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "legendFormat": "engine_getBlobsV1 max", - "range": true, - "refId": "E", - "useBackend": false + "editorMode": "code", + "exemplar": false, + "expr": "reth_static_files_segment_entries{$instance_label=\"$instance\"}", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" } ], - "title": "Engine API getBlobs Latency", - "type": "timeseries" + "title": "Entries per segment", + "type": "table" }, { "datasource": { @@ -6897,105 +6594,149 @@ "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" + "mode": "thresholds" }, "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false + "align": "left", + "cellOptions": { + "type": "auto" + }, + "footer": { + "reducers": [] + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "unit", + "value": "locale" + }, + { + "id": "displayName", + "value": "Files" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "segment" }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "properties": [ + { + "id": "displayName", + "value": "Segment" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Time" }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "properties": [ + { + "id": "custom.hideFrom.viz", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "instance" }, - "thresholdsStyle": { - "mode": "off" - } + "properties": [ + { + "id": "custom.hideFrom.viz", + "value": true + } + ] }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ + { + "matcher": { + "id": "byName", + "options": "job" + }, + "properties": [ { - "color": "green", - "value": null + "id": "custom.hideFrom.viz", + "value": true } ] }, - "unit": "none" - }, - "overrides": [] + { + "matcher": { + "id": "byName", + "options": "__name__" + }, + "properties": [ + { + "id": "custom.hideFrom.viz", + "value": true + } + ] + } + ] }, "gridPos": { "h": 8, - "w": 12, - "x": 0, - "y": 236 + "w": 8, + "x": 16, + "y": 186 }, - "id": 1000, + "id": 205, "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "cellHeight": "sm", + "showHeader": true }, + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "editorMode": "builder", - "expr": "reth_engine_api_blob_metrics_blob_count{instance=~\"$instance\"}", - "legendFormat": "Blobs Found", - "range": true, + "editorMode": "code", + "exemplar": false, + "expr": "reth_static_files_segment_files{$instance_label=\"$instance\"}", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "builder", - "expr": "reth_engine_api_blob_metrics_blob_misses{instance=~\"$instance\"}", - "hide": false, - "legendFormat": "Blobs Missed", - "range": true, - "refId": "B" } ], - "title": "Blob Count and Misses", - "type": "timeseries" + "title": "Files per segment", + "type": "table" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "description": "Counts the number of failed response deliveries due to client request termination.", + "description": "The size of the static files over time", "fieldConfig": { "defaults": { "color": { @@ -7024,7 +6765,8 @@ "scaleDistribution": { "type": "linear" }, - "showPoints": "never", + "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -7039,7 +6781,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -7047,17 +6790,17 @@ } ] }, - "unit": "none" + "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 8, - "w": 24, + "w": 12, "x": 0, - "y": 236 + "y": 194 }, - "id": 249, + "id": 206, "options": { "legend": { "calcs": [], @@ -7066,52 +6809,34 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "expr": "consensus_engine_beacon_failed_new_payload_response_deliveries{instance=~\"$instance\"}", - "legendFormat": "Failed NewPayload Deliveries", + "editorMode": "code", + "expr": "sum by (job) ( reth_static_files_segment_size{$instance_label=\"$instance\"} )", + "legendFormat": "__auto", + "range": true, "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "expr": "consensus_engine_beacon_failed_forkchoice_updated_response_deliveries{instance=~\"$instance\"}", - "legendFormat": "Failed ForkchoiceUpdated Deliveries", - "refId": "B" } ], - "title": "Failed Engine API Response Deliveries", + "title": "Static Files growth", "type": "timeseries" }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 244 - }, - "id": 214, - "panels": [], - "title": "State Root Task", - "type": "row" - }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "description": "The maximum time the static files operation which commits a writer took.", "fieldConfig": { "defaults": { "color": { @@ -7125,7 +6850,7 @@ "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, - "drawStyle": "line", + "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { @@ -7141,6 +6866,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -7155,24 +6881,26 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", "value": 80 } ] - } + }, + "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 245 + "x": 12, + "y": 194 }, - "id": 216, + "id": 207, "options": { "legend": { "calcs": [], @@ -7181,46 +6909,48 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "reth_tree_root_proof_calculation_storage_targets_histogram{instance=\"$instance\",quantile=~\"(0|0.5|0.9|0.95|1)\"}", - "instant": false, - "legendFormat": "{{type}} storage proof targets p{{quantile}}", - "range": true, - "refId": "Branch Nodes" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "reth_tree_root_proof_calculation_account_targets_histogram{instance=\"$instance\",quantile=~\"(0|0.5|0.9|0.95|1)\"}", - "hide": false, - "instant": false, - "legendFormat": "{{type}} account proof targets p{{quantile}}", + "expr": "max(max_over_time(reth_static_files_jar_provider_write_duration_seconds{$instance_label=\"$instance\", operation=\"commit-writer\", quantile=\"1\"}[$__interval]) > 0) by (segment)", + "legendFormat": "{{segment}}", "range": true, - "refId": "Leaf Nodes" + "refId": "A" } ], - "title": "Proof Targets", + "title": "Max writer commit time", "type": "timeseries" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 202 + }, + "id": 79, + "panels": [], + "repeat": "instance", + "title": "Blockchain Tree", + "type": "row" + }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "description": "The block number of the tip of the canonical chain from the blockchain tree.", "fieldConfig": { "defaults": { "color": { @@ -7250,6 +6980,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -7264,7 +6995,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -7278,10 +7010,10 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 245 + "x": 0, + "y": 203 }, - "id": 255, + "id": 74, "options": { "legend": { "calcs": [], @@ -7290,26 +7022,27 @@ "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "editorMode": "code", - "expr": "reth_tree_root_proofs_processed_histogram{instance=\"$instance\",quantile=~\"(0|0.5|0.9|0.95|1)\"}", - "instant": false, - "legendFormat": "{{quantile}} percentile", + "editorMode": "builder", + "expr": "reth_blockchain_tree_canonical_chain_height{$instance_label=\"$instance\"}", + "hide": false, + "legendFormat": "Canonical chain height", "range": true, - "refId": "Branch Nodes" + "refId": "B" } ], - "title": "Proofs Processed", + "title": "Canonical chain height", "type": "timeseries" }, { @@ -7317,6 +7050,7 @@ "type": "prometheus", "uid": "${datasource}" }, + "description": "Total number of blocks in the tree's block buffer", "fieldConfig": { "defaults": { "color": { @@ -7346,6 +7080,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -7360,25 +7095,25 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", "value": 80 } ] - }, - "unit": "s" + } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 253 + "x": 12, + "y": 203 }, - "id": 254, + "id": 80, "options": { "legend": { "calcs": [], @@ -7387,26 +7122,27 @@ "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "editorMode": "code", - "expr": "reth_tree_root_proof_calculation_duration_histogram{instance=\"$instance\",quantile=~\"(0|0.5|0.9|0.95|1)\"}", - "instant": false, - "legendFormat": "{{quantile}} percentile", + "editorMode": "builder", + "expr": "reth_blockchain_tree_block_buffer_blocks{$instance_label=\"$instance\"}", + "hide": false, + "legendFormat": "Buffered blocks", "range": true, - "refId": "Branch Nodes" + "refId": "B" } ], - "title": "Proof calculation duration", + "title": "Block buffer blocks", "type": "timeseries" }, { @@ -7443,6 +7179,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -7457,7 +7194,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -7472,38 +7210,39 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 253 + "x": 0, + "y": 211 }, - "id": 256, + "id": 1002, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "reth_tree_root_inflight_multiproofs_histogram{instance=\"$instance\",quantile=~\"(0|0.5|0.9|0.95|1)\"}", + "expr": "increase(reth_blockchain_tree_reorgs{$instance_label=\"$instance\"}[$__rate_interval])", "instant": false, - "legendFormat": "{{quantile}} percentile", + "legendFormat": "__auto", "range": true, - "refId": "Branch Nodes" + "refId": "A" } ], - "title": "In-flight MultiProof requests", + "title": "Reorgs", "type": "timeseries" }, { @@ -7540,6 +7279,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -7554,7 +7294,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -7569,45 +7310,60 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 261 + "x": 12, + "y": 211 }, - "id": 257, + "id": 190, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "reth_tree_root_pending_multiproofs_histogram{instance=\"$instance\",quantile=~\"(0|0.5|0.9|0.95|1)\"}", + "expr": "reth_blockchain_tree_latest_reorg_depth{$instance_label=\"$instance\"}", "instant": false, - "legendFormat": "{{quantile}} percentile", + "legendFormat": "__auto", "range": true, - "refId": "Branch Nodes" + "refId": "A" } ], - "title": "Pending MultiProof requests", + "title": "Latest Reorg Depth", "type": "timeseries" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 219 + }, + "id": 108, + "panels": [], + "title": "RPC server", + "type": "row" + }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "description": "", "fieldConfig": { "defaults": { "color": { @@ -7622,7 +7378,7 @@ "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, @@ -7636,7 +7392,8 @@ "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", + "showPoints": "never", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -7651,7 +7408,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -7659,17 +7417,42 @@ } ] }, - "unit": "none" + "unit": "short" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "http" + }, + "properties": [ + { + "id": "displayName", + "value": "HTTP" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ws" + }, + "properties": [ + { + "id": "displayName", + "value": "WebSocket" + } + ] + } + ] }, "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 261 + "x": 0, + "y": 220 }, - "id": 259, + "id": 109, "options": { "legend": { "calcs": [], @@ -7678,28 +7461,123 @@ "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, + "disableTextWrap": false, "editorMode": "code", - "expr": "reth_sparse_state_trie_multiproof_skipped_account_nodes{instance=\"$instance\",quantile=~\"(0|0.5|0.9|0.95|1)\"}", - "instant": false, - "legendFormat": "Account {{quantile}} percentile", + "expr": "sum(reth_rpc_server_connections_connections_opened_total{$instance_label=\"$instance\"} - reth_rpc_server_connections_connections_closed_total{$instance_label=\"$instance\"}) by (transport)", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "{{transport}}", "range": true, - "refId": "Branch Nodes" + "refId": "A", + "useBackend": false } ], - "title": "Redundant multiproof account nodes", + "title": "Active Connections", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 220 + }, + "id": 111, + "maxDataPoints": 25, + "options": { + "calculate": false, + "cellGap": 1, + "cellValues": { + "unit": "s" + }, + "color": { + "exponent": 0.2, + "fill": "dark-orange", + "min": 0, + "mode": "opacity", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto", + "value": "Latency time" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisLabel": "Quantile", + "axisPlacement": "left", + "reverse": false, + "unit": "percentunit" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "avg(max_over_time(reth_rpc_server_connections_request_time_seconds{$instance_label=\"$instance\"}[$__rate_interval]) > 0) by (quantile)", + "format": "time_series", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Request Latency time", + "type": "heatmap" + }, { "datasource": { "type": "prometheus", @@ -7718,7 +7596,7 @@ "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, - "drawStyle": "line", + "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { @@ -7734,6 +7612,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -7748,7 +7627,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -7756,7 +7636,7 @@ } ] }, - "unit": "none" + "unit": "s" }, "overrides": [] }, @@ -7764,9 +7644,9 @@ "h": 8, "w": 12, "x": 0, - "y": 269 + "y": 228 }, - "id": 260, + "id": 120, "options": { "legend": { "calcs": [], @@ -7775,26 +7655,27 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "reth_sparse_state_trie_multiproof_skipped_storage_nodes{instance=\"$instance\",quantile=~\"(0|0.5|0.9|0.95|1)\"}", + "expr": "max(max_over_time(reth_rpc_server_calls_time_seconds{$instance_label=\"$instance\"}[$__rate_interval])) by (method) > 0", "instant": false, - "legendFormat": "Storage {{quantile}} percentile", + "legendFormat": "__auto", "range": true, - "refId": "Branch Nodes" + "refId": "A" } ], - "title": "Redundant multiproof storage nodes", + "title": "Maximum call latency per method", "type": "timeseries" }, { @@ -7802,58 +7683,19 @@ "type": "prometheus", "uid": "${datasource}" }, + "description": "", "fieldConfig": { "defaults": { - "color": { - "mode": "palette-classic" - }, "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, "scaleDistribution": { "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" + } }, "overrides": [] }, @@ -7861,38 +7703,70 @@ "h": 8, "w": 12, "x": 12, - "y": 269 + "y": 228 }, - "id": 261, + "id": 112, + "maxDataPoints": 25, "options": { + "calculate": false, + "cellGap": 1, + "cellValues": { + "unit": "s" + }, + "color": { + "exponent": 0.2, + "fill": "dark-orange", + "min": 0, + "mode": "opacity", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "show": true + }, + "rowsFrame": { + "layout": "auto", + "value": "Latency time" }, "tooltip": { "mode": "single", - "sort": "none" + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisLabel": "Quantile", + "axisPlacement": "left", + "reverse": false, + "unit": "percentunit" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "reth_sparse_state_trie_multiproof_total_storage_nodes{instance=\"$instance\",quantile=~\"(0|0.5|0.9|0.95|1)\"}", + "exemplar": false, + "expr": "avg(max_over_time(reth_rpc_server_calls_time_seconds{$instance_label=\"$instance\"}[$__rate_interval]) > 0) by (quantile)", + "format": "time_series", "instant": false, - "legendFormat": "Storage {{quantile}} percentile", + "legendFormat": "{{quantile}}", "range": true, - "refId": "Branch Nodes" + "refId": "A" } ], - "title": "Total multiproof storage nodes", - "type": "timeseries" + "title": "Call Latency time", + "type": "heatmap" }, { "datasource": { @@ -7928,6 +7802,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -7942,25 +7817,62 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", "value": 80 } ] - }, - "unit": "none" + } }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*cached items.*/" + }, + "properties": [ + { + "id": "custom.axisLabel", + "value": "Items" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*consumers.*/" + }, + "properties": [ + { + "id": "custom.axisLabel", + "value": "Queued consumers" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.memory usage*/" + }, + "properties": [ + { + "id": "unit", + "value": "decbytes" + } + ] + } + ] }, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 277 + "y": 236 }, - "id": 262, + "id": 198, "options": { "legend": { "calcs": [], @@ -7969,35 +7881,157 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.2.1", "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_rpc_eth_cache_cached_count{$instance_label=\"$instance\", cache=\"headers\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Headers cache cached items", + "range": true, + "refId": "A", + "useBackend": false + }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "editorMode": "code", - "expr": "reth_sparse_state_trie_multiproof_total_account_nodes{instance=\"$instance\",quantile=~\"(0|0.5|0.9|0.95|1)\"}", + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_rpc_eth_cache_queued_consumers_count{$instance_label=\"$instance\", cache=\"receipts\"}", + "fullMetaSearch": false, "hide": false, + "includeNullMetadata": true, "instant": false, - "legendFormat": "Account {{quantile}} percentile", + "legendFormat": "Receipts cache queued consumers", "range": true, - "refId": "A" + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_rpc_eth_cache_queued_consumers_count{$instance_label=\"$instance\", cache=\"headers\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Headers cache queued consumers", + "range": true, + "refId": "C", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_rpc_eth_cache_queued_consumers_count{$instance_label=\"$instance\", cache=\"blocks\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Block cache queued consumers", + "range": true, + "refId": "D", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_rpc_eth_cache_memory_usage{$instance_label=\"$instance\", cache=\"blocks\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Blocks cache memory usage", + "range": true, + "refId": "E", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_rpc_eth_cache_cached_count{$instance_label=\"$instance\", cache=\"receipts\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Receipts cache cached items", + "range": true, + "refId": "F", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_rpc_eth_cache_memory_usage{$instance_label=\"$instance\", cache=\"receipts\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Receipts cache memory usage", + "range": true, + "refId": "G", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_rpc_eth_cache_cached_count{$instance_label=\"$instance\", cache=\"blocks\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Block cache cached items", + "range": true, + "refId": "H", + "useBackend": false } ], - "title": "Total multiproof account nodes", + "title": "RPC Cache Metrics", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "description": "How much time is spent in the multiproof task", "fieldConfig": { "defaults": { "color": { @@ -8020,13 +8054,14 @@ "viz": false }, "insertNulls": false, - "lineInterpolation": "linear", + "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -8041,7 +8076,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -8049,7 +8085,7 @@ } ] }, - "unit": "s" + "unit": "reqps" }, "overrides": [] }, @@ -8057,9 +8093,9 @@ "h": 8, "w": 12, "x": 12, - "y": 277 + "y": 236 }, - "id": 263, + "id": 246, "options": { "legend": { "calcs": [], @@ -8068,11 +8104,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { @@ -8080,15 +8117,14 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "reth_tree_root_multiproof_task_total_duration_histogram{instance=\"$instance\",quantile=~\"(0|0.5|0.9|0.95|1)\"}", - "hide": false, + "expr": "sum(rate(reth_rpc_server_calls_successful_total{instance =~ \"$instance\"}[$__rate_interval])) by (method) > 0", "instant": false, - "legendFormat": "Task duration {{quantile}} percentile", + "legendFormat": "{{method}}", "range": true, "refId": "A" } ], - "title": "Proof fetching total duration", + "title": "RPC Throughput", "type": "timeseries" }, { @@ -8097,20 +8133,19 @@ "h": 1, "w": 24, "x": 0, - "y": 285 + "y": 244 }, - "id": 68, + "id": 24, "panels": [], "repeat": "instance", - "title": "Payload Builder", + "title": "Downloader: Headers", "type": "row" }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "description": "Number of active jobs", "fieldConfig": { "defaults": { "color": { @@ -8134,12 +8169,13 @@ }, "insertNulls": false, "lineInterpolation": "linear", - "lineWidth": 3, + "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -8154,7 +8190,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -8162,16 +8199,41 @@ } ] } - }, - "overrides": [] + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "C" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "D" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] }, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 286 + "y": 245 }, - "id": 60, + "id": 26, "options": { "legend": { "calcs": [], @@ -8180,11 +8242,12 @@ "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { @@ -8192,13 +8255,50 @@ "uid": "${datasource}" }, "editorMode": "builder", - "expr": "reth_payloads_active_jobs{instance=~\"$instance\"}", - "legendFormat": "Active Jobs", + "expr": "reth_downloaders_headers_total_downloaded{$instance_label=\"$instance\"}", + "legendFormat": "Downloaded", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "reth_downloaders_headers_total_flushed{$instance_label=\"$instance\"}", + "hide": false, + "legendFormat": "Flushed", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "rate(reth_downloaders_headers_total_downloaded{$instance_label=\"$instance\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "Downloaded/s", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "rate(reth_downloaders_headers_total_flushed{$instance_label=\"$instance\"}[$__rate_interval])", + "hide": false, + "legendFormat": "Flushed/s", + "range": true, + "refId": "D" } ], - "title": "Active Jobs", + "title": "I/O", "type": "timeseries" }, { @@ -8206,7 +8306,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "Total number of initiated jobs", + "description": "Internal errors in the header downloader. These are expected to happen from time to time.", "fieldConfig": { "defaults": { "color": { @@ -8230,12 +8330,13 @@ }, "insertNulls": false, "lineInterpolation": "linear", - "lineWidth": 3, + "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -8250,14 +8351,16 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", "value": 80 } ] - } + }, + "unit": "cps" }, "overrides": [] }, @@ -8265,9 +8368,9 @@ "h": 8, "w": 12, "x": 12, - "y": 286 + "y": 245 }, - "id": 62, + "id": 33, "options": { "legend": { "calcs": [], @@ -8276,25 +8379,50 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", - "expr": "reth_payloads_initiated_jobs{instance=~\"$instance\"}", - "legendFormat": "Initiated Jobs", + "expr": "rate(reth_downloaders_headers_timeout_errors{$instance_label=\"$instance\"}[$__rate_interval])", + "legendFormat": "Request timed out", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "rate(reth_downloaders_headers_unexpected_errors{$instance_label=\"$instance\"}[$__rate_interval])", + "hide": false, + "legendFormat": "Unexpected error", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "rate(reth_downloaders_headers_validation_errors{$instance_label=\"$instance\"}[$__rate_interval])", + "hide": false, + "legendFormat": "Invalid response", + "range": true, + "refId": "C" } ], - "title": "Initiated Jobs", + "title": "Errors", "type": "timeseries" }, { @@ -8302,7 +8430,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "Total number of failed jobs", + "description": "The number of connected peers and in-progress requests for headers.", "fieldConfig": { "defaults": { "color": { @@ -8326,12 +8454,13 @@ }, "insertNulls": false, "lineInterpolation": "linear", - "lineWidth": 3, + "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -8346,7 +8475,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -8361,9 +8491,9 @@ "h": 8, "w": 12, "x": 0, - "y": 294 + "y": 253 }, - "id": 64, + "id": 36, "options": { "legend": { "calcs": [], @@ -8372,25 +8502,38 @@ "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", - "expr": "reth_payloads_failed_jobs{instance=~\"$instance\"}", - "legendFormat": "Failed Jobs", + "expr": "reth_downloaders_headers_in_flight_requests{$instance_label=\"$instance\"}", + "legendFormat": "In flight requests", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "reth_network_connected_peers{$instance_label=\"$instance\"}", + "hide": false, + "legendFormat": "Connected peers", + "range": true, + "refId": "B" } ], - "title": "Failed Jobs", + "title": "Requests", "type": "timeseries" }, { @@ -8399,18 +8542,20 @@ "h": 1, "w": 24, "x": 0, - "y": 302 + "y": 261 }, - "id": 97, + "id": 32, "panels": [], - "title": "Process", + "repeat": "instance", + "title": "Downloader: Bodies", "type": "row" }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, + "description": "The internal state of the headers downloader: the number of downloaded headers, and the number of headers sent to the header stage.", "fieldConfig": { "defaults": { "color": { @@ -8440,6 +8585,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -8454,7 +8600,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -8462,18 +8609,38 @@ } ] }, - "unit": "decbytes" + "unit": "locale" }, "overrides": [ { "matcher": { - "id": "byName", - "options": "Retained" + "id": "byFrameRefID", + "options": "C" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "ops" + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "D" }, "properties": [ { "id": "custom.axisPlacement", "value": "right" + }, + { + "id": "unit", + "value": "ops" } ] } @@ -8483,9 +8650,9 @@ "h": 8, "w": 12, "x": 0, - "y": 303 + "y": 262 }, - "id": 98, + "id": 30, "options": { "legend": { "calcs": [], @@ -8494,11 +8661,12 @@ "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { @@ -8506,22 +8674,20 @@ "uid": "${datasource}" }, "editorMode": "builder", - "expr": "reth_jemalloc_active{instance=~\"$instance\"}", - "instant": false, - "legendFormat": "Active", + "expr": "reth_downloaders_bodies_total_downloaded{$instance_label=\"$instance\"}", + "legendFormat": "Downloaded", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", - "expr": "reth_jemalloc_allocated{instance=~\"$instance\"}", + "expr": "reth_downloaders_bodies_total_flushed{$instance_label=\"$instance\"}", "hide": false, - "instant": false, - "legendFormat": "Allocated", + "legendFormat": "Flushed", "range": true, "refId": "B" }, @@ -8531,160 +8697,70 @@ "uid": "${datasource}" }, "editorMode": "builder", - "expr": "reth_jemalloc_mapped{instance=~\"$instance\"}", + "expr": "rate(reth_downloaders_bodies_total_flushed{$instance_label=\"$instance\"}[$__rate_interval])", "hide": false, - "instant": false, - "legendFormat": "Mapped", + "legendFormat": "Flushed/s", "range": true, "refId": "C" }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "builder", - "expr": "reth_jemalloc_metadata{instance=~\"$instance\"}", - "hide": false, - "instant": false, - "legendFormat": "Metadata", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "builder", - "expr": "reth_jemalloc_resident{instance=~\"$instance\"}", - "hide": false, - "instant": false, - "legendFormat": "Resident", - "range": true, - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "builder", - "expr": "reth_jemalloc_retained{instance=~\"$instance\"}", - "hide": false, - "instant": false, - "legendFormat": "Retained", - "range": true, - "refId": "F" - } - ], - "title": "Jemalloc Memory", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] + "expr": "rate(reth_downloaders_bodies_total_downloaded{$instance_label=\"$instance\"}[$__rate_interval])", + "hide": false, + "legendFormat": "Downloaded/s", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" }, - "unit": "decbytes" + "editorMode": "builder", + "expr": "reth_downloaders_bodies_buffered_responses{$instance_label=\"$instance\"}", + "hide": false, + "legendFormat": "Buffered responses", + "range": true, + "refId": "E" }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 303 - }, - "id": 101, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "reth_downloaders_bodies_buffered_blocks{$instance_label=\"$instance\"}", + "hide": false, + "legendFormat": "Buffered blocks", + "range": true, + "refId": "F" }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.4.0", - "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "editorMode": "code", - "expr": "reth_process_resident_memory_bytes{instance=~\"$instance\"}", - "instant": false, - "legendFormat": "Resident", + "editorMode": "builder", + "expr": "reth_downloaders_bodies_queued_blocks{$instance_label=\"$instance\"}", + "hide": false, + "legendFormat": "Queued blocks", "range": true, - "refId": "A" + "refId": "G" } ], - "title": "Memory", + "title": "I/O", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "description": "100% = 1 core", + "description": "Internal errors in the bodies downloader. These are expected to happen from time to time.", "fieldConfig": { "defaults": { "color": { @@ -8714,6 +8790,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -8724,29 +8801,27 @@ } }, "mappings": [], + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 80 + "color": "green", + "value": 0 } ] }, - "unit": "percentunit" + "unit": "cps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 311 + "x": 12, + "y": 262 }, - "id": 99, + "id": 28, "options": { "legend": { "calcs": [], @@ -8755,11 +8830,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { @@ -8767,22 +8843,45 @@ "uid": "${datasource}" }, "editorMode": "builder", - "expr": "avg(rate(reth_process_cpu_seconds_total{instance=~\"$instance\"}[1m]))", - "instant": false, - "legendFormat": "Process", + "expr": "rate(reth_downloaders_bodies_timeout_errors{$instance_label=\"$instance\"}[$__rate_interval])", + "legendFormat": "Request timed out", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "rate(reth_downloaders_bodies_unexpected_errors{$instance_label=\"$instance\"}[$__rate_interval])", + "hide": false, + "legendFormat": "Unexpected error", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "rate(reth_downloaders_bodies_validation_errors{$instance_label=\"$instance\"}[$__rate_interval])", + "hide": false, + "legendFormat": "Invalid response", + "range": true, + "refId": "C" } ], - "title": "CPU", + "title": "Errors", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "description": "", + "description": "The number of connected peers and in-progress requests for bodies.", "fieldConfig": { "defaults": { "color": { @@ -8812,6 +8911,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -8826,25 +8926,25 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", "value": 80 } ] - }, - "unit": "none" + } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 311 + "x": 0, + "y": 270 }, - "id": 100, + "id": 35, "options": { "legend": { "calcs": [], @@ -8853,11 +8953,12 @@ "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { @@ -8865,34 +8966,33 @@ "uid": "${datasource}" }, "editorMode": "builder", - "expr": "reth_process_open_fds{instance=~\"$instance\"}", - "instant": false, - "legendFormat": "Open", + "expr": "reth_downloaders_bodies_in_flight_requests{$instance_label=\"$instance\"}", + "legendFormat": "In flight requests", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "reth_network_connected_peers{$instance_label=\"$instance\"}", + "hide": false, + "legendFormat": "Connected peers", + "range": true, + "refId": "B" } ], - "title": "File Descriptors", + "title": "Requests", "type": "timeseries" }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 319 - }, - "id": 105, - "panels": [], - "title": "Pruning", - "type": "row" - }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "description": "The number of blocks and size in bytes of those blocks", "fieldConfig": { "defaults": { "color": { @@ -8922,7 +9022,8 @@ "type": "linear" }, "showPoints": "auto", - "spanNulls": true, + "showValues": false, + "spanNulls": false, "stacking": { "group": "A", "mode": "none" @@ -8932,12 +9033,12 @@ } }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -8945,52 +9046,83 @@ } ] }, - "unit": "s" + "unit": "bytes" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "blocks" + } + ] + } + ] }, "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 320 + "x": 12, + "y": 270 }, - "id": 106, + "id": 73, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": false + "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "editorMode": "code", - "expr": "rate(reth_pruner_duration_seconds_sum{instance=~\"$instance\"}[$__rate_interval]) / rate(reth_pruner_duration_seconds_count{instance=~\"$instance\"}[$__rate_interval])", - "instant": false, - "legendFormat": "__auto", + "editorMode": "builder", + "expr": "reth_downloaders_bodies_buffered_blocks_size_bytes{$instance_label=\"$instance\"}", + "hide": false, + "legendFormat": "Buffered blocks size ", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "reth_downloaders_bodies_buffered_blocks{$instance_label=\"$instance\"}", + "hide": false, + "legendFormat": "Buffered blocks", + "range": true, + "refId": "B" } ], - "title": "Pruner duration, total", + "title": "Downloader buffer", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, + "description": "The number of blocks in a request and size in bytes of those block responses", "fieldConfig": { "defaults": { "color": { @@ -9020,7 +9152,8 @@ "type": "linear" }, "showPoints": "auto", - "spanNulls": true, + "showValues": false, + "spanNulls": false, "stacking": { "group": "A", "mode": "none" @@ -9030,12 +9163,12 @@ } }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -9043,17 +9176,34 @@ } ] }, - "unit": "s" + "unit": "bytes" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "blocks" + } + ] + } + ] }, "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 320 + "x": 0, + "y": 278 }, - "id": 107, + "id": 102, "options": { "legend": { "calcs": [], @@ -9062,11 +9212,12 @@ "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { @@ -9074,21 +9225,60 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "rate(reth_pruner_segments_duration_seconds_sum{instance=~\"$instance\"}[$__rate_interval]) / rate(reth_pruner_segments_duration_seconds_count{instance=~\"$instance\"}[$__rate_interval])", + "expr": "reth_downloaders_bodies_response_response_size_bytes{$instance_label=\"$instance\"}", + "hide": false, + "legendFormat": "Response size", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "reth_downloaders_bodies_response_response_length{$instance_label=\"$instance\"}", + "hide": false, + "legendFormat": "Individual response length (number of bodies in response)", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "reth_downloaders_bodies_response_response_size_bytes / reth_downloaders_bodies_response_response_length{$instance_label=\"$instance\"}", + "hide": false, "instant": false, - "legendFormat": "{{segment}}", + "legendFormat": "Mean body size in response", "range": true, - "refId": "A" + "refId": "C" } ], - "title": "Pruner duration, per segment", + "title": "Block body response sizes", "type": "timeseries" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 286 + }, + "id": 226, + "panels": [], + "title": "Eth Requests", + "type": "row" + }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, + "description": "", "fieldConfig": { "defaults": { "color": { @@ -9117,8 +9307,9 @@ "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", - "spanNulls": true, + "showPoints": "never", + "showValues": false, + "spanNulls": false, "stacking": { "group": "A", "mode": "none" @@ -9132,7 +9323,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -9140,17 +9332,42 @@ } ] }, - "unit": "none" + "unit": "short" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "http" + }, + "properties": [ + { + "id": "displayName", + "value": "HTTP" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ws" + }, + "properties": [ + { + "id": "displayName", + "value": "WebSocket" + } + ] + } + ] }, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 328 + "y": 287 }, - "id": 217, + "id": 225, "options": { "legend": { "calcs": [], @@ -9159,45 +9376,38 @@ "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "editorMode": "code", - "expr": "reth_pruner_segments_highest_pruned_block{instance=~\"$instance\"}", - "instant": false, - "legendFormat": "{{segment}}", + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(reth_network_eth_headers_requests_received_total{$instance_label=\"$instance\"}[$__rate_interval])", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "Headers Requests/s", "range": true, - "refId": "A" + "refId": "A", + "useBackend": false } ], - "title": "Highest pruned block, per segment", + "title": "Headers Requests Received", "type": "timeseries" }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 336 - }, - "id": 108, - "panels": [], - "title": "RPC server", - "type": "row" - }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "description": "", "fieldConfig": { @@ -9214,7 +9424,7 @@ "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 10, + "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, @@ -9229,6 +9439,7 @@ "type": "linear" }, "showPoints": "never", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -9243,7 +9454,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -9283,10 +9495,10 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 337 + "x": 12, + "y": 287 }, - "id": 109, + "id": 227, "options": { "legend": { "calcs": [], @@ -9295,11 +9507,13 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, + "maxHeight": 600, "mode": "multi", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { @@ -9307,115 +9521,26 @@ "uid": "${datasource}" }, "disableTextWrap": false, - "editorMode": "code", - "expr": "sum(reth_rpc_server_connections_connections_opened_total{instance=~\"$instance\"} - reth_rpc_server_connections_connections_closed_total{instance=~\"$instance\"}) by (transport)", + "editorMode": "builder", + "expr": "rate(reth_network_eth_receipts_requests_received_total{$instance_label=\"$instance\"}[$__rate_interval])", "format": "time_series", "fullMetaSearch": false, "includeNullMetadata": true, - "legendFormat": "{{transport}}", + "legendFormat": "Receipts Requests/s", "range": true, "refId": "A", "useBackend": false } ], - "title": "Active Connections", + "title": "Receipts Requests Received", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "description": "", - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 337 - }, - "id": 111, - "maxDataPoints": 25, - "options": { - "calculate": false, - "cellGap": 1, - "cellValues": { - "unit": "s" - }, - "color": { - "exponent": 0.2, - "fill": "dark-orange", - "min": 0, - "mode": "opacity", - "reverse": false, - "scale": "exponential", - "scheme": "Oranges", - "steps": 128 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto", - "value": "Latency time" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": false - }, - "yAxis": { - "axisLabel": "Quantile", - "axisPlacement": "left", - "reverse": false, - "unit": "percentunit" - } - }, - "pluginVersion": "11.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "avg(max_over_time(reth_rpc_server_connections_request_time_seconds{instance=~\"$instance\"}[$__rate_interval]) > 0) by (quantile)", - "format": "time_series", - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Request Latency time", - "type": "heatmap" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, "fieldConfig": { "defaults": { "color": { @@ -9429,7 +9554,7 @@ "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, - "drawStyle": "points", + "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { @@ -9444,7 +9569,8 @@ "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", + "showPoints": "never", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -9459,7 +9585,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -9467,142 +9594,84 @@ } ] }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 345 - }, - "id": 120, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "unit": "short" }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "max(max_over_time(reth_rpc_server_calls_time_seconds{instance=~\"$instance\"}[$__rate_interval])) by (method) > 0", - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Maximum call latency per method", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "http" + }, + "properties": [ + { + "id": "displayName", + "value": "HTTP" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ws" }, - "scaleDistribution": { - "type": "linear" - } + "properties": [ + { + "id": "displayName", + "value": "WebSocket" + } + ] } - }, - "overrides": [] + ] }, "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 345 + "x": 0, + "y": 295 }, - "id": 112, - "maxDataPoints": 25, + "id": 235, "options": { - "calculate": false, - "cellGap": 1, - "cellValues": { - "unit": "s" - }, - "color": { - "exponent": 0.2, - "fill": "dark-orange", - "min": 0, - "mode": "opacity", - "reverse": false, - "scale": "exponential", - "scheme": "Oranges", - "steps": 128 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto", - "value": "Latency time" + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": false - }, - "yAxis": { - "axisLabel": "Quantile", - "axisPlacement": "left", - "reverse": false, - "unit": "percentunit" + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "none" } }, - "pluginVersion": "11.2.0", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "editorMode": "code", - "exemplar": false, - "expr": "avg(max_over_time(reth_rpc_server_calls_time_seconds{instance=~\"$instance\"}[$__rate_interval]) > 0) by (quantile)", + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(reth_network_eth_bodies_requests_received_total{$instance_label=\"$instance\"}[$__rate_interval])", "format": "time_series", - "instant": false, - "legendFormat": "{{quantile}}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "Bodies Requests/s", "range": true, - "refId": "A" + "refId": "A", + "useBackend": false } ], - "title": "Call Latency time", - "type": "heatmap" + "title": "Bodies Requests Received", + "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, + "description": "", "fieldConfig": { "defaults": { "color": { @@ -9631,7 +9700,8 @@ "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", + "showPoints": "never", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -9646,49 +9716,39 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", "value": 80 } ] - } + }, + "unit": "short" }, "overrides": [ { "matcher": { - "id": "byRegexp", - "options": "/.*cached items.*/" - }, - "properties": [ - { - "id": "custom.axisLabel", - "value": "Items" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/.*consumers.*/" + "id": "byName", + "options": "http" }, "properties": [ { - "id": "custom.axisLabel", - "value": "Queued consumers" + "id": "displayName", + "value": "HTTP" } ] }, { "matcher": { - "id": "byRegexp", - "options": "/.memory usage*/" + "id": "byName", + "options": "ws" }, "properties": [ { - "id": "unit", - "value": "decbytes" + "id": "displayName", + "value": "WebSocket" } ] } @@ -9697,10 +9757,10 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 353 + "x": 12, + "y": 295 }, - "id": 198, + "id": 234, "options": { "legend": { "calcs": [], @@ -9709,11 +9769,13 @@ "showLegend": true }, "tooltip": { - "mode": "single", + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { @@ -9722,143 +9784,39 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "reth_rpc_eth_cache_cached_count{instance=\"$instance\", cache=\"headers\"}", + "expr": "rate(reth_network_eth_node_data_requests_received_total{$instance_label=\"$instance\"}[$__rate_interval])", + "format": "time_series", "fullMetaSearch": false, "includeNullMetadata": true, - "instant": false, - "legendFormat": "Headers cache cached items", + "legendFormat": "Node Data Requests/s", "range": true, "refId": "A", "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_rpc_eth_cache_queued_consumers_count{instance=\"$instance\", cache=\"receipts\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Receipts cache queued consumers", - "range": true, - "refId": "B", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_rpc_eth_cache_queued_consumers_count{instance=\"$instance\", cache=\"headers\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Headers cache queued consumers", - "range": true, - "refId": "C", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_rpc_eth_cache_queued_consumers_count{instance=\"$instance\", cache=\"blocks\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Block cache queued consumers", - "range": true, - "refId": "D", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_rpc_eth_cache_memory_usage{instance=\"$instance\", cache=\"blocks\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Blocks cache memory usage", - "range": true, - "refId": "E", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_rpc_eth_cache_cached_count{instance=\"$instance\", cache=\"receipts\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Receipts cache cached items", - "range": true, - "refId": "F", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_rpc_eth_cache_memory_usage{instance=\"$instance\", cache=\"receipts\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Receipts cache memory usage", - "range": true, - "refId": "G", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_rpc_eth_cache_cached_count{instance=\"$instance\", cache=\"blocks\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Block cache cached items", - "range": true, - "refId": "H", - "useBackend": false } ], - "title": "RPC Cache Metrics", + "title": "Node Data Requests Received", "type": "timeseries" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 303 + }, + "id": 68, + "panels": [], + "repeat": "instance", + "title": "Payload Builder", + "type": "row" + }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, + "description": "Number of active jobs", "fieldConfig": { "defaults": { "color": { @@ -9881,13 +9839,14 @@ "viz": false }, "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 1, + "lineInterpolation": "linear", + "lineWidth": 3, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -9902,25 +9861,25 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", "value": 80 } ] - }, - "unit": "reqps" + } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 353 + "x": 0, + "y": 304 }, - "id": 246, + "id": 60, "options": { "legend": { "calcs": [], @@ -9929,47 +9888,34 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "editorMode": "code", - "expr": "sum(rate(reth_rpc_server_calls_successful_total{instance =~ \"$instance\"}[$__rate_interval])) by (method) > 0", - "instant": false, - "legendFormat": "{{method}}", + "editorMode": "builder", + "expr": "reth_payloads_active_jobs{$instance_label=\"$instance\"}", + "legendFormat": "Active Jobs", "range": true, "refId": "A" } ], - "title": "RPC Throughput", + "title": "Active Jobs", "type": "timeseries" }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 361 - }, - "id": 236, - "panels": [], - "title": "Execution Extensions", - "type": "row" - }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "description": "The total number of canonical state notifications sent to ExExes.", + "description": "Total number of initiated jobs", "fieldConfig": { "defaults": { "color": { @@ -9993,12 +9939,13 @@ }, "insertNulls": false, "lineInterpolation": "linear", - "lineWidth": 1, + "lineWidth": 3, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -10013,7 +9960,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -10027,10 +9975,10 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 362 + "x": 12, + "y": 304 }, - "id": 237, + "id": 62, "options": { "legend": { "calcs": [], @@ -10039,11 +9987,12 @@ "showLegend": true }, "tooltip": { - "mode": "multi", + "hideZeros": false, + "mode": "single", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { @@ -10051,22 +10000,21 @@ "uid": "${datasource}" }, "editorMode": "builder", - "expr": "reth_exex_notifications_sent_total{instance=~\"$instance\"}", - "hide": false, - "legendFormat": "Total Notifications Sent", + "expr": "reth_payloads_initiated_jobs{$instance_label=\"$instance\"}", + "legendFormat": "Initiated Jobs", "range": true, - "refId": "B" + "refId": "A" } ], - "title": "Total Notifications Sent", + "title": "Initiated Jobs", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "description": "The total number of events ExExes have sent to the manager.", + "description": "Total number of failed jobs", "fieldConfig": { "defaults": { "color": { @@ -10090,12 +10038,13 @@ }, "insertNulls": false, "lineInterpolation": "linear", - "lineWidth": 1, + "lineWidth": 3, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -10110,7 +10059,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -10124,10 +10074,10 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 362 + "x": 0, + "y": 312 }, - "id": 238, + "id": 64, "options": { "legend": { "calcs": [], @@ -10136,11 +10086,12 @@ "showLegend": true }, "tooltip": { - "mode": "multi", + "hideZeros": false, + "mode": "single", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { @@ -10148,22 +10099,33 @@ "uid": "${datasource}" }, "editorMode": "builder", - "expr": "reth_exex_events_sent_total{instance=~\"$instance\"}", - "hide": false, - "legendFormat": "Total Events Sent", + "expr": "reth_payloads_failed_jobs{$instance_label=\"$instance\"}", + "legendFormat": "Failed Jobs", "range": true, - "refId": "B" + "refId": "A" } ], - "title": "Total Events Sent", + "title": "Failed Jobs", "type": "timeseries" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 320 + }, + "id": 105, + "panels": [], + "title": "Pruning", + "type": "row" + }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "description": "Current and Maximum capacity of the internal state notifications buffer.", "fieldConfig": { "defaults": { "color": { @@ -10193,7 +10155,8 @@ "type": "linear" }, "showPoints": "auto", - "spanNulls": false, + "showValues": false, + "spanNulls": true, "stacking": { "group": "A", "mode": "none" @@ -10203,18 +10166,21 @@ } }, "mappings": [], + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", "value": 80 } ] - } + }, + "unit": "s" }, "overrides": [] }, @@ -10222,57 +10188,45 @@ "h": 8, "w": 12, "x": 0, - "y": 370 + "y": 321 }, - "id": 239, + "id": 106, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { - "mode": "multi", + "hideZeros": false, + "mode": "single", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "editorMode": "builder", - "expr": "reth_exex_manager_current_capacity{instance=~\"$instance\"}", - "hide": false, - "legendFormat": "Current size", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "builder", - "expr": "max_over_time(reth_exex_manager_max_capacity{instance=~\"$instance\"}[1h])", - "hide": false, - "legendFormat": "Max size", + "editorMode": "code", + "expr": "rate(reth_pruner_duration_seconds_sum{$instance_label=\"$instance\"}[$__rate_interval]) / rate(reth_pruner_duration_seconds_count{$instance_label=\"$instance\"}[$__rate_interval])", + "instant": false, + "legendFormat": "__auto", "range": true, - "refId": "C" + "refId": "A" } ], - "title": "Current and Max Capacity", + "title": "Pruner duration, total", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "description": "Current size of the internal state notifications buffer.", "fieldConfig": { "defaults": { "color": { @@ -10302,7 +10256,8 @@ "type": "linear" }, "showPoints": "auto", - "spanNulls": false, + "showValues": false, + "spanNulls": true, "stacking": { "group": "A", "mode": "none" @@ -10312,18 +10267,21 @@ } }, "mappings": [], + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", "value": 80 } ] - } + }, + "unit": "s" }, "overrides": [] }, @@ -10331,9 +10289,9 @@ "h": 8, "w": 12, "x": 12, - "y": 370 + "y": 321 }, - "id": 219, + "id": 107, "options": { "legend": { "calcs": [], @@ -10342,45 +10300,80 @@ "showLegend": true }, "tooltip": { - "mode": "multi", + "hideZeros": false, + "mode": "single", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "editorMode": "builder", - "expr": "reth_exex_manager_buffer_size{instance=~\"$instance\"}", - "hide": false, - "legendFormat": "Max size", + "editorMode": "code", + "expr": "rate(reth_pruner_segments_duration_seconds_sum{$instance_label=\"$instance\"}[$__rate_interval]) / rate(reth_pruner_segments_duration_seconds_count{$instance_label=\"$instance\"}[$__rate_interval])", + "instant": false, + "legendFormat": "{{segment}}", "range": true, - "refId": "B" + "refId": "A" } ], - "title": "Buffer Size", + "title": "Pruner duration, per segment", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "description": "Total number of ExExes installed in the node", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -10396,43 +10389,40 @@ "h": 8, "w": 12, "x": 0, - "y": 378 + "y": 329 }, - "id": 220, + "id": 217, "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } }, - "pluginVersion": "11.2.0", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "editorMode": "builder", - "expr": "reth_exex_manager_num_exexs{instance=~\"$instance\"}", - "hide": false, - "legendFormat": "Number of ExExs", + "description": "Archive and full nodes prune headers, transactions and receipts in MDBX (hot db) after they have been written to static files (cold db). Full nodes additionally prune history indices.", + "editorMode": "code", + "expr": "reth_pruner_segments_highest_pruned_block{$instance_label=\"$instance\"}", + "instant": false, + "legendFormat": "{{segment}}", "range": true, "refId": "A" } ], - "title": "Number of ExExes", - "type": "stat" + "title": "Highest pruned block, per segment", + "type": "timeseries" }, { "collapsed": false, @@ -10440,19 +10430,18 @@ "h": 1, "w": 24, "x": 0, - "y": 386 + "y": 337 }, - "id": 241, + "id": 97, "panels": [], - "title": "Execution Extensions Write-Ahead Log", + "title": "Process", "type": "row" }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "description": "", "fieldConfig": { "defaults": { "color": { @@ -10482,6 +10471,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -10491,30 +10481,44 @@ "mode": "off" } }, - "fieldMinMax": false, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", "value": 80 } ] - } + }, + "unit": "decbytes" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Retained" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] }, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 387 + "y": 338 }, - "id": 243, + "id": 98, "options": { "legend": { "calcs": [], @@ -10523,22 +10527,35 @@ "showLegend": true }, "tooltip": { - "mode": "multi", + "hideZeros": false, + "mode": "single", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "reth_jemalloc_active{$instance_label=\"$instance\"}", + "instant": false, + "legendFormat": "Active", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - "editorMode": "code", - "expr": "reth_exex_wal_lowest_committed_block_height{instance=~\"$instance\"}", + "editorMode": "builder", + "expr": "reth_jemalloc_allocated{$instance_label=\"$instance\"}", "hide": false, "instant": false, - "legendFormat": "Lowest Block", + "legendFormat": "Allocated", "range": true, "refId": "B" }, @@ -10547,16 +10564,55 @@ "type": "prometheus", "uid": "${datasource}" }, - "editorMode": "code", - "expr": "reth_exex_wal_highest_committed_block_height{instance=~\"$instance\"}", + "editorMode": "builder", + "expr": "reth_jemalloc_mapped{$instance_label=\"$instance\"}", "hide": false, "instant": false, - "legendFormat": "Highest Block", + "legendFormat": "Mapped", "range": true, "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "reth_jemalloc_metadata{$instance_label=\"$instance\"}", + "hide": false, + "instant": false, + "legendFormat": "Metadata", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "reth_jemalloc_resident{$instance_label=\"$instance\"}", + "hide": false, + "instant": false, + "legendFormat": "Resident", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "reth_jemalloc_retained{$instance_label=\"$instance\"}", + "hide": false, + "instant": false, + "legendFormat": "Retained", + "range": true, + "refId": "F" } ], - "title": "Current Committed Block Heights", + "title": "Jemalloc Memory", "type": "timeseries" }, { @@ -10594,6 +10650,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -10603,20 +10660,21 @@ "mode": "off" } }, - "fieldMinMax": false, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", "value": 80 } ] - } + }, + "unit": "decbytes" }, "overrides": [] }, @@ -10624,9 +10682,9 @@ "h": 8, "w": 12, "x": 12, - "y": 387 + "y": 338 }, - "id": 244, + "id": 101, "options": { "legend": { "calcs": [], @@ -10635,40 +10693,27 @@ "showLegend": true }, "tooltip": { - "mode": "multi", + "hideZeros": false, + "mode": "single", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "reth_exex_wal_committed_blocks_count{instance=~\"$instance\"}", - "hide": false, - "instant": false, - "legendFormat": "Committed Blocks", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "reth_exex_wal_notifications_count{instance=~\"$instance\"}", - "hide": false, + "expr": "reth_process_resident_memory_bytes{$instance_label=\"$instance\"}", "instant": false, - "legendFormat": "Notifications", + "legendFormat": "Resident", "range": true, - "refId": "B" + "refId": "A" } ], - "title": "Number of entities", + "title": "Memory", "type": "timeseries" }, { @@ -10676,7 +10721,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "", + "description": "100% = 1 core", "fieldConfig": { "defaults": { "color": { @@ -10706,6 +10751,7 @@ "type": "linear" }, "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -10715,13 +10761,13 @@ "mode": "off" } }, - "fieldMinMax": false, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -10729,7 +10775,7 @@ } ] }, - "unit": "bytes" + "unit": "percentunit" }, "overrides": [] }, @@ -10737,53 +10783,40 @@ "h": 8, "w": 12, "x": 0, - "y": 395 + "y": 346 }, - "id": 245, + "id": 99, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": false + "showLegend": true }, "tooltip": { - "mode": "multi", + "hideZeros": false, + "mode": "single", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "editorMode": "code", - "expr": "reth_exex_wal_size_bytes{instance=~\"$instance\"}", - "hide": false, + "editorMode": "builder", + "expr": "avg(rate(reth_process_cpu_seconds_total{$instance_label=\"$instance\"}[1m]))", "instant": false, - "legendFormat": "__auto", + "legendFormat": "Process", "range": true, - "refId": "C" + "refId": "A" } ], - "title": "Total size of all notifications", + "title": "CPU", "type": "timeseries" }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 403 - }, - "id": 226, - "panels": [], - "title": "Eth Requests", - "type": "row" - }, { "datasource": { "type": "prometheus", @@ -10818,7 +10851,8 @@ "scaleDistribution": { "type": "linear" }, - "showPoints": "never", + "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -10833,7 +10867,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -10841,42 +10876,17 @@ } ] }, - "unit": "short" + "unit": "none" }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "http" - }, - "properties": [ - { - "id": "displayName", - "value": "HTTP" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "ws" - }, - "properties": [ - { - "id": "displayName", - "value": "WebSocket" - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 404 + "x": 12, + "y": 346 }, - "id": 225, + "id": 100, "options": { "legend": { "calcs": [], @@ -10885,31 +10895,27 @@ "showLegend": true }, "tooltip": { - "maxHeight": 600, - "mode": "multi", + "hideZeros": false, + "mode": "single", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "disableTextWrap": false, "editorMode": "builder", - "expr": "rate(reth_network_eth_headers_requests_received_total{instance=~\"$instance\"}[$__rate_interval])", - "format": "time_series", - "fullMetaSearch": false, - "includeNullMetadata": true, - "legendFormat": "Headers Requests/s", + "expr": "reth_process_open_fds{$instance_label=\"$instance\"}", + "instant": false, + "legendFormat": "Open", "range": true, - "refId": "A", - "useBackend": false + "refId": "A" } ], - "title": "Headers Requests Received", + "title": "File Descriptors", "type": "timeseries" }, { @@ -10917,7 +10923,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "", + "description": "Tracks the number of critical tasks currently ran by the executor.", "fieldConfig": { "defaults": { "color": { @@ -10946,7 +10952,8 @@ "scaleDistribution": { "type": "linear" }, - "showPoints": "never", + "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -10961,50 +10968,26 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { - "color": "red", - "value": 80 + "color": "semi-dark-red", + "value": 0 } ] }, - "unit": "short" + "unit": "tasks" }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "http" - }, - "properties": [ - { - "id": "displayName", - "value": "HTTP" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "ws" - }, - "properties": [ - { - "id": "displayName", - "value": "WebSocket" - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 404 + "x": 0, + "y": 354 }, - "id": 227, + "id": 248, "options": { "legend": { "calcs": [], @@ -11013,31 +10996,28 @@ "showLegend": true }, "tooltip": { - "maxHeight": 600, - "mode": "multi", + "hideZeros": false, + "mode": "single", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "rate(reth_network_eth_receipts_requests_received_total{instance=~\"$instance\"}[$__rate_interval])", - "format": "time_series", - "fullMetaSearch": false, - "includeNullMetadata": true, - "legendFormat": "Receipts Requests/s", + "editorMode": "code", + "expr": "reth_executor_spawn_critical_tasks_total{$instance_label=\"$instance\"}- reth_executor_spawn_finished_critical_tasks_total{$instance_label=\"$instance\"}", + "hide": false, + "instant": false, + "legendFormat": "Tasks running", "range": true, - "refId": "A", - "useBackend": false + "refId": "C" } ], - "title": "Receipts Requests Received", + "title": "Task Executor critical tasks", "type": "timeseries" }, { @@ -11045,7 +11025,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "", + "description": "Tracks the number of regular tasks currently ran by the executor.", "fieldConfig": { "defaults": { "color": { @@ -11074,7 +11054,8 @@ "scaleDistribution": { "type": "linear" }, - "showPoints": "never", + "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -11089,38 +11070,27 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { - "color": "red", + "color": "semi-dark-red", "value": 80 } ] }, - "unit": "short" + "unit": "tasks/s" }, "overrides": [ { "matcher": { - "id": "byName", - "options": "http" - }, - "properties": [ - { - "id": "displayName", - "value": "HTTP" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "ws" + "id": "byFrameRefID", + "options": "C" }, "properties": [ { - "id": "displayName", - "value": "WebSocket" + "id": "unit", + "value": "tasks" } ] } @@ -11129,10 +11099,10 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 412 + "x": 12, + "y": 354 }, - "id": 235, + "id": 247, "options": { "legend": { "calcs": [], @@ -11140,40 +11110,55 @@ "placement": "bottom", "showLegend": true }, - "tooltip": { - "maxHeight": 600, - "mode": "multi", + "tooltip": { + "hideZeros": false, + "mode": "single", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "disableTextWrap": false, - "editorMode": "builder", - "expr": "rate(reth_network_eth_bodies_requests_received_total{instance=~\"$instance\"}[$__rate_interval])", - "format": "time_series", + "editorMode": "code", + "exemplar": false, + "expr": "rate(reth_executor_spawn_regular_tasks_total{$instance_label=\"$instance\"}[$__rate_interval])", "fullMetaSearch": false, - "includeNullMetadata": true, - "legendFormat": "Bodies Requests/s", + "hide": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "Tasks started", "range": true, "refId": "A", "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_executor_spawn_regular_tasks_total{$instance_label=\"$instance\"} - reth_executor_spawn_finished_regular_tasks_total{$instance_label=\"$instance\"}", + "hide": false, + "instant": false, + "legendFormat": "Tasks running", + "range": true, + "refId": "C" } ], - "title": "Bodies Requests Received", + "title": "Task Executor regular tasks", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, - "description": "", + "description": "Tracks the number of regular blocking tasks currently ran by the executor.", "fieldConfig": { "defaults": { "color": { @@ -11202,7 +11187,8 @@ "scaleDistribution": { "type": "linear" }, - "showPoints": "never", + "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", @@ -11217,38 +11203,27 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { - "color": "red", + "color": "semi-dark-red", "value": 80 } ] }, - "unit": "short" + "unit": "tasks/s" }, "overrides": [ { "matcher": { - "id": "byName", - "options": "http" - }, - "properties": [ - { - "id": "displayName", - "value": "HTTP" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "ws" + "id": "byFrameRefID", + "options": "C" }, "properties": [ { - "id": "displayName", - "value": "WebSocket" + "id": "unit", + "value": "tasks" } ] } @@ -11257,10 +11232,10 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 412 + "x": 0, + "y": 362 }, - "id": 234, + "id": 1007, "options": { "legend": { "calcs": [], @@ -11269,36 +11244,864 @@ "showLegend": true }, "tooltip": { - "maxHeight": 600, - "mode": "multi", + "hideZeros": false, + "mode": "single", "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_PROMETHEUS}" }, "disableTextWrap": false, - "editorMode": "builder", - "expr": "rate(reth_network_eth_node_data_requests_received_total{instance=~\"$instance\"}[$__rate_interval])", - "format": "time_series", + "editorMode": "code", + "exemplar": false, + "expr": "rate(reth_executor_spawn_regular_blocking_tasks_total{$instance_label=\"$instance\"}[$__rate_interval])", "fullMetaSearch": false, - "includeNullMetadata": true, - "legendFormat": "Node Data Requests/s", + "hide": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "Tasks started", "range": true, "refId": "A", "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_executor_spawn_regular_blocking_tasks_total{$instance_label=\"$instance\"} - reth_executor_spawn_finished_regular_blocking_tasks_total{$instance_label=\"$instance\"}", + "hide": false, + "instant": false, + "legendFormat": "Tasks running", + "range": true, + "refId": "C" } ], - "title": "Node Data Requests Received", + "title": "Task Executor regular blocking tasks", "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 370 + }, + "id": 236, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The total number of canonical state notifications sent to ExExes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 370 + }, + "id": 237, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "reth_exex_notifications_sent_total{$instance_label=\"$instance\"}", + "hide": false, + "legendFormat": "Total Notifications Sent", + "range": true, + "refId": "B" + } + ], + "title": "Total Notifications Sent", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The total number of events ExExes have sent to the manager.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 370 + }, + "id": 238, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "reth_exex_events_sent_total{$instance_label=\"$instance\"}", + "hide": false, + "legendFormat": "Total Events Sent", + "range": true, + "refId": "B" + } + ], + "title": "Total Events Sent", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Current and Maximum capacity of the internal state notifications buffer.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 378 + }, + "id": 239, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "reth_exex_manager_current_capacity{$instance_label=\"$instance\"}", + "hide": false, + "legendFormat": "Current size", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "max_over_time(reth_exex_manager_max_capacity{$instance_label=\"$instance\"}[1h])", + "hide": false, + "legendFormat": "Max size", + "range": true, + "refId": "C" + } + ], + "title": "Current and Max Capacity", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Current size of the internal state notifications buffer.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 378 + }, + "id": 219, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "reth_exex_manager_buffer_size{$instance_label=\"$instance\"}", + "hide": false, + "legendFormat": "Max size", + "range": true, + "refId": "B" + } + ], + "title": "Buffer Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total number of ExExes installed in the node", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 386 + }, + "id": 220, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "reth_exex_manager_num_exexs{$instance_label=\"$instance\"}", + "hide": false, + "legendFormat": "Number of ExExs", + "range": true, + "refId": "A" + } + ], + "title": "Number of ExExes", + "type": "stat" + } + ], + "title": "Execution Extensions", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 371 + }, + "id": 241, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 371 + }, + "id": 243, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "reth_exex_wal_lowest_committed_block_height{$instance_label=\"$instance\"}", + "hide": false, + "instant": false, + "legendFormat": "Lowest Block", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_exex_wal_highest_committed_block_height{$instance_label=\"$instance\"}", + "hide": false, + "instant": false, + "legendFormat": "Highest Block", + "range": true, + "refId": "C" + } + ], + "title": "Current Committed Block Heights", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 371 + }, + "id": 244, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_exex_wal_committed_blocks_count{$instance_label=\"$instance\"}", + "hide": false, + "instant": false, + "legendFormat": "Committed Blocks", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "reth_exex_wal_notifications_count{$instance_label=\"$instance\"}", + "hide": false, + "instant": false, + "legendFormat": "Notifications", + "range": true, + "refId": "B" + } + ], + "title": "Number of entities", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 379 + }, + "id": 245, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "reth_exex_wal_size_bytes{$instance_label=\"$instance\"}", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "C" + } + ], + "title": "Total size of all notifications", + "type": "timeseries" + } + ], + "title": "Execution Extensions Write-Ahead Log", + "type": "row" } ], "refresh": "5s", - "schemaVersion": 40, + "schemaVersion": 42, "tags": [], "templating": { "list": [ @@ -11308,38 +12111,19 @@ "type": "prometheus", "uid": "${datasource}" }, - "definition": "query_result(reth_info)", - "includeAll": false, - "label": "Job", - "name": "job", - "options": [], - "query": { - "qryType": 3, - "query": "query_result(reth_info)", - "refId": "PrometheusVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "/.*job=\\\"([^\\\"]*).*/", - "type": "query" - }, - { - "current": {}, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "definition": "query_result(reth_info{job=\"${job}\"})", + "definition": "label_values(reth_info,$instance_label)", "includeAll": false, - "label": "Instance (auto-selected)", + "label": "Instance", "name": "instance", "options": [], "query": { - "qryType": 3, - "query": "query_result(reth_info{job=\"${job}\"})", + "qryType": 1, + "query": "label_values(reth_info,$instance_label)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, - "regex": "/.*instance=\\\"([^\\\"]*).*/", + "regex": "", + "sort": 1, "type": "query" }, { @@ -11352,17 +12136,37 @@ "refresh": 1, "regex": "", "type": "datasource" + }, + { + "hide": 2, + "label": "Instance Label", + "name": "instance_label", + "query": "${VAR_INSTANCE_LABEL}", + "skipUrlSync": true, + "type": "constant", + "current": { + "value": "${VAR_INSTANCE_LABEL}", + "text": "${VAR_INSTANCE_LABEL}", + "selected": false + }, + "options": [ + { + "value": "${VAR_INSTANCE_LABEL}", + "text": "${VAR_INSTANCE_LABEL}", + "selected": false + } + ] } ] }, "time": { - "from": "now-1h", + "from": "now-12h", "to": "now" }, "timepicker": {}, "timezone": "", "title": "Reth", "uid": "2k8BXz24x", - "version": 2, + "version": 3, "weekStart": "" } diff --git a/etc/grafana/dashboards/reth-mempool.json b/etc/grafana/dashboards/reth-mempool.json index ca2dffeecac..188161d27a3 100644 --- a/etc/grafana/dashboards/reth-mempool.json +++ b/etc/grafana/dashboards/reth-mempool.json @@ -1,3962 +1,3984 @@ { - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "label": "Prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } - ], - "__elements": {}, - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "11.2.0" - }, - { - "type": "panel", - "id": "piechart", - "name": "Pie chart", - "version": "" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "1.0.0" - }, - { - "type": "panel", - "id": "stat", - "name": "Stat", - "version": "" - }, - { - "type": "panel", - "id": "timeseries", - "name": "Time series", - "version": "" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "description": "Metrics for transaction P2P gossip and the local view of mempool data", - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": null, - "links": [], - "liveNow": false, - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 96, - "panels": [], - "repeat": "instance", - "repeatDirection": "h", - "title": "Overview", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 3, - "x": 0, - "y": 1 - }, - "id": 22, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "text": { - "valueSize": 20 - }, - "textMode": "name", - "wideLayout": true - }, - "pluginVersion": "11.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "builder", - "exemplar": false, - "expr": "reth_info{instance=~\"$instance\"}", - "instant": true, - "legendFormat": "{{version}}", - "range": false, - "refId": "A" + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" } - ], - "title": "Version", - "transparent": true, - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 3, - "y": 1 - }, - "id": 192, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "text": { - "valueSize": 20 - }, - "textMode": "name", - "wideLayout": true - }, - "pluginVersion": "11.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "builder", - "exemplar": false, - "expr": "reth_info{instance=~\"$instance\"}", - "instant": true, - "legendFormat": "{{build_timestamp}}", - "range": false, - "refId": "A" - } - ], - "title": "Build Timestamp", - "transparent": true, - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 3, - "x": 9, - "y": 1 - }, - "id": 193, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "text": { - "valueSize": 20 - }, - "textMode": "name", - "wideLayout": true - }, - "pluginVersion": "11.2.0", - "targets": [ + ], + "__elements": {}, + "__requires": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "builder", - "exemplar": false, - "expr": "reth_info{instance=~\"$instance\"}", - "instant": true, - "legendFormat": "{{git_sha}}", - "range": false, - "refId": "A" - } - ], - "title": "Git SHA", - "transparent": true, - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 2, - "x": 12, - "y": 1 - }, - "id": 195, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "text": { - "valueSize": 20 - }, - "textMode": "name", - "wideLayout": true - }, - "pluginVersion": "11.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "builder", - "exemplar": false, - "expr": "reth_info{instance=~\"$instance\"}", - "instant": true, - "legendFormat": "{{build_profile}}", - "range": false, - "refId": "A" - } - ], - "title": "Build Profile", - "transparent": true, - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 5, - "x": 14, - "y": 1 - }, - "id": 196, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "text": { - "valueSize": 20 - }, - "textMode": "name", - "wideLayout": true - }, - "pluginVersion": "11.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "builder", - "exemplar": false, - "expr": "reth_info{instance=~\"$instance\"}", - "instant": true, - "legendFormat": "{{target_triple}}", - "range": false, - "refId": "A" - } - ], - "title": "Target Triple", - "transparent": true, - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 5, - "x": 19, - "y": 1 - }, - "id": 197, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "text": { - "valueSize": 20 - }, - "textMode": "name", - "wideLayout": true - }, - "pluginVersion": "11.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "builder", - "exemplar": false, - "expr": "reth_info{instance=~\"$instance\"}", - "instant": true, - "legendFormat": "{{cargo_features}}", - "range": false, - "refId": "A" - } - ], - "title": "Cargo Features", - "transparent": true, - "type": "stat" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 4 - }, - "id": 89, - "panels": [], - "repeat": "instance", - "repeatDirection": "h", - "title": "Transaction Pool", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Tracks the entries, byte size, failed inserts and file deletes of the blob store", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 5 - }, - "id": 115, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_transaction_pool_blobstore_entries{instance=~\"$instance\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Entries", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_transaction_pool_blobstore_byte_size{instance=~\"$instance\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Bytesize", - "range": true, - "refId": "B", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_transaction_pool_blobstore_failed_inserts{instance=~\"$instance\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Failed Inserts", - "range": true, - "refId": "C", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_transaction_pool_blobstore_failed_deletes{instance=~\"$instance\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Failed Deletes", - "range": true, - "refId": "D", - "useBackend": false - } - ], - "title": "Blob store", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Tracks a heuristic of the memory footprint of the various transaction pool sub-pools", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 5 - }, - "id": 210, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "builder", - "expr": "reth_transaction_pool_basefee_pool_size_bytes{instance=~\"$instance\"}", - "legendFormat": "Base Fee Pool Size", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "builder", - "expr": "reth_transaction_pool_pending_pool_size_bytes{instance=~\"$instance\"}", - "hide": false, - "legendFormat": "Pending Pool Size", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "builder", - "expr": "reth_transaction_pool_queued_pool_size_bytes{instance=~\"$instance\"}", - "hide": false, - "legendFormat": "Queued Pool Size", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "builder", - "expr": "reth_transaction_pool_blob_pool_size_bytes{instance=~\"$instance\"}", - "legendFormat": "Blob Pool Size", - "range": true, - "refId": "D" - } - ], - "title": "Subpool Sizes in Bytes", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Transaction pool maintenance metrics", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 13 - }, - "id": 91, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_transaction_pool_dirty_accounts{instance=~\"$instance\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Dirty Accounts", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_transaction_pool_drift_count{instance=~\"$instance\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Drift Count", - "range": true, - "refId": "B", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_transaction_pool_reinserted_transactions{instance=~\"$instance\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Reinserted Transactions", - "range": true, - "refId": "C", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_transaction_pool_deleted_tracked_finalized_blobs{instance=~\"$instance\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Deleted Tracked Finalized Blobs", - "range": true, - "refId": "D", - "useBackend": false - } - ], - "title": "TxPool Maintenance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Tracks the number of transactions in the various transaction pool sub-pools", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "11.2.0" }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 13 - }, - "id": 92, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "builder", - "expr": "reth_transaction_pool_basefee_pool_transactions{instance=~\"$instance\"}", - "legendFormat": "Base Fee Pool Transactions", - "range": true, - "refId": "A" + "type": "panel", + "id": "piechart", + "name": "Pie chart", + "version": "" }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "builder", - "expr": "reth_transaction_pool_pending_pool_transactions{instance=~\"$instance\"}", - "hide": false, - "legendFormat": "Pending Pool Transactions", - "range": true, - "refId": "B" + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "builder", - "expr": "reth_transaction_pool_queued_pool_transactions{instance=~\"$instance\"}", - "hide": false, - "legendFormat": "Queued Pool Transactions", - "range": true, - "refId": "C" + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "builder", - "expr": "reth_transaction_pool_blob_pool_transactions{instance=~\"$instance\"}", - "legendFormat": "Blob Pool Transactions", - "range": true, - "refId": "D" + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" } - ], - "title": "Subpool Transaction Count", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Tracks the number of transactions per second that are inserted and removed from the transaction pool, as well as the number of invalid transactions per second.\n\nBad transactions are a subset of invalid transactions, these will never be successfully imported. The remaining invalid transactions have a chance of being imported, for example transactions with nonce gaps.\n\n", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": true, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [ - { - "matcher": { - "id": "byFrameRefID", - "options": "B" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 21 - }, - "id": 93, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "rate(reth_transaction_pool_inserted_transactions{instance=~\"$instance\"}[$__rate_interval])", - "fullMetaSearch": false, - "includeNullMetadata": true, - "legendFormat": "Inserted Transactions", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "rate(reth_transaction_pool_removed_transactions{instance=~\"$instance\"}[$__rate_interval])", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "legendFormat": "Removed Transactions", - "range": true, - "refId": "B", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "rate(reth_transaction_pool_invalid_transactions{instance=~\"$instance\"}[$__rate_interval])", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "legendFormat": "Invalid Transactions", - "range": true, - "refId": "C", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "rate(reth_network_bad_imports{instance=\"$instance\"}[$__rate_interval])", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "Bad Transactions", - "range": true, - "refId": "D", - "useBackend": false - } - ], - "title": "Inserted Transactions", - "type": "timeseries" }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Number of transactions about to be imported into the pool.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 21 - }, - "id": 94, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "builder", - "expr": "reth_network_pending_pool_imports{instance=~\"$instance\"}", - "hide": false, - "legendFormat": "Transactions Pending Import", - "range": true, - "refId": "C" + "description": "Metrics for transaction P2P gossip and the local view of mempool data", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 96, + "panels": [], + "repeat": "instance", + "repeatDirection": "h", + "title": "Overview", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 0, + "y": 1 + }, + "id": 22, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": { + "valueSize": 20 + }, + "textMode": "name", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "reth_info{$instance_label=\"$instance\"}", + "instant": true, + "legendFormat": "{{version}}", + "range": false, + "refId": "A" + } + ], + "title": "Version", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 3, + "y": 1 + }, + "id": 192, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": { + "valueSize": 20 + }, + "textMode": "name", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "reth_info{$instance_label=\"$instance\"}", + "instant": true, + "legendFormat": "{{build_timestamp}}", + "range": false, + "refId": "A" + } + ], + "title": "Build Timestamp", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 9, + "y": 1 + }, + "id": 193, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": { + "valueSize": 20 + }, + "textMode": "name", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "reth_info{$instance_label=\"$instance\"}", + "instant": true, + "legendFormat": "{{git_sha}}", + "range": false, + "refId": "A" + } + ], + "title": "Git SHA", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 12, + "y": 1 + }, + "id": 195, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": { + "valueSize": 20 + }, + "textMode": "name", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "reth_info{$instance_label=\"$instance\"}", + "instant": true, + "legendFormat": "{{build_profile}}", + "range": false, + "refId": "A" + } + ], + "title": "Build Profile", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 14, + "y": 1 + }, + "id": 196, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": { + "valueSize": 20 + }, + "textMode": "name", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "reth_info{$instance_label=\"$instance\"}", + "instant": true, + "legendFormat": "{{target_triple}}", + "range": false, + "refId": "A" + } + ], + "title": "Target Triple", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 19, + "y": 1 + }, + "id": 197, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": { + "valueSize": 20 + }, + "textMode": "name", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "reth_info{$instance_label=\"$instance\"}", + "instant": true, + "legendFormat": "{{cargo_features}}", + "range": false, + "refId": "A" + } + ], + "title": "Cargo Features", + "transparent": true, + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 89, + "panels": [], + "repeat": "instance", + "repeatDirection": "h", + "title": "Transaction Pool", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Tracks the entries, byte size, failed inserts and file deletes of the blob store", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 5 + }, + "id": 115, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_transaction_pool_blobstore_entries{$instance_label=\"$instance\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Entries", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_transaction_pool_blobstore_byte_size{$instance_label=\"$instance\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Bytesize", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_transaction_pool_blobstore_failed_inserts{$instance_label=\"$instance\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Failed Inserts", + "range": true, + "refId": "C", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_transaction_pool_blobstore_failed_deletes{$instance_label=\"$instance\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Failed Deletes", + "range": true, + "refId": "D", + "useBackend": false + } + ], + "title": "Blob store", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Tracks a heuristic of the memory footprint of the various transaction pool sub-pools", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 5 + }, + "id": 210, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "reth_transaction_pool_basefee_pool_size_bytes{$instance_label=\"$instance\"}", + "legendFormat": "Base Fee Pool Size", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "reth_transaction_pool_pending_pool_size_bytes{$instance_label=\"$instance\"}", + "hide": false, + "legendFormat": "Pending Pool Size", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "reth_transaction_pool_queued_pool_size_bytes{$instance_label=\"$instance\"}", + "hide": false, + "legendFormat": "Queued Pool Size", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "reth_transaction_pool_blob_pool_size_bytes{$instance_label=\"$instance\"}", + "legendFormat": "Blob Pool Size", + "range": true, + "refId": "D" + } + ], + "title": "Subpool Sizes in Bytes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Transaction pool maintenance metrics", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 13 + }, + "id": 91, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_transaction_pool_dirty_accounts{$instance_label=\"$instance\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Dirty Accounts", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_transaction_pool_drift_count{$instance_label=\"$instance\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Drift Count", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_transaction_pool_reinserted_transactions{$instance_label=\"$instance\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Reinserted Transactions", + "range": true, + "refId": "C", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_transaction_pool_deleted_tracked_finalized_blobs{$instance_label=\"$instance\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Deleted Tracked Finalized Blobs", + "range": true, + "refId": "D", + "useBackend": false + } + ], + "title": "TxPool Maintenance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Tracks the number of transactions in the various transaction pool sub-pools", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 13 + }, + "id": 92, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "reth_transaction_pool_basefee_pool_transactions{$instance_label=\"$instance\"}", + "legendFormat": "Base Fee Pool Transactions", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "reth_transaction_pool_pending_pool_transactions{$instance_label=\"$instance\"}", + "hide": false, + "legendFormat": "Pending Pool Transactions", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "reth_transaction_pool_queued_pool_transactions{$instance_label=\"$instance\"}", + "hide": false, + "legendFormat": "Queued Pool Transactions", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "reth_transaction_pool_blob_pool_transactions{$instance_label=\"$instance\"}", + "legendFormat": "Blob Pool Transactions", + "range": true, + "refId": "D" + } + ], + "title": "Subpool Transaction Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Tracks the number of transactions per second that are inserted and removed from the transaction pool, as well as the number of invalid transactions per second.\n\nBad transactions are a subset of invalid transactions, these will never be successfully imported. The remaining invalid transactions have a chance of being imported, for example transactions with nonce gaps.\n\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": true, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 93, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(reth_transaction_pool_inserted_transactions{$instance_label=\"$instance\"}[$__rate_interval])", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "Inserted Transactions", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(reth_transaction_pool_removed_transactions{$instance_label=\"$instance\"}[$__rate_interval])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "Removed Transactions", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(reth_transaction_pool_invalid_transactions{$instance_label=\"$instance\"}[$__rate_interval])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "Invalid Transactions", + "range": true, + "refId": "C", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(reth_network_bad_imports{$instance_label=\"$instance\"}[$__rate_interval])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "Bad Transactions", + "range": true, + "refId": "D", + "useBackend": false + } + ], + "title": "Inserted Transactions", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of transactions about to be imported into the pool.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 94, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "reth_network_pending_pool_imports{$instance_label=\"$instance\"}", + "hide": false, + "legendFormat": "Transactions Pending Import", + "range": true, + "refId": "C" + } + ], + "title": "Pending Pool Imports", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Tracks the number of incoming transaction messages in the channel from the network to the transaction pool.\n\nMempool messages sent over this channel are `GetPooledTransactions` requests, `NewPooledTransactionHashes` announcements (gossip), and `Transactions` (gossip)\n\nTx - `NetworkManager`\n\\nRx - `TransactionsManager`", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": true, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "mps" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "C" + }, + "properties": [ + { + "id": "unit", + "value": "events" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 29 + }, + "id": 95, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "rate(reth_network_pool_transactions_messages_sent_total{$instance_label=\"$instance\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "Tx", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "rate(reth_network_pool_transactions_messages_received_total{$instance_label=\"$instance\"}[$__rate_interval])", + "hide": false, + "legendFormat": "Rx", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "reth_network_pool_transactions_messages_sent_total{$instance_label=\"$instance\"} - reth_network_pool_transactions_messages_received_total{$instance_label=\"$instance\"}", + "hide": false, + "legendFormat": "Messages in Channel", + "range": true, + "refId": "C" + } + ], + "title": "Incoming Gossip and Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Measures the message send rate (MPS) for queued outgoing messages. Outgoing messages are added to the queue before being sent to other peers, and this metric helps track the rate of message dispatch.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "mps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 29 + }, + "id": 219, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(reth_network_queued_outgoing_messages{$instance_label=\"$instance\"}[$__rate_interval])", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Queued Messages per Second", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Queued Outgoing Messages", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "All Transactions metrics", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 37 + }, + "id": 116, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_transaction_pool_all_transactions_by_hash{$instance_label=\"$instance\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "All transactions by hash", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_transaction_pool_all_transactions_by_id{$instance_label=\"$instance\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "All transactions by id", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_transaction_pool_all_transactions_by_all_senders{$instance_label=\"$instance\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "All transactions by all senders", + "range": true, + "refId": "C", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_transaction_pool_blob_transactions_nonce_gaps{$instance_label=\"$instance\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Blob transactions nonce gaps", + "range": true, + "refId": "D", + "useBackend": false + } + ], + "title": "All Transactions metrics", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Egress RLPx capability traffic (default only `eth` capability)\n\nDropped - session channels are bounded, if there's no capacity, the message will be dropped.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "mps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 37 + }, + "id": 217, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(reth_network_total_outgoing_peer_messages_dropped{$instance_label=\"$instance\"}[$__rate_interval])", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Dropped", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Outgoing Capability Messages", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total number of times a transaction is sent/announced that is already in the local pool.\n\nThis reflects the redundancy in the mempool.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 45 + }, + "id": 213, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(reth_network_occurrences_hashes_already_in_pool{$instance_label=\"$instance\"}[$__rate_interval])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "Freq Announced Transactions Already in Pool", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(reth_network_occurrences_transactions_already_in_pool{$instance_label=\"$instance\"}[$__rate_interval])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "Freq Received Transactions Already in Pool ", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Frequency of Transactions Already in Pool", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Currently active outgoing GetPooledTransactions requests.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 45 + }, + "id": 104, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "reth_network_inflight_transaction_requests{$instance_label=\"$instance\"}", + "hide": false, + "legendFormat": "Inflight Transaction Requests", + "range": true, + "refId": "C" + } + ], + "title": "Inflight Transaction Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Duration of one call to poll `TransactionsManager` future, and its nested function calls.\n\nNetwork Events - stream peer session updates from `NetworkManager`;\nTransaction Events - stream txns gossip from `NetworkManager`;\nPending Transactions - stream hashes of txns successfully inserted into pending set in `TransactionPool`;\nPending Pool Imports - flush txns to pool from `TransactionsManager`;\nFetch Events - stream fetch txn events (success case wraps a tx) from `TransactionFetcher`;\nFetch Pending Hashes - search for hashes announced by an idle peer in cache for hashes pending fetch;\n(Transactions Commands - stream commands from testnet to fetch/serve/propagate txns)\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 53 + }, + "id": 200, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_network_acc_duration_poll_network_events{$instance_label=\"$instance\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Network Events", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_network_acc_duration_poll_transaction_events{$instance_label=\"$instance\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Transaction Events", + "range": true, + "refId": "C", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "reth_network_acc_duration_poll_imported_transactions{$instance_label=\"$instance\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Pending Transactions", + "range": true, + "refId": "D", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_network_acc_duration_poll_pending_pool_imports{$instance_label=\"$instance\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Pending Pool Imports", + "range": true, + "refId": "E", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_network_acc_duration_poll_fetch_events{$instance_label=\"$instance\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Fetch Events", + "range": true, + "refId": "F", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_network_acc_duration_poll_commands{$instance_label=\"$instance\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Commands", + "range": true, + "refId": "G", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_network_acc_duration_fetch_pending_hashes{$instance_label=\"$instance\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Fetch Pending Hashes", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_network_duration_poll_tx_manager{$instance_label=\"$instance\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Total Transactions Manager Future", + "range": true, + "refId": "H", + "useBackend": false + } + ], + "title": "Transactions Manager Poll Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 53 + }, + "id": 199, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "reth_network_hashes_pending_fetch{$instance_label=\"$instance\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Hashes in Pending Fetch Cache", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "reth_network_inflight_transaction_requests{$instance_label=\"$instance\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Hashes in Inflight Requests", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(reth_network_hashes_inflight_transaction_requests{$instance_label=\"$instance\"}) + sum(reth_network_hashes_pending_fetch{$instance_label=\"$instance\"})", + "hide": false, + "instant": false, + "legendFormat": "Total Hashes in Transaction Fetcher", + "range": true, + "refId": "C" + } + ], + "title": "Transaction Fetcher Hashes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Durations of one call to poll `NetworkManager` future, and its nested function calls.\n\nNetwork Handle Message - stream network handle messages from `TransactionsManager`;\nSwarm Events - stream transaction gossip from `Swarm`", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 61 + }, + "id": 209, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_network_acc_duration_poll_network_handle{$instance_label=\"$instance\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Network Handle Messages", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_network_acc_duration_poll_swarm{$instance_label=\"$instance\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Swarm Events", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_network_duration_poll_network_manager{$instance_label=\"$instance\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Total Network Manager Future", + "range": true, + "refId": "C", + "useBackend": false + } + ], + "title": "Network Manager Poll Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Frequency of a peer sending a transaction that has already been marked as seen by that peer. This could for example be the case if a transaction is sent/announced to the peer at the same time that the peer sends/announces the same transaction to us.\n\nThis reflects the latency in the mempool.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 61 + }, + "id": 208, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(reth_network_occurrences_hash_already_seen_by_peer{$instance_label=\"$instance\"}[$__rate_interval])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "Freq Announced Transactions Already Seen by Peer", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(reth_network_occurrences_of_transaction_already_seen_by_peer{$instance_label=\"$instance\"}[$__rate_interval])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "Freq Received Transactions Already Seen by Peer", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Frequency of Transactions Already Marked as Seen by Peer", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of all transactions of all sub-pools by type", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 69 + }, + "id": 218, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "reth_transaction_pool_total_legacy_transactions{$instance_label=\"$instance\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "Legacy", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "reth_transaction_pool_total_eip2930_transactions{$instance_label=\"$instance\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "EIP-2930", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "reth_transaction_pool_total_eip1559_transactions{$instance_label=\"$instance\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "EIP-1559", + "range": true, + "refId": "C", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "reth_transaction_pool_total_eip4844_transactions{$instance_label=\"$instance\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "EIP-4844", + "range": true, + "refId": "D", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "reth_transaction_pool_total_eip7702_transactions{$instance_label=\"$instance\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "EIP-7702", + "range": true, + "refId": "E", + "useBackend": false + } + ], + "title": "Transactions by Type in Pool", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Duration of one call to `TransactionFetcher::on_fetch_pending_hashes`.\n\nFind Peer - find an idle fallback peer for a hash pending fetch.\n\nFill Request - fill `GetPooledTransactions` request, for the found peer, with more hashes from cache of hashes pending fetch. ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 69 + }, + "id": 215, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_network_duration_find_idle_fallback_peer_for_any_pending_hash{$instance_label=\"$instance\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Find Idle Peer", + "range": true, + "refId": "C", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_network_duration_fill_request_from_hashes_pending_fetch{$instance_label=\"$instance\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Fill Request", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Fetch Hashes Pending Fetch Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Frequency of transaction types seen in announcements", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 77 + }, + "id": 214, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(reth_network_transaction_fetcher_legacy_sum{$instance_label=\"$instance\"}[$__rate_interval])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "Legacy", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(reth_network_transaction_fetcher_eip2930_sum{$instance_label=\"$instance\"}[$__rate_interval])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "Eip2930", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(reth_network_transaction_fetcher_eip1559_sum{$instance_label=\"$instance\"}[$__rate_interval])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "Eip1559", + "range": true, + "refId": "C", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(reth_network_transaction_fetcher_eip4844_sum{$instance_label=\"$instance\"}[$__rate_interval])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "Eip4844", + "range": true, + "refId": "D", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(reth_network_transaction_fetcher_eip7702_sum{$instance_label=\"$instance\"}[$__rate_interval])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "Eip7702", + "range": true, + "refId": "E", + "useBackend": false + } + ], + "title": "Announced Transactions by Type", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of transactions evicted in each pool", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 77 + }, + "id": 220, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "disableTextWrap": false, + "editorMode": "code", + "expr": "reth_transaction_pool_pending_transactions_evicted{$instance_label=\"$instance\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "PendingPool", + "range": true, + "refId": "A", + "useBackend": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "reth_transaction_pool_basefee_transactions_evicted{$instance_label=\"$instance\"}", + "hide": false, + "instant": false, + "legendFormat": "BasefeePool", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "reth_transaction_pool_blob_transactions_evicted{$instance_label=\"$instance\"}", + "hide": false, + "instant": false, + "legendFormat": "BlobPool", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "reth_transaction_pool_queued_transactions_evicted{$instance_label=\"$instance\"}", + "hide": false, + "instant": false, + "legendFormat": "QueuedPool", + "range": true, + "refId": "D" + } + ], + "title": "Evicted Transactions", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 85 + }, + "id": 6, + "panels": [], + "repeat": "instance", + "title": "Networking", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The number of tracked peers in the discovery modules (dnsdisc and discv4)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 86 + }, + "id": 18, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "reth_network_tracked_peers{$instance_label=\"$instance\"}", + "legendFormat": "Tracked Peers", + "range": true, + "refId": "A" + } + ], + "title": "Discovery: Tracked peers", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The number of incoming and outgoing connections, as well as the number of peers we are currently connected to. Outgoing and incoming connections also count peers we are trying to connect to.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 86 + }, + "id": 16, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "reth_network_pending_outgoing_connections{$instance_label=\"$instance\"}", + "legendFormat": "Pending Outgoing Connections", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "reth_network_outgoing_connections{$instance_label=\"$instance\"}", + "legendFormat": "Outgoing Connections", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "reth_network_total_pending_connections{$instance_label=\"$instance\"}", + "legendFormat": "Total Pending Connections", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "reth_network_incoming_connections{$instance_label=\"$instance\"}", + "hide": false, + "legendFormat": "Incoming Connections", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "reth_network_connected_peers{$instance_label=\"$instance\"}", + "hide": false, + "legendFormat": "Connected Peers", + "range": true, + "refId": "E" + } + ], + "title": "Connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Internal errors in the P2P module. These are expected to happen from time to time. High error rates should not cause alarm if the node is peering otherwise.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 86 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": ["value"] + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "rate(reth_p2pstream_disconnected_errors{$instance_label=\"$instance\"}[$__rate_interval])", + "legendFormat": "P2P Stream Disconnected", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "rate(reth_network_pending_session_failures{$instance_label=\"$instance\"}[$__rate_interval])", + "hide": false, + "legendFormat": "Failed Pending Sessions", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "rate(reth_network_invalid_messages_received_total{$instance_label=\"$instance\"}[$__rate_interval])", + "hide": false, + "legendFormat": "Invalid Messages", + "range": true, + "refId": "C" + } + ], + "title": "P2P Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 94 + }, + "id": 54, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "pieType": "pie", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "reth_network_useless_peer{$instance_label=\"$instance\"}", + "legendFormat": "UselessPeer", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "reth_network_subprotocol_specific{$instance_label=\"$instance\"}", + "hide": false, + "legendFormat": "SubprotocolSpecific", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "reth_network_already_connected{$instance_label=\"$instance\"}", + "hide": false, + "legendFormat": "AlreadyConnected", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "reth_network_client_quitting{$instance_label=\"$instance\"}", + "hide": false, + "legendFormat": "ClientQuitting", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "reth_network_unexpected_identity{$instance_label=\"$instance\"}", + "hide": false, + "legendFormat": "UnexpectedHandshakeIdentity", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "reth_network_disconnect_requested{$instance_label=\"$instance\"}", + "hide": false, + "legendFormat": "DisconnectRequested", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "reth_network_null_node_identity{$instance_label=\"$instance\"}", + "hide": false, + "legendFormat": "NullNodeIdentity", + "range": true, + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "reth_network_tcp_subsystem_error{$instance_label=\"$instance\"}", + "hide": false, + "legendFormat": "TCPSubsystemError", + "range": true, + "refId": "H" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "reth_network_incompatible{$instance_label=\"$instance\"}", + "hide": false, + "legendFormat": "IncompatibleP2PVersion", + "range": true, + "refId": "I" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "reth_network_protocol_breach{$instance_label=\"$instance\"}", + "hide": false, + "legendFormat": "ProtocolBreach", + "range": true, + "refId": "J" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "reth_network_too_many_peers{$instance_label=\"$instance\"}", + "hide": false, + "legendFormat": "TooManyPeers", + "range": true, + "refId": "K" + } + ], + "title": "Peer Disconnect Reasons", + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of successful outgoing dial attempts.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 14, + "x": 8, + "y": 94 + }, + "id": 103, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "reth_network_total_dial_successes{$instance_label=\"$instance\"}", + "legendFormat": "Total Dial Successes", + "range": true, + "refId": "A" + } + ], + "title": "Total Dial Success", + "type": "timeseries" } - ], - "title": "Pending Pool Imports", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Tracks the number of incoming transaction messages in the channel from the network to the transaction pool.\n\nMempool messages sent over this channel are `GetPooledTransactions` requests, `NewPooledTransactionHashes` announcements (gossip), and `Transactions` (gossip)\n\nTx - `NetworkManager`\n\\nRx - `TransactionsManager`", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": true, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" + ], + "refresh": "30s", + "revision": 1, + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "hide": 2, + "label": "Instance Label", + "name": "instance_label", + "query": "job", + "skipUrlSync": false, + "type": "constant" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(reth_info,$instance_label)", + "hide": 0, + "includeAll": false, + "label": "Instance", + "multi": false, + "name": "instance", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(reth_info,$instance_label)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": {}, + "includeAll": false, + "label": "Datasource", + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "mps" - }, - "overrides": [ - { - "matcher": { - "id": "byFrameRefID", - "options": "B" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - }, - { - "matcher": { - "id": "byFrameRefID", - "options": "C" - }, - "properties": [ - { - "id": "unit", - "value": "events" - } - ] - } ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 29 - }, - "id": 95, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "builder", - "expr": "rate(reth_network_pool_transactions_messages_sent_total{instance=~\"$instance\"}[$__rate_interval])", - "hide": false, - "instant": false, - "legendFormat": "Tx", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "builder", - "expr": "rate(reth_network_pool_transactions_messages_received_total{instance=~\"$instance\"}[$__rate_interval])", - "hide": false, - "legendFormat": "Rx", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "reth_network_pool_transactions_messages_sent_total{instance=~\"$instance\"} - reth_network_pool_transactions_messages_received_total{instance=~\"$instance\"}", - "hide": false, - "legendFormat": "Messages in Channel", - "range": true, - "refId": "C" - } - ], - "title": "Incoming Gossip and Requests", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Measures the message send rate (MPS) for queued outgoing messages. Outgoing messages are added to the queue before being sent to other peers, and this metric helps track the rate of message dispatch.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "mps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 29 - }, - "id": 219, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "rate(reth_network_queued_outgoing_messages{instance=\"$instance\"}[$__rate_interval])", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Queued Messages per Second", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Queued Outgoing Messages", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "All Transactions metrics", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 37 - }, - "id": 116, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_transaction_pool_all_transactions_by_hash{instance=~\"$instance\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "All transactions by hash", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_transaction_pool_all_transactions_by_id{instance=~\"$instance\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "All transactions by id", - "range": true, - "refId": "B", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_transaction_pool_all_transactions_by_all_senders{instance=~\"$instance\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "All transactions by all senders", - "range": true, - "refId": "C", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_transaction_pool_blob_transactions_nonce_gaps{instance=~\"$instance\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Blob transactions nonce gaps", - "range": true, - "refId": "D", - "useBackend": false - } - ], - "title": "All Transactions metrics", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Egress RLPx capability traffic (default only `eth` capability)\n\nDropped - session channels are bounded, if there's no capacity, the message will be dropped.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "mps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 37 - }, - "id": 217, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "rate(reth_network_total_outgoing_peer_messages_dropped{instance=\"$instance\"}[$__rate_interval])", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Dropped", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Outgoing Capability Messages", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Total number of times a transaction is sent/announced that is already in the local pool.\n\nThis reflects the redundancy in the mempool.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "cps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 45 - }, - "id": 213, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "rate(reth_network_occurrences_hashes_already_in_pool{instance=\"$instance\"}[$__rate_interval])", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "Freq Announced Transactions Already in Pool", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "rate(reth_network_occurrences_transactions_already_in_pool{instance=\"$instance\"}[$__rate_interval])", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "Freq Received Transactions Already in Pool ", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "Frequency of Transactions Already in Pool", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Currently active outgoing GetPooledTransactions requests.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 45 - }, - "id": 104, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "builder", - "expr": "reth_network_inflight_transaction_requests{instance=~\"$instance\"}", - "hide": false, - "legendFormat": "Inflight Transaction Requests", - "range": true, - "refId": "C" - } - ], - "title": "Inflight Transaction Requests", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Duration of one call to poll `TransactionsManager` future, and its nested function calls.\n\nNetwork Events - stream peer session updates from `NetworkManager`;\nTransaction Events - stream txns gossip from `NetworkManager`;\nPending Transactions - stream hashes of txns successfully inserted into pending set in `TransactionPool`;\nPending Pool Imports - flush txns to pool from `TransactionsManager`;\nFetch Events - stream fetch txn events (success case wraps a tx) from `TransactionFetcher`;\nFetch Pending Hashes - search for hashes announced by an idle peer in cache for hashes pending fetch;\n(Transactions Commands - stream commands from testnet to fetch/serve/propagate txns)\n", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 53 - }, - "id": 200, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_network_acc_duration_poll_network_events{instance=\"$instance\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Network Events", - "range": true, - "refId": "B", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_network_acc_duration_poll_transaction_events{instance=\"$instance\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Transaction Events", - "range": true, - "refId": "C", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "reth_network_acc_duration_poll_imported_transactions{instance=\"$instance\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Pending Transactions", - "range": true, - "refId": "D", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_network_acc_duration_poll_pending_pool_imports{instance=\"$instance\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Pending Pool Imports", - "range": true, - "refId": "E", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_network_acc_duration_poll_fetch_events{instance=\"$instance\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Fetch Events", - "range": true, - "refId": "F", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_network_acc_duration_poll_commands{instance=\"$instance\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Commands", - "range": true, - "refId": "G", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_network_acc_duration_fetch_pending_hashes{instance=\"$instance\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Fetch Pending Hashes", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_network_duration_poll_tx_manager{instance=\"$instance\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Total Transactions Manager Future", - "range": true, - "refId": "H", - "useBackend": false - } - ], - "title": "Transactions Manager Poll Duration", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 53 - }, - "id": 199, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "reth_network_hashes_pending_fetch{instance=~\"$instance\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Hashes in Pending Fetch Cache", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "reth_network_hashes_inflight_transaction_requests{instance=~\"$instance\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Hashes in Inflight Requests", - "range": true, - "refId": "B", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "sum(reth_network_hashes_inflight_transaction_requests{instance=~\"$instance\"}) + sum(reth_network_hashes_pending_fetch{instance=~\"$instance\"})", - "hide": false, - "instant": false, - "legendFormat": "Total Hashes in Transaction Fetcher", - "range": true, - "refId": "C" - } - ], - "title": "Transaction Fetcher Hashes", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Durations of one call to poll `NetworkManager` future, and its nested function calls.\n\nNetwork Handle Message - stream network handle messages from `TransactionsManager`;\nSwarm Events - stream transaction gossip from `Swarm`", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 61 - }, - "id": 209, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_network_acc_duration_poll_network_handle{instance=\"$instance\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Network Handle Messages", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_network_acc_duration_poll_swarm{instance=\"$instance\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Swarm Events", - "range": true, - "refId": "B", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_network_duration_poll_network_manager{instance=\"$instance\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Total Network Manager Future", - "range": true, - "refId": "C", - "useBackend": false - } - ], - "title": "Network Manager Poll Duration", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Frequency of a peer sending a transaction that has already been marked as seen by that peer. This could for example be the case if a transaction is sent/announced to the peer at the same time that the peer sends/announces the same transaction to us.\n\nThis reflects the latency in the mempool.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "cps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 61 - }, - "id": 208, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "rate(reth_network_occurrences_hash_already_seen_by_peer{instance=\"$instance\"}[$__rate_interval])", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "Freq Announced Transactions Already Seen by Peer", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "rate(reth_network_occurrences_of_transaction_already_seen_by_peer{instance=\"$instance\"}[$__rate_interval])", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "Freq Received Transactions Already Seen by Peer", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "Frequency of Transactions Already Marked as Seen by Peer", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Number of all transactions of all sub-pools by type", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 69 - }, - "id": 218, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "reth_transaction_pool_total_legacy_transactions{instance=\"$instance\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "Legacy", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "reth_transaction_pool_total_eip2930_transactions{instance=\"$instance\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "EIP-2930", - "range": true, - "refId": "B", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "reth_transaction_pool_total_eip1559_transactions{instance=\"$instance\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "EIP-1559", - "range": true, - "refId": "C", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "reth_transaction_pool_total_eip4844_transactions{instance=\"$instance\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "EIP-4844", - "range": true, - "refId": "D", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "reth_transaction_pool_total_eip7702_transactions{instance=\"$instance\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "EIP-7702", - "range": true, - "refId": "E", - "useBackend": false - } - ], - "title": "Transactions by Type in Pool", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Duration of one call to `TransactionFetcher::on_fetch_pending_hashes`.\n\nFind Peer - find an idle fallback peer for a hash pending fetch.\n\nFill Request - fill `GetPooledTransactions` request, for the found peer, with more hashes from cache of hashes pending fetch. ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 69 - }, - "id": 215, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_network_duration_find_idle_fallback_peer_for_any_pending_hash{instance=\"$instance\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Find Idle Peer", - "range": true, - "refId": "C", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_network_duration_fill_request_from_hashes_pending_fetch{instance=\"$instance\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Fill Request", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "Fetch Hashes Pending Fetch Duration", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Frequency of transaction types seen in announcements", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "cps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 77 - }, - "id": 214, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "rate(reth_network_transaction_fetcher_legacy_sum{instance=\"$instance\"}[$__rate_interval])", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "Legacy", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "rate(reth_network_transaction_fetcher_eip2930_sum{instance=\"$instance\"}[$__rate_interval])", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "Eip2930", - "range": true, - "refId": "B", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "rate(reth_network_transaction_fetcher_eip1559_sum{instance=\"$instance\"}[$__rate_interval])", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "Eip1559", - "range": true, - "refId": "C", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "rate(reth_network_transaction_fetcher_eip4844_sum{instance=\"$instance\"}[$__rate_interval])", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": false, - "instant": false, - "legendFormat": "Eip4844", - "range": true, - "refId": "D", - "useBackend": false - } - ], - "title": "Announced Transactions by Type", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Number of transactions evicted in each pool", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 77 - }, - "id": 220, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "disableTextWrap": false, - "editorMode": "code", - "expr": "reth_transaction_pool_pending_transactions_evicted{instance=\"$instance\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "legendFormat": "PendingPool", - "range": true, - "refId": "A", - "useBackend": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - } - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "reth_transaction_pool_basefee_transactions_evicted{instance=\"$instance\"}", - "hide": false, - "instant": false, - "legendFormat": "BasefeePool", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "reth_transaction_pool_blob_transactions_evicted{instance=\"$instance\"}", - "hide": false, - "instant": false, - "legendFormat": "BlobPool", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "reth_transaction_pool_queued_transactions_evicted{instance=\"$instance\"}", - "hide": false, - "instant": false, - "legendFormat": "QueuedPool", - "range": true, - "refId": "D" - } - ], - "title": "Evicted Transactions", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 85 - }, - "id": 6, - "panels": [], - "repeat": "instance", - "title": "Networking", - "type": "row" }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "The number of tracked peers in the discovery modules (dnsdisc and discv4)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 86 - }, - "id": 18, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "builder", - "expr": "reth_network_tracked_peers{instance=~\"$instance\"}", - "legendFormat": "Tracked Peers", - "range": true, - "refId": "A" - } - ], - "title": "Discovery: Tracked peers", - "type": "timeseries" + "time": { + "from": "now-1h", + "to": "now" }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "The number of incoming and outgoing connections, as well as the number of peers we are currently connected to. Outgoing and incoming connections also count peers we are trying to connect to.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 86 - }, - "id": 16, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "builder", - "expr": "reth_network_pending_outgoing_connections{instance=~\"$instance\"}", - "legendFormat": "Pending Outgoing Connections", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "builder", - "expr": "reth_network_outgoing_connections{instance=~\"$instance\"}", - "legendFormat": "Outgoing Connections", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "builder", - "expr": "reth_network_total_pending_connections{instance=~\"$instance\"}", - "legendFormat": "Total Pending Connections", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "builder", - "expr": "reth_network_incoming_connections{instance=~\"$instance\"}", - "hide": false, - "legendFormat": "Incoming Connections", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "builder", - "expr": "reth_network_connected_peers{instance=~\"$instance\"}", - "hide": false, - "legendFormat": "Connected Peers", - "range": true, - "refId": "E" - } - ], - "title": "Connections", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Internal errors in the P2P module. These are expected to happen from time to time. High error rates should not cause alarm if the node is peering otherwise.", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "red", - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "cps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 86 - }, - "id": 8, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "values": [ - "value" - ] - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "builder", - "expr": "rate(reth_p2pstream_disconnected_errors{instance=~\"$instance\"}[$__rate_interval])", - "legendFormat": "P2P Stream Disconnected", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "builder", - "expr": "rate(reth_network_pending_session_failures{instance=~\"$instance\"}[$__rate_interval])", - "hide": false, - "legendFormat": "Failed Pending Sessions", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "builder", - "expr": "rate(reth_network_invalid_messages_received_total{instance=~\"$instance\"}[$__rate_interval])", - "hide": false, - "legendFormat": "Invalid Messages", - "range": true, - "refId": "C" - } - ], - "title": "P2P Errors", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - } - }, - "mappings": [] - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 94 - }, - "id": 54, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "pieType": "pie", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "builder", - "expr": "reth_network_useless_peer{instance=~\"$instance\"}", - "legendFormat": "UselessPeer", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "builder", - "expr": "reth_network_subprotocol_specific{instance=~\"$instance\"}", - "hide": false, - "legendFormat": "SubprotocolSpecific", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "builder", - "expr": "reth_network_already_connected{instance=~\"$instance\"}", - "hide": false, - "legendFormat": "AlreadyConnected", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "builder", - "expr": "reth_network_client_quitting{instance=~\"$instance\"}", - "hide": false, - "legendFormat": "ClientQuitting", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "builder", - "expr": "reth_network_unexpected_identity{instance=~\"$instance\"}", - "hide": false, - "legendFormat": "UnexpectedHandshakeIdentity", - "range": true, - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "builder", - "expr": "reth_network_disconnect_requested{instance=~\"$instance\"}", - "hide": false, - "legendFormat": "DisconnectRequested", - "range": true, - "refId": "F" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "builder", - "expr": "reth_network_null_node_identity{instance=~\"$instance\"}", - "hide": false, - "legendFormat": "NullNodeIdentity", - "range": true, - "refId": "G" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "builder", - "expr": "reth_network_tcp_subsystem_error{instance=~\"$instance\"}", - "hide": false, - "legendFormat": "TCPSubsystemError", - "range": true, - "refId": "H" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "builder", - "expr": "reth_network_incompatible{instance=~\"$instance\"}", - "hide": false, - "legendFormat": "IncompatibleP2PVersion", - "range": true, - "refId": "I" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "builder", - "expr": "reth_network_protocol_breach{instance=~\"$instance\"}", - "hide": false, - "legendFormat": "ProtocolBreach", - "range": true, - "refId": "J" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "builder", - "expr": "reth_network_too_many_peers{instance=~\"$instance\"}", - "hide": false, - "legendFormat": "TooManyPeers", - "range": true, - "refId": "K" - } - ], - "title": "Peer Disconnect Reasons", - "type": "piechart" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "Number of successful outgoing dial attempts.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 14, - "x": 8, - "y": 94 - }, - "id": 103, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "builder", - "expr": "reth_network_total_dial_successes{instance=~\"$instance\"}", - "legendFormat": "Total Dial Successes", - "range": true, - "refId": "A" - } - ], - "title": "Total Dial Success", - "type": "timeseries" - } - ], - "refresh": "30s", - "revision": 1, - "schemaVersion": 39, - "tags": [], - "templating": { - "list": [ - { - "current": {}, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "definition": "query_result(reth_info)", - "hide": 0, - "includeAll": false, - "multi": false, - "name": "instance", - "options": [], - "query": { - "query": "query_result(reth_info)", - "refId": "PrometheusVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "/.*instance=\\\"([^\\\"]*).*/", - "skipUrlSync": false, - "sort": 0, - "type": "query" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Reth - Transaction Pool", - "uid": "bee34f59-c79c-4669-a000-198057b3703d", - "version": 2, - "weekStart": "" -} \ No newline at end of file + "timepicker": {}, + "timezone": "", + "title": "Reth - Transaction Pool", + "uid": "bee34f59-c79c-4669-a000-198057b3703d", + "version": 2, + "weekStart": "" +} diff --git a/etc/grafana/dashboards/reth-performance.json b/etc/grafana/dashboards/reth-performance.json deleted file mode 100644 index 02d890dceef..00000000000 --- a/etc/grafana/dashboards/reth-performance.json +++ /dev/null @@ -1,346 +0,0 @@ -{ - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "label": "Prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } - ], - "__elements": {}, - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "11.1.0" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "1.0.0" - }, - { - "type": "panel", - "id": "timeseries", - "name": "Time series", - "version": "" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": null, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 3, - "panels": [], - "title": "Block Validation", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "This tracks the proportion of various tasks that take up time during block validation", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 25, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "percent" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 1 - }, - "id": 1, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_sync_block_validation_state_root_duration{instance=\"$instance\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "State Root Duration", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_sync_execution_execution_duration{instance=\"$instance\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Execution Duration", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "Block Validation Overview", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "This tracks the total block validation latency, as well as the latency for validation sub-tasks ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 25, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 1 - }, - "id": 2, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_sync_block_validation_state_root_duration{instance=\"$instance\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "State Root Duration", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_sync_execution_execution_duration{instance=\"$instance\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Execution Duration", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "Block Validation Latency", - "type": "timeseries" - } - ], - "refresh": "30s", - "schemaVersion": 39, - "tags": [], - "templating": { - "list": [ - { - "current": {}, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "definition": "query_result(reth_info)", - "hide": 0, - "includeAll": false, - "label": "instance", - "multi": false, - "name": "instance", - "options": [], - "query": { - "qryType": 3, - "query": "query_result(reth_info)", - "refId": "PrometheusVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "/.*instance=\\\"([^\\\"]*).*/", - "skipUrlSync": false, - "sort": 0, - "type": "query" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": {}, - "timezone": "browser", - "title": "Reth - Performance", - "uid": "bdywb3xjphfy8a", - "version": 2, - "weekStart": "" -} diff --git a/etc/lighthouse.yml b/etc/lighthouse.yml index ccd520ffa26..fc76b1fc776 100644 --- a/etc/lighthouse.yml +++ b/etc/lighthouse.yml @@ -1,5 +1,5 @@ -version: "3.9" name: reth + services: lighthouse: restart: unless-stopped @@ -13,11 +13,17 @@ services: - "9000:9000/tcp" # p2p - "9000:9000/udp" # p2p volumes: - - lighthousedata:/root/.lighthouse + - lighthouse_data:/root/.lighthouse - ./jwttoken:/root/jwt:ro # For Sepolia: # - Replace `--network mainnet` with `--network sepolia` - # - Use different checkpoint sync URL: `--checkpoint-sync-url https://sepolia.checkpoint-sync.ethpandaops.io` + # - Use different checkpoint sync URL: `--checkpoint-sync-url https://checkpoint-sync.sepolia.ethpandaops.io` + # For Holesky: + # - Replace `--network mainnet` with `--network holesky` + # - Use different checkpoint sync URL: `--checkpoint-sync-url https://checkpoint-sync.holesky.ethpandaops.io` + # For Hoodi: + # - Replace `--network mainnet` with `--network hoodi` + # - Use different checkpoint sync URL: `--checkpoint-sync-url https://checkpoint-sync.hoodi.ethpandaops.io` command: > lighthouse bn --network mainnet @@ -43,5 +49,5 @@ services: - --metrics-port=9091 volumes: - lighthousedata: + lighthouse_data: driver: local diff --git a/examples/beacon-api-sidecar-fetcher/Cargo.toml b/examples/beacon-api-sidecar-fetcher/Cargo.toml index cfa7d51ff06..90fa08efab8 100644 --- a/examples/beacon-api-sidecar-fetcher/Cargo.toml +++ b/examples/beacon-api-sidecar-fetcher/Cargo.toml @@ -6,12 +6,12 @@ edition.workspace = true license.workspace = true [dependencies] -reth.workspace = true reth-ethereum = { workspace = true, features = ["node", "pool", "cli"] } alloy-rpc-types-beacon.workspace = true alloy-primitives.workspace = true alloy-consensus.workspace = true +alloy-eips.workspace = true clap.workspace = true eyre.workspace = true diff --git a/examples/beacon-api-sidecar-fetcher/src/main.rs b/examples/beacon-api-sidecar-fetcher/src/main.rs index 981da3f2a5c..4ec1727bc4e 100644 --- a/examples/beacon-api-sidecar-fetcher/src/main.rs +++ b/examples/beacon-api-sidecar-fetcher/src/main.rs @@ -1,7 +1,7 @@ //! Run with //! //! ```sh -//! cargo run -p beacon-api-beacon-sidecar-fetcher --node --full +//! cargo run -p example-beacon-api-sidecar-fetcher -- node --full //! ``` //! //! This launches a regular reth instance and subscribes to payload attributes event stream. @@ -22,10 +22,9 @@ use alloy_primitives::B256; use clap::Parser; use futures_util::{stream::FuturesUnordered, StreamExt}; use mined_sidecar::MinedSidecarStream; -use reth::builder::NodeHandle; use reth_ethereum::{ cli::{chainspec::EthereumChainSpecParser, interface::Cli}, - node::EthereumNode, + node::{builder::NodeHandle, EthereumNode}, provider::CanonStateSubscriptions, }; @@ -38,7 +37,7 @@ fn main() { let NodeHandle { node, node_exit_future } = builder.node(EthereumNode::default()).launch().await?; - let notifications: reth::providers::CanonStateNotificationStream = + let notifications: reth_ethereum::provider::CanonStateNotificationStream = node.provider.canonical_state_stream(); let pool = node.pool.clone(); @@ -86,10 +85,7 @@ pub struct BeaconSidecarConfig { impl Default for BeaconSidecarConfig { /// Default setup for lighthouse client fn default() -> Self { - Self { - cl_addr: IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), // Equivalent to Ipv4Addr::LOCALHOST - cl_port: 5052, - } + Self { cl_addr: IpAddr::V4(Ipv4Addr::LOCALHOST), cl_port: 5052 } } } diff --git a/examples/beacon-api-sidecar-fetcher/src/mined_sidecar.rs b/examples/beacon-api-sidecar-fetcher/src/mined_sidecar.rs index 2de0213980f..cc3ba9abf88 100644 --- a/examples/beacon-api-sidecar-fetcher/src/mined_sidecar.rs +++ b/examples/beacon-api-sidecar-fetcher/src/mined_sidecar.rs @@ -1,8 +1,6 @@ use crate::BeaconSidecarConfig; -use alloy_consensus::{ - transaction::PooledTransaction, BlockHeader, Signed, Transaction as _, TxEip4844WithSidecar, - Typed2718, -}; +use alloy_consensus::{Signed, Transaction as _, TxEip4844WithSidecar, Typed2718}; +use alloy_eips::eip7594::BlobTransactionSidecarVariant; use alloy_primitives::B256; use alloy_rpc_types_beacon::sidecar::{BeaconBlobBundle, SidecarIterator}; use eyre::Result; @@ -12,6 +10,7 @@ use reth_ethereum::{ pool::{BlobStoreError, TransactionPoolExt}, primitives::RecoveredBlock, provider::CanonStateNotification, + PooledTransactionVariant, }; use serde::{Deserialize, Serialize}; use std::{ @@ -31,7 +30,7 @@ pub struct BlockMetadata { #[derive(Debug, Clone)] pub struct MinedBlob { - pub transaction: Signed, + pub transaction: Signed>, pub block_metadata: BlockMetadata, } @@ -99,12 +98,12 @@ where St: Stream + Send + Unpin + 'static, P: TransactionPoolExt + Unpin + 'static, { - fn process_block(&mut self, block: &RecoveredBlock) { + fn process_block(&mut self, block: &RecoveredBlock) { let txs: Vec<_> = block .body() .transactions() .filter(|tx| tx.is_eip4844()) - .map(|tx| (tx.clone(), tx.blob_versioned_hashes().unwrap().len())) + .map(|tx| (tx.clone(), tx.blob_count().unwrap_or(0) as usize)) .collect(); let mut all_blobs_available = true; @@ -118,7 +117,7 @@ where Ok(blobs) => { actions_to_queue.reserve_exact(txs.len()); for ((tx, _), sidecar) in txs.iter().zip(blobs.into_iter()) { - if let PooledTransaction::Eip4844(transaction) = tx + if let PooledTransactionVariant::Eip4844(transaction) = tx .clone() .try_into_pooled_eip4844(Arc::unwrap_or_clone(sidecar)) .expect("should not fail to convert blob tx if it is already eip4844") @@ -203,9 +202,9 @@ where .map(|tx| { let transaction_hash = *tx.tx_hash(); let block_metadata = BlockMetadata { - block_hash: new.tip().hash(), - block_number: new.tip().number(), - gas_used: new.tip().gas_used(), + block_hash: block.hash(), + block_number: block.number, + gas_used: block.gas_used, }; BlobTransactionEvent::Reorged(ReorgedBlob { transaction_hash, @@ -231,8 +230,8 @@ where async fn fetch_blobs_for_block( client: reqwest::Client, url: String, - block: RecoveredBlock, - txs: Vec<(reth::primitives::TransactionSigned, usize)>, + block: RecoveredBlock, + txs: Vec<(reth_ethereum::TransactionSigned, usize)>, ) -> Result, SideCarError> { let response = match client.get(url).header("Accept", "application/json").send().await { Ok(response) => response, @@ -273,9 +272,9 @@ async fn fetch_blobs_for_block( .iter() .filter_map(|(tx, blob_len)| { sidecar_iterator.next_sidecar(*blob_len).and_then(|sidecar| { - if let PooledTransaction::Eip4844(transaction) = tx + if let PooledTransactionVariant::Eip4844(transaction) = tx .clone() - .try_into_pooled_eip4844(sidecar) + .try_into_pooled_eip4844(BlobTransactionSidecarVariant::Eip4844(sidecar)) .expect("should not fail to convert blob tx if it is already eip4844") { let block_metadata = BlockMetadata { diff --git a/examples/beacon-api-sse/src/main.rs b/examples/beacon-api-sse/src/main.rs index 46bb0ddd444..fee20e09b1f 100644 --- a/examples/beacon-api-sse/src/main.rs +++ b/examples/beacon-api-sse/src/main.rs @@ -5,7 +5,7 @@ //! Run with //! //! ```sh -//! cargo run -p beacon-api-sse -- node +//! cargo run -p example-beacon-api-sse -- node //! ``` //! //! This launches a regular reth instance and subscribes to payload attributes event stream. diff --git a/examples/bsc-p2p/Cargo.toml b/examples/bsc-p2p/Cargo.toml index f6f5677dc2a..a3e2ba1d6a5 100644 --- a/examples/bsc-p2p/Cargo.toml +++ b/examples/bsc-p2p/Cargo.toml @@ -29,7 +29,6 @@ alloy-rpc-types = { workspace = true, features = ["engine"] } # misc bytes.workspace = true -derive_more.workspace = true futures.workspace = true secp256k1 = { workspace = true, features = ["global-context", "std", "recovery"] } serde = { workspace = true, features = ["derive"], optional = true } diff --git a/examples/bsc-p2p/src/block_import/mod.rs b/examples/bsc-p2p/src/block_import/mod.rs index ba7820bd327..a017372dccb 100644 --- a/examples/bsc-p2p/src/block_import/mod.rs +++ b/examples/bsc-p2p/src/block_import/mod.rs @@ -1,6 +1,7 @@ #![allow(unused)] use handle::ImportHandle; use reth_engine_primitives::EngineTypes; +use reth_eth_wire::NewBlock; use reth_network::import::{BlockImport, BlockImportOutcome, NewBlockEvent}; use reth_network_peers::PeerId; use reth_payload_primitives::{BuiltPayload, PayloadTypes}; @@ -25,8 +26,12 @@ impl BscBlockImport { } } -impl BlockImport> for BscBlockImport { - fn on_new_block(&mut self, peer_id: PeerId, incoming_block: NewBlockEvent>) { +impl BlockImport>> for BscBlockImport { + fn on_new_block( + &mut self, + peer_id: PeerId, + incoming_block: NewBlockEvent>>, + ) { if let NewBlockEvent::Block(block) = incoming_block { let _ = self.handle.send_block(block, peer_id); } @@ -43,7 +48,7 @@ impl BlockImport> for BscBlockImport { impl fmt::Debug for BscBlockImport { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("BscBlockImport") - .field("engine_handle", &"BeaconConsensusEngineHandle") + .field("engine_handle", &"ConsensusEngineHandle") .field("service_handle", &"BscBlockImportHandle") .finish() } diff --git a/examples/bsc-p2p/src/block_import/parlia.rs b/examples/bsc-p2p/src/block_import/parlia.rs index ec7459ca8b9..a985895aa6c 100644 --- a/examples/bsc-p2p/src/block_import/parlia.rs +++ b/examples/bsc-p2p/src/block_import/parlia.rs @@ -32,7 +32,7 @@ where { /// Determines the head block hash according to Parlia consensus rules: /// 1. Follow the highest block number - /// 2. For same height blocks, pick the one with lower hash + /// 2. For the same height blocks, pick the one with the lower hash pub(crate) fn canonical_head( &self, hash: B256, diff --git a/examples/bsc-p2p/src/block_import/service.rs b/examples/bsc-p2p/src/block_import/service.rs index e3d05886105..35003423e73 100644 --- a/examples/bsc-p2p/src/block_import/service.rs +++ b/examples/bsc-p2p/src/block_import/service.rs @@ -2,7 +2,8 @@ use super::handle::ImportHandle; use crate::block_import::parlia::{ParliaConsensus, ParliaConsensusErr}; use alloy_rpc_types::engine::{ForkchoiceState, PayloadStatusEnum}; use futures::{future::Either, stream::FuturesUnordered, StreamExt}; -use reth_engine_primitives::{BeaconConsensusEngineHandle, EngineTypes}; +use reth_engine_primitives::{ConsensusEngineHandle, EngineTypes}; +use reth_eth_wire::NewBlock; use reth_network::{ import::{BlockImportError, BlockImportEvent, BlockImportOutcome, BlockValidation}, message::NewBlockMessage, @@ -25,13 +26,13 @@ pub type BscBlock = <<::BuiltPayload as BuiltPayload>::Primitives as NodePrimitives>::Block; /// Network message containing a new block -pub(crate) type BlockMsg = NewBlockMessage>; +pub(crate) type BlockMsg = NewBlockMessage>>; /// Import outcome for a block -pub(crate) type Outcome = BlockImportOutcome>; +pub(crate) type Outcome = BlockImportOutcome>>; /// Import event for a block -pub(crate) type ImportEvent = BlockImportEvent>; +pub(crate) type ImportEvent = BlockImportEvent>>; /// Future that processes a block import and returns its outcome type PayloadFut = Pin> + Send + Sync>>; @@ -51,7 +52,7 @@ where T: PayloadTypes, { /// The handle to communicate with the engine service - engine: BeaconConsensusEngineHandle, + engine: ConsensusEngineHandle, /// The consensus implementation consensus: Arc>, /// Receive the new block from the network @@ -70,7 +71,7 @@ where /// Create a new block import service pub fn new( consensus: Arc>, - engine: BeaconConsensusEngineHandle, + engine: ConsensusEngineHandle, ) -> (Self, ImportHandle) { let (to_import, from_network) = mpsc::unbounded_channel(); let (to_network, import_outcome) = mpsc::unbounded_channel(); @@ -356,7 +357,7 @@ mod tests { async fn new(responses: EngineResponses) -> Self { let consensus = Arc::new(ParliaConsensus::new(MockProvider)); let (to_engine, from_engine) = mpsc::unbounded_channel(); - let engine_handle = BeaconConsensusEngineHandle::new(to_engine); + let engine_handle = ConsensusEngineHandle::new(to_engine); handle_engine_msg(from_engine, responses).await; @@ -371,7 +372,7 @@ mod tests { /// Run a block import test with the given event assertion async fn assert_block_import(&mut self, assert_fn: F) where - F: Fn(&BlockImportEvent>) -> bool, + F: Fn(&BlockImportEvent>>) -> bool, { let block_msg = create_test_block(); self.handle.send_block(block_msg, PeerId::random()).unwrap(); @@ -400,7 +401,7 @@ mod tests { } /// Creates a test block message - fn create_test_block() -> NewBlockMessage { + fn create_test_block() -> NewBlockMessage> { let block: reth_primitives::Block = Block::default(); let new_block = NewBlock { block: block.clone(), td: U128::ZERO }; NewBlockMessage { hash: block.header.hash_slow(), block: Arc::new(new_block) } diff --git a/examples/bsc-p2p/src/handshake.rs b/examples/bsc-p2p/src/handshake.rs index 5d5d7d04cbf..1f619afd3b5 100644 --- a/examples/bsc-p2p/src/handshake.rs +++ b/examples/bsc-p2p/src/handshake.rs @@ -4,8 +4,9 @@ use futures::SinkExt; use reth_eth_wire::{ errors::{EthHandshakeError, EthStreamError}, handshake::{EthRlpxHandshake, EthereumEthHandshake, UnauthEth}, + UnifiedStatus, }; -use reth_eth_wire_types::{DisconnectReason, EthVersion, Status}; +use reth_eth_wire_types::{DisconnectReason, EthVersion}; use reth_ethereum_forks::ForkFilter; use std::{future::Future, pin::Pin}; use tokio::time::{timeout, Duration}; @@ -21,8 +22,8 @@ impl BscHandshake { /// Negotiate the upgrade status message. pub async fn upgrade_status( unauth: &mut dyn UnauthEth, - negotiated_status: Status, - ) -> Result { + negotiated_status: UnifiedStatus, + ) -> Result { if negotiated_status.version > EthVersion::Eth66 { // Send upgrade status message allowing peer to broadcast transactions let upgrade_msg = UpgradeStatus { @@ -66,10 +67,10 @@ impl EthRlpxHandshake for BscHandshake { fn handshake<'a>( &'a self, unauth: &'a mut dyn UnauthEth, - status: Status, + status: UnifiedStatus, fork_filter: ForkFilter, timeout_limit: Duration, - ) -> Pin> + 'a + Send>> { + ) -> Pin> + 'a + Send>> { Box::pin(async move { let fut = async { let negotiated_status = diff --git a/examples/bsc-p2p/src/upgrade_status.rs b/examples/bsc-p2p/src/upgrade_status.rs index eadbdcd6ace..c31cf1751ac 100644 --- a/examples/bsc-p2p/src/upgrade_status.rs +++ b/examples/bsc-p2p/src/upgrade_status.rs @@ -44,7 +44,7 @@ impl UpgradeStatus { } /// The extension to define whether to enable or disable the flag. -/// This flag currently is ignored, and will be supported later. +/// This flag is currently ignored, and will be supported later. #[derive(Debug, Clone, PartialEq, Eq, RlpEncodable, RlpDecodable)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] pub struct UpgradeStatusExtension { diff --git a/examples/custom-beacon-withdrawals/Cargo.toml b/examples/custom-beacon-withdrawals/Cargo.toml index ca69b4f029b..c36a5ee915a 100644 --- a/examples/custom-beacon-withdrawals/Cargo.toml +++ b/examples/custom-beacon-withdrawals/Cargo.toml @@ -6,7 +6,6 @@ edition.workspace = true license.workspace = true [dependencies] -reth.workspace = true reth-ethereum = { workspace = true, features = ["node", "node-api", "evm", "cli"] } alloy-evm.workspace = true diff --git a/examples/custom-beacon-withdrawals/src/main.rs b/examples/custom-beacon-withdrawals/src/main.rs index f501c598bd3..1d93226dd6a 100644 --- a/examples/custom-beacon-withdrawals/src/main.rs +++ b/examples/custom-beacon-withdrawals/src/main.rs @@ -7,24 +7,23 @@ use alloy_eips::eip4895::Withdrawal; use alloy_evm::{ block::{BlockExecutorFactory, BlockExecutorFor, ExecutableTx}, eth::{EthBlockExecutionCtx, EthBlockExecutor}, + precompiles::PrecompilesMap, + revm::context::{result::ResultAndState, Block as _}, EthEvm, EthEvmFactory, }; use alloy_sol_macro::sol; use alloy_sol_types::SolCall; -use reth::{ - builder::{components::ExecutorBuilder, BuilderContext}, - primitives::SealedBlock, -}; use reth_ethereum::{ chainspec::ChainSpec, cli::interface::Cli, evm::{ primitives::{ execute::{BlockExecutionError, BlockExecutor, InternalBlockExecutionError}, - Database, Evm, EvmEnv, InspectorFor, NextBlockEnvAttributes, OnStateHook, + Database, Evm, EvmEnv, EvmEnvFor, ExecutionCtxFor, InspectorFor, + NextBlockEnvAttributes, OnStateHook, }, revm::{ - context::{result::ExecutionResult, TxEnv}, + context::TxEnv, db::State, primitives::{address, hardfork::SpecId, Address}, DatabaseCommit, @@ -32,13 +31,15 @@ use reth_ethereum::{ EthBlockAssembler, EthEvmConfig, RethReceiptBuilder, }, node::{ - api::{ConfigureEvm, FullNodeTypes, NodeTypes}, + api::{ConfigureEngineEvm, ConfigureEvm, ExecutableTxIterator, FullNodeTypes, NodeTypes}, + builder::{components::ExecutorBuilder, BuilderContext}, node::EthereumAddOns, EthereumNode, }, - primitives::{Header, SealedHeader}, + primitives::{Header, SealedBlock, SealedHeader}, provider::BlockExecutionResult, - EthPrimitives, Receipt, TransactionSigned, + rpc::types::engine::ExecutionData, + Block, EthPrimitives, Receipt, TransactionSigned, }; use std::{fmt::Display, sync::Arc}; @@ -101,7 +102,7 @@ impl BlockExecutorFactory for CustomEvmConfig { fn create_executor<'a, DB, I>( &'a self, - evm: EthEvm<&'a mut State, I>, + evm: EthEvm<&'a mut State, I, PrecompilesMap>, ctx: EthBlockExecutionCtx<'a>, ) -> impl BlockExecutorFor<'a, Self, DB, I> where @@ -134,7 +135,7 @@ impl ConfigureEvm for CustomEvmConfig { self.inner.block_assembler() } - fn evm_env(&self, header: &Header) -> EvmEnv { + fn evm_env(&self, header: &Header) -> Result, Self::Error> { self.inner.evm_env(header) } @@ -146,7 +147,10 @@ impl ConfigureEvm for CustomEvmConfig { self.inner.next_evm_env(parent, attributes) } - fn context_for_block<'a>(&self, block: &'a SealedBlock) -> EthBlockExecutionCtx<'a> { + fn context_for_block<'a>( + &self, + block: &'a SealedBlock, + ) -> Result, Self::Error> { self.inner.context_for_block(block) } @@ -154,11 +158,31 @@ impl ConfigureEvm for CustomEvmConfig { &self, parent: &SealedHeader, attributes: Self::NextBlockEnvCtx, - ) -> EthBlockExecutionCtx<'_> { + ) -> Result, Self::Error> { self.inner.context_for_next_block(parent, attributes) } } +impl ConfigureEngineEvm for CustomEvmConfig { + fn evm_env_for_payload(&self, payload: &ExecutionData) -> Result, Self::Error> { + self.inner.evm_env_for_payload(payload) + } + + fn context_for_payload<'a>( + &self, + payload: &'a ExecutionData, + ) -> Result, Self::Error> { + self.inner.context_for_payload(payload) + } + + fn tx_iterator_for_payload( + &self, + payload: &ExecutionData, + ) -> Result, Self::Error> { + self.inner.tx_iterator_for_payload(payload) + } +} + pub struct CustomBlockExecutor<'a, Evm> { /// Inner Ethereum execution strategy. inner: EthBlockExecutor<'a, Evm, &'a Arc, &'a RethReceiptBuilder>, @@ -177,12 +201,19 @@ where self.inner.apply_pre_execution_changes() } - fn execute_transaction_with_result_closure( + fn execute_transaction_without_commit( + &mut self, + tx: impl ExecutableTx, + ) -> Result::HaltReason>, BlockExecutionError> { + self.inner.execute_transaction_without_commit(tx) + } + + fn commit_transaction( &mut self, + output: ResultAndState<::HaltReason>, tx: impl ExecutableTx, - f: impl FnOnce(&ExecutionResult<::HaltReason>), ) -> Result { - self.inner.execute_transaction_with_result_closure(tx, f) + self.inner.commit_transaction(output, tx) } fn finish(mut self) -> Result<(Self::Evm, BlockExecutionResult), BlockExecutionError> { @@ -240,7 +271,7 @@ pub fn apply_withdrawals_contract_call( // Clean-up post system tx context state.remove(&SYSTEM_ADDRESS); - state.remove(&evm.block().beneficiary); + state.remove(&evm.block().beneficiary()); evm.db_mut().commit(state); diff --git a/examples/custom-dev-node/Cargo.toml b/examples/custom-dev-node/Cargo.toml index 60a57a1ec90..ad0ba9aba9c 100644 --- a/examples/custom-dev-node/Cargo.toml +++ b/examples/custom-dev-node/Cargo.toml @@ -6,7 +6,6 @@ edition.workspace = true license.workspace = true [dependencies] -reth.workspace = true reth-ethereum = { workspace = true, features = ["node", "test-utils"] } futures-util.workspace = true diff --git a/examples/custom-dev-node/src/main.rs b/examples/custom-dev-node/src/main.rs index 892d017477a..c5441a2b388 100644 --- a/examples/custom-dev-node/src/main.rs +++ b/examples/custom-dev-node/src/main.rs @@ -8,18 +8,16 @@ use std::sync::Arc; use alloy_genesis::Genesis; use alloy_primitives::{b256, hex}; use futures_util::StreamExt; -use reth::{ - builder::{NodeBuilder, NodeHandle}, - tasks::TaskManager, -}; use reth_ethereum::{ chainspec::ChainSpec, node::{ + builder::{NodeBuilder, NodeHandle}, core::{args::RpcServerArgs, node_config::NodeConfig}, EthereumNode, }, provider::CanonStateSubscriptions, rpc::api::eth::helpers::EthTransactions, + tasks::TaskManager, }; #[tokio::main] @@ -35,7 +33,7 @@ async fn main() -> eyre::Result<()> { let NodeHandle { node, node_exit_future: _ } = NodeBuilder::new(node_config) .testing_node(tasks.executor()) .node(EthereumNode::default()) - .launch() + .launch_with_debug_capabilities() .await?; let mut notifications = node.provider.canonical_state_stream(); diff --git a/examples/custom-engine-types/Cargo.toml b/examples/custom-engine-types/Cargo.toml index 73e233190fc..d6f41980dd8 100644 --- a/examples/custom-engine-types/Cargo.toml +++ b/examples/custom-engine-types/Cargo.toml @@ -6,14 +6,11 @@ edition.workspace = true license.workspace = true [dependencies] -reth.workspace = true reth-payload-builder.workspace = true reth-basic-payload-builder.workspace = true reth-ethereum-payload-builder.workspace = true -reth-engine-local.workspace = true reth-ethereum = { workspace = true, features = ["test-utils", "node", "node-api", "pool"] } reth-tracing.workspace = true -reth-trie-db.workspace = true alloy-genesis.workspace = true alloy-rpc-types = { workspace = true, features = ["engine"] } alloy-primitives.workspace = true diff --git a/examples/custom-engine-types/src/main.rs b/examples/custom-engine-types/src/main.rs index 4f146073ef3..ca724e52af2 100644 --- a/examples/custom-engine-types/src/main.rs +++ b/examples/custom-engine-types/src/main.rs @@ -23,31 +23,25 @@ use alloy_primitives::{Address, B256}; use alloy_rpc_types::{ engine::{ ExecutionData, ExecutionPayloadEnvelopeV2, ExecutionPayloadEnvelopeV3, - ExecutionPayloadEnvelopeV4, ExecutionPayloadV1, PayloadAttributes as EthPayloadAttributes, - PayloadId, + ExecutionPayloadEnvelopeV4, ExecutionPayloadEnvelopeV5, ExecutionPayloadV1, + PayloadAttributes as EthPayloadAttributes, PayloadId, }, Withdrawal, }; -use reth::{ - builder::{ - components::{BasicPayloadServiceBuilder, ComponentsBuilder, PayloadBuilderBuilder}, - node::NodeTypes, - rpc::{EngineValidatorBuilder, RpcAddOns}, - BuilderContext, FullNodeTypes, Node, NodeAdapter, NodeBuilder, NodeComponentsBuilder, - }, - rpc::types::engine::ExecutionPayload, - tasks::TaskManager, -}; use reth_basic_payload_builder::{BuildArguments, BuildOutcome, PayloadBuilder, PayloadConfig}; -use reth_engine_local::payload::UnsupportedLocalAttributes; use reth_ethereum::{ chainspec::{Chain, ChainSpec, ChainSpecProvider}, node::{ api::{ payload::{EngineApiMessageVersion, EngineObjectValidationError, PayloadOrAttributes}, - validate_version_specific_fields, AddOnsContext, EngineTypes, EngineValidator, - FullNodeComponents, InvalidPayloadAttributesError, NewPayloadError, PayloadAttributes, - PayloadBuilderAttributes, PayloadTypes, PayloadValidator, + validate_version_specific_fields, AddOnsContext, EngineApiValidator, EngineTypes, + FullNodeComponents, FullNodeTypes, InvalidPayloadAttributesError, NewPayloadError, + NodeTypes, PayloadAttributes, PayloadBuilderAttributes, PayloadTypes, PayloadValidator, + }, + builder::{ + components::{BasicPayloadServiceBuilder, ComponentsBuilder, PayloadBuilderBuilder}, + rpc::{PayloadValidatorBuilder, RpcAddOns}, + BuilderContext, Node, NodeAdapter, NodeBuilder, }, core::{args::RpcServerArgs, node_config::NodeConfig}, node::{ @@ -57,14 +51,15 @@ use reth_ethereum::{ EthEvmConfig, EthereumEthApiBuilder, }, pool::{PoolTransaction, TransactionPool}, - primitives::{RecoveredBlock, SealedBlock}, + primitives::{Block, RecoveredBlock, SealedBlock}, provider::{EthStorage, StateProviderFactory}, - Block, EthPrimitives, TransactionSigned, + rpc::types::engine::ExecutionPayload, + tasks::TaskManager, + EthPrimitives, TransactionSigned, }; use reth_ethereum_payload_builder::{EthereumBuilderConfig, EthereumExecutionPayloadValidator}; use reth_payload_builder::{EthBuiltPayload, EthPayloadBuilderAttributes, PayloadBuilderError}; use reth_tracing::{RethTracer, Tracer}; -use reth_trie_db::MerklePatriciaTrie; use serde::{Deserialize, Serialize}; use std::{convert::Infallible, sync::Arc}; use thiserror::Error; @@ -79,9 +74,6 @@ pub struct CustomPayloadAttributes { pub custom: u64, } -// TODO(mattsse): remove this tmp workaround -impl UnsupportedLocalAttributes for CustomPayloadAttributes {} - /// Custom error type used in payload attributes validation #[derive(Debug, Error)] pub enum CustomError { @@ -176,6 +168,7 @@ impl EngineTypes for CustomEngineTypes { type ExecutionPayloadEnvelopeV2 = ExecutionPayloadEnvelopeV2; type ExecutionPayloadEnvelopeV3 = ExecutionPayloadEnvelopeV3; type ExecutionPayloadEnvelopeV4 = ExecutionPayloadEnvelopeV4; + type ExecutionPayloadEnvelopeV5 = ExecutionPayloadEnvelopeV5; } /// Custom engine validator @@ -197,9 +190,8 @@ impl CustomEngineValidator { } } -impl PayloadValidator for CustomEngineValidator { - type Block = Block; - type ExecutionData = ExecutionData; +impl PayloadValidator for CustomEngineValidator { + type Block = reth_ethereum::Block; fn ensure_well_formed_payload( &self, @@ -208,16 +200,22 @@ impl PayloadValidator for CustomEngineValidator { let sealed_block = self.inner.ensure_well_formed_payload(payload)?; sealed_block.try_recover().map_err(|e| NewPayloadError::Other(e.into())) } + + fn validate_payload_attributes_against_header( + &self, + _attr: &CustomPayloadAttributes, + _header: &::Header, + ) -> Result<(), InvalidPayloadAttributesError> { + // skip default timestamp validation + Ok(()) + } } -impl EngineValidator for CustomEngineValidator -where - T: PayloadTypes, -{ +impl EngineApiValidator for CustomEngineValidator { fn validate_version_specific_fields( &self, version: EngineApiMessageVersion, - payload_or_attrs: PayloadOrAttributes<'_, Self::ExecutionData, T::PayloadAttributes>, + payload_or_attrs: PayloadOrAttributes<'_, ExecutionData, CustomPayloadAttributes>, ) -> Result<(), EngineObjectValidationError> { validate_version_specific_fields(self.chain_spec(), version, payload_or_attrs) } @@ -225,12 +223,12 @@ where fn ensure_well_formed_attributes( &self, version: EngineApiMessageVersion, - attributes: &T::PayloadAttributes, + attributes: &CustomPayloadAttributes, ) -> Result<(), EngineObjectValidationError> { validate_version_specific_fields( self.chain_spec(), version, - PayloadOrAttributes::::PayloadAttributes( + PayloadOrAttributes::::PayloadAttributes( attributes, ), )?; @@ -244,15 +242,6 @@ where Ok(()) } - - fn validate_payload_attributes_against_header( - &self, - _attr: &::PayloadAttributes, - _header: &::Header, - ) -> Result<(), InvalidPayloadAttributesError> { - // skip default timestamp validation - Ok(()) - } } /// Custom engine validator builder @@ -260,15 +249,9 @@ where #[non_exhaustive] pub struct CustomEngineValidatorBuilder; -impl EngineValidatorBuilder for CustomEngineValidatorBuilder +impl PayloadValidatorBuilder for CustomEngineValidatorBuilder where - N: FullNodeComponents< - Types: NodeTypes< - Payload = CustomEngineTypes, - ChainSpec = ChainSpec, - Primitives = EthPrimitives, - >, - >, + N: FullNodeComponents, { type Validator = CustomEngineValidator; @@ -285,7 +268,6 @@ struct MyCustomNode; impl NodeTypes for MyCustomNode { type Primitives = EthPrimitives; type ChainSpec = ChainSpec; - type StateCommitment = MerklePatriciaTrie; type Storage = EthStorage; type Payload = CustomEngineTypes; } @@ -298,14 +280,7 @@ pub type MyNodeAddOns = RpcAddOns Node for MyCustomNode where - N: FullNodeTypes< - Types: NodeTypes< - Payload = CustomEngineTypes, - ChainSpec = ChainSpec, - Primitives = EthPrimitives, - Storage = EthStorage, - >, - >, + N: FullNodeTypes, { type ComponentsBuilder = ComponentsBuilder< N, @@ -315,9 +290,7 @@ where EthereumExecutorBuilder, EthereumConsensusBuilder, >; - type AddOns = MyNodeAddOns< - NodeAdapter>::Components>, - >; + type AddOns = MyNodeAddOns>; fn components_builder(&self) -> Self::ComponentsBuilder { ComponentsBuilder::default() diff --git a/examples/custom-evm/Cargo.toml b/examples/custom-evm/Cargo.toml index c3a14e56576..84ba29cf5c6 100644 --- a/examples/custom-evm/Cargo.toml +++ b/examples/custom-evm/Cargo.toml @@ -6,7 +6,6 @@ edition.workspace = true license.workspace = true [dependencies] -reth.workspace = true reth-ethereum = { workspace = true, features = ["test-utils", "node", "evm", "pool"] } reth-tracing.workspace = true alloy-evm.workspace = true diff --git a/examples/custom-evm/src/main.rs b/examples/custom-evm/src/main.rs index 79d9eab348e..e32f0be6bd5 100644 --- a/examples/custom-evm/src/main.rs +++ b/examples/custom-evm/src/main.rs @@ -2,27 +2,27 @@ #![warn(unused_crate_dependencies)] -use alloy_evm::{eth::EthEvmContext, EvmFactory}; -use alloy_genesis::Genesis; -use alloy_primitives::{address, Address, Bytes}; -use reth::{ - builder::{components::ExecutorBuilder, BuilderContext, NodeBuilder}, - tasks::TaskManager, +use alloy_evm::{ + eth::EthEvmContext, + precompiles::PrecompilesMap, + revm::{ + handler::EthPrecompiles, + precompile::{Precompile, PrecompileId}, + }, + EvmFactory, }; +use alloy_genesis::Genesis; +use alloy_primitives::{address, Bytes}; use reth_ethereum::{ chainspec::{Chain, ChainSpec}, evm::{ primitives::{Database, EvmEnv}, revm::{ - context::{Cfg, Context, TxEnv}, - context_interface::{ - result::{EVMError, HaltReason}, - ContextTr, - }, - handler::{EthPrecompiles, PrecompileProvider}, + context::{BlockEnv, Context, TxEnv}, + context_interface::result::{EVMError, HaltReason}, inspector::{Inspector, NoOpInspector}, - interpreter::{interpreter::EthInterpreter, InputsImpl, InterpreterResult}, - precompile::{PrecompileFn, PrecompileOutput, PrecompileResult, Precompiles}, + interpreter::interpreter::EthInterpreter, + precompile::{PrecompileOutput, PrecompileResult, Precompiles}, primitives::hardfork::SpecId, MainBuilder, MainContext, }, @@ -30,10 +30,12 @@ use reth_ethereum::{ }, node::{ api::{FullNodeTypes, NodeTypes}, + builder::{components::ExecutorBuilder, BuilderContext, NodeBuilder}, core::{args::RpcServerArgs, node_config::NodeConfig}, node::EthereumAddOns, EthereumNode, }, + tasks::TaskManager, EthPrimitives, }; use reth_tracing::{RethTracer, Tracer}; @@ -46,20 +48,27 @@ pub struct MyEvmFactory; impl EvmFactory for MyEvmFactory { type Evm, EthInterpreter>> = - EthEvm; + EthEvm; type Tx = TxEnv; type Error = EVMError; type HaltReason = HaltReason; type Context = EthEvmContext; type Spec = SpecId; + type BlockEnv = BlockEnv; + type Precompiles = PrecompilesMap; fn create_evm(&self, db: DB, input: EvmEnv) -> Self::Evm { - let evm = Context::mainnet() + let spec = input.cfg_env.spec; + let mut evm = Context::mainnet() .with_db(db) .with_cfg(input.cfg_env) .with_block(input.block_env) .build_mainnet_with_inspector(NoOpInspector {}) - .with_precompiles(CustomPrecompiles::new()); + .with_precompiles(PrecompilesMap::from_static(EthPrecompiles::default().precompiles)); + + if spec == SpecId::PRAGUE { + evm = evm.with_precompiles(PrecompilesMap::from_static(prague_custom())); + } EthEvm::new(evm, false) } @@ -83,7 +92,7 @@ impl ExecutorBuilder for MyExecutorBuilder where Node: FullNodeTypes>, { - type EVM = EthEvmConfig; + type EVM = EthEvmConfig; async fn build_evm(self, ctx: &BuilderContext) -> eyre::Result { let evm_config = @@ -92,70 +101,22 @@ where } } -/// A custom precompile that contains static precompiles. -#[derive(Clone)] -pub struct CustomPrecompiles { - pub precompiles: EthPrecompiles, -} - -impl CustomPrecompiles { - /// Given a [`PrecompileProvider`] and cache for a specific precompiles, create a - /// wrapper that can be used inside Evm. - fn new() -> Self { - Self { precompiles: EthPrecompiles::default() } - } -} - -/// Returns precompiles for Fjor spec. +/// Returns precompiles for Prague spec. pub fn prague_custom() -> &'static Precompiles { static INSTANCE: OnceLock = OnceLock::new(); INSTANCE.get_or_init(|| { let mut precompiles = Precompiles::prague().clone(); // Custom precompile. - precompiles.extend([( + let precompile = Precompile::new( + PrecompileId::custom("custom"), address!("0x0000000000000000000000000000000000000999"), - |_, _| -> PrecompileResult { - PrecompileResult::Ok(PrecompileOutput::new(0, Bytes::new())) - } as PrecompileFn, - ) - .into()]); + |_, _| PrecompileResult::Ok(PrecompileOutput::new(0, Bytes::new())), + ); + precompiles.extend([precompile]); precompiles }) } -impl PrecompileProvider for CustomPrecompiles { - type Output = InterpreterResult; - - fn set_spec(&mut self, spec: ::Spec) -> bool { - let spec_id = spec.clone().into(); - if spec_id == SpecId::PRAGUE { - self.precompiles = EthPrecompiles { precompiles: prague_custom(), spec: spec.into() } - } else { - PrecompileProvider::::set_spec(&mut self.precompiles, spec); - } - true - } - - fn run( - &mut self, - context: &mut CTX, - address: &Address, - inputs: &InputsImpl, - is_static: bool, - gas_limit: u64, - ) -> Result, String> { - self.precompiles.run(context, address, inputs, is_static, gas_limit) - } - - fn warm_addresses(&self) -> Box> { - self.precompiles.warm_addresses() - } - - fn contains(&self, address: &Address) -> bool { - self.precompiles.contains(address) - } -} - #[tokio::main] async fn main() -> eyre::Result<()> { let _guard = RethTracer::new().init()?; diff --git a/examples/custom-hardforks/Cargo.toml b/examples/custom-hardforks/Cargo.toml new file mode 100644 index 00000000000..78060f6af62 --- /dev/null +++ b/examples/custom-hardforks/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "custom-hardforks" +license.workspace = true +version = "0.1.0" +edition = "2021" +publish = false + +[dependencies] +# Core Reth dependencies for chain specs and hardforks +reth-chainspec.workspace = true +reth-network-peers.workspace = true +alloy-genesis.workspace = true +alloy-consensus.workspace = true +alloy-primitives.workspace = true +alloy-eips.workspace = true +serde = { version = "1.0", features = ["derive"] } diff --git a/examples/custom-hardforks/src/chainspec.rs b/examples/custom-hardforks/src/chainspec.rs new file mode 100644 index 00000000000..d51db59fddb --- /dev/null +++ b/examples/custom-hardforks/src/chainspec.rs @@ -0,0 +1,149 @@ +//! Custom chain specification integrating hardforks. +//! +//! This demonstrates how to build a `ChainSpec` with custom hardforks, +//! implementing required traits for integration with Reth's chain management. + +use alloy_eips::eip7840::BlobParams; +use alloy_genesis::Genesis; +use alloy_primitives::{B256, U256}; +use reth_chainspec::{ + hardfork, BaseFeeParams, Chain, ChainSpec, DepositContract, EthChainSpec, EthereumHardfork, + EthereumHardforks, ForkCondition, Hardfork, Hardforks, +}; +use reth_network_peers::NodeRecord; +use serde::{Deserialize, Serialize}; + +// Define custom hardfork variants using Reth's `hardfork!` macro. +// Each variant represents a protocol upgrade (e.g., enabling new features). +hardfork!( + /// Custom hardforks for the example chain. + /// + /// These are inspired by Ethereum's upgrades but customized for demonstration. + /// Add new variants here to extend the chain's hardfork set. + CustomHardfork { + /// Enables basic custom features (e.g., a new precompile). + BasicUpgrade, + /// Enables advanced features (e.g., state modifications). + AdvancedUpgrade, + } +); + +// Implement the `Hardfork` trait for each variant. +// This defines the name and any custom logic (e.g., feature toggles). +// Note: The hardfork! macro already implements Hardfork, so no manual impl needed. + +// Configuration for hardfork activation. +// This struct holds settings like activation blocks and is serializable for config files. +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub struct CustomHardforkConfig { + /// Block number to activate BasicUpgrade. + pub basic_upgrade_block: u64, + /// Block number to activate AdvancedUpgrade. + pub advanced_upgrade_block: u64, +} + +// Custom chain spec wrapping Reth's `ChainSpec` with our hardforks. +#[derive(Debug, Clone)] +pub struct CustomChainSpec { + pub inner: ChainSpec, +} + +impl CustomChainSpec { + /// Creates a custom chain spec from a genesis file. + /// + /// This parses the [`ChainSpec`] and adds the custom hardforks. + pub fn from_genesis(genesis: Genesis) -> Self { + let extra = genesis.config.extra_fields.deserialize_as::().unwrap(); + + let mut inner = ChainSpec::from_genesis(genesis); + inner.hardforks.insert( + CustomHardfork::BasicUpgrade, + ForkCondition::Timestamp(extra.basic_upgrade_block), + ); + inner.hardforks.insert( + CustomHardfork::AdvancedUpgrade, + ForkCondition::Timestamp(extra.advanced_upgrade_block), + ); + Self { inner } + } +} + +// Implement `Hardforks` to integrate custom hardforks with Reth's system. +impl Hardforks for CustomChainSpec { + fn fork(&self, fork: H) -> ForkCondition { + self.inner.fork(fork) + } + + fn forks_iter(&self) -> impl Iterator { + self.inner.forks_iter() + } + + fn fork_id(&self, head: &reth_chainspec::Head) -> reth_chainspec::ForkId { + self.inner.fork_id(head) + } + + fn latest_fork_id(&self) -> reth_chainspec::ForkId { + self.inner.latest_fork_id() + } + + fn fork_filter(&self, head: reth_chainspec::Head) -> reth_chainspec::ForkFilter { + self.inner.fork_filter(head) + } +} + +// Implement `EthChainSpec` for compatibility with Ethereum-based nodes. +impl EthChainSpec for CustomChainSpec { + type Header = alloy_consensus::Header; + + fn chain(&self) -> Chain { + self.inner.chain() + } + + fn base_fee_params_at_timestamp(&self, timestamp: u64) -> BaseFeeParams { + self.inner.base_fee_params_at_timestamp(timestamp) + } + + fn blob_params_at_timestamp(&self, timestamp: u64) -> Option { + self.inner.blob_params_at_timestamp(timestamp) + } + + fn deposit_contract(&self) -> Option<&DepositContract> { + self.inner.deposit_contract() + } + + fn genesis_hash(&self) -> B256 { + self.inner.genesis_hash() + } + + fn prune_delete_limit(&self) -> usize { + self.inner.prune_delete_limit() + } + + fn display_hardforks(&self) -> Box { + Box::new(self.inner.display_hardforks()) + } + + fn genesis_header(&self) -> &Self::Header { + self.inner.genesis_header() + } + + fn genesis(&self) -> &Genesis { + self.inner.genesis() + } + + fn bootnodes(&self) -> Option> { + self.inner.bootnodes() + } + + fn final_paris_total_difficulty(&self) -> Option { + self.inner.final_paris_total_difficulty() + } +} + +// Implement `EthereumHardforks` to support Ethereum hardfork queries. +impl EthereumHardforks for CustomChainSpec { + fn ethereum_fork_activation(&self, fork: EthereumHardfork) -> ForkCondition { + self.inner.ethereum_fork_activation(fork) + } +} diff --git a/examples/custom-hardforks/src/main.rs b/examples/custom-hardforks/src/main.rs new file mode 100644 index 00000000000..588f260c616 --- /dev/null +++ b/examples/custom-hardforks/src/main.rs @@ -0,0 +1,5 @@ +//! Example that showcases how to inject custom hardforks. + +pub mod chainspec; + +fn main() {} diff --git a/examples/custom-inspector/Cargo.toml b/examples/custom-inspector/Cargo.toml index cd9cab56be3..22bf3c7a246 100644 --- a/examples/custom-inspector/Cargo.toml +++ b/examples/custom-inspector/Cargo.toml @@ -6,7 +6,6 @@ edition.workspace = true license.workspace = true [dependencies] -reth.workspace = true reth-ethereum = { workspace = true, features = ["node", "evm", "pool", "cli"] } alloy-rpc-types-eth.workspace = true clap = { workspace = true, features = ["derive"] } diff --git a/examples/custom-inspector/src/main.rs b/examples/custom-inspector/src/main.rs index 2227c22c76f..f7accf0e8c0 100644 --- a/examples/custom-inspector/src/main.rs +++ b/examples/custom-inspector/src/main.rs @@ -16,7 +16,6 @@ use alloy_primitives::Address; use alloy_rpc_types_eth::{state::EvmOverrides, TransactionRequest}; use clap::Parser; use futures_util::StreamExt; -use reth::builder::NodeHandle; use reth_ethereum::{ cli::{chainspec::EthereumChainSpecParser, interface::Cli}, evm::{ @@ -28,7 +27,7 @@ use reth_ethereum::{ interpreter::{interpreter::EthInterpreter, interpreter_types::Jumps, Interpreter}, }, }, - node::EthereumNode, + node::{builder::FullNodeFor, EthereumNode}, pool::TransactionPool, rpc::api::eth::helpers::Call, }; @@ -37,8 +36,10 @@ fn main() { Cli::::parse() .run(|builder, args| async move { // launch the node - let NodeHandle { node, node_exit_future } = - builder.node(EthereumNode::default()).launch().await?; + let handle = builder.node(EthereumNode::default()).launch().await?; + + let node: FullNodeFor = handle.node; + let node_exit_future = handle.node_exit_future; // create a new subscription to pending transactions let mut pending_transactions = node.pool.new_pending_pool_transactions_listener(); @@ -55,43 +56,43 @@ fn main() { let tx = event.transaction; println!("Transaction received: {tx:?}"); - if let Some(recipient) = tx.to() { - if args.is_match(&recipient) { - // convert the pool transaction - let call_request = - TransactionRequest::from_recovered_transaction(tx.to_consensus()); - - let evm_config = node.evm_config.clone(); - - let result = eth_api - .spawn_with_call_at( - call_request, - BlockNumberOrTag::Latest.into(), - EvmOverrides::default(), - move |db, evm_env, tx_env| { - let mut dummy_inspector = DummyInspector::default(); - let mut evm = evm_config.evm_with_env_and_inspector( - db, - evm_env, - &mut dummy_inspector, - ); - // execute the transaction on a blocking task and await - // the - // inspector result - let _ = evm.transact(tx_env)?; - Ok(dummy_inspector) - }, - ) - .await; - - if let Ok(ret_val) = result { - let hash = tx.hash(); - println!( - "Inspector result for transaction {}: \n {}", - hash, - ret_val.ret_val.join("\n") - ); - } + if let Some(recipient) = tx.to() && + args.is_match(&recipient) + { + // convert the pool transaction + let call_request = + TransactionRequest::from_recovered_transaction(tx.to_consensus()); + + let evm_config = node.evm_config.clone(); + + let result = eth_api + .spawn_with_call_at( + call_request, + BlockNumberOrTag::Latest.into(), + EvmOverrides::default(), + move |db, evm_env, tx_env| { + let mut dummy_inspector = DummyInspector::default(); + let mut evm = evm_config.evm_with_env_and_inspector( + db, + evm_env, + &mut dummy_inspector, + ); + // execute the transaction on a blocking task and await + // the + // inspector result + let _ = evm.transact(tx_env)?; + Ok(dummy_inspector) + }, + ) + .await; + + if let Ok(ret_val) = result { + let hash = tx.hash(); + println!( + "Inspector result for transaction {}: \n {}", + hash, + ret_val.ret_val.join("\n") + ); } } } diff --git a/examples/custom-node-components/Cargo.toml b/examples/custom-node-components/Cargo.toml index 039abea201b..dd467c09ae2 100644 --- a/examples/custom-node-components/Cargo.toml +++ b/examples/custom-node-components/Cargo.toml @@ -6,7 +6,6 @@ edition.workspace = true license.workspace = true [dependencies] -reth.workspace = true reth-ethereum = { workspace = true, features = ["node", "pool", "node-api", "cli"] } reth-tracing.workspace = true diff --git a/examples/custom-node-components/src/main.rs b/examples/custom-node-components/src/main.rs index 1e5fa177f36..b6b8fb3cdf2 100644 --- a/examples/custom-node-components/src/main.rs +++ b/examples/custom-node-components/src/main.rs @@ -2,11 +2,15 @@ #![warn(unused_crate_dependencies)] -use reth::builder::{components::PoolBuilder, BuilderContext, FullNodeTypes}; use reth_ethereum::{ chainspec::ChainSpec, cli::interface::Cli, - node::{api::NodeTypes, node::EthereumAddOns, EthereumNode}, + node::{ + api::{FullNodeTypes, NodeTypes}, + builder::{components::PoolBuilder, BuilderContext}, + node::EthereumAddOns, + EthereumNode, + }, pool::{ blobstore::InMemoryBlobStore, EthTransactionPool, PoolConfig, TransactionValidationTaskExecutor, diff --git a/examples/custom-node/Cargo.toml b/examples/custom-node/Cargo.toml index 886f2509fe0..fe1f0006256 100644 --- a/examples/custom-node/Cargo.toml +++ b/examples/custom-node/Cargo.toml @@ -12,12 +12,14 @@ reth-codecs.workspace = true reth-network-peers.workspace = true reth-node-builder.workspace = true reth-optimism-forks.workspace = true -reth-optimism-consensus.workspace = true -reth-op = { workspace = true, features = ["node", "pool"] } +reth-optimism-flashblocks.workspace = true +reth-db-api.workspace = true +reth-op = { workspace = true, features = ["node", "pool", "rpc"] } reth-payload-builder.workspace = true reth-rpc-api.workspace = true +reth-engine-primitives.workspace = true reth-rpc-engine-api.workspace = true -reth-ethereum = { workspace = true, features = ["node-api", "network", "evm", "pool"] } +reth-ethereum = { workspace = true, features = ["node-api", "network", "evm", "pool", "trie", "storage-api", "provider"] } # revm revm.workspace = true @@ -32,9 +34,12 @@ alloy-op-evm.workspace = true alloy-primitives.workspace = true alloy-rlp.workspace = true alloy-serde.workspace = true +alloy-network.workspace = true alloy-rpc-types-engine.workspace = true +alloy-rpc-types-eth.workspace = true op-alloy-consensus.workspace = true op-alloy-rpc-types-engine.workspace = true +op-alloy-rpc-types.workspace = true op-revm.workspace = true # misc @@ -43,12 +48,9 @@ derive_more.workspace = true eyre.workspace = true jsonrpsee.workspace = true serde.workspace = true - +thiserror.workspace = true modular-bitfield.workspace = true -[dev-dependencies] -test-fuzz.workspace = true - [features] arbitrary = [ "alloy-consensus/arbitrary", @@ -63,5 +65,8 @@ arbitrary = [ "revm/arbitrary", "reth-ethereum/arbitrary", "alloy-rpc-types-engine/arbitrary", + "reth-db-api/arbitrary", + "alloy-rpc-types-eth/arbitrary", + "op-alloy-rpc-types/arbitrary", ] default = [] diff --git a/examples/custom-node/src/chainspec.rs b/examples/custom-node/src/chainspec.rs index 3ac6b51f149..4291b3549e4 100644 --- a/examples/custom-node/src/chainspec.rs +++ b/examples/custom-node/src/chainspec.rs @@ -14,6 +14,12 @@ pub struct CustomChainSpec { genesis_header: SealedHeader, } +impl CustomChainSpec { + pub const fn inner(&self) -> &OpChainSpec { + &self.inner + } +} + impl Hardforks for CustomChainSpec { fn fork(&self, fork: H) -> reth_ethereum::chainspec::ForkCondition { self.inner.fork(fork) @@ -44,15 +50,8 @@ impl Hardforks for CustomChainSpec { impl EthChainSpec for CustomChainSpec { type Header = CustomHeader; - fn base_fee_params_at_block( - &self, - block_number: u64, - ) -> reth_ethereum::chainspec::BaseFeeParams { - self.inner.base_fee_params_at_block(block_number) - } - - fn blob_params_at_timestamp(&self, timestamp: u64) -> Option { - self.inner.blob_params_at_timestamp(timestamp) + fn chain(&self) -> reth_ethereum::chainspec::Chain { + self.inner.chain() } fn base_fee_params_at_timestamp( @@ -62,38 +61,38 @@ impl EthChainSpec for CustomChainSpec { self.inner.base_fee_params_at_timestamp(timestamp) } - fn bootnodes(&self) -> Option> { - self.inner.bootnodes() - } - - fn chain(&self) -> reth_ethereum::chainspec::Chain { - self.inner.chain() + fn blob_params_at_timestamp(&self, timestamp: u64) -> Option { + self.inner.blob_params_at_timestamp(timestamp) } fn deposit_contract(&self) -> Option<&reth_ethereum::chainspec::DepositContract> { self.inner.deposit_contract() } - fn display_hardforks(&self) -> Box { - self.inner.display_hardforks() + fn genesis_hash(&self) -> revm_primitives::B256 { + self.genesis_header.hash() } fn prune_delete_limit(&self) -> usize { self.inner.prune_delete_limit() } - fn genesis(&self) -> &Genesis { - self.inner.genesis() - } - - fn genesis_hash(&self) -> revm_primitives::B256 { - self.genesis_header.hash() + fn display_hardforks(&self) -> Box { + self.inner.display_hardforks() } fn genesis_header(&self) -> &Self::Header { &self.genesis_header } + fn genesis(&self) -> &Genesis { + self.inner.genesis() + } + + fn bootnodes(&self) -> Option> { + self.inner.bootnodes() + } + fn final_paris_total_difficulty(&self) -> Option { self.inner.get_final_paris_total_difficulty() } diff --git a/examples/custom-node/src/consensus.rs b/examples/custom-node/src/consensus.rs deleted file mode 100644 index b9a8d2e1636..00000000000 --- a/examples/custom-node/src/consensus.rs +++ /dev/null @@ -1,27 +0,0 @@ -use std::sync::Arc; - -use reth_node_builder::{ - components::ConsensusBuilder, BuilderContext, FullNodeTypes, NodePrimitives, NodeTypes, -}; -use reth_op::DepositReceipt; -use reth_optimism_consensus::OpBeaconConsensus; -use reth_optimism_forks::OpHardforks; - -#[derive(Debug, Default, Clone)] -pub struct CustomConsensusBuilder; - -impl ConsensusBuilder for CustomConsensusBuilder -where - Node: FullNodeTypes< - Types: NodeTypes< - ChainSpec: OpHardforks, - Primitives: NodePrimitives, - >, - >, -{ - type Consensus = Arc::ChainSpec>>; - - async fn build_consensus(self, ctx: &BuilderContext) -> eyre::Result { - Ok(Arc::new(OpBeaconConsensus::new(ctx.chain_spec()))) - } -} diff --git a/examples/custom-node/src/engine.rs b/examples/custom-node/src/engine.rs index ab938be82d4..0c80e52a661 100644 --- a/examples/custom-node/src/engine.rs +++ b/examples/custom-node/src/engine.rs @@ -1,30 +1,48 @@ -use crate::primitives::CustomNodePrimitives; +use crate::{ + chainspec::CustomChainSpec, + evm::CustomEvmConfig, + primitives::{CustomHeader, CustomNodePrimitives, CustomTransaction}, + CustomNode, +}; +use alloy_eips::eip2718::WithEncoded; use op_alloy_rpc_types_engine::{OpExecutionData, OpExecutionPayload}; -use reth_chain_state::ExecutedBlockWithTrieUpdates; +use reth_chain_state::ExecutedBlock; +use reth_engine_primitives::EngineApiValidator; use reth_ethereum::{ node::api::{ - BuiltPayload, ExecutionPayload, NodePrimitives, PayloadAttributes, - PayloadBuilderAttributes, PayloadTypes, + validate_version_specific_fields, AddOnsContext, BuiltPayload, EngineApiMessageVersion, + EngineObjectValidationError, ExecutionPayload, FullNodeComponents, NewPayloadError, + NodePrimitives, PayloadAttributes, PayloadBuilderAttributes, PayloadOrAttributes, + PayloadTypes, PayloadValidator, }, - primitives::SealedBlock, + primitives::{RecoveredBlock, SealedBlock}, + storage::StateProviderFactory, + trie::{KeccakKeyHasher, KeyHasher}, }; -use reth_op::{ - node::{OpBuiltPayload, OpPayloadAttributes, OpPayloadBuilderAttributes}, - OpTransactionSigned, +use reth_node_builder::{rpc::PayloadValidatorBuilder, InvalidPayloadAttributesError}; +use reth_op::node::{ + engine::OpEngineValidator, payload::OpAttributes, OpBuiltPayload, OpEngineTypes, + OpPayloadAttributes, OpPayloadBuilderAttributes, }; use revm_primitives::U256; use serde::{Deserialize, Serialize}; +use std::sync::Arc; +use thiserror::Error; #[derive(Debug, Clone, Copy, Serialize, Deserialize)] pub struct CustomPayloadTypes; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct CustomExecutionData { - inner: OpExecutionData, - extension: u64, + pub inner: OpExecutionData, + pub extension: u64, } impl ExecutionPayload for CustomExecutionData { + fn parent_hash(&self) -> revm_primitives::B256 { + self.inner.parent_hash() + } + fn block_hash(&self) -> revm_primitives::B256 { self.inner.block_hash() } @@ -33,24 +51,20 @@ impl ExecutionPayload for CustomExecutionData { self.inner.block_number() } - fn parent_hash(&self) -> revm_primitives::B256 { - self.inner.parent_hash() + fn withdrawals(&self) -> Option<&Vec> { + None } - fn gas_used(&self) -> u64 { - self.inner.gas_used() + fn parent_beacon_block_root(&self) -> Option { + self.inner.parent_beacon_block_root() } fn timestamp(&self) -> u64 { self.inner.timestamp() } - fn parent_beacon_block_root(&self) -> Option { - self.inner.parent_beacon_block_root() - } - - fn withdrawals(&self) -> Option<&Vec> { - None + fn gas_used(&self) -> u64 { + self.inner.gas_used() } } @@ -62,10 +76,6 @@ pub struct CustomPayloadAttributes { } impl PayloadAttributes for CustomPayloadAttributes { - fn parent_beacon_block_root(&self) -> Option { - self.inner.parent_beacon_block_root() - } - fn timestamp(&self) -> u64 { self.inner.timestamp() } @@ -73,12 +83,16 @@ impl PayloadAttributes for CustomPayloadAttributes { fn withdrawals(&self) -> Option<&Vec> { self.inner.withdrawals() } + + fn parent_beacon_block_root(&self) -> Option { + self.inner.parent_beacon_block_root() + } } #[derive(Debug, Clone)] pub struct CustomPayloadBuilderAttributes { - inner: OpPayloadBuilderAttributes, - _extension: u64, + pub inner: OpPayloadBuilderAttributes, + pub extension: u64, } impl PayloadBuilderAttributes for CustomPayloadBuilderAttributes { @@ -95,38 +109,47 @@ impl PayloadBuilderAttributes for CustomPayloadBuilderAttributes { { let CustomPayloadAttributes { inner, extension } = rpc_payload_attributes; - Ok(Self { - inner: OpPayloadBuilderAttributes::try_new(parent, inner, version)?, - _extension: extension, - }) + Ok(Self { inner: OpPayloadBuilderAttributes::try_new(parent, inner, version)?, extension }) + } + + fn payload_id(&self) -> alloy_rpc_types_engine::PayloadId { + self.inner.payload_id() } fn parent(&self) -> revm_primitives::B256 { self.inner.parent() } + fn timestamp(&self) -> u64 { + self.inner.timestamp() + } + fn parent_beacon_block_root(&self) -> Option { self.inner.parent_beacon_block_root() } - fn payload_id(&self) -> alloy_rpc_types_engine::PayloadId { - self.inner.payload_id() + fn suggested_fee_recipient(&self) -> revm_primitives::Address { + self.inner.suggested_fee_recipient() } fn prev_randao(&self) -> revm_primitives::B256 { self.inner.prev_randao() } - fn suggested_fee_recipient(&self) -> revm_primitives::Address { - self.inner.suggested_fee_recipient() + fn withdrawals(&self) -> &alloy_eips::eip4895::Withdrawals { + self.inner.withdrawals() } +} - fn timestamp(&self) -> u64 { - self.inner.timestamp() +impl OpAttributes for CustomPayloadBuilderAttributes { + type Transaction = CustomTransaction; + + fn no_tx_pool(&self) -> bool { + self.inner.no_tx_pool } - fn withdrawals(&self) -> &alloy_eips::eip4895::Withdrawals { - self.inner.withdrawals() + fn sequencer_transactions(&self) -> &[WithEncoded] { + &self.inner.transactions } } @@ -140,14 +163,14 @@ impl BuiltPayload for CustomBuiltPayload { self.0.block() } - fn executed_block(&self) -> Option> { - self.0.executed_block() - } - fn fees(&self) -> U256 { self.0.fees() } + fn executed_block(&self) -> Option> { + self.0.executed_block() + } + fn requests(&self) -> Option { self.0.requests() } @@ -162,10 +185,10 @@ impl From } impl PayloadTypes for CustomPayloadTypes { - type BuiltPayload = CustomBuiltPayload; + type ExecutionData = CustomExecutionData; + type BuiltPayload = OpBuiltPayload; type PayloadAttributes = CustomPayloadAttributes; type PayloadBuilderAttributes = CustomPayloadBuilderAttributes; - type ExecutionData = CustomExecutionData; fn block_to_payload( block: SealedBlock< @@ -179,3 +202,118 @@ impl PayloadTypes for CustomPayloadTypes { CustomExecutionData { inner: OpExecutionData { payload, sidecar }, extension } } } + +/// Custom engine validator +#[derive(Debug, Clone)] +pub struct CustomEngineValidator

{ + inner: OpEngineValidator, +} + +impl

CustomEngineValidator

+where + P: Send + Sync + Unpin + 'static, +{ + /// Instantiates a new validator. + pub fn new(chain_spec: Arc, provider: P) -> Self { + Self { inner: OpEngineValidator::new::(chain_spec, provider) } + } + + /// Returns the chain spec used by the validator. + #[inline] + fn chain_spec(&self) -> &CustomChainSpec { + self.inner.chain_spec() + } +} + +impl

PayloadValidator for CustomEngineValidator

+where + P: StateProviderFactory + Send + Sync + Unpin + 'static, +{ + type Block = crate::primitives::block::Block; + + fn ensure_well_formed_payload( + &self, + payload: CustomExecutionData, + ) -> Result, NewPayloadError> { + let sealed_block = PayloadValidator::::ensure_well_formed_payload( + &self.inner, + payload.inner, + )?; + let (block, senders) = sealed_block.split_sealed(); + let (header, body) = block.split_sealed_header_body(); + let header = CustomHeader { inner: header.into_header(), extension: payload.extension }; + let body = body.map_ommers(|_| CustomHeader::default()); + let block = SealedBlock::::from_parts_unhashed(header, body); + + Ok(block.with_senders(senders)) + } + + fn validate_payload_attributes_against_header( + &self, + _attr: &CustomPayloadAttributes, + _header: &::Header, + ) -> Result<(), InvalidPayloadAttributesError> { + // skip default timestamp validation + Ok(()) + } +} + +impl

EngineApiValidator for CustomEngineValidator

+where + P: StateProviderFactory + Send + Sync + Unpin + 'static, +{ + fn validate_version_specific_fields( + &self, + version: EngineApiMessageVersion, + payload_or_attrs: PayloadOrAttributes<'_, CustomExecutionData, CustomPayloadAttributes>, + ) -> Result<(), EngineObjectValidationError> { + validate_version_specific_fields(self.chain_spec(), version, payload_or_attrs) + } + + fn ensure_well_formed_attributes( + &self, + version: EngineApiMessageVersion, + attributes: &CustomPayloadAttributes, + ) -> Result<(), EngineObjectValidationError> { + validate_version_specific_fields( + self.chain_spec(), + version, + PayloadOrAttributes::::PayloadAttributes(attributes), + )?; + + // custom validation logic - ensure that the custom field is not zero + // if attributes.extension == 0 { + // return Err(EngineObjectValidationError::invalid_params( + // CustomError::CustomFieldIsNotZero, + // )) + // } + + Ok(()) + } +} + +/// Custom error type used in payload attributes validation +#[derive(Debug, Error)] +pub enum CustomError { + #[error("Custom field is not zero")] + CustomFieldIsNotZero, +} + +/// Custom engine validator builder +#[derive(Debug, Default, Clone, Copy)] +#[non_exhaustive] +pub struct CustomEngineValidatorBuilder; + +impl PayloadValidatorBuilder for CustomEngineValidatorBuilder +where + N: FullNodeComponents, +{ + type Validator = CustomEngineValidator; + + async fn build(self, ctx: &AddOnsContext<'_, N>) -> eyre::Result { + Ok(CustomEngineValidator::new::( + ctx.config.chain.clone(), + ctx.node.provider().clone(), + )) + } +} diff --git a/examples/custom-node/src/engine_api.rs b/examples/custom-node/src/engine_api.rs index 69ff05171e4..1a947d8ec5d 100644 --- a/examples/custom-node/src/engine_api.rs +++ b/examples/custom-node/src/engine_api.rs @@ -1,9 +1,7 @@ use crate::{ - chainspec::CustomChainSpec, - engine::{ - CustomBuiltPayload, CustomExecutionData, CustomPayloadAttributes, CustomPayloadTypes, - }, + engine::{CustomExecutionData, CustomPayloadAttributes, CustomPayloadTypes}, primitives::CustomNodePrimitives, + CustomNode, }; use alloy_rpc_types_engine::{ ExecutionPayloadV3, ForkchoiceState, ForkchoiceUpdated, PayloadId, PayloadStatus, @@ -11,11 +9,10 @@ use alloy_rpc_types_engine::{ use async_trait::async_trait; use jsonrpsee::{core::RpcResult, proc_macros::rpc, RpcModule}; use reth_ethereum::node::api::{ - AddOnsContext, BeaconConsensusEngineHandle, EngineApiMessageVersion, FullNodeComponents, - NodeTypes, + AddOnsContext, ConsensusEngineHandle, EngineApiMessageVersion, FullNodeComponents, }; use reth_node_builder::rpc::EngineApiBuilder; -use reth_op::node::node::OpStorage; +use reth_op::node::OpBuiltPayload; use reth_payload_builder::PayloadStore; use reth_rpc_api::IntoEngineApiRpcModule; use reth_rpc_engine_api::EngineApiError; @@ -27,15 +24,20 @@ pub struct CustomExecutionPayloadInput {} #[derive(Clone, serde::Serialize)] pub struct CustomExecutionPayloadEnvelope { execution_payload: ExecutionPayloadV3, + extension: u64, } -impl From for CustomExecutionPayloadEnvelope { - fn from(value: CustomBuiltPayload) -> Self { - let sealed_block = value.0.into_sealed_block(); +impl From> for CustomExecutionPayloadEnvelope { + fn from(value: OpBuiltPayload) -> Self { + let sealed_block = value.into_sealed_block(); let hash = sealed_block.hash(); + let extension = sealed_block.header().extension; let block = sealed_block.into_block(); - Self { execution_payload: ExecutionPayloadV3::from_block_unchecked(hash, &block.clone()) } + Self { + execution_payload: ExecutionPayloadV3::from_block_unchecked(hash, &block), + extension, + } } } @@ -61,13 +63,13 @@ pub struct CustomEngineApi { } struct CustomEngineApiInner { - beacon_consensus: BeaconConsensusEngineHandle, + beacon_consensus: ConsensusEngineHandle, payload_store: PayloadStore, } impl CustomEngineApiInner { fn new( - beacon_consensus: BeaconConsensusEngineHandle, + beacon_consensus: ConsensusEngineHandle, payload_store: PayloadStore, ) -> Self { Self { beacon_consensus, payload_store } @@ -122,19 +124,12 @@ where } } -#[derive(Debug, Default)] +#[derive(Debug, Default, Clone)] pub struct CustomEngineApiBuilder {} impl EngineApiBuilder for CustomEngineApiBuilder where - N: FullNodeComponents< - Types: NodeTypes< - Payload = CustomPayloadTypes, - ChainSpec = CustomChainSpec, - Primitives = CustomNodePrimitives, - Storage = OpStorage, - >, - >, + N: FullNodeComponents, { type EngineApi = CustomEngineApi; diff --git a/examples/custom-node/src/evm.rs b/examples/custom-node/src/evm.rs deleted file mode 100644 index 83bb813590d..00000000000 --- a/examples/custom-node/src/evm.rs +++ /dev/null @@ -1,178 +0,0 @@ -use crate::chainspec::CustomChainSpec; -use alloy_consensus::{Block, Header}; -use alloy_evm::{ - block::{ - BlockExecutionError, BlockExecutionResult, BlockExecutor, BlockExecutorFactory, - BlockExecutorFor, ExecutableTx, OnStateHook, - }, - Database, Evm, EvmEnv, -}; -use alloy_op_evm::{OpBlockExecutionCtx, OpBlockExecutor, OpEvm}; -use op_revm::{OpSpecId, OpTransaction}; -use reth_ethereum::{ - evm::primitives::{ - execute::{BlockAssembler, BlockAssemblerInput}, - InspectorFor, - }, - node::api::ConfigureEvm, - primitives::{Receipt, SealedBlock, SealedHeader}, -}; -use reth_op::{ - chainspec::OpChainSpec, - node::{ - OpBlockAssembler, OpEvmConfig, OpEvmFactory, OpNextBlockEnvAttributes, OpRethReceiptBuilder, - }, - DepositReceipt, OpPrimitives, OpReceipt, OpTransactionSigned, -}; -use revm::{ - context::{result::ExecutionResult, TxEnv}, - database::State, -}; -use std::sync::Arc; - -pub struct CustomBlockExecutor { - inner: OpBlockExecutor>, -} - -impl<'db, DB, E> BlockExecutor for CustomBlockExecutor -where - DB: Database + 'db, - E: Evm, Tx = OpTransaction>, -{ - type Transaction = OpTransactionSigned; - type Receipt = OpReceipt; - type Evm = E; - - fn apply_pre_execution_changes(&mut self) -> Result<(), BlockExecutionError> { - self.inner.apply_pre_execution_changes() - } - - fn execute_transaction_with_result_closure( - &mut self, - tx: impl ExecutableTx, - f: impl FnOnce(&ExecutionResult<::HaltReason>), - ) -> Result { - self.inner.execute_transaction_with_result_closure(tx, f) - } - - fn finish(self) -> Result<(Self::Evm, BlockExecutionResult), BlockExecutionError> { - self.inner.finish() - } - - fn set_state_hook(&mut self, _hook: Option>) { - self.inner.set_state_hook(_hook) - } - - fn evm_mut(&mut self) -> &mut Self::Evm { - self.inner.evm_mut() - } - - fn evm(&self) -> &Self::Evm { - self.inner.evm() - } -} - -#[derive(Clone, Debug)] -pub struct CustomBlockAssembler { - inner: OpBlockAssembler, -} - -impl BlockAssembler for CustomBlockAssembler -where - F: for<'a> BlockExecutorFactory< - ExecutionCtx<'a> = OpBlockExecutionCtx, - Transaction = OpTransactionSigned, - Receipt: Receipt + DepositReceipt, - >, -{ - // TODO: use custom block here - type Block = Block; - - fn assemble_block( - &self, - input: BlockAssemblerInput<'_, '_, F>, - ) -> Result { - let block = self.inner.assemble_block(input)?; - - Ok(block) - } -} - -#[derive(Debug, Clone)] -pub struct CustomEvmConfig { - inner: OpEvmConfig, - block_assembler: CustomBlockAssembler, -} - -impl BlockExecutorFactory for CustomEvmConfig { - type EvmFactory = OpEvmFactory; - type ExecutionCtx<'a> = OpBlockExecutionCtx; - type Transaction = OpTransactionSigned; - type Receipt = OpReceipt; - - fn evm_factory(&self) -> &Self::EvmFactory { - self.inner.evm_factory() - } - - fn create_executor<'a, DB, I>( - &'a self, - evm: OpEvm<&'a mut State, I>, - ctx: OpBlockExecutionCtx, - ) -> impl BlockExecutorFor<'a, Self, DB, I> - where - DB: Database + 'a, - I: InspectorFor> + 'a, - { - CustomBlockExecutor { - inner: OpBlockExecutor::new( - evm, - ctx, - self.inner.chain_spec().clone(), - *self.inner.executor_factory.receipt_builder(), - ), - } - } -} - -impl ConfigureEvm for CustomEvmConfig { - type Primitives = OpPrimitives; - type Error = ::Error; - type NextBlockEnvCtx = ::NextBlockEnvCtx; - type BlockExecutorFactory = Self; - type BlockAssembler = CustomBlockAssembler; - - fn block_executor_factory(&self) -> &Self::BlockExecutorFactory { - self - } - - fn block_assembler(&self) -> &Self::BlockAssembler { - &self.block_assembler - } - - fn evm_env(&self, header: &Header) -> EvmEnv { - self.inner.evm_env(header) - } - - fn next_evm_env( - &self, - parent: &Header, - attributes: &OpNextBlockEnvAttributes, - ) -> Result, Self::Error> { - self.inner.next_evm_env(parent, attributes) - } - - fn context_for_block( - &self, - block: &SealedBlock>, - ) -> OpBlockExecutionCtx { - self.inner.context_for_block(block) - } - - fn context_for_next_block( - &self, - parent: &SealedHeader, - attributes: Self::NextBlockEnvCtx, - ) -> OpBlockExecutionCtx { - self.inner.context_for_next_block(parent, attributes) - } -} diff --git a/examples/custom-node/src/evm/alloy.rs b/examples/custom-node/src/evm/alloy.rs new file mode 100644 index 00000000000..d8df842cfc5 --- /dev/null +++ b/examples/custom-node/src/evm/alloy.rs @@ -0,0 +1,126 @@ +use crate::evm::{CustomTxEnv, PaymentTxEnv}; +use alloy_evm::{precompiles::PrecompilesMap, Database, Evm, EvmEnv, EvmFactory}; +use alloy_op_evm::{OpEvm, OpEvmFactory}; +use alloy_primitives::{Address, Bytes}; +use op_revm::{ + precompiles::OpPrecompiles, L1BlockInfo, OpContext, OpHaltReason, OpSpecId, OpTransaction, + OpTransactionError, +}; +use reth_ethereum::evm::revm::{ + context::{result::ResultAndState, BlockEnv, CfgEnv}, + handler::PrecompileProvider, + interpreter::InterpreterResult, + Context, Inspector, Journal, +}; +use revm::{context_interface::result::EVMError, inspector::NoOpInspector}; +use std::error::Error; + +/// EVM context contains data that EVM needs for execution of [`CustomTxEnv`]. +pub type CustomContext = + Context, CfgEnv, DB, Journal, L1BlockInfo>; + +pub struct CustomEvm { + inner: OpEvm, +} + +impl CustomEvm { + pub fn new(op: OpEvm) -> Self { + Self { inner: op } + } +} + +impl Evm for CustomEvm +where + DB: Database, + I: Inspector>, + P: PrecompileProvider, Output = InterpreterResult>, +{ + type DB = DB; + type Tx = CustomTxEnv; + type Error = EVMError; + type HaltReason = OpHaltReason; + type Spec = OpSpecId; + type BlockEnv = BlockEnv; + type Precompiles = P; + type Inspector = I; + + fn block(&self) -> &BlockEnv { + self.inner.block() + } + + fn chain_id(&self) -> u64 { + self.inner.chain_id() + } + + fn transact_raw( + &mut self, + tx: Self::Tx, + ) -> Result, Self::Error> { + match tx { + CustomTxEnv::Op(tx) => self.inner.transact_raw(tx), + CustomTxEnv::Payment(..) => todo!(), + } + } + + fn transact_system_call( + &mut self, + caller: Address, + contract: Address, + data: Bytes, + ) -> Result, Self::Error> { + self.inner.transact_system_call(caller, contract, data) + } + + fn finish(self) -> (Self::DB, EvmEnv) { + self.inner.finish() + } + + fn set_inspector_enabled(&mut self, enabled: bool) { + self.inner.set_inspector_enabled(enabled) + } + + fn components(&self) -> (&Self::DB, &Self::Inspector, &Self::Precompiles) { + self.inner.components() + } + + fn components_mut(&mut self) -> (&mut Self::DB, &mut Self::Inspector, &mut Self::Precompiles) { + self.inner.components_mut() + } +} + +#[derive(Default, Debug, Clone, Copy)] +pub struct CustomEvmFactory(pub OpEvmFactory); + +impl CustomEvmFactory { + pub fn new() -> Self { + Self::default() + } +} + +impl EvmFactory for CustomEvmFactory { + type Evm>> = CustomEvm; + type Context = OpContext; + type Tx = CustomTxEnv; + type Error = EVMError; + type HaltReason = OpHaltReason; + type Spec = OpSpecId; + type BlockEnv = BlockEnv; + type Precompiles = PrecompilesMap; + + fn create_evm( + &self, + db: DB, + input: EvmEnv, + ) -> Self::Evm { + CustomEvm::new(self.0.create_evm(db, input)) + } + + fn create_evm_with_inspector>>( + &self, + db: DB, + input: EvmEnv, + inspector: I, + ) -> Self::Evm { + CustomEvm::new(self.0.create_evm_with_inspector(db, input, inspector)) + } +} diff --git a/examples/custom-node/src/evm/assembler.rs b/examples/custom-node/src/evm/assembler.rs new file mode 100644 index 00000000000..dd1cfd3cb46 --- /dev/null +++ b/examples/custom-node/src/evm/assembler.rs @@ -0,0 +1,41 @@ +use crate::{ + chainspec::CustomChainSpec, + evm::executor::CustomBlockExecutionCtx, + primitives::{Block, CustomHeader, CustomTransaction}, +}; +use alloy_evm::block::{BlockExecutionError, BlockExecutorFactory}; +use reth_ethereum::{ + evm::primitives::execute::{BlockAssembler, BlockAssemblerInput}, + primitives::Receipt, +}; +use reth_op::{node::OpBlockAssembler, DepositReceipt}; +use std::sync::Arc; + +#[derive(Clone, Debug)] +pub struct CustomBlockAssembler { + block_assembler: OpBlockAssembler, +} + +impl CustomBlockAssembler { + pub const fn new(chain_spec: Arc) -> Self { + Self { block_assembler: OpBlockAssembler::new(chain_spec) } + } +} + +impl BlockAssembler for CustomBlockAssembler +where + F: for<'a> BlockExecutorFactory< + ExecutionCtx<'a> = CustomBlockExecutionCtx, + Transaction = CustomTransaction, + Receipt: Receipt + DepositReceipt, + >, +{ + type Block = Block; + + fn assemble_block( + &self, + input: BlockAssemblerInput<'_, '_, F, CustomHeader>, + ) -> Result { + Ok(self.block_assembler.assemble_block(input)?.map_header(From::from)) + } +} diff --git a/examples/custom-node/src/evm/builder.rs b/examples/custom-node/src/evm/builder.rs new file mode 100644 index 00000000000..fe7e7cf7113 --- /dev/null +++ b/examples/custom-node/src/evm/builder.rs @@ -0,0 +1,22 @@ +use crate::{chainspec::CustomChainSpec, evm::CustomEvmConfig, primitives::CustomNodePrimitives}; +use reth_ethereum::node::api::FullNodeTypes; +use reth_node_builder::{components::ExecutorBuilder, BuilderContext, NodeTypes}; +use std::{future, future::Future}; + +#[derive(Debug, Clone, Default)] +#[non_exhaustive] +pub struct CustomExecutorBuilder; + +impl ExecutorBuilder for CustomExecutorBuilder +where + Node::Types: NodeTypes, +{ + type EVM = CustomEvmConfig; + + fn build_evm( + self, + ctx: &BuilderContext, + ) -> impl Future> + Send { + future::ready(Ok(CustomEvmConfig::new(ctx.chain_spec()))) + } +} diff --git a/examples/custom-node/src/evm/config.rs b/examples/custom-node/src/evm/config.rs new file mode 100644 index 00000000000..a7dee31a835 --- /dev/null +++ b/examples/custom-node/src/evm/config.rs @@ -0,0 +1,177 @@ +use crate::{ + chainspec::CustomChainSpec, + engine::{CustomExecutionData, CustomPayloadBuilderAttributes}, + evm::{alloy::CustomEvmFactory, executor::CustomBlockExecutionCtx, CustomBlockAssembler}, + primitives::{Block, CustomHeader, CustomNodePrimitives, CustomTransaction}, +}; +use alloy_consensus::BlockHeader; +use alloy_eips::{eip2718::WithEncoded, Decodable2718}; +use alloy_evm::EvmEnv; +use alloy_op_evm::OpBlockExecutionCtx; +use alloy_rpc_types_engine::PayloadError; +use op_revm::OpSpecId; +use reth_engine_primitives::ExecutableTxIterator; +use reth_ethereum::{ + chainspec::EthChainSpec, + node::api::{BuildNextEnv, ConfigureEvm, PayloadBuilderError}, + primitives::{SealedBlock, SealedHeader}, +}; +use reth_node_builder::{ConfigureEngineEvm, NewPayloadError}; +use reth_op::{ + chainspec::OpHardforks, + evm::primitives::{EvmEnvFor, ExecutionCtxFor}, + node::{OpEvmConfig, OpNextBlockEnvAttributes, OpRethReceiptBuilder}, + primitives::SignedTransaction, +}; +use reth_optimism_flashblocks::ExecutionPayloadBaseV1; +use reth_rpc_api::eth::helpers::pending_block::BuildPendingEnv; +use std::sync::Arc; + +#[derive(Debug, Clone)] +pub struct CustomEvmConfig { + pub(super) inner: OpEvmConfig, + pub(super) block_assembler: CustomBlockAssembler, + pub(super) custom_evm_factory: CustomEvmFactory, +} + +impl CustomEvmConfig { + pub fn new(chain_spec: Arc) -> Self { + Self { + inner: OpEvmConfig::new( + Arc::new(chain_spec.inner().clone()), + OpRethReceiptBuilder::default(), + ), + block_assembler: CustomBlockAssembler::new(chain_spec), + custom_evm_factory: CustomEvmFactory::new(), + } + } +} + +impl ConfigureEvm for CustomEvmConfig { + type Primitives = CustomNodePrimitives; + type Error = ::Error; + type NextBlockEnvCtx = CustomNextBlockEnvAttributes; + type BlockExecutorFactory = Self; + type BlockAssembler = CustomBlockAssembler; + + fn block_executor_factory(&self) -> &Self::BlockExecutorFactory { + self + } + + fn block_assembler(&self) -> &Self::BlockAssembler { + &self.block_assembler + } + + fn evm_env(&self, header: &CustomHeader) -> Result, Self::Error> { + self.inner.evm_env(header) + } + + fn next_evm_env( + &self, + parent: &CustomHeader, + attributes: &CustomNextBlockEnvAttributes, + ) -> Result, Self::Error> { + self.inner.next_evm_env(parent, &attributes.inner) + } + + fn context_for_block( + &self, + block: &SealedBlock, + ) -> Result { + Ok(CustomBlockExecutionCtx { + inner: OpBlockExecutionCtx { + parent_hash: block.header().parent_hash(), + parent_beacon_block_root: block.header().parent_beacon_block_root(), + extra_data: block.header().extra_data().clone(), + }, + extension: block.extension, + }) + } + + fn context_for_next_block( + &self, + parent: &SealedHeader, + attributes: Self::NextBlockEnvCtx, + ) -> Result { + Ok(CustomBlockExecutionCtx { + inner: OpBlockExecutionCtx { + parent_hash: parent.hash(), + parent_beacon_block_root: attributes.inner.parent_beacon_block_root, + extra_data: attributes.inner.extra_data, + }, + extension: attributes.extension, + }) + } +} + +impl ConfigureEngineEvm for CustomEvmConfig { + fn evm_env_for_payload( + &self, + payload: &CustomExecutionData, + ) -> Result, Self::Error> { + self.inner.evm_env_for_payload(&payload.inner) + } + + fn context_for_payload<'a>( + &self, + payload: &'a CustomExecutionData, + ) -> Result, Self::Error> { + Ok(CustomBlockExecutionCtx { + inner: self.inner.context_for_payload(&payload.inner)?, + extension: payload.extension, + }) + } + + fn tx_iterator_for_payload( + &self, + payload: &CustomExecutionData, + ) -> Result, Self::Error> { + Ok(payload.inner.payload.transactions().clone().into_iter().map(|encoded| { + let tx = CustomTransaction::decode_2718_exact(encoded.as_ref()) + .map_err(Into::into) + .map_err(PayloadError::Decode)?; + let signer = tx.try_recover().map_err(NewPayloadError::other)?; + Ok::<_, NewPayloadError>(WithEncoded::new(encoded, tx.with_signer(signer))) + })) + } +} + +/// Additional parameters required for executing next block of custom transactions. +#[derive(Debug, Clone)] +pub struct CustomNextBlockEnvAttributes { + inner: OpNextBlockEnvAttributes, + extension: u64, +} + +impl From for CustomNextBlockEnvAttributes { + fn from(value: ExecutionPayloadBaseV1) -> Self { + Self { inner: value.into(), extension: 0 } + } +} + +impl BuildPendingEnv for CustomNextBlockEnvAttributes { + fn build_pending_env(parent: &SealedHeader) -> Self { + Self { + inner: OpNextBlockEnvAttributes::build_pending_env(parent), + extension: parent.extension, + } + } +} + +impl BuildNextEnv + for CustomNextBlockEnvAttributes +where + H: BlockHeader, + ChainSpec: EthChainSpec + OpHardforks, +{ + fn build_next_env( + attributes: &CustomPayloadBuilderAttributes, + parent: &SealedHeader, + chain_spec: &ChainSpec, + ) -> Result { + let inner = + OpNextBlockEnvAttributes::build_next_env(&attributes.inner, parent, chain_spec)?; + + Ok(CustomNextBlockEnvAttributes { inner, extension: attributes.extension }) + } +} diff --git a/examples/custom-node/src/evm/env.rs b/examples/custom-node/src/evm/env.rs new file mode 100644 index 00000000000..53a2b4e3f15 --- /dev/null +++ b/examples/custom-node/src/evm/env.rs @@ -0,0 +1,340 @@ +use crate::primitives::{CustomTransaction, TxPayment}; +use alloy_eips::{eip2930::AccessList, Typed2718}; +use alloy_evm::{FromRecoveredTx, FromTxWithEncoded, IntoTxEnv}; +use alloy_op_evm::block::OpTxEnv; +use alloy_primitives::{Address, Bytes, TxKind, B256, U256}; +use op_alloy_consensus::OpTxEnvelope; +use op_revm::OpTransaction; +use reth_ethereum::evm::{primitives::TransactionEnv, revm::context::TxEnv}; + +/// An Optimism transaction extended by [`PaymentTxEnv`] that can be fed to [`Evm`]. +/// +/// [`Evm`]: alloy_evm::Evm +#[derive(Clone, Debug)] +pub enum CustomTxEnv { + Op(OpTransaction), + Payment(PaymentTxEnv), +} + +/// A transaction environment is a set of information related to an Ethereum transaction that can be +/// fed to [`Evm`] for execution. +/// +/// [`Evm`]: alloy_evm::Evm +#[derive(Clone, Debug, Default)] +pub struct PaymentTxEnv(pub TxEnv); + +impl revm::context::Transaction for CustomTxEnv { + type AccessListItem<'a> + = ::AccessListItem<'a> + where + Self: 'a; + type Authorization<'a> + = ::Authorization<'a> + where + Self: 'a; + + fn tx_type(&self) -> u8 { + match self { + Self::Op(tx) => tx.tx_type(), + Self::Payment(tx) => tx.tx_type(), + } + } + + fn caller(&self) -> Address { + match self { + Self::Op(tx) => tx.caller(), + Self::Payment(tx) => tx.caller(), + } + } + + fn gas_limit(&self) -> u64 { + match self { + Self::Op(tx) => tx.gas_limit(), + Self::Payment(tx) => tx.gas_limit(), + } + } + + fn value(&self) -> U256 { + match self { + Self::Op(tx) => tx.value(), + Self::Payment(tx) => tx.value(), + } + } + + fn input(&self) -> &Bytes { + match self { + Self::Op(tx) => tx.input(), + Self::Payment(tx) => tx.input(), + } + } + + fn nonce(&self) -> u64 { + match self { + Self::Op(tx) => revm::context::Transaction::nonce(tx), + Self::Payment(tx) => revm::context::Transaction::nonce(tx), + } + } + + fn kind(&self) -> TxKind { + match self { + Self::Op(tx) => tx.kind(), + Self::Payment(tx) => tx.kind(), + } + } + + fn chain_id(&self) -> Option { + match self { + Self::Op(tx) => tx.chain_id(), + Self::Payment(tx) => tx.chain_id(), + } + } + + fn gas_price(&self) -> u128 { + match self { + Self::Op(tx) => tx.gas_price(), + Self::Payment(tx) => tx.gas_price(), + } + } + + fn access_list(&self) -> Option>> { + Some(match self { + Self::Op(tx) => tx.base.access_list.iter(), + Self::Payment(tx) => tx.0.access_list.iter(), + }) + } + + fn blob_versioned_hashes(&self) -> &[B256] { + match self { + Self::Op(tx) => tx.blob_versioned_hashes(), + Self::Payment(tx) => tx.blob_versioned_hashes(), + } + } + + fn max_fee_per_blob_gas(&self) -> u128 { + match self { + Self::Op(tx) => tx.max_fee_per_blob_gas(), + Self::Payment(tx) => tx.max_fee_per_blob_gas(), + } + } + + fn authorization_list_len(&self) -> usize { + match self { + Self::Op(tx) => tx.authorization_list_len(), + Self::Payment(tx) => tx.authorization_list_len(), + } + } + + fn authorization_list(&self) -> impl Iterator> { + match self { + Self::Op(tx) => tx.base.authorization_list.iter(), + Self::Payment(tx) => tx.0.authorization_list.iter(), + } + } + + fn max_priority_fee_per_gas(&self) -> Option { + match self { + Self::Op(tx) => tx.max_priority_fee_per_gas(), + Self::Payment(tx) => tx.max_priority_fee_per_gas(), + } + } +} + +impl revm::context::Transaction for PaymentTxEnv { + type AccessListItem<'a> + = ::AccessListItem<'a> + where + Self: 'a; + type Authorization<'a> + = ::Authorization<'a> + where + Self: 'a; + + fn tx_type(&self) -> u8 { + self.0.tx_type() + } + + fn caller(&self) -> Address { + self.0.caller() + } + + fn gas_limit(&self) -> u64 { + self.0.gas_limit() + } + + fn value(&self) -> U256 { + self.0.value() + } + + fn input(&self) -> &Bytes { + self.0.input() + } + + fn nonce(&self) -> u64 { + revm::context::Transaction::nonce(&self.0) + } + + fn kind(&self) -> TxKind { + self.0.kind() + } + + fn chain_id(&self) -> Option { + self.0.chain_id() + } + + fn gas_price(&self) -> u128 { + self.0.gas_price() + } + + fn access_list(&self) -> Option>> { + self.0.access_list() + } + + fn blob_versioned_hashes(&self) -> &[B256] { + self.0.blob_versioned_hashes() + } + + fn max_fee_per_blob_gas(&self) -> u128 { + self.0.max_fee_per_blob_gas() + } + + fn authorization_list_len(&self) -> usize { + self.0.authorization_list_len() + } + + fn authorization_list(&self) -> impl Iterator> { + self.0.authorization_list() + } + + fn max_priority_fee_per_gas(&self) -> Option { + self.0.max_priority_fee_per_gas() + } +} + +impl TransactionEnv for PaymentTxEnv { + fn set_gas_limit(&mut self, gas_limit: u64) { + self.0.set_gas_limit(gas_limit); + } + + fn nonce(&self) -> u64 { + self.0.nonce() + } + + fn set_nonce(&mut self, nonce: u64) { + self.0.set_nonce(nonce); + } + + fn set_access_list(&mut self, access_list: AccessList) { + self.0.set_access_list(access_list); + } +} + +impl TransactionEnv for CustomTxEnv { + fn set_gas_limit(&mut self, gas_limit: u64) { + match self { + Self::Op(tx) => tx.set_gas_limit(gas_limit), + Self::Payment(tx) => tx.set_gas_limit(gas_limit), + } + } + + fn nonce(&self) -> u64 { + match self { + Self::Op(tx) => tx.nonce(), + Self::Payment(tx) => tx.nonce(), + } + } + + fn set_nonce(&mut self, nonce: u64) { + match self { + Self::Op(tx) => tx.set_nonce(nonce), + Self::Payment(tx) => tx.set_nonce(nonce), + } + } + + fn set_access_list(&mut self, access_list: AccessList) { + match self { + Self::Op(tx) => tx.set_access_list(access_list), + Self::Payment(tx) => tx.set_access_list(access_list), + } + } +} + +impl FromRecoveredTx for TxEnv { + fn from_recovered_tx(tx: &TxPayment, caller: Address) -> Self { + let TxPayment { + chain_id, + nonce, + gas_limit, + max_fee_per_gas, + max_priority_fee_per_gas, + to, + value, + } = tx; + Self { + tx_type: tx.ty(), + caller, + gas_limit: *gas_limit, + gas_price: *max_fee_per_gas, + gas_priority_fee: Some(*max_priority_fee_per_gas), + kind: TxKind::Call(*to), + value: *value, + nonce: *nonce, + chain_id: Some(*chain_id), + ..Default::default() + } + } +} + +impl FromTxWithEncoded for TxEnv { + fn from_encoded_tx(tx: &TxPayment, sender: Address, _encoded: Bytes) -> Self { + Self::from_recovered_tx(tx, sender) + } +} + +impl FromRecoveredTx for CustomTxEnv { + fn from_recovered_tx(tx: &OpTxEnvelope, sender: Address) -> Self { + Self::Op(OpTransaction::from_recovered_tx(tx, sender)) + } +} + +impl FromTxWithEncoded for CustomTxEnv { + fn from_encoded_tx(tx: &OpTxEnvelope, sender: Address, encoded: Bytes) -> Self { + Self::Op(OpTransaction::from_encoded_tx(tx, sender, encoded)) + } +} + +impl FromRecoveredTx for CustomTxEnv { + fn from_recovered_tx(tx: &CustomTransaction, sender: Address) -> Self { + match tx { + CustomTransaction::Op(tx) => Self::from_recovered_tx(tx, sender), + CustomTransaction::Payment(tx) => { + Self::Payment(PaymentTxEnv(TxEnv::from_recovered_tx(tx.tx(), sender))) + } + } + } +} + +impl FromTxWithEncoded for CustomTxEnv { + fn from_encoded_tx(tx: &CustomTransaction, sender: Address, encoded: Bytes) -> Self { + match tx { + CustomTransaction::Op(tx) => Self::from_encoded_tx(tx, sender, encoded), + CustomTransaction::Payment(tx) => { + Self::Payment(PaymentTxEnv(TxEnv::from_encoded_tx(tx.tx(), sender, encoded))) + } + } + } +} + +impl IntoTxEnv for CustomTxEnv { + fn into_tx_env(self) -> Self { + self + } +} + +impl OpTxEnv for CustomTxEnv { + fn encoded_bytes(&self) -> Option<&Bytes> { + match self { + Self::Op(tx) => tx.encoded_bytes(), + Self::Payment(_) => None, + } + } +} diff --git a/examples/custom-node/src/evm/executor.rs b/examples/custom-node/src/evm/executor.rs new file mode 100644 index 00000000000..5288e1d67a5 --- /dev/null +++ b/examples/custom-node/src/evm/executor.rs @@ -0,0 +1,123 @@ +use crate::{ + evm::{ + alloy::{CustomEvm, CustomEvmFactory}, + CustomEvmConfig, CustomTxEnv, + }, + primitives::CustomTransaction, +}; +use alloy_consensus::transaction::Recovered; +use alloy_evm::{ + block::{ + BlockExecutionError, BlockExecutionResult, BlockExecutor, BlockExecutorFactory, + BlockExecutorFor, ExecutableTx, OnStateHook, + }, + precompiles::PrecompilesMap, + Database, Evm, +}; +use alloy_op_evm::{OpBlockExecutionCtx, OpBlockExecutor}; +use reth_ethereum::evm::primitives::InspectorFor; +use reth_op::{chainspec::OpChainSpec, node::OpRethReceiptBuilder, OpReceipt}; +use revm::{context::result::ResultAndState, database::State}; +use std::sync::Arc; + +pub struct CustomBlockExecutor { + inner: OpBlockExecutor>, +} + +impl<'db, DB, E> BlockExecutor for CustomBlockExecutor +where + DB: Database + 'db, + E: Evm, Tx = CustomTxEnv>, +{ + type Transaction = CustomTransaction; + type Receipt = OpReceipt; + type Evm = E; + + fn apply_pre_execution_changes(&mut self) -> Result<(), BlockExecutionError> { + self.inner.apply_pre_execution_changes() + } + + fn execute_transaction_without_commit( + &mut self, + tx: impl ExecutableTx, + ) -> Result::HaltReason>, BlockExecutionError> { + match tx.tx() { + CustomTransaction::Op(op_tx) => self + .inner + .execute_transaction_without_commit(Recovered::new_unchecked(op_tx, *tx.signer())), + CustomTransaction::Payment(..) => todo!(), + } + } + + fn commit_transaction( + &mut self, + output: ResultAndState<::HaltReason>, + tx: impl ExecutableTx, + ) -> Result { + match tx.tx() { + CustomTransaction::Op(op_tx) => { + self.inner.commit_transaction(output, Recovered::new_unchecked(op_tx, *tx.signer())) + } + CustomTransaction::Payment(..) => todo!(), + } + } + + fn finish(self) -> Result<(Self::Evm, BlockExecutionResult), BlockExecutionError> { + self.inner.finish() + } + + fn set_state_hook(&mut self, _hook: Option>) { + self.inner.set_state_hook(_hook) + } + + fn evm_mut(&mut self) -> &mut Self::Evm { + self.inner.evm_mut() + } + + fn evm(&self) -> &Self::Evm { + self.inner.evm() + } +} + +impl BlockExecutorFactory for CustomEvmConfig { + type EvmFactory = CustomEvmFactory; + type ExecutionCtx<'a> = CustomBlockExecutionCtx; + type Transaction = CustomTransaction; + type Receipt = OpReceipt; + + fn evm_factory(&self) -> &Self::EvmFactory { + &self.custom_evm_factory + } + + fn create_executor<'a, DB, I>( + &'a self, + evm: CustomEvm<&'a mut State, I, PrecompilesMap>, + ctx: CustomBlockExecutionCtx, + ) -> impl BlockExecutorFor<'a, Self, DB, I> + where + DB: Database + 'a, + I: InspectorFor> + 'a, + { + CustomBlockExecutor { + inner: OpBlockExecutor::new( + evm, + ctx.inner, + self.inner.chain_spec().clone(), + *self.inner.executor_factory.receipt_builder(), + ), + } + } +} + +/// Additional parameters for executing custom transactions. +#[derive(Debug, Clone)] +pub struct CustomBlockExecutionCtx { + pub inner: OpBlockExecutionCtx, + pub extension: u64, +} + +impl From for OpBlockExecutionCtx { + fn from(value: CustomBlockExecutionCtx) -> Self { + value.inner + } +} diff --git a/examples/custom-node/src/evm/mod.rs b/examples/custom-node/src/evm/mod.rs new file mode 100644 index 00000000000..7e4ac45c325 --- /dev/null +++ b/examples/custom-node/src/evm/mod.rs @@ -0,0 +1,13 @@ +mod alloy; +mod assembler; +mod builder; +mod config; +mod env; +mod executor; + +pub use alloy::{CustomContext, CustomEvm}; +pub use assembler::CustomBlockAssembler; +pub use builder::CustomExecutorBuilder; +pub use config::CustomEvmConfig; +pub use env::{CustomTxEnv, PaymentTxEnv}; +pub use executor::CustomBlockExecutor; diff --git a/examples/custom-node/src/lib.rs b/examples/custom-node/src/lib.rs index e6a2eab8612..4210ac9b767 100644 --- a/examples/custom-node/src/lib.rs +++ b/examples/custom-node/src/lib.rs @@ -1,4 +1,4 @@ -//! This example shows how implement a custom node. +//! This example shows how to implement a custom node. //! //! A node consists of: //! - primitives: block,header,transactions @@ -7,59 +7,80 @@ #![cfg_attr(not(test), warn(unused_crate_dependencies))] +use crate::{ + engine::{CustomEngineValidatorBuilder, CustomPayloadTypes}, + engine_api::CustomEngineApiBuilder, + evm::CustomExecutorBuilder, + pool::CustomPooledTransaction, + primitives::CustomTransaction, + rpc::CustomRpcTypes, +}; use chainspec::CustomChainSpec; -use consensus::CustomConsensusBuilder; -use engine::CustomPayloadTypes; -use pool::CustomPoolBuilder; use primitives::CustomNodePrimitives; use reth_ethereum::node::api::{FullNodeTypes, NodeTypes}; -use reth_node_builder::{components::ComponentsBuilder, Node, NodeComponentsBuilder}; -use reth_op::node::{node::OpStorage, OpNode}; +use reth_node_builder::{ + components::{BasicPayloadServiceBuilder, ComponentsBuilder}, + Node, NodeAdapter, +}; +use reth_op::{ + node::{ + node::{OpConsensusBuilder, OpNetworkBuilder, OpPayloadBuilder, OpPoolBuilder}, + txpool, OpAddOns, OpNode, + }, + rpc::OpEthApiBuilder, +}; pub mod chainspec; -pub mod consensus; pub mod engine; pub mod engine_api; pub mod evm; -pub mod network; pub mod pool; pub mod primitives; +pub mod rpc; #[derive(Debug, Clone)] -pub struct CustomNode {} +pub struct CustomNode { + inner: OpNode, +} impl NodeTypes for CustomNode { type Primitives = CustomNodePrimitives; type ChainSpec = CustomChainSpec; - type StateCommitment = ::StateCommitment; type Storage = ::Storage; type Payload = CustomPayloadTypes; } impl Node for CustomNode where - N: FullNodeTypes< - Types: NodeTypes< - Payload = CustomPayloadTypes, - ChainSpec = CustomChainSpec, - Primitives = CustomNodePrimitives, - Storage = OpStorage, - >, - >, - ComponentsBuilder: - NodeComponentsBuilder, + N: FullNodeTypes, { - type ComponentsBuilder = - ComponentsBuilder; + type ComponentsBuilder = ComponentsBuilder< + N, + OpPoolBuilder>, + BasicPayloadServiceBuilder, + OpNetworkBuilder, + CustomExecutorBuilder, + OpConsensusBuilder, + >; - type AddOns = (); + type AddOns = OpAddOns< + NodeAdapter, + OpEthApiBuilder, + CustomEngineValidatorBuilder, + CustomEngineApiBuilder, + >; fn components_builder(&self) -> Self::ComponentsBuilder { ComponentsBuilder::default() .node_types::() - .pool(CustomPoolBuilder::default()) - .consensus(CustomConsensusBuilder) + .pool(OpPoolBuilder::default()) + .executor(CustomExecutorBuilder::default()) + .payload(BasicPayloadServiceBuilder::new(OpPayloadBuilder::new(false))) + .network(OpNetworkBuilder::new(false, false)) + .consensus(OpConsensusBuilder::default()) } - fn add_ons(&self) -> Self::AddOns {} + fn add_ons(&self) -> Self::AddOns { + self.inner.add_ons_builder().build() + } } diff --git a/examples/custom-node/src/network.rs b/examples/custom-node/src/network.rs deleted file mode 100644 index 0a58c338aa5..00000000000 --- a/examples/custom-node/src/network.rs +++ /dev/null @@ -1,96 +0,0 @@ -use crate::{ - chainspec::CustomChainSpec, - primitives::{ - CustomHeader, CustomNodePrimitives, CustomTransactionEnvelope, ExtendedOpTxEnvelope, - }, -}; -use alloy_consensus::{Block, BlockBody}; -use eyre::Result; -use op_alloy_consensus::OpPooledTransaction; -use reth_ethereum::{ - chainspec::{EthChainSpec, Hardforks}, - network::{NetworkConfig, NetworkHandle, NetworkManager, NetworkPrimitives}, - node::api::{FullNodeTypes, NodeTypes, TxTy}, - pool::{PoolTransaction, TransactionPool}, -}; -use reth_node_builder::{components::NetworkBuilder, BuilderContext}; -use reth_op::{primitives::ExtendedTxEnvelope, OpReceipt}; - -#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Hash)] -#[non_exhaustive] -pub struct CustomNetworkPrimitives; - -impl NetworkPrimitives for CustomNetworkPrimitives { - type BlockHeader = CustomHeader; - type BlockBody = BlockBody, CustomHeader>; - type Block = Block, CustomHeader>; - type BroadcastedTransaction = ExtendedOpTxEnvelope; - type PooledTransaction = ExtendedTxEnvelope; - type Receipt = OpReceipt; -} - -#[derive(Default)] -pub struct CustomNetworkBuilder {} - -impl CustomNetworkBuilder { - fn network_config( - &self, - ctx: &BuilderContext, - ) -> eyre::Result::Provider, CustomNetworkPrimitives>> - where - Node: FullNodeTypes>, - { - let args = &ctx.config().network; - let network_builder = ctx - .network_config_builder()? - // apply discovery settings - .apply(|mut builder| { - let rlpx_socket = (args.addr, args.port).into(); - if args.discovery.disable_discovery { - builder = builder.disable_discv4_discovery(); - } - if !args.discovery.disable_discovery { - builder = builder.discovery_v5( - args.discovery.discovery_v5_builder( - rlpx_socket, - ctx.config() - .network - .resolved_bootnodes() - .or_else(|| ctx.chain_spec().bootnodes()) - .unwrap_or_default(), - ), - ); - } - - builder - }); - - let network_config = ctx.build_network_config(network_builder); - - Ok(network_config) - } -} - -impl NetworkBuilder for CustomNetworkBuilder -where - Node: FullNodeTypes< - Types: NodeTypes, - >, - Pool: TransactionPool< - Transaction: PoolTransaction< - Consensus = TxTy, - Pooled = ExtendedTxEnvelope, - >, - > + Unpin - + 'static, -{ - type Network = NetworkHandle; - - async fn build_network(self, ctx: &BuilderContext, pool: Pool) -> Result { - let network_config = self.network_config(ctx)?; - let network = NetworkManager::builder(network_config).await?; - let handle = ctx.start_network(network, pool); - - Ok(handle) - } -} diff --git a/examples/custom-node/src/pool.rs b/examples/custom-node/src/pool.rs index 5b024089d32..8828803a0f3 100644 --- a/examples/custom-node/src/pool.rs +++ b/examples/custom-node/src/pool.rs @@ -1,214 +1,93 @@ -// use jsonrpsee::tracing::{debug, info}; -use crate::primitives::CustomTransactionEnvelope; -use op_alloy_consensus::{interop::SafetyLevel, OpTxEnvelope}; -use reth_chain_state::CanonStateSubscriptions; -use reth_node_builder::{ - components::{PoolBuilder, PoolBuilderConfigOverrides}, - node::{FullNodeTypes, NodeTypes}, - BuilderContext, NodePrimitives, +use crate::primitives::{CustomTransaction, TxPayment}; +use alloy_consensus::{ + crypto::RecoveryError, + error::ValueError, + transaction::{SignerRecoverable, TxHashRef}, + Signed, TransactionEnvelope, }; -use reth_op::{ - node::txpool::{ - conditional::MaybeConditionalTransaction, - interop::MaybeInteropTransaction, - supervisor::{SupervisorClient, DEFAULT_SUPERVISOR_URL}, - OpPooledTransaction, OpTransactionPool, OpTransactionValidator, - }, - pool::{ - blobstore::DiskFileBlobStore, CoinbaseTipOrdering, EthPoolTransaction, - TransactionValidationTaskExecutor, - }, - primitives::ExtendedTxEnvelope, +use alloy_primitives::{Address, Sealed, B256}; +use op_alloy_consensus::{OpPooledTransaction, OpTransaction, TxDeposit}; +use reth_ethereum::primitives::{ + serde_bincode_compat::RlpBincode, InMemorySize, SignedTransaction, }; -use reth_optimism_forks::OpHardforks; -#[derive(Debug, Clone)] -pub struct CustomPoolBuilder< - T = OpPooledTransaction>, -> { - /// Enforced overrides that are applied to the pool config. - pub pool_config_overrides: PoolBuilderConfigOverrides, - /// Enable transaction conditionals. - pub enable_tx_conditional: bool, - /// Supervisor client url - pub supervisor_http: String, - /// Supervisor safety level - pub supervisor_safety_level: SafetyLevel, - /// Marker for the pooled transaction type. - _pd: core::marker::PhantomData, +#[derive(Clone, Debug, TransactionEnvelope)] +#[envelope(tx_type_name = CustomPooledTxType)] +pub enum CustomPooledTransaction { + /// A regular Optimism transaction as defined by [`OpPooledTransaction`]. + #[envelope(flatten)] + Op(OpPooledTransaction), + /// A [`TxPayment`] tagged with type 0x2A (decimal 42). + #[envelope(ty = 42)] + Payment(Signed), } -impl Default for CustomPoolBuilder { - fn default() -> Self { - Self { - pool_config_overrides: Default::default(), - enable_tx_conditional: false, - supervisor_http: DEFAULT_SUPERVISOR_URL.to_string(), - supervisor_safety_level: SafetyLevel::CrossUnsafe, - _pd: Default::default(), +impl From for CustomTransaction { + fn from(tx: CustomPooledTransaction) -> Self { + match tx { + CustomPooledTransaction::Op(tx) => Self::Op(tx.into()), + CustomPooledTransaction::Payment(tx) => Self::Payment(tx), } } } -impl CustomPoolBuilder { - /// Sets the enable_tx_conditional flag on the pool builder. - pub const fn with_enable_tx_conditional(mut self, enable_tx_conditional: bool) -> Self { - self.enable_tx_conditional = enable_tx_conditional; - self +impl TryFrom for CustomPooledTransaction { + type Error = ValueError; + + fn try_from(tx: CustomTransaction) -> Result { + match tx { + CustomTransaction::Op(op) => Ok(Self::Op( + OpPooledTransaction::try_from(op).map_err(|op| op.map(CustomTransaction::Op))?, + )), + CustomTransaction::Payment(payment) => Ok(Self::Payment(payment)), + } } +} - /// Sets the [PoolBuilderConfigOverrides] on the pool builder. - pub fn with_pool_config_overrides( - mut self, - pool_config_overrides: PoolBuilderConfigOverrides, - ) -> Self { - self.pool_config_overrides = pool_config_overrides; - self +impl RlpBincode for CustomPooledTransaction {} + +impl OpTransaction for CustomPooledTransaction { + fn is_deposit(&self) -> bool { + false } - /// Sets the supervisor client - pub fn with_supervisor( - mut self, - supervisor_client: String, - supervisor_safety_level: SafetyLevel, - ) -> Self { - self.supervisor_http = supervisor_client; - self.supervisor_safety_level = supervisor_safety_level; - self + fn as_deposit(&self) -> Option<&Sealed> { + None } } -impl PoolBuilder for CustomPoolBuilder -where - Node: FullNodeTypes>, - ::Primitives: - NodePrimitives>, - T: EthPoolTransaction> - + MaybeConditionalTransaction - + MaybeInteropTransaction, -{ - type Pool = OpTransactionPool; - - async fn build_pool(self, ctx: &BuilderContext) -> eyre::Result { - let Self { pool_config_overrides, .. } = self; - let data_dir = ctx.config().datadir(); - let blob_store = DiskFileBlobStore::open(data_dir.blobstore(), Default::default())?; - // supervisor used for interop - if ctx.chain_spec().is_interop_active_at_timestamp(ctx.head().timestamp) && - self.supervisor_http == DEFAULT_SUPERVISOR_URL - { - // info!(target: "reth::cli", - // url=%DEFAULT_SUPERVISOR_URL, - // "Default supervisor url is used, consider changing --rollup.supervisor-http." - // ); +impl SignerRecoverable for CustomPooledTransaction { + fn recover_signer(&self) -> Result { + match self { + CustomPooledTransaction::Op(tx) => SignerRecoverable::recover_signer(tx), + CustomPooledTransaction::Payment(tx) => SignerRecoverable::recover_signer(tx), } - let supervisor_client = SupervisorClient::builder(self.supervisor_http.clone()) - .minimum_safety(self.supervisor_safety_level) - .build() - .await; - - let validator = TransactionValidationTaskExecutor::eth_builder(ctx.provider().clone()) - .no_eip4844() - .with_head_timestamp(ctx.head().timestamp) - .kzg_settings(ctx.kzg_settings()?) - .set_tx_fee_cap(ctx.config().rpc.rpc_tx_fee_cap) - .with_additional_tasks( - pool_config_overrides - .additional_validation_tasks - .unwrap_or_else(|| ctx.config().txpool.additional_validation_tasks), - ) - .build_with_tasks(ctx.task_executor().clone(), blob_store.clone()) - .map(|validator| { - OpTransactionValidator::new(validator) - // In --dev mode we can't require gas fees because we're unable to decode - // the L1 block info - .require_l1_data_gas_fee(!ctx.config().dev.dev) - .with_supervisor(supervisor_client.clone()) - }); - - let transaction_pool = reth_ethereum::pool::Pool::new( - validator, - CoinbaseTipOrdering::default(), - blob_store, - pool_config_overrides.apply(ctx.pool_config()), - ); - // info!(target: "reth::cli", "Transaction pool initialized";); - - // spawn txpool maintenance tasks - { - let pool = transaction_pool.clone(); - let chain_events = ctx.provider().canonical_state_stream(); - let client = ctx.provider().clone(); - if !ctx.config().txpool.disable_transactions_backup { - // Use configured backup path or default to data dir - let transactions_path = ctx - .config() - .txpool - .transactions_backup_path - .clone() - .unwrap_or_else(|| data_dir.txpool_transactions()); - - let transactions_backup_config = - reth_ethereum::pool::maintain::LocalTransactionBackupConfig::with_local_txs_backup(transactions_path); + } - ctx.task_executor().spawn_critical_with_graceful_shutdown_signal( - "local transactions backup task", - |shutdown| { - reth_ethereum::pool::maintain::backup_local_transactions_task( - shutdown, - pool.clone(), - transactions_backup_config, - ) - }, - ); - } + fn recover_signer_unchecked(&self) -> Result { + match self { + CustomPooledTransaction::Op(tx) => SignerRecoverable::recover_signer_unchecked(tx), + CustomPooledTransaction::Payment(tx) => SignerRecoverable::recover_signer_unchecked(tx), + } + } +} - // spawn the main maintenance task - ctx.task_executor().spawn_critical( - "txpool maintenance task", - reth_ethereum::pool::maintain::maintain_transaction_pool_future( - client, - pool.clone(), - chain_events, - ctx.task_executor().clone(), - reth_ethereum::pool::maintain::MaintainPoolConfig { - max_tx_lifetime: pool.config().max_queued_lifetime, - no_local_exemptions: transaction_pool - .config() - .local_transactions_config - .no_exemptions, - ..Default::default() - }, - ), - ); - // debug!(target: "reth::cli", "Spawned txpool maintenance task"); +impl TxHashRef for CustomPooledTransaction { + fn tx_hash(&self) -> &B256 { + match self { + CustomPooledTransaction::Op(tx) => tx.tx_hash(), + CustomPooledTransaction::Payment(tx) => tx.hash(), + } + } +} - // spawn the Op txpool maintenance task - let chain_events = ctx.provider().canonical_state_stream(); - ctx.task_executor().spawn_critical( - "Op txpool interop maintenance task", - reth_op::node::txpool::maintain::maintain_transaction_pool_interop_future( - pool.clone(), - chain_events, - supervisor_client, - ), - ); - // debug!(target: "reth::cli", "Spawned Op interop txpool maintenance task"); +impl SignedTransaction for CustomPooledTransaction {} - if self.enable_tx_conditional { - // spawn the Op txpool maintenance task - let chain_events = ctx.provider().canonical_state_stream(); - ctx.task_executor().spawn_critical( - "Op txpool conditional maintenance task", - reth_op::node::txpool::maintain::maintain_transaction_pool_conditional_future( - pool, - chain_events, - ), - ); - // debug!(target: "reth::cli", "Spawned Op conditional txpool maintenance task"); - } +impl InMemorySize for CustomPooledTransaction { + fn size(&self) -> usize { + match self { + CustomPooledTransaction::Op(tx) => InMemorySize::size(tx), + CustomPooledTransaction::Payment(tx) => InMemorySize::size(tx), } - - Ok(transaction_pool) } } diff --git a/examples/custom-node/src/primitives/block.rs b/examples/custom-node/src/primitives/block.rs index 3de2831c410..d8db04e245c 100644 --- a/examples/custom-node/src/primitives/block.rs +++ b/examples/custom-node/src/primitives/block.rs @@ -1,9 +1,7 @@ -use crate::primitives::{CustomHeader, CustomTransactionEnvelope, ExtendedOpTxEnvelope}; +use crate::primitives::{CustomHeader, CustomTransaction}; /// The Block type of this node -pub type Block = - alloy_consensus::Block, CustomHeader>; +pub type Block = alloy_consensus::Block; /// The body type of this node -pub type BlockBody = - alloy_consensus::BlockBody, CustomHeader>; +pub type BlockBody = alloy_consensus::BlockBody; diff --git a/examples/custom-node/src/primitives/header.rs b/examples/custom-node/src/primitives/header.rs index 7bdb4a8d73c..946bad51894 100644 --- a/examples/custom-node/src/primitives/header.rs +++ b/examples/custom-node/src/primitives/header.rs @@ -1,10 +1,8 @@ use alloy_consensus::Header; -use alloy_primitives::{ - private::derive_more, Address, BlockNumber, Bloom, Bytes, Sealable, B256, B64, U256, -}; +use alloy_primitives::{Address, BlockNumber, Bloom, Bytes, Sealable, B256, B64, U256}; use alloy_rlp::{Encodable, RlpDecodable, RlpEncodable}; use reth_codecs::Compact; -use reth_ethereum::primitives::{BlockHeader, InMemorySize}; +use reth_ethereum::primitives::{serde_bincode_compat::RlpBincode, BlockHeader, InMemorySize}; use revm_primitives::keccak256; use serde::{Deserialize, Serialize}; @@ -36,7 +34,11 @@ pub struct CustomHeader { pub extension: u64, } -impl CustomHeader {} +impl From

for CustomHeader { + fn from(value: Header) -> Self { + CustomHeader { inner: value, extension: 0 } + } +} impl AsRef for CustomHeader { fn as_ref(&self) -> &Self { @@ -162,34 +164,21 @@ impl reth_codecs::Compact for CustomHeader { } } -impl BlockHeader for CustomHeader {} +impl reth_db_api::table::Compress for CustomHeader { + type Compressed = Vec; -mod serde_bincode_compat { - use alloy_consensus::serde_bincode_compat::Header; - use reth_ethereum::primitives::serde_bincode_compat::SerdeBincodeCompat; - use serde::{Deserialize, Serialize}; - - #[derive(Serialize, Deserialize, Debug)] - pub struct CustomHeader<'a> { - inner: Header<'a>, - extension: u64, + fn compress_to_buf>(&self, buf: &mut B) { + let _ = Compact::to_compact(self, buf); } +} - impl From> for super::CustomHeader { - fn from(value: CustomHeader) -> Self { - Self { inner: value.inner.into(), extension: value.extension } - } +impl reth_db_api::table::Decompress for CustomHeader { + fn decompress(value: &[u8]) -> Result { + let (obj, _) = Compact::from_compact(value, value.len()); + Ok(obj) } +} - impl SerdeBincodeCompat for super::CustomHeader { - type BincodeRepr<'a> = CustomHeader<'a>; - - fn as_repr(&self) -> Self::BincodeRepr<'_> { - CustomHeader { inner: self.inner.as_repr(), extension: self.extension } - } +impl BlockHeader for CustomHeader {} - fn from_repr(repr: Self::BincodeRepr<'_>) -> Self { - repr.into() - } - } -} +impl RlpBincode for CustomHeader {} diff --git a/examples/custom-node/src/primitives/mod.rs b/examples/custom-node/src/primitives/mod.rs index dd9a2228a23..773ff4888cc 100644 --- a/examples/custom-node/src/primitives/mod.rs +++ b/examples/custom-node/src/primitives/mod.rs @@ -22,6 +22,6 @@ impl NodePrimitives for CustomNodePrimitives { type Block = Block; type BlockHeader = CustomHeader; type BlockBody = BlockBody; - type SignedTx = ExtendedOpTxEnvelope; + type SignedTx = CustomTransaction; type Receipt = OpReceipt; } diff --git a/examples/custom-node/src/primitives/tx.rs b/examples/custom-node/src/primitives/tx.rs index d862281bf69..fe763e079e5 100644 --- a/examples/custom-node/src/primitives/tx.rs +++ b/examples/custom-node/src/primitives/tx.rs @@ -1,226 +1,144 @@ -use super::{TxCustom, TxTypeCustom}; +use super::TxPayment; use alloy_consensus::{ - crypto::{ - secp256k1::{recover_signer, recover_signer_unchecked}, - RecoveryError, - }, - SignableTransaction, Signed, Transaction, + crypto::RecoveryError, + transaction::{SignerRecoverable, TxHashRef}, + Signed, TransactionEnvelope, }; -use alloy_eips::{eip2718::Eip2718Result, Decodable2718, Encodable2718, Typed2718}; -use alloy_primitives::{keccak256, Signature, TxHash}; -use alloy_rlp::{BufMut, Decodable, Encodable, Result as RlpResult}; -use op_alloy_consensus::OpTxEnvelope; +use alloy_eips::Encodable2718; +use alloy_primitives::{Sealed, Signature, B256}; +use alloy_rlp::BufMut; +use op_alloy_consensus::{OpTxEnvelope, TxDeposit}; use reth_codecs::{ - alloy::transaction::{FromTxCompact, ToTxCompact}, + alloy::transaction::{CompactEnvelope, FromTxCompact, ToTxCompact}, Compact, }; -use reth_ethereum::primitives::{serde_bincode_compat::SerdeBincodeCompat, InMemorySize}; -use reth_op::primitives::{ExtendedTxEnvelope, SignedTransaction}; -use revm_primitives::{Address, Bytes}; -use serde::{Deserialize, Serialize}; - -/// A [`SignedTransaction`] implementation that combines the [`OpTxEnvelope`] and another -/// transaction type. -pub type ExtendedOpTxEnvelope = ExtendedTxEnvelope; - -#[derive(Debug, Clone, Serialize, Deserialize, Hash, Eq, PartialEq)] -pub struct CustomTransactionEnvelope { - pub inner: Signed, +use reth_ethereum::primitives::{serde_bincode_compat::RlpBincode, InMemorySize}; +use reth_op::{primitives::SignedTransaction, OpTransaction}; +use revm_primitives::Address; + +/// Either [`OpTxEnvelope`] or [`TxPayment`]. +#[derive(Debug, Clone, TransactionEnvelope)] +#[envelope(tx_type_name = TxTypeCustom)] +pub enum CustomTransaction { + /// A regular Optimism transaction as defined by [`OpTxEnvelope`]. + #[envelope(flatten)] + Op(OpTxEnvelope), + /// A [`TxPayment`] tagged with type 0x2A (decimal 42). + #[envelope(ty = 42)] + Payment(Signed), } -impl Transaction for CustomTransactionEnvelope { - fn chain_id(&self) -> Option { - self.inner.tx().chain_id() - } - - fn nonce(&self) -> u64 { - self.inner.tx().nonce() - } - - fn gas_limit(&self) -> u64 { - self.inner.tx().gas_limit() - } - - fn gas_price(&self) -> Option { - self.inner.tx().gas_price() - } - - fn max_fee_per_gas(&self) -> u128 { - self.inner.tx().max_fee_per_gas() - } - - fn max_priority_fee_per_gas(&self) -> Option { - self.inner.tx().max_priority_fee_per_gas() - } - - fn max_fee_per_blob_gas(&self) -> Option { - self.inner.tx().max_fee_per_blob_gas() - } - - fn priority_fee_or_price(&self) -> u128 { - self.inner.tx().priority_fee_or_price() - } - - fn effective_gas_price(&self, base_fee: Option) -> u128 { - self.inner.tx().effective_gas_price(base_fee) - } - - fn is_dynamic_fee(&self) -> bool { - self.inner.tx().is_dynamic_fee() - } - - fn kind(&self) -> revm_primitives::TxKind { - self.inner.tx().kind() - } - - fn is_create(&self) -> bool { - false - } - - fn value(&self) -> revm_primitives::U256 { - self.inner.tx().value() - } - - fn input(&self) -> &Bytes { - // CustomTransactions have no input data - static EMPTY_BYTES: Bytes = Bytes::new(); - &EMPTY_BYTES - } - - fn access_list(&self) -> Option<&alloy_eips::eip2930::AccessList> { - self.inner.tx().access_list() - } - - fn blob_versioned_hashes(&self) -> Option<&[revm_primitives::B256]> { - self.inner.tx().blob_versioned_hashes() - } - - fn authorization_list(&self) -> Option<&[alloy_eips::eip7702::SignedAuthorization]> { - self.inner.tx().authorization_list() - } -} - -impl SignedTransaction for CustomTransactionEnvelope { - fn tx_hash(&self) -> &TxHash { - self.inner.hash() - } +impl RlpBincode for CustomTransaction {} - fn recover_signer(&self) -> Result { - let signature_hash = self.inner.signature_hash(); - recover_signer(self.inner.signature(), signature_hash) - } - - fn recover_signer_unchecked_with_buf( - &self, - buf: &mut Vec, - ) -> Result { - self.inner.tx().encode_for_signing(buf); - let signature_hash = keccak256(buf); - recover_signer_unchecked(self.inner.signature(), signature_hash) - } -} - -impl Typed2718 for CustomTransactionEnvelope { - fn ty(&self) -> u8 { - self.inner.tx().ty() - } -} - -impl Decodable2718 for CustomTransactionEnvelope { - fn typed_decode(ty: u8, buf: &mut &[u8]) -> Eip2718Result { - Ok(Self { inner: Signed::::typed_decode(ty, buf)? }) - } - - fn fallback_decode(buf: &mut &[u8]) -> Eip2718Result { - Ok(Self { inner: Signed::::fallback_decode(buf)? }) - } -} - -impl Encodable2718 for CustomTransactionEnvelope { - fn encode_2718_len(&self) -> usize { - self.inner.encode_2718_len() - } - - fn encode_2718(&self, out: &mut dyn BufMut) { - self.inner.encode_2718(out) - } -} - -impl Decodable for CustomTransactionEnvelope { - fn decode(buf: &mut &[u8]) -> RlpResult { - let inner = Signed::::decode_2718(buf)?; - Ok(CustomTransactionEnvelope { inner }) - } -} - -impl Encodable for CustomTransactionEnvelope { - fn encode(&self, out: &mut dyn BufMut) { - self.inner.tx().encode(out) +impl reth_codecs::alloy::transaction::Envelope for CustomTransaction { + fn signature(&self) -> &Signature { + match self { + CustomTransaction::Op(tx) => reth_codecs::alloy::transaction::Envelope::signature(tx), + CustomTransaction::Payment(tx) => tx.signature(), + } } -} -impl InMemorySize for CustomTransactionEnvelope { - fn size(&self) -> usize { - self.inner.tx().size() + fn tx_type(&self) -> Self::TxType { + match self { + CustomTransaction::Op(tx) => TxTypeCustom::Op(tx.tx_type()), + CustomTransaction::Payment(_) => TxTypeCustom::Payment, + } } } -impl FromTxCompact for CustomTransactionEnvelope { +impl FromTxCompact for CustomTransaction { type TxType = TxTypeCustom; - fn from_tx_compact(buf: &[u8], _tx_type: Self::TxType, signature: Signature) -> (Self, &[u8]) + fn from_tx_compact(buf: &[u8], tx_type: Self::TxType, signature: Signature) -> (Self, &[u8]) where Self: Sized, { - let (tx, buf) = TxCustom::from_compact(buf, buf.len()); - let tx = Signed::new_unhashed(tx, signature); - (CustomTransactionEnvelope { inner: tx }, buf) + match tx_type { + TxTypeCustom::Op(tx_type) => { + let (tx, buf) = OpTxEnvelope::from_tx_compact(buf, tx_type, signature); + (Self::Op(tx), buf) + } + TxTypeCustom::Payment => { + let (tx, buf) = TxPayment::from_compact(buf, buf.len()); + let tx = Signed::new_unhashed(tx, signature); + (Self::Payment(tx), buf) + } + } } } -impl ToTxCompact for CustomTransactionEnvelope { +impl ToTxCompact for CustomTransaction { fn to_tx_compact(&self, buf: &mut (impl BufMut + AsMut<[u8]>)) { - self.inner.tx().to_compact(buf); + match self { + CustomTransaction::Op(tx) => tx.to_tx_compact(buf), + CustomTransaction::Payment(tx) => { + tx.tx().to_compact(buf); + } + } } } -#[derive(Debug, Serialize, Deserialize)] -pub struct BincodeCompatSignedTxCustom(pub Signed); +impl Compact for CustomTransaction { + fn to_compact(&self, buf: &mut B) -> usize + where + B: BufMut + AsMut<[u8]>, + { + ::to_compact(self, buf) + } -impl SerdeBincodeCompat for CustomTransactionEnvelope { - type BincodeRepr<'a> = BincodeCompatSignedTxCustom; + fn from_compact(buf: &[u8], len: usize) -> (Self, &[u8]) { + ::from_compact(buf, len) + } +} - fn as_repr(&self) -> Self::BincodeRepr<'_> { - BincodeCompatSignedTxCustom(self.inner.clone()) +impl OpTransaction for CustomTransaction { + fn is_deposit(&self) -> bool { + match self { + CustomTransaction::Op(op) => op.is_deposit(), + CustomTransaction::Payment(_) => false, + } } - fn from_repr(repr: Self::BincodeRepr<'_>) -> Self { - Self { inner: repr.0.clone() } + fn as_deposit(&self) -> Option<&Sealed> { + match self { + CustomTransaction::Op(op) => op.as_deposit(), + CustomTransaction::Payment(_) => None, + } } } -impl reth_codecs::alloy::transaction::Envelope for CustomTransactionEnvelope { - fn signature(&self) -> &Signature { - self.inner.signature() +impl SignerRecoverable for CustomTransaction { + fn recover_signer(&self) -> Result { + match self { + CustomTransaction::Op(tx) => SignerRecoverable::recover_signer(tx), + CustomTransaction::Payment(tx) => SignerRecoverable::recover_signer(tx), + } } - fn tx_type(&self) -> Self::TxType { - TxTypeCustom::Custom + fn recover_signer_unchecked(&self) -> Result { + match self { + CustomTransaction::Op(tx) => SignerRecoverable::recover_signer_unchecked(tx), + CustomTransaction::Payment(tx) => SignerRecoverable::recover_signer_unchecked(tx), + } } } -impl Compact for CustomTransactionEnvelope { - fn to_compact(&self, buf: &mut B) -> usize - where - B: alloy_rlp::bytes::BufMut + AsMut<[u8]>, - { - self.inner.tx().to_compact(buf) +impl TxHashRef for CustomTransaction { + fn tx_hash(&self) -> &B256 { + match self { + CustomTransaction::Op(tx) => TxHashRef::tx_hash(tx), + CustomTransaction::Payment(tx) => tx.hash(), + } } +} - fn from_compact(buf: &[u8], len: usize) -> (Self, &[u8]) { - let (signature, rest) = Signature::from_compact(buf, len); - let (inner, buf) = ::from_compact(rest, len); - let signed = Signed::new_unhashed(inner, signature); - (CustomTransactionEnvelope { inner: signed }, buf) +impl SignedTransaction for CustomTransaction {} + +impl InMemorySize for CustomTransaction { + fn size(&self) -> usize { + match self { + CustomTransaction::Op(tx) => InMemorySize::size(tx), + CustomTransaction::Payment(tx) => InMemorySize::size(tx), + } } } diff --git a/examples/custom-node/src/primitives/tx_custom.rs b/examples/custom-node/src/primitives/tx_custom.rs index 7d6b4b103b4..8729378bd59 100644 --- a/examples/custom-node/src/primitives/tx_custom.rs +++ b/examples/custom-node/src/primitives/tx_custom.rs @@ -1,13 +1,13 @@ -use crate::primitives::{TxTypeCustom, TRANSFER_TX_TYPE_ID}; +use crate::primitives::PAYMENT_TX_TYPE_ID; use alloy_consensus::{ transaction::{RlpEcdsaDecodableTx, RlpEcdsaEncodableTx}, SignableTransaction, Transaction, }; use alloy_eips::{eip2930::AccessList, eip7702::SignedAuthorization, Typed2718}; -use alloy_primitives::{Bytes, ChainId, Signature, TxKind, B256, U256}; +use alloy_primitives::{Address, Bytes, ChainId, Signature, TxKind, B256, U256}; use alloy_rlp::{BufMut, Decodable, Encodable}; use core::mem; -use reth_ethereum::primitives::{serde_bincode_compat::SerdeBincodeCompat, InMemorySize}; +use reth_ethereum::primitives::{serde_bincode_compat::RlpBincode, InMemorySize}; /// A transaction with a priority fee ([EIP-1559](https://eips.ethereum.org/EIPS/eip-1559)). #[derive( @@ -22,8 +22,8 @@ use reth_ethereum::primitives::{serde_bincode_compat::SerdeBincodeCompat, InMemo reth_codecs::Compact, )] #[serde(rename_all = "camelCase")] -#[doc(alias = "CustomTransaction", alias = "TransactionCustom", alias = "CustomTx")] -pub struct TxCustom { +#[doc(alias = "PaymentTransaction", alias = "TransactionPayment", alias = "PaymentTx")] +pub struct TxPayment { /// EIP-155: Simple replay attack protection #[serde(with = "alloy_serde::quantity")] pub chain_id: ChainId, @@ -59,37 +59,23 @@ pub struct TxCustom { /// This is also known as `GasTipCap` #[serde(with = "alloy_serde::quantity")] pub max_priority_fee_per_gas: u128, - /// The 160-bit address of the message call’s recipient or, for a contract creation - /// transaction, ∅, used here to denote the only member of B0 ; formally Tt. - #[serde(default)] - pub to: TxKind, + /// The 160-bit address of the message call’s recipient. + pub to: Address, /// A scalar value equal to the number of Wei to /// be transferred to the message call’s recipient or, /// in the case of contract creation, as an endowment /// to the newly created account; formally Tv. pub value: U256, - /// The accessList specifies a list of addresses and storage keys; - /// these addresses and storage keys are added into the `accessed_addresses` - /// and `accessed_storage_keys` global sets (introduced in EIP-2929). - /// A gas cost is charged, though at a discount relative to the cost of - /// accessing outside the list. - pub access_list: AccessList, - /// Input has two uses depending if `to` field is Create or Call. - /// pub init: An unlimited size byte array specifying the - /// EVM-code for the account initialisation procedure CREATE, - /// data: An unlimited size byte array specifying the - /// input data of the message call, formally Td. - pub input: Bytes, } -impl TxCustom { +impl TxPayment { /// Get the transaction type #[doc(alias = "transaction_type")] - pub const fn tx_type() -> TxTypeCustom { - TxTypeCustom::Custom + pub const fn tx_type() -> super::tx::TxTypeCustom { + super::tx::TxTypeCustom::Payment } - /// Calculates a heuristic for the in-memory size of the [TxCustom] + /// Calculates a heuristic for the in-memory size of the [TxPayment] /// transaction. #[inline] pub fn size(&self) -> usize { @@ -98,14 +84,12 @@ impl TxCustom { mem::size_of::() + // gas_limit mem::size_of::() + // max_fee_per_gas mem::size_of::() + // max_priority_fee_per_gas - self.to.size() + // to - mem::size_of::() + // value - self.access_list.size() + // access_list - self.input.len() // input + mem::size_of::
() + // to + mem::size_of::() // value } } -impl RlpEcdsaEncodableTx for TxCustom { +impl RlpEcdsaEncodableTx for TxPayment { /// Outputs the length of the transaction's fields, without a RLP header. fn rlp_encoded_fields_length(&self) -> usize { self.chain_id.length() + @@ -114,9 +98,7 @@ impl RlpEcdsaEncodableTx for TxCustom { self.max_fee_per_gas.length() + self.gas_limit.length() + self.to.length() + - self.value.length() + - self.input.0.length() + - self.access_list.length() + self.value.length() } /// Encodes only the transaction's fields into the desired buffer, without @@ -129,15 +111,13 @@ impl RlpEcdsaEncodableTx for TxCustom { self.gas_limit.encode(out); self.to.encode(out); self.value.encode(out); - self.input.0.encode(out); - self.access_list.encode(out); } } -impl RlpEcdsaDecodableTx for TxCustom { - const DEFAULT_TX_TYPE: u8 = { Self::tx_type() as u8 }; +impl RlpEcdsaDecodableTx for TxPayment { + const DEFAULT_TX_TYPE: u8 = { PAYMENT_TX_TYPE_ID }; - /// Decodes the inner [TxCustom] fields from RLP bytes. + /// Decodes the inner [TxPayment] fields from RLP bytes. /// /// NOTE: This assumes a RLP header has already been decoded, and _just_ /// decodes the following RLP fields in the following order: @@ -160,13 +140,11 @@ impl RlpEcdsaDecodableTx for TxCustom { gas_limit: Decodable::decode(buf)?, to: Decodable::decode(buf)?, value: Decodable::decode(buf)?, - input: Decodable::decode(buf)?, - access_list: Decodable::decode(buf)?, }) } } -impl Transaction for TxCustom { +impl Transaction for TxPayment { #[inline] fn chain_id(&self) -> Option { Some(self.chain_id) @@ -228,12 +206,12 @@ impl Transaction for TxCustom { #[inline] fn kind(&self) -> TxKind { - self.to + TxKind::Call(self.to) } #[inline] fn is_create(&self) -> bool { - self.to.is_create() + false } #[inline] @@ -243,12 +221,14 @@ impl Transaction for TxCustom { #[inline] fn input(&self) -> &Bytes { - &self.input + // No input data + static EMPTY_BYTES: Bytes = Bytes::new(); + &EMPTY_BYTES } #[inline] fn access_list(&self) -> Option<&AccessList> { - Some(&self.access_list) + None } #[inline] @@ -262,19 +242,19 @@ impl Transaction for TxCustom { } } -impl Typed2718 for TxCustom { +impl Typed2718 for TxPayment { fn ty(&self) -> u8 { - TRANSFER_TX_TYPE_ID + PAYMENT_TX_TYPE_ID } } -impl SignableTransaction for TxCustom { +impl SignableTransaction for TxPayment { fn set_chain_id(&mut self, chain_id: ChainId) { self.chain_id = chain_id; } fn encode_for_signing(&self, out: &mut dyn alloy_rlp::BufMut) { - out.put_u8(Self::tx_type() as u8); + out.put_u8(Self::tx_type().ty()); self.encode(out) } @@ -283,7 +263,7 @@ impl SignableTransaction for TxCustom { } } -impl Encodable for TxCustom { +impl Encodable for TxPayment { fn encode(&self, out: &mut dyn BufMut) { self.rlp_encode(out); } @@ -293,29 +273,16 @@ impl Encodable for TxCustom { } } -impl Decodable for TxCustom { +impl Decodable for TxPayment { fn decode(buf: &mut &[u8]) -> alloy_rlp::Result { Self::rlp_decode(buf) } } -impl InMemorySize for TxCustom { +impl InMemorySize for TxPayment { fn size(&self) -> usize { - TxCustom::size(self) + TxPayment::size(self) } } -#[derive(Debug, serde::Serialize, serde::Deserialize)] -pub struct BincodeCompatTxCustom(pub TxCustom); - -impl SerdeBincodeCompat for TxCustom { - type BincodeRepr<'a> = BincodeCompatTxCustom; - - fn as_repr(&self) -> Self::BincodeRepr<'_> { - BincodeCompatTxCustom(self.clone()) - } - - fn from_repr(repr: Self::BincodeRepr<'_>) -> Self { - repr.0.clone() - } -} +impl RlpBincode for TxPayment {} diff --git a/examples/custom-node/src/primitives/tx_type.rs b/examples/custom-node/src/primitives/tx_type.rs index 36160024792..46c7de3f5cd 100644 --- a/examples/custom-node/src/primitives/tx_type.rs +++ b/examples/custom-node/src/primitives/tx_type.rs @@ -1,21 +1,8 @@ +use crate::primitives::TxTypeCustom; use alloy_primitives::bytes::{Buf, BufMut}; use reth_codecs::{txtype::COMPACT_EXTENDED_IDENTIFIER_FLAG, Compact}; -use serde::{Deserialize, Serialize}; -pub const TRANSFER_TX_TYPE_ID: u8 = 42; - -/// An enum for the custom transaction type(s) -#[repr(u8)] -#[derive(Debug, Clone, Serialize, Deserialize, Hash, Eq, PartialEq)] -pub enum TxTypeCustom { - Custom = TRANSFER_TX_TYPE_ID, -} - -impl From for u8 { - fn from(value: TxTypeCustom) -> Self { - value as Self - } -} +pub const PAYMENT_TX_TYPE_ID: u8 = 42; impl Compact for TxTypeCustom { fn to_compact(&self, buf: &mut B) -> usize @@ -23,26 +10,30 @@ impl Compact for TxTypeCustom { B: BufMut + AsMut<[u8]>, { match self { - Self::Custom => { - buf.put_u8(TRANSFER_TX_TYPE_ID); + Self::Op(ty) => ty.to_compact(buf), + Self::Payment => { + buf.put_u8(PAYMENT_TX_TYPE_ID); COMPACT_EXTENDED_IDENTIFIER_FLAG } } } fn from_compact(mut buf: &[u8], identifier: usize) -> (Self, &[u8]) { - ( - match identifier { - COMPACT_EXTENDED_IDENTIFIER_FLAG => { + match identifier { + COMPACT_EXTENDED_IDENTIFIER_FLAG => ( + { let extended_identifier = buf.get_u8(); match extended_identifier { - TRANSFER_TX_TYPE_ID => Self::Custom, + PAYMENT_TX_TYPE_ID => Self::Payment, _ => panic!("Unsupported TxType identifier: {extended_identifier}"), } - } - _ => panic!("Unknown identifier for TxType: {identifier}"), - }, - buf, - ) + }, + buf, + ), + v => { + let (inner, buf) = TxTypeCustom::from_compact(buf, v); + (inner, buf) + } + } } } diff --git a/examples/custom-node/src/rpc.rs b/examples/custom-node/src/rpc.rs new file mode 100644 index 00000000000..8259297367d --- /dev/null +++ b/examples/custom-node/src/rpc.rs @@ -0,0 +1,53 @@ +use crate::{ + evm::CustomTxEnv, + primitives::{CustomHeader, CustomTransaction}, +}; +use alloy_consensus::error::ValueError; +use alloy_network::TxSigner; +use op_alloy_consensus::OpTxEnvelope; +use op_alloy_rpc_types::{OpTransactionReceipt, OpTransactionRequest}; +use reth_op::rpc::RpcTypes; +use reth_rpc_api::eth::{ + transaction::TryIntoTxEnv, EthTxEnvError, SignTxRequestError, SignableTxRequest, TryIntoSimTx, +}; +use revm::context::{BlockEnv, CfgEnv}; + +#[derive(Debug, Clone, Copy, Default)] +#[non_exhaustive] +pub struct CustomRpcTypes; + +impl RpcTypes for CustomRpcTypes { + type Header = alloy_rpc_types_eth::Header; + type Receipt = OpTransactionReceipt; + type TransactionRequest = OpTransactionRequest; + type TransactionResponse = op_alloy_rpc_types::Transaction; +} + +impl TryIntoSimTx for OpTransactionRequest { + fn try_into_sim_tx(self) -> Result> { + Ok(CustomTransaction::Op(self.try_into_sim_tx()?)) + } +} + +impl TryIntoTxEnv for OpTransactionRequest { + type Err = EthTxEnvError; + + fn try_into_tx_env( + self, + cfg_env: &CfgEnv, + block_env: &BlockEnv, + ) -> Result { + Ok(CustomTxEnv::Op(self.try_into_tx_env(cfg_env, block_env)?)) + } +} + +impl SignableTxRequest for OpTransactionRequest { + async fn try_build_and_sign( + self, + signer: impl TxSigner + Send, + ) -> Result { + Ok(CustomTransaction::Op( + SignableTxRequest::::try_build_and_sign(self, signer).await?, + )) + } +} diff --git a/examples/custom-payload-builder/Cargo.toml b/examples/custom-payload-builder/Cargo.toml index 3bab8b9cec9..f7a24eb6852 100644 --- a/examples/custom-payload-builder/Cargo.toml +++ b/examples/custom-payload-builder/Cargo.toml @@ -6,7 +6,6 @@ edition.workspace = true license.workspace = true [dependencies] -reth.workspace = true reth-basic-payload-builder.workspace = true reth-payload-builder.workspace = true reth-ethereum = { workspace = true, features = ["node", "pool", "cli"] } diff --git a/examples/custom-payload-builder/src/generator.rs b/examples/custom-payload-builder/src/generator.rs index e8d5bb62d9f..324d685b1ab 100644 --- a/examples/custom-payload-builder/src/generator.rs +++ b/examples/custom-payload-builder/src/generator.rs @@ -1,6 +1,5 @@ use crate::job::EmptyBlockPayloadJob; use alloy_eips::BlockNumberOrTag; -use reth::tasks::TaskSpawner; use reth_basic_payload_builder::{ BasicPayloadJobGeneratorConfig, HeaderForPayload, PayloadBuilder, PayloadConfig, }; @@ -8,6 +7,7 @@ use reth_ethereum::{ node::api::{Block, PayloadBuilderAttributes}, primitives::SealedHeader, provider::{BlockReaderIdExt, BlockSource, StateProviderFactory}, + tasks::TaskSpawner, }; use reth_payload_builder::{PayloadBuilderError, PayloadJobGenerator}; use std::sync::Arc; diff --git a/examples/custom-payload-builder/src/job.rs b/examples/custom-payload-builder/src/job.rs index f511766b0d2..abb6e89668f 100644 --- a/examples/custom-payload-builder/src/job.rs +++ b/examples/custom-payload-builder/src/job.rs @@ -1,7 +1,9 @@ use futures_util::Future; -use reth::tasks::TaskSpawner; use reth_basic_payload_builder::{HeaderForPayload, PayloadBuilder, PayloadConfig}; -use reth_ethereum::node::api::PayloadKind; +use reth_ethereum::{ + node::api::{PayloadBuilderAttributes, PayloadKind}, + tasks::TaskSpawner, +}; use reth_payload_builder::{KeepPayloadJobAlive, PayloadBuilderError, PayloadJob}; use std::{ @@ -45,6 +47,10 @@ where Ok(self.config.attributes.clone()) } + fn payload_timestamp(&self) -> Result { + Ok(self.config.attributes.timestamp()) + } + fn resolve_kind( &mut self, _kind: PayloadKind, diff --git a/examples/custom-payload-builder/src/main.rs b/examples/custom-payload-builder/src/main.rs index 73798d20eab..c38b46a5b9c 100644 --- a/examples/custom-payload-builder/src/main.rs +++ b/examples/custom-payload-builder/src/main.rs @@ -1,4 +1,4 @@ -//! Example for how hook into the node via the CLI extension mechanism without registering +//! Example for how to hook into the node via the CLI extension mechanism without registering //! additional arguments //! //! Run with @@ -12,13 +12,13 @@ #![warn(unused_crate_dependencies)] use crate::generator::EmptyBlockPayloadJobGenerator; -use reth::builder::{components::PayloadServiceBuilder, BuilderContext}; use reth_basic_payload_builder::BasicPayloadJobGeneratorConfig; use reth_ethereum::{ chainspec::ChainSpec, cli::interface::Cli, node::{ api::{node::FullNodeTypes, NodeTypes}, + builder::{components::PayloadServiceBuilder, BuilderContext}, core::cli::config::PayloadBuilderConfig, node::EthereumAddOns, EthEngineTypes, EthEvmConfig, EthereumNode, diff --git a/examples/custom-rlpx-subprotocol/Cargo.toml b/examples/custom-rlpx-subprotocol/Cargo.toml index d396b99eb79..06e8b77950c 100644 --- a/examples/custom-rlpx-subprotocol/Cargo.toml +++ b/examples/custom-rlpx-subprotocol/Cargo.toml @@ -8,8 +8,7 @@ license.workspace = true [dependencies] tokio = { workspace = true, features = ["full"] } futures.workspace = true -reth-ethereum = { workspace = true, features = ["node", "network"] } -reth.workspace = true +reth-ethereum = { workspace = true, features = ["node", "network", "cli"] } tokio-stream.workspace = true eyre.workspace = true tracing.workspace = true diff --git a/examples/custom-rlpx-subprotocol/src/main.rs b/examples/custom-rlpx-subprotocol/src/main.rs index 4e4bea532b0..91ec308abf6 100644 --- a/examples/custom-rlpx-subprotocol/src/main.rs +++ b/examples/custom-rlpx-subprotocol/src/main.rs @@ -14,7 +14,6 @@ mod subprotocol; use std::net::{Ipv4Addr, SocketAddr, SocketAddrV4}; -use reth::builder::NodeHandle; use reth_ethereum::{ network::{ api::{test_utils::PeersHandleProvider, NetworkInfo}, @@ -22,7 +21,7 @@ use reth_ethereum::{ protocol::IntoRlpxSubProtocol, NetworkConfig, NetworkManager, NetworkProtocols, }, - node::EthereumNode, + node::{builder::NodeHandle, EthereumNode}, }; use subprotocol::{ connection::CustomCommand, @@ -35,7 +34,7 @@ use tokio::sync::{mpsc, oneshot}; use tracing::info; fn main() -> eyre::Result<()> { - reth::cli::Cli::parse_args().run(|builder, _args| async move { + reth_ethereum::cli::Cli::parse_args().run(|builder, _args| async move { // launch the node let NodeHandle { node, node_exit_future } = builder.node(EthereumNode::default()).launch().await?; diff --git a/examples/custom-rlpx-subprotocol/src/subprotocol/protocol/handler.rs b/examples/custom-rlpx-subprotocol/src/subprotocol/protocol/handler.rs index e18a63673a0..8a6dead2cbc 100644 --- a/examples/custom-rlpx-subprotocol/src/subprotocol/protocol/handler.rs +++ b/examples/custom-rlpx-subprotocol/src/subprotocol/protocol/handler.rs @@ -4,7 +4,7 @@ use reth_ethereum::network::{api::PeerId, protocol::ProtocolHandler}; use std::net::SocketAddr; use tokio::sync::mpsc; -/// Protocol state is an helper struct to store the protocol events. +/// Protocol state is a helper struct to store the protocol events. #[derive(Clone, Debug)] pub(crate) struct ProtocolState { pub(crate) events: mpsc::UnboundedSender, diff --git a/examples/custom-rlpx-subprotocol/src/subprotocol/protocol/proto.rs b/examples/custom-rlpx-subprotocol/src/subprotocol/protocol/proto.rs index 495c4357823..19508c17035 100644 --- a/examples/custom-rlpx-subprotocol/src/subprotocol/protocol/proto.rs +++ b/examples/custom-rlpx-subprotocol/src/subprotocol/protocol/proto.rs @@ -1,4 +1,4 @@ -//! Simple RLPx Ping Pong protocol that also support sending messages, +//! Simple RLPx Ping Pong protocol that also supports sending messages, //! following [RLPx specs](https://github.com/ethereum/devp2p/blob/master/rlpx.md) use alloy_primitives::bytes::{Buf, BufMut, BytesMut}; diff --git a/examples/db-access/src/main.rs b/examples/db-access/src/main.rs index 74739a5d926..339aa1ae3d1 100644 --- a/examples/db-access/src/main.rs +++ b/examples/db-access/src/main.rs @@ -7,9 +7,9 @@ use reth_ethereum::{ primitives::{AlloyBlockHeader, SealedBlock, SealedHeader}, provider::{ providers::ReadOnlyConfig, AccountReader, BlockReader, BlockSource, HeaderProvider, - ReceiptProvider, StateProvider, TransactionsProvider, + ReceiptProvider, StateProvider, TransactionVariant, TransactionsProvider, }, - rpc::eth::primitives::{Filter, FilteredParams}, + rpc::eth::primitives::Filter, TransactionSigned, }; @@ -57,20 +57,15 @@ fn header_provider_example(provider: T, number: u64) -> eyre: // Can query the header by number let header = provider.header_by_number(number)?.ok_or(eyre::eyre!("header not found"))?; - // We can convert a header to a sealed header which contains the hash w/o needing to re-compute + // We can convert a header to a sealed header which contains the hash w/o needing to recompute // it every time. let sealed_header = SealedHeader::seal_slow(header); // Can also query the header by hash! let header_by_hash = - provider.header(&sealed_header.hash())?.ok_or(eyre::eyre!("header by hash not found"))?; + provider.header(sealed_header.hash())?.ok_or(eyre::eyre!("header by hash not found"))?; assert_eq!(sealed_header.header(), &header_by_hash); - // The header's total difficulty is stored in a separate table, so we have a separate call for - // it. This is not needed for post PoS transition chains. - let td = provider.header_td_by_number(number)?.ok_or(eyre::eyre!("header td not found"))?; - assert!(!td.is_zero()); - // Can query headers by range as well, already sealed! let headers = provider.sealed_headers_range(100..200)?; assert_eq!(headers.len(), 100); @@ -123,10 +118,12 @@ fn block_provider_example>( let block = provider.block(number.into())?.ok_or(eyre::eyre!("block num not found"))?; assert_eq!(block.number, number); - // Can query a block with its senders, this is useful when you'd want to execute a block and do + // Can query a block with its senders, this is useful when you want to execute a block and do // not want to manually recover the senders for each transaction (as each transaction is // stored on disk with its v,r,s but not its `from` field.). - let block = provider.block(number.into())?.ok_or(eyre::eyre!("block num not found"))?; + let _recovered_block = provider + .sealed_block_with_senders(number.into(), TransactionVariant::WithHash)? + .ok_or(eyre::eyre!("block num not found"))?; // Can seal the block to cache the hash, like the Header above. let sealed_block = SealedBlock::seal_slow(block.clone()); @@ -143,20 +140,12 @@ fn block_provider_example>( .ok_or(eyre::eyre!("block by hash not found"))?; assert_eq!(block, block_by_hash2); - // Or you can also specify the datasource. For this provider this always return `None`, but + // Or you can also specify the datasource. For this provider this always returns `None`, but // the blockchain tree is also able to access pending state not available in the db yet. let block_by_hash3 = provider .find_block_by_hash(sealed_block.hash(), BlockSource::Any)? .ok_or(eyre::eyre!("block hash not found"))?; assert_eq!(block, block_by_hash3); - - // Can query the block's ommers/uncles - let _ommers = provider.ommers(number.into())?; - - // Can query the block's withdrawals (via the `WithdrawalsProvider`) - let _withdrawals = - provider.withdrawals_by_block(sealed_block.hash().into(), sealed_block.timestamp)?; - Ok(()) } @@ -186,7 +175,7 @@ fn receipts_provider_example< .receipts_by_block(100.into())? .ok_or(eyre::eyre!("no receipts found for block"))?; - // Can check if a address/topic filter is present in a header, if it is we query the block and + // Can check if an address/topic filter is present in a header, if it is we query the block and // receipts and do something with the data // 1. get the bloom from the header let header = provider.header_by_number(header_num)?.unwrap(); @@ -201,21 +190,14 @@ fn receipts_provider_example< // TODO: Make it clearer how to choose between event_signature(topic0) (event name) and the // other 3 indexed topics. This API is a bit clunky and not obvious to use at the moment. let filter = Filter::new().address(addr).event_signature(topic); - let filter_params = FilteredParams::new(Some(filter)); - let address_filter = FilteredParams::address_filter(&addr.into()); - let topics_filter = FilteredParams::topics_filter(&[topic.into()]); // 3. If the address & topics filters match do something. We use the outer check against the // bloom filter stored in the header to avoid having to query the receipts table when there // is no instance of any event that matches the filter in the header. - if FilteredParams::matches_address(bloom, &address_filter) && - FilteredParams::matches_topics(bloom, &topics_filter) - { + if filter.matches_bloom(bloom) { let receipts = provider.receipt(header_num)?.ok_or(eyre::eyre!("receipt not found"))?; for log in &receipts.logs { - if filter_params.filter_address(&log.address) && - filter_params.filter_topics(log.topics()) - { + if filter.matches(log) { // Do something with the log e.g. decode it. println!("Matching log found! {log:?}") } diff --git a/examples/engine-api-access/Cargo.toml b/examples/engine-api-access/Cargo.toml new file mode 100644 index 00000000000..3e1f185077f --- /dev/null +++ b/examples/engine-api-access/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "example-engine-api-access" +version = "0.0.0" +publish = false +edition.workspace = true +license.workspace = true + +[dependencies] +# reth +reth-db = { workspace = true, features = ["op", "test-utils"] } +reth-node-builder.workspace = true +reth-optimism-node.workspace = true +reth-optimism-chainspec.workspace = true + +tokio = { workspace = true, features = ["sync"] } diff --git a/examples/engine-api-access/src/main.rs b/examples/engine-api-access/src/main.rs new file mode 100644 index 00000000000..5f43d94bf6e --- /dev/null +++ b/examples/engine-api-access/src/main.rs @@ -0,0 +1,47 @@ +//! Example demonstrating how to access the Engine API instance during construction. +//! +//! Run with +//! +//! ```sh +//! cargo run -p example-engine-api-access +//! ``` + +use reth_db::test_utils::create_test_rw_db; +use reth_node_builder::{EngineApiExt, FullNodeComponents, NodeBuilder, NodeConfig}; +use reth_optimism_chainspec::BASE_MAINNET; +use reth_optimism_node::{ + args::RollupArgs, node::OpEngineValidatorBuilder, OpAddOns, OpEngineApiBuilder, OpNode, +}; +use tokio::sync::oneshot; + +#[tokio::main] +async fn main() { + // Op node configuration and setup + let config = NodeConfig::new(BASE_MAINNET.clone()); + let db = create_test_rw_db(); + let args = RollupArgs::default(); + let op_node = OpNode::new(args); + + let (engine_api_tx, _engine_api_rx) = oneshot::channel(); + + let engine_api = + EngineApiExt::new(OpEngineApiBuilder::::default(), move |api| { + let _ = engine_api_tx.send(api); + }); + + let _builder = NodeBuilder::new(config) + .with_database(db) + .with_types::() + .with_components(op_node.components()) + .with_add_ons(OpAddOns::default().with_engine_api(engine_api)) + .on_component_initialized(move |ctx| { + let _provider = ctx.provider(); + Ok(()) + }) + .on_node_started(|_full_node| Ok(())) + .on_rpc_started(|_ctx, handles| { + let _client = handles.rpc.http_client(); + Ok(()) + }) + .check_launch(); +} diff --git a/examples/exex-hello-world/Cargo.toml b/examples/exex-hello-world/Cargo.toml index 4751605b078..4d59c2c98c7 100644 --- a/examples/exex-hello-world/Cargo.toml +++ b/examples/exex-hello-world/Cargo.toml @@ -7,8 +7,7 @@ license.workspace = true [dependencies] # reth -reth.workspace = true -reth-ethereum = { workspace = true, features = ["full"] } +reth-ethereum = { workspace = true, features = ["full", "cli"] } reth-tracing.workspace = true eyre.workspace = true diff --git a/examples/exex-hello-world/src/main.rs b/examples/exex-hello-world/src/main.rs index cb560669d35..2c89fb72627 100644 --- a/examples/exex-hello-world/src/main.rs +++ b/examples/exex-hello-world/src/main.rs @@ -8,20 +8,28 @@ use clap::Parser; use futures::TryStreamExt; -use reth::rpc::eth::core::EthApiFor; use reth_ethereum::{ + chainspec::EthereumHardforks, exex::{ExExContext, ExExEvent, ExExNotification}, - node::{api::FullNodeComponents, EthereumNode}, + node::{ + api::{FullNodeComponents, NodeTypes}, + builder::rpc::RpcHandle, + EthereumNode, + }, + rpc::api::eth::helpers::FullEthApi, }; use reth_tracing::tracing::info; use tokio::sync::oneshot; +/// Additional CLI arguments #[derive(Parser)] struct ExExArgs { + /// whether to launch an op-reth node #[arg(long)] optimism: bool, } +/// A basic subscription loop of new blocks. async fn my_exex(mut ctx: ExExContext) -> eyre::Result<()> { while let Some(notification) = ctx.notifications.try_next().await? { match ¬ification { @@ -44,22 +52,44 @@ async fn my_exex(mut ctx: ExExContext) -> eyre:: Ok(()) } -/// This is an example of how to access the `EthApi` inside an ExEx. It receives the `EthApi` once -/// the node is launched fully. -async fn ethapi_exex( +/// This is an example of how to access the [`RpcHandle`] inside an ExEx. It receives the +/// [`RpcHandle`] once the node is launched fully. +/// +/// This function supports both Opstack Eth API and ethereum Eth API. +/// +/// The received handle gives access to the `EthApi` has full access to all eth api functionality +/// [`FullEthApi`]. And also gives access to additional eth-related rpc method handlers, such as eth +/// filter. +async fn ethapi_exex( mut ctx: ExExContext, - ethapi_rx: oneshot::Receiver>, + rpc_handle: oneshot::Receiver>, ) -> eyre::Result<()> where - Node: FullNodeComponents, + Node: FullNodeComponents>, + EthApi: FullEthApi, { // Wait for the ethapi to be sent from the main function - let _ethapi = ethapi_rx.await?; - info!("Received ethapi inside exex"); + let rpc_handle = rpc_handle.await?; + info!("Received rpc handle inside exex"); + + // obtain the ethapi from the rpc handle + let ethapi = rpc_handle.eth_api(); + + // EthFilter type that provides all eth_getlogs related logic + let _eth_filter = rpc_handle.eth_handlers().filter.clone(); + // EthPubSub type that provides all eth_subscribe logic + let _eth_pubsub = rpc_handle.eth_handlers().pubsub.clone(); + // The TraceApi type that provides all the trace_ handlers + let _trace_api = rpc_handle.trace_api(); + // The DebugApi type that provides all the trace_ handlers + let _debug_api = rpc_handle.debug_api(); while let Some(notification) = ctx.notifications.try_next().await? { if let Some(committed_chain) = notification.committed_chain() { ctx.events.send(ExExEvent::FinishedHeight(committed_chain.tip().num_hash()))?; + + // can use the eth api to interact with the node + let _rpc_block = ethapi.rpc_block(committed_chain.tip().hash().into(), true).await?; } } @@ -71,30 +101,42 @@ fn main() -> eyre::Result<()> { if args.optimism { reth_op::cli::Cli::parse_args().run(|builder, _| { + let (rpc_handle_tx, rpc_handle_rx) = oneshot::channel(); Box::pin(async move { let handle = builder .node(reth_op::node::OpNode::default()) .install_exex("my-exex", async move |ctx| Ok(my_exex(ctx))) + .install_exex("ethapi-exex", async move |ctx| { + Ok(ethapi_exex(ctx, rpc_handle_rx)) + }) .launch() .await?; + // Retrieve the rpc handle from the node and send it to the exex + rpc_handle_tx + .send(handle.node.add_ons_handle.clone()) + .expect("Failed to send ethapi to ExEx"); + handle.wait_for_node_exit().await }) }) } else { - reth::cli::Cli::parse_args().run(|builder, _| { + reth_ethereum::cli::Cli::parse_args().run(|builder, _| { Box::pin(async move { - let (ethapi_tx, ethapi_rx) = oneshot::channel(); + let (rpc_handle_tx, rpc_handle_rx) = oneshot::channel(); let handle = builder .node(EthereumNode::default()) .install_exex("my-exex", async move |ctx| Ok(my_exex(ctx))) - .install_exex("ethapi-exex", async move |ctx| Ok(ethapi_exex(ctx, ethapi_rx))) + .install_exex("ethapi-exex", async move |ctx| { + Ok(ethapi_exex(ctx, rpc_handle_rx)) + }) .launch() .await?; - // Retrieve the ethapi from the node and send it to the exex - let ethapi = handle.node.add_ons_handle.eth_api(); - ethapi_tx.send(ethapi.clone()).expect("Failed to send ethapi to ExEx"); + // Retrieve the rpc handle from the node and send it to the exex + rpc_handle_tx + .send(handle.node.add_ons_handle.clone()) + .expect("Failed to send ethapi to ExEx"); handle.wait_for_node_exit().await }) diff --git a/examples/exex-subscription/Cargo.toml b/examples/exex-subscription/Cargo.toml index 2ebb6e10be5..9593409b215 100644 --- a/examples/exex-subscription/Cargo.toml +++ b/examples/exex-subscription/Cargo.toml @@ -20,4 +20,5 @@ clap = { workspace = true, features = ["derive"] } jsonrpsee = { workspace = true, features = ["server", "macros"] } tokio.workspace = true serde.workspace = true +serde_json.workspace = true tracing.workspace = true diff --git a/examples/exex-subscription/src/main.rs b/examples/exex-subscription/src/main.rs index 875b9d30b1d..e39408a3dc0 100644 --- a/examples/exex-subscription/src/main.rs +++ b/examples/exex-subscription/src/main.rs @@ -1,18 +1,16 @@ #![allow(dead_code)] -//! An ExEx example that installs a new RPC subscription endpoint that emit storage changes for a +//! An ExEx example that installs a new RPC subscription endpoint that emits storage changes for a //! requested address. #[allow(dead_code)] use alloy_primitives::{Address, U256}; -use clap::Parser; use futures::TryStreamExt; use jsonrpsee::{ - core::SubscriptionResult, proc_macros::rpc, tracing, PendingSubscriptionSink, - SubscriptionMessage, + core::SubscriptionResult, proc_macros::rpc, PendingSubscriptionSink, SubscriptionMessage, }; use reth_ethereum::{ exex::{ExExContext, ExExEvent, ExExNotification}, - node::{api::FullNodeComponents, EthereumNode}, + node::{api::FullNodeComponents, builder::NodeHandleFor, EthereumNode}, }; use std::collections::HashMap; use tokio::sync::{mpsc, oneshot}; @@ -85,7 +83,9 @@ impl StorageWatcherApiServer for StorageWatcherRpc { let Ok(mut rx) = resp_rx.await else { return }; while let Some(diff) = rx.recv().await { - let msg = SubscriptionMessage::from_json(&diff).expect("serialize"); + let msg = SubscriptionMessage::from( + serde_json::value::to_raw_value(&diff).expect("serialize"), + ); if sink.send(msg).await.is_err() { break; } @@ -164,19 +164,13 @@ async fn my_exex( Ok(()) } -#[derive(Parser, Debug)] -struct Args { - #[arg(long)] - enable_ext: bool, -} - fn main() -> eyre::Result<()> { - reth_ethereum::cli::Cli::parse_args().run(|builder, _args| async move { + reth_ethereum::cli::Cli::parse_args().run(|builder, _| async move { let (subscriptions_tx, subscriptions_rx) = mpsc::unbounded_channel::(); let rpc = StorageWatcherRpc::new(subscriptions_tx.clone()); - let handle = builder + let handle: NodeHandleFor = builder .node(EthereumNode::default()) .extend_rpc_modules(move |ctx| { ctx.modules.merge_configured(StorageWatcherApiServer::into_rpc(rpc))?; diff --git a/examples/full-contract-state/Cargo.toml b/examples/full-contract-state/Cargo.toml new file mode 100644 index 00000000000..f4f61244a29 --- /dev/null +++ b/examples/full-contract-state/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "example-full-contract-state" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true +exclude.workspace = true + +[dependencies] +reth-ethereum = { workspace = true, features = ["node"] } +eyre.workspace = true + +[lints] +workspace = true diff --git a/examples/full-contract-state/README.md b/examples/full-contract-state/README.md new file mode 100644 index 00000000000..c0cd4a0def4 --- /dev/null +++ b/examples/full-contract-state/README.md @@ -0,0 +1,69 @@ +# Full Contract State Example + +This example demonstrates how to extract the complete state of a specific contract from the reth database. + +## What it does + +The example shows how to: + +1. **Connect to a reth database** - Uses the recommended builder pattern to create a read-only provider +2. **Get basic account information** - Retrieves balance, nonce, and code hash for the contract address +3. **Get contract bytecode** - Fetches the actual contract bytecode if the account is a contract +4. **Iterate through all storage slots** - Uses database cursors to efficiently retrieve all storage key-value pairs + +## Prerequisites + +- A reth database with some data (you can sync a node or use a pre-synced database) +- Set the `RETH_DATADIR` environment variable to point to your reth data directory +- Set the `CONTRACT_ADDRESS` environment variable to provide target contract address + +## Usage + +```bash +# Set your reth data directory +export RETH_DATADIR="/path/to/your/reth/datadir" +# Set target contract address +export CONTRACT_ADDRESS="0x0..." + +# Run the example +cargo run --example full-contract-state +``` + +## Code Structure + +The example consists of: + +- **`ContractState` struct** - Holds all contract state information +- **`extract_contract_state` function** - Main function that extracts contract state +- **`main` function** - Sets up the provider and demonstrates usage + +## Key Concepts + +### Provider Pattern +The example uses reth's provider pattern: +- `ProviderFactory` - Creates database connections +- `DatabaseProvider` - Provides low-level database access +- `StateProvider` - Provides high-level state access + +### Database Cursors +For efficient storage iteration, the example uses database cursors: +- `cursor_dup_read` - Creates a cursor for duplicate key tables +- `seek_exact` - Positions cursor at specific key +- `next_dup` - Iterates through duplicate entries + +## Output + +The example will print: +- Contract address +- Account balance +- Account nonce +- Code hash +- Number of storage slots +- All storage key-value pairs + +## Error Handling + +The example includes proper error handling: +- Returns `None` if the contract doesn't exist +- Uses `ProviderResult` for database operation errors +- Gracefully handles missing bytecode or storage diff --git a/examples/full-contract-state/src/main.rs b/examples/full-contract-state/src/main.rs new file mode 100644 index 00000000000..0a0cdf81adb --- /dev/null +++ b/examples/full-contract-state/src/main.rs @@ -0,0 +1,94 @@ +//! Example demonstrating how to extract the full state of a specific contract from the reth +//! database. +//! +//! This example shows how to: +//! 1. Connect to a reth database +//! 2. Get basic account information (balance, nonce, code hash) +//! 3. Get contract bytecode +//! 4. Iterate through all storage slots for the contract + +use reth_ethereum::{ + chainspec::ChainSpecBuilder, + evm::revm::primitives::{Address, B256, U256}, + node::EthereumNode, + primitives::{Account, Bytecode}, + provider::{ + db::{ + cursor::{DbCursorRO, DbDupCursorRO}, + tables, + transaction::DbTx, + }, + providers::ReadOnlyConfig, + ProviderResult, + }, + storage::{DBProvider, StateProvider}, +}; +use std::{collections::HashMap, str::FromStr}; + +/// Represents the complete state of a contract including account info, bytecode, and storage +#[derive(Debug, Clone)] +pub struct ContractState { + /// The address of the contract + pub address: Address, + /// Basic account information (balance, nonce, code hash) + pub account: Account, + /// Contract bytecode (None if not a contract or doesn't exist) + pub bytecode: Option, + /// All storage slots for the contract + pub storage: HashMap, +} + +/// Extract the full state of a specific contract +pub fn extract_contract_state( + provider: &P, + state_provider: &dyn StateProvider, + contract_address: Address, +) -> ProviderResult> { + let account = state_provider.basic_account(&contract_address)?; + let Some(account) = account else { + return Ok(None); + }; + + let bytecode = state_provider.account_code(&contract_address)?; + + let mut storage_cursor = provider.tx_ref().cursor_dup_read::()?; + let mut storage = HashMap::new(); + + if let Some((_, first_entry)) = storage_cursor.seek_exact(contract_address)? { + storage.insert(first_entry.key, first_entry.value); + + while let Some((_, entry)) = storage_cursor.next_dup()? { + storage.insert(entry.key, entry.value); + } + } + + Ok(Some(ContractState { address: contract_address, account, bytecode, storage })) +} + +fn main() -> eyre::Result<()> { + let address = std::env::var("CONTRACT_ADDRESS")?; + let contract_address = Address::from_str(&address)?; + + let datadir = std::env::var("RETH_DATADIR")?; + let spec = ChainSpecBuilder::mainnet().build(); + let factory = EthereumNode::provider_factory_builder() + .open_read_only(spec.into(), ReadOnlyConfig::from_datadir(datadir))?; + + let provider = factory.provider()?; + let state_provider = factory.latest()?; + let contract_state = + extract_contract_state(&provider, state_provider.as_ref(), contract_address)?; + + if let Some(state) = contract_state { + println!("Contract: {}", state.address); + println!("Balance: {}", state.account.balance); + println!("Nonce: {}", state.account.nonce); + println!("Code hash: {:?}", state.account.bytecode_hash); + println!("Storage slots: {}", state.storage.len()); + for (key, value) in &state.storage { + println!("\t{key}: {value}"); + } + } + + Ok(()) +} diff --git a/examples/manual-p2p/src/main.rs b/examples/manual-p2p/src/main.rs index 8fb970abcea..41fb846c940 100644 --- a/examples/manual-p2p/src/main.rs +++ b/examples/manual-p2p/src/main.rs @@ -19,8 +19,8 @@ use reth_ethereum::{ network::{ config::rng_secret_key, eth_wire::{ - EthMessage, EthStream, HelloMessage, P2PStream, Status, UnauthedEthStream, - UnauthedP2PStream, + EthMessage, EthStream, HelloMessage, P2PStream, UnauthedEthStream, UnauthedP2PStream, + UnifiedStatus, }, EthNetworkPrimitives, }, @@ -101,26 +101,30 @@ async fn handshake_p2p( } // Perform a ETH Wire handshake with a peer -async fn handshake_eth(p2p_stream: AuthedP2PStream) -> eyre::Result<(AuthedEthStream, Status)> { +async fn handshake_eth( + p2p_stream: AuthedP2PStream, +) -> eyre::Result<(AuthedEthStream, UnifiedStatus)> { let fork_filter = MAINNET.fork_filter(Head { timestamp: MAINNET.fork(EthereumHardfork::Shanghai).as_timestamp().unwrap(), ..Default::default() }); - let status = Status::builder() + let unified_status = UnifiedStatus::builder() .chain(Chain::mainnet()) .genesis(MAINNET_GENESIS_HASH) .forkid(MAINNET.hardfork_fork_id(EthereumHardfork::Shanghai).unwrap()) .build(); - let status = - Status { version: p2p_stream.shared_capabilities().eth()?.version().try_into()?, ..status }; + let status = UnifiedStatus { + version: p2p_stream.shared_capabilities().eth()?.version().try_into()?, + ..unified_status + }; let eth_unauthed = UnauthedEthStream::new(p2p_stream); Ok(eth_unauthed.handshake(status, fork_filter).await?) } // Snoop by greedily capturing all broadcasts that the peer emits -// note: this node cannot handle request so will be disconnected by peer when challenged +// note: this node cannot handle request so it will be disconnected by peer when challenged async fn snoop(peer: NodeRecord, mut eth_stream: AuthedEthStream) { while let Some(Ok(update)) = eth_stream.next().await { match update { diff --git a/examples/network-proxy/src/main.rs b/examples/network-proxy/src/main.rs index 461fe348360..51ba8e2b4a4 100644 --- a/examples/network-proxy/src/main.rs +++ b/examples/network-proxy/src/main.rs @@ -81,6 +81,7 @@ async fn main() -> eyre::Result<()> { IncomingEthRequest::GetBlockBodies { .. } => {} IncomingEthRequest::GetNodeData { .. } => {} IncomingEthRequest::GetReceipts { .. } => {} + IncomingEthRequest::GetReceipts69 { .. } => {} } } transaction_message = transactions_rx.recv() => { diff --git a/examples/node-builder-api/Cargo.toml b/examples/node-builder-api/Cargo.toml new file mode 100644 index 00000000000..287456ec04e --- /dev/null +++ b/examples/node-builder-api/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "example-node-builder-api" +version = "0.0.0" +publish = false +edition.workspace = true +license.workspace = true + +[dependencies] +reth-ethereum = { workspace = true, features = ["node", "pool", "node-api", "cli", "test-utils"] } diff --git a/examples/node-builder-api/src/main.rs b/examples/node-builder-api/src/main.rs new file mode 100644 index 00000000000..f0d937a2d97 --- /dev/null +++ b/examples/node-builder-api/src/main.rs @@ -0,0 +1,29 @@ +//! This example showcases various Nodebuilder use cases + +use reth_ethereum::{ + cli::interface::Cli, + node::{builder::components::NoopNetworkBuilder, node::EthereumAddOns, EthereumNode}, +}; + +/// Maps the ethereum node's network component to the noop implementation. +/// +/// This installs the [`NoopNetworkBuilder`] that does not launch a real network. +pub fn noop_network() { + Cli::parse_args() + .run(|builder, _| async move { + let handle = builder + // use the default ethereum node types + .with_types::() + // Configure the components of the node + // use default ethereum components but use the Noop network that does nothing but + .with_components(EthereumNode::components().network(NoopNetworkBuilder::eth())) + .with_add_ons(EthereumAddOns::default()) + .launch() + .await?; + + handle.wait_for_node_exit().await + }) + .unwrap(); +} + +fn main() {} diff --git a/examples/node-custom-rpc/Cargo.toml b/examples/node-custom-rpc/Cargo.toml index aaba2fc0167..5b8aa98dbcf 100644 --- a/examples/node-custom-rpc/Cargo.toml +++ b/examples/node-custom-rpc/Cargo.toml @@ -11,5 +11,6 @@ reth-ethereum = { workspace = true, features = ["node", "pool", "cli"] } clap = { workspace = true, features = ["derive"] } jsonrpsee = { workspace = true, features = ["server", "macros"] } tokio.workspace = true +serde_json.workspace = true + [dev-dependencies] -tokio.workspace = true diff --git a/examples/node-custom-rpc/src/main.rs b/examples/node-custom-rpc/src/main.rs index c5cef667441..7ab271b4cc5 100644 --- a/examples/node-custom-rpc/src/main.rs +++ b/examples/node-custom-rpc/src/main.rs @@ -32,7 +32,9 @@ fn main() { Cli::::parse() .run(|builder, args| async move { let handle = builder + // configure default ethereum node .node(EthereumNode::default()) + // extend the rpc modules with our custom `TxpoolExt` endpoints .extend_rpc_modules(move |ctx| { if !args.enable_ext { return Ok(()) @@ -50,7 +52,8 @@ fn main() { Ok(()) }) - .launch() + // launch the node with custom rpc + .launch_with_debug_capabilities() .await?; handle.wait_for_node_exit().await @@ -76,6 +79,10 @@ pub trait TxpoolExtApi { #[method(name = "transactionCount")] fn transaction_count(&self) -> RpcResult; + /// Clears the transaction pool. + #[method(name = "clearTxpool")] + fn clear_txpool(&self) -> RpcResult<()>; + /// Creates a subscription that returns the number of transactions in the pool every 10s. #[subscription(name = "subscribeTransactionCount", item = usize)] fn subscribe_transaction_count( @@ -84,7 +91,7 @@ pub trait TxpoolExtApi { ) -> SubscriptionResult; } -/// The type that implements the `txpool` rpc namespace trait +/// The type that implements the `txpoolExt` rpc namespace trait pub struct TxpoolExt { pool: Pool, } @@ -98,6 +105,12 @@ where Ok(self.pool.pool_size().total) } + fn clear_txpool(&self) -> RpcResult<()> { + let all_tx_hashes = self.pool.all_transaction_hashes(); + self.pool.remove_transactions(all_tx_hashes); + Ok(()) + } + fn subscribe_transaction_count( &self, pending_subscription_sink: PendingSubscriptionSink, @@ -109,7 +122,7 @@ where let sink = match pending_subscription_sink.accept().await { Ok(sink) => sink, Err(e) => { - println!("failed to accept subscription: {}", e); + println!("failed to accept subscription: {e}"); return; } }; @@ -117,8 +130,9 @@ where loop { sleep(Duration::from_secs(delay)).await; - let msg = SubscriptionMessage::from_json(&pool.pool_size().total) - .expect("Failed to serialize `usize`"); + let msg = SubscriptionMessage::from( + serde_json::value::to_raw_value(&pool.pool_size().total).expect("serialize"), + ); let _ = sink.send(msg).await; } }); @@ -144,6 +158,12 @@ mod tests { Ok(self.pool.pool_size().total) } + fn clear_txpool(&self) -> RpcResult<()> { + let all_tx_hashes = self.pool.all_transaction_hashes(); + self.pool.remove_transactions(all_tx_hashes); + Ok(()) + } + fn subscribe_transaction_count( &self, pending: PendingSubscriptionSink, @@ -164,8 +184,10 @@ mod tests { // Send pool size repeatedly, with a 10-second delay loop { sleep(Duration::from_millis(delay)).await; - let message = SubscriptionMessage::from_json(&pool.pool_size().total) - .expect("serialize usize"); + let message = SubscriptionMessage::from( + serde_json::value::to_raw_value(&pool.pool_size().total) + .expect("serialize usize"), + ); // Just ignore errors if a client has dropped let _ = sink.send(message).await; @@ -184,6 +206,16 @@ mod tests { assert_eq!(count, 0); } + #[tokio::test(flavor = "multi_thread")] + async fn test_call_clear_txpool_http() { + let server_addr = start_server().await; + let uri = format!("http://{server_addr}"); + let client = HttpClientBuilder::default().build(&uri).unwrap(); + TxpoolExtApiClient::clear_txpool(&client).await.unwrap(); + let count = TxpoolExtApiClient::transaction_count(&client).await.unwrap(); + assert_eq!(count, 0); + } + #[tokio::test(flavor = "multi_thread")] async fn test_subscribe_transaction_count_ws() { let server_addr = start_server().await; diff --git a/examples/node-event-hooks/src/main.rs b/examples/node-event-hooks/src/main.rs index 60bc8c13250..fc72b936f5f 100644 --- a/examples/node-event-hooks/src/main.rs +++ b/examples/node-event-hooks/src/main.rs @@ -1,4 +1,4 @@ -//! Example for how hook into the node via the CLI extension mechanism without registering +//! Example for how to hook into the node via the CLI extension mechanism without registering //! additional arguments //! //! Run with diff --git a/examples/op-db-access/Cargo.toml b/examples/op-db-access/Cargo.toml new file mode 100644 index 00000000000..ae06e600b9c --- /dev/null +++ b/examples/op-db-access/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "example-op-db-access" +version = "0.0.0" +publish = false +edition.workspace = true +license.workspace = true + +[dependencies] +reth-op = { workspace = true, features = ["node"] } + +eyre.workspace = true diff --git a/examples/op-db-access/src/main.rs b/examples/op-db-access/src/main.rs new file mode 100644 index 00000000000..6afe8d25b35 --- /dev/null +++ b/examples/op-db-access/src/main.rs @@ -0,0 +1,23 @@ +//! Shows how to manually access the database + +use reth_op::{chainspec::BASE_MAINNET, node::OpNode, provider::providers::ReadOnlyConfig}; + +// Providers are zero-cost abstractions on top of an opened MDBX Transaction +// exposing a familiar API to query the chain's information without requiring knowledge +// of the inner tables. +// +// These abstractions do not include any caching and the user is responsible for doing that. +// Other parts of the code which include caching are parts of the `EthApi` abstraction. +fn main() -> eyre::Result<()> { + // The path to data directory, e.g. "~/.local/reth/share/base" + let datadir = std::env::var("RETH_DATADIR")?; + + // Instantiate a provider factory for Ethereum mainnet using the provided datadir path. + let factory = OpNode::provider_factory_builder() + .open_read_only(BASE_MAINNET.clone(), ReadOnlyConfig::from_datadir(datadir))?; + + // obtain a provider access that has direct access to the database. + let _provider = factory.provider(); + + Ok(()) +} diff --git a/examples/polygon-p2p/src/main.rs b/examples/polygon-p2p/src/main.rs index 8882a9f6c80..d4301ec0124 100644 --- a/examples/polygon-p2p/src/main.rs +++ b/examples/polygon-p2p/src/main.rs @@ -1,4 +1,4 @@ -//! Example for how hook into the polygon p2p network +//! Example for how to hook into the polygon p2p network //! //! Run with //! @@ -67,13 +67,13 @@ async fn main() { let net_handle = net_manager.handle(); let mut events = net_handle.event_listener(); - // NetworkManager is a long running task, let's spawn it + // NetworkManager is a long-running task, let's spawn it tokio::spawn(net_manager); info!("Looking for Polygon peers..."); while let Some(evt) = events.next().await { // For the sake of the example we only print the session established event - // with the chain specific details + // with the chain-specific details if let NetworkEvent::ActivePeerSession { info, .. } = evt { let SessionInfo { status, client_version, .. } = info; let chain = status.chain; @@ -81,5 +81,5 @@ async fn main() { } // More events here } - // We will be disconnected from peers since we are not able to answer to network requests + // We will be disconnected from peers since we are not able to respond to network requests } diff --git a/examples/precompile-cache/Cargo.toml b/examples/precompile-cache/Cargo.toml index 143e8f63598..637f99c1d1a 100644 --- a/examples/precompile-cache/Cargo.toml +++ b/examples/precompile-cache/Cargo.toml @@ -6,7 +6,6 @@ edition.workspace = true license.workspace = true [dependencies] -reth.workspace = true reth-ethereum = { workspace = true, features = ["test-utils", "evm", "node-api", "node"] } reth-tracing.workspace = true alloy-evm.workspace = true diff --git a/examples/precompile-cache/src/main.rs b/examples/precompile-cache/src/main.rs index f8ec197381d..fe748db4636 100644 --- a/examples/precompile-cache/src/main.rs +++ b/examples/precompile-cache/src/main.rs @@ -2,76 +2,75 @@ #![warn(unused_crate_dependencies)] -use alloy_evm::{eth::EthEvmContext, EvmFactory}; +use alloy_evm::{ + eth::EthEvmContext, + precompiles::{DynPrecompile, Precompile, PrecompileInput, PrecompilesMap}, + revm::{handler::EthPrecompiles, precompile::PrecompileId}, + Evm, EvmFactory, +}; use alloy_genesis::Genesis; -use alloy_primitives::{Address, Bytes}; +use alloy_primitives::Bytes; use parking_lot::RwLock; -use reth::{ - builder::{components::ExecutorBuilder, BuilderContext, NodeBuilder}, - tasks::TaskManager, -}; use reth_ethereum::{ chainspec::{Chain, ChainSpec}, evm::{ primitives::{Database, EvmEnv}, revm::{ - context::{Cfg, Context, TxEnv}, - context_interface::{ - result::{EVMError, HaltReason}, - ContextTr, - }, - handler::{EthPrecompiles, PrecompileProvider}, + context::{BlockEnv, Context, TxEnv}, + context_interface::result::{EVMError, HaltReason}, inspector::{Inspector, NoOpInspector}, - interpreter::{interpreter::EthInterpreter, InputsImpl, InterpreterResult}, + interpreter::interpreter::EthInterpreter, + precompile::PrecompileResult, primitives::hardfork::SpecId, MainBuilder, MainContext, }, }, node::{ api::{FullNodeTypes, NodeTypes}, + builder::{components::ExecutorBuilder, BuilderContext, NodeBuilder}, core::{args::RpcServerArgs, node_config::NodeConfig}, evm::EthEvm, node::EthereumAddOns, EthEvmConfig, EthereumNode, }, + tasks::TaskManager, EthPrimitives, }; use reth_tracing::{RethTracer, Tracer}; use schnellru::{ByLength, LruMap}; -use std::{collections::HashMap, sync::Arc}; +use std::sync::Arc; /// Type alias for the LRU cache used within the [`PrecompileCache`]. -type PrecompileLRUCache = LruMap<(SpecId, Bytes, u64), Result>; - -type WrappedEthEvm = EthEvm>; +type PrecompileLRUCache = LruMap<(Bytes, u64), PrecompileResult>; /// A cache for precompile inputs / outputs. /// -/// This assumes that the precompile is a standard precompile, as in `StandardPrecompileFn`, meaning -/// its inputs are only `(Bytes, u64)`. -/// -/// NOTE: This does not work with "context stateful precompiles", ie `ContextStatefulPrecompile` or -/// `ContextStatefulPrecompileMut`. They are explicitly banned. -#[derive(Debug, Default)] +/// This cache works with standard precompiles that take input data and gas limit as parameters. +/// The cache key is composed of the input bytes and gas limit, and the cached value is the +/// precompile execution result. +#[derive(Debug)] pub struct PrecompileCache { /// Caches for each precompile input / output. - cache: HashMap, + cache: PrecompileLRUCache, } /// Custom EVM factory. -#[derive(Debug, Clone, Default)] +#[derive(Debug, Clone)] #[non_exhaustive] pub struct MyEvmFactory { precompile_cache: Arc>, } impl EvmFactory for MyEvmFactory { - type Evm, EthInterpreter>> = WrappedEthEvm; + type Evm, EthInterpreter>> = + EthEvm; type Tx = TxEnv; type Error = EVMError; type HaltReason = HaltReason; type Context = EthEvmContext; type Spec = SpecId; + type BlockEnv = BlockEnv; + type Precompiles = PrecompilesMap; fn create_evm(&self, db: DB, input: EvmEnv) -> Self::Evm { let new_cache = self.precompile_cache.clone(); @@ -81,9 +80,15 @@ impl EvmFactory for MyEvmFactory { .with_cfg(input.cfg_env) .with_block(input.block_env) .build_mainnet_with_inspector(NoOpInspector {}) - .with_precompiles(WrappedPrecompile::new(EthPrecompiles::default(), new_cache)); + .with_precompiles(PrecompilesMap::from_static(EthPrecompiles::default().precompiles)); - EthEvm::new(evm, false) + let mut evm = EthEvm::new(evm, false); + + evm.precompiles_mut().map_precompiles(|_, precompile| { + WrappedPrecompile::wrap(precompile, new_cache.clone()) + }); + + evm } fn create_evm_with_inspector, EthInterpreter>>( @@ -98,94 +103,81 @@ impl EvmFactory for MyEvmFactory { /// A custom precompile that contains the cache and precompile it wraps. #[derive(Clone)] -pub struct WrappedPrecompile

{ +pub struct WrappedPrecompile { /// The precompile to wrap. - precompile: P, + precompile: DynPrecompile, /// The cache to use. cache: Arc>, - /// The spec id to use. - spec: SpecId, } -impl

WrappedPrecompile

{ - /// Given a [`PrecompileProvider`] and cache for a specific precompiles, create a +impl WrappedPrecompile { + fn new(precompile: DynPrecompile, cache: Arc>) -> Self { + Self { precompile, cache } + } + + /// Given a [`DynPrecompile`] and cache for a specific precompiles, create a /// wrapper that can be used inside Evm. - fn new(precompile: P, cache: Arc>) -> Self { - WrappedPrecompile { precompile, cache: cache.clone(), spec: SpecId::default() } + fn wrap(precompile: DynPrecompile, cache: Arc>) -> DynPrecompile { + let precompile_id = precompile.precompile_id().clone(); + let wrapped = Self::new(precompile, cache); + (precompile_id, move |input: PrecompileInput<'_>| -> PrecompileResult { + wrapped.call(input) + }) + .into() } } -impl> PrecompileProvider - for WrappedPrecompile

-{ - type Output = P::Output; - - fn set_spec(&mut self, spec: ::Spec) -> bool { - self.precompile.set_spec(spec.clone()); - self.spec = spec.into(); - true +impl Precompile for WrappedPrecompile { + fn precompile_id(&self) -> &PrecompileId { + self.precompile.precompile_id() } - fn run( - &mut self, - context: &mut CTX, - address: &Address, - inputs: &InputsImpl, - is_static: bool, - gas_limit: u64, - ) -> Result, String> { + fn call(&self, input: PrecompileInput<'_>) -> PrecompileResult { let mut cache = self.cache.write(); - let key = (self.spec, inputs.input.clone(), gas_limit); + let key = (Bytes::copy_from_slice(input.data), input.gas); // get the result if it exists - if let Some(precompiles) = cache.cache.get_mut(address) { - if let Some(result) = precompiles.get(&key) { - return result.clone().map(Some) - } + if let Some(result) = cache.cache.get(&key) { + return result.clone() } // call the precompile if cache miss - let output = self.precompile.run(context, address, inputs, is_static, gas_limit); - - if let Some(output) = output.clone().transpose() { - // insert the result into the cache - cache - .cache - .entry(*address) - .or_insert(PrecompileLRUCache::new(ByLength::new(1024))) - .insert(key, output); - } - - output - } + let output = self.precompile.call(input); - fn warm_addresses(&self) -> Box> { - self.precompile.warm_addresses() - } + // insert the result into the cache + cache.cache.insert(key, output.clone()); - fn contains(&self, address: &Address) -> bool { - self.precompile.contains(address) + output } } /// Builds a regular ethereum block executor that uses the custom EVM. -#[derive(Debug, Default, Clone)] +#[derive(Debug, Clone)] #[non_exhaustive] pub struct MyExecutorBuilder { /// The precompile cache to use for all executors. precompile_cache: Arc>, } +impl Default for MyExecutorBuilder { + fn default() -> Self { + let precompile_cache = PrecompileCache { + cache: LruMap::<(Bytes, u64), PrecompileResult>::new(ByLength::new(100)), + }; + Self { precompile_cache: Arc::new(RwLock::new(precompile_cache)) } + } +} + impl ExecutorBuilder for MyExecutorBuilder where Node: FullNodeTypes>, { - type EVM = EthEvmConfig; + type EVM = EthEvmConfig; async fn build_evm(self, ctx: &BuilderContext) -> eyre::Result { let evm_config = EthEvmConfig::new_with_evm_factory( ctx.chain_spec(), - MyEvmFactory { precompile_cache: self.precompile_cache.clone() }, + MyEvmFactory { precompile_cache: self.precompile_cache }, ); Ok(evm_config) } diff --git a/examples/rpc-db/Cargo.toml b/examples/rpc-db/Cargo.toml index 0a27ef565b0..3928a30f892 100644 --- a/examples/rpc-db/Cargo.toml +++ b/examples/rpc-db/Cargo.toml @@ -8,7 +8,6 @@ license.workspace = true [dependencies] futures.workspace = true jsonrpsee.workspace = true -reth.workspace = true reth-ethereum = { workspace = true, features = ["test-utils", "node", "provider", "pool", "network", "rpc"] } tokio = { workspace = true, features = ["full"] } eyre.workspace = true diff --git a/examples/rpc-db/src/main.rs b/examples/rpc-db/src/main.rs index 16ba5b6e406..97bd1debdcc 100644 --- a/examples/rpc-db/src/main.rs +++ b/examples/rpc-db/src/main.rs @@ -1,4 +1,4 @@ -//! Example illustrating how to run the ETH JSON RPC API as standalone over a DB file. +//! Example illustrating how to run the ETH JSON RPC API as a standalone over a DB file. //! //! Run with //! @@ -16,9 +16,9 @@ use std::{path::Path, sync::Arc}; -use reth::beacon_consensus::EthBeaconConsensus; use reth_ethereum::{ chainspec::ChainSpecBuilder, + consensus::EthBeaconConsensus, network::api::noop::NoopNetwork, node::{api::NodeTypesWithDBAdapter, EthEvmConfig, EthereumNode}, pool::noop::NoopTransactionPool, @@ -31,17 +31,17 @@ use reth_ethereum::{ builder::{RethRpcModule, RpcModuleBuilder, RpcServerConfig, TransportRpcModuleConfig}, EthApiBuilder, }, + tasks::TokioTaskExecutor, }; // Configuring the network parts, ideally also wouldn't need to think about this. use myrpc_ext::{MyRpcExt, MyRpcExtApiServer}; -use reth::tasks::TokioTaskExecutor; // Custom rpc extension pub mod myrpc_ext; #[tokio::main] async fn main() -> eyre::Result<()> { - // 1. Setup the DB + // 1. Set up the DB let db_path = std::env::var("RETH_DB_PATH")?; let db_path = Path::new(&db_path); let db = Arc::new(open_db_read_only( @@ -55,7 +55,7 @@ async fn main() -> eyre::Result<()> { StaticFileProvider::read_only(db_path.join("static_files"), true)?, ); - // 2. Setup the blockchain provider using only the database provider and a noop for the tree to + // 2. Set up the blockchain provider using only the database provider and a noop for the tree to // satisfy trait bounds. Tree is not used in this example since we are only operating on the // disk and don't handle new blocks/live sync etc, which is done by the blockchain tree. let provider = BlockchainProvider::new(factory)?; @@ -65,7 +65,7 @@ async fn main() -> eyre::Result<()> { // Rest is just noops that do nothing .with_noop_pool() .with_noop_network() - .with_executor(TokioTaskExecutor::default()) + .with_executor(Box::new(TokioTaskExecutor::default())) .with_evm_config(EthEvmConfig::new(spec.clone())) .with_consensus(EthBeaconConsensus::new(spec.clone())); diff --git a/examples/rpc-db/src/myrpc_ext.rs b/examples/rpc-db/src/myrpc_ext.rs index e472fa38b4c..68681ad587e 100644 --- a/examples/rpc-db/src/myrpc_ext.rs +++ b/examples/rpc-db/src/myrpc_ext.rs @@ -1,11 +1,10 @@ // Reth block related imports -use reth_ethereum::{provider::BlockReaderIdExt, Block}; +use reth_ethereum::{provider::BlockReaderIdExt, rpc::eth::EthResult, Block}; // Rpc related imports use jsonrpsee::proc_macros::rpc; -use reth::rpc::server_types::eth::EthResult; -/// trait interface for a custom rpc namespace: `MyRpc` +/// trait interface for a custom rpc namespace: `myrpcExt` /// /// This defines an additional namespace where all methods are configured as trait functions. #[rpc(server, namespace = "myrpcExt")] @@ -15,14 +14,14 @@ pub trait MyRpcExtApi { fn custom_method(&self) -> EthResult>; } -/// The type that implements `myRpc` rpc namespace trait +/// The type that implements `myrpcExt` rpc namespace trait pub struct MyRpcExt { pub provider: Provider, } impl MyRpcExtApiServer for MyRpcExt where - Provider: BlockReaderIdExt + 'static, + Provider: BlockReaderIdExt + 'static, { /// Showcasing how to implement a custom rpc method /// using the provider. diff --git a/examples/txpool-tracing/Cargo.toml b/examples/txpool-tracing/Cargo.toml index f53f2ad05d7..df72dd193f9 100644 --- a/examples/txpool-tracing/Cargo.toml +++ b/examples/txpool-tracing/Cargo.toml @@ -6,9 +6,12 @@ edition.workspace = true license.workspace = true [dependencies] -reth.workspace = true -reth-ethereum = { workspace = true, features = ["node", "pool", "cli"] } +reth-ethereum = { workspace = true, features = ["node", "pool", "cli", "rpc"] } + +alloy-primitives.workspace = true alloy-rpc-types-trace.workspace = true +alloy-network.workspace = true + clap = { workspace = true, features = ["derive"] } futures-util.workspace = true -alloy-primitives.workspace = true +eyre.workspace = true diff --git a/examples/txpool-tracing/src/main.rs b/examples/txpool-tracing/src/main.rs index a7986a675a5..f510a3f68b8 100644 --- a/examples/txpool-tracing/src/main.rs +++ b/examples/txpool-tracing/src/main.rs @@ -14,13 +14,15 @@ use alloy_primitives::Address; use alloy_rpc_types_trace::{parity::TraceType, tracerequest::TraceCallRequest}; use clap::Parser; use futures_util::StreamExt; -use reth::{builder::NodeHandle, rpc::types::TransactionRequest}; use reth_ethereum::{ cli::{chainspec::EthereumChainSpecParser, interface::Cli}, - node::EthereumNode, + node::{builder::NodeHandle, EthereumNode}, pool::TransactionPool, + rpc::eth::primitives::TransactionRequest, }; +mod submit; + fn main() { Cli::::parse() .run(|builder, args| async move { @@ -42,17 +44,17 @@ fn main() { let tx = event.transaction; println!("Transaction received: {tx:?}"); - if let Some(recipient) = tx.to() { - if args.is_match(&recipient) { - // trace the transaction with `trace_call` - let callrequest = - TransactionRequest::from_recovered_transaction(tx.to_consensus()); - let tracerequest = TraceCallRequest::new(callrequest) - .with_trace_type(TraceType::Trace); - if let Ok(trace_result) = traceapi.trace_call(tracerequest).await { - let hash = tx.hash(); - println!("trace result for transaction {hash}: {trace_result:?}"); - } + if let Some(recipient) = tx.to() && + args.is_match(&recipient) + { + // trace the transaction with `trace_call` + let callrequest = + TransactionRequest::from_recovered_transaction(tx.to_consensus()); + let tracerequest = + TraceCallRequest::new(callrequest).with_trace_type(TraceType::Trace); + if let Ok(trace_result) = traceapi.trace_call(tracerequest).await { + let hash = tx.hash(); + println!("trace result for transaction {hash}: {trace_result:?}"); } } } @@ -66,7 +68,7 @@ fn main() { /// Our custom cli args extension that adds one flag to reth default CLI. #[derive(Debug, Clone, Default, clap::Args)] struct RethCliTxpoolExt { - /// recipients addresses that we want to trace + /// recipients' addresses that we want to trace #[arg(long, value_delimiter = ',')] pub recipients: Vec

, } diff --git a/examples/txpool-tracing/src/submit.rs b/examples/txpool-tracing/src/submit.rs new file mode 100644 index 00000000000..f3e0de16edb --- /dev/null +++ b/examples/txpool-tracing/src/submit.rs @@ -0,0 +1,124 @@ +//! Transaction submission functionality for the txpool tracing example +#![allow(unused)] +#![allow(clippy::too_many_arguments)] + +use alloy_network::{Ethereum, EthereumWallet, NetworkWallet, TransactionBuilder}; +use alloy_primitives::{Address, TxHash, U256}; +use futures_util::StreamExt; +use reth_ethereum::{ + node::api::{FullNodeComponents, NodeTypes}, + pool::{ + AddedTransactionOutcome, PoolTransaction, TransactionEvent, TransactionOrigin, + TransactionPool, + }, + primitives::SignerRecoverable, + rpc::eth::primitives::TransactionRequest, + EthPrimitives, TransactionSigned, +}; + +/// Submit a transaction to the transaction pool +/// +/// This function demonstrates how to create, sign, and submit a transaction +/// to the reth transaction pool. +pub async fn submit_transaction( + node: &FC, + wallet: &EthereumWallet, + to: Address, + data: Vec, + nonce: u64, + chain_id: u64, + gas_limit: u64, + max_priority_fee_per_gas: u128, + max_fee_per_gas: u128, +) -> eyre::Result +where + // This enforces `EthPrimitives` types for this node, which unlocks the proper conversions when + FC: FullNodeComponents>, +{ + // Create the transaction request + let request = TransactionRequest::default() + .with_to(to) + .with_input(data) + .with_nonce(nonce) + .with_chain_id(chain_id) + .with_gas_limit(gas_limit) + .with_max_priority_fee_per_gas(max_priority_fee_per_gas) + .with_max_fee_per_gas(max_fee_per_gas); + + // Sign the transaction + let transaction: TransactionSigned = + NetworkWallet::::sign_request(wallet, request).await?.into(); + // Get the transaction hash before submitting + let tx_hash = *transaction.hash(); + + // Recover the transaction + let transaction = transaction.try_into_recovered()?; + + let mut tx_events = node + .pool() + .add_consensus_transaction_and_subscribe(transaction, TransactionOrigin::Local) + .await + .map_err(|e| eyre::eyre!("Pool error: {e}"))?; + + // Wait for the transaction to be added to the pool + while let Some(event) = tx_events.next().await { + match event { + TransactionEvent::Mined(_) => { + println!("Transaction was mined: {:?}", tx_events.hash()); + break; + } + TransactionEvent::Pending => { + println!("Transaction added to pending pool: {:?}", tx_events.hash()); + break; + } + TransactionEvent::Discarded => { + return Err(eyre::eyre!("Transaction discarded: {:?}", tx_events.hash(),)); + } + _ => { + // Continue waiting for added or rejected event + } + } + } + + Ok(tx_hash) +} + +/// Helper function to submit a simple ETH transfer transaction +/// +/// This will first populate a tx request, sign it then submit to the pool in the required format. +pub async fn submit_eth_transfer( + node: &FC, + wallet: &EthereumWallet, + to: Address, + value: U256, + nonce: u64, + chain_id: u64, + gas_limit: u64, + max_priority_fee_per_gas: u128, + max_fee_per_gas: u128, +) -> eyre::Result +where + FC: FullNodeComponents>, +{ + // Create the transaction request for ETH transfer + let request = TransactionRequest::default() + .with_to(to) + .with_value(value) + .with_nonce(nonce) + .with_chain_id(chain_id) + .with_gas_limit(gas_limit) + .with_max_priority_fee_per_gas(max_priority_fee_per_gas) + .with_max_fee_per_gas(max_fee_per_gas); + + // Sign the transaction + let transaction: TransactionSigned = + NetworkWallet::::sign_request(wallet, request).await?.into(); + // Recover the transaction + let transaction = transaction.try_into_recovered()?; + + // Submit the transaction to the pool + node.pool() + .add_consensus_transaction(transaction, TransactionOrigin::Local) + .await + .map_err(|e| eyre::eyre!("Pool error: {e}")) +} diff --git a/flake.lock b/flake.lock new file mode 100644 index 00000000000..4efd90828f9 --- /dev/null +++ b/flake.lock @@ -0,0 +1,116 @@ +{ + "nodes": { + "crane": { + "locked": { + "lastModified": 1760924934, + "narHash": "sha256-tuuqY5aU7cUkR71sO2TraVKK2boYrdW3gCSXUkF4i44=", + "owner": "ipetkov", + "repo": "crane", + "rev": "c6b4d5308293d0d04fcfeee92705017537cad02f", + "type": "github" + }, + "original": { + "owner": "ipetkov", + "repo": "crane", + "type": "github" + } + }, + "fenix": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ], + "rust-analyzer-src": "rust-analyzer-src" + }, + "locked": { + "lastModified": 1761720242, + "narHash": "sha256-Zi9nWw68oUDMVOhf/+Z97wVbNV2K7eEAGZugQKqU7xw=", + "owner": "nix-community", + "repo": "fenix", + "rev": "8e4d32f4cc12b3f106af6e4515b36ac046a1ec91", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "fenix", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1731603435, + "narHash": "sha256-CqCX4JG7UiHvkrBTpYC3wcEurvbtTADLbo3Ns2CEoL8=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "8b27c1239e5c421a2bbc2c65d52e4a6fbf2ff296", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "24.11", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "crane": "crane", + "fenix": "fenix", + "nixpkgs": "nixpkgs", + "utils": "utils" + } + }, + "rust-analyzer-src": { + "flake": false, + "locked": { + "lastModified": 1761686505, + "narHash": "sha256-jX6UrGS/hABDaM4jdx3+xgH3KCHP2zKHeTa8CD5myEo=", + "owner": "rust-lang", + "repo": "rust-analyzer", + "rev": "d08d54f3c10dfa41033eb780c3bddb50e09d30fc", + "type": "github" + }, + "original": { + "owner": "rust-lang", + "ref": "nightly", + "repo": "rust-analyzer", + "type": "github" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 00000000000..512b69e3660 --- /dev/null +++ b/flake.nix @@ -0,0 +1,131 @@ +{ + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/24.11"; + utils.url = "github:numtide/flake-utils"; + crane.url = "github:ipetkov/crane"; + + fenix = { + url = "github:nix-community/fenix"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + }; + + outputs = + { + nixpkgs, + utils, + crane, + fenix, + ... + }: + utils.lib.eachDefaultSystem ( + system: + let + pkgs = import nixpkgs { inherit system; }; + + # A useful helper for folding a list of `prevSet -> newSet` functions + # into an attribute set. + composeAttrOverrides = defaultAttrs: overrides: builtins.foldl' + (acc: f: acc // (f acc)) defaultAttrs overrides; + + cargoTarget = pkgs.stdenv.hostPlatform.rust.rustcTargetSpec; + cargoTargetEnvVar = builtins.replaceStrings ["-"] ["_"] + (pkgs.lib.toUpper cargoTarget); + + cargoTOML = builtins.fromTOML (builtins.readFile ./Cargo.toml); + packageVersion = cargoTOML.workspace.package.version; + + rustStable = fenix.packages.${system}.stable.withComponents [ + "cargo" "rustc" "rust-src" "clippy" + ]; + + rustNightly = fenix.packages.${system}.latest; + + craneLib = (crane.mkLib pkgs).overrideToolchain rustStable; + + nativeBuildInputs = [ + pkgs.pkg-config + pkgs.libgit2 + pkgs.perl + ]; + + withClang = prev: { + buildInputs = prev.buildInputs or [] ++ [ + pkgs.clang + ]; + LIBCLANG_PATH = "${pkgs.libclang.lib}/lib"; + }; + + withMaxPerf = prev: { + cargoBuildCommand = "cargo build --profile=maxperf"; + cargoExtraArgs = prev.cargoExtraArgs or "" + " --features=jemalloc,asm-keccak"; + RUSTFLAGS = prev.RUSTFLAGS or [] ++ [ + "-Ctarget-cpu=native" + ]; + }; + + withMold = prev: { + buildInputs = prev.buildInputs or [] ++ [ + pkgs.mold + ]; + "CARGO_TARGET_${cargoTargetEnvVar}_LINKER" = "${pkgs.llvmPackages.clangUseLLVM}/bin/clang"; + RUSTFLAGS = prev.RUSTFLAGS or [] ++ [ + "-Clink-arg=-fuse-ld=${pkgs.mold}/bin/mold" + ]; + }; + + withOp = prev: { + cargoExtraArgs = prev.cargoExtraArgs or "" + " -p op-reth --bin=op-reth"; + }; + + mkReth = overrides: craneLib.buildPackage (composeAttrOverrides { + pname = "reth"; + version = packageVersion; + src = ./.; + inherit nativeBuildInputs; + doCheck = false; + } overrides); + + in + { + packages = rec { + + reth = mkReth ([ + withClang + withMaxPerf + ] ++ pkgs.lib.optionals pkgs.stdenv.isLinux [ + withMold + ]); + + op-reth = mkReth ([ + withClang + withMaxPerf + withOp + ] ++ pkgs.lib.optionals pkgs.stdenv.isLinux [ + withMold + ]); + + default = reth; + }; + + devShell = let + overrides = [ + withClang + ] ++ pkgs.lib.optionals pkgs.stdenv.isLinux [ + withMold + ]; + in craneLib.devShell (composeAttrOverrides { + packages = nativeBuildInputs ++ [ + rustNightly.rust-analyzer + rustNightly.rustfmt + pkgs.cargo-nextest + ]; + + # Remove the hardening added by nix to fix jmalloc compilation error. + # More info: https://github.com/tikv/jemallocator/issues/108 + hardeningDisable = [ "fortify" ]; + + } overrides); + } + ); +} diff --git a/rustfmt.toml b/rustfmt.toml index 68c3c93033d..bf86a535083 100644 --- a/rustfmt.toml +++ b/rustfmt.toml @@ -1,3 +1,4 @@ +style_edition = "2021" reorder_imports = true imports_granularity = "Crate" use_small_heuristics = "Max" diff --git a/testing/ef-tests/.gitignore b/testing/ef-tests/.gitignore index eae5bd973fc..0bf9998816a 100644 --- a/testing/ef-tests/.gitignore +++ b/testing/ef-tests/.gitignore @@ -1 +1,2 @@ -ethereum-tests \ No newline at end of file +ethereum-tests +execution-spec-tests \ No newline at end of file diff --git a/testing/ef-tests/Cargo.toml b/testing/ef-tests/Cargo.toml index 003fbc82e6c..e9cf465a98d 100644 --- a/testing/ef-tests/Cargo.toml +++ b/testing/ef-tests/Cargo.toml @@ -28,11 +28,11 @@ reth-evm.workspace = true reth-evm-ethereum.workspace = true reth-ethereum-consensus.workspace = true reth-revm = { workspace = true, features = ["std", "witness"] } -reth-stateless = { workspace = true } +reth-stateless = { workspace = true, features = ["secp256k1"] } reth-tracing.workspace = true reth-trie.workspace = true reth-trie-db.workspace = true -revm = { workspace = true, features = ["secp256k1", "blst", "c-kzg"] } +revm = { workspace = true, features = ["secp256k1", "blst", "c-kzg", "memory_limit"] } alloy-rlp.workspace = true alloy-primitives.workspace = true @@ -45,9 +45,3 @@ serde.workspace = true serde_json.workspace = true thiserror.workspace = true rayon.workspace = true -tracing.workspace = true -# TODO: When we build for a windows target on an ubuntu runner, crunchy tries to -# get the wrong path, update this when the workflow has been updated -# -# See: https://github.com/eira-fransham/crunchy/issues/13 -crunchy = "=0.2.2" diff --git a/testing/ef-tests/src/cases/blockchain_test.rs b/testing/ef-tests/src/cases/blockchain_test.rs index 5e32ee873e6..c54ef2ad7b1 100644 --- a/testing/ef-tests/src/cases/blockchain_test.rs +++ b/testing/ef-tests/src/cases/blockchain_test.rs @@ -10,47 +10,57 @@ use reth_chainspec::ChainSpec; use reth_consensus::{Consensus, HeaderValidator}; use reth_db_common::init::{insert_genesis_hashes, insert_genesis_history, insert_genesis_state}; use reth_ethereum_consensus::{validate_block_post_execution, EthBeaconConsensus}; -use reth_ethereum_primitives::Block; +use reth_ethereum_primitives::{Block, TransactionSigned}; use reth_evm::{execute::Executor, ConfigureEvm}; -use reth_evm_ethereum::execute::EthExecutorProvider; -use reth_primitives_traits::{RecoveredBlock, SealedBlock}; +use reth_evm_ethereum::EthEvmConfig; +use reth_primitives_traits::{Block as BlockTrait, RecoveredBlock, SealedBlock}; use reth_provider::{ test_utils::create_test_provider_factory_with_chain_spec, BlockWriter, DatabaseProviderFactory, ExecutionOutcome, HeaderProvider, HistoryWriter, OriginalValuesKnown, StateProofProvider, - StateWriter, StorageLocation, + StateWriter, StaticFileProviderFactory, StaticFileSegment, StaticFileWriter, }; use reth_revm::{database::StateProviderDatabase, witness::ExecutionWitnessRecord, State}; -use reth_stateless::{validation::stateless_validation, ExecutionWitness}; +use reth_stateless::{ + trie::StatelessSparseTrie, validation::stateless_validation_with_trie, ExecutionWitness, + UncompressedPublicKey, +}; use reth_trie::{HashedPostState, KeccakKeyHasher, StateRoot}; use reth_trie_db::DatabaseStateRoot; -use std::{collections::BTreeMap, fs, path::Path, sync::Arc}; +use std::{ + collections::BTreeMap, + fs, + path::{Path, PathBuf}, + sync::Arc, +}; /// A handler for the blockchain test suite. #[derive(Debug)] pub struct BlockchainTests { - suite: String, + suite_path: PathBuf, } impl BlockchainTests { - /// Create a new handler for a subset of the blockchain test suite. - pub const fn new(suite: String) -> Self { - Self { suite } + /// Create a new suite for tests with blockchain tests format. + pub const fn new(suite_path: PathBuf) -> Self { + Self { suite_path } } } impl Suite for BlockchainTests { type Case = BlockchainTestCase; - fn suite_name(&self) -> String { - format!("BlockchainTests/{}", self.suite) + fn suite_path(&self) -> &Path { + &self.suite_path } } /// An Ethereum blockchain test. #[derive(Debug, PartialEq, Eq)] pub struct BlockchainTestCase { - tests: BTreeMap, - skip: bool, + /// The tests within this test case. + pub tests: BTreeMap, + /// Whether to skip this test case. + pub skip: bool, } impl BlockchainTestCase { @@ -91,39 +101,45 @@ impl BlockchainTestCase { } /// Execute a single `BlockchainTest`, validating the outcome against the - /// expectations encoded in the JSON file. - fn run_single_case(name: &str, case: &BlockchainTest) -> Result<(), Error> { + /// expectations encoded in the JSON file. Returns the list of executed blocks + /// with their execution witnesses. + pub fn run_single_case( + name: &str, + case: &BlockchainTest, + ) -> Result, ExecutionWitness)>, Error> { let expectation = Self::expected_failure(case); match run_case(case) { // All blocks executed successfully. - Ok(()) => { + Ok(program_inputs) => { // Check if the test case specifies that it should have failed if let Some((block, msg)) = expectation { Err(Error::Assertion(format!( "Test case: {name}\nExpected failure at block {block} - {msg}, but all blocks succeeded", ))) } else { - Ok(()) + Ok(program_inputs) } } // A block processing failure occurred. - Err(Error::BlockProcessingFailed { block_number }) => match expectation { - // It happened on exactly the block we were told to fail on - Some((expected, _)) if block_number == expected => Ok(()), - - // Uncle side‑chain edge case, we accept as long as it failed. - // But we don't check the exact block number. - _ if Self::is_uncle_sidechain_case(name) => Ok(()), - - // Expected failure, but block number does not match - Some((expected, _)) => Err(Error::Assertion(format!( - "Test case: {name}\nExpected failure at block {expected}\nGot failure at block {block_number}", - ))), - - // No failure expected at all - bubble up original error. - None => Err(Error::BlockProcessingFailed { block_number }), - }, + Err(Error::BlockProcessingFailed { block_number, partial_program_inputs, err }) => { + match expectation { + // It happened on exactly the block we were told to fail on + Some((expected, _)) if block_number == expected => Ok(partial_program_inputs), + + // Uncle side‑chain edge case, we accept as long as it failed. + // But we don't check the exact block number. + _ if Self::is_uncle_sidechain_case(name) => Ok(partial_program_inputs), + + // Expected failure, but block number does not match + Some((expected, _)) => Err(Error::Assertion(format!( + "Test case: {name}\nExpected failure at block {expected}\nGot failure at block {block_number}", + ))), + + // No failure expected at all - bubble up original error. + None => Err(Error::BlockProcessingFailed { block_number, partial_program_inputs, err }), + } + } // Non‑processing error – forward as‑is. // @@ -131,7 +147,7 @@ impl BlockchainTestCase { // Since it is unexpected, we treat it as a test failure. // // One reason for this happening is when one forgets to wrap the error from `run_case` - // so that it produces a `Error::BlockProcessingFailed` + // so that it produces an `Error::BlockProcessingFailed` Err(other) => Err(other), } } @@ -157,7 +173,7 @@ impl Case for BlockchainTestCase { fn run(&self) -> Result<(), Error> { // If the test is marked for skipping, return a Skipped error immediately. if self.skip { - return Err(Error::Skipped) + return Err(Error::Skipped); } // Iterate through test cases, filtering by the network type to exclude specific forks. @@ -165,14 +181,14 @@ impl Case for BlockchainTestCase { .iter() .filter(|(_, case)| !Self::excluded_fork(case.network)) .par_bridge() - .try_for_each(|(name, case)| Self::run_single_case(name, case))?; + .try_for_each(|(name, case)| Self::run_single_case(name, case).map(|_| ()))?; Ok(()) } } -/// Executes a single `BlockchainTest`, returning an error if the blockchain state -/// does not match the expected outcome after all blocks are executed. +/// Executes a single `BlockchainTest` returning an error as soon as any block has a consensus +/// validation failure. /// /// A `BlockchainTest` represents a self-contained scenario: /// - It initializes a fresh blockchain state. @@ -181,9 +197,13 @@ impl Case for BlockchainTestCase { /// outcome. /// /// Returns: -/// - `Ok(())` if all blocks execute successfully and the final state is correct. -/// - `Err(Error)` if any block fails to execute correctly, or if the post-state validation fails. -fn run_case(case: &BlockchainTest) -> Result<(), Error> { +/// - `Ok(_)` if all blocks execute successfully, returning recovered blocks and full block +/// execution witness. +/// - `Err(Error)` if any block fails to execute correctly, returning a partial block execution +/// witness if the error is of variant `BlockProcessingFailed`. +fn run_case( + case: &BlockchainTest, +) -> Result, ExecutionWitness)>, Error> { // Create a new test database and initialize a provider for the test case. let chain_spec: Arc = Arc::new(case.network.into()); let factory = create_test_provider_factory_with_chain_spec(chain_spec.clone()); @@ -198,21 +218,28 @@ fn run_case(case: &BlockchainTest) -> Result<(), Error> { .unwrap(); provider - .insert_block(genesis_block.clone(), StorageLocation::Database) - .map_err(|_| Error::BlockProcessingFailed { block_number: 0 })?; + .insert_block(genesis_block.clone()) + .map_err(|err| Error::block_failed(0, Default::default(), err))?; + + // Increment block number for receipts static file + provider + .static_file_provider() + .latest_writer(StaticFileSegment::Receipts) + .and_then(|mut writer| writer.increment_block(0)) + .map_err(|err| Error::block_failed(0, Default::default(), err))?; let genesis_state = case.pre.clone().into_genesis_state(); insert_genesis_state(&provider, genesis_state.iter()) - .map_err(|_| Error::BlockProcessingFailed { block_number: 0 })?; + .map_err(|err| Error::block_failed(0, Default::default(), err))?; insert_genesis_hashes(&provider, genesis_state.iter()) - .map_err(|_| Error::BlockProcessingFailed { block_number: 0 })?; + .map_err(|err| Error::block_failed(0, Default::default(), err))?; insert_genesis_history(&provider, genesis_state.iter()) - .map_err(|_| Error::BlockProcessingFailed { block_number: 0 })?; + .map_err(|err| Error::block_failed(0, Default::default(), err))?; // Decode blocks let blocks = decode_blocks(&case.blocks)?; - let executor_provider = EthExecutorProvider::ethereum(chain_spec.clone()); + let executor_provider = EthEvmConfig::ethereum(chain_spec.clone()); let mut parent = genesis_block; let mut program_inputs = Vec::new(); @@ -222,12 +249,19 @@ fn run_case(case: &BlockchainTest) -> Result<(), Error> { // Insert the block into the database provider - .insert_block(block.clone(), StorageLocation::Database) - .map_err(|_| Error::BlockProcessingFailed { block_number })?; + .insert_block(block.clone()) + .map_err(|err| Error::block_failed(block_number, Default::default(), err))?; + // Commit static files, so we can query the headers for stateless execution below + provider + .static_file_provider() + .commit() + .map_err(|err| Error::block_failed(block_number, Default::default(), err))?; // Consensus checks before block execution - pre_execution_checks(chain_spec.clone(), &parent, block) - .map_err(|_| Error::BlockProcessingFailed { block_number })?; + pre_execution_checks(chain_spec.clone(), &parent, block).map_err(|err| { + program_inputs.push((block.clone(), execution_witness_with_parent(&parent))); + Error::block_failed(block_number, program_inputs.clone(), err) + })?; let mut witness_record = ExecutionWitnessRecord::default(); @@ -237,14 +271,14 @@ fn run_case(case: &BlockchainTest) -> Result<(), Error> { let executor = executor_provider.batch_executor(state_db); let output = executor - .execute_with_state_closure(&(*block).clone(), |statedb: &State<_>| { + .execute_with_state_closure_always(&(*block).clone(), |statedb: &State<_>| { witness_record.record_executed_state(statedb); }) - .map_err(|_| Error::BlockProcessingFailed { block_number })?; + .map_err(|err| Error::block_failed(block_number, program_inputs.clone(), err))?; // Consensus checks after block execution validate_block_post_execution(block, &chain_spec, &output.receipts, &output.requests) - .map_err(|_| Error::BlockProcessingFailed { block_number })?; + .map_err(|err| Error::block_failed(block_number, program_inputs.clone(), err))?; // Generate the stateless witness // TODO: Most of this code is copy-pasted from debug_executionWitness @@ -278,52 +312,71 @@ fn run_case(case: &BlockchainTest) -> Result<(), Error> { HashedPostState::from_bundle_state::(output.state.state()); let (computed_state_root, _) = StateRoot::overlay_root_with_updates(provider.tx_ref(), hashed_state.clone()) - .map_err(|_| Error::BlockProcessingFailed { block_number })?; + .map_err(|err| Error::block_failed(block_number, program_inputs.clone(), err))?; if computed_state_root != block.state_root { - return Err(Error::BlockProcessingFailed { block_number }) + return Err(Error::block_failed( + block_number, + program_inputs.clone(), + Error::Assertion("state root mismatch".to_string()), + )); } // Commit the post state/state diff to the database provider - .write_state( - &ExecutionOutcome::single(block.number, output), - OriginalValuesKnown::Yes, - StorageLocation::Database, - ) - .map_err(|_| Error::BlockProcessingFailed { block_number })?; + .write_state(&ExecutionOutcome::single(block.number, output), OriginalValuesKnown::Yes) + .map_err(|err| Error::block_failed(block_number, program_inputs.clone(), err))?; provider .write_hashed_state(&hashed_state.into_sorted()) - .map_err(|_| Error::BlockProcessingFailed { block_number })?; + .map_err(|err| Error::block_failed(block_number, program_inputs.clone(), err))?; provider .update_history_indices(block.number..=block.number) - .map_err(|_| Error::BlockProcessingFailed { block_number })?; + .map_err(|err| Error::block_failed(block_number, program_inputs.clone(), err))?; // Since there were no errors, update the parent block parent = block.clone() } - // Validate the post-state for the test case. - // - // If we get here then it means that the post-state root checks - // made after we execute each block was successful. - // - // If an error occurs here, then it is: - // - Either an issue with the test setup - // - Possibly an error in the test case where the post-state root in the last block does not - // match the post-state values. - let expected_post_state = case.post_state.as_ref().ok_or(Error::MissingPostState)?; - for (&address, account) in expected_post_state { - account.assert_db(address, provider.tx_ref())?; + match &case.post_state { + Some(expected_post_state) => { + // Validate the post-state for the test case. + // + // If we get here then it means that the post-state root checks + // made after we execute each block was successful. + // + // If an error occurs here, then it is: + // - Either an issue with the test setup + // - Possibly an error in the test case where the post-state root in the last block does + // not match the post-state values. + for (address, account) in expected_post_state { + account.assert_db(*address, provider.tx_ref())?; + } + } + None => { + // Some tests may not have post-state (e.g., state-heavy benchmark tests). + // In this case, we can skip the post-state validation. + } } // Now validate using the stateless client if everything else passes - for (block, execution_witness) in program_inputs { - stateless_validation(block, execution_witness, chain_spec.clone()) - .expect("stateless validation failed"); + for (recovered_block, execution_witness) in &program_inputs { + let block = recovered_block.clone().into_block(); + + // Recover the actual public keys from the transaction signatures + let public_keys = recover_signers(block.body().transactions()) + .expect("Failed to recover public keys from transaction signatures"); + + stateless_validation_with_trie::( + block, + public_keys, + execution_witness.clone(), + chain_spec.clone(), + EthEvmConfig::new(chain_spec.clone()), + ) + .expect("stateless validation failed"); } - Ok(()) + Ok(program_inputs) } fn decode_blocks( @@ -336,12 +389,12 @@ fn decode_blocks( let block_number = (block_index + 1) as u64; let decoded = SealedBlock::::decode(&mut block.rlp.as_ref()) - .map_err(|_| Error::BlockProcessingFailed { block_number })?; + .map_err(|err| Error::block_failed(block_number, Default::default(), err))?; let recovered_block = decoded .clone() .try_recover() - .map_err(|_| Error::BlockProcessingFailed { block_number })?; + .map_err(|err| Error::block_failed(block_number, Default::default(), err))?; blocks.push(recovered_block); } @@ -370,6 +423,26 @@ fn pre_execution_checks( Ok(()) } +/// Recover public keys from transaction signatures. +fn recover_signers<'a, I>(txs: I) -> Result, Box> +where + I: IntoIterator, +{ + txs.into_iter() + .enumerate() + .map(|(i, tx)| { + tx.signature() + .recover_from_prehash(&tx.signature_hash()) + .map(|keys| { + UncompressedPublicKey( + keys.to_encoded_point(false).as_bytes().try_into().unwrap(), + ) + }) + .map_err(|e| format!("failed to recover signature for tx #{i}: {e}").into()) + }) + .collect::, _>>() +} + /// Returns whether the test at the given path should be skipped. /// /// Some tests are edge cases that cannot happen on mainnet, while others are skipped for @@ -390,7 +463,7 @@ pub fn should_skip(path: &Path) -> bool { | "typeTwoBerlin.json" // Test checks if nonce overflows. We are handling this correctly but we are not parsing - // exception in testsuite There are more nonce overflow tests that are in internal + // exception in testsuite. There are more nonce overflow tests that are internal // call/create, and those tests are passing and are enabled. | "CreateTransactionHighNonce.json" @@ -436,3 +509,9 @@ fn path_contains(path_str: &str, rhs: &[&str]) -> bool { let rhs = rhs.join(std::path::MAIN_SEPARATOR_STR); path_str.contains(&rhs) } + +fn execution_witness_with_parent(parent: &RecoveredBlock) -> ExecutionWitness { + let mut serialized_header = Vec::new(); + parent.header().encode(&mut serialized_header); + ExecutionWitness { headers: vec![serialized_header.into()], ..Default::default() } +} diff --git a/testing/ef-tests/src/lib.rs b/testing/ef-tests/src/lib.rs index ca5e47d2d3b..fc9beda0f84 100644 --- a/testing/ef-tests/src/lib.rs +++ b/testing/ef-tests/src/lib.rs @@ -5,7 +5,7 @@ html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256", issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] use reth_revm as _; use revm as _; diff --git a/testing/ef-tests/src/models.rs b/testing/ef-tests/src/models.rs index 6cad5331e59..49c49bf1936 100644 --- a/testing/ef-tests/src/models.rs +++ b/testing/ef-tests/src/models.rs @@ -5,7 +5,7 @@ use alloy_consensus::Header as RethHeader; use alloy_eips::eip4895::Withdrawals; use alloy_genesis::GenesisAccount; use alloy_primitives::{keccak256, Address, Bloom, Bytes, B256, B64, U256}; -use reth_chainspec::{ChainSpec, ChainSpecBuilder}; +use reth_chainspec::{ChainSpec, ChainSpecBuilder, EthereumHardfork, ForkCondition}; use reth_db_api::{cursor::DbDupCursorRO, tables, transaction::DbTx}; use reth_primitives_traits::SealedHeader; use serde::Deserialize; @@ -294,9 +294,14 @@ pub enum ForkSpec { /// London London, /// Paris aka The Merge + #[serde(alias = "Paris")] Merge, + /// Paris to Shanghai at time 15k + ParisToShanghaiAtTime15k, /// Shanghai Shanghai, + /// Shanghai to Cancun at time 15k + ShanghaiToCancunAtTime15k, /// Merge EOF test #[serde(alias = "Merge+3540+3670")] MergeEOF, @@ -308,39 +313,63 @@ pub enum ForkSpec { MergePush0, /// Cancun Cancun, + /// Cancun to Prague at time 15k + CancunToPragueAtTime15k, /// Prague Prague, } impl From for ChainSpec { fn from(fork_spec: ForkSpec) -> Self { - let spec_builder = ChainSpecBuilder::mainnet(); + let spec_builder = ChainSpecBuilder::mainnet().reset(); match fork_spec { ForkSpec::Frontier => spec_builder.frontier_activated(), - ForkSpec::Homestead | ForkSpec::FrontierToHomesteadAt5 => { - spec_builder.homestead_activated() - } - ForkSpec::EIP150 | ForkSpec::HomesteadToDaoAt5 | ForkSpec::HomesteadToEIP150At5 => { - spec_builder.tangerine_whistle_activated() - } + ForkSpec::FrontierToHomesteadAt5 => spec_builder + .frontier_activated() + .with_fork(EthereumHardfork::Homestead, ForkCondition::Block(5)), + ForkSpec::Homestead => spec_builder.homestead_activated(), + ForkSpec::HomesteadToDaoAt5 => spec_builder + .homestead_activated() + .with_fork(EthereumHardfork::Dao, ForkCondition::Block(5)), + ForkSpec::HomesteadToEIP150At5 => spec_builder + .homestead_activated() + .with_fork(EthereumHardfork::Tangerine, ForkCondition::Block(5)), + ForkSpec::EIP150 => spec_builder.tangerine_whistle_activated(), ForkSpec::EIP158 => spec_builder.spurious_dragon_activated(), - ForkSpec::Byzantium | - ForkSpec::EIP158ToByzantiumAt5 | - ForkSpec::ConstantinopleFix | - ForkSpec::ByzantiumToConstantinopleFixAt5 => spec_builder.byzantium_activated(), + ForkSpec::EIP158ToByzantiumAt5 => spec_builder + .spurious_dragon_activated() + .with_fork(EthereumHardfork::Byzantium, ForkCondition::Block(5)), + ForkSpec::Byzantium => spec_builder.byzantium_activated(), + ForkSpec::ByzantiumToConstantinopleAt5 => spec_builder + .byzantium_activated() + .with_fork(EthereumHardfork::Constantinople, ForkCondition::Block(5)), + ForkSpec::ByzantiumToConstantinopleFixAt5 => spec_builder + .byzantium_activated() + .with_fork(EthereumHardfork::Petersburg, ForkCondition::Block(5)), + ForkSpec::Constantinople => spec_builder.constantinople_activated(), + ForkSpec::ConstantinopleFix => spec_builder.petersburg_activated(), ForkSpec::Istanbul => spec_builder.istanbul_activated(), ForkSpec::Berlin => spec_builder.berlin_activated(), - ForkSpec::London | ForkSpec::BerlinToLondonAt5 => spec_builder.london_activated(), + ForkSpec::BerlinToLondonAt5 => spec_builder + .berlin_activated() + .with_fork(EthereumHardfork::London, ForkCondition::Block(5)), + ForkSpec::London => spec_builder.london_activated(), ForkSpec::Merge | ForkSpec::MergeEOF | ForkSpec::MergeMeterInitCode | ForkSpec::MergePush0 => spec_builder.paris_activated(), + ForkSpec::ParisToShanghaiAtTime15k => spec_builder + .paris_activated() + .with_fork(EthereumHardfork::Shanghai, ForkCondition::Timestamp(15_000)), ForkSpec::Shanghai => spec_builder.shanghai_activated(), + ForkSpec::ShanghaiToCancunAtTime15k => spec_builder + .shanghai_activated() + .with_fork(EthereumHardfork::Cancun, ForkCondition::Timestamp(15_000)), ForkSpec::Cancun => spec_builder.cancun_activated(), - ForkSpec::ByzantiumToConstantinopleAt5 | ForkSpec::Constantinople => { - panic!("Overridden with PETERSBURG") - } + ForkSpec::CancunToPragueAtTime15k => spec_builder + .cancun_activated() + .with_fork(EthereumHardfork::Prague, ForkCondition::Timestamp(15_000)), ForkSpec::Prague => spec_builder.prague_activated(), } .build() diff --git a/testing/ef-tests/src/result.rs b/testing/ef-tests/src/result.rs index a1bed359b07..481d1fe7700 100644 --- a/testing/ef-tests/src/result.rs +++ b/testing/ef-tests/src/result.rs @@ -2,7 +2,10 @@ use crate::Case; use reth_db::DatabaseError; +use reth_ethereum_primitives::Block; +use reth_primitives_traits::RecoveredBlock; use reth_provider::ProviderError; +use reth_stateless::ExecutionWitness; use std::path::{Path, PathBuf}; use thiserror::Error; @@ -17,16 +20,19 @@ pub enum Error { /// The test was skipped #[error("test was skipped")] Skipped, - /// No post state found in test - #[error("no post state found for validation")] - MissingPostState, /// Block processing failed /// Note: This includes but is not limited to execution. /// For example, the header number could be incorrect. - #[error("block {block_number} failed to process")] + #[error("block {block_number} failed to process: {err}")] BlockProcessingFailed { /// The block number for the block that failed block_number: u64, + /// Contains the inputs necessary for the block stateless validation guest program used in + /// zkVMs to prove the block is invalid. + partial_program_inputs: Vec<(RecoveredBlock, ExecutionWitness)>, + /// The specific error + #[source] + err: Box, }, /// An IO error occurred #[error("an error occurred interacting with the file system at {path}: {error}")] @@ -63,6 +69,17 @@ pub enum Error { ConsensusError(#[from] reth_consensus::ConsensusError), } +impl Error { + /// Create a new [`Error::BlockProcessingFailed`] error. + pub fn block_failed( + block_number: u64, + partial_program_inputs: Vec<(RecoveredBlock, ExecutionWitness)>, + err: impl std::error::Error + Send + Sync + 'static, + ) -> Self { + Self::BlockProcessingFailed { block_number, partial_program_inputs, err: Box::new(err) } + } +} + /// The result of running a test. #[derive(Debug)] pub struct CaseResult { diff --git a/testing/ef-tests/src/suite.rs b/testing/ef-tests/src/suite.rs index 237ca935baf..0b3ed447a24 100644 --- a/testing/ef-tests/src/suite.rs +++ b/testing/ef-tests/src/suite.rs @@ -12,25 +12,28 @@ pub trait Suite { /// The type of test cases in this suite. type Case: Case; - /// The name of the test suite used to locate the individual test cases. - /// - /// # Example - /// - /// - `GeneralStateTests` - /// - `BlockchainTests/InvalidBlocks` - /// - `BlockchainTests/TransitionTests` - fn suite_name(&self) -> String; + /// The path to the test suite directory. + fn suite_path(&self) -> &Path; + + /// Run all test cases in the suite. + fn run(&self) { + let suite_path = self.suite_path(); + for entry in WalkDir::new(suite_path).min_depth(1).max_depth(1) { + let entry = entry.expect("Failed to read directory"); + if entry.file_type().is_dir() { + self.run_only(entry.file_name().to_string_lossy().as_ref()); + } + } + } - /// Load and run each contained test case. + /// Load and run each contained test case for the provided sub-folder. /// /// # Note /// /// This recursively finds every test description in the resulting path. - fn run(&self) { + fn run_only(&self, name: &str) { // Build the path to the test suite directory - let suite_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("ethereum-tests") - .join(self.suite_name()); + let suite_path = self.suite_path().join(name); // Verify that the path exists assert!(suite_path.exists(), "Test suite path does not exist: {suite_path:?}"); @@ -48,7 +51,7 @@ pub trait Suite { let results = Cases { test_cases }.run(); // Assert that all tests in the suite pass - assert_tests_pass(&self.suite_name(), &suite_path, &results); + assert_tests_pass(name, &suite_path, &results); } } diff --git a/testing/ef-tests/tests/tests.rs b/testing/ef-tests/tests/tests.rs index a1838d43e51..2728246901a 100644 --- a/testing/ef-tests/tests/tests.rs +++ b/testing/ef-tests/tests/tests.rs @@ -2,13 +2,19 @@ #![cfg(feature = "ef-tests")] use ef_tests::{cases::blockchain_test::BlockchainTests, suite::Suite}; +use std::path::PathBuf; macro_rules! general_state_test { ($test_name:ident, $dir:ident) => { #[test] fn $test_name() { reth_tracing::init_test_tracing(); - BlockchainTests::new(format!("GeneralStateTests/{}", stringify!($dir))).run(); + let suite_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("ethereum-tests") + .join("BlockchainTests"); + + BlockchainTests::new(suite_path) + .run_only(&format!("GeneralStateTests/{}", stringify!($dir))); } }; } @@ -83,10 +89,24 @@ macro_rules! blockchain_test { #[test] fn $test_name() { reth_tracing::init_test_tracing(); - BlockchainTests::new(format!("{}", stringify!($dir))).run(); + let suite_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("ethereum-tests") + .join("BlockchainTests"); + + BlockchainTests::new(suite_path).run_only(stringify!($dir)); } }; } blockchain_test!(valid_blocks, ValidBlocks); blockchain_test!(invalid_blocks, InvalidBlocks); + +#[test] +fn eest_fixtures() { + reth_tracing::init_test_tracing(); + let suite_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("execution-spec-tests") + .join("blockchain_tests"); + + BlockchainTests::new(suite_path).run(); +} diff --git a/testing/runner/Cargo.toml b/testing/runner/Cargo.toml new file mode 100644 index 00000000000..0b6893fd8b9 --- /dev/null +++ b/testing/runner/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "ef-test-runner" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true +exclude.workspace = true + +[dependencies] +clap = { workspace = true, features = ["derive"] } +ef-tests.path = "../ef-tests" + +[lints] +workspace = true diff --git a/testing/runner/src/main.rs b/testing/runner/src/main.rs new file mode 100644 index 00000000000..a36c443850c --- /dev/null +++ b/testing/runner/src/main.rs @@ -0,0 +1,17 @@ +//! Command-line interface for running tests. +use std::path::PathBuf; + +use clap::Parser; +use ef_tests::{cases::blockchain_test::BlockchainTests, Suite}; + +/// Command-line arguments for the test runner. +#[derive(Debug, Parser)] +pub struct TestRunnerCommand { + /// Path to the test suite + suite_path: PathBuf, +} + +fn main() { + let cmd = TestRunnerCommand::parse(); + BlockchainTests::new(cmd.suite_path.join("blockchain_tests")).run(); +} diff --git a/testing/testing-utils/Cargo.toml b/testing/testing-utils/Cargo.toml index eb4cb4e4449..06e73631ef8 100644 --- a/testing/testing-utils/Cargo.toml +++ b/testing/testing-utils/Cargo.toml @@ -23,7 +23,3 @@ alloy-eips.workspace = true rand.workspace = true secp256k1 = { workspace = true, features = ["rand"] } rand_08.workspace = true - -[dev-dependencies] -alloy-eips.workspace = true -reth-primitives-traits.workspace = true diff --git a/testing/testing-utils/src/generators.rs b/testing/testing-utils/src/generators.rs index 1a9297f8e1f..52aa8eab665 100644 --- a/testing/testing-utils/src/generators.rs +++ b/testing/testing-utils/src/generators.rs @@ -87,7 +87,7 @@ pub fn rng_with_seed(seed: &[u8]) -> StdRng { /// The parent hash of the first header /// in the result will be equal to `head`. /// -/// The headers are assumed to not be correct if validated. +/// The headers are assumed not to be correct if validated. pub fn random_header_range( rng: &mut R, range: Range, @@ -118,7 +118,7 @@ pub fn random_block_with_parent( /// Generate a random [`SealedHeader`]. /// -/// The header is assumed to not be correct if validated. +/// The header is assumed not to be correct if validated. pub fn random_header(rng: &mut R, number: u64, parent: Option) -> SealedHeader { let header = alloy_consensus::Header { number, @@ -453,6 +453,7 @@ pub fn random_receipt( rng: &mut R, transaction: &TransactionSigned, logs_count: Option, + topics_count: Option, ) -> Receipt { let success = rng.random::(); let logs_count = logs_count.unwrap_or_else(|| rng.random::()); @@ -462,7 +463,7 @@ pub fn random_receipt( success, cumulative_gas_used: rng.random_range(0..=transaction.gas_limit()), logs: if success { - (0..logs_count).map(|_| random_log(rng, None, None)).collect() + (0..logs_count).map(|_| random_log(rng, None, topics_count)).collect() } else { vec![] }, @@ -487,7 +488,10 @@ mod tests { use alloy_consensus::TxEip1559; use alloy_eips::eip2930::AccessList; use alloy_primitives::{hex, Signature}; - use reth_primitives_traits::crypto::secp256k1::{public_key_to_address, sign_message}; + use reth_primitives_traits::{ + crypto::secp256k1::{public_key_to_address, sign_message}, + SignerRecoverable, + }; use std::str::FromStr; #[test] diff --git a/testing/testing-utils/src/lib.rs b/testing/testing-utils/src/lib.rs index c593d306468..8baf40d1b63 100644 --- a/testing/testing-utils/src/lib.rs +++ b/testing/testing-utils/src/lib.rs @@ -5,7 +5,7 @@ html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256", issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] -#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(not(test), warn(unused_crate_dependencies))] pub mod genesis_allocator; diff --git a/typos.toml b/typos.toml new file mode 100644 index 00000000000..25f54392661 --- /dev/null +++ b/typos.toml @@ -0,0 +1,39 @@ +[files] +extend-exclude = [ + ".git", + "target", + "crates/storage/libmdbx-rs/mdbx-sys/libmdbx", + "Cargo.toml", + "Cargo.lock", + "testing/ef-tests", +] + +[default] +extend-ignore-re = [ + # Hex strings of various lengths + "(?i)0x[0-9a-f]{8}", # 8 hex chars + "(?i)0x[0-9a-f]{40}", # 40 hex chars + "(?i)0x[0-9a-f]{64}", # 64 hex chars + "(?i)[0-9a-f]{8}", # 8 hex chars without 0x + "(?i)[0-9a-f]{40}", # 40 hex chars without 0x + "(?i)[0-9a-f]{64}", # 64 hex chars without 0x + # Ordinals in identifiers + "[0-9]+nd", + "[0-9]+th", + "[0-9]+st", + "[0-9]+rd", +] + +[default.extend-words] +# These are valid identifiers/terms that should be allowed +crate = "crate" +ser = "ser" +ratatui = "ratatui" +seeked = "seeked" # Past tense of seek, used in trie iterator +Seeked = "Seeked" # Type name in trie iterator +Whe = "Whe" # Part of base64 encoded signature +hel = "hel" # Part of hostname bootnode-hetzner-hel +ONL = "ONL" # Part of base64 encoded ENR +Iy = "Iy" # Part of base64 encoded ENR +flate = "flate" # zlib-flate is a valid tool name +Pn = "Pn" # Part of UPnP (Universal Plug and Play)

(&mut self, predicate: P) -> Option> + where + P: FnOnce(&SparseSubtrie) -> bool, + { + match self { + Self::Revealed(subtrie) if predicate(subtrie) => { + let Self::Revealed(subtrie) = core::mem::take(self) else { unreachable!() }; + Some(subtrie) + } + Self::Revealed(_) | Self::Blind(_) => None, + } + } + + /// Shrinks the capacity of the subtrie's node storage. + /// Works for both revealed and blind tries with allocated storage. + pub(crate) fn shrink_nodes_to(&mut self, size: usize) { + match self { + Self::Revealed(trie) | Self::Blind(Some(trie)) => { + trie.shrink_nodes_to(size); + } + Self::Blind(None) => {} + } + } + + /// Shrinks the capacity of the subtrie's value storage. + /// Works for both revealed and blind tries with allocated storage. + pub(crate) fn shrink_values_to(&mut self, size: usize) { + match self { + Self::Revealed(trie) | Self::Blind(Some(trie)) => { + trie.shrink_values_to(size); + } + Self::Blind(None) => {} + } + } +} diff --git a/crates/trie/sparse-parallel/src/metrics.rs b/crates/trie/sparse-parallel/src/metrics.rs new file mode 100644 index 00000000000..892c8fbe2ae --- /dev/null +++ b/crates/trie/sparse-parallel/src/metrics.rs @@ -0,0 +1,23 @@ +//! Metrics for the parallel sparse trie +use reth_metrics::{metrics::Histogram, Metrics}; + +/// Metrics for the parallel sparse trie +#[derive(Metrics, Clone)] +#[metrics(scope = "parallel_sparse_trie")] +pub(crate) struct ParallelSparseTrieMetrics { + /// A histogram for the number of subtries updated when calculating hashes. + pub(crate) subtries_updated: Histogram, + /// A histogram for the time it took to update lower subtrie hashes. + pub(crate) subtrie_hash_update_latency: Histogram, + /// A histogram for the time it took to update the upper subtrie hashes. + pub(crate) subtrie_upper_hash_latency: Histogram, +} + +impl PartialEq for ParallelSparseTrieMetrics { + fn eq(&self, _other: &Self) -> bool { + // It does not make sense to compare metrics, so return true, all are equal + true + } +} + +impl Eq for ParallelSparseTrieMetrics {} diff --git a/crates/trie/sparse-parallel/src/trie.rs b/crates/trie/sparse-parallel/src/trie.rs new file mode 100644 index 00000000000..133cdfece4c --- /dev/null +++ b/crates/trie/sparse-parallel/src/trie.rs @@ -0,0 +1,6843 @@ +use crate::LowerSparseSubtrie; +use alloc::borrow::Cow; +use alloy_primitives::{ + map::{Entry, HashMap}, + B256, +}; +use alloy_rlp::Decodable; +use alloy_trie::{BranchNodeCompact, TrieMask, EMPTY_ROOT_HASH}; +use reth_execution_errors::{SparseTrieErrorKind, SparseTrieResult}; +use reth_trie_common::{ + prefix_set::{PrefixSet, PrefixSetMut}, + BranchNodeRef, ExtensionNodeRef, LeafNodeRef, Nibbles, RlpNode, TrieNode, CHILD_INDEX_RANGE, +}; +use reth_trie_sparse::{ + provider::{RevealedNode, TrieNodeProvider}, + LeafLookup, LeafLookupError, RevealedSparseNode, RlpNodeStackItem, SparseNode, SparseNodeType, + SparseTrieInterface, SparseTrieUpdates, TrieMasks, +}; +use smallvec::SmallVec; +use std::{ + cmp::{Ord, Ordering, PartialOrd}, + sync::mpsc, +}; +use tracing::{debug, instrument, trace}; + +/// The maximum length of a path, in nibbles, which belongs to the upper subtrie of a +/// [`ParallelSparseTrie`]. All longer paths belong to a lower subtrie. +pub const UPPER_TRIE_MAX_DEPTH: usize = 2; + +/// Number of lower subtries which are managed by the [`ParallelSparseTrie`]. +pub const NUM_LOWER_SUBTRIES: usize = 16usize.pow(UPPER_TRIE_MAX_DEPTH as u32); + +/// Configuration for controlling when parallelism is enabled in [`ParallelSparseTrie`] operations. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct ParallelismThresholds { + /// Minimum number of nodes to reveal before parallel processing is enabled. + /// When `reveal_nodes` has fewer nodes than this threshold, they will be processed serially. + pub min_revealed_nodes: usize, + /// Minimum number of changed keys (prefix set length) before parallel processing is enabled + /// for hash updates. When updating subtrie hashes with fewer changed keys than this threshold, + /// the updates will be processed serially. + pub min_updated_nodes: usize, +} + +/// A revealed sparse trie with subtries that can be updated in parallel. +/// +/// ## Structure +/// +/// The trie is divided into two tiers for efficient parallel processing: +/// - **Upper subtrie**: Contains nodes with paths shorter than [`UPPER_TRIE_MAX_DEPTH`] +/// - **Lower subtries**: An array of [`NUM_LOWER_SUBTRIES`] subtries, each handling nodes with +/// paths of at least [`UPPER_TRIE_MAX_DEPTH`] nibbles +/// +/// Node placement is determined by path depth: +/// - Paths with < [`UPPER_TRIE_MAX_DEPTH`] nibbles go to the upper subtrie +/// - Paths with >= [`UPPER_TRIE_MAX_DEPTH`] nibbles go to lower subtries, indexed by their first +/// [`UPPER_TRIE_MAX_DEPTH`] nibbles. +/// +/// Each lower subtrie tracks its root via the `path` field, which represents the shortest path +/// in that subtrie. This path will have at least [`UPPER_TRIE_MAX_DEPTH`] nibbles, but may be +/// longer when an extension node in the upper trie "reaches into" the lower subtrie. For example, +/// if the upper trie has an extension from `0x1` to `0x12345`, then the lower subtrie for prefix +/// `0x12` will have its root at path `0x12345` rather than at `0x12`. +/// +/// ## Node Revealing +/// +/// The trie uses lazy loading to efficiently handle large state tries. Nodes can be: +/// - **Blind nodes**: Stored as hashes ([`SparseNode::Hash`]), representing unloaded trie parts +/// - **Revealed nodes**: Fully loaded nodes (Branch, Extension, Leaf) with complete structure +/// +/// Note: An empty trie contains an `EmptyRoot` node at the root path, rather than no nodes at all. +/// A trie with no nodes is blinded, its root may be `EmptyRoot` or some other node type. +/// +/// Revealing is generally done using pre-loaded node data provided to via `reveal_nodes`. In +/// certain cases, such as edge-cases when updating/removing leaves, nodes are revealed on-demand. +/// +/// ## Leaf Operations +/// +/// **Update**: When updating a leaf, the new value is stored in the appropriate subtrie's values +/// map. If the leaf is new, the trie structure is updated by walking to the leaf from the root, +/// creating necessary intermediate branch nodes. +/// +/// **Removal**: Leaf removal may require parent node modifications. The algorithm walks up the +/// trie, removing nodes that become empty and converting single-child branches to extensions. +/// +/// During leaf operations the overall structure of the trie may change, causing nodes to be moved +/// from the upper to lower trie or vice-versa. +/// +/// The `prefix_set` is modified during both leaf updates and removals to track changed leaf paths. +/// +/// ## Root Hash Calculation +/// +/// Root hash computation follows a bottom-up approach: +/// 1. Update hashes for all modified lower subtries (can be done in parallel) +/// 2. Update hashes for the upper subtrie (which may reference lower subtrie hashes) +/// 3. Calculate the final root hash from the upper subtrie's root node +/// +/// The `prefix_set` tracks which paths have been modified, enabling incremental updates instead of +/// recalculating the entire trie. +/// +/// ## Invariants +/// +/// - Each leaf entry in the `subtries` and `upper_trie` collection must have a corresponding entry +/// in `values` collection. If the root node is a leaf, it must also have an entry in `values`. +/// - All keys in `values` collection are full leaf paths. +#[derive(Clone, PartialEq, Eq, Debug)] +pub struct ParallelSparseTrie { + /// This contains the trie nodes for the upper part of the trie. + upper_subtrie: Box, + /// An array containing the subtries at the second level of the trie. + lower_subtries: [LowerSparseSubtrie; NUM_LOWER_SUBTRIES], + /// Set of prefixes (key paths) that have been marked as updated. + /// This is used to track which parts of the trie need to be recalculated. + prefix_set: PrefixSetMut, + /// Optional tracking of trie updates for later use. + updates: Option, + /// When a bit is set, the corresponding child subtree is stored in the database. + branch_node_tree_masks: HashMap, + /// When a bit is set, the corresponding child is stored as a hash in the database. + branch_node_hash_masks: HashMap, + /// Reusable buffer pool used for collecting [`SparseTrieUpdatesAction`]s during hash + /// computations. + update_actions_buffers: Vec>, + /// Thresholds controlling when parallelism is enabled for different operations. + parallelism_thresholds: ParallelismThresholds, + /// Metrics for the parallel sparse trie. + #[cfg(feature = "metrics")] + metrics: crate::metrics::ParallelSparseTrieMetrics, +} + +impl Default for ParallelSparseTrie { + fn default() -> Self { + Self { + upper_subtrie: Box::new(SparseSubtrie { + nodes: HashMap::from_iter([(Nibbles::default(), SparseNode::Empty)]), + ..Default::default() + }), + lower_subtries: [const { LowerSparseSubtrie::Blind(None) }; NUM_LOWER_SUBTRIES], + prefix_set: PrefixSetMut::default(), + updates: None, + branch_node_tree_masks: HashMap::default(), + branch_node_hash_masks: HashMap::default(), + update_actions_buffers: Vec::default(), + parallelism_thresholds: Default::default(), + #[cfg(feature = "metrics")] + metrics: Default::default(), + } + } +} + +impl SparseTrieInterface for ParallelSparseTrie { + fn with_root( + mut self, + root: TrieNode, + masks: TrieMasks, + retain_updates: bool, + ) -> SparseTrieResult { + // A fresh/cleared `ParallelSparseTrie` has a `SparseNode::Empty` at its root in the upper + // subtrie. Delete that so we can reveal the new root node. + let path = Nibbles::default(); + let _removed_root = self.upper_subtrie.nodes.remove(&path).expect("root node should exist"); + debug_assert_eq!(_removed_root, SparseNode::Empty); + + self = self.with_updates(retain_updates); + + self.reveal_upper_node(Nibbles::default(), &root, masks)?; + Ok(self) + } + + fn with_updates(mut self, retain_updates: bool) -> Self { + self.updates = retain_updates.then(Default::default); + self + } + + fn reveal_nodes(&mut self, mut nodes: Vec) -> SparseTrieResult<()> { + if nodes.is_empty() { + return Ok(()) + } + + // Sort nodes first by their subtrie, and secondarily by their path. This allows for + // grouping nodes by their subtrie using `chunk_by`. + nodes.sort_unstable_by( + |RevealedSparseNode { path: path_a, .. }, RevealedSparseNode { path: path_b, .. }| { + let subtrie_type_a = SparseSubtrieType::from_path(path_a); + let subtrie_type_b = SparseSubtrieType::from_path(path_b); + subtrie_type_a.cmp(&subtrie_type_b).then(path_a.cmp(path_b)) + }, + ); + + // Update the top-level branch node masks. This is simple and can't be done in parallel. + for RevealedSparseNode { path, masks, .. } in &nodes { + if let Some(tree_mask) = masks.tree_mask { + self.branch_node_tree_masks.insert(*path, tree_mask); + } + if let Some(hash_mask) = masks.hash_mask { + self.branch_node_hash_masks.insert(*path, hash_mask); + } + } + + // Due to the sorting all upper subtrie nodes will be at the front of the slice. We split + // them off from the rest to be handled specially by + // `ParallelSparseTrie::reveal_upper_node`. + let num_upper_nodes = nodes + .iter() + .position(|n| !SparseSubtrieType::path_len_is_upper(n.path.len())) + .unwrap_or(nodes.len()); + + let upper_nodes = &nodes[..num_upper_nodes]; + let lower_nodes = &nodes[num_upper_nodes..]; + + // Reserve the capacity of the upper subtrie's `nodes` HashMap before iterating, so we don't + // end up making many small capacity changes as we loop. + self.upper_subtrie.nodes.reserve(upper_nodes.len()); + for node in upper_nodes { + self.reveal_upper_node(node.path, &node.node, node.masks)?; + } + + if !self.is_reveal_parallelism_enabled(lower_nodes.len()) { + for node in lower_nodes { + if let Some(subtrie) = self.lower_subtrie_for_path_mut(&node.path) { + subtrie.reveal_node(node.path, &node.node, node.masks)?; + } else { + panic!("upper subtrie node {node:?} found amongst lower nodes"); + } + } + return Ok(()) + } + + #[cfg(not(feature = "std"))] + unreachable!("nostd is checked by is_reveal_parallelism_enabled"); + + #[cfg(feature = "std")] + // Reveal lower subtrie nodes in parallel + { + use rayon::iter::{IndexedParallelIterator, IntoParallelIterator, ParallelIterator}; + + // Group the nodes by lower subtrie. This must be collected into a Vec in order for + // rayon's `zip` to be happy. + let node_groups: Vec<_> = lower_nodes + .chunk_by(|node_a, node_b| { + SparseSubtrieType::from_path(&node_a.path) == + SparseSubtrieType::from_path(&node_b.path) + }) + .collect(); + + // Take the lower subtries in the same order that the nodes were grouped into, so that + // the two can be zipped together. This also must be collected into a Vec for rayon's + // `zip` to be happy. + let lower_subtries: Vec<_> = node_groups + .iter() + .map(|nodes| { + // NOTE: chunk_by won't produce empty groups + let node = &nodes[0]; + let idx = + SparseSubtrieType::from_path(&node.path).lower_index().unwrap_or_else( + || panic!("upper subtrie node {node:?} found amongst lower nodes"), + ); + // due to the nodes being sorted secondarily on their path, and chunk_by keeping + // the first element of each group, the `path` here will necessarily be the + // shortest path being revealed for each subtrie. Therefore we can reveal the + // subtrie itself using this path and retain correct behavior. + self.lower_subtries[idx].reveal(&node.path); + (idx, self.lower_subtries[idx].take_revealed().expect("just revealed")) + }) + .collect(); + + let (tx, rx) = mpsc::channel(); + + // Zip the lower subtries and their corresponding node groups, and reveal lower subtrie + // nodes in parallel + lower_subtries + .into_par_iter() + .zip(node_groups.into_par_iter()) + .map(|((subtrie_idx, mut subtrie), nodes)| { + // reserve space in the HashMap ahead of time; doing it on a node-by-node basis + // can cause multiple re-allocations as the hashmap grows. + subtrie.nodes.reserve(nodes.len()); + + for node in nodes { + // Reveal each node in the subtrie, returning early on any errors + let res = subtrie.reveal_node(node.path, &node.node, node.masks); + if res.is_err() { + return (subtrie_idx, subtrie, res) + } + } + (subtrie_idx, subtrie, Ok(())) + }) + .for_each_init(|| tx.clone(), |tx, result| tx.send(result).unwrap()); + + drop(tx); + + // Take back all lower subtries which were sent to the rayon pool, collecting the last + // seen error in the process and returning that. If we don't fully drain the channel + // then we lose lower sparse tries, putting the whole ParallelSparseTrie in an + // inconsistent state. + let mut any_err = Ok(()); + for (subtrie_idx, subtrie, res) in rx { + self.lower_subtries[subtrie_idx] = LowerSparseSubtrie::Revealed(subtrie); + if res.is_err() { + any_err = res; + } + } + + any_err + } + } + + fn update_leaf( + &mut self, + full_path: Nibbles, + value: Vec, + provider: P, + ) -> SparseTrieResult<()> { + self.prefix_set.insert(full_path); + let existing = self.upper_subtrie.inner.values.insert(full_path, value.clone()); + if existing.is_some() { + // upper trie structure unchanged, return immediately + return Ok(()) + } + + let retain_updates = self.updates_enabled(); + + // Start at the root, traversing until we find either the node to update or a subtrie to + // update. + // + // We first traverse the upper subtrie for two levels, and moving any created nodes to a + // lower subtrie if necessary. + // + // We use `next` to keep track of the next node that we need to traverse to, and + // `new_nodes` to keep track of any nodes that were created during the traversal. + let mut new_nodes = Vec::new(); + let mut next = Some(Nibbles::default()); + + // Traverse the upper subtrie to find the node to update or the subtrie to update. + // + // We stop when the next node to traverse would be in a lower subtrie, or if there are no + // more nodes to traverse. + while let Some(current) = + next.filter(|next| SparseSubtrieType::path_len_is_upper(next.len())) + { + // Traverse the next node, keeping track of any changed nodes and the next step in the + // trie + match self.upper_subtrie.update_next_node(current, &full_path, retain_updates)? { + LeafUpdateStep::Continue { next_node } => { + next = Some(next_node); + } + LeafUpdateStep::Complete { inserted_nodes, reveal_path } => { + new_nodes.extend(inserted_nodes); + + if let Some(reveal_path) = reveal_path { + let subtrie = self.subtrie_for_path_mut(&reveal_path); + if subtrie.nodes.get(&reveal_path).expect("node must exist").is_hash() { + debug!( + target: "trie::parallel_sparse", + child_path = ?reveal_path, + leaf_full_path = ?full_path, + "Extension node child not revealed in update_leaf, falling back to db", + ); + if let Some(RevealedNode { node, tree_mask, hash_mask }) = + provider.trie_node(&reveal_path)? + { + let decoded = TrieNode::decode(&mut &node[..])?; + trace!( + target: "trie::parallel_sparse", + ?reveal_path, + ?decoded, + ?tree_mask, + ?hash_mask, + "Revealing child (from upper)", + ); + subtrie.reveal_node( + reveal_path, + &decoded, + TrieMasks { hash_mask, tree_mask }, + )?; + } else { + return Err(SparseTrieErrorKind::NodeNotFoundInProvider { + path: reveal_path, + } + .into()) + } + } + } + + next = None; + } + LeafUpdateStep::NodeNotFound => { + next = None; + } + } + } + + // Move nodes from upper subtrie to lower subtries + for node_path in &new_nodes { + // Skip nodes that belong in the upper subtrie + if SparseSubtrieType::path_len_is_upper(node_path.len()) { + continue + } + + let node = + self.upper_subtrie.nodes.remove(node_path).expect("node belongs to upper subtrie"); + + // If it's a leaf node, extract its value before getting mutable reference to subtrie. + let leaf_value = if let SparseNode::Leaf { key, .. } = &node { + let mut leaf_full_path = *node_path; + leaf_full_path.extend(key); + Some(( + leaf_full_path, + self.upper_subtrie + .inner + .values + .remove(&leaf_full_path) + .expect("leaf nodes have associated values entries"), + )) + } else { + None + }; + + // Get or create the subtrie with the exact node path (not truncated to 2 nibbles). + let subtrie = self.subtrie_for_path_mut(node_path); + + // Insert the leaf value if we have one + if let Some((leaf_full_path, value)) = leaf_value { + subtrie.inner.values.insert(leaf_full_path, value); + } + + // Insert the node into the lower subtrie + subtrie.nodes.insert(*node_path, node); + } + + // If we reached the max depth of the upper trie, we may have had more nodes to insert. + if let Some(next_path) = next.filter(|n| !SparseSubtrieType::path_len_is_upper(n.len())) { + // The value was inserted into the upper subtrie's `values` at the top of this method. + // At this point we know the value is not in the upper subtrie, and the call to + // `update_leaf` below will insert it into the lower subtrie. So remove it from the + // upper subtrie. + self.upper_subtrie.inner.values.remove(&full_path); + + // Use subtrie_for_path to ensure the subtrie has the correct path. + // + // The next_path here represents where we need to continue traversal, which may + // be longer than 2 nibbles if we're following an extension node. + let subtrie = self.subtrie_for_path_mut(&next_path); + + // Create an empty root at the subtrie path if the subtrie is empty + if subtrie.nodes.is_empty() { + subtrie.nodes.insert(subtrie.path, SparseNode::Empty); + } + + // If we didn't update the target leaf, we need to call update_leaf on the subtrie + // to ensure that the leaf is updated correctly. + subtrie.update_leaf(full_path, value, provider, retain_updates)?; + } + + Ok(()) + } + + fn remove_leaf( + &mut self, + full_path: &Nibbles, + provider: P, + ) -> SparseTrieResult<()> { + // When removing a leaf node it's possibly necessary to modify its parent node, and possibly + // the parent's parent node. It is not ever necessary to descend further than that; once an + // extension node is hit it must terminate in a branch or the root, which won't need further + // updates. So the situation with maximum updates is: + // + // - Leaf + // - Branch with 2 children, one being this leaf + // - Extension + // + // ...which will result in just a leaf or extension, depending on what the branch's other + // child is. + // + // Therefore, first traverse the trie in order to find the leaf node and at most its parent + // and grandparent. + + let leaf_path; + let leaf_subtrie; + + let mut branch_parent_path: Option = None; + let mut branch_parent_node: Option = None; + + let mut ext_grandparent_path: Option = None; + let mut ext_grandparent_node: Option = None; + + let mut curr_path = Nibbles::new(); // start traversal from root + let mut curr_subtrie = self.upper_subtrie.as_mut(); + let mut curr_subtrie_is_upper = true; + + // List of node paths which need to have their hashes reset + let mut paths_to_reset_hashes = Vec::new(); + + loop { + let curr_node = curr_subtrie.nodes.get_mut(&curr_path).unwrap(); + + match Self::find_next_to_leaf(&curr_path, curr_node, full_path) { + FindNextToLeafOutcome::NotFound => return Ok(()), // leaf isn't in the trie + FindNextToLeafOutcome::BlindedNode(hash) => { + return Err(SparseTrieErrorKind::BlindedNode { path: curr_path, hash }.into()) + } + FindNextToLeafOutcome::Found => { + // this node is the target leaf + leaf_path = curr_path; + leaf_subtrie = curr_subtrie; + break; + } + FindNextToLeafOutcome::ContinueFrom(next_path) => { + // Any branches/extensions along the path to the leaf will have their `hash` + // field unset, as it will no longer be valid once the leaf is removed. + match curr_node { + SparseNode::Branch { hash, .. } => { + if hash.is_some() { + paths_to_reset_hashes + .push((SparseSubtrieType::from_path(&curr_path), curr_path)); + } + + // If there is already an extension leading into a branch, then that + // extension is no longer relevant. + match (&branch_parent_path, &ext_grandparent_path) { + (Some(branch), Some(ext)) if branch.len() > ext.len() => { + ext_grandparent_path = None; + ext_grandparent_node = None; + } + _ => (), + }; + branch_parent_path = Some(curr_path); + branch_parent_node = Some(curr_node.clone()); + } + SparseNode::Extension { hash, .. } => { + if hash.is_some() { + paths_to_reset_hashes + .push((SparseSubtrieType::from_path(&curr_path), curr_path)); + } + + // We can assume a new branch node will be found after the extension, so + // there's no need to modify branch_parent_path/node even if it's + // already set. + ext_grandparent_path = Some(curr_path); + ext_grandparent_node = Some(curr_node.clone()); + } + SparseNode::Empty | SparseNode::Hash(_) | SparseNode::Leaf { .. } => { + unreachable!( + "find_next_to_leaf only continues to a branch or extension" + ) + } + } + + curr_path = next_path; + + // If we were previously looking at the upper trie, and the new path is in the + // lower trie, we need to pull out a ref to the lower trie. + if curr_subtrie_is_upper && + let SparseSubtrieType::Lower(idx) = + SparseSubtrieType::from_path(&curr_path) + { + curr_subtrie = self.lower_subtries[idx] + .as_revealed_mut() + .expect("lower subtrie is revealed"); + curr_subtrie_is_upper = false; + } + } + }; + } + + // We've traversed to the leaf and collected its ancestors as necessary. Remove the leaf + // from its SparseSubtrie and reset the hashes of the nodes along the path. + self.prefix_set.insert(*full_path); + leaf_subtrie.inner.values.remove(full_path); + for (subtrie_type, path) in paths_to_reset_hashes { + let node = match subtrie_type { + SparseSubtrieType::Upper => self.upper_subtrie.nodes.get_mut(&path), + SparseSubtrieType::Lower(idx) => self.lower_subtries[idx] + .as_revealed_mut() + .expect("lower subtrie is revealed") + .nodes + .get_mut(&path), + } + .expect("node exists"); + + match node { + SparseNode::Extension { hash, .. } | SparseNode::Branch { hash, .. } => { + *hash = None + } + SparseNode::Empty | SparseNode::Hash(_) | SparseNode::Leaf { .. } => { + unreachable!("only branch and extension node hashes can be reset") + } + } + } + self.remove_node(&leaf_path); + + // If the leaf was at the root replace its node with the empty value. We can stop execution + // here, all remaining logic is related to the ancestors of the leaf. + if leaf_path.is_empty() { + self.upper_subtrie.nodes.insert(leaf_path, SparseNode::Empty); + return Ok(()) + } + + // If there is a parent branch node (very likely, unless the leaf is at the root) execute + // any required changes for that node, relative to the removed leaf. + if let (Some(branch_path), &Some(SparseNode::Branch { mut state_mask, .. })) = + (&branch_parent_path, &branch_parent_node) + { + let child_nibble = leaf_path.get_unchecked(branch_path.len()); + state_mask.unset_bit(child_nibble); + + let new_branch_node = if state_mask.count_bits() == 1 { + // If only one child is left set in the branch node, we need to collapse it. Get + // full path of the only child node left. + let remaining_child_path = { + let mut p = *branch_path; + p.push_unchecked( + state_mask.first_set_bit_index().expect("state mask is not empty"), + ); + p + }; + + trace!( + target: "trie::parallel_sparse", + ?leaf_path, + ?branch_path, + ?remaining_child_path, + "Branch node has only one child", + ); + + // If the remaining child node is not yet revealed then we have to reveal it here, + // otherwise it's not possible to know how to collapse the branch. + let remaining_child_node = self.reveal_remaining_child_on_leaf_removal( + provider, + full_path, + &remaining_child_path, + true, // recurse_into_extension + )?; + + let (new_branch_node, remove_child) = Self::branch_changes_on_leaf_removal( + branch_path, + &remaining_child_path, + &remaining_child_node, + ); + + if remove_child { + self.move_value_on_leaf_removal( + branch_path, + &new_branch_node, + &remaining_child_path, + ); + self.remove_node(&remaining_child_path); + } + + if let Some(updates) = self.updates.as_mut() { + updates.updated_nodes.remove(branch_path); + updates.removed_nodes.insert(*branch_path); + } + + new_branch_node + } else { + // If more than one child is left set in the branch, we just re-insert it with the + // updated state_mask. + SparseNode::new_branch(state_mask) + }; + + let branch_subtrie = self.subtrie_for_path_mut(branch_path); + branch_subtrie.nodes.insert(*branch_path, new_branch_node.clone()); + branch_parent_node = Some(new_branch_node); + }; + + // If there is a grandparent extension node then there will necessarily be a parent branch + // node. Execute any required changes for the extension node, relative to the (possibly now + // replaced with a leaf or extension) branch node. + if let (Some(ext_path), Some(SparseNode::Extension { key: shortkey, .. })) = + (ext_grandparent_path, &ext_grandparent_node) + { + let ext_subtrie = self.subtrie_for_path_mut(&ext_path); + let branch_path = branch_parent_path.as_ref().unwrap(); + + if let Some(new_ext_node) = Self::extension_changes_on_leaf_removal( + &ext_path, + shortkey, + branch_path, + branch_parent_node.as_ref().unwrap(), + ) { + ext_subtrie.nodes.insert(ext_path, new_ext_node.clone()); + self.move_value_on_leaf_removal(&ext_path, &new_ext_node, branch_path); + self.remove_node(branch_path); + } + } + + Ok(()) + } + + #[instrument(level = "trace", target = "trie::sparse::parallel", skip(self))] + fn root(&mut self) -> B256 { + trace!(target: "trie::parallel_sparse", "Calculating trie root hash"); + + // Update all lower subtrie hashes + self.update_subtrie_hashes(); + + // Update hashes for the upper subtrie using our specialized function + // that can access both upper and lower subtrie nodes + let mut prefix_set = core::mem::take(&mut self.prefix_set).freeze(); + let root_rlp = self.update_upper_subtrie_hashes(&mut prefix_set); + + // Return the root hash + root_rlp.as_hash().unwrap_or(EMPTY_ROOT_HASH) + } + + #[instrument(level = "trace", target = "trie::sparse::parallel", skip(self))] + fn update_subtrie_hashes(&mut self) { + trace!(target: "trie::parallel_sparse", "Updating subtrie hashes"); + + // Take changed subtries according to the prefix set + let mut prefix_set = core::mem::take(&mut self.prefix_set).freeze(); + let num_changed_keys = prefix_set.len(); + let (mut changed_subtries, unchanged_prefix_set) = + self.take_changed_lower_subtries(&mut prefix_set); + + // update metrics + #[cfg(feature = "metrics")] + self.metrics.subtries_updated.record(changed_subtries.len() as f64); + + // Update the prefix set with the keys that didn't have matching subtries + self.prefix_set = unchanged_prefix_set; + + // Update subtrie hashes serially parallelism is not enabled + if !self.is_update_parallelism_enabled(num_changed_keys) { + for changed_subtrie in &mut changed_subtries { + changed_subtrie.subtrie.update_hashes( + &mut changed_subtrie.prefix_set, + &mut changed_subtrie.update_actions_buf, + &self.branch_node_tree_masks, + &self.branch_node_hash_masks, + ); + } + + self.insert_changed_subtries(changed_subtries); + return + } + + #[cfg(not(feature = "std"))] + unreachable!("nostd is checked by is_update_parallelism_enabled"); + + #[cfg(feature = "std")] + // Update subtrie hashes in parallel + { + use rayon::iter::{IntoParallelIterator, ParallelIterator}; + use tracing::debug_span; + + let (tx, rx) = mpsc::channel(); + + let branch_node_tree_masks = &self.branch_node_tree_masks; + let branch_node_hash_masks = &self.branch_node_hash_masks; + let span = tracing::Span::current(); + changed_subtries + .into_par_iter() + .map(|mut changed_subtrie| { + let _enter = debug_span!( + target: "trie::parallel_sparse", + parent: span.clone(), + "subtrie", + index = changed_subtrie.index + ) + .entered(); + + #[cfg(feature = "metrics")] + let start = std::time::Instant::now(); + changed_subtrie.subtrie.update_hashes( + &mut changed_subtrie.prefix_set, + &mut changed_subtrie.update_actions_buf, + branch_node_tree_masks, + branch_node_hash_masks, + ); + #[cfg(feature = "metrics")] + self.metrics.subtrie_hash_update_latency.record(start.elapsed()); + changed_subtrie + }) + .for_each_init(|| tx.clone(), |tx, result| tx.send(result).unwrap()); + + drop(tx); + self.insert_changed_subtries(rx); + } + } + + fn get_leaf_value(&self, full_path: &Nibbles) -> Option<&Vec> { + self.subtrie_for_path(full_path).and_then(|subtrie| subtrie.inner.values.get(full_path)) + } + + fn updates_ref(&self) -> Cow<'_, SparseTrieUpdates> { + self.updates.as_ref().map_or(Cow::Owned(SparseTrieUpdates::default()), Cow::Borrowed) + } + + fn take_updates(&mut self) -> SparseTrieUpdates { + self.updates.take().unwrap_or_default() + } + + fn wipe(&mut self) { + self.upper_subtrie.wipe(); + self.lower_subtries = [const { LowerSparseSubtrie::Blind(None) }; NUM_LOWER_SUBTRIES]; + self.prefix_set = PrefixSetMut::all(); + self.updates = self.updates.is_some().then(SparseTrieUpdates::wiped); + } + + fn clear(&mut self) { + self.upper_subtrie.clear(); + self.upper_subtrie.nodes.insert(Nibbles::default(), SparseNode::Empty); + for subtrie in &mut self.lower_subtries { + subtrie.clear(); + } + self.prefix_set.clear(); + self.updates = None; + self.branch_node_tree_masks.clear(); + self.branch_node_hash_masks.clear(); + // `update_actions_buffers` doesn't need to be cleared; we want to reuse the Vecs it has + // buffered, and all of those are already inherently cleared when they get used. + } + + fn find_leaf( + &self, + full_path: &Nibbles, + expected_value: Option<&Vec>, + ) -> Result { + // Inclusion proof + // + // First, do a quick check if the value exists in either the upper or lower subtrie's values + // map. We assume that if there exists a leaf node, then its value will be in the `values` + // map. + if let Some(actual_value) = std::iter::once(self.upper_subtrie.as_ref()) + .chain(self.lower_subtrie_for_path(full_path)) + .filter_map(|subtrie| subtrie.inner.values.get(full_path)) + .next() + { + // We found the leaf, check if the value matches (if expected value was provided) + return expected_value + .is_none_or(|v| v == actual_value) + .then_some(LeafLookup::Exists) + .ok_or_else(|| LeafLookupError::ValueMismatch { + path: *full_path, + expected: expected_value.cloned(), + actual: actual_value.clone(), + }) + } + + // If the value does not exist in the `values` map, then this means that the leaf either: + // - Does not exist in the trie + // - Is missing from the witness + // We traverse the trie to find the location where this leaf would have been, showing + // that it is not in the trie. Or we find a blinded node, showing that the witness is + // not complete. + let mut curr_path = Nibbles::new(); // start traversal from root + let mut curr_subtrie = self.upper_subtrie.as_ref(); + let mut curr_subtrie_is_upper = true; + + loop { + let curr_node = curr_subtrie.nodes.get(&curr_path).unwrap(); + + match Self::find_next_to_leaf(&curr_path, curr_node, full_path) { + FindNextToLeafOutcome::NotFound => return Ok(LeafLookup::NonExistent), + FindNextToLeafOutcome::BlindedNode(hash) => { + // We hit a blinded node - cannot determine if leaf exists + return Err(LeafLookupError::BlindedNode { path: curr_path, hash }); + } + FindNextToLeafOutcome::Found => { + panic!("target leaf {full_path:?} found at path {curr_path:?}, even though value wasn't in values hashmap"); + } + FindNextToLeafOutcome::ContinueFrom(next_path) => { + curr_path = next_path; + // If we were previously looking at the upper trie, and the new path is in the + // lower trie, we need to pull out a ref to the lower trie. + if curr_subtrie_is_upper && + let Some(lower_subtrie) = self.lower_subtrie_for_path(&curr_path) + { + curr_subtrie = lower_subtrie; + curr_subtrie_is_upper = false; + } + } + } + } + } + + fn shrink_nodes_to(&mut self, size: usize) { + // Distribute the capacity across upper and lower subtries + // + // Always include upper subtrie, plus any lower subtries + let total_subtries = 1 + NUM_LOWER_SUBTRIES; + let size_per_subtrie = size / total_subtries; + + // Shrink the upper subtrie + self.upper_subtrie.shrink_nodes_to(size_per_subtrie); + + // Shrink lower subtries (works for both revealed and blind with allocation) + for subtrie in &mut self.lower_subtries { + subtrie.shrink_nodes_to(size_per_subtrie); + } + + // shrink masks maps + self.branch_node_hash_masks.shrink_to(size); + self.branch_node_tree_masks.shrink_to(size); + } + + fn shrink_values_to(&mut self, size: usize) { + // Distribute the capacity across upper and lower subtries + // + // Always include upper subtrie, plus any lower subtries + let total_subtries = 1 + NUM_LOWER_SUBTRIES; + let size_per_subtrie = size / total_subtries; + + // Shrink the upper subtrie + self.upper_subtrie.shrink_values_to(size_per_subtrie); + + // Shrink lower subtries (works for both revealed and blind with allocation) + for subtrie in &mut self.lower_subtries { + subtrie.shrink_values_to(size_per_subtrie); + } + } +} + +impl ParallelSparseTrie { + /// Sets the thresholds that control when parallelism is used during operations. + pub const fn with_parallelism_thresholds(mut self, thresholds: ParallelismThresholds) -> Self { + self.parallelism_thresholds = thresholds; + self + } + + /// Returns true if retaining updates is enabled for the overall trie. + const fn updates_enabled(&self) -> bool { + self.updates.is_some() + } + + /// Returns true if parallelism should be enabled for revealing the given number of nodes. + /// Will always return false in nostd builds. + const fn is_reveal_parallelism_enabled(&self, num_nodes: usize) -> bool { + #[cfg(not(feature = "std"))] + return false; + + num_nodes >= self.parallelism_thresholds.min_revealed_nodes + } + + /// Returns true if parallelism should be enabled for updating hashes with the given number + /// of changed keys. Will always return false in nostd builds. + const fn is_update_parallelism_enabled(&self, num_changed_keys: usize) -> bool { + #[cfg(not(feature = "std"))] + return false; + + num_changed_keys >= self.parallelism_thresholds.min_updated_nodes + } + + /// Creates a new revealed sparse trie from the given root node. + /// + /// This function initializes the internal structures and then reveals the root. + /// It is a convenient method to create a trie when you already have the root node available. + /// + /// # Arguments + /// + /// * `root` - The root node of the trie + /// * `masks` - Trie masks for root branch node + /// * `retain_updates` - Whether to track updates + /// + /// # Returns + /// + /// Self if successful, or an error if revealing fails. + pub fn from_root( + root: TrieNode, + masks: TrieMasks, + retain_updates: bool, + ) -> SparseTrieResult { + Self::default().with_root(root, masks, retain_updates) + } + + /// Returns a reference to the lower `SparseSubtrie` for the given path, or None if the + /// path belongs to the upper trie, or if the lower subtrie for the path doesn't exist or is + /// blinded. + fn lower_subtrie_for_path(&self, path: &Nibbles) -> Option<&SparseSubtrie> { + match SparseSubtrieType::from_path(path) { + SparseSubtrieType::Upper => None, + SparseSubtrieType::Lower(idx) => self.lower_subtries[idx].as_revealed_ref(), + } + } + + /// Returns a mutable reference to the lower `SparseSubtrie` for the given path, or None if the + /// path belongs to the upper trie. + /// + /// This method will create/reveal a new lower subtrie for the given path if one isn't already. + /// If one does exist, but its path field is longer than the given path, then the field will be + /// set to the given path. + fn lower_subtrie_for_path_mut(&mut self, path: &Nibbles) -> Option<&mut SparseSubtrie> { + match SparseSubtrieType::from_path(path) { + SparseSubtrieType::Upper => None, + SparseSubtrieType::Lower(idx) => { + self.lower_subtries[idx].reveal(path); + Some(self.lower_subtries[idx].as_revealed_mut().expect("just revealed")) + } + } + } + + /// Returns a reference to either the lower or upper `SparseSubtrie` for the given path, + /// depending on the path's length. + /// + /// Returns `None` if a lower subtrie does not exist for the given path. + fn subtrie_for_path(&self, path: &Nibbles) -> Option<&SparseSubtrie> { + // We can't just call `lower_subtrie_for_path` and return `upper_subtrie` if it returns + // None, because Rust complains about double mutable borrowing `self`. + if SparseSubtrieType::path_len_is_upper(path.len()) { + Some(&self.upper_subtrie) + } else { + self.lower_subtrie_for_path(path) + } + } + + /// Returns a mutable reference to either the lower or upper `SparseSubtrie` for the given path, + /// depending on the path's length. + /// + /// This method will create/reveal a new lower subtrie for the given path if one isn't already. + /// If one does exist, but its path field is longer than the given path, then the field will be + /// set to the given path. + fn subtrie_for_path_mut(&mut self, path: &Nibbles) -> &mut SparseSubtrie { + // We can't just call `lower_subtrie_for_path` and return `upper_subtrie` if it returns + // None, because Rust complains about double mutable borrowing `self`. + if SparseSubtrieType::path_len_is_upper(path.len()) { + &mut self.upper_subtrie + } else { + self.lower_subtrie_for_path_mut(path).unwrap() + } + } + + /// Returns the next node in the traversal path from the given path towards the leaf for the + /// given full leaf path, or an error if any node along the traversal path is not revealed. + /// + /// + /// ## Panics + /// + /// If `from_path` is not a prefix of `leaf_full_path`. + fn find_next_to_leaf( + from_path: &Nibbles, + from_node: &SparseNode, + leaf_full_path: &Nibbles, + ) -> FindNextToLeafOutcome { + debug_assert!(leaf_full_path.len() >= from_path.len()); + debug_assert!(leaf_full_path.starts_with(from_path)); + + match from_node { + // If empty node is found it means the subtrie doesn't have any nodes in it, let alone + // the target leaf. + SparseNode::Empty => FindNextToLeafOutcome::NotFound, + SparseNode::Hash(hash) => FindNextToLeafOutcome::BlindedNode(*hash), + SparseNode::Leaf { key, .. } => { + let mut found_full_path = *from_path; + found_full_path.extend(key); + + if &found_full_path == leaf_full_path { + return FindNextToLeafOutcome::Found + } + FindNextToLeafOutcome::NotFound + } + SparseNode::Extension { key, .. } => { + if leaf_full_path.len() == from_path.len() { + return FindNextToLeafOutcome::NotFound + } + + let mut child_path = *from_path; + child_path.extend(key); + + if !leaf_full_path.starts_with(&child_path) { + return FindNextToLeafOutcome::NotFound + } + FindNextToLeafOutcome::ContinueFrom(child_path) + } + SparseNode::Branch { state_mask, .. } => { + if leaf_full_path.len() == from_path.len() { + return FindNextToLeafOutcome::NotFound + } + + let nibble = leaf_full_path.get_unchecked(from_path.len()); + if !state_mask.is_bit_set(nibble) { + return FindNextToLeafOutcome::NotFound + } + + let mut child_path = *from_path; + child_path.push_unchecked(nibble); + + FindNextToLeafOutcome::ContinueFrom(child_path) + } + } + } + + /// Called when a child node has collapsed into its parent as part of `remove_leaf`. If the + /// new parent node is a leaf, then the previous child also was, and if the previous child was + /// on a lower subtrie while the parent is on an upper then the leaf value needs to be moved to + /// the upper. + fn move_value_on_leaf_removal( + &mut self, + parent_path: &Nibbles, + new_parent_node: &SparseNode, + prev_child_path: &Nibbles, + ) { + // If the parent path isn't in the upper then it doesn't matter what the new node is, + // there's no situation where a leaf value needs to be moved. + if SparseSubtrieType::from_path(parent_path).lower_index().is_some() { + return; + } + + if let SparseNode::Leaf { key, .. } = new_parent_node { + let Some(prev_child_subtrie) = self.lower_subtrie_for_path_mut(prev_child_path) else { + return; + }; + + let mut leaf_full_path = *parent_path; + leaf_full_path.extend(key); + + let val = prev_child_subtrie.inner.values.remove(&leaf_full_path).expect("ParallelSparseTrie is in an inconsistent state, expected value on subtrie which wasn't found"); + self.upper_subtrie.inner.values.insert(leaf_full_path, val); + } + } + + /// Used by `remove_leaf` to ensure that when a node is removed from a lower subtrie that any + /// externalities are handled. These can include: + /// - Removing the lower subtrie completely, if it is now empty. + /// - Updating the `path` field of the lower subtrie to indicate that its root node has changed. + /// + /// This method assumes that the caller will deal with putting all other nodes in the trie into + /// a consistent state after the removal of this one. + /// + /// ## Panics + /// + /// - If the removed node was not a leaf or extension. + fn remove_node(&mut self, path: &Nibbles) { + let subtrie = self.subtrie_for_path_mut(path); + let node = subtrie.nodes.remove(path); + + let Some(idx) = SparseSubtrieType::from_path(path).lower_index() else { + // When removing a node from the upper trie there's nothing special we need to do to fix + // its path field; the upper trie's path is always empty. + return; + }; + + match node { + Some(SparseNode::Leaf { .. }) => { + // If the leaf was the final node in its lower subtrie then we can blind the + // subtrie, effectively marking it as empty. + if subtrie.nodes.is_empty() { + self.lower_subtries[idx].clear(); + } + } + Some(SparseNode::Extension { key, .. }) => { + // If the removed extension was the root node of a lower subtrie then the lower + // subtrie's `path` needs to be updated to be whatever node the extension used to + // point to. + if &subtrie.path == path { + subtrie.path.extend(&key); + } + } + _ => panic!("Expected to remove a leaf or extension, but removed {node:?}"), + } + } + + /// Given the path to a parent branch node and a child node which is the sole remaining child on + /// that branch after removing a leaf, returns a node to replace the parent branch node and a + /// boolean indicating if the child should be deleted. + /// + /// ## Panics + /// + /// - If either parent or child node is not already revealed. + /// - If parent's path is not a prefix of the child's path. + fn branch_changes_on_leaf_removal( + parent_path: &Nibbles, + remaining_child_path: &Nibbles, + remaining_child_node: &SparseNode, + ) -> (SparseNode, bool) { + debug_assert!(remaining_child_path.len() > parent_path.len()); + debug_assert!(remaining_child_path.starts_with(parent_path)); + + let remaining_child_nibble = remaining_child_path.get_unchecked(parent_path.len()); + + // If we swap the branch node out either an extension or leaf, depending on + // what its remaining child is. + match remaining_child_node { + SparseNode::Empty | SparseNode::Hash(_) => { + panic!("remaining child must have been revealed already") + } + // If the only child is a leaf node, we downgrade the branch node into a + // leaf node, prepending the nibble to the key, and delete the old + // child. + SparseNode::Leaf { key, .. } => { + let mut new_key = Nibbles::from_nibbles_unchecked([remaining_child_nibble]); + new_key.extend(key); + (SparseNode::new_leaf(new_key), true) + } + // If the only child node is an extension node, we downgrade the branch + // node into an even longer extension node, prepending the nibble to the + // key, and delete the old child. + SparseNode::Extension { key, .. } => { + let mut new_key = Nibbles::from_nibbles_unchecked([remaining_child_nibble]); + new_key.extend(key); + (SparseNode::new_ext(new_key), true) + } + // If the only child is a branch node, we downgrade the current branch + // node into a one-nibble extension node. + SparseNode::Branch { .. } => ( + SparseNode::new_ext(Nibbles::from_nibbles_unchecked([remaining_child_nibble])), + false, + ), + } + } + + /// Given the path to a parent extension and its key, and a child node (not necessarily on this + /// subtrie), returns an optional replacement parent node. If a replacement is returned then the + /// child node should be deleted. + /// + /// ## Panics + /// + /// - If either parent or child node is not already revealed. + /// - If parent's path is not a prefix of the child's path. + fn extension_changes_on_leaf_removal( + parent_path: &Nibbles, + parent_key: &Nibbles, + child_path: &Nibbles, + child: &SparseNode, + ) -> Option { + debug_assert!(child_path.len() > parent_path.len()); + debug_assert!(child_path.starts_with(parent_path)); + + // If the parent node is an extension node, we need to look at its child to see + // if we need to merge it. + match child { + SparseNode::Empty | SparseNode::Hash(_) => { + panic!("child must be revealed") + } + // For a leaf node, we collapse the extension node into a leaf node, + // extending the key. While it's impossible to encounter an extension node + // followed by a leaf node in a complete trie, it's possible here because we + // could have downgraded the extension node's child into a leaf node from a + // branch in a previous call to `branch_changes_on_leaf_removal`. + SparseNode::Leaf { key, .. } => { + let mut new_key = *parent_key; + new_key.extend(key); + Some(SparseNode::new_leaf(new_key)) + } + // Similar to the leaf node, for an extension node, we collapse them into one + // extension node, extending the key. + SparseNode::Extension { key, .. } => { + let mut new_key = *parent_key; + new_key.extend(key); + Some(SparseNode::new_ext(new_key)) + } + // For a branch node, we just leave the extension node as-is. + SparseNode::Branch { .. } => None, + } + } + + /// Called when a leaf is removed on a branch which has only one other remaining child. That + /// child must be revealed in order to properly collapse the branch. + /// + /// If `recurse_into_extension` is true, and the remaining child is an extension node, then its + /// child will be ensured to be revealed as well. + /// + /// ## Returns + /// + /// The node of the remaining child, whether it was already revealed or not. + fn reveal_remaining_child_on_leaf_removal( + &mut self, + provider: P, + full_path: &Nibbles, // only needed for logs + remaining_child_path: &Nibbles, + recurse_into_extension: bool, + ) -> SparseTrieResult { + let remaining_child_subtrie = self.subtrie_for_path_mut(remaining_child_path); + + let remaining_child_node = + match remaining_child_subtrie.nodes.get(remaining_child_path).unwrap() { + SparseNode::Hash(_) => { + debug!( + target: "trie::parallel_sparse", + child_path = ?remaining_child_path, + leaf_full_path = ?full_path, + "Node child not revealed in remove_leaf, falling back to db", + ); + if let Some(RevealedNode { node, tree_mask, hash_mask }) = + provider.trie_node(remaining_child_path)? + { + let decoded = TrieNode::decode(&mut &node[..])?; + trace!( + target: "trie::parallel_sparse", + ?remaining_child_path, + ?decoded, + ?tree_mask, + ?hash_mask, + "Revealing remaining blinded branch child" + ); + remaining_child_subtrie.reveal_node( + *remaining_child_path, + &decoded, + TrieMasks { hash_mask, tree_mask }, + )?; + remaining_child_subtrie.nodes.get(remaining_child_path).unwrap().clone() + } else { + return Err(SparseTrieErrorKind::NodeNotFoundInProvider { + path: *remaining_child_path, + } + .into()) + } + } + node => node.clone(), + }; + + // If `recurse_into_extension` is true, and the remaining child is an extension node, then + // its child will be ensured to be revealed as well. This is required for generation of + // trie updates; without revealing the grandchild branch it's not always possible to know + // if the tree mask bit should be set for the child extension on its parent branch. + if let SparseNode::Extension { key, .. } = &remaining_child_node && + recurse_into_extension + { + let mut remaining_grandchild_path = *remaining_child_path; + remaining_grandchild_path.extend(key); + + trace!( + target: "trie::parallel_sparse", + remaining_grandchild_path = ?remaining_grandchild_path, + child_path = ?remaining_child_path, + leaf_full_path = ?full_path, + "Revealing child of extension node, which is the last remaining child of the branch" + ); + + self.reveal_remaining_child_on_leaf_removal( + provider, + full_path, + &remaining_grandchild_path, + false, // recurse_into_extension + )?; + } + + Ok(remaining_child_node) + } + + /// Drains any [`SparseTrieUpdatesAction`]s from the given subtrie, and applies each action to + /// the given `updates` set. If the given set is None then this is a no-op. + #[instrument(level = "trace", target = "trie::parallel_sparse", skip_all)] + fn apply_subtrie_update_actions( + &mut self, + update_actions: impl Iterator, + ) { + if let Some(updates) = self.updates.as_mut() { + for action in update_actions { + match action { + SparseTrieUpdatesAction::InsertRemoved(path) => { + updates.updated_nodes.remove(&path); + updates.removed_nodes.insert(path); + } + SparseTrieUpdatesAction::RemoveUpdated(path) => { + updates.updated_nodes.remove(&path); + } + SparseTrieUpdatesAction::InsertUpdated(path, branch_node) => { + updates.updated_nodes.insert(path, branch_node); + } + } + } + }; + } + + /// Updates hashes for the upper subtrie, using nodes from both upper and lower subtries. + #[instrument(level = "trace", target = "trie::parallel_sparse", skip_all, ret)] + fn update_upper_subtrie_hashes(&mut self, prefix_set: &mut PrefixSet) -> RlpNode { + trace!(target: "trie::parallel_sparse", "Updating upper subtrie hashes"); + + debug_assert!(self.upper_subtrie.inner.buffers.path_stack.is_empty()); + self.upper_subtrie.inner.buffers.path_stack.push(RlpNodePathStackItem { + path: Nibbles::default(), // Start from root + is_in_prefix_set: None, + }); + + #[cfg(feature = "metrics")] + let start = std::time::Instant::now(); + + let mut update_actions_buf = + self.updates_enabled().then(|| self.update_actions_buffers.pop().unwrap_or_default()); + + while let Some(stack_item) = self.upper_subtrie.inner.buffers.path_stack.pop() { + let path = stack_item.path; + let node = if path.len() < UPPER_TRIE_MAX_DEPTH { + self.upper_subtrie.nodes.get_mut(&path).expect("upper subtrie node must exist") + } else { + let index = path_subtrie_index_unchecked(&path); + let node = self.lower_subtries[index] + .as_revealed_mut() + .expect("lower subtrie must exist") + .nodes + .get_mut(&path) + .expect("lower subtrie node must exist"); + // Lower subtrie root node hashes must be computed before updating upper subtrie + // hashes + debug_assert!( + node.hash().is_some(), + "Lower subtrie root node at path {path:?} has no hash" + ); + node + }; + + // Calculate the RLP node for the current node using upper subtrie + self.upper_subtrie.inner.rlp_node( + prefix_set, + &mut update_actions_buf, + stack_item, + node, + &self.branch_node_tree_masks, + &self.branch_node_hash_masks, + ); + } + + // If there were any branch node updates as a result of calculating the RLP node for the + // upper trie then apply them to the top-level set. + if let Some(mut update_actions_buf) = update_actions_buf { + self.apply_subtrie_update_actions( + #[allow(clippy::iter_with_drain)] + update_actions_buf.drain(..), + ); + self.update_actions_buffers.push(update_actions_buf); + } + + #[cfg(feature = "metrics")] + self.metrics.subtrie_upper_hash_latency.record(start.elapsed()); + + debug_assert_eq!(self.upper_subtrie.inner.buffers.rlp_node_stack.len(), 1); + self.upper_subtrie.inner.buffers.rlp_node_stack.pop().unwrap().rlp_node + } + + /// Returns: + /// 1. List of lower [subtries](SparseSubtrie) that have changed according to the provided + /// [prefix set](PrefixSet). See documentation of [`ChangedSubtrie`] for more details. Lower + /// subtries whose root node is missing a hash will also be returned; this is required to + /// handle cases where extensions/leafs get shortened and therefore moved from the upper to a + /// lower subtrie. + /// 2. Prefix set of keys that do not belong to any lower subtrie. + /// + /// This method helps optimize hash recalculations by identifying which specific + /// lower subtries need to be updated. Each lower subtrie can then be updated in parallel. + /// + /// IMPORTANT: The method removes the subtries from `lower_subtries`, and the caller is + /// responsible for returning them back into the array. + #[instrument(level = "trace", target = "trie::parallel_sparse", skip_all, fields(prefix_set_len = prefix_set.len()))] + fn take_changed_lower_subtries( + &mut self, + prefix_set: &mut PrefixSet, + ) -> (Vec, PrefixSetMut) { + // Fast-path: If the prefix set is empty then no subtries can have been changed. Just return + // empty values. + if prefix_set.is_empty() && !prefix_set.all() { + return Default::default(); + } + + // Clone the prefix set to iterate over its keys. Cloning is cheap, it's just an Arc. + let prefix_set_clone = prefix_set.clone(); + let mut prefix_set_iter = prefix_set_clone.into_iter().copied().peekable(); + let mut changed_subtries = Vec::new(); + let mut unchanged_prefix_set = PrefixSetMut::default(); + let updates_enabled = self.updates_enabled(); + + for (index, subtrie) in self.lower_subtries.iter_mut().enumerate() { + if let Some(subtrie) = subtrie.take_revealed_if(|subtrie| { + prefix_set.contains(&subtrie.path) || + subtrie.nodes.get(&subtrie.path).is_some_and(|n| n.hash().is_none()) + }) { + let prefix_set = if prefix_set.all() { + unchanged_prefix_set = PrefixSetMut::all(); + PrefixSetMut::all() + } else { + // Take those keys from the original prefix set that start with the subtrie path + // + // Subtries are stored in the order of their paths, so we can use the same + // prefix set iterator. + let mut new_prefix_set = Vec::new(); + while let Some(key) = prefix_set_iter.peek() { + if key.starts_with(&subtrie.path) { + // If the key starts with the subtrie path, add it to the new prefix set + new_prefix_set.push(prefix_set_iter.next().unwrap()); + } else if new_prefix_set.is_empty() && key < &subtrie.path { + // If we didn't yet have any keys that belong to this subtrie, and the + // current key is still less than the subtrie path, add it to the + // unchanged prefix set + unchanged_prefix_set.insert(prefix_set_iter.next().unwrap()); + } else { + // If we're past the subtrie path, we're done with this subtrie. Do not + // advance the iterator, the next key will be processed either by the + // next subtrie or inserted into the unchanged prefix set. + break + } + } + PrefixSetMut::from(new_prefix_set) + } + .freeze(); + + // We need the full path of root node of the lower subtrie to the unchanged prefix + // set, so that we don't skip it when calculating hashes for the upper subtrie. + match subtrie.nodes.get(&subtrie.path) { + Some(SparseNode::Extension { key, .. } | SparseNode::Leaf { key, .. }) => { + unchanged_prefix_set.insert(subtrie.path.join(key)); + } + Some(SparseNode::Branch { .. }) => { + unchanged_prefix_set.insert(subtrie.path); + } + _ => {} + } + + let update_actions_buf = + updates_enabled.then(|| self.update_actions_buffers.pop().unwrap_or_default()); + + changed_subtries.push(ChangedSubtrie { + index, + subtrie, + prefix_set, + update_actions_buf, + }); + } + } + + // Extend the unchanged prefix set with the remaining keys that are not part of any subtries + unchanged_prefix_set.extend_keys(prefix_set_iter); + + (changed_subtries, unchanged_prefix_set) + } + + /// Returns an iterator over all nodes in the trie in no particular order. + #[cfg(test)] + fn all_nodes(&self) -> impl IntoIterator { + let mut nodes = vec![]; + for subtrie in self.lower_subtries.iter().filter_map(LowerSparseSubtrie::as_revealed_ref) { + nodes.extend(subtrie.nodes.iter()) + } + nodes.extend(self.upper_subtrie.nodes.iter()); + nodes + } + + /// Reveals a trie node in the upper trie if it has not been revealed before. When revealing + /// branch/extension nodes this may recurse into a lower trie to reveal a child. + /// + /// This function decodes a trie node and inserts it into the trie structure. It handles + /// different node types (leaf, extension, branch) by appropriately adding them to the trie and + /// recursively revealing their children. + /// + /// # Arguments + /// + /// * `path` - The path where the node should be revealed + /// * `node` - The trie node to reveal + /// * `masks` - Trie masks for branch nodes + /// + /// # Returns + /// + /// `Ok(())` if successful, or an error if the node was not revealed. + fn reveal_upper_node( + &mut self, + path: Nibbles, + node: &TrieNode, + masks: TrieMasks, + ) -> SparseTrieResult<()> { + // If there is no subtrie for the path it means the path is UPPER_TRIE_MAX_DEPTH or less + // nibbles, and so belongs to the upper trie. + self.upper_subtrie.reveal_node(path, node, masks)?; + + // The previous upper_trie.reveal_node call will not have revealed any child nodes via + // reveal_node_or_hash if the child node would be found on a lower subtrie. We handle that + // here by manually checking the specific cases where this could happen, and calling + // reveal_node_or_hash for each. + match node { + TrieNode::Branch(branch) => { + // If a branch is at the cutoff level of the trie then it will be in the upper trie, + // but all of its children will be in a lower trie. Check if a child node would be + // in the lower subtrie, and reveal accordingly. + if !SparseSubtrieType::path_len_is_upper(path.len() + 1) { + let mut stack_ptr = branch.as_ref().first_child_index(); + for idx in CHILD_INDEX_RANGE { + if branch.state_mask.is_bit_set(idx) { + let mut child_path = path; + child_path.push_unchecked(idx); + self.lower_subtrie_for_path_mut(&child_path) + .expect("child_path must have a lower subtrie") + .reveal_node_or_hash(child_path, &branch.stack[stack_ptr])?; + stack_ptr += 1; + } + } + } + } + TrieNode::Extension(ext) => { + let mut child_path = path; + child_path.extend(&ext.key); + if let Some(subtrie) = self.lower_subtrie_for_path_mut(&child_path) { + subtrie.reveal_node_or_hash(child_path, &ext.child)?; + } + } + TrieNode::EmptyRoot | TrieNode::Leaf(_) => (), + } + + Ok(()) + } + + /// Return updated subtries back to the trie after executing any actions required on the + /// top-level `SparseTrieUpdates`. + #[instrument(level = "trace", target = "trie::parallel_sparse", skip_all)] + fn insert_changed_subtries( + &mut self, + changed_subtries: impl IntoIterator, + ) { + for ChangedSubtrie { index, subtrie, update_actions_buf, .. } in changed_subtries { + if let Some(mut update_actions_buf) = update_actions_buf { + self.apply_subtrie_update_actions( + #[allow(clippy::iter_with_drain)] + update_actions_buf.drain(..), + ); + self.update_actions_buffers.push(update_actions_buf); + } + + self.lower_subtries[index] = LowerSparseSubtrie::Revealed(subtrie); + } + } +} + +/// This is a subtrie of the [`ParallelSparseTrie`] that contains a map from path to sparse trie +/// nodes. +#[derive(Clone, PartialEq, Eq, Debug, Default)] +pub struct SparseSubtrie { + /// The root path of this subtrie. + /// + /// This is the _full_ path to this subtrie, meaning it includes the first + /// [`UPPER_TRIE_MAX_DEPTH`] nibbles that we also use for indexing subtries in the + /// [`ParallelSparseTrie`]. + /// + /// There should be a node for this path in `nodes` map. + pub(crate) path: Nibbles, + /// The map from paths to sparse trie nodes within this subtrie. + nodes: HashMap, + /// Subset of fields for mutable access while `nodes` field is also being mutably borrowed. + inner: SparseSubtrieInner, +} + +/// Returned by the `find_next_to_leaf` method to indicate either that the leaf has been found, +/// traversal should be continued from the given path, or the leaf is not in the trie. +enum FindNextToLeafOutcome { + /// `Found` indicates that the leaf was found at the given path. + Found, + /// `ContinueFrom` indicates that traversal should continue from the given path. + ContinueFrom(Nibbles), + /// `NotFound` indicates that there is no way to traverse to the leaf, as it is not in the + /// trie. + NotFound, + /// `BlindedNode` indicates that the node is blinded with the contained hash and cannot be + /// traversed. + BlindedNode(B256), +} + +impl SparseSubtrie { + /// Creates a new empty subtrie with the specified root path. + pub(crate) fn new(path: Nibbles) -> Self { + Self { path, ..Default::default() } + } + + /// Returns true if this subtrie has any nodes, false otherwise. + pub(crate) fn is_empty(&self) -> bool { + self.nodes.is_empty() + } + + /// Returns true if the current path and its child are both found in the same level. + fn is_child_same_level(current_path: &Nibbles, child_path: &Nibbles) -> bool { + let current_level = core::mem::discriminant(&SparseSubtrieType::from_path(current_path)); + let child_level = core::mem::discriminant(&SparseSubtrieType::from_path(child_path)); + current_level == child_level + } + + /// Updates or inserts a leaf node at the specified key path with the provided RLP-encoded + /// value. + /// + /// If the leaf did not previously exist, this method adjusts the trie structure by inserting + /// new leaf nodes, splitting branch nodes, or collapsing extension nodes as needed. + /// + /// # Returns + /// + /// Returns the `Ok` if the update is successful. + /// + /// Note: If an update requires revealing a blinded node, an error is returned if the blinded + /// provider returns an error. + pub fn update_leaf( + &mut self, + full_path: Nibbles, + value: Vec, + provider: impl TrieNodeProvider, + retain_updates: bool, + ) -> SparseTrieResult<()> { + debug_assert!(full_path.starts_with(&self.path)); + let existing = self.inner.values.insert(full_path, value); + if existing.is_some() { + // trie structure unchanged, return immediately + return Ok(()) + } + + // Here we are starting at the root of the subtrie, and traversing from there. + let mut current = Some(self.path); + while let Some(current_path) = current { + match self.update_next_node(current_path, &full_path, retain_updates)? { + LeafUpdateStep::Continue { next_node } => { + current = Some(next_node); + } + LeafUpdateStep::Complete { reveal_path, .. } => { + if let Some(reveal_path) = reveal_path && + self.nodes.get(&reveal_path).expect("node must exist").is_hash() + { + debug!( + target: "trie::parallel_sparse", + child_path = ?reveal_path, + leaf_full_path = ?full_path, + "Extension node child not revealed in update_leaf, falling back to db", + ); + if let Some(RevealedNode { node, tree_mask, hash_mask }) = + provider.trie_node(&reveal_path)? + { + let decoded = TrieNode::decode(&mut &node[..])?; + trace!( + target: "trie::parallel_sparse", + ?reveal_path, + ?decoded, + ?tree_mask, + ?hash_mask, + "Revealing child (from lower)", + ); + self.reveal_node( + reveal_path, + &decoded, + TrieMasks { hash_mask, tree_mask }, + )?; + } else { + return Err(SparseTrieErrorKind::NodeNotFoundInProvider { + path: reveal_path, + } + .into()) + } + } + + current = None; + } + LeafUpdateStep::NodeNotFound => { + current = None; + } + } + } + + Ok(()) + } + + /// Processes the current node, returning what to do next in the leaf update process. + /// + /// This will add or update any nodes in the trie as necessary. + /// + /// Returns a `LeafUpdateStep` containing the next node to process (if any) and + /// the paths of nodes that were inserted during this step. + fn update_next_node( + &mut self, + mut current: Nibbles, + path: &Nibbles, + retain_updates: bool, + ) -> SparseTrieResult { + debug_assert!(path.starts_with(&self.path)); + debug_assert!(current.starts_with(&self.path)); + debug_assert!(path.starts_with(¤t)); + let Some(node) = self.nodes.get_mut(¤t) else { + return Ok(LeafUpdateStep::NodeNotFound); + }; + match node { + SparseNode::Empty => { + // We need to insert the node with a different path and key depending on the path of + // the subtrie. + let path = path.slice(self.path.len()..); + *node = SparseNode::new_leaf(path); + Ok(LeafUpdateStep::complete_with_insertions(vec![current], None)) + } + SparseNode::Hash(hash) => { + Err(SparseTrieErrorKind::BlindedNode { path: current, hash: *hash }.into()) + } + SparseNode::Leaf { key: current_key, .. } => { + current.extend(current_key); + + // this leaf is being updated + debug_assert!( + ¤t != path, + "we already checked leaf presence in the beginning" + ); + + // find the common prefix + let common = current.common_prefix_length(path); + + // update existing node + let new_ext_key = current.slice(current.len() - current_key.len()..common); + *node = SparseNode::new_ext(new_ext_key); + + // create a branch node and corresponding leaves + self.nodes.reserve(3); + let branch_path = current.slice(..common); + let new_leaf_path = path.slice(..=common); + let existing_leaf_path = current.slice(..=common); + + self.nodes.insert( + branch_path, + SparseNode::new_split_branch( + current.get_unchecked(common), + path.get_unchecked(common), + ), + ); + self.nodes.insert(new_leaf_path, SparseNode::new_leaf(path.slice(common + 1..))); + self.nodes + .insert(existing_leaf_path, SparseNode::new_leaf(current.slice(common + 1..))); + + Ok(LeafUpdateStep::complete_with_insertions( + vec![branch_path, new_leaf_path, existing_leaf_path], + None, + )) + } + SparseNode::Extension { key, .. } => { + current.extend(key); + + if !path.starts_with(¤t) { + // find the common prefix + let common = current.common_prefix_length(path); + *key = current.slice(current.len() - key.len()..common); + + // If branch node updates retention is enabled, we need to query the + // extension node child to later set the hash mask for a parent branch node + // correctly. + let reveal_path = retain_updates.then_some(current); + + // create state mask for new branch node + // NOTE: this might overwrite the current extension node + self.nodes.reserve(3); + let branch_path = current.slice(..common); + let new_leaf_path = path.slice(..=common); + let branch = SparseNode::new_split_branch( + current.get_unchecked(common), + path.get_unchecked(common), + ); + + self.nodes.insert(branch_path, branch); + + // create new leaf + let new_leaf = SparseNode::new_leaf(path.slice(common + 1..)); + self.nodes.insert(new_leaf_path, new_leaf); + + let mut inserted_nodes = vec![branch_path, new_leaf_path]; + + // recreate extension to previous child if needed + let key = current.slice(common + 1..); + if !key.is_empty() { + let ext_path = current.slice(..=common); + self.nodes.insert(ext_path, SparseNode::new_ext(key)); + inserted_nodes.push(ext_path); + } + + return Ok(LeafUpdateStep::complete_with_insertions(inserted_nodes, reveal_path)) + } + + Ok(LeafUpdateStep::continue_with(current)) + } + SparseNode::Branch { state_mask, .. } => { + let nibble = path.get_unchecked(current.len()); + current.push_unchecked(nibble); + if !state_mask.is_bit_set(nibble) { + state_mask.set_bit(nibble); + let new_leaf = SparseNode::new_leaf(path.slice(current.len()..)); + self.nodes.insert(current, new_leaf); + return Ok(LeafUpdateStep::complete_with_insertions(vec![current], None)) + } + + // If the nibble is set, we can continue traversing the branch. + Ok(LeafUpdateStep::continue_with(current)) + } + } + } + + /// Internal implementation of the method of the same name on `ParallelSparseTrie`. + fn reveal_node( + &mut self, + path: Nibbles, + node: &TrieNode, + masks: TrieMasks, + ) -> SparseTrieResult<()> { + debug_assert!(path.starts_with(&self.path)); + + // If the node is already revealed and it's not a hash node, do nothing. + if self.nodes.get(&path).is_some_and(|node| !node.is_hash()) { + return Ok(()) + } + + match node { + TrieNode::EmptyRoot => { + // For an empty root, ensure that we are at the root path, and at the upper subtrie. + debug_assert!(path.is_empty()); + debug_assert!(self.path.is_empty()); + self.nodes.insert(path, SparseNode::Empty); + } + TrieNode::Branch(branch) => { + // For a branch node, iterate over all potential children + let mut stack_ptr = branch.as_ref().first_child_index(); + for idx in CHILD_INDEX_RANGE { + if branch.state_mask.is_bit_set(idx) { + let mut child_path = path; + child_path.push_unchecked(idx); + if Self::is_child_same_level(&path, &child_path) { + // Reveal each child node or hash it has, but only if the child is on + // the same level as the parent. + self.reveal_node_or_hash(child_path, &branch.stack[stack_ptr])?; + } + stack_ptr += 1; + } + } + // Update the branch node entry in the nodes map, handling cases where a blinded + // node is now replaced with a revealed node. + match self.nodes.entry(path) { + Entry::Occupied(mut entry) => match entry.get() { + // Replace a hash node with a fully revealed branch node. + SparseNode::Hash(hash) => { + entry.insert(SparseNode::Branch { + state_mask: branch.state_mask, + // Memoize the hash of a previously blinded node in a new branch + // node. + hash: Some(*hash), + store_in_db_trie: Some( + masks.hash_mask.is_some_and(|mask| !mask.is_empty()) || + masks.tree_mask.is_some_and(|mask| !mask.is_empty()), + ), + }); + } + // Branch node already exists, or an extension node was placed where a + // branch node was before. + SparseNode::Branch { .. } | SparseNode::Extension { .. } => {} + // All other node types can't be handled. + node @ (SparseNode::Empty | SparseNode::Leaf { .. }) => { + return Err(SparseTrieErrorKind::Reveal { + path: *entry.key(), + node: Box::new(node.clone()), + } + .into()) + } + }, + Entry::Vacant(entry) => { + entry.insert(SparseNode::new_branch(branch.state_mask)); + } + } + } + TrieNode::Extension(ext) => match self.nodes.entry(path) { + Entry::Occupied(mut entry) => match entry.get() { + // Replace a hash node with a revealed extension node. + SparseNode::Hash(hash) => { + let mut child_path = *entry.key(); + child_path.extend(&ext.key); + entry.insert(SparseNode::Extension { + key: ext.key, + // Memoize the hash of a previously blinded node in a new extension + // node. + hash: Some(*hash), + store_in_db_trie: None, + }); + if Self::is_child_same_level(&path, &child_path) { + self.reveal_node_or_hash(child_path, &ext.child)?; + } + } + // Extension node already exists, or an extension node was placed where a branch + // node was before. + SparseNode::Extension { .. } | SparseNode::Branch { .. } => {} + // All other node types can't be handled. + node @ (SparseNode::Empty | SparseNode::Leaf { .. }) => { + return Err(SparseTrieErrorKind::Reveal { + path: *entry.key(), + node: Box::new(node.clone()), + } + .into()) + } + }, + Entry::Vacant(entry) => { + let mut child_path = *entry.key(); + child_path.extend(&ext.key); + entry.insert(SparseNode::new_ext(ext.key)); + if Self::is_child_same_level(&path, &child_path) { + self.reveal_node_or_hash(child_path, &ext.child)?; + } + } + }, + TrieNode::Leaf(leaf) => match self.nodes.entry(path) { + Entry::Occupied(mut entry) => match entry.get() { + // Replace a hash node with a revealed leaf node and store leaf node value. + SparseNode::Hash(hash) => { + let mut full = *entry.key(); + full.extend(&leaf.key); + self.inner.values.insert(full, leaf.value.clone()); + entry.insert(SparseNode::Leaf { + key: leaf.key, + // Memoize the hash of a previously blinded node in a new leaf + // node. + hash: Some(*hash), + }); + } + // Leaf node already exists. + SparseNode::Leaf { .. } => {} + // All other node types can't be handled. + node @ (SparseNode::Empty | + SparseNode::Extension { .. } | + SparseNode::Branch { .. }) => { + return Err(SparseTrieErrorKind::Reveal { + path: *entry.key(), + node: Box::new(node.clone()), + } + .into()) + } + }, + Entry::Vacant(entry) => { + let mut full = *entry.key(); + full.extend(&leaf.key); + entry.insert(SparseNode::new_leaf(leaf.key)); + self.inner.values.insert(full, leaf.value.clone()); + } + }, + } + + Ok(()) + } + + /// Reveals either a node or its hash placeholder based on the provided child data. + /// + /// When traversing the trie, we often encounter references to child nodes that + /// are either directly embedded or represented by their hash. This method + /// handles both cases: + /// + /// 1. If the child data represents a hash (32+1=33 bytes), store it as a hash node + /// 2. Otherwise, decode the data as a [`TrieNode`] and recursively reveal it using + /// `reveal_node` + /// + /// # Returns + /// + /// Returns `Ok(())` if successful, or an error if the node cannot be revealed. + /// + /// # Error Handling + /// + /// Will error if there's a conflict between a new hash node and an existing one + /// at the same path + fn reveal_node_or_hash(&mut self, path: Nibbles, child: &[u8]) -> SparseTrieResult<()> { + if child.len() == B256::len_bytes() + 1 { + let hash = B256::from_slice(&child[1..]); + match self.nodes.entry(path) { + Entry::Occupied(entry) => match entry.get() { + // Hash node with a different hash can't be handled. + SparseNode::Hash(previous_hash) if previous_hash != &hash => { + return Err(SparseTrieErrorKind::Reveal { + path: *entry.key(), + node: Box::new(SparseNode::Hash(hash)), + } + .into()) + } + _ => {} + }, + Entry::Vacant(entry) => { + entry.insert(SparseNode::Hash(hash)); + } + } + return Ok(()) + } + + self.reveal_node(path, &TrieNode::decode(&mut &child[..])?, TrieMasks::none()) + } + + /// Recalculates and updates the RLP hashes for the changed nodes in this subtrie. + /// + /// The function starts from the subtrie root, traverses down to leaves, and then calculates + /// the hashes from leaves back up to the root. It uses a stack from [`SparseSubtrieBuffers`] to + /// track the traversal and accumulate RLP encodings. + /// + /// # Parameters + /// + /// - `prefix_set`: The set of trie paths whose nodes have changed. + /// - `update_actions`: A buffer which `SparseTrieUpdatesAction`s will be written to in the + /// event that any changes to the top-level updates are required. If None then update + /// retention is disabled. + /// - `branch_node_tree_masks`: The tree masks for branch nodes + /// - `branch_node_hash_masks`: The hash masks for branch nodes + /// + /// # Returns + /// + /// A tuple containing the root node of the updated subtrie. + /// + /// # Panics + /// + /// If the node at the root path does not exist. + #[instrument(level = "trace", target = "trie::parallel_sparse", skip_all, fields(root = ?self.path), ret)] + fn update_hashes( + &mut self, + prefix_set: &mut PrefixSet, + update_actions: &mut Option>, + branch_node_tree_masks: &HashMap, + branch_node_hash_masks: &HashMap, + ) -> RlpNode { + trace!(target: "trie::parallel_sparse", "Updating subtrie hashes"); + + debug_assert!(prefix_set.iter().all(|path| path.starts_with(&self.path))); + + debug_assert!(self.inner.buffers.path_stack.is_empty()); + self.inner + .buffers + .path_stack + .push(RlpNodePathStackItem { path: self.path, is_in_prefix_set: None }); + + while let Some(stack_item) = self.inner.buffers.path_stack.pop() { + let path = stack_item.path; + let node = self + .nodes + .get_mut(&path) + .unwrap_or_else(|| panic!("node at path {path:?} does not exist")); + + self.inner.rlp_node( + prefix_set, + update_actions, + stack_item, + node, + branch_node_tree_masks, + branch_node_hash_masks, + ); + } + + debug_assert_eq!(self.inner.buffers.rlp_node_stack.len(), 1); + self.inner.buffers.rlp_node_stack.pop().unwrap().rlp_node + } + + /// Removes all nodes and values from the subtrie, resetting it to a blank state + /// with only an empty root node. This is used when a storage root is deleted. + fn wipe(&mut self) { + self.nodes = HashMap::from_iter([(Nibbles::default(), SparseNode::Empty)]); + self.inner.clear(); + } + + /// Clears the subtrie, keeping the data structures allocated. + pub(crate) fn clear(&mut self) { + self.nodes.clear(); + self.inner.clear(); + } + + /// Shrinks the capacity of the subtrie's node storage. + pub(crate) fn shrink_nodes_to(&mut self, size: usize) { + self.nodes.shrink_to(size); + } + + /// Shrinks the capacity of the subtrie's value storage. + pub(crate) fn shrink_values_to(&mut self, size: usize) { + self.inner.values.shrink_to(size); + } +} + +/// Helper type for [`SparseSubtrie`] to mutably access only a subset of fields from the original +/// struct. +#[derive(Clone, PartialEq, Eq, Debug, Default)] +struct SparseSubtrieInner { + /// Map from leaf key paths to their values. + /// All values are stored here instead of directly in leaf nodes. + values: HashMap>, + /// Reusable buffers for [`SparseSubtrie::update_hashes`]. + buffers: SparseSubtrieBuffers, +} + +impl SparseSubtrieInner { + /// Computes the RLP encoding and its hash for a single (trie node)[`SparseNode`]. + /// + /// # Deferred Processing + /// + /// When an extension or a branch node depends on child nodes that haven't been computed yet, + /// the function pushes the current node back onto the path stack along with its children, + /// then returns early. This allows the iterative algorithm to process children first before + /// retrying the parent. + /// + /// # Parameters + /// + /// - `prefix_set`: Set of prefixes (key paths) that have been marked as updated + /// - `update_actions`: A buffer which `SparseTrieUpdatesAction`s will be written to in the + /// event that any changes to the top-level updates are required. If None then update + /// retention is disabled. + /// - `stack_item`: The stack item to process + /// - `node`: The sparse node to process (will be mutated to update hash) + /// - `branch_node_tree_masks`: The tree masks for branch nodes + /// - `branch_node_hash_masks`: The hash masks for branch nodes + /// + /// # Side Effects + /// + /// - Updates the node's hash field after computing RLP + /// - Pushes nodes to [`SparseSubtrieBuffers::path_stack`] to manage traversal + /// - May push items onto the path stack for deferred processing + /// + /// # Exit condition + /// + /// Once all nodes have been processed and all RLPs and hashes calculated, pushes the root node + /// onto the [`SparseSubtrieBuffers::rlp_node_stack`] and exits. + fn rlp_node( + &mut self, + prefix_set: &mut PrefixSet, + update_actions: &mut Option>, + mut stack_item: RlpNodePathStackItem, + node: &mut SparseNode, + branch_node_tree_masks: &HashMap, + branch_node_hash_masks: &HashMap, + ) { + let path = stack_item.path; + trace!( + target: "trie::parallel_sparse", + ?path, + ?node, + "Calculating node RLP" + ); + + // Check if the path is in the prefix set. + // First, check the cached value. If it's `None`, then check the prefix set, and update + // the cached value. + let mut prefix_set_contains = |path: &Nibbles| { + *stack_item.is_in_prefix_set.get_or_insert_with(|| prefix_set.contains(path)) + }; + + let (rlp_node, node_type) = match node { + SparseNode::Empty => (RlpNode::word_rlp(&EMPTY_ROOT_HASH), SparseNodeType::Empty), + SparseNode::Hash(hash) => { + // Return pre-computed hash of a blinded node immediately + (RlpNode::word_rlp(hash), SparseNodeType::Hash) + } + SparseNode::Leaf { key, hash } => { + let mut path = path; + path.extend(key); + let value = self.values.get(&path); + if let Some(hash) = hash.filter(|_| !prefix_set_contains(&path) || value.is_none()) + { + // If the node hash is already computed, and either the node path is not in + // the prefix set or the leaf doesn't belong to the current trie (its value is + // absent), return the pre-computed hash + (RlpNode::word_rlp(&hash), SparseNodeType::Leaf) + } else { + // Encode the leaf node and update its hash + let value = self.values.get(&path).unwrap(); + self.buffers.rlp_buf.clear(); + let rlp_node = LeafNodeRef { key, value }.rlp(&mut self.buffers.rlp_buf); + *hash = rlp_node.as_hash(); + (rlp_node, SparseNodeType::Leaf) + } + } + SparseNode::Extension { key, hash, store_in_db_trie } => { + let mut child_path = path; + child_path.extend(key); + if let Some((hash, store_in_db_trie)) = + hash.zip(*store_in_db_trie).filter(|_| !prefix_set_contains(&path)) + { + // If the node hash is already computed, and the node path is not in + // the prefix set, return the pre-computed hash + ( + RlpNode::word_rlp(&hash), + SparseNodeType::Extension { store_in_db_trie: Some(store_in_db_trie) }, + ) + } else if self.buffers.rlp_node_stack.last().is_some_and(|e| e.path == child_path) { + // Top of the stack has the child node, we can encode the extension node and + // update its hash + let RlpNodeStackItem { path: _, rlp_node: child, node_type: child_node_type } = + self.buffers.rlp_node_stack.pop().unwrap(); + self.buffers.rlp_buf.clear(); + let rlp_node = + ExtensionNodeRef::new(key, &child).rlp(&mut self.buffers.rlp_buf); + *hash = rlp_node.as_hash(); + + let store_in_db_trie_value = child_node_type.store_in_db_trie(); + + trace!( + target: "trie::parallel_sparse", + ?path, + ?child_path, + ?child_node_type, + "Extension node" + ); + + *store_in_db_trie = store_in_db_trie_value; + + ( + rlp_node, + SparseNodeType::Extension { + // Inherit the `store_in_db_trie` flag from the child node, which is + // always the branch node + store_in_db_trie: store_in_db_trie_value, + }, + ) + } else { + // Need to defer processing until child is computed, on the next + // invocation update the node's hash. + self.buffers.path_stack.extend([ + RlpNodePathStackItem { + path, + is_in_prefix_set: Some(prefix_set_contains(&path)), + }, + RlpNodePathStackItem { path: child_path, is_in_prefix_set: None }, + ]); + return + } + } + SparseNode::Branch { state_mask, hash, store_in_db_trie } => { + if let Some((hash, store_in_db_trie)) = + hash.zip(*store_in_db_trie).filter(|_| !prefix_set_contains(&path)) + { + // If the node hash is already computed, and the node path is not in + // the prefix set, return the pre-computed hash + self.buffers.rlp_node_stack.push(RlpNodeStackItem { + path, + rlp_node: RlpNode::word_rlp(&hash), + node_type: SparseNodeType::Branch { + store_in_db_trie: Some(store_in_db_trie), + }, + }); + return + } + + let retain_updates = update_actions.is_some() && prefix_set_contains(&path); + + self.buffers.branch_child_buf.clear(); + // Walk children in a reverse order from `f` to `0`, so we pop the `0` first + // from the stack and keep walking in the sorted order. + for bit in CHILD_INDEX_RANGE.rev() { + if state_mask.is_bit_set(bit) { + let mut child = path; + child.push_unchecked(bit); + self.buffers.branch_child_buf.push(child); + } + } + + self.buffers + .branch_value_stack_buf + .resize(self.buffers.branch_child_buf.len(), Default::default()); + let mut added_children = false; + + let mut tree_mask = TrieMask::default(); + let mut hash_mask = TrieMask::default(); + let mut hashes = Vec::new(); + for (i, child_path) in self.buffers.branch_child_buf.iter().enumerate() { + if self.buffers.rlp_node_stack.last().is_some_and(|e| &e.path == child_path) { + let RlpNodeStackItem { + path: _, + rlp_node: child, + node_type: child_node_type, + } = self.buffers.rlp_node_stack.pop().unwrap(); + + // Update the masks only if we need to retain trie updates + if retain_updates { + // SAFETY: it's a child, so it's never empty + let last_child_nibble = child_path.last().unwrap(); + + // Determine whether we need to set trie mask bit. + let should_set_tree_mask_bit = if let Some(store_in_db_trie) = + child_node_type.store_in_db_trie() + { + // A branch or an extension node explicitly set the + // `store_in_db_trie` flag + store_in_db_trie + } else { + // A blinded node has the tree mask bit set + child_node_type.is_hash() && + branch_node_tree_masks + .get(&path) + .is_some_and(|mask| mask.is_bit_set(last_child_nibble)) + }; + if should_set_tree_mask_bit { + tree_mask.set_bit(last_child_nibble); + } + + // Set the hash mask. If a child node is a revealed branch node OR + // is a blinded node that has its hash mask bit set according to the + // database, set the hash mask bit and save the hash. + let hash = child.as_hash().filter(|_| { + child_node_type.is_branch() || + (child_node_type.is_hash() && + branch_node_hash_masks.get(&path).is_some_and( + |mask| mask.is_bit_set(last_child_nibble), + )) + }); + if let Some(hash) = hash { + hash_mask.set_bit(last_child_nibble); + hashes.push(hash); + } + } + + // Insert children in the resulting buffer in a normal order, + // because initially we iterated in reverse. + // SAFETY: i < len and len is never 0 + let original_idx = self.buffers.branch_child_buf.len() - i - 1; + self.buffers.branch_value_stack_buf[original_idx] = child; + added_children = true; + } else { + // Need to defer processing until children are computed, on the next + // invocation update the node's hash. + debug_assert!(!added_children); + self.buffers.path_stack.push(RlpNodePathStackItem { + path, + is_in_prefix_set: Some(prefix_set_contains(&path)), + }); + self.buffers.path_stack.extend( + self.buffers + .branch_child_buf + .drain(..) + .map(|path| RlpNodePathStackItem { path, is_in_prefix_set: None }), + ); + return + } + } + + trace!( + target: "trie::parallel_sparse", + ?path, + ?tree_mask, + ?hash_mask, + "Branch node masks" + ); + + // Top of the stack has all children node, we can encode the branch node and + // update its hash + self.buffers.rlp_buf.clear(); + let branch_node_ref = + BranchNodeRef::new(&self.buffers.branch_value_stack_buf, *state_mask); + let rlp_node = branch_node_ref.rlp(&mut self.buffers.rlp_buf); + *hash = rlp_node.as_hash(); + + // Save a branch node update only if it's not a root node, and we need to + // persist updates. + let store_in_db_trie_value = if let Some(update_actions) = + update_actions.as_mut().filter(|_| retain_updates && !path.is_empty()) + { + let store_in_db_trie = !tree_mask.is_empty() || !hash_mask.is_empty(); + if store_in_db_trie { + // Store in DB trie if there are either any children that are stored in + // the DB trie, or any children represent hashed values + hashes.reverse(); + let branch_node = BranchNodeCompact::new( + *state_mask, + tree_mask, + hash_mask, + hashes, + hash.filter(|_| path.is_empty()), + ); + update_actions + .push(SparseTrieUpdatesAction::InsertUpdated(path, branch_node)); + } else if branch_node_tree_masks.get(&path).is_some_and(|mask| !mask.is_empty()) || + branch_node_hash_masks.get(&path).is_some_and(|mask| !mask.is_empty()) + { + // If new tree and hash masks are empty, but previously they weren't, we + // need to remove the node update and add the node itself to the list of + // removed nodes. + update_actions.push(SparseTrieUpdatesAction::InsertRemoved(path)); + } else if branch_node_tree_masks.get(&path).is_none_or(|mask| mask.is_empty()) && + branch_node_hash_masks.get(&path).is_none_or(|mask| mask.is_empty()) + { + // If new tree and hash masks are empty, and they were previously empty + // as well, we need to remove the node update. + update_actions.push(SparseTrieUpdatesAction::RemoveUpdated(path)); + } + + store_in_db_trie + } else { + false + }; + *store_in_db_trie = Some(store_in_db_trie_value); + + ( + rlp_node, + SparseNodeType::Branch { store_in_db_trie: Some(store_in_db_trie_value) }, + ) + } + }; + + self.buffers.rlp_node_stack.push(RlpNodeStackItem { path, rlp_node, node_type }); + trace!( + target: "trie::parallel_sparse", + ?path, + ?node_type, + "Added node to RLP node stack" + ); + } + + /// Clears the subtrie, keeping the data structures allocated. + fn clear(&mut self) { + self.values.clear(); + self.buffers.clear(); + } +} + +/// Represents the outcome of processing a node during leaf insertion +#[derive(Clone, Debug, PartialEq, Eq, Default)] +pub enum LeafUpdateStep { + /// Continue traversing to the next node + Continue { + /// The next node path to process + next_node: Nibbles, + }, + /// Update is complete with nodes inserted + Complete { + /// The node paths that were inserted during this step + inserted_nodes: Vec, + /// Path to a node which may need to be revealed + reveal_path: Option, + }, + /// The node was not found + #[default] + NodeNotFound, +} + +impl LeafUpdateStep { + /// Creates a step to continue with the next node + pub const fn continue_with(next_node: Nibbles) -> Self { + Self::Continue { next_node } + } + + /// Creates a step indicating completion with inserted nodes + pub const fn complete_with_insertions( + inserted_nodes: Vec, + reveal_path: Option, + ) -> Self { + Self::Complete { inserted_nodes, reveal_path } + } +} + +/// Sparse Subtrie Type. +/// +/// Used to determine the type of subtrie a certain path belongs to: +/// - Paths in the range `0x..=0xf` belong to the upper subtrie. +/// - Paths in the range `0x00..` belong to one of the lower subtries. The index of the lower +/// subtrie is determined by the first [`UPPER_TRIE_MAX_DEPTH`] nibbles of the path. +/// +/// There can be at most [`NUM_LOWER_SUBTRIES`] lower subtries. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum SparseSubtrieType { + /// Upper subtrie with paths in the range `0x..=0xf` + Upper, + /// Lower subtrie with paths in the range `0x00..`. Includes the index of the subtrie, + /// according to the path prefix. + Lower(usize), +} + +impl SparseSubtrieType { + /// Returns true if a node at a path of the given length would be placed in the upper subtrie. + /// + /// Nodes with paths shorter than [`UPPER_TRIE_MAX_DEPTH`] nibbles belong to the upper subtrie, + /// while longer paths belong to the lower subtries. + pub const fn path_len_is_upper(len: usize) -> bool { + len < UPPER_TRIE_MAX_DEPTH + } + + /// Returns the type of subtrie based on the given path. + pub fn from_path(path: &Nibbles) -> Self { + if Self::path_len_is_upper(path.len()) { + Self::Upper + } else { + Self::Lower(path_subtrie_index_unchecked(path)) + } + } + + /// Returns the index of the lower subtrie, if it exists. + pub const fn lower_index(&self) -> Option { + match self { + Self::Upper => None, + Self::Lower(index) => Some(*index), + } + } +} + +impl Ord for SparseSubtrieType { + /// Orders two [`SparseSubtrieType`]s such that `Upper` is less than `Lower(_)`, and `Lower`s + /// are ordered by their index. + fn cmp(&self, other: &Self) -> Ordering { + match (self, other) { + (Self::Upper, Self::Upper) => Ordering::Equal, + (Self::Upper, Self::Lower(_)) => Ordering::Less, + (Self::Lower(_), Self::Upper) => Ordering::Greater, + (Self::Lower(idx_a), Self::Lower(idx_b)) if idx_a == idx_b => Ordering::Equal, + (Self::Lower(idx_a), Self::Lower(idx_b)) => idx_a.cmp(idx_b), + } + } +} + +impl PartialOrd for SparseSubtrieType { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +/// Collection of reusable buffers for calculating subtrie hashes. +/// +/// These buffers reduce allocations when computing RLP representations during trie updates. +#[derive(Clone, PartialEq, Eq, Debug, Default)] +pub struct SparseSubtrieBuffers { + /// Stack of RLP node paths + path_stack: Vec, + /// Stack of RLP nodes + rlp_node_stack: Vec, + /// Reusable branch child path + branch_child_buf: SmallVec<[Nibbles; 16]>, + /// Reusable branch value stack + branch_value_stack_buf: SmallVec<[RlpNode; 16]>, + /// Reusable RLP buffer + rlp_buf: Vec, +} + +impl SparseSubtrieBuffers { + /// Clears all buffers. + fn clear(&mut self) { + self.path_stack.clear(); + self.path_stack.shrink_to_fit(); + + self.rlp_node_stack.clear(); + self.rlp_node_stack.shrink_to_fit(); + + self.branch_child_buf.clear(); + self.branch_child_buf.shrink_to_fit(); + + self.branch_value_stack_buf.clear(); + self.branch_value_stack_buf.shrink_to_fit(); + + self.rlp_buf.clear(); + self.rlp_buf.shrink_to_fit(); + } +} + +/// RLP node path stack item. +#[derive(Clone, PartialEq, Eq, Debug)] +pub struct RlpNodePathStackItem { + /// Path to the node. + pub path: Nibbles, + /// Whether the path is in the prefix set. If [`None`], then unknown yet. + pub is_in_prefix_set: Option, +} + +/// Changed subtrie. +#[derive(Debug)] +struct ChangedSubtrie { + /// Lower subtrie index in the range [0, [`NUM_LOWER_SUBTRIES`]). + index: usize, + /// Changed subtrie + subtrie: Box, + /// Prefix set of keys that belong to the subtrie. + prefix_set: PrefixSet, + /// Reusable buffer for collecting [`SparseTrieUpdatesAction`]s during computations. Will be + /// None if update retention is disabled. + update_actions_buf: Option>, +} + +/// Convert first [`UPPER_TRIE_MAX_DEPTH`] nibbles of the path into a lower subtrie index in the +/// range [0, [`NUM_LOWER_SUBTRIES`]). +/// +/// # Panics +/// +/// If the path is shorter than [`UPPER_TRIE_MAX_DEPTH`] nibbles. +fn path_subtrie_index_unchecked(path: &Nibbles) -> usize { + debug_assert_eq!(UPPER_TRIE_MAX_DEPTH, 2); + path.get_byte_unchecked(0) as usize +} + +/// Used by lower subtries to communicate updates to the top-level [`SparseTrieUpdates`] set. +#[derive(Clone, Debug, Eq, PartialEq)] +enum SparseTrieUpdatesAction { + /// Remove the path from the `updated_nodes`, if it was present, and add it to `removed_nodes`. + InsertRemoved(Nibbles), + /// Remove the path from the `updated_nodes`, if it was present, leaving `removed_nodes` + /// unaffected. + RemoveUpdated(Nibbles), + /// Insert the branch node into `updated_nodes`. + InsertUpdated(Nibbles, BranchNodeCompact), +} + +#[cfg(test)] +mod tests { + use super::{ + path_subtrie_index_unchecked, LowerSparseSubtrie, ParallelSparseTrie, SparseSubtrie, + SparseSubtrieType, + }; + use crate::trie::ChangedSubtrie; + use alloy_primitives::{ + b256, hex, + map::{B256Set, DefaultHashBuilder, HashMap}, + B256, U256, + }; + use alloy_rlp::{Decodable, Encodable}; + use alloy_trie::{BranchNodeCompact, Nibbles}; + use assert_matches::assert_matches; + use itertools::Itertools; + use proptest::{prelude::*, sample::SizeRange}; + use proptest_arbitrary_interop::arb; + use reth_execution_errors::{SparseTrieError, SparseTrieErrorKind}; + use reth_primitives_traits::Account; + use reth_provider::{test_utils::create_test_provider_factory, TrieWriter}; + use reth_trie::{ + hashed_cursor::{noop::NoopHashedAccountCursor, HashedPostStateAccountCursor}, + node_iter::{TrieElement, TrieNodeIter}, + trie_cursor::{noop::NoopAccountTrieCursor, TrieCursor, TrieCursorFactory}, + walker::TrieWalker, + HashedPostState, + }; + use reth_trie_common::{ + prefix_set::PrefixSetMut, + proof::{ProofNodes, ProofRetainer}, + updates::TrieUpdates, + BranchNode, ExtensionNode, HashBuilder, LeafNode, RlpNode, TrieMask, TrieNode, + EMPTY_ROOT_HASH, + }; + use reth_trie_db::DatabaseTrieCursorFactory; + use reth_trie_sparse::{ + provider::{DefaultTrieNodeProvider, RevealedNode, TrieNodeProvider}, + LeafLookup, LeafLookupError, RevealedSparseNode, SerialSparseTrie, SparseNode, + SparseTrieInterface, SparseTrieUpdates, TrieMasks, + }; + use std::collections::{BTreeMap, BTreeSet}; + + /// Pad nibbles to the length of a B256 hash with zeros on the right. + fn pad_nibbles_right(mut nibbles: Nibbles) -> Nibbles { + nibbles.extend(&Nibbles::from_nibbles_unchecked(vec![ + 0; + B256::len_bytes() * 2 - nibbles.len() + ])); + nibbles + } + + /// Mock trie node provider for testing that allows pre-setting nodes at specific paths. + /// + /// This provider can be used in tests to simulate trie nodes that need to be revealed + /// during trie operations, particularly when collapsing branch nodes during leaf removal. + #[derive(Debug, Clone)] + struct MockTrieNodeProvider { + /// Mapping from path to revealed node data + nodes: HashMap, + } + + impl MockTrieNodeProvider { + /// Creates a new empty mock provider + fn new() -> Self { + Self { nodes: HashMap::default() } + } + + /// Adds a revealed node at the specified path + fn add_revealed_node(&mut self, path: Nibbles, node: RevealedNode) { + self.nodes.insert(path, node); + } + } + + impl TrieNodeProvider for MockTrieNodeProvider { + fn trie_node(&self, path: &Nibbles) -> Result, SparseTrieError> { + Ok(self.nodes.get(path).cloned()) + } + } + + fn create_account(nonce: u64) -> Account { + Account { nonce, ..Default::default() } + } + + fn encode_account_value(nonce: u64) -> Vec { + let account = Account { nonce, ..Default::default() }; + let trie_account = account.into_trie_account(EMPTY_ROOT_HASH); + let mut buf = Vec::new(); + trie_account.encode(&mut buf); + buf + } + + /// Test context that provides helper methods for trie testing + #[derive(Default)] + struct ParallelSparseTrieTestContext; + + impl ParallelSparseTrieTestContext { + /// Assert that a lower subtrie exists at the given path + fn assert_subtrie_exists(&self, trie: &ParallelSparseTrie, path: &Nibbles) { + let idx = path_subtrie_index_unchecked(path); + assert!( + trie.lower_subtries[idx].as_revealed_ref().is_some(), + "Expected lower subtrie at path {path:?} to exist", + ); + } + + /// Get a lower subtrie, panicking if it doesn't exist + fn get_subtrie<'a>( + &self, + trie: &'a ParallelSparseTrie, + path: &Nibbles, + ) -> &'a SparseSubtrie { + let idx = path_subtrie_index_unchecked(path); + trie.lower_subtries[idx] + .as_revealed_ref() + .unwrap_or_else(|| panic!("Lower subtrie at path {path:?} should exist")) + } + + /// Assert that a lower subtrie has a specific path field value + fn assert_subtrie_path( + &self, + trie: &ParallelSparseTrie, + subtrie_prefix: impl AsRef<[u8]>, + expected_path: impl AsRef<[u8]>, + ) { + let subtrie_prefix = Nibbles::from_nibbles(subtrie_prefix); + let expected_path = Nibbles::from_nibbles(expected_path); + let idx = path_subtrie_index_unchecked(&subtrie_prefix); + + let subtrie = trie.lower_subtries[idx].as_revealed_ref().unwrap_or_else(|| { + panic!("Lower subtrie at prefix {subtrie_prefix:?} should exist") + }); + + assert_eq!( + subtrie.path, expected_path, + "Subtrie at prefix {subtrie_prefix:?} should have path {expected_path:?}, but has {:?}", + subtrie.path + ); + } + + /// Create test leaves with consecutive account values + fn create_test_leaves(&self, paths: &[&[u8]]) -> Vec<(Nibbles, Vec)> { + paths + .iter() + .enumerate() + .map(|(i, path)| (Nibbles::from_nibbles(path), encode_account_value(i as u64 + 1))) + .collect() + } + + /// Create a single test leaf with the given path and value nonce + fn create_test_leaf(&self, path: impl AsRef<[u8]>, value_nonce: u64) -> (Nibbles, Vec) { + (Nibbles::from_nibbles(path), encode_account_value(value_nonce)) + } + + /// Update multiple leaves in the trie + fn update_leaves( + &self, + trie: &mut ParallelSparseTrie, + leaves: impl IntoIterator)>, + ) { + for (path, value) in leaves { + trie.update_leaf(path, value, DefaultTrieNodeProvider).unwrap(); + } + } + + /// Create an assertion builder for a subtrie + fn assert_subtrie<'a>( + &self, + trie: &'a ParallelSparseTrie, + path: Nibbles, + ) -> SubtrieAssertion<'a> { + self.assert_subtrie_exists(trie, &path); + let subtrie = self.get_subtrie(trie, &path); + SubtrieAssertion::new(subtrie) + } + + /// Create an assertion builder for the upper subtrie + fn assert_upper_subtrie<'a>(&self, trie: &'a ParallelSparseTrie) -> SubtrieAssertion<'a> { + SubtrieAssertion::new(&trie.upper_subtrie) + } + + /// Assert the root, trie updates, and nodes against the hash builder output. + fn assert_with_hash_builder( + &self, + trie: &mut ParallelSparseTrie, + hash_builder_root: B256, + hash_builder_updates: TrieUpdates, + hash_builder_proof_nodes: ProofNodes, + ) { + assert_eq!(trie.root(), hash_builder_root); + pretty_assertions::assert_eq!( + BTreeMap::from_iter(trie.updates_ref().updated_nodes.clone()), + BTreeMap::from_iter(hash_builder_updates.account_nodes) + ); + assert_eq_parallel_sparse_trie_proof_nodes(trie, hash_builder_proof_nodes); + } + } + + /// Assertion builder for subtrie structure + struct SubtrieAssertion<'a> { + subtrie: &'a SparseSubtrie, + } + + impl<'a> SubtrieAssertion<'a> { + fn new(subtrie: &'a SparseSubtrie) -> Self { + Self { subtrie } + } + + fn has_branch(self, path: &Nibbles, expected_mask_bits: &[u8]) -> Self { + match self.subtrie.nodes.get(path) { + Some(SparseNode::Branch { state_mask, .. }) => { + for bit in expected_mask_bits { + assert!( + state_mask.is_bit_set(*bit), + "Expected branch at {path:?} to have bit {bit} set, instead mask is: {state_mask:?}", + ); + } + } + node => panic!("Expected branch node at {path:?}, found {node:?}"), + } + self + } + + fn has_leaf(self, path: &Nibbles, expected_key: &Nibbles) -> Self { + match self.subtrie.nodes.get(path) { + Some(SparseNode::Leaf { key, .. }) => { + assert_eq!( + *key, *expected_key, + "Expected leaf at {path:?} to have key {expected_key:?}, found {key:?}", + ); + } + node => panic!("Expected leaf node at {path:?}, found {node:?}"), + } + self + } + + fn has_extension(self, path: &Nibbles, expected_key: &Nibbles) -> Self { + match self.subtrie.nodes.get(path) { + Some(SparseNode::Extension { key, .. }) => { + assert_eq!( + *key, *expected_key, + "Expected extension at {path:?} to have key {expected_key:?}, found {key:?}", + ); + } + node => panic!("Expected extension node at {path:?}, found {node:?}"), + } + self + } + + fn has_hash(self, path: &Nibbles, expected_hash: &B256) -> Self { + match self.subtrie.nodes.get(path) { + Some(SparseNode::Hash(hash)) => { + assert_eq!( + *hash, *expected_hash, + "Expected hash at {path:?} to be {expected_hash:?}, found {hash:?}", + ); + } + node => panic!("Expected hash node at {path:?}, found {node:?}"), + } + self + } + + fn has_value(self, path: &Nibbles, expected_value: &[u8]) -> Self { + let actual = self.subtrie.inner.values.get(path); + assert_eq!( + actual.map(|v| v.as_slice()), + Some(expected_value), + "Expected value at {path:?} to be {expected_value:?}, found {actual:?}", + ); + self + } + + fn has_no_value(self, path: &Nibbles) -> Self { + let actual = self.subtrie.inner.values.get(path); + assert!(actual.is_none(), "Expected no value at {path:?}, but found {actual:?}"); + self + } + } + + fn create_leaf_node(key: impl AsRef<[u8]>, value_nonce: u64) -> TrieNode { + TrieNode::Leaf(LeafNode::new(Nibbles::from_nibbles(key), encode_account_value(value_nonce))) + } + + fn create_extension_node(key: impl AsRef<[u8]>, child_hash: B256) -> TrieNode { + TrieNode::Extension(ExtensionNode::new( + Nibbles::from_nibbles(key), + RlpNode::word_rlp(&child_hash), + )) + } + + fn create_branch_node_with_children( + children_indices: &[u8], + child_hashes: impl IntoIterator, + ) -> TrieNode { + let mut stack = Vec::new(); + let mut state_mask = TrieMask::default(); + + for (&idx, hash) in children_indices.iter().zip(child_hashes.into_iter()) { + state_mask.set_bit(idx); + stack.push(hash); + } + + TrieNode::Branch(BranchNode::new(stack, state_mask)) + } + + /// Calculate the state root by feeding the provided state to the hash builder and retaining the + /// proofs for the provided targets. + /// + /// Returns the state root and the retained proof nodes. + fn run_hash_builder( + state: impl IntoIterator + Clone, + trie_cursor: impl TrieCursor, + destroyed_accounts: B256Set, + proof_targets: impl IntoIterator, + ) -> (B256, TrieUpdates, ProofNodes, HashMap, HashMap) + { + let mut account_rlp = Vec::new(); + + let mut hash_builder = HashBuilder::default() + .with_updates(true) + .with_proof_retainer(ProofRetainer::from_iter(proof_targets)); + + let mut prefix_set = PrefixSetMut::default(); + prefix_set.extend_keys(state.clone().into_iter().map(|(nibbles, _)| nibbles)); + prefix_set.extend_keys(destroyed_accounts.iter().map(Nibbles::unpack)); + let walker = TrieWalker::<_>::state_trie(trie_cursor, prefix_set.freeze()) + .with_deletions_retained(true); + let hashed_post_state = HashedPostState::default() + .with_accounts(state.into_iter().map(|(nibbles, account)| { + (nibbles.pack().into_inner().unwrap().into(), Some(account)) + })) + .into_sorted(); + let mut node_iter = TrieNodeIter::state_trie( + walker, + HashedPostStateAccountCursor::new( + NoopHashedAccountCursor::default(), + hashed_post_state.accounts(), + ), + ); + + while let Some(node) = node_iter.try_next().unwrap() { + match node { + TrieElement::Branch(branch) => { + hash_builder.add_branch(branch.key, branch.value, branch.children_are_in_trie); + } + TrieElement::Leaf(key, account) => { + let account = account.into_trie_account(EMPTY_ROOT_HASH); + account.encode(&mut account_rlp); + + hash_builder.add_leaf(Nibbles::unpack(key), &account_rlp); + account_rlp.clear(); + } + } + } + let root = hash_builder.root(); + let proof_nodes = hash_builder.take_proof_nodes(); + let branch_node_hash_masks = hash_builder + .updated_branch_nodes + .clone() + .unwrap_or_default() + .iter() + .map(|(path, node)| (*path, node.hash_mask)) + .collect(); + let branch_node_tree_masks = hash_builder + .updated_branch_nodes + .clone() + .unwrap_or_default() + .iter() + .map(|(path, node)| (*path, node.tree_mask)) + .collect(); + + let mut trie_updates = TrieUpdates::default(); + let removed_keys = node_iter.walker.take_removed_keys(); + trie_updates.finalize(hash_builder, removed_keys, destroyed_accounts); + + (root, trie_updates, proof_nodes, branch_node_hash_masks, branch_node_tree_masks) + } + + /// Returns a `ParallelSparseTrie` pre-loaded with the given nodes, as well as leaf values + /// inferred from any provided leaf nodes. + fn new_test_trie(nodes: Nodes) -> ParallelSparseTrie + where + Nodes: Iterator, + { + let mut trie = ParallelSparseTrie::default().with_updates(true); + + for (path, node) in nodes { + let subtrie = trie.subtrie_for_path_mut(&path); + if let SparseNode::Leaf { key, .. } = &node { + let mut full_key = path; + full_key.extend(key); + subtrie.inner.values.insert(full_key, "LEAF VALUE".into()); + } + subtrie.nodes.insert(path, node); + } + trie + } + + fn parallel_sparse_trie_nodes( + sparse_trie: &ParallelSparseTrie, + ) -> impl IntoIterator { + let lower_sparse_nodes = sparse_trie + .lower_subtries + .iter() + .filter_map(|subtrie| subtrie.as_revealed_ref()) + .flat_map(|subtrie| subtrie.nodes.iter()); + + let upper_sparse_nodes = sparse_trie.upper_subtrie.nodes.iter(); + + lower_sparse_nodes.chain(upper_sparse_nodes).sorted_by_key(|(path, _)| *path) + } + + /// Assert that the parallel sparse trie nodes and the proof nodes from the hash builder are + /// equal. + fn assert_eq_parallel_sparse_trie_proof_nodes( + sparse_trie: &ParallelSparseTrie, + proof_nodes: ProofNodes, + ) { + let proof_nodes = proof_nodes + .into_nodes_sorted() + .into_iter() + .map(|(path, node)| (path, TrieNode::decode(&mut node.as_ref()).unwrap())); + + let all_sparse_nodes = parallel_sparse_trie_nodes(sparse_trie); + + for ((proof_node_path, proof_node), (sparse_node_path, sparse_node)) in + proof_nodes.zip(all_sparse_nodes) + { + assert_eq!(&proof_node_path, sparse_node_path); + + let equals = match (&proof_node, &sparse_node) { + // Both nodes are empty + (TrieNode::EmptyRoot, SparseNode::Empty) => true, + // Both nodes are branches and have the same state mask + ( + TrieNode::Branch(BranchNode { state_mask: proof_state_mask, .. }), + SparseNode::Branch { state_mask: sparse_state_mask, .. }, + ) => proof_state_mask == sparse_state_mask, + // Both nodes are extensions and have the same key + ( + TrieNode::Extension(ExtensionNode { key: proof_key, .. }), + SparseNode::Extension { key: sparse_key, .. }, + ) | + // Both nodes are leaves and have the same key + ( + TrieNode::Leaf(LeafNode { key: proof_key, .. }), + SparseNode::Leaf { key: sparse_key, .. }, + ) => proof_key == sparse_key, + // Empty and hash nodes are specific to the sparse trie, skip them + (_, SparseNode::Empty | SparseNode::Hash(_)) => continue, + _ => false, + }; + assert!( + equals, + "path: {proof_node_path:?}\nproof node: {proof_node:?}\nsparse node: {sparse_node:?}" + ); + } + } + + #[test] + fn test_get_changed_subtries_empty() { + let mut trie = ParallelSparseTrie::default(); + let mut prefix_set = PrefixSetMut::from([Nibbles::default()]).freeze(); + + let (subtries, unchanged_prefix_set) = trie.take_changed_lower_subtries(&mut prefix_set); + assert!(subtries.is_empty()); + assert_eq!(unchanged_prefix_set, PrefixSetMut::from(prefix_set.iter().copied())); + } + + #[test] + fn test_get_changed_subtries() { + // Create a trie with three subtries + let mut trie = ParallelSparseTrie::default(); + let subtrie_1 = Box::new(SparseSubtrie::new(Nibbles::from_nibbles([0x0, 0x0]))); + let subtrie_1_index = path_subtrie_index_unchecked(&subtrie_1.path); + let subtrie_2 = Box::new(SparseSubtrie::new(Nibbles::from_nibbles([0x1, 0x0]))); + let subtrie_2_index = path_subtrie_index_unchecked(&subtrie_2.path); + let subtrie_3 = Box::new(SparseSubtrie::new(Nibbles::from_nibbles([0x3, 0x0]))); + let subtrie_3_index = path_subtrie_index_unchecked(&subtrie_3.path); + + // Add subtries at specific positions + trie.lower_subtries[subtrie_1_index] = LowerSparseSubtrie::Revealed(subtrie_1.clone()); + trie.lower_subtries[subtrie_2_index] = LowerSparseSubtrie::Revealed(subtrie_2.clone()); + trie.lower_subtries[subtrie_3_index] = LowerSparseSubtrie::Revealed(subtrie_3); + + let unchanged_prefix_set = PrefixSetMut::from([ + Nibbles::from_nibbles([0x0]), + Nibbles::from_nibbles([0x2, 0x0, 0x0]), + ]); + // Create a prefix set with the keys that match only the second subtrie + let mut prefix_set = PrefixSetMut::from([ + // Match second subtrie + Nibbles::from_nibbles([0x1, 0x0, 0x0]), + Nibbles::from_nibbles([0x1, 0x0, 0x1, 0x0]), + ]); + prefix_set.extend(unchanged_prefix_set); + let mut prefix_set = prefix_set.freeze(); + + // Second subtrie should be removed and returned + let (subtries, unchanged_prefix_set) = trie.take_changed_lower_subtries(&mut prefix_set); + assert_eq!( + subtries + .into_iter() + .map(|ChangedSubtrie { index, subtrie, prefix_set, .. }| { + (index, subtrie, prefix_set.iter().copied().collect::>()) + }) + .collect::>(), + vec![( + subtrie_2_index, + subtrie_2, + vec![ + Nibbles::from_nibbles([0x1, 0x0, 0x0]), + Nibbles::from_nibbles([0x1, 0x0, 0x1, 0x0]) + ] + )] + ); + assert_eq!(unchanged_prefix_set, unchanged_prefix_set); + assert!(trie.lower_subtries[subtrie_2_index].as_revealed_ref().is_none()); + + // First subtrie should remain unchanged + assert_eq!(trie.lower_subtries[subtrie_1_index], LowerSparseSubtrie::Revealed(subtrie_1)); + } + + #[test] + fn test_get_changed_subtries_all() { + // Create a trie with three subtries + let mut trie = ParallelSparseTrie::default(); + let subtrie_1 = Box::new(SparseSubtrie::new(Nibbles::from_nibbles([0x0, 0x0]))); + let subtrie_1_index = path_subtrie_index_unchecked(&subtrie_1.path); + let subtrie_2 = Box::new(SparseSubtrie::new(Nibbles::from_nibbles([0x1, 0x0]))); + let subtrie_2_index = path_subtrie_index_unchecked(&subtrie_2.path); + let subtrie_3 = Box::new(SparseSubtrie::new(Nibbles::from_nibbles([0x3, 0x0]))); + let subtrie_3_index = path_subtrie_index_unchecked(&subtrie_3.path); + + // Add subtries at specific positions + trie.lower_subtries[subtrie_1_index] = LowerSparseSubtrie::Revealed(subtrie_1.clone()); + trie.lower_subtries[subtrie_2_index] = LowerSparseSubtrie::Revealed(subtrie_2.clone()); + trie.lower_subtries[subtrie_3_index] = LowerSparseSubtrie::Revealed(subtrie_3.clone()); + + // Create a prefix set that matches any key + let mut prefix_set = PrefixSetMut::all().freeze(); + + // All subtries should be removed and returned + let (subtries, unchanged_prefix_set) = trie.take_changed_lower_subtries(&mut prefix_set); + assert_eq!( + subtries + .into_iter() + .map(|ChangedSubtrie { index, subtrie, prefix_set, .. }| { + (index, subtrie, prefix_set.all()) + }) + .collect::>(), + vec![ + (subtrie_1_index, subtrie_1, true), + (subtrie_2_index, subtrie_2, true), + (subtrie_3_index, subtrie_3, true) + ] + ); + assert_eq!(unchanged_prefix_set, PrefixSetMut::all()); + + assert!(trie.lower_subtries.iter().all(|subtrie| subtrie.as_revealed_ref().is_none())); + } + + #[test] + fn test_sparse_subtrie_type() { + assert_eq!(SparseSubtrieType::from_path(&Nibbles::new()), SparseSubtrieType::Upper); + assert_eq!( + SparseSubtrieType::from_path(&Nibbles::from_nibbles([0])), + SparseSubtrieType::Upper + ); + assert_eq!( + SparseSubtrieType::from_path(&Nibbles::from_nibbles([15])), + SparseSubtrieType::Upper + ); + assert_eq!( + SparseSubtrieType::from_path(&Nibbles::from_nibbles([0, 0])), + SparseSubtrieType::Lower(0) + ); + assert_eq!( + SparseSubtrieType::from_path(&Nibbles::from_nibbles([0, 0, 0])), + SparseSubtrieType::Lower(0) + ); + assert_eq!( + SparseSubtrieType::from_path(&Nibbles::from_nibbles([0, 1])), + SparseSubtrieType::Lower(1) + ); + assert_eq!( + SparseSubtrieType::from_path(&Nibbles::from_nibbles([0, 1, 0])), + SparseSubtrieType::Lower(1) + ); + assert_eq!( + SparseSubtrieType::from_path(&Nibbles::from_nibbles([0, 15])), + SparseSubtrieType::Lower(15) + ); + assert_eq!( + SparseSubtrieType::from_path(&Nibbles::from_nibbles([15, 0])), + SparseSubtrieType::Lower(240) + ); + assert_eq!( + SparseSubtrieType::from_path(&Nibbles::from_nibbles([15, 1])), + SparseSubtrieType::Lower(241) + ); + assert_eq!( + SparseSubtrieType::from_path(&Nibbles::from_nibbles([15, 15])), + SparseSubtrieType::Lower(255) + ); + assert_eq!( + SparseSubtrieType::from_path(&Nibbles::from_nibbles([15, 15, 15])), + SparseSubtrieType::Lower(255) + ); + } + + #[test] + fn test_reveal_node_leaves() { + let mut trie = ParallelSparseTrie::default(); + + // Reveal leaf in the upper trie + { + let path = Nibbles::from_nibbles([0x1]); + let node = create_leaf_node([0x2, 0x3], 42); + let masks = TrieMasks::none(); + + trie.reveal_nodes(vec![RevealedSparseNode { path, node, masks }]).unwrap(); + + assert_matches!( + trie.upper_subtrie.nodes.get(&path), + Some(SparseNode::Leaf { key, hash: None }) + if key == &Nibbles::from_nibbles([0x2, 0x3]) + ); + + let full_path = Nibbles::from_nibbles([0x1, 0x2, 0x3]); + assert_eq!( + trie.upper_subtrie.inner.values.get(&full_path), + Some(&encode_account_value(42)) + ); + } + + // Reveal leaf in a lower trie + { + let path = Nibbles::from_nibbles([0x1, 0x2]); + let node = create_leaf_node([0x3, 0x4], 42); + let masks = TrieMasks::none(); + + trie.reveal_nodes(vec![RevealedSparseNode { path, node, masks }]).unwrap(); + + // Check that the lower subtrie was created + let idx = path_subtrie_index_unchecked(&path); + assert!(trie.lower_subtries[idx].as_revealed_ref().is_some()); + + // Check that the lower subtrie's path was correctly set + let lower_subtrie = trie.lower_subtries[idx].as_revealed_ref().unwrap(); + assert_eq!(lower_subtrie.path, path); + + assert_matches!( + lower_subtrie.nodes.get(&path), + Some(SparseNode::Leaf { key, hash: None }) + if key == &Nibbles::from_nibbles([0x3, 0x4]) + ); + } + + // Reveal leaf in a lower trie with a longer path, shouldn't result in the subtrie's root + // path changing. + { + let path = Nibbles::from_nibbles([0x1, 0x2, 0x3]); + let node = create_leaf_node([0x4, 0x5], 42); + let masks = TrieMasks::none(); + + trie.reveal_nodes(vec![RevealedSparseNode { path, node, masks }]).unwrap(); + + // Check that the lower subtrie's path hasn't changed + let idx = path_subtrie_index_unchecked(&path); + let lower_subtrie = trie.lower_subtries[idx].as_revealed_ref().unwrap(); + assert_eq!(lower_subtrie.path, Nibbles::from_nibbles([0x1, 0x2])); + } + } + + #[test] + fn test_reveal_node_extension_all_upper() { + let path = Nibbles::new(); + let child_hash = B256::repeat_byte(0xab); + let node = create_extension_node([0x1], child_hash); + let masks = TrieMasks::none(); + let trie = ParallelSparseTrie::from_root(node, masks, true).unwrap(); + + assert_matches!( + trie.upper_subtrie.nodes.get(&path), + Some(SparseNode::Extension { key, hash: None, .. }) + if key == &Nibbles::from_nibbles([0x1]) + ); + + // Child path should be in upper trie + let child_path = Nibbles::from_nibbles([0x1]); + assert_eq!(trie.upper_subtrie.nodes.get(&child_path), Some(&SparseNode::Hash(child_hash))); + } + + #[test] + fn test_reveal_node_extension_cross_level() { + let path = Nibbles::new(); + let child_hash = B256::repeat_byte(0xcd); + let node = create_extension_node([0x1, 0x2, 0x3], child_hash); + let masks = TrieMasks::none(); + let trie = ParallelSparseTrie::from_root(node, masks, true).unwrap(); + + // Extension node should be in upper trie + assert_matches!( + trie.upper_subtrie.nodes.get(&path), + Some(SparseNode::Extension { key, hash: None, .. }) + if key == &Nibbles::from_nibbles([0x1, 0x2, 0x3]) + ); + + // Child path (0x1, 0x2, 0x3) should be in lower trie + let child_path = Nibbles::from_nibbles([0x1, 0x2, 0x3]); + let idx = path_subtrie_index_unchecked(&child_path); + assert!(trie.lower_subtries[idx].as_revealed_ref().is_some()); + + let lower_subtrie = trie.lower_subtries[idx].as_revealed_ref().unwrap(); + assert_eq!(lower_subtrie.path, child_path); + assert_eq!(lower_subtrie.nodes.get(&child_path), Some(&SparseNode::Hash(child_hash))); + } + + #[test] + fn test_reveal_node_extension_cross_level_boundary() { + let mut trie = ParallelSparseTrie::default(); + let path = Nibbles::from_nibbles([0x1]); + let child_hash = B256::repeat_byte(0xcd); + let node = create_extension_node([0x2], child_hash); + let masks = TrieMasks::none(); + + trie.reveal_nodes(vec![RevealedSparseNode { path, node, masks }]).unwrap(); + + // Extension node should be in upper trie + assert_matches!( + trie.upper_subtrie.nodes.get(&path), + Some(SparseNode::Extension { key, hash: None, .. }) + if key == &Nibbles::from_nibbles([0x2]) + ); + + // Child path (0x1, 0x2) should be in lower trie + let child_path = Nibbles::from_nibbles([0x1, 0x2]); + let idx = path_subtrie_index_unchecked(&child_path); + assert!(trie.lower_subtries[idx].as_revealed_ref().is_some()); + + let lower_subtrie = trie.lower_subtries[idx].as_revealed_ref().unwrap(); + assert_eq!(lower_subtrie.path, child_path); + assert_eq!(lower_subtrie.nodes.get(&child_path), Some(&SparseNode::Hash(child_hash))); + } + + #[test] + fn test_reveal_node_branch_all_upper() { + let path = Nibbles::new(); + let child_hashes = [ + RlpNode::word_rlp(&B256::repeat_byte(0x11)), + RlpNode::word_rlp(&B256::repeat_byte(0x22)), + ]; + let node = create_branch_node_with_children(&[0x0, 0x5], child_hashes.clone()); + let masks = TrieMasks::none(); + let trie = ParallelSparseTrie::from_root(node, masks, true).unwrap(); + + // Branch node should be in upper trie + assert_matches!( + trie.upper_subtrie.nodes.get(&path), + Some(SparseNode::Branch { state_mask, hash: None, .. }) + if *state_mask == 0b0000000000100001.into() + ); + + // Children should be in upper trie (paths of length 2) + let child_path_0 = Nibbles::from_nibbles([0x0]); + let child_path_5 = Nibbles::from_nibbles([0x5]); + assert_eq!( + trie.upper_subtrie.nodes.get(&child_path_0), + Some(&SparseNode::Hash(child_hashes[0].as_hash().unwrap())) + ); + assert_eq!( + trie.upper_subtrie.nodes.get(&child_path_5), + Some(&SparseNode::Hash(child_hashes[1].as_hash().unwrap())) + ); + } + + #[test] + fn test_reveal_node_branch_cross_level() { + let mut trie = ParallelSparseTrie::default(); + let path = Nibbles::from_nibbles([0x1]); // Exactly 1 nibbles - boundary case + let child_hashes = [ + RlpNode::word_rlp(&B256::repeat_byte(0x33)), + RlpNode::word_rlp(&B256::repeat_byte(0x44)), + RlpNode::word_rlp(&B256::repeat_byte(0x55)), + ]; + let node = create_branch_node_with_children(&[0x0, 0x7, 0xf], child_hashes.clone()); + let masks = TrieMasks::none(); + + trie.reveal_nodes(vec![RevealedSparseNode { path, node, masks }]).unwrap(); + + // Branch node should be in upper trie + assert_matches!( + trie.upper_subtrie.nodes.get(&path), + Some(SparseNode::Branch { state_mask, hash: None, .. }) + if *state_mask == 0b1000000010000001.into() + ); + + // All children should be in lower tries since they have paths of length 3 + let child_paths = [ + Nibbles::from_nibbles([0x1, 0x0]), + Nibbles::from_nibbles([0x1, 0x7]), + Nibbles::from_nibbles([0x1, 0xf]), + ]; + + for (i, child_path) in child_paths.iter().enumerate() { + let idx = path_subtrie_index_unchecked(child_path); + let lower_subtrie = trie.lower_subtries[idx].as_revealed_ref().unwrap(); + assert_eq!(&lower_subtrie.path, child_path); + assert_eq!( + lower_subtrie.nodes.get(child_path), + Some(&SparseNode::Hash(child_hashes[i].as_hash().unwrap())), + ); + } + } + + #[test] + fn test_update_subtrie_hashes_prefix_set_matching() { + // Create a trie and reveal leaf nodes using reveal_nodes + let mut trie = ParallelSparseTrie::default(); + + // Create dummy leaf nodes. + let leaf_1_full_path = Nibbles::from_nibbles([0; 64]); + let leaf_1_path = leaf_1_full_path.slice(..2); + let leaf_1_key = leaf_1_full_path.slice(2..); + let leaf_2_full_path = Nibbles::from_nibbles([vec![0, 1], vec![0; 62]].concat()); + let leaf_2_path = leaf_2_full_path.slice(..2); + let leaf_2_key = leaf_2_full_path.slice(2..); + let leaf_3_full_path = Nibbles::from_nibbles([vec![0, 2], vec![0; 62]].concat()); + let leaf_3_path = leaf_3_full_path.slice(..2); + let leaf_3_key = leaf_3_full_path.slice(2..); + let leaf_1 = create_leaf_node(leaf_1_key.to_vec(), 1); + let leaf_2 = create_leaf_node(leaf_2_key.to_vec(), 2); + let leaf_3 = create_leaf_node(leaf_3_key.to_vec(), 3); + + // Create branch node with hashes for each leaf. + let child_hashes = [ + RlpNode::word_rlp(&B256::repeat_byte(0x00)), + RlpNode::word_rlp(&B256::repeat_byte(0x11)), + // deliberately omit hash for leaf_3 + ]; + let branch_path = Nibbles::from_nibbles([0x0]); + let branch_node = create_branch_node_with_children(&[0x0, 0x1, 0x2], child_hashes); + + // Reveal nodes using reveal_nodes + trie.reveal_nodes(vec![ + RevealedSparseNode { path: branch_path, node: branch_node, masks: TrieMasks::none() }, + RevealedSparseNode { path: leaf_1_path, node: leaf_1, masks: TrieMasks::none() }, + RevealedSparseNode { path: leaf_2_path, node: leaf_2, masks: TrieMasks::none() }, + RevealedSparseNode { path: leaf_3_path, node: leaf_3, masks: TrieMasks::none() }, + ]) + .unwrap(); + + // Calculate subtrie indexes + let subtrie_1_index = SparseSubtrieType::from_path(&leaf_1_path).lower_index().unwrap(); + let subtrie_2_index = SparseSubtrieType::from_path(&leaf_2_path).lower_index().unwrap(); + let subtrie_3_index = SparseSubtrieType::from_path(&leaf_3_path).lower_index().unwrap(); + + let mut unchanged_prefix_set = PrefixSetMut::from([ + Nibbles::from_nibbles([0x0]), + leaf_2_full_path, + Nibbles::from_nibbles([0x3, 0x0, 0x0]), + ]); + // Create a prefix set with the keys that match only the second subtrie + let mut prefix_set = PrefixSetMut::from([ + // Match second subtrie + Nibbles::from_nibbles([0x0, 0x1, 0x0]), + Nibbles::from_nibbles([0x0, 0x1, 0x1, 0x0]), + ]); + prefix_set.extend(unchanged_prefix_set.clone()); + trie.prefix_set = prefix_set; + + // Update subtrie hashes + trie.update_subtrie_hashes(); + + // We expect that leaf 3 (0x02) should have been added to the prefix set, because it is + // missing a hash and is the root node of a lower subtrie, and therefore would need to have + // that hash calculated by `update_upper_subtrie_hashes`. + unchanged_prefix_set.insert(leaf_3_full_path); + + // Check that the prefix set was updated + assert_eq!( + trie.prefix_set.clone().freeze().into_iter().collect::>(), + unchanged_prefix_set.freeze().into_iter().collect::>() + ); + // Check that subtries were returned back to the array + assert!(trie.lower_subtries[subtrie_1_index].as_revealed_ref().is_some()); + assert!(trie.lower_subtries[subtrie_2_index].as_revealed_ref().is_some()); + assert!(trie.lower_subtries[subtrie_3_index].as_revealed_ref().is_some()); + } + + #[test] + fn test_subtrie_update_hashes() { + let mut subtrie = Box::new(SparseSubtrie::new(Nibbles::from_nibbles([0x0, 0x0]))); + + // Create leaf nodes with paths 0x0...0, 0x00001...0, 0x0010...0 + let leaf_1_full_path = Nibbles::from_nibbles([0; 64]); + let leaf_1_path = leaf_1_full_path.slice(..5); + let leaf_1_key = leaf_1_full_path.slice(5..); + let leaf_2_full_path = Nibbles::from_nibbles([vec![0, 0, 0, 0, 1], vec![0; 59]].concat()); + let leaf_2_path = leaf_2_full_path.slice(..5); + let leaf_2_key = leaf_2_full_path.slice(5..); + let leaf_3_full_path = Nibbles::from_nibbles([vec![0, 0, 1], vec![0; 61]].concat()); + let leaf_3_path = leaf_3_full_path.slice(..3); + let leaf_3_key = leaf_3_full_path.slice(3..); + + let account_1 = create_account(1); + let account_2 = create_account(2); + let account_3 = create_account(3); + let leaf_1 = create_leaf_node(leaf_1_key.to_vec(), account_1.nonce); + let leaf_2 = create_leaf_node(leaf_2_key.to_vec(), account_2.nonce); + let leaf_3 = create_leaf_node(leaf_3_key.to_vec(), account_3.nonce); + + // Create bottom branch node + let branch_1_path = Nibbles::from_nibbles([0, 0, 0, 0]); + let branch_1 = create_branch_node_with_children( + &[0, 1], + vec![ + RlpNode::from_rlp(&alloy_rlp::encode(&leaf_1)), + RlpNode::from_rlp(&alloy_rlp::encode(&leaf_2)), + ], + ); + + // Create an extension node + let extension_path = Nibbles::from_nibbles([0, 0, 0]); + let extension_key = Nibbles::from_nibbles([0]); + let extension = create_extension_node( + extension_key.to_vec(), + RlpNode::from_rlp(&alloy_rlp::encode(&branch_1)).as_hash().unwrap(), + ); + + // Create top branch node + let branch_2_path = Nibbles::from_nibbles([0, 0]); + let branch_2 = create_branch_node_with_children( + &[0, 1], + vec![ + RlpNode::from_rlp(&alloy_rlp::encode(&extension)), + RlpNode::from_rlp(&alloy_rlp::encode(&leaf_3)), + ], + ); + + // Reveal nodes + subtrie.reveal_node(branch_2_path, &branch_2, TrieMasks::none()).unwrap(); + subtrie.reveal_node(leaf_1_path, &leaf_1, TrieMasks::none()).unwrap(); + subtrie.reveal_node(extension_path, &extension, TrieMasks::none()).unwrap(); + subtrie.reveal_node(branch_1_path, &branch_1, TrieMasks::none()).unwrap(); + subtrie.reveal_node(leaf_2_path, &leaf_2, TrieMasks::none()).unwrap(); + subtrie.reveal_node(leaf_3_path, &leaf_3, TrieMasks::none()).unwrap(); + + // Run hash builder for two leaf nodes + let (_, _, proof_nodes, _, _) = run_hash_builder( + [ + (leaf_1_full_path, account_1), + (leaf_2_full_path, account_2), + (leaf_3_full_path, account_3), + ], + NoopAccountTrieCursor::default(), + Default::default(), + [ + branch_1_path, + extension_path, + branch_2_path, + leaf_1_full_path, + leaf_2_full_path, + leaf_3_full_path, + ], + ); + + // Update hashes for the subtrie + subtrie.update_hashes( + &mut PrefixSetMut::from([leaf_1_full_path, leaf_2_full_path, leaf_3_full_path]) + .freeze(), + &mut None, + &HashMap::default(), + &HashMap::default(), + ); + + // Compare hashes between hash builder and subtrie + let hash_builder_branch_1_hash = + RlpNode::from_rlp(proof_nodes.get(&branch_1_path).unwrap().as_ref()).as_hash().unwrap(); + let subtrie_branch_1_hash = subtrie.nodes.get(&branch_1_path).unwrap().hash().unwrap(); + assert_eq!(hash_builder_branch_1_hash, subtrie_branch_1_hash); + + let hash_builder_extension_hash = + RlpNode::from_rlp(proof_nodes.get(&extension_path).unwrap().as_ref()) + .as_hash() + .unwrap(); + let subtrie_extension_hash = subtrie.nodes.get(&extension_path).unwrap().hash().unwrap(); + assert_eq!(hash_builder_extension_hash, subtrie_extension_hash); + + let hash_builder_branch_2_hash = + RlpNode::from_rlp(proof_nodes.get(&branch_2_path).unwrap().as_ref()).as_hash().unwrap(); + let subtrie_branch_2_hash = subtrie.nodes.get(&branch_2_path).unwrap().hash().unwrap(); + assert_eq!(hash_builder_branch_2_hash, subtrie_branch_2_hash); + + let subtrie_leaf_1_hash = subtrie.nodes.get(&leaf_1_path).unwrap().hash().unwrap(); + let hash_builder_leaf_1_hash = + RlpNode::from_rlp(proof_nodes.get(&leaf_1_path).unwrap().as_ref()).as_hash().unwrap(); + assert_eq!(hash_builder_leaf_1_hash, subtrie_leaf_1_hash); + + let hash_builder_leaf_2_hash = + RlpNode::from_rlp(proof_nodes.get(&leaf_2_path).unwrap().as_ref()).as_hash().unwrap(); + let subtrie_leaf_2_hash = subtrie.nodes.get(&leaf_2_path).unwrap().hash().unwrap(); + assert_eq!(hash_builder_leaf_2_hash, subtrie_leaf_2_hash); + + let hash_builder_leaf_3_hash = + RlpNode::from_rlp(proof_nodes.get(&leaf_3_path).unwrap().as_ref()).as_hash().unwrap(); + let subtrie_leaf_3_hash = subtrie.nodes.get(&leaf_3_path).unwrap().hash().unwrap(); + assert_eq!(hash_builder_leaf_3_hash, subtrie_leaf_3_hash); + } + + #[test] + fn test_remove_leaf_branch_becomes_extension() { + // + // 0x: Extension (Key = 5) + // 0x5: └── Branch (Mask = 1001) + // 0x50: ├── 0 -> Extension (Key = 23) + // 0x5023: │ └── Branch (Mask = 0101) + // 0x50231: │ ├── 1 -> Leaf + // 0x50233: │ └── 3 -> Leaf + // 0x53: └── 3 -> Leaf (Key = 7) + // + // After removing 0x53, extension+branch+extension become a single extension + // + let mut trie = new_test_trie( + [ + (Nibbles::default(), SparseNode::new_ext(Nibbles::from_nibbles([0x5]))), + (Nibbles::from_nibbles([0x5]), SparseNode::new_branch(TrieMask::new(0b1001))), + ( + Nibbles::from_nibbles([0x5, 0x0]), + SparseNode::new_ext(Nibbles::from_nibbles([0x2, 0x3])), + ), + ( + Nibbles::from_nibbles([0x5, 0x0, 0x2, 0x3]), + SparseNode::new_branch(TrieMask::new(0b0101)), + ), + ( + Nibbles::from_nibbles([0x5, 0x0, 0x2, 0x3, 0x1]), + SparseNode::new_leaf(Nibbles::new()), + ), + ( + Nibbles::from_nibbles([0x5, 0x0, 0x2, 0x3, 0x3]), + SparseNode::new_leaf(Nibbles::new()), + ), + ( + Nibbles::from_nibbles([0x5, 0x3]), + SparseNode::new_leaf(Nibbles::from_nibbles([0x7])), + ), + ] + .into_iter(), + ); + + let provider = MockTrieNodeProvider::new(); + + // Remove the leaf with a full path of 0x537 + let leaf_full_path = Nibbles::from_nibbles([0x5, 0x3, 0x7]); + trie.remove_leaf(&leaf_full_path, provider).unwrap(); + + let upper_subtrie = &trie.upper_subtrie; + let lower_subtrie_50 = trie.lower_subtries[0x50].as_revealed_ref().unwrap(); + + // Check that the `SparseSubtrie` the leaf was removed from was itself removed, as it is now + // empty. + assert_matches!(trie.lower_subtries[0x53].as_revealed_ref(), None); + + // Check that the leaf node was removed, and that its parent/grandparent were modified + // appropriately. + assert_matches!( + upper_subtrie.nodes.get(&Nibbles::from_nibbles([])), + Some(SparseNode::Extension{ key, ..}) + if key == &Nibbles::from_nibbles([0x5, 0x0, 0x2, 0x3]) + ); + assert_matches!(upper_subtrie.nodes.get(&Nibbles::from_nibbles([0x5])), None); + assert_matches!(lower_subtrie_50.nodes.get(&Nibbles::from_nibbles([0x5, 0x0])), None); + assert_matches!( + lower_subtrie_50.nodes.get(&Nibbles::from_nibbles([0x5, 0x0, 0x2, 0x3])), + Some(SparseNode::Branch{ state_mask, .. }) + if *state_mask == 0b0101.into() + ); + } + + #[test] + fn test_remove_leaf_branch_becomes_leaf() { + // + // 0x: Branch (Mask = 0011) + // 0x0: ├── 0 -> Leaf (Key = 12) + // 0x1: └── 1 -> Leaf (Key = 34) + // + // After removing 0x012, branch becomes a leaf + // + let mut trie = new_test_trie( + [ + (Nibbles::default(), SparseNode::new_branch(TrieMask::new(0b0011))), + ( + Nibbles::from_nibbles([0x0]), + SparseNode::new_leaf(Nibbles::from_nibbles([0x1, 0x2])), + ), + ( + Nibbles::from_nibbles([0x1]), + SparseNode::new_leaf(Nibbles::from_nibbles([0x3, 0x4])), + ), + ] + .into_iter(), + ); + + // Add the branch node to updated_nodes to simulate it being modified earlier + if let Some(updates) = trie.updates.as_mut() { + updates + .updated_nodes + .insert(Nibbles::default(), BranchNodeCompact::new(0b11, 0, 0, vec![], None)); + } + + let provider = MockTrieNodeProvider::new(); + + // Remove the leaf with a full path of 0x012 + let leaf_full_path = Nibbles::from_nibbles([0x0, 0x1, 0x2]); + trie.remove_leaf(&leaf_full_path, provider).unwrap(); + + let upper_subtrie = &trie.upper_subtrie; + + // Check that the leaf's value was removed + assert_matches!(upper_subtrie.inner.values.get(&leaf_full_path), None); + + // Check that the branch node collapsed into a leaf node with the remaining child's key + assert_matches!( + upper_subtrie.nodes.get(&Nibbles::default()), + Some(SparseNode::Leaf{ key, ..}) + if key == &Nibbles::from_nibbles([0x1, 0x3, 0x4]) + ); + + // Check that the remaining child node was removed + assert_matches!(upper_subtrie.nodes.get(&Nibbles::from_nibbles([0x1])), None); + // Check that the removed child node was also removed + assert_matches!(upper_subtrie.nodes.get(&Nibbles::from_nibbles([0x0])), None); + + // Check that updates were tracked correctly when branch collapsed + let updates = trie.updates.as_ref().unwrap(); + + // The branch at root should be marked as removed since it collapsed + assert!(updates.removed_nodes.contains(&Nibbles::default())); + + // The branch should no longer be in updated_nodes + assert!(!updates.updated_nodes.contains_key(&Nibbles::default())); + } + + #[test] + fn test_remove_leaf_extension_becomes_leaf() { + // + // 0x: Extension (Key = 5) + // 0x5: └── Branch (Mask = 0011) + // 0x50: ├── 0 -> Leaf (Key = 12) + // 0x51: └── 1 -> Leaf (Key = 34) + // + // After removing 0x5012, extension+branch becomes a leaf + // + let mut trie = new_test_trie( + [ + (Nibbles::default(), SparseNode::new_ext(Nibbles::from_nibbles([0x5]))), + (Nibbles::from_nibbles([0x5]), SparseNode::new_branch(TrieMask::new(0b0011))), + ( + Nibbles::from_nibbles([0x5, 0x0]), + SparseNode::new_leaf(Nibbles::from_nibbles([0x1, 0x2])), + ), + ( + Nibbles::from_nibbles([0x5, 0x1]), + SparseNode::new_leaf(Nibbles::from_nibbles([0x3, 0x4])), + ), + ] + .into_iter(), + ); + + let provider = MockTrieNodeProvider::new(); + + // Remove the leaf with a full path of 0x5012 + let leaf_full_path = Nibbles::from_nibbles([0x5, 0x0, 0x1, 0x2]); + trie.remove_leaf(&leaf_full_path, provider).unwrap(); + + let upper_subtrie = &trie.upper_subtrie; + + // Check that both lower subtries were removed. 0x50 should have been removed because + // removing its leaf made it empty. 0x51 should have been removed after its own leaf was + // collapsed into the upper trie, leaving it also empty. + assert_matches!(trie.lower_subtries[0x50].as_revealed_ref(), None); + assert_matches!(trie.lower_subtries[0x51].as_revealed_ref(), None); + + // Check that the other leaf's value was moved to the upper trie + let other_leaf_full_value = Nibbles::from_nibbles([0x5, 0x1, 0x3, 0x4]); + assert_matches!(upper_subtrie.inner.values.get(&other_leaf_full_value), Some(_)); + + // Check that the extension node collapsed into a leaf node + assert_matches!( + upper_subtrie.nodes.get(&Nibbles::default()), + Some(SparseNode::Leaf{ key, ..}) + if key == &Nibbles::from_nibbles([0x5, 0x1, 0x3, 0x4]) + ); + + // Check that intermediate nodes were removed + assert_matches!(upper_subtrie.nodes.get(&Nibbles::from_nibbles([0x5])), None); + } + + #[test] + fn test_remove_leaf_branch_on_branch() { + // + // 0x: Branch (Mask = 0101) + // 0x0: ├── 0 -> Leaf (Key = 12) + // 0x2: └── 2 -> Branch (Mask = 0011) + // 0x20: ├── 0 -> Leaf (Key = 34) + // 0x21: └── 1 -> Leaf (Key = 56) + // + // After removing 0x2034, the inner branch becomes a leaf + // + let mut trie = new_test_trie( + [ + (Nibbles::default(), SparseNode::new_branch(TrieMask::new(0b0101))), + ( + Nibbles::from_nibbles([0x0]), + SparseNode::new_leaf(Nibbles::from_nibbles([0x1, 0x2])), + ), + (Nibbles::from_nibbles([0x2]), SparseNode::new_branch(TrieMask::new(0b0011))), + ( + Nibbles::from_nibbles([0x2, 0x0]), + SparseNode::new_leaf(Nibbles::from_nibbles([0x3, 0x4])), + ), + ( + Nibbles::from_nibbles([0x2, 0x1]), + SparseNode::new_leaf(Nibbles::from_nibbles([0x5, 0x6])), + ), + ] + .into_iter(), + ); + + let provider = MockTrieNodeProvider::new(); + + // Remove the leaf with a full path of 0x2034 + let leaf_full_path = Nibbles::from_nibbles([0x2, 0x0, 0x3, 0x4]); + trie.remove_leaf(&leaf_full_path, provider).unwrap(); + + let upper_subtrie = &trie.upper_subtrie; + + // Check that both lower subtries were removed. 0x20 should have been removed because + // removing its leaf made it empty. 0x21 should have been removed after its own leaf was + // collapsed into the upper trie, leaving it also empty. + assert_matches!(trie.lower_subtries[0x20].as_revealed_ref(), None); + assert_matches!(trie.lower_subtries[0x21].as_revealed_ref(), None); + + // Check that the other leaf's value was moved to the upper trie + let other_leaf_full_value = Nibbles::from_nibbles([0x2, 0x1, 0x5, 0x6]); + assert_matches!(upper_subtrie.inner.values.get(&other_leaf_full_value), Some(_)); + + // Check that the root branch still exists unchanged + assert_matches!( + upper_subtrie.nodes.get(&Nibbles::default()), + Some(SparseNode::Branch{ state_mask, .. }) + if *state_mask == 0b0101.into() + ); + + // Check that the inner branch became an extension + assert_matches!( + upper_subtrie.nodes.get(&Nibbles::from_nibbles([0x2])), + Some(SparseNode::Leaf{ key, ..}) + if key == &Nibbles::from_nibbles([0x1, 0x5, 0x6]) + ); + } + + #[test] + fn test_remove_leaf_lower_subtrie_root_path_update() { + // + // 0x: Extension (Key = 123, root of lower subtrie) + // 0x123: └── Branch (Mask = 0011000) + // 0x1233: ├── 3 -> Leaf (Key = []) + // 0x1234: └── 4 -> Extension (Key = 5) + // 0x12345: └── Branch (Mask = 0011) + // 0x123450: ├── 0 -> Leaf (Key = []) + // 0x123451: └── 1 -> Leaf (Key = []) + // + // After removing leaf at 0x1233, the branch at 0x123 becomes an extension to 0x12345, which + // then gets merged with the root extension at 0x. The lower subtrie's `path` field should + // be updated from 0x123 to 0x12345. + // + let mut trie = new_test_trie( + [ + (Nibbles::default(), SparseNode::new_ext(Nibbles::from_nibbles([0x1, 0x2, 0x3]))), + ( + Nibbles::from_nibbles([0x1, 0x2, 0x3]), + SparseNode::new_branch(TrieMask::new(0b0011000)), + ), + ( + Nibbles::from_nibbles([0x1, 0x2, 0x3, 0x3]), + SparseNode::new_leaf(Nibbles::default()), + ), + ( + Nibbles::from_nibbles([0x1, 0x2, 0x3, 0x4]), + SparseNode::new_ext(Nibbles::from_nibbles([0x5])), + ), + ( + Nibbles::from_nibbles([0x1, 0x2, 0x3, 0x4, 0x5]), + SparseNode::new_branch(TrieMask::new(0b0011)), + ), + ( + Nibbles::from_nibbles([0x1, 0x2, 0x3, 0x4, 0x5, 0x0]), + SparseNode::new_leaf(Nibbles::default()), + ), + ( + Nibbles::from_nibbles([0x1, 0x2, 0x3, 0x4, 0x5, 0x1]), + SparseNode::new_leaf(Nibbles::default()), + ), + ] + .into_iter(), + ); + + let provider = MockTrieNodeProvider::new(); + + // Verify initial state - the lower subtrie's path should be 0x123 + let lower_subtrie_root_path = Nibbles::from_nibbles([0x1, 0x2, 0x3]); + assert_matches!( + trie.lower_subtrie_for_path_mut(&lower_subtrie_root_path), + Some(subtrie) + if subtrie.path == lower_subtrie_root_path + ); + + // Remove the leaf at 0x1233 + let leaf_full_path = Nibbles::from_nibbles([0x1, 0x2, 0x3, 0x3]); + trie.remove_leaf(&leaf_full_path, provider).unwrap(); + + // After removal: + // 1. The branch at 0x123 should become an extension to 0x12345 + // 2. That extension should merge with the root extension at 0x + // 3. The lower subtrie's path should be updated to 0x12345 + let lower_subtrie = trie.lower_subtries[0x12].as_revealed_ref().unwrap(); + assert_eq!(lower_subtrie.path, Nibbles::from_nibbles([0x1, 0x2, 0x3, 0x4, 0x5])); + + // Verify the root extension now points all the way to 0x12345 + assert_matches!( + trie.upper_subtrie.nodes.get(&Nibbles::default()), + Some(SparseNode::Extension { key, .. }) + if key == &Nibbles::from_nibbles([0x1, 0x2, 0x3, 0x4, 0x5]) + ); + + // Verify the branch at 0x12345 hasn't been modified + assert_matches!( + lower_subtrie.nodes.get(&Nibbles::from_nibbles([0x1, 0x2, 0x3, 0x4, 0x5])), + Some(SparseNode::Branch { state_mask, .. }) + if state_mask == &TrieMask::new(0b0011) + ); + } + + #[test] + fn test_remove_leaf_remaining_child_needs_reveal() { + // + // 0x: Branch (Mask = 0011) + // 0x0: ├── 0 -> Leaf (Key = 12) + // 0x1: └── 1 -> Hash (blinded leaf) + // + // After removing 0x012, the hash node needs to be revealed to collapse the branch + // + let mut trie = new_test_trie( + [ + (Nibbles::default(), SparseNode::new_branch(TrieMask::new(0b0011))), + ( + Nibbles::from_nibbles([0x0]), + SparseNode::new_leaf(Nibbles::from_nibbles([0x1, 0x2])), + ), + (Nibbles::from_nibbles([0x1]), SparseNode::Hash(B256::repeat_byte(0xab))), + ] + .into_iter(), + ); + + // Create a mock provider that will reveal the blinded leaf + let mut provider = MockTrieNodeProvider::new(); + let revealed_leaf = create_leaf_node([0x3, 0x4], 42); + let mut encoded = Vec::new(); + revealed_leaf.encode(&mut encoded); + provider.add_revealed_node( + Nibbles::from_nibbles([0x1]), + RevealedNode { node: encoded.into(), tree_mask: None, hash_mask: None }, + ); + + // Remove the leaf with a full path of 0x012 + let leaf_full_path = Nibbles::from_nibbles([0x0, 0x1, 0x2]); + trie.remove_leaf(&leaf_full_path, provider).unwrap(); + + let upper_subtrie = &trie.upper_subtrie; + + // Check that the leaf value was removed + assert_matches!(upper_subtrie.inner.values.get(&leaf_full_path), None); + + // Check that the branch node collapsed into a leaf node with the revealed child's key + assert_matches!( + upper_subtrie.nodes.get(&Nibbles::default()), + Some(SparseNode::Leaf{ key, ..}) + if key == &Nibbles::from_nibbles([0x1, 0x3, 0x4]) + ); + + // Check that the remaining child node was removed (since it was merged) + assert_matches!(upper_subtrie.nodes.get(&Nibbles::from_nibbles([0x1])), None); + } + + #[test] + fn test_remove_leaf_root() { + // + // 0x: Leaf (Key = 123) + // + // After removing 0x123, the trie becomes empty + // + let mut trie = new_test_trie(std::iter::once(( + Nibbles::default(), + SparseNode::new_leaf(Nibbles::from_nibbles([0x1, 0x2, 0x3])), + ))); + + let provider = MockTrieNodeProvider::new(); + + // Remove the leaf with a full key of 0x123 + let leaf_full_path = Nibbles::from_nibbles([0x1, 0x2, 0x3]); + trie.remove_leaf(&leaf_full_path, provider).unwrap(); + + let upper_subtrie = &trie.upper_subtrie; + + // Check that the leaf value was removed + assert_matches!(upper_subtrie.inner.values.get(&leaf_full_path), None); + + // Check that the root node was changed to Empty + assert_matches!(upper_subtrie.nodes.get(&Nibbles::default()), Some(SparseNode::Empty)); + } + + #[test] + fn test_remove_leaf_unsets_hash_along_path() { + // + // Creates a trie structure: + // 0x: Branch (with hash set) + // 0x0: ├── Extension (with hash set) + // 0x01: │ └── Branch (with hash set) + // 0x012: │ ├── Leaf (Key = 34, with hash set) + // 0x013: │ ├── Leaf (Key = 56, with hash set) + // 0x014: │ └── Leaf (Key = 78, with hash set) + // 0x1: └── Leaf (Key = 78, with hash set) + // + // When removing leaf at 0x01234, all nodes along the path (root branch, + // extension at 0x0, branch at 0x01) should have their hash field unset + // + + let mut trie = new_test_trie( + [ + ( + Nibbles::default(), + SparseNode::Branch { + state_mask: TrieMask::new(0b0011), + hash: Some(B256::repeat_byte(0x10)), + store_in_db_trie: None, + }, + ), + ( + Nibbles::from_nibbles([0x0]), + SparseNode::Extension { + key: Nibbles::from_nibbles([0x1]), + hash: Some(B256::repeat_byte(0x20)), + store_in_db_trie: None, + }, + ), + ( + Nibbles::from_nibbles([0x0, 0x1]), + SparseNode::Branch { + state_mask: TrieMask::new(0b11100), + hash: Some(B256::repeat_byte(0x30)), + store_in_db_trie: None, + }, + ), + ( + Nibbles::from_nibbles([0x0, 0x1, 0x2]), + SparseNode::Leaf { + key: Nibbles::from_nibbles([0x3, 0x4]), + hash: Some(B256::repeat_byte(0x40)), + }, + ), + ( + Nibbles::from_nibbles([0x0, 0x1, 0x3]), + SparseNode::Leaf { + key: Nibbles::from_nibbles([0x5, 0x6]), + hash: Some(B256::repeat_byte(0x50)), + }, + ), + ( + Nibbles::from_nibbles([0x0, 0x1, 0x4]), + SparseNode::Leaf { + key: Nibbles::from_nibbles([0x6, 0x7]), + hash: Some(B256::repeat_byte(0x60)), + }, + ), + ( + Nibbles::from_nibbles([0x1]), + SparseNode::Leaf { + key: Nibbles::from_nibbles([0x7, 0x8]), + hash: Some(B256::repeat_byte(0x70)), + }, + ), + ] + .into_iter(), + ); + + let provider = MockTrieNodeProvider::new(); + + // Remove a leaf which does not exist; this should have no effect. + trie.remove_leaf(&Nibbles::from_nibbles([0x0, 0x1, 0x2, 0x3, 0x4, 0xF]), &provider) + .unwrap(); + for (path, node) in trie.all_nodes() { + assert!(node.hash().is_some(), "path {path:?} should still have a hash"); + } + + // Remove the leaf at path 0x01234 + let leaf_full_path = Nibbles::from_nibbles([0x0, 0x1, 0x2, 0x3, 0x4]); + trie.remove_leaf(&leaf_full_path, &provider).unwrap(); + + let upper_subtrie = &trie.upper_subtrie; + let lower_subtrie_10 = trie.lower_subtries[0x01].as_revealed_ref().unwrap(); + + // Verify that hash fields are unset for all nodes along the path to the removed leaf + assert_matches!( + upper_subtrie.nodes.get(&Nibbles::default()), + Some(SparseNode::Branch { hash: None, .. }) + ); + assert_matches!( + upper_subtrie.nodes.get(&Nibbles::from_nibbles([0x0])), + Some(SparseNode::Extension { hash: None, .. }) + ); + assert_matches!( + lower_subtrie_10.nodes.get(&Nibbles::from_nibbles([0x0, 0x1])), + Some(SparseNode::Branch { hash: None, .. }) + ); + + // Verify that nodes not on the path still have their hashes + assert_matches!( + upper_subtrie.nodes.get(&Nibbles::from_nibbles([0x1])), + Some(SparseNode::Leaf { hash: Some(_), .. }) + ); + assert_matches!( + lower_subtrie_10.nodes.get(&Nibbles::from_nibbles([0x0, 0x1, 0x3])), + Some(SparseNode::Leaf { hash: Some(_), .. }) + ); + assert_matches!( + lower_subtrie_10.nodes.get(&Nibbles::from_nibbles([0x0, 0x1, 0x4])), + Some(SparseNode::Leaf { hash: Some(_), .. }) + ); + } + + #[test] + fn test_remove_leaf_remaining_extension_node_child_is_revealed() { + let branch_path = Nibbles::from_nibbles([0x4, 0xf, 0x8, 0x8, 0x0, 0x7]); + let removed_branch_path = Nibbles::from_nibbles([0x4, 0xf, 0x8, 0x8, 0x0, 0x7, 0x2]); + + // Convert the logs into reveal_nodes call on a fresh ParallelSparseTrie + let nodes = vec![ + // Branch at 0x4f8807 + RevealedSparseNode { + path: branch_path, + node: { + TrieNode::Branch(BranchNode::new( + vec![ + RlpNode::word_rlp(&B256::from(hex!( + "dede882d52f0e0eddfb5b89293a10c87468b4a73acd0d4ae550054a92353f6d5" + ))), + RlpNode::word_rlp(&B256::from(hex!( + "8746f18e465e2eed16117306b6f2eef30bc9d2978aee4a7838255e39c41a3222" + ))), + RlpNode::word_rlp(&B256::from(hex!( + "35a4ea861548af5f0262a9b6d619b4fc88fce6531cbd004eab1530a73f34bbb1" + ))), + RlpNode::word_rlp(&B256::from(hex!( + "47d5c2bf9eea5c1ee027e4740c2b86159074a27d52fd2f6a8a8c86c77e48006f" + ))), + RlpNode::word_rlp(&B256::from(hex!( + "eb76a359b216e1d86b1f2803692a9fe8c3d3f97a9fe6a82b396e30344febc0c1" + ))), + RlpNode::word_rlp(&B256::from(hex!( + "437656f2697f167b23e33cb94acc8550128cfd647fc1579d61e982cb7616b8bc" + ))), + RlpNode::word_rlp(&B256::from(hex!( + "45a1ac2faf15ea8a4da6f921475974e0379f39c3d08166242255a567fa88ce6c" + ))), + RlpNode::word_rlp(&B256::from(hex!( + "7dbb299d714d3dfa593f53bc1b8c66d5c401c30a0b5587b01254a56330361395" + ))), + RlpNode::word_rlp(&B256::from(hex!( + "ae407eb14a74ed951c9949c1867fb9ee9ba5d5b7e03769eaf3f29c687d080429" + ))), + RlpNode::word_rlp(&B256::from(hex!( + "768d0fe1003f0e85d3bc76e4a1fa0827f63b10ca9bca52d56c2b1cceb8eb8b08" + ))), + RlpNode::word_rlp(&B256::from(hex!( + "e5127935143493d5094f4da6e4f7f5a0f62d524fbb61e7bb9fb63d8a166db0f3" + ))), + RlpNode::word_rlp(&B256::from(hex!( + "7f3698297308664fbc1b9e2c41d097fbd57d8f364c394f6ad7c71b10291fbf42" + ))), + RlpNode::word_rlp(&B256::from(hex!( + "4a2bc7e19cec63cb5ef5754add0208959b50bcc79f13a22a370f77b277dbe6db" + ))), + RlpNode::word_rlp(&B256::from(hex!( + "40764b8c48de59258e62a3371909a107e76e1b5e847cfa94dbc857e9fd205103" + ))), + RlpNode::word_rlp(&B256::from(hex!( + "2985dca29a7616920d95c43ab62eb013a40e6a0c88c284471e4c3bd22f3b9b25" + ))), + RlpNode::word_rlp(&B256::from(hex!( + "1b6511f7a385e79477239f7dd4a49f52082ecac05aa5bd0de18b1d55fe69d10c" + ))), + ], + TrieMask::new(0b1111111111111111), + )) + }, + masks: TrieMasks { + hash_mask: Some(TrieMask::new(0b1111111111111111)), + tree_mask: Some(TrieMask::new(0b0011110100100101)), + }, + }, + // Branch at 0x4f88072 + RevealedSparseNode { + path: removed_branch_path, + node: { + let stack = vec![ + RlpNode::word_rlp(&B256::from(hex!( + "15fd4993a41feff1af3b629b32572ab05acddd97c681d82ec2eb89c8a8e3ab9e" + ))), + RlpNode::word_rlp(&B256::from(hex!( + "a272b0b94ced4e6ec7adb41719850cf4a167ad8711d0dda6a810d129258a0d94" + ))), + ]; + let branch_node = BranchNode::new(stack, TrieMask::new(0b0001000000000100)); + TrieNode::Branch(branch_node) + }, + masks: TrieMasks { + hash_mask: Some(TrieMask::new(0b0000000000000000)), + tree_mask: Some(TrieMask::new(0b0000000000000100)), + }, + }, + // Extension at 0x4f880722 + RevealedSparseNode { + path: Nibbles::from_nibbles([0x4, 0xf, 0x8, 0x8, 0x0, 0x7, 0x2, 0x2]), + node: { + let extension_node = ExtensionNode::new( + Nibbles::from_nibbles([0x6]), + RlpNode::word_rlp(&B256::from(hex!( + "56fab2b106a97eae9c7197f86d03bca292da6e0ac725b783082f7d950cc4e0fc" + ))), + ); + TrieNode::Extension(extension_node) + }, + masks: TrieMasks { hash_mask: None, tree_mask: None }, + }, + // Leaf at 0x4f88072c + RevealedSparseNode { + path: Nibbles::from_nibbles([0x4, 0xf, 0x8, 0x8, 0x0, 0x7, 0x2, 0xc]), + node: { + let leaf_node = LeafNode::new( + Nibbles::from_nibbles([ + 0x0, 0x7, 0x7, 0xf, 0x8, 0x6, 0x6, 0x1, 0x3, 0x0, 0x8, 0x8, 0xd, 0xf, + 0xc, 0xa, 0xe, 0x6, 0x4, 0x8, 0xa, 0xb, 0xe, 0x8, 0x3, 0x1, 0xf, 0xa, + 0xd, 0xc, 0xa, 0x5, 0x5, 0xa, 0xd, 0x4, 0x3, 0xa, 0xb, 0x1, 0x6, 0x5, + 0xd, 0x1, 0x6, 0x8, 0x0, 0xd, 0xd, 0x5, 0x6, 0x7, 0xb, 0x5, 0xd, 0x6, + ]), + hex::decode("8468d3971d").unwrap(), + ); + TrieNode::Leaf(leaf_node) + }, + masks: TrieMasks { hash_mask: None, tree_mask: None }, + }, + ]; + + // Create a fresh ParallelSparseTrie + let mut trie = ParallelSparseTrie::from_root( + TrieNode::Extension(ExtensionNode::new( + Nibbles::from_nibbles([0x4, 0xf, 0x8, 0x8, 0x0, 0x7]), + RlpNode::word_rlp(&B256::from(hex!( + "56fab2b106a97eae9c7197f86d03bca292da6e0ac725b783082f7d950cc4e0fc" + ))), + )), + TrieMasks::none(), + true, + ) + .unwrap(); + + // Call reveal_nodes + trie.reveal_nodes(nodes).unwrap(); + + // Remove the leaf at "0x4f88072c077f86613088dfcae648abe831fadca55ad43ab165d1680dd567b5d6" + let leaf_key = Nibbles::from_nibbles([ + 0x4, 0xf, 0x8, 0x8, 0x0, 0x7, 0x2, 0xc, 0x0, 0x7, 0x7, 0xf, 0x8, 0x6, 0x6, 0x1, 0x3, + 0x0, 0x8, 0x8, 0xd, 0xf, 0xc, 0xa, 0xe, 0x6, 0x4, 0x8, 0xa, 0xb, 0xe, 0x8, 0x3, 0x1, + 0xf, 0xa, 0xd, 0xc, 0xa, 0x5, 0x5, 0xa, 0xd, 0x4, 0x3, 0xa, 0xb, 0x1, 0x6, 0x5, 0xd, + 0x1, 0x6, 0x8, 0x0, 0xd, 0xd, 0x5, 0x6, 0x7, 0xb, 0x5, 0xd, 0x6, + ]); + + let mut provider = MockTrieNodeProvider::new(); + let revealed_branch = create_branch_node_with_children(&[], []); + let mut encoded = Vec::new(); + revealed_branch.encode(&mut encoded); + provider.add_revealed_node( + Nibbles::from_nibbles([0x4, 0xf, 0x8, 0x8, 0x0, 0x7, 0x2, 0x2, 0x6]), + RevealedNode { + node: encoded.into(), + tree_mask: None, + // Give it a fake hashmask so that it appears like it will be stored in the db + hash_mask: Some(TrieMask::new(0b1111)), + }, + ); + + trie.remove_leaf(&leaf_key, provider).unwrap(); + + // Calculate root so that updates are calculated. + trie.root(); + + // Take updates and assert they are correct + let updates = trie.take_updates(); + assert_eq!( + updates.removed_nodes.into_iter().collect::>(), + vec![removed_branch_path] + ); + assert_eq!(updates.updated_nodes.len(), 1); + let updated_node = updates.updated_nodes.get(&branch_path).unwrap(); + + // Second bit must be set, indicating that the extension's child is in the db + assert_eq!(updated_node.tree_mask, TrieMask::new(0b011110100100101),) + } + + #[test] + fn test_parallel_sparse_trie_root() { + // Step 1: Create the trie structure + // Extension node at 0x with key 0x2 (goes to upper subtrie) + let extension_path = Nibbles::new(); + let extension_key = Nibbles::from_nibbles([0x2]); + + // Branch node at 0x2 with children 0 and 1 (goes to upper subtrie) + let branch_path = Nibbles::from_nibbles([0x2]); + + // Leaf nodes at 0x20 and 0x21 (go to lower subtries) + let leaf_1_path = Nibbles::from_nibbles([0x2, 0x0]); + let leaf_1_key = Nibbles::from_nibbles(vec![0; 62]); // Remaining key + let leaf_1_full_path = Nibbles::from_nibbles([vec![0x2, 0x0], vec![0; 62]].concat()); + + let leaf_2_path = Nibbles::from_nibbles([0x2, 0x1]); + let leaf_2_key = Nibbles::from_nibbles(vec![0; 62]); // Remaining key + let leaf_2_full_path = Nibbles::from_nibbles([vec![0x2, 0x1], vec![0; 62]].concat()); + + // Create accounts + let account_1 = create_account(1); + let account_2 = create_account(2); + + // Create leaf nodes + let leaf_1 = create_leaf_node(leaf_1_key.to_vec(), account_1.nonce); + let leaf_2 = create_leaf_node(leaf_2_key.to_vec(), account_2.nonce); + + // Create branch node with children at indices 0 and 1 + let branch = create_branch_node_with_children( + &[0, 1], + vec![ + RlpNode::from_rlp(&alloy_rlp::encode(&leaf_1)), + RlpNode::from_rlp(&alloy_rlp::encode(&leaf_2)), + ], + ); + + // Create extension node pointing to branch + let extension = create_extension_node( + extension_key.to_vec(), + RlpNode::from_rlp(&alloy_rlp::encode(&branch)).as_hash().unwrap(), + ); + + // Step 2: Reveal nodes in the trie + let mut trie = ParallelSparseTrie::from_root(extension, TrieMasks::none(), true).unwrap(); + trie.reveal_nodes(vec![ + RevealedSparseNode { path: branch_path, node: branch, masks: TrieMasks::none() }, + RevealedSparseNode { path: leaf_1_path, node: leaf_1, masks: TrieMasks::none() }, + RevealedSparseNode { path: leaf_2_path, node: leaf_2, masks: TrieMasks::none() }, + ]) + .unwrap(); + + // Step 3: Reset hashes for all revealed nodes to test actual hash calculation + // Reset upper subtrie node hashes + trie.upper_subtrie.nodes.get_mut(&extension_path).unwrap().set_hash(None); + trie.upper_subtrie.nodes.get_mut(&branch_path).unwrap().set_hash(None); + + // Reset lower subtrie node hashes + let leaf_1_subtrie_idx = path_subtrie_index_unchecked(&leaf_1_path); + let leaf_2_subtrie_idx = path_subtrie_index_unchecked(&leaf_2_path); + + trie.lower_subtries[leaf_1_subtrie_idx] + .as_revealed_mut() + .unwrap() + .nodes + .get_mut(&leaf_1_path) + .unwrap() + .set_hash(None); + trie.lower_subtries[leaf_2_subtrie_idx] + .as_revealed_mut() + .unwrap() + .nodes + .get_mut(&leaf_2_path) + .unwrap() + .set_hash(None); + + // Step 4: Add changed leaf node paths to prefix set + trie.prefix_set.insert(leaf_1_full_path); + trie.prefix_set.insert(leaf_2_full_path); + + // Step 5: Calculate root using our implementation + let root = trie.root(); + + // Step 6: Calculate root using HashBuilder for comparison + let (hash_builder_root, _, _proof_nodes, _, _) = run_hash_builder( + [(leaf_1_full_path, account_1), (leaf_2_full_path, account_2)], + NoopAccountTrieCursor::default(), + Default::default(), + [extension_path, branch_path, leaf_1_full_path, leaf_2_full_path], + ); + + // Step 7: Verify the roots match + assert_eq!(root, hash_builder_root); + + // Verify hashes were computed + let leaf_1_subtrie = trie.lower_subtries[leaf_1_subtrie_idx].as_revealed_ref().unwrap(); + let leaf_2_subtrie = trie.lower_subtries[leaf_2_subtrie_idx].as_revealed_ref().unwrap(); + assert!(trie.upper_subtrie.nodes.get(&extension_path).unwrap().hash().is_some()); + assert!(trie.upper_subtrie.nodes.get(&branch_path).unwrap().hash().is_some()); + assert!(leaf_1_subtrie.nodes.get(&leaf_1_path).unwrap().hash().is_some()); + assert!(leaf_2_subtrie.nodes.get(&leaf_2_path).unwrap().hash().is_some()); + } + + #[test] + fn sparse_trie_empty_update_one() { + let ctx = ParallelSparseTrieTestContext; + + let key = Nibbles::unpack(B256::with_last_byte(42)); + let value = || Account::default(); + let value_encoded = || { + let mut account_rlp = Vec::new(); + value().into_trie_account(EMPTY_ROOT_HASH).encode(&mut account_rlp); + account_rlp + }; + + let (hash_builder_root, hash_builder_updates, hash_builder_proof_nodes, _, _) = + run_hash_builder( + [(key, value())], + NoopAccountTrieCursor::default(), + Default::default(), + [key], + ); + + let mut sparse = ParallelSparseTrie::default().with_updates(true); + ctx.update_leaves(&mut sparse, [(key, value_encoded())]); + ctx.assert_with_hash_builder( + &mut sparse, + hash_builder_root, + hash_builder_updates, + hash_builder_proof_nodes, + ); + } + + #[test] + fn sparse_trie_empty_update_multiple_lower_nibbles() { + let ctx = ParallelSparseTrieTestContext; + + let paths = (0..=16).map(|b| Nibbles::unpack(B256::with_last_byte(b))).collect::>(); + let value = || Account::default(); + let value_encoded = || { + let mut account_rlp = Vec::new(); + value().into_trie_account(EMPTY_ROOT_HASH).encode(&mut account_rlp); + account_rlp + }; + + let (hash_builder_root, hash_builder_updates, hash_builder_proof_nodes, _, _) = + run_hash_builder( + paths.iter().copied().zip(std::iter::repeat_with(value)), + NoopAccountTrieCursor::default(), + Default::default(), + paths.clone(), + ); + + let mut sparse = ParallelSparseTrie::default().with_updates(true); + ctx.update_leaves( + &mut sparse, + paths.into_iter().zip(std::iter::repeat_with(value_encoded)), + ); + + ctx.assert_with_hash_builder( + &mut sparse, + hash_builder_root, + hash_builder_updates, + hash_builder_proof_nodes, + ); + } + + #[test] + fn sparse_trie_empty_update_multiple_upper_nibbles() { + let paths = (239..=255).map(|b| Nibbles::unpack(B256::repeat_byte(b))).collect::>(); + let value = || Account::default(); + let value_encoded = || { + let mut account_rlp = Vec::new(); + value().into_trie_account(EMPTY_ROOT_HASH).encode(&mut account_rlp); + account_rlp + }; + + let (hash_builder_root, hash_builder_updates, hash_builder_proof_nodes, _, _) = + run_hash_builder( + paths.iter().copied().zip(std::iter::repeat_with(value)), + NoopAccountTrieCursor::default(), + Default::default(), + paths.clone(), + ); + + let provider = DefaultTrieNodeProvider; + let mut sparse = ParallelSparseTrie::default().with_updates(true); + for path in &paths { + sparse.update_leaf(*path, value_encoded(), &provider).unwrap(); + } + let sparse_root = sparse.root(); + let sparse_updates = sparse.take_updates(); + + assert_eq!(sparse_root, hash_builder_root); + assert_eq!(sparse_updates.updated_nodes, hash_builder_updates.account_nodes); + assert_eq_parallel_sparse_trie_proof_nodes(&sparse, hash_builder_proof_nodes); + } + + #[test] + fn sparse_trie_empty_update_multiple() { + let ctx = ParallelSparseTrieTestContext; + + let paths = (0..=255) + .map(|b| { + Nibbles::unpack(if b % 2 == 0 { + B256::repeat_byte(b) + } else { + B256::with_last_byte(b) + }) + }) + .collect::>(); + let value = || Account::default(); + let value_encoded = || { + let mut account_rlp = Vec::new(); + value().into_trie_account(EMPTY_ROOT_HASH).encode(&mut account_rlp); + account_rlp + }; + + let (hash_builder_root, hash_builder_updates, hash_builder_proof_nodes, _, _) = + run_hash_builder( + paths.iter().sorted_unstable().copied().zip(std::iter::repeat_with(value)), + NoopAccountTrieCursor::default(), + Default::default(), + paths.clone(), + ); + + let mut sparse = ParallelSparseTrie::default().with_updates(true); + ctx.update_leaves( + &mut sparse, + paths.iter().copied().zip(std::iter::repeat_with(value_encoded)), + ); + ctx.assert_with_hash_builder( + &mut sparse, + hash_builder_root, + hash_builder_updates, + hash_builder_proof_nodes, + ); + } + + #[test] + fn sparse_trie_empty_update_repeated() { + let ctx = ParallelSparseTrieTestContext; + + let paths = (0..=255).map(|b| Nibbles::unpack(B256::repeat_byte(b))).collect::>(); + let old_value = Account { nonce: 1, ..Default::default() }; + let old_value_encoded = { + let mut account_rlp = Vec::new(); + old_value.into_trie_account(EMPTY_ROOT_HASH).encode(&mut account_rlp); + account_rlp + }; + let new_value = Account { nonce: 2, ..Default::default() }; + let new_value_encoded = { + let mut account_rlp = Vec::new(); + new_value.into_trie_account(EMPTY_ROOT_HASH).encode(&mut account_rlp); + account_rlp + }; + + let (hash_builder_root, hash_builder_updates, hash_builder_proof_nodes, _, _) = + run_hash_builder( + paths.iter().copied().zip(std::iter::repeat_with(|| old_value)), + NoopAccountTrieCursor::default(), + Default::default(), + paths.clone(), + ); + + let mut sparse = ParallelSparseTrie::default().with_updates(true); + ctx.update_leaves( + &mut sparse, + paths.iter().copied().zip(std::iter::repeat(old_value_encoded)), + ); + ctx.assert_with_hash_builder( + &mut sparse, + hash_builder_root, + hash_builder_updates, + hash_builder_proof_nodes, + ); + + let (hash_builder_root, hash_builder_updates, hash_builder_proof_nodes, _, _) = + run_hash_builder( + paths.iter().copied().zip(std::iter::repeat(new_value)), + NoopAccountTrieCursor::default(), + Default::default(), + paths.clone(), + ); + + ctx.update_leaves( + &mut sparse, + paths.iter().copied().zip(std::iter::repeat(new_value_encoded)), + ); + ctx.assert_with_hash_builder( + &mut sparse, + hash_builder_root, + hash_builder_updates, + hash_builder_proof_nodes, + ); + } + + #[test] + fn sparse_trie_remove_leaf() { + let ctx = ParallelSparseTrieTestContext; + let provider = DefaultTrieNodeProvider; + let mut sparse = ParallelSparseTrie::default(); + + let value = alloy_rlp::encode_fixed_size(&U256::ZERO).to_vec(); + + ctx.update_leaves( + &mut sparse, + [ + (Nibbles::from_nibbles([0x5, 0x0, 0x2, 0x3, 0x1]), value.clone()), + (Nibbles::from_nibbles([0x5, 0x0, 0x2, 0x3, 0x3]), value.clone()), + (Nibbles::from_nibbles([0x5, 0x2, 0x0, 0x1, 0x3]), value.clone()), + (Nibbles::from_nibbles([0x5, 0x3, 0x1, 0x0, 0x2]), value.clone()), + (Nibbles::from_nibbles([0x5, 0x3, 0x3, 0x0, 0x2]), value.clone()), + (Nibbles::from_nibbles([0x5, 0x3, 0x3, 0x2, 0x0]), value), + ], + ); + + // Extension (Key = 5) + // └── Branch (Mask = 1011) + // ├── 0 -> Extension (Key = 23) + // │ └── Branch (Mask = 0101) + // │ ├── 1 -> Leaf (Key = 1, Path = 50231) + // │ └── 3 -> Leaf (Key = 3, Path = 50233) + // ├── 2 -> Leaf (Key = 013, Path = 52013) + // └── 3 -> Branch (Mask = 0101) + // ├── 1 -> Leaf (Key = 3102, Path = 53102) + // └── 3 -> Branch (Mask = 1010) + // ├── 0 -> Leaf (Key = 3302, Path = 53302) + // └── 2 -> Leaf (Key = 3320, Path = 53320) + pretty_assertions::assert_eq!( + parallel_sparse_trie_nodes(&sparse) + .into_iter() + .map(|(k, v)| (*k, v.clone())) + .collect::>(), + BTreeMap::from_iter([ + (Nibbles::default(), SparseNode::new_ext(Nibbles::from_nibbles([0x5]))), + (Nibbles::from_nibbles([0x5]), SparseNode::new_branch(0b1101.into())), + ( + Nibbles::from_nibbles([0x5, 0x0]), + SparseNode::new_ext(Nibbles::from_nibbles([0x2, 0x3])) + ), + ( + Nibbles::from_nibbles([0x5, 0x0, 0x2, 0x3]), + SparseNode::new_branch(0b1010.into()) + ), + ( + Nibbles::from_nibbles([0x5, 0x0, 0x2, 0x3, 0x1]), + SparseNode::new_leaf(Nibbles::default()) + ), + ( + Nibbles::from_nibbles([0x5, 0x0, 0x2, 0x3, 0x3]), + SparseNode::new_leaf(Nibbles::default()) + ), + ( + Nibbles::from_nibbles([0x5, 0x2]), + SparseNode::new_leaf(Nibbles::from_nibbles([0x0, 0x1, 0x3])) + ), + (Nibbles::from_nibbles([0x5, 0x3]), SparseNode::new_branch(0b1010.into())), + ( + Nibbles::from_nibbles([0x5, 0x3, 0x1]), + SparseNode::new_leaf(Nibbles::from_nibbles([0x0, 0x2])) + ), + (Nibbles::from_nibbles([0x5, 0x3, 0x3]), SparseNode::new_branch(0b0101.into())), + ( + Nibbles::from_nibbles([0x5, 0x3, 0x3, 0x0]), + SparseNode::new_leaf(Nibbles::from_nibbles([0x2])) + ), + ( + Nibbles::from_nibbles([0x5, 0x3, 0x3, 0x2]), + SparseNode::new_leaf(Nibbles::from_nibbles([0x0])) + ) + ]) + ); + + sparse.remove_leaf(&Nibbles::from_nibbles([0x5, 0x2, 0x0, 0x1, 0x3]), &provider).unwrap(); + + // Extension (Key = 5) + // └── Branch (Mask = 1001) + // ├── 0 -> Extension (Key = 23) + // │ └── Branch (Mask = 0101) + // │ ├── 1 -> Leaf (Key = 0231, Path = 50231) + // │ └── 3 -> Leaf (Key = 0233, Path = 50233) + // └── 3 -> Branch (Mask = 0101) + // ├── 1 -> Leaf (Key = 3102, Path = 53102) + // └── 3 -> Branch (Mask = 1010) + // ├── 0 -> Leaf (Key = 3302, Path = 53302) + // └── 2 -> Leaf (Key = 3320, Path = 53320) + pretty_assertions::assert_eq!( + parallel_sparse_trie_nodes(&sparse) + .into_iter() + .map(|(k, v)| (*k, v.clone())) + .collect::>(), + BTreeMap::from_iter([ + (Nibbles::default(), SparseNode::new_ext(Nibbles::from_nibbles([0x5]))), + (Nibbles::from_nibbles([0x5]), SparseNode::new_branch(0b1001.into())), + ( + Nibbles::from_nibbles([0x5, 0x0]), + SparseNode::new_ext(Nibbles::from_nibbles([0x2, 0x3])) + ), + ( + Nibbles::from_nibbles([0x5, 0x0, 0x2, 0x3]), + SparseNode::new_branch(0b1010.into()) + ), + ( + Nibbles::from_nibbles([0x5, 0x0, 0x2, 0x3, 0x1]), + SparseNode::new_leaf(Nibbles::default()) + ), + ( + Nibbles::from_nibbles([0x5, 0x0, 0x2, 0x3, 0x3]), + SparseNode::new_leaf(Nibbles::default()) + ), + (Nibbles::from_nibbles([0x5, 0x3]), SparseNode::new_branch(0b1010.into())), + ( + Nibbles::from_nibbles([0x5, 0x3, 0x1]), + SparseNode::new_leaf(Nibbles::from_nibbles([0x0, 0x2])) + ), + (Nibbles::from_nibbles([0x5, 0x3, 0x3]), SparseNode::new_branch(0b0101.into())), + ( + Nibbles::from_nibbles([0x5, 0x3, 0x3, 0x0]), + SparseNode::new_leaf(Nibbles::from_nibbles([0x2])) + ), + ( + Nibbles::from_nibbles([0x5, 0x3, 0x3, 0x2]), + SparseNode::new_leaf(Nibbles::from_nibbles([0x0])) + ) + ]) + ); + + sparse.remove_leaf(&Nibbles::from_nibbles([0x5, 0x0, 0x2, 0x3, 0x1]), &provider).unwrap(); + + // Extension (Key = 5) + // └── Branch (Mask = 1001) + // ├── 0 -> Leaf (Key = 0233, Path = 50233) + // └── 3 -> Branch (Mask = 0101) + // ├── 1 -> Leaf (Key = 3102, Path = 53102) + // └── 3 -> Branch (Mask = 1010) + // ├── 0 -> Leaf (Key = 3302, Path = 53302) + // └── 2 -> Leaf (Key = 3320, Path = 53320) + pretty_assertions::assert_eq!( + parallel_sparse_trie_nodes(&sparse) + .into_iter() + .map(|(k, v)| (*k, v.clone())) + .collect::>(), + BTreeMap::from_iter([ + (Nibbles::default(), SparseNode::new_ext(Nibbles::from_nibbles([0x5]))), + (Nibbles::from_nibbles([0x5]), SparseNode::new_branch(0b1001.into())), + ( + Nibbles::from_nibbles([0x5, 0x0]), + SparseNode::new_leaf(Nibbles::from_nibbles([0x2, 0x3, 0x3])) + ), + (Nibbles::from_nibbles([0x5, 0x3]), SparseNode::new_branch(0b1010.into())), + ( + Nibbles::from_nibbles([0x5, 0x3, 0x1]), + SparseNode::new_leaf(Nibbles::from_nibbles([0x0, 0x2])) + ), + (Nibbles::from_nibbles([0x5, 0x3, 0x3]), SparseNode::new_branch(0b0101.into())), + ( + Nibbles::from_nibbles([0x5, 0x3, 0x3, 0x0]), + SparseNode::new_leaf(Nibbles::from_nibbles([0x2])) + ), + ( + Nibbles::from_nibbles([0x5, 0x3, 0x3, 0x2]), + SparseNode::new_leaf(Nibbles::from_nibbles([0x0])) + ) + ]) + ); + + sparse.remove_leaf(&Nibbles::from_nibbles([0x5, 0x3, 0x1, 0x0, 0x2]), &provider).unwrap(); + + // Extension (Key = 5) + // └── Branch (Mask = 1001) + // ├── 0 -> Leaf (Key = 0233, Path = 50233) + // └── 3 -> Branch (Mask = 1010) + // ├── 0 -> Leaf (Key = 3302, Path = 53302) + // └── 2 -> Leaf (Key = 3320, Path = 53320) + pretty_assertions::assert_eq!( + parallel_sparse_trie_nodes(&sparse) + .into_iter() + .map(|(k, v)| (*k, v.clone())) + .collect::>(), + BTreeMap::from_iter([ + (Nibbles::default(), SparseNode::new_ext(Nibbles::from_nibbles([0x5]))), + (Nibbles::from_nibbles([0x5]), SparseNode::new_branch(0b1001.into())), + ( + Nibbles::from_nibbles([0x5, 0x0]), + SparseNode::new_leaf(Nibbles::from_nibbles([0x2, 0x3, 0x3])) + ), + ( + Nibbles::from_nibbles([0x5, 0x3]), + SparseNode::new_ext(Nibbles::from_nibbles([0x3])) + ), + (Nibbles::from_nibbles([0x5, 0x3, 0x3]), SparseNode::new_branch(0b0101.into())), + ( + Nibbles::from_nibbles([0x5, 0x3, 0x3, 0x0]), + SparseNode::new_leaf(Nibbles::from_nibbles([0x2])) + ), + ( + Nibbles::from_nibbles([0x5, 0x3, 0x3, 0x2]), + SparseNode::new_leaf(Nibbles::from_nibbles([0x0])) + ) + ]) + ); + + sparse.remove_leaf(&Nibbles::from_nibbles([0x5, 0x3, 0x3, 0x2, 0x0]), &provider).unwrap(); + + // Extension (Key = 5) + // └── Branch (Mask = 1001) + // ├── 0 -> Leaf (Key = 0233, Path = 50233) + // └── 3 -> Leaf (Key = 3302, Path = 53302) + pretty_assertions::assert_eq!( + parallel_sparse_trie_nodes(&sparse) + .into_iter() + .map(|(k, v)| (*k, v.clone())) + .collect::>(), + BTreeMap::from_iter([ + (Nibbles::default(), SparseNode::new_ext(Nibbles::from_nibbles([0x5]))), + (Nibbles::from_nibbles([0x5]), SparseNode::new_branch(0b1001.into())), + ( + Nibbles::from_nibbles([0x5, 0x0]), + SparseNode::new_leaf(Nibbles::from_nibbles([0x2, 0x3, 0x3])) + ), + ( + Nibbles::from_nibbles([0x5, 0x3]), + SparseNode::new_leaf(Nibbles::from_nibbles([0x3, 0x0, 0x2])) + ), + ]) + ); + + sparse.remove_leaf(&Nibbles::from_nibbles([0x5, 0x0, 0x2, 0x3, 0x3]), &provider).unwrap(); + + // Leaf (Key = 53302) + pretty_assertions::assert_eq!( + parallel_sparse_trie_nodes(&sparse) + .into_iter() + .map(|(k, v)| (*k, v.clone())) + .collect::>(), + BTreeMap::from_iter([( + Nibbles::default(), + SparseNode::new_leaf(Nibbles::from_nibbles([0x5, 0x3, 0x3, 0x0, 0x2])) + ),]) + ); + + sparse.remove_leaf(&Nibbles::from_nibbles([0x5, 0x3, 0x3, 0x0, 0x2]), &provider).unwrap(); + + // Empty + pretty_assertions::assert_eq!( + parallel_sparse_trie_nodes(&sparse) + .into_iter() + .map(|(k, v)| (*k, v.clone())) + .collect::>(), + BTreeMap::from_iter([(Nibbles::default(), SparseNode::Empty)]) + ); + } + + #[test] + fn sparse_trie_remove_leaf_blinded() { + let leaf = LeafNode::new( + Nibbles::default(), + alloy_rlp::encode_fixed_size(&U256::from(1)).to_vec(), + ); + let branch = TrieNode::Branch(BranchNode::new( + vec![ + RlpNode::word_rlp(&B256::repeat_byte(1)), + RlpNode::from_raw_rlp(&alloy_rlp::encode(leaf.clone())).unwrap(), + ], + TrieMask::new(0b11), + )); + + let provider = DefaultTrieNodeProvider; + let mut sparse = ParallelSparseTrie::from_root( + branch.clone(), + TrieMasks { hash_mask: Some(TrieMask::new(0b01)), tree_mask: None }, + false, + ) + .unwrap(); + + // Reveal a branch node and one of its children + // + // Branch (Mask = 11) + // ├── 0 -> Hash (Path = 0) + // └── 1 -> Leaf (Path = 1) + sparse + .reveal_nodes(vec![ + RevealedSparseNode { + path: Nibbles::default(), + node: branch, + masks: TrieMasks { hash_mask: None, tree_mask: Some(TrieMask::new(0b01)) }, + }, + RevealedSparseNode { + path: Nibbles::from_nibbles([0x1]), + node: TrieNode::Leaf(leaf), + masks: TrieMasks::none(), + }, + ]) + .unwrap(); + + // Removing a blinded leaf should result in an error + assert_matches!( + sparse.remove_leaf(&Nibbles::from_nibbles([0x0]), &provider).map_err(|e| e.into_kind()), + Err(SparseTrieErrorKind::BlindedNode { path, hash }) if path == Nibbles::from_nibbles([0x0]) && hash == B256::repeat_byte(1) + ); + } + + #[test] + fn sparse_trie_remove_leaf_non_existent() { + let leaf = LeafNode::new( + Nibbles::default(), + alloy_rlp::encode_fixed_size(&U256::from(1)).to_vec(), + ); + let branch = TrieNode::Branch(BranchNode::new( + vec![ + RlpNode::word_rlp(&B256::repeat_byte(1)), + RlpNode::from_raw_rlp(&alloy_rlp::encode(leaf.clone())).unwrap(), + ], + TrieMask::new(0b11), + )); + + let provider = DefaultTrieNodeProvider; + let mut sparse = ParallelSparseTrie::from_root( + branch.clone(), + TrieMasks { hash_mask: Some(TrieMask::new(0b01)), tree_mask: None }, + false, + ) + .unwrap(); + + // Reveal a branch node and one of its children + // + // Branch (Mask = 11) + // ├── 0 -> Hash (Path = 0) + // └── 1 -> Leaf (Path = 1) + sparse + .reveal_nodes(vec![ + RevealedSparseNode { + path: Nibbles::default(), + node: branch, + masks: TrieMasks { hash_mask: None, tree_mask: Some(TrieMask::new(0b01)) }, + }, + RevealedSparseNode { + path: Nibbles::from_nibbles([0x1]), + node: TrieNode::Leaf(leaf), + masks: TrieMasks::none(), + }, + ]) + .unwrap(); + + // Removing a non-existent leaf should be a noop + let sparse_old = sparse.clone(); + assert_matches!(sparse.remove_leaf(&Nibbles::from_nibbles([0x2]), &provider), Ok(())); + assert_eq!(sparse, sparse_old); + } + + #[test] + fn sparse_trie_fuzz() { + // Having only the first 3 nibbles set, we narrow down the range of keys + // to 4096 different hashes. It allows us to generate collisions more likely + // to test the sparse trie updates. + const KEY_NIBBLES_LEN: usize = 3; + + fn test(updates: Vec<(BTreeMap, BTreeSet)>) { + { + let mut state = BTreeMap::default(); + let default_provider = DefaultTrieNodeProvider; + let provider_factory = create_test_provider_factory(); + let mut sparse = ParallelSparseTrie::default().with_updates(true); + + for (update, keys_to_delete) in updates { + // Insert state updates into the sparse trie and calculate the root + for (key, account) in update.clone() { + let account = account.into_trie_account(EMPTY_ROOT_HASH); + let mut account_rlp = Vec::new(); + account.encode(&mut account_rlp); + sparse.update_leaf(key, account_rlp, &default_provider).unwrap(); + } + // We need to clone the sparse trie, so that all updated branch nodes are + // preserved, and not only those that were changed after the last call to + // `root()`. + let mut updated_sparse = sparse.clone(); + let sparse_root = updated_sparse.root(); + let sparse_updates = updated_sparse.take_updates(); + + // Insert state updates into the hash builder and calculate the root + state.extend(update); + let provider = provider_factory.provider().unwrap(); + let trie_cursor = DatabaseTrieCursorFactory::new(provider.tx_ref()); + let (hash_builder_root, hash_builder_updates, hash_builder_proof_nodes, _, _) = + run_hash_builder( + state.clone(), + trie_cursor.account_trie_cursor().unwrap(), + Default::default(), + state.keys().copied(), + ); + + // Extract account nodes before moving hash_builder_updates + let hash_builder_account_nodes = hash_builder_updates.account_nodes.clone(); + + // Write trie updates to the database + let provider_rw = provider_factory.provider_rw().unwrap(); + provider_rw.write_trie_updates(hash_builder_updates).unwrap(); + provider_rw.commit().unwrap(); + + // Assert that the sparse trie root matches the hash builder root + assert_eq!(sparse_root, hash_builder_root); + // Assert that the sparse trie updates match the hash builder updates + pretty_assertions::assert_eq!( + BTreeMap::from_iter(sparse_updates.updated_nodes), + BTreeMap::from_iter(hash_builder_account_nodes) + ); + // Assert that the sparse trie nodes match the hash builder proof nodes + assert_eq_parallel_sparse_trie_proof_nodes( + &updated_sparse, + hash_builder_proof_nodes, + ); + + // Delete some keys from both the hash builder and the sparse trie and check + // that the sparse trie root still matches the hash builder root + for key in &keys_to_delete { + state.remove(key).unwrap(); + sparse.remove_leaf(key, &default_provider).unwrap(); + } + + // We need to clone the sparse trie, so that all updated branch nodes are + // preserved, and not only those that were changed after the last call to + // `root()`. + let mut updated_sparse = sparse.clone(); + let sparse_root = updated_sparse.root(); + let sparse_updates = updated_sparse.take_updates(); + + let provider = provider_factory.provider().unwrap(); + let trie_cursor = DatabaseTrieCursorFactory::new(provider.tx_ref()); + let (hash_builder_root, hash_builder_updates, hash_builder_proof_nodes, _, _) = + run_hash_builder( + state.clone(), + trie_cursor.account_trie_cursor().unwrap(), + keys_to_delete + .iter() + .map(|nibbles| B256::from_slice(&nibbles.pack())) + .collect(), + state.keys().copied(), + ); + + // Extract account nodes before moving hash_builder_updates + let hash_builder_account_nodes = hash_builder_updates.account_nodes.clone(); + + // Write trie updates to the database + let provider_rw = provider_factory.provider_rw().unwrap(); + provider_rw.write_trie_updates(hash_builder_updates).unwrap(); + provider_rw.commit().unwrap(); + + // Assert that the sparse trie root matches the hash builder root + assert_eq!(sparse_root, hash_builder_root); + // Assert that the sparse trie updates match the hash builder updates + pretty_assertions::assert_eq!( + BTreeMap::from_iter(sparse_updates.updated_nodes), + BTreeMap::from_iter(hash_builder_account_nodes) + ); + // Assert that the sparse trie nodes match the hash builder proof nodes + assert_eq_parallel_sparse_trie_proof_nodes( + &updated_sparse, + hash_builder_proof_nodes, + ); + } + } + } + + fn transform_updates( + updates: Vec>, + mut rng: impl rand::Rng, + ) -> Vec<(BTreeMap, BTreeSet)> { + let mut keys = BTreeSet::new(); + updates + .into_iter() + .map(|update| { + keys.extend(update.keys().copied()); + + let keys_to_delete_len = update.len() / 2; + let keys_to_delete = (0..keys_to_delete_len) + .map(|_| { + let key = + *rand::seq::IteratorRandom::choose(keys.iter(), &mut rng).unwrap(); + keys.take(&key).unwrap() + }) + .collect(); + + (update, keys_to_delete) + }) + .collect::>() + } + + proptest!(ProptestConfig::with_cases(10), |( + updates in proptest::collection::vec( + proptest::collection::btree_map( + any_with::(SizeRange::new(KEY_NIBBLES_LEN..=KEY_NIBBLES_LEN)).prop_map(pad_nibbles_right), + arb::(), + 1..50, + ), + 1..50, + ).prop_perturb(transform_updates) + )| { + test(updates) + }); + } + + #[test] + fn sparse_trie_fuzz_vs_serial() { + // Having only the first 3 nibbles set, we narrow down the range of keys + // to 4096 different hashes. It allows us to generate collisions more likely + // to test the sparse trie updates. + const KEY_NIBBLES_LEN: usize = 3; + + fn test(updates: Vec<(BTreeMap, BTreeSet)>) { + let default_provider = DefaultTrieNodeProvider; + let mut serial = SerialSparseTrie::default().with_updates(true); + let mut parallel = ParallelSparseTrie::default().with_updates(true); + + for (update, keys_to_delete) in updates { + // Perform leaf updates on both tries + for (key, account) in update.clone() { + let account = account.into_trie_account(EMPTY_ROOT_HASH); + let mut account_rlp = Vec::new(); + account.encode(&mut account_rlp); + serial.update_leaf(key, account_rlp.clone(), &default_provider).unwrap(); + parallel.update_leaf(key, account_rlp, &default_provider).unwrap(); + } + + // Calculate roots and assert their equality + let serial_root = serial.root(); + let parallel_root = parallel.root(); + assert_eq!(parallel_root, serial_root); + + // Assert that both tries produce the same updates + let serial_updates = serial.take_updates(); + let parallel_updates = parallel.take_updates(); + pretty_assertions::assert_eq!( + BTreeMap::from_iter(parallel_updates.updated_nodes), + BTreeMap::from_iter(serial_updates.updated_nodes), + ); + pretty_assertions::assert_eq!( + BTreeSet::from_iter(parallel_updates.removed_nodes), + BTreeSet::from_iter(serial_updates.removed_nodes), + ); + + // Perform leaf removals on both tries + for key in &keys_to_delete { + parallel.remove_leaf(key, &default_provider).unwrap(); + serial.remove_leaf(key, &default_provider).unwrap(); + } + + // Calculate roots and assert their equality + let serial_root = serial.root(); + let parallel_root = parallel.root(); + assert_eq!(parallel_root, serial_root); + + // Assert that both tries produce the same updates + let serial_updates = serial.take_updates(); + let parallel_updates = parallel.take_updates(); + pretty_assertions::assert_eq!( + BTreeMap::from_iter(parallel_updates.updated_nodes), + BTreeMap::from_iter(serial_updates.updated_nodes), + ); + pretty_assertions::assert_eq!( + BTreeSet::from_iter(parallel_updates.removed_nodes), + BTreeSet::from_iter(serial_updates.removed_nodes), + ); + } + } + + fn transform_updates( + updates: Vec>, + mut rng: impl rand::Rng, + ) -> Vec<(BTreeMap, BTreeSet)> { + let mut keys = BTreeSet::new(); + updates + .into_iter() + .map(|update| { + keys.extend(update.keys().copied()); + + let keys_to_delete_len = update.len() / 2; + let keys_to_delete = (0..keys_to_delete_len) + .map(|_| { + let key = + *rand::seq::IteratorRandom::choose(keys.iter(), &mut rng).unwrap(); + keys.take(&key).unwrap() + }) + .collect(); + + (update, keys_to_delete) + }) + .collect::>() + } + + proptest!(ProptestConfig::with_cases(10), |( + updates in proptest::collection::vec( + proptest::collection::btree_map( + any_with::(SizeRange::new(KEY_NIBBLES_LEN..=KEY_NIBBLES_LEN)).prop_map(pad_nibbles_right), + arb::(), + 1..50, + ), + 1..50, + ).prop_perturb(transform_updates) + )| { + test(updates) + }); + } + + #[test] + fn sparse_trie_two_leaves_at_lower_roots() { + let provider = DefaultTrieNodeProvider; + let mut trie = ParallelSparseTrie::default().with_updates(true); + let key_50 = Nibbles::unpack(hex!( + "0x5000000000000000000000000000000000000000000000000000000000000000" + )); + let key_51 = Nibbles::unpack(hex!( + "0x5100000000000000000000000000000000000000000000000000000000000000" + )); + + let account = Account::default().into_trie_account(EMPTY_ROOT_HASH); + let mut account_rlp = Vec::new(); + account.encode(&mut account_rlp); + + // Add a leaf and calculate the root. + trie.update_leaf(key_50, account_rlp.clone(), &provider).unwrap(); + trie.root(); + + // Add a second leaf and assert that the root is the expected value. + trie.update_leaf(key_51, account_rlp.clone(), &provider).unwrap(); + + let expected_root = + hex!("0xdaf0ef9f91a2f179bb74501209effdb5301db1697bcab041eca2234b126e25de"); + let root = trie.root(); + assert_eq!(root, expected_root); + assert_eq!(SparseTrieUpdates::default(), trie.take_updates()); + } + + /// We have three leaves that share the same prefix: 0x00, 0x01 and 0x02. Hash builder trie has + /// only nodes 0x00 and 0x01, and we have proofs for them. Node B is new and inserted in the + /// sparse trie first. + /// + /// 1. Reveal the hash builder proof to leaf 0x00 in the sparse trie. + /// 2. Insert leaf 0x01 into the sparse trie. + /// 3. Reveal the hash builder proof to leaf 0x02 in the sparse trie. + /// + /// The hash builder proof to the leaf 0x02 didn't have the leaf 0x01 at the corresponding + /// nibble of the branch node, so we need to adjust the branch node instead of fully + /// replacing it. + #[test] + fn sparse_trie_reveal_node_1() { + let key1 = || pad_nibbles_right(Nibbles::from_nibbles_unchecked([0x00])); + let key2 = || pad_nibbles_right(Nibbles::from_nibbles_unchecked([0x01])); + let key3 = || pad_nibbles_right(Nibbles::from_nibbles_unchecked([0x02])); + let value = || Account::default(); + let value_encoded = || { + let mut account_rlp = Vec::new(); + value().into_trie_account(EMPTY_ROOT_HASH).encode(&mut account_rlp); + account_rlp + }; + + // Generate the proof for the root node and initialize the sparse trie with it + let (_, _, hash_builder_proof_nodes, branch_node_hash_masks, branch_node_tree_masks) = + run_hash_builder( + [(key1(), value()), (key3(), value())], + NoopAccountTrieCursor::default(), + Default::default(), + [Nibbles::default()], + ); + + let provider = DefaultTrieNodeProvider; + let mut sparse = ParallelSparseTrie::from_root( + TrieNode::decode(&mut &hash_builder_proof_nodes.nodes_sorted()[0].1[..]).unwrap(), + TrieMasks { + hash_mask: branch_node_hash_masks.get(&Nibbles::default()).copied(), + tree_mask: branch_node_tree_masks.get(&Nibbles::default()).copied(), + }, + false, + ) + .unwrap(); + + // Generate the proof for the first key and reveal it in the sparse trie + let (_, _, hash_builder_proof_nodes, branch_node_hash_masks, branch_node_tree_masks) = + run_hash_builder( + [(key1(), value()), (key3(), value())], + NoopAccountTrieCursor::default(), + Default::default(), + [key1()], + ); + let revealed_nodes: Vec = hash_builder_proof_nodes + .nodes_sorted() + .into_iter() + .map(|(path, node)| { + let hash_mask = branch_node_hash_masks.get(&path).copied(); + let tree_mask = branch_node_tree_masks.get(&path).copied(); + RevealedSparseNode { + path, + node: TrieNode::decode(&mut &node[..]).unwrap(), + masks: TrieMasks { hash_mask, tree_mask }, + } + }) + .collect(); + sparse.reveal_nodes(revealed_nodes).unwrap(); + + // Check that the branch node exists with only two nibbles set + assert_eq!( + sparse.upper_subtrie.nodes.get(&Nibbles::default()), + Some(&SparseNode::new_branch(0b101.into())) + ); + + // Insert the leaf for the second key + sparse.update_leaf(key2(), value_encoded(), &provider).unwrap(); + + // Check that the branch node was updated and another nibble was set + assert_eq!( + sparse.upper_subtrie.nodes.get(&Nibbles::default()), + Some(&SparseNode::new_branch(0b111.into())) + ); + + // Generate the proof for the third key and reveal it in the sparse trie + let (_, _, hash_builder_proof_nodes, branch_node_hash_masks, branch_node_tree_masks) = + run_hash_builder( + [(key1(), value()), (key3(), value())], + NoopAccountTrieCursor::default(), + Default::default(), + [key3()], + ); + let revealed_nodes: Vec = hash_builder_proof_nodes + .nodes_sorted() + .into_iter() + .map(|(path, node)| { + let hash_mask = branch_node_hash_masks.get(&path).copied(); + let tree_mask = branch_node_tree_masks.get(&path).copied(); + RevealedSparseNode { + path, + node: TrieNode::decode(&mut &node[..]).unwrap(), + masks: TrieMasks { hash_mask, tree_mask }, + } + }) + .collect(); + sparse.reveal_nodes(revealed_nodes).unwrap(); + + // Check that nothing changed in the branch node + assert_eq!( + sparse.upper_subtrie.nodes.get(&Nibbles::default()), + Some(&SparseNode::new_branch(0b111.into())) + ); + + // Generate the nodes for the full trie with all three key using the hash builder, and + // compare them to the sparse trie + let (_, _, hash_builder_proof_nodes, _, _) = run_hash_builder( + [(key1(), value()), (key2(), value()), (key3(), value())], + NoopAccountTrieCursor::default(), + Default::default(), + [key1(), key2(), key3()], + ); + + assert_eq_parallel_sparse_trie_proof_nodes(&sparse, hash_builder_proof_nodes); + } + + /// We have three leaves: 0x0000, 0x0101, and 0x0102. Hash builder trie has all nodes, and we + /// have proofs for them. + /// + /// 1. Reveal the hash builder proof to leaf 0x00 in the sparse trie. + /// 2. Remove leaf 0x00 from the sparse trie (that will remove the branch node and create an + /// extension node with the key 0x0000). + /// 3. Reveal the hash builder proof to leaf 0x0101 in the sparse trie. + /// + /// The hash builder proof to the leaf 0x0101 had a branch node in the path, but we turned it + /// into an extension node, so it should ignore this node. + #[test] + fn sparse_trie_reveal_node_2() { + let key1 = || pad_nibbles_right(Nibbles::from_nibbles_unchecked([0x00, 0x00])); + let key2 = || pad_nibbles_right(Nibbles::from_nibbles_unchecked([0x01, 0x01])); + let key3 = || pad_nibbles_right(Nibbles::from_nibbles_unchecked([0x01, 0x02])); + let value = || Account::default(); + + // Generate the proof for the root node and initialize the sparse trie with it + let (_, _, hash_builder_proof_nodes, branch_node_hash_masks, branch_node_tree_masks) = + run_hash_builder( + [(key1(), value()), (key2(), value()), (key3(), value())], + NoopAccountTrieCursor::default(), + Default::default(), + [Nibbles::default()], + ); + + let provider = DefaultTrieNodeProvider; + let mut sparse = ParallelSparseTrie::from_root( + TrieNode::decode(&mut &hash_builder_proof_nodes.nodes_sorted()[0].1[..]).unwrap(), + TrieMasks { + hash_mask: branch_node_hash_masks.get(&Nibbles::default()).copied(), + tree_mask: branch_node_tree_masks.get(&Nibbles::default()).copied(), + }, + false, + ) + .unwrap(); + + // Generate the proof for the children of the root branch node and reveal it in the sparse + // trie + let (_, _, hash_builder_proof_nodes, branch_node_hash_masks, branch_node_tree_masks) = + run_hash_builder( + [(key1(), value()), (key2(), value()), (key3(), value())], + NoopAccountTrieCursor::default(), + Default::default(), + [key1(), Nibbles::from_nibbles_unchecked([0x01])], + ); + let revealed_nodes: Vec = hash_builder_proof_nodes + .nodes_sorted() + .into_iter() + .map(|(path, node)| { + let hash_mask = branch_node_hash_masks.get(&path).copied(); + let tree_mask = branch_node_tree_masks.get(&path).copied(); + RevealedSparseNode { + path, + node: TrieNode::decode(&mut &node[..]).unwrap(), + masks: TrieMasks { hash_mask, tree_mask }, + } + }) + .collect(); + sparse.reveal_nodes(revealed_nodes).unwrap(); + + // Check that the branch node exists + assert_eq!( + sparse.upper_subtrie.nodes.get(&Nibbles::default()), + Some(&SparseNode::new_branch(0b11.into())) + ); + + // Remove the leaf for the first key + sparse.remove_leaf(&key1(), &provider).unwrap(); + + // Check that the branch node was turned into an extension node + assert_eq!( + sparse.upper_subtrie.nodes.get(&Nibbles::default()), + Some(&SparseNode::new_ext(Nibbles::from_nibbles_unchecked([0x01]))) + ); + + // Generate the proof for the third key and reveal it in the sparse trie + let (_, _, hash_builder_proof_nodes, branch_node_hash_masks, branch_node_tree_masks) = + run_hash_builder( + [(key1(), value()), (key2(), value()), (key3(), value())], + NoopAccountTrieCursor::default(), + Default::default(), + [key2()], + ); + let revealed_nodes: Vec = hash_builder_proof_nodes + .nodes_sorted() + .into_iter() + .map(|(path, node)| { + let hash_mask = branch_node_hash_masks.get(&path).copied(); + let tree_mask = branch_node_tree_masks.get(&path).copied(); + RevealedSparseNode { + path, + node: TrieNode::decode(&mut &node[..]).unwrap(), + masks: TrieMasks { hash_mask, tree_mask }, + } + }) + .collect(); + sparse.reveal_nodes(revealed_nodes).unwrap(); + + // Check that nothing changed in the extension node + assert_eq!( + sparse.upper_subtrie.nodes.get(&Nibbles::default()), + Some(&SparseNode::new_ext(Nibbles::from_nibbles_unchecked([0x01]))) + ); + } + + /// We have two leaves that share the same prefix: 0x0001 and 0x0002, and a leaf with a + /// different prefix: 0x0100. Hash builder trie has only the first two leaves, and we have + /// proofs for them. + /// + /// 1. Insert the leaf 0x0100 into the sparse trie, and check that the root extension node was + /// turned into a branch node. + /// 2. Reveal the leaf 0x0001 in the sparse trie, and check that the root branch node wasn't + /// overwritten with the extension node from the proof. + #[test] + fn sparse_trie_reveal_node_3() { + let key1 = || pad_nibbles_right(Nibbles::from_nibbles_unchecked([0x00, 0x01])); + let key2 = || pad_nibbles_right(Nibbles::from_nibbles_unchecked([0x00, 0x02])); + let key3 = || pad_nibbles_right(Nibbles::from_nibbles_unchecked([0x01, 0x00])); + let value = || Account::default(); + let value_encoded = || { + let mut account_rlp = Vec::new(); + value().into_trie_account(EMPTY_ROOT_HASH).encode(&mut account_rlp); + account_rlp + }; + + // Generate the proof for the root node and initialize the sparse trie with it + let (_, _, hash_builder_proof_nodes, branch_node_hash_masks, branch_node_tree_masks) = + run_hash_builder( + [(key1(), value()), (key2(), value())], + NoopAccountTrieCursor::default(), + Default::default(), + [Nibbles::default()], + ); + + let provider = DefaultTrieNodeProvider; + let mut sparse = ParallelSparseTrie::from_root( + TrieNode::decode(&mut &hash_builder_proof_nodes.nodes_sorted()[0].1[..]).unwrap(), + TrieMasks { + hash_mask: branch_node_hash_masks.get(&Nibbles::default()).copied(), + tree_mask: branch_node_tree_masks.get(&Nibbles::default()).copied(), + }, + false, + ) + .unwrap(); + + // Check that the root extension node exists + assert_matches!( + sparse.upper_subtrie.nodes.get(&Nibbles::default()), + Some(SparseNode::Extension { key, hash: None, store_in_db_trie: None }) if *key == Nibbles::from_nibbles([0x00]) + ); + + // Insert the leaf with a different prefix + sparse.update_leaf(key3(), value_encoded(), &provider).unwrap(); + + // Check that the extension node was turned into a branch node + assert_matches!( + sparse.upper_subtrie.nodes.get(&Nibbles::default()), + Some(SparseNode::Branch { state_mask, hash: None, store_in_db_trie: None }) if *state_mask == TrieMask::new(0b11) + ); + + // Generate the proof for the first key and reveal it in the sparse trie + let (_, _, hash_builder_proof_nodes, branch_node_hash_masks, branch_node_tree_masks) = + run_hash_builder( + [(key1(), value()), (key2(), value())], + NoopAccountTrieCursor::default(), + Default::default(), + [key1()], + ); + let revealed_nodes: Vec = hash_builder_proof_nodes + .nodes_sorted() + .into_iter() + .map(|(path, node)| { + let hash_mask = branch_node_hash_masks.get(&path).copied(); + let tree_mask = branch_node_tree_masks.get(&path).copied(); + RevealedSparseNode { + path, + node: TrieNode::decode(&mut &node[..]).unwrap(), + masks: TrieMasks { hash_mask, tree_mask }, + } + }) + .collect(); + sparse.reveal_nodes(revealed_nodes).unwrap(); + + // Check that the branch node wasn't overwritten by the extension node in the proof + assert_matches!( + sparse.upper_subtrie.nodes.get(&Nibbles::default()), + Some(SparseNode::Branch { state_mask, hash: None, store_in_db_trie: None }) if *state_mask == TrieMask::new(0b11) + ); + } + + #[test] + fn test_update_leaf_cross_level() { + let ctx = ParallelSparseTrieTestContext; + let mut trie = + ParallelSparseTrie::from_root(TrieNode::EmptyRoot, TrieMasks::none(), true).unwrap(); + + // Test adding leaves that demonstrate the cross-level behavior + // Based on the example: leaves 0x1234, 0x1245, 0x1334, 0x1345 + // + // Final trie structure: + // Upper trie: + // 0x: Extension { key: 0x1 } + // └── 0x1: Branch { state_mask: 0x1100 } + // └── Subtrie (0x12): pointer to lower subtrie + // └── Subtrie (0x13): pointer to lower subtrie + // + // Lower subtrie (0x12): + // 0x12: Branch { state_mask: 0x8 | 0x10 } + // ├── 0x123: Leaf { key: 0x4 } + // └── 0x124: Leaf { key: 0x5 } + // + // Lower subtrie (0x13): + // 0x13: Branch { state_mask: 0x8 | 0x10 } + // ├── 0x133: Leaf { key: 0x4 } + // └── 0x134: Leaf { key: 0x5 } + + // First add leaf 0x1345 - this should create a leaf in upper trie at 0x + let (leaf1_path, value1) = ctx.create_test_leaf([0x1, 0x3, 0x4, 0x5], 1); + trie.update_leaf(leaf1_path, value1.clone(), DefaultTrieNodeProvider).unwrap(); + + // Verify upper trie has a leaf at the root with key 1345 + ctx.assert_upper_subtrie(&trie) + .has_leaf(&Nibbles::default(), &Nibbles::from_nibbles([0x1, 0x3, 0x4, 0x5])) + .has_value(&leaf1_path, &value1); + + // Add leaf 0x1234 - this should go first in the upper subtrie + let (leaf2_path, value2) = ctx.create_test_leaf([0x1, 0x2, 0x3, 0x4], 2); + trie.update_leaf(leaf2_path, value2.clone(), DefaultTrieNodeProvider).unwrap(); + + // Upper trie should now have a branch at 0x1 + ctx.assert_upper_subtrie(&trie) + .has_branch(&Nibbles::from_nibbles([0x1]), &[0x2, 0x3]) + .has_no_value(&leaf1_path) + .has_no_value(&leaf2_path); + + // Add leaf 0x1245 - this should cause a branch and create the 0x12 subtrie + let (leaf3_path, value3) = ctx.create_test_leaf([0x1, 0x2, 0x4, 0x5], 3); + trie.update_leaf(leaf3_path, value3.clone(), DefaultTrieNodeProvider).unwrap(); + + // Verify lower subtrie at 0x12 exists with correct structure + ctx.assert_subtrie(&trie, Nibbles::from_nibbles([0x1, 0x2])) + .has_branch(&Nibbles::from_nibbles([0x1, 0x2]), &[0x3, 0x4]) + .has_leaf(&Nibbles::from_nibbles([0x1, 0x2, 0x3]), &Nibbles::from_nibbles([0x4])) + .has_leaf(&Nibbles::from_nibbles([0x1, 0x2, 0x4]), &Nibbles::from_nibbles([0x5])) + .has_value(&leaf2_path, &value2) + .has_value(&leaf3_path, &value3); + + // Add leaf 0x1334 - this should create another lower subtrie + let (leaf4_path, value4) = ctx.create_test_leaf([0x1, 0x3, 0x3, 0x4], 4); + trie.update_leaf(leaf4_path, value4.clone(), DefaultTrieNodeProvider).unwrap(); + + // Verify lower subtrie at 0x13 exists with correct values + ctx.assert_subtrie(&trie, Nibbles::from_nibbles([0x1, 0x3])) + .has_value(&leaf1_path, &value1) + .has_value(&leaf4_path, &value4); + + // Verify the 0x12 subtrie still has its values + ctx.assert_subtrie(&trie, Nibbles::from_nibbles([0x1, 0x2])) + .has_value(&leaf2_path, &value2) + .has_value(&leaf3_path, &value3); + + // Upper trie has no values + ctx.assert_upper_subtrie(&trie) + .has_extension(&Nibbles::default(), &Nibbles::from_nibbles([0x1])) + .has_branch(&Nibbles::from_nibbles([0x1]), &[0x2, 0x3]) + .has_no_value(&leaf1_path) + .has_no_value(&leaf2_path) + .has_no_value(&leaf3_path) + .has_no_value(&leaf4_path); + } + + #[test] + fn test_update_leaf_split_at_level_boundary() { + let ctx = ParallelSparseTrieTestContext; + let mut trie = + ParallelSparseTrie::from_root(TrieNode::EmptyRoot, TrieMasks::none(), true).unwrap(); + + // This test demonstrates what happens when we insert leaves that cause + // splitting exactly at the upper/lower trie boundary (2 nibbles). + // + // Final trie structure: + // Upper trie: + // 0x: Extension { key: 0x12 } + // └── Subtrie (0x12): pointer to lower subtrie + // + // Lower subtrie (0x12): + // 0x12: Branch { state_mask: 0x4 | 0x8 } + // ├── 0x122: Leaf { key: 0x4 } + // └── 0x123: Leaf { key: 0x4 } + + // First insert a leaf that ends exactly at the boundary (2 nibbles) + let (first_leaf_path, first_value) = ctx.create_test_leaf([0x1, 0x2, 0x2, 0x4], 1); + + trie.update_leaf(first_leaf_path, first_value.clone(), DefaultTrieNodeProvider).unwrap(); + + // In an empty trie, the first leaf becomes the root, regardless of path length + ctx.assert_upper_subtrie(&trie) + .has_leaf(&Nibbles::default(), &Nibbles::from_nibbles([0x1, 0x2, 0x2, 0x4])) + .has_value(&first_leaf_path, &first_value); + + // Now insert another leaf that shares the same 2-nibble prefix + let (second_leaf_path, second_value) = ctx.create_test_leaf([0x1, 0x2, 0x3, 0x4], 2); + + trie.update_leaf(second_leaf_path, second_value.clone(), DefaultTrieNodeProvider).unwrap(); + + // Now both leaves should be in a lower subtrie at index [0x1, 0x2] + ctx.assert_subtrie(&trie, Nibbles::from_nibbles([0x1, 0x2])) + .has_branch(&Nibbles::from_nibbles([0x1, 0x2]), &[0x2, 0x3]) + .has_leaf(&Nibbles::from_nibbles([0x1, 0x2, 0x2]), &Nibbles::from_nibbles([0x4])) + .has_leaf(&Nibbles::from_nibbles([0x1, 0x2, 0x3]), &Nibbles::from_nibbles([0x4])) + .has_value(&first_leaf_path, &first_value) + .has_value(&second_leaf_path, &second_value); + + // Upper subtrie should no longer have these values + ctx.assert_upper_subtrie(&trie) + .has_no_value(&first_leaf_path) + .has_no_value(&second_leaf_path); + } + + #[test] + fn test_update_subtrie_with_multiple_leaves() { + let ctx = ParallelSparseTrieTestContext; + let mut trie = + ParallelSparseTrie::from_root(TrieNode::EmptyRoot, TrieMasks::none(), true).unwrap(); + + // First, add multiple leaves that will create a subtrie structure + // All leaves share the prefix [0x1, 0x2] to ensure they create a subtrie + // + // This should result in a trie with the following structure: + // 0x: Extension { key: 0x12 } + // └── Subtrie (0x12): + // 0x12: Branch { state_mask: 0x3 | 0x4 } + // ├── 0x123: Branch { state_mask: 0x4 | 0x5 } + // │ ├── 0x1234: Leaf { key: 0x } + // │ └── 0x1235: Leaf { key: 0x } + // └── 0x124: Branch { state_mask: 0x6 | 0x7 } + // ├── 0x1246: Leaf { key: 0x } + // └── 0x1247: Leaf { key: 0x } + let leaves = ctx.create_test_leaves(&[ + &[0x1, 0x2, 0x3, 0x4], + &[0x1, 0x2, 0x3, 0x5], + &[0x1, 0x2, 0x4, 0x6], + &[0x1, 0x2, 0x4, 0x7], + ]); + + // Insert all leaves + ctx.update_leaves(&mut trie, leaves.clone()); + + // Verify the upper subtrie has an extension node at the root with key 0x12 + ctx.assert_upper_subtrie(&trie) + .has_extension(&Nibbles::default(), &Nibbles::from_nibbles([0x1, 0x2])); + + // Verify the subtrie structure using fluent assertions + ctx.assert_subtrie(&trie, Nibbles::from_nibbles([0x1, 0x2])) + .has_branch(&Nibbles::from_nibbles([0x1, 0x2]), &[0x3, 0x4]) + .has_branch(&Nibbles::from_nibbles([0x1, 0x2, 0x3]), &[0x4, 0x5]) + .has_branch(&Nibbles::from_nibbles([0x1, 0x2, 0x4]), &[0x6, 0x7]) + .has_value(&leaves[0].0, &leaves[0].1) + .has_value(&leaves[1].0, &leaves[1].1) + .has_value(&leaves[2].0, &leaves[2].1) + .has_value(&leaves[3].0, &leaves[3].1); + + // Now update one of the leaves with a new value + let updated_path = Nibbles::from_nibbles([0x1, 0x2, 0x3, 0x4]); + let (_, updated_value) = ctx.create_test_leaf([0x1, 0x2, 0x3, 0x4], 100); + + trie.update_leaf(updated_path, updated_value.clone(), DefaultTrieNodeProvider).unwrap(); + + // Verify the subtrie structure is maintained and value is updated + // The branch structure should remain the same and all values should be present + ctx.assert_subtrie(&trie, Nibbles::from_nibbles([0x1, 0x2])) + .has_branch(&Nibbles::from_nibbles([0x1, 0x2]), &[0x3, 0x4]) + .has_value(&updated_path, &updated_value) + .has_value(&leaves[1].0, &leaves[1].1) + .has_value(&leaves[2].0, &leaves[2].1) + .has_value(&leaves[3].0, &leaves[3].1); + + // Add a new leaf that extends an existing branch + let (new_leaf_path, new_leaf_value) = ctx.create_test_leaf([0x1, 0x2, 0x3, 0x6], 200); + + trie.update_leaf(new_leaf_path, new_leaf_value.clone(), DefaultTrieNodeProvider).unwrap(); + + // Verify the branch at [0x1, 0x2, 0x3] now has an additional child + ctx.assert_subtrie(&trie, Nibbles::from_nibbles([0x1, 0x2])) + .has_branch(&Nibbles::from_nibbles([0x1, 0x2, 0x3]), &[0x4, 0x5, 0x6]) + .has_value(&new_leaf_path, &new_leaf_value); + } + + #[test] + fn test_update_subtrie_extension_node_subtrie() { + let ctx = ParallelSparseTrieTestContext; + let mut trie = + ParallelSparseTrie::from_root(TrieNode::EmptyRoot, TrieMasks::none(), true).unwrap(); + + // All leaves share the prefix [0x1, 0x2] to ensure they create a subtrie + // + // This should result in a trie with the following structure + // 0x: Extension { key: 0x123 } + // └── Subtrie (0x12): + // 0x123: Branch { state_mask: 0x3 | 0x4 } + // ├── 0x123: Leaf { key: 0x4 } + // └── 0x124: Leaf { key: 0x5 } + let leaves = ctx.create_test_leaves(&[&[0x1, 0x2, 0x3, 0x4], &[0x1, 0x2, 0x3, 0x5]]); + + // Insert all leaves + ctx.update_leaves(&mut trie, leaves.clone()); + + // Verify the upper subtrie has an extension node at the root with key 0x123 + ctx.assert_upper_subtrie(&trie) + .has_extension(&Nibbles::default(), &Nibbles::from_nibbles([0x1, 0x2, 0x3])); + + // Verify the lower subtrie structure + ctx.assert_subtrie(&trie, Nibbles::from_nibbles([0x1, 0x2])) + .has_branch(&Nibbles::from_nibbles([0x1, 0x2, 0x3]), &[0x4, 0x5]) + .has_value(&leaves[0].0, &leaves[0].1) + .has_value(&leaves[1].0, &leaves[1].1); + } + + #[test] + fn update_subtrie_extension_node_cross_level() { + let ctx = ParallelSparseTrieTestContext; + let mut trie = + ParallelSparseTrie::from_root(TrieNode::EmptyRoot, TrieMasks::none(), true).unwrap(); + + // First, add multiple leaves that will create a subtrie structure + // All leaves share the prefix [0x1, 0x2] to ensure they create a branch node and subtrie + // + // This should result in a trie with the following structure + // 0x: Extension { key: 0x12 } + // └── Subtrie (0x12): + // 0x12: Branch { state_mask: 0x3 | 0x4 } + // ├── 0x123: Leaf { key: 0x4 } + // └── 0x124: Leaf { key: 0x5 } + let leaves = ctx.create_test_leaves(&[&[0x1, 0x2, 0x3, 0x4], &[0x1, 0x2, 0x4, 0x5]]); + + // Insert all leaves + ctx.update_leaves(&mut trie, leaves.clone()); + + // Verify the upper subtrie has an extension node at the root with key 0x12 + ctx.assert_upper_subtrie(&trie) + .has_extension(&Nibbles::default(), &Nibbles::from_nibbles([0x1, 0x2])); + + // Verify the lower subtrie structure + ctx.assert_subtrie(&trie, Nibbles::from_nibbles([0x1, 0x2])) + .has_branch(&Nibbles::from_nibbles([0x1, 0x2]), &[0x3, 0x4]) + .has_leaf(&Nibbles::from_nibbles([0x1, 0x2, 0x3]), &Nibbles::from_nibbles([0x4])) + .has_leaf(&Nibbles::from_nibbles([0x1, 0x2, 0x4]), &Nibbles::from_nibbles([0x5])) + .has_value(&leaves[0].0, &leaves[0].1) + .has_value(&leaves[1].0, &leaves[1].1); + } + + #[test] + fn test_update_single_nibble_paths() { + let ctx = ParallelSparseTrieTestContext; + let mut trie = + ParallelSparseTrie::from_root(TrieNode::EmptyRoot, TrieMasks::none(), true).unwrap(); + + // Test edge case: single nibble paths that create branches in upper trie + // + // Final trie structure: + // Upper trie: + // 0x: Branch { state_mask: 0x1 | 0x2 | 0x4 | 0x8 } + // ├── 0x0: Leaf { key: 0x } + // ├── 0x1: Leaf { key: 0x } + // ├── 0x2: Leaf { key: 0x } + // └── 0x3: Leaf { key: 0x } + + // Insert leaves with single nibble paths + let (leaf1_path, value1) = ctx.create_test_leaf([0x0], 1); + let (leaf2_path, value2) = ctx.create_test_leaf([0x1], 2); + let (leaf3_path, value3) = ctx.create_test_leaf([0x2], 3); + let (leaf4_path, value4) = ctx.create_test_leaf([0x3], 4); + + ctx.update_leaves( + &mut trie, + [ + (leaf1_path, value1.clone()), + (leaf2_path, value2.clone()), + (leaf3_path, value3.clone()), + (leaf4_path, value4.clone()), + ], + ); + + // Verify upper trie has a branch at root with 4 children + ctx.assert_upper_subtrie(&trie) + .has_branch(&Nibbles::default(), &[0x0, 0x1, 0x2, 0x3]) + .has_leaf(&Nibbles::from_nibbles([0x0]), &Nibbles::default()) + .has_leaf(&Nibbles::from_nibbles([0x1]), &Nibbles::default()) + .has_leaf(&Nibbles::from_nibbles([0x2]), &Nibbles::default()) + .has_leaf(&Nibbles::from_nibbles([0x3]), &Nibbles::default()) + .has_value(&leaf1_path, &value1) + .has_value(&leaf2_path, &value2) + .has_value(&leaf3_path, &value3) + .has_value(&leaf4_path, &value4); + } + + #[test] + fn test_update_deep_extension_chain() { + let ctx = ParallelSparseTrieTestContext; + let mut trie = + ParallelSparseTrie::from_root(TrieNode::EmptyRoot, TrieMasks::none(), true).unwrap(); + + // Test edge case: deep extension chains that span multiple levels + // + // Final trie structure: + // Upper trie: + // 0x: Extension { key: 0x111111 } + // └── Subtrie (0x11): pointer to lower subtrie + // + // Lower subtrie (0x11): + // 0x111111: Branch { state_mask: 0x1 | 0x2 } + // ├── 0x1111110: Leaf { key: 0x } + // └── 0x1111111: Leaf { key: 0x } + + // Create leaves with a long common prefix + let (leaf1_path, value1) = ctx.create_test_leaf([0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x0], 1); + let (leaf2_path, value2) = ctx.create_test_leaf([0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1], 2); + + ctx.update_leaves(&mut trie, [(leaf1_path, value1.clone()), (leaf2_path, value2.clone())]); + + // Verify upper trie has extension with the full common prefix + ctx.assert_upper_subtrie(&trie).has_extension( + &Nibbles::default(), + &Nibbles::from_nibbles([0x1, 0x1, 0x1, 0x1, 0x1, 0x1]), + ); + + // Verify lower subtrie has branch structure + ctx.assert_subtrie(&trie, Nibbles::from_nibbles([0x1, 0x1])) + .has_branch(&Nibbles::from_nibbles([0x1, 0x1, 0x1, 0x1, 0x1, 0x1]), &[0x0, 0x1]) + .has_leaf( + &Nibbles::from_nibbles([0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x0]), + &Nibbles::default(), + ) + .has_leaf( + &Nibbles::from_nibbles([0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1]), + &Nibbles::default(), + ) + .has_value(&leaf1_path, &value1) + .has_value(&leaf2_path, &value2); + } + + #[test] + fn test_update_branch_with_all_nibbles() { + let ctx = ParallelSparseTrieTestContext; + let mut trie = + ParallelSparseTrie::from_root(TrieNode::EmptyRoot, TrieMasks::none(), true).unwrap(); + + // Test edge case: branch node with all 16 possible nibble children + // + // Final trie structure: + // Upper trie: + // 0x: Extension { key: 0xA } + // └── Subtrie (0xA0): pointer to lower subtrie + // + // Lower subtrie (0xA0): + // 0xA0: Branch { state_mask: 0xFFFF } (all 16 children) + // ├── 0xA00: Leaf { key: 0x } + // ├── 0xA01: Leaf { key: 0x } + // ├── 0xA02: Leaf { key: 0x } + // ... (all nibbles 0x0 through 0xF) + // └── 0xA0F: Leaf { key: 0x } + + // Create leaves for all 16 possible nibbles + let mut leaves = Vec::new(); + for nibble in 0x0..=0xF { + let (path, value) = ctx.create_test_leaf([0xA, 0x0, nibble], nibble as u64 + 1); + leaves.push((path, value)); + } + + // Insert all leaves + ctx.update_leaves(&mut trie, leaves.iter().cloned()); + + // Verify upper trie structure + ctx.assert_upper_subtrie(&trie) + .has_extension(&Nibbles::default(), &Nibbles::from_nibbles([0xA, 0x0])); + + // Verify lower subtrie has branch with all 16 children + let mut subtrie_assert = + ctx.assert_subtrie(&trie, Nibbles::from_nibbles([0xA, 0x0])).has_branch( + &Nibbles::from_nibbles([0xA, 0x0]), + &[0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xA, 0xB, 0xC, 0xD, 0xE, 0xF], + ); + + // Verify all leaves exist + for (i, (path, value)) in leaves.iter().enumerate() { + subtrie_assert = subtrie_assert + .has_leaf(&Nibbles::from_nibbles([0xA, 0x0, i as u8]), &Nibbles::default()) + .has_value(path, value); + } + } + + #[test] + fn test_update_creates_multiple_subtries() { + let ctx = ParallelSparseTrieTestContext; + let mut trie = + ParallelSparseTrie::from_root(TrieNode::EmptyRoot, TrieMasks::none(), true).unwrap(); + + // Test edge case: updates that create multiple subtries at once + // + // Final trie structure: + // Upper trie: + // 0x: Extension { key: 0x0 } + // └── 0x0: Branch { state_mask: 0xF } + // ├── Subtrie (0x00): pointer + // ├── Subtrie (0x01): pointer + // ├── Subtrie (0x02): pointer + // └── Subtrie (0x03): pointer + // + // Each lower subtrie has leaves: + // 0xXY: Leaf { key: 0xZ... } + + // Create leaves that will force multiple subtries + let leaves = [ + ctx.create_test_leaf([0x0, 0x0, 0x1, 0x2], 1), + ctx.create_test_leaf([0x0, 0x1, 0x3, 0x4], 2), + ctx.create_test_leaf([0x0, 0x2, 0x5, 0x6], 3), + ctx.create_test_leaf([0x0, 0x3, 0x7, 0x8], 4), + ]; + + // Insert all leaves + ctx.update_leaves(&mut trie, leaves.iter().cloned()); + + // Verify upper trie has extension then branch + ctx.assert_upper_subtrie(&trie) + .has_extension(&Nibbles::default(), &Nibbles::from_nibbles([0x0])) + .has_branch(&Nibbles::from_nibbles([0x0]), &[0x0, 0x1, 0x2, 0x3]); + + // Verify each subtrie exists and contains its leaf + for (i, (leaf_path, leaf_value)) in leaves.iter().enumerate() { + let subtrie_path = Nibbles::from_nibbles([0x0, i as u8]); + ctx.assert_subtrie(&trie, subtrie_path) + .has_leaf( + &subtrie_path, + &Nibbles::from_nibbles(match i { + 0 => vec![0x1, 0x2], + 1 => vec![0x3, 0x4], + 2 => vec![0x5, 0x6], + 3 => vec![0x7, 0x8], + _ => unreachable!(), + }), + ) + .has_value(leaf_path, leaf_value); + } + } + + #[test] + fn test_update_extension_to_branch_transformation() { + let ctx = ParallelSparseTrieTestContext; + let mut trie = + ParallelSparseTrie::from_root(TrieNode::EmptyRoot, TrieMasks::none(), true).unwrap(); + + // Test edge case: extension node transforms to branch when split + // + // Initial state after first two leaves: + // Upper trie: + // 0x: Extension { key: 0xFF0 } + // └── Subtrie (0xFF): pointer + // + // After third leaf (0xF0...): + // Upper trie: + // 0x: Extension { key: 0xF } + // └── 0xF: Branch { state_mask: 0x10 | 0x8000 } + // ├── Subtrie (0xF0): pointer + // └── Subtrie (0xFF): pointer + + // First two leaves share prefix 0xFF0 + let (leaf1_path, value1) = ctx.create_test_leaf([0xF, 0xF, 0x0, 0x1], 1); + let (leaf2_path, value2) = ctx.create_test_leaf([0xF, 0xF, 0x0, 0x2], 2); + let (leaf3_path, value3) = ctx.create_test_leaf([0xF, 0x0, 0x0, 0x3], 3); + + ctx.update_leaves(&mut trie, [(leaf1_path, value1.clone()), (leaf2_path, value2.clone())]); + + // Verify initial extension structure + ctx.assert_upper_subtrie(&trie) + .has_extension(&Nibbles::default(), &Nibbles::from_nibbles([0xF, 0xF, 0x0])); + + // Add leaf that splits the extension + ctx.update_leaves(&mut trie, [(leaf3_path, value3.clone())]); + + // Verify transformed structure + ctx.assert_upper_subtrie(&trie) + .has_extension(&Nibbles::default(), &Nibbles::from_nibbles([0xF])) + .has_branch(&Nibbles::from_nibbles([0xF]), &[0x0, 0xF]); + + // Verify subtries + ctx.assert_subtrie(&trie, Nibbles::from_nibbles([0xF, 0xF])) + .has_branch(&Nibbles::from_nibbles([0xF, 0xF, 0x0]), &[0x1, 0x2]) + .has_leaf(&Nibbles::from_nibbles([0xF, 0xF, 0x0, 0x1]), &Nibbles::default()) + .has_leaf(&Nibbles::from_nibbles([0xF, 0xF, 0x0, 0x2]), &Nibbles::default()) + .has_value(&leaf1_path, &value1) + .has_value(&leaf2_path, &value2); + + ctx.assert_subtrie(&trie, Nibbles::from_nibbles([0xF, 0x0])) + .has_leaf(&Nibbles::from_nibbles([0xF, 0x0]), &Nibbles::from_nibbles([0x0, 0x3])) + .has_value(&leaf3_path, &value3); + } + + #[test] + fn test_update_upper_extension_reveal_lower_hash_node() { + let ctx = ParallelSparseTrieTestContext; + + // Test edge case: extension pointing to hash node that gets updated to branch + // and reveals the hash node from lower trie + // + // Setup: + // Upper trie: + // 0x: Extension { key: 0xAB } + // └── Subtrie (0xAB): pointer + // Lower trie (0xAB): + // 0xAB: Hash + // + // After update: + // Upper trie: + // 0x: Extension { key: 0xA } + // └── 0xA: Branch { state_mask: 0b100000000001 } + // ├── 0xA0: Leaf { value: ... } + // └── 0xAB: pointer + // Lower trie (0xAB): + // 0xAB: Branch { state_mask: 0b11 } + // ├── 0xAB1: Hash + // └── 0xAB2: Hash + + // Create a mock provider that will provide the hash node + let mut provider = MockTrieNodeProvider::new(); + + // Create revealed branch which will get revealed and add it to the mock provider + let child_hashes = [ + RlpNode::word_rlp(&B256::repeat_byte(0x11)), + RlpNode::word_rlp(&B256::repeat_byte(0x22)), + ]; + let revealed_branch = create_branch_node_with_children(&[0x1, 0x2], child_hashes); + let mut encoded = Vec::new(); + revealed_branch.encode(&mut encoded); + provider.add_revealed_node( + Nibbles::from_nibbles([0xA, 0xB]), + RevealedNode { node: encoded.into(), tree_mask: None, hash_mask: None }, + ); + + let mut trie = new_test_trie( + [ + (Nibbles::default(), SparseNode::new_ext(Nibbles::from_nibbles([0xA, 0xB]))), + (Nibbles::from_nibbles([0xA, 0xB]), SparseNode::Hash(B256::repeat_byte(0x42))), + ] + .into_iter(), + ); + + // Now add a leaf that will force the hash node to become a branch + let (leaf_path, value) = ctx.create_test_leaf([0xA, 0x0], 1); + trie.update_leaf(leaf_path, value, provider).unwrap(); + + // Verify the structure: extension should now terminate in a branch on the upper trie + ctx.assert_upper_subtrie(&trie) + .has_extension(&Nibbles::default(), &Nibbles::from_nibbles([0xA])) + .has_branch(&Nibbles::from_nibbles([0xA]), &[0x0, 0xB]); + + // Verify the lower trie now has a branch structure + ctx.assert_subtrie(&trie, Nibbles::from_nibbles([0xA, 0xB])) + .has_branch(&Nibbles::from_nibbles([0xA, 0xB]), &[0x1, 0x2]) + .has_hash(&Nibbles::from_nibbles([0xA, 0xB, 0x1]), &B256::repeat_byte(0x11)) + .has_hash(&Nibbles::from_nibbles([0xA, 0xB, 0x2]), &B256::repeat_byte(0x22)); + } + + #[test] + fn test_update_long_shared_prefix_at_boundary() { + let ctx = ParallelSparseTrieTestContext; + let mut trie = + ParallelSparseTrie::from_root(TrieNode::EmptyRoot, TrieMasks::none(), true).unwrap(); + + // Test edge case: leaves with long shared prefix that ends exactly at 2-nibble boundary + // + // Final trie structure: + // Upper trie: + // 0x: Extension { key: 0xAB } + // └── Subtrie (0xAB): pointer to lower subtrie + // + // Lower subtrie (0xAB): + // 0xAB: Branch { state_mask: 0x1000 | 0x2000 } + // ├── 0xABC: Leaf { key: 0xDEF } + // └── 0xABD: Leaf { key: 0xEF0 } + + // Create leaves that share exactly 2 nibbles + let (leaf1_path, value1) = ctx.create_test_leaf([0xA, 0xB, 0xC, 0xD, 0xE, 0xF], 1); + let (leaf2_path, value2) = ctx.create_test_leaf([0xA, 0xB, 0xD, 0xE, 0xF, 0x0], 2); + + trie.update_leaf(leaf1_path, value1.clone(), DefaultTrieNodeProvider).unwrap(); + trie.update_leaf(leaf2_path, value2.clone(), DefaultTrieNodeProvider).unwrap(); + + // Verify upper trie structure + ctx.assert_upper_subtrie(&trie) + .has_extension(&Nibbles::default(), &Nibbles::from_nibbles([0xA, 0xB])); + + // Verify lower subtrie structure + ctx.assert_subtrie(&trie, Nibbles::from_nibbles([0xA, 0xB])) + .has_branch(&Nibbles::from_nibbles([0xA, 0xB]), &[0xC, 0xD]) + .has_leaf( + &Nibbles::from_nibbles([0xA, 0xB, 0xC]), + &Nibbles::from_nibbles([0xD, 0xE, 0xF]), + ) + .has_leaf( + &Nibbles::from_nibbles([0xA, 0xB, 0xD]), + &Nibbles::from_nibbles([0xE, 0xF, 0x0]), + ) + .has_value(&leaf1_path, &value1) + .has_value(&leaf2_path, &value2); + } + + #[test] + fn test_update_branch_to_extension_collapse() { + let ctx = ParallelSparseTrieTestContext; + let mut trie = + ParallelSparseTrie::from_root(TrieNode::EmptyRoot, TrieMasks::none(), true).unwrap(); + + // Test creating a trie with leaves that share a long common prefix + // + // Initial state with 3 leaves (0x1234, 0x2345, 0x2356): + // Upper trie: + // 0x: Branch { state_mask: 0x6 } + // ├── 0x1: Leaf { key: 0x234 } + // └── 0x2: Extension { key: 0x3 } + // └── Subtrie (0x23): pointer + // Lower subtrie (0x23): + // 0x23: Branch { state_mask: 0x30 } + // ├── 0x234: Leaf { key: 0x5 } + // └── 0x235: Leaf { key: 0x6 } + // + // Then we create a new trie with leaves (0x1234, 0x1235, 0x1236): + // Expected structure: + // Upper trie: + // 0x: Extension { key: 0x123 } + // └── Subtrie (0x12): pointer + // Lower subtrie (0x12): + // 0x123: Branch { state_mask: 0x70 } // bits 4, 5, 6 set + // ├── 0x1234: Leaf { key: 0x } + // ├── 0x1235: Leaf { key: 0x } + // └── 0x1236: Leaf { key: 0x } + + // Create initial leaves + let (leaf1_path, value1) = ctx.create_test_leaf([0x1, 0x2, 0x3, 0x4], 1); + let (leaf2_path, value2) = ctx.create_test_leaf([0x2, 0x3, 0x4, 0x5], 2); + let (leaf3_path, value3) = ctx.create_test_leaf([0x2, 0x3, 0x5, 0x6], 3); + + trie.update_leaf(leaf1_path, value1, DefaultTrieNodeProvider).unwrap(); + trie.update_leaf(leaf2_path, value2, DefaultTrieNodeProvider).unwrap(); + trie.update_leaf(leaf3_path, value3, DefaultTrieNodeProvider).unwrap(); + + // Verify initial structure has branch at root + ctx.assert_upper_subtrie(&trie).has_branch(&Nibbles::default(), &[0x1, 0x2]); + + // Now update to create a pattern where extension is more efficient + // Replace leaves to all share prefix 0x123 + let (new_leaf1_path, new_value1) = ctx.create_test_leaf([0x1, 0x2, 0x3, 0x4], 10); + let (new_leaf2_path, new_value2) = ctx.create_test_leaf([0x1, 0x2, 0x3, 0x5], 11); + let (new_leaf3_path, new_value3) = ctx.create_test_leaf([0x1, 0x2, 0x3, 0x6], 12); + + // Clear and add new leaves + let mut trie = + ParallelSparseTrie::from_root(TrieNode::EmptyRoot, TrieMasks::none(), true).unwrap(); + trie.update_leaf(new_leaf1_path, new_value1.clone(), DefaultTrieNodeProvider).unwrap(); + trie.update_leaf(new_leaf2_path, new_value2.clone(), DefaultTrieNodeProvider).unwrap(); + trie.update_leaf(new_leaf3_path, new_value3.clone(), DefaultTrieNodeProvider).unwrap(); + + // Verify new structure has extension + ctx.assert_upper_subtrie(&trie) + .has_extension(&Nibbles::default(), &Nibbles::from_nibbles([0x1, 0x2, 0x3])); + + // Verify lower subtrie path was correctly updated to 0x123 + ctx.assert_subtrie_path(&trie, [0x1, 0x2], [0x1, 0x2, 0x3]); + + // Verify lower subtrie - all three leaves should be properly inserted + ctx.assert_subtrie(&trie, Nibbles::from_nibbles([0x1, 0x2])) + .has_branch(&Nibbles::from_nibbles([0x1, 0x2, 0x3]), &[0x4, 0x5, 0x6]) // All three children + .has_leaf(&Nibbles::from_nibbles([0x1, 0x2, 0x3, 0x4]), &Nibbles::default()) + .has_leaf(&Nibbles::from_nibbles([0x1, 0x2, 0x3, 0x5]), &Nibbles::default()) + .has_leaf(&Nibbles::from_nibbles([0x1, 0x2, 0x3, 0x6]), &Nibbles::default()) + .has_value(&new_leaf1_path, &new_value1) + .has_value(&new_leaf2_path, &new_value2) + .has_value(&new_leaf3_path, &new_value3); + } + + #[test] + fn test_update_shared_prefix_patterns() { + let ctx = ParallelSparseTrieTestContext; + let mut trie = + ParallelSparseTrie::from_root(TrieNode::EmptyRoot, TrieMasks::none(), true).unwrap(); + + // Test edge case: different patterns of shared prefixes + // + // Final trie structure: + // Upper trie: + // 0x: Branch { state_mask: 0x6 } + // ├── 0x1: Leaf { key: 0x234 } + // └── 0x2: Extension { key: 0x3 } + // └── Subtrie (0x23): pointer + // + // Lower subtrie (0x23): + // 0x23: Branch { state_mask: 0x10 | 0x20 } + // ├── 0x234: Leaf { key: 0x5 } + // └── 0x235: Leaf { key: 0x6 } + + // Create leaves with different shared prefix patterns + let (leaf1_path, value1) = ctx.create_test_leaf([0x1, 0x2, 0x3, 0x4], 1); + let (leaf2_path, value2) = ctx.create_test_leaf([0x2, 0x3, 0x4, 0x5], 2); + let (leaf3_path, value3) = ctx.create_test_leaf([0x2, 0x3, 0x5, 0x6], 3); + + trie.update_leaf(leaf1_path, value1, DefaultTrieNodeProvider).unwrap(); + trie.update_leaf(leaf2_path, value2.clone(), DefaultTrieNodeProvider).unwrap(); + trie.update_leaf(leaf3_path, value3.clone(), DefaultTrieNodeProvider).unwrap(); + + // Verify upper trie structure + ctx.assert_upper_subtrie(&trie) + .has_branch(&Nibbles::default(), &[0x1, 0x2]) + .has_leaf(&Nibbles::from_nibbles([0x1]), &Nibbles::from_nibbles([0x2, 0x3, 0x4])) + .has_extension(&Nibbles::from_nibbles([0x2]), &Nibbles::from_nibbles([0x3])); + + // Verify lower subtrie structure + ctx.assert_subtrie(&trie, Nibbles::from_nibbles([0x2, 0x3])) + .has_branch(&Nibbles::from_nibbles([0x2, 0x3]), &[0x4, 0x5]) + .has_leaf(&Nibbles::from_nibbles([0x2, 0x3, 0x4]), &Nibbles::from_nibbles([0x5])) + .has_leaf(&Nibbles::from_nibbles([0x2, 0x3, 0x5]), &Nibbles::from_nibbles([0x6])) + .has_value(&leaf2_path, &value2) + .has_value(&leaf3_path, &value3); + } + + #[test] + fn test_progressive_branch_creation() { + let ctx = ParallelSparseTrieTestContext; + let mut trie = + ParallelSparseTrie::from_root(TrieNode::EmptyRoot, TrieMasks::none(), true).unwrap(); + + // Test starting with a single leaf and progressively adding leaves + // that create branch nodes at shorter and shorter paths + // + // Step 1: Add leaf at 0x12345 + // Upper trie: + // 0x: Leaf { key: 0x12345 } + // + // Step 2: Add leaf at 0x12346 + // Upper trie: + // 0x: Extension { key: 0x1234 } + // └── Subtrie (0x12): pointer + // Lower subtrie (0x12): + // 0x1234: Branch { state_mask: 0x60 } // bits 5 and 6 set + // ├── 0x12345: Leaf { key: 0x } + // └── 0x12346: Leaf { key: 0x } + // + // Step 3: Add leaf at 0x1235 + // Lower subtrie (0x12) updates to: + // 0x123: Branch { state_mask: 0x30 } // bits 4 and 5 set + // ├── 0x1234: Branch { state_mask: 0x60 } + // │ ├── 0x12345: Leaf { key: 0x } + // │ └── 0x12346: Leaf { key: 0x } + // └── 0x1235: Leaf { key: 0x } + // + // Step 4: Add leaf at 0x124 + // Lower subtrie (0x12) updates to: + // 0x12: Branch { state_mask: 0x18 } // bits 3 and 4 set + // ├── 0x123: Branch { state_mask: 0x30 } + // │ ├── 0x1234: Branch { state_mask: 0x60 } + // │ │ ├── 0x12345: Leaf { key: 0x } + // │ │ └── 0x12346: Leaf { key: 0x } + // │ └── 0x1235: Leaf { key: 0x } + // └── 0x124: Leaf { key: 0x } + + // Step 1: Add first leaf - initially stored as leaf in upper trie + let (leaf1_path, value1) = ctx.create_test_leaf([0x1, 0x2, 0x3, 0x4, 0x5], 1); + trie.update_leaf(leaf1_path, value1.clone(), DefaultTrieNodeProvider).unwrap(); + + // Verify leaf node in upper trie (optimized single-leaf case) + ctx.assert_upper_subtrie(&trie) + .has_leaf(&Nibbles::default(), &Nibbles::from_nibbles([0x1, 0x2, 0x3, 0x4, 0x5])) + .has_value(&leaf1_path, &value1); + + // Step 2: Add leaf at 0x12346 - creates branch at 0x1234 + let (leaf2_path, value2) = ctx.create_test_leaf([0x1, 0x2, 0x3, 0x4, 0x6], 2); + trie.update_leaf(leaf2_path, value2.clone(), DefaultTrieNodeProvider).unwrap(); + + // Verify extension now goes to 0x1234 + ctx.assert_upper_subtrie(&trie) + .has_extension(&Nibbles::default(), &Nibbles::from_nibbles([0x1, 0x2, 0x3, 0x4])); + + // Verify subtrie path updated to 0x1234 + ctx.assert_subtrie_path(&trie, [0x1, 0x2], [0x1, 0x2, 0x3, 0x4]); + + ctx.assert_subtrie(&trie, Nibbles::from_nibbles([0x1, 0x2])) + .has_branch(&Nibbles::from_nibbles([0x1, 0x2, 0x3, 0x4]), &[0x5, 0x6]) + .has_leaf(&Nibbles::from_nibbles([0x1, 0x2, 0x3, 0x4, 0x5]), &Nibbles::default()) + .has_leaf(&Nibbles::from_nibbles([0x1, 0x2, 0x3, 0x4, 0x6]), &Nibbles::default()) + .has_value(&leaf1_path, &value1) + .has_value(&leaf2_path, &value2); + + // Step 3: Add leaf at 0x1235 - creates branch at 0x123 + let (leaf3_path, value3) = ctx.create_test_leaf([0x1, 0x2, 0x3, 0x5], 3); + trie.update_leaf(leaf3_path, value3.clone(), DefaultTrieNodeProvider).unwrap(); + + // Verify extension now goes to 0x123 + ctx.assert_upper_subtrie(&trie) + .has_extension(&Nibbles::default(), &Nibbles::from_nibbles([0x1, 0x2, 0x3])); + + // Verify subtrie path updated to 0x123 + ctx.assert_subtrie_path(&trie, [0x1, 0x2], [0x1, 0x2, 0x3]); + + ctx.assert_subtrie(&trie, Nibbles::from_nibbles([0x1, 0x2])) + .has_branch(&Nibbles::from_nibbles([0x1, 0x2, 0x3]), &[0x4, 0x5]) + .has_branch(&Nibbles::from_nibbles([0x1, 0x2, 0x3, 0x4]), &[0x5, 0x6]) + .has_leaf(&Nibbles::from_nibbles([0x1, 0x2, 0x3, 0x5]), &Nibbles::default()) + .has_value(&leaf1_path, &value1) + .has_value(&leaf2_path, &value2) + .has_value(&leaf3_path, &value3); + + // Step 4: Add leaf at 0x124 - creates branch at 0x12 (subtrie root) + let (leaf4_path, value4) = ctx.create_test_leaf([0x1, 0x2, 0x4], 4); + trie.update_leaf(leaf4_path, value4.clone(), DefaultTrieNodeProvider).unwrap(); + + // Verify extension now goes to 0x12 + ctx.assert_upper_subtrie(&trie) + .has_extension(&Nibbles::default(), &Nibbles::from_nibbles([0x1, 0x2])); + + // Verify subtrie path updated to 0x12 + ctx.assert_subtrie_path(&trie, [0x1, 0x2], [0x1, 0x2]); + + // Verify final structure + ctx.assert_subtrie(&trie, Nibbles::from_nibbles([0x1, 0x2])) + .has_branch(&Nibbles::from_nibbles([0x1, 0x2]), &[0x3, 0x4]) + .has_branch(&Nibbles::from_nibbles([0x1, 0x2, 0x3]), &[0x4, 0x5]) + .has_branch(&Nibbles::from_nibbles([0x1, 0x2, 0x3, 0x4]), &[0x5, 0x6]) + .has_leaf(&Nibbles::from_nibbles([0x1, 0x2, 0x4]), &Nibbles::default()) + .has_value(&leaf1_path, &value1) + .has_value(&leaf2_path, &value2) + .has_value(&leaf3_path, &value3) + .has_value(&leaf4_path, &value4); + } + + #[test] + fn test_update_max_depth_paths() { + let ctx = ParallelSparseTrieTestContext; + let mut trie = + ParallelSparseTrie::from_root(TrieNode::EmptyRoot, TrieMasks::none(), true).unwrap(); + + // Test edge case: very long paths (64 nibbles - max for addresses/storage) + // + // Final trie structure: + // Upper trie: + // 0x: Extension { key: 0xFF } + // └── Subtrie (0xFF): pointer + // + // Lower subtrie (0xFF): + // Has very long paths with slight differences at the end + + // Create two 64-nibble paths that differ only in the last nibble + let mut path1_nibbles = vec![0xF; 63]; + path1_nibbles.push(0x0); + let mut path2_nibbles = vec![0xF; 63]; + path2_nibbles.push(0x1); + + let (leaf1_path, value1) = ctx.create_test_leaf(&path1_nibbles, 1); + let (leaf2_path, value2) = ctx.create_test_leaf(&path2_nibbles, 2); + + trie.update_leaf(leaf1_path, value1.clone(), DefaultTrieNodeProvider).unwrap(); + trie.update_leaf(leaf2_path, value2.clone(), DefaultTrieNodeProvider).unwrap(); + + // The common prefix of 63 F's will create a very long extension + let extension_key = vec![0xF; 63]; + ctx.assert_upper_subtrie(&trie) + .has_extension(&Nibbles::default(), &Nibbles::from_nibbles(&extension_key)); + + // Verify the subtrie has the branch at the end + ctx.assert_subtrie(&trie, Nibbles::from_nibbles([0xF, 0xF])) + .has_branch(&Nibbles::from_nibbles(&path1_nibbles[..63]), &[0x0, 0x1]) + .has_value(&leaf1_path, &value1) + .has_value(&leaf2_path, &value2); + } + + #[test] + fn test_hoodie_block_1_data() { + // Reveal node at path Nibbles(0x) - root branch node + let root_branch_stack = vec![ + hex!("a0550b6aba4dd4582a2434d2cbdad8d3007d09f622d7a6e6eaa7a49385823c2fa2"), + hex!("a04788a4975a9e1efd29b834fd80fdfe8a57cc1b1c5ace6d30ce5a36a15e0092b3"), + hex!("a093aeccf87da304e6f7d09edc5d7bd3a552808866d2149dd0940507a8f9bfa910"), + hex!("a08b5b423ba68d0dec2eca1f408076f9170678505eb4a5db2abbbd83bb37666949"), + hex!("a08592f62216af4218098a78acad7cf472a727fb55e6c27d3cfdf2774d4518eb83"), + hex!("a0ef02aeee845cb64c11f85edc1a3094227c26445952554b8a9248915d80c746c3"), + hex!("a0df2529ee3a1ce4df5a758cf17e6a86d0fb5ea22ab7071cf60af6412e9b0a428a"), + hex!("a0acaa1092db69cd5a63676685827b3484c4b80dc1d3361f6073bbb9240101e144"), + hex!("a09c3f2bb2a729d71f246a833353ade65667716bb330e0127a3299a42d11200f93"), + hex!("a0ce978470f4c0b1f8069570563a14d2b79d709add2db4bf22dd9b6aed3271c566"), + hex!("a095f783cd1d464a60e3c8adcadc28c6eb9fec7306664df39553be41dccc909606"), + hex!("a0a9083f5fb914b255e1feb5d951a4dfddacf3c8003ef1d1ec6a13bb6ba5b2ac62"), + hex!("a0fec113d537d8577cd361e0cabf5e95ef58f1cc34318292fdecce9fae57c3e094"), + hex!("a08b7465f5fe8b3e3c0d087cb7521310d4065ef2a0ee43bf73f68dee8a5742b3dd"), + hex!("a0c589aa1ae3d5fd87d8640957f7d5184a4ac06f393b453a8e8ed7e8fba0d385c8"), + hex!("a0b516d6f3352f87beab4ed6e7322f191fc7a147686500ef4de7dd290ad784ef51"), + ]; + + let root_branch_rlp_stack: Vec = root_branch_stack + .iter() + .map(|hex_str| RlpNode::from_raw_rlp(&hex_str[..]).unwrap()) + .collect(); + + let root_branch_node = BranchNode::new( + root_branch_rlp_stack, + TrieMask::new(0b1111111111111111), // state_mask: all 16 children present + ); + + let root_branch_masks = TrieMasks { + hash_mask: Some(TrieMask::new(0b1111111111111111)), + tree_mask: Some(TrieMask::new(0b1111111111111111)), + }; + + let mut trie = ParallelSparseTrie::from_root( + TrieNode::Branch(root_branch_node), + root_branch_masks, + true, + ) + .unwrap(); + + // Reveal node at path Nibbles(0x3) - branch node + let branch_0x3_stack = vec![ + hex!("a09da7d9755fe0c558b3c3de9fdcdf9f28ae641f38c9787b05b73ab22ae53af3e2"), + hex!("a0d9990bf0b810d1145ecb2b011fd68c63cc85564e6724166fd4a9520180706e5f"), + hex!("a0f60eb4b12132a40df05d9bbdb88bbde0185a3f097f3c76bf4200c23eda26cf86"), + hex!("a0ca976997ddaf06f18992f6207e4f6a05979d07acead96568058789017cc6d06b"), + hex!("a04d78166b48044fdc28ed22d2fd39c8df6f8aaa04cb71d3a17286856f6893ff83"), + hex!("a021d4f90c34d3f1706e78463b6482bca77a3aa1cd059a3f326c42a1cfd30b9b60"), + hex!("a0fc3b71c33e2e6b77c5e494c1db7fdbb447473f003daf378c7a63ba9bf3f0049d"), + hex!("a0e33ed2be194a3d93d343e85642447c93a9d0cfc47a016c2c23d14c083be32a7c"), + hex!("a07b8e7a21c1178d28074f157b50fca85ee25c12568ff8e9706dcbcdacb77bf854"), + hex!("a0973274526811393ea0bf4811ca9077531db00d06b86237a2ecd683f55ba4bcb0"), + hex!("a03a93d726d7487874e51b52d8d534c63aa2a689df18e3b307c0d6cb0a388b00f3"), + hex!("a06aa67101d011d1c22fe739ef83b04b5214a3e2f8e1a2625d8bfdb116b447e86f"), + hex!("a02dd545b33c62d33a183e127a08a4767fba891d9f3b94fc20a2ca02600d6d1fff"), + hex!("a0fe6db87d00f06d53bff8169fa497571ff5af1addfb715b649b4d79dd3e394b04"), + hex!("a0d9240a9d2d5851d05a97ff3305334dfdb0101e1e321fc279d2bb3cad6afa8fc8"), + hex!("a01b69c6ab5173de8a8ec53a6ebba965713a4cc7feb86cb3e230def37c230ca2b2"), + ]; + + let branch_0x3_rlp_stack: Vec = branch_0x3_stack + .iter() + .map(|hex_str| RlpNode::from_raw_rlp(&hex_str[..]).unwrap()) + .collect(); + + let branch_0x3_node = BranchNode::new( + branch_0x3_rlp_stack, + TrieMask::new(0b1111111111111111), // state_mask: all 16 children present + ); + + let branch_0x3_masks = TrieMasks { + hash_mask: Some(TrieMask::new(0b0100010000010101)), + tree_mask: Some(TrieMask::new(0b0100000000000000)), + }; + + // Reveal node at path Nibbles(0x37) - leaf node + let leaf_path = Nibbles::from_nibbles([0x3, 0x7]); + let leaf_key = Nibbles::unpack( + &hex!("d65eaa92c6bc4c13a5ec45527f0c18ea8932588728769ec7aecfe6d9f32e42")[..], + ); + let leaf_value = hex!("f8440180a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a0f57acd40259872606d76197ef052f3d35588dadf919ee1f0e3cb9b62d3f4b02c").to_vec(); + + let leaf_node = LeafNode::new(leaf_key, leaf_value); + let leaf_masks = TrieMasks::none(); + + trie.reveal_nodes(vec![ + RevealedSparseNode { + path: Nibbles::from_nibbles([0x3]), + node: TrieNode::Branch(branch_0x3_node), + masks: branch_0x3_masks, + }, + RevealedSparseNode { + path: leaf_path, + node: TrieNode::Leaf(leaf_node), + masks: leaf_masks, + }, + ]) + .unwrap(); + + // Update leaf with its new value + let mut leaf_full_path = leaf_path; + leaf_full_path.extend(&leaf_key); + + let leaf_new_value = vec![ + 248, 68, 1, 128, 160, 224, 163, 152, 169, 122, 160, 155, 102, 53, 41, 0, 47, 28, 205, + 190, 199, 5, 215, 108, 202, 22, 138, 70, 196, 178, 193, 208, 18, 96, 95, 63, 238, 160, + 245, 122, 205, 64, 37, 152, 114, 96, 109, 118, 25, 126, 240, 82, 243, 211, 85, 136, + 218, 223, 145, 158, 225, 240, 227, 203, 155, 98, 211, 244, 176, 44, + ]; + + trie.update_leaf(leaf_full_path, leaf_new_value.clone(), DefaultTrieNodeProvider).unwrap(); + + // Sanity checks before calculating the root + assert_eq!( + Some(&leaf_new_value), + trie.lower_subtrie_for_path(&leaf_path).unwrap().inner.values.get(&leaf_full_path) + ); + assert!(trie.upper_subtrie.inner.values.is_empty()); + + // Assert the root hash matches the expected value + let expected_root = + b256!("0x29b07de8376e9ce7b3a69e9b102199869514d3f42590b5abc6f7d48ec9b8665c"); + assert_eq!(trie.root(), expected_root); + } + + #[test] + fn find_leaf_existing_leaf() { + // Create a simple trie with one leaf + let provider = DefaultTrieNodeProvider; + let mut sparse = ParallelSparseTrie::default(); + let path = Nibbles::from_nibbles([0x1, 0x2, 0x3]); + let value = b"test_value".to_vec(); + + sparse.update_leaf(path, value.clone(), &provider).unwrap(); + + // Check that the leaf exists + let result = sparse.find_leaf(&path, None); + assert_matches!(result, Ok(LeafLookup::Exists)); + + // Check with expected value matching + let result = sparse.find_leaf(&path, Some(&value)); + assert_matches!(result, Ok(LeafLookup::Exists)); + } + + #[test] + fn find_leaf_value_mismatch() { + // Create a simple trie with one leaf + let provider = DefaultTrieNodeProvider; + let mut sparse = ParallelSparseTrie::default(); + let path = Nibbles::from_nibbles([0x1, 0x2, 0x3]); + let value = b"test_value".to_vec(); + let wrong_value = b"wrong_value".to_vec(); + + sparse.update_leaf(path, value, &provider).unwrap(); + + // Check with wrong expected value + let result = sparse.find_leaf(&path, Some(&wrong_value)); + assert_matches!( + result, + Err(LeafLookupError::ValueMismatch { path: p, expected: Some(e), actual: _a }) if p == path && e == wrong_value + ); + } + + #[test] + fn find_leaf_not_found_empty_trie() { + // Empty trie + let sparse = ParallelSparseTrie::default(); + let path = Nibbles::from_nibbles([0x1, 0x2, 0x3]); + + // Leaf should not exist + let result = sparse.find_leaf(&path, None); + assert_matches!(result, Ok(LeafLookup::NonExistent)); + } + + #[test] + fn find_leaf_empty_trie() { + let sparse = ParallelSparseTrie::default(); + let path = Nibbles::from_nibbles_unchecked([0x1, 0x2, 0x3, 0x4]); + + let result = sparse.find_leaf(&path, None); + assert_matches!(result, Ok(LeafLookup::NonExistent)); + } + + #[test] + fn find_leaf_exists_no_value_check() { + let provider = DefaultTrieNodeProvider; + let mut sparse = ParallelSparseTrie::default(); + let path = Nibbles::from_nibbles_unchecked([0x1, 0x2, 0x3, 0x4]); + sparse.update_leaf(path, encode_account_value(0), &provider).unwrap(); + + let result = sparse.find_leaf(&path, None); + assert_matches!(result, Ok(LeafLookup::Exists)); + } + + #[test] + fn find_leaf_exists_with_value_check_ok() { + let provider = DefaultTrieNodeProvider; + let mut sparse = ParallelSparseTrie::default(); + let path = Nibbles::from_nibbles_unchecked([0x1, 0x2, 0x3, 0x4]); + let value = encode_account_value(0); + sparse.update_leaf(path, value.clone(), &provider).unwrap(); + + let result = sparse.find_leaf(&path, Some(&value)); + assert_matches!(result, Ok(LeafLookup::Exists)); + } + + #[test] + fn find_leaf_exclusion_branch_divergence() { + let provider = DefaultTrieNodeProvider; + let mut sparse = ParallelSparseTrie::default(); + let path1 = Nibbles::from_nibbles_unchecked([0x1, 0x2, 0x3, 0x4]); // Creates branch at 0x12 + let path2 = Nibbles::from_nibbles_unchecked([0x1, 0x2, 0x5, 0x6]); // Belongs to same branch + let search_path = Nibbles::from_nibbles_unchecked([0x1, 0x2, 0x7, 0x8]); // Diverges at nibble 7 + + sparse.update_leaf(path1, encode_account_value(0), &provider).unwrap(); + sparse.update_leaf(path2, encode_account_value(1), &provider).unwrap(); + + let result = sparse.find_leaf(&search_path, None); + assert_matches!(result, Ok(LeafLookup::NonExistent)) + } + + #[test] + fn find_leaf_exclusion_extension_divergence() { + let provider = DefaultTrieNodeProvider; + let mut sparse = ParallelSparseTrie::default(); + // This will create an extension node at root with key 0x12 + let path1 = Nibbles::from_nibbles_unchecked([0x1, 0x2, 0x3, 0x4, 0x5, 0x6]); + // This path diverges from the extension key + let search_path = Nibbles::from_nibbles_unchecked([0x1, 0x2, 0x7, 0x8]); + + sparse.update_leaf(path1, encode_account_value(0), &provider).unwrap(); + + let result = sparse.find_leaf(&search_path, None); + assert_matches!(result, Ok(LeafLookup::NonExistent)) + } + + #[test] + fn find_leaf_exclusion_leaf_divergence() { + let provider = DefaultTrieNodeProvider; + let mut sparse = ParallelSparseTrie::default(); + let existing_leaf_path = Nibbles::from_nibbles_unchecked([0x1, 0x2, 0x3, 0x4]); + let search_path = Nibbles::from_nibbles_unchecked([0x1, 0x2, 0x3, 0x4, 0x5, 0x6]); + + sparse.update_leaf(existing_leaf_path, encode_account_value(0), &provider).unwrap(); + + let result = sparse.find_leaf(&search_path, None); + assert_matches!(result, Ok(LeafLookup::NonExistent)) + } + + #[test] + fn find_leaf_exclusion_path_ends_at_branch() { + let provider = DefaultTrieNodeProvider; + let mut sparse = ParallelSparseTrie::default(); + let path1 = Nibbles::from_nibbles_unchecked([0x1, 0x2, 0x3, 0x4]); // Creates branch at 0x12 + let path2 = Nibbles::from_nibbles_unchecked([0x1, 0x2, 0x5, 0x6]); + let search_path = Nibbles::from_nibbles_unchecked([0x1, 0x2]); // Path of the branch itself + + sparse.update_leaf(path1, encode_account_value(0), &provider).unwrap(); + sparse.update_leaf(path2, encode_account_value(1), &provider).unwrap(); + + let result = sparse.find_leaf(&search_path, None); + assert_matches!(result, Ok(LeafLookup::NonExistent)); + } + + #[test] + fn find_leaf_error_blinded_node_at_leaf_path() { + // Scenario: The node *at* the leaf path is blinded. + let blinded_hash = B256::repeat_byte(0xBB); + let leaf_path = Nibbles::from_nibbles_unchecked([0x1, 0x2, 0x3, 0x4]); + + let sparse = new_test_trie( + [ + ( + // Ext 0x12 + Nibbles::default(), + SparseNode::new_ext(Nibbles::from_nibbles_unchecked([0x1, 0x2])), + ), + ( + // Ext 0x123 + Nibbles::from_nibbles_unchecked([0x1, 0x2]), + SparseNode::new_ext(Nibbles::from_nibbles_unchecked([0x3])), + ), + ( + // Branch at 0x123, child 4 + Nibbles::from_nibbles_unchecked([0x1, 0x2, 0x3]), + SparseNode::new_branch(TrieMask::new(0b10000)), + ), + ( + // Blinded node at 0x1234 + leaf_path, + SparseNode::Hash(blinded_hash), + ), + ] + .into_iter(), + ); + + let result = sparse.find_leaf(&leaf_path, None); + + // Should error because it hit the blinded node exactly at the leaf path + assert_matches!(result, Err(LeafLookupError::BlindedNode { path, hash }) + if path == leaf_path && hash == blinded_hash + ); + } + + #[test] + fn find_leaf_error_blinded_node() { + let blinded_hash = B256::repeat_byte(0xAA); + let path_to_blind = Nibbles::from_nibbles_unchecked([0x1]); + let search_path = Nibbles::from_nibbles_unchecked([0x1, 0x2, 0x3, 0x4]); + + let sparse = new_test_trie( + [ + // Root is a branch with child 0x1 (blinded) and 0x5 (revealed leaf) + // So we set Bit 1 and Bit 5 in the state_mask + (Nibbles::default(), SparseNode::new_branch(TrieMask::new(0b100010))), + (path_to_blind, SparseNode::Hash(blinded_hash)), + ( + Nibbles::from_nibbles_unchecked([0x5]), + SparseNode::new_leaf(Nibbles::from_nibbles_unchecked([0x6, 0x7, 0x8])), + ), + ] + .into_iter(), + ); + + let result = sparse.find_leaf(&search_path, None); + + // Should error because it hit the blinded node at path 0x1 + assert_matches!(result, Err(LeafLookupError::BlindedNode { path, hash }) + if path == path_to_blind && hash == blinded_hash + ); + } +} diff --git a/crates/trie/sparse/Cargo.toml b/crates/trie/sparse/Cargo.toml index 3726fb4632b..b2c7ee0f566 100644 --- a/crates/trie/sparse/Cargo.toml +++ b/crates/trie/sparse/Cargo.toml @@ -16,7 +16,8 @@ workspace = true reth-primitives-traits.workspace = true reth-execution-errors.workspace = true reth-trie-common.workspace = true -tracing.workspace = true +tracing = { workspace = true, features = ["attributes"] } +alloy-trie.workspace = true # alloy alloy-primitives.workspace = true @@ -24,6 +25,7 @@ alloy-rlp.workspace = true # misc auto_impl.workspace = true +rayon = { workspace = true, optional = true } smallvec = { workspace = true, features = ["const_new"] } # metrics @@ -53,12 +55,14 @@ rand_08.workspace = true [features] default = ["std", "metrics"] std = [ - "reth-storage-api/std", - "reth-primitives-traits/std", - "reth-execution-errors/std", - "reth-trie-common/std", + "dep:rayon", "alloy-primitives/std", "alloy-rlp/std", + "alloy-trie/std", + "reth-execution-errors/std", + "reth-primitives-traits/std", + "reth-storage-api/std", + "reth-trie-common/std", "tracing/std", ] metrics = ["dep:reth-metrics", "dep:metrics", "std"] @@ -72,9 +76,10 @@ test-utils = [ ] arbitrary = [ "std", + "alloy-primitives/arbitrary", + "alloy-trie/arbitrary", "reth-primitives-traits/arbitrary", "reth-trie-common/arbitrary", - "alloy-primitives/arbitrary", "smallvec/arbitrary", ] diff --git a/crates/trie/sparse/benches/rlp_node.rs b/crates/trie/sparse/benches/rlp_node.rs index 113392fca54..9f2337f31b8 100644 --- a/crates/trie/sparse/benches/rlp_node.rs +++ b/crates/trie/sparse/benches/rlp_node.rs @@ -7,7 +7,7 @@ use proptest::{prelude::*, test_runner::TestRunner}; use rand::{seq::IteratorRandom, Rng}; use reth_testing_utils::generators; use reth_trie::Nibbles; -use reth_trie_sparse::RevealedSparseTrie; +use reth_trie_sparse::{provider::DefaultTrieNodeProvider, SerialSparseTrie, SparseTrieInterface}; fn update_rlp_node_level(c: &mut Criterion) { let mut rng = generators::rng(); @@ -22,10 +22,15 @@ fn update_rlp_node_level(c: &mut Criterion) { .current(); // Create a sparse trie with `size` leaves - let mut sparse = RevealedSparseTrie::default(); + let provider = DefaultTrieNodeProvider; + let mut sparse = SerialSparseTrie::default(); for (key, value) in &state { sparse - .update_leaf(Nibbles::unpack(key), alloy_rlp::encode_fixed_size(value).to_vec()) + .update_leaf( + Nibbles::unpack(key), + alloy_rlp::encode_fixed_size(value).to_vec(), + &provider, + ) .unwrap(); } sparse.root(); @@ -39,6 +44,7 @@ fn update_rlp_node_level(c: &mut Criterion) { .update_leaf( Nibbles::unpack(key), alloy_rlp::encode_fixed_size(&rng.random::()).to_vec(), + &provider, ) .unwrap(); } diff --git a/crates/trie/sparse/benches/root.rs b/crates/trie/sparse/benches/root.rs index f4d461ae51a..9eaf54c2d0f 100644 --- a/crates/trie/sparse/benches/root.rs +++ b/crates/trie/sparse/benches/root.rs @@ -7,13 +7,13 @@ use proptest::{prelude::*, strategy::ValueTree, test_runner::TestRunner}; use reth_trie::{ hashed_cursor::{noop::NoopHashedStorageCursor, HashedPostStateStorageCursor}, node_iter::{TrieElement, TrieNodeIter}, - trie_cursor::{noop::NoopStorageTrieCursor, InMemoryStorageTrieCursor}, + trie_cursor::{noop::NoopStorageTrieCursor, InMemoryTrieCursor}, updates::StorageTrieUpdates, walker::TrieWalker, HashedStorage, }; use reth_trie_common::{HashBuilder, Nibbles}; -use reth_trie_sparse::SparseTrie; +use reth_trie_sparse::{provider::DefaultTrieNodeProvider, SerialSparseTrie, SparseTrie}; fn calculate_root_from_leaves(c: &mut Criterion) { let mut group = c.benchmark_group("calculate root from leaves"); @@ -40,13 +40,15 @@ fn calculate_root_from_leaves(c: &mut Criterion) { }); // sparse trie + let provider = DefaultTrieNodeProvider; group.bench_function(BenchmarkId::new("sparse trie", size), |b| { - b.iter_with_setup(SparseTrie::revealed_empty, |mut sparse| { + b.iter_with_setup(SparseTrie::::revealed_empty, |mut sparse| { for (key, value) in &state { sparse .update_leaf( Nibbles::unpack(key), alloy_rlp::encode_fixed_size(value).to_vec(), + &provider, ) .unwrap(); } @@ -131,11 +133,10 @@ fn calculate_root_from_leaves_repeated(c: &mut Criterion) { ) }; - let walker = TrieWalker::storage_trie( - InMemoryStorageTrieCursor::new( - B256::ZERO, - NoopStorageTrieCursor::default(), - Some(&trie_updates_sorted), + let walker = TrieWalker::<_>::storage_trie( + InMemoryTrieCursor::new( + Some(NoopStorageTrieCursor::default()), + &trie_updates_sorted.storage_nodes, ), prefix_set, ); @@ -177,6 +178,7 @@ fn calculate_root_from_leaves_repeated(c: &mut Criterion) { }); // sparse trie + let provider = DefaultTrieNodeProvider; let benchmark_id = BenchmarkId::new( "sparse trie", format!( @@ -186,12 +188,13 @@ fn calculate_root_from_leaves_repeated(c: &mut Criterion) { group.bench_function(benchmark_id, |b| { b.iter_with_setup( || { - let mut sparse = SparseTrie::revealed_empty(); + let mut sparse = SparseTrie::::revealed_empty(); for (key, value) in &init_state { sparse .update_leaf( Nibbles::unpack(key), alloy_rlp::encode_fixed_size(value).to_vec(), + &provider, ) .unwrap(); } @@ -205,6 +208,7 @@ fn calculate_root_from_leaves_repeated(c: &mut Criterion) { .update_leaf( Nibbles::unpack(key), alloy_rlp::encode_fixed_size(value).to_vec(), + &provider, ) .unwrap(); } diff --git a/crates/trie/sparse/benches/update.rs b/crates/trie/sparse/benches/update.rs new file mode 100644 index 00000000000..dff0260a9a4 --- /dev/null +++ b/crates/trie/sparse/benches/update.rs @@ -0,0 +1,104 @@ +#![allow(missing_docs)] + +use alloy_primitives::{B256, U256}; +use criterion::{criterion_group, criterion_main, BatchSize, BenchmarkId, Criterion}; +use proptest::{prelude::*, strategy::ValueTree}; +use rand::seq::IteratorRandom; +use reth_trie_common::Nibbles; +use reth_trie_sparse::{provider::DefaultTrieNodeProvider, SerialSparseTrie, SparseTrie}; + +const LEAF_COUNTS: [usize; 2] = [1_000, 5_000]; + +fn update_leaf(c: &mut Criterion) { + let mut group = c.benchmark_group("update_leaf"); + + for leaf_count in LEAF_COUNTS { + group.bench_function(BenchmarkId::from_parameter(leaf_count), |b| { + let leaves = generate_leaves(leaf_count); + // Start with an empty trie + let provider = DefaultTrieNodeProvider; + + b.iter_batched( + || { + let mut trie = SparseTrie::::revealed_empty(); + // Pre-populate with data + for (path, value) in leaves.iter().cloned() { + trie.update_leaf(path, value, &provider).unwrap(); + } + + let new_leaves = leaves + .iter() + // Update 10% of existing leaves with new values + .choose_multiple(&mut rand::rng(), leaf_count / 10) + .into_iter() + .map(|(path, _)| { + ( + path, + alloy_rlp::encode_fixed_size(&U256::from(path.len() * 2)).to_vec(), + ) + }) + .collect::>(); + + (trie, new_leaves) + }, + |(mut trie, new_leaves)| { + for (path, new_value) in new_leaves { + trie.update_leaf(*path, new_value, &provider).unwrap(); + } + trie + }, + BatchSize::LargeInput, + ); + }); + } +} + +fn remove_leaf(c: &mut Criterion) { + let mut group = c.benchmark_group("remove_leaf"); + + for leaf_count in LEAF_COUNTS { + group.bench_function(BenchmarkId::from_parameter(leaf_count), |b| { + let leaves = generate_leaves(leaf_count); + // Start with an empty trie + let provider = DefaultTrieNodeProvider; + + b.iter_batched( + || { + let mut trie = SparseTrie::::revealed_empty(); + // Pre-populate with data + for (path, value) in leaves.iter().cloned() { + trie.update_leaf(path, value, &provider).unwrap(); + } + + let delete_leaves = leaves + .iter() + .map(|(path, _)| path) + // Remove 10% leaves + .choose_multiple(&mut rand::rng(), leaf_count / 10); + + (trie, delete_leaves) + }, + |(mut trie, delete_leaves)| { + for path in delete_leaves { + trie.remove_leaf(path, &provider).unwrap(); + } + trie + }, + BatchSize::LargeInput, + ); + }); + } +} + +fn generate_leaves(size: usize) -> Vec<(Nibbles, Vec)> { + proptest::collection::hash_map(any::(), any::(), size) + .new_tree(&mut Default::default()) + .unwrap() + .current() + .iter() + .map(|(key, value)| (Nibbles::unpack(key), alloy_rlp::encode_fixed_size(value).to_vec())) + .collect() +} + +criterion_group!(benches, update_leaf, remove_leaf); +criterion_main!(benches); diff --git a/crates/trie/sparse/src/lib.rs b/crates/trie/sparse/src/lib.rs index 617622d194f..6b175970481 100644 --- a/crates/trie/sparse/src/lib.rs +++ b/crates/trie/sparse/src/lib.rs @@ -11,7 +11,10 @@ pub use state::*; mod trie; pub use trie::*; -pub mod blinded; +mod traits; +pub use traits::*; + +pub mod provider; #[cfg(feature = "metrics")] mod metrics; diff --git a/crates/trie/sparse/src/metrics.rs b/crates/trie/sparse/src/metrics.rs index 44f9c9dc958..8dc64ddc599 100644 --- a/crates/trie/sparse/src/metrics.rs +++ b/crates/trie/sparse/src/metrics.rs @@ -16,24 +16,25 @@ pub(crate) struct SparseStateTrieMetrics { /// Number of total storage nodes, including those that were skipped. pub(crate) multiproof_total_storage_nodes: u64, /// The actual metrics we will record into the histogram - pub(crate) histograms: SparseStateTrieHistograms, + pub(crate) histograms: SparseStateTrieInnerMetrics, } impl SparseStateTrieMetrics { /// Record the metrics into the histograms - pub(crate) fn record(&self) { + pub(crate) fn record(&mut self) { + use core::mem::take; self.histograms .multiproof_skipped_account_nodes - .record(self.multiproof_skipped_account_nodes as f64); + .record(take(&mut self.multiproof_skipped_account_nodes) as f64); self.histograms .multiproof_total_account_nodes - .record(self.multiproof_total_account_nodes as f64); + .record(take(&mut self.multiproof_total_account_nodes) as f64); self.histograms .multiproof_skipped_storage_nodes - .record(self.multiproof_skipped_storage_nodes as f64); + .record(take(&mut self.multiproof_skipped_storage_nodes) as f64); self.histograms .multiproof_total_storage_nodes - .record(self.multiproof_total_storage_nodes as f64); + .record(take(&mut self.multiproof_total_storage_nodes) as f64); } /// Increment the skipped account nodes counter by the given count @@ -60,7 +61,7 @@ impl SparseStateTrieMetrics { /// Metrics for the sparse state trie #[derive(Metrics)] #[metrics(scope = "sparse_state_trie")] -pub(crate) struct SparseStateTrieHistograms { +pub(crate) struct SparseStateTrieInnerMetrics { /// Histogram of account nodes that were skipped during a multiproof reveal due to being /// redundant (i.e. they were already revealed) pub(crate) multiproof_skipped_account_nodes: Histogram, diff --git a/crates/trie/sparse/src/blinded.rs b/crates/trie/sparse/src/provider.rs similarity index 58% rename from crates/trie/sparse/src/blinded.rs rename to crates/trie/sparse/src/provider.rs index b42012eb8ea..405b3a84747 100644 --- a/crates/trie/sparse/src/blinded.rs +++ b/crates/trie/sparse/src/provider.rs @@ -4,13 +4,13 @@ use alloy_primitives::{Bytes, B256}; use reth_execution_errors::SparseTrieError; use reth_trie_common::{Nibbles, TrieMask}; -/// Factory for instantiating blinded node providers. +/// Factory for instantiating trie node providers. #[auto_impl::auto_impl(&)] -pub trait BlindedProviderFactory { +pub trait TrieNodeProviderFactory { /// Type capable of fetching blinded account nodes. - type AccountNodeProvider: BlindedProvider; + type AccountNodeProvider: TrieNodeProvider; /// Type capable of fetching blinded storage nodes. - type StorageNodeProvider: BlindedProvider; + type StorageNodeProvider: TrieNodeProvider; /// Returns blinded account node provider. fn account_node_provider(&self) -> Self::AccountNodeProvider; @@ -30,36 +30,36 @@ pub struct RevealedNode { pub hash_mask: Option, } -/// Trie node provider for retrieving blinded nodes. +/// Trie node provider for retrieving trie nodes. #[auto_impl::auto_impl(&)] -pub trait BlindedProvider { - /// Retrieve blinded node by path. - fn blinded_node(&self, path: &Nibbles) -> Result, SparseTrieError>; +pub trait TrieNodeProvider { + /// Retrieve trie node by path. + fn trie_node(&self, path: &Nibbles) -> Result, SparseTrieError>; } -/// Default blinded node provider factory that creates [`DefaultBlindedProvider`]. +/// Default trie node provider factory that creates [`DefaultTrieNodeProviderFactory`]. #[derive(PartialEq, Eq, Clone, Default, Debug)] -pub struct DefaultBlindedProviderFactory; +pub struct DefaultTrieNodeProviderFactory; -impl BlindedProviderFactory for DefaultBlindedProviderFactory { - type AccountNodeProvider = DefaultBlindedProvider; - type StorageNodeProvider = DefaultBlindedProvider; +impl TrieNodeProviderFactory for DefaultTrieNodeProviderFactory { + type AccountNodeProvider = DefaultTrieNodeProvider; + type StorageNodeProvider = DefaultTrieNodeProvider; fn account_node_provider(&self) -> Self::AccountNodeProvider { - DefaultBlindedProvider + DefaultTrieNodeProvider } fn storage_node_provider(&self, _account: B256) -> Self::StorageNodeProvider { - DefaultBlindedProvider + DefaultTrieNodeProvider } } -/// Default blinded node provider that always returns `Ok(None)`. +/// Default trie node provider that always returns `Ok(None)`. #[derive(PartialEq, Eq, Clone, Default, Debug)] -pub struct DefaultBlindedProvider; +pub struct DefaultTrieNodeProvider; -impl BlindedProvider for DefaultBlindedProvider { - fn blinded_node(&self, _path: &Nibbles) -> Result, SparseTrieError> { +impl TrieNodeProvider for DefaultTrieNodeProvider { + fn trie_node(&self, _path: &Nibbles) -> Result, SparseTrieError> { Ok(None) } } diff --git a/crates/trie/sparse/src/state.rs b/crates/trie/sparse/src/state.rs index edbc7d04309..f142385c3cd 100644 --- a/crates/trie/sparse/src/state.rs +++ b/crates/trie/sparse/src/state.rs @@ -1,37 +1,92 @@ use crate::{ - blinded::{BlindedProvider, BlindedProviderFactory, DefaultBlindedProviderFactory}, - LeafLookup, RevealedSparseTrie, SparseTrie, TrieMasks, + provider::{TrieNodeProvider, TrieNodeProviderFactory}, + traits::SparseTrieInterface, + RevealedSparseNode, SerialSparseTrie, SparseTrie, TrieMasks, }; use alloc::{collections::VecDeque, vec::Vec}; use alloy_primitives::{ - hex, map::{B256Map, HashMap, HashSet}, Bytes, B256, }; use alloy_rlp::{Decodable, Encodable}; -use core::{fmt, iter::Peekable}; +use alloy_trie::proof::DecodedProofNodes; use reth_execution_errors::{SparseStateTrieErrorKind, SparseStateTrieResult, SparseTrieErrorKind}; use reth_primitives_traits::Account; use reth_trie_common::{ proof::ProofNodes, updates::{StorageTrieUpdates, TrieUpdates}, - MultiProof, Nibbles, RlpNode, StorageMultiProof, TrieAccount, TrieMask, TrieNode, - EMPTY_ROOT_HASH, TRIE_ACCOUNT_RLP_MAX_SIZE, + DecodedMultiProof, DecodedStorageMultiProof, MultiProof, Nibbles, RlpNode, StorageMultiProof, + TrieAccount, TrieMask, TrieNode, EMPTY_ROOT_HASH, TRIE_ACCOUNT_RLP_MAX_SIZE, }; -use tracing::trace; +use tracing::{instrument, trace}; + +/// Provides type-safe re-use of cleared [`SparseStateTrie`]s, which helps to save allocations +/// across payload runs. +#[derive(Debug)] +pub struct ClearedSparseStateTrie< + A = SerialSparseTrie, // Account trie implementation + S = SerialSparseTrie, // Storage trie implementation +>(SparseStateTrie); + +impl ClearedSparseStateTrie +where + A: SparseTrieInterface, + S: SparseTrieInterface, +{ + /// Creates a [`ClearedSparseStateTrie`] by clearing all the existing internal state of a + /// [`SparseStateTrie`] and then storing that instance for later re-use. + pub fn from_state_trie(mut trie: SparseStateTrie) -> Self { + trie.state = trie.state.clear(); + trie.revealed_account_paths.clear(); + trie.storage.clear(); + trie.account_rlp_buf.clear(); + Self(trie) + } + + /// Shrink the cleared sparse trie's capacity to the given node and value size. + /// This helps reduce memory usage when the trie has excess capacity. + /// The capacity is distributed equally across the account trie and all storage tries. + pub fn shrink_to(&mut self, node_size: usize, value_size: usize) { + // Count total number of storage tries (active + cleared + default) + let storage_tries_count = self.0.storage.tries.len() + self.0.storage.cleared_tries.len(); + + // Total tries = 1 account trie + all storage tries + let total_tries = 1 + storage_tries_count; + + // Distribute capacity equally among all tries + let node_size_per_trie = node_size / total_tries; + let value_size_per_trie = value_size / total_tries; + + // Shrink the account trie + self.0.state.shrink_nodes_to(node_size_per_trie); + self.0.state.shrink_values_to(value_size_per_trie); + // Give storage tries the remaining capacity after account trie allocation + let storage_node_size = node_size.saturating_sub(node_size_per_trie); + let storage_value_size = value_size.saturating_sub(value_size_per_trie); + + // Shrink all storage tries (they will redistribute internally) + self.0.storage.shrink_to(storage_node_size, storage_value_size); + } + + /// Returns the cleared [`SparseStateTrie`], consuming this instance. + pub fn into_inner(self) -> SparseStateTrie { + self.0 + } +} + +#[derive(Debug)] /// Sparse state trie representing lazy-loaded Ethereum state trie. -pub struct SparseStateTrie { - /// Blinded node provider factory. - provider_factory: F, +pub struct SparseStateTrie< + A = SerialSparseTrie, // Account trie implementation + S = SerialSparseTrie, // Storage trie implementation +> { /// Sparse account trie. - state: SparseTrie, - /// Sparse storage tries. - storages: B256Map>, + state: SparseTrie, /// Collection of revealed account trie paths. revealed_account_paths: HashSet, - /// Collection of revealed storage trie paths, per account. - revealed_storage_paths: B256Map>, + /// State related to storage tries. + storage: StorageTries, /// Flag indicating whether trie updates should be retained. retain_updates: bool, /// Reusable buffer for RLP encoding of trie accounts. @@ -41,15 +96,16 @@ pub struct SparseStateTrie Default for SparseStateTrie +where + A: Default, + S: Default, +{ fn default() -> Self { Self { - provider_factory: DefaultBlindedProviderFactory, state: Default::default(), - storages: Default::default(), revealed_account_paths: Default::default(), - revealed_storage_paths: Default::default(), + storage: Default::default(), retain_updates: false, account_rlp_buf: Vec::with_capacity(TRIE_ACCOUNT_RLP_MAX_SIZE), #[cfg(feature = "metrics")] @@ -58,19 +114,6 @@ impl Default for SparseStateTrie { } } -impl fmt::Debug for SparseStateTrie